@haaaiawd/second-nature 0.2.2 → 0.2.5

package/runtime/connectors/services/connector-cooldown-port.js

@@ -0,0 +1,123 @@
+/**
+ * ConnectorCooldownPort — Durable cooldown ledger for repeated terminal failures.
+ *
+ * Core logic: Track terminal failures per platform/capability and block replay
+ * for a bounded window after repeated failures. Successful recovery is allowed
+ * to bypass stale cooldown.
+ *
+ * Design authority:
+ * - `.anws/v8/04_SYSTEM_DESIGN/connector-system.md §6`
+ * - `.anws/v8/04_SYSTEM_DESIGN/body-tool-system.md §4`
+ *
+ * Dependencies:
+ * - `src/storage/v8-state-stores.js` (read/write connector cooldown state)
+ * - `src/connectors/base/failure-taxonomy.js` (FailureClass, retryable lookup)
+ *
+ * Boundary:
+ * - Does not execute connectors; only records/read cooldown state.
+ * - Does not permanently blacklist platforms; cooldown expires.
+ */
+import { readConnectorCooldownState, writeConnectorCooldownState, } from "../../storage/v8-state-stores.js";
+// ───────────────────────────────────────────────────────────────
+// Config
+// ───────────────────────────────────────────────────────────────
+const DEFAULT_COOLDOWN_MS = 5 * 60 * 1000; // 5 minutes
+const TERMINAL_FAILURE_THRESHOLD = 2;
+const RETRYABLE_FAILURE_CLASSES = new Set([
+    "transport_failure",
+    "rate_limited",
+    "timeout",
+    "concurrency_conflict",
+]);
+// ───────────────────────────────────────────────────────────────
+// Helpers
+// ───────────────────────────────────────────────────────────────
+function makeCooldownId(platformId, capabilityId) {
+    return `cooldown_${platformId}_${capabilityId}`;
+}
+function addMs(iso, ms) {
+    return new Date(new Date(iso).getTime() + ms).toISOString();
+}
+function isAfter(a, b) {
+    return new Date(a).getTime() > new Date(b).getTime();
+}
+// ───────────────────────────────────────────────────────────────
+// Public API
+// ───────────────────────────────────────────────────────────────
+export function createConnectorCooldownPort(db) {
+    return {
+        async isBlocked(platformId, intent) {
+            const read = await readConnectorCooldownState(db, platformId, intent);
+            if (read.degraded) {
+                // Fail-closed: if we cannot read cooldown state, prevent replay to avoid hammering
+                return {
+                    blocked: true,
+                    reason: "cooldown_state_unreadable",
+                };
+            }
+            if (!read.row) {
+                return { blocked: false };
+            }
+            const now = new Date().toISOString();
+            const blocked = isAfter(read.row.blockedUntil, now);
+            return {
+                blocked,
+                retryAfterMs: blocked
+                    ? Math.max(0, new Date(read.row.blockedUntil).getTime() - new Date(now).getTime())
+                    : undefined,
+            };
+        },
+        async markFailure(platformId, intent, failureClass, retryAfterMs) {
+            const id = makeCooldownId(platformId, intent);
+            const now = new Date().toISOString();
+            const existing = await readConnectorCooldownState(db, platformId, intent);
+            const isRetryable = RETRYABLE_FAILURE_CLASSES.has(failureClass);
+            let failureCount = 1;
+            let terminalCount = isRetryable ? 0 : 1;
+            let blockedUntil = now;
+            if (!existing.degraded && existing.row) {
+                failureCount = existing.row.failureCount + 1;
+                terminalCount = (existing.row.terminalCount ?? 0) + (isRetryable ? 0 : 1);
+                // Extend blocked window if already blocked
+                if (isAfter(existing.row.blockedUntil, now)) {
+                    blockedUntil = existing.row.blockedUntil;
+                }
+            }
+            if (retryAfterMs && retryAfterMs > 0) {
+                // Rate-limit or explicit retry-after takes precedence
+                blockedUntil = addMs(now, retryAfterMs);
+            }
+            else if (!isRetryable && terminalCount >= TERMINAL_FAILURE_THRESHOLD) {
+                // Repeated terminal failures enter bounded cooldown
+                blockedUntil = addMs(now, DEFAULT_COOLDOWN_MS);
+            }
+            else if (isRetryable) {
+                // Retryable failures do not accumulate terminal cooldown
+                blockedUntil = now;
+            }
+            await writeConnectorCooldownState(db, {
+                id,
+                platformId,
+                capabilityId: intent,
+                failureClass,
+                retryAfterMs: retryAfterMs ?? null,
+                blockedUntil,
+                failureCount,
+                terminalCount,
+                sourceRefs: [
+                    {
+                        uri: `sn://cooldown/${platformId}/${intent}`,
+                        family: "audit",
+                        id,
+                        redactionClass: "none",
+                        resolveStatus: "resolvable",
+                    },
+                ],
+                payloadJson: JSON.stringify({ markedAt: now, failureCount, terminalCount, isRetryable }),
+                createdAt: existing.row?.createdAt ?? now,
+                updatedAt: now,
+                redactionClass: "none",
+            });
+        },
+    };
+}

package/runtime/connectors/services/connector-executor-adapter.js

@@ -19,6 +19,7 @@ import { parseConnectorManifestV6 } from "../manifest/manifest-parser.js";
 import fs from "node:fs";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
+import { createConnectorCooldownPort } from "./connector-cooldown-port.js";
 const DEFAULT_AGENT_WORLD_USERNAME = "nyx_ha";
 const DEFAULT_AGENT_WORLD_PROFILE_PATH_TEMPLATE = "/api/agents/profile/{username}";
 function readString(value) {
@@ -104,7 +105,7 @@ async function fetchAgentWorldJson(input) {
         body: input.body === undefined ? undefined : JSON.stringify(input.body),
     });
     if (!resp.ok) {
-        throw { code: "api_error", detail: `agent-world ${input.label}: ${resp.status}` };
+        throw { code: "api_error", status: resp.status, detail: `agent-world ${input.label}: ${resp.status}` };
     }
     return resp.json();
 }
@@ -147,7 +148,7 @@ async function fetchEvoMapJson(input) {
         body: input.body === undefined ? undefined : JSON.stringify(input.body),
     });
     if (!resp.ok) {
-        throw { code: "api_error", detail: `evomap ${input.label}: ${resp.status}` };
+        throw { code: "api_error", status: resp.status, detail: `evomap ${input.label}: ${resp.status}` };
     }
     return resp.json();
 }
@@ -256,6 +257,8 @@ function createDeclarativeHttpRunner(manifest, credential) {
                     body: method !== "GET" && request.payload ? JSON.stringify(request.payload) : undefined,
                 });
                 if (!resp.ok) {
+                    const status = resp.status;
+                    const body = await resp.text().catch(() => "");
                     return {
                         platformId: request.platformId,
                         channel: plan.channel,
@@ -263,7 +266,8 @@ function createDeclarativeHttpRunner(manifest, credential) {
                         success: false,
                         error: {
                             code: "api_error",
-                            detail: `HTTP ${resp.status}: ${await resp.text().catch(() => "")}`,
+                            status,
+                            detail: `HTTP ${status}${body ? `: ${body.slice(0, 200)}` : ""}`,
                         },
                     };
                 }
@@ -618,7 +622,8 @@ export function createConnectorExecutorAdapter(options) {
     registry.register({ ...agentWorldManifest });
     registry.register({ ...instreetManifest });
     registerWorkspaceManifests(registry, options.workspaceRoot);
-    const routeContextPort = createCredentialRouteContextPort(vault);
+    const cooldownPort = createConnectorCooldownPort(options.stateDb);
+    const routeContextPort = createCredentialRouteContextPort(vault, options.stateDb);
     const routePlanner = new ConnectorRoutePlanner(registry, routeContextPort, new ChannelHealthStore());
     const telemetry = new ExecutionTelemetry(options.observabilityDb);
     const executionRunner = createAdaptiveExecutionRunner(vault, options.workspaceRoot);
@@ -626,6 +631,7 @@ export function createConnectorExecutorAdapter(options) {
         routePlanner,
         executionRunner,
         telemetry,
+        cooldownPort,
         effectCommitLedger: new InMemoryEffectCommitLedger(),
         retryPolicy: { maxRetries: 2, jitter: true },
     });

package/runtime/connectors/services/credential-route-context.d.ts

@@ -3,8 +3,9 @@
  *
  * Loads decrypted credentials from state DB and maps them to the
  * CredentialContext shape expected by ConnectorRoutePlanner.
- * Cooldown is stubbed (always unblocked) until a cooldown ledger is modeled.
+ * Cooldown state is loaded from connector_cooldown_state table.
  */
 import type { RouteContextPort } from "../base/contract.js";
 import type { CredentialVault } from "../../storage/services/credential-vault.js";
-export declare function createCredentialRouteContextPort(vault: CredentialVault): RouteContextPort;
+import type { StateDatabase } from "../../storage/db/index.js";
+export declare function createCredentialRouteContextPort(vault: CredentialVault, db: StateDatabase): RouteContextPort;

package/runtime/connectors/services/credential-route-context.js

@@ -1,4 +1,5 @@
-export function createCredentialRouteContextPort(vault) {
+import { readConnectorCooldownState } from "../../storage/v8-state-stores.js";
+export function createCredentialRouteContextPort(vault, db) {
     return {
         async loadCredentialState(platformId) {
             const ctx = await vault.loadCredentialContext(platformId);
@@ -12,8 +13,23 @@ export function createCredentialRouteContextPort(vault) {
             }
             return ctx;
         },
-        async loadCooldownState() {
-            return { blocked: false };
+        async loadCooldownState(platformId, intent) {
+            const read = await readConnectorCooldownState(db, platformId, intent);
+            if (read.degraded) {
+                // Fail-closed on unreadable cooldown state
+                return { blocked: true, reason: "cooldown_state_unreadable" };
+            }
+            if (!read.row) {
+                return { blocked: false };
+            }
+            const now = new Date().toISOString();
+            const blocked = new Date(read.row.blockedUntil).getTime() > new Date(now).getTime();
+            return {
+                blocked,
+                retryAfterMs: blocked
+                    ? Math.max(0, new Date(read.row.blockedUntil).getTime() - new Date(now).getTime())
+                    : undefined,
+            };
         },
     };
 }

package/runtime/core/second-nature/action/action-closure-recorder.d.ts

@@ -56,6 +56,8 @@ export declare function recordRememberClosure(db: StateDatabase, cycleId: string
 export declare function recordPolicyOutcomeClosure(db: StateDatabase, cycleId: string, closureStatus: ClosureStatus, reason: V8ReasonCode, params: {
     proposalId?: string;
     decisionId?: string;
+    platformId?: string;
+    capabilityId?: string;
     downgradedActionKind?: string;
     postProcessing?: string[];
     nextState?: string;
@@ -63,6 +65,8 @@ export declare function recordPolicyOutcomeClosure(db: StateDatabase, cycleId: s
 export declare function recordExecutionClosure(db: StateDatabase, cycleId: string, closureStatus: "completed" | "failed", reason: V8ReasonCode, params: {
     proposalId: string;
     decisionId: string;
+    platformId?: string;
+    capabilityId?: string;
     executionResultRef?: string;
     outputSummary?: string;
     nextState?: string;

package/runtime/core/second-nature/action/action-closure-recorder.js

@@ -46,6 +46,7 @@ export async function recordNoActionClosure(db, cycleId, noActionReason, options
         id: closureId,
         createdAt: now,
         cycleId,
+        platformId: "heartbeat",
         status: "no_action",
         reason: noActionReason,
         nextState: "await_next_cycle",
@@ -122,6 +123,8 @@ export async function recordPolicyOutcomeClosure(db, cycleId, closureStatus, rea
         id: closureId,
         createdAt: now,
         cycleId,
+        platformId: params.platformId ?? "heartbeat",
+        capabilityId: params.capabilityId,
         proposalId: params.proposalId,
         decisionId: params.decisionId,
         status: closureStatus,
@@ -161,6 +164,8 @@ export async function recordExecutionClosure(db, cycleId, closureStatus, reason,
         id: closureId,
         createdAt: now,
         cycleId,
+        platformId: params.platformId ?? "heartbeat",
+        capabilityId: params.capabilityId,
         proposalId: params.proposalId,
         decisionId: params.decisionId,
         status: closureStatus,

package/runtime/core/second-nature/action/action-proposal-builder.js

@@ -136,6 +136,7 @@ export async function buildActionProposal(db, judgmentVerdictId, options) {
             id: closureId,
             createdAt: now,
             cycleId,
+            platformId: "heartbeat",
             status: "completed",
             reason: "remember_for_review",
             nextState: "pending_daily_review",

package/runtime/core/second-nature/control-plane/heartbeat-orchestrator.d.ts

@@ -22,6 +22,7 @@
  * Test coverage: tests/unit/control-plane/heartbeat-cycle-trace.test.ts
  */
 import type { StateDatabase } from "../../../storage/db/index.js";
+import { type DailyRhythmState } from "../quiet-dream/daily-rhythm-scheduler.js";
 import type { SourceRef, DegradedOperationResult, V8ReasonCode } from "../../../shared/types/v8-contracts.js";
 export interface HeartbeatOrchestrationRequest {
     workspaceRoot: string;
@@ -34,5 +35,6 @@ export interface HeartbeatOrchestrationResult {
     closureRef?: SourceRef;
     noActionReason?: V8ReasonCode;
     degraded?: DegradedOperationResult;
+    rhythmState?: DailyRhythmState;
 }
 export declare function runHeartbeatCycle(db: StateDatabase, request: HeartbeatOrchestrationRequest): Promise<HeartbeatOrchestrationResult | DegradedOperationResult>;