chainlesschain 0.165.1 → 0.165.2

package/src/lib/background-agent-keeper-client.js

@@ -5,6 +5,8 @@ import {
   BACKGROUND_AGENT_KEEPER_ARM,
   BACKGROUND_AGENT_KEEPER_ARMED,
   BACKGROUND_AGENT_KEEPER_HELLO,
+  BACKGROUND_AGENT_KEEPER_HEARTBEAT,
+  BACKGROUND_AGENT_KEEPER_HEARTBEAT_INTERVAL_MS,
   BACKGROUND_AGENT_KEEPER_PROTOCOL_VERSION,
   BACKGROUND_AGENT_KEEPER_READY,
   BACKGROUND_AGENT_KEEPER_RETIRE,
@@ -76,11 +78,17 @@ export async function connectBackgroundAgentKeeper(options = {}) {
   const retireTimeoutMs = resolveBackgroundAgentKeeperRetireTimeoutMs(
     options.retireTimeoutMs,
   );
+  const heartbeatIntervalMs = Math.max(
+    10,
+    Number(options.heartbeatIntervalMs) ||
+      BACKGROUND_AGENT_KEEPER_HEARTBEAT_INTERVAL_MS,
+  );
   const socket = await connectWithDeadline(options.pipePath, connectTimeoutMs);
   const pending = new Map();
   let ready = false;
   let closing = false;
   let activeTurn = null;
+  let heartbeatTimer = null;
   let readyResolve;
   let readyReject;
   const readyPromise = new Promise((resolve, reject) => {
@@ -89,6 +97,8 @@ export async function connectBackgroundAgentKeeper(options = {}) {
   });
 
   const failPending = (error) => {
+    clearInterval(heartbeatTimer);
+    heartbeatTimer = null;
     readyReject(error);
     for (const entry of pending.values()) {
       clearTimeout(entry.timer);
@@ -171,6 +181,23 @@ export async function connectBackgroundAgentKeeper(options = {}) {
   );
   helloTimer.unref?.();
   await readyPromise.finally(() => clearTimeout(helloTimer));
+  const sendHeartbeat = () => {
+    try {
+      writeMessage(
+        socket,
+        createBackgroundAgentKeeperMessage(BACKGROUND_AGENT_KEEPER_HEARTBEAT, {
+          id: hello.id,
+          workerGeneration: hello.workerGeneration,
+          workerPid: hello.workerPid,
+        }),
+      );
+    } catch (error) {
+      socket.destroy(error);
+    }
+  };
+  sendHeartbeat();
+  heartbeatTimer = setInterval(sendHeartbeat, heartbeatIntervalMs);
+  heartbeatTimer.unref?.();
 
   const request = (
     type,
@@ -241,6 +268,8 @@ export async function connectBackgroundAgentKeeper(options = {}) {
     close() {
       if (closing) return;
       closing = true;
+      clearInterval(heartbeatTimer);
+      heartbeatTimer = null;
       socket.end();
     },
     activeTurn: () => activeTurn,

package/src/lib/background-agent-keeper-protocol.js

@@ -5,11 +5,15 @@ import { posix } from "node:path";
 export const BACKGROUND_AGENT_KEEPER_PROTOCOL_VERSION = 1;
 export const BACKGROUND_AGENT_KEEPER_HELLO = "background-agent-keeper-hello";
 export const BACKGROUND_AGENT_KEEPER_READY = "background-agent-keeper-ready";
+export const BACKGROUND_AGENT_KEEPER_HEARTBEAT =
+  "background-agent-keeper-heartbeat";
 export const BACKGROUND_AGENT_KEEPER_ARM = "background-agent-keeper-arm";
 export const BACKGROUND_AGENT_KEEPER_ARMED = "background-agent-keeper-armed";
 export const BACKGROUND_AGENT_KEEPER_RETIRE = "background-agent-keeper-retire";
 export const BACKGROUND_AGENT_KEEPER_RETIRED =
   "background-agent-keeper-retired";
+export const BACKGROUND_AGENT_KEEPER_HEARTBEAT_INTERVAL_MS = 1_000;
+export const BACKGROUND_AGENT_KEEPER_HEARTBEAT_TIMEOUT_MS = 15_000;
 
 // RETIRE is the only keeper request that performs destructive OS cleanup. Its
 // deadline is deliberately independent from the short HELLO/ARM request

package/src/lib/background-agent-supervisor.js

@@ -3051,22 +3051,10 @@ function isDestructiveIdentityRetired(identity) {
   return ["dead", "reused"].includes(identity?.status);
 }
 
-function isRecoverableCreationTimeIdentityProbe(identity) {
-  // A one-shot `ps -o lstart=` failure is observational: it says nothing about
-  // whether the exact target is still present. Other unverifiable outcomes can
-  // mean a missing durable anchor, a vanished group owner, PID reuse with a
-  // surviving group, or an untrusted group snapshot; those remain immediately
-  // fail-closed rather than being promoted into the resend path.
-  return (
-    identity?.status === "unverifiable" &&
-    identity?.reason === "creation-time-probe-failed"
-  );
-}
-
-function isRecoverableInitialDestructiveIdentityProbe(identity) {
+function isRecoverableDestructiveIdentityProbe(identity) {
   // These outcomes come only from read-only OS observations. Retrying them
   // cannot address a process, and the caller still requires two consecutive
-  // fresh exact matches before it may issue the first signal.
+  // fresh exact matches before it may issue or reissue a signal.
   return (
     identity?.status === "unverifiable" &&
     ["creation-time-probe-failed", "process-group-state-probe-failed"].includes(
@@ -3137,7 +3125,7 @@ function retryRecoverableInitialPosixIdentity({
     }
     if (
       lastIdentity?.status !== "match" &&
-      !isRecoverableInitialDestructiveIdentityProbe(lastIdentity)
+      !isRecoverableDestructiveIdentityProbe(lastIdentity)
     ) {
       return outcome("failed", "preflight-identity");
     }
@@ -3154,7 +3142,7 @@ function retryRecoverableInitialPosixIdentity({
     }
     if (
       lastIdentity?.status !== "match" &&
-      !isRecoverableInitialDestructiveIdentityProbe(lastIdentity)
+      !isRecoverableDestructiveIdentityProbe(lastIdentity)
     ) {
       return outcome("failed", "preflight-identity");
     }
@@ -3172,8 +3160,9 @@ function retryRecoverableInitialPosixIdentity({
 
 /**
  * Recover from a transient POSIX EPERM without ever signalling from unknown
- * identity evidence. Probe-only waits may bridge a transient creation-time
- * lookup failure, but a resend requires two consecutive fresh exact matches.
+ * identity evidence. Probe-only waits may bridge a transient creation-time or
+ * process-group-state lookup failure, but a resend requires two consecutive
+ * fresh exact matches.
  */
 function retryExactPosixSignalAfterPermissionError({
   inspectIdentity,
@@ -3203,7 +3192,7 @@ function retryExactPosixSignalAfterPermissionError({
     }
     if (
       lastIdentity?.status !== "match" &&
-      !isRecoverableCreationTimeIdentityProbe(lastIdentity)
+      !isRecoverableDestructiveIdentityProbe(lastIdentity)
     ) {
       return outcome("failed", "identity");
     }
@@ -3226,7 +3215,7 @@ function retryExactPosixSignalAfterPermissionError({
     }
     if (
       lastIdentity?.status !== "match" &&
-      !isRecoverableCreationTimeIdentityProbe(lastIdentity)
+      !isRecoverableDestructiveIdentityProbe(lastIdentity)
     ) {
       return outcome("failed", "identity");
     }
@@ -3306,7 +3295,7 @@ function signalExactBackgroundProcessTree(
   if (lastIdentity.status !== "match") {
     if (
       platform === "win32" ||
-      !isRecoverableInitialDestructiveIdentityProbe(lastIdentity)
+      !isRecoverableDestructiveIdentityProbe(lastIdentity)
     ) {
       retryStoppedBy = "preflight-identity";
       throw failure(
@@ -3663,7 +3652,7 @@ export function stopBackgroundAgent(id) {
           identityProbeRetries: 0,
         };
         try {
-          const preflightIdentity = inspectIdentity();
+          let preflightIdentity = inspectIdentity();
           activeTerminationContext.lastIdentityStatus =
             preflightIdentity.status;
           activeTerminationContext.lastIdentityReason =
@@ -3681,14 +3670,55 @@ export function stopBackgroundAgent(id) {
             continue;
           }
           if (preflightIdentity.status !== "match") {
-            activeTerminationContext.retryStoppedBy = "preflight-identity";
-            throw new Error(
-              `destructive process identity is ${preflightIdentity.status}: ${preflightIdentity.reason || "unknown"}`,
-            );
+            if (
+              process.platform === "win32" ||
+              !isRecoverableDestructiveIdentityProbe(preflightIdentity)
+            ) {
+              activeTerminationContext.retryStoppedBy = "preflight-identity";
+              throw new Error(
+                `destructive process identity is ${preflightIdentity.status}: ${preflightIdentity.reason || "unknown"}`,
+              );
+            }
+            permissionRetryDeadline =
+              _deps.posixPermissionRetryNowMs() +
+              Math.max(0, Number(_deps.processExitWaitDeadlineMs) || 0);
+            const retry = retryRecoverableInitialPosixIdentity({
+              inspectIdentity,
+              initialIdentity: preflightIdentity,
+              deadline: permissionRetryDeadline,
+            });
+            activeTerminationContext.identityProbeRetries +=
+              retry.identityProbeRetries;
+            preflightIdentity = retry.lastIdentity;
+            activeTerminationContext.lastIdentityStatus =
+              preflightIdentity?.status;
+            activeTerminationContext.lastIdentityReason =
+              preflightIdentity?.reason || null;
+            activeTerminationContext.retryStoppedBy = retry.retryStoppedBy;
+            if (retry.status === "retired") {
+              if (
+                preflightIdentity?.status === "reused" &&
+                primaryWorker?.pid === target.pid &&
+                !signalledPids.has(target.pid)
+              ) {
+                workerPidReused = true;
+              }
+              target.retiredBeforeWait = true;
+              signalledPids.add(target.pid);
+              continue;
+            }
+            if (retry.status !== "matched") {
+              throw new Error(
+                `destructive process identity is ${preflightIdentity?.status || "unknown"}: ${preflightIdentity?.reason || "unknown"}`,
+              );
+            }
+          }
+          if (!(permissionRetryDeadline > 0)) {
+            permissionRetryDeadline =
+              _deps.posixPermissionRetryNowMs() +
+              Math.max(0, Number(_deps.processExitWaitDeadlineMs) || 0);
           }
-          permissionRetryDeadline =
-            _deps.posixPermissionRetryNowMs() +
-            Math.max(0, Number(_deps.processExitWaitDeadlineMs) || 0);
+          activeTerminationContext.retryStoppedBy = "initial-signal";
           target.signalledAsGroup = sendInitialSignal() === "process-group";
           signalledPids.add(target.pid);
         } catch (error) {

package/src/lib/cowork-task-runner.js

@@ -58,6 +58,7 @@ export const _deps = {
 
 const DEFAULT_MAX_ITERATIONS = 50;
 const DEFAULT_TOKEN_BUDGET = 100_000;
+const WORKFLOW_EFFECT_ID_RE = /^sha256:[a-f0-9]{64}$/;
 const COWORK_MCP_CALL_LEDGER_UNSUPPORTED =
   "CC_COWORK_MCP_CALL_LEDGER_UNSUPPORTED";
 const COWORK_MCP_SESSION_UNVERIFIED = "CC_COWORK_MCP_SESSION_UNVERIFIED";
@@ -74,6 +75,44 @@ function coworkMcpAdmissionError(code, message, sessionId, cause = null) {
   return error;
 }
 
+function assertWorkflowEffectId(value) {
+  if (value == null) return;
+  if (typeof value !== "string" || !WORKFLOW_EFFECT_ID_RE.test(value)) {
+    const error = new TypeError(
+      "workflowEffectId must be a canonical sha256 identity",
+    );
+    error.code = "CC_COWORK_WORKFLOW_EFFECT_ID_INVALID";
+    throw error;
+  }
+}
+
+function workflowEffectEvidence(subAgent, workflowEffectId) {
+  if (!workflowEffectId) return {};
+  const attempts =
+    typeof subAgent?.providerRequestAttempts === "function"
+      ? subAgent.providerRequestAttempts()
+      : [];
+  const receipts =
+    typeof subAgent?.providerRequestReceipts === "function"
+      ? subAgent.providerRequestReceipts()
+      : [];
+  const nestedEffectAttempts =
+    typeof subAgent?.nestedEffectAttempts === "function"
+      ? subAgent.nestedEffectAttempts()
+      : [];
+  const nestedEffectSettlements =
+    typeof subAgent?.nestedEffectSettlements === "function"
+      ? subAgent.nestedEffectSettlements()
+      : [];
+  return {
+    workflowEffectId,
+    providerRequestAttempts: attempts,
+    providerRequestReceipts: receipts,
+    nestedEffectAttempts,
+    nestedEffectSettlements,
+  };
+}
+
 function assertCoworkMcpEventUsageAdmission(event, sessionId) {
   if (event?.type !== "session_start") return;
   const data =
@@ -477,6 +516,12 @@ export function prepareCoworkMcpRuntime(mcp, options = {}) {
  * @param {number} [options.maxIterations] - Override iteration limit
  * @param {number} [options.tokenBudget] - Override token budget
  * @param {string} [options.mcpSessionId] - Stable MCP ledger session for resume
+ * @param {string} [options.workflowEffectId] - Canonical durable workflow effect
+ * @param {boolean} [options.strictUsageTelemetry] - Require synchronous durable call observers
+ * @param {function} [options.onUsageBoundary] - Provider-call durable start observer
+ * @param {function} [options.onUsageSettlement] - Provider-call durable settlement observer
+ * @param {function} [options.onToolCallBoundary] - Tool-call durable start observer
+ * @param {function} [options.onToolCallSettlement] - Tool-call durable settlement observer
  * @param {(request: object) => boolean|Promise<boolean>} [options.approveMcpLocalCodeExecution]
  * @returns {Promise<{ taskId: string, status: string, result: object }>}
  */
@@ -493,12 +538,19 @@ export async function runCoworkTask(options = {}) {
     onProgress = null,
     signal = null,
     mcpSessionId = null,
+    workflowEffectId = null,
+    strictUsageTelemetry = false,
+    onUsageBoundary = null,
+    onUsageSettlement = null,
+    onToolCallBoundary = null,
+    onToolCallSettlement = null,
     approveMcpLocalCodeExecution = null,
   } = options;
 
   if (!userMessage || typeof userMessage !== "string") {
     throw new Error("userMessage is required");
   }
+  assertWorkflowEffectId(workflowEffectId);
 
   // An explicit canonical authority must be admitted before mounting local MCP
   // servers, constructing a child context, or making any provider call.
@@ -582,6 +634,16 @@ export async function runCoworkTask(options = {}) {
       cwd,
       onProgress,
       signal,
+      ...(workflowEffectId ? { workflowEffectId } : {}),
+      ...(strictUsageTelemetry
+        ? {
+            strictUsageTelemetry: true,
+            onUsageBoundary,
+            onUsageSettlement,
+            onToolCallBoundary,
+            onToolCallSettlement,
+          }
+        : {}),
       extraToolDefinitions: mcp.extraToolDefinitions,
       externalToolDescriptors: mcp.externalToolDescriptors,
       externalToolExecutors: mcp.externalToolExecutors,
@@ -632,17 +694,25 @@ export async function runCoworkTask(options = {}) {
         templateId: template.id,
         templateName: template.name,
         ...(mcpRuntime.sessionId ? { mcpSessionId: mcpRuntime.sessionId } : {}),
+        ...workflowEffectEvidence(subAgent, workflowEffectId),
         result,
       };
       _appendHistory(cwd, entry, userMessage);
       return entry;
     } catch (err) {
+      // A durable workflow already marked this outer effect as dispatched.
+      // Converting a child/provider exception into an ordinary failed result
+      // would authorize the step retry path even though the physical provider
+      // outcome may be unknown. Propagate so the runtime keeps it pending and
+      // requires reconciliation.
+      if (workflowEffectId) throw err;
       const entry = {
         taskId,
         status: "failed",
         templateId: template.id,
         templateName: template.name,
         ...(mcpRuntime.sessionId ? { mcpSessionId: mcpRuntime.sessionId } : {}),
+        ...workflowEffectEvidence(subAgent, workflowEffectId),
         result: {
           summary: `Task failed: ${err.message}`,
           artifacts: [],

package/src/lib/cowork-workflow.js

@@ -66,6 +66,8 @@ export const COWORK_WORKFLOW_RUN_ADMISSION_INVALID_CODE =
   "COWORK_WORKFLOW_RUN_ADMISSION_INVALID";
 export const COWORK_WORKFLOW_RUN_RESULT_INVALID_CODE =
   "COWORK_WORKFLOW_RUN_RESULT_INVALID";
+export const COWORK_WORKFLOW_CONTROL_SIGNAL_CODE =
+  "COWORK_WORKFLOW_CONTROL_SIGNAL";
 
 const DYNAMIC_WORKFLOW_RUN_ADMISSION_SCHEMA =
   "cc-dynamic-workflow-run-admission/v1";
@@ -484,7 +486,9 @@ function normalizeWorkflowRunAdmission(
     authority.definitionDigest !== definitionDigest ||
     value.executionLocation.authoritySchema !==
       EXECUTION_LOCATION_AUTHORITY_SCHEMA ||
-    value.executionLocation.authority !== "verified-session-start" ||
+    !["verified-session-start", "verified-session-location-handoff"].includes(
+      value.executionLocation.authority,
+    ) ||
     value.executionLocation.bindingSchema !==
       EXECUTION_LOCATION_BINDING_SCHEMA ||
     typeof value.executionLocation.location !== "string" ||
@@ -534,7 +538,7 @@ function normalizeWorkflowRunAdmission(
     }),
     executionLocation: Object.freeze({
       authoritySchema: EXECUTION_LOCATION_AUTHORITY_SCHEMA,
-      authority: "verified-session-start",
+      authority: value.executionLocation.authority,
       session: Object.freeze({
         sessionId: session.sessionId,
         headHash: session.headHash,
@@ -1307,6 +1311,7 @@ export async function runStepWithRetry({
   taskSemaphore,
   runTask = _deps.runTask,
   recordId = step.id,
+  iteration = 1,
   onTaskStart,
   admitted = false,
 }) {
@@ -1336,6 +1341,12 @@ export async function runStepWithRetry({
           files: step.files || [],
           cwd,
           llmOptions,
+          workflowEffect: Object.freeze({
+            stepId: recordId,
+            iteration,
+            attempt,
+            timeoutMs,
+          }),
           ...(abortController ? { signal: abortController.signal } : {}),
         }),
       );
@@ -1353,20 +1364,38 @@ export async function runStepWithRetry({
       lastErr = null;
     } catch (err) {
       if (release) release();
+      if (err?.code === COWORK_WORKFLOW_CONTROL_SIGNAL_CODE) throw err;
       if (err?.code === COWORK_WORKFLOW_RUN_RESULT_INVALID_CODE) throw err;
       if (admitted && err?.code === STEP_TIMEOUT_CODE && taskPromise != null) {
         abortController.abort(err);
         // A terminal admitted record must never be written while the physical
         // task can still produce side effects. Confirm settlement before a
         // retry or final failure; a runner that ignores abort safely blocks.
+        // A late completed result is authoritative and must not be replayed.
         try {
-          await taskPromise;
-        } catch {
-          // The timeout remains the attempt's externally visible failure.
+          const lateRawEntry = await taskPromise;
+          const lateEntry = normalizeAdmittedTaskEntry(lateRawEntry);
+          if (lateEntry.status === "completed") {
+            return { ok: true, entry: lateEntry, attempts: attempt };
+          }
+          lastEntry = lateEntry;
+          lastErr = null;
+        } catch (lateError) {
+          if (
+            lateError?.code === COWORK_WORKFLOW_CONTROL_SIGNAL_CODE ||
+            lateError?.code === COWORK_WORKFLOW_RUN_RESULT_INVALID_CODE
+          ) {
+            throw lateError;
+          }
+          // The timeout remains the attempt's externally visible failure when
+          // a non-authoritative runner error arrives after the deadline.
+          lastErr = err;
+          lastEntry = null;
         }
+      } else {
+        lastErr = err; // threw or timed out → retry-eligible
+        lastEntry = null;
       }
-      lastErr = err; // threw or timed out → retry-eligible
-      lastEntry = null;
     }
     if (attempt <= maxRetries) {
       const delay = retryDelayFor(step, attempt);
@@ -1503,6 +1532,7 @@ export async function runLoopStep({
       taskSemaphore,
       runTask,
       recordId,
+      iteration: iter,
       onTaskStart: (event) => {
         if (startAnnounced) return;
         startAnnounced = true;
@@ -1760,8 +1790,11 @@ export async function runPipeline({
           admitted,
         });
       } catch (err) {
-        if (err?.code === COWORK_WORKFLOW_RUN_RESULT_INVALID_CODE) {
-          fatalError = err;
+        if (
+          err?.code === COWORK_WORKFLOW_RUN_RESULT_INVALID_CODE ||
+          err?.code === COWORK_WORKFLOW_CONTROL_SIGNAL_CODE
+        ) {
+          fatalError ||= err;
           halted = true;
           active--;
           pump();

package/src/lib/doctor-checkup.js

@@ -462,7 +462,6 @@ async function pluginSection(opts, _deps2) {
   const checks = [];
   try {
     const { listInstalled } = await import("./plugin-runtime/install.js");
-    const { readPluginLock } = await import("./plugin-runtime/signature.js");
     const plugins = listInstalled({ cwd: opts.cwd || process.cwd() }) || [];
     if (plugins.length === 0) {
       checks.push(
@@ -475,22 +474,47 @@ async function pluginSection(opts, _deps2) {
       );
     } else {
       let unsigned = 0;
+      const runtimeBlocked = [];
+      const retainedRecovery = [];
       for (const p of plugins) {
-        try {
-          const lock = p.dir ? readPluginLock(p.dir) : null;
-          if (!lock || !lock.signature) unsigned++;
-        } catch {
-          unsigned++;
+        if (p.activePointer?.recoveryPath) {
+          retainedRecovery.push(`${p.name} (${p.activePointer.recoveryPath})`);
+        }
+        if (p.runtimeBlocked === true) {
+          runtimeBlocked.push(
+            `${p.name} (${p.activePointer?.status || "unknown pointer state"})`,
+          );
+          continue;
         }
+        if (p.integrity?.signature?.present !== true) unsigned++;
       }
       checks.push(
         check(
           "plugins",
           `Installed plugins: ${plugins.length}`,
-          unsigned > 0 ? CHECK_LEVELS.WARN : CHECK_LEVELS.OK,
-          unsigned > 0
-            ? `${unsigned} without a signature lock — they will be refused if plugins.requireSignedPlugins is enabled`
-            : "all have signature locks",
+          unsigned > 0 ||
+            runtimeBlocked.length > 0 ||
+            retainedRecovery.length > 0
+            ? CHECK_LEVELS.WARN
+            : CHECK_LEVELS.OK,
+          [
+            runtimeBlocked.length > 0
+              ? `runtime blocked: ${runtimeBlocked.join(", ")}`
+              : null,
+            retainedRecovery.length > 0
+              ? `retained install recovery: ${retainedRecovery.join(", ")}; remove the blocked plugin before another mutation`
+              : null,
+            unsigned > 0
+              ? `${unsigned} active without a signature lock — they will be refused if plugins.requireSignedPlugins is enabled`
+              : null,
+            runtimeBlocked.length === 0 &&
+            retainedRecovery.length === 0 &&
+            unsigned === 0
+              ? "all active plugins have signature locks"
+              : null,
+          ]
+            .filter(Boolean)
+            .join("; "),
         ),
       );
     }