chainlesschain 0.162.180 → 0.162.182

package/src/lib/background-agent-supervisor.js

@@ -20,6 +20,7 @@ import {
   finishAgentWorktree,
   validateAgentWorktree,
 } from "./agent-worktree.js";
+import { rejectPendingBackgroundInteractions } from "./background-interaction-journal.js";
 import executionBroker from "./process-execution-broker/index.js";
 
 export const DEFAULT_HEARTBEAT_INTERVAL_MS = 5000;
@@ -464,6 +465,67 @@ export function effectiveBackgroundAgentState(state, options = {}) {
         /* best-effort */
       }
     }
+    if (
+      options.persist !== false &&
+      state.sessionId &&
+      !isSameProcess(state.pid, state.startedAt)
+    ) {
+      const pending = state.pendingQuestion;
+      const requestId = pending?.requestId || pending?.intId;
+      try {
+        const recovery = rejectPendingBackgroundInteractions(
+          state.sessionId,
+          state.id,
+          {
+            fallbackRequest: requestId
+              ? {
+                  requestId,
+                  binding: pending.binding,
+                  payload: {
+                    kind: "question",
+                    question: pending.question || pending.prompt || "",
+                    options: pending.options || null,
+                    multiSelect: pending.multiSelect === true,
+                    timeoutMs: pending.timeoutMs,
+                  },
+                  createdAt: pending.askedAt,
+                }
+              : null,
+            code: "INTERACTION_WORKER_LOST",
+            message:
+              "The background worker was lost before the interaction settled",
+          },
+        );
+        const recoveredRecord = requestId
+          ? recovery.journal.get(requestId)
+          : null;
+        if (
+          recovery.changed ||
+          (recoveredRecord && recoveredRecord.status !== "pending")
+        ) {
+          next = {
+            ...next,
+            phase: null,
+            pendingQuestion: null,
+            interactionRecovery: {
+              status: "rejected",
+              requestIds: recovery.rejected.map((record) => record.requestId),
+              recoveredAt: t,
+            },
+          };
+        }
+      } catch (error) {
+        next = {
+          ...next,
+          interactionRecovery: {
+            status: "failed",
+            code: error?.code || "INTERACTION_RECOVERY_FAILED",
+            message: error?.message || String(error),
+            recoveredAt: t,
+          },
+        };
+      }
+    }
     if (options.persist !== false) {
       try {
         writeBackgroundAgentState(next);
@@ -838,11 +900,17 @@ export function launchBackgroundAgent({
   };
   writeBackgroundAgentState(state);
   let child;
+  let workerLogFd;
   try {
+    // Preserve loader/bootstrap failures that happen before the worker can
+    // open its own append handle. This also gives detached Windows targets
+    // concrete inheritable stdout/stderr handles instead of relying on the
+    // hosted process' possibly absent standard handles.
+    workerLogFd = openSync(job.logFile, "a");
     child = _deps.spawn(process.execPath, [worker, jobFile], {
       cwd,
       detached: true,
-      stdio: "ignore",
+      stdio: ["ignore", workerLogFd, workerLogFd],
       windowsHide: true,
       origin: "background-agent:worker",
       policy: "allow",
@@ -853,6 +921,10 @@ export function launchBackgroundAgent({
     rmSync(jobFile, { force: true });
     rmSync(statePath(id), { force: true });
     throw error;
+  } finally {
+    if (Number.isInteger(workerLogFd)) {
+      closeSync(workerLogFd);
+    }
   }
   // Async spawn failures (EPERM, cwd raced away between the check and the
   // spawn, …) arrive as an 'error' event on the detached child. Reap them
@@ -875,13 +947,42 @@ export function launchBackgroundAgent({
         /* best-effort */
       }
     });
+    child.on("exit", (code, signal) => {
+      try {
+        rmSync(jobFile, { force: true });
+        const current = readBackgroundAgentState(id);
+        // A healthy worker persists completed/failed before exiting. If the
+        // process (or its platform wrapper) exits while the state still says
+        // running, bootstrap or finalization failed and the session must not
+        // remain a phantom live task until heartbeat reconciliation.
+        if (current && current.status === "running") {
+          writeBackgroundAgentState({
+            ...current,
+            status: "failed",
+            endedAt: Date.now(),
+            exitCode: Number.isInteger(code) ? code : null,
+            signal: signal || null,
+            lostReason: `worker-exited-before-finalize: exit=${
+              Number.isInteger(code) ? code : "null"
+            } signal=${signal || "none"}`,
+          });
+        }
+      } catch {
+        /* best-effort */
+      }
+    });
   }
   child.unref();
   const current = readBackgroundAgentState(id) || state;
   const withPid = {
     ...current,
-    pid: child.pid,
-    workerPid: child.pid,
+    // A platform wrapper (notably the Windows Job Object adapter) can return
+    // from spawn after the real worker has already claimed the state file.
+    // Preserve that authoritative process.pid instead of racing it back to
+    // the wrapper pid. With an ordinary direct spawn both fields are still
+    // null here and are initialized from child.pid as before.
+    pid: current.pid ?? child.pid,
+    workerPid: current.workerPid ?? child.pid,
   };
   writeBackgroundAgentState(withPid);
   return withPid;

package/src/lib/background-interaction-journal.js

@@ -0,0 +1,407 @@
+/**
+ * Durable pending/settlement journal for background human interactions.
+ *
+ * Each mutation appends a full snapshot to the session's tamper-evident JSONL
+ * transcript. The worker persists a terminal settlement before delivering it
+ * to the suspended child, so retries can be classified as the same idempotent
+ * answer or a conflicting second settlement even after a process restart.
+ */
+
+import { createHash } from "node:crypto";
+import {
+  appendEvent as storeAppendEvent,
+  readEvents as storeReadEvents,
+} from "../harness/jsonl-session-store.js";
+import {
+  normalizeInteractionBinding,
+  sameInteractionBinding,
+} from "./interaction-binding.js";
+
+export const BACKGROUND_INTERACTION_JOURNAL_EVENT =
+  "background_interaction_journal";
+export const BACKGROUND_INTERACTION_JOURNAL_VERSION = 1;
+export const MAX_BACKGROUND_INTERACTION_RECORDS = 256;
+
+const TERMINAL_STATUSES = new Set(["resolved", "rejected", "cancelled"]);
+
+export const _deps = {
+  appendEvent: storeAppendEvent,
+  readEvents: storeReadEvents,
+};
+
+function journalError(message, code, details = {}) {
+  const error = new Error(message);
+  error.code = code;
+  Object.assign(error, details);
+  return error;
+}
+
+function cloneJson(value) {
+  if (value === undefined) return null;
+  return JSON.parse(JSON.stringify(value));
+}
+
+function canonicalize(value) {
+  if (value === undefined) return "__undefined__";
+  if (value === null || typeof value !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) {
+    return `[${value.map((entry) => canonicalize(entry)).join(",")}]`;
+  }
+  return `{${Object.keys(value)
+    .sort()
+    .map((key) => `${JSON.stringify(key)}:${canonicalize(value[key])}`)
+    .join(",")}}`;
+}
+
+function fingerprint(value) {
+  return createHash("sha256").update(canonicalize(value)).digest("hex");
+}
+
+function normalizeError(error, fallbackCode = "INTERACTION_REJECTED") {
+  if (!error) return null;
+  if (typeof error === "string") {
+    return { code: fallbackCode, message: error };
+  }
+  return {
+    code: String(error.code || fallbackCode),
+    message: String(error.message || error),
+  };
+}
+
+function normalizeSettlement(outcome = {}) {
+  const status = String(outcome.status || "");
+  if (!TERMINAL_STATUSES.has(status)) {
+    throw journalError(
+      `Invalid background interaction settlement status: ${status || "<missing>"}`,
+      "INTERACTION_SETTLEMENT_INVALID",
+    );
+  }
+  return {
+    status,
+    answer: status === "resolved" ? cloneJson(outcome.answer) : null,
+    error:
+      status === "resolved"
+        ? null
+        : normalizeError(
+            outcome.error,
+            status === "cancelled"
+              ? "INTERACTION_CANCELLED"
+              : "INTERACTION_REJECTED",
+          ),
+  };
+}
+
+export class BackgroundInteractionJournal {
+  constructor(options = {}) {
+    this.backgroundAgentId = String(options.backgroundAgentId || "");
+    if (!this.backgroundAgentId) {
+      throw journalError(
+        "backgroundAgentId is required for the interaction journal",
+        "INTERACTION_JOURNAL_INVALID",
+      );
+    }
+    this._now = options.now || Date.now;
+    this.records = [];
+  }
+
+  static fromJSON(snapshot, options = {}) {
+    if (
+      !snapshot ||
+      snapshot.version !== BACKGROUND_INTERACTION_JOURNAL_VERSION ||
+      typeof snapshot.backgroundAgentId !== "string" ||
+      !Array.isArray(snapshot.records)
+    ) {
+      throw journalError(
+        "Invalid background interaction journal snapshot",
+        "INTERACTION_JOURNAL_CORRUPT",
+      );
+    }
+    const journal = new BackgroundInteractionJournal({
+      backgroundAgentId: snapshot.backgroundAgentId,
+      now: options.now,
+    });
+    journal.records = snapshot.records.map((record) => ({
+      requestId: String(record.requestId || ""),
+      binding: normalizeInteractionBinding(record.binding),
+      payload: cloneJson(record.payload || {}),
+      payloadFingerprint: String(record.payloadFingerprint || ""),
+      status: String(record.status || "pending"),
+      createdAt: Number(record.createdAt) || 0,
+      settledAt: record.settledAt == null ? null : Number(record.settledAt),
+      settlement: record.settlement ? cloneJson(record.settlement) : null,
+      settlementFingerprint: record.settlementFingerprint
+        ? String(record.settlementFingerprint)
+        : null,
+    }));
+    return journal;
+  }
+
+  toJSON() {
+    return {
+      version: BACKGROUND_INTERACTION_JOURNAL_VERSION,
+      backgroundAgentId: this.backgroundAgentId,
+      records: cloneJson(this.records),
+    };
+  }
+
+  get(requestId) {
+    return (
+      this.records.find((record) => record.requestId === String(requestId)) ||
+      null
+    );
+  }
+
+  pending() {
+    return this.records.filter((record) => record.status === "pending");
+  }
+
+  recordPending({ requestId, binding, payload = {}, createdAt } = {}) {
+    const id = String(requestId || "");
+    const normalizedBinding = normalizeInteractionBinding(binding);
+    if (!id || normalizedBinding.sequence === null) {
+      throw journalError(
+        "A requestId and complete interaction binding are required",
+        "INTERACTION_REQUEST_INVALID",
+      );
+    }
+    const safePayload = cloneJson(payload);
+    const payloadFingerprint = fingerprint(safePayload);
+    const existing = this.get(id);
+    if (existing) {
+      if (
+        sameInteractionBinding(existing.binding, normalizedBinding) &&
+        existing.payloadFingerprint === payloadFingerprint
+      ) {
+        return { applied: false, record: cloneJson(existing) };
+      }
+      throw journalError(
+        `Interaction request ${id} was replayed with different binding or payload`,
+        "INTERACTION_REQUEST_CONFLICT",
+        { requestId: id },
+      );
+    }
+
+    const record = {
+      requestId: id,
+      binding: normalizedBinding,
+      payload: safePayload,
+      payloadFingerprint,
+      status: "pending",
+      createdAt: Number(createdAt) || this._now(),
+      settledAt: null,
+      settlement: null,
+      settlementFingerprint: null,
+    };
+    this.records.push(record);
+    this._prune();
+    return { applied: true, record: cloneJson(record) };
+  }
+
+  settle(requestId, binding, outcome = {}) {
+    const id = String(requestId || "");
+    const record = this.get(id);
+    if (!record) {
+      throw journalError(
+        `Unknown background interaction request: ${id || "<missing>"}`,
+        "INTERACTION_REQUEST_UNKNOWN",
+        { requestId: id },
+      );
+    }
+    if (!sameInteractionBinding(record.binding, binding)) {
+      throw journalError(
+        `Interaction response binding does not match request ${id}`,
+        "INTERACTION_BINDING_MISMATCH",
+        { requestId: id },
+      );
+    }
+    const settlement = normalizeSettlement(outcome);
+    const settlementFingerprint = fingerprint(settlement);
+    if (record.status !== "pending") {
+      if (record.settlementFingerprint === settlementFingerprint) {
+        return { applied: false, record: cloneJson(record) };
+      }
+      throw journalError(
+        `Interaction request ${id} is already settled`,
+        "INTERACTION_ALREADY_SETTLED",
+        { requestId: id, status: record.status },
+      );
+    }
+
+    record.status = settlement.status;
+    record.settledAt = this._now();
+    record.settlement = settlement;
+    record.settlementFingerprint = settlementFingerprint;
+    return { applied: true, record: cloneJson(record) };
+  }
+
+  rejectPending(error = {}) {
+    const rejected = [];
+    for (const record of this.pending()) {
+      const result = this.settle(record.requestId, record.binding, {
+        status: "rejected",
+        error,
+      });
+      rejected.push(result.record);
+    }
+    return rejected;
+  }
+
+  _prune() {
+    while (this.records.length > MAX_BACKGROUND_INTERACTION_RECORDS) {
+      const terminalIndex = this.records.findIndex((record) =>
+        TERMINAL_STATUSES.has(record.status),
+      );
+      if (terminalIndex === -1) {
+        throw journalError(
+          `Background interaction journal exceeded ${MAX_BACKGROUND_INTERACTION_RECORDS} pending requests`,
+          "INTERACTION_PENDING_LIMIT",
+        );
+      }
+      this.records.splice(terminalIndex, 1);
+    }
+  }
+}
+
+export class BackgroundInteractionPersistenceError extends Error {
+  constructor(operation, sessionId, cause) {
+    const detail =
+      cause?.message || String(cause || "unknown persistence error");
+    super(
+      `Background interaction journal ${operation} failed for ${sessionId}: ${detail}`,
+      { cause },
+    );
+    this.name = "BackgroundInteractionPersistenceError";
+    this.code =
+      operation === "read"
+        ? "INTERACTION_JOURNAL_READ_FAILED"
+        : "INTERACTION_JOURNAL_PERSIST_FAILED";
+    this.operation = operation;
+    this.sessionId = sessionId;
+  }
+}
+
+export function persistBackgroundInteractionJournal(
+  sessionId,
+  journal,
+  options = {},
+) {
+  const failIfUnavailable = options.failIfUnavailable !== false;
+  try {
+    if (!sessionId || !(journal instanceof BackgroundInteractionJournal)) {
+      throw new TypeError("sessionId and interaction journal are required");
+    }
+    _deps.appendEvent(
+      sessionId,
+      BACKGROUND_INTERACTION_JOURNAL_EVENT,
+      journal.toJSON(),
+    );
+    return true;
+  } catch (error) {
+    if (!failIfUnavailable) return false;
+    throw new BackgroundInteractionPersistenceError(
+      "write",
+      sessionId || "<missing-session>",
+      error,
+    );
+  }
+}
+
+export function loadBackgroundInteractionJournal(
+  sessionId,
+  backgroundAgentId,
+  options = {},
+) {
+  const failIfUnavailable = options.failIfUnavailable !== false;
+  try {
+    if (!sessionId || !backgroundAgentId) {
+      throw new TypeError("sessionId and backgroundAgentId are required");
+    }
+    const events = _deps.readEvents(sessionId) || [];
+    for (let index = events.length - 1; index >= 0; index--) {
+      const event = events[index];
+      if (
+        event?.type === BACKGROUND_INTERACTION_JOURNAL_EVENT &&
+        event.data?.backgroundAgentId === backgroundAgentId
+      ) {
+        return BackgroundInteractionJournal.fromJSON(event.data, options);
+      }
+    }
+    return new BackgroundInteractionJournal({
+      backgroundAgentId,
+      now: options.now,
+    });
+  } catch (error) {
+    if (!failIfUnavailable) {
+      return new BackgroundInteractionJournal({
+        backgroundAgentId,
+        now: options.now,
+      });
+    }
+    throw new BackgroundInteractionPersistenceError(
+      "read",
+      sessionId || "<missing-session>",
+      error,
+    );
+  }
+}
+
+/**
+ * Clone-mutate-persist transaction. The live journal is not changed when the
+ * append fails, so a caller may safely retry the same settlement.
+ */
+export function updateBackgroundInteractionJournal(
+  sessionId,
+  journal,
+  mutate,
+  options = {},
+) {
+  const draft = BackgroundInteractionJournal.fromJSON(journal.toJSON(), {
+    now: options.now || journal._now,
+  });
+  const result = mutate(draft);
+  if (!options.persistIf || options.persistIf(result) !== false) {
+    persistBackgroundInteractionJournal(sessionId, draft, options);
+  }
+  return { journal: draft, result };
+}
+
+/**
+ * Recovery entry point used when a worker is observed lost or starts again.
+ * An optional state-file fallback lets pre-journal pendingQuestion records be
+ * imported once and deterministically rejected.
+ */
+export function rejectPendingBackgroundInteractions(
+  sessionId,
+  backgroundAgentId,
+  options = {},
+) {
+  let journal = loadBackgroundInteractionJournal(
+    sessionId,
+    backgroundAgentId,
+    options,
+  );
+  const fallback = options.fallbackRequest;
+  const hasFallback =
+    fallback?.requestId &&
+    normalizeInteractionBinding(fallback.binding).sequence !== null &&
+    !journal.get(fallback.requestId);
+  if (!journal.pending().length && !hasFallback) {
+    return { journal, rejected: [], changed: false };
+  }
+  const mutation = updateBackgroundInteractionJournal(
+    sessionId,
+    journal,
+    (draft) => {
+      if (hasFallback) draft.recordPending(fallback);
+      return draft.rejectPending({
+        code: options.code || "INTERACTION_WORKER_LOST",
+        message:
+          options.message ||
+          "The background worker exited before the interaction settled",
+      });
+    },
+    options,
+  );
+  journal = mutation.journal;
+  return { journal, rejected: mutation.result, changed: true };
+}

package/src/lib/background-interaction-resolver.js

@@ -13,6 +13,16 @@
  */
 
 import { randomUUID } from "node:crypto";
+import {
+  hasCompleteInteractionBinding,
+  normalizeInteractionBinding,
+  sameInteractionBinding,
+} from "./interaction-binding.js";
+
+export {
+  normalizeInteractionBinding,
+  sameInteractionBinding,
+} from "./interaction-binding.js";
 
 export const BACKGROUND_INTERACTION_PROTOCOL_VERSION = 1;
 export const DEFAULT_BACKGROUND_INTERACTION_TIMEOUT_MS = 60_000;
@@ -30,31 +40,6 @@ function asNullableString(value) {
   return String(value);
 }
 
-function normalizeBinding(binding = {}) {
-  return {
-    backgroundAgentId: asNullableString(binding.backgroundAgentId),
-    sessionId: asNullableString(binding.sessionId),
-    turnId: asNullableString(binding.turnId),
-    toolUseId: asNullableString(binding.toolUseId),
-    sequence:
-      Number.isSafeInteger(binding.sequence) && binding.sequence > 0
-        ? binding.sequence
-        : null,
-  };
-}
-
-function sameBinding(left, right) {
-  const a = normalizeBinding(left);
-  const b = normalizeBinding(right);
-  return (
-    a.backgroundAgentId === b.backgroundAgentId &&
-    a.sessionId === b.sessionId &&
-    a.turnId === b.turnId &&
-    a.toolUseId === b.toolUseId &&
-    a.sequence === b.sequence
-  );
-}
-
 function validRequest(message) {
   return (
     message &&
@@ -65,12 +50,16 @@ function validRequest(message) {
     message.requestId.length <= 256 &&
     message.payload &&
     typeof message.payload === "object" &&
-    normalizeBinding(message.binding).sequence !== null
+    hasCompleteInteractionBinding(message.binding)
   );
 }
 
 function sendIpc(target, message, onError = () => {}) {
-  if (!target || typeof target.send !== "function" || target.connected === false)
+  if (
+    !target ||
+    typeof target.send !== "function" ||
+    target.connected === false
+  )
     return false;
   try {
     target.send(message, (error) => {
@@ -138,7 +127,8 @@ export function createBackgroundInteractionClient(options = {}) {
       return;
     }
     const entry = pending.get(message.requestId);
-    if (!entry || !sameBinding(entry.binding, message.binding)) return;
+    if (!entry || !sameInteractionBinding(entry.binding, message.binding))
+      return;
 
     if (message.status === "resolved") {
       settle(entry, entry.resolve, message.answer);
@@ -170,7 +160,7 @@ export function createBackgroundInteractionClient(options = {}) {
 
       sequence += 1;
       const requestId = `${backgroundAgentId}:${idFactory()}`;
-      const binding = normalizeBinding({
+      const binding = normalizeInteractionBinding({
         backgroundAgentId,
         sessionId: payload.sessionId ?? sessionId,
         turnId: payload.turnId ?? turnId,
@@ -301,7 +291,7 @@ export function attachInteractionRequestHandler(
       protocolVersion: BACKGROUND_INTERACTION_PROTOCOL_VERSION,
       requestId:
         typeof message?.requestId === "string" ? message.requestId : "",
-      binding: normalizeBinding(message?.binding),
+      binding: normalizeInteractionBinding(message?.binding),
       status: "rejected",
       error: { code, message: detail },
     };
@@ -315,7 +305,7 @@ export function attachInteractionRequestHandler(
       message.protocolVersion === BACKGROUND_INTERACTION_PROTOCOL_VERSION
     ) {
       const active = inflight.get(message.requestId);
-      if (active && sameBinding(active.binding, message.binding)) {
+      if (active && sameInteractionBinding(active.binding, message.binding)) {
         inflight.delete(message.requestId);
         active.controller.abort(message.reason || "cancelled");
       }
@@ -331,12 +321,11 @@ export function attachInteractionRequestHandler(
       return;
     }
 
-    const binding = normalizeBinding(message.binding);
+    const binding = normalizeInteractionBinding(message.binding);
     const expectedAgentId = asNullableString(options.backgroundAgentId);
     const expectedSessionId = asNullableString(options.sessionId);
     if (
-      (expectedAgentId &&
-        binding.backgroundAgentId !== expectedAgentId) ||
+      (expectedAgentId && binding.backgroundAgentId !== expectedAgentId) ||
       (expectedSessionId && binding.sessionId !== expectedSessionId)
     ) {
       rejectRequest(

package/src/lib/background-session-transport.js

@@ -198,7 +198,8 @@ export function startBackgroundSessionServer(opts) {
             case "interaction_response": {
               try {
                 // Support both field names: requestId (client) and intId (wire legacy)
-                const { requestId, intId, answer, result, error } = message;
+                const { requestId, intId, binding, answer, result, error } =
+                  message;
                 const resolvedId = requestId || intId;
                 if (!resolvedId) {
                   writeMessage(socket, {
@@ -208,14 +209,23 @@ export function startBackgroundSessionServer(opts) {
                   return;
                 }
                 // Single-object callback matching worker expectation: { requestId, answer, error }
-                onInteractionResponse?.({
+                const settlement = onInteractionResponse?.({
                   requestId: resolvedId,
+                  binding,
                   answer: answer ?? result,
                   error,
                 });
                 writeMessage(socket, {
                   type: "received",
                   requestId: resolvedId,
+                  accepted:
+                    typeof settlement === "object"
+                      ? settlement.accepted === true
+                      : settlement === true,
+                  duplicate:
+                    typeof settlement === "object"
+                      ? settlement.duplicate === true
+                      : false,
                 });
               } catch (err) {
                 writeMessage(socket, {