@awak-app/simy-cli 0.5.71 → 0.5.73

package/src/runner.js

@@ -1,4 +1,5 @@
 import { execFile, execFileSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
 import { EventEmitter, once } from "node:events";
 import { access } from "node:fs/promises";
 import path from "node:path";
@@ -20,7 +21,7 @@ import {
 } from "./orchestrator/execution-io.js";
 import { summarizeCodingLoopEvent } from "./orchestrator/presentation.js";
 import { PR_PREPARATION_CHECKS } from "./orchestrator/pr-preparation.js";
-import { increaseProviderTokenBudget, refreshBudgetState } from "./orchestrator/budget.js";
+import { budgetState, extendProgressCeilings, increaseProviderTokenBudget, providerBudgetTokens, refreshBudgetState } from "./orchestrator/budget.js";
 import { appendEvent } from "./orchestrator/shared.js";
 import {
   createProviderStreamDecoder,
@@ -42,7 +43,7 @@ import { LocalRunRegistry } from "./run-registry.js";
 import { webApiHeaders, webApiUrl } from "./web-api.js";
 import { normalizeGitHubRemote } from "./workspace-context.js";
 import { CLI_VERSION } from "./cli-contract.js";
-import { reviseObjective, verifyObjective } from "./orchestrator/outcome-monitor.js";
+import { freezeObjective, reviseObjective, verifyObjective } from "./orchestrator/outcome-monitor.js";
 import { hasUserOutcomeProof } from "./orchestrator/completion-contract.js";
 import { createSqmChecker } from "./sqm/index.js";
 import {
@@ -51,6 +52,10 @@ import {
   normalizeWorkflowStartIntent,
 } from "./workflow-observation-sender.js";
 import { saveCodingRunSnapshot } from "./coding-run-store.js";
+import { acquireDesktopProcessLock } from "./desktop-process-lock.js";
+import {
+  assertRecoveryActor, recoveryActorBinding, refreshRecoveryAudit, refreshRecoveryBudgetAudit,
+} from "./coding-run-recovery.js";
 
 const execFileAsync = promisify(execFile);
 const MAX_LOCAL_LOG_LINES = 1_000;
@@ -133,6 +138,7 @@ export function createRun({ runId, request, session, apiOrigin, localStateRoot =
       metadata: {
         local_orchestrator_version: 1,
         local_output_persisted: false,
+        recovery_actor: recoveryActorBinding(session, apiOrigin, normalizedRequest),
         ...(normalizedRequest.workflow_observation_binding
           ? {
               workflow_observation: {
@@ -188,6 +194,10 @@ export function restoreRun({ snapshot, session, apiOrigin, localPath = null, loc
       max_attempts: charter.max_attempts,
       retry_budget: charter.retry_budget,
       token_budget: charter.token_budget,
+      continuation_policy: charter.continuation_policy,
+      provider_token_ceiling: charter.provider_token_ceiling,
+      attempt_ceiling: charter.attempt_ceiling,
+      max_no_progress_attempts: charter.max_no_progress_attempts,
       ui_evidence_root: String(charter.ui_evidence_root || ""),
       acceptance_criteria: Array.isArray(charter.acceptance_criteria)
         ? charter.acceptance_criteria
@@ -245,6 +255,15 @@ export function restoreRun({ snapshot, session, apiOrigin, localPath = null, loc
   const objectiveUnverified = !objectiveCheck.valid || objectiveCheck.legacy || savedGoalUnproven;
   const restoredState = restoredFromState || objectiveUnverified ? "waiting_human" : persistedState;
   if (restoredFromState) {
+    restoredSnapshot.metadata ??= {};
+    restoredSnapshot.metadata.recovery = {
+      version: 1,
+      status: ["dispatching", "coding"].includes(restoredFromState)
+        ? "needs_reconciliation" : "ready",
+      interrupted_state: restoredFromState,
+      interrupted_attempt: restoredSnapshot.events?.findLast((item) =>
+        Number.isInteger(item?.detail?.attempt_number))?.detail.attempt_number ?? null,
+    };
     restoredSnapshot.events = Array.isArray(restoredSnapshot.events)
       ? restoredSnapshot.events
       : [];
@@ -302,6 +321,63 @@ export async function publishRestoredLocalCodingRun(run) {
   return run.snapshot;
 }
 
+export async function prepareLocalCodingRunRecovery(run) {
+  assertRecoveryActor(run);
+  if (run.operation || run.child) throw new Error("The selected run is still active.");
+  const repositoryPath = await resolveRepositoryPath(run.request);
+  const git = async (args) => (await execFileAsync("git", args, {
+    cwd: repositoryPath, maxBuffer: 16 * 1024 * 1024,
+  })).stdout;
+  const head = (await git(["rev-parse", "HEAD"])).trim();
+  const status = await git(["status", "--porcelain=v1", "--untracked-files=all"]);
+  const diff = await git(["diff", "HEAD", "--binary"]);
+  return {
+    version: 1, run_id: run.id,
+    objective: verifyObjective(run.snapshot.charter),
+    recovery: structuredClone(run.snapshot.metadata?.recovery || { status: "ready" }),
+    checkout: { head_sha: head, status_sha256: executionTextSha256(status), diff_sha256: executionTextSha256(diff) },
+    attempts_recorded: run.snapshot.attempts.length,
+    last_pr_url: run.snapshot.attempts.at(-1)?.pr_url || null,
+  };
+}
+
+// Only the controller's direct human decision or a trusted reconciler may
+// provide this result. A refreshed checkout alone cannot prove an external
+// publication failed or authorize replaying it.
+export async function reconcileLocalCodingRun(run, { operationStatus, observation, evidenceRefs, authorize } = {}) {
+  assertRecoveryActor(run);
+  if (run.operation || run.child || !RESUMABLE_STATES.has(run.status)) {
+    throw new Error("Interrupted operation reconciliation is available only while the run is waiting.");
+  }
+  if (run.snapshot.metadata?.recovery?.status !== "needs_reconciliation") {
+    throw new Error("This run has no interrupted operation requiring reconciliation.");
+  }
+  if (!["completed", "not_started", "failed", "none"].includes(operationStatus) ||
+      !String(observation || "").trim() || !Array.isArray(evidenceRefs) ||
+      !evidenceRefs.some((item) => typeof item === "string" && item.trim())) {
+    throw new Error("An authoritative operation status, observation and evidence references are required; unknown success cannot be replayed.");
+  }
+  const descriptor = await prepareLocalCodingRunRecovery(run);
+  await authorize?.();
+  assertRecoveryActor(run);
+  if (run.operation || run.child || !RESUMABLE_STATES.has(run.status) || run.snapshot.metadata?.recovery?.status !== 'needs_reconciliation') {
+    throw new Error('The interrupted run changed while reconciliation evidence was inspected.');
+  }
+  run.snapshot.metadata.recovery = {
+    ...descriptor.recovery, status: "ready", operation_status: operationStatus,
+    reconciled_at: new Date().toISOString(), checkout: descriptor.checkout,
+    observation_sha256: executionTextSha256(String(observation).trim()),
+    evidence_refs: evidenceRefs.filter((item) => typeof item === "string" && item.trim()),
+    actor: recoveryActorBinding(run.session, run.apiOrigin, run.request),
+  };
+  appendEvent(run.snapshot, "waiting_human", "The interrupted operation was reconciled before continuation.", {
+    code: "interrupted_operation_reconciled", operation_status: operationStatus,
+    interrupted_attempt: descriptor.recovery.interrupted_attempt,
+  });
+  await updateRun(run, "waiting_human");
+  return run.snapshot.metadata.recovery;
+}
+
 export async function startLocalCodingRun(
   run,
   dependencies = {},
@@ -329,18 +405,29 @@ function assertRunCanContinue(run) {
   if (!RESUMABLE_STATES.has(run.status)) {
     throw new Error("Human guidance is available only when the selected run is waiting.");
   }
+  assertRecoveryActor(run);
+  if (run.snapshot.metadata?.recovery?.status === "needs_reconciliation") {
+    const error = new Error("Reconcile the interrupted operation before continuing the same run; its side effects may already exist.");
+    error.code = "interrupted_operation_unverified";
+    throw error;
+  }
+  if (verifyObjective(run.snapshot.charter).legacy) {
+    const error = new Error("Explicitly confirm the saved objective before continuing this legacy run.");
+    error.code = "objective_baseline_unverified";
+    throw error;
+  }
 
-  const nextAttempt = run.snapshot.attempts.length + 1;
   const budget = refreshBudgetState(run.snapshot);
   if (budget.token_exhausted) {
     throw new Error("The Agentic Loop run used its token budget. Start a new run to continue.");
   }
-  if (nextAttempt > run.snapshot.charter.max_attempts) {
+  if (budget.retry_exhausted) {
     throw new Error("The Agentic Loop run used its retry budget. Start a new run to continue.");
   }
 }
 
 export async function increaseLocalCodingRunProviderTokenBudget(run, value) {
+  assertRecoveryActor(run);
   if (run.operation || run.child) {
     throw new Error("Provider token budget can change only while the run is waiting.");
   }
@@ -352,7 +439,12 @@ export async function increaseLocalCodingRunProviderTokenBudget(run, value) {
     throw new Error("The Agentic Loop run also used its retry budget. Start a new run to continue.");
   }
   const previousBudget = currentBudget.token_budget;
+  const previousAudit = run.snapshot.final_audit;
+  const resolvedBudgetAudit = previousAudit?.findings?.some(
+    (finding) => finding.code === "TOKEN_BUDGET_EXHAUSTED",
+  ) ? structuredClone(previousAudit) : null;
   const budget = increaseProviderTokenBudget(run.snapshot, value);
+  refreshRecoveryBudgetAudit(run.snapshot, budget);
   run.request.token_budget = budget.token_budget;
   appendEvent(
     run.snapshot,
@@ -361,6 +453,7 @@ export async function increaseLocalCodingRunProviderTokenBudget(run, value) {
     {
       code: "provider_token_budget_increased",
       previous_provider_token_budget: previousBudget,
+      ...(resolvedBudgetAudit ? { previous_final_audit: resolvedBudgetAudit } : {}),
       provider_token_budget: budget.token_budget,
       budget,
     },
@@ -369,6 +462,29 @@ export async function increaseLocalCodingRunProviderTokenBudget(run, value) {
   return budget;
 }
 
+export async function extendLocalCodingRunCeilings(run, changes) {
+  assertRecoveryActor(run);
+  if (run.operation || run.child || !RESUMABLE_STATES.has(run.status)) {
+    throw new Error("Hard ceilings can change only while the selected run is waiting and idle.");
+  }
+  const previous = budgetState(run.snapshot);
+  const budget = extendProgressCeilings(run.snapshot, changes);
+  refreshRecoveryBudgetAudit(run.snapshot, budget);
+  run.request.provider_token_ceiling = run.snapshot.charter.provider_token_ceiling;
+  run.request.attempt_ceiling = run.snapshot.charter.attempt_ceiling;
+  appendEvent(run.snapshot, run.status, "The human operator explicitly increased this saved run's hard ceilings.", {
+    code: "continuation_ceilings_extended",
+    previous_provider_token_ceiling: previous.hard_token_ceiling,
+    previous_attempt_ceiling: previous.hard_attempt_ceiling,
+    provider_token_ceiling: budget.hard_token_ceiling,
+    attempt_ceiling: budget.hard_attempt_ceiling,
+    budget,
+  });
+  // Extending consent is separate from continuing work, including a stopped run.
+  await updateRun(run, run.status);
+  return budget;
+}
+
 export async function continueLocalCodingRunAfterRepositoryApproval(
   run,
   { repository, localPath } = {},
@@ -437,11 +553,39 @@ export function queueLocalCodingGuidance(run, guidance) {
 
 export async function applyLocalHumanDecision(
   run,
-  { message, designApproval = false, requiredChecks = [], acceptanceCriteria = [] } = {},
+  { message, designApproval = false, requiredChecks = [], acceptanceCriteria = [], confirmObjective = false, confirmedObjective = null } = {},
   dependencies = {},
 ) {
   const guidance = String(message || "").trim();
   if (!guidance) throw new Error("A human decision note is required.");
+  if (confirmObjective) {
+    assertRecoveryActor(run);
+    if (run.operation || run.child || !RESUMABLE_STATES.has(run.status)) {
+      throw new Error("Objective confirmation is available only while the saved run is waiting.");
+    }
+    const objective = verifyObjective(run.snapshot.charter);
+    if (!objective.valid) throw new Error("Restore the saved objective or explicitly revise its changed criteria; an invalid integrity chain cannot be rebaselined.");
+    if (objective.legacy) {
+      const charter = run.snapshot.charter;
+      if (confirmedObjective?.originalRequest !== charter.thread_state.original_request ||
+          confirmedObjective?.userGoal !== charter.thread_state.user_goal ||
+          JSON.stringify(confirmedObjective?.acceptanceCriteria) !== JSON.stringify(charter.acceptance_criteria)) {
+        throw new Error("Confirm the exact saved original request, user goal and acceptance criteria to establish a legacy baseline.");
+      }
+      freezeObjective(run.snapshot.charter);
+      refreshRecoveryAudit(run.snapshot, {
+        reason: "objective_baseline_confirmed",
+        stale: (finding) => finding.code === "OBJECTIVE_BASELINE_UNVERIFIED",
+        summary: "The saved objective was confirmed; remaining verification or approval is still required.",
+      });
+      run.snapshot.done_gate_passed = false;
+      run.snapshot.implementation_gate_passed = false;
+      appendEvent(run.snapshot, "waiting_human", "The original saved objective was explicitly confirmed.", {
+        code: "objective_baseline_confirmed", note_sha256: executionTextSha256(guidance),
+      });
+      await updateRun(run, "waiting_human");
+    }
+  }
   assertRunCanContinue(run);
   const charter = run.snapshot.charter;
   if (acceptanceCriteria.length > 0) {
@@ -540,10 +684,12 @@ export async function stopLocalCodingRun(run) {
 async function signalAndWaitForClose(child, signal, timeoutMs) {
   const close = once(child, "close").then(() => "closed");
   signalExecutorProcess(child, signal);
-  return Promise.race([
-    close,
-    new Promise((resolve) => setTimeout(() => resolve("timeout"), timeoutMs)),
-  ]);
+  let timer;
+  try {
+    return await Promise.race([close,
+      new Promise((resolve) => { timer = setTimeout(() => resolve("timeout"), timeoutMs); }),
+    ]);
+  } finally { clearTimeout(timer); }
 }
 
 async function waitForLocalStop(run) {
@@ -598,13 +744,17 @@ async function executeLocalCodingRun(
   { executeAttempt, executeIndependentAudit, collectEvidence } = {},
   { humanGuidance = "", resume = false } = {},
 ) {
-  const operation = runLocalCodingRun(run, {
-    executeAttempt,
-    executeIndependentAudit,
-    collectEvidence,
-    humanGuidance,
-    resume,
-  });
+  const operation = (async () => {
+    assertRecoveryActor(run);
+    const lease = run.localStateRoot ? await acquireDesktopProcessLock(
+      path.join(run.localStateRoot, ".controllers", executionTextSha256(run.id)),
+    ) : null;
+    try {
+      return await runLocalCodingRun(run, {
+        executeAttempt, executeIndependentAudit, collectEvidence, humanGuidance, resume,
+      });
+    } finally { await lease?.release(); }
+  })();
   run.operation = operation;
   try {
     return await operation;
@@ -646,6 +796,16 @@ async function runLocalCodingRun(
     await updateRun(run, run.status);
   }
 
+  const recovery = run.snapshot.metadata?.recovery;
+  if (resume && recovery?.checkout) {
+    const recoveredGuidance = [
+      "Resume the same saved objective and approvals. Reconcile existing work before taking another action.",
+      `Interrupted operation status: ${recovery.operation_status}. Checkout at reconciliation: ${JSON.stringify(recovery.checkout)}.`,
+      "Never duplicate a completed external operation; inspect its existing identity and evidence. If its current outcome is unknown, stop for reconciliation instead of replaying it.",
+    ].join("\n");
+    humanGuidance = [recoveredGuidance, humanGuidance].filter(Boolean).join("\n\n");
+  }
+
   const executor =
     executeAttempt ||
     ((context) =>
@@ -687,6 +847,7 @@ async function runLocalCodingRun(
       deviceId: run.session?.device_id,
     });
   run.sqmChecker = sqmChecker;
+  refreshSqmCheckerSession(run, sqmChecker);
 
   try {
     await runCodingLoop({
@@ -762,6 +923,7 @@ export async function recheckLocalCodingRun(run, { collectEvidence } = {}) {
       deviceId: run.session?.device_id,
     });
   run.sqmChecker = sqmChecker;
+  refreshSqmCheckerSession(run, sqmChecker);
   await recheckPrReadiness({
     snapshot: run.snapshot,
     collectEvidence: evidenceCollector,
@@ -800,6 +962,16 @@ function sqmBundleEndpoint(session) {
   return new URL("sqm/knowledge-bundle", session.api_base_url).href;
 }
 
+function refreshSqmCheckerSession(run, checker) {
+  assertRecoveryActor(run);
+  if (!run.session?.token || !checker.updateSession) return;
+  checker.updateSession({ token: run.session.token,
+    accountId: run.session.account_id || run.session.auth_user_id || run.session.device_id,
+    organizationId: run.session.organization_id || run.session.org_id,
+    deviceId: run.session.device_id, endpoint: sqmBundleEndpoint(run.session), webOrigin: run.apiOrigin,
+  });
+}
+
 function sqmEvidenceEndpoint(session) {
   if (process.env.SIMY_SQM_EVIDENCE_URL?.trim()) return process.env.SIMY_SQM_EVIDENCE_URL.trim();
   if (!session?.api_base_url) return undefined;
@@ -813,8 +985,9 @@ export async function executeProcessAttempt({
   run,
   marker = "SIMY_RESULT_JSON:",
   attemptNumber = null,
+  resolveCommandImpl = resolveDesktopExecutorCommand,
 }) {
-  const command = await resolveDesktopExecutorCommand({
+  const command = await resolveCommandImpl({
     backend,
     instruction,
     repositoryPath,
@@ -824,6 +997,8 @@ export async function executeProcessAttempt({
   const stderr = [];
   const tokenUsageRecords = [];
   const phase = marker === "SIMY_AUDIT_JSON:" ? "reviewer" : "executor";
+  const initialBudget = budgetState(run.snapshot);
+  const invocationId = randomUUID();
   const observationSender =
     phase === "executor" &&
     !["accepted", "already_recorded"].includes(run.snapshot.metadata?.workflow_observation?.status)
@@ -839,6 +1014,8 @@ export async function executeProcessAttempt({
   return new Promise((resolve) => {
     let settled = false;
     let terminalError = null;
+    let terminalErrorCode = null;
+    const checkpointWrites = [];
     const child = command.spawn(command.bin, command.args, {
       cwd: repositoryPath,
       env: { ...process.env, ...command.env },
@@ -853,13 +1030,42 @@ export async function executeProcessAttempt({
       onLine: (line) => emitOutput(run, line, { backend }),
       onEvent: (event) => observationSender?.observe(event),
       onTerminalError: (message) => { terminalError ||= message; },
-      onUsage: (usage) =>
+      onUsage: (usage) => {
         tokenUsageRecords.push({
           ...usage,
           phase,
           ...(Number.isInteger(attemptNumber) ? { attempt_number: attemptNumber } : {}),
           backend,
-        }),
+          ...(initialBudget.continuation_policy === "progress" ? { invocation_id: invocationId } : {}),
+        });
+        if (initialBudget.continuation_policy !== "progress") return;
+        const currentUsed = initialBudget.tokens_used + tokenUsageRecords.reduce((sum, record) => sum + providerBudgetTokens(record), 0);
+        run.snapshot.metadata.provider_usage_checkpoints ??= {};
+        run.snapshot.metadata.provider_usage_checkpoints[invocationId] = {
+          invocation_id: invocationId, phase, attempt_number: attemptNumber, started_at: startedAt,
+          token_usage: { records: structuredClone(tokenUsageRecords) },
+          total_provider_tokens: currentUsed,
+          hard_token_ceiling: initialBudget.hard_token_ceiling,
+        };
+        refreshBudgetState(run.snapshot);
+        if (run.localStateRoot) checkpointWrites.push(saveCodingRunSnapshot(run.localStateRoot, run.snapshot).catch(() => {
+          emitOutput(run, "The Provider usage checkpoint could not be persisted; recovery usage remains unverified.", { backend });
+        }));
+        if (currentUsed < initialBudget.hard_token_ceiling || terminalErrorCode || run.stopRequested) return;
+        terminalErrorCode = "TOKEN_BUDGET_EXHAUSTED";
+        terminalError = `Provider token hard ceiling reached (${currentUsed} / ${initialBudget.hard_token_ceiling}). Extend the same run's ceiling explicitly before continuing.`;
+        appendEvent(run.snapshot, run.status, terminalError, {
+          code: "provider_token_ceiling_reached", phase, attempt_number: attemptNumber,
+          provider_tokens_used: currentUsed, provider_token_ceiling: initialBudget.hard_token_ceiling,
+        });
+        if (!settled) {
+          run.controlState = "stopping";
+          void signalAndWaitForClose(child, "SIGINT", 1_500).then(async result => {
+            if (result === "timeout" && !settled && run.child === child) result = await signalAndWaitForClose(child, "SIGTERM", 1_500);
+            if (result === "timeout" && !settled && run.child === child) await signalAndWaitForClose(child, "SIGKILL", 750);
+          }).catch(() => {});
+        }
+      },
     });
     const stderrDecoder = createProviderStreamDecoder({
       backend,
@@ -884,6 +1090,7 @@ export async function executeProcessAttempt({
       stdoutDecoder.flush();
       stderrDecoder.flush();
       run.child = null;
+      await Promise.all(checkpointWrites);
       // The run must always resolve: a WorkFlow observation failure (or a
       // sender bug) never changes or blocks the Provider result. flush() is
       // bounded by per-request timeouts and is designed not to reject.
@@ -920,6 +1127,7 @@ export async function executeProcessAttempt({
       resolve({
         exitCode: terminalError && exitCode === 0 ? 1 : exitCode,
         error: error || terminalError,
+        ...(terminalErrorCode ? { errorCode: terminalErrorCode } : {}),
         result: Object.keys(result).length > 0 ? result : parse(rawStdout),
         tokenUsage:
           tokenUsageRecords.length > 0 ? { records: tokenUsageRecords } : {},

package/src/runtime-diagnostics.js

@@ -0,0 +1,77 @@
+import { appendFileSync, chmodSync, mkdirSync, renameSync, rmSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { StringDecoder } from 'node:string_decoder';
+import { redactExecutionText } from './orchestrator/execution-io.js';
+
+// Local diagnostics are deliberately bounded and private. Use the same credential
+// policy as execution history, with URL queries removed (login nonces live there).
+export function createRuntimeDiagnostics(root, { name = 'daemon', maxBytes = 1024 * 1024, backups = 2 } = {}) {
+  const directory = join(root, 'logs');
+  const file = join(directory, `${name}.log`);
+  const write = value => {
+    try {
+      mkdirSync(directory, { recursive: true, mode: 0o700 });
+      const text = redactExecutionText(String(value)
+        .replace(/(\b(?:set-cookie|cookie)\s*:\s*)[^\r\n]*/gi, '$1[REDACTED]')
+        .replace(/(https?:\/\/[^\s?#]+)[?#][^\s]*/g, '$1?[REDACTED]'), { maxChars: 8192 }).text;
+      // Do not retain fragments of PEM keys, whose boundaries may span writes.
+      const safe = /PRIVATE KEY|^[A-Za-z0-9+/=]{64,}$/.test(text.trim()) ? '[REDACTED KEY MATERIAL]' : text;
+      const record = Buffer.from(`${new Date().toISOString()} ${safe.replace(/[\r\n]+/g, ' ')}\n`);
+      const bounded = record.subarray(0, maxBytes);
+      let size = 0;
+      try { size = statSync(file).size; } catch {}
+      if (size + bounded.length > maxBytes) {
+        rmSync(`${file}.${backups}`, { force: true });
+        for (let index = backups - 1; index >= 1; index--) {
+          try { renameSync(`${file}.${index}`, `${file}.${index + 1}`); } catch (error) { if (error.code !== 'ENOENT') throw error; }
+        }
+        try { renameSync(file, `${file}.1`); } catch (error) { if (error.code !== 'ENOENT') throw error; }
+      }
+      appendFileSync(file, bounded, { mode: 0o600 });
+      chmodSync(file, 0o600);
+    } catch {
+      // Diagnostics must never prevent startup/recovery or recurse via stderr.
+    }
+  };
+  const stream = label => {
+    const decoder = new StringDecoder('utf8');
+    let pending = '', dropping = false, privateKey = false;
+    const writeChunk = chunk => {
+      const text = decoder.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
+      for (const part of text.split(/(?<=\n)/)) {
+        const end = part.endsWith('\n');
+        if (!dropping) {
+          pending += part;
+          if (pending.length > 8192) {
+            // Drop the whole oversized line, rather than persisting an unredacted
+            // partial credential split across chunks or the truncation boundary.
+            write(`${label}: [oversized diagnostic line omitted]`);
+            pending = ''; dropping = !end;
+          } else if (end) {
+            if (/-----BEGIN .*PRIVATE KEY-----/.test(pending)) privateKey = true;
+            write(`${label}: ${privateKey ? '[REDACTED KEY MATERIAL]' : pending.trimEnd()}`);
+            if (/-----END .*PRIVATE KEY-----/.test(pending)) privateKey = false;
+            pending = '';
+          }
+        } else if (end) dropping = false;
+      }
+    };
+    return { write: writeChunk, flush() { if (pending && !dropping) write(`${label}: ${privateKey ? '[REDACTED KEY MATERIAL]' : pending}`); pending = ''; dropping = false; } };
+  };
+  return { file, write, stream };
+}
+
+export function captureRuntimeOutput(diagnostics, streams = { stdout: process.stdout, stderr: process.stderr }) {
+  const restores = [];
+  for (const [label, target] of Object.entries(streams)) {
+    const sink = diagnostics.stream(label);
+    const original = target.write;
+    target.write = function (chunk, ...args) { sink.write(chunk); return original.call(this, chunk, ...args); };
+    restores.push(() => { target.write = original; sink.flush(); });
+  }
+  const fatal = error => diagnostics.write(`uncaught: ${error?.stack || error}`);
+  process.on('uncaughtExceptionMonitor', fatal);
+  const restore = () => { for (const callback of restores) callback(); process.removeListener('uncaughtExceptionMonitor', fatal); process.removeListener('exit', restore); };
+  process.once('exit', restore);
+  return restore;
+}

package/src/session-supervision/codex-app-server.js

@@ -2,7 +2,7 @@ import { associationContract } from '../codex-plugins/simy/hooks/action-associat
 import { spawn } from 'node:child_process';
 import { realpath } from 'node:fs/promises';
 import { inspectBackendExecutable } from '../backend-executable.js';
-import { assessedCheckpointContract, understandingCheckpointContract } from '../codex-plugins/simy/hooks/delivery-loop.mjs';
+import { assessedCheckpointContract, deliveryBudgetState, understandingCheckpointContract } from '../codex-plugins/simy/hooks/delivery-loop.mjs';
 import { boundConversation, boundProviderGoal, textContent } from './conversation-context.js';
 import { JsonDataUrlFilter } from './json-data-url-filter.js';
 
@@ -422,7 +422,8 @@ export async function executeCodexAppServer(record, { openClient = openCodexAppS
       enrolledScopes: run.managed.approvalScopes,
       lastDependencyResolution: run.managed.lastDependencyResolution,
       ...(authorizedWait ? { approvedAction: { approvalId: approved.id, action: approved.action, oneShot: true } } : {}),
-      budgets: { deadline: run.deadline, continuations: run.continuations, maxContinuations: run.maxContinuations, failures: run.failures, maxFailures: run.maxFailures } };
+      budgets: { ...deliveryBudgetState(run), continuations: run.continuations, failures: run.failures, maxFailures: run.maxFailures,
+        softDeadline: run.deadline, softMaxContinuations: run.maxContinuations } };
     const approvalInstruction = authorizedWait ? `SIMY recorded a matching, one-shot scoped user approval ${approved.id} for this exact selected action. The decision outcome "wait" describes its pre-approval assessment; the approved action is authorized now. Execute it once, then report what actually happened. ` : '';
     const prompt = run.managed.phase === 'action_association' ? associationContract(run.managed.actionAssociation.job) : run.managed.phase === 'understanding' ? `Understand the ORIGINAL user request in this exact Codex conversation. This first/clarifying turn is read-only and must not perform task work. Context: ${JSON.stringify({ ...context, priorUnderstanding: run.managed.sessionUnderstanding, userAnswer: run.managed.understandingAnswer })}\n${understandingCheckpointContract(decision)} Do not write SIMY state directly.` : `Continue this existing SIMY managed task in the exact Codex conversation. Context: ${JSON.stringify(context)}\n${approvalInstruction}Execute ONLY the selected decision action. Do not perform any action from the NEXT plan in this turn. For a next local action within the enrolled task use the exact scope "task"; do not relabel external, broad, or destructive work as local. Preserve provider permissions. End with one JSON checkpoint. ${assessedCheckpointContract(decision)} Do not write SIMY state directly.`;
     const initialSandbox = run.managed.providerSandbox || resumed.sandbox;

package/src/session-supervision/index.js

@@ -141,7 +141,7 @@ export async function discoverSessions({ root, openClient = openCodexAppServer,
 }
 
 // Explicit execution authority: legacy monitor/tracked records stay read-only.
-export async function setSessionAutoContinue({ root, provider, session, enabled, ...options }) {
+export async function setSessionAutoContinue({ root, provider, session, enabled, continuationPolicy, hardMaxMinutes, hardMaxContinuations, providerTokenCeiling, ...options }) {
   if (!['codex', 'claude'].includes(provider) || typeof enabled !== 'boolean') throw new Error('Unsupported automatic continuation request.');
   if (enabled) await trackSession({ root, provider, session, tracked: true, ...options });
   return updateRecord(root, provider, session, async previous => {
@@ -152,8 +152,20 @@ export async function setSessionAutoContinue({ root, provider, session, enabled,
     if (enabled && provider === 'claude' && !await liveClaudeHook(previous,
       { session, cwd: previous.cwd, observation: { source: previous.admission?.source } }, options.inspectProcess || processIdentity)) throw new Error('Claude needs a live matching SIMY SessionStart hook; cold CLI takeover is unavailable.');
     const epoch = previous.autoContinue === enabled && previous.autoContinueEpoch ? previous.autoContinueEpoch : randomUUID();
+    const settings = { autonomyPolicy: 'balanced', maxMinutes: 30, maxContinuations: 6, maxFailures: 3,
+      ...previous.autoContinueSettings,
+      ...(continuationPolicy === undefined ? {} : { continuationPolicy }),
+      ...(hardMaxMinutes === undefined ? {} : { hardMaxMinutes }),
+      ...(hardMaxContinuations === undefined ? {} : { hardMaxContinuations }),
+      ...(providerTokenCeiling === undefined ? {} : { providerTokenCeiling }) };
+    if (settings.continuationPolicy !== undefined && !['bounded', 'progress'].includes(settings.continuationPolicy)) throw new Error('Choose a continuation policy.');
+    for (const [key, maximum] of [['hardMaxMinutes', 1440], ['hardMaxContinuations', 50], ['providerTokenCeiling', 15_000_000]]) {
+      if (settings[key] !== undefined && (!Number.isSafeInteger(settings[key]) || settings[key] < 1 || settings[key] > maximum)) throw new Error('Invalid continuation ceiling.');
+    }
+    if (providerTokenCeiling !== undefined && provider !== 'codex') throw new Error('Token ceilings require host-observed Codex usage.');
+    if (previous.run && [continuationPolicy, hardMaxMinutes, hardMaxContinuations, providerTokenCeiling].some(value => value !== undefined)) throw new Error('Change an existing task through its explicit continuation controls.');
     const next = { ...previous, autoContinue: enabled, autoContinueEpoch: epoch,
-      ...(enabled ? { tracked: true, monitor: true, autoContinueSettings: { autonomyPolicy: 'balanced', maxMinutes: 30, maxContinuations: 6, maxFailures: 3 } } : { tracked: false, monitor: false }) };
+      ...(enabled ? { tracked: true, monitor: true, autoContinueSettings: settings } : { tracked: false, monitor: false }) };
     // Adopt existing supervision without restarting it or renewing its budgets.
     if (next.run?.managed) next.run.managed.autoContinueEpoch = epoch;
     return next;
@@ -209,7 +221,7 @@ export async function monitorSession({ root, provider, session, monitor = true,
 }
 
 export async function enrollSession({ root, provider, session, objective, acceptanceCriteria, contextBootstrap = false, autonomyPolicy = 'balanced',
-  maxMinutes = 30, maxContinuations = 6, maxFailures = 3, maxNoProgressMinutes, maxTokens, autoContinueEpoch, openClient = openCodexAppServer,
+  maxMinutes = 30, maxContinuations = 6, maxFailures = 3, maxNoProgressMinutes, maxTokens, continuationPolicy, hardMaxMinutes, hardMaxContinuations, providerTokenCeiling, autoContinueEpoch, openClient = openCodexAppServer,
   readClaude = readClaudeSession, inspectProcess = processIdentity, now = Date.now() }) {
   if (!['codex', 'claude'].includes(provider)) throw new Error('This provider does not have a verified exact-session enrollment channel.');
   if (maxTokens !== undefined && provider !== 'codex') throw new Error('Token-budgeted supervision requires host-observed Codex usage; Claude live hooks cannot enforce this budget.');
@@ -235,7 +247,7 @@ export async function enrollSession({ root, provider, session, objective, accept
   const admission = provider === 'codex' ? { source: 'codex_app_server', provider, session, cwd, model: metadata.model } : undefined;
   const liveRecord = provider === 'claude' ? await readRecord(root, provider, session) : null;
   const run = await startDelivery({ root, provider, session, cwd, objective: objective.trim(), nextStep: 'Inspect current evidence and propose the next assessed action.',
-    maxMinutes, maxContinuations, maxFailures, now, admission, inspectProcess, autoContinueEpoch,
+    maxMinutes, maxContinuations, maxFailures, continuationPolicy, hardMaxMinutes, hardMaxContinuations, providerTokenCeiling, now, admission, inspectProcess, autoContinueEpoch,
     liveHookAdmission: liveRecord ? { hookStartedAt: liveRecord.hookStartedAt, pid: liveRecord.process.pid, identity: liveRecord.process.identity } : undefined,
     managed: { acceptanceCriteria, contextBootstrap, autonomyPolicy, maxNoProgressMinutes, maxTokens, approvalScopes: ['task'], approvalPolicy: 'wait',
       initialAction: initialPlan.alternatives[0].action, initialPlan, remainingWork: objective.trim(),

package/src/sqm/bundle-store.js

@@ -1,3 +1,4 @@
+import { safeSqmDiagnostic } from "./diagnostics.js";
 import { createHash, createPublicKey } from "node:crypto";
 import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
 import { homedir } from "node:os";
@@ -5,6 +6,7 @@ import path from "node:path";
 
 import { verifyBundleIntegrity } from "./canonical.js";
 import { validateKnowledgeBundle } from "./validation.js";
+import { fetchSqmRead, sqmDiagnostic } from "./request-recovery.js";
 
 const DEFAULT_TTL_MS = 5 * 60 * 1000;
 export const SQM_BUNDLE_FETCH_TIMEOUT_MS = 15_000;
@@ -35,6 +37,9 @@ export async function loadKnowledgeBundle({
   fetchImpl = globalThis.fetch,
   now = Date.now(),
   publicKeys = publicKeysFromEnvironment(),
+  signal,
+  recoveryOptions,
+  onRecoveryEvent,
 }) {
   if (!organizationId) {
     return result(null, ["SQM organization identity is unavailable; shadow checks were skipped."], "unavailable");
@@ -49,7 +54,7 @@ export async function loadKnowledgeBundle({
     Date.parse(cached.bundle.expires_at) > now;
   if (!offline && endpoint && (!fresh || refresh || cached?.bundle.system_scope)) {
     try {
-      const fetched = await fetchBundle({ endpoint, repository, cliVersion, token, organizationId, etag: cached?.etag, fetchImpl });
+      const fetched = await fetchBundle({ endpoint, repository, cliVersion, token, organizationId, etag: cached?.etag, fetchImpl, signal, recoveryOptions, onRecoveryEvent });
       if (fetched.notModified) {
         if (!cached) throw new Error("SQM Cloud returned 304 without a verified local bundle.");
         if (Date.parse(cached.bundle.expires_at) <= now) throw nonFallbackError("SQM Cloud returned 304 for an expired knowledge bundle.");
@@ -81,9 +86,10 @@ export async function loadKnowledgeBundle({
       });
       return result(bundle, [], "cloud");
     } catch (error) {
+      if (signal?.aborted) throw signal.reason;
       if (error?.code === "incompatible_cli") throw error;
       if (error?.fallbackAllowed !== false && cached && !cached.bundle.system_scope && Date.parse(cached.bundle.expires_at) > now) return result(cached.bundle, [`SQM Cloud unavailable; using last verified bundle: ${message(error)}`], "stale_cache");
-      return result(null, [`SQM knowledge could not be used; shadow checks were skipped: ${message(error)}`], "unavailable");
+      return { ...result(null, [`SQM knowledge could not be used; shadow checks were skipped: ${message(error)}`], "unavailable"), diagnostic: { ...safeSqmDiagnostic(error, "knowledge_bundle", "knowledge_bundle_unavailable"), ...sqmDiagnostic(error, "knowledge_bundle") } };
     }
   }
   if (cached) {
@@ -166,7 +172,7 @@ function trustedDiscoveryEndpoint(endpoint) {
     (url.protocol === "http:" && ["127.0.0.1", "localhost", "[::1]"].includes(url.hostname));
 }
 
-async function fetchBundle({ endpoint, repository, cliVersion, token, organizationId, etag, fetchImpl }) {
+async function fetchBundle({ endpoint, repository, cliVersion, token, organizationId, etag, fetchImpl, signal, recoveryOptions, onRecoveryEvent }) {
   const url = new URL(endpoint);
   url.searchParams.set("repository", repository);
   url.searchParams.set("cli_version", cliVersion);
@@ -175,10 +181,8 @@ async function fetchBundle({ endpoint, repository, cliVersion, token, organizati
   if (token) headers.Authorization = `Bearer ${token}`;
   if (organizationId) headers["x-simy-org-id"] = organizationId;
   if (etag) headers["If-None-Match"] = etag;
-  const response = await fetchImpl(url, {
-    headers,
-    signal: AbortSignal.timeout(SQM_BUNDLE_FETCH_TIMEOUT_MS),
-  });
+  const response = await fetchSqmRead(url, { ...recoveryOptions, headers, signal, fetchImpl,
+    requestTimeoutMs: SQM_BUNDLE_FETCH_TIMEOUT_MS, onRecoveryEvent });
   const responseEtag = response.headers?.get?.("etag") || null;
   const signingKeyId = response.headers?.get?.("x-simy-sqm-key-id") || null;
   const signingPublicKey = response.headers?.get?.("x-simy-sqm-public-key") || null;
@@ -200,6 +204,9 @@ async function fetchBundle({ endpoint, repository, cliVersion, token, organizati
       );
     }
     const error = new Error(`SQM Cloud returned HTTP ${response.status}${body ? `: ${body.slice(0, 300)}` : ""}.`);
+    error.status = response.status;
+    Object.assign(error, safeSqmDiagnostic({ status: response.status, code: parsed?.error?.code || parsed?.detail?.code,
+      requestId: response.headers?.get?.("x-request-id") }, "knowledge_bundle", "knowledge_bundle_http_error"));
     error.fallbackAllowed = response.status >= 500 && response.status <= 599;
     throw error;
   }