@awak-app/simy-cli 0.6.16 → 0.6.18

package/src/sqm/command.js

@@ -1,6 +1,6 @@
 import { sqmConversationIdentity } from './conversation-identity.js';
 import { safeSqmDiagnostic } from "./diagnostics.js";
-import { createSqmCheckRecorder } from "./history-store.js";
+import { createSqmHistoryStore, createSqmCheckRecorder } from "./history-store.js";
 import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
 import { homedir } from "node:os";
 import path from "node:path";
@@ -8,6 +8,7 @@ import path from "node:path";
 import { isSessionValid, readSession } from "../session-store.js";
 import { resolveWebOrigin } from "../web-origin.js";
 import { submitLocalEvidence } from "./local-evidence.js";
+import { verifySignedEvidence } from "./proof.js";
 import { uploadRunEvidence } from "./evidence-client.js";
 import { evaluateKnowledgeBundle } from "./checkers.js";
 import { runSqmCheck, runSqmMinCheck } from "./index.js";
@@ -24,7 +25,7 @@ export async function runSqmCommand(argv, { cliVersion }) {
   if (argv[0] === "submit-evidence") return await submitLocalEvidence(argv);
   if (argv[0] === "lint") return await runLintCommand(argv);
   if (argv[0] === "replay") return await runReplayCommand(argv);
-  if (!["min-check", "check"].includes(argv[0])) throw new Error("Usage: simy sqm <min-check|check|lint|replay> [options]");
+  if (!["min-check", "check", "upload-proof"].includes(argv[0])) throw new Error("Usage: simy sqm <min-check|check|upload-proof|lint|replay> [options]");
   const webOrigin = resolveWebOrigin(option(argv, "--host"));
   const authorizationLogger = argv.includes("--json")
     ? { log: message => console.warn(message), error: message => console.error(message) }
@@ -40,6 +41,28 @@ export async function runSqmCommand(argv, { cliVersion }) {
   const accountId = process.env.SIMY_SQM_ACCOUNT_ID || validSession?.account_id || validSession?.auth_user_id || validSession?.device_id;
   const deviceId = process.env.SIMY_SQM_DEVICE_ID || validSession?.device_id;
   if (!organizationId) throw new Error(`SQM needs an organization-bound SIMY session. Run \`simy --host ${webOrigin}\` once, then retry.`);
+  if (argv[0] === "upload-proof") {
+    if (argv.includes("--offline") || argv.includes("--no-upload")) throw new Error("upload-proof requires online evidence storage.");
+    const proofPath = path.resolve(requiredOption(argv, "--proof"));
+    const evidencePath = path.resolve(option(argv, "--signed-evidence") || `${proofPath}.signed.json`);
+    const [proof, signedEvidence] = await Promise.all([readJson(proofPath), readJson(evidencePath)]);
+    verifySignedEvidence(proof, signedEvidence);
+    if (!accountId || !deviceId || proof.organization_id !== organizationId ||
+        signedEvidence.signer.account_id !== accountId || signedEvidence.signer.device_id !== deviceId) {
+      throw new Error("Saved SQM evidence does not belong to the current organization, account and device.");
+    }
+    const evidenceEndpoint = process.env.SIMY_SQM_EVIDENCE_URL || (validSession?.api_base_url ? new URL("sqm/run-evidence", validSession.api_base_url).href : undefined);
+    const upload = await uploadRetainedProof({ endpoint: evidenceEndpoint, token, proof, signedEvidence });
+    await saveUploadReceipt(`${proofPath}.upload.json`, upload);
+    if (upload.uploaded) {
+      try {
+        await createSqmHistoryStore().reconcileUpload({ webOrigin, accountId, organizationId }, { proof, signedEvidence, upload });
+      } catch { upload.history_warning = "local_history_update_failed"; }
+    }
+    if (argv.includes("--json")) console.log(JSON.stringify(upload, null, 2));
+    else console.log(upload.uploaded ? `Cloud evidence: stored · ${upload.evidence_id}` : `Cloud evidence: local only (${upload.reason}). Original proof retained.`);
+    return upload.uploaded ? 0 : 2;
+  }
   const endpoint = process.env.SIMY_SQM_BUNDLE_URL || (validSession?.api_base_url ? new URL("sqm/knowledge-bundle", validSession.api_base_url).href : undefined);
   const { providerSession, conversationProvider } = await sqmConversationIdentity({ session: option(argv, "--session"), provider: option(argv, "--provider") });
   let checkedDescriptions = [];
@@ -130,19 +153,21 @@ export async function runSqmCommand(argv, { cliVersion }) {
     if (!argv.includes("--no-upload")) {
       const evidenceEndpoint = process.env.SIMY_SQM_EVIDENCE_URL || (validSession?.api_base_url ? new URL("sqm/run-evidence", validSession.api_base_url).href : undefined);
       stage = "evidence_upload";
-      upload = await uploadRunEvidence({ endpoint: evidenceEndpoint, token, proof: result.proof, signedEvidence: result.signed_evidence });
+      upload = await uploadRetainedProof({ endpoint: evidenceEndpoint, token, proof: result.proof, signedEvidence: result.signed_evidence });
     }
+    await saveUploadReceipt(`${proofPath}.upload.json`, upload);
     result.upload = upload;
     result.artifacts = {
       min_check: minPath,
       proof: proofPath,
       signed_evidence: evidencePath,
       diagnostics: diagnosticsPath,
+      upload: `${proofPath}.upload.json`,
     };
     await save(result);
     if (argv.includes("--json")) console.log(JSON.stringify(result, null, 2));
-    else printFull(result);
-    return result.proof.outcome.status === "failed" ? 1 : 0;
+    else printFull(result, webOrigin);
+    return result.proof.outcome.status === "failed" ? 1 : !argv.includes("--no-upload") && !upload.uploaded ? 2 : 0;
   } catch (error) {
     await save({ status: "failed", diagnostic: { ...safeSqmDiagnostic(error, stage), ...sqmDiagnostic(error, stage) } });
     throw error;
@@ -153,6 +178,7 @@ function printHelp() {
   console.log(`Usage:
   simy sqm min-check [options]
   simy sqm check [options]
+  simy sqm upload-proof --proof <path> [--signed-evidence <path>] [--host <origin>] [--json]
   simy sqm submit-evidence --file <delivery.json> [--host <origin>] [--dry-run]
   simy sqm lint --bundle <path>
   simy sqm replay --bundle <path> --dataset <path>
@@ -162,6 +188,8 @@ Two-stage SQM flow:
   check      Full state/invariant/stress/strength/scenario verification. It
              verifies the pinned min-check identity, writes proof.json, and
              produces independent per-run signed evidence.
+  upload-proof  Explicitly resend the original signed proof without rerunning
+             checks. Saves the storage acknowledgement beside the proof.
   lint       Report broad matchers and missing evaluation gates before publish.
   replay     Evaluate a bundle against labelled historical repository snapshots.
 
@@ -271,14 +299,17 @@ function printUnavailable(result, argv) {
   return 2;
 }
 
-function printFull(result) {
+function printFull(result, webOrigin) {
   console.log(`SQM full check: ${result.proof.outcome.status} · ${result.findings.length} finding(s)`);
   console.log(formatSqmCoverage(result.proof));
   console.log(`Proof ${result.proof.proof_id} (${result.proof.integrity.digest})`);
   console.log(`Per-run signature: valid · ${result.signature_verification.key_id}`);
   console.log(`Artifacts: ${result.artifacts.proof} · ${result.artifacts.signed_evidence} · ${result.artifacts.diagnostics}`);
   if (result.upload.uploaded) console.log(`Cloud evidence: stored · ${result.upload.evidence_id || result.signed_evidence.evidence_id}`);
-  else console.log(`Cloud evidence: local only (${result.upload.reason})`);
+  else {
+    console.log(`Cloud evidence: local only (${result.upload.reason})`);
+    if (result.upload.reason !== "disabled") console.log(`Storage not confirmed. Recover with: simy sqm upload-proof --proof ${shellArgument(result.artifacts.proof)} --signed-evidence ${shellArgument(result.artifacts.signed_evidence)} --host ${shellArgument(webOrigin)}`);
+  }
 }
 
 function defaultMinResultPath() {
@@ -300,3 +331,23 @@ function option(argv, name) {
   if (!value || value.startsWith("--")) throw new Error(`${name} requires a value.`);
   return value;
 }
+
+async function uploadRetainedProof(options) {
+  try { return await uploadRunEvidence(options); }
+  catch (error) {
+    const diagnostic = safeSqmDiagnostic(error, "evidence_upload", "evidence_upload_failed");
+    return { uploaded: false, reason: diagnostic.code, diagnostic };
+  }
+}
+
+function shellArgument(value) {
+  return "'" + String(value).replaceAll("'", "'\\''") + "'";
+}
+
+async function saveUploadReceipt(target, upload) {
+  try { await atomicJson(target, upload); }
+  catch {
+    upload.receipt_warning = "local_receipt_write_failed";
+    console.warn(`Could not save the local storage receipt. Cloud storage is ${upload.uploaded ? "confirmed" : "unconfirmed"}.`);
+  }
+}

package/src/sqm/evidence-client.js

@@ -26,5 +26,15 @@ export async function uploadRunEvidence({ endpoint, token, proof, signedEvidence
       requestId: response.headers?.get?.("x-request-id") }, "evidence_upload", "evidence_upload_failed"));
     throw error;
   }
-  return { uploaded: true, ...payload };
+  const evidence = payload?.evidence;
+  if (payload?.signature_valid !== true || typeof evidence?.id !== "string" || !evidence.id ||
+      evidence.proof_id !== proof.proof_id || evidence.evidence_id !== signedEvidence.evidence_id ||
+      evidence.proof_digest !== proof.integrity.digest) {
+    const error = new Error("SQM storage response did not acknowledge the submitted evidence identity.");
+    error.code = "evidence_acknowledgement_invalid";
+    error.stage = "evidence_upload";
+    error.retryable = false;
+    throw error;
+  }
+  return { uploaded: true, evidence_id: evidence.evidence_id, record_id: evidence.id };
 }

package/src/sqm/failure-circuit.js

@@ -19,7 +19,7 @@ export function createSqmFailureCircuitStore({ root = defaultSessionRoot() } = {
     async read(scope) {
       try {
         const state = JSON.parse(await readFile(path(scope), "utf8"));
-        if (!validVersion(state.version) || !Number.isInteger(state.failures) || state.failures < 0 || state.failures > SQM_FAILURE_PAUSE_THRESHOLD || (state.inFlight !== undefined && typeof state.inFlight !== "boolean"))
+        if (!validVersion(state.version) || !Number.isInteger(state.failures) || state.failures < 0 || state.failures > SQM_FAILURE_PAUSE_THRESHOLD || (state.inFlight !== undefined && typeof state.inFlight !== "boolean") || (state.providerFailures !== undefined && (!Number.isInteger(state.providerFailures) || state.providerFailures < 0 || state.providerFailures > 2)) || (state.providerRetryAt !== undefined && (!Number.isFinite(state.providerRetryAt) || state.providerRetryAt < 0)))
           throw new Error("Invalid saved SQM failure state.");
         return state;
       } catch (error) { if (error.code === "ENOENT") return null; throw error; }
@@ -42,7 +42,7 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
   const states = new Map();
   const unreadableScopes = new Set();
   const save = async (scope, state) => {
-    try { await store?.write(scope, { version: state.version, failures: state.failures, inFlight: state.inFlight === true }); state.storageError = false; }
+    try { await store?.write(scope, { version: state.version, failures: state.failures, inFlight: state.inFlight === true, ...(state.providerFailures ? { providerFailures: state.providerFailures, providerRetryAt: state.providerRetryAt } : {}) }); state.storageError = false; }
     catch (error) { state.storageError = true; throw error; }
   };
   const load = async scope => {
@@ -52,18 +52,20 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
       let saved;
       try { saved = await store?.read(scope); unreadableScopes.delete(key); }
       catch (error) { unreadableScopes.add(key); throw error; }
-      state = saved ? { version: saved.version, failures: saved.failures, inFlight: saved.inFlight === true } : { version, failures: 0, inFlight: false };
+      state = saved ? { version: saved.version, failures: saved.failures, inFlight: saved.inFlight === true, providerFailures: saved.providerFailures || 0, providerRetryAt: saved.providerRetryAt || 0 } : { version, failures: 0, inFlight: false };
       states.set(key, state);
       // An unfinished review from this same version means the last host exited
       // without a result or graceful cancellation. Count it once on restart.
       if (state.inFlight && compareVersions(version, state.version) <= 0) {
         state.failures = Math.min(SQM_FAILURE_PAUSE_THRESHOLD, state.failures + 1);
+        if (state.providerFailures === 1) state.providerFailures = 2;
         state.inFlight = false;
         await save(scope, state);
       }
     }
     if (compareVersions(version, state.version) > 0) {
       state.version = version;
+      state.providerFailures = 0; state.providerRetryAt = 0;
       state.failures = 0;
       state.inFlight = false;
       await save(scope, state);
@@ -71,7 +73,22 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
     return state;
   };
   return {
-    async canClaim(scope) { return (await load(scope)).failures < SQM_FAILURE_PAUSE_THRESHOLD; },
+    async canClaim(scope, now = Date.now()) {
+      const state = await load(scope);
+      return state.failures < SQM_FAILURE_PAUSE_THRESHOLD && (state.providerFailures || 0) < 2 && now >= (state.providerRetryAt || 0);
+    },
+    async providerFailure(scope, retryAt, retryAttempt = 0) {
+      const state = await load(scope);
+      state.providerFailures = retryAttempt === 1 ? 2 : Math.min(2, (state.providerFailures || 0) + 1);
+      state.providerRetryAt = retryAt;
+      await save(scope, state);
+    },
+    async providerRetryUsed(scope) {
+      const state = await load(scope);
+      if (state.providerFailures) return;
+      state.providerFailures = 1; state.providerRetryAt = 0;
+      await save(scope, state);
+    },
     async begin(scope) {
       const state = await load(scope);
       state.inFlight = true;
@@ -85,20 +102,25 @@ export function createSqmFailureCircuit({ version = CLI_VERSION, store = null }
     async failure(scope) {
       const state = await load(scope);
       state.failures = Math.min(SQM_FAILURE_PAUSE_THRESHOLD, state.failures + 1);
+      if (state.inFlight && state.providerFailures === 1) state.providerFailures = 2;
       state.inFlight = false;
       await save(scope, state);
     },
     async success(scope) {
       const state = await load(scope);
       state.failures = 0;
+      state.providerFailures = 0; state.providerRetryAt = 0;
       state.inFlight = false;
       await save(scope, state);
     },
     snapshot(scope) {
       const state = states.get(scopeKey(scope));
+      const paused = (state?.failures || 0) >= SQM_FAILURE_PAUSE_THRESHOLD || (state?.providerFailures || 0) >= 2;
       return { consecutive_failures: state?.failures || 0,
-        paused_until_update: (state?.failures || 0) >= SQM_FAILURE_PAUSE_THRESHOLD,
-        paused_at_version: (state?.failures || 0) >= SQM_FAILURE_PAUSE_THRESHOLD ? state.version : null,
+        paused_until_update: paused,
+        provider_retry_exhausted: (state?.providerFailures || 0) >= 2,
+        provider_recovery_pending: state?.providerFailures === 1,
+        paused_at_version: paused ? state.version : null,
         storage_unavailable: state?.storageError === true || unreadableScopes.has(scopeKey(scope)) };
     },
   };

package/src/sqm/history-store.js

@@ -7,7 +7,8 @@ import { basename, join } from "node:path";
 import { formatSqmCoverage, sqmCoverageWarnings } from "./report.js";
 import { defaultSessionRoot } from "../session-store.js";
 
-const LIMIT = 100;
+// Limit the desktop projection, never the durable history.
+const DISPLAY_LIMIT = 100;
 const hash = value => createHash("sha256").update(value).digest("hex");
 
 export function sqmHistoryScope(webOrigin, session) {
@@ -28,13 +29,14 @@ export function createSqmHistoryStore({ root = defaultSessionRoot(), now = () =>
     if (cached && now() - cached.at < 3000) return cached.items;
     let names;
     try { names = await readdir(dir); } catch (error) { if (error.code === "ENOENT") return []; throw error; }
-    const items = (await Promise.all(names.filter(name => /^[a-f0-9]{64}\.json$/.test(name)).map(async name => {
-      try {
-        const text = await readFile(join(dir, name), "utf8");
-        if (text.length > 16_384) return null;
-        return metadata(JSON.parse(text));
-      } catch { return null; }
-    }))).filter(Boolean).sort((a, b) => b.startedAt.localeCompare(a.startedAt));
+    const items = [];
+    const files = names.filter(name => /^[a-f0-9]{64}\.json$/.test(name));
+    // Retained history can outgrow the process file-descriptor limit.
+    for (let offset = 0; offset < files.length; offset += 16) {
+      items.push(...(await Promise.all(files.slice(offset, offset + 16)
+        .map(name => readItem(dir, name)))).filter(Boolean));
+    }
+    items.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
     cache.set(dir, { at: now(), items });
     return items;
   }
@@ -52,31 +54,59 @@ export function createSqmHistoryStore({ root = defaultSessionRoot(), now = () =>
           task_id: item.id, provider_diagnostic: item.providerDiagnostic,
           http_status: item.errorStatus ?? item.httpStatus, at: item.finishedAt || item.startedAt });
       } finally { await rm(temp, { force: true }); cache.delete(dir); }
-      const items = await list(scope);
-      for (const kind of ["analysis", "check"]) {
-        await Promise.all(items.filter(row => row.kind === kind).slice(LIMIT).map(row => rm(join(dir, `${hash(row.key)}.json`), { force: true })));
-      }
-      cache.delete(dir);
       return true;
     },
+    async reconcileUpload(scope, { proof, signedEvidence, upload }) {
+      if (!upload?.uploaded) return 0;
+      let updated = 0;
+      for (const row of await list(scope)) {
+        if (row.kind !== "check" || row.proofId !== proof.proof_id ||
+            row.proofDigest !== proof.integrity.digest || row.evidenceId !== signedEvidence.evidence_id) continue;
+        const previousFailure = row.errorStage === "evidence_upload" ? row.error : null;
+        const next = { ...row, result: (row.result || "").replace(/Evidence upload: [^\n]*/u,
+          `Evidence upload: uploaded · ${upload.evidence_id} · record ${upload.record_id}`) };
+        if (previousFailure) {
+          next.result += `\nPrevious storage failure (recovered): ${previousFailure}`;
+          for (const key of ["error", "errorCode", "errorStage", "errorStatus", "requestId"]) delete next[key];
+        }
+        await this.record(scope, next);
+        updated++;
+      }
+      return updated;
+    },
     async items(scope) {
       // A saved running row is not evidence that this process owns its lease.
       return (await list(scope)).map(item => ({ ...item, state: item.state === "running" ? "interrupted" : item.state }));
     },
     async detail(scope, key) {
-      const item = (await this.items(scope)).find(item => item.key === key);
-      if (!item) return null;
+      const dir = directory(scope);
+      if (!dir || typeof key !== "string") return null;
+      const saved = await readItem(dir, `${hash(key)}.json`);
+      if (!saved || saved.key !== key) return null;
+      const item = { ...saved, state: saved.state === "running" ? "interrupted" : saved.state };
       return { key, instruction: item.instruction || `SQM local review · ${item.provider || "local"}`,
         result: item.result || "", error: item.state === "interrupted" ? "This client stopped before the local run finished. Check the web workspace for its current retry status." : item.error || "", links: item.webUrl ? [{ label: "Analysis details", url: item.webUrl }] : [] };
     },
   };
 }
 
+async function readItem(dir, name) {
+  try {
+    const text = await readFile(join(dir, name), "utf8");
+    if (text.length > 16_384) return null;
+    return metadata(JSON.parse(text));
+  } catch (error) {
+    // One damaged or unreadable archived entry must not hide healthy history.
+    if (["ENOENT", "EISDIR", "EACCES", "EPERM"].includes(error.code) || error instanceof SyntaxError) return null;
+    throw error;
+  }
+}
+
 function metadata(entry) {
   if (!entry || !["analysis", "check"].includes(entry.kind) || typeof entry.key !== "string" || !entry.key.startsWith(entry.kind === "analysis" ? "sqm:" : "sqm-check:")) return null;
   if (!["running", "completed", "failed", "unavailable", "interrupted", "verification_pending"].includes(entry.state) || !Number.isFinite(Date.parse(entry.startedAt))) return null;
   const item = {};
-  for (const name of ["key", "id", "kind", "label", "state", "provider", "phase", "repository", "startedAt", "finishedAt", "instruction", "result", "error", "errorCode", "errorStage", "requestId", "webUrl", "providerSession", "conversationProvider", "outcome"]) {
+  for (const name of ["key", "id", "kind", "label", "state", "provider", "phase", "repository", "startedAt", "finishedAt", "instruction", "result", "error", "errorCode", "errorStage", "requestId", "webUrl", "providerSession", "conversationProvider", "outcome", "proofId", "evidenceId", "proofDigest"]) {
     if (typeof entry[name] === "string") item[name] = entry[name].slice(0, ["instruction", "result", "error"].includes(name) ? 4096 : 512);
   }
   if (Number.isInteger(entry.errorStatus) && entry.errorStatus >= 400 && entry.errorStatus <= 599) item.errorStatus = entry.errorStatus;
@@ -92,7 +122,7 @@ function metadata(entry) {
 export function projectSqmHistory(items, live = { activeCount: 0, items: [] }) {
   const analyses = new Map(items.filter(item => item.kind === "analysis").map(item => [item.key, item]));
   for (const item of live.items || []) analyses.set(item.key, { ...analyses.get(item.key), ...item });
-  const rows = items => items.sort((a, b) => (b.startedAt || "").localeCompare(a.startedAt || "")).slice(0, LIMIT)
+  const rows = items => items.sort((a, b) => (b.startedAt || "").localeCompare(a.startedAt || "")).slice(0, DISPLAY_LIMIT)
     .map(({ instruction, result, error, ...item }) => item);
   return { sqmTasks: { activeCount: live.activeCount || 0, items: rows([...analyses.values()]) },
     sqmChecks: { activeCount: 0, items: rows(items.filter(item => item.kind === "check")) } };
@@ -117,12 +147,13 @@ export function createSqmCheckRecorder({ root, scope, phase, cwd, baseBranch, pr
           minCheck.matched_modules.reduce((total, module) => total + (module.scenario_ids?.length || 0), 0),
         noSystemKnowledge: minCheck.matched_modules.length === 0 && minCheck.coverage_reasons?.includes("no_system_knowledge") === true,
         outcomeStatus: minCheck.status }) : [];
-    const summary = proof ? [`Full check: ${proof.outcome.status}`, formatSqmCoverage(proof), `Evidence upload: ${result.upload?.uploaded ? "uploaded" : "kept locally"}`].join("\n")
+    const summary = proof ? [`Full check: ${proof.outcome.status}`, formatSqmCoverage(proof), `Proof: ${proof.proof_id}`, `Evidence upload: ${result.upload?.uploaded ? `uploaded · ${result.upload.evidence_id} · record ${result.upload.record_id}` : "kept locally"}`].join("\n")
       : state === "unavailable" ? "SQM could not obtain a verified result. See the recorded diagnostics below."
       : state === "failed" && !identity ? "The command did not finish. See the recorded diagnostics below."
       : `Min-check: ${result.min_check?.status || result.status} · ${(result.preliminary_findings || result.min_check?.preliminary_findings)?.length || 0} preliminary finding(s) · ${(result.matched_modules || result.min_check?.matched_modules)?.length || 0} matched module(s).\nThis is a preflight; run the full check for signed verification.`;
     try {
       await history.record(scope, { key, id: key, kind: "check", phase, state, startedAt, providerSession, conversationProvider, outcome,
+        proofId: proof?.proof_id, proofDigest: proof?.integrity?.digest, evidenceId: result.signed_evidence?.evidence_id,
         checked: result.checked_descriptions || presentation.checked || [],
         noApplicableChecks: (proof?.outcome.reasons || minCheck.coverage_reasons || []).includes('no_applicable_rules'),
         findings: (proof?.findings || result.preliminary_findings || minCheck.preliminary_findings || []).length, finishedAt: new Date().toISOString(), repository,

package/src/sqm/provider-diagnostics.js

@@ -10,6 +10,7 @@ export function safeProviderDiagnostic(value) {
   if (Number.isInteger(value.exitCode) && value.exitCode >= 0 && value.exitCode <= 255) result.exitCode = value.exitCode;
   if (SIGNALS.has(value.signal)) result.signal = value.signal;
   if (Number.isInteger(value.outputBytes) && value.outputBytes >= 0 && value.outputBytes <= 2147483647) result.outputBytes = value.outputBytes;
+  if (["before_turn", "turn_started", "turn_output"].includes(value.phase)) result.phase = value.phase;
   if (value.retryAttempt === 0 || value.retryAttempt === 1) result.retryAttempt = value.retryAttempt;
   if (typeof value.model === "string" && /^(?:gpt-6-(?:astra|sol)|gpt-6\.1-sol|claude-(?:sonnet|opus|haiku)-[0-9]+(?:-[0-9]+){0,3})$/.test(value.model) && value.model.length <= 64) result.model = value.model;
   return result;

package/src/sqm/review-worker.js

@@ -15,10 +15,17 @@ import {
   webApiUrl,
 } from "../web-api.js";
 
+const RECOVERABLE_PROVIDER_FAILURES = new Set([
+  "provider_model_unsupported", "provider_process_failed", "provider_network_unavailable",
+  "provider_rate_limited", "provider_execution_error", "codex_local_review_failed", "claude_local_review_failed",
+]);
+const isRecoverableProviderFailure = failure => ["analysis", "adjudication"].includes(failure.stage) && RECOVERABLE_PROVIDER_FAILURES.has(failure.code);
 const DEFAULT_LEASE_SECONDS = 300;
 // Claim can retrieve and freeze organization-scoped Slack evidence (45s budget).
 const DEFAULT_REQUEST_TIMEOUT_MS = 75_000;
-const DEFAULT_EXECUTION_TIMEOUT_MS = 240_000;
+const DEFAULT_EXECUTION_TIMEOUT_MS = 360_000;
+const CODEX_ATTEMPT_TIMEOUT_MS = 180_000;
+const MIN_STALL_RETRY_MS = 60_000;
 // Keep analysis independent of interactive config. Availability varies by
 // account and CLI; only an explicit model rejection may try the next model.
 // Background SQM and conversation triage use Sol models to limit usage.
@@ -124,13 +131,16 @@ export function createSqmLocalReviewWorker({
   const pauses = new Map();
   const contextKey = (provider, scope) => JSON.stringify([scope?.webOrigin, scope?.accountId, scope?.organizationId, provider]);
   const currentPause = () => pauses.get(contextKey(getProvider(), getHistoryScope()));
-  const pauseForFailure = (error, provider, scope, recoveryKey) => {
+  const pauseForFailure = async (error, provider, scope, recoveryKey) => {
     const failure = failureDetails.get(error) || reviewFailure(error, "claim", provider);
-    if (!["provider_sign_in_required", "provider_model_unsupported", "codex_local_unavailable", "claude_local_unavailable", "desktop_access_required"].includes(failure.code) && ![401, 403].includes(failure.status)) return;
+    const recoverable = isRecoverableProviderFailure(failure) || failure.providerDiagnostic?.retryAttempt === 1
+      || (["analysis", "adjudication"].includes(failure.stage) && failureCircuit.snapshot(scope).provider_recovery_pending);
+    if (!recoverable && !["provider_sign_in_required", "provider_model_unsupported", "codex_local_unavailable", "claude_local_unavailable", "desktop_access_required"].includes(failure.code) && ![401, 403].includes(failure.status)) return;
     const key = contextKey(provider, scope);
     const failures = Math.min((pauses.get(key)?.failures || 0) + 1, 4);
     const delay = Math.min(30 * 60_000, Math.max(1000, providerFailureCooldownMs) * 2 ** (failures - 1));
-    pauses.set(key, { code: failure.code, stage: failure.stage, status: failure.status, failures, recoveryKey,
+    if (recoverable) await failureCircuit.providerFailure(scope, now() + delay, failure.providerDiagnostic?.retryAttempt);
+    pauses.set(key, { recoverable, code: failure.code, stage: failure.stage, status: failure.status, failures, recoveryKey,
       retryAt: now() + delay, message: `${formatFailure(failure)}\nNew SQM claims are paused. After fixing this cause, refresh the local AI connection or wait for the next recovery attempt.` });
     if (pauses.size > 32) pauses.delete(pauses.keys().next().value);
   };
@@ -167,7 +177,10 @@ export function createSqmLocalReviewWorker({
     for (let attempt = 0; ; attempt += 1) {
       setStage(false);
       if (reviewOptions.signal?.aborted) throw providerFailure("analysis_cancelled");
-      const review = await runReview(nextRequest, provider, reviewOptions);
+      const review = await runReview(nextRequest, provider, {
+        ...reviewOptions, allowAutomaticRetry: !failureCircuit.snapshot(reviewOptions.historyScope).provider_recovery_pending,
+      });
+      if (review.retryAttempt === 1) await failureCircuit.providerRetryUsed(reviewOptions.historyScope);
       if (reviewOptions.signal?.aborted) throw providerFailure("analysis_cancelled");
       setStage(true);
       try {
@@ -175,7 +188,11 @@ export function createSqmLocalReviewWorker({
         if (reviewOptions.signal?.aborted) throw providerFailure("analysis_cancelled");
         return { review, response };
       } catch (error) {
-        if (attempt >= 1 || error.status !== 422 || error.code !== code || !error.validationFeedback) throw error;
+        if (attempt >= 1 || failureCircuit.snapshot(reviewOptions.historyScope).provider_recovery_pending ||
+          error.status !== 422 || error.code !== code || !error.validationFeedback) throw error;
+        // Native recovery, cooldown recovery and model-output correction share
+        // one durable retry budget across analysis and adjudication.
+        await failureCircuit.providerRetryUsed(reviewOptions.historyScope);
         nextRequest = { ...request, prompt: `${request.prompt}\n\nThe previous result failed validation. Return a corrected complete JSON object using only the supplied evidence. Fix these constraints:\n${error.validationFeedback}` };
       }
     }
@@ -199,7 +216,7 @@ export function createSqmLocalReviewWorker({
     let stage = "analysis";
     const signal = reviewController?.signal;
     const request = body => api.request(body, { signal });
-    const reviewOptions = { signal, providerReportingSupported: claim.capabilities?.reported_executor_provider === true,
+    const reviewOptions = { signal, historyScope, beforeAutomaticRetry: () => failureCircuit.providerRetryUsed(historyScope), allowAutomaticRetry: !failureCircuit.snapshot(historyScope).provider_recovery_pending, providerReportingSupported: claim.capabilities?.reported_executor_provider === true,
       beforeLeaseDispatch: () => request({ action: "heartbeat", ...leaseBody(claim, provider) }) };
     const providerReport = (review, selectedProvider = provider) => claim.capabilities?.reported_executor_provider === true
       ? { reported_executor_provider: review.provider || selectedProvider } : {};
@@ -256,9 +273,11 @@ export function createSqmLocalReviewWorker({
       const circuit = failureCircuit.snapshot(getHistoryScope());
       return { active_job_id: activeJobId, polling: Boolean(pollPromise), closed, suspended,
         paused: pausedEnvironment !== null || circuit.paused_until_update || circuit.storage_unavailable,
-        pause_reason: circuit.paused_until_update ? "consecutive_failures_until_update" : circuit.storage_unavailable ? "failure_state_unavailable" : pauseReason,
+        pause_reason: circuit.provider_retry_exhausted ? "provider_retry_failed_until_update" : circuit.paused_until_update ? "consecutive_failures_until_update" : circuit.storage_unavailable ? "failure_state_unavailable" : pauseReason,
         ...circuit, issue: issueContext === contextKey(getProvider(), getHistoryScope()) ? issue : null,
-        ...(pause ? { provider_pause: { code: pause.code, status: pause.status, retry_at: new Date(pause.retryAt).toISOString(), message: pause.message } } : {}) };
+        ...(pause ? { provider_pause: { code: pause.code, status: pause.status,
+          retry_at: circuit.provider_retry_exhausted ? null : new Date(pause.retryAt).toISOString(),
+          message: circuit.provider_retry_exhausted ? "The provider retry failed. New SQM reviews are paused until SIMY is updated." : pause.message } } : {}) };
     },
     desktopTasks() { return { activeCount: activeJobId ? 1 : 0, items: visibleHistory().map(item => ({ ...item, canStop: item.id === activeJobId })) }; },
     desktopTaskDetail(key) {
@@ -270,8 +289,15 @@ export function createSqmLocalReviewWorker({
     poll() {
       if (closed || suspended || pollPromise) return pollPromise || Promise.resolve(null);
       const environmentKey = getEnvironmentKey();
-      if (pausedEnvironment !== null && pausedEnvironment === environmentKey) return Promise.resolve(null);
-      if (pausedEnvironment !== null) { pausedEnvironment = null; pauseReason = null; pauses.delete(contextKey(getProvider(), getHistoryScope())); issue = null; }
+      if (pausedEnvironment !== null && pausedEnvironment === environmentKey) {
+        const pause = currentPause();
+        // Known provider failures already have a bounded cooldown. Do not let
+        // the environment gate suppress their recovery probe indefinitely.
+        if (!pause?.recoverable || (pause.recoveryKey === getRecoveryKey(getProvider()) && now() < pause.retryAt)) return Promise.resolve(null);
+        pausedEnvironment = null; pauseReason = null;
+      } else if (pausedEnvironment !== null) {
+        pausedEnvironment = null; pauseReason = null; pauses.delete(contextKey(getProvider(), getHistoryScope())); issue = null;
+      }
       const provider = getProvider();
       if (provider == null) return Promise.resolve(null);
       assertProvider(provider);
@@ -286,7 +312,7 @@ export function createSqmLocalReviewWorker({
       const workspaceUrl = getWorkspaceUrl();
       reviewController = new AbortController();
       pollPromise = (async () => {
-        if (!await failureCircuit.canClaim(historyScope)) return null;
+        if (!await failureCircuit.canClaim(historyScope, now())) return null;
         if (reviewController.signal.aborted) throw providerFailure("analysis_cancelled");
         const claim = await api.request({
           action: "claim",
@@ -312,7 +338,7 @@ export function createSqmLocalReviewWorker({
             return null;
           }
           try { await failureCircuit.failure(historyScope); } catch (storageError) { onError(storageError); }
-          pauseForFailure(error, provider, historyScope, recoveryKey);
+          try { await pauseForFailure(error, provider, historyScope, recoveryKey); } catch (storageError) { onError(storageError); }
           if (error.jobId && /^[a-f0-9-]{36}$/i.test(error.jobId) && !history.some(item => item.id === error.jobId)) {
             const scope = historyScope;
             const failure = reviewFailure(error, "claim", provider);
@@ -322,7 +348,7 @@ export function createSqmLocalReviewWorker({
             history.unshift(item); historyScopes.set(item, JSON.stringify(scope)); history.splice(50);
             await persist(scope, item);
           }
-          if (["provider_model_unsupported", "codex_local_unavailable", "claude_local_unavailable", "provider_sign_in_required", "provider_status_unknown"].includes(error.code)) {
+          if (isRecoverableProviderFailure(failureDetails.get(error) || reviewFailure(error, "claim", provider)) || ["provider_model_unsupported", "codex_local_unavailable", "claude_local_unavailable", "provider_sign_in_required", "provider_status_unknown"].includes(error.code)) {
             pausedEnvironment = environmentKey; pauseReason = error.code;
           }
           issueContext = key;
@@ -409,6 +435,10 @@ async function runSingleProviderReview(
     terminationGraceMs = 1_000,
     beforeDispatch = async () => {},
     deadlineAt,
+    codexAttemptTimeoutMs = CODEX_ATTEMPT_TIMEOUT_MS,
+    minStallRetryMs = MIN_STALL_RETRY_MS,
+    allowAutomaticRetry = true,
+    beforeAutomaticRetry = async () => {},
   } = {},
 ) {
   assertProvider(provider);
@@ -435,7 +465,7 @@ async function runSingleProviderReview(
     const models = provider === "codex" ? SQM_CODEX_MODELS : [null];
     let processResult;
     let selectedModel;
-    let modelIndex = 0, retryAttempt = 0, claudeTurnLimit = 1;
+    let modelIndex = 0, retryAttempt = 0, claudeTurnLimit = 1, stallRetried = false;
     while (modelIndex < models.length) {
       const model = models[modelIndex];
       await beforeDispatch();
@@ -472,7 +502,7 @@ async function runSingleProviderReview(
             "-",
           ],
           prompt,
-          { spawnImpl, executionTimeoutMs: remaining, label, cwd: workDir, environment, provider, signal, readOnlyAnalysis, model, retryAttempt,
+          { spawnImpl, executionTimeoutMs: remaining, stallTimeoutMs: provider === "codex" && readOnlyAnalysis && allowAutomaticRetry && !stallRetried && remaining > codexAttemptTimeoutMs ? codexAttemptTimeoutMs : null, label, cwd: workDir, environment, provider, signal, readOnlyAnalysis, model, retryAttempt,
             terminationGraceMs: Number.isInteger(terminationGraceMs) ? Math.max(1, Math.min(terminationGraceMs, 5_000)) : 1_000 },
         );
         selectedModel = model;
@@ -481,12 +511,19 @@ async function runSingleProviderReview(
         // Repair only an explicit native structured-generation limit. Keep
         // initial calls unchanged, the original deadline and dispatch guard,
         // and never retry authentication/schema/process/security failures.
-        if (provider === "claude" && readOnlyAnalysis && retryAttempt === 0
+        if (allowAutomaticRetry && provider === "claude" && readOnlyAnalysis && retryAttempt === 0
           && ["provider_turn_limit_reached", "provider_structured_output_exhausted"].includes(error.code)) {
+          await beforeAutomaticRetry();
           retryAttempt = 1;
           if (error.code === "provider_turn_limit_reached") claudeTurnLimit = 3;
           continue;
         }
+        if (allowAutomaticRetry && provider === "codex" && readOnlyAnalysis && !stallRetried && error.code === "analysis_timeout"
+          && ["before_turn", "turn_started"].includes(error.providerDiagnostic?.phase) && deadline - Date.now() >= minStallRetryMs) {
+          await beforeAutomaticRetry();
+          stallRetried = true; retryAttempt = 1;
+          continue;
+        }
         if (error.code !== "provider_model_unsupported" || model === models.at(-1)) throw error;
         modelIndex += 1;
       }
@@ -513,6 +550,7 @@ async function runSingleProviderReview(
     return {
       output,
       model: provider === "codex" ? selectedModel : `${provider}-local/${inspection.installed_version || "unknown"}`,
+      ...(processResult.diagnostic.retryAttempt === 1 ? { retryAttempt: 1 } : {}),
     };
   } catch (error) {
     if (error instanceof SqmLocalReviewError || error?.code === 'desktop_access_required') throw error;
@@ -537,7 +575,7 @@ function codexAnalysisArguments(system) {
     '-c', `developer_instructions=${JSON.stringify(system)}`];
 }
 
-function spawnReview(command, args, prompt, { spawnImpl, executionTimeoutMs, cwd, environment, provider, signal, readOnlyAnalysis, terminationGraceMs, model, retryAttempt }) {
+function spawnReview(command, args, prompt, { spawnImpl, executionTimeoutMs, stallTimeoutMs, cwd, environment, provider, signal, readOnlyAnalysis, terminationGraceMs, model, retryAttempt }) {
   return new Promise((resolve, reject) => {
     if (signal?.aborted) return reject(providerFailure("analysis_cancelled"));
     let child;
@@ -546,7 +584,7 @@ function spawnReview(command, args, prompt, { spawnImpl, executionTimeoutMs, cwd
         env: safeReviewEnvironment(environment, provider), cwd, windowsHide: true, detached: process.platform !== "win32", stdio: ["pipe", "pipe", "pipe"],
       });
     } catch { reject(providerFailure("provider_spawn_failed", { model })); return; }
-    let stdout = "", stderrPending = "", outputBytes = 0;
+    let stdout = "", stderrPending = "", outputBytes = 0, phase = "before_turn";
     let settled = false, terminalError = null, killTimer = null;
     let diagnostic = safeProviderDiagnostic({ model, retryAttempt });
     const halt = error => {
@@ -563,12 +601,18 @@ function spawnReview(command, args, prompt, { spawnImpl, executionTimeoutMs, cwd
     const cancelled = () => halt(providerFailure("analysis_cancelled"));
     const timer = setTimeout(() => halt(providerFailure("analysis_timeout")), executionTimeoutMs);
     timer.unref?.();
+    // The short watchdog applies only while the turn has produced no output.
+    // Once output arrives, the original total deadline still bounds the work.
+    const stallTimer = stallTimeoutMs == null ? null : setTimeout(() => {
+      if (phase !== "turn_output") halt(providerFailure("analysis_timeout"));
+    }, stallTimeoutMs);
+    stallTimer?.unref?.();
     const finish = (error = null) => {
       if (settled) return;
       settled = true;
-      clearTimeout(timer); clearTimeout(killTimer);
+      clearTimeout(timer); clearTimeout(stallTimer); clearTimeout(killTimer);
       signal?.removeEventListener("abort", cancelled);
-      diagnostic = safeProviderDiagnostic({ ...diagnostic, outputBytes });
+      diagnostic = safeProviderDiagnostic({ ...diagnostic, outputBytes, ...(provider === "codex" ? { phase } : {}) });
       if (error) {
         error.providerDiagnostic = safeProviderDiagnostic({ ...error.providerDiagnostic, ...diagnostic });
         reject(error);
@@ -600,9 +644,14 @@ function spawnReview(command, args, prompt, { spawnImpl, executionTimeoutMs, cwd
       if (outputBytes > 2 * 1024 * 1024) { halt(providerFailure("provider_output_too_large")); return; }
       stdout += chunk;
       if (readOnlyAnalysis && provider === "codex") {
+        let observed = "before_turn";
         for (const line of stdout.split("\n").filter(Boolean)) {
           let event;
           try { event = JSON.parse(line); } catch { continue; }
+          if (event.type === "turn.started") observed = observed === "before_turn" ? "turn_started" : observed;
+          else if (observed !== "before_turn" && ["turn.completed", "item.completed"].includes(event.type)) observed = "turn_output";
+          phase = observed;
+          if (phase === "turn_output") clearTimeout(stallTimer);
           if (["error", "turn.failed"].includes(event.type)) {
             const error = nativeFailure(event.message ?? event.error?.message);
             if (error) { halt(error); return; }