@volter/supercode-orchestrator 0.5.187 → 0.5.188

package/README.md

@@ -42,6 +42,17 @@ Rules the codec keeps (enforced in Rust, observed here):
   since import (UNI-22); it copies the store byte for byte when they did not.
 - Files the IR does not model are never touched by `save` and are carried by path on a migration.
 
+## Local native session recovery
+
+`supercode sessions recover --once --retry-after 300` runs a bounded opt-in
+recovery pass over this user's local Claude Code and Codex panes. The shared
+terminal classifier identifies the current transient failure; the machine's
+`panes.retry` door owns the guarded native input and retained receipt. This
+package owns the retry interval, durable input IDs and reporting. A separate
+SQLite owner lease prevents overlapping passes without retaining a lock after a
+process dies. It does not change a board's dispatch policy or restart an
+intentionally suspended session. See the [session guide](../../docs/guides/sessions.md#retry-transient-native-session-failures).
+
 ## OpenClaw, and the two migration verbs (ORC-12)
 
 ```js

package/package.json

@@ -1,9 +1,13 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.187",
+  "version": "0.5.188",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {
+    "./session-recovery": {
+      "types": "./session-recovery.d.ts",
+      "import": "./session-recovery.mjs"
+    },
     ".": {
       "types": "./index.d.ts",
       "import": "./index.mjs"
@@ -24,6 +28,7 @@
     "check-types": "node scripts/generate-types.mjs --check"
   },
   "dependencies": {
+    "@volter/supercode-terminal": "^0.3.8",
     "@volter/supercode-harness-sdk": "^0.3.101",
     "@volter/teams": "0.2.84",
     "@volter/ztrack": "^2.4.23",

package/session-recovery.d.ts

@@ -0,0 +1,2 @@
+/** One explicitly requested, bounded local Claude Code/Codex recovery pass. */
+export function runSessionRecovery(args?: string[]): Promise<number>;

package/session-recovery.mjs

@@ -0,0 +1,157 @@
+// The optional local-session retry policy. The machine owns observation and native input receipts;
+// this orchestrator command owns when to ask for another turn. One invocation is bounded to eight seconds.
+import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { DatabaseSync } from 'node:sqlite';
+import { openLocalMachine, StreamRpc } from '@volter/teams/client';
+import { supercodeHome } from '@volter/teams/home';
+import { classifyRecoverableFailure } from '@volter/supercode-terminal';
+
+const HELP = `supercode sessions recover [--once] [--home DIR] [--retry-after SECONDS]
+
+Retry current transient Claude Code/Codex failures in this user's local panes.
+Each pass reads every live panel and submits continue only for the current failed
+turn with an empty composer. Suspended panes, user input and completed turns are
+preserved. Delivery is confirmed by the machine's retained native receipt.
+
+The default retry interval is 300 seconds per pane. Schedule this short command
+with your service manager for recurring recovery. Policy and receipts are retained
+under <supercode-home>/session-recovery; the machine daemon does not choose retries.
+`;
+
+function options(args) {
+  const out = { home: supercodeHome(), retryAfter: 300 };
+  for (let index = 0; index < args.length; index++) {
+    const arg = args[index];
+    if (arg === '--once') continue;
+    if (arg === '--help' || arg === '-h') return { help: true };
+    if (arg === '--home' || arg === '--retry-after') {
+      const value = args[++index];
+      if (!value || value.startsWith('--')) throw new Error(`${arg} needs a value`);
+      if (arg === '--home') out.home = resolve(value);
+      else {
+        out.retryAfter = Number(value);
+        if (!Number.isInteger(out.retryAfter) || out.retryAfter < 30 || out.retryAfter > 86400) throw new Error('--retry-after must be 30..86400 seconds');
+      }
+    } else throw new Error(`unknown recovery option: ${arg}`);
+  }
+  return out;
+}
+
+function readState(path) {
+  try { return JSON.parse(readFileSync(path, 'utf8')); }
+  catch (error) { if (error.code === 'ENOENT') return { version: 1, attempts: {} }; throw error; }
+}
+
+/** Run the CLI's one short recovery pass. Stable message IDs survive command and daemon restarts. */
+export async function runSessionRecovery(args = []) {
+  let opts;
+  try { opts = options(args); } catch (error) { process.stderr.write(`${error.message}\n${HELP}`); return 2; }
+  if (opts.help) { process.stdout.write(HELP); return 0; }
+  const started = Date.now(), deadline = started + 8000;
+  const summary = { at: new Date(started).toISOString(), scanned: 0, continued: [], confirmed: [], pending: [], skipped: [], errors: [] };
+  const directory = join(opts.home, 'session-recovery');
+  mkdirSync(directory, { recursive: true });
+  const statePath = join(directory, 'state.json'), logPath = join(directory, 'activity.jsonl');
+  let channel, owner, timer;
+  const log = value => {
+    if (existsSync(logPath) && statSync(logPath).size > 2_000_000) {
+      // Keep one previous log without making rotation fail on Windows when it already exists.
+      writeFileSync(`${logPath}.previous`, readFileSync(logPath)); writeFileSync(logPath, '');
+    }
+    appendFileSync(logPath, `${JSON.stringify(value)}\n`);
+  };
+  try {
+    // A separate SQLite owner lock lasts only for this pass. The state is committed before each side effect;
+    // a process death releases ownership without losing the idempotency key of input already submitted.
+    owner = new DatabaseSync(join(directory, 'owner.sqlite'));
+    try { owner.exec('BEGIN IMMEDIATE'); }
+    catch (error) {
+      if (/locked|busy/iu.test(error.message)) { process.stdout.write(`${JSON.stringify({ ...summary, skipped: ['another recovery pass is running'] })}\n`); return 0; }
+      throw error;
+    }
+    timer = setTimeout(() => {
+      log({ ...summary, durationMs: Date.now() - started, error: 'recovery pass exceeded its eight-second deadline' });
+      channel?.close(); owner?.close(); process.exit(1);
+    }, 8500);
+    const machine = JSON.parse(readFileSync(join(opts.home, 'teams', 'machine.json'), 'utf8'));
+    channel = await openLocalMachine(machine.socket, { timeoutMs: 1000 });
+    const rpc = new StreamRpc(await channel.open('node'));
+    const call = (method, params, maximum = 1800) => {
+      const timeoutMs = Math.min(maximum, deadline - Date.now());
+      if (timeoutMs <= 0) throw new Error(`recovery deadline reached before ${method}`);
+      return rpc.call(method, params, { timeoutMs });
+    };
+    const panes = await call('panes.processes', {});
+    const state = readState(statePath);
+    if (!state.attempts || typeof state.attempts !== 'object' || Array.isArray(state.attempts)) throw new Error('invalid recovery state: attempts must be an object');
+    const save = () => {
+      const temporary = `${statePath}.${process.pid}.tmp`;
+      writeFileSync(temporary, `${JSON.stringify(state, null, 2)}\n`); renameSync(temporary, statePath);
+    };
+    const current = new Map(panes.map(pane => [pane.pane, pane.pid]));
+    for (const [pane, attempt] of Object.entries(state.attempts)) if (!current.has(pane) || current.get(pane) !== attempt.pid) delete state.attempts[pane];
+    save();
+    const claim = { pid: process.pid, ppid: process.ppid, session: process.env.CODEX_THREAD_ID || process.env.CLAUDE_CODE_SESSION_ID || null };
+    const retry = (pane, ids, statusOnly = false) => call('panes.retry', {
+      pane: pane.pane, expected_pid: pane.pid, message_ids: ids, status_only: statusOnly, foreign: true, claim,
+    }, statusOnly ? 1500 : 3500);
+
+    await Promise.all(panes.map(async pane => {
+      try {
+        const panel = await call('panes.capture', { pane: pane.pane, lines: 100 });
+        summary.scanned++;
+        if (!['codex', 'claude-code'].includes(pane.kind)) return;
+        const previous = state.attempts[pane.pane];
+        if (previous && ['queued', 'sending'].includes(previous.status)) {
+          const receipt = await retry(pane, previous.messageIds, true);
+          if (receipt.delivered) {
+            previous.status = 'confirmed'; previous.lastStatus = receipt.receipt; save();
+            summary.confirmed.push({ pane: pane.pane, message_ids: previous.messageIds });
+            log({ at: new Date().toISOString(), event: 'continue-confirmed', pane: pane.pane, receipt });
+          } else if (receipt.refused || receipt.skipped) {
+            previous.status = 'refused'; previous.lastStatus = receipt.reason ?? receipt.skipped; save();
+          } else if (receipt.known || Date.now() - previous.at < opts.retryAfter * 1000) {
+            summary.pending.push({ pane: pane.pane, reason: receipt.reason ?? 'awaiting native acknowledgement', ageSeconds: Math.round((Date.now() - previous.at) / 1000) });
+            return;
+          } else {
+            // A timed-out request which was never accepted may be retried, only after another current-error read.
+            previous.status = 'not-submitted'; save();
+          }
+        }
+        const failure = classifyRecoverableFailure(panel.text);
+        if (!failure) { summary.skipped.push({ pane: pane.pane, reason: 'no current recoverable failure' }); return; }
+        if (Date.now() - (state.attempts[pane.pane]?.at ?? 0) < opts.retryAfter * 1000) {
+          summary.skipped.push({ pane: pane.pane, reason: 'retry interval has not elapsed' }); return;
+        }
+        // An unanswered submission keeps its original ID even when the daemon cannot find it yet.
+        // If it later completes, the same envelope is acknowledged rather than pasted a second time.
+        const messageIds = previous?.status === 'not-submitted' ? previous.messageIds : [`recover-${randomUUID()}`];
+        const attempt = { at: Date.now(), pid: pane.pid, failure, messageIds, status: 'sending' };
+        state.attempts[pane.pane] = attempt; save();
+        log({ at: new Date().toISOString(), event: 'continue-attempt', pane: pane.pane, ...attempt });
+        let result = await retry(pane, attempt.messageIds);
+        const until = Math.min(Date.now() + 1200, deadline - 300);
+        while (result.queued && Date.now() < until) {
+          await new Promise(done => setTimeout(done, 200));
+          result = await retry(pane, attempt.messageIds, true);
+        }
+        attempt.status = result.delivered ? 'confirmed' : result.refused ? 'refused' : result.skipped ? 'skipped' : result.queued || result.known ? 'queued' : 'refused';
+        attempt.lastStatus = result.reason ?? result.skipped ?? result.receipt; save();
+        log({ at: new Date().toISOString(), event: 'continue-result', pane: pane.pane, result });
+        if (result.delivered) summary.continued.push({ pane: pane.pane, failure: failure.kind });
+        else if (result.skipped) summary.skipped.push({ pane: pane.pane, reason: result.skipped });
+        else if (!result.refused && (result.queued || result.known)) summary.pending.push({ pane: pane.pane, reason: result.reason ?? 'awaiting native acknowledgement', ageSeconds: Math.round((Date.now() - attempt.at) / 1000) });
+        else summary.errors.push({ pane: pane.pane, message: result.reason ?? JSON.stringify(result) });
+      } catch (error) { summary.errors.push({ pane: pane.pane, message: error.message }); }
+    }));
+  } catch (error) { summary.errors.push({ message: error.message }); }
+  finally {
+    clearTimeout(timer); channel?.close(); owner?.close();
+  }
+  summary.durationMs = Date.now() - started;
+  log(summary); writeFileSync(join(directory, 'last-run.json'), `${JSON.stringify(summary, null, 2)}\n`);
+  process.stdout.write(`${JSON.stringify(summary)}\n`);
+  return summary.errors.length ? 1 : 0;
+}