@volter/supercode-orchestrator 0.5.119 → 0.5.120

package/board/ci.mjs

@@ -145,9 +145,22 @@ async function repositoryWorkspace(board, id) {
     // Resolution without create names a solo worktree even before it exists on
     // this machine. Its portable repository and branch still identify the
     // pushed PR. Only an actual directory can supply local source evidence.
-    return { ...resolved, here: here && checkoutPresent(resolved.path) };
+    if (here && !checkoutPresent(resolved.path)) return { ...resolved, here: await recreatedWorktree(workspace, resolved) };
+    return { ...resolved, here };
   }
-  return workspace.workspace_path ? { path: workspace.workspace_path, here } : null;
+  return workspace.workspace_path ? { path: workspace.workspace_path, here: here && checkoutPresent(workspace.workspace_path) } : null;
+}
+
+/** A card's declared solo worktree that is missing here (deleted, or never made on this machine) while its branch is on
+ *  the remote is made again from the remote branch (the daemon's resolve with create), so its review reads the pushed
+ *  head instead of waiting on a checkout nobody will restore. Answers whether the checkout is now present. */
+async function recreatedWorktree(workspace, resolved) {
+  if (workspace.workspace_kind !== 'worktree' || !resolved?.branch || !resolved.root || !checkoutPresent(resolved.root)) return false;
+  let pushed = '';
+  try { pushed = await run('git', ['ls-remote', '--heads', 'origin', `refs/heads/${resolved.branch}`], resolved.root); } catch { return false; }
+  if (!pushed) return false;
+  // a worktree that cannot be made here leaves the card reviewed at its pushed head, as a card on another machine is
+  try { const made = await resolveWorkspace(workspace, true); return made.path === resolved.path && checkoutPresent(made.path); } catch { return false; }
 }
 
 const pullRequestLists = new Map();

package/board/dispatch.mjs

@@ -39,7 +39,7 @@ import { homedir, hostname, tmpdir, totalmem } from 'node:os';
 import { basename, dirname, join } from 'node:path';
 import { createHash, randomUUID } from 'node:crypto';
 import { fileURLToPath } from 'node:url';
-import { view, boardCopyOf, boardPath, claimer, claimHolds, copyRefusal, event, json, listBoards, logsDir, newClaim, now, openBoard, setActor, tx } from './store.mjs';
+import { view, boardCopyOf, boardPath, claimer, claimHolds, copyRefusal, event, json, listBoards, logsDir, newClaim, now, openBoard, setActor, storageFault, tx } from './store.mjs';
 import { doors, fire, loadWorkflow, resumeOf, runSpec, runWorkingSeconds, requestSessionClose, retireSupersededRuns, refreshCardInputs, refreshRelationAnswers, transitionsFor, scopeOf, statusOf, projectStatus, storedStatus, templateValue, workflowFor } from './workflow.mjs';
 import { checkWorkspace, assertWorkspaceOwner, notePresence, resolveWorkspace } from './workspace.mjs';
 import { capacity, harnessDefaults, holdings } from './capacity.mjs';
@@ -188,7 +188,22 @@ async function subscriberAlive(fleet, address) {
   const session = sessionAt(address);
   if (!session) return false;
   await readBefore(fleet, session);
-  return sessionAlive(fleet, session) ?? null;
+  const alive = sessionAlive(fleet, session);
+  if (alive == null && subscriberGone(fleet, address)) return false;
+  return alive ?? null;
+}
+/**
+ * A subscriber that is gone: no transcript, no process and no pane. Its machine's discovery answered whole (no source
+ * failed, nothing truncated) without the session, the fleet lists no pane for it, and the reading came from a daemon
+ * whose index names a Claude subagent by its own file (#1423, marked `subagent_rows: 'own'` on its discovery): an older
+ * daemon's index dropped a parent session that ran a subagent, so its absence there is never read as gone.
+ */
+function subscriberGone(fleet, address) {
+  if (fleet.alive.has(address) || fleet.native?.has(address)) return false;
+  const session = sessionAt(address);
+  const seen = session && discoveries.get(nativeKey(session.machine, session.harness, session.session_id));
+  if (!seen?.complete) return false;
+  return !seen.sessions.some((r) => r.locator?.session_id === session.session_id && r.locator?.harness === session.harness);
 }
 const paneLists = new Map();
 /** A session the fleet reading did not publish: alive by what was read of it before (readBefore), else unknown (null). */
@@ -242,8 +257,8 @@ function nativeSessions(machine, harness, sessionId) {
 async function readNative(machine, harness, sessionId) {
   const key = nativeKey(machine, harness, sessionId), seen = discoveries.get(key);
   if (seen) return seen.sessions;
-  const sessions = await discoverNative(machine, harness, sessionId);
-  discoveries.set(key, { at: Date.now(), sessions });
+  const { sessions, complete } = await discoverNative(machine, harness, sessionId);
+  discoveries.set(key, { at: Date.now(), sessions, complete });
   return sessions;
 }
 // A native discovery is bounded as the fleet reading is: one that does not answer in time is unknown, read next round.
@@ -255,7 +270,9 @@ async function discoverNative(machine, harness, sessionId) {
   const result = await doorOn(machine, 'harness.v1.sessions.discover', { harnesses: [harness], query: sessionId, limit: 2000 }, { timeout: NATIVE_DISCOVERY_MS });
   const page = result.code === 0 ? lastJson(result.stdout) : null;
   if (!Array.isArray(page?.sessions)) throw unavailable(`native liveness unavailable on ${machine}: ${(result.stderr || result.stdout).trim().slice(-300)}`);
-  return page.sessions;
+  // whole: every source read, nothing truncated, from an index that keeps every session's own row (subscriberGone)
+  const complete = page.subagent_rows === 'own' && !(Array.isArray(page.failed) && page.failed.length) && !page.receipt?.truncated && page.next_cursor == null;
+  return { sessions: page.sessions, complete };
 }
 function nativeReading(rows, session) {
   const row = rows.find(r => r.locator?.session_id === session.session_id && r.locator?.harness === session.harness);
@@ -792,6 +809,9 @@ async function startCards(board, fleet, lanes, settings, report, entry, effects,
     } catch (error) {
       // a launch whose run ended meanwhile is no failure of the card's current run (its successor): said, nothing fired
       if (error.launchPending || error.launchSuperseded) { report.queued.push([task.id, error.message]); continue; }
+      // the board's store could not be written (a full or failing volume): no fault of the card, which keeps its failure
+      // count and is launched again next round (its launch key reconciles a session the refused write did not record)
+      if (storageFault(error)) { report.queued.push([task.id, `the board's store refused a write (${error.message}); the launch is tried again next round`]); continue; }
       const payload = { error: String(error.message).slice(-500), origin: originOf(error), ...(error.resumeFailed ? { resume_failed: true, ...error.resumeFailed } : {}) };
       let r;
       try { r = launchFailed(board, entry, task.id, error, payload); } catch { r = { effects: [] }; /* no transition for it */ }
@@ -991,7 +1011,7 @@ async function turnAtClose(session, pane = null, { launchWindow = null } = {}) {
     return `pane ${session.pane} names no process to read`;
   } else {
     if (!session.session_id) return 'its session is not known yet';
-    try { activity = nativeReading(await discoverNative(machine, session.harness ?? 'claude-code', session.session_id), session).row?.activity ?? null; }
+    try { activity = nativeReading((await discoverNative(machine, session.harness ?? 'claude-code', session.session_id)).sessions, session).row?.activity ?? null; }
     catch (error) { return `its session could not be read just now (${String(error.message).slice(0, 160)})`; }
     if (!activity) return `its session ${addressOf(session)} is not in its machine's discovery`;
   }
@@ -2426,8 +2446,11 @@ function sessionRecord(db) {
  */
 async function endWorkerSubscriptions(board, fleet, report) {
   // each worker subscriber's liveness is read before the write
-  const aliveNow = new Map();
-  if (fleet.ok) for (const { chat_id: address } of board.db.prepare("SELECT DISTINCT chat_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) aliveNow.set(address, await subscriberAlive(fleet, address));
+  const aliveNow = new Map(), goneNow = new Map();
+  if (fleet.ok) for (const { chat_id: address } of board.db.prepare("SELECT DISTINCT chat_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) {
+    aliveNow.set(address, await subscriberAlive(fleet, address));
+    if (aliveNow.get(address) === false && subscriberGone(fleet, address)) goneNow.set(address, now());
+  }
   tx(board, (db) => {
     const { held, latest } = sessionRecord(db);
     const remove = db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?");
@@ -2436,7 +2459,9 @@ async function endWorkerSubscriptions(board, fleet, report) {
       const address = row.chat_id, record = latest.get(address);
       // a session alive, or whose liveness could not be read (unknown, never taken as ended), keeps its subscription
       if (!record || record.session.released || held.has(address) || (fleet.ok && aliveNow.get(address) !== false)) continue;
-      const evidence = record.session.close_confirmed_at ? { by: 'board_close', run: record.run, closed_at: record.session.close_confirmed_at } : null;
+      // the board closed it, or it is gone (no transcript, process or pane, read whole: subscriberGone)
+      const evidence = record.session.close_confirmed_at ? { by: 'board_close', run: record.run, closed_at: record.session.close_confirmed_at }
+        : goneNow.has(address) ? { by: 'gone', run: record.run, read_at: goneNow.get(address), machine: sessionAt(address)?.machine ?? null } : null;
       if (!evidence || !remove.run(row.task_id, address, row.thread_id).changes) continue;
       if (!ended.has(address)) ended.set(address, { record, evidence, cards: [] });
       ended.get(address).cards.push(row.task_id);

package/board/store.mjs

@@ -329,6 +329,14 @@ export function publishMarked(root) {
 export const contention = (error) => [5, 6].includes(Number(error?.errcode) & 255)
   || ['SQLITE_BUSY', 'SQLITE_LOCKED'].includes(error?.code)
   || /^database (?:table |schema )?is locked$/i.test(error?.message ?? '');
+// The store's volume refusing a write (SQLite IOERR or FULL and their extended codes; ENOSPC or EIO from a file write):
+// a fault of the machine's storage at that minute, never of the card or the operation that met it. A transaction it
+// rolled back is tried again a bounded number of times (STORAGE_TRIES); one still refused fails cleanly, and its caller
+// leaves the work for the next round instead of counting it against a card.
+export const storageFault = (error) => [10, 13].includes(Number(error?.errcode) & 255)
+  || ['SQLITE_IOERR', 'SQLITE_FULL', 'ENOSPC', 'EIO'].includes(error?.code)
+  || /^(?:disk I\/O error|database or disk is full)$/i.test(error?.message ?? '');
+export const STORAGE_TRIES = 3;
 const pause = new Int32Array(new SharedArrayBuffer(4));
 /** Run `fn` until it answers without meeting another process's write (a short, randomized wait between tries). */
 export function retryContention(fn) {
@@ -349,7 +357,7 @@ export const view = (board, fn) => inTx.has(board.db) ? fn(board.db) : retryCont
 const inTx = new WeakSet();
 function sqliteTx(db, fn) {
   if (inTx.has(db)) return fn(db);
-  for (;;) {
+  for (let faults = 0; ;) {
     try {
       db.exec('BEGIN');
       inTx.add(db);
@@ -359,6 +367,7 @@ function sqliteTx(db, fn) {
       return out;
     } catch(error) {
       try {db.exec('ROLLBACK');} catch {}
+      if (storageFault(error) && ++faults < STORAGE_TRIES) { Atomics.wait(pause, 0, 0, 50 * faults); continue; }
       if (!contention(error))throw error;
       Atomics.wait(pause, 0, 0, 1 + Math.random() * 20);
     } finally { inTx.delete(db); }

package/board/ztrack.mjs

@@ -14,7 +14,7 @@ import { createHash, randomBytes } from 'node:crypto';
 import { existsSync, linkSync, mkdirSync, readFileSync, realpathSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
 import { basename, dirname, join, resolve } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
-import { actorOf, contention, event, json, now, principalOf } from './store.mjs';
+import { actorOf, contention, event, json, now, principalOf, storageFault, STORAGE_TRIES } from './store.mjs';
 import { GitDocument } from './git-document.mjs';
 import { noteStep, recordSlow } from '@volter/supercode-harness-sdk/slow-log';
 import * as slowLog from '@volter/supercode-harness-sdk/slow-log';
@@ -418,7 +418,7 @@ class ZtrackBoard {
     // A pull request merged on the remote (a page edited in Git Notes) is in the document before this write reads it.
     if (this.git) this.git.paths = this.docs.map((doc) => doc.path);
     try { timedStep('board git pull', () => this.git?.pull()); } catch (error) { current().err(`board: ${error.message}\n`); }
-    let txStarted = 0;
+    let txStarted = 0, faults = 0;
     for (let attempt = 0; ; attempt++) {
       let outcome;
       try {
@@ -458,6 +458,8 @@ class ZtrackBoard {
         try { this.conn.exec('ROLLBACK'); } catch { /* not begun */ }
         this.active = false; this.seen = null; this.loadErrors();
         if (error === RETRY || sqliteContention(error)) { backoff(attempt); noteStep('board lock wait', performance.now() - txStarted); continue; }
+        // the store's volume refused the write (store.mjs storageFault): rolled back, so tried again a bounded number of times
+        if (storageFault(error) && ++faults < STORAGE_TRIES) { sleep(50 * faults); continue; }
         throw error;
       } finally { this.active = false; }
       this.seed = null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.119",
+  "version": "0.5.120",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {