@volter/supercode-orchestrator 0.4.103 → 0.4.105

package/board/dispatch.mjs

@@ -176,6 +176,16 @@ export function lanesOf(profiles = {}) {
 
 function supervise(board, fleet, settings, report, entry, effects) {
   const rows = board.db.prepare(entry.workflow.run ? "SELECT id FROM tasks WHERE current_run_id IS NOT NULL" : "SELECT id FROM tasks WHERE status = 'running'").all();
+  // The catalog can lag a pane. Read the actual machines once, before any board mutation.
+  const observed = new Map();
+  for (const { id } of rows) {
+    const s = currentRun(board.db, id)?.metadata?.supercode;
+    if (!s?.pane || observed.has(s.machine)) continue;
+    const r = sc(['teams', 'panes', 'ls', '--machine', s.machine ?? localMachine()], { timeout: 60_000 });
+    const list = r.code === 0 ? json(r.stdout.trim(), null) : null;
+    observed.set(s.machine, Array.isArray(list) ? new Set(list.map((p) => p.pane)) : null);
+    if (!Array.isArray(list)) report.failed.push([id, `pane liveness unavailable; retaining run: ${(r.stderr || r.stdout).trim().slice(-200)}`]);
+  }
   for (const { id } of rows) {
     tx(board, (db) => {
       const task = getTask(db, id);
@@ -197,7 +207,9 @@ function supervise(board, fleet, settings, report, entry, effects) {
       };
       const limits = limitsOf(db, board, entry, task, session);
       // Hermes's orphan reconciliation: a running card whose claim bookkeeping is broken, with nothing live behind it
-      const live = session?.surface === 'headless' ? pidAlive(session.pid) : session?.session_id ? (fleet.ok ? sessionAlive(fleet, session) : true) : false;
+      const paneReading = session?.pane ? observed.get(session.machine) : undefined;
+      const paneLive = paneReading instanceof Set ? paneReading.has(session.pane) : null;
+      const live = session?.surface === 'headless' ? pidAlive(session.pid) : session?.pane ? paneLive !== false : session?.session_id ? (fleet.ok ? sessionAlive(fleet, session) : true) : false;
       // (whether it is reconciled is the workflow's: its `orphaned` transitions and their guard)
       if ((!task.claim_lock || !task.claim_expires) && !live) {
         observe('orphaned', { reason: 'orphaned_running', claim_lock: task.claim_lock, claim_expires: task.claim_expires, error: 'orphaned running card (broken claim bookkeeping)' }, 'reclaimed');
@@ -226,20 +238,21 @@ function supervise(board, fleet, settings, report, entry, effects) {
         return;
       }
       if (session?.session_id) {
-        if (!fleet.ok) return; // no reading, no verdict
-        const alive = sessionAlive(fleet, session);
-        const machineUp = fleet.online.has(session.machine);
+        if (session.pane && paneLive === null) return; // no machine reading, no loss verdict
+        if (!session.pane && !fleet.ok) return;
+        const alive = session.pane ? paneLive : sessionAlive(fleet, session);
+        const machineUp = paneReading instanceof Set || fleet.online.has(session.machine);
         const young = t - Number(session.launched_at ?? 0) < Number(limits.grace ?? settings.grace);
         const missing = !alive && machineUp && !young;
         const next = { ...session };
         // a machine that is off the fleet: how long, for the workflow's machine_offline (the claim is kept meanwhile)
-        if (machineUp) delete next.offline_since; else next.offline_since ??= t;
+        if (machineUp) next.offline_since = null; else next.offline_since ??= t;
         if (!machineUp) {
           setRunSession(db, runRow.id, next);
           observe('machine_offline', { machine: session.machine, seconds: t - next.offline_since }, null);
           if (getTask(db, id)?.current_run_id !== runRow.id) return; // the workflow moved it
         }
-        if (alive) { next.seen_at ??= t; delete next.missing_since; }
+        if (alive) { next.seen_at ??= t; next.missing_since = null; }
         if (missing) next.missing_since ??= t;
         if (missing && t - next.missing_since >= Number(limits.grace ?? settings.grace)) {
           // lost: gone from an online machine for the whole grace. A resume that never came up is not resumable.
@@ -500,7 +513,7 @@ export function closeRequested(board, report, { taskId = null, caller = null } =
         if (current.close_error !== error) event(db, row.task_id, 'session_close_failed', { session_id: session.session_id, pane: session.pane, error, retry: true }, row.id);
       } else {
         const { close_error: _error, ...rest } = current;
-        setRunSession(db, row.id, { ...rest, closed_at: at, close_confirmed_at: at });
+        setRunSession(db, row.id, { ...rest, close_error: null, closed_at: at, close_confirmed_at: at });
         const address = addressOf(session);
         const subscribed = active.some((r) => addressOf(json(r.metadata, {})?.supercode) === address && r.id !== row.id);
         if (!subscribed) db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ?").run(row.task_id, address);
@@ -651,6 +664,12 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
   // the slot's last session: kept when live, resumed when lost, never another slot's (a reviewer's is its own)
   // a close its session's own command asked for is carried out first, so its pane is gone before the slot's next
   // session starts (never two processes on one session)
+  // Every old pane in the slot must be settled, including a failed launch with no session id.
+  closeRequested(board, { undelivered: [] }, { taskId: task.id });
+  const pending = view(board, (db) => db.prepare('SELECT id, ended_at, metadata FROM task_runs WHERE task_id = ? AND id != ?').all(task.id, runId))
+    .map((r) => ({ ...r, session: json(r.metadata, {})?.supercode }))
+    .find((r) => r.session?.pane && (r.session.slot ?? 'worker') === slot && (r.ended_at || r.session.close_requested) && !r.session.close_confirmed_at);
+  if (pending) throw new Error(`slot ${slot} still owns pane ${pending.session.pane} from run ${pending.id}; closure must be confirmed before another session starts: ${pending.session.close_error ?? 'close pending'}`);
   let prev = view(board, (db) => lastSession(db, task.id, slot));
   if (prev?.close_requested && !prev.closed_at) {
     const report = { undelivered: [] };
@@ -671,7 +690,10 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
   };
   let session = null;
   let replaced = null;
-  if (prev?.forgotten) replaced = { session_id: prev.session_id, why: 'the workflow let it go' };
+  if (prev?.forgotten) {
+    if (prev.pane && panePresent(prev.machine, prev.pane)) throw new Error(`forgotten session ${prev.session_id} still has live pane ${prev.pane}; close it before replacement`);
+    replaced = { session_id: prev.session_id, why: 'the workflow let it go' };
+  }
   else if (prev && (prev.harness ?? 'claude-code') !== harness) {
     // the card was reassigned to a lane of another harness: that harness starts it fresh (the card's text and thread
     // are its context), and the last session, if still open, is closed so two harnesses never work one card
@@ -680,7 +702,9 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
   } else if (prev && prev.machine === machine) {
     // a session the board closed, or one whose close its own command asked for (the next tick's sweep closes that
     // run's pane), is not kept, whatever a fleet reading taken before the close still lists
-    if (!prev.closed_at && !prev.close_requested && sessionAlive(fleet, prev)) {
+    const paneLive = prev.pane ? panePresent(prev.machine, prev.pane) : false;
+    if (paneLive && (prev.closed_at || prev.close_requested || prev.forgotten)) throw new Error(`session ${prev.session_id} still has live pane ${prev.pane}; cannot resume or replace it`);
+    if (!prev.closed_at && !prev.close_requested && (prev.pane ? paneLive : sessionAlive(fleet, prev))) {
       // the session is kept; the heartbeat's count belongs to the run that ended, so the new run starts it again
       const { missing_since: _m, goal_turns: _t, goal_nudged_at: _n, close_requested: _q, closed_at: _c, close_confirmed_at: _cc, close_error: _ce, close_attempts: _ca, close_attempted_at: _ct, ...kept } = prev;
       session = { ...kept, mode: 'adopt', launched_at: now(), seen_at: now() };
@@ -759,12 +783,18 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
   return session;
 }
 
-/** Whether a pane just launched on `machine` is still there after the harness had a moment to start. */
+/** A machine reading, not a missing catalog entry, establishes whether a pane is alive. */
+function panePresent(machine, pane) {
+  const r = sc(['teams', 'panes', 'ls', '--machine', machine ?? localMachine()], { timeout: 60_000 });
+  const panes = r.code === 0 ? json(r.stdout.trim(), null) : null;
+  if (!Array.isArray(panes)) throw new Error(`cannot establish pane ${pane} liveness on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
+  return panes.some((p) => p.pane === pane);
+}
+
+/** Whether a pane just launched is still there after the harness had a moment to start. */
 function paneSurvives(machine, pane) {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 6000);
-  const r = sc(['teams', 'panes', 'ls', '--machine', machine], { timeout: 60_000 });
-  const panes = r.code === 0 ? json(r.stdout.trim(), null) : null;
-  return Array.isArray(panes) ? panes.some((p) => p.pane === pane) : true; // no reading: the grace decides
+  return panePresent(machine, pane);
 }
 
 /** Hermes's own worker argv and task environment, run on this machine; watched by its pid. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.4.103",
+  "version": "0.4.105",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {