@volter/supercode-orchestrator 0.5.130 → 0.5.132

package/board/cli.mjs

@@ -64,6 +64,7 @@ async function boardPublishing(root) {
 }
 import { dispatchingOf, knowLocalMachine, requestAdoption, alertManagers, boardAddress, boardLabel, boardSettings, boardCaller, callerSession, commandEnv, reconcileCallerLaunch, closeRequested, lanesOf, localMachine, observeFleet, paused, plan, startCard, stopDispatching, supercodeBin, tick, deliver, startDelivery } from './dispatch.mjs';
 import { watchBoard } from './wake.mjs';
+import { onPanesChanged } from './machine-fleet.mjs';
 import { streamBoardPublications } from './event-stream.mjs';
 import { completionOwnerOf, followHome, isLatestReviewer, loadWorkflow, readHome, requestSessionClose, scopeOf, statusOf, useWorkflow, workflowEntry, workflowFor } from './workflow.mjs';
 import { evaluate } from './expr.mjs';
@@ -1702,6 +1703,9 @@ async function serve(root, args = {}) {
     if (watched !== undefined && board) heldDocumentsWatched(board, id ?? key, watched);
     else if (uncertain && !board) heldDocumentsChanged();
   } });
+  // a machine's panes changing in its record (a session came up under its pane, a pane or its session ended) is what a
+  // round binds a launch, verifies an adoption and reads a session's end from: the next round is due then
+  onPanesChanged(() => events?.wake('machine-record'));
   // the owner holds its boards only now, its watch installed (watch before load): opened once, changed by events
   holdBoards(root);
   // a pause of the collector holds the loop like any synchronous step: each is a step, so a block line names it
@@ -1723,7 +1727,7 @@ async function serve(root, args = {}) {
   };
   await registerOwner();
   onLocalReopen(registerOwner);
-  log('dispatch wakes on board writes, document edits and board mail; the interval sweep reads each machine\'s own door');
+  log('dispatch wakes on board writes, document edits, board mail and a change of a machine\'s panes in its record; the interval sweep reads each machine\'s own door');
   const stuck = {};
   const warned = {};
   let pausedSaid = false;

package/board/dispatch.mjs

@@ -7,8 +7,9 @@
 // - `pane`: an interactive session of the lane's harness (Claude Code or Codex) on the card's machine (`supercode open
 //   --new … --on <machine>`), its session id recorded on the run: one the dispatcher chose (Claude Code's
 //   `--session-id`) or the one the harness named, read from the fleet by its pane (Codex). It closes when its run ends. It is live
-//   while supercode's fleet lists it; one missing for the whole grace on an online machine is lost, and the next start
-//   resumes it in its own harness. Nothing asks the session about itself.
+//   while supercode's fleet lists it; one whose pane its machine saw end is lost at once, one missing with no end seen
+//   for the whole grace on an online machine is lost then, and the next start resumes it in its own harness. Nothing
+//   asks the session about itself.
 // - `headless`: the profile's worker for one query on this machine (`bin/hermes-worker.mjs`, Hermes's worker
 //   argv and task environment), watched by its pid as Hermes watches its own.
 //
@@ -572,16 +573,25 @@ async function supervise(board, fleet, settings, report, entry, effects) {
         // read one grace after it went, not two (an adopted session that ended inside the grace of its adoption waited
         // out both)
         const young = !session.seen_at && t - Number(session.launched_at ?? 0) < Number(limits.grace ?? settings.grace);
-        const missing = !alive && machineUp && !young;
+        // its pane's end as its machine saw it (the record's `ended`: tmux's own exit log, or a pane kept dead): the
+        // session is known to be gone, and nothing is waited for. The grace is for a session that is only missing from
+        // a reading, with no end seen: it may be read again. A launch the board keyed is told how it ended by its
+        // machine's receipt (above: a clean end is a session someone closed, never a loss to recover from), so its
+        // receipt is what is waited for, as it was, on a round whose reading of it failed
+        const receipted = Boolean(session.launch_key) && Boolean(fleet.capabilities?.get(normalMachine(session.machine))?.launches);
+        const ended = !alive && machineUp && session.pane && !receipted ? fleet.ended?.get(normalMachine(session.machine))?.get(session.pane) ?? null : null;
+        const missing = !alive && machineUp && (!young || Boolean(ended));
         const next = { ...session };
         if (alive) { next.seen_at ??= t; next.missing_since = null; }
         if (missing) next.missing_since ??= t;
-        if (missing && t - next.missing_since >= Number(limits.grace ?? settings.grace)) {
-          // lost: gone from an online machine for the whole grace. A resume that never came up is not resumable.
+        if (missing && (ended || t - next.missing_since >= Number(limits.grace ?? settings.grace))) {
+          // lost: its pane's end was seen, or it is gone from an online machine for the whole grace. A resume that never
+          // came up is not resumable.
           if (next.mode === 'resume' && !next.seen_at) next.resumable = false;
           setRunSession(db, runRow.id, next);
           observe('session_lost', { session_id: next.session_id, machine: next.machine, pane: next.pane, mode: next.mode, seen: Boolean(next.seen_at),
-            never_came_up: next.mode === 'fresh' && !next.seen_at, error: `session ${next.session_id} is gone from ${next.machine}` }, 'lost');
+            never_came_up: next.mode === 'fresh' && !next.seen_at, ...(ended ? { pane_ended: ended } : {}),
+            error: `session ${next.session_id} is gone from ${next.machine}${ended ? `: its pane ${next.pane} ended${ended.how ? ` (${ended.how})` : ''}` : ''}` }, 'lost');
           return;
         }
         if (JSON.stringify(next) !== JSON.stringify(session)) setRunSession(db, runRow.id, next);
@@ -632,7 +642,7 @@ async function stopSession(board, session) {
     return;
   }
   if (session.pane) {
-    const r = await runtimeDoor(normalMachine(session.machine??localMachine())===localMachine()?null:session.machine,'harness.v1.panes.close',{pane:session.pane},{env:commandEnv(),timeout:60_000,as:boardAddress(board.label)});
+    const r = await runtimeDoor(normalMachine(session.machine??localMachine())===localMachine()?null:session.machine,'harness.v1.panes.close',{pane:session.pane,...(session.launch_key?{launch:session.launch_key}:{})},{env:commandEnv(),timeout:60_000,as:boardAddress(board.label)});
     // a door that timed out or could not be reached says nothing about the pane: the close waits for the next round
     const transient = doorTransient(r);
     if (r.code !== 0 || json(r.stdout.trim(), null)?.closed !== session.pane) throw new Error(`${transient ? 'close deferred: ' : ''}closing pane ${session.pane}: ${(r.stderr || r.stdout || `exit ${r.code}`).trim().slice(-500)}`);
@@ -1062,6 +1072,8 @@ async function turnAtClose(session, pane = null, { launchWindow = null } = {}) {
   return `its session ${who} is in a turn (${activity.turn ?? 'unknown'}); it closes once the turn ends`;
 }
 
+/** The closes this process is making now, by run: each one's outcome, for a pass that meets it under way. */
+const closesUnderway = new Map();
 export async function closeRequested(board, report, { taskId = null, caller = null, ttl = null, fleet = null } = {}) {
   const rows = view(board, (db) => db.prepare(`SELECT id, task_id, ended_at, metadata FROM task_runs
     WHERE metadata IS NOT NULL ${taskId ? 'AND task_id = ?' : ''}`).all(...(taskId ? [taskId] : [])));
@@ -1129,17 +1141,43 @@ export async function closeRequested(board, report, { taskId = null, caller = nu
     }
     // A final refusal stops further close attempts. Only an authorized observation of this exact pane may establish
     // absence: panes.list can be filtered or carry missing context keys. Unknown/invisible keeps the original failure.
-    const finalFailure = closeRefused(session);
+    // A refusal's reason can end (the machine's daemon was replaced by one that reads the launch's starter): the same
+    // reading says whether this board's close of the pane would be admitted now (`yours`), and a close it would admit
+    // is tried again, unless an unended run works in the pane. A machine that does not say stays refused.
+    let finalFailure = closeRefused(session);
     if (finalFailure) {
       if (!session.pane) continue;
-      const r = await doorOn(session.machine ?? localMachine(), 'harness.v1.panes.presence', { pane: session.pane }, { timeout: 10_000, as: boardAddress(board.label) });
+      const r = await doorOn(session.machine ?? localMachine(), 'harness.v1.panes.presence', { pane: session.pane, ...(session.launch_key ? { launch: session.launch_key } : {}) }, { timeout: 10_000, as: boardAddress(board.label) });
       const observed = r.code === 0 ? json(r.stdout.trim(), null) : null;
-      if (observed?.pane !== session.pane || observed.present !== false) continue;
+      if (observed?.pane !== session.pane) continue;
+      if (observed.present === true && observed.yours === true) {
+        if (view(board, (db) => db.prepare(`SELECT metadata FROM task_runs r WHERE r.ended_at IS NULL OR EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id)`).all())
+          .some((r) => { const s = json(r.metadata, {})?.supercode; return s?.pane === session.pane && normalMachine(s.machine) === normalMachine(session.machine); })) continue;
+        finalFailure = false;
+      } else if (observed.present !== false) continue;
     }
     const active = view(board, (db) => db.prepare(`SELECT r.id, r.metadata FROM task_runs r WHERE r.ended_at IS NULL OR EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id)`).all());
     // Persist retries even for historical ends that did not have a close_session action. The close is claimed in the
     // same transaction, before the pane or process is touched: of two rounds (or a round and a verb) that read one close,
     // only the one whose claim committed closes it, and the other never records the pane it finds gone as a failure.
+    // One close of a run at a time in this process too. The claim below keeps another process off it, never this one
+    // (claimHolds: its own claim does not hold it), and the owner makes a close in more than one pass side by side: a
+    // round's, and each `workflow close` it answers. A second pass that began while the first was stopping the session
+    // found the pane still listed and no session under it, and answered "close deferred: pane … runs …, whose session
+    // could not be read" for a session that was being closed (the system test's close loop, 1 take in 7). It waits
+    // for the close under way and reports what that one did.
+    const flight = `${board.root}\u0000${board.slug ?? ''}\u0000${row.id}`;
+    const underway = closesUnderway.get(flight);
+    if (underway) {
+      const made = await underway;
+      if (made.error) report.undelivered.push([row.task_id, addressOf(session), made.error, 'close']);
+      else if (made.closed) (report.closed ??= []).push({ task_id: row.task_id, ...session });
+      continue;
+    }
+    let settle;
+    closesUnderway.set(flight, new Promise((resolve) => { settle = resolve; }));
+    let error = null, closed = false;
+    try {
     const claimed = tx(board, (db) => {
       const current = json(db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(row.id)?.metadata, {})?.supercode;
       if (!current || current.close_confirmed_at || current.close_transferred_to || current.pane !== session.pane || current.session_id !== session.session_id || claimHolds(current.close_claim)) return false;
@@ -1150,7 +1188,6 @@ export async function closeRequested(board, report, { taskId = null, caller = nu
       return true;
     });
     if (!claimed) continue;
-    let error = null;
     try {
       const machine = session.machine ?? localMachine();
       if (session.surface === 'native' && !session.pane) {
@@ -1197,7 +1234,7 @@ export async function closeRequested(board, report, { taskId = null, caller = nu
         setRunSession(db, row.id, { ...current, close_attempted_at: at, close_attempts: Number(current.close_attempts ?? 0) + 1, close_error: error, ...(final ? { close_failed_at: at } : {}) });
         // Retain every attempt's latest fact, but avoid an unbounded thread of identical failures.
         if (current.close_error !== error) event(db, row.task_id, 'session_close_failed', { session_id: session.session_id, pane: session.pane, error, retry: !final }, row.id);
-        if (final) event(db, row.task_id, 'run_incident', { observation: 'close_failed', run: row.id, reason: `the board could not close ${addressOf(session)} and will not try again: ${error}`, session: session.session_id, machine: session.machine, pane: session.pane ?? null, evidence: { error } }, row.id);
+        if (final && current.close_error !== error) event(db, row.task_id, 'run_incident', { observation: 'close_failed', run: row.id, reason: `the board could not close ${addressOf(session)} and will not try again: ${error}`, session: session.session_id, machine: session.machine, pane: session.pane ?? null, evidence: { error } }, row.id);
       } else if (!current.close_confirmed_at) {
         // a close another closer already confirmed (a claim that expired under it) is said once, by that closer
         const { close_error: _error, ...rest } = current;
@@ -1216,7 +1253,8 @@ export async function closeRequested(board, report, { taskId = null, caller = nu
       }
     });
     if (error) report.undelivered.push([row.task_id, addressOf(session), error, 'close']);
-    else (report.closed ??= []).push({ task_id: row.task_id, ...session });
+    else { closed = true; (report.closed ??= []).push({ task_id: row.task_id, ...session }); }
+    } finally { closesUnderway.delete(flight); settle({ error, closed }); }
   }
 }
 

package/board/machine-fleet.mjs

@@ -11,6 +11,33 @@ import { noteStep } from '@volter/supercode-harness-sdk/slow-log';
 import { machineCall, machineRecordStream } from './doors.mjs';
 import { RecordView } from './record-view.mjs';
 const recordView = new RecordView(machineRecordStream);
+/**
+ * What a round decides from in a machine's record: its live panes, the session under each and whether it runs, and
+ * the panes it saw end. Not a session's turn, which changes all day and starts, binds and ends nothing.
+ */
+const paneFacts = (held) => {
+  const record = held?.record;
+  if (!record?.panes) return '';
+  const under = new Map((record.panes.activities ?? []).map(({ pid, activity }) => [pid, activity]));
+  return JSON.stringify([
+    (record.panes.panes ?? []).map((pane) => [pane.pane, Boolean(pane.dead), under.get(pane.pid)?.session_id ?? null, under.get(pane.pid)?.presence === 'running']).sort(),
+    (record.panes.ended ?? []).map((exit) => exit.pane).sort(),
+  ]);
+};
+const paneListeners = new Set();
+let factsHeld = new Map();
+recordView.onApplied = (records) => {
+  const next = new Map(Object.entries(records).map(([machine, held]) => [machine, paneFacts(held)]));
+  const changed = [...next].filter(([machine, facts]) => factsHeld.has(machine) && factsHeld.get(machine) !== facts).map(([machine]) => machine);
+  factsHeld = next;
+  if (changed.length) for (const listener of paneListeners) { try { listener(changed); } catch { /* a listener cannot stop the record's reading */ } }
+};
+/**
+ * Told when a machine's panes change in its record (a pane or the session under it came up, ended or was closed):
+ * the dispatcher's next round is due then, not at its interval's sweep. Returns its unsubscribe. The record's stream is
+ * opened by a round's first reading and held from then on.
+ */
+export function onPanesChanged(listener) { paneListeners.add(listener); return () => paneListeners.delete(listener); }
 const DEADLINE_MS = 10_000;
 const within = (promise, ms) => Promise.race([promise, new Promise((_, reject) => setTimeout(() => reject(Object.assign(new Error(`no answer within ${ms} ms`), { code: 'timeout' })), ms).unref?.())]);
 // One machine's reading, from its held record: its panes, what a board launch needs of it, and the session under each
@@ -27,7 +54,7 @@ function readOne(name, held) {
   if (notCurrent) return { machine: name, error: `its panes are not current: ${notCurrent}`, code: 'stale' };
   const panes = record.panes?.panes ?? [];
   const capabilities = record.capabilities ?? { guard: [], launches: false };
-  return { machine: name, panes, activities: record.panes?.activities ?? [], capabilities: { guard: capabilities.guard ?? [], launches: Boolean(capabilities.launches), launchesRefused: capabilities.launches ? null : 'its record names no keyed launch door' } };
+  return { machine: name, panes, activities: record.panes?.activities ?? [], ended: record.panes?.ended ?? [], capabilities: { guard: capabilities.guard ?? [], launches: Boolean(capabilities.launches), launchesRefused: capabilities.launches ? null : 'its record names no keyed launch door' } };
 }
 /** One reading of `machines` (normalized names) and this machine, `local`: every record, from this machine's daemon. */
 export async function readMachineFleet(machines, local) {
@@ -47,7 +74,8 @@ export async function readMachineFleet(machines, local) {
   const rows = names.map((name) => heldError ? { machine: name, error: `this machine's daemon did not give the records it holds: ${heldError}`, code: 'unavailable' } : readOne(name, held[name]));
   noteStep(`fleet reading (${names.length} machine${names.length === 1 ? '' : 's'}, held records)`, performance.now() - started);
   // `panes`: each answering machine's live pane ids, whether or not a session was read in them
-  const online = new Set(), unknown = new Set(), alive = new Set(), native = new Map(), sessions = [], capabilities = new Map(), panes = new Map();
+  // `ended`: the panes each machine saw end (its tmux's exit log, or a pane its host keeps dead), with how each ended
+  const online = new Set(), unknown = new Set(), alive = new Set(), native = new Map(), sessions = [], capabilities = new Map(), panes = new Map(), ended = new Map();
   let localError = null;
   for (const row of rows) {
     if (row.error) {
@@ -59,6 +87,10 @@ export async function readMachineFleet(machines, local) {
     capabilities.set(row.machine, row.capabilities ?? { guard: [], launches: false });
     const live = (row.panes ?? []).filter((pane) => !pane.dead && pane.pid);
     panes.set(row.machine, new Set((row.panes ?? []).filter((pane) => !pane.dead).map((pane) => pane.pane)));
+    ended.set(row.machine, new Map([
+      ...(row.ended ?? []).filter((exit) => exit?.pane && !panes.get(row.machine).has(exit.pane)).map((exit) => [exit.pane, { failed: exit.failed ?? null, how: exit.how ?? null }]),
+      ...(row.panes ?? []).filter((pane) => pane.dead).map((pane) => [pane.pane, { failed: null, how: 'its program ended (the pane is kept for its last screen)' }]),
+    ]));
     for (const { pid, activity } of row.activities ?? []) {
       const pane = live.find((item) => item.pid === pid);
       if (!pane || !activity?.session_id || !activity.harness) continue;
@@ -73,5 +105,5 @@ export async function readMachineFleet(machines, local) {
     }
   }
   const ok = online.has(local);
-  return { ok, presenceKnown: ok, online, unknown, alive, native, sessions, capabilities, panes, error: ok ? null : `machine ${local}'s door did not answer: ${localError ?? 'no row'}` };
+  return { ok, presenceKnown: ok, online, unknown, alive, native, sessions, capabilities, panes, ended, error: ok ? null : `machine ${local}'s door did not answer: ${localError ?? 'no row'}` };
 }

package/board/record-view.mjs

@@ -88,6 +88,8 @@ export class RecordView {
     if (this.ready) {
       for (const waiter of this.waiters) { clearTimeout(waiter.timer); waiter.resolve(this.records); }
       this.waiters.clear();
+      // whoever follows the records' changes (the dispatcher's wake) is told once a snapshot or a page of changes is whole
+      try { this.onApplied?.(this.records); } catch { /* a follower cannot fail the stream */ }
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.130",
+  "version": "0.5.132",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {