@volter/supercode-orchestrator 0.5.129 → 0.5.131

package/board/cli.mjs

@@ -64,6 +64,7 @@ async function boardPublishing(root) {
 }
 import { dispatchingOf, knowLocalMachine, requestAdoption, alertManagers, boardAddress, boardLabel, boardSettings, boardCaller, callerSession, commandEnv, reconcileCallerLaunch, closeRequested, lanesOf, localMachine, observeFleet, paused, plan, startCard, stopDispatching, supercodeBin, tick, deliver, startDelivery } from './dispatch.mjs';
 import { watchBoard } from './wake.mjs';
+import { onPanesChanged } from './machine-fleet.mjs';
 import { streamBoardPublications } from './event-stream.mjs';
 import { completionOwnerOf, followHome, isLatestReviewer, loadWorkflow, readHome, requestSessionClose, scopeOf, statusOf, useWorkflow, workflowEntry, workflowFor } from './workflow.mjs';
 import { evaluate } from './expr.mjs';
@@ -1702,6 +1703,9 @@ async function serve(root, args = {}) {
     if (watched !== undefined && board) heldDocumentsWatched(board, id ?? key, watched);
     else if (uncertain && !board) heldDocumentsChanged();
   } });
+  // a machine's panes changing in its record (a session came up under its pane, a pane or its session ended) is what a
+  // round binds a launch, verifies an adoption and reads a session's end from: the next round is due then
+  onPanesChanged(() => events?.wake('machine-record'));
   // the owner holds its boards only now, its watch installed (watch before load): opened once, changed by events
   holdBoards(root);
   // a pause of the collector holds the loop like any synchronous step: each is a step, so a block line names it
@@ -1723,7 +1727,7 @@ async function serve(root, args = {}) {
   };
   await registerOwner();
   onLocalReopen(registerOwner);
-  log('dispatch wakes on board writes, document edits and board mail; the interval sweep reads each machine\'s own door');
+  log('dispatch wakes on board writes, document edits, board mail and a change of a machine\'s panes in its record; the interval sweep reads each machine\'s own door');
   const stuck = {};
   const warned = {};
   let pausedSaid = false;

package/board/dispatch.mjs

@@ -7,8 +7,9 @@
 // - `pane`: an interactive session of the lane's harness (Claude Code or Codex) on the card's machine (`supercode open
 //   --new … --on <machine>`), its session id recorded on the run: one the dispatcher chose (Claude Code's
 //   `--session-id`) or the one the harness named, read from the fleet by its pane (Codex). It closes when its run ends. It is live
-//   while supercode's fleet lists it; one missing for the whole grace on an online machine is lost, and the next start
-//   resumes it in its own harness. Nothing asks the session about itself.
+//   while supercode's fleet lists it; one whose pane its machine saw end is lost at once, one missing with no end seen
+//   for the whole grace on an online machine is lost then, and the next start resumes it in its own harness. Nothing
+//   asks the session about itself.
 // - `headless`: the profile's worker for one query on this machine (`bin/hermes-worker.mjs`, Hermes's worker
 //   argv and task environment), watched by its pid as Hermes watches its own.
 //
@@ -179,7 +180,11 @@ const addressOf = (s) => s?.address ?? (s?.session_id && s.machine ? `sc:${s.mac
 // A session in a live pane of its machine is the fleet's reading; any other run is read on its own machine: a headless
 // worker by its pid, a pane by that machine's pane list, a session in no pane (an adopted native one) by that machine's
 // harness discovery. Null only when the machine cannot answer.
-const sessionAlive = (fleet, session) => fleet.native?.has(addressOf(session)) ? fleet.native.get(addressOf(session)).alive : unpublishedAlive(session);
+// A pane is read from the machine's record when the fleet reading holds its panes (current, or the machine is not in
+// it): the pane list asked through its door serves READING_MS, so a pane that ended since that reading was read live
+// for up to those 15 s more, and a session's loss was read that much late.
+const paneInRecord = (fleet, session) => session?.surface !== 'headless' && session?.pane && session.machine && fleet?.panes?.has(normalMachine(session.machine)) ? fleet.panes.get(normalMachine(session.machine)).has(session.pane) : null;
+const sessionAlive = (fleet, session) => fleet.native?.has(addressOf(session)) ? fleet.native.get(addressOf(session)).alive : paneInRecord(fleet, session) ?? unpublishedAlive(session);
 /** A subscriber's address as the session it names, for sessionAlive (its machine, harness and native id). */
 const sessionAt = (address) => { const m = /^sc:([^:]+):([^:]+):(.+)$/.exec(String(address)); return m && HARNESSES[m[2]] ? { address, machine: normalMachine(m[1]), harness: m[2], session_id: m[3] } : null; };
 /** A subscriber's liveness, read first (readBefore): true, false, or null when it could not be read (unknown, never dead). */
@@ -563,17 +568,30 @@ async function supervise(board, fleet, settings, report, entry, effects) {
         const alive = sessionAlive(fleet,session);
         if (alive == null) return;
         const machineUp = fleet.online.has(session.machine);
-        const young = t - Number(session.launched_at ?? 0) < Number(limits.grace ?? settings.grace);
-        const missing = !alive && machineUp && !young;
+        // the grace after a launch is for a session that has not come up yet: one already seen live (bound at its
+        // launch, verified at its adoption) has, and is missed from the first reading it is gone from, so its loss is
+        // read one grace after it went, not two (an adopted session that ended inside the grace of its adoption waited
+        // out both)
+        const young = !session.seen_at && t - Number(session.launched_at ?? 0) < Number(limits.grace ?? settings.grace);
+        // its pane's end as its machine saw it (the record's `ended`: tmux's own exit log, or a pane kept dead): the
+        // session is known to be gone, and nothing is waited for. The grace is for a session that is only missing from
+        // a reading, with no end seen: it may be read again. A launch the board keyed is told how it ended by its
+        // machine's receipt (above: a clean end is a session someone closed, never a loss to recover from), so its
+        // receipt is what is waited for, as it was, on a round whose reading of it failed
+        const receipted = Boolean(session.launch_key) && Boolean(fleet.capabilities?.get(normalMachine(session.machine))?.launches);
+        const ended = !alive && machineUp && session.pane && !receipted ? fleet.ended?.get(normalMachine(session.machine))?.get(session.pane) ?? null : null;
+        const missing = !alive && machineUp && (!young || Boolean(ended));
         const next = { ...session };
         if (alive) { next.seen_at ??= t; next.missing_since = null; }
         if (missing) next.missing_since ??= t;
-        if (missing && t - next.missing_since >= Number(limits.grace ?? settings.grace)) {
-          // lost: gone from an online machine for the whole grace. A resume that never came up is not resumable.
+        if (missing && (ended || t - next.missing_since >= Number(limits.grace ?? settings.grace))) {
+          // lost: its pane's end was seen, or it is gone from an online machine for the whole grace. A resume that never
+          // came up is not resumable.
           if (next.mode === 'resume' && !next.seen_at) next.resumable = false;
           setRunSession(db, runRow.id, next);
           observe('session_lost', { session_id: next.session_id, machine: next.machine, pane: next.pane, mode: next.mode, seen: Boolean(next.seen_at),
-            never_came_up: next.mode === 'fresh' && !next.seen_at, error: `session ${next.session_id} is gone from ${next.machine}` }, 'lost');
+            never_came_up: next.mode === 'fresh' && !next.seen_at, ...(ended ? { pane_ended: ended } : {}),
+            error: `session ${next.session_id} is gone from ${next.machine}${ended ? `: its pane ${next.pane} ended${ended.how ? ` (${ended.how})` : ''}` : ''}` }, 'lost');
           return;
         }
         if (JSON.stringify(next) !== JSON.stringify(session)) setRunSession(db, runRow.id, next);
@@ -1054,6 +1072,8 @@ async function turnAtClose(session, pane = null, { launchWindow = null } = {}) {
   return `its session ${who} is in a turn (${activity.turn ?? 'unknown'}); it closes once the turn ends`;
 }
 
+/** The closes this process is making now, by run: each one's outcome, for a pass that meets it under way. */
+const closesUnderway = new Map();
 export async function closeRequested(board, report, { taskId = null, caller = null, ttl = null, fleet = null } = {}) {
   const rows = view(board, (db) => db.prepare(`SELECT id, task_id, ended_at, metadata FROM task_runs
     WHERE metadata IS NOT NULL ${taskId ? 'AND task_id = ?' : ''}`).all(...(taskId ? [taskId] : [])));
@@ -1132,6 +1152,24 @@ export async function closeRequested(board, report, { taskId = null, caller = nu
     // Persist retries even for historical ends that did not have a close_session action. The close is claimed in the
     // same transaction, before the pane or process is touched: of two rounds (or a round and a verb) that read one close,
     // only the one whose claim committed closes it, and the other never records the pane it finds gone as a failure.
+    // One close of a run at a time in this process too. The claim below keeps another process off it, never this one
+    // (claimHolds: its own claim does not hold it), and the owner makes a close in more than one pass side by side: a
+    // round's, and each `workflow close` it answers. A second pass that began while the first was stopping the session
+    // found the pane still listed and no session under it, and answered "close deferred: pane … runs …, whose session
+    // could not be read" for a session that was being closed (the system test's close loop, 1 take in 7). It waits
+    // for the close under way and reports what that one did.
+    const flight = `${board.root}\u0000${board.slug ?? ''}\u0000${row.id}`;
+    const underway = closesUnderway.get(flight);
+    if (underway) {
+      const made = await underway;
+      if (made.error) report.undelivered.push([row.task_id, addressOf(session), made.error, 'close']);
+      else if (made.closed) (report.closed ??= []).push({ task_id: row.task_id, ...session });
+      continue;
+    }
+    let settle;
+    closesUnderway.set(flight, new Promise((resolve) => { settle = resolve; }));
+    let error = null, closed = false;
+    try {
     const claimed = tx(board, (db) => {
       const current = json(db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(row.id)?.metadata, {})?.supercode;
       if (!current || current.close_confirmed_at || current.close_transferred_to || current.pane !== session.pane || current.session_id !== session.session_id || claimHolds(current.close_claim)) return false;
@@ -1142,7 +1180,6 @@ export async function closeRequested(board, report, { taskId = null, caller = nu
       return true;
     });
     if (!claimed) continue;
-    let error = null;
     try {
       const machine = session.machine ?? localMachine();
       if (session.surface === 'native' && !session.pane) {
@@ -1208,7 +1245,8 @@ export async function closeRequested(board, report, { taskId = null, caller = nu
       }
     });
     if (error) report.undelivered.push([row.task_id, addressOf(session), error, 'close']);
-    else (report.closed ??= []).push({ task_id: row.task_id, ...session });
+    else { closed = true; (report.closed ??= []).push({ task_id: row.task_id, ...session }); }
+    } finally { closesUnderway.delete(flight); settle({ error, closed }); }
   }
 }
 

package/board/machine-fleet.mjs

@@ -11,6 +11,33 @@ import { noteStep } from '@volter/supercode-harness-sdk/slow-log';
 import { machineCall, machineRecordStream } from './doors.mjs';
 import { RecordView } from './record-view.mjs';
 const recordView = new RecordView(machineRecordStream);
+/**
+ * What a round decides from in a machine's record: its live panes, the session under each and whether it runs, and
+ * the panes it saw end. Not a session's turn, which changes all day and starts, binds and ends nothing.
+ */
+const paneFacts = (held) => {
+  const record = held?.record;
+  if (!record?.panes) return '';
+  const under = new Map((record.panes.activities ?? []).map(({ pid, activity }) => [pid, activity]));
+  return JSON.stringify([
+    (record.panes.panes ?? []).map((pane) => [pane.pane, Boolean(pane.dead), under.get(pane.pid)?.session_id ?? null, under.get(pane.pid)?.presence === 'running']).sort(),
+    (record.panes.ended ?? []).map((exit) => exit.pane).sort(),
+  ]);
+};
+const paneListeners = new Set();
+let factsHeld = new Map();
+recordView.onApplied = (records) => {
+  const next = new Map(Object.entries(records).map(([machine, held]) => [machine, paneFacts(held)]));
+  const changed = [...next].filter(([machine, facts]) => factsHeld.has(machine) && factsHeld.get(machine) !== facts).map(([machine]) => machine);
+  factsHeld = next;
+  if (changed.length) for (const listener of paneListeners) { try { listener(changed); } catch { /* a listener cannot stop the record's reading */ } }
+};
+/**
+ * Told when a machine's panes change in its record (a pane or the session under it came up, ended or was closed):
+ * the dispatcher's next round is due then, not at its interval's sweep. Returns its unsubscribe. The record's stream is
+ * opened by a round's first reading and held from then on.
+ */
+export function onPanesChanged(listener) { paneListeners.add(listener); return () => paneListeners.delete(listener); }
 const DEADLINE_MS = 10_000;
 const within = (promise, ms) => Promise.race([promise, new Promise((_, reject) => setTimeout(() => reject(Object.assign(new Error(`no answer within ${ms} ms`), { code: 'timeout' })), ms).unref?.())]);
 // One machine's reading, from its held record: its panes, what a board launch needs of it, and the session under each
@@ -27,7 +54,7 @@ function readOne(name, held) {
   if (notCurrent) return { machine: name, error: `its panes are not current: ${notCurrent}`, code: 'stale' };
   const panes = record.panes?.panes ?? [];
   const capabilities = record.capabilities ?? { guard: [], launches: false };
-  return { machine: name, panes, activities: record.panes?.activities ?? [], capabilities: { guard: capabilities.guard ?? [], launches: Boolean(capabilities.launches), launchesRefused: capabilities.launches ? null : 'its record names no keyed launch door' } };
+  return { machine: name, panes, activities: record.panes?.activities ?? [], ended: record.panes?.ended ?? [], capabilities: { guard: capabilities.guard ?? [], launches: Boolean(capabilities.launches), launchesRefused: capabilities.launches ? null : 'its record names no keyed launch door' } };
 }
 /** One reading of `machines` (normalized names) and this machine, `local`: every record, from this machine's daemon. */
 export async function readMachineFleet(machines, local) {
@@ -47,7 +74,8 @@ export async function readMachineFleet(machines, local) {
   const rows = names.map((name) => heldError ? { machine: name, error: `this machine's daemon did not give the records it holds: ${heldError}`, code: 'unavailable' } : readOne(name, held[name]));
   noteStep(`fleet reading (${names.length} machine${names.length === 1 ? '' : 's'}, held records)`, performance.now() - started);
   // `panes`: each answering machine's live pane ids, whether or not a session was read in them
-  const online = new Set(), unknown = new Set(), alive = new Set(), native = new Map(), sessions = [], capabilities = new Map(), panes = new Map();
+  // `ended`: the panes each machine saw end (its tmux's exit log, or a pane its host keeps dead), with how each ended
+  const online = new Set(), unknown = new Set(), alive = new Set(), native = new Map(), sessions = [], capabilities = new Map(), panes = new Map(), ended = new Map();
   let localError = null;
   for (const row of rows) {
     if (row.error) {
@@ -59,6 +87,10 @@ export async function readMachineFleet(machines, local) {
     capabilities.set(row.machine, row.capabilities ?? { guard: [], launches: false });
     const live = (row.panes ?? []).filter((pane) => !pane.dead && pane.pid);
     panes.set(row.machine, new Set((row.panes ?? []).filter((pane) => !pane.dead).map((pane) => pane.pane)));
+    ended.set(row.machine, new Map([
+      ...(row.ended ?? []).filter((exit) => exit?.pane && !panes.get(row.machine).has(exit.pane)).map((exit) => [exit.pane, { failed: exit.failed ?? null, how: exit.how ?? null }]),
+      ...(row.panes ?? []).filter((pane) => pane.dead).map((pane) => [pane.pane, { failed: null, how: 'its program ended (the pane is kept for its last screen)' }]),
+    ]));
     for (const { pid, activity } of row.activities ?? []) {
       const pane = live.find((item) => item.pid === pid);
       if (!pane || !activity?.session_id || !activity.harness) continue;
@@ -73,5 +105,5 @@ export async function readMachineFleet(machines, local) {
     }
   }
   const ok = online.has(local);
-  return { ok, presenceKnown: ok, online, unknown, alive, native, sessions, capabilities, panes, error: ok ? null : `machine ${local}'s door did not answer: ${localError ?? 'no row'}` };
+  return { ok, presenceKnown: ok, online, unknown, alive, native, sessions, capabilities, panes, ended, error: ok ? null : `machine ${local}'s door did not answer: ${localError ?? 'no row'}` };
 }

package/board/record-view.mjs

@@ -88,6 +88,8 @@ export class RecordView {
     if (this.ready) {
       for (const waiter of this.waiters) { clearTimeout(waiter.timer); waiter.resolve(this.records); }
       this.waiters.clear();
+      // whoever follows the records' changes (the dispatcher's wake) is told once a snapshot or a page of changes is whole
+      try { this.onApplied?.(this.records); } catch { /* a follower cannot fail the stream */ }
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.129",
+  "version": "0.5.131",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {
@@ -24,7 +24,7 @@
     "check-types": "node scripts/generate-types.mjs --check"
   },
   "dependencies": {
-    "@volter/supercode-harness-sdk": "^0.3.60",
+    "@volter/supercode-harness-sdk": "^0.3.89",
     "@volter/teams": "0.2.52",
     "@volter/ztrack": "^2.4.23",
     "yaml": "^2.9.0"