@volter/supercode-orchestrator 0.4.118 → 0.4.120

package/board/dispatch.mjs

@@ -116,11 +116,13 @@ export function observeFleet() {
   const list = sc(['message', 'list', '--json']);
   const sessions = list.code === 0 ? json(list.stdout.trim(), null) : null;
   if (!Array.isArray(sessions)) return { ok: false, alive: new Set(), online: new Set(), error: `message list: ${(list.stderr || list.stdout).trim().slice(-300)}` };
-  const machines = sc(['teams', 'machines', 'list', '--json']);
-  const rows = machines.code === 0 ? lastJson(machines.stdout)?.items : null;
+  const machines = sc(['teams', 'machines', 'events', '--json']);
+  const presence = machines.code === 0 ? lastJson(machines.stdout) : null;
   const online = new Set([localMachine()]);
-  for (const m of rows ?? []) if (m.enrollment === 'active' && m.connection === 'online') online.add(normalMachine(m.name));
-  return { ok: true, sessions, alive: new Set(sessions.map((s) => s.address).filter(Boolean)), online, error: rows ? null : 'teams machines list gave no answer; only this machine counts as online' };
+  for (const m of presence?.items ?? []) if (m.linked) online.add(normalMachine(m.name));
+  return { ok: true, sessions, alive: new Set(sessions.map((s) => s.address).filter(Boolean)), online,
+    presenceKnown: presence?.connected === true, machineEvents: presence,
+    error: presence?.connected ? null : 'machine event stream unavailable; retaining existing runs until availability is known' };
 }
 
 // how the launcher opens each harness a pane lane can run, and how it names the session: Claude Code is told the id
@@ -199,27 +201,39 @@ export function lanesOf(profiles = {}) {
   return out;
 }
 
+const onMachine = machine => normalMachine(machine ?? localMachine()) === localMachine() ? [] : ['--machine', machine];
+const unavailable = message => Object.assign(new Error(message), { launchDeferred: true });
+
 // ------------------------------------------------------------------ supervise
 
 function supervise(board, fleet, settings, report, entry, effects) {
   const rows = board.db.prepare(entry.workflow.run ? "SELECT id FROM tasks WHERE current_run_id IS NOT NULL" : "SELECT id FROM tasks WHERE status = 'running'").all();
   // The catalog can lag a pane. Read the actual machines once, before any board mutation.
   const observed = new Map();
+  const exits = new Map();
   for (const { id } of rows) {
     const s = currentRun(board.db, id)?.metadata?.supercode;
     if (!s?.pane || observed.has(s.machine)) continue;
-    const r = sc(['teams', 'panes', 'ls', '--machine', s.machine ?? localMachine()], { timeout: 60_000 });
+    const r = sc(['teams', 'panes', 'ls', ...onMachine(s.machine)], { timeout: 60_000 });
     const list = r.code === 0 ? json(r.stdout.trim(), null) : null;
-    observed.set(s.machine, Array.isArray(list) ? new Set(list.map((p) => p.pane)) : null);
+    observed.set(s.machine, Array.isArray(list) ? new Set(list.filter(p => !p.dead).map((p) => p.pane)) : null);
     if (!Array.isArray(list)) report.failed.push([id, `pane liveness unavailable; retaining run: ${(r.stderr || r.stdout).trim().slice(-200)}`]);
   }
+  for (const {id} of rows) {
+    const session=currentRun(board.db,id)?.metadata?.supercode;
+    const panes=observed.get(session?.machine);
+    if(session?.launch_key && panes instanceof Set && !panes.has(session.pane)) {
+      try { const receipt=launchReceipt(board,session);if(receipt?.live===false&&receipt.exit)exits.set(id,receipt.exit); }
+      catch(error){report.failed.push([id,`exit receipt unavailable; retaining liveness observation: ${error.message}`]);}
+    }
+  }
   for (const { id } of rows) {
     tx(board, (db) => {
       const task = getTask(db, id);
       if (!task || (entry.workflow.run ? !task.current_run_id : task.status !== 'running')) return;
       const runRow = currentRun(db, id);
       let session = runRow?.metadata?.supercode ?? null;
-      if (session?.launch_pending) {
+      if (session?.launch_pending || session?.launch_deferred) {
         extendClaim(db, id, { ttl: settings.ttl, reason: 'launch_receipt_pending' });
         return; // An uncertain launch owns this run until a machine receipt settles it.
       }
@@ -247,6 +261,30 @@ function supervise(board, fleet, settings, report, entry, effects) {
         }
       };
       const limits = limitsOf(db, board, entry, task, session);
+      // Availability is the same retained linked/unlinked feed subscribers hear.
+      // A failed stream is unknown, not evidence that every remote machine left.
+      if (session?.session_id && session.surface !== 'headless') {
+        const panes = session.pane ? observed.get(session.machine) : null;
+        const reachable = panes instanceof Set || fleet.online.has(session.machine);
+        if (!reachable && !fleet.presenceKnown) return;
+        if (!reachable) {
+          const next = {...session, offline_since: session.offline_since ?? t};
+          setRunSession(db,runRow.id,next);
+          observe('machine_offline',{machine:session.machine,seconds:t-next.offline_since},null);
+          const age = Number(entry.params.block_age_seconds ?? 3600);
+          if (t-Number(next.offline_notified_at ?? next.offline_since)>=age) {
+            setRunSession(db,runRow.id,{...currentRun(db,id)?.metadata?.supercode,offline_notified_at:t});
+            observe('machine_wait_aged',{machine:session.machine,seconds:t-next.offline_since,reason:`machine ${session.machine} has been unlinked for ${t-next.offline_since}s`},null);
+          }
+          if (getTask(db,id)?.current_run_id===runRow.id) extendClaim(db,id,{ttl:settings.ttl,reason:'machine_offline'});
+          return;
+        }
+        if (session.offline_since != null) {
+          setRunSession(db,runRow.id,{...session,offline_since:null,offline_notified_at:null});
+          observe('machine_online',{machine:session.machine,reason:'machine linked again'},null);
+          session=currentRun(db,id)?.metadata?.supercode ?? session;
+        }
+      }
       // Hermes's orphan reconciliation: a running card whose claim bookkeeping is broken, with nothing live behind it
       const paneReading = session?.pane ? observed.get(session.machine) : undefined;
       const paneLive = paneReading instanceof Set ? paneReading.has(session.pane) : null;
@@ -279,6 +317,11 @@ function supervise(board, fleet, settings, report, entry, effects) {
         return;
       }
       if (session?.session_id) {
+        if(exits.has(id)) {
+          const exit=exits.get(id);
+          observe('exited',{code:exit.failed?1:0,error:`native session ${session.session_id} ended: ${exit.how}`,session_id:session.session_id,pane:session.pane},'crashed');
+          return;
+        }
         if (session.pane && paneLive === null) return; // no machine reading, no loss verdict
         if (!session.pane && !fleet.ok) return;
         const alive = session.pane ? paneLive : sessionAlive(fleet, session);
@@ -286,13 +329,6 @@ function supervise(board, fleet, settings, report, entry, effects) {
         const young = t - Number(session.launched_at ?? 0) < Number(limits.grace ?? settings.grace);
         const missing = !alive && machineUp && !young;
         const next = { ...session };
-        // a machine that is off the fleet: how long, for the workflow's machine_offline (the claim is kept meanwhile)
-        if (machineUp) next.offline_since = null; else next.offline_since ??= t;
-        if (!machineUp) {
-          setRunSession(db, runRow.id, next);
-          observe('machine_offline', { machine: session.machine, seconds: t - next.offline_since }, null);
-          if (getTask(db, id)?.current_run_id !== runRow.id) return; // the workflow moved it
-        }
         if (alive) { next.seen_at ??= t; next.missing_since = null; }
         if (missing) next.missing_since ??= t;
         if (missing && t - next.missing_since >= Number(limits.grace ?? settings.grace)) {
@@ -502,9 +538,23 @@ export function launchRun(board, entry, id, lane, fleet, settings) {
     if (slot === 'worker' || (worker?.session_id && worker.session_id === reviewer?.session_id)) throw new Error('a reviewer must have a different session from the implementer');
   }
   const meta = { status, role: spec.role, slot };
-  return surface === 'pane'
-    ? launchPane(board, task, runId, lane, fleet, { input: prompt, slot, meta })
-    : launchHeadless(board, task, runId, lane, settings, { prompt, skills: spec.skills ?? [], meta });
+  try {
+    const result = surface === 'pane'
+      ? launchPane(board, task, runId, lane, fleet, { input: prompt, slot, meta })
+      : launchHeadless(board, task, runId, lane, settings, { prompt, skills: spec.skills ?? [], meta });
+    tx(board,db=>setRunSession(db,runId,{launch_deferred:false,launch_deferred_error:null}));
+    return result;
+  } catch(error) {
+    if(error.launchDeferred) {
+      tx(board,db=>{
+        const previous=currentRun(db,id)?.metadata?.supercode;
+        setRunSession(db,runId,{...meta,launch_deferred:true,launch_deferred_error:error.message});
+        if(previous?.launch_deferred_error!==error.message)event(db,id,'launch_deferred',{reason:error.message},runId);
+      });
+      error.launchPending=true;
+    }
+    throw error;
+  }
 }
 
 /** Start one card now by a person's word (a triage card's session, a manager's start): the workflow's start event. */
@@ -570,9 +620,9 @@ export function closeRequested(board, report, { taskId = null, caller = null } =
     try {
       const machine = session.machine ?? localMachine();
       if (!panes.has(machine)) {
-        const r = sc(['teams', 'panes', 'ls', '--machine', machine], { timeout: 60_000, as: boardAddress(board.label) });
+        const r = sc(['teams', 'panes', 'ls', ...onMachine(machine)], { timeout: 60_000, as: boardAddress(board.label) });
         const list = r.code === 0 ? json(r.stdout.trim(), null) : null;
-        panes.set(machine, Array.isArray(list) ? { live: new Set(list.map((p) => p.pane)) } : { error: `cannot read panes on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}` });
+        panes.set(machine, Array.isArray(list) ? { live: new Set(list.filter(p => !p.dead).map((p) => p.pane)) } : { error: `cannot read panes on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}` });
       }
       const observed = panes.get(machine);
       if (observed.error) throw new Error(observed.error);
@@ -618,7 +668,7 @@ function observeIdle(board, fleet, settings, report, entry, effects) {
       if (!(entry.workflow.statuses[status]?.on?.session_idle || entry.workflow.on?.session_idle || entry.workflow.run?.on?.session_idle)) return;
       const runRow = currentRun(db, id);
       const session = runRow?.metadata?.supercode;
-      if (!session?.session_id || session.surface !== 'pane') return;
+      if (!session?.session_id || session.surface !== 'pane' || session.offline_since != null) return;
       const live = fleet.sessions.find((x) => x.address?.endsWith(`:${session.session_id}`));
       const observedAt = now();
       const lastAt = Math.floor(Number(live?.last_message_at_ms) / 1000);
@@ -727,6 +777,7 @@ function launcherCheck(db, task, lane, fleet) {
   if (!task.machine) return 'the card names no machine; a pane session runs on the machine its card names';
   const machine = normalMachine(task.machine);
   if (!fleet.ok) return `supercode could not read the fleet: ${fleet.error}`;
+  if (machine !== localMachine() && !fleet.presenceKnown) return 'machine event stream unavailable; launch waits for a current availability reading';
   if (!fleet.online.has(machine)) return `machine ${machine} is offline; the card starts when it is back`;
   // a card on this machine gets its workspace made here (scratch, dir or worktree, as Hermes makes it); another
   // machine's card names its directory there
@@ -781,12 +832,12 @@ const getTaskNow = (board, id) => view(board, (db) => getTask(db, id));
 
 /** A transport error is not a negative launch receipt. The key remains owned by its run. */
 function launchReceipt(board, session) {
-  const r = sc(['teams', 'launches', 'get', session.launch_key, '--machine', session.machine],
+  const r = sc(['teams', 'launches', 'get', session.launch_key, ...onMachine(session.machine)],
     { timeout: 30_000, as: boardAddress(board.label) });
-  if (r.code !== 0) throw new Error(`launch receipt unavailable on ${session.machine}: install the matching Teams CLI and machine capability (teams launches get): ${(r.stderr || r.stdout).trim().slice(-300)}`);
+  if (r.code !== 0) throw unavailable(`launch receipt unavailable on ${session.machine}: install the matching Teams CLI and machine capability (teams launches get): ${(r.stderr || r.stdout).trim().slice(-300)}`);
   if (r.stdout.trim() === 'null') return null;
   const receipt = lastJson(r.stdout);
-  if (!receipt || receipt.key !== session.launch_key) throw new Error(`invalid launch receipt for ${session.launch_key} on ${session.machine}`);
+  if (!receipt || receipt.key !== session.launch_key) throw unavailable(`invalid launch receipt for ${session.launch_key} on ${session.machine}`);
   return receipt;
 }
 
@@ -834,7 +885,22 @@ export function reconcileCallerLaunch(board, sid) {
   }
 }
 
-function reconcileLaunches(board, fleet, report, entry, effects, settings) {
+function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes) {
+  const deferred=view(board,db=>db.prepare("SELECT t.id FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
+  for(const {id} of deferred) {
+    try {
+      const task=getTaskNow(board,id),status=view(board,db=>statusOf(db,task)),spec=runSpec(entry,status,task);
+      const profile=spec&&view(board,db=>evaluate(entry.workflow.roles?.[spec.role]??'card.assignee',scopeOf(db,board,task,{params:entry.params})));
+      const lane=lanes?.[profile];if(!lane){report.queued.push([id,'launch waits for its configured lane']);continue;}
+      const why=launcherCheck(board.db,task,lane,fleet);if(why){report.queued.push([id,why]);continue;}
+      const session=launchRun(board,entry,id,lane,fleet,settings);
+      if(session)report.started.push([id,session.address??session.session_id,session.mode]);
+    }catch(error){
+      if(error.launchPending){report.queued.push([id,error.message]);continue;}
+      const moved=tx(board,db=>fire(db,board,entry,id,'spawn_failed',{roles:['platform'],payload:{error:error.message}}));effects.push(...moved.effects);
+      report.failed.push([id,error.message]);
+    }
+  }
   const rows = view(board, (db) => db.prepare(`SELECT id, task_id, ended_at, metadata FROM task_runs
     WHERE json_extract(metadata, '$.supercode.launch_pending') = 1
     AND json_extract(metadata, '$.supercode.close_confirmed_at') IS NULL`).all());
@@ -883,15 +949,15 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
   const pending = view(board, (db) => db.prepare('SELECT id, ended_at, metadata FROM task_runs WHERE task_id = ? AND id != ?').all(task.id, runId))
     .map((r) => ({ ...r, session: json(r.metadata, {})?.supercode }))
     .find((r) => (r.session?.pane || r.session?.launch_key) && !r.session.close_transferred_to && (r.session.slot ?? 'worker') === slot && (r.ended_at || r.session.close_requested) && !r.session.close_confirmed_at);
-  if (pending) throw new Error(`slot ${slot} still owns pane ${pending.session.pane} from run ${pending.id}; closure must be confirmed before another session starts: ${pending.session.close_error ?? 'close pending'}`);
+  if (pending) throw unavailable(`slot ${slot} still owns pane ${pending.session.pane} from run ${pending.id}; closure must be confirmed before another session starts: ${pending.session.close_error ?? 'close pending'}`);
   let prev = view(board, (db) => lastSession(db, task.id, slot));
   if (prev?.session_id && !prev.forgotten && (prev.harness ?? 'claude-code') === harness && prev.machine === machine) {
     const fresh = observeFleet();
-    if (!fresh.ok) throw new Error(`cannot check for a manual continuation of ${prev.session_id}: ${fresh.error}`);
-    const r = sc(['teams', 'panes', 'ls', '--machine', machine], { timeout: 60_000 });
+    if (!fresh.ok) throw unavailable(`cannot check for a manual continuation of ${prev.session_id}: ${fresh.error}`);
+    const r = sc(['teams', 'panes', 'ls', ...onMachine(machine)], { timeout: 60_000 });
     const listed = r.code === 0 ? json(r.stdout.trim(), null) : null;
-    if (!Array.isArray(listed)) throw new Error(`cannot read panes before continuing ${prev.session_id}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
-    const pane = continuationPane(fresh, prev, new Set(listed.map((p) => p.pane)));
+    if (!Array.isArray(listed)) throw unavailable(`cannot read panes before continuing ${prev.session_id}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
+    const pane = continuationPane(fresh, prev, new Set(listed.filter(p => !p.dead).map((p) => p.pane)));
     if (pane && pane !== prev.pane) prev = { ...prev, pane, close_requested: null, closed_at: null, close_confirmed_at: null, close_error: null };
     fleet = fresh;
   }
@@ -900,7 +966,7 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
     closeRequested(board, report, { taskId: task.id });
     prev = view(board, (db) => lastSession(db, task.id, slot));
     // a pane that could not be closed still holds the session: no second process is started on it
-    if (prev?.close_requested && !prev.closed_at) throw new Error(`the slot's session ${prev.session_id} asked to be closed and its pane ${prev.pane} could not be: ${report.undelivered.map((u) => u[2]).join('; ') || 'no reason given'}`);
+    if (prev?.close_requested && !prev.closed_at) throw unavailable(`the slot's session ${prev.session_id} asked to be closed and its pane ${prev.pane} could not be: ${report.undelivered.map((u) => u[2]).join('; ') || 'no reason given'}`);
   }
   const openOwned = (key, sid, mode, args) => {
     launchReceipt(board, { machine, launch_key: key }); // Prove the exact receipt door before starting a process.
@@ -1015,10 +1081,10 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
 
 /** A machine reading, not a missing catalog entry, establishes whether a pane is alive. */
 function panePresent(machine, pane) {
-  const r = sc(['teams', 'panes', 'ls', '--machine', machine ?? localMachine()], { timeout: 60_000 });
+  const r = sc(['teams', 'panes', 'ls', ...onMachine(machine)], { timeout: 60_000 });
   const panes = r.code === 0 ? json(r.stdout.trim(), null) : null;
-  if (!Array.isArray(panes)) throw new Error(`cannot establish pane ${pane} liveness on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
-  return panes.some((p) => p.pane === pane);
+  if (!Array.isArray(panes)) throw unavailable(`cannot establish pane ${pane} liveness on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
+  return panes.some((p) => p.pane === pane && !p.dead);
 }
 
 /** Whether a pane just launched is still there after the harness had a moment to start. */
@@ -1438,6 +1504,7 @@ function readMailbox(board, address, report) {
     try { renameSync(path, join(box, 'cur', name)); } catch { continue; } // another reader took it
     // an answer to a question the board asked is what a card waits on (read by the card's workflow), not a verb
     if (envelope.in_reply_to) continue;
+    if (envelope.from === `sc:${localMachine()}:operator:teams-events` && json(envelope.body, {})?.machine_event) { report.mailed.push([envelope.from, 'machine_event']); continue; }
     const argv = json(envelope.body, {})?.workflow;
     let answer;
     if (!Array.isArray(argv) || !argv.length || NOT_BY_MAIL.has(argv[0])) answer = `not applied: a mailed card update is one \`supercode workflow\` verb other than ${[...NOT_BY_MAIL].join(', ')}`;
@@ -1548,10 +1615,14 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
       // where the tick's time goes, phase by phase (ms), printed by `serve --verbose` and in `--json`
       let mark = Date.now();
       const phase = (name) => { const t = Date.now(); report.ms[name] = t - mark; mark = t; };
+      if (!fleet.machineEvents?.subscriptions?.includes(boardAddress(board.label))) {
+        const subscription = sc(['teams','machines','subscribe',boardAddress(board.label),'--json']);
+        if (subscription.code !== 0) report.failed.push(['machine_events',(subscription.stderr || subscription.stdout).trim().slice(-200)]);
+      }
       readMailbox(board, boardAddress(board.label), report); phase('mailbox');
       await refreshRelationAnswers(board); phase('answers');
       const effects = [];
-      reconcileLaunches(board, fleet, report, entry, effects, settings); supervise(board, fleet, settings, report, entry, effects); await runEffects(board, effects.splice(0), { retryNotices: true }); closeRequested(board, report); phase('supervise');
+      reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); await runEffects(board, effects.splice(0), { retryNotices: true }); closeRequested(board, report); phase('supervise');
       report.unsettled = [];
       report.promoted = recomputeReady(board, effects, report.unsettled);
       await sendBoardMessages(board, report);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.4.118",
+  "version": "0.4.120",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {