@volter/supercode-orchestrator 0.4.119 → 0.4.120

package/board/dispatch.mjs

@@ -201,27 +201,39 @@ export function lanesOf(profiles = {}) {
   return out;
 }
 
+const onMachine = machine => normalMachine(machine ?? localMachine()) === localMachine() ? [] : ['--machine', machine];
+const unavailable = message => Object.assign(new Error(message), { launchDeferred: true });
+
 // ------------------------------------------------------------------ supervise
 
 function supervise(board, fleet, settings, report, entry, effects) {
   const rows = board.db.prepare(entry.workflow.run ? "SELECT id FROM tasks WHERE current_run_id IS NOT NULL" : "SELECT id FROM tasks WHERE status = 'running'").all();
   // The catalog can lag a pane. Read the actual machines once, before any board mutation.
   const observed = new Map();
+  const exits = new Map();
   for (const { id } of rows) {
     const s = currentRun(board.db, id)?.metadata?.supercode;
     if (!s?.pane || observed.has(s.machine)) continue;
-    const r = sc(['teams', 'panes', 'ls', '--machine', s.machine ?? localMachine()], { timeout: 60_000 });
+    const r = sc(['teams', 'panes', 'ls', ...onMachine(s.machine)], { timeout: 60_000 });
     const list = r.code === 0 ? json(r.stdout.trim(), null) : null;
-    observed.set(s.machine, Array.isArray(list) ? new Set(list.map((p) => p.pane)) : null);
+    observed.set(s.machine, Array.isArray(list) ? new Set(list.filter(p => !p.dead).map((p) => p.pane)) : null);
     if (!Array.isArray(list)) report.failed.push([id, `pane liveness unavailable; retaining run: ${(r.stderr || r.stdout).trim().slice(-200)}`]);
   }
+  for (const {id} of rows) {
+    const session=currentRun(board.db,id)?.metadata?.supercode;
+    const panes=observed.get(session?.machine);
+    if(session?.launch_key && panes instanceof Set && !panes.has(session.pane)) {
+      try { const receipt=launchReceipt(board,session);if(receipt?.live===false&&receipt.exit)exits.set(id,receipt.exit); }
+      catch(error){report.failed.push([id,`exit receipt unavailable; retaining liveness observation: ${error.message}`]);}
+    }
+  }
   for (const { id } of rows) {
     tx(board, (db) => {
       const task = getTask(db, id);
       if (!task || (entry.workflow.run ? !task.current_run_id : task.status !== 'running')) return;
       const runRow = currentRun(db, id);
       let session = runRow?.metadata?.supercode ?? null;
-      if (session?.launch_pending) {
+      if (session?.launch_pending || session?.launch_deferred) {
         extendClaim(db, id, { ttl: settings.ttl, reason: 'launch_receipt_pending' });
         return; // An uncertain launch owns this run until a machine receipt settles it.
       }
@@ -305,6 +317,11 @@ function supervise(board, fleet, settings, report, entry, effects) {
         return;
       }
       if (session?.session_id) {
+        if(exits.has(id)) {
+          const exit=exits.get(id);
+          observe('exited',{code:exit.failed?1:0,error:`native session ${session.session_id} ended: ${exit.how}`,session_id:session.session_id,pane:session.pane},'crashed');
+          return;
+        }
         if (session.pane && paneLive === null) return; // no machine reading, no loss verdict
         if (!session.pane && !fleet.ok) return;
         const alive = session.pane ? paneLive : sessionAlive(fleet, session);
@@ -521,9 +538,23 @@ export function launchRun(board, entry, id, lane, fleet, settings) {
     if (slot === 'worker' || (worker?.session_id && worker.session_id === reviewer?.session_id)) throw new Error('a reviewer must have a different session from the implementer');
   }
   const meta = { status, role: spec.role, slot };
-  return surface === 'pane'
-    ? launchPane(board, task, runId, lane, fleet, { input: prompt, slot, meta })
-    : launchHeadless(board, task, runId, lane, settings, { prompt, skills: spec.skills ?? [], meta });
+  try {
+    const result = surface === 'pane'
+      ? launchPane(board, task, runId, lane, fleet, { input: prompt, slot, meta })
+      : launchHeadless(board, task, runId, lane, settings, { prompt, skills: spec.skills ?? [], meta });
+    tx(board,db=>setRunSession(db,runId,{launch_deferred:false,launch_deferred_error:null}));
+    return result;
+  } catch(error) {
+    if(error.launchDeferred) {
+      tx(board,db=>{
+        const previous=currentRun(db,id)?.metadata?.supercode;
+        setRunSession(db,runId,{...meta,launch_deferred:true,launch_deferred_error:error.message});
+        if(previous?.launch_deferred_error!==error.message)event(db,id,'launch_deferred',{reason:error.message},runId);
+      });
+      error.launchPending=true;
+    }
+    throw error;
+  }
 }
 
 /** Start one card now by a person's word (a triage card's session, a manager's start): the workflow's start event. */
@@ -589,9 +620,9 @@ export function closeRequested(board, report, { taskId = null, caller = null } =
     try {
       const machine = session.machine ?? localMachine();
       if (!panes.has(machine)) {
-        const r = sc(['teams', 'panes', 'ls', '--machine', machine], { timeout: 60_000, as: boardAddress(board.label) });
+        const r = sc(['teams', 'panes', 'ls', ...onMachine(machine)], { timeout: 60_000, as: boardAddress(board.label) });
         const list = r.code === 0 ? json(r.stdout.trim(), null) : null;
-        panes.set(machine, Array.isArray(list) ? { live: new Set(list.map((p) => p.pane)) } : { error: `cannot read panes on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}` });
+        panes.set(machine, Array.isArray(list) ? { live: new Set(list.filter(p => !p.dead).map((p) => p.pane)) } : { error: `cannot read panes on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}` });
       }
       const observed = panes.get(machine);
       if (observed.error) throw new Error(observed.error);
@@ -801,12 +832,12 @@ const getTaskNow = (board, id) => view(board, (db) => getTask(db, id));
 
 /** A transport error is not a negative launch receipt. The key remains owned by its run. */
 function launchReceipt(board, session) {
-  const r = sc(['teams', 'launches', 'get', session.launch_key, '--machine', session.machine],
+  const r = sc(['teams', 'launches', 'get', session.launch_key, ...onMachine(session.machine)],
     { timeout: 30_000, as: boardAddress(board.label) });
-  if (r.code !== 0) throw new Error(`launch receipt unavailable on ${session.machine}: install the matching Teams CLI and machine capability (teams launches get): ${(r.stderr || r.stdout).trim().slice(-300)}`);
+  if (r.code !== 0) throw unavailable(`launch receipt unavailable on ${session.machine}: install the matching Teams CLI and machine capability (teams launches get): ${(r.stderr || r.stdout).trim().slice(-300)}`);
   if (r.stdout.trim() === 'null') return null;
   const receipt = lastJson(r.stdout);
-  if (!receipt || receipt.key !== session.launch_key) throw new Error(`invalid launch receipt for ${session.launch_key} on ${session.machine}`);
+  if (!receipt || receipt.key !== session.launch_key) throw unavailable(`invalid launch receipt for ${session.launch_key} on ${session.machine}`);
   return receipt;
 }
 
@@ -854,7 +885,22 @@ export function reconcileCallerLaunch(board, sid) {
   }
 }
 
-function reconcileLaunches(board, fleet, report, entry, effects, settings) {
+function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes) {
+  const deferred=view(board,db=>db.prepare("SELECT t.id FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
+  for(const {id} of deferred) {
+    try {
+      const task=getTaskNow(board,id),status=view(board,db=>statusOf(db,task)),spec=runSpec(entry,status,task);
+      const profile=spec&&view(board,db=>evaluate(entry.workflow.roles?.[spec.role]??'card.assignee',scopeOf(db,board,task,{params:entry.params})));
+      const lane=lanes?.[profile];if(!lane){report.queued.push([id,'launch waits for its configured lane']);continue;}
+      const why=launcherCheck(board.db,task,lane,fleet);if(why){report.queued.push([id,why]);continue;}
+      const session=launchRun(board,entry,id,lane,fleet,settings);
+      if(session)report.started.push([id,session.address??session.session_id,session.mode]);
+    }catch(error){
+      if(error.launchPending){report.queued.push([id,error.message]);continue;}
+      const moved=tx(board,db=>fire(db,board,entry,id,'spawn_failed',{roles:['platform'],payload:{error:error.message}}));effects.push(...moved.effects);
+      report.failed.push([id,error.message]);
+    }
+  }
   const rows = view(board, (db) => db.prepare(`SELECT id, task_id, ended_at, metadata FROM task_runs
     WHERE json_extract(metadata, '$.supercode.launch_pending') = 1
     AND json_extract(metadata, '$.supercode.close_confirmed_at') IS NULL`).all());
@@ -903,15 +949,15 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
   const pending = view(board, (db) => db.prepare('SELECT id, ended_at, metadata FROM task_runs WHERE task_id = ? AND id != ?').all(task.id, runId))
     .map((r) => ({ ...r, session: json(r.metadata, {})?.supercode }))
     .find((r) => (r.session?.pane || r.session?.launch_key) && !r.session.close_transferred_to && (r.session.slot ?? 'worker') === slot && (r.ended_at || r.session.close_requested) && !r.session.close_confirmed_at);
-  if (pending) throw new Error(`slot ${slot} still owns pane ${pending.session.pane} from run ${pending.id}; closure must be confirmed before another session starts: ${pending.session.close_error ?? 'close pending'}`);
+  if (pending) throw unavailable(`slot ${slot} still owns pane ${pending.session.pane} from run ${pending.id}; closure must be confirmed before another session starts: ${pending.session.close_error ?? 'close pending'}`);
   let prev = view(board, (db) => lastSession(db, task.id, slot));
   if (prev?.session_id && !prev.forgotten && (prev.harness ?? 'claude-code') === harness && prev.machine === machine) {
     const fresh = observeFleet();
-    if (!fresh.ok) throw new Error(`cannot check for a manual continuation of ${prev.session_id}: ${fresh.error}`);
-    const r = sc(['teams', 'panes', 'ls', '--machine', machine], { timeout: 60_000 });
+    if (!fresh.ok) throw unavailable(`cannot check for a manual continuation of ${prev.session_id}: ${fresh.error}`);
+    const r = sc(['teams', 'panes', 'ls', ...onMachine(machine)], { timeout: 60_000 });
     const listed = r.code === 0 ? json(r.stdout.trim(), null) : null;
-    if (!Array.isArray(listed)) throw new Error(`cannot read panes before continuing ${prev.session_id}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
-    const pane = continuationPane(fresh, prev, new Set(listed.map((p) => p.pane)));
+    if (!Array.isArray(listed)) throw unavailable(`cannot read panes before continuing ${prev.session_id}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
+    const pane = continuationPane(fresh, prev, new Set(listed.filter(p => !p.dead).map((p) => p.pane)));
     if (pane && pane !== prev.pane) prev = { ...prev, pane, close_requested: null, closed_at: null, close_confirmed_at: null, close_error: null };
     fleet = fresh;
   }
@@ -920,7 +966,7 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
     closeRequested(board, report, { taskId: task.id });
     prev = view(board, (db) => lastSession(db, task.id, slot));
     // a pane that could not be closed still holds the session: no second process is started on it
-    if (prev?.close_requested && !prev.closed_at) throw new Error(`the slot's session ${prev.session_id} asked to be closed and its pane ${prev.pane} could not be: ${report.undelivered.map((u) => u[2]).join('; ') || 'no reason given'}`);
+    if (prev?.close_requested && !prev.closed_at) throw unavailable(`the slot's session ${prev.session_id} asked to be closed and its pane ${prev.pane} could not be: ${report.undelivered.map((u) => u[2]).join('; ') || 'no reason given'}`);
   }
   const openOwned = (key, sid, mode, args) => {
     launchReceipt(board, { machine, launch_key: key }); // Prove the exact receipt door before starting a process.
@@ -1035,10 +1081,10 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
 
 /** A machine reading, not a missing catalog entry, establishes whether a pane is alive. */
 function panePresent(machine, pane) {
-  const r = sc(['teams', 'panes', 'ls', '--machine', machine ?? localMachine()], { timeout: 60_000 });
+  const r = sc(['teams', 'panes', 'ls', ...onMachine(machine)], { timeout: 60_000 });
   const panes = r.code === 0 ? json(r.stdout.trim(), null) : null;
-  if (!Array.isArray(panes)) throw new Error(`cannot establish pane ${pane} liveness on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
-  return panes.some((p) => p.pane === pane);
+  if (!Array.isArray(panes)) throw unavailable(`cannot establish pane ${pane} liveness on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
+  return panes.some((p) => p.pane === pane && !p.dead);
 }
 
 /** Whether a pane just launched is still there after the harness had a moment to start. */
@@ -1576,7 +1622,7 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
       readMailbox(board, boardAddress(board.label), report); phase('mailbox');
       await refreshRelationAnswers(board); phase('answers');
       const effects = [];
-      reconcileLaunches(board, fleet, report, entry, effects, settings); supervise(board, fleet, settings, report, entry, effects); await runEffects(board, effects.splice(0), { retryNotices: true }); closeRequested(board, report); phase('supervise');
+      reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); await runEffects(board, effects.splice(0), { retryNotices: true }); closeRequested(board, report); phase('supervise');
       report.unsettled = [];
       report.promoted = recomputeReady(board, effects, report.unsettled);
       await sendBoardMessages(board, report);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.4.119",
+  "version": "0.4.120",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {