@volter/supercode-orchestrator 0.5.25 → 0.5.27

package/bin/agent-door.mjs

@@ -2,7 +2,6 @@
 // The orchestrator's synchronous host reaches Teams through the installed neutral SDK.
 import { fromContext } from '@volter/teams/context';
 import { supercodeHome } from '@volter/teams/home';
-import { createHash } from 'node:crypto';
 import { openMachine, normalMachineName } from '@volter/teams/machine-door';
 import { hostname } from 'node:os';
 import { StreamRpc } from '@volter/teams/client';
@@ -31,10 +30,21 @@ try{
   }
   else if(op==='reader')result=await team.agents.delegateReader(args[0],{},{...options,ifMatch:args[1]});
   else if(op==='bindContext'){
-    const key=createHash('sha256').update(`${scope.origin}/${scope.team_id}/${args[0]}`).digest('hex');
-    const channel=await openMachine({machine:reach(args[2]),context:context??process.env.SUPERCODE_CONTEXT});
-    try{const rpc=new StreamRpc(await channel.open('node'));result=await rpc.call('harness.v1.agents.bindSession',{session:args[1],context:`agent-${key.slice(0,32)}`,assignments:args[3]??[],intended:args[4]??null});}
-    finally{channel.close();}
+    // The binding names the agent's reader context on the session's machine, so it installs that context itself: a
+    // launch installs one before its session starts, but an adopted session never had one, and a context installed a
+    // day before has expired. An ended agent has neither a reader nor a live session: Teams revoked its credentials
+    // and ended its sessions when it ended, so there is nothing to bind.
+    const agent=(await team.agents.get(args[0],options)).data;
+    if(agent.state==='ended')result={bound:false,reason:`agent ${agent.id} ended`};
+    else{
+      const reader=await team.agents.delegateReader(agent.id,{expires_in_ms:86400000},{...options,ifMatch:agent.revision});
+      const channel=await openMachine({machine:reach(args[2]),context:context??process.env.SUPERCODE_CONTEXT});
+      try{
+        const rpc=new StreamRpc(await channel.open('node'));
+        const installed=await rpc.call('harness.v1.agents.context',{origin:scope.origin,team:scope.team_id,agent:agent.id,credential:reader.data});
+        result=await rpc.call('harness.v1.agents.bindSession',{session:args[1],context:installed.context,assignments:args[3]??[],intended:args[4]??null});
+      }finally{channel.close();}
+    }
   }
   else if(op==='readerContext'){
     const identity=await team.agents.get(args[0],options);

package/board/cli.mjs

@@ -7,6 +7,7 @@
 import { taskStatusOf } from './published.mjs';
 import { appendFileSync, copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
 import { spawnSync } from 'node:child_process';
+import { recordSlow, takeSteps } from '@volter/supercode-harness-sdk/slow-log';
 import { basename, join, resolve } from 'node:path';
 import { homedir, tmpdir, userInfo } from 'node:os';
 import { read } from '../orchestration.mjs';
@@ -298,13 +299,13 @@ const metadataOf = (v) => {
 };
 
 /**
- * Every workflow call says when this orchestrator is not the published release: a build no release names (a hand
- * install of a branch) or one older than npm's latest. Action is required until it is installed; the latest is read
- * from the registry at most once an hour.
+ * What is wrong with this orchestrator's version, or null: a build no release names (a hand install of a branch) or one
+ * older than npm's latest. Every workflow call says it and the dispatcher tells its managers; action is required until
+ * the release is installed. The latest is read from the registry at most once an hour.
  */
-function sayReleaseBehind() {
+function releaseBehind() {
   let running;
-  try { running = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version; } catch { return; }
+  try { running = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version; } catch { return null; }
   const cache = join(tmpdir(), 'supercode-orchestrator-latest.json');
   let latest = null;
   let fresh = false;
@@ -315,13 +316,18 @@ function sayReleaseBehind() {
     latest = r.status === 0 ? r.stdout.trim() || null : null;
     try { writeFileSync(cache, JSON.stringify({ at: Date.now(), latest })); } catch { /* read again next call */ }
   }
-  if (!latest || running === latest) return;
+  if (!latest || running === latest) return null;
   const parts = (v) => v.split('-')[0].split('.').map(Number);
   const [a, b] = [parts(running), parts(latest)];
   const older = a[0] - b[0] || a[1] - b[1] || a[2] - b[2];
   const fix = `npm install -g @volter/supercode-orchestrator@${latest} (and each install's pinned copy)`;
-  if (running.includes('-')) process.stderr.write(`board: error: orchestrator ${running} is not a published release (latest ${latest}); action required: ${fix}\n`);
-  else if (older < 0) process.stderr.write(`board: error: orchestrator ${running} is behind the published ${latest}; action required: ${fix}\n`);
+  if (running.includes('-')) return { key: `${running}>${latest}`, text: `orchestrator ${running} is not a published release (latest ${latest}); action required: ${fix}` };
+  if (older < 0) return { key: `${running}>${latest}`, text: `orchestrator ${running} is behind the published ${latest}; action required: ${fix}` };
+  return null;
+}
+function sayReleaseBehind() {
+  const behind = releaseBehind();
+  if (behind) process.stderr.write(`board: error: ${behind.text}\n`);
 }
 
 export async function runWorkflow(argv) {
@@ -1394,7 +1400,17 @@ async function serve(root, args = {}) {
   const FAILED_ROUNDS_ALERT = 5;
   const failing = new Map();
   let round = 0;
+  // A dispatcher that is not the published release is told up the chain, once per version pair: it keeps running, and
+  // its manager hears that action is required (a caller's stderr line alone reached no one).
+  const versionsTold = new Set();
+  const tellReleaseBehind = async () => {
+    const behind = releaseBehind();
+    if (!behind || versionsTold.has(behind.key)) return;
+    // the send's key makes a retry safe; a pair counts as told only once a manager got it
+    if (await alertManagers(root, 'default', `The dispatcher of ${root} runs ${behind.text}`, `release-behind-${behind.key}`) > 0) versionsTold.add(behind.key);
+  };
   const watchRounds = async (reports) => {
+    await tellReleaseBehind().catch(() => {});
     if (reports.every((r) => !r.error)) {
       const temporary = `${heartbeat}.${process.pid}.tmp`;
       try {
@@ -1491,6 +1507,8 @@ async function serve(root, args = {}) {
         log(`tick failed: ${error.message}`);
         await watchRounds([{ board: '*', error: `tick failed: ${error.message}` }]);
       }
+      // each round is a line of its own in the slow log, with the steps it spent its time on
+      recordSlow({ kind: 'call', name: 'dispatch round', ms: Date.now() - began, slowest: null, extra: { steps: takeSteps() } });
       // a newer install is looked for after each tick and through the wait before the next
       const newer = () => { const now = installed(); return running && now && now !== running ? now : null; };
       let moved = newer();

package/board/dispatch.mjs

@@ -143,6 +143,16 @@ const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0,
 // headless worker by its pid, a pane by that machine's pane list. Null only when the machine cannot answer.
 const sessionAlive = (fleet, session) => fleet.native?.has(addressOf(session)) ? fleet.native.get(addressOf(session)).alive : unpublishedAlive(session);
 const paneLists = new Map();
+// A run the agent sync holds names only its machine and native session (`harness:id`), never a pane or pid. Teams'
+// fleet answers for a session it published; one it never published (a gateway worker in its profile's directory,
+// outside every synced root) is read through its machine's native discovery, as a native close reads it.
+function syncRowAlive(fleet, row) {
+  const machine = row.machine || localMachine(), at = row.session_ref.indexOf(':');
+  const session = { machine, harness: row.session_ref.slice(0, at), session_id: row.session_ref.slice(at + 1), address: `sc:${machine}:${row.session_ref}` };
+  const alive = sessionAlive(fleet, session);
+  if (alive !== null) return alive;
+  try { return nativeReading(nativeSessions(machine, session.harness), session).alive; } catch { return null; }
+}
 function unpublishedAlive(session) {
   if (!session) return null;
   if (session.surface === 'headless' && session.pid) return pidAlive(session.pid);
@@ -302,7 +312,10 @@ function supervise(board, fleet, settings, report, entry, effects) {
         }
       };
       const limits = limitsOf(db, board, entry, task, session);
-      if (runRow && (session?.launch_pending || session?.launch_deferred || (!session?.session_id && !session?.pid))) {
+      // A verified adoption binds the run to its session; a launch key still pending beside it is the reconciler's to
+      // retire at its machine, and no longer what the run waits on.
+      const adopted = session?.mode === 'adopt' && Boolean(session.address);
+      if (runRow && ((session?.launch_pending && !adopted) || session?.launch_deferred || (!session?.session_id && !session?.pid))) {
         const age = t - Number(session?.launched_at ?? runRow.started_at);
         if (age >= settings.launchWindow) {
           const reason = session?.launch_deferred_error ?? session?.launch_error ?? 'no session or process was bound to the dispatched run';
@@ -795,13 +808,16 @@ export function closeRequested(board, report, { taskId = null, caller = null } =
         receipt = launchReceipt(board, session);
         if (!receipt) receipt = cancelUnissuedLaunch(board, session);
       } catch (error) { receiptError = error.message; }
-      if (unissuedLaunch(receipt, session) && retireUnissuedLaunch(board, row, session, receipt)) {
+      const retired = unissuedLaunch(receipt, session) && retireUnissuedLaunch(board, row, session, receipt);
+      if (retired === true) {
         (report.closed ??= []).push({ task_id: row.task_id, ...session, receipt });
         continue;
       }
+      // the key was never issued, and the session adopted onto the run is closed (or released) as its own below
+      if (retired === 'released') session = json(board.db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(row.id)?.metadata, {})?.supercode;
       // A receipt that names its pane and reads it not live is terminal enough to settle, with or without a status (a
       // daemon that predates launch statuses answers none): the board's own pane reading below confirms the close
-      if (receipt?.pane && receipt.status !== 'issued' && (receipt.status || typeof receipt.live === 'boolean' || receipt.exit)) { bindLaunch(board, row, session, receipt.pane, session.session_id); session = { ...session, pane: receipt.pane }; }
+      else if (receipt?.pane && receipt.status !== 'issued' && (receipt.status || typeof receipt.live === 'boolean' || receipt.exit)) { bindLaunch(board, row, session, receipt.pane, session.session_id); session = { ...session, pane: receipt.pane }; }
       else {
         const error = `close deferred: ${receiptError ?? `awaiting terminal launch receipt ${session.launch_key} (${receipt?.status ?? 'unknown'})`}`;
         tx(board, (db) => {
@@ -1187,12 +1203,24 @@ function unissuedLaunch(receipt, session) {
     receipt?.outcome === 'not_issued' && receipt?.live === false && receipt?.pane === null;
 }
 
+/**
+ * Retire a launch key its machine confirmed it never issued. Returns true when the run's launch is settled and closed,
+ * 'released' when the key is retired but the run keeps a session adopted onto it, false when the run is not this key's.
+ */
 function retireUnissuedLaunch(board, row, session, receipt, { entry = null, effects = [] } = {}) {
   if (!unissuedLaunch(receipt, session)) return false;
   const result = tx(board, db => {
     const current = db.prepare('SELECT ended_at, metadata FROM task_runs WHERE id=?').get(row.id);
     const saved = json(current?.metadata, {})?.supercode;
-    if (!saved || saved.launch_key !== session.launch_key || saved.pane || saved.address || saved.pid) return { retired: false, effects: [] };
+    if (!saved || saved.launch_key !== session.launch_key) return { retired: false, effects: [] };
+    // A session adopted onto a run whose launch was still pending did not come from that launch (the machine never
+    // issued the key): the key is retired and the adopted session stays the run's, supervised and closed as its own.
+    if (saved.mode === 'adopt' && saved.address && saved.launch_pending) {
+      setRunSession(db, row.id, { launch_pending: false, launch_cancellation: receipt, launch_error: null, launch_key: null });
+      event(db, row.task_id, 'session_launch_cancelled', { key: session.launch_key, machine: session.machine, receipt, kept: saved.address }, row.id);
+      return { retired: 'released', effects: [] };
+    }
+    if (saved.pane || saved.address || saved.pid) return { retired: false, effects: [] };
     if (!current.ended_at && (!entry || getTask(db, row.task_id)?.current_run_id !== row.id)) return { retired: false, effects: [] };
     if (saved.close_confirmed_at) return { retired: true, effects: [] };
     const collected = [];
@@ -1313,13 +1341,16 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
   for(const [org,context] of orgs)for(const row of pendingAgentRuns(board.root,org)){
     const launch=agentRuns.find(run=>String(run.id)===String(row.run_id));
     try{synchronizeAgentRun(board.root,org,row,{context,intended:json(launch?.metadata,{})?.supercode?.agent_profile_intent});}
-    catch(error){if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&sessionAlive(fleet,{address:`sc:${row.machine||localMachine()}:${row.session_ref}`,machine:row.machine||localMachine()})===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
+    catch(error){if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&syncRowAlive(fleet,row)===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
   }
-  const deferred=view(board,db=>db.prepare("SELECT t.id,r.started_at FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
+  const deferred=view(board,db=>db.prepare("SELECT t.id,r.id run_id,COALESCE(json_extract(r.metadata,'$.supercode.launched_at'),r.started_at) launched_at FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
   // A deferred launch was never issued (a reading or door was not yet available): it is the start its manager, creator
-  // or workflow asked for, carried out within the launch window; supervise surfaces it once when the window ends
-  for(const {id,started_at} of deferred) {
-    if (now() - started_at >= settings.launchWindow) continue;
+  // or workflow asked for, carried out within the launch window; supervise surfaces it once when the window ends. The
+  // window counts only time the fleet could be read: a launch waiting on an unreadable fleet was never tried, so its
+  // clock moves with the wait instead of running out (every resumed card failed so in the 22:17Z event-stream outage).
+  for(const {id,run_id,launched_at} of deferred) {
+    if (now() - launched_at >= settings.launchWindow) continue;
+    if (!fleet.ok) { tx(board,db=>setRunSession(db,run_id,{launched_at:now()})); report.queued.push([id,`launch waits for the fleet: ${fleet.error}`]); continue; }
     try {
       const task=getTaskNow(board,id),status=view(board,db=>statusOf(db,task)),spec=runSpec(entry,status,task);
       const profile=spec&&view(board,db=>evaluate(entry.workflow.roles?.[spec.role]??'card.assignee',scopeOf(db,board,task,{params:entry.params})));
@@ -1758,7 +1789,8 @@ function adoptSession(board, id, who, { workspace = null, ttl, assignee = null,
     const session = { surface: found.pane ? 'pane' : 'native', session_id: sessionId, harness, machine: host, pane: found.pane ?? null, address: found.address, mode: 'adopt', launched_at: now(), seen_at: now(),
       ...(agent ? { agent_id: agent.identity.id, agent_org: team.org, agent_context: context ?? null,
         agent_profile_intent: { ...structuredClone(agent.intended.resolved), worker: { ...agent.intended.resolved?.worker, harness: lane.harness, model: lane.model ?? null, cwd: dir ?? task.workspace_path ?? null } } } : {}) };
-    setRunSession(db, runId, { ...(currentRun(db, id)?.metadata?.supercode ?? {}), ...session, slot: currentRun(db, id)?.metadata?.supercode?.slot ?? beforeSession?.slot ?? expected?.session ?? 'worker', close_requested: null, closed_at: null, close_confirmed_at: null, close_error: null, missing_since: null });
+    // a deferred launch was never issued: the adopted session is the run's, and no later round launches another
+    setRunSession(db, runId, { ...(currentRun(db, id)?.metadata?.supercode ?? {}), ...session, launch_deferred: false, slot: currentRun(db, id)?.metadata?.supercode?.slot ?? beforeSession?.slot ?? expected?.session ?? 'worker', close_requested: null, closed_at: null, close_confirmed_at: null, close_error: null, missing_since: null });
     event(db, id, 'session_adopted', { session_id: sessionId, machine: host, pane: session.pane, address: found.address, mode: 'adopt', role: currentRun(db, id)?.metadata?.supercode?.role }, runId);
     subscribeIn(db, id, found.address, 'worker');
     for (const p of db.prepare('SELECT parent_id FROM task_links WHERE child_id = ?').all(id)) subscribeIn(db, p.parent_id, found.address, 'dependency');

package/board/event-fleet.mjs

@@ -2,10 +2,10 @@
 import { readFileSync, writeFileSync, renameSync } from 'node:fs';
 import { join } from 'node:path';
 import { randomUUID } from 'node:crypto';
-import { eventClient } from '@volter/teams/event-client';
+import { eventClient, retryAfterMs } from '@volter/teams/event-client';
 const pathOf=root=>join(root,'.session-events.json');
 const normalize=value=>String(value??'').toLowerCase().replace(/[^a-z0-9-]/g,'-').replace(/-+/g,'-').replace(/^-|-$/g,'');
-const wait=signal=>new Promise(resolve=>{const timer=setTimeout(resolve,5000);signal.addEventListener('abort',()=>{clearTimeout(timer);resolve();},{once:true});});
+const wait=(signal,ms)=>new Promise(resolve=>{const timer=setTimeout(resolve,ms);signal.addEventListener('abort',()=>{clearTimeout(timer);resolve();},{once:true});});
 
 export function followFleet({root,context,supercodeHome,wake,log}){
   const abort=new AbortController();let resources={},cursor=null,connected=false,snapshot=null;
@@ -20,20 +20,27 @@ export function followFleet({root,context,supercodeHome,wake,log}){
     target[key]={sequence:frame.sequence,data:event.data};return true;
   };
   void(async()=>{
+    // Refused for load, the follower waits out the server's retry_after, doubling while it is refused (to 5 min), and
+    // keeps its cursor: asking every 5 s spent the slot it waited for on the next refusal.
+    let refused=0;
     while(!abort.signal.aborted){
+      let pause=5000;
       try{
         const {team}=eventClient({context,supercodeHome});
         // Every connection begins with a complete, ACL-filtered snapshot. A process restart cannot trust an old cache.
         for await(const frame of team.events.follow({kinds:['session.started','session.ended','session.changed','resource.changed']}, {signal:abort.signal,after:cursor})){
           if(frame.type==='snapshot_begin'){snapshot={};connected=false;persist();}
           else if(frame.type==='snapshot_item')apply(snapshot??resources,frame);
-          else if(frame.type==='snapshot_end'){resources=snapshot??resources;snapshot=null;cursor=frame.cursor;connected=true;persist();wake('session-events-snapshot');}
+          else if(frame.type==='snapshot_end'){resources=snapshot??resources;snapshot=null;cursor=frame.cursor;connected=true;refused=0;persist();wake('session-events-snapshot');}
           else if(frame.type==='event'){const changed=apply(resources,frame);cursor=frame.cursor;connected=true;persist();if(changed)wake('session-events');}
-          else if(frame.type==='checkpoint'){const recovered=!connected;cursor=frame.cursor;connected=true;persist();if(recovered)wake('session-events-reconnected');}
+          else if(frame.type==='checkpoint'){const recovered=!connected;cursor=frame.cursor;connected=true;refused=0;persist();if(recovered)wake('session-events-reconnected');}
           else if(frame.terminal){cursor=null;connected=false;persist();wake('session-events-unavailable');break;}
         }
-      }catch(error){if(!abort.signal.aborted)log(`dispatch session events unavailable: ${error.code??error.message}`);if(error.status===409)cursor=null;}
-      connected=false;persist();wake('session-events-unavailable');if(!abort.signal.aborted)await wait(abort.signal);
+      }catch(error){
+        const asked=retryAfterMs(error);if(asked){pause=Math.min(300000,asked*2**refused);refused++;}
+        if(!abort.signal.aborted)log(`dispatch session events unavailable: ${error.code??error.message}${asked?` (retrying in ${Math.round(pause/1000)} s)`:''}`);if(error.status===409)cursor=null;
+      }
+      connected=false;persist();wake('session-events-unavailable');if(!abort.signal.aborted)await wait(abort.signal,pause);
     }
   })();
   return {close(){abort.abort();connected=false;persist();}};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.25",
+  "version": "0.5.27",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {
@@ -24,8 +24,8 @@
     "check-types": "node scripts/generate-types.mjs --check"
   },
   "dependencies": {
-    "@volter/supercode-harness-sdk": "^0.3.58",
-    "@volter/teams": "0.2.11",
+    "@volter/supercode-harness-sdk": "^0.3.60",
+    "@volter/teams": "0.2.12",
     "@volter/ztrack": "^2.4.23",
     "yaml": "^2.9.0"
   },