@volter/supercode-orchestrator 0.5.25 → 0.5.26

package/board/cli.mjs

@@ -7,6 +7,7 @@
 import { taskStatusOf } from './published.mjs';
 import { appendFileSync, copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
 import { spawnSync } from 'node:child_process';
+import { recordSlow, takeSteps } from '@volter/supercode-harness-sdk/slow-log';
 import { basename, join, resolve } from 'node:path';
 import { homedir, tmpdir, userInfo } from 'node:os';
 import { read } from '../orchestration.mjs';
@@ -298,13 +299,13 @@ const metadataOf = (v) => {
 };
 
 /**
- * Every workflow call says when this orchestrator is not the published release: a build no release names (a hand
- * install of a branch) or one older than npm's latest. Action is required until it is installed; the latest is read
- * from the registry at most once an hour.
+ * What is wrong with this orchestrator's version, or null: a build no release names (a hand install of a branch) or one
+ * older than npm's latest. Every workflow call says it and the dispatcher tells its managers; action is required until
+ * the release is installed. The latest is read from the registry at most once an hour.
  */
-function sayReleaseBehind() {
+function releaseBehind() {
   let running;
-  try { running = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version; } catch { return; }
+  try { running = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version; } catch { return null; }
   const cache = join(tmpdir(), 'supercode-orchestrator-latest.json');
   let latest = null;
   let fresh = false;
@@ -315,13 +316,18 @@ function sayReleaseBehind() {
     latest = r.status === 0 ? r.stdout.trim() || null : null;
     try { writeFileSync(cache, JSON.stringify({ at: Date.now(), latest })); } catch { /* read again next call */ }
   }
-  if (!latest || running === latest) return;
+  if (!latest || running === latest) return null;
   const parts = (v) => v.split('-')[0].split('.').map(Number);
   const [a, b] = [parts(running), parts(latest)];
   const older = a[0] - b[0] || a[1] - b[1] || a[2] - b[2];
   const fix = `npm install -g @volter/supercode-orchestrator@${latest} (and each install's pinned copy)`;
-  if (running.includes('-')) process.stderr.write(`board: error: orchestrator ${running} is not a published release (latest ${latest}); action required: ${fix}\n`);
-  else if (older < 0) process.stderr.write(`board: error: orchestrator ${running} is behind the published ${latest}; action required: ${fix}\n`);
+  if (running.includes('-')) return { key: `${running}>${latest}`, text: `orchestrator ${running} is not a published release (latest ${latest}); action required: ${fix}` };
+  if (older < 0) return { key: `${running}>${latest}`, text: `orchestrator ${running} is behind the published ${latest}; action required: ${fix}` };
+  return null;
+}
+function sayReleaseBehind() {
+  const behind = releaseBehind();
+  if (behind) process.stderr.write(`board: error: ${behind.text}\n`);
 }
 
 export async function runWorkflow(argv) {
@@ -1394,7 +1400,17 @@ async function serve(root, args = {}) {
   const FAILED_ROUNDS_ALERT = 5;
   const failing = new Map();
   let round = 0;
+  // A dispatcher that is not the published release is told up the chain, once per version pair: it keeps running, and
+  // its manager hears that action is required (a caller's stderr line alone reached no one).
+  const versionsTold = new Set();
+  const tellReleaseBehind = async () => {
+    const behind = releaseBehind();
+    if (!behind || versionsTold.has(behind.key)) return;
+    // the send's key makes a retry safe; a pair counts as told only once a manager got it
+    if (await alertManagers(root, 'default', `The dispatcher of ${root} runs ${behind.text}`, `release-behind-${behind.key}`) > 0) versionsTold.add(behind.key);
+  };
   const watchRounds = async (reports) => {
+    await tellReleaseBehind().catch(() => {});
     if (reports.every((r) => !r.error)) {
       const temporary = `${heartbeat}.${process.pid}.tmp`;
       try {
@@ -1491,6 +1507,8 @@ async function serve(root, args = {}) {
         log(`tick failed: ${error.message}`);
         await watchRounds([{ board: '*', error: `tick failed: ${error.message}` }]);
       }
+      // each round is a line of its own in the slow log, with the steps it spent its time on
+      recordSlow({ kind: 'call', name: 'dispatch round', ms: Date.now() - began, slowest: null, extra: { steps: takeSteps() } });
       // a newer install is looked for after each tick and through the wait before the next
       const newer = () => { const now = installed(); return running && now && now !== running ? now : null; };
       let moved = newer();

package/board/dispatch.mjs

@@ -1315,11 +1315,14 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
     try{synchronizeAgentRun(board.root,org,row,{context,intended:json(launch?.metadata,{})?.supercode?.agent_profile_intent});}
     catch(error){if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&sessionAlive(fleet,{address:`sc:${row.machine||localMachine()}:${row.session_ref}`,machine:row.machine||localMachine()})===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
   }
-  const deferred=view(board,db=>db.prepare("SELECT t.id,r.started_at FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
+  const deferred=view(board,db=>db.prepare("SELECT t.id,r.id run_id,COALESCE(json_extract(r.metadata,'$.supercode.launched_at'),r.started_at) launched_at FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
   // A deferred launch was never issued (a reading or door was not yet available): it is the start its manager, creator
-  // or workflow asked for, carried out within the launch window; supervise surfaces it once when the window ends
-  for(const {id,started_at} of deferred) {
-    if (now() - started_at >= settings.launchWindow) continue;
+  // or workflow asked for, carried out within the launch window; supervise surfaces it once when the window ends. The
+  // window counts only time the fleet could be read: a launch waiting on an unreadable fleet was never tried, so its
+  // clock moves with the wait instead of running out (every resumed card failed so in the 22:17Z event-stream outage).
+  for(const {id,run_id,launched_at} of deferred) {
+    if (now() - launched_at >= settings.launchWindow) continue;
+    if (!fleet.ok) { tx(board,db=>setRunSession(db,run_id,{launched_at:now()})); report.queued.push([id,`launch waits for the fleet: ${fleet.error}`]); continue; }
     try {
       const task=getTaskNow(board,id),status=view(board,db=>statusOf(db,task)),spec=runSpec(entry,status,task);
       const profile=spec&&view(board,db=>evaluate(entry.workflow.roles?.[spec.role]??'card.assignee',scopeOf(db,board,task,{params:entry.params})));

package/board/event-fleet.mjs

@@ -2,10 +2,10 @@
 import { readFileSync, writeFileSync, renameSync } from 'node:fs';
 import { join } from 'node:path';
 import { randomUUID } from 'node:crypto';
-import { eventClient } from '@volter/teams/event-client';
+import { eventClient, retryAfterMs } from '@volter/teams/event-client';
 const pathOf=root=>join(root,'.session-events.json');
 const normalize=value=>String(value??'').toLowerCase().replace(/[^a-z0-9-]/g,'-').replace(/-+/g,'-').replace(/^-|-$/g,'');
-const wait=signal=>new Promise(resolve=>{const timer=setTimeout(resolve,5000);signal.addEventListener('abort',()=>{clearTimeout(timer);resolve();},{once:true});});
+const wait=(signal,ms)=>new Promise(resolve=>{const timer=setTimeout(resolve,ms);signal.addEventListener('abort',()=>{clearTimeout(timer);resolve();},{once:true});});
 
 export function followFleet({root,context,supercodeHome,wake,log}){
   const abort=new AbortController();let resources={},cursor=null,connected=false,snapshot=null;
@@ -20,20 +20,27 @@ export function followFleet({root,context,supercodeHome,wake,log}){
     target[key]={sequence:frame.sequence,data:event.data};return true;
   };
   void(async()=>{
+    // Refused for load, the follower waits out the server's retry_after, doubling while it is refused (to 5 min), and
+    // keeps its cursor: asking every 5 s spent the slot it waited for on the next refusal.
+    let refused=0;
     while(!abort.signal.aborted){
+      let pause=5000;
       try{
         const {team}=eventClient({context,supercodeHome});
         // Every connection begins with a complete, ACL-filtered snapshot. A process restart cannot trust an old cache.
         for await(const frame of team.events.follow({kinds:['session.started','session.ended','session.changed','resource.changed']}, {signal:abort.signal,after:cursor})){
           if(frame.type==='snapshot_begin'){snapshot={};connected=false;persist();}
           else if(frame.type==='snapshot_item')apply(snapshot??resources,frame);
-          else if(frame.type==='snapshot_end'){resources=snapshot??resources;snapshot=null;cursor=frame.cursor;connected=true;persist();wake('session-events-snapshot');}
+          else if(frame.type==='snapshot_end'){resources=snapshot??resources;snapshot=null;cursor=frame.cursor;connected=true;refused=0;persist();wake('session-events-snapshot');}
           else if(frame.type==='event'){const changed=apply(resources,frame);cursor=frame.cursor;connected=true;persist();if(changed)wake('session-events');}
-          else if(frame.type==='checkpoint'){const recovered=!connected;cursor=frame.cursor;connected=true;persist();if(recovered)wake('session-events-reconnected');}
+          else if(frame.type==='checkpoint'){const recovered=!connected;cursor=frame.cursor;connected=true;refused=0;persist();if(recovered)wake('session-events-reconnected');}
           else if(frame.terminal){cursor=null;connected=false;persist();wake('session-events-unavailable');break;}
         }
-      }catch(error){if(!abort.signal.aborted)log(`dispatch session events unavailable: ${error.code??error.message}`);if(error.status===409)cursor=null;}
-      connected=false;persist();wake('session-events-unavailable');if(!abort.signal.aborted)await wait(abort.signal);
+      }catch(error){
+        const asked=retryAfterMs(error);if(asked){pause=Math.min(300000,asked*2**refused);refused++;}
+        if(!abort.signal.aborted)log(`dispatch session events unavailable: ${error.code??error.message}${asked?` (retrying in ${Math.round(pause/1000)} s)`:''}`);if(error.status===409)cursor=null;
+      }
+      connected=false;persist();wake('session-events-unavailable');if(!abort.signal.aborted)await wait(abort.signal,pause);
     }
   })();
   return {close(){abort.abort();connected=false;persist();}};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.25",
+  "version": "0.5.26",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {
@@ -24,8 +24,8 @@
     "check-types": "node scripts/generate-types.mjs --check"
   },
   "dependencies": {
-    "@volter/supercode-harness-sdk": "^0.3.58",
-    "@volter/teams": "0.2.11",
+    "@volter/supercode-harness-sdk": "^0.3.60",
+    "@volter/teams": "0.2.12",
     "@volter/ztrack": "^2.4.23",
     "yaml": "^2.9.0"
   },