@volter/supercode-orchestrator 0.5.24 → 0.5.26

package/board/cli.mjs

@@ -7,6 +7,7 @@
 import { taskStatusOf } from './published.mjs';
 import { appendFileSync, copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
 import { spawnSync } from 'node:child_process';
+import { recordSlow, takeSteps } from '@volter/supercode-harness-sdk/slow-log';
 import { basename, join, resolve } from 'node:path';
 import { homedir, tmpdir, userInfo } from 'node:os';
 import { read } from '../orchestration.mjs';
@@ -298,13 +299,13 @@ const metadataOf = (v) => {
 };
 
 /**
- * Every workflow call says when this orchestrator is not the published release: a build no release names (a hand
- * install of a branch) or one older than npm's latest. Action is required until it is installed; the latest is read
- * from the registry at most once an hour.
+ * What is wrong with this orchestrator's version, or null: a build no release names (a hand install of a branch) or one
+ * older than npm's latest. Every workflow call says it and the dispatcher tells its managers; action is required until
+ * the release is installed. The latest is read from the registry at most once an hour.
  */
-function sayReleaseBehind() {
+function releaseBehind() {
   let running;
-  try { running = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version; } catch { return; }
+  try { running = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version; } catch { return null; }
   const cache = join(tmpdir(), 'supercode-orchestrator-latest.json');
   let latest = null;
   let fresh = false;
@@ -315,13 +316,18 @@ function sayReleaseBehind() {
     latest = r.status === 0 ? r.stdout.trim() || null : null;
     try { writeFileSync(cache, JSON.stringify({ at: Date.now(), latest })); } catch { /* read again next call */ }
   }
-  if (!latest || running === latest) return;
+  if (!latest || running === latest) return null;
   const parts = (v) => v.split('-')[0].split('.').map(Number);
   const [a, b] = [parts(running), parts(latest)];
   const older = a[0] - b[0] || a[1] - b[1] || a[2] - b[2];
   const fix = `npm install -g @volter/supercode-orchestrator@${latest} (and each install's pinned copy)`;
-  if (running.includes('-')) process.stderr.write(`board: error: orchestrator ${running} is not a published release (latest ${latest}); action required: ${fix}\n`);
-  else if (older < 0) process.stderr.write(`board: error: orchestrator ${running} is behind the published ${latest}; action required: ${fix}\n`);
+  if (running.includes('-')) return { key: `${running}>${latest}`, text: `orchestrator ${running} is not a published release (latest ${latest}); action required: ${fix}` };
+  if (older < 0) return { key: `${running}>${latest}`, text: `orchestrator ${running} is behind the published ${latest}; action required: ${fix}` };
+  return null;
+}
+function sayReleaseBehind() {
+  const behind = releaseBehind();
+  if (behind) process.stderr.write(`board: error: ${behind.text}\n`);
 }
 
 export async function runWorkflow(argv) {
@@ -1394,7 +1400,17 @@ async function serve(root, args = {}) {
   const FAILED_ROUNDS_ALERT = 5;
   const failing = new Map();
   let round = 0;
+  // A dispatcher that is not the published release is told up the chain, once per version pair: it keeps running, and
+  // its manager hears that action is required (a caller's stderr line alone reached no one).
+  const versionsTold = new Set();
+  const tellReleaseBehind = async () => {
+    const behind = releaseBehind();
+    if (!behind || versionsTold.has(behind.key)) return;
+    // the send's key makes a retry safe; a pair counts as told only once a manager got it
+    if (await alertManagers(root, 'default', `The dispatcher of ${root} runs ${behind.text}`, `release-behind-${behind.key}`) > 0) versionsTold.add(behind.key);
+  };
   const watchRounds = async (reports) => {
+    await tellReleaseBehind().catch(() => {});
     if (reports.every((r) => !r.error)) {
       const temporary = `${heartbeat}.${process.pid}.tmp`;
       try {
@@ -1402,7 +1418,10 @@ async function serve(root, args = {}) {
         renameSync(temporary, heartbeat);
       } catch { /* the next round writes it */ }
     }
-    for (const r of reports) {
+    // A whole tick that failed is counted as board '*'; a round in which every board succeeded clears it too, so its
+    // recovery is said and a later failure starts a count of its own.
+    const settled = reports.every((r) => !r.error) && failing.has('*') ? [...reports, { board: '*' }] : reports;
+    for (const r of settled) {
       const run = failing.get(r.board) ?? { count: 0, since: Date.now(), told: 0 };
       if (!r.error) {
         if (run.count >= FAILED_ROUNDS_ALERT) await alertManagers(root, r.board === '*' ? 'default' : r.board, `The dispatcher's rounds on board ${r.board} succeed again after ${run.count} failed rounds since ${new Date(run.since).toISOString()}.`, `dispatcher-recovered-${r.board}-${run.since}`);
@@ -1488,6 +1507,8 @@ async function serve(root, args = {}) {
         log(`tick failed: ${error.message}`);
         await watchRounds([{ board: '*', error: `tick failed: ${error.message}` }]);
       }
+      // each round is a line of its own in the slow log, with the steps it spent its time on
+      recordSlow({ kind: 'call', name: 'dispatch round', ms: Date.now() - began, slowest: null, extra: { steps: takeSteps() } });
       // a newer install is looked for after each tick and through the wait before the next
       const newer = () => { const now = installed(); return running && now && now !== running ? now : null; };
       let moved = newer();

package/board/dispatch.mjs

@@ -1074,11 +1074,22 @@ export async function mergeReviewed(board, e) {
     return true;
   }
   const entry = await loadWorkflow(board.root);
-  tx(board, (db) => {
+  // A fault other than the card's refusal keeps the effect for the next round, and the round goes on past it.
+  try { tx(board, (db) => {
     event(db, e.id, 'reviewed_merged', { merges });
-    const r = fire(db, board, entry, e.id, 'reviewed_merged', { roles: ['platform'], payload: { merges, summary: merges.length ? `Merged at the reviewed heads: ${merges.map((m) => `${m.pr} (${m.reviewed.slice(0, 8)} → ${String(m.merge).slice(0, 8)})`).join(', ')}` : 'The pass named no PR to merge.' } });
+    // A card that moved after its pass cannot take the event: that is said to the manager once, never thrown, so the
+    // kept effect does not stop every round after it (as merge_refused, above).
+    let r;
+    try { r = fire(db, board, entry, e.id, 'reviewed_merged', { roles: ['platform'], payload: { merges, summary: merges.length ? `Merged at the reviewed heads: ${merges.map((m) => `${m.pr} (${m.reviewed.slice(0, 8)} → ${String(m.merge).slice(0, 8)})`).join(', ')}` : 'The pass named no PR to merge.' } }); }
+    catch (error) {
+      // only the card's refusal is said and consumed; anything else (a write contention, a fault) is thrown and retried
+      if (!error.noTransition && !error.refusal && !/does not apply|there is for/.test(error.message)) throw error;
+      const to = [].concat(entry.params.managers ?? []).find(Boolean);
+      if (to) event(db, e.id, 'board_message', { to, text: `${e.id}: its reviewed PRs are merged (${merges.map((m) => m.pr).join(', ') || 'none'}), but the card could not take reviewed_merged: ${String(error.message).slice(0, 300)}` }, null);
+      return;
+    }
     for (const eff of r.effects) db.prepare('INSERT INTO supercode_effects (effect) VALUES (?)').run(JSON.stringify(eff));
-  });
+  }); } catch (error) { process.stderr.write(`board: ${e.id}: reviewed_merged kept for the next round: ${String(error.message).slice(0, 300)}\n`); return false; }
   return true;
 }
 
@@ -1304,11 +1315,14 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
     try{synchronizeAgentRun(board.root,org,row,{context,intended:json(launch?.metadata,{})?.supercode?.agent_profile_intent});}
     catch(error){if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&sessionAlive(fleet,{address:`sc:${row.machine||localMachine()}:${row.session_ref}`,machine:row.machine||localMachine()})===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
   }
-  const deferred=view(board,db=>db.prepare("SELECT t.id,r.started_at FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
+  const deferred=view(board,db=>db.prepare("SELECT t.id,r.id run_id,COALESCE(json_extract(r.metadata,'$.supercode.launched_at'),r.started_at) launched_at FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
   // A deferred launch was never issued (a reading or door was not yet available): it is the start its manager, creator
-  // or workflow asked for, carried out within the launch window; supervise surfaces it once when the window ends
-  for(const {id,started_at} of deferred) {
-    if (now() - started_at >= settings.launchWindow) continue;
+  // or workflow asked for, carried out within the launch window; supervise surfaces it once when the window ends. The
+  // window counts only time the fleet could be read: a launch waiting on an unreadable fleet was never tried, so its
+  // clock moves with the wait instead of running out (every resumed card failed so in the 22:17Z event-stream outage).
+  for(const {id,run_id,launched_at} of deferred) {
+    if (now() - launched_at >= settings.launchWindow) continue;
+    if (!fleet.ok) { tx(board,db=>setRunSession(db,run_id,{launched_at:now()})); report.queued.push([id,`launch waits for the fleet: ${fleet.error}`]); continue; }
     try {
       const task=getTaskNow(board,id),status=view(board,db=>statusOf(db,task)),spec=runSpec(entry,status,task);
       const profile=spec&&view(board,db=>evaluate(entry.workflow.roles?.[spec.role]??'card.assignee',scopeOf(db,board,task,{params:entry.params})));

package/board/event-fleet.mjs

@@ -2,10 +2,10 @@
 import { readFileSync, writeFileSync, renameSync } from 'node:fs';
 import { join } from 'node:path';
 import { randomUUID } from 'node:crypto';
-import { eventClient } from '@volter/teams/event-client';
+import { eventClient, retryAfterMs } from '@volter/teams/event-client';
 const pathOf=root=>join(root,'.session-events.json');
 const normalize=value=>String(value??'').toLowerCase().replace(/[^a-z0-9-]/g,'-').replace(/-+/g,'-').replace(/^-|-$/g,'');
-const wait=signal=>new Promise(resolve=>{const timer=setTimeout(resolve,5000);signal.addEventListener('abort',()=>{clearTimeout(timer);resolve();},{once:true});});
+const wait=(signal,ms)=>new Promise(resolve=>{const timer=setTimeout(resolve,ms);signal.addEventListener('abort',()=>{clearTimeout(timer);resolve();},{once:true});});
 
 export function followFleet({root,context,supercodeHome,wake,log}){
   const abort=new AbortController();let resources={},cursor=null,connected=false,snapshot=null;
@@ -20,20 +20,27 @@ export function followFleet({root,context,supercodeHome,wake,log}){
     target[key]={sequence:frame.sequence,data:event.data};return true;
   };
   void(async()=>{
+    // Refused for load, the follower waits out the server's retry_after, doubling while it is refused (to 5 min), and
+    // keeps its cursor: asking every 5 s spent the slot it waited for on the next refusal.
+    let refused=0;
     while(!abort.signal.aborted){
+      let pause=5000;
       try{
         const {team}=eventClient({context,supercodeHome});
         // Every connection begins with a complete, ACL-filtered snapshot. A process restart cannot trust an old cache.
         for await(const frame of team.events.follow({kinds:['session.started','session.ended','session.changed','resource.changed']}, {signal:abort.signal,after:cursor})){
           if(frame.type==='snapshot_begin'){snapshot={};connected=false;persist();}
           else if(frame.type==='snapshot_item')apply(snapshot??resources,frame);
-          else if(frame.type==='snapshot_end'){resources=snapshot??resources;snapshot=null;cursor=frame.cursor;connected=true;persist();wake('session-events-snapshot');}
+          else if(frame.type==='snapshot_end'){resources=snapshot??resources;snapshot=null;cursor=frame.cursor;connected=true;refused=0;persist();wake('session-events-snapshot');}
           else if(frame.type==='event'){const changed=apply(resources,frame);cursor=frame.cursor;connected=true;persist();if(changed)wake('session-events');}
-          else if(frame.type==='checkpoint'){const recovered=!connected;cursor=frame.cursor;connected=true;persist();if(recovered)wake('session-events-reconnected');}
+          else if(frame.type==='checkpoint'){const recovered=!connected;cursor=frame.cursor;connected=true;refused=0;persist();if(recovered)wake('session-events-reconnected');}
           else if(frame.terminal){cursor=null;connected=false;persist();wake('session-events-unavailable');break;}
         }
-      }catch(error){if(!abort.signal.aborted)log(`dispatch session events unavailable: ${error.code??error.message}`);if(error.status===409)cursor=null;}
-      connected=false;persist();wake('session-events-unavailable');if(!abort.signal.aborted)await wait(abort.signal);
+      }catch(error){
+        const asked=retryAfterMs(error);if(asked){pause=Math.min(300000,asked*2**refused);refused++;}
+        if(!abort.signal.aborted)log(`dispatch session events unavailable: ${error.code??error.message}${asked?` (retrying in ${Math.round(pause/1000)} s)`:''}`);if(error.status===409)cursor=null;
+      }
+      connected=false;persist();wake('session-events-unavailable');if(!abort.signal.aborted)await wait(abort.signal,pause);
     }
   })();
   return {close(){abort.abort();connected=false;persist();}};

package/board/store.mjs

@@ -9,6 +9,7 @@
 // This module opens a board in its backing and gives its callers one primitive, a write transaction; the transitions
 // are in `engine.mjs`. Hermes's own tools may write a `kanban.db` at the same time: every write there takes SQLite's
 // native transaction boundary and retries an optimistic conflict.
+import { noteStep } from '@volter/supercode-harness-sdk/slow-log';
 import { randomBytes } from 'node:crypto';
 import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';
 import { hostname } from 'node:os';
@@ -200,9 +201,15 @@ export function openBoard(root, slug = DEFAULT_BOARD, { create = false, backing
  * A read inside the transaction is the state it writes over.
  */
 export const tx = (board, fn) => {
-  const result = board.tx(db => {const result=fn(db);recordCardPublications(db,actor);return result;});
-  signalBoardWrite(board.root, actor);
-  return result;
+  // Each transaction is a step named by the function that asked for it, so a round's slow-log line says which of its
+  // steps make its transactions (about a hundred a round, at 150 ms each, on 2026-10-02).
+  const caller = /at (?:async )?([\w.$<>]+) /.exec(String(new Error().stack).split('\n')[2] ?? '')?.[1] ?? 'unknown';
+  const started = performance.now();
+  try {
+    const result = board.tx(db => {const result=fn(db);recordCardPublications(db,actor);return result;});
+    signalBoardWrite(board.root, actor);
+    return result;
+  } finally { noteStep(`board tx from ${caller}`, performance.now() - started); }
 };
 // SQLite's own contention (BUSY, LOCKED, and their extended codes) and nothing else: a refusal whose words happen to
 // say "blocked" or "busy" reaches its caller at once instead of being retried forever.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.24",
+  "version": "0.5.26",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {
@@ -24,8 +24,8 @@
     "check-types": "node scripts/generate-types.mjs --check"
   },
   "dependencies": {
-    "@volter/supercode-harness-sdk": "^0.3.58",
-    "@volter/teams": "0.2.11",
+    "@volter/supercode-harness-sdk": "^0.3.60",
+    "@volter/teams": "0.2.12",
     "@volter/ztrack": "^2.4.23",
     "yaml": "^2.9.0"
   },