@volter/supercode-orchestrator 0.5.72 → 0.5.74

package/board/cli.mjs

@@ -14,9 +14,10 @@ import { taskStatusOf } from './published.mjs';
 import { createServer } from 'node:net';
 import { chmodSync, existsSync, mkdirSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, copyFileSync, readFileSync, appendFileSync, writeFileSync } from 'node:fs';
 import { notRuntime, ownHome } from './owner-files.mjs';
-import { spawnSync } from 'node:child_process';
+import { execFile, spawnSync } from 'node:child_process';
 import { holdLock } from './lock-holder.mjs';
 import { takeEpoch } from './fence.mjs';
+import { dispatcherNotice, holdOwnerHealth, holdRound, holdSwitch } from './owner-health.mjs';
 import { boardTool } from './tools.mjs';
 import { recordSlow, takeSteps } from '@volter/supercode-harness-sdk/slow-log';
 import { basename, dirname, join, resolve } from 'node:path';
@@ -27,19 +28,23 @@ import { isLocalWorkspace } from './workspace.mjs';
 import { openZtrackBoard } from './ztrack.mjs';
 import * as engine from './engine.mjs';
 
+// A mailbox read through the mailbox's own command, awaited (never a wait that holds the event loop). Gap A31: a
+// helper process until the daemon owns mail (order 3 PR 4).
+const mailRead = (verb) => new Promise((resolve) => execFile(supercodeBin(), ['message', verb, '--json'], { encoding: 'utf8', env: commandEnv(), timeout: 15_000, maxBuffer: 64 * 1024 * 1024 }, (_, stdout) => resolve(stdout ?? '')));
+
 /** The messages on this machine the mailbox still has to deliver (`supercode message waiting`), by message id. */
-function mailWaiting() {
-  const r = spawnSync(supercodeBin(), ['message', 'waiting', '--json'], { encoding: 'utf8', env: commandEnv(), timeout: 15_000 });
-  try { return new Map((JSON.parse(r.stdout) ?? []).map((mail) => [mail.id, mail])); } catch { return new Map(); }
+async function mailWaiting() {
+  const stdout = await mailRead('waiting');
+  try { return new Map((JSON.parse(stdout) ?? []).map((mail) => [mail.id, mail])); } catch { return new Map(); }
 }
 
 /**
  * The messages on this machine whose delivery the mailbox gave up on (`supercode message expired`), by message id:
  * filed by the board, never delivered within the mailbox's bound, kept unread. Empty when the mailbox does not answer.
  */
-function mailExpiries() {
-  const r = spawnSync(supercodeBin(), ['message', 'expired', '--json'], { encoding: 'utf8', env: commandEnv(), timeout: 15_000 });
-  try { return new Map((JSON.parse(r.stdout) ?? []).map((expiry) => [expiry.id, expiry])); } catch { return new Map(); }
+async function mailExpiries() {
+  const stdout = await mailRead('expired');
+  try { return new Map((JSON.parse(stdout) ?? []).map((expiry) => [expiry.id, expiry])); } catch { return new Map(); }
 }
 
 /**
@@ -318,34 +323,6 @@ const metadataOf = (v) => {
   return got;
 };
 
-/**
- * What is wrong with this orchestrator's version, or null: a build no release names (a hand install of a branch) or one
- * older than npm's latest. Every workflow call says it and the dispatcher tells its managers; action is required until
- * the release is installed. The latest is read from the registry at most once an hour.
- */
-function releaseBehind() {
-  let running;
-  try { running = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version; } catch { return null; }
-  const cache = join(tmpdir(), 'supercode-orchestrator-latest.json');
-  let latest = null;
-  let fresh = false;
-  try { const seen = JSON.parse(readFileSync(cache, 'utf8')); if (Date.now() - seen.at < 3_600_000) { fresh = true; latest = seen.latest; } } catch { /* read below */ }
-  if (!fresh) {
-    // a failed read is kept for the hour too: offline, no call waits on the registry more than once an hour
-    const r = spawnSync('npm', ['view', '@volter/supercode-orchestrator', 'version'], { encoding: 'utf8', timeout: 5000 });
-    latest = r.status === 0 ? r.stdout.trim() || null : null;
-    try { writeFileSync(cache, JSON.stringify({ at: Date.now(), latest })); } catch { /* read again next call */ }
-  }
-  if (!latest || running === latest) return null;
-  const parts = (v) => v.split('-')[0].split('.').map(Number);
-  const [a, b] = [parts(running), parts(latest)];
-  const older = a[0] - b[0] || a[1] - b[1] || a[2] - b[2];
-  const fix = `npm install -g @volter/supercode-orchestrator@${latest} (and each install's pinned copy)`;
-  if (running.includes('-')) return { key: `${running}>${latest}`, text: `orchestrator ${running} is not a published release (latest ${latest}); action required: ${fix}` };
-  if (older < 0) return { key: `${running}>${latest}`, text: `orchestrator ${running} is behind the published ${latest}; action required: ${fix}` };
-  return null;
-}
-
 /** One verb in full, in the call it runs in (call.mjs): a caller's in the owner, or the process's own here. */
 export async function runVerb(argv) {
   await knowLocalMachine();
@@ -361,7 +338,7 @@ export async function runVerb(argv) {
   const slug = viewTag ? DEFAULT_BOARD : named;
   const open = (create = false) => { const b = openBoard(root, slug, { create, document: args.document ?? null, backing: create ? args.backing ?? null : null }); b.label = boardLabel(root, slug); return b; };
   const withBoard = (fn, create = false) => { const b = open(create); try { return fn(b); } finally { b.close(); } };
-  const withBoardAsync = async (fn) => { const b = open(); try { return await fn(b); } finally { b.close(); } };
+  const withBoardAsync = async (fn, create = false) => { const b = open(create); try { return await fn(b); } finally { b.close(); } };
   const id = rest[0];
   const need = (n, what) => { if (rest.length < n) throw new Error(`${verb} needs ${what}`); };
   const ids = (first = rest) => [...first, ...(args.ids ?? [])];
@@ -374,10 +351,12 @@ export async function runVerb(argv) {
   current().recorded = sid && boardExists(root, slug) ? withBoard((b) => b.db.prepare("SELECT json_extract(metadata, '$.supercode.address') AS address FROM task_runs WHERE json_extract(metadata, '$.supercode.session_id') = ? ORDER BY id DESC LIMIT 1").get(sid)?.address) ?? null : null;
   if (sid) setActor(callerAddress());
   setPrincipal(await callerPrincipal(root, sid, { teams: !sid && !READS.has(verb) }));
+  // the dispatcher as the owner holds it, said once per call (owner-health.mjs; nothing is read for it)
+  if (!current().said.has(`dispatcher:${slug}`)) { current().said.add(`dispatcher:${slug}`); for (const line of dispatcherNotice(slug)) current().err(`${line}\n`); }
   if (sid && !READS.has(verb) && boardExists(root, slug)) {
     // binding the caller's launch is a write: only the owner, answering the caller's own verb, makes it (a local read
     // never writes)
-    withBoard((b) => { if (isCallersCall()) reconcileCallerLaunch(b, sid); const own = ownCard(b.db, sid); if (own) authorize(b.db, verb, rest, args, own); });
+    await withBoardAsync(async (b) => { if (isCallersCall()) await reconcileCallerLaunch(b, sid); const own = ownCard(b.db, sid); if (own) authorize(b.db, verb, rest, args, own); });
   }
   // the board's workflow (the IR's): what each verb does, and who may send it, is its
   // every verb reads the board through its workflow (a list settles cards too), so every verb loads it
@@ -469,7 +448,7 @@ export async function runVerb(argv) {
       // --goal / --no-goal are the create's words; without either, the workflow's initial decides
       // a person filing for themselves sponsors it; an agent filing a person's request names them (proven) or records them unverified
       const sponsorship = args.sponsor || args['requested-by'] ? await requestedSponsor(args) : principalOf()?.startsWith('person:') ? { sponsor: principalOf().slice('person:'.length) } : {};
-      const created = withBoard((b) => engine.createCard(b, {
+      const created = await withBoardAsync(async (b) => engine.createCard(b, await engine.prepareCard(b, {
         title: rest.join(' '), body: textFrom(args['body-file'], args.body), assignee: args.assignee === 'none' ? null : args.assignee, machine: args.machine,
         workspace, branch: args.branch, parents: args.parent, priority: args.priority, tenant: args.tenant,
         triage: args.triage, idempotencyKey: args['idempotency-key'], maxRuntime: duration(args['max-runtime']),
@@ -479,7 +458,7 @@ export async function runVerb(argv) {
         project: args.project ?? meta.project_id ?? null, goal: args.goal ? { maxTurns: args['goal-max-turns'] ? Number(args['goal-max-turns']) : null } : args['no-goal'] ? false : null,
         tags: [...(viewTag ? [viewTag] : []), ...(args.tag ?? [])],
         ...sponsorship,
-      }), true);
+      })), true);
       await withBoardAsync(b=>engine.runEffects(b,created.effects??[]));
       let answer = { id: created.id, existing: created.existing };
       if (args.adopt) {
@@ -878,10 +857,9 @@ export async function runVerb(argv) {
 
     case 'diagnostics': case 'diag': {
       const { settings } = await laneState(root).catch(() => ({ settings: boardSettings({}) }));
-      const fleet = observeFleet(root);
+      const fleet = await observeFleet(root);
       const publishing = await boardPublishing(root);
-      const expiredMail = mailExpiries();
-      const waitingMail = mailWaiting();
+      const [expiredMail, waitingMail] = await Promise.all([await mailExpiries(), await mailWaiting()]);
       const order = ['warning', 'error', 'critical'];
       const found = withBoard((b) => {
         const cards = args.task ? [engine.getTask(b.db, args.task)].filter(Boolean) : engine.listCards(b.db, {});
@@ -1143,7 +1121,7 @@ export async function runVerb(argv) {
     }
     case 'close': {
       need(1, 'a card id');
-      const closed = withBoard((b) => closeSession(b, id));
+      const closed = await withBoardAsync((b) => closeSession(b, id));
       say(args, closed ? closed.map((c) => `closed ${c.session_id} (pane ${c.pane} on ${c.machine})`).join('\n') : `${id} has no live session`, { ok: true, closed });
       return 0;
     }
@@ -1450,13 +1428,13 @@ function authorize(db, verb, rest, args, own) {
 }
 
 /** Close the live sessions of a card that is not running (its worker's and its reviewers'), freeing its machine's place. */
-function closeSession(board, id) {
+async function closeSession(board, id) {
   const task = engine.getTask(board.db, id);
   if (!task) throw new Error(`no card ${id}`);
   if (task.current_run_id != null) throw new Error(`${id} has an active run; pause, block or reclaim it before closing its session`);
   tx(board, (db) => requestSessionClose(db, id));
   const report = { undelivered: [], closed: [] };
-  closeRequested(board, report, { taskId: id, caller: callerSession() });
+  await closeRequested(board, report, { taskId: id, caller: callerSession() });
   if (report.undelivered.length) throw new Error(`close remains queued for retry: ${report.undelivered.map((r) => r[2]).join('; ')}`);
   return report.closed.length ? report.closed : null;
 }
@@ -1487,6 +1465,7 @@ function setDispatching(root, slug, on, reason) {
       db.prepare("INSERT INTO supercode_board_state (key, value) VALUES ('dispatching', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(JSON.stringify(value));
     });
   } finally { board.close(); }
+  holdSwitch(slug, value);
   return value;
 }
 
@@ -1508,6 +1487,10 @@ async function answerVerbs(root, log) {
   ownHome(home);
   // the lock held, this owner takes its fencing epoch and fences every board, before it answers anything (fence.mjs)
   const epoch = takeEpoch(root, () => listBoards(root).map((slug) => openBoard(root, slug)));
+  // what the owner holds of its dispatcher from here on: each board's switch, and its predecessor's last round, read once
+  let lastRound = null;
+  try { const beat = JSON.parse(readFileSync(join(root, 'dispatcher-heartbeat.json'), 'utf8')); lastRound = { at: beat.at, errors: beat.errors ?? [] }; } catch { /* no round recorded yet */ }
+  holdOwnerHealth(listBoards(root).map((slug) => [slug, dispatchingOf(root, slug)]), lastRound);
   log(`owner epoch ${epoch}`);
   const path = boardOwnerSocket(root);
   try { unlinkSync(path); } catch { /* none left */ }
@@ -1596,17 +1579,7 @@ async function serve(root, args = {}) {
   const FAILED_ROUNDS_ALERT = 5;
   const failing = new Map();
   let round = 0;
-  // A dispatcher that is not the published release is told up the chain, once per version pair: it keeps running, and
-  // its manager hears that action is required (a caller's stderr line alone reached no one).
-  const versionsTold = new Set();
-  const tellReleaseBehind = async () => {
-    const behind = releaseBehind();
-    if (!behind || versionsTold.has(behind.key)) return;
-    // the send's key makes a retry safe; a pair counts as told only once a manager got it
-    if (await alertManagers(root, 'default', `The dispatcher of ${root} runs ${behind.text}`, `release-behind-${behind.key}`) > 0) versionsTold.add(behind.key);
-  };
   const watchRounds = async (reports) => {
-    await tellReleaseBehind().catch(() => {});
     if (reports.every((r) => !r.error)) {
       // A publication the team refused holds this home's publishing where it is (nothing after it is skipped): a fact of
       // the same kind as rounds that stopped, so it goes where every board verb's error line reads it. Only a refusal
@@ -1617,7 +1590,9 @@ async function serve(root, args = {}) {
       const heldLine = refused ? [`publishing to the team is held: the team refused ${refused.card}'s ${refused.kind} (sequence ${refused.sequence}) with ${refused.status}${refused.code ? ` ${refused.code}` : ''}: ${String(refused.error ?? '').split('\n')[0].slice(0, 200)} (tried ${refused.attempts} time(s)); nothing of this home after it is published until the team takes it`] : [];
       const temporary = `${heartbeat}.${process.pid}.tmp`;
       try {
-        writeFileSync(temporary, `${JSON.stringify({ at: new Date().toISOString(), round, pid: process.pid, version: installed(), boards: reports.map((r) => ({ board: r.board, started: r.started?.length ?? 0, queued: r.queued?.length ?? 0, failed: r.failed?.length ?? 0, ms: r.ms ?? null })), errors: [...heldLine, ...reports.flatMap((r) => [...(r.failed ?? []), ...(r.unsettled ?? [])].map(([t, why]) => `${r.board} ${t}: ${String(why).split('\n')[0].slice(0, 300)}`))].slice(0, 20) })}\n`);
+        const beat = { at: new Date().toISOString(), round, pid: process.pid, version: installed(), boards: reports.map((r) => ({ board: r.board, started: r.started?.length ?? 0, queued: r.queued?.length ?? 0, failed: r.failed?.length ?? 0, ms: r.ms ?? null })), errors: [...heldLine, ...reports.flatMap((r) => [...(r.failed ?? []), ...(r.unsettled ?? [])].map(([t, why]) => `${r.board} ${t}: ${String(why).split('\n')[0].slice(0, 300)}`))].slice(0, 20) };
+        holdRound(beat.at, beat.errors);
+        writeFileSync(temporary, `${JSON.stringify(beat)}\n`);
         renameSync(temporary, heartbeat);
       } catch { /* the next round writes it */ }
     }
@@ -1650,6 +1625,9 @@ async function serve(root, args = {}) {
     onChange: (name) => log(`home changed (${name}): read again on its next use`),
     onLost: (error) => { log(`the home's files can no longer be watched (${error.message}); stopping, for the keeper to start the owner again`); stop(); },
   });
+  // Hermes's install is located once per process (a launch's memory snapshot reads it): here, before any verb is
+  // answered, so no verb or round waits on it
+  try { (await import('../hermes-cli.mjs')).hermesInstall(); } catch { /* none installed */ }
   let closeVerbs;
   try { closeVerbs = await answerVerbs(root, log); } catch (error) { unfollowHome(); throw error; }
   log(`owning the boards of ${root} on ${localMachine()} (orchestrator ${running ?? 'unknown'}); dispatching where a board's dispatching is on (\`supercode workflow dispatching\`)`);
@@ -1738,6 +1716,8 @@ async function serve(root, args = {}) {
       if (moved) { log(`orchestrator ${moved} is installed (this serve runs ${running}); stopping, for its keeper to start the new one`); break; }
     }
   } finally {
+    // a fetch in flight lands, and what it brought is merged, while this owner still holds the home's lock
+    try { const { settleDocuments } = await import('./git-document.mjs'); await settleDocuments(); } catch { /* nothing in flight */ }
     closeVerbs();
     unfollowHome();
     await closeDoors();