@volter/supercode-orchestrator 0.4.27 → 0.4.29

package/board/cli.mjs

@@ -1020,6 +1020,7 @@ async function serve(root, args = {}) {
     while (!stopped) {
       pausedSaid = Boolean(paused(root)) && pausedSaid;
       let interval = 60;
+      const began = Date.now();
       try {
         const state = await laneState(root);
         const settings = { ...state.settings, ...override };
@@ -1027,6 +1028,7 @@ async function serve(root, args = {}) {
         for (const r of await tick({ root, lanes: state.lanes, settings })) {
           const said = summary(r);
           if (!said.endsWith('nothing to do') || args.verbose) log(said);
+          if (args.verbose && r.ms) log(`  took ${Object.entries(r.ms).map(([k, v]) => `${k} ${(v / 1000).toFixed(1)}s`).join(', ')}`);
           if (r.paused && !pausedSaid) { log(`new starts paused by ${r.paused} (\`supercode workflow resume\` lifts it)`); pausedSaid = true; }
           // Hermes's stuck warning: ready work for six ticks running and nothing started
           stuck[r.board] = r.ready_waiting && !r.started.length && !r.paused ? (stuck[r.board] ?? 0) + 1 : 0;
@@ -1038,7 +1040,9 @@ async function serve(root, args = {}) {
       } catch (error) {
         log(`tick failed: ${error.message}`);
       }
-      for (let waited = 0; waited < interval && !stopped; waited += 1) await new Promise((done) => setTimeout(done, 1000));
+      // the next tick is due an interval after this one began (a long tick is not followed by a full interval's wait)
+      const rest = Math.max(1, interval - Math.floor((Date.now() - began) / 1000));
+      for (let waited = 0; waited < rest && !stopped; waited += 1) await new Promise((done) => setTimeout(done, 1000));
     }
   } finally {
     lock.release();

package/board/dispatch.mjs

@@ -475,7 +475,7 @@ function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
   if (prev && prev.machine === machine) {
     if (sessionAlive(fleet, prev)) {
       // the session is kept; the heartbeat's count belongs to the run that ended, so the new run starts it again
-      const { missing_since: _m, goal_turns: _t, goal_nudged_at: _n, idle_since: _i, worked_since_ask: _w, ...kept } = prev;
+      const { missing_since: _m, goal_turns: _t, goal_nudged_at: _n, ...kept } = prev;
       session = { ...kept, mode: 'adopt', launched_at: now(), seen_at: now() };
     } else if (prev.resumable !== false) {
       const r = sc([...base, '--key', `board-${task.id}-r${runId}-resume`, '--input',
@@ -677,14 +677,16 @@ function notice(kind, task, p, role) {
   }
 }
 
-function send(from, label, to, body, key, { queue = false } = {}) {
+async function send(from, label, to, body, key, { queue = false } = {}) {
   const request = {
     op: 'send', from, from_name: `${label} board`, to, queue, id: key,
     body: `[Board notice from the ${label} board] ${body}\n\n(An automated notice from the board's dispatcher; replies to this address go nowhere.)`,
   };
-  const r = sc(['message', 'remote-deliver'], { input: JSON.stringify(request), timeout: 60_000 });
+  // a delivery to a busy session can take a minute on a loaded machine; what it answered is the verdict (a process
+  // killed after it answered `code: 0` delivered)
+  const r = await scAsync(['message', 'remote-deliver'], { input: JSON.stringify(request), timeout: 180_000 });
   const answer = lastJson(r.stdout) ?? {};
-  if (r.code !== 0 || answer.code !== 0) throw new Error((answer.text || r.stderr || 'no answer').trim().slice(-300));
+  if (answer.code !== 0) throw new Error((answer.text || r.stderr || `no answer (exit ${r.code})`).trim().slice(-300));
 }
 
 /** The board's own address: where its notices come from and where card updates are mailed. */
@@ -693,20 +695,20 @@ export const boardAddress = (label) => `sc:${localMachine()}:board:${label.repla
 /** Consecutive failed sends after which a subscription is dropped (Hermes's notifier drops a dead chat after 12). */
 const MAX_SEND_FAILURES = 12;
 
-function notify(board, fleet, report) {
+async function notify(board, fleet, report) {
   const from = boardAddress(board.label);
-  for (const sub of subscriptions(board.db)) {
-    if (sub.platform !== 'supercode') continue;
+  // each subscription's notices go in order; different subscribers are told side by side
+  await pool(subscriptions(board.db).filter((sub) => sub.platform === 'supercode'), SEND_CONCURRENCY, async (sub) => {
     const role = ['worker', 'dependency'].includes(sub.delivery_metadata?.role) ? sub.delivery_metadata.role : 'creator';
     // a session that worked or watched a card and is gone hears nothing more; a creator is retried until it is back
     if (role !== 'creator' && fleet.ok && !fleet.alive.has(sub.chat_id)) {
       tx(board, (db) => db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ?").run(sub.task_id, sub.chat_id));
-      continue;
+      return;
     }
     const kinds = MAIL_KINDS[role];
     const events = board.db.prepare(`SELECT id, kind, payload, created_at FROM task_events WHERE task_id = ? AND id > ? AND kind IN (${kinds.map(() => '?').join(', ')}) ORDER BY id`)
       .all(sub.task_id, sub.last_event_id, ...kinds);
-    if (!events.length) continue;
+    if (!events.length) return;
     const task = tx(board, (db) => getTask(db, sub.task_id));
     let cursor = Number(sub.last_event_id);
     let failed = null;
@@ -721,7 +723,7 @@ function notify(board, fleet, report) {
         } else text = notice(ev.kind, task, p, role);
         if (text) {
           // Hermes's delivery modes: `notify` is passive (read at the session's next turn), `wake`/`notify+wake` start its turn
-          send(from, board.label, sub.chat_id, text, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: sub.delivery_mode === 'notify' });
+          await send(from, board.label, sub.chat_id, text, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: sub.delivery_mode === 'notify' });
           report.notified.push([sub.task_id, ev.kind, sub.chat_id]);
         }
         cursor = ev.id;
@@ -739,7 +741,7 @@ function notify(board, fleet, report) {
       if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures }), sub.task_id, sub.chat_id);
       if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(sub.task_id, sub.chat_id);
     });
-  }
+  });
 }
 
 // Hermes's terminal kinds for a chat subscriber, in its order of silence: archived and unblocked move the cursor and say nothing
@@ -905,7 +907,7 @@ function readMailbox(board, address, report) {
       answer = `${clean.slice(0, 2).join(' ')}: ${(r.status === 0 ? r.stdout : r.stderr || r.stdout).trim().slice(-600) || `exit ${r.status}`}`;
     }
     report.mailed.push([envelope.from, argv?.[0] ?? '?']);
-    try { send(address, board.label, envelope.from, answer, `board-answer-${envelope.id}`); } catch (error) { report.undelivered.push(['mail', envelope.from, String(error.message).slice(-200)]); }
+    report.answers.push(send(address, board.label, envelope.from, answer, `board-answer-${envelope.id}`).catch((error) => { report.undelivered.push(['mail', envelope.from, String(error.message).slice(-200)]); }));
   }
 }
 
@@ -915,14 +917,17 @@ function readMailbox(board, address, report) {
  * The board's heartbeat for a pane session: a running card's session never sits idle without deciding. Each tick, a
  * session that is idle while its card is still running is asked to decide: `request-review` when the work is done,
  * `block` with its kind when it cannot go on (waiting on a person, a capability, another card), or keep working. A
- * goal card (Hermes's goal loop) is told to keep going. A session is asked once it has been idle a whole tick; one
- * seen working since the last ask answered it, and its count starts again. After the turn budget (`goal_max_turns`, else
+ * goal card (Hermes's goal loop) is told to keep going. Idle is judged by the session's last message time (supercode
+ * reads it from the transcript), never by its status label: a session is asked once its last message is a whole tick
+ * old; one whose messages ran on a whole tick past the last ask answered it by working, and its count starts again
+ * (a reply to the ask alone is not work). After the turn budget (`goal_max_turns`, else
  * `kanban.goal_max_turns`) of asks in a row with no work and no decision (a goal card: of turns), the card is blocked `needs_input` and its creator is told.
  * Whether the work is done is the session's call through its card, not an auxiliary judge's.
  */
-function pursueGoals(board, fleet, settings, report) {
+async function pursueGoals(board, fleet, settings, report) {
   if (!fleet.ok) return;
   const cards = board.db.prepare("SELECT id FROM tasks WHERE status = 'running'").all();
+  const nudges = [];
   for (const { id } of cards) {
     const nudge = tx(board, (db) => {
       const task = getTask(db, id);
@@ -930,23 +935,22 @@ function pursueGoals(board, fleet, settings, report) {
       const session = runRow?.metadata?.supercode;
       if (!session?.session_id || session.surface !== 'pane') return null;
       const live = fleet.sessions.find((x) => x.address?.endsWith(`:${session.session_id}`));
-      if (!live) return null;
-      // a session seen working since the last ask answered it by working: its count starts again (a goal card keeps
-      // Hermes's budget of turns); one idle for less than a whole tick is between turns, not idle
-      if (live.status !== 'idle') {
-        if (session.idle_since || (session.goal_nudged_at && !session.worked_since_ask)) setRunSession(db, runRow.id, { ...session, idle_since: null, worked_since_ask: true });
-        return null;
-      }
-      if (!session.idle_since) { setRunSession(db, runRow.id, { ...session, idle_since: now() }); return null; }
-      if (now() - session.idle_since < settings.interval) return null;
+      // a supercode that reports no last message time gives no reading, and no reading gives no verdict
+      const lastAt = Math.floor(Number(live.last_message_at_ms) / 1000);
+      if (!(lastAt > 0)) return null;
+      // a message within the last tick is a session working or between turns, not idle
+      if (now() - lastAt < settings.interval) return null;
       if (session.goal_nudged_at && now() - session.goal_nudged_at < settings.interval * 2) return null;
-      const turns = session.worked_since_ask && !task.goal_mode ? 0 : Number(session.goal_turns ?? 0);
+      // messages that ran on a whole tick past the last ask answered it by working: the count starts again (a goal
+      // card keeps Hermes's budget of turns)
+      const worked = Boolean(session.goal_nudged_at) && lastAt >= session.goal_nudged_at + settings.interval;
+      const turns = worked && !task.goal_mode ? 0 : Number(session.goal_turns ?? 0);
       const budget = task.goal_max_turns ?? settings.goalMaxTurns;
       if (turns >= budget) {
         blockIn(db, id, { reason: `its session went idle ${budget} times without deciding done or blocked; review it`, kind: 'needs_input' });
         return null;
       }
-      setRunSession(db, runRow.id, { ...session, goal_turns: turns + 1, goal_nudged_at: now(), worked_since_ask: false });
+      setRunSession(db, runRow.id, { ...session, goal_turns: turns + 1, goal_nudged_at: now() });
       event(db, id, task.goal_mode ? 'goal_continued' : 'decision_asked', { turn: turns + 1, budget }, runRow.id);
       const local = normalMachine(task.machine) === localMachine();
       const where = local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(board.label)}`;
@@ -957,11 +961,12 @@ function pursueGoals(board, fleet, settings, report) {
           + `A session that stays idle without deciding has its card blocked for its creator after ${budget}.`;
       return { address: live.address, text, kind: task.goal_mode ? 'goal_continued' : 'decision_asked' };
     });
-    if (nudge) {
-      try { send(boardAddress(board.label), board.label, nudge.address, nudge.text, `board-goal-${id}-${now()}`); report.notified.push([id, nudge.kind, nudge.address]); }
-      catch (error) { report.undelivered.push([id, nudge.address, String(error.message).slice(-200)]); }
-    }
+    if (nudge) nudges.push({ id, ...nudge });
   }
+  await pool(nudges, SEND_CONCURRENCY, async (nudge) => {
+    try { await send(boardAddress(board.label), board.label, nudge.address, nudge.text, `board-goal-${nudge.id}-${now()}`); report.notified.push([nudge.id, nudge.kind, nudge.address]); }
+    catch (error) { report.undelivered.push([nudge.id, nudge.address, String(error.message).slice(-200)]); }
+  });
 }
 
 /** Hermes's auto-decompose: triage cards with a pane lane get a session that splits them, a few per tick. */
@@ -990,6 +995,32 @@ export function boardLabel(root, slug) {
   return slug && slug !== 'default' ? `${home}/${slug}` : home;
 }
 
+/** `sc` without blocking the tick: notices to many sessions go out side by side. */
+function scAsync(args, { input = null, timeout = 120_000 } = {}) {
+  return new Promise((done) => {
+    const child = spawnChild(supercodeBin(), args, { cwd: tmpdir(), env: process.env, stdio: ['pipe', 'pipe', 'pipe'] });
+    let stdout = '';
+    let stderr = '';
+    const timer = setTimeout(() => { stderr += ` timed out after ${timeout}ms`; child.kill('SIGKILL'); }, timeout);
+    child.stdout.setEncoding('utf8').on('data', (c) => { stdout += c; });
+    child.stderr.setEncoding('utf8').on('data', (c) => { stderr += c; });
+    child.on('error', (error) => { clearTimeout(timer); done({ code: null, stdout, stderr: stderr + error.message }); });
+    child.on('close', (code) => { clearTimeout(timer); done({ code, stdout, stderr }); });
+    child.stdin.end(input ?? '');
+  });
+}
+
+// how many deliveries a tick has in flight at once (each is a `supercode message` process of a few seconds)
+const SEND_CONCURRENCY = 8;
+
+/** Run `fn` over `items`, at most `n` at a time. */
+async function pool(items, n, fn) {
+  let next = 0;
+  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+    while (next < items.length) { const i = next; next += 1; await fn(items[i]); }
+  }));
+}
+
 /**
  * One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
  * `only` limits starting to one card (a create that answers whether its card started). Answers one report per board.
@@ -1005,7 +1036,9 @@ const CORRUPT = /malformed|not a database|corrupt|disk image/i;
  */
 export async function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true }) {
   const needFleet = Object.values(lanes).some((l) => l.surface === 'pane');
+  const fleetAt = Date.now();
   const fleet = needFleet ? observeFleet() : { ok: false, alive: new Set(), online: new Set(), sessions: [], error: 'no pane lanes' };
+  const fleetMs = Date.now() - fleetAt;
   const boards = slugs ?? listBoards(root);
   // the running cards of every board of the home, for kanban.max_in_progress (host-wide, as Hermes counts it)
   let hostRunning = 0;
@@ -1014,7 +1047,9 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
   }
   const out = [];
   for (const slug of boards) {
-    const report = { board: slug, mailed: [], started: [], queued: [], failed: [], lost: [], crashed: [], timed_out: [], reclaimed: [], auto_blocked: [], promoted: 0, unassigned: [], nonspawnable: [], notified: [], undelivered: [], fleet: fleet.ok ? 'read' : fleet.error };
+    const report = { board: slug, ms: { fleet: fleetMs }, mailed: [], started: [], queued: [], failed: [], lost: [], crashed: [], timed_out: [], reclaimed: [], auto_blocked: [], promoted: 0, unassigned: [], nonspawnable: [], notified: [], undelivered: [], fleet: fleet.ok ? 'read' : fleet.error };
+    // the mailbox's answers in flight (not part of the report the caller prints)
+    Object.defineProperty(report, 'answers', { value: [], enumerable: false });
     const path = boardPath(root, slug);
     const marker = `${path}.corrupt-seen`;
     if (existsSync(marker) && readFileSync(marker, 'utf8').trim() === fingerprint(path)) { report.error = 'skipped: the board file was corrupt and has not changed since (`supercode workflow repair`)'; out.push(report); continue; }
@@ -1024,14 +1059,18 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
     try {
       board = openBoard(root, slug);
       board.label = boardLabel(root, slug);
-      readMailbox(board, boardAddress(board.label), report);
-      supervise(board, fleet, settings, report);
+      // where the tick's time goes, phase by phase (ms), printed by `serve --verbose` and in `--json`
+      let mark = Date.now();
+      const phase = (name) => { const t = Date.now(); report.ms[name] = t - mark; mark = t; };
+      readMailbox(board, boardAddress(board.label), report); phase('mailbox');
+      supervise(board, fleet, settings, report); phase('supervise');
       report.promoted = recomputeReady(board, settings.failureLimit);
       const before = report.started.length;
       if (dispatch) start(board, fleet, lanes, settings, report, { only, hostRunning });
-      hostRunning += report.started.length - before;
-      if (dispatch && !only && !report.paused) { pursueGoals(board, fleet, settings, report); autoDecompose(board, lanes, settings, report); }
-      notify(board, fleet, report);
+      hostRunning += report.started.length - before; phase('start');
+      if (dispatch && !only && !report.paused) { await pursueGoals(board, fleet, settings, report); autoDecompose(board, lanes, settings, report); }
+      phase('goals');
+      await notify(board, fleet, report); phase('notify');
       fireHooks(board, settings, report);
       if (String(board.db.prepare('PRAGMA journal_mode').get()?.journal_mode).toLowerCase() === 'wal') board.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get();
       report.ready_waiting = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks t LEFT JOIN supercode_cards c ON c.task_id = t.id WHERE t.status = 'ready' AND t.assignee IS NOT NULL AND c.subtask_of IS NULL").get().n);
@@ -1042,6 +1081,7 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
       report.error = error.message;
       if (CORRUPT.test(error.message)) writeFileSync(marker, fingerprint(path));
     } finally {
+      await Promise.all(report.answers);
       board?.close();
       lock.release();
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.4.27",
+  "version": "0.4.29",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {