@volter/supercode-orchestrator 0.5.33 → 0.5.35

package/bin/runtime.mjs

@@ -674,6 +674,15 @@ async function main() {
     return;
   }
   const root = resolve(args.root ?? defaultRoot());
+  // `SUPERCODE_BOARD_DISPATCHER=serve`: whoever started this daemon also runs `workflow serve` on the home, the board's
+  // one dispatcher, so this daemon runs no dispatch round (board.mjs). It is this process's alone: the sessions it
+  // launches do not inherit it. Any other value is refused before this daemon takes its lease or a lock.
+  const boardDispatcher = process.env.SUPERCODE_BOARD_DISPATCHER || null;
+  delete process.env.SUPERCODE_BOARD_DISPATCHER;
+  if (boardDispatcher !== null && boardDispatcher !== 'serve') {
+    say('error', { root, message: `SUPERCODE_BOARD_DISPATCHER is \`${boardDispatcher}\`: the one value it takes is \`serve\` (a \`workflow serve\` beside this daemon dispatches the board); unset, the board workflow's dispatch.serve decides` });
+    process.exit(2);
+  }
   let crashed = false;
   let gatewayLock = null;
   if (!args.once) {
@@ -746,11 +755,6 @@ async function main() {
   // binary that spawned this daemon, so the worker doors it opens are that
   // build's, not whatever `supercode` a PATH lookup happens to find.
   const client = new SupercodeHarnessClient(process.env.SUPERCODE_BIN ? { command: process.env.SUPERCODE_BIN } : {});
-  // `SUPERCODE_BOARD_DISPATCHER=serve`: whoever started this daemon also runs `workflow serve` on the home, the board's
-  // one dispatcher, so this daemon runs no dispatch round (board.mjs). It is this process's alone: the sessions it
-  // launches do not inherit it.
-  const boardDispatcher = process.env.SUPERCODE_BOARD_DISPATCHER || null;
-  delete process.env.SUPERCODE_BOARD_DISPATCHER;
   const orchestrator = new Orchestrator({
     boardDispatcher,
     executeJob: args.executorUrl ? httpJobExecutor({ url: args.executorUrl, tokenFile: args.executorTokenFile, root }) : null,

package/board/cli.mjs

@@ -1358,7 +1358,7 @@ function closeSession(board, id) {
 export function summary(r) {
   const parts = [`board ${r.board}:`];
   if (r.error) parts.push(`error ${r.error}`);
-  for (const [k, label] of [['mailed', 'mailed updates'], ['started', 'started'], ['queued', 'waiting'], ['failed', 'failed'], ['lost', 'lost'], ['crashed', 'crashed'], ['timed_out', 'timed out'], ['reclaimed', 'reclaimed'], ['auto_blocked', 'gave up'], ['notified', 'notices'], ['closed', 'closed sessions'], ['undelivered', 'undelivered (retrying)']]) {
+  for (const [k, label] of [['mailed', 'mailed updates'], ['started', 'started'], ['queued', 'waiting'], ['failed', 'failed'], ['lost', 'lost'], ['crashed', 'crashed'], ['timed_out', 'timed out'], ['reclaimed', 'reclaimed'], ['auto_blocked', 'gave up'], ['notified', 'notices'], ['closed', 'closed sessions'], ['undelivered', 'undelivered (retrying)'], ['subscriptions_ended', 'worker subscriptions to ended sessions ended']]) {
     if (r[k]?.length) parts.push(`${label} ${r[k].length}`);
   }
   if (r.promoted) parts.push(`promoted ${r.promoted}`);
@@ -1444,6 +1444,8 @@ async function serve(root, args = {}) {
   const stuck = {};
   const warned = {};
   let pausedSaid = false;
+  // a board whose cards name an assignee that is no profile of this home is said once, until it has none
+  const laneSaid = new Set();
   let wakeReason = null;
   try {
     while (!stopped) {
@@ -1498,6 +1500,8 @@ async function serve(root, args = {}) {
           stuck[r.board] = r.ready_waiting && !r.started.length && !r.paused ? (stuck[r.board] || Date.now()) : 0;
           if (stuck[r.board] && Date.now() - stuck[r.board] >= interval * 6000 && Date.now() - (warned[r.board] ?? 0) >= 300_000) { log(`board ${r.board} stuck: ${r.ready_waiting} ready card(s) and nothing started for ${Math.floor((Date.now() - stuck[r.board]) / 1000)}s; \`supercode workflow diagnostics\``); warned[r.board] = Date.now(); }
           for (const [t, why] of r.queued) log(`  ${t} waits: ${why}`);
+          if (r.nonspawnable?.length && !laneSaid.has(r.board)) { laneSaid.add(r.board); log(`board ${r.board}: ${r.nonspawnable.length} card(s) name an assignee that is no profile of this home, and wait`); }
+          if (!r.nonspawnable?.length) laneSaid.delete(r.board);
           for (const [t, why] of r.failed) log(`  ${t} did not start: ${why}`);
           for (const [t, why] of r.unsettled ?? []) log(`  ${t} could not be read: ${why}`);
           for (const [t, s, m] of r.started) log(`  ${t} started ${s} (${m})`);

package/board/dispatch.mjs

@@ -910,9 +910,15 @@ export function closeRequested(board, report, { taskId = null, caller = null, tt
         const { close_error: _error, ...rest } = current;
         setRunSession(db, row.id, { ...rest, close_error: null, closed_at: at, close_confirmed_at: at });
         const address = addressOf(session);
-        const subscribed = active.some((r) => addressOf(json(r.metadata, {})?.supercode) === address && r.id !== row.id);
-        if (!subscribed) db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ?").run(row.task_id, address);
-        event(db, row.task_id, 'session_closed', { session_id: session.session_id, pane: session.pane, machine: session.machine }, row.id);
+        // The closed session stops hearing every card it heard as their worker: its own and each subtask (a launch
+        // subscribes it to all of them), unless another unended run still works in it, read in this transaction.
+        // Removing only the card's own left every subtask notifying a closed session, each attempt a failed delivery,
+        // every round. A later run that resumes it by id subscribes it again at its launch (and each round while it is
+        // the card's current run).
+        const subscribed = db.prepare('SELECT r.id, r.metadata FROM task_runs r WHERE r.ended_at IS NULL OR EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id)').all()
+          .some((r) => addressOf(json(r.metadata, {})?.supercode) === address && r.id !== row.id);
+        const unsubscribed = subscribed ? 0 : db.prepare("DELETE FROM kanban_notify_subs WHERE platform = 'supercode' AND chat_id = ? AND (task_id = ? OR json_extract(delivery_metadata, '$.role') = 'worker')").run(address, row.task_id).changes;
+        event(db, row.task_id, 'session_closed', { session_id: session.session_id, pane: session.pane, machine: session.machine, unsubscribed }, row.id);
       }
     });
     if (error) report.undelivered.push([row.task_id, addressOf(session), error, 'close']);
@@ -1966,7 +1972,12 @@ async function send(board, from, label, to, body, key, { queue = false, ask = fa
   // killed after it answered `code: 0` delivered)
   const r = await scAsync(['message', 'remote-deliver'], { input: JSON.stringify(request), timeout: 180_000 });
   const answer = lastJson(r.stdout) ?? {};
-  if (answer.code !== 0) throw new Error((answer.text || r.stderr || `no answer (exit ${r.code})`).trim().slice(-300));
+  if (answer.code !== 0) {
+    const error = new Error((answer.text || r.stderr || `no answer (exit ${r.code})`).trim().slice(-300));
+    // the delivery door's own answer that the recipient runs nowhere (mail_route / mail_send), never a local failure
+    if (typeof answer.code === 'number' && / is no longer running\. Nothing was sent\./.test(String(answer.text ?? ''))) error.notRunning = true;
+    throw error;
+  }
   return answer.receipt ?? { delivered: false, reason: 'mail door returned no delivery receipt' };
 }
 
@@ -2064,8 +2075,73 @@ function ensureSubscriptions(board, entry) {
 /** The event kinds an audience hears (the workflow's notify). */
 const heardBy = (entry, audience) => Object.keys(entry.workflow.notify?.[audience] ?? {});
 
+/**
+ * The board's record of each session address: `held`, every address an unended or current run works in, and `latest`,
+ * the latest run that names it. A Claude or Codex session resumed by id keeps its address, so a run that resumes it
+ * holds it again.
+ */
+function sessionRecord(db) {
+  const held = new Set(), latest = new Map();
+  for (const r of db.prepare('SELECT r.id, r.task_id, r.ended_at, r.metadata, t.id AS current FROM task_runs r LEFT JOIN tasks t ON t.current_run_id = r.id ORDER BY r.id').all()) {
+    const session = json(r.metadata, {})?.supercode, address = addressOf(session);
+    if (!address) continue;
+    if (!r.ended_at || r.current) held.add(address);
+    latest.set(address, { run: r.id, task_id: r.task_id, session });
+  }
+  return { held, latest };
+}
+
+/**
+ * Whether the recipient's machine says, through its native discovery, that the session is not running (`persisted`:
+ * its transcript is on disk and nothing runs it). Asked only after the delivery door answered that the session is no
+ * longer running, once per machine and harness a round; null when the machine cannot answer.
+ */
+function nativeEnded(address, discovered) {
+  const m = String(address).match(/^sc:([^:]+):([^:]+):(.+)$/);
+  if (!m) return null;
+  const [, machine, harness, session_id] = m, key = `${normalMachine(machine)}\u0000${harness}`;
+  if (!discovered.has(key)) { try { discovered.set(key, nativeSessions(machine, harness)); } catch { discovered.set(key, null); } }
+  const rows = discovered.get(key);
+  if (!rows) return null;
+  return nativeReading(rows, { session_id, harness }).alive === false ? { by: 'native_discovery', presence: 'persisted', machine } : null;
+}
+
+/**
+ * The worker subscriptions of sessions that have ended, removed in one transaction, with one notice_unsubscribed event
+ * per session naming its cards. A session has ended when no unended or current run works in it, the fleet does not list
+ * it running, its latest run is not a release (a released session is a person's own, left running), and either the
+ * board closed it (that run's confirmed close) or its machine's native discovery read it `persisted` this round
+ * (`natively`). Everything is read inside the transaction, so a run that took the session meanwhile keeps it. A run
+ * that resumes the session subscribes it again at its launch and each round while it is the card's current run.
+ * A creator's or manager's subscription is never ended here: its notices wait for the session to come back.
+ */
+function endWorkerSubscriptions(board, fleet, report, natively = new Map()) {
+  tx(board, (db) => {
+    const { held, latest } = sessionRecord(db);
+    const remove = db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?");
+    const ended = new Map();
+    for (const row of db.prepare("SELECT task_id, chat_id, thread_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) {
+      const address = row.chat_id, record = latest.get(address);
+      if (!record || record.session.released || held.has(address) || (fleet.ok && fleet.alive.has(address))) continue;
+      const evidence = record.session.close_confirmed_at ? { by: 'board_close', run: record.run, closed_at: record.session.close_confirmed_at } : natively.get(address);
+      if (!evidence || !remove.run(row.task_id, address, row.thread_id).changes) continue;
+      if (!ended.has(address)) ended.set(address, { record, evidence, cards: [] });
+      ended.get(address).cards.push(row.task_id);
+    }
+    for (const [address, { record, evidence, cards }] of ended) {
+      event(db, record.task_id, 'notice_unsubscribed', { address, role: 'worker', reason: 'the session has ended', evidence, cards, count: cards.length }, record.run);
+      for (const card of cards) (report.subscriptions_ended ??= []).push([card, address]);
+    }
+  });
+}
+
 async function notify(board, fleet, report, entry) {
   const from = boardAddress(board.label);
+  endWorkerSubscriptions(board, fleet, report);
+  const { held, latest } = view(board, sessionRecord);
+  // A recipient whose delivery failed this round is not asked again this round for its other cards: those are left for
+  // the next round untried (a closed session subscribed to 21 subtasks was a delivery process for each, every round)
+  const refused = new Map(), discovered = new Map(), natively = new Map();
   // each subscription's notices go in order; different subscribers are told side by side
   await pool(subscriptions(board.db).filter((sub) => sub.platform === 'supercode'), SEND_CONCURRENCY, async (sub) => {
     if (sub.delivery_metadata?.dependent_card && sub.delivery_metadata.waiting_recipient) return;
@@ -2085,7 +2161,7 @@ async function notify(board, fleet, report, entry) {
     // the one the next run gets): the notice is filed in its mailbox, read when it next works
     const working = role !== 'worker' || view(board, (db) => addressOf(currentRun(db, sub.task_id)?.metadata?.supercode) === sub.chat_id);
     let cursor = Number(sub.last_event_id);
-    let failed = null;
+    let failed = null, untried = false;
     try {
       // an event older than the workflow's notice_max_age_seconds is history, not news: it is passed over, never mailed
       const maxAge = Number(entry.params.notice_max_age_seconds) || null;
@@ -2113,10 +2189,17 @@ async function notify(board, fleet, report, entry) {
         } else text = noticeOf(entry, role, ev.kind, task, p);
         if (text) {
           const body = ['worker', 'dependency'].includes(role) ? identifyNotice(ev, text) : text;
-          const receipt = await send(board, from, board.label, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: String(sub.chat_id).includes(':claude-code:') ? false : !working || (!entry.workflow.run && sub.delivery_mode === 'notify'), subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
-          // A notice the recipient's mailbox took is filed: the machine delivers it at the session's next turn, so the
-          // board does not send it again.
-          if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'notice was not filed');
+          if (refused.has(sub.chat_id)) { untried = true; break; }
+          let receipt;
+          try {
+            receipt = await send(board, from, board.label, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: String(sub.chat_id).includes(':claude-code:') ? false : !working || (!entry.workflow.run && sub.delivery_mode === 'notify'), subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
+            // A notice the recipient's mailbox took is filed: the machine delivers it at the session's next turn, so the
+            // board does not send it again.
+            if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'notice was not filed');
+          } catch (error) {
+            if (!refused.has(sub.chat_id)) refused.set(sub.chat_id, error);
+            throw error;
+          }
           tx(board, (db) => event(db, sub.task_id, receipt.delivered ? 'notice_delivered' : 'notice_filed', { source_event: ev.id, kind: ev.kind, address: sub.chat_id, receipt }));
           report.notified.push([sub.task_id, ev.kind, sub.chat_id]);
           // the cursor follows each notice sent, so a restart mid-backlog resumes after it
@@ -2125,6 +2208,14 @@ async function notify(board, fleet, report, entry) {
         cursor = ev.id;
       }
     } catch (error) {
+      // One "no longer running" answer is not an ended session (a Claude session is resumed under the same id): for a
+      // worker no run holds, and that is not a released session, the machine's native discovery is asked, and a
+      // `persisted` reading ends its worker subscriptions after this round's deliveries.
+      const record = latest.get(sub.chat_id);
+      if (error.notRunning && role === 'worker' && record && !record.session.released && !held.has(sub.chat_id)) {
+        if (!natively.has(sub.chat_id)) natively.set(sub.chat_id, nativeEnded(sub.chat_id, discovered));
+        if (natively.get(sub.chat_id)) return;
+      }
       failed = String(error.message).slice(-200);
       tx(board, (db) => {
         const kind = 'notice_failed';
@@ -2140,12 +2231,16 @@ async function notify(board, fleet, report, entry) {
       const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
       const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
       db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(cursor, ...identity, cursor);
+      // a round that left this subscription untried (a delivery to its recipient failed on another card) says nothing of it
+      if (untried) return;
       const failures = failed ? Number(sub.delivery_metadata?.failures ?? 0) + 1 : 0;
       if (failures >= MAX_SEND_FAILURES) { db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity); return; }
       if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures }), ...identity);
       if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity);
     });
   });
+  const persisted = new Map([...natively].filter(([, reading]) => reading));
+  if (persisted.size) endWorkerSubscriptions(board, fleet, report, persisted);
 }
 
 // Hermes's terminal kinds for a chat subscriber, in its order of silence: archived and unblocked move the cursor and say nothing

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.33",
+  "version": "0.5.35",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {