@volter/supercode-orchestrator 0.5.34 → 0.5.36

package/board/cli.mjs

@@ -15,7 +15,7 @@ import { actorOf, view, backingPaths, layOut, backingOf, boardExists, boardPath,
 import { isLocalWorkspace } from './workspace.mjs';
 import { openZtrackBoard } from './ztrack.mjs';
 import * as engine from './engine.mjs';
-import { requestAdoption, alertManagers, boardAddress, boardLabel, boardSettings, callerSession, commandEnv, reconcileCallerLaunch, closeRequested, lanesOf, localMachine, observeFleet, paused, plan, startCard, supercodeBin, tick } from './dispatch.mjs';
+import { requestAdoption, alertManagers, boardAddress, boardLabel, boardSettings, callerSession, commandEnv, reconcileCallerLaunch, closeRequested, lanesOf, localMachine, observeFleet, paused, plan, startCard, stopDispatching, supercodeBin, tick } from './dispatch.mjs';
 import { watchBoard } from './wake.mjs';
 import { streamBoardPublications } from './event-stream.mjs';
 import { completionOwnerOf, isLatestReviewer, loadWorkflow, requestSessionClose, scopeOf, statusOf, useWorkflow, workflowEntry, workflowFor } from './workflow.mjs';
@@ -1358,7 +1358,7 @@ function closeSession(board, id) {
 export function summary(r) {
   const parts = [`board ${r.board}:`];
   if (r.error) parts.push(`error ${r.error}`);
-  for (const [k, label] of [['mailed', 'mailed updates'], ['started', 'started'], ['queued', 'waiting'], ['failed', 'failed'], ['lost', 'lost'], ['crashed', 'crashed'], ['timed_out', 'timed out'], ['reclaimed', 'reclaimed'], ['auto_blocked', 'gave up'], ['notified', 'notices'], ['closed', 'closed sessions'], ['undelivered', 'undelivered (retrying)']]) {
+  for (const [k, label] of [['mailed', 'mailed updates'], ['started', 'started'], ['queued', 'waiting'], ['failed', 'failed'], ['lost', 'lost'], ['crashed', 'crashed'], ['timed_out', 'timed out'], ['reclaimed', 'reclaimed'], ['auto_blocked', 'gave up'], ['notified', 'notices'], ['closed', 'closed sessions'], ['undelivered', 'undelivered (retrying)'], ['subscriptions_ended', 'worker subscriptions to ended sessions ended']]) {
     if (r[k]?.length) parts.push(`${label} ${r[k].length}`);
   }
   if (r.promoted) parts.push(`promoted ${r.promoted}`);
@@ -1379,9 +1379,9 @@ async function serve(root, args = {}) {
   if (args.pidfile) writeFileSync(args.pidfile, `${process.pid}\n`);
   let stopped = false;
   let events;
-  const stop = () => { stopped = true; events?.close(); };
-  process.on('SIGTERM', stop);
-  process.on('SIGINT', stop);
+  // a signal ends the round in progress with nothing new started (dispatch.mjs stopDispatching) and no round after it
+  const stop = () => { stopped = true; stopDispatching(); events?.close(); };
+  for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) process.on(signal, stop);
   // what the serve says is kept in the home too (logs/workflow-serve.log), so why it stopped outlives its pane
   const kept = join(root, 'logs', 'workflow-serve.log');
   const log = (text) => {
@@ -1488,7 +1488,7 @@ async function serve(root, args = {}) {
         interval = settings.interval;
         const reports = await tick({ root, lanes: state.lanes, settings, params, deploymentReceipt: true });
         round += 1;
-        await watchRounds(reports);
+        if (!stopped) await watchRounds(reports);
         for (const r of reports) {
           nextMailAt = Math.min(nextMailAt, r.next_mail_at ?? Infinity);
           const said = summary(r);
@@ -1509,7 +1509,7 @@ async function serve(root, args = {}) {
         }
       } catch (error) {
         log(`tick failed: ${error.message}`);
-        await watchRounds([{ board: '*', error: `tick failed: ${error.message}` }]);
+        if (!stopped) await watchRounds([{ board: '*', error: `tick failed: ${error.message}` }]);
       }
       // each round is a line of its own in the slow log, with the steps it spent its time on
       recordSlow({ kind: 'call', name: 'dispatch round', ms: Date.now() - began, slowest: null, extra: { steps: takeSteps() } });
@@ -1529,7 +1529,7 @@ async function serve(root, args = {}) {
     }
   } finally {
     events?.close();
-    process.off('SIGTERM', stop); process.off('SIGINT', stop);
+    for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) process.off(signal, stop);
     if (args.pidfile) { try { unlinkSync(args.pidfile); } catch { /* gone */ } }
   }
   log(stopped ? 'stopped (a signal)' : 'stopped');

package/board/dispatch.mjs

@@ -910,9 +910,15 @@ export function closeRequested(board, report, { taskId = null, caller = null, tt
         const { close_error: _error, ...rest } = current;
         setRunSession(db, row.id, { ...rest, close_error: null, closed_at: at, close_confirmed_at: at });
         const address = addressOf(session);
-        const subscribed = active.some((r) => addressOf(json(r.metadata, {})?.supercode) === address && r.id !== row.id);
-        if (!subscribed) db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ?").run(row.task_id, address);
-        event(db, row.task_id, 'session_closed', { session_id: session.session_id, pane: session.pane, machine: session.machine }, row.id);
+        // The closed session stops hearing every card it heard as their worker: its own and each subtask (a launch
+        // subscribes it to all of them), unless another unended run still works in it, read in this transaction.
+        // Removing only the card's own left every subtask notifying a closed session, each attempt a failed delivery,
+        // every round. A later run that resumes it by id subscribes it again at its launch (and each round while it is
+        // the card's current run).
+        const subscribed = db.prepare('SELECT r.id, r.metadata FROM task_runs r WHERE r.ended_at IS NULL OR EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id)').all()
+          .some((r) => addressOf(json(r.metadata, {})?.supercode) === address && r.id !== row.id);
+        const unsubscribed = subscribed ? 0 : db.prepare("DELETE FROM kanban_notify_subs WHERE platform = 'supercode' AND chat_id = ? AND (task_id = ? OR json_extract(delivery_metadata, '$.role') = 'worker')").run(address, row.task_id).changes;
+        event(db, row.task_id, 'session_closed', { session_id: session.session_id, pane: session.pane, machine: session.machine, unsubscribed }, row.id);
       }
     });
     if (error) report.undelivered.push([row.task_id, addressOf(session), error, 'close']);
@@ -1966,7 +1972,12 @@ async function send(board, from, label, to, body, key, { queue = false, ask = fa
   // killed after it answered `code: 0` delivered)
   const r = await scAsync(['message', 'remote-deliver'], { input: JSON.stringify(request), timeout: 180_000 });
   const answer = lastJson(r.stdout) ?? {};
-  if (answer.code !== 0) throw new Error((answer.text || r.stderr || `no answer (exit ${r.code})`).trim().slice(-300));
+  if (answer.code !== 0) {
+    const error = new Error((answer.text || r.stderr || `no answer (exit ${r.code})`).trim().slice(-300));
+    // the delivery door's own answer that the recipient runs nowhere (mail_route / mail_send), never a local failure
+    if (typeof answer.code === 'number' && / is no longer running\. Nothing was sent\./.test(String(answer.text ?? ''))) error.notRunning = true;
+    throw error;
+  }
   return answer.receipt ?? { delivered: false, reason: 'mail door returned no delivery receipt' };
 }
 
@@ -1998,6 +2009,7 @@ export async function sendBoardMessages(board, report, taskId = null) {
   const planned = tx(board, db => pendingRequestMail(db, taskId));
   if (Number.isFinite(planned.nextAt)) report.next_mail_at = Math.min(report.next_mail_at ?? Infinity, planned.nextAt);
   for (const mail of planned.ready) {
+    if (stopping) break;
     const first = mail.members[0];
     try {
       const task = view(board, db => getTask(db, first.task_id));
@@ -2064,8 +2076,73 @@ function ensureSubscriptions(board, entry) {
 /** The event kinds an audience hears (the workflow's notify). */
 const heardBy = (entry, audience) => Object.keys(entry.workflow.notify?.[audience] ?? {});
 
+/**
+ * The board's record of each session address: `held`, every address an unended or current run works in, and `latest`,
+ * the latest run that names it. A Claude or Codex session resumed by id keeps its address, so a run that resumes it
+ * holds it again.
+ */
+function sessionRecord(db) {
+  const held = new Set(), latest = new Map();
+  for (const r of db.prepare('SELECT r.id, r.task_id, r.ended_at, r.metadata, t.id AS current FROM task_runs r LEFT JOIN tasks t ON t.current_run_id = r.id ORDER BY r.id').all()) {
+    const session = json(r.metadata, {})?.supercode, address = addressOf(session);
+    if (!address) continue;
+    if (!r.ended_at || r.current) held.add(address);
+    latest.set(address, { run: r.id, task_id: r.task_id, session });
+  }
+  return { held, latest };
+}
+
+/**
+ * Whether the recipient's machine says, through its native discovery, that the session is not running (`persisted`:
+ * its transcript is on disk and nothing runs it). Asked only after the delivery door answered that the session is no
+ * longer running, once per machine and harness a round; null when the machine cannot answer.
+ */
+function nativeEnded(address, discovered) {
+  const m = String(address).match(/^sc:([^:]+):([^:]+):(.+)$/);
+  if (!m) return null;
+  const [, machine, harness, session_id] = m, key = `${normalMachine(machine)}\u0000${harness}`;
+  if (!discovered.has(key)) { try { discovered.set(key, nativeSessions(machine, harness)); } catch { discovered.set(key, null); } }
+  const rows = discovered.get(key);
+  if (!rows) return null;
+  return nativeReading(rows, { session_id, harness }).alive === false ? { by: 'native_discovery', presence: 'persisted', machine } : null;
+}
+
+/**
+ * The worker subscriptions of sessions that have ended, removed in one transaction, with one notice_unsubscribed event
+ * per session naming its cards. A session has ended when no unended or current run works in it, the fleet does not list
+ * it running, its latest run is not a release (a released session is a person's own, left running), and either the
+ * board closed it (that run's confirmed close) or its machine's native discovery read it `persisted` this round
+ * (`natively`). Everything is read inside the transaction, so a run that took the session meanwhile keeps it. A run
+ * that resumes the session subscribes it again at its launch and each round while it is the card's current run.
+ * A creator's or manager's subscription is never ended here: its notices wait for the session to come back.
+ */
+function endWorkerSubscriptions(board, fleet, report, natively = new Map()) {
+  tx(board, (db) => {
+    const { held, latest } = sessionRecord(db);
+    const remove = db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?");
+    const ended = new Map();
+    for (const row of db.prepare("SELECT task_id, chat_id, thread_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) {
+      const address = row.chat_id, record = latest.get(address);
+      if (!record || record.session.released || held.has(address) || (fleet.ok && fleet.alive.has(address))) continue;
+      const evidence = record.session.close_confirmed_at ? { by: 'board_close', run: record.run, closed_at: record.session.close_confirmed_at } : natively.get(address);
+      if (!evidence || !remove.run(row.task_id, address, row.thread_id).changes) continue;
+      if (!ended.has(address)) ended.set(address, { record, evidence, cards: [] });
+      ended.get(address).cards.push(row.task_id);
+    }
+    for (const [address, { record, evidence, cards }] of ended) {
+      event(db, record.task_id, 'notice_unsubscribed', { address, role: 'worker', reason: 'the session has ended', evidence, cards, count: cards.length }, record.run);
+      for (const card of cards) (report.subscriptions_ended ??= []).push([card, address]);
+    }
+  });
+}
+
 async function notify(board, fleet, report, entry) {
   const from = boardAddress(board.label);
+  endWorkerSubscriptions(board, fleet, report);
+  const { held, latest } = view(board, sessionRecord);
+  // A recipient whose delivery failed this round is not asked again this round for its other cards: those are left for
+  // the next round untried (a closed session subscribed to 21 subtasks was a delivery process for each, every round)
+  const refused = new Map(), discovered = new Map(), natively = new Map();
   // each subscription's notices go in order; different subscribers are told side by side
   await pool(subscriptions(board.db).filter((sub) => sub.platform === 'supercode'), SEND_CONCURRENCY, async (sub) => {
     if (sub.delivery_metadata?.dependent_card && sub.delivery_metadata.waiting_recipient) return;
@@ -2085,7 +2162,7 @@ async function notify(board, fleet, report, entry) {
     // the one the next run gets): the notice is filed in its mailbox, read when it next works
     const working = role !== 'worker' || view(board, (db) => addressOf(currentRun(db, sub.task_id)?.metadata?.supercode) === sub.chat_id);
     let cursor = Number(sub.last_event_id);
-    let failed = null;
+    let failed = null, untried = false;
     try {
       // an event older than the workflow's notice_max_age_seconds is history, not news: it is passed over, never mailed
       const maxAge = Number(entry.params.notice_max_age_seconds) || null;
@@ -2113,10 +2190,17 @@ async function notify(board, fleet, report, entry) {
         } else text = noticeOf(entry, role, ev.kind, task, p);
         if (text) {
           const body = ['worker', 'dependency'].includes(role) ? identifyNotice(ev, text) : text;
-          const receipt = await send(board, from, board.label, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: String(sub.chat_id).includes(':claude-code:') ? false : !working || (!entry.workflow.run && sub.delivery_mode === 'notify'), subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
-          // A notice the recipient's mailbox took is filed: the machine delivers it at the session's next turn, so the
-          // board does not send it again.
-          if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'notice was not filed');
+          if (refused.has(sub.chat_id) || stopping) { untried = true; break; }
+          let receipt;
+          try {
+            receipt = await send(board, from, board.label, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: String(sub.chat_id).includes(':claude-code:') ? false : !working || (!entry.workflow.run && sub.delivery_mode === 'notify'), subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
+            // A notice the recipient's mailbox took is filed: the machine delivers it at the session's next turn, so the
+            // board does not send it again.
+            if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'notice was not filed');
+          } catch (error) {
+            if (!refused.has(sub.chat_id)) refused.set(sub.chat_id, error);
+            throw error;
+          }
           tx(board, (db) => event(db, sub.task_id, receipt.delivered ? 'notice_delivered' : 'notice_filed', { source_event: ev.id, kind: ev.kind, address: sub.chat_id, receipt }));
           report.notified.push([sub.task_id, ev.kind, sub.chat_id]);
           // the cursor follows each notice sent, so a restart mid-backlog resumes after it
@@ -2125,6 +2209,14 @@ async function notify(board, fleet, report, entry) {
         cursor = ev.id;
       }
     } catch (error) {
+      // One "no longer running" answer is not an ended session (a Claude session is resumed under the same id): for a
+      // worker no run holds, and that is not a released session, the machine's native discovery is asked, and a
+      // `persisted` reading ends its worker subscriptions after this round's deliveries.
+      const record = latest.get(sub.chat_id);
+      if (error.notRunning && role === 'worker' && record && !record.session.released && !held.has(sub.chat_id)) {
+        if (!natively.has(sub.chat_id)) natively.set(sub.chat_id, nativeEnded(sub.chat_id, discovered));
+        if (natively.get(sub.chat_id)) return;
+      }
       failed = String(error.message).slice(-200);
       tx(board, (db) => {
         const kind = 'notice_failed';
@@ -2140,12 +2232,16 @@ async function notify(board, fleet, report, entry) {
       const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
       const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
       db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(cursor, ...identity, cursor);
+      // a round that left this subscription untried (a delivery to its recipient failed on another card) says nothing of it
+      if (untried) return;
       const failures = failed ? Number(sub.delivery_metadata?.failures ?? 0) + 1 : 0;
       if (failures >= MAX_SEND_FAILURES) { db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity); return; }
       if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures }), ...identity);
       if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity);
     });
   });
+  const persisted = new Map([...natively].filter(([, reading]) => reading));
+  if (persisted.size) endWorkerSubscriptions(board, fleet, report, persisted);
 }
 
 // Hermes's terminal kinds for a chat subscriber, in its order of silence: archived and unblocked move the cursor and say nothing
@@ -2359,6 +2455,11 @@ function scAsync(args, { input = null, timeout = 120_000 } = {}) {
   });
 }
 
+// A dispatcher told to stop (its serve's SIGTERM, SIGINT or SIGHUP) starts nothing more in the round it is in: no further
+// board, card start, board message or notice. What it leaves untried is not a failure; the next dispatcher sends it.
+let stopping = false;
+export function stopDispatching() { stopping = true; }
+
 // how many deliveries a tick has in flight at once (each is a `supercode message` process of a few seconds)
 const SEND_CONCURRENCY = 8;
 
@@ -2366,7 +2467,7 @@ const SEND_CONCURRENCY = 8;
 async function pool(items, n, fn) {
   let next = 0;
   await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
-    while (next < items.length) { const i = next; next += 1; await fn(items[i]); }
+    while (next < items.length && !stopping) { const i = next; next += 1; await fn(items[i]); }
   }));
 }
 
@@ -2503,6 +2604,7 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
   const held = dispatch ? homeHoldings(root, boards, lanes, fleet, entry) : [];
   const out = [];
   for (const slug of boards) {
+    if (stopping) break;
     const report = { board: slug, ms: { fleet: fleetMs }, mailed: [], started: [], queued: [], failed: [], lost: [], crashed: [], timed_out: [], reclaimed: [], auto_blocked: [], promoted: 0, unassigned: [], nonspawnable: [], notified: [], undelivered: [], fleet: fleet.ok ? 'read' : fleet.error };
     // the mailbox's answers in flight (not part of the report the caller prints)
     Object.defineProperty(report, 'answers', { value: [], enumerable: false });
@@ -2533,7 +2635,7 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
       tx(board,db=>auditEndedRuns(db,board,entry));
       if (dispatch) await activatePullRequestReviews(board, entry, report, effects, { only });
       await sendBoardMessages(board, report);
-      if (dispatch) startCards(board, fleet, lanes, settings, report, entry, effects, { only, held });
+      if (dispatch && !stopping) startCards(board, fleet, lanes, settings, report, entry, effects, { only, held });
       phase('start');
       if (dispatch && !only && !report.paused) observeIdle(board, fleet, settings, report, entry, effects);
       // what the moves asked for, once they are committed (messages to actors, closed panes, cleaned workspaces)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.34",
+  "version": "0.5.36",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {