@volter/supercode-orchestrator 0.5.37 → 0.5.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,11 +16,11 @@
16
16
  // subscriber's cursor moves only after its notice was delivered, so two dispatchers over one board never both
17
17
  // launch and a crash between the two steps is answered by the next tick.
18
18
  import { monitorsRuns, failedObservation, surfaceIncident, auditEndedRuns, originOf } from './incidents.mjs';
19
- import { readEventFleet } from './event-fleet.mjs';
20
- let eventFleetRoot = null;
19
+ import { readMachineFleet } from './machine-fleet.mjs';
20
+ let fleetRoot = null;
21
21
  import { inspectCompletionEvidence, noCodeChange, openReviewPullRequests, reviewDue, withCompletionEvidence, reviewCandidate, workflowRevision, retainWorkflowTick } from './ci.mjs';
22
22
  import { runtimeDoor } from './runtime-door.mjs';
23
- import { agentScope, localCardAgent, localReaderContext, localAgentSessions, synchronizeLocalAgents, runBudgetRefusal, runsAwaitingUsage, recordRunUsage, agentPreamble, agentEnvironment, recordAgentRun, pendingAgentRuns, synchronizeAgentRun, abandonAgentRun,agentProfileMismatch,activeAgentAssignments,finishAgentAssignment,synchronizeAgentEnds, agentRunForSession } from '../agents.mjs';
23
+ import { agentScope, localCardAgent, localReaderContext, localAgentSessions, synchronizeLocalAgents, runBudgetRefusal, runsAwaitingUsage, recordRunUsage, agentPreamble, agentEnvironment, recordAgentRun, pendingAgentRuns, synchronizeAgentRun, claimAgentRun, releaseAgentRun, abandonAgentRun,activeAgentAssignments,finishAgentAssignment,synchronizeAgentEnds, agentRunForSession } from '../agents.mjs';
24
24
  import { materializeAgentHome } from '../agent-home.mjs';
25
25
  import { memorySnapshotSync } from '../hermes-cli.mjs';
26
26
  import { createRequire } from 'node:module';
@@ -39,7 +39,7 @@ import { identifyNotice } from './notices.mjs';
39
39
  import { pendingRequestMail } from './request-mail.mjs';
40
40
  import {
41
41
  cardContext, claimTtl, currentRun, DEFAULT_FAILURE_LIMIT, extendClaim, getTask, lastSession,
42
- recomputeReady, runEffects, setRunSession, subscribeIn, subscriptions, reconcileDependencySubscriptions,
42
+ recomputeReady, runEffects, queueEffects, setRunSession, subscribeIn, subscriptions, reconcileDependencySubscriptions,
43
43
  } from './engine.mjs';
44
44
 
45
45
  const WORKER = fileURLToPath(new URL('../bin/hermes-worker.mjs', import.meta.url));
@@ -115,9 +115,22 @@ export function callerSession() {
115
115
  return claude;
116
116
  }
117
117
 
118
- export function observeFleet(root = eventFleetRoot) {
119
- const fleet = readEventFleet(root);
120
- presenceReading = { known: fleet.presenceKnown === true, online: new Set([localMachine(), ...fleet.online]) };
118
+ /** The fleet as the home's machines answer for themselves (machine-fleet.mjs): this machine, and every machine an open
119
+ * card of the home names or one of its notice subscribers is on. */
120
+ export function observeFleet(root = fleetRoot) {
121
+ const machines = new Set();
122
+ for (const slug of root ? listBoards(root) : []) {
123
+ try {
124
+ const b = openBoard(root, slug);
125
+ try {
126
+ const live = "t.status NOT IN ('done', 'cancelled', 'archived') AND NOT COALESCE(c.archived, 0)";
127
+ for (const { machine } of b.db.prepare(`SELECT DISTINCT c.machine FROM supercode_cards c JOIN tasks t ON t.id = c.task_id WHERE c.machine IS NOT NULL AND c.machine != '' AND ${live}`).all()) machines.add(normalMachine(machine));
128
+ for (const { chat_id: address } of b.db.prepare(`SELECT DISTINCT s.chat_id FROM kanban_notify_subs s JOIN tasks t ON t.id = s.task_id JOIN supercode_cards c ON c.task_id = t.id WHERE s.platform = 'supercode' AND s.chat_id LIKE 'sc:%' AND ${live}`).all()) machines.add(normalMachine(address.split(':')[1]));
129
+ } finally { b.close(); }
130
+ } catch { /* its own tick reports it */ }
131
+ }
132
+ const fleet = readMachineFleet(machines, localMachine());
133
+ presenceReading = { known: fleet.presenceKnown === true, online: new Set([localMachine(), ...fleet.online]), unknown: fleet.unknown ?? new Set() };
121
134
  notePresence(presenceReading);
122
135
  return fleet;
123
136
  }
@@ -139,26 +152,30 @@ const adoptedOnly = (s) => s?.mode === 'adopt' && !s.launch_key && s.surface !==
139
152
  const addressOf = (s) => s?.address ?? (s?.session_id && s.machine ? `sc:${s.machine}:${s.harness ?? 'claude-code'}:${s.session_id}` : null);
140
153
  const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
141
154
 
142
- // A run Teams never published (one outside a synced root, a scratch workspace's) is read on its own machine: a
143
- // headless worker by its pid, a pane by that machine's pane list. Null only when the machine cannot answer.
155
+ // A session in a live pane of its machine is the fleet's reading; any other run is read on its own machine: a headless
156
+ // worker by its pid, a pane by that machine's pane list, a session in no pane (an adopted native one) by that machine's
157
+ // harness discovery. Null only when the machine cannot answer.
144
158
  const sessionAlive = (fleet, session) => fleet.native?.has(addressOf(session)) ? fleet.native.get(addressOf(session)).alive : unpublishedAlive(session);
159
+ /** A subscriber's address as the session it names, for sessionAlive (its machine, harness and native id). */
160
+ const sessionAt = (address) => { const m = /^sc:([^:]+):([^:]+):(.+)$/.exec(String(address)); return m && HARNESSES[m[2]] ? { address, machine: normalMachine(m[1]), harness: m[2], session_id: m[3] } : null; };
161
+ const subscriberAlive = (fleet, address) => fleet.alive.has(address) || (sessionAt(address) ? sessionAlive(fleet, sessionAt(address)) === true : false);
145
162
  const paneLists = new Map();
146
- // A run the agent sync holds names only its machine and native session (`harness:id`), never a pane or pid. Teams'
147
- // fleet answers for a session it published; one it never published (a gateway worker in its profile's directory,
148
- // outside every synced root) is read through its machine's native discovery, as a native close reads it.
163
+ // A run the agent sync holds names only its machine and native session (`harness:id`), never a pane or pid.
149
164
  function syncRowAlive(fleet, row) {
150
165
  const machine = row.machine || localMachine(), at = row.session_ref.indexOf(':');
151
- const session = { machine, harness: row.session_ref.slice(0, at), session_id: row.session_ref.slice(at + 1), address: `sc:${machine}:${row.session_ref}` };
152
- const alive = sessionAlive(fleet, session);
153
- if (alive !== null) return alive;
154
- try { return nativeReading(nativeSessions(machine, session.harness), session).alive; } catch { return null; }
166
+ return sessionAlive(fleet, { machine, harness: row.session_ref.slice(0, at), session_id: row.session_ref.slice(at + 1), address: `sc:${machine}:${row.session_ref}` });
155
167
  }
156
168
  function unpublishedAlive(session) {
157
169
  if (!session) return null;
158
170
  if (session.surface === 'headless' && session.pid) return pidAlive(session.pid);
159
- if (!session.pane || !session.machine) return null;
171
+ if (!session.machine) return null;
172
+ if (!session.pane) {
173
+ if (!session.session_id) return null;
174
+ try { return nativeReading(nativeSessions(session.machine, session.harness ?? 'claude-code'), session).alive; } catch { return null; }
175
+ }
160
176
  let entry = paneLists.get(session.machine);
161
- if (!entry || Date.now() - entry.at > 5000) {
177
+ if (!entry || Date.now() - entry.at > READING_MS) {
178
+ if (readingsOnly) return null;
162
179
  const r = doorOn(session.machine, 'harness.v1.panes.list', {}, { timeout: 60_000 });
163
180
  const panes = r.code === 0 ? json(r.stdout.trim(), null) : null;
164
181
  entry = { at: Date.now(), panes: Array.isArray(panes) ? panes : null };
@@ -170,19 +187,34 @@ function unpublishedAlive(session) {
170
187
  /** Native activity comes from the machine's harness discovery, not catalogue age. One discovery (up to 2000
171
188
  * sessions, seconds each) serves a round's every native session on that machine, as paneLists serves its panes. */
172
189
  const discoveries = new Map();
190
+ // A machine's pane list and harness discovery are read before a board write opens, never inside one: a door that
191
+ // takes seconds would hold the board's write lock past a worker's own command's busy wait. A write that needs a
192
+ // reading not taken yet reads it as unknown. One reading serves READING_MS.
193
+ const READING_MS = 15_000;
194
+ let readingsOnly = 0;
195
+ const txRead = (board, fn) => { readingsOnly += 1; try { return tx(board, fn); } finally { readingsOnly -= 1; } };
196
+ /** Take the readings a board write about `session` will need, before the write. */
197
+ function readBefore(fleet, session) {
198
+ if (!session?.session_id || !session.machine || session.surface === 'headless') return;
199
+ try { nativeSessions(session.machine, session.harness ?? 'claude-code'); } catch { /* unread: unknown inside the write */ }
200
+ sessionAlive(fleet, session);
201
+ }
173
202
  function nativeSessions(machine, harness) {
174
203
  const key = `${normalMachine(machine)}\u0000${harness}`, seen = discoveries.get(key);
175
- if (seen && Date.now() - seen.at <= 5000) return seen.sessions;
204
+ if (seen && Date.now() - seen.at <= READING_MS) return seen.sessions;
205
+ if (readingsOnly) throw unavailable('native liveness is read before a board write, not inside it');
176
206
  const sessions = discoverNative(machine, harness);
177
207
  discoveries.set(key, { at: Date.now(), sessions });
178
208
  return sessions;
179
209
  }
210
+ // A native discovery is bounded as the fleet reading is: one that does not answer in time is unknown, read next round.
211
+ const NATIVE_DISCOVERY_MS = 15_000;
180
212
  function discoverNative(machine, harness) {
181
213
  if (offlineMachine(machine)) throw unavailable(`native liveness unavailable: machine ${normalMachine(machine)} is offline`);
182
214
  // This machine's discovery is its harness door, as every other harness read here is; another machine's is supercode's fleet arm
183
215
  const result = normalMachine(machine) === localMachine()
184
- ? doorOn(machine, 'harness.v1.sessions.discover', { harnesses: [harness], limit: 2000 }, { timeout: 120_000 })
185
- : sc(['discover', '--on', machine, '--harness', harness, '--limit', '2000'], { timeout: 120_000 });
216
+ ? doorOn(machine, 'harness.v1.sessions.discover', { harnesses: [harness], limit: 2000 }, { timeout: NATIVE_DISCOVERY_MS })
217
+ : sc(['discover', '--on', machine, '--harness', harness, '--limit', '2000'], { timeout: NATIVE_DISCOVERY_MS });
186
218
  const page = result.code === 0 ? lastJson(result.stdout) : null;
187
219
  if (!Array.isArray(page?.sessions)) throw unavailable(`native liveness unavailable on ${machine}: ${(result.stderr || result.stdout).trim().slice(-300)}`);
188
220
  return page.sessions;
@@ -256,8 +288,9 @@ export function lanesOf(profiles = {}) {
256
288
  return out;
257
289
  }
258
290
 
259
- // The latest presence reading (observeFleet). A machine it reads as offline is not asked: each remote call would
260
- // wait out its timeout, and one offline machine held every close and launch on this one.
291
+ // The latest presence reading (observeFleet). A machine that did not answer it (offline, or unread this round) is not
292
+ // asked again until the next reading: each remote call would wait out its timeout, and one silent machine held every
293
+ // close and launch on this one. Its door answers `machine_offline`, which every caller tries again next round.
261
294
  let presenceReading = null;
262
295
  const offlineMachine = (machine) => {
263
296
  const m = normalMachine(machine ?? localMachine());
@@ -265,7 +298,7 @@ const offlineMachine = (machine) => {
265
298
  };
266
299
  /** A machine's public door (`harness.v1.*`), answered at once when the machine reads as offline. */
267
300
  const doorOn = (machine, method, params, options) => offlineMachine(machine)
268
- ? { code: 1, stdout: '', stderr: `machine ${normalMachine(machine)} is offline` }
301
+ ? { code: 1, stdout: '', stderr: JSON.stringify({ code: 'machine_offline', message: `machine ${normalMachine(machine)} did not answer the fleet reading` }) }
269
302
  : runtimeDoor(normalMachine(machine ?? localMachine()) === localMachine() ? null : machine, method, params, options);
270
303
  const unavailable = message => Object.assign(new Error(message), { launchDeferred: true });
271
304
 
@@ -282,8 +315,9 @@ function supervise(board, fleet, settings, report, entry, effects) {
282
315
  try { const receipt = launchReceipt(board, session); if (receipt?.live === false && receipt.exit) exits.set(id, receipt.exit); }
283
316
  catch (error) { report.failed.push([id, `exit receipt unavailable; retaining liveness observation: ${error.message}`]); }
284
317
  }
318
+ for (const { id } of rows) readBefore(fleet, currentRun(board.db, id)?.metadata?.supercode);
285
319
  for (const { id } of rows) {
286
- tx(board, (db) => {
320
+ txRead(board, (db) => {
287
321
  const task = getTask(db, id);
288
322
  if (!task || (entry.workflow.run ? !task.current_run_id : task.status !== 'running')) return;
289
323
  const runRow = currentRun(db, id);
@@ -329,11 +363,11 @@ function supervise(board, fleet, settings, report, entry, effects) {
329
363
  } else extendClaim(db, id, { ttl: settings.ttl, reason: 'launch_receipt_pending' });
330
364
  return;
331
365
  }
332
- // Availability is the same retained linked/unlinked feed subscribers hear.
333
- // A failed stream is unknown, not evidence that every remote machine left.
366
+ // Availability is the machine's own door (observeFleet). A machine whose door did not answer is unknown, not
367
+ // evidence that it left: its run is kept. Only a relay's positive answer that it is not linked reads as offline.
334
368
  if (session?.session_id && session.surface !== 'headless') {
335
369
  const reachable = fleet.online.has(session.machine);
336
- if (!reachable && !fleet.presenceKnown) return;
370
+ if (!reachable && (!fleet.presenceKnown || fleet.unknown?.has(session.machine))) return;
337
371
  if (!reachable) {
338
372
  const next = {...session, offline_since: session.offline_since ?? t};
339
373
  setRunSession(db,runRow.id,next);
@@ -736,7 +770,7 @@ export function launchRun(board, entry, id, lane, fleet, settings) {
736
770
  export async function startCard(board, id, { roles = ['manager'], lanes, settings } = {}) {
737
771
  // the board's own published fleet projection, as a tick reads it: a manager's or creator's start that continues the
738
772
  // slot's session checks that session through it
739
- eventFleetRoot = board.root;
773
+ fleetRoot = board.root;
740
774
  const entry = await loadWorkflow(board.root);
741
775
  const fleet = observeFleet();
742
776
  const before = getTaskNow(board, id), checked = checkWorkspace(board, before);
@@ -925,16 +959,21 @@ function observeIdle(board, fleet, settings, report, entry, effects) {
925
959
  return Number(discovered.get(key)?.find((row) => row.locator?.session_id === session.session_id)?.updated_at_ms) || null;
926
960
  };
927
961
  const managers = [].concat(entry.params.managers ?? []).filter(Boolean);
928
- for (const { id } of board.db.prepare(entry.workflow.run ? "SELECT id FROM tasks WHERE current_run_id IS NOT NULL" : "SELECT id FROM tasks WHERE status = 'running'").all()) {
929
- tx(board, (db) => {
962
+ const running = board.db.prepare(entry.workflow.run ? "SELECT id FROM tasks WHERE current_run_id IS NOT NULL" : "SELECT id FROM tasks WHERE status = 'running'").all();
963
+ for (const { id } of running) { const session = currentRun(board.db, id)?.metadata?.supercode; readBefore(fleet, session); if (session?.session_id) updatedAt(session); }
964
+ for (const { id } of running) {
965
+ txRead(board, (db) => {
930
966
  const task = getTask(db, id);
931
967
  const status = statusOf(db, task);
932
968
  if (!(entry.workflow.statuses[status]?.on?.session_idle || entry.workflow.on?.session_idle || entry.workflow.run?.on?.session_idle)) return;
933
969
  const runRow = currentRun(db, id);
934
970
  const session = runRow?.metadata?.supercode;
935
971
  if (!session?.session_id || !['pane', 'native'].includes(session.surface) || session.offline_since != null) return;
936
- if (session.surface === 'native' && fleet.native?.get(addressOf(session))?.alive !== true) return;
937
- const live = fleet.sessions.find((x) => x.address === addressOf(session));
972
+ if (session.surface === 'native' && sessionAlive(fleet, session) !== true) return;
973
+ // a session in no pane has its turn on its machine's discovery row, which sessionAlive just read
974
+ let nativeRow = null;
975
+ if (session.surface === 'native') try { nativeRow = nativeReading(nativeSessions(session.machine, session.harness ?? 'claude-code'), session).row; } catch { /* unread: no turn */ }
976
+ const live = fleet.sessions.find((x) => x.address === addressOf(session)) ?? (nativeRow ? { address: addressOf(session), turn: nativeRow.activity?.turn ?? null } : undefined);
938
977
  const observedAt = now();
939
978
  const lastAtMs = Number(live?.last_message_at_ms) || updatedAt(session);
940
979
  const lastAt = Math.floor(Number(lastAtMs) / 1000);
@@ -989,7 +1028,7 @@ function observeIdle(board, fleet, settings, report, entry, effects) {
989
1028
  const observation = retain('dispatched', { turns_after: Number(stored?.goal_turns ?? 0), nudge_queued: r.effects.some(e => e.kind === 'send_to_actor'), run_ended: after?.id !== runRow.id });
990
1029
  event(db, id, 'session_idle_observed', observation, runRow.id);
991
1030
  if (after?.id === runRow.id) setRunSession(db, runRow.id, { goal_nudged_at: observedAt });
992
- report.notified.push([id, 'session_idle', live.address]);
1031
+ report.notified.push([id, 'session_idle', addressOf(session)]);
993
1032
  } catch (error) {
994
1033
  retain('dispatch_failed', { error: String(error.message).slice(-200) });
995
1034
  report.failed.push([id, `session_idle: ${String(error.message).slice(-200)}`]);
@@ -1034,7 +1073,9 @@ export async function sessionNotice(board, e) {
1034
1073
  : { lifecycle_notice_error: null, ...(session.lifecycle_notice_pending === e.durableId ? { lifecycle_notice_pending: null } : {}) }) });
1035
1074
  if (!error || session?.lifecycle_notice_error !== error) event(db, e.id, error ? 'session_notice_failed' : receipt?.delivered ? 'session_notice_delivered' : 'session_notice_queued', { address: e.address, verb: e.verb, source_event: e.sourceEvent, notice: e.durableId, receipt, ...(error ? { error, retry: true } : {}) }, e.runId);
1036
1075
  });
1037
- return !error;
1076
+ // the attempt's reason goes with it, onto the effect's row (runEffects counts the attempt)
1077
+ if (error) throw new Error(error);
1078
+ return true;
1038
1079
  }
1039
1080
 
1040
1081
  /** A session effect a committed move asked for: a message to the card's current actor, or closing a slot's pane. */
@@ -1054,7 +1095,10 @@ export async function sessionEffect(board, e) {
1054
1095
  const payload = { notice: e.durableId, address, receipt };
1055
1096
  if (previous?.kind !== kind || JSON.stringify(json(previous?.payload, {})?.receipt) !== JSON.stringify(receipt)) event(db, e.id, kind, payload, e.runId);
1056
1097
  });
1057
- return receipt.delivered;
1098
+ // a send its recipient's mailbox took (filed, or queued for a session between turns) is accepted: the mailbox
1099
+ // delivers it from there, and the board does not send it again
1100
+ if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'the actor notice was not filed');
1101
+ return true;
1058
1102
  }
1059
1103
  if (e.kind === 'close_session' || e.kind === 'run_ended') {
1060
1104
  tx(board, (db) => requestSessionClose(db, e.id, e.kind === 'run_ended' ? { runId: e.runId } : { slot: e.slot, through: e.runId }));
@@ -1069,6 +1113,7 @@ export async function sessionEffect(board, e) {
1069
1113
  * Once every PR is merged the card is sent `reviewed_merged` with the merge commits. Answers false (kept, tried again
1070
1114
  * next round) on anything GitHub did not decide.
1071
1115
  */
1116
+ const MERGE_BUDGET_MS = 300_000;
1072
1117
  export async function mergeReviewed(board, e) {
1073
1118
  const { candidatePullRequests, mergeReviewedPullRequest } = await import('./ci.mjs');
1074
1119
  const pass = view(board, (db) => db.prepare("SELECT id, payload FROM task_events WHERE task_id = ? AND kind = 'review_pass' ORDER BY id DESC LIMIT 1").get(e.id));
@@ -1076,11 +1121,16 @@ export async function mergeReviewed(board, e) {
1076
1121
  const ci = typeof recorded === 'string' ? json(recorded, null) : recorded;
1077
1122
  const prs = candidatePullRequests(ci).filter((p) => p?.pr?.url && p.commit);
1078
1123
  const merges = [];
1124
+ // a merge attempt spends at most MERGE_BUDGET on starting GitHub calls (each bounded by its own timeout), inside its
1125
+ // effect's claim; what is left is the next attempt's
1126
+ const began = Date.now();
1079
1127
  for (const p of prs) {
1128
+ if (Date.now() - began > MERGE_BUDGET_MS) throw new Error(`the merge attempt spent its ${MERGE_BUDGET_MS / 1000} s; the rest is merged next attempt`);
1080
1129
  let answer;
1081
1130
  try { answer = mergeReviewedPullRequest(p.pr.url, p.commit); } catch (error) {
1082
- tx(board, (db) => event(db, e.id, 'merge_pending', { pr: p.pr.url, reviewed: p.commit, reason: String(error.message).slice(-300) }));
1083
- return false;
1131
+ const reason = String(error.message).slice(-300);
1132
+ tx(board, (db) => event(db, e.id, 'merge_pending', { pr: p.pr.url, reviewed: p.commit, reason }));
1133
+ throw new Error(`merging ${p.pr.url} at ${p.commit}: ${reason}`);
1084
1134
  }
1085
1135
  if (answer.merged) { merges.push({ pr: p.pr.url, reviewed: p.commit, merge: answer.sha }); continue; }
1086
1136
  const reason = answer.moved ? `${p.pr.url} is no longer at its reviewed head ${p.commit}${answer.head ? ` (now ${answer.head})` : ''}; what was pushed after the verdict needs its own review` : `${p.pr.url} was closed without merging`;
@@ -1121,7 +1171,7 @@ function launcherCheck(db, task, lane, fleet) {
1121
1171
  if (!task.machine) return 'the card names no machine; a pane session runs on the machine its card names';
1122
1172
  const machine = normalMachine(task.machine);
1123
1173
  if (!fleet.ok) return `supercode could not read the fleet: ${fleet.error}`;
1124
- if (machine !== localMachine() && !fleet.presenceKnown) return 'machine event stream unavailable; launch waits for a current availability reading';
1174
+ if (fleet.unknown?.has(machine)) return `machine ${machine}'s door did not answer; the card starts when it does`;
1125
1175
  if (!fleet.online.has(machine)) return `machine ${machine} is offline; the card starts when it is back`;
1126
1176
  // a card on this machine gets its workspace made here (scratch, dir or worktree, as Hermes makes it); another
1127
1177
  // machine's card names its directory there
@@ -1266,7 +1316,7 @@ function bindLaunch(board, row, session, pane, sid) {
1266
1316
 
1267
1317
  /** A fast worker can reach its first board door before open's reply. Bind only the daemon's keyed pane. */
1268
1318
  export function reconcileCallerLaunch(board, sid) {
1269
- eventFleetRoot = board.root;
1319
+ fleetRoot = board.root;
1270
1320
  if (view(board, (db) => db.prepare(`SELECT 1 FROM task_runs r JOIN tasks t ON t.current_run_id=r.id
1271
1321
  WHERE json_extract(r.metadata, '$.supercode.session_id')=? AND COALESCE(json_extract(r.metadata, '$.supercode.launch_pending'),0)=0`).get(sid))) return;
1272
1322
  const rows = view(board, (db) => db.prepare(`SELECT r.id, r.task_id, r.metadata FROM task_runs r
@@ -1290,7 +1340,9 @@ export function reconcileCallerLaunch(board, sid) {
1290
1340
  }
1291
1341
  }
1292
1342
 
1293
- function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes) {
1343
+ /** The board's agent records joined to Teams: declarations, ends, session ownership and usage. Teams I/O, so it runs in
1344
+ * the delivery pass (`deliver`), never in a dispatch round. */
1345
+ function synchronizeAgents(board, fleet, report, entry) {
1294
1346
  // A delayed launch receipt/catalogue event binds the same declared agent, never another identity.
1295
1347
  // A resumed session is held by its newest run; an earlier run of the same session no longer owns it.
1296
1348
  const agentRuns=[...new Map(view(board,db=>db.prepare("SELECT id,task_id,metadata,ended_at FROM task_runs WHERE json_extract(metadata,'$.supercode.agent_id') IS NOT NULL ORDER BY id").all())
@@ -1299,15 +1351,6 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
1299
1351
  for(const row of agentRuns){
1300
1352
  const session=json(row.metadata,{})?.supercode;
1301
1353
  if(!session?.session_id||!session.agent_org)continue;
1302
- const observation=fleet.sessions?.find(item=>item.address===addressOf(session))?.identity;
1303
- if(observation?.agent?.id===session.agent_id&&session.agent_profile_revision!==observation.revision){
1304
- const mismatch=agentProfileMismatch(session.agent_profile_intent,observation.resolved);
1305
- tx(board,db=>{
1306
- // Rule 4: a run whose recorded profile differs from its agent's intent is shown and not counted.
1307
- setRunSession(db,row.id,{agent_profile_recorded:observation.resolved,agent_profile_revision:observation.revision,agent_profile_mismatch:mismatch,counted:!mismatch.differences.length});
1308
- event(db,row.task_id,mismatch.differences.length?'agent_profile_drift':'agent_profile_recorded',{agent:session.agent_id,session:session.session_id,comparison:mismatch},row.id);
1309
- });
1310
- }
1311
1354
  const ref=`${session.harness}:${session.session_id}`;
1312
1355
  // An ended run whose session the agent store already holds, published, for another agent lost that claim when it
1313
1356
  // ran: no round can change an ended run or a published owner, so the disagreement is history. It is said once on
@@ -1327,10 +1370,11 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
1327
1370
  orgs.set(session.agent_org,session.agent_context);
1328
1371
  }
1329
1372
  // Identities minted while Teams was absent are declared under their ids before their sessions are joined.
1330
- const context=process.env.SUPERCODE_CONTEXT,scope=agentScope(context);
1373
+ const context=process.env.SUPERCODE_CONTEXT;
1374
+ let scope;try{scope=agentScope(context);}catch(error){scope={offline:true};report.failed.push(['agents',`agent declarations wait: ${error.message}`]);}
1331
1375
  if(!scope.offline){
1332
1376
  orgs.set(scope.org,orgs.get(scope.org)??context);
1333
- for(const pending of synchronizeLocalAgents({root:board.root,profiles:entry.profiles,context}))report.queued.push([pending.agent,`agent declaration pending: ${pending.error}`]);
1377
+ for(const pending of synchronizeLocalAgents({root:board.root,profiles:entry.profiles,context}))(pending.failed?report.failed:report.queued).push([pending.agent,`agent declaration ${pending.failed?`failed for good after ${pending.attempts} refusals`:'pending'}: ${pending.error}`]);
1334
1378
  }
1335
1379
  orgs.delete('local');
1336
1380
  // G3: an ended run's session usage, read once from its harness's own records on its machine
@@ -1357,9 +1401,14 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
1357
1401
  for(const [org,context] of orgs)for(const pending of synchronizeAgentEnds(board.root,org,{context}))report.queued.push([pending.agent,`agent end synchronization pending: ${pending.error}`]);
1358
1402
  for(const [org,context] of orgs)for(const row of pendingAgentRuns(board.root,org)){
1359
1403
  const launch=agentRuns.find(run=>String(run.id)===String(row.run_id));
1404
+ // each run is claimed by compare-and-set on its own `published` before it is joined; another pass's claim skips it
1405
+ if(!claimAgentRun(board.root,org,row))continue;
1360
1406
  try{synchronizeAgentRun(board.root,org,row,{context,intended:json(launch?.metadata,{})?.supercode?.agent_profile_intent});}
1361
- catch(error){if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&syncRowAlive(fleet,row)===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
1407
+ catch(error){releaseAgentRun(board.root,org,row);if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&syncRowAlive(fleet,row)===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
1362
1408
  }
1409
+ }
1410
+
1411
+ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes) {
1363
1412
  const deferred=view(board,db=>db.prepare("SELECT t.id,r.id run_id,COALESCE(json_extract(r.metadata,'$.supercode.launched_at'),r.started_at) launched_at FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
1364
1413
  // A deferred launch was never issued (a reading or door was not yet available): it is the start its manager, creator
1365
1414
  // or workflow asked for, carried out within the launch window; supervise surfaces it once when the window ends. The
@@ -1436,7 +1485,8 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
1436
1485
  if (lane.harness === 'codex' && lane.reasoning_effort) tail.push('-c', `model_reasoning_effort=${JSON.stringify(lane.reasoning_effort)}`);
1437
1486
  const harness = HARNESSES[lane.harness] ? lane.harness : 'claude-code';
1438
1487
  const how = HARNESSES[harness];
1439
- const base = ['open', '--new', how.program, '--detach', `--window=${window}`, '--on', task.machine, '--cwd', task.workspace_path, '--env', 'SUPERCODE_BOARD_PROCESS_GUARD=1',
1488
+ // A card on this machine opens through this machine's own daemon; only another machine is reached with `--on`.
1489
+ const base = ['open', '--new', how.program, '--detach', `--window=${window}`, ...(machine === localMachine() ? [] : ['--on', task.machine]), '--cwd', task.workspace_path, '--env', 'SUPERCODE_BOARD_PROCESS_GUARD=1',
1440
1490
  ...Object.entries(meta.agent_env??{}).flatMap(([key,value])=>['--env',`${key}=${value}`])];
1441
1491
  // the slot's last session: kept when live, resumed when lost, never another slot's (a reviewer's is its own)
1442
1492
  // a close its session's own command asked for is carried out first, so its pane is gone before the slot's next
@@ -1497,10 +1547,14 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
1497
1547
  };
1498
1548
  let session = null;
1499
1549
  let replaced = null;
1500
- if (prev?.surface === 'native' && !prev.pane && !prev.close_confirmed_at) {
1550
+ // A session in no pane is read on its machine before anything replaces it. One the board released (an adopted
1551
+ // session it did not launch, left running at its run's end) is its owner's while it runs: it is never resumed beside
1552
+ // itself, and the launch waits for the owner to end it.
1553
+ if (prev?.surface === 'native' && !prev.pane && (!prev.close_confirmed_at || prev.released)) {
1501
1554
  const reading = nativeReading(nativeSessions(prev.machine, prev.harness), prev);
1502
1555
  if (reading.alive == null) throw unavailable(`cannot establish native continuation liveness for ${addressOf(prev)}`);
1503
1556
  fleet.native ??= new Map(); fleet.native.set(addressOf(prev), reading);
1557
+ if (reading.alive && prev.released) throw unavailable(`released session ${addressOf(prev)} still runs; the launch waits for its owner to end it`);
1504
1558
  if (reading.alive && (prev.closed_at || prev.close_requested || prev.forgotten)) throw unavailable(`native session ${addressOf(prev)} is still live; its close must finish before replacement`);
1505
1559
  }
1506
1560
  if (prev?.forgotten) {
@@ -1750,25 +1804,13 @@ function adoptionOpenTo(db, id, request, open) {
1750
1804
  return Boolean(claim) && claim.request === request && claim.lock === claimer();
1751
1805
  }
1752
1806
 
1753
- /**
1754
- * A session on this machine that Teams never published (one outside every synced root: a profile's own directory) is
1755
- * found through this machine's own doors: its panes, and the native session each pane's process runs. Only a full
1756
- * address is looked up here; liveness is still verified below as for any other session.
1757
- */
1758
- function localListing(who) {
1807
+ /** A session in no pane (a native process its machine's harness discovery holds as running), by its full address. */
1808
+ function nativeListing(who) {
1759
1809
  const m = /^sc:([^:]+):([^:]+):(.+)$/.exec(String(who));
1760
- if (!m || normalMachine(m[1]) !== localMachine()) return null;
1761
- const [, , harness, sessionId] = m;
1762
- const listed = runtimeDoor(null, 'harness.v1.panes.list', {}, { timeout: 60_000 });
1763
- const panes = (listed.code === 0 ? json(listed.stdout.trim(), []) : []) ?? [];
1764
- const live = panes.filter((p) => !p.dead && p.pid);
1765
- if (!live.length) return null;
1766
- const r = runtimeDoor(null, 'harness.v1.sessions.activity_under', { pids: live.map((p) => p.pid) }, { timeout: 30_000 });
1767
- const row = (r.code === 0 ? json(r.stdout.trim(), {})?.activities ?? [] : []).find((a) => a.activity?.session_id === sessionId && (a.activity?.harness ?? harness) === harness);
1768
- // Not found in a pane of this machine: not listed, so a session opened moments ago is looked for again next tick.
1769
- const pane = row ? live.find((p) => p.pid === row.pid)?.pane ?? null : null;
1770
- if (!pane) return null;
1771
- return { address: `sc:${localMachine()}:${harness}:${sessionId}`, machine: localMachine(), harness, session_id: sessionId, pane, name: null };
1810
+ if (!m || !HARNESSES[m[2]]) return null;
1811
+ const [, host, harness, session_id] = m;
1812
+ try { return nativeReading(nativeSessions(host, harness), { session_id, harness }).alive === true ? { address: who, machine: normalMachine(host), harness, session_id, pane: null, name: null } : null; }
1813
+ catch { return null; }
1772
1814
  }
1773
1815
 
1774
1816
  /** The verb an adoption with no current run fires: the workflow's own `adopt` where it declares one, else its dispatch start. */
@@ -1790,7 +1832,7 @@ function adoptedRole(db, board, entry, task, run) {
1790
1832
  function adoptSession(board, id, who, { request = null, open = request, progress = {}, workspace = null, ttl, assignee = null, role = null, fleet = null, lanes = null } = {}) {
1791
1833
  fleet ??= observeFleet();
1792
1834
  if (!fleet.ok) throw new Error(`supercode could not read the fleet: ${fleet.error}`);
1793
- const found = fleet.sessions.find((s) => s.address === who || s.address.endsWith(`:${who}`) || s.name === who) ?? localListing(who);
1835
+ const found = fleet.sessions.find((s) => s.address === who || s.address.endsWith(`:${who}`) || s.name === who) ?? nativeListing(who);
1794
1836
  if (!found) throw Object.assign(new Error(`the fleet lists no live session ${who}`), { notListed: true });
1795
1837
  const [, host, harness, sessionId] = found.address.split(':');
1796
1838
  if (!HARNESSES[harness]) throw new Error(`${found.address} is ${harness}; the board keeps and resumes ${Object.keys(HARNESSES).join(' and ')} sessions`);
@@ -1918,7 +1960,7 @@ function reconnectContinuedSessions(board, fleet, entry, report) {
1918
1960
  // Preserve the ended interval as evidence before reopening supervision on the same run.
1919
1961
  event(db,row.task_id,'session_continued',{session_id:saved.session_id,from_pane:saved.pane,pane,prior_outcome:row.outcome,prior_ended_at:row.ended_at,incident:saved.incident??null},row.id);
1920
1962
  db.prepare("UPDATE task_runs SET ended_at=NULL,status='running',outcome=NULL,error=NULL,metadata=?,claim_lock=?,claim_expires=? WHERE id=?")
1921
- .run(JSON.stringify({...json(row.metadata,{}),supercode:session}),claimer(),now()+900,row.id);
1963
+ .run(JSON.stringify({...json(db.prepare('SELECT metadata FROM task_runs WHERE id=?').get(row.id)?.metadata,{}),supercode:session}),claimer(),now()+900,row.id); // re-read: the run's observations just recorded stay
1922
1964
  db.prepare('UPDATE tasks SET current_run_id=?,claim_lock=?,claim_expires=? WHERE id=?').run(row.id,claimer(),now()+900,row.task_id);
1923
1965
  projectStatus(db,board,entry,row.task_id);
1924
1966
  event(db,row.task_id,'dispatch_released',{reason:'verified manual continuation',session:addressOf(session),incident:saved.incident??null});
@@ -1983,10 +2025,43 @@ export async function alertManagers(root, slug, text, key) {
1983
2025
  * and marked board_message_sent. A question goes under the id its card waits on; a resend of a sent one is a no-op
1984
2026
  * (the id is the send's idempotency key). One that fails is tried again next tick.
1985
2027
  */
2028
+ const MAIL_READY_SECONDS = 300;
2029
+ /** Each board message not yet sent nor abandoned gets its effect row (once: the insert reads that none exists). */
2030
+ export function queueBoardMail(db) {
2031
+ db.prepare(`INSERT INTO supercode_effects (effect)
2032
+ SELECT json_object('kind', 'board_message', 'id', original.task_id, 'event', original.id) FROM task_events original
2033
+ WHERE original.kind = 'board_message'
2034
+ AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind = 'board_message_sent' AND sent.task_id = original.task_id AND json_extract(sent.payload, '$.event') = original.id)
2035
+ AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind = 'board_message_abandoned' AND gone.task_id = original.task_id AND json_extract(gone.payload, '$.event') = original.id)
2036
+ AND NOT EXISTS (SELECT 1 FROM supercode_effects e WHERE json_extract(e.effect, '$.kind') = 'board_message' AND json_extract(e.effect, '$.event') = original.id)`).run();
2037
+ }
2038
+
2039
+ /**
2040
+ * One board message's effect: its mail (with any batch it is in) sent under its fixed message id, so a second sender
2041
+ * is dropped by the mailbox. Answers true once it is sent, 'wait' while its batch window is open; throws the failure.
2042
+ */
2043
+ export async function sendBoardMessage(board, e) {
2044
+ const state = () => view(board, (db) => db.prepare(`SELECT
2045
+ EXISTS (SELECT 1 FROM task_events WHERE kind='board_message_sent' AND task_id=? AND json_extract(payload,'$.event')=?) AS sent,
2046
+ EXISTS (SELECT 1 FROM task_events WHERE kind='board_message_abandoned' AND task_id=? AND json_extract(payload,'$.event')=?) AS gone`).get(e.id, e.event, e.id, e.event));
2047
+ if (state().sent || state().gone || !e.event) return true;
2048
+ const report = { undelivered: [] };
2049
+ await sendBoardMessages(board, report, e.id);
2050
+ if (state().sent) return true;
2051
+ const failed = report.undelivered.find(([task]) => task === e.id);
2052
+ if (failed) throw new Error(failed[2]);
2053
+ // a mail waits for its batch window (at most a minute); one still not ready well after that is an attempt that failed
2054
+ const recorded = view(board, (db) => db.prepare('SELECT created_at FROM task_events WHERE id = ?').get(e.event)?.created_at);
2055
+ if (recorded && now() - Number(recorded) > MAIL_READY_SECONDS) throw new Error(`the mail was not ready to send ${now() - Number(recorded)} s after it was recorded`);
2056
+ return 'wait';
2057
+ }
2058
+
1986
2059
  export async function sendBoardMessages(board, report, taskId = null) {
1987
2060
  const pending = view(board, db => db.prepare(`SELECT 1 FROM task_events original WHERE kind='board_message'
1988
2061
  AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind='board_message_sent'
1989
- AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id) LIMIT 1`).get());
2062
+ AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
2063
+ AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind='board_message_abandoned'
2064
+ AND gone.task_id=original.task_id AND json_extract(gone.payload,'$.event')=original.id) LIMIT 1`).get());
1990
2065
  if (!pending) return;
1991
2066
  const planned = tx(board, db => pendingRequestMail(db, taskId));
1992
2067
  if (Number.isFinite(planned.nextAt)) report.next_mail_at = Math.min(report.next_mail_at ?? Infinity, planned.nextAt);
@@ -1999,17 +2074,13 @@ export async function sendBoardMessages(board, report, taskId = null) {
1999
2074
  const receipt = await send(board, boardAddress(board.label), board.label, mail.to, mail.text, mail.message_id ?? `board-message-${first.id}`, { ask: Boolean(mail.message_id), subject });
2000
2075
  if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'message was not filed');
2001
2076
  tx(board, db => {
2002
- for (const member of mail.members) event(db, member.task_id, 'board_message_sent', { event: member.id, to: mail.to, receipt, ...(mail.message_id ? { message_id: mail.message_id, request_id: member.message_id ?? mail.message_id } : {}) });
2077
+ for (const member of mail.members) if (!db.prepare("SELECT 1 FROM task_events WHERE kind='board_message_sent' AND task_id=? AND json_extract(payload,'$.event')=?").get(member.task_id, member.id)) event(db, member.task_id, 'board_message_sent', { event: member.id, to: mail.to, receipt, ...(mail.message_id ? { message_id: mail.message_id, request_id: member.message_id ?? mail.message_id } : {}) });
2003
2078
  });
2004
2079
  } catch (error) {
2005
2080
  const reason = String(error.message).slice(-200);
2006
2081
  for (const member of mail.members) report.undelivered.push([member.task_id, mail.to, reason]);
2007
2082
  tx(board, db => {
2008
- for (const member of mail.members) {
2009
- const previous = db.prepare("SELECT payload,created_at FROM task_events WHERE task_id=? AND kind='board_message_failed' AND json_extract(payload,'$.event')=? ORDER BY id DESC LIMIT 1").get(member.task_id, member.id);
2010
- if (json(previous?.payload, {})?.reason !== reason || now() - (previous?.created_at ?? 0) >= 60)
2011
- event(db, member.task_id, 'board_message_failed', { event: member.id, to: mail.to, message_id: mail.message_id, reason });
2012
- }
2083
+ for (const member of mail.members) event(db, member.task_id, 'board_message_failed', { event: member.id, to: mail.to, message_id: mail.message_id, reason });
2013
2084
  });
2014
2085
  }
2015
2086
  }
@@ -2018,8 +2089,10 @@ export async function sendBoardMessages(board, report, taskId = null) {
2018
2089
  /** The board's own address: where its notices come from and where card updates are mailed. */
2019
2090
  export const boardAddress = (label) => `sc:${localMachine()}:board:${label.replace(/[^A-Za-z0-9_.-]/g, '-')}`;
2020
2091
 
2021
- /** Consecutive failed sends after which a subscription is dropped (Hermes's notifier drops a dead chat after 12). */
2092
+ /** Consecutive failed sends after which a chat-platform subscription is dropped (Hermes's notifier drops a dead chat after 12). */
2022
2093
  const MAX_SEND_FAILURES = 12;
2094
+ /** Attempts a supercode notice gets before it is abandoned (its subscription stays). */
2095
+ const NOTICE_ATTEMPTS = 6;
2023
2096
 
2024
2097
  /**
2025
2098
  * What an audience is told of an event: the workflow's `notify[audience][kind]` template over the card, the event's
@@ -2099,13 +2172,16 @@ function nativeEnded(address, discovered) {
2099
2172
  * A creator's or manager's subscription is never ended here: its notices wait for the session to come back.
2100
2173
  */
2101
2174
  function endWorkerSubscriptions(board, fleet, report, natively = new Map()) {
2175
+ // each worker subscriber's liveness is read before the write
2176
+ const aliveNow = new Map();
2177
+ if (fleet.ok) for (const { chat_id: address } of board.db.prepare("SELECT DISTINCT chat_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) aliveNow.set(address, subscriberAlive(fleet, address));
2102
2178
  tx(board, (db) => {
2103
2179
  const { held, latest } = sessionRecord(db);
2104
2180
  const remove = db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?");
2105
2181
  const ended = new Map();
2106
2182
  for (const row of db.prepare("SELECT task_id, chat_id, thread_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) {
2107
2183
  const address = row.chat_id, record = latest.get(address);
2108
- if (!record || record.session.released || held.has(address) || (fleet.ok && fleet.alive.has(address))) continue;
2184
+ if (!record || record.session.released || held.has(address) || (fleet.ok && aliveNow.get(address) === true)) continue;
2109
2185
  const evidence = record.session.close_confirmed_at ? { by: 'board_close', run: record.run, closed_at: record.session.close_confirmed_at } : natively.get(address);
2110
2186
  if (!evidence || !remove.run(row.task_id, address, row.thread_id).changes) continue;
2111
2187
  if (!ended.has(address)) ended.set(address, { record, evidence, cards: [] });
@@ -2128,11 +2204,13 @@ async function notify(board, fleet, report, entry) {
2128
2204
  // each subscription's notices go in order; different subscribers are told side by side
2129
2205
  await pool(subscriptions(board.db).filter((sub) => sub.platform === 'supercode'), SEND_CONCURRENCY, async (sub) => {
2130
2206
  if (sub.delivery_metadata?.dependent_card && sub.delivery_metadata.waiting_recipient) return;
2207
+ // a subscription whose last send failed waits out its retry time (as an effect does) before the next attempt
2208
+ if (Number(sub.delivery_metadata?.next_at ?? 0) > now()) return;
2131
2209
  const role = ['worker', 'dependency', 'manager'].includes(sub.delivery_metadata?.role) ? sub.delivery_metadata.role : 'creator';
2132
2210
  // a session the fleet does not list this tick is told next tick: one reading is not a loss (a loaded box drops
2133
2211
  // sessions from a read, and a dropped subscription never came back), and a session that is replaced on its card
2134
2212
  // has its subscription moved when the new one starts
2135
- if (role !== 'creator' && fleet.ok && !fleet.alive.has(sub.chat_id)) return;
2213
+ if (role !== 'creator' && fleet.ok && !subscriberAlive(fleet, sub.chat_id)) return;
2136
2214
  const kinds = heardBy(entry, role);
2137
2215
  if (!kinds.length) return;
2138
2216
  const events = board.db.prepare(`SELECT id, kind, payload, created_at, run_id FROM task_events WHERE task_id = ? AND id > ? AND kind IN (${kinds.map(() => '?').join(', ')}) ORDER BY id`)
@@ -2144,7 +2222,13 @@ async function notify(board, fleet, report, entry) {
2144
2222
  // the one the next run gets): the notice is filed in its mailbox, read when it next works
2145
2223
  const working = role !== 'worker' || view(board, (db) => addressOf(currentRun(db, sub.task_id)?.metadata?.supercode) === sub.chat_id);
2146
2224
  let cursor = Number(sub.last_event_id);
2147
- let failed = null, untried = false;
2225
+ // A notice is sent first, under its fixed key (the mailbox drops a second send of it), and the cursor then only moves
2226
+ // forward past it: two passes may both send one notice, and its recipient gets it once; a send that fails, or a pass
2227
+ // that dies before its send, leaves the cursor behind the notice, so it is sent again.
2228
+ const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
2229
+ const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
2230
+ const forward = (to) => tx(board, (db) => db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(to, ...identity, to));
2231
+ let failed = null, untried = false, trying = null;
2148
2232
  try {
2149
2233
  // an event older than the workflow's notice_max_age_seconds is history, not news: it is passed over, never mailed
2150
2234
  const maxAge = Number(entry.params.notice_max_age_seconds) || null;
@@ -2155,7 +2239,6 @@ async function notify(board, fleet, report, entry) {
2155
2239
  const sendable = (ev) => { const p = json(ev.payload, {}) ?? {}; return !(maxAge && now() - Number(ev.created_at) > maxAge) && !(p.by && p.by === sub.chat_id); };
2156
2240
  const latest = new Map(events.filter((ev) => !['commented', 'answered'].includes(ev.kind) && sendable(ev)).map((ev) => [ev.kind, ev.id]));
2157
2241
  const sentBefore = board.db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND kind IN ('notice_delivered', 'notice_filed') AND json_extract(payload, '$.source_event') = ? AND json_extract(payload, '$.address') = ? LIMIT 1");
2158
- const advance = (id) => tx(board, (db) => db.prepare("UPDATE kanban_notify_subs SET last_event_id = ? WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ? AND last_event_id < ?").run(id, sub.task_id, sub.chat_id, sub.thread_id ?? '', id));
2159
2242
  for (const ev of events) {
2160
2243
  const p = json(ev.payload, {}) ?? {};
2161
2244
  if (maxAge && now() - Number(ev.created_at) > maxAge) { cursor = ev.id; (report.stale_notices_passed ??= 0); report.stale_notices_passed += 1; continue; }
@@ -2173,6 +2256,7 @@ async function notify(board, fleet, report, entry) {
2173
2256
  if (text) {
2174
2257
  const body = ['worker', 'dependency'].includes(role) ? identifyNotice(ev, text) : text;
2175
2258
  if (refused.has(sub.chat_id) || stopping) { untried = true; break; }
2259
+ trying = ev.id;
2176
2260
  let receipt;
2177
2261
  try {
2178
2262
  receipt = await send(board, from, board.label, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: String(sub.chat_id).includes(':claude-code:') ? false : !working || (!entry.workflow.run && sub.delivery_mode === 'notify'), subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
@@ -2180,13 +2264,16 @@ async function notify(board, fleet, report, entry) {
2180
2264
  // board does not send it again.
2181
2265
  if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'notice was not filed');
2182
2266
  } catch (error) {
2267
+ // two passes may send one notice at once, and the second send can fail while the first is taken: a notice
2268
+ // another pass already recorded as taken is no failure
2269
+ if (sentBefore.get(sub.task_id, ev.id, sub.chat_id)) { cursor = ev.id; forward(ev.id); continue; }
2183
2270
  if (!refused.has(sub.chat_id)) refused.set(sub.chat_id, error);
2184
2271
  throw error;
2185
2272
  }
2186
- tx(board, (db) => event(db, sub.task_id, receipt.delivered ? 'notice_delivered' : 'notice_filed', { source_event: ev.id, kind: ev.kind, address: sub.chat_id, receipt }));
2273
+ // recorded once, whichever pass's send the mailbox took
2274
+ tx(board, (db) => { if (!db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND kind IN ('notice_delivered', 'notice_filed') AND json_extract(payload, '$.source_event') = ? AND json_extract(payload, '$.address') = ?").get(sub.task_id, ev.id, sub.chat_id)) event(db, sub.task_id, receipt.delivered ? 'notice_delivered' : 'notice_filed', { source_event: ev.id, kind: ev.kind, address: sub.chat_id, receipt }); });
2187
2275
  report.notified.push([sub.task_id, ev.kind, sub.chat_id]);
2188
- // the cursor follows each notice sent, so a restart mid-backlog resumes after it
2189
- advance(ev.id);
2276
+ forward(ev.id);
2190
2277
  }
2191
2278
  cursor = ev.id;
2192
2279
  }
@@ -2209,16 +2296,22 @@ async function notify(board, fleet, report, entry) {
2209
2296
  });
2210
2297
  report.undelivered.push([sub.task_id, sub.chat_id, failed]);
2211
2298
  }
2212
- // the cursor moves past what was delivered (a failed delivery is retried next tick; the mail id dedupes)
2299
+ // the cursor moves forward past what was passed over (a failed delivery is retried next pass)
2300
+ forward(cursor);
2213
2301
  tx(board, (db) => {
2214
- const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
2215
- const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
2216
- db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(cursor, ...identity, cursor);
2217
2302
  // a round that left this subscription untried (a delivery to its recipient failed on another card) says nothing of it
2218
2303
  if (untried) return;
2304
+ // A notice is tried NOTICE_ATTEMPTS times, waiting as an effect does between attempts. After its last failed
2305
+ // attempt it is abandoned (its terminal state, `notice_abandoned`, recorded once) and the cursor moves past it;
2306
+ // the subscription stays, so the notices after it are still told (dropping it lost them when the round
2307
+ // subscribed the session again from the card's newest event).
2219
2308
  const failures = failed ? Number(sub.delivery_metadata?.failures ?? 0) + 1 : 0;
2220
- if (failures >= MAX_SEND_FAILURES) { db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity); return; }
2221
- if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures }), ...identity);
2309
+ if (failed && failures >= NOTICE_ATTEMPTS && trying) {
2310
+ event(db, sub.task_id, 'notice_abandoned', { source_event: trying, address: sub.chat_id, attempts: failures, error: failed });
2311
+ db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ?, delivery_metadata = ? ${where} AND last_event_id < ?`).run(trying, JSON.stringify({ ...sub.delivery_metadata, failures: 0, next_at: 0 }), ...identity, trying);
2312
+ return;
2313
+ }
2314
+ if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures, next_at: failed ? now() + Math.min(1800, 30 * 2 ** (failures - 1)) : 0 }), ...identity);
2222
2315
  if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity);
2223
2316
  });
2224
2317
  });
@@ -2459,7 +2552,7 @@ const CORRUPT = /malformed|not a database|corrupt|disk image/i;
2459
2552
 
2460
2553
  /** What a tick would start on each board of the home, planned through the workflow's `dispatch`, writing nothing. */
2461
2554
  export async function plan({ root, lanes, settings, slugs = null, params = {} }) {
2462
- eventFleetRoot = root;
2555
+ fleetRoot = root;
2463
2556
  const loaded = await loadWorkflow(root, { fresh: true });
2464
2557
  const entry = { ...loaded, params: { ...loaded.params, ...params } };
2465
2558
  const fleet = Object.values(lanes).some((l) => l.surface === 'pane') ? observeFleet() : { ok: false, alive: new Set(), online: new Set(), sessions: [], error: 'no pane lanes' };
@@ -2559,20 +2652,81 @@ function capMax(cap, group, entry, lanes) {
2559
2652
  catch { return null; }
2560
2653
  }
2561
2654
 
2655
+ const ORCHESTRATOR_ENTRY = fileURLToPath(new URL('../bin/orchestrator.mjs', import.meta.url));
2656
+ const deliveries = new Map();
2657
+ /**
2658
+ * Start the home's delivery pass (`workflow deliver`) in a process of its own, unless one is already running: the
2659
+ * dispatcher's loop calls this after each round and never waits for it. `log(level, line)` hears what it failed at.
2660
+ * Answers whether one was started.
2661
+ */
2662
+ export function startDelivery(root, { log = () => {} } = {}) {
2663
+ if (deliveries.has(root)) return false;
2664
+ const child = spawnChild(process.execPath, [ORCHESTRATOR_ENTRY, 'workflow', 'deliver', '--root', root, '--json'], { stdio: ['ignore', 'pipe', 'pipe'], env: process.env });
2665
+ deliveries.set(root, child);
2666
+ let out = '', err = '';
2667
+ child.stdout.setEncoding('utf8').on('data', (c) => { out += c; });
2668
+ child.stderr.setEncoding('utf8').on('data', (c) => { err = (err + c).slice(-2000); });
2669
+ const done = () => {
2670
+ deliveries.delete(root);
2671
+ let reports = null; try { reports = JSON.parse(out); } catch { log('warn', `delivery pass failed: ${(err || out).trim().slice(-300) || 'no answer'}`); return; }
2672
+ for (const r of reports) {
2673
+ if (r.error) log('warn', `delivery on board ${r.board} failed: ${r.error}`);
2674
+ for (const [task, to, why] of r.undelivered ?? []) log('warn', `board ${r.board}: a delivery for ${task} to ${to} waits: ${why}`);
2675
+ log('info', `delivery on board ${r.board}: ${r.effects_waiting ?? 0} effect(s) waiting, ${r.effects_failed ?? 0} failed for good; took ${Object.entries(r.ms ?? {}).map(([k, v]) => `${k} ${(v / 1000).toFixed(1)}s`).join(', ')}`);
2676
+ }
2677
+ };
2678
+ child.on('error', (error) => { log('warn', `delivery pass did not start: ${error.message}`); deliveries.delete(root); });
2679
+ child.on('close', done);
2680
+ return true;
2681
+ }
2682
+
2683
+ /**
2684
+ * The delivery pass over every board of the home at `root`: what rounds recorded for after them (each due effect, with
2685
+ * its bounded attempts), board mail, subscription notices, and the board's agent records joined to Teams. It runs in
2686
+ * its own process (`workflow deliver`, started by serve after a round, one at a time), never inside a dispatch round,
2687
+ * so no recipient, door or Teams answer holds a round. Answers one report per board.
2688
+ */
2689
+ export async function deliver({ root, slugs = null }) {
2690
+ fleetRoot = root;
2691
+ const entry = await loadWorkflow(root, { fresh: true });
2692
+ const fleet = observeFleet();
2693
+ const out = [];
2694
+ for (const slug of slugs ?? listBoards(root)) {
2695
+ const report = { board: slug, ms: {}, mailed: [], queued: [], failed: [], notified: [], undelivered: [], fleet: fleet.ok ? 'read' : fleet.error };
2696
+ let board;
2697
+ try {
2698
+ board = openBoard(root, slug);
2699
+ board.label = boardLabel(root, slug);
2700
+ setActor(boardAddress(board.label));
2701
+ let mark = Date.now();
2702
+ const phase = (name) => { const t = Date.now(); report.ms[name] = t - mark; mark = t; };
2703
+ // board mail is sent as effect rows, one per message, each claimed as any effect is
2704
+ tx(board, (db) => queueBoardMail(db)); phase('mail');
2705
+ report.effects = await runEffects(board, [], { pending: true }); phase('effects');
2706
+ await notify(board, fleet, report, entry); phase('notify');
2707
+ synchronizeAgents(board, fleet, report, entry); phase('agents');
2708
+ report.effects_failed = Number(board.db.prepare('SELECT COUNT(*) AS n FROM supercode_effects WHERE failed_at IS NOT NULL').get().n);
2709
+ report.effects_waiting = Number(board.db.prepare('SELECT COUNT(*) AS n FROM supercode_effects WHERE failed_at IS NULL').get().n);
2710
+ } catch (error) { report.error = error.message; }
2711
+ finally { board?.close(); }
2712
+ out.push(report);
2713
+ }
2714
+ return out;
2715
+ }
2716
+
2562
2717
  /**
2563
2718
  * One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
2564
2719
  * `only` limits starting to one card (a create that answers whether its card started). Each board is ticked under its
2565
2720
  * version-checked card claims; no dispatcher file lock. Two rounds may run at once (the dispatcher's and a one-shot
2566
2721
  * `workflow dispatch`). An adoption request and a session close are claimed in a transaction (`newClaim`) before
2567
2722
  * anything is done for them outside the board; card starts, supervision, review requests, mailed verbs and manual
2568
- * continuations rely on their per-card transactions. Two rounds on one board at once can still both act on: queued
2569
- * effects in `runEffects` (send_to_actor, session_notice, merge_reviewed, declare_card_agent), board mail,
2570
- * subscription notices (a doubled notice_delivered has been seen live), deferred-launch retries (two panes),
2571
- * launch-receipt work (cancelUnissuedLaunch, bindLaunch), the Teams agent-run sync in reconcileLaunches, and
2572
- * board-only writes that do not re-check (a doubled agent_profile_recorded). Answers one report per board.
2723
+ * continuations rely on their per-card transactions. Two rounds on one board at once can still both act on:
2724
+ * deferred-launch retries (two panes), launch-receipt work (cancelUnissuedLaunch, bindLaunch), and board-only writes
2725
+ * that do not re-check. A round runs no effect and delivers nothing: it records effects (`queueEffects`) and the
2726
+ * delivery pass (`deliver`) carries them out. Answers one report per board.
2573
2727
  */
2574
2728
  export async function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true, params = {}, deploymentReceipt = false }) {
2575
- eventFleetRoot = root;
2729
+ fleetRoot = root;
2576
2730
  // a verb's flags (dispatch --max, --failure-limit) are params of this tick, read by the workflow as its own
2577
2731
  const deployment = deploymentReceipt ? workflowRevision(root) : null;
2578
2732
  const loaded = await loadWorkflow(root, { fresh: true });
@@ -2611,20 +2765,18 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
2611
2765
  readMailbox(board, boardAddress(board.label), report); phase('mailbox');
2612
2766
  await refreshRelationAnswers(board); phase('answers');
2613
2767
  const effects = [];
2614
- completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); await runEffects(board, effects.splice(0), { retryNotices: true }); closeRequested(board, report, { ttl: settings.ttl }); phase('supervise');
2768
+ completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); queueEffects(board, effects.splice(0)); closeRequested(board, report, { ttl: settings.ttl }); phase('supervise');
2615
2769
  report.unsettled = [];
2616
2770
  report.promoted = recomputeReady(board, effects, report.unsettled);
2617
2771
  tx(board,db=>auditEndedRuns(db,board,entry));
2618
2772
  if (dispatch) await activatePullRequestReviews(board, entry, report, effects, { only });
2619
- await sendBoardMessages(board, report);
2620
2773
  if (dispatch && !stopping) startCards(board, fleet, lanes, settings, report, entry, effects, { only, held });
2621
2774
  phase('start');
2622
2775
  if (dispatch && !only && !report.paused) observeIdle(board, fleet, settings, report, entry, effects);
2623
- // what the moves asked for, once they are committed (messages to actors, closed panes, cleaned workspaces)
2624
- await runEffects(board, effects);
2776
+ // what the moves asked for is recorded with them; the delivery pass (`deliver`) carries it out, outside the round
2777
+ queueEffects(board, effects);
2625
2778
  phase('goals');
2626
2779
  ensureSubscriptions(board, entry);
2627
- await notify(board, fleet, report, entry); phase('notify');
2628
2780
  fireHooks(board, settings, report);
2629
2781
  if (String(board.db.prepare('PRAGMA journal_mode').get()?.journal_mode).toLowerCase() === 'wal') board.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get();
2630
2782
  report.ready_waiting = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks t LEFT JOIN supercode_cards c ON c.task_id = t.id WHERE t.status = 'ready' AND t.assignee IS NOT NULL AND c.subtask_of IS NULL").get().n);