@volter/supercode-orchestrator 0.5.38 → 0.5.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/agents.mjs CHANGED
@@ -38,6 +38,11 @@ function store(root){
38
38
  CREATE TABLE IF NOT EXISTS agent_end_outbox(org TEXT NOT NULL,agent_id TEXT NOT NULL,reason TEXT NOT NULL,created_at INTEGER NOT NULL,PRIMARY KEY(org,agent_id));
39
39
  CREATE TABLE IF NOT EXISTS agent_runtime_sessions(org TEXT NOT NULL,runtime_ref TEXT NOT NULL,session_ref TEXT NOT NULL,PRIMARY KEY(org,runtime_ref));
40
40
  CREATE TABLE IF NOT EXISTS agent_runs(org TEXT NOT NULL,agent_id TEXT NOT NULL,session_ref TEXT NOT NULL,machine TEXT,run_id TEXT,profile_json TEXT,published INTEGER NOT NULL DEFAULT 0,ended_at INTEGER,end_reason TEXT,main INTEGER NOT NULL DEFAULT 1,starter_session TEXT,usage_json TEXT,PRIMARY KEY(org,session_ref));`);
41
+ // A declaration Teams refused is asked again a bounded number of times (synchronizeLocalAgents), then left failed.
42
+ const columns=new Set(db.prepare('PRAGMA table_info(agent_intents)').all().map(c=>c.name));
43
+ for(const [column,type] of Object.entries({declare_attempts:'INTEGER NOT NULL DEFAULT 0',declare_next_at:'INTEGER NOT NULL DEFAULT 0',declare_failed_at:'INTEGER',declare_error:'TEXT'}))if(!columns.has(column))db.exec(`ALTER TABLE agent_intents ADD COLUMN ${column} ${type}`);
44
+ // An end is claimed by the pass sending it (synchronizeAgentEnds).
45
+ if(!new Set(db.prepare('PRAGMA table_info(agent_end_outbox)').all().map(c=>c.name)).has('claimed_at'))db.exec('ALTER TABLE agent_end_outbox ADD COLUMN claimed_at INTEGER');
41
46
  return db;
42
47
  }
43
48
  const digest=value=>createHash('sha256').update(value).digest('hex');
@@ -86,13 +91,16 @@ export function synchronizeAgentEnds(root,org,{context}={}){
86
91
  try{
87
92
  for(const row of db.prepare('SELECT * FROM agent_end_outbox WHERE org=? ORDER BY created_at LIMIT 100').all(org)){
88
93
  if(db.prepare("SELECT 1 FROM agent_assignments WHERE org=? AND agent_id=? AND state='active'").get(org,row.agent_id)){db.prepare('DELETE FROM agent_end_outbox WHERE org=? AND agent_id=?').run(org,row.agent_id);continue;}
89
- if(db.prepare('SELECT 1 FROM agent_runs WHERE org=? AND agent_id=? AND published=0').get(org,row.agent_id)){pending.push({agent:row.agent_id,error:'session ownership publication remains pending'});continue;}
94
+ if(db.prepare('SELECT 1 FROM agent_runs WHERE org=? AND agent_id=? AND published<=0').get(org,row.agent_id)){pending.push({agent:row.agent_id,error:'session ownership publication remains pending'});continue;}
95
+ // claimed by compare-and-set on its claimed_at (free, or a claim older than SYNC_CLAIM_MS from a pass that died);
96
+ // deleted only after Teams ended the agent (the end is idempotent by its digest), handed back when it failed
97
+ if(db.prepare('UPDATE agent_end_outbox SET claimed_at=? WHERE org=? AND agent_id=? AND (claimed_at IS NULL OR claimed_at<?)').run(Date.now(),org,row.agent_id,Date.now()-SYNC_CLAIM_MS).changes!==1)continue;
90
98
  try{
91
99
  agentDoor('end',[row.agent_id,row.reason,digest(`${org}/${row.agent_id}/end/${row.created_at}`)],{context});
92
100
  db.prepare('UPDATE agent_runs SET ended_at=COALESCE(ended_at,?),end_reason=COALESCE(end_reason,?) WHERE org=? AND agent_id=?').run(Date.now(),row.reason,org,row.agent_id);
93
101
  db.prepare('DELETE FROM agent_end_outbox WHERE org=? AND agent_id=?').run(org,row.agent_id);
94
102
  }
95
- catch(error){pending.push({agent:row.agent_id,error:error.message});}
103
+ catch(error){db.prepare('UPDATE agent_end_outbox SET claimed_at=NULL WHERE org=? AND agent_id=?').run(org,row.agent_id);pending.push({agent:row.agent_id,error:error.message});}
96
104
  }
97
105
  return pending;
98
106
  }finally{db.close();}
@@ -349,21 +357,36 @@ function adoptLocalOrg(root,scope){
349
357
  db.exec('COMMIT');
350
358
  }catch(error){try{db.exec('ROLLBACK');}catch{}throw error;}finally{db.close();}
351
359
  }
360
+ const DECLARE_ATTEMPTS=6;
361
+ // A sync claim outlasts one Teams call chain (each door call bounded at 25-60 s).
362
+ const SYNC_CLAIM_MS=600000;
352
363
  /** Declare every locally minted identity once Teams is reachable, under the id its sessions already carry. */
353
364
  export function synchronizeLocalAgents({root,profiles,context,machine=null,declarations={}}){
354
365
  const scope=agentScope(context);if(scope.offline)return [];
355
366
  adoptLocalOrg(root,scope);
356
367
  const db=store(root);let rows;
357
- try{rows=db.prepare('SELECT * FROM agent_intents WHERE org=? AND declared=0 AND declaration_json IS NOT NULL ORDER BY created_at').all(scope.org);}finally{db.close();}
368
+ try{rows=db.prepare('SELECT * FROM agent_intents WHERE org=? AND declared=0 AND declaration_json IS NOT NULL AND declare_failed_at IS NULL AND declare_next_at<=? ORDER BY created_at').all(scope.org,Date.now());}finally{db.close();}
358
369
  const pending=[],base={root,org:scope.org,server:scope.server,profiles,context,machine};
359
370
  for(const row of rows){
360
371
  const declaration=JSON.parse(row.declaration_json);
372
+ // claimed by compare-and-set on its own next attempt time, set ahead only while it is due: a second pass skips it
373
+ const claim=store(root);let claimed;
374
+ try{claimed=claim.prepare('UPDATE agent_intents SET declare_next_at=? WHERE org=? AND id=? AND declared=0 AND declare_failed_at IS NULL AND declare_next_at<=?').run(Date.now()+SYNC_CLAIM_MS,row.org,row.id,Date.now()).changes===1;}finally{claim.close();}
375
+ if(!claimed)continue;
361
376
  try{
362
377
  if(declaration.kind==='root')ensureRootAgent({...base,profile:declaration.profile});
363
378
  else if(declaration.kind==='card')ensureCardAgent({...base,profile:declaration.profile,card:declaration.card,role:declaration.role,sponsor:declaration.sponsor,starterSession:declaration.starterSession});
364
379
  else if(declaration.kind==='profile')ensureProfileAgent({...base,profile:declaration.profile});
365
380
  else if(declaration.kind==='declared')ensureProfileAgent({...base,profile:declaration.profile,agent:declaration.agent,declarations});
366
- }catch(error){if(!teamsUnavailable(error))pending.push({agent:row.id,error:error.message});else return [...pending,{agent:row.id,error:error.message}];}
381
+ }catch(error){
382
+ // Teams unreachable is no answer about this declaration: the rest wait for the next pass, and no attempt is counted.
383
+ if(teamsUnavailable(error)){const db=store(root);try{db.prepare('UPDATE agent_intents SET declare_next_at=? WHERE org=? AND id=?').run(row.declare_next_at??0,row.org,row.id);}finally{db.close();}return [...pending,{agent:row.id,error:error.message}];}
384
+ // A refusal is this declaration's own: asked again after a doubling wait (one minute, at most an hour), and after
385
+ // DECLARE_ATTEMPTS refusals left failed, said once, and never asked again.
386
+ const made=Number(row.declare_attempts??0)+1,failed=made>=DECLARE_ATTEMPTS,db=store(root);
387
+ try{db.prepare('UPDATE agent_intents SET declare_attempts=?,declare_next_at=?,declare_failed_at=?,declare_error=? WHERE org=? AND id=?').run(made,Date.now()+Math.min(3600000,60000*2**(made-1)),failed?Date.now():null,String(error.message).slice(0,500),row.org,row.id);}finally{db.close();}
388
+ pending.push({agent:row.id,error:error.message,attempts:made,failed});
389
+ }
367
390
  }
368
391
  return pending;
369
392
  }
@@ -377,7 +400,7 @@ export function recordNativeAgentSession(root,runtimeRef,session) {
377
400
  db.exec('BEGIN IMMEDIATE');
378
401
  db.prepare('INSERT INTO agent_runtime_sessions(org,runtime_ref,session_ref) VALUES (?,?,?) ON CONFLICT(org,runtime_ref) DO UPDATE SET session_ref=excluded.session_ref').run(owner.org,runtimeRef,session);
379
402
  // A published native session remains history; an unpublished lease-only row is not a session.
380
- db.prepare('DELETE FROM agent_runs WHERE org=? AND session_ref=? AND published=0').run(owner.org,runtimeRef);
403
+ db.prepare('DELETE FROM agent_runs WHERE org=? AND session_ref=? AND published<=0').run(owner.org,runtimeRef);
381
404
  db.exec('COMMIT');
382
405
  }catch(error){try{db.exec('ROLLBACK');}catch{}throw error;}finally{db.close();}
383
406
  return agentRunForSession(root,[runtimeRef]);
@@ -421,7 +444,13 @@ export function recordAgentRun(root,org,id,{session,machine=null,run,profile=nul
421
444
  db.exec('COMMIT');
422
445
  }catch(error){try{db.exec('ROLLBACK');}catch{}throw error;}finally{db.close();}
423
446
  }
424
- export function pendingAgentRuns(root,org){const db=store(root);try{return db.prepare('SELECT * FROM agent_runs WHERE org=? AND published=0 ORDER BY rowid LIMIT 100').all(org).filter(row=>!(retryOf(root,org,row)?.next_at>Date.now()));}finally{db.close();}}
447
+ // A run waiting to be joined has published=0; a pass joining it holds published=-(the time it claimed it), by
448
+ // compare-and-set, and a claim older than SYNC_CLAIM_MS (a pass that died) is free again.
449
+ export function pendingAgentRuns(root,org){const db=store(root);try{return db.prepare('SELECT * FROM agent_runs WHERE org=? AND (published=0 OR (published<0 AND -published<?)) ORDER BY rowid LIMIT 100').all(org,Date.now()-SYNC_CLAIM_MS).filter(row=>!(retryOf(root,org,row)?.next_at>Date.now()));}finally{db.close();}}
450
+ /** Claim one waiting run for this pass; false when another pass holds it. */
451
+ export function claimAgentRun(root,org,row){const db=store(root);try{return db.prepare('UPDATE agent_runs SET published=? WHERE org=? AND session_ref=? AND published=?').run(-Date.now(),org,row.session_ref,row.published).changes===1;}finally{db.close();}}
452
+ /** Give a claimed run back, unjoined. */
453
+ export function releaseAgentRun(root,org,row){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=0 WHERE org=? AND session_ref=? AND published<0').run(org,row.session_ref);}finally{db.close();}}
425
454
  /** Session references come from the bound runtime, never from a model's requested agent id. */
426
455
  export function agentRunForSession(root, references) {
427
456
  const db = store(root);
@@ -468,7 +497,7 @@ export function runBudgetRefusal(root,agent,budgets,now){
468
497
  return null;
469
498
  }
470
499
  /** A run whose session Teams never had and that no longer runs here can never be owned there: it leaves the sync. */
471
- export function abandonAgentRun(root,org,session){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=2 WHERE org=? AND session_ref=? AND published=0').run(org,session);db.prepare('DELETE FROM agent_run_retries WHERE org=? AND session_ref=?').run(org,session);}finally{db.close();}}
500
+ export function abandonAgentRun(root,org,session){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=2 WHERE org=? AND session_ref=? AND published<=0').run(org,session);db.prepare('DELETE FROM agent_run_retries WHERE org=? AND session_ref=?').run(org,session);}finally{db.close();}}
472
501
  export function publishAgentRun(root,org,session){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=1 WHERE org=? AND session_ref=?').run(org,session);}finally{db.close();}}
473
502
  export function agentPreamble(agent,{main=true,previous=[]}={}){
474
503
  const {identity:a,intended,assignments=[]}=agent;
package/board/cli.mjs CHANGED
@@ -15,7 +15,7 @@ import { actorOf, view, backingPaths, layOut, backingOf, boardExists, boardPath,
15
15
  import { isLocalWorkspace } from './workspace.mjs';
16
16
  import { openZtrackBoard } from './ztrack.mjs';
17
17
  import * as engine from './engine.mjs';
18
- import { requestAdoption, alertManagers, boardAddress, boardLabel, boardSettings, callerSession, commandEnv, reconcileCallerLaunch, closeRequested, lanesOf, localMachine, observeFleet, paused, plan, startCard, stopDispatching, supercodeBin, tick } from './dispatch.mjs';
18
+ import { requestAdoption, alertManagers, boardAddress, boardLabel, boardSettings, callerSession, commandEnv, reconcileCallerLaunch, closeRequested, lanesOf, localMachine, observeFleet, paused, plan, startCard, stopDispatching, supercodeBin, tick, deliver, startDelivery } from './dispatch.mjs';
19
19
  import { watchBoard } from './wake.mjs';
20
20
  import { streamBoardPublications } from './event-stream.mjs';
21
21
  import { completionOwnerOf, isLatestReviewer, loadWorkflow, requestSessionClose, scopeOf, statusOf, useWorkflow, workflowEntry, workflowFor } from './workflow.mjs';
@@ -950,6 +950,14 @@ export async function runWorkflow(argv) {
950
950
  process.stdout.write(`${lines.join('\n')}\n`);
951
951
  return 0;
952
952
  }
953
+ case 'deliver': {
954
+ // the delivery pass: due effects, board mail, subscription notices and the agent records' Teams sync (serve
955
+ // starts it after each round in a process of its own; never part of a round)
956
+ const reports = await deliver({ root, slugs: args.board ? [slug] : null });
957
+ if (args.json) say(args, '', reports);
958
+ else for (const r of reports) process.stdout.write(`${r.board}: ${r.error ? `error: ${r.error}` : `${r.effects_waiting ?? 0} effect(s) waiting, ${r.effects_failed ?? 0} failed for good, ${r.notified.length} notified`}\n`);
959
+ return reports.some((r) => r.error) ? 1 : 0;
960
+ }
953
961
  case 'daemon': case 'serve': return serve(root, args);
954
962
 
955
963
  case 'gc': {
@@ -1488,6 +1496,8 @@ async function serve(root, args = {}) {
1488
1496
  interval = settings.interval;
1489
1497
  const reports = await tick({ root, lanes: state.lanes, settings, params, deploymentReceipt: true });
1490
1498
  round += 1;
1499
+ // what the round recorded is delivered by a pass of its own, which this loop never waits for
1500
+ if (!stopped) startDelivery(root, { log: (level, line) => { if (level === 'warn' || args.verbose) log(line); } });
1491
1501
  if (!stopped) await watchRounds(reports);
1492
1502
  for (const r of reports) {
1493
1503
  nextMailAt = Math.min(nextMailAt, r.next_mail_at ?? Infinity);
@@ -20,7 +20,7 @@ import { readMachineFleet } from './machine-fleet.mjs';
20
20
  let fleetRoot = null;
21
21
  import { inspectCompletionEvidence, noCodeChange, openReviewPullRequests, reviewDue, withCompletionEvidence, reviewCandidate, workflowRevision, retainWorkflowTick } from './ci.mjs';
22
22
  import { runtimeDoor } from './runtime-door.mjs';
23
- import { agentScope, localCardAgent, localReaderContext, localAgentSessions, synchronizeLocalAgents, runBudgetRefusal, runsAwaitingUsage, recordRunUsage, agentPreamble, agentEnvironment, recordAgentRun, pendingAgentRuns, synchronizeAgentRun, abandonAgentRun,activeAgentAssignments,finishAgentAssignment,synchronizeAgentEnds, agentRunForSession } from '../agents.mjs';
23
+ import { agentScope, localCardAgent, localReaderContext, localAgentSessions, synchronizeLocalAgents, runBudgetRefusal, runsAwaitingUsage, recordRunUsage, agentPreamble, agentEnvironment, recordAgentRun, pendingAgentRuns, synchronizeAgentRun, claimAgentRun, releaseAgentRun, abandonAgentRun,activeAgentAssignments,finishAgentAssignment,synchronizeAgentEnds, agentRunForSession } from '../agents.mjs';
24
24
  import { materializeAgentHome } from '../agent-home.mjs';
25
25
  import { memorySnapshotSync } from '../hermes-cli.mjs';
26
26
  import { createRequire } from 'node:module';
@@ -39,7 +39,7 @@ import { identifyNotice } from './notices.mjs';
39
39
  import { pendingRequestMail } from './request-mail.mjs';
40
40
  import {
41
41
  cardContext, claimTtl, currentRun, DEFAULT_FAILURE_LIMIT, extendClaim, getTask, lastSession,
42
- recomputeReady, runEffects, setRunSession, subscribeIn, subscriptions, reconcileDependencySubscriptions,
42
+ recomputeReady, runEffects, queueEffects, setRunSession, subscribeIn, subscriptions, reconcileDependencySubscriptions,
43
43
  } from './engine.mjs';
44
44
 
45
45
  const WORKER = fileURLToPath(new URL('../bin/hermes-worker.mjs', import.meta.url));
@@ -174,7 +174,8 @@ function unpublishedAlive(session) {
174
174
  try { return nativeReading(nativeSessions(session.machine, session.harness ?? 'claude-code'), session).alive; } catch { return null; }
175
175
  }
176
176
  let entry = paneLists.get(session.machine);
177
- if (!entry || Date.now() - entry.at > 5000) {
177
+ if (!entry || Date.now() - entry.at > READING_MS) {
178
+ if (readingsOnly) return null;
178
179
  const r = doorOn(session.machine, 'harness.v1.panes.list', {}, { timeout: 60_000 });
179
180
  const panes = r.code === 0 ? json(r.stdout.trim(), null) : null;
180
181
  entry = { at: Date.now(), panes: Array.isArray(panes) ? panes : null };
@@ -186,9 +187,22 @@ function unpublishedAlive(session) {
186
187
  /** Native activity comes from the machine's harness discovery, not catalogue age. One discovery (up to 2000
187
188
  * sessions, seconds each) serves a round's every native session on that machine, as paneLists serves its panes. */
188
189
  const discoveries = new Map();
190
+ // A machine's pane list and harness discovery are read before a board write opens, never inside one: a door that
191
+ // takes seconds would hold the board's write lock past a worker's own command's busy wait. A write that needs a
192
+ // reading not taken yet reads it as unknown. One reading serves READING_MS.
193
+ const READING_MS = 15_000;
194
+ let readingsOnly = 0;
195
+ const txRead = (board, fn) => { readingsOnly += 1; try { return tx(board, fn); } finally { readingsOnly -= 1; } };
196
+ /** Take the readings a board write about `session` will need, before the write. */
197
+ function readBefore(fleet, session) {
198
+ if (!session?.session_id || !session.machine || session.surface === 'headless') return;
199
+ try { nativeSessions(session.machine, session.harness ?? 'claude-code'); } catch { /* unread: unknown inside the write */ }
200
+ sessionAlive(fleet, session);
201
+ }
189
202
  function nativeSessions(machine, harness) {
190
203
  const key = `${normalMachine(machine)}\u0000${harness}`, seen = discoveries.get(key);
191
- if (seen && Date.now() - seen.at <= 5000) return seen.sessions;
204
+ if (seen && Date.now() - seen.at <= READING_MS) return seen.sessions;
205
+ if (readingsOnly) throw unavailable('native liveness is read before a board write, not inside it');
192
206
  const sessions = discoverNative(machine, harness);
193
207
  discoveries.set(key, { at: Date.now(), sessions });
194
208
  return sessions;
@@ -301,8 +315,9 @@ function supervise(board, fleet, settings, report, entry, effects) {
301
315
  try { const receipt = launchReceipt(board, session); if (receipt?.live === false && receipt.exit) exits.set(id, receipt.exit); }
302
316
  catch (error) { report.failed.push([id, `exit receipt unavailable; retaining liveness observation: ${error.message}`]); }
303
317
  }
318
+ for (const { id } of rows) readBefore(fleet, currentRun(board.db, id)?.metadata?.supercode);
304
319
  for (const { id } of rows) {
305
- tx(board, (db) => {
320
+ txRead(board, (db) => {
306
321
  const task = getTask(db, id);
307
322
  if (!task || (entry.workflow.run ? !task.current_run_id : task.status !== 'running')) return;
308
323
  const runRow = currentRun(db, id);
@@ -944,8 +959,10 @@ function observeIdle(board, fleet, settings, report, entry, effects) {
944
959
  return Number(discovered.get(key)?.find((row) => row.locator?.session_id === session.session_id)?.updated_at_ms) || null;
945
960
  };
946
961
  const managers = [].concat(entry.params.managers ?? []).filter(Boolean);
947
- for (const { id } of board.db.prepare(entry.workflow.run ? "SELECT id FROM tasks WHERE current_run_id IS NOT NULL" : "SELECT id FROM tasks WHERE status = 'running'").all()) {
948
- tx(board, (db) => {
962
+ const running = board.db.prepare(entry.workflow.run ? "SELECT id FROM tasks WHERE current_run_id IS NOT NULL" : "SELECT id FROM tasks WHERE status = 'running'").all();
963
+ for (const { id } of running) { const session = currentRun(board.db, id)?.metadata?.supercode; readBefore(fleet, session); if (session?.session_id) updatedAt(session); }
964
+ for (const { id } of running) {
965
+ txRead(board, (db) => {
949
966
  const task = getTask(db, id);
950
967
  const status = statusOf(db, task);
951
968
  if (!(entry.workflow.statuses[status]?.on?.session_idle || entry.workflow.on?.session_idle || entry.workflow.run?.on?.session_idle)) return;
@@ -1056,7 +1073,9 @@ export async function sessionNotice(board, e) {
1056
1073
  : { lifecycle_notice_error: null, ...(session.lifecycle_notice_pending === e.durableId ? { lifecycle_notice_pending: null } : {}) }) });
1057
1074
  if (!error || session?.lifecycle_notice_error !== error) event(db, e.id, error ? 'session_notice_failed' : receipt?.delivered ? 'session_notice_delivered' : 'session_notice_queued', { address: e.address, verb: e.verb, source_event: e.sourceEvent, notice: e.durableId, receipt, ...(error ? { error, retry: true } : {}) }, e.runId);
1058
1075
  });
1059
- return !error;
1076
+ // the attempt's reason goes with it, onto the effect's row (runEffects counts the attempt)
1077
+ if (error) throw new Error(error);
1078
+ return true;
1060
1079
  }
1061
1080
 
1062
1081
  /** A session effect a committed move asked for: a message to the card's current actor, or closing a slot's pane. */
@@ -1076,7 +1095,10 @@ export async function sessionEffect(board, e) {
1076
1095
  const payload = { notice: e.durableId, address, receipt };
1077
1096
  if (previous?.kind !== kind || JSON.stringify(json(previous?.payload, {})?.receipt) !== JSON.stringify(receipt)) event(db, e.id, kind, payload, e.runId);
1078
1097
  });
1079
- return receipt.delivered;
1098
+ // a send its recipient's mailbox took (filed, or queued for a session between turns) is accepted: the mailbox
1099
+ // delivers it from there, and the board does not send it again
1100
+ if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'the actor notice was not filed');
1101
+ return true;
1080
1102
  }
1081
1103
  if (e.kind === 'close_session' || e.kind === 'run_ended') {
1082
1104
  tx(board, (db) => requestSessionClose(db, e.id, e.kind === 'run_ended' ? { runId: e.runId } : { slot: e.slot, through: e.runId }));
@@ -1091,6 +1113,7 @@ export async function sessionEffect(board, e) {
1091
1113
  * Once every PR is merged the card is sent `reviewed_merged` with the merge commits. Answers false (kept, tried again
1092
1114
  * next round) on anything GitHub did not decide.
1093
1115
  */
1116
+ const MERGE_BUDGET_MS = 300_000;
1094
1117
  export async function mergeReviewed(board, e) {
1095
1118
  const { candidatePullRequests, mergeReviewedPullRequest } = await import('./ci.mjs');
1096
1119
  const pass = view(board, (db) => db.prepare("SELECT id, payload FROM task_events WHERE task_id = ? AND kind = 'review_pass' ORDER BY id DESC LIMIT 1").get(e.id));
@@ -1098,11 +1121,16 @@ export async function mergeReviewed(board, e) {
1098
1121
  const ci = typeof recorded === 'string' ? json(recorded, null) : recorded;
1099
1122
  const prs = candidatePullRequests(ci).filter((p) => p?.pr?.url && p.commit);
1100
1123
  const merges = [];
1124
+ // a merge attempt spends at most MERGE_BUDGET on starting GitHub calls (each bounded by its own timeout), inside its
1125
+ // effect's claim; what is left is the next attempt's
1126
+ const began = Date.now();
1101
1127
  for (const p of prs) {
1128
+ if (Date.now() - began > MERGE_BUDGET_MS) throw new Error(`the merge attempt spent its ${MERGE_BUDGET_MS / 1000} s; the rest is merged next attempt`);
1102
1129
  let answer;
1103
1130
  try { answer = mergeReviewedPullRequest(p.pr.url, p.commit); } catch (error) {
1104
- tx(board, (db) => event(db, e.id, 'merge_pending', { pr: p.pr.url, reviewed: p.commit, reason: String(error.message).slice(-300) }));
1105
- return false;
1131
+ const reason = String(error.message).slice(-300);
1132
+ tx(board, (db) => event(db, e.id, 'merge_pending', { pr: p.pr.url, reviewed: p.commit, reason }));
1133
+ throw new Error(`merging ${p.pr.url} at ${p.commit}: ${reason}`);
1106
1134
  }
1107
1135
  if (answer.merged) { merges.push({ pr: p.pr.url, reviewed: p.commit, merge: answer.sha }); continue; }
1108
1136
  const reason = answer.moved ? `${p.pr.url} is no longer at its reviewed head ${p.commit}${answer.head ? ` (now ${answer.head})` : ''}; what was pushed after the verdict needs its own review` : `${p.pr.url} was closed without merging`;
@@ -1312,7 +1340,9 @@ export function reconcileCallerLaunch(board, sid) {
1312
1340
  }
1313
1341
  }
1314
1342
 
1315
- function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes) {
1343
+ /** The board's agent records joined to Teams: declarations, ends, session ownership and usage. Teams I/O, so it runs in
1344
+ * the delivery pass (`deliver`), never in a dispatch round. */
1345
+ function synchronizeAgents(board, fleet, report, entry) {
1316
1346
  // A delayed launch receipt/catalogue event binds the same declared agent, never another identity.
1317
1347
  // A resumed session is held by its newest run; an earlier run of the same session no longer owns it.
1318
1348
  const agentRuns=[...new Map(view(board,db=>db.prepare("SELECT id,task_id,metadata,ended_at FROM task_runs WHERE json_extract(metadata,'$.supercode.agent_id') IS NOT NULL ORDER BY id").all())
@@ -1344,7 +1374,7 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
1344
1374
  let scope;try{scope=agentScope(context);}catch(error){scope={offline:true};report.failed.push(['agents',`agent declarations wait: ${error.message}`]);}
1345
1375
  if(!scope.offline){
1346
1376
  orgs.set(scope.org,orgs.get(scope.org)??context);
1347
- for(const pending of synchronizeLocalAgents({root:board.root,profiles:entry.profiles,context}))report.queued.push([pending.agent,`agent declaration pending: ${pending.error}`]);
1377
+ for(const pending of synchronizeLocalAgents({root:board.root,profiles:entry.profiles,context}))(pending.failed?report.failed:report.queued).push([pending.agent,`agent declaration ${pending.failed?`failed for good after ${pending.attempts} refusals`:'pending'}: ${pending.error}`]);
1348
1378
  }
1349
1379
  orgs.delete('local');
1350
1380
  // G3: an ended run's session usage, read once from its harness's own records on its machine
@@ -1371,9 +1401,14 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
1371
1401
  for(const [org,context] of orgs)for(const pending of synchronizeAgentEnds(board.root,org,{context}))report.queued.push([pending.agent,`agent end synchronization pending: ${pending.error}`]);
1372
1402
  for(const [org,context] of orgs)for(const row of pendingAgentRuns(board.root,org)){
1373
1403
  const launch=agentRuns.find(run=>String(run.id)===String(row.run_id));
1404
+ // each run is claimed by compare-and-set on its own `published` before it is joined; another pass's claim skips it
1405
+ if(!claimAgentRun(board.root,org,row))continue;
1374
1406
  try{synchronizeAgentRun(board.root,org,row,{context,intended:json(launch?.metadata,{})?.supercode?.agent_profile_intent});}
1375
- catch(error){if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&syncRowAlive(fleet,row)===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
1407
+ catch(error){releaseAgentRun(board.root,org,row);if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&syncRowAlive(fleet,row)===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
1376
1408
  }
1409
+ }
1410
+
1411
+ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes) {
1377
1412
  const deferred=view(board,db=>db.prepare("SELECT t.id,r.id run_id,COALESCE(json_extract(r.metadata,'$.supercode.launched_at'),r.started_at) launched_at FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
1378
1413
  // A deferred launch was never issued (a reading or door was not yet available): it is the start its manager, creator
1379
1414
  // or workflow asked for, carried out within the launch window; supervise surfaces it once when the window ends. The
@@ -1925,7 +1960,7 @@ function reconnectContinuedSessions(board, fleet, entry, report) {
1925
1960
  // Preserve the ended interval as evidence before reopening supervision on the same run.
1926
1961
  event(db,row.task_id,'session_continued',{session_id:saved.session_id,from_pane:saved.pane,pane,prior_outcome:row.outcome,prior_ended_at:row.ended_at,incident:saved.incident??null},row.id);
1927
1962
  db.prepare("UPDATE task_runs SET ended_at=NULL,status='running',outcome=NULL,error=NULL,metadata=?,claim_lock=?,claim_expires=? WHERE id=?")
1928
- .run(JSON.stringify({...json(row.metadata,{}),supercode:session}),claimer(),now()+900,row.id);
1963
+ .run(JSON.stringify({...json(db.prepare('SELECT metadata FROM task_runs WHERE id=?').get(row.id)?.metadata,{}),supercode:session}),claimer(),now()+900,row.id); // re-read: the run's observations just recorded stay
1929
1964
  db.prepare('UPDATE tasks SET current_run_id=?,claim_lock=?,claim_expires=? WHERE id=?').run(row.id,claimer(),now()+900,row.task_id);
1930
1965
  projectStatus(db,board,entry,row.task_id);
1931
1966
  event(db,row.task_id,'dispatch_released',{reason:'verified manual continuation',session:addressOf(session),incident:saved.incident??null});
@@ -1990,10 +2025,43 @@ export async function alertManagers(root, slug, text, key) {
1990
2025
  * and marked board_message_sent. A question goes under the id its card waits on; a resend of a sent one is a no-op
1991
2026
  * (the id is the send's idempotency key). One that fails is tried again next tick.
1992
2027
  */
2028
+ const MAIL_READY_SECONDS = 300;
2029
+ /** Each board message not yet sent nor abandoned gets its effect row (once: the insert reads that none exists). */
2030
+ export function queueBoardMail(db) {
2031
+ db.prepare(`INSERT INTO supercode_effects (effect)
2032
+ SELECT json_object('kind', 'board_message', 'id', original.task_id, 'event', original.id) FROM task_events original
2033
+ WHERE original.kind = 'board_message'
2034
+ AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind = 'board_message_sent' AND sent.task_id = original.task_id AND json_extract(sent.payload, '$.event') = original.id)
2035
+ AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind = 'board_message_abandoned' AND gone.task_id = original.task_id AND json_extract(gone.payload, '$.event') = original.id)
2036
+ AND NOT EXISTS (SELECT 1 FROM supercode_effects e WHERE json_extract(e.effect, '$.kind') = 'board_message' AND json_extract(e.effect, '$.event') = original.id)`).run();
2037
+ }
2038
+
2039
+ /**
2040
+ * One board message's effect: its mail (with any batch it is in) sent under its fixed message id, so a second sender
2041
+ * is dropped by the mailbox. Answers true once it is sent, 'wait' while its batch window is open; throws the failure.
2042
+ */
2043
+ export async function sendBoardMessage(board, e) {
2044
+ const state = () => view(board, (db) => db.prepare(`SELECT
2045
+ EXISTS (SELECT 1 FROM task_events WHERE kind='board_message_sent' AND task_id=? AND json_extract(payload,'$.event')=?) AS sent,
2046
+ EXISTS (SELECT 1 FROM task_events WHERE kind='board_message_abandoned' AND task_id=? AND json_extract(payload,'$.event')=?) AS gone`).get(e.id, e.event, e.id, e.event));
2047
+ if (state().sent || state().gone || !e.event) return true;
2048
+ const report = { undelivered: [] };
2049
+ await sendBoardMessages(board, report, e.id);
2050
+ if (state().sent) return true;
2051
+ const failed = report.undelivered.find(([task]) => task === e.id);
2052
+ if (failed) throw new Error(failed[2]);
2053
+ // a mail waits for its batch window (at most a minute); one still not ready well after that is an attempt that failed
2054
+ const recorded = view(board, (db) => db.prepare('SELECT created_at FROM task_events WHERE id = ?').get(e.event)?.created_at);
2055
+ if (recorded && now() - Number(recorded) > MAIL_READY_SECONDS) throw new Error(`the mail was not ready to send ${now() - Number(recorded)} s after it was recorded`);
2056
+ return 'wait';
2057
+ }
2058
+
1993
2059
  export async function sendBoardMessages(board, report, taskId = null) {
1994
2060
  const pending = view(board, db => db.prepare(`SELECT 1 FROM task_events original WHERE kind='board_message'
1995
2061
  AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind='board_message_sent'
1996
- AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id) LIMIT 1`).get());
2062
+ AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
2063
+ AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind='board_message_abandoned'
2064
+ AND gone.task_id=original.task_id AND json_extract(gone.payload,'$.event')=original.id) LIMIT 1`).get());
1997
2065
  if (!pending) return;
1998
2066
  const planned = tx(board, db => pendingRequestMail(db, taskId));
1999
2067
  if (Number.isFinite(planned.nextAt)) report.next_mail_at = Math.min(report.next_mail_at ?? Infinity, planned.nextAt);
@@ -2006,17 +2074,13 @@ export async function sendBoardMessages(board, report, taskId = null) {
2006
2074
  const receipt = await send(board, boardAddress(board.label), board.label, mail.to, mail.text, mail.message_id ?? `board-message-${first.id}`, { ask: Boolean(mail.message_id), subject });
2007
2075
  if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'message was not filed');
2008
2076
  tx(board, db => {
2009
- for (const member of mail.members) event(db, member.task_id, 'board_message_sent', { event: member.id, to: mail.to, receipt, ...(mail.message_id ? { message_id: mail.message_id, request_id: member.message_id ?? mail.message_id } : {}) });
2077
+ for (const member of mail.members) if (!db.prepare("SELECT 1 FROM task_events WHERE kind='board_message_sent' AND task_id=? AND json_extract(payload,'$.event')=?").get(member.task_id, member.id)) event(db, member.task_id, 'board_message_sent', { event: member.id, to: mail.to, receipt, ...(mail.message_id ? { message_id: mail.message_id, request_id: member.message_id ?? mail.message_id } : {}) });
2010
2078
  });
2011
2079
  } catch (error) {
2012
2080
  const reason = String(error.message).slice(-200);
2013
2081
  for (const member of mail.members) report.undelivered.push([member.task_id, mail.to, reason]);
2014
2082
  tx(board, db => {
2015
- for (const member of mail.members) {
2016
- const previous = db.prepare("SELECT payload,created_at FROM task_events WHERE task_id=? AND kind='board_message_failed' AND json_extract(payload,'$.event')=? ORDER BY id DESC LIMIT 1").get(member.task_id, member.id);
2017
- if (json(previous?.payload, {})?.reason !== reason || now() - (previous?.created_at ?? 0) >= 60)
2018
- event(db, member.task_id, 'board_message_failed', { event: member.id, to: mail.to, message_id: mail.message_id, reason });
2019
- }
2083
+ for (const member of mail.members) event(db, member.task_id, 'board_message_failed', { event: member.id, to: mail.to, message_id: mail.message_id, reason });
2020
2084
  });
2021
2085
  }
2022
2086
  }
@@ -2025,8 +2089,10 @@ export async function sendBoardMessages(board, report, taskId = null) {
2025
2089
  /** The board's own address: where its notices come from and where card updates are mailed. */
2026
2090
  export const boardAddress = (label) => `sc:${localMachine()}:board:${label.replace(/[^A-Za-z0-9_.-]/g, '-')}`;
2027
2091
 
2028
- /** Consecutive failed sends after which a subscription is dropped (Hermes's notifier drops a dead chat after 12). */
2092
+ /** Consecutive failed sends after which a chat-platform subscription is dropped (Hermes's notifier drops a dead chat after 12). */
2029
2093
  const MAX_SEND_FAILURES = 12;
2094
+ /** Attempts a supercode notice gets before it is abandoned (its subscription stays). */
2095
+ const NOTICE_ATTEMPTS = 6;
2030
2096
 
2031
2097
  /**
2032
2098
  * What an audience is told of an event: the workflow's `notify[audience][kind]` template over the card, the event's
@@ -2106,13 +2172,16 @@ function nativeEnded(address, discovered) {
2106
2172
  * A creator's or manager's subscription is never ended here: its notices wait for the session to come back.
2107
2173
  */
2108
2174
  function endWorkerSubscriptions(board, fleet, report, natively = new Map()) {
2175
+ // each worker subscriber's liveness is read before the write
2176
+ const aliveNow = new Map();
2177
+ if (fleet.ok) for (const { chat_id: address } of board.db.prepare("SELECT DISTINCT chat_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) aliveNow.set(address, subscriberAlive(fleet, address));
2109
2178
  tx(board, (db) => {
2110
2179
  const { held, latest } = sessionRecord(db);
2111
2180
  const remove = db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?");
2112
2181
  const ended = new Map();
2113
2182
  for (const row of db.prepare("SELECT task_id, chat_id, thread_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) {
2114
2183
  const address = row.chat_id, record = latest.get(address);
2115
- if (!record || record.session.released || held.has(address) || (fleet.ok && subscriberAlive(fleet, address))) continue;
2184
+ if (!record || record.session.released || held.has(address) || (fleet.ok && aliveNow.get(address) === true)) continue;
2116
2185
  const evidence = record.session.close_confirmed_at ? { by: 'board_close', run: record.run, closed_at: record.session.close_confirmed_at } : natively.get(address);
2117
2186
  if (!evidence || !remove.run(row.task_id, address, row.thread_id).changes) continue;
2118
2187
  if (!ended.has(address)) ended.set(address, { record, evidence, cards: [] });
@@ -2135,6 +2204,8 @@ async function notify(board, fleet, report, entry) {
2135
2204
  // each subscription's notices go in order; different subscribers are told side by side
2136
2205
  await pool(subscriptions(board.db).filter((sub) => sub.platform === 'supercode'), SEND_CONCURRENCY, async (sub) => {
2137
2206
  if (sub.delivery_metadata?.dependent_card && sub.delivery_metadata.waiting_recipient) return;
2207
+ // a subscription whose last send failed waits out its retry time (as an effect does) before the next attempt
2208
+ if (Number(sub.delivery_metadata?.next_at ?? 0) > now()) return;
2138
2209
  const role = ['worker', 'dependency', 'manager'].includes(sub.delivery_metadata?.role) ? sub.delivery_metadata.role : 'creator';
2139
2210
  // a session the fleet does not list this tick is told next tick: one reading is not a loss (a loaded box drops
2140
2211
  // sessions from a read, and a dropped subscription never came back), and a session that is replaced on its card
@@ -2151,7 +2222,13 @@ async function notify(board, fleet, report, entry) {
2151
2222
  // the one the next run gets): the notice is filed in its mailbox, read when it next works
2152
2223
  const working = role !== 'worker' || view(board, (db) => addressOf(currentRun(db, sub.task_id)?.metadata?.supercode) === sub.chat_id);
2153
2224
  let cursor = Number(sub.last_event_id);
2154
- let failed = null, untried = false;
2225
+ // A notice is sent first, under its fixed key (the mailbox drops a second send of it), and the cursor then only moves
2226
+ // forward past it: two passes may both send one notice, and its recipient gets it once; a send that fails, or a pass
2227
+ // that dies before its send, leaves the cursor behind the notice, so it is sent again.
2228
+ const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
2229
+ const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
2230
+ const forward = (to) => tx(board, (db) => db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(to, ...identity, to));
2231
+ let failed = null, untried = false, trying = null;
2155
2232
  try {
2156
2233
  // an event older than the workflow's notice_max_age_seconds is history, not news: it is passed over, never mailed
2157
2234
  const maxAge = Number(entry.params.notice_max_age_seconds) || null;
@@ -2160,9 +2237,12 @@ async function notify(board, fleet, report, entry) {
2160
2237
  // and answers, each its own message, in full. A notice the board already recorded as taken by this address is never sent again.
2161
2238
  // only events that would be told count: a stale one or the recipient's own act never supersedes one that would be
2162
2239
  const sendable = (ev) => { const p = json(ev.payload, {}) ?? {}; return !(maxAge && now() - Number(ev.created_at) > maxAge) && !(p.by && p.by === sub.chat_id); };
2163
- const latest = new Map(events.filter((ev) => !['commented', 'answered'].includes(ev.kind) && sendable(ev)).map((ev) => [ev.kind, ev.id]));
2240
+ // News is coalesced by its subject, not its kind alone: a later `assigned` says what an earlier one did, but each
2241
+ // subtask added, dependency linked or unlinked is its own fact, keyed by the card it names; a comment or an
2242
+ // answer is always its own message.
2243
+ const subjectOf = (ev) => { const p = json(ev.payload, {}) ?? {}; return ['commented', 'answered'].includes(ev.kind) ? `${ev.kind}:${ev.id}` : `${ev.kind}:${p.subtask ?? p.parent ?? p.child ?? ''}`; };
2244
+ const latest = new Map(events.filter((ev) => sendable(ev)).map((ev) => [subjectOf(ev), ev.id]));
2164
2245
  const sentBefore = board.db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND kind IN ('notice_delivered', 'notice_filed') AND json_extract(payload, '$.source_event') = ? AND json_extract(payload, '$.address') = ? LIMIT 1");
2165
- const advance = (id) => tx(board, (db) => db.prepare("UPDATE kanban_notify_subs SET last_event_id = ? WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ? AND last_event_id < ?").run(id, sub.task_id, sub.chat_id, sub.thread_id ?? '', id));
2166
2246
  for (const ev of events) {
2167
2247
  const p = json(ev.payload, {}) ?? {};
2168
2248
  if (maxAge && now() - Number(ev.created_at) > maxAge) { cursor = ev.id; (report.stale_notices_passed ??= 0); report.stale_notices_passed += 1; continue; }
@@ -2170,7 +2250,7 @@ async function notify(board, fleet, report, entry) {
2170
2250
  if (sentBefore.get(sub.task_id, ev.id, sub.chat_id)) { cursor = ev.id; continue; }
2171
2251
  // nobody is told of their own act
2172
2252
  if (p.by && p.by === sub.chat_id) { cursor = ev.id; continue; }
2173
- if (latest.has(ev.kind) && latest.get(ev.kind) !== ev.id) { cursor = ev.id; (report.superseded_notices_passed ??= 0); report.superseded_notices_passed += 1; continue; }
2253
+ if (latest.has(subjectOf(ev)) && latest.get(subjectOf(ev)) !== ev.id) { cursor = ev.id; (report.superseded_notices_passed ??= 0); report.superseded_notices_passed += 1; continue; }
2174
2254
  let text;
2175
2255
  if (ev.kind === 'commented') {
2176
2256
  if (p.author === sub.chat_id) { cursor = ev.id; continue; }
@@ -2180,6 +2260,7 @@ async function notify(board, fleet, report, entry) {
2180
2260
  if (text) {
2181
2261
  const body = ['worker', 'dependency'].includes(role) ? identifyNotice(ev, text) : text;
2182
2262
  if (refused.has(sub.chat_id) || stopping) { untried = true; break; }
2263
+ trying = ev.id;
2183
2264
  let receipt;
2184
2265
  try {
2185
2266
  receipt = await send(board, from, board.label, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: String(sub.chat_id).includes(':claude-code:') ? false : !working || (!entry.workflow.run && sub.delivery_mode === 'notify'), subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
@@ -2187,13 +2268,16 @@ async function notify(board, fleet, report, entry) {
2187
2268
  // board does not send it again.
2188
2269
  if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'notice was not filed');
2189
2270
  } catch (error) {
2271
+ // two passes may send one notice at once, and the second send can fail while the first is taken: a notice
2272
+ // another pass already recorded as taken is no failure
2273
+ if (sentBefore.get(sub.task_id, ev.id, sub.chat_id)) { cursor = ev.id; forward(ev.id); continue; }
2190
2274
  if (!refused.has(sub.chat_id)) refused.set(sub.chat_id, error);
2191
2275
  throw error;
2192
2276
  }
2193
- tx(board, (db) => event(db, sub.task_id, receipt.delivered ? 'notice_delivered' : 'notice_filed', { source_event: ev.id, kind: ev.kind, address: sub.chat_id, receipt }));
2277
+ // recorded once, whichever pass's send the mailbox took
2278
+ tx(board, (db) => { if (!db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND kind IN ('notice_delivered', 'notice_filed') AND json_extract(payload, '$.source_event') = ? AND json_extract(payload, '$.address') = ?").get(sub.task_id, ev.id, sub.chat_id)) event(db, sub.task_id, receipt.delivered ? 'notice_delivered' : 'notice_filed', { source_event: ev.id, kind: ev.kind, address: sub.chat_id, receipt }); });
2194
2279
  report.notified.push([sub.task_id, ev.kind, sub.chat_id]);
2195
- // the cursor follows each notice sent, so a restart mid-backlog resumes after it
2196
- advance(ev.id);
2280
+ forward(ev.id);
2197
2281
  }
2198
2282
  cursor = ev.id;
2199
2283
  }
@@ -2216,16 +2300,23 @@ async function notify(board, fleet, report, entry) {
2216
2300
  });
2217
2301
  report.undelivered.push([sub.task_id, sub.chat_id, failed]);
2218
2302
  }
2219
- // the cursor moves past what was delivered (a failed delivery is retried next tick; the mail id dedupes)
2303
+ // the cursor moves forward past what was passed over (a failed delivery is retried next pass)
2304
+ forward(cursor);
2220
2305
  tx(board, (db) => {
2221
- const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
2222
- const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
2223
- db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(cursor, ...identity, cursor);
2224
2306
  // a round that left this subscription untried (a delivery to its recipient failed on another card) says nothing of it
2225
2307
  if (untried) return;
2226
- const failures = failed ? Number(sub.delivery_metadata?.failures ?? 0) + 1 : 0;
2227
- if (failures >= MAX_SEND_FAILURES) { db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity); return; }
2228
- if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures }), ...identity);
2308
+ // A notice is tried NOTICE_ATTEMPTS times, waiting as an effect does between attempts. After its last failed
2309
+ // attempt it is abandoned (its terminal state, `notice_abandoned`, recorded once) and the cursor moves past it;
2310
+ // the subscription stays, so the notices after it are still told (dropping it lost them when the round
2311
+ // subscribed the session again from the card's newest event).
2312
+ // failures are counted per notice: a failure of a later notice starts its own count
2313
+ const failures = failed ? (Number(sub.delivery_metadata?.failing_event) === trying ? Number(sub.delivery_metadata?.failures ?? 0) : 0) + 1 : 0;
2314
+ if (failed && failures >= NOTICE_ATTEMPTS && trying) {
2315
+ event(db, sub.task_id, 'notice_abandoned', { source_event: trying, address: sub.chat_id, attempts: failures, error: failed });
2316
+ db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ?, delivery_metadata = ? ${where} AND last_event_id < ?`).run(trying, JSON.stringify({ ...sub.delivery_metadata, failures: 0, next_at: 0 }), ...identity, trying);
2317
+ return;
2318
+ }
2319
+ if (failures !== Number(sub.delivery_metadata?.failures ?? 0) || (failed && Number(sub.delivery_metadata?.failing_event) !== trying)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures, failing_event: failed ? trying : null, next_at: failed ? now() + Math.min(1800, 30 * 2 ** (failures - 1)) : 0 }), ...identity);
2229
2320
  if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity);
2230
2321
  });
2231
2322
  });
@@ -2566,17 +2657,78 @@ function capMax(cap, group, entry, lanes) {
2566
2657
  catch { return null; }
2567
2658
  }
2568
2659
 
2660
+ const ORCHESTRATOR_ENTRY = fileURLToPath(new URL('../bin/orchestrator.mjs', import.meta.url));
2661
+ const deliveries = new Map();
2662
+ /**
2663
+ * Start the home's delivery pass (`workflow deliver`) in a process of its own, unless one is already running: the
2664
+ * dispatcher's loop calls this after each round and never waits for it. `log(level, line)` hears what it failed at.
2665
+ * Answers whether one was started.
2666
+ */
2667
+ export function startDelivery(root, { log = () => {} } = {}) {
2668
+ if (deliveries.has(root)) return false;
2669
+ const child = spawnChild(process.execPath, [ORCHESTRATOR_ENTRY, 'workflow', 'deliver', '--root', root, '--json'], { stdio: ['ignore', 'pipe', 'pipe'], env: process.env });
2670
+ deliveries.set(root, child);
2671
+ let out = '', err = '';
2672
+ child.stdout.setEncoding('utf8').on('data', (c) => { out += c; });
2673
+ child.stderr.setEncoding('utf8').on('data', (c) => { err = (err + c).slice(-2000); });
2674
+ const done = () => {
2675
+ deliveries.delete(root);
2676
+ let reports = null; try { reports = JSON.parse(out); } catch { log('warn', `delivery pass failed: ${(err || out).trim().slice(-300) || 'no answer'}`); return; }
2677
+ for (const r of reports) {
2678
+ if (r.error) log('warn', `delivery on board ${r.board} failed: ${r.error}`);
2679
+ for (const [task, to, why] of r.undelivered ?? []) log('warn', `board ${r.board}: a delivery for ${task} to ${to} waits: ${why}`);
2680
+ log('info', `delivery on board ${r.board}: ${r.effects_waiting ?? 0} effect(s) waiting, ${r.effects_failed ?? 0} failed for good; took ${Object.entries(r.ms ?? {}).map(([k, v]) => `${k} ${(v / 1000).toFixed(1)}s`).join(', ')}`);
2681
+ }
2682
+ };
2683
+ child.on('error', (error) => { log('warn', `delivery pass did not start: ${error.message}`); deliveries.delete(root); });
2684
+ child.on('close', done);
2685
+ return true;
2686
+ }
2687
+
2688
+ /**
2689
+ * The delivery pass over every board of the home at `root`: what rounds recorded for after them (each due effect, with
2690
+ * its bounded attempts), board mail, subscription notices, and the board's agent records joined to Teams. It runs in
2691
+ * its own process (`workflow deliver`, started by serve after a round, one at a time), never inside a dispatch round,
2692
+ * so no recipient, door or Teams answer holds a round. Answers one report per board.
2693
+ */
2694
+ export async function deliver({ root, slugs = null }) {
2695
+ fleetRoot = root;
2696
+ const entry = await loadWorkflow(root, { fresh: true });
2697
+ const fleet = observeFleet();
2698
+ const out = [];
2699
+ for (const slug of slugs ?? listBoards(root)) {
2700
+ const report = { board: slug, ms: {}, mailed: [], queued: [], failed: [], notified: [], undelivered: [], fleet: fleet.ok ? 'read' : fleet.error };
2701
+ let board;
2702
+ try {
2703
+ board = openBoard(root, slug);
2704
+ board.label = boardLabel(root, slug);
2705
+ setActor(boardAddress(board.label));
2706
+ let mark = Date.now();
2707
+ const phase = (name) => { const t = Date.now(); report.ms[name] = t - mark; mark = t; };
2708
+ // board mail is sent as effect rows, one per message, each claimed as any effect is
2709
+ tx(board, (db) => queueBoardMail(db)); phase('mail');
2710
+ report.effects = await runEffects(board, [], { pending: true }); phase('effects');
2711
+ await notify(board, fleet, report, entry); phase('notify');
2712
+ synchronizeAgents(board, fleet, report, entry); phase('agents');
2713
+ report.effects_failed = Number(board.db.prepare('SELECT COUNT(*) AS n FROM supercode_effects WHERE failed_at IS NOT NULL').get().n);
2714
+ report.effects_waiting = Number(board.db.prepare('SELECT COUNT(*) AS n FROM supercode_effects WHERE failed_at IS NULL').get().n);
2715
+ } catch (error) { report.error = error.message; }
2716
+ finally { board?.close(); }
2717
+ out.push(report);
2718
+ }
2719
+ return out;
2720
+ }
2721
+
2569
2722
  /**
2570
2723
  * One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
2571
2724
  * `only` limits starting to one card (a create that answers whether its card started). Each board is ticked under its
2572
2725
  * version-checked card claims; no dispatcher file lock. Two rounds may run at once (the dispatcher's and a one-shot
2573
2726
  * `workflow dispatch`). An adoption request and a session close are claimed in a transaction (`newClaim`) before
2574
2727
  * anything is done for them outside the board; card starts, supervision, review requests, mailed verbs and manual
2575
- * continuations rely on their per-card transactions. Two rounds on one board at once can still both act on: queued
2576
- * effects in `runEffects` (send_to_actor, session_notice, merge_reviewed, declare_card_agent), board mail,
2577
- * subscription notices (a doubled notice_delivered has been seen live), deferred-launch retries (two panes),
2578
- * launch-receipt work (cancelUnissuedLaunch, bindLaunch), the Teams agent-run sync in reconcileLaunches, and
2579
- * board-only writes that do not re-check. Answers one report per board.
2728
+ * continuations rely on their per-card transactions. Two rounds on one board at once can still both act on:
2729
+ * deferred-launch retries (two panes), launch-receipt work (cancelUnissuedLaunch, bindLaunch), and board-only writes
2730
+ * that do not re-check. A round runs no effect and delivers nothing: it records effects (`queueEffects`) and the
2731
+ * delivery pass (`deliver`) carries them out. Answers one report per board.
2580
2732
  */
2581
2733
  export async function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true, params = {}, deploymentReceipt = false }) {
2582
2734
  fleetRoot = root;
@@ -2618,20 +2770,18 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
2618
2770
  readMailbox(board, boardAddress(board.label), report); phase('mailbox');
2619
2771
  await refreshRelationAnswers(board); phase('answers');
2620
2772
  const effects = [];
2621
- completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); await runEffects(board, effects.splice(0), { retryNotices: true }); closeRequested(board, report, { ttl: settings.ttl }); phase('supervise');
2773
+ completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); queueEffects(board, effects.splice(0)); closeRequested(board, report, { ttl: settings.ttl }); phase('supervise');
2622
2774
  report.unsettled = [];
2623
2775
  report.promoted = recomputeReady(board, effects, report.unsettled);
2624
2776
  tx(board,db=>auditEndedRuns(db,board,entry));
2625
2777
  if (dispatch) await activatePullRequestReviews(board, entry, report, effects, { only });
2626
- await sendBoardMessages(board, report);
2627
2778
  if (dispatch && !stopping) startCards(board, fleet, lanes, settings, report, entry, effects, { only, held });
2628
2779
  phase('start');
2629
2780
  if (dispatch && !only && !report.paused) observeIdle(board, fleet, settings, report, entry, effects);
2630
- // what the moves asked for, once they are committed (messages to actors, closed panes, cleaned workspaces)
2631
- await runEffects(board, effects);
2781
+ // what the moves asked for is recorded with them; the delivery pass (`deliver`) carries it out, outside the round
2782
+ queueEffects(board, effects);
2632
2783
  phase('goals');
2633
2784
  ensureSubscriptions(board, entry);
2634
- await notify(board, fleet, report, entry); phase('notify');
2635
2785
  fireHooks(board, settings, report);
2636
2786
  if (String(board.db.prepare('PRAGMA journal_mode').get()?.journal_mode).toLowerCase() === 'wal') board.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get();
2637
2787
  report.ready_waiting = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks t LEFT JOIN supercode_cards c ON c.task_id = t.id WHERE t.status = 'ready' AND t.assignee IS NOT NULL AND c.subtask_of IS NULL").get().n);
package/board/engine.mjs CHANGED
@@ -463,41 +463,105 @@ export async function goto(board, id, status, { roles = null, payload = {} } = {
463
463
  return { ...(await act(board, id, name, { roles, payload })), event: name };
464
464
  }
465
465
 
466
- /** The I/O a committed move asked for. */
467
- export async function runEffects(board, effects = [], { retryNotices = false } = {}) {
466
+ // A delivery (a notice, an actor message, an agent declaration) is tried at most EFFECT_ATTEMPTS times, waiting twice as
467
+ // long after each failed attempt (from 30 s, at most 30 min). Its last failed attempt is its terminal state: the row
468
+ // keeps `failed_at` and the error, the card records `effect_failed` once, diagnose shows it, and nothing tries it again.
469
+ // A board-state effect the dispatcher owns (OWNED: merging the reviewed PRs, cleaning a workspace) is never abandoned:
470
+ // it keeps trying at the capped wait, and at EFFECT_ATTEMPTS failed attempts its card records `effect_stalled` and the
471
+ // managers are mailed once, so a passed card whose merge keeps failing is seen, and still heals when the cause clears.
472
+ export const EFFECT_ATTEMPTS = 6;
473
+ const OWNED = new Set(['merge_reviewed', 'cleanup_workspace']);
474
+ const effectWait = (attempts) => Math.min(1800, 30 * 2 ** Math.max(0, attempts - 1));
475
+ // A delivery pass claims a row by compare-and-set on its next_at (set ahead only while the row is due) before trying it,
476
+ // so two passes (serve's, the hosted loop's, a person's `workflow deliver`, a verb's own) never try one effect at once.
477
+ // The claim outlasts any one attempt: each effect's I/O is bounded by its own timeouts (a send's 180 s, a merge's
478
+ // MERGE_BUDGET plus one in-flight GitHub call), all well inside EFFECT_CLAIM.
479
+ const EFFECT_CLAIM = 900;
480
+
481
+ /**
482
+ * The effects a committed move asked for. Board writes the round owns (invalidating descendants, asking a session's
483
+ * close, stopping a local worker) are made here, in the round; everything that reaches outside the board is recorded
484
+ * to run after it, in the delivery pass, never inside a dispatch round.
485
+ */
486
+ export function queueEffects(board, effects = []) {
487
+ // a board message is its own effect row (dispatch.mjs queueBoardMail), so the move's mail hint is not recorded
488
+ const fresh = effects.filter((e) => !e.durableId && e.kind !== 'run' && !(e.kind === 'board_message' && !e.event));
489
+ const now_ = fresh.filter((e) => ['invalidate_descendants', 'close_session', 'run_ended', 'stop_run'].includes(e.kind));
490
+ const later = fresh.filter((e) => !now_.includes(e));
491
+ for (const e of now_) {
492
+ if (e.kind === 'invalidate_descendants') tx(board, (db) => invalidateDescendants(db, board, e.id));
493
+ else if (e.kind === 'close_session') tx(board, (db) => requestSessionClose(db, e.id, { slot: e.slot, through: e.runId }));
494
+ else if (e.kind === 'run_ended') { if (e.outcome === 'reclaimed') stopWorker(board, e.runId); tx(board, (db) => requestSessionClose(db, e.id, { runId: e.runId })); }
495
+ else if (e.kind === 'stop_run') stopWorker(board, e.runId);
496
+ }
497
+ if (later.length) tx(board, (db) => { for (const e of later) db.prepare('INSERT INTO supercode_effects (effect) VALUES (?)').run(JSON.stringify(e)); });
498
+ }
499
+
500
+ /** The I/O a committed move asked for: the move's own `effects`, and with `pending` every recorded effect that is due. */
501
+ export async function runEffects(board, effects = [], { pending: due = false } = {}) {
468
502
  const requested = new Set(effects.map((e) => e.durableId).filter(Boolean));
469
- const pending = board.db.prepare('SELECT id, effect FROM supercode_effects ORDER BY id').all()
470
- .filter((p) => retryNotices || requested.has(p.id));
471
- const durable = new Set(pending.map((p) => p.id));
472
- const combined = [...pending.map((p) => ({ ...json(p.effect, {}), durableId: p.id })), ...effects.filter((e) => !e.durableId || !durable.has(e.durableId))];
503
+ const t = now();
504
+ const rows = board.db.prepare('SELECT id, effect, attempts, next_at FROM supercode_effects WHERE failed_at IS NULL ORDER BY id').all()
505
+ .filter((p) => (requested.has(p.id) || due) && Number(p.next_at ?? 0) <= t);
506
+ const durable = new Set(rows.map((p) => p.id));
507
+ const attempts = new Map(rows.map((p) => [p.id, Number(p.attempts ?? 0)]));
508
+ const combined = [...rows.map((p) => ({ ...json(p.effect, {}), durableId: p.id })), ...effects.filter((e) => !e.durableId || (!durable.has(e.durableId) && !requested.has(e.durableId)))];
473
509
  combined.sort((a,b) => Number(b.kind === 'session_notice') - Number(a.kind === 'session_notice'));
510
+ // what this call did, for its caller's report: effect ids done, tried and kept, and skipped as another's claim
511
+ const did = { done: [], kept: [], claimed_elsewhere: [] };
474
512
  for (const e of combined) {
475
- if (e.kind === 'session_notice') {
476
- const { sessionNotice } = await import('./dispatch.mjs');
477
- if (!await sessionNotice(board, e)) continue;
478
- }
479
- else if (e.kind === 'cleanup_workspace') cleanupWorkspace(board, e.id);
480
- else if (e.kind === 'declare_card_agent') {
481
- const {declareCardAgentEffect}=await import('./agent-declarations.mjs');
482
- if(!declareCardAgentEffect(board,e))continue;
483
- }
484
- else if (e.kind === 'board_message') {
485
- const { sendBoardMessages } = await import('./dispatch.mjs');
486
- await sendBoardMessages(board, { undelivered: [] }, e.id);
487
- }
488
- else if (e.kind === 'merge_reviewed') {
489
- const { mergeReviewed } = await import('./dispatch.mjs');
490
- if (await mergeReviewed(board, e) === false) continue;
491
- }
492
- else if (e.kind === 'invalidate_descendants') tx(board, (db) => invalidateDescendants(db, board, e.id));
493
- else if (e.kind === 'stop_run') stopWorker(board, e.runId);
494
- else if (e.kind === 'close_session' || e.kind === 'send_to_actor' || e.kind === 'run_ended') {
495
- if (e.kind === 'run_ended' && e.outcome === 'reclaimed') stopWorker(board, e.runId);
496
- const { sessionEffect } = await import('./dispatch.mjs');
497
- if (await sessionEffect(board, e) === false) continue;
498
- }
499
- if (e.durableId) tx(board, (db) => db.prepare('DELETE FROM supercode_effects WHERE id = ?').run(e.durableId));
513
+ // the claim: taken only while the row is due and not failed; another pass holding it answers no change
514
+ if (e.durableId && !tx(board, (db) => db.prepare('UPDATE supercode_effects SET next_at = ? WHERE id = ? AND failed_at IS NULL AND next_at <= ?').run(now() + EFFECT_CLAIM, e.durableId, now()).changes)) { did.claimed_elsewhere.push(e.durableId); continue; }
515
+ let done, error = null;
516
+ try { done = await runEffect(board, e); } catch (thrown) { done = false; error = String(thrown?.message ?? thrown).slice(0, 500); }
517
+ if (!e.durableId) continue;
518
+ // a mail inside its batch window is not an attempt: it is looked at again in a few seconds
519
+ if (done === 'wait') { tx(board, (db) => db.prepare('UPDATE supercode_effects SET next_at = ? WHERE id = ?').run(now() + 5, e.durableId)); did.kept.push(e.durableId); continue; }
520
+ if (done) { tx(board, (db) => db.prepare('DELETE FROM supercode_effects WHERE id = ?').run(e.durableId)); did.done.push(`${e.durableId}:${e.kind}`); continue; }
521
+ did.kept.push(e.durableId);
522
+ const made = (attempts.get(e.durableId) ?? 0) + 1;
523
+ tx(board, (db) => {
524
+ const row = db.prepare('SELECT failed_at, last_error FROM supercode_effects WHERE id = ?').get(e.durableId);
525
+ if (!row || row.failed_at) return;
526
+ const reason = error ?? row.last_error ?? `${e.kind} was not accepted`;
527
+ const card = e.id && db.prepare('SELECT 1 FROM tasks WHERE id = ?').get(e.id) ? e.id : null;
528
+ if (OWNED.has(e.kind)) {
529
+ db.prepare('UPDATE supercode_effects SET attempts = ?, next_at = ?, last_error = ? WHERE id = ?').run(made, now() + effectWait(made), reason, e.durableId);
530
+ if (made === EFFECT_ATTEMPTS && card) {
531
+ event(db, card, 'effect_stalled', { effect: e.durableId, kind: e.kind, attempts: made, error: reason, retry_every_seconds: effectWait(made) }, e.runId ?? null);
532
+ const to = [].concat(workflowFor(board.root).params?.managers ?? []).find(Boolean);
533
+ if (to) event(db, card, 'board_message', { to, text: `${card}: the board's ${e.kind.replace('_', ' ')} has failed ${made} times and keeps trying every ${effectWait(made) / 60} min: ${reason}` }, null);
534
+ }
535
+ } else if (made >= EFFECT_ATTEMPTS) {
536
+ db.prepare('UPDATE supercode_effects SET attempts = ?, failed_at = ?, last_error = ? WHERE id = ?').run(made, now(), reason, e.durableId);
537
+ if (card) event(db, card, 'effect_failed', { effect: e.durableId, kind: e.kind, attempts: made, error: reason, address: e.address ?? null }, e.runId ?? null);
538
+ if (card && e.kind === 'board_message' && e.event) event(db, card, 'board_message_abandoned', { event: e.event, attempts: made, reason });
539
+ } else db.prepare('UPDATE supercode_effects SET attempts = ?, next_at = ?, last_error = ? WHERE id = ?').run(made, now() + effectWait(made), reason, e.durableId);
540
+ });
541
+ }
542
+ return did;
543
+ }
544
+
545
+ /** One effect's I/O: true when it is done, false when it is not done yet (the caller counts the attempt). */
546
+ async function runEffect(board, e) {
547
+ if (e.kind === 'session_notice') { const { sessionNotice } = await import('./dispatch.mjs'); return await sessionNotice(board, e) !== false; }
548
+ if (e.kind === 'cleanup_workspace') { cleanupWorkspace(board, e.id); return true; }
549
+ if (e.kind === 'declare_card_agent') { const { declareCardAgentEffect } = await import('./agent-declarations.mjs'); return declareCardAgentEffect(board, e) !== false; }
550
+ if (e.kind === 'board_message') {
551
+ const { sendBoardMessage, sendBoardMessages } = await import('./dispatch.mjs');
552
+ // a verb's own ask sends its card's mail at once; the mail's own effect row then finds it sent
553
+ if (!e.event) { await sendBoardMessages(board, { undelivered: [] }, e.id); return true; }
554
+ return await sendBoardMessage(board, e);
500
555
  }
556
+ if (e.kind === 'merge_reviewed') { const { mergeReviewed } = await import('./dispatch.mjs'); return await mergeReviewed(board, e) !== false; }
557
+ if (e.kind === 'invalidate_descendants') { tx(board, (db) => invalidateDescendants(db, board, e.id)); return true; }
558
+ if (e.kind === 'stop_run') { stopWorker(board, e.runId); return true; }
559
+ if (e.kind === 'close_session' || e.kind === 'send_to_actor' || e.kind === 'run_ended') {
560
+ if (e.kind === 'run_ended' && e.outcome === 'reclaimed') stopWorker(board, e.runId);
561
+ const { sessionEffect } = await import('./dispatch.mjs');
562
+ return await sessionEffect(board, e) !== false;
563
+ }
564
+ return true;
501
565
  }
502
566
 
503
567
  // a run taken back stops its one-shot worker on this host (Hermes's _terminate_reclaimed_worker); a pane session is
@@ -903,6 +967,18 @@ export function diagnose(db, task, { failureLimit = DEFAULT_FAILURE_LIMIT, stran
903
967
  const session = lastSession(db, task.id);
904
968
  if (session?.machine && !fleet.online.has(session.machine) && !fleet.unknown?.has(session.machine)) out.push({ kind: 'machine_offline', severity: 'warning', title: `its machine ${session.machine} is offline`, data: { machine: session.machine }, actions: [] });
905
969
  }
970
+ // what the board could not carry out for the card: a delivery failed for good, or a board effect that keeps failing
971
+ for (const row of all(db, "SELECT id, effect, attempts, failed_at, last_error FROM supercode_effects WHERE json_extract(effect, '$.id') = ? AND (failed_at IS NOT NULL OR attempts >= ?)", task.id, EFFECT_ATTEMPTS)) {
972
+ const kind = json(row.effect, {})?.kind ?? 'effect';
973
+ out.push(row.failed_at
974
+ ? { kind: 'effect_failed', severity: 'warning', title: `its ${kind} failed for good after ${row.attempts} attempts: ${row.last_error ?? 'no reason recorded'}`, data: { effect: row.id, kind, attempts: row.attempts }, actions: [] }
975
+ : { kind: 'effect_stalled', severity: 'error', title: `its ${kind} has failed ${row.attempts} times and is still tried: ${row.last_error ?? 'no reason recorded'}`, data: { effect: row.id, kind, attempts: row.attempts }, actions: [] });
976
+ }
977
+ // notices the board gave up on: each one, with the event it was about and who was not told
978
+ for (const row of all(db, "SELECT payload FROM task_events WHERE task_id = ? AND kind = 'notice_abandoned' ORDER BY id DESC LIMIT 20", task.id)) {
979
+ const p = json(row.payload, {}) ?? {};
980
+ out.push({ kind: 'notice_abandoned', severity: 'warning', title: `the notice of event ${p.source_event} to ${p.address} was abandoned after ${p.attempts} attempts: ${p.error ?? 'no reason recorded'}`, data: { event: p.source_event, address: p.address, attempts: p.attempts }, actions: [] });
981
+ }
906
982
  const order = ['critical', 'error', 'warning'];
907
983
  return out.sort((a, b) => order.indexOf(a.severity) - order.indexOf(b.severity));
908
984
  }
@@ -3,7 +3,9 @@ import { event, json, now } from './store.mjs';
3
3
  export function pendingRequestMail(db, taskId = null) {
4
4
  const pending = db.prepare(`SELECT id, task_id, payload, created_at FROM task_events original
5
5
  WHERE kind='board_message' AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind='board_message_sent'
6
- AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id) ORDER BY id`).all()
6
+ AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
7
+ AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind='board_message_abandoned'
8
+ AND gone.task_id=original.task_id AND json_extract(gone.payload,'$.event')=original.id) ORDER BY id`).all()
7
9
  .map(row => ({ ...row, mail: json(row.payload, {}) ?? {} }));
8
10
  if (!pending.length) return { ready: [], nextAt: Infinity };
9
11
  const batches = db.prepare("SELECT payload FROM task_events WHERE kind='board_message_batch' AND id>=? ORDER BY id").all(pending[0].id)
package/board/store.mjs CHANGED
@@ -76,6 +76,9 @@ CREATE TABLE IF NOT EXISTS supercode_effects (id INTEGER PRIMARY KEY AUTOINCREME
76
76
  // `stage`: the card's workflow status where kanban.db stores another (a `reviewing` card is `running` to Hermes)
77
77
  // `since`: when the card entered its workflow status; `after_fired`: which of the status's `after` moves have run since
78
78
  // `tags`: the card's ordered tags (Hermes's `tenant` holds the first); `sponsor`, `sponsor_evidence`, `requested_by_unverified`: C4
79
+ // An effect's attempts: how many were made, when the next may be, and, once its attempts are spent, when it failed and
80
+ // why (engine.mjs runEffects). A failed effect is kept as the record of its terminal state and never tried again.
81
+ const EFFECT_COLUMNS = { attempts: 'INTEGER NOT NULL DEFAULT 0', next_at: 'INTEGER NOT NULL DEFAULT 0', failed_at: 'INTEGER', last_error: 'TEXT' };
79
82
  const OURS_COLUMNS = { workspace_anchor: 'TEXT', workspace_root: 'TEXT', subtask_of: 'TEXT', stage: 'TEXT', since: 'INTEGER', after_fired: 'TEXT', blocked_by: 'TEXT', paused: 'TEXT', archived: 'INTEGER DEFAULT 0', every: 'TEXT', next_run_at: 'INTEGER', tags: 'TEXT', sponsor: 'TEXT', sponsor_evidence: 'TEXT', requested_by_unverified: 'TEXT' };
80
83
 
81
84
  // ------------------------------------------------------------------ backings
@@ -140,6 +143,8 @@ export function layOut(db, { fresh }) {
140
143
  db.exec(PUBLICATION_SCHEMA);
141
144
  const have = new Set(db.prepare('PRAGMA table_info(supercode_cards)').all().map((c) => c.name));
142
145
  for (const [column, type] of Object.entries(OURS_COLUMNS)) if (!have.has(column)) db.exec(`ALTER TABLE supercode_cards ADD COLUMN ${column} ${type}`);
146
+ const effectColumns = new Set(db.prepare('PRAGMA table_info(supercode_effects)').all().map((c) => c.name));
147
+ for (const [column, type] of Object.entries(EFFECT_COLUMNS)) if (!effectColumns.has(column)) db.exec(`ALTER TABLE supercode_effects ADD COLUMN ${column} ${type}`);
143
148
  }
144
149
 
145
150
  /**
package/board.mjs CHANGED
@@ -13,7 +13,7 @@
13
13
  import { spawn } from 'node:child_process';
14
14
  import { fileURLToPath } from 'node:url';
15
15
  import { summary } from './board/cli.mjs';
16
- import { commandEnv, notifyChats, supercodeBin } from './board/dispatch.mjs';
16
+ import { commandEnv, notifyChats, startDelivery, supercodeBin } from './board/dispatch.mjs';
17
17
  import { watchBoard } from './board/wake.mjs';
18
18
  import { workflowEntry } from './board/workflow.mjs';
19
19
  import { evaluate, holds } from './board/expr.mjs';
@@ -79,6 +79,8 @@ export async function startBoard({ root, orchestration = null, log = () => {}, a
79
79
  try {
80
80
  const { reports, error } = await dispatchOnce(root);
81
81
  if (error) { log('warn', `board: tick failed: ${error}`); return; }
82
+ // what the round recorded is delivered by a pass of its own, which this loop never waits for
83
+ startDelivery(root, { log: (level, line) => log(level, `board: ${line}`) });
82
84
  // chat-platform subscriptions are delivered here, through the daemon's own channel adapters
83
85
  try { await notifyChats(root, adapters, log, wake); } catch (error) { log('warn', `board: chat notices failed: ${error.message}`); }
84
86
  for (const r of reports) {
package/loop.mjs CHANGED
@@ -24,7 +24,7 @@ import { harnessHomes } from './mcp/verbs.mjs';
24
24
  import { checkMonitor, runJobScript } from './gates.mjs';
25
25
  import { startBoard } from './board.mjs';
26
26
  import { SenderIdentityRecord } from '@volter/teams/sender-identity';
27
- import { agentDoor,agentScope,teamsUnavailable,ensureProfileAgent,localProfileAgent,localAgentRecord,localAgentSessions,synchronizeLocalAgents,ensureForkAgent,agentRunForKey,agentIntent,agentPreamble,agentEnvironment,recordAgentRun,recordNativeAgentSession,pendingAgentRuns,synchronizeAgentRun,agentRunForSession } from './agents.mjs';
27
+ import { agentDoor,agentScope,teamsUnavailable,ensureProfileAgent,localProfileAgent,localAgentRecord,localAgentSessions,synchronizeLocalAgents,ensureForkAgent,agentRunForKey,agentIntent,agentPreamble,agentEnvironment,recordAgentRun,recordNativeAgentSession,pendingAgentRuns,claimAgentRun,releaseAgentRun,synchronizeAgentRun,agentRunForSession } from './agents.mjs';
28
28
  import { materializeAgentHome } from './agent-home.mjs';
29
29
  import { localMachine } from './board/dispatch.mjs';
30
30
  import { execFile } from 'node:child_process';
@@ -201,8 +201,9 @@ export class Orchestrator {
201
201
  if(scope.offline)return;
202
202
  for(const pending of synchronizeLocalAgents({root:this.state.root,profiles:this.state.profiles,context,declarations:this.state.agents??{}}))this.log('warn',`agent ${pending.agent} declaration pending: ${pending.error}`);
203
203
  for(const row of pendingAgentRuns(this.state.root,scope.org)){
204
+ if(!claimAgentRun(this.state.root,scope.org,row))continue;
204
205
  try{synchronizeAgentRun(this.state.root,scope.org,row,{context,intended:row.profile_json?JSON.parse(row.profile_json):undefined});}
205
- catch(error){this.log('warn',`agent session ${row.session_ref}: ${error.message}`);if(error.code==='rate_limited')break;}
206
+ catch(error){releaseAgentRun(this.state.root,scope.org,row);this.log('warn',`agent session ${row.session_ref}: ${error.message}`);if(error.code==='rate_limited')break;}
206
207
  }
207
208
  }catch(error){this.log('warn',`agent session synchronization: ${error.message}`);}
208
209
  }
@@ -521,8 +522,8 @@ export class Orchestrator {
521
522
  if(!INSTRUCTION_ARGUMENTS[worker.harness])(this.runtimeAgentPreambles??=new Map()).set(lid,agentPrompt);
522
523
  const session=`${worker.harness}:${runtime.handle.runtime_id}`;
523
524
  recordAgentRun(this.state.root,scope.org,agent.identity.id,{session,machine:null,run:run_key??lid,profile:agent.intended.resolved,main,starterSession:starter_session});
524
- if(agent.synced!==false)try{synchronizeAgentRun(this.state.root,scope.org,{agent_id:agent.identity.id,session_ref:session,machine:null,main},{context,intended:agent.intended.resolved});}
525
- catch(error){this.log('warn',`agent session ${session} retained for synchronization: ${error.message}`);}
525
+ if(agent.synced!==false&&claimAgentRun(this.state.root,scope.org,{session_ref:session,published:0}))try{synchronizeAgentRun(this.state.root,scope.org,{agent_id:agent.identity.id,session_ref:session,machine:null,main},{context,intended:agent.intended.resolved});}
526
+ catch(error){releaseAgentRun(this.state.root,scope.org,{session_ref:session});this.log('warn',`agent session ${session} retained for synchronization: ${error.message}`);}
526
527
  this.enqueue(events.worker({
527
528
  profile: profileName, keyString, runtime_id: runtime.handle.runtime_id,
528
529
  ev: { kind: 'started', runtime_id: runtime.handle.runtime_id, session_id: runtime.handle.runtime_id, connection: runtime.connection, harness: worker.harness, steer },
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@volter/supercode-orchestrator",
3
- "version": "0.5.38",
3
+ "version": "0.5.40",
4
4
  "type": "module",
5
5
  "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
6
6
  "exports": {