@volter/supercode-orchestrator 0.5.38 → 0.5.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents.mjs +36 -7
- package/board/cli.mjs +11 -1
- package/board/dispatch.mjs +190 -45
- package/board/engine.mjs +102 -31
- package/board/request-mail.mjs +3 -1
- package/board/store.mjs +5 -0
- package/board.mjs +3 -1
- package/loop.mjs +5 -4
- package/package.json +1 -1
package/agents.mjs
CHANGED
|
@@ -38,6 +38,11 @@ function store(root){
|
|
|
38
38
|
CREATE TABLE IF NOT EXISTS agent_end_outbox(org TEXT NOT NULL,agent_id TEXT NOT NULL,reason TEXT NOT NULL,created_at INTEGER NOT NULL,PRIMARY KEY(org,agent_id));
|
|
39
39
|
CREATE TABLE IF NOT EXISTS agent_runtime_sessions(org TEXT NOT NULL,runtime_ref TEXT NOT NULL,session_ref TEXT NOT NULL,PRIMARY KEY(org,runtime_ref));
|
|
40
40
|
CREATE TABLE IF NOT EXISTS agent_runs(org TEXT NOT NULL,agent_id TEXT NOT NULL,session_ref TEXT NOT NULL,machine TEXT,run_id TEXT,profile_json TEXT,published INTEGER NOT NULL DEFAULT 0,ended_at INTEGER,end_reason TEXT,main INTEGER NOT NULL DEFAULT 1,starter_session TEXT,usage_json TEXT,PRIMARY KEY(org,session_ref));`);
|
|
41
|
+
// A declaration Teams refused is asked again a bounded number of times (synchronizeLocalAgents), then left failed.
|
|
42
|
+
const columns=new Set(db.prepare('PRAGMA table_info(agent_intents)').all().map(c=>c.name));
|
|
43
|
+
for(const [column,type] of Object.entries({declare_attempts:'INTEGER NOT NULL DEFAULT 0',declare_next_at:'INTEGER NOT NULL DEFAULT 0',declare_failed_at:'INTEGER',declare_error:'TEXT'}))if(!columns.has(column))db.exec(`ALTER TABLE agent_intents ADD COLUMN ${column} ${type}`);
|
|
44
|
+
// An end is claimed by the pass sending it (synchronizeAgentEnds).
|
|
45
|
+
if(!new Set(db.prepare('PRAGMA table_info(agent_end_outbox)').all().map(c=>c.name)).has('claimed_at'))db.exec('ALTER TABLE agent_end_outbox ADD COLUMN claimed_at INTEGER');
|
|
41
46
|
return db;
|
|
42
47
|
}
|
|
43
48
|
const digest=value=>createHash('sha256').update(value).digest('hex');
|
|
@@ -86,13 +91,16 @@ export function synchronizeAgentEnds(root,org,{context}={}){
|
|
|
86
91
|
try{
|
|
87
92
|
for(const row of db.prepare('SELECT * FROM agent_end_outbox WHERE org=? ORDER BY created_at LIMIT 100').all(org)){
|
|
88
93
|
if(db.prepare("SELECT 1 FROM agent_assignments WHERE org=? AND agent_id=? AND state='active'").get(org,row.agent_id)){db.prepare('DELETE FROM agent_end_outbox WHERE org=? AND agent_id=?').run(org,row.agent_id);continue;}
|
|
89
|
-
if(db.prepare('SELECT 1 FROM agent_runs WHERE org=? AND agent_id=? AND published
|
|
94
|
+
if(db.prepare('SELECT 1 FROM agent_runs WHERE org=? AND agent_id=? AND published<=0').get(org,row.agent_id)){pending.push({agent:row.agent_id,error:'session ownership publication remains pending'});continue;}
|
|
95
|
+
// claimed by compare-and-set on its claimed_at (free, or a claim older than SYNC_CLAIM_MS from a pass that died);
|
|
96
|
+
// deleted only after Teams ended the agent (the end is idempotent by its digest), handed back when it failed
|
|
97
|
+
if(db.prepare('UPDATE agent_end_outbox SET claimed_at=? WHERE org=? AND agent_id=? AND (claimed_at IS NULL OR claimed_at<?)').run(Date.now(),org,row.agent_id,Date.now()-SYNC_CLAIM_MS).changes!==1)continue;
|
|
90
98
|
try{
|
|
91
99
|
agentDoor('end',[row.agent_id,row.reason,digest(`${org}/${row.agent_id}/end/${row.created_at}`)],{context});
|
|
92
100
|
db.prepare('UPDATE agent_runs SET ended_at=COALESCE(ended_at,?),end_reason=COALESCE(end_reason,?) WHERE org=? AND agent_id=?').run(Date.now(),row.reason,org,row.agent_id);
|
|
93
101
|
db.prepare('DELETE FROM agent_end_outbox WHERE org=? AND agent_id=?').run(org,row.agent_id);
|
|
94
102
|
}
|
|
95
|
-
catch(error){pending.push({agent:row.agent_id,error:error.message});}
|
|
103
|
+
catch(error){db.prepare('UPDATE agent_end_outbox SET claimed_at=NULL WHERE org=? AND agent_id=?').run(org,row.agent_id);pending.push({agent:row.agent_id,error:error.message});}
|
|
96
104
|
}
|
|
97
105
|
return pending;
|
|
98
106
|
}finally{db.close();}
|
|
@@ -349,21 +357,36 @@ function adoptLocalOrg(root,scope){
|
|
|
349
357
|
db.exec('COMMIT');
|
|
350
358
|
}catch(error){try{db.exec('ROLLBACK');}catch{}throw error;}finally{db.close();}
|
|
351
359
|
}
|
|
360
|
+
const DECLARE_ATTEMPTS=6;
|
|
361
|
+
// A sync claim outlasts one Teams call chain (each door call bounded at 25-60 s).
|
|
362
|
+
const SYNC_CLAIM_MS=600000;
|
|
352
363
|
/** Declare every locally minted identity once Teams is reachable, under the id its sessions already carry. */
|
|
353
364
|
export function synchronizeLocalAgents({root,profiles,context,machine=null,declarations={}}){
|
|
354
365
|
const scope=agentScope(context);if(scope.offline)return [];
|
|
355
366
|
adoptLocalOrg(root,scope);
|
|
356
367
|
const db=store(root);let rows;
|
|
357
|
-
try{rows=db.prepare('SELECT * FROM agent_intents WHERE org=? AND declared=0 AND declaration_json IS NOT NULL ORDER BY created_at').all(scope.org);}finally{db.close();}
|
|
368
|
+
try{rows=db.prepare('SELECT * FROM agent_intents WHERE org=? AND declared=0 AND declaration_json IS NOT NULL AND declare_failed_at IS NULL AND declare_next_at<=? ORDER BY created_at').all(scope.org,Date.now());}finally{db.close();}
|
|
358
369
|
const pending=[],base={root,org:scope.org,server:scope.server,profiles,context,machine};
|
|
359
370
|
for(const row of rows){
|
|
360
371
|
const declaration=JSON.parse(row.declaration_json);
|
|
372
|
+
// claimed by compare-and-set on its own next attempt time, set ahead only while it is due: a second pass skips it
|
|
373
|
+
const claim=store(root);let claimed;
|
|
374
|
+
try{claimed=claim.prepare('UPDATE agent_intents SET declare_next_at=? WHERE org=? AND id=? AND declared=0 AND declare_failed_at IS NULL AND declare_next_at<=?').run(Date.now()+SYNC_CLAIM_MS,row.org,row.id,Date.now()).changes===1;}finally{claim.close();}
|
|
375
|
+
if(!claimed)continue;
|
|
361
376
|
try{
|
|
362
377
|
if(declaration.kind==='root')ensureRootAgent({...base,profile:declaration.profile});
|
|
363
378
|
else if(declaration.kind==='card')ensureCardAgent({...base,profile:declaration.profile,card:declaration.card,role:declaration.role,sponsor:declaration.sponsor,starterSession:declaration.starterSession});
|
|
364
379
|
else if(declaration.kind==='profile')ensureProfileAgent({...base,profile:declaration.profile});
|
|
365
380
|
else if(declaration.kind==='declared')ensureProfileAgent({...base,profile:declaration.profile,agent:declaration.agent,declarations});
|
|
366
|
-
}catch(error){
|
|
381
|
+
}catch(error){
|
|
382
|
+
// Teams unreachable is no answer about this declaration: the rest wait for the next pass, and no attempt is counted.
|
|
383
|
+
if(teamsUnavailable(error)){const db=store(root);try{db.prepare('UPDATE agent_intents SET declare_next_at=? WHERE org=? AND id=?').run(row.declare_next_at??0,row.org,row.id);}finally{db.close();}return [...pending,{agent:row.id,error:error.message}];}
|
|
384
|
+
// A refusal is this declaration's own: asked again after a doubling wait (one minute, at most an hour), and after
|
|
385
|
+
// DECLARE_ATTEMPTS refusals left failed, said once, and never asked again.
|
|
386
|
+
const made=Number(row.declare_attempts??0)+1,failed=made>=DECLARE_ATTEMPTS,db=store(root);
|
|
387
|
+
try{db.prepare('UPDATE agent_intents SET declare_attempts=?,declare_next_at=?,declare_failed_at=?,declare_error=? WHERE org=? AND id=?').run(made,Date.now()+Math.min(3600000,60000*2**(made-1)),failed?Date.now():null,String(error.message).slice(0,500),row.org,row.id);}finally{db.close();}
|
|
388
|
+
pending.push({agent:row.id,error:error.message,attempts:made,failed});
|
|
389
|
+
}
|
|
367
390
|
}
|
|
368
391
|
return pending;
|
|
369
392
|
}
|
|
@@ -377,7 +400,7 @@ export function recordNativeAgentSession(root,runtimeRef,session) {
|
|
|
377
400
|
db.exec('BEGIN IMMEDIATE');
|
|
378
401
|
db.prepare('INSERT INTO agent_runtime_sessions(org,runtime_ref,session_ref) VALUES (?,?,?) ON CONFLICT(org,runtime_ref) DO UPDATE SET session_ref=excluded.session_ref').run(owner.org,runtimeRef,session);
|
|
379
402
|
// A published native session remains history; an unpublished lease-only row is not a session.
|
|
380
|
-
db.prepare('DELETE FROM agent_runs WHERE org=? AND session_ref=? AND published
|
|
403
|
+
db.prepare('DELETE FROM agent_runs WHERE org=? AND session_ref=? AND published<=0').run(owner.org,runtimeRef);
|
|
381
404
|
db.exec('COMMIT');
|
|
382
405
|
}catch(error){try{db.exec('ROLLBACK');}catch{}throw error;}finally{db.close();}
|
|
383
406
|
return agentRunForSession(root,[runtimeRef]);
|
|
@@ -421,7 +444,13 @@ export function recordAgentRun(root,org,id,{session,machine=null,run,profile=nul
|
|
|
421
444
|
db.exec('COMMIT');
|
|
422
445
|
}catch(error){try{db.exec('ROLLBACK');}catch{}throw error;}finally{db.close();}
|
|
423
446
|
}
|
|
424
|
-
|
|
447
|
+
// A run waiting to be joined has published=0; a pass joining it holds published=-(the time it claimed it), by
|
|
448
|
+
// compare-and-set, and a claim older than SYNC_CLAIM_MS (a pass that died) is free again.
|
|
449
|
+
export function pendingAgentRuns(root,org){const db=store(root);try{return db.prepare('SELECT * FROM agent_runs WHERE org=? AND (published=0 OR (published<0 AND -published<?)) ORDER BY rowid LIMIT 100').all(org,Date.now()-SYNC_CLAIM_MS).filter(row=>!(retryOf(root,org,row)?.next_at>Date.now()));}finally{db.close();}}
|
|
450
|
+
/** Claim one waiting run for this pass; false when another pass holds it. */
|
|
451
|
+
export function claimAgentRun(root,org,row){const db=store(root);try{return db.prepare('UPDATE agent_runs SET published=? WHERE org=? AND session_ref=? AND published=?').run(-Date.now(),org,row.session_ref,row.published).changes===1;}finally{db.close();}}
|
|
452
|
+
/** Give a claimed run back, unjoined. */
|
|
453
|
+
export function releaseAgentRun(root,org,row){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=0 WHERE org=? AND session_ref=? AND published<0').run(org,row.session_ref);}finally{db.close();}}
|
|
425
454
|
/** Session references come from the bound runtime, never from a model's requested agent id. */
|
|
426
455
|
export function agentRunForSession(root, references) {
|
|
427
456
|
const db = store(root);
|
|
@@ -468,7 +497,7 @@ export function runBudgetRefusal(root,agent,budgets,now){
|
|
|
468
497
|
return null;
|
|
469
498
|
}
|
|
470
499
|
/** A run whose session Teams never had and that no longer runs here can never be owned there: it leaves the sync. */
|
|
471
|
-
export function abandonAgentRun(root,org,session){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=2 WHERE org=? AND session_ref=? AND published
|
|
500
|
+
export function abandonAgentRun(root,org,session){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=2 WHERE org=? AND session_ref=? AND published<=0').run(org,session);db.prepare('DELETE FROM agent_run_retries WHERE org=? AND session_ref=?').run(org,session);}finally{db.close();}}
|
|
472
501
|
export function publishAgentRun(root,org,session){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=1 WHERE org=? AND session_ref=?').run(org,session);}finally{db.close();}}
|
|
473
502
|
export function agentPreamble(agent,{main=true,previous=[]}={}){
|
|
474
503
|
const {identity:a,intended,assignments=[]}=agent;
|
package/board/cli.mjs
CHANGED
|
@@ -15,7 +15,7 @@ import { actorOf, view, backingPaths, layOut, backingOf, boardExists, boardPath,
|
|
|
15
15
|
import { isLocalWorkspace } from './workspace.mjs';
|
|
16
16
|
import { openZtrackBoard } from './ztrack.mjs';
|
|
17
17
|
import * as engine from './engine.mjs';
|
|
18
|
-
import { requestAdoption, alertManagers, boardAddress, boardLabel, boardSettings, callerSession, commandEnv, reconcileCallerLaunch, closeRequested, lanesOf, localMachine, observeFleet, paused, plan, startCard, stopDispatching, supercodeBin, tick } from './dispatch.mjs';
|
|
18
|
+
import { requestAdoption, alertManagers, boardAddress, boardLabel, boardSettings, callerSession, commandEnv, reconcileCallerLaunch, closeRequested, lanesOf, localMachine, observeFleet, paused, plan, startCard, stopDispatching, supercodeBin, tick, deliver, startDelivery } from './dispatch.mjs';
|
|
19
19
|
import { watchBoard } from './wake.mjs';
|
|
20
20
|
import { streamBoardPublications } from './event-stream.mjs';
|
|
21
21
|
import { completionOwnerOf, isLatestReviewer, loadWorkflow, requestSessionClose, scopeOf, statusOf, useWorkflow, workflowEntry, workflowFor } from './workflow.mjs';
|
|
@@ -950,6 +950,14 @@ export async function runWorkflow(argv) {
|
|
|
950
950
|
process.stdout.write(`${lines.join('\n')}\n`);
|
|
951
951
|
return 0;
|
|
952
952
|
}
|
|
953
|
+
case 'deliver': {
|
|
954
|
+
// the delivery pass: due effects, board mail, subscription notices and the agent records' Teams sync (serve
|
|
955
|
+
// starts it after each round in a process of its own; never part of a round)
|
|
956
|
+
const reports = await deliver({ root, slugs: args.board ? [slug] : null });
|
|
957
|
+
if (args.json) say(args, '', reports);
|
|
958
|
+
else for (const r of reports) process.stdout.write(`${r.board}: ${r.error ? `error: ${r.error}` : `${r.effects_waiting ?? 0} effect(s) waiting, ${r.effects_failed ?? 0} failed for good, ${r.notified.length} notified`}\n`);
|
|
959
|
+
return reports.some((r) => r.error) ? 1 : 0;
|
|
960
|
+
}
|
|
953
961
|
case 'daemon': case 'serve': return serve(root, args);
|
|
954
962
|
|
|
955
963
|
case 'gc': {
|
|
@@ -1488,6 +1496,8 @@ async function serve(root, args = {}) {
|
|
|
1488
1496
|
interval = settings.interval;
|
|
1489
1497
|
const reports = await tick({ root, lanes: state.lanes, settings, params, deploymentReceipt: true });
|
|
1490
1498
|
round += 1;
|
|
1499
|
+
// what the round recorded is delivered by a pass of its own, which this loop never waits for
|
|
1500
|
+
if (!stopped) startDelivery(root, { log: (level, line) => { if (level === 'warn' || args.verbose) log(line); } });
|
|
1491
1501
|
if (!stopped) await watchRounds(reports);
|
|
1492
1502
|
for (const r of reports) {
|
|
1493
1503
|
nextMailAt = Math.min(nextMailAt, r.next_mail_at ?? Infinity);
|
package/board/dispatch.mjs
CHANGED
|
@@ -20,7 +20,7 @@ import { readMachineFleet } from './machine-fleet.mjs';
|
|
|
20
20
|
let fleetRoot = null;
|
|
21
21
|
import { inspectCompletionEvidence, noCodeChange, openReviewPullRequests, reviewDue, withCompletionEvidence, reviewCandidate, workflowRevision, retainWorkflowTick } from './ci.mjs';
|
|
22
22
|
import { runtimeDoor } from './runtime-door.mjs';
|
|
23
|
-
import { agentScope, localCardAgent, localReaderContext, localAgentSessions, synchronizeLocalAgents, runBudgetRefusal, runsAwaitingUsage, recordRunUsage, agentPreamble, agentEnvironment, recordAgentRun, pendingAgentRuns, synchronizeAgentRun, abandonAgentRun,activeAgentAssignments,finishAgentAssignment,synchronizeAgentEnds, agentRunForSession } from '../agents.mjs';
|
|
23
|
+
import { agentScope, localCardAgent, localReaderContext, localAgentSessions, synchronizeLocalAgents, runBudgetRefusal, runsAwaitingUsage, recordRunUsage, agentPreamble, agentEnvironment, recordAgentRun, pendingAgentRuns, synchronizeAgentRun, claimAgentRun, releaseAgentRun, abandonAgentRun,activeAgentAssignments,finishAgentAssignment,synchronizeAgentEnds, agentRunForSession } from '../agents.mjs';
|
|
24
24
|
import { materializeAgentHome } from '../agent-home.mjs';
|
|
25
25
|
import { memorySnapshotSync } from '../hermes-cli.mjs';
|
|
26
26
|
import { createRequire } from 'node:module';
|
|
@@ -39,7 +39,7 @@ import { identifyNotice } from './notices.mjs';
|
|
|
39
39
|
import { pendingRequestMail } from './request-mail.mjs';
|
|
40
40
|
import {
|
|
41
41
|
cardContext, claimTtl, currentRun, DEFAULT_FAILURE_LIMIT, extendClaim, getTask, lastSession,
|
|
42
|
-
recomputeReady, runEffects, setRunSession, subscribeIn, subscriptions, reconcileDependencySubscriptions,
|
|
42
|
+
recomputeReady, runEffects, queueEffects, setRunSession, subscribeIn, subscriptions, reconcileDependencySubscriptions,
|
|
43
43
|
} from './engine.mjs';
|
|
44
44
|
|
|
45
45
|
const WORKER = fileURLToPath(new URL('../bin/hermes-worker.mjs', import.meta.url));
|
|
@@ -174,7 +174,8 @@ function unpublishedAlive(session) {
|
|
|
174
174
|
try { return nativeReading(nativeSessions(session.machine, session.harness ?? 'claude-code'), session).alive; } catch { return null; }
|
|
175
175
|
}
|
|
176
176
|
let entry = paneLists.get(session.machine);
|
|
177
|
-
if (!entry || Date.now() - entry.at >
|
|
177
|
+
if (!entry || Date.now() - entry.at > READING_MS) {
|
|
178
|
+
if (readingsOnly) return null;
|
|
178
179
|
const r = doorOn(session.machine, 'harness.v1.panes.list', {}, { timeout: 60_000 });
|
|
179
180
|
const panes = r.code === 0 ? json(r.stdout.trim(), null) : null;
|
|
180
181
|
entry = { at: Date.now(), panes: Array.isArray(panes) ? panes : null };
|
|
@@ -186,9 +187,22 @@ function unpublishedAlive(session) {
|
|
|
186
187
|
/** Native activity comes from the machine's harness discovery, not catalogue age. One discovery (up to 2000
|
|
187
188
|
* sessions, seconds each) serves a round's every native session on that machine, as paneLists serves its panes. */
|
|
188
189
|
const discoveries = new Map();
|
|
190
|
+
// A machine's pane list and harness discovery are read before a board write opens, never inside one: a door that
|
|
191
|
+
// takes seconds would hold the board's write lock past a worker's own command's busy wait. A write that needs a
|
|
192
|
+
// reading not taken yet reads it as unknown. One reading serves READING_MS.
|
|
193
|
+
const READING_MS = 15_000;
|
|
194
|
+
let readingsOnly = 0;
|
|
195
|
+
const txRead = (board, fn) => { readingsOnly += 1; try { return tx(board, fn); } finally { readingsOnly -= 1; } };
|
|
196
|
+
/** Take the readings a board write about `session` will need, before the write. */
|
|
197
|
+
function readBefore(fleet, session) {
|
|
198
|
+
if (!session?.session_id || !session.machine || session.surface === 'headless') return;
|
|
199
|
+
try { nativeSessions(session.machine, session.harness ?? 'claude-code'); } catch { /* unread: unknown inside the write */ }
|
|
200
|
+
sessionAlive(fleet, session);
|
|
201
|
+
}
|
|
189
202
|
function nativeSessions(machine, harness) {
|
|
190
203
|
const key = `${normalMachine(machine)}\u0000${harness}`, seen = discoveries.get(key);
|
|
191
|
-
if (seen && Date.now() - seen.at <=
|
|
204
|
+
if (seen && Date.now() - seen.at <= READING_MS) return seen.sessions;
|
|
205
|
+
if (readingsOnly) throw unavailable('native liveness is read before a board write, not inside it');
|
|
192
206
|
const sessions = discoverNative(machine, harness);
|
|
193
207
|
discoveries.set(key, { at: Date.now(), sessions });
|
|
194
208
|
return sessions;
|
|
@@ -301,8 +315,9 @@ function supervise(board, fleet, settings, report, entry, effects) {
|
|
|
301
315
|
try { const receipt = launchReceipt(board, session); if (receipt?.live === false && receipt.exit) exits.set(id, receipt.exit); }
|
|
302
316
|
catch (error) { report.failed.push([id, `exit receipt unavailable; retaining liveness observation: ${error.message}`]); }
|
|
303
317
|
}
|
|
318
|
+
for (const { id } of rows) readBefore(fleet, currentRun(board.db, id)?.metadata?.supercode);
|
|
304
319
|
for (const { id } of rows) {
|
|
305
|
-
|
|
320
|
+
txRead(board, (db) => {
|
|
306
321
|
const task = getTask(db, id);
|
|
307
322
|
if (!task || (entry.workflow.run ? !task.current_run_id : task.status !== 'running')) return;
|
|
308
323
|
const runRow = currentRun(db, id);
|
|
@@ -944,8 +959,10 @@ function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
944
959
|
return Number(discovered.get(key)?.find((row) => row.locator?.session_id === session.session_id)?.updated_at_ms) || null;
|
|
945
960
|
};
|
|
946
961
|
const managers = [].concat(entry.params.managers ?? []).filter(Boolean);
|
|
947
|
-
|
|
948
|
-
|
|
962
|
+
const running = board.db.prepare(entry.workflow.run ? "SELECT id FROM tasks WHERE current_run_id IS NOT NULL" : "SELECT id FROM tasks WHERE status = 'running'").all();
|
|
963
|
+
for (const { id } of running) { const session = currentRun(board.db, id)?.metadata?.supercode; readBefore(fleet, session); if (session?.session_id) updatedAt(session); }
|
|
964
|
+
for (const { id } of running) {
|
|
965
|
+
txRead(board, (db) => {
|
|
949
966
|
const task = getTask(db, id);
|
|
950
967
|
const status = statusOf(db, task);
|
|
951
968
|
if (!(entry.workflow.statuses[status]?.on?.session_idle || entry.workflow.on?.session_idle || entry.workflow.run?.on?.session_idle)) return;
|
|
@@ -1056,7 +1073,9 @@ export async function sessionNotice(board, e) {
|
|
|
1056
1073
|
: { lifecycle_notice_error: null, ...(session.lifecycle_notice_pending === e.durableId ? { lifecycle_notice_pending: null } : {}) }) });
|
|
1057
1074
|
if (!error || session?.lifecycle_notice_error !== error) event(db, e.id, error ? 'session_notice_failed' : receipt?.delivered ? 'session_notice_delivered' : 'session_notice_queued', { address: e.address, verb: e.verb, source_event: e.sourceEvent, notice: e.durableId, receipt, ...(error ? { error, retry: true } : {}) }, e.runId);
|
|
1058
1075
|
});
|
|
1059
|
-
|
|
1076
|
+
// the attempt's reason goes with it, onto the effect's row (runEffects counts the attempt)
|
|
1077
|
+
if (error) throw new Error(error);
|
|
1078
|
+
return true;
|
|
1060
1079
|
}
|
|
1061
1080
|
|
|
1062
1081
|
/** A session effect a committed move asked for: a message to the card's current actor, or closing a slot's pane. */
|
|
@@ -1076,7 +1095,10 @@ export async function sessionEffect(board, e) {
|
|
|
1076
1095
|
const payload = { notice: e.durableId, address, receipt };
|
|
1077
1096
|
if (previous?.kind !== kind || JSON.stringify(json(previous?.payload, {})?.receipt) !== JSON.stringify(receipt)) event(db, e.id, kind, payload, e.runId);
|
|
1078
1097
|
});
|
|
1079
|
-
|
|
1098
|
+
// a send its recipient's mailbox took (filed, or queued for a session between turns) is accepted: the mailbox
|
|
1099
|
+
// delivers it from there, and the board does not send it again
|
|
1100
|
+
if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'the actor notice was not filed');
|
|
1101
|
+
return true;
|
|
1080
1102
|
}
|
|
1081
1103
|
if (e.kind === 'close_session' || e.kind === 'run_ended') {
|
|
1082
1104
|
tx(board, (db) => requestSessionClose(db, e.id, e.kind === 'run_ended' ? { runId: e.runId } : { slot: e.slot, through: e.runId }));
|
|
@@ -1091,6 +1113,7 @@ export async function sessionEffect(board, e) {
|
|
|
1091
1113
|
* Once every PR is merged the card is sent `reviewed_merged` with the merge commits. Answers false (kept, tried again
|
|
1092
1114
|
* next round) on anything GitHub did not decide.
|
|
1093
1115
|
*/
|
|
1116
|
+
const MERGE_BUDGET_MS = 300_000;
|
|
1094
1117
|
export async function mergeReviewed(board, e) {
|
|
1095
1118
|
const { candidatePullRequests, mergeReviewedPullRequest } = await import('./ci.mjs');
|
|
1096
1119
|
const pass = view(board, (db) => db.prepare("SELECT id, payload FROM task_events WHERE task_id = ? AND kind = 'review_pass' ORDER BY id DESC LIMIT 1").get(e.id));
|
|
@@ -1098,11 +1121,16 @@ export async function mergeReviewed(board, e) {
|
|
|
1098
1121
|
const ci = typeof recorded === 'string' ? json(recorded, null) : recorded;
|
|
1099
1122
|
const prs = candidatePullRequests(ci).filter((p) => p?.pr?.url && p.commit);
|
|
1100
1123
|
const merges = [];
|
|
1124
|
+
// a merge attempt spends at most MERGE_BUDGET on starting GitHub calls (each bounded by its own timeout), inside its
|
|
1125
|
+
// effect's claim; what is left is the next attempt's
|
|
1126
|
+
const began = Date.now();
|
|
1101
1127
|
for (const p of prs) {
|
|
1128
|
+
if (Date.now() - began > MERGE_BUDGET_MS) throw new Error(`the merge attempt spent its ${MERGE_BUDGET_MS / 1000} s; the rest is merged next attempt`);
|
|
1102
1129
|
let answer;
|
|
1103
1130
|
try { answer = mergeReviewedPullRequest(p.pr.url, p.commit); } catch (error) {
|
|
1104
|
-
|
|
1105
|
-
|
|
1131
|
+
const reason = String(error.message).slice(-300);
|
|
1132
|
+
tx(board, (db) => event(db, e.id, 'merge_pending', { pr: p.pr.url, reviewed: p.commit, reason }));
|
|
1133
|
+
throw new Error(`merging ${p.pr.url} at ${p.commit}: ${reason}`);
|
|
1106
1134
|
}
|
|
1107
1135
|
if (answer.merged) { merges.push({ pr: p.pr.url, reviewed: p.commit, merge: answer.sha }); continue; }
|
|
1108
1136
|
const reason = answer.moved ? `${p.pr.url} is no longer at its reviewed head ${p.commit}${answer.head ? ` (now ${answer.head})` : ''}; what was pushed after the verdict needs its own review` : `${p.pr.url} was closed without merging`;
|
|
@@ -1312,7 +1340,9 @@ export function reconcileCallerLaunch(board, sid) {
|
|
|
1312
1340
|
}
|
|
1313
1341
|
}
|
|
1314
1342
|
|
|
1315
|
-
|
|
1343
|
+
/** The board's agent records joined to Teams: declarations, ends, session ownership and usage. Teams I/O, so it runs in
|
|
1344
|
+
* the delivery pass (`deliver`), never in a dispatch round. */
|
|
1345
|
+
function synchronizeAgents(board, fleet, report, entry) {
|
|
1316
1346
|
// A delayed launch receipt/catalogue event binds the same declared agent, never another identity.
|
|
1317
1347
|
// A resumed session is held by its newest run; an earlier run of the same session no longer owns it.
|
|
1318
1348
|
const agentRuns=[...new Map(view(board,db=>db.prepare("SELECT id,task_id,metadata,ended_at FROM task_runs WHERE json_extract(metadata,'$.supercode.agent_id') IS NOT NULL ORDER BY id").all())
|
|
@@ -1344,7 +1374,7 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
|
|
|
1344
1374
|
let scope;try{scope=agentScope(context);}catch(error){scope={offline:true};report.failed.push(['agents',`agent declarations wait: ${error.message}`]);}
|
|
1345
1375
|
if(!scope.offline){
|
|
1346
1376
|
orgs.set(scope.org,orgs.get(scope.org)??context);
|
|
1347
|
-
for(const pending of synchronizeLocalAgents({root:board.root,profiles:entry.profiles,context}))report.queued.push([pending.agent,`agent declaration pending: ${pending.error}`]);
|
|
1377
|
+
for(const pending of synchronizeLocalAgents({root:board.root,profiles:entry.profiles,context}))(pending.failed?report.failed:report.queued).push([pending.agent,`agent declaration ${pending.failed?`failed for good after ${pending.attempts} refusals`:'pending'}: ${pending.error}`]);
|
|
1348
1378
|
}
|
|
1349
1379
|
orgs.delete('local');
|
|
1350
1380
|
// G3: an ended run's session usage, read once from its harness's own records on its machine
|
|
@@ -1371,9 +1401,14 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
|
|
|
1371
1401
|
for(const [org,context] of orgs)for(const pending of synchronizeAgentEnds(board.root,org,{context}))report.queued.push([pending.agent,`agent end synchronization pending: ${pending.error}`]);
|
|
1372
1402
|
for(const [org,context] of orgs)for(const row of pendingAgentRuns(board.root,org)){
|
|
1373
1403
|
const launch=agentRuns.find(run=>String(run.id)===String(row.run_id));
|
|
1404
|
+
// each run is claimed by compare-and-set on its own `published` before it is joined; another pass's claim skips it
|
|
1405
|
+
if(!claimAgentRun(board.root,org,row))continue;
|
|
1374
1406
|
try{synchronizeAgentRun(board.root,org,row,{context,intended:json(launch?.metadata,{})?.supercode?.agent_profile_intent});}
|
|
1375
|
-
catch(error){if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&syncRowAlive(fleet,row)===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
|
|
1407
|
+
catch(error){releaseAgentRun(board.root,org,row);if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&syncRowAlive(fleet,row)===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
|
|
1376
1408
|
}
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes) {
|
|
1377
1412
|
const deferred=view(board,db=>db.prepare("SELECT t.id,r.id run_id,COALESCE(json_extract(r.metadata,'$.supercode.launched_at'),r.started_at) launched_at FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
|
|
1378
1413
|
// A deferred launch was never issued (a reading or door was not yet available): it is the start its manager, creator
|
|
1379
1414
|
// or workflow asked for, carried out within the launch window; supervise surfaces it once when the window ends. The
|
|
@@ -1925,7 +1960,7 @@ function reconnectContinuedSessions(board, fleet, entry, report) {
|
|
|
1925
1960
|
// Preserve the ended interval as evidence before reopening supervision on the same run.
|
|
1926
1961
|
event(db,row.task_id,'session_continued',{session_id:saved.session_id,from_pane:saved.pane,pane,prior_outcome:row.outcome,prior_ended_at:row.ended_at,incident:saved.incident??null},row.id);
|
|
1927
1962
|
db.prepare("UPDATE task_runs SET ended_at=NULL,status='running',outcome=NULL,error=NULL,metadata=?,claim_lock=?,claim_expires=? WHERE id=?")
|
|
1928
|
-
.run(JSON.stringify({...json(row.metadata,{}),supercode:session}),claimer(),now()+900,row.id);
|
|
1963
|
+
.run(JSON.stringify({...json(db.prepare('SELECT metadata FROM task_runs WHERE id=?').get(row.id)?.metadata,{}),supercode:session}),claimer(),now()+900,row.id); // re-read: the run's observations just recorded stay
|
|
1929
1964
|
db.prepare('UPDATE tasks SET current_run_id=?,claim_lock=?,claim_expires=? WHERE id=?').run(row.id,claimer(),now()+900,row.task_id);
|
|
1930
1965
|
projectStatus(db,board,entry,row.task_id);
|
|
1931
1966
|
event(db,row.task_id,'dispatch_released',{reason:'verified manual continuation',session:addressOf(session),incident:saved.incident??null});
|
|
@@ -1990,10 +2025,43 @@ export async function alertManagers(root, slug, text, key) {
|
|
|
1990
2025
|
* and marked board_message_sent. A question goes under the id its card waits on; a resend of a sent one is a no-op
|
|
1991
2026
|
* (the id is the send's idempotency key). One that fails is tried again next tick.
|
|
1992
2027
|
*/
|
|
2028
|
+
const MAIL_READY_SECONDS = 300;
|
|
2029
|
+
/** Each board message not yet sent nor abandoned gets its effect row (once: the insert reads that none exists). */
|
|
2030
|
+
export function queueBoardMail(db) {
|
|
2031
|
+
db.prepare(`INSERT INTO supercode_effects (effect)
|
|
2032
|
+
SELECT json_object('kind', 'board_message', 'id', original.task_id, 'event', original.id) FROM task_events original
|
|
2033
|
+
WHERE original.kind = 'board_message'
|
|
2034
|
+
AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind = 'board_message_sent' AND sent.task_id = original.task_id AND json_extract(sent.payload, '$.event') = original.id)
|
|
2035
|
+
AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind = 'board_message_abandoned' AND gone.task_id = original.task_id AND json_extract(gone.payload, '$.event') = original.id)
|
|
2036
|
+
AND NOT EXISTS (SELECT 1 FROM supercode_effects e WHERE json_extract(e.effect, '$.kind') = 'board_message' AND json_extract(e.effect, '$.event') = original.id)`).run();
|
|
2037
|
+
}
|
|
2038
|
+
|
|
2039
|
+
/**
|
|
2040
|
+
* One board message's effect: its mail (with any batch it is in) sent under its fixed message id, so a second sender
|
|
2041
|
+
* is dropped by the mailbox. Answers true once it is sent, 'wait' while its batch window is open; throws the failure.
|
|
2042
|
+
*/
|
|
2043
|
+
export async function sendBoardMessage(board, e) {
|
|
2044
|
+
const state = () => view(board, (db) => db.prepare(`SELECT
|
|
2045
|
+
EXISTS (SELECT 1 FROM task_events WHERE kind='board_message_sent' AND task_id=? AND json_extract(payload,'$.event')=?) AS sent,
|
|
2046
|
+
EXISTS (SELECT 1 FROM task_events WHERE kind='board_message_abandoned' AND task_id=? AND json_extract(payload,'$.event')=?) AS gone`).get(e.id, e.event, e.id, e.event));
|
|
2047
|
+
if (state().sent || state().gone || !e.event) return true;
|
|
2048
|
+
const report = { undelivered: [] };
|
|
2049
|
+
await sendBoardMessages(board, report, e.id);
|
|
2050
|
+
if (state().sent) return true;
|
|
2051
|
+
const failed = report.undelivered.find(([task]) => task === e.id);
|
|
2052
|
+
if (failed) throw new Error(failed[2]);
|
|
2053
|
+
// a mail waits for its batch window (at most a minute); one still not ready well after that is an attempt that failed
|
|
2054
|
+
const recorded = view(board, (db) => db.prepare('SELECT created_at FROM task_events WHERE id = ?').get(e.event)?.created_at);
|
|
2055
|
+
if (recorded && now() - Number(recorded) > MAIL_READY_SECONDS) throw new Error(`the mail was not ready to send ${now() - Number(recorded)} s after it was recorded`);
|
|
2056
|
+
return 'wait';
|
|
2057
|
+
}
|
|
2058
|
+
|
|
1993
2059
|
export async function sendBoardMessages(board, report, taskId = null) {
|
|
1994
2060
|
const pending = view(board, db => db.prepare(`SELECT 1 FROM task_events original WHERE kind='board_message'
|
|
1995
2061
|
AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind='board_message_sent'
|
|
1996
|
-
AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
|
|
2062
|
+
AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
|
|
2063
|
+
AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind='board_message_abandoned'
|
|
2064
|
+
AND gone.task_id=original.task_id AND json_extract(gone.payload,'$.event')=original.id) LIMIT 1`).get());
|
|
1997
2065
|
if (!pending) return;
|
|
1998
2066
|
const planned = tx(board, db => pendingRequestMail(db, taskId));
|
|
1999
2067
|
if (Number.isFinite(planned.nextAt)) report.next_mail_at = Math.min(report.next_mail_at ?? Infinity, planned.nextAt);
|
|
@@ -2006,17 +2074,13 @@ export async function sendBoardMessages(board, report, taskId = null) {
|
|
|
2006
2074
|
const receipt = await send(board, boardAddress(board.label), board.label, mail.to, mail.text, mail.message_id ?? `board-message-${first.id}`, { ask: Boolean(mail.message_id), subject });
|
|
2007
2075
|
if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'message was not filed');
|
|
2008
2076
|
tx(board, db => {
|
|
2009
|
-
for (const member of mail.members) event(db, member.task_id, 'board_message_sent', { event: member.id, to: mail.to, receipt, ...(mail.message_id ? { message_id: mail.message_id, request_id: member.message_id ?? mail.message_id } : {}) });
|
|
2077
|
+
for (const member of mail.members) if (!db.prepare("SELECT 1 FROM task_events WHERE kind='board_message_sent' AND task_id=? AND json_extract(payload,'$.event')=?").get(member.task_id, member.id)) event(db, member.task_id, 'board_message_sent', { event: member.id, to: mail.to, receipt, ...(mail.message_id ? { message_id: mail.message_id, request_id: member.message_id ?? mail.message_id } : {}) });
|
|
2010
2078
|
});
|
|
2011
2079
|
} catch (error) {
|
|
2012
2080
|
const reason = String(error.message).slice(-200);
|
|
2013
2081
|
for (const member of mail.members) report.undelivered.push([member.task_id, mail.to, reason]);
|
|
2014
2082
|
tx(board, db => {
|
|
2015
|
-
for (const member of mail.members) {
|
|
2016
|
-
const previous = db.prepare("SELECT payload,created_at FROM task_events WHERE task_id=? AND kind='board_message_failed' AND json_extract(payload,'$.event')=? ORDER BY id DESC LIMIT 1").get(member.task_id, member.id);
|
|
2017
|
-
if (json(previous?.payload, {})?.reason !== reason || now() - (previous?.created_at ?? 0) >= 60)
|
|
2018
|
-
event(db, member.task_id, 'board_message_failed', { event: member.id, to: mail.to, message_id: mail.message_id, reason });
|
|
2019
|
-
}
|
|
2083
|
+
for (const member of mail.members) event(db, member.task_id, 'board_message_failed', { event: member.id, to: mail.to, message_id: mail.message_id, reason });
|
|
2020
2084
|
});
|
|
2021
2085
|
}
|
|
2022
2086
|
}
|
|
@@ -2025,8 +2089,10 @@ export async function sendBoardMessages(board, report, taskId = null) {
|
|
|
2025
2089
|
/** The board's own address: where its notices come from and where card updates are mailed. */
|
|
2026
2090
|
export const boardAddress = (label) => `sc:${localMachine()}:board:${label.replace(/[^A-Za-z0-9_.-]/g, '-')}`;
|
|
2027
2091
|
|
|
2028
|
-
/** Consecutive failed sends after which a subscription is dropped (Hermes's notifier drops a dead chat after 12). */
|
|
2092
|
+
/** Consecutive failed sends after which a chat-platform subscription is dropped (Hermes's notifier drops a dead chat after 12). */
|
|
2029
2093
|
const MAX_SEND_FAILURES = 12;
|
|
2094
|
+
/** Attempts a supercode notice gets before it is abandoned (its subscription stays). */
|
|
2095
|
+
const NOTICE_ATTEMPTS = 6;
|
|
2030
2096
|
|
|
2031
2097
|
/**
|
|
2032
2098
|
* What an audience is told of an event: the workflow's `notify[audience][kind]` template over the card, the event's
|
|
@@ -2106,13 +2172,16 @@ function nativeEnded(address, discovered) {
|
|
|
2106
2172
|
* A creator's or manager's subscription is never ended here: its notices wait for the session to come back.
|
|
2107
2173
|
*/
|
|
2108
2174
|
function endWorkerSubscriptions(board, fleet, report, natively = new Map()) {
|
|
2175
|
+
// each worker subscriber's liveness is read before the write
|
|
2176
|
+
const aliveNow = new Map();
|
|
2177
|
+
if (fleet.ok) for (const { chat_id: address } of board.db.prepare("SELECT DISTINCT chat_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) aliveNow.set(address, subscriberAlive(fleet, address));
|
|
2109
2178
|
tx(board, (db) => {
|
|
2110
2179
|
const { held, latest } = sessionRecord(db);
|
|
2111
2180
|
const remove = db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?");
|
|
2112
2181
|
const ended = new Map();
|
|
2113
2182
|
for (const row of db.prepare("SELECT task_id, chat_id, thread_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) {
|
|
2114
2183
|
const address = row.chat_id, record = latest.get(address);
|
|
2115
|
-
if (!record || record.session.released || held.has(address) || (fleet.ok &&
|
|
2184
|
+
if (!record || record.session.released || held.has(address) || (fleet.ok && aliveNow.get(address) === true)) continue;
|
|
2116
2185
|
const evidence = record.session.close_confirmed_at ? { by: 'board_close', run: record.run, closed_at: record.session.close_confirmed_at } : natively.get(address);
|
|
2117
2186
|
if (!evidence || !remove.run(row.task_id, address, row.thread_id).changes) continue;
|
|
2118
2187
|
if (!ended.has(address)) ended.set(address, { record, evidence, cards: [] });
|
|
@@ -2135,6 +2204,8 @@ async function notify(board, fleet, report, entry) {
|
|
|
2135
2204
|
// each subscription's notices go in order; different subscribers are told side by side
|
|
2136
2205
|
await pool(subscriptions(board.db).filter((sub) => sub.platform === 'supercode'), SEND_CONCURRENCY, async (sub) => {
|
|
2137
2206
|
if (sub.delivery_metadata?.dependent_card && sub.delivery_metadata.waiting_recipient) return;
|
|
2207
|
+
// a subscription whose last send failed waits out its retry time (as an effect does) before the next attempt
|
|
2208
|
+
if (Number(sub.delivery_metadata?.next_at ?? 0) > now()) return;
|
|
2138
2209
|
const role = ['worker', 'dependency', 'manager'].includes(sub.delivery_metadata?.role) ? sub.delivery_metadata.role : 'creator';
|
|
2139
2210
|
// a session the fleet does not list this tick is told next tick: one reading is not a loss (a loaded box drops
|
|
2140
2211
|
// sessions from a read, and a dropped subscription never came back), and a session that is replaced on its card
|
|
@@ -2151,7 +2222,13 @@ async function notify(board, fleet, report, entry) {
|
|
|
2151
2222
|
// the one the next run gets): the notice is filed in its mailbox, read when it next works
|
|
2152
2223
|
const working = role !== 'worker' || view(board, (db) => addressOf(currentRun(db, sub.task_id)?.metadata?.supercode) === sub.chat_id);
|
|
2153
2224
|
let cursor = Number(sub.last_event_id);
|
|
2154
|
-
|
|
2225
|
+
// A notice is sent first, under its fixed key (the mailbox drops a second send of it), and the cursor then only moves
|
|
2226
|
+
// forward past it: two passes may both send one notice, and its recipient gets it once; a send that fails, or a pass
|
|
2227
|
+
// that dies before its send, leaves the cursor behind the notice, so it is sent again.
|
|
2228
|
+
const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
|
|
2229
|
+
const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
|
|
2230
|
+
const forward = (to) => tx(board, (db) => db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(to, ...identity, to));
|
|
2231
|
+
let failed = null, untried = false, trying = null;
|
|
2155
2232
|
try {
|
|
2156
2233
|
// an event older than the workflow's notice_max_age_seconds is history, not news: it is passed over, never mailed
|
|
2157
2234
|
const maxAge = Number(entry.params.notice_max_age_seconds) || null;
|
|
@@ -2162,7 +2239,6 @@ async function notify(board, fleet, report, entry) {
|
|
|
2162
2239
|
const sendable = (ev) => { const p = json(ev.payload, {}) ?? {}; return !(maxAge && now() - Number(ev.created_at) > maxAge) && !(p.by && p.by === sub.chat_id); };
|
|
2163
2240
|
const latest = new Map(events.filter((ev) => !['commented', 'answered'].includes(ev.kind) && sendable(ev)).map((ev) => [ev.kind, ev.id]));
|
|
2164
2241
|
const sentBefore = board.db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND kind IN ('notice_delivered', 'notice_filed') AND json_extract(payload, '$.source_event') = ? AND json_extract(payload, '$.address') = ? LIMIT 1");
|
|
2165
|
-
const advance = (id) => tx(board, (db) => db.prepare("UPDATE kanban_notify_subs SET last_event_id = ? WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ? AND last_event_id < ?").run(id, sub.task_id, sub.chat_id, sub.thread_id ?? '', id));
|
|
2166
2242
|
for (const ev of events) {
|
|
2167
2243
|
const p = json(ev.payload, {}) ?? {};
|
|
2168
2244
|
if (maxAge && now() - Number(ev.created_at) > maxAge) { cursor = ev.id; (report.stale_notices_passed ??= 0); report.stale_notices_passed += 1; continue; }
|
|
@@ -2180,6 +2256,7 @@ async function notify(board, fleet, report, entry) {
|
|
|
2180
2256
|
if (text) {
|
|
2181
2257
|
const body = ['worker', 'dependency'].includes(role) ? identifyNotice(ev, text) : text;
|
|
2182
2258
|
if (refused.has(sub.chat_id) || stopping) { untried = true; break; }
|
|
2259
|
+
trying = ev.id;
|
|
2183
2260
|
let receipt;
|
|
2184
2261
|
try {
|
|
2185
2262
|
receipt = await send(board, from, board.label, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: String(sub.chat_id).includes(':claude-code:') ? false : !working || (!entry.workflow.run && sub.delivery_mode === 'notify'), subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
|
|
@@ -2187,13 +2264,16 @@ async function notify(board, fleet, report, entry) {
|
|
|
2187
2264
|
// board does not send it again.
|
|
2188
2265
|
if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'notice was not filed');
|
|
2189
2266
|
} catch (error) {
|
|
2267
|
+
// two passes may send one notice at once, and the second send can fail while the first is taken: a notice
|
|
2268
|
+
// another pass already recorded as taken is no failure
|
|
2269
|
+
if (sentBefore.get(sub.task_id, ev.id, sub.chat_id)) { cursor = ev.id; forward(ev.id); continue; }
|
|
2190
2270
|
if (!refused.has(sub.chat_id)) refused.set(sub.chat_id, error);
|
|
2191
2271
|
throw error;
|
|
2192
2272
|
}
|
|
2193
|
-
|
|
2273
|
+
// recorded once, whichever pass's send the mailbox took
|
|
2274
|
+
tx(board, (db) => { if (!db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND kind IN ('notice_delivered', 'notice_filed') AND json_extract(payload, '$.source_event') = ? AND json_extract(payload, '$.address') = ?").get(sub.task_id, ev.id, sub.chat_id)) event(db, sub.task_id, receipt.delivered ? 'notice_delivered' : 'notice_filed', { source_event: ev.id, kind: ev.kind, address: sub.chat_id, receipt }); });
|
|
2194
2275
|
report.notified.push([sub.task_id, ev.kind, sub.chat_id]);
|
|
2195
|
-
|
|
2196
|
-
advance(ev.id);
|
|
2276
|
+
forward(ev.id);
|
|
2197
2277
|
}
|
|
2198
2278
|
cursor = ev.id;
|
|
2199
2279
|
}
|
|
@@ -2216,16 +2296,22 @@ async function notify(board, fleet, report, entry) {
|
|
|
2216
2296
|
});
|
|
2217
2297
|
report.undelivered.push([sub.task_id, sub.chat_id, failed]);
|
|
2218
2298
|
}
|
|
2219
|
-
// the cursor moves past what was
|
|
2299
|
+
// the cursor moves forward past what was passed over (a failed delivery is retried next pass)
|
|
2300
|
+
forward(cursor);
|
|
2220
2301
|
tx(board, (db) => {
|
|
2221
|
-
const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
|
|
2222
|
-
const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
|
|
2223
|
-
db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(cursor, ...identity, cursor);
|
|
2224
2302
|
// a round that left this subscription untried (a delivery to its recipient failed on another card) says nothing of it
|
|
2225
2303
|
if (untried) return;
|
|
2304
|
+
// A notice is tried NOTICE_ATTEMPTS times, waiting as an effect does between attempts. After its last failed
|
|
2305
|
+
// attempt it is abandoned (its terminal state, `notice_abandoned`, recorded once) and the cursor moves past it;
|
|
2306
|
+
// the subscription stays, so the notices after it are still told (dropping it lost them when the round
|
|
2307
|
+
// subscribed the session again from the card's newest event).
|
|
2226
2308
|
const failures = failed ? Number(sub.delivery_metadata?.failures ?? 0) + 1 : 0;
|
|
2227
|
-
if (failures >=
|
|
2228
|
-
|
|
2309
|
+
if (failed && failures >= NOTICE_ATTEMPTS && trying) {
|
|
2310
|
+
event(db, sub.task_id, 'notice_abandoned', { source_event: trying, address: sub.chat_id, attempts: failures, error: failed });
|
|
2311
|
+
db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ?, delivery_metadata = ? ${where} AND last_event_id < ?`).run(trying, JSON.stringify({ ...sub.delivery_metadata, failures: 0, next_at: 0 }), ...identity, trying);
|
|
2312
|
+
return;
|
|
2313
|
+
}
|
|
2314
|
+
if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures, next_at: failed ? now() + Math.min(1800, 30 * 2 ** (failures - 1)) : 0 }), ...identity);
|
|
2229
2315
|
if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity);
|
|
2230
2316
|
});
|
|
2231
2317
|
});
|
|
@@ -2566,17 +2652,78 @@ function capMax(cap, group, entry, lanes) {
|
|
|
2566
2652
|
catch { return null; }
|
|
2567
2653
|
}
|
|
2568
2654
|
|
|
2655
|
+
const ORCHESTRATOR_ENTRY = fileURLToPath(new URL('../bin/orchestrator.mjs', import.meta.url));
|
|
2656
|
+
const deliveries = new Map();
|
|
2657
|
+
/**
|
|
2658
|
+
* Start the home's delivery pass (`workflow deliver`) in a process of its own, unless one is already running: the
|
|
2659
|
+
* dispatcher's loop calls this after each round and never waits for it. `log(level, line)` hears what it failed at.
|
|
2660
|
+
* Answers whether one was started.
|
|
2661
|
+
*/
|
|
2662
|
+
export function startDelivery(root, { log = () => {} } = {}) {
|
|
2663
|
+
if (deliveries.has(root)) return false;
|
|
2664
|
+
const child = spawnChild(process.execPath, [ORCHESTRATOR_ENTRY, 'workflow', 'deliver', '--root', root, '--json'], { stdio: ['ignore', 'pipe', 'pipe'], env: process.env });
|
|
2665
|
+
deliveries.set(root, child);
|
|
2666
|
+
let out = '', err = '';
|
|
2667
|
+
child.stdout.setEncoding('utf8').on('data', (c) => { out += c; });
|
|
2668
|
+
child.stderr.setEncoding('utf8').on('data', (c) => { err = (err + c).slice(-2000); });
|
|
2669
|
+
const done = () => {
|
|
2670
|
+
deliveries.delete(root);
|
|
2671
|
+
let reports = null; try { reports = JSON.parse(out); } catch { log('warn', `delivery pass failed: ${(err || out).trim().slice(-300) || 'no answer'}`); return; }
|
|
2672
|
+
for (const r of reports) {
|
|
2673
|
+
if (r.error) log('warn', `delivery on board ${r.board} failed: ${r.error}`);
|
|
2674
|
+
for (const [task, to, why] of r.undelivered ?? []) log('warn', `board ${r.board}: a delivery for ${task} to ${to} waits: ${why}`);
|
|
2675
|
+
log('info', `delivery on board ${r.board}: ${r.effects_waiting ?? 0} effect(s) waiting, ${r.effects_failed ?? 0} failed for good; took ${Object.entries(r.ms ?? {}).map(([k, v]) => `${k} ${(v / 1000).toFixed(1)}s`).join(', ')}`);
|
|
2676
|
+
}
|
|
2677
|
+
};
|
|
2678
|
+
child.on('error', (error) => { log('warn', `delivery pass did not start: ${error.message}`); deliveries.delete(root); });
|
|
2679
|
+
child.on('close', done);
|
|
2680
|
+
return true;
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2683
|
+
/**
|
|
2684
|
+
* The delivery pass over every board of the home at `root`: what rounds recorded for after them (each due effect, with
|
|
2685
|
+
* its bounded attempts), board mail, subscription notices, and the board's agent records joined to Teams. It runs in
|
|
2686
|
+
* its own process (`workflow deliver`, started by serve after a round, one at a time), never inside a dispatch round,
|
|
2687
|
+
* so no recipient, door or Teams answer holds a round. Answers one report per board.
|
|
2688
|
+
*/
|
|
2689
|
+
export async function deliver({ root, slugs = null }) {
|
|
2690
|
+
fleetRoot = root;
|
|
2691
|
+
const entry = await loadWorkflow(root, { fresh: true });
|
|
2692
|
+
const fleet = observeFleet();
|
|
2693
|
+
const out = [];
|
|
2694
|
+
for (const slug of slugs ?? listBoards(root)) {
|
|
2695
|
+
const report = { board: slug, ms: {}, mailed: [], queued: [], failed: [], notified: [], undelivered: [], fleet: fleet.ok ? 'read' : fleet.error };
|
|
2696
|
+
let board;
|
|
2697
|
+
try {
|
|
2698
|
+
board = openBoard(root, slug);
|
|
2699
|
+
board.label = boardLabel(root, slug);
|
|
2700
|
+
setActor(boardAddress(board.label));
|
|
2701
|
+
let mark = Date.now();
|
|
2702
|
+
const phase = (name) => { const t = Date.now(); report.ms[name] = t - mark; mark = t; };
|
|
2703
|
+
// board mail is sent as effect rows, one per message, each claimed as any effect is
|
|
2704
|
+
tx(board, (db) => queueBoardMail(db)); phase('mail');
|
|
2705
|
+
report.effects = await runEffects(board, [], { pending: true }); phase('effects');
|
|
2706
|
+
await notify(board, fleet, report, entry); phase('notify');
|
|
2707
|
+
synchronizeAgents(board, fleet, report, entry); phase('agents');
|
|
2708
|
+
report.effects_failed = Number(board.db.prepare('SELECT COUNT(*) AS n FROM supercode_effects WHERE failed_at IS NOT NULL').get().n);
|
|
2709
|
+
report.effects_waiting = Number(board.db.prepare('SELECT COUNT(*) AS n FROM supercode_effects WHERE failed_at IS NULL').get().n);
|
|
2710
|
+
} catch (error) { report.error = error.message; }
|
|
2711
|
+
finally { board?.close(); }
|
|
2712
|
+
out.push(report);
|
|
2713
|
+
}
|
|
2714
|
+
return out;
|
|
2715
|
+
}
|
|
2716
|
+
|
|
2569
2717
|
/**
|
|
2570
2718
|
* One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
|
|
2571
2719
|
* `only` limits starting to one card (a create that answers whether its card started). Each board is ticked under its
|
|
2572
2720
|
* version-checked card claims; no dispatcher file lock. Two rounds may run at once (the dispatcher's and a one-shot
|
|
2573
2721
|
* `workflow dispatch`). An adoption request and a session close are claimed in a transaction (`newClaim`) before
|
|
2574
2722
|
* anything is done for them outside the board; card starts, supervision, review requests, mailed verbs and manual
|
|
2575
|
-
* continuations rely on their per-card transactions. Two rounds on one board at once can still both act on:
|
|
2576
|
-
*
|
|
2577
|
-
*
|
|
2578
|
-
*
|
|
2579
|
-
* board-only writes that do not re-check. Answers one report per board.
|
|
2723
|
+
* continuations rely on their per-card transactions. Two rounds on one board at once can still both act on:
|
|
2724
|
+
* deferred-launch retries (two panes), launch-receipt work (cancelUnissuedLaunch, bindLaunch), and board-only writes
|
|
2725
|
+
* that do not re-check. A round runs no effect and delivers nothing: it records effects (`queueEffects`) and the
|
|
2726
|
+
* delivery pass (`deliver`) carries them out. Answers one report per board.
|
|
2580
2727
|
*/
|
|
2581
2728
|
export async function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true, params = {}, deploymentReceipt = false }) {
|
|
2582
2729
|
fleetRoot = root;
|
|
@@ -2618,20 +2765,18 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
|
|
|
2618
2765
|
readMailbox(board, boardAddress(board.label), report); phase('mailbox');
|
|
2619
2766
|
await refreshRelationAnswers(board); phase('answers');
|
|
2620
2767
|
const effects = [];
|
|
2621
|
-
completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects);
|
|
2768
|
+
completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); queueEffects(board, effects.splice(0)); closeRequested(board, report, { ttl: settings.ttl }); phase('supervise');
|
|
2622
2769
|
report.unsettled = [];
|
|
2623
2770
|
report.promoted = recomputeReady(board, effects, report.unsettled);
|
|
2624
2771
|
tx(board,db=>auditEndedRuns(db,board,entry));
|
|
2625
2772
|
if (dispatch) await activatePullRequestReviews(board, entry, report, effects, { only });
|
|
2626
|
-
await sendBoardMessages(board, report);
|
|
2627
2773
|
if (dispatch && !stopping) startCards(board, fleet, lanes, settings, report, entry, effects, { only, held });
|
|
2628
2774
|
phase('start');
|
|
2629
2775
|
if (dispatch && !only && !report.paused) observeIdle(board, fleet, settings, report, entry, effects);
|
|
2630
|
-
// what the moves asked for
|
|
2631
|
-
|
|
2776
|
+
// what the moves asked for is recorded with them; the delivery pass (`deliver`) carries it out, outside the round
|
|
2777
|
+
queueEffects(board, effects);
|
|
2632
2778
|
phase('goals');
|
|
2633
2779
|
ensureSubscriptions(board, entry);
|
|
2634
|
-
await notify(board, fleet, report, entry); phase('notify');
|
|
2635
2780
|
fireHooks(board, settings, report);
|
|
2636
2781
|
if (String(board.db.prepare('PRAGMA journal_mode').get()?.journal_mode).toLowerCase() === 'wal') board.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get();
|
|
2637
2782
|
report.ready_waiting = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks t LEFT JOIN supercode_cards c ON c.task_id = t.id WHERE t.status = 'ready' AND t.assignee IS NOT NULL AND c.subtask_of IS NULL").get().n);
|
package/board/engine.mjs
CHANGED
|
@@ -463,41 +463,105 @@ export async function goto(board, id, status, { roles = null, payload = {} } = {
|
|
|
463
463
|
return { ...(await act(board, id, name, { roles, payload })), event: name };
|
|
464
464
|
}
|
|
465
465
|
|
|
466
|
-
|
|
467
|
-
|
|
466
|
+
// A delivery (a notice, an actor message, an agent declaration) is tried at most EFFECT_ATTEMPTS times, waiting twice as
|
|
467
|
+
// long after each failed attempt (from 30 s, at most 30 min). Its last failed attempt is its terminal state: the row
|
|
468
|
+
// keeps `failed_at` and the error, the card records `effect_failed` once, diagnose shows it, and nothing tries it again.
|
|
469
|
+
// A board-state effect the dispatcher owns (OWNED: merging the reviewed PRs, cleaning a workspace) is never abandoned:
|
|
470
|
+
// it keeps trying at the capped wait, and at EFFECT_ATTEMPTS failed attempts its card records `effect_stalled` and the
|
|
471
|
+
// managers are mailed once, so a passed card whose merge keeps failing is seen, and still heals when the cause clears.
|
|
472
|
+
export const EFFECT_ATTEMPTS = 6;
|
|
473
|
+
const OWNED = new Set(['merge_reviewed', 'cleanup_workspace']);
|
|
474
|
+
const effectWait = (attempts) => Math.min(1800, 30 * 2 ** Math.max(0, attempts - 1));
|
|
475
|
+
// A delivery pass claims a row by compare-and-set on its next_at (set ahead only while the row is due) before trying it,
|
|
476
|
+
// so two passes (serve's, the hosted loop's, a person's `workflow deliver`, a verb's own) never try one effect at once.
|
|
477
|
+
// The claim outlasts any one attempt: each effect's I/O is bounded by its own timeouts (a send's 180 s, a merge's
|
|
478
|
+
// MERGE_BUDGET plus one in-flight GitHub call), all well inside EFFECT_CLAIM.
|
|
479
|
+
const EFFECT_CLAIM = 900;
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* The effects a committed move asked for. Board writes the round owns (invalidating descendants, asking a session's
|
|
483
|
+
* close, stopping a local worker) are made here, in the round; everything that reaches outside the board is recorded
|
|
484
|
+
* to run after it, in the delivery pass, never inside a dispatch round.
|
|
485
|
+
*/
|
|
486
|
+
export function queueEffects(board, effects = []) {
|
|
487
|
+
// a board message is its own effect row (dispatch.mjs queueBoardMail), so the move's mail hint is not recorded
|
|
488
|
+
const fresh = effects.filter((e) => !e.durableId && e.kind !== 'run' && !(e.kind === 'board_message' && !e.event));
|
|
489
|
+
const now_ = fresh.filter((e) => ['invalidate_descendants', 'close_session', 'run_ended', 'stop_run'].includes(e.kind));
|
|
490
|
+
const later = fresh.filter((e) => !now_.includes(e));
|
|
491
|
+
for (const e of now_) {
|
|
492
|
+
if (e.kind === 'invalidate_descendants') tx(board, (db) => invalidateDescendants(db, board, e.id));
|
|
493
|
+
else if (e.kind === 'close_session') tx(board, (db) => requestSessionClose(db, e.id, { slot: e.slot, through: e.runId }));
|
|
494
|
+
else if (e.kind === 'run_ended') { if (e.outcome === 'reclaimed') stopWorker(board, e.runId); tx(board, (db) => requestSessionClose(db, e.id, { runId: e.runId })); }
|
|
495
|
+
else if (e.kind === 'stop_run') stopWorker(board, e.runId);
|
|
496
|
+
}
|
|
497
|
+
if (later.length) tx(board, (db) => { for (const e of later) db.prepare('INSERT INTO supercode_effects (effect) VALUES (?)').run(JSON.stringify(e)); });
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
/** The I/O a committed move asked for: the move's own `effects`, and with `pending` every recorded effect that is due. */
|
|
501
|
+
export async function runEffects(board, effects = [], { pending: due = false } = {}) {
|
|
468
502
|
const requested = new Set(effects.map((e) => e.durableId).filter(Boolean));
|
|
469
|
-
const
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
const
|
|
503
|
+
const t = now();
|
|
504
|
+
const rows = board.db.prepare('SELECT id, effect, attempts, next_at FROM supercode_effects WHERE failed_at IS NULL ORDER BY id').all()
|
|
505
|
+
.filter((p) => (requested.has(p.id) || due) && Number(p.next_at ?? 0) <= t);
|
|
506
|
+
const durable = new Set(rows.map((p) => p.id));
|
|
507
|
+
const attempts = new Map(rows.map((p) => [p.id, Number(p.attempts ?? 0)]));
|
|
508
|
+
const combined = [...rows.map((p) => ({ ...json(p.effect, {}), durableId: p.id })), ...effects.filter((e) => !e.durableId || (!durable.has(e.durableId) && !requested.has(e.durableId)))];
|
|
473
509
|
combined.sort((a,b) => Number(b.kind === 'session_notice') - Number(a.kind === 'session_notice'));
|
|
510
|
+
// what this call did, for its caller's report: effect ids done, tried and kept, and skipped as another's claim
|
|
511
|
+
const did = { done: [], kept: [], claimed_elsewhere: [] };
|
|
474
512
|
for (const e of combined) {
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
}
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
const
|
|
490
|
-
if (
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
if (
|
|
498
|
-
|
|
499
|
-
|
|
513
|
+
// the claim: taken only while the row is due and not failed; another pass holding it answers no change
|
|
514
|
+
if (e.durableId && !tx(board, (db) => db.prepare('UPDATE supercode_effects SET next_at = ? WHERE id = ? AND failed_at IS NULL AND next_at <= ?').run(now() + EFFECT_CLAIM, e.durableId, now()).changes)) { did.claimed_elsewhere.push(e.durableId); continue; }
|
|
515
|
+
let done, error = null;
|
|
516
|
+
try { done = await runEffect(board, e); } catch (thrown) { done = false; error = String(thrown?.message ?? thrown).slice(0, 500); }
|
|
517
|
+
if (!e.durableId) continue;
|
|
518
|
+
// a mail inside its batch window is not an attempt: it is looked at again in a few seconds
|
|
519
|
+
if (done === 'wait') { tx(board, (db) => db.prepare('UPDATE supercode_effects SET next_at = ? WHERE id = ?').run(now() + 5, e.durableId)); did.kept.push(e.durableId); continue; }
|
|
520
|
+
if (done) { tx(board, (db) => db.prepare('DELETE FROM supercode_effects WHERE id = ?').run(e.durableId)); did.done.push(`${e.durableId}:${e.kind}`); continue; }
|
|
521
|
+
did.kept.push(e.durableId);
|
|
522
|
+
const made = (attempts.get(e.durableId) ?? 0) + 1;
|
|
523
|
+
tx(board, (db) => {
|
|
524
|
+
const row = db.prepare('SELECT failed_at, last_error FROM supercode_effects WHERE id = ?').get(e.durableId);
|
|
525
|
+
if (!row || row.failed_at) return;
|
|
526
|
+
const reason = error ?? row.last_error ?? `${e.kind} was not accepted`;
|
|
527
|
+
const card = e.id && db.prepare('SELECT 1 FROM tasks WHERE id = ?').get(e.id) ? e.id : null;
|
|
528
|
+
if (OWNED.has(e.kind)) {
|
|
529
|
+
db.prepare('UPDATE supercode_effects SET attempts = ?, next_at = ?, last_error = ? WHERE id = ?').run(made, now() + effectWait(made), reason, e.durableId);
|
|
530
|
+
if (made === EFFECT_ATTEMPTS && card) {
|
|
531
|
+
event(db, card, 'effect_stalled', { effect: e.durableId, kind: e.kind, attempts: made, error: reason, retry_every_seconds: effectWait(made) }, e.runId ?? null);
|
|
532
|
+
const to = [].concat(workflowFor(board.root).params?.managers ?? []).find(Boolean);
|
|
533
|
+
if (to) event(db, card, 'board_message', { to, text: `${card}: the board's ${e.kind.replace('_', ' ')} has failed ${made} times and keeps trying every ${effectWait(made) / 60} min: ${reason}` }, null);
|
|
534
|
+
}
|
|
535
|
+
} else if (made >= EFFECT_ATTEMPTS) {
|
|
536
|
+
db.prepare('UPDATE supercode_effects SET attempts = ?, failed_at = ?, last_error = ? WHERE id = ?').run(made, now(), reason, e.durableId);
|
|
537
|
+
if (card) event(db, card, 'effect_failed', { effect: e.durableId, kind: e.kind, attempts: made, error: reason, address: e.address ?? null }, e.runId ?? null);
|
|
538
|
+
if (card && e.kind === 'board_message' && e.event) event(db, card, 'board_message_abandoned', { event: e.event, attempts: made, reason });
|
|
539
|
+
} else db.prepare('UPDATE supercode_effects SET attempts = ?, next_at = ?, last_error = ? WHERE id = ?').run(made, now() + effectWait(made), reason, e.durableId);
|
|
540
|
+
});
|
|
500
541
|
}
|
|
542
|
+
return did;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
/** One effect's I/O: true when it is done, false when it is not done yet (the caller counts the attempt). */
|
|
546
|
+
async function runEffect(board, e) {
|
|
547
|
+
if (e.kind === 'session_notice') { const { sessionNotice } = await import('./dispatch.mjs'); return await sessionNotice(board, e) !== false; }
|
|
548
|
+
if (e.kind === 'cleanup_workspace') { cleanupWorkspace(board, e.id); return true; }
|
|
549
|
+
if (e.kind === 'declare_card_agent') { const { declareCardAgentEffect } = await import('./agent-declarations.mjs'); return declareCardAgentEffect(board, e) !== false; }
|
|
550
|
+
if (e.kind === 'board_message') {
|
|
551
|
+
const { sendBoardMessage, sendBoardMessages } = await import('./dispatch.mjs');
|
|
552
|
+
// a verb's own ask sends its card's mail at once; the mail's own effect row then finds it sent
|
|
553
|
+
if (!e.event) { await sendBoardMessages(board, { undelivered: [] }, e.id); return true; }
|
|
554
|
+
return await sendBoardMessage(board, e);
|
|
555
|
+
}
|
|
556
|
+
if (e.kind === 'merge_reviewed') { const { mergeReviewed } = await import('./dispatch.mjs'); return await mergeReviewed(board, e) !== false; }
|
|
557
|
+
if (e.kind === 'invalidate_descendants') { tx(board, (db) => invalidateDescendants(db, board, e.id)); return true; }
|
|
558
|
+
if (e.kind === 'stop_run') { stopWorker(board, e.runId); return true; }
|
|
559
|
+
if (e.kind === 'close_session' || e.kind === 'send_to_actor' || e.kind === 'run_ended') {
|
|
560
|
+
if (e.kind === 'run_ended' && e.outcome === 'reclaimed') stopWorker(board, e.runId);
|
|
561
|
+
const { sessionEffect } = await import('./dispatch.mjs');
|
|
562
|
+
return await sessionEffect(board, e) !== false;
|
|
563
|
+
}
|
|
564
|
+
return true;
|
|
501
565
|
}
|
|
502
566
|
|
|
503
567
|
// a run taken back stops its one-shot worker on this host (Hermes's _terminate_reclaimed_worker); a pane session is
|
|
@@ -903,6 +967,13 @@ export function diagnose(db, task, { failureLimit = DEFAULT_FAILURE_LIMIT, stran
|
|
|
903
967
|
const session = lastSession(db, task.id);
|
|
904
968
|
if (session?.machine && !fleet.online.has(session.machine) && !fleet.unknown?.has(session.machine)) out.push({ kind: 'machine_offline', severity: 'warning', title: `its machine ${session.machine} is offline`, data: { machine: session.machine }, actions: [] });
|
|
905
969
|
}
|
|
970
|
+
// what the board could not carry out for the card: a delivery failed for good, or a board effect that keeps failing
|
|
971
|
+
for (const row of all(db, "SELECT id, effect, attempts, failed_at, last_error FROM supercode_effects WHERE json_extract(effect, '$.id') = ? AND (failed_at IS NOT NULL OR attempts >= ?)", task.id, EFFECT_ATTEMPTS)) {
|
|
972
|
+
const kind = json(row.effect, {})?.kind ?? 'effect';
|
|
973
|
+
out.push(row.failed_at
|
|
974
|
+
? { kind: 'effect_failed', severity: 'warning', title: `its ${kind} failed for good after ${row.attempts} attempts: ${row.last_error ?? 'no reason recorded'}`, data: { effect: row.id, kind, attempts: row.attempts }, actions: [] }
|
|
975
|
+
: { kind: 'effect_stalled', severity: 'error', title: `its ${kind} has failed ${row.attempts} times and is still tried: ${row.last_error ?? 'no reason recorded'}`, data: { effect: row.id, kind, attempts: row.attempts }, actions: [] });
|
|
976
|
+
}
|
|
906
977
|
const order = ['critical', 'error', 'warning'];
|
|
907
978
|
return out.sort((a, b) => order.indexOf(a.severity) - order.indexOf(b.severity));
|
|
908
979
|
}
|
package/board/request-mail.mjs
CHANGED
|
@@ -3,7 +3,9 @@ import { event, json, now } from './store.mjs';
|
|
|
3
3
|
export function pendingRequestMail(db, taskId = null) {
|
|
4
4
|
const pending = db.prepare(`SELECT id, task_id, payload, created_at FROM task_events original
|
|
5
5
|
WHERE kind='board_message' AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind='board_message_sent'
|
|
6
|
-
AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
|
|
6
|
+
AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
|
|
7
|
+
AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind='board_message_abandoned'
|
|
8
|
+
AND gone.task_id=original.task_id AND json_extract(gone.payload,'$.event')=original.id) ORDER BY id`).all()
|
|
7
9
|
.map(row => ({ ...row, mail: json(row.payload, {}) ?? {} }));
|
|
8
10
|
if (!pending.length) return { ready: [], nextAt: Infinity };
|
|
9
11
|
const batches = db.prepare("SELECT payload FROM task_events WHERE kind='board_message_batch' AND id>=? ORDER BY id").all(pending[0].id)
|
package/board/store.mjs
CHANGED
|
@@ -76,6 +76,9 @@ CREATE TABLE IF NOT EXISTS supercode_effects (id INTEGER PRIMARY KEY AUTOINCREME
|
|
|
76
76
|
// `stage`: the card's workflow status where kanban.db stores another (a `reviewing` card is `running` to Hermes)
|
|
77
77
|
// `since`: when the card entered its workflow status; `after_fired`: which of the status's `after` moves have run since
|
|
78
78
|
// `tags`: the card's ordered tags (Hermes's `tenant` holds the first); `sponsor`, `sponsor_evidence`, `requested_by_unverified`: C4
|
|
79
|
+
// An effect's attempts: how many were made, when the next may be, and, once its attempts are spent, when it failed and
|
|
80
|
+
// why (engine.mjs runEffects). A failed effect is kept as the record of its terminal state and never tried again.
|
|
81
|
+
const EFFECT_COLUMNS = { attempts: 'INTEGER NOT NULL DEFAULT 0', next_at: 'INTEGER NOT NULL DEFAULT 0', failed_at: 'INTEGER', last_error: 'TEXT' };
|
|
79
82
|
const OURS_COLUMNS = { workspace_anchor: 'TEXT', workspace_root: 'TEXT', subtask_of: 'TEXT', stage: 'TEXT', since: 'INTEGER', after_fired: 'TEXT', blocked_by: 'TEXT', paused: 'TEXT', archived: 'INTEGER DEFAULT 0', every: 'TEXT', next_run_at: 'INTEGER', tags: 'TEXT', sponsor: 'TEXT', sponsor_evidence: 'TEXT', requested_by_unverified: 'TEXT' };
|
|
80
83
|
|
|
81
84
|
// ------------------------------------------------------------------ backings
|
|
@@ -140,6 +143,8 @@ export function layOut(db, { fresh }) {
|
|
|
140
143
|
db.exec(PUBLICATION_SCHEMA);
|
|
141
144
|
const have = new Set(db.prepare('PRAGMA table_info(supercode_cards)').all().map((c) => c.name));
|
|
142
145
|
for (const [column, type] of Object.entries(OURS_COLUMNS)) if (!have.has(column)) db.exec(`ALTER TABLE supercode_cards ADD COLUMN ${column} ${type}`);
|
|
146
|
+
const effectColumns = new Set(db.prepare('PRAGMA table_info(supercode_effects)').all().map((c) => c.name));
|
|
147
|
+
for (const [column, type] of Object.entries(EFFECT_COLUMNS)) if (!effectColumns.has(column)) db.exec(`ALTER TABLE supercode_effects ADD COLUMN ${column} ${type}`);
|
|
143
148
|
}
|
|
144
149
|
|
|
145
150
|
/**
|
package/board.mjs
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import { spawn } from 'node:child_process';
|
|
14
14
|
import { fileURLToPath } from 'node:url';
|
|
15
15
|
import { summary } from './board/cli.mjs';
|
|
16
|
-
import { commandEnv, notifyChats, supercodeBin } from './board/dispatch.mjs';
|
|
16
|
+
import { commandEnv, notifyChats, startDelivery, supercodeBin } from './board/dispatch.mjs';
|
|
17
17
|
import { watchBoard } from './board/wake.mjs';
|
|
18
18
|
import { workflowEntry } from './board/workflow.mjs';
|
|
19
19
|
import { evaluate, holds } from './board/expr.mjs';
|
|
@@ -79,6 +79,8 @@ export async function startBoard({ root, orchestration = null, log = () => {}, a
|
|
|
79
79
|
try {
|
|
80
80
|
const { reports, error } = await dispatchOnce(root);
|
|
81
81
|
if (error) { log('warn', `board: tick failed: ${error}`); return; }
|
|
82
|
+
// what the round recorded is delivered by a pass of its own, which this loop never waits for
|
|
83
|
+
startDelivery(root, { log: (level, line) => log(level, `board: ${line}`) });
|
|
82
84
|
// chat-platform subscriptions are delivered here, through the daemon's own channel adapters
|
|
83
85
|
try { await notifyChats(root, adapters, log, wake); } catch (error) { log('warn', `board: chat notices failed: ${error.message}`); }
|
|
84
86
|
for (const r of reports) {
|
package/loop.mjs
CHANGED
|
@@ -24,7 +24,7 @@ import { harnessHomes } from './mcp/verbs.mjs';
|
|
|
24
24
|
import { checkMonitor, runJobScript } from './gates.mjs';
|
|
25
25
|
import { startBoard } from './board.mjs';
|
|
26
26
|
import { SenderIdentityRecord } from '@volter/teams/sender-identity';
|
|
27
|
-
import { agentDoor,agentScope,teamsUnavailable,ensureProfileAgent,localProfileAgent,localAgentRecord,localAgentSessions,synchronizeLocalAgents,ensureForkAgent,agentRunForKey,agentIntent,agentPreamble,agentEnvironment,recordAgentRun,recordNativeAgentSession,pendingAgentRuns,synchronizeAgentRun,agentRunForSession } from './agents.mjs';
|
|
27
|
+
import { agentDoor,agentScope,teamsUnavailable,ensureProfileAgent,localProfileAgent,localAgentRecord,localAgentSessions,synchronizeLocalAgents,ensureForkAgent,agentRunForKey,agentIntent,agentPreamble,agentEnvironment,recordAgentRun,recordNativeAgentSession,pendingAgentRuns,claimAgentRun,releaseAgentRun,synchronizeAgentRun,agentRunForSession } from './agents.mjs';
|
|
28
28
|
import { materializeAgentHome } from './agent-home.mjs';
|
|
29
29
|
import { localMachine } from './board/dispatch.mjs';
|
|
30
30
|
import { execFile } from 'node:child_process';
|
|
@@ -201,8 +201,9 @@ export class Orchestrator {
|
|
|
201
201
|
if(scope.offline)return;
|
|
202
202
|
for(const pending of synchronizeLocalAgents({root:this.state.root,profiles:this.state.profiles,context,declarations:this.state.agents??{}}))this.log('warn',`agent ${pending.agent} declaration pending: ${pending.error}`);
|
|
203
203
|
for(const row of pendingAgentRuns(this.state.root,scope.org)){
|
|
204
|
+
if(!claimAgentRun(this.state.root,scope.org,row))continue;
|
|
204
205
|
try{synchronizeAgentRun(this.state.root,scope.org,row,{context,intended:row.profile_json?JSON.parse(row.profile_json):undefined});}
|
|
205
|
-
catch(error){this.log('warn',`agent session ${row.session_ref}: ${error.message}`);if(error.code==='rate_limited')break;}
|
|
206
|
+
catch(error){releaseAgentRun(this.state.root,scope.org,row);this.log('warn',`agent session ${row.session_ref}: ${error.message}`);if(error.code==='rate_limited')break;}
|
|
206
207
|
}
|
|
207
208
|
}catch(error){this.log('warn',`agent session synchronization: ${error.message}`);}
|
|
208
209
|
}
|
|
@@ -521,8 +522,8 @@ export class Orchestrator {
|
|
|
521
522
|
if(!INSTRUCTION_ARGUMENTS[worker.harness])(this.runtimeAgentPreambles??=new Map()).set(lid,agentPrompt);
|
|
522
523
|
const session=`${worker.harness}:${runtime.handle.runtime_id}`;
|
|
523
524
|
recordAgentRun(this.state.root,scope.org,agent.identity.id,{session,machine:null,run:run_key??lid,profile:agent.intended.resolved,main,starterSession:starter_session});
|
|
524
|
-
if(agent.synced!==false)try{synchronizeAgentRun(this.state.root,scope.org,{agent_id:agent.identity.id,session_ref:session,machine:null,main},{context,intended:agent.intended.resolved});}
|
|
525
|
-
catch(error){this.log('warn',`agent session ${session} retained for synchronization: ${error.message}`);}
|
|
525
|
+
if(agent.synced!==false&&claimAgentRun(this.state.root,scope.org,{session_ref:session,published:0}))try{synchronizeAgentRun(this.state.root,scope.org,{agent_id:agent.identity.id,session_ref:session,machine:null,main},{context,intended:agent.intended.resolved});}
|
|
526
|
+
catch(error){releaseAgentRun(this.state.root,scope.org,{session_ref:session});this.log('warn',`agent session ${session} retained for synchronization: ${error.message}`);}
|
|
526
527
|
this.enqueue(events.worker({
|
|
527
528
|
profile: profileName, keyString, runtime_id: runtime.handle.runtime_id,
|
|
528
529
|
ev: { kind: 'started', runtime_id: runtime.handle.runtime_id, session_id: runtime.handle.runtime_id, connection: runtime.connection, harness: worker.harness, steer },
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-orchestrator",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.39",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
|
|
6
6
|
"exports": {
|