@volter/supercode-orchestrator 0.5.37 → 0.5.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents.mjs +36 -7
- package/bin/machine-fleet.mjs +29 -0
- package/board/cli.mjs +12 -2
- package/board/dispatch.mjs +260 -108
- package/board/engine.mjs +103 -32
- package/board/machine-fleet.mjs +49 -0
- package/board/request-mail.mjs +3 -1
- package/board/store.mjs +5 -0
- package/board/wake.mjs +2 -4
- package/board.mjs +3 -1
- package/loop.mjs +5 -4
- package/package.json +1 -1
- package/board/event-fleet.mjs +0 -66
package/board/dispatch.mjs
CHANGED
|
@@ -16,11 +16,11 @@
|
|
|
16
16
|
// subscriber's cursor moves only after its notice was delivered, so two dispatchers over one board never both
|
|
17
17
|
// launch and a crash between the two steps is answered by the next tick.
|
|
18
18
|
import { monitorsRuns, failedObservation, surfaceIncident, auditEndedRuns, originOf } from './incidents.mjs';
|
|
19
|
-
import {
|
|
20
|
-
let
|
|
19
|
+
import { readMachineFleet } from './machine-fleet.mjs';
|
|
20
|
+
let fleetRoot = null;
|
|
21
21
|
import { inspectCompletionEvidence, noCodeChange, openReviewPullRequests, reviewDue, withCompletionEvidence, reviewCandidate, workflowRevision, retainWorkflowTick } from './ci.mjs';
|
|
22
22
|
import { runtimeDoor } from './runtime-door.mjs';
|
|
23
|
-
import { agentScope, localCardAgent, localReaderContext, localAgentSessions, synchronizeLocalAgents, runBudgetRefusal, runsAwaitingUsage, recordRunUsage, agentPreamble, agentEnvironment, recordAgentRun, pendingAgentRuns, synchronizeAgentRun, abandonAgentRun,
|
|
23
|
+
import { agentScope, localCardAgent, localReaderContext, localAgentSessions, synchronizeLocalAgents, runBudgetRefusal, runsAwaitingUsage, recordRunUsage, agentPreamble, agentEnvironment, recordAgentRun, pendingAgentRuns, synchronizeAgentRun, claimAgentRun, releaseAgentRun, abandonAgentRun,activeAgentAssignments,finishAgentAssignment,synchronizeAgentEnds, agentRunForSession } from '../agents.mjs';
|
|
24
24
|
import { materializeAgentHome } from '../agent-home.mjs';
|
|
25
25
|
import { memorySnapshotSync } from '../hermes-cli.mjs';
|
|
26
26
|
import { createRequire } from 'node:module';
|
|
@@ -39,7 +39,7 @@ import { identifyNotice } from './notices.mjs';
|
|
|
39
39
|
import { pendingRequestMail } from './request-mail.mjs';
|
|
40
40
|
import {
|
|
41
41
|
cardContext, claimTtl, currentRun, DEFAULT_FAILURE_LIMIT, extendClaim, getTask, lastSession,
|
|
42
|
-
recomputeReady, runEffects, setRunSession, subscribeIn, subscriptions, reconcileDependencySubscriptions,
|
|
42
|
+
recomputeReady, runEffects, queueEffects, setRunSession, subscribeIn, subscriptions, reconcileDependencySubscriptions,
|
|
43
43
|
} from './engine.mjs';
|
|
44
44
|
|
|
45
45
|
const WORKER = fileURLToPath(new URL('../bin/hermes-worker.mjs', import.meta.url));
|
|
@@ -115,9 +115,22 @@ export function callerSession() {
|
|
|
115
115
|
return claude;
|
|
116
116
|
}
|
|
117
117
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
118
|
+
/** The fleet as the home's machines answer for themselves (machine-fleet.mjs): this machine, and every machine an open
|
|
119
|
+
* card of the home names or one of its notice subscribers is on. */
|
|
120
|
+
export function observeFleet(root = fleetRoot) {
|
|
121
|
+
const machines = new Set();
|
|
122
|
+
for (const slug of root ? listBoards(root) : []) {
|
|
123
|
+
try {
|
|
124
|
+
const b = openBoard(root, slug);
|
|
125
|
+
try {
|
|
126
|
+
const live = "t.status NOT IN ('done', 'cancelled', 'archived') AND NOT COALESCE(c.archived, 0)";
|
|
127
|
+
for (const { machine } of b.db.prepare(`SELECT DISTINCT c.machine FROM supercode_cards c JOIN tasks t ON t.id = c.task_id WHERE c.machine IS NOT NULL AND c.machine != '' AND ${live}`).all()) machines.add(normalMachine(machine));
|
|
128
|
+
for (const { chat_id: address } of b.db.prepare(`SELECT DISTINCT s.chat_id FROM kanban_notify_subs s JOIN tasks t ON t.id = s.task_id JOIN supercode_cards c ON c.task_id = t.id WHERE s.platform = 'supercode' AND s.chat_id LIKE 'sc:%' AND ${live}`).all()) machines.add(normalMachine(address.split(':')[1]));
|
|
129
|
+
} finally { b.close(); }
|
|
130
|
+
} catch { /* its own tick reports it */ }
|
|
131
|
+
}
|
|
132
|
+
const fleet = readMachineFleet(machines, localMachine());
|
|
133
|
+
presenceReading = { known: fleet.presenceKnown === true, online: new Set([localMachine(), ...fleet.online]), unknown: fleet.unknown ?? new Set() };
|
|
121
134
|
notePresence(presenceReading);
|
|
122
135
|
return fleet;
|
|
123
136
|
}
|
|
@@ -139,26 +152,30 @@ const adoptedOnly = (s) => s?.mode === 'adopt' && !s.launch_key && s.surface !==
|
|
|
139
152
|
const addressOf = (s) => s?.address ?? (s?.session_id && s.machine ? `sc:${s.machine}:${s.harness ?? 'claude-code'}:${s.session_id}` : null);
|
|
140
153
|
const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
141
154
|
|
|
142
|
-
// A
|
|
143
|
-
//
|
|
155
|
+
// A session in a live pane of its machine is the fleet's reading; any other run is read on its own machine: a headless
|
|
156
|
+
// worker by its pid, a pane by that machine's pane list, a session in no pane (an adopted native one) by that machine's
|
|
157
|
+
// harness discovery. Null only when the machine cannot answer.
|
|
144
158
|
const sessionAlive = (fleet, session) => fleet.native?.has(addressOf(session)) ? fleet.native.get(addressOf(session)).alive : unpublishedAlive(session);
|
|
159
|
+
/** A subscriber's address as the session it names, for sessionAlive (its machine, harness and native id). */
|
|
160
|
+
const sessionAt = (address) => { const m = /^sc:([^:]+):([^:]+):(.+)$/.exec(String(address)); return m && HARNESSES[m[2]] ? { address, machine: normalMachine(m[1]), harness: m[2], session_id: m[3] } : null; };
|
|
161
|
+
const subscriberAlive = (fleet, address) => fleet.alive.has(address) || (sessionAt(address) ? sessionAlive(fleet, sessionAt(address)) === true : false);
|
|
145
162
|
const paneLists = new Map();
|
|
146
|
-
// A run the agent sync holds names only its machine and native session (`harness:id`), never a pane or pid.
|
|
147
|
-
// fleet answers for a session it published; one it never published (a gateway worker in its profile's directory,
|
|
148
|
-
// outside every synced root) is read through its machine's native discovery, as a native close reads it.
|
|
163
|
+
// A run the agent sync holds names only its machine and native session (`harness:id`), never a pane or pid.
|
|
149
164
|
function syncRowAlive(fleet, row) {
|
|
150
165
|
const machine = row.machine || localMachine(), at = row.session_ref.indexOf(':');
|
|
151
|
-
|
|
152
|
-
const alive = sessionAlive(fleet, session);
|
|
153
|
-
if (alive !== null) return alive;
|
|
154
|
-
try { return nativeReading(nativeSessions(machine, session.harness), session).alive; } catch { return null; }
|
|
166
|
+
return sessionAlive(fleet, { machine, harness: row.session_ref.slice(0, at), session_id: row.session_ref.slice(at + 1), address: `sc:${machine}:${row.session_ref}` });
|
|
155
167
|
}
|
|
156
168
|
function unpublishedAlive(session) {
|
|
157
169
|
if (!session) return null;
|
|
158
170
|
if (session.surface === 'headless' && session.pid) return pidAlive(session.pid);
|
|
159
|
-
if (!session.
|
|
171
|
+
if (!session.machine) return null;
|
|
172
|
+
if (!session.pane) {
|
|
173
|
+
if (!session.session_id) return null;
|
|
174
|
+
try { return nativeReading(nativeSessions(session.machine, session.harness ?? 'claude-code'), session).alive; } catch { return null; }
|
|
175
|
+
}
|
|
160
176
|
let entry = paneLists.get(session.machine);
|
|
161
|
-
if (!entry || Date.now() - entry.at >
|
|
177
|
+
if (!entry || Date.now() - entry.at > READING_MS) {
|
|
178
|
+
if (readingsOnly) return null;
|
|
162
179
|
const r = doorOn(session.machine, 'harness.v1.panes.list', {}, { timeout: 60_000 });
|
|
163
180
|
const panes = r.code === 0 ? json(r.stdout.trim(), null) : null;
|
|
164
181
|
entry = { at: Date.now(), panes: Array.isArray(panes) ? panes : null };
|
|
@@ -170,19 +187,34 @@ function unpublishedAlive(session) {
|
|
|
170
187
|
/** Native activity comes from the machine's harness discovery, not catalogue age. One discovery (up to 2000
|
|
171
188
|
* sessions, seconds each) serves a round's every native session on that machine, as paneLists serves its panes. */
|
|
172
189
|
const discoveries = new Map();
|
|
190
|
+
// A machine's pane list and harness discovery are read before a board write opens, never inside one: a door that
|
|
191
|
+
// takes seconds would hold the board's write lock past a worker's own command's busy wait. A write that needs a
|
|
192
|
+
// reading not taken yet reads it as unknown. One reading serves READING_MS.
|
|
193
|
+
const READING_MS = 15_000;
|
|
194
|
+
let readingsOnly = 0;
|
|
195
|
+
const txRead = (board, fn) => { readingsOnly += 1; try { return tx(board, fn); } finally { readingsOnly -= 1; } };
|
|
196
|
+
/** Take the readings a board write about `session` will need, before the write. */
|
|
197
|
+
function readBefore(fleet, session) {
|
|
198
|
+
if (!session?.session_id || !session.machine || session.surface === 'headless') return;
|
|
199
|
+
try { nativeSessions(session.machine, session.harness ?? 'claude-code'); } catch { /* unread: unknown inside the write */ }
|
|
200
|
+
sessionAlive(fleet, session);
|
|
201
|
+
}
|
|
173
202
|
function nativeSessions(machine, harness) {
|
|
174
203
|
const key = `${normalMachine(machine)}\u0000${harness}`, seen = discoveries.get(key);
|
|
175
|
-
if (seen && Date.now() - seen.at <=
|
|
204
|
+
if (seen && Date.now() - seen.at <= READING_MS) return seen.sessions;
|
|
205
|
+
if (readingsOnly) throw unavailable('native liveness is read before a board write, not inside it');
|
|
176
206
|
const sessions = discoverNative(machine, harness);
|
|
177
207
|
discoveries.set(key, { at: Date.now(), sessions });
|
|
178
208
|
return sessions;
|
|
179
209
|
}
|
|
210
|
+
// A native discovery is bounded as the fleet reading is: one that does not answer in time is unknown, read next round.
|
|
211
|
+
const NATIVE_DISCOVERY_MS = 15_000;
|
|
180
212
|
function discoverNative(machine, harness) {
|
|
181
213
|
if (offlineMachine(machine)) throw unavailable(`native liveness unavailable: machine ${normalMachine(machine)} is offline`);
|
|
182
214
|
// This machine's discovery is its harness door, as every other harness read here is; another machine's is supercode's fleet arm
|
|
183
215
|
const result = normalMachine(machine) === localMachine()
|
|
184
|
-
? doorOn(machine, 'harness.v1.sessions.discover', { harnesses: [harness], limit: 2000 }, { timeout:
|
|
185
|
-
: sc(['discover', '--on', machine, '--harness', harness, '--limit', '2000'], { timeout:
|
|
216
|
+
? doorOn(machine, 'harness.v1.sessions.discover', { harnesses: [harness], limit: 2000 }, { timeout: NATIVE_DISCOVERY_MS })
|
|
217
|
+
: sc(['discover', '--on', machine, '--harness', harness, '--limit', '2000'], { timeout: NATIVE_DISCOVERY_MS });
|
|
186
218
|
const page = result.code === 0 ? lastJson(result.stdout) : null;
|
|
187
219
|
if (!Array.isArray(page?.sessions)) throw unavailable(`native liveness unavailable on ${machine}: ${(result.stderr || result.stdout).trim().slice(-300)}`);
|
|
188
220
|
return page.sessions;
|
|
@@ -256,8 +288,9 @@ export function lanesOf(profiles = {}) {
|
|
|
256
288
|
return out;
|
|
257
289
|
}
|
|
258
290
|
|
|
259
|
-
// The latest presence reading (observeFleet). A machine
|
|
260
|
-
// wait out its timeout, and one
|
|
291
|
+
// The latest presence reading (observeFleet). A machine that did not answer it (offline, or unread this round) is not
|
|
292
|
+
// asked again until the next reading: each remote call would wait out its timeout, and one silent machine held every
|
|
293
|
+
// close and launch on this one. Its door answers `machine_offline`, which every caller tries again next round.
|
|
261
294
|
let presenceReading = null;
|
|
262
295
|
const offlineMachine = (machine) => {
|
|
263
296
|
const m = normalMachine(machine ?? localMachine());
|
|
@@ -265,7 +298,7 @@ const offlineMachine = (machine) => {
|
|
|
265
298
|
};
|
|
266
299
|
/** A machine's public door (`harness.v1.*`), answered at once when the machine reads as offline. */
|
|
267
300
|
const doorOn = (machine, method, params, options) => offlineMachine(machine)
|
|
268
|
-
? { code: 1, stdout: '', stderr: `machine ${normalMachine(machine)}
|
|
301
|
+
? { code: 1, stdout: '', stderr: JSON.stringify({ code: 'machine_offline', message: `machine ${normalMachine(machine)} did not answer the fleet reading` }) }
|
|
269
302
|
: runtimeDoor(normalMachine(machine ?? localMachine()) === localMachine() ? null : machine, method, params, options);
|
|
270
303
|
const unavailable = message => Object.assign(new Error(message), { launchDeferred: true });
|
|
271
304
|
|
|
@@ -282,8 +315,9 @@ function supervise(board, fleet, settings, report, entry, effects) {
|
|
|
282
315
|
try { const receipt = launchReceipt(board, session); if (receipt?.live === false && receipt.exit) exits.set(id, receipt.exit); }
|
|
283
316
|
catch (error) { report.failed.push([id, `exit receipt unavailable; retaining liveness observation: ${error.message}`]); }
|
|
284
317
|
}
|
|
318
|
+
for (const { id } of rows) readBefore(fleet, currentRun(board.db, id)?.metadata?.supercode);
|
|
285
319
|
for (const { id } of rows) {
|
|
286
|
-
|
|
320
|
+
txRead(board, (db) => {
|
|
287
321
|
const task = getTask(db, id);
|
|
288
322
|
if (!task || (entry.workflow.run ? !task.current_run_id : task.status !== 'running')) return;
|
|
289
323
|
const runRow = currentRun(db, id);
|
|
@@ -329,11 +363,11 @@ function supervise(board, fleet, settings, report, entry, effects) {
|
|
|
329
363
|
} else extendClaim(db, id, { ttl: settings.ttl, reason: 'launch_receipt_pending' });
|
|
330
364
|
return;
|
|
331
365
|
}
|
|
332
|
-
// Availability is the
|
|
333
|
-
//
|
|
366
|
+
// Availability is the machine's own door (observeFleet). A machine whose door did not answer is unknown, not
|
|
367
|
+
// evidence that it left: its run is kept. Only a relay's positive answer that it is not linked reads as offline.
|
|
334
368
|
if (session?.session_id && session.surface !== 'headless') {
|
|
335
369
|
const reachable = fleet.online.has(session.machine);
|
|
336
|
-
if (!reachable && !fleet.presenceKnown) return;
|
|
370
|
+
if (!reachable && (!fleet.presenceKnown || fleet.unknown?.has(session.machine))) return;
|
|
337
371
|
if (!reachable) {
|
|
338
372
|
const next = {...session, offline_since: session.offline_since ?? t};
|
|
339
373
|
setRunSession(db,runRow.id,next);
|
|
@@ -736,7 +770,7 @@ export function launchRun(board, entry, id, lane, fleet, settings) {
|
|
|
736
770
|
export async function startCard(board, id, { roles = ['manager'], lanes, settings } = {}) {
|
|
737
771
|
// the board's own published fleet projection, as a tick reads it: a manager's or creator's start that continues the
|
|
738
772
|
// slot's session checks that session through it
|
|
739
|
-
|
|
773
|
+
fleetRoot = board.root;
|
|
740
774
|
const entry = await loadWorkflow(board.root);
|
|
741
775
|
const fleet = observeFleet();
|
|
742
776
|
const before = getTaskNow(board, id), checked = checkWorkspace(board, before);
|
|
@@ -925,16 +959,21 @@ function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
925
959
|
return Number(discovered.get(key)?.find((row) => row.locator?.session_id === session.session_id)?.updated_at_ms) || null;
|
|
926
960
|
};
|
|
927
961
|
const managers = [].concat(entry.params.managers ?? []).filter(Boolean);
|
|
928
|
-
|
|
929
|
-
|
|
962
|
+
const running = board.db.prepare(entry.workflow.run ? "SELECT id FROM tasks WHERE current_run_id IS NOT NULL" : "SELECT id FROM tasks WHERE status = 'running'").all();
|
|
963
|
+
for (const { id } of running) { const session = currentRun(board.db, id)?.metadata?.supercode; readBefore(fleet, session); if (session?.session_id) updatedAt(session); }
|
|
964
|
+
for (const { id } of running) {
|
|
965
|
+
txRead(board, (db) => {
|
|
930
966
|
const task = getTask(db, id);
|
|
931
967
|
const status = statusOf(db, task);
|
|
932
968
|
if (!(entry.workflow.statuses[status]?.on?.session_idle || entry.workflow.on?.session_idle || entry.workflow.run?.on?.session_idle)) return;
|
|
933
969
|
const runRow = currentRun(db, id);
|
|
934
970
|
const session = runRow?.metadata?.supercode;
|
|
935
971
|
if (!session?.session_id || !['pane', 'native'].includes(session.surface) || session.offline_since != null) return;
|
|
936
|
-
if (session.surface === 'native' && fleet
|
|
937
|
-
|
|
972
|
+
if (session.surface === 'native' && sessionAlive(fleet, session) !== true) return;
|
|
973
|
+
// a session in no pane has its turn on its machine's discovery row, which sessionAlive just read
|
|
974
|
+
let nativeRow = null;
|
|
975
|
+
if (session.surface === 'native') try { nativeRow = nativeReading(nativeSessions(session.machine, session.harness ?? 'claude-code'), session).row; } catch { /* unread: no turn */ }
|
|
976
|
+
const live = fleet.sessions.find((x) => x.address === addressOf(session)) ?? (nativeRow ? { address: addressOf(session), turn: nativeRow.activity?.turn ?? null } : undefined);
|
|
938
977
|
const observedAt = now();
|
|
939
978
|
const lastAtMs = Number(live?.last_message_at_ms) || updatedAt(session);
|
|
940
979
|
const lastAt = Math.floor(Number(lastAtMs) / 1000);
|
|
@@ -989,7 +1028,7 @@ function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
989
1028
|
const observation = retain('dispatched', { turns_after: Number(stored?.goal_turns ?? 0), nudge_queued: r.effects.some(e => e.kind === 'send_to_actor'), run_ended: after?.id !== runRow.id });
|
|
990
1029
|
event(db, id, 'session_idle_observed', observation, runRow.id);
|
|
991
1030
|
if (after?.id === runRow.id) setRunSession(db, runRow.id, { goal_nudged_at: observedAt });
|
|
992
|
-
report.notified.push([id, 'session_idle',
|
|
1031
|
+
report.notified.push([id, 'session_idle', addressOf(session)]);
|
|
993
1032
|
} catch (error) {
|
|
994
1033
|
retain('dispatch_failed', { error: String(error.message).slice(-200) });
|
|
995
1034
|
report.failed.push([id, `session_idle: ${String(error.message).slice(-200)}`]);
|
|
@@ -1034,7 +1073,9 @@ export async function sessionNotice(board, e) {
|
|
|
1034
1073
|
: { lifecycle_notice_error: null, ...(session.lifecycle_notice_pending === e.durableId ? { lifecycle_notice_pending: null } : {}) }) });
|
|
1035
1074
|
if (!error || session?.lifecycle_notice_error !== error) event(db, e.id, error ? 'session_notice_failed' : receipt?.delivered ? 'session_notice_delivered' : 'session_notice_queued', { address: e.address, verb: e.verb, source_event: e.sourceEvent, notice: e.durableId, receipt, ...(error ? { error, retry: true } : {}) }, e.runId);
|
|
1036
1075
|
});
|
|
1037
|
-
|
|
1076
|
+
// the attempt's reason goes with it, onto the effect's row (runEffects counts the attempt)
|
|
1077
|
+
if (error) throw new Error(error);
|
|
1078
|
+
return true;
|
|
1038
1079
|
}
|
|
1039
1080
|
|
|
1040
1081
|
/** A session effect a committed move asked for: a message to the card's current actor, or closing a slot's pane. */
|
|
@@ -1054,7 +1095,10 @@ export async function sessionEffect(board, e) {
|
|
|
1054
1095
|
const payload = { notice: e.durableId, address, receipt };
|
|
1055
1096
|
if (previous?.kind !== kind || JSON.stringify(json(previous?.payload, {})?.receipt) !== JSON.stringify(receipt)) event(db, e.id, kind, payload, e.runId);
|
|
1056
1097
|
});
|
|
1057
|
-
|
|
1098
|
+
// a send its recipient's mailbox took (filed, or queued for a session between turns) is accepted: the mailbox
|
|
1099
|
+
// delivers it from there, and the board does not send it again
|
|
1100
|
+
if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'the actor notice was not filed');
|
|
1101
|
+
return true;
|
|
1058
1102
|
}
|
|
1059
1103
|
if (e.kind === 'close_session' || e.kind === 'run_ended') {
|
|
1060
1104
|
tx(board, (db) => requestSessionClose(db, e.id, e.kind === 'run_ended' ? { runId: e.runId } : { slot: e.slot, through: e.runId }));
|
|
@@ -1069,6 +1113,7 @@ export async function sessionEffect(board, e) {
|
|
|
1069
1113
|
* Once every PR is merged the card is sent `reviewed_merged` with the merge commits. Answers false (kept, tried again
|
|
1070
1114
|
* next round) on anything GitHub did not decide.
|
|
1071
1115
|
*/
|
|
1116
|
+
const MERGE_BUDGET_MS = 300_000;
|
|
1072
1117
|
export async function mergeReviewed(board, e) {
|
|
1073
1118
|
const { candidatePullRequests, mergeReviewedPullRequest } = await import('./ci.mjs');
|
|
1074
1119
|
const pass = view(board, (db) => db.prepare("SELECT id, payload FROM task_events WHERE task_id = ? AND kind = 'review_pass' ORDER BY id DESC LIMIT 1").get(e.id));
|
|
@@ -1076,11 +1121,16 @@ export async function mergeReviewed(board, e) {
|
|
|
1076
1121
|
const ci = typeof recorded === 'string' ? json(recorded, null) : recorded;
|
|
1077
1122
|
const prs = candidatePullRequests(ci).filter((p) => p?.pr?.url && p.commit);
|
|
1078
1123
|
const merges = [];
|
|
1124
|
+
// a merge attempt spends at most MERGE_BUDGET on starting GitHub calls (each bounded by its own timeout), inside its
|
|
1125
|
+
// effect's claim; what is left is the next attempt's
|
|
1126
|
+
const began = Date.now();
|
|
1079
1127
|
for (const p of prs) {
|
|
1128
|
+
if (Date.now() - began > MERGE_BUDGET_MS) throw new Error(`the merge attempt spent its ${MERGE_BUDGET_MS / 1000} s; the rest is merged next attempt`);
|
|
1080
1129
|
let answer;
|
|
1081
1130
|
try { answer = mergeReviewedPullRequest(p.pr.url, p.commit); } catch (error) {
|
|
1082
|
-
|
|
1083
|
-
|
|
1131
|
+
const reason = String(error.message).slice(-300);
|
|
1132
|
+
tx(board, (db) => event(db, e.id, 'merge_pending', { pr: p.pr.url, reviewed: p.commit, reason }));
|
|
1133
|
+
throw new Error(`merging ${p.pr.url} at ${p.commit}: ${reason}`);
|
|
1084
1134
|
}
|
|
1085
1135
|
if (answer.merged) { merges.push({ pr: p.pr.url, reviewed: p.commit, merge: answer.sha }); continue; }
|
|
1086
1136
|
const reason = answer.moved ? `${p.pr.url} is no longer at its reviewed head ${p.commit}${answer.head ? ` (now ${answer.head})` : ''}; what was pushed after the verdict needs its own review` : `${p.pr.url} was closed without merging`;
|
|
@@ -1121,7 +1171,7 @@ function launcherCheck(db, task, lane, fleet) {
|
|
|
1121
1171
|
if (!task.machine) return 'the card names no machine; a pane session runs on the machine its card names';
|
|
1122
1172
|
const machine = normalMachine(task.machine);
|
|
1123
1173
|
if (!fleet.ok) return `supercode could not read the fleet: ${fleet.error}`;
|
|
1124
|
-
if (
|
|
1174
|
+
if (fleet.unknown?.has(machine)) return `machine ${machine}'s door did not answer; the card starts when it does`;
|
|
1125
1175
|
if (!fleet.online.has(machine)) return `machine ${machine} is offline; the card starts when it is back`;
|
|
1126
1176
|
// a card on this machine gets its workspace made here (scratch, dir or worktree, as Hermes makes it); another
|
|
1127
1177
|
// machine's card names its directory there
|
|
@@ -1266,7 +1316,7 @@ function bindLaunch(board, row, session, pane, sid) {
|
|
|
1266
1316
|
|
|
1267
1317
|
/** A fast worker can reach its first board door before open's reply. Bind only the daemon's keyed pane. */
|
|
1268
1318
|
export function reconcileCallerLaunch(board, sid) {
|
|
1269
|
-
|
|
1319
|
+
fleetRoot = board.root;
|
|
1270
1320
|
if (view(board, (db) => db.prepare(`SELECT 1 FROM task_runs r JOIN tasks t ON t.current_run_id=r.id
|
|
1271
1321
|
WHERE json_extract(r.metadata, '$.supercode.session_id')=? AND COALESCE(json_extract(r.metadata, '$.supercode.launch_pending'),0)=0`).get(sid))) return;
|
|
1272
1322
|
const rows = view(board, (db) => db.prepare(`SELECT r.id, r.task_id, r.metadata FROM task_runs r
|
|
@@ -1290,7 +1340,9 @@ export function reconcileCallerLaunch(board, sid) {
|
|
|
1290
1340
|
}
|
|
1291
1341
|
}
|
|
1292
1342
|
|
|
1293
|
-
|
|
1343
|
+
/** The board's agent records joined to Teams: declarations, ends, session ownership and usage. Teams I/O, so it runs in
|
|
1344
|
+
* the delivery pass (`deliver`), never in a dispatch round. */
|
|
1345
|
+
function synchronizeAgents(board, fleet, report, entry) {
|
|
1294
1346
|
// A delayed launch receipt/catalogue event binds the same declared agent, never another identity.
|
|
1295
1347
|
// A resumed session is held by its newest run; an earlier run of the same session no longer owns it.
|
|
1296
1348
|
const agentRuns=[...new Map(view(board,db=>db.prepare("SELECT id,task_id,metadata,ended_at FROM task_runs WHERE json_extract(metadata,'$.supercode.agent_id') IS NOT NULL ORDER BY id").all())
|
|
@@ -1299,15 +1351,6 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
|
|
|
1299
1351
|
for(const row of agentRuns){
|
|
1300
1352
|
const session=json(row.metadata,{})?.supercode;
|
|
1301
1353
|
if(!session?.session_id||!session.agent_org)continue;
|
|
1302
|
-
const observation=fleet.sessions?.find(item=>item.address===addressOf(session))?.identity;
|
|
1303
|
-
if(observation?.agent?.id===session.agent_id&&session.agent_profile_revision!==observation.revision){
|
|
1304
|
-
const mismatch=agentProfileMismatch(session.agent_profile_intent,observation.resolved);
|
|
1305
|
-
tx(board,db=>{
|
|
1306
|
-
// Rule 4: a run whose recorded profile differs from its agent's intent is shown and not counted.
|
|
1307
|
-
setRunSession(db,row.id,{agent_profile_recorded:observation.resolved,agent_profile_revision:observation.revision,agent_profile_mismatch:mismatch,counted:!mismatch.differences.length});
|
|
1308
|
-
event(db,row.task_id,mismatch.differences.length?'agent_profile_drift':'agent_profile_recorded',{agent:session.agent_id,session:session.session_id,comparison:mismatch},row.id);
|
|
1309
|
-
});
|
|
1310
|
-
}
|
|
1311
1354
|
const ref=`${session.harness}:${session.session_id}`;
|
|
1312
1355
|
// An ended run whose session the agent store already holds, published, for another agent lost that claim when it
|
|
1313
1356
|
// ran: no round can change an ended run or a published owner, so the disagreement is history. It is said once on
|
|
@@ -1327,10 +1370,11 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
|
|
|
1327
1370
|
orgs.set(session.agent_org,session.agent_context);
|
|
1328
1371
|
}
|
|
1329
1372
|
// Identities minted while Teams was absent are declared under their ids before their sessions are joined.
|
|
1330
|
-
const context=process.env.SUPERCODE_CONTEXT
|
|
1373
|
+
const context=process.env.SUPERCODE_CONTEXT;
|
|
1374
|
+
let scope;try{scope=agentScope(context);}catch(error){scope={offline:true};report.failed.push(['agents',`agent declarations wait: ${error.message}`]);}
|
|
1331
1375
|
if(!scope.offline){
|
|
1332
1376
|
orgs.set(scope.org,orgs.get(scope.org)??context);
|
|
1333
|
-
for(const pending of synchronizeLocalAgents({root:board.root,profiles:entry.profiles,context}))report.queued.push([pending.agent,`agent declaration pending: ${pending.error}`]);
|
|
1377
|
+
for(const pending of synchronizeLocalAgents({root:board.root,profiles:entry.profiles,context}))(pending.failed?report.failed:report.queued).push([pending.agent,`agent declaration ${pending.failed?`failed for good after ${pending.attempts} refusals`:'pending'}: ${pending.error}`]);
|
|
1334
1378
|
}
|
|
1335
1379
|
orgs.delete('local');
|
|
1336
1380
|
// G3: an ended run's session usage, read once from its harness's own records on its machine
|
|
@@ -1357,9 +1401,14 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
|
|
|
1357
1401
|
for(const [org,context] of orgs)for(const pending of synchronizeAgentEnds(board.root,org,{context}))report.queued.push([pending.agent,`agent end synchronization pending: ${pending.error}`]);
|
|
1358
1402
|
for(const [org,context] of orgs)for(const row of pendingAgentRuns(board.root,org)){
|
|
1359
1403
|
const launch=agentRuns.find(run=>String(run.id)===String(row.run_id));
|
|
1404
|
+
// each run is claimed by compare-and-set on its own `published` before it is joined; another pass's claim skips it
|
|
1405
|
+
if(!claimAgentRun(board.root,org,row))continue;
|
|
1360
1406
|
try{synchronizeAgentRun(board.root,org,row,{context,intended:json(launch?.metadata,{})?.supercode?.agent_profile_intent});}
|
|
1361
|
-
catch(error){if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&syncRowAlive(fleet,row)===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
|
|
1407
|
+
catch(error){releaseAgentRun(board.root,org,row);if(error.code==='rate_limited'){report.queued.push([row.run_id,`agent session sync pending: ${error.message}`]);break;}const gone=error.claimRefused&&error.status===404&&fleet.ok&&syncRowAlive(fleet,row)===false;if(gone)abandonAgentRun(board.root,org,row.session_ref);report.failed.push([row.run_id,`agent session sync failed: ${error.message}${gone?`; ${row.session_ref} is stopped and Teams refuses it, so it leaves the sync`:''}`]);}
|
|
1362
1408
|
}
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes) {
|
|
1363
1412
|
const deferred=view(board,db=>db.prepare("SELECT t.id,r.id run_id,COALESCE(json_extract(r.metadata,'$.supercode.launched_at'),r.started_at) launched_at FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
|
|
1364
1413
|
// A deferred launch was never issued (a reading or door was not yet available): it is the start its manager, creator
|
|
1365
1414
|
// or workflow asked for, carried out within the launch window; supervise surfaces it once when the window ends. The
|
|
@@ -1436,7 +1485,8 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
1436
1485
|
if (lane.harness === 'codex' && lane.reasoning_effort) tail.push('-c', `model_reasoning_effort=${JSON.stringify(lane.reasoning_effort)}`);
|
|
1437
1486
|
const harness = HARNESSES[lane.harness] ? lane.harness : 'claude-code';
|
|
1438
1487
|
const how = HARNESSES[harness];
|
|
1439
|
-
|
|
1488
|
+
// A card on this machine opens through this machine's own daemon; only another machine is reached with `--on`.
|
|
1489
|
+
const base = ['open', '--new', how.program, '--detach', `--window=${window}`, ...(machine === localMachine() ? [] : ['--on', task.machine]), '--cwd', task.workspace_path, '--env', 'SUPERCODE_BOARD_PROCESS_GUARD=1',
|
|
1440
1490
|
...Object.entries(meta.agent_env??{}).flatMap(([key,value])=>['--env',`${key}=${value}`])];
|
|
1441
1491
|
// the slot's last session: kept when live, resumed when lost, never another slot's (a reviewer's is its own)
|
|
1442
1492
|
// a close its session's own command asked for is carried out first, so its pane is gone before the slot's next
|
|
@@ -1497,10 +1547,14 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
1497
1547
|
};
|
|
1498
1548
|
let session = null;
|
|
1499
1549
|
let replaced = null;
|
|
1500
|
-
|
|
1550
|
+
// A session in no pane is read on its machine before anything replaces it. One the board released (an adopted
|
|
1551
|
+
// session it did not launch, left running at its run's end) is its owner's while it runs: it is never resumed beside
|
|
1552
|
+
// itself, and the launch waits for the owner to end it.
|
|
1553
|
+
if (prev?.surface === 'native' && !prev.pane && (!prev.close_confirmed_at || prev.released)) {
|
|
1501
1554
|
const reading = nativeReading(nativeSessions(prev.machine, prev.harness), prev);
|
|
1502
1555
|
if (reading.alive == null) throw unavailable(`cannot establish native continuation liveness for ${addressOf(prev)}`);
|
|
1503
1556
|
fleet.native ??= new Map(); fleet.native.set(addressOf(prev), reading);
|
|
1557
|
+
if (reading.alive && prev.released) throw unavailable(`released session ${addressOf(prev)} still runs; the launch waits for its owner to end it`);
|
|
1504
1558
|
if (reading.alive && (prev.closed_at || prev.close_requested || prev.forgotten)) throw unavailable(`native session ${addressOf(prev)} is still live; its close must finish before replacement`);
|
|
1505
1559
|
}
|
|
1506
1560
|
if (prev?.forgotten) {
|
|
@@ -1750,25 +1804,13 @@ function adoptionOpenTo(db, id, request, open) {
|
|
|
1750
1804
|
return Boolean(claim) && claim.request === request && claim.lock === claimer();
|
|
1751
1805
|
}
|
|
1752
1806
|
|
|
1753
|
-
/**
|
|
1754
|
-
|
|
1755
|
-
* found through this machine's own doors: its panes, and the native session each pane's process runs. Only a full
|
|
1756
|
-
* address is looked up here; liveness is still verified below as for any other session.
|
|
1757
|
-
*/
|
|
1758
|
-
function localListing(who) {
|
|
1807
|
+
/** A session in no pane (a native process its machine's harness discovery holds as running), by its full address. */
|
|
1808
|
+
function nativeListing(who) {
|
|
1759
1809
|
const m = /^sc:([^:]+):([^:]+):(.+)$/.exec(String(who));
|
|
1760
|
-
if (!m ||
|
|
1761
|
-
const [, , harness,
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
const live = panes.filter((p) => !p.dead && p.pid);
|
|
1765
|
-
if (!live.length) return null;
|
|
1766
|
-
const r = runtimeDoor(null, 'harness.v1.sessions.activity_under', { pids: live.map((p) => p.pid) }, { timeout: 30_000 });
|
|
1767
|
-
const row = (r.code === 0 ? json(r.stdout.trim(), {})?.activities ?? [] : []).find((a) => a.activity?.session_id === sessionId && (a.activity?.harness ?? harness) === harness);
|
|
1768
|
-
// Not found in a pane of this machine: not listed, so a session opened moments ago is looked for again next tick.
|
|
1769
|
-
const pane = row ? live.find((p) => p.pid === row.pid)?.pane ?? null : null;
|
|
1770
|
-
if (!pane) return null;
|
|
1771
|
-
return { address: `sc:${localMachine()}:${harness}:${sessionId}`, machine: localMachine(), harness, session_id: sessionId, pane, name: null };
|
|
1810
|
+
if (!m || !HARNESSES[m[2]]) return null;
|
|
1811
|
+
const [, host, harness, session_id] = m;
|
|
1812
|
+
try { return nativeReading(nativeSessions(host, harness), { session_id, harness }).alive === true ? { address: who, machine: normalMachine(host), harness, session_id, pane: null, name: null } : null; }
|
|
1813
|
+
catch { return null; }
|
|
1772
1814
|
}
|
|
1773
1815
|
|
|
1774
1816
|
/** The verb an adoption with no current run fires: the workflow's own `adopt` where it declares one, else its dispatch start. */
|
|
@@ -1790,7 +1832,7 @@ function adoptedRole(db, board, entry, task, run) {
|
|
|
1790
1832
|
function adoptSession(board, id, who, { request = null, open = request, progress = {}, workspace = null, ttl, assignee = null, role = null, fleet = null, lanes = null } = {}) {
|
|
1791
1833
|
fleet ??= observeFleet();
|
|
1792
1834
|
if (!fleet.ok) throw new Error(`supercode could not read the fleet: ${fleet.error}`);
|
|
1793
|
-
const found = fleet.sessions.find((s) => s.address === who || s.address.endsWith(`:${who}`) || s.name === who) ??
|
|
1835
|
+
const found = fleet.sessions.find((s) => s.address === who || s.address.endsWith(`:${who}`) || s.name === who) ?? nativeListing(who);
|
|
1794
1836
|
if (!found) throw Object.assign(new Error(`the fleet lists no live session ${who}`), { notListed: true });
|
|
1795
1837
|
const [, host, harness, sessionId] = found.address.split(':');
|
|
1796
1838
|
if (!HARNESSES[harness]) throw new Error(`${found.address} is ${harness}; the board keeps and resumes ${Object.keys(HARNESSES).join(' and ')} sessions`);
|
|
@@ -1918,7 +1960,7 @@ function reconnectContinuedSessions(board, fleet, entry, report) {
|
|
|
1918
1960
|
// Preserve the ended interval as evidence before reopening supervision on the same run.
|
|
1919
1961
|
event(db,row.task_id,'session_continued',{session_id:saved.session_id,from_pane:saved.pane,pane,prior_outcome:row.outcome,prior_ended_at:row.ended_at,incident:saved.incident??null},row.id);
|
|
1920
1962
|
db.prepare("UPDATE task_runs SET ended_at=NULL,status='running',outcome=NULL,error=NULL,metadata=?,claim_lock=?,claim_expires=? WHERE id=?")
|
|
1921
|
-
.run(JSON.stringify({...json(row.metadata,{}),supercode:session}),claimer(),now()+900,row.id);
|
|
1963
|
+
.run(JSON.stringify({...json(db.prepare('SELECT metadata FROM task_runs WHERE id=?').get(row.id)?.metadata,{}),supercode:session}),claimer(),now()+900,row.id); // re-read: the run's observations just recorded stay
|
|
1922
1964
|
db.prepare('UPDATE tasks SET current_run_id=?,claim_lock=?,claim_expires=? WHERE id=?').run(row.id,claimer(),now()+900,row.task_id);
|
|
1923
1965
|
projectStatus(db,board,entry,row.task_id);
|
|
1924
1966
|
event(db,row.task_id,'dispatch_released',{reason:'verified manual continuation',session:addressOf(session),incident:saved.incident??null});
|
|
@@ -1983,10 +2025,43 @@ export async function alertManagers(root, slug, text, key) {
|
|
|
1983
2025
|
* and marked board_message_sent. A question goes under the id its card waits on; a resend of a sent one is a no-op
|
|
1984
2026
|
* (the id is the send's idempotency key). One that fails is tried again next tick.
|
|
1985
2027
|
*/
|
|
2028
|
+
const MAIL_READY_SECONDS = 300;
|
|
2029
|
+
/** Each board message not yet sent nor abandoned gets its effect row (once: the insert reads that none exists). */
|
|
2030
|
+
export function queueBoardMail(db) {
|
|
2031
|
+
db.prepare(`INSERT INTO supercode_effects (effect)
|
|
2032
|
+
SELECT json_object('kind', 'board_message', 'id', original.task_id, 'event', original.id) FROM task_events original
|
|
2033
|
+
WHERE original.kind = 'board_message'
|
|
2034
|
+
AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind = 'board_message_sent' AND sent.task_id = original.task_id AND json_extract(sent.payload, '$.event') = original.id)
|
|
2035
|
+
AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind = 'board_message_abandoned' AND gone.task_id = original.task_id AND json_extract(gone.payload, '$.event') = original.id)
|
|
2036
|
+
AND NOT EXISTS (SELECT 1 FROM supercode_effects e WHERE json_extract(e.effect, '$.kind') = 'board_message' AND json_extract(e.effect, '$.event') = original.id)`).run();
|
|
2037
|
+
}
|
|
2038
|
+
|
|
2039
|
+
/**
|
|
2040
|
+
* One board message's effect: its mail (with any batch it is in) sent under its fixed message id, so a second sender
|
|
2041
|
+
* is dropped by the mailbox. Answers true once it is sent, 'wait' while its batch window is open; throws the failure.
|
|
2042
|
+
*/
|
|
2043
|
+
export async function sendBoardMessage(board, e) {
|
|
2044
|
+
const state = () => view(board, (db) => db.prepare(`SELECT
|
|
2045
|
+
EXISTS (SELECT 1 FROM task_events WHERE kind='board_message_sent' AND task_id=? AND json_extract(payload,'$.event')=?) AS sent,
|
|
2046
|
+
EXISTS (SELECT 1 FROM task_events WHERE kind='board_message_abandoned' AND task_id=? AND json_extract(payload,'$.event')=?) AS gone`).get(e.id, e.event, e.id, e.event));
|
|
2047
|
+
if (state().sent || state().gone || !e.event) return true;
|
|
2048
|
+
const report = { undelivered: [] };
|
|
2049
|
+
await sendBoardMessages(board, report, e.id);
|
|
2050
|
+
if (state().sent) return true;
|
|
2051
|
+
const failed = report.undelivered.find(([task]) => task === e.id);
|
|
2052
|
+
if (failed) throw new Error(failed[2]);
|
|
2053
|
+
// a mail waits for its batch window (at most a minute); one still not ready well after that is an attempt that failed
|
|
2054
|
+
const recorded = view(board, (db) => db.prepare('SELECT created_at FROM task_events WHERE id = ?').get(e.event)?.created_at);
|
|
2055
|
+
if (recorded && now() - Number(recorded) > MAIL_READY_SECONDS) throw new Error(`the mail was not ready to send ${now() - Number(recorded)} s after it was recorded`);
|
|
2056
|
+
return 'wait';
|
|
2057
|
+
}
|
|
2058
|
+
|
|
1986
2059
|
export async function sendBoardMessages(board, report, taskId = null) {
|
|
1987
2060
|
const pending = view(board, db => db.prepare(`SELECT 1 FROM task_events original WHERE kind='board_message'
|
|
1988
2061
|
AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind='board_message_sent'
|
|
1989
|
-
AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
|
|
2062
|
+
AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
|
|
2063
|
+
AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind='board_message_abandoned'
|
|
2064
|
+
AND gone.task_id=original.task_id AND json_extract(gone.payload,'$.event')=original.id) LIMIT 1`).get());
|
|
1990
2065
|
if (!pending) return;
|
|
1991
2066
|
const planned = tx(board, db => pendingRequestMail(db, taskId));
|
|
1992
2067
|
if (Number.isFinite(planned.nextAt)) report.next_mail_at = Math.min(report.next_mail_at ?? Infinity, planned.nextAt);
|
|
@@ -1999,17 +2074,13 @@ export async function sendBoardMessages(board, report, taskId = null) {
|
|
|
1999
2074
|
const receipt = await send(board, boardAddress(board.label), board.label, mail.to, mail.text, mail.message_id ?? `board-message-${first.id}`, { ask: Boolean(mail.message_id), subject });
|
|
2000
2075
|
if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'message was not filed');
|
|
2001
2076
|
tx(board, db => {
|
|
2002
|
-
for (const member of mail.members) event(db, member.task_id, 'board_message_sent', { event: member.id, to: mail.to, receipt, ...(mail.message_id ? { message_id: mail.message_id, request_id: member.message_id ?? mail.message_id } : {}) });
|
|
2077
|
+
for (const member of mail.members) if (!db.prepare("SELECT 1 FROM task_events WHERE kind='board_message_sent' AND task_id=? AND json_extract(payload,'$.event')=?").get(member.task_id, member.id)) event(db, member.task_id, 'board_message_sent', { event: member.id, to: mail.to, receipt, ...(mail.message_id ? { message_id: mail.message_id, request_id: member.message_id ?? mail.message_id } : {}) });
|
|
2003
2078
|
});
|
|
2004
2079
|
} catch (error) {
|
|
2005
2080
|
const reason = String(error.message).slice(-200);
|
|
2006
2081
|
for (const member of mail.members) report.undelivered.push([member.task_id, mail.to, reason]);
|
|
2007
2082
|
tx(board, db => {
|
|
2008
|
-
for (const member of mail.members) {
|
|
2009
|
-
const previous = db.prepare("SELECT payload,created_at FROM task_events WHERE task_id=? AND kind='board_message_failed' AND json_extract(payload,'$.event')=? ORDER BY id DESC LIMIT 1").get(member.task_id, member.id);
|
|
2010
|
-
if (json(previous?.payload, {})?.reason !== reason || now() - (previous?.created_at ?? 0) >= 60)
|
|
2011
|
-
event(db, member.task_id, 'board_message_failed', { event: member.id, to: mail.to, message_id: mail.message_id, reason });
|
|
2012
|
-
}
|
|
2083
|
+
for (const member of mail.members) event(db, member.task_id, 'board_message_failed', { event: member.id, to: mail.to, message_id: mail.message_id, reason });
|
|
2013
2084
|
});
|
|
2014
2085
|
}
|
|
2015
2086
|
}
|
|
@@ -2018,8 +2089,10 @@ export async function sendBoardMessages(board, report, taskId = null) {
|
|
|
2018
2089
|
/** The board's own address: where its notices come from and where card updates are mailed. */
|
|
2019
2090
|
export const boardAddress = (label) => `sc:${localMachine()}:board:${label.replace(/[^A-Za-z0-9_.-]/g, '-')}`;
|
|
2020
2091
|
|
|
2021
|
-
/** Consecutive failed sends after which a subscription is dropped (Hermes's notifier drops a dead chat after 12). */
|
|
2092
|
+
/** Consecutive failed sends after which a chat-platform subscription is dropped (Hermes's notifier drops a dead chat after 12). */
|
|
2022
2093
|
const MAX_SEND_FAILURES = 12;
|
|
2094
|
+
/** Attempts a supercode notice gets before it is abandoned (its subscription stays). */
|
|
2095
|
+
const NOTICE_ATTEMPTS = 6;
|
|
2023
2096
|
|
|
2024
2097
|
/**
|
|
2025
2098
|
* What an audience is told of an event: the workflow's `notify[audience][kind]` template over the card, the event's
|
|
@@ -2099,13 +2172,16 @@ function nativeEnded(address, discovered) {
|
|
|
2099
2172
|
* A creator's or manager's subscription is never ended here: its notices wait for the session to come back.
|
|
2100
2173
|
*/
|
|
2101
2174
|
function endWorkerSubscriptions(board, fleet, report, natively = new Map()) {
|
|
2175
|
+
// each worker subscriber's liveness is read before the write
|
|
2176
|
+
const aliveNow = new Map();
|
|
2177
|
+
if (fleet.ok) for (const { chat_id: address } of board.db.prepare("SELECT DISTINCT chat_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) aliveNow.set(address, subscriberAlive(fleet, address));
|
|
2102
2178
|
tx(board, (db) => {
|
|
2103
2179
|
const { held, latest } = sessionRecord(db);
|
|
2104
2180
|
const remove = db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?");
|
|
2105
2181
|
const ended = new Map();
|
|
2106
2182
|
for (const row of db.prepare("SELECT task_id, chat_id, thread_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) {
|
|
2107
2183
|
const address = row.chat_id, record = latest.get(address);
|
|
2108
|
-
if (!record || record.session.released || held.has(address) || (fleet.ok &&
|
|
2184
|
+
if (!record || record.session.released || held.has(address) || (fleet.ok && aliveNow.get(address) === true)) continue;
|
|
2109
2185
|
const evidence = record.session.close_confirmed_at ? { by: 'board_close', run: record.run, closed_at: record.session.close_confirmed_at } : natively.get(address);
|
|
2110
2186
|
if (!evidence || !remove.run(row.task_id, address, row.thread_id).changes) continue;
|
|
2111
2187
|
if (!ended.has(address)) ended.set(address, { record, evidence, cards: [] });
|
|
@@ -2128,11 +2204,13 @@ async function notify(board, fleet, report, entry) {
|
|
|
2128
2204
|
// each subscription's notices go in order; different subscribers are told side by side
|
|
2129
2205
|
await pool(subscriptions(board.db).filter((sub) => sub.platform === 'supercode'), SEND_CONCURRENCY, async (sub) => {
|
|
2130
2206
|
if (sub.delivery_metadata?.dependent_card && sub.delivery_metadata.waiting_recipient) return;
|
|
2207
|
+
// a subscription whose last send failed waits out its retry time (as an effect does) before the next attempt
|
|
2208
|
+
if (Number(sub.delivery_metadata?.next_at ?? 0) > now()) return;
|
|
2131
2209
|
const role = ['worker', 'dependency', 'manager'].includes(sub.delivery_metadata?.role) ? sub.delivery_metadata.role : 'creator';
|
|
2132
2210
|
// a session the fleet does not list this tick is told next tick: one reading is not a loss (a loaded box drops
|
|
2133
2211
|
// sessions from a read, and a dropped subscription never came back), and a session that is replaced on its card
|
|
2134
2212
|
// has its subscription moved when the new one starts
|
|
2135
|
-
if (role !== 'creator' && fleet.ok && !fleet
|
|
2213
|
+
if (role !== 'creator' && fleet.ok && !subscriberAlive(fleet, sub.chat_id)) return;
|
|
2136
2214
|
const kinds = heardBy(entry, role);
|
|
2137
2215
|
if (!kinds.length) return;
|
|
2138
2216
|
const events = board.db.prepare(`SELECT id, kind, payload, created_at, run_id FROM task_events WHERE task_id = ? AND id > ? AND kind IN (${kinds.map(() => '?').join(', ')}) ORDER BY id`)
|
|
@@ -2144,7 +2222,13 @@ async function notify(board, fleet, report, entry) {
|
|
|
2144
2222
|
// the one the next run gets): the notice is filed in its mailbox, read when it next works
|
|
2145
2223
|
const working = role !== 'worker' || view(board, (db) => addressOf(currentRun(db, sub.task_id)?.metadata?.supercode) === sub.chat_id);
|
|
2146
2224
|
let cursor = Number(sub.last_event_id);
|
|
2147
|
-
|
|
2225
|
+
// A notice is sent first, under its fixed key (the mailbox drops a second send of it), and the cursor then only moves
|
|
2226
|
+
// forward past it: two passes may both send one notice, and its recipient gets it once; a send that fails, or a pass
|
|
2227
|
+
// that dies before its send, leaves the cursor behind the notice, so it is sent again.
|
|
2228
|
+
const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
|
|
2229
|
+
const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
|
|
2230
|
+
const forward = (to) => tx(board, (db) => db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(to, ...identity, to));
|
|
2231
|
+
let failed = null, untried = false, trying = null;
|
|
2148
2232
|
try {
|
|
2149
2233
|
// an event older than the workflow's notice_max_age_seconds is history, not news: it is passed over, never mailed
|
|
2150
2234
|
const maxAge = Number(entry.params.notice_max_age_seconds) || null;
|
|
@@ -2155,7 +2239,6 @@ async function notify(board, fleet, report, entry) {
|
|
|
2155
2239
|
const sendable = (ev) => { const p = json(ev.payload, {}) ?? {}; return !(maxAge && now() - Number(ev.created_at) > maxAge) && !(p.by && p.by === sub.chat_id); };
|
|
2156
2240
|
const latest = new Map(events.filter((ev) => !['commented', 'answered'].includes(ev.kind) && sendable(ev)).map((ev) => [ev.kind, ev.id]));
|
|
2157
2241
|
const sentBefore = board.db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND kind IN ('notice_delivered', 'notice_filed') AND json_extract(payload, '$.source_event') = ? AND json_extract(payload, '$.address') = ? LIMIT 1");
|
|
2158
|
-
const advance = (id) => tx(board, (db) => db.prepare("UPDATE kanban_notify_subs SET last_event_id = ? WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ? AND last_event_id < ?").run(id, sub.task_id, sub.chat_id, sub.thread_id ?? '', id));
|
|
2159
2242
|
for (const ev of events) {
|
|
2160
2243
|
const p = json(ev.payload, {}) ?? {};
|
|
2161
2244
|
if (maxAge && now() - Number(ev.created_at) > maxAge) { cursor = ev.id; (report.stale_notices_passed ??= 0); report.stale_notices_passed += 1; continue; }
|
|
@@ -2173,6 +2256,7 @@ async function notify(board, fleet, report, entry) {
|
|
|
2173
2256
|
if (text) {
|
|
2174
2257
|
const body = ['worker', 'dependency'].includes(role) ? identifyNotice(ev, text) : text;
|
|
2175
2258
|
if (refused.has(sub.chat_id) || stopping) { untried = true; break; }
|
|
2259
|
+
trying = ev.id;
|
|
2176
2260
|
let receipt;
|
|
2177
2261
|
try {
|
|
2178
2262
|
receipt = await send(board, from, board.label, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: String(sub.chat_id).includes(':claude-code:') ? false : !working || (!entry.workflow.run && sub.delivery_mode === 'notify'), subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
|
|
@@ -2180,13 +2264,16 @@ async function notify(board, fleet, report, entry) {
|
|
|
2180
2264
|
// board does not send it again.
|
|
2181
2265
|
if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'notice was not filed');
|
|
2182
2266
|
} catch (error) {
|
|
2267
|
+
// two passes may send one notice at once, and the second send can fail while the first is taken: a notice
|
|
2268
|
+
// another pass already recorded as taken is no failure
|
|
2269
|
+
if (sentBefore.get(sub.task_id, ev.id, sub.chat_id)) { cursor = ev.id; forward(ev.id); continue; }
|
|
2183
2270
|
if (!refused.has(sub.chat_id)) refused.set(sub.chat_id, error);
|
|
2184
2271
|
throw error;
|
|
2185
2272
|
}
|
|
2186
|
-
|
|
2273
|
+
// recorded once, whichever pass's send the mailbox took
|
|
2274
|
+
tx(board, (db) => { if (!db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND kind IN ('notice_delivered', 'notice_filed') AND json_extract(payload, '$.source_event') = ? AND json_extract(payload, '$.address') = ?").get(sub.task_id, ev.id, sub.chat_id)) event(db, sub.task_id, receipt.delivered ? 'notice_delivered' : 'notice_filed', { source_event: ev.id, kind: ev.kind, address: sub.chat_id, receipt }); });
|
|
2187
2275
|
report.notified.push([sub.task_id, ev.kind, sub.chat_id]);
|
|
2188
|
-
|
|
2189
|
-
advance(ev.id);
|
|
2276
|
+
forward(ev.id);
|
|
2190
2277
|
}
|
|
2191
2278
|
cursor = ev.id;
|
|
2192
2279
|
}
|
|
@@ -2209,16 +2296,22 @@ async function notify(board, fleet, report, entry) {
|
|
|
2209
2296
|
});
|
|
2210
2297
|
report.undelivered.push([sub.task_id, sub.chat_id, failed]);
|
|
2211
2298
|
}
|
|
2212
|
-
// the cursor moves past what was
|
|
2299
|
+
// the cursor moves forward past what was passed over (a failed delivery is retried next pass)
|
|
2300
|
+
forward(cursor);
|
|
2213
2301
|
tx(board, (db) => {
|
|
2214
|
-
const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
|
|
2215
|
-
const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
|
|
2216
|
-
db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(cursor, ...identity, cursor);
|
|
2217
2302
|
// a round that left this subscription untried (a delivery to its recipient failed on another card) says nothing of it
|
|
2218
2303
|
if (untried) return;
|
|
2304
|
+
// A notice is tried NOTICE_ATTEMPTS times, waiting as an effect does between attempts. After its last failed
|
|
2305
|
+
// attempt it is abandoned (its terminal state, `notice_abandoned`, recorded once) and the cursor moves past it;
|
|
2306
|
+
// the subscription stays, so the notices after it are still told (dropping it lost them when the round
|
|
2307
|
+
// subscribed the session again from the card's newest event).
|
|
2219
2308
|
const failures = failed ? Number(sub.delivery_metadata?.failures ?? 0) + 1 : 0;
|
|
2220
|
-
if (failures >=
|
|
2221
|
-
|
|
2309
|
+
if (failed && failures >= NOTICE_ATTEMPTS && trying) {
|
|
2310
|
+
event(db, sub.task_id, 'notice_abandoned', { source_event: trying, address: sub.chat_id, attempts: failures, error: failed });
|
|
2311
|
+
db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ?, delivery_metadata = ? ${where} AND last_event_id < ?`).run(trying, JSON.stringify({ ...sub.delivery_metadata, failures: 0, next_at: 0 }), ...identity, trying);
|
|
2312
|
+
return;
|
|
2313
|
+
}
|
|
2314
|
+
if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures, next_at: failed ? now() + Math.min(1800, 30 * 2 ** (failures - 1)) : 0 }), ...identity);
|
|
2222
2315
|
if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity);
|
|
2223
2316
|
});
|
|
2224
2317
|
});
|
|
@@ -2459,7 +2552,7 @@ const CORRUPT = /malformed|not a database|corrupt|disk image/i;
|
|
|
2459
2552
|
|
|
2460
2553
|
/** What a tick would start on each board of the home, planned through the workflow's `dispatch`, writing nothing. */
|
|
2461
2554
|
export async function plan({ root, lanes, settings, slugs = null, params = {} }) {
|
|
2462
|
-
|
|
2555
|
+
fleetRoot = root;
|
|
2463
2556
|
const loaded = await loadWorkflow(root, { fresh: true });
|
|
2464
2557
|
const entry = { ...loaded, params: { ...loaded.params, ...params } };
|
|
2465
2558
|
const fleet = Object.values(lanes).some((l) => l.surface === 'pane') ? observeFleet() : { ok: false, alive: new Set(), online: new Set(), sessions: [], error: 'no pane lanes' };
|
|
@@ -2559,20 +2652,81 @@ function capMax(cap, group, entry, lanes) {
|
|
|
2559
2652
|
catch { return null; }
|
|
2560
2653
|
}
|
|
2561
2654
|
|
|
2655
|
+
const ORCHESTRATOR_ENTRY = fileURLToPath(new URL('../bin/orchestrator.mjs', import.meta.url));
|
|
2656
|
+
const deliveries = new Map();
|
|
2657
|
+
/**
|
|
2658
|
+
* Start the home's delivery pass (`workflow deliver`) in a process of its own, unless one is already running: the
|
|
2659
|
+
* dispatcher's loop calls this after each round and never waits for it. `log(level, line)` hears what it failed at.
|
|
2660
|
+
* Answers whether one was started.
|
|
2661
|
+
*/
|
|
2662
|
+
export function startDelivery(root, { log = () => {} } = {}) {
|
|
2663
|
+
if (deliveries.has(root)) return false;
|
|
2664
|
+
const child = spawnChild(process.execPath, [ORCHESTRATOR_ENTRY, 'workflow', 'deliver', '--root', root, '--json'], { stdio: ['ignore', 'pipe', 'pipe'], env: process.env });
|
|
2665
|
+
deliveries.set(root, child);
|
|
2666
|
+
let out = '', err = '';
|
|
2667
|
+
child.stdout.setEncoding('utf8').on('data', (c) => { out += c; });
|
|
2668
|
+
child.stderr.setEncoding('utf8').on('data', (c) => { err = (err + c).slice(-2000); });
|
|
2669
|
+
const done = () => {
|
|
2670
|
+
deliveries.delete(root);
|
|
2671
|
+
let reports = null; try { reports = JSON.parse(out); } catch { log('warn', `delivery pass failed: ${(err || out).trim().slice(-300) || 'no answer'}`); return; }
|
|
2672
|
+
for (const r of reports) {
|
|
2673
|
+
if (r.error) log('warn', `delivery on board ${r.board} failed: ${r.error}`);
|
|
2674
|
+
for (const [task, to, why] of r.undelivered ?? []) log('warn', `board ${r.board}: a delivery for ${task} to ${to} waits: ${why}`);
|
|
2675
|
+
log('info', `delivery on board ${r.board}: ${r.effects_waiting ?? 0} effect(s) waiting, ${r.effects_failed ?? 0} failed for good; took ${Object.entries(r.ms ?? {}).map(([k, v]) => `${k} ${(v / 1000).toFixed(1)}s`).join(', ')}`);
|
|
2676
|
+
}
|
|
2677
|
+
};
|
|
2678
|
+
child.on('error', (error) => { log('warn', `delivery pass did not start: ${error.message}`); deliveries.delete(root); });
|
|
2679
|
+
child.on('close', done);
|
|
2680
|
+
return true;
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2683
|
+
/**
|
|
2684
|
+
* The delivery pass over every board of the home at `root`: what rounds recorded for after them (each due effect, with
|
|
2685
|
+
* its bounded attempts), board mail, subscription notices, and the board's agent records joined to Teams. It runs in
|
|
2686
|
+
* its own process (`workflow deliver`, started by serve after a round, one at a time), never inside a dispatch round,
|
|
2687
|
+
* so no recipient, door or Teams answer holds a round. Answers one report per board.
|
|
2688
|
+
*/
|
|
2689
|
+
export async function deliver({ root, slugs = null }) {
|
|
2690
|
+
fleetRoot = root;
|
|
2691
|
+
const entry = await loadWorkflow(root, { fresh: true });
|
|
2692
|
+
const fleet = observeFleet();
|
|
2693
|
+
const out = [];
|
|
2694
|
+
for (const slug of slugs ?? listBoards(root)) {
|
|
2695
|
+
const report = { board: slug, ms: {}, mailed: [], queued: [], failed: [], notified: [], undelivered: [], fleet: fleet.ok ? 'read' : fleet.error };
|
|
2696
|
+
let board;
|
|
2697
|
+
try {
|
|
2698
|
+
board = openBoard(root, slug);
|
|
2699
|
+
board.label = boardLabel(root, slug);
|
|
2700
|
+
setActor(boardAddress(board.label));
|
|
2701
|
+
let mark = Date.now();
|
|
2702
|
+
const phase = (name) => { const t = Date.now(); report.ms[name] = t - mark; mark = t; };
|
|
2703
|
+
// board mail is sent as effect rows, one per message, each claimed as any effect is
|
|
2704
|
+
tx(board, (db) => queueBoardMail(db)); phase('mail');
|
|
2705
|
+
report.effects = await runEffects(board, [], { pending: true }); phase('effects');
|
|
2706
|
+
await notify(board, fleet, report, entry); phase('notify');
|
|
2707
|
+
synchronizeAgents(board, fleet, report, entry); phase('agents');
|
|
2708
|
+
report.effects_failed = Number(board.db.prepare('SELECT COUNT(*) AS n FROM supercode_effects WHERE failed_at IS NOT NULL').get().n);
|
|
2709
|
+
report.effects_waiting = Number(board.db.prepare('SELECT COUNT(*) AS n FROM supercode_effects WHERE failed_at IS NULL').get().n);
|
|
2710
|
+
} catch (error) { report.error = error.message; }
|
|
2711
|
+
finally { board?.close(); }
|
|
2712
|
+
out.push(report);
|
|
2713
|
+
}
|
|
2714
|
+
return out;
|
|
2715
|
+
}
|
|
2716
|
+
|
|
2562
2717
|
/**
|
|
2563
2718
|
* One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
|
|
2564
2719
|
* `only` limits starting to one card (a create that answers whether its card started). Each board is ticked under its
|
|
2565
2720
|
* version-checked card claims; no dispatcher file lock. Two rounds may run at once (the dispatcher's and a one-shot
|
|
2566
2721
|
* `workflow dispatch`). An adoption request and a session close are claimed in a transaction (`newClaim`) before
|
|
2567
2722
|
* anything is done for them outside the board; card starts, supervision, review requests, mailed verbs and manual
|
|
2568
|
-
* continuations rely on their per-card transactions. Two rounds on one board at once can still both act on:
|
|
2569
|
-
*
|
|
2570
|
-
*
|
|
2571
|
-
*
|
|
2572
|
-
* board-only writes that do not re-check (a doubled agent_profile_recorded). Answers one report per board.
|
|
2723
|
+
* continuations rely on their per-card transactions. Two rounds on one board at once can still both act on:
|
|
2724
|
+
* deferred-launch retries (two panes), launch-receipt work (cancelUnissuedLaunch, bindLaunch), and board-only writes
|
|
2725
|
+
* that do not re-check. A round runs no effect and delivers nothing: it records effects (`queueEffects`) and the
|
|
2726
|
+
* delivery pass (`deliver`) carries them out. Answers one report per board.
|
|
2573
2727
|
*/
|
|
2574
2728
|
export async function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true, params = {}, deploymentReceipt = false }) {
|
|
2575
|
-
|
|
2729
|
+
fleetRoot = root;
|
|
2576
2730
|
// a verb's flags (dispatch --max, --failure-limit) are params of this tick, read by the workflow as its own
|
|
2577
2731
|
const deployment = deploymentReceipt ? workflowRevision(root) : null;
|
|
2578
2732
|
const loaded = await loadWorkflow(root, { fresh: true });
|
|
@@ -2611,20 +2765,18 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
|
|
|
2611
2765
|
readMailbox(board, boardAddress(board.label), report); phase('mailbox');
|
|
2612
2766
|
await refreshRelationAnswers(board); phase('answers');
|
|
2613
2767
|
const effects = [];
|
|
2614
|
-
completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects);
|
|
2768
|
+
completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); queueEffects(board, effects.splice(0)); closeRequested(board, report, { ttl: settings.ttl }); phase('supervise');
|
|
2615
2769
|
report.unsettled = [];
|
|
2616
2770
|
report.promoted = recomputeReady(board, effects, report.unsettled);
|
|
2617
2771
|
tx(board,db=>auditEndedRuns(db,board,entry));
|
|
2618
2772
|
if (dispatch) await activatePullRequestReviews(board, entry, report, effects, { only });
|
|
2619
|
-
await sendBoardMessages(board, report);
|
|
2620
2773
|
if (dispatch && !stopping) startCards(board, fleet, lanes, settings, report, entry, effects, { only, held });
|
|
2621
2774
|
phase('start');
|
|
2622
2775
|
if (dispatch && !only && !report.paused) observeIdle(board, fleet, settings, report, entry, effects);
|
|
2623
|
-
// what the moves asked for
|
|
2624
|
-
|
|
2776
|
+
// what the moves asked for is recorded with them; the delivery pass (`deliver`) carries it out, outside the round
|
|
2777
|
+
queueEffects(board, effects);
|
|
2625
2778
|
phase('goals');
|
|
2626
2779
|
ensureSubscriptions(board, entry);
|
|
2627
|
-
await notify(board, fleet, report, entry); phase('notify');
|
|
2628
2780
|
fireHooks(board, settings, report);
|
|
2629
2781
|
if (String(board.db.prepare('PRAGMA journal_mode').get()?.journal_mode).toLowerCase() === 'wal') board.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get();
|
|
2630
2782
|
report.ready_waiting = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks t LEFT JOIN supercode_cards c ON c.task_id = t.id WHERE t.status = 'ready' AND t.assignee IS NOT NULL AND c.subtask_of IS NULL").get().n);
|