@volter/supercode-orchestrator 0.4.28 → 0.4.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/board/cli.mjs +70 -43
- package/board/dispatch.mjs +342 -287
- package/board/engine.mjs +151 -310
- package/board/expr.mjs +212 -0
- package/board/store.mjs +4 -2
- package/board/tools.mjs +17 -8
- package/board/workflow.mjs +442 -0
- package/board.mjs +8 -4
- package/kanban-worker.mjs +2 -1
- package/loop.mjs +1 -1
- package/package.json +1 -1
package/board/dispatch.mjs
CHANGED
|
@@ -20,11 +20,13 @@ import { hostname, tmpdir } from 'node:os';
|
|
|
20
20
|
import { basename, dirname, join } from 'node:path';
|
|
21
21
|
import { randomUUID } from 'node:crypto';
|
|
22
22
|
import { fileURLToPath } from 'node:url';
|
|
23
|
-
import { boardPath, event, json, listBoards, now, openBoard, tx } from './store.mjs';
|
|
23
|
+
import { boardPath, claimer, event, json, listBoards, now, openBoard, tx } from './store.mjs';
|
|
24
24
|
import { holdBoardTickLock } from '../hermes-locks.mjs';
|
|
25
|
+
import { doors, fire, loadWorkflow, resumeOf, scopeOf, statusOf, templateValue, workflowFor } from './workflow.mjs';
|
|
26
|
+
import { evaluate, holds, render } from './expr.mjs';
|
|
25
27
|
import {
|
|
26
|
-
|
|
27
|
-
recomputeReady,
|
|
28
|
+
cardContext, claimTtl, currentRun, DEFAULT_FAILURE_LIMIT, extendClaim, getTask, lastSession,
|
|
29
|
+
recomputeReady, runEffects, setRunSession, subscribeIn, subscriptions,
|
|
28
30
|
} from './engine.mjs';
|
|
29
31
|
|
|
30
32
|
const WORKER = fileURLToPath(new URL('../bin/hermes-worker.mjs', import.meta.url));
|
|
@@ -145,9 +147,8 @@ export function lanesOf(profiles = {}) {
|
|
|
145
147
|
|
|
146
148
|
// ------------------------------------------------------------------ supervise
|
|
147
149
|
|
|
148
|
-
function supervise(board, fleet, settings, report) {
|
|
150
|
+
function supervise(board, fleet, settings, report, entry, effects) {
|
|
149
151
|
const rows = board.db.prepare("SELECT id FROM tasks WHERE status = 'running'").all();
|
|
150
|
-
const stop = [];
|
|
151
152
|
for (const { id } of rows) {
|
|
152
153
|
tx(board, (db) => {
|
|
153
154
|
const task = getTask(db, id);
|
|
@@ -155,28 +156,34 @@ function supervise(board, fleet, settings, report) {
|
|
|
155
156
|
const runRow = currentRun(db, id);
|
|
156
157
|
const session = runRow?.metadata?.supercode ?? null;
|
|
157
158
|
const t = now();
|
|
159
|
+
// what the dispatcher saw becomes a platform event; the card's workflow says what it does
|
|
160
|
+
const observe = (name, payload, bucket) => {
|
|
161
|
+
try {
|
|
162
|
+
const r = fire(db, board, entry, id, name, { roles: ['platform'], payload });
|
|
163
|
+
effects.push(...r.effects);
|
|
164
|
+
if (bucket) report[bucket].push(id);
|
|
165
|
+
if (r.to !== r.from && entry.workflow.statuses[r.to]?.stored_as === 'blocked') report.auto_blocked.push(id);
|
|
166
|
+
} catch (error) {
|
|
167
|
+
// an observation no transition takes is nothing to do (a guard the workflow keeps, such as a grace)
|
|
168
|
+
if (!error.noTransition) report.failed.push([id, `${name}: ${String(error.message).slice(-200)}`]);
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
const limits = limitsOf(db, board, entry, task, session);
|
|
158
172
|
// Hermes's orphan reconciliation: a running card whose claim bookkeeping is broken, with nothing live behind it
|
|
159
173
|
const live = session?.surface === 'headless' ? pidAlive(session.pid) : session?.session_id ? (fleet.ok ? sessionAlive(fleet, session) : true) : false;
|
|
160
174
|
if (settings.reconcileOrphans && (!task.claim_lock || !task.claim_expires) && !live) {
|
|
161
|
-
|
|
162
|
-
db.prepare('INSERT INTO task_comments (task_id, author, body, created_at) VALUES (?, ?, ?, ?)').run(id, 'dispatcher', 'Requeued: the card was running with no claim and nothing live behind it.', t);
|
|
163
|
-
report.reclaimed.push(id);
|
|
175
|
+
observe('orphaned', { reason: 'orphaned_running', claim_lock: task.claim_lock, claim_expires: task.claim_expires, error: 'orphaned running card (broken claim bookkeeping)' }, 'reclaimed');
|
|
164
176
|
return;
|
|
165
177
|
}
|
|
166
|
-
// a
|
|
167
|
-
if (
|
|
168
|
-
|
|
169
|
-
release(db, id, { outcome: 'timed_out', error: `ran past max_runtime ${task.max_runtime_seconds}s`, kind: 'timed_out', payload: { limit_seconds: task.max_runtime_seconds } });
|
|
170
|
-
if (recordFailure(db, id, `timed out after ${task.max_runtime_seconds}s`, { failureLimit: settings.failureLimit, outcome: 'timed_out' })) report.auto_blocked.push(id);
|
|
171
|
-
report.timed_out.push(id);
|
|
178
|
+
// a run past its runtime limit (the status's `run.limits.runtime`), as Hermes times its workers out
|
|
179
|
+
if (limits.runtime && runRow && t - Number(runRow.started_at) > Number(limits.runtime)) {
|
|
180
|
+
observe('timed_out', { limit_seconds: limits.runtime, error: `ran past max_runtime ${limits.runtime}s` }, 'timed_out');
|
|
172
181
|
return;
|
|
173
182
|
}
|
|
174
183
|
if (session?.surface === 'headless') {
|
|
175
|
-
// Hermes's stale check: a worker running past the
|
|
176
|
-
if (
|
|
177
|
-
|
|
178
|
-
release(db, id, { outcome: 'stale', error: `no heartbeat after ${t - Number(runRow.started_at)}s running`, kind: 'stale', payload: { timeout_seconds: settings.staleTimeout, pid: session.pid } });
|
|
179
|
-
report.reclaimed.push(id);
|
|
184
|
+
// Hermes's stale check: a worker running past the limit with no heartbeat in the last hour is stopped
|
|
185
|
+
if (limits.stale && runRow && t - Number(runRow.started_at) >= Number(limits.stale) && !(task.last_heartbeat_at && t - Number(task.last_heartbeat_at) < 3600)) {
|
|
186
|
+
observe('stale', { timeout_seconds: limits.stale, pid: session.pid, error: `no heartbeat after ${t - Number(runRow.started_at)}s running` }, 'reclaimed');
|
|
180
187
|
return;
|
|
181
188
|
}
|
|
182
189
|
if (pidAlive(session.pid)) {
|
|
@@ -186,39 +193,32 @@ function supervise(board, fleet, settings, report) {
|
|
|
186
193
|
if (t - Number(session.launched_at ?? 0) < settings.crashGrace) return;
|
|
187
194
|
// how the worker ended, from the exit code its wrapper wrote (Hermes reads its reaped children's)
|
|
188
195
|
const code = session.exit_file && existsSync(session.exit_file) ? Number(readFileSync(session.exit_file, 'utf8').trim()) : null;
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
release(db, id, { outcome: 'rate_limited', error: 'worker hit a rate limit (exit 75)', kind: 'rate_limited', payload: { worker_pid: session.pid, exit_code: 75 } });
|
|
192
|
-
report.crashed.push(id);
|
|
193
|
-
return;
|
|
194
|
-
}
|
|
195
|
-
const clean = code === 0;
|
|
196
|
-
const why = clean ? `worker ${session.pid} exited cleanly (rc=0) without completing or blocking the card` : `worker ${session.pid} exited${code === null ? '' : ` with ${code}`} without completing the card`;
|
|
197
|
-
release(db, id, { outcome: 'crashed', error: why, kind: 'crashed', payload: { worker_pid: session.pid, exit_code: code, protocol_violation: clean } });
|
|
198
|
-
// a clean exit that left the card running is a protocol violation: Hermes trips the breaker on the first
|
|
199
|
-
if (clean) db.prepare('UPDATE tasks SET consecutive_failures = ? WHERE id = ?').run(Math.max(0, (task.max_retries ?? settings.failureLimit) - 1), id);
|
|
200
|
-
if (recordFailure(db, id, why, { failureLimit: settings.failureLimit })) report.auto_blocked.push(id);
|
|
201
|
-
report.crashed.push(id);
|
|
196
|
+
const why = code === 75 ? 'worker hit a rate limit (exit 75)' : code === 0 ? `worker ${session.pid} exited cleanly (rc=0) without completing or blocking the card` : `worker ${session.pid} exited${code === null ? '' : ` with ${code}`} without completing the card`;
|
|
197
|
+
observe('exited', { code, worker_pid: session.pid, exit_code: code, protocol_violation: code === 0, error: why }, 'crashed');
|
|
202
198
|
return;
|
|
203
199
|
}
|
|
204
200
|
if (session?.session_id) {
|
|
205
201
|
if (!fleet.ok) return; // no reading, no verdict
|
|
206
202
|
const alive = sessionAlive(fleet, session);
|
|
207
203
|
const machineUp = fleet.online.has(session.machine);
|
|
208
|
-
const young = t - Number(session.launched_at ?? 0) < settings.grace;
|
|
204
|
+
const young = t - Number(session.launched_at ?? 0) < Number(limits.grace ?? settings.grace);
|
|
209
205
|
const missing = !alive && machineUp && !young;
|
|
210
206
|
const next = { ...session };
|
|
207
|
+
// a machine that is off the fleet: how long, for the workflow's machine_offline (the claim is kept meanwhile)
|
|
208
|
+
if (machineUp) delete next.offline_since; else next.offline_since ??= t;
|
|
209
|
+
if (!machineUp) {
|
|
210
|
+
setRunSession(db, runRow.id, next);
|
|
211
|
+
observe('machine_offline', { machine: session.machine, seconds: t - next.offline_since }, null);
|
|
212
|
+
if (getTask(db, id)?.current_run_id !== runRow.id) return; // the workflow moved it
|
|
213
|
+
}
|
|
211
214
|
if (alive) { next.seen_at ??= t; delete next.missing_since; }
|
|
212
215
|
if (missing) next.missing_since ??= t;
|
|
213
|
-
if (missing && t - next.missing_since >= settings.grace) {
|
|
214
|
-
// lost: gone from an online machine for the whole grace. A resume that never came up is not resumable
|
|
215
|
-
// a fresh start that never came up is a failure. The card goes back to ready and the spawn resumes it.
|
|
216
|
+
if (missing && t - next.missing_since >= Number(limits.grace ?? settings.grace)) {
|
|
217
|
+
// lost: gone from an online machine for the whole grace. A resume that never came up is not resumable.
|
|
216
218
|
if (next.mode === 'resume' && !next.seen_at) next.resumable = false;
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
report.lost.push(id);
|
|
221
|
-
if (next.mode === 'fresh' && !next.seen_at && recordFailure(db, id, `session ${next.session_id} never came up on ${next.machine}`, { failureLimit: settings.failureLimit, outcome: 'crashed' })) report.auto_blocked.push(id);
|
|
219
|
+
setRunSession(db, runRow.id, next);
|
|
220
|
+
observe('session_lost', { session_id: next.session_id, machine: next.machine, pane: next.pane, mode: next.mode, seen: Boolean(next.seen_at),
|
|
221
|
+
never_came_up: next.mode === 'fresh' && !next.seen_at, error: `session ${next.session_id} is gone from ${next.machine}` }, 'lost');
|
|
222
222
|
return;
|
|
223
223
|
}
|
|
224
224
|
if (JSON.stringify(next) !== JSON.stringify(session)) setRunSession(db, runRow.id, next);
|
|
@@ -229,13 +229,21 @@ function supervise(board, fleet, settings, report) {
|
|
|
229
229
|
}
|
|
230
230
|
// claimed with nothing launched (a dispatcher that died between the two, or another implementation's claim):
|
|
231
231
|
// the claim's own expiry answers it, as Hermes's does
|
|
232
|
-
if (task.claim_expires && Number(task.claim_expires) < t) {
|
|
233
|
-
release(db, id, { outcome: 'reclaimed', error: `stale_lock=${task.claim_lock}`, kind: 'reclaimed', payload: { stale_lock: task.claim_lock, claim_expires: task.claim_expires } });
|
|
234
|
-
report.reclaimed.push(id);
|
|
235
|
-
}
|
|
232
|
+
if (task.claim_expires && Number(task.claim_expires) < t) observe('orphaned', { stale_lock: task.claim_lock, claim_expires: task.claim_expires, error: `stale_lock=${task.claim_lock}` }, 'reclaimed');
|
|
236
233
|
});
|
|
237
234
|
}
|
|
238
|
-
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** A running card's limits: its status's `run.limits`, each evaluated for the card (seconds; null for none). */
|
|
238
|
+
function limitsOf(db, board, entry, task, session) {
|
|
239
|
+
const status = statusOf(db, task);
|
|
240
|
+
const scope = scopeOf(db, board, task, { params: entry.params, extra: { run: { surface: session?.surface ?? null, local: true } } });
|
|
241
|
+
const out = {};
|
|
242
|
+
for (const [name, expr] of Object.entries(entry.workflow.statuses[status]?.run?.limits ?? {})) {
|
|
243
|
+
const v = evaluate(expr, scope);
|
|
244
|
+
out[name] = v === null || v === undefined || v === '' ? null : Number(v);
|
|
245
|
+
}
|
|
246
|
+
return out;
|
|
239
247
|
}
|
|
240
248
|
|
|
241
249
|
function stopSession(session) {
|
|
@@ -251,6 +259,208 @@ function stopSession(session) {
|
|
|
251
259
|
if (session.pane) sc(['teams', 'panes', 'close', session.pane, ...(session.machine ? ['--machine', session.machine] : [])], { timeout: 60_000 });
|
|
252
260
|
}
|
|
253
261
|
|
|
262
|
+
// ------------------------------------------------------------------ starting, as the workflow's dispatch says
|
|
263
|
+
|
|
264
|
+
/** The command form a card's session uses to reach its board: `--root` on the board's machine, `--at` elsewhere. */
|
|
265
|
+
export function whereOf(board, task) {
|
|
266
|
+
const local = !task.machine || normalMachine(task.machine) === localMachine();
|
|
267
|
+
return local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(board.label)}`;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// what the platform tells expressions about this machine and this tick
|
|
271
|
+
const hostOf = (pressure) => ({ memory_pressure: pressure, max_in_progress: derivedMaxInProgress() });
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Start what the workflow's dispatcher may start this tick: cards whose status has a transition for `dispatch.start`,
|
|
275
|
+
* that `dispatch.eligible` admits, in `dispatch.order`, under `dispatch.limits` (a cap per group, slots reserved for
|
|
276
|
+
* the cards a reserve names while any waits). Each is sent the start event (its validators are Hermes's respawn
|
|
277
|
+
* guard) and what its new status runs is launched.
|
|
278
|
+
*/
|
|
279
|
+
function startCards(board, fleet, lanes, settings, report, entry, effects, { only = null, hostRunning = 0 } = {}) {
|
|
280
|
+
const stop = paused(board.root);
|
|
281
|
+
if (stop) { report.paused = stop; return; }
|
|
282
|
+
const { workflow, params } = entry;
|
|
283
|
+
const d = workflow.dispatch;
|
|
284
|
+
const pressure = memoryPressure();
|
|
285
|
+
const tickState = { local_started: 0 };
|
|
286
|
+
const startable = new Set(Object.entries(workflow.statuses).filter(([, st]) => st.on?.[d.start]).map(([name]) => name));
|
|
287
|
+
const everyCard = board.db.prepare("SELECT id FROM tasks WHERE status != 'archived'").all().map((r) => getTask(board.db, r.id));
|
|
288
|
+
const scopeFor = (task, extra = {}) => scopeOf(board.db, board, task, { params, extra: { host: hostOf(pressure), tick: tickState, ...extra } });
|
|
289
|
+
// who would run it: the role of the status its start moves it to, and that role's lane
|
|
290
|
+
const planOf = (task, scope) => {
|
|
291
|
+
const status = statusOf(board.db, task);
|
|
292
|
+
const t = (workflow.statuses[status]?.on?.[d.start] ?? []).find((x) => (!x.by?.length || x.by.includes('platform')) && holds(x.guard ?? null, scope));
|
|
293
|
+
if (!t?.target) return null;
|
|
294
|
+
const target = String(templateValue(t.target, scope));
|
|
295
|
+
const runSpec = workflow.statuses[target]?.run ?? null;
|
|
296
|
+
const profile = runSpec ? evaluate(workflow.roles?.[runSpec.role] ?? 'card.assignee', scope) : null;
|
|
297
|
+
return { target, runSpec, profile, lane: profile ? lanes[profile] ?? null : null };
|
|
298
|
+
};
|
|
299
|
+
const candidates = [];
|
|
300
|
+
for (const task of everyCard) {
|
|
301
|
+
if (only && task.id !== only) continue;
|
|
302
|
+
if (!startable.has(statusOf(board.db, task))) continue;
|
|
303
|
+
const plan = planOf(task, scopeFor(task));
|
|
304
|
+
if (!plan) continue;
|
|
305
|
+
const scope = scopeFor(task, { lane: plan.lane ?? {}, run: { surface: plan.lane?.surface ?? null, local: plan.lane?.surface !== 'pane' || normalMachine(task.machine ?? '') === localMachine() } });
|
|
306
|
+
if (!holds(d.eligible ?? null, scope)) continue;
|
|
307
|
+
candidates.push({ task, plan, scope });
|
|
308
|
+
}
|
|
309
|
+
const keyOf = (c) => (d.order ?? []).map((o) => { const desc = o.startsWith('-'); const v = evaluate(desc ? o.slice(1) : o, c.scope); return { v, desc }; });
|
|
310
|
+
candidates.sort((a, b) => {
|
|
311
|
+
const ka = keyOf(a); const kb = keyOf(b);
|
|
312
|
+
for (let i = 0; i < ka.length; i += 1) {
|
|
313
|
+
if (ka[i].v === kb[i].v) continue;
|
|
314
|
+
const lt = (ka[i].v ?? -Infinity) < (kb[i].v ?? -Infinity);
|
|
315
|
+
return (lt ? -1 : 1) * (ka[i].desc ? -1 : 1);
|
|
316
|
+
}
|
|
317
|
+
return 0;
|
|
318
|
+
});
|
|
319
|
+
// the caps: how many counted cards each group holds now
|
|
320
|
+
const limits = (d.limits ?? []).map((l) => ({ ...l, tally: new Map() }));
|
|
321
|
+
const cardScope = new Map(everyCard.map((c) => [c.id, scopeFor(c, { lane: lanes[c.assignee] ?? {}, fleet_alive: fleet.ok ? sessionAlive(fleet, lastSession(board.db, c.id)) : false })]));
|
|
322
|
+
for (const l of limits) {
|
|
323
|
+
for (const [, sc] of cardScope) {
|
|
324
|
+
sc.card.session_live = sc.fleet_alive;
|
|
325
|
+
if (!holds(l.counts, sc)) continue;
|
|
326
|
+
const g = JSON.stringify(evaluate(l.group, sc));
|
|
327
|
+
l.tally.set(g, (l.tally.get(g) ?? 0) + 1);
|
|
328
|
+
}
|
|
329
|
+
// the home's other boards count toward a group that is the whole home (kanban.max_in_progress across boards)
|
|
330
|
+
const here = [...cardScope.values()].filter((sc) => holds(l.counts, sc)).length;
|
|
331
|
+
if (evaluate(l.group, { card: {} }) === 'home') l.tally.set(JSON.stringify('home'), (l.tally.get(JSON.stringify('home')) ?? 0) + Math.max(0, hostRunning - here));
|
|
332
|
+
}
|
|
333
|
+
const reserveWaits = (l) => l.reserve && candidates.some((c) => holds(l.reserve.for, c.scope));
|
|
334
|
+
for (const c of candidates) {
|
|
335
|
+
const { task, plan, scope } = c;
|
|
336
|
+
if (!plan.profile) { report.unassigned.push(task.id); tx(board, (db) => noteGuard(db, task.id, 'no profile runs it (no assignee, and the workflow names none)')); continue; }
|
|
337
|
+
if (!plan.lane) { report.nonspawnable.push(task.id); tx(board, (db) => noteGuard(db, task.id, `${plan.profile} is no lane of this home`)); continue; }
|
|
338
|
+
let why = null;
|
|
339
|
+
for (const l of limits) {
|
|
340
|
+
const max = evaluate(l.max, scope);
|
|
341
|
+
if (max === null || max === undefined || max === '') continue;
|
|
342
|
+
const g = JSON.stringify(evaluate(l.group, scope));
|
|
343
|
+
const held = reserveWaits(l) && !holds(l.reserve.for, scope) ? Number(evaluate(l.reserve.slots, scope) ?? 0) : 0;
|
|
344
|
+
if ((l.tally.get(g) ?? 0) >= Number(max) - held) { why = `${l.description ?? 'a cap'} (${max}${held ? `, ${held} held for others` : ''})`; break; }
|
|
345
|
+
}
|
|
346
|
+
// the launcher's own conditions: a lane that can start this card where it names
|
|
347
|
+
if (!why) why = launcherCheck(board.db, task, plan.lane, fleet);
|
|
348
|
+
if (why) { tx(board, (db) => noteGuard(db, task.id, why)); report.queued.push([task.id, why]); continue; }
|
|
349
|
+
let moved;
|
|
350
|
+
try {
|
|
351
|
+
moved = tx(board, (db) => {
|
|
352
|
+
if (!task.assignee && plan.profile) {
|
|
353
|
+
db.prepare('UPDATE tasks SET assignee = ? WHERE id = ?').run(plan.profile, task.id);
|
|
354
|
+
event(db, task.id, 'assigned', { assignee: plan.profile, source: 'workflow role' });
|
|
355
|
+
}
|
|
356
|
+
return fire(db, board, entry, task.id, d.start, { roles: ['platform'], payload: { lock: claimer() }, extra: { host: hostOf(pressure), tick: tickState } });
|
|
357
|
+
});
|
|
358
|
+
} catch (error) {
|
|
359
|
+
const reason = error.refusal ?? String(error.message).slice(-300);
|
|
360
|
+
tx(board, (db) => noteGuard(db, task.id, reason));
|
|
361
|
+
report.queued.push([task.id, reason]);
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
effects.push(...moved.effects);
|
|
365
|
+
for (const l of limits) if (holds(l.counts, cardScope.get(task.id) ?? scope)) { const g = JSON.stringify(evaluate(l.group, scope)); l.tally.set(g, (l.tally.get(g) ?? 0) + 1); }
|
|
366
|
+
try {
|
|
367
|
+
const session = launchRun(board, entry, task.id, plan.lane, fleet, settings);
|
|
368
|
+
if (session) {
|
|
369
|
+
if (session.surface !== 'pane' || normalMachine(task.machine ?? '') === localMachine()) tickState.local_started += 1;
|
|
370
|
+
report.started.push([task.id, session.address ?? `pid ${session.pid}`, session.mode]);
|
|
371
|
+
}
|
|
372
|
+
} catch (error) {
|
|
373
|
+
try { const r = tx(board, (db) => fire(db, board, entry, task.id, 'spawn_failed', { roles: ['platform'], payload: { error: String(error.message).slice(-500) } })); effects.push(...r.effects); } catch { /* no transition for it */ }
|
|
374
|
+
report.failed.push([task.id, String(error.message).slice(-300)]);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/** Launch what the card's status runs: its role's lane, in the run's session slot, told the run's prompt and doors. */
|
|
380
|
+
export function launchRun(board, entry, id, lane, fleet, settings) {
|
|
381
|
+
const task = getTaskNow(board, id);
|
|
382
|
+
const status = tx(board, (db) => statusOf(db, task));
|
|
383
|
+
const spec = entry.workflow.statuses[status]?.run;
|
|
384
|
+
if (!spec) return null;
|
|
385
|
+
const runId = task.current_run_id;
|
|
386
|
+
const surface = lane.surface === 'pane' ? 'pane' : 'headless';
|
|
387
|
+
const scope = tx(board, (db) => scopeOf(db, board, task, { params: entry.params }));
|
|
388
|
+
const where = whereOf(board, task);
|
|
389
|
+
const prompt = render(spec.prompt, { ...scope, where, context: tx(board, (db) => cardContext(db, id)), doors: doors(entry.workflow, status, spec.role, id, where, surface) });
|
|
390
|
+
// the run's session record knows its status, role and slot (a reviewer's slot is never the implementer's)
|
|
391
|
+
const meta = { status, role: spec.role, slot: spec.session };
|
|
392
|
+
return surface === 'pane'
|
|
393
|
+
? launchPane(board, task, runId, lane, fleet, { input: prompt, slot: spec.session, meta })
|
|
394
|
+
: launchHeadless(board, task, runId, lane, settings, { prompt, skills: spec.skills ?? [], meta });
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/** Start one card now by a person's word (a triage card's session, a manager's start): the workflow's start event. */
|
|
398
|
+
export async function startCard(board, id, { roles = ['manager'], lanes, settings } = {}) {
|
|
399
|
+
const entry = await loadWorkflow(board.root);
|
|
400
|
+
const fleet = observeFleet();
|
|
401
|
+
const moved = tx(board, (db) => fire(db, board, entry, id, entry.workflow.dispatch.start, { roles, payload: { lock: claimer() } }));
|
|
402
|
+
const task = getTaskNow(board, id);
|
|
403
|
+
const spec = entry.workflow.statuses[moved.to]?.run;
|
|
404
|
+
const profile = spec ? tx(board, (db) => evaluate(entry.workflow.roles?.[spec.role] ?? 'card.assignee', scopeOf(db, board, task, { params: entry.params }))) : null;
|
|
405
|
+
const lane = profile ? lanes[profile] : null;
|
|
406
|
+
if (spec && !lane) throw new Error(`${id} is ${moved.to}, run by ${spec.role} (${profile ?? 'nobody'}), which is no lane of this home`);
|
|
407
|
+
return lane ? launchRun(board, entry, id, lane, fleet, settings) : null;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* The heartbeat, observed: a running pane session whose last message is a whole tick old is sent `session_idle` with
|
|
412
|
+
* how many times it has been asked this run and whether it worked since the last ask; the workflow decides the rest.
|
|
413
|
+
*/
|
|
414
|
+
function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
415
|
+
if (!fleet.ok) return;
|
|
416
|
+
for (const { id } of board.db.prepare("SELECT id FROM tasks WHERE status = 'running'").all()) {
|
|
417
|
+
tx(board, (db) => {
|
|
418
|
+
const task = getTask(db, id);
|
|
419
|
+
const status = statusOf(db, task);
|
|
420
|
+
if (!(entry.workflow.statuses[status]?.on?.session_idle || entry.workflow.on?.session_idle)) return;
|
|
421
|
+
const runRow = currentRun(db, id);
|
|
422
|
+
const session = runRow?.metadata?.supercode;
|
|
423
|
+
if (!session?.session_id || session.surface !== 'pane') return;
|
|
424
|
+
const live = fleet.sessions.find((x) => x.address?.endsWith(`:${session.session_id}`));
|
|
425
|
+
// a supercode that reports no last message time gives no reading, and no reading gives no verdict
|
|
426
|
+
const lastAt = Math.floor(Number(live?.last_message_at_ms) / 1000);
|
|
427
|
+
if (!(lastAt > 0)) return;
|
|
428
|
+
if (now() - lastAt < settings.interval) return;
|
|
429
|
+
if (session.goal_nudged_at && now() - session.goal_nudged_at < settings.interval * 2) return;
|
|
430
|
+
const worked = Boolean(session.goal_nudged_at) && lastAt >= session.goal_nudged_at + settings.interval;
|
|
431
|
+
try {
|
|
432
|
+
const r = fire(db, board, entry, id, 'session_idle', { roles: ['platform'], payload: { turns: Number(session.goal_turns ?? 0), worked }, extra: { where: whereOf(board, task) } });
|
|
433
|
+
effects.push(...r.effects);
|
|
434
|
+
const after = currentRun(db, id);
|
|
435
|
+
if (after?.id === runRow.id) setRunSession(db, runRow.id, { ...(after.metadata?.supercode ?? session), goal_nudged_at: now() });
|
|
436
|
+
report.notified.push([id, 'session_idle', live.address]);
|
|
437
|
+
} catch (error) { report.failed.push([id, `session_idle: ${String(error.message).slice(-200)}`]); }
|
|
438
|
+
});
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/** A session effect a committed move asked for: a message to the card's current actor, or closing a slot's pane. */
|
|
443
|
+
export async function sessionEffect(board, e) {
|
|
444
|
+
if (e.kind === 'send_to_actor') {
|
|
445
|
+
const run = tx(board, (db) => currentRun(db, e.id));
|
|
446
|
+
const session = run?.metadata?.supercode;
|
|
447
|
+
const address = session?.address ?? (session?.session_id && session.machine ? `sc:${session.machine}:claude-code:${session.session_id}` : null);
|
|
448
|
+
if (!address) return;
|
|
449
|
+
await send(boardAddress(board.label), board.label, address, e.text, `board-actor-${e.id}-${now()}`);
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
if (e.kind === 'close_session') {
|
|
453
|
+
const rows = tx(board, (db) => db.prepare('SELECT id, metadata FROM task_runs WHERE task_id = ? ORDER BY id DESC').all(e.id));
|
|
454
|
+
for (const row of rows) {
|
|
455
|
+
const session = json(row.metadata, {})?.supercode;
|
|
456
|
+
if ((session?.slot ?? 'worker') !== e.slot || !session?.pane || session.closed_at) continue;
|
|
457
|
+
stopSession(session);
|
|
458
|
+
tx(board, (db) => setRunSession(db, row.id, { ...session, closed_at: now() }));
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
254
464
|
// ------------------------------------------------------------------ start
|
|
255
465
|
|
|
256
466
|
const running = (db, sql, ...args) => Number(db.prepare(`SELECT COUNT(*) AS n FROM tasks t LEFT JOIN supercode_cards c ON c.task_id = t.id WHERE t.status = 'running' ${sql}`).get(...args).n);
|
|
@@ -269,15 +479,11 @@ function machineLoad(db, machine, fleet) {
|
|
|
269
479
|
}
|
|
270
480
|
|
|
271
481
|
/** Why a card cannot start now, or null. */
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
const cap = lane.capacity ?? settings.perProfile;
|
|
275
|
-
if (cap && running(db, 'AND t.assignee = ?', lane.name) >= cap) return `${lane.name} is at capacity (${cap}); the card starts when one of its sessions ends`;
|
|
482
|
+
/** What the launcher itself needs to start this card (the caps are the workflow's `dispatch.limits`). */
|
|
483
|
+
function launcherCheck(db, task, lane, fleet) {
|
|
276
484
|
if (lane.surface !== 'pane') return null;
|
|
277
485
|
if (!task.machine) return 'the card names no machine; a pane session runs on the machine its card names';
|
|
278
486
|
const machine = normalMachine(task.machine);
|
|
279
|
-
const mcap = settings.machines[machine]?.capacity;
|
|
280
|
-
if (mcap && machineLoad(db, machine, fleet) >= mcap) return `machine ${machine} holds ${mcap} of the board's sessions (its capacity); the card starts when one is closed`;
|
|
281
487
|
if (!fleet.ok) return `supercode could not read the fleet: ${fleet.error}`;
|
|
282
488
|
if (!fleet.online.has(machine)) return `machine ${machine} is offline; the card starts when it is back`;
|
|
283
489
|
// a card on this machine gets its workspace made here (scratch, dir or worktree, as Hermes makes it); another
|
|
@@ -293,17 +499,6 @@ function guard(db, task, lane, fleet, settings, budget = null) {
|
|
|
293
499
|
const BLOCKER = /\b(quota|rate[\s_-]?limit|429|403|auth\w*|unauthorized|forbidden|billing|subscription|access[\s_]denied|permission[\s_]denied|invalid[\s_]api[\s_]key)\b/i;
|
|
294
500
|
const PR_URL = /https?:\/\/github\.com\/[^/\s]+\/[^/\s]+\/pull\/\d+/i;
|
|
295
501
|
|
|
296
|
-
function respawnGuard(db, task, settings, lane) {
|
|
297
|
-
const t = now();
|
|
298
|
-
const last = db.prepare('SELECT outcome, ended_at FROM task_runs WHERE task_id = ? AND ended_at IS NOT NULL ORDER BY ended_at DESC LIMIT 1').get(task.id);
|
|
299
|
-
if (last?.outcome === 'rate_limited') return settings.rateLimitCooldown > 0 && t - Number(last.ended_at) < settings.rateLimitCooldown ? 'rate_limit_cooldown' : null;
|
|
300
|
-
if (task.last_failure_error && BLOCKER.test(task.last_failure_error)) return 'blocker_auth';
|
|
301
|
-
if (lane === 'review') return null;
|
|
302
|
-
const done = db.prepare("SELECT ended_at FROM task_runs WHERE task_id = ? AND outcome = 'completed' AND ended_at >= ? ORDER BY ended_at DESC LIMIT 1").get(task.id, t - 3600);
|
|
303
|
-
if (done && !db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND created_at >= ? AND kind IN ('status', 'promoted', 'unblocked', 'reclaimed') LIMIT 1").get(task.id, done.ended_at)) return 'recent_success';
|
|
304
|
-
for (const c of db.prepare('SELECT body FROM task_comments WHERE task_id = ? AND created_at >= ?').all(task.id, t - 86400)) if (PR_URL.test(c.body ?? '')) return 'active_pr';
|
|
305
|
-
return null;
|
|
306
|
-
}
|
|
307
502
|
|
|
308
503
|
/** This machine's memory pressure as Hermes reads it (Linux's MemAvailable; `unknown` elsewhere, which restricts nothing). */
|
|
309
504
|
function memoryPressure() {
|
|
@@ -331,135 +526,15 @@ export function paused(root) {
|
|
|
331
526
|
return [join(root, 'ESTOP'), join(process.env.HOME ?? '', '.hermes', 'ESTOP')].find((p) => existsSync(p)) ?? null;
|
|
332
527
|
}
|
|
333
528
|
|
|
334
|
-
/**
|
|
335
|
-
* The tick's spawn budget, as Hermes counts it: `kanban.max_spawn` against this board's running cards, and
|
|
336
|
-
* `kanban.max_in_progress` against every board of the home (workers share one machine's memory).
|
|
337
|
-
*/
|
|
338
|
-
function spawnBudget(board, settings, hostRunning) {
|
|
339
|
-
const here = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks WHERE status = 'running'").get().n);
|
|
340
|
-
const limits = [];
|
|
341
|
-
if (settings.maxSpawn) limits.push([settings.maxSpawn - here, `the board runs ${settings.maxSpawn} cards at most (kanban.max_spawn)`]);
|
|
342
|
-
if (settings.maxInProgress) limits.push([settings.maxInProgress - hostRunning, `the home runs ${settings.maxInProgress} cards at most across its boards (kanban.max_in_progress)`]);
|
|
343
|
-
if (!limits.length) return null;
|
|
344
|
-
const [left, why] = limits.sort((a, b) => a[0] - b[0])[0];
|
|
345
|
-
return { left, why };
|
|
346
|
-
}
|
|
347
529
|
|
|
348
|
-
function start(board, fleet, lanes, settings, report, { only = null, hostRunning = 0 } = {}) {
|
|
349
|
-
const stop = paused(board.root);
|
|
350
|
-
if (stop) { report.paused = stop; return; }
|
|
351
|
-
const pressure = memoryPressure();
|
|
352
|
-
let localStarted = 0;
|
|
353
|
-
const order = 'ORDER BY priority DESC, created_at ASC';
|
|
354
|
-
const ready = board.db.prepare(`SELECT id, status, assignee FROM tasks WHERE status = 'ready' AND claim_lock IS NULL ${order}`).all();
|
|
355
|
-
const review = settings.reviewDispatch ? board.db.prepare(`SELECT id, status, assignee FROM tasks WHERE status = 'review' AND claim_lock IS NULL ${order}`).all() : [];
|
|
356
|
-
const budget = spawnBudget(board, settings, hostRunning);
|
|
357
|
-
// Hermes's review reservation: while review work waits, one slot of the budget is held back from the ready lane
|
|
358
|
-
const reviewWaits = review.some((r) => lanes[r.assignee || settings.defaultAssignee]);
|
|
359
|
-
const readyBudget = budget && budget.left > 0 && reviewWaits ? { left: budget.left - 1, why: `${budget.why}; one slot is held for review` } : budget;
|
|
360
|
-
for (const row of [...ready, ...review]) {
|
|
361
|
-
const lanesBudget = row.status === 'review' ? budget : readyBudget;
|
|
362
|
-
if (only && row.id !== only) continue;
|
|
363
|
-
if (board.db.prepare('SELECT subtask_of FROM supercode_cards WHERE task_id = ?').get(row.id)?.subtask_of) continue; // worked by its arc's session
|
|
364
|
-
const assignee = row.assignee || settings.defaultAssignee;
|
|
365
|
-
if (!assignee) { report.unassigned.push(row.id); continue; }
|
|
366
|
-
const lane = lanes[assignee];
|
|
367
|
-
if (!lane) { report.nonspawnable.push(row.id); continue; }
|
|
368
|
-
const claimed = tx(board, (db) => {
|
|
369
|
-
const task = getTask(db, row.id);
|
|
370
|
-
if (!task || task.status !== row.status || task.claim_lock) return null;
|
|
371
|
-
let why = guard(db, task, lane, fleet, settings, lanesBudget);
|
|
372
|
-
// this machine's memory bounds what starts here: every one-shot worker, and a pane session on this machine
|
|
373
|
-
const here = lane.surface !== 'pane' || normalMachine(task.machine ?? '') === localMachine();
|
|
374
|
-
if (!why && here && (pressure === 'critical' || (pressure === 'elevated' && localStarted >= 1))) why = `this machine's memory pressure is ${pressure}; the card starts when it eases`;
|
|
375
|
-
// Hermes's respawn guard, on every lane (a pane card whose session is still live is kept, never started twice)
|
|
376
|
-
if (!why) why = respawnGuard(db, task, settings, row.status);
|
|
377
|
-
if (why) { noteGuard(db, row.id, why); report.queued.push([row.id, why]); return null; }
|
|
378
|
-
if (!row.assignee) {
|
|
379
|
-
db.prepare('UPDATE tasks SET assignee = ? WHERE id = ?').run(assignee, row.id);
|
|
380
|
-
event(db, row.id, 'assigned', { assignee, source: 'kanban.default_assignee' });
|
|
381
|
-
}
|
|
382
|
-
const runId = claimIn(db, row.id, { ttl: settings.ttl, expectStatus: row.status });
|
|
383
|
-
return runId ? { task: getTask(db, row.id), runId } : null;
|
|
384
|
-
});
|
|
385
|
-
if (!claimed) continue;
|
|
386
|
-
try {
|
|
387
|
-
const session = lane.surface === 'pane'
|
|
388
|
-
? launchPane(board, claimed.task, claimed.runId, lane, fleet, { input: row.status === 'review' ? reviewAsk(board, claimed.task) : null })
|
|
389
|
-
: launchHeadless(board, claimed.task, claimed.runId, lane, settings);
|
|
390
|
-
if (lane.surface !== 'pane' || normalMachine(claimed.task.machine ?? '') === localMachine()) localStarted += 1;
|
|
391
|
-
if (budget) budget.left -= 1;
|
|
392
|
-
if (readyBudget && readyBudget !== budget) readyBudget.left -= 1;
|
|
393
|
-
report.started.push([row.id, session.address ?? `pid ${session.pid}`, session.mode]);
|
|
394
|
-
} catch (error) {
|
|
395
|
-
tx(board, (db) => {
|
|
396
|
-
release(db, row.id, { outcome: 'spawn_failed', error: String(error.message).slice(-500), kind: 'spawn_failed', payload: { error: String(error.message).slice(-300) } });
|
|
397
|
-
if (recordFailure(db, row.id, error.message, { failureLimit: settings.failureLimit, outcome: 'spawn_failed' })) report.auto_blocked.push(row.id);
|
|
398
|
-
});
|
|
399
|
-
report.failed.push([row.id, String(error.message).slice(-300)]);
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
530
|
|
|
404
|
-
/** What a session started from the review lane is asked to do: verify the handoff and give the verdict. */
|
|
405
|
-
function reviewAsk(board, task) {
|
|
406
|
-
const local = normalMachine(task.machine) === localMachine();
|
|
407
|
-
const where = local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(board.label)}`;
|
|
408
|
-
return `You are reviewing card ${task.id} on the ${board.label} board: its implementer asked for review. Verify the handoff against the card's acceptance, `
|
|
409
|
-
+ `then give the verdict: \`supercode workflow complete ${task.id} --summary "<what you verified>" ${where}\` approves it, and `
|
|
410
|
-
+ `\`supercode workflow request-changes ${task.id} "<what must change>" ${where}\` sends it back.\n\n${tx(board, (db) => cardContext(db, task.id))}`;
|
|
411
|
-
}
|
|
412
531
|
|
|
413
|
-
/** What a session started on a triage card is asked to do: write its spec, or split it into cards it waits on. */
|
|
414
|
-
function triageAsk(board, task, label, purpose) {
|
|
415
|
-
const local = normalMachine(task.machine) === localMachine();
|
|
416
|
-
const where = local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(label)}`;
|
|
417
|
-
const context = tx(board, (db) => cardContext(db, task.id));
|
|
418
|
-
return purpose === 'decompose'
|
|
419
|
-
? `Card ${task.id} on the ${label} board is in triage: a rough idea to split. Read it, then file the cards it breaks into and make it wait on them: `
|
|
420
|
-
+ `\`supercode workflow decompose ${task.id} --child "<title>[@<profile>]" --child "..." ${where}\` (each child gets this card's machine and directory). `
|
|
421
|
-
+ `The card itself stays and wakes when they are done.\n\n${context}`
|
|
422
|
-
: `Card ${task.id} on the ${label} board is in triage: a rough idea. Write its spec (the goal, the approach, what will show it is done), then file it: `
|
|
423
|
-
+ `\`supercode workflow specify ${task.id} --title "<a tightened title>" --body "<the spec>" ${where}\`. The card then waits for its parents and starts.\n\n${context}`;
|
|
424
|
-
}
|
|
425
532
|
|
|
426
|
-
/**
|
|
427
|
-
* Start a session on a triage card to specify or decompose it (where Hermes asks an auxiliary model, supercode asks a
|
|
428
|
-
* session in the card's lane, which then keeps the card's memory when the card starts). Answers the session.
|
|
429
|
-
*/
|
|
430
|
-
export function startTriageSession(board, id, lanes, settings, purpose) {
|
|
431
|
-
const fleet = observeFleet();
|
|
432
|
-
const task = tx(board, (db) => getTask(db, id));
|
|
433
|
-
if (!task) throw new Error(`no card ${id}`);
|
|
434
|
-
if (task.status !== 'triage') throw new Error(`${id} is ${task.status}; only a triage card is ${purpose === 'decompose' ? 'decomposed' : 'specified'}`);
|
|
435
|
-
const lane = lanes[task.assignee || settings.defaultAssignee];
|
|
436
|
-
if (!lane || lane.surface !== 'pane') throw new Error(`${id} has no pane lane to start a session in; give its spec directly (${purpose === 'decompose' ? '--child' : '--title/--body'})`);
|
|
437
|
-
const runId = tx(board, (db) => {
|
|
438
|
-
const why = guard(db, task, lane, fleet, settings);
|
|
439
|
-
if (why) throw new Error(`${id} cannot start a session now: ${why}`);
|
|
440
|
-
return claimIn(db, id, { ttl: settings.ttl, expectStatus: 'triage' });
|
|
441
|
-
});
|
|
442
|
-
if (!runId) throw new Error(`${id} could not be claimed`);
|
|
443
|
-
return launchPane(board, getTaskNow(board, id), runId, lane, fleet, { input: triageAsk(board, task, board.label, purpose) });
|
|
444
|
-
}
|
|
445
533
|
|
|
446
534
|
const getTaskNow = (board, id) => tx(board, (db) => getTask(db, id));
|
|
447
535
|
|
|
448
|
-
function opening(board, task, label) {
|
|
449
|
-
const context = tx(board, (db) => cardContext(db, task.id));
|
|
450
|
-
const skills = task.skills?.length ? `\nSkills pinned to this card: ${task.skills.join(', ')}.` : '';
|
|
451
|
-
const at = `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}`;
|
|
452
|
-
// the board's own door is on the machine that keeps it; a session elsewhere writes to its card through its manager
|
|
453
|
-
const local = normalMachine(task.machine) === localMachine();
|
|
454
|
-
const where = local ? at : `--at ${boardAddress(label)}`;
|
|
455
|
-
const door = `\nYour card is yours to keep current: \`supercode workflow comment ${task.id} "<text>" ${where}\`, \`block ${task.id} "<why>" --kind <needs_input|capability|transient|dependency> ${where}\` `
|
|
456
|
-
+ `and \`request-review ${task.id} --summary "<what>" ${where}\` update it; \`create "<title>" --subtask-of ${task.id} ${where}\` files a subtask, and \`complete <subtask> ${where}\` marks one done`
|
|
457
|
-
+ `${local ? '' : ' (each is filed with the board and answered by a board notice)'}. Its reviewers, not you, mark the card itself done.`;
|
|
458
|
-
return `You are the session working card ${task.id} on the ${label} board. The board started you; updates to the card `
|
|
459
|
-
+ `(comments, reassignment, blocks, completion) and to the cards it waits on reach you as board notices.${door}${skills}\n\n${context}`;
|
|
460
|
-
}
|
|
461
536
|
|
|
462
|
-
function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
|
|
537
|
+
function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'worker', meta = {} } = {}) {
|
|
463
538
|
const machine = normalMachine(task.machine);
|
|
464
539
|
if (machine === localMachine() && !(task.workspace_kind === 'dir' && task.workspace_path && existsSync(task.workspace_path))) {
|
|
465
540
|
const made = resolveLocalWorkspace(board, task);
|
|
@@ -469,7 +544,8 @@ function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
|
|
|
469
544
|
const model = task.model_override ?? lane.model;
|
|
470
545
|
const tail = model ? ['--model', model] : [];
|
|
471
546
|
const base = ['open', '--new', 'claude', '--detach', '--on', task.machine, '--cwd', task.workspace_path];
|
|
472
|
-
|
|
547
|
+
// the slot's last session: kept when live, resumed when lost, never another slot's (a reviewer's is its own)
|
|
548
|
+
const prev = tx(board, (db) => lastSession(db, task.id, slot));
|
|
473
549
|
let session = null;
|
|
474
550
|
let replaced = null;
|
|
475
551
|
if (prev && prev.machine === machine) {
|
|
@@ -477,6 +553,8 @@ function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
|
|
|
477
553
|
// the session is kept; the heartbeat's count belongs to the run that ended, so the new run starts it again
|
|
478
554
|
const { missing_since: _m, goal_turns: _t, goal_nudged_at: _n, ...kept } = prev;
|
|
479
555
|
session = { ...kept, mode: 'adopt', launched_at: now(), seen_at: now() };
|
|
556
|
+
// a kept session is told what this run is for (its card came back to it, with what happened since)
|
|
557
|
+
if (input) deliverNow(board, prev.address ?? `sc:${prev.machine}:claude-code:${prev.session_id}`, input, `board-run-${task.id}-r${runId}`);
|
|
480
558
|
} else if (prev.resumable !== false) {
|
|
481
559
|
const r = sc([...base, '--key', `board-${task.id}-r${runId}-resume`, '--input',
|
|
482
560
|
`The board resumed this session after it was lost; you are still on card ${task.id}: ${task.title}. Carry on.`, '--', '--resume', prev.session_id, ...tail], { timeout: 180_000 });
|
|
@@ -489,12 +567,13 @@ function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
|
|
|
489
567
|
} else if (prev) replaced = { session_id: prev.session_id, why: `the card moved from ${prev.machine} to ${machine}` };
|
|
490
568
|
if (!session) {
|
|
491
569
|
const sid = randomUUID();
|
|
492
|
-
const r = sc([...base, '--key', `board-${task.id}-r${runId}-${sid.slice(0, 8)}`, '--input', input ??
|
|
570
|
+
const r = sc([...base, '--key', `board-${task.id}-r${runId}-${sid.slice(0, 8)}`, '--input', input ?? `Card ${task.id}: ${task.title}`, '--', '--session-id', sid, ...tail], { timeout: 180_000 });
|
|
493
571
|
if (r.code !== 0) throw new Error(`supercode open --on ${task.machine}: ${(r.stderr || r.stdout).trim().slice(-500)}`);
|
|
494
572
|
session = { surface: 'pane', session_id: sid, machine, pane: lastJson(r.stdout)?.pane ?? null, mode: 'fresh', launched_at: now(), ...(replaced ? { replaces: replaced.session_id } : {}) };
|
|
495
573
|
}
|
|
496
574
|
session.harness = 'claude-code';
|
|
497
575
|
session.address = `sc:${machine}:claude-code:${session.session_id}`;
|
|
576
|
+
Object.assign(session, meta);
|
|
498
577
|
const kind = { adopt: 'session_adopted', resume: 'session_resumed', fresh: replaced ? 'session_replaced' : 'session_started' }[session.mode];
|
|
499
578
|
tx(board, (db) => {
|
|
500
579
|
setRunSession(db, runId, session);
|
|
@@ -520,10 +599,10 @@ function paneSurvives(machine, pane) {
|
|
|
520
599
|
}
|
|
521
600
|
|
|
522
601
|
/** Hermes's own worker argv and task environment, run on this machine; watched by its pid. */
|
|
523
|
-
function launchHeadless(board, task, runId, lane, settings) {
|
|
602
|
+
function launchHeadless(board, task, runId, lane, settings, { prompt = null, skills: runSkills = [], meta = {} } = {}) {
|
|
524
603
|
const workspace = resolveLocalWorkspace(board, task);
|
|
525
|
-
|
|
526
|
-
const skills =
|
|
604
|
+
// the card's own skills and the ones its status's run names (a reviewer's, for one)
|
|
605
|
+
const skills = [...new Set([...task.skills, ...runSkills])];
|
|
527
606
|
const args = ['-p', lane.name, '--cli', '--accept-hooks', ...skills.flatMap((s) => ['--skills', s]),
|
|
528
607
|
...(task.model_override ? ['-m', task.model_override, ...(task.provider_override ? ['--provider', task.provider_override] : [])] : []),
|
|
529
608
|
...(task.reasoning_effort ? ['--reasoning', task.reasoning_effort] : []), 'chat', '-q', `work kanban task ${task.id}`];
|
|
@@ -541,12 +620,12 @@ function launchHeadless(board, task, runId, lane, settings) {
|
|
|
541
620
|
...(task.goal_mode ? { HERMES_KANBAN_GOAL_MODE: '1', ...(task.goal_max_turns ? { HERMES_KANBAN_GOAL_MAX_TURNS: String(task.goal_max_turns) } : {}) } : {}),
|
|
542
621
|
...process.env, HERMES_HOME: lane.dir, HERMES_PROFILE: lane.name, HERMES_KANBAN_TASK: task.id, HERMES_KANBAN_WORKSPACE: workspace,
|
|
543
622
|
HERMES_KANBAN_RUN_ID: String(runId), HERMES_KANBAN_DB: board.path, HERMES_KANBAN_BOARD: board.slug, HERMES_SESSION_SOURCE: 'kanban',
|
|
544
|
-
HERMES_KANBAN_CLAIM_LOCK:
|
|
623
|
+
HERMES_KANBAN_CLAIM_LOCK: claimer(), ...(prompt ? { SUPERCODE_BOARD_PROMPT: prompt } : {}), ...(task.tenant ? { HERMES_TENANT: task.tenant } : {}),
|
|
545
624
|
...(task.branch_name ? { HERMES_KANBAN_BRANCH: task.branch_name } : {}),
|
|
546
625
|
},
|
|
547
626
|
});
|
|
548
627
|
child.unref();
|
|
549
|
-
const session = { surface: 'headless', pid: child.pid, machine: localMachine(), mode: 'fresh', launched_at: now(), exit_file: exitFile };
|
|
628
|
+
const session = { surface: 'headless', pid: child.pid, machine: localMachine(), mode: 'fresh', launched_at: now(), exit_file: exitFile, ...meta };
|
|
550
629
|
tx(board, (db) => {
|
|
551
630
|
db.prepare('UPDATE tasks SET worker_pid = ? WHERE id = ?').run(child.pid, task.id);
|
|
552
631
|
db.prepare('UPDATE task_runs SET worker_pid = ? WHERE id = ?').run(child.pid, runId);
|
|
@@ -620,20 +699,18 @@ export function adoptSession(board, id, who, { workspace = null, ttl, assignee =
|
|
|
620
699
|
if (task.machine && normalMachine(task.machine) !== host) throw new Error(`${id} runs on ${task.machine}, and ${found.address} is on ${host}; move the card first`);
|
|
621
700
|
const was = lastSession(db, id);
|
|
622
701
|
if (was && was.session_id !== sessionId && sessionAlive(fleet, was)) throw new Error(`${id} already has a live session, ${was.address}`);
|
|
623
|
-
if (!task.machine) db.prepare('INSERT
|
|
702
|
+
if (!task.machine) db.prepare('INSERT INTO supercode_cards (task_id, machine) VALUES (?, ?) ON CONFLICT(task_id) DO UPDATE SET machine = excluded.machine').run(id, host);
|
|
624
703
|
// the lane that resumes it if it is lost, set in the same transaction (assigning first would let a tick start another)
|
|
625
704
|
if (assignee && assignee !== task.assignee) { db.prepare('UPDATE tasks SET assignee = ? WHERE id = ?').run(assignee, id); event(db, id, 'assigned', { assignee }); }
|
|
626
705
|
if (dir && !(task.workspace_kind === 'dir' && task.workspace_path)) {
|
|
627
706
|
db.prepare("UPDATE tasks SET workspace_kind = 'dir', workspace_path = ? WHERE id = ?").run(dir.replace(/^dir:/, ''), id);
|
|
628
707
|
}
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
}
|
|
633
|
-
const runId = claimIn(db, id, { ttl, expectStatus: 'ready' });
|
|
708
|
+
// the workflow's adopt event: the card is claimed for its implementer, whose session this now is
|
|
709
|
+
fire(db, board, workflowFor(board.root), id, 'adopt', { roles: ['manager'], payload: { lock: claimer() } });
|
|
710
|
+
const runId = getTask(db, id).current_run_id;
|
|
634
711
|
if (!runId) throw new Error(`${id} waits on parents that are not done`);
|
|
635
712
|
const session = { surface: 'pane', session_id: sessionId, harness, machine: host, pane: found.pane ?? null, address: found.address, mode: 'adopt', launched_at: now(), seen_at: now() };
|
|
636
|
-
setRunSession(db, runId, session);
|
|
713
|
+
setRunSession(db, runId, { ...(currentRun(db, id)?.metadata?.supercode ?? {}), ...session, slot: 'worker' });
|
|
637
714
|
event(db, id, 'session_adopted', { session_id: sessionId, machine: host, pane: session.pane, address: found.address, mode: 'adopt' }, runId);
|
|
638
715
|
subscribeIn(db, id, found.address, 'worker');
|
|
639
716
|
for (const p of db.prepare('SELECT parent_id FROM task_links WHERE child_id = ?').all(id)) subscribeIn(db, p.parent_id, found.address, 'dependency');
|
|
@@ -677,14 +754,23 @@ function notice(kind, task, p, role) {
|
|
|
677
754
|
}
|
|
678
755
|
}
|
|
679
756
|
|
|
680
|
-
|
|
757
|
+
// a notice sent within a transaction's caller that cannot await (the launcher): the same door, synchronously
|
|
758
|
+
function deliverNow(board, to, body, key) {
|
|
759
|
+
const request = { op: 'send', from: boardAddress(board.label), from_name: `${board.label} board`, to, queue: false, id: key,
|
|
760
|
+
body: `[Board notice from the ${board.label} board] ${body}\n\n(An automated notice from the board's dispatcher; replies to this address go nowhere.)` };
|
|
761
|
+
sc(['message', 'remote-deliver'], { input: JSON.stringify(request), timeout: 120_000 });
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
async function send(from, label, to, body, key, { queue = false } = {}) {
|
|
681
765
|
const request = {
|
|
682
766
|
op: 'send', from, from_name: `${label} board`, to, queue, id: key,
|
|
683
767
|
body: `[Board notice from the ${label} board] ${body}\n\n(An automated notice from the board's dispatcher; replies to this address go nowhere.)`,
|
|
684
768
|
};
|
|
685
|
-
|
|
769
|
+
// a delivery to a busy session can take a minute on a loaded machine; what it answered is the verdict (a process
|
|
770
|
+
// killed after it answered `code: 0` delivered)
|
|
771
|
+
const r = await scAsync(['message', 'remote-deliver'], { input: JSON.stringify(request), timeout: 180_000 });
|
|
686
772
|
const answer = lastJson(r.stdout) ?? {};
|
|
687
|
-
if (
|
|
773
|
+
if (answer.code !== 0) throw new Error((answer.text || r.stderr || `no answer (exit ${r.code})`).trim().slice(-300));
|
|
688
774
|
}
|
|
689
775
|
|
|
690
776
|
/** The board's own address: where its notices come from and where card updates are mailed. */
|
|
@@ -693,20 +779,20 @@ export const boardAddress = (label) => `sc:${localMachine()}:board:${label.repla
|
|
|
693
779
|
/** Consecutive failed sends after which a subscription is dropped (Hermes's notifier drops a dead chat after 12). */
|
|
694
780
|
const MAX_SEND_FAILURES = 12;
|
|
695
781
|
|
|
696
|
-
function notify(board, fleet, report) {
|
|
782
|
+
async function notify(board, fleet, report) {
|
|
697
783
|
const from = boardAddress(board.label);
|
|
698
|
-
|
|
699
|
-
|
|
784
|
+
// each subscription's notices go in order; different subscribers are told side by side
|
|
785
|
+
await pool(subscriptions(board.db).filter((sub) => sub.platform === 'supercode'), SEND_CONCURRENCY, async (sub) => {
|
|
700
786
|
const role = ['worker', 'dependency'].includes(sub.delivery_metadata?.role) ? sub.delivery_metadata.role : 'creator';
|
|
701
787
|
// a session that worked or watched a card and is gone hears nothing more; a creator is retried until it is back
|
|
702
788
|
if (role !== 'creator' && fleet.ok && !fleet.alive.has(sub.chat_id)) {
|
|
703
789
|
tx(board, (db) => db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ?").run(sub.task_id, sub.chat_id));
|
|
704
|
-
|
|
790
|
+
return;
|
|
705
791
|
}
|
|
706
792
|
const kinds = MAIL_KINDS[role];
|
|
707
793
|
const events = board.db.prepare(`SELECT id, kind, payload, created_at FROM task_events WHERE task_id = ? AND id > ? AND kind IN (${kinds.map(() => '?').join(', ')}) ORDER BY id`)
|
|
708
794
|
.all(sub.task_id, sub.last_event_id, ...kinds);
|
|
709
|
-
if (!events.length)
|
|
795
|
+
if (!events.length) return;
|
|
710
796
|
const task = tx(board, (db) => getTask(db, sub.task_id));
|
|
711
797
|
let cursor = Number(sub.last_event_id);
|
|
712
798
|
let failed = null;
|
|
@@ -721,7 +807,7 @@ function notify(board, fleet, report) {
|
|
|
721
807
|
} else text = notice(ev.kind, task, p, role);
|
|
722
808
|
if (text) {
|
|
723
809
|
// Hermes's delivery modes: `notify` is passive (read at the session's next turn), `wake`/`notify+wake` start its turn
|
|
724
|
-
send(from, board.label, sub.chat_id, text, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: sub.delivery_mode === 'notify' });
|
|
810
|
+
await send(from, board.label, sub.chat_id, text, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: sub.delivery_mode === 'notify' });
|
|
725
811
|
report.notified.push([sub.task_id, ev.kind, sub.chat_id]);
|
|
726
812
|
}
|
|
727
813
|
cursor = ev.id;
|
|
@@ -739,7 +825,7 @@ function notify(board, fleet, report) {
|
|
|
739
825
|
if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures }), sub.task_id, sub.chat_id);
|
|
740
826
|
if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(sub.task_id, sub.chat_id);
|
|
741
827
|
});
|
|
742
|
-
}
|
|
828
|
+
});
|
|
743
829
|
}
|
|
744
830
|
|
|
745
831
|
// Hermes's terminal kinds for a chat subscriber, in its order of silence: archived and unblocked move the cursor and say nothing
|
|
@@ -905,83 +991,13 @@ function readMailbox(board, address, report) {
|
|
|
905
991
|
answer = `${clean.slice(0, 2).join(' ')}: ${(r.status === 0 ? r.stdout : r.stderr || r.stdout).trim().slice(-600) || `exit ${r.status}`}`;
|
|
906
992
|
}
|
|
907
993
|
report.mailed.push([envelope.from, argv?.[0] ?? '?']);
|
|
908
|
-
|
|
994
|
+
report.answers.push(send(address, board.label, envelope.from, answer, `board-answer-${envelope.id}`).catch((error) => { report.undelivered.push(['mail', envelope.from, String(error.message).slice(-200)]); }));
|
|
909
995
|
}
|
|
910
996
|
}
|
|
911
997
|
|
|
912
998
|
// ------------------------------------------------------------------ goals and triage
|
|
913
999
|
|
|
914
|
-
/**
|
|
915
|
-
* The board's heartbeat for a pane session: a running card's session never sits idle without deciding. Each tick, a
|
|
916
|
-
* session that is idle while its card is still running is asked to decide: `request-review` when the work is done,
|
|
917
|
-
* `block` with its kind when it cannot go on (waiting on a person, a capability, another card), or keep working. A
|
|
918
|
-
* goal card (Hermes's goal loop) is told to keep going. Idle is judged by the session's last message time (supercode
|
|
919
|
-
* reads it from the transcript), never by its status label: a session is asked once its last message is a whole tick
|
|
920
|
-
* old; one whose messages ran on a whole tick past the last ask answered it by working, and its count starts again
|
|
921
|
-
* (a reply to the ask alone is not work). After the turn budget (`goal_max_turns`, else
|
|
922
|
-
* `kanban.goal_max_turns`) of asks in a row with no work and no decision (a goal card: of turns), the card is blocked `needs_input` and its creator is told.
|
|
923
|
-
* Whether the work is done is the session's call through its card, not an auxiliary judge's.
|
|
924
|
-
*/
|
|
925
|
-
function pursueGoals(board, fleet, settings, report) {
|
|
926
|
-
if (!fleet.ok) return;
|
|
927
|
-
const cards = board.db.prepare("SELECT id FROM tasks WHERE status = 'running'").all();
|
|
928
|
-
for (const { id } of cards) {
|
|
929
|
-
const nudge = tx(board, (db) => {
|
|
930
|
-
const task = getTask(db, id);
|
|
931
|
-
const runRow = currentRun(db, id);
|
|
932
|
-
const session = runRow?.metadata?.supercode;
|
|
933
|
-
if (!session?.session_id || session.surface !== 'pane') return null;
|
|
934
|
-
const live = fleet.sessions.find((x) => x.address?.endsWith(`:${session.session_id}`));
|
|
935
|
-
// a supercode that reports no last message time gives no reading, and no reading gives no verdict
|
|
936
|
-
const lastAt = Math.floor(Number(live.last_message_at_ms) / 1000);
|
|
937
|
-
if (!(lastAt > 0)) return null;
|
|
938
|
-
// a message within the last tick is a session working or between turns, not idle
|
|
939
|
-
if (now() - lastAt < settings.interval) return null;
|
|
940
|
-
if (session.goal_nudged_at && now() - session.goal_nudged_at < settings.interval * 2) return null;
|
|
941
|
-
// messages that ran on a whole tick past the last ask answered it by working: the count starts again (a goal
|
|
942
|
-
// card keeps Hermes's budget of turns)
|
|
943
|
-
const worked = Boolean(session.goal_nudged_at) && lastAt >= session.goal_nudged_at + settings.interval;
|
|
944
|
-
const turns = worked && !task.goal_mode ? 0 : Number(session.goal_turns ?? 0);
|
|
945
|
-
const budget = task.goal_max_turns ?? settings.goalMaxTurns;
|
|
946
|
-
if (turns >= budget) {
|
|
947
|
-
blockIn(db, id, { reason: `its session went idle ${budget} times without deciding done or blocked; review it`, kind: 'needs_input' });
|
|
948
|
-
return null;
|
|
949
|
-
}
|
|
950
|
-
setRunSession(db, runRow.id, { ...session, goal_turns: turns + 1, goal_nudged_at: now() });
|
|
951
|
-
event(db, id, task.goal_mode ? 'goal_continued' : 'decision_asked', { turn: turns + 1, budget }, runRow.id);
|
|
952
|
-
const local = normalMachine(task.machine) === localMachine();
|
|
953
|
-
const where = local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(board.label)}`;
|
|
954
|
-
const decide = `When it is done, \`supercode workflow request-review ${id} --summary "<what>" ${where}\`; if you cannot go on, \`supercode workflow block ${id} "<why>" --kind <needs_input|capability|transient|dependency> ${where}\`.`;
|
|
955
|
-
const text = task.goal_mode
|
|
956
|
-
? `Card ${id} (${task.title}) is not done yet: keep working toward it (goal turn ${turns + 1} of ${budget}). ${decide}`
|
|
957
|
-
: `Card ${id} (${task.title}) is still running and you are idle: decide (${turns + 1} of ${budget}). If there is more to do, keep working. ${decide} `
|
|
958
|
-
+ `A session that stays idle without deciding has its card blocked for its creator after ${budget}.`;
|
|
959
|
-
return { address: live.address, text, kind: task.goal_mode ? 'goal_continued' : 'decision_asked' };
|
|
960
|
-
});
|
|
961
|
-
if (nudge) {
|
|
962
|
-
try { send(boardAddress(board.label), board.label, nudge.address, nudge.text, `board-goal-${id}-${now()}`); report.notified.push([id, nudge.kind, nudge.address]); }
|
|
963
|
-
catch (error) { report.undelivered.push([id, nudge.address, String(error.message).slice(-200)]); }
|
|
964
|
-
}
|
|
965
|
-
}
|
|
966
|
-
}
|
|
967
1000
|
|
|
968
|
-
/** Hermes's auto-decompose: triage cards with a pane lane get a session that splits them, a few per tick. */
|
|
969
|
-
function autoDecompose(board, lanes, settings, report) {
|
|
970
|
-
if (!settings.autoDecompose) return;
|
|
971
|
-
let started = 0;
|
|
972
|
-
for (const { id, assignee } of board.db.prepare("SELECT id, assignee FROM tasks WHERE status = 'triage' ORDER BY priority DESC, created_at ASC").all()) {
|
|
973
|
-
if (started >= settings.autoDecomposePerTick) break;
|
|
974
|
-
const lane = lanes[assignee || settings.defaultAssignee];
|
|
975
|
-
if (lane?.surface !== 'pane') continue;
|
|
976
|
-
try {
|
|
977
|
-
const session = startTriageSession(board, id, lanes, settings, 'decompose');
|
|
978
|
-
report.started.push([id, session.address, 'decompose']);
|
|
979
|
-
started += 1;
|
|
980
|
-
} catch (error) {
|
|
981
|
-
report.queued.push([id, String(error.message).slice(-300)]);
|
|
982
|
-
}
|
|
983
|
-
}
|
|
984
|
-
}
|
|
985
1001
|
|
|
986
1002
|
// ------------------------------------------------------------------ the tick
|
|
987
1003
|
|
|
@@ -991,6 +1007,32 @@ export function boardLabel(root, slug) {
|
|
|
991
1007
|
return slug && slug !== 'default' ? `${home}/${slug}` : home;
|
|
992
1008
|
}
|
|
993
1009
|
|
|
1010
|
+
/** `sc` without blocking the tick: notices to many sessions go out side by side. */
|
|
1011
|
+
function scAsync(args, { input = null, timeout = 120_000 } = {}) {
|
|
1012
|
+
return new Promise((done) => {
|
|
1013
|
+
const child = spawnChild(supercodeBin(), args, { cwd: tmpdir(), env: process.env, stdio: ['pipe', 'pipe', 'pipe'] });
|
|
1014
|
+
let stdout = '';
|
|
1015
|
+
let stderr = '';
|
|
1016
|
+
const timer = setTimeout(() => { stderr += ` timed out after ${timeout}ms`; child.kill('SIGKILL'); }, timeout);
|
|
1017
|
+
child.stdout.setEncoding('utf8').on('data', (c) => { stdout += c; });
|
|
1018
|
+
child.stderr.setEncoding('utf8').on('data', (c) => { stderr += c; });
|
|
1019
|
+
child.on('error', (error) => { clearTimeout(timer); done({ code: null, stdout, stderr: stderr + error.message }); });
|
|
1020
|
+
child.on('close', (code) => { clearTimeout(timer); done({ code, stdout, stderr }); });
|
|
1021
|
+
child.stdin.end(input ?? '');
|
|
1022
|
+
});
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
// how many deliveries a tick has in flight at once (each is a `supercode message` process of a few seconds)
|
|
1026
|
+
const SEND_CONCURRENCY = 8;
|
|
1027
|
+
|
|
1028
|
+
/** Run `fn` over `items`, at most `n` at a time. */
|
|
1029
|
+
async function pool(items, n, fn) {
|
|
1030
|
+
let next = 0;
|
|
1031
|
+
await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
|
|
1032
|
+
while (next < items.length) { const i = next; next += 1; await fn(items[i]); }
|
|
1033
|
+
}));
|
|
1034
|
+
}
|
|
1035
|
+
|
|
994
1036
|
/**
|
|
995
1037
|
* One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
|
|
996
1038
|
* `only` limits starting to one card (a create that answers whether its card started). Answers one report per board.
|
|
@@ -1005,8 +1047,11 @@ const CORRUPT = /malformed|not a database|corrupt|disk image/i;
|
|
|
1005
1047
|
* per-tick lock (Hermes's `.dispatch.lock`), skipped when another dispatcher holds it. Answers one report per board.
|
|
1006
1048
|
*/
|
|
1007
1049
|
export async function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true }) {
|
|
1050
|
+
const entry = await loadWorkflow(root, { fresh: true });
|
|
1008
1051
|
const needFleet = Object.values(lanes).some((l) => l.surface === 'pane');
|
|
1052
|
+
const fleetAt = Date.now();
|
|
1009
1053
|
const fleet = needFleet ? observeFleet() : { ok: false, alive: new Set(), online: new Set(), sessions: [], error: 'no pane lanes' };
|
|
1054
|
+
const fleetMs = Date.now() - fleetAt;
|
|
1010
1055
|
const boards = slugs ?? listBoards(root);
|
|
1011
1056
|
// the running cards of every board of the home, for kanban.max_in_progress (host-wide, as Hermes counts it)
|
|
1012
1057
|
let hostRunning = 0;
|
|
@@ -1015,7 +1060,9 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
|
|
|
1015
1060
|
}
|
|
1016
1061
|
const out = [];
|
|
1017
1062
|
for (const slug of boards) {
|
|
1018
|
-
const report = { board: slug, mailed: [], started: [], queued: [], failed: [], lost: [], crashed: [], timed_out: [], reclaimed: [], auto_blocked: [], promoted: 0, unassigned: [], nonspawnable: [], notified: [], undelivered: [], fleet: fleet.ok ? 'read' : fleet.error };
|
|
1063
|
+
const report = { board: slug, ms: { fleet: fleetMs }, mailed: [], started: [], queued: [], failed: [], lost: [], crashed: [], timed_out: [], reclaimed: [], auto_blocked: [], promoted: 0, unassigned: [], nonspawnable: [], notified: [], undelivered: [], fleet: fleet.ok ? 'read' : fleet.error };
|
|
1064
|
+
// the mailbox's answers in flight (not part of the report the caller prints)
|
|
1065
|
+
Object.defineProperty(report, 'answers', { value: [], enumerable: false });
|
|
1019
1066
|
const path = boardPath(root, slug);
|
|
1020
1067
|
const marker = `${path}.corrupt-seen`;
|
|
1021
1068
|
if (existsSync(marker) && readFileSync(marker, 'utf8').trim() === fingerprint(path)) { report.error = 'skipped: the board file was corrupt and has not changed since (`supercode workflow repair`)'; out.push(report); continue; }
|
|
@@ -1025,14 +1072,21 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
|
|
|
1025
1072
|
try {
|
|
1026
1073
|
board = openBoard(root, slug);
|
|
1027
1074
|
board.label = boardLabel(root, slug);
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1075
|
+
// where the tick's time goes, phase by phase (ms), printed by `serve --verbose` and in `--json`
|
|
1076
|
+
let mark = Date.now();
|
|
1077
|
+
const phase = (name) => { const t = Date.now(); report.ms[name] = t - mark; mark = t; };
|
|
1078
|
+
readMailbox(board, boardAddress(board.label), report); phase('mailbox');
|
|
1079
|
+
const effects = [];
|
|
1080
|
+
supervise(board, fleet, settings, report, entry, effects); phase('supervise');
|
|
1081
|
+
report.promoted = recomputeReady(board);
|
|
1031
1082
|
const before = report.started.length;
|
|
1032
|
-
if (dispatch)
|
|
1033
|
-
hostRunning += report.started.length - before;
|
|
1034
|
-
if (dispatch && !only && !report.paused)
|
|
1035
|
-
|
|
1083
|
+
if (dispatch) startCards(board, fleet, lanes, settings, report, entry, effects, { only, hostRunning });
|
|
1084
|
+
hostRunning += report.started.length - before; phase('start');
|
|
1085
|
+
if (dispatch && !only && !report.paused) observeIdle(board, fleet, settings, report, entry, effects);
|
|
1086
|
+
// what the moves asked for, once they are committed (messages to actors, closed panes, cleaned workspaces)
|
|
1087
|
+
await runEffects(board, effects);
|
|
1088
|
+
phase('goals');
|
|
1089
|
+
await notify(board, fleet, report); phase('notify');
|
|
1036
1090
|
fireHooks(board, settings, report);
|
|
1037
1091
|
if (String(board.db.prepare('PRAGMA journal_mode').get()?.journal_mode).toLowerCase() === 'wal') board.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get();
|
|
1038
1092
|
report.ready_waiting = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks t LEFT JOIN supercode_cards c ON c.task_id = t.id WHERE t.status = 'ready' AND t.assignee IS NOT NULL AND c.subtask_of IS NULL").get().n);
|
|
@@ -1043,6 +1097,7 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
|
|
|
1043
1097
|
report.error = error.message;
|
|
1044
1098
|
if (CORRUPT.test(error.message)) writeFileSync(marker, fingerprint(path));
|
|
1045
1099
|
} finally {
|
|
1100
|
+
await Promise.all(report.answers);
|
|
1046
1101
|
board?.close();
|
|
1047
1102
|
lock.release();
|
|
1048
1103
|
}
|