@volter/supercode-orchestrator 0.4.29 → 0.4.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/board/cli.mjs +65 -42
- package/board/dispatch.mjs +293 -274
- package/board/engine.mjs +151 -310
- package/board/expr.mjs +212 -0
- package/board/store.mjs +4 -2
- package/board/tools.mjs +17 -8
- package/board/workflow.mjs +442 -0
- package/board.mjs +8 -4
- package/kanban-worker.mjs +2 -1
- package/loop.mjs +1 -1
- package/package.json +1 -1
package/board/dispatch.mjs
CHANGED
|
@@ -20,11 +20,13 @@ import { hostname, tmpdir } from 'node:os';
|
|
|
20
20
|
import { basename, dirname, join } from 'node:path';
|
|
21
21
|
import { randomUUID } from 'node:crypto';
|
|
22
22
|
import { fileURLToPath } from 'node:url';
|
|
23
|
-
import { boardPath, event, json, listBoards, now, openBoard, tx } from './store.mjs';
|
|
23
|
+
import { boardPath, claimer, event, json, listBoards, now, openBoard, tx } from './store.mjs';
|
|
24
24
|
import { holdBoardTickLock } from '../hermes-locks.mjs';
|
|
25
|
+
import { doors, fire, loadWorkflow, resumeOf, scopeOf, statusOf, templateValue, workflowFor } from './workflow.mjs';
|
|
26
|
+
import { evaluate, holds, render } from './expr.mjs';
|
|
25
27
|
import {
|
|
26
|
-
|
|
27
|
-
recomputeReady,
|
|
28
|
+
cardContext, claimTtl, currentRun, DEFAULT_FAILURE_LIMIT, extendClaim, getTask, lastSession,
|
|
29
|
+
recomputeReady, runEffects, setRunSession, subscribeIn, subscriptions,
|
|
28
30
|
} from './engine.mjs';
|
|
29
31
|
|
|
30
32
|
const WORKER = fileURLToPath(new URL('../bin/hermes-worker.mjs', import.meta.url));
|
|
@@ -145,9 +147,8 @@ export function lanesOf(profiles = {}) {
|
|
|
145
147
|
|
|
146
148
|
// ------------------------------------------------------------------ supervise
|
|
147
149
|
|
|
148
|
-
function supervise(board, fleet, settings, report) {
|
|
150
|
+
function supervise(board, fleet, settings, report, entry, effects) {
|
|
149
151
|
const rows = board.db.prepare("SELECT id FROM tasks WHERE status = 'running'").all();
|
|
150
|
-
const stop = [];
|
|
151
152
|
for (const { id } of rows) {
|
|
152
153
|
tx(board, (db) => {
|
|
153
154
|
const task = getTask(db, id);
|
|
@@ -155,28 +156,34 @@ function supervise(board, fleet, settings, report) {
|
|
|
155
156
|
const runRow = currentRun(db, id);
|
|
156
157
|
const session = runRow?.metadata?.supercode ?? null;
|
|
157
158
|
const t = now();
|
|
159
|
+
// what the dispatcher saw becomes a platform event; the card's workflow says what it does
|
|
160
|
+
const observe = (name, payload, bucket) => {
|
|
161
|
+
try {
|
|
162
|
+
const r = fire(db, board, entry, id, name, { roles: ['platform'], payload });
|
|
163
|
+
effects.push(...r.effects);
|
|
164
|
+
if (bucket) report[bucket].push(id);
|
|
165
|
+
if (r.to !== r.from && entry.workflow.statuses[r.to]?.stored_as === 'blocked') report.auto_blocked.push(id);
|
|
166
|
+
} catch (error) {
|
|
167
|
+
// an observation no transition takes is nothing to do (a guard the workflow keeps, such as a grace)
|
|
168
|
+
if (!error.noTransition) report.failed.push([id, `${name}: ${String(error.message).slice(-200)}`]);
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
const limits = limitsOf(db, board, entry, task, session);
|
|
158
172
|
// Hermes's orphan reconciliation: a running card whose claim bookkeeping is broken, with nothing live behind it
|
|
159
173
|
const live = session?.surface === 'headless' ? pidAlive(session.pid) : session?.session_id ? (fleet.ok ? sessionAlive(fleet, session) : true) : false;
|
|
160
174
|
if (settings.reconcileOrphans && (!task.claim_lock || !task.claim_expires) && !live) {
|
|
161
|
-
|
|
162
|
-
db.prepare('INSERT INTO task_comments (task_id, author, body, created_at) VALUES (?, ?, ?, ?)').run(id, 'dispatcher', 'Requeued: the card was running with no claim and nothing live behind it.', t);
|
|
163
|
-
report.reclaimed.push(id);
|
|
175
|
+
observe('orphaned', { reason: 'orphaned_running', claim_lock: task.claim_lock, claim_expires: task.claim_expires, error: 'orphaned running card (broken claim bookkeeping)' }, 'reclaimed');
|
|
164
176
|
return;
|
|
165
177
|
}
|
|
166
|
-
// a
|
|
167
|
-
if (
|
|
168
|
-
|
|
169
|
-
release(db, id, { outcome: 'timed_out', error: `ran past max_runtime ${task.max_runtime_seconds}s`, kind: 'timed_out', payload: { limit_seconds: task.max_runtime_seconds } });
|
|
170
|
-
if (recordFailure(db, id, `timed out after ${task.max_runtime_seconds}s`, { failureLimit: settings.failureLimit, outcome: 'timed_out' })) report.auto_blocked.push(id);
|
|
171
|
-
report.timed_out.push(id);
|
|
178
|
+
// a run past its runtime limit (the status's `run.limits.runtime`), as Hermes times its workers out
|
|
179
|
+
if (limits.runtime && runRow && t - Number(runRow.started_at) > Number(limits.runtime)) {
|
|
180
|
+
observe('timed_out', { limit_seconds: limits.runtime, error: `ran past max_runtime ${limits.runtime}s` }, 'timed_out');
|
|
172
181
|
return;
|
|
173
182
|
}
|
|
174
183
|
if (session?.surface === 'headless') {
|
|
175
|
-
// Hermes's stale check: a worker running past the
|
|
176
|
-
if (
|
|
177
|
-
|
|
178
|
-
release(db, id, { outcome: 'stale', error: `no heartbeat after ${t - Number(runRow.started_at)}s running`, kind: 'stale', payload: { timeout_seconds: settings.staleTimeout, pid: session.pid } });
|
|
179
|
-
report.reclaimed.push(id);
|
|
184
|
+
// Hermes's stale check: a worker running past the limit with no heartbeat in the last hour is stopped
|
|
185
|
+
if (limits.stale && runRow && t - Number(runRow.started_at) >= Number(limits.stale) && !(task.last_heartbeat_at && t - Number(task.last_heartbeat_at) < 3600)) {
|
|
186
|
+
observe('stale', { timeout_seconds: limits.stale, pid: session.pid, error: `no heartbeat after ${t - Number(runRow.started_at)}s running` }, 'reclaimed');
|
|
180
187
|
return;
|
|
181
188
|
}
|
|
182
189
|
if (pidAlive(session.pid)) {
|
|
@@ -186,39 +193,32 @@ function supervise(board, fleet, settings, report) {
|
|
|
186
193
|
if (t - Number(session.launched_at ?? 0) < settings.crashGrace) return;
|
|
187
194
|
// how the worker ended, from the exit code its wrapper wrote (Hermes reads its reaped children's)
|
|
188
195
|
const code = session.exit_file && existsSync(session.exit_file) ? Number(readFileSync(session.exit_file, 'utf8').trim()) : null;
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
release(db, id, { outcome: 'rate_limited', error: 'worker hit a rate limit (exit 75)', kind: 'rate_limited', payload: { worker_pid: session.pid, exit_code: 75 } });
|
|
192
|
-
report.crashed.push(id);
|
|
193
|
-
return;
|
|
194
|
-
}
|
|
195
|
-
const clean = code === 0;
|
|
196
|
-
const why = clean ? `worker ${session.pid} exited cleanly (rc=0) without completing or blocking the card` : `worker ${session.pid} exited${code === null ? '' : ` with ${code}`} without completing the card`;
|
|
197
|
-
release(db, id, { outcome: 'crashed', error: why, kind: 'crashed', payload: { worker_pid: session.pid, exit_code: code, protocol_violation: clean } });
|
|
198
|
-
// a clean exit that left the card running is a protocol violation: Hermes trips the breaker on the first
|
|
199
|
-
if (clean) db.prepare('UPDATE tasks SET consecutive_failures = ? WHERE id = ?').run(Math.max(0, (task.max_retries ?? settings.failureLimit) - 1), id);
|
|
200
|
-
if (recordFailure(db, id, why, { failureLimit: settings.failureLimit })) report.auto_blocked.push(id);
|
|
201
|
-
report.crashed.push(id);
|
|
196
|
+
const why = code === 75 ? 'worker hit a rate limit (exit 75)' : code === 0 ? `worker ${session.pid} exited cleanly (rc=0) without completing or blocking the card` : `worker ${session.pid} exited${code === null ? '' : ` with ${code}`} without completing the card`;
|
|
197
|
+
observe('exited', { code, worker_pid: session.pid, exit_code: code, protocol_violation: code === 0, error: why }, 'crashed');
|
|
202
198
|
return;
|
|
203
199
|
}
|
|
204
200
|
if (session?.session_id) {
|
|
205
201
|
if (!fleet.ok) return; // no reading, no verdict
|
|
206
202
|
const alive = sessionAlive(fleet, session);
|
|
207
203
|
const machineUp = fleet.online.has(session.machine);
|
|
208
|
-
const young = t - Number(session.launched_at ?? 0) < settings.grace;
|
|
204
|
+
const young = t - Number(session.launched_at ?? 0) < Number(limits.grace ?? settings.grace);
|
|
209
205
|
const missing = !alive && machineUp && !young;
|
|
210
206
|
const next = { ...session };
|
|
207
|
+
// a machine that is off the fleet: how long, for the workflow's machine_offline (the claim is kept meanwhile)
|
|
208
|
+
if (machineUp) delete next.offline_since; else next.offline_since ??= t;
|
|
209
|
+
if (!machineUp) {
|
|
210
|
+
setRunSession(db, runRow.id, next);
|
|
211
|
+
observe('machine_offline', { machine: session.machine, seconds: t - next.offline_since }, null);
|
|
212
|
+
if (getTask(db, id)?.current_run_id !== runRow.id) return; // the workflow moved it
|
|
213
|
+
}
|
|
211
214
|
if (alive) { next.seen_at ??= t; delete next.missing_since; }
|
|
212
215
|
if (missing) next.missing_since ??= t;
|
|
213
|
-
if (missing && t - next.missing_since >= settings.grace) {
|
|
214
|
-
// lost: gone from an online machine for the whole grace. A resume that never came up is not resumable
|
|
215
|
-
// a fresh start that never came up is a failure. The card goes back to ready and the spawn resumes it.
|
|
216
|
+
if (missing && t - next.missing_since >= Number(limits.grace ?? settings.grace)) {
|
|
217
|
+
// lost: gone from an online machine for the whole grace. A resume that never came up is not resumable.
|
|
216
218
|
if (next.mode === 'resume' && !next.seen_at) next.resumable = false;
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
report.lost.push(id);
|
|
221
|
-
if (next.mode === 'fresh' && !next.seen_at && recordFailure(db, id, `session ${next.session_id} never came up on ${next.machine}`, { failureLimit: settings.failureLimit, outcome: 'crashed' })) report.auto_blocked.push(id);
|
|
219
|
+
setRunSession(db, runRow.id, next);
|
|
220
|
+
observe('session_lost', { session_id: next.session_id, machine: next.machine, pane: next.pane, mode: next.mode, seen: Boolean(next.seen_at),
|
|
221
|
+
never_came_up: next.mode === 'fresh' && !next.seen_at, error: `session ${next.session_id} is gone from ${next.machine}` }, 'lost');
|
|
222
222
|
return;
|
|
223
223
|
}
|
|
224
224
|
if (JSON.stringify(next) !== JSON.stringify(session)) setRunSession(db, runRow.id, next);
|
|
@@ -229,13 +229,21 @@ function supervise(board, fleet, settings, report) {
|
|
|
229
229
|
}
|
|
230
230
|
// claimed with nothing launched (a dispatcher that died between the two, or another implementation's claim):
|
|
231
231
|
// the claim's own expiry answers it, as Hermes's does
|
|
232
|
-
if (task.claim_expires && Number(task.claim_expires) < t) {
|
|
233
|
-
release(db, id, { outcome: 'reclaimed', error: `stale_lock=${task.claim_lock}`, kind: 'reclaimed', payload: { stale_lock: task.claim_lock, claim_expires: task.claim_expires } });
|
|
234
|
-
report.reclaimed.push(id);
|
|
235
|
-
}
|
|
232
|
+
if (task.claim_expires && Number(task.claim_expires) < t) observe('orphaned', { stale_lock: task.claim_lock, claim_expires: task.claim_expires, error: `stale_lock=${task.claim_lock}` }, 'reclaimed');
|
|
236
233
|
});
|
|
237
234
|
}
|
|
238
|
-
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** A running card's limits: its status's `run.limits`, each evaluated for the card (seconds; null for none). */
|
|
238
|
+
function limitsOf(db, board, entry, task, session) {
|
|
239
|
+
const status = statusOf(db, task);
|
|
240
|
+
const scope = scopeOf(db, board, task, { params: entry.params, extra: { run: { surface: session?.surface ?? null, local: true } } });
|
|
241
|
+
const out = {};
|
|
242
|
+
for (const [name, expr] of Object.entries(entry.workflow.statuses[status]?.run?.limits ?? {})) {
|
|
243
|
+
const v = evaluate(expr, scope);
|
|
244
|
+
out[name] = v === null || v === undefined || v === '' ? null : Number(v);
|
|
245
|
+
}
|
|
246
|
+
return out;
|
|
239
247
|
}
|
|
240
248
|
|
|
241
249
|
function stopSession(session) {
|
|
@@ -251,6 +259,211 @@ function stopSession(session) {
|
|
|
251
259
|
if (session.pane) sc(['teams', 'panes', 'close', session.pane, ...(session.machine ? ['--machine', session.machine] : [])], { timeout: 60_000 });
|
|
252
260
|
}
|
|
253
261
|
|
|
262
|
+
// ------------------------------------------------------------------ starting, as the workflow's dispatch says
|
|
263
|
+
|
|
264
|
+
/** The command form a card's session uses to reach its board: `--root` on the board's machine, `--at` elsewhere. */
|
|
265
|
+
export function whereOf(board, task) {
|
|
266
|
+
const local = !task.machine || normalMachine(task.machine) === localMachine();
|
|
267
|
+
return local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(board.label)}`;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// what the platform tells expressions about this machine and this tick
|
|
271
|
+
const hostOf = (pressure) => ({ memory_pressure: pressure, max_in_progress: derivedMaxInProgress() });
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Start what the workflow's dispatcher may start this tick: cards whose status has a transition for `dispatch.start`,
|
|
275
|
+
* that `dispatch.eligible` admits, in `dispatch.order`, under `dispatch.limits` (a cap per group, slots reserved for
|
|
276
|
+
* the cards a reserve names while any waits). Each is sent the start event (its validators are Hermes's respawn
|
|
277
|
+
* guard) and what its new status runs is launched.
|
|
278
|
+
*/
|
|
279
|
+
function startCards(board, fleet, lanes, settings, report, entry, effects, { only = null, hostRunning = 0 } = {}) {
|
|
280
|
+
const stop = paused(board.root);
|
|
281
|
+
if (stop) { report.paused = stop; return; }
|
|
282
|
+
const { workflow, params } = entry;
|
|
283
|
+
const d = workflow.dispatch;
|
|
284
|
+
const pressure = memoryPressure();
|
|
285
|
+
const tickState = { local_started: 0 };
|
|
286
|
+
const startable = new Set(Object.entries(workflow.statuses).filter(([, st]) => st.on?.[d.start]).map(([name]) => name));
|
|
287
|
+
const everyCard = board.db.prepare("SELECT id FROM tasks WHERE status != 'archived'").all().map((r) => getTask(board.db, r.id));
|
|
288
|
+
const scopeFor = (task, extra = {}) => scopeOf(board.db, board, task, { params, extra: { host: hostOf(pressure), tick: tickState, ...extra } });
|
|
289
|
+
// who would run it: the role of the status its start moves it to, and that role's lane
|
|
290
|
+
const planOf = (task, scope) => {
|
|
291
|
+
const status = statusOf(board.db, task);
|
|
292
|
+
const t = (workflow.statuses[status]?.on?.[d.start] ?? []).find((x) => (!x.by?.length || x.by.includes('platform')) && holds(x.guard ?? null, scope));
|
|
293
|
+
if (!t?.target) return null;
|
|
294
|
+
const target = String(templateValue(t.target, scope));
|
|
295
|
+
const runSpec = workflow.statuses[target]?.run ?? null;
|
|
296
|
+
const profile = runSpec ? evaluate(workflow.roles?.[runSpec.role] ?? 'card.assignee', scope) : null;
|
|
297
|
+
return { target, runSpec, profile, lane: profile ? lanes[profile] ?? null : null };
|
|
298
|
+
};
|
|
299
|
+
const candidates = [];
|
|
300
|
+
for (const task of everyCard) {
|
|
301
|
+
if (only && task.id !== only) continue;
|
|
302
|
+
if (!startable.has(statusOf(board.db, task))) continue;
|
|
303
|
+
const plan = planOf(task, scopeFor(task));
|
|
304
|
+
if (!plan) continue;
|
|
305
|
+
const scope = scopeFor(task, { lane: plan.lane ?? {}, run: { surface: plan.lane?.surface ?? null, local: plan.lane?.surface !== 'pane' || normalMachine(task.machine ?? '') === localMachine() } });
|
|
306
|
+
if (!holds(d.eligible ?? null, scope)) continue;
|
|
307
|
+
candidates.push({ task, plan, scope });
|
|
308
|
+
}
|
|
309
|
+
const keyOf = (c) => (d.order ?? []).map((o) => { const desc = o.startsWith('-'); const v = evaluate(desc ? o.slice(1) : o, c.scope); return { v, desc }; });
|
|
310
|
+
candidates.sort((a, b) => {
|
|
311
|
+
const ka = keyOf(a); const kb = keyOf(b);
|
|
312
|
+
for (let i = 0; i < ka.length; i += 1) {
|
|
313
|
+
if (ka[i].v === kb[i].v) continue;
|
|
314
|
+
const lt = (ka[i].v ?? -Infinity) < (kb[i].v ?? -Infinity);
|
|
315
|
+
return (lt ? -1 : 1) * (ka[i].desc ? -1 : 1);
|
|
316
|
+
}
|
|
317
|
+
return 0;
|
|
318
|
+
});
|
|
319
|
+
// the caps: how many counted cards each group holds now
|
|
320
|
+
const limits = (d.limits ?? []).map((l) => ({ ...l, tally: new Map() }));
|
|
321
|
+
const cardScope = new Map(everyCard.map((c) => [c.id, scopeFor(c, { lane: lanes[c.assignee] ?? {}, fleet_alive: fleet.ok ? sessionAlive(fleet, lastSession(board.db, c.id)) : false })]));
|
|
322
|
+
for (const l of limits) {
|
|
323
|
+
for (const [, sc] of cardScope) {
|
|
324
|
+
sc.card.session_live = sc.fleet_alive;
|
|
325
|
+
if (!holds(l.counts, sc)) continue;
|
|
326
|
+
const g = JSON.stringify(evaluate(l.group, sc));
|
|
327
|
+
l.tally.set(g, (l.tally.get(g) ?? 0) + 1);
|
|
328
|
+
}
|
|
329
|
+
// the home's other boards count toward a group that is the whole home (kanban.max_in_progress across boards)
|
|
330
|
+
const here = [...cardScope.values()].filter((sc) => holds(l.counts, sc)).length;
|
|
331
|
+
if (evaluate(l.group, { card: {} }) === 'home') l.tally.set(JSON.stringify('home'), (l.tally.get(JSON.stringify('home')) ?? 0) + Math.max(0, hostRunning - here));
|
|
332
|
+
}
|
|
333
|
+
const reserveWaits = (l) => l.reserve && candidates.some((c) => holds(l.reserve.for, c.scope));
|
|
334
|
+
for (const c of candidates) {
|
|
335
|
+
const { task, plan, scope } = c;
|
|
336
|
+
if (!plan.profile) { report.unassigned.push(task.id); tx(board, (db) => noteGuard(db, task.id, 'no profile runs it (no assignee, and the workflow names none)')); continue; }
|
|
337
|
+
if (!plan.lane) { report.nonspawnable.push(task.id); tx(board, (db) => noteGuard(db, task.id, `${plan.profile} is no lane of this home`)); continue; }
|
|
338
|
+
let why = null;
|
|
339
|
+
for (const l of limits) {
|
|
340
|
+
const max = evaluate(l.max, scope);
|
|
341
|
+
if (max === null || max === undefined || max === '') continue;
|
|
342
|
+
const g = JSON.stringify(evaluate(l.group, scope));
|
|
343
|
+
const held = reserveWaits(l) && !holds(l.reserve.for, scope) ? Number(evaluate(l.reserve.slots, scope) ?? 0) : 0;
|
|
344
|
+
if ((l.tally.get(g) ?? 0) >= Number(max) - held) { why = `${l.description ?? 'a cap'} (${max}${held ? `, ${held} held for others` : ''})`; break; }
|
|
345
|
+
}
|
|
346
|
+
// the launcher's own conditions: a lane that can start this card where it names
|
|
347
|
+
if (!why) why = launcherCheck(board.db, task, plan.lane, fleet);
|
|
348
|
+
if (why) { tx(board, (db) => noteGuard(db, task.id, why)); report.queued.push([task.id, why]); continue; }
|
|
349
|
+
let moved;
|
|
350
|
+
try {
|
|
351
|
+
moved = tx(board, (db) => {
|
|
352
|
+
if (!task.assignee && plan.profile) {
|
|
353
|
+
db.prepare('UPDATE tasks SET assignee = ? WHERE id = ?').run(plan.profile, task.id);
|
|
354
|
+
event(db, task.id, 'assigned', { assignee: plan.profile, source: 'workflow role' });
|
|
355
|
+
}
|
|
356
|
+
return fire(db, board, entry, task.id, d.start, { roles: ['platform'], payload: { lock: claimer() }, extra: { host: hostOf(pressure), tick: tickState } });
|
|
357
|
+
});
|
|
358
|
+
} catch (error) {
|
|
359
|
+
const reason = error.refusal ?? String(error.message).slice(-300);
|
|
360
|
+
tx(board, (db) => noteGuard(db, task.id, reason));
|
|
361
|
+
report.queued.push([task.id, reason]);
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
effects.push(...moved.effects);
|
|
365
|
+
for (const l of limits) if (holds(l.counts, cardScope.get(task.id) ?? scope)) { const g = JSON.stringify(evaluate(l.group, scope)); l.tally.set(g, (l.tally.get(g) ?? 0) + 1); }
|
|
366
|
+
try {
|
|
367
|
+
const session = launchRun(board, entry, task.id, plan.lane, fleet, settings);
|
|
368
|
+
if (session) {
|
|
369
|
+
if (session.surface !== 'pane' || normalMachine(task.machine ?? '') === localMachine()) tickState.local_started += 1;
|
|
370
|
+
report.started.push([task.id, session.address ?? `pid ${session.pid}`, session.mode]);
|
|
371
|
+
}
|
|
372
|
+
} catch (error) {
|
|
373
|
+
try { const r = tx(board, (db) => fire(db, board, entry, task.id, 'spawn_failed', { roles: ['platform'], payload: { error: String(error.message).slice(-500) } })); effects.push(...r.effects); } catch { /* no transition for it */ }
|
|
374
|
+
report.failed.push([task.id, String(error.message).slice(-300)]);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/** Launch what the card's status runs: its role's lane, in the run's session slot, told the run's prompt and doors. */
|
|
380
|
+
export function launchRun(board, entry, id, lane, fleet, settings) {
|
|
381
|
+
const task = getTaskNow(board, id);
|
|
382
|
+
const status = tx(board, (db) => statusOf(db, task));
|
|
383
|
+
const spec = entry.workflow.statuses[status]?.run;
|
|
384
|
+
if (!spec) return null;
|
|
385
|
+
const runId = task.current_run_id;
|
|
386
|
+
const surface = lane.surface === 'pane' ? 'pane' : 'headless';
|
|
387
|
+
const scope = tx(board, (db) => scopeOf(db, board, task, { params: entry.params }));
|
|
388
|
+
const where = whereOf(board, task);
|
|
389
|
+
const prompt = render(spec.prompt, { ...scope, where, context: tx(board, (db) => cardContext(db, id)), doors: doors(entry.workflow, status, spec.role, id, where, surface) });
|
|
390
|
+
// the run's session record knows its status, role and slot (a reviewer's slot is never the implementer's)
|
|
391
|
+
const meta = { status, role: spec.role, slot: spec.session };
|
|
392
|
+
return surface === 'pane'
|
|
393
|
+
? launchPane(board, task, runId, lane, fleet, { input: prompt, slot: spec.session, meta })
|
|
394
|
+
: launchHeadless(board, task, runId, lane, settings, { prompt, skills: spec.skills ?? [], meta });
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/** Start one card now by a person's word (a triage card's session, a manager's start): the workflow's start event. */
|
|
398
|
+
export async function startCard(board, id, { roles = ['manager'], lanes, settings } = {}) {
|
|
399
|
+
const entry = await loadWorkflow(board.root);
|
|
400
|
+
const fleet = observeFleet();
|
|
401
|
+
const moved = tx(board, (db) => fire(db, board, entry, id, entry.workflow.dispatch.start, { roles, payload: { lock: claimer() } }));
|
|
402
|
+
const task = getTaskNow(board, id);
|
|
403
|
+
const spec = entry.workflow.statuses[moved.to]?.run;
|
|
404
|
+
const profile = spec ? tx(board, (db) => evaluate(entry.workflow.roles?.[spec.role] ?? 'card.assignee', scopeOf(db, board, task, { params: entry.params }))) : null;
|
|
405
|
+
const lane = profile ? lanes[profile] : null;
|
|
406
|
+
if (spec && !lane) throw new Error(`${id} is ${moved.to}, run by ${spec.role} (${profile ?? 'nobody'}), which is no lane of this home`);
|
|
407
|
+
return lane ? launchRun(board, entry, id, lane, fleet, settings) : null;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* The heartbeat, observed: a running pane session whose last message is a whole tick old is sent `session_idle` with
|
|
412
|
+
* how many times it has been asked this run and whether it worked since the last ask; the workflow decides the rest.
|
|
413
|
+
*/
|
|
414
|
+
function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
415
|
+
if (!fleet.ok) return;
|
|
416
|
+
for (const { id } of board.db.prepare("SELECT id FROM tasks WHERE status = 'running'").all()) {
|
|
417
|
+
tx(board, (db) => {
|
|
418
|
+
const task = getTask(db, id);
|
|
419
|
+
const status = statusOf(db, task);
|
|
420
|
+
if (!(entry.workflow.statuses[status]?.on?.session_idle || entry.workflow.on?.session_idle)) return;
|
|
421
|
+
const runRow = currentRun(db, id);
|
|
422
|
+
const session = runRow?.metadata?.supercode;
|
|
423
|
+
if (!session?.session_id || session.surface !== 'pane') return;
|
|
424
|
+
const live = fleet.sessions.find((x) => x.address?.endsWith(`:${session.session_id}`));
|
|
425
|
+
// a supercode that reports no last message time gives no reading, and no reading gives no verdict
|
|
426
|
+
const lastAt = Math.floor(Number(live?.last_message_at_ms) / 1000);
|
|
427
|
+
if (!(lastAt > 0)) return;
|
|
428
|
+
if (now() - lastAt < settings.interval) return;
|
|
429
|
+
if (session.goal_nudged_at && now() - session.goal_nudged_at < settings.interval * 2) return;
|
|
430
|
+
const worked = Boolean(session.goal_nudged_at) && lastAt >= session.goal_nudged_at + settings.interval;
|
|
431
|
+
try {
|
|
432
|
+
const where = whereOf(board, task);
|
|
433
|
+
// the ask names what the session may do, as its run's prompt did
|
|
434
|
+
const said = doors(entry.workflow, status, session.role ?? entry.workflow.statuses[status]?.run?.role ?? 'implementer', id, where, session.surface === 'pane' ? 'pane' : 'headless');
|
|
435
|
+
const r = fire(db, board, entry, id, 'session_idle', { roles: ['platform'], payload: { turns: Number(session.goal_turns ?? 0), worked }, extra: { where, doors: said } });
|
|
436
|
+
effects.push(...r.effects);
|
|
437
|
+
const after = currentRun(db, id);
|
|
438
|
+
if (after?.id === runRow.id) setRunSession(db, runRow.id, { ...(after.metadata?.supercode ?? session), goal_nudged_at: now() });
|
|
439
|
+
report.notified.push([id, 'session_idle', live.address]);
|
|
440
|
+
} catch (error) { report.failed.push([id, `session_idle: ${String(error.message).slice(-200)}`]); }
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/** A session effect a committed move asked for: a message to the card's current actor, or closing a slot's pane. */
|
|
446
|
+
export async function sessionEffect(board, e) {
|
|
447
|
+
if (e.kind === 'send_to_actor') {
|
|
448
|
+
const run = tx(board, (db) => currentRun(db, e.id));
|
|
449
|
+
const session = run?.metadata?.supercode;
|
|
450
|
+
const address = session?.address ?? (session?.session_id && session.machine ? `sc:${session.machine}:claude-code:${session.session_id}` : null);
|
|
451
|
+
if (!address) return;
|
|
452
|
+
await send(boardAddress(board.label), board.label, address, e.text, `board-actor-${e.id}-${now()}`);
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
if (e.kind === 'close_session') {
|
|
456
|
+
const rows = tx(board, (db) => db.prepare('SELECT id, metadata FROM task_runs WHERE task_id = ? ORDER BY id DESC').all(e.id));
|
|
457
|
+
for (const row of rows) {
|
|
458
|
+
const session = json(row.metadata, {})?.supercode;
|
|
459
|
+
if ((session?.slot ?? 'worker') !== e.slot || !session?.pane || session.closed_at) continue;
|
|
460
|
+
stopSession(session);
|
|
461
|
+
tx(board, (db) => setRunSession(db, row.id, { ...session, closed_at: now() }));
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
|
|
254
467
|
// ------------------------------------------------------------------ start
|
|
255
468
|
|
|
256
469
|
const running = (db, sql, ...args) => Number(db.prepare(`SELECT COUNT(*) AS n FROM tasks t LEFT JOIN supercode_cards c ON c.task_id = t.id WHERE t.status = 'running' ${sql}`).get(...args).n);
|
|
@@ -269,15 +482,11 @@ function machineLoad(db, machine, fleet) {
|
|
|
269
482
|
}
|
|
270
483
|
|
|
271
484
|
/** Why a card cannot start now, or null. */
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
const cap = lane.capacity ?? settings.perProfile;
|
|
275
|
-
if (cap && running(db, 'AND t.assignee = ?', lane.name) >= cap) return `${lane.name} is at capacity (${cap}); the card starts when one of its sessions ends`;
|
|
485
|
+
/** What the launcher itself needs to start this card (the caps are the workflow's `dispatch.limits`). */
|
|
486
|
+
function launcherCheck(db, task, lane, fleet) {
|
|
276
487
|
if (lane.surface !== 'pane') return null;
|
|
277
488
|
if (!task.machine) return 'the card names no machine; a pane session runs on the machine its card names';
|
|
278
489
|
const machine = normalMachine(task.machine);
|
|
279
|
-
const mcap = settings.machines[machine]?.capacity;
|
|
280
|
-
if (mcap && machineLoad(db, machine, fleet) >= mcap) return `machine ${machine} holds ${mcap} of the board's sessions (its capacity); the card starts when one is closed`;
|
|
281
490
|
if (!fleet.ok) return `supercode could not read the fleet: ${fleet.error}`;
|
|
282
491
|
if (!fleet.online.has(machine)) return `machine ${machine} is offline; the card starts when it is back`;
|
|
283
492
|
// a card on this machine gets its workspace made here (scratch, dir or worktree, as Hermes makes it); another
|
|
@@ -293,17 +502,6 @@ function guard(db, task, lane, fleet, settings, budget = null) {
|
|
|
293
502
|
const BLOCKER = /\b(quota|rate[\s_-]?limit|429|403|auth\w*|unauthorized|forbidden|billing|subscription|access[\s_]denied|permission[\s_]denied|invalid[\s_]api[\s_]key)\b/i;
|
|
294
503
|
const PR_URL = /https?:\/\/github\.com\/[^/\s]+\/[^/\s]+\/pull\/\d+/i;
|
|
295
504
|
|
|
296
|
-
function respawnGuard(db, task, settings, lane) {
|
|
297
|
-
const t = now();
|
|
298
|
-
const last = db.prepare('SELECT outcome, ended_at FROM task_runs WHERE task_id = ? AND ended_at IS NOT NULL ORDER BY ended_at DESC LIMIT 1').get(task.id);
|
|
299
|
-
if (last?.outcome === 'rate_limited') return settings.rateLimitCooldown > 0 && t - Number(last.ended_at) < settings.rateLimitCooldown ? 'rate_limit_cooldown' : null;
|
|
300
|
-
if (task.last_failure_error && BLOCKER.test(task.last_failure_error)) return 'blocker_auth';
|
|
301
|
-
if (lane === 'review') return null;
|
|
302
|
-
const done = db.prepare("SELECT ended_at FROM task_runs WHERE task_id = ? AND outcome = 'completed' AND ended_at >= ? ORDER BY ended_at DESC LIMIT 1").get(task.id, t - 3600);
|
|
303
|
-
if (done && !db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND created_at >= ? AND kind IN ('status', 'promoted', 'unblocked', 'reclaimed') LIMIT 1").get(task.id, done.ended_at)) return 'recent_success';
|
|
304
|
-
for (const c of db.prepare('SELECT body FROM task_comments WHERE task_id = ? AND created_at >= ?').all(task.id, t - 86400)) if (PR_URL.test(c.body ?? '')) return 'active_pr';
|
|
305
|
-
return null;
|
|
306
|
-
}
|
|
307
505
|
|
|
308
506
|
/** This machine's memory pressure as Hermes reads it (Linux's MemAvailable; `unknown` elsewhere, which restricts nothing). */
|
|
309
507
|
function memoryPressure() {
|
|
@@ -331,135 +529,15 @@ export function paused(root) {
|
|
|
331
529
|
return [join(root, 'ESTOP'), join(process.env.HOME ?? '', '.hermes', 'ESTOP')].find((p) => existsSync(p)) ?? null;
|
|
332
530
|
}
|
|
333
531
|
|
|
334
|
-
/**
|
|
335
|
-
* The tick's spawn budget, as Hermes counts it: `kanban.max_spawn` against this board's running cards, and
|
|
336
|
-
* `kanban.max_in_progress` against every board of the home (workers share one machine's memory).
|
|
337
|
-
*/
|
|
338
|
-
function spawnBudget(board, settings, hostRunning) {
|
|
339
|
-
const here = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks WHERE status = 'running'").get().n);
|
|
340
|
-
const limits = [];
|
|
341
|
-
if (settings.maxSpawn) limits.push([settings.maxSpawn - here, `the board runs ${settings.maxSpawn} cards at most (kanban.max_spawn)`]);
|
|
342
|
-
if (settings.maxInProgress) limits.push([settings.maxInProgress - hostRunning, `the home runs ${settings.maxInProgress} cards at most across its boards (kanban.max_in_progress)`]);
|
|
343
|
-
if (!limits.length) return null;
|
|
344
|
-
const [left, why] = limits.sort((a, b) => a[0] - b[0])[0];
|
|
345
|
-
return { left, why };
|
|
346
|
-
}
|
|
347
532
|
|
|
348
|
-
function start(board, fleet, lanes, settings, report, { only = null, hostRunning = 0 } = {}) {
|
|
349
|
-
const stop = paused(board.root);
|
|
350
|
-
if (stop) { report.paused = stop; return; }
|
|
351
|
-
const pressure = memoryPressure();
|
|
352
|
-
let localStarted = 0;
|
|
353
|
-
const order = 'ORDER BY priority DESC, created_at ASC';
|
|
354
|
-
const ready = board.db.prepare(`SELECT id, status, assignee FROM tasks WHERE status = 'ready' AND claim_lock IS NULL ${order}`).all();
|
|
355
|
-
const review = settings.reviewDispatch ? board.db.prepare(`SELECT id, status, assignee FROM tasks WHERE status = 'review' AND claim_lock IS NULL ${order}`).all() : [];
|
|
356
|
-
const budget = spawnBudget(board, settings, hostRunning);
|
|
357
|
-
// Hermes's review reservation: while review work waits, one slot of the budget is held back from the ready lane
|
|
358
|
-
const reviewWaits = review.some((r) => lanes[r.assignee || settings.defaultAssignee]);
|
|
359
|
-
const readyBudget = budget && budget.left > 0 && reviewWaits ? { left: budget.left - 1, why: `${budget.why}; one slot is held for review` } : budget;
|
|
360
|
-
for (const row of [...ready, ...review]) {
|
|
361
|
-
const lanesBudget = row.status === 'review' ? budget : readyBudget;
|
|
362
|
-
if (only && row.id !== only) continue;
|
|
363
|
-
if (board.db.prepare('SELECT subtask_of FROM supercode_cards WHERE task_id = ?').get(row.id)?.subtask_of) continue; // worked by its arc's session
|
|
364
|
-
const assignee = row.assignee || settings.defaultAssignee;
|
|
365
|
-
if (!assignee) { report.unassigned.push(row.id); continue; }
|
|
366
|
-
const lane = lanes[assignee];
|
|
367
|
-
if (!lane) { report.nonspawnable.push(row.id); continue; }
|
|
368
|
-
const claimed = tx(board, (db) => {
|
|
369
|
-
const task = getTask(db, row.id);
|
|
370
|
-
if (!task || task.status !== row.status || task.claim_lock) return null;
|
|
371
|
-
let why = guard(db, task, lane, fleet, settings, lanesBudget);
|
|
372
|
-
// this machine's memory bounds what starts here: every one-shot worker, and a pane session on this machine
|
|
373
|
-
const here = lane.surface !== 'pane' || normalMachine(task.machine ?? '') === localMachine();
|
|
374
|
-
if (!why && here && (pressure === 'critical' || (pressure === 'elevated' && localStarted >= 1))) why = `this machine's memory pressure is ${pressure}; the card starts when it eases`;
|
|
375
|
-
// Hermes's respawn guard, on every lane (a pane card whose session is still live is kept, never started twice)
|
|
376
|
-
if (!why) why = respawnGuard(db, task, settings, row.status);
|
|
377
|
-
if (why) { noteGuard(db, row.id, why); report.queued.push([row.id, why]); return null; }
|
|
378
|
-
if (!row.assignee) {
|
|
379
|
-
db.prepare('UPDATE tasks SET assignee = ? WHERE id = ?').run(assignee, row.id);
|
|
380
|
-
event(db, row.id, 'assigned', { assignee, source: 'kanban.default_assignee' });
|
|
381
|
-
}
|
|
382
|
-
const runId = claimIn(db, row.id, { ttl: settings.ttl, expectStatus: row.status });
|
|
383
|
-
return runId ? { task: getTask(db, row.id), runId } : null;
|
|
384
|
-
});
|
|
385
|
-
if (!claimed) continue;
|
|
386
|
-
try {
|
|
387
|
-
const session = lane.surface === 'pane'
|
|
388
|
-
? launchPane(board, claimed.task, claimed.runId, lane, fleet, { input: row.status === 'review' ? reviewAsk(board, claimed.task) : null })
|
|
389
|
-
: launchHeadless(board, claimed.task, claimed.runId, lane, settings);
|
|
390
|
-
if (lane.surface !== 'pane' || normalMachine(claimed.task.machine ?? '') === localMachine()) localStarted += 1;
|
|
391
|
-
if (budget) budget.left -= 1;
|
|
392
|
-
if (readyBudget && readyBudget !== budget) readyBudget.left -= 1;
|
|
393
|
-
report.started.push([row.id, session.address ?? `pid ${session.pid}`, session.mode]);
|
|
394
|
-
} catch (error) {
|
|
395
|
-
tx(board, (db) => {
|
|
396
|
-
release(db, row.id, { outcome: 'spawn_failed', error: String(error.message).slice(-500), kind: 'spawn_failed', payload: { error: String(error.message).slice(-300) } });
|
|
397
|
-
if (recordFailure(db, row.id, error.message, { failureLimit: settings.failureLimit, outcome: 'spawn_failed' })) report.auto_blocked.push(row.id);
|
|
398
|
-
});
|
|
399
|
-
report.failed.push([row.id, String(error.message).slice(-300)]);
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
533
|
|
|
404
|
-
/** What a session started from the review lane is asked to do: verify the handoff and give the verdict. */
|
|
405
|
-
function reviewAsk(board, task) {
|
|
406
|
-
const local = normalMachine(task.machine) === localMachine();
|
|
407
|
-
const where = local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(board.label)}`;
|
|
408
|
-
return `You are reviewing card ${task.id} on the ${board.label} board: its implementer asked for review. Verify the handoff against the card's acceptance, `
|
|
409
|
-
+ `then give the verdict: \`supercode workflow complete ${task.id} --summary "<what you verified>" ${where}\` approves it, and `
|
|
410
|
-
+ `\`supercode workflow request-changes ${task.id} "<what must change>" ${where}\` sends it back.\n\n${tx(board, (db) => cardContext(db, task.id))}`;
|
|
411
|
-
}
|
|
412
534
|
|
|
413
|
-
/** What a session started on a triage card is asked to do: write its spec, or split it into cards it waits on. */
|
|
414
|
-
function triageAsk(board, task, label, purpose) {
|
|
415
|
-
const local = normalMachine(task.machine) === localMachine();
|
|
416
|
-
const where = local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(label)}`;
|
|
417
|
-
const context = tx(board, (db) => cardContext(db, task.id));
|
|
418
|
-
return purpose === 'decompose'
|
|
419
|
-
? `Card ${task.id} on the ${label} board is in triage: a rough idea to split. Read it, then file the cards it breaks into and make it wait on them: `
|
|
420
|
-
+ `\`supercode workflow decompose ${task.id} --child "<title>[@<profile>]" --child "..." ${where}\` (each child gets this card's machine and directory). `
|
|
421
|
-
+ `The card itself stays and wakes when they are done.\n\n${context}`
|
|
422
|
-
: `Card ${task.id} on the ${label} board is in triage: a rough idea. Write its spec (the goal, the approach, what will show it is done), then file it: `
|
|
423
|
-
+ `\`supercode workflow specify ${task.id} --title "<a tightened title>" --body "<the spec>" ${where}\`. The card then waits for its parents and starts.\n\n${context}`;
|
|
424
|
-
}
|
|
425
535
|
|
|
426
|
-
/**
|
|
427
|
-
* Start a session on a triage card to specify or decompose it (where Hermes asks an auxiliary model, supercode asks a
|
|
428
|
-
* session in the card's lane, which then keeps the card's memory when the card starts). Answers the session.
|
|
429
|
-
*/
|
|
430
|
-
export function startTriageSession(board, id, lanes, settings, purpose) {
|
|
431
|
-
const fleet = observeFleet();
|
|
432
|
-
const task = tx(board, (db) => getTask(db, id));
|
|
433
|
-
if (!task) throw new Error(`no card ${id}`);
|
|
434
|
-
if (task.status !== 'triage') throw new Error(`${id} is ${task.status}; only a triage card is ${purpose === 'decompose' ? 'decomposed' : 'specified'}`);
|
|
435
|
-
const lane = lanes[task.assignee || settings.defaultAssignee];
|
|
436
|
-
if (!lane || lane.surface !== 'pane') throw new Error(`${id} has no pane lane to start a session in; give its spec directly (${purpose === 'decompose' ? '--child' : '--title/--body'})`);
|
|
437
|
-
const runId = tx(board, (db) => {
|
|
438
|
-
const why = guard(db, task, lane, fleet, settings);
|
|
439
|
-
if (why) throw new Error(`${id} cannot start a session now: ${why}`);
|
|
440
|
-
return claimIn(db, id, { ttl: settings.ttl, expectStatus: 'triage' });
|
|
441
|
-
});
|
|
442
|
-
if (!runId) throw new Error(`${id} could not be claimed`);
|
|
443
|
-
return launchPane(board, getTaskNow(board, id), runId, lane, fleet, { input: triageAsk(board, task, board.label, purpose) });
|
|
444
|
-
}
|
|
445
536
|
|
|
446
537
|
const getTaskNow = (board, id) => tx(board, (db) => getTask(db, id));
|
|
447
538
|
|
|
448
|
-
function opening(board, task, label) {
|
|
449
|
-
const context = tx(board, (db) => cardContext(db, task.id));
|
|
450
|
-
const skills = task.skills?.length ? `\nSkills pinned to this card: ${task.skills.join(', ')}.` : '';
|
|
451
|
-
const at = `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}`;
|
|
452
|
-
// the board's own door is on the machine that keeps it; a session elsewhere writes to its card through its manager
|
|
453
|
-
const local = normalMachine(task.machine) === localMachine();
|
|
454
|
-
const where = local ? at : `--at ${boardAddress(label)}`;
|
|
455
|
-
const door = `\nYour card is yours to keep current: \`supercode workflow comment ${task.id} "<text>" ${where}\`, \`block ${task.id} "<why>" --kind <needs_input|capability|transient|dependency> ${where}\` `
|
|
456
|
-
+ `and \`request-review ${task.id} --summary "<what>" ${where}\` update it; \`create "<title>" --subtask-of ${task.id} ${where}\` files a subtask, and \`complete <subtask> ${where}\` marks one done`
|
|
457
|
-
+ `${local ? '' : ' (each is filed with the board and answered by a board notice)'}. Its reviewers, not you, mark the card itself done.`;
|
|
458
|
-
return `You are the session working card ${task.id} on the ${label} board. The board started you; updates to the card `
|
|
459
|
-
+ `(comments, reassignment, blocks, completion) and to the cards it waits on reach you as board notices.${door}${skills}\n\n${context}`;
|
|
460
|
-
}
|
|
461
539
|
|
|
462
|
-
function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
|
|
540
|
+
function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'worker', meta = {} } = {}) {
|
|
463
541
|
const machine = normalMachine(task.machine);
|
|
464
542
|
if (machine === localMachine() && !(task.workspace_kind === 'dir' && task.workspace_path && existsSync(task.workspace_path))) {
|
|
465
543
|
const made = resolveLocalWorkspace(board, task);
|
|
@@ -469,7 +547,8 @@ function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
|
|
|
469
547
|
const model = task.model_override ?? lane.model;
|
|
470
548
|
const tail = model ? ['--model', model] : [];
|
|
471
549
|
const base = ['open', '--new', 'claude', '--detach', '--on', task.machine, '--cwd', task.workspace_path];
|
|
472
|
-
|
|
550
|
+
// the slot's last session: kept when live, resumed when lost, never another slot's (a reviewer's is its own)
|
|
551
|
+
const prev = tx(board, (db) => lastSession(db, task.id, slot));
|
|
473
552
|
let session = null;
|
|
474
553
|
let replaced = null;
|
|
475
554
|
if (prev && prev.machine === machine) {
|
|
@@ -477,6 +556,8 @@ function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
|
|
|
477
556
|
// the session is kept; the heartbeat's count belongs to the run that ended, so the new run starts it again
|
|
478
557
|
const { missing_since: _m, goal_turns: _t, goal_nudged_at: _n, ...kept } = prev;
|
|
479
558
|
session = { ...kept, mode: 'adopt', launched_at: now(), seen_at: now() };
|
|
559
|
+
// a kept session is told what this run is for (its card came back to it, with what happened since)
|
|
560
|
+
if (input) deliverNow(board, prev.address ?? `sc:${prev.machine}:claude-code:${prev.session_id}`, input, `board-run-${task.id}-r${runId}`);
|
|
480
561
|
} else if (prev.resumable !== false) {
|
|
481
562
|
const r = sc([...base, '--key', `board-${task.id}-r${runId}-resume`, '--input',
|
|
482
563
|
`The board resumed this session after it was lost; you are still on card ${task.id}: ${task.title}. Carry on.`, '--', '--resume', prev.session_id, ...tail], { timeout: 180_000 });
|
|
@@ -489,12 +570,13 @@ function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
|
|
|
489
570
|
} else if (prev) replaced = { session_id: prev.session_id, why: `the card moved from ${prev.machine} to ${machine}` };
|
|
490
571
|
if (!session) {
|
|
491
572
|
const sid = randomUUID();
|
|
492
|
-
const r = sc([...base, '--key', `board-${task.id}-r${runId}-${sid.slice(0, 8)}`, '--input', input ??
|
|
573
|
+
const r = sc([...base, '--key', `board-${task.id}-r${runId}-${sid.slice(0, 8)}`, '--input', input ?? `Card ${task.id}: ${task.title}`, '--', '--session-id', sid, ...tail], { timeout: 180_000 });
|
|
493
574
|
if (r.code !== 0) throw new Error(`supercode open --on ${task.machine}: ${(r.stderr || r.stdout).trim().slice(-500)}`);
|
|
494
575
|
session = { surface: 'pane', session_id: sid, machine, pane: lastJson(r.stdout)?.pane ?? null, mode: 'fresh', launched_at: now(), ...(replaced ? { replaces: replaced.session_id } : {}) };
|
|
495
576
|
}
|
|
496
577
|
session.harness = 'claude-code';
|
|
497
578
|
session.address = `sc:${machine}:claude-code:${session.session_id}`;
|
|
579
|
+
Object.assign(session, meta);
|
|
498
580
|
const kind = { adopt: 'session_adopted', resume: 'session_resumed', fresh: replaced ? 'session_replaced' : 'session_started' }[session.mode];
|
|
499
581
|
tx(board, (db) => {
|
|
500
582
|
setRunSession(db, runId, session);
|
|
@@ -520,10 +602,10 @@ function paneSurvives(machine, pane) {
|
|
|
520
602
|
}
|
|
521
603
|
|
|
522
604
|
/** Hermes's own worker argv and task environment, run on this machine; watched by its pid. */
|
|
523
|
-
function launchHeadless(board, task, runId, lane, settings) {
|
|
605
|
+
function launchHeadless(board, task, runId, lane, settings, { prompt = null, skills: runSkills = [], meta = {} } = {}) {
|
|
524
606
|
const workspace = resolveLocalWorkspace(board, task);
|
|
525
|
-
|
|
526
|
-
const skills =
|
|
607
|
+
// the card's own skills and the ones its status's run names (a reviewer's, for one)
|
|
608
|
+
const skills = [...new Set([...task.skills, ...runSkills])];
|
|
527
609
|
const args = ['-p', lane.name, '--cli', '--accept-hooks', ...skills.flatMap((s) => ['--skills', s]),
|
|
528
610
|
...(task.model_override ? ['-m', task.model_override, ...(task.provider_override ? ['--provider', task.provider_override] : [])] : []),
|
|
529
611
|
...(task.reasoning_effort ? ['--reasoning', task.reasoning_effort] : []), 'chat', '-q', `work kanban task ${task.id}`];
|
|
@@ -541,12 +623,12 @@ function launchHeadless(board, task, runId, lane, settings) {
|
|
|
541
623
|
...(task.goal_mode ? { HERMES_KANBAN_GOAL_MODE: '1', ...(task.goal_max_turns ? { HERMES_KANBAN_GOAL_MAX_TURNS: String(task.goal_max_turns) } : {}) } : {}),
|
|
542
624
|
...process.env, HERMES_HOME: lane.dir, HERMES_PROFILE: lane.name, HERMES_KANBAN_TASK: task.id, HERMES_KANBAN_WORKSPACE: workspace,
|
|
543
625
|
HERMES_KANBAN_RUN_ID: String(runId), HERMES_KANBAN_DB: board.path, HERMES_KANBAN_BOARD: board.slug, HERMES_SESSION_SOURCE: 'kanban',
|
|
544
|
-
HERMES_KANBAN_CLAIM_LOCK:
|
|
626
|
+
HERMES_KANBAN_CLAIM_LOCK: claimer(), ...(prompt ? { SUPERCODE_BOARD_PROMPT: prompt } : {}), ...(task.tenant ? { HERMES_TENANT: task.tenant } : {}),
|
|
545
627
|
...(task.branch_name ? { HERMES_KANBAN_BRANCH: task.branch_name } : {}),
|
|
546
628
|
},
|
|
547
629
|
});
|
|
548
630
|
child.unref();
|
|
549
|
-
const session = { surface: 'headless', pid: child.pid, machine: localMachine(), mode: 'fresh', launched_at: now(), exit_file: exitFile };
|
|
631
|
+
const session = { surface: 'headless', pid: child.pid, machine: localMachine(), mode: 'fresh', launched_at: now(), exit_file: exitFile, ...meta };
|
|
550
632
|
tx(board, (db) => {
|
|
551
633
|
db.prepare('UPDATE tasks SET worker_pid = ? WHERE id = ?').run(child.pid, task.id);
|
|
552
634
|
db.prepare('UPDATE task_runs SET worker_pid = ? WHERE id = ?').run(child.pid, runId);
|
|
@@ -620,20 +702,18 @@ export function adoptSession(board, id, who, { workspace = null, ttl, assignee =
|
|
|
620
702
|
if (task.machine && normalMachine(task.machine) !== host) throw new Error(`${id} runs on ${task.machine}, and ${found.address} is on ${host}; move the card first`);
|
|
621
703
|
const was = lastSession(db, id);
|
|
622
704
|
if (was && was.session_id !== sessionId && sessionAlive(fleet, was)) throw new Error(`${id} already has a live session, ${was.address}`);
|
|
623
|
-
if (!task.machine) db.prepare('INSERT
|
|
705
|
+
if (!task.machine) db.prepare('INSERT INTO supercode_cards (task_id, machine) VALUES (?, ?) ON CONFLICT(task_id) DO UPDATE SET machine = excluded.machine').run(id, host);
|
|
624
706
|
// the lane that resumes it if it is lost, set in the same transaction (assigning first would let a tick start another)
|
|
625
707
|
if (assignee && assignee !== task.assignee) { db.prepare('UPDATE tasks SET assignee = ? WHERE id = ?').run(assignee, id); event(db, id, 'assigned', { assignee }); }
|
|
626
708
|
if (dir && !(task.workspace_kind === 'dir' && task.workspace_path)) {
|
|
627
709
|
db.prepare("UPDATE tasks SET workspace_kind = 'dir', workspace_path = ? WHERE id = ?").run(dir.replace(/^dir:/, ''), id);
|
|
628
710
|
}
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
}
|
|
633
|
-
const runId = claimIn(db, id, { ttl, expectStatus: 'ready' });
|
|
711
|
+
// the workflow's adopt event: the card is claimed for its implementer, whose session this now is
|
|
712
|
+
fire(db, board, workflowFor(board.root), id, 'adopt', { roles: ['manager'], payload: { lock: claimer() } });
|
|
713
|
+
const runId = getTask(db, id).current_run_id;
|
|
634
714
|
if (!runId) throw new Error(`${id} waits on parents that are not done`);
|
|
635
715
|
const session = { surface: 'pane', session_id: sessionId, harness, machine: host, pane: found.pane ?? null, address: found.address, mode: 'adopt', launched_at: now(), seen_at: now() };
|
|
636
|
-
setRunSession(db, runId, session);
|
|
716
|
+
setRunSession(db, runId, { ...(currentRun(db, id)?.metadata?.supercode ?? {}), ...session, slot: 'worker' });
|
|
637
717
|
event(db, id, 'session_adopted', { session_id: sessionId, machine: host, pane: session.pane, address: found.address, mode: 'adopt' }, runId);
|
|
638
718
|
subscribeIn(db, id, found.address, 'worker');
|
|
639
719
|
for (const p of db.prepare('SELECT parent_id FROM task_links WHERE child_id = ?').all(id)) subscribeIn(db, p.parent_id, found.address, 'dependency');
|
|
@@ -677,6 +757,13 @@ function notice(kind, task, p, role) {
|
|
|
677
757
|
}
|
|
678
758
|
}
|
|
679
759
|
|
|
760
|
+
// a notice sent within a transaction's caller that cannot await (the launcher): the same door, synchronously
|
|
761
|
+
function deliverNow(board, to, body, key) {
|
|
762
|
+
const request = { op: 'send', from: boardAddress(board.label), from_name: `${board.label} board`, to, queue: false, id: key,
|
|
763
|
+
body: `[Board notice from the ${board.label} board] ${body}\n\n(An automated notice from the board's dispatcher; replies to this address go nowhere.)` };
|
|
764
|
+
sc(['message', 'remote-deliver'], { input: JSON.stringify(request), timeout: 120_000 });
|
|
765
|
+
}
|
|
766
|
+
|
|
680
767
|
async function send(from, label, to, body, key, { queue = false } = {}) {
|
|
681
768
|
const request = {
|
|
682
769
|
op: 'send', from, from_name: `${label} board`, to, queue, id: key,
|
|
@@ -913,79 +1000,7 @@ function readMailbox(board, address, report) {
|
|
|
913
1000
|
|
|
914
1001
|
// ------------------------------------------------------------------ goals and triage
|
|
915
1002
|
|
|
916
|
-
/**
|
|
917
|
-
* The board's heartbeat for a pane session: a running card's session never sits idle without deciding. Each tick, a
|
|
918
|
-
* session that is idle while its card is still running is asked to decide: `request-review` when the work is done,
|
|
919
|
-
* `block` with its kind when it cannot go on (waiting on a person, a capability, another card), or keep working. A
|
|
920
|
-
* goal card (Hermes's goal loop) is told to keep going. Idle is judged by the session's last message time (supercode
|
|
921
|
-
* reads it from the transcript), never by its status label: a session is asked once its last message is a whole tick
|
|
922
|
-
* old; one whose messages ran on a whole tick past the last ask answered it by working, and its count starts again
|
|
923
|
-
* (a reply to the ask alone is not work). After the turn budget (`goal_max_turns`, else
|
|
924
|
-
* `kanban.goal_max_turns`) of asks in a row with no work and no decision (a goal card: of turns), the card is blocked `needs_input` and its creator is told.
|
|
925
|
-
* Whether the work is done is the session's call through its card, not an auxiliary judge's.
|
|
926
|
-
*/
|
|
927
|
-
async function pursueGoals(board, fleet, settings, report) {
|
|
928
|
-
if (!fleet.ok) return;
|
|
929
|
-
const cards = board.db.prepare("SELECT id FROM tasks WHERE status = 'running'").all();
|
|
930
|
-
const nudges = [];
|
|
931
|
-
for (const { id } of cards) {
|
|
932
|
-
const nudge = tx(board, (db) => {
|
|
933
|
-
const task = getTask(db, id);
|
|
934
|
-
const runRow = currentRun(db, id);
|
|
935
|
-
const session = runRow?.metadata?.supercode;
|
|
936
|
-
if (!session?.session_id || session.surface !== 'pane') return null;
|
|
937
|
-
const live = fleet.sessions.find((x) => x.address?.endsWith(`:${session.session_id}`));
|
|
938
|
-
// a supercode that reports no last message time gives no reading, and no reading gives no verdict
|
|
939
|
-
const lastAt = Math.floor(Number(live.last_message_at_ms) / 1000);
|
|
940
|
-
if (!(lastAt > 0)) return null;
|
|
941
|
-
// a message within the last tick is a session working or between turns, not idle
|
|
942
|
-
if (now() - lastAt < settings.interval) return null;
|
|
943
|
-
if (session.goal_nudged_at && now() - session.goal_nudged_at < settings.interval * 2) return null;
|
|
944
|
-
// messages that ran on a whole tick past the last ask answered it by working: the count starts again (a goal
|
|
945
|
-
// card keeps Hermes's budget of turns)
|
|
946
|
-
const worked = Boolean(session.goal_nudged_at) && lastAt >= session.goal_nudged_at + settings.interval;
|
|
947
|
-
const turns = worked && !task.goal_mode ? 0 : Number(session.goal_turns ?? 0);
|
|
948
|
-
const budget = task.goal_max_turns ?? settings.goalMaxTurns;
|
|
949
|
-
if (turns >= budget) {
|
|
950
|
-
blockIn(db, id, { reason: `its session went idle ${budget} times without deciding done or blocked; review it`, kind: 'needs_input' });
|
|
951
|
-
return null;
|
|
952
|
-
}
|
|
953
|
-
setRunSession(db, runRow.id, { ...session, goal_turns: turns + 1, goal_nudged_at: now() });
|
|
954
|
-
event(db, id, task.goal_mode ? 'goal_continued' : 'decision_asked', { turn: turns + 1, budget }, runRow.id);
|
|
955
|
-
const local = normalMachine(task.machine) === localMachine();
|
|
956
|
-
const where = local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(board.label)}`;
|
|
957
|
-
const decide = `When it is done, \`supercode workflow request-review ${id} --summary "<what>" ${where}\`; if you cannot go on, \`supercode workflow block ${id} "<why>" --kind <needs_input|capability|transient|dependency> ${where}\`.`;
|
|
958
|
-
const text = task.goal_mode
|
|
959
|
-
? `Card ${id} (${task.title}) is not done yet: keep working toward it (goal turn ${turns + 1} of ${budget}). ${decide}`
|
|
960
|
-
: `Card ${id} (${task.title}) is still running and you are idle: decide (${turns + 1} of ${budget}). If there is more to do, keep working. ${decide} `
|
|
961
|
-
+ `A session that stays idle without deciding has its card blocked for its creator after ${budget}.`;
|
|
962
|
-
return { address: live.address, text, kind: task.goal_mode ? 'goal_continued' : 'decision_asked' };
|
|
963
|
-
});
|
|
964
|
-
if (nudge) nudges.push({ id, ...nudge });
|
|
965
|
-
}
|
|
966
|
-
await pool(nudges, SEND_CONCURRENCY, async (nudge) => {
|
|
967
|
-
try { await send(boardAddress(board.label), board.label, nudge.address, nudge.text, `board-goal-${nudge.id}-${now()}`); report.notified.push([nudge.id, nudge.kind, nudge.address]); }
|
|
968
|
-
catch (error) { report.undelivered.push([nudge.id, nudge.address, String(error.message).slice(-200)]); }
|
|
969
|
-
});
|
|
970
|
-
}
|
|
971
1003
|
|
|
972
|
-
/** Hermes's auto-decompose: triage cards with a pane lane get a session that splits them, a few per tick. */
|
|
973
|
-
function autoDecompose(board, lanes, settings, report) {
|
|
974
|
-
if (!settings.autoDecompose) return;
|
|
975
|
-
let started = 0;
|
|
976
|
-
for (const { id, assignee } of board.db.prepare("SELECT id, assignee FROM tasks WHERE status = 'triage' ORDER BY priority DESC, created_at ASC").all()) {
|
|
977
|
-
if (started >= settings.autoDecomposePerTick) break;
|
|
978
|
-
const lane = lanes[assignee || settings.defaultAssignee];
|
|
979
|
-
if (lane?.surface !== 'pane') continue;
|
|
980
|
-
try {
|
|
981
|
-
const session = startTriageSession(board, id, lanes, settings, 'decompose');
|
|
982
|
-
report.started.push([id, session.address, 'decompose']);
|
|
983
|
-
started += 1;
|
|
984
|
-
} catch (error) {
|
|
985
|
-
report.queued.push([id, String(error.message).slice(-300)]);
|
|
986
|
-
}
|
|
987
|
-
}
|
|
988
|
-
}
|
|
989
1004
|
|
|
990
1005
|
// ------------------------------------------------------------------ the tick
|
|
991
1006
|
|
|
@@ -1035,6 +1050,7 @@ const CORRUPT = /malformed|not a database|corrupt|disk image/i;
|
|
|
1035
1050
|
* per-tick lock (Hermes's `.dispatch.lock`), skipped when another dispatcher holds it. Answers one report per board.
|
|
1036
1051
|
*/
|
|
1037
1052
|
export async function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true }) {
|
|
1053
|
+
const entry = await loadWorkflow(root, { fresh: true });
|
|
1038
1054
|
const needFleet = Object.values(lanes).some((l) => l.surface === 'pane');
|
|
1039
1055
|
const fleetAt = Date.now();
|
|
1040
1056
|
const fleet = needFleet ? observeFleet() : { ok: false, alive: new Set(), online: new Set(), sessions: [], error: 'no pane lanes' };
|
|
@@ -1063,12 +1079,15 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
|
|
|
1063
1079
|
let mark = Date.now();
|
|
1064
1080
|
const phase = (name) => { const t = Date.now(); report.ms[name] = t - mark; mark = t; };
|
|
1065
1081
|
readMailbox(board, boardAddress(board.label), report); phase('mailbox');
|
|
1066
|
-
|
|
1067
|
-
|
|
1082
|
+
const effects = [];
|
|
1083
|
+
supervise(board, fleet, settings, report, entry, effects); phase('supervise');
|
|
1084
|
+
report.promoted = recomputeReady(board);
|
|
1068
1085
|
const before = report.started.length;
|
|
1069
|
-
if (dispatch)
|
|
1086
|
+
if (dispatch) startCards(board, fleet, lanes, settings, report, entry, effects, { only, hostRunning });
|
|
1070
1087
|
hostRunning += report.started.length - before; phase('start');
|
|
1071
|
-
if (dispatch && !only && !report.paused)
|
|
1088
|
+
if (dispatch && !only && !report.paused) observeIdle(board, fleet, settings, report, entry, effects);
|
|
1089
|
+
// what the moves asked for, once they are committed (messages to actors, closed panes, cleaned workspaces)
|
|
1090
|
+
await runEffects(board, effects);
|
|
1072
1091
|
phase('goals');
|
|
1073
1092
|
await notify(board, fleet, report); phase('notify');
|
|
1074
1093
|
fireHooks(board, settings, report);
|