@volter/supercode-orchestrator 0.4.17 → 0.4.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/board/cli.mjs +791 -120
- package/board/dispatch.mjs +165 -9
- package/board/engine.mjs +225 -7
- package/board.mjs +0 -1
- package/package.json +1 -1
package/board/dispatch.mjs
CHANGED
|
@@ -106,6 +106,11 @@ export function boardSettings(kanban = {}) {
|
|
|
106
106
|
grace: pos(kanban.session_grace_seconds) ?? 120,
|
|
107
107
|
machines: Object.fromEntries(Object.entries(kanban.machines ?? {}).map(([name, m]) => [normalMachine(name), { capacity: pos(m?.capacity) }])),
|
|
108
108
|
interval: Math.max(Number(kanban.dispatch_interval_seconds) || 60, 1),
|
|
109
|
+
staleTimeout: pos(kanban.dispatch_stale_timeout_seconds),
|
|
110
|
+
autoDecompose: kanban.auto_decompose !== false,
|
|
111
|
+
autoDecomposePerTick: pos(kanban.auto_decompose_per_tick) ?? 3,
|
|
112
|
+
goalMaxTurns: pos(kanban.goal_max_turns) ?? 20,
|
|
113
|
+
rateLimitCooldown: Number.isInteger(Number(process.env.HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS)) && process.env.HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS !== '' ? Number(process.env.HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS) : 300,
|
|
109
114
|
};
|
|
110
115
|
}
|
|
111
116
|
|
|
@@ -141,6 +146,13 @@ function supervise(board, fleet, settings, report) {
|
|
|
141
146
|
return;
|
|
142
147
|
}
|
|
143
148
|
if (session?.surface === 'headless') {
|
|
149
|
+
// Hermes's stale check: a worker running past the timeout with no heartbeat in the last hour is stopped
|
|
150
|
+
if (settings.staleTimeout && runRow && t - Number(runRow.started_at) >= settings.staleTimeout && !(task.last_heartbeat_at && t - Number(task.last_heartbeat_at) < 3600)) {
|
|
151
|
+
stop.push(session);
|
|
152
|
+
release(db, id, { outcome: 'stale', error: `no heartbeat after ${t - Number(runRow.started_at)}s running`, kind: 'stale', payload: { timeout_seconds: settings.staleTimeout, pid: session.pid } });
|
|
153
|
+
report.reclaimed.push(id);
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
144
156
|
if (pidAlive(session.pid)) {
|
|
145
157
|
if (Number(task.claim_expires) <= t + 60) extendClaim(db, id, { ttl: settings.ttl, reason: 'pid_alive', extra: { worker_pid: session.pid } });
|
|
146
158
|
return;
|
|
@@ -228,6 +240,38 @@ function guard(db, task, lane, fleet, settings) {
|
|
|
228
240
|
return null;
|
|
229
241
|
}
|
|
230
242
|
|
|
243
|
+
// Hermes's respawn guard, for a one-shot (headless) worker: no respawn while the last failure is an auth or quota
|
|
244
|
+
// wall, after a completion nobody requeued, or while a pull request is open on the card. A pane session is resumed
|
|
245
|
+
// rather than respawned, so it never runs twice and these do not apply.
|
|
246
|
+
const BLOCKER = /\b(quota|rate[\s_-]?limit|429|403|auth\w*|unauthorized|forbidden|billing|subscription|access[\s_]denied|permission[\s_]denied|invalid[\s_]api[\s_]key)\b/i;
|
|
247
|
+
const PR_URL = /https?:\/\/github\.com\/[^/\s]+\/[^/\s]+\/pull\/\d+/i;
|
|
248
|
+
|
|
249
|
+
function respawnGuard(db, task, settings, lane) {
|
|
250
|
+
const t = now();
|
|
251
|
+
const last = db.prepare('SELECT outcome, ended_at FROM task_runs WHERE task_id = ? AND ended_at IS NOT NULL ORDER BY ended_at DESC LIMIT 1').get(task.id);
|
|
252
|
+
if (last?.outcome === 'rate_limited') return settings.rateLimitCooldown > 0 && t - Number(last.ended_at) < settings.rateLimitCooldown ? 'rate_limit_cooldown' : null;
|
|
253
|
+
if (task.last_failure_error && BLOCKER.test(task.last_failure_error)) return 'blocker_auth';
|
|
254
|
+
if (lane === 'review') return null;
|
|
255
|
+
const done = db.prepare("SELECT ended_at FROM task_runs WHERE task_id = ? AND outcome = 'completed' AND ended_at >= ? ORDER BY ended_at DESC LIMIT 1").get(task.id, t - 3600);
|
|
256
|
+
if (done && !db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND created_at >= ? AND kind IN ('status', 'promoted', 'unblocked', 'reclaimed') LIMIT 1").get(task.id, done.ended_at)) return 'recent_success';
|
|
257
|
+
for (const c of db.prepare('SELECT body FROM task_comments WHERE task_id = ? AND created_at >= ?').all(task.id, t - 86400)) if (PR_URL.test(c.body ?? '')) return 'active_pr';
|
|
258
|
+
return null;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** This machine's memory pressure as Hermes reads it (Linux's MemAvailable; `unknown` elsewhere, which restricts nothing). */
|
|
262
|
+
function memoryPressure() {
|
|
263
|
+
try {
|
|
264
|
+
const info = Object.fromEntries(readFileSync('/proc/meminfo', 'utf8').split('\n').map((l) => /^(\w+):\s+(\d+)/.exec(l)).filter(Boolean).map((m) => [m[1], Number(m[2])]));
|
|
265
|
+
const avail = info.MemAvailable;
|
|
266
|
+
const total = info.MemTotal;
|
|
267
|
+
if (!Number.isFinite(avail)) return 'unknown';
|
|
268
|
+
const fraction = total > 0 ? avail / total : null;
|
|
269
|
+
if (avail < 64 * 1024 || (fraction !== null && fraction < 0.05)) return 'critical';
|
|
270
|
+
if (avail < 128 * 1024 || (fraction !== null && fraction < 0.15)) return 'elevated';
|
|
271
|
+
return 'ok';
|
|
272
|
+
} catch { return 'unknown'; }
|
|
273
|
+
}
|
|
274
|
+
|
|
231
275
|
/** Record a guard reason once per change, not once per tick. */
|
|
232
276
|
function noteGuard(db, id, reason) {
|
|
233
277
|
const last = db.prepare("SELECT payload FROM task_events WHERE task_id = ? AND kind IN ('respawn_guarded', 'claimed') ORDER BY id DESC LIMIT 1").get(id);
|
|
@@ -237,6 +281,8 @@ function noteGuard(db, id, reason) {
|
|
|
237
281
|
|
|
238
282
|
function start(board, fleet, lanes, settings, report, { only = null } = {}) {
|
|
239
283
|
const lanesWanted = settings.reviewDispatch ? "('ready', 'review')" : "('ready')";
|
|
284
|
+
const pressure = memoryPressure();
|
|
285
|
+
let headlessStarted = 0;
|
|
240
286
|
const rows = board.db.prepare(`SELECT id, status, assignee FROM tasks WHERE status IN ${lanesWanted} AND claim_lock IS NULL ORDER BY priority DESC, created_at ASC`).all();
|
|
241
287
|
for (const row of rows) {
|
|
242
288
|
if (only && row.id !== only) continue;
|
|
@@ -248,7 +294,12 @@ function start(board, fleet, lanes, settings, report, { only = null } = {}) {
|
|
|
248
294
|
const claimed = tx(board, (db) => {
|
|
249
295
|
const task = getTask(db, row.id);
|
|
250
296
|
if (!task || task.status !== row.status || task.claim_lock) return null;
|
|
251
|
-
|
|
297
|
+
let why = guard(db, task, lane, fleet, settings);
|
|
298
|
+
if (!why && lane.surface !== 'pane') {
|
|
299
|
+
// a one-shot worker runs on this machine: its memory, and Hermes's respawn guard
|
|
300
|
+
if (pressure === 'critical' || (pressure === 'elevated' && headlessStarted >= 1)) why = `this machine's memory pressure is ${pressure}; the card starts when it eases`;
|
|
301
|
+
else why = respawnGuard(db, task, settings, row.status);
|
|
302
|
+
}
|
|
252
303
|
if (why) { noteGuard(db, row.id, why); report.queued.push([row.id, why]); return null; }
|
|
253
304
|
if (!row.assignee) {
|
|
254
305
|
db.prepare('UPDATE tasks SET assignee = ? WHERE id = ?').run(assignee, row.id);
|
|
@@ -259,7 +310,10 @@ function start(board, fleet, lanes, settings, report, { only = null } = {}) {
|
|
|
259
310
|
});
|
|
260
311
|
if (!claimed) continue;
|
|
261
312
|
try {
|
|
262
|
-
const session = lane.surface === 'pane'
|
|
313
|
+
const session = lane.surface === 'pane'
|
|
314
|
+
? launchPane(board, claimed.task, claimed.runId, lane, fleet, { input: row.status === 'review' ? reviewAsk(board, claimed.task) : null })
|
|
315
|
+
: launchHeadless(board, claimed.task, claimed.runId, lane, settings);
|
|
316
|
+
if (lane.surface !== 'pane') headlessStarted += 1;
|
|
263
317
|
report.started.push([row.id, session.address ?? `pid ${session.pid}`, session.mode]);
|
|
264
318
|
} catch (error) {
|
|
265
319
|
tx(board, (db) => {
|
|
@@ -271,6 +325,50 @@ function start(board, fleet, lanes, settings, report, { only = null } = {}) {
|
|
|
271
325
|
}
|
|
272
326
|
}
|
|
273
327
|
|
|
328
|
+
/** What a session started from the review lane is asked to do: verify the handoff and give the verdict. */
|
|
329
|
+
function reviewAsk(board, task) {
|
|
330
|
+
const local = normalMachine(task.machine) === localMachine();
|
|
331
|
+
const where = local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(board.label)}`;
|
|
332
|
+
return `You are reviewing card ${task.id} on the ${board.label} board: its implementer asked for review. Verify the handoff against the card's acceptance, `
|
|
333
|
+
+ `then give the verdict: \`supercode workflow complete ${task.id} --summary "<what you verified>" ${where}\` approves it, and `
|
|
334
|
+
+ `\`supercode workflow request-changes ${task.id} "<what must change>" ${where}\` sends it back.\n\n${tx(board, (db) => cardContext(db, task.id))}`;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/** What a session started on a triage card is asked to do: write its spec, or split it into cards it waits on. */
|
|
338
|
+
function triageAsk(board, task, label, purpose) {
|
|
339
|
+
const local = normalMachine(task.machine) === localMachine();
|
|
340
|
+
const where = local ? `--root ${board.root}${board.slug !== 'default' ? ` --board ${board.slug}` : ''}` : `--at ${boardAddress(label)}`;
|
|
341
|
+
const context = tx(board, (db) => cardContext(db, task.id));
|
|
342
|
+
return purpose === 'decompose'
|
|
343
|
+
? `Card ${task.id} on the ${label} board is in triage: a rough idea to split. Read it, then file the cards it breaks into and make it wait on them: `
|
|
344
|
+
+ `\`supercode workflow decompose ${task.id} --child "<title>[@<profile>]" --child "..." ${where}\` (each child gets this card's machine and directory). `
|
|
345
|
+
+ `The card itself stays and wakes when they are done.\n\n${context}`
|
|
346
|
+
: `Card ${task.id} on the ${label} board is in triage: a rough idea. Write its spec (the goal, the approach, what will show it is done), then file it: `
|
|
347
|
+
+ `\`supercode workflow specify ${task.id} --title "<a tightened title>" --body "<the spec>" ${where}\`. The card then waits for its parents and starts.\n\n${context}`;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Start a session on a triage card to specify or decompose it (where Hermes asks an auxiliary model, supercode asks a
|
|
352
|
+
* session in the card's lane, which then keeps the card's memory when the card starts). Answers the session.
|
|
353
|
+
*/
|
|
354
|
+
export function startTriageSession(board, id, lanes, settings, purpose) {
|
|
355
|
+
const fleet = observeFleet();
|
|
356
|
+
const task = tx(board, (db) => getTask(db, id));
|
|
357
|
+
if (!task) throw new Error(`no card ${id}`);
|
|
358
|
+
if (task.status !== 'triage') throw new Error(`${id} is ${task.status}; only a triage card is ${purpose === 'decompose' ? 'decomposed' : 'specified'}`);
|
|
359
|
+
const lane = lanes[task.assignee || settings.defaultAssignee];
|
|
360
|
+
if (!lane || lane.surface !== 'pane') throw new Error(`${id} has no pane lane to start a session in; give its spec directly (${purpose === 'decompose' ? '--child' : '--title/--body'})`);
|
|
361
|
+
const runId = tx(board, (db) => {
|
|
362
|
+
const why = guard(db, task, lane, fleet, settings);
|
|
363
|
+
if (why) throw new Error(`${id} cannot start a session now: ${why}`);
|
|
364
|
+
return claimIn(db, id, { ttl: settings.ttl, expectStatus: 'triage' });
|
|
365
|
+
});
|
|
366
|
+
if (!runId) throw new Error(`${id} could not be claimed`);
|
|
367
|
+
return launchPane(board, getTaskNow(board, id), runId, lane, fleet, { input: triageAsk(board, task, board.label, purpose) });
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
const getTaskNow = (board, id) => tx(board, (db) => getTask(db, id));
|
|
371
|
+
|
|
274
372
|
function opening(board, task, label) {
|
|
275
373
|
const context = tx(board, (db) => cardContext(db, task.id));
|
|
276
374
|
const skills = task.skills?.length ? `\nSkills pinned to this card: ${task.skills.join(', ')}.` : '';
|
|
@@ -279,13 +377,13 @@ function opening(board, task, label) {
|
|
|
279
377
|
const local = normalMachine(task.machine) === localMachine();
|
|
280
378
|
const where = local ? at : `--at ${boardAddress(label)}`;
|
|
281
379
|
const door = `\nYour card is yours to keep current: \`supercode workflow comment ${task.id} "<text>" ${where}\`, \`block ${task.id} "<why>" --kind <needs_input|capability|transient|dependency> ${where}\` `
|
|
282
|
-
+ `and \`review ${task.id} --summary "<what>" ${where}\` update it; \`create "<title>" --subtask-of ${task.id} ${where}\` files a subtask, and \`complete <subtask> ${where}\` marks one done`
|
|
380
|
+
+ `and \`request-review ${task.id} --summary "<what>" ${where}\` update it; \`create "<title>" --subtask-of ${task.id} ${where}\` files a subtask, and \`complete <subtask> ${where}\` marks one done`
|
|
283
381
|
+ `${local ? '' : ' (each is filed with the board and answered by a board notice)'}. Its reviewers, not you, mark the card itself done.`;
|
|
284
382
|
return `You are the session working card ${task.id} on the ${label} board. The board started you; updates to the card `
|
|
285
383
|
+ `(comments, reassignment, blocks, completion) and to the cards it waits on reach you as board notices.${door}${skills}\n\n${context}`;
|
|
286
384
|
}
|
|
287
385
|
|
|
288
|
-
function launchPane(board, task, runId, lane, fleet) {
|
|
386
|
+
function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
|
|
289
387
|
const machine = normalMachine(task.machine);
|
|
290
388
|
const model = task.model_override ?? lane.model;
|
|
291
389
|
const tail = model ? ['--model', model] : [];
|
|
@@ -309,7 +407,7 @@ function launchPane(board, task, runId, lane, fleet) {
|
|
|
309
407
|
} else if (prev) replaced = { session_id: prev.session_id, why: `the card moved from ${prev.machine} to ${machine}` };
|
|
310
408
|
if (!session) {
|
|
311
409
|
const sid = randomUUID();
|
|
312
|
-
const r = sc([...base, '--key', `board-${task.id}-r${runId}-${sid.slice(0, 8)}`, '--input', opening(board, task, board.label), '--', '--session-id', sid, ...tail], { timeout: 180_000 });
|
|
410
|
+
const r = sc([...base, '--key', `board-${task.id}-r${runId}-${sid.slice(0, 8)}`, '--input', input ?? opening(board, task, board.label), '--', '--session-id', sid, ...tail], { timeout: 180_000 });
|
|
313
411
|
if (r.code !== 0) throw new Error(`supercode open --on ${task.machine}: ${(r.stderr || r.stdout).trim().slice(-500)}`);
|
|
314
412
|
session = { surface: 'pane', session_id: sid, machine, pane: lastJson(r.stdout)?.pane ?? null, mode: 'fresh', launched_at: now(), ...(replaced ? { replaces: replaced.session_id } : {}) };
|
|
315
413
|
}
|
|
@@ -342,7 +440,9 @@ function paneSurvives(machine, pane) {
|
|
|
342
440
|
/** Hermes's own worker argv and task environment, run on this machine; watched by its pid. */
|
|
343
441
|
function launchHeadless(board, task, runId, lane, settings) {
|
|
344
442
|
const workspace = resolveLocalWorkspace(board, task);
|
|
345
|
-
const
|
|
443
|
+
const reviewing = tx(board, (db) => json(db.prepare("SELECT payload FROM task_events WHERE task_id = ? AND run_id = ? AND kind = 'claimed'").get(task.id, runId)?.payload, {})?.source_status === 'review');
|
|
444
|
+
const skills = reviewing ? [...new Set([...task.skills, 'sdlc-review'])] : task.skills;
|
|
445
|
+
const args = ['-p', lane.name, '--cli', '--accept-hooks', ...skills.flatMap((s) => ['--skills', s]),
|
|
346
446
|
...(task.model_override ? ['-m', task.model_override, ...(task.provider_override ? ['--provider', task.provider_override] : [])] : []),
|
|
347
447
|
...(task.reasoning_effort ? ['--reasoning', task.reasoning_effort] : []), 'chat', '-q', `work kanban task ${task.id}`];
|
|
348
448
|
const logDir = join(dirname(board.path), board.slug === 'default' ? 'kanban' : '', 'logs');
|
|
@@ -398,7 +498,7 @@ function resolveLocalWorkspace(board, task) {
|
|
|
398
498
|
* never as the session describes itself. From then on the board watches it, resumes it when lost, and tells it of
|
|
399
499
|
* its card. `who` is an address, a session id, or a `name@machine` the fleet lists.
|
|
400
500
|
*/
|
|
401
|
-
export function
|
|
501
|
+
export function adoptSession(board, id, who, { workspace = null, ttl } = {}) {
|
|
402
502
|
const fleet = observeFleet();
|
|
403
503
|
if (!fleet.ok) throw new Error(`supercode could not read the fleet: ${fleet.error}`);
|
|
404
504
|
const found = fleet.sessions.find((s) => s.address === who || s.address.endsWith(`:${who}`) || s.name === who);
|
|
@@ -528,7 +628,7 @@ function notify(board, fleet, report) {
|
|
|
528
628
|
// ------------------------------------------------------------------ the board's mailbox
|
|
529
629
|
|
|
530
630
|
const ENTRY = fileURLToPath(new URL('../bin/orchestrator.mjs', import.meta.url));
|
|
531
|
-
const NOT_BY_MAIL = new Set(['serve', 'dispatch', 'init', 'attach']);
|
|
631
|
+
const NOT_BY_MAIL = new Set(['serve', 'daemon', 'dispatch', 'init', 'adopt', 'boards', 'gc', 'repair', 'watch', 'tail', 'attach']);
|
|
532
632
|
|
|
533
633
|
/** The board's mailbox on this machine: the directory supercode files mail for `address` in, or null. */
|
|
534
634
|
function mailboxOf(address) {
|
|
@@ -572,6 +672,62 @@ function readMailbox(board, address, report) {
|
|
|
572
672
|
}
|
|
573
673
|
}
|
|
574
674
|
|
|
675
|
+
// ------------------------------------------------------------------ goals and triage
|
|
676
|
+
|
|
677
|
+
/**
|
|
678
|
+
* Hermes's goal loop for a pane session: a goal card whose session sits idle while the card is still running is told
|
|
679
|
+
* to keep going, up to the card's turn budget; when the budget is spent the card is blocked for review. Whether the
|
|
680
|
+
* work is done is the session's call through its card (`request-review`), not an auxiliary judge's.
|
|
681
|
+
*/
|
|
682
|
+
function pursueGoals(board, fleet, settings, report) {
|
|
683
|
+
if (!fleet.ok) return;
|
|
684
|
+
const cards = board.db.prepare("SELECT id FROM tasks WHERE status = 'running' AND goal_mode = 1").all();
|
|
685
|
+
for (const { id } of cards) {
|
|
686
|
+
const nudge = tx(board, (db) => {
|
|
687
|
+
const task = getTask(db, id);
|
|
688
|
+
const runRow = currentRun(db, id);
|
|
689
|
+
const session = runRow?.metadata?.supercode;
|
|
690
|
+
if (!session?.session_id || session.surface !== 'pane') return null;
|
|
691
|
+
const live = fleet.sessions.find((x) => x.address?.endsWith(`:${session.session_id}`));
|
|
692
|
+
if (live?.status !== 'idle') return null;
|
|
693
|
+
if (session.goal_nudged_at && now() - session.goal_nudged_at < settings.interval * 2) return null;
|
|
694
|
+
const turns = Number(session.goal_turns ?? 0);
|
|
695
|
+
const budget = task.goal_max_turns ?? settings.goalMaxTurns;
|
|
696
|
+
if (turns >= budget) {
|
|
697
|
+
db.prepare("UPDATE tasks SET status = 'blocked', claim_lock = NULL, claim_expires = NULL WHERE id = ?").run(id);
|
|
698
|
+
const ended = endRun(db, id, { outcome: 'blocked', summary: `goal turn budget (${budget}) spent` });
|
|
699
|
+
event(db, id, 'blocked', { reason: `goal turn budget (${budget}) spent; review it`, kind: 'needs_input', source_status: 'ready' }, ended);
|
|
700
|
+
return null;
|
|
701
|
+
}
|
|
702
|
+
setRunSession(db, runRow.id, { ...session, goal_turns: turns + 1, goal_nudged_at: now() });
|
|
703
|
+
event(db, id, 'goal_continued', { turn: turns + 1, budget }, runRow.id);
|
|
704
|
+
return { address: live.address, text: `Card ${id} (${task.title}) is not done yet: keep working toward it (goal turn ${turns + 1} of ${budget}). When it is done, \`request-review\` it; if you are stuck, \`block\` it with its kind.` };
|
|
705
|
+
});
|
|
706
|
+
if (nudge) {
|
|
707
|
+
try { send(boardAddress(board.label), board.label, nudge.address, nudge.text, `board-goal-${id}-${now()}`); report.notified.push([id, 'goal_continued', nudge.address]); }
|
|
708
|
+
catch (error) { report.undelivered.push([id, nudge.address, String(error.message).slice(-200)]); }
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
/** Hermes's auto-decompose: triage cards with a pane lane get a session that splits them, a few per tick. */
|
|
714
|
+
function autoDecompose(board, lanes, settings, report) {
|
|
715
|
+
if (!settings.autoDecompose) return;
|
|
716
|
+
let started = 0;
|
|
717
|
+
for (const { id, assignee } of board.db.prepare("SELECT id, assignee FROM tasks WHERE status = 'triage' ORDER BY priority DESC, created_at ASC").all()) {
|
|
718
|
+
if (started >= settings.autoDecomposePerTick) break;
|
|
719
|
+
const lane = lanes[assignee || settings.defaultAssignee];
|
|
720
|
+
if (lane?.surface !== 'pane') continue;
|
|
721
|
+
try {
|
|
722
|
+
const session = startTriageSession(board, id, lanes, settings, 'decompose');
|
|
723
|
+
report.started.push([id, session.address, 'decompose']);
|
|
724
|
+
started += 1;
|
|
725
|
+
} catch (error) {
|
|
726
|
+
report.queued.push([id, String(error.message).slice(-300)]);
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
|
|
575
731
|
// ------------------------------------------------------------------ the tick
|
|
576
732
|
|
|
577
733
|
/** The board's name in notices: the home's owner folder (`…/volter-manager/home` → `volter-manager`), and the slug. */
|
|
@@ -598,10 +754,10 @@ export function tick({ root, lanes, settings, only = null, slugs = null, dispatc
|
|
|
598
754
|
supervise(board, fleet, settings, report);
|
|
599
755
|
report.promoted = recomputeReady(board, settings.failureLimit);
|
|
600
756
|
if (dispatch) start(board, fleet, lanes, settings, report, { only });
|
|
757
|
+
if (dispatch && !only) { pursueGoals(board, fleet, settings, report); autoDecompose(board, lanes, settings, report); }
|
|
601
758
|
notify(board, fleet, report);
|
|
602
759
|
// what a Hermes gateway serves around its dispatcher and this one does not: said by the caller
|
|
603
760
|
report.chat_subscriptions = Number(board.db.prepare("SELECT COUNT(*) AS n FROM kanban_notify_subs WHERE platform != 'supercode'").get().n);
|
|
604
|
-
report.triage = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks WHERE status = 'triage'").get().n);
|
|
605
761
|
} catch (error) {
|
|
606
762
|
report.error = error.message;
|
|
607
763
|
} finally {
|
package/board/engine.mjs
CHANGED
|
@@ -105,7 +105,8 @@ function retryStatus(db, taskId) {
|
|
|
105
105
|
const row = one(db, 'SELECT current_run_id FROM tasks WHERE id = ?', taskId);
|
|
106
106
|
if (!row?.current_run_id) return 'ready';
|
|
107
107
|
const claimed = one(db, "SELECT payload FROM task_events WHERE task_id = ? AND run_id = ? AND kind = 'claimed' ORDER BY id DESC LIMIT 1", taskId, row.current_run_id);
|
|
108
|
-
|
|
108
|
+
const source = json(claimed?.payload, {})?.source_status;
|
|
109
|
+
return source === 'review' || source === 'triage' ? source : 'ready';
|
|
109
110
|
}
|
|
110
111
|
|
|
111
112
|
/** The phase a card blocked out of comes back to (Hermes reads it off the latest interruption). */
|
|
@@ -151,12 +152,13 @@ export function createCard(board, spec) {
|
|
|
151
152
|
while (one(db, 'SELECT 1 FROM tasks WHERE id = ?', id)) id = newTaskId();
|
|
152
153
|
const t = now();
|
|
153
154
|
run(db, `INSERT INTO tasks (id, title, body, assignee, status, priority, created_by, created_at, workspace_kind, workspace_path, branch_name,
|
|
154
|
-
tenant, idempotency_key, max_runtime_seconds, skills, max_retries, model_override, provider_override, reasoning_effort, session_id
|
|
155
|
-
|
|
155
|
+
tenant, idempotency_key, max_runtime_seconds, skills, max_retries, model_override, provider_override, reasoning_effort, session_id,
|
|
156
|
+
project_id, goal_mode, goal_max_turns)
|
|
157
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
156
158
|
id, title, spec.body ?? null, spec.assignee || null, status, Number(spec.priority) || 0, spec.createdBy ?? 'user', t, ws.kind, ws.path,
|
|
157
159
|
spec.branch ?? null, spec.tenant ?? null, spec.idempotencyKey ?? null, spec.maxRuntime ?? null,
|
|
158
160
|
spec.skills?.length ? JSON.stringify(spec.skills) : null, spec.maxRetries ?? null, spec.model ?? null, spec.provider ?? null,
|
|
159
|
-
spec.reasoning ?? null, spec.sessionId ?? null);
|
|
161
|
+
spec.reasoning ?? null, spec.sessionId ?? null, spec.project ?? null, spec.goal ? 1 : 0, spec.goal?.maxTurns ?? null);
|
|
160
162
|
const machine = spec.machine ?? arc?.machine ?? null;
|
|
161
163
|
if (machine || arc) run(db, 'INSERT OR REPLACE INTO supercode_cards (task_id, machine, subtask_of) VALUES (?, ?, ?)', id, machine, arc?.id ?? null);
|
|
162
164
|
for (const p of parents) run(db, 'INSERT OR IGNORE INTO task_links (parent_id, child_id) VALUES (?, ?)', p, id);
|
|
@@ -250,7 +252,8 @@ export function claim(board, id, { ttl, lock = claimer(), expectStatus = 'ready'
|
|
|
250
252
|
|
|
251
253
|
/** `claim` inside the caller's own transaction (a dispatcher's guard and claim see one state). */
|
|
252
254
|
export function claimIn(db, id, { ttl = claimTtl(), lock = claimer(), expectStatus = null } = {}) {
|
|
253
|
-
|
|
255
|
+
// a triage card is claimed to be specified, which does not wait on its parents
|
|
256
|
+
if (expectStatus !== 'triage' && !parentsDone(db, id)) {
|
|
254
257
|
run(db, "UPDATE tasks SET status = 'todo' WHERE id = ? AND status = 'ready'", id);
|
|
255
258
|
event(db, id, 'claim_rejected', { reason: 'parents_not_done' });
|
|
256
259
|
return null;
|
|
@@ -268,7 +271,7 @@ export function claimIn(db, id, { ttl = claimTtl(), lock = claimer(), expectStat
|
|
|
268
271
|
id, task.assignee, task.current_step_key, lock, expires, task.max_runtime_seconds, t);
|
|
269
272
|
const runId = Number(r.lastInsertRowid);
|
|
270
273
|
run(db, 'UPDATE tasks SET current_run_id = ? WHERE id = ?', runId, id);
|
|
271
|
-
event(db, id, 'claimed', { lock, expires, run_id: runId, source_status:
|
|
274
|
+
event(db, id, 'claimed', { lock, expires, run_id: runId, source_status: ['review', 'triage'].includes(task.status) ? task.status : 'ready' }, runId);
|
|
272
275
|
return runId;
|
|
273
276
|
}
|
|
274
277
|
|
|
@@ -323,6 +326,10 @@ export function complete(board, id, { result = null, summary = null, metadata =
|
|
|
323
326
|
if (runId === null && (said || metadata || task.status === 'review')) runId = synthesizeRun(db, id, { outcome: 'completed', summary: said ?? (task.status === 'review' ? 'Review approved without additional evidence.' : null), metadata });
|
|
324
327
|
const first = String(said ?? '').trim().split('\n')[0].slice(0, 400);
|
|
325
328
|
event(db, id, 'completed', { result_len: result ? result.length : 0, summary: first || null }, runId);
|
|
329
|
+
// card ids the handoff names that no card has (Hermes flags them the same way)
|
|
330
|
+
const named = [...new Set(`${summary ?? ''} ${result ?? ''}`.match(/\bt_[a-f0-9]{8,}\b/g) ?? [])];
|
|
331
|
+
const phantom = named.filter((t) => !one(db, 'SELECT 1 FROM tasks WHERE id = ?', t));
|
|
332
|
+
if (phantom.length) event(db, id, 'suspected_hallucinated_references', { phantom_refs: phantom, source: 'completion_summary' }, runId);
|
|
326
333
|
recomputeIn(db);
|
|
327
334
|
});
|
|
328
335
|
}
|
|
@@ -380,7 +387,9 @@ export function requestChanges(board, id, { reason = null, reviewer = null, impl
|
|
|
380
387
|
return tx(board, (db) => {
|
|
381
388
|
const task = must(db, id);
|
|
382
389
|
if (task.status !== 'review' && !(task.status === 'running' && retryStatus(db, id) === 'review')) throw new Error(`${id} is not in review`);
|
|
383
|
-
|
|
390
|
+
// back to the implementer the review request recorded, not to the reviewer now holding it
|
|
391
|
+
const asked = json(one(db, "SELECT payload FROM task_events WHERE task_id = ? AND kind = 'review_requested' ORDER BY id DESC LIMIT 1", id)?.payload, {}) ?? {};
|
|
392
|
+
const back = implementer ?? asked.implementer ?? task.assignee;
|
|
384
393
|
run(db, "UPDATE tasks SET status = ?, assignee = ?, claim_lock = NULL, claim_expires = NULL, worker_pid = NULL WHERE id = ?", parentsDone(db, id) ? 'ready' : 'todo', back, id);
|
|
385
394
|
const runId = endRun(db, id, { outcome: 'changes_requested', summary: reason });
|
|
386
395
|
event(db, id, 'changes_requested', { reason, reviewer, implementer: back }, runId);
|
|
@@ -545,3 +554,212 @@ export function cardContext(db, id) {
|
|
|
545
554
|
}
|
|
546
555
|
return out.join('\n');
|
|
547
556
|
}
|
|
557
|
+
|
|
558
|
+
// ------------------------------------------------------------------ the rest of Hermes's verbs
|
|
559
|
+
|
|
560
|
+
/** A triage card gets its spec (a tightened title and a body) and waits in todo for its parents, as Hermes's specifier leaves it. */
|
|
561
|
+
export function specify(board, id, { title = null, body = null, author = 'user' } = {}) {
|
|
562
|
+
return tx(board, (db) => {
|
|
563
|
+
const task = must(db, id);
|
|
564
|
+
if (task.status !== 'triage' && task.status !== 'running') throw new Error(`${id} is ${task.status}; only a triage card is specified`);
|
|
565
|
+
if (title && String(title).trim()) run(db, 'UPDATE tasks SET title = ? WHERE id = ?', String(title).trim(), id);
|
|
566
|
+
if (body !== null) run(db, 'UPDATE tasks SET body = ? WHERE id = ?', body, id);
|
|
567
|
+
run(db, "UPDATE tasks SET status = 'todo', claim_lock = NULL, claim_expires = NULL, worker_pid = NULL WHERE id = ?", id);
|
|
568
|
+
const runId = endRun(db, id, { outcome: 'specified', summary: 'specified' });
|
|
569
|
+
event(db, id, 'specified', { author, title_changed: Boolean(title), body_len: body ? body.length : 0 }, runId);
|
|
570
|
+
recomputeIn(db);
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* A triage card fans out into children it waits on (Hermes's decomposer: the root stays and is the child of every
|
|
576
|
+
* leaf, so it wakes when the graph is done). `children` are `{ title, body?, assignee? }`; answers their ids.
|
|
577
|
+
*/
|
|
578
|
+
export function decompose(board, id, children, { author = 'user' } = {}) {
|
|
579
|
+
if (!children.length) throw new Error('decompose names at least one child (--child "<title>[@<assignee>]")');
|
|
580
|
+
const task = tx(board, (db) => must(db, id));
|
|
581
|
+
if (task.status !== 'triage' && task.status !== 'running') throw new Error(`${id} is ${task.status}; only a triage card is decomposed`);
|
|
582
|
+
const ids = children.map((c) => createCard(board, {
|
|
583
|
+
title: c.title, body: c.body ?? null, assignee: c.assignee ?? task.assignee, machine: task.machine,
|
|
584
|
+
workspace: task.workspace_path ? `${task.workspace_kind}:${task.workspace_path}` : task.workspace_kind, tenant: task.tenant, createdBy: author,
|
|
585
|
+
}).id);
|
|
586
|
+
tx(board, (db) => {
|
|
587
|
+
for (const child of ids) run(db, 'INSERT OR IGNORE INTO task_links (parent_id, child_id) VALUES (?, ?)', child, id);
|
|
588
|
+
run(db, "UPDATE tasks SET status = 'todo', claim_lock = NULL, claim_expires = NULL, worker_pid = NULL WHERE id = ?", id);
|
|
589
|
+
const runId = endRun(db, id, { outcome: 'decomposed', summary: `decomposed into ${ids.join(', ')}` });
|
|
590
|
+
event(db, id, 'decomposed', { children: ids, author }, runId);
|
|
591
|
+
});
|
|
592
|
+
return ids;
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/** Hermes's swarm: a completed root as the shared blackboard, workers after it, a verifier after them all, a synthesizer last. */
|
|
596
|
+
export function swarm(board, { goal, workers, verifier, synthesizer, tenant = null, createdBy = 'swarm-orchestrator', priority = 0, idempotencyKey = null }) {
|
|
597
|
+
if (!String(goal ?? '').trim()) throw new Error('a swarm has a goal');
|
|
598
|
+
if (!workers.length) throw new Error('a swarm has at least one --worker PROFILE:TITLE[:SKILL,SKILL]');
|
|
599
|
+
const protocol = (root) => `\n\n## Swarm protocol\n- Swarm root / shared blackboard: \`${root}\`.\n- Read sibling/parent handoffs from the board's context before working.\n`
|
|
600
|
+
+ `- Put machine-readable facts in completion metadata.\n- Put cross-worker notes on the root card as comments.\n- Goal: ${goal.trim()}\n`;
|
|
601
|
+
const root = createCard(board, { title: `Swarm: ${goal.trim()}`.slice(0, 200), body: goal, tenant, createdBy, priority, idempotencyKey, initialStatus: 'blocked' });
|
|
602
|
+
if (root.existing) return { root_id: root.id, existing: true };
|
|
603
|
+
tx(board, (db) => {
|
|
604
|
+
run(db, "UPDATE tasks SET status = 'done', completed_at = ? WHERE id = ?", now(), root.id);
|
|
605
|
+
const runId = synthesizeRun(db, root.id, { outcome: 'completed', summary: 'swarm blackboard', metadata: { swarm: true } });
|
|
606
|
+
event(db, root.id, 'completed', { result_len: 0, summary: 'swarm blackboard' }, runId);
|
|
607
|
+
});
|
|
608
|
+
const workerIds = workers.map((w) => createCard(board, { title: w.title, body: `${w.title}${protocol(root.id)}`, assignee: w.profile, skills: w.skills, parents: [root.id], tenant, createdBy, priority }).id);
|
|
609
|
+
const verifierId = createCard(board, { title: 'Verify swarm outputs', body: `Verify the workers' outputs.${protocol(root.id)}`, assignee: verifier, parents: workerIds, tenant, createdBy, priority }).id;
|
|
610
|
+
const synthesizerId = createCard(board, { title: 'Synthesize swarm outputs', body: `Synthesize the verified outputs.${protocol(root.id)}`, assignee: synthesizer, parents: [verifierId], tenant, createdBy, priority }).id;
|
|
611
|
+
return { root_id: root.id, worker_ids: workerIds, verifier_id: verifierId, synthesizer_id: synthesizerId };
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/** Hermes's `edit`: correct a done card's result (and its completed run's summary and metadata). */
|
|
615
|
+
export function editResult(board, id, { result, summary = null, metadata = undefined }) {
|
|
616
|
+
return tx(board, (db) => {
|
|
617
|
+
const task = must(db, id);
|
|
618
|
+
if (task.status !== 'done') throw new Error(`${id} is ${task.status}; edit corrects a done card's result`);
|
|
619
|
+
run(db, 'UPDATE tasks SET result = ? WHERE id = ?', result, id);
|
|
620
|
+
const said = summary ?? result;
|
|
621
|
+
const last = one(db, "SELECT id FROM task_runs WHERE task_id = ? AND outcome = 'completed' ORDER BY COALESCE(ended_at, started_at, 0) DESC, id DESC LIMIT 1", id);
|
|
622
|
+
let runId = last ? Number(last.id) : null;
|
|
623
|
+
if (runId === null) runId = synthesizeRun(db, id, { outcome: 'completed', summary: said, metadata: metadata ?? null });
|
|
624
|
+
else {
|
|
625
|
+
run(db, 'UPDATE task_runs SET summary = ? WHERE id = ?', said, runId);
|
|
626
|
+
if (metadata !== undefined) run(db, 'UPDATE task_runs SET metadata = ? WHERE id = ?', JSON.stringify(metadata), runId);
|
|
627
|
+
}
|
|
628
|
+
event(db, id, 'edited', { fields: ['result', 'summary', ...(metadata !== undefined ? ['metadata'] : [])], result_len: result ? result.length : 0, summary: String(said ?? '').split('\n')[0].slice(0, 400) || null }, runId);
|
|
629
|
+
});
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
/** Send a card in review back to its implementer (ready, or todo while parents are open). */
|
|
633
|
+
export function reopenReview(board, id) {
|
|
634
|
+
return tx(board, (db) => {
|
|
635
|
+
const task = must(db, id);
|
|
636
|
+
if (task.status !== 'review') throw new Error(`${id} is ${task.status}, not in review`);
|
|
637
|
+
const handoff = json(one(db, "SELECT payload FROM task_events WHERE task_id = ? AND kind = 'review_requested' ORDER BY id DESC LIMIT 1", id)?.payload, {}) ?? {};
|
|
638
|
+
const status = parentsDone(db, id) ? 'ready' : 'todo';
|
|
639
|
+
const implementer = typeof handoff.implementer === 'string' && handoff.implementer.trim() ? handoff.implementer : null;
|
|
640
|
+
run(db, `UPDATE tasks SET status = ?, current_run_id = NULL, claim_lock = NULL, claim_expires = NULL, worker_pid = NULL${implementer ? ', assignee = ?' : ''} WHERE id = ?`, ...(implementer ? [status, implementer, id] : [status, id]));
|
|
641
|
+
event(db, id, 'review_reopened', { status, ...(implementer ? { implementer } : {}) });
|
|
642
|
+
return status;
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
/** A running card's worker says it is alive (Hermes's `heartbeat`); the dispatcher does not rely on it. */
|
|
647
|
+
export function heartbeat(board, id, note = null) {
|
|
648
|
+
return tx(board, (db) => {
|
|
649
|
+
const task = must(db, id);
|
|
650
|
+
if (task.status !== 'running') throw new Error(`${id} is ${task.status}, not running`);
|
|
651
|
+
const t = now();
|
|
652
|
+
run(db, 'UPDATE tasks SET last_heartbeat_at = ? WHERE id = ?', t, id);
|
|
653
|
+
if (task.current_run_id) run(db, 'UPDATE task_runs SET last_heartbeat_at = ? WHERE id = ?', t, task.current_run_id);
|
|
654
|
+
event(db, id, 'heartbeat', note ? { note } : null, task.current_run_id);
|
|
655
|
+
});
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
/** A card's model (and provider) for its next start, or cleared. */
|
|
659
|
+
export function setModel(board, id, model = null, provider = null) {
|
|
660
|
+
if (provider && !model) throw new Error('a provider needs a model');
|
|
661
|
+
return tx(board, (db) => {
|
|
662
|
+
must(db, id);
|
|
663
|
+
run(db, 'UPDATE tasks SET model_override = ?, provider_override = ? WHERE id = ?', model || null, provider || null, id);
|
|
664
|
+
event(db, id, 'model_override_set', { model: model || null, provider: provider || null });
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
export function addAttachment(board, id, { file, name, contentType = null, author = 'user', dir }) {
|
|
669
|
+
return tx(board, (db) => {
|
|
670
|
+
must(db, id);
|
|
671
|
+
const r = run(db, 'INSERT INTO task_attachments (task_id, filename, stored_path, content_type, size, uploaded_by, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)',
|
|
672
|
+
id, name, file.path, contentType, file.size, author, now());
|
|
673
|
+
event(db, id, 'attachment_added', { id: Number(r.lastInsertRowid), filename: name, size: file.size });
|
|
674
|
+
return Number(r.lastInsertRowid);
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
export const attachments = (db, id) => all(db, 'SELECT * FROM task_attachments WHERE task_id = ? ORDER BY id', id);
|
|
679
|
+
|
|
680
|
+
export function removeAttachment(board, attachmentId) {
|
|
681
|
+
return tx(board, (db) => {
|
|
682
|
+
const row = one(db, 'SELECT * FROM task_attachments WHERE id = ?', attachmentId);
|
|
683
|
+
if (!row) throw new Error(`no attachment ${attachmentId}`);
|
|
684
|
+
run(db, 'DELETE FROM task_attachments WHERE id = ?', attachmentId);
|
|
685
|
+
event(db, row.task_id, 'attachment_removed', { id: Number(attachmentId), filename: row.filename });
|
|
686
|
+
return row;
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
/** Per status, per assignee, and the oldest ready card's age (Hermes's `stats`). */
|
|
691
|
+
export function stats(db) {
|
|
692
|
+
const byStatus = {};
|
|
693
|
+
for (const r of all(db, "SELECT status, COUNT(*) AS n FROM tasks WHERE status != 'archived' GROUP BY status")) byStatus[r.status] = Number(r.n);
|
|
694
|
+
const byAssignee = {};
|
|
695
|
+
for (const r of all(db, "SELECT assignee, status, COUNT(*) AS n FROM tasks WHERE status != 'archived' AND assignee IS NOT NULL GROUP BY assignee, status")) (byAssignee[r.assignee] ??= {})[r.status] = Number(r.n);
|
|
696
|
+
const byMachine = {};
|
|
697
|
+
for (const r of all(db, "SELECT c.machine, t.status, COUNT(*) AS n FROM tasks t JOIN supercode_cards c ON c.task_id = t.id WHERE t.status != 'archived' AND c.machine IS NOT NULL GROUP BY c.machine, t.status")) (byMachine[r.machine] ??= {})[r.status] = Number(r.n);
|
|
698
|
+
const oldest = one(db, "SELECT MIN(created_at) AS ts FROM tasks WHERE status = 'ready'")?.ts;
|
|
699
|
+
const t = now();
|
|
700
|
+
return { by_status: byStatus, by_assignee: byAssignee, by_machine: byMachine, oldest_ready_age_seconds: oldest ? t - Number(oldest) : null, now: t };
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
export function runs(db, id, { stateType = null, stateName = null } = {}) {
|
|
704
|
+
must(db, id);
|
|
705
|
+
const where = stateType === 'status' ? ' AND status = ?' : stateType === 'outcome' ? ' AND outcome = ?' : '';
|
|
706
|
+
return all(db, `SELECT * FROM task_runs WHERE task_id = ?${where} ORDER BY id`, ...(where ? [id, stateName] : [id])).map((r) => ({ ...r, metadata: json(r.metadata, null) }));
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
/** Delete the thread events of done and archived cards older than `seconds` (Hermes's `gc` of events). */
|
|
710
|
+
export function gcEvents(board, seconds) {
|
|
711
|
+
return tx(board, (db) => Number(run(db, "DELETE FROM task_events WHERE created_at < ? AND task_id IN (SELECT id FROM tasks WHERE status IN ('done', 'archived'))", now() - seconds).changes));
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
// ------------------------------------------------------------------ diagnostics
|
|
715
|
+
|
|
716
|
+
const hours = (s) => s / 3600;
|
|
717
|
+
|
|
718
|
+
/**
|
|
719
|
+
* Distress signals on a card, computed on demand and never stored (Hermes's diagnostics, the same kinds where the
|
|
720
|
+
* same thing can go wrong, and supercode's own where only its sessions can): each `{ kind, severity, title, data,
|
|
721
|
+
* actions }`, worst first.
|
|
722
|
+
*/
|
|
723
|
+
export function diagnose(db, task, { failureLimit = DEFAULT_FAILURE_LIMIT, strandedSeconds = 30 * 60, blockedHours = 24, fleet = null } = {}) {
|
|
724
|
+
const out = [];
|
|
725
|
+
const t = now();
|
|
726
|
+
const events = all(db, 'SELECT kind, payload, created_at FROM task_events WHERE task_id = ? ORDER BY id', task.id);
|
|
727
|
+
const failures = Number(task.consecutive_failures || 0);
|
|
728
|
+
if (failures >= Math.max(1, (task.max_retries ?? failureLimit) - 1) && task.status !== 'done') {
|
|
729
|
+
out.push({ kind: 'repeated_failures', severity: task.status === 'blocked' ? 'error' : 'warning', title: `${failures} consecutive failures`,
|
|
730
|
+
data: { failures, last_error: task.last_failure_error }, actions: [`supercode workflow reclaim ${task.id}`, `supercode workflow reassign ${task.id} <profile>`] });
|
|
731
|
+
}
|
|
732
|
+
const phantom = [...events].reverse().find((e) => e.kind === 'suspected_hallucinated_references');
|
|
733
|
+
if (phantom && !events.some((e) => e.kind === 'edited' && e.created_at > phantom.created_at)) {
|
|
734
|
+
out.push({ kind: 'prose_phantom_refs', severity: 'warning', title: 'its completion summary names card ids no card has', data: json(phantom.payload, {}) ?? {}, actions: [`supercode workflow edit ${task.id} --result <corrected>`] });
|
|
735
|
+
}
|
|
736
|
+
const lost = events.filter((e) => e.kind === 'session_lost' && t - e.created_at < 24 * 3600).length;
|
|
737
|
+
if (lost >= 2) out.push({ kind: 'repeated_session_loss', severity: 'warning', title: `its session was lost ${lost}x in 24h`, data: { lost }, actions: [`supercode workflow move ${task.id} <machine>`] });
|
|
738
|
+
const crashes = events.filter((e) => e.kind === 'crashed' && t - e.created_at < 24 * 3600).length;
|
|
739
|
+
if (crashes >= 2) out.push({ kind: 'repeated_crashes', severity: 'error', title: `its worker crashed ${crashes}x in 24h`, data: { crashes }, actions: [`supercode workflow log ${task.id}`] });
|
|
740
|
+
if (task.status === 'blocked') {
|
|
741
|
+
const since = [...events].reverse().find((e) => e.kind === 'blocked' || e.kind === 'gave_up')?.created_at ?? task.created_at;
|
|
742
|
+
if (hours(t - since) >= blockedHours) out.push({ kind: 'stuck_in_blocked', severity: 'warning', title: `blocked for ${Math.floor(hours(t - since))}h`, data: { hours: Math.floor(hours(t - since)) }, actions: [`supercode workflow unblock ${task.id}`] });
|
|
743
|
+
const cycles = events.filter((e) => e.kind === 'unblocked' && t - e.created_at < 24 * 3600).length;
|
|
744
|
+
if (cycles >= 3) out.push({ kind: 'block_unblock_cycling', severity: 'warning', title: `block→unblock cycled ${cycles}x in 24h`, data: { cycles }, actions: [] });
|
|
745
|
+
}
|
|
746
|
+
if (task.status === 'ready' && task.assignee) {
|
|
747
|
+
const since = [...events].reverse().find((e) => ['created', 'promoted', 'unblocked', 'reclaimed', 'session_lost', 'promoted_manual'].includes(e.kind))?.created_at ?? task.created_at;
|
|
748
|
+
const age = t - since;
|
|
749
|
+
if (age >= strandedSeconds) {
|
|
750
|
+
const guard = json([...events].reverse().find((e) => e.kind === 'respawn_guarded')?.payload, {})?.reason;
|
|
751
|
+
out.push({ kind: 'stranded_in_ready', severity: age >= strandedSeconds * 6 ? 'critical' : age >= strandedSeconds * 2 ? 'error' : 'warning',
|
|
752
|
+
title: `ready for ${Math.floor(age / 60)}m with no session`, data: { age_seconds: age, ...(guard ? { waiting_because: guard } : {}) }, actions: [`supercode workflow reassign ${task.id} <profile>`] });
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
if (task.status === 'review') {
|
|
756
|
+
const children = all(db, 'SELECT child_id FROM task_links WHERE parent_id = ?', task.id).length;
|
|
757
|
+
if (children) out.push({ kind: 'review_dependency_deadlock', severity: 'warning', title: `review handoff holds ${children} dependent card(s)`, data: { children }, actions: [`supercode workflow complete ${task.id}`] });
|
|
758
|
+
}
|
|
759
|
+
if (task.status === 'running' && fleet?.ok) {
|
|
760
|
+
const session = lastSession(db, task.id);
|
|
761
|
+
if (session?.machine && !fleet.online.has(session.machine)) out.push({ kind: 'machine_offline', severity: 'warning', title: `its machine ${session.machine} is offline`, data: { machine: session.machine }, actions: [] });
|
|
762
|
+
}
|
|
763
|
+
const order = ['critical', 'error', 'warning'];
|
|
764
|
+
return out.sort((a, b) => order.indexOf(a.severity) - order.indexOf(b.severity));
|
|
765
|
+
}
|
package/board.mjs
CHANGED
|
@@ -54,7 +54,6 @@ export async function startBoard({ root, config = {}, log = () => {} }) {
|
|
|
54
54
|
for (const [task, why] of r.failed ?? []) log('warn', `board ${r.board}: ${task} did not start: ${why}`);
|
|
55
55
|
for (const [task, to, why] of r.undelivered ?? []) log('warn', `board ${r.board}: a notice for ${task} to ${to} was not delivered: ${why}`);
|
|
56
56
|
if (r.chat_subscriptions && !said.has(`subs:${r.board}`)) { said.add(`subs:${r.board}`); log('warn', `board ${r.board}: ${r.chat_subscriptions} subscription(s) name a chat platform, which Hermes's gateway notifier serves and this dispatcher does not (it tells supercode sessions)`); }
|
|
57
|
-
if (r.triage && !said.has(`triage:${r.board}`)) { said.add(`triage:${r.board}`); log('warn', `board ${r.board}: ${r.triage} triage card(s) wait for decomposition, which Hermes's gateway runs on its auxiliary model and this dispatcher does not`); }
|
|
58
57
|
if (r.nonspawnable?.length && !said.has(`lane:${r.board}`)) { said.add(`lane:${r.board}`); log('warn', `board ${r.board}: ${r.nonspawnable.length} card(s) name an assignee that is no profile of this home, and wait`); }
|
|
59
58
|
}
|
|
60
59
|
} finally {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-orchestrator",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.18",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
|
|
6
6
|
"exports": {
|