@volter/supercode-orchestrator 0.4.18 → 0.4.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapters/chat-sdk.mjs +9 -5
- package/board/cli.mjs +28 -4
- package/board/dispatch.mjs +260 -24
- package/board/engine.mjs +48 -2
- package/board/store.mjs +7 -2
- package/board.mjs +12 -2
- package/hermes-locks.mjs +9 -1
- package/loop.mjs +1 -1
- package/package.json +1 -1
package/adapters/chat-sdk.mjs
CHANGED
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
// leaves the IR's queue as the single one. `extra.concurrency` overrides it.
|
|
58
58
|
|
|
59
59
|
import { existsSync, readFileSync } from 'node:fs';
|
|
60
|
-
import { join } from 'node:path';
|
|
60
|
+
import { basename, join } from 'node:path';
|
|
61
61
|
import { parseDotenv } from '../orchestration.mjs';
|
|
62
62
|
import { TOKEN_SCOPES, acquireTokenLock, releaseTokenLock } from '../hermes-locks.mjs';
|
|
63
63
|
import { chunks } from './chunks.mjs';
|
|
@@ -438,15 +438,19 @@ export class ChatSdkAdapter {
|
|
|
438
438
|
// A platform that caps a message's length gets the text in parts, in order, as Hermes's own adapters send it
|
|
439
439
|
// (Discord refuses past 2000 characters, and a fire's report was cut off there). The first part is the handle.
|
|
440
440
|
const limit = MESSAGE_LIMIT[this.platform];
|
|
441
|
+
const parts = limit ? chunks(content.text ?? '', limit) : [content.text ?? ''];
|
|
442
|
+
// files ride with the last part, as the Chat SDK uploads them (`{ markdown, files }`): `{ path }` read here, or `{ data, filename }`
|
|
443
|
+
const files = (content.files ?? []).map((f) => ({ data: f.data ?? readFileSync(f.path), filename: f.filename ?? basename(f.path), ...(f.mime ? { mimeType: f.mime } : {}) }));
|
|
441
444
|
let first;
|
|
442
|
-
for (const [index, text] of
|
|
445
|
+
for (const [index, text] of parts.entries()) {
|
|
446
|
+
const message = files.length && index === parts.length - 1 ? { markdown: text, files } : text;
|
|
443
447
|
let sent;
|
|
444
448
|
if (thread) {
|
|
445
|
-
sent = content.reply_to && index === 0 ? await thread.reply(String(content.reply_to),
|
|
449
|
+
sent = content.reply_to && index === 0 ? await thread.reply(String(content.reply_to), message) : await thread.post(message);
|
|
446
450
|
} else if (threadId) {
|
|
447
|
-
sent = await this.sdkAdapter.postMessage(threadId,
|
|
451
|
+
sent = await this.sdkAdapter.postMessage(threadId, message);
|
|
448
452
|
} else {
|
|
449
|
-
sent = await this.sdkAdapter.postChannelMessage(this.channelId(String(target.chat_id)),
|
|
453
|
+
sent = await this.sdkAdapter.postChannelMessage(this.channelId(String(target.chat_id)), message);
|
|
450
454
|
}
|
|
451
455
|
first ??= sent;
|
|
452
456
|
}
|
package/board/cli.mjs
CHANGED
|
@@ -12,7 +12,7 @@ import { read } from '../orchestration.mjs';
|
|
|
12
12
|
import { holdDispatcherLock } from '../hermes-locks.mjs';
|
|
13
13
|
import { boardPath, DEFAULT_BOARD, event, json, listBoards, now, openBoard, tx } from './store.mjs';
|
|
14
14
|
import * as engine from './engine.mjs';
|
|
15
|
-
import { adoptSession, boardLabel, boardSettings, lanesOf, localMachine, observeFleet, startTriageSession, tick } from './dispatch.mjs';
|
|
15
|
+
import { adoptSession, boardLabel, boardSettings, lanesOf, localMachine, observeFleet, paused, startTriageSession, tick } from './dispatch.mjs';
|
|
16
16
|
|
|
17
17
|
const HELP = `supercode workflow <verb> [--root <home>] [--board <slug>] [--json]
|
|
18
18
|
|
|
@@ -34,6 +34,7 @@ Hermes Kanban's verbs, with their flags:
|
|
|
34
34
|
diagnostics [--task] [--severity] stats assignees log <id> [--tail N] tail <id> watch [--assignee] [--kinds]
|
|
35
35
|
notify-subscribe <id> --platform P --chat-id C [--thread-id] notify-list [<id>] notify-unsubscribe <id> --platform P --chat-id C
|
|
36
36
|
dispatch [--dry-run] [--max N] [--failure-limit N] daemon gc [--event-retention-days] [--log-retention-days] repair
|
|
37
|
+
pause [reason] · resume (Hermes's emergency stop, the ESTOP file: new starts stop, running sessions go on)
|
|
37
38
|
|
|
38
39
|
supercode's own:
|
|
39
40
|
create … --machine M --subtask-of ARC --notify ADDRESS|--no-notify --no-start
|
|
@@ -244,7 +245,7 @@ export async function runWorkflow(argv) {
|
|
|
244
245
|
}), true);
|
|
245
246
|
let answer = { id: created.id, existing: created.existing };
|
|
246
247
|
if (!args['no-start'] && !created.existing && !args['subtask-of']) {
|
|
247
|
-
const [report] = tick({ root, lanes, settings, only: created.id, slugs: [slug] });
|
|
248
|
+
const [report] = await tick({ root, lanes, settings, only: created.id, slugs: [slug] });
|
|
248
249
|
const started = report.started.find(([t]) => t === created.id);
|
|
249
250
|
const queued = report.queued.find(([t]) => t === created.id);
|
|
250
251
|
const failed = report.failed.find(([t]) => t === created.id);
|
|
@@ -626,7 +627,7 @@ export async function runWorkflow(argv) {
|
|
|
626
627
|
say(args, would.length ? would.map((w) => `would start ${w.id} as ${w.surface} for ${w.assignee}${w.machine ? ` on ${w.machine}` : ''}`).join('\n') : 'nothing would start', would);
|
|
627
628
|
return 0;
|
|
628
629
|
}
|
|
629
|
-
const reports = tick({ root, lanes, settings });
|
|
630
|
+
const reports = await tick({ root, lanes, settings });
|
|
630
631
|
if (args.json) say(args, '', reports);
|
|
631
632
|
else for (const r of reports) process.stdout.write(`${summary(r)}\n`);
|
|
632
633
|
return reports.some((r) => r.error) ? 1 : 0;
|
|
@@ -677,6 +678,18 @@ export async function runWorkflow(argv) {
|
|
|
677
678
|
return report.status === 'corrupt' ? 1 : 0;
|
|
678
679
|
}
|
|
679
680
|
|
|
681
|
+
case 'pause': {
|
|
682
|
+
// Hermes's emergency stop: new starts stop on every board of the home; running sessions go on
|
|
683
|
+
writeFileSync(join(root, 'ESTOP'), `${JSON.stringify({ engaged_at: new Date().toISOString(), reason: rest.join(' ') || null })}\n`);
|
|
684
|
+
say(args, `New starts paused (${join(root, 'ESTOP')}); \`supercode workflow resume\` lifts it`, { ok: true, paused: join(root, 'ESTOP') });
|
|
685
|
+
return 0;
|
|
686
|
+
}
|
|
687
|
+
case 'resume': {
|
|
688
|
+
const lifted = [join(root, 'ESTOP'), join(process.env.HOME ?? '', '.hermes', 'ESTOP')].filter((p) => existsSync(p));
|
|
689
|
+
for (const p of lifted) unlinkSync(p);
|
|
690
|
+
say(args, lifted.length ? `Resumed (removed ${lifted.join(', ')})` : 'Not paused', { ok: true, lifted });
|
|
691
|
+
return 0;
|
|
692
|
+
}
|
|
680
693
|
case 'adopt': {
|
|
681
694
|
need(2, 'a card id and a session');
|
|
682
695
|
const { settings } = await laneState(root);
|
|
@@ -909,6 +922,9 @@ function authorize(db, verb, rest, args, own) {
|
|
|
909
922
|
return;
|
|
910
923
|
}
|
|
911
924
|
if (verb === 'boards') { if (!['list', 'ls', 'show', 'current', undefined].includes(rest[0])) refuse('does not change the home\'s boards'); return; }
|
|
925
|
+
// verbs that act on the board rather than on a card are the manager's and the board's, never a card's session's
|
|
926
|
+
if (['pause', 'resume', 'swarm', 'dispatch', 'daemon', 'serve', 'gc', 'repair', 'adopt', 'close', 'move', 'reassign', 'assign', 'set-model', 'claim', 'reclaim', 'archive', 'promote', 'schedule', 'link', 'unlink', 'edit', 'reopen-review', 'attach-rm'].includes(verb)) refuse(`does not ${verb}; the fleet's manager does`);
|
|
927
|
+
if (args.all) refuse(`does not ${verb} every card; only its own`);
|
|
912
928
|
const card = engine.getTask(db, rest[0]);
|
|
913
929
|
if (!card) return; // the verb itself answers "no card"
|
|
914
930
|
const mine = card.id === own.id;
|
|
@@ -961,15 +977,23 @@ async function serve(root) {
|
|
|
961
977
|
process.on('SIGINT', stop);
|
|
962
978
|
const log = (text) => process.stdout.write(`${new Date().toISOString()} ${text}\n`);
|
|
963
979
|
log(`dispatching the boards of ${root} from ${localMachine()}`);
|
|
980
|
+
const stuck = {};
|
|
981
|
+
const warned = {};
|
|
982
|
+
let pausedSaid = false;
|
|
964
983
|
try {
|
|
965
984
|
while (!stopped) {
|
|
985
|
+
pausedSaid = Boolean(paused(root)) && pausedSaid;
|
|
966
986
|
let interval = 60;
|
|
967
987
|
try {
|
|
968
988
|
const { lanes, settings } = await laneState(root);
|
|
969
989
|
interval = settings.interval;
|
|
970
|
-
for (const r of tick({ root, lanes, settings })) {
|
|
990
|
+
for (const r of await tick({ root, lanes, settings })) {
|
|
971
991
|
const said = summary(r);
|
|
972
992
|
if (!said.endsWith('nothing to do')) log(said);
|
|
993
|
+
if (r.paused && !pausedSaid) { log(`new starts paused by ${r.paused} (\`supercode workflow resume\` lifts it)`); pausedSaid = true; }
|
|
994
|
+
// Hermes's stuck warning: ready work for six ticks running and nothing started
|
|
995
|
+
stuck[r.board] = r.ready_waiting && !r.started.length && !r.paused ? (stuck[r.board] ?? 0) + 1 : 0;
|
|
996
|
+
if (stuck[r.board] >= 6 && Date.now() - (warned[r.board] ?? 0) >= 300_000) { log(`board ${r.board} stuck: ${r.ready_waiting} ready card(s) and nothing started for ${stuck[r.board]} ticks; \`supercode workflow diagnostics\``); warned[r.board] = Date.now(); }
|
|
973
997
|
for (const [t, why] of r.failed) log(` ${t} did not start: ${why}`);
|
|
974
998
|
for (const [t, s, m] of r.started) log(` ${t} started ${s} (${m})`);
|
|
975
999
|
for (const [t, to, why] of r.undelivered) log(` notice for ${t} to ${to} not delivered: ${why}`);
|
package/board/dispatch.mjs
CHANGED
|
@@ -15,12 +15,13 @@
|
|
|
15
15
|
// subscriber's cursor moves only after its notice was delivered, so two dispatchers over one board never both
|
|
16
16
|
// launch and a crash between the two steps is answered by the next tick.
|
|
17
17
|
import { spawn as spawnChild, spawnSync } from 'node:child_process';
|
|
18
|
-
import { existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync } from 'node:fs';
|
|
18
|
+
import { existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
19
19
|
import { hostname, tmpdir } from 'node:os';
|
|
20
20
|
import { basename, dirname, join } from 'node:path';
|
|
21
21
|
import { randomUUID } from 'node:crypto';
|
|
22
22
|
import { fileURLToPath } from 'node:url';
|
|
23
|
-
import { event, json, listBoards, now, openBoard, tx } from './store.mjs';
|
|
23
|
+
import { boardPath, event, json, listBoards, now, openBoard, tx } from './store.mjs';
|
|
24
|
+
import { holdBoardTickLock } from '../hermes-locks.mjs';
|
|
24
25
|
import {
|
|
25
26
|
cardContext, claimIn, claimTtl, currentRun, DEFAULT_FAILURE_LIMIT, extendClaim, getTask, lastSession,
|
|
26
27
|
recomputeReady, recordFailure, release, setRunSession, subscribeIn, subscriptions,
|
|
@@ -110,6 +111,11 @@ export function boardSettings(kanban = {}) {
|
|
|
110
111
|
autoDecompose: kanban.auto_decompose !== false,
|
|
111
112
|
autoDecomposePerTick: pos(kanban.auto_decompose_per_tick) ?? 3,
|
|
112
113
|
goalMaxTurns: pos(kanban.goal_max_turns) ?? 20,
|
|
114
|
+
maxSpawn: pos(kanban.max_spawn),
|
|
115
|
+
reconcileOrphans: kanban.reconcile_orphans !== false,
|
|
116
|
+
logRotateBytes: pos(kanban.worker_log_rotate_bytes) ?? 2 * 1024 * 1024,
|
|
117
|
+
logBackups: Number.isInteger(Number(kanban.worker_log_backup_count)) && Number(kanban.worker_log_backup_count) >= 0 ? Number(kanban.worker_log_backup_count) : 1,
|
|
118
|
+
hooks: kanban.hooks && typeof kanban.hooks === 'object' ? kanban.hooks : {},
|
|
113
119
|
rateLimitCooldown: Number.isInteger(Number(process.env.HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS)) && process.env.HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS !== '' ? Number(process.env.HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS) : 300,
|
|
114
120
|
};
|
|
115
121
|
}
|
|
@@ -137,6 +143,14 @@ function supervise(board, fleet, settings, report) {
|
|
|
137
143
|
const runRow = currentRun(db, id);
|
|
138
144
|
const session = runRow?.metadata?.supercode ?? null;
|
|
139
145
|
const t = now();
|
|
146
|
+
// Hermes's orphan reconciliation: a running card whose claim bookkeeping is broken, with nothing live behind it
|
|
147
|
+
const live = session?.surface === 'headless' ? pidAlive(session.pid) : session?.session_id ? (fleet.ok ? sessionAlive(fleet, session) : true) : false;
|
|
148
|
+
if (settings.reconcileOrphans && (!task.claim_lock || !task.claim_expires) && !live) {
|
|
149
|
+
release(db, id, { outcome: 'reclaimed', error: 'orphaned running card (broken claim bookkeeping)', kind: 'reconciled', payload: { reason: 'orphaned_running', claim_lock: task.claim_lock, claim_expires: task.claim_expires } });
|
|
150
|
+
db.prepare('INSERT INTO task_comments (task_id, author, body, created_at) VALUES (?, ?, ?, ?)').run(id, 'dispatcher', 'Requeued: the card was running with no claim and nothing live behind it.', t);
|
|
151
|
+
report.reclaimed.push(id);
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
140
154
|
// a card past its max runtime is stopped and counted, as Hermes times its workers out
|
|
141
155
|
if (task.max_runtime_seconds && runRow && t - Number(runRow.started_at) > Number(task.max_runtime_seconds)) {
|
|
142
156
|
stop.push(session);
|
|
@@ -158,8 +172,20 @@ function supervise(board, fleet, settings, report) {
|
|
|
158
172
|
return;
|
|
159
173
|
}
|
|
160
174
|
if (t - Number(session.launched_at ?? 0) < 30) return;
|
|
161
|
-
|
|
162
|
-
|
|
175
|
+
// how the worker ended, from the exit code its wrapper wrote (Hermes reads its reaped children's)
|
|
176
|
+
const code = session.exit_file && existsSync(session.exit_file) ? Number(readFileSync(session.exit_file, 'utf8').trim()) : null;
|
|
177
|
+
if (code === 75) {
|
|
178
|
+
// a quota wall: back to ready with no failure counted; the respawn guard waits out the cooldown
|
|
179
|
+
release(db, id, { outcome: 'rate_limited', error: 'worker hit a rate limit (exit 75)', kind: 'rate_limited', payload: { worker_pid: session.pid, exit_code: 75 } });
|
|
180
|
+
report.crashed.push(id);
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
const clean = code === 0;
|
|
184
|
+
const why = clean ? `worker ${session.pid} exited cleanly (rc=0) without completing or blocking the card` : `worker ${session.pid} exited${code === null ? '' : ` with ${code}`} without completing the card`;
|
|
185
|
+
release(db, id, { outcome: 'crashed', error: why, kind: 'crashed', payload: { worker_pid: session.pid, exit_code: code, protocol_violation: clean } });
|
|
186
|
+
// a clean exit that left the card running is a protocol violation: Hermes trips the breaker on the first
|
|
187
|
+
if (clean) db.prepare('UPDATE tasks SET consecutive_failures = ? WHERE id = ?').run(Math.max(0, (task.max_retries ?? settings.failureLimit) - 1), id);
|
|
188
|
+
if (recordFailure(db, id, why, { failureLimit: settings.failureLimit })) report.auto_blocked.push(id);
|
|
163
189
|
report.crashed.push(id);
|
|
164
190
|
return;
|
|
165
191
|
}
|
|
@@ -202,7 +228,14 @@ function supervise(board, fleet, settings, report) {
|
|
|
202
228
|
|
|
203
229
|
function stopSession(session) {
|
|
204
230
|
if (!session) return;
|
|
205
|
-
if (session.surface === 'headless' && pidAlive(session.pid)) {
|
|
231
|
+
if (session.surface === 'headless' && pidAlive(session.pid)) {
|
|
232
|
+
// SIGTERM to the worker's process group, then SIGKILL if it is still there after a grace, as Hermes stops its own
|
|
233
|
+
const signal = (sig) => { try { process.kill(-session.pid, sig); } catch { try { process.kill(session.pid, sig); } catch { /* gone */ } } };
|
|
234
|
+
signal('SIGTERM');
|
|
235
|
+
for (let waited = 0; waited < 10 && pidAlive(session.pid); waited += 1) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 500);
|
|
236
|
+
if (pidAlive(session.pid)) signal('SIGKILL');
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
206
239
|
if (session.pane) sc(['teams', 'panes', 'close', session.pane, ...(session.machine ? ['--machine', session.machine] : [])], { timeout: 60_000 });
|
|
207
240
|
}
|
|
208
241
|
|
|
@@ -224,8 +257,8 @@ function machineLoad(db, machine, fleet) {
|
|
|
224
257
|
}
|
|
225
258
|
|
|
226
259
|
/** Why a card cannot start now, or null. */
|
|
227
|
-
function guard(db, task, lane, fleet, settings) {
|
|
228
|
-
if (
|
|
260
|
+
function guard(db, task, lane, fleet, settings, budget = null) {
|
|
261
|
+
if (budget && budget.left <= 0) return budget.why;
|
|
229
262
|
const cap = lane.capacity ?? settings.perProfile;
|
|
230
263
|
if (cap && running(db, 'AND t.assignee = ?', lane.name) >= cap) return `${lane.name} is at capacity (${cap}); the card starts when one of its sessions ends`;
|
|
231
264
|
if (lane.surface !== 'pane') return null;
|
|
@@ -235,7 +268,9 @@ function guard(db, task, lane, fleet, settings) {
|
|
|
235
268
|
if (mcap && machineLoad(db, machine, fleet) >= mcap) return `machine ${machine} holds ${mcap} of the board's sessions (its capacity); the card starts when one is closed`;
|
|
236
269
|
if (!fleet.ok) return `supercode could not read the fleet: ${fleet.error}`;
|
|
237
270
|
if (!fleet.online.has(machine)) return `machine ${machine} is offline; the card starts when it is back`;
|
|
238
|
-
|
|
271
|
+
// a card on this machine gets its workspace made here (scratch, dir or worktree, as Hermes makes it); another
|
|
272
|
+
// machine's card names its directory there
|
|
273
|
+
if (machine !== localMachine() && (task.workspace_kind !== 'dir' || !task.workspace_path)) return `a pane session on ${machine} starts in a directory there: the card's workspace is dir:<absolute path>, not ${task.workspace_kind}`;
|
|
239
274
|
if (lane.harness !== 'claude-code') return `a pane session's id is chosen at launch, which ${lane.harness} does not take; the lane's worker is claude-code`;
|
|
240
275
|
return null;
|
|
241
276
|
}
|
|
@@ -279,12 +314,39 @@ function noteGuard(db, id, reason) {
|
|
|
279
314
|
event(db, id, 'respawn_guarded', { reason });
|
|
280
315
|
}
|
|
281
316
|
|
|
282
|
-
|
|
283
|
-
|
|
317
|
+
/** Hermes's emergency stop: the `ESTOP` sentinel in the home or in `~/.hermes` stops new starts (running ones go on). */
|
|
318
|
+
export function paused(root) {
|
|
319
|
+
return [join(root, 'ESTOP'), join(process.env.HOME ?? '', '.hermes', 'ESTOP')].find((p) => existsSync(p)) ?? null;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* The tick's spawn budget, as Hermes counts it: `kanban.max_spawn` against this board's running cards, and
|
|
324
|
+
* `kanban.max_in_progress` against every board of the home (workers share one machine's memory).
|
|
325
|
+
*/
|
|
326
|
+
function spawnBudget(board, settings, hostRunning) {
|
|
327
|
+
const here = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks WHERE status = 'running'").get().n);
|
|
328
|
+
const limits = [];
|
|
329
|
+
if (settings.maxSpawn) limits.push([settings.maxSpawn - here, `the board runs ${settings.maxSpawn} cards at most (kanban.max_spawn)`]);
|
|
330
|
+
if (settings.maxInProgress) limits.push([settings.maxInProgress - hostRunning, `the home runs ${settings.maxInProgress} cards at most across its boards (kanban.max_in_progress)`]);
|
|
331
|
+
if (!limits.length) return null;
|
|
332
|
+
const [left, why] = limits.sort((a, b) => a[0] - b[0])[0];
|
|
333
|
+
return { left, why };
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function start(board, fleet, lanes, settings, report, { only = null, hostRunning = 0 } = {}) {
|
|
337
|
+
const stop = paused(board.root);
|
|
338
|
+
if (stop) { report.paused = stop; return; }
|
|
284
339
|
const pressure = memoryPressure();
|
|
285
340
|
let headlessStarted = 0;
|
|
286
|
-
const
|
|
287
|
-
|
|
341
|
+
const order = 'ORDER BY priority DESC, created_at ASC';
|
|
342
|
+
const ready = board.db.prepare(`SELECT id, status, assignee FROM tasks WHERE status = 'ready' AND claim_lock IS NULL ${order}`).all();
|
|
343
|
+
const review = settings.reviewDispatch ? board.db.prepare(`SELECT id, status, assignee FROM tasks WHERE status = 'review' AND claim_lock IS NULL ${order}`).all() : [];
|
|
344
|
+
const budget = spawnBudget(board, settings, hostRunning);
|
|
345
|
+
// Hermes's review reservation: while review work waits, one slot of the budget is held back from the ready lane
|
|
346
|
+
const reviewWaits = review.some((r) => lanes[r.assignee || settings.defaultAssignee]);
|
|
347
|
+
const readyBudget = budget && budget.left > 0 && reviewWaits ? { left: budget.left - 1, why: `${budget.why}; one slot is held for review` } : budget;
|
|
348
|
+
for (const row of [...ready, ...review]) {
|
|
349
|
+
const lanesBudget = row.status === 'review' ? budget : readyBudget;
|
|
288
350
|
if (only && row.id !== only) continue;
|
|
289
351
|
if (board.db.prepare('SELECT subtask_of FROM supercode_cards WHERE task_id = ?').get(row.id)?.subtask_of) continue; // worked by its arc's session
|
|
290
352
|
const assignee = row.assignee || settings.defaultAssignee;
|
|
@@ -294,7 +356,7 @@ function start(board, fleet, lanes, settings, report, { only = null } = {}) {
|
|
|
294
356
|
const claimed = tx(board, (db) => {
|
|
295
357
|
const task = getTask(db, row.id);
|
|
296
358
|
if (!task || task.status !== row.status || task.claim_lock) return null;
|
|
297
|
-
let why = guard(db, task, lane, fleet, settings);
|
|
359
|
+
let why = guard(db, task, lane, fleet, settings, lanesBudget);
|
|
298
360
|
if (!why && lane.surface !== 'pane') {
|
|
299
361
|
// a one-shot worker runs on this machine: its memory, and Hermes's respawn guard
|
|
300
362
|
if (pressure === 'critical' || (pressure === 'elevated' && headlessStarted >= 1)) why = `this machine's memory pressure is ${pressure}; the card starts when it eases`;
|
|
@@ -314,6 +376,8 @@ function start(board, fleet, lanes, settings, report, { only = null } = {}) {
|
|
|
314
376
|
? launchPane(board, claimed.task, claimed.runId, lane, fleet, { input: row.status === 'review' ? reviewAsk(board, claimed.task) : null })
|
|
315
377
|
: launchHeadless(board, claimed.task, claimed.runId, lane, settings);
|
|
316
378
|
if (lane.surface !== 'pane') headlessStarted += 1;
|
|
379
|
+
if (budget) budget.left -= 1;
|
|
380
|
+
if (readyBudget && readyBudget !== budget) readyBudget.left -= 1;
|
|
317
381
|
report.started.push([row.id, session.address ?? `pid ${session.pid}`, session.mode]);
|
|
318
382
|
} catch (error) {
|
|
319
383
|
tx(board, (db) => {
|
|
@@ -385,6 +449,11 @@ function opening(board, task, label) {
|
|
|
385
449
|
|
|
386
450
|
function launchPane(board, task, runId, lane, fleet, { input = null } = {}) {
|
|
387
451
|
const machine = normalMachine(task.machine);
|
|
452
|
+
if (machine === localMachine() && !(task.workspace_kind === 'dir' && task.workspace_path && existsSync(task.workspace_path))) {
|
|
453
|
+
const made = resolveLocalWorkspace(board, task);
|
|
454
|
+
tx(board, (db) => db.prepare('UPDATE tasks SET workspace_path = ? WHERE id = ?').run(made, task.id));
|
|
455
|
+
task = { ...task, workspace_path: made };
|
|
456
|
+
}
|
|
388
457
|
const model = task.model_override ?? lane.model;
|
|
389
458
|
const tail = model ? ['--model', model] : [];
|
|
390
459
|
const base = ['open', '--new', 'claude', '--detach', '--on', task.machine, '--cwd', task.workspace_path];
|
|
@@ -447,10 +516,16 @@ function launchHeadless(board, task, runId, lane, settings) {
|
|
|
447
516
|
...(task.reasoning_effort ? ['--reasoning', task.reasoning_effort] : []), 'chat', '-q', `work kanban task ${task.id}`];
|
|
448
517
|
const logDir = join(dirname(board.path), board.slug === 'default' ? 'kanban' : '', 'logs');
|
|
449
518
|
mkdirSync(logDir, { recursive: true });
|
|
450
|
-
const
|
|
451
|
-
|
|
519
|
+
const logFile = join(logDir, `${task.id}.log`);
|
|
520
|
+
rotateLog(logFile, settings.logRotateBytes, settings.logBackups);
|
|
521
|
+
const out = openSync(logFile, 'a');
|
|
522
|
+
const exitFile = join(logDir, `${task.id}.r${runId}.exit`);
|
|
523
|
+
// the worker runs under a shell that records its exit code, so a later tick can tell a clean exit from a crash
|
|
524
|
+
const child = spawnChild('/bin/sh', ['-c', '"$0" "$@"; echo $? > "$SUPERCODE_BOARD_EXIT_FILE"', process.execPath, WORKER, ...args], {
|
|
452
525
|
cwd: workspace, detached: true, stdio: ['ignore', out, out],
|
|
453
526
|
env: {
|
|
527
|
+
SUPERCODE_BOARD_EXIT_FILE: exitFile,
|
|
528
|
+
...(task.goal_mode ? { HERMES_KANBAN_GOAL_MODE: '1', ...(task.goal_max_turns ? { HERMES_KANBAN_GOAL_MAX_TURNS: String(task.goal_max_turns) } : {}) } : {}),
|
|
454
529
|
...process.env, HERMES_HOME: lane.dir, HERMES_PROFILE: lane.name, HERMES_KANBAN_TASK: task.id, HERMES_KANBAN_WORKSPACE: workspace,
|
|
455
530
|
HERMES_KANBAN_RUN_ID: String(runId), HERMES_KANBAN_DB: board.path, HERMES_KANBAN_BOARD: board.slug, HERMES_SESSION_SOURCE: 'kanban',
|
|
456
531
|
HERMES_KANBAN_CLAIM_LOCK: `${hostname()}:${process.pid}`, ...(task.tenant ? { HERMES_TENANT: task.tenant } : {}),
|
|
@@ -458,7 +533,7 @@ function launchHeadless(board, task, runId, lane, settings) {
|
|
|
458
533
|
},
|
|
459
534
|
});
|
|
460
535
|
child.unref();
|
|
461
|
-
const session = { surface: 'headless', pid: child.pid, machine:
|
|
536
|
+
const session = { surface: 'headless', pid: child.pid, machine: localMachine(), mode: 'fresh', launched_at: now(), exit_file: exitFile };
|
|
462
537
|
tx(board, (db) => {
|
|
463
538
|
db.prepare('UPDATE tasks SET worker_pid = ? WHERE id = ?').run(child.pid, task.id);
|
|
464
539
|
db.prepare('UPDATE task_runs SET worker_pid = ? WHERE id = ?').run(child.pid, runId);
|
|
@@ -468,6 +543,16 @@ function launchHeadless(board, task, runId, lane, settings) {
|
|
|
468
543
|
return session;
|
|
469
544
|
}
|
|
470
545
|
|
|
546
|
+
/** Hermes's worker log rotation: past `max` bytes the log moves to `.1` (… `.n`), the oldest dropped. */
|
|
547
|
+
function rotateLog(file, max, backups) {
|
|
548
|
+
try {
|
|
549
|
+
if (!existsSync(file) || statSync(file).size <= max) return;
|
|
550
|
+
if (backups === 0) { unlinkSync(file); return; }
|
|
551
|
+
for (let n = backups - 1; n >= 1; n -= 1) if (existsSync(`${file}.${n}`)) renameSync(`${file}.${n}`, `${file}.${n + 1}`);
|
|
552
|
+
renameSync(file, `${file}.1`);
|
|
553
|
+
} catch { /* a log that cannot rotate keeps growing */ }
|
|
554
|
+
}
|
|
555
|
+
|
|
471
556
|
/** A headless card's directory on this machine, made as Hermes makes it (scratch, dir, or a git worktree). */
|
|
472
557
|
function resolveLocalWorkspace(board, task) {
|
|
473
558
|
const root = board.root;
|
|
@@ -583,6 +668,9 @@ function send(from, label, to, body, key) {
|
|
|
583
668
|
/** The board's own address: where its notices come from and where card updates are mailed. */
|
|
584
669
|
export const boardAddress = (label) => `sc:${localMachine()}:board:${label.replace(/[^A-Za-z0-9_.-]/g, '-')}`;
|
|
585
670
|
|
|
671
|
+
/** Consecutive failed sends after which a subscription is dropped (Hermes's notifier drops a dead chat after 12). */
|
|
672
|
+
const MAX_SEND_FAILURES = 12;
|
|
673
|
+
|
|
586
674
|
function notify(board, fleet, report) {
|
|
587
675
|
const from = boardAddress(board.label);
|
|
588
676
|
for (const sub of subscriptions(board.db)) {
|
|
@@ -599,6 +687,7 @@ function notify(board, fleet, report) {
|
|
|
599
687
|
if (!events.length) continue;
|
|
600
688
|
const task = tx(board, (db) => getTask(db, sub.task_id));
|
|
601
689
|
let cursor = Number(sub.last_event_id);
|
|
690
|
+
let failed = null;
|
|
602
691
|
try {
|
|
603
692
|
for (const ev of events) {
|
|
604
693
|
const p = json(ev.payload, {}) ?? {};
|
|
@@ -615,16 +704,135 @@ function notify(board, fleet, report) {
|
|
|
615
704
|
cursor = ev.id;
|
|
616
705
|
}
|
|
617
706
|
} catch (error) {
|
|
618
|
-
|
|
707
|
+
failed = String(error.message).slice(-200);
|
|
708
|
+
report.undelivered.push([sub.task_id, sub.chat_id, failed]);
|
|
619
709
|
}
|
|
620
710
|
// the cursor moves past what was delivered (a failed delivery is retried next tick; the mail id dedupes)
|
|
621
711
|
tx(board, (db) => {
|
|
622
|
-
|
|
623
|
-
|
|
712
|
+
const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ?";
|
|
713
|
+
db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(cursor, sub.task_id, sub.chat_id, cursor);
|
|
714
|
+
const failures = failed ? Number(sub.delivery_metadata?.failures ?? 0) + 1 : 0;
|
|
715
|
+
if (failures >= MAX_SEND_FAILURES) { db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(sub.task_id, sub.chat_id); return; }
|
|
716
|
+
if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures }), sub.task_id, sub.chat_id);
|
|
717
|
+
if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(sub.task_id, sub.chat_id);
|
|
624
718
|
});
|
|
625
719
|
}
|
|
626
720
|
}
|
|
627
721
|
|
|
722
|
+
// Hermes's terminal kinds for a chat subscriber, in its order of silence: archived and unblocked move the cursor and say nothing
|
|
723
|
+
const CHAT_KINDS = ['completed', 'blocked', 'gave_up', 'crashed', 'timed_out', 'status', 'archived', 'unblocked', 'block_loop_detected', 'review_requested', 'changes_requested'];
|
|
724
|
+
|
|
725
|
+
/**
|
|
726
|
+
* A chat-platform subscription's events, sent through the orchestrator's own channel adapters (the daemon hosts them;
|
|
727
|
+
* `serve` hosts none, as a Hermes CLI without its gateway delivers none). Answers what was sent and what failed.
|
|
728
|
+
*/
|
|
729
|
+
export async function notifyChats(root, adapters, log = () => {}) {
|
|
730
|
+
const sent = [];
|
|
731
|
+
for (const slug of listBoards(root)) {
|
|
732
|
+
const board = openBoard(root, slug);
|
|
733
|
+
board.label = boardLabel(root, slug);
|
|
734
|
+
try {
|
|
735
|
+
for (const sub of subscriptions(board.db)) {
|
|
736
|
+
const adapter = adapters?.[sub.platform];
|
|
737
|
+
if (sub.platform === 'supercode' || !adapter) continue;
|
|
738
|
+
const events = board.db.prepare(`SELECT id, kind, payload FROM task_events WHERE task_id = ? AND id > ? AND kind IN (${CHAT_KINDS.map(() => '?').join(', ')}) ORDER BY id`).all(sub.task_id, sub.last_event_id, ...CHAT_KINDS);
|
|
739
|
+
if (!events.length) continue;
|
|
740
|
+
const task = tx(board, (db) => getTask(db, sub.task_id));
|
|
741
|
+
let cursor = Number(sub.last_event_id);
|
|
742
|
+
let failed = false;
|
|
743
|
+
for (const ev of events) {
|
|
744
|
+
const payload = json(ev.payload, {}) ?? {};
|
|
745
|
+
const text = ['archived', 'unblocked'].includes(ev.kind) ? null : notice(ev.kind, task, payload, 'creator');
|
|
746
|
+
if (text) {
|
|
747
|
+
// a completed card's deliverables go with it as uploads, as Hermes's notifier sends them
|
|
748
|
+
const files = ev.kind === 'completed' ? artifactsOf(payload, task).map((path) => ({ path })) : [];
|
|
749
|
+
try {
|
|
750
|
+
await adapter.send({ kind: 'explicit', platform: sub.platform, chat_id: sub.chat_id, ...(sub.thread_id ? { thread_id: sub.thread_id } : {}) }, { text: `${slug !== 'default' ? `[${slug}] ` : ''}${text}`, ...(files.length ? { files } : {}) });
|
|
751
|
+
sent.push([sub.task_id, ev.kind, `${sub.platform}:${sub.chat_id}`]);
|
|
752
|
+
} catch (error) {
|
|
753
|
+
failed = true;
|
|
754
|
+
log('warn', `board ${slug}: a notice for ${sub.task_id} to ${sub.platform}:${sub.chat_id} failed: ${error.message}`);
|
|
755
|
+
break;
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
cursor = ev.id;
|
|
759
|
+
}
|
|
760
|
+
tx(board, (db) => {
|
|
761
|
+
const where = 'WHERE task_id = ? AND platform = ? AND chat_id = ? AND thread_id = ?';
|
|
762
|
+
const key = [sub.task_id, sub.platform, sub.chat_id, sub.thread_id ?? ''];
|
|
763
|
+
db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(cursor, ...key, cursor);
|
|
764
|
+
const failures = failed ? Number(sub.delivery_metadata?.failures ?? 0) + 1 : 0;
|
|
765
|
+
if (failures >= MAX_SEND_FAILURES) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...key);
|
|
766
|
+
else if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures }), ...key);
|
|
767
|
+
if (task?.status === 'archived' && !failed) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...key);
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
} finally {
|
|
771
|
+
board.close();
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
return sent;
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
/** A completed card's artifact files: its completion's `artifacts`, then absolute paths its summary or result names that exist. */
|
|
778
|
+
function artifactsOf(payload, task) {
|
|
779
|
+
const seen = new Set();
|
|
780
|
+
const out = [];
|
|
781
|
+
const add = (p) => {
|
|
782
|
+
const path = String(p ?? '').replace(/^~(?=\/)/, process.env.HOME ?? '~');
|
|
783
|
+
if (!path.startsWith('/') || seen.has(path)) return;
|
|
784
|
+
try { if (!statSync(path).isFile()) return; } catch { return; }
|
|
785
|
+
seen.add(path);
|
|
786
|
+
out.push(path);
|
|
787
|
+
};
|
|
788
|
+
for (const p of Array.isArray(payload.artifacts) ? payload.artifacts : []) add(p);
|
|
789
|
+
for (const text of [payload.summary, task?.result]) for (const m of String(text ?? '').matchAll(/(?:^|\s)(~?\/[^\s'"`]+\.[A-Za-z0-9]{1,8})/g)) add(m[1]);
|
|
790
|
+
return out;
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
// ------------------------------------------------------------------ hooks
|
|
794
|
+
|
|
795
|
+
// Hermes's lifecycle hooks, from the board's own events (so a change any writer made fires them, Hermes's CLI's too)
|
|
796
|
+
const HOOK_OF = {
|
|
797
|
+
claimed: 'kanban_task_claimed', completed: 'kanban_task_completed', blocked: 'kanban_task_blocked',
|
|
798
|
+
spawned: 'on_kanban_worker_spawned', session_started: 'on_kanban_worker_spawned', session_resumed: 'on_kanban_worker_spawned', session_replaced: 'on_kanban_worker_spawned', session_adopted: 'on_kanban_worker_spawned',
|
|
799
|
+
crashed: 'on_kanban_worker_exited', rate_limited: 'on_kanban_worker_exited', session_lost: 'on_kanban_worker_exited', timed_out: 'on_kanban_worker_exited',
|
|
800
|
+
stale: 'on_kanban_worker_stale_claim', reclaimed: 'on_kanban_worker_stale_claim',
|
|
801
|
+
assigned: 'on_kanban_task_updated', edited: 'on_kanban_task_updated', model_override_set: 'on_kanban_task_updated', machine_set: 'on_kanban_task_updated',
|
|
802
|
+
linked: 'on_kanban_task_updated', unlinked: 'on_kanban_task_updated', unblocked: 'on_kanban_task_updated', promoted: 'on_kanban_task_updated',
|
|
803
|
+
archived: 'on_kanban_task_updated', review_requested: 'on_kanban_task_updated', changes_requested: 'on_kanban_task_updated',
|
|
804
|
+
};
|
|
805
|
+
|
|
806
|
+
/**
|
|
807
|
+
* `kanban.hooks.<hook>: <command>` (or a list): each command runs with the event as one JSON object on its stdin, and
|
|
808
|
+
* nothing waits for it. Hermes runs Python plugins at these points; supercode runs any program.
|
|
809
|
+
*/
|
|
810
|
+
function fireHooks(board, settings, report) {
|
|
811
|
+
const hooks = settings.hooks ?? {};
|
|
812
|
+
if (!Object.keys(hooks).length) return;
|
|
813
|
+
const run = (hook, payload) => {
|
|
814
|
+
for (const command of [hooks[hook]].flat().filter((c) => typeof c === 'string' && c.trim())) {
|
|
815
|
+
const child = spawnChild('/bin/sh', ['-c', command], { detached: true, stdio: ['pipe', 'ignore', 'ignore'], cwd: board.root });
|
|
816
|
+
child.on('error', () => {});
|
|
817
|
+
child.stdin.on('error', () => {});
|
|
818
|
+
child.stdin.end(JSON.stringify({ hook, board: board.slug, ...payload }));
|
|
819
|
+
child.unref();
|
|
820
|
+
}
|
|
821
|
+
};
|
|
822
|
+
tx(board, (db) => {
|
|
823
|
+
db.exec('CREATE TABLE IF NOT EXISTS supercode_board_state (key TEXT PRIMARY KEY, value TEXT)');
|
|
824
|
+
const max = Number(db.prepare('SELECT COALESCE(MAX(id), 0) AS m FROM task_events').get().m);
|
|
825
|
+
const row = db.prepare("SELECT value FROM supercode_board_state WHERE key = 'hooks_cursor'").get();
|
|
826
|
+
const cursor = row ? Number(row.value) : max; // the first tick starts at the present, never replays history
|
|
827
|
+
for (const ev of db.prepare('SELECT e.id, e.task_id, e.run_id, e.kind, e.payload, e.created_at, t.assignee FROM task_events e LEFT JOIN tasks t ON t.id = e.task_id WHERE e.id > ? ORDER BY e.id').all(cursor)) {
|
|
828
|
+
const hook = HOOK_OF[ev.kind];
|
|
829
|
+
if (hook && hooks[hook]) run(hook, { task_id: ev.task_id, run_id: ev.run_id, event: ev.kind, assignee: ev.assignee, payload: json(ev.payload, null), at: ev.created_at });
|
|
830
|
+
}
|
|
831
|
+
db.prepare("INSERT INTO supercode_board_state (key, value) VALUES ('hooks_cursor', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(String(max));
|
|
832
|
+
});
|
|
833
|
+
if (hooks.on_kanban_dispatch_tick) run('on_kanban_dispatch_tick', { started: report.started.length, reclaimed: report.reclaimed.length, crashed: report.crashed.length, timed_out: report.timed_out.length, promoted: report.promoted });
|
|
834
|
+
}
|
|
835
|
+
|
|
628
836
|
// ------------------------------------------------------------------ the board's mailbox
|
|
629
837
|
|
|
630
838
|
const ENTRY = fileURLToPath(new URL('../bin/orchestrator.mjs', import.meta.url));
|
|
@@ -740,12 +948,32 @@ export function boardLabel(root, slug) {
|
|
|
740
948
|
* One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
|
|
741
949
|
* `only` limits starting to one card (a create that answers whether its card started). Answers one report per board.
|
|
742
950
|
*/
|
|
743
|
-
|
|
951
|
+
/** A board file's fingerprint: a corrupt board is skipped until the file changes (Hermes retries it then). */
|
|
952
|
+
const fingerprint = (path) => { try { const st = statSync(path); return `${st.size}:${st.mtimeMs}`; } catch { return 'missing'; } };
|
|
953
|
+
const CORRUPT = /malformed|not a database|corrupt|disk image/i;
|
|
954
|
+
|
|
955
|
+
/**
|
|
956
|
+
* One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
|
|
957
|
+
* `only` limits starting to one card (a create that answers whether its card started). Each board is ticked under its
|
|
958
|
+
* per-tick lock (Hermes's `.dispatch.lock`), skipped when another dispatcher holds it. Answers one report per board.
|
|
959
|
+
*/
|
|
960
|
+
export async function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true }) {
|
|
744
961
|
const needFleet = Object.values(lanes).some((l) => l.surface === 'pane');
|
|
745
|
-
const fleet = needFleet ? observeFleet() : { ok: false, alive: new Set(), online: new Set(), error: 'no pane lanes' };
|
|
962
|
+
const fleet = needFleet ? observeFleet() : { ok: false, alive: new Set(), online: new Set(), sessions: [], error: 'no pane lanes' };
|
|
963
|
+
const boards = slugs ?? listBoards(root);
|
|
964
|
+
// the running cards of every board of the home, for kanban.max_in_progress (host-wide, as Hermes counts it)
|
|
965
|
+
let hostRunning = 0;
|
|
966
|
+
for (const slug of boards) {
|
|
967
|
+
try { const b = openBoard(root, slug); try { hostRunning += Number(b.db.prepare("SELECT COUNT(*) AS n FROM tasks WHERE status = 'running'").get().n); } finally { b.close(); } } catch { /* its own tick reports it */ }
|
|
968
|
+
}
|
|
746
969
|
const out = [];
|
|
747
|
-
for (const slug of
|
|
970
|
+
for (const slug of boards) {
|
|
748
971
|
const report = { board: slug, mailed: [], started: [], queued: [], failed: [], lost: [], crashed: [], timed_out: [], reclaimed: [], auto_blocked: [], promoted: 0, unassigned: [], nonspawnable: [], notified: [], undelivered: [], fleet: fleet.ok ? 'read' : fleet.error };
|
|
972
|
+
const path = boardPath(root, slug);
|
|
973
|
+
const marker = `${path}.corrupt-seen`;
|
|
974
|
+
if (existsSync(marker) && readFileSync(marker, 'utf8').trim() === fingerprint(path)) { report.error = 'skipped: the board file was corrupt and has not changed since (`supercode workflow repair`)'; out.push(report); continue; }
|
|
975
|
+
let lock = null;
|
|
976
|
+
try { lock = await holdBoardTickLock(path); } catch (error) { report.skipped_locked = error.message; out.push(report); continue; }
|
|
749
977
|
let board;
|
|
750
978
|
try {
|
|
751
979
|
board = openBoard(root, slug);
|
|
@@ -753,15 +981,23 @@ export function tick({ root, lanes, settings, only = null, slugs = null, dispatc
|
|
|
753
981
|
readMailbox(board, boardAddress(board.label), report);
|
|
754
982
|
supervise(board, fleet, settings, report);
|
|
755
983
|
report.promoted = recomputeReady(board, settings.failureLimit);
|
|
756
|
-
|
|
757
|
-
if (dispatch
|
|
984
|
+
const before = report.started.length;
|
|
985
|
+
if (dispatch) start(board, fleet, lanes, settings, report, { only, hostRunning });
|
|
986
|
+
hostRunning += report.started.length - before;
|
|
987
|
+
if (dispatch && !only && !report.paused) { pursueGoals(board, fleet, settings, report); autoDecompose(board, lanes, settings, report); }
|
|
758
988
|
notify(board, fleet, report);
|
|
989
|
+
fireHooks(board, settings, report);
|
|
990
|
+
if (String(board.db.prepare('PRAGMA journal_mode').get()?.journal_mode).toLowerCase() === 'wal') board.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get();
|
|
991
|
+
report.ready_waiting = Number(board.db.prepare("SELECT COUNT(*) AS n FROM tasks t LEFT JOIN supercode_cards c ON c.task_id = t.id WHERE t.status = 'ready' AND t.assignee IS NOT NULL AND c.subtask_of IS NULL").get().n);
|
|
759
992
|
// what a Hermes gateway serves around its dispatcher and this one does not: said by the caller
|
|
760
993
|
report.chat_subscriptions = Number(board.db.prepare("SELECT COUNT(*) AS n FROM kanban_notify_subs WHERE platform != 'supercode'").get().n);
|
|
994
|
+
if (existsSync(marker)) unlinkSync(marker);
|
|
761
995
|
} catch (error) {
|
|
762
996
|
report.error = error.message;
|
|
997
|
+
if (CORRUPT.test(error.message)) writeFileSync(marker, fingerprint(path));
|
|
763
998
|
} finally {
|
|
764
999
|
board?.close();
|
|
1000
|
+
lock.release();
|
|
765
1001
|
}
|
|
766
1002
|
out.push(report);
|
|
767
1003
|
}
|
package/board/engine.mjs
CHANGED
|
@@ -7,6 +7,10 @@
|
|
|
7
7
|
//
|
|
8
8
|
// Each function takes an open board (`store.mjs`) and runs in one write transaction. Nothing here launches,
|
|
9
9
|
// observes or sends: the dispatcher (`dispatch.mjs`) does, and records what it did through these.
|
|
10
|
+
import { spawnSync } from 'node:child_process';
|
|
11
|
+
import { existsSync, rmSync } from 'node:fs';
|
|
12
|
+
import { join, resolve } from 'node:path';
|
|
13
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
10
14
|
import { claimer, event, json, newTaskId, now, tx } from './store.mjs';
|
|
11
15
|
|
|
12
16
|
export const STATUSES = ['triage', 'todo', 'scheduled', 'ready', 'running', 'blocked', 'review', 'done', 'archived'];
|
|
@@ -125,6 +129,17 @@ function resumeStatus(db, taskId) {
|
|
|
125
129
|
* machine its session runs on. Answers its id; with an `idempotencyKey` a live card already filed under it is
|
|
126
130
|
* answered instead.
|
|
127
131
|
*/
|
|
132
|
+
/** A project in the home's `projects.db` (Hermes's registry), by id or slug; null when there is none. */
|
|
133
|
+
function projectOf(root, idOrSlug) {
|
|
134
|
+
const file = join(root, 'projects.db');
|
|
135
|
+
if (!existsSync(file)) return null;
|
|
136
|
+
const db = new DatabaseSync(file, { readOnly: true });
|
|
137
|
+
try {
|
|
138
|
+
return db.prepare('SELECT id, slug, name, primary_path FROM projects WHERE id = ?').get(idOrSlug)
|
|
139
|
+
?? db.prepare('SELECT id, slug, name, primary_path FROM projects WHERE slug = ?').get(String(idOrSlug).toLowerCase()) ?? null;
|
|
140
|
+
} catch { return null; } finally { db.close(); }
|
|
141
|
+
}
|
|
142
|
+
|
|
128
143
|
export function createCard(board, spec) {
|
|
129
144
|
const title = String(spec.title ?? '').trim();
|
|
130
145
|
if (!title) throw new Error('a card has a title');
|
|
@@ -150,6 +165,17 @@ export function createCard(board, spec) {
|
|
|
150
165
|
else if (parents.some((p) => !['done', 'archived'].includes(one(db, 'SELECT status FROM tasks WHERE id = ?', p).status))) status = 'todo';
|
|
151
166
|
let id = newTaskId();
|
|
152
167
|
while (one(db, 'SELECT 1 FROM tasks WHERE id = ?', id)) id = newTaskId();
|
|
168
|
+
// a project (Hermes's projects registry, `projects.db` in the home) anchors the card's worktree in its repository
|
|
169
|
+
const project = spec.project ? projectOf(board.root, spec.project) : null;
|
|
170
|
+
if (project) {
|
|
171
|
+
if (ws.kind === 'scratch') ws.kind = 'worktree';
|
|
172
|
+
if (ws.kind === 'worktree' && !ws.path && project.primary_path) ws.path = join(project.primary_path, '.worktrees', id);
|
|
173
|
+
if (ws.kind === 'worktree' && !spec.branch) {
|
|
174
|
+
const t = title.toLowerCase().replace(/[^a-z0-9._-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40).replace(/-+$/, '');
|
|
175
|
+
spec = { ...spec, branch: `${project.slug}/${id}${t ? `-${t}` : ''}` };
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
spec = { ...spec, project: project?.id ?? null };
|
|
153
179
|
const t = now();
|
|
154
180
|
run(db, `INSERT INTO tasks (id, title, body, assignee, status, priority, created_by, created_at, workspace_kind, workspace_path, branch_name,
|
|
155
181
|
tenant, idempotency_key, max_runtime_seconds, skills, max_retries, model_override, provider_override, reasoning_effort, session_id,
|
|
@@ -314,7 +340,26 @@ export function recordFailure(db, id, error, { failureLimit = DEFAULT_FAILURE_LI
|
|
|
314
340
|
return true;
|
|
315
341
|
}
|
|
316
342
|
|
|
317
|
-
export function complete(board, id,
|
|
343
|
+
export function complete(board, id, opts = {}) {
|
|
344
|
+
completeIn(board, id, opts);
|
|
345
|
+
cleanupWorkspace(board, id);
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
/**
|
|
349
|
+
* A completed card's workspace goes, as Hermes cleans up on completion: a scratch directory under the board's own
|
|
350
|
+
* workspaces, or a worktree (its branch stays); never while a child still waits on it, never a `dir`.
|
|
351
|
+
*/
|
|
352
|
+
function cleanupWorkspace(board, id) {
|
|
353
|
+
const task = tx(board, (db) => getTask(db, id));
|
|
354
|
+
if (!task?.workspace_path || !['scratch', 'worktree'].includes(task.workspace_kind)) return;
|
|
355
|
+
const waiting = tx(board, (db) => one(db, "SELECT 1 FROM task_links l JOIN tasks t ON t.id = l.child_id WHERE l.parent_id = ? AND t.status NOT IN ('done', 'archived') LIMIT 1", id));
|
|
356
|
+
if (waiting || !existsSync(task.workspace_path)) return;
|
|
357
|
+
if (task.workspace_kind === 'worktree') { spawnSync('git', ['-C', task.workspace_path, 'worktree', 'remove', '--force', task.workspace_path], { encoding: 'utf8' }); return; }
|
|
358
|
+
const managed = resolve(board.slug === 'default' ? join(board.root, 'kanban', 'workspaces') : join(board.root, 'kanban', 'boards', board.slug, 'workspaces'));
|
|
359
|
+
if (resolve(task.workspace_path).startsWith(`${managed}/`)) rmSync(task.workspace_path, { recursive: true, force: true });
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function completeIn(board, id, { result = null, summary = null, metadata = null } = {}) {
|
|
318
363
|
return tx(board, (db) => {
|
|
319
364
|
const task = must(db, id);
|
|
320
365
|
if (!parentsDone(db, id)) throw new Error(`${id} waits on parents that are not done`);
|
|
@@ -325,7 +370,8 @@ export function complete(board, id, { result = null, summary = null, metadata =
|
|
|
325
370
|
let runId = endRun(db, id, { outcome: 'completed', status: 'done', summary: said, metadata: metadata ?? undefined });
|
|
326
371
|
if (runId === null && (said || metadata || task.status === 'review')) runId = synthesizeRun(db, id, { outcome: 'completed', summary: said ?? (task.status === 'review' ? 'Review approved without additional evidence.' : null), metadata });
|
|
327
372
|
const first = String(said ?? '').trim().split('\n')[0].slice(0, 400);
|
|
328
|
-
|
|
373
|
+
const artifacts = Array.isArray(metadata?.artifacts) ? metadata.artifacts.filter((a) => typeof a === 'string' && a.trim()) : [];
|
|
374
|
+
event(db, id, 'completed', { result_len: result ? result.length : 0, summary: first || null, ...(artifacts.length ? { artifacts } : {}) }, runId);
|
|
329
375
|
// card ids the handoff names that no card has (Hermes flags them the same way)
|
|
330
376
|
const named = [...new Set(`${summary ?? ''} ${result ?? ''}`.match(/\bt_[a-f0-9]{8,}\b/g) ?? [])];
|
|
331
377
|
const phantom = named.filter((t) => !one(db, 'SELECT 1 FROM tasks WHERE id = ?', t));
|
package/board/store.mjs
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// `engine.mjs`. Hermes's own tools may write the same file at the same time: every write takes SQLite's writer
|
|
10
10
|
// lock (`BEGIN IMMEDIATE`) and retries while the other holds it.
|
|
11
11
|
import { randomBytes } from 'node:crypto';
|
|
12
|
-
import { existsSync, mkdirSync, readdirSync } from 'node:fs';
|
|
12
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync } from 'node:fs';
|
|
13
13
|
import { hostname } from 'node:os';
|
|
14
14
|
import { dirname, join } from 'node:path';
|
|
15
15
|
import { DatabaseSync } from 'node:sqlite';
|
|
@@ -78,7 +78,12 @@ export function listBoards(root) {
|
|
|
78
78
|
const out = existsSync(boardPath(root)) ? [DEFAULT_BOARD] : [];
|
|
79
79
|
const named = join(root, 'kanban', 'boards');
|
|
80
80
|
if (existsSync(named)) {
|
|
81
|
-
for (const slug of readdirSync(named))
|
|
81
|
+
for (const slug of readdirSync(named)) {
|
|
82
|
+
if (slug.startsWith('_') || !existsSync(boardPath(root, slug))) continue;
|
|
83
|
+
// a board archived in its metadata is not dispatched (Hermes lists boards without the archived)
|
|
84
|
+
try { if (JSON.parse(readFileSync(join(named, slug, 'board.json'), 'utf8'))?.archived) continue; } catch { /* no metadata: live */ }
|
|
85
|
+
out.push(slug);
|
|
86
|
+
}
|
|
82
87
|
}
|
|
83
88
|
return out;
|
|
84
89
|
}
|
package/board.mjs
CHANGED
|
@@ -11,6 +11,7 @@ import { spawn } from 'node:child_process';
|
|
|
11
11
|
import { fileURLToPath } from 'node:url';
|
|
12
12
|
import { holdDispatcherLock } from './hermes-locks.mjs';
|
|
13
13
|
import { summary } from './board/cli.mjs';
|
|
14
|
+
import { notifyChats } from './board/dispatch.mjs';
|
|
14
15
|
|
|
15
16
|
const ENTRY = fileURLToPath(new URL('./bin/orchestrator.mjs', import.meta.url));
|
|
16
17
|
|
|
@@ -30,7 +31,7 @@ function dispatchOnce(root) {
|
|
|
30
31
|
}
|
|
31
32
|
|
|
32
33
|
/** Start the board's dispatcher for the home at `root`; answers `{ tick, stop }`, or null when it does not dispatch. */
|
|
33
|
-
export async function startBoard({ root, config = {}, log = () => {} }) {
|
|
34
|
+
export async function startBoard({ root, config = {}, log = () => {}, adapters = {} }) {
|
|
34
35
|
const kanban = config.kanban ?? {};
|
|
35
36
|
const off = String(process.env.HERMES_KANBAN_DISPATCH_IN_GATEWAY ?? '').trim().toLowerCase();
|
|
36
37
|
if (['0', 'false', 'no', 'off'].includes(off)) { log('info', 'board: not dispatched (HERMES_KANBAN_DISPATCH_IN_GATEWAY)'); return null; }
|
|
@@ -42,18 +43,27 @@ export async function startBoard({ root, config = {}, log = () => {} }) {
|
|
|
42
43
|
let running = false;
|
|
43
44
|
let stopped = false;
|
|
44
45
|
const said = new Set();
|
|
46
|
+
const stuck = {};
|
|
47
|
+
const warned = {};
|
|
45
48
|
const tick = async () => {
|
|
46
49
|
if (running || stopped) return;
|
|
47
50
|
running = true;
|
|
48
51
|
try {
|
|
49
52
|
const { reports, error } = await dispatchOnce(root);
|
|
50
53
|
if (error) { log('warn', `board: tick failed: ${error}`); return; }
|
|
54
|
+
// chat-platform subscriptions are delivered here, through the daemon's own channel adapters
|
|
55
|
+
try { await notifyChats(root, adapters, log); } catch (error) { log('warn', `board: chat notices failed: ${error.message}`); }
|
|
51
56
|
for (const r of reports) {
|
|
52
57
|
const line = summary(r);
|
|
58
|
+
stuck[r.board] = r.ready_waiting && !r.started?.length && !r.paused ? (stuck[r.board] ?? 0) + 1 : 0;
|
|
59
|
+
if (stuck[r.board] >= 6 && Date.now() - (warned[r.board] ?? 0) >= 300_000) { log('warn', `board ${r.board} stuck: ${r.ready_waiting} ready card(s) and nothing started for ${stuck[r.board]} ticks`); warned[r.board] = Date.now(); }
|
|
60
|
+
if (r.paused && !said.has(`paused:${r.board}`)) { said.add(`paused:${r.board}`); log('info', `board ${r.board}: new starts paused by ${r.paused}`); }
|
|
61
|
+
if (!r.paused) said.delete(`paused:${r.board}`);
|
|
53
62
|
if (!line.endsWith('nothing to do')) log(r.error ? 'warn' : 'info', line);
|
|
54
63
|
for (const [task, why] of r.failed ?? []) log('warn', `board ${r.board}: ${task} did not start: ${why}`);
|
|
55
64
|
for (const [task, to, why] of r.undelivered ?? []) log('warn', `board ${r.board}: a notice for ${task} to ${to} was not delivered: ${why}`);
|
|
56
|
-
|
|
65
|
+
const unserved = r.chat_subscriptions && !Object.keys(adapters).length;
|
|
66
|
+
if (unserved && !said.has(`subs:${r.board}`)) { said.add(`subs:${r.board}`); log('warn', `board ${r.board}: ${r.chat_subscriptions} subscription(s) name a chat platform and this daemon hosts no channel adapter to tell them`); }
|
|
57
67
|
if (r.nonspawnable?.length && !said.has(`lane:${r.board}`)) { said.add(`lane:${r.board}`); log('warn', `board ${r.board}: ${r.nonspawnable.length} card(s) name an assignee that is no profile of this home, and wait`); }
|
|
58
68
|
}
|
|
59
69
|
} finally {
|
package/hermes-locks.mjs
CHANGED
|
@@ -13,7 +13,7 @@ import { spawn, spawnSync } from 'node:child_process';
|
|
|
13
13
|
import { createHash } from 'node:crypto';
|
|
14
14
|
import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, unlinkSync, writeSync } from 'node:fs';
|
|
15
15
|
import { homedir } from 'node:os';
|
|
16
|
-
import { join, resolve } from 'node:path';
|
|
16
|
+
import { dirname, join, resolve } from 'node:path';
|
|
17
17
|
|
|
18
18
|
/** Hermes's gateway kind, the `kind` of every record it writes. */
|
|
19
19
|
export const GATEWAY_KIND = 'hermes-gateway';
|
|
@@ -154,6 +154,14 @@ export function holdDispatcherLock(root, { python = 'python3' } = {}) {
|
|
|
154
154
|
return holdFileLock(path, root, { python, what: `the board at ${resolve(root)} is dispatched by`, tail: 'one dispatcher serves a board' });
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
+
/**
|
|
158
|
+
* Hold one board's per-tick lock, `<kanban.db>.dispatch.lock` (Hermes's `_dispatch_tick_lock`), for one tick: a
|
|
159
|
+
* one-shot `dispatch` never ticks a board beside another dispatcher. Rejects, naming the holder, when it is held.
|
|
160
|
+
*/
|
|
161
|
+
export function holdBoardTickLock(dbPath, { python = 'python3' } = {}) {
|
|
162
|
+
return holdFileLock(`${dbPath}.dispatch.lock`, dirname(dbPath), { python, what: `the board ${dbPath} is being ticked by`, tail: 'one tick at a time' });
|
|
163
|
+
}
|
|
164
|
+
|
|
157
165
|
function holdFileLock(path, home, { python, what, tail }) {
|
|
158
166
|
return new Promise((done, fail) => {
|
|
159
167
|
let child;
|
package/loop.mjs
CHANGED
|
@@ -85,7 +85,7 @@ export class Orchestrator {
|
|
|
85
85
|
// reject unhandled, which Node treats as fatal and which stopped the daemon
|
|
86
86
|
if (this.autoTick) { this.timer = setInterval(() => { this.tick().catch(() => undefined); }, TICK_MS); this.timer.unref?.(); }
|
|
87
87
|
// the home's board, dispatched as a Hermes gateway dispatches it (docs/plans/hermes-compat.md row 14)
|
|
88
|
-
if (this.autoTick) this.board = await startBoard({ root: this.state.root, config: this.state.profiles.default?.residue?.config ?? {}, log: this.log });
|
|
88
|
+
if (this.autoTick) this.board = await startBoard({ root: this.state.root, config: this.state.profiles.default?.residue?.config ?? {}, log: this.log, adapters: this.adapters });
|
|
89
89
|
}
|
|
90
90
|
|
|
91
91
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-orchestrator",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.19",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
|
|
6
6
|
"exports": {
|