@volter/supercode-orchestrator 0.5.37 → 0.5.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents.mjs +36 -7
- package/bin/machine-fleet.mjs +29 -0
- package/board/cli.mjs +12 -2
- package/board/dispatch.mjs +260 -108
- package/board/engine.mjs +103 -32
- package/board/machine-fleet.mjs +49 -0
- package/board/request-mail.mjs +3 -1
- package/board/store.mjs +5 -0
- package/board/wake.mjs +2 -4
- package/board.mjs +3 -1
- package/loop.mjs +5 -4
- package/package.json +1 -1
- package/board/event-fleet.mjs +0 -66
package/board/engine.mjs
CHANGED
|
@@ -463,41 +463,105 @@ export async function goto(board, id, status, { roles = null, payload = {} } = {
|
|
|
463
463
|
return { ...(await act(board, id, name, { roles, payload })), event: name };
|
|
464
464
|
}
|
|
465
465
|
|
|
466
|
-
|
|
467
|
-
|
|
466
|
+
// A delivery (a notice, an actor message, an agent declaration) is tried at most EFFECT_ATTEMPTS times, waiting twice as
|
|
467
|
+
// long after each failed attempt (from 30 s, at most 30 min). Its last failed attempt is its terminal state: the row
|
|
468
|
+
// keeps `failed_at` and the error, the card records `effect_failed` once, diagnose shows it, and nothing tries it again.
|
|
469
|
+
// A board-state effect the dispatcher owns (OWNED: merging the reviewed PRs, cleaning a workspace) is never abandoned:
|
|
470
|
+
// it keeps trying at the capped wait, and at EFFECT_ATTEMPTS failed attempts its card records `effect_stalled` and the
|
|
471
|
+
// managers are mailed once, so a passed card whose merge keeps failing is seen, and still heals when the cause clears.
|
|
472
|
+
export const EFFECT_ATTEMPTS = 6;
|
|
473
|
+
const OWNED = new Set(['merge_reviewed', 'cleanup_workspace']);
|
|
474
|
+
const effectWait = (attempts) => Math.min(1800, 30 * 2 ** Math.max(0, attempts - 1));
|
|
475
|
+
// A delivery pass claims a row by compare-and-set on its next_at (set ahead only while the row is due) before trying it,
|
|
476
|
+
// so two passes (serve's, the hosted loop's, a person's `workflow deliver`, a verb's own) never try one effect at once.
|
|
477
|
+
// The claim outlasts any one attempt: each effect's I/O is bounded by its own timeouts (a send's 180 s, a merge's
|
|
478
|
+
// MERGE_BUDGET plus one in-flight GitHub call), all well inside EFFECT_CLAIM.
|
|
479
|
+
const EFFECT_CLAIM = 900;
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* The effects a committed move asked for. Board writes the round owns (invalidating descendants, asking a session's
|
|
483
|
+
* close, stopping a local worker) are made here, in the round; everything that reaches outside the board is recorded
|
|
484
|
+
* to run after it, in the delivery pass, never inside a dispatch round.
|
|
485
|
+
*/
|
|
486
|
+
export function queueEffects(board, effects = []) {
|
|
487
|
+
// a board message is its own effect row (dispatch.mjs queueBoardMail), so the move's mail hint is not recorded
|
|
488
|
+
const fresh = effects.filter((e) => !e.durableId && e.kind !== 'run' && !(e.kind === 'board_message' && !e.event));
|
|
489
|
+
const now_ = fresh.filter((e) => ['invalidate_descendants', 'close_session', 'run_ended', 'stop_run'].includes(e.kind));
|
|
490
|
+
const later = fresh.filter((e) => !now_.includes(e));
|
|
491
|
+
for (const e of now_) {
|
|
492
|
+
if (e.kind === 'invalidate_descendants') tx(board, (db) => invalidateDescendants(db, board, e.id));
|
|
493
|
+
else if (e.kind === 'close_session') tx(board, (db) => requestSessionClose(db, e.id, { slot: e.slot, through: e.runId }));
|
|
494
|
+
else if (e.kind === 'run_ended') { if (e.outcome === 'reclaimed') stopWorker(board, e.runId); tx(board, (db) => requestSessionClose(db, e.id, { runId: e.runId })); }
|
|
495
|
+
else if (e.kind === 'stop_run') stopWorker(board, e.runId);
|
|
496
|
+
}
|
|
497
|
+
if (later.length) tx(board, (db) => { for (const e of later) db.prepare('INSERT INTO supercode_effects (effect) VALUES (?)').run(JSON.stringify(e)); });
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
/** The I/O a committed move asked for: the move's own `effects`, and with `pending` every recorded effect that is due. */
|
|
501
|
+
export async function runEffects(board, effects = [], { pending: due = false } = {}) {
|
|
468
502
|
const requested = new Set(effects.map((e) => e.durableId).filter(Boolean));
|
|
469
|
-
const
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
const
|
|
503
|
+
const t = now();
|
|
504
|
+
const rows = board.db.prepare('SELECT id, effect, attempts, next_at FROM supercode_effects WHERE failed_at IS NULL ORDER BY id').all()
|
|
505
|
+
.filter((p) => (requested.has(p.id) || due) && Number(p.next_at ?? 0) <= t);
|
|
506
|
+
const durable = new Set(rows.map((p) => p.id));
|
|
507
|
+
const attempts = new Map(rows.map((p) => [p.id, Number(p.attempts ?? 0)]));
|
|
508
|
+
const combined = [...rows.map((p) => ({ ...json(p.effect, {}), durableId: p.id })), ...effects.filter((e) => !e.durableId || (!durable.has(e.durableId) && !requested.has(e.durableId)))];
|
|
473
509
|
combined.sort((a,b) => Number(b.kind === 'session_notice') - Number(a.kind === 'session_notice'));
|
|
510
|
+
// what this call did, for its caller's report: effect ids done, tried and kept, and skipped as another's claim
|
|
511
|
+
const did = { done: [], kept: [], claimed_elsewhere: [] };
|
|
474
512
|
for (const e of combined) {
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
}
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
const
|
|
490
|
-
if (
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
if (
|
|
498
|
-
|
|
499
|
-
|
|
513
|
+
// the claim: taken only while the row is due and not failed; another pass holding it answers no change
|
|
514
|
+
if (e.durableId && !tx(board, (db) => db.prepare('UPDATE supercode_effects SET next_at = ? WHERE id = ? AND failed_at IS NULL AND next_at <= ?').run(now() + EFFECT_CLAIM, e.durableId, now()).changes)) { did.claimed_elsewhere.push(e.durableId); continue; }
|
|
515
|
+
let done, error = null;
|
|
516
|
+
try { done = await runEffect(board, e); } catch (thrown) { done = false; error = String(thrown?.message ?? thrown).slice(0, 500); }
|
|
517
|
+
if (!e.durableId) continue;
|
|
518
|
+
// a mail inside its batch window is not an attempt: it is looked at again in a few seconds
|
|
519
|
+
if (done === 'wait') { tx(board, (db) => db.prepare('UPDATE supercode_effects SET next_at = ? WHERE id = ?').run(now() + 5, e.durableId)); did.kept.push(e.durableId); continue; }
|
|
520
|
+
if (done) { tx(board, (db) => db.prepare('DELETE FROM supercode_effects WHERE id = ?').run(e.durableId)); did.done.push(`${e.durableId}:${e.kind}`); continue; }
|
|
521
|
+
did.kept.push(e.durableId);
|
|
522
|
+
const made = (attempts.get(e.durableId) ?? 0) + 1;
|
|
523
|
+
tx(board, (db) => {
|
|
524
|
+
const row = db.prepare('SELECT failed_at, last_error FROM supercode_effects WHERE id = ?').get(e.durableId);
|
|
525
|
+
if (!row || row.failed_at) return;
|
|
526
|
+
const reason = error ?? row.last_error ?? `${e.kind} was not accepted`;
|
|
527
|
+
const card = e.id && db.prepare('SELECT 1 FROM tasks WHERE id = ?').get(e.id) ? e.id : null;
|
|
528
|
+
if (OWNED.has(e.kind)) {
|
|
529
|
+
db.prepare('UPDATE supercode_effects SET attempts = ?, next_at = ?, last_error = ? WHERE id = ?').run(made, now() + effectWait(made), reason, e.durableId);
|
|
530
|
+
if (made === EFFECT_ATTEMPTS && card) {
|
|
531
|
+
event(db, card, 'effect_stalled', { effect: e.durableId, kind: e.kind, attempts: made, error: reason, retry_every_seconds: effectWait(made) }, e.runId ?? null);
|
|
532
|
+
const to = [].concat(workflowFor(board.root).params?.managers ?? []).find(Boolean);
|
|
533
|
+
if (to) event(db, card, 'board_message', { to, text: `${card}: the board's ${e.kind.replace('_', ' ')} has failed ${made} times and keeps trying every ${effectWait(made) / 60} min: ${reason}` }, null);
|
|
534
|
+
}
|
|
535
|
+
} else if (made >= EFFECT_ATTEMPTS) {
|
|
536
|
+
db.prepare('UPDATE supercode_effects SET attempts = ?, failed_at = ?, last_error = ? WHERE id = ?').run(made, now(), reason, e.durableId);
|
|
537
|
+
if (card) event(db, card, 'effect_failed', { effect: e.durableId, kind: e.kind, attempts: made, error: reason, address: e.address ?? null }, e.runId ?? null);
|
|
538
|
+
if (card && e.kind === 'board_message' && e.event) event(db, card, 'board_message_abandoned', { event: e.event, attempts: made, reason });
|
|
539
|
+
} else db.prepare('UPDATE supercode_effects SET attempts = ?, next_at = ?, last_error = ? WHERE id = ?').run(made, now() + effectWait(made), reason, e.durableId);
|
|
540
|
+
});
|
|
500
541
|
}
|
|
542
|
+
return did;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
/** One effect's I/O: true when it is done, false when it is not done yet (the caller counts the attempt). */
|
|
546
|
+
async function runEffect(board, e) {
|
|
547
|
+
if (e.kind === 'session_notice') { const { sessionNotice } = await import('./dispatch.mjs'); return await sessionNotice(board, e) !== false; }
|
|
548
|
+
if (e.kind === 'cleanup_workspace') { cleanupWorkspace(board, e.id); return true; }
|
|
549
|
+
if (e.kind === 'declare_card_agent') { const { declareCardAgentEffect } = await import('./agent-declarations.mjs'); return declareCardAgentEffect(board, e) !== false; }
|
|
550
|
+
if (e.kind === 'board_message') {
|
|
551
|
+
const { sendBoardMessage, sendBoardMessages } = await import('./dispatch.mjs');
|
|
552
|
+
// a verb's own ask sends its card's mail at once; the mail's own effect row then finds it sent
|
|
553
|
+
if (!e.event) { await sendBoardMessages(board, { undelivered: [] }, e.id); return true; }
|
|
554
|
+
return await sendBoardMessage(board, e);
|
|
555
|
+
}
|
|
556
|
+
if (e.kind === 'merge_reviewed') { const { mergeReviewed } = await import('./dispatch.mjs'); return await mergeReviewed(board, e) !== false; }
|
|
557
|
+
if (e.kind === 'invalidate_descendants') { tx(board, (db) => invalidateDescendants(db, board, e.id)); return true; }
|
|
558
|
+
if (e.kind === 'stop_run') { stopWorker(board, e.runId); return true; }
|
|
559
|
+
if (e.kind === 'close_session' || e.kind === 'send_to_actor' || e.kind === 'run_ended') {
|
|
560
|
+
if (e.kind === 'run_ended' && e.outcome === 'reclaimed') stopWorker(board, e.runId);
|
|
561
|
+
const { sessionEffect } = await import('./dispatch.mjs');
|
|
562
|
+
return await sessionEffect(board, e) !== false;
|
|
563
|
+
}
|
|
564
|
+
return true;
|
|
501
565
|
}
|
|
502
566
|
|
|
503
567
|
// a run taken back stops its one-shot worker on this host (Hermes's _terminate_reclaimed_worker); a pane session is
|
|
@@ -901,7 +965,14 @@ export function diagnose(db, task, { failureLimit = DEFAULT_FAILURE_LIMIT, stran
|
|
|
901
965
|
}
|
|
902
966
|
if (task.status === 'running' && fleet?.ok) {
|
|
903
967
|
const session = lastSession(db, task.id);
|
|
904
|
-
if (session?.machine && !fleet.online.has(session.machine)) out.push({ kind: 'machine_offline', severity: 'warning', title: `its machine ${session.machine} is offline`, data: { machine: session.machine }, actions: [] });
|
|
968
|
+
if (session?.machine && !fleet.online.has(session.machine) && !fleet.unknown?.has(session.machine)) out.push({ kind: 'machine_offline', severity: 'warning', title: `its machine ${session.machine} is offline`, data: { machine: session.machine }, actions: [] });
|
|
969
|
+
}
|
|
970
|
+
// what the board could not carry out for the card: a delivery failed for good, or a board effect that keeps failing
|
|
971
|
+
for (const row of all(db, "SELECT id, effect, attempts, failed_at, last_error FROM supercode_effects WHERE json_extract(effect, '$.id') = ? AND (failed_at IS NOT NULL OR attempts >= ?)", task.id, EFFECT_ATTEMPTS)) {
|
|
972
|
+
const kind = json(row.effect, {})?.kind ?? 'effect';
|
|
973
|
+
out.push(row.failed_at
|
|
974
|
+
? { kind: 'effect_failed', severity: 'warning', title: `its ${kind} failed for good after ${row.attempts} attempts: ${row.last_error ?? 'no reason recorded'}`, data: { effect: row.id, kind, attempts: row.attempts }, actions: [] }
|
|
975
|
+
: { kind: 'effect_stalled', severity: 'error', title: `its ${kind} has failed ${row.attempts} times and is still tried: ${row.last_error ?? 'no reason recorded'}`, data: { effect: row.id, kind, attempts: row.attempts }, actions: [] });
|
|
905
976
|
}
|
|
906
977
|
const order = ['critical', 'error', 'warning'];
|
|
907
978
|
return out.sort((a, b) => order.indexOf(a.severity) - order.indexOf(b.severity));
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// The fleet as each machine the board works on answers for itself: its panes (`harness.v1.panes.list`) and the harness
|
|
2
|
+
// session beneath each live pane's process (`harness.v1.sessions.activity_under`). This is the reading the machine's
|
|
3
|
+
// connector publishes to Teams, taken at its source. Teams' session feed is for viewers; dispatch never waits on it
|
|
4
|
+
// (D137, D138).
|
|
5
|
+
//
|
|
6
|
+
// Every machine is read at once, each within DEADLINE_MS (bin/machine-fleet.mjs), so one hung machine or relay costs a
|
|
7
|
+
// reading that long, not the sum of them. A machine that answered is `online`. A remote machine that did not is
|
|
8
|
+
// `unknown` (its runs are kept, as an unread machine's always were), unless the relay positively answered that it is
|
|
9
|
+
// not linked (`machine_offline`): then it is offline. This machine's door not answering is no reading at all.
|
|
10
|
+
import { spawnSync } from 'node:child_process';
|
|
11
|
+
import { fileURLToPath } from 'node:url';
|
|
12
|
+
|
|
13
|
+
const entry = fileURLToPath(new URL('../bin/machine-fleet.mjs', import.meta.url));
|
|
14
|
+
const DEADLINE_MS = 10_000;
|
|
15
|
+
|
|
16
|
+
/** One reading of `machines` (normalized names) and this machine, `local`. */
|
|
17
|
+
export function readMachineFleet(machines, local) {
|
|
18
|
+
const names = [...new Set([local, ...machines])];
|
|
19
|
+
const reply = spawnSync(process.execPath, [entry], { input: JSON.stringify({ machines: names.map((name) => ({ name, at: name === local ? null : name })), deadlineMs: DEADLINE_MS }),
|
|
20
|
+
encoding: 'utf8', timeout: DEADLINE_MS + 5_000, maxBuffer: 64 * 1024 * 1024 });
|
|
21
|
+
let rows = null;
|
|
22
|
+
try { rows = JSON.parse(String(reply.stdout).trim().split('\n').pop()); } catch { /* no reading */ }
|
|
23
|
+
const online = new Set(), unknown = new Set(), alive = new Set(), native = new Map(), sessions = [];
|
|
24
|
+
if (!Array.isArray(rows)) {
|
|
25
|
+
const error = String(reply.stderr || reply.error?.message || 'no answer').trim().slice(-300);
|
|
26
|
+
return { ok: false, presenceKnown: false, online, unknown: new Set(names), alive, native, sessions, error: `the fleet reading did not answer: ${error}` };
|
|
27
|
+
}
|
|
28
|
+
let localError = null;
|
|
29
|
+
for (const row of rows) {
|
|
30
|
+
if (row.error) {
|
|
31
|
+
if (row.machine === local) localError = row.error;
|
|
32
|
+
else if (row.code !== 'machine_offline') unknown.add(row.machine);
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
online.add(row.machine);
|
|
36
|
+
const live = (row.panes ?? []).filter((pane) => !pane.dead && pane.pid);
|
|
37
|
+
for (const { pid, activity } of row.activities ?? []) {
|
|
38
|
+
const pane = live.find((item) => item.pid === pid);
|
|
39
|
+
if (!pane || !activity?.session_id || !activity.harness) continue;
|
|
40
|
+
const address = `sc:${row.machine}:${activity.harness}:${activity.session_id}`, running = activity.presence === 'running';
|
|
41
|
+
native.set(address, { alive: running ? true : null, states: [running ? true : null] });
|
|
42
|
+
if (running) alive.add(address);
|
|
43
|
+
sessions.push({ address, machine: row.machine, harness: activity.harness, session_id: activity.session_id, pane: pane.pane, name: pane.title ?? pane.label ?? null,
|
|
44
|
+
state: running ? 'running' : 'unknown', turn: activity.turn ?? null, native_state: activity.evidence?.native_state ?? null });
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const ok = online.has(local);
|
|
48
|
+
return { ok, presenceKnown: ok, online, unknown, alive, native, sessions, error: ok ? null : `machine ${local}'s door did not answer: ${localError ?? 'no row'}` };
|
|
49
|
+
}
|
package/board/request-mail.mjs
CHANGED
|
@@ -3,7 +3,9 @@ import { event, json, now } from './store.mjs';
|
|
|
3
3
|
export function pendingRequestMail(db, taskId = null) {
|
|
4
4
|
const pending = db.prepare(`SELECT id, task_id, payload, created_at FROM task_events original
|
|
5
5
|
WHERE kind='board_message' AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind='board_message_sent'
|
|
6
|
-
AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
|
|
6
|
+
AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
|
|
7
|
+
AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind='board_message_abandoned'
|
|
8
|
+
AND gone.task_id=original.task_id AND json_extract(gone.payload,'$.event')=original.id) ORDER BY id`).all()
|
|
7
9
|
.map(row => ({ ...row, mail: json(row.payload, {}) ?? {} }));
|
|
8
10
|
if (!pending.length) return { ready: [], nextAt: Infinity };
|
|
9
11
|
const batches = db.prepare("SELECT payload FROM task_events WHERE kind='board_message_batch' AND id>=? ORDER BY id").all(pending[0].id)
|
package/board/store.mjs
CHANGED
|
@@ -76,6 +76,9 @@ CREATE TABLE IF NOT EXISTS supercode_effects (id INTEGER PRIMARY KEY AUTOINCREME
|
|
|
76
76
|
// `stage`: the card's workflow status where kanban.db stores another (a `reviewing` card is `running` to Hermes)
|
|
77
77
|
// `since`: when the card entered its workflow status; `after_fired`: which of the status's `after` moves have run since
|
|
78
78
|
// `tags`: the card's ordered tags (Hermes's `tenant` holds the first); `sponsor`, `sponsor_evidence`, `requested_by_unverified`: C4
|
|
79
|
+
// An effect's attempts: how many were made, when the next may be, and, once its attempts are spent, when it failed and
|
|
80
|
+
// why (engine.mjs runEffects). A failed effect is kept as the record of its terminal state and never tried again.
|
|
81
|
+
const EFFECT_COLUMNS = { attempts: 'INTEGER NOT NULL DEFAULT 0', next_at: 'INTEGER NOT NULL DEFAULT 0', failed_at: 'INTEGER', last_error: 'TEXT' };
|
|
79
82
|
const OURS_COLUMNS = { workspace_anchor: 'TEXT', workspace_root: 'TEXT', subtask_of: 'TEXT', stage: 'TEXT', since: 'INTEGER', after_fired: 'TEXT', blocked_by: 'TEXT', paused: 'TEXT', archived: 'INTEGER DEFAULT 0', every: 'TEXT', next_run_at: 'INTEGER', tags: 'TEXT', sponsor: 'TEXT', sponsor_evidence: 'TEXT', requested_by_unverified: 'TEXT' };
|
|
80
83
|
|
|
81
84
|
// ------------------------------------------------------------------ backings
|
|
@@ -140,6 +143,8 @@ export function layOut(db, { fresh }) {
|
|
|
140
143
|
db.exec(PUBLICATION_SCHEMA);
|
|
141
144
|
const have = new Set(db.prepare('PRAGMA table_info(supercode_cards)').all().map((c) => c.name));
|
|
142
145
|
for (const [column, type] of Object.entries(OURS_COLUMNS)) if (!have.has(column)) db.exec(`ALTER TABLE supercode_cards ADD COLUMN ${column} ${type}`);
|
|
146
|
+
const effectColumns = new Set(db.prepare('PRAGMA table_info(supercode_effects)').all().map((c) => c.name));
|
|
147
|
+
for (const [column, type] of Object.entries(EFFECT_COLUMNS)) if (!effectColumns.has(column)) db.exec(`ALTER TABLE supercode_effects ADD COLUMN ${column} ${type}`);
|
|
143
148
|
}
|
|
144
149
|
|
|
145
150
|
/**
|
package/board/wake.mjs
CHANGED
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
import { watch, existsSync, readFileSync, readdirSync, writeFileSync, renameSync, unlinkSync, realpathSync } from 'node:fs';
|
|
4
4
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
5
5
|
import { randomUUID } from 'node:crypto';
|
|
6
|
-
import { followFleet } from './event-fleet.mjs';
|
|
7
6
|
import { boardEventRelay } from './event-relay.mjs';
|
|
8
7
|
import { fileURLToPath } from 'node:url';
|
|
9
8
|
|
|
@@ -27,7 +26,7 @@ export function watchBoard({ root, binary, env, log = () => {} }) {
|
|
|
27
26
|
if (!pending) pending = { at, sources: new Set() };
|
|
28
27
|
pending.at = Math.min(pending.at, at);
|
|
29
28
|
pending.sources.add(source);
|
|
30
|
-
|
|
29
|
+
void relay?.wake();
|
|
31
30
|
waiter?.();
|
|
32
31
|
};
|
|
33
32
|
const directory = (path, key, accept) => {
|
|
@@ -94,7 +93,6 @@ export function watchBoard({ root, binary, env, log = () => {} }) {
|
|
|
94
93
|
refresh();
|
|
95
94
|
const supercodeHome = env.SUPERCODE_HOME || (env.XDG_CONFIG_HOME ? join(env.XDG_CONFIG_HOME,'supercode') : join(env.HOME ?? '', '.config','supercode'));
|
|
96
95
|
relay = boardEventRelay({root,supercodeHome,log});
|
|
97
|
-
const fleet = followFleet({root,supercodeHome,wake,log});
|
|
98
96
|
void relay.wake();
|
|
99
97
|
return {
|
|
100
98
|
refresh,
|
|
@@ -111,7 +109,7 @@ export function watchBoard({ root, binary, env, log = () => {} }) {
|
|
|
111
109
|
return result ? { at: result.at, sources: [...result.sources] } : null;
|
|
112
110
|
},
|
|
113
111
|
close() {
|
|
114
|
-
closed = true; waiter?.();
|
|
112
|
+
closed = true; waiter?.(); relay.close();
|
|
115
113
|
for (const handle of watchers.values()) handle.close();
|
|
116
114
|
watchers.clear();
|
|
117
115
|
},
|
package/board.mjs
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import { spawn } from 'node:child_process';
|
|
14
14
|
import { fileURLToPath } from 'node:url';
|
|
15
15
|
import { summary } from './board/cli.mjs';
|
|
16
|
-
import { commandEnv, notifyChats, supercodeBin } from './board/dispatch.mjs';
|
|
16
|
+
import { commandEnv, notifyChats, startDelivery, supercodeBin } from './board/dispatch.mjs';
|
|
17
17
|
import { watchBoard } from './board/wake.mjs';
|
|
18
18
|
import { workflowEntry } from './board/workflow.mjs';
|
|
19
19
|
import { evaluate, holds } from './board/expr.mjs';
|
|
@@ -79,6 +79,8 @@ export async function startBoard({ root, orchestration = null, log = () => {}, a
|
|
|
79
79
|
try {
|
|
80
80
|
const { reports, error } = await dispatchOnce(root);
|
|
81
81
|
if (error) { log('warn', `board: tick failed: ${error}`); return; }
|
|
82
|
+
// what the round recorded is delivered by a pass of its own, which this loop never waits for
|
|
83
|
+
startDelivery(root, { log: (level, line) => log(level, `board: ${line}`) });
|
|
82
84
|
// chat-platform subscriptions are delivered here, through the daemon's own channel adapters
|
|
83
85
|
try { await notifyChats(root, adapters, log, wake); } catch (error) { log('warn', `board: chat notices failed: ${error.message}`); }
|
|
84
86
|
for (const r of reports) {
|
package/loop.mjs
CHANGED
|
@@ -24,7 +24,7 @@ import { harnessHomes } from './mcp/verbs.mjs';
|
|
|
24
24
|
import { checkMonitor, runJobScript } from './gates.mjs';
|
|
25
25
|
import { startBoard } from './board.mjs';
|
|
26
26
|
import { SenderIdentityRecord } from '@volter/teams/sender-identity';
|
|
27
|
-
import { agentDoor,agentScope,teamsUnavailable,ensureProfileAgent,localProfileAgent,localAgentRecord,localAgentSessions,synchronizeLocalAgents,ensureForkAgent,agentRunForKey,agentIntent,agentPreamble,agentEnvironment,recordAgentRun,recordNativeAgentSession,pendingAgentRuns,synchronizeAgentRun,agentRunForSession } from './agents.mjs';
|
|
27
|
+
import { agentDoor,agentScope,teamsUnavailable,ensureProfileAgent,localProfileAgent,localAgentRecord,localAgentSessions,synchronizeLocalAgents,ensureForkAgent,agentRunForKey,agentIntent,agentPreamble,agentEnvironment,recordAgentRun,recordNativeAgentSession,pendingAgentRuns,claimAgentRun,releaseAgentRun,synchronizeAgentRun,agentRunForSession } from './agents.mjs';
|
|
28
28
|
import { materializeAgentHome } from './agent-home.mjs';
|
|
29
29
|
import { localMachine } from './board/dispatch.mjs';
|
|
30
30
|
import { execFile } from 'node:child_process';
|
|
@@ -201,8 +201,9 @@ export class Orchestrator {
|
|
|
201
201
|
if(scope.offline)return;
|
|
202
202
|
for(const pending of synchronizeLocalAgents({root:this.state.root,profiles:this.state.profiles,context,declarations:this.state.agents??{}}))this.log('warn',`agent ${pending.agent} declaration pending: ${pending.error}`);
|
|
203
203
|
for(const row of pendingAgentRuns(this.state.root,scope.org)){
|
|
204
|
+
if(!claimAgentRun(this.state.root,scope.org,row))continue;
|
|
204
205
|
try{synchronizeAgentRun(this.state.root,scope.org,row,{context,intended:row.profile_json?JSON.parse(row.profile_json):undefined});}
|
|
205
|
-
catch(error){this.log('warn',`agent session ${row.session_ref}: ${error.message}`);if(error.code==='rate_limited')break;}
|
|
206
|
+
catch(error){releaseAgentRun(this.state.root,scope.org,row);this.log('warn',`agent session ${row.session_ref}: ${error.message}`);if(error.code==='rate_limited')break;}
|
|
206
207
|
}
|
|
207
208
|
}catch(error){this.log('warn',`agent session synchronization: ${error.message}`);}
|
|
208
209
|
}
|
|
@@ -521,8 +522,8 @@ export class Orchestrator {
|
|
|
521
522
|
if(!INSTRUCTION_ARGUMENTS[worker.harness])(this.runtimeAgentPreambles??=new Map()).set(lid,agentPrompt);
|
|
522
523
|
const session=`${worker.harness}:${runtime.handle.runtime_id}`;
|
|
523
524
|
recordAgentRun(this.state.root,scope.org,agent.identity.id,{session,machine:null,run:run_key??lid,profile:agent.intended.resolved,main,starterSession:starter_session});
|
|
524
|
-
if(agent.synced!==false)try{synchronizeAgentRun(this.state.root,scope.org,{agent_id:agent.identity.id,session_ref:session,machine:null,main},{context,intended:agent.intended.resolved});}
|
|
525
|
-
catch(error){this.log('warn',`agent session ${session} retained for synchronization: ${error.message}`);}
|
|
525
|
+
if(agent.synced!==false&&claimAgentRun(this.state.root,scope.org,{session_ref:session,published:0}))try{synchronizeAgentRun(this.state.root,scope.org,{agent_id:agent.identity.id,session_ref:session,machine:null,main},{context,intended:agent.intended.resolved});}
|
|
526
|
+
catch(error){releaseAgentRun(this.state.root,scope.org,{session_ref:session});this.log('warn',`agent session ${session} retained for synchronization: ${error.message}`);}
|
|
526
527
|
this.enqueue(events.worker({
|
|
527
528
|
profile: profileName, keyString, runtime_id: runtime.handle.runtime_id,
|
|
528
529
|
ev: { kind: 'started', runtime_id: runtime.handle.runtime_id, session_id: runtime.handle.runtime_id, connection: runtime.connection, harness: worker.harness, steer },
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-orchestrator",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.39",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
|
|
6
6
|
"exports": {
|
package/board/event-fleet.mjs
DELETED
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
// A projection of retained Teams events. Dispatch reads this projection, never a periodically fetched catalogue.
|
|
2
|
-
import { readFileSync, writeFileSync, renameSync } from 'node:fs';
|
|
3
|
-
import { join } from 'node:path';
|
|
4
|
-
import { randomUUID } from 'node:crypto';
|
|
5
|
-
import { eventClient, retryAfterMs } from '@volter/teams/event-client';
|
|
6
|
-
const pathOf=root=>join(root,'.session-events.json');
|
|
7
|
-
const normalize=value=>String(value??'').toLowerCase().replace(/[^a-z0-9-]/g,'-').replace(/-+/g,'-').replace(/^-|-$/g,'');
|
|
8
|
-
const wait=(signal,ms)=>new Promise(resolve=>{const timer=setTimeout(resolve,ms);signal.addEventListener('abort',()=>{clearTimeout(timer);resolve();},{once:true});});
|
|
9
|
-
|
|
10
|
-
export function followFleet({root,context,supercodeHome,wake,log}){
|
|
11
|
-
const abort=new AbortController();let resources={},cursor=null,connected=false,snapshot=null;
|
|
12
|
-
const persist=()=>{
|
|
13
|
-
const path=pathOf(root),temporary=`${path}.${randomUUID()}.tmp`;
|
|
14
|
-
writeFileSync(temporary,JSON.stringify({pid:process.pid,connected,cursor,resources}),{mode:0o600});renameSync(temporary,path);
|
|
15
|
-
};
|
|
16
|
-
const apply=(target,frame)=>{
|
|
17
|
-
const event=frame.data;if(!event||!['session','machine'].includes(event.resource?.kind))return false;
|
|
18
|
-
const key=`${event.resource.kind}:${event.resource.id}`,old=target[key];
|
|
19
|
-
if(old&&old.sequence>=frame.sequence)return false;
|
|
20
|
-
target[key]={sequence:frame.sequence,data:event.data};return true;
|
|
21
|
-
};
|
|
22
|
-
void(async()=>{
|
|
23
|
-
// Refused for load, the follower waits out the server's retry_after, doubling while it is refused (to 5 min), and
|
|
24
|
-
// keeps its cursor: asking every 5 s spent the slot it waited for on the next refusal.
|
|
25
|
-
let refused=0;
|
|
26
|
-
while(!abort.signal.aborted){
|
|
27
|
-
let pause=5000;
|
|
28
|
-
try{
|
|
29
|
-
const {team}=eventClient({context,supercodeHome});
|
|
30
|
-
// Every connection begins with a complete, ACL-filtered snapshot. A process restart cannot trust an old cache.
|
|
31
|
-
for await(const frame of team.events.follow({kinds:['session.started','session.ended','session.changed','resource.changed']}, {signal:abort.signal,after:cursor,caller:'dispatch session-events'})){
|
|
32
|
-
if(frame.type==='snapshot_begin'){snapshot={};connected=false;persist();}
|
|
33
|
-
else if(frame.type==='snapshot_item')apply(snapshot??resources,frame);
|
|
34
|
-
else if(frame.type==='snapshot_end'){resources=snapshot??resources;snapshot=null;cursor=frame.cursor;connected=true;refused=0;persist();wake('session-events-snapshot');}
|
|
35
|
-
else if(frame.type==='event'){const changed=apply(resources,frame);cursor=frame.cursor;connected=true;persist();if(changed)wake('session-events');}
|
|
36
|
-
else if(frame.type==='checkpoint'){const recovered=!connected;cursor=frame.cursor;connected=true;refused=0;persist();if(recovered)wake('session-events-reconnected');}
|
|
37
|
-
else if(frame.terminal){cursor=null;connected=false;persist();wake('session-events-unavailable');break;}
|
|
38
|
-
}
|
|
39
|
-
}catch(error){
|
|
40
|
-
const asked=retryAfterMs(error);if(asked){pause=Math.min(300000,asked*2**refused);refused++;}
|
|
41
|
-
if(!abort.signal.aborted)log(`dispatch session events unavailable: ${error.code??error.message}${asked?` (retrying in ${Math.round(pause/1000)} s)`:''}`);if(error.status===409)cursor=null;
|
|
42
|
-
}
|
|
43
|
-
connected=false;persist();wake('session-events-unavailable');if(!abort.signal.aborted)await wait(abort.signal,pause);
|
|
44
|
-
}
|
|
45
|
-
})();
|
|
46
|
-
return {close(){abort.abort();connected=false;persist();}};
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
export function readEventFleet(root){
|
|
50
|
-
let cache;try{cache=JSON.parse(readFileSync(pathOf(root),'utf8'));process.kill(cache.pid,0);}catch{return {ok:false,alive:new Set(),online:new Set(),sessions:[],native:new Map(),error:'published session events unavailable',presenceKnown:false};}
|
|
51
|
-
const machines=new Map(),online=new Set();
|
|
52
|
-
for(const [key,value] of Object.entries(cache.resources??{}))if(key.startsWith('machine:')){machines.set(value.data.id,normalize(value.data.name));if(cache.connected&&value.data.linked)online.add(normalize(value.data.name));}
|
|
53
|
-
const sessions=[],native=new Map(),alive=new Set();
|
|
54
|
-
for(const [key,value] of Object.entries(cache.resources??{}))if(key.startsWith('session:')){
|
|
55
|
-
for(const binding of value.data.bindings??[]){
|
|
56
|
-
const machine=machines.get(binding.machine_id);if(!machine||!binding.harness||!binding.native_session_id)continue;
|
|
57
|
-
const address=`sc:${machine}:${binding.harness}:${binding.native_session_id}`;
|
|
58
|
-
const known=cache.connected&&online.has(machine),state=known?(binding.state==='running'?true:binding.state==='stopped'?false:null):null;
|
|
59
|
-
const previous=native.get(address),states=[...(previous?.states??[]),state];
|
|
60
|
-
const combined=states.includes(true)?true:states.includes(null)?null:states.every(value=>value===false)?false:null;
|
|
61
|
-
native.set(address,{alive:combined,states});if(combined===true)alive.add(address);
|
|
62
|
-
sessions.push({address,machine,harness:binding.harness,session_id:binding.native_session_id,pane:binding.pane??null,name:binding.name??null,state:binding.state,turn:binding.turn??null,native_state:binding.evidence?.native_state??null,identity:value.data.agent_identity??null});
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
return {ok:!!cache.connected,presenceKnown:!!cache.connected,online,alive,native,sessions,error:cache.connected?null:'published session events disconnected'};
|
|
66
|
-
}
|