@volter/supercode-orchestrator 0.5.39 → 0.5.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/machine-fleet.mjs +13 -4
- package/board/dispatch.mjs +99 -135
- package/board/engine.mjs +5 -0
- package/board/machine-fleet.mjs +9 -5
- package/board/workflow.mjs +0 -3
- package/package.json +1 -1
package/bin/machine-fleet.mjs
CHANGED
|
@@ -12,15 +12,24 @@ async function read({ name, at }) {
|
|
|
12
12
|
let channel;
|
|
13
13
|
try {
|
|
14
14
|
channel = await openMachine({ machine: at });
|
|
15
|
-
const
|
|
15
|
+
const node = new StreamRpc(await channel.open('node'));
|
|
16
|
+
const panes = await node.call('harness.v1.panes.list', {});
|
|
17
|
+
// What a board launch needs of this machine: its board process guard (describe), and its keyed launch door, asked
|
|
18
|
+
// for a key no launch has (an answer of null is the door; an unknown method is its absence)
|
|
19
|
+
const description = await node.call('harness.v1.machine.describe', {}).catch(() => null);
|
|
20
|
+
const keyed = await node.call('harness.v1.launches.get', { key: 'board-capability-reading' }).then(() => null, (error) => String(error?.message ?? error).slice(0, 200));
|
|
21
|
+
const capabilities = { guard: description?.boardProcessPolicy?.version === 1 ? description.boardProcessPolicy.harnesses ?? [] : [], launches: keyed === null, launchesRefused: keyed };
|
|
16
22
|
const live = (Array.isArray(panes) ? panes : []).filter((pane) => !pane.dead && pane.pid);
|
|
17
|
-
|
|
23
|
+
// the session under each live pane: the harness stream, an operator's; a caller the machine admits only to its panes
|
|
24
|
+
// (a box another custodian enrolled, granting view and control) reads its panes and no sessions
|
|
25
|
+
let activities = [], sessionsUnread = null;
|
|
18
26
|
if (live.length) {
|
|
19
27
|
const client = new SupercodeHarnessClient({ transport: harnessTransport(channel) });
|
|
20
28
|
try { await client.start?.(); activities = (await client.request('harness.v1.sessions.activity_under', { pids: live.map((pane) => pane.pid) }))?.activities ?? []; }
|
|
21
|
-
|
|
29
|
+
catch (error) { sessionsUnread = String(error?.message ?? error).slice(0, 200); }
|
|
30
|
+
finally { await client.close?.().catch(() => {}); }
|
|
22
31
|
}
|
|
23
|
-
return { machine: name, panes, activities };
|
|
32
|
+
return { machine: name, panes, activities, capabilities, sessionsUnread };
|
|
24
33
|
} finally { channel?.close(); }
|
|
25
34
|
}
|
|
26
35
|
const rows = await Promise.all(machines.map((machine) => within(read(machine), deadlineMs)
|
package/board/dispatch.mjs
CHANGED
|
@@ -312,6 +312,8 @@ function supervise(board, fleet, settings, report, entry, effects) {
|
|
|
312
312
|
for (const { id } of rows) {
|
|
313
313
|
const session = currentRun(board.db, id)?.metadata?.supercode;
|
|
314
314
|
if (!session?.launch_key || !fleet.ok || sessionAlive(fleet, session) !== false) continue;
|
|
315
|
+
// how a keyed launch ended, where its machine keeps that record: read when it can be, never waited on
|
|
316
|
+
if (!fleet.capabilities?.get(normalMachine(session.machine))?.launches) continue;
|
|
315
317
|
try { const receipt = launchReceipt(board, session); if (receipt?.live === false && receipt.exit) exits.set(id, receipt.exit); }
|
|
316
318
|
catch (error) { report.failed.push([id, `exit receipt unavailable; retaining liveness observation: ${error.message}`]); }
|
|
317
319
|
}
|
|
@@ -351,11 +353,16 @@ function supervise(board, fleet, settings, report, entry, effects) {
|
|
|
351
353
|
const adopted = session?.mode === 'adopt' && Boolean(session.address);
|
|
352
354
|
if (runRow && ((session?.launch_pending && !adopted) || session?.launch_deferred || (!session?.session_id && !session?.pid))) {
|
|
353
355
|
const age = t - Number(session?.launched_at ?? runRow.started_at);
|
|
356
|
+
// a launch whose answer was lost may still have opened its pane: the window ends it only when this round's
|
|
357
|
+
// fleet reading shows neither its pane nor its session on its machine
|
|
358
|
+
const seen = session?.launch_key ? launchReading(fleet, session) : 'absent';
|
|
359
|
+
if (age >= settings.launchWindow && seen !== 'absent') { extendClaim(db, id, { ttl: settings.ttl, reason: seen === 'present' ? 'launch_binding' : 'launch_unread' }); return; }
|
|
354
360
|
if (age >= settings.launchWindow) {
|
|
355
361
|
const reason = session?.launch_deferred_error ?? session?.launch_error ?? 'no session or process was bound to the dispatched run';
|
|
356
362
|
const error = `launch window ${settings.launchWindow}s expired after ${age}s: ${reason}`;
|
|
357
363
|
if (!session?.launch_expired_at) {
|
|
358
|
-
|
|
364
|
+
// a fresh launch the reading never showed has no session to resume: its next run starts fresh
|
|
365
|
+
setRunSession(db, runRow.id, { launch_expired_at: t, launch_error: error, ...(session?.launch_key && session.mode === 'fresh' ? { forgotten: true } : {}) });
|
|
359
366
|
event(db, id, 'session_launch_expired', { error, age_seconds: age, window_seconds: settings.launchWindow, key: session?.launch_key ?? null, pane: session?.pane ?? null }, runRow.id);
|
|
360
367
|
}
|
|
361
368
|
observe('spawn_failed', { error, launch_window_seconds: settings.launchWindow }, null);
|
|
@@ -799,7 +806,7 @@ export async function startCard(board, id, { roles = ['manager'], lanes, setting
|
|
|
799
806
|
|
|
800
807
|
/** Reconcile durable close intent and ended runs, including old unverified closed_at records.
|
|
801
808
|
* Reads and process I/O stay outside the writer. An active successor owns its pane until it ends. */
|
|
802
|
-
export function closeRequested(board, report, { taskId = null, caller = null, ttl = null } = {}) {
|
|
809
|
+
export function closeRequested(board, report, { taskId = null, caller = null, ttl = null, fleet = null } = {}) {
|
|
803
810
|
const rows = view(board, (db) => db.prepare(`SELECT id, task_id, ended_at, metadata FROM task_runs
|
|
804
811
|
WHERE metadata IS NOT NULL ${taskId ? 'AND task_id = ?' : ''}`).all(...(taskId ? [taskId] : [])));
|
|
805
812
|
// Repair pre-transfer history once: only identities with an older unended reference need a write.
|
|
@@ -821,29 +828,29 @@ export function closeRequested(board, report, { taskId = null, caller = null, tt
|
|
|
821
828
|
for (const row of rows) {
|
|
822
829
|
let session = json(board.db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(row.id)?.metadata, {})?.supercode;
|
|
823
830
|
if (!session || session.close_transferred_to || session.close_confirmed_at) continue;
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
(
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
831
|
+
// A keyed launch is reconciled by its machine's own daemon (D137), never by a door the close waits on. A session
|
|
832
|
+
// adopted onto a run whose launch was still pending did not come from that launch: the key is let go and the
|
|
833
|
+
// session is the run's, released or closed as its own below. A launch that never bound a pane left nothing the
|
|
834
|
+
// board can name on its machine: its close is settled here. One with a pane is closed as any pane: panes.close,
|
|
835
|
+
// then the pane read absent.
|
|
836
|
+
if (session.launch_key && session.launch_pending && (row.ended_at || session.close_requested)) {
|
|
837
|
+
if (session.mode === 'adopt' && session.address) {
|
|
838
|
+
tx(board, (db) => setRunSession(db, row.id, { launch_pending: false, launch_key: null, launch_error: null }));
|
|
839
|
+
session = json(board.db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(row.id)?.metadata, {})?.supercode;
|
|
840
|
+
} else if (!session.pane && !session.address && !session.pid) {
|
|
841
|
+
// settled only on a fleet reading that shows no session with its id on its machine; a lost open answer may
|
|
842
|
+
// have opened one, and that is closed as its own once it is bound
|
|
843
|
+
if (launchReading(fleet, session) !== 'absent') { report.undelivered.push([row.task_id, session.launch_key, 'close waits for a fleet reading that shows its launch absent', 'close']); continue; }
|
|
844
|
+
const settled = tx(board, (db) => {
|
|
845
|
+
const current = json(db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(row.id)?.metadata, {})?.supercode;
|
|
846
|
+
if (!current || current.launch_key !== session.launch_key || current.pane || current.close_confirmed_at) return false;
|
|
847
|
+
const at = now();
|
|
848
|
+
setRunSession(db, row.id, { launch_pending: false, close_requested: current.close_requested ?? at, closed_at: at, close_confirmed_at: at, close_error: null,
|
|
849
|
+
launch_error: current.launch_error ?? 'no pane was bound to its launch', forgotten: current.mode === 'fresh' ? true : Boolean(current.forgotten) });
|
|
850
|
+
event(db, row.task_id, 'session_closed', { key: session.launch_key, machine: session.machine, reason: 'no pane was bound to its launch, and the fleet reading shows neither a pane nor its session' }, row.id);
|
|
851
|
+
return true;
|
|
845
852
|
});
|
|
846
|
-
report.
|
|
853
|
+
if (settled) (report.closed ??= []).push({ task_id: row.task_id, ...session });
|
|
847
854
|
continue;
|
|
848
855
|
}
|
|
849
856
|
}
|
|
@@ -1049,10 +1056,7 @@ export async function sessionNotice(board, e) {
|
|
|
1049
1056
|
if (prior?.close_confirmed_at) {
|
|
1050
1057
|
tx(board, (db) => {
|
|
1051
1058
|
// A closed session has nothing to be told; its next run starts with its own prompt.
|
|
1052
|
-
|
|
1053
|
-
event(db, e.id, 'session_notice_not_required', { address: e.address, verb: e.verb, notice: e.durableId,
|
|
1054
|
-
reason: unissued ? 'machine confirmed launch was not issued; no session exists to notify' : 'session closed',
|
|
1055
|
-
...(unissued ? { receipt: prior.launch_cancellation } : {}) }, e.runId);
|
|
1059
|
+
event(db, e.id, 'session_notice_not_required', { address: e.address, verb: e.verb, notice: e.durableId, reason: 'session closed' }, e.runId);
|
|
1056
1060
|
if (prior.lifecycle_notice_pending === e.durableId) setRunSession(db, e.runId, { lifecycle_notice_pending: null });
|
|
1057
1061
|
});
|
|
1058
1062
|
return true;
|
|
@@ -1172,6 +1176,11 @@ function launcherCheck(db, task, lane, fleet) {
|
|
|
1172
1176
|
const machine = normalMachine(task.machine);
|
|
1173
1177
|
if (!fleet.ok) return `supercode could not read the fleet: ${fleet.error}`;
|
|
1174
1178
|
if (fleet.unknown?.has(machine)) return `machine ${machine}'s door did not answer; the card starts when it does`;
|
|
1179
|
+
// a machine is scheduled only when it can take a board launch: its board process guard runs this harness, and its
|
|
1180
|
+
// daemon keeps keyed launches (it reconciles them itself). One without them is not scheduled, and says why.
|
|
1181
|
+
const can = fleet.capabilities?.get(machine);
|
|
1182
|
+
if (fleet.online.has(machine) && can && !can.launches) return `machine ${machine} cannot take a board launch: its keyed launch door (harness.v1.launches.get) answered ${can.launchesRefused ?? 'nothing'} (an older daemon has none; a box another custodian runs keeps it for its operators)`;
|
|
1183
|
+
if (fleet.online.has(machine) && can && !can.guard.includes(HARNESSES[lane.harness] ? lane.harness : 'claude-code')) return `machine ${machine} has no native board process guard for ${lane.harness ?? 'claude-code'}; no unguarded session is launched`;
|
|
1175
1184
|
if (!fleet.online.has(machine)) return `machine ${machine} is offline; the card starts when it is back`;
|
|
1176
1185
|
// a card on this machine gets its workspace made here (scratch, dir or worktree, as Hermes makes it); another
|
|
1177
1186
|
// machine's card names its directory there
|
|
@@ -1233,68 +1242,28 @@ const getTaskNow = (board, id) => view(board, (db) => getTask(db, id));
|
|
|
1233
1242
|
|
|
1234
1243
|
|
|
1235
1244
|
/** A transport error is not a negative launch receipt. The key remains owned by its run. */
|
|
1245
|
+
/**
|
|
1246
|
+
* Where the round's fleet reading puts a launch's session: 'present' when its machine lists the launch's pane or a
|
|
1247
|
+
* session with its id, 'absent' when its machine answered and lists neither, null when its machine was not read.
|
|
1248
|
+
*/
|
|
1249
|
+
function launchReading(fleet, session) {
|
|
1250
|
+
const machine = normalMachine(session?.machine ?? localMachine());
|
|
1251
|
+
if (!fleet?.ok || !fleet.online?.has(machine) || !fleet.panes?.has(machine)) return null;
|
|
1252
|
+
const sessionSeen = session.session_id && fleet.sessions?.some((f) => f.machine === machine && f.session_id === session.session_id);
|
|
1253
|
+
return (session.pane && fleet.panes.get(machine).has(session.pane)) || sessionSeen ? 'present' : 'absent';
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
/** How a keyed launch ended, as its machine recorded it; read only where the machine has the door (supervise). */
|
|
1236
1257
|
function launchReceipt(board, session) {
|
|
1237
1258
|
const r = doorOn(session.machine, 'harness.v1.launches.get',{key:session.launch_key},
|
|
1238
1259
|
{ timeout: 30_000, as: boardAddress(board.label) });
|
|
1239
|
-
if (r.code !== 0) throw unavailable(`launch receipt unavailable on ${session.machine}:
|
|
1260
|
+
if (r.code !== 0) throw unavailable(`launch receipt unavailable on ${session.machine}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
|
|
1240
1261
|
if (r.stdout.trim() === 'null') return null;
|
|
1241
1262
|
const receipt = lastJson(r.stdout);
|
|
1242
1263
|
if (!receipt || receipt.key !== session.launch_key) throw unavailable(`invalid launch receipt for ${session.launch_key} on ${session.machine}`);
|
|
1243
1264
|
return receipt;
|
|
1244
1265
|
}
|
|
1245
1266
|
|
|
1246
|
-
function cancelUnissuedLaunch(board, session) {
|
|
1247
|
-
const result = doorOn(session.machine, 'harness.v1.launches.cancel', {key:session.launch_key},
|
|
1248
|
-
{ timeout: 30_000, as: boardAddress(board.label) });
|
|
1249
|
-
if (result.code !== 0) throw unavailable(`launch cancellation unavailable on ${session.machine}; retain ${session.launch_key}: ${(result.stderr || result.stdout).trim().slice(-300)}`);
|
|
1250
|
-
const receipt = lastJson(result.stdout);
|
|
1251
|
-
if (!receipt || receipt.key !== session.launch_key) throw unavailable(`invalid cancellation receipt for ${session.launch_key}`);
|
|
1252
|
-
return receipt;
|
|
1253
|
-
}
|
|
1254
|
-
|
|
1255
|
-
function unissuedLaunch(receipt, session) {
|
|
1256
|
-
return receipt?.key === session?.launch_key && receipt?.status === 'cancelled' && receipt?.cancelled === true &&
|
|
1257
|
-
receipt?.outcome === 'not_issued' && receipt?.live === false && receipt?.pane === null;
|
|
1258
|
-
}
|
|
1259
|
-
|
|
1260
|
-
/**
|
|
1261
|
-
* Retire a launch key its machine confirmed it never issued. Returns true when the run's launch is settled and closed,
|
|
1262
|
-
* 'released' when the key is retired but the run keeps a session adopted onto it, false when the run is not this key's.
|
|
1263
|
-
*/
|
|
1264
|
-
function retireUnissuedLaunch(board, row, session, receipt, { entry = null, effects = [] } = {}) {
|
|
1265
|
-
if (!unissuedLaunch(receipt, session)) return false;
|
|
1266
|
-
const result = tx(board, db => {
|
|
1267
|
-
const current = db.prepare('SELECT ended_at, metadata FROM task_runs WHERE id=?').get(row.id);
|
|
1268
|
-
const saved = json(current?.metadata, {})?.supercode;
|
|
1269
|
-
if (!saved || saved.launch_key !== session.launch_key) return { retired: false, effects: [] };
|
|
1270
|
-
// A session adopted onto a run whose launch was still pending did not come from that launch (the machine never
|
|
1271
|
-
// issued the key): the key is retired and the adopted session stays the run's, supervised and closed as its own.
|
|
1272
|
-
if (saved.mode === 'adopt' && saved.address && saved.launch_pending) {
|
|
1273
|
-
setRunSession(db, row.id, { launch_pending: false, launch_cancellation: receipt, launch_error: null, launch_key: null });
|
|
1274
|
-
event(db, row.task_id, 'session_launch_cancelled', { key: session.launch_key, machine: session.machine, receipt, kept: saved.address }, row.id);
|
|
1275
|
-
return { retired: 'released', effects: [] };
|
|
1276
|
-
}
|
|
1277
|
-
if (saved.pane || saved.address || saved.pid) return { retired: false, effects: [] };
|
|
1278
|
-
if (!current.ended_at && (!entry || getTask(db, row.task_id)?.current_run_id !== row.id)) return { retired: false, effects: [] };
|
|
1279
|
-
if (saved.close_confirmed_at) return { retired: true, effects: [] };
|
|
1280
|
-
const collected = [];
|
|
1281
|
-
const at = now();
|
|
1282
|
-
setRunSession(db, row.id, { launch_pending: false, launch_cancellation: receipt, launch_error: 'machine confirmed launch was not issued',
|
|
1283
|
-
close_requested: saved.close_requested ?? at, closed_at: at, close_confirmed_at: at, close_error: null,
|
|
1284
|
-
forgotten: saved.mode === 'fresh' ? true : Boolean(saved.forgotten) });
|
|
1285
|
-
event(db, row.task_id, 'session_launch_cancelled', { key: session.launch_key, machine: session.machine, receipt }, row.id);
|
|
1286
|
-
if (!current.ended_at) {
|
|
1287
|
-
const payload = { error: `launch ${session.launch_key} was not issued; its machine retired the key` };
|
|
1288
|
-
const moved = monitorsRuns(entry) ? (surfaceIncident(db,board,entry,row.task_id,'spawn_failed',payload), {effects:[]}) : fire(db,board,entry,row.task_id,'spawn_failed',{roles:['platform'],payload});
|
|
1289
|
-
collected.push(...moved.effects);
|
|
1290
|
-
}
|
|
1291
|
-
event(db, row.task_id, 'session_closed', { key: session.launch_key, machine: session.machine, reason: 'launch not issued', receipt }, row.id);
|
|
1292
|
-
return { retired: true, effects: collected };
|
|
1293
|
-
});
|
|
1294
|
-
effects.push(...result.effects);
|
|
1295
|
-
return result.retired;
|
|
1296
|
-
}
|
|
1297
|
-
|
|
1298
1267
|
function bindLaunch(board, row, session, pane, sid) {
|
|
1299
1268
|
return tx(board, (db) => {
|
|
1300
1269
|
const current = db.prepare('SELECT ended_at, metadata FROM task_runs WHERE id=?').get(row.id);
|
|
@@ -1331,12 +1300,10 @@ export function reconcileCallerLaunch(board, sid) {
|
|
|
1331
1300
|
if (!candidates.length) return;
|
|
1332
1301
|
const pane = process.env.SUPERCODE_TEAMS_PANE || observeFleet().sessions?.find((s) => s.address === `sc:${machine}:${harness}:${sid}`)?.pane;
|
|
1333
1302
|
if (!pane) return;
|
|
1303
|
+
// the caller's own pane (its environment, or this machine's reading) binds the run whose launch recorded that pane
|
|
1334
1304
|
for (const row of candidates) {
|
|
1335
1305
|
const s = json(row.metadata, {})?.supercode;
|
|
1336
|
-
|
|
1337
|
-
if (receipt?.live && receipt.pane === pane) {
|
|
1338
|
-
if (bindLaunch(board, row, s, pane, sid)) return;
|
|
1339
|
-
}
|
|
1306
|
+
if ((s.pane === pane || s.session_id === sid) && bindLaunch(board, row, s, pane, sid)) return;
|
|
1340
1307
|
}
|
|
1341
1308
|
}
|
|
1342
1309
|
|
|
@@ -1433,35 +1400,29 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
|
|
|
1433
1400
|
const rows = view(board, (db) => db.prepare(`SELECT id, task_id, started_at, ended_at, metadata FROM task_runs
|
|
1434
1401
|
WHERE json_extract(metadata, '$.supercode.launch_pending') = 1
|
|
1435
1402
|
AND json_extract(metadata, '$.supercode.close_confirmed_at') IS NULL`).all());
|
|
1403
|
+
// A keyed launch whose session is not bound yet is bound from this round's fleet reading: its pane, and the session
|
|
1404
|
+
// the reading finds in it. Its machine's daemon reconciles the key itself (D137); no door is asked here. A launch with
|
|
1405
|
+
// no pane waits for its launch window (supervise surfaces it then); a pane read absent on its online machine after
|
|
1406
|
+
// the grace ended before its session was bound.
|
|
1436
1407
|
for (const row of rows) {
|
|
1437
1408
|
const s = json(row.metadata, {})?.supercode;
|
|
1438
1409
|
try {
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
const moved = tx(board, (db) => {
|
|
1456
|
-
if (getTask(db, row.task_id)?.current_run_id !== row.id) return null;
|
|
1457
|
-
setRunSession(db, row.id, { launch_pending: false });
|
|
1458
|
-
const payload = { error: `launch ${s.launch_key} ended before session binding` };
|
|
1459
|
-
if (monitorsRuns(entry)) { surfaceIncident(db, board, entry, row.task_id, 'spawn_failed', payload); return { effects: [] }; }
|
|
1460
|
-
return fire(db, board, entry, row.task_id, 'spawn_failed', { roles: ['platform'], payload });
|
|
1461
|
-
});
|
|
1462
|
-
if (moved) effects.push(...moved.effects);
|
|
1463
|
-
}
|
|
1464
|
-
} else if (!sid) report.queued.push([row.task_id, `pane ${receipt.pane} is live; awaiting ${s.harness} session identity`]);
|
|
1410
|
+
// found by its pane (chosen before open) or by its session id (named before open, for Claude Code)
|
|
1411
|
+
const found = fleet.sessions?.find((f) => f.machine === normalMachine(s.machine) && String(f.address).startsWith(`sc:${s.machine}:${s.harness}:`)
|
|
1412
|
+
&& ((s.pane && f.pane === s.pane) || (s.session_id && f.session_id === s.session_id)));
|
|
1413
|
+
const sid = s.session_id ?? found?.address.split(':')[3];
|
|
1414
|
+
if (found && sid) { bindLaunch(board, row, s, found.pane ?? s.pane, sid); continue; }
|
|
1415
|
+
if (!row.ended_at && now() - Number(s.launched_at) > settings.grace && launchReading(fleet, s) === 'absent') {
|
|
1416
|
+
const moved = tx(board, (db) => {
|
|
1417
|
+
if (getTask(db, row.task_id)?.current_run_id !== row.id) return null;
|
|
1418
|
+
// a fresh launch that never showed in a reading has no session to resume: its next run starts fresh
|
|
1419
|
+
setRunSession(db, row.id, { launch_pending: false, ...(s.mode === 'fresh' ? { forgotten: true } : {}) });
|
|
1420
|
+
const payload = { error: `launch ${s.launch_key} ended before session binding` };
|
|
1421
|
+
if (monitorsRuns(entry)) { surfaceIncident(db, board, entry, row.task_id, 'spawn_failed', payload); return { effects: [] }; }
|
|
1422
|
+
return fire(db, board, entry, row.task_id, 'spawn_failed', { roles: ['platform'], payload });
|
|
1423
|
+
});
|
|
1424
|
+
if (moved) effects.push(...moved.effects);
|
|
1425
|
+
} else report.queued.push([row.task_id, `launch ${s.launch_key}${s.pane ? ` (pane ${s.pane})` : ''} awaits its ${s.harness} session in the fleet reading`]);
|
|
1465
1426
|
} catch (error) { report.failed.push([row.task_id, `launch reconciliation retained: ${error.message}`]); }
|
|
1466
1427
|
}
|
|
1467
1428
|
}
|
|
@@ -1499,7 +1460,8 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
1499
1460
|
if (pending) throw unavailable(`slot ${slot} still owns pane ${pending.session.pane} from run ${pending.id}; closure must be confirmed before another session starts: ${pending.session.close_error ?? 'close pending'}`);
|
|
1500
1461
|
let prev = view(board, (db) => lastSession(db, task.id, slot));
|
|
1501
1462
|
if (prev?.session_id && !prev.forgotten && (prev.harness ?? 'claude-code') === harness && prev.machine === machine) {
|
|
1502
|
-
|
|
1463
|
+
// the round's own fleet reading: a launch never reads the fleet again
|
|
1464
|
+
const fresh = fleet;
|
|
1503
1465
|
if (!fresh.ok) throw unavailable(`cannot check for a manual continuation of ${prev.session_id}: ${fresh.error}`);
|
|
1504
1466
|
const r = doorOn(machine, 'harness.v1.panes.list',{}, { timeout: 60_000 });
|
|
1505
1467
|
const listed = r.code === 0 ? json(r.stdout.trim(), null) : null;
|
|
@@ -1515,30 +1477,27 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
1515
1477
|
// a pane that could not be closed still holds the session: no second process is started on it
|
|
1516
1478
|
if (prev?.close_requested && !prev.closed_at) throw unavailable(`the slot's session ${prev.session_id} asked to be closed and its pane ${prev.pane} could not be: ${report.undelivered.map((u) => u[2]).join('; ') || 'no reason given'}`);
|
|
1517
1479
|
}
|
|
1480
|
+
// The machine was checked before the card was claimed (launcherCheck: its board process guard and keyed launch door,
|
|
1481
|
+
// from the round's fleet reading); the launch asks no door of it. Open's reply names the pane; a session named at
|
|
1482
|
+
// launch (Claude Code's) is bound with it, one that names itself (Codex) is bound from a later round's fleet reading.
|
|
1483
|
+
// The pane id is chosen from the launch key and recorded before open runs, so a launch whose answer is lost (a
|
|
1484
|
+
// timeout, a closed stream) is still found by its pane in a later round's fleet reading, and by its session id.
|
|
1518
1485
|
const openOwned = (key, sid, mode, args) => {
|
|
1519
|
-
const
|
|
1520
|
-
const policy = description.code === 0 ? json(description.stdout,null)?.boardProcessPolicy : null;
|
|
1521
|
-
if (policy?.version !== 1 || !policy.harnesses?.includes(harness)) throw unavailable(`machine ${machine} has no native board process guard for ${harness}; no unguarded session was launched`);
|
|
1522
|
-
launchReceipt(board, { machine, launch_key: key }); // Prove the exact receipt door before starting a process.
|
|
1486
|
+
const chosen = `p_${createHash('sha256').update(key).digest('hex').slice(0, 16)}`;
|
|
1523
1487
|
tx(board, (db) => setRunSession(db, runId, { ...meta, surface: 'pane', machine, harness, slot,
|
|
1524
|
-
session_id: sid, mode, launch_key: key, launch_pending: true, launched_at: now() }));
|
|
1525
|
-
const result = sc(args, { timeout: 180_000, as: boardAddress(board.label) });
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
const pane = facts?.pane ?? lastJson(result.stdout)?.pane ?? null;
|
|
1529
|
-
// The daemon's keyed receipt is authoritative even when the stream carrying open's reply closed.
|
|
1530
|
-
if (pane) tx(board, (db) => {
|
|
1488
|
+
session_id: sid, mode, launch_key: key, pane: chosen, launch_pending: true, launched_at: now() }));
|
|
1489
|
+
const result = sc([...args.slice(0, args.indexOf('--key')), '--pane', chosen, ...args.slice(args.indexOf('--key'))], { timeout: 180_000, as: boardAddress(board.label) });
|
|
1490
|
+
const pane = lastJson(result.stdout)?.pane ?? chosen;
|
|
1491
|
+
if (pane !== chosen) tx(board, (db) => {
|
|
1531
1492
|
const before = json(db.prepare('SELECT metadata FROM task_runs WHERE id=?').get(runId)?.metadata, {})?.supercode ?? {};
|
|
1532
1493
|
setRunSession(db, runId, { ...before, pane });
|
|
1533
1494
|
});
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
if (pane && discovered && facts?.live === true) {
|
|
1537
|
-
return { surface: 'pane', session_id: discovered, machine, harness, pane, mode, launch_key: key,
|
|
1495
|
+
if (pane && sid && result.code === 0) {
|
|
1496
|
+
return { surface: 'pane', session_id: sid, machine, harness, pane, mode, launch_key: key,
|
|
1538
1497
|
launch_pending: false, launch_error: null, launched_at: now() };
|
|
1539
1498
|
}
|
|
1540
|
-
const reason =
|
|
1541
|
-
`launch ${key} is
|
|
1499
|
+
const reason = result.code === 0 ? `launch ${key} opened pane ${pane}; its ${harness} session is bound from the fleet reading` :
|
|
1500
|
+
`launch ${key} (pane ${pane}) gave no answer; it is bound from the fleet reading if it opened: ${(result.stderr || result.stdout).trim().slice(-300)}`;
|
|
1542
1501
|
tx(board, (db) => {
|
|
1543
1502
|
setRunSession(db, runId, { launch_pending: true, launch_error: reason });
|
|
1544
1503
|
event(db, task.id, 'session_launch_pending', { key, pane, reason }, runId);
|
|
@@ -2237,7 +2196,11 @@ async function notify(board, fleet, report, entry) {
|
|
|
2237
2196
|
// and answers, each its own message, in full. A notice the board already recorded as taken by this address is never sent again.
|
|
2238
2197
|
// only events that would be told count: a stale one or the recipient's own act never supersedes one that would be
|
|
2239
2198
|
const sendable = (ev) => { const p = json(ev.payload, {}) ?? {}; return !(maxAge && now() - Number(ev.created_at) > maxAge) && !(p.by && p.by === sub.chat_id); };
|
|
2240
|
-
|
|
2199
|
+
// News is coalesced by its subject, not its kind alone: a later `assigned` says what an earlier one did, but each
|
|
2200
|
+
// subtask added, dependency linked or unlinked is its own fact, keyed by the card it names; a comment or an
|
|
2201
|
+
// answer is always its own message.
|
|
2202
|
+
const subjectOf = (ev) => { const p = json(ev.payload, {}) ?? {}; return ['commented', 'answered'].includes(ev.kind) ? `${ev.kind}:${ev.id}` : `${ev.kind}:${p.subtask ?? p.parent ?? p.child ?? ''}`; };
|
|
2203
|
+
const latest = new Map(events.filter((ev) => sendable(ev)).map((ev) => [subjectOf(ev), ev.id]));
|
|
2241
2204
|
const sentBefore = board.db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND kind IN ('notice_delivered', 'notice_filed') AND json_extract(payload, '$.source_event') = ? AND json_extract(payload, '$.address') = ? LIMIT 1");
|
|
2242
2205
|
for (const ev of events) {
|
|
2243
2206
|
const p = json(ev.payload, {}) ?? {};
|
|
@@ -2246,7 +2209,7 @@ async function notify(board, fleet, report, entry) {
|
|
|
2246
2209
|
if (sentBefore.get(sub.task_id, ev.id, sub.chat_id)) { cursor = ev.id; continue; }
|
|
2247
2210
|
// nobody is told of their own act
|
|
2248
2211
|
if (p.by && p.by === sub.chat_id) { cursor = ev.id; continue; }
|
|
2249
|
-
if (latest.has(ev
|
|
2212
|
+
if (latest.has(subjectOf(ev)) && latest.get(subjectOf(ev)) !== ev.id) { cursor = ev.id; (report.superseded_notices_passed ??= 0); report.superseded_notices_passed += 1; continue; }
|
|
2250
2213
|
let text;
|
|
2251
2214
|
if (ev.kind === 'commented') {
|
|
2252
2215
|
if (p.author === sub.chat_id) { cursor = ev.id; continue; }
|
|
@@ -2305,13 +2268,14 @@ async function notify(board, fleet, report, entry) {
|
|
|
2305
2268
|
// attempt it is abandoned (its terminal state, `notice_abandoned`, recorded once) and the cursor moves past it;
|
|
2306
2269
|
// the subscription stays, so the notices after it are still told (dropping it lost them when the round
|
|
2307
2270
|
// subscribed the session again from the card's newest event).
|
|
2308
|
-
|
|
2271
|
+
// failures are counted per notice: a failure of a later notice starts its own count
|
|
2272
|
+
const failures = failed ? (Number(sub.delivery_metadata?.failing_event) === trying ? Number(sub.delivery_metadata?.failures ?? 0) : 0) + 1 : 0;
|
|
2309
2273
|
if (failed && failures >= NOTICE_ATTEMPTS && trying) {
|
|
2310
2274
|
event(db, sub.task_id, 'notice_abandoned', { source_event: trying, address: sub.chat_id, attempts: failures, error: failed });
|
|
2311
2275
|
db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ?, delivery_metadata = ? ${where} AND last_event_id < ?`).run(trying, JSON.stringify({ ...sub.delivery_metadata, failures: 0, next_at: 0 }), ...identity, trying);
|
|
2312
2276
|
return;
|
|
2313
2277
|
}
|
|
2314
|
-
if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures, next_at: failed ? now() + Math.min(1800, 30 * 2 ** (failures - 1)) : 0 }), ...identity);
|
|
2278
|
+
if (failures !== Number(sub.delivery_metadata?.failures ?? 0) || (failed && Number(sub.delivery_metadata?.failing_event) !== trying)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures, failing_event: failed ? trying : null, next_at: failed ? now() + Math.min(1800, 30 * 2 ** (failures - 1)) : 0 }), ...identity);
|
|
2315
2279
|
if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity);
|
|
2316
2280
|
});
|
|
2317
2281
|
});
|
|
@@ -2721,7 +2685,7 @@ export async function deliver({ root, slugs = null }) {
|
|
|
2721
2685
|
* `workflow dispatch`). An adoption request and a session close are claimed in a transaction (`newClaim`) before
|
|
2722
2686
|
* anything is done for them outside the board; card starts, supervision, review requests, mailed verbs and manual
|
|
2723
2687
|
* continuations rely on their per-card transactions. Two rounds on one board at once can still both act on:
|
|
2724
|
-
* deferred-launch retries (two panes), launch
|
|
2688
|
+
* deferred-launch retries (two panes), launch binding (bindLaunch), and board-only writes
|
|
2725
2689
|
* that do not re-check. A round runs no effect and delivers nothing: it records effects (`queueEffects`) and the
|
|
2726
2690
|
* delivery pass (`deliver`) carries them out. Answers one report per board.
|
|
2727
2691
|
*/
|
|
@@ -2765,7 +2729,7 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
|
|
|
2765
2729
|
readMailbox(board, boardAddress(board.label), report); phase('mailbox');
|
|
2766
2730
|
await refreshRelationAnswers(board); phase('answers');
|
|
2767
2731
|
const effects = [];
|
|
2768
|
-
completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); queueEffects(board, effects.splice(0)); closeRequested(board, report, { ttl: settings.ttl }); phase('supervise');
|
|
2732
|
+
completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); queueEffects(board, effects.splice(0)); closeRequested(board, report, { ttl: settings.ttl, fleet }); phase('supervise');
|
|
2769
2733
|
report.unsettled = [];
|
|
2770
2734
|
report.promoted = recomputeReady(board, effects, report.unsettled);
|
|
2771
2735
|
tx(board,db=>auditEndedRuns(db,board,entry));
|
package/board/engine.mjs
CHANGED
|
@@ -974,6 +974,11 @@ export function diagnose(db, task, { failureLimit = DEFAULT_FAILURE_LIMIT, stran
|
|
|
974
974
|
? { kind: 'effect_failed', severity: 'warning', title: `its ${kind} failed for good after ${row.attempts} attempts: ${row.last_error ?? 'no reason recorded'}`, data: { effect: row.id, kind, attempts: row.attempts }, actions: [] }
|
|
975
975
|
: { kind: 'effect_stalled', severity: 'error', title: `its ${kind} has failed ${row.attempts} times and is still tried: ${row.last_error ?? 'no reason recorded'}`, data: { effect: row.id, kind, attempts: row.attempts }, actions: [] });
|
|
976
976
|
}
|
|
977
|
+
// notices the board gave up on: each one, with the event it was about and who was not told
|
|
978
|
+
for (const row of all(db, "SELECT payload FROM task_events WHERE task_id = ? AND kind = 'notice_abandoned' ORDER BY id DESC LIMIT 20", task.id)) {
|
|
979
|
+
const p = json(row.payload, {}) ?? {};
|
|
980
|
+
out.push({ kind: 'notice_abandoned', severity: 'warning', title: `the notice of event ${p.source_event} to ${p.address} was abandoned after ${p.attempts} attempts: ${p.error ?? 'no reason recorded'}`, data: { event: p.source_event, address: p.address, attempts: p.attempts }, actions: [] });
|
|
981
|
+
}
|
|
977
982
|
const order = ['critical', 'error', 'warning'];
|
|
978
983
|
return out.sort((a, b) => order.indexOf(a.severity) - order.indexOf(b.severity));
|
|
979
984
|
}
|
package/board/machine-fleet.mjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// The fleet as each machine the board works on answers for itself: its panes (`harness.v1.panes.list`) and the harness
|
|
2
2
|
// session beneath each live pane's process (`harness.v1.sessions.activity_under`). This is the reading the machine's
|
|
3
|
-
// connector publishes to Teams, taken at its source.
|
|
4
|
-
// (
|
|
3
|
+
// connector publishes to Teams, taken at its source. Each machine's reading also says what a board launch needs of it
|
|
4
|
+
// (`capabilities`: its board process guard's harnesses, and whether it has the keyed launch door). Teams' session feed
|
|
5
|
+
// is for viewers; dispatch never waits on it (D137, D138).
|
|
5
6
|
//
|
|
6
7
|
// Every machine is read at once, each within DEADLINE_MS (bin/machine-fleet.mjs), so one hung machine or relay costs a
|
|
7
8
|
// reading that long, not the sum of them. A machine that answered is `online`. A remote machine that did not is
|
|
@@ -20,10 +21,11 @@ export function readMachineFleet(machines, local) {
|
|
|
20
21
|
encoding: 'utf8', timeout: DEADLINE_MS + 5_000, maxBuffer: 64 * 1024 * 1024 });
|
|
21
22
|
let rows = null;
|
|
22
23
|
try { rows = JSON.parse(String(reply.stdout).trim().split('\n').pop()); } catch { /* no reading */ }
|
|
23
|
-
|
|
24
|
+
// `panes`: each answering machine's live pane ids, whether or not a session was read in them
|
|
25
|
+
const online = new Set(), unknown = new Set(), alive = new Set(), native = new Map(), sessions = [], capabilities = new Map(), panes = new Map();
|
|
24
26
|
if (!Array.isArray(rows)) {
|
|
25
27
|
const error = String(reply.stderr || reply.error?.message || 'no answer').trim().slice(-300);
|
|
26
|
-
return { ok: false, presenceKnown: false, online, unknown: new Set(names), alive, native, sessions, error: `the fleet reading did not answer: ${error}` };
|
|
28
|
+
return { ok: false, presenceKnown: false, online, unknown: new Set(names), alive, native, sessions, capabilities, panes, error: `the fleet reading did not answer: ${error}` };
|
|
27
29
|
}
|
|
28
30
|
let localError = null;
|
|
29
31
|
for (const row of rows) {
|
|
@@ -33,7 +35,9 @@ export function readMachineFleet(machines, local) {
|
|
|
33
35
|
continue;
|
|
34
36
|
}
|
|
35
37
|
online.add(row.machine);
|
|
38
|
+
capabilities.set(row.machine, row.capabilities ?? { guard: [], launches: false });
|
|
36
39
|
const live = (row.panes ?? []).filter((pane) => !pane.dead && pane.pid);
|
|
40
|
+
panes.set(row.machine, new Set((row.panes ?? []).filter((pane) => !pane.dead).map((pane) => pane.pane)));
|
|
37
41
|
for (const { pid, activity } of row.activities ?? []) {
|
|
38
42
|
const pane = live.find((item) => item.pid === pid);
|
|
39
43
|
if (!pane || !activity?.session_id || !activity.harness) continue;
|
|
@@ -45,5 +49,5 @@ export function readMachineFleet(machines, local) {
|
|
|
45
49
|
}
|
|
46
50
|
}
|
|
47
51
|
const ok = online.has(local);
|
|
48
|
-
return { ok, presenceKnown: ok, online, unknown, alive, native, sessions, error: ok ? null : `machine ${local}'s door did not answer: ${localError ?? 'no row'}` };
|
|
52
|
+
return { ok, presenceKnown: ok, online, unknown, alive, native, sessions, capabilities, panes, error: ok ? null : `machine ${local}'s door did not answer: ${localError ?? 'no row'}` };
|
|
49
53
|
}
|
package/board/workflow.mjs
CHANGED
|
@@ -374,9 +374,6 @@ function queueTransitionNotice(db, board, entry, task, from, name, payload, befo
|
|
|
374
374
|
const afterState = beforeRun && json(one(db, 'SELECT metadata FROM task_runs WHERE id=?', beforeRun.id)?.metadata, {})?.supercode?.state;
|
|
375
375
|
if (!beforeRun || (state(task, from) === state(after, to) && beforeState === afterState) || effects.some((e) => e.kind === 'session_notice' && e.runId === beforeRun.id)) return;
|
|
376
376
|
const session = json(beforeRun.metadata, {})?.supercode;
|
|
377
|
-
const cancellation = session?.launch_cancellation;
|
|
378
|
-
if (cancellation?.key === session?.launch_key && cancellation?.status === 'cancelled' && cancellation?.cancelled === true &&
|
|
379
|
-
cancellation?.outcome === 'not_issued' && cancellation?.live === false && cancellation?.pane === null) return;
|
|
380
377
|
const address = session?.address ?? (session?.session_id && session.machine ? `sc:${session.machine}:${session.harness ?? 'claude-code'}:${session.session_id}` : null);
|
|
381
378
|
if (!address && !session?.launch_key && !session?.pane) return;
|
|
382
379
|
// The session that made this change learns it from its own command's answer; a notice about
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-orchestrator",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.41",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
|
|
6
6
|
"exports": {
|