@volter/supercode-orchestrator 0.4.148 → 0.4.150
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/board/ci.mjs +147 -56
- package/board/cli.mjs +68 -61
- package/board/dispatch.mjs +129 -58
- package/board/engine.mjs +69 -25
- package/board/store.mjs +6 -3
- package/board/tools.mjs +6 -3
- package/board/workflow.mjs +30 -45
- package/board/workspace.mjs +32 -0
- package/board/ztrack.mjs +802 -542
- package/cron.mjs +8 -2
- package/hermes-locks.mjs +1 -20
- package/loop.mjs +23 -2
- package/mcp/hermes-kanban-tools.json +47 -18
- package/package.json +2 -2
- package/board/rows.mjs +0 -132
package/board/dispatch.mjs
CHANGED
|
@@ -15,17 +15,17 @@
|
|
|
15
15
|
// Every side effect follows a committed transition: a card is claimed before its session is launched, and a
|
|
16
16
|
// subscriber's cursor moves only after its notice was delivered, so two dispatchers over one board never both
|
|
17
17
|
// launch and a crash between the two steps is answered by the next tick.
|
|
18
|
-
import { inspectCompletionEvidence,
|
|
18
|
+
import { inspectCompletionEvidence, openReviewPullRequests, reviewDue, withCompletionEvidence, reviewCandidate, workflowRevision, retainWorkflowTick } from './ci.mjs';
|
|
19
19
|
import { createRequire } from 'node:module';
|
|
20
20
|
import { spawn as spawnChild, spawnSync } from 'node:child_process';
|
|
21
21
|
import { existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
22
|
-
import { hostname, tmpdir } from 'node:os';
|
|
22
|
+
import { hostname, tmpdir, totalmem } from 'node:os';
|
|
23
23
|
import { basename, dirname, join } from 'node:path';
|
|
24
24
|
import { createHash, randomUUID } from 'node:crypto';
|
|
25
25
|
import { fileURLToPath } from 'node:url';
|
|
26
26
|
import { view, boardPath, claimer, event, json, listBoards, logsDir, now, openBoard, setActor, tx } from './store.mjs';
|
|
27
27
|
import { doors, fire, loadWorkflow, resumeOf, runSpec, runWorkingSeconds, requestSessionClose, retireSupersededRuns, refreshCardInputs, refreshRelationAnswers, transitionsFor, scopeOf, statusOf, templateValue, workflowFor } from './workflow.mjs';
|
|
28
|
-
import { checkWorkspace, assertWorkspaceOwner, resolveWorkspace } from './workspace.mjs';
|
|
28
|
+
import { checkWorkspace, assertWorkspaceOwner, notePresence, resolveWorkspace } from './workspace.mjs';
|
|
29
29
|
import { evaluate, holds, render } from './expr.mjs';
|
|
30
30
|
import { identifyNotice } from './notices.mjs';
|
|
31
31
|
import { pendingRequestMail } from './request-mail.mjs';
|
|
@@ -123,6 +123,8 @@ export function observeFleet() {
|
|
|
123
123
|
const presence = machines.code === 0 ? lastJson(machines.stdout) : null;
|
|
124
124
|
const online = new Set([localMachine()]);
|
|
125
125
|
for (const m of presence?.items ?? []) if (m.linked) online.add(normalMachine(m.name));
|
|
126
|
+
presenceReading = { known: presence?.connected === true, online };
|
|
127
|
+
notePresence(presenceReading);
|
|
126
128
|
return { ok: true, sessions, alive: new Set(sessions.map((s) => s.address).filter(Boolean)), online,
|
|
127
129
|
presenceKnown: presence?.connected === true, machineEvents: presence,
|
|
128
130
|
error: presence?.connected ? null : 'machine event stream unavailable; retaining existing runs until availability is known' };
|
|
@@ -143,6 +145,7 @@ const sessionAlive = (fleet, session) => fleet.native?.has(addressOf(session)) ?
|
|
|
143
145
|
|
|
144
146
|
/** Native activity comes from the machine's harness discovery, not catalogue age. */
|
|
145
147
|
function nativeSessions(machine, harness) {
|
|
148
|
+
if (offlineMachine(machine)) throw unavailable(`native liveness unavailable: machine ${normalMachine(machine)} is offline`);
|
|
146
149
|
const result = sc(['discover', '--on', machine, '--harness', harness, '--limit', '2000'], { timeout: 120_000 });
|
|
147
150
|
const page = result.code === 0 ? lastJson(result.stdout) : null;
|
|
148
151
|
if (!Array.isArray(page?.sessions)) throw unavailable(`native liveness unavailable on ${machine}: ${(result.stderr || result.stdout).trim().slice(-300)}`);
|
|
@@ -173,13 +176,12 @@ function pidAlive(pid) {
|
|
|
173
176
|
|
|
174
177
|
/**
|
|
175
178
|
* Hermes's default for kanban.max_in_progress when none is set: one worker per 512 MiB of the host's memory, between
|
|
176
|
-
* two and eight
|
|
179
|
+
* two and eight. The memory is the host's own reading (os.totalmem: /proc/meminfo on Linux, hw.memsize on macOS), so
|
|
180
|
+
* the cap holds on every host, not only where /proc exists.
|
|
177
181
|
*/
|
|
178
182
|
function derivedMaxInProgress() {
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
return Number.isFinite(total) && total > 0 ? Math.max(2, Math.min(Math.floor(total / 1024 / 512), 8)) : null;
|
|
182
|
-
} catch { return null; }
|
|
183
|
+
const total = totalmem();
|
|
184
|
+
return Number.isFinite(total) && total > 0 ? Math.max(2, Math.min(Math.floor(total / 1024 / 1024 / 512), 8)) : null;
|
|
183
185
|
}
|
|
184
186
|
|
|
185
187
|
/**
|
|
@@ -219,6 +221,17 @@ export function lanesOf(profiles = {}) {
|
|
|
219
221
|
}
|
|
220
222
|
|
|
221
223
|
const onMachine = machine => normalMachine(machine ?? localMachine()) === localMachine() ? [] : ['--machine', machine];
|
|
224
|
+
// The latest presence reading (observeFleet). A machine it reads as offline is not asked: each remote call would
|
|
225
|
+
// wait out its timeout, and one offline machine held every close and launch on this one.
|
|
226
|
+
let presenceReading = null;
|
|
227
|
+
const offlineMachine = (machine) => {
|
|
228
|
+
const m = normalMachine(machine ?? localMachine());
|
|
229
|
+
return m !== localMachine() && presenceReading?.known === true && !presenceReading.online.has(m);
|
|
230
|
+
};
|
|
231
|
+
/** `supercode <args>` for `machine`: on another machine through its Teams server, answered at once when it is offline. */
|
|
232
|
+
const scOn = (machine, args, options) => offlineMachine(machine)
|
|
233
|
+
? { code: 1, stdout: '', stderr: `machine ${normalMachine(machine)} is offline` }
|
|
234
|
+
: sc([...args, ...onMachine(machine)], options);
|
|
222
235
|
const unavailable = message => Object.assign(new Error(message), { launchDeferred: true });
|
|
223
236
|
|
|
224
237
|
// ------------------------------------------------------------------ supervise
|
|
@@ -242,7 +255,7 @@ function supervise(board, fleet, settings, report, entry, effects) {
|
|
|
242
255
|
for (const { id } of rows) {
|
|
243
256
|
const s = currentRun(board.db, id)?.metadata?.supercode;
|
|
244
257
|
if (!s?.pane || observed.has(s.machine)) continue;
|
|
245
|
-
const r =
|
|
258
|
+
const r = scOn(s.machine, ['teams', 'panes', 'ls'], { timeout: 60_000 });
|
|
246
259
|
const list = r.code === 0 ? json(r.stdout.trim(), null) : null;
|
|
247
260
|
observed.set(s.machine, Array.isArray(list) ? new Set(list.filter(p => !p.dead).map((p) => p.pane)) : null);
|
|
248
261
|
if (!Array.isArray(list)) report.failed.push([id, `pane liveness unavailable; retaining run: ${(r.stderr || r.stdout).trim().slice(-200)}`]);
|
|
@@ -333,13 +346,13 @@ function supervise(board, fleet, settings, report, entry, effects) {
|
|
|
333
346
|
return;
|
|
334
347
|
}
|
|
335
348
|
// a run past its runtime limit (the status's `run.limits.runtime`), as Hermes times its workers out
|
|
336
|
-
if (limits.runtime && runRow &&
|
|
349
|
+
if (limits.runtime && runRow && runWorkingSeconds(runRow, t) > Number(limits.runtime)) {
|
|
337
350
|
observe('timed_out', { limit_seconds: limits.runtime, error: `ran past max_runtime ${limits.runtime}s` }, 'timed_out');
|
|
338
351
|
return;
|
|
339
352
|
}
|
|
340
353
|
if (session?.surface === 'headless') {
|
|
341
354
|
// Hermes's stale check: a worker running past the limit with no heartbeat in the last hour is stopped
|
|
342
|
-
if (limits.stale && runRow &&
|
|
355
|
+
if (limits.stale && runRow && runWorkingSeconds(runRow, t) >= Number(limits.stale) && !(task.last_heartbeat_at && t - Number(task.last_heartbeat_at) < 3600)) {
|
|
343
356
|
observe('stale', { timeout_seconds: limits.stale, pid: session.pid, error: `no heartbeat after ${t - Number(runRow.started_at)}s running` }, 'reclaimed');
|
|
344
357
|
return;
|
|
345
358
|
}
|
|
@@ -357,7 +370,17 @@ function supervise(board, fleet, settings, report, entry, effects) {
|
|
|
357
370
|
if (session?.session_id) {
|
|
358
371
|
if(exits.has(id)) {
|
|
359
372
|
const exit=exits.get(id);
|
|
360
|
-
|
|
373
|
+
// an interactive session that ended cleanly was closed (by a person, or by itself): the workflow says what
|
|
374
|
+
// follows, and it is not a crash to restart
|
|
375
|
+
const ended = { code: exit.failed ? 1 : 0, error: `native session ${session.session_id} ended: ${exit.how}`, session_id: session.session_id, pane: session.pane };
|
|
376
|
+
if (!exit.failed) {
|
|
377
|
+
try {
|
|
378
|
+
const r = fire(db, board, entry, id, 'session_closed', { roles: ['platform'], payload: ended });
|
|
379
|
+
effects.push(...r.effects); (report.closed_by_hand ??= []).push(id);
|
|
380
|
+
return;
|
|
381
|
+
} catch (error) { if (!error.noTransition) { report.failed.push([id, `session_closed: ${String(error.message).slice(-200)}`]); return; } }
|
|
382
|
+
}
|
|
383
|
+
observe('exited', ended, 'crashed');
|
|
361
384
|
return;
|
|
362
385
|
}
|
|
363
386
|
if (session.pane && paneLive === null) return; // no machine reading, no loss verdict
|
|
@@ -413,6 +436,7 @@ function stopSession(board, session) {
|
|
|
413
436
|
if (pidAlive(session.pid)) signal('SIGKILL');
|
|
414
437
|
return;
|
|
415
438
|
}
|
|
439
|
+
if (session.surface !== 'headless' && offlineMachine(session.machine)) throw new Error(`close deferred: machine ${normalMachine(session.machine)} is offline`);
|
|
416
440
|
if (!session.pane && session.surface === 'native') {
|
|
417
441
|
const r = sc(['close', addressOf(session)], { timeout: 120_000, as: boardAddress(board.label) });
|
|
418
442
|
if (r.code !== 0) throw new Error(`closing native session ${addressOf(session)}: ${(r.stderr || r.stdout).trim().slice(-500)}`);
|
|
@@ -527,7 +551,8 @@ function startCards(board, fleet, lanes, settings, report, entry, effects, { onl
|
|
|
527
551
|
if (why) { note(task.id, why); report.queued.push([task.id, why]); continue; }
|
|
528
552
|
if (dry) {
|
|
529
553
|
report.would.push({ id: task.id, status: plan.target, role: plan.runSpec?.role ?? null, assignee: plan.profile, machine: task.machine ?? null, surface: plan.lane.surface });
|
|
530
|
-
|
|
554
|
+
// a card this tick starts counts toward its groups (its scope was read before the start, with no run yet)
|
|
555
|
+
for (const l of limits) { const g = JSON.stringify(evaluate(l.group, scope)); l.tally.set(g, (l.tally.get(g) ?? 0) + 1); }
|
|
531
556
|
continue;
|
|
532
557
|
}
|
|
533
558
|
let moved, checked;
|
|
@@ -549,7 +574,7 @@ function startCards(board, fleet, lanes, settings, report, entry, effects, { onl
|
|
|
549
574
|
continue;
|
|
550
575
|
}
|
|
551
576
|
effects.push(...moved.effects);
|
|
552
|
-
for (const l of limits)
|
|
577
|
+
for (const l of limits) { const g = JSON.stringify(evaluate(l.group, scope)); l.tally.set(g, (l.tally.get(g) ?? 0) + 1); }
|
|
553
578
|
try {
|
|
554
579
|
const session = launchRun(board, entry, task.id, plan.lane, fleet, settings);
|
|
555
580
|
if (session) {
|
|
@@ -690,20 +715,17 @@ export function closeRequested(board, report, { taskId = null, caller = null } =
|
|
|
690
715
|
if (reading.alive) {
|
|
691
716
|
const protectedNow = active.some(r => addressOf(json(r.metadata, {})?.supercode) === addressOf(session));
|
|
692
717
|
if (protectedNow) throw new Error('close deferred: native session is owned by an unended run');
|
|
693
|
-
if (session.lifecycle_notice_pending) throw new Error(`close deferred: lifecycle notice ${session.lifecycle_notice_pending} has not been delivered`);
|
|
694
718
|
stopSession(board, session);
|
|
695
719
|
}
|
|
696
720
|
} else {
|
|
697
721
|
if (!panes.has(machine)) {
|
|
698
|
-
const r =
|
|
722
|
+
const r = scOn(machine, ['teams', 'panes', 'ls'], { timeout: 60_000, as: boardAddress(board.label) });
|
|
699
723
|
const list = r.code === 0 ? json(r.stdout.trim(), null) : null;
|
|
700
724
|
panes.set(machine, Array.isArray(list) ? { live: new Set(list.filter(p => !p.dead).map((p) => p.pane)) } : { error: `cannot read panes on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}` });
|
|
701
725
|
}
|
|
702
726
|
const observed = panes.get(machine);
|
|
703
727
|
if (observed.error) throw new Error(observed.error);
|
|
704
728
|
if (observed.live.has(session.pane)) {
|
|
705
|
-
const pendingNotice = view(board, (db) => db.prepare("SELECT id FROM supercode_effects WHERE json_extract(effect, '$.kind')='session_notice' AND json_extract(effect, '$.runId')=? LIMIT 1").get(row.id));
|
|
706
|
-
if (pendingNotice || session.lifecycle_notice_pending) throw new Error(`close deferred: lifecycle notice ${pendingNotice?.id ?? session.lifecycle_notice_pending} has not been delivered: ${session.lifecycle_notice_error ?? 'pending'}`);
|
|
707
729
|
const protectedNow = view(board, (db) => db.prepare(`SELECT metadata FROM task_runs r WHERE r.ended_at IS NULL OR EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id)`).all())
|
|
708
730
|
.some((r) => { const s = json(r.metadata, {})?.supercode; return s?.pane === session.pane && normalMachine(s.machine) === normalMachine(session.machine); });
|
|
709
731
|
if (protectedNow) throw new Error(`close deferred: pane ${session.pane} is owned by an unended run`);
|
|
@@ -744,7 +766,6 @@ function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
744
766
|
if (!(entry.workflow.statuses[status]?.on?.session_idle || entry.workflow.on?.session_idle || entry.workflow.run?.on?.session_idle)) return;
|
|
745
767
|
const runRow = currentRun(db, id);
|
|
746
768
|
const session = runRow?.metadata?.supercode;
|
|
747
|
-
if (session?.wait_started_at != null) return;
|
|
748
769
|
if (!session?.session_id || !['pane', 'native'].includes(session.surface) || session.offline_since != null) return;
|
|
749
770
|
if (session.surface === 'native' && fleet.native?.get(addressOf(session))?.alive !== true) return;
|
|
750
771
|
const live = fleet.sessions.find((x) => x.address === addressOf(session));
|
|
@@ -801,9 +822,10 @@ export async function sessionNotice(board, e) {
|
|
|
801
822
|
const prior = view(board, (db) => json(db.prepare('SELECT metadata FROM task_runs WHERE id=?').get(e.runId)?.metadata, {})?.supercode);
|
|
802
823
|
if (prior?.close_confirmed_at) {
|
|
803
824
|
tx(board, (db) => {
|
|
825
|
+
// A closed session has nothing to be told; its next run starts with its own prompt.
|
|
804
826
|
const unissued = unissuedLaunch(prior.launch_cancellation, prior);
|
|
805
|
-
event(db, e.id,
|
|
806
|
-
|
|
827
|
+
event(db, e.id, 'session_notice_not_required', { address: e.address, verb: e.verb, notice: e.durableId,
|
|
828
|
+
reason: unissued ? 'machine confirmed launch was not issued; no session exists to notify' : 'session closed',
|
|
807
829
|
...(unissued ? { receipt: prior.launch_cancellation } : {}) }, e.runId);
|
|
808
830
|
if (prior.lifecycle_notice_pending === e.durableId) setRunSession(db, e.runId, { lifecycle_notice_pending: null });
|
|
809
831
|
});
|
|
@@ -814,13 +836,16 @@ export async function sessionNotice(board, e) {
|
|
|
814
836
|
try {
|
|
815
837
|
if (!e.address) throw new Error('lifecycle notice awaits the owned launch session identity');
|
|
816
838
|
receipt = await send(boardAddress(board.label), board.label, e.address, e.text, effectMessageKey(e));
|
|
817
|
-
|
|
839
|
+
// A notice filed in the session's mailbox is done: the session reads its mail at its next
|
|
840
|
+
// turn, and a resumed session first. Only a notice no mailbox took is retried.
|
|
841
|
+
if (!receipt.delivered && !receipt.queued) error = receipt.reason ?? 'notice was not filed'; }
|
|
818
842
|
catch (failure) { error = String(failure.message).slice(-500); }
|
|
819
843
|
tx(board, (db) => {
|
|
820
844
|
const session = json(db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(e.runId)?.metadata, {})?.supercode;
|
|
821
845
|
if (session) setRunSession(db, e.runId, { ...session,
|
|
822
|
-
...(error ? { lifecycle_notice_error: error }
|
|
823
|
-
|
|
846
|
+
...(error ? { lifecycle_notice_error: error }
|
|
847
|
+
: { lifecycle_notice_error: null, ...(session.lifecycle_notice_pending === e.durableId ? { lifecycle_notice_pending: null } : {}) }) });
|
|
848
|
+
if (!error || session?.lifecycle_notice_error !== error) event(db, e.id, error ? 'session_notice_failed' : receipt?.delivered ? 'session_notice_delivered' : 'session_notice_queued', { address: e.address, verb: e.verb, source_event: e.sourceEvent, notice: e.durableId, receipt, ...(error ? { error, retry: true } : {}) }, e.runId);
|
|
824
849
|
});
|
|
825
850
|
return !error;
|
|
826
851
|
}
|
|
@@ -830,10 +855,6 @@ export async function sessionEffect(board, e) {
|
|
|
830
855
|
if (e.kind === 'send_to_actor') {
|
|
831
856
|
const row = view(board, db => e.runId ? db.prepare('SELECT metadata FROM task_runs WHERE id=?').get(e.runId) : null);
|
|
832
857
|
const session = json(row?.metadata, {})?.supercode;
|
|
833
|
-
if (session?.wait_started_at != null) {
|
|
834
|
-
tx(board, db => event(db, e.id, 'actor_notice_suppressed', { notice: e.durableId, reason: 'run is waiting', run_id: e.runId }, e.runId));
|
|
835
|
-
return true;
|
|
836
|
-
}
|
|
837
858
|
const address = addressOf(session);
|
|
838
859
|
let receipt;
|
|
839
860
|
try {
|
|
@@ -894,8 +915,16 @@ const BLOCKER = /\b(quota|rate[\s_-]?limit|429|403|auth\w*|unauthorized|forbidde
|
|
|
894
915
|
const PR_URL = /https?:\/\/github\.com\/[^/\s]+\/[^/\s]+\/pull\/\d+/i;
|
|
895
916
|
|
|
896
917
|
|
|
897
|
-
/**
|
|
918
|
+
/**
|
|
919
|
+
* This machine's memory pressure: on Linux as Hermes reads it (MemAvailable); on macOS the kernel's own level
|
|
920
|
+
* (kern.memorystatus_vm_pressure_level: 1 normal, 2 warning, 4 critical); `unknown` elsewhere, which restricts nothing.
|
|
921
|
+
*/
|
|
898
922
|
function memoryPressure() {
|
|
923
|
+
if (process.platform === 'darwin') {
|
|
924
|
+
const r = spawnSync('sysctl', ['-n', 'kern.memorystatus_vm_pressure_level'], { encoding: 'utf8', timeout: 5_000 });
|
|
925
|
+
const level = Number(String(r.stdout ?? '').trim());
|
|
926
|
+
return level >= 4 ? 'critical' : level >= 2 ? 'elevated' : level === 1 ? 'ok' : 'unknown';
|
|
927
|
+
}
|
|
899
928
|
try {
|
|
900
929
|
const info = Object.fromEntries(readFileSync('/proc/meminfo', 'utf8').split('\n').map((l) => /^(\w+):\s+(\d+)/.exec(l)).filter(Boolean).map((m) => [m[1], Number(m[2])]));
|
|
901
930
|
const avail = info.MemAvailable;
|
|
@@ -933,7 +962,7 @@ const getTaskNow = (board, id) => view(board, (db) => getTask(db, id));
|
|
|
933
962
|
|
|
934
963
|
/** A transport error is not a negative launch receipt. The key remains owned by its run. */
|
|
935
964
|
function launchReceipt(board, session) {
|
|
936
|
-
const r =
|
|
965
|
+
const r = scOn(session.machine, ['teams', 'launches', 'get', session.launch_key],
|
|
937
966
|
{ timeout: 30_000, as: boardAddress(board.label) });
|
|
938
967
|
if (r.code !== 0) throw unavailable(`launch receipt unavailable on ${session.machine}: install the matching Teams CLI and machine capability (teams launches get): ${(r.stderr || r.stdout).trim().slice(-300)}`);
|
|
939
968
|
if (r.stdout.trim() === 'null') return null;
|
|
@@ -943,7 +972,7 @@ function launchReceipt(board, session) {
|
|
|
943
972
|
}
|
|
944
973
|
|
|
945
974
|
function cancelUnissuedLaunch(board, session) {
|
|
946
|
-
const result =
|
|
975
|
+
const result = scOn(session.machine, ['teams', 'launches', 'cancel', session.launch_key],
|
|
947
976
|
{ timeout: 30_000, as: boardAddress(board.label) });
|
|
948
977
|
if (result.code !== 0) throw unavailable(`launch cancellation unavailable on ${session.machine}; retain ${session.launch_key}: ${(result.stderr || result.stdout).trim().slice(-300)}`);
|
|
949
978
|
const receipt = lastJson(result.stdout);
|
|
@@ -1078,6 +1107,10 @@ function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes
|
|
|
1078
1107
|
|
|
1079
1108
|
function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'worker', meta = {} } = {}) {
|
|
1080
1109
|
const machine = normalMachine(task.machine);
|
|
1110
|
+
// Whether a board session opens a terminal window on its machine is the workflow's choice, stated on every launch;
|
|
1111
|
+
// no machine default decides it.
|
|
1112
|
+
const window = workflowFor(board.root).params.session_window;
|
|
1113
|
+
if (typeof window !== 'boolean') throw unavailable('the workflow declares no params.session_window (true or false): whether a board session opens a window on its machine is the workflow\'s choice');
|
|
1081
1114
|
{
|
|
1082
1115
|
const made = resolveLocalWorkspace(board, task);
|
|
1083
1116
|
tx(board, (db) => {
|
|
@@ -1090,7 +1123,7 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
1090
1123
|
if (lane.harness === 'codex' && lane.reasoning_effort) tail.push('-c', `model_reasoning_effort=${JSON.stringify(lane.reasoning_effort)}`);
|
|
1091
1124
|
const harness = HARNESSES[lane.harness] ? lane.harness : 'claude-code';
|
|
1092
1125
|
const how = HARNESSES[harness];
|
|
1093
|
-
const base = ['open', '--new', how.program, '--detach', '--on', task.machine, '--cwd', task.workspace_path];
|
|
1126
|
+
const base = ['open', '--new', how.program, '--detach', `--window=${window}`, '--on', task.machine, '--cwd', task.workspace_path];
|
|
1094
1127
|
// the slot's last session: kept when live, resumed when lost, never another slot's (a reviewer's is its own)
|
|
1095
1128
|
// a close its session's own command asked for is carried out first, so its pane is gone before the slot's next
|
|
1096
1129
|
// session starts (never two processes on one session)
|
|
@@ -1104,7 +1137,7 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
1104
1137
|
if (prev?.session_id && !prev.forgotten && (prev.harness ?? 'claude-code') === harness && prev.machine === machine) {
|
|
1105
1138
|
const fresh = observeFleet();
|
|
1106
1139
|
if (!fresh.ok) throw unavailable(`cannot check for a manual continuation of ${prev.session_id}: ${fresh.error}`);
|
|
1107
|
-
const r =
|
|
1140
|
+
const r = scOn(machine, ['teams', 'panes', 'ls'], { timeout: 60_000 });
|
|
1108
1141
|
const listed = r.code === 0 ? json(r.stdout.trim(), null) : null;
|
|
1109
1142
|
if (!Array.isArray(listed)) throw unavailable(`cannot read panes before continuing ${prev.session_id}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
|
|
1110
1143
|
const pane = continuationPane(fresh, prev, new Set(listed.filter(p => !p.dead).map((p) => p.pane)));
|
|
@@ -1237,7 +1270,7 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
1237
1270
|
|
|
1238
1271
|
/** A machine reading, not a missing catalog entry, establishes whether a pane is alive. */
|
|
1239
1272
|
function panePresent(machine, pane) {
|
|
1240
|
-
const r =
|
|
1273
|
+
const r = scOn(machine, ['teams', 'panes', 'ls'], { timeout: 60_000 });
|
|
1241
1274
|
const panes = r.code === 0 ? json(r.stdout.trim(), null) : null;
|
|
1242
1275
|
if (!Array.isArray(panes)) throw unavailable(`cannot establish pane ${pane} liveness on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
|
|
1243
1276
|
return panes.some((p) => p.pane === pane && !p.dead);
|
|
@@ -1314,14 +1347,39 @@ function resolveLocalWorkspace(board, task) {
|
|
|
1314
1347
|
* never as the session describes itself. From then on the board watches it, resumes it when lost, and tells it of
|
|
1315
1348
|
* its card. `who` is an address, a session id, or a `name@machine` the fleet lists.
|
|
1316
1349
|
*/
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1350
|
+
/**
|
|
1351
|
+
* The manager asks the board to adopt session `who` for card `id`: the card is held at once (no replacement can be
|
|
1352
|
+
* launched), and the dispatcher's next tick verifies the session from the fleet reading it already takes and binds
|
|
1353
|
+
* it (`completeAdoptions`). The verb does no process I/O, so it answers as fast as any board write.
|
|
1354
|
+
*/
|
|
1355
|
+
export function requestAdoption(board, id, who, { workspace = null, assignee = null, role = null } = {}) {
|
|
1356
|
+
if (!who) throw new Error('name the session to adopt');
|
|
1320
1357
|
tx(board, db => {
|
|
1321
1358
|
if (!getTask(db, id)) throw new Error(`no card ${id}`);
|
|
1322
|
-
event(db, id, 'dispatch_held', { session: who, reason: 'adoption requested' });
|
|
1359
|
+
event(db, id, 'dispatch_held', { session: who, reason: 'adoption requested', workspace, assignee, role });
|
|
1323
1360
|
});
|
|
1324
|
-
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
/** Adoptions the manager requested and no tick has answered yet, completed from this tick's fleet reading. */
|
|
1364
|
+
export function completeAdoptions(board, fleet, settings, report) {
|
|
1365
|
+
const pending = view(board, db => db.prepare(`SELECT e.task_id, e.payload FROM task_events e WHERE e.kind = 'dispatch_held'
|
|
1366
|
+
AND json_extract(e.payload, '$.reason') = 'adoption requested'
|
|
1367
|
+
AND e.id = (SELECT MAX(id) FROM task_events l WHERE l.task_id = e.task_id AND l.kind IN ('dispatch_held','dispatch_released','session_adopted','adoption_failed'))`).all());
|
|
1368
|
+
for (const row of pending) {
|
|
1369
|
+
const asked = json(row.payload, {}) ?? {};
|
|
1370
|
+
try {
|
|
1371
|
+
const got = adoptSession(board, row.task_id, asked.session, { workspace: asked.workspace ?? null, assignee: asked.assignee ?? null, role: asked.role ?? null, ttl: settings.ttl, fleet: fleet?.ok ? fleet : null });
|
|
1372
|
+
(report.adopted ??= []).push([row.task_id, got.address]);
|
|
1373
|
+
} catch (error) {
|
|
1374
|
+
const message = String(error.message).slice(-400);
|
|
1375
|
+
tx(board, db => event(db, row.task_id, 'adoption_failed', { session: asked.session, error: message }));
|
|
1376
|
+
report.failed.push([row.task_id, `adoption of ${asked.session}: ${message}`]);
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
function adoptSession(board, id, who, { workspace = null, ttl, assignee = null, role = null, fleet = null } = {}) {
|
|
1382
|
+
fleet ??= observeFleet();
|
|
1325
1383
|
if (!fleet.ok) throw new Error(`supercode could not read the fleet: ${fleet.error}`);
|
|
1326
1384
|
const found = fleet.sessions.find((s) => s.address === who || s.address.endsWith(`:${who}`) || s.name === who);
|
|
1327
1385
|
if (!found) throw new Error(`the fleet lists no live session ${who}`);
|
|
@@ -1424,7 +1482,7 @@ export async function sendBoardMessages(board, report, taskId = null) {
|
|
|
1424
1482
|
const task = view(board, db => getTask(db, first.task_id));
|
|
1425
1483
|
const subject = mail.requester ? `State requests · ${mail.members.length} request(s) · ${task?.title || first.task_id}` : `${task?.title || first.task_id} · ${mail.message_id ? 'Question' : 'Update'}`;
|
|
1426
1484
|
const receipt = await send(boardAddress(board.label), board.label, mail.to, mail.text, mail.message_id ?? `board-message-${first.id}`, { ask: Boolean(mail.message_id), subject });
|
|
1427
|
-
if (!receipt.delivered) throw new Error(receipt.reason ?? '
|
|
1485
|
+
if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'message was not filed');
|
|
1428
1486
|
tx(board, db => {
|
|
1429
1487
|
for (const member of mail.members) event(db, member.task_id, 'board_message_sent', { event: member.id, to: mail.to, receipt, ...(mail.message_id ? { message_id: mail.message_id, request_id: member.message_id ?? mail.message_id } : {}) });
|
|
1430
1488
|
});
|
|
@@ -1502,8 +1560,11 @@ async function notify(board, fleet, report, entry) {
|
|
|
1502
1560
|
if (!events.length) return;
|
|
1503
1561
|
// the card as the workflow names it (its status the workflow's: triaging, reviewing, stopped; `stored_as` kanban.db's)
|
|
1504
1562
|
const task = view(board, (db) => { const t = getTask(db, sub.task_id); return t ? { ...t, status: statusOf(db, t), stored_as: t.status } : t; });
|
|
1563
|
+
// a worker whose run has ended is not woken by a notice (a second session would start working the card beside
|
|
1564
|
+
// the one the next run gets): the notice is filed in its mailbox, read when it next works
|
|
1565
|
+
const working = role !== 'worker' || view(board, (db) => addressOf(currentRun(db, sub.task_id)?.metadata?.supercode) === sub.chat_id);
|
|
1505
1566
|
let cursor = Number(sub.last_event_id);
|
|
1506
|
-
let failed = null
|
|
1567
|
+
let failed = null;
|
|
1507
1568
|
try {
|
|
1508
1569
|
for (const ev of events) {
|
|
1509
1570
|
const p = json(ev.payload, {}) ?? {};
|
|
@@ -1517,9 +1578,11 @@ async function notify(board, fleet, report, entry) {
|
|
|
1517
1578
|
} else text = noticeOf(entry, role, ev.kind, task, p);
|
|
1518
1579
|
if (text) {
|
|
1519
1580
|
const body = ['worker', 'dependency'].includes(role) ? identifyNotice(ev, text) : text;
|
|
1520
|
-
const receipt = await send(from, board.label, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: !entry.workflow.run && sub.delivery_mode === 'notify', subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
|
|
1521
|
-
|
|
1522
|
-
|
|
1581
|
+
const receipt = await send(from, board.label, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: !working || (!entry.workflow.run && sub.delivery_mode === 'notify'), subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
|
|
1582
|
+
// A notice the recipient's mailbox took is filed: the machine delivers it at the session's next turn, so the
|
|
1583
|
+
// board does not send it again.
|
|
1584
|
+
if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'notice was not filed');
|
|
1585
|
+
tx(board, (db) => event(db, sub.task_id, receipt.delivered ? 'notice_delivered' : 'notice_filed', { source_event: ev.id, kind: ev.kind, address: sub.chat_id, receipt }));
|
|
1523
1586
|
report.notified.push([sub.task_id, ev.kind, sub.chat_id]);
|
|
1524
1587
|
}
|
|
1525
1588
|
cursor = ev.id;
|
|
@@ -1527,7 +1590,7 @@ async function notify(board, fleet, report, entry) {
|
|
|
1527
1590
|
} catch (error) {
|
|
1528
1591
|
failed = String(error.message).slice(-200);
|
|
1529
1592
|
tx(board, (db) => {
|
|
1530
|
-
const kind =
|
|
1593
|
+
const kind = 'notice_failed';
|
|
1531
1594
|
const previous = db.prepare('SELECT payload FROM task_events WHERE task_id=? AND kind=? ORDER BY id DESC LIMIT 1').get(sub.task_id, kind);
|
|
1532
1595
|
const payload = { after_event: cursor, address: sub.chat_id, error: failed, retry: true };
|
|
1533
1596
|
if (JSON.stringify(json(previous?.payload, {})) !== JSON.stringify(payload)) event(db, sub.task_id, kind, payload);
|
|
@@ -1539,7 +1602,7 @@ async function notify(board, fleet, report, entry) {
|
|
|
1539
1602
|
const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
|
|
1540
1603
|
const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
|
|
1541
1604
|
db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(cursor, ...identity, cursor);
|
|
1542
|
-
const failures = failed
|
|
1605
|
+
const failures = failed ? Number(sub.delivery_metadata?.failures ?? 0) + 1 : 0;
|
|
1543
1606
|
if (failures >= MAX_SEND_FAILURES) { db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity); return; }
|
|
1544
1607
|
if (failures !== Number(sub.delivery_metadata?.failures ?? 0)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures }), ...identity);
|
|
1545
1608
|
if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity);
|
|
@@ -1778,38 +1841,46 @@ export async function plan({ root, lanes, settings, slugs = null, params = {} })
|
|
|
1778
1841
|
}
|
|
1779
1842
|
|
|
1780
1843
|
/** An opened PR asks for the declared review transition once the arc's tasks are closed. */
|
|
1844
|
+
// A task written in the card's own Tasks section that did not become a subtask holds its review too.
|
|
1781
1845
|
async function activatePullRequestReviews(board, entry, report, effects, { only = null } = {}) {
|
|
1782
1846
|
const candidates = view(board, db => db.prepare(`SELECT t.id FROM tasks t JOIN supercode_cards c ON c.task_id=t.id
|
|
1783
|
-
WHERE c.subtask_of IS NULL AND
|
|
1847
|
+
WHERE c.subtask_of IS NULL AND c.every IS NULL
|
|
1784
1848
|
AND t.workspace_kind!='scratch' ${only ? 'AND t.id=?' : ''}`).all(...(only ? [only] : [])));
|
|
1785
1849
|
for (const { id } of candidates) {
|
|
1786
1850
|
const eligible = db => {
|
|
1787
1851
|
const task = getTask(db, id);
|
|
1788
1852
|
if (!task) return false;
|
|
1789
1853
|
const scope = scopeOf(db, board, task, { params: entry.params });
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1854
|
+
if (!transitionsFor(entry.workflow, statusOf(db, task), 'request_review').some(t => !t.by || t.by.includes('platform')) ||
|
|
1855
|
+
scope.card.dispatch_hold || scope.card.paused || scope.card.blocked_by.length) return false;
|
|
1856
|
+
// The card's Tasks are its subtasks; an open one holds the review, and says so.
|
|
1857
|
+
const open = scope.card.subtasks.filter(t => !['done', 'cancelled'].includes(t.status));
|
|
1858
|
+
return open.length ? open : true;
|
|
1793
1859
|
};
|
|
1794
|
-
|
|
1860
|
+
const ready = view(board, eligible);
|
|
1861
|
+
if (!ready) continue;
|
|
1862
|
+
if (ready !== true) { report.queued.push([id, `review waits: ${ready.length} open task(s) in the card's Tasks: ${ready[0].id} ${ready[0].title}`]); continue; }
|
|
1863
|
+
let payload = null;
|
|
1795
1864
|
try {
|
|
1796
|
-
|
|
1797
|
-
|
|
1865
|
+
payload = openReviewPullRequests(board, id);
|
|
1866
|
+
// The arc's open PRs ask for one review only while some head has no verdict: after changes are
|
|
1867
|
+
// requested the next run is the implementer's, until a new head is pushed.
|
|
1868
|
+
if (!payload || !reviewDue(board, id, payload.pr.map((url, index) => ({ url, head: payload.pr_head[index] })))) continue;
|
|
1798
1869
|
const evidence = inspectCompletionEvidence(board, id, entry, payload, 'request_review');
|
|
1799
1870
|
if (!evidence.ready) throw new Error(evidence.reason);
|
|
1800
1871
|
refreshCardInputs(board, id, { commits: true });
|
|
1801
1872
|
const moved = tx(board, db => {
|
|
1802
1873
|
const task = getTask(db, id);
|
|
1803
|
-
if (
|
|
1874
|
+
if (eligible(db) !== true) return null;
|
|
1804
1875
|
return withCompletionEvidence(board, id, evidence, () => fire(db, board, entry, id, 'request_review', {
|
|
1805
|
-
roles: ['platform'], payload: { ...payload, run_id: task.current_run_id, summary: `Opened
|
|
1876
|
+
roles: ['platform'], payload: { ...payload, run_id: task.current_run_id, summary: `Opened ${payload.pr.map((url, index) => `${url} at ${payload.pr_head[index]}`).join(', ')}`, ci: evidence }
|
|
1806
1877
|
}));
|
|
1807
1878
|
});
|
|
1808
1879
|
if (moved) {
|
|
1809
1880
|
effects.push(...moved.effects);
|
|
1810
|
-
(report.review_requested ??= []).push({ id,
|
|
1881
|
+
(report.review_requested ??= []).push({ id, prs: payload.pr, heads: payload.pr_head });
|
|
1811
1882
|
}
|
|
1812
|
-
} catch (error) { report.queued.push([id, `PR review waits: ${error.message}`]); }
|
|
1883
|
+
} catch (error) { report.queued.push([id, `PR review waits${payload ? ` (${payload.pr.join(', ')})` : ''}: ${error.message}`]); }
|
|
1813
1884
|
}
|
|
1814
1885
|
}
|
|
1815
1886
|
|
|
@@ -1857,7 +1928,7 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
|
|
|
1857
1928
|
readMailbox(board, boardAddress(board.label), report); phase('mailbox');
|
|
1858
1929
|
await refreshRelationAnswers(board); phase('answers');
|
|
1859
1930
|
const effects = [];
|
|
1860
|
-
reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); await runEffects(board, effects.splice(0), { retryNotices: true }); closeRequested(board, report); phase('supervise');
|
|
1931
|
+
completeAdoptions(board, fleet, settings, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); await runEffects(board, effects.splice(0), { retryNotices: true }); closeRequested(board, report); phase('supervise');
|
|
1861
1932
|
report.unsettled = [];
|
|
1862
1933
|
report.promoted = recomputeReady(board, effects, report.unsettled);
|
|
1863
1934
|
if (dispatch) await activatePullRequestReviews(board, entry, report, effects, { only });
|