@volter/supercode-orchestrator 0.4.102 → 0.4.104
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/board/cli.mjs +8 -21
- package/board/dispatch.mjs +100 -41
- package/board/engine.mjs +7 -5
- package/board/workflow.mjs +20 -1
- package/package.json +1 -1
package/board/cli.mjs
CHANGED
|
@@ -12,8 +12,8 @@ import { read } from '../orchestration.mjs';
|
|
|
12
12
|
import { view, backingPaths, layOut, backingOf, boardExists, boardPath, DEFAULT_BOARD, event, json, listBoards, logsDir, now, openBoard, setActor, tx } from './store.mjs';
|
|
13
13
|
import { openZtrackBoard } from './ztrack.mjs';
|
|
14
14
|
import * as engine from './engine.mjs';
|
|
15
|
-
import { adoptSession, boardAddress, boardLabel, boardSettings, callerSession, lanesOf, localMachine, observeFleet, paused, plan, startCard, tick } from './dispatch.mjs';
|
|
16
|
-
import { loadWorkflow, scopeOf, statusOf, useWorkflow, workflowEntry, workflowFor } from './workflow.mjs';
|
|
15
|
+
import { adoptSession, boardAddress, boardLabel, boardSettings, callerSession, closeRequested, lanesOf, localMachine, observeFleet, paused, plan, startCard, tick } from './dispatch.mjs';
|
|
16
|
+
import { loadWorkflow, requestSessionClose, scopeOf, statusOf, useWorkflow, workflowEntry, workflowFor } from './workflow.mjs';
|
|
17
17
|
import { evaluate } from './expr.mjs';
|
|
18
18
|
|
|
19
19
|
const HELP = `supercode workflow <verb> [--root <home>] [--board <slug>] [--json]
|
|
@@ -1104,32 +1104,19 @@ function authorize(db, verb, rest, args, own) {
|
|
|
1104
1104
|
/** Close the live sessions of a card that is not running (its worker's and its reviewers'), freeing its machine's place. */
|
|
1105
1105
|
function closeSession(board, id) {
|
|
1106
1106
|
const task = engine.getTask(board.db, id);
|
|
1107
|
-
const rows = board.db.prepare('SELECT id, metadata FROM task_runs WHERE task_id = ? AND metadata IS NOT NULL ORDER BY id DESC').all(id);
|
|
1108
1107
|
if (!task) throw new Error(`no card ${id}`);
|
|
1109
1108
|
if (task.current_run_id != null) throw new Error(`${id} has an active run; pause, block or reclaim it before closing its session`);
|
|
1110
|
-
|
|
1111
|
-
const
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
seen.add(session.session_id);
|
|
1116
|
-
// the board closes its card's pane, whichever session ran `workflow close`
|
|
1117
|
-
const r = spawnSync(process.env.SUPERCODE_BIN || 'supercode', ['teams', 'panes', 'close', session.pane, '--machine', session.machine], { cwd: tmpdir(), encoding: 'utf8', timeout: 60_000, env: { ...process.env, SUPERCODE_CALLER: boardAddress(board.label) } });
|
|
1118
|
-
if (r.status !== 0) throw new Error(`closing pane ${session.pane} on ${session.machine}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
|
|
1119
|
-
tx(board, (db) => {
|
|
1120
|
-
const whole = json(db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(row.id)?.metadata, {}) ?? {};
|
|
1121
|
-
db.prepare('UPDATE task_runs SET metadata = ? WHERE id = ?').run(JSON.stringify({ ...whole, supercode: { ...whole.supercode, closed_at: now() } }), row.id);
|
|
1122
|
-
event(db, id, 'session_closed', { session_id: session.session_id, machine: session.machine, pane: session.pane });
|
|
1123
|
-
});
|
|
1124
|
-
closed.push(session);
|
|
1125
|
-
}
|
|
1126
|
-
return closed.length ? closed : null;
|
|
1109
|
+
tx(board, (db) => requestSessionClose(db, id));
|
|
1110
|
+
const report = { undelivered: [], closed: [] };
|
|
1111
|
+
closeRequested(board, report, { taskId: id, caller: callerSession() });
|
|
1112
|
+
if (report.undelivered.length) throw new Error(`close remains queued for retry: ${report.undelivered.map((r) => r[2]).join('; ')}`);
|
|
1113
|
+
return report.closed.length ? report.closed : null;
|
|
1127
1114
|
}
|
|
1128
1115
|
|
|
1129
1116
|
export function summary(r) {
|
|
1130
1117
|
const parts = [`board ${r.board}:`];
|
|
1131
1118
|
if (r.error) parts.push(`error ${r.error}`);
|
|
1132
|
-
for (const [k, label] of [['mailed', 'mailed updates'], ['started', 'started'], ['queued', 'waiting'], ['failed', 'failed'], ['lost', 'lost'], ['crashed', 'crashed'], ['timed_out', 'timed out'], ['reclaimed', 'reclaimed'], ['auto_blocked', 'gave up'], ['notified', 'notices']]) {
|
|
1119
|
+
for (const [k, label] of [['mailed', 'mailed updates'], ['started', 'started'], ['queued', 'waiting'], ['failed', 'failed'], ['lost', 'lost'], ['crashed', 'crashed'], ['timed_out', 'timed out'], ['reclaimed', 'reclaimed'], ['auto_blocked', 'gave up'], ['notified', 'notices'], ['closed', 'closed sessions'], ['undelivered', 'undelivered (retrying)']]) {
|
|
1133
1120
|
if (r[k]?.length) parts.push(`${label} ${r[k].length}`);
|
|
1134
1121
|
}
|
|
1135
1122
|
if (r.promoted) parts.push(`promoted ${r.promoted}`);
|
package/board/dispatch.mjs
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
//
|
|
7
7
|
// - `pane`: an interactive session of the lane's harness (Claude Code or Codex) on the card's machine (`supercode open
|
|
8
8
|
// --new … --on <machine>`), its session id recorded on the run: one the dispatcher chose (Claude Code's
|
|
9
|
-
// `--session-id`) or the one the harness named, read from the fleet by its pane (Codex). It
|
|
9
|
+
// `--session-id`) or the one the harness named, read from the fleet by its pane (Codex). It closes when its run ends. It is live
|
|
10
10
|
// while supercode's fleet lists it; one missing for the whole grace on an online machine is lost, and the next start
|
|
11
11
|
// resumes it in its own harness. Nothing asks the session about itself.
|
|
12
12
|
// - `headless`: the profile's worker for one query on this machine (`bin/hermes-worker.mjs`, Hermes's worker
|
|
@@ -22,7 +22,7 @@ import { basename, dirname, join } from 'node:path';
|
|
|
22
22
|
import { randomUUID } from 'node:crypto';
|
|
23
23
|
import { fileURLToPath } from 'node:url';
|
|
24
24
|
import { view, boardPath, claimer, event, json, listBoards, logsDir, now, openBoard, setActor, tx } from './store.mjs';
|
|
25
|
-
import { doors, fire, loadWorkflow, resumeOf, runSpec, refreshRelationAnswers, transitionsFor, scopeOf, statusOf, templateValue, workflowFor } from './workflow.mjs';
|
|
25
|
+
import { doors, fire, loadWorkflow, resumeOf, runSpec, requestSessionClose, refreshRelationAnswers, transitionsFor, scopeOf, statusOf, templateValue, workflowFor } from './workflow.mjs';
|
|
26
26
|
import { evaluate, holds, render } from './expr.mjs';
|
|
27
27
|
import {
|
|
28
28
|
cardContext, claimTtl, currentRun, DEFAULT_FAILURE_LIMIT, extendClaim, getTask, lastSession,
|
|
@@ -284,7 +284,10 @@ function stopSession(board, session) {
|
|
|
284
284
|
if (pidAlive(session.pid)) signal('SIGKILL');
|
|
285
285
|
return;
|
|
286
286
|
}
|
|
287
|
-
if (session.pane)
|
|
287
|
+
if (session.pane) {
|
|
288
|
+
const r = sc(['teams', 'panes', 'close', session.pane, ...(session.machine ? ['--machine', session.machine] : [])], { timeout: 60_000, as: boardAddress(board.label) });
|
|
289
|
+
if (r.code !== 0 || lastJson(r.stdout)?.closed !== session.pane) throw new Error(`closing pane ${session.pane}: ${(r.stderr || r.stdout || `exit ${r.code}`).trim().slice(-500)}`);
|
|
290
|
+
}
|
|
288
291
|
}
|
|
289
292
|
|
|
290
293
|
// ------------------------------------------------------------------ starting, as the workflow's dispatch says
|
|
@@ -458,18 +461,54 @@ export async function startCard(board, id, { roles = ['manager'], lanes, setting
|
|
|
458
461
|
* The heartbeat, observed: a running pane session whose last message is a whole tick old is sent `session_idle` with
|
|
459
462
|
* how many times it has been asked this run and whether it worked since the last ask; the workflow decides the rest.
|
|
460
463
|
*/
|
|
461
|
-
/**
|
|
462
|
-
|
|
463
|
-
|
|
464
|
+
/** Reconcile durable close intent and ended runs, including old unverified closed_at records.
|
|
465
|
+
* Reads and process I/O stay outside the writer. An active successor owns its pane until it ends. */
|
|
466
|
+
export function closeRequested(board, report, { taskId = null, caller = null } = {}) {
|
|
467
|
+
const rows = view(board, (db) => db.prepare(`SELECT id, task_id, ended_at, metadata FROM task_runs
|
|
468
|
+
WHERE metadata IS NOT NULL ${taskId ? 'AND task_id = ?' : ''}`).all(...(taskId ? [taskId] : [])));
|
|
469
|
+
const panes = new Map();
|
|
464
470
|
for (const row of rows) {
|
|
465
471
|
const session = json(row.metadata, {})?.supercode;
|
|
466
|
-
if (!session?.pane) continue;
|
|
467
|
-
|
|
468
|
-
const
|
|
472
|
+
if (!session?.pane || session.close_confirmed_at || (!row.ended_at && !session.close_requested)) continue;
|
|
473
|
+
if (caller && session.session_id === caller) continue; // let the command that ended its own run return
|
|
474
|
+
const active = view(board, (db) => db.prepare(`SELECT r.id, r.metadata FROM tasks t JOIN task_runs r ON r.id = t.current_run_id`).all());
|
|
475
|
+
if (active.some((r) => {
|
|
476
|
+
const s = json(r.metadata, {})?.supercode;
|
|
477
|
+
return r.id !== row.id && s?.pane === session.pane && normalMachine(s.machine) === normalMachine(session.machine);
|
|
478
|
+
})) continue;
|
|
479
|
+
// Persist retries even for historical ends that did not have a close_session action.
|
|
480
|
+
tx(board, (db) => requestSessionClose(db, row.task_id, { runId: row.id }));
|
|
481
|
+
let error = null;
|
|
482
|
+
try {
|
|
483
|
+
const machine = session.machine ?? localMachine();
|
|
484
|
+
if (!panes.has(machine)) {
|
|
485
|
+
const r = sc(['teams', 'panes', 'ls', '--machine', machine], { timeout: 60_000, as: boardAddress(board.label) });
|
|
486
|
+
const list = r.code === 0 ? json(r.stdout.trim(), null) : null;
|
|
487
|
+
panes.set(machine, Array.isArray(list) ? { live: new Set(list.map((p) => p.pane)) } : { error: `cannot read panes on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}` });
|
|
488
|
+
}
|
|
489
|
+
const observed = panes.get(machine);
|
|
490
|
+
if (observed.error) throw new Error(observed.error);
|
|
491
|
+
if (observed.live.has(session.pane)) { stopSession(board, session); observed.live.delete(session.pane); }
|
|
492
|
+
} catch (e) { error = String(e.message).slice(-500); }
|
|
469
493
|
tx(board, (db) => {
|
|
470
|
-
|
|
471
|
-
|
|
494
|
+
const current = json(db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(row.id)?.metadata, {})?.supercode;
|
|
495
|
+
if (!current || current.pane !== session.pane) return;
|
|
496
|
+
const at = now();
|
|
497
|
+
if (error) {
|
|
498
|
+
setRunSession(db, row.id, { ...current, close_attempted_at: at, close_attempts: Number(current.close_attempts ?? 0) + 1, close_error: error });
|
|
499
|
+
// Retain every attempt's latest fact, but avoid an unbounded thread of identical failures.
|
|
500
|
+
if (current.close_error !== error) event(db, row.task_id, 'session_close_failed', { session_id: session.session_id, pane: session.pane, error, retry: true }, row.id);
|
|
501
|
+
} else {
|
|
502
|
+
const { close_error: _error, ...rest } = current;
|
|
503
|
+
setRunSession(db, row.id, { ...rest, close_error: null, closed_at: at, close_confirmed_at: at });
|
|
504
|
+
const address = addressOf(session);
|
|
505
|
+
const subscribed = active.some((r) => addressOf(json(r.metadata, {})?.supercode) === address && r.id !== row.id);
|
|
506
|
+
if (!subscribed) db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ?").run(row.task_id, address);
|
|
507
|
+
event(db, row.task_id, 'session_closed', { session_id: session.session_id, pane: session.pane, machine: session.machine }, row.id);
|
|
508
|
+
}
|
|
472
509
|
});
|
|
510
|
+
if (error) report.undelivered.push([row.task_id, addressOf(session), error]);
|
|
511
|
+
else (report.closed ??= []).push({ task_id: row.task_id, ...session });
|
|
473
512
|
}
|
|
474
513
|
}
|
|
475
514
|
|
|
@@ -514,25 +553,9 @@ export async function sessionEffect(board, e) {
|
|
|
514
553
|
await send(boardAddress(board.label), board.label, address, e.text, `board-actor-${e.id}-${now()}`);
|
|
515
554
|
return;
|
|
516
555
|
}
|
|
517
|
-
if (e.kind === 'close_session') {
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
// a run newer than the one that left the status is its successor, already started in this tick: not closed
|
|
521
|
-
if (e.runId && row.id > e.runId) continue;
|
|
522
|
-
const session = json(row.metadata, {})?.supercode;
|
|
523
|
-
if ((session?.slot ?? 'worker') !== e.slot || !session?.pane || session.closed_at) continue;
|
|
524
|
-
// the session that sent the event is not closed under its own command: the dispatcher closes it next tick
|
|
525
|
-
const caller = callerSession();
|
|
526
|
-
if (caller && caller === session.session_id) { tx(board, (db) => setRunSession(db, row.id, { ...session, close_requested: now() })); return; }
|
|
527
|
-
stopSession(board, session);
|
|
528
|
-
// a closed session hears nothing more of the card
|
|
529
|
-
const address = addressOf(session);
|
|
530
|
-
tx(board, (db) => {
|
|
531
|
-
setRunSession(db, row.id, { ...session, closed_at: now() });
|
|
532
|
-
db.prepare("DELETE FROM kanban_notify_subs WHERE task_id = ? AND platform = 'supercode' AND chat_id = ?").run(e.id, address);
|
|
533
|
-
});
|
|
534
|
-
return;
|
|
535
|
-
}
|
|
556
|
+
if (e.kind === 'close_session' || e.kind === 'run_ended') {
|
|
557
|
+
tx(board, (db) => requestSessionClose(db, e.id, e.kind === 'run_ended' ? { runId: e.runId } : { slot: e.slot, through: e.runId }));
|
|
558
|
+
// The serve tick drains the queue after the originating command has returned. Failures remain durable.
|
|
536
559
|
}
|
|
537
560
|
}
|
|
538
561
|
|
|
@@ -628,17 +651,36 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
628
651
|
// the slot's last session: kept when live, resumed when lost, never another slot's (a reviewer's is its own)
|
|
629
652
|
// a close its session's own command asked for is carried out first, so its pane is gone before the slot's next
|
|
630
653
|
// session starts (never two processes on one session)
|
|
654
|
+
// Every old pane in the slot must be settled, including a failed launch with no session id.
|
|
655
|
+
closeRequested(board, { undelivered: [] }, { taskId: task.id });
|
|
656
|
+
const pending = view(board, (db) => db.prepare('SELECT id, ended_at, metadata FROM task_runs WHERE task_id = ? AND id != ?').all(task.id, runId))
|
|
657
|
+
.map((r) => ({ ...r, session: json(r.metadata, {})?.supercode }))
|
|
658
|
+
.find((r) => r.session?.pane && (r.session.slot ?? 'worker') === slot && (r.ended_at || r.session.close_requested) && !r.session.close_confirmed_at);
|
|
659
|
+
if (pending) throw new Error(`slot ${slot} still owns pane ${pending.session.pane} from run ${pending.id}; closure must be confirmed before another session starts: ${pending.session.close_error ?? 'close pending'}`);
|
|
631
660
|
let prev = view(board, (db) => lastSession(db, task.id, slot));
|
|
632
661
|
if (prev?.close_requested && !prev.closed_at) {
|
|
633
662
|
const report = { undelivered: [] };
|
|
634
|
-
closeRequested(board, report);
|
|
663
|
+
closeRequested(board, report, { taskId: task.id });
|
|
635
664
|
prev = view(board, (db) => lastSession(db, task.id, slot));
|
|
636
665
|
// a pane that could not be closed still holds the session: no second process is started on it
|
|
637
666
|
if (prev?.close_requested && !prev.closed_at) throw new Error(`the slot's session ${prev.session_id} asked to be closed and its pane ${prev.pane} could not be: ${report.undelivered.map((u) => u[2]).join('; ') || 'no reason given'}`);
|
|
638
667
|
}
|
|
668
|
+
// Record the pane before waiting for the harness to name itself. A failed launch still owns a process.
|
|
669
|
+
const rememberPane = (pane, sessionId, mode) => {
|
|
670
|
+
if (!pane) return;
|
|
671
|
+
tx(board, (db) => {
|
|
672
|
+
const row = db.prepare('SELECT metadata, ended_at FROM task_runs WHERE id = ?').get(runId);
|
|
673
|
+
const before = json(row?.metadata, {})?.supercode ?? {};
|
|
674
|
+
setRunSession(db, runId, { ...before, ...meta, surface: 'pane', session_id: sessionId, harness, machine, pane, slot, mode, launched_at: now() });
|
|
675
|
+
if (row?.ended_at) requestSessionClose(db, task.id, { runId });
|
|
676
|
+
});
|
|
677
|
+
};
|
|
639
678
|
let session = null;
|
|
640
679
|
let replaced = null;
|
|
641
|
-
if (prev?.forgotten)
|
|
680
|
+
if (prev?.forgotten) {
|
|
681
|
+
if (prev.pane && panePresent(prev.machine, prev.pane)) throw new Error(`forgotten session ${prev.session_id} still has live pane ${prev.pane}; close it before replacement`);
|
|
682
|
+
replaced = { session_id: prev.session_id, why: 'the workflow let it go' };
|
|
683
|
+
}
|
|
642
684
|
else if (prev && (prev.harness ?? 'claude-code') !== harness) {
|
|
643
685
|
// the card was reassigned to a lane of another harness: that harness starts it fresh (the card's text and thread
|
|
644
686
|
// are its context), and the last session, if still open, is closed so two harnesses never work one card
|
|
@@ -647,9 +689,11 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
647
689
|
} else if (prev && prev.machine === machine) {
|
|
648
690
|
// a session the board closed, or one whose close its own command asked for (the next tick's sweep closes that
|
|
649
691
|
// run's pane), is not kept, whatever a fleet reading taken before the close still lists
|
|
650
|
-
|
|
692
|
+
const paneLive = prev.pane ? panePresent(prev.machine, prev.pane) : false;
|
|
693
|
+
if (paneLive && (prev.closed_at || prev.close_requested || prev.forgotten)) throw new Error(`session ${prev.session_id} still has live pane ${prev.pane}; cannot resume or replace it`);
|
|
694
|
+
if (!prev.closed_at && !prev.close_requested && (prev.pane ? paneLive : sessionAlive(fleet, prev))) {
|
|
651
695
|
// the session is kept; the heartbeat's count belongs to the run that ended, so the new run starts it again
|
|
652
|
-
const { missing_since: _m, goal_turns: _t, goal_nudged_at: _n, close_requested: _q, closed_at: _c, ...kept } = prev;
|
|
696
|
+
const { missing_since: _m, goal_turns: _t, goal_nudged_at: _n, close_requested: _q, closed_at: _c, close_confirmed_at: _cc, close_error: _ce, close_attempts: _ca, close_attempted_at: _ct, ...kept } = prev;
|
|
653
697
|
session = { ...kept, mode: 'adopt', launched_at: now(), seen_at: now() };
|
|
654
698
|
// a kept session is told what this run is for (its card came back to it, with what happened since)
|
|
655
699
|
if (input) deliverNow(board, addressOf(prev), input, `board-run-${task.id}-r${runId}`);
|
|
@@ -661,6 +705,7 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
661
705
|
const its = HARNESSES[prev.harness ?? 'claude-code'] ?? HARNESSES['claude-code'];
|
|
662
706
|
const r = sc(['open', '--new', its.program, ...base.slice(3), '--key', `board-${task.id}-r${runId}-resume`, '--input', told, '--', ...its.resume(prev.session_id), ...tail], { timeout: 180_000, as: boardAddress(board.label) });
|
|
663
707
|
const pane = r.code === 0 ? lastJson(r.stdout)?.pane ?? null : null;
|
|
708
|
+
rememberPane(pane, prev.session_id, 'resume');
|
|
664
709
|
// a resume the harness refuses (its transcript is gone) ends the pane at once; another machine's launch
|
|
665
710
|
// answers before that shows, so the pane is looked for again a moment later
|
|
666
711
|
if (pane && paneSurvives(task.machine, pane)) session = { surface: 'pane', session_id: prev.session_id, harness: prev.harness ?? 'claude-code', machine, pane, mode: 'resume', launched_at: now() };
|
|
@@ -680,10 +725,11 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
680
725
|
const r = sc([...base, '--key', launchKey, '--input', input ?? `Card ${task.id}: ${task.title}`, '--', ...(sid ? how.fresh(sid) : how.fresh()), ...tail], { timeout: 180_000, as: boardAddress(board.label) });
|
|
681
726
|
if (r.code !== 0) throw new Error(`supercode open --on ${task.machine}: ${(r.stderr || r.stdout).trim().slice(-500)}`);
|
|
682
727
|
const pane = lastJson(r.stdout)?.pane ?? null;
|
|
728
|
+
rememberPane(pane, sid, 'fresh');
|
|
683
729
|
const receipt=sc(['teams','rpc','launches.get',JSON.stringify({key:launchKey}),'--machine',task.machine],{timeout:30_000,as:boardAddress(board.label)});
|
|
684
730
|
const facts=receipt.code===0?lastJson(receipt.stdout):null;
|
|
685
731
|
if (!pane || facts?.inputDelivered===false || facts?.live===false || (!how.namesItself && !paneSurvives(task.machine,pane))) {
|
|
686
|
-
|
|
732
|
+
// The failed run's durable close queue owns its pane, even if the close door is unavailable.
|
|
687
733
|
throw new Error(`${harness} launch did not become a working session: ${facts?.inputError??(!pane?'no pane returned':'pane ended or initial input was not delivered')}`);
|
|
688
734
|
}
|
|
689
735
|
// a harness that names its own session is found in the fleet by the pane the board opened for it
|
|
@@ -693,7 +739,7 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
693
739
|
}
|
|
694
740
|
if (!sid) {
|
|
695
741
|
// a pane whose session the board cannot name is closed, never left working beside the next start's
|
|
696
|
-
|
|
742
|
+
// The run retains its pane for cleanup even though no harness session id arrived.
|
|
697
743
|
throw new Error(`${harness} in pane ${pane ?? '(none)'} on ${task.machine} named no session within 90 s; the pane is closed`);
|
|
698
744
|
}
|
|
699
745
|
session = { surface: 'pane', session_id: sid, machine, pane, mode: 'fresh', launched_at: now(), ...(replaced ? { replaces: replaced.session_id } : {}) };
|
|
@@ -702,7 +748,12 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
702
748
|
session.address = `sc:${machine}:${session.harness}:${session.session_id}`;
|
|
703
749
|
Object.assign(session, meta);
|
|
704
750
|
const kind = { adopt: 'session_adopted', resume: 'session_resumed', fresh: replaced ? 'session_replaced' : 'session_started' }[session.mode];
|
|
705
|
-
tx(board, (db) => {
|
|
751
|
+
const recorded = tx(board, (db) => {
|
|
752
|
+
const row = db.prepare('SELECT ended_at FROM task_runs WHERE id = ?').get(runId);
|
|
753
|
+
if (row?.ended_at || getTask(db, task.id)?.current_run_id !== runId) {
|
|
754
|
+
requestSessionClose(db, task.id, { runId });
|
|
755
|
+
return false;
|
|
756
|
+
}
|
|
706
757
|
setRunSession(db, runId, session);
|
|
707
758
|
event(db, task.id, kind, { session_id: session.session_id, machine, pane: session.pane, address: session.address, mode: session.mode, ...(replaced ? { replaced } : {}) }, runId);
|
|
708
759
|
// the working session hears updates to its card; the slot's previous session no longer does (a reviewer starting
|
|
@@ -713,16 +764,24 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
713
764
|
// it sees the cards its own depends on change, and works its subtasks
|
|
714
765
|
for (const p of db.prepare('SELECT parent_id FROM task_links WHERE child_id = ?').all(task.id)) subscribeIn(db, p.parent_id, session.address, 'dependency');
|
|
715
766
|
for (const t of db.prepare('SELECT task_id FROM supercode_cards WHERE subtask_of = ?').all(task.id)) subscribeIn(db, t.task_id, session.address, 'worker');
|
|
767
|
+
return true;
|
|
716
768
|
});
|
|
769
|
+
if (!recorded) throw new Error(`run ${runId} ended while its session launched; its pane is queued for closure`);
|
|
717
770
|
return session;
|
|
718
771
|
}
|
|
719
772
|
|
|
720
|
-
/**
|
|
773
|
+
/** A machine reading, not a missing catalog entry, establishes whether a pane is alive. */
|
|
774
|
+
function panePresent(machine, pane) {
|
|
775
|
+
const r = sc(['teams', 'panes', 'ls', '--machine', machine ?? localMachine()], { timeout: 60_000 });
|
|
776
|
+
const panes = r.code === 0 ? json(r.stdout.trim(), null) : null;
|
|
777
|
+
if (!Array.isArray(panes)) throw new Error(`cannot establish pane ${pane} liveness on ${machine}: ${(r.stderr || r.stdout).trim().slice(-300)}`);
|
|
778
|
+
return panes.some((p) => p.pane === pane);
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/** Whether a pane just launched is still there after the harness had a moment to start. */
|
|
721
782
|
function paneSurvives(machine, pane) {
|
|
722
783
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 6000);
|
|
723
|
-
|
|
724
|
-
const panes = r.code === 0 ? json(r.stdout.trim(), null) : null;
|
|
725
|
-
return Array.isArray(panes) ? panes.some((p) => p.pane === pane) : true; // no reading: the grace decides
|
|
784
|
+
return panePresent(machine, pane);
|
|
726
785
|
}
|
|
727
786
|
|
|
728
787
|
/** Hermes's own worker argv and task environment, run on this machine; watched by its pid. */
|
package/board/engine.mjs
CHANGED
|
@@ -12,7 +12,7 @@ import { existsSync, rmSync } from 'node:fs';
|
|
|
12
12
|
import { join, resolve } from 'node:path';
|
|
13
13
|
import { DatabaseSync } from 'node:sqlite';
|
|
14
14
|
import { view, claimer, event, json, newTaskId, now, tx } from './store.mjs';
|
|
15
|
-
import { refreshCardInputs, eventTo, fire, loadWorkflow, setStatus, settle, startNew, statusOf, runEvent, workflowFor } from './workflow.mjs';
|
|
15
|
+
import { refreshCardInputs, requestSessionClose, eventTo, fire, loadWorkflow, setStatus, settle, startNew, statusOf, runEvent, workflowFor } from './workflow.mjs';
|
|
16
16
|
|
|
17
17
|
export const STATUSES = ['triage', 'todo', 'scheduled', 'ready', 'running', 'blocked', 'review', 'done', 'archived'];
|
|
18
18
|
export const WORKSPACE_KINDS = ['scratch', 'dir', 'worktree'];
|
|
@@ -77,12 +77,13 @@ export function endRun(db, taskId, { outcome, status = null, summary = null, err
|
|
|
77
77
|
const was = one(db, 'SELECT metadata FROM task_runs WHERE id = ?', runId)?.metadata ?? null;
|
|
78
78
|
const keep = metadata === undefined ? was : (metadata ? JSON.stringify(metadata) : null);
|
|
79
79
|
// a run taken back (reclaim, archive, a reopened ancestor) stops its one-shot worker on this machine, as Hermes
|
|
80
|
-
// terminates a reclaimed worker (a board's one-shot workers run where the board is);
|
|
80
|
+
// terminates a reclaimed worker (a board's one-shot workers run where the board is); pane sessions close through the dispatcher’s durable queue
|
|
81
81
|
const worker = outcome === 'reclaimed' ? json(was, {})?.supercode : null;
|
|
82
82
|
const lock = one(db, 'SELECT claim_lock FROM task_runs WHERE id = ?', runId)?.claim_lock ?? one(db, 'SELECT claim_lock FROM tasks WHERE id = ?', taskId)?.claim_lock;
|
|
83
83
|
if (worker?.surface === 'headless' && worker.pid && String(lock ?? '').startsWith(`${claimer().split(':')[0]}:`)) run(db, 'INSERT INTO supercode_effects(effect) VALUES(?)', JSON.stringify({kind:'stop_run',id:taskId,runId}));
|
|
84
84
|
run(db, `UPDATE task_runs SET status = ?, outcome = ?, summary = ?, error = ?, metadata = ?, ended_at = ?, claim_lock = NULL, claim_expires = NULL, worker_pid = NULL
|
|
85
85
|
WHERE id = ? AND ended_at IS NULL`, status ?? outcome, outcome, summary, error, keep, now(), runId);
|
|
86
|
+
requestSessionClose(db, taskId, { runId });
|
|
86
87
|
run(db, 'UPDATE tasks SET current_run_id = NULL WHERE id = ?', taskId);
|
|
87
88
|
return runId;
|
|
88
89
|
}
|
|
@@ -355,8 +356,9 @@ export async function runEffects(board, effects = []) {
|
|
|
355
356
|
for (const e of [...pending.map((p) => ({ ...json(p.effect, {}), durableId: p.id })), ...effects]) {
|
|
356
357
|
if (e.kind === 'cleanup_workspace') cleanupWorkspace(board, e.id);
|
|
357
358
|
else if (e.kind === 'invalidate_descendants') tx(board, (db) => invalidateDescendants(db, board, e.id));
|
|
358
|
-
else if (
|
|
359
|
-
else if (e.kind === 'close_session' || e.kind === 'send_to_actor') {
|
|
359
|
+
else if (e.kind === 'stop_run') stopWorker(board, e.runId);
|
|
360
|
+
else if (e.kind === 'close_session' || e.kind === 'send_to_actor' || e.kind === 'run_ended') {
|
|
361
|
+
if (e.kind === 'run_ended' && e.outcome === 'reclaimed') stopWorker(board, e.runId);
|
|
360
362
|
const { sessionEffect } = await import('./dispatch.mjs');
|
|
361
363
|
await sessionEffect(board, e);
|
|
362
364
|
}
|
|
@@ -365,7 +367,7 @@ export async function runEffects(board, effects = []) {
|
|
|
365
367
|
}
|
|
366
368
|
|
|
367
369
|
// a run taken back stops its one-shot worker on this host (Hermes's _terminate_reclaimed_worker); a pane session is
|
|
368
|
-
//
|
|
370
|
+
// stopped through its board-owned pane close queue
|
|
369
371
|
function stopWorker(board, runId) {
|
|
370
372
|
if (!runId) return;
|
|
371
373
|
const row = view(board, (db) => one(db, 'SELECT metadata, claim_lock FROM task_runs WHERE id = ?', runId));
|
package/board/workflow.mjs
CHANGED
|
@@ -589,6 +589,7 @@ const ACTIONS = {
|
|
|
589
589
|
const runId = Number(row.current_run_id);
|
|
590
590
|
run(ctx.db, `UPDATE task_runs SET status = ?, outcome = ?, summary = COALESCE(?, summary), error = COALESCE(?, error), ended_at = ?, claim_lock = NULL, claim_expires = NULL, worker_pid = NULL
|
|
591
591
|
WHERE id = ? AND ended_at IS NULL`, outcome, outcome, summary, error ?? ctx.scope.event?.error ?? null, t, runId);
|
|
592
|
+
requestSessionClose(ctx.db, ctx.id, { runId });
|
|
592
593
|
ctx.runId = runId;
|
|
593
594
|
ctx.lastOutcome = outcome;
|
|
594
595
|
ctx.effects.push({ kind: 'run_ended', id: ctx.id, runId, outcome });
|
|
@@ -671,7 +672,11 @@ const ACTIONS = {
|
|
|
671
672
|
cleanup_workspace(ctx) { ctx.effects.push({ kind: 'cleanup_workspace', id: ctx.id }); },
|
|
672
673
|
stop_run(ctx) { ctx.effects.push({ kind: 'stop_run', id: ctx.id, runId: ctx.runId }); },
|
|
673
674
|
// the run leaving the status is the one whose session closes, never a run started after it (a reclaim's successor)
|
|
674
|
-
close_session(ctx, slot) {
|
|
675
|
+
close_session(ctx, slot) {
|
|
676
|
+
const through = ctx.runId ?? one(ctx.db, 'SELECT MAX(id) AS id FROM task_runs WHERE task_id = ?', ctx.id)?.id ?? 0;
|
|
677
|
+
requestSessionClose(ctx.db, ctx.id, { slot, through });
|
|
678
|
+
ctx.effects.push({ kind: 'close_session', id: ctx.id, slot, runId: through });
|
|
679
|
+
},
|
|
675
680
|
// the slot's last session is let go: its next run starts a new session (a card's context is otherwise kept: a
|
|
676
681
|
// session that cannot be resumed stops the card, it is never replaced on the board's own account)
|
|
677
682
|
forget_session(ctx, slot) {
|
|
@@ -707,6 +712,20 @@ const ACTIONS = {
|
|
|
707
712
|
send_to_actor(ctx, text) { ctx.effects.push({ kind: 'send_to_actor', id: ctx.id, text }); },
|
|
708
713
|
};
|
|
709
714
|
|
|
715
|
+
/** Persist close intent in the same transaction as the transition; I/O can fail or never run. */
|
|
716
|
+
export function requestSessionClose(db, id, { runId = null, slot = null, through = null } = {}) {
|
|
717
|
+
for (const row of all(db, 'SELECT id, metadata FROM task_runs WHERE task_id = ?', id)) {
|
|
718
|
+
if ((runId != null && row.id !== runId) || (through != null && row.id > through)) continue;
|
|
719
|
+
const whole = json(row.metadata, {}) ?? {};
|
|
720
|
+
const session = whole.supercode;
|
|
721
|
+
if (!session?.pane || session.close_confirmed_at || (slot != null && (session.slot ?? 'worker') !== slot)) continue;
|
|
722
|
+
if (session.close_requested && !session.closed_at) continue;
|
|
723
|
+
whole.supercode = { ...session, close_requested: session.close_requested ?? now() };
|
|
724
|
+
delete whole.supercode.closed_at; // older versions recorded success without checking the receipt
|
|
725
|
+
run(db, 'UPDATE task_runs SET metadata = ? WHERE id = ?', JSON.stringify(whole), row.id);
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
|
|
710
729
|
function runSession(ctx, change) {
|
|
711
730
|
const runId = ctx.runId ?? one(ctx.db, 'SELECT current_run_id FROM tasks WHERE id = ?', ctx.id)?.current_run_id;
|
|
712
731
|
if (!runId) return;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-orchestrator",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.104",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
|
|
6
6
|
"exports": {
|