@volter/supercode-orchestrator 0.5.47 → 0.5.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/bin/runtime.mjs +23 -31
- package/board/cli.mjs +12 -2
- package/board/dispatch.mjs +100 -265
- package/board/engine.mjs +17 -19
- package/board/request-mail.mjs +2 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -525,10 +525,12 @@ failed/timeout fires without replaying them; a restart advances a stale claim
|
|
|
525
525
|
once. `callOperator` addresses the live socket, with no cold-write fallback.
|
|
526
526
|
`Orchestrator({executeJob})` is the corresponding in-process API for embedders.
|
|
527
527
|
|
|
528
|
-
The daemon
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
528
|
+
The daemon serves a home only while it holds the home's claim, Hermes's
|
|
529
|
+
`gateway.lock` (an flock the kernel releases when its holder dies). A second
|
|
530
|
+
daemon on the home waits for the claim and serves the home as soon as the first
|
|
531
|
+
is gone, however it ended; nothing fails closed or waits for a person. The
|
|
532
|
+
serving daemon records itself in `orchestrator.lock` for `supercode orchestrator
|
|
533
|
+
status` and the operator verbs; a normal stop removes only its own record. Native `jobs.resume` computes the first fire
|
|
532
534
|
for a job that was created paused with no next-run timestamp.
|
|
533
535
|
|
|
534
536
|
## Hosted daemon lifecycle
|
package/bin/runtime.mjs
CHANGED
|
@@ -53,7 +53,7 @@
|
|
|
53
53
|
// Everything it prints is one JSON object per line on stdout.
|
|
54
54
|
|
|
55
55
|
import { SenderIdentityRecord } from '@volter/teams/sender-identity';
|
|
56
|
-
import { existsSync,
|
|
56
|
+
import { existsSync, readFileSync, writeFileSync, unlinkSync } from 'node:fs';
|
|
57
57
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
58
58
|
import { spawn, spawnSync } from 'node:child_process';
|
|
59
59
|
import { fileURLToPath } from 'node:url';
|
|
@@ -75,7 +75,8 @@ import { httpJobExecutor } from '../hosted.mjs';
|
|
|
75
75
|
import { SupercodeHarnessClient } from '@volter/supercode-harness-sdk';
|
|
76
76
|
|
|
77
77
|
/** How long a starting daemon waits for a stopping predecessor to release the home's lease. */
|
|
78
|
-
|
|
78
|
+
/** How often a daemon waiting for the home's claim asks for it again. */
|
|
79
|
+
const CLAIM_RETRY_MS = 2_000;
|
|
79
80
|
|
|
80
81
|
/// Adapter selector -> constructor. A `platforms:` block picks its adapter by
|
|
81
82
|
/// `extra.adapter` when it has one and by the block's own name otherwise, so
|
|
@@ -388,16 +389,16 @@ function profileReport(name, profile) {
|
|
|
388
389
|
};
|
|
389
390
|
}
|
|
390
391
|
|
|
391
|
-
// ---------------------------------------------------------------- the
|
|
392
|
+
// ---------------------------------------------------------------- who serves the home
|
|
392
393
|
|
|
393
|
-
/** The
|
|
394
|
+
/** The record of the daemon serving the home, shared with `supercode orchestrator status|stop` (Rust `Lease`). */
|
|
394
395
|
const readLock = readLease;
|
|
395
396
|
|
|
396
397
|
function writeLock(root, lease) {
|
|
397
398
|
writeFileSync(join(root, LOCK_FILE), `${JSON.stringify(lease, null, 2)}\n`, { flag: 'wx', mode: 0o600 });
|
|
398
399
|
}
|
|
399
400
|
|
|
400
|
-
/** Only ever removes
|
|
401
|
+
/** Only ever removes the record this process wrote. */
|
|
401
402
|
function clearLock(root) {
|
|
402
403
|
const held = readLock(root);
|
|
403
404
|
if (held && held.pid !== process.pid) return;
|
|
@@ -677,35 +678,26 @@ async function main() {
|
|
|
677
678
|
let crashed = false;
|
|
678
679
|
let gatewayLock = null;
|
|
679
680
|
if (!args.once) {
|
|
680
|
-
// One
|
|
681
|
-
//
|
|
682
|
-
//
|
|
683
|
-
//
|
|
684
|
-
//
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
const guard = join(root, 'orchestrator.acquire');
|
|
692
|
-
mkdirSync(guard);
|
|
693
|
-
try {
|
|
694
|
-
const previous = readLock(root);
|
|
695
|
-
if (previous && leaseIsLive(previous)) {
|
|
696
|
-
say('error', { root, message: `already serving ${root} (pid ${previous.pid})` });
|
|
697
|
-
throw new Error(`already serving ${root}`);
|
|
681
|
+
// One runtime serves a home: whoever holds its claim, Hermes's own gateway lock (`<home>/gateway.lock`, an flock the
|
|
682
|
+
// kernel releases the moment its holder dies, however it died). So its cron jobs fire once, its channel adapters
|
|
683
|
+
// are held once and its bindings are driven once, and a Hermes gateway on the home refuses while this daemon
|
|
684
|
+
// serves it. A second daemon does not exit and nothing waits for a person: it waits for the claim and serves the
|
|
685
|
+
// home as soon as the first is gone (a service manager's restart, a crash, kill -9).
|
|
686
|
+
for (let said = false; ;) {
|
|
687
|
+
try { gatewayLock = await holdGatewayLock(root); break; }
|
|
688
|
+
catch (error) {
|
|
689
|
+
if (!/ is served by /.test(error.message)) { say('error', { root, message: error.message }); process.exit(1); }
|
|
690
|
+
if (!said) { say('log', { level: 'info', message: `${error.message}; this daemon serves the home when that one ends` }); said = true; }
|
|
691
|
+
await new Promise((resolve) => setTimeout(resolve, CLAIM_RETRY_MS));
|
|
698
692
|
}
|
|
699
|
-
crashed = Boolean(previous);
|
|
700
|
-
if (previous) unlinkSync(join(root, LOCK_FILE));
|
|
701
|
-
writeLock(root, { pid: process.pid, started_at: new Date().toISOString(), root, ...thisHost(), executor_url: args.executorUrl ?? null });
|
|
702
|
-
} finally {
|
|
703
|
-
rmdirSync(guard);
|
|
704
693
|
}
|
|
705
|
-
//
|
|
706
|
-
//
|
|
694
|
+
// Who serves the home now (`orchestrator.lock`), for `supercode orchestrator status` and the operator verbs' live
|
|
695
|
+
// door. A record left behind is a predecessor that never stopped cleanly: this start replays for it (§4.7).
|
|
696
|
+
const previous = readLock(root);
|
|
697
|
+
crashed = Boolean(previous && previous.pid !== process.pid);
|
|
698
|
+
if (previous) unlinkSync(join(root, LOCK_FILE));
|
|
699
|
+
writeLock(root, { pid: process.pid, started_at: new Date().toISOString(), root, ...thisHost(), executor_url: args.executorUrl ?? null });
|
|
707
700
|
try {
|
|
708
|
-
gatewayLock = await holdGatewayLock(root);
|
|
709
701
|
// the home in both forms before it is loaded: a Hermes home is adopted as it stands
|
|
710
702
|
for (const change of adoptHome(root)) say('log', { level: 'info', message: `home: ${change}` });
|
|
711
703
|
} catch (error) {
|
package/board/cli.mjs
CHANGED
|
@@ -17,6 +17,15 @@ import { openZtrackBoard } from './ztrack.mjs';
|
|
|
17
17
|
import * as engine from './engine.mjs';
|
|
18
18
|
import { runtimeDoor } from './runtime-door.mjs';
|
|
19
19
|
|
|
20
|
+
/**
|
|
21
|
+
* The messages on this machine whose delivery the mailbox gave up on (`supercode message expired`), by message id:
|
|
22
|
+
* filed by the board, never delivered within the mailbox's bound, kept unread. Empty when the mailbox does not answer.
|
|
23
|
+
*/
|
|
24
|
+
function mailExpiries() {
|
|
25
|
+
const r = spawnSync(supercodeBin(), ['message', 'expired', '--json'], { encoding: 'utf8', env: commandEnv(), timeout: 15_000 });
|
|
26
|
+
try { return new Map((JSON.parse(r.stdout) ?? []).map((expiry) => [expiry.id, expiry])); } catch { return new Map(); }
|
|
27
|
+
}
|
|
28
|
+
|
|
20
29
|
/**
|
|
21
30
|
* What this machine's connector reports of publishing this board home (`harness.v1.boards.list`): whether the team
|
|
22
31
|
* refused its credential, the publication it is retrying and the ones it abandoned. Null when no daemon answers or the
|
|
@@ -833,11 +842,12 @@ export async function runWorkflow(argv) {
|
|
|
833
842
|
const { settings } = await laneState(root).catch(() => ({ settings: boardSettings({}) }));
|
|
834
843
|
const fleet = observeFleet(root);
|
|
835
844
|
const publishing = boardPublishing(root);
|
|
845
|
+
const expiredMail = mailExpiries();
|
|
836
846
|
const order = ['warning', 'error', 'critical'];
|
|
837
847
|
const found = withBoard((b) => {
|
|
838
848
|
const cards = args.task ? [engine.getTask(b.db, args.task)].filter(Boolean) : engine.listCards(b.db, {});
|
|
839
849
|
return [...(args.task ? [] : engine.diagnosePublishing(publishing).map((d) => ({ task: '(board)', title_of_task: null, ...d }))),
|
|
840
|
-
...cards.flatMap((t) => engine.diagnose(b.db, t, { failureLimit: settings.failureLimit, fleet, launchWindowSeconds: settings.launchWindow, publishing }).map((d) => ({ task: t.id, title_of_task: t.title, ...d })))]
|
|
850
|
+
...cards.flatMap((t) => engine.diagnose(b.db, t, { failureLimit: settings.failureLimit, fleet, launchWindowSeconds: settings.launchWindow, publishing, expiredMail }).map((d) => ({ task: t.id, title_of_task: t.title, ...d })))]
|
|
841
851
|
.filter((d) => !args.severity || order.indexOf(d.severity) >= order.indexOf(args.severity));
|
|
842
852
|
});
|
|
843
853
|
if (args.json) { say(args, '', found); return 0; }
|
|
@@ -1514,7 +1524,7 @@ async function serve(root, args = {}) {
|
|
|
1514
1524
|
for (const [t, why] of r.failed) log(` ${t} did not start: ${why}`);
|
|
1515
1525
|
for (const [t, why] of r.unsettled ?? []) log(` ${t} could not be read: ${why}`);
|
|
1516
1526
|
for (const [t, s, m] of r.started) log(` ${t} started ${s} (${m})`);
|
|
1517
|
-
for (const [t, to, why, what] of r.undelivered) log(what === 'close' ? ` close of ${to} for ${t} waits: ${why}` : ` notice for ${t} to ${to} not
|
|
1527
|
+
for (const [t, to, why, what] of r.undelivered) log(what === 'close' ? ` close of ${to} for ${t} waits: ${why}` : ` notice for ${t} to ${to} not filed: ${why}`);
|
|
1518
1528
|
}
|
|
1519
1529
|
} catch (error) {
|
|
1520
1530
|
log(`tick failed: ${error.message}`);
|
package/board/dispatch.mjs
CHANGED
|
@@ -1068,7 +1068,11 @@ function effectMessageKey(effect) {
|
|
|
1068
1068
|
return `board-effect-${createHash('sha256').update(identity).digest('hex')}`;
|
|
1069
1069
|
}
|
|
1070
1070
|
|
|
1071
|
-
/**
|
|
1071
|
+
/**
|
|
1072
|
+
* A lifecycle notice to the session whose run changed, filed in its mailbox (the mailbox delivers it). The verb that
|
|
1073
|
+
* recorded it files it at once after its commit; the delivery pass files any a verb left. It waits only for the
|
|
1074
|
+
* session's address (a launch not yet bound).
|
|
1075
|
+
*/
|
|
1072
1076
|
export async function sessionNotice(board, e) {
|
|
1073
1077
|
const prior = view(board, (db) => json(db.prepare('SELECT metadata FROM task_runs WHERE id=?').get(e.runId)?.metadata, {})?.supercode);
|
|
1074
1078
|
if (prior?.close_confirmed_at) {
|
|
@@ -1079,24 +1083,14 @@ export async function sessionNotice(board, e) {
|
|
|
1079
1083
|
});
|
|
1080
1084
|
return true;
|
|
1081
1085
|
}
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
if (!e.address) throw new Error('lifecycle notice awaits the owned launch session identity');
|
|
1086
|
-
receipt = await send(board, boardAddress(board.label), board.label, e.address, e.text, effectMessageKey(e));
|
|
1087
|
-
// A notice filed in the session's mailbox is done: the session reads its mail at its next
|
|
1088
|
-
// turn, and a resumed session first. Only a notice no mailbox took is retried.
|
|
1089
|
-
if (!receipt.delivered && !receipt.queued) error = receipt.reason ?? 'notice was not filed'; }
|
|
1090
|
-
catch (failure) { error = String(failure.message).slice(-500); }
|
|
1086
|
+
const address = e.address ?? addressOf(prior);
|
|
1087
|
+
if (!address) throw new Error('lifecycle notice awaits the owned launch session identity');
|
|
1088
|
+
const filed = fileMail(board, address, e.text, effectMessageKey(e));
|
|
1091
1089
|
tx(board, (db) => {
|
|
1092
1090
|
const session = json(db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(e.runId)?.metadata, {})?.supercode;
|
|
1093
|
-
if (session) setRunSession(db, e.runId, {
|
|
1094
|
-
|
|
1095
|
-
: { lifecycle_notice_error: null, ...(session.lifecycle_notice_pending === e.durableId ? { lifecycle_notice_pending: null } : {}) }) });
|
|
1096
|
-
if (!error || session?.lifecycle_notice_error !== error) event(db, e.id, error ? 'session_notice_failed' : receipt?.delivered ? 'session_notice_delivered' : 'session_notice_queued', { address: e.address, verb: e.verb, source_event: e.sourceEvent, notice: e.durableId, receipt, ...(error ? { error, retry: true } : {}) }, e.runId);
|
|
1091
|
+
if (session?.lifecycle_notice_pending === e.durableId) setRunSession(db, e.runId, { lifecycle_notice_pending: null });
|
|
1092
|
+
if (filed.new) event(db, e.id, 'session_notice_filed', { address, verb: e.verb, source_event: e.sourceEvent, notice: e.durableId, message_id: filed.message_id }, e.runId);
|
|
1097
1093
|
});
|
|
1098
|
-
// the attempt's reason goes with it, onto the effect's row (runEffects counts the attempt)
|
|
1099
|
-
if (error) throw new Error(error);
|
|
1100
1094
|
return true;
|
|
1101
1095
|
}
|
|
1102
1096
|
|
|
@@ -1106,20 +1100,10 @@ export async function sessionEffect(board, e) {
|
|
|
1106
1100
|
const row = view(board, db => e.runId ? db.prepare('SELECT metadata FROM task_runs WHERE id=?').get(e.runId) : null);
|
|
1107
1101
|
const session = json(row?.metadata, {})?.supercode;
|
|
1108
1102
|
const address = addressOf(session);
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
} catch (error) { receipt = { delivered: false, reason: error.message }; }
|
|
1114
|
-
const kind = receipt.delivered ? 'actor_notice_delivered' : receipt.queued ? 'actor_notice_queued' : 'actor_notice_failed';
|
|
1115
|
-
tx(board, db => {
|
|
1116
|
-
const previous = db.prepare("SELECT kind,payload FROM task_events WHERE task_id=? AND kind IN ('actor_notice_delivered','actor_notice_queued','actor_notice_failed') ORDER BY id DESC LIMIT 1").get(e.id);
|
|
1117
|
-
const payload = { notice: e.durableId, address, receipt };
|
|
1118
|
-
if (previous?.kind !== kind || JSON.stringify(json(previous?.payload, {})?.receipt) !== JSON.stringify(receipt)) event(db, e.id, kind, payload, e.runId);
|
|
1119
|
-
});
|
|
1120
|
-
// a send its recipient's mailbox took (filed, or queued for a session between turns) is accepted: the mailbox
|
|
1121
|
-
// delivers it from there, and the board does not send it again
|
|
1122
|
-
if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'the actor notice was not filed');
|
|
1103
|
+
if (!address) throw new Error('actor session identity unavailable');
|
|
1104
|
+
// filed in the actor's mailbox, which delivers it
|
|
1105
|
+
const filed = fileMail(board, address, e.text, effectMessageKey(e));
|
|
1106
|
+
if (filed.new) tx(board, db => event(db, e.id, 'actor_notice_filed', { notice: e.durableId, address, message_id: filed.message_id }, e.runId));
|
|
1123
1107
|
return true;
|
|
1124
1108
|
}
|
|
1125
1109
|
if (e.kind === 'close_session' || e.kind === 'run_ended') {
|
|
@@ -1550,17 +1534,8 @@ function launchPane(board, task, runId, lane, fleet, { input = null, slot = 'wor
|
|
|
1550
1534
|
session = { ...kept, mode: 'adopt', launched_at: now(), seen_at: now() };
|
|
1551
1535
|
// a kept session is told what this run is for (its card came back to it, with what happened since)
|
|
1552
1536
|
tx(board, (db) => setRunSession(db, runId, { ...session, ...meta }));
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
catch (error) {
|
|
1556
|
-
tx(board, (db) => {
|
|
1557
|
-
const effect = { kind: 'session_notice', id: task.id, runId, address: addressOf(prev), text: input, verb: 'start', messageKey: `board-run-${task.id}-r${runId}` };
|
|
1558
|
-
const saved = db.prepare('INSERT INTO supercode_effects(effect) VALUES(?)').run(JSON.stringify(effect));
|
|
1559
|
-
setRunSession(db, runId, { lifecycle_notice_pending: Number(saved.lastInsertRowid), lifecycle_notice_error: error.message });
|
|
1560
|
-
event(db, task.id, 'session_notice_failed', { verb: 'start', error: error.message, retry: true }, runId);
|
|
1561
|
-
});
|
|
1562
|
-
}
|
|
1563
|
-
}
|
|
1537
|
+
// filed in its mailbox, which delivers it
|
|
1538
|
+
if (input) fileMail(board, addressOf(prev), input, `board-run-${task.id}-r${runId}`);
|
|
1564
1539
|
} else if (prev.resumable !== false) {
|
|
1565
1540
|
const authorization = view(board, db => currentRun(db,task.id)?.metadata?.supercode?.restart_authorized);
|
|
1566
1541
|
const released = view(board,db=>{
|
|
@@ -1950,40 +1925,29 @@ function reconnectContinuedSessions(board, fleet, entry, report) {
|
|
|
1950
1925
|
// ------------------------------------------------------------------ notify
|
|
1951
1926
|
|
|
1952
1927
|
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
async function send(board, from, label, to, body, key, { queue = false, ask = false, subject } = {}) {
|
|
1965
|
-
if (boardCopyOf(board)) throw copyRefusal(board, 'sends no mail to sessions or managers');
|
|
1928
|
+
/**
|
|
1929
|
+
* File one message from the board in its receiver's mailbox, and stop there (D138 step 6, D140): a local write through
|
|
1930
|
+
* the mailbox's filing door (`supercode message file`), idempotent by the message id `key` fixes, with no network and
|
|
1931
|
+
* nothing to try again. The mailbox delivers it: the machine daemon's watch loop hands it to its session by the
|
|
1932
|
+
* session's own door, and carries mail for another machine there through Teams. `wake: false` leaves it for the
|
|
1933
|
+
* session's next turn. A question the board waits on (`ask`) is answered by a reply to the board's address; a notice
|
|
1934
|
+
* is not. Throws only when the local write failed.
|
|
1935
|
+
*/
|
|
1936
|
+
function fileMail(board, to, body, key, { ask = false, wake = true, subject } = {}) {
|
|
1937
|
+
if (boardCopyOf(board)) throw copyRefusal(board, 'files no mail for sessions or managers');
|
|
1966
1938
|
const request = {
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
body: ask ? `[A question from the ${label} board] ${body}` : `[Board notice from the ${label} board] ${body}\n\n(An automated notice from the board's dispatcher; replies to this address go nowhere.)`,
|
|
1939
|
+
from: boardAddress(board.label), from_name: `${board.label} board`, to, id: key, wake, subject: String(subject ?? String(body).split('\n')[0]).slice(0, 160),
|
|
1940
|
+
body: ask ? `[A question from the ${board.label} board] ${body}` : `[Board notice from the ${board.label} board] ${body}\n\n(An automated notice from the board's dispatcher; replies to this address go nowhere.)`,
|
|
1970
1941
|
};
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
if (answer.code !== 0) {
|
|
1976
|
-
const error = new Error((answer.text || r.stderr || `no answer (exit ${r.code})`).trim().slice(-300));
|
|
1977
|
-
// the delivery door's own answer that the recipient runs nowhere (mail_route / mail_send), never a local failure
|
|
1978
|
-
if (typeof answer.code === 'number' && / is no longer running\. Nothing was sent\./.test(String(answer.text ?? ''))) error.notRunning = true;
|
|
1979
|
-
throw error;
|
|
1980
|
-
}
|
|
1981
|
-
return answer.receipt ?? { delivered: false, reason: 'mail door returned no delivery receipt' };
|
|
1942
|
+
const r = sc(['message', 'file'], { input: JSON.stringify(request), timeout: 30_000 });
|
|
1943
|
+
const answer = lastJson(r.stdout);
|
|
1944
|
+
if (answer?.code !== 0) throw new Error((answer?.text || r.stderr || `no answer from the mailbox's filing door (exit ${r.code})`).trim().slice(-300));
|
|
1945
|
+
return { filed: true, message_id: answer.message_id, new: answer.new === true };
|
|
1982
1946
|
}
|
|
1983
1947
|
|
|
1984
1948
|
/**
|
|
1985
1949
|
* Mail the workflow's managers (`params.managers`) about the dispatcher itself, not a card: its rounds failing, for one.
|
|
1986
|
-
* Answers how many
|
|
1950
|
+
* Answers how many it was filed for; a filing that fails is skipped, never a reason to fail the caller.
|
|
1987
1951
|
*/
|
|
1988
1952
|
export async function alertManagers(root, slug, text, key) {
|
|
1989
1953
|
let managers = [];
|
|
@@ -1991,54 +1955,26 @@ export async function alertManagers(root, slug, text, key) {
|
|
|
1991
1955
|
const label = boardLabel(root, slug);
|
|
1992
1956
|
let told = 0;
|
|
1993
1957
|
for (const to of managers) {
|
|
1994
|
-
try {
|
|
1958
|
+
try { fileMail({ root, label }, to, text, key, { subject: text.split('\n')[0] }); told += 1; } catch { /* a filing that failed here is said by the next alert */ }
|
|
1995
1959
|
}
|
|
1996
1960
|
return told;
|
|
1997
1961
|
}
|
|
1998
1962
|
|
|
1999
1963
|
/**
|
|
2000
|
-
* The messages the workflow has the board send (`tell`, `ask`: board_message events)
|
|
2001
|
-
* and marked board_message_sent. A question goes under the id its card
|
|
2002
|
-
*
|
|
2003
|
-
|
|
2004
|
-
const MAIL_READY_SECONDS = 300;
|
|
2005
|
-
/** Each board message not yet sent nor abandoned gets its effect row (once: the insert reads that none exists). */
|
|
2006
|
-
export function queueBoardMail(db) {
|
|
2007
|
-
db.prepare(`INSERT INTO supercode_effects (effect)
|
|
2008
|
-
SELECT json_object('kind', 'board_message', 'id', original.task_id, 'event', original.id) FROM task_events original
|
|
2009
|
-
WHERE original.kind = 'board_message'
|
|
2010
|
-
AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind = 'board_message_sent' AND sent.task_id = original.task_id AND json_extract(sent.payload, '$.event') = original.id)
|
|
2011
|
-
AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind = 'board_message_abandoned' AND gone.task_id = original.task_id AND json_extract(gone.payload, '$.event') = original.id)
|
|
2012
|
-
AND NOT EXISTS (SELECT 1 FROM supercode_effects e WHERE json_extract(e.effect, '$.kind') = 'board_message' AND json_extract(e.effect, '$.event') = original.id)`).run();
|
|
2013
|
-
}
|
|
2014
|
-
|
|
2015
|
-
/**
|
|
2016
|
-
* One board message's effect: its mail (with any batch it is in) sent under its fixed message id, so a second sender
|
|
2017
|
-
* is dropped by the mailbox. Answers true once it is sent, 'wait' while its batch window is open; throws the failure.
|
|
1964
|
+
* The messages the workflow has the board send (`tell`, `ask`: board_message events) not filed yet, filed now (each
|
|
1965
|
+
* with any batch it is in, `pendingRequestMail`) and marked board_message_sent. A question goes under the id its card
|
|
1966
|
+
* waits on, so a second filing of it files nothing new. A mail inside its batch window waits for the next pass; one
|
|
1967
|
+
* whose filing failed here is reported and filed by the next pass.
|
|
2018
1968
|
*/
|
|
2019
|
-
export
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
if (
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
if (failed) throw new Error(failed[2]);
|
|
2029
|
-
// a mail waits for its batch window (at most a minute); one still not ready well after that is an attempt that failed
|
|
2030
|
-
const recorded = view(board, (db) => db.prepare('SELECT created_at FROM task_events WHERE id = ?').get(e.event)?.created_at);
|
|
2031
|
-
if (recorded && now() - Number(recorded) > MAIL_READY_SECONDS) throw new Error(`the mail was not ready to send ${now() - Number(recorded)} s after it was recorded`);
|
|
2032
|
-
return 'wait';
|
|
2033
|
-
}
|
|
2034
|
-
|
|
2035
|
-
export async function sendBoardMessages(board, report, taskId = null) {
|
|
2036
|
-
const pending = view(board, db => db.prepare(`SELECT 1 FROM task_events original WHERE kind='board_message'
|
|
2037
|
-
AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind='board_message_sent'
|
|
2038
|
-
AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
|
|
2039
|
-
AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind='board_message_abandoned'
|
|
2040
|
-
AND gone.task_id=original.task_id AND json_extract(gone.payload,'$.event')=original.id) LIMIT 1`).get());
|
|
2041
|
-
if (!pending) return;
|
|
1969
|
+
export function fileBoardMail(board, report, taskId = null) {
|
|
1970
|
+
// a message older than the workflow's notice_max_age_seconds is history, not news (a backlog replayed after an
|
|
1971
|
+
// upgrade or a long stop): it is passed over, recorded once, never filed
|
|
1972
|
+
const maxAge = Number(workflowFor(board.root)?.params?.notice_max_age_seconds) || null;
|
|
1973
|
+
if (maxAge) tx(board, db => {
|
|
1974
|
+
for (const row of db.prepare(`SELECT id, task_id FROM task_events original WHERE kind='board_message' AND created_at < ?
|
|
1975
|
+
AND NOT EXISTS (SELECT 1 FROM task_events done WHERE done.kind IN ('board_message_sent', 'board_message_passed') AND done.task_id=original.task_id AND json_extract(done.payload,'$.event')=original.id)`).all(now() - maxAge))
|
|
1976
|
+
event(db, row.task_id, 'board_message_passed', { event: row.id, reason: `older than notice_max_age_seconds (${maxAge})` });
|
|
1977
|
+
});
|
|
2042
1978
|
const planned = tx(board, db => pendingRequestMail(db, taskId));
|
|
2043
1979
|
if (Number.isFinite(planned.nextAt)) report.next_mail_at = Math.min(report.next_mail_at ?? Infinity, planned.nextAt);
|
|
2044
1980
|
for (const mail of planned.ready) {
|
|
@@ -2047,17 +1983,13 @@ export async function sendBoardMessages(board, report, taskId = null) {
|
|
|
2047
1983
|
try {
|
|
2048
1984
|
const task = view(board, db => getTask(db, first.task_id));
|
|
2049
1985
|
const subject = mail.requester ? `State requests · ${mail.members.length} request(s) · ${task?.title || first.task_id}` : `${task?.title || first.task_id} · ${mail.message_id ? 'Question' : 'Update'}`;
|
|
2050
|
-
const
|
|
2051
|
-
if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'message was not filed');
|
|
1986
|
+
const filed = fileMail(board, mail.to, mail.text, mail.message_id ?? `board-message-${first.id}`, { ask: Boolean(mail.message_id), subject });
|
|
2052
1987
|
tx(board, db => {
|
|
2053
|
-
for (const member of mail.members) if (!db.prepare("SELECT 1 FROM task_events WHERE kind='board_message_sent' AND task_id=? AND json_extract(payload,'$.event')=?").get(member.task_id, member.id)) event(db, member.task_id, 'board_message_sent', { event: member.id, to: mail.to,
|
|
1988
|
+
for (const member of mail.members) if (!db.prepare("SELECT 1 FROM task_events WHERE kind='board_message_sent' AND task_id=? AND json_extract(payload,'$.event')=?").get(member.task_id, member.id)) event(db, member.task_id, 'board_message_sent', { event: member.id, to: mail.to, message_id: filed.message_id, ...(mail.message_id ? { request_id: member.message_id ?? mail.message_id } : {}) });
|
|
2054
1989
|
});
|
|
1990
|
+
(report.mailed ??= []).push([first.task_id, mail.to]);
|
|
2055
1991
|
} catch (error) {
|
|
2056
|
-
const
|
|
2057
|
-
for (const member of mail.members) report.undelivered.push([member.task_id, mail.to, reason]);
|
|
2058
|
-
tx(board, db => {
|
|
2059
|
-
for (const member of mail.members) event(db, member.task_id, 'board_message_failed', { event: member.id, to: mail.to, message_id: mail.message_id, reason });
|
|
2060
|
-
});
|
|
1992
|
+
for (const member of mail.members) report.undelivered.push([member.task_id, mail.to, String(error.message).slice(-200)]);
|
|
2061
1993
|
}
|
|
2062
1994
|
}
|
|
2063
1995
|
}
|
|
@@ -2067,8 +1999,6 @@ export const boardAddress = (label) => `sc:${localMachine()}:board:${label.repla
|
|
|
2067
1999
|
|
|
2068
2000
|
/** Consecutive failed sends after which a chat-platform subscription is dropped (Hermes's notifier drops a dead chat after 12). */
|
|
2069
2001
|
const MAX_SEND_FAILURES = 12;
|
|
2070
|
-
/** Attempts a supercode notice gets before it is abandoned (its subscription stays). */
|
|
2071
|
-
const NOTICE_ATTEMPTS = 6;
|
|
2072
2002
|
|
|
2073
2003
|
/**
|
|
2074
2004
|
* What an audience is told of an event: the workflow's `notify[audience][kind]` template over the card, the event's
|
|
@@ -2123,31 +2053,17 @@ function sessionRecord(db) {
|
|
|
2123
2053
|
return { held, latest };
|
|
2124
2054
|
}
|
|
2125
2055
|
|
|
2126
|
-
/**
|
|
2127
|
-
* Whether the recipient's machine says, through its native discovery, that the session is not running (`persisted`:
|
|
2128
|
-
* its transcript is on disk and nothing runs it). Asked only after the delivery door answered that the session is no
|
|
2129
|
-
* longer running, once per machine and harness a round; null when the machine cannot answer.
|
|
2130
|
-
*/
|
|
2131
|
-
function nativeEnded(address, discovered) {
|
|
2132
|
-
const m = String(address).match(/^sc:([^:]+):([^:]+):(.+)$/);
|
|
2133
|
-
if (!m) return null;
|
|
2134
|
-
const [, machine, harness, session_id] = m, key = `${normalMachine(machine)}\u0000${harness}`;
|
|
2135
|
-
if (!discovered.has(key)) { try { discovered.set(key, nativeSessions(machine, harness)); } catch { discovered.set(key, null); } }
|
|
2136
|
-
const rows = discovered.get(key);
|
|
2137
|
-
if (!rows) return null;
|
|
2138
|
-
return nativeReading(rows, { session_id, harness }).alive === false ? { by: 'native_discovery', presence: 'persisted', machine } : null;
|
|
2139
|
-
}
|
|
2140
|
-
|
|
2141
2056
|
/**
|
|
2142
2057
|
* The worker subscriptions of sessions that have ended, removed in one transaction, with one notice_unsubscribed event
|
|
2143
2058
|
* per session naming its cards. A session has ended when no unended or current run works in it, the fleet does not list
|
|
2144
|
-
* it running, its latest run is not a release (a released session is a person's own, left running), and
|
|
2145
|
-
*
|
|
2146
|
-
*
|
|
2059
|
+
* it running, its latest run is not a release (a released session is a person's own, left running), and the board
|
|
2060
|
+
* closed it (that run's confirmed close). Everything is read inside the transaction, so a run that took the session
|
|
2061
|
+
* meanwhile keeps it. A worker session that ended any other way keeps its subscriptions, and is passed over while the
|
|
2062
|
+
* fleet does not list it (`notify`); a run
|
|
2147
2063
|
* that resumes the session subscribes it again at its launch and each round while it is the card's current run.
|
|
2148
2064
|
* A creator's or manager's subscription is never ended here: its notices wait for the session to come back.
|
|
2149
2065
|
*/
|
|
2150
|
-
function endWorkerSubscriptions(board, fleet, report
|
|
2066
|
+
function endWorkerSubscriptions(board, fleet, report) {
|
|
2151
2067
|
// each worker subscriber's liveness is read before the write
|
|
2152
2068
|
const aliveNow = new Map();
|
|
2153
2069
|
if (fleet.ok) for (const { chat_id: address } of board.db.prepare("SELECT DISTINCT chat_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) aliveNow.set(address, subscriberAlive(fleet, address));
|
|
@@ -2158,7 +2074,7 @@ function endWorkerSubscriptions(board, fleet, report, natively = new Map()) {
|
|
|
2158
2074
|
for (const row of db.prepare("SELECT task_id, chat_id, thread_id FROM kanban_notify_subs WHERE platform = 'supercode' AND json_extract(delivery_metadata, '$.role') = 'worker'").all()) {
|
|
2159
2075
|
const address = row.chat_id, record = latest.get(address);
|
|
2160
2076
|
if (!record || record.session.released || held.has(address) || (fleet.ok && aliveNow.get(address) === true)) continue;
|
|
2161
|
-
const evidence = record.session.close_confirmed_at ? { by: 'board_close', run: record.run, closed_at: record.session.close_confirmed_at } :
|
|
2077
|
+
const evidence = record.session.close_confirmed_at ? { by: 'board_close', run: record.run, closed_at: record.session.close_confirmed_at } : null;
|
|
2162
2078
|
if (!evidence || !remove.run(row.task_id, address, row.thread_id).changes) continue;
|
|
2163
2079
|
if (!ended.has(address)) ended.set(address, { record, evidence, cards: [] });
|
|
2164
2080
|
ended.get(address).cards.push(row.task_id);
|
|
@@ -2170,63 +2086,54 @@ function endWorkerSubscriptions(board, fleet, report, natively = new Map()) {
|
|
|
2170
2086
|
});
|
|
2171
2087
|
}
|
|
2172
2088
|
|
|
2173
|
-
|
|
2174
|
-
|
|
2089
|
+
/**
|
|
2090
|
+
* Each subscriber's notices, filed in its mailbox (D138 step 6, D140): the dispatcher's job ends at the filing, a local
|
|
2091
|
+
* write under the notice's fixed key; the mailbox delivers it to the session, on this machine or through Teams on
|
|
2092
|
+
* another. Each subscription's notices go in order, and its cursor moves past a notice once it is filed (or passed
|
|
2093
|
+
* over); a filing that failed here leaves the cursor before it, and the next pass files it.
|
|
2094
|
+
*/
|
|
2095
|
+
function notify(board, fleet, report, entry) {
|
|
2175
2096
|
endWorkerSubscriptions(board, fleet, report);
|
|
2176
|
-
const
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
const refused = new Map(), discovered = new Map(), natively = new Map();
|
|
2180
|
-
// each subscription's notices go in order; different subscribers are told side by side
|
|
2181
|
-
await pool(subscriptions(board.db).filter((sub) => sub.platform === 'supercode'), SEND_CONCURRENCY, async (sub) => {
|
|
2182
|
-
if (sub.delivery_metadata?.dependent_card && sub.delivery_metadata.waiting_recipient) return;
|
|
2183
|
-
// a subscription whose last send failed waits out its retry time (as an effect does) before the next attempt
|
|
2184
|
-
if (Number(sub.delivery_metadata?.next_at ?? 0) > now()) return;
|
|
2097
|
+
for (const sub of subscriptions(board.db).filter((s) => s.platform === 'supercode')) {
|
|
2098
|
+
if (stopping) break;
|
|
2099
|
+
if (sub.delivery_metadata?.dependent_card && sub.delivery_metadata.waiting_recipient) continue;
|
|
2185
2100
|
const role = ['worker', 'dependency', 'manager'].includes(sub.delivery_metadata?.role) ? sub.delivery_metadata.role : 'creator';
|
|
2186
|
-
// a session the fleet does not list this
|
|
2101
|
+
// a session the fleet does not list this pass is told next pass: one reading is not a loss (a loaded box drops
|
|
2187
2102
|
// sessions from a read, and a dropped subscription never came back), and a session that is replaced on its card
|
|
2188
2103
|
// has its subscription moved when the new one starts
|
|
2189
|
-
if (role !== 'creator' && fleet.ok && !subscriberAlive(fleet, sub.chat_id))
|
|
2104
|
+
if (role !== 'creator' && fleet.ok && !subscriberAlive(fleet, sub.chat_id)) continue;
|
|
2190
2105
|
const kinds = heardBy(entry, role);
|
|
2191
|
-
if (!kinds.length)
|
|
2106
|
+
if (!kinds.length) continue;
|
|
2192
2107
|
const events = board.db.prepare(`SELECT id, kind, payload, created_at, run_id FROM task_events WHERE task_id = ? AND id > ? AND kind IN (${kinds.map(() => '?').join(', ')}) ORDER BY id`)
|
|
2193
2108
|
.all(sub.task_id, sub.last_event_id, ...kinds);
|
|
2194
|
-
if (!events.length)
|
|
2109
|
+
if (!events.length) continue;
|
|
2195
2110
|
// the card as the workflow names it (its status the workflow's: triaging, reviewing, stopped; `stored_as` kanban.db's)
|
|
2196
2111
|
const task = view(board, (db) => { const t = getTask(db, sub.task_id); return t ? { ...t, status: statusOf(db, t), stored_as: t.status } : t; });
|
|
2197
2112
|
// a worker whose run has ended is not woken by a notice (a second session would start working the card beside
|
|
2198
|
-
// the one the next run gets): the notice is filed
|
|
2113
|
+
// the one the next run gets): the notice is filed for its next turn
|
|
2199
2114
|
const working = role !== 'worker' || view(board, (db) => addressOf(currentRun(db, sub.task_id)?.metadata?.supercode) === sub.chat_id);
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
// forward past it: two passes may both send one notice, and its recipient gets it once; a send that fails, or a pass
|
|
2203
|
-
// that dies before its send, leaves the cursor behind the notice, so it is sent again.
|
|
2115
|
+
// a Claude session always starts a turn when mail reaches it, so its mail is always delivered now
|
|
2116
|
+
const wake = String(sub.chat_id).includes(':claude-code:') || (working && !(!entry.workflow.run && sub.delivery_mode === 'notify'));
|
|
2204
2117
|
const where = "WHERE task_id = ? AND platform = 'supercode' AND chat_id = ? AND thread_id = ?";
|
|
2205
2118
|
const identity = [sub.task_id, sub.chat_id, sub.thread_id ?? ''];
|
|
2206
2119
|
const forward = (to) => tx(board, (db) => db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ? ${where} AND last_event_id < ?`).run(to, ...identity, to));
|
|
2207
|
-
|
|
2120
|
+
// an event older than the workflow's notice_max_age_seconds is history, not news: it is passed over, never filed
|
|
2121
|
+
const maxAge = Number(entry.params.notice_max_age_seconds) || null;
|
|
2122
|
+
const stale = (ev) => maxAge && now() - Number(ev.created_at) > maxAge;
|
|
2123
|
+
// A backlog (a restart, a pass that did not run) is told as it stands now: news is coalesced by its subject, not its
|
|
2124
|
+
// kind alone (a later `assigned` says what an earlier one did, but each subtask added, dependency linked or unlinked
|
|
2125
|
+
// is its own fact, keyed by the card it names); a comment or an answer is always its own message. Only events that
|
|
2126
|
+
// would be told count: a stale one or the recipient's own act never supersedes one that would be.
|
|
2127
|
+
const subjectOf = (ev) => { const p = json(ev.payload, {}) ?? {}; return ['commented', 'answered'].includes(ev.kind) ? `${ev.kind}:${ev.id}` : `${ev.kind}:${p.subtask ?? p.parent ?? p.child ?? ''}`; };
|
|
2128
|
+
const latest = new Map(events.filter((ev) => !stale(ev) && !((json(ev.payload, {}) ?? {}).by === sub.chat_id)).map((ev) => [subjectOf(ev), ev.id]));
|
|
2129
|
+
let cursor = Number(sub.last_event_id);
|
|
2208
2130
|
try {
|
|
2209
|
-
// an event older than the workflow's notice_max_age_seconds is history, not news: it is passed over, never mailed
|
|
2210
|
-
const maxAge = Number(entry.params.notice_max_age_seconds) || null;
|
|
2211
|
-
// A backlog (a restart, a recipient that could not be reached) is told as it stands now: of one kind of news about
|
|
2212
|
-
// the card, only the latest (a later adoption_failed, blocked or moved says what the earlier one did); comments
|
|
2213
|
-
// and answers, each its own message, in full. A notice the board already recorded as taken by this address is never sent again.
|
|
2214
|
-
// only events that would be told count: a stale one or the recipient's own act never supersedes one that would be
|
|
2215
|
-
const sendable = (ev) => { const p = json(ev.payload, {}) ?? {}; return !(maxAge && now() - Number(ev.created_at) > maxAge) && !(p.by && p.by === sub.chat_id); };
|
|
2216
|
-
// News is coalesced by its subject, not its kind alone: a later `assigned` says what an earlier one did, but each
|
|
2217
|
-
// subtask added, dependency linked or unlinked is its own fact, keyed by the card it names; a comment or an
|
|
2218
|
-
// answer is always its own message.
|
|
2219
|
-
const subjectOf = (ev) => { const p = json(ev.payload, {}) ?? {}; return ['commented', 'answered'].includes(ev.kind) ? `${ev.kind}:${ev.id}` : `${ev.kind}:${p.subtask ?? p.parent ?? p.child ?? ''}`; };
|
|
2220
|
-
const latest = new Map(events.filter((ev) => sendable(ev)).map((ev) => [subjectOf(ev), ev.id]));
|
|
2221
|
-
const sentBefore = board.db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND kind IN ('notice_delivered', 'notice_filed') AND json_extract(payload, '$.source_event') = ? AND json_extract(payload, '$.address') = ? LIMIT 1");
|
|
2222
2131
|
for (const ev of events) {
|
|
2223
2132
|
const p = json(ev.payload, {}) ?? {};
|
|
2224
|
-
if (
|
|
2225
|
-
|
|
2226
|
-
if (sentBefore.get(sub.task_id, ev.id, sub.chat_id)) { cursor = ev.id; continue; }
|
|
2133
|
+
if (stale(ev)) { cursor = ev.id; report.stale_notices_passed = (report.stale_notices_passed ?? 0) + 1; continue; }
|
|
2227
2134
|
// nobody is told of their own act
|
|
2228
2135
|
if (p.by && p.by === sub.chat_id) { cursor = ev.id; continue; }
|
|
2229
|
-
if (latest.
|
|
2136
|
+
if (latest.get(subjectOf(ev)) !== ev.id) { cursor = ev.id; report.superseded_notices_passed = (report.superseded_notices_passed ?? 0) + 1; continue; }
|
|
2230
2137
|
let text;
|
|
2231
2138
|
if (ev.kind === 'commented') {
|
|
2232
2139
|
if (p.author === sub.chat_id) { cursor = ev.id; continue; }
|
|
@@ -2235,69 +2142,23 @@ async function notify(board, fleet, report, entry) {
|
|
|
2235
2142
|
} else text = noticeOf(entry, role, ev.kind, task, p);
|
|
2236
2143
|
if (text) {
|
|
2237
2144
|
const body = ['worker', 'dependency'].includes(role) ? identifyNotice(ev, text) : text;
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
try {
|
|
2242
|
-
receipt = await send(board, from, board.label, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { queue: String(sub.chat_id).includes(':claude-code:') ? false : !working || (!entry.workflow.run && sub.delivery_mode === 'notify'), subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
|
|
2243
|
-
// A notice the recipient's mailbox took is filed: the machine delivers it at the session's next turn, so the
|
|
2244
|
-
// board does not send it again.
|
|
2245
|
-
if (!receipt.delivered && !receipt.queued) throw new Error(receipt.reason ?? 'notice was not filed');
|
|
2246
|
-
} catch (error) {
|
|
2247
|
-
// two passes may send one notice at once, and the second send can fail while the first is taken: a notice
|
|
2248
|
-
// another pass already recorded as taken is no failure
|
|
2249
|
-
if (sentBefore.get(sub.task_id, ev.id, sub.chat_id)) { cursor = ev.id; forward(ev.id); continue; }
|
|
2250
|
-
if (!refused.has(sub.chat_id)) refused.set(sub.chat_id, error);
|
|
2251
|
-
throw error;
|
|
2252
|
-
}
|
|
2253
|
-
// recorded once, whichever pass's send the mailbox took
|
|
2254
|
-
tx(board, (db) => { if (!db.prepare("SELECT 1 FROM task_events WHERE task_id = ? AND kind IN ('notice_delivered', 'notice_filed') AND json_extract(payload, '$.source_event') = ? AND json_extract(payload, '$.address') = ?").get(sub.task_id, ev.id, sub.chat_id)) event(db, sub.task_id, receipt.delivered ? 'notice_delivered' : 'notice_filed', { source_event: ev.id, kind: ev.kind, address: sub.chat_id, receipt }); });
|
|
2145
|
+
// filed under its fixed key: a second pass filing the same notice files nothing new
|
|
2146
|
+
const filed = fileMail(board, sub.chat_id, body, `board-${sub.task_id}-${ev.id}-${sub.chat_id.slice(-12)}`, { wake, subject: `${ev.kind.replaceAll('_', ' ')} · run ${ev.run_id ?? 'none'} · event ${ev.id} · ${task?.title || sub.task_id}` });
|
|
2147
|
+
if (filed.new) tx(board, (db) => event(db, sub.task_id, 'notice_filed', { source_event: ev.id, kind: ev.kind, address: sub.chat_id, message_id: filed.message_id, wake }));
|
|
2255
2148
|
report.notified.push([sub.task_id, ev.kind, sub.chat_id]);
|
|
2256
|
-
forward(ev.id);
|
|
2257
2149
|
}
|
|
2258
2150
|
cursor = ev.id;
|
|
2151
|
+
forward(cursor);
|
|
2259
2152
|
}
|
|
2260
2153
|
} catch (error) {
|
|
2261
|
-
|
|
2262
|
-
// worker no run holds, and that is not a released session, the machine's native discovery is asked, and a
|
|
2263
|
-
// `persisted` reading ends its worker subscriptions after this round's deliveries.
|
|
2264
|
-
const record = latest.get(sub.chat_id);
|
|
2265
|
-
if (error.notRunning && role === 'worker' && record && !record.session.released && !held.has(sub.chat_id)) {
|
|
2266
|
-
if (!natively.has(sub.chat_id)) natively.set(sub.chat_id, nativeEnded(sub.chat_id, discovered));
|
|
2267
|
-
if (natively.get(sub.chat_id)) return;
|
|
2268
|
-
}
|
|
2269
|
-
failed = String(error.message).slice(-200);
|
|
2270
|
-
tx(board, (db) => {
|
|
2271
|
-
const kind = 'notice_failed';
|
|
2272
|
-
const previous = db.prepare('SELECT payload FROM task_events WHERE task_id=? AND kind=? ORDER BY id DESC LIMIT 1').get(sub.task_id, kind);
|
|
2273
|
-
const payload = { after_event: cursor, address: sub.chat_id, error: failed, retry: true };
|
|
2274
|
-
const { by: _, ...said } = json(previous?.payload, {}) ?? {};
|
|
2275
|
-
if (JSON.stringify(said) !== JSON.stringify(payload)) event(db, sub.task_id, kind, payload);
|
|
2276
|
-
});
|
|
2277
|
-
report.undelivered.push([sub.task_id, sub.chat_id, failed]);
|
|
2154
|
+
report.undelivered.push([sub.task_id, sub.chat_id, String(error.message).slice(-200)]);
|
|
2278
2155
|
}
|
|
2279
|
-
// the cursor moves forward past what was passed over (a failed delivery is retried next pass)
|
|
2280
2156
|
forward(cursor);
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
// the subscription stays, so the notices after it are still told (dropping it lost them when the round
|
|
2287
|
-
// subscribed the session again from the card's newest event).
|
|
2288
|
-
// failures are counted per notice: a failure of a later notice starts its own count
|
|
2289
|
-
const failures = failed ? (Number(sub.delivery_metadata?.failing_event) === trying ? Number(sub.delivery_metadata?.failures ?? 0) : 0) + 1 : 0;
|
|
2290
|
-
if (failed && failures >= NOTICE_ATTEMPTS && trying) {
|
|
2291
|
-
event(db, sub.task_id, 'notice_abandoned', { source_event: trying, address: sub.chat_id, attempts: failures, error: failed });
|
|
2292
|
-
db.prepare(`UPDATE kanban_notify_subs SET last_event_id = ?, delivery_metadata = ? ${where} AND last_event_id < ?`).run(trying, JSON.stringify({ ...sub.delivery_metadata, failures: 0, next_at: 0 }), ...identity, trying);
|
|
2293
|
-
return;
|
|
2294
|
-
}
|
|
2295
|
-
if (failures !== Number(sub.delivery_metadata?.failures ?? 0) || (failed && Number(sub.delivery_metadata?.failing_event) !== trying)) db.prepare(`UPDATE kanban_notify_subs SET delivery_metadata = ? ${where}`).run(JSON.stringify({ ...sub.delivery_metadata, failures, failing_event: failed ? trying : null, next_at: failed ? now() + Math.min(1800, 30 * 2 ** (failures - 1)) : 0 }), ...identity);
|
|
2296
|
-
if (task?.status === 'archived' && cursor >= events.at(-1).id) db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity);
|
|
2297
|
-
});
|
|
2298
|
-
});
|
|
2299
|
-
const persisted = new Map([...natively].filter(([, reading]) => reading));
|
|
2300
|
-
if (persisted.size) endWorkerSubscriptions(board, fleet, report, persisted);
|
|
2157
|
+
// a subscription past its card's last event ends with the card: a worker's when the card is done or archived (its
|
|
2158
|
+
// work is over, and its session may never run again), anyone's when it is archived
|
|
2159
|
+
const over = task && (task.stored_as === 'archived' || (role === 'worker' && task.stored_as === 'done'));
|
|
2160
|
+
if (over && cursor >= events.at(-1).id) tx(board, (db) => db.prepare(`DELETE FROM kanban_notify_subs ${where}`).run(...identity));
|
|
2161
|
+
}
|
|
2301
2162
|
}
|
|
2302
2163
|
|
|
2303
2164
|
// Hermes's terminal kinds for a chat subscriber, in its order of silence: archived and unblocked move the cursor and say nothing
|
|
@@ -2496,37 +2357,11 @@ export function boardLabel(root, slug) {
|
|
|
2496
2357
|
return slug && slug !== 'default' ? `${home}/${slug}` : home;
|
|
2497
2358
|
}
|
|
2498
2359
|
|
|
2499
|
-
/** `sc` without blocking the tick: notices to many sessions go out side by side. */
|
|
2500
|
-
function scAsync(args, { input = null, timeout = 120_000 } = {}) {
|
|
2501
|
-
return new Promise((done) => {
|
|
2502
|
-
const child = spawnChild(supercodeBin(), args, { cwd: tmpdir(), env: commandEnv(), stdio: ['pipe', 'pipe', 'pipe'] });
|
|
2503
|
-
let stdout = '';
|
|
2504
|
-
let stderr = '';
|
|
2505
|
-
const timer = setTimeout(() => { stderr += ` timed out after ${timeout}ms`; child.kill('SIGKILL'); }, timeout);
|
|
2506
|
-
child.stdout.setEncoding('utf8').on('data', (c) => { stdout += c; });
|
|
2507
|
-
child.stderr.setEncoding('utf8').on('data', (c) => { stderr += c; });
|
|
2508
|
-
child.on('error', (error) => { clearTimeout(timer); done({ code: null, stdout, stderr: stderr + error.message }); });
|
|
2509
|
-
child.on('close', (code) => { clearTimeout(timer); done({ code, stdout, stderr }); });
|
|
2510
|
-
child.stdin.end(input ?? '');
|
|
2511
|
-
});
|
|
2512
|
-
}
|
|
2513
|
-
|
|
2514
2360
|
// A dispatcher told to stop (its serve's SIGTERM, SIGINT or SIGHUP) starts nothing more in the round it is in: no further
|
|
2515
|
-
// board, card start, board message or notice. What it leaves
|
|
2361
|
+
// board, card start, board message or notice. What it leaves unfiled is not a failure; the next dispatcher files it.
|
|
2516
2362
|
let stopping = false;
|
|
2517
2363
|
export function stopDispatching() { stopping = true; }
|
|
2518
2364
|
|
|
2519
|
-
// how many deliveries a tick has in flight at once (each is a `supercode message` process of a few seconds)
|
|
2520
|
-
const SEND_CONCURRENCY = 8;
|
|
2521
|
-
|
|
2522
|
-
/** Run `fn` over `items`, at most `n` at a time. */
|
|
2523
|
-
async function pool(items, n, fn) {
|
|
2524
|
-
let next = 0;
|
|
2525
|
-
await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
|
|
2526
|
-
while (next < items.length && !stopping) { const i = next; next += 1; await fn(items[i]); }
|
|
2527
|
-
}));
|
|
2528
|
-
}
|
|
2529
|
-
|
|
2530
2365
|
/** A board file's fingerprint: a corrupt board is skipped until the file changes (Hermes retries it then). */
|
|
2531
2366
|
const fingerprint = (path) => { try { const st = statSync(path); return `${st.size}:${st.mtimeMs}`; } catch { return 'missing'; } };
|
|
2532
2367
|
const CORRUPT = /malformed|not a database|corrupt|disk image/i;
|
|
@@ -2681,10 +2516,10 @@ export async function deliver({ root, slugs = null }) {
|
|
|
2681
2516
|
setActor(boardAddress(board.label));
|
|
2682
2517
|
let mark = Date.now();
|
|
2683
2518
|
const phase = (name) => { const t = Date.now(); report.ms[name] = t - mark; mark = t; };
|
|
2684
|
-
// board mail
|
|
2685
|
-
|
|
2519
|
+
// board mail and notices are filed in their receivers' mailboxes (local writes); the mailbox delivers them
|
|
2520
|
+
fileBoardMail(board, report); phase('mail');
|
|
2686
2521
|
report.effects = await runEffects(board, [], { pending: true }); phase('effects');
|
|
2687
|
-
|
|
2522
|
+
notify(board, fleet, report, entry); phase('notify');
|
|
2688
2523
|
synchronizeAgents(board, report); phase('agents');
|
|
2689
2524
|
report.effects_failed = Number(board.db.prepare('SELECT COUNT(*) AS n FROM supercode_effects WHERE failed_at IS NOT NULL').get().n);
|
|
2690
2525
|
report.effects_waiting = Number(board.db.prepare('SELECT COUNT(*) AS n FROM supercode_effects WHERE failed_at IS NULL').get().n);
|
package/board/engine.mjs
CHANGED
|
@@ -477,8 +477,8 @@ const OWNED = new Set(['merge_reviewed', 'cleanup_workspace']);
|
|
|
477
477
|
const effectWait = (attempts) => Math.min(1800, 30 * 2 ** Math.max(0, attempts - 1));
|
|
478
478
|
// A delivery pass claims a row by compare-and-set on its next_at (set ahead only while the row is due) before trying it,
|
|
479
479
|
// so two passes (serve's, the hosted loop's, a person's `workflow deliver`, a verb's own) never try one effect at once.
|
|
480
|
-
// The claim outlasts any one attempt: each effect's I/O is bounded by its own timeouts (a
|
|
481
|
-
//
|
|
480
|
+
// The claim outlasts any one attempt: each effect's I/O is bounded by its own timeouts (a merge's MERGE_BUDGET plus one
|
|
481
|
+
// in-flight GitHub call), all well inside EFFECT_CLAIM.
|
|
482
482
|
const EFFECT_CLAIM = 900;
|
|
483
483
|
|
|
484
484
|
/**
|
|
@@ -487,8 +487,9 @@ const EFFECT_CLAIM = 900;
|
|
|
487
487
|
* to run after it, in the delivery pass, never inside a dispatch round.
|
|
488
488
|
*/
|
|
489
489
|
export function queueEffects(board, effects = []) {
|
|
490
|
-
//
|
|
491
|
-
|
|
490
|
+
// board mail is filed by the delivery pass from its board_message events (dispatch.mjs fileBoardMail), so the move's
|
|
491
|
+
// mail hint is not recorded
|
|
492
|
+
const fresh = effects.filter((e) => !e.durableId && e.kind !== 'run' && e.kind !== 'board_message');
|
|
492
493
|
const now_ = fresh.filter((e) => ['invalidate_descendants', 'close_session', 'run_ended', 'stop_run'].includes(e.kind));
|
|
493
494
|
const later = fresh.filter((e) => !now_.includes(e));
|
|
494
495
|
for (const e of now_) {
|
|
@@ -518,8 +519,6 @@ export async function runEffects(board, effects = [], { pending: due = false } =
|
|
|
518
519
|
let done, error = null;
|
|
519
520
|
try { done = await runEffect(board, e); } catch (thrown) { done = false; error = String(thrown?.message ?? thrown).slice(0, 500); }
|
|
520
521
|
if (!e.durableId) continue;
|
|
521
|
-
// a mail inside its batch window is not an attempt: it is looked at again in a few seconds
|
|
522
|
-
if (done === 'wait') { tx(board, (db) => db.prepare('UPDATE supercode_effects SET next_at = ? WHERE id = ?').run(now() + 5, e.durableId)); did.kept.push(e.durableId); continue; }
|
|
523
522
|
if (done) { tx(board, (db) => db.prepare('DELETE FROM supercode_effects WHERE id = ?').run(e.durableId)); did.done.push(`${e.durableId}:${e.kind}`); continue; }
|
|
524
523
|
did.kept.push(e.durableId);
|
|
525
524
|
const made = (attempts.get(e.durableId) ?? 0) + 1;
|
|
@@ -538,7 +537,6 @@ export async function runEffects(board, effects = [], { pending: due = false } =
|
|
|
538
537
|
} else if (made >= EFFECT_ATTEMPTS) {
|
|
539
538
|
db.prepare('UPDATE supercode_effects SET attempts = ?, failed_at = ?, last_error = ? WHERE id = ?').run(made, now(), reason, e.durableId);
|
|
540
539
|
if (card) event(db, card, 'effect_failed', { effect: e.durableId, kind: e.kind, attempts: made, error: reason, address: e.address ?? null }, e.runId ?? null);
|
|
541
|
-
if (card && e.kind === 'board_message' && e.event) event(db, card, 'board_message_abandoned', { event: e.event, attempts: made, reason });
|
|
542
540
|
} else db.prepare('UPDATE supercode_effects SET attempts = ?, next_at = ?, last_error = ? WHERE id = ?').run(made, now() + effectWait(made), reason, e.durableId);
|
|
543
541
|
});
|
|
544
542
|
}
|
|
@@ -550,12 +548,8 @@ async function runEffect(board, e) {
|
|
|
550
548
|
if (e.kind === 'session_notice') { const { sessionNotice } = await import('./dispatch.mjs'); return await sessionNotice(board, e) !== false; }
|
|
551
549
|
if (e.kind === 'cleanup_workspace') { cleanupWorkspace(board, e.id); return true; }
|
|
552
550
|
if (e.kind === 'declare_card_agent') { const { declareCardAgentEffect } = await import('./agent-declarations.mjs'); return declareCardAgentEffect(board, e) !== false; }
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
// a verb's own ask sends its card's mail at once; the mail's own effect row then finds it sent
|
|
556
|
-
if (!e.event) { await sendBoardMessages(board, { undelivered: [] }, e.id); return true; }
|
|
557
|
-
return await sendBoardMessage(board, e);
|
|
558
|
-
}
|
|
551
|
+
// a verb's own ask files its card's mail at once (a local write; the mailbox delivers it)
|
|
552
|
+
if (e.kind === 'board_message') { const { fileBoardMail } = await import('./dispatch.mjs'); fileBoardMail(board, { undelivered: [] }, e.id); return true; }
|
|
559
553
|
if (e.kind === 'merge_reviewed') { const { mergeReviewed } = await import('./dispatch.mjs'); return await mergeReviewed(board, e) !== false; }
|
|
560
554
|
if (e.kind === 'invalidate_descendants') { tx(board, (db) => invalidateDescendants(db, board, e.id)); return true; }
|
|
561
555
|
if (e.kind === 'stop_run') { stopWorker(board, e.runId); return true; }
|
|
@@ -928,7 +922,7 @@ export function diagnosePublishing(publishing) {
|
|
|
928
922
|
* same thing can go wrong, and supercode's own where only its sessions can): each `{ kind, severity, title, data,
|
|
929
923
|
* actions }`, worst first.
|
|
930
924
|
*/
|
|
931
|
-
export function diagnose(db, task, { failureLimit = DEFAULT_FAILURE_LIMIT, strandedSeconds = 30 * 60, blockedHours = 24, fleet = null, launchWindowSeconds = 300, publishing = null } = {}) {
|
|
925
|
+
export function diagnose(db, task, { failureLimit = DEFAULT_FAILURE_LIMIT, strandedSeconds = 30 * 60, blockedHours = 24, fleet = null, launchWindowSeconds = 300, publishing = null, expiredMail = null } = {}) {
|
|
932
926
|
const out = [];
|
|
933
927
|
const t = now();
|
|
934
928
|
const events = all(db, 'SELECT kind, payload, created_at FROM task_events WHERE task_id = ? ORDER BY id', task.id);
|
|
@@ -985,15 +979,19 @@ export function diagnose(db, task, { failureLimit = DEFAULT_FAILURE_LIMIT, stran
|
|
|
985
979
|
? { kind: 'effect_failed', severity: 'warning', title: `its ${kind} failed for good after ${row.attempts} attempts: ${row.last_error ?? 'no reason recorded'}`, data: { effect: row.id, kind, attempts: row.attempts }, actions: [] }
|
|
986
980
|
: { kind: 'effect_stalled', severity: 'error', title: `its ${kind} has failed ${row.attempts} times and is still tried: ${row.last_error ?? 'no reason recorded'}`, data: { effect: row.id, kind, attempts: row.attempts }, actions: [] });
|
|
987
981
|
}
|
|
988
|
-
// notices the board gave up on: each one, with the event it was about and who was not told
|
|
989
|
-
for (const row of all(db, "SELECT payload FROM task_events WHERE task_id = ? AND kind = 'notice_abandoned' ORDER BY id DESC LIMIT 20", task.id)) {
|
|
990
|
-
const p = json(row.payload, {}) ?? {};
|
|
991
|
-
out.push({ kind: 'notice_abandoned', severity: 'warning', title: `the notice of event ${p.source_event} to ${p.address} was abandoned after ${p.attempts} attempts: ${p.error ?? 'no reason recorded'}`, data: { event: p.source_event, address: p.address, attempts: p.attempts }, actions: [] });
|
|
992
|
-
}
|
|
993
982
|
// publications of the card the machine's connector abandoned: the team refused each one past its attempts
|
|
994
983
|
for (const a of (publishing?.abandoned ?? []).filter((a) => a.card === task.id).slice(0, 20)) {
|
|
995
984
|
out.push({ kind: 'publication_abandoned', severity: 'warning', title: `its ${a.kind} (sequence ${a.sequence}) was not published to the team: refused ${a.attempts} times (${a.status} ${a.code ?? ''}): ${a.error}`, data: { publication: a.publication, sequence: a.sequence, status: a.status, code: a.code }, actions: [] });
|
|
996
985
|
}
|
|
986
|
+
// mail the board filed for the card whose delivery the mailbox gave up on (its wake expired): it stays unread there
|
|
987
|
+
if (expiredMail?.size) {
|
|
988
|
+
for (const row of all(db, "SELECT kind, payload FROM task_events WHERE task_id = ? AND kind IN ('notice_filed', 'session_notice_filed', 'actor_notice_filed', 'board_message_sent') ORDER BY id DESC LIMIT 200", task.id)) {
|
|
989
|
+
const p = json(row.payload, {}) ?? {}, expiry = p.message_id && expiredMail.get(p.message_id);
|
|
990
|
+
if (!expiry) continue;
|
|
991
|
+
const what = row.kind === 'notice_filed' ? `its ${p.kind} notice` : row.kind === 'board_message_sent' ? 'its board mail' : `its ${row.kind.replace('_filed', '').replace('_', ' ')}`;
|
|
992
|
+
out.push({ kind: 'mail_expired', severity: 'warning', title: `${what} to ${expiry.to} was never delivered: ${expiry.reason} (${expiry.attempts} attempts); it stays unread in that mailbox`, data: { message_id: p.message_id, to: expiry.to, attempts: expiry.attempts, expired_at_ms: expiry.expired_at_ms }, actions: [] });
|
|
993
|
+
}
|
|
994
|
+
}
|
|
997
995
|
const order = ['critical', 'error', 'warning'];
|
|
998
996
|
return out.sort((a, b) => order.indexOf(a.severity) - order.indexOf(b.severity));
|
|
999
997
|
}
|
package/board/request-mail.mjs
CHANGED
|
@@ -2,10 +2,8 @@ import { event, json, now } from './store.mjs';
|
|
|
2
2
|
|
|
3
3
|
export function pendingRequestMail(db, taskId = null) {
|
|
4
4
|
const pending = db.prepare(`SELECT id, task_id, payload, created_at FROM task_events original
|
|
5
|
-
WHERE kind='board_message' AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind
|
|
6
|
-
AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id)
|
|
7
|
-
AND NOT EXISTS (SELECT 1 FROM task_events gone WHERE gone.kind='board_message_abandoned'
|
|
8
|
-
AND gone.task_id=original.task_id AND json_extract(gone.payload,'$.event')=original.id) ORDER BY id`).all()
|
|
5
|
+
WHERE kind='board_message' AND NOT EXISTS (SELECT 1 FROM task_events sent WHERE sent.kind IN ('board_message_sent', 'board_message_passed')
|
|
6
|
+
AND sent.task_id=original.task_id AND json_extract(sent.payload,'$.event')=original.id) ORDER BY id`).all()
|
|
9
7
|
.map(row => ({ ...row, mail: json(row.payload, {}) ?? {} }));
|
|
10
8
|
if (!pending.length) return { ready: [], nextAt: Infinity };
|
|
11
9
|
const batches = db.prepare("SELECT payload FROM task_events WHERE kind='board_message_batch' AND id>=? ORDER BY id").all(pending[0].id)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-orchestrator",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.49",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
|
|
6
6
|
"exports": {
|