bareloop 0.32.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +4 -1
- package/bareloop.context.md +163 -82
- package/package.json +1 -1
- package/src/authorflow.js +5 -3
- package/src/authorjob.js +3 -1
- package/src/authorrun.js +36 -8
- package/src/authorscout.js +10 -6
- package/src/bundlerun.js +266 -0
- package/src/cli.js +24 -352
- package/src/interviewrun.js +6 -4
- package/src/invoke.js +22 -0
- package/src/job.js +2 -2
- package/src/judged.js +27 -5
- package/src/kinds.js +1 -0
- package/src/ledger.js +3 -4
- package/src/monthly.js +112 -35
- package/src/panel/authorroutes.js +12 -2
- package/src/panel/authorsession.js +24 -4
- package/src/panel/server.js +18 -13
- package/src/planrun.js +10 -8
- package/src/providerrows.js +87 -1
- package/src/providers.js +4 -4
- package/src/run.js +7 -2
- package/src/runlist.js +94 -7
- package/src/source.js +8 -3
- package/src/u-watchdog.mjs +296 -0
- package/src/userrun.js +207 -65
- package/types/authorflow.d.ts +8 -2
- package/types/authorjob.d.ts +6 -1
- package/types/authorscout.d.ts +11 -2
- package/types/bundlerun.d.ts +26 -0
- package/types/invoke.d.ts +8 -0
- package/types/judged.d.ts +10 -3
- package/types/monthly.d.ts +57 -10
- package/types/panel/authorroutes.d.ts +15 -0
- package/types/planrun.d.ts +13 -1
- package/types/providerrows.d.ts +66 -0
- package/types/providers.d.ts +1 -2
- package/types/run.d.ts +14 -1
- package/types/runlist.d.ts +79 -3
- package/types/source.d.ts +8 -3
- package/types/userrun.d.ts +49 -0
package/src/providers.js
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
// `provider` name to the bare-agent constructor it drives, the env var
|
|
3
3
|
// carrying its key, and the per-tier model table — including any
|
|
4
4
|
// provider-specific request-key gating. Before this module existed, both
|
|
5
|
-
// runners constructed `AnthropicProvider` BY NAME (`
|
|
6
|
-
// `
|
|
5
|
+
// runners constructed `AnthropicProvider` BY NAME (the old `buildProviders` in
|
|
6
|
+
// `src/cli.js` and `scripts/run-u.mjs`'s top-level wiring, both since replaced
|
|
7
|
+
// by `buildRunnerProviders` below) and any second
|
|
7
8
|
// provider would have meant scattering `if (provider === …)` across both.
|
|
8
9
|
//
|
|
9
10
|
// hamr's ruling (PRD 30.7, 2026-09-09): `deepseek-chat` is THE secondary
|
|
@@ -265,8 +266,7 @@ export function makeProvider(providerName, { apiKey, model, baseUrl } = {}) {
|
|
|
265
266
|
|
|
266
267
|
/**
|
|
267
268
|
* Build the `{provider, providerFor, judgeProvider}` triple both runners
|
|
268
|
-
* need (`src/
|
|
269
|
-
* wiring) — ONE seam instead of two copies of the same construction and
|
|
269
|
+
* need (`src/authorrun.js` and `src/panel/authorsession.js` call it) — ONE seam instead of two copies of the same construction and
|
|
270
270
|
* tier-memoization logic. `model`/`tierModels` are already-RESOLVED inputs
|
|
271
271
|
* (the caller runs `resolveWorkerModel` itself, exactly as both runners did
|
|
272
272
|
* before this factory existed — that decision stays where each runner's own
|
package/src/run.js
CHANGED
|
@@ -95,6 +95,11 @@ async function primitiveSmoke(workdir) {
|
|
|
95
95
|
* metered claude-json text provider for the toolless drafter); ignored on every Loop-driven provider
|
|
96
96
|
* @param {(type: string, data?: object) => object} opts.emit spine emitter
|
|
97
97
|
* @param {(tier: string) => any} [opts.providerFor] P: per-step model-tier provider factory (forwarded to the plan flow)
|
|
98
|
+
* @param {{in: number, out: number}|null} [opts.rates] the CUSTOMER's own price for the worker's key row
|
|
99
|
+
* (`ratesFor`, src/providerrows.js — USD per 1K tokens), resolved once by the door and handed to every
|
|
100
|
+
* worker/planner Loop as bare-agent's `rates` (rounds then read `rateSource:'caller'`). Absent = the
|
|
101
|
+
* built-in guess, exactly as before. A Loop-driven provider only: a native session reports its own cost.
|
|
102
|
+
* @param {{in: number, out: number}|null} [opts.judgeRates] the same, for the judge's own key row
|
|
98
103
|
* @param {string|null} [opts.judgeModel] SOFTGREEN — WHICH model the judge provider drives
|
|
99
104
|
* (PRD item 32.1), resolved by the CALLER (`resolveJudge`, src/judged.js) and passed
|
|
100
105
|
* through untouched. Travels with `judgeProvider`; absent, a judged stage stops as a
|
|
@@ -265,7 +270,7 @@ async function primitiveSmoke(workdir) {
|
|
|
265
270
|
* deliberately NOT in the resumable/checkpoint set (`src/reuse.js`) — an
|
|
266
271
|
* unknown in-flight state is not a known-safe resume point.
|
|
267
272
|
*/
|
|
268
|
-
export async function runJob(rawSpec, { approvals, workdir, provider, nativeProvider, providerFor, judgeProvider = null, judgeModel = null, emit, capRuns = 3, strikeLimit, shellCapUsd = 2, closeTimeoutMs, closeDir = null, layerRoot = false, readShim = false, scout = true, bridge = null, draftSpentUsd = 0, draftSpendComplete = true, priorSpentUsd = 0, priorSpendComplete = true, priorWallMs = 0, resumeSeed = null, resumeGrades = [], resumeReplans = null, resumeBranch = null, humanRuling = null, heldRuling = null, reviewDoor = null, doorRerun = null, resumable = true }) {
|
|
273
|
+
export async function runJob(rawSpec, { approvals, workdir, provider, nativeProvider, providerFor, judgeProvider = null, judgeModel = null, rates = null, judgeRates = null, emit, capRuns = 3, strikeLimit, shellCapUsd = 2, closeTimeoutMs, closeDir = null, layerRoot = false, readShim = false, scout = true, bridge = null, draftSpentUsd = 0, draftSpendComplete = true, priorSpentUsd = 0, priorSpendComplete = true, priorWallMs = 0, resumeSeed = null, resumeGrades = [], resumeReplans = null, resumeBranch = null, humanRuling = null, heldRuling = null, reviewDoor = null, doorRerun = null, resumable = true }) {
|
|
269
274
|
// THE READ SHIM's ARM, resolved at the door — the FIRST thing this entry does,
|
|
270
275
|
// before the ledger, before the approval gate, before a byte of the spec is read.
|
|
271
276
|
// An unrecognised spelling throws here at zero cost instead of being coerced by
|
|
@@ -532,7 +537,7 @@ export async function runJob(rawSpec, { approvals, workdir, provider, nativeProv
|
|
|
532
537
|
// accounts it natively (F12) and the job-end money contract is unchanged.
|
|
533
538
|
{
|
|
534
539
|
const outcome = await runPlan(job, {
|
|
535
|
-
workdir, provider, nativeProvider, providerFor, judgeProvider, judgeModel, emit: meter, capRuns, ...(strikeLimit !== undefined ? { strikeLimit } : {}), closeTimeoutMs, closeDir, layerRoot, readShim, scout, bridge, priorWallMs: chainWallMs, resumeSeed, resumeGrades, resumeReplans, resumeBranch, humanRuling, heldRuling, reviewDoor, doorRerun, priorSpentUsd: chainFoldUsd, resumable,
|
|
540
|
+
workdir, provider, nativeProvider, providerFor, judgeProvider, judgeModel, rates, judgeRates, emit: meter, capRuns, ...(strikeLimit !== undefined ? { strikeLimit } : {}), closeTimeoutMs, closeDir, layerRoot, readShim, scout, bridge, priorWallMs: chainWallMs, resumeSeed, resumeGrades, resumeReplans, resumeBranch, humanRuling, heldRuling, reviewDoor, doorRerun, priorSpentUsd: chainFoldUsd, resumable,
|
|
536
541
|
// hamr's ruling 2026-09-28 — ONE cap covers drafting + run: the run's
|
|
537
542
|
// enforced ceiling is the signed `budgetUsd` (unchanged, still in the
|
|
538
543
|
// spec hash) MINUS whatever drafting already spent on this job before
|
package/src/runlist.js
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// caller doesn't know one) — a relative path here would resolve differently
|
|
14
14
|
// depending on who later reads the file. `via` says how the row was minted:
|
|
15
15
|
// `'run-u'` (src/userrun.js, the person-path run), `'bundle'`
|
|
16
|
-
// (src/
|
|
16
|
+
// (src/userrun.js's bundle door, `bareloop run`), or `'backfill'` (`bareloop runs
|
|
17
17
|
// backfill`, reconstructed from an archived spine already on disk).
|
|
18
18
|
//
|
|
19
19
|
// SECRETS: a row carries only paths/ids/names — never a key, a prompt, or any
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
|
|
30
30
|
import {
|
|
31
31
|
existsSync, mkdirSync, appendFileSync, chmodSync, readdirSync, statSync,
|
|
32
|
+
readFileSync,
|
|
32
33
|
} from 'node:fs';
|
|
33
34
|
import { homedir } from 'node:os';
|
|
34
35
|
import {
|
|
@@ -57,7 +58,9 @@ export function runlistPath(home) {
|
|
|
57
58
|
}
|
|
58
59
|
|
|
59
60
|
/**
|
|
60
|
-
* @typedef {{ at: string, runid: string, job: string, spine: string, patient: string|null, via: 'run-u'|'bundle'|'backfill' }} RunRow
|
|
61
|
+
* @typedef {{ at: string, runid: string, job: string, spine: string, patient: string|null, via: 'run-u'|'bundle'|'backfill', pid?: number, capUsd?: number }} RunRow
|
|
62
|
+
* `pid`/`capUsd` (optional; older and backfilled rows carry neither): the runner's process id and the
|
|
63
|
+
* leg's $ cap — what the monthly limit holds while that pid is a live bareloop runner (src/monthly.js).
|
|
61
64
|
*/
|
|
62
65
|
|
|
63
66
|
/**
|
|
@@ -87,7 +90,8 @@ export function appendRun(row, opts = {}) {
|
|
|
87
90
|
mkdirSync(home, { recursive: true, mode: 0o700 });
|
|
88
91
|
if (existsSync(path)) {
|
|
89
92
|
const { records } = parseJsonl(path);
|
|
90
|
-
|
|
93
|
+
// only RUN rows count: a settled/released event names the same runid for another purpose
|
|
94
|
+
if (records.some((r) => r && !r.type && r.runid === row.runid)) {
|
|
91
95
|
return { appended: false, reason: 'duplicate-runid' };
|
|
92
96
|
}
|
|
93
97
|
}
|
|
@@ -96,18 +100,86 @@ export function appendRun(row, opts = {}) {
|
|
|
96
100
|
return { appended: true };
|
|
97
101
|
}
|
|
98
102
|
|
|
103
|
+
/**
|
|
104
|
+
* @typedef {{ runid: string, type: 'settled'|'released', by: string, at: string, reason?: string, spentUsd?: number, spendComplete?: boolean }} RunEvent
|
|
105
|
+
* A claim's write-back, appended (never rewritten) beside the run rows. `settled` = the run's claim on the
|
|
106
|
+
* monthly limit is over (`spentUsd`/`spendComplete` are its final figure, `by` is WHO wrote it: the run itself
|
|
107
|
+
* at its `job-end`, or the next run that found its process gone, `reason: 'process gone'`). `released` = the
|
|
108
|
+
* claim was taken and then given back unspent, its run never started. The month's totals keep reading real
|
|
109
|
+
* spend off the spines; an event only releases the hold and records who wrote it.
|
|
110
|
+
*/
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Append one {@link RunEvent}. Not locked: one short line, one append. THROWS on IO failure.
|
|
114
|
+
* @param {RunEvent} event
|
|
115
|
+
* @param {{ home?: string }} [opts]
|
|
116
|
+
* @returns {void}
|
|
117
|
+
*/
|
|
118
|
+
export function appendRunEvent(event, opts = {}) {
|
|
119
|
+
const path = runlistPath(opts.home);
|
|
120
|
+
mkdirSync(runlistHome(opts.home), { recursive: true, mode: 0o700 });
|
|
121
|
+
appendFileSync(path, `${JSON.stringify(event)}\n`);
|
|
122
|
+
try { chmodSync(path, 0o600); } catch { /* best-effort perms */ }
|
|
123
|
+
}
|
|
124
|
+
|
|
99
125
|
/**
|
|
100
126
|
* Tolerant reader — reuses {@link parseJsonl} (never a second hand-rolled
|
|
101
127
|
* parser). An absent file reads as an empty list, not an error: a fresh
|
|
102
|
-
* install has never run `appendRun` yet.
|
|
128
|
+
* install has never run `appendRun` yet. FOLDS the file: `rows` are the RUNS
|
|
129
|
+
* only (an event line is never listed as a run, and a run whose claim was
|
|
130
|
+
* `released` never started, so it is not listed either); `events` are the
|
|
131
|
+
* write-backs, in file order.
|
|
103
132
|
* @param {{ home?: string }} [opts]
|
|
104
|
-
* @returns {{ rows: RunRow[], skipped: number }}
|
|
133
|
+
* @returns {{ rows: RunRow[], skipped: number, events: RunEvent[] }}
|
|
105
134
|
*/
|
|
106
135
|
export function readRunList(opts = {}) {
|
|
107
136
|
const path = runlistPath(opts.home);
|
|
108
|
-
if (!existsSync(path)) return { rows: [], skipped: 0 };
|
|
137
|
+
if (!existsSync(path)) return { rows: [], skipped: 0, events: [] };
|
|
109
138
|
const { records, skipped } = parseJsonl(path);
|
|
110
|
-
|
|
139
|
+
// the FIRST settle for a runid is authoritative: two runs may both close the same dead claim, and the
|
|
140
|
+
// second note is a duplicate (a `released` line likewise); file order decides, later copies are ignored
|
|
141
|
+
const seen = new Set();
|
|
142
|
+
/** @type {RunEvent[]} */
|
|
143
|
+
const events = records.filter((r) => {
|
|
144
|
+
if (!r || (r.type !== 'settled' && r.type !== 'released')) return false;
|
|
145
|
+
const key = `${r.type}:${r.runid}`;
|
|
146
|
+
if (seen.has(key)) return false;
|
|
147
|
+
seen.add(key);
|
|
148
|
+
return true;
|
|
149
|
+
});
|
|
150
|
+
const released = new Set(events.filter((e) => e.type === 'released').map((e) => e.runid));
|
|
151
|
+
const rows = records.filter((r) => r && !r.type && !released.has(r.runid));
|
|
152
|
+
return { rows, skipped, events };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Is this pid a live bareloop runner? `process.kill(pid, 0)` says whether the
|
|
157
|
+
* pid exists (EPERM = it exists, just not ours); `/proc/<pid>/cmdline` says whether it is a
|
|
158
|
+
* bareloop runner — a recycled pid that now belongs to some other program is NOT. A runner is
|
|
159
|
+
* a node executable (argv[0] is `node`, `nodejs`, `node22`, ...) with a later argv entry named
|
|
160
|
+
* `bareloop`, `bareloop.mjs`, `run-u.mjs` or `u-watchdog.mjs` (the basename, so the npm bin
|
|
161
|
+
* symlink counts), or a process whose argv[0] is itself one of those names. A non-node program
|
|
162
|
+
* that merely has such a name in its arguments (`nvim /x/bareloop`, `git -C /x/bareloop`) is not
|
|
163
|
+
* a runner. Residual, accepted: a recycled pid that is a node program with such an argument
|
|
164
|
+
* still reads as a runner. The wrong direction to err is "not a runner" on a real live run — it
|
|
165
|
+
* would release that run's cap. Where `/proc` is unreadable (not Linux, or not ours) the
|
|
166
|
+
* existence test alone decides: alive. The ONE spelling — the monthly limit's holds, and
|
|
167
|
+
* `--resume`'s old-pid check share it.
|
|
168
|
+
* @param {number} pid
|
|
169
|
+
* @returns {boolean}
|
|
170
|
+
*/
|
|
171
|
+
export function isLiveRunner(pid) {
|
|
172
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
173
|
+
try { process.kill(pid, 0); } catch (/** @type {any} */ e) { if (e?.code !== 'EPERM') return false; }
|
|
174
|
+
/** @type {string|null} */
|
|
175
|
+
let cmdline = null;
|
|
176
|
+
try { cmdline = readFileSync(`/proc/${pid}/cmdline`, 'utf8'); } catch { /* no /proc, or not ours */ }
|
|
177
|
+
if (cmdline === null) return true;
|
|
178
|
+
const names = ['bareloop', 'bareloop.mjs', 'run-u.mjs', 'u-watchdog.mjs'];
|
|
179
|
+
const [argv0 = '', ...rest] = cmdline.split('\0');
|
|
180
|
+
const first = basename(argv0);
|
|
181
|
+
if (names.includes(first)) return true;
|
|
182
|
+
return first.startsWith('node') && rest.some((a) => names.includes(basename(a)));
|
|
111
183
|
}
|
|
112
184
|
|
|
113
185
|
/**
|
|
@@ -141,6 +213,21 @@ function runidForSpine(spinePath) {
|
|
|
141
213
|
// the panel) because the monthly limit reads the same rule (src/monthly.js).
|
|
142
214
|
export const DIED_MTIME_MS = 10 * 60 * 1000;
|
|
143
215
|
|
|
216
|
+
/**
|
|
217
|
+
* The ONE "is this run alive" rule, for the panel's glyph ([▶] vs [?]) and the monthly limit's hold. A
|
|
218
|
+
* row that carries its runner's `pid` is alive exactly while that pid is a live bareloop runner
|
|
219
|
+
* ({@link isLiveRunner}) however quiet its spine; only a row with NO pid (an older row) falls back to
|
|
220
|
+
* the spine-mtime rule above. A run that has ENDED (a `job-end` on its spine) is a different fact the
|
|
221
|
+
* callers read off the spine itself — this answers "is a process still running it", nothing more.
|
|
222
|
+
* @param {{ pid?: number, spine: string }} row
|
|
223
|
+
* @param {number} [nowMs] the clock for the mtime fallback
|
|
224
|
+
* @returns {boolean}
|
|
225
|
+
*/
|
|
226
|
+
export function runIsAlive(row, nowMs = Date.now()) {
|
|
227
|
+
if (Number.isInteger(row.pid)) return isLiveRunner(/** @type {number} */ (row.pid));
|
|
228
|
+
try { return nowMs - statSync(row.spine).mtimeMs <= DIED_MTIME_MS; } catch { return false; }
|
|
229
|
+
}
|
|
230
|
+
|
|
144
231
|
// A backfill scan of a real patients directory can be arbitrarily deep
|
|
145
232
|
// (person-path runs archive several levels down: `<dir>/<proj>/out/
|
|
146
233
|
// source-<x>/<proj>-bareloop/u-<id>.jsonl`, 4 levels under the dir a person
|
package/src/source.js
CHANGED
|
@@ -441,11 +441,15 @@ async function fetchOnce(url, timeoutMs) {
|
|
|
441
441
|
* never hands a repo source's destination to `copyOut` — wiring it into
|
|
442
442
|
* `writeScope` is M3/M4's job, not this one.
|
|
443
443
|
* @param {{source: string, into: string, destination?: string,
|
|
444
|
-
* fetchTimeoutMs?: number
|
|
445
|
-
* callers omit it and get
|
|
444
|
+
* fetchTimeoutMs?: number, afterScan?: () => (void|Promise<void>)}} args
|
|
445
|
+
* `fetchTimeoutMs` is test-only — production callers omit it and get
|
|
446
|
+
* `PROVIDER_TIMEOUT_MS`. `afterScan` is a test seam too: called exactly once,
|
|
447
|
+
* after the secret scan finished and before the freeze loop re-reads any
|
|
448
|
+
* file's bytes, so a test can swap a file inside that window
|
|
449
|
+
* deterministically. Production never sets it; absent, behaviour is unchanged.
|
|
446
450
|
* @returns {Promise<{stop: null, into: string, tree: string, manifestPath: string, manifest: object}|SourceRefusal>}
|
|
447
451
|
*/
|
|
448
|
-
export async function prepareSource({ source, into, destination, fetchTimeoutMs = PROVIDER_TIMEOUT_MS }) {
|
|
452
|
+
export async function prepareSource({ source, into, destination, fetchTimeoutMs = PROVIDER_TIMEOUT_MS, afterScan }) {
|
|
449
453
|
const intoAbs = resolve(into);
|
|
450
454
|
if (existsSync(intoAbs)) {
|
|
451
455
|
return refuse('into-exists', `${intoAbs} already exists — a source door writes a FRESH tree, never reuses one (the export worktree rule)`);
|
|
@@ -674,6 +678,7 @@ export async function prepareSource({ source, into, destination, fetchTimeoutMs
|
|
|
674
678
|
// (verified: 0o755 in, 0o755 out; a plain `writeFile` defaults to 0o644),
|
|
675
679
|
// so the replacement chmod's the destination from a fresh `stat` to keep
|
|
676
680
|
// that behaviour unchanged.
|
|
681
|
+
if (afterScan) await afterScan();
|
|
677
682
|
for (let i = 0; i < frozen.files.length; i += 1) {
|
|
678
683
|
const f = frozen.files[i];
|
|
679
684
|
const dest = join(treeDir, treeRel(f.rel));
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
// The OUTSIDE watchdog (F67) — the only stop that survives a frozen run.
|
|
2
|
+
//
|
|
3
|
+
// MEASURED reason it exists. U run ms3jh76q spent 81.5 minutes emitting nothing,
|
|
4
|
+
// then died `read ETIMEDOUT` — the kernel giving up on a socket the process never
|
|
5
|
+
// read. The in-process stall fuse (F66) did not fire once, and it is not broken:
|
|
6
|
+
// firing was reproduced against that run's exact shape (four rounds, then
|
|
7
|
+
// silence). It never RAN, because a timer in the same event loop as whatever
|
|
8
|
+
// froze that loop shares its fate. Every guard bareloop had lived inside the run:
|
|
9
|
+
//
|
|
10
|
+
// bare-agent's timeoutMs idle SOCKET timer — resets on bytes, saw nothing wrong
|
|
11
|
+
// the wall clock read BETWEEN rounds — no round ever arrived
|
|
12
|
+
// the stall fuse (F66) a setTimeout — cannot fire on a blocked event loop
|
|
13
|
+
//
|
|
14
|
+
// So this one is a separate process and shares nothing with the run. It reads one
|
|
15
|
+
// file's mtime and calls kill(2). That is the whole design.
|
|
16
|
+
//
|
|
17
|
+
// It is deliberately DUMB, and each omission is load-bearing:
|
|
18
|
+
// - it parses no JSON (a malformed spine must not blind the guard)
|
|
19
|
+
// - it imports nothing from src/ (no shared code means no shared failure)
|
|
20
|
+
// - it holds no model of what the run is doing (nothing to get out of sync)
|
|
21
|
+
// - it reads bytes, not meaning (cost is one stat(2) per poll)
|
|
22
|
+
//
|
|
23
|
+
// TWO triggers, ONE reconciled decision (hamr's ruling, 2026-07-30: "the kill from
|
|
24
|
+
// outside should check for activity/bytes or other markers for activity, not a
|
|
25
|
+
// silent kill"):
|
|
26
|
+
// STALE no progress for longer than the worst LEGAL silence → the run is wedged
|
|
27
|
+
// DEAD past the deadline (wall + stages × close timeout) AND the spine flat for
|
|
28
|
+
// the whole dead window → the process is producing nothing, and its own
|
|
29
|
+
// deadline has already funded every legal silence
|
|
30
|
+
// Past the deadline but still ACTIVE is NOT a kill any more. The in-process fuses
|
|
31
|
+
// and the money cap own bounding a run that is alive; this guard owns processes
|
|
32
|
+
// that are dead. A live run is never killed from outside purely by the clock —
|
|
33
|
+
// that is the point of the change, and the previous behaviour could destroy a
|
|
34
|
+
// verdict mid-close. Overshoot is reported LOUD on every poll instead. The full
|
|
35
|
+
// decision table sits above the decision code.
|
|
36
|
+
//
|
|
37
|
+
// The kill is RECORDED next to the spine, and it is never silent: before the
|
|
38
|
+
// signal goes out this process says which deadline passed, which marker it
|
|
39
|
+
// checked, its last value and age, and why it judged the process dead. A run
|
|
40
|
+
// stopped by the arbiter must never read as a mystery crash — that is the
|
|
41
|
+
// distinction between a governance stop and a casualty, and it decides whether the
|
|
42
|
+
// row is evidence or noise (F45/F48).
|
|
43
|
+
//
|
|
44
|
+
// Usage:
|
|
45
|
+
// node src/u-watchdog.mjs --spine <path> --pid <n>
|
|
46
|
+
// [--stale-ms N] [--wall-ms N] [--grace-ms N] [--dead-ms N] [--poll-ms N] [--term-grace-ms N]
|
|
47
|
+
import { statSync, writeFileSync, renameSync, unlinkSync } from 'node:fs';
|
|
48
|
+
|
|
49
|
+
const arg = (/** @type {string} */ n, /** @type {string|null} */ dflt = null) => {
|
|
50
|
+
const i = process.argv.indexOf(`--${n}`);
|
|
51
|
+
return i === -1 ? dflt : (process.argv[i + 1] ?? dflt);
|
|
52
|
+
};
|
|
53
|
+
// A flag the operator PASSED is never quietly replaced by a default. An ABSENT flag
|
|
54
|
+
// takes its default (`--wall-ms` omitted is the legal unbounded choice, and run-u
|
|
55
|
+
// omits it on a spec with no `maxWallMs`); a flag that is present and unreadable —
|
|
56
|
+
// `0`, a negative, `undefined`, `soon` — is an operator error and stops this process
|
|
57
|
+
// LOUD. Silently defaulting it disarms a trigger while the caller's own banner still
|
|
58
|
+
// claims the bound, which is the guard reading zero and saying nothing: the
|
|
59
|
+
// blind-instrument class, and W-2's case besides (a wall remainder of nothing is a
|
|
60
|
+
// decision, not an unbounded launch).
|
|
61
|
+
const num = (/** @type {string} */ n, /** @type {number|null} */ dflt) => {
|
|
62
|
+
const v = arg(n);
|
|
63
|
+
if (v === null) return dflt;
|
|
64
|
+
const x = Number(v);
|
|
65
|
+
if (!Number.isFinite(x) || x <= 0) {
|
|
66
|
+
console.error(`[u-watchdog] REFUSED: --${n} ${JSON.stringify(v)} is not a positive number of milliseconds. It is NOT defaulted — a defaulted bound disarms this guard's trigger while the run's banner still advertises one. Pass a real remainder, or omit the flag to take the default deliberately (an omitted --wall-ms is the unbounded choice, and the STALE trigger stays armed either way).`);
|
|
67
|
+
process.exit(2);
|
|
68
|
+
}
|
|
69
|
+
return x;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
const spine = arg('spine');
|
|
73
|
+
const pid = Number(arg('pid'));
|
|
74
|
+
if (!spine || !Number.isInteger(pid) || pid <= 0) {
|
|
75
|
+
console.error('usage: u-watchdog.mjs --spine <path> --pid <n> [--stale-ms N] [--wall-ms N] [--grace-ms N] [--dead-ms N] [--poll-ms N] [--term-grace-ms N]');
|
|
76
|
+
process.exit(2);
|
|
77
|
+
}
|
|
78
|
+
// The watched pid must be this process's PARENT. kill(pid, 0) answers "is SOME
|
|
79
|
+
// process holding this pid" — after a SIGKILL/OOM the kernel can recycle the
|
|
80
|
+
// run's pid onto a stranger, and an armed guard would SIGTERM/SIGKILL it and
|
|
81
|
+
// leave a marker blaming the run. The parent link cannot be recycled: run-u
|
|
82
|
+
// spawns this guard as its direct child, and `process.ppid` flips to init
|
|
83
|
+
// within one poll of the parent dying — verified live, including under SIGKILL.
|
|
84
|
+
// A guard pointed at a non-parent refuses LOUD at startup (exit 2, no marker):
|
|
85
|
+
// an armed guard aimed at a stranger is worse than no guard, and a silent no-op
|
|
86
|
+
// is the blind-instrument class.
|
|
87
|
+
if (process.ppid !== pid) {
|
|
88
|
+
console.error(`[u-watchdog] REFUSED: --pid ${pid} is not this process's parent (ppid ${process.ppid}) — the liveness check is the parent link, which pid reuse cannot forge`);
|
|
89
|
+
process.exit(2);
|
|
90
|
+
}
|
|
91
|
+
// Defaults are DERIVED, not chosen: the stale window is twice the in-process fuse
|
|
92
|
+
// (F66, hamr's 5 min), so the inner guard always gets its three tries first and
|
|
93
|
+
// this one only speaks when the inner one could not. It stays a flag because a
|
|
94
|
+
// threshold is arbiter territory — a number the agent picks off its own sample is
|
|
95
|
+
// fitted, not set.
|
|
96
|
+
const staleMs = num('stale-ms', 600_000) ?? 600_000;
|
|
97
|
+
const wallMs = num('wall-ms', null);
|
|
98
|
+
// The wall's grace covers the CLOSE, which legitimately runs after the last round
|
|
99
|
+
// and is the longest legal silence a healthy run produces. It is sized the same way
|
|
100
|
+
// the run's own clock sizes its enforced deadline (src/clock.js: maxWallMs +
|
|
101
|
+
// closeStages × closeTimeoutMs) and the same way the stale window is sized —
|
|
102
|
+
// STAGES × the close timeout, because a staged close (PRD v1.28) gives EVERY stage
|
|
103
|
+
// the full timeout. The 900_000 default is ONE stage: the single-predicate floor,
|
|
104
|
+
// not a number any staged job should run on. A caller with a staged close passes
|
|
105
|
+
// its own (scripts/run-u.mjs does; a 4-stage close needs 60min, not 15min, and the
|
|
106
|
+
// stale window already got this arithmetic under F67 while the grace did not).
|
|
107
|
+
const graceMs = num('grace-ms', 900_000) ?? 900_000;
|
|
108
|
+
// How long the spine must be flat, PAST the deadline, before the process is judged
|
|
109
|
+
// dead. Derived, not chosen: it is the in-process stall fuse's own window (F66,
|
|
110
|
+
// hamr's 5 minutes). Past the deadline the grace has ALREADY paid
|
|
111
|
+
// for every legal close stage, so there is no legal silence left to protect — one
|
|
112
|
+
// fuse window is patience, not budget. Never above the stale window, which would
|
|
113
|
+
// make it inert.
|
|
114
|
+
const deadMs = Math.min(num('dead-ms', 300_000) ?? 300_000, staleMs);
|
|
115
|
+
const pollMs = num('poll-ms', 5_000) ?? 5_000;
|
|
116
|
+
// How long SIGTERM gets before SIGKILL. A flag, not a constant, for the same reason
|
|
117
|
+
// every other window here is one — and because a test cannot exercise the escalation
|
|
118
|
+
// against a ten-minute wait. The DEFAULT is unchanged (10s): long enough for a live run
|
|
119
|
+
// to flush its spine, short enough that a run swallowing the polite signal cannot
|
|
120
|
+
// outlive its own guard.
|
|
121
|
+
const termGraceMs = num('term-grace-ms', 10_000) ?? 10_000;
|
|
122
|
+
|
|
123
|
+
const startedAt = Date.now();
|
|
124
|
+
const iso = (/** @type {number} */ ms) => new Date(ms).toISOString();
|
|
125
|
+
// Liveness is the PARENT LINK, not kill(pid, 0): once the parent dies this
|
|
126
|
+
// process is reparented and ppid stops matching — reuse-proof where a pid probe
|
|
127
|
+
// is not (see the startup refusal above).
|
|
128
|
+
const alive = () => process.ppid === pid;
|
|
129
|
+
|
|
130
|
+
// ── THE ACTIVITY MARKER ─────────────────────────────────────────────────────────
|
|
131
|
+
// "Is it dead" is answered by EVIDENCE, never by the clock alone. ONE marker:
|
|
132
|
+
//
|
|
133
|
+
// spine PROGRESS. Bytes reaching the run's own append-only log. Growth here is
|
|
134
|
+
// the run doing the job. One stat(2): size OR mtime moving counts.
|
|
135
|
+
//
|
|
136
|
+
// ONE on purpose. A second, CPU-liveness marker (utime+stime+cutime+cstime from
|
|
137
|
+
// /proc/<pid>/stat) was built here and then MEASURED broken in BOTH directions, so
|
|
138
|
+
// it is deleted rather than kept as decoration:
|
|
139
|
+
// - it read DEAD on a live run. A close runs its suite in a CHILD, and a child's
|
|
140
|
+
// ticks are credited to the parent only when the parent REAPS it — so the
|
|
141
|
+
// busiest minutes of a multi-minute close show a parent whose own ticks never
|
|
142
|
+
// move.
|
|
143
|
+
// - it read ALIVE on a dead one. run-u's event-loop lag sampler wakes this very
|
|
144
|
+
// process every second, moving its ticks roughly every 3.3s; at the production
|
|
145
|
+
// constants (deadMs 300_000 against pollMs 5_000) the marker could therefore
|
|
146
|
+
// never go flat, and the deadline kill could effectively never fire. The kill
|
|
147
|
+
// tests only passed at --dead-ms 500 — three orders of magnitude below what
|
|
148
|
+
// production runs on.
|
|
149
|
+
// An instrument that cannot see the variable makes every reading unusable, and a
|
|
150
|
+
// veto that is always ON is not a safety, it is a disarmed trigger. hamr's ruling,
|
|
151
|
+
// 2026-07-30: "keep what is simpler/available" — the spine, which is the marker
|
|
152
|
+
// that actually distinguishes progress from silence.
|
|
153
|
+
|
|
154
|
+
/** spine size+mtime in one stat(2). A missing file reads as the watchdog's own
|
|
155
|
+
* start: a run that wedges BEFORE its first event is the most stalled run there
|
|
156
|
+
* is, and treating "no file" as "no news is good news" would blind the guard to
|
|
157
|
+
* exactly that case. */
|
|
158
|
+
const readSpine = () => { try { const st = statSync(spine); return { mtimeMs: st.mtimeMs, size: st.size }; } catch { return { mtimeMs: startedAt, size: -1 }; } };
|
|
159
|
+
|
|
160
|
+
let spineSeen = readSpine();
|
|
161
|
+
let spineMovedAt = spineSeen.mtimeMs;
|
|
162
|
+
|
|
163
|
+
/** advance the marker's last-movement stamp. Called once per poll, before any
|
|
164
|
+
* decision reads it. */
|
|
165
|
+
function sampleMarkers(/** @type {number} */ now) {
|
|
166
|
+
const st = readSpine();
|
|
167
|
+
if (st.size !== spineSeen.size) spineMovedAt = now;
|
|
168
|
+
if (st.mtimeMs > spineMovedAt) spineMovedAt = st.mtimeMs;
|
|
169
|
+
spineSeen = st;
|
|
170
|
+
}
|
|
171
|
+
/** what every log line and every kill record says about the marker: its value and
|
|
172
|
+
* its age. Kept under a `markers` key — the record is a read-once forensic
|
|
173
|
+
* artifact, and a reader comparing an old kill to a new one should see the shape
|
|
174
|
+
* hold and the CPU entry simply be gone. */
|
|
175
|
+
const markerReport = (/** @type {number} */ now) => ({
|
|
176
|
+
spine: { path: spine, bytes: spineSeen.size < 0 ? 'no file yet' : spineSeen.size, lastMovedAt: iso(spineMovedAt), coldMs: now - spineMovedAt },
|
|
177
|
+
});
|
|
178
|
+
const spineColdMs = (/** @type {number} */ now) => now - spineMovedAt;
|
|
179
|
+
|
|
180
|
+
function stop(/** @type {string} */ reason, /** @type {string} */ judgement, /** @type {object} */ detail) {
|
|
181
|
+
const now = Date.now();
|
|
182
|
+
const markers = markerReport(now);
|
|
183
|
+
const record = {
|
|
184
|
+
watchdog: 'u-watchdog', reason, killed: true, pid, spine,
|
|
185
|
+
// Was the target still THERE when the decision was taken (F72 park B — the checks
|
|
186
|
+
// that ran before the signal belong in the record, not only in the reasoning). It
|
|
187
|
+
// is the parent link, not `kill(pid, 0)`: a pid probe answers "is SOME process
|
|
188
|
+
// holding this pid" and cannot survive pid reuse (see the startup refusal). A
|
|
189
|
+
// `false` here says the run died on its own between the last poll and this line —
|
|
190
|
+
// a different event from stopping a running process, and the record must be able
|
|
191
|
+
// to say which one happened.
|
|
192
|
+
pidAlive: alive(),
|
|
193
|
+
at: iso(now),
|
|
194
|
+
startedAt: iso(startedAt),
|
|
195
|
+
elapsedMs: now - startedAt,
|
|
196
|
+
// Every bound this guard was ARMED with, so the decision is reproducible from the
|
|
197
|
+
// record alone — including the two that shape what happens next (`pollMs`, which
|
|
198
|
+
// says how many flat observations the age above represents, and `termGraceMs`,
|
|
199
|
+
// which says how long the run had to die politely).
|
|
200
|
+
staleMs, wallMs, graceMs, deadMs, pollMs, termGraceMs,
|
|
201
|
+
lastEventAt: iso(spineMovedAt),
|
|
202
|
+
judgement, markers,
|
|
203
|
+
...detail,
|
|
204
|
+
};
|
|
205
|
+
// Write the verdict BEFORE the kill: if this process dies mid-stop, the record of why
|
|
206
|
+
// must already be on disk. A kill with no explanation is the failure mode this whole
|
|
207
|
+
// file exists to prevent.
|
|
208
|
+
//
|
|
209
|
+
// TEMP + RENAME, because the record is read by a script that runs after a crash
|
|
210
|
+
// (run-u.mjs / run-reuse.mjs both `JSON.parse` it): rename is atomic within a
|
|
211
|
+
// directory, so a reader can only ever see the whole file or no file — never the
|
|
212
|
+
// half-written one a direct write leaves if this process is killed mid-write.
|
|
213
|
+
//
|
|
214
|
+
// And it is WRAPPED, because a guard that can be disarmed through its own logging is
|
|
215
|
+
// not a guard (F70: the instrument carrying the failure mode it exists to catch). The
|
|
216
|
+
// kill below runs whether or not this succeeded; a failure is named on stderr so a
|
|
217
|
+
// missing marker is never read as "the guard never fired".
|
|
218
|
+
const reportPath = `${spine}.watchdog.json`;
|
|
219
|
+
const tmpPath = `${reportPath}.tmp`;
|
|
220
|
+
try {
|
|
221
|
+
writeFileSync(tmpPath, `${JSON.stringify(record, null, 2)}\n`);
|
|
222
|
+
renameSync(tmpPath, reportPath);
|
|
223
|
+
} catch (e) {
|
|
224
|
+
console.error(`[u-watchdog] REPORT WRITE FAILED (${reportPath}): ${e?.message ?? e} — the kill still proceeds; this stderr IS the record now`);
|
|
225
|
+
try { unlinkSync(tmpPath); } catch { /* nothing to clean, or unwritable too */ }
|
|
226
|
+
}
|
|
227
|
+
// NEVER a silent kill (hamr, 2026-07-30). Three lines, in the order a reader
|
|
228
|
+
// needs them: what fired, which bound it was measured against, and the exact
|
|
229
|
+
// marker evidence that says the process is dead rather than slow.
|
|
230
|
+
console.error(`[u-watchdog] KILL ${reason}: pid ${pid} judged DEAD — ${judgement}`);
|
|
231
|
+
console.error(`[u-watchdog] deadline: ${JSON.stringify({ elapsedMs: record.elapsedMs, wallMs, graceMs, staleMs, deadMs, ...detail })}`);
|
|
232
|
+
console.error(`[u-watchdog] marker checked: ${JSON.stringify(markers)}`);
|
|
233
|
+
console.error(`[u-watchdog] SIGTERM -> pid ${pid}; SIGKILL in ${termGraceMs}ms if it is still there`);
|
|
234
|
+
try { process.kill(pid, 'SIGTERM'); } catch { /* already gone */ }
|
|
235
|
+
// SIGTERM lets the run flush its spine. A wedged event loop cannot run a signal
|
|
236
|
+
// handler, so SIGKILL is the one that actually lands — but only after giving the
|
|
237
|
+
// recoverable case its chance. A run that INSTALLS a handler and swallows the signal
|
|
238
|
+
// reaches the same place: the polite signal is an opportunity, never a veto.
|
|
239
|
+
setTimeout(() => {
|
|
240
|
+
if (alive()) {
|
|
241
|
+
console.error(`[u-watchdog] SIGKILL -> pid ${pid}: it survived the ${termGraceMs}ms grace`);
|
|
242
|
+
try { process.kill(pid, 'SIGKILL'); } catch { /* raced */ }
|
|
243
|
+
}
|
|
244
|
+
process.exit(1);
|
|
245
|
+
}, termGraceMs);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
console.error(`[u-watchdog] watching pid ${pid} — marker [spine], stale ${Math.round(staleMs / 1000)}s${wallMs ? `, wall ${Math.round(wallMs / 1000)}s (+${Math.round(graceMs / 1000)}s grace, then ${Math.round(deadMs / 1000)}s of a flat spine)` : ', no wall'}`);
|
|
249
|
+
|
|
250
|
+
// ── THE DECISION ────────────────────────────────────────────────────────────────
|
|
251
|
+
// One table, evaluated every poll after the marker is sampled. "Past deadline"
|
|
252
|
+
// means elapsed >= wallMs + graceMs, where the grace is stages × close timeout.
|
|
253
|
+
//
|
|
254
|
+
// past deadline | spine (progress) | action
|
|
255
|
+
// --------------+-------------------+--------------------------------------------
|
|
256
|
+
// any | cold >= staleMs | KILL 'stale'
|
|
257
|
+
// no | cold < staleMs | quiet — the run is inside its worst legal
|
|
258
|
+
// | | silence
|
|
259
|
+
// yes | cold < deadMs | LOUD, no kill (still progressing)
|
|
260
|
+
// yes | cold >= deadMs | KILL 'wall-dead'
|
|
261
|
+
//
|
|
262
|
+
// Two things this table says on purpose:
|
|
263
|
+
// 1. Only BYTES hold a kill off. Nothing else is consulted — in particular not
|
|
264
|
+
// CPU, which was measured unable to tell a working process from a spun-out
|
|
265
|
+
// one and, worse, unable to go flat at all at the production constants (see
|
|
266
|
+
// THE ACTIVITY MARKER above). A hard-frozen event loop burns 100% CPU forever
|
|
267
|
+
// and emits nothing; it is F67's own case, and it must die on both triggers.
|
|
268
|
+
// 2. Past the deadline with the spine still moving, NOTHING is killed. The
|
|
269
|
+
// in-process fuses (F66), the wall clock and the money cap own bounding a run
|
|
270
|
+
// that is alive; this guard owns processes that are dead. It says so loudly
|
|
271
|
+
// every poll rather than going quiet — the overshoot is still a fact the
|
|
272
|
+
// operator must see, it is just not this process's to end.
|
|
273
|
+
const tick = setInterval(() => {
|
|
274
|
+
if (!alive()) { clearInterval(tick); process.exit(0); } // the run finished on its own
|
|
275
|
+
const now = Date.now();
|
|
276
|
+
sampleMarkers(now);
|
|
277
|
+
const spineCold = spineColdMs(now);
|
|
278
|
+
const elapsed = now - startedAt;
|
|
279
|
+
|
|
280
|
+
if (spineCold >= staleMs) {
|
|
281
|
+
clearInterval(tick);
|
|
282
|
+
stop('stale', `the spine has not grown for ${Math.round(spineCold / 1000)}s, past the ${Math.round(staleMs / 1000)}s window sized above the worst LEGAL silence (a full staged close) — the run is producing nothing, whatever it is or is not burning CPU on`, { coldMs: spineCold });
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
if (wallMs === null || elapsed < wallMs + graceMs) return;
|
|
286
|
+
|
|
287
|
+
const overMs = elapsed - (wallMs + graceMs);
|
|
288
|
+
if (spineCold < deadMs) {
|
|
289
|
+
// LOUD, every poll. hamr: the outside kill checks for activity/bytes, not the
|
|
290
|
+
// clock alone — so an overrun that is still working gets named, not executed.
|
|
291
|
+
console.error(`[u-watchdog] PAST DEADLINE by ${Math.round(overMs / 1000)}s (wall ${Math.round(wallMs / 1000)}s + ${Math.round(graceMs / 1000)}s close grace) — NOT killing: still ACTIVE on spine (grew ${Math.round(spineCold / 1000)}s ago, ${spineSeen.size < 0 ? 'no file yet' : `${spineSeen.size} bytes`}). This guard stops DEAD processes; a live run is bounded by its own fuses and its money cap.`);
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
clearInterval(tick);
|
|
295
|
+
stop('wall-dead', `${Math.round(overMs / 1000)}s past the deadline (wall ${Math.round(wallMs / 1000)}s + ${Math.round(graceMs / 1000)}s close grace, which has already funded every legal close stage) AND the spine flat for at least ${Math.round(deadMs / 1000)}s — no bytes are reaching the run's own log, so there is nothing left that could still be producing a verdict`, { requestedMs: wallMs, enforcedMs: wallMs + graceMs, pastDeadlineMs: overMs });
|
|
296
|
+
}, pollMs);
|