@feigi/fleet-ctl 3.37.2 → 3.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/board.mjs +9 -6
- package/scripts/compute-board.mjs +13 -8
- package/scripts/fleet-state.mjs +164 -67
- package/scripts/fleet-tick.mjs +130 -17
- package/scripts/ledger.mjs +35 -10
- package/skills/run-team/SKILL.md +61 -20
package/package.json
CHANGED
package/scripts/board.mjs
CHANGED
|
@@ -1134,19 +1134,22 @@ export async function gather({ ledgerFile, prevFile, stateFile = null, scriptDir
|
|
|
1134
1134
|
// reading. Defaulted rather than resolved here for gather()'s own reason —
|
|
1135
1135
|
// every caller in this file passes the instance's answer, and a second
|
|
1136
1136
|
// resolution could name a different workspace's beat.
|
|
1137
|
-
// `ticked`
|
|
1138
|
-
// "no try/catch, no tryRun shape" rule just above
|
|
1139
|
-
//
|
|
1140
|
-
//
|
|
1137
|
+
// `ticked`, `acted` and `controller` ride beside `beat`, same file same
|
|
1138
|
+
// read, and under the same "no try/catch, no tryRun shape" rule just above:
|
|
1139
|
+
// a run is judged from all of its own marks, and the record's `at` is what
|
|
1140
|
+
// tells this run's marks from the previous run's (fleet-state.mjs's
|
|
1141
|
+
// assessBeat has the rule).
|
|
1141
1142
|
const priorState = stateFile ? readState(stateFile, NAME) : null;
|
|
1142
1143
|
const beat = priorState?.beat ?? null;
|
|
1143
1144
|
const ticked = priorState?.ticked ?? null;
|
|
1145
|
+
const acted = priorState?.acted ?? null;
|
|
1146
|
+
const controller = priorState?.controller ?? null;
|
|
1144
1147
|
// Whose stall it is, off the controller record in the same file — judged
|
|
1145
1148
|
// here because it reads the process table, and only when the mark is
|
|
1146
1149
|
// stalled, so a healthy board reads none. computeBoard() words it.
|
|
1147
1150
|
const now = Date.now();
|
|
1148
|
-
const stallOwnerOf = isStalled(assessBeat({ beat, ticked, now })) ? stallOwner(priorState, NAME) : null;
|
|
1149
|
-
return { ledger, issues, prs, merged, ci, prev, repo, repoUrl, workspace, port, spend, beat, ticked, stallOwner: stallOwnerOf, poolOk,
|
|
1151
|
+
const stallOwnerOf = isStalled(assessBeat({ beat, ticked, acted, controller, now })) ? stallOwner(priorState, NAME) : null;
|
|
1152
|
+
return { ledger, issues, prs, merged, ci, prev, repo, repoUrl, workspace, port, spend, beat, ticked, acted, controller, stallOwner: stallOwnerOf, poolOk,
|
|
1150
1153
|
poolCapped, prsCapped,
|
|
1151
1154
|
now, interval };
|
|
1152
1155
|
}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// fleet-state.mjs owns the heartbeat's `beat` key, and the rule for reading
|
|
8
8
|
// that key travels with it rather than being copied here. That module's own
|
|
9
9
|
// I/O (its path probe, its file read) is never called from this file; only
|
|
10
|
-
// assessBeat/isStalled/stallReport are, all of which are functions of their
|
|
10
|
+
// assessBeat/isStalled/isReported/stallReport are, all of which are functions of their
|
|
11
11
|
// arguments. Importing the rule is what keeps the cockpit's stall wording and
|
|
12
12
|
// fleet-tick's identical — two spellings of "this run is dead" would be two
|
|
13
13
|
// answers the operator has to reconcile at 3am.
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
// it to its ticket number regardless; see the same paragraph below.
|
|
25
25
|
// fleet-tick.mjs's I/O and main() never run here, main() being guarded on
|
|
26
26
|
// argv[1].
|
|
27
|
-
import { assessBeat, isStalled, stallReport, NO_OWNER } from "./fleet-state.mjs";
|
|
27
|
+
import { assessBeat, isStalled, isReported, stallReport, NO_OWNER } from "./fleet-state.mjs";
|
|
28
28
|
import { parseToken, rowPr, premisesOf, HALT_CAUSES, REVIEW, REVIEWED } from "./ledger-grammar.mjs";
|
|
29
29
|
import { latestFinisherAttempts } from "./fleet-tick.mjs";
|
|
30
30
|
|
|
@@ -382,9 +382,14 @@ function splitNumbered(line) {
|
|
|
382
382
|
// `owner` is whose stall it is, fleet-state.mjs's stallOwner() answer: it
|
|
383
383
|
// reads the process table, which is I/O, so gather() judges it and this
|
|
384
384
|
// function only words it. Absent is no controller named.
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
385
|
+
//
|
|
386
|
+
// The previous run's recorded stop is shown too, until this run's first tick
|
|
387
|
+
// lands: it is that run's end, worded as such, with nothing of this run's on
|
|
388
|
+
// it.
|
|
389
|
+
function stall(marks, tickets, pool, { ledgerOk, poolOk, owner }, now) {
|
|
390
|
+
const verdict = assessBeat({ ...marks, now });
|
|
391
|
+
if (!isReported(verdict)) return null;
|
|
392
|
+
if (!isStalled(verdict)) return { ...verdict, text: stallReport(verdict) };
|
|
388
393
|
const claimed = ledgerOk ? tickets.filter((t) => t.column !== "POOL" && t.column !== "MERGED").length : null;
|
|
389
394
|
const supply = poolOk ? pool : null;
|
|
390
395
|
return { ...verdict, claimed, supply, owner, text: stallReport(verdict, { claimed, supply, owner }) };
|
|
@@ -584,9 +589,9 @@ export function computeBoard(inputs) {
|
|
|
584
589
|
// beaten and one that is beating normally — the page hides the banner on
|
|
585
590
|
// either, because a banner that fires on every healthy tick is a banner
|
|
586
591
|
// the operator learns to read past.
|
|
587
|
-
//
|
|
592
|
+
// The marks pass through bare, unlike `spend` two fields up: that
|
|
588
593
|
// coalesce is load-bearing because the value is serialised into
|
|
589
|
-
// board.json as-is, while these
|
|
594
|
+
// board.json as-is, while these are only ever fed to assessBeat(),
|
|
590
595
|
// which already treats `undefined` and `null` identically — a `?? null`
|
|
591
596
|
// here would cost a line to say nothing stall() does not already do.
|
|
592
597
|
//
|
|
@@ -594,7 +599,7 @@ export function computeBoard(inputs) {
|
|
|
594
599
|
// `ledgerState`'s own `?? "read"` default just above: every caller that
|
|
595
600
|
// builds inputs by hand (every test in this suite) means a read that
|
|
596
601
|
// succeeded, and only gather() — which now sets both — can say otherwise.
|
|
597
|
-
liveness: stall(inputs.beat, inputs.ticked, tickets, pool, {
|
|
602
|
+
liveness: stall({ beat: inputs.beat, ticked: inputs.ticked, acted: inputs.acted, controller: inputs.controller }, tickets, pool, {
|
|
598
603
|
ledgerOk: (ledger.state ?? "read") === "read",
|
|
599
604
|
poolOk: inputs.poolOk ?? true,
|
|
600
605
|
owner: inputs.stallOwner ?? NO_OWNER,
|
package/scripts/fleet-state.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// The heartbeat's shared state file, owned jointly by fleet-tick.mjs,
|
|
2
|
-
// fleet-heartbeat.mjs and ledger.mjs
|
|
2
|
+
// fleet-heartbeat.mjs and ledger.mjs — so the path and the file's shape
|
|
3
3
|
// live here rather than in spellings that drift. Same rule arg.mjs states for
|
|
4
4
|
// the digits guard: what travels between scripts is the rule, never a copy of
|
|
5
5
|
// it.
|
|
@@ -35,6 +35,18 @@
|
|
|
35
35
|
// when rotate wrote it — a mark older than `at` is the previous
|
|
36
36
|
// run's. Removed, not left stale, by a rotate that finds no omp
|
|
37
37
|
// ancestor.
|
|
38
|
+
// acted ledger.mjs only. When a ledger write last landed: every
|
|
39
|
+
// subcommand that writes the ledger stamps it, rotate in the same
|
|
40
|
+
// write as `controller`. A busy run writes its ledger on every
|
|
41
|
+
// dispatch and settle, far more often than it ticks, and never
|
|
42
|
+
// beats while there is work, so this is the mark that tells a busy
|
|
43
|
+
// run from a dead one.
|
|
44
|
+
// seen fleet-tick only. {member: firstSeenAtMs}: the first tick that saw
|
|
45
|
+
// each live ledger token, so a later tick can say how long a token
|
|
46
|
+
// has stayed live — the ledger records that a member was dispatched,
|
|
47
|
+
// never when. Pruned to the live set on every tick, so a settled
|
|
48
|
+
// member's stamp does not outlive it. Carried, never written, by
|
|
49
|
+
// the heartbeat and by ledger.mjs.
|
|
38
50
|
//
|
|
39
51
|
// One writer per key. A key two scripts wrote would need locking to be
|
|
40
52
|
// correct, and none of them is in a position to hold one. That holds per key,
|
|
@@ -105,7 +117,7 @@ export function readState(path, name) {
|
|
|
105
117
|
// to judge and nothing to report. A zeroed mark would date the beat to the
|
|
106
118
|
// epoch and read as decades overdue on a run that has simply not started
|
|
107
119
|
// one — the cry-wolf direction this key must never fail in.
|
|
108
|
-
const fresh = { quiet: 0, elapsed: 0, digest: "", beat: null, ticked: null, controller: null, rest: {} };
|
|
120
|
+
const fresh = { quiet: 0, elapsed: 0, digest: "", beat: null, ticked: null, acted: null, controller: null, seen: {}, rest: {} };
|
|
109
121
|
// The same state, flagged: the file EXISTS and could not be used. A caller
|
|
110
122
|
// that acts on the absence of a record — ledger.mjs rotate — must tell this
|
|
111
123
|
// from `fresh`, which is a run that has simply not written one yet.
|
|
@@ -172,7 +184,8 @@ export function readState(path, name) {
|
|
|
172
184
|
};
|
|
173
185
|
// `ticked` carries no interval — fleet-tick has no promise to fail, only an
|
|
174
186
|
// occurrence — so it is a mark of one field, valid or absent, same "never
|
|
175
|
-
// zero" rule as `mark` above for the same cry-wolf reason.
|
|
187
|
+
// zero" rule as `mark` above for the same cry-wolf reason. `acted` is the
|
|
188
|
+
// same shape for the same reason.
|
|
176
189
|
const tick = (v) => {
|
|
177
190
|
if (!isRecord(v)) return null;
|
|
178
191
|
const at = num(v.at);
|
|
@@ -197,14 +210,26 @@ export function readState(path, name) {
|
|
|
197
210
|
at: Number.isInteger(v.at) && v.at > 0 ? v.at : null,
|
|
198
211
|
};
|
|
199
212
|
};
|
|
200
|
-
|
|
213
|
+
// `seen` is validated per entry, not as a unit: each stamp is a separate
|
|
214
|
+
// member's age, so one junk entry costs only that member its age. A stamp
|
|
215
|
+
// that cannot be read restarts that member's clock, which is the late
|
|
216
|
+
// direction — never the early one, where a healthy member reads as stuck.
|
|
217
|
+
const stamps = (v) => {
|
|
218
|
+
if (!isRecord(v)) return {};
|
|
219
|
+
const out = {};
|
|
220
|
+
for (const [member, at] of Object.entries(v)) if (Number.isInteger(at) && at > 0) out[member] = at;
|
|
221
|
+
return out;
|
|
222
|
+
};
|
|
223
|
+
const { quiet, elapsed, digest, beat, ticked, acted, controller, seen, ...rest } = parsed;
|
|
201
224
|
return {
|
|
202
225
|
quiet: num(quiet),
|
|
203
226
|
elapsed: num(elapsed),
|
|
204
227
|
digest: typeof digest === "string" ? digest : "",
|
|
205
228
|
beat: mark(beat),
|
|
206
229
|
ticked: tick(ticked),
|
|
230
|
+
acted: tick(acted),
|
|
207
231
|
controller: ctl(controller),
|
|
232
|
+
seen: stamps(seen),
|
|
208
233
|
rest,
|
|
209
234
|
};
|
|
210
235
|
}
|
|
@@ -252,11 +277,15 @@ export function writeState(path, name, prev, patch) {
|
|
|
252
277
|
// every hold and would otherwise erase fleet-tick's `ticked` the first
|
|
253
278
|
// time a heartbeat lands after a busy stretch.
|
|
254
279
|
...(prev.ticked ? { ticked: prev.ticked } : {}),
|
|
255
|
-
// And again for ledger.mjs
|
|
280
|
+
// And again for ledger.mjs's `controller` and `acted`, which both scripts
|
|
256
281
|
// above would otherwise erase on their next write. A patch REMOVES a key
|
|
257
282
|
// by naming it with the value `undefined`, which JSON.stringify drops —
|
|
258
283
|
// the way rotate retires a record no omp ancestor replaces.
|
|
259
284
|
...(prev.controller ? { controller: prev.controller } : {}),
|
|
285
|
+
...(prev.acted ? { acted: prev.acted } : {}),
|
|
286
|
+
// `seen` is fleet-tick's, and the heartbeat and ledger.mjs write this file
|
|
287
|
+
// too: carried the same way, so neither erases the ages the tick keeps.
|
|
288
|
+
...(prev.seen && Object.keys(prev.seen).length > 0 ? { seen: prev.seen } : {}),
|
|
260
289
|
...patch,
|
|
261
290
|
};
|
|
262
291
|
const tmp = `${path}.${process.pid}.tmp`;
|
|
@@ -269,7 +298,7 @@ export function writeState(path, name, prev, patch) {
|
|
|
269
298
|
// Best-effort and never fatal: a cleanup that threw would turn a reported,
|
|
270
299
|
// survived write failure into the crash the comment above rules out.
|
|
271
300
|
try { rmSync(tmp, { force: true }); } catch {}
|
|
272
|
-
console.error(`${name}: WARNING could not write ${path} (${e.message}) — back-off progress will not persist`);
|
|
301
|
+
console.error(`${name}: WARNING could not write ${path} (${e.message}) — back-off progress and the liveness ages the tick stamps will not persist`);
|
|
273
302
|
return false;
|
|
274
303
|
}
|
|
275
304
|
}
|
|
@@ -310,55 +339,102 @@ export const BEAT_GRACE = 2;
|
|
|
310
339
|
// here and imported there: every script that reads or writes the heartbeat
|
|
311
340
|
// file already imports this module, and a magic "1200" typed twice is exactly
|
|
312
341
|
// the drift the module header's "never a copy of it" rule exists to close.
|
|
313
|
-
// It doubles as the freshness window for `ticked`
|
|
314
|
-
//
|
|
315
|
-
// worst-case gap this design ever tolerates when only the
|
|
316
|
-
// beating, times the same grace multiplier as `beat`.
|
|
342
|
+
// It doubles as the freshness window for `ticked`, `acted` and the controller
|
|
343
|
+
// record below: none of them carries a promised interval, so each is judged
|
|
344
|
+
// against the worst-case gap this design ever tolerates when only the
|
|
345
|
+
// heartbeat is beating, times the same grace multiplier as `beat`.
|
|
317
346
|
export const DEFAULT_CEILING_S = 1200;
|
|
318
347
|
|
|
319
|
-
|
|
348
|
+
const WINDOW_MS = DEFAULT_CEILING_S * 1000 * BEAT_GRACE;
|
|
349
|
+
|
|
350
|
+
// A mark older than the controller record's `at` was left by the run before
|
|
351
|
+
// the one that wrote the record — rotate replaced the record after that run's
|
|
352
|
+
// last beat, tick and ledger write — so it says nothing about this run. With
|
|
353
|
+
// no record, or a record with no `at`, nothing can be placed and every mark
|
|
354
|
+
// counts as this run's.
|
|
355
|
+
function thisRun(mark, controller) {
|
|
356
|
+
if (mark == null) return null;
|
|
357
|
+
return controller == null || controller.at === null || mark.at >= controller.at ? mark : null;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// What the marks say, as a value. `now` is a parameter so every caller's
|
|
320
361
|
// verdict is deterministic and testable — the same reason compute-board.mjs
|
|
321
|
-
// takes one.
|
|
362
|
+
// takes one. Only this run's marks decide it.
|
|
322
363
|
//
|
|
323
|
-
// none no mark
|
|
324
|
-
// stall: there is no
|
|
325
|
-
// would fire on every first tick
|
|
326
|
-
// the line.
|
|
364
|
+
// none no mark of this run and no controller record. A fresh run, or
|
|
365
|
+
// one whose heartbeat never ran. NOT a stall: there is no run to
|
|
366
|
+
// judge, and reporting one here would fire on every first tick
|
|
367
|
+
// and train the reader to ignore the line.
|
|
327
368
|
// beating seen within the interval it promised, plus grace — OR a recent
|
|
328
|
-
//
|
|
329
|
-
// drains (SKILL.md: "beat when
|
|
330
|
-
//
|
|
331
|
-
//
|
|
332
|
-
//
|
|
333
|
-
// stopped a reason
|
|
334
|
-
// recorded stop IS the end of the run and waiting for it to go
|
|
369
|
+
// tick, ledger write or controller record covers for it. `beat`
|
|
370
|
+
// is only refreshed when the queue drains (SKILL.md: "beat when
|
|
371
|
+
// there is nothing to do"), so a busy run can go its whole length
|
|
372
|
+
// without one; its ticks and ledger writes are the liveness
|
|
373
|
+
// signal for exactly that case.
|
|
374
|
+
// stopped this run recorded a reason. Reported whatever the age, because
|
|
375
|
+
// a recorded stop IS the end of the run and waiting for it to go
|
|
335
376
|
// stale first would sit on the one report that knows its cause.
|
|
336
|
-
// stale overdue with no reason recorded AND no recent tick
|
|
337
|
-
// abrupt death: a crashed harness, a closed
|
|
338
|
-
//
|
|
339
|
-
//
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
377
|
+
// stale overdue with no reason recorded AND no recent tick or ledger
|
|
378
|
+
// write either. The abrupt death: a crashed harness, a closed
|
|
379
|
+
// terminal, an OOM kill — or a live controller that stopped
|
|
380
|
+
// acting. Readers say exactly that rather than naming a cause
|
|
381
|
+
// they cannot know. A run with a controller record and no beat of
|
|
382
|
+
// its own is judged off its latest tick, ledger write or the
|
|
383
|
+
// record itself, so one that dies before it ever beats still goes
|
|
384
|
+
// stale; `beatless` marks that verdict, and `mark` names which.
|
|
385
|
+
// ended the PREVIOUS run's recorded stop, while this run has not
|
|
386
|
+
// ticked yet. It is that run's end, never this run's verdict:
|
|
387
|
+
// reported until this run's first tick, and never after.
|
|
388
|
+
export function assessBeat({ beat = null, ticked = null, acted = null, controller = null, now }) {
|
|
389
|
+
const ours = thisRun(beat, controller);
|
|
390
|
+
if (ours) {
|
|
391
|
+
const seen = since(ours.at, ours.interval * 1000, now);
|
|
392
|
+
if (ours.stopped) return { ...seen, kind: "stopped", reason: ours.stopped };
|
|
393
|
+
return { ...seen, kind: now > stallsAt({ beat, ticked, acted, controller }) ? "stale" : "beating", reason: null };
|
|
394
|
+
}
|
|
395
|
+
const marks = [
|
|
396
|
+
["controller", controller?.at ?? null],
|
|
397
|
+
["ticked", thisRun(ticked, controller)?.at ?? null],
|
|
398
|
+
["acted", thisRun(acted, controller)?.at ?? null],
|
|
399
|
+
].filter(([, at]) => at !== null);
|
|
400
|
+
// Without a record a mark-less beat is no run at all, whatever ticks or
|
|
401
|
+
// ledger writes some earlier run left beside it.
|
|
402
|
+
const judged = controller !== null && marks.length > 0;
|
|
403
|
+
if (judged && now > stallsAt({ beat, ticked, acted, controller })) {
|
|
404
|
+
const [mark, at] = marks.reduce((a, b) => (b[1] >= a[1] ? b : a));
|
|
405
|
+
return { ...since(at, WINDOW_MS, now), kind: "stale", reason: null, beatless: true, mark };
|
|
406
|
+
}
|
|
407
|
+
if (beat?.stopped && thisRun(ticked, controller) === null) {
|
|
408
|
+
return { ...since(beat.at, beat.interval * 1000, now), kind: "ended", reason: beat.stopped };
|
|
409
|
+
}
|
|
410
|
+
return judged ? { kind: "beating" } : { kind: "none" };
|
|
350
411
|
}
|
|
351
412
|
|
|
352
|
-
//
|
|
353
|
-
//
|
|
354
|
-
//
|
|
355
|
-
//
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
413
|
+
// A mark's age against the window it is judged by. Clamped at zero rather
|
|
414
|
+
// than left signed: a mark from the future is a clock that moved, not a beat
|
|
415
|
+
// that is minus-five-minutes overdue, and a negative age formatted into the
|
|
416
|
+
// line reads as a defect in this code.
|
|
417
|
+
function since(at, intervalMs, now) {
|
|
418
|
+
const ageMs = Math.max(0, now - at);
|
|
419
|
+
return { at, ageMs, intervalMs, overdueMs: Math.max(0, ageMs - intervalMs) };
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// The instant this run turns `stale`: past `beat`'s own interval plus grace
|
|
423
|
+
// AND past the ceiling window plus grace of every tick, ledger write and
|
|
424
|
+
// controller record, whichever ends latest. Only this run's marks count — the
|
|
425
|
+
// previous run's are partitioned off by the record's `at` exactly as
|
|
426
|
+
// assessBeat() does. An absent mark contributes nothing, and null means this
|
|
427
|
+
// run has no mark at all. Exported for the readers that must say WHEN a
|
|
428
|
+
// beating run would stop counting, not only whether it has.
|
|
429
|
+
export function stallsAt({ beat = null, ticked = null, acted = null, controller = null }) {
|
|
430
|
+
const ends = [ticked, acted]
|
|
431
|
+
.map((m) => thisRun(m, controller))
|
|
432
|
+
.filter((m) => m !== null)
|
|
433
|
+
.map((m) => m.at + WINDOW_MS);
|
|
434
|
+
if (controller?.at != null) ends.push(controller.at + WINDOW_MS);
|
|
435
|
+
const ours = thisRun(beat, controller);
|
|
436
|
+
if (ours) ends.push(ours.at + ours.interval * 1000 * BEAT_GRACE);
|
|
437
|
+
return ends.length > 0 ? Math.max(...ends) : null;
|
|
362
438
|
}
|
|
363
439
|
|
|
364
440
|
// Whether this verdict is something to report. Exported because both readers
|
|
@@ -369,6 +445,13 @@ export function isStalled(verdict) {
|
|
|
369
445
|
return verdict.kind === "stopped" || verdict.kind === "stale";
|
|
370
446
|
}
|
|
371
447
|
|
|
448
|
+
// Whether this verdict prints at all: a stall, or the previous run's end. The
|
|
449
|
+
// second never reaches isStalled(), whose readers go on to count claims and
|
|
450
|
+
// judge the controller — both this run's, and neither the previous run's.
|
|
451
|
+
export function isReported(verdict) {
|
|
452
|
+
return isStalled(verdict) || verdict.kind === "ended";
|
|
453
|
+
}
|
|
454
|
+
|
|
372
455
|
// Whole minutes, floored, because the numbers here are tens of minutes to
|
|
373
456
|
// hours and a seconds-precise age in a stall line is false precision: the
|
|
374
457
|
// mark is written once per hold, so its resolution is minutes anyway. That
|
|
@@ -419,26 +502,26 @@ function mins(ms) {
|
|
|
419
502
|
//
|
|
420
503
|
// A mark older than the record's `at` was left by the run BEFORE the one
|
|
421
504
|
// that wrote the record: rotate replaced the record after that run's last
|
|
422
|
-
// beat and
|
|
423
|
-
//
|
|
424
|
-
// run alive. That stall reads the verdict rotate reached on the
|
|
425
|
-
// replaced, `prior`: dead is a controller gone whose claims this run
|
|
426
|
-
// ancestor is this same session's earlier run and so alive, none is
|
|
427
|
-
// A mark written since the record — any later tick
|
|
428
|
-
// controller's own, so the record itself is judged,
|
|
429
|
-
// a reused pid is gone and the NEXT run resumes;
|
|
430
|
-
// reader, is alive. A record with no `at` cannot
|
|
431
|
-
// judged too.
|
|
505
|
+
// beat, tick and ledger write. When every mark is that old, the record names
|
|
506
|
+
// a controller that has left no mark yet, so judging it would call every
|
|
507
|
+
// previous run alive. That stall reads the verdict rotate reached on the
|
|
508
|
+
// record it replaced, `prior`: dead is a controller gone whose claims this run
|
|
509
|
+
// resumes, ancestor is this same session's earlier run and so alive, none is
|
|
510
|
+
// unknown. A mark written since the record — any later beat, tick or ledger
|
|
511
|
+
// write — is the recorded controller's own, so the record itself is judged,
|
|
512
|
+
// through proc.mjs: dead or a reused pid is gone and the NEXT run resumes;
|
|
513
|
+
// alive, or an ancestor of the reader, is alive. A record with no `at` cannot
|
|
514
|
+
// place the mark and is judged too.
|
|
432
515
|
//
|
|
433
516
|
// `judge` is replaceable for the same reason proc.mjs's process source is: a
|
|
434
517
|
// reader under a live omp session would otherwise find that session. A
|
|
435
518
|
// process table that cannot be read leaves the stall unattributed, said on
|
|
436
519
|
// stderr, rather than calling a controller nobody could judge dead or alive.
|
|
437
520
|
export const NO_OWNER = Object.freeze({ kind: "none" });
|
|
438
|
-
export function stallOwner({ controller, beat, ticked }, name, judge = judgeRecord) {
|
|
521
|
+
export function stallOwner({ controller, beat = null, ticked = null, acted = null }, name, judge = judgeRecord) {
|
|
439
522
|
if (!controller) return NO_OWNER;
|
|
440
|
-
const before = (mark) => mark === null
|
|
441
|
-
if (controller.at !== null && before(beat) && before(ticked)) {
|
|
523
|
+
const before = (mark) => thisRun(mark, controller) === null;
|
|
524
|
+
if (controller.at !== null && before(beat) && before(ticked) && before(acted)) {
|
|
442
525
|
if (controller.prior === "dead") return { kind: "gone", resumes: "this run" };
|
|
443
526
|
if (controller.prior === "ancestor") return { kind: "alive", pid: null };
|
|
444
527
|
return NO_OWNER;
|
|
@@ -456,9 +539,13 @@ export function stallOwner({ controller, beat, ticked }, name, judge = judgeReco
|
|
|
456
539
|
return NO_OWNER;
|
|
457
540
|
}
|
|
458
541
|
|
|
459
|
-
// The report. Returns null for anything
|
|
542
|
+
// The report. Returns null for anything isReported() rejects, so a caller that
|
|
460
543
|
// skipped the predicate cannot print a stall for a healthy run by accident.
|
|
461
544
|
//
|
|
545
|
+
// The previous run's end is one line of its own and carries nothing of this
|
|
546
|
+
// run's: no claims, no supply, no owner. It names when that run stopped and
|
|
547
|
+
// the reason it recorded, and nothing more.
|
|
548
|
+
//
|
|
462
549
|
// What it names is the acceptance criterion itself: a bare "stale" tells the
|
|
463
550
|
// maintainer nothing they can act on, so the line carries when the beat was
|
|
464
551
|
// last seen, how overdue that is against the interval it PROMISED, whether a
|
|
@@ -478,22 +565,32 @@ export function stallOwner({ controller, beat, ticked }, name, judge = judgeReco
|
|
|
478
565
|
// cockpit's own cards) and either source can fail, and "0 claimed" off a
|
|
479
566
|
// failed read is the same lie as "supply 0" off one — it says the night cost
|
|
480
567
|
// nothing when it may have stranded the whole pool.
|
|
481
|
-
export function stallReport(verdict, { claimed, supply, owner = NO_OWNER }) {
|
|
482
|
-
if (!
|
|
568
|
+
export function stallReport(verdict, { claimed, supply, owner = NO_OWNER } = {}) {
|
|
569
|
+
if (!isReported(verdict)) return null;
|
|
570
|
+
const when = (at) => new Date(at).toISOString();
|
|
571
|
+
if (verdict.kind === "ended") {
|
|
572
|
+
return `previous run stopped deliberately at ${when(verdict.at)} (${mins(verdict.ageMs)} ago) — recorded reason: ${verdict.reason}`;
|
|
573
|
+
}
|
|
483
574
|
const n = (v) => (v == null ? "unknown" : String(v));
|
|
484
575
|
// Present tense for the deliberate stop and past for the abrupt one, because
|
|
485
576
|
// that is the actual difference: one run said why it was going, the other
|
|
486
577
|
// was cut off mid-beat and left nothing behind to ask.
|
|
487
578
|
const why = verdict.kind === "stopped"
|
|
488
579
|
? `stopped deliberately — recorded reason: ${verdict.reason}`
|
|
489
|
-
: "the beat stopped without a recorded reason";
|
|
580
|
+
: verdict.beatless ? "no beat, tick or ledger write since, and no recorded reason" : "the beat stopped without a recorded reason";
|
|
490
581
|
const head = owner.kind === "alive"
|
|
491
582
|
? `heartbeat not beating, controller alive (${owner.pid === null ? "this session's earlier run" : `pid ${owner.pid}`})`
|
|
492
583
|
: owner.kind === "gone" ? "heartbeat STALLED, controller gone" : "heartbeat STALLED";
|
|
493
584
|
const tail = owner.kind === "gone"
|
|
494
585
|
? `; ${owner.resumes} resumes its claims without a PR, and its open PRs return through the fold-in`
|
|
495
586
|
: "";
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
587
|
+
const seen = verdict.beatless
|
|
588
|
+
? `no beat this run, last ${MARK_NAMES[verdict.mark]} ${when(verdict.at)}`
|
|
589
|
+
+ ` (${mins(verdict.ageMs)} ago, ${mins(verdict.overdueMs)} past the ${mins(verdict.intervalMs)} window)`
|
|
590
|
+
: `last beat ${when(verdict.at)}`
|
|
591
|
+
+ ` (${mins(verdict.ageMs)} ago, ${mins(verdict.overdueMs)} past the ${mins(verdict.intervalMs)} interval it promised)`;
|
|
592
|
+
return `${head}: ${seen} — ${why}; ${n(claimed)} ticket(s) claimed and in flight, pool supply ${n(supply)}${tail}`;
|
|
499
593
|
}
|
|
594
|
+
|
|
595
|
+
// What a beatless stall names as this run's last sign of life.
|
|
596
|
+
const MARK_NAMES = { controller: "controller record", ticked: "tick", acted: "ledger write" };
|
package/scripts/fleet-tick.mjs
CHANGED
|
@@ -12,7 +12,9 @@
|
|
|
12
12
|
// actual/target and a named ACTION per role — `PULL #<n> #<n>`, `DISPATCH
|
|
13
13
|
// fix-pr PR#<pr>`, `DISPATCH review PR#<pr>`, `DISPATCH merge-bot`, `HOLD (…)`,
|
|
14
14
|
// `REFRESHED shortlist: …` — with run-team's guards applied in code rather than
|
|
15
|
-
// recalled from its table. You cannot forget what a script prints at you.
|
|
15
|
+
// recalled from its table. You cannot forget what a script prints at you. A
|
|
16
|
+
// drained run with nothing left in flight ends on one more: `ACTION: run over`,
|
|
17
|
+
// naming the stop the controller records last.
|
|
16
18
|
//
|
|
17
19
|
// WHERE THE NUMBERS COME FROM — nobody states them:
|
|
18
20
|
//
|
|
@@ -56,6 +58,14 @@
|
|
|
56
58
|
// file reads DEFAULT-ONLY, the way the router (not in the tree
|
|
57
59
|
// yet) is to read it. pr-cost.mjs writes the main
|
|
58
60
|
// workspace's file by default, the one read here.
|
|
61
|
+
// seen `seen` in the heartbeat's state file, this script's own key:
|
|
62
|
+
// when it first saw each live token. The ledger records that a
|
|
63
|
+
// member was dispatched, never when, so the age of a live token
|
|
64
|
+
// exists nowhere else. A token live past its family's threshold
|
|
65
|
+
// prints one `CHECK-LIVENESS` line — advice to the controller to
|
|
66
|
+
// take the registry census, never an action: it resets no
|
|
67
|
+
// `quiet`, changes no row and frees no slot, and the tick never
|
|
68
|
+
// settles or kills anything.
|
|
59
69
|
//
|
|
60
70
|
// The pure half below is `reconcile()` over the counts `deriveRun()` reads off
|
|
61
71
|
// the ledger; main() does the I/O. Split so the guard table and the reading are
|
|
@@ -912,6 +922,64 @@ export function refreshWhy({ draining, status, entries, unclaimed: k, implCap })
|
|
|
912
922
|
return null;
|
|
913
923
|
}
|
|
914
924
|
|
|
925
|
+
// How long, in minutes, a live token of each family may run before the tick
|
|
926
|
+
// asks the controller to look at it. Each default is that family's p99
|
|
927
|
+
// run time: a token past it is rare among members that finished, so a line
|
|
928
|
+
// is worth a look and not noise. Measured 2026-10-11. The four member
|
|
929
|
+
// families are the maintainer's reading of the omp rows of
|
|
930
|
+
// member-outcomes.tsv, one family each; `review` is
|
|
931
|
+
// the nearest-rank p99 of (last - first entry timestamp) over every
|
|
932
|
+
// review-pr-*.jsonl under ~/.omp/agent/sessions — 77.09 min over 4751
|
|
933
|
+
// transcripts, rounded up. Re-derive by re-running that query; no script
|
|
934
|
+
// carries it.
|
|
935
|
+
export const LIVENESS_DEFAULTS = { impl: 78, "fix-pr": 56, "finisher-pr": 57, "merge-bot": 101, review: 78 };
|
|
936
|
+
// One flag per family, so a controller can tune what the defaults get wrong.
|
|
937
|
+
const LIVENESS_FLAGS = {
|
|
938
|
+
impl: "liveness-impl", "fix-pr": "liveness-fix-pr", "finisher-pr": "liveness-finisher",
|
|
939
|
+
"merge-bot": "liveness-merge-bot", review: "liveness-review",
|
|
940
|
+
};
|
|
941
|
+
|
|
942
|
+
// A live name's family: a member token's own, or `review` for the in-flight
|
|
943
|
+
// `review:PR#<n>` a `review=` launch is named by (it carries no member token).
|
|
944
|
+
const livenessFamily = (name) => (name.startsWith("review:") ? "review" : parseMember(name)?.family ?? null);
|
|
945
|
+
|
|
946
|
+
// The `seen` map for this tick: one stamp per live name, carried when it can
|
|
947
|
+
// be trusted and `now` when not. A stamp is the age of the token, so the rule
|
|
948
|
+
// is which ones still describe THIS run's token:
|
|
949
|
+
// - an `ancestor` rotate carried this run's tokens across — they are old, and
|
|
950
|
+
// their age must survive it;
|
|
951
|
+
// - `dead` and `none` carry nothing, so a stamp older than the record's `at`
|
|
952
|
+
// belongs to a token of the previous run, even one that shares its name
|
|
953
|
+
// (`merge-bot-1` is reused), and restarts at zero;
|
|
954
|
+
// - no record, or one with no `at`, leaves nothing to compare, so the stamp
|
|
955
|
+
// stands.
|
|
956
|
+
// A name that is no longer live is dropped: a settled member's stamp must not
|
|
957
|
+
// age a later token of the same name.
|
|
958
|
+
export function stampSeen({ live, prev, controller, now }) {
|
|
959
|
+
const thisRun = (at) => controller === null || controller.prior === "ancestor" || controller.at === null || at >= controller.at;
|
|
960
|
+
const next = {};
|
|
961
|
+
for (const name of live) next[name] = prev[name] !== undefined && thisRun(prev[name]) ? prev[name] : now;
|
|
962
|
+
return next;
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
// The CHECK-LIVENESS lines: one per live name past its family's threshold, as
|
|
966
|
+
// `{member, line}`. The age shows only as a multiple of the threshold, so the
|
|
967
|
+
// line — and the digest `--fold-unchanged` compares — changes when a member
|
|
968
|
+
// crosses into the next bucket and never on the minute between.
|
|
969
|
+
export function livenessLines({ live, seen, thresholds, now }) {
|
|
970
|
+
const out = [];
|
|
971
|
+
for (const member of live) {
|
|
972
|
+
const family = livenessFamily(member);
|
|
973
|
+
if (family === null || seen[member] === undefined) continue;
|
|
974
|
+
const minutes = thresholds[family];
|
|
975
|
+
const k = Math.floor(Math.max(0, now - seen[member]) / (minutes * 60_000));
|
|
976
|
+
if (k < 1) continue;
|
|
977
|
+
const label = family === "finisher-pr" ? "finisher" : family;
|
|
978
|
+
out.push({ member, line: `CHECK-LIVENESS ${member} (${label} > ${k === 1 ? "" : `${k}×`}p99 ${k * minutes}m)` });
|
|
979
|
+
}
|
|
980
|
+
return out;
|
|
981
|
+
}
|
|
982
|
+
|
|
915
983
|
// --------------------------------------------------------------------------
|
|
916
984
|
// I/O. Everything below runs only as a CLI — importing this file must never
|
|
917
985
|
// parse argv or touch the network, or the pure half stops being unit-testable.
|
|
@@ -926,7 +994,7 @@ import { parseArgs } from "node:util";
|
|
|
926
994
|
import { makeDie, isDigits } from "./arg.mjs";
|
|
927
995
|
import { gitEnv } from "./git-env.mjs";
|
|
928
996
|
import { fleetFile, FleetDirUnresolvable } from "./fleet-dir.mjs";
|
|
929
|
-
import { statePath, readState, writeState, assessBeat, isStalled, stallReport, stallOwner } from "./fleet-state.mjs";
|
|
997
|
+
import { statePath, readState, writeState, assessBeat, isStalled, isReported, stallReport, stallOwner } from "./fleet-state.mjs";
|
|
930
998
|
import { checkMainCheckout, describe } from "./main-checkout.mjs";
|
|
931
999
|
|
|
932
1000
|
const NAME = "fleet-tick";
|
|
@@ -943,6 +1011,9 @@ const MERGEABLE_STATES = ["MERGEABLE", "CONFLICTING", "UNKNOWN"];
|
|
|
943
1011
|
// cap, because refusing would suppress the one announcement a dead run leaves
|
|
944
1012
|
// behind. So it discloses instead — the line says the count is a floor.
|
|
945
1013
|
const CLAIMED_LIMIT = 200;
|
|
1014
|
+
// The tick's last line once a draining run has nothing left in flight. The
|
|
1015
|
+
// tick cannot write the stop — `beat` is fleet-heartbeat's — so it names it.
|
|
1016
|
+
export const RUN_OVER = `ACTION: run over → fleet-heartbeat.mjs --stop "<reason>"`;
|
|
946
1017
|
const MAX_BUFFER = 64 * 1024 * 1024;
|
|
947
1018
|
// Every gh spawn below is killed at this bound, so a gh that never answers is
|
|
948
1019
|
// a failed read on that call's own failure path — the open-PR read refuses the
|
|
@@ -987,6 +1058,8 @@ const OPTIONS = {
|
|
|
987
1058
|
// under parseArgs, and no path is legitimately empty, so empty resolves to
|
|
988
1059
|
// the default.
|
|
989
1060
|
state: { type: "string", default: "" },
|
|
1061
|
+
// Minutes a live token of each family may run before CHECK-LIVENESS names it.
|
|
1062
|
+
...Object.fromEntries(Object.entries(LIVENESS_FLAGS).map(([family, flag]) => [flag, { type: "string", default: String(LIVENESS_DEFAULTS[family]) }])),
|
|
990
1063
|
};
|
|
991
1064
|
|
|
992
1065
|
function options() {
|
|
@@ -1017,6 +1090,7 @@ function options() {
|
|
|
1017
1090
|
implCap: positive("implementer-cap"), reviewerCap,
|
|
1018
1091
|
maxReviews: values["max-reviews"] === undefined ? reviewerCap : positive("max-reviews"),
|
|
1019
1092
|
fold: values["fold-unchanged"], state: values.state || statePath(NAME),
|
|
1093
|
+
liveness: Object.fromEntries(Object.entries(LIVENESS_FLAGS).map(([family, flag]) => [family, positive(flag)])),
|
|
1020
1094
|
};
|
|
1021
1095
|
}
|
|
1022
1096
|
|
|
@@ -1242,7 +1316,7 @@ function claimed() {
|
|
|
1242
1316
|
}
|
|
1243
1317
|
|
|
1244
1318
|
function main() {
|
|
1245
|
-
const { fold, state: path, ...caps } = options();
|
|
1319
|
+
const { fold, state: path, liveness, ...caps } = options();
|
|
1246
1320
|
const listPath = shortlistPath();
|
|
1247
1321
|
const current = readShortlist(listPath);
|
|
1248
1322
|
// Beside the shortlist, off the same git probe rather than a second one.
|
|
@@ -1255,9 +1329,24 @@ function main() {
|
|
|
1255
1329
|
// Whose stall it is comes off the controller record, judged only once
|
|
1256
1330
|
// there is a stall to name, so a healthy tick reads no process table.
|
|
1257
1331
|
const priorState = readState(path, NAME);
|
|
1258
|
-
const
|
|
1259
|
-
|
|
1260
|
-
|
|
1332
|
+
const tickedAt = Date.now();
|
|
1333
|
+
const { beat, ticked, acted, controller } = priorState;
|
|
1334
|
+
const verdict = assessBeat({ beat, ticked, acted, controller, now: tickedAt });
|
|
1335
|
+
// `ticked`, fleet-tick's own liveness key, written here and on its own,
|
|
1336
|
+
// ahead of the gh and ledger reads that can refuse this tick: a refused
|
|
1337
|
+
// tick was still invoked, and a run whose every tick refuses on a gh outage
|
|
1338
|
+
// is a run working, not a dead one. Only the usage refusals above leave no
|
|
1339
|
+
// mark — they never reached the state file — and a file that exists but
|
|
1340
|
+
// could not be read is left as it is, as ledger.mjs's `acted` stamp leaves
|
|
1341
|
+
// it: rotate refuses on such a file because a record it may hold cannot be
|
|
1342
|
+
// known, and a write here would replace that record with a fresh one.
|
|
1343
|
+
// Judged before it is written, so the previous run's end prints on this
|
|
1344
|
+
// run's first tick and on no later one.
|
|
1345
|
+
if (!priorState.degraded) writeState(path, NAME, priorState, { ticked: { at: tickedAt } });
|
|
1346
|
+
if (isReported(verdict)) {
|
|
1347
|
+
console.log(stallReport(verdict, isStalled(verdict)
|
|
1348
|
+
? { claimed: claimed(), supply: current.scanned, owner: stallOwner(priorState, NAME) }
|
|
1349
|
+
: {}));
|
|
1261
1350
|
}
|
|
1262
1351
|
|
|
1263
1352
|
// Every read happens before anything prints: a partial tick is worse than no
|
|
@@ -1310,20 +1399,40 @@ function main() {
|
|
|
1310
1399
|
...caps, ...run, heads, supply, shortlistStatus: current.status, refresh: fresh, mainCheckout, router,
|
|
1311
1400
|
});
|
|
1312
1401
|
const said = describe(mainCheckout, run.live);
|
|
1313
|
-
|
|
1402
|
+
// Re-read rather than reusing priorState: this must be the freshest
|
|
1403
|
+
// snapshot, or this tick reverts whatever the heartbeat wrote while the
|
|
1404
|
+
// reads above were in flight. Read before the lines are built because the
|
|
1405
|
+
// liveness lines need the stamps it holds, and nothing between here and the
|
|
1406
|
+
// write below is I/O.
|
|
1407
|
+
const prev = readState(path, NAME);
|
|
1408
|
+
const now = Date.now();
|
|
1409
|
+
// Advisory only: a stale token prints a line asking the controller for the
|
|
1410
|
+
// registry census. It is not a row — it resets no `quiet`, moves no count
|
|
1411
|
+
// and frees no slot — and nothing here settles, kills or writes the ledger.
|
|
1412
|
+
const seen = stampSeen({ live: run.live, prev: prev.seen, controller: prev.controller, now });
|
|
1413
|
+
const stale = livenessLines({ live: run.live, seen, thresholds: liveness, now });
|
|
1414
|
+
const lines = [...(said === null ? [] : [said]), ...stale.map((s) => s.line), ...formatLines(rows)];
|
|
1415
|
+
// The run is over once it is draining and nothing is left in flight: no
|
|
1416
|
+
// open PR and no implementer still holding a claim. Printed last and read
|
|
1417
|
+
// as actionable, so it never folds away: the stop it names is the run's
|
|
1418
|
+
// last act, and only the controller can write it.
|
|
1419
|
+
const over = run.draining !== null && prs.length === 0 && run.implLive === 0;
|
|
1420
|
+
if (over) lines.push(RUN_OVER);
|
|
1314
1421
|
|
|
1315
1422
|
// The back-off streak and the fold digest, written on EVERY tick: an edge
|
|
1316
1423
|
// tick that dispatched is the clearest possible "there is work here". Digest,
|
|
1317
|
-
// not the lines themselves, so the file stays readable.
|
|
1318
|
-
// reusing priorState: this must be the freshest snapshot, or this tick
|
|
1319
|
-
// reverts whatever the heartbeat wrote while the reads above were in flight.
|
|
1320
|
-
const prev = readState(path, NAME);
|
|
1424
|
+
// not the lines themselves, so the file stays readable.
|
|
1321
1425
|
const digest = createHash("sha256").update(lines.join("\n")).digest("hex");
|
|
1322
|
-
const acts = actionable(rows);
|
|
1323
|
-
// `
|
|
1324
|
-
//
|
|
1325
|
-
//
|
|
1326
|
-
|
|
1426
|
+
const acts = actionable(rows) || over;
|
|
1427
|
+
// `seen` is pruned to the live set on the same write; an empty one removes
|
|
1428
|
+
// the key. `ticked` was written at the top of the tick, unless the file
|
|
1429
|
+
// could not be read then: this write replaces such a file, so it carries
|
|
1430
|
+
// the stamp the top of the tick left out.
|
|
1431
|
+
writeState(path, NAME, prev, {
|
|
1432
|
+
quiet: acts ? 0 : prev.quiet + 1, digest,
|
|
1433
|
+
seen: Object.keys(seen).length > 0 ? seen : undefined,
|
|
1434
|
+
...(priorState.degraded ? { ticked: { at: tickedAt } } : {}),
|
|
1435
|
+
});
|
|
1327
1436
|
|
|
1328
1437
|
// Fold only when BOTH hold: nothing to act on, and nothing new to say. A
|
|
1329
1438
|
// main-checkout hold still holding dispatch stays named on the folded line
|
|
@@ -1333,7 +1442,11 @@ function main() {
|
|
|
1333
1442
|
if (fold && !acts && digest === prev.digest) {
|
|
1334
1443
|
const why = mainCheckoutHoldReason(mainCheckout);
|
|
1335
1444
|
const held = why === null ? "" : `; HOLD (${why}) persists`;
|
|
1336
|
-
|
|
1445
|
+
// A stale member persists the same way: advisory, so it never resets
|
|
1446
|
+
// `quiet`, and named on the folded line for as long as it stays live
|
|
1447
|
+
// past its threshold. The digest already moves when its bucket does.
|
|
1448
|
+
const aged = stale.map((s) => `; CHECK-LIVENESS ${s.member} persists`).join("");
|
|
1449
|
+
console.log(`fleet-tick: unchanged, nothing to act on (quiet=${prev.quiet + 1})${held}${aged} — full rows on the next change`);
|
|
1337
1450
|
return;
|
|
1338
1451
|
}
|
|
1339
1452
|
for (const line of lines) console.log(line);
|
package/scripts/ledger.mjs
CHANGED
|
@@ -32,6 +32,12 @@
|
|
|
32
32
|
// `filed` kept 2-5 of 8 rows — and the lock is what lets a second kind of
|
|
33
33
|
// writer exist at all. The reasoning for each part of the lock lives at
|
|
34
34
|
// acquireLock() below.
|
|
35
|
+
//
|
|
36
|
+
// Every write subcommand also stamps `acted` in the heartbeat file beside the
|
|
37
|
+
// ledger whenever its call ends without a refusal — rotate in the same write
|
|
38
|
+
// as its controller record. A run's ledger writes are its densest sign of
|
|
39
|
+
// life, so the stall readers judge a busy run by them (fleet-state.mjs has
|
|
40
|
+
// the rule).
|
|
35
41
|
|
|
36
42
|
import { readFileSync, writeFileSync, mkdirSync, existsSync, renameSync, openSync, writeSync, closeSync, unlinkSync, linkSync } from "node:fs";
|
|
37
43
|
import { dirname, resolve, join, basename } from "node:path";
|
|
@@ -929,6 +935,18 @@ if (cmd === "read") {
|
|
|
929
935
|
die(`unknown subcommand '${cmd}' — expected row, filed, ruled, check, read, dispatch, settle, review, drain or rotate`);
|
|
930
936
|
}
|
|
931
937
|
|
|
938
|
+
// Reached only by an arm that did not die, so every write subcommand that did
|
|
939
|
+
// not refuse is stamped — one that found nothing to change (a repeat `drain`,
|
|
940
|
+
// a duplicate `filed`) is still the controller acting. Still under the write
|
|
941
|
+
// lock, so two ledger calls never race on the key. A heartbeat file that
|
|
942
|
+
// exists and cannot be read is left as it is for the readers that refuse on
|
|
943
|
+
// it; readState() has already said why.
|
|
944
|
+
if (WRITE_COMMANDS.has(cmd) && cmd !== "rotate") {
|
|
945
|
+
const beatFile = stateFileIn(dirname(file));
|
|
946
|
+
const state = readState(beatFile, NAME);
|
|
947
|
+
if (!state.degraded) writeState(beatFile, NAME, state, { acted: { at: Date.now() } });
|
|
948
|
+
}
|
|
949
|
+
|
|
932
950
|
// `check` alone of the subcommands leaves its arm early: the already-filed
|
|
933
951
|
// answer is complete before the near-miss ranking and the tracker query below
|
|
934
952
|
// it, whose results that answer would only discard. A function is what makes
|
|
@@ -1570,9 +1588,10 @@ function runCheck() {
|
|
|
1570
1588
|
// and keep dispatching — and rotating under it would split its run across
|
|
1571
1589
|
// two ledgers. A dead one, a reused pid, or this session's own earlier run
|
|
1572
1590
|
// allows the rotation. With no record the guard falls back to the mark, read
|
|
1573
|
-
// through fleet-state.mjs's own reader and verdict: still beating
|
|
1574
|
-
//
|
|
1575
|
-
//
|
|
1591
|
+
// through fleet-state.mjs's own reader and verdict: still beating — a beat
|
|
1592
|
+
// inside its window, or past it with a tick or ledger write inside
|
|
1593
|
+
// theirs — refuses; a stalled mark, a recorded stop, or no beat at all allows
|
|
1594
|
+
// the rotation, and that fallback proves "not beating", never "dead".
|
|
1576
1595
|
//
|
|
1577
1596
|
// A heartbeat file that exists but cannot be read or parsed gets neither
|
|
1578
1597
|
// verdict: a record it may hold cannot be known, so rotate refuses rather
|
|
@@ -1582,7 +1601,8 @@ function runCheck() {
|
|
|
1582
1601
|
// record with this run's own, carrying the verdict as `prior` — the next
|
|
1583
1602
|
// step reads `prior`, because the new record always names this run's own
|
|
1584
1603
|
// live ancestor — and the time of this write as `at`, so a reader can tell a
|
|
1585
|
-
// mark the previous run left from one written since.
|
|
1604
|
+
// mark the previous run left from one written since. The same write stamps
|
|
1605
|
+
// `acted` at that time, as every ledger write does. No omp ancestor removes
|
|
1586
1606
|
// the key: an older run's record left in place would be judged again, as
|
|
1587
1607
|
// dead, by a run that cannot know whether the run after it is still live.
|
|
1588
1608
|
// This run's own record is computed before anything moves too, so a process
|
|
@@ -1591,7 +1611,7 @@ function runCheck() {
|
|
|
1591
1611
|
function runRotate() {
|
|
1592
1612
|
const beatFile = stateFileIn(dirname(file));
|
|
1593
1613
|
const state = readState(beatFile, NAME);
|
|
1594
|
-
const { beat, ticked, controller } = state;
|
|
1614
|
+
const { beat, ticked, acted, controller } = state;
|
|
1595
1615
|
// A file that exists and could not be used holds no record we can know of:
|
|
1596
1616
|
// reading it as "no record" would let a live controller's ledger be moved
|
|
1597
1617
|
// by exactly the fault that hid its record.
|
|
@@ -1612,10 +1632,14 @@ function runRotate() {
|
|
|
1612
1632
|
die(`refusing to rotate ${file}: the controller that owns it is alive — controller pid ${controller.pid} (started ${controller.lstart}) recorded in ${beatFile} is alive and not an ancestor of this run, whatever its heartbeat mark says; one controller per checkout — end that omp session, or start the run from inside it`);
|
|
1613
1633
|
}
|
|
1614
1634
|
// The fallback guards a ledger, as it always has: with none to move there
|
|
1615
|
-
// is nothing for a beating mark to protect.
|
|
1635
|
+
// is nothing for a beating mark to protect. A ledger write is the
|
|
1636
|
+
// controller acting, and counts the way a beat does.
|
|
1616
1637
|
const exists = existsSync(file);
|
|
1617
|
-
if (prior === "none" && exists && assessBeat({ beat, ticked, now: Date.now() }).kind === "beating") {
|
|
1618
|
-
|
|
1638
|
+
if (prior === "none" && exists && assessBeat({ beat, ticked, acted, now: Date.now() }).kind === "beating") {
|
|
1639
|
+
const [what, mark] = [["beat", beat], ["tick", ticked], ["ledger write", acted]]
|
|
1640
|
+
.filter(([, m]) => m !== null)
|
|
1641
|
+
.reduce((a, b) => (b[1].at > a[1].at ? b : a));
|
|
1642
|
+
die(`refusing to rotate ${file}: the controller that owns it is still beating or acting — last ${what} ${new Date(mark.at).toISOString()} in ${beatFile}; it counts as stalled after ${new Date(stallsAt({ beat, ticked, acted })).toISOString()}`);
|
|
1619
1643
|
}
|
|
1620
1644
|
// Read before anything moves: a ledger whose live tokens cannot be read is
|
|
1621
1645
|
// refused with nothing moved, never archived with those tokens dropped.
|
|
@@ -1670,8 +1694,9 @@ function runRotate() {
|
|
|
1670
1694
|
// Re-read rather than reuse the read above: the heartbeat file has other
|
|
1671
1695
|
// writers, and a patch built from an older view would undo what they wrote
|
|
1672
1696
|
// since.
|
|
1673
|
-
const
|
|
1674
|
-
|
|
1697
|
+
const now = Date.now();
|
|
1698
|
+
const record = ours === null ? undefined : { ...ours, prior, at: now };
|
|
1699
|
+
if (!writeState(beatFile, NAME, readState(beatFile, NAME), { controller: record, acted: { at: now } })) {
|
|
1675
1700
|
die(`${archive === null ? "found no ledger" : `rotated ${file} -> ${archive}`} but could not record this run's controller in ${beatFile} — the next run judges whatever record that file still holds, and none if it cannot be read`);
|
|
1676
1701
|
}
|
|
1677
1702
|
console.log(JSON.stringify({
|
package/skills/run-team/SKILL.md
CHANGED
|
@@ -441,9 +441,10 @@ phase, or in any later one, asks the maintainer which tickets to take.
|
|
|
441
441
|
controller can stop beating and keep dispatching. A dead one, a pid
|
|
442
442
|
reused by another process, or this session's own earlier run lets it
|
|
443
443
|
rotate. With no record the heartbeat mark decides, as the only guard
|
|
444
|
-
there is: still beating refuses, naming the
|
|
445
|
-
count as stalled; a stalled mark, a
|
|
446
|
-
lets it rotate — and that fallback
|
|
444
|
+
there is: still beating or acting refuses, naming the latest beat, tick or
|
|
445
|
+
ledger write and when it would count as stalled; a stalled mark, a
|
|
446
|
+
recorded `--stop`, or no beat at all lets it rotate — and that fallback
|
|
447
|
+
proves "not beating", never "dead", and
|
|
447
448
|
guards a ledger only: with none to move the mark is not consulted. With no
|
|
448
449
|
ledger the record is still judged, so a live recorded controller refuses
|
|
449
450
|
even then. It also refuses, nothing moved, when the stamped archive name
|
|
@@ -454,10 +455,12 @@ phase, or in any later one, asks the maintainer which tickets to take.
|
|
|
454
455
|
above it, its start time, as `prior` the verdict it reached on the
|
|
455
456
|
record it found (`dead`, `ancestor` or `none`), and as `at` when it wrote
|
|
456
457
|
it — or to remove the key when
|
|
457
|
-
no `omp` process is above it.
|
|
458
|
+
no `omp` process is above it. The same write stamps `acted`, the time of
|
|
459
|
+
this run's latest ledger write, which every write subcommand stamps from
|
|
460
|
+
then on. That write can fail, and then exit 2 follows
|
|
458
461
|
a move: `rotated <file> -> <archive>` on stderr, then the refusal that the
|
|
459
462
|
record could not be written. Run the rotation before this run's own first
|
|
460
|
-
tick or
|
|
463
|
+
tick, beat or ledger write, each of which writes a mark the fallback reads. Older
|
|
461
464
|
archives, hand-named ones included, are left alone.
|
|
462
465
|
|
|
463
466
|
**Derive the Recipe — the Recipe derivation step — before the fold-in below
|
|
@@ -1671,7 +1674,7 @@ ticket's line — never a hand edit.
|
|
|
1671
1674
|
| Label seen (persistent Monitor) | nothing to record |
|
|
1672
1675
|
| CI run terminal | `ci=<run-id>:<attempt>:<conclusion>` on the row; then the finisher gate (below) |
|
|
1673
1676
|
| Merge-bot pass report | `held-behind:#<lower>` rows; `ledger.mjs settle merge-bot-<n>=done`; `reap.sh --apply`. (The bot may also have written `conflict-hold:#<pr>` onto a held PR's own row earlier in this same pass, before reporting — a token the bot wrote in this pass, never one `reap.sh` writes.) |
|
|
1674
|
-
| Drain | `ledger.mjs drain "<reason>"`; release the claims (below); `settle impl-<N>=released` |
|
|
1677
|
+
| Drain | `ledger.mjs drain "<reason>"`; release the claims (below); `settle impl-<N>=released`; then the rest of **End of run** (below), `--stop` last |
|
|
1675
1678
|
| Heartbeat | nothing to record |
|
|
1676
1679
|
|
|
1677
1680
|
Then run the tick, act on every line it prints, and arm the beat. What a wake
|
|
@@ -1739,6 +1742,19 @@ owes beyond its row:
|
|
|
1739
1742
|
claims. The marker is on disk, so the tick holds the implementer row at
|
|
1740
1743
|
`HOLD (draining)` from then on — for a replacement controller too — and keeps
|
|
1741
1744
|
the review, fix-applier and merge rows firing until the open PRs merge.
|
|
1745
|
+
The drain starts the end of the run; it does not end it. **End of run**, in
|
|
1746
|
+
this order:
|
|
1747
|
+
1. `ledger.mjs drain "<reason>"`.
|
|
1748
|
+
2. Release every claim that never became a PR, settling each
|
|
1749
|
+
`impl-<N>=released`.
|
|
1750
|
+
3. Act on the review, fix-applier and merge rows until no PR is open. Once
|
|
1751
|
+
the run is draining with no open PR and no live implementer, the tick's
|
|
1752
|
+
last line is `ACTION: run over → fleet-heartbeat.mjs --stop "<reason>"`.
|
|
1753
|
+
4. Scrape the run's member facts, append every `tier-outcomes.mjs` row, and
|
|
1754
|
+
run `metrics-refresh.mjs` (above).
|
|
1755
|
+
5. Commit the metrics files and open their PR (above).
|
|
1756
|
+
6. `~/.fleet/bin/fleet-run fleet-heartbeat.mjs --stop "<reason>"` — last,
|
|
1757
|
+
because every reader takes a recorded stop as the end of the run.
|
|
1742
1758
|
- **Monitor: CI run completes** → bind it (`ci-state.mjs --pr <N>`) and **record
|
|
1743
1759
|
it as `ci=<run-id>:<attempt>:<conclusion>`, not just "CI green"** — in the
|
|
1744
1760
|
ticket's ledger row, which outlives your context. `row` **replaces the whole
|
|
@@ -1906,6 +1922,16 @@ depth** guard table applied in code. Act on each line as it reads:
|
|
|
1906
1922
|
baseline was recorded. Yours to fix: run Phase 0's `--record`. A baseline
|
|
1907
1923
|
that vanished mid-run is not a fresh run — report it to the maintainer
|
|
1908
1924
|
before recording over whatever the tree now holds.
|
|
1925
|
+
- `CHECK-LIVENESS <member> (<family> > p99 <t>m)`, printed above the rows — a
|
|
1926
|
+
live ledger token has stayed live past its family's p99 run time, counted
|
|
1927
|
+
from the first tick that saw it (`> 2×p99 …` from the second multiple of the
|
|
1928
|
+
threshold on; `review:PR#<n>` names an in-flight `review=` launch). Advice,
|
|
1929
|
+
not an action: it is not actionable, resets no `quiet`, changes no row and
|
|
1930
|
+
frees no slot, and the tick settles and kills nothing. Take the census the
|
|
1931
|
+
**Member unaccounted** row of **Failure handling** gives before you touch the
|
|
1932
|
+
token. It repeats every tick while the token stays live past the threshold.
|
|
1933
|
+
`--liveness-impl`, `--liveness-fix-pr`, `--liveness-finisher`,
|
|
1934
|
+
`--liveness-merge-bot` and `--liveness-review` set the thresholds, in minutes.
|
|
1909
1935
|
|
|
1910
1936
|
**Tier guards under Pull.** The per-dispatch `tier-check.mjs --batch` is
|
|
1911
1937
|
phase 2's own step: a mismatch it finds is the
|
|
@@ -1972,6 +1998,10 @@ ticket's own remedy. A dirty or unknown main checkout still folds — clearing i
|
|
|
1972
1998
|
is the maintainer's, so it is nothing for you to act on — but the folded line
|
|
1973
1999
|
names it (`…; HOLD (main checkout dirty) persists — …`) for as long as it holds
|
|
1974
2000
|
dispatch.
|
|
2001
|
+
A live member past its family's threshold folds the same way — it is advice, so
|
|
2002
|
+
nothing for the heartbeat to act on by itself — and the folded line names each
|
|
2003
|
+
such member (`…; CHECK-LIVENESS fix-pr-5 persists — …`) for as long as it stays
|
|
2004
|
+
live past the threshold.
|
|
1975
2005
|
|
|
1976
2006
|
**When you stop on purpose, say why — one command, and it is the last thing
|
|
1977
2007
|
you owe the next run.** The beat leaves a dated mark on every hold, so a run
|
|
@@ -1983,34 +2013,41 @@ than being detected, and nothing else can write the reason afterwards:
|
|
|
1983
2013
|
~/.fleet/bin/fleet-run fleet-heartbeat.mjs --stop "budget exhausted"
|
|
1984
2014
|
```
|
|
1985
2015
|
|
|
1986
|
-
It holds nothing and returns at once. Use it when the budget goes, when
|
|
1987
|
-
can see the context ceiling coming
|
|
2016
|
+
It holds nothing and returns at once. Use it when the budget goes, or when
|
|
2017
|
+
you can see the context ceiling coming. A drained run records it as the last
|
|
2018
|
+
step of **End of run** (above), never at the drain itself.
|
|
1988
2019
|
Skipping it is not fatal — the next reader still reports the stop, it just
|
|
1989
2020
|
reports that the beat stopped *without a recorded reason*, which is all it can
|
|
1990
2021
|
honestly say about a death that named nothing.
|
|
1991
2022
|
|
|
1992
2023
|
**A stall line names whose stall it is.** `fleet-tick`
|
|
1993
|
-
prints it before its own rows when
|
|
1994
|
-
interval
|
|
1995
|
-
|
|
2024
|
+
prints it before its own rows when this run has gone quiet: its beat overdue
|
|
2025
|
+
against the interval it recorded with no tick or ledger write inside the
|
|
2026
|
+
40-minute window either — or, with no beat yet, its latest tick, ledger write
|
|
2027
|
+
or controller record past that window. A beat, tick or ledger write older than
|
|
2028
|
+
the controller record rotate wrote is the previous run's and counts for
|
|
2029
|
+
nothing. The cockpit shows the same line while it is up. It names when this
|
|
2030
|
+
run was last seen, how overdue that is, how many
|
|
1996
2031
|
tickets are still claimed and in flight, and how much supply the last read found
|
|
1997
2032
|
— because the claimed ones keep the `in-progress` label, the candidate scan
|
|
1998
2033
|
EXCLUDES that label, and those tickets are therefore invisible to your own
|
|
1999
2034
|
shortlist and to the maintainer's. Its opening words come off the controller
|
|
2000
|
-
record
|
|
2001
|
-
`prior` rotate recorded; any later line is judged off the record itself:
|
|
2035
|
+
record:
|
|
2002
2036
|
|
|
2003
2037
|
- `heartbeat STALLED, controller gone` — the run that stranded them is dead.
|
|
2004
|
-
Its claims without a PR come back through
|
|
2005
|
-
|
|
2006
|
-
through the fold-in.
|
|
2038
|
+
Its claims without a PR come back through the next run's phase 0
|
|
2039
|
+
stranded-claim step, and its open PRs through the fold-in.
|
|
2007
2040
|
- `heartbeat not beating, controller alive (pid N)`, or `(this session's
|
|
2008
2041
|
earlier run)` — a controller that stopped beating and kept going. Nothing
|
|
2009
|
-
died, so nothing is resumed;
|
|
2010
|
-
|
|
2042
|
+
died, so nothing is resumed; when that controller is you, the fix is to
|
|
2043
|
+
tick on every wake and beat when there is nothing to do.
|
|
2011
2044
|
- `heartbeat STALLED` alone — no record says whether that controller lives.
|
|
2012
2045
|
The line is detection only.
|
|
2013
2046
|
|
|
2047
|
+
Your first tick may print `previous run stopped deliberately at <time> (<age>
|
|
2048
|
+
ago) — recorded reason: <reason>` instead: the stop the previous run recorded.
|
|
2049
|
+
It is that run's end, not a verdict on yours, and no later tick prints it.
|
|
2050
|
+
|
|
2014
2051
|
Read the line and carry on; nothing in the fleet releases, relabels or
|
|
2015
2052
|
reaps the stranded tickets for you, and nothing restarts the run that stranded them.
|
|
2016
2053
|
|
|
@@ -4016,6 +4053,7 @@ Plan it as a PR the run's own merge queue lands, never as an edit-and-continue.
|
|
|
4016
4053
|
| Member silent or truncated | Send to ping or resume — same unit of work; see the state machine below |
|
|
4017
4054
|
| Member idle with work outstanding | Read the PR first, *then* ping. Idle ≠ done |
|
|
4018
4055
|
| Member **killed** (spend limit, API error, crash) | Confirm it is dead first — a frozen transcript does not establish that; see below — then new member, new name, prompt carries inherited state |
|
|
4056
|
+
| Member **unaccounted** (a live token the tick flags `CHECK-LIVENESS`, or any live token nothing has reported on) | Registry census first: `read history://` once. A row other than `running` → `read agent://<member>` and the artifacts (PR head, worktree, labels), settle the real outcome with the existing vocabulary, and resume a `parked` member with `write agent://<member>` when its work is worth continuing. A `running` row → the mtime discriminator below, two polls: moved means BLOCKED and alive; frozen at both polls **and** no artifact movement → `write proc://<member>/kill`, `ledger.mjs settle <member> killed`, fresh name. Three independent alive-signals — the first poll, the second poll, the artifacts — must all be silent before a kill; the tick settles and kills nothing |
|
|
4019
4057
|
|
|
4020
4058
|
A red PR never silently becomes `ready-to-merge`.
|
|
4021
4059
|
|
|
@@ -4035,8 +4073,8 @@ moved: **mtimes that moved between the two mean BLOCKED, a live member
|
|
|
4035
4073
|
still holding its claim and its worktree, so the killed row above does not
|
|
4036
4074
|
apply to it; mtimes unchanged at both are the dead signature.** Only the first
|
|
4037
4075
|
of those two verdicts is conclusive — a member wedged on a call that writes
|
|
4038
|
-
nothing freezes its scratch dir too — so settle the dead one on the
|
|
4039
|
-
|
|
4076
|
+
nothing freezes its scratch dir too — so settle the dead one on the Member
|
|
4077
|
+
unaccounted row above, never on the two listings alone. Measured once: a refuter
|
|
4040
4078
|
wedged forever on `until grep -q` for a marker `node --test` never writes, its
|
|
4041
4079
|
transcript's record counts unchanged across two polls while its own scratch
|
|
4042
4080
|
files carried fresh mtimes throughout — alive the entire time it read as dead.
|
|
@@ -4073,6 +4111,9 @@ One git-ignored `.fleet/ledger.md` per run, updated at **every** state change vi
|
|
|
4073
4111
|
durable thing in the run: it compacts, and a controller that loses the dispatch
|
|
4074
4112
|
map or the filed list redoes finished work. Two duplicate tickets shipped in one
|
|
4075
4113
|
run from exactly that.
|
|
4114
|
+
Compaction parks every in-flight member and nothing reports it, so on resume
|
|
4115
|
+
take the registry census first (**Member unaccounted**, in **Failure
|
|
4116
|
+
handling**) before you trust a live token.
|
|
4076
4117
|
|
|
4077
4118
|
**The file lives one run.** Phase 0 rotates it before anything reads it, so
|
|
4078
4119
|
no exclusion, filing, ruling or drain marker outlives the run that wrote it,
|