@feigi/fleet-ctl 3.38.0 → 3.39.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/board.mjs +9 -6
- package/scripts/compute-board.mjs +13 -8
- package/scripts/fleet-state.mjs +145 -68
- package/scripts/fleet-tick.mjs +38 -11
- package/scripts/ledger.mjs +35 -10
- package/skills/run-team/SKILL.md +41 -18
package/package.json
CHANGED
package/scripts/board.mjs
CHANGED
|
@@ -1134,19 +1134,22 @@ export async function gather({ ledgerFile, prevFile, stateFile = null, scriptDir
|
|
|
1134
1134
|
// reading. Defaulted rather than resolved here for gather()'s own reason —
|
|
1135
1135
|
// every caller in this file passes the instance's answer, and a second
|
|
1136
1136
|
// resolution could name a different workspace's beat.
|
|
1137
|
-
// `ticked`
|
|
1138
|
-
// "no try/catch, no tryRun shape" rule just above
|
|
1139
|
-
//
|
|
1140
|
-
//
|
|
1137
|
+
// `ticked`, `acted` and `controller` ride beside `beat`, same file same
|
|
1138
|
+
// read, and under the same "no try/catch, no tryRun shape" rule just above:
|
|
1139
|
+
// a run is judged from all of its own marks, and the record's `at` is what
|
|
1140
|
+
// tells this run's marks from the previous run's (fleet-state.mjs's
|
|
1141
|
+
// assessBeat has the rule).
|
|
1141
1142
|
const priorState = stateFile ? readState(stateFile, NAME) : null;
|
|
1142
1143
|
const beat = priorState?.beat ?? null;
|
|
1143
1144
|
const ticked = priorState?.ticked ?? null;
|
|
1145
|
+
const acted = priorState?.acted ?? null;
|
|
1146
|
+
const controller = priorState?.controller ?? null;
|
|
1144
1147
|
// Whose stall it is, off the controller record in the same file — judged
|
|
1145
1148
|
// here because it reads the process table, and only when the mark is
|
|
1146
1149
|
// stalled, so a healthy board reads none. computeBoard() words it.
|
|
1147
1150
|
const now = Date.now();
|
|
1148
|
-
const stallOwnerOf = isStalled(assessBeat({ beat, ticked, now })) ? stallOwner(priorState, NAME) : null;
|
|
1149
|
-
return { ledger, issues, prs, merged, ci, prev, repo, repoUrl, workspace, port, spend, beat, ticked, stallOwner: stallOwnerOf, poolOk,
|
|
1151
|
+
const stallOwnerOf = isStalled(assessBeat({ beat, ticked, acted, controller, now })) ? stallOwner(priorState, NAME) : null;
|
|
1152
|
+
return { ledger, issues, prs, merged, ci, prev, repo, repoUrl, workspace, port, spend, beat, ticked, acted, controller, stallOwner: stallOwnerOf, poolOk,
|
|
1150
1153
|
poolCapped, prsCapped,
|
|
1151
1154
|
now, interval };
|
|
1152
1155
|
}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// fleet-state.mjs owns the heartbeat's `beat` key, and the rule for reading
|
|
8
8
|
// that key travels with it rather than being copied here. That module's own
|
|
9
9
|
// I/O (its path probe, its file read) is never called from this file; only
|
|
10
|
-
// assessBeat/isStalled/stallReport are, all of which are functions of their
|
|
10
|
+
// assessBeat/isStalled/isReported/stallReport are, all of which are functions of their
|
|
11
11
|
// arguments. Importing the rule is what keeps the cockpit's stall wording and
|
|
12
12
|
// fleet-tick's identical — two spellings of "this run is dead" would be two
|
|
13
13
|
// answers the operator has to reconcile at 3am.
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
// it to its ticket number regardless; see the same paragraph below.
|
|
25
25
|
// fleet-tick.mjs's I/O and main() never run here, main() being guarded on
|
|
26
26
|
// argv[1].
|
|
27
|
-
import { assessBeat, isStalled, stallReport, NO_OWNER } from "./fleet-state.mjs";
|
|
27
|
+
import { assessBeat, isStalled, isReported, stallReport, NO_OWNER } from "./fleet-state.mjs";
|
|
28
28
|
import { parseToken, rowPr, premisesOf, HALT_CAUSES, REVIEW, REVIEWED } from "./ledger-grammar.mjs";
|
|
29
29
|
import { latestFinisherAttempts } from "./fleet-tick.mjs";
|
|
30
30
|
|
|
@@ -382,9 +382,14 @@ function splitNumbered(line) {
|
|
|
382
382
|
// `owner` is whose stall it is, fleet-state.mjs's stallOwner() answer: it
|
|
383
383
|
// reads the process table, which is I/O, so gather() judges it and this
|
|
384
384
|
// function only words it. Absent is no controller named.
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
385
|
+
//
|
|
386
|
+
// The previous run's recorded stop is shown too, until this run's first tick
|
|
387
|
+
// lands: it is that run's end, worded as such, with nothing of this run's on
|
|
388
|
+
// it.
|
|
389
|
+
function stall(marks, tickets, pool, { ledgerOk, poolOk, owner }, now) {
|
|
390
|
+
const verdict = assessBeat({ ...marks, now });
|
|
391
|
+
if (!isReported(verdict)) return null;
|
|
392
|
+
if (!isStalled(verdict)) return { ...verdict, text: stallReport(verdict) };
|
|
388
393
|
const claimed = ledgerOk ? tickets.filter((t) => t.column !== "POOL" && t.column !== "MERGED").length : null;
|
|
389
394
|
const supply = poolOk ? pool : null;
|
|
390
395
|
return { ...verdict, claimed, supply, owner, text: stallReport(verdict, { claimed, supply, owner }) };
|
|
@@ -584,9 +589,9 @@ export function computeBoard(inputs) {
|
|
|
584
589
|
// beaten and one that is beating normally — the page hides the banner on
|
|
585
590
|
// either, because a banner that fires on every healthy tick is a banner
|
|
586
591
|
// the operator learns to read past.
|
|
587
|
-
//
|
|
592
|
+
// The marks pass through bare, unlike `spend` two fields up: that
|
|
588
593
|
// coalesce is load-bearing because the value is serialised into
|
|
589
|
-
// board.json as-is, while these
|
|
594
|
+
// board.json as-is, while these are only ever fed to assessBeat(),
|
|
590
595
|
// which already treats `undefined` and `null` identically — a `?? null`
|
|
591
596
|
// here would cost a line to say nothing stall() does not already do.
|
|
592
597
|
//
|
|
@@ -594,7 +599,7 @@ export function computeBoard(inputs) {
|
|
|
594
599
|
// `ledgerState`'s own `?? "read"` default just above: every caller that
|
|
595
600
|
// builds inputs by hand (every test in this suite) means a read that
|
|
596
601
|
// succeeded, and only gather() — which now sets both — can say otherwise.
|
|
597
|
-
liveness: stall(inputs.beat, inputs.ticked, tickets, pool, {
|
|
602
|
+
liveness: stall({ beat: inputs.beat, ticked: inputs.ticked, acted: inputs.acted, controller: inputs.controller }, tickets, pool, {
|
|
598
603
|
ledgerOk: (ledger.state ?? "read") === "read",
|
|
599
604
|
poolOk: inputs.poolOk ?? true,
|
|
600
605
|
owner: inputs.stallOwner ?? NO_OWNER,
|
package/scripts/fleet-state.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// The heartbeat's shared state file, owned jointly by fleet-tick.mjs,
|
|
2
|
-
// fleet-heartbeat.mjs and ledger.mjs
|
|
2
|
+
// fleet-heartbeat.mjs and ledger.mjs — so the path and the file's shape
|
|
3
3
|
// live here rather than in spellings that drift. Same rule arg.mjs states for
|
|
4
4
|
// the digits guard: what travels between scripts is the rule, never a copy of
|
|
5
5
|
// it.
|
|
@@ -35,12 +35,18 @@
|
|
|
35
35
|
// when rotate wrote it — a mark older than `at` is the previous
|
|
36
36
|
// run's. Removed, not left stale, by a rotate that finds no omp
|
|
37
37
|
// ancestor.
|
|
38
|
+
// acted ledger.mjs only. When a ledger write last landed: every
|
|
39
|
+
// subcommand that writes the ledger stamps it, rotate in the same
|
|
40
|
+
// write as `controller`. A busy run writes its ledger on every
|
|
41
|
+
// dispatch and settle, far more often than it ticks, and never
|
|
42
|
+
// beats while there is work, so this is the mark that tells a busy
|
|
43
|
+
// run from a dead one.
|
|
38
44
|
// seen fleet-tick only. {member: firstSeenAtMs}: the first tick that saw
|
|
39
45
|
// each live ledger token, so a later tick can say how long a token
|
|
40
46
|
// has stayed live — the ledger records that a member was dispatched,
|
|
41
47
|
// never when. Pruned to the live set on every tick, so a settled
|
|
42
48
|
// member's stamp does not outlive it. Carried, never written, by
|
|
43
|
-
// the heartbeat and by
|
|
49
|
+
// the heartbeat and by ledger.mjs.
|
|
44
50
|
//
|
|
45
51
|
// One writer per key. A key two scripts wrote would need locking to be
|
|
46
52
|
// correct, and none of them is in a position to hold one. That holds per key,
|
|
@@ -111,7 +117,7 @@ export function readState(path, name) {
|
|
|
111
117
|
// to judge and nothing to report. A zeroed mark would date the beat to the
|
|
112
118
|
// epoch and read as decades overdue on a run that has simply not started
|
|
113
119
|
// one — the cry-wolf direction this key must never fail in.
|
|
114
|
-
const fresh = { quiet: 0, elapsed: 0, digest: "", beat: null, ticked: null, controller: null, seen: {}, rest: {} };
|
|
120
|
+
const fresh = { quiet: 0, elapsed: 0, digest: "", beat: null, ticked: null, acted: null, controller: null, seen: {}, rest: {} };
|
|
115
121
|
// The same state, flagged: the file EXISTS and could not be used. A caller
|
|
116
122
|
// that acts on the absence of a record — ledger.mjs rotate — must tell this
|
|
117
123
|
// from `fresh`, which is a run that has simply not written one yet.
|
|
@@ -178,7 +184,8 @@ export function readState(path, name) {
|
|
|
178
184
|
};
|
|
179
185
|
// `ticked` carries no interval — fleet-tick has no promise to fail, only an
|
|
180
186
|
// occurrence — so it is a mark of one field, valid or absent, same "never
|
|
181
|
-
// zero" rule as `mark` above for the same cry-wolf reason.
|
|
187
|
+
// zero" rule as `mark` above for the same cry-wolf reason. `acted` is the
|
|
188
|
+
// same shape for the same reason.
|
|
182
189
|
const tick = (v) => {
|
|
183
190
|
if (!isRecord(v)) return null;
|
|
184
191
|
const at = num(v.at);
|
|
@@ -213,13 +220,14 @@ export function readState(path, name) {
|
|
|
213
220
|
for (const [member, at] of Object.entries(v)) if (Number.isInteger(at) && at > 0) out[member] = at;
|
|
214
221
|
return out;
|
|
215
222
|
};
|
|
216
|
-
const { quiet, elapsed, digest, beat, ticked, controller, seen, ...rest } = parsed;
|
|
223
|
+
const { quiet, elapsed, digest, beat, ticked, acted, controller, seen, ...rest } = parsed;
|
|
217
224
|
return {
|
|
218
225
|
quiet: num(quiet),
|
|
219
226
|
elapsed: num(elapsed),
|
|
220
227
|
digest: typeof digest === "string" ? digest : "",
|
|
221
228
|
beat: mark(beat),
|
|
222
229
|
ticked: tick(ticked),
|
|
230
|
+
acted: tick(acted),
|
|
223
231
|
controller: ctl(controller),
|
|
224
232
|
seen: stamps(seen),
|
|
225
233
|
rest,
|
|
@@ -269,12 +277,13 @@ export function writeState(path, name, prev, patch) {
|
|
|
269
277
|
// every hold and would otherwise erase fleet-tick's `ticked` the first
|
|
270
278
|
// time a heartbeat lands after a busy stretch.
|
|
271
279
|
...(prev.ticked ? { ticked: prev.ticked } : {}),
|
|
272
|
-
// And again for ledger.mjs
|
|
280
|
+
// And again for ledger.mjs's `controller` and `acted`, which both scripts
|
|
273
281
|
// above would otherwise erase on their next write. A patch REMOVES a key
|
|
274
282
|
// by naming it with the value `undefined`, which JSON.stringify drops —
|
|
275
283
|
// the way rotate retires a record no omp ancestor replaces.
|
|
276
284
|
...(prev.controller ? { controller: prev.controller } : {}),
|
|
277
|
-
|
|
285
|
+
...(prev.acted ? { acted: prev.acted } : {}),
|
|
286
|
+
// `seen` is fleet-tick's, and the heartbeat and ledger.mjs write this file
|
|
278
287
|
// too: carried the same way, so neither erases the ages the tick keeps.
|
|
279
288
|
...(prev.seen && Object.keys(prev.seen).length > 0 ? { seen: prev.seen } : {}),
|
|
280
289
|
...patch,
|
|
@@ -330,55 +339,102 @@ export const BEAT_GRACE = 2;
|
|
|
330
339
|
// here and imported there: every script that reads or writes the heartbeat
|
|
331
340
|
// file already imports this module, and a magic "1200" typed twice is exactly
|
|
332
341
|
// the drift the module header's "never a copy of it" rule exists to close.
|
|
333
|
-
// It doubles as the freshness window for `ticked`
|
|
334
|
-
//
|
|
335
|
-
// worst-case gap this design ever tolerates when only the
|
|
336
|
-
// beating, times the same grace multiplier as `beat`.
|
|
342
|
+
// It doubles as the freshness window for `ticked`, `acted` and the controller
|
|
343
|
+
// record below: none of them carries a promised interval, so each is judged
|
|
344
|
+
// against the worst-case gap this design ever tolerates when only the
|
|
345
|
+
// heartbeat is beating, times the same grace multiplier as `beat`.
|
|
337
346
|
export const DEFAULT_CEILING_S = 1200;
|
|
338
347
|
|
|
339
|
-
|
|
348
|
+
const WINDOW_MS = DEFAULT_CEILING_S * 1000 * BEAT_GRACE;
|
|
349
|
+
|
|
350
|
+
// A mark older than the controller record's `at` was left by the run before
|
|
351
|
+
// the one that wrote the record — rotate replaced the record after that run's
|
|
352
|
+
// last beat, tick and ledger write — so it says nothing about this run. With
|
|
353
|
+
// no record, or a record with no `at`, nothing can be placed and every mark
|
|
354
|
+
// counts as this run's.
|
|
355
|
+
function thisRun(mark, controller) {
|
|
356
|
+
if (mark == null) return null;
|
|
357
|
+
return controller == null || controller.at === null || mark.at >= controller.at ? mark : null;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// What the marks say, as a value. `now` is a parameter so every caller's
|
|
340
361
|
// verdict is deterministic and testable — the same reason compute-board.mjs
|
|
341
|
-
// takes one.
|
|
362
|
+
// takes one. Only this run's marks decide it.
|
|
342
363
|
//
|
|
343
|
-
// none no mark
|
|
344
|
-
// stall: there is no
|
|
345
|
-
// would fire on every first tick
|
|
346
|
-
// the line.
|
|
364
|
+
// none no mark of this run and no controller record. A fresh run, or
|
|
365
|
+
// one whose heartbeat never ran. NOT a stall: there is no run to
|
|
366
|
+
// judge, and reporting one here would fire on every first tick
|
|
367
|
+
// and train the reader to ignore the line.
|
|
347
368
|
// beating seen within the interval it promised, plus grace — OR a recent
|
|
348
|
-
//
|
|
349
|
-
// drains (SKILL.md: "beat when
|
|
350
|
-
//
|
|
351
|
-
//
|
|
352
|
-
//
|
|
353
|
-
// stopped a reason
|
|
354
|
-
// recorded stop IS the end of the run and waiting for it to go
|
|
369
|
+
// tick, ledger write or controller record covers for it. `beat`
|
|
370
|
+
// is only refreshed when the queue drains (SKILL.md: "beat when
|
|
371
|
+
// there is nothing to do"), so a busy run can go its whole length
|
|
372
|
+
// without one; its ticks and ledger writes are the liveness
|
|
373
|
+
// signal for exactly that case.
|
|
374
|
+
// stopped this run recorded a reason. Reported whatever the age, because
|
|
375
|
+
// a recorded stop IS the end of the run and waiting for it to go
|
|
355
376
|
// stale first would sit on the one report that knows its cause.
|
|
356
|
-
// stale overdue with no reason recorded AND no recent tick
|
|
357
|
-
// abrupt death: a crashed harness, a closed
|
|
358
|
-
//
|
|
359
|
-
//
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
377
|
+
// stale overdue with no reason recorded AND no recent tick or ledger
|
|
378
|
+
// write either. The abrupt death: a crashed harness, a closed
|
|
379
|
+
// terminal, an OOM kill — or a live controller that stopped
|
|
380
|
+
// acting. Readers say exactly that rather than naming a cause
|
|
381
|
+
// they cannot know. A run with a controller record and no beat of
|
|
382
|
+
// its own is judged off its latest tick, ledger write or the
|
|
383
|
+
// record itself, so one that dies before it ever beats still goes
|
|
384
|
+
// stale; `beatless` marks that verdict, and `mark` names which.
|
|
385
|
+
// ended the PREVIOUS run's recorded stop, while this run has not
|
|
386
|
+
// ticked yet. It is that run's end, never this run's verdict:
|
|
387
|
+
// reported until this run's first tick, and never after.
|
|
388
|
+
export function assessBeat({ beat = null, ticked = null, acted = null, controller = null, now }) {
|
|
389
|
+
const ours = thisRun(beat, controller);
|
|
390
|
+
if (ours) {
|
|
391
|
+
const seen = since(ours.at, ours.interval * 1000, now);
|
|
392
|
+
if (ours.stopped) return { ...seen, kind: "stopped", reason: ours.stopped };
|
|
393
|
+
return { ...seen, kind: now > stallsAt({ beat, ticked, acted, controller }) ? "stale" : "beating", reason: null };
|
|
394
|
+
}
|
|
395
|
+
const marks = [
|
|
396
|
+
["controller", controller?.at ?? null],
|
|
397
|
+
["ticked", thisRun(ticked, controller)?.at ?? null],
|
|
398
|
+
["acted", thisRun(acted, controller)?.at ?? null],
|
|
399
|
+
].filter(([, at]) => at !== null);
|
|
400
|
+
// Without a record a mark-less beat is no run at all, whatever ticks or
|
|
401
|
+
// ledger writes some earlier run left beside it.
|
|
402
|
+
const judged = controller !== null && marks.length > 0;
|
|
403
|
+
if (judged && now > stallsAt({ beat, ticked, acted, controller })) {
|
|
404
|
+
const [mark, at] = marks.reduce((a, b) => (b[1] >= a[1] ? b : a));
|
|
405
|
+
return { ...since(at, WINDOW_MS, now), kind: "stale", reason: null, beatless: true, mark };
|
|
406
|
+
}
|
|
407
|
+
if (beat?.stopped && thisRun(ticked, controller) === null) {
|
|
408
|
+
return { ...since(beat.at, beat.interval * 1000, now), kind: "ended", reason: beat.stopped };
|
|
409
|
+
}
|
|
410
|
+
return judged ? { kind: "beating" } : { kind: "none" };
|
|
370
411
|
}
|
|
371
412
|
|
|
372
|
-
//
|
|
373
|
-
//
|
|
374
|
-
//
|
|
375
|
-
//
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
413
|
+
// A mark's age against the window it is judged by. Clamped at zero rather
|
|
414
|
+
// than left signed: a mark from the future is a clock that moved, not a beat
|
|
415
|
+
// that is minus-five-minutes overdue, and a negative age formatted into the
|
|
416
|
+
// line reads as a defect in this code.
|
|
417
|
+
function since(at, intervalMs, now) {
|
|
418
|
+
const ageMs = Math.max(0, now - at);
|
|
419
|
+
return { at, ageMs, intervalMs, overdueMs: Math.max(0, ageMs - intervalMs) };
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// The instant this run turns `stale`: past `beat`'s own interval plus grace
|
|
423
|
+
// AND past the ceiling window plus grace of every tick, ledger write and
|
|
424
|
+
// controller record, whichever ends latest. Only this run's marks count — the
|
|
425
|
+
// previous run's are partitioned off by the record's `at` exactly as
|
|
426
|
+
// assessBeat() does. An absent mark contributes nothing, and null means this
|
|
427
|
+
// run has no mark at all. Exported for the readers that must say WHEN a
|
|
428
|
+
// beating run would stop counting, not only whether it has.
|
|
429
|
+
export function stallsAt({ beat = null, ticked = null, acted = null, controller = null }) {
|
|
430
|
+
const ends = [ticked, acted]
|
|
431
|
+
.map((m) => thisRun(m, controller))
|
|
432
|
+
.filter((m) => m !== null)
|
|
433
|
+
.map((m) => m.at + WINDOW_MS);
|
|
434
|
+
if (controller?.at != null) ends.push(controller.at + WINDOW_MS);
|
|
435
|
+
const ours = thisRun(beat, controller);
|
|
436
|
+
if (ours) ends.push(ours.at + ours.interval * 1000 * BEAT_GRACE);
|
|
437
|
+
return ends.length > 0 ? Math.max(...ends) : null;
|
|
382
438
|
}
|
|
383
439
|
|
|
384
440
|
// Whether this verdict is something to report. Exported because both readers
|
|
@@ -389,6 +445,13 @@ export function isStalled(verdict) {
|
|
|
389
445
|
return verdict.kind === "stopped" || verdict.kind === "stale";
|
|
390
446
|
}
|
|
391
447
|
|
|
448
|
+
// Whether this verdict prints at all: a stall, or the previous run's end. The
|
|
449
|
+
// second never reaches isStalled(), whose readers go on to count claims and
|
|
450
|
+
// judge the controller — both this run's, and neither the previous run's.
|
|
451
|
+
export function isReported(verdict) {
|
|
452
|
+
return isStalled(verdict) || verdict.kind === "ended";
|
|
453
|
+
}
|
|
454
|
+
|
|
392
455
|
// Whole minutes, floored, because the numbers here are tens of minutes to
|
|
393
456
|
// hours and a seconds-precise age in a stall line is false precision: the
|
|
394
457
|
// mark is written once per hold, so its resolution is minutes anyway. That
|
|
@@ -439,26 +502,26 @@ function mins(ms) {
|
|
|
439
502
|
//
|
|
440
503
|
// A mark older than the record's `at` was left by the run BEFORE the one
|
|
441
504
|
// that wrote the record: rotate replaced the record after that run's last
|
|
442
|
-
// beat and
|
|
443
|
-
//
|
|
444
|
-
// run alive. That stall reads the verdict rotate reached on the
|
|
445
|
-
// replaced, `prior`: dead is a controller gone whose claims this run
|
|
446
|
-
// ancestor is this same session's earlier run and so alive, none is
|
|
447
|
-
// A mark written since the record — any later tick
|
|
448
|
-
// controller's own, so the record itself is judged,
|
|
449
|
-
// a reused pid is gone and the NEXT run resumes;
|
|
450
|
-
// reader, is alive. A record with no `at` cannot
|
|
451
|
-
// judged too.
|
|
505
|
+
// beat, tick and ledger write. When every mark is that old, the record names
|
|
506
|
+
// a controller that has left no mark yet, so judging it would call every
|
|
507
|
+
// previous run alive. That stall reads the verdict rotate reached on the
|
|
508
|
+
// record it replaced, `prior`: dead is a controller gone whose claims this run
|
|
509
|
+
// resumes, ancestor is this same session's earlier run and so alive, none is
|
|
510
|
+
// unknown. A mark written since the record — any later beat, tick or ledger
|
|
511
|
+
// write — is the recorded controller's own, so the record itself is judged,
|
|
512
|
+
// through proc.mjs: dead or a reused pid is gone and the NEXT run resumes;
|
|
513
|
+
// alive, or an ancestor of the reader, is alive. A record with no `at` cannot
|
|
514
|
+
// place the mark and is judged too.
|
|
452
515
|
//
|
|
453
516
|
// `judge` is replaceable for the same reason proc.mjs's process source is: a
|
|
454
517
|
// reader under a live omp session would otherwise find that session. A
|
|
455
518
|
// process table that cannot be read leaves the stall unattributed, said on
|
|
456
519
|
// stderr, rather than calling a controller nobody could judge dead or alive.
|
|
457
520
|
export const NO_OWNER = Object.freeze({ kind: "none" });
|
|
458
|
-
export function stallOwner({ controller, beat, ticked }, name, judge = judgeRecord) {
|
|
521
|
+
export function stallOwner({ controller, beat = null, ticked = null, acted = null }, name, judge = judgeRecord) {
|
|
459
522
|
if (!controller) return NO_OWNER;
|
|
460
|
-
const before = (mark) => mark === null
|
|
461
|
-
if (controller.at !== null && before(beat) && before(ticked)) {
|
|
523
|
+
const before = (mark) => thisRun(mark, controller) === null;
|
|
524
|
+
if (controller.at !== null && before(beat) && before(ticked) && before(acted)) {
|
|
462
525
|
if (controller.prior === "dead") return { kind: "gone", resumes: "this run" };
|
|
463
526
|
if (controller.prior === "ancestor") return { kind: "alive", pid: null };
|
|
464
527
|
return NO_OWNER;
|
|
@@ -476,9 +539,13 @@ export function stallOwner({ controller, beat, ticked }, name, judge = judgeReco
|
|
|
476
539
|
return NO_OWNER;
|
|
477
540
|
}
|
|
478
541
|
|
|
479
|
-
// The report. Returns null for anything
|
|
542
|
+
// The report. Returns null for anything isReported() rejects, so a caller that
|
|
480
543
|
// skipped the predicate cannot print a stall for a healthy run by accident.
|
|
481
544
|
//
|
|
545
|
+
// The previous run's end is one line of its own and carries nothing of this
|
|
546
|
+
// run's: no claims, no supply, no owner. It names when that run stopped and
|
|
547
|
+
// the reason it recorded, and nothing more.
|
|
548
|
+
//
|
|
482
549
|
// What it names is the acceptance criterion itself: a bare "stale" tells the
|
|
483
550
|
// maintainer nothing they can act on, so the line carries when the beat was
|
|
484
551
|
// last seen, how overdue that is against the interval it PROMISED, whether a
|
|
@@ -498,22 +565,32 @@ export function stallOwner({ controller, beat, ticked }, name, judge = judgeReco
|
|
|
498
565
|
// cockpit's own cards) and either source can fail, and "0 claimed" off a
|
|
499
566
|
// failed read is the same lie as "supply 0" off one — it says the night cost
|
|
500
567
|
// nothing when it may have stranded the whole pool.
|
|
501
|
-
export function stallReport(verdict, { claimed, supply, owner = NO_OWNER }) {
|
|
502
|
-
if (!
|
|
568
|
+
export function stallReport(verdict, { claimed, supply, owner = NO_OWNER } = {}) {
|
|
569
|
+
if (!isReported(verdict)) return null;
|
|
570
|
+
const when = (at) => new Date(at).toISOString();
|
|
571
|
+
if (verdict.kind === "ended") {
|
|
572
|
+
return `previous run stopped deliberately at ${when(verdict.at)} (${mins(verdict.ageMs)} ago) — recorded reason: ${verdict.reason}`;
|
|
573
|
+
}
|
|
503
574
|
const n = (v) => (v == null ? "unknown" : String(v));
|
|
504
575
|
// Present tense for the deliberate stop and past for the abrupt one, because
|
|
505
576
|
// that is the actual difference: one run said why it was going, the other
|
|
506
577
|
// was cut off mid-beat and left nothing behind to ask.
|
|
507
578
|
const why = verdict.kind === "stopped"
|
|
508
579
|
? `stopped deliberately — recorded reason: ${verdict.reason}`
|
|
509
|
-
: "the beat stopped without a recorded reason";
|
|
580
|
+
: verdict.beatless ? "no beat, tick or ledger write since, and no recorded reason" : "the beat stopped without a recorded reason";
|
|
510
581
|
const head = owner.kind === "alive"
|
|
511
582
|
? `heartbeat not beating, controller alive (${owner.pid === null ? "this session's earlier run" : `pid ${owner.pid}`})`
|
|
512
583
|
: owner.kind === "gone" ? "heartbeat STALLED, controller gone" : "heartbeat STALLED";
|
|
513
584
|
const tail = owner.kind === "gone"
|
|
514
585
|
? `; ${owner.resumes} resumes its claims without a PR, and its open PRs return through the fold-in`
|
|
515
586
|
: "";
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
587
|
+
const seen = verdict.beatless
|
|
588
|
+
? `no beat this run, last ${MARK_NAMES[verdict.mark]} ${when(verdict.at)}`
|
|
589
|
+
+ ` (${mins(verdict.ageMs)} ago, ${mins(verdict.overdueMs)} past the ${mins(verdict.intervalMs)} window)`
|
|
590
|
+
: `last beat ${when(verdict.at)}`
|
|
591
|
+
+ ` (${mins(verdict.ageMs)} ago, ${mins(verdict.overdueMs)} past the ${mins(verdict.intervalMs)} interval it promised)`;
|
|
592
|
+
return `${head}: ${seen} — ${why}; ${n(claimed)} ticket(s) claimed and in flight, pool supply ${n(supply)}${tail}`;
|
|
519
593
|
}
|
|
594
|
+
|
|
595
|
+
// What a beatless stall names as this run's last sign of life.
|
|
596
|
+
const MARK_NAMES = { controller: "controller record", ticked: "tick", acted: "ledger write" };
|
package/scripts/fleet-tick.mjs
CHANGED
|
@@ -12,7 +12,9 @@
|
|
|
12
12
|
// actual/target and a named ACTION per role — `PULL #<n> #<n>`, `DISPATCH
|
|
13
13
|
// fix-pr PR#<pr>`, `DISPATCH review PR#<pr>`, `DISPATCH merge-bot`, `HOLD (…)`,
|
|
14
14
|
// `REFRESHED shortlist: …` — with run-team's guards applied in code rather than
|
|
15
|
-
// recalled from its table. You cannot forget what a script prints at you.
|
|
15
|
+
// recalled from its table. You cannot forget what a script prints at you. A
|
|
16
|
+
// drained run with nothing left in flight ends on one more: `ACTION: run over`,
|
|
17
|
+
// naming the stop the controller records last.
|
|
16
18
|
//
|
|
17
19
|
// WHERE THE NUMBERS COME FROM — nobody states them:
|
|
18
20
|
//
|
|
@@ -992,7 +994,7 @@ import { parseArgs } from "node:util";
|
|
|
992
994
|
import { makeDie, isDigits } from "./arg.mjs";
|
|
993
995
|
import { gitEnv } from "./git-env.mjs";
|
|
994
996
|
import { fleetFile, FleetDirUnresolvable } from "./fleet-dir.mjs";
|
|
995
|
-
import { statePath, readState, writeState, assessBeat, isStalled, stallReport, stallOwner } from "./fleet-state.mjs";
|
|
997
|
+
import { statePath, readState, writeState, assessBeat, isStalled, isReported, stallReport, stallOwner } from "./fleet-state.mjs";
|
|
996
998
|
import { checkMainCheckout, describe } from "./main-checkout.mjs";
|
|
997
999
|
|
|
998
1000
|
const NAME = "fleet-tick";
|
|
@@ -1009,6 +1011,9 @@ const MERGEABLE_STATES = ["MERGEABLE", "CONFLICTING", "UNKNOWN"];
|
|
|
1009
1011
|
// cap, because refusing would suppress the one announcement a dead run leaves
|
|
1010
1012
|
// behind. So it discloses instead — the line says the count is a floor.
|
|
1011
1013
|
const CLAIMED_LIMIT = 200;
|
|
1014
|
+
// The tick's last line once a draining run has nothing left in flight. The
|
|
1015
|
+
// tick cannot write the stop — `beat` is fleet-heartbeat's — so it names it.
|
|
1016
|
+
export const RUN_OVER = `ACTION: run over → fleet-heartbeat.mjs --stop "<reason>"`;
|
|
1012
1017
|
const MAX_BUFFER = 64 * 1024 * 1024;
|
|
1013
1018
|
// Every gh spawn below is killed at this bound, so a gh that never answers is
|
|
1014
1019
|
// a failed read on that call's own failure path — the open-PR read refuses the
|
|
@@ -1324,9 +1329,24 @@ function main() {
|
|
|
1324
1329
|
// Whose stall it is comes off the controller record, judged only once
|
|
1325
1330
|
// there is a stall to name, so a healthy tick reads no process table.
|
|
1326
1331
|
const priorState = readState(path, NAME);
|
|
1327
|
-
const
|
|
1328
|
-
|
|
1329
|
-
|
|
1332
|
+
const tickedAt = Date.now();
|
|
1333
|
+
const { beat, ticked, acted, controller } = priorState;
|
|
1334
|
+
const verdict = assessBeat({ beat, ticked, acted, controller, now: tickedAt });
|
|
1335
|
+
// `ticked`, fleet-tick's own liveness key, written here and on its own,
|
|
1336
|
+
// ahead of the gh and ledger reads that can refuse this tick: a refused
|
|
1337
|
+
// tick was still invoked, and a run whose every tick refuses on a gh outage
|
|
1338
|
+
// is a run working, not a dead one. Only the usage refusals above leave no
|
|
1339
|
+
// mark — they never reached the state file — and a file that exists but
|
|
1340
|
+
// could not be read is left as it is, as ledger.mjs's `acted` stamp leaves
|
|
1341
|
+
// it: rotate refuses on such a file because a record it may hold cannot be
|
|
1342
|
+
// known, and a write here would replace that record with a fresh one.
|
|
1343
|
+
// Judged before it is written, so the previous run's end prints on this
|
|
1344
|
+
// run's first tick and on no later one.
|
|
1345
|
+
if (!priorState.degraded) writeState(path, NAME, priorState, { ticked: { at: tickedAt } });
|
|
1346
|
+
if (isReported(verdict)) {
|
|
1347
|
+
console.log(stallReport(verdict, isStalled(verdict)
|
|
1348
|
+
? { claimed: claimed(), supply: current.scanned, owner: stallOwner(priorState, NAME) }
|
|
1349
|
+
: {}));
|
|
1330
1350
|
}
|
|
1331
1351
|
|
|
1332
1352
|
// Every read happens before anything prints: a partial tick is worse than no
|
|
@@ -1392,19 +1412,26 @@ function main() {
|
|
|
1392
1412
|
const seen = stampSeen({ live: run.live, prev: prev.seen, controller: prev.controller, now });
|
|
1393
1413
|
const stale = livenessLines({ live: run.live, seen, thresholds: liveness, now });
|
|
1394
1414
|
const lines = [...(said === null ? [] : [said]), ...stale.map((s) => s.line), ...formatLines(rows)];
|
|
1415
|
+
// The run is over once it is draining and nothing is left in flight: no
|
|
1416
|
+
// open PR and no implementer still holding a claim. Printed last and read
|
|
1417
|
+
// as actionable, so it never folds away: the stop it names is the run's
|
|
1418
|
+
// last act, and only the controller can write it.
|
|
1419
|
+
const over = run.draining !== null && prs.length === 0 && run.implLive === 0;
|
|
1420
|
+
if (over) lines.push(RUN_OVER);
|
|
1395
1421
|
|
|
1396
1422
|
// The back-off streak and the fold digest, written on EVERY tick: an edge
|
|
1397
1423
|
// tick that dispatched is the clearest possible "there is work here". Digest,
|
|
1398
1424
|
// not the lines themselves, so the file stays readable.
|
|
1399
1425
|
const digest = createHash("sha256").update(lines.join("\n")).digest("hex");
|
|
1400
|
-
const acts = actionable(rows);
|
|
1401
|
-
// `
|
|
1402
|
-
//
|
|
1403
|
-
//
|
|
1404
|
-
//
|
|
1426
|
+
const acts = actionable(rows) || over;
|
|
1427
|
+
// `seen` is pruned to the live set on the same write; an empty one removes
|
|
1428
|
+
// the key. `ticked` was written at the top of the tick, unless the file
|
|
1429
|
+
// could not be read then: this write replaces such a file, so it carries
|
|
1430
|
+
// the stamp the top of the tick left out.
|
|
1405
1431
|
writeState(path, NAME, prev, {
|
|
1406
|
-
quiet: acts ? 0 : prev.quiet + 1, digest,
|
|
1432
|
+
quiet: acts ? 0 : prev.quiet + 1, digest,
|
|
1407
1433
|
seen: Object.keys(seen).length > 0 ? seen : undefined,
|
|
1434
|
+
...(priorState.degraded ? { ticked: { at: tickedAt } } : {}),
|
|
1408
1435
|
});
|
|
1409
1436
|
|
|
1410
1437
|
// Fold only when BOTH hold: nothing to act on, and nothing new to say. A
|
package/scripts/ledger.mjs
CHANGED
|
@@ -32,6 +32,12 @@
|
|
|
32
32
|
// `filed` kept 2-5 of 8 rows — and the lock is what lets a second kind of
|
|
33
33
|
// writer exist at all. The reasoning for each part of the lock lives at
|
|
34
34
|
// acquireLock() below.
|
|
35
|
+
//
|
|
36
|
+
// Every write subcommand also stamps `acted` in the heartbeat file beside the
|
|
37
|
+
// ledger whenever its call ends without a refusal — rotate in the same write
|
|
38
|
+
// as its controller record. A run's ledger writes are its densest sign of
|
|
39
|
+
// life, so the stall readers judge a busy run by them (fleet-state.mjs has
|
|
40
|
+
// the rule).
|
|
35
41
|
|
|
36
42
|
import { readFileSync, writeFileSync, mkdirSync, existsSync, renameSync, openSync, writeSync, closeSync, unlinkSync, linkSync } from "node:fs";
|
|
37
43
|
import { dirname, resolve, join, basename } from "node:path";
|
|
@@ -929,6 +935,18 @@ if (cmd === "read") {
|
|
|
929
935
|
die(`unknown subcommand '${cmd}' — expected row, filed, ruled, check, read, dispatch, settle, review, drain or rotate`);
|
|
930
936
|
}
|
|
931
937
|
|
|
938
|
+
// Reached only by an arm that did not die, so every write subcommand that did
|
|
939
|
+
// not refuse is stamped — one that found nothing to change (a repeat `drain`,
|
|
940
|
+
// a duplicate `filed`) is still the controller acting. Still under the write
|
|
941
|
+
// lock, so two ledger calls never race on the key. A heartbeat file that
|
|
942
|
+
// exists and cannot be read is left as it is for the readers that refuse on
|
|
943
|
+
// it; readState() has already said why.
|
|
944
|
+
if (WRITE_COMMANDS.has(cmd) && cmd !== "rotate") {
|
|
945
|
+
const beatFile = stateFileIn(dirname(file));
|
|
946
|
+
const state = readState(beatFile, NAME);
|
|
947
|
+
if (!state.degraded) writeState(beatFile, NAME, state, { acted: { at: Date.now() } });
|
|
948
|
+
}
|
|
949
|
+
|
|
932
950
|
// `check` alone of the subcommands leaves its arm early: the already-filed
|
|
933
951
|
// answer is complete before the near-miss ranking and the tracker query below
|
|
934
952
|
// it, whose results that answer would only discard. A function is what makes
|
|
@@ -1570,9 +1588,10 @@ function runCheck() {
|
|
|
1570
1588
|
// and keep dispatching — and rotating under it would split its run across
|
|
1571
1589
|
// two ledgers. A dead one, a reused pid, or this session's own earlier run
|
|
1572
1590
|
// allows the rotation. With no record the guard falls back to the mark, read
|
|
1573
|
-
// through fleet-state.mjs's own reader and verdict: still beating
|
|
1574
|
-
//
|
|
1575
|
-
//
|
|
1591
|
+
// through fleet-state.mjs's own reader and verdict: still beating — a beat
|
|
1592
|
+
// inside its window, or past it with a tick or ledger write inside
|
|
1593
|
+
// theirs — refuses; a stalled mark, a recorded stop, or no beat at all allows
|
|
1594
|
+
// the rotation, and that fallback proves "not beating", never "dead".
|
|
1576
1595
|
//
|
|
1577
1596
|
// A heartbeat file that exists but cannot be read or parsed gets neither
|
|
1578
1597
|
// verdict: a record it may hold cannot be known, so rotate refuses rather
|
|
@@ -1582,7 +1601,8 @@ function runCheck() {
|
|
|
1582
1601
|
// record with this run's own, carrying the verdict as `prior` — the next
|
|
1583
1602
|
// step reads `prior`, because the new record always names this run's own
|
|
1584
1603
|
// live ancestor — and the time of this write as `at`, so a reader can tell a
|
|
1585
|
-
// mark the previous run left from one written since.
|
|
1604
|
+
// mark the previous run left from one written since. The same write stamps
|
|
1605
|
+
// `acted` at that time, as every ledger write does. No omp ancestor removes
|
|
1586
1606
|
// the key: an older run's record left in place would be judged again, as
|
|
1587
1607
|
// dead, by a run that cannot know whether the run after it is still live.
|
|
1588
1608
|
// This run's own record is computed before anything moves too, so a process
|
|
@@ -1591,7 +1611,7 @@ function runCheck() {
|
|
|
1591
1611
|
function runRotate() {
|
|
1592
1612
|
const beatFile = stateFileIn(dirname(file));
|
|
1593
1613
|
const state = readState(beatFile, NAME);
|
|
1594
|
-
const { beat, ticked, controller } = state;
|
|
1614
|
+
const { beat, ticked, acted, controller } = state;
|
|
1595
1615
|
// A file that exists and could not be used holds no record we can know of:
|
|
1596
1616
|
// reading it as "no record" would let a live controller's ledger be moved
|
|
1597
1617
|
// by exactly the fault that hid its record.
|
|
@@ -1612,10 +1632,14 @@ function runRotate() {
|
|
|
1612
1632
|
die(`refusing to rotate ${file}: the controller that owns it is alive — controller pid ${controller.pid} (started ${controller.lstart}) recorded in ${beatFile} is alive and not an ancestor of this run, whatever its heartbeat mark says; one controller per checkout — end that omp session, or start the run from inside it`);
|
|
1613
1633
|
}
|
|
1614
1634
|
// The fallback guards a ledger, as it always has: with none to move there
|
|
1615
|
-
// is nothing for a beating mark to protect.
|
|
1635
|
+
// is nothing for a beating mark to protect. A ledger write is the
|
|
1636
|
+
// controller acting, and counts the way a beat does.
|
|
1616
1637
|
const exists = existsSync(file);
|
|
1617
|
-
if (prior === "none" && exists && assessBeat({ beat, ticked, now: Date.now() }).kind === "beating") {
|
|
1618
|
-
|
|
1638
|
+
if (prior === "none" && exists && assessBeat({ beat, ticked, acted, now: Date.now() }).kind === "beating") {
|
|
1639
|
+
const [what, mark] = [["beat", beat], ["tick", ticked], ["ledger write", acted]]
|
|
1640
|
+
.filter(([, m]) => m !== null)
|
|
1641
|
+
.reduce((a, b) => (b[1].at > a[1].at ? b : a));
|
|
1642
|
+
die(`refusing to rotate ${file}: the controller that owns it is still beating or acting — last ${what} ${new Date(mark.at).toISOString()} in ${beatFile}; it counts as stalled after ${new Date(stallsAt({ beat, ticked, acted })).toISOString()}`);
|
|
1619
1643
|
}
|
|
1620
1644
|
// Read before anything moves: a ledger whose live tokens cannot be read is
|
|
1621
1645
|
// refused with nothing moved, never archived with those tokens dropped.
|
|
@@ -1670,8 +1694,9 @@ function runRotate() {
|
|
|
1670
1694
|
// Re-read rather than reuse the read above: the heartbeat file has other
|
|
1671
1695
|
// writers, and a patch built from an older view would undo what they wrote
|
|
1672
1696
|
// since.
|
|
1673
|
-
const
|
|
1674
|
-
|
|
1697
|
+
const now = Date.now();
|
|
1698
|
+
const record = ours === null ? undefined : { ...ours, prior, at: now };
|
|
1699
|
+
if (!writeState(beatFile, NAME, readState(beatFile, NAME), { controller: record, acted: { at: now } })) {
|
|
1675
1700
|
die(`${archive === null ? "found no ledger" : `rotated ${file} -> ${archive}`} but could not record this run's controller in ${beatFile} — the next run judges whatever record that file still holds, and none if it cannot be read`);
|
|
1676
1701
|
}
|
|
1677
1702
|
console.log(JSON.stringify({
|
package/skills/run-team/SKILL.md
CHANGED
|
@@ -441,9 +441,10 @@ phase, or in any later one, asks the maintainer which tickets to take.
|
|
|
441
441
|
controller can stop beating and keep dispatching. A dead one, a pid
|
|
442
442
|
reused by another process, or this session's own earlier run lets it
|
|
443
443
|
rotate. With no record the heartbeat mark decides, as the only guard
|
|
444
|
-
there is: still beating refuses, naming the
|
|
445
|
-
count as stalled; a stalled mark, a
|
|
446
|
-
lets it rotate — and that fallback
|
|
444
|
+
there is: still beating or acting refuses, naming the latest beat, tick or
|
|
445
|
+
ledger write and when it would count as stalled; a stalled mark, a
|
|
446
|
+
recorded `--stop`, or no beat at all lets it rotate — and that fallback
|
|
447
|
+
proves "not beating", never "dead", and
|
|
447
448
|
guards a ledger only: with none to move the mark is not consulted. With no
|
|
448
449
|
ledger the record is still judged, so a live recorded controller refuses
|
|
449
450
|
even then. It also refuses, nothing moved, when the stamped archive name
|
|
@@ -454,10 +455,12 @@ phase, or in any later one, asks the maintainer which tickets to take.
|
|
|
454
455
|
above it, its start time, as `prior` the verdict it reached on the
|
|
455
456
|
record it found (`dead`, `ancestor` or `none`), and as `at` when it wrote
|
|
456
457
|
it — or to remove the key when
|
|
457
|
-
no `omp` process is above it.
|
|
458
|
+
no `omp` process is above it. The same write stamps `acted`, the time of
|
|
459
|
+
this run's latest ledger write, which every write subcommand stamps from
|
|
460
|
+
then on. That write can fail, and then exit 2 follows
|
|
458
461
|
a move: `rotated <file> -> <archive>` on stderr, then the refusal that the
|
|
459
462
|
record could not be written. Run the rotation before this run's own first
|
|
460
|
-
tick or
|
|
463
|
+
tick, beat or ledger write, each of which writes a mark the fallback reads. Older
|
|
461
464
|
archives, hand-named ones included, are left alone.
|
|
462
465
|
|
|
463
466
|
**Derive the Recipe — the Recipe derivation step — before the fold-in below
|
|
@@ -1671,7 +1674,7 @@ ticket's line — never a hand edit.
|
|
|
1671
1674
|
| Label seen (persistent Monitor) | nothing to record |
|
|
1672
1675
|
| CI run terminal | `ci=<run-id>:<attempt>:<conclusion>` on the row; then the finisher gate (below) |
|
|
1673
1676
|
| Merge-bot pass report | `held-behind:#<lower>` rows; `ledger.mjs settle merge-bot-<n>=done`; `reap.sh --apply`. (The bot may also have written `conflict-hold:#<pr>` onto a held PR's own row earlier in this same pass, before reporting — a token the bot wrote in this pass, never one `reap.sh` writes.) |
|
|
1674
|
-
| Drain | `ledger.mjs drain "<reason>"`; release the claims (below); `settle impl-<N>=released` |
|
|
1677
|
+
| Drain | `ledger.mjs drain "<reason>"`; release the claims (below); `settle impl-<N>=released`; then the rest of **End of run** (below), `--stop` last |
|
|
1675
1678
|
| Heartbeat | nothing to record |
|
|
1676
1679
|
|
|
1677
1680
|
Then run the tick, act on every line it prints, and arm the beat. What a wake
|
|
@@ -1739,6 +1742,19 @@ owes beyond its row:
|
|
|
1739
1742
|
claims. The marker is on disk, so the tick holds the implementer row at
|
|
1740
1743
|
`HOLD (draining)` from then on — for a replacement controller too — and keeps
|
|
1741
1744
|
the review, fix-applier and merge rows firing until the open PRs merge.
|
|
1745
|
+
The drain starts the end of the run; it does not end it. **End of run**, in
|
|
1746
|
+
this order:
|
|
1747
|
+
1. `ledger.mjs drain "<reason>"`.
|
|
1748
|
+
2. Release every claim that never became a PR, settling each
|
|
1749
|
+
`impl-<N>=released`.
|
|
1750
|
+
3. Act on the review, fix-applier and merge rows until no PR is open. Once
|
|
1751
|
+
the run is draining with no open PR and no live implementer, the tick's
|
|
1752
|
+
last line is `ACTION: run over → fleet-heartbeat.mjs --stop "<reason>"`.
|
|
1753
|
+
4. Scrape the run's member facts, append every `tier-outcomes.mjs` row, and
|
|
1754
|
+
run `metrics-refresh.mjs` (above).
|
|
1755
|
+
5. Commit the metrics files and open their PR (above).
|
|
1756
|
+
6. `~/.fleet/bin/fleet-run fleet-heartbeat.mjs --stop "<reason>"` — last,
|
|
1757
|
+
because every reader takes a recorded stop as the end of the run.
|
|
1742
1758
|
- **Monitor: CI run completes** → bind it (`ci-state.mjs --pr <N>`) and **record
|
|
1743
1759
|
it as `ci=<run-id>:<attempt>:<conclusion>`, not just "CI green"** — in the
|
|
1744
1760
|
ticket's ledger row, which outlives your context. `row` **replaces the whole
|
|
@@ -1997,34 +2013,41 @@ than being detected, and nothing else can write the reason afterwards:
|
|
|
1997
2013
|
~/.fleet/bin/fleet-run fleet-heartbeat.mjs --stop "budget exhausted"
|
|
1998
2014
|
```
|
|
1999
2015
|
|
|
2000
|
-
It holds nothing and returns at once. Use it when the budget goes, when
|
|
2001
|
-
can see the context ceiling coming
|
|
2016
|
+
It holds nothing and returns at once. Use it when the budget goes, or when
|
|
2017
|
+
you can see the context ceiling coming. A drained run records it as the last
|
|
2018
|
+
step of **End of run** (above), never at the drain itself.
|
|
2002
2019
|
Skipping it is not fatal — the next reader still reports the stop, it just
|
|
2003
2020
|
reports that the beat stopped *without a recorded reason*, which is all it can
|
|
2004
2021
|
honestly say about a death that named nothing.
|
|
2005
2022
|
|
|
2006
2023
|
**A stall line names whose stall it is.** `fleet-tick`
|
|
2007
|
-
prints it before its own rows when
|
|
2008
|
-
interval
|
|
2009
|
-
|
|
2024
|
+
prints it before its own rows when this run has gone quiet: its beat overdue
|
|
2025
|
+
against the interval it recorded with no tick or ledger write inside the
|
|
2026
|
+
40-minute window either — or, with no beat yet, its latest tick, ledger write
|
|
2027
|
+
or controller record past that window. A beat, tick or ledger write older than
|
|
2028
|
+
the controller record rotate wrote is the previous run's and counts for
|
|
2029
|
+
nothing. The cockpit shows the same line while it is up. It names when this
|
|
2030
|
+
run was last seen, how overdue that is, how many
|
|
2010
2031
|
tickets are still claimed and in flight, and how much supply the last read found
|
|
2011
2032
|
— because the claimed ones keep the `in-progress` label, the candidate scan
|
|
2012
2033
|
EXCLUDES that label, and those tickets are therefore invisible to your own
|
|
2013
2034
|
shortlist and to the maintainer's. Its opening words come off the controller
|
|
2014
|
-
record
|
|
2015
|
-
`prior` rotate recorded; any later line is judged off the record itself:
|
|
2035
|
+
record:
|
|
2016
2036
|
|
|
2017
2037
|
- `heartbeat STALLED, controller gone` — the run that stranded them is dead.
|
|
2018
|
-
Its claims without a PR come back through
|
|
2019
|
-
|
|
2020
|
-
through the fold-in.
|
|
2038
|
+
Its claims without a PR come back through the next run's phase 0
|
|
2039
|
+
stranded-claim step, and its open PRs through the fold-in.
|
|
2021
2040
|
- `heartbeat not beating, controller alive (pid N)`, or `(this session's
|
|
2022
2041
|
earlier run)` — a controller that stopped beating and kept going. Nothing
|
|
2023
|
-
died, so nothing is resumed;
|
|
2024
|
-
|
|
2042
|
+
died, so nothing is resumed; when that controller is you, the fix is to
|
|
2043
|
+
tick on every wake and beat when there is nothing to do.
|
|
2025
2044
|
- `heartbeat STALLED` alone — no record says whether that controller lives.
|
|
2026
2045
|
The line is detection only.
|
|
2027
2046
|
|
|
2047
|
+
Your first tick may print `previous run stopped deliberately at <time> (<age>
|
|
2048
|
+
ago) — recorded reason: <reason>` instead: the stop the previous run recorded.
|
|
2049
|
+
It is that run's end, not a verdict on yours, and no later tick prints it.
|
|
2050
|
+
|
|
2028
2051
|
Read the line and carry on; nothing in the fleet releases, relabels or
|
|
2029
2052
|
reaps the stranded tickets for you, and nothing restarts the run that stranded them.
|
|
2030
2053
|
|