@feigi/fleet-ctl 3.38.0 → 3.39.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@feigi/fleet-ctl",
3
- "version": "3.38.0",
3
+ "version": "3.39.0",
4
4
  "description": "Agent fleet: run-team controller, merge bot, PR reviewer, and the ticket pipeline they share",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
package/scripts/board.mjs CHANGED
@@ -1134,19 +1134,22 @@ export async function gather({ ledgerFile, prevFile, stateFile = null, scriptDir
1134
1134
  // reading. Defaulted rather than resolved here for gather()'s own reason —
1135
1135
  // every caller in this file passes the instance's answer, and a second
1136
1136
  // resolution could name a different workspace's beat.
1137
- // `ticked` rides beside `beat`, same file same read, and under the same
1138
- // "no try/catch, no tryRun shape" rule just above — fleet-tick.mjs's own
1139
- // liveness key, for the busy-stretch case `beat` alone
1140
- // cannot see (fleet-state.mjs's assessBeat has the rule).
1137
+ // `ticked`, `acted` and `controller` ride beside `beat`, same file same
1138
+ // read, and under the same "no try/catch, no tryRun shape" rule just above:
1139
+ // a run is judged from all of its own marks, and the record's `at` is what
1140
+ // tells this run's marks from the previous run's (fleet-state.mjs's
1141
+ // assessBeat has the rule).
1141
1142
  const priorState = stateFile ? readState(stateFile, NAME) : null;
1142
1143
  const beat = priorState?.beat ?? null;
1143
1144
  const ticked = priorState?.ticked ?? null;
1145
+ const acted = priorState?.acted ?? null;
1146
+ const controller = priorState?.controller ?? null;
1144
1147
  // Whose stall it is, off the controller record in the same file — judged
1145
1148
  // here because it reads the process table, and only when the mark is
1146
1149
  // stalled, so a healthy board reads none. computeBoard() words it.
1147
1150
  const now = Date.now();
1148
- const stallOwnerOf = isStalled(assessBeat({ beat, ticked, now })) ? stallOwner(priorState, NAME) : null;
1149
- return { ledger, issues, prs, merged, ci, prev, repo, repoUrl, workspace, port, spend, beat, ticked, stallOwner: stallOwnerOf, poolOk,
1151
+ const stallOwnerOf = isStalled(assessBeat({ beat, ticked, acted, controller, now })) ? stallOwner(priorState, NAME) : null;
1152
+ return { ledger, issues, prs, merged, ci, prev, repo, repoUrl, workspace, port, spend, beat, ticked, acted, controller, stallOwner: stallOwnerOf, poolOk,
1150
1153
  poolCapped, prsCapped,
1151
1154
  now, interval };
1152
1155
  }
@@ -7,7 +7,7 @@
7
7
  // fleet-state.mjs owns the heartbeat's `beat` key, and the rule for reading
8
8
  // that key travels with it rather than being copied here. That module's own
9
9
  // I/O (its path probe, its file read) is never called from this file; only
10
- // assessBeat/isStalled/stallReport are, all of which are functions of their
10
+ // assessBeat/isStalled/isReported/stallReport are, all of which are functions of their
11
11
  // arguments. Importing the rule is what keeps the cockpit's stall wording and
12
12
  // fleet-tick's identical — two spellings of "this run is dead" would be two
13
13
  // answers the operator has to reconcile at 3am.
@@ -24,7 +24,7 @@
24
24
  // it to its ticket number regardless; see the same paragraph below.
25
25
  // fleet-tick.mjs's I/O and main() never run here, main() being guarded on
26
26
  // argv[1].
27
- import { assessBeat, isStalled, stallReport, NO_OWNER } from "./fleet-state.mjs";
27
+ import { assessBeat, isStalled, isReported, stallReport, NO_OWNER } from "./fleet-state.mjs";
28
28
  import { parseToken, rowPr, premisesOf, HALT_CAUSES, REVIEW, REVIEWED } from "./ledger-grammar.mjs";
29
29
  import { latestFinisherAttempts } from "./fleet-tick.mjs";
30
30
 
@@ -382,9 +382,14 @@ function splitNumbered(line) {
382
382
  // `owner` is whose stall it is, fleet-state.mjs's stallOwner() answer: it
383
383
  // reads the process table, which is I/O, so gather() judges it and this
384
384
  // function only words it. Absent is no controller named.
385
- function stall(beat, ticked, tickets, pool, { ledgerOk, poolOk, owner }, now) {
386
- const verdict = assessBeat({ beat, ticked, now });
387
- if (!isStalled(verdict)) return null;
385
+ //
386
+ // The previous run's recorded stop is shown too, until this run's first tick
387
+ // lands: it is that run's end, worded as such, with nothing of this run's on
388
+ // it.
389
+ function stall(marks, tickets, pool, { ledgerOk, poolOk, owner }, now) {
390
+ const verdict = assessBeat({ ...marks, now });
391
+ if (!isReported(verdict)) return null;
392
+ if (!isStalled(verdict)) return { ...verdict, text: stallReport(verdict) };
388
393
  const claimed = ledgerOk ? tickets.filter((t) => t.column !== "POOL" && t.column !== "MERGED").length : null;
389
394
  const supply = poolOk ? pool : null;
390
395
  return { ...verdict, claimed, supply, owner, text: stallReport(verdict, { claimed, supply, owner }) };
@@ -584,9 +589,9 @@ export function computeBoard(inputs) {
584
589
  // beaten and one that is beating normally — the page hides the banner on
585
590
  // either, because a banner that fires on every healthy tick is a banner
586
591
  // the operator learns to read past.
587
- // `beat`/`ticked` pass through bare, unlike `spend` two fields up: that
592
+ // The marks pass through bare, unlike `spend` two fields up: that
588
593
  // coalesce is load-bearing because the value is serialised into
589
- // board.json as-is, while these two are only ever fed to assessBeat(),
594
+ // board.json as-is, while these are only ever fed to assessBeat(),
590
595
  // which already treats `undefined` and `null` identically — a `?? null`
591
596
  // here would cost a line to say nothing stall() does not already do.
592
597
  //
@@ -594,7 +599,7 @@ export function computeBoard(inputs) {
594
599
  // `ledgerState`'s own `?? "read"` default just above: every caller that
595
600
  // builds inputs by hand (every test in this suite) means a read that
596
601
  // succeeded, and only gather() — which now sets both — can say otherwise.
597
- liveness: stall(inputs.beat, inputs.ticked, tickets, pool, {
602
+ liveness: stall({ beat: inputs.beat, ticked: inputs.ticked, acted: inputs.acted, controller: inputs.controller }, tickets, pool, {
598
603
  ledgerOk: (ledger.state ?? "read") === "read",
599
604
  poolOk: inputs.poolOk ?? true,
600
605
  owner: inputs.stallOwner ?? NO_OWNER,
@@ -1,5 +1,5 @@
1
1
  // The heartbeat's shared state file, owned jointly by fleet-tick.mjs,
2
- // fleet-heartbeat.mjs and ledger.mjs rotate — so the path and the file's shape
2
+ // fleet-heartbeat.mjs and ledger.mjs — so the path and the file's shape
3
3
  // live here rather than in spellings that drift. Same rule arg.mjs states for
4
4
  // the digits guard: what travels between scripts is the rule, never a copy of
5
5
  // it.
@@ -35,12 +35,18 @@
35
35
  // when rotate wrote it — a mark older than `at` is the previous
36
36
  // run's. Removed, not left stale, by a rotate that finds no omp
37
37
  // ancestor.
38
+ // acted ledger.mjs only. When a ledger write last landed: every
39
+ // subcommand that writes the ledger stamps it, rotate in the same
40
+ // write as `controller`. A busy run writes its ledger on every
41
+ // dispatch and settle, far more often than it ticks, and never
42
+ // beats while there is work, so this is the mark that tells a busy
43
+ // run from a dead one.
38
44
  // seen fleet-tick only. {member: firstSeenAtMs}: the first tick that saw
39
45
  // each live ledger token, so a later tick can say how long a token
40
46
  // has stayed live — the ledger records that a member was dispatched,
41
47
  // never when. Pruned to the live set on every tick, so a settled
42
48
  // member's stamp does not outlive it. Carried, never written, by
43
- // the heartbeat and by rotate.
49
+ // the heartbeat and by ledger.mjs.
44
50
  //
45
51
  // One writer per key. A key two scripts wrote would need locking to be
46
52
  // correct, and none of them is in a position to hold one. That holds per key,
@@ -111,7 +117,7 @@ export function readState(path, name) {
111
117
  // to judge and nothing to report. A zeroed mark would date the beat to the
112
118
  // epoch and read as decades overdue on a run that has simply not started
113
119
  // one — the cry-wolf direction this key must never fail in.
114
- const fresh = { quiet: 0, elapsed: 0, digest: "", beat: null, ticked: null, controller: null, seen: {}, rest: {} };
120
+ const fresh = { quiet: 0, elapsed: 0, digest: "", beat: null, ticked: null, acted: null, controller: null, seen: {}, rest: {} };
115
121
  // The same state, flagged: the file EXISTS and could not be used. A caller
116
122
  // that acts on the absence of a record — ledger.mjs rotate — must tell this
117
123
  // from `fresh`, which is a run that has simply not written one yet.
@@ -178,7 +184,8 @@ export function readState(path, name) {
178
184
  };
179
185
  // `ticked` carries no interval — fleet-tick has no promise to fail, only an
180
186
  // occurrence — so it is a mark of one field, valid or absent, same "never
181
- // zero" rule as `mark` above for the same cry-wolf reason.
187
+ // zero" rule as `mark` above for the same cry-wolf reason. `acted` is the
188
+ // same shape for the same reason.
182
189
  const tick = (v) => {
183
190
  if (!isRecord(v)) return null;
184
191
  const at = num(v.at);
@@ -213,13 +220,14 @@ export function readState(path, name) {
213
220
  for (const [member, at] of Object.entries(v)) if (Number.isInteger(at) && at > 0) out[member] = at;
214
221
  return out;
215
222
  };
216
- const { quiet, elapsed, digest, beat, ticked, controller, seen, ...rest } = parsed;
223
+ const { quiet, elapsed, digest, beat, ticked, acted, controller, seen, ...rest } = parsed;
217
224
  return {
218
225
  quiet: num(quiet),
219
226
  elapsed: num(elapsed),
220
227
  digest: typeof digest === "string" ? digest : "",
221
228
  beat: mark(beat),
222
229
  ticked: tick(ticked),
230
+ acted: tick(acted),
223
231
  controller: ctl(controller),
224
232
  seen: stamps(seen),
225
233
  rest,
@@ -269,12 +277,13 @@ export function writeState(path, name, prev, patch) {
269
277
  // every hold and would otherwise erase fleet-tick's `ticked` the first
270
278
  // time a heartbeat lands after a busy stretch.
271
279
  ...(prev.ticked ? { ticked: prev.ticked } : {}),
272
- // And again for ledger.mjs rotate's `controller`, which both scripts
280
+ // And again for ledger.mjs's `controller` and `acted`, which both scripts
273
281
  // above would otherwise erase on their next write. A patch REMOVES a key
274
282
  // by naming it with the value `undefined`, which JSON.stringify drops —
275
283
  // the way rotate retires a record no omp ancestor replaces.
276
284
  ...(prev.controller ? { controller: prev.controller } : {}),
277
- // `seen` is fleet-tick's, and the heartbeat and rotate write this file
285
+ ...(prev.acted ? { acted: prev.acted } : {}),
286
+ // `seen` is fleet-tick's, and the heartbeat and ledger.mjs write this file
278
287
  // too: carried the same way, so neither erases the ages the tick keeps.
279
288
  ...(prev.seen && Object.keys(prev.seen).length > 0 ? { seen: prev.seen } : {}),
280
289
  ...patch,
@@ -330,55 +339,102 @@ export const BEAT_GRACE = 2;
330
339
  // here and imported there: every script that reads or writes the heartbeat
331
340
  // file already imports this module, and a magic "1200" typed twice is exactly
332
341
  // the drift the module header's "never a copy of it" rule exists to close.
333
- // It doubles as the freshness window for `ticked` below: fleet-tick's own
334
- // invocations carry no promised interval, so they are judged against the
335
- // worst-case gap this design ever tolerates when only the heartbeat is
336
- // beating, times the same grace multiplier as `beat`.
342
+ // It doubles as the freshness window for `ticked`, `acted` and the controller
343
+ // record below: none of them carries a promised interval, so each is judged
344
+ // against the worst-case gap this design ever tolerates when only the
345
+ // heartbeat is beating, times the same grace multiplier as `beat`.
337
346
  export const DEFAULT_CEILING_S = 1200;
338
347
 
339
- // What the mark says, as a value. `now` is a parameter so every caller's
348
+ const WINDOW_MS = DEFAULT_CEILING_S * 1000 * BEAT_GRACE;
349
+
350
+ // A mark older than the controller record's `at` was left by the run before
351
+ // the one that wrote the record — rotate replaced the record after that run's
352
+ // last beat, tick and ledger write — so it says nothing about this run. With
353
+ // no record, or a record with no `at`, nothing can be placed and every mark
354
+ // counts as this run's.
355
+ function thisRun(mark, controller) {
356
+ if (mark == null) return null;
357
+ return controller == null || controller.at === null || mark.at >= controller.at ? mark : null;
358
+ }
359
+
360
+ // What the marks say, as a value. `now` is a parameter so every caller's
340
361
  // verdict is deterministic and testable — the same reason compute-board.mjs
341
- // takes one.
362
+ // takes one. Only this run's marks decide it.
342
363
  //
343
- // none no mark. A fresh run, or one whose heartbeat never ran. NOT a
344
- // stall: there is no beat to have stopped, and reporting one here
345
- // would fire on every first tick and train the reader to ignore
346
- // the line.
364
+ // none no mark of this run and no controller record. A fresh run, or
365
+ // one whose heartbeat never ran. NOT a stall: there is no run to
366
+ // judge, and reporting one here would fire on every first tick
367
+ // and train the reader to ignore the line.
347
368
  // beating seen within the interval it promised, plus grace — OR a recent
348
- // `ticked` covers for it. `beat` is only refreshed when the queue
349
- // drains (SKILL.md: "beat when there is nothing to do"), so a
350
- // busy stretch that outlasts `beat`'s own grace window is not a dead
351
- // run; fleet-tick's own edge-triggered invocations are the other
352
- // liveness signal for exactly that case.
353
- // stopped a reason was recorded. Reported whatever the age, because a
354
- // recorded stop IS the end of the run and waiting for it to go
369
+ // tick, ledger write or controller record covers for it. `beat`
370
+ // is only refreshed when the queue drains (SKILL.md: "beat when
371
+ // there is nothing to do"), so a busy run can go its whole length
372
+ // without one; its ticks and ledger writes are the liveness
373
+ // signal for exactly that case.
374
+ // stopped this run recorded a reason. Reported whatever the age, because
375
+ // a recorded stop IS the end of the run and waiting for it to go
355
376
  // stale first would sit on the one report that knows its cause.
356
- // stale overdue with no reason recorded AND no recent tick either. The
357
- // abrupt death: a crashed harness, a closed terminal, an OOM
358
- // kill. Readers say exactly that rather than naming a cause they
359
- // cannot know.
360
- export function assessBeat({ beat, ticked, now }) {
361
- if (!beat) return { kind: "none" };
362
- // Clamped at zero rather than left signed: a mark from the future is a
363
- // clock that moved, not a beat that is minus-five-minutes overdue, and a
364
- // negative age formatted into the line reads as a defect in this code.
365
- const ageMs = Math.max(0, now - beat.at);
366
- const intervalMs = beat.interval * 1000;
367
- const seen = { at: beat.at, ageMs, intervalMs, overdueMs: Math.max(0, ageMs - intervalMs) };
368
- if (beat.stopped) return { ...seen, kind: "stopped", reason: beat.stopped };
369
- return { ...seen, kind: now > stallsAt({ beat, ticked }) ? "stale" : "beating", reason: null };
377
+ // stale overdue with no reason recorded AND no recent tick or ledger
378
+ // write either. The abrupt death: a crashed harness, a closed
379
+ // terminal, an OOM kill — or a live controller that stopped
380
+ // acting. Readers say exactly that rather than naming a cause
381
+ // they cannot know. A run with a controller record and no beat of
382
+ // its own is judged off its latest tick, ledger write or the
383
+ // record itself, so one that dies before it ever beats still goes
384
+ // stale; `beatless` marks that verdict, and `mark` names which.
385
+ // ended the PREVIOUS run's recorded stop, while this run has not
386
+ // ticked yet. It is that run's end, never this run's verdict:
387
+ // reported until this run's first tick, and never after.
388
+ export function assessBeat({ beat = null, ticked = null, acted = null, controller = null, now }) {
389
+ const ours = thisRun(beat, controller);
390
+ if (ours) {
391
+ const seen = since(ours.at, ours.interval * 1000, now);
392
+ if (ours.stopped) return { ...seen, kind: "stopped", reason: ours.stopped };
393
+ return { ...seen, kind: now > stallsAt({ beat, ticked, acted, controller }) ? "stale" : "beating", reason: null };
394
+ }
395
+ const marks = [
396
+ ["controller", controller?.at ?? null],
397
+ ["ticked", thisRun(ticked, controller)?.at ?? null],
398
+ ["acted", thisRun(acted, controller)?.at ?? null],
399
+ ].filter(([, at]) => at !== null);
400
+ // Without a record a mark-less beat is no run at all, whatever ticks or
401
+ // ledger writes some earlier run left beside it.
402
+ const judged = controller !== null && marks.length > 0;
403
+ if (judged && now > stallsAt({ beat, ticked, acted, controller })) {
404
+ const [mark, at] = marks.reduce((a, b) => (b[1] >= a[1] ? b : a));
405
+ return { ...since(at, WINDOW_MS, now), kind: "stale", reason: null, beatless: true, mark };
406
+ }
407
+ if (beat?.stopped && thisRun(ticked, controller) === null) {
408
+ return { ...since(beat.at, beat.interval * 1000, now), kind: "ended", reason: beat.stopped };
409
+ }
410
+ return judged ? { kind: "beating" } : { kind: "none" };
370
411
  }
371
412
 
372
- // The instant an unstopped mark turns `stale`: past `beat`'s own interval
373
- // plus grace AND past a tick's ceiling window plus grace, whichever ends
374
- // later. An absent `ticked` — a state file fleet-tick has not yet written to
375
- // — contributes nothing, which is `beat` alone deciding it. A present one
376
- // only ever postpones the stall; it can never manufacture a stale verdict
377
- // `beat` alone would not have reached. Exported for the readers that must
378
- // say WHEN a beating mark would stop counting, not only whether it has.
379
- export function stallsAt({ beat, ticked }) {
380
- const beatEnds = beat.at + beat.interval * 1000 * BEAT_GRACE;
381
- return ticked ? Math.max(beatEnds, ticked.at + DEFAULT_CEILING_S * 1000 * BEAT_GRACE) : beatEnds;
413
+ // A mark's age against the window it is judged by. Clamped at zero rather
414
+ // than left signed: a mark from the future is a clock that moved, not a beat
415
+ // that is minus-five-minutes overdue, and a negative age formatted into the
416
+ // line reads as a defect in this code.
417
+ function since(at, intervalMs, now) {
418
+ const ageMs = Math.max(0, now - at);
419
+ return { at, ageMs, intervalMs, overdueMs: Math.max(0, ageMs - intervalMs) };
420
+ }
421
+
422
+ // The instant this run turns `stale`: past `beat`'s own interval plus grace
423
+ // AND past the ceiling window plus grace of every tick, ledger write and
424
+ // controller record, whichever ends latest. Only this run's marks count — the
425
+ // previous run's are partitioned off by the record's `at` exactly as
426
+ // assessBeat() does. An absent mark contributes nothing, and null means this
427
+ // run has no mark at all. Exported for the readers that must say WHEN a
428
+ // beating run would stop counting, not only whether it has.
429
+ export function stallsAt({ beat = null, ticked = null, acted = null, controller = null }) {
430
+ const ends = [ticked, acted]
431
+ .map((m) => thisRun(m, controller))
432
+ .filter((m) => m !== null)
433
+ .map((m) => m.at + WINDOW_MS);
434
+ if (controller?.at != null) ends.push(controller.at + WINDOW_MS);
435
+ const ours = thisRun(beat, controller);
436
+ if (ours) ends.push(ours.at + ours.interval * 1000 * BEAT_GRACE);
437
+ return ends.length > 0 ? Math.max(...ends) : null;
382
438
  }
383
439
 
384
440
  // Whether this verdict is something to report. Exported because both readers
@@ -389,6 +445,13 @@ export function isStalled(verdict) {
389
445
  return verdict.kind === "stopped" || verdict.kind === "stale";
390
446
  }
391
447
 
448
+ // Whether this verdict prints at all: a stall, or the previous run's end. The
449
+ // second never reaches isStalled(), whose readers go on to count claims and
450
+ // judge the controller — both this run's, and neither the previous run's.
451
+ export function isReported(verdict) {
452
+ return isStalled(verdict) || verdict.kind === "ended";
453
+ }
454
+
392
455
  // Whole minutes, floored, because the numbers here are tens of minutes to
393
456
  // hours and a seconds-precise age in a stall line is false precision: the
394
457
  // mark is written once per hold, so its resolution is minutes anyway. That
@@ -439,26 +502,26 @@ function mins(ms) {
439
502
  //
440
503
  // A mark older than the record's `at` was left by the run BEFORE the one
441
504
  // that wrote the record: rotate replaced the record after that run's last
442
- // beat and tick. That is a run's first tick, and by then the record names
443
- // this run's own live controller, so judging it would call every previous
444
- // run alive. That stall reads the verdict rotate reached on the record it
445
- // replaced, `prior`: dead is a controller gone whose claims this run resumes,
446
- // ancestor is this same session's earlier run and so alive, none is unknown.
447
- // A mark written since the record — any later tick — is the recorded
448
- // controller's own, so the record itself is judged, through proc.mjs: dead or
449
- // a reused pid is gone and the NEXT run resumes; alive, or an ancestor of the
450
- // reader, is alive. A record with no `at` cannot place the mark and is
451
- // judged too.
505
+ // beat, tick and ledger write. When every mark is that old, the record names
506
+ // a controller that has left no mark yet, so judging it would call every
507
+ // previous run alive. That stall reads the verdict rotate reached on the
508
+ // record it replaced, `prior`: dead is a controller gone whose claims this run
509
+ // resumes, ancestor is this same session's earlier run and so alive, none is
510
+ // unknown. A mark written since the record — any later beat, tick or ledger
511
+ // write — is the recorded controller's own, so the record itself is judged,
512
+ // through proc.mjs: dead or a reused pid is gone and the NEXT run resumes;
513
+ // alive, or an ancestor of the reader, is alive. A record with no `at` cannot
514
+ // place the mark and is judged too.
452
515
  //
453
516
  // `judge` is replaceable for the same reason proc.mjs's process source is: a
454
517
  // reader under a live omp session would otherwise find that session. A
455
518
  // process table that cannot be read leaves the stall unattributed, said on
456
519
  // stderr, rather than calling a controller nobody could judge dead or alive.
457
520
  export const NO_OWNER = Object.freeze({ kind: "none" });
458
- export function stallOwner({ controller, beat, ticked }, name, judge = judgeRecord) {
521
+ export function stallOwner({ controller, beat = null, ticked = null, acted = null }, name, judge = judgeRecord) {
459
522
  if (!controller) return NO_OWNER;
460
- const before = (mark) => mark === null || mark.at < controller.at;
461
- if (controller.at !== null && before(beat) && before(ticked)) {
523
+ const before = (mark) => thisRun(mark, controller) === null;
524
+ if (controller.at !== null && before(beat) && before(ticked) && before(acted)) {
462
525
  if (controller.prior === "dead") return { kind: "gone", resumes: "this run" };
463
526
  if (controller.prior === "ancestor") return { kind: "alive", pid: null };
464
527
  return NO_OWNER;
@@ -476,9 +539,13 @@ export function stallOwner({ controller, beat, ticked }, name, judge = judgeReco
476
539
  return NO_OWNER;
477
540
  }
478
541
 
479
- // The report. Returns null for anything isStalled() rejects, so a caller that
542
+ // The report. Returns null for anything isReported() rejects, so a caller that
480
543
  // skipped the predicate cannot print a stall for a healthy run by accident.
481
544
  //
545
+ // The previous run's end is one line of its own and carries nothing of this
546
+ // run's: no claims, no supply, no owner. It names when that run stopped and
547
+ // the reason it recorded, and nothing more.
548
+ //
482
549
  // What it names is the acceptance criterion itself: a bare "stale" tells the
483
550
  // maintainer nothing they can act on, so the line carries when the beat was
484
551
  // last seen, how overdue that is against the interval it PROMISED, whether a
@@ -498,22 +565,32 @@ export function stallOwner({ controller, beat, ticked }, name, judge = judgeReco
498
565
  // cockpit's own cards) and either source can fail, and "0 claimed" off a
499
566
  // failed read is the same lie as "supply 0" off one — it says the night cost
500
567
  // nothing when it may have stranded the whole pool.
501
- export function stallReport(verdict, { claimed, supply, owner = NO_OWNER }) {
502
- if (!isStalled(verdict)) return null;
568
+ export function stallReport(verdict, { claimed, supply, owner = NO_OWNER } = {}) {
569
+ if (!isReported(verdict)) return null;
570
+ const when = (at) => new Date(at).toISOString();
571
+ if (verdict.kind === "ended") {
572
+ return `previous run stopped deliberately at ${when(verdict.at)} (${mins(verdict.ageMs)} ago) — recorded reason: ${verdict.reason}`;
573
+ }
503
574
  const n = (v) => (v == null ? "unknown" : String(v));
504
575
  // Present tense for the deliberate stop and past for the abrupt one, because
505
576
  // that is the actual difference: one run said why it was going, the other
506
577
  // was cut off mid-beat and left nothing behind to ask.
507
578
  const why = verdict.kind === "stopped"
508
579
  ? `stopped deliberately — recorded reason: ${verdict.reason}`
509
- : "the beat stopped without a recorded reason";
580
+ : verdict.beatless ? "no beat, tick or ledger write since, and no recorded reason" : "the beat stopped without a recorded reason";
510
581
  const head = owner.kind === "alive"
511
582
  ? `heartbeat not beating, controller alive (${owner.pid === null ? "this session's earlier run" : `pid ${owner.pid}`})`
512
583
  : owner.kind === "gone" ? "heartbeat STALLED, controller gone" : "heartbeat STALLED";
513
584
  const tail = owner.kind === "gone"
514
585
  ? `; ${owner.resumes} resumes its claims without a PR, and its open PRs return through the fold-in`
515
586
  : "";
516
- return `${head}: last beat ${new Date(verdict.at).toISOString()}`
517
- + ` (${mins(verdict.ageMs)} ago, ${mins(verdict.overdueMs)} past the ${mins(verdict.intervalMs)} interval it promised)`
518
- + ` — ${why}; ${n(claimed)} ticket(s) claimed and in flight, pool supply ${n(supply)}${tail}`;
587
+ const seen = verdict.beatless
588
+ ? `no beat this run, last ${MARK_NAMES[verdict.mark]} ${when(verdict.at)}`
589
+ + ` (${mins(verdict.ageMs)} ago, ${mins(verdict.overdueMs)} past the ${mins(verdict.intervalMs)} window)`
590
+ : `last beat ${when(verdict.at)}`
591
+ + ` (${mins(verdict.ageMs)} ago, ${mins(verdict.overdueMs)} past the ${mins(verdict.intervalMs)} interval it promised)`;
592
+ return `${head}: ${seen} — ${why}; ${n(claimed)} ticket(s) claimed and in flight, pool supply ${n(supply)}${tail}`;
519
593
  }
594
+
595
+ // What a beatless stall names as this run's last sign of life.
596
+ const MARK_NAMES = { controller: "controller record", ticked: "tick", acted: "ledger write" };
@@ -12,7 +12,9 @@
12
12
  // actual/target and a named ACTION per role — `PULL #<n> #<n>`, `DISPATCH
13
13
  // fix-pr PR#<pr>`, `DISPATCH review PR#<pr>`, `DISPATCH merge-bot`, `HOLD (…)`,
14
14
  // `REFRESHED shortlist: …` — with run-team's guards applied in code rather than
15
- // recalled from its table. You cannot forget what a script prints at you.
15
+ // recalled from its table. You cannot forget what a script prints at you. A
16
+ // drained run with nothing left in flight ends on one more: `ACTION: run over`,
17
+ // naming the stop the controller records last.
16
18
  //
17
19
  // WHERE THE NUMBERS COME FROM — nobody states them:
18
20
  //
@@ -992,7 +994,7 @@ import { parseArgs } from "node:util";
992
994
  import { makeDie, isDigits } from "./arg.mjs";
993
995
  import { gitEnv } from "./git-env.mjs";
994
996
  import { fleetFile, FleetDirUnresolvable } from "./fleet-dir.mjs";
995
- import { statePath, readState, writeState, assessBeat, isStalled, stallReport, stallOwner } from "./fleet-state.mjs";
997
+ import { statePath, readState, writeState, assessBeat, isStalled, isReported, stallReport, stallOwner } from "./fleet-state.mjs";
996
998
  import { checkMainCheckout, describe } from "./main-checkout.mjs";
997
999
 
998
1000
  const NAME = "fleet-tick";
@@ -1009,6 +1011,9 @@ const MERGEABLE_STATES = ["MERGEABLE", "CONFLICTING", "UNKNOWN"];
1009
1011
  // cap, because refusing would suppress the one announcement a dead run leaves
1010
1012
  // behind. So it discloses instead — the line says the count is a floor.
1011
1013
  const CLAIMED_LIMIT = 200;
1014
+ // The tick's last line once a draining run has nothing left in flight. The
1015
+ // tick cannot write the stop — `beat` is fleet-heartbeat's — so it names it.
1016
+ export const RUN_OVER = `ACTION: run over → fleet-heartbeat.mjs --stop "<reason>"`;
1012
1017
  const MAX_BUFFER = 64 * 1024 * 1024;
1013
1018
  // Every gh spawn below is killed at this bound, so a gh that never answers is
1014
1019
  // a failed read on that call's own failure path — the open-PR read refuses the
@@ -1324,9 +1329,24 @@ function main() {
1324
1329
  // Whose stall it is comes off the controller record, judged only once
1325
1330
  // there is a stall to name, so a healthy tick reads no process table.
1326
1331
  const priorState = readState(path, NAME);
1327
- const verdict = assessBeat({ beat: priorState.beat, ticked: priorState.ticked, now: Date.now() });
1328
- if (isStalled(verdict)) {
1329
- console.log(stallReport(verdict, { claimed: claimed(), supply: current.scanned, owner: stallOwner(priorState, NAME) }));
1332
+ const tickedAt = Date.now();
1333
+ const { beat, ticked, acted, controller } = priorState;
1334
+ const verdict = assessBeat({ beat, ticked, acted, controller, now: tickedAt });
1335
+ // `ticked`, fleet-tick's own liveness key, written here and on its own,
1336
+ // ahead of the gh and ledger reads that can refuse this tick: a refused
1337
+ // tick was still invoked, and a run whose every tick refuses on a gh outage
1338
+ // is a run working, not a dead one. Only the usage refusals above leave no
1339
+ // mark — they never reached the state file — and a file that exists but
1340
+ // could not be read is left as it is, as ledger.mjs's `acted` stamp leaves
1341
+ // it: rotate refuses on such a file because a record it may hold cannot be
1342
+ // known, and a write here would replace that record with a fresh one.
1343
+ // Judged before it is written, so the previous run's end prints on this
1344
+ // run's first tick and on no later one.
1345
+ if (!priorState.degraded) writeState(path, NAME, priorState, { ticked: { at: tickedAt } });
1346
+ if (isReported(verdict)) {
1347
+ console.log(stallReport(verdict, isStalled(verdict)
1348
+ ? { claimed: claimed(), supply: current.scanned, owner: stallOwner(priorState, NAME) }
1349
+ : {}));
1330
1350
  }
1331
1351
 
1332
1352
  // Every read happens before anything prints: a partial tick is worse than no
@@ -1392,19 +1412,26 @@ function main() {
1392
1412
  const seen = stampSeen({ live: run.live, prev: prev.seen, controller: prev.controller, now });
1393
1413
  const stale = livenessLines({ live: run.live, seen, thresholds: liveness, now });
1394
1414
  const lines = [...(said === null ? [] : [said]), ...stale.map((s) => s.line), ...formatLines(rows)];
1415
+ // The run is over once it is draining and nothing is left in flight: no
1416
+ // open PR and no implementer still holding a claim. Printed last and read
1417
+ // as actionable, so it never folds away: the stop it names is the run's
1418
+ // last act, and only the controller can write it.
1419
+ const over = run.draining !== null && prs.length === 0 && run.implLive === 0;
1420
+ if (over) lines.push(RUN_OVER);
1395
1421
 
1396
1422
  // The back-off streak and the fold digest, written on EVERY tick: an edge
1397
1423
  // tick that dispatched is the clearest possible "there is work here". Digest,
1398
1424
  // not the lines themselves, so the file stays readable.
1399
1425
  const digest = createHash("sha256").update(lines.join("\n")).digest("hex");
1400
- const acts = actionable(rows);
1401
- // `ticked`, fleet-tick's own liveness key: written on every tick that gets
1402
- // past its refusals — a refused tick (exit 2) writes none; this tick
1403
- // reaching here IS the occurrence. `seen` is pruned to the live set on the
1404
- // same write; an empty one removes the key.
1426
+ const acts = actionable(rows) || over;
1427
+ // `seen` is pruned to the live set on the same write; an empty one removes
1428
+ // the key. `ticked` was written at the top of the tick, unless the file
1429
+ // could not be read then: this write replaces such a file, so it carries
1430
+ // the stamp the top of the tick left out.
1405
1431
  writeState(path, NAME, prev, {
1406
- quiet: acts ? 0 : prev.quiet + 1, digest, ticked: { at: now },
1432
+ quiet: acts ? 0 : prev.quiet + 1, digest,
1407
1433
  seen: Object.keys(seen).length > 0 ? seen : undefined,
1434
+ ...(priorState.degraded ? { ticked: { at: tickedAt } } : {}),
1408
1435
  });
1409
1436
 
1410
1437
  // Fold only when BOTH hold: nothing to act on, and nothing new to say. A
@@ -32,6 +32,12 @@
32
32
  // `filed` kept 2-5 of 8 rows — and the lock is what lets a second kind of
33
33
  // writer exist at all. The reasoning for each part of the lock lives at
34
34
  // acquireLock() below.
35
+ //
36
+ // Every write subcommand also stamps `acted` in the heartbeat file beside the
37
+ // ledger whenever its call ends without a refusal — rotate in the same write
38
+ // as its controller record. A run's ledger writes are its densest sign of
39
+ // life, so the stall readers judge a busy run by them (fleet-state.mjs has
40
+ // the rule).
35
41
 
36
42
  import { readFileSync, writeFileSync, mkdirSync, existsSync, renameSync, openSync, writeSync, closeSync, unlinkSync, linkSync } from "node:fs";
37
43
  import { dirname, resolve, join, basename } from "node:path";
@@ -929,6 +935,18 @@ if (cmd === "read") {
929
935
  die(`unknown subcommand '${cmd}' — expected row, filed, ruled, check, read, dispatch, settle, review, drain or rotate`);
930
936
  }
931
937
 
938
+ // Reached only by an arm that did not die, so every write subcommand that did
939
+ // not refuse is stamped — one that found nothing to change (a repeat `drain`,
940
+ // a duplicate `filed`) is still the controller acting. Still under the write
941
+ // lock, so two ledger calls never race on the key. A heartbeat file that
942
+ // exists and cannot be read is left as it is for the readers that refuse on
943
+ // it; readState() has already said why.
944
+ if (WRITE_COMMANDS.has(cmd) && cmd !== "rotate") {
945
+ const beatFile = stateFileIn(dirname(file));
946
+ const state = readState(beatFile, NAME);
947
+ if (!state.degraded) writeState(beatFile, NAME, state, { acted: { at: Date.now() } });
948
+ }
949
+
932
950
  // `check` alone of the subcommands leaves its arm early: the already-filed
933
951
  // answer is complete before the near-miss ranking and the tracker query below
934
952
  // it, whose results that answer would only discard. A function is what makes
@@ -1570,9 +1588,10 @@ function runCheck() {
1570
1588
  // and keep dispatching — and rotating under it would split its run across
1571
1589
  // two ledgers. A dead one, a reused pid, or this session's own earlier run
1572
1590
  // allows the rotation. With no record the guard falls back to the mark, read
1573
- // through fleet-state.mjs's own reader and verdict: still beating refuses; a
1574
- // stalled mark, a recorded stop, or no mark at all allows the rotation, and
1575
- // that fallback proves "not beating", never "dead".
1591
+ // through fleet-state.mjs's own reader and verdict: still beating — a beat
1592
+ // inside its window, or past it with a tick or ledger write inside
1593
+ // theirs — refuses; a stalled mark, a recorded stop, or no beat at all allows
1594
+ // the rotation, and that fallback proves "not beating", never "dead".
1576
1595
  //
1577
1596
  // A heartbeat file that exists but cannot be read or parsed gets neither
1578
1597
  // verdict: a record it may hold cannot be known, so rotate refuses rather
@@ -1582,7 +1601,8 @@ function runCheck() {
1582
1601
  // record with this run's own, carrying the verdict as `prior` — the next
1583
1602
  // step reads `prior`, because the new record always names this run's own
1584
1603
  // live ancestor — and the time of this write as `at`, so a reader can tell a
1585
- // mark the previous run left from one written since. No omp ancestor removes
1604
+ // mark the previous run left from one written since. The same write stamps
1605
+ // `acted` at that time, as every ledger write does. No omp ancestor removes
1586
1606
  // the key: an older run's record left in place would be judged again, as
1587
1607
  // dead, by a run that cannot know whether the run after it is still live.
1588
1608
  // This run's own record is computed before anything moves too, so a process
@@ -1591,7 +1611,7 @@ function runCheck() {
1591
1611
  function runRotate() {
1592
1612
  const beatFile = stateFileIn(dirname(file));
1593
1613
  const state = readState(beatFile, NAME);
1594
- const { beat, ticked, controller } = state;
1614
+ const { beat, ticked, acted, controller } = state;
1595
1615
  // A file that exists and could not be used holds no record we can know of:
1596
1616
  // reading it as "no record" would let a live controller's ledger be moved
1597
1617
  // by exactly the fault that hid its record.
@@ -1612,10 +1632,14 @@ function runRotate() {
1612
1632
  die(`refusing to rotate ${file}: the controller that owns it is alive — controller pid ${controller.pid} (started ${controller.lstart}) recorded in ${beatFile} is alive and not an ancestor of this run, whatever its heartbeat mark says; one controller per checkout — end that omp session, or start the run from inside it`);
1613
1633
  }
1614
1634
  // The fallback guards a ledger, as it always has: with none to move there
1615
- // is nothing for a beating mark to protect.
1635
+ // is nothing for a beating mark to protect. A ledger write is the
1636
+ // controller acting, and counts the way a beat does.
1616
1637
  const exists = existsSync(file);
1617
- if (prior === "none" && exists && assessBeat({ beat, ticked, now: Date.now() }).kind === "beating") {
1618
- die(`refusing to rotate ${file}: the controller that owns it is still beating — last beat ${new Date(beat.at).toISOString()} in ${beatFile}; it counts as stalled after ${new Date(stallsAt({ beat, ticked })).toISOString()}`);
1638
+ if (prior === "none" && exists && assessBeat({ beat, ticked, acted, now: Date.now() }).kind === "beating") {
1639
+ const [what, mark] = [["beat", beat], ["tick", ticked], ["ledger write", acted]]
1640
+ .filter(([, m]) => m !== null)
1641
+ .reduce((a, b) => (b[1].at > a[1].at ? b : a));
1642
+ die(`refusing to rotate ${file}: the controller that owns it is still beating or acting — last ${what} ${new Date(mark.at).toISOString()} in ${beatFile}; it counts as stalled after ${new Date(stallsAt({ beat, ticked, acted })).toISOString()}`);
1619
1643
  }
1620
1644
  // Read before anything moves: a ledger whose live tokens cannot be read is
1621
1645
  // refused with nothing moved, never archived with those tokens dropped.
@@ -1670,8 +1694,9 @@ function runRotate() {
1670
1694
  // Re-read rather than reuse the read above: the heartbeat file has other
1671
1695
  // writers, and a patch built from an older view would undo what they wrote
1672
1696
  // since.
1673
- const record = ours === null ? undefined : { ...ours, prior, at: Date.now() };
1674
- if (!writeState(beatFile, NAME, readState(beatFile, NAME), { controller: record })) {
1697
+ const now = Date.now();
1698
+ const record = ours === null ? undefined : { ...ours, prior, at: now };
1699
+ if (!writeState(beatFile, NAME, readState(beatFile, NAME), { controller: record, acted: { at: now } })) {
1675
1700
  die(`${archive === null ? "found no ledger" : `rotated ${file} -> ${archive}`} but could not record this run's controller in ${beatFile} — the next run judges whatever record that file still holds, and none if it cannot be read`);
1676
1701
  }
1677
1702
  console.log(JSON.stringify({
@@ -441,9 +441,10 @@ phase, or in any later one, asks the maintainer which tickets to take.
441
441
  controller can stop beating and keep dispatching. A dead one, a pid
442
442
  reused by another process, or this session's own earlier run lets it
443
443
  rotate. With no record the heartbeat mark decides, as the only guard
444
- there is: still beating refuses, naming the last beat and when it would
445
- count as stalled; a stalled mark, a recorded `--stop`, or no mark at all
446
- lets it rotate — and that fallback proves "not beating", never "dead", and
444
+ there is: still beating or acting refuses, naming the latest beat, tick or
445
+ ledger write and when it would count as stalled; a stalled mark, a
446
+ recorded `--stop`, or no beat at all lets it rotate — and that fallback
447
+ proves "not beating", never "dead", and
447
448
  guards a ledger only: with none to move the mark is not consulted. With no
448
449
  ledger the record is still judged, so a live recorded controller refuses
449
450
  even then. It also refuses, nothing moved, when the stamped archive name
@@ -454,10 +455,12 @@ phase, or in any later one, asks the maintainer which tickets to take.
454
455
  above it, its start time, as `prior` the verdict it reached on the
455
456
  record it found (`dead`, `ancestor` or `none`), and as `at` when it wrote
456
457
  it — or to remove the key when
457
- no `omp` process is above it. That write can fail, and then exit 2 follows
458
+ no `omp` process is above it. The same write stamps `acted`, the time of
459
+ this run's latest ledger write, which every write subcommand stamps from
460
+ then on. That write can fail, and then exit 2 follows
458
461
  a move: `rotated <file> -> <archive>` on stderr, then the refusal that the
459
462
  record could not be written. Run the rotation before this run's own first
460
- tick or beat, either of which writes the mark the fallback reads. Older
463
+ tick, beat or ledger write, each of which writes a mark the fallback reads. Older
461
464
  archives, hand-named ones included, are left alone.
462
465
 
463
466
  **Derive the Recipe — the Recipe derivation step — before the fold-in below
@@ -1671,7 +1674,7 @@ ticket's line — never a hand edit.
1671
1674
  | Label seen (persistent Monitor) | nothing to record |
1672
1675
  | CI run terminal | `ci=<run-id>:<attempt>:<conclusion>` on the row; then the finisher gate (below) |
1673
1676
  | Merge-bot pass report | `held-behind:#<lower>` rows; `ledger.mjs settle merge-bot-<n>=done`; `reap.sh --apply`. (The bot may also have written `conflict-hold:#<pr>` onto a held PR's own row earlier in this same pass, before reporting — a token the bot wrote in this pass, never one `reap.sh` writes.) |
1674
- | Drain | `ledger.mjs drain "<reason>"`; release the claims (below); `settle impl-<N>=released` |
1677
+ | Drain | `ledger.mjs drain "<reason>"`; release the claims (below); `settle impl-<N>=released`; then the rest of **End of run** (below), `--stop` last |
1675
1678
  | Heartbeat | nothing to record |
1676
1679
 
1677
1680
  Then run the tick, act on every line it prints, and arm the beat. What a wake
@@ -1739,6 +1742,19 @@ owes beyond its row:
1739
1742
  claims. The marker is on disk, so the tick holds the implementer row at
1740
1743
  `HOLD (draining)` from then on — for a replacement controller too — and keeps
1741
1744
  the review, fix-applier and merge rows firing until the open PRs merge.
1745
+ The drain starts the end of the run; it does not end it. **End of run**, in
1746
+ this order:
1747
+ 1. `ledger.mjs drain "<reason>"`.
1748
+ 2. Release every claim that never became a PR, settling each
1749
+ `impl-<N>=released`.
1750
+ 3. Act on the review, fix-applier and merge rows until no PR is open. Once
1751
+ the run is draining with no open PR and no live implementer, the tick's
1752
+ last line is `ACTION: run over → fleet-heartbeat.mjs --stop "<reason>"`.
1753
+ 4. Scrape the run's member facts, append every `tier-outcomes.mjs` row, and
1754
+ run `metrics-refresh.mjs` (above).
1755
+ 5. Commit the metrics files and open their PR (above).
1756
+ 6. `~/.fleet/bin/fleet-run fleet-heartbeat.mjs --stop "<reason>"` — last,
1757
+ because every reader takes a recorded stop as the end of the run.
1742
1758
  - **Monitor: CI run completes** → bind it (`ci-state.mjs --pr <N>`) and **record
1743
1759
  it as `ci=<run-id>:<attempt>:<conclusion>`, not just "CI green"** — in the
1744
1760
  ticket's ledger row, which outlives your context. `row` **replaces the whole
@@ -1997,34 +2013,41 @@ than being detected, and nothing else can write the reason afterwards:
1997
2013
  ~/.fleet/bin/fleet-run fleet-heartbeat.mjs --stop "budget exhausted"
1998
2014
  ```
1999
2015
 
2000
- It holds nothing and returns at once. Use it when the budget goes, when you
2001
- can see the context ceiling coming, or when you drain the run deliberately.
2016
+ It holds nothing and returns at once. Use it when the budget goes, or when
2017
+ you can see the context ceiling coming. A drained run records it as the last
2018
+ step of **End of run** (above), never at the drain itself.
2002
2019
  Skipping it is not fatal — the next reader still reports the stop, it just
2003
2020
  reports that the beat stopped *without a recorded reason*, which is all it can
2004
2021
  honestly say about a death that named nothing.
2005
2022
 
2006
2023
  **A stall line names whose stall it is.** `fleet-tick`
2007
- prints it before its own rows when the mark it finds is overdue against the
2008
- interval that mark recorded, and the cockpit shows the same line while it is
2009
- up. It names when the beat was last seen, how overdue that is, how many
2024
+ prints it before its own rows when this run has gone quiet: its beat overdue
2025
+ against the interval it recorded with no tick or ledger write inside the
2026
+ 40-minute window either — or, with no beat yet, its latest tick, ledger write
2027
+ or controller record past that window. A beat, tick or ledger write older than
2028
+ the controller record rotate wrote is the previous run's and counts for
2029
+ nothing. The cockpit shows the same line while it is up. It names when this
2030
+ run was last seen, how overdue that is, how many
2010
2031
  tickets are still claimed and in flight, and how much supply the last read found
2011
2032
  — because the claimed ones keep the `in-progress` label, the candidate scan
2012
2033
  EXCLUDES that label, and those tickets are therefore invisible to your own
2013
2034
  shortlist and to the maintainer's. Its opening words come off the controller
2014
- record. Your first tick's line is the PREVIOUS run's stall, read off the
2015
- `prior` rotate recorded; any later line is judged off the record itself:
2035
+ record:
2016
2036
 
2017
2037
  - `heartbeat STALLED, controller gone` — the run that stranded them is dead.
2018
- Its claims without a PR come back through phase 0's stranded-claim step —
2019
- this run's on your first tick, the next run's otherwise — and its open PRs
2020
- through the fold-in.
2038
+ Its claims without a PR come back through the next run's phase 0
2039
+ stranded-claim step, and its open PRs through the fold-in.
2021
2040
  - `heartbeat not beating, controller alive (pid N)`, or `(this session's
2022
2041
  earlier run)` — a controller that stopped beating and kept going. Nothing
2023
- died, so nothing is resumed; on a later tick of your own, that controller
2024
- is you, and the fix is to beat.
2042
+ died, so nothing is resumed; when that controller is you, the fix is to
2043
+ tick on every wake and beat when there is nothing to do.
2025
2044
  - `heartbeat STALLED` alone — no record says whether that controller lives.
2026
2045
  The line is detection only.
2027
2046
 
2047
+ Your first tick may print `previous run stopped deliberately at <time> (<age>
2048
+ ago) — recorded reason: <reason>` instead: the stop the previous run recorded.
2049
+ It is that run's end, not a verdict on yours, and no later tick prints it.
2050
+
2028
2051
  Read the line and carry on; nothing in the fleet releases, relabels or
2029
2052
  reaps the stranded tickets for you, and nothing restarts the run that stranded them.
2030
2053