@feigi/fleet-ctl 3.42.1 → 3.43.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@feigi/fleet-ctl",
3
- "version": "3.42.1",
3
+ "version": "3.43.0",
4
4
  "description": "Agent fleet: run-team controller, merge bot, PR reviewer, and the ticket pipeline they share",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -199,7 +199,8 @@ testcmd=$("$script_dir/derive-testcmd.sh" . test --at origin/main 2>&1) || die "
199
199
  echo " test entrypoint → $testcmd" >&2
200
200
 
201
201
  # The Test entrypoint is a shell COMMAND, not an argv: the runner hands it to
202
- # `sh -c`, so a compound one (`cd sub && make test`) keeps its meaning, with the
202
+ # the suite-slot wrapper, which hands it to `sh -c`, so a compound one
203
+ # (`cd sub && make test`) keeps its meaning, with the
203
204
  # runner's own arguments appended as "$@". It reaches the runner's source
204
205
  # single-quoted — each embedded `'` closed, escaped and reopened — so nothing
205
206
  # in it expands at write time. `LC_ALL=C` so sed reads the command as bytes.
@@ -385,12 +386,16 @@ else
385
386
  # times in one run — including by an agent whose parent was briefed but did
386
387
  # not pass them down. Anyone who finds the worktree finds the runner. The
387
388
  # isolation triple is per claim and language-neutral; everything else is the
388
- # Test entrypoint's own business, so the runner is a thin exec of it.
389
+ # Test entrypoint's own business, so the runner is a thin exec of it — through
390
+ # the suite-slot wrapper, which bounds how many full runs the host executes
391
+ # at once: a run with no arguments takes a slot, a narrowed one passes
392
+ # through. By the Resolver's own door, never a path frozen into a runner
393
+ # that is written once and never rewritten.
389
394
  cat > "$runner" <<SH
390
395
  #!/bin/sh
391
396
  # agent-test template: $tmpl_stamp
392
397
  export TEST_COMPOSE_PROJECT=ab-$issue TEST_POSTGRES_PORT=$pg TEST_OLLAMA_PORT=$ollama
393
- exec sh -c '$quoted "\$@"' agent-test "\$@"
398
+ exec "\$HOME/.fleet/bin/fleet-run" suite-slot.mjs '$quoted' "\$@"
394
399
  SH
395
400
  chmod +x "$runner"
396
401
  excl="$(git rev-parse --git-common-dir)/info/exclude"
@@ -1240,14 +1240,14 @@ function premiseState(noun, target, label) {
1240
1240
  // An Exclusion whose premise sends the tick to a refresh. Lifted, for a
1241
1241
  // ticket the current file does not already carry (if it does, the refresh
1242
1242
  // that lifted it has run): a `behind-pr:#M` reads off the open list already
1243
- // in hand; a `behind-issue:#M` costs one `gh issue view`; a branch-named
1244
- // `behind-pr` costs one `gh pr view`, whose MERGED or CLOSED lifts it —
1245
- // before its PR exists, "no open PR" is not "merged". Standing, for a
1246
- // branch-named `behind-pr` whose ticket the current file still carries: the
1247
- // file may predate the exclusion, and only the refresh's own probe can drop
1248
- // the ticket from it, so any answer short of MERGED or CLOSED, an
1249
- // unanswered probe included, asks for that refresh. An unanswered probe
1250
- // never lifts an exclusion.
1243
+ // in hand; a numeric `behind-issue:#M` costs one `gh issue view`; a
1244
+ // branch-named `behind-pr` costs one `gh pr view`, whose MERGED or CLOSED
1245
+ // lifts it — before its PR exists, "no open PR" is not "merged". Standing,
1246
+ // for a branch-named `behind-pr` or a numeric `behind-issue:#M` whose ticket
1247
+ // the current file still carries: the file may predate the exclusion, and
1248
+ // only the refresh's own probe can drop the ticket from it, so any answer
1249
+ // short of MERGED or CLOSED, an unanswered probe included, asks for that
1250
+ // refresh. An unanswered probe never lifts an exclusion.
1251
1251
  function liftedPremise(excluded, entries, prs) {
1252
1252
  const inFile = new Set(entries.map((e) => e.n));
1253
1253
  const open = new Set(prs.map((p) => p.number));
@@ -1260,12 +1260,15 @@ function liftedPremise(excluded, entries, prs) {
1260
1260
  if (!lifted && inFile.has(n)) return `behind-pr:#${target} not merged, #${n} still on the shortlist`;
1261
1261
  continue;
1262
1262
  }
1263
- if (inFile.has(n) || !isDigits(target)) continue;
1264
- if (kind === "pr" && !open.has(Number(target))) return `behind-pr:#${target} no longer open`;
1265
- if (kind === "issue") {
1263
+ if (kind === "issue" && isDigits(target)) {
1266
1264
  const st = premiseState("issue", target, `behind-issue:#${target}`);
1267
- if (st === "CLOSED" || st === "MERGED") return `behind-issue:#${target} ${st}`;
1265
+ const lifted = st === "CLOSED" || st === "MERGED";
1266
+ if (lifted && !inFile.has(n)) return `behind-issue:#${target} ${st}`;
1267
+ if (!lifted && inFile.has(n)) return `behind-issue:#${target} not closed, #${n} still on the shortlist`;
1268
+ continue;
1268
1269
  }
1270
+ if (inFile.has(n) || !isDigits(target)) continue;
1271
+ if (kind === "pr" && !open.has(Number(target))) return `behind-pr:#${target} no longer open`;
1269
1272
  }
1270
1273
  }
1271
1274
  return null;
@@ -1409,7 +1412,7 @@ function main() {
1409
1412
  }
1410
1413
 
1411
1414
  // Pull from the current file first; refresh when it is short, absent, an
1412
- // exclusion has lifted, or a branch-named one still names a ticket on it.
1415
+ // exclusion has lifted, or a branch-named or issue one still names a ticket on it.
1413
1416
  // A refresh that succeeds replaces the heads outright: the PULL is read off
1414
1417
  // what shortlist.mjs just probed, never merged with the file's own heads,
1415
1418
  // or a ticket the refresh dropped would be pulled anyway.
@@ -40,7 +40,7 @@
40
40
  // life, so the stall readers judge a busy run by them (fleet-state.mjs has
41
41
  // the rule).
42
42
 
43
- import { readFileSync, writeFileSync, mkdirSync, existsSync, renameSync, openSync, writeSync, closeSync, unlinkSync, linkSync } from "node:fs";
43
+ import { readFileSync, writeFileSync, mkdirSync, existsSync, renameSync, unlinkSync, linkSync } from "node:fs";
44
44
  import { dirname, resolve, join, basename } from "node:path";
45
45
  import { fileURLToPath } from "node:url";
46
46
  import { spawnSync, execFileSync } from "node:child_process";
@@ -49,7 +49,8 @@ import { fleetFile, FleetDirUnresolvable } from "./fleet-dir.mjs";
49
49
  import { parseMember, parseToken, memberTokens, nextMergeBot, nextRetry, liveOnLedger, retiredOnLedger, isRetirable, agentDefinition, CELL, tierValues, rowPr, rowWriteRefusal, isReviewToken, addedReviewToken, newSplitRow, splitRowClause, premisesOf, ROWS, DISPATCHED, FILED, RULED, DRAIN, RETIRED, headerRe, escapeText, unescapeText } from "./ledger-grammar.mjs";
50
50
  import { deriveRun, LedgerError, labelOffMember } from "./fleet-tick.mjs";
51
51
  import { stateFileIn, readState, writeState, assessBeat, stallsAt } from "./fleet-state.mjs";
52
- import { isDead, processSource, ancestry, controllerOf, judgeController, ProcUnreadable } from "./proc.mjs";
52
+ import { processSource, ancestry, controllerOf, judgeController, ProcUnreadable } from "./proc.mjs";
53
+ import { attemptLock, releaseLockFile, reapPathOf } from "./pid-lock.mjs";
53
54
 
54
55
  const NAME = "ledger";
55
56
  // The plugin's own agent definitions — the directory tier-check.mjs reads
@@ -530,55 +531,15 @@ function save(d) {
530
531
  // discards the other's row. Reads (`check`, `read`) take no lock — the rename
531
532
  // already hands them either the old file or the new one, never a torn one.
532
533
  //
533
- // `<file>.lock` is created O_EXCL and holds its owner's pid. Creating it and
534
- // writing the pid are two steps, so a waiter can read an empty or partial
535
- // lock in between: anything that is not a canonical positive integer counts
536
- // as a LIVE holder — keep waiting, and never hand it to process.kill, where
537
- // NaN is a TypeError crash rather than a wait. That empty or partial lock
538
- // never clears on its own, either: nothing revisits and finishes writing it
539
- // — the writer that would have finished died before the second step. Same
540
- // permanent wedge as a leftover reap lock below, same fix: the operator
541
- // removes it by hand once nothing is writing.
542
- //
543
- // A holder that died without releasing — SIGKILL always, and so does any
544
- // signal this file leaves unhandled (SIGQUIT, SIGABRT, a segfault: only a
545
- // normal exit and the three catchable signals below run the `exit` event
546
- // that releases it) — is taken over, but only under a second O_EXCL lock,
547
- // `<file>.lock.reap`, and only after RE-READING the lock under it, finding
548
- // the same dead pid, and RE-CHECKING that pid is still dead. Unlink-then-
549
- // create, or a bare rename, lets two waiters that both saw the dead pid each
550
- // remove the other's FRESH lock and write at once; the re-read is what makes
551
- // the takeover safe. The re-check narrows the one window the re-read leaves,
552
- // and the order the two run in is what narrows it: isDead(deadPid)
553
- // runs before the re-read, not after, so a live writer that takes the lock in
554
- // the gap between them is caught by the read that follows, not hidden behind
555
- // one taken before it arrived. Read first, and a recycled pid that came back
556
- // to life in that same gap would still read back as itself and still check
557
- // dead — the order this replaces would delete that live holder's fresh lock. A
558
- // recycled pid that became a new writer this same way — dead holder's lock
559
- // reaped by someone else, then re-created by the new owner of that very pid —
560
- // still writes the same content and would still pass the re-read alone, but
561
- // the isDead() ahead of it now catches that writer live, before the read ever
562
- // runs. Only that exact pid reused a second time, inside this narrower gap,
563
- // can still slip through. Under the reap lock no other waiter can replace the
564
- // lock's content outside that gap, so a pid still dead immediately before the
565
- // read that follows it is the dead holder, and a live one is left alone to be
566
- // waited on like any live holder. A reap lock is held for one liveness check,
567
- // one read and one unlink, and one left behind by any of those same abrupt
568
- // deaths in that window is deliberately NOT reclaimed: waiters time out naming
569
- // the holder, and the operator removes it. Fail closed.
570
- //
571
- // A dead holder whose pid was recycled reads as live until the timeout.
572
- // Accepted: an mtime/age heuristic instead would take over a live writer on a
573
- // slow disk, which is the row loss this lock exists to prevent.
574
- //
575
- // A live-looking holder can also be a zombie: a pid whose process exited but
576
- // whose parent never `wait()`-ed on it. The OS keeps that pid reserved until
577
- // reaped, so `process.kill(pid, 0)` still succeeds and isDead() reads it as
578
- // alive — but unlike the two wedges above, that clears on its own once the
579
- // pid is reaped (the parent's own exit is normally enough to trigger this),
580
- // and the next writer's death check then takes the lock over like any other
581
- // dead holder. No hand removal needed.
534
+ // `<file>.lock` is a pid-stamped lock and `<file>.lock.reap` its takeover
535
+ // lock; pid-lock.mjs owns the protocol — the O_EXCL create, what an empty or
536
+ // partial lock and a leftover reap lock mean, the dead-holder takeover and
537
+ // why it is ordered the way it is, the zombie case. What stays here is the
538
+ // ledger's own policy: how long to wait, and what to say on expiry. A holder
539
+ // that died without releasing is taken over, and only a normal exit and the
540
+ // three catchable signals below run the `exit` event that releases the lock —
541
+ // SIGKILL always, and any signal this file leaves unhandled (SIGQUIT,
542
+ // SIGABRT, a segfault), leave it for the takeover.
582
543
  //
583
544
  // The wait is bounded — 10 s by default — and the timeout is die(), exit 2,
584
545
  // having written nothing. `LEDGER_LOCK_TIMEOUT_MS` overrides it and, like
@@ -590,7 +551,7 @@ function save(d) {
590
551
  // same-run accidental duplicate was filed minutes apart, and a
591
552
  // claim/reserve subcommand closing that window was ruled out.
592
553
  const LOCK = `${file}.lock`;
593
- const REAP = `${LOCK}.reap`;
554
+ const REAP = reapPathOf(LOCK);
594
555
  const LOCK_WAIT_MS = (() => {
595
556
  const v = process.env.LEDGER_LOCK_TIMEOUT_MS;
596
557
  const ms = isDigits(String(v)) ? Number(v) : 0;
@@ -598,62 +559,10 @@ const LOCK_WAIT_MS = (() => {
598
559
  })();
599
560
  let lockHeld = false;
600
561
 
601
- const parsePid = (text) => (/^[1-9][0-9]*$/.test(text) ? Number(text) : null);
602
-
603
- // null when there is no lock to read — it was released between our failed
604
- // create and this read, and the next create attempt is the right response.
605
- function readLock() {
606
- try {
607
- return readFileSync(LOCK, "utf8");
608
- } catch (e) {
609
- if (e.code === "ENOENT") return null;
610
- die(`cannot read ${LOCK}: ${e.message}`);
611
- }
612
- }
613
-
614
- // true when the takeover ran (whatever it found under the reap lock), so the
615
- // caller retries the create at once; false when another waiter holds the
616
- // reap lock, so the caller waits like any other contended lock.
617
- function reapDeadHolder(deadPid) {
618
- let fd;
619
- try {
620
- fd = openSync(REAP, "wx");
621
- } catch (e) {
622
- if (e.code === "EEXIST") return false;
623
- die(`cannot create ${REAP}: ${e.message}`);
624
- }
625
- try {
626
- closeSync(fd);
627
- // Dead again, not just when the caller looked: that pid may have been
628
- // reused since, so isDead runs first, fresh, right here — before the
629
- // exact-match re-read, not after it. Exact match only: a waiter that
630
- // got here first may already have reaped the dead lock and taken a
631
- // fresh one of its own.
632
- if (isDead(deadPid) && readLock() === String(deadPid)) {
633
- // Already gone is already the outcome this call wants: an operator
634
- // clearing a wedged lock by hand between the read above and here must
635
- // not turn our own cleanup into an uncaught crash.
636
- try {
637
- unlinkSync(LOCK);
638
- } catch (e) {
639
- if (e.code !== "ENOENT") throw e;
640
- }
641
- }
642
- } finally {
643
- unlinkSync(REAP);
644
- }
645
- return true;
646
- }
647
-
648
562
  function releaseLock() {
649
563
  if (!lockHeld) return;
650
564
  lockHeld = false;
651
- try {
652
- if (readFileSync(LOCK, "utf8") === String(process.pid)) unlinkSync(LOCK);
653
- } catch {
654
- // Nothing to release, or nothing this process can do about it on its way
655
- // out: exit handlers cannot fail usefully.
656
- }
565
+ releaseLockFile(LOCK);
657
566
  }
658
567
 
659
568
  async function acquireLock() {
@@ -666,28 +575,12 @@ async function acquireLock() {
666
575
  for (const sig of ["SIGTERM", "SIGINT", "SIGHUP"]) process.on(sig, () => process.exit(2));
667
576
  const deadline = Date.now() + LOCK_WAIT_MS;
668
577
  for (;;) {
669
- let fd = null;
670
- try {
671
- fd = openSync(LOCK, "wx");
672
- } catch (e) {
673
- if (e.code !== "EEXIST") die(`cannot create ${LOCK}: ${e.message}`);
674
- }
675
- if (fd !== null) {
676
- try {
677
- writeSync(fd, String(process.pid));
678
- } catch (e) {
679
- // Ours, and empty or partial: every waiter reads it as live, and
680
- // releaseLock() will not match it, so it goes now or never.
681
- closeSync(fd);
682
- unlinkSync(LOCK);
683
- die(`cannot write ${LOCK}: ${e.message}`);
684
- }
685
- closeSync(fd);
578
+ const attempt = attemptLock(LOCK, die);
579
+ if (attempt.state === "taken") {
686
580
  lockHeld = true;
687
581
  return;
688
582
  }
689
- const holder = readLock();
690
- if (holder === null) {
583
+ if (attempt.state === "vanished") {
691
584
  // Ordinarily transient — released between our failed create above and
692
585
  // this read — so retrying the create at once beats sleeping first.
693
586
  // But that same shape recurs FOREVER if reading it fails for a reason
@@ -699,14 +592,13 @@ async function acquireLock() {
699
592
  if (Date.now() >= deadline) die(`cannot lock ${file} — ${LOCK} exists but cannot be read`);
700
593
  continue;
701
594
  }
702
- const pid = parsePid(holder);
703
- const dead = pid !== null && isDead(pid);
704
- if (dead && reapDeadHolder(pid)) continue;
595
+ if (attempt.state === "reaped") continue;
596
+ const { holder, pid, dead } = attempt;
705
597
  if (Date.now() >= deadline) {
706
598
  // `dead` here but still not taken over means this call's own reap
707
599
  // attempt found `${REAP}` already there right now — the same instant a
708
600
  // reaper SIGKILLed between its own create and unlink leaves behind
709
- // forever (see the write-lock header comment above `LOCK`/`REAP`), and
601
+ // forever (see pid-lock.mjs's header comment), and
710
602
  // indistinguishable at this instant from a live reaper mid-takeover.
711
603
  // Naming the pid alone points an operator at a process that no longer
712
604
  // exists; naming `${REAP}` is what lets them find the file
@@ -715,7 +607,7 @@ async function acquireLock() {
715
607
  if (dead) {
716
608
  die(`cannot lock ${file} — held by pid ${pid}, which is dead; ${REAP} is blocking its takeover — if nothing is currently writing, remove it by hand`);
717
609
  }
718
- // No pid at all is the other permanent wedge the header describes: an
610
+ // No pid at all is the other permanent wedge pid-lock.mjs's header describes: an
719
611
  // empty or partial `${LOCK}` whose writer died before writing its pid,
720
612
  // read as live forever by design. There is no process to point at, so
721
613
  // name `${LOCK}` itself — the file the same runbook says to remove.
@@ -0,0 +1,158 @@
1
+ // The pid-stamped lock file both the ledger's write lock and the suite slots
2
+ // are built on: one attempt to take it, the dead-holder takeover, and the
3
+ // release. Waiting is NOT here — how long to wait, and what to do on
4
+ // expiry, is the caller's, because the two callers differ (the ledger waits
5
+ // on one lock and dies; the suite wrapper cycles several and exits 75). Each
6
+ // caller passes its own die(), so every refusal speaks under its own NAME.
7
+ //
8
+ // `<lock>` is created O_EXCL and holds its owner's pid. Creating it and
9
+ // writing the pid are two steps, so a waiter can read an empty or partial
10
+ // lock in between: anything that is not a canonical positive integer counts
11
+ // as a LIVE holder — keep waiting, and never hand it to process.kill, where
12
+ // NaN is a TypeError crash rather than a wait. That empty or partial lock
13
+ // never clears on its own, either: nothing revisits and finishes writing it
14
+ // — the writer that would have finished died before the second step. Same
15
+ // permanent wedge as a leftover reap lock below, same fix: the operator
16
+ // removes it by hand once nothing is writing.
17
+ //
18
+ // A holder that died without releasing — SIGKILL always, and so does any
19
+ // signal its process leaves unhandled (SIGQUIT, SIGABRT, a segfault: only a
20
+ // normal exit and the signals a caller handles run the `exit` event that
21
+ // releases it) — is taken over, but only under a second O_EXCL lock,
22
+ // `<lock>.reap`, and only after RE-READING the lock under it, finding
23
+ // the same dead pid, and RE-CHECKING that pid is still dead. Unlink-then-
24
+ // create, or a bare rename, lets two waiters that both saw the dead pid each
25
+ // remove the other's FRESH lock and write at once; the re-read is what makes
26
+ // the takeover safe. The re-check narrows the one window the re-read leaves,
27
+ // and the order the two run in is what narrows it: isDead(deadPid)
28
+ // runs before the re-read, not after, so a live writer that takes the lock in
29
+ // the gap between them is caught by the read that follows, not hidden behind
30
+ // one taken before it arrived. Read first, and a recycled pid that came back
31
+ // to life in that same gap would still read back as itself and still check
32
+ // dead — the order this replaces would delete that live holder's fresh lock. A
33
+ // recycled pid that became a new writer this same way — dead holder's lock
34
+ // reaped by someone else, then re-created by the new owner of that very pid —
35
+ // still writes the same content and would still pass the re-read alone, but
36
+ // the isDead() ahead of it now catches that writer live, before the read ever
37
+ // runs. Only that exact pid reused a second time, inside this narrower gap,
38
+ // can still slip through. Under the reap lock no other waiter can replace the
39
+ // lock's content outside that gap, so a pid still dead immediately before the
40
+ // read that follows it is the dead holder, and a live one is left alone to be
41
+ // waited on like any live holder. A reap lock is held for one liveness check,
42
+ // one read and one unlink, and one left behind by any of those same abrupt
43
+ // deaths in that window is deliberately NOT reclaimed: waiters time out naming
44
+ // the holder, and the operator removes it. Fail closed.
45
+ //
46
+ // A dead holder whose pid was recycled reads as live until the timeout.
47
+ // Accepted: an mtime/age heuristic instead would take over a live writer on a
48
+ // slow disk, which is the row loss the ledger's lock exists to prevent.
49
+ //
50
+ // A live-looking holder can also be a zombie: a pid whose process exited but
51
+ // whose parent never `wait()`-ed on it. The OS keeps that pid reserved until
52
+ // reaped, so `process.kill(pid, 0)` still succeeds and isDead() reads it as
53
+ // alive — but unlike the two wedges above, that clears on its own once the
54
+ // pid is reaped (the parent's own exit is normally enough to trigger this),
55
+ // and the next writer's death check then takes the lock over like any other
56
+ // dead holder. No hand removal needed.
57
+
58
+ import { readFileSync, openSync, writeSync, closeSync, unlinkSync } from "node:fs";
59
+ import { isDead } from "./proc.mjs";
60
+
61
+ export const parsePid = (text) => (/^[1-9][0-9]*$/.test(text) ? Number(text) : null);
62
+
63
+ export const reapPathOf = (lock) => `${lock}.reap`;
64
+
65
+ // null when there is no lock to read — it was released between a failed
66
+ // create and this read, and the next create attempt is the right response.
67
+ export function readLock(lock, die) {
68
+ try {
69
+ return readFileSync(lock, "utf8");
70
+ } catch (e) {
71
+ if (e.code === "ENOENT") return null;
72
+ die(`cannot read ${lock}: ${e.message}`);
73
+ }
74
+ }
75
+
76
+ // true when the takeover ran (whatever it found under the reap lock), so the
77
+ // caller retries the create at once; false when another waiter holds the
78
+ // reap lock, so the caller waits like any other contended lock.
79
+ export function reapDeadHolder(lock, deadPid, die) {
80
+ const reap = reapPathOf(lock);
81
+ let fd;
82
+ try {
83
+ fd = openSync(reap, "wx");
84
+ } catch (e) {
85
+ if (e.code === "EEXIST") return false;
86
+ die(`cannot create ${reap}: ${e.message}`);
87
+ }
88
+ try {
89
+ closeSync(fd);
90
+ // Dead again, not just when the caller looked: that pid may have been
91
+ // reused since, so isDead runs first, fresh, right here — before the
92
+ // exact-match re-read, not after it. Exact match only: a waiter that
93
+ // got here first may already have reaped the dead lock and taken a
94
+ // fresh one of its own.
95
+ if (isDead(deadPid) && readLock(lock, die) === String(deadPid)) {
96
+ // Already gone is already the outcome this call wants: an operator
97
+ // clearing a wedged lock by hand between the read above and here must
98
+ // not turn our own cleanup into an uncaught crash.
99
+ try {
100
+ unlinkSync(lock);
101
+ } catch (e) {
102
+ if (e.code !== "ENOENT") throw e;
103
+ }
104
+ }
105
+ } finally {
106
+ unlinkSync(reap);
107
+ }
108
+ return true;
109
+ }
110
+
111
+ // One attempt, never a wait. Answers with `state`:
112
+ // "taken" — the lock is ours, stamped with this process's pid.
113
+ // "vanished" — it existed at the create and was gone at the read: released
114
+ // in between, so the next create attempt is the right response.
115
+ // "reaped" — its holder was dead and the takeover ran, so create again
116
+ // at once.
117
+ // "held" — somebody else's. `holder` is the file's text as read, `pid`
118
+ // its parse (null when it is not a canonical pid) and `dead`
119
+ // whether that pid is gone while the takeover could not run —
120
+ // another waiter holds the reap lock.
121
+ export function attemptLock(lock, die) {
122
+ let fd = null;
123
+ try {
124
+ fd = openSync(lock, "wx");
125
+ } catch (e) {
126
+ if (e.code !== "EEXIST") die(`cannot create ${lock}: ${e.message}`);
127
+ }
128
+ if (fd !== null) {
129
+ try {
130
+ writeSync(fd, String(process.pid));
131
+ } catch (e) {
132
+ // Ours, and empty or partial: every waiter reads it as live, and
133
+ // releaseLockFile() will not match it, so it goes now or never.
134
+ closeSync(fd);
135
+ unlinkSync(lock);
136
+ die(`cannot write ${lock}: ${e.message}`);
137
+ }
138
+ closeSync(fd);
139
+ return { state: "taken" };
140
+ }
141
+ const holder = readLock(lock, die);
142
+ if (holder === null) return { state: "vanished" };
143
+ const pid = parsePid(holder);
144
+ const dead = pid !== null && isDead(pid);
145
+ if (dead && reapDeadHolder(lock, pid, die)) return { state: "reaped" };
146
+ return { state: "held", holder, pid, dead };
147
+ }
148
+
149
+ // Removes `lock` only while it still names this process. Nothing to release,
150
+ // or nothing this process can do about it on its way out: exit handlers cannot
151
+ // fail usefully.
152
+ export function releaseLockFile(lock) {
153
+ try {
154
+ if (readFileSync(lock, "utf8") === String(process.pid)) unlinkSync(lock);
155
+ } catch {
156
+ // see above
157
+ }
158
+ }
@@ -142,6 +142,7 @@ export const TEST_RUN_SCHEMA = {
142
142
  cancelled: { type: "integer" },
143
143
  skipped: { type: "integer" },
144
144
  error: { type: "string", description: "Only when no summary was printed: what happened instead." },
145
+ slotWaitMs: { type: "integer", description: "Milliseconds the command waited for a suite slot, copied from the `SUITE_SLOT_WAIT_MS=<n>` line it printed; absent when the log holds no such line." },
145
146
  },
146
147
  };
147
148
 
@@ -321,6 +322,17 @@ export function resolveTestCmd(explicit, snap) {
321
322
  );
322
323
  }
323
324
 
325
+ // The review's one run goes through the suite-slot wrapper, by the Resolver,
326
+ // so it counts against the host-wide bound on concurrent full runs. The
327
+ // command is one word to the wrapper — it hands the string to `sh -c` itself —
328
+ // so it is single-quoted here, each embedded `'` closed, escaped and reopened.
329
+ function slotted(cmd) {
330
+ return `~/.fleet/bin/fleet-run suite-slot.mjs '${cmd.replaceAll("'", "'\\''")}'`;
331
+ }
332
+
333
+ // The status suite-slot.mjs exits with when its wait for a slot expired.
334
+ const NO_SLOT_EXIT = 75;
335
+
324
336
  export function decodeArgs(a) {
325
337
  if (typeof a !== "string") return a || {};
326
338
  try {
@@ -486,6 +498,14 @@ cannot be told apart from a regression.`;
486
498
  // TEST_RUN_EXIT line would otherwise read as green.
487
499
  function testRunReason(run) {
488
500
  if (!run) return "the review's shared test run returned nothing — no counts exist, so no dimension's tests ran";
501
+ // The wrapper's wait cap expired: it launched nothing, so there are no
502
+ // counts to judge and the exit status is the wrapper's, not the suite's. A
503
+ // run that printed counts and happened to exit 75 is the suite's own answer
504
+ // and judged below like any other.
505
+ if (run.exitCode === NO_SLOT_EXIT && typeof run.tests !== "number") {
506
+ const waited = /no slot after (\d+)s/.exec(run.error ?? "")?.[1];
507
+ return `no suite slot within ${waited === undefined ? "the wait cap" : `${waited} s`} — the host is at its suite bound; not a test failure`;
508
+ }
489
509
  const cmd = run.command || "the test command";
490
510
  if (typeof run.tests !== "number")
491
511
  return `\`${cmd}\` produced no counts — the shared test run crashed, hit its deadline, or printed no summary${run.error ? ` (${run.error})` : ""} — a failed run, not a pass`;
@@ -1003,14 +1023,25 @@ block below is absolute and cd-chained on purpose — run it exactly as written,
1003
1023
  and add nothing relative to it.
1004
1024
 
1005
1025
  Run it as ONE blocking foreground command — never backgrounded, never polled —
1006
- with a command deadline of 1800 seconds:
1026
+ with a command deadline of 3600 seconds:
1007
1027
 
1008
- { cd "${snap.path}" && ${testCmd}; } > "${logPath}" 2>&1; echo "TEST_RUN_EXIT=$?"
1028
+ { cd "${snap.path}" && ${slotted(testCmd)}; } > "${logPath}" 2>&1; echo "TEST_RUN_EXIT=$?"
1009
1029
 
1010
1030
  Run it exactly once: not again when it fails, and not again when it hits the
1011
1031
  deadline. A failing run is a result every specialist reads, and a second
1012
1032
  launch is the load this step exists to remove.
1013
1033
 
1034
+ The command first waits for a free suite slot — the host bounds how many full
1035
+ suites run at once — and that wait is not part of the run: it can take up to
1036
+ 1800 seconds before the suite starts, which is why the deadline above is longer
1037
+ than a run's own budget. When it gets a slot it prints a
1038
+ \`SUITE_SLOT_WAIT_MS=<n>\` line to the log; report \`slotWaitMs\` = that number,
1039
+ copied as printed, and omit it when the log has no such line. If no slot came
1040
+ free it launched nothing: TEST_RUN_EXIT is 75 and the log holds
1041
+ \`suite-slot: no slot after <n>s\`. That is not a test result — report
1042
+ \`exitCode\` 75, omit every count, copy that line into \`error\`, and do not run
1043
+ the command again.
1044
+
1014
1045
  Do not substitute a command of your own. In a repo that has a shared test stack,
1015
1046
  a bare runner picks up a default config whose setup can tear a sibling's
1016
1047
  container down mid-run; a guessed glob is worse in every repo, because one
@@ -0,0 +1,174 @@
1
+ #!/usr/bin/env node
2
+ // A host-wide bound on concurrent FULL test-suite runs.
3
+ //
4
+ // suite-slot.mjs '<entrypoint>' [args…]
5
+ //
6
+ // Runs `sh -c '<entrypoint> "$@"' suite-slot args…` — the entrypoint is a shell
7
+ // COMMAND, so a compound one (`cd sub && make test`) keeps its meaning and the
8
+ // caller's own arguments land after it, argv-preserved: a path with a space or
9
+ // a shell metacharacter in it reaches the entrypoint as one word, never
10
+ // re-split by a shell.
11
+ //
12
+ // Which runs take a slot is decided by ARITY, never by what the entrypoint is:
13
+ // - no appended arguments: the full suite. It takes one of N slots, waiting
14
+ // for one if all are held.
15
+ // - appended arguments: a narrowed run. It takes no slot and queues behind
16
+ // nothing, so a verifier's narrowed run never waits on a full one.
17
+ // - FLEET_SUITE_SLOT already set: an enclosing wrapper holds a slot and
18
+ // exported its lock path, so this call is nested inside it. It runs
19
+ // straight through, arguments and all, and takes no second slot — a
20
+ // wrapped command inside a wrapped command is one run, not a deadlock.
21
+ //
22
+ // A slot is `<home>/.fleet/suite-slots/<i>.lock` for `i` in 1..N: a
23
+ // pid-stamped O_EXCL lock, the very primitive the ledger's write lock is
24
+ // (pid-lock.mjs owns the protocol, the dead-holder takeover included). Under
25
+ // the home directory rather than a workspace's `.fleet/` because the bound is
26
+ // host-wide and a review's snapshot is no repository. The holder stamps its own
27
+ // pid and releases on exit; a holder killed without releasing is taken over by
28
+ // the next waiter, so a SIGKILLed wrapper never wedges a slot. What the
29
+ // takeover cannot clear — an empty or partial lock whose writer died before
30
+ // stamping its pid, and a leftover `<i>.lock.reap` — is named on the way out
31
+ // of an expired wait, for the operator to remove by hand once nothing is
32
+ // running.
33
+ //
34
+ // Environment:
35
+ // FLEET_SUITE_SLOTS number of slots N, a positive integer. Default 2.
36
+ // FLEET_SUITE_WAIT_S seconds to wait for a slot, a non-negative integer.
37
+ // Default 1800. Waiting never counts against the run.
38
+ // FLEET_SUITE_SLOT set by the holder for its child; see re-entry above.
39
+ // HOME where `.fleet/suite-slots/` lives.
40
+ //
41
+ // Exit status: the entrypoint's own, with a child killed by signal N read as
42
+ // 128+N. 2 is a refusal of this script's own (usage, a malformed variable, a
43
+ // slot directory it cannot create); 75 is an expired wait, having launched
44
+ // nothing, with `suite-slot: no slot after <n>s` on stderr. On acquisition it
45
+ // prints `SUITE_SLOT_WAIT_MS=<n>` to stderr — the milliseconds the wait took,
46
+ // a few at most when a slot was free.
47
+ //
48
+ // A waiter cycles the slots every 2 s. While the entrypoint runs, SIGTERM,
49
+ // SIGINT and SIGHUP are forwarded to it and the wrapper exits when it does.
50
+
51
+ import { spawn } from "node:child_process";
52
+ import { mkdirSync } from "node:fs";
53
+ import { constants as osConstants, homedir } from "node:os";
54
+ import { join } from "node:path";
55
+ import { makeDie, isFlagLike, isDigits, writeAll } from "./arg.mjs";
56
+ import { attemptLock, releaseLockFile, reapPathOf } from "./pid-lock.mjs";
57
+
58
+ const NAME = "suite-slot";
59
+ const die = makeDie(NAME);
60
+
61
+ const DEFAULT_SLOTS = 2;
62
+ const DEFAULT_WAIT_S = 1800;
63
+ const CYCLE_MS = 2000;
64
+ const EXIT_NO_SLOT = 75;
65
+ const FORWARDED = ["SIGTERM", "SIGINT", "SIGHUP"];
66
+
67
+ const [entrypoint, ...args] = process.argv.slice(2);
68
+ if (entrypoint === undefined || isFlagLike(entrypoint)) {
69
+ const why = entrypoint === undefined || entrypoint.trim() === "" ? "no entrypoint given" : `${JSON.stringify(entrypoint)} is a flag, not an entrypoint`;
70
+ die(`usage: suite-slot.mjs '<entrypoint>' [args…] — ${why}`);
71
+ }
72
+
73
+ // An unset variable takes the default; a set one must be a whole number in
74
+ // range. A malformed value is refused rather than defaulted: this bound is the
75
+ // host's protection, and a typo that quietly widens it or hangs the wait is
76
+ // worse than a run that did not start.
77
+ function intEnv(name, fallback, min) {
78
+ const raw = process.env[name];
79
+ if (raw === undefined) return fallback;
80
+ if (!isDigits(raw) || Number(raw) < min) {
81
+ die(`${name} must be an integer of at least ${min}, got ${JSON.stringify(raw)}`);
82
+ }
83
+ return Number(raw);
84
+ }
85
+
86
+ // Where a signal goes: the running child once there is one, otherwise nowhere
87
+ // — the wrapper holds nothing yet, so it simply ends.
88
+ let signalTarget = null;
89
+ for (const sig of FORWARDED) {
90
+ process.on(sig, () => {
91
+ if (signalTarget) signalTarget.kill(sig);
92
+ else process.exit(128 + osConstants.signals[sig]);
93
+ });
94
+ }
95
+
96
+ // The entrypoint, run to completion with the caller's streams. Resolves to the
97
+ // status the wrapper should exit with. Asynchronous on purpose: a blocking
98
+ // spawn holds the event loop, so the signal handlers could not run until the
99
+ // child had already finished.
100
+ function run(env) {
101
+ return new Promise((resolve) => {
102
+ const child = spawn("sh", ["-c", `${entrypoint} "$@"`, NAME, ...args], { stdio: "inherit", env });
103
+ signalTarget = child;
104
+ child.on("error", (e) => die(`cannot run ${JSON.stringify(entrypoint)}: ${e.message}`));
105
+ child.on("close", (code, signal) => resolve(signal ? 128 + osConstants.signals[signal] : code));
106
+ });
107
+ }
108
+
109
+ if (process.env.FLEET_SUITE_SLOT !== undefined || args.length > 0) {
110
+ process.exit(await run(process.env));
111
+ }
112
+
113
+ const slots = intEnv("FLEET_SUITE_SLOTS", DEFAULT_SLOTS, 1);
114
+ const waitMs = intEnv("FLEET_SUITE_WAIT_S", DEFAULT_WAIT_S, 0) * 1000;
115
+ const dir = join(homedir(), ".fleet", "suite-slots");
116
+ try {
117
+ mkdirSync(dir, { recursive: true });
118
+ } catch (e) {
119
+ die(`cannot create ${dir}: ${e.message}`);
120
+ }
121
+ const lockOf = (i) => join(dir, `${i}.lock`);
122
+
123
+ // One pass over the slots. The first one taken is returned as its path; a
124
+ // slot whose holder vanished or was reaped is tried once more at once, and
125
+ // counts as busy for this pass if it keeps changing under us — the next pass
126
+ // is 2 s away, so a path that never settles cannot spin. `held` collects what
127
+ // each busy slot answered, for the message an expired wait leaves behind.
128
+ function pass(held) {
129
+ held.length = 0;
130
+ for (let i = 1; i <= slots; i++) {
131
+ let seen = null;
132
+ for (let tries = 0; tries < 2; tries++) {
133
+ seen = attemptLock(lockOf(i), die);
134
+ if (seen.state === "taken") return lockOf(i);
135
+ if (seen.state === "held") break;
136
+ }
137
+ held.push({ lock: lockOf(i), ...seen });
138
+ }
139
+ return null;
140
+ }
141
+
142
+ // What an operator needs to act on a wait that expired. A live holder needs
143
+ // nothing — the host is simply at its bound — so only the two wedges the
144
+ // takeover cannot clear are named.
145
+ function wedges(held) {
146
+ const lines = [];
147
+ for (const h of held) {
148
+ if (h.state !== "held") continue;
149
+ if (h.pid === null) {
150
+ lines.push(`${h.lock} holds ${JSON.stringify(h.holder)}, not a pid, so it is never taken over — if nothing is running, remove it by hand`);
151
+ } else if (h.dead) {
152
+ lines.push(`${h.lock} is held by dead pid ${h.pid}; ${reapPathOf(h.lock)} is blocking its takeover — if nothing is running, remove it by hand`);
153
+ }
154
+ }
155
+ return lines;
156
+ }
157
+
158
+ const started = Date.now();
159
+ const held = [];
160
+ let slotPath = pass(held);
161
+ while (slotPath === null) {
162
+ const left = waitMs - (Date.now() - started);
163
+ if (left <= 0) {
164
+ const detail = wedges(held).map((l) => `\n${NAME}: ${l}`).join("");
165
+ writeAll(2, `${NAME}: no slot after ${waitMs / 1000}s${detail}\n`);
166
+ process.exit(EXIT_NO_SLOT);
167
+ }
168
+ await new Promise((r) => setTimeout(r, Math.min(CYCLE_MS, left)));
169
+ slotPath = pass(held);
170
+ }
171
+ process.on("exit", () => releaseLockFile(slotPath));
172
+ writeAll(2, `SUITE_SLOT_WAIT_MS=${Date.now() - started}\n`);
173
+
174
+ process.exit(await run({ ...process.env, FLEET_SUITE_SLOT: slotPath }));
@@ -671,7 +671,8 @@ phase, or in any later one, asks the maintainer which tickets to take.
671
671
  `shortlist.mjs` itself when unclaimed entries fall below the implementer
672
672
  cap, when the file is missing or empty, when an `excluded` premise has
673
673
  lifted, and when a branch-named `behind-pr` exclusion whose PR is not yet
674
- MERGED or CLOSED names a ticket the file still carries, and prints
674
+ MERGED or CLOSED, or a `behind-issue` exclusion whose issue is not yet
675
+ closed, names a ticket the file still carries, and prints
675
676
  `REFRESHED shortlist: <n> entries; <k> lifted`. Phase 0 runs it by hand
676
677
  once because a dead run's shortlist is stale by definition.
677
678
 
@@ -1006,10 +1007,12 @@ its re-derive-once rule.
1006
1007
 
1007
1008
  **Materialize the isolation envelope as a file, not a briefing.** The claim
1008
1009
  writes the runner, `agent-test`, into the fresh worktree it creates: an exec of
1009
- the Recipe's Test entrypoint that exports one port triple derived from `<N>`
1010
+ the Recipe's Test entrypoint, through the suite-slot wrapper (see **The suite
1011
+ slot bound**), that exports one port triple derived from `<N>`
1010
1012
  (so collisions stay impossible), kept out of `git status` through the
1011
1013
  repository's exclude file. A repository that tracks its own `agent-test` keeps
1012
- it byte-identical, with whatever isolation that repository built. Brief members
1014
+ it byte-identical, with whatever isolation that repository built, and takes the
1015
+ suite slot bound for its own full runs itself: nothing else will. Brief members
1013
1016
  with `./agent-test` and nothing else — anyone who finds the worktree finds the
1014
1017
  runner, including grandchildren you never dispatched. Its arguments land after
1015
1018
  the Test entrypoint, so what one may name — a test file, a test name — is that
@@ -1862,8 +1865,10 @@ review units (in-flight `review=` tokens plus unsettled `fix-pr-` ones), the
1862
1865
  merge bot, merge holds, tier mismatches and the drain marker come off
1863
1866
  `.fleet/ledger.md`; the unclaimed heads and the supply come off
1864
1867
  `.fleet/shortlist.json`, which it refreshes itself when the unclaimed count is
1865
- below the implementer cap, when the file is missing or empty, or when an
1866
- Exclusion's premise has lifted; the open PRs come off `gh`. A ledger token it
1868
+ below the implementer cap, when the file is missing or empty, when an
1869
+ Exclusion's premise has lifted, or when a branch-named `behind-pr` or a
1870
+ `behind-issue` Exclusion still not lifted names a ticket the file carries; the
1871
+ open PRs come off `gh`. A ledger token it
1867
1872
  cannot read, it refuses rather than guessing — a member counted live or gone on
1868
1873
  a guess is the over- or under-dispatch it exists to prevent — so correct that
1869
1874
  row with `ledger.mjs row` and tick again. The caps default to 2 implementers
@@ -2574,11 +2579,54 @@ The fix-applier prompt and every refuter brief carry the rule and the safe
2574
2579
  invocation form; the merge bot's brief is the wrong seat, because a bot
2575
2580
  reads gates and never injects a fault.
2576
2581
 
2582
+ **The suite slot bound.** At most `FLEET_SUITE_SLOTS` full suite runs execute on
2583
+ a host at once — 2 by default, whatever the member caps count — because each run
2584
+ already spreads over most of the host's cores. The bound is a wrapper,
2585
+ `suite-slot.mjs`, reached through the Resolver:
2586
+ `~/.fleet/bin/fleet-run suite-slot.mjs '<entrypoint>' [args…]` runs the
2587
+ entrypoint under `sh -c` with the arguments after it. It decides by arity. No
2588
+ appended arguments is a full run, and waits for a free slot. Appended arguments
2589
+ are a narrowed run, which takes none, so a verifier's `<testCmd> <args>` never
2590
+ queues behind a full run — only its fall-back to the full command does. A
2591
+ wrapper that finds `FLEET_SUITE_SLOT` set, which a slot holder exports to its
2592
+ child, is nested inside a held slot and runs straight through, so a wrapped
2593
+ command inside a wrapped command is one run, not a deadlock. A slot is a
2594
+ pid-stamped lock file in `~/.fleet/suite-slots/` — under the host's `~/.fleet/`
2595
+ and not a workspace's, because the bound is host-wide and a review's snapshot is
2596
+ no repository.
2597
+
2598
+ Each seat that runs the full suite is wrapped where its command is written. A
2599
+ claim's runner execs the Test entrypoint through the wrapper. A repository that
2600
+ tracks its own runner wraps it itself, as this repository's `agent-test` does
2601
+ for a bare run. The review wraps its own one run, so its args carry the bare
2602
+ command; a wrapped string handed to it anyway is harmless, the inner wrapper
2603
+ being the nested case. The fix-applier and the finisher run what you hand them,
2604
+ so hand them the wrapped string as `<testCmd>` —
2605
+ `~/.fleet/bin/fleet-run suite-slot.mjs '<command>'`, each `'` inside the command
2606
+ written `'\''` — and it is still one command for every gate.
2607
+
2608
+ The wait is capped and never counts against the run: `FLEET_SUITE_WAIT_S`
2609
+ seconds, 1800 by default, then exit **75** with `suite-slot: no slot after <n>s`
2610
+ on stderr, having launched nothing. On getting a slot the wrapper prints
2611
+ `SUITE_SLOT_WAIT_MS=<n>` — the wait in milliseconds — to stderr, which the
2612
+ review's test-run block captures into its log and the test-run agent reports as
2613
+ `slotWaitMs`. **Exit 75 is not a test result.** In a review it leaves every
2614
+ dimension unrun, for the reason "no suite slot within <n> s — the host is at its
2615
+ suite bound; not a test failure", with no retry inside the run: re-review it
2616
+ under the next runner letter (`review-pr-<pr#>-b` after a first runner), as for
2617
+ a runner that reported `failed`. Tell every implementer, fix-applier and
2618
+ finisher, in its brief, in one line: exit 75 is not a test result; run the
2619
+ command again.
2620
+
2621
+ A slot holder killed without releasing is taken over by the next waiter under
2622
+ the same `.reap` discipline as the ledger's lock; the wedges that discipline
2623
+ cannot clear are in the runbook under **Writer policy**.
2624
+
2577
2625
  **Where `testCmd` comes from:** the repository's Test entrypoint, out of the
2578
2626
  Recipe cache phase 0's Recipe derivation step proved —
2579
2627
  `~/.fleet/bin/fleet-run derive-testcmd.sh . test --at origin/main` prints it — and the one you
2580
2628
  hand specialists per **Give specialists a stack-free test command** above. Pass
2581
- the same string to the review, to the fix-applier, and to the finisher — whose
2629
+ the same command to the review, to the fix-applier, and to the finisher — whose
2582
2630
  duty-2 mutation
2583
2631
  gate runs it too — so every gate runs one command. Omit it from the review args
2584
2632
  and the review (`review-core.mjs`, in a runner) reads the same cache itself,
@@ -4363,6 +4411,13 @@ removal needed. The other two shapes do not clear on their own — remove
4363
4411
  `<file>.lock` or `<file>.lock.reap` by hand once nothing is writing. Reads
4364
4412
  (`check`, `read`) take no lock.
4365
4413
 
4414
+ **The suite slots wedge the same ways, and clear the same way.**
4415
+ `~/.fleet/suite-slots/<i>.lock` is built on the ledger's lock primitive, so an empty or
4416
+ unparseable `<i>.lock` and a leftover `<i>.lock.reap` are permanent, a zombie
4417
+ holder is transient, and a dead holder is taken over. A wait that expired with
4418
+ exit 75 names each wedged file — remove it by hand once no suite run is
4419
+ running, the way `<file>.lock` is removed above.
4420
+
4366
4421
  **The gap the lock does not close:** `check` → `gh issue create` → `filed` is
4367
4422
  three commands, and the lock covers each one alone, so two filers can both read
4368
4423
  `clean` in the seconds before either records. Accepted — the measured