@feigi/fleet-ctl 3.23.1 → 3.23.2

package/agents/fleet-recipe-deriver.agent.md

@@ -89,6 +89,13 @@ no cache was written. Exit 2 means no verdict on the proof: it could not be
 attempted (no `origin/main`, not a repository) or the cache could not be
 written — report it, do not work around it.
 
+The script bounds each command it runs itself — the Install step, both Test
+runs and the mutation — and refuses one still running at that bound as
+`timed out`. **Never wrap the `recipe-prove.mjs` call in a timeout of your
+own**: an outside kill leaves no refusal to report. A deadline on the tool
+call that runs it does the same when it is shorter than the script's bound,
+so give that call `timeout: 0` and wait for the script.
+
 You may revise and re-run after a `NOT PROVEN` you can act on — a wrong
 command, a count line you mis-copied, a mutation that changed nothing — **at
 most four `recipe-prove.mjs` runs in all**, the first one included. Never
@@ -104,6 +111,8 @@ Your last message is exactly one of:
   verbatim, the commands you tried, and one line naming the cause:
   **vacuous** (the suite ran no tests, or none a mutation could reach),
   **did not run** (a command not found or not executable — a toolchain this
-  machine lacks), **dirty install** (the Install step changes the tree), or
-  **unreadable** (you could not tell from the repository how it is built or
+  machine lacks), **dirty install** (the Install step changes the tree),
+  **timed out** (a command the proof runs — the Install step, the Test
+  entrypoint or the mutation — hung until the script's own bound stopped it),
+  or **unreadable** (you could not tell from the repository how it is built or
   tested).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feigi/fleet-ctl",
-  "version": "3.23.1",
+  "version": "3.23.2",
   "description": "Agent fleet: run-team controller, merge bot, PR reviewer, and the ticket pipeline they share",
   "license": "Apache-2.0",
   "repository": {

package/scripts/is-cli.test.mjs

@@ -67,7 +67,7 @@ test("isCLI defaults to the live process.argv[1]", () => {
 // symlinked run must match it, not merely be non-zero.
 const CLI_SCRIPTS = [
   "diff-stats", "fleet-heartbeat", "fleet-tick", "frontmatter-check", "main-checkout",
-  "member-outcomes", "tier-check", "tier-outcomes", "tier-roles", "board", "dispositions-check",
+  "member-outcomes", "tier-check", "tier-outcomes", "tier-roles", "board", "dispositions-check", "recipe-prove",
 ];
 
 for (const name of CLI_SCRIPTS) {

package/scripts/recipe-derivation-prose.test.mjs

@@ -50,6 +50,7 @@ test("an invalid cache is re-derived once; a failed derivation stalls the run in
   assert.match(s, phrase("A red suite never triggers it: a failing test is a finding, not a stale Recipe."));
   assert.match(s, phrase("`RECIPE NOT PROVEN` from any derivation, or a second invalid-cache refusal after this run's one re-derivation, halts the run"));
   assert.match(s, phrase("It is never a loop: no third derivation, and never a cache written by hand."));
+  assert.match(s, phrase("an Install step, Test entrypoint or mutation that hangs until the proof's own bound refuses it as timed out."));
 });
 
 test("the claim paragraph sends an absent or invalid cache to the derivation step, and infers no Install step", () => {
@@ -92,8 +93,20 @@ test("every flag the deriver's procedure passes is one recipe-prove.mjs accepts"
   for (const flag of used) assert.ok(accepted.has(flag), `the deriver passes ${flag}, which recipe-prove.mjs does not accept`);
 });
 
-test("the deriver reports one of two lines, naming vacuity as a cause", () => {
+test("the deriver reports one of two lines, naming vacuity and a timeout as causes", () => {
   assert.match(DERIVER, /`RECIPE PROVEN` followed by the JSON line/);
   assert.match(DERIVER, /`RECIPE NOT PROVEN` followed by the last run's `NOT PROVEN` reason/);
   assert.match(DERIVER, /\*\*vacuous\*\*/);
+  assert.match(DERIVER, /\*\*timed out\*\*/);
+  assert.match(DERIVER, phrase("the Install step, the Test entrypoint or the mutation — hung until the script's own bound stopped it"));
+});
+
+// recipe-prove.mjs bounds its own commands and refuses one that overruns; an
+// outside kill would end the run with no refusal for the deriver to report.
+test("the deriver is told the script bounds its own commands, and never to wrap the call in a timeout", () => {
+  assert.match(DERIVER, phrase("The script bounds each command it runs itself"));
+  assert.match(DERIVER, phrase("**Never wrap the `recipe-prove.mjs` call in a timeout of your own**"));
+  assert.match(DERIVER, phrase("refuses one still running at that bound as `timed out`"));
+  assert.match(DERIVER, phrase("so give that call `timeout: 0` and wait for the script"));
+  assert.match(PROVER, /timed out: still running after/, "recipe-prove.mjs no longer names a timeout the way the deriver's cause reads it");
 });

package/scripts/recipe-prove.mjs

@@ -34,6 +34,12 @@
 //   With neither, nothing is proven: the refusal still names where the test
 //   run's output is, so the caller can read the count line off it.
 //
+// Each of those commands — the Install step, both Test runs, the mutation —
+// runs under a time bound of its own, 20 minutes, that RECIPE_PROVE_TIMEOUT
+// (whole seconds) can shorten and never lengthen. A command still running at
+// the bound is killed, and the proof is NOT PROVEN for it, the refusal naming
+// the bound and the command's log.
+//
 // On proof, the cache is written atomically to <workspace>/.fleet/recipe.json
 // — <workspace> being the directory holding the repository's common git dir,
 // the place derive-testcmd.sh reads it from — and then read back through
@@ -59,6 +65,7 @@ import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, readFileSync,
 import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { isCLI } from "./is-cli.mjs";
 import { gitEnv, workspaceDirFromGitCommonDir } from "./git-env.mjs";
 import { isDigits } from "./arg.mjs";
 
@@ -73,6 +80,27 @@ const READER = join(dirname(fileURLToPath(import.meta.url)), "derive-testcmd.sh"
 // at that repository's tree, while the cache still landed here.
 const ENV = gitEnv();
 
+// The bound on each Recipe command. A command that never returns — a stalled
+// fetch, a watch-mode runner — would otherwise hang the Recipe derivation,
+// which a run waits on before its first claim, with no refusal ever printed.
+// 20 minutes is chosen against the false failure: far above a healthy install
+// or suite, and still a bound.
+//
+// The bound is per command, never for the whole proof: the longest a proof
+// can take is every command it runs taking its full bound.
+//
+// RECIPE_PROVE_TIMEOUT can only SHORTEN it: a knob that could lengthen the
+// bound is one more way for configuration to remove it. A value that is not a
+// positive whole number of seconds below the default is not an error and not
+// a bound either — the default stands, in silence. isDigits() is arg.mjs's
+// own predicate, so `3.5`, `-5` and `5e3` are refused flat, never coerced.
+const COMMAND_DEFAULT_SECONDS = 20 * 60;
+export function commandBudget(override) {
+  const seconds = isDigits(String(override)) ? Number(override) : 0;
+  return (seconds > 0 && seconds < COMMAND_DEFAULT_SECONDS ? seconds : COMMAND_DEFAULT_SECONDS) * 1000;
+}
+const COMMAND_TIMEOUT_MS = commandBudget(process.env.RECIPE_PROVE_TIMEOUT);
+
 class Refusal extends Error {
   constructor(code, message) {
     super(message);
@@ -133,10 +161,25 @@ function git(args, cwd) {
 // killed by a signal (an OOM kill, a timeout wrapper) has no exit status, so
 // it is refused here and never reported as one: a crashed run must not count
 // as a red suite.
+//
+// A command that overruns COMMAND_TIMEOUT_MS is killed with SIGKILL, never the
+// default SIGTERM: a command that traps or ignores SIGTERM would keep spawnSync
+// waiting on it forever. spawnSync reports that kill as an ETIMEDOUT `error`
+// alongside the signal, so the timeout is read first: through the `error`
+// branch it would be a `sh` that could not start, and through the `signal`
+// one a kill with no cause named. Whatever it printed before the kill stays in
+// the log, which is opened for append. Only the `sh` itself is killed:
+// spawnSync cannot kill a process tree, so a child the command started may
+// outlive the refusal.
 function sh(cmd, cwd, log) {
   const fd = openSync(log, "a");
   try {
-    const r = spawnSync("sh", ["-c", cmd], { cwd, env: ENV, stdio: ["ignore", fd, fd] });
+    const r = spawnSync("sh", ["-c", cmd], {
+      cwd, env: ENV, stdio: ["ignore", fd, fd], timeout: COMMAND_TIMEOUT_MS, killSignal: "SIGKILL",
+    });
+    if (r.error?.code === "ETIMEDOUT") {
+      throw notProven(`'${cmd}' timed out: still running after ${COMMAND_TIMEOUT_MS / 1000}s, the bound on each Recipe command, so it was stopped; output: ${log}`);
+    }
     if (r.error) throw cannot(`could not start sh: ${r.error.message}`);
     if (r.signal) throw notProven(`'${cmd}' was killed by ${r.signal}, so it has no exit status to read; output: ${log}`);
     return r.status;
@@ -326,19 +369,22 @@ function main(argv) {
   process.stdout.write(`${NAME}: PROVEN — Recipe cache written to ${cache}\n${JSON.stringify(recipe)}\n`);
 }
 
-try {
-  main(process.argv.slice(2));
-} catch (e) {
-  // A fault outside the proof — the cache or its temp file could not be
-  // written — is "no verdict" (2), never the NOT PROVEN (1) a stack trace's
-  // exit status would read as.
-  if (!(e instanceof Refusal)) {
-    process.stderr.write(`${NAME}: ${e.message}\n`);
-    process.exit(2);
+// Only run main() as a CLI, never when imported by a test (see is-cli.mjs).
+if (isCLI(import.meta.url)) {
+  try {
+    main(process.argv.slice(2));
+  } catch (e) {
+    // A fault outside the proof — the cache or its temp file could not be
+    // written — is "no verdict" (2), never the NOT PROVEN (1) a stack trace's
+    // exit status would read as.
+    if (!(e instanceof Refusal)) {
+      process.stderr.write(`${NAME}: ${e.message}\n`);
+      process.exit(2);
+    }
+    // The closing sentence on a line of its own: most reasons end in a log path,
+    // and a `.` riding on one would read as part of it.
+    const msg = e.code === 1 ? `NOT PROVEN — ${e.message}\nNo Recipe cache written.` : e.message;
+    process.stderr.write(`${NAME}: ${msg}\n`);
+    process.exit(e.code);
   }
-  // The closing sentence on a line of its own: most reasons end in a log path,
-  // and a `.` riding on one would read as part of it.
-  const msg = e.code === 1 ? `NOT PROVEN — ${e.message}\nNo Recipe cache written.` : e.message;
-  process.stderr.write(`${NAME}: ${msg}\n`);
-  process.exit(e.code);
 }

package/scripts/recipe-prove.test.mjs

@@ -5,6 +5,7 @@ import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, sy
 import { join } from "node:path";
 import { tempDir } from "./temp-dir.mjs";
 import { writeExecStub } from "./exec-stub.mjs";
+import { commandBudget } from "./recipe-prove.mjs";
 
 // recipe-prove.mjs is the Recipe derivation step's proof and the ONE writer of
 // the Recipe cache: the deriving agent chooses the Install step and the Test
@@ -324,7 +325,7 @@ test("the throwaway worktree is removed afterwards, proven or not", () => {
   const { dir } = repo(MAVEN_FILES);
   prove(dir, MAVEN_PROOF);
   prove(dir, ["--install", "exit 3", ...MAVEN_PROOF.slice(2)]);
-  assert.equal(git(dir, "worktree", "list", "--porcelain").split("\n").filter((l) => l.startsWith("worktree ")).length, 1);
+  assert.equal(worktrees(dir), 1);
 });
 
 test("an ambient GIT_DIR naming another repository does not change the answer", () => {
@@ -367,10 +368,13 @@ test("a proof that cannot be attempted exits 2, distinct from a Recipe that is n
   const missing = prove(half, ["--install", "--test", "--test", "mvn -q test"]);
   assert.equal(missing.status, 2);
   assert.match(missing.err, /^recipe-prove: usage:/);
-  assert.equal(git(half, "worktree", "list", "--porcelain").split("\n").filter((l) => l.startsWith("worktree ")).length, 1,
+  assert.equal(worktrees(half), 1,
     "a refusal at the argument boundary creates no worktree");
 });
 
+// The number of worktrees the repository at `dir` has registered, the main checkout included.
+const worktrees = (dir) => git(dir, "worktree", "list", "--porcelain").split("\n").filter((l) => l.startsWith("worktree ")).length;
+
 // Every temp dir the proof made, scanned for the throwaway worktree it names `wt`.
 function assertNoWorktreeLeft(tmp) {
   for (const d of readdirSync(tmp)) {
@@ -581,7 +585,7 @@ test("read-only install output cannot turn a proof that held into a crash", { sk
   assert.equal(r.status, 0, r.err);
   assert.equal(JSON.parse(readFileSync(cachePath(dir), "utf8")).derivedAt, head);
   assert.deepEqual(logDirs(r.tmp), []);
-  assert.equal(git(dir, "worktree", "list", "--porcelain").split("\n").filter((l) => l.startsWith("worktree ")).length, 1);
+  assert.equal(worktrees(dir), 1);
 });
 
 test("read-only install output does not replace a refusal's reason with a stack trace", { skip: IGNORES_MODES }, () => {
@@ -721,3 +725,115 @@ test("the mutation is credited only with what it changed, never with what the un
   assert.equal(r.status, 0, r.err);
   assert.match(JSON.parse(readFileSync(cachePath(real), "utf8")).mutation, /changed state\.txt$/);
 });
+
+// Every Recipe command runs under a time bound, shortened here through
+// RECIPE_PROVE_TIMEOUT so a hang costs seconds. Each case below hangs one of
+// the four commands the proof runs; a bound that missed one would leave that
+// case to the helper's own 60s spawn timeout, which reads as status null.
+const timedOut = (cmd, seconds) =>
+  new RegExp(`NOT PROVEN — '${cmd.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}' timed out: still running after ${seconds}s, the bound on each Recipe command`);
+
+const HANGS = [
+  {
+    name: "an Install step",
+    files: MAVEN_FILES,
+    seconds: 1,
+    hung: "echo fetching deps; exec sleep 30",
+    args: (hung) => ["--install", hung, ...MAVEN_PROOF.slice(2)],
+    log: "install.log",
+    output: /fetching deps/,
+  },
+  {
+    name: "a Test entrypoint",
+    files: MAVEN_FILES,
+    seconds: 1,
+    hung: "echo 'Tests run: 1,'; exec sleep 30",
+    args: (hung) => ["--install", "true", "--test", hung, "--count-line", "Tests run: 1,", "--test-count", "1"],
+    log: "test.log",
+    output: /Tests run: 1,/,
+  },
+  // The two below run a healthy command first, so their bound leaves it room.
+  {
+    name: "a mutation command",
+    files: GO_FILES,
+    seconds: 3,
+    hung: "echo mutating; exec sleep 30",
+    args: (hung) => [...GO_PROOF.slice(0, 4), "--mutate", hung, "--mutation", "never finishes"],
+    log: "mutate.log",
+    output: /mutating/,
+  },
+  {
+    name: "a mutated Test run",
+    files: GO_FILES,
+    seconds: 3,
+    hung: "if grep -q 'a - b' calc.go; then echo mutated; exec sleep 30; else go test ./...; fi",
+    args: (hung) => ["--install", "true", "--test", hung, ...GO_PROOF.slice(4)],
+    log: "test-mutated.log",
+    output: /mutated/,
+  },
+];
+
+for (const c of HANGS) {
+  test(`${c.name} that never finishes is refused as timed out, NOT PROVEN, naming the bound and its log`, () => {
+    const { dir } = repo(c.files);
+    // A cache from an earlier proof: a refusal must leave its bytes alone.
+    mkdirSync(join(dir, ".fleet"));
+    writeFileSync(cachePath(dir), '{"earlier":"cache"}\n');
+    const start = Date.now();
+    const r = prove(dir, c.args(c.hung), { env: { RECIPE_PROVE_TIMEOUT: String(c.seconds) } });
+    const elapsed = Date.now() - start;
+    assert.equal(r.status, 1, `status ${r.status} after ${elapsed} ms: ${r.err}`);
+    assert.match(r.err, timedOut(c.hung, c.seconds));
+    assert.doesNotMatch(r.err, /could not start sh/);
+    assert.doesNotMatch(r.err, /was killed by/, "a timeout is named as one, not as a bare signal");
+    assert.ok(elapsed < 20_000, `the 30s command was not cut at the ${c.seconds}s bound: ${elapsed} ms`);
+    // The log the refusal names is kept, with what the command printed before the kill.
+    const log = r.err.match(/output: (\S+)$/m);
+    assert.ok(log, r.err);
+    assert.ok(log[1].endsWith(`/${c.log}`), `${log[1]} is not the ${c.log}`);
+    assert.match(readFileSync(log[1], "utf8"), c.output);
+    // The throwaway worktree is removed and unregistered, and the cache untouched.
+    assertNoWorktreeLeft(r.tmp);
+    assert.equal(worktrees(dir), 1, git(dir, "worktree", "list"));
+    assert.equal(readFileSync(cachePath(dir), "utf8"), '{"earlier":"cache"}\n');
+  });
+}
+
+// SIGTERM, the default kill signal, is one a command can trap: spawnSync then
+// waits on it forever. The bound must not depend on the command's consent.
+test("a command that ignores SIGTERM is still refused at the bound, not waited on", () => {
+  const { dir } = repo(MAVEN_FILES);
+  const hung = "trap '' TERM; while :; do sleep 1; done";
+  const start = Date.now();
+  const r = prove(dir, ["--install", "true", "--test", hung, "--count-line", "x", "--test-count", "1"], { env: { RECIPE_PROVE_TIMEOUT: "1" } });
+  const elapsed = Date.now() - start;
+  assert.equal(r.status, 1, `status ${r.status} after ${elapsed} ms: ${r.err}`);
+  assert.match(r.err, timedOut(hung, 1));
+  assert.ok(elapsed < 20_000, `the bound did not stop a command that traps SIGTERM: ${elapsed} ms`);
+  assert.equal(existsSync(cachePath(dir)), false);
+  assert.equal(worktrees(dir), 1, git(dir, "worktree", "list"));
+});
+
+// The bound is per command, never one budget for the whole proof: two
+// commands each under it, together over it, still prove.
+test("commands that each finish inside the bound still prove, whatever their total", () => {
+  const { dir } = repo(MAVEN_FILES);
+  const r = prove(dir, ["--install", "sleep 3", "--test", "sleep 3; mvn -q test", ...MAVEN_PROOF.slice(4)], { env: { RECIPE_PROVE_TIMEOUT: "5" } });
+  assert.equal(r.status, 0, r.err);
+  assert.equal(JSON.parse(readFileSync(cachePath(dir), "utf8")).testCount, 1);
+});
+
+// RECIPE_PROVE_TIMEOUT may only shorten the 20-minute default: a value that is
+// not a whole number of seconds strictly between 0 and the default leaves the
+// default standing. `3.5` and `-5` are numbers Number() would take; a bound
+// they set would be configuration lengthening or removing nothing it may.
+test("RECIPE_PROVE_TIMEOUT can only shorten the default bound", () => {
+  const DEFAULT_MS = 20 * 60 * 1000;
+  assert.equal(commandBudget(undefined), DEFAULT_MS);
+  for (const ignored of ["", "0", "abc", "3.5", "-5", " 5", "5e3", "1200", "1201", "86400"]) {
+    assert.equal(commandBudget(ignored), DEFAULT_MS, `RECIPE_PROVE_TIMEOUT=${JSON.stringify(ignored)} changed the bound`);
+  }
+  assert.equal(commandBudget("1"), 1000);
+  assert.equal(commandBudget("1199"), 1_199_000);
+  assert.equal(commandBudget("007"), 7000);
+});

package/skills/run-team/SKILL.md

@@ -403,9 +403,11 @@ phase, or in any later one, asks the maintainer which tickets to take.
    re-derivation, halts the run — no Pull, no claim, no review dispatch — and
    you report `STALL — Recipe not proven: <cause>` to the maintainer with the
    deriver's reason verbatim. A repository the fleet cannot run needs human
-   hands, the `ready-for-human` cause: a toolchain this machine lacks, or a
-   suite that runs no tests the proof can see. It is never a loop: no third
-   derivation, and never a cache written by hand.
+   hands, the `ready-for-human` cause: a toolchain this machine lacks, a
+   suite that runs no tests the proof can see, or an Install step, Test
+   entrypoint or mutation that hangs until the proof's own bound refuses it as
+   timed out. It is never a loop: no third derivation, and never a cache
+   written by hand.
 
    **Fold in every PR a prior run left open, before shortlisting.** A chore PR
    carrying that run's own metrics, or ticket work whose review was deferred —