@mmerterden/multi-agent-pipeline 17.4.0 → 17.5.0

package/pipeline/schemas/prefs.schema.json

@@ -969,6 +969,28 @@
             }
           }
         },
+        "baseBranchEvidence": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "v17.5.0+ - Phase 0 Step 3 base-branch evidence collection (refs/features/base-branch-evidence.md). Candidates are collected with the evidence behind them (issue version field, linked release issue, the branch convention learned from the refs that exist, recent branches, repo default), ranked, and shown. Interactive runs always ask; the evidence only reorders the rows.",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "default": true,
+              "description": "Collect issue-derived candidates at all. Off falls back to the develop/release/main sort order, which is still surfaced through the picker."
+            },
+            "preferLinkedRelease": {
+              "type": "boolean",
+              "default": false,
+              "description": "On boards where opening a development sub-task requires selecting the related release issue, that link is the authoritative base-branch answer and the version field is corroboration. Raises the linked-release weight above the version-field one rather than adding a second rule."
+            },
+            "autopilotAsksOnIssue": {
+              "type": "boolean",
+              "default": false,
+              "description": "OFF by default, and it is an outward-facing write. When on, an autopilot run whose base-branch derivation is ambiguous posts ONE comment on the Jira or GitHub issue asking which branch to develop from, then halts on circuit-breaker trigger 6 and waits for resume. A question, never a state change: no transition, no resolution, no assignee, no close. The body uses Ref:, never Closes:/Fixes:/Resolves:, and its human-facing copy follows outputLanguage. Autopilot never posts the question and then answers it itself."
+            }
+          }
+        },
         "autopilotCircuitBreaker": {
           "type": "object",
           "additionalProperties": false,

package/pipeline/schemas/token-budget.json

@@ -4,7 +4,7 @@
   "description": "Per-phase token ceilings for the lazy-loaded pipeline docs, enforced by smoke-token-budget.sh. Only the ACTIVE phase is loaded at run time and nothing truncates a phase document, so these numbers govern what may be written, not what a run receives. Two rules, and the split is the point. `max_tokens` and `total_max_tokens` are committed constants a human owns: computing them would end the gate, because a ceiling that is always current x k can never fail. The warn tier is NOT stored - the gate derives it per phase from that phase's own git history (median + 3*MAD of its historical per-commit deltas, which is robust to the one 980-token release that makes sigma meaningless on phase-4), because a soft line maintained by hand rots, and this one rotted twice: it was reset at v13.6.0 when five lines had gone permanently amber, and six of eight were amber again by v17.1.0. A ceiling more than 25% above the measurement is stale and fails, which is the 'only ratchets down' rule the SKILL.md grace list always had and this budget never did. Change history: docs/token-budget-history.md.",
   "phases": {
     "phase-0-init": {
-      "max_tokens": 13100
+      "max_tokens": 13400
     },
     "phase-1-analysis": {
       "max_tokens": 4600
@@ -28,5 +28,5 @@
       "max_tokens": 6350
     }
   },
-  "total_max_tokens": 62700
+  "total_max_tokens": 63000
 }

package/pipeline/scripts/autopilot-runner.mjs

@@ -12,7 +12,8 @@
  *   2. INTAKE   autopilot-intake.mjs writes the ordered queue. Read-only.
  *   3. ARM      autopilot-arming.mjs answers spend and credentials.
  *   4. TAKE     the head of the queue, and only if this repo is free.
- *   5. RUN      one `claude --bg` child, waited on. Stops at an open PR.
+ *   5. RUN      one `claude --bg` child, then SUPERVISED to the end. `--bg`
+ *               returns immediately, so the spawn proves only that a run began.
  *   6. RECORD   attempted.jsonl, clear the pid file, refresh status.json.
  *
  * LIVENESS WITHOUT A LEASE. Lease arithmetic exists to arbitrate between rival
@@ -26,9 +27,9 @@
  * runner was live and do nothing, forever.
  *
  * WHAT IT WILL NOT DO. It does not merge. It does not remove a worktree holding
- * uncommitted work - that is stashed to a branch and the worktree kept, because
- * losing a day of edits is worse than 750 MB. It does not remove a worktree it
- * cannot prove it created: the measured tree here had 19 worktrees with no state
+ * uncommitted work - that is stashed under a named message and the worktree
+ * kept, because losing a day of edits is worse than 750 MB. It does not remove
+ * a worktree it cannot prove it created: the measured tree here had 19 worktrees with no state
  * file, several of them hand-made and one in active use.
  *
  * Usage:
@@ -48,6 +49,8 @@ import {
   mkdirSync,
   rmSync,
   chmodSync,
+  readdirSync,
+  statSync,
 } from "node:fs";
 import { join } from "node:path";
 import { homedir } from "node:os";
@@ -62,6 +65,17 @@ const ROOT = process.env.MA_AUTOPILOT_ROOT || join(homedir(), ".claude", "autopi
 const SCRIPTS = process.env.MA_AP_SCRIPTS || import.meta.dirname;
 const CLAUDE_BIN = process.env.MA_AP_CLAUDE_BIN || "claude";
 const DRY = process.argv.includes("--dry-run");
+const POLL_MS = Number(process.env.MA_AP_POLL_MS || 15000);
+// A run that has not finished in this long is not going to, and holding the
+// queue behind it is worse than recording it as stuck. launchd restarts the
+// tick, so nothing is lost by returning.
+const MAX_RUN_MS = Number(process.env.MA_AP_MAX_RUN_MS || 6 * 60 * 60 * 1000);
+// `claude agents --json` lists a session a moment AFTER `--bg` returns, so an
+// absence in the first few seconds is not evidence of anything.
+const REGISTER_GRACE_MS = Number(process.env.MA_AP_GRACE_MS || 60000);
+// A session in one of these is still working. Anything else - and a row with
+// no status at all is not "anything else" - ends supervision.
+const LIVE_STATUSES = new Set(["busy", "running", "waiting"]);
 
 const log = (m) => process.stdout.write(`autopilot-runner: ${m}\n`);
 
@@ -122,17 +136,32 @@ function pidAlive(pid) {
   }
 }
 
-function sessionLive(sessionId) {
-  if (!sessionId) return false;
-  const out = process.env.MA_AP_AGENTS_JSON ?? run(CLAUDE_BIN, ["agents", "--json"]);
+/** Block this thread. The tick IS the supervisor, so it has nothing else to do. */
+function sleepSync(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/**
+ * The session's row in `claude agents --json`.
+ *
+ * Three answers, not two, and the third is the one that matters: `undefined`
+ * means the list could not be read, which is NOT the same as the session being
+ * gone. Collapsing them is how a live run gets retired because one command's
+ * output changed shape.
+ *
+ * @returns {object|null|undefined} the row, null when listed-and-absent,
+ *   undefined when the list itself could not be read
+ */
+export function agentRow(sessionId, agentsJson) {
+  if (!sessionId) return null;
+  const out = agentsJson ?? process.env.MA_AP_AGENTS_JSON ?? run(CLAUDE_BIN, ["agents", "--json"]);
+  if (!out || !out.trim()) return undefined;
   try {
-    const rows = JSON.parse(out || "[]");
+    const rows = JSON.parse(out);
     const list = Array.isArray(rows) ? rows : rows.agents || [];
-    return list.some((a) => a.sessionId === sessionId);
+    return list.find((a) => a.sessionId === sessionId) || null;
   } catch {
-    // Cannot tell. Treated as NOT live on purpose: the alternative is a queue
-    // that stops forever because one command's output changed shape.
-    return false;
+    return undefined;
   }
 }
 
@@ -143,20 +172,55 @@ function sessionLive(sessionId) {
  * restart: a pid recorded before the current boot is stale by definition and
  * needs no probe at all.
  */
-export function holderIsLive(holder, { now = bootTime(), agentsJson } = {}) {
-  if (!holder || !holder.pid) return false;
-  if (holder.bootTime && now && holder.bootTime !== now) return false;
-  if (!pidAlive(holder.pid)) return false;
-  if (agentsJson !== undefined) {
+export function holderLiveness(holder, { now = bootTime(), agentsJson } = {}) {
+  if (!holder || !holder.pid) return "dead";
+  if (holder.bootTime && now && holder.bootTime !== now) return "dead";
+  if (!pidAlive(holder.pid)) return "dead";
+  const row = agentRow(holder.sessionId, agentsJson);
+  if (row === undefined) return "unknown";
+  return row ? "live" : "dead";
+}
+
+export function holderIsLive(holder, opts = {}) {
+  return holderLiveness(holder, opts) === "live";
+}
+
+/**
+ * The run's own state file, which is what actually knows the worktree.
+ *
+ * The runner does not create the worktree - Phase 0 of the run does - and the
+ * runner's `taskId` (`ap-<epoch>`) is not the run's task id. So the link is made
+ * from disk: a state file whose `worktreePath` sits under this repo and which
+ * was written after this claim started. `gc-abandoned.sh` builds the same index
+ * for the same reason.
+ *
+ * Returns null rather than guessing. Every field it fills is one `retire()` can
+ * act on; without them retire still records the attempt, it just cannot clean
+ * anything up.
+ */
+export function findRunState(repoPath, startedAtSec, logsRoot) {
+  const root = logsRoot || join(homedir(), ".claude", "logs", "multi-agent");
+  if (!repoPath || !existsSync(root)) return null;
+  let best = null;
+  for (const dir of readdirSync(root, { withFileTypes: true })) {
+    if (!dir.isDirectory()) continue;
+    const p = join(root, dir.name, "agent-state.json");
+    if (!existsSync(p)) continue;
+    let st;
     try {
-      const rows = JSON.parse(agentsJson || "[]");
-      const list = Array.isArray(rows) ? rows : rows.agents || [];
-      return list.some((a) => a.sessionId === holder.sessionId);
+      st = statSync(p);
     } catch {
-      return false;
+      continue;
     }
+    // A second of slack: the claim is stamped before the child is spawned.
+    if (st.mtimeMs < (startedAtSec - 1) * 1000) continue;
+    const doc = readJson(p, null);
+    const wt = doc && doc.worktreePath;
+    if (!wt || !(wt === repoPath || wt.startsWith(repoPath + "/"))) continue;
+    if (!best || st.mtimeMs > best.mtimeMs)
+      best = { statePath: p, worktree: wt, mtimeMs: st.mtimeMs };
   }
-  return sessionLive(holder.sessionId);
+  return best;
 }
 
 /**
@@ -172,17 +236,31 @@ function retire(holder) {
   if (wt && existsSync(wt)) {
     const dirty = run("git", ["-C", wt, "status", "--porcelain"]).trim();
     if (dirty) {
-      // A day of edits is worth more than 750 MB. The branch name is derived
-      // from the task so it can be found without remembering this ran.
-      const branch = `autopilot/abandoned/${holder.taskId || item.id || "unknown"}`;
-      run("git", ["-C", wt, "stash", "push", "-u", "-m", branch]);
-      run("git", ["-C", wt, "branch", branch]);
-      stashed = branch;
-      log(`kept ${wt}: uncommitted work stashed to ${branch}`);
+      // A day of edits is worth more than 750 MB. The stash MESSAGE carries the
+      // task, so `git stash list` finds it without remembering this ran.
+      //
+      // No branch is created. A branch made after `stash push` points at HEAD
+      // and therefore contains none of the stashed work - verified: the branch's
+      // tree held the ORIGINAL file and no untracked file at all. Branching at
+      // `refs/stash` is no better, because with `-u` the untracked files live in
+      // the stash commit's third parent, not in its tree. One honest reference
+      // beats two, and this is the contract gc-abandoned.sh already keeps.
+      const label = `autopilot/abandoned/${holder.taskId || item.id || "unknown"}`;
+      run("git", ["-C", wt, "stash", "push", "-u", "-m", label]);
+      stashed = label;
+      log(`kept ${wt}: uncommitted work stashed as "${label}" (git stash list)`);
     } else if (holder.createdWorktree) {
-      run("git", ["-C", wt, "worktree", "remove", "--force", wt]);
-      run("git", ["-C", holder.repoPath || wt, "worktree", "prune"]);
-      log(`removed ${wt}`);
+      // A clean worktree is only safe to remove once nothing is writing to it.
+      // Supervision can end while the run does not - a timeout, or an agent
+      // list that could not be read - and `worktree remove --force` under a
+      // live session is the one mistake here with nothing to undo it.
+      if (agentRow(holder.sessionId)) {
+        log(`kept ${wt}: session ${holder.sessionId} is still listed`);
+      } else {
+        run("git", ["-C", wt, "worktree", "remove", "--force", wt]);
+        run("git", ["-C", holder.repoPath || wt, "worktree", "prune"]);
+        log(`removed ${wt}`);
+      }
     }
   }
 
@@ -209,20 +287,156 @@ function refreshStatus() {
   if (existsSync(s)) run("bash", [s, "--write"]);
 }
 
+/**
+ * Put the slot back and drop the claim. One place, so every exit agrees.
+ *
+ * It re-reads queue.json rather than trusting a copy, because intake runs
+ * between the read and here. The recovery path calls this too: `retire()` used
+ * to record the outcome and delete the claim while leaving the item in
+ * `running`, and nothing else on this machine ever removes one - intake copies
+ * the list forward verbatim. One crashed runner was enough to wedge the queue
+ * at "1/1 slots in use" on every tick from then on.
+ */
+function releaseSlot(id, pidPath, queue = { queued: [], running: [] }) {
+  const after = readJson(join(ROOT, "queue.json"), queue);
+  writeState("queue.json", {
+    ...after,
+    running: (after.running || []).filter((r) => r.id !== id),
+  });
+  rmSync(pidPath, { force: true });
+  refreshStatus();
+}
+
+/** The PR a run opened, wherever its state file recorded it. */
+export function prFromState(state) {
+  const pr = state && state.pr;
+  if (!pr) return null;
+  if (typeof pr === "string") return pr.startsWith("http") ? pr : null;
+  if (typeof pr === "object") return pr.url || null;
+  return null;
+}
+
+/**
+ * What to call this attempt.
+ *
+ * The run's own state is the authority; the supervisor only says why waiting
+ * stopped. "finished with no PR" is deliberately not "failed": a run can stop
+ * legitimately without opening one, and calling that a failure would put a
+ * healthy item into the retry path.
+ */
+export function outcomeFor(supervised, state, prUrl) {
+  if (supervised.reason === "timeout") return "timed-out";
+  if (supervised.reason === "unknown") return "unknown";
+  if (supervised.reason === "needs-input") return "needs-input";
+  if (prUrl) return "pr-opened";
+  const status = state && state.status;
+  if (status === "complete" || status === "completed") return "completed-no-pr";
+  if (status === "failed") return "failed";
+  return state ? `stopped-${status || "unknown"}` : "no-state";
+}
+
+/**
+ * Wait for the background run, and learn what it made while waiting.
+ *
+ * Polling `claude agents --json` rather than waiting on the child, because the
+ * child is gone: `--bg` returns immediately. Three things end the wait - the
+ * session leaves the list, it reports a status that is not live, or the ceiling
+ * is reached. An unreadable list ends it too, but says "unknown" rather than
+ * inventing an outcome.
+ *
+ * The claim is enriched on the way: the first poll that finds the run's state
+ * file writes `worktree` and `statePath` into runner.pid, which is what lets a
+ * LATER tick's retire() actually clean up. Those fields were read by retire()
+ * from the day it was written and never once set by anything.
+ */
+function supervise(sessionId, claim, item) {
+  const startedMs = Date.now();
+  const deadline = startedMs + MAX_RUN_MS;
+  let sawLive = false;
+  let sawBusy = false;
+  let reason;
+
+  for (;;) {
+    const row = agentRow(sessionId);
+
+    if (row === undefined) {
+      reason = "unknown";
+      break;
+    }
+    if (row) {
+      sawLive = true;
+      if (!claim.statePath) {
+        const found = findRunState(claim.repoPath, claim.startedAt);
+        if (found) {
+          claim.statePath = found.statePath;
+          claim.worktree = found.worktree;
+          // Phase 0 opened it, not this runner - but this runner is the only
+          // thing that will be around to remove it.
+          claim.createdWorktree = true;
+          writeState("runner.pid", claim);
+          log(`tracking ${found.worktree}`);
+        }
+      }
+      if (row.status === "busy" || row.status === "running") sawBusy = true;
+      if (row.status === "waiting") {
+        // Only terminal once the run has actually been working. A session can
+        // read as "waiting" in the second after launch, and ending supervision
+        // there would abandon every run at birth.
+        if (sawBusy) {
+          reason = "needs-input";
+          break;
+        }
+      } else if (row.status && !LIVE_STATUSES.has(row.status)) {
+        reason = "finished";
+        break;
+      }
+    } else if (sawLive || Date.now() - startedMs > REGISTER_GRACE_MS) {
+      reason = "gone";
+      break;
+    }
+
+    if (Date.now() >= deadline) {
+      reason = "timeout";
+      break;
+    }
+    sleepSync(POLL_MS);
+  }
+
+  if (!claim.statePath) {
+    const found = findRunState(claim.repoPath, claim.startedAt);
+    if (found) {
+      claim.statePath = found.statePath;
+      claim.worktree = found.worktree;
+      claim.createdWorktree = true;
+    }
+  }
+  const waitedSec = Math.round((Date.now() - startedMs) / 1000);
+  log(`${item.id}: supervision ended (${reason}) after ${waitedSec}s`);
+  return { reason, waitedSec };
+}
+
 function main() {
   const pidPath = join(ROOT, "runner.pid");
 
   // ---- 1. RECOVER --------------------------------------------------------
   const holder = readJson(pidPath, null);
   if (holder) {
-    if (holderIsLive(holder)) {
+    const liveness = holderLiveness(holder);
+    if (liveness === "live") {
       log(`another runner is live (pid ${holder.pid}) - nothing to do`);
       return 0;
     }
+    if (liveness === "unknown") {
+      // The pid answers but the agent list could not be read. "I cannot tell"
+      // is not "it died": retiring here deletes a running run's claim and
+      // records an outcome that never happened.
+      log(`cannot read the agent list - leaving pid ${holder.pid} alone this tick`);
+      return 0;
+    }
     log(`previous runner is gone (pid ${holder.pid}) - retiring its item`);
     if (!DRY) {
       retire(holder);
-      rmSync(pidPath, { force: true });
+      releaseSlot(holder.item && holder.item.id, pidPath);
     }
   }
 
@@ -332,20 +546,53 @@ function main() {
     ],
     { cwd: next.localPath || process.cwd(), encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] },
   );
+  if (child.status !== 0) {
+    const why = (child.stderr || child.error?.message || "").trim().slice(0, 300);
+    record({ source: next.source, id: next.id, taskId, outcome: "launch-failed", note: why });
+    releaseSlot(next.id, pidPath, queue);
+    log(`${next.id}: launch-failed ${why}`);
+    return 0;
+  }
 
-  // ---- 6. RECORD ---------------------------------------------------------
-  const outcome = child.status === 0 ? "pr-opened" : "build-failed";
-  const prUrl = /https:\/\/\S*\/pull(-requests)?\/\d+/.exec(child.stdout || "")?.[0] || null;
-  record({ source: next.source, id: next.id, taskId, outcome, prUrl, usd: 0 });
+  // ---- 5b. SUPERVISE -----------------------------------------------------
+  // `claude --bg` returns as soon as the session is started - `claude --help`
+  // says so in as many words. The spawn above therefore proves only that a run
+  // BEGAN. Everything below used to run immediately after it: the outcome was
+  // recorded as "pr-opened" before any work happened, the slot was freed within
+  // a second, and the per-repo guard three steps up became decorative because
+  // `running` was already empty when the next tick read it.
+  const supervised = supervise(sessionId, claim, next);
 
-  const after = readJson(join(ROOT, "queue.json"), queue);
-  writeState("queue.json", {
-    ...after,
-    running: (after.running || []).filter((r) => r.id !== next.id),
+  // ---- 6. RECORD ---------------------------------------------------------
+  // From the run's own state file, not from the child's exit code and not from
+  // its stdout - with `--bg` the stdout is a session id, so the PR regex that
+  // used to read it could never match.
+  const finalState = claim.statePath ? readJson(claim.statePath, null) : null;
+  const prUrl = prFromState(finalState);
+  const outcome = outcomeFor(supervised, finalState, prUrl);
+  record({
+    source: next.source,
+    id: next.id,
+    taskId,
+    outcome,
+    prUrl,
+    waitedSec: supervised.waitedSec,
+    usd: 0,
   });
-  rmSync(join(ROOT, "runner.pid"), { force: true });
-  refreshStatus();
-  log(`${next.id}: ${outcome}${prUrl ? ` ${prUrl}` : ""}`);
+
+  // "gone" and "finished" are the two reasons that mean the run ENDED. The
+  // other three - a ceiling, an unreadable list, a session parked on a question
+  // - say only that waiting stopped, and the run may still hold its worktree.
+  // Deleting the claim there would strand it: the worktree and state path this
+  // tick just learned are the only record, and the next tick would take another
+  // item in the same repo. Keeping it means the next tick's RECOVER decides,
+  // with retire() free to clean up and releaseSlot to free the slot.
+  if (supervised.reason === "gone" || supervised.reason === "finished") {
+    releaseSlot(next.id, pidPath, queue);
+  } else {
+    log(`${next.id}: claim kept for the next tick (${supervised.reason})`);
+  }
+  log(`${next.id}: ${outcome}${prUrl ? ` ${prUrl}` : ""} (${supervised.waitedSec}s)`);
   return 0;
 }