@mmerterden/multi-agent-pipeline 17.6.0 → 18.0.0

package/pipeline/scripts/autopilot-intake.mjs

@@ -39,6 +39,7 @@ import { execFileSync } from "node:child_process";
 import { existsSync, readFileSync, mkdirSync, writeFileSync, renameSync, chmodSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { homedir } from "node:os";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
 
 const ROOT = process.env.MA_AUTOPILOT_ROOT || join(homedir(), ".claude", "autopilot");
 
@@ -382,6 +383,6 @@ function main(argv) {
   return 0;
 }
 
-if (import.meta.url === `file://${process.argv[1]}`) {
+if (invokedDirectly(import.meta.url)) {
   process.exit(main(process.argv));
 }

package/pipeline/scripts/autopilot-runner.mjs

@@ -51,10 +51,16 @@ import {
   chmodSync,
   readdirSync,
   statSync,
+  openSync,
+  readSync,
+  closeSync,
+  truncateSync,
 } from "node:fs";
 import { join } from "node:path";
 import { homedir } from "node:os";
 import { randomUUID } from "node:crypto";
+import { runMain } from "../lib/fatal.mjs";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
 
 const ROOT = process.env.MA_AUTOPILOT_ROOT || join(homedir(), ".claude", "autopilot");
 // Siblings resolve from THIS file's own directory, not from ~/.claude/scripts.
@@ -76,6 +82,25 @@ const REGISTER_GRACE_MS = Number(process.env.MA_AP_GRACE_MS || 60000);
 // A session in one of these is still working. Anything else - and a row with
 // no status at all is not "anything else" - ends supervision.
 const LIVE_STATUSES = new Set(["busy", "running", "waiting"]);
+// launchd appends this process's stdout to runner.log forever. On a laptop that
+// is a file nobody ever looks at; on a machine that ticks every few minutes for
+// months it is the thing that fills the disk, and a full disk stops the runs it
+// was logging.
+const LOG_MAX_BYTES = Number(process.env.MA_AP_LOG_MAX_BYTES || 5 * 1024 * 1024);
+// Consecutive failed attempts before the runner stops taking NEW work. The
+// failure this guards against is a machine-level one - an expired token, a full
+// disk, a `claude` that no longer launches - where every item fails the same
+// way and the queue is consumed one worthless run at a time. Zero disables it.
+const BREAKER_LIMIT = Number(process.env.MA_AP_BREAKER_LIMIT || 3);
+// How long an open breaker waits before letting ONE attempt through. Long
+// enough that a broken machine is not burning the queue (default 30 min, so at
+// a 15-minute tick it is every other tick at most), short enough that a machine
+// fixed at 3am is working again by morning without anyone touching it.
+const BREAKER_COOLDOWN_SEC = Number(process.env.MA_AP_BREAKER_COOLDOWN_SEC || 1800);
+// Outcomes that mean the attempt produced nothing. `needs-input` is NOT here:
+// a run parked on a question did work and is waiting for a person, which is the
+// system behaving correctly.
+const FAILED_OUTCOMES = new Set(["launch-failed", "timed-out", "no-state", "failed", "unknown"]);
 
 const log = (m) => process.stdout.write(`autopilot-runner: ${m}\n`);
 
@@ -120,6 +145,111 @@ function record(entry) {
   chmodSync(p, 0o600);
 }
 
+/**
+ * Keep runner.log bounded without breaking the writer.
+ *
+ * launchd holds the file open with O_APPEND, so RENAMING the log leaves that fd
+ * writing to the renamed inode and the new file stays empty forever - the
+ * rotation that looks right is the one that silently stops logging. Truncating
+ * the SAME inode is what works: the open fd keeps writing, at offset zero.
+ * The tail is copied aside first, because the last few hundred lines are the
+ * ones somebody is about to need.
+ */
+function rotateLog() {
+  const p = join(ROOT, "runner.log");
+  let size;
+  try {
+    size = statSync(p).size;
+  } catch {
+    // No log yet (a runner that has never been started by launchd), or it is
+    // unreadable. Nothing to rotate either way.
+    return;
+  }
+  if (size <= LOG_MAX_BYTES) return;
+  try {
+    const keep = Math.min(size, 256 * 1024);
+    const fd = openSync(p, "r");
+    const buf = Buffer.alloc(keep);
+    readSync(fd, buf, 0, keep, size - keep);
+    closeSync(fd);
+    writeFileSync(join(ROOT, "runner.log.1"), buf, { mode: 0o600 });
+    // Truncate in place. Not unlink, not rename.
+    truncateSync(p, 0);
+    log(
+      `runner.log reached ${Math.round(size / 1048576)}MB - tail kept in runner.log.1, log truncated`,
+    );
+  } catch {
+    // A log that cannot be rotated is not a reason to skip the tick.
+  }
+}
+
+/**
+ * One line per tick, machine-readable, next to the human log. The human log
+ * answers "what happened just now"; this answers "how has it been behaving for
+ * a week", which is the question a server actually raises and the one prose
+ * cannot be asked.
+ */
+function tick(entry) {
+  try {
+    ensureRoot();
+    const p = join(ROOT, "ticks.jsonl");
+    appendFileSync(p, JSON.stringify({ at: new Date().toISOString(), ...entry }) + "\n", {
+      mode: 0o600,
+    });
+  } catch {
+    // Telemetry never fails a tick.
+  }
+}
+
+/**
+ * How many attempts in a row produced nothing.
+ *
+ * Read from attempted.jsonl rather than kept as a counter, because a counter is
+ * a second piece of state that can disagree with the ledger - and the ledger is
+ * what a person reads when they ask why the runner stopped.
+ *
+ * @returns {{count:number, last:string|null}}
+ */
+export function consecutiveFailures(lines) {
+  let count = 0;
+  let last = null;
+  for (let i = lines.length - 1; i >= 0; i--) {
+    const row = lines[i];
+    if (!row || typeof row.outcome !== "string") continue;
+    // A blocked item never ran: arming refused it, which says nothing about
+    // whether a run would have worked.
+    if (row.outcome.startsWith("blocked-")) continue;
+    if (FAILED_OUTCOMES.has(row.outcome)) {
+      count++;
+      if (!last) last = row.outcome;
+      continue;
+    }
+    break;
+  }
+  return { count, last };
+}
+
+function readAttempted() {
+  const p = join(ROOT, "attempted.jsonl");
+  if (!existsSync(p)) return [];
+  try {
+    return readFileSync(p, "utf-8")
+      .split("\n")
+      .filter(Boolean)
+      .slice(-50)
+      .map((l) => {
+        try {
+          return JSON.parse(l);
+        } catch {
+          return null;
+        }
+      })
+      .filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
 /** Seconds since the epoch at which this machine booted. */
 export function bootTime() {
   const out = run("sysctl", ["-n", "kern.boottime"]);
@@ -446,6 +576,61 @@ function main() {
     return 0;
   }
 
+  rotateLog();
+
+  // ---- 1b. BREAKER -------------------------------------------------------
+  // Three failures in a row is not three unlucky items, it is one broken
+  // machine: an expired token, a `claude` that no longer launches, a full disk.
+  // Left alone the runner eats the whole queue at a few minutes an item and
+  // records a failure for each, which destroys the evidence of WHICH item was
+  // first and leaves nothing to resume. Stopping keeps the queue intact.
+  const attempts = readAttempted();
+  const breaker = consecutiveFailures(attempts);
+  if (BREAKER_LIMIT > 0 && breaker.count >= BREAKER_LIMIT) {
+    // HALF-OPEN, not latched. The first version of this returned here on every
+    // tick, and the only writer of attempted.jsonl is downstream of this
+    // return - so once it opened, no new attempt could ever be recorded, the
+    // consecutive count could never fall, and the runner was stopped for good.
+    // The log line said "until an attempt succeeds" and the feature doc said
+    // "one successful attempt clears it": both described a state the code made
+    // unreachable.
+    //
+    // A breaker that cannot re-close is not a breaker, it is an off switch with
+    // a misleading label. So after a cooldown one probe is let through: if the
+    // machine recovered it succeeds and the count resets on its own; if it did
+    // not, that probe fails, becomes the new most recent attempt, and the
+    // breaker closes again for another cooldown. One wasted run per cooldown is
+    // the price of not needing a human to notice.
+    const lastAt = attempts.length ? Number(attempts[attempts.length - 1].at || 0) : 0;
+    const sinceSec = lastAt ? Math.floor(Date.now() / 1000) - lastAt : Infinity;
+    const probeDue = sinceSec >= BREAKER_COOLDOWN_SEC;
+    if (!probeDue) {
+      const why = `${breaker.count} attempts in a row produced nothing (last: ${breaker.last})`;
+      const waitMin = Math.ceil((BREAKER_COOLDOWN_SEC - sinceSec) / 60);
+      log(`circuit breaker open - ${why}`);
+      log(
+        `  one probe run is allowed in ${waitMin} min; check the token, disk and \`claude\` binary`,
+      );
+      log("  to override for one tick: MA_AP_BREAKER_LIMIT=0");
+      if (!DRY) {
+        writeState("queue.json", {
+          ...readJson(join(ROOT, "queue.json"), { queued: [], running: [] }),
+          blockedReason: `circuit breaker: ${why} - probe in ${waitMin} min`,
+        });
+        refreshStatus();
+      }
+      tick({
+        action: "breaker-open",
+        failures: breaker.count,
+        lastOutcome: breaker.last,
+        probeInSec: BREAKER_COOLDOWN_SEC - sinceSec,
+      });
+      return 0;
+    }
+    log(`circuit breaker half-open - ${breaker.count} failures, letting one probe run through`);
+    tick({ action: "breaker-probe", failures: breaker.count, lastOutcome: breaker.last });
+  }
+
   // ---- 2. INTAKE ---------------------------------------------------------
   const intake = join(SCRIPTS, "autopilot-intake.mjs");
   if (existsSync(intake)) run(process.execPath, [intake]);
@@ -455,6 +640,7 @@ function main() {
   const slots = Number(config.slots ?? 1);
   if (running.length >= slots) {
     log(`${running.length}/${slots} slots in use - nothing to take`);
+    tick({ action: "slots-full", running: running.length, slots });
     return 0;
   }
 
@@ -466,6 +652,11 @@ function main() {
   const next = (queue.queued || []).find((i) => !busyRepos.has(i.repo));
   if (!next) {
     log(queue.emptyReason || "every queued item belongs to a repo already in flight");
+    tick({
+      action: "nothing-to-take",
+      queued: (queue.queued || []).length,
+      running: running.length,
+    });
     return 0;
   }
 
@@ -593,9 +784,22 @@ function main() {
     log(`${next.id}: claim kept for the next tick (${supervised.reason})`);
   }
   log(`${next.id}: ${outcome}${prUrl ? ` ${prUrl}` : ""} (${supervised.waitedSec}s)`);
+  tick({
+    action: "ran",
+    source: next.source,
+    id: next.id,
+    taskId,
+    outcome,
+    reason: supervised.reason,
+    waitedSec: supervised.waitedSec,
+    prOpened: Boolean(prUrl),
+    consecutiveFailuresBefore: breaker.count,
+  });
   return 0;
 }
 
-if (import.meta.url === `file://${process.argv[1]}`) {
-  process.exit(main());
+if (invokedDirectly(import.meta.url)) {
+  runMain("autopilot-runner", () => {
+    process.exit(main());
+  });
 }

package/pipeline/scripts/build-references.mjs

@@ -22,6 +22,7 @@
 // Exit codes: 0 ok, 1 coverage failure, 2 usage / parse error.
 
 import { readFileSync } from "node:fs";
+import { runMain } from "../lib/fatal.mjs";
 
 const HEADERS = {
   tr: ["Tür", "Kaynak", "URL / Yol", "Sürüm / Ref", "Rol", "Erişim", "Notlar"],
@@ -385,4 +386,4 @@ function main() {
   process.stdout.write(`${renderTable(rows, lang)}\n`);
 }
 
-main();
+runMain("build-references", main);

package/pipeline/scripts/build-stack-plugins.mjs

@@ -33,7 +33,8 @@ import {
   mkdirSync,
   statSync,
 } from "node:fs";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import { spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
 
@@ -65,7 +66,14 @@ if (!HOME) {
   process.exit(2);
 }
 const PLUGINS_REPO = getArg("--plugins-repo", join(HOME, "multi-agent-plugins"));
-const PIPE_ROOT = getArg("--pipeline", join(HOME, "multi-agent-pipeline"));
+// The pipeline root is THIS file's repo, found by walking up from the script,
+// not a guess at where the checkout lives. It defaulted to
+// $HOME/multi-agent-pipeline, which is true on the maintainer's machine and
+// nowhere else: on a CI runner the checkout is under the workspace directory,
+// so --check-routing reported "source not found" and three tests failed on
+// every clean host while passing locally. That is the whole class of bug CI
+// exists to catch, and it survived because CI was asleep.
+const PIPE_ROOT = getArg("--pipeline", join(dirname(fileURLToPath(import.meta.url)), "..", ".."));
 const EXTERNAL = join(PIPE_ROOT, "pipeline/skills/shared/external");
 const DRY = args.includes("--dry-run");
 

package/pipeline/scripts/capture-evidence.sh

@@ -207,7 +207,10 @@ case "$MODE" in
       # stale video from yesterday's run attached as today's evidence is worse
       # than no video, because nobody re-checks an artefact that is present.
       SRC=""
-      for cand in $(find test-results cypress/videos -type f \( -name '*.webm' -o -name '*.mp4' \) 2>/dev/null); do
+      # `while read`, not `for cand in $(find ...)`: word splitting turns one
+      # path with a space into two candidates, both of which fail `[ -f ]`, and
+      # the evidence goes missing without a word said.
+      while IFS= read -r cand; do
         [ -f "$cand" ] || continue
         MT=$(date -r "$cand" +%s 2>/dev/null || echo 0)
         [ "$MT" -ge "$SINCE" ] 2>/dev/null || continue
@@ -215,7 +218,9 @@ case "$MODE" in
           PREV=$(date -r "$SRC" +%s 2>/dev/null || echo 0)
           [ "$MT" -gt "$PREV" ] 2>/dev/null && SRC="$cand"
         fi
-      done
+      done <<EOF
+$(find test-results cypress/videos -type f \( -name '*.webm' -o -name '*.mp4' \) 2>/dev/null)
+EOF
       [ -n "$SRC" ] || { echo "capture-evidence: the suite recorded no video after the marker (is video enabled in the project's runner config?)" >&2; exit 4; }
       case "$SRC" in
         *.webm)

package/pipeline/scripts/classify-plan-safety.mjs

@@ -197,5 +197,6 @@ if (recommendPause) {
   summary = `Low-risk plan  -  autopilot safe`;
 }
 
+// See run-metrics.mjs: exiting here would truncate this write when stdout is a
+// pipe, and 0 is the default code anyway.
 process.stdout.write(JSON.stringify({ score, recommendPause, reasons, summary }, null, 2) + "\n");
-process.exit(0);