@mmerterden/multi-agent-pipeline 20.2.1 → 20.3.0

package/pipeline/scripts/autopilot-awake.mjs

@@ -0,0 +1,255 @@
+#!/usr/bin/env node
+/**
+ * autopilot-awake.mjs  -  the launchd job that keeps the Mac awake while
+ * autopilot is on.
+ *
+ * The schedule's StartInterval does not wake a sleeping Mac. The in-flight lock
+ * (`caffeinate -s -w <runner pid>`, _autopilot-ops.mjs) covers a run while it is
+ * going, and nothing covers the gap between ticks: the machine sleeps, the next
+ * tick never fires, and the queue stops with nothing to say so. This job closes
+ * that gap. It is a second, separate launchd agent that runs
+ * `/usr/bin/caffeinate -s -i` (plus `-d` with `awake.display: true`) for as long
+ * as it is loaded, and it is loaded exactly while the mode is on.
+ *
+ *   render  [--display|--no-display]   print the plist
+ *   install [--display|--no-display]   write it into LaunchAgents and load it;
+ *                                      without a flag, awake.display from the
+ *                                      config decides. awake.enabled: false
+ *                                      removes the job instead.
+ *   remove                             unload it and delete the plist
+ *   status  [--json]                   { held, display }
+ *
+ * Injectable for tests: MA_AP_LAUNCH_AGENTS (the LaunchAgents dir),
+ * MA_AP_LAUNCHCTL_BIN, MA_AP_PS_BIN, MA_AP_AWAKE_TEMPLATE, MA_AUTOPILOT_ROOT.
+ *
+ * Exit: 0 on success, 1 when launchd refused the job, 2 on usage.
+ */
+
+import { spawnSync } from "node:child_process";
+import {
+  chmodSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+
+export const AWAKE_LABEL = "com.multi-agent.autopilot-awake";
+export const CAFFEINATE = "/usr/bin/caffeinate";
+const TEMPLATE_NAME = "multi-agent-autopilot-awake.plist.template";
+const HERE = dirname(fileURLToPath(import.meta.url));
+const LAUNCHCTL_TIMEOUT_MS = 15000;
+
+function home(env) {
+  return env.HOME || homedir();
+}
+
+function readJson(p) {
+  try {
+    return JSON.parse(readFileSync(p, "utf-8"));
+  } catch {
+    return null;
+  }
+}
+
+/** The config's `awake` block with its defaults: on, display allowed to sleep. */
+export function awakeSettings(config) {
+  const a = config?.awake;
+  return { enabled: a?.enabled !== false, display: a?.display === true };
+}
+
+/**
+ * `-s` holds system sleep off and macOS honours it on AC only; `-i` holds idle
+ * sleep off on battery as well; `-d` keeps the display on and is opt-in,
+ * because nothing autopilot runs needs a lit screen.
+ */
+export function caffeinateArgs({ display = false } = {}) {
+  return display ? ["-s", "-i", "-d"] : ["-s", "-i"];
+}
+
+/** The installed tree keeps templates/ beside scripts/; the repo keeps it in install/. */
+export function templatePath(env = process.env) {
+  const candidates = [
+    env.MA_AP_AWAKE_TEMPLATE,
+    join(HERE, "..", "templates", TEMPLATE_NAME),
+    join(HERE, "..", "..", "install", "templates", TEMPLATE_NAME),
+    ...[".claude", ".copilot", ".codex"].map((d) => join(home(env), d, "templates", TEMPLATE_NAME)),
+  ].filter(Boolean);
+  return candidates.find((p) => existsSync(p)) || null;
+}
+
+function xmlEscape(s) {
+  return String(s).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+
+export function renderAwakePlist({ display = false, label = AWAKE_LABEL, env = process.env } = {}) {
+  const tpl = templatePath(env);
+  if (!tpl) throw new Error(`${TEMPLATE_NAME} is not installed - re-run the installer`);
+  const args = [CAFFEINATE, ...caffeinateArgs({ display })]
+    .map((a) => `    <string>${xmlEscape(a)}</string>`)
+    .join("\n");
+  return readFileSync(tpl, "utf-8")
+    .replace(/\{\{LABEL\}\}/g, xmlEscape(label))
+    .replace(/\{\{ARGS\}\}/g, args);
+}
+
+export function launchAgentsDir(env = process.env) {
+  return env.MA_AP_LAUNCH_AGENTS || join(home(env), "Library", "LaunchAgents");
+}
+
+export function awakePlistPath(env = process.env) {
+  return join(launchAgentsDir(env), `${AWAKE_LABEL}.plist`);
+}
+
+function autopilotConfig(env) {
+  const root = env.MA_AUTOPILOT_ROOT || join(home(env), ".claude", "autopilot");
+  return readJson(join(root, "config.json"));
+}
+
+function launchctl(args, env) {
+  const r = spawnSync(env.MA_AP_LAUNCHCTL_BIN || "launchctl", args, {
+    encoding: "utf-8",
+    stdio: ["ignore", "pipe", "pipe"],
+    timeout: LAUNCHCTL_TIMEOUT_MS,
+  });
+  return { ok: !r.error && r.status === 0, stdout: String(r.stdout || "") };
+}
+
+function domain() {
+  return `gui/${process.getuid()}`;
+}
+
+/** Unload the job and delete its plist. Absent is success. */
+export function removeAwake({ env = process.env } = {}) {
+  const path = awakePlistPath(env);
+  if (!launchctl(["bootout", `${domain()}/${AWAKE_LABEL}`], env).ok && existsSync(path)) {
+    launchctl(["unload", path], env);
+  }
+  rmSync(path, { force: true });
+  return { path, removed: true };
+}
+
+/**
+ * Write the plist and load it. A job already loaded is booted out first, so a
+ * changed display answer takes effect now rather than at the next login.
+ *
+ * @returns {{path:string|null, loaded:boolean, display:boolean, enabled:boolean}}
+ */
+export function installAwake({ env = process.env, display } = {}) {
+  const settings = awakeSettings(autopilotConfig(env));
+  if (!settings.enabled) {
+    removeAwake({ env });
+    return { path: null, loaded: false, display: false, enabled: false };
+  }
+  const shown = typeof display === "boolean" ? display : settings.display;
+  const xml = renderAwakePlist({ display: shown, env });
+  const path = awakePlistPath(env);
+  mkdirSync(dirname(path), { recursive: true });
+  const tmp = `${path}.${process.pid}.tmp`;
+  writeFileSync(tmp, xml, { mode: 0o644 });
+  chmodSync(tmp, 0o644);
+  launchctl(["bootout", `${domain()}/${AWAKE_LABEL}`], env);
+  renameSync(tmp, path);
+  const loaded =
+    launchctl(["bootstrap", domain(), path], env).ok || launchctl(["load", path], env).ok;
+  return { path, loaded, display: shown, enabled: true };
+}
+
+/** `caffeinate -s -i [-d]` and nothing after it: the in-flight lock carries `-w <pid>`. */
+const AWAKE_PS = /(^|\/)caffeinate -s -i( -d)?\s*$/;
+
+/**
+ * Held when launchd reports a running PID for the label, or, when launchd
+ * cannot say, when a `caffeinate -s -i` process is running. The display answer
+ * comes from the installed plist, else from that process's arguments.
+ *
+ * @returns {{held:boolean, display:boolean}}
+ */
+export function awakeStatus({ env = process.env } = {}) {
+  const path = awakePlistPath(env);
+  let plistDisplay = null;
+  if (existsSync(path)) {
+    try {
+      plistDisplay = /<string>-d<\/string>/.test(readFileSync(path, "utf-8"));
+    } catch {
+      plistDisplay = null;
+    }
+  }
+  const listed = launchctl(["list", AWAKE_LABEL], env);
+  if (listed.ok && /"PID"\s*=\s*\d+/.test(listed.stdout)) {
+    return { held: true, display: plistDisplay === true };
+  }
+  const ps = spawnSync(env.MA_AP_PS_BIN || "ps", ["-axo", "command="], {
+    encoding: "utf-8",
+    stdio: ["ignore", "pipe", "ignore"],
+    timeout: LAUNCHCTL_TIMEOUT_MS,
+  });
+  const line = String(ps.stdout || "")
+    .split("\n")
+    .map((l) => l.trim().replace(/^\d+\s+/, ""))
+    .find((l) => AWAKE_PS.test(l));
+  if (line) return { held: true, display: plistDisplay ?? / -d\s*$/.test(line) };
+  return { held: false, display: plistDisplay === true };
+}
+
+function displayFlag(args) {
+  if (args.includes("--display")) return true;
+  if (args.includes("--no-display")) return false;
+  return undefined;
+}
+
+function main(argv) {
+  const [cmd, ...rest] = argv;
+  const env = process.env;
+  switch (cmd) {
+    case "render":
+      process.stdout.write(renderAwakePlist({ display: displayFlag(rest) === true, env }));
+      return 0;
+    case "install": {
+      const r = installAwake({ env, display: displayFlag(rest) });
+      if (!r.enabled) {
+        console.log("awake: disabled in the config (awake.enabled: false) - no job installed");
+        return 0;
+      }
+      if (!r.loaded) {
+        console.error(`awake: launchd refused ${r.path}`);
+        return 1;
+      }
+      console.log(`awake: ${r.path} loaded (display ${r.display ? "kept on" : "may sleep"})`);
+      return 0;
+    }
+    case "remove":
+      removeAwake({ env });
+      console.log(`awake: ${AWAKE_LABEL} removed`);
+      return 0;
+    case "status": {
+      const s = awakeStatus({ env });
+      if (rest.includes("--json")) console.log(JSON.stringify(s));
+      else
+        console.log(
+          `awake: ${s.held ? "held" : "not held"}, display ${s.display ? "on" : "may sleep"}`,
+        );
+      return 0;
+    }
+    default:
+      console.error(
+        "usage: autopilot-awake.mjs {render|install|remove|status} [--display|--no-display|--json]",
+      );
+      return 2;
+  }
+}
+
+if (invokedDirectly(import.meta.url)) {
+  try {
+    process.exitCode = main(process.argv.slice(2));
+  } catch (e) {
+    console.error(`awake: ${e?.message || e}`);
+    process.exitCode = 1;
+  }
+}

package/pipeline/scripts/autopilot-intake.mjs

@@ -32,7 +32,9 @@
  *
  * Exit: 0 always when it can read its own state. An empty queue is an answer,
  * not an error - "no item carries the label yet" is the single most common
- * reason nothing is happening and it is not a fault.
+ * reason nothing is happening and it is not a fault. 3 when config.json breaks
+ * autopilot-config.schema.json: queue.json then holds no queued item and a
+ * `blockedReason` naming the field.
  */
 
 import { execFileSync } from "node:child_process";
@@ -40,6 +42,8 @@ import { existsSync, readFileSync, mkdirSync, writeFileSync, renameSync, chmodSy
 import { join, dirname } from "node:path";
 import { homedir } from "node:os";
 import { invokedDirectly } from "../lib/invoked-directly.mjs";
+import { OUTCOME, isParked, isRetryable, isTerminal } from "./_autopilot-outcomes.mjs";
+import { configRefusal } from "./_autopilot-config.mjs";
 
 const ROOT = process.env.MA_AUTOPILOT_ROOT || join(homedir(), ".claude", "autopilot");
 
@@ -55,16 +59,13 @@ const PRIORITY = {
   lowest: 10,
 };
 
-/**
- * An outcome that means the item is finished with us. Anything not listed is
- * either retryable or a waiting state, and the difference matters: treating
- * `died` as terminal drops an item because a laptop lid closed once.
+/*
+ * What an outcome means - finished, worth another attempt, or waiting for a
+ * person - is _autopilot-outcomes.mjs, the one contract the runner writes and
+ * this file reads. A `blocked-*` outcome is none of the three: the token being
+ * expired is not the item's fault, and it becomes workable again the moment the
+ * token is replaced, so it is queued again without counting an attempt.
  */
-const TERMINAL = new Set(["pr-opened", "merged", "skipped"]);
-/** Counted against `maxAttempts`. A credential block is NOT one of these - the
- *  token being expired is not the item's fault, and it becomes workable again
- *  the moment the token is replaced. */
-const RETRYABLE = new Set(["died", "build-failed"]);
 
 export function priorityScore(name) {
   if (!name) return 0;
@@ -137,7 +138,14 @@ export function orderQueue(items, { maxConsecutivePerRepo = 5 } = {}) {
   return out;
 }
 
-/** Attempt history, folded into one record per item. */
+/**
+ * Attempt history, folded into one record per item.
+ *
+ * Every RETRYABLE outcome counts toward `maxAttempts`, and counts ONCE per run:
+ * a run whose supervision timed out is recorded as `timed-out`, and when a
+ * later tick retires it the same run is recorded again as `died`. Both carry
+ * the runner's `taskId`, which is what makes them one attempt and not two.
+ */
 export function foldAttempts(lines) {
   const byKey = new Map();
   for (const raw of lines) {
@@ -150,8 +158,14 @@ export function foldAttempts(lines) {
       continue; // a truncated last line is normal on an append-only log
     }
     const key = `${rec.source}:${rec.id}`;
-    const cur = byKey.get(key) || { attempts: 0, last: null, usd: 0, at: 0 };
-    if (RETRYABLE.has(rec.outcome)) cur.attempts += 1;
+    const cur = byKey.get(key) || { attempts: 0, last: null, usd: 0, at: 0, runs: new Set() };
+    if (isRetryable(rec.outcome)) {
+      if (!rec.taskId) cur.attempts += 1;
+      else if (!cur.runs.has(rec.taskId)) {
+        cur.runs.add(rec.taskId);
+        cur.attempts += 1;
+      }
+    }
     cur.usd += Number(rec.usd || 0);
     if ((rec.at || 0) >= cur.at) {
       cur.at = rec.at || 0;
@@ -164,6 +178,18 @@ export function foldAttempts(lines) {
   return byKey;
 }
 
+/**
+ * The key a GitHub item was recorded under when its id carried only the short
+ * repo name (`app#5`). Read as a fallback so an item finished under the old id
+ * is not queued again; where two repos share the short name this errs toward
+ * leaving the item alone, never toward running it twice.
+ */
+function legacyKey(c) {
+  if (c?.source !== "github" || typeof c.id !== "string") return null;
+  const m = /^[^/#]+\/([^/#]+)#(\d+)$/.exec(c.id);
+  return m ? `github:${m[1]}#${m[2]}` : null;
+}
+
 /**
  * Which candidates may enter the queue, and WHY each excluded one was excluded.
  * Every drop carries a reason: a queue that silently shortens is the thing that
@@ -174,17 +200,24 @@ export function selectable(candidates, history, { maxAttempts = 2 } = {}) {
   const awaiting = [];
   const dropped = [];
   for (const c of candidates) {
-    const h = history.get(`${c.source}:${c.id}`);
+    const h = history.get(`${c.source}:${c.id}`) ?? history.get(legacyKey(c));
     if (!h) {
       queued.push(c);
       continue;
     }
-    if (TERMINAL.has(h.last)) {
+    if (isTerminal(h.last)) {
       dropped.push({ ...c, reason: `already ${h.last}`, prUrl: h.prUrl });
       continue;
     }
-    if (h.last === "awaiting-answer") {
-      awaiting.push({ ...c, askedAt: h.at, gapFingerprint: h.gapFingerprint });
+    // Parked for a person: never re-run on its own. `awaiting-answer` is the
+    // maturity question; the others wait just the same, and say why.
+    if (isParked(h.last)) {
+      awaiting.push({
+        ...c,
+        askedAt: h.at,
+        gapFingerprint: h.gapFingerprint,
+        ...(h.last === OUTCOME.AWAITING_ANSWER ? {} : { waitingOn: h.last }),
+      });
       continue;
     }
     if (h.attempts >= maxAttempts) {
@@ -234,10 +267,21 @@ function fetchGitHub(repo) {
   } catch {
     return [];
   }
+  return githubCandidates(repo, rows);
+}
+
+/**
+ * Queue items from `gh issue list` rows. The id carries the full owner/repo:
+ * two repos may share a short name, and an id made from the short name alone
+ * would let one repo's finished issue drop the other's. `ref` stays short -
+ * the run is launched in the repo's own checkout, which names the repo.
+ */
+export function githubCandidates(repo, rows) {
+  const label = repo.githubLabel || "agent-queue";
   const short = repo.nameWithOwner.split("/").pop();
-  return rows.map((r) => ({
+  return (rows || []).map((r) => ({
     source: "github",
-    id: `${short}#${r.number}`,
+    id: `${repo.nameWithOwner}#${r.number}`,
     ref: `${short}#${r.number}`,
     repo: repo.nameWithOwner,
     localPath: repo.localPath,
@@ -278,7 +322,12 @@ function fetchJira(repo) {
   } catch {
     return [];
   }
-  return rows.map((i) => ({
+  return jiraCandidates(repo, rows);
+}
+
+/** Queue items from a Jira search's `issues`. The issue key is the id. */
+export function jiraCandidates(repo, issues) {
+  return (issues || []).map((i) => ({
     source: "jira",
     id: i.key,
     ref: i.key,
@@ -292,14 +341,42 @@ function fetchJira(repo) {
   }));
 }
 
-function gather(config) {
+/**
+ * Every candidate from every configured repo.
+ *
+ * A Jira issue is one item whichever repo's search returned it. The default
+ * JQL (assignee plus label) is the same for every repo, so without this each
+ * repo with Jira on would queue - and run, and open a PR for - the same issue.
+ * The first repo in config order takes it; each other match is returned in
+ * `duplicates` with the reason, so the queue says why it is not listed twice.
+ *
+ * @returns {{items: object[], duplicates: object[]}}
+ */
+export function gather(
+  config,
+  { fetchGitHub: gh = fetchGitHub, fetchJira: jira = fetchJira } = {},
+) {
   const items = [];
+  const duplicates = [];
+  const jiraOwner = new Map();
   for (const repo of config.repos || []) {
     const sources = repo.sources || ["github"];
-    if (sources.includes("github")) items.push(...fetchGitHub(repo));
-    if (sources.includes("jira")) items.push(...fetchJira(repo));
+    if (sources.includes("github")) items.push(...gh(repo));
+    if (!sources.includes("jira")) continue;
+    for (const it of jira(repo)) {
+      const owner = jiraOwner.get(it.id);
+      if (owner === undefined) {
+        jiraOwner.set(it.id, it.repo);
+        items.push(it);
+      } else if (owner !== it.repo) {
+        duplicates.push({
+          ...it,
+          reason: `also matched for ${owner}, queued there - give each repo a jiraJql that matches only its own issues`,
+        });
+      }
+    }
   }
-  return items;
+  return { items, duplicates };
 }
 
 // ---------------------------------------------------------------------------
@@ -313,6 +390,17 @@ function atomicWrite(name, text) {
   renameSync(tmp, dst);
 }
 
+function readQueue() {
+  const qPath = join(ROOT, "queue.json");
+  if (!existsSync(qPath)) return { queued: [], running: [] };
+  try {
+    const q = JSON.parse(readFileSync(qPath, "utf-8"));
+    return q && typeof q === "object" && !Array.isArray(q) ? q : { queued: [], running: [] };
+  } catch {
+    return { queued: [], running: [] };
+  }
+}
+
 function main(argv) {
   const args = argv.slice(2);
   const has = (f) => args.includes(f);
@@ -326,10 +414,31 @@ function main(argv) {
   }
 
   const cfgPath = join(ROOT, "config.json");
-  const config = existsSync(cfgPath) ? JSON.parse(readFileSync(cfgPath, "utf-8")) : { repos: [] };
+  let config = { repos: [] };
+  let refusal = null;
+  if (existsSync(cfgPath)) {
+    try {
+      config = JSON.parse(readFileSync(cfgPath, "utf-8"));
+      refusal = configRefusal(config);
+    } catch (e) {
+      refusal = `autopilot config.json cannot be read: ${e?.message || e}`;
+    }
+  }
+  // A ceiling that is not a number compares false with everything, which would
+  // retry an item forever. Nothing is queued until the config is corrected.
+  if (refusal) {
+    const queue = { ...readQueue(), lastTickAt: Math.floor(Date.now() / 1000) };
+    queue.queued = [];
+    queue.blockedReason = refusal;
+    if (!has("--dry-run")) atomicWrite("queue.json", JSON.stringify(queue, null, 2));
+    process.stderr.write(`autopilot-intake: ${refusal}\n`);
+    return 3;
+  }
 
   const fromFile = val("--from");
-  const candidates = fromFile ? JSON.parse(readFileSync(fromFile, "utf-8")) : gather(config);
+  const { items: candidates, duplicates } = fromFile
+    ? { items: JSON.parse(readFileSync(fromFile, "utf-8")), duplicates: [] }
+    : gather(config);
 
   const attemptPath = join(ROOT, "attempted.jsonl");
   const history = foldAttempts(
@@ -347,22 +456,14 @@ function main(argv) {
   // The existing queue's `running` list is preserved: intake decides what is
   // NEXT and has no opinion about what is already in flight. Overwriting it here
   // would make a scan look like a crash to the status reader.
-  let running = [];
-  const qPath = join(ROOT, "queue.json");
-  if (existsSync(qPath)) {
-    try {
-      running = JSON.parse(readFileSync(qPath, "utf-8")).running || [];
-    } catch {
-      running = [];
-    }
-  }
+  const running = readQueue().running || [];
 
   const queue = {
     lastTickAt: Math.floor(Date.now() / 1000),
     running,
     queued: ordered,
     awaiting,
-    dropped,
+    dropped: [...dropped, ...duplicates],
     emptyReason:
       ordered.length === 0 && awaiting.length === 0
         ? candidates.length === 0