@mmerterden/multi-agent-pipeline 20.2.1 → 20.3.0

package/pipeline/scripts/_autopilot-ops.mjs

@@ -0,0 +1,567 @@
+/**
+ * _autopilot-ops.mjs  -  the operations a runner tick owns besides the run.
+ *
+ *   SLEEP LOCK   a sleep inhibitor held while a session is in flight, spawned
+ *                by the tick and killed by the pid it spawned.
+ *   PARALLEL CAP how many runner-launched sessions may be working at once.
+ *   DAILY JOBS   the cleanup report and the opt-in digest, each at most once
+ *                per local date, stamped in daily.json.
+ *
+ * Each one is a pure function or a small effect with an injectable binary, so
+ * test/autopilot-operations.test.mjs drives them without a real `caffeinate`,
+ * `claude` or tracker.
+ */
+
+import { spawn, spawnSync } from "node:child_process";
+import {
+  closeSync,
+  existsSync,
+  linkSync,
+  openSync,
+  readFileSync,
+  readdirSync,
+  realpathSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+  writeSync,
+} from "node:fs";
+import { basename, join } from "node:path";
+import { classifyOutcome, isBlocked, isParked } from "./_autopilot-outcomes.mjs";
+import { attemptSpend } from "./_cost.mjs";
+import { logsRoot as defaultLogsRoot } from "./_run-paths.mjs";
+import { readRegularJson } from "../lib/regular-file.mjs";
+
+// Reports and digests are small, but one a day for a year is a directory
+// nobody reads. The newest ones answer every question anyone asks of them.
+const KEEP_DAILY_FILES = 14;
+
+function readJson(p, fallback) {
+  try {
+    return JSON.parse(readFileSync(p, "utf-8"));
+  } catch {
+    return fallback;
+  }
+}
+
+function writePrivate(p, text) {
+  writeFileSync(p, text, { mode: 0o600 });
+}
+
+function pidAlive(pid) {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (e) {
+    return e.code === "EPERM";
+  }
+}
+
+// ---- SLEEP LOCK ------------------------------------------------------------
+
+/**
+ * The inhibitor, tied to the runner's pid so it cannot outlive the runner even
+ * when the runner is killed before it can release it.
+ *
+ * `caffeinate -s -w <pid>`. `-s` is the assertion that prevents SYSTEM sleep
+ * and is honoured only on AC power. `-w` makes caffeinate exit by itself when
+ * the runner exits. Holding sleep off between ticks, and on battery, is the
+ * awake agent's job (autopilot-awake.mjs), not this lock's: this one covers the
+ * run in flight whether or not that agent is installed.
+ *
+ * macOS only (ADR-0012): any other platform holds nothing.
+ *
+ * @returns {{bin:string, args:string[]}|null}
+ */
+export function inhibitorCommand(platform, runnerPid, env = process.env) {
+  if (platform !== "darwin") return null;
+  return { bin: env.MA_AP_INHIBIT_BIN || "caffeinate", args: ["-s", "-w", String(runnerPid)] };
+}
+
+/**
+ * Hold the sleep inhibitor for the rest of this session. `release()` is
+ * idempotent, runs on process exit as well, and kills only the process group
+ * this call spawned. inhibitor.json records it for autopilot-status and for the
+ * next tick's sweep.
+ *
+ * @returns {{held:boolean, pid:number|null, release:() => void}}
+ */
+export function holdSleep({
+  root,
+  env = process.env,
+  platform = env.MA_AP_PLATFORM || process.platform,
+  runnerPid = process.pid,
+  log = () => {},
+} = {}) {
+  const none = { held: false, pid: null, release() {} };
+  const cmd = inhibitorCommand(platform, runnerPid, env);
+  if (!cmd) {
+    log(`sleep lock: not held on ${platform} - macOS only, see ADR-0012`);
+    return none;
+  }
+  let child;
+  try {
+    // Its own process group, so release() takes the waiter with it.
+    child = spawn(cmd.bin, cmd.args, { stdio: "ignore", detached: true });
+  } catch (e) {
+    log(`sleep lock: ${cmd.bin} could not start (${e?.message || e}) - running without it`);
+    return none;
+  }
+  child.on("error", () => {});
+  if (!child.pid) {
+    log(`sleep lock: ${cmd.bin} is not available - running without it`);
+    return none;
+  }
+  child.unref();
+  const record = join(root, "inhibitor.json");
+  try {
+    writePrivate(
+      record,
+      JSON.stringify({
+        pid: child.pid,
+        runnerPid,
+        bin: cmd.bin,
+        platform,
+        since: Math.floor(Date.now() / 1000),
+      }),
+    );
+  } catch {
+    // The inhibitor works without its record; only the status line loses it.
+  }
+  let released = false;
+  const release = () => {
+    if (released) return;
+    released = true;
+    process.removeListener("exit", release);
+    for (const target of [-child.pid, child.pid]) {
+      try {
+        process.kill(target, "SIGTERM");
+        break;
+      } catch {
+        // Already gone, or the group signal is not permitted here: try the pid.
+      }
+    }
+    rmSync(record, { force: true });
+  };
+  process.on("exit", release);
+  return { held: true, pid: child.pid, release };
+}
+
+/**
+ * A record left by a runner that is gone. Its inhibitor exits by itself when
+ * that runner's pid does, so this only matters when it did not; it is killed
+ * only when the recorded pid is still running the recorded binary.
+ */
+export function sweepInhibitor(root, log = () => {}) {
+  const record = join(root, "inhibitor.json");
+  const rec = readJson(record, null);
+  if (!rec) return;
+  if (rec.runnerPid !== process.pid && pidAlive(rec.runnerPid)) return;
+  if (pidAlive(rec.pid)) {
+    const r = spawnSync("ps", ["-p", String(rec.pid), "-o", "command="], { encoding: "utf-8" });
+    if (rec.bin && String(r.stdout || "").includes(basename(rec.bin))) {
+      try {
+        process.kill(rec.pid, "SIGTERM");
+        log(`sleep lock: stopped inhibitor pid ${rec.pid} left by runner ${rec.runnerPid}`);
+      } catch {
+        // It exited between the check and the signal.
+      }
+    }
+  }
+  rmSync(record, { force: true });
+}
+
+// ---- PARALLEL CAP ----------------------------------------------------------
+
+/** The configured cap, or null for unlimited (absent, null, or not a positive integer). */
+export function parallelCap(config) {
+  const n = config?.maxParallelAgents;
+  return Number.isInteger(n) && n > 0 ? n : null;
+}
+
+/**
+ * Runner-launched sessions working right now: every entry the runner still
+ * supervises, plus each parked entry whose session is busy again (a person
+ * answered and it resumed). A parked session that is waiting is idle and does
+ * not count; one that has left the list does not either.
+ *
+ * @returns {number|undefined} undefined when the agent list cannot be read
+ */
+export function agentsInFlight(running, agentsJson) {
+  const entries = Array.isArray(running) ? running : [];
+  const active = entries.filter((r) => r?.state !== "parked").length;
+  const parked = entries.filter((r) => r?.state === "parked" && r.sessionId);
+  if (!parked.length) return active;
+  let rows;
+  try {
+    const doc = JSON.parse(String(agentsJson ?? ""));
+    rows = Array.isArray(doc) ? doc : doc?.agents || [];
+  } catch {
+    return undefined;
+  }
+  const working = new Set(["busy", "running"]);
+  const busy = parked.filter((p) =>
+    rows.some((a) => a?.sessionId === p.sessionId && working.has(a?.status)),
+  ).length;
+  return active + busy;
+}
+
+/**
+ * Create the claim file only if it does not exist. Two ticks that both passed
+ * the recovery check race here, and exactly one of them wins; the other sees
+ * false and leaves without launching.
+ *
+ * The claim is written whole to a private temp file first and then hard-linked
+ * into place: link() fails with EEXIST when the name is taken, so the exclusive
+ * create and the content arrive together, and no reader ever sees an empty or
+ * half-written claim.
+ */
+export function claimExclusive(path, obj) {
+  const tmp = `${path}.${process.pid}.${Date.now()}.claim`;
+  const fd = openSync(tmp, "wx", 0o600);
+  try {
+    writeSync(fd, JSON.stringify(obj, null, 2));
+  } finally {
+    closeSync(fd);
+  }
+  try {
+    linkSync(tmp, path);
+    return true;
+  } catch (e) {
+    if (e.code === "EEXIST") return false;
+    throw e;
+  } finally {
+    rmSync(tmp, { force: true });
+  }
+}
+
+// ---- DAILY JOBS ------------------------------------------------------------
+
+/** YYYY-MM-DD in the machine's own time zone: a day is the operator's day. */
+export function localDate(ms = Date.now()) {
+  const d = new Date(ms);
+  const pad = (n) => String(n).padStart(2, "0");
+  return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
+}
+
+export function dailyDue(root, job, date) {
+  return readJson(join(root, "daily.json"), {})?.[job] !== date;
+}
+
+export function markDaily(root, job, date) {
+  const p = join(root, "daily.json");
+  const doc = readJson(p, {}) || {};
+  writePrivate(p, JSON.stringify({ ...doc, [job]: date }, null, 2));
+}
+
+/** Keep the newest KEEP_DAILY_FILES of `<prefix><date>.json|.md`. */
+export function pruneDaily(root, prefix, keep = KEEP_DAILY_FILES) {
+  let names;
+  try {
+    names = readdirSync(root);
+  } catch {
+    return;
+  }
+  const dates = [
+    ...new Set(
+      names
+        .map((n) => new RegExp(`^${prefix}(\\d{4}-\\d{2}-\\d{2})\\.(json|md)$`).exec(n)?.[1])
+        .filter(Boolean),
+    ),
+  ].sort();
+  for (const d of dates.slice(0, Math.max(0, dates.length - keep))) {
+    for (const ext of ["json", "md"]) rmSync(join(root, `${prefix}${d}.${ext}`), { force: true });
+  }
+}
+
+/** The newest `<prefix><date>.json` in root, or "" when there is none. */
+export function latestDaily(root, prefix) {
+  try {
+    const hit = readdirSync(root)
+      .filter((n) => n.startsWith(prefix) && n.endsWith(".json"))
+      .sort()
+      .at(-1);
+    return hit ? join(root, hit) : "";
+  } catch {
+    return "";
+  }
+}
+
+function realOr(p) {
+  try {
+    return realpathSync(p);
+  } catch {
+    return p;
+  }
+}
+
+function readJsonl(p) {
+  if (!existsSync(p)) return [];
+  try {
+    return readFileSync(p, "utf-8")
+      .split("\n")
+      .filter(Boolean)
+      .map((l) => {
+        try {
+          return JSON.parse(l);
+        } catch {
+          return null;
+        }
+      })
+      .filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * The runner task ids whose item is parked now: the item's newest attempt that
+ * ran (blocked rows are refusals, not attempts) is a PARKED outcome. A parked
+ * run leaves the queue once its session ends, so the queue alone does not hold
+ * its worktree; the attempt log does.
+ */
+function parkedTaskIds(root) {
+  const latest = new Map();
+  const itemOf = new Map();
+  for (const r of readJsonl(join(root, "attempted.jsonl"))) {
+    if (!r || typeof r.outcome !== "string" || isBlocked(r.outcome)) continue;
+    const item = `${r.source}:${r.id}`;
+    latest.set(item, r.outcome);
+    if (typeof r.taskId === "string" && r.taskId) itemOf.set(r.taskId, item);
+  }
+  const parked = new Set();
+  for (const [taskId, item] of itemOf) if (isParked(latest.get(item))) parked.add(taskId);
+  return parked;
+}
+
+/**
+ * For every worktree a run state names, the session of the newest such state.
+ * Run directories are nested under their project or flat (_run-paths.mjs).
+ */
+function worktreeSessions(logs) {
+  const newest = new Map();
+  const offer = (statePath) => {
+    const doc = readRegularJson(statePath, null);
+    if (!doc || typeof doc.worktreePath !== "string" || !doc.worktreePath) return;
+    let mtimeMs;
+    try {
+      mtimeMs = statSync(statePath).mtimeMs;
+    } catch {
+      return;
+    }
+    const wt = realOr(doc.worktreePath);
+    const cur = newest.get(wt);
+    if (!cur || mtimeMs > cur.mtimeMs) {
+      newest.set(wt, {
+        sessionId: typeof doc.sessionId === "string" ? doc.sessionId : null,
+        mtimeMs,
+      });
+    }
+  };
+  const dirs = (p) => {
+    try {
+      return readdirSync(p, { withFileTypes: true }).filter((e) => e.isDirectory());
+    } catch {
+      return [];
+    }
+  };
+  for (const e of dirs(logs)) {
+    const top = join(logs, e.name);
+    if (existsSync(join(top, "agent-state.json"))) offer(join(top, "agent-state.json"));
+    for (const k of dirs(top)) {
+      const p = join(top, k.name, "agent-state.json");
+      if (existsSync(p)) offer(p);
+    }
+  }
+  return newest;
+}
+
+/**
+ * Worktrees this runner created, from worktrees.jsonl, minus any the queue
+ * still holds and any whose item is parked for a person. The ledger is written
+ * only for a run whose state carries the session id the runner launched it
+ * with - the same proof `createdWorktree` rests on - and that proof is checked
+ * again here: a path is listed only while the newest run state naming it
+ * carries the ledger row's session. A worktree an attended run later opened at
+ * the same path, a hand-made one, or one no state names never appears here.
+ */
+export function runnerWorktrees(root, queue, { logsRoot = defaultLogsRoot() } = {}) {
+  const held = new Set(
+    (queue?.running || []).map((r) => (r?.worktree ? realOr(r.worktree) : null)).filter(Boolean),
+  );
+  const parked = parkedTaskIds(root);
+  const rows = readJsonl(join(root, "worktrees.jsonl")).filter(
+    (row) => typeof row?.worktree === "string" && row.worktree,
+  );
+  for (const row of rows) {
+    if (typeof row.taskId === "string" && parked.has(row.taskId)) held.add(realOr(row.worktree));
+  }
+  const owners = worktreeSessions(logsRoot);
+  const seen = new Set();
+  for (const row of rows) {
+    const p = realOr(row.worktree);
+    if (held.has(p) || !existsSync(p)) continue;
+    const owner = owners.get(p);
+    if (!owner || !row.sessionId || owner.sessionId !== row.sessionId) continue;
+    seen.add(p);
+  }
+  return [...seen].sort();
+}
+
+const GC_ACTIONS = {
+  "would-remove": "would be removed",
+  "would-stash": "would be stashed (uncommitted work), worktree kept",
+  "would-mark": "would be marked abandoned (no removable worktree)",
+  removed: "removed",
+  "remove-failed": "could not be removed (git worktree remove refused), directory kept",
+  stashed: "stashed (uncommitted work), worktree kept",
+  marked: "marked abandoned (no removable worktree)",
+  unattributed: "no run state on record, left alone",
+};
+
+/**
+ * The daily cleanup report. gc-abandoned.sh decides, restricted with `--only`
+ * to the worktrees this runner created; its dry run is the default, and
+ * `gc.autoDelete: true` is the only thing that adds `--yes`.
+ *
+ * @returns {{json:string, md:string, candidates:number, mode:string}}
+ */
+export function gcReport({ root, gcScript, logsRoot, config, queue, date, log = () => {} }) {
+  const mode = config?.gc?.autoDelete === true ? "delete" : "dry-run";
+  const worktrees = runnerWorktrees(root, queue, { logsRoot });
+  let candidates = [];
+  let summary = "no runner-created worktree on record";
+  if (worktrees.length && !existsSync(gcScript)) {
+    summary = `${basename(gcScript)} is not installed beside the runner`;
+  } else if (worktrees.length) {
+    const only = join(root, "gc-only.txt");
+    const out = join(root, "gc-report.jsonl");
+    writePrivate(only, worktrees.join("\n") + "\n");
+    rmSync(out, { force: true });
+    const r = spawnSync(
+      "bash",
+      [
+        gcScript,
+        "--logs",
+        logsRoot,
+        "--only",
+        only,
+        "--report",
+        out,
+        ...(mode === "delete" ? ["--yes"] : []),
+      ],
+      { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], timeout: 10 * 60 * 1000 },
+    );
+    candidates = readJsonl(out);
+    summary =
+      String(r.stdout || "")
+        .trim()
+        .split("\n")
+        .at(-1) || `gc-abandoned exited ${r.status}`;
+    rmSync(out, { force: true });
+    rmSync(only, { force: true });
+  }
+  const doc = {
+    date,
+    generatedAt: new Date().toISOString(),
+    mode,
+    autoDelete: mode === "delete",
+    considered: worktrees.length,
+    candidates,
+    summary,
+  };
+  const json = join(root, `gc-report-${date}.json`);
+  const md = join(root, `gc-report-${date}.md`);
+  writePrivate(json, JSON.stringify(doc, null, 2) + "\n");
+  const lines = [
+    `# Cleanup report ${date}`,
+    "",
+    `Mode: ${mode === "delete" ? "delete (gc.autoDelete is on)" : "dry run - nothing was removed"}.`,
+    `Runner-created worktrees considered: ${worktrees.length}.`,
+    "",
+    ...(candidates.length
+      ? candidates.map(
+          (c) =>
+            `- ${c.path || c.task}: ${GC_ACTIONS[c.action] || c.action} - ${c.reason}` +
+            (Number.isFinite(c.idleDays) ? `, idle ${c.idleDays}d` : ""),
+        )
+      : ["Nothing to clean up."]),
+    "",
+    summary,
+    "",
+  ];
+  writePrivate(md, lines.join("\n"));
+  pruneDaily(root, "gc-report-");
+  log(`cleanup report: ${candidates.length} candidate(s), ${mode} - ${json}`);
+  return { json, md, candidates: candidates.length, mode };
+}
+
+/**
+ * One day of the queue, from the attempt log and the parked entries.
+ *
+ * `items` counts distinct items that RAN: a blocked row is an arming refusal,
+ * written on every tick it holds, and counting it would make a quiet day look
+ * busy. Blocked rows are still counted in `byClass`, since a day of refusals is
+ * the thing the digest exists to surface. Cost follows the spend ceiling's
+ * rule (_cost.mjs attemptSpend): each run once, at its latest cumulative
+ * figure, and an unmeasured run as unknown, never as zero.
+ */
+export function buildDigest({ lines, queue, now = Date.now() / 1000, ceilingUsd = null }) {
+  const cutoff = now - 86400;
+  const rows = (lines || []).filter((r) => r && Number(r.at || 0) >= cutoff);
+  const byClass = {};
+  const byOutcome = {};
+  const items = new Set();
+  const prsOpened = [];
+  for (const r of rows) {
+    const cls = classifyOutcome(r.outcome) || "unclassified";
+    byClass[cls] = (byClass[cls] || 0) + 1;
+    byOutcome[r.outcome] = (byOutcome[r.outcome] || 0) + 1;
+    if (!isBlocked(r.outcome)) items.add(`${r.source}:${r.id}`);
+    if (typeof r.prUrl === "string" && r.prUrl && !prsOpened.includes(r.prUrl)) {
+      prsOpened.push(r.prUrl);
+    }
+  }
+  const { usd, unknown: unknownRuns } = attemptSpend(rows);
+  const awaitingAnswer = (queue?.running || [])
+    .filter((e) => e?.state === "parked")
+    .map((e) => ({ id: e.id, outcome: e.outcome ?? null, parkedAt: e.parkedAt ?? null }));
+  return {
+    windowStart: new Date(cutoff * 1000).toISOString(),
+    windowEnd: new Date(now * 1000).toISOString(),
+    items: items.size,
+    attempts: rows.filter((r) => !isBlocked(r.outcome)).length,
+    byClass,
+    byOutcome,
+    prsOpened,
+    awaitingAnswer,
+    cost: { usd: Math.round(usd * 100) / 100, unknownRuns, ceilingUsd },
+  };
+}
+
+/** The digest as a comment body: plain text, every number named. */
+export function digestText(d) {
+  const classes = Object.entries(d.byClass)
+    .map(([k, v]) => `${k} ${v}`)
+    .join(", ");
+  const out = [
+    `Autopilot daily digest (${d.windowStart} to ${d.windowEnd})`,
+    "",
+    `${d.items} item(s) attempted in ${d.attempts} run(s). Outcomes: ${classes || "none"}.`,
+    `Cost: $${d.cost.usd.toFixed(2)}` +
+      (d.cost.ceilingUsd !== null ? ` of a $${Number(d.cost.ceilingUsd).toFixed(2)} ceiling` : "") +
+      (d.cost.unknownRuns ? `, ${d.cost.unknownRuns} run(s) not measured` : "") +
+      ".",
+    "",
+    `PRs opened (${d.prsOpened.length}):`,
+    ...(d.prsOpened.length ? d.prsOpened.map((u) => `- ${u}`) : ["- none"]),
+    "",
+    `Waiting for an answer (${d.awaitingAnswer.length}):`,
+    ...(d.awaitingAnswer.length
+      ? d.awaitingAnswer.map((p) => `- ${p.id}${p.outcome ? ` (${p.outcome})` : ""}`)
+      : ["- none"]),
+    "",
+  ];
+  return out.join("\n");
+}