@mmerterden/multi-agent-pipeline 17.0.0 → 17.3.0

package/pipeline/scripts/autopilot-runner.mjs

@@ -0,0 +1,354 @@
+#!/usr/bin/env node
+/**
+ * autopilot-runner.mjs  -  one tick. Recover, take one item, run it, record it.
+ *
+ * The loop is launchd, not `while true`. A `while true` that crashes is a dead
+ * queue with no signal; a tick that crashes is repeated `StartInterval` seconds
+ * later by something whose whole job is restarting it. That choice is also what
+ * makes step 1 the mutual exclusion: two overlapping ticks cannot take the same
+ * item, because the second one sees the first one's pid file and leaves.
+ *
+ *   1. RECOVER  a previous runner's pid file decides whether anything is live.
+ *   2. INTAKE   autopilot-intake.mjs writes the ordered queue. Read-only.
+ *   3. ARM      autopilot-arming.mjs answers spend and credentials.
+ *   4. TAKE     the head of the queue, and only if this repo is free.
+ *   5. RUN      one `claude --bg` child, waited on. Stops at an open PR.
+ *   6. RECORD   attempted.jsonl, clear the pid file, refresh status.json.
+ *
+ * LIVENESS WITHOUT A LEASE. Lease arithmetic exists to arbitrate between rival
+ * writers, and here there are none: one machine, one queue, one writer. The
+ * question is only "is the runner I recorded still alive", and three cheap facts
+ * answer it exactly - the pid responds to signal 0, the machine has not rebooted
+ * since the pid was recorded, and the session is in `claude agents --json`. The
+ * middle one is what makes a reboot safe without probing anything: after a
+ * restart pids start low, so a recorded 4711 may well belong to something real
+ * and unrelated, and a runner that trusted `kill -0` alone would decide another
+ * runner was live and do nothing, forever.
+ *
+ * WHAT IT WILL NOT DO. It does not merge. It does not remove a worktree holding
+ * uncommitted work - that is stashed to a branch and the worktree kept, because
+ * losing a day of edits is worse than 750 MB. It does not remove a worktree it
+ * cannot prove it created: the measured tree here had 19 worktrees with no state
+ * file, several of them hand-made and one in active use.
+ *
+ * Usage:
+ *   autopilot-runner.mjs              one tick
+ *   autopilot-runner.mjs --dry-run    everything except the spawn, printed
+ *
+ * Exit: 0 for every normal outcome INCLUDING "nothing to do" and "not armed".
+ * launchd logs a non-zero exit as a failure, and a quiet queue is not one.
+ */
+
+import { execFileSync, spawnSync } from "node:child_process";
+import {
+  existsSync,
+  readFileSync,
+  writeFileSync,
+  appendFileSync,
+  mkdirSync,
+  rmSync,
+  chmodSync,
+} from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { randomUUID } from "node:crypto";
+
+const ROOT = process.env.MA_AUTOPILOT_ROOT || join(homedir(), ".claude", "autopilot");
+// Siblings resolve from THIS file's own directory, not from ~/.claude/scripts.
+// All five autopilot scripts install into whichever host tree took them, so a
+// runner living in ~/.codex/scripts must find the intake beside it - naming one
+// host's path means a Codex-only or Copilot-only tick finds nothing and does
+// nothing, with no error anywhere. smoke-autopilot-hosts.sh covers this.
+const SCRIPTS = process.env.MA_AP_SCRIPTS || import.meta.dirname;
+const CLAUDE_BIN = process.env.MA_AP_CLAUDE_BIN || "claude";
+const DRY = process.argv.includes("--dry-run");
+
+const log = (m) => process.stdout.write(`autopilot-runner: ${m}\n`);
+
+function run(cmd, args, opts = {}) {
+  try {
+    return execFileSync(cmd, args, {
+      encoding: "utf-8",
+      stdio: ["ignore", "pipe", "pipe"],
+      ...opts,
+    });
+  } catch (e) {
+    return e.stdout || "";
+  }
+}
+
+function readJson(p, fallback) {
+  if (!existsSync(p)) return fallback;
+  try {
+    return JSON.parse(readFileSync(p, "utf-8"));
+  } catch {
+    return fallback;
+  }
+}
+
+function ensureRoot() {
+  mkdirSync(ROOT, { recursive: true, mode: 0o700 });
+}
+
+function writeState(name, obj) {
+  ensureRoot();
+  const p = join(ROOT, name);
+  writeFileSync(p, JSON.stringify(obj, null, 2), { mode: 0o600 });
+  chmodSync(p, 0o600);
+}
+
+function record(entry) {
+  ensureRoot();
+  const p = join(ROOT, "attempted.jsonl");
+  appendFileSync(p, JSON.stringify({ at: Math.floor(Date.now() / 1000), ...entry }) + "\n", {
+    mode: 0o600,
+  });
+  chmodSync(p, 0o600);
+}
+
+/** Seconds since the epoch at which this machine booted. */
+export function bootTime() {
+  const out = run("sysctl", ["-n", "kern.boottime"]);
+  const m = /sec\s*=\s*(\d+)/.exec(out);
+  return m ? Number(m[1]) : 0;
+}
+
+function pidAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (e) {
+    return e.code === "EPERM"; // alive but not ours
+  }
+}
+
+function sessionLive(sessionId) {
+  if (!sessionId) return false;
+  const out = process.env.MA_AP_AGENTS_JSON ?? run(CLAUDE_BIN, ["agents", "--json"]);
+  try {
+    const rows = JSON.parse(out || "[]");
+    const list = Array.isArray(rows) ? rows : rows.agents || [];
+    return list.some((a) => a.sessionId === sessionId);
+  } catch {
+    // Cannot tell. Treated as NOT live on purpose: the alternative is a queue
+    // that stops forever because one command's output changed shape.
+    return false;
+  }
+}
+
+/**
+ * Is the recorded runner still running?
+ *
+ * All three conditions, and the boot check is the one that matters after a
+ * restart: a pid recorded before the current boot is stale by definition and
+ * needs no probe at all.
+ */
+export function holderIsLive(holder, { now = bootTime(), agentsJson } = {}) {
+  if (!holder || !holder.pid) return false;
+  if (holder.bootTime && now && holder.bootTime !== now) return false;
+  if (!pidAlive(holder.pid)) return false;
+  if (agentsJson !== undefined) {
+    try {
+      const rows = JSON.parse(agentsJson || "[]");
+      const list = Array.isArray(rows) ? rows : rows.agents || [];
+      return list.some((a) => a.sessionId === holder.sessionId);
+    } catch {
+      return false;
+    }
+  }
+  return sessionLive(holder.sessionId);
+}
+
+/**
+ * Retire a run whose session is gone. The worktree is removed BY NAME, never by
+ * sweeping: gc-worktrees.sh deliberately skips registered worktrees and always
+ * will, and the runner is the only thing that knows which one it created.
+ */
+function retire(holder) {
+  const item = holder.item || {};
+  const wt = holder.worktree;
+  let stashed = null;
+
+  if (wt && existsSync(wt)) {
+    const dirty = run("git", ["-C", wt, "status", "--porcelain"]).trim();
+    if (dirty) {
+      // A day of edits is worth more than 750 MB. The branch name is derived
+      // from the task so it can be found without remembering this ran.
+      const branch = `autopilot/abandoned/${holder.taskId || item.id || "unknown"}`;
+      run("git", ["-C", wt, "stash", "push", "-u", "-m", branch]);
+      run("git", ["-C", wt, "branch", branch]);
+      stashed = branch;
+      log(`kept ${wt}: uncommitted work stashed to ${branch}`);
+    } else if (holder.createdWorktree) {
+      run("git", ["-C", wt, "worktree", "remove", "--force", wt]);
+      run("git", ["-C", holder.repoPath || wt, "worktree", "prune"]);
+      log(`removed ${wt}`);
+    }
+  }
+
+  if (holder.statePath && existsSync(holder.statePath)) {
+    const st = readJson(holder.statePath, null);
+    if (st) {
+      st.status = "abandoned";
+      st.abandonedBy = "autopilot-runner";
+      writeFileSync(holder.statePath, JSON.stringify(st, null, 2));
+    }
+  }
+
+  record({
+    source: item.source,
+    id: item.id,
+    taskId: holder.taskId,
+    outcome: "died",
+    stashedTo: stashed,
+  });
+}
+
+function refreshStatus() {
+  const s = join(SCRIPTS, "autopilot-status.sh");
+  if (existsSync(s)) run("bash", [s, "--write"]);
+}
+
+function main() {
+  const pidPath = join(ROOT, "runner.pid");
+
+  // ---- 1. RECOVER --------------------------------------------------------
+  const holder = readJson(pidPath, null);
+  if (holder) {
+    if (holderIsLive(holder)) {
+      log(`another runner is live (pid ${holder.pid}) - nothing to do`);
+      return 0;
+    }
+    log(`previous runner is gone (pid ${holder.pid}) - retiring its item`);
+    if (!DRY) {
+      retire(holder);
+      rmSync(pidPath, { force: true });
+    }
+  }
+
+  const config = readJson(join(ROOT, "config.json"), null);
+  if (!config) {
+    log("not configured on this machine - nothing to do");
+    return 0;
+  }
+
+  // ---- 2. INTAKE ---------------------------------------------------------
+  const intake = join(SCRIPTS, "autopilot-intake.mjs");
+  if (existsSync(intake)) run(process.execPath, [intake]);
+  const queue = readJson(join(ROOT, "queue.json"), { queued: [], running: [] });
+
+  const running = queue.running || [];
+  const slots = Number(config.slots ?? 1);
+  if (running.length >= slots) {
+    log(`${running.length}/${slots} slots in use - nothing to take`);
+    return 0;
+  }
+
+  // Per-repo concurrency is always 1, whatever `slots` says: two runs in one
+  // checkout contend on .git/index.lock, and that is a named failure rather
+  // than a slow path. It also means the queue steps around a repo YOU are
+  // working in instead of competing with you for it.
+  const busyRepos = new Set(running.map((r) => r.repo));
+  const next = (queue.queued || []).find((i) => !busyRepos.has(i.repo));
+  if (!next) {
+    log(queue.emptyReason || "every queued item belongs to a repo already in flight");
+    return 0;
+  }
+
+  // ---- 3. ARM ------------------------------------------------------------
+  const armScript = join(SCRIPTS, "autopilot-arming.mjs");
+  if (existsSync(armScript)) {
+    const out = run(process.execPath, [armScript, "check", "--source", next.source, "--json"]);
+    const arm = (() => {
+      try {
+        return JSON.parse(out);
+      } catch {
+        return { armed: true };
+      }
+    })();
+    if (!arm.armed) {
+      log(`not armed: ${arm.reason}`);
+      if (!DRY) {
+        record({ source: next.source, id: next.id, outcome: `blocked-${arm.blockedBy}` });
+        writeState("queue.json", { ...queue, blockedReason: arm.reason });
+        refreshStatus();
+      }
+      return 0;
+    }
+  }
+
+  // ---- 4. TAKE -----------------------------------------------------------
+  const sessionId = randomUUID();
+  const taskId = `ap-${Date.now()}`;
+  const claim = {
+    pid: process.pid,
+    sessionId,
+    bootTime: bootTime(),
+    taskId,
+    item: next,
+    repoPath: next.localPath,
+    startedAt: Math.floor(Date.now() / 1000),
+  };
+
+  if (DRY) {
+    log(`would take ${next.source}:${next.id} (${next.repo}) as session ${sessionId}`);
+    return 0;
+  }
+
+  writeState("runner.pid", claim);
+  writeState("queue.json", {
+    ...queue,
+    running: [
+      ...running,
+      {
+        id: next.id,
+        repo: next.repo,
+        depth: config.depthRouter === "auto" ? undefined : "full",
+        stack: next.stack,
+        startedAt: claim.startedAt,
+        state: "running",
+        url: next.url,
+      },
+    ],
+    queued: (queue.queued || []).filter((i) => i.id !== next.id),
+  });
+  refreshStatus();
+
+  // ---- 5. RUN ------------------------------------------------------------
+  // The child is a normal pipeline run in the mode that already exists: one
+  // item, unattended, stopping at an open PR. Nothing about the per-item
+  // `autopilot` mode changes because a queue is calling it.
+  log(`running ${next.source}:${next.id}`);
+  const child = spawnSync(
+    CLAUDE_BIN,
+    [
+      "--bg",
+      "--session-id",
+      sessionId,
+      "--permission-prompts",
+      "none",
+      "-p",
+      `/multi-agent ${next.ref || next.id} autopilot`,
+    ],
+    { cwd: next.localPath || process.cwd(), encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] },
+  );
+
+  // ---- 6. RECORD ---------------------------------------------------------
+  const outcome = child.status === 0 ? "pr-opened" : "build-failed";
+  const prUrl = /https:\/\/\S*\/pull(-requests)?\/\d+/.exec(child.stdout || "")?.[0] || null;
+  record({ source: next.source, id: next.id, taskId, outcome, prUrl, usd: 0 });
+
+  const after = readJson(join(ROOT, "queue.json"), queue);
+  writeState("queue.json", {
+    ...after,
+    running: (after.running || []).filter((r) => r.id !== next.id),
+  });
+  rmSync(join(ROOT, "runner.pid"), { force: true });
+  refreshStatus();
+  log(`${next.id}: ${outcome}${prUrl ? ` ${prUrl}` : ""}`);
+  return 0;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exit(main());
+}

package/pipeline/scripts/autopilot-status.sh

@@ -0,0 +1,213 @@
+#!/usr/bin/env bash
+# autopilot-status.sh  -  what continuous mode is doing, in one place.
+#
+# ONE producer, three renderings. The menu bar, the terminal and the session hook
+# all describe the same thing, and the moment each computes its own answer they
+# start disagreeing - which is exactly the failure this release spent its time on:
+# a run reading `in_progress` while its PR was already open.
+#
+#   (default)    a paragraph for a human reading the terminal
+#   --short      one line, for autopilot-off and the session hook
+#   --json       status.json, which is what bin/menubar renders
+#   --write      compute and write status.json, print nothing
+#   --subjects   one line per in-flight item for the native task widget
+#
+# `--json` and `--write` produce the SAME bytes. A menu bar fed by a different
+# code path than the terminal is a second source of truth with a nicer font.
+#
+# Exit: 0 always when it can read the state, 2 on usage. Never non-zero because
+# the mode is off - "off" is an answer.
+
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+. "$HERE/../lib/autopilot-state.sh" 2>/dev/null || . "$HOME/.claude/lib/autopilot-state.sh"
+
+MODE="human"
+case "${1:-}" in
+  "") ;;
+  --short) MODE="short" ;;
+  --json) MODE="json" ;;
+  --write) MODE="write" ;;
+  --subjects) MODE="subjects" ;;
+  -h | --help)
+    grep -E '^#( |$)' "$0" | sed -E 's/^# ?//'
+    exit 0
+    ;;
+  *)
+    echo "autopilot-status: unknown option $1" >&2
+    exit 2
+    ;;
+esac
+
+command -v jq >/dev/null 2>&1 || { echo "autopilot-status: jq is required"; exit 0; }
+
+ON=false; ma_ap_is_on && ON=true
+CONFIGURED=false; ma_ap_is_configured && CONFIGURED=true
+POWER=$(ma_ap_power)
+SLOTS=$(ma_ap_cfg '.slots' 1)
+CEILING=$(ma_ap_slot_ceiling)
+INTERVAL=$(ma_ap_cfg '.scanIntervalSeconds' 120)
+
+QUEUE=$(ma_ap_read queue.json 2>/dev/null || echo '{}')
+RUNNING=$(printf '%s' "$QUEUE" | jq -c '[.running[]? | select((.state // "running") == "running")]' 2>/dev/null || echo '[]')
+QUEUED=$(printf '%s' "$QUEUE" | jq -c '[.queued[]?]' 2>/dev/null || echo '[]')
+AWAITING=$(printf '%s' "$QUEUE" | jq -c '[.awaiting[]?]' 2>/dev/null || echo '[]')
+LAST_TICK=$(printf '%s' "$QUEUE" | jq -r '.lastTickAt // empty' 2>/dev/null)
+EMPTY_REASON=$(printf '%s' "$QUEUE" | jq -r '.emptyReason // empty' 2>/dev/null)
+BLOCKED=$(printf '%s' "$QUEUE" | jq -r '.blockedReason // empty' 2>/dev/null)
+
+# Spend and the arming verdict come from the one script that computes them, so
+# the terminal, the menu bar and the runner cannot disagree about whether the
+# queue is allowed to take an item.
+ARM='{}'
+ARMING=$(ma_ap_asset scripts/autopilot-arming.mjs 2>/dev/null || true)
+if [ "$CONFIGURED" = true ] && [ -n "$ARMING" ]; then
+  ARM=$(node "$ARMING" check --json 2>/dev/null || echo '{}')
+fi
+SPEND=$(printf '%s' "$ARM" | jq -r '.spend.usd // 0' 2>/dev/null || echo 0)
+CEILING_USD=$(printf '%s' "$ARM" | jq -r '.ceiling // 0' 2>/dev/null || echo 0)
+
+# Recently finished, from the append-only attempt log rather than from a second
+# list somebody has to remember to prune. Bounded to a day: this answers "what
+# happened while I was away", and an unbounded list stops answering that.
+CUTOFF=$(($(date +%s) - 86400))
+REPORTS='[]'
+if [ -f "$(ma_ap_root)/attempted.jsonl" ]; then
+  REPORTS=$(jq -sc --argjson c "$CUTOFF" '
+    [ .[] | select((.at // 0) >= $c) | select(.prUrl != null and .prUrl != "")
+      | {id: .id, kind: "pr", url: .prUrl, outcome: (.outcome // "open"), finishedAt: .at} ]
+    | sort_by(-.finishedAt) | .[0:10]' "$(ma_ap_root)/attempted.jsonl" 2>/dev/null || echo '[]')
+fi
+TODAY_N=$(printf '%s' "$REPORTS" | jq 'length' 2>/dev/null || echo 0)
+TODAY_USD=$(printf '%s' "$QUEUE" | jq -r '.todayUsd // 0' 2>/dev/null || echo 0)
+
+NOTE=""
+if [ "$ON" = true ]; then
+  if [ "$POWER" = "ac" ]; then NOTE="prizde, uyku engelleniyor"; else NOTE="bataryada, uyku serbest - prize takınca devam eder"; fi
+fi
+if ma_ap_is_orphaned; then
+  NOTE="config açık ama launchd'de iş yok - /multi-agent:autopilot-on ile yeniden kur"
+fi
+
+build_json() {
+  jq -nc \
+    --argjson on "$ON" --argjson slots "$SLOTS" \
+    --argjson running "$RUNNING" --argjson queued "$QUEUED" --argjson awaiting "$AWAITING" \
+    --argjson reports "$REPORTS" --argjson todayCount "$TODAY_N" --argjson todayUsd "$TODAY_USD" \
+    --arg power "$POWER" --arg note "$NOTE" \
+    --arg emptyReason "$EMPTY_REASON" --arg blockedReason "$BLOCKED" \
+    --argjson interval "$INTERVAL" --argjson spendUsd "${SPEND:-0}" \
+    --argjson ceilingUsd "${CEILING_USD:-0}" \
+    --argjson lastTickAt "${LAST_TICK:-null}" \
+    '{on:$on, slots:$slots, running:$running, queued:$queued, awaiting:$awaiting,
+      reports:$reports, todayCount:$todayCount, todayUsd:$todayUsd,
+      lastTickAt:$lastTickAt, power:$power, note:$note,
+      scanIntervalSeconds:$interval, emptyReason:$emptyReason,
+      blockedReason:$blockedReason, spendUsd:$spendUsd, ceilingUsd:$ceilingUsd}'
+}
+
+case "$MODE" in
+  json)
+    build_json
+    exit 0
+    ;;
+  write)
+    build_json | ma_ap_write status.json
+    exit 0
+    ;;
+esac
+
+# Refresh status.json as a side effect of being asked, so the menu bar is current
+# the moment anyone looks at the terminal - but ONLY when the mode is already
+# configured. Asking a question may not answer it into existence: on a machine
+# that never turned this on, `ma_ap_write` would create ~/.claude/autopilot/ and
+# quietly break the guarantee that a clean install writes nothing. Caught by
+# running this file on this machine, which then had the directory.
+if [ "$CONFIGURED" = true ]; then
+  build_json | ma_ap_write status.json 2>/dev/null || true
+fi
+
+NR=$(printf '%s' "$RUNNING" | jq 'length')
+NQ=$(printf '%s' "$QUEUED" | jq 'length')
+NA=$(printf '%s' "$AWAITING" | jq 'length')
+
+if [ "$MODE" = "subjects" ]; then
+  # One line per item, with the numbers INSIDE the string: the widget takes
+  # exactly one string per row, so anything the reader needs has to travel in it.
+  printf '%s' "$RUNNING" | jq -r '.[] |
+    "\(.id) · \(.depth // "full" | ascii_upcase[0:1] + .[1:]) · Faz \(.phase // "?")\(if .phaseTotal then "/\(.phaseTotal)" else "" end) \(.phaseName // "") · \(.stack // "")"'
+  printf '%s' "$AWAITING" | jq -r '.[] | "\(.id) · cevap bekliyor"'
+  exit 0
+fi
+
+if [ "$MODE" = "short" ]; then
+  if [ "$ON" != true ]; then
+    if [ "$CONFIGURED" = true ]; then
+      echo "autopilot: kapalı ($(ma_ap_cfg '.repos | length' 0) repo seçili)"
+    else
+      echo "autopilot: kurulu değil"
+    fi
+    exit 0
+  fi
+  echo "autopilot: açık · $NR koşuyor, $NQ sırada, $NA cevap bekliyor · $NOTE"
+  exit 0
+fi
+
+# --- human -------------------------------------------------------------------
+if [ "$CONFIGURED" != true ]; then
+  echo "autopilot: bu makinede kurulu değil."
+  echo "  Açmak için: /multi-agent:autopilot-on"
+  exit 0
+fi
+
+NREPOS=$(ma_ap_cfg '.repos | length' 0)
+if [ "$ON" = true ]; then
+  echo "autopilot: AÇIK · $NREPOS repo · $SLOTS slot (bu makine $CEILING kaldırır) · her ${INTERVAL}sn tarıyor"
+else
+  echo "autopilot: KAPALI · $NREPOS repo seçili, zamanlama yok"
+  echo "  Açmak için: /multi-agent:autopilot-on"
+fi
+[ -n "$NOTE" ] && echo "  $NOTE"
+
+echo
+if [ "$NR" -gt 0 ]; then
+  echo "Koşuyor ($NR):"
+  printf '%s' "$RUNNING" | jq -r '.[] | "  \(.id) · \(.phaseName // .phase // "?") · \(.stack // "")"'
+fi
+if [ "$NQ" -gt 0 ]; then
+  echo "Sırada ($NQ):"
+  printf '%s' "$QUEUED" | jq -r '.[0:8][] | "  \(.id)"'
+fi
+if [ "$NA" -gt 0 ]; then
+  echo "Cevap bekliyor ($NA) - açık soruları yorumladık, cevap gelince devam edecek:"
+  printf '%s' "$AWAITING" | jq -r '.[] | "  \(.id)"'
+fi
+if [ "$NR" -eq 0 ] && [ "$NQ" -eq 0 ] && [ "$NA" -eq 0 ]; then
+  echo "Sırada iş yok."
+  # The single most common "why is nothing happening" answer, and it is not a
+  # fault: no item carries the label yet. When intake recorded a more specific
+  # reason, that one is truer and goes first.
+  if [ -n "$EMPTY_REASON" ]; then
+    echo "  $EMPTY_REASON"
+  fi
+  echo "  Bir madde almasi icin: GitHub issue'ya ya da sana atanmis Jira maddesine label ekle."
+fi
+
+# The queue refusing to take work is not the same as having none, and reading
+# one as the other is the whole reason this line exists separately.
+if [ -n "$BLOCKED" ]; then
+  echo
+  echo "Yeni madde alınmıyor: $BLOCKED"
+fi
+if [ "$CEILING_USD" != "0" ]; then
+  echo "Son 24 saat: \$$SPEND / \$$CEILING_USD"
+fi
+
+if [ "$TODAY_N" -gt 0 ]; then
+  echo
+  echo "Son 24 saat ($TODAY_N PR):"
+  printf '%s' "$REPORTS" | jq -r '.[0:6][] | "  \(.id) · \(.url)"'
+fi
+exit 0