@mmerterden/multi-agent-pipeline 17.0.0 → 17.1.0

package/pipeline/scripts/autopilot-intake.mjs

@@ -0,0 +1,383 @@
+#!/usr/bin/env node
+/**
+ * autopilot-intake.mjs  -  sources to one ordered queue. Read-only on the world.
+ *
+ * This file answers "what is next" and nothing else. It never dispatches, never
+ * writes to a ticket, never touches a worktree: the runner does that, and
+ * keeping the two apart is what makes the ordering testable without a network,
+ * a simulator, or a willingness to open a real PR while a gate runs.
+ *
+ * ORDER (descending precedence). Deterministic, no model call:
+ *
+ *   1. explicit rank from the label (`agent-queue:1` ... `agent-queue:9`)
+ *   2. repo grouping - once a repo is at the head its items keep going until it
+ *      is empty or `maxConsecutivePerRepo` is reached. This is not cosmetic: the
+ *      code graph, repo map and knowledge index are per-repo and are rebuilt per
+ *      run today, so draining a repo is what lets items 2..N skip that setup.
+ *   3. priority - the Jira ladder from jira/SKILL.md, or a GitHub priority label
+ *   4. created ASCENDING - oldest first. Note this is the OPPOSITE of the `jira`
+ *      picker, which shows newest first: a human picking from a list wants what
+ *      just landed, an unattended queue must not starve what has been waiting.
+ *   5. `source:id` lexicographic, so equal inputs give one identical order and a
+ *      re-scan does not reshuffle the queue under a running item.
+ *
+ * Grouping never moves an item across a rank boundary; rank is global, grouping
+ * is within a rank.
+ *
+ * Usage:
+ *   autopilot-intake.mjs                 fetch, order, write queue.json
+ *   autopilot-intake.mjs --dry-run       fetch, order, print, write NOTHING
+ *   autopilot-intake.mjs --from <file>   candidates from a file, no fetching
+ *   autopilot-intake.mjs --order-only    candidates on stdin, ordered JSON out
+ *
+ * Exit: 0 always when it can read its own state. An empty queue is an answer,
+ * not an error - "no item carries the label yet" is the single most common
+ * reason nothing is happening and it is not a fault.
+ */
+
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync, mkdirSync, writeFileSync, renameSync, chmodSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { homedir } from "node:os";
+
+const ROOT = process.env.MA_AUTOPILOT_ROOT || join(homedir(), ".claude", "autopilot");
+
+/** Jira's ladder, copied in score form from jira/SKILL.md so both rank the same. */
+const PRIORITY = {
+  blocker: 100,
+  critical: 90,
+  highest: 95,
+  high: 80,
+  medium: 50,
+  normal: 50,
+  low: 30,
+  lowest: 10,
+};
+
+/**
+ * An outcome that means the item is finished with us. Anything not listed is
+ * either retryable or a waiting state, and the difference matters: treating
+ * `died` as terminal drops an item because a laptop lid closed once.
+ */
+const TERMINAL = new Set(["pr-opened", "merged", "skipped"]);
+/** Counted against `maxAttempts`. A credential block is NOT one of these - the
+ *  token being expired is not the item's fault, and it becomes workable again
+ *  the moment the token is replaced. */
+const RETRYABLE = new Set(["died", "build-failed"]);
+
+export function priorityScore(name) {
+  if (!name) return 0;
+  return PRIORITY[String(name).trim().toLowerCase()] ?? 0;
+}
+
+/** `agent-queue:3` -> 3. Any other label shape -> null (ordered by the rules). */
+export function rankFromLabels(labels, base) {
+  for (const l of labels || []) {
+    const name = typeof l === "string" ? l : l?.name;
+    if (!name) continue;
+    const m = new RegExp(`^${base}:([1-9])$`).exec(name);
+    if (m) return Number(m[1]);
+  }
+  return null;
+}
+
+function baseCompare(a, b) {
+  const pa = a.priority ?? 0;
+  const pb = b.priority ?? 0;
+  if (pa !== pb) return pb - pa; // higher score first
+  const ca = a.createdAt || "";
+  const cb = b.createdAt || "";
+  if (ca !== cb) return ca < cb ? -1 : 1; // oldest first
+  return `${a.source}:${a.id}` < `${b.source}:${b.id}` ? -1 : 1;
+}
+
+/**
+ * The whole ordering, as a pure function of its arguments. Everything above
+ * this line fetches; everything below is what the gate can exercise with a
+ * fixture and what a reader can reason about.
+ */
+export function orderQueue(items, { maxConsecutivePerRepo = 5 } = {}) {
+  const byRank = new Map();
+  for (const it of items) {
+    const r = it.rank ?? 10;
+    if (!byRank.has(r)) byRank.set(r, []);
+    byRank.get(r).push(it);
+  }
+
+  const out = [];
+  for (const rank of [...byRank.keys()].sort((x, y) => x - y)) {
+    const pool = byRank.get(rank).slice().sort(baseCompare);
+    let current = null;
+    let run = 0;
+    while (pool.length > 0) {
+      let idx = -1;
+      if (current !== null && run < maxConsecutivePerRepo) {
+        idx = pool.findIndex((i) => i.repo === current);
+      }
+      if (idx === -1 && current !== null && run >= maxConsecutivePerRepo) {
+        // The ceiling has to ACTIVELY hand the turn to another repo. The first
+        // version of this loop fell back to pool[0] here, which is still the
+        // busy repo's next item whenever that repo dominates the pool - so the
+        // ceiling was computed, compared, and then did nothing. Twelve items in
+        // one repo still shipped before a single item in another.
+        idx = pool.findIndex((i) => i.repo !== current);
+      }
+      if (idx === -1) idx = 0; // only this repo is left; the ceiling is moot
+      const item = pool[idx];
+      if (item.repo !== current) {
+        current = item.repo;
+        run = 0;
+      }
+      pool.splice(idx, 1);
+      out.push(item);
+      run += 1;
+    }
+  }
+  return out;
+}
+
+/** Attempt history, folded into one record per item. */
+export function foldAttempts(lines) {
+  const byKey = new Map();
+  for (const raw of lines) {
+    const s = raw.trim();
+    if (!s) continue;
+    let rec;
+    try {
+      rec = JSON.parse(s);
+    } catch {
+      continue; // a truncated last line is normal on an append-only log
+    }
+    const key = `${rec.source}:${rec.id}`;
+    const cur = byKey.get(key) || { attempts: 0, last: null, usd: 0, at: 0 };
+    if (RETRYABLE.has(rec.outcome)) cur.attempts += 1;
+    cur.usd += Number(rec.usd || 0);
+    if ((rec.at || 0) >= cur.at) {
+      cur.at = rec.at || 0;
+      cur.last = rec.outcome;
+      cur.prUrl = rec.prUrl || cur.prUrl;
+      cur.gapFingerprint = rec.gapFingerprint ?? cur.gapFingerprint;
+    }
+    byKey.set(key, cur);
+  }
+  return byKey;
+}
+
+/**
+ * Which candidates may enter the queue, and WHY each excluded one was excluded.
+ * Every drop carries a reason: a queue that silently shortens is the thing that
+ * makes "why is nothing happening" unanswerable.
+ */
+export function selectable(candidates, history, { maxAttempts = 2 } = {}) {
+  const queued = [];
+  const awaiting = [];
+  const dropped = [];
+  for (const c of candidates) {
+    const h = history.get(`${c.source}:${c.id}`);
+    if (!h) {
+      queued.push(c);
+      continue;
+    }
+    if (TERMINAL.has(h.last)) {
+      dropped.push({ ...c, reason: `already ${h.last}`, prUrl: h.prUrl });
+      continue;
+    }
+    if (h.last === "awaiting-answer") {
+      awaiting.push({ ...c, askedAt: h.at, gapFingerprint: h.gapFingerprint });
+      continue;
+    }
+    if (h.attempts >= maxAttempts) {
+      dropped.push({ ...c, reason: `${h.attempts} attempts, ceiling ${maxAttempts}` });
+      continue;
+    }
+    queued.push({ ...c, attempts: h.attempts });
+  }
+  return { queued, awaiting, dropped };
+}
+
+// ---------------------------------------------------------------------------
+// Fetching. Everything below talks to the world.
+// ---------------------------------------------------------------------------
+
+function sh(cmd, args, opts = {}) {
+  try {
+    return execFileSync(cmd, args, {
+      encoding: "utf-8",
+      stdio: ["ignore", "pipe", "pipe"],
+      ...opts,
+    });
+  } catch (e) {
+    return e.stdout || "";
+  }
+}
+
+function fetchGitHub(repo) {
+  const label = repo.githubLabel || "agent-queue";
+  const out = sh("gh", [
+    "issue",
+    "list",
+    "--repo",
+    repo.nameWithOwner,
+    "--label",
+    label,
+    "--state",
+    "open",
+    "--json",
+    "number,title,labels,url,createdAt",
+    "--limit",
+    "100",
+  ]);
+  let rows;
+  try {
+    rows = JSON.parse(out || "[]");
+  } catch {
+    return [];
+  }
+  const short = repo.nameWithOwner.split("/").pop();
+  return rows.map((r) => ({
+    source: "github",
+    id: `${short}#${r.number}`,
+    ref: `${short}#${r.number}`,
+    repo: repo.nameWithOwner,
+    localPath: repo.localPath,
+    title: r.title,
+    url: r.url,
+    createdAt: r.createdAt,
+    rank: rankFromLabels(r.labels, label),
+    // GitHub has no priority field; a `priority:high` label is the convention
+    // and its absence is an honest 0 rather than a guess.
+    priority: priorityScore(
+      (r.labels || [])
+        .map((l) => l.name || "")
+        .find((n) => n.startsWith("priority:"))
+        ?.split(":")[1],
+    ),
+  }));
+}
+
+function fetchJira(repo) {
+  const label = repo.jiraLabel || "agent-queue";
+  const jql =
+    repo.jiraJql ||
+    `assignee = currentUser() AND labels = "${label}" AND resolution = EMPTY ` +
+      `AND status not in (Done, Closed, Cancelled) ORDER BY priority DESC, created ASC`;
+  // One helper owns Jira search: URL encoding and JQL injection are already
+  // solved there, and a second copy here would be a second place to get them
+  // wrong. It prints one JSON document on stdout.
+  const helper = join(homedir(), ".claude", "scripts", "jira-search.sh");
+  if (!existsSync(helper)) return [];
+  const out = sh("bash", [helper, "--jql", jql, "--fields", "key,summary,priority,created"]);
+  let rows;
+  try {
+    rows = JSON.parse(out || "{}").issues || [];
+  } catch {
+    return [];
+  }
+  return rows.map((i) => ({
+    source: "jira",
+    id: i.key,
+    ref: i.key,
+    repo: repo.nameWithOwner,
+    localPath: repo.localPath,
+    title: i.fields?.summary,
+    url: null,
+    createdAt: i.fields?.created,
+    rank: null,
+    priority: priorityScore(i.fields?.priority?.name),
+  }));
+}
+
+function gather(config) {
+  const items = [];
+  for (const repo of config.repos || []) {
+    const sources = repo.sources || ["github"];
+    if (sources.includes("github")) items.push(...fetchGitHub(repo));
+    if (sources.includes("jira")) items.push(...fetchJira(repo));
+  }
+  return items;
+}
+
+// ---------------------------------------------------------------------------
+
+function atomicWrite(name, text) {
+  mkdirSync(ROOT, { recursive: true, mode: 0o700 });
+  const dst = join(ROOT, name);
+  const tmp = join(dirname(dst), `.${name}.${process.pid}`);
+  writeFileSync(tmp, text, { mode: 0o600 });
+  chmodSync(tmp, 0o600);
+  renameSync(tmp, dst);
+}
+
+function main(argv) {
+  const args = argv.slice(2);
+  const has = (f) => args.includes(f);
+  const val = (f) => (args.includes(f) ? args[args.indexOf(f) + 1] : null);
+
+  if (has("--order-only")) {
+    const raw = readFileSync(0, "utf-8");
+    const items = JSON.parse(raw || "[]");
+    process.stdout.write(JSON.stringify(orderQueue(items), null, 2) + "\n");
+    return 0;
+  }
+
+  const cfgPath = join(ROOT, "config.json");
+  const config = existsSync(cfgPath) ? JSON.parse(readFileSync(cfgPath, "utf-8")) : { repos: [] };
+
+  const fromFile = val("--from");
+  const candidates = fromFile ? JSON.parse(readFileSync(fromFile, "utf-8")) : gather(config);
+
+  const attemptPath = join(ROOT, "attempted.jsonl");
+  const history = foldAttempts(
+    existsSync(attemptPath) ? readFileSync(attemptPath, "utf-8").split("\n") : [],
+  );
+
+  const { queued, awaiting, dropped } = selectable(candidates, history, {
+    maxAttempts: config.maxAttempts ?? 2,
+  });
+
+  const ordered = orderQueue(queued, {
+    maxConsecutivePerRepo: config.maxConsecutivePerRepo ?? 5,
+  });
+
+  // The existing queue's `running` list is preserved: intake decides what is
+  // NEXT and has no opinion about what is already in flight. Overwriting it here
+  // would make a scan look like a crash to the status reader.
+  let running = [];
+  const qPath = join(ROOT, "queue.json");
+  if (existsSync(qPath)) {
+    try {
+      running = JSON.parse(readFileSync(qPath, "utf-8")).running || [];
+    } catch {
+      running = [];
+    }
+  }
+
+  const queue = {
+    lastTickAt: Math.floor(Date.now() / 1000),
+    running,
+    queued: ordered,
+    awaiting,
+    dropped,
+    emptyReason:
+      ordered.length === 0 && awaiting.length === 0
+        ? candidates.length === 0
+          ? "no item carries the label yet"
+          : "every candidate is finished, waiting or past its attempt ceiling"
+        : null,
+  };
+
+  const text = JSON.stringify(queue, null, 2);
+  if (has("--dry-run")) {
+    process.stdout.write(text + "\n");
+    return 0;
+  }
+  atomicWrite("queue.json", text);
+  process.stdout.write(
+    `autopilot-intake: ${ordered.length} queued, ${awaiting.length} awaiting, ${dropped.length} dropped\n`,
+  );
+  return 0;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exit(main(process.argv));
+}