@mmerterden/multi-agent-pipeline 20.36.0 → 20.37.1

package/pipeline/scripts/runs-index.mjs

@@ -81,7 +81,7 @@ const PHASE_WAITING_FROM = PHASE_CONTRACT.thresholds.waitingFromPhase;
  */
 const LAST_PHASE = PHASE_CONTRACT.phases[PHASE_CONTRACT.phases.length - 1].id;
 
-export const CONTRACT_VERSION = "1.3.0";
+export const CONTRACT_VERSION = "1.4.0";
 
 const STATE_MAX_BYTES = 16 * 1024 * 1024;
 
@@ -93,14 +93,22 @@ const GROUPS = {
   stopped: "Stopped mid-development",
   question: "Left at a question",
   unknown: "Status not recorded",
+  done: "Finished",
 };
 
+/**
+ * A run in one of these finished or was given up; it asks nothing of anyone.
+ * `failed` is not here: a failed run can still be resumed or killed.
+ */
+const DONE_STATUSES = new Set(["complete", "completed", "done", "abandoned", "cancelled"]);
+
 function parseArgs(argv) {
-  const flags = { json: false, redact: false, group: null, taskId: null };
+  const flags = { json: false, redact: false, all: false, group: null, taskId: null };
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i];
     if (a === "--json") flags.json = true;
     else if (a === "--redact") flags.redact = true;
+    else if (a === "--all") flags.all = true;
     else if (a === "--group") flags.group = argv[++i];
     else if (a.startsWith("--group=")) flags.group = a.slice(8);
     else if (a === "--task-id") flags.taskId = argv[++i];
@@ -379,11 +387,12 @@ function summarisePhases(tracker) {
  * The group a run belongs to, per status/SKILL.md 3b.
  *
  * @param {object|null} state
- * @returns {"waiting"|"stopped"|"question"|"unknown"}
+ * @returns {"waiting"|"stopped"|"question"|"unknown"|"done"}
  */
 function groupOf(state) {
   const status = state?.status;
   if (!status) return "unknown";
+  if (DONE_STATUSES.has(status)) return "done";
   const phase = Number(state?.currentPhase);
   const prUrl = state?.pr?.url ?? state?.prUrl ?? null;
   if (status === "awaiting_input" || status === "awaiting-user-test-main-checkout")
@@ -525,7 +534,17 @@ function fmtDuration(startedAt) {
   return `${Math.floor(h / 24)}d`;
 }
 
-function renderHuman(records) {
+/**
+ * The Phase column. A phase past the contract's last one was recorded under an
+ * older, longer numbering; "current/last" would read as overflow, so it shows
+ * as the recorded number alone.
+ */
+export function phaseCell(phase) {
+  if (phase === null) return " -";
+  return phase > LAST_PHASE ? `${phase} old` : `${phase}/${LAST_PHASE}`;
+}
+
+function renderHuman(records, { all = false } = {}) {
   const out = [];
   const total = records.length;
   out.push(`multi-agent runs  -  ${total} under ${logsRoot()}`);
@@ -541,13 +560,20 @@ function renderHuman(records) {
   for (const key of Object.keys(GROUPS)) {
     const rows = records.filter((r) => r.group === key);
     if (!rows.length) continue;
+    // Neither group asks anything of anyone, and together they are most of a
+    // machine's runs: a count keeps the actionable groups readable.
+    if ((key === "done" || key === "unknown") && !all) {
+      out.push("");
+      out.push(`${GROUPS[key]}  (${rows.length})  -  --all lists them`);
+      continue;
+    }
     out.push("");
     out.push(`${GROUPS[key]}  (${rows.length})`);
     out.push(
       "  ID                              Phase  Status            Branch                    Age    ~USD",
     );
     for (const r of rows) {
-      const phase = r.currentPhase === null ? " -" : `${r.currentPhase}/${LAST_PHASE}`;
+      const phase = phaseCell(r.currentPhase);
       out.push(
         "  " +
           [
@@ -651,7 +677,7 @@ function main() {
   const flags = parseArgs(process.argv.slice(2));
   if (flags.help) {
     process.stdout.write(
-      "Usage: runs-index.mjs [--json [--redact]] [--group waiting|stopped|question|unknown] [--task-id <id>]\n",
+      "Usage: runs-index.mjs [--json [--redact]] [--all] [--group waiting|stopped|question|unknown|done] [--task-id <id>]\n",
     );
     return 0;
   }
@@ -670,7 +696,8 @@ function main() {
       JSON.stringify(envelope(records, { redact: flags.redact }), null, 2) + "\n",
     );
   } else {
-    process.stdout.write(renderHuman(records) + "\n");
+    // A group asked for by name is listed, not folded into its count.
+    process.stdout.write(renderHuman(records, { all: flags.all || flags.group !== null }) + "\n");
   }
   return 0;
 }

package/pipeline/scripts/scope-check-gate.mjs

@@ -99,6 +99,17 @@ const LINE_COMMENT = /(\.(py|pyi|ya?ml|sh|bash|zsh|rb|toml|coffee)|(^|\/)(GNU)?[
 const TIGHT_LANGS =
   /\.(js|jsx|mjs|cjs|ts|tsx|java|go|cs|swift|kt|kts|json|rs|dart|scala|py|pyi|c|h|cc|cpp|hpp|m|mm)$/;
 const LEADING_OPERATOR_LANGS = /\.(kt|kts|swift|go|scala)$/;
+// A newline ends the statement here, so joining two lines changes the program.
+const NEWLINE_TERMINATED = new RegExp(
+  [
+    String.raw`\.(sh|bash|zsh|ksh|fish|rb|rake|gemspec|toml|dockerfile)$`,
+    String.raw`(^|\/)\.envrc$`,
+    String.raw`(^|\/)[^/]*Rakefile$`,
+    String.raw`(^|\/)(Gemfile|Podfile|Fastfile|Appfile|Matchfile|Snapfile|Scanfile|Gymfile|Deliverfile|Pluginfile|Dangerfile|Brewfile|Vagrantfile|Jenkinsfile)$`,
+    String.raw`(^|\/)(Dockerfile|Containerfile)([.-][^/]*)?$`,
+  ].join("|"),
+  "i",
+);
 const BREAK_KEEPERS = /\b(return|throw|yield|break|continue)$/;
 const LEADING_OPERATOR = /^[+\-*/%&|^.?]/;
 const TIGHT = /\s*([(){}[\],;:])\s*/g;
@@ -135,8 +146,9 @@ function foldLine(line, hashComments) {
 /**
  * The layout-free form of some lines. Each line is folded on its own (linear
  * in the hunk), then the lines are joined by a space, except that a break
- * after return, throw, yield, break or continue, and in Kotlin, Swift, Go or
- * Scala a break before a leading operator, stays a break. Space next to
+ * after return, throw, yield, break or continue, in Kotlin, Swift, Go or
+ * Scala a break before a leading operator, and every break in shell, Ruby,
+ * TOML and Dockerfiles, stays a break. Space next to
  * brackets, commas, semicolons and colons is dropped only in TIGHT_LANGS, and
  * never in a hunk that holds a C preprocessor line.
  */
@@ -150,6 +162,7 @@ function outsideStrings(text, file = "") {
       return;
     }
     const keepBreak =
+      NEWLINE_TERMINATED.test(file) ||
       BREAK_KEEPERS.test(lines[i - 1]) ||
       (LEADING_OPERATOR_LANGS.test(file) && LEADING_OPERATOR.test(line));
     joined += (keepBreak ? "\u23ce" : " ") + line;

package/pipeline/scripts/status-overview.mjs

@@ -0,0 +1,401 @@
+#!/usr/bin/env node
+/**
+ * @file status-overview.mjs  -  what is happening on this machine right now,
+ * in one answer: the live Claude Code sessions and the run each one drives,
+ * the tokens they used, which accounts and credentials are set up, and the
+ * run list.
+ *
+ * Sources, all read on every call:
+ *
+ *   1. sessions: `claude agents --json` (lib/claude-sessions.mjs), matched to
+ *      a run by the session id it recorded, else by a working directory inside
+ *      the run's worktree (activity-snapshot.mjs matchRun);
+ *   2. tokens: the `usage` block of every assistant message in the session's
+ *      transcript and its subagent transcripts, counted once per message id
+ *      (a message split into several lines repeats its usage). The last-24h
+ *      figure covers every transcript written in that window. Prices come from
+ *      cost-table.json; cache writes are priced at the input rate, so the
+ *      estimate errs low;
+ *   3. accounts: the github.com logins gh holds (usage-identity.mjs) and the
+ *      author git would write in the current directory;
+ *   4. credentials: each logical name in prefs.global.keychainMapping is
+ *      "registered" when its service name is in `credential-store.sh list`.
+ *      Only service names are read: the store is never asked for a value, and
+ *      nothing about a value (length, prefix) is printed;
+ *   5. runs: runs-index.mjs, the same records /multi-agent:status always used.
+ *
+ * Usage:
+ *   status-overview.mjs [--json] [--all]
+ *
+ * Env: MA_CREDENTIAL_STORE (the store script), MA_PREFS (the preferences
+ * file), CLAUDE_PROJECTS_ROOT (the transcript root), MA_AP_AGENTS_JSON (the
+ * session list, see claude-sessions.mjs).
+ *
+ * Exit: 0 printed, 2 usage.
+ *
+ * @module pipeline/scripts/status-overview
+ */
+
+import { createReadStream, existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { homedir } from "node:os";
+import { basename, join } from "node:path";
+import { createInterface } from "node:readline";
+
+import { listSessions } from "../lib/claude-sessions.mjs";
+import { runMain } from "../lib/fatal.mjs";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+import { costUsd } from "./_cost.mjs";
+import { matchRun } from "./activity-snapshot.mjs";
+import { buildIndex, phaseCell } from "./runs-index.mjs";
+import { listAccounts } from "./usage-identity.mjs";
+
+export const CONTRACT_VERSION = "1.0.0";
+export const WINDOW_MS = 24 * 3_600_000;
+const SAFE_SESSION_ID = /^[A-Za-z0-9-]{8,64}$/;
+/**
+ * States of a session that is still open. Wider than claude-sessions.mjs
+ * LIVE_STATUSES, which asks whether a session is working: an idle or blocked
+ * session is not working, but it is open and it is what this overview counts.
+ */
+const OPEN = new Set(["busy", "running", "waiting", "idle", "blocked"]);
+
+const projectsRoot = (env) => env.CLAUDE_PROJECTS_ROOT || join(homedir(), ".claude", "projects");
+const prefsPath = (env) =>
+  env.MA_PREFS || join(homedir(), ".claude", "multi-agent-preferences.json");
+const storePath = (env) =>
+  env.MA_CREDENTIAL_STORE || new URL("../lib/credential-store.sh", import.meta.url).pathname;
+
+function listDir(dir) {
+  try {
+    return readdirSync(dir);
+  } catch {
+    return [];
+  }
+}
+
+let PRICES = null;
+/** The cost-table rate for a wire model id, by its `modelId`, else by rung name in the id. */
+export function rateFor(model) {
+  if (!PRICES) {
+    try {
+      PRICES = JSON.parse(
+        readFileSync(new URL("./cost-table.json", import.meta.url), "utf8"),
+      ).prices;
+    } catch {
+      PRICES = {};
+    }
+  }
+  if (typeof model !== "string" || !model) return null;
+  for (const rate of Object.values(PRICES)) if (rate?.modelId === model) return rate;
+  for (const [rung, rate] of Object.entries(PRICES)) {
+    if (/^[a-z]+$/.test(rung) && model.includes(`-${rung}-`)) return rate;
+  }
+  return null;
+}
+
+const emptyUsage = () => ({ in: 0, out: 0, cacheRead: 0, cacheWrite: 0, usd: 0, unpriced: 0 });
+
+function addUsage(acc, model, u) {
+  const tin = Number(u.input_tokens) || 0;
+  const tout = Number(u.output_tokens) || 0;
+  const read = Number(u.cache_read_input_tokens) || 0;
+  const write = Number(u.cache_creation_input_tokens) || 0;
+  acc.in += tin;
+  acc.out += tout;
+  acc.cacheRead += read;
+  acc.cacheWrite += write;
+  const usd = costUsd(rateFor(model), tin + write, tout, read);
+  if (usd === null) acc.unpriced++;
+  else acc.usd += usd;
+}
+
+/**
+ * Token usage in one transcript: the whole file, and the part at or after
+ * `since`. Each message id counts once in the file; `windowSeen`, shared
+ * across files, keeps a message a resumed or forked session copied into a
+ * second transcript from counting twice in the window. A file that cannot be
+ * read adds nothing and comes back `readable: false`.
+ *
+ * @returns {Promise<{total: object, window: object, readable: boolean}>}
+ */
+export async function transcriptUsage(path, since = 0, windowSeen = new Set()) {
+  const total = emptyUsage();
+  const window = emptyUsage();
+  const seen = new Set();
+  try {
+    await scan(path, (rec, msg, id) => {
+      if (id) {
+        if (seen.has(id)) return;
+        seen.add(id);
+      }
+      addUsage(total, msg.model, msg.usage);
+      const t = Date.parse(rec.timestamp);
+      if (!Number.isFinite(t) || t < since) return;
+      if (id) {
+        if (windowSeen.has(id)) return;
+        windowSeen.add(id);
+      }
+      addUsage(window, msg.model, msg.usage);
+    });
+  } catch {
+    return { total: emptyUsage(), window: emptyUsage(), readable: false };
+  }
+  return { total, window, readable: true };
+}
+
+async function scan(path, visit) {
+  const lines = createInterface({ input: createReadStream(path), crlfDelay: Infinity });
+  for await (const line of lines) {
+    if (!line.includes('"usage"')) continue;
+    let rec;
+    try {
+      rec = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    const msg = rec?.message;
+    if (rec?.type !== "assistant" || !msg?.usage || typeof msg.usage !== "object") continue;
+    // The host writes its own notices as "<synthetic>" messages; no model ran.
+    if (msg.model === "<synthetic>") continue;
+    visit(rec, msg, typeof msg.id === "string" ? msg.id : null);
+  }
+}
+
+function merge(a, b) {
+  for (const k of Object.keys(a)) a[k] += b[k];
+  return a;
+}
+
+/** The session transcript and its subagent transcripts, or [] when the id is unsafe or absent. */
+export function sessionFiles(root, sessionId) {
+  if (typeof sessionId !== "string" || !SAFE_SESSION_ID.test(sessionId)) return [];
+  for (const slug of listDir(root)) {
+    const main = join(root, slug, `${sessionId}.jsonl`);
+    if (!existsSync(main)) continue;
+    const sub = join(root, slug, sessionId, "subagents");
+    return [
+      main,
+      ...listDir(sub)
+        .filter((f) => f.endsWith(".jsonl"))
+        .map((f) => join(sub, f)),
+    ];
+  }
+  return [];
+}
+
+/** Every transcript under the root written at or after `since`. */
+function recentFiles(root, since) {
+  const out = [];
+  const visit = (dir, depth) => {
+    for (const name of listDir(dir)) {
+      const p = join(dir, name);
+      let st;
+      try {
+        st = statSync(p);
+      } catch {
+        continue;
+      }
+      if (st.isDirectory() && depth < 3) visit(p, depth + 1);
+      else if (st.isFile() && name.endsWith(".jsonl") && st.mtimeMs >= since) out.push(p);
+    }
+  };
+  visit(root, 0);
+  return out;
+}
+
+/** Logical credential names and whether each is registered; values are never read. */
+export function credentialStatus({ env = process.env } = {}) {
+  let mapping;
+  try {
+    mapping = JSON.parse(readFileSync(prefsPath(env), "utf8"))?.global?.keychainMapping ?? {};
+  } catch {
+    return { readable: false, items: [] };
+  }
+  const listed = spawnSync("bash", [storePath(env), "list"], {
+    encoding: "utf8",
+    timeout: 15000,
+    env,
+  });
+  if (listed.status !== 0) return { readable: false, items: [] };
+  const services = new Set(String(listed.stdout).split("\n").filter(Boolean));
+  const items = Object.entries(mapping)
+    .filter(([k, v]) => typeof k === "string" && typeof v === "string" && v)
+    .map(([name, service]) => ({ name, registered: services.has(service) }))
+    .sort((a, b) => a.name.localeCompare(b.name));
+  return { readable: true, items };
+}
+
+/** The author git would write in the current directory. */
+function gitIdentity(env) {
+  const read = (key) => {
+    const r = spawnSync("git", ["config", "--get", key], { encoding: "utf8", timeout: 5000, env });
+    return r.status === 0 ? r.stdout.trim() || null : null;
+  };
+  const name = read("user.name");
+  return name ? { name, email: read("user.email") } : null;
+}
+
+/**
+ * The whole overview as one document.
+ *
+ * @param {{env?: object, now?: number, all?: boolean}} [opts]
+ */
+export async function statusOverview({ env = process.env, now = Date.now(), all = false } = {}) {
+  const root = projectsRoot(env);
+  const since = now - WINDOW_MS;
+  const runs = buildIndex();
+  const rows = listSessions(env);
+
+  // One read per transcript: a live session's file is usually also one of the
+  // files written in the window.
+  const open = (Array.isArray(rows) ? rows : []).filter((s) => {
+    const state = s?.status ?? s?.state ?? null;
+    return !state || OPEN.has(state);
+  });
+  const files = new Map();
+  for (const s of open) for (const f of sessionFiles(root, s?.sessionId)) files.set(f, null);
+  for (const f of recentFiles(root, since)) files.set(f, null);
+  const windowSeen = new Set();
+  const last24h = emptyUsage();
+  let unreadable = 0;
+  for (const f of files.keys()) {
+    const u = await transcriptUsage(f, since, windowSeen);
+    if (!u.readable) unreadable++;
+    files.set(f, u);
+    merge(last24h, u.window);
+  }
+
+  const sessions = [];
+  for (const s of open) {
+    const state = s?.status ?? s?.state ?? null;
+    const { run, matchedBy } = matchRun(s?.sessionId, s?.cwd ?? null, null, runs);
+    const usage = emptyUsage();
+    for (const f of sessionFiles(root, s?.sessionId))
+      merge(usage, files.get(f)?.total ?? emptyUsage());
+    sessions.push({
+      sessionId: typeof s?.sessionId === "string" ? s.sessionId : null,
+      kind: typeof s?.kind === "string" ? s.kind : null,
+      state,
+      project: s?.cwd ? basename(s.cwd) : null,
+      startedAt: s?.startedAt ?? null,
+      run: run ? { taskId: run.taskId, currentPhase: run.currentPhase, status: run.status } : null,
+      matchedBy,
+      usage,
+    });
+  }
+
+  const accounts = listAccounts({ env });
+  return {
+    contractVersion: CONTRACT_VERSION,
+    generatedAt: new Date(now).toISOString(),
+    sessions,
+    sessionListReadable: Array.isArray(rows),
+    usage: {
+      liveSessions: sessions.reduce((acc, s) => merge(acc, { ...s.usage }), emptyUsage()),
+      last24h,
+      unreadableTranscripts: unreadable,
+    },
+    accounts: {
+      github: accounts.map((a) => ({ login: a.login, active: a.active })),
+      git: gitIdentity(env),
+    },
+    credentials: credentialStatus({ env }),
+    runs: all ? runs : runs.filter((r) => r.group !== "done" && r.group !== "unknown"),
+    runCounts: runs.reduce((acc, r) => ({ ...acc, [r.group]: (acc[r.group] ?? 0) + 1 }), {}),
+  };
+}
+
+const n = (x) =>
+  x >= 1e9
+    ? `${(x / 1e9).toFixed(1)}B`
+    : x >= 1e6
+      ? `${(x / 1e6).toFixed(1)}M`
+      : x >= 1e3
+        ? `${(x / 1e3).toFixed(1)}k`
+        : String(x);
+const usageLine = (u) =>
+  `in ${n(u.in)}  out ${n(u.out)}  cache read ${n(u.cacheRead)}  cache write ${n(u.cacheWrite)}  ~$${u.usd.toFixed(2)}` +
+  (u.unpriced ? `  (${u.unpriced} unpriced)` : "");
+
+export function renderHuman(doc) {
+  const out = [];
+  const busy = doc.sessions.filter((s) => s.state === "busy" || s.state === "running").length;
+  out.push(`Live sessions  (${doc.sessions.length}, ${busy} busy)`);
+  if (!doc.sessionListReadable)
+    out.push("  the session list could not be read (claude agents --json)");
+  for (const s of doc.sessions) {
+    const run = s.run ? `${s.run.taskId} ${phaseCell(s.run.currentPhase)}` : "no run";
+    out.push(
+      `  ${(s.sessionId ?? "-").slice(0, 8)}  ${(s.kind ?? "-").padEnd(11)}  ${(s.state ?? "-").padEnd(7)}  ` +
+        `${(s.project ?? "-").padEnd(24).slice(0, 24)}  ${run.padEnd(22)}  ~$${s.usage.usd.toFixed(2)}`,
+    );
+  }
+  out.push("");
+  out.push("Tokens");
+  out.push(`  live sessions  ${usageLine(doc.usage.liveSessions)}`);
+  out.push(`  last 24h       ${usageLine(doc.usage.last24h)}`);
+  if (doc.usage.unreadableTranscripts)
+    out.push(
+      `  ${doc.usage.unreadableTranscripts} transcript(s) could not be read; the totals leave them out`,
+    );
+  out.push("");
+  out.push("Accounts");
+  const gh = doc.accounts.github;
+  out.push(
+    `  github  ${gh.length ? gh.map((a) => (a.active ? `${a.login} (active)` : a.login)).join(", ") : "none logged in"}`,
+  );
+  if (doc.accounts.git)
+    out.push(`  git     ${doc.accounts.git.name} <${doc.accounts.git.email ?? "-"}>`);
+  out.push("");
+  const c = doc.credentials;
+  if (!c.readable) out.push("Credentials  could not be read");
+  else {
+    const missing = c.items.filter((i) => !i.registered).map((i) => i.name);
+    out.push(`Credentials  ${c.items.length - missing.length}/${c.items.length} registered`);
+    out.push(
+      `  ${
+        c.items
+          .filter((i) => i.registered)
+          .map((i) => i.name)
+          .join(", ") || "-"
+      }`,
+    );
+    if (missing.length) out.push(`  missing: ${missing.join(", ")}`);
+  }
+  out.push("");
+  const counts = Object.entries(doc.runCounts)
+    .map(([g, k]) => `${g} ${k}`)
+    .join(", ");
+  out.push(`Runs  (${counts || "none"})  -  runs-index.mjs lists them`);
+  const driven = new Set(doc.sessions.map((x) => x.run?.taskId).filter(Boolean));
+  for (const r of doc.runs.filter((x) => x.group === "waiting" || driven.has(x.taskId))) {
+    out.push(
+      `  ${r.taskId.padEnd(30).slice(0, 30)}  ${phaseCell(r.currentPhase).padStart(5)}  ${r.group}`,
+    );
+  }
+  return out.join("\n");
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const flags = { json: false, all: false };
+  for (const a of args) {
+    if (a === "--json") flags.json = true;
+    else if (a === "--all") flags.all = true;
+    else if (a === "-h" || a === "--help") {
+      process.stdout.write("Usage: status-overview.mjs [--json] [--all]\n");
+      return 0;
+    } else {
+      process.stderr.write(`status-overview: unknown argument ${a}\n`);
+      return 2;
+    }
+  }
+  const doc = await statusOverview({ all: flags.all });
+  process.stdout.write((flags.json ? JSON.stringify(doc, null, 2) : renderHuman(doc)) + "\n");
+  return 0;
+}
+
+if (invokedDirectly(import.meta.url)) {
+  runMain("status-overview", main);
+}