@mmerterden/multi-agent-pipeline 20.29.0 → 20.33.0

package/pipeline/scripts/run-results.mjs

@@ -0,0 +1,373 @@
+#!/usr/bin/env node
+/**
+ * @file run-results.mjs  -  what a run produced, read from its own files: the
+ * diff summary, the review findings, the test outcome, the screenshots and,
+ * for an analysis run, the documents it wrote.
+ *
+ * Every source is a file or a state field the phases already write; nothing is
+ * computed here that a phase did not record:
+ *
+ *   diff      agent-state `diffRisk` (diff-risk-score.mjs, Phase 3), else the
+ *             file and line counts of `.review-diff.txt`
+ *   findings  `triage-output.json` (Phase 3 triage), else agent-state
+ *             `review.findings` (a standalone /multi-agent:review)
+ *   tests     agent-state `uiTest` and the `test-summary` gate entries
+ *             (test-summary.mjs)
+ *   evidence  `.pipeline/evidence/*.png|jpg` and agent-state `visualEvidence`
+ *   analysis  agent-state `analysisSpec.outputs` (the documents' text) and
+ *             `analysis.docPath[]`
+ *
+ * Each file is looked for in the run's log directory, its `artifacts/` copy
+ * (worktree-finalize.sh salvages there) and, while the run is live, its
+ * worktree. The state's worktreePath is used only when git lists it as a
+ * worktree of the state's projectRoot. A part with no source is null, never
+ * an empty success.
+ *
+ * An analysis document is read only when its real path sits inside one of
+ * those roots and it is Markdown or HTML; any other path in the state keeps
+ * its name with `text: null`, so a state file cannot point the reader at an
+ * arbitrary file.
+ *
+ * Text a person or tool wrote (finding text, test names, document bodies) is
+ * scrubbed and run through the credential redactor; paths are reduced to file
+ * names.
+ *
+ * Usage:
+ *   run-results.mjs <task-id> [--json]
+ *   run-results.mjs <task-id> --evidence <name>      one image, base64, as JSON
+ *
+ * Exit: 0 printed, 2 usage, 3 no such run or image.
+ *
+ * @module pipeline/scripts/run-results
+ */
+
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from "node:fs";
+import { basename, join, sep } from "node:path";
+import { selectRuns } from "./runs-index.mjs";
+import { createRedactor } from "./transcript-index.mjs";
+import { readRegularJson } from "../lib/regular-file.mjs";
+import { runMain } from "../lib/fatal.mjs";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+
+export const CONTRACT_VERSION = "1.0.0";
+export const EVIDENCE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}\.(?:png|jpe?g)$/i;
+const MAX_IMAGE_BYTES = 8 * 1024 * 1024;
+const MAX_DOC_CHARS = 200_000;
+const MAX_FINDINGS = 200;
+const DOC_EXT = /\.(?:md|markdown|html?)$/i;
+
+export class ResultsError extends Error {}
+
+function readState(dir) {
+  for (const p of [join(dir, "agent-state.json"), join(dir, "artifacts", "agent-state.json")]) {
+    if (existsSync(p)) return readRegularJson(p, null) ?? {};
+  }
+  return {};
+}
+
+const realOrNull = (p) => {
+  try {
+    return realpathSync(p);
+  } catch {
+    return null;
+  }
+};
+
+/**
+ * The state's worktreePath when git lists it as a worktree of the state's
+ * projectRoot, else null.
+ */
+export function verifiedWorktree(state) {
+  const wt = typeof state.worktreePath === "string" ? realOrNull(state.worktreePath) : null;
+  const root = typeof state.projectRoot === "string" ? realOrNull(state.projectRoot) : null;
+  if (!wt || !root) return null;
+  let listing;
+  try {
+    listing = execFileSync("git", ["-C", root, "worktree", "list", "--porcelain"], {
+      encoding: "utf8",
+      timeout: 5000,
+      stdio: ["ignore", "pipe", "ignore"],
+    });
+  } catch {
+    return null;
+  }
+  for (const line of listing.split("\n")) {
+    if (line.startsWith("worktree ") && realOrNull(line.slice(9)) === wt) return wt;
+  }
+  return null;
+}
+
+/** The directories a run's files may be in, most durable first. */
+function roots(run, state) {
+  const out = [run.dir, join(run.dir, "artifacts")];
+  const wt = verifiedWorktree(state);
+  if (wt) out.push(wt);
+  return out;
+}
+
+/** The real path of `p` when it is a Markdown or HTML file inside one of `dirs`. */
+function containedDoc(p, dirs) {
+  const real = realOrNull(p);
+  if (!real || !DOC_EXT.test(real)) return null;
+  for (const d of dirs) {
+    const rd = realOrNull(d);
+    if (rd && real.startsWith(rd + sep)) return real;
+  }
+  return null;
+}
+
+function firstExisting(dirs, rel) {
+  for (const d of dirs) {
+    const p = join(d, rel);
+    try {
+      if (statSync(p).isFile()) return p;
+    } catch {
+      // Not in this root.
+    }
+  }
+  return null;
+}
+
+function diffPart(state, dirs) {
+  const r = state.diffRisk;
+  if (r && typeof r === "object" && Number.isInteger(r.files)) {
+    return {
+      files: r.files,
+      added: Number.isInteger(r.loc_added) ? r.loc_added : null,
+      removed: Number.isInteger(r.loc_removed) ? r.loc_removed : null,
+      maxScore: typeof r.max_score === "number" ? r.max_score : null,
+      signals: Array.isArray(r.signals) ? r.signals.map(String) : [],
+      changed: [],
+      source: "diffRisk",
+    };
+  }
+  const p = firstExisting(dirs, ".review-diff.txt");
+  if (!p) return null;
+  const files = new Set();
+  let added = 0;
+  let removed = 0;
+  for (const line of readFileSync(p, "utf8").split("\n")) {
+    const m = /^\+\+\+ b\/(.+)$/.exec(line);
+    if (m) files.add(m[1]);
+    else if (line.startsWith("+") && !line.startsWith("+++")) added++;
+    else if (line.startsWith("-") && !line.startsWith("---")) removed++;
+  }
+  return {
+    files: files.size,
+    added,
+    removed,
+    maxScore: null,
+    signals: [],
+    changed: [...files].slice(0, 200).map((f) => basename(f)),
+    source: ".review-diff.txt",
+  };
+}
+
+function findingsPart(state, dirs, R) {
+  const p = firstExisting(dirs, "triage-output.json");
+  const triage = p ? readRegularJson(p, null) : null;
+  const shape = (f, verdict) => ({
+    severity: String(f?.severity ?? "unknown"),
+    file: f?.file ? basename(String(f.file)) : null,
+    line: Number.isInteger(f?.line) ? f.line : null,
+    issue: R.excerpt(f?.issue ?? f?.title ?? f?.message ?? "", 400),
+    fix: f?.fix ? R.excerpt(f.fix, 400) : null,
+    reviewer: f?.reviewer ? String(f.reviewer) : null,
+    verdict,
+  });
+  if (triage && typeof triage === "object") {
+    const items = [];
+    for (const v of ["accepted", "deferred", "rejected"]) {
+      for (const f of Array.isArray(triage[v]) ? triage[v] : []) items.push(shape(f, v));
+    }
+    return {
+      approved: typeof triage.approved === "boolean" ? triage.approved : null,
+      items: items.slice(0, MAX_FINDINGS),
+      total: items.length,
+      source: "triage-output.json",
+    };
+  }
+  const review = state.review;
+  if (review && Array.isArray(review.findings)) {
+    const items = review.findings.map((f) => shape(f, f?.status ?? f?.verdict ?? "accepted"));
+    return {
+      approved: null,
+      items: items.slice(0, MAX_FINDINGS),
+      total: items.length,
+      source: "agent-state review",
+    };
+  }
+  return null;
+}
+
+function testsPart(state, R) {
+  const gates = Array.isArray(state.gates) ? state.gates : [];
+  const summaries = gates.filter(
+    (g) => g && (g.gate === "test-summary" || g.name === "test-summary"),
+  );
+  const last = summaries.at(-1) ?? null;
+  const ui = state.uiTest && typeof state.uiTest === "object" ? state.uiTest : null;
+  if (!last && !ui) return null;
+  const count = (k) => {
+    const v = last?.counts?.[k] ?? last?.[k];
+    return Number.isInteger(v) ? v : null;
+  };
+  return {
+    status: last?.status ?? last?.verdict ?? ui?.status ?? null,
+    passed: count("passed"),
+    failed: count("failed"),
+    skipped: count("skipped"),
+    uiTest: ui
+      ? {
+          ran: ui.ran === true,
+          status: ui.status ?? null,
+          notRunReason: ui.notRunReason ? R.excerpt(ui.notRunReason, 200) : null,
+        }
+      : null,
+    source: last ? "test-summary gate" : "agent-state uiTest",
+  };
+}
+
+function evidenceDirs(dirs) {
+  return dirs.map((d) => join(d, ".pipeline", "evidence")).filter((d) => existsSync(d));
+}
+
+function evidencePart(state, dirs) {
+  const names = new Map();
+  for (const d of evidenceDirs(dirs)) {
+    let list;
+    try {
+      list = readdirSync(d);
+    } catch {
+      continue;
+    }
+    for (const f of list) if (EVIDENCE_NAME.test(f) && !names.has(f)) names.set(f, null);
+  }
+  const ve = state.visualEvidence;
+  for (const side of ["before", "after"]) {
+    for (const e of Array.isArray(ve?.[side]) ? ve[side] : []) {
+      const f = typeof e?.file === "string" ? basename(e.file) : null;
+      if (f && names.has(f)) names.set(f, side);
+    }
+  }
+  return [...names.entries()].sort().map(([name, phase]) => ({ name, phase }));
+}
+
+function analysisPart(state, dirs, R) {
+  const outputs = state.analysisSpec?.outputs;
+  const paths = [
+    ...(Array.isArray(outputs?.localPaths) ? outputs.localPaths : []),
+    ...(Array.isArray(state.analysis?.docPath) ? state.analysis.docPath : []),
+  ].filter((p) => typeof p === "string");
+  if (!paths.length && !outputs) return null;
+  const documents = [];
+  for (const p of [...new Set(paths)]) {
+    const real = containedDoc(p, dirs);
+    try {
+      if (!real || !statSync(real).isFile()) throw new Error("not readable here");
+      const text = readFileSync(real, "utf8").slice(0, MAX_DOC_CHARS);
+      documents.push({ name: basename(p), text: R.clean(text) });
+    } catch {
+      documents.push({ name: basename(p), text: null });
+    }
+  }
+  return {
+    documents,
+    confluencePages: Array.isArray(outputs?.confluencePageUrls)
+      ? outputs.confluencePageUrls.length
+      : 0,
+    jiraIssueKey:
+      typeof outputs?.jiraIssueKey === "string" &&
+      /^[A-Z][A-Z0-9_]*-[0-9]+$/.test(outputs.jiraIssueKey)
+        ? outputs.jiraIssueKey
+        : null,
+  };
+}
+
+function runFor(taskId) {
+  const { found, records } = selectRuns({ taskId });
+  if (!found || !records.length) throw new ResultsError("no such run");
+  return records[0];
+}
+
+/** Everything a run produced, as one document. */
+export function runResults(taskId) {
+  const run = runFor(taskId);
+  const state = readState(run.dir);
+  const dirs = roots(run, state);
+  const R = createRedactor();
+  return {
+    contractVersion: CONTRACT_VERSION,
+    taskId: run.taskId,
+    diff: diffPart(state, dirs),
+    findings: findingsPart(state, dirs, R),
+    tests: testsPart(state, R),
+    evidence: evidencePart(state, dirs),
+    analysis: analysisPart(state, dirs, R),
+  };
+}
+
+/**
+ * One screenshot as base64. The name must be a bare image file name found in
+ * one of the run's evidence directories; its real path must stay inside it.
+ */
+export function runEvidence(taskId, name) {
+  if (typeof name !== "string" || !EVIDENCE_NAME.test(name))
+    throw new ResultsError("no such image");
+  const run = runFor(taskId);
+  const state = readState(run.dir);
+  for (const d of evidenceDirs(roots(run, state))) {
+    const p = join(d, name);
+    let real;
+    let realDir;
+    try {
+      real = realpathSync(p);
+      realDir = realpathSync(d);
+    } catch {
+      continue;
+    }
+    if (!real.startsWith(realDir + sep)) continue;
+    const st = statSync(real);
+    if (!st.isFile() || st.size > MAX_IMAGE_BYTES) continue;
+    const mime = /\.png$/i.test(name) ? "image/png" : "image/jpeg";
+    return {
+      contractVersion: CONTRACT_VERSION,
+      taskId: run.taskId,
+      name,
+      mime,
+      data: readFileSync(real).toString("base64"),
+    };
+  }
+  throw new ResultsError("no such image");
+}
+
+function main(argv) {
+  const [taskId, ...rest] = argv;
+  if (!taskId) {
+    process.stderr.write("usage: run-results.mjs <task-id> [--json] | --evidence <name>\n");
+    return 2;
+  }
+  try {
+    if (rest[0] === "--evidence") {
+      process.stdout.write(JSON.stringify(runEvidence(taskId, rest[1])) + "\n");
+      return 0;
+    }
+    if (rest.some((a) => a !== "--json")) {
+      process.stderr.write("usage: run-results.mjs <task-id> [--json] | --evidence <name>\n");
+      return 2;
+    }
+    process.stdout.write(JSON.stringify(runResults(taskId), null, 2) + "\n");
+    return 0;
+  } catch (err) {
+    if (!(err instanceof ResultsError)) throw err;
+    process.stderr.write(`run-results: ${err.message}\n`);
+    return 3;
+  }
+}
+
+if (invokedDirectly(import.meta.url)) {
+  runMain("run-results", () => {
+    process.exitCode = main(process.argv.slice(2));
+  });
+}

package/pipeline/scripts/runs-index.mjs

@@ -81,12 +81,12 @@ const PHASE_WAITING_FROM = PHASE_CONTRACT.thresholds.waitingFromPhase;
  */
 const LAST_PHASE = PHASE_CONTRACT.phases[PHASE_CONTRACT.phases.length - 1].id;
 
-export const CONTRACT_VERSION = "1.2.0";
+export const CONTRACT_VERSION = "1.3.0";
 
 const STATE_MAX_BYTES = 16 * 1024 * 1024;
 
 /** The run fields runs-index.schema.json marks x-sensitivity local. */
-export const LOCAL_RUN_FIELDS = ["dir", "worktreePath", "prUrl", "duplicateOf"];
+export const LOCAL_RUN_FIELDS = ["dir", "worktreePath", "prUrl", "duplicateOf", "channelUrls"];
 
 const GROUPS = {
   waiting: "Waiting on you",
@@ -226,6 +226,57 @@ function runTypeOf(state, kind) {
   return "development";
 }
 
+const JIRA_KEY = /^[A-Z][A-Z0-9_]{0,30}-[0-9]{1,9}$/;
+const GH_PART = /^[A-Za-z0-9_.-]{1,100}$/;
+
+/**
+ * Where a run's work came from: its Jira key, its GitHub issue as
+ * owner/repo#N, or free text. A GitHub issue that got a Jira carries both.
+ */
+export function sourceOf(state) {
+  const jira =
+    typeof state?.jiraId === "string" && JIRA_KEY.test(state.jiraId) ? state.jiraId : null;
+  const g = state?.githubIssue;
+  const issue =
+    g &&
+    GH_PART.test(String(g.owner)) &&
+    GH_PART.test(String(g.repo)) &&
+    Number.isInteger(g.number) &&
+    g.number > 0
+      ? `${g.owner}/${g.repo}#${g.number}`
+      : null;
+  if (issue) return { kind: "github", jira, issue };
+  if (jira) return { kind: "jira", jira, issue: null };
+  if (state?.inputType === "free-text") return { kind: "text", jira: null, issue: null };
+  return null;
+}
+
+const CHANNELS = ["pr", "jira", "confluence", "wiki", "issue"];
+const CHANNEL_STATES = new Set(["posted", "skipped", "failed"]);
+
+/** Where Phase 5 sent the report: each adapter's outcome, in a fixed order. */
+export function channelsOf(state) {
+  const r = state?.channelResults;
+  if (!r || typeof r !== "object") return { channels: [], urls: {} };
+  const channels = [];
+  const urls = {};
+  for (const c of CHANNELS) {
+    const v = r[c];
+    if (!v || !CHANNEL_STATES.has(v.status)) continue;
+    channels.push({ channel: c, status: v.status });
+    if (typeof v.url === "string" && /^https:\/\/\S+$/.test(v.url)) urls[c] = v.url;
+  }
+  return { channels, urls };
+}
+
+/** The run's PR number, from the state or its PR address; null when neither names one. */
+export function prNumberOf(state, prUrl) {
+  const n = state?.pr?.number;
+  if (Number.isInteger(n) && n > 0) return n;
+  const m = /\/(?:pull|pull-requests)\/(\d{1,9})(?:[/?#]|$)/.exec(String(prUrl ?? ""));
+  return m && Number(m[1]) > 0 ? Number(m[1]) : null;
+}
+
 /** The issue summary Phase 0 recorded, else the input the run was launched with. */
 function titleOf(state) {
   return textOrNull(state?.title) ?? textOrNull(state?.launchRequest?.input);
@@ -388,6 +439,7 @@ export function buildIndex() {
     const tracker = readJson(trackerPath);
     const phases = summarisePhases(tracker);
     const prUrl = state?.pr?.url ?? state?.prUrl ?? null;
+    const prNumber = prNumberOf(state, prUrl);
     const kind = statePath ? "pipeline" : trackerPath ? "tracker-only" : "empty";
     const currentPhase = Number.isFinite(Number(state?.currentPhase))
       ? Number(state.currentPhase)
@@ -443,7 +495,15 @@ export function buildIndex() {
       // exactly what nobody measured.
       unlockedWrites: Number.isInteger(tracker?.unlockedWrites) ? tracker.unlockedWrites : 0,
       runType: runTypeOf(state, kind),
+      mode:
+        typeof state?.mode === "string" && /^[a-z][a-z-]{0,30}$/.test(state.mode)
+          ? state.mode
+          : null,
       title: titleOf(state),
+      source: sourceOf(state),
+      prNumber,
+      channels: channelsOf(state).channels,
+      channelUrls: channelsOf(state).urls,
       progress: runProgress(phases.phases),
       currentTask: currentTaskOf(phases.phases, currentPhase),
       // Whether the session the run was launched under is alive, from