@mmerterden/multi-agent-pipeline 20.26.0 → 20.27.0

package/pipeline/scripts/render-flow-run.mjs

@@ -0,0 +1,583 @@
+#!/usr/bin/env node
+/**
+ * render-flow-run.mjs  -  the "Test Run (Xray)" content block of /multi-agent:channels.
+ *
+ * A flow runner records every UI scenario run in `<RUN_ROOT>/history.jsonl`:
+ * a "run" line with per-scenario results, and, once the run is delivered to
+ * Xray, a "delivery" line with the same runId naming the Test Execution it
+ * created per domain. This script reads that file and renders the block that
+ * lets a reader get from the dev task to the run that proves it.
+ *
+ * It is a read-only consumer. It never calls the tracker, never creates or
+ * updates an execution and never invents a key: a domain with no execution in
+ * the delivery line is reported as such. The only process it starts is `git`,
+ * to list the task branch's commits for discovery.
+ *
+ * Run resolution, first hit wins:
+ *   1. --flow-run <runId>[,<runId>...]
+ *   2. state.evidence.flowRuns[] in agent-state.json
+ *   3. discovery: runs whose recorded commit is on the task branch (in
+ *      base..branch) or whose results carry ticket == the task's Jira key.
+ *      Discovery is a suggestion; the caller shows the candidates for the user
+ *      to untick.
+ *
+ * Usage:
+ *   node render-flow-run.mjs [--flow-run id,id] [--state agent-state.json]
+ *        [--history path] [--run-root dir] [--config scenario.config.yaml]
+ *        [--repo dir] [--branch b] [--base b] [--ticket KEY]
+ *        [--jira-base url] [--lang tr|en] [--heading-level 2|3] [--json]
+ *
+ * Without --json it prints the markdown block, or nothing when the block is
+ * unavailable. Exit 0 in both cases; exit 2 on a usage error.
+ */
+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+import { homedir } from "node:os";
+import { spawnSync } from "node:child_process";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+
+export const RUN = "run";
+export const DELIVERY = "delivery";
+const CONFIG_NAME = "scenario.config.yaml";
+const XRAY_STATUS = {
+  passed: "PASS",
+  expectedFailure: "PASS",
+  failed: "FAIL",
+  skipped: "TODO",
+  indeterminate: "ABORTED",
+};
+const ALWAYS_COUNTED = ["PASS", "FAIL", "ABORTED"];
+
+const LABELS = {
+  tr: {
+    heading: "Test Koşumu",
+    field: "Alan",
+    value: "Değer",
+    execution: "Test Execution",
+    setPlan: "Test Set / Plan",
+    run: "Koşu",
+    result: "Sonuç",
+    scenario: "Senaryo",
+    xrayTest: "Xray testi",
+    status: "Durum",
+    notDelivered: "Xray'e teslim edilmedi",
+    deliverHint: "Teslim etmek için",
+    skipped: "atlandı",
+    noExecution: "execution oluşturulmadı",
+    noTestKey: "Xray testi yok (xray-sync)",
+    missingHistory: (path) => `koşu geçmişi yok: ${path}`,
+    noMatch: (what) => `bu iş için koşu bulunamadı (${what})`,
+  },
+  en: {
+    heading: "Test Run",
+    field: "Field",
+    value: "Value",
+    execution: "Test Execution",
+    setPlan: "Test Set / Plan",
+    run: "Run",
+    result: "Result",
+    scenario: "Scenario",
+    xrayTest: "Xray test",
+    status: "Status",
+    notDelivered: "Not delivered to Xray",
+    deliverHint: "To deliver it",
+    skipped: "skipped",
+    noExecution: "no execution created",
+    noTestKey: "No Xray test (xray-sync)",
+    missingHistory: (path) => `no run history: ${path}`,
+    noMatch: (what) => `no run found for this task (${what})`,
+  },
+};
+
+export function readHistory(path) {
+  let text;
+  try {
+    text = readFileSync(path, "utf-8");
+  } catch {
+    return { exists: false, entries: [] };
+  }
+  const entries = [];
+  for (const line of text.split(/\r?\n/)) {
+    let value;
+    try {
+      value = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    // A run id is a path segment and a command argument; anything else is not a run.
+    if (
+      value &&
+      typeof value === "object" &&
+      !Array.isArray(value) &&
+      typeof value.runId === "string" &&
+      RUN_ID.test(value.runId) &&
+      !/^\.+$/.test(value.runId)
+    )
+      entries.push(value);
+  }
+  return { exists: true, entries };
+}
+
+const RUN_ID = /^[A-Za-z0-9._-]+$/;
+
+/** runId -> { run, delivery }; delivery is the latest delivery line for that runId, or null. */
+export function pairRuns(entries) {
+  const deliveries = new Map();
+  entries.forEach((entry, index) => {
+    if (entry.type !== DELIVERY) return;
+    const current = deliveries.get(entry.runId);
+    const order = [entry.at || "", index];
+    if (!current || compareOrder(order, current.order) >= 0)
+      deliveries.set(entry.runId, { entry, order });
+  });
+  const paired = new Map();
+  for (const entry of entries) {
+    if (entry.type !== RUN) continue;
+    const delivery = deliveries.get(entry.runId)?.entry;
+    paired.set(entry.runId, {
+      run: entry,
+      delivery: delivery
+        ? {
+            executions: delivery.executions || {},
+            skipped: delivery.skipped || {},
+            at: delivery.at || null,
+          }
+        : null,
+    });
+  }
+  return paired;
+}
+
+function compareOrder([atA, indexA], [atB, indexB]) {
+  if (atA !== atB) return atA < atB ? -1 : 1;
+  return indexA - indexB;
+}
+
+function commitMatches(recorded, branchCommits) {
+  if (!recorded) return false;
+  const short = String(recorded).toLowerCase();
+  if (short.length < 7) return false;
+  return branchCommits.some((sha) => {
+    const full = String(sha).toLowerCase();
+    return full.startsWith(short) || short.startsWith(full);
+  });
+}
+
+function carriesTicket(run, ticket) {
+  if (!ticket) return false;
+  if (run.ticket === ticket) return true;
+  return (run.results || []).some((result) => result.ticket === ticket);
+}
+
+export function resolveRuns({
+  flag = [],
+  stateRuns = [],
+  entries = [],
+  branchCommits = [],
+  ticket = null,
+}) {
+  if (flag.length) return { source: "flag", runIds: [...flag] };
+  if (stateRuns.length) return { source: "state", runIds: [...stateRuns] };
+  const runIds = entries
+    .filter((entry) => entry.type === RUN)
+    .filter((run) => commitMatches(run.commit, branchCommits) || carriesTicket(run, ticket))
+    .map((run) => run.runId);
+  return runIds.length
+    ? { source: "discovery", runIds: [...new Set(runIds)] }
+    : { source: null, runIds: [] };
+}
+
+/** Scalars from a block-style YAML mapping, keyed by dotted path. Lists and flow style are skipped. */
+export function yamlScalars(text) {
+  const values = {};
+  const stack = [];
+  for (const raw of text.split(/\r?\n/)) {
+    if (!raw.trim() || raw.trimStart().startsWith("#") || raw.trimStart().startsWith("- "))
+      continue;
+    const match = raw.match(/^(\s*)([A-Za-z0-9_.-]+):(?:\s+(.*))?$/);
+    if (!match) continue;
+    const indent = match[1].length;
+    while (stack.length && stack[stack.length - 1].indent >= indent) stack.pop();
+    const path = [...stack.map((item) => item.key), match[2]].join(".");
+    const value = (match[3] || "").replace(/\s+#.*$/, "").trim();
+    if (value === "") {
+      stack.push({ indent, key: match[2] });
+    } else {
+      values[path] = value.replace(/^(["'])(.*)\1$/, "$2");
+    }
+  }
+  return values;
+}
+
+function findConfig(start) {
+  let dir = resolve(start);
+  for (;;) {
+    const candidate = join(dir, CONFIG_NAME);
+    if (existsSync(candidate)) return candidate;
+    const parent = dirname(dir);
+    if (parent === dir) return null;
+    dir = parent;
+  }
+}
+
+function newestPluginFile(relative, home) {
+  const cache = join(home, ".claude", "plugins", "cache");
+  let best = null;
+  const visit = (dir, depth) => {
+    if (depth > 3) return;
+    let names;
+    try {
+      names = readdirSync(dir);
+    } catch {
+      return;
+    }
+    for (const name of names) {
+      const child = join(dir, name);
+      const candidate = join(child, relative);
+      if (existsSync(candidate)) {
+        const mtime = statSync(candidate).mtimeMs;
+        if (!best || mtime > best.mtime) best = { path: candidate, mtime };
+      } else {
+        visit(child, depth + 1);
+      }
+    }
+  };
+  visit(cache, 0);
+  return best?.path || null;
+}
+
+function resolveHook(value, configDir, env, home) {
+  if (!value) return null;
+  if (value.startsWith("plugin:")) {
+    const relative = value.slice("plugin:".length);
+    if (env.FLOW_HOOKS_HOME) {
+      const direct = join(env.FLOW_HOOKS_HOME, relative.replace(/^hooks\//, ""));
+      if (existsSync(direct)) return direct;
+    }
+    return newestPluginFile(relative, home);
+  }
+  return isAbsolute(value) ? value : join(configDir, value);
+}
+
+/** jiraBase, the xray deliver hook and each domain's setKey / planKey, from the runner's own settings. */
+export function readFlowSettings({ configPath, env = process.env, home = homedir() }) {
+  const settings = { jiraBase: null, xrayHook: null, xray: {} };
+  let values = {};
+  if (configPath && existsSync(configPath)) values = yamlScalars(readFileSync(configPath, "utf-8"));
+  const base = env.JIRA_BASE || values["deliver.jiraBase"] || null;
+  settings.jiraBase = base ? base.replace(/\/+$/, "") : null;
+  if (!configPath) return settings;
+  const configDir = dirname(configPath);
+  settings.xrayHook = resolveHook(values["deliver.hooks.xray"], configDir, env, home);
+  const data = values["catalog.data"];
+  if (!data) return settings;
+  const xrayDir = join(isAbsolute(data) ? data : join(configDir, data), "xray");
+  let files;
+  try {
+    files = readdirSync(xrayDir).filter((name) => name.endsWith(".yaml"));
+  } catch {
+    return settings;
+  }
+  for (const name of files) {
+    const domain = name.slice(0, -".yaml".length);
+    const domainValues = yamlScalars(readFileSync(join(xrayDir, name), "utf-8"));
+    const keys = {};
+    if (domainValues.setKey) keys.setKey = domainValues.setKey;
+    if (domainValues.planKey) keys.planKey = domainValues.planKey;
+    if (Object.keys(keys).length) settings.xray[domain] = keys;
+  }
+  return settings;
+}
+
+const ISSUE_KEY = /^[A-Z][A-Z0-9]*-\d+$/;
+
+// History values are written by another tool; every one is escaped for a table cell,
+// and only a well-formed issue key becomes a link.
+function cell(text) {
+  return String(text)
+    .replace(/\r?\n/g, " ")
+    .replace(/[\\`*_[\]()!<>|#~]/g, "\\$&");
+}
+
+// A value made only of safe characters reads best as code; anything else is
+// escaped plain text, because a backtick in it would end the code span.
+function code(value, safe = /^[A-Za-z0-9._/:-]+$/) {
+  const text = String(value);
+  return safe.test(text) ? `\`${text}\`` : cell(text);
+}
+
+function keyRef(key, jiraBase) {
+  return jiraBase && ISSUE_KEY.test(String(key))
+    ? `[${key}](${jiraBase}/browse/${key})`
+    : cell(key);
+}
+
+function shellQuote(value) {
+  return `'${String(value).replace(/'/g, "'\\''")}'`;
+}
+
+function runDomains(run) {
+  const named = run.domains || [];
+  const fromResults = (run.results || []).map((result) => result.domain).filter(Boolean);
+  return [...new Set([...named, ...fromResults])].sort();
+}
+
+function executionCell(entry, labels, jiraBase) {
+  const { run, delivery } = entry;
+  if (!delivery) return labels.notDelivered;
+  const parts = runDomains(run).map((domain) => {
+    const key = delivery.executions[domain];
+    if (key) return `${keyRef(key, jiraBase)} (${cell(domain)})`;
+    const reason = delivery.skipped[domain];
+    if (reason) return `${cell(domain)}: ${labels.skipped} (${code(reason)})`;
+    return `${cell(domain)}: ${labels.noExecution} (lane: ${cell(run.lane || "?")})`;
+  });
+  for (const [domain, key] of Object.entries(delivery.executions)) {
+    if (!runDomains(run).includes(domain)) parts.push(`${keyRef(key, jiraBase)} (${cell(domain)})`);
+  }
+  return parts.join(" · ") || labels.noExecution;
+}
+
+function setPlanCell(run, xray, jiraBase) {
+  const parts = [];
+  for (const domain of runDomains(run)) {
+    const keys = xray[domain];
+    if (!keys) continue;
+    const named = [];
+    if (keys.setKey) named.push(`Set ${keyRef(keys.setKey, jiraBase)}`);
+    if (keys.planKey) named.push(`Plan ${keyRef(keys.planKey, jiraBase)}`);
+    parts.push(
+      runDomains(run).length > 1 ? `${cell(domain)}: ${named.join(" · ")}` : named.join(" · "),
+    );
+  }
+  return parts.join("; ");
+}
+
+function runCell(run) {
+  const parts = [code(run.runId)];
+  if (run.lane) parts.push(cell(run.lane));
+  if (run.commit) parts.push(code(String(run.commit).slice(0, 7), /^[0-9a-fA-F]+$/));
+  if (run.branch) parts.push(code(run.branch));
+  return parts.join(" · ");
+}
+
+function resultCell(run) {
+  const counts = {};
+  for (const result of run.results || []) {
+    const status = XRAY_STATUS[result.status] || "TODO";
+    counts[status] = (counts[status] || 0) + 1;
+  }
+  const extra = Object.keys(counts)
+    .filter((status) => !ALWAYS_COUNTED.includes(status))
+    .sort();
+  return [...ALWAYS_COUNTED, ...extra]
+    .map((status) => `${status} ${counts[status] || 0}`)
+    .join(" · ");
+}
+
+function deliverCommand(run, runRoot, xrayHook) {
+  const hook = xrayHook ? shellQuote(xrayHook) : "<xray_deliver.py>";
+  return `python3 ${hook} --targets xray --run-dir ${shellQuote(join(runRoot, String(run.runId)))}`;
+}
+
+/** One block for the resolved runs. `runs` are pairRuns() values; unknown ids are filtered by the caller. */
+export function renderBlock(
+  runs,
+  { lang = "tr", jiraBase = null, xray = {}, runRoot, xrayHook = null, headingLevel = 3 } = {},
+) {
+  const labels = LABELS[lang] || LABELS.en;
+  const lines = [`${"#".repeat(headingLevel)} ${labels.heading}`];
+  for (const entry of runs) {
+    const { run } = entry;
+    lines.push("");
+    if (runs.length > 1) lines.push(`**${labels.run} ${cell(run.runId)}**`, "");
+    lines.push(`| ${labels.field} | ${labels.value} |`, "| --- | --- |");
+    lines.push(`| ${labels.execution} | ${executionCell(entry, labels, jiraBase)} |`);
+    const setPlan = setPlanCell(run, xray, jiraBase);
+    if (setPlan) lines.push(`| ${labels.setPlan} | ${setPlan} |`);
+    lines.push(`| ${labels.run} | ${runCell(run)} |`);
+    lines.push(`| ${labels.result} | ${resultCell(run)} |`);
+    const results = run.results || [];
+    if (results.length) {
+      lines.push(
+        "",
+        `| ${labels.scenario} | ${labels.xrayTest} | ${labels.status} |`,
+        "| --- | --- | --- |",
+      );
+      for (const result of results) {
+        const test = result.testKey ? keyRef(result.testKey, jiraBase) : labels.noTestKey;
+        lines.push(`| ${cell(result.id)} | ${test} | ${cell(result.status || "?")} |`);
+      }
+    }
+    if (!entry.delivery) {
+      lines.push(
+        "",
+        `${labels.deliverHint}:`,
+        "",
+        "```",
+        deliverCommand(run, runRoot, xrayHook),
+        "```",
+      );
+    }
+  }
+  return lines.join("\n") + "\n";
+}
+
+export function branchCommitsFor({ repo, branch, base }) {
+  if (!repo || !branch || !base) return [];
+  // A ref starting with "-" would be read by git as an option.
+  if (String(base).startsWith("-") || String(branch).startsWith("-")) return [];
+  const listed = spawnSync(
+    "git",
+    ["-C", repo, "rev-list", "--end-of-options", `${base}..${branch}`],
+    {
+      encoding: "utf-8",
+    },
+  );
+  if (listed.status !== 0) return [];
+  return listed.stdout.split("\n").filter(Boolean);
+}
+
+export function buildReport({
+  historyPath,
+  flag = [],
+  stateRuns = [],
+  branchCommits = [],
+  ticket = null,
+  lang = "tr",
+  jiraBase = null,
+  xray = {},
+  xrayHook = null,
+  runRoot = null,
+  headingLevel = 3,
+}) {
+  const labels = LABELS[lang] || LABELS.en;
+  const root = runRoot || dirname(historyPath);
+  const history = readHistory(historyPath);
+  const empty = {
+    available: false,
+    source: null,
+    runIds: [],
+    missing: [],
+    suggestion: false,
+    candidates: [],
+    markdown: "",
+  };
+  if (!history.exists) return { ...empty, reason: labels.missingHistory(historyPath) };
+  const paired = pairRuns(history.entries);
+  const resolved = resolveRuns({
+    flag,
+    stateRuns,
+    entries: history.entries,
+    branchCommits,
+    ticket,
+  });
+  const found = resolved.runIds.filter((id) => paired.has(id));
+  const missing = resolved.runIds.filter((id) => !paired.has(id));
+  if (!found.length) {
+    const what =
+      [ticket, missing.length ? missing.join(", ") : null].filter(Boolean).join(" · ") ||
+      historyPath;
+    return { ...empty, source: resolved.source, missing, reason: labels.noMatch(what) };
+  }
+  const runs = found.map((id) => paired.get(id));
+  return {
+    available: true,
+    reason: null,
+    source: resolved.source,
+    suggestion: resolved.source === "discovery",
+    runIds: found,
+    missing,
+    candidates: runs.map(({ run, delivery }) => ({
+      runId: run.runId,
+      commit: run.commit || null,
+      lane: run.lane || null,
+      delivered: Boolean(delivery && Object.keys(delivery.executions).length),
+    })),
+    markdown: renderBlock(runs, { lang, jiraBase, xray, runRoot: root, xrayHook, headingLevel }),
+  };
+}
+
+function parseArgs(argv) {
+  const opts = { flowRuns: [], json: false };
+  const valued = new Set([
+    "--flow-run",
+    "--state",
+    "--history",
+    "--run-root",
+    "--config",
+    "--repo",
+    "--branch",
+    "--base",
+    "--ticket",
+    "--jira-base",
+    "--lang",
+    "--heading-level",
+  ]);
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--json") {
+      opts.json = true;
+    } else if (valued.has(arg)) {
+      const value = argv[++i];
+      if (value === undefined) throw new Error(`${arg} needs a value`);
+      if (arg === "--flow-run")
+        opts.flowRuns.push(
+          ...value
+            .split(",")
+            .map((id) => id.trim())
+            .filter(Boolean),
+        );
+      else opts[arg.slice(2).replace(/-(\w)/g, (_, c) => c.toUpperCase())] = value;
+    } else {
+      throw new Error(`unknown argument: ${arg}`);
+    }
+  }
+  return opts;
+}
+
+function readState(path) {
+  if (!path) return {};
+  try {
+    return JSON.parse(readFileSync(path, "utf-8"));
+  } catch {
+    return {};
+  }
+}
+
+function main(argv) {
+  let opts;
+  try {
+    opts = parseArgs(argv);
+  } catch (error) {
+    process.stderr.write(`render-flow-run: ${error.message}\n`);
+    return 2;
+  }
+  const state = readState(opts.state);
+  const runRoot = opts.runRoot || join(homedir(), "FlowRuns");
+  const historyPath = opts.history || join(runRoot, "history.jsonl");
+  const repo = opts.repo || state.worktreePath || state.projectRoot || process.cwd();
+  const configPath = opts.config || process.env.FLOW_CONFIG || findConfig(repo);
+  const settings = readFlowSettings({ configPath, env: process.env });
+  const ticket =
+    opts.ticket || (/^[A-Z][A-Z0-9]+-\d+$/.test(state.taskId || "") ? state.taskId : null);
+  const branch = opts.branch || state.branch || null;
+  const base = opts.base || state.baseBranch || null;
+  const report = buildReport({
+    historyPath,
+    flag: opts.flowRuns,
+    stateRuns: Array.isArray(state.evidence?.flowRuns) ? state.evidence.flowRuns : [],
+    branchCommits: branchCommitsFor({ repo, branch, base }),
+    ticket,
+    lang: opts.lang || "tr",
+    jiraBase: opts.jiraBase ? opts.jiraBase.replace(/\/+$/, "") : settings.jiraBase,
+    xray: settings.xray,
+    xrayHook: settings.xrayHook,
+    runRoot: opts.history && !opts.runRoot ? dirname(historyPath) : runRoot,
+    headingLevel: Number(opts.headingLevel) === 2 ? 2 : 3,
+  });
+  process.stdout.write(opts.json ? `${JSON.stringify(report, null, 2)}\n` : report.markdown);
+  return 0;
+}
+
+if (invokedDirectly(import.meta.url)) process.exit(main(process.argv.slice(2)));