canary-test-cli 7.2.0 → 8.0.0

package/agents/skills/claude-code/canary-screech/scripts/cli.mjs

@@ -0,0 +1,128 @@
+#!/usr/bin/env node
+// canary-screech -- broken-main siren (#591).
+//
+// The cross-run member of the failure-surfacing family. canary-fail-fast aborts
+// inside one run; canary-test-reporter summarises one run; neither can say the
+// default branch went red, because that fact only exists in the SEQUENCE of
+// runs. This skill reads that sequence out of the run-history store and emits a
+// one-page blast: culprit commit range, failure cluster, owning area, a
+// quarantine-or-revert recommendation, and a chat-ready block.
+//
+// It emits. It does not post: no Slack, no Teams, no webhook, no `gh`. Output
+// is a markdown artifact (--out) plus a `::error` GitHub Actions annotation,
+// the same channel canary-fail-fast uses.
+//
+// Advisory by default (#508 D3): exit 0 whatever it finds, unless --strict.
+// Under --strict: 1 = red, 3 = abstained (zero runs for the branch), 0 = green.
+//
+// Invoked via `canary skills run canary-screech -- --history <jsonl> [...]`.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import {
+  createParser,
+  formatUsageError,
+  EXIT_USAGE,
+} from '../../../lib/parse-args.mjs';
+import { loadRuns, runsForBranch } from './history.mjs';
+import { assessBranch } from './redness.mjs';
+import { clusterFailures } from './cluster.mjs';
+import { renderBlast } from './blast.mjs';
+
+const PREFIX = 'canary-screech:';
+
+/** Exit code reserved family-wide for "abstained" (#508 D4). */
+const EXIT_ABSTAINED = 3;
+
+const USAGE =
+  'usage: canary-screech [-h] --history PATH [--branch NAME] [--out PATH] [--strict]\n' +
+  '\n' +
+  'Broken-main siren: one-page blast when the default branch goes red.';
+
+export const CLI_SPEC = {
+  prog: 'canary-screech',
+  booleans: { '--strict': 'strict' },
+  values: {
+    '--history': { key: 'history' },
+    '--branch': { key: 'branch' },
+    '--out': { key: 'out' },
+  },
+  defaults: { branch: 'main' },
+  required: ['--history'],
+};
+
+const parseArgs = createParser(CLI_SPEC);
+
+/**
+ * Write the artifact, or return the message explaining why it could not be.
+ * Extracted so `main` stays under the complexity the perf gate allows.
+ *
+ * @returns {string|null} null on success
+ */
+function writeArtifact(out, markdown) {
+  try {
+    fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true });
+    fs.writeFileSync(out, markdown, 'utf8');
+    return null;
+  } catch (exc) {
+    return `cannot write artifact: ${exc.message}`;
+  }
+}
+
+/** The `--strict` exit contract. Advisory callers never reach this. */
+function strictExitFor(state) {
+  if (state === 'abstained') return EXIT_ABSTAINED;
+  return state === 'red' ? 1 : 0;
+}
+
+export function main(argv = []) {
+  const { opts: args, help, error } = parseArgs(argv);
+
+  if (help) {
+    console.log(USAGE);
+    return 0;
+  }
+  if (error) {
+    console.error(formatUsageError(CLI_SPEC.prog, error));
+    return EXIT_USAGE;
+  }
+
+  let runs;
+  try {
+    runs = runsForBranch(loadRuns(args.history), args.branch);
+  } catch (exc) {
+    // A bad --history path is an error, never "the branch looks fine".
+    console.error(`${PREFIX} ${exc.message}`);
+    return 1;
+  }
+
+  const assessment = assessBranch(runs);
+  const cluster =
+    assessment.state === 'red'
+      ? clusterFailures(assessment.firstRed, assessment.culpritRange)
+      : null;
+  const blast = renderBlast({ branch: args.branch, assessment, cluster });
+
+  console.log(blast.markdown);
+  for (const annotation of blast.annotations) console.log(annotation);
+
+  if (args.out) {
+    const failure = writeArtifact(args.out, blast.markdown);
+    if (failure) {
+      console.error(`${PREFIX} ${failure}`);
+      return 1;
+    }
+  }
+
+  return args.strict ? strictExitFor(assessment.state) : 0;
+}
+
+// Direct execution (the skill runner execs this file via its shebang).
+//
+// `process.exitCode`, not `process.exit()`: a large payload exceeds the pipe
+// buffer, and `process.exit` tears the process down mid-write, leaving
+// truncated output that still exits 0 (#791).
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/agents/skills/claude-code/canary-screech/scripts/cluster.mjs

@@ -0,0 +1,97 @@
+// cluster -- what broke together, who owns it, and what to do about it. Pure.
+//
+// The one-pager's value is not the list of failing tests (the run log already
+// has that). It is the SHAPE of the break: whether one thing broke in one
+// place, or many things broke everywhere. Those two shapes want opposite
+// responses, which is why the recommendation is derived from the shape rather
+// than from the failure count.
+
+const UNCATEGORIZED = 'uncategorized';
+
+/** A test row counts as a failure when it did not pass. */
+function isFailure(test) {
+  const status = String(test?.status ?? '').toLowerCase();
+  return (
+    status === 'failed' || status === 'unexpected' || status === 'timedout'
+  );
+}
+
+/**
+ * @typedef {object} Cluster
+ * @property {string} category
+ * @property {object[]} tests
+ */
+
+/** Failures grouped by category, largest group first. */
+function byCategory(failures) {
+  const groups = new Map();
+  for (const test of failures) {
+    const category = test.failure_category || UNCATEGORIZED;
+    if (!groups.has(category)) groups.set(category, []);
+    groups.get(category).push(test);
+  }
+  // Ties broken by name so the one-pager is stable across runs -- a report that
+  // reshuffles itself is a report nobody diffs.
+  return [...groups.entries()]
+    .map(([category, tests]) => ({ category, tests }))
+    .sort(
+      (a, b) =>
+        b.tests.length - a.tests.length || a.category.localeCompare(b.category),
+    );
+}
+
+/** Failure counts per owning area, busiest first. Areas are often absent. */
+function byArea(failures) {
+  const counts = new Map();
+  for (const test of failures) {
+    if (test.area) counts.set(test.area, (counts.get(test.area) ?? 0) + 1);
+  }
+  return [...counts.entries()]
+    .map(([area, count]) => ({ area, count }))
+    .sort((a, b) => b.count - a.count || a.area.localeCompare(b.area));
+}
+
+/**
+ * The response the break's shape argues for.
+ *
+ * @param {{area: string, count: number}[]} areas
+ * @param {{commits: string[], bounded: boolean}} culpritRange
+ */
+function recommendFor(areas, culpritRange) {
+  // No area data at all. A recommendation here would be derived from nothing,
+  // which is exactly the confident-on-absent-data shape this repo keeps getting
+  // burned by -- so it declines to make one.
+  if (!areas.length) return 'investigate';
+
+  // One place, one attributable commit: the cheap, complete fix is a revert.
+  const commits = culpritRange?.commits ?? [];
+  const attributable = culpritRange?.bounded === true && commits.length === 1;
+  if (areas.length === 1 && attributable) return 'revert';
+
+  // Several areas, or an un-attributable range: nothing is cleanly revertable,
+  // so isolate the failures and keep the branch moving.
+  return 'quarantine';
+}
+
+/**
+ * Cluster a red run's failures and derive the response.
+ *
+ * @param {object} run the first red run
+ * @param {{commits: string[], bounded: boolean}} culpritRange
+ * @returns {{clusters: Cluster[], areas: {area: string, count: number}[],
+ *           owningArea: string|null,
+ *           recommendation: 'revert'|'quarantine'|'investigate'}}
+ */
+export function clusterFailures(run, culpritRange) {
+  const failures = (run?.tests ?? []).filter(isFailure);
+  const areas = byArea(failures);
+  return {
+    clusters: byCategory(failures),
+    areas,
+    // The busiest area, or null when nothing recorded one. With several areas
+    // in play this is a pointer, not an owner -- which is why the
+    // recommendation refuses to treat it as one.
+    owningArea: areas[0]?.area ?? null,
+    recommendation: recommendFor(areas, culpritRange),
+  };
+}

package/agents/skills/claude-code/canary-screech/scripts/history.mjs

@@ -0,0 +1,73 @@
+// history -- read the run-history store canary-screech watches.
+//
+// The store is `test-results/reports/history-v2.jsonl`: one RunRecord JSON
+// object per line (the shape in `ts/src/history/record.ts`). This module is the
+// only part of the skill that touches the filesystem; everything downstream is
+// pure.
+//
+// Deliberately NOT tolerant of a missing file. Returning `[]` there would be
+// byte-identical to a genuinely empty store, and the caller would print an
+// abstention -- a plausible-looking, wrong answer -- when the truth is that
+// `--history` points at nothing. A typo'd path must look like a typo'd path.
+
+import fs from 'node:fs';
+
+/** The fields this skill reads. Everything else in the record is ignored. */
+
+/**
+ * Parse the JSONL store into records.
+ *
+ * @param {string} file path to history-v2.jsonl
+ * @returns {object[]} one record per non-blank line, in file order
+ */
+export function loadRuns(file) {
+  if (!fs.existsSync(file)) {
+    throw new Error(`history store not found: ${file}`);
+  }
+  const lines = fs.readFileSync(file, 'utf8').split(/\r\n|\r|\n/);
+  const runs = [];
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    let record;
+    try {
+      record = JSON.parse(line);
+    } catch (exc) {
+      throw new Error(
+        `malformed history record at line ${i + 1}: ${exc.message}`,
+      );
+    }
+    if (
+      record === null ||
+      typeof record !== 'object' ||
+      Array.isArray(record)
+    ) {
+      throw new Error(
+        `malformed history record at line ${i + 1}: not an object`,
+      );
+    }
+    runs.push(record);
+  }
+  return runs;
+}
+
+/**
+ * The runs for one branch, oldest first.
+ *
+ * Ordered by `timestamp`, not by file order: the store is append-only per
+ * writer, and several suites append to it concurrently, so file order is not
+ * chronological order. Getting this backwards would name the wrong run as the
+ * first red one, which is the whole culprit range.
+ *
+ * @param {object[]} runs
+ * @param {string} branch
+ * @returns {object[]}
+ */
+export function runsForBranch(runs, branch) {
+  return runs
+    .filter((r) => r.branch === branch)
+    .slice()
+    .sort((a, b) =>
+      String(a.timestamp ?? '').localeCompare(String(b.timestamp ?? '')),
+    );
+}

package/agents/skills/claude-code/canary-screech/scripts/redness.mjs

@@ -0,0 +1,94 @@
+// redness -- is this branch red, and which commits are implicated. Pure.
+//
+// "Red" is a property of the LATEST run on the branch, but the culprit range is
+// a property of the transition into red, so the walk goes backwards from the
+// latest run to the FIRST consecutive red one. Naming the latest red run as the
+// culprit is the obvious mistake: by the time a human looks, main has usually
+// taken several more commits while staying broken, and reverting the newest of
+// them fixes nothing.
+//
+// Split into four small functions rather than one readable-looking block: the
+// perf gate scores the block at cyclomatic 12 / 53 lines, and the shape it is
+// objecting to is real -- three distinct verdicts sharing one return statement.
+
+/** A run counts as failing when it recorded at least one failed test. */
+function isRed(run) {
+  return Number(run.failed ?? 0) > 0;
+}
+
+/**
+ * @typedef {object} CulpritRange
+ * @property {string|null} from  last known-green commit, or null if unknown
+ * @property {string|null} to    commit of the first red run
+ * @property {string[]} commits  distinct commits inside the range
+ * @property {boolean} bounded   false when no green run precedes the break
+ */
+
+/** Index of the first run in the unbroken red tail ending at the latest run. */
+function firstRedIndex(runs) {
+  let i = runs.length - 1;
+  while (i > 0 && isRed(runs[i - 1])) i -= 1;
+  return i;
+}
+
+/** The red verdict, with the range the break is attributable to. */
+function redVerdict(runs, index) {
+  const firstRed = runs[index];
+  const lastGreen = index > 0 ? runs[index - 1] : null;
+
+  // The commits this store can actually SEE inside the range. The interval is
+  // `(lastGreen, firstRed]`, and its interior may hold commits that no run ever
+  // observed -- enumerating those needs git, which this skill deliberately does
+  // not shell out to. So the list is the observed suspect only, and
+  // `bounded: false` is how "the lower bound is unknown" is said out loud
+  // rather than implied away. `clusterFailures` accepts a longer list so a
+  // git-aware caller can widen the range without changing this contract.
+  const commits = firstRed.commit_sha ? [firstRed.commit_sha] : [];
+
+  return {
+    state: 'red',
+    latest: runs[runs.length - 1],
+    firstRed,
+    lastGreen,
+    culpritRange: {
+      from: lastGreen?.commit_sha ?? null,
+      to: firstRed.commit_sha ?? null,
+      commits,
+      bounded: lastGreen !== null,
+    },
+  };
+}
+
+/**
+ * Assess a branch from its runs, oldest first.
+ *
+ * @param {object[]} runs oldest-first runs for ONE branch
+ * @returns {{state: 'red'|'green'|'abstained', latest: object|null,
+ *           firstRed: object|null, lastGreen: object|null,
+ *           culpritRange: CulpritRange|null}}
+ */
+export function assessBranch(runs) {
+  // Zero denominator. Not green -- nothing was observed, so nothing is known.
+  if (!runs.length) {
+    return {
+      state: 'abstained',
+      latest: null,
+      firstRed: null,
+      lastGreen: null,
+      culpritRange: null,
+    };
+  }
+
+  const latest = runs[runs.length - 1];
+  if (!isRed(latest)) {
+    return {
+      state: 'green',
+      latest,
+      firstRed: null,
+      lastGreen: latest,
+      culpritRange: null,
+    };
+  }
+
+  return redVerdict(runs, firstRedIndex(runs));
+}