canary-test-cli 7.1.0 → 8.0.0

package/agents/skills/claude-code/canary-screech/scripts/blast.mjs

@@ -0,0 +1,125 @@
+// blast -- render the one-page broken-main blast. Pure.
+//
+// Three outputs from one assessment, because the same fact has three audiences:
+//   markdown     the standalone artifact a human opens
+//   annotations  a single `::error` line so the GitHub Checks UI shows it
+//   chatBlock    plain text the on-call pastes into whatever chat they use
+//
+// The chat block is emitted, never posted. The skill has no credentials, no
+// webhook, and no write access to anything but the artifact path it was given.
+
+const CROSS = '\u274c'; // cross mark
+const CHECK = '\u2705'; // white heavy check mark
+const DASH = '\u2014'; // em dash
+
+/** The first meaningful line of an error blob, clipped. */
+function firstLine(error, limit = 160) {
+  if (!error) return '(no error message)';
+  for (const raw of String(error).split(/\r\n|\r|\n/)) {
+    const line = raw.trim();
+    if (line) return line.slice(0, limit);
+  }
+  return '(no error message)';
+}
+
+const NEXT_STEP = {
+  revert: 'Revert the culprit commit. One area, one attributable commit.',
+  quarantine:
+    'Quarantine the failing tests and open a tracking issue. The break is not cleanly attributable to one commit.',
+  investigate:
+    'Investigate before acting. The failing tests carry no owning area, so neither a revert nor a quarantine can be aimed.',
+};
+
+/**
+ * Render the blast.
+ *
+ * @param {{branch: string, assessment: object, cluster: object|null}} input
+ * @returns {{markdown: string, annotations: string[], chatBlock: string}}
+ */
+export function renderBlast({ branch, assessment, cluster }) {
+  if (assessment.state === 'abstained') {
+    const text =
+      `${CROSS} canary-screech ABSTAINED ${DASH} the run-history store holds no run for \`${branch}\`.\n\n` +
+      'Zero observations is not a healthy branch. Point `--history` at a store ' +
+      'that this branch actually writes to, or wire the suite to record runs.';
+    return {
+      markdown: `# canary-screech ${DASH} ${branch}\n\n${text}\n`,
+      annotations: [],
+      chatBlock: text,
+    };
+  }
+
+  if (assessment.state === 'green') {
+    const text = `${CHECK} \`${branch}\` is green as of ${assessment.latest.commit_sha ?? '(unknown commit)'}.`;
+    return {
+      markdown: `# canary-screech ${DASH} ${branch}\n\n${text}\n`,
+      annotations: [],
+      chatBlock: text,
+    };
+  }
+
+  const { culpritRange, firstRed } = assessment;
+  const from =
+    culpritRange.from ?? '(unknown - no green run recorded before the break)';
+  const to = culpritRange.to ?? '(unknown)';
+  const failureCount = cluster.clusters.reduce((n, c) => n + c.tests.length, 0);
+
+  const chatLines = [
+    `${CROSS} ${branch} is RED ${DASH} ${failureCount} failing test${failureCount === 1 ? '' : 's'}`,
+    `culprit range: ${from}..${to}`,
+    `owning area:   ${cluster.owningArea ?? '(none recorded)'}`,
+    `recommendation: ${cluster.recommendation.toUpperCase()} ${DASH} ${NEXT_STEP[cluster.recommendation]}`,
+    `first red run: ${firstRed.run_id} (${firstRed.suite}) at ${firstRed.timestamp}`,
+  ];
+  const chatBlock = chatLines.join('\n');
+
+  const lines = [
+    `# ${CROSS} canary-screech ${DASH} \`${branch}\` is red`,
+    '',
+    `**First red run:** \`${firstRed.run_id}\` (${firstRed.suite}) at ${firstRed.timestamp}`,
+    '',
+    '## Culprit commit range',
+    '',
+    `\`${from}\` .. \`${to}\``,
+    '',
+  ];
+  if (!culpritRange.bounded) {
+    lines.push(
+      '> The lower bound is unknown: the store holds no green run for this ' +
+        'branch before the break, so the range below is open-ended.',
+      '',
+    );
+  }
+  lines.push('## Failure cluster', '');
+  for (const c of cluster.clusters) {
+    lines.push(`### ${c.category} (${c.tests.length})`, '');
+    for (const t of c.tests) {
+      lines.push(`- \`${t.test_name}\` ${DASH} ${firstLine(t.error_text)}`);
+    }
+    lines.push('');
+  }
+  lines.push(
+    '## Owning area',
+    '',
+    cluster.areas.length
+      ? cluster.areas.map((a) => `- \`${a.area}\` (${a.count})`).join('\n')
+      : '_No failing test carries an `area`, so ownership could not be derived._',
+    '',
+    '## Recommendation',
+    '',
+    `**${cluster.recommendation.toUpperCase()}** ${DASH} ${NEXT_STEP[cluster.recommendation]}`,
+    '',
+    '## Chat-ready block',
+    '',
+    '```text',
+    chatBlock,
+    '```',
+    '',
+  );
+
+  const annotations = [
+    `::error title=Broken branch::${branch} is red ${DASH} ${failureCount} failing test${failureCount === 1 ? '' : 's'} in ${cluster.owningArea ?? 'an unrecorded area'}; culprit ${from}..${to}; recommendation: ${cluster.recommendation}`,
+  ];
+
+  return { markdown: lines.join('\n'), annotations, chatBlock };
+}

package/agents/skills/claude-code/canary-screech/scripts/cli.mjs

@@ -0,0 +1,128 @@
+#!/usr/bin/env node
+// canary-screech -- broken-main siren (#591).
+//
+// The cross-run member of the failure-surfacing family. canary-fail-fast aborts
+// inside one run; canary-test-reporter summarises one run; neither can say the
+// default branch went red, because that fact only exists in the SEQUENCE of
+// runs. This skill reads that sequence out of the run-history store and emits a
+// one-page blast: culprit commit range, failure cluster, owning area, a
+// quarantine-or-revert recommendation, and a chat-ready block.
+//
+// It emits. It does not post: no Slack, no Teams, no webhook, no `gh`. Output
+// is a markdown artifact (--out) plus a `::error` GitHub Actions annotation,
+// the same channel canary-fail-fast uses.
+//
+// Advisory by default (#508 D3): exit 0 whatever it finds, unless --strict.
+// Under --strict: 1 = red, 3 = abstained (zero runs for the branch), 0 = green.
+//
+// Invoked via `canary skills run canary-screech -- --history <jsonl> [...]`.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import {
+  createParser,
+  formatUsageError,
+  EXIT_USAGE,
+} from '../../../lib/parse-args.mjs';
+import { loadRuns, runsForBranch } from './history.mjs';
+import { assessBranch } from './redness.mjs';
+import { clusterFailures } from './cluster.mjs';
+import { renderBlast } from './blast.mjs';
+
+const PREFIX = 'canary-screech:';
+
+/** Exit code reserved family-wide for "abstained" (#508 D4). */
+const EXIT_ABSTAINED = 3;
+
+const USAGE =
+  'usage: canary-screech [-h] --history PATH [--branch NAME] [--out PATH] [--strict]\n' +
+  '\n' +
+  'Broken-main siren: one-page blast when the default branch goes red.';
+
+export const CLI_SPEC = {
+  prog: 'canary-screech',
+  booleans: { '--strict': 'strict' },
+  values: {
+    '--history': { key: 'history' },
+    '--branch': { key: 'branch' },
+    '--out': { key: 'out' },
+  },
+  defaults: { branch: 'main' },
+  required: ['--history'],
+};
+
+const parseArgs = createParser(CLI_SPEC);
+
+/**
+ * Write the artifact, or return the message explaining why it could not be.
+ * Extracted so `main` stays under the complexity the perf gate allows.
+ *
+ * @returns {string|null} null on success
+ */
+function writeArtifact(out, markdown) {
+  try {
+    fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true });
+    fs.writeFileSync(out, markdown, 'utf8');
+    return null;
+  } catch (exc) {
+    return `cannot write artifact: ${exc.message}`;
+  }
+}
+
+/** The `--strict` exit contract. Advisory callers never reach this. */
+function strictExitFor(state) {
+  if (state === 'abstained') return EXIT_ABSTAINED;
+  return state === 'red' ? 1 : 0;
+}
+
+export function main(argv = []) {
+  const { opts: args, help, error } = parseArgs(argv);
+
+  if (help) {
+    console.log(USAGE);
+    return 0;
+  }
+  if (error) {
+    console.error(formatUsageError(CLI_SPEC.prog, error));
+    return EXIT_USAGE;
+  }
+
+  let runs;
+  try {
+    runs = runsForBranch(loadRuns(args.history), args.branch);
+  } catch (exc) {
+    // A bad --history path is an error, never "the branch looks fine".
+    console.error(`${PREFIX} ${exc.message}`);
+    return 1;
+  }
+
+  const assessment = assessBranch(runs);
+  const cluster =
+    assessment.state === 'red'
+      ? clusterFailures(assessment.firstRed, assessment.culpritRange)
+      : null;
+  const blast = renderBlast({ branch: args.branch, assessment, cluster });
+
+  console.log(blast.markdown);
+  for (const annotation of blast.annotations) console.log(annotation);
+
+  if (args.out) {
+    const failure = writeArtifact(args.out, blast.markdown);
+    if (failure) {
+      console.error(`${PREFIX} ${failure}`);
+      return 1;
+    }
+  }
+
+  return args.strict ? strictExitFor(assessment.state) : 0;
+}
+
+// Direct execution (the skill runner execs this file via its shebang).
+//
+// `process.exitCode`, not `process.exit()`: a large payload exceeds the pipe
+// buffer, and `process.exit` tears the process down mid-write, leaving
+// truncated output that still exits 0 (#791).
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/agents/skills/claude-code/canary-screech/scripts/cluster.mjs

@@ -0,0 +1,97 @@
+// cluster -- what broke together, who owns it, and what to do about it. Pure.
+//
+// The one-pager's value is not the list of failing tests (the run log already
+// has that). It is the SHAPE of the break: whether one thing broke in one
+// place, or many things broke everywhere. Those two shapes want opposite
+// responses, which is why the recommendation is derived from the shape rather
+// than from the failure count.
+
+const UNCATEGORIZED = 'uncategorized';
+
+/** A test row counts as a failure when it did not pass. */
+function isFailure(test) {
+  const status = String(test?.status ?? '').toLowerCase();
+  return (
+    status === 'failed' || status === 'unexpected' || status === 'timedout'
+  );
+}
+
+/**
+ * @typedef {object} Cluster
+ * @property {string} category
+ * @property {object[]} tests
+ */
+
+/** Failures grouped by category, largest group first. */
+function byCategory(failures) {
+  const groups = new Map();
+  for (const test of failures) {
+    const category = test.failure_category || UNCATEGORIZED;
+    if (!groups.has(category)) groups.set(category, []);
+    groups.get(category).push(test);
+  }
+  // Ties broken by name so the one-pager is stable across runs -- a report that
+  // reshuffles itself is a report nobody diffs.
+  return [...groups.entries()]
+    .map(([category, tests]) => ({ category, tests }))
+    .sort(
+      (a, b) =>
+        b.tests.length - a.tests.length || a.category.localeCompare(b.category),
+    );
+}
+
+/** Failure counts per owning area, busiest first. Areas are often absent. */
+function byArea(failures) {
+  const counts = new Map();
+  for (const test of failures) {
+    if (test.area) counts.set(test.area, (counts.get(test.area) ?? 0) + 1);
+  }
+  return [...counts.entries()]
+    .map(([area, count]) => ({ area, count }))
+    .sort((a, b) => b.count - a.count || a.area.localeCompare(b.area));
+}
+
+/**
+ * The response the break's shape argues for.
+ *
+ * @param {{area: string, count: number}[]} areas
+ * @param {{commits: string[], bounded: boolean}} culpritRange
+ */
+function recommendFor(areas, culpritRange) {
+  // No area data at all. A recommendation here would be derived from nothing,
+  // which is exactly the confident-on-absent-data shape this repo keeps getting
+  // burned by -- so it declines to make one.
+  if (!areas.length) return 'investigate';
+
+  // One place, one attributable commit: the cheap, complete fix is a revert.
+  const commits = culpritRange?.commits ?? [];
+  const attributable = culpritRange?.bounded === true && commits.length === 1;
+  if (areas.length === 1 && attributable) return 'revert';
+
+  // Several areas, or an un-attributable range: nothing is cleanly revertable,
+  // so isolate the failures and keep the branch moving.
+  return 'quarantine';
+}
+
+/**
+ * Cluster a red run's failures and derive the response.
+ *
+ * @param {object} run the first red run
+ * @param {{commits: string[], bounded: boolean}} culpritRange
+ * @returns {{clusters: Cluster[], areas: {area: string, count: number}[],
+ *           owningArea: string|null,
+ *           recommendation: 'revert'|'quarantine'|'investigate'}}
+ */
+export function clusterFailures(run, culpritRange) {
+  const failures = (run?.tests ?? []).filter(isFailure);
+  const areas = byArea(failures);
+  return {
+    clusters: byCategory(failures),
+    areas,
+    // The busiest area, or null when nothing recorded one. With several areas
+    // in play this is a pointer, not an owner -- which is why the
+    // recommendation refuses to treat it as one.
+    owningArea: areas[0]?.area ?? null,
+    recommendation: recommendFor(areas, culpritRange),
+  };
+}

package/agents/skills/claude-code/canary-screech/scripts/history.mjs

@@ -0,0 +1,73 @@
+// history -- read the run-history store canary-screech watches.
+//
+// The store is `test-results/reports/history-v2.jsonl`: one RunRecord JSON
+// object per line (the shape in `ts/src/history/record.ts`). This module is the
+// only part of the skill that touches the filesystem; everything downstream is
+// pure.
+//
+// Deliberately NOT tolerant of a missing file. Returning `[]` there would be
+// byte-identical to a genuinely empty store, and the caller would print an
+// abstention -- a plausible-looking, wrong answer -- when the truth is that
+// `--history` points at nothing. A typo'd path must look like a typo'd path.
+
+import fs from 'node:fs';
+
+/** The fields this skill reads. Everything else in the record is ignored. */
+
+/**
+ * Parse the JSONL store into records.
+ *
+ * @param {string} file path to history-v2.jsonl
+ * @returns {object[]} one record per non-blank line, in file order
+ */
+export function loadRuns(file) {
+  if (!fs.existsSync(file)) {
+    throw new Error(`history store not found: ${file}`);
+  }
+  const lines = fs.readFileSync(file, 'utf8').split(/\r\n|\r|\n/);
+  const runs = [];
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    let record;
+    try {
+      record = JSON.parse(line);
+    } catch (exc) {
+      throw new Error(
+        `malformed history record at line ${i + 1}: ${exc.message}`,
+      );
+    }
+    if (
+      record === null ||
+      typeof record !== 'object' ||
+      Array.isArray(record)
+    ) {
+      throw new Error(
+        `malformed history record at line ${i + 1}: not an object`,
+      );
+    }
+    runs.push(record);
+  }
+  return runs;
+}
+
+/**
+ * The runs for one branch, oldest first.
+ *
+ * Ordered by `timestamp`, not by file order: the store is append-only per
+ * writer, and several suites append to it concurrently, so file order is not
+ * chronological order. Getting this backwards would name the wrong run as the
+ * first red one, which is the whole culprit range.
+ *
+ * @param {object[]} runs
+ * @param {string} branch
+ * @returns {object[]}
+ */
+export function runsForBranch(runs, branch) {
+  return runs
+    .filter((r) => r.branch === branch)
+    .slice()
+    .sort((a, b) =>
+      String(a.timestamp ?? '').localeCompare(String(b.timestamp ?? '')),
+    );
+}

package/agents/skills/claude-code/canary-screech/scripts/redness.mjs

@@ -0,0 +1,94 @@
+// redness -- is this branch red, and which commits are implicated. Pure.
+//
+// "Red" is a property of the LATEST run on the branch, but the culprit range is
+// a property of the transition into red, so the walk goes backwards from the
+// latest run to the FIRST consecutive red one. Naming the latest red run as the
+// culprit is the obvious mistake: by the time a human looks, main has usually
+// taken several more commits while staying broken, and reverting the newest of
+// them fixes nothing.
+//
+// Split into four small functions rather than one readable-looking block: the
+// perf gate scores the block at cyclomatic 12 / 53 lines, and the shape it is
+// objecting to is real -- three distinct verdicts sharing one return statement.
+
+/** A run counts as failing when it recorded at least one failed test. */
+function isRed(run) {
+  return Number(run.failed ?? 0) > 0;
+}
+
+/**
+ * @typedef {object} CulpritRange
+ * @property {string|null} from  last known-green commit, or null if unknown
+ * @property {string|null} to    commit of the first red run
+ * @property {string[]} commits  distinct commits inside the range
+ * @property {boolean} bounded   false when no green run precedes the break
+ */
+
+/** Index of the first run in the unbroken red tail ending at the latest run. */
+function firstRedIndex(runs) {
+  let i = runs.length - 1;
+  while (i > 0 && isRed(runs[i - 1])) i -= 1;
+  return i;
+}
+
+/** The red verdict, with the range the break is attributable to. */
+function redVerdict(runs, index) {
+  const firstRed = runs[index];
+  const lastGreen = index > 0 ? runs[index - 1] : null;
+
+  // The commits this store can actually SEE inside the range. The interval is
+  // `(lastGreen, firstRed]`, and its interior may hold commits that no run ever
+  // observed -- enumerating those needs git, which this skill deliberately does
+  // not shell out to. So the list is the observed suspect only, and
+  // `bounded: false` is how "the lower bound is unknown" is said out loud
+  // rather than implied away. `clusterFailures` accepts a longer list so a
+  // git-aware caller can widen the range without changing this contract.
+  const commits = firstRed.commit_sha ? [firstRed.commit_sha] : [];
+
+  return {
+    state: 'red',
+    latest: runs[runs.length - 1],
+    firstRed,
+    lastGreen,
+    culpritRange: {
+      from: lastGreen?.commit_sha ?? null,
+      to: firstRed.commit_sha ?? null,
+      commits,
+      bounded: lastGreen !== null,
+    },
+  };
+}
+
+/**
+ * Assess a branch from its runs, oldest first.
+ *
+ * @param {object[]} runs oldest-first runs for ONE branch
+ * @returns {{state: 'red'|'green'|'abstained', latest: object|null,
+ *           firstRed: object|null, lastGreen: object|null,
+ *           culpritRange: CulpritRange|null}}
+ */
+export function assessBranch(runs) {
+  // Zero denominator. Not green -- nothing was observed, so nothing is known.
+  if (!runs.length) {
+    return {
+      state: 'abstained',
+      latest: null,
+      firstRed: null,
+      lastGreen: null,
+      culpritRange: null,
+    };
+  }
+
+  const latest = runs[runs.length - 1];
+  if (!isRed(latest)) {
+    return {
+      state: 'green',
+      latest,
+      firstRed: null,
+      lastGreen: latest,
+      culpritRange: null,
+    };
+  }
+
+  return redVerdict(runs, firstRedIndex(runs));
+}