canary-test-cli 7.1.0 → 8.0.0

package/agents/skills/claude-code/canary-katana/scripts/alarm.mjs

@@ -0,0 +1,296 @@
+// alarm -- fire only when a deletion removes the last coverage of a hot symbol.
+//
+// Most test deletions are legitimate, so katana is silent by default and
+// records everything. It alarms in exactly one situation: the deleted test was
+// the *last* test covering a symbol `critical-areas.json` marks high-risk. When
+// that file is missing or malformed the alarm degrades to recording-only and
+// says so; a gate that manufactures failures on missing data gets muted, and a
+// muted gate is worse than no gate.
+
+import fs from 'node:fs';
+
+import { isTestFile } from './diffscan.mjs';
+
+export const DEGRADED_NOTICE =
+  'critical-area data unavailable, recording only, not alarming';
+
+// risk_score at or above this makes a name-matched last-coverage loss CRITICAL;
+// below it the loss is still real but ranked HIGH.
+const CRITICAL_RISK = 0.7;
+
+// Directory names too generic to imply a coverage relationship on their own.
+const GENERIC_DIRS = new Set([
+  'src',
+  'lib',
+  'app',
+  'apps',
+  'packages',
+  'pkg',
+  'tests',
+  'test',
+  '__tests__',
+  'e2e',
+  'spec',
+  'dist',
+  'build',
+]);
+
+const CODE_SUFFIXES = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.py'];
+
+export const Fidelity = {
+  NAME_MATCHED: { value: 'name-matched', rank: 0 },
+  HEURISTIC: { value: 'heuristic', rank: 1 },
+};
+
+export const Severity = {
+  CRITICAL: { value: 'critical', sortKey: 0 },
+  HIGH: { value: 'high', sortKey: 1 },
+  MEDIUM: { value: 'medium', sortKey: 2 },
+};
+
+/**
+ * @typedef {{available: boolean, areas: Array<Record<string, any>>, reason: string}} CriticalAreas
+ */
+
+/** JSON-contract shape of a finding. */
+export function findingToDict(f) {
+  return {
+    kind: f.kind,
+    test: f.test,
+    file: f.file,
+    area: f.area,
+    fidelity: f.fidelity.value,
+    severity: f.severity.value,
+    evidence: f.evidence,
+  };
+}
+
+/**
+ * Load critical-areas.json; unavailable (not throwing) on any problem.
+ * @returns {CriticalAreas}
+ */
+export function loadCriticalAreas(filePath) {
+  if (filePath === null || filePath === undefined) {
+    return {
+      available: false,
+      areas: [],
+      reason: 'critical-area file not provided',
+    };
+  }
+  if (!fs.existsSync(filePath)) {
+    return {
+      available: false,
+      areas: [],
+      reason: `critical-area file not found: ${filePath}`,
+    };
+  }
+  let data;
+  try {
+    data = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+  } catch (exc) {
+    return {
+      available: false,
+      areas: [],
+      reason: `critical-area file malformed: ${exc.message}`,
+    };
+  }
+  const areas =
+    data &&
+    typeof data === 'object' &&
+    !Array.isArray(data) &&
+    Array.isArray(data.areas)
+      ? data.areas
+      : [];
+  return { available: true, areas: [...areas], reason: '' };
+}
+
+const norm = (text) => text.toLowerCase().replace(/[^a-z0-9]/g, '');
+
+/**
+ * Symbols an area path exposes: its basename minus code suffix, plus parts.
+ * `src/loyalty/points.service.ts` -> {points.service, points, service}.
+ */
+export function areaSymbols(areaPath) {
+  let base = areaPath.replace(/\\/g, '/').split('/').pop();
+  for (const suffix of CODE_SUFFIXES) {
+    if (base.endsWith(suffix)) {
+      base = base.slice(0, -suffix.length);
+      break;
+    }
+  }
+  const symbols = new Set([base]);
+  for (const part of base.split('.')) if (part) symbols.add(part);
+  return symbols;
+}
+
+const areaNormSymbols = (areaPath) => {
+  const out = new Set();
+  for (const s of areaSymbols(areaPath)) {
+    const n = norm(s);
+    if (n.length >= 4) out.add(n);
+  }
+  return out;
+};
+
+const dirsOf = (p) => {
+  const parts = p.replace(/\\/g, '/').split('/').filter(Boolean);
+  return parts.slice(0, -1);
+};
+
+const significantDirs = (p) =>
+  new Set(dirsOf(p).filter((d) => !GENERIC_DIRS.has(d)));
+
+const nameCovers = (testName, normSymbols) => {
+  const normalized = norm(testName);
+  return [...normSymbols].some((sym) => normalized.includes(sym));
+};
+
+// Heavy/ignored directories never worth walking for test files (#395).
+const SKIP_DIRS = new Set([
+  '.git',
+  'node_modules',
+  '__pycache__',
+  '.venv',
+  'venv',
+  'dist',
+  'build',
+  '.mypy_cache',
+  '.pytest_cache',
+  '.tox',
+  'coverage',
+  '.next',
+  '.turbo',
+]);
+
+/**
+ * Enumerate repo test files, pruning heavy dirs. Deterministic (sorted).
+ * @returns {[string, string][]} [relPosixPath, absolutePath] pairs.
+ */
+export function repoTestFiles(repo) {
+  const results = [];
+  const walk = (absDir, relDir) => {
+    let entries;
+    try {
+      entries = fs.readdirSync(absDir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    const dirs = [];
+    const files = [];
+    for (const e of entries) {
+      if (e.isDirectory()) {
+        if (!SKIP_DIRS.has(e.name)) dirs.push(e.name);
+      } else if (e.isFile()) {
+        files.push(e.name);
+      }
+    }
+    for (const name of files.sort()) {
+      const rel = relDir ? `${relDir}/${name}` : name;
+      if (isTestFile(rel)) results.push([rel, `${absDir}/${name}`]);
+    }
+    for (const name of dirs.sort()) {
+      walk(`${absDir}/${name}`, relDir ? `${relDir}/${name}` : name);
+    }
+  };
+  walk(String(repo), '');
+  return results;
+}
+
+const PY_TEST_DEF = /^\s*(?:async\s+)?def\s+(test\w*)\s*\(/gm;
+const JS_TEST_CALL =
+  /\b(?:describe|context|it|test)(?:\.\w+)?\s*\(\s*(['"`])(.*?)\1/g;
+
+function testNames(text) {
+  const names = [];
+  for (const m of text.matchAll(PY_TEST_DEF)) names.push(m[1]);
+  for (const m of text.matchAll(JS_TEST_CALL)) names.push(m[2]);
+  return names;
+}
+
+const readTextSafe = (p) => {
+  try {
+    return fs.readFileSync(p, 'utf8');
+  } catch {
+    return null;
+  }
+};
+
+function nameCoverageRemains(repo, normSymbols) {
+  for (const [, abs] of repoTestFiles(repo)) {
+    const text = readTextSafe(abs);
+    if (text === null) continue;
+    if (testNames(text).some((name) => nameCovers(name, normSymbols)))
+      return true;
+  }
+  return false;
+}
+
+function dirCoverageRemains(repo, areaDirs) {
+  for (const [rel, abs] of repoTestFiles(repo)) {
+    if (!dirsOf(rel).some((d) => areaDirs.has(d))) continue;
+    const text = readTextSafe(abs);
+    if (text === null) continue;
+    if (testNames(text).length) return true;
+  }
+  return false;
+}
+
+/** Return last-coverage-removed findings; empty when data is unavailable. */
+export function buildFindings(deletions, areas, repo) {
+  if (!areas.available) return []; // silent by default: never alarm on degraded data
+  const findings = [];
+
+  for (const deletion of deletions) {
+    let best = null;
+    for (const area of areas.areas) {
+      const areaPath = area.path || '';
+      const risk = Number.parseFloat(area.risk_score) || 0.0;
+      const normSymbols = areaNormSymbols(areaPath);
+
+      let fidelity;
+      let severity;
+      if (normSymbols.size && nameCovers(deletion.name, normSymbols)) {
+        if (nameCoverageRemains(repo, normSymbols)) continue;
+        fidelity = Fidelity.NAME_MATCHED;
+        severity = risk >= CRITICAL_RISK ? Severity.CRITICAL : Severity.HIGH;
+      } else {
+        const areaDirs = significantDirs(areaPath);
+        const delDirs = new Set(dirsOf(deletion.file));
+        if (![...areaDirs].some((d) => delDirs.has(d))) continue;
+        if (dirCoverageRemains(repo, areaDirs)) continue;
+        fidelity = Fidelity.HEURISTIC;
+        severity = Severity.MEDIUM;
+      }
+
+      const candidate = {
+        kind: 'last-coverage-removed',
+        test: deletion.name,
+        file: deletion.file,
+        area: areaPath,
+        fidelity,
+        severity,
+        evidence: `${deletion.name} was the last test covering ${areaPath}`,
+      };
+      // Keep the best candidate per deletion: lower (fidelity.rank, sortKey)
+      // wins, element-wise (name-matched outranks heuristic; then severity).
+      if (best === null) {
+        best = candidate;
+      } else {
+        const better =
+          fidelity.rank !== best.fidelity.rank
+            ? fidelity.rank < best.fidelity.rank
+            : severity.sortKey < best.severity.sortKey;
+        if (better) best = candidate;
+      }
+    }
+    if (best !== null) findings.push(best);
+  }
+
+  findings.sort(
+    (a, b) =>
+      a.severity.sortKey - b.severity.sortKey ||
+      a.file.localeCompare(b.file) ||
+      a.test.localeCompare(b.test),
+  );
+  return findings;
+}

package/agents/skills/claude-code/canary-katana/scripts/cli.mjs

@@ -0,0 +1,247 @@
+#!/usr/bin/env node
+// canary-katana -- quarantine deleted and newly-skipped tests, with provenance.
+//
+// Captures every removed or skipped test into an append-only ledger (who
+// deleted it, when, in which commit, and why), and alarms in exactly one case:
+// the removed test was the last coverage of a symbol `critical-areas.json`
+// marks high-risk.
+//
+// Advisory by default (always exit 0). `--strict` exits 1 only on a real alarm;
+// a degraded run (no critical-area data) stays exit 0 even under `--strict` --
+// a gate that fails on missing data gets muted, and a muted gate is worse than
+// none.
+//
+// Invoked via `canary skills run canary-katana -- [options]`.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import {
+  createParser,
+  formatUsageError,
+  EXIT_USAGE,
+} from '../../../lib/parse-args.mjs';
+import * as diffscan from './diffscan.mjs';
+import * as alarm from './alarm.mjs';
+import * as ledger from './ledger.mjs';
+
+// --- no-silent-abstention (#508 D2, skill-CLI convention half) ---------------
+//
+// Skill CLIs are deliberately self-contained -- no engine import -- so they
+// cannot call `gateOutcome`. They honour the doctrine by CONVENTION, emitting
+// the same greppable line the engine helper does; the skill-layer conformance
+// registry (agents/skills/test/gate-conformance.test.ts) holds them to it.
+//
+// U+26A0 / U+2014 as escapes so this source stays ASCII, matching
+// ts/src/core/gate-result.ts.
+const ABSTAINED_LINE =
+  '\u{26A0} Abstained \u{2014} verified zero items; this is not a pass.';
+
+const PREFIX = 'canary-katana:';
+
+const USAGE =
+  'usage: canary-katana [-h] [--repo PATH] [--diff-file PATH] [--ledger PATH]\n' +
+  '                     [--critical-areas PATH] [--json] [--strict] [--no-write]\n' +
+  '\n' +
+  'Quarantine deleted and newly-skipped tests into an append-only ledger with\n' +
+  'provenance, and alarm when a removal drops the last coverage of a critical\n' +
+  'area.\n' +
+  '\n' +
+  'options:\n' +
+  '  -h, --help             show this help message and exit\n' +
+  '  --repo PATH            repository to inspect (default: .)\n' +
+  '  --diff-file PATH       read the diff from a file instead of git\n' +
+  '  --ledger PATH          ledger location (default: <repo>/.canary/quarantine.json)\n' +
+  '  --critical-areas PATH  critical-areas.json used to raise alarms\n' +
+  '  --json                 emit machine-readable output instead of human text\n' +
+  '  --strict               exit 1 on a real alarm (degraded runs stay 0)\n' +
+  '  --no-write             do not append to the ledger (read-only run)';
+
+/**
+ * katana takes no positionals, so any leftover token -- dashed or not -- is a
+ * usage error rather than something silently ignored, and `--` has nothing to
+ * protect. The four shared invariants (null-prototype lookup, empty-value
+ * rejection, arity, `--flag=value`) live in the shared parser; see #479 for why
+ * they stopped living here.
+ */
+export const CLI_SPEC = {
+  prog: 'canary-katana',
+  booleans: {
+    '--json': 'json',
+    '--strict': 'strict',
+    '--no-write': 'noWrite',
+  },
+  values: {
+    '--repo': { key: 'repo' },
+    '--diff-file': { key: 'diffFile' },
+    '--ledger': { key: 'ledger' },
+    '--critical-areas': { key: 'criticalAreas' },
+  },
+  defaults: { repo: '.' },
+};
+
+const parseArgs = createParser(CLI_SPEC);
+
+/**
+ * Return { text, base }. `base` is a git ref when one is resolvable. With
+ * diffFile the diff is read verbatim and git is still consulted (best-effort)
+ * for provenance; without it the diff is computed from the repo's own history.
+ */
+function loadDiff(repo, diffFile) {
+  if (diffFile) {
+    if (!fs.existsSync(diffFile)) {
+      const err = new Error(`diff file not found: ${diffFile}`);
+      err.notFound = true;
+      throw err;
+    }
+    const text = fs.readFileSync(diffFile, 'utf8');
+    let base = null;
+    try {
+      base = diffscan.resolveBase(repo, null);
+    } catch {
+      base = null; // non-git repo: provenance stays unknown
+    }
+    return { text, base };
+  }
+  const base = diffscan.resolveBase(repo, null);
+  return { text: diffscan.diffText(repo, base), base };
+}
+
+function provenance(repo, base, file) {
+  if (base === null) return null;
+  try {
+    return diffscan.commitForFile(repo, base, file);
+  } catch {
+    return null; // missing history is unknown, not fatal
+  }
+}
+
+function toEntries(repo, base, deletions) {
+  return deletions.map((d) => {
+    const commit = provenance(repo, base, d.file);
+    return ledger.LedgerEntry({
+      test: d.name,
+      file: d.file,
+      kind: d.kind,
+      marker: d.marker,
+      commit: commit ? commit.sha : '',
+      author: commit ? commit.author : 'unknown',
+      date: commit ? commit.date : '',
+      reason: commit ? commit.subject : '',
+      // The `Ticket:` trailer is one way an issue link arrives; a quarantine
+      // producer writing a caused row is the other. Both land in `issue`.
+      issue: commit ? commit.ticket : '',
+    });
+  });
+}
+
+function renderText(deletions, findings, degraded, scanned) {
+  // #508: katana's denominator is the DIFF it read, not the deletions it found.
+  // Zero deletions in a 500-line diff is a real result; zero deletions in an
+  // EMPTY diff means nothing was examined at all. `0 deletion(s) captured` reads
+  // identically in both cases, which is precisely the shape the doctrine bans.
+  if (!scanned) {
+    return (
+      `${ABSTAINED_LINE} The diff was empty, so no deleted test could be ` +
+      'captured. Check --repo/--diff-file, or that the range actually ' +
+      'contains changes.'
+    );
+  }
+  const lines = [`${deletions.length} deletion(s) captured.`];
+  if (degraded) lines.push(alarm.DEGRADED_NOTICE);
+  for (const f of findings) {
+    lines.push(
+      `  [${f.severity.value}] ${f.file}::${f.test} removed the last coverage of ${f.area}`,
+    );
+  }
+  return lines.join('\n');
+}
+
+export function main(argv = []) {
+  const { opts: args, help, error } = parseArgs(argv);
+
+  // FIRST, before loadDiff and before any ledger write: a usage request or a
+  // typo must never mutate the working tree.
+  if (help) {
+    console.log(USAGE);
+    return 0;
+  }
+  if (error) {
+    console.error(formatUsageError(CLI_SPEC.prog, error));
+    return EXIT_USAGE;
+  }
+
+  const repo = args.repo;
+  // `!= null`, not a truthiness test: "--ledger was not given" and "--ledger
+  // was given an empty path" are different situations, and only the first one
+  // may fall back to the default. (An empty value is rejected at parse time,
+  // so this branch is now unreachable with '' -- the explicit null check keeps
+  // it that way if the parser ever loosens.)
+  const ledgerPath =
+    args.ledger != null
+      ? args.ledger
+      : path.join(repo, '.canary', 'quarantine.json');
+
+  let diff;
+  let base;
+  try {
+    ({ text: diff, base } = loadDiff(repo, args.diffFile));
+  } catch (exc) {
+    if (exc.notFound) {
+      console.error(`${PREFIX} ${exc.message}`);
+      return 1;
+    }
+    console.error(`${PREFIX} could not read diff: ${exc.message}`);
+    return 1;
+  }
+
+  // Non-blank diff text is the denominator probe: `loadDiff` succeeding does
+  // not mean it returned anything to scan.
+  const scanned = diff.trim().length > 0;
+  const deletions = diffscan.findDeletions(diff);
+  const entries = toEntries(repo, base, deletions);
+
+  if (!args.noWrite) {
+    try {
+      ledger.appendEntries(ledgerPath, entries);
+    } catch (exc) {
+      console.error(`${PREFIX} ${exc.message}`);
+      return 1;
+    }
+  }
+
+  const areas = alarm.loadCriticalAreas(args.criticalAreas);
+  const degraded = !areas.available;
+  const findings = alarm.buildFindings(deletions, areas, repo);
+
+  if (args.json) {
+    const payload = {
+      schema_version: ledger.SCHEMA_VERSION,
+      captured: deletions.map(diffscan.deletionToDict),
+      findings: findings.map(alarm.findingToDict),
+      ledger: String(ledgerPath),
+    };
+    if (degraded) payload.degraded_notice = alarm.DEGRADED_NOTICE;
+    // Additive (#508): a consumer can distinguish "no deletions" from "nothing
+    // examined" without parsing prose.
+    payload.checked = scanned ? 1 : 0;
+    payload.abstained = !scanned;
+    console.log(JSON.stringify(payload, null, 2));
+  } else {
+    console.log(renderText(deletions, findings, degraded, scanned));
+  }
+
+  // Advisory by default (D3); --strict inherits EXIT_ABSTAINED (3) on an empty
+  // diff, distinct from 1 ("captured a real deletion").
+  if (args.strict && !scanned) return 3;
+  return args.strict && findings.length ? 1 : 0;
+}
+
+// Direct execution (the skill runner execs this file via its shebang).
+//
+// `process.exitCode`, not `process.exit()`: a large `--json` payload exceeds
+// the pipe buffer, and `process.exit` tears the process down mid-write, leaving
+// truncated JSON that still exits 0 (#791).
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exitCode = main(process.argv.slice(2));
+}