canary-test-cli 7.0.0 → 7.2.0

package/agents/skills/claude-code/canary-katana/scripts/cli.mjs

@@ -0,0 +1,247 @@
+#!/usr/bin/env node
+// canary-katana -- quarantine deleted and newly-skipped tests, with provenance.
+//
+// Captures every removed or skipped test into an append-only ledger (who
+// deleted it, when, in which commit, and why), and alarms in exactly one case:
+// the removed test was the last coverage of a symbol `critical-areas.json`
+// marks high-risk.
+//
+// Advisory by default (always exit 0). `--strict` exits 1 only on a real alarm;
+// a degraded run (no critical-area data) stays exit 0 even under `--strict` --
+// a gate that fails on missing data gets muted, and a muted gate is worse than
+// none.
+//
+// Invoked via `canary skills run canary-katana -- [options]`.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import {
+  createParser,
+  formatUsageError,
+  EXIT_USAGE,
+} from '../../../lib/parse-args.mjs';
+import * as diffscan from './diffscan.mjs';
+import * as alarm from './alarm.mjs';
+import * as ledger from './ledger.mjs';
+
+// --- no-silent-abstention (#508 D2, skill-CLI convention half) ---------------
+//
+// Skill CLIs are deliberately self-contained -- no engine import -- so they
+// cannot call `gateOutcome`. They honour the doctrine by CONVENTION, emitting
+// the same greppable line the engine helper does; the skill-layer conformance
+// registry (agents/skills/test/gate-conformance.test.ts) holds them to it.
+//
+// U+26A0 / U+2014 as escapes so this source stays ASCII, matching
+// ts/src/core/gate-result.ts.
+const ABSTAINED_LINE =
+  '\u{26A0} Abstained \u{2014} verified zero items; this is not a pass.';
+
+const PREFIX = 'canary-katana:';
+
+const USAGE =
+  'usage: canary-katana [-h] [--repo PATH] [--diff-file PATH] [--ledger PATH]\n' +
+  '                     [--critical-areas PATH] [--json] [--strict] [--no-write]\n' +
+  '\n' +
+  'Quarantine deleted and newly-skipped tests into an append-only ledger with\n' +
+  'provenance, and alarm when a removal drops the last coverage of a critical\n' +
+  'area.\n' +
+  '\n' +
+  'options:\n' +
+  '  -h, --help             show this help message and exit\n' +
+  '  --repo PATH            repository to inspect (default: .)\n' +
+  '  --diff-file PATH       read the diff from a file instead of git\n' +
+  '  --ledger PATH          ledger location (default: <repo>/.canary/quarantine.json)\n' +
+  '  --critical-areas PATH  critical-areas.json used to raise alarms\n' +
+  '  --json                 emit machine-readable output instead of human text\n' +
+  '  --strict               exit 1 on a real alarm (degraded runs stay 0)\n' +
+  '  --no-write             do not append to the ledger (read-only run)';
+
+/**
+ * katana takes no positionals, so any leftover token -- dashed or not -- is a
+ * usage error rather than something silently ignored, and `--` has nothing to
+ * protect. The four shared invariants (null-prototype lookup, empty-value
+ * rejection, arity, `--flag=value`) live in the shared parser; see #479 for why
+ * they stopped living here.
+ */
+export const CLI_SPEC = {
+  prog: 'canary-katana',
+  booleans: {
+    '--json': 'json',
+    '--strict': 'strict',
+    '--no-write': 'noWrite',
+  },
+  values: {
+    '--repo': { key: 'repo' },
+    '--diff-file': { key: 'diffFile' },
+    '--ledger': { key: 'ledger' },
+    '--critical-areas': { key: 'criticalAreas' },
+  },
+  defaults: { repo: '.' },
+};
+
+const parseArgs = createParser(CLI_SPEC);
+
+/**
+ * Return { text, base }. `base` is a git ref when one is resolvable. With
+ * diffFile the diff is read verbatim and git is still consulted (best-effort)
+ * for provenance; without it the diff is computed from the repo's own history.
+ */
+function loadDiff(repo, diffFile) {
+  if (diffFile) {
+    if (!fs.existsSync(diffFile)) {
+      const err = new Error(`diff file not found: ${diffFile}`);
+      err.notFound = true;
+      throw err;
+    }
+    const text = fs.readFileSync(diffFile, 'utf8');
+    let base = null;
+    try {
+      base = diffscan.resolveBase(repo, null);
+    } catch {
+      base = null; // non-git repo: provenance stays unknown
+    }
+    return { text, base };
+  }
+  const base = diffscan.resolveBase(repo, null);
+  return { text: diffscan.diffText(repo, base), base };
+}
+
+function provenance(repo, base, file) {
+  if (base === null) return null;
+  try {
+    return diffscan.commitForFile(repo, base, file);
+  } catch {
+    return null; // missing history is unknown, not fatal
+  }
+}
+
+function toEntries(repo, base, deletions) {
+  return deletions.map((d) => {
+    const commit = provenance(repo, base, d.file);
+    return ledger.LedgerEntry({
+      test: d.name,
+      file: d.file,
+      kind: d.kind,
+      marker: d.marker,
+      commit: commit ? commit.sha : '',
+      author: commit ? commit.author : 'unknown',
+      date: commit ? commit.date : '',
+      reason: commit ? commit.subject : '',
+      // The `Ticket:` trailer is one way an issue link arrives; a quarantine
+      // producer writing a caused row is the other. Both land in `issue`.
+      issue: commit ? commit.ticket : '',
+    });
+  });
+}
+
+function renderText(deletions, findings, degraded, scanned) {
+  // #508: katana's denominator is the DIFF it read, not the deletions it found.
+  // Zero deletions in a 500-line diff is a real result; zero deletions in an
+  // EMPTY diff means nothing was examined at all. `0 deletion(s) captured` reads
+  // identically in both cases, which is precisely the shape the doctrine bans.
+  if (!scanned) {
+    return (
+      `${ABSTAINED_LINE} The diff was empty, so no deleted test could be ` +
+      'captured. Check --repo/--diff-file, or that the range actually ' +
+      'contains changes.'
+    );
+  }
+  const lines = [`${deletions.length} deletion(s) captured.`];
+  if (degraded) lines.push(alarm.DEGRADED_NOTICE);
+  for (const f of findings) {
+    lines.push(
+      `  [${f.severity.value}] ${f.file}::${f.test} removed the last coverage of ${f.area}`,
+    );
+  }
+  return lines.join('\n');
+}
+
+export function main(argv = []) {
+  const { opts: args, help, error } = parseArgs(argv);
+
+  // FIRST, before loadDiff and before any ledger write: a usage request or a
+  // typo must never mutate the working tree.
+  if (help) {
+    console.log(USAGE);
+    return 0;
+  }
+  if (error) {
+    console.error(formatUsageError(CLI_SPEC.prog, error));
+    return EXIT_USAGE;
+  }
+
+  const repo = args.repo;
+  // `!= null`, not a truthiness test: "--ledger was not given" and "--ledger
+  // was given an empty path" are different situations, and only the first one
+  // may fall back to the default. (An empty value is rejected at parse time,
+  // so this branch is now unreachable with '' -- the explicit null check keeps
+  // it that way if the parser ever loosens.)
+  const ledgerPath =
+    args.ledger != null
+      ? args.ledger
+      : path.join(repo, '.canary', 'quarantine.json');
+
+  let diff;
+  let base;
+  try {
+    ({ text: diff, base } = loadDiff(repo, args.diffFile));
+  } catch (exc) {
+    if (exc.notFound) {
+      console.error(`${PREFIX} ${exc.message}`);
+      return 1;
+    }
+    console.error(`${PREFIX} could not read diff: ${exc.message}`);
+    return 1;
+  }
+
+  // Non-blank diff text is the denominator probe: `loadDiff` succeeding does
+  // not mean it returned anything to scan.
+  const scanned = diff.trim().length > 0;
+  const deletions = diffscan.findDeletions(diff);
+  const entries = toEntries(repo, base, deletions);
+
+  if (!args.noWrite) {
+    try {
+      ledger.appendEntries(ledgerPath, entries);
+    } catch (exc) {
+      console.error(`${PREFIX} ${exc.message}`);
+      return 1;
+    }
+  }
+
+  const areas = alarm.loadCriticalAreas(args.criticalAreas);
+  const degraded = !areas.available;
+  const findings = alarm.buildFindings(deletions, areas, repo);
+
+  if (args.json) {
+    const payload = {
+      schema_version: ledger.SCHEMA_VERSION,
+      captured: deletions.map(diffscan.deletionToDict),
+      findings: findings.map(alarm.findingToDict),
+      ledger: String(ledgerPath),
+    };
+    if (degraded) payload.degraded_notice = alarm.DEGRADED_NOTICE;
+    // Additive (#508): a consumer can distinguish "no deletions" from "nothing
+    // examined" without parsing prose.
+    payload.checked = scanned ? 1 : 0;
+    payload.abstained = !scanned;
+    console.log(JSON.stringify(payload, null, 2));
+  } else {
+    console.log(renderText(deletions, findings, degraded, scanned));
+  }
+
+  // Advisory by default (D3); --strict inherits EXIT_ABSTAINED (3) on an empty
+  // diff, distinct from 1 ("captured a real deletion").
+  if (args.strict && !scanned) return 3;
+  return args.strict && findings.length ? 1 : 0;
+}
+
+// Direct execution (the skill runner execs this file via its shebang).
+//
+// `process.exitCode`, not `process.exit()`: a large `--json` payload exceeds
+// the pipe buffer, and `process.exit` tears the process down mid-write, leaving
+// truncated JSON that still exits 0 (#791).
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/agents/skills/claude-code/canary-katana/scripts/ledger.mjs

@@ -0,0 +1,183 @@
+// ledger -- the quarantine record for tests that are out of the suite.
+//
+// Every captured deletion is written with its provenance (who, when, what
+// commit, why) so a test that vanishes leaves a trail instead of a silent gap.
+// The ledger is de-duplicated: re-running the capture on the same change adds
+// nothing, and a batch of new entries is sorted for a stable on-disk order.
+//
+// SCHEMA v2 (#771) adds the WHY a row is out, alongside the provenance of how it
+// left: `cause`, `issue`, `expiry`. Two fields deserve their separation --
+// `reason` is DERIVED (the commit subject, "what change did this"), `cause` is
+// ASSERTED (a judgement someone made, "why is it out"). Collapsing them would
+// dress an auto-derived string up as a claim a person stands behind.
+//
+// v2 also makes the ledger no longer purely append-only, for one narrow case
+// documented at `appendEntries`: a row that states a cause SUPERSEDES a causeless
+// row for the same (test, file). See the comment there for why leaving both is
+// worse than replacing one.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+export const SCHEMA_VERSION = 2;
+
+// The fields, in the order they define a row's identity for de-duplication.
+const FIELDS = [
+  'test',
+  'file',
+  'kind',
+  'marker',
+  'commit',
+  'author',
+  'date',
+  'reason',
+  'cause',
+  'issue',
+  'expiry',
+];
+
+/**
+ * Why a test is out of the suite. The two middle values are the ones where the
+ * TEST IS CORRECT and someone else owns the fix, so a producer must require an
+ * `issue` for them -- an untracked "the product is broken" note decays into an
+ * unexplained skip within a release.
+ */
+export const CAUSES = ['flaky', 'product-defect', 'blocked-data', 'obsolete'];
+
+/** Causes for which a row without an `issue` is not a real record. */
+export const CAUSES_REQUIRING_ISSUE = ['product-defect', 'blocked-data'];
+
+/**
+ * @typedef {{test: string, file: string, kind: string, marker: string,
+ *            commit: string, author: string, date: string, reason: string,
+ *            cause: string, issue: string, expiry: string}} LedgerRow
+ */
+
+/** Normalize an entry-like object into a row with the canonical field order. */
+export function LedgerEntry(fields) {
+  const row = {};
+  for (const f of FIELDS) row[f] = fields[f] ?? '';
+  // v1 wrote the tracker link as `ticket` (#781). It is the same fact under a
+  // different name, so it migrates onto `issue` rather than being dropped or
+  // kept alongside -- two fields answering "what is this waiting on" is how a
+  // consumer ends up reading the empty one. `Ticket:` survives as the name of
+  // the COMMIT TRAILER that populates it, which is a mechanism, not a schema.
+  if (!row.issue && typeof fields.ticket === 'string')
+    row.issue = fields.ticket;
+  return row;
+}
+
+// The subset that defines a row's IDENTITY for de-duplication.
+//
+// `issue` and `expiry` are excluded, for the reason #781 excluded `ticket`:
+// identity is what happened -- which test, in which file, muted how, by which
+// commit and why -- and a tracker link is an attribute of that event rather
+// than part of it. Including them would also break the append-only guarantee
+// across this schema change, since rows written before the fields existed key
+// as `''` and the same capture re-run with an issue present would hash
+// differently and append a duplicate of a row already on disk.
+//
+// `cause` IS identity, and deliberately so: `appendEntries` decides supersede
+// on whether a row states one, so a caused and a causeless row for the same
+// test must remain distinguishable here for that decision to have anything to
+// act on.
+const IDENTITY_FIELDS = FIELDS.filter((f) => f !== 'issue' && f !== 'expiry');
+
+const key = (row) => IDENTITY_FIELDS.map((f) => row[f] ?? '').join('\u0000');
+
+/** The (test, file) pair a supersede decision is made on. */
+const pair = (row) => `${row.test ?? ''}\u0000${row.file ?? ''}`;
+
+/**
+ * Load the ledger document, or an empty one when the file is absent.
+ * Throws on unparseable JSON or a non-object top level -- a corrupt ledger is a
+ * hard error the caller must surface, not silently overwrite.
+ *
+ * Every row is normalized through `LedgerEntry`, so a v1 file read here comes
+ * back with the v2 fields present and empty. That is what makes writing
+ * `schema_version: 2` honest: the version claims "these rows have these fields",
+ * and after normalization they do. Stamping the version over un-migrated rows
+ * would make it a promise the file does not keep.
+ * @returns {{schema_version: number, entries: LedgerRow[]}}
+ */
+export function load(filePath) {
+  if (!fs.existsSync(filePath)) {
+    return { schema_version: SCHEMA_VERSION, entries: [] };
+  }
+  let data;
+  try {
+    data = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+  } catch (exc) {
+    throw new Error(`ledger is not valid JSON: ${filePath}: ${exc.message}`);
+  }
+  if (data === null || typeof data !== 'object' || Array.isArray(data)) {
+    throw new Error(`ledger top level must be an object: ${filePath}`);
+  }
+  if (!('schema_version' in data)) data.schema_version = SCHEMA_VERSION;
+  if (!('entries' in data)) data.entries = [];
+  if (!Array.isArray(data.entries)) {
+    throw new Error(`ledger entries must be an array: ${filePath}`);
+  }
+  data.entries = data.entries.map((e) => LedgerEntry(e));
+  return data;
+}
+
+/**
+ * Append `entries` to the ledger at `filePath` and persist it. New entries are
+ * sorted by (file, test) and de-duplicated against the batch and disk.
+ *
+ * ONE ROW PER (test, file) MAY STATE A CAUSE, and a caused row wins:
+ *
+ *   - a new row WITH a cause replaces a causeless row for the same pair
+ *   - a new row WITHOUT a cause is dropped when a caused row already exists
+ *
+ * Without this, katana recording `{kind: 'skipped', cause: ''}` and a quarantine
+ * writer recording `{kind: 'skipped', cause: 'product-defect', issue: '#123'}`
+ * differ in `key()` and BOTH persist. A consumer that fails on an unlinked
+ * quarantine (canary-ci-ready does) then fails on the causeless row while the
+ * linked row sits beside it -- the ledger contradicting itself about one test.
+ *
+ * History is not lost to this: rows that differ in cause-bearing state are the
+ * only ones that collapse. Two caused rows, or two causeless rows, keep the
+ * full-field identity and both remain.
+ */
+export function appendEntries(filePath, entries) {
+  const doc = load(filePath);
+  const existing = doc.entries;
+  const seen = new Set(existing.map(key));
+  const causedPairs = new Set(existing.filter((r) => r.cause).map(pair));
+
+  const newRows = entries
+    .map((e) => LedgerEntry(e))
+    .sort(
+      (a, b) => a.file.localeCompare(b.file) || a.test.localeCompare(b.test),
+    );
+
+  for (const row of newRows) {
+    const k = key(row);
+    if (seen.has(k)) continue;
+    const p = pair(row);
+
+    if (row.cause) {
+      // Supersede: drop any causeless row for this pair, then take its place.
+      for (let i = existing.length - 1; i >= 0; i--) {
+        if (!existing[i].cause && pair(existing[i]) === p) {
+          seen.delete(key(existing[i]));
+          existing.splice(i, 1);
+        }
+      }
+      causedPairs.add(p);
+    } else if (causedPairs.has(p)) {
+      // A causeless row must never sit next to a caused one for the same test.
+      continue;
+    }
+
+    seen.add(k);
+    existing.push(row);
+  }
+
+  doc.schema_version = SCHEMA_VERSION;
+  fs.mkdirSync(path.dirname(path.resolve(filePath)), { recursive: true });
+  fs.writeFileSync(filePath, `${JSON.stringify(doc, null, 2)}\n`, 'utf8');
+  return doc;
+}

package/agents/skills/claude-code/canary-pr-guardian/SKILL.md

@@ -0,0 +1,129 @@
+---
+name: canary-pr-guardian
+description: >
+  PR/pre-commit test-guardian orchestrator: runs the deterministic Tier-0
+  diff-coverage pass, audits affected tests via canary-test-reviewer (Tier 1),
+  and — at the desk with authorTests opt-in — authors missing tests via
+  canary-test-author (Tier 2), staging them and blocking the commit once for
+  human review. Use to guard a change's test quality before it lands.
+---
+
+# Canary: PR Guardian
+
+Guards a change's test quality before it lands. Composes the deterministic
+Tier-0 diff-coverage engine with two native agents — `canary-test-reviewer`
+(read-only audit) and `canary-test-author` (authoring) — under a strict
+write-safety model. This is the **Option A** driver: the engine never calls an
+LLM; **this skill** invokes the agents in-session and enforces
+stage-and-block-once.
+
+**On the tier numbers.** Here they are the values of
+`canary guardian pr-check --tier 0|1|2`, not a repo-wide capability scale — Tier
+1 means "the `--tier 1` pass," which is the agent audit. `Tier-0` is the one
+number with a repo-wide meaning (deterministic, no network, no agent), and
+`Tier-1`/`Tier-2` are guardian-local by
+[ADR 0015](../../../../docs/knowledge/decisions/0015-skill-capability-vocabulary.md).
+Do not carry them into other skills.
+
+## When to Use
+
+- Before opening or updating a PR, to check that new/changed code is tested.
+- As a pre-commit companion when `preCommit.authorTests: true` is set and you
+  want the guardian to author the missing tests for you (at the desk only).
+- NOT in CI for Tier-2 write-back — that is a NON-GOAL. CI runs Tier-0 only (the
+  `CANARY_GUARDIAN_AGENT` env is unset there).
+
+## Safety model (non-negotiable)
+
+- **NEVER commit or push.** This skill only authors and `git add`s. The human
+  reviews the staged tests and re-commits.
+- **Honor every `skipped` reason** from `author-plan` verbatim (opt-in-off /
+  tier / fork / collision / loop-guard). Never override a skip.
+- **Block once.** When `block.block == true`, print the block message and stop —
+  leave the staged tests for the human. The loop-guard sentinel you write in
+  Phase 3 is what stops the guardian re-authoring over its own output on the
+  next run. It is stamped with the current `HEAD` and expires by itself once the
+  human's review commit moves `HEAD` — never delete it yourself.
+- **Authoring is opt-in.** No `preCommit.authorTests: true` ⇒ no writes, ever.
+
+## Phases
+
+### Phase 0 — Deterministic scope
+
+Run the Tier-0 pass and read its findings:
+
+```bash
+canary guardian pr-check --format json
+```
+
+Findings are `untested-new-code` gaps. If there are none, report clean and stop.
+
+### Phase 1 — Quality audit (Tier ≥ 1, read-only)
+
+Export the availability signal so the probe reports the ceiling, then audit the
+affected tests with `canary-test-reviewer`:
+
+```bash
+export CANARY_GUARDIAN_AGENT=1   # or 2 when authoring is enabled
+```
+
+If this checkout is a **fork** (an untrusted/read-only context — detect it, e.g.
+`git config --get remote.origin.url` pointing at a fork, or a CI fork PR), arm
+the fork guard so the engine's safety layer never authors on it:
+
+```bash
+export CANARY_GUARDIAN_IS_FORK=1   # any value other than "0"/unset means fork
+```
+
+Leave `CANARY_GUARDIAN_IS_FORK` unset (or `0`) at your own desk on a trusted
+checkout. The guard fails CLOSED: any ambiguous value is treated as a fork and
+authoring is skipped.
+
+Use the `canary-test-reviewer` agent to review the affected tests. This is
+**read-only** — surface weak-test findings; write nothing.
+
+### Phase 2 — Authoring plan (Tier 2 + opt-in)
+
+Ask the engine's safety layer for the plan (intents + block decision):
+
+```bash
+canary guardian author-plan --json
+```
+
+The JSON is `{"intents": [...], "block": {...}}`. Each intent carries `status`
+(`planned` | `authored` | `skipped`), `target_path`, `requirement`, and a
+`skip_reason` when skipped. **Do not author anything the plan skipped** — the
+Engine guards (opt-in, fork, collision, loop-guard) are authoritative.
+
+### Phase 3 — Author, stage, block once
+
+For each intent with `status: "planned"`, use the `canary-test-author` agent
+with the intent's `requirement` as the task and `target_path` as the
+destination. **Never overwrite an existing file at `target_path`.** The engine's
+collision guard runs at plan time, so between planning and writing another
+PR/session may have created the file (a TOCTOU window). If the target already
+exists at write time, skip that intent and report it — do not clobber it. Then
+stage the authored files:
+
+```bash
+git add <target_path>
+```
+
+When `block.block == true`, record the guardian-authored paths in the loop-guard
+sentinel BEFORE blocking, by running the deterministic producer command once per
+authored path:
+
+```bash
+canary guardian mark-authored --path <target_path> [--path <target_path> ...]
+```
+
+This is a real CLI step (not something you `touch` yourself): it writes the
+sentinel inside the real git dir and records exactly the paths you authored,
+stamped with the current `HEAD`. `author-plan` reads it on the next run and
+returns a `loop-guard` skip **while `HEAD` is unchanged**, so the guardian never
+authors on top of its own output — and once the human commits the reviewed
+tests, `HEAD` moves and authoring re-enables itself.
+
+Then print `block.message` (the "N test(s) authored & staged — review and
+re-commit" notice) and **stop**. Do not commit. The human reviews the staged
+tests and re-commits.

package/agents/skills/claude-code/canary-pr-guardian/skill.yaml

@@ -0,0 +1,17 @@
+name: canary-pr-guardian
+version: '1.0.0'
+description:
+  PR/pre-commit test-guardian orchestrator — Tier-0 diff-coverage, Tier-1
+  quality audit via canary-test-reviewer, and at-desk Tier-2 authoring via
+  canary-test-author with stage-and-block-once review.
+stability: static
+triggers:
+  - manual
+platforms:
+  - claude-code
+type: rigid
+tools: []
+tier: 1
+depends_on:
+  - canary-test-reviewer
+  - canary-test-author