canary-test-cli 7.1.0 → 7.2.0

package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs

@@ -0,0 +1,268 @@
+#!/usr/bin/env node
+// canary-cassandra -- vacuous-test detection (#755).
+//
+// Finds tests that PASS WITHOUT PROVING ANYTHING: an assertion that compares a
+// value with itself (VAC-001), a test that never invokes the target it claims
+// to cover (VAC-002), and a test whose every assertion is an absence observed
+// on a bystander (VAC-003).
+//
+//   <paths>    files or directories to scan (default: the current directory).
+//   --json     emit machine-readable findings instead of human text.
+//   --strict   exit 1 when there are findings (default is advisory: exit 0).
+//
+// Deterministic: no LLM, no network, no test execution. The rules themselves
+// live in the engine (`core/vacuity-scanner`), which is also what `canary
+// vacuity-check` and the promotion gate run -- see engine.mjs for why this one
+// skill delegates where its three siblings self-host.
+//
+// Invoked via `canary skills run canary-cassandra -- [paths] [--json] [--strict]`.
+
+import fs from 'node:fs';
+
+import {
+  createParser,
+  formatUsageError,
+  EXIT_USAGE,
+} from '../../../lib/parse-args.mjs';
+import { loadEngine } from './engine.mjs';
+
+export const SCHEMA_VERSION = 1;
+
+const PREFIX = 'canary-cassandra:';
+
+/** Reserved CLI-wide: exit 3 means "abstained -- verified zero items". */
+const EXIT_ABSTAINED = 3;
+
+// U+2192 written as an escape so this source stays ASCII, matching the family.
+const ARROW = '\u{2192}';
+
+const USAGE =
+  'usage: canary-cassandra [-h] [--json] [--strict] [--] [path ...]\n' +
+  '\n' +
+  'Vacuous-test detection: finds tests that pass without proving anything.\n' +
+  '\n' +
+  'positional arguments:\n' +
+  '  path        files or directories to scan (default: the current directory)\n' +
+  '\n' +
+  'options:\n' +
+  '  -h, --help  show this help message and exit\n' +
+  '  --json      emit machine-readable findings instead of human text\n' +
+  '  --strict    exit 1 when there are findings (default is advisory: exit 0)\n' +
+  '\n' +
+  'rules:\n' +
+  '  VAC-001 (critical)  assertion compares a value with itself\n' +
+  '  VAC-002 (warning)   the test never invokes the target it covers\n' +
+  '  VAC-003 (warning)   every assertion is an absence, on a bystander\n' +
+  '\n' +
+  'The denominator is TESTS read, not files. A zero denominator exits 3 under\n' +
+  '--strict; it is never reported as a clean scan.';
+
+/**
+ * The `--` terminator and a lone `-` come with declaring positionals, so a file
+ * literally named `--json` stays reachable. Shared with the other skill CLIs
+ * via `lib/parse-args.mjs`; `test/skill-cli-conformance.test.ts` asserts this
+ * export exists so a hand-rolled parser cannot land.
+ */
+export const CLI_SPEC = {
+  prog: 'canary-cassandra',
+  booleans: { '--json': 'json', '--strict': 'strict' },
+  positionals: { key: 'paths', defaults: ['.'] },
+};
+
+const parseArgs = createParser(CLI_SPEC);
+
+// Resolved once, at load, so `main` stays synchronous like every sibling's --
+// the conformance suite calls `main(argv)` and reads a number back. A failed
+// resolution is carried, not thrown: `--help` must answer even with no engine.
+const ENGINE = await loadEngine();
+
+/** Every test file the given paths contribute, de-duplicated and ordered. */
+function collectFiles(paths, engine) {
+  const seen = new Set();
+  const files = [];
+  for (const entry of paths) {
+    const found = engine.isDir(entry)
+      ? engine.collectTestFiles(entry)
+      : [entry];
+    for (const file of found) {
+      if (seen.has(file)) continue;
+      seen.add(file);
+      files.push(file);
+    }
+  }
+  return files;
+}
+
+/** Run the engine scanner over each file, accumulating one gate result. */
+function scanFiles(files, engine) {
+  const findings = [];
+  const skipped = [];
+  let checked = 0;
+  for (const file of files) {
+    const r = engine.scanVacuity(file);
+    checked += r.checked;
+    findings.push(...r.findings);
+    if (r.skipped) skipped.push(...r.skipped);
+  }
+  return { checked, findings, skipped };
+}
+
+/** The sibling finding envelope, plus the two fields only cassandra has. */
+function toJson(f) {
+  return {
+    file: f.file,
+    line: f.line,
+    rule_id: f.rule,
+    severity: f.severity,
+    // `snippet` in the sibling envelope is "the locus, in one line". For a
+    // vacuity finding that is the test, not the source line: the defect is the
+    // test as a whole, and a single line of it reads as a lint hit.
+    snippet: f.test,
+    why: f.message,
+    suggestion: f.suggestion,
+    fidelity: f.fidelity ?? null,
+  };
+}
+
+function summary(result, filesScanned, outcome) {
+  const bySeverity = {};
+  for (const f of result.findings) {
+    bySeverity[f.severity] = (bySeverity[f.severity] || 0) + 1;
+  }
+  return {
+    files_scanned: filesScanned,
+    // The denominator that matters. A healthy `files_scanned` over zero tests
+    // is the subtler zero, and the one a file-counting scanner prints a clean
+    // tick on.
+    tests_checked: result.checked,
+    abstained: outcome.abstained,
+    findings: result.findings.length,
+    by_severity: bySeverity,
+    skipped: result.skipped.length,
+  };
+}
+
+function renderText(result, outcome) {
+  const lines = [];
+  for (const f of result.findings) {
+    const tier = f.fidelity ? ` [${f.fidelity}]` : '';
+    lines.push(
+      `[${f.severity.toUpperCase()}] ${f.file}:${f.line} (${f.rule})${tier}`,
+    );
+    lines.push(`  ${f.test}: ${f.message}`);
+    lines.push(`  ${ARROW} ${f.suggestion}`);
+    lines.push('');
+  }
+  lines.push(outcome.summaryLine);
+  if (outcome.abstained) {
+    lines.push(
+      'No test was read, so nothing here is proven. Point at a directory ' +
+        'holding test files, or pass one directly.',
+    );
+  } else if (!result.findings.length) {
+    lines.push(
+      'Advisory by default. Re-run with --strict to fail on findings.',
+    );
+  }
+  return lines.join('\n');
+}
+
+/** Exit 1 with a named reason. "Cannot verify" is a finding, not a skip. */
+function fail(message) {
+  console.error(`${PREFIX} ${message}`);
+  return 1;
+}
+
+/** Emit the result in the caller's mode. */
+function report(result, files, outcome, json) {
+  if (!json) {
+    console.log(renderText(result, outcome));
+    return;
+  }
+  console.log(
+    JSON.stringify(
+      {
+        schema_version: SCHEMA_VERSION,
+        findings: result.findings.map(toJson),
+        skipped: result.skipped,
+        summary: summary(result, files.length, outcome),
+      },
+      null,
+      2,
+    ),
+  );
+}
+
+/**
+ * Scan the given paths.
+ *
+ * Returns the scan plus its gate outcome, or a numeric exit code when nothing
+ * could be scanned at all -- an unresolvable engine and a missing path are
+ * FAILURES, never a clean result with an empty finding list.
+ */
+function scan(paths) {
+  if (!ENGINE.ok) return fail(ENGINE.error);
+  for (const entry of paths) {
+    if (!fs.existsSync(entry)) return fail(`path not found: ${entry}`);
+  }
+  const files = collectFiles(paths, ENGINE);
+  const result = scanFiles(files, ENGINE);
+  if (files.length === 0) {
+    result.skipped.push({
+      name: paths.join(', '),
+      reason: `no test file matched (looked for ${ENGINE.SCANNABLE_DESC})`,
+    });
+  }
+  // The same helper the engine's own surfaces use, rather than a hand-copied
+  // abstention line: this CLI already imports the engine, so the doctrine is
+  // enforced by the code instead of by convention.
+  const outcome = ENGINE.gateOutcome(
+    {
+      checked: result.checked,
+      findings: result.findings,
+      skipped: result.skipped,
+    },
+    'advisory',
+    { noun: 'test(s)' },
+  );
+  return { files, result, outcome };
+}
+
+export function main(argv = []) {
+  const { positionals: paths, opts, help, error } = parseArgs(argv);
+
+  // Usage resolves before any filesystem or engine work, so `--help` answers
+  // even in an install where the engine never resolved.
+  if (help) {
+    console.log(USAGE);
+    return 0;
+  }
+  if (error) {
+    console.error(formatUsageError(CLI_SPEC.prog, error));
+    return EXIT_USAGE;
+  }
+
+  const scanned = scan(paths);
+  if (typeof scanned === 'number') return scanned;
+  const { files, result, outcome } = scanned;
+  report(result, files, outcome, opts.json);
+
+  // Advisory by default (D3): findings are loud, the exit is not. Under
+  // --strict the exit-code contract applies, and a collapsed denominator takes
+  // EXIT_ABSTAINED (3) -- distinct from 1, "found something real".
+  if (!opts.strict) return 0;
+  if (outcome.abstained) return EXIT_ABSTAINED;
+  return result.findings.length ? 1 : 0;
+}
+
+// Direct execution (the skill runner execs this file via its shebang).
+//
+// `process.exitCode` rather than `process.exit()`: a `--json` payload over a
+// large suite exceeds the pipe buffer, and `process.exit` tears the process
+// down mid-write, truncating it to ~64KB. Truncated JSON that still exits 0 is
+// a machine-readable result a consumer cannot parse but a shell reads as
+// success -- the exact class of quiet failure the whole family guards against.
+// Setting the code lets node drain stdout and exit with the same status.
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/agents/skills/claude-code/canary-cassandra/scripts/engine.mjs

@@ -0,0 +1,95 @@
+// Engine resolution for the cassandra skill CLI (#755).
+//
+// WHY THIS SKILL IMPORTS THE ENGINE AND ITS SIBLINGS DO NOT
+//
+// savant, blackhawk and katana each own their detection outright, so their CLIs
+// are self-contained by construction. Cassandra's detection is NOT its own: it
+// already lives in `ts/src/core/vacuity-scanner.ts` and is already exercised by
+// `canary vacuity-check` and by the promotion gate. That module's docstring
+// states the position deliberately -- #605 accepted that `static_linter` and
+// `quality_scorer` already overlap and a third half-enforcer would be the real
+// defect -- so shipping a second, hand-copied vacuity scanner inside this skill
+// would trade #755's asymmetry for a worse one: two detectors that disagree.
+//
+// So the missing piece was never the detection. It was the `cli:` entry point,
+// and this file is what lets one exist without forking the rules.
+//
+// The engine sits three-plus levels above this script in both layouts canary
+// ships, and the lookup is a fixed, ordered list rather than a search: an
+// ambiguous resolution would make which rules ran depend on the install.
+//
+//   repo checkout : <root>/ts/dist/core/...        (after `npm --prefix ts run build`)
+//   npm package   : <pkg>/dist/engine/core/...     (staged by npm/scripts/build-engine.mjs)
+//
+// A failure to resolve is reported as a failure. It is never absorbed into a
+// clean scan -- "the detector could not load" and "the detector found nothing"
+// must not print the same thing.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+
+// scripts -> canary-cassandra -> claude-code -> skills -> agents -> <root>
+const ROOT = path.resolve(HERE, '..', '..', '..', '..', '..');
+
+/** Candidate engine directories, highest priority first. */
+export function engineCandidates(root = ROOT, env = process.env) {
+  const candidates = [];
+  // An explicit override exists for overlay installs, where the skill is
+  // deployed away from the engine that owns its rules.
+  if (env.CANARY_ENGINE_DIR)
+    candidates.push(path.resolve(env.CANARY_ENGINE_DIR));
+  candidates.push(path.join(root, 'ts', 'dist'));
+  candidates.push(path.join(root, 'dist', 'engine'));
+  return candidates;
+}
+
+const MODULES = [
+  ['core', 'vacuity-scanner.js'],
+  ['core', 'test-files.js'],
+  ['core', 'gate-result.js'],
+];
+
+/** The first candidate directory holding every module we need, or null. */
+export function resolveEngineDir(candidates = engineCandidates()) {
+  for (const dir of candidates) {
+    if (MODULES.every((parts) => fs.existsSync(path.join(dir, ...parts)))) {
+      return dir;
+    }
+  }
+  return null;
+}
+
+/**
+ * Load the engine halves this CLI delegates to.
+ *
+ * @returns `{ ok: true, ... }` on success, or `{ ok: false, error }` naming
+ *   every directory that was tried -- a zero-denominator report of its own.
+ */
+export async function loadEngine(candidates = engineCandidates()) {
+  const dir = resolveEngineDir(candidates);
+  if (dir === null) {
+    return {
+      ok: false,
+      error:
+        'the vacuity engine could not be located, so nothing was scanned. ' +
+        `Tried: ${candidates.join(', ')}. In a Canary checkout run ` +
+        '`npm --prefix ts run build`; otherwise reinstall canary-test-cli, or ' +
+        'set CANARY_ENGINE_DIR to the directory holding core/.',
+    };
+  }
+  const load = (parts) => import(pathToFileURL(path.join(dir, ...parts)).href);
+  const [vacuity, testFiles, gate] = await Promise.all(MODULES.map(load));
+  return {
+    ok: true,
+    dir,
+    scanVacuity: vacuity.scanVacuity,
+    collectTestFiles: testFiles.collectTestFiles,
+    isDir: testFiles.isDir,
+    SCANNABLE_DESC: testFiles.SCANNABLE_DESC,
+    gateOutcome: gate.gateOutcome,
+    EXIT_ABSTAINED: gate.EXIT_ABSTAINED,
+  };
+}

package/agents/skills/claude-code/canary-ci-ready/SKILL.md

@@ -0,0 +1,161 @@
+---
+name: canary-ci-ready
+description: >
+  Analyses a test suite for CI readiness: coverage depth, flakiness, assertion
+  quality, critical path coverage, and suite runtime. Accepts documented
+  failures (quarantined tests with linked open issues count as verified).
+  Investigates config/auth failures using the consuming repo's declared
+  user_catalog_skill.
+---
+
+# Canary: CI Ready
+
+Analyses a test suite across five dimensions and produces a readiness score. Use
+this before promoting a suite to CI, or as the convergence gate in
+`/canary-test-pipeline`.
+
+## When to Use
+
+- Before wiring a new test suite into CI for the first time
+
+- When a CI run is failing and you need to understand why
+
+- As part of `/canary-test-pipeline` (Phase 0 and convergence gate)
+
+- When asked "is this suite ready for CI?"
+
+## The Five Checks
+
+Run all five checks and score each pass / warn / fail.
+
+### 1. Coverage depth
+
+Read `.canary/test-inventory.json` if present. If absent or older than 7 days,
+run `canary coverage` to generate fresh data.
+
+Default threshold: depth ≥ 2 for all endpoints in critical areas. Override with
+`--threshold <n>`.
+
+- **pass** — all critical-area endpoints at depth ≥ threshold
+
+- **warn** — some endpoints at depth 1 (hit but unasserted)
+
+- **fail** — any critical-area endpoint at depth 0
+
+### 2. Flakiness
+
+Read `test-results/quarantine-ledger.json` (or the path in
+`.canary/company.json` under `quarantine_ledger_path` if set).
+
+A quarantined test is acceptable only when it has a linked open issue (Jira or
+GitHub). Check issue state:
+
+- Linked issue **open** → counts as verified (documented, tracked)
+
+- Linked issue **closed** → flag: quarantine should be resolved
+
+- **No linked issue** → fail: unlinked quarantine blocks CI-ready
+
+### 3. Assertion quality
+
+Read depth scores from the inventory. In critical-area endpoints:
+
+- **pass** — all tests at depth ≥ 2 (shaped assertions: result.ok or equivalent)
+
+- **warn** — some tests at depth 1 (status-only assertions)
+
+- **fail** — majority of critical-path tests at depth 1
+
+### 4. Critical path coverage
+
+Only run this check if `.canary/critical-areas.json` is present.
+
+Cross-reference the top 5 risk-scored areas from `critical-areas.json` against
+`test-inventory.json`:
+
+- **pass** — all top-5 areas have at least one test at depth ≥ 1
+
+- **warn** — one area uncovered
+
+- **fail** — two or more top areas uncovered
+
+- **skip** — `critical-areas.json` absent (note this in output, not a failure)
+
+### 5. Suite runtime
+
+Read `test-results/run-history.ndjson`. Use the p95 of the last 10 runs.
+
+**With harness MCP available:** score the p95 against trend history rather than
+an absolute clock. Call `get_perf_baselines` and compare this run's p95 to the
+recorded baseline for the suite:
+
+- **pass** — p95 within the baseline's tolerance, or an improvement
+- **warn** — p95 regressed past tolerance but under 2× the baseline
+- **fail** — p95 at or over 2× the baseline
+- **skip** — no baseline recorded yet (this is a baseline-capture run, not a
+  failure — say so in the output)
+
+After scoring, record the run back into the baseline with
+`update_perf_baselines` so the trend keeps moving. Do **not** record a run that
+failed for unrelated reasons — a broken run's runtime is not a data point.
+
+Regression beats absolute here. A suite that has always taken 11 minutes is a
+fact of life; a suite that went from 3 minutes to 11 this week is the actual
+signal, and an absolute threshold cannot tell those apart — it fails the first
+forever and stays silent on the second until it crosses the line.
+
+**Fallback (no MCP):** judge against absolute thresholds.
+
+- **pass** — p95 under the configured timeout (default: 5 minutes)
+- **warn** — p95 between 5–10 minutes
+- **fail** — p95 over 10 minutes, or no run history (cannot assess)
+
+State which mode was used in the output — "runtime vs. baseline" or "runtime vs.
+absolute threshold" — so a reader knows whether a pass means "no regression" or
+merely "under the clock".
+
+## User Catalog Investigation
+
+When a test fails with an auth, permission, or configuration error:
+
+1. Read `user_catalog_skill` from `.canary/company.json`
+2. If present: invoke `canary skills run <user_catalog_skill>` with the required
+   attributes from the error context; surface any matching user as a suggestion
+3. If absent, or no matching user found: present constructively —
+
+   > "This failure may be a test user or test data configuration issue. Check
+   > your user catalog if you have one, or set up the required test data before
+   > re-running."
+
+Never reference a specific catalog skill by name in output.
+
+## Output Format
+
+```text
+CI Readiness — <repo-name>
+
+  ✓ / ⚠ / ✗   <check name>   <brief finding>
+  ...
+
+  Score: N/5 — CI-READY  or  NOT CI-READY
+
+  Runtime scored vs. baseline  |  vs. absolute threshold
+
+  <gap list with suggested next actions>
+```
+
+Score of 5/5 = CI-READY. Any fail = NOT CI-READY. Warns do not block.
+
+## Flags
+
+- `--threshold <n>` — minimum depth for coverage check (default: 2)
+
+## Related skills
+
+- `/canary-test-pipeline` — orchestrates this skill as Phase 0 and convergence
+  gate
+
+- `/canary-critical-areas` — produces `critical-areas.json` used by check 4
+
+- `canary-unquarantine` (overlay) — resolves quarantined tests once bugs are
+  fixed

package/agents/skills/claude-code/canary-ci-ready/skill.yaml

@@ -0,0 +1,14 @@
+name: canary-ci-ready
+version: '1.0.0'
+description:
+  Analyse a test suite for CI readiness across coverage depth, flakiness,
+  assertion quality, critical-path coverage, and runtime; documented/quarantined
+  failures count as verified.
+stability: static
+triggers:
+  - manual
+platforms:
+  - claude-code
+type: rigid
+tools: []
+tier: 1