canary-test-cli 7.0.0 → 7.2.0

package/agents/skills/claude-code/canary-savant/scripts/cli.mjs

@@ -0,0 +1,274 @@
+#!/usr/bin/env node
+// canary-savant -- order-dependence and isolation detector (Tier-1 static scan).
+//
+// Phase 1 ships the always-on static "suspect" tier: an AST-lite scan that
+// flags the shared-state smells that predict order-dependent tests -- module-
+// level mutables written by tests, setup without teardown, mutated process
+// singletons, order-coupled names -- with no test execution. The opt-in
+// dynamic confirmer (--confirm) lands in a later phase.
+//
+//   <paths>    files or directories to scan (default: the current directory).
+//   --json     emit machine-readable findings instead of human text.
+//   --strict   exit 1 when there are findings (default is advisory: exit 0).
+//
+// Tier-0 in the real sense -- no LLM, no network, no secrets, no dependency on
+// any other skill.
+//
+// Invoked via `canary skills run canary-savant -- [paths] [--json] [--strict]`.
+
+import fs from 'node:fs';
+import { scanPaths, toJson } from './scanner.mjs';
+import { confirm, locatePolluters, realPolluterSeams } from './runner.mjs';
+import { RULES } from './rules.mjs';
+import {
+  createParser,
+  formatUsageError,
+  EXIT_USAGE,
+} from '../../../lib/parse-args.mjs';
+
+export const SCHEMA_VERSION = 1;
+
+// --- no-silent-abstention (#508 D2, skill-CLI convention half) ---------------
+//
+// Skill CLIs are deliberately self-contained -- no engine import, no shared
+// module -- so they cannot call `gateOutcome`. They honour the doctrine by
+// CONVENTION instead, emitting the same greppable line the engine helper does.
+// The skill-layer conformance registry (agents/skills/test/gate-conformance.
+// test.ts) is what holds them to it: a row whose fixture collapses the
+// denominator and asserts the loud outcome.
+//
+// U+26A0 / U+2014 are written as escapes so this source stays ASCII, matching
+// ts/src/core/gate-result.ts.
+const ABSTAINED_LINE =
+  '\u{26A0} Abstained \u{2014} verified zero items; this is not a pass.';
+
+const PREFIX = 'canary-savant:';
+
+// The rules block is GENERATED from RULES, never hand-typed, so a new rule
+// appears in --help the moment it is registered.
+const USAGE =
+  'usage: canary-savant [-h] [--json] [--strict] [--confirm] [--seed N] [--]\n' +
+  '                     [path ...]\n' +
+  '\n' +
+  'Order-dependence and isolation detector: flags the shared-state smells that\n' +
+  'predict order-dependent tests, and optionally confirms them dynamically.\n' +
+  '\n' +
+  'positional arguments:\n' +
+  '  path        files or directories to scan (default: the current directory)\n' +
+  '\n' +
+  'options:\n' +
+  '  -h, --help  show this help message and exit\n' +
+  '  --json      emit machine-readable findings instead of human text\n' +
+  '  --strict    exit 1 when there are findings (default is advisory: exit 0)\n' +
+  '  --confirm   run the opt-in Tier-2 dynamic confirmer (executes the suite)\n' +
+  '  --seed N    shuffle seed for --confirm (default: random)\n' +
+  '\n' +
+  'rules:\n' +
+  RULES.map((r) => `  ${r.ruleId} (${r.severity})`).join('\n');
+
+function summary(result) {
+  const bySeverity = {};
+  for (const f of result.findings) {
+    bySeverity[f.severity] = (bySeverity[f.severity] || 0) + 1;
+  }
+  return {
+    files_scanned: result.filesScanned,
+    findings: result.findings.length,
+    by_severity: bySeverity,
+    suppressed: result.suppressed ?? 0,
+  };
+}
+
+// A trailing "N suppressed" note keeps inline-ignored lines visible but out of
+// the actionable total - the pattern canary-blackhawk (#393) and the
+// PR-guardian sticky comment use.
+function suppressedNote(result) {
+  const n = result.suppressed ?? 0;
+  return n ? `\n${n} suppressed (inline savant-ignore).` : '';
+}
+
+function renderText(result) {
+  const count = result.findings.length;
+  const files = result.filesScanned;
+  const fp = files === 1 ? '' : 's';
+  // #508: zero suspects over zero scanned files is an ABSENT result, not a
+  // clean one. Findings outrank abstention, so this is the no-findings path.
+  if (!count && !files) {
+    return (
+      `${ABSTAINED_LINE} No file matched the given paths, so there is ` +
+      'nothing to report. Point at a directory that holds test files, or ' +
+      'pass a file directly.'
+    );
+  }
+  if (!count) {
+    return (
+      `No order-dependence suspects (${files} file${fp} scanned).` +
+      suppressedNote(result)
+    );
+  }
+  const sp = count === 1 ? '' : 's';
+  const lines = [
+    `${count} order-dependence suspect${sp} in ${files} file${fp}:`,
+    '',
+  ];
+  for (const f of result.findings) {
+    lines.push(`  ${f.file}:${f.line}  [${f.severity}] ${f.ruleId}`);
+    lines.push(`      ${f.snippet}`);
+    lines.push(`      why: ${f.why}`);
+  }
+  lines.push('');
+  lines.push(
+    'Advisory by default. Re-run with --strict to fail the step on findings.',
+  );
+  return lines.join('\n') + suppressedNote(result);
+}
+
+/**
+ * savant takes paths, so it gets the `--` end-of-options terminator and treats
+ * a lone `-` as a positional, as argparse does.
+ *
+ * `--seed` is the flag that made the shared int type worth having: a bad value
+ * used to decay to `Math.floor(Math.random() * 1e6)` at exit 0, silently
+ * randomizing the one flag whose entire purpose is reproducibility. The shared
+ * parser rejects a non-integer AND an integer past the safe range (#479).
+ */
+export const CLI_SPEC = {
+  prog: 'canary-savant',
+  booleans: {
+    '--json': 'json',
+    '--strict': 'strict',
+    '--confirm': 'confirm',
+  },
+  values: { '--seed': { key: 'seed', type: 'int' } },
+  positionals: { key: 'paths', defaults: ['.'] },
+};
+
+const parseArgs = createParser(CLI_SPEC);
+
+export function renderConfirm(dyn) {
+  if (
+    dyn.status === 'no_plugin' ||
+    dyn.status === 'baseline_red' ||
+    dyn.status === 'unknown_framework'
+  ) {
+    return `\nTier 2 (dynamic): skipped - ${dyn.message}`;
+  }
+  const lines = [`\nTier 2 (dynamic): seed ${dyn.seed}`];
+  if (!dyn.victims.length) {
+    lines.push('  No order-dependence confirmed under this seed.');
+  } else {
+    let namedAny = false;
+    for (const v of dyn.victims) {
+      lines.push(`  order-dependent: ${v.victim}`);
+      if (v.polluter) {
+        lines.push(`    polluted by: ${v.polluter}`);
+        if (v.reproduce) lines.push(`    reproduce: ${v.reproduce}`);
+        namedAny = true;
+      } else if (v.note) {
+        lines.push(`    ${v.note}`);
+      } else if (v.exhausted) {
+        lines.push(
+          `    no single culprit isolated; smallest reproducing prefix: ` +
+            `${v.minimalPrefix?.length ?? '?'} test(s)`,
+        );
+      }
+    }
+    if (!namedAny && dyn.reproduce) {
+      lines.push(`  reproduce: ${dyn.reproduce}`);
+    }
+    if (dyn.framework === 'vitest') {
+      lines.push(
+        '  (vitest: victims detected; polluter bisect is pytest-only)',
+      );
+    }
+  }
+  if (dyn.nondeterministic.length) {
+    lines.push(
+      `  (${dyn.nondeterministic.length} nondeterministic flake(s) - not order, handed off)`,
+    );
+  }
+  return lines.join('\n');
+}
+
+export function main(argv = []) {
+  const { positionals: paths, opts, help, error } = parseArgs(argv);
+
+  // Usage and parse errors resolve before any filesystem work, so `--help`
+  // never reports a missing path and a typo never half-runs a scan.
+  if (help) {
+    console.log(USAGE);
+    return 0;
+  }
+  if (error) {
+    console.error(formatUsageError(CLI_SPEC.prog, error));
+    return EXIT_USAGE;
+  }
+
+  for (const entry of paths) {
+    if (!fs.existsSync(entry)) {
+      console.error(`${PREFIX} path not found: ${entry}`);
+      return 1;
+    }
+  }
+
+  const result = scanPaths(paths);
+
+  let dyn;
+  if (opts.confirm) {
+    const seed = Number.isFinite(opts.seed)
+      ? opts.seed
+      : Math.floor(Math.random() * 1e6);
+    dyn = confirm(paths, { seed });
+    // Phase 3: name the polluter behind each confirmed victim. pytest only -
+    // vitest has no CLI-driven ordered per-test execution, so it gets victim
+    // detection (Phase 2) but not polluter bisection.
+    if (
+      dyn.status === 'ok' &&
+      dyn.victims.length &&
+      dyn.framework === 'pytest'
+    ) {
+      dyn.victims = locatePolluters(
+        dyn.victims,
+        dyn.order,
+        realPolluterSeams(paths),
+      );
+    }
+  }
+
+  if (opts.json) {
+    const payload = {
+      schema_version: SCHEMA_VERSION,
+      findings: result.findings.map(toJson),
+      summary: summary(result),
+    };
+    if (dyn) {
+      payload.dynamic = {
+        status: dyn.status,
+        seed: dyn.seed,
+        victims: dyn.victims,
+        nondeterministic: dyn.nondeterministic,
+        reproduce: dyn.reproduce,
+        ...(dyn.message ? { message: dyn.message } : {}),
+      };
+    }
+    console.log(JSON.stringify(payload, null, 2));
+  } else {
+    console.log(renderText(result) + (dyn ? renderConfirm(dyn) : ''));
+  }
+
+  const hasViolation =
+    result.findings.length > 0 || (dyn?.victims.length ?? 0) > 0;
+  // Advisory by default (D3); --strict inherits EXIT_ABSTAINED (3) on a
+  // collapsed denominator, distinct from 1 ("found something real").
+  if (opts.strict && !result.filesScanned) return 3;
+  return opts.strict && hasViolation ? 1 : 0;
+}
+
+// Direct execution (the skill runner execs this file via its shebang).
+//
+// `process.exitCode`, not `process.exit()`: a large `--json` payload exceeds
+// the pipe buffer, and `process.exit` tears the process down mid-write, leaving
+// truncated JSON that still exits 0 (#791).
+if (import.meta.url === `file://${process.argv[1]}`) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/agents/skills/claude-code/canary-savant/scripts/restoration.mjs

@@ -0,0 +1,274 @@
+// SV003 restoration analysis: does the file restore the global it mutates?
+//
+// SV003's `why` asserts persistence ("the change persists across tests").
+// Dogfooding (#493) showed 37 of 51 self-scan findings were in files using
+// the textbook save-in-beforeEach / restore-in-afterEach pattern -- the rule
+// was asserting a consequence it never checked. This module supplies the
+// check; when it finds restoration, the scanner suppresses the finding.
+//
+// Chosen heuristic -- deliberately conservative, because a false skip hides
+// real pollution while a false flag is merely advisory noise. A mutation of
+// family F (process.env / os.environ / sys.modules) with key K is considered
+// restored only when there is POSITIVE evidence:
+//
+//  1. Snapshot write-back: the mutation's own RHS is a plain identifier (or
+//     one index/property off it) that some line saves FROM the same family
+//     (`saved[v] = process.env[v]`, `origCI = process.env.CI`,
+//     `old = os.environ.copy()`). That line IS the restore, not a pollution.
+//  2. Teardown restore: inside a teardown region, the same family is
+//     restored -- assigned or deleted with key K, or with a computed key /
+//     update() / clear() / Object.assign(family, ...) which restores the
+//     whole family (the save-restore loop pattern).
+//
+// Teardown regions: JS afterEach/afterAll call bodies (paren-balanced,
+// string-aware, capped at 50 lines if unclosed -- an unbalanced count must
+// not swallow the file into "teardown"); Python teardown_*/tearDown* def
+// bodies, code after a fixture `yield` (indentation-scoped), and addCleanup
+// lines. Detection is language-agnostic like the rest of the scanner; a JS
+// generator `yield` could open a phantom region, but a same-family restore
+// idiom inside one is overwhelmingly teardown-intent anyway.
+//
+// Known non-suppressors, on purpose:
+// - vi.stubEnv/vi.unstubAllEnvs and monkeypatch only undo their OWN
+//   mutations, never a direct `process.env.X = ...` / `os.environ[...] =`,
+//   so their presence must not launder a direct mutation. (Mutations made
+//   THROUGH them never match SV003's assign patterns in the first place.)
+// - `addCleanup(os.environ.pop, 'K')` (function-reference form) is not
+//   recognized; only idioms where the restore call is spelled out are. A
+//   missed restore is a false flag, the safe direction.
+
+import { SINGLETON_FAMILIES } from './rules.mjs';
+import {
+  stringLiteralRanges,
+  inStringLiteral,
+  execOutsideStrings,
+} from './string-literals.mjs';
+
+// An unclosed JS teardown region (unbalanced parens, e.g. a regex literal
+// confusing the counter) extends at most this many lines past its opener.
+const REGION_CAP_LINES = 50;
+
+const splitLines = (text) => text.split(/\r\n|\r|\n/);
+
+/** Every match of `pattern` in `line` whose start index is code. */
+function execAllOutsideStrings(pattern, line, ranges) {
+  const flags = pattern.flags.includes('g')
+    ? pattern.flags
+    : `${pattern.flags}g`;
+  const re = new RegExp(pattern.source, flags);
+  const matches = [];
+  let match;
+  while ((match = re.exec(line)) !== null) {
+    if (!inStringLiteral(ranges, match.index)) matches.push(match);
+    if (re.lastIndex === match.index) re.lastIndex += 1;
+  }
+  return matches;
+}
+
+/** 'FOO' from a quoted bracket expression; null for a computed key. */
+function literalKey(expr) {
+  if (expr == null) return null;
+  const m = /^\s*(['"`])(.*)\1\s*$/.exec(expr);
+  return m ? m[2] : null;
+}
+
+/** Key from an assign/delete match: dot-property group or bracket literal. */
+function keyOf(match) {
+  // process.env has (dotKey, bracketExpr); the Python families have a single
+  // bracket/arg group. A dot-property is always a literal key.
+  if (match.length > 2) return match[1] ?? literalKey(match[2]);
+  return literalKey(match[1]);
+}
+
+/**
+ * Classify the singleton mutation on `line`, if any.
+ * @param {string} line
+ * @param {Array<[number, number]>} ranges string ranges for `line`
+ * @returns {{family: string, key: string|null, rhs: string}|null}
+ */
+export function classifyMutation(line, ranges) {
+  for (const family of SINGLETON_FAMILIES) {
+    const match = execOutsideStrings(family.assign, line, ranges);
+    if (!match) continue;
+    return {
+      family: family.id,
+      key: keyOf(match),
+      rhs: line.slice(match.index + match[0].length),
+    };
+  }
+  return null;
+}
+
+// The mutation's RHS must be a bare identifier plus at most one index or
+// property -- `saved[v]`, `origCI`, `snap.CI` -- for write-back detection.
+const RHS_IDENT = /^\s*([A-Za-z_$][\w$]*)\s*(?:\[[^\]]*\]|\.\w+)?\s*[;,]?\s*$/;
+
+const RE_META = /[.*+?^${}()|[\]\\]/g;
+const escapeRe = (s) => s.replace(RE_META, '\\$&');
+
+/**
+ * True when the mutation is a write-back from a snapshot of the same family:
+ * its RHS reads a variable that some line saved from the family via a PURE
+ * snapshot (`x = process.env[...]`, `x = os.environ.copy()`,
+ * `x = dict(os.environ)`, `x = { ...process.env }`). Expressions
+ * (`x = process.env.A + '/y'`) are not snapshots and do not count.
+ * @param {{family: string, rhs: string}} mutation from classifyMutation
+ * @param {string[]} lines the file's lines
+ * @returns {boolean}
+ */
+export function isSnapshotWriteBack(mutation, lines) {
+  const rhs = RHS_IDENT.exec(mutation.rhs);
+  if (!rhs) return false;
+  const family = SINGLETON_FAMILIES.find((f) => f.id === mutation.family);
+  const saveEvidence = new RegExp(
+    `(?:^|[^.\\w$])${escapeRe(rhs[1])}\\s*(?:\\[[^\\]]*\\]|\\.\\w+)?\\s*=\\s*` +
+      `(?:\\{\\s*\\.\\.\\.\\s*${family.token}\\s*\\}` +
+      `|dict\\(\\s*${family.token}\\s*\\)` +
+      `|${family.token}(?:\\s*\\[[^\\]]*\\]|\\.\\w+\\([^)]*\\)|\\.\\w+)?` +
+      `)\\s*[;,)]?\\s*$`,
+  );
+  return lines.some((l) => saveEvidence.test(l));
+}
+
+// --- Teardown-region collection ----------------------------------------------
+
+const JS_TEARDOWN_TOKEN = /\b(?:afterEach|afterAll)\s*\(/;
+const PY_TEARDOWN_DEF = /^(\s*)def\s+(?:teardown\w*|tearDown\w*)\s*\(/;
+const PY_YIELD = /^(\s*)yield\b/;
+// #733: an in-test `finally` restore is TIGHTER than an afterEach -- the
+// window in which the global is dirty is the try block, not the whole test --
+// yet only framework hooks counted, so the better idiom was the flagged one.
+const JS_FINALLY = /\bfinally\s*\{/;
+const PY_FINALLY = /^(\s*)finally\s*:/;
+
+const indentOf = (line) => /^\s*/.exec(line)[0].length;
+const isBlank = (line) => line.trim() === '';
+
+/** JS: afterEach/afterAll call bodies, paren-balanced and string-aware. */
+function collectJsRegions(lines, rangesByLine, region) {
+  lines.forEach((line, i) => {
+    const token = execOutsideStrings(JS_TEARDOWN_TOKEN, line, rangesByLine[i]);
+    if (!token) return;
+    let depth = 0;
+    let col = token.index + token[0].length - 1; // the opening paren
+    for (let j = i; j < lines.length && j <= i + REGION_CAP_LINES; j += 1) {
+      region.add(j);
+      const text = lines[j];
+      const start = j === i ? col : 0;
+      for (let k = start; k < text.length; k += 1) {
+        if (inStringLiteral(rangesByLine[j], k)) continue;
+        if (text[k] === '(') depth += 1;
+        else if (text[k] === ')') {
+          depth -= 1;
+          if (depth === 0) return;
+        }
+      }
+    }
+  });
+}
+
+/** JS: `finally { ... }` bodies, brace-balanced and string-aware (#733). */
+function collectJsFinallyRegions(lines, rangesByLine, region) {
+  lines.forEach((line, i) => {
+    const token = execOutsideStrings(JS_FINALLY, line, rangesByLine[i]);
+    if (!token) return;
+    let depth = 0;
+    // Count from the finally's own `{`, never the line start -- the common
+    // spelling `} finally {` opens with the TRY block's closing brace.
+    let col = token.index + token[0].length - 1;
+    for (let j = i; j < lines.length && j <= i + REGION_CAP_LINES; j += 1) {
+      region.add(j);
+      const text = lines[j];
+      const start = j === i ? col : 0;
+      for (let k = start; k < text.length; k += 1) {
+        if (inStringLiteral(rangesByLine[j], k)) continue;
+        if (text[k] === '{') depth += 1;
+        else if (text[k] === '}') {
+          depth -= 1;
+          if (depth === 0) return;
+        }
+      }
+    }
+  });
+}
+
+/** Python: teardown def bodies, post-yield code, addCleanup lines. */
+function collectPyRegions(lines, rangesByLine, region) {
+  lines.forEach((line, i) => {
+    const def = PY_TEARDOWN_DEF.exec(line);
+    if (def) {
+      const indent = def[1].length;
+      for (let j = i + 1; j < lines.length; j += 1) {
+        if (!isBlank(lines[j]) && indentOf(lines[j]) <= indent) break;
+        region.add(j);
+      }
+    }
+    const yielded = PY_YIELD.exec(line);
+    if (yielded) {
+      const indent = yielded[1].length;
+      for (let j = i + 1; j < lines.length; j += 1) {
+        if (!isBlank(lines[j]) && indentOf(lines[j]) < indent) break;
+        region.add(j);
+      }
+    }
+    // `finally:` is indentation-scoped, like the def and yield regions (#733).
+    const fin = PY_FINALLY.exec(line);
+    if (fin) {
+      const indent = fin[1].length;
+      for (let j = i + 1; j < lines.length; j += 1) {
+        if (!isBlank(lines[j]) && indentOf(lines[j]) <= indent) break;
+        region.add(j);
+      }
+    }
+    if (execOutsideStrings(/\baddCleanup\b/, line, rangesByLine[i])) {
+      region.add(i);
+    }
+  });
+}
+
+/**
+ * Analyze which globals the file restores in teardown.
+ * @param {string} text file contents
+ * @returns {{restores: (family: string, key: string|null) => boolean}}
+ */
+export function analyzeRestoration(text) {
+  const lines = splitLines(text);
+  const rangesByLine = lines.map((l) => stringLiteralRanges(l));
+  const region = new Set();
+  collectJsRegions(lines, rangesByLine, region);
+  collectJsFinallyRegions(lines, rangesByLine, region);
+  collectPyRegions(lines, rangesByLine, region);
+
+  const restoresAll = new Set();
+  const restoredKeys = new Map(); // family id -> Set<key>
+  const record = (familyId, key) => {
+    if (key == null) {
+      restoresAll.add(familyId);
+      return;
+    }
+    if (!restoredKeys.has(familyId)) restoredKeys.set(familyId, new Set());
+    restoredKeys.get(familyId).add(key);
+  };
+
+  for (const i of region) {
+    const line = lines[i];
+    const ranges = rangesByLine[i];
+    for (const family of SINGLETON_FAMILIES) {
+      for (const pattern of [family.assign, ...family.deletes]) {
+        for (const match of execAllOutsideStrings(pattern, line, ranges)) {
+          record(family.id, keyOf(match));
+        }
+      }
+      for (const pattern of family.restoreAll) {
+        if (execOutsideStrings(pattern, line, ranges)) record(family.id, null);
+      }
+    }
+  }
+
+  return {
+    restores(familyId, key) {
+      if (restoresAll.has(familyId)) return true;
+      return key != null && (restoredKeys.get(familyId)?.has(key) ?? false);
+    },
+  };
+}