canary-test-cli 7.2.0 → 8.0.0

package/dist/engine/batwoman-cli.js

@@ -0,0 +1,119 @@
+/**
+ * `canary batwoman --issue N [--json]` (spec Phase 4).
+ *
+ * The first surface a human meets, and the point where every earlier phase is
+ * wired together: the closure adapter names the change, the probes decide per
+ * file, and the renderer says it in the reader's register.
+ *
+ * **The two output paths are not interchangeable.** The human report is
+ * persona-shaped prose; `--json` is the CI wrapper's input and is
+ * persona-independent, because a machine consumer's parser must not shift with
+ * whoever happened to run the command. Both carry every changed file, and
+ * neither carries an aggregate a reader could mistake for a pass -- `--json`
+ * especially, since a convenient `passed: true` is exactly the field a CI
+ * wrapper would grow later (spec criteria 2 and 3).
+ *
+ * A failure to resolve the closure exits non-zero and prints nothing to
+ * stdout: there is no report to give. A failure to read run history does not,
+ * because "could not tell" is a real verdict over a known set of files, and
+ * losing the whole report to it would be worse than reporting the abstentions.
+ */
+import { Command, InvalidArgumentError } from 'commander';
+import { auditIssue, } from './analysis/batwoman/audit.js';
+import { renderReport } from './analysis/batwoman/render.js';
+import { tallyVerdicts } from './analysis/batwoman/verdict.js';
+import { resolvePersona } from './core/persona.js';
+import { CliExitError } from './cli-common.js';
+/**
+ * `owner/name` for the repository under audit.
+ *
+ * Read from the environment GitHub Actions already sets, so the CI wrapper
+ * needs no argument. Outside Actions it is required explicitly rather than
+ * guessed from a git remote: a wrong repo would silently audit someone else's
+ * run history and report it as this one's.
+ */
+function repoFromEnv(env) {
+    const repo = env['GITHUB_REPOSITORY'];
+    if (repo === undefined || repo.trim() === '') {
+        throw new Error('no repository to audit: set GITHUB_REPOSITORY to "owner/name". It is ' +
+            'not inferred from a git remote, because auditing the wrong ' +
+            "repository's run history would report a confident answer about the " +
+            'wrong thing.');
+    }
+    return repo;
+}
+/** The machine shape. Deliberately flat, and deliberately without a verdict. */
+function toJson({ closure, verdicts }, repo) {
+    const tally = tallyVerdicts(verdicts);
+    return JSON.stringify({
+        issue: closure.header.issue,
+        pullRequest: closure.pullRequest,
+        repo,
+        mergeSha: closure.header.mergeSha,
+        mergeSubject: closure.header.mergeSubject,
+        mergedAt: closure.header.mergedAt.toISOString(),
+        // `changed` and `byStatus` only. There is no `assessed`, no `passed`,
+        // and no score: a consumer wanting a subtotal has to add the statuses up
+        // in the open, where a reviewer can see which ones it folded together.
+        summary: { changed: tally.changed, byStatus: tally.byStatus },
+        files: verdicts.map((v) => ({
+            file: v.file,
+            status: v.status,
+            explanation: v.explanation,
+            ...('evidence' in v && v.evidence !== undefined
+                ? { evidence: v.evidence }
+                : {}),
+        })),
+    }, null, 2);
+}
+/** Build the `batwoman` subcommand. */
+export function buildBatwomanCommand(deps, cli = {}) {
+    const wiring = {
+        repo: () => repoFromEnv(deps.env),
+        root: () => deps.cwd(),
+        ...cli,
+    };
+    const command = new Command('batwoman');
+    command
+        .description('Report whether the files a closed issue changed have executed since ' +
+        'the fix merged. Never asserts the fix is correct.')
+        .requiredOption('--issue <number>', 'the closed issue to audit', (raw) => {
+        const n = Number.parseInt(raw, 10);
+        if (!Number.isInteger(n) || n <= 0) {
+            // commander's own error type, so a bad flag exits as a USAGE error
+            // (2) with a clean message, rather than as an unexpected crash with a
+            // stack trace in front of the user.
+            throw new InvalidArgumentError(`--issue must be a positive integer, got "${raw}"`);
+        }
+        return n;
+    })
+        .option('--json', 'emit the machine shape instead of the report')
+        .action(async (opts) => {
+        let audit;
+        let repo;
+        try {
+            repo = wiring.repo();
+            audit = await auditIssue(opts.issue, repo, wiring.root(), wiring);
+        }
+        catch (err) {
+            // Nothing is printed to stdout: there is no report to give, and half a
+            // report would be read as a whole one. Only a failure to resolve the
+            // closure lands here -- an unreadable run history becomes per-file
+            // abstentions inside the audit, which are worth printing.
+            deps.err(err instanceof Error ? err.message : String(err));
+            throw new CliExitError(1);
+        }
+        if (opts.json === true) {
+            deps.out(toJson(audit, repo));
+            return;
+        }
+        deps.out(renderReport({
+            header: audit.closure.header,
+            repo,
+            verdicts: audit.verdicts,
+            persona: resolvePersona(wiring.registry === undefined ? {} : { registry: wiring.registry }),
+        }));
+    });
+    return command;
+}
+//# sourceMappingURL=batwoman-cli.js.map

package/dist/engine/ci-ready-cli.js

@@ -0,0 +1,71 @@
+/**
+ * `canary ci-ready`: the deterministic scorer behind the canary-ci-ready skill.
+ *
+ * Reads its inputs under `--root` and hands them to the pure scoring in
+ * `core/ci-ready.ts`. Exit codes follow the CLI-wide gate contract:
+ *   - 3 (EXIT_ABSTAINED): every check skipped, so nothing was scored.
+ *   - 1: at least one check failed.
+ *   - 0: nothing failed. That covers both `ready` and `incomplete`; the text
+ *     output says loudly which one it is.
+ */
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { Command } from 'commander';
+import { CliExitError } from './cli-common.js';
+import { EXIT_ABSTAINED } from './core/gate-result.js';
+import { scoreCiReady } from './core/ci-ready.js';
+import { NdjsonHistoryStore } from './history/ndjson-store.js';
+/** Same default location `canary history` writes to (DEFAULT_HISTORY_FILE in history/cli.ts). */
+const HISTORY_FILE = join('test-results', 'reports', 'history-v2.jsonl');
+function readRuns(root) {
+    const path = join(root, HISTORY_FILE);
+    return existsSync(path) ? new NdjsonHistoryStore(path).readAll() : null;
+}
+function renderText(report) {
+    const lines = [
+        `CI readiness: ${report.verdict} — ${report.checked} of ${report.checks.length} checks scored`,
+    ];
+    for (const c of report.checks)
+        lines.push(`  ${c.verdict.padEnd(4)}  ${c.name}: ${c.reason}`);
+    const skipped = report.checks.length - report.checked;
+    if (report.verdict === 'incomplete') {
+        lines.push(`Incomplete: ${skipped} check(s) skipped for missing inputs, so this is not a full readiness result.`);
+    }
+    if (report.verdict === 'abstained') {
+        lines.push('Abstained: no check had an input to score.');
+    }
+    return lines;
+}
+function exitCodeFor(report) {
+    if (report.verdict === 'abstained')
+        return EXIT_ABSTAINED;
+    return report.verdict === 'not-ready' ? 1 : 0;
+}
+export function buildCiReadyCommand(deps) {
+    const command = new Command('ci-ready');
+    command
+        .description('Score CI readiness across five checks; checks with no input report skip, never pass.')
+        .option('--root <dir>', 'Repository root to read inputs from (default: current directory).')
+        .option('--json', 'Output the verdict and every check as JSON.')
+        .action((opts) => {
+        const root = opts.root ?? deps.cwd();
+        const report = scoreCiReady({
+            runs: readRuns(root),
+            historyPath: HISTORY_FILE,
+            hasInventory: existsSync(join(root, '.canary', 'test-inventory.json')),
+            hasCriticalAreas: existsSync(join(root, '.canary', 'critical-areas.json')),
+        });
+        if (opts.json === true) {
+            deps.out(JSON.stringify(report, null, 2));
+        }
+        else {
+            for (const line of renderText(report))
+                deps.out(line);
+        }
+        const code = exitCodeFor(report);
+        if (code !== 0)
+            throw new CliExitError(code);
+    });
+    return command;
+}
+//# sourceMappingURL=ci-ready-cli.js.map

package/dist/engine/cli-commands.js

@@ -369,6 +369,14 @@ export function migrateCmd(opts, deps) {
                 note: r.note,
             })),
             installed_workflows: report.installed_workflows.map((r) => r.to_dict()),
+            // #504 part 1 (spec test 28), additive: every key above keeps its name
+            // and value. `existing_suites` closes a #585 gap -- it was on the
+            // report but never in JSON, leaving a scripted consumer with
+            // `would_create: []` and no reason attached. `workspace` is `null`
+            // for a single-package repo; `shapes` is the set deployment used.
+            existing_suites: report.existing_suites,
+            workspace: report.workspace,
+            shapes: report.shapes,
         }));
         return;
     }
@@ -597,6 +605,7 @@ export function flakeCheckCmd(path, opts, deps) {
  */
 export function vacuityCheckCmd(path, opts, deps) {
     const json = opts.json === true;
+    const verbose = opts.verbose === true;
     const files = isDir(path) ? collectTestFiles(path) : [path];
     const findings = [];
     const skipped = [];
@@ -614,10 +623,26 @@ export function vacuityCheckCmd(path, opts, deps) {
             reason: `no test file matched (looked for ${SCANNABLE_DESC})`,
         });
     }
-    const result = { checked, findings };
-    if (skipped.length > 0)
-        result.skipped = skipped;
-    const outcome = gateOutcome(result, 'advisory', { noun: 'test(s)' });
+    // #860: the summary line carries one entry per skip REASON with its count,
+    // not one per test -- 232 titles on one line is how skips get ignored. The
+    // per-test list stays reachable (--verbose, --json), so every skip remains
+    // countable and locatable; only the summary line is compacted.
+    const byReason = new Map();
+    for (const s of skipped)
+        byReason.set(s.reason, (byReason.get(s.reason) ?? 0) + 1);
+    // The suffix is built here rather than by `skippedSuffix`, whose count is the
+    // number of ENTRIES -- fed per-reason groups it would report "1 skipped" for
+    // 13 skipped tests.
+    const groups = [...byReason]
+        .map(([reason, n]) => `${n} test(s) [${reason}]`)
+        .join('; ');
+    const outcome = gateOutcome({ checked, findings }, 'advisory', {
+        noun: 'test(s)',
+    });
+    const summaryLine = skipped.length === 0
+        ? outcome.summaryLine
+        : `${outcome.summaryLine} (${skipped.length} skipped: ${groups})` +
+            (verbose ? '' : ` ${pc.dim('(--verbose to list skipped tests)')}`);
     // `advisory` keeps findings at exit 0; the abstention still has to be loud, so
     // the exit code for a zero denominator is taken from the gate contract.
     const exitCode = outcome.abstained ? EXIT_ABSTAINED : 0;
@@ -639,9 +664,14 @@ export function vacuityCheckCmd(path, opts, deps) {
         deps.out(`  ${f.test}: ${f.message}`);
         deps.out(`  ${pc.dim(`${ARROW} ${f.suggestion}`)}\n`);
     }
+    if (verbose && skipped.length > 0) {
+        deps.out(pc.bold('Skipped:'));
+        for (const s of skipped)
+            deps.out(`  ${s.name} ${pc.dim(`[${s.reason}]`)}`);
+    }
     deps.out(outcome.abstained
-        ? pc.bold(pc.yellow(outcome.summaryLine))
-        : `${pc.bold(outcome.summaryLine)}`);
+        ? pc.bold(pc.yellow(summaryLine))
+        : `${pc.bold(summaryLine)}`);
     if (exitCode !== 0)
         throw new CliExitError(exitCode);
 }
@@ -793,7 +823,16 @@ export async function ticketUpdateCmd(opts, deps) {
     let reportData = {};
     if (opts.result) {
         try {
-            reportData = JSON.parse(readFileSync(opts.result, 'utf-8'));
+            const parsed = JSON.parse(readFileSync(opts.result, 'utf-8'));
+            // Valid JSON is not necessarily a report. A bare `null` or a top-level
+            // array parses fine and then gets indexed: `null` threw a raw TypeError
+            // out of the handler, and an array silently defaulted every field. Both
+            // belong on the same "could not read result file" path as a parse error.
+            if (parsed === null ||
+                typeof parsed !== 'object' ||
+                Array.isArray(parsed))
+                throw new Error('expected a JSON object at the top level');
+            reportData = parsed;
         }
         catch (exc) {
             deps.out(pc.red(`Could not read result file '${opts.result}': ${exc instanceof Error ? exc.message : String(exc)}`));

package/dist/engine/cli.core.js

@@ -20,6 +20,10 @@ import { Command, Option } from 'commander';
 import { CliExitError, normalizeUsageExit } from './cli-common.js';
 import { createAnalyzeCommand } from './analysis/cli.js';
 import { doctorCmd, feedbackCmd, flakeCheckCmd, listFrameworksCmd, healTestCmd, initCmd, migrateCmd, overlayCmd, promoteCheckCmd, recommendFrameworkCmd, reviewTestCmd, runCmd, setupCmd, ticketUpdateCmd, uninstallCmd, upgradeCmd, vacuityCheckCmd, versionCmd, } from './cli-commands.js';
+import { buildBatwomanCommand } from './batwoman-cli.js';
+import { buildCiReadyCommand } from './ci-ready-cli.js';
+import { buildScalingCurveCommand } from './scaling-curve-cli.js';
+import { buildPermissionMatrixCommand } from './permission-matrix-cli.js';
 import { buildCompanyKnowledgeCommand } from './company-knowledge-cli.js';
 import { createGuardianCommand } from './guardian/cli.js';
 import { createHistoryCommand } from './history/cli.js';
@@ -145,6 +149,7 @@ export function createCanaryCommand(depsInit = {}) {
         .description('Find tests that pass without proving anything -- advisory, no LLM required.')
         .argument('<path>', 'Test file or directory to scan.')
         .option('--json', 'Output the verdict and its denominator as JSON.')
+        .option('--verbose', 'List every skipped test with its file:line.')
         .action((path, opts) => {
         vacuityCheckCmd(path, opts, deps);
     });
@@ -157,6 +162,13 @@ export function createCanaryCommand(depsInit = {}) {
         .option('--dry-run', 'Show what would change without writing.')
         .option('--json', 'Output results as JSON.')
         .action((path, opts) => {
+        // heal-test applies only pattern fixes, so --no-pattern leaves nothing to
+        // do. Guarded here rather than in healTestCmd to keep that function under
+        // the complexity threshold the perf ratchet enforces.
+        if (opts.pattern === false) {
+            deps.out('Pattern fixes disabled (--no-pattern); file left unchanged.');
+            return;
+        }
         healTestCmd(path, opts, deps);
     });
     program
@@ -223,6 +235,10 @@ export function createCanaryCommand(depsInit = {}) {
     program.addCommand(buildSkillsCommand(deps));
     program.addCommand(buildWorkflowCommand(deps));
     program.addCommand(buildCompanyKnowledgeCommand(deps));
+    program.addCommand(buildBatwomanCommand(deps));
+    program.addCommand(buildScalingCurveCommand(deps));
+    program.addCommand(buildPermissionMatrixCommand(deps));
+    program.addCommand(buildCiReadyCommand(deps));
     // Propagate the usage-exit normalization to every top-level command (the
     // sub-apps also set it on their own subcommands internally).
     for (const sub of program.commands) {

package/dist/engine/company-knowledge-cli.js

@@ -78,8 +78,16 @@ function ckShowCmd(opts, deps) {
 export function ckInitCmd(opts, deps) {
     const canaryDir = join(deps.cwd(), '.canary');
     const outPath = join(canaryDir, 'company.json');
-    // Existing values become the shown defaults (load() returns empty when absent).
-    const existing = CompanyKnowledge.load(deps.cwd(), null, deps.home());
+    // Existing values become the shown defaults (load() returns empty when
+    // absent). `--force` means "start from scratch", which is what the
+    // already-exists banner promises and what the flag's own help text says, so
+    // it must drop those defaults entirely -- otherwise every prompt still falls
+    // back to the old value, the doc-URL accumulator and brand map are still
+    // pre-seeded, and --force only silences the banner while rewriting the old
+    // file verbatim, with no way to clear a field.
+    const existing = opts.force
+        ? new CompanyKnowledge()
+        : CompanyKnowledge.load(deps.cwd(), null, deps.home());
     if (existsSync(outPath) && !opts.force) {
         deps.out(`${pc.yellow(WARN)}  ${pc.bold(outPath)} already exists.\nExisting values will be shown as defaults. Pass ${pc.bold('--force')} to start from scratch.`);
         deps.out('');

package/dist/engine/core/ci-ready.js

@@ -0,0 +1,112 @@
+/** Matches `canary analyze flaky`'s defaults: a 30-run window, 10% flake rate. */
+const FLAKY_WINDOW_RUNS = 30;
+const FLAKY_FAIL_RATE = 0.1;
+const INVENTORY = '.canary/test-inventory.json';
+const CRITICAL_AREAS = '.canary/critical-areas.json';
+function inventoryMissingReason() {
+    return `no ${INVENTORY}: nothing in canary produces it (the documented \`canary coverage\` command does not exist)`;
+}
+/** Per-test flake rate across the window, for tests that appeared at all. */
+function flakeRates(runs) {
+    const seen = new Map();
+    for (const run of runs) {
+        for (const t of run.tests ?? []) {
+            const s = seen.get(t.test_name) ?? { present: 0, flaky: 0 };
+            s.present += 1;
+            if (t.status === 'flaky')
+                s.flaky += 1;
+            seen.set(t.test_name, s);
+        }
+    }
+    const rates = new Map();
+    for (const [name, s] of seen) {
+        if (s.flaky > 0)
+            rates.set(name, s.flaky / s.present);
+    }
+    return rates;
+}
+function scoreFlakiness(runs, historyPath) {
+    const name = 'flakiness';
+    if (runs === null || runs.length === 0) {
+        return {
+            name,
+            verdict: 'skip',
+            reason: `no runs recorded in ${historyPath}`,
+        };
+    }
+    const window = runs.slice(-FLAKY_WINDOW_RUNS);
+    const rates = flakeRates(window);
+    if (rates.size === 0) {
+        return {
+            name,
+            verdict: 'pass',
+            reason: `0 flaky tests across ${window.length} run(s)`,
+        };
+    }
+    let worst = ['', 0];
+    for (const entry of rates)
+        if (entry[1] > worst[1])
+            worst = entry;
+    const pct = Math.round(worst[1] * 100);
+    const verdict = worst[1] >= FLAKY_FAIL_RATE ? 'fail' : 'warn';
+    return {
+        name,
+        verdict,
+        reason: `${rates.size} flaky test(s) across ${window.length} run(s); worst is ${worst[0]} at ${pct}%`,
+    };
+}
+function scoreInventoryCheck(name, hasInventory) {
+    const reason = hasInventory
+        ? `${INVENTORY} is present, but no documented schema exists to score it against`
+        : inventoryMissingReason();
+    return { name, verdict: 'skip', reason };
+}
+function scoreCriticalPaths(hasCriticalAreas, hasInventory) {
+    const name = 'critical-paths';
+    if (!hasCriticalAreas) {
+        return {
+            name,
+            verdict: 'skip',
+            reason: `no ${CRITICAL_AREAS}, and cross-referencing it also needs ${INVENTORY}`,
+        };
+    }
+    return {
+        name,
+        verdict: 'skip',
+        reason: scoreInventoryCheck(name, hasInventory).reason,
+    };
+}
+function scoreRuntime() {
+    return {
+        name: 'suite-runtime',
+        verdict: 'skip',
+        reason: 'the run-history store records no durations, so a p95 runtime cannot be computed',
+    };
+}
+function readinessVerdict(checks) {
+    const scored = checks.filter((c) => c.verdict !== 'skip');
+    if (scored.length === 0)
+        return 'abstained';
+    if (scored.some((c) => c.verdict === 'fail'))
+        return 'not-ready';
+    if (scored.length === checks.length &&
+        scored.every((c) => c.verdict === 'pass')) {
+        return 'ready';
+    }
+    return 'incomplete';
+}
+export function scoreCiReady(inputs) {
+    const checks = [
+        scoreInventoryCheck('coverage-depth', inputs.hasInventory),
+        scoreFlakiness(inputs.runs, inputs.historyPath),
+        scoreInventoryCheck('assertion-quality', inputs.hasInventory),
+        scoreCriticalPaths(inputs.hasCriticalAreas, inputs.hasInventory),
+        scoreRuntime(),
+    ];
+    return {
+        verdict: readinessVerdict(checks),
+        checked: checks.filter((c) => c.verdict !== 'skip').length,
+        checks,
+    };
+}
+//# sourceMappingURL=ci-ready.js.map

package/dist/engine/core/company-knowledge.js

@@ -448,6 +448,11 @@ function parseLayer(data, source) {
     const warns = [];
     const confluence_spaces = validateStrings(dictGet(data, 'confluence_spaces', []), 'confluence_spaces', (v) => _SPACE_OR_PROJECT_RE.test(v), (v) => v.toUpperCase(), warns);
     const jira_projects = validateStrings(dictGet(data, 'jira_projects', []), 'jira_projects', (v) => _SPACE_OR_PROJECT_RE.test(v), (v) => v.toUpperCase(), warns);
+    // `internal_doc_urls` cannot go through `validateStrings` (each entry needs
+    // the URL validator's own per-entry warning), but it is still a list field
+    // and must abstain out loud like one: a type-confused value used to be
+    // dropped in silence, so the operator's reference docs vanished from every
+    // generated prompt with nothing in `company-knowledge show` to say why.
     const rawUrls = dictGet(data, 'internal_doc_urls', []);
     const internal_doc_urls = [];
     if (Array.isArray(rawUrls)) {
@@ -459,6 +464,9 @@ function parseLayer(data, source) {
                 internal_doc_urls.push(validated);
         }
     }
+    else {
+        warns.push(`internal_doc_urls: expected list, got ${pyTypeName(rawUrls)} ${EMDASH} skipped`);
+    }
     const internal_domains = validateStrings(dictGet(data, 'internal_domains', []), 'internal_domains', (v) => _DOMAIN_RE.test(v), (v) => v.toLowerCase(), warns);
     const mcp_servers = validateStrings(dictGet(data, 'mcp_servers', []), 'mcp_servers', (v) => _MCP_SERVER_RE.test(v), null, warns);
     const claude_code_skills = validateStrings(dictGet(data, 'claude_code_skills', []), 'claude_code_skills', (v) => _SKILL_RE.test(v), (v) => v.toLowerCase(), warns);