canary-test-cli 6.4.0 → 6.5.0

package/dist/doctor.d.ts

@@ -1,5 +1,8 @@
 import type { CommandDeps } from './overlay-commands.js';
+import { type EngineCheckDeps } from './engine-checks.js';
 import { type CommandRunner, type UrlProbe } from './doctor-manifest.js';
+import { EXIT_ABSTAINED, type SkipEntry } from './gate-result.js';
+export { EXIT_ABSTAINED };
 /** Outcome of a single doctor check. */
 export type CheckStatus = 'pass' | 'fail' | 'skip' | 'info';
 /** A single doctor check result, rendered as one output line. */
@@ -21,6 +24,17 @@ export interface DoctorDeps extends CommandDeps {
     probeUrl?: UrlProbe;
     runCommand?: CommandRunner;
     timeoutMs?: number;
+    /**
+     * Override the engine check set. A test seam only: it is the one input a
+     * hermetic run cannot control (engine checks read the real environment), and
+     * the abstention fixtures need a run whose denominator is exactly zero.
+     */
+    runEngineChecks?: (deps: EngineCheckDeps) => Promise<CheckResult[]>;
+}
+/** One printed section: a header and its check results. */
+export interface CheckGroup {
+    header: string;
+    results: CheckResult[];
 }
 /**
  * The `canary doctor --json` machine contract (issue #318). Canary-owned and
@@ -46,6 +60,12 @@ export interface JsonReport {
     allPassed: boolean;
     /** Non-fatal advisories (e.g. an unknown `--audience`); empty when none. */
     warnings: string[];
+    /** Checks that produced a verdict (`pass + fail`) -- the denominator (#508). */
+    checked: number;
+    /** Skipped checks, always reported; never folded into `allPassed` (D7). */
+    skipped: SkipEntry[];
+    /** True when nothing was verified: `allPassed` is false and the exit is 3. */
+    abstained: boolean;
 }
 /** `--json` requests machine output on stdout instead of the human report. */
 export declare function parseJsonFlag(args: readonly string[]): boolean;
@@ -59,9 +79,37 @@ export declare function parseJsonFlag(args: readonly string[]): boolean;
  * guess why their filter matched nothing.
  */
 export declare function unknownAudienceHint(audience: string | null, known: readonly string[]): string | null;
+/** Doctor's denominator, and the decision that follows from it (#508 D7). */
+export interface DoctorSummary {
+    /** Checks that actually produced a verdict: `pass + fail`. */
+    checked: number;
+    passed: number;
+    failed: number;
+    /** Every skipped check, always visible, never folded into `passed`. */
+    skipped: SkipEntry[];
+    abstained: boolean;
+    exitCode: number;
+    summaryLine: string;
+}
+/**
+ * Compute doctor's denominator and summary line (#508 Wave 3, D7 / #505).
+ *
+ * Doctor used to count only failures, so a run in which EVERY check was
+ * skipped printed `All checks passed.` and exited 0 -- the doctrine violation
+ * that started #508. Here the denominator is explicit: `info` results are not
+ * verifications (they report context, not evidence), so a run with nothing but
+ * skips and info abstains.
+ *
+ * The decision (exit code + `abstained`) comes from `gateOutcome` and is never
+ * re-derived here; only the failure line keeps doctor's own vocabulary
+ * ("check(s) failed" rather than "finding(s)"), rendered with the helper's own
+ * skip suffix so the two can never drift.
+ */
+export declare function summarizeChecks(groups: readonly CheckGroup[]): DoctorSummary;
 /**
- * Run `canary doctor`. Returns a process exit code: 0 when every check passed
- * or was skipped/info, non-zero when any check failed. A malformed manifest for
- * one overlay never blocks engine checks or other overlays.
+ * Run `canary doctor`. Returns a process exit code: 0 when at least one check
+ * ran and none failed, 1 when any check failed, and `EXIT_ABSTAINED` (3) when
+ * nothing was actually verified (#508). A malformed manifest for one overlay
+ * never blocks engine checks or other overlays.
  */
 export declare function runDoctor(args: readonly string[], deps?: DoctorDeps): Promise<number>;

package/dist/doctor.js

@@ -33,8 +33,10 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.EXIT_ABSTAINED = void 0;
 exports.parseJsonFlag = parseJsonFlag;
 exports.unknownAudienceHint = unknownAudienceHint;
+exports.summarizeChecks = summarizeChecks;
 exports.runDoctor = runDoctor;
 /**
  * `canary doctor` — environment self-check (Phase 2).
@@ -57,6 +59,8 @@ const os = __importStar(require("node:os"));
 const engine_checks_js_1 = require("./engine-checks.js");
 const doctor_manifest_js_1 = require("./doctor-manifest.js");
 const registry = __importStar(require("./overlays-registry.js"));
+const gate_result_js_1 = require("./gate-result.js");
+Object.defineProperty(exports, "EXIT_ABSTAINED", { enumerable: true, get: function () { return gate_result_js_1.EXIT_ABSTAINED; } });
 const SYMBOL = {
     pass: '✓',
     fail: '✗',
@@ -141,9 +145,60 @@ async function overlayResults(entry, deps, audience) {
     return { group: { header, results }, loadedChecks: load.checks };
 }
 /**
- * Run `canary doctor`. Returns a process exit code: 0 when every check passed
- * or was skipped/info, non-zero when any check failed. A malformed manifest for
- * one overlay never blocks engine checks or other overlays.
+ * Compute doctor's denominator and summary line (#508 Wave 3, D7 / #505).
+ *
+ * Doctor used to count only failures, so a run in which EVERY check was
+ * skipped printed `All checks passed.` and exited 0 -- the doctrine violation
+ * that started #508. Here the denominator is explicit: `info` results are not
+ * verifications (they report context, not evidence), so a run with nothing but
+ * skips and info abstains.
+ *
+ * The decision (exit code + `abstained`) comes from `gateOutcome` and is never
+ * re-derived here; only the failure line keeps doctor's own vocabulary
+ * ("check(s) failed" rather than "finding(s)"), rendered with the helper's own
+ * skip suffix so the two can never drift.
+ */
+function summarizeChecks(groups) {
+    const results = groups.flatMap((g) => g.results);
+    const failures = results.filter((r) => r.status === 'fail');
+    const passed = results.filter((r) => r.status === 'pass').length;
+    const skipped = results
+        .filter((r) => r.status === 'skip')
+        .map((r) => ({ name: r.id, reason: r.remedy ?? r.label }));
+    const checked = passed + failures.length;
+    const outcome = (0, gate_result_js_1.gateOutcome)({ checked, findings: failures, skipped }, 'gate');
+    return {
+        checked,
+        passed,
+        failed: failures.length,
+        skipped,
+        abstained: outcome.abstained,
+        exitCode: outcome.exitCode,
+        summaryLine: failures.length > 0
+            ? `${failures.length} check(s) failed${(0, gate_result_js_1.skippedSuffix)(skipped)}`
+            : outcome.summaryLine,
+    };
+}
+/**
+ * Remediation for an abstained run: why the denominator collapsed, and the
+ * first fix step. Required of every abstaining surface (spec: "an abstaining
+ * surface must say _why_ ... and the first fix step").
+ */
+function abstentionRemedy(summary) {
+    return summary.skipped.length > 0
+        ? 'Every registered check was skipped or informational, so doctor verified ' +
+            'nothing. Grant command-check consent (re-run `canary overlay add ' +
+            '<name> --yes`) or install an overlay whose checks apply here, then ' +
+            're-run.'
+        : 'No check was registered, so doctor verified nothing. Install an ' +
+            'overlay that ships a `.canary/doctor.json` (`canary overlay add ' +
+            '<source>`), then re-run.';
+}
+/**
+ * Run `canary doctor`. Returns a process exit code: 0 when at least one check
+ * ran and none failed, 1 when any check failed, and `EXIT_ABSTAINED` (3) when
+ * nothing was actually verified (#508). A malformed manifest for one overlay
+ * never blocks engine checks or other overlays.
  */
 async function runDoctor(args, deps = {}) {
     const out = deps.out ?? process.stdout;
@@ -157,8 +212,9 @@ async function runDoctor(args, deps = {}) {
         getLatestVersion: deps.getLatestVersion,
         timeoutMs: deps.timeoutMs,
     };
+    const engineChecks = deps.runEngineChecks ?? engine_checks_js_1.runEngineChecks;
     const groups = [
-        { header: 'Engine', results: await (0, engine_checks_js_1.runEngineChecks)(engineDeps) },
+        { header: 'Engine', results: await engineChecks(engineDeps) },
     ];
     let reg;
     try {
@@ -177,7 +233,10 @@ async function runDoctor(args, deps = {}) {
     // tell them the valid vocabulary instead of silently filtering to only
     // the audience-less checks and leaving them to wonder why.
     const audienceHint = unknownAudienceHint(audience, (0, doctor_manifest_js_1.collectAudiences)(allChecks));
-    const failures = groups.reduce((n, g) => n + g.results.filter((r) => r.status === 'fail').length, 0);
+    // #508: the denominator, not just the failure count. `summarizeChecks` is
+    // the only path to a summary line and an exit code, so doctor structurally
+    // cannot print a bare success over zero verified checks.
+    const summary = summarizeChecks(groups);
     // Issue #318: `--json` emits the canary-owned machine contract instead of
     // the human report — nothing else is written to stdout, so the whole stream
     // parses as one JSON object.
@@ -191,11 +250,16 @@ async function runDoctor(args, deps = {}) {
                 ...(r.remedy !== undefined ? { remedy: r.remedy } : {}),
                 group: g.header,
             }))),
-            allPassed: failures === 0,
+            // #508: an abstained run is NOT "all passed" -- zero verified checks is
+            // an absent measurement, never a green.
+            allPassed: summary.failed === 0 && !summary.abstained,
             warnings: audienceHint ? [audienceHint] : [],
+            checked: summary.checked,
+            skipped: summary.skipped,
+            abstained: summary.abstained,
         };
         out.write(`${JSON.stringify(report, null, 2)}\n`);
-        return failures === 0 ? 0 : 1;
+        return summary.exitCode;
     }
     out.write('canary doctor\n');
     if (audienceHint) {
@@ -207,6 +271,9 @@ async function runDoctor(args, deps = {}) {
             out.write(renderCheck(result));
         }
     }
-    out.write(`\n${failures === 0 ? 'All checks passed.' : `${failures} check(s) failed.`}\n`);
-    return failures === 0 ? 0 : 1;
+    out.write(`\n${summary.summaryLine}\n`);
+    if (summary.abstained) {
+        out.write(`  ${abstentionRemedy(summary)}\n`);
+    }
+    return summary.exitCode;
 }

package/dist/engine/analysis/cli.js

@@ -27,8 +27,9 @@ import { mkdirSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { Command, Option } from 'commander';
 import { jsonIndent2, normalizeUsageExit } from '../cli-common.js';
+import { gateOutcome } from '../core/gate-result.js';
 import { AnalysisEngine } from './engine.js';
-import { buildAreaHealthReport, buildCommonFailuresReport, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
+import { buildCommonFailuresReport, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
 import { NdjsonHistoryStore } from '../history/ndjson-store.js';
 const DEFAULT_HISTORY_PATH = 'test-results/reports/history-v2.jsonl';
 /** Process-backed defaults for production. */
@@ -53,6 +54,41 @@ export function defaultAnalyzeDeps() {
         },
     };
 }
+/**
+ * The shared denominator guard for every history-backed analyze report
+ * (#508 Wave 4a).
+ *
+ * The denominator is the number of RUNS in the store, never the number of
+ * result rows: zero flaky rows across 500 runs is a genuine clean fleet, while
+ * zero rows across zero runs is an absent measurement. Only `countRuns()`
+ * separates them -- keying off `rows.length` would abstain on every healthy
+ * fleet, which is the fastest way to get a doctrine muted (the katana lesson).
+ *
+ * Advisory (D3): the exit stays 0. On the human path the abstention line
+ * REPLACES the all-clear report; on `--json` the payload is a bare array with
+ * nowhere to put an `abstained` field, so stdout is left byte-identical and the
+ * notice rides stderr -- the same split `analyze` already uses for its
+ * `--db-url` note.
+ *
+ * Returns true when the caller should stop (the store was empty).
+ */
+function abstainOnEmptyHistory(store, deps, json, what) {
+    if (store.countRuns() > 0)
+        return false;
+    const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
+    const notice = `${outcome.summaryLine} No run history to analyze, so "${what}" is ` +
+        `unknown rather than clean. Record runs first ` +
+        `(\`canary history push\`, or a reporter that writes ` +
+        `${DEFAULT_HISTORY_PATH}), then re-run.`;
+    if (json) {
+        deps.out(jsonIndent2([]));
+        deps.err(notice);
+    }
+    else {
+        deps.out(notice);
+    }
+    return true;
+}
 function writeArtifacts(artifacts, output) {
     mkdirSync(output, { recursive: true });
     for (const [name, content] of Object.entries(artifacts)) {
@@ -61,6 +97,9 @@ function writeArtifacts(artifacts, output) {
 }
 function flakyCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
+    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'flake rate')) {
+        return;
+    }
     const rows = store.queryFlaky(opts.window, opts.suite ?? null, opts.minRate);
     if (opts.json) {
         deps.out(jsonIndent2(rows));
@@ -71,6 +110,9 @@ function flakyCmd(opts, deps) {
 }
 function spikesCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
+    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'failure spikes')) {
+        return;
+    }
     const rows = [];
     for (const r of store.readAll()) {
         if (opts.since && (r.timestamp ?? '') < opts.since)
@@ -92,12 +134,25 @@ function spikesCmd(opts, deps) {
     }
 }
 function areaHealthCmd(opts, deps) {
-    // Faithful to Python: always builds from an empty row set and never branches
-    // on --json.
-    deps.out(buildAreaHealthReport([], opts.weeks));
+    // #508 Wave 4a: this command builds its report from a HARDCODED empty row
+    // set (faithful to the Python original, which did the same and never branched
+    // on --json). Its denominator is therefore UNCONDITIONALLY zero -- with a
+    // thousand runs recorded it still renders "no area health data", which reads
+    // as a measured all-clear and is not one. So it always abstains, whatever the
+    // store holds. Wiring a real row set is a separate scope call: it changes the
+    // port's contract, and #515 deferred it for exactly that reason.
+    void opts;
+    const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
+    deps.out(`${outcome.summaryLine} \`analyze area-health\` computes no rows in this ` +
+        `build -- its report is a template, not a measurement, so a clean-looking ` +
+        `result here would be a fiction. Use \`analyze digest\` for the reports ` +
+        `that are wired, and track the area-health row set as unimplemented.`);
 }
 function commonFailuresCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
+    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'common failures')) {
+        return;
+    }
     const rows = [];
     for (const record of store.readAll()) {
         if (opts.since && (record.timestamp ?? '') < opts.since)
@@ -122,7 +177,11 @@ function commonFailuresCmd(opts, deps) {
     }
 }
 function regressionCandidatesCmd(opts, deps) {
-    const engine = new AnalysisEngine(deps.makeStore(opts.dbUrl));
+    const store = deps.makeStore(opts.dbUrl);
+    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'regression candidates')) {
+        return;
+    }
+    const engine = new AnalysisEngine(store);
     const candidates = engine.detectRegressionCandidates(null, opts.minGreen, opts.recentFailures);
     if (opts.json) {
         deps.out(jsonIndent2(candidates));
@@ -132,7 +191,11 @@ function regressionCandidatesCmd(opts, deps) {
     }
 }
 function digestCmd(opts, deps) {
-    const engine = new AnalysisEngine(deps.makeStore(opts.dbUrl));
+    const store = deps.makeStore(opts.dbUrl);
+    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'fleet health')) {
+        return;
+    }
+    const engine = new AnalysisEngine(store);
     const result = engine.run({
         window: opts.window,
         delta: opts.delta,

package/dist/engine/cli-commands.js

@@ -14,6 +14,7 @@ import { existsSync, readdirSync, readFileSync, statSync, writeFileSync, } from
 import { basename, join, resolve } from 'node:path';
 import pc from 'picocolors';
 import { CliExit, jsonIndent2 } from './cli-common.js';
+import { gateOutcome } from './core/gate-result.js';
 import { ckInitCmd } from './company-knowledge-cli.js';
 import { extractFrameworkHint } from './core/classifier.js';
 import { VALID_CATEGORIES, buildFeedback } from './core/feedback.js';
@@ -393,8 +394,39 @@ function findingPayload(f) {
         suggestion: f.suggestion,
     };
 }
+/**
+ * The denominator guard shared by the file-scanning gates (#508 Wave 4a).
+ *
+ * `review-test` and `flake-check` used to render a green all-clear whenever
+ * their finding list was empty -- indistinguishable from a run that scanned a
+ * directory matching zero test files. That is the #503 shape: a gate that
+ * verified nothing reporting a pass. Both are GATES (they carry an exit-code
+ * contract), so a collapsed denominator exits 3.
+ *
+ * Returns without throwing when at least one file was collected; the caller's
+ * normal rendering continues. `--json` keeps a parseable array on stdout -- only
+ * the exit code and the stderr notice carry the abstention.
+ */
+function abstainOnZeroFiles(files, path, deps, json) {
+    if (files.length > 0)
+        return;
+    const outcome = gateOutcome({ checked: 0, findings: [] }, 'gate');
+    const remedy = `No test file matched under ${path} (looked for test_*.py, ` +
+        `*.spec.ts/js, *.test.ts/js). Point at a directory that holds tests, ` +
+        `or pass a single file directly.`;
+    if (json) {
+        deps.out(jsonIndent2([]));
+        deps.err(`${outcome.summaryLine} ${remedy}`);
+    }
+    else {
+        deps.out(pc.bold(pc.yellow(outcome.summaryLine)));
+        deps.out(`  ${remedy}`);
+    }
+    throw new CliExit(outcome.exitCode);
+}
 export function reviewTestCmd(path, opts, deps) {
     const files = isDir(path) ? collectTestFiles(path) : [path];
+    abstainOnZeroFiles(files, path, deps, opts.json === true);
     const linter = deps.makeLinter();
     const allFindings = [];
     for (const f of files)
@@ -431,6 +463,7 @@ export function reviewTestCmd(path, opts, deps) {
 }
 export function flakeCheckCmd(path, opts, deps) {
     const files = isDir(path) ? collectTestFiles(path) : [path];
+    abstainOnZeroFiles(files, path, deps, opts.json === true);
     const linter = deps.makeLinter();
     const allFindings = [];
     for (const f of files)

package/dist/engine/core/gate-result.js

@@ -25,8 +25,15 @@ const EMDASH = '\u{2014}'; // em dash
 // C0 controls (incl. \n, ESC) and DEL: a skip name must never be able to
 // forge output lines or smuggle ANSI sequences into the summary.
 const CONTROL_CHARS = /[\u0000-\u001F\u007F]/g;
-/** D7: skipped entries render in EVERY summary line. */
-function skippedSuffix(skipped) {
+/**
+ * D7: skipped entries render in EVERY summary line.
+ *
+ * Exported so a surface with its own failure vocabulary (doctor says
+ * "check(s) failed", not "finding(s)") can render the identical skip suffix
+ * instead of re-deriving the format -- the decision still comes from
+ * {@link gateOutcome}, only the noun differs.
+ */
+export function skippedSuffix(skipped) {
     if (!skipped || skipped.length === 0)
         return '';
     const names = skipped