canary-test-cli 7.0.0 → 7.1.0

package/dist/engine/analysis/cli.js

@@ -13,13 +13,13 @@
  *   - `json.dumps(x, indent=2)` -> {@link jsonIndent2} (byte-exact + ensure_ascii).
  *   - The report builders are byte-exact ports (Markdown), so the human-readable
  *     paths match the oracle exactly.
- *   - INTENTIONAL DEVIATION: the TS analysis engine (`engine.ts`) operates on the
- *     LOCAL NDJSON store only -- the JS Supabase SDK is async and the engine's
- *     query surface is synchronous. `--db-url` is still accepted (faithful CLI
- *     surface) but the TS port always reads the local store, exactly as the
- *     Python `isinstance(store, LocalHistoryStore)` read-path does for the
- *     spikes/common-failures commands. No Python analyze test exercises a remote
- *     store.
+ *   - `--db-url` / `CANARY_HISTORY_DB_URL` are HONOURED as of #711 (ADR 0013
+ *     Decision 4): the engine is async, so analyze selects its backend through
+ *     the shared `makeStore` factory like every other history consumer. Three
+ *     reports (spikes, common-failures, regression-candidates) are computed by
+ *     walking raw run records, which only the local backend exposes; against a
+ *     remote backend each names itself as unverifiable instead of rendering an
+ *     empty report. No Python analyze test exercised a remote store.
  *   - `area-health` accepts `--json` but ignores it -- faithful to the Python
  *     command, which never branches on `output_json`.
  */
@@ -29,29 +29,22 @@ import { Command, InvalidArgumentError, Option } from 'commander';
 import { jsonIndent2, normalizeUsageExit } from '../cli-common.js';
 import { gateOutcome } from '../core/gate-result.js';
 import { AnalysisEngine } from './engine.js';
-import { buildCommonFailuresReport, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
-import { NdjsonHistoryStore } from '../history/ndjson-store.js';
+import { buildCommonFailuresReport, buildFlakyTestsReport, buildRegressionCandidatesReport, buildFailureSpikesReport, } from './reports.js';
+import { makeStore } from '../history/store.js';
 const DEFAULT_HISTORY_PATH = 'test-results/reports/history-v2.jsonl';
 /** Process-backed defaults for production. */
 export function defaultAnalyzeDeps() {
-    const err = (s) => {
-        process.stderr.write(`${s}\n`);
-    };
     return {
         out: (s) => process.stdout.write(`${s}\n`),
-        err,
+        err: (s) => process.stderr.write(`${s}\n`),
         env: process.env,
-        // The ported analysis engine's query surface is synchronous, so it cannot
-        // drive the async Supabase store; analyze reads local NDJSON only. Python
-        // honors --db-url / CANARY_HISTORY_DB_URL via make_store, so warn (to stderr,
-        // not stdout -- keeps --json clean) rather than SILENTLY reading a different
-        // data source. Full remote support is deferred with the async engine port.
-        makeStore: (dbUrl) => {
-            if (dbUrl) {
-                err('note: --db-url is ignored by analyze; it reads local NDJSON only.');
-            }
-            return new NdjsonHistoryStore(DEFAULT_HISTORY_PATH);
-        },
+        // #711: analyze now goes through the shared factory, so --db-url and
+        // CANARY_HISTORY_DB_URL select the backend for real. Until the engine went
+        // async this could not be done -- the engine held the synchronous store
+        // contract -- and the flag was accepted with a printed apology instead.
+        // Where a remote backend cannot answer a given section, the command says so
+        // by name (see `cannotVerifyRawRecords`) rather than rendering an empty one.
+        makeStore: (dbUrl) => makeStore(dbUrl, DEFAULT_HISTORY_PATH),
     };
 }
 /**
@@ -72,8 +65,14 @@ export function defaultAnalyzeDeps() {
  *
  * Returns true when the caller should stop (the store was empty).
  */
-function abstainOnEmptyHistory(store, deps, json, what) {
-    if (store.countRuns() > 0)
+async function abstainOnEmptyHistory(store, deps, json, what) {
+    // #711: `countRuns` is an OPTIONAL capability, and a backend that cannot
+    // report its denominator has an UNKNOWN one, not a zero one (ADR 0013
+    // Decision 3). Abstaining here would fire on every remote query and mute the
+    // doctrine -- the katana lesson again, from the other direction.
+    if (!store.countRuns)
+        return false;
+    if ((await store.countRuns()) > 0)
         return false;
     const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
     const notice = `${outcome.summaryLine} No run history to analyze, so "${what}" is ` +
@@ -89,6 +88,45 @@ function abstainOnEmptyHistory(store, deps, json, what) {
     }
     return true;
 }
+/**
+ * The capability guard for the sections built on raw run records (#711).
+ *
+ * `spikes`, `common-failures` and `regression-candidates` are computed by
+ * walking whole run records rather than by any aggregate query, so they exist
+ * only where the backend implements the optional `readAll()`. The local NDJSON
+ * store does; the remote Supabase store does not.
+ *
+ * Against a backend without it these commands used to emit an empty report,
+ * which is indistinguishable on screen from a measured all-clear — the exact
+ * false-green shape #508 exists to kill, and one that only became reachable
+ * once `--db-url` started being honoured. So the command names the section it
+ * could not compute, and names the ones that DO work against this backend so
+ * the answer is actionable rather than just a refusal.
+ *
+ * Advisory, matching `abstainOnEmptyHistory`: the exit stays 0, the human path
+ * gets the notice instead of the report, and `--json` keeps stdout a parseable
+ * empty array with the notice on stderr.
+ *
+ * Returns true when the caller should stop.
+ */
+function cannotVerifyRawRecords(store, deps, json, what) {
+    if (store.readAll)
+        return false;
+    const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
+    const notice = `${outcome.summaryLine} cannot verify: this history backend does not ` +
+        `expose raw run records, so "${what}" is UNKNOWN rather than clean. ` +
+        `\`analyze flaky\` works against this backend today; drop --db-url ` +
+        `(and CANARY_HISTORY_DB_URL) to analyze the local ${DEFAULT_HISTORY_PATH} ` +
+        `instead.`;
+    if (json) {
+        deps.out(jsonIndent2([]));
+        deps.err(notice);
+    }
+    else {
+        deps.out(notice);
+    }
+    return true;
+}
 // --- unit-bearing flags (#673) -----------------------------------------------
 //
 // #670 put the unit in the NAME one layer down -- `analysis/reports.ts` takes
@@ -190,26 +228,29 @@ function writeArtifacts(artifacts, output) {
         writeFileSync(join(output, name), content, 'utf-8');
     }
 }
-function flakyCmd(opts, deps) {
+async function flakyCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
-    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'flake rate')) {
+    if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'flake rate')) {
         return;
     }
-    const rows = store.queryFlaky(opts.windowRuns, opts.suite ?? null, opts.minRatePct);
+    const rows = await store.queryFlaky(opts.windowRuns, opts.suite ?? null, opts.minRatePct);
     if (opts.json) {
         deps.out(jsonIndent2(rows));
     }
     else {
-        deps.out(buildFlakyReport(rows, opts.windowRuns, opts.minRatePct));
+        deps.out(buildFlakyTestsReport(rows, opts.windowRuns, opts.minRatePct));
     }
 }
-function spikesCmd(opts, deps) {
+async function spikesCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
-    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'failure spikes')) {
+    if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'failure spikes')) {
+        return;
+    }
+    if (cannotVerifyRawRecords(store, deps, opts.json === true, 'failure spikes')) {
         return;
     }
     const rows = [];
-    for (const r of store.readAll()) {
+    for (const r of await store.readAll()) {
         if (opts.since && (r.timestamp ?? '') < opts.since)
             continue;
         rows.push({
@@ -225,7 +266,7 @@ function spikesCmd(opts, deps) {
         deps.out(jsonIndent2(rows));
     }
     else {
-        deps.out(buildSpikesReport(rows, opts.deltaPp));
+        deps.out(buildFailureSpikesReport(rows, opts.deltaPp));
     }
 }
 function areaHealthCmd(opts, deps) {
@@ -243,13 +284,16 @@ function areaHealthCmd(opts, deps) {
         `result here would be a fiction. Use \`analyze digest\` for the reports ` +
         `that are wired, and track the area-health row set as unimplemented.`);
 }
-function commonFailuresCmd(opts, deps) {
+async function commonFailuresCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
-    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'common failures')) {
+    if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'common failures')) {
+        return;
+    }
+    if (cannotVerifyRawRecords(store, deps, opts.json === true, 'common failures')) {
         return;
     }
     const rows = [];
-    for (const record of store.readAll()) {
+    for (const record of await store.readAll()) {
         if (opts.since && (record.timestamp ?? '') < opts.since)
             continue;
         for (const t of record.tests ?? []) {
@@ -271,13 +315,16 @@ function commonFailuresCmd(opts, deps) {
         deps.out(buildCommonFailuresReport(rows, opts.minSuites));
     }
 }
-function regressionCandidatesCmd(opts, deps) {
+async function regressionCandidatesCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
-    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'regression candidates')) {
+    if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'regression candidates')) {
+        return;
+    }
+    if (cannotVerifyRawRecords(store, deps, opts.json === true, 'regression candidates')) {
         return;
     }
     const engine = new AnalysisEngine(store);
-    const candidates = engine.detectRegressionCandidates(null, opts.minGreen, opts.recentFailures);
+    const candidates = await engine.detectRegressionCandidates(null, opts.minGreen, opts.recentFailures);
     if (opts.json) {
         deps.out(jsonIndent2(candidates));
     }
@@ -285,13 +332,13 @@ function regressionCandidatesCmd(opts, deps) {
         deps.out(buildRegressionCandidatesReport(candidates));
     }
 }
-function digestCmd(opts, deps) {
+async function digestCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
-    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'fleet health')) {
+    if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'fleet health')) {
         return;
     }
     const engine = new AnalysisEngine(store);
-    const result = engine.run({
+    const result = await engine.run({
         windowRuns: opts.windowRuns,
         deltaPp: opts.deltaPp,
         weeks: opts.weeks,
@@ -299,6 +346,19 @@ function digestCmd(opts, deps) {
         suite: opts.suite ?? null,
     });
     writeArtifacts(result.artifacts, opts.output);
+    // Unlike the single-section commands, digest can partially succeed: the flaky
+    // leaderboard rides a contract method and is real even on a backend with no
+    // raw-record access. So it renders what it measured and names what it could
+    // not -- a digest silently missing three of its five sections would be the
+    // worst of both worlds. The notice rides stderr so --json and the Slack path
+    // stay byte-clean for their consumers.
+    if (result.degraded.length > 0) {
+        const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
+        deps.err(`${outcome.summaryLine} cannot verify: this history backend does not ` +
+            `expose raw run records, so ${result.degraded.join(', ')} ` +
+            `${result.degraded.length === 1 ? 'is' : 'are'} UNKNOWN rather than ` +
+            `clean in this digest. The flake leaderboard above is measured.`);
+    }
     if (opts.json) {
         deps.out(jsonIndent2({
             flaky_count: result.flaky.length,
@@ -328,7 +388,9 @@ function printSlack(flakyCount, regCount, deps) {
 // description: a blank one is how the silent unit survived this long, since the
 // value's meaning lived only in the report it eventually printed.
 const DB_URL_ENV = 'CANARY_HISTORY_DB_URL';
-const DB_URL_DESC = 'History store URL (accepted, but analyze reads local NDJSON).';
+const DB_URL_DESC = 'History store URL. Defaults to the local NDJSON store; a remote backend ' +
+    'cannot answer the spikes, common-failures or regression-candidates ' +
+    'reports, which say so rather than reporting zero.';
 const JSON_DESC = 'Emit the rows as JSON instead of a Markdown report.';
 const WINDOW_RUNS_DESC = 'Rolling window measured in RUNS, not days.';
 const MIN_RATE_PCT_DESC = 'Minimum flake rate to report, in PERCENT 0-100 (10 means 10%, not 0.1).';
@@ -355,8 +417,8 @@ export function createAnalyzeCommand(depsInit = {}) {
         .addOption(new Option('--min-rate <percent>', 'Deprecated alias for --min-rate-pct.').argParser(parsePercentScale('--min-rate', 'percent')))
         .addOption(new Option('--db-url <url>', DB_URL_DESC).env(DB_URL_ENV))
         .option('--json', JSON_DESC)
-        .action((opts, cmd) => {
-        flakyCmd(resolveUnitFlags(opts, cmd, deps, [WINDOW_ALIAS, MIN_RATE_ALIAS]), deps);
+        .action(async (opts, cmd) => {
+        await flakyCmd(resolveUnitFlags(opts, cmd, deps, [WINDOW_ALIAS, MIN_RATE_ALIAS]), deps);
     });
     program
         .command('spikes')
@@ -368,8 +430,8 @@ export function createAnalyzeCommand(depsInit = {}) {
         .addOption(new Option('--delta <points>', 'Deprecated alias for --delta-pp.').argParser(parsePercentScale('--delta', 'percentage points')))
         .addOption(new Option('--db-url <url>', DB_URL_DESC).env(DB_URL_ENV))
         .option('--json', JSON_DESC)
-        .action((opts, cmd) => {
-        spikesCmd(resolveUnitFlags(opts, cmd, deps, [DELTA_ALIAS]), deps);
+        .action(async (opts, cmd) => {
+        await spikesCmd(resolveUnitFlags(opts, cmd, deps, [DELTA_ALIAS]), deps);
     });
     program
         .command('area-health')
@@ -391,8 +453,8 @@ export function createAnalyzeCommand(depsInit = {}) {
         .argParser((v) => Number.parseInt(v, 10)))
         .addOption(new Option('--db-url <url>', DB_URL_DESC).env(DB_URL_ENV))
         .option('--json', JSON_DESC)
-        .action((opts) => {
-        commonFailuresCmd(opts, deps);
+        .action(async (opts) => {
+        await commonFailuresCmd(opts, deps);
     });
     program
         .command('regression-candidates')
@@ -405,8 +467,8 @@ export function createAnalyzeCommand(depsInit = {}) {
         .argParser((v) => Number.parseInt(v, 10)))
         .addOption(new Option('--db-url <url>', DB_URL_DESC).env(DB_URL_ENV))
         .option('--json', JSON_DESC)
-        .action((opts) => {
-        regressionCandidatesCmd(opts, deps);
+        .action(async (opts) => {
+        await regressionCandidatesCmd(opts, deps);
     });
     program
         .command('digest')
@@ -430,8 +492,8 @@ export function createAnalyzeCommand(depsInit = {}) {
         .option('--json', 'Emit per-section counts as JSON instead of the digest.')
         .option('--slack', 'Emit a short Slack-formatted summary.')
         .addOption(new Option('--db-url <url>', DB_URL_DESC).env(DB_URL_ENV))
-        .action((opts, cmd) => {
-        digestCmd(resolveUnitFlags(opts, cmd, deps, [WINDOW_ALIAS, DELTA_ALIAS]), deps);
+        .action(async (opts, cmd) => {
+        await digestCmd(resolveUnitFlags(opts, cmd, deps, [WINDOW_ALIAS, DELTA_ALIAS]), deps);
     });
     for (const sub of program.commands) {
         sub.exitOverride(normalizeUsageExit);

package/dist/engine/analysis/engine.js

@@ -9,7 +9,7 @@
  * `run()` (the Python engine initialises `area_rows = []` and never appends), so
  * the area-health artifact always renders the empty-data message.
  */
-import { buildAreaHealthReport, buildCommonFailuresReport, buildDigest, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
+import { buildAreaHealthReport, buildCommonFailuresReport, buildDigest, buildFlakyTestsReport, buildRegressionCandidatesReport, buildFailureSpikesReport, } from './reports.js';
 import { detectRegressions } from '../history/detector.js';
 import { def } from '../util/coalesce.js';
 function isReadable(store) {
@@ -26,12 +26,16 @@ function toCommonFailureRow(record, t) {
         error_text: t.error_text,
     };
 }
+/** Report-language names for the sections that need raw-record access. */
+const SECTION_SPIKES = 'failure spikes';
+const SECTION_COMMON_FAILURES = 'common failures';
+const SECTION_REGRESSIONS = 'regression candidates';
 export class AnalysisEngine {
     store;
     constructor(store) {
         this.store = store;
     }
-    run(opts = {}) {
+    async run(opts = {}) {
         const windowRuns = opts.windowRuns ?? 30;
         const deltaPp = opts.deltaPp ?? 20.0;
         const weeks = opts.weeks ?? 4;
@@ -40,11 +44,20 @@ export class AnalysisEngine {
         const minGreen = opts.minGreen ?? 5;
         const recentFailures = opts.recentFailures ?? 3;
         const suite = opts.suite ?? null;
-        const flaky = this.store.queryFlaky(windowRuns, suite, minFlakeRatePct);
-        const suitesToQuery = suite ? [suite] : this.discoverSuites();
+        const flaky = (await this.store.queryFlaky(windowRuns, suite, minFlakeRatePct));
+        // Sections that need raw-record access are UNKNOWN, not empty, on a backend
+        // that does not offer it (#711). `flaky` above and a suite-scoped `spikes`
+        // below ride real contract methods, so they stay measured either way.
+        const degraded = [];
+        const readable = isReadable(this.store);
+        const suitesToQuery = suite ? [suite] : await this.discoverSuites();
+        // With no explicit --suite, the suite list itself comes from raw records —
+        // so spikes is only degraded in that case, not whenever readAll is absent.
+        if (!readable && !suite)
+            degraded.push(SECTION_SPIKES);
         const spikesRows = [];
         for (const s of suitesToQuery) {
-            const summary = this.store.querySummary(s, windowRuns * 2);
+            const summary = await this.store.querySummary(s, windowRuns * 2);
             for (const row of def(summary.runs, [])) {
                 // query_summary rows omit suite; the spikes builder groups by it, so
                 // tag each pooled row with the suite it came from (matches Python).
@@ -59,8 +72,12 @@ export class AnalysisEngine {
         }
         // Faithful to Python: area rows are never populated by run().
         const areaRows = [];
-        const commonRows = this.queryCommonFailures(suite);
-        const regressionCandidates = this.detectRegressionCandidates(suite, minGreen, recentFailures);
+        const commonRows = await this.queryCommonFailures(suite);
+        if (!readable)
+            degraded.push(SECTION_COMMON_FAILURES);
+        const regressionCandidates = await this.detectRegressionCandidates(suite, minGreen, recentFailures);
+        if (!readable)
+            degraded.push(SECTION_REGRESSIONS);
         const digest = buildDigest({
             flaky,
             spikes: spikesRows,
@@ -73,8 +90,8 @@ export class AnalysisEngine {
             minSuites,
         });
         const artifacts = {
-            'flaky.md': buildFlakyReport(flaky, windowRuns, minFlakeRatePct),
-            'spikes.md': buildSpikesReport(spikesRows, deltaPp),
+            'flaky.md': buildFlakyTestsReport(flaky, windowRuns, minFlakeRatePct),
+            'spikes.md': buildFailureSpikesReport(spikesRows, deltaPp),
             'area-health.md': buildAreaHealthReport(areaRows, weeks),
             'common-failures.md': buildCommonFailuresReport(commonRows, minSuites),
             'regression-candidates.md': buildRegressionCandidatesReport(regressionCandidates),
@@ -88,23 +105,24 @@ export class AnalysisEngine {
             regressionCandidates,
             digestMd: digest,
             artifacts,
+            degraded,
         };
     }
-    discoverSuites() {
+    async discoverSuites() {
         if (!isReadable(this.store))
             return [];
         const suites = new Set();
-        for (const r of this.store.readAll()) {
+        for (const r of await this.store.readAll()) {
             if (r.suite)
                 suites.add(r.suite);
         }
         return [...suites];
     }
-    queryCommonFailures(suite) {
+    async queryCommonFailures(suite) {
         if (!isReadable(this.store))
             return [];
         const rows = [];
-        for (const record of this.store.readAll()) {
+        for (const record of await this.store.readAll()) {
             if (suite && record.suite !== suite)
                 continue;
             for (const t of def(record.tests, [])) {
@@ -114,11 +132,11 @@ export class AnalysisEngine {
         }
         return rows;
     }
-    detectRegressionCandidates(suite, minGreen, recentFailures) {
+    async detectRegressionCandidates(suite, minGreen, recentFailures) {
         if (!isReadable(this.store))
             return [];
         const testNames = new Set();
-        for (const record of this.store.readAll()) {
+        for (const record of await this.store.readAll()) {
             if (suite && record.suite !== suite)
                 continue;
             for (const t of def(record.tests, []))
@@ -126,7 +144,7 @@ export class AnalysisEngine {
         }
         const candidates = [];
         for (const name of testNames) {
-            const timeline = this.store.queryTimeline(name);
+            const timeline = await this.store.queryTimeline(name);
             const result = detectRegressions(timeline, minGreen, recentFailures);
             if (result.is_regression) {
                 candidates.push({

package/dist/engine/analysis/reports.js

@@ -16,7 +16,7 @@ export { round1 };
 // ---------------------------------------------------------------------------
 // Flaky report
 // ---------------------------------------------------------------------------
-export function buildFlakyReport(rows, windowRuns, minRatePct, limit = 20) {
+export function buildFlakyTestsReport(rows, windowRuns, minRatePct, limit = 20) {
     if (rows.length === 0) {
         return `No tests above ${pyFloat(minRatePct)}% flake rate in the last ${windowRuns} runs.\n`;
     }
@@ -75,7 +75,7 @@ function detectSpikes(rows, deltaPp) {
     }
     return spikes;
 }
-export function buildSpikesReport(rows, deltaPp) {
+export function buildFailureSpikesReport(rows, deltaPp) {
     if (rows.length === 0) {
         return 'No run data available for spike detection.\n';
     }
@@ -210,8 +210,9 @@ function regressionRow(r) {
 export function buildDigest(args) {
     const sections = [
         '# Fleet Health Digest\n',
-        '## Flaky Tests\n\n' + buildFlakyReport(args.flaky, args.windowRuns, 10.0),
-        '## Spikes\n\n' + buildSpikesReport(args.spikes, args.deltaPp),
+        '## Flaky Tests\n\n' +
+            buildFlakyTestsReport(args.flaky, args.windowRuns, 10.0),
+        '## Spikes\n\n' + buildFailureSpikesReport(args.spikes, args.deltaPp),
         '## Area Health\n\n' + buildAreaHealthReport(args.areaHealth, args.weeks),
         '## Common Failures\n\n' +
             buildCommonFailuresReport(args.commonFailures, args.minSuites),