canary-test-cli 6.8.1 → 7.1.0

package/dist/engine/analysis/cli.js

@@ -13,45 +13,38 @@
  *   - `json.dumps(x, indent=2)` -> {@link jsonIndent2} (byte-exact + ensure_ascii).
  *   - The report builders are byte-exact ports (Markdown), so the human-readable
  *     paths match the oracle exactly.
- *   - INTENTIONAL DEVIATION: the TS analysis engine (`engine.ts`) operates on the
- *     LOCAL NDJSON store only -- the JS Supabase SDK is async and the engine's
- *     query surface is synchronous. `--db-url` is still accepted (faithful CLI
- *     surface) but the TS port always reads the local store, exactly as the
- *     Python `isinstance(store, LocalHistoryStore)` read-path does for the
- *     spikes/common-failures commands. No Python analyze test exercises a remote
- *     store.
+ *   - `--db-url` / `CANARY_HISTORY_DB_URL` are HONOURED as of #711 (ADR 0013
+ *     Decision 4): the engine is async, so analyze selects its backend through
+ *     the shared `makeStore` factory like every other history consumer. Three
+ *     reports (spikes, common-failures, regression-candidates) are computed by
+ *     walking raw run records, which only the local backend exposes; against a
+ *     remote backend each names itself as unverifiable instead of rendering an
+ *     empty report. No Python analyze test exercised a remote store.
  *   - `area-health` accepts `--json` but ignores it -- faithful to the Python
  *     command, which never branches on `output_json`.
  */
 import { mkdirSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
-import { Command, Option } from 'commander';
+import { Command, InvalidArgumentError, Option } from 'commander';
 import { jsonIndent2, normalizeUsageExit } from '../cli-common.js';
 import { gateOutcome } from '../core/gate-result.js';
 import { AnalysisEngine } from './engine.js';
-import { buildCommonFailuresReport, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
-import { NdjsonHistoryStore } from '../history/ndjson-store.js';
+import { buildCommonFailuresReport, buildFlakyTestsReport, buildRegressionCandidatesReport, buildFailureSpikesReport, } from './reports.js';
+import { makeStore } from '../history/store.js';
 const DEFAULT_HISTORY_PATH = 'test-results/reports/history-v2.jsonl';
 /** Process-backed defaults for production. */
 export function defaultAnalyzeDeps() {
-    const err = (s) => {
-        process.stderr.write(`${s}\n`);
-    };
     return {
         out: (s) => process.stdout.write(`${s}\n`),
-        err,
+        err: (s) => process.stderr.write(`${s}\n`),
         env: process.env,
-        // The ported analysis engine's query surface is synchronous, so it cannot
-        // drive the async Supabase store; analyze reads local NDJSON only. Python
-        // honors --db-url / CANARY_HISTORY_DB_URL via make_store, so warn (to stderr,
-        // not stdout -- keeps --json clean) rather than SILENTLY reading a different
-        // data source. Full remote support is deferred with the async engine port.
-        makeStore: (dbUrl) => {
-            if (dbUrl) {
-                err('note: --db-url is ignored by analyze; it reads local NDJSON only.');
-            }
-            return new NdjsonHistoryStore(DEFAULT_HISTORY_PATH);
-        },
+        // #711: analyze now goes through the shared factory, so --db-url and
+        // CANARY_HISTORY_DB_URL select the backend for real. Until the engine went
+        // async this could not be done -- the engine held the synchronous store
+        // contract -- and the flag was accepted with a printed apology instead.
+        // Where a remote backend cannot answer a given section, the command says so
+        // by name (see `cannotVerifyRawRecords`) rather than rendering an empty one.
+        makeStore: (dbUrl) => makeStore(dbUrl, DEFAULT_HISTORY_PATH),
     };
 }
 /**
@@ -72,8 +65,14 @@ export function defaultAnalyzeDeps() {
  *
  * Returns true when the caller should stop (the store was empty).
  */
-function abstainOnEmptyHistory(store, deps, json, what) {
-    if (store.countRuns() > 0)
+async function abstainOnEmptyHistory(store, deps, json, what) {
+    // #711: `countRuns` is an OPTIONAL capability, and a backend that cannot
+    // report its denominator has an UNKNOWN one, not a zero one (ADR 0013
+    // Decision 3). Abstaining here would fire on every remote query and mute the
+    // doctrine -- the katana lesson again, from the other direction.
+    if (!store.countRuns)
+        return false;
+    if ((await store.countRuns()) > 0)
         return false;
     const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
     const notice = `${outcome.summaryLine} No run history to analyze, so "${what}" is ` +
@@ -89,32 +88,169 @@ function abstainOnEmptyHistory(store, deps, json, what) {
     }
     return true;
 }
+/**
+ * The capability guard for the sections built on raw run records (#711).
+ *
+ * `spikes`, `common-failures` and `regression-candidates` are computed by
+ * walking whole run records rather than by any aggregate query, so they exist
+ * only where the backend implements the optional `readAll()`. The local NDJSON
+ * store does; the remote Supabase store does not.
+ *
+ * Against a backend without it these commands used to emit an empty report,
+ * which is indistinguishable on screen from a measured all-clear — the exact
+ * false-green shape #508 exists to kill, and one that only became reachable
+ * once `--db-url` started being honoured. So the command names the section it
+ * could not compute, and names the ones that DO work against this backend so
+ * the answer is actionable rather than just a refusal.
+ *
+ * Advisory, matching `abstainOnEmptyHistory`: the exit stays 0, the human path
+ * gets the notice instead of the report, and `--json` keeps stdout a parseable
+ * empty array with the notice on stderr.
+ *
+ * Returns true when the caller should stop.
+ */
+function cannotVerifyRawRecords(store, deps, json, what) {
+    if (store.readAll)
+        return false;
+    const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
+    const notice = `${outcome.summaryLine} cannot verify: this history backend does not ` +
+        `expose raw run records, so "${what}" is UNKNOWN rather than clean. ` +
+        `\`analyze flaky\` works against this backend today; drop --db-url ` +
+        `(and CANARY_HISTORY_DB_URL) to analyze the local ${DEFAULT_HISTORY_PATH} ` +
+        `instead.`;
+    if (json) {
+        deps.out(jsonIndent2([]));
+        deps.err(notice);
+    }
+    else {
+        deps.out(notice);
+    }
+    return true;
+}
+// --- unit-bearing flags (#673) -----------------------------------------------
+//
+// #670 put the unit in the NAME one layer down -- `analysis/reports.ts` takes
+// `windowRuns`, `deltaPp`, `minRatePct` -- and stopped at the CLI boundary
+// because renaming a flag is user-visible. The flags carried the same silence
+// they always had: the report prints `window: 30 runs`, `20.0pp increase` and
+// `>= 10.0%`, so the unit only ever arrived AFTER the user had guessed. This
+// section closes that gap using the same convention: `<measure><Unit>` in
+// camelCase becomes `--<measure>-<unit>` on the command line.
+//
+// `--delta` is the sharpest case. It is a percentage-POINT threshold that looks
+// like a percentage, so `--delta 20` against a failure rate moving 5% -> 6% is
+// the difference between firing and staying silent, with nothing on screen to
+// say which reading applied.
+/** One decimal or integer, no sign: the only shape these thresholds accept. */
+const UNSIGNED_NUMBER = /^(\d+(\.\d+)?|\.\d+)$/;
+/**
+ * A whole count of runs, >= 1.
+ *
+ * The bare `Number.parseInt` this replaces mapped `--window seven` to `NaN`,
+ * which the query layer happily compared against and matched nothing -- a
+ * silent abstention wearing a clean fleet's clothes. A value that cannot mean
+ * what the flag says is a usage error instead.
+ */
+function parseRuns(flag) {
+    return (raw) => {
+        if (!/^\d+$/.test(raw.trim()) || Number.parseInt(raw, 10) < 1) {
+            throw new InvalidArgumentError(`${flag} takes a whole number of RUNS, at least 1 (e.g. 30); ` +
+                `got "${raw}".`);
+        }
+        return Number.parseInt(raw, 10);
+    };
+}
+/**
+ * A 0-100 value on a percentage scale (`percent` for a rate, `percentage
+ * points` for a delta). Rejecting >100 is what catches the fraction habit from
+ * the other direction; `0.1` is still legal (a tenth of a percent), so the help
+ * text carries that half of the warning.
+ */
+function parsePercentScale(flag, unit) {
+    return (raw) => {
+        const n = Number.parseFloat(raw);
+        if (!UNSIGNED_NUMBER.test(raw.trim()) || !Number.isFinite(n) || n > 100) {
+            throw new InvalidArgumentError(`${flag} takes ${unit} between 0 and 100 (e.g. 10 means ten ${unit}, ` +
+                `not 0.1); got "${raw}".`);
+        }
+        return n;
+    };
+}
+const WINDOW_ALIAS = {
+    canonical: 'windowRuns',
+    legacy: 'window',
+    canonicalFlag: '--window-runs',
+    legacyFlag: '--window',
+    unit: 'a count of runs, not days',
+};
+const DELTA_ALIAS = {
+    canonical: 'deltaPp',
+    legacy: 'delta',
+    canonicalFlag: '--delta-pp',
+    legacyFlag: '--delta',
+    unit: 'percentage points, not percent',
+};
+const MIN_RATE_ALIAS = {
+    canonical: 'minRatePct',
+    legacy: 'minRate',
+    canonicalFlag: '--min-rate-pct',
+    legacyFlag: '--min-rate',
+    unit: 'a percentage, 0-100',
+};
+/**
+ * Fold any deprecated spellings the user typed onto their canonical keys.
+ *
+ * The canonical flag wins when BOTH are given -- the alias is the legacy path,
+ * so an explicit new-style flag is the more deliberate statement of intent.
+ * `getOptionValueSource` is what separates "the user typed it" from "commander
+ * filled in the default"; comparing values would let a canonical default that
+ * happens to equal the alias silently discard the alias.
+ */
+function resolveUnitFlags(opts, cmd, deps, aliases) {
+    // Commander hands option bags back as plain objects keyed by camelCase flag
+    // name; the per-command interfaces describe them but carry no index
+    // signature, so the alias keys are reached through one local widening.
+    const merged = { ...opts };
+    for (const alias of aliases) {
+        if (merged[alias.legacy] === undefined)
+            continue;
+        deps.err(`note: ${alias.legacyFlag} is deprecated; use ${alias.canonicalFlag} ` +
+            `(the value is ${alias.unit}).`);
+        if (cmd.getOptionValueSource(alias.canonical) !== 'cli') {
+            merged[alias.canonical] = merged[alias.legacy];
+        }
+    }
+    return merged;
+}
 function writeArtifacts(artifacts, output) {
     mkdirSync(output, { recursive: true });
     for (const [name, content] of Object.entries(artifacts)) {
         writeFileSync(join(output, name), content, 'utf-8');
     }
 }
-function flakyCmd(opts, deps) {
+async function flakyCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
-    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'flake rate')) {
+    if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'flake rate')) {
         return;
     }
-    const rows = store.queryFlaky(opts.window, opts.suite ?? null, opts.minRate);
+    const rows = await store.queryFlaky(opts.windowRuns, opts.suite ?? null, opts.minRatePct);
     if (opts.json) {
         deps.out(jsonIndent2(rows));
     }
     else {
-        deps.out(buildFlakyReport(rows, opts.window, opts.minRate));
+        deps.out(buildFlakyTestsReport(rows, opts.windowRuns, opts.minRatePct));
     }
 }
-function spikesCmd(opts, deps) {
+async function spikesCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
-    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'failure spikes')) {
+    if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'failure spikes')) {
+        return;
+    }
+    if (cannotVerifyRawRecords(store, deps, opts.json === true, 'failure spikes')) {
         return;
     }
     const rows = [];
-    for (const r of store.readAll()) {
+    for (const r of await store.readAll()) {
         if (opts.since && (r.timestamp ?? '') < opts.since)
             continue;
         rows.push({
@@ -130,7 +266,7 @@ function spikesCmd(opts, deps) {
         deps.out(jsonIndent2(rows));
     }
     else {
-        deps.out(buildSpikesReport(rows, opts.delta));
+        deps.out(buildFailureSpikesReport(rows, opts.deltaPp));
     }
 }
 function areaHealthCmd(opts, deps) {
@@ -148,13 +284,16 @@ function areaHealthCmd(opts, deps) {
         `result here would be a fiction. Use \`analyze digest\` for the reports ` +
         `that are wired, and track the area-health row set as unimplemented.`);
 }
-function commonFailuresCmd(opts, deps) {
+async function commonFailuresCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
-    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'common failures')) {
+    if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'common failures')) {
+        return;
+    }
+    if (cannotVerifyRawRecords(store, deps, opts.json === true, 'common failures')) {
         return;
     }
     const rows = [];
-    for (const record of store.readAll()) {
+    for (const record of await store.readAll()) {
         if (opts.since && (record.timestamp ?? '') < opts.since)
             continue;
         for (const t of record.tests ?? []) {
@@ -176,13 +315,16 @@ function commonFailuresCmd(opts, deps) {
         deps.out(buildCommonFailuresReport(rows, opts.minSuites));
     }
 }
-function regressionCandidatesCmd(opts, deps) {
+async function regressionCandidatesCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
-    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'regression candidates')) {
+    if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'regression candidates')) {
+        return;
+    }
+    if (cannotVerifyRawRecords(store, deps, opts.json === true, 'regression candidates')) {
         return;
     }
     const engine = new AnalysisEngine(store);
-    const candidates = engine.detectRegressionCandidates(null, opts.minGreen, opts.recentFailures);
+    const candidates = await engine.detectRegressionCandidates(null, opts.minGreen, opts.recentFailures);
     if (opts.json) {
         deps.out(jsonIndent2(candidates));
     }
@@ -190,20 +332,33 @@ function regressionCandidatesCmd(opts, deps) {
         deps.out(buildRegressionCandidatesReport(candidates));
     }
 }
-function digestCmd(opts, deps) {
+async function digestCmd(opts, deps) {
     const store = deps.makeStore(opts.dbUrl);
-    if (abstainOnEmptyHistory(store, deps, opts.json === true, 'fleet health')) {
+    if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'fleet health')) {
         return;
     }
     const engine = new AnalysisEngine(store);
-    const result = engine.run({
-        window: opts.window,
-        delta: opts.delta,
+    const result = await engine.run({
+        windowRuns: opts.windowRuns,
+        deltaPp: opts.deltaPp,
         weeks: opts.weeks,
         minSuites: opts.minSuites,
         suite: opts.suite ?? null,
     });
     writeArtifacts(result.artifacts, opts.output);
+    // Unlike the single-section commands, digest can partially succeed: the flaky
+    // leaderboard rides a contract method and is real even on a backend with no
+    // raw-record access. So it renders what it measured and names what it could
+    // not -- a digest silently missing three of its five sections would be the
+    // worst of both worlds. The notice rides stderr so --json and the Slack path
+    // stay byte-clean for their consumers.
+    if (result.degraded.length > 0) {
+        const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
+        deps.err(`${outcome.summaryLine} cannot verify: this history backend does not ` +
+            `expose raw run records, so ${result.degraded.join(', ')} ` +
+            `${result.degraded.length === 1 ? 'is' : 'are'} UNKNOWN rather than ` +
+            `clean in this digest. The flake leaderboard above is measured.`);
+    }
     if (opts.json) {
         deps.out(jsonIndent2({
             flaky_count: result.flaky.length,
@@ -229,6 +384,18 @@ function printSlack(flakyCount, regCount, deps) {
     deps.out(lines.join('\n'));
 }
 // --- assembly ----------------------------------------------------------------
+// Option help shared across subcommands. Every analyze option carries a
+// description: a blank one is how the silent unit survived this long, since the
+// value's meaning lived only in the report it eventually printed.
+const DB_URL_ENV = 'CANARY_HISTORY_DB_URL';
+const DB_URL_DESC = 'History store URL. Defaults to the local NDJSON store; a remote backend ' +
+    'cannot answer the spikes, common-failures or regression-candidates ' +
+    'reports, which say so rather than reporting zero.';
+const JSON_DESC = 'Emit the rows as JSON instead of a Markdown report.';
+const WINDOW_RUNS_DESC = 'Rolling window measured in RUNS, not days.';
+const MIN_RATE_PCT_DESC = 'Minimum flake rate to report, in PERCENT 0-100 (10 means 10%, not 0.1).';
+const DELTA_PP_DESC = 'Spike threshold as a PERCENTAGE-POINT rise in failure rate ' +
+    '(20 means 5% -> 25%, not 5% -> 6%).';
 /** Build a fresh `analyze` command wired to `depsInit`. */
 export function createAnalyzeCommand(depsInit = {}) {
     const deps = { ...defaultAnalyzeDeps(), ...depsInit };
@@ -239,89 +406,94 @@ export function createAnalyzeCommand(depsInit = {}) {
     program
         .command('flaky')
         .description('Fleet-wide flake leaderboard.')
-        .addOption(new Option('-w, --window <n>')
+        .addOption(new Option('-w, --window-runs <runs>', WINDOW_RUNS_DESC)
         .default(30)
-        .argParser((v) => Number.parseInt(v, 10)))
-        .option('-s, --suite <suite>')
-        .addOption(new Option('--min-rate <pct>')
+        .argParser(parseRuns('--window-runs')))
+        .addOption(new Option('--window <runs>', 'Deprecated alias for --window-runs.').argParser(parseRuns('--window')))
+        .option('-s, --suite <suite>', 'Filter to a specific suite.')
+        .addOption(new Option('--min-rate-pct <percent>', MIN_RATE_PCT_DESC)
         .default(10.0)
-        .argParser((v) => Number.parseFloat(v)))
-        .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
-        .option('--json')
-        .action((opts) => {
-        flakyCmd(opts, deps);
+        .argParser(parsePercentScale('--min-rate-pct', 'percent')))
+        .addOption(new Option('--min-rate <percent>', 'Deprecated alias for --min-rate-pct.').argParser(parsePercentScale('--min-rate', 'percent')))
+        .addOption(new Option('--db-url <url>', DB_URL_DESC).env(DB_URL_ENV))
+        .option('--json', JSON_DESC)
+        .action(async (opts, cmd) => {
+        await flakyCmd(resolveUnitFlags(opts, cmd, deps, [WINDOW_ALIAS, MIN_RATE_ALIAS]), deps);
     });
     program
         .command('spikes')
         .description('Recent failure spikes across suites.')
         .option('--since <date>', 'ISO date filter, e.g. 2026-06-01')
-        .addOption(new Option('--delta <pp>')
+        .addOption(new Option('--delta-pp <points>', DELTA_PP_DESC)
         .default(20.0)
-        .argParser((v) => Number.parseFloat(v)))
-        .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
-        .option('--json')
-        .action((opts) => {
-        spikesCmd(opts, deps);
+        .argParser(parsePercentScale('--delta-pp', 'percentage points')))
+        .addOption(new Option('--delta <points>', 'Deprecated alias for --delta-pp.').argParser(parsePercentScale('--delta', 'percentage points')))
+        .addOption(new Option('--db-url <url>', DB_URL_DESC).env(DB_URL_ENV))
+        .option('--json', JSON_DESC)
+        .action(async (opts, cmd) => {
+        await spikesCmd(resolveUnitFlags(opts, cmd, deps, [DELTA_ALIAS]), deps);
     });
     program
         .command('area-health')
         .description('Area degradation trends over time.')
-        .addOption(new Option('--weeks <n>')
+        .addOption(new Option('--weeks <weeks>', 'Trend window, in whole weeks.')
         .default(4)
         .argParser((v) => Number.parseInt(v, 10)))
-        .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
-        .option('--json')
+        .addOption(new Option('--db-url <url>', DB_URL_DESC).env(DB_URL_ENV))
+        .option('--json', JSON_DESC)
         .action((opts) => {
         areaHealthCmd(opts, deps);
     });
     program
         .command('common-failures')
         .description('Cross-suite failure fingerprinting.')
-        .option('--since <date>')
-        .addOption(new Option('--min-suites <n>')
+        .option('--since <date>', 'ISO date filter, e.g. 2026-06-01')
+        .addOption(new Option('--min-suites <suites>', 'Report a failure only once it appears in this many SUITES.')
         .default(2)
         .argParser((v) => Number.parseInt(v, 10)))
-        .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
-        .option('--json')
-        .action((opts) => {
-        commonFailuresCmd(opts, deps);
+        .addOption(new Option('--db-url <url>', DB_URL_DESC).env(DB_URL_ENV))
+        .option('--json', JSON_DESC)
+        .action(async (opts) => {
+        await commonFailuresCmd(opts, deps);
     });
     program
         .command('regression-candidates')
         .description('Tests newly and consistently broken after a green streak.')
-        .addOption(new Option('--min-green <n>')
+        .addOption(new Option('--min-green <runs>', 'Length of the prior green streak, in RUNS.')
         .default(5)
         .argParser((v) => Number.parseInt(v, 10)))
-        .addOption(new Option('--recent-failures <n>')
+        .addOption(new Option('--recent-failures <runs>', 'Consecutive failing RUNS required after the streak.')
         .default(3)
         .argParser((v) => Number.parseInt(v, 10)))
-        .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
-        .option('--json')
-        .action((opts) => {
-        regressionCandidatesCmd(opts, deps);
+        .addOption(new Option('--db-url <url>', DB_URL_DESC).env(DB_URL_ENV))
+        .option('--json', JSON_DESC)
+        .action(async (opts) => {
+        await regressionCandidatesCmd(opts, deps);
     });
     program
         .command('digest')
         .description('Combined digest of all five report types.')
-        .addOption(new Option('--window <n>')
+        .addOption(new Option('--window-runs <runs>', WINDOW_RUNS_DESC)
         .default(30)
-        .argParser((v) => Number.parseInt(v, 10)))
-        .addOption(new Option('--delta <pp>')
+        .argParser(parseRuns('--window-runs')))
+        .addOption(new Option('--window <runs>', 'Deprecated alias for --window-runs.').argParser(parseRuns('--window')))
+        .addOption(new Option('--delta-pp <points>', DELTA_PP_DESC)
         .default(20.0)
-        .argParser((v) => Number.parseFloat(v)))
-        .addOption(new Option('--weeks <n>')
+        .argParser(parsePercentScale('--delta-pp', 'percentage points')))
+        .addOption(new Option('--delta <points>', 'Deprecated alias for --delta-pp.').argParser(parsePercentScale('--delta', 'percentage points')))
+        .addOption(new Option('--weeks <weeks>', 'Area-health trend window, in whole weeks.')
         .default(4)
         .argParser((v) => Number.parseInt(v, 10)))
-        .addOption(new Option('--min-suites <n>')
+        .addOption(new Option('--min-suites <suites>', 'Report a failure only once it appears in this many SUITES.')
         .default(2)
         .argParser((v) => Number.parseInt(v, 10)))
-        .option('--suite <suite>')
-        .addOption(new Option('--output <dir>').default('test-results/analysis'))
-        .option('--json')
-        .option('--slack')
-        .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
-        .action((opts) => {
-        digestCmd(opts, deps);
+        .option('--suite <suite>', 'Filter to a specific suite.')
+        .addOption(new Option('--output <dir>', 'Directory the Markdown artifacts are ' + 'written to.').default('test-results/analysis'))
+        .option('--json', 'Emit per-section counts as JSON instead of the digest.')
+        .option('--slack', 'Emit a short Slack-formatted summary.')
+        .addOption(new Option('--db-url <url>', DB_URL_DESC).env(DB_URL_ENV))
+        .action(async (opts, cmd) => {
+        await digestCmd(resolveUnitFlags(opts, cmd, deps, [WINDOW_ALIAS, DELTA_ALIAS]), deps);
     });
     for (const sub of program.commands) {
         sub.exitOverride(normalizeUsageExit);

package/dist/engine/analysis/engine.js

@@ -9,7 +9,7 @@
  * `run()` (the Python engine initialises `area_rows = []` and never appends), so
  * the area-health artifact always renders the empty-data message.
  */
-import { buildAreaHealthReport, buildCommonFailuresReport, buildDigest, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
+import { buildAreaHealthReport, buildCommonFailuresReport, buildDigest, buildFlakyTestsReport, buildRegressionCandidatesReport, buildFailureSpikesReport, } from './reports.js';
 import { detectRegressions } from '../history/detector.js';
 import { def } from '../util/coalesce.js';
 function isReadable(store) {
@@ -26,25 +26,38 @@ function toCommonFailureRow(record, t) {
         error_text: t.error_text,
     };
 }
+/** Report-language names for the sections that need raw-record access. */
+const SECTION_SPIKES = 'failure spikes';
+const SECTION_COMMON_FAILURES = 'common failures';
+const SECTION_REGRESSIONS = 'regression candidates';
 export class AnalysisEngine {
     store;
     constructor(store) {
         this.store = store;
     }
-    run(opts = {}) {
-        const window = opts.window ?? 30;
-        const delta = opts.delta ?? 20.0;
+    async run(opts = {}) {
+        const windowRuns = opts.windowRuns ?? 30;
+        const deltaPp = opts.deltaPp ?? 20.0;
         const weeks = opts.weeks ?? 4;
         const minSuites = opts.minSuites ?? 2;
-        const minFlakeRate = opts.minFlakeRate ?? 10.0;
+        const minFlakeRatePct = opts.minFlakeRatePct ?? 10.0;
         const minGreen = opts.minGreen ?? 5;
         const recentFailures = opts.recentFailures ?? 3;
         const suite = opts.suite ?? null;
-        const flaky = this.store.queryFlaky(window, suite, minFlakeRate);
-        const suitesToQuery = suite ? [suite] : this.discoverSuites();
+        const flaky = (await this.store.queryFlaky(windowRuns, suite, minFlakeRatePct));
+        // Sections that need raw-record access are UNKNOWN, not empty, on a backend
+        // that does not offer it (#711). `flaky` above and a suite-scoped `spikes`
+        // below ride real contract methods, so they stay measured either way.
+        const degraded = [];
+        const readable = isReadable(this.store);
+        const suitesToQuery = suite ? [suite] : await this.discoverSuites();
+        // With no explicit --suite, the suite list itself comes from raw records —
+        // so spikes is only degraded in that case, not whenever readAll is absent.
+        if (!readable && !suite)
+            degraded.push(SECTION_SPIKES);
         const spikesRows = [];
         for (const s of suitesToQuery) {
-            const summary = this.store.querySummary(s, window * 2);
+            const summary = await this.store.querySummary(s, windowRuns * 2);
             for (const row of def(summary.runs, [])) {
                 // query_summary rows omit suite; the spikes builder groups by it, so
                 // tag each pooled row with the suite it came from (matches Python).
@@ -59,22 +72,26 @@ export class AnalysisEngine {
         }
         // Faithful to Python: area rows are never populated by run().
         const areaRows = [];
-        const commonRows = this.queryCommonFailures(suite);
-        const regressionCandidates = this.detectRegressionCandidates(suite, minGreen, recentFailures);
+        const commonRows = await this.queryCommonFailures(suite);
+        if (!readable)
+            degraded.push(SECTION_COMMON_FAILURES);
+        const regressionCandidates = await this.detectRegressionCandidates(suite, minGreen, recentFailures);
+        if (!readable)
+            degraded.push(SECTION_REGRESSIONS);
         const digest = buildDigest({
             flaky,
             spikes: spikesRows,
             areaHealth: areaRows,
             commonFailures: commonRows,
             regressionCandidates,
-            window,
-            delta,
+            windowRuns,
+            deltaPp,
             weeks,
             minSuites,
         });
         const artifacts = {
-            'flaky.md': buildFlakyReport(flaky, window, minFlakeRate),
-            'spikes.md': buildSpikesReport(spikesRows, delta),
+            'flaky.md': buildFlakyTestsReport(flaky, windowRuns, minFlakeRatePct),
+            'spikes.md': buildFailureSpikesReport(spikesRows, deltaPp),
             'area-health.md': buildAreaHealthReport(areaRows, weeks),
             'common-failures.md': buildCommonFailuresReport(commonRows, minSuites),
             'regression-candidates.md': buildRegressionCandidatesReport(regressionCandidates),
@@ -88,23 +105,24 @@ export class AnalysisEngine {
             regressionCandidates,
             digestMd: digest,
             artifacts,
+            degraded,
         };
     }
-    discoverSuites() {
+    async discoverSuites() {
         if (!isReadable(this.store))
             return [];
         const suites = new Set();
-        for (const r of this.store.readAll()) {
+        for (const r of await this.store.readAll()) {
             if (r.suite)
                 suites.add(r.suite);
         }
         return [...suites];
     }
-    queryCommonFailures(suite) {
+    async queryCommonFailures(suite) {
         if (!isReadable(this.store))
             return [];
         const rows = [];
-        for (const record of this.store.readAll()) {
+        for (const record of await this.store.readAll()) {
             if (suite && record.suite !== suite)
                 continue;
             for (const t of def(record.tests, [])) {
@@ -114,11 +132,11 @@ export class AnalysisEngine {
         }
         return rows;
     }
-    detectRegressionCandidates(suite, minGreen, recentFailures) {
+    async detectRegressionCandidates(suite, minGreen, recentFailures) {
         if (!isReadable(this.store))
             return [];
         const testNames = new Set();
-        for (const record of this.store.readAll()) {
+        for (const record of await this.store.readAll()) {
             if (suite && record.suite !== suite)
                 continue;
             for (const t of def(record.tests, []))
@@ -126,7 +144,7 @@ export class AnalysisEngine {
         }
         const candidates = [];
         for (const name of testNames) {
-            const timeline = this.store.queryTimeline(name);
+            const timeline = await this.store.queryTimeline(name);
             const result = detectRegressions(timeline, minGreen, recentFailures);
             if (result.is_regression) {
                 candidates.push({