canary-test-cli 5.15.0 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/agent/frameworks/registry.json +655 -0
  2. package/bin/canary.js +20 -15
  3. package/dist/doctor-manifest.d.ts +94 -0
  4. package/dist/doctor.d.ts +67 -0
  5. package/dist/engine/analysis/cli.js +270 -0
  6. package/dist/engine/analysis/engine.js +146 -0
  7. package/dist/engine/analysis/reports.js +0 -0
  8. package/dist/engine/analysis/rows.js +9 -0
  9. package/dist/engine/cli-commands.js +618 -0
  10. package/dist/engine/cli-common.js +60 -0
  11. package/dist/engine/cli.core.js +208 -0
  12. package/dist/engine/cli.js +31 -0
  13. package/dist/engine/company-knowledge-cli.js +201 -0
  14. package/dist/engine/core/ci-env.js +33 -0
  15. package/dist/engine/core/classifier.js +192 -0
  16. package/dist/engine/core/company-knowledge.js +765 -0
  17. package/dist/engine/core/config-validation.js +74 -0
  18. package/dist/engine/core/detection.js +48 -0
  19. package/dist/engine/core/domain-scanner.js +212 -0
  20. package/dist/engine/core/environment-detect.js +410 -0
  21. package/dist/engine/core/executor.js +181 -0
  22. package/dist/engine/core/feedback.js +93 -0
  23. package/dist/engine/core/fixture-scanner.js +173 -0
  24. package/dist/engine/core/framework-registry.js +123 -0
  25. package/dist/engine/core/mcp-validator.js +218 -0
  26. package/dist/engine/core/metadata-scanner.js +147 -0
  27. package/dist/engine/core/migrator.js +1112 -0
  28. package/dist/engine/core/overlays.js +176 -0
  29. package/dist/engine/core/pattern-healer.js +147 -0
  30. package/dist/engine/core/pattern-matcher.js +255 -0
  31. package/dist/engine/core/quality-scorer.js +213 -0
  32. package/dist/engine/core/recommender.js +152 -0
  33. package/dist/engine/core/reporter.js +211 -0
  34. package/dist/engine/core/scaffolder.js +236 -0
  35. package/dist/engine/core/skill-registry.js +522 -0
  36. package/dist/engine/core/static-linter.js +237 -0
  37. package/dist/engine/core/ticket-updater.js +639 -0
  38. package/dist/engine/core/workflow-discovery.js +693 -0
  39. package/dist/engine/guardian/agent-tier.js +338 -0
  40. package/dist/engine/guardian/analysis-emit.js +201 -0
  41. package/dist/engine/guardian/cli.js +787 -0
  42. package/dist/engine/guardian/coverage.js +1055 -0
  43. package/dist/engine/guardian/delta-emitter.js +46 -0
  44. package/dist/engine/guardian/diff-extractor.js +257 -0
  45. package/dist/engine/guardian/hard-gate.js +373 -0
  46. package/dist/engine/guardian/impact-mapper.js +121 -0
  47. package/dist/engine/guardian/pr-check.js +975 -0
  48. package/dist/engine/guardian/pr-comment.js +200 -0
  49. package/dist/engine/guardian/summary-emitter.js +94 -0
  50. package/dist/engine/guardian/tier.js +58 -0
  51. package/dist/engine/history/cli.js +303 -0
  52. package/dist/engine/history/detector.js +68 -0
  53. package/dist/engine/history/ndjson-store.js +177 -0
  54. package/dist/engine/history/record.js +14 -0
  55. package/dist/engine/history/schema.js +59 -0
  56. package/dist/engine/history/store.js +47 -0
  57. package/dist/engine/history/supabase-store.js +113 -0
  58. package/dist/engine/main-deps.js +105 -0
  59. package/dist/engine/mcp-server.js +647 -0
  60. package/dist/engine/package.json +4 -0
  61. package/dist/engine/skills-cli.js +181 -0
  62. package/dist/engine/ui/banner.js +50 -0
  63. package/dist/engine/util/coalesce.js +12 -0
  64. package/dist/engine/util/round.js +43 -0
  65. package/dist/engine/workflow-cli.js +242 -0
  66. package/dist/engine-checks.d.ts +49 -0
  67. package/dist/overlay-commands.d.ts +81 -0
  68. package/dist/overlay-conflicts.d.ts +33 -0
  69. package/dist/overlay-lint.d.ts +19 -0
  70. package/dist/overlays-registry.d.ts +74 -0
  71. package/dist/reporters/testtracker.d.ts +89 -0
  72. package/dist/reporters/testtracker.js +195 -0
  73. package/dist/router.d.ts +12 -0
  74. package/dist/router.js +4 -4
  75. package/dist/skill-requirements.d.ts +57 -0
  76. package/dist/source-spec.d.ts +20 -0
  77. package/package.json +30 -6
  78. package/bin/canary +0 -0
  79. package/scripts/install.js +0 -104
package/bin/canary.js CHANGED
@@ -6,35 +6,40 @@ const path = require('node:path');
6
6
  const fs = require('node:fs');
7
7
  const { isTsCommand, route } = require('../dist/router.js');
8
8
 
9
- function getBinaryPath(platform) {
10
- const name = platform === 'win32' ? 'canary.exe' : 'canary';
11
- return path.join(__dirname, name);
9
+ /**
10
+ * Absolute path to the bundled TypeScript engine's runnable entry
11
+ * (`dist/engine/cli.js`, generated by scripts/build-engine.mjs). Runs under the
12
+ * current `node` via `execFileSync(process.execPath, [enginePath, ...])`.
13
+ */
14
+ function getEnginePath() {
15
+ return path.join(__dirname, '..', 'dist', 'engine', 'cli.js');
12
16
  }
13
17
 
14
18
  /**
15
- * Forward a command to the bundled Python binary. Returns the exit code
16
- * (0 on success, the binary's status on failure, 1 when the binary is missing).
19
+ * Forward a command to the bundled TypeScript engine. Returns the exit code
20
+ * (0 on success, the engine's status on failure, 1 when the engine is missing).
17
21
  * Dependencies are injectable for testing.
18
22
  */
19
- function forwardToBinary(
23
+ function forwardToEngine(
20
24
  argv,
21
25
  {
22
26
  execFile = execFileSync,
23
27
  existsSync = fs.existsSync,
24
- platform = process.platform,
28
+ execPath = process.execPath,
25
29
  stderr = process.stderr,
26
30
  } = {},
27
31
  ) {
28
- const binaryPath = getBinaryPath(platform);
29
- if (!existsSync(binaryPath)) {
32
+ const enginePath = getEnginePath();
33
+ if (!existsSync(enginePath)) {
30
34
  stderr.write(
31
- `canary binary not found at ${binaryPath}.\n` +
32
- `Try reinstalling: npm install -g canary-test-cli\n`,
35
+ `canary engine not found at ${enginePath}.\n` +
36
+ `The package looks incomplete; try reinstalling: npm install -g canary-test-cli\n`,
33
37
  );
34
38
  return 1;
35
39
  }
36
40
  try {
37
- execFile(binaryPath, argv, { stdio: 'inherit' });
41
+ // Inherit stdio AND cwd so the engine operates on the user's project.
42
+ execFile(execPath, [enginePath, ...argv], { stdio: 'inherit' });
38
43
  return 0;
39
44
  } catch (err) {
40
45
  return err.status ?? 1;
@@ -43,14 +48,14 @@ function forwardToBinary(
43
48
 
44
49
  /**
45
50
  * Dispatch one invocation: TS-handled commands (e.g. `overlay`, `doctor`) go to
46
- * the router; everything else forwards verbatim to the Python binary. Returns
51
+ * the router; everything else forwards verbatim to the bundled engine. Returns
47
52
  * the process exit code, or a Promise of one for async commands (`doctor`).
48
53
  */
49
54
  function run(argv, deps = {}) {
50
55
  if (isTsCommand(argv)) {
51
56
  return route(argv, deps) ?? 0;
52
57
  }
53
- return forwardToBinary(argv, deps);
58
+ return forwardToEngine(argv, deps);
54
59
  }
55
60
 
56
61
  function main() {
@@ -60,5 +65,5 @@ function main() {
60
65
  );
61
66
  }
62
67
 
63
- module.exports = { getBinaryPath, forwardToBinary, run };
68
+ module.exports = { getEnginePath, forwardToEngine, run };
64
69
  if (require.main === module) main();
@@ -0,0 +1,94 @@
1
+ import type { CheckResult } from './doctor.js';
2
+ export declare const CHECK_TYPES: readonly ["file-exists", "url-reachable", "command-succeeds"];
3
+ export type CheckType = (typeof CHECK_TYPES)[number];
4
+ /** A single validated check from an overlay's `doctor.json`. */
5
+ export interface ManifestCheck {
6
+ id: string;
7
+ type: CheckType;
8
+ /** Shown under the check line when it fails. */
9
+ remedy: string;
10
+ /**
11
+ * Free-form audience tags (#319 B). A check with none runs for every
12
+ * audience; otherwise it runs only when `--audience <tag>` matches one.
13
+ * Declared as `audience:` in doctor.json (legacy alias: `persona:`).
14
+ *
15
+ * NB: unrelated to harness's persona system (`run_persona` /
16
+ * `generate_persona_artifacts`). This is a canary-local check-grouping tag;
17
+ * the field was renamed off "persona" to end that collision.
18
+ */
19
+ audience?: string[];
20
+ /** `file-exists`: path relative to the overlay clone. */
21
+ path?: string;
22
+ /** `url-reachable`: the URL to probe. */
23
+ url?: string;
24
+ /** `command-succeeds`: argv array run (no shell) in the clone dir. */
25
+ command?: string[];
26
+ }
27
+ /** Result of loading a manifest: either its checks, or a single failing check. */
28
+ export type ManifestLoad = {
29
+ ok: true;
30
+ checks: ManifestCheck[];
31
+ } | {
32
+ ok: false;
33
+ failure: CheckResult;
34
+ };
35
+ /** Path to an overlay's manifest. */
36
+ export declare function manifestPath(cloneDir: string): string;
37
+ /**
38
+ * Stable fingerprint of a manifest's `command-succeeds` checks — the (id,
39
+ * command) pairs, sorted by id. Returns null when there are no such checks
40
+ * (nothing to gate). Consent is re-requested when this value changes.
41
+ */
42
+ export declare function commandSucceedsHash(checks: ManifestCheck[]): string | null;
43
+ /**
44
+ * Load and validate `<clone>/.canary/doctor.json`. A missing file is not an
45
+ * error (no checks). A malformed file or an invalid check degrades to a single
46
+ * failing check ({@link ManifestLoad} `ok: false`) — never a throw.
47
+ */
48
+ export declare function loadManifest(cloneDir: string): ManifestLoad;
49
+ /**
50
+ * The distinct audience tags declared across a set of checks, in first-seen
51
+ * order and de-duplicated case-insensitively (original casing preserved for
52
+ * display). This is the discoverable audience *vocabulary* — the engine ships
53
+ * none of its own, so it is derived entirely from overlay manifests. Used to
54
+ * tell a user which `--audience` values actually mean something (issue #294).
55
+ */
56
+ export declare function collectAudiences(checks: ManifestCheck[]): string[];
57
+ /**
58
+ * Keep checks that should run for `audience`: a null audience runs everything;
59
+ * otherwise keep checks with no audience plus those whose audience list
60
+ * contains the tag (case-insensitive).
61
+ */
62
+ export declare function filterByAudience(checks: ManifestCheck[], audience: string | null): ManifestCheck[];
63
+ /** Default per-check timeout for url and command checks. */
64
+ export declare const DEFAULT_CHECK_TIMEOUT_MS = 10000;
65
+ /** Probe a URL for reachability (injectable). Resolves true on a 2xx/3xx. */
66
+ export type UrlProbe = (url: string, timeoutMs: number) => Promise<boolean>;
67
+ /** Run a command (injectable). `ok` = exit 0; `timedOut` = killed at the timeout. */
68
+ export type CommandRunner = (command: string[], cwd: string, timeoutMs: number, extraEnv?: NodeJS.ProcessEnv) => {
69
+ ok: boolean;
70
+ timedOut: boolean;
71
+ detail?: string;
72
+ };
73
+ /** Context for executing checks against one overlay clone. */
74
+ export interface RunContext {
75
+ cloneDir: string;
76
+ /**
77
+ * Directory `canary doctor` was invoked from (the consuming repo root).
78
+ * Exposed to `command-succeeds` checks as `CANARY_INVOCATION_DIR` so a check
79
+ * can anchor to consuming-repo runtime artifacts instead of the overlay
80
+ * clone, which is always the cwd (#378).
81
+ */
82
+ invocationDir?: string;
83
+ /** Whether this overlay's `command-succeeds` checks may execute. */
84
+ consentGranted: boolean;
85
+ timeoutMs?: number;
86
+ probeUrl?: UrlProbe;
87
+ runCommand?: CommandRunner;
88
+ }
89
+ /**
90
+ * Execute one validated check against its overlay clone, under a bounded
91
+ * timeout. `command-succeeds` is skipped (not failed) unless consent is
92
+ * granted. Never throws.
93
+ */
94
+ export declare function runCheck(check: ManifestCheck, ctx: RunContext): Promise<CheckResult>;
@@ -0,0 +1,67 @@
1
+ import type { CommandDeps } from './overlay-commands.js';
2
+ import { type CommandRunner, type UrlProbe } from './doctor-manifest.js';
3
+ /** Outcome of a single doctor check. */
4
+ export type CheckStatus = 'pass' | 'fail' | 'skip' | 'info';
5
+ /** A single doctor check result, rendered as one output line. */
6
+ export interface CheckResult {
7
+ /** Stable identifier (engine check id, or the manifest check's `id`). */
8
+ id: string;
9
+ status: CheckStatus;
10
+ /** Human-readable label for the check line. */
11
+ label: string;
12
+ /** Shown indented under the line when the check does not pass. */
13
+ remedy?: string;
14
+ }
15
+ /** Dependencies for `doctor`, injectable for tests (real ones by default). */
16
+ export interface DoctorDeps extends CommandDeps {
17
+ /** Current working directory (for project `.canary/` and `.mcp.json`). */
18
+ cwd?: string;
19
+ currentVersion?: string;
20
+ getLatestVersion?: () => Promise<string | null>;
21
+ probeUrl?: UrlProbe;
22
+ runCommand?: CommandRunner;
23
+ timeoutMs?: number;
24
+ }
25
+ /**
26
+ * The `canary doctor --json` machine contract (issue #318). Canary-owned and
27
+ * intentionally distinct from `harness doctor --json`: only `allPassed` matches
28
+ * harness's top-level shape; per-check fields (`id`/`label`/`remedy`/`group`)
29
+ * and the `skip` status tier are canary's own. `version` guards the contract so
30
+ * consumers can detect a breaking change.
31
+ */
32
+ export interface JsonReportCheck {
33
+ /** Stable check identifier (engine check id, or the manifest check's `id`). */
34
+ id: string;
35
+ status: CheckStatus;
36
+ label: string;
37
+ /** Present only when the check did not pass. */
38
+ remedy?: string;
39
+ /** The section the check belongs to, e.g. `"Engine"` or `"Overlay: acme"`. */
40
+ group: string;
41
+ }
42
+ export interface JsonReport {
43
+ version: 1;
44
+ checks: JsonReportCheck[];
45
+ /** True iff no check failed — mirrors `harness doctor --json`'s one shared field. */
46
+ allPassed: boolean;
47
+ /** Non-fatal advisories (e.g. an unknown `--audience`); empty when none. */
48
+ warnings: string[];
49
+ }
50
+ /** `--json` requests machine output on stdout instead of the human report. */
51
+ export declare function parseJsonFlag(args: readonly string[]): boolean;
52
+ /**
53
+ * Fail-loud hint for an unrecognized `--audience` value (issue #294). Returns
54
+ * null when there is nothing to say — no audience was passed, or the passed
55
+ * audience is part of the known vocabulary. Otherwise returns a one-line,
56
+ * actionable message: the engine ships no audience vocabulary, so this lists
57
+ * the tags overlays actually declared (or says none are defined) instead of
58
+ * silently running only the audience-less checks and leaving the user to
59
+ * guess why their filter matched nothing.
60
+ */
61
+ export declare function unknownAudienceHint(audience: string | null, known: readonly string[]): string | null;
62
+ /**
63
+ * Run `canary doctor`. Returns a process exit code: 0 when every check passed
64
+ * or was skipped/info, non-zero when any check failed. A malformed manifest for
65
+ * one overlay never blocks engine checks or other overlays.
66
+ */
67
+ export declare function runDoctor(args: readonly string[], deps?: DoctorDeps): Promise<number>;
@@ -0,0 +1,270 @@
1
+ /**
2
+ * CLI subcommands for `canary analyze` -- faithful port of
3
+ * `agent/analysis/cli.py` (the `analyze_app` Typer sub-app), wired to the
4
+ * already-ported analysis engine + report builders (`engine.ts`, `reports.ts`,
5
+ * `rows.ts`) and the local NDJSON history store.
6
+ *
7
+ * Follows the guardian CLI conventions (see `../cli-common.ts`): a
8
+ * {@link createAnalyzeCommand} factory wired to an injectable {@link AnalyzeDeps},
9
+ * and `normalizeUsageExit` on every command so usage errors exit 2. No command
10
+ * raises a business exit -- every analyze subcommand returns 0 (matching Python).
11
+ *
12
+ * Python->TS fidelity notes:
13
+ * - `json.dumps(x, indent=2)` -> {@link jsonIndent2} (byte-exact + ensure_ascii).
14
+ * - The report builders are byte-exact ports (Markdown), so the human-readable
15
+ * paths match the oracle exactly.
16
+ * - INTENTIONAL DEVIATION: the TS analysis engine (`engine.ts`) operates on the
17
+ * LOCAL NDJSON store only -- the JS Supabase SDK is async and the engine's
18
+ * query surface is synchronous. `--db-url` is still accepted (faithful CLI
19
+ * surface) but the TS port always reads the local store, exactly as the
20
+ * Python `isinstance(store, LocalHistoryStore)` read-path does for the
21
+ * spikes/common-failures commands. No Python analyze test exercises a remote
22
+ * store.
23
+ * - `area-health` accepts `--json` but ignores it -- faithful to the Python
24
+ * command, which never branches on `output_json`.
25
+ */
26
+ import { mkdirSync, writeFileSync } from 'node:fs';
27
+ import { join } from 'node:path';
28
+ import { Command, Option } from 'commander';
29
+ import { jsonIndent2, normalizeUsageExit } from '../cli-common.js';
30
+ import { AnalysisEngine } from './engine.js';
31
+ import { buildAreaHealthReport, buildCommonFailuresReport, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
32
+ import { NdjsonHistoryStore } from '../history/ndjson-store.js';
33
+ const DEFAULT_HISTORY_PATH = 'test-results/reports/history-v2.jsonl';
34
+ /** Process-backed defaults for production. */
35
+ export function defaultAnalyzeDeps() {
36
+ const err = (s) => {
37
+ process.stderr.write(`${s}\n`);
38
+ };
39
+ return {
40
+ out: (s) => process.stdout.write(`${s}\n`),
41
+ err,
42
+ env: process.env,
43
+ // The ported analysis engine's query surface is synchronous, so it cannot
44
+ // drive the async Supabase store; analyze reads local NDJSON only. Python
45
+ // honors --db-url / CANARY_HISTORY_DB_URL via make_store, so warn (to stderr,
46
+ // not stdout -- keeps --json clean) rather than SILENTLY reading a different
47
+ // data source. Full remote support is deferred with the async engine port.
48
+ makeStore: (dbUrl) => {
49
+ if (dbUrl) {
50
+ err('note: --db-url is ignored by analyze; it reads local NDJSON only.');
51
+ }
52
+ return new NdjsonHistoryStore(DEFAULT_HISTORY_PATH);
53
+ },
54
+ };
55
+ }
56
+ function writeArtifacts(artifacts, output) {
57
+ mkdirSync(output, { recursive: true });
58
+ for (const [name, content] of Object.entries(artifacts)) {
59
+ writeFileSync(join(output, name), content, 'utf-8');
60
+ }
61
+ }
62
+ function flakyCmd(opts, deps) {
63
+ const store = deps.makeStore(opts.dbUrl);
64
+ const rows = store.queryFlaky(opts.window, opts.suite ?? null, opts.minRate);
65
+ if (opts.json) {
66
+ deps.out(jsonIndent2(rows));
67
+ }
68
+ else {
69
+ deps.out(buildFlakyReport(rows, opts.window, opts.minRate));
70
+ }
71
+ }
72
+ function spikesCmd(opts, deps) {
73
+ const store = deps.makeStore(opts.dbUrl);
74
+ const rows = [];
75
+ for (const r of store.readAll()) {
76
+ if (opts.since && (r.timestamp ?? '') < opts.since)
77
+ continue;
78
+ rows.push({
79
+ suite: r.suite ?? '',
80
+ timestamp: r.timestamp ?? '',
81
+ passed: r.passed ?? 0,
82
+ failed: r.failed ?? 0,
83
+ flaky: r.flaky ?? 0,
84
+ total: r.total ?? 0,
85
+ });
86
+ }
87
+ if (opts.json) {
88
+ deps.out(jsonIndent2(rows));
89
+ }
90
+ else {
91
+ deps.out(buildSpikesReport(rows, opts.delta));
92
+ }
93
+ }
94
+ function areaHealthCmd(opts, deps) {
95
+ // Faithful to Python: always builds from an empty row set and never branches
96
+ // on --json.
97
+ deps.out(buildAreaHealthReport([], opts.weeks));
98
+ }
99
+ function commonFailuresCmd(opts, deps) {
100
+ const store = deps.makeStore(opts.dbUrl);
101
+ const rows = [];
102
+ for (const record of store.readAll()) {
103
+ if (opts.since && (record.timestamp ?? '') < opts.since)
104
+ continue;
105
+ for (const t of record.tests ?? []) {
106
+ if ((t.status === 'failed' || t.status === 'flaky') && t.error_text) {
107
+ rows.push({
108
+ test_name: t.test_name,
109
+ suite: record.suite ?? '',
110
+ failure_category: t.failure_category ?? 'other',
111
+ error_text: t.error_text ?? '',
112
+ run_count: 1,
113
+ });
114
+ }
115
+ }
116
+ }
117
+ if (opts.json) {
118
+ deps.out(jsonIndent2(rows));
119
+ }
120
+ else {
121
+ deps.out(buildCommonFailuresReport(rows, opts.minSuites));
122
+ }
123
+ }
124
+ function regressionCandidatesCmd(opts, deps) {
125
+ const engine = new AnalysisEngine(deps.makeStore(opts.dbUrl));
126
+ const candidates = engine.detectRegressionCandidates(null, opts.minGreen, opts.recentFailures);
127
+ if (opts.json) {
128
+ deps.out(jsonIndent2(candidates));
129
+ }
130
+ else {
131
+ deps.out(buildRegressionCandidatesReport(candidates));
132
+ }
133
+ }
134
+ function digestCmd(opts, deps) {
135
+ const engine = new AnalysisEngine(deps.makeStore(opts.dbUrl));
136
+ const result = engine.run({
137
+ window: opts.window,
138
+ delta: opts.delta,
139
+ weeks: opts.weeks,
140
+ minSuites: opts.minSuites,
141
+ suite: opts.suite ?? null,
142
+ });
143
+ writeArtifacts(result.artifacts, opts.output);
144
+ if (opts.json) {
145
+ deps.out(jsonIndent2({
146
+ flaky_count: result.flaky.length,
147
+ spike_count: result.spikes.length,
148
+ regression_count: result.regressionCandidates.length,
149
+ }));
150
+ }
151
+ else if (opts.slack) {
152
+ printSlack(result.flaky.length, result.regressionCandidates.length, deps);
153
+ }
154
+ else {
155
+ deps.out(result.digestMd);
156
+ deps.out(`\nArtifacts written to ${opts.output}/`);
157
+ }
158
+ }
159
+ function printSlack(flakyCount, regCount, deps) {
160
+ // U+2022 bullet, U+2265 >=.
161
+ const lines = [
162
+ '*Fleet Health Digest*',
163
+ `\u{2022} Flakeys \u{2265} 10%: ${flakyCount}`,
164
+ `\u{2022} Regression candidates: ${regCount}`,
165
+ ];
166
+ deps.out(lines.join('\n'));
167
+ }
168
+ // --- assembly ----------------------------------------------------------------
169
+ /** Build a fresh `analyze` command wired to `depsInit`. */
170
+ export function createAnalyzeCommand(depsInit = {}) {
171
+ const deps = { ...defaultAnalyzeDeps(), ...depsInit };
172
+ const program = new Command('analyze');
173
+ program
174
+ .description('Cross-suite fleet health analysis.')
175
+ .exitOverride(normalizeUsageExit);
176
+ program
177
+ .command('flaky')
178
+ .description('Fleet-wide flake leaderboard.')
179
+ .addOption(new Option('-w, --window <n>')
180
+ .default(30)
181
+ .argParser((v) => Number.parseInt(v, 10)))
182
+ .option('-s, --suite <suite>')
183
+ .addOption(new Option('--min-rate <pct>')
184
+ .default(10.0)
185
+ .argParser((v) => Number.parseFloat(v)))
186
+ .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
187
+ .option('--json')
188
+ .action((opts) => {
189
+ flakyCmd(opts, deps);
190
+ });
191
+ program
192
+ .command('spikes')
193
+ .description('Recent failure spikes across suites.')
194
+ .option('--since <date>', 'ISO date filter, e.g. 2026-06-01')
195
+ .addOption(new Option('--delta <pp>')
196
+ .default(20.0)
197
+ .argParser((v) => Number.parseFloat(v)))
198
+ .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
199
+ .option('--json')
200
+ .action((opts) => {
201
+ spikesCmd(opts, deps);
202
+ });
203
+ program
204
+ .command('area-health')
205
+ .description('Area degradation trends over time.')
206
+ .addOption(new Option('--weeks <n>')
207
+ .default(4)
208
+ .argParser((v) => Number.parseInt(v, 10)))
209
+ .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
210
+ .option('--json')
211
+ .action((opts) => {
212
+ areaHealthCmd(opts, deps);
213
+ });
214
+ program
215
+ .command('common-failures')
216
+ .description('Cross-suite failure fingerprinting.')
217
+ .option('--since <date>')
218
+ .addOption(new Option('--min-suites <n>')
219
+ .default(2)
220
+ .argParser((v) => Number.parseInt(v, 10)))
221
+ .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
222
+ .option('--json')
223
+ .action((opts) => {
224
+ commonFailuresCmd(opts, deps);
225
+ });
226
+ program
227
+ .command('regression-candidates')
228
+ .description('Tests newly and consistently broken after a green streak.')
229
+ .addOption(new Option('--min-green <n>')
230
+ .default(5)
231
+ .argParser((v) => Number.parseInt(v, 10)))
232
+ .addOption(new Option('--recent-failures <n>')
233
+ .default(3)
234
+ .argParser((v) => Number.parseInt(v, 10)))
235
+ .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
236
+ .option('--json')
237
+ .action((opts) => {
238
+ regressionCandidatesCmd(opts, deps);
239
+ });
240
+ program
241
+ .command('digest')
242
+ .description('Combined digest of all five report types.')
243
+ .addOption(new Option('--window <n>')
244
+ .default(30)
245
+ .argParser((v) => Number.parseInt(v, 10)))
246
+ .addOption(new Option('--delta <pp>')
247
+ .default(20.0)
248
+ .argParser((v) => Number.parseFloat(v)))
249
+ .addOption(new Option('--weeks <n>')
250
+ .default(4)
251
+ .argParser((v) => Number.parseInt(v, 10)))
252
+ .addOption(new Option('--min-suites <n>')
253
+ .default(2)
254
+ .argParser((v) => Number.parseInt(v, 10)))
255
+ .option('--suite <suite>')
256
+ .addOption(new Option('--output <dir>').default('test-results/analysis'))
257
+ .option('--json')
258
+ .option('--slack')
259
+ .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
260
+ .action((opts) => {
261
+ digestCmd(opts, deps);
262
+ });
263
+ for (const sub of program.commands) {
264
+ sub.exitOverride(normalizeUsageExit);
265
+ }
266
+ return program;
267
+ }
268
+ /** The production `analyze` command (process-backed defaults). */
269
+ export const analyzeCommand = createAnalyzeCommand();
270
+ //# sourceMappingURL=cli.js.map
@@ -0,0 +1,146 @@
1
+ /**
2
+ * AnalysisEngine — queries the history store and runs all report types.
3
+ *
4
+ * Faithful TS port of `agent/analysis/engine.py`. Thin coordinator: calls the
5
+ * store's query methods, passes results to the pure builders in reports.ts, and
6
+ * returns structured data plus Markdown artifacts.
7
+ *
8
+ * Fidelity note: as in Python, `areaHealth` is intentionally never populated by
9
+ * `run()` (the Python engine initialises `area_rows = []` and never appends), so
10
+ * the area-health artifact always renders the empty-data message.
11
+ */
12
+ import { buildAreaHealthReport, buildCommonFailuresReport, buildDigest, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
13
+ import { detectRegressions } from '../history/detector.js';
14
+ import { def } from '../util/coalesce.js';
15
+ function isReadable(store) {
16
+ return typeof store.readAll === 'function';
17
+ }
18
+ function isFailureWithError(t) {
19
+ return ((t.status === 'failed' || t.status === 'flaky') && Boolean(t.error_text));
20
+ }
21
+ function toCommonFailureRow(record, t) {
22
+ return {
23
+ test_name: t.test_name,
24
+ suite: def(record.suite, ''),
25
+ failure_category: def(t.failure_category, 'other'),
26
+ error_text: t.error_text,
27
+ };
28
+ }
29
+ export class AnalysisEngine {
30
+ store;
31
+ constructor(store) {
32
+ this.store = store;
33
+ }
34
+ run(opts = {}) {
35
+ const window = opts.window ?? 30;
36
+ const delta = opts.delta ?? 20.0;
37
+ const weeks = opts.weeks ?? 4;
38
+ const minSuites = opts.minSuites ?? 2;
39
+ const minFlakeRate = opts.minFlakeRate ?? 10.0;
40
+ const minGreen = opts.minGreen ?? 5;
41
+ const recentFailures = opts.recentFailures ?? 3;
42
+ const suite = opts.suite ?? null;
43
+ const flaky = this.store.queryFlaky(window, suite, minFlakeRate);
44
+ const suitesToQuery = suite ? [suite] : this.discoverSuites();
45
+ const spikesRows = [];
46
+ for (const s of suitesToQuery) {
47
+ const summary = this.store.querySummary(s, window * 2);
48
+ for (const row of def(summary.runs, [])) {
49
+ // query_summary rows omit suite; the spikes builder groups by it, so
50
+ // tag each pooled row with the suite it came from (matches Python).
51
+ spikesRows.push({
52
+ suite: s,
53
+ timestamp: row.timestamp,
54
+ total: row.total,
55
+ failed: row.failed,
56
+ flaky: row.flaky,
57
+ });
58
+ }
59
+ }
60
+ // Faithful to Python: area rows are never populated by run().
61
+ const areaRows = [];
62
+ const commonRows = this.queryCommonFailures(suite);
63
+ const regressionCandidates = this.detectRegressionCandidates(suite, minGreen, recentFailures);
64
+ const digest = buildDigest({
65
+ flaky,
66
+ spikes: spikesRows,
67
+ areaHealth: areaRows,
68
+ commonFailures: commonRows,
69
+ regressionCandidates,
70
+ window,
71
+ delta,
72
+ weeks,
73
+ minSuites,
74
+ });
75
+ const artifacts = {
76
+ 'flaky.md': buildFlakyReport(flaky, window, minFlakeRate),
77
+ 'spikes.md': buildSpikesReport(spikesRows, delta),
78
+ 'area-health.md': buildAreaHealthReport(areaRows, weeks),
79
+ 'common-failures.md': buildCommonFailuresReport(commonRows, minSuites),
80
+ 'regression-candidates.md': buildRegressionCandidatesReport(regressionCandidates),
81
+ 'digest.md': digest,
82
+ };
83
+ return {
84
+ flaky,
85
+ spikes: spikesRows,
86
+ areaHealth: areaRows,
87
+ commonFailures: commonRows,
88
+ regressionCandidates,
89
+ digestMd: digest,
90
+ artifacts,
91
+ };
92
+ }
93
+ discoverSuites() {
94
+ if (!isReadable(this.store))
95
+ return [];
96
+ const suites = new Set();
97
+ for (const r of this.store.readAll()) {
98
+ if (r.suite)
99
+ suites.add(r.suite);
100
+ }
101
+ return [...suites];
102
+ }
103
+ queryCommonFailures(suite) {
104
+ if (!isReadable(this.store))
105
+ return [];
106
+ const rows = [];
107
+ for (const record of this.store.readAll()) {
108
+ if (suite && record.suite !== suite)
109
+ continue;
110
+ for (const t of def(record.tests, [])) {
111
+ if (isFailureWithError(t))
112
+ rows.push(toCommonFailureRow(record, t));
113
+ }
114
+ }
115
+ return rows;
116
+ }
117
+ detectRegressionCandidates(suite, minGreen, recentFailures) {
118
+ if (!isReadable(this.store))
119
+ return [];
120
+ const testNames = new Set();
121
+ for (const record of this.store.readAll()) {
122
+ if (suite && record.suite !== suite)
123
+ continue;
124
+ for (const t of def(record.tests, []))
125
+ testNames.add(t.test_name);
126
+ }
127
+ const candidates = [];
128
+ for (const name of testNames) {
129
+ const timeline = this.store.queryTimeline(name);
130
+ const result = detectRegressions(timeline, minGreen, recentFailures);
131
+ if (result.is_regression) {
132
+ candidates.push({
133
+ test_name: name,
134
+ suite: timeline.length > 0 ? timeline[0].suite : '',
135
+ area: null,
136
+ green_streak: result.green_streak,
137
+ ...(result.first_failure_commit
138
+ ? { first_failure_commit: result.first_failure_commit }
139
+ : {}),
140
+ });
141
+ }
142
+ }
143
+ return candidates;
144
+ }
145
+ }
146
+ //# sourceMappingURL=engine.js.map
Binary file
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Row shapes consumed by the report builders.
3
+ *
4
+ * These mirror the pre-fetched query rows that the Python `reports.py` builders
5
+ * receive from the history store — one shape per report. Ported field-for-field
6
+ * so the TS builders produce identical output (see the parity harness).
7
+ */
8
+ export {};
9
+ //# sourceMappingURL=rows.js.map