canary-test-cli 5.15.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/frameworks/registry.json +655 -0
- package/bin/canary.js +20 -15
- package/dist/doctor-manifest.d.ts +94 -0
- package/dist/doctor.d.ts +67 -0
- package/dist/engine/analysis/cli.js +270 -0
- package/dist/engine/analysis/engine.js +146 -0
- package/dist/engine/analysis/reports.js +0 -0
- package/dist/engine/analysis/rows.js +9 -0
- package/dist/engine/cli-commands.js +618 -0
- package/dist/engine/cli-common.js +60 -0
- package/dist/engine/cli.core.js +208 -0
- package/dist/engine/cli.js +31 -0
- package/dist/engine/company-knowledge-cli.js +201 -0
- package/dist/engine/core/ci-env.js +33 -0
- package/dist/engine/core/classifier.js +192 -0
- package/dist/engine/core/company-knowledge.js +765 -0
- package/dist/engine/core/config-validation.js +74 -0
- package/dist/engine/core/detection.js +48 -0
- package/dist/engine/core/domain-scanner.js +212 -0
- package/dist/engine/core/environment-detect.js +410 -0
- package/dist/engine/core/executor.js +181 -0
- package/dist/engine/core/feedback.js +93 -0
- package/dist/engine/core/fixture-scanner.js +173 -0
- package/dist/engine/core/framework-registry.js +123 -0
- package/dist/engine/core/mcp-validator.js +218 -0
- package/dist/engine/core/metadata-scanner.js +147 -0
- package/dist/engine/core/migrator.js +1112 -0
- package/dist/engine/core/overlays.js +176 -0
- package/dist/engine/core/pattern-healer.js +147 -0
- package/dist/engine/core/pattern-matcher.js +255 -0
- package/dist/engine/core/quality-scorer.js +213 -0
- package/dist/engine/core/recommender.js +152 -0
- package/dist/engine/core/reporter.js +211 -0
- package/dist/engine/core/scaffolder.js +236 -0
- package/dist/engine/core/skill-registry.js +522 -0
- package/dist/engine/core/static-linter.js +237 -0
- package/dist/engine/core/ticket-updater.js +639 -0
- package/dist/engine/core/workflow-discovery.js +693 -0
- package/dist/engine/guardian/agent-tier.js +338 -0
- package/dist/engine/guardian/analysis-emit.js +201 -0
- package/dist/engine/guardian/cli.js +787 -0
- package/dist/engine/guardian/coverage.js +1055 -0
- package/dist/engine/guardian/delta-emitter.js +46 -0
- package/dist/engine/guardian/diff-extractor.js +257 -0
- package/dist/engine/guardian/hard-gate.js +373 -0
- package/dist/engine/guardian/impact-mapper.js +121 -0
- package/dist/engine/guardian/pr-check.js +975 -0
- package/dist/engine/guardian/pr-comment.js +200 -0
- package/dist/engine/guardian/summary-emitter.js +94 -0
- package/dist/engine/guardian/tier.js +58 -0
- package/dist/engine/history/cli.js +303 -0
- package/dist/engine/history/detector.js +68 -0
- package/dist/engine/history/ndjson-store.js +177 -0
- package/dist/engine/history/record.js +14 -0
- package/dist/engine/history/schema.js +59 -0
- package/dist/engine/history/store.js +47 -0
- package/dist/engine/history/supabase-store.js +113 -0
- package/dist/engine/main-deps.js +105 -0
- package/dist/engine/mcp-server.js +647 -0
- package/dist/engine/package.json +4 -0
- package/dist/engine/skills-cli.js +181 -0
- package/dist/engine/ui/banner.js +50 -0
- package/dist/engine/util/coalesce.js +12 -0
- package/dist/engine/util/round.js +43 -0
- package/dist/engine/workflow-cli.js +242 -0
- package/dist/engine-checks.d.ts +49 -0
- package/dist/overlay-commands.d.ts +81 -0
- package/dist/overlay-conflicts.d.ts +33 -0
- package/dist/overlay-lint.d.ts +19 -0
- package/dist/overlays-registry.d.ts +74 -0
- package/dist/reporters/testtracker.d.ts +89 -0
- package/dist/reporters/testtracker.js +195 -0
- package/dist/router.d.ts +12 -0
- package/dist/router.js +4 -4
- package/dist/skill-requirements.d.ts +57 -0
- package/dist/source-spec.d.ts +20 -0
- package/package.json +30 -6
- package/bin/canary +0 -0
- package/scripts/install.js +0 -104
package/bin/canary.js
CHANGED
|
@@ -6,35 +6,40 @@ const path = require('node:path');
|
|
|
6
6
|
const fs = require('node:fs');
|
|
7
7
|
const { isTsCommand, route } = require('../dist/router.js');
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
/**
|
|
10
|
+
* Absolute path to the bundled TypeScript engine's runnable entry
|
|
11
|
+
* (`dist/engine/cli.js`, generated by scripts/build-engine.mjs). Runs under the
|
|
12
|
+
* current `node` via `execFileSync(process.execPath, [enginePath, ...])`.
|
|
13
|
+
*/
|
|
14
|
+
function getEnginePath() {
|
|
15
|
+
return path.join(__dirname, '..', 'dist', 'engine', 'cli.js');
|
|
12
16
|
}
|
|
13
17
|
|
|
14
18
|
/**
|
|
15
|
-
* Forward a command to the bundled
|
|
16
|
-
* (0 on success, the
|
|
19
|
+
* Forward a command to the bundled TypeScript engine. Returns the exit code
|
|
20
|
+
* (0 on success, the engine's status on failure, 1 when the engine is missing).
|
|
17
21
|
* Dependencies are injectable for testing.
|
|
18
22
|
*/
|
|
19
|
-
function
|
|
23
|
+
function forwardToEngine(
|
|
20
24
|
argv,
|
|
21
25
|
{
|
|
22
26
|
execFile = execFileSync,
|
|
23
27
|
existsSync = fs.existsSync,
|
|
24
|
-
|
|
28
|
+
execPath = process.execPath,
|
|
25
29
|
stderr = process.stderr,
|
|
26
30
|
} = {},
|
|
27
31
|
) {
|
|
28
|
-
const
|
|
29
|
-
if (!existsSync(
|
|
32
|
+
const enginePath = getEnginePath();
|
|
33
|
+
if (!existsSync(enginePath)) {
|
|
30
34
|
stderr.write(
|
|
31
|
-
`canary
|
|
32
|
-
`
|
|
35
|
+
`canary engine not found at ${enginePath}.\n` +
|
|
36
|
+
`The package looks incomplete; try reinstalling: npm install -g canary-test-cli\n`,
|
|
33
37
|
);
|
|
34
38
|
return 1;
|
|
35
39
|
}
|
|
36
40
|
try {
|
|
37
|
-
|
|
41
|
+
// Inherit stdio AND cwd so the engine operates on the user's project.
|
|
42
|
+
execFile(execPath, [enginePath, ...argv], { stdio: 'inherit' });
|
|
38
43
|
return 0;
|
|
39
44
|
} catch (err) {
|
|
40
45
|
return err.status ?? 1;
|
|
@@ -43,14 +48,14 @@ function forwardToBinary(
|
|
|
43
48
|
|
|
44
49
|
/**
|
|
45
50
|
* Dispatch one invocation: TS-handled commands (e.g. `overlay`, `doctor`) go to
|
|
46
|
-
* the router; everything else forwards verbatim to the
|
|
51
|
+
* the router; everything else forwards verbatim to the bundled engine. Returns
|
|
47
52
|
* the process exit code, or a Promise of one for async commands (`doctor`).
|
|
48
53
|
*/
|
|
49
54
|
function run(argv, deps = {}) {
|
|
50
55
|
if (isTsCommand(argv)) {
|
|
51
56
|
return route(argv, deps) ?? 0;
|
|
52
57
|
}
|
|
53
|
-
return
|
|
58
|
+
return forwardToEngine(argv, deps);
|
|
54
59
|
}
|
|
55
60
|
|
|
56
61
|
function main() {
|
|
@@ -60,5 +65,5 @@ function main() {
|
|
|
60
65
|
);
|
|
61
66
|
}
|
|
62
67
|
|
|
63
|
-
module.exports = {
|
|
68
|
+
module.exports = { getEnginePath, forwardToEngine, run };
|
|
64
69
|
if (require.main === module) main();
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import type { CheckResult } from './doctor.js';
|
|
2
|
+
export declare const CHECK_TYPES: readonly ["file-exists", "url-reachable", "command-succeeds"];
|
|
3
|
+
export type CheckType = (typeof CHECK_TYPES)[number];
|
|
4
|
+
/** A single validated check from an overlay's `doctor.json`. */
|
|
5
|
+
export interface ManifestCheck {
|
|
6
|
+
id: string;
|
|
7
|
+
type: CheckType;
|
|
8
|
+
/** Shown under the check line when it fails. */
|
|
9
|
+
remedy: string;
|
|
10
|
+
/**
|
|
11
|
+
* Free-form audience tags (#319 B). A check with none runs for every
|
|
12
|
+
* audience; otherwise it runs only when `--audience <tag>` matches one.
|
|
13
|
+
* Declared as `audience:` in doctor.json (legacy alias: `persona:`).
|
|
14
|
+
*
|
|
15
|
+
* NB: unrelated to harness's persona system (`run_persona` /
|
|
16
|
+
* `generate_persona_artifacts`). This is a canary-local check-grouping tag;
|
|
17
|
+
* the field was renamed off "persona" to end that collision.
|
|
18
|
+
*/
|
|
19
|
+
audience?: string[];
|
|
20
|
+
/** `file-exists`: path relative to the overlay clone. */
|
|
21
|
+
path?: string;
|
|
22
|
+
/** `url-reachable`: the URL to probe. */
|
|
23
|
+
url?: string;
|
|
24
|
+
/** `command-succeeds`: argv array run (no shell) in the clone dir. */
|
|
25
|
+
command?: string[];
|
|
26
|
+
}
|
|
27
|
+
/** Result of loading a manifest: either its checks, or a single failing check. */
|
|
28
|
+
export type ManifestLoad = {
|
|
29
|
+
ok: true;
|
|
30
|
+
checks: ManifestCheck[];
|
|
31
|
+
} | {
|
|
32
|
+
ok: false;
|
|
33
|
+
failure: CheckResult;
|
|
34
|
+
};
|
|
35
|
+
/** Path to an overlay's manifest. */
|
|
36
|
+
export declare function manifestPath(cloneDir: string): string;
|
|
37
|
+
/**
|
|
38
|
+
* Stable fingerprint of a manifest's `command-succeeds` checks — the (id,
|
|
39
|
+
* command) pairs, sorted by id. Returns null when there are no such checks
|
|
40
|
+
* (nothing to gate). Consent is re-requested when this value changes.
|
|
41
|
+
*/
|
|
42
|
+
export declare function commandSucceedsHash(checks: ManifestCheck[]): string | null;
|
|
43
|
+
/**
|
|
44
|
+
* Load and validate `<clone>/.canary/doctor.json`. A missing file is not an
|
|
45
|
+
* error (no checks). A malformed file or an invalid check degrades to a single
|
|
46
|
+
* failing check ({@link ManifestLoad} `ok: false`) — never a throw.
|
|
47
|
+
*/
|
|
48
|
+
export declare function loadManifest(cloneDir: string): ManifestLoad;
|
|
49
|
+
/**
|
|
50
|
+
* The distinct audience tags declared across a set of checks, in first-seen
|
|
51
|
+
* order and de-duplicated case-insensitively (original casing preserved for
|
|
52
|
+
* display). This is the discoverable audience *vocabulary* — the engine ships
|
|
53
|
+
* none of its own, so it is derived entirely from overlay manifests. Used to
|
|
54
|
+
* tell a user which `--audience` values actually mean something (issue #294).
|
|
55
|
+
*/
|
|
56
|
+
export declare function collectAudiences(checks: ManifestCheck[]): string[];
|
|
57
|
+
/**
|
|
58
|
+
* Keep checks that should run for `audience`: a null audience runs everything;
|
|
59
|
+
* otherwise keep checks with no audience plus those whose audience list
|
|
60
|
+
* contains the tag (case-insensitive).
|
|
61
|
+
*/
|
|
62
|
+
export declare function filterByAudience(checks: ManifestCheck[], audience: string | null): ManifestCheck[];
|
|
63
|
+
/** Default per-check timeout for url and command checks. */
|
|
64
|
+
export declare const DEFAULT_CHECK_TIMEOUT_MS = 10000;
|
|
65
|
+
/** Probe a URL for reachability (injectable). Resolves true on a 2xx/3xx. */
|
|
66
|
+
export type UrlProbe = (url: string, timeoutMs: number) => Promise<boolean>;
|
|
67
|
+
/** Run a command (injectable). `ok` = exit 0; `timedOut` = killed at the timeout. */
|
|
68
|
+
export type CommandRunner = (command: string[], cwd: string, timeoutMs: number, extraEnv?: NodeJS.ProcessEnv) => {
|
|
69
|
+
ok: boolean;
|
|
70
|
+
timedOut: boolean;
|
|
71
|
+
detail?: string;
|
|
72
|
+
};
|
|
73
|
+
/** Context for executing checks against one overlay clone. */
|
|
74
|
+
export interface RunContext {
|
|
75
|
+
cloneDir: string;
|
|
76
|
+
/**
|
|
77
|
+
* Directory `canary doctor` was invoked from (the consuming repo root).
|
|
78
|
+
* Exposed to `command-succeeds` checks as `CANARY_INVOCATION_DIR` so a check
|
|
79
|
+
* can anchor to consuming-repo runtime artifacts instead of the overlay
|
|
80
|
+
* clone, which is always the cwd (#378).
|
|
81
|
+
*/
|
|
82
|
+
invocationDir?: string;
|
|
83
|
+
/** Whether this overlay's `command-succeeds` checks may execute. */
|
|
84
|
+
consentGranted: boolean;
|
|
85
|
+
timeoutMs?: number;
|
|
86
|
+
probeUrl?: UrlProbe;
|
|
87
|
+
runCommand?: CommandRunner;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Execute one validated check against its overlay clone, under a bounded
|
|
91
|
+
* timeout. `command-succeeds` is skipped (not failed) unless consent is
|
|
92
|
+
* granted. Never throws.
|
|
93
|
+
*/
|
|
94
|
+
export declare function runCheck(check: ManifestCheck, ctx: RunContext): Promise<CheckResult>;
|
package/dist/doctor.d.ts
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { CommandDeps } from './overlay-commands.js';
|
|
2
|
+
import { type CommandRunner, type UrlProbe } from './doctor-manifest.js';
|
|
3
|
+
/** Outcome of a single doctor check. */
|
|
4
|
+
export type CheckStatus = 'pass' | 'fail' | 'skip' | 'info';
|
|
5
|
+
/** A single doctor check result, rendered as one output line. */
|
|
6
|
+
export interface CheckResult {
|
|
7
|
+
/** Stable identifier (engine check id, or the manifest check's `id`). */
|
|
8
|
+
id: string;
|
|
9
|
+
status: CheckStatus;
|
|
10
|
+
/** Human-readable label for the check line. */
|
|
11
|
+
label: string;
|
|
12
|
+
/** Shown indented under the line when the check does not pass. */
|
|
13
|
+
remedy?: string;
|
|
14
|
+
}
|
|
15
|
+
/** Dependencies for `doctor`, injectable for tests (real ones by default). */
|
|
16
|
+
export interface DoctorDeps extends CommandDeps {
|
|
17
|
+
/** Current working directory (for project `.canary/` and `.mcp.json`). */
|
|
18
|
+
cwd?: string;
|
|
19
|
+
currentVersion?: string;
|
|
20
|
+
getLatestVersion?: () => Promise<string | null>;
|
|
21
|
+
probeUrl?: UrlProbe;
|
|
22
|
+
runCommand?: CommandRunner;
|
|
23
|
+
timeoutMs?: number;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* The `canary doctor --json` machine contract (issue #318). Canary-owned and
|
|
27
|
+
* intentionally distinct from `harness doctor --json`: only `allPassed` matches
|
|
28
|
+
* harness's top-level shape; per-check fields (`id`/`label`/`remedy`/`group`)
|
|
29
|
+
* and the `skip` status tier are canary's own. `version` guards the contract so
|
|
30
|
+
* consumers can detect a breaking change.
|
|
31
|
+
*/
|
|
32
|
+
export interface JsonReportCheck {
|
|
33
|
+
/** Stable check identifier (engine check id, or the manifest check's `id`). */
|
|
34
|
+
id: string;
|
|
35
|
+
status: CheckStatus;
|
|
36
|
+
label: string;
|
|
37
|
+
/** Present only when the check did not pass. */
|
|
38
|
+
remedy?: string;
|
|
39
|
+
/** The section the check belongs to, e.g. `"Engine"` or `"Overlay: acme"`. */
|
|
40
|
+
group: string;
|
|
41
|
+
}
|
|
42
|
+
export interface JsonReport {
|
|
43
|
+
version: 1;
|
|
44
|
+
checks: JsonReportCheck[];
|
|
45
|
+
/** True iff no check failed — mirrors `harness doctor --json`'s one shared field. */
|
|
46
|
+
allPassed: boolean;
|
|
47
|
+
/** Non-fatal advisories (e.g. an unknown `--audience`); empty when none. */
|
|
48
|
+
warnings: string[];
|
|
49
|
+
}
|
|
50
|
+
/** `--json` requests machine output on stdout instead of the human report. */
|
|
51
|
+
export declare function parseJsonFlag(args: readonly string[]): boolean;
|
|
52
|
+
/**
|
|
53
|
+
* Fail-loud hint for an unrecognized `--audience` value (issue #294). Returns
|
|
54
|
+
* null when there is nothing to say — no audience was passed, or the passed
|
|
55
|
+
* audience is part of the known vocabulary. Otherwise returns a one-line,
|
|
56
|
+
* actionable message: the engine ships no audience vocabulary, so this lists
|
|
57
|
+
* the tags overlays actually declared (or says none are defined) instead of
|
|
58
|
+
* silently running only the audience-less checks and leaving the user to
|
|
59
|
+
* guess why their filter matched nothing.
|
|
60
|
+
*/
|
|
61
|
+
export declare function unknownAudienceHint(audience: string | null, known: readonly string[]): string | null;
|
|
62
|
+
/**
|
|
63
|
+
* Run `canary doctor`. Returns a process exit code: 0 when every check passed
|
|
64
|
+
* or was skipped/info, non-zero when any check failed. A malformed manifest for
|
|
65
|
+
* one overlay never blocks engine checks or other overlays.
|
|
66
|
+
*/
|
|
67
|
+
export declare function runDoctor(args: readonly string[], deps?: DoctorDeps): Promise<number>;
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLI subcommands for `canary analyze` -- faithful port of
|
|
3
|
+
* `agent/analysis/cli.py` (the `analyze_app` Typer sub-app), wired to the
|
|
4
|
+
* already-ported analysis engine + report builders (`engine.ts`, `reports.ts`,
|
|
5
|
+
* `rows.ts`) and the local NDJSON history store.
|
|
6
|
+
*
|
|
7
|
+
* Follows the guardian CLI conventions (see `../cli-common.ts`): a
|
|
8
|
+
* {@link createAnalyzeCommand} factory wired to an injectable {@link AnalyzeDeps},
|
|
9
|
+
* and `normalizeUsageExit` on every command so usage errors exit 2. No command
|
|
10
|
+
* raises a business exit -- every analyze subcommand returns 0 (matching Python).
|
|
11
|
+
*
|
|
12
|
+
* Python->TS fidelity notes:
|
|
13
|
+
* - `json.dumps(x, indent=2)` -> {@link jsonIndent2} (byte-exact + ensure_ascii).
|
|
14
|
+
* - The report builders are byte-exact ports (Markdown), so the human-readable
|
|
15
|
+
* paths match the oracle exactly.
|
|
16
|
+
* - INTENTIONAL DEVIATION: the TS analysis engine (`engine.ts`) operates on the
|
|
17
|
+
* LOCAL NDJSON store only -- the JS Supabase SDK is async and the engine's
|
|
18
|
+
* query surface is synchronous. `--db-url` is still accepted (faithful CLI
|
|
19
|
+
* surface) but the TS port always reads the local store, exactly as the
|
|
20
|
+
* Python `isinstance(store, LocalHistoryStore)` read-path does for the
|
|
21
|
+
* spikes/common-failures commands. No Python analyze test exercises a remote
|
|
22
|
+
* store.
|
|
23
|
+
* - `area-health` accepts `--json` but ignores it -- faithful to the Python
|
|
24
|
+
* command, which never branches on `output_json`.
|
|
25
|
+
*/
|
|
26
|
+
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
27
|
+
import { join } from 'node:path';
|
|
28
|
+
import { Command, Option } from 'commander';
|
|
29
|
+
import { jsonIndent2, normalizeUsageExit } from '../cli-common.js';
|
|
30
|
+
import { AnalysisEngine } from './engine.js';
|
|
31
|
+
import { buildAreaHealthReport, buildCommonFailuresReport, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
|
|
32
|
+
import { NdjsonHistoryStore } from '../history/ndjson-store.js';
|
|
33
|
+
const DEFAULT_HISTORY_PATH = 'test-results/reports/history-v2.jsonl';
|
|
34
|
+
/** Process-backed defaults for production. */
|
|
35
|
+
export function defaultAnalyzeDeps() {
|
|
36
|
+
const err = (s) => {
|
|
37
|
+
process.stderr.write(`${s}\n`);
|
|
38
|
+
};
|
|
39
|
+
return {
|
|
40
|
+
out: (s) => process.stdout.write(`${s}\n`),
|
|
41
|
+
err,
|
|
42
|
+
env: process.env,
|
|
43
|
+
// The ported analysis engine's query surface is synchronous, so it cannot
|
|
44
|
+
// drive the async Supabase store; analyze reads local NDJSON only. Python
|
|
45
|
+
// honors --db-url / CANARY_HISTORY_DB_URL via make_store, so warn (to stderr,
|
|
46
|
+
// not stdout -- keeps --json clean) rather than SILENTLY reading a different
|
|
47
|
+
// data source. Full remote support is deferred with the async engine port.
|
|
48
|
+
makeStore: (dbUrl) => {
|
|
49
|
+
if (dbUrl) {
|
|
50
|
+
err('note: --db-url is ignored by analyze; it reads local NDJSON only.');
|
|
51
|
+
}
|
|
52
|
+
return new NdjsonHistoryStore(DEFAULT_HISTORY_PATH);
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
function writeArtifacts(artifacts, output) {
|
|
57
|
+
mkdirSync(output, { recursive: true });
|
|
58
|
+
for (const [name, content] of Object.entries(artifacts)) {
|
|
59
|
+
writeFileSync(join(output, name), content, 'utf-8');
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
function flakyCmd(opts, deps) {
|
|
63
|
+
const store = deps.makeStore(opts.dbUrl);
|
|
64
|
+
const rows = store.queryFlaky(opts.window, opts.suite ?? null, opts.minRate);
|
|
65
|
+
if (opts.json) {
|
|
66
|
+
deps.out(jsonIndent2(rows));
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
deps.out(buildFlakyReport(rows, opts.window, opts.minRate));
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
function spikesCmd(opts, deps) {
|
|
73
|
+
const store = deps.makeStore(opts.dbUrl);
|
|
74
|
+
const rows = [];
|
|
75
|
+
for (const r of store.readAll()) {
|
|
76
|
+
if (opts.since && (r.timestamp ?? '') < opts.since)
|
|
77
|
+
continue;
|
|
78
|
+
rows.push({
|
|
79
|
+
suite: r.suite ?? '',
|
|
80
|
+
timestamp: r.timestamp ?? '',
|
|
81
|
+
passed: r.passed ?? 0,
|
|
82
|
+
failed: r.failed ?? 0,
|
|
83
|
+
flaky: r.flaky ?? 0,
|
|
84
|
+
total: r.total ?? 0,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
if (opts.json) {
|
|
88
|
+
deps.out(jsonIndent2(rows));
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
deps.out(buildSpikesReport(rows, opts.delta));
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
function areaHealthCmd(opts, deps) {
|
|
95
|
+
// Faithful to Python: always builds from an empty row set and never branches
|
|
96
|
+
// on --json.
|
|
97
|
+
deps.out(buildAreaHealthReport([], opts.weeks));
|
|
98
|
+
}
|
|
99
|
+
function commonFailuresCmd(opts, deps) {
|
|
100
|
+
const store = deps.makeStore(opts.dbUrl);
|
|
101
|
+
const rows = [];
|
|
102
|
+
for (const record of store.readAll()) {
|
|
103
|
+
if (opts.since && (record.timestamp ?? '') < opts.since)
|
|
104
|
+
continue;
|
|
105
|
+
for (const t of record.tests ?? []) {
|
|
106
|
+
if ((t.status === 'failed' || t.status === 'flaky') && t.error_text) {
|
|
107
|
+
rows.push({
|
|
108
|
+
test_name: t.test_name,
|
|
109
|
+
suite: record.suite ?? '',
|
|
110
|
+
failure_category: t.failure_category ?? 'other',
|
|
111
|
+
error_text: t.error_text ?? '',
|
|
112
|
+
run_count: 1,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
if (opts.json) {
|
|
118
|
+
deps.out(jsonIndent2(rows));
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
deps.out(buildCommonFailuresReport(rows, opts.minSuites));
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
function regressionCandidatesCmd(opts, deps) {
|
|
125
|
+
const engine = new AnalysisEngine(deps.makeStore(opts.dbUrl));
|
|
126
|
+
const candidates = engine.detectRegressionCandidates(null, opts.minGreen, opts.recentFailures);
|
|
127
|
+
if (opts.json) {
|
|
128
|
+
deps.out(jsonIndent2(candidates));
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
deps.out(buildRegressionCandidatesReport(candidates));
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
function digestCmd(opts, deps) {
|
|
135
|
+
const engine = new AnalysisEngine(deps.makeStore(opts.dbUrl));
|
|
136
|
+
const result = engine.run({
|
|
137
|
+
window: opts.window,
|
|
138
|
+
delta: opts.delta,
|
|
139
|
+
weeks: opts.weeks,
|
|
140
|
+
minSuites: opts.minSuites,
|
|
141
|
+
suite: opts.suite ?? null,
|
|
142
|
+
});
|
|
143
|
+
writeArtifacts(result.artifacts, opts.output);
|
|
144
|
+
if (opts.json) {
|
|
145
|
+
deps.out(jsonIndent2({
|
|
146
|
+
flaky_count: result.flaky.length,
|
|
147
|
+
spike_count: result.spikes.length,
|
|
148
|
+
regression_count: result.regressionCandidates.length,
|
|
149
|
+
}));
|
|
150
|
+
}
|
|
151
|
+
else if (opts.slack) {
|
|
152
|
+
printSlack(result.flaky.length, result.regressionCandidates.length, deps);
|
|
153
|
+
}
|
|
154
|
+
else {
|
|
155
|
+
deps.out(result.digestMd);
|
|
156
|
+
deps.out(`\nArtifacts written to ${opts.output}/`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
function printSlack(flakyCount, regCount, deps) {
|
|
160
|
+
// U+2022 bullet, U+2265 >=.
|
|
161
|
+
const lines = [
|
|
162
|
+
'*Fleet Health Digest*',
|
|
163
|
+
`\u{2022} Flakeys \u{2265} 10%: ${flakyCount}`,
|
|
164
|
+
`\u{2022} Regression candidates: ${regCount}`,
|
|
165
|
+
];
|
|
166
|
+
deps.out(lines.join('\n'));
|
|
167
|
+
}
|
|
168
|
+
// --- assembly ----------------------------------------------------------------
|
|
169
|
+
/** Build a fresh `analyze` command wired to `depsInit`. */
|
|
170
|
+
export function createAnalyzeCommand(depsInit = {}) {
|
|
171
|
+
const deps = { ...defaultAnalyzeDeps(), ...depsInit };
|
|
172
|
+
const program = new Command('analyze');
|
|
173
|
+
program
|
|
174
|
+
.description('Cross-suite fleet health analysis.')
|
|
175
|
+
.exitOverride(normalizeUsageExit);
|
|
176
|
+
program
|
|
177
|
+
.command('flaky')
|
|
178
|
+
.description('Fleet-wide flake leaderboard.')
|
|
179
|
+
.addOption(new Option('-w, --window <n>')
|
|
180
|
+
.default(30)
|
|
181
|
+
.argParser((v) => Number.parseInt(v, 10)))
|
|
182
|
+
.option('-s, --suite <suite>')
|
|
183
|
+
.addOption(new Option('--min-rate <pct>')
|
|
184
|
+
.default(10.0)
|
|
185
|
+
.argParser((v) => Number.parseFloat(v)))
|
|
186
|
+
.addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
|
|
187
|
+
.option('--json')
|
|
188
|
+
.action((opts) => {
|
|
189
|
+
flakyCmd(opts, deps);
|
|
190
|
+
});
|
|
191
|
+
program
|
|
192
|
+
.command('spikes')
|
|
193
|
+
.description('Recent failure spikes across suites.')
|
|
194
|
+
.option('--since <date>', 'ISO date filter, e.g. 2026-06-01')
|
|
195
|
+
.addOption(new Option('--delta <pp>')
|
|
196
|
+
.default(20.0)
|
|
197
|
+
.argParser((v) => Number.parseFloat(v)))
|
|
198
|
+
.addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
|
|
199
|
+
.option('--json')
|
|
200
|
+
.action((opts) => {
|
|
201
|
+
spikesCmd(opts, deps);
|
|
202
|
+
});
|
|
203
|
+
program
|
|
204
|
+
.command('area-health')
|
|
205
|
+
.description('Area degradation trends over time.')
|
|
206
|
+
.addOption(new Option('--weeks <n>')
|
|
207
|
+
.default(4)
|
|
208
|
+
.argParser((v) => Number.parseInt(v, 10)))
|
|
209
|
+
.addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
|
|
210
|
+
.option('--json')
|
|
211
|
+
.action((opts) => {
|
|
212
|
+
areaHealthCmd(opts, deps);
|
|
213
|
+
});
|
|
214
|
+
program
|
|
215
|
+
.command('common-failures')
|
|
216
|
+
.description('Cross-suite failure fingerprinting.')
|
|
217
|
+
.option('--since <date>')
|
|
218
|
+
.addOption(new Option('--min-suites <n>')
|
|
219
|
+
.default(2)
|
|
220
|
+
.argParser((v) => Number.parseInt(v, 10)))
|
|
221
|
+
.addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
|
|
222
|
+
.option('--json')
|
|
223
|
+
.action((opts) => {
|
|
224
|
+
commonFailuresCmd(opts, deps);
|
|
225
|
+
});
|
|
226
|
+
program
|
|
227
|
+
.command('regression-candidates')
|
|
228
|
+
.description('Tests newly and consistently broken after a green streak.')
|
|
229
|
+
.addOption(new Option('--min-green <n>')
|
|
230
|
+
.default(5)
|
|
231
|
+
.argParser((v) => Number.parseInt(v, 10)))
|
|
232
|
+
.addOption(new Option('--recent-failures <n>')
|
|
233
|
+
.default(3)
|
|
234
|
+
.argParser((v) => Number.parseInt(v, 10)))
|
|
235
|
+
.addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
|
|
236
|
+
.option('--json')
|
|
237
|
+
.action((opts) => {
|
|
238
|
+
regressionCandidatesCmd(opts, deps);
|
|
239
|
+
});
|
|
240
|
+
program
|
|
241
|
+
.command('digest')
|
|
242
|
+
.description('Combined digest of all five report types.')
|
|
243
|
+
.addOption(new Option('--window <n>')
|
|
244
|
+
.default(30)
|
|
245
|
+
.argParser((v) => Number.parseInt(v, 10)))
|
|
246
|
+
.addOption(new Option('--delta <pp>')
|
|
247
|
+
.default(20.0)
|
|
248
|
+
.argParser((v) => Number.parseFloat(v)))
|
|
249
|
+
.addOption(new Option('--weeks <n>')
|
|
250
|
+
.default(4)
|
|
251
|
+
.argParser((v) => Number.parseInt(v, 10)))
|
|
252
|
+
.addOption(new Option('--min-suites <n>')
|
|
253
|
+
.default(2)
|
|
254
|
+
.argParser((v) => Number.parseInt(v, 10)))
|
|
255
|
+
.option('--suite <suite>')
|
|
256
|
+
.addOption(new Option('--output <dir>').default('test-results/analysis'))
|
|
257
|
+
.option('--json')
|
|
258
|
+
.option('--slack')
|
|
259
|
+
.addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
|
|
260
|
+
.action((opts) => {
|
|
261
|
+
digestCmd(opts, deps);
|
|
262
|
+
});
|
|
263
|
+
for (const sub of program.commands) {
|
|
264
|
+
sub.exitOverride(normalizeUsageExit);
|
|
265
|
+
}
|
|
266
|
+
return program;
|
|
267
|
+
}
|
|
268
|
+
/** The production `analyze` command (process-backed defaults). */
|
|
269
|
+
export const analyzeCommand = createAnalyzeCommand();
|
|
270
|
+
//# sourceMappingURL=cli.js.map
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AnalysisEngine — queries the history store and runs all report types.
|
|
3
|
+
*
|
|
4
|
+
* Faithful TS port of `agent/analysis/engine.py`. Thin coordinator: calls the
|
|
5
|
+
* store's query methods, passes results to the pure builders in reports.ts, and
|
|
6
|
+
* returns structured data plus Markdown artifacts.
|
|
7
|
+
*
|
|
8
|
+
* Fidelity note: as in Python, `areaHealth` is intentionally never populated by
|
|
9
|
+
* `run()` (the Python engine initialises `area_rows = []` and never appends), so
|
|
10
|
+
* the area-health artifact always renders the empty-data message.
|
|
11
|
+
*/
|
|
12
|
+
import { buildAreaHealthReport, buildCommonFailuresReport, buildDigest, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
|
|
13
|
+
import { detectRegressions } from '../history/detector.js';
|
|
14
|
+
import { def } from '../util/coalesce.js';
|
|
15
|
+
function isReadable(store) {
|
|
16
|
+
return typeof store.readAll === 'function';
|
|
17
|
+
}
|
|
18
|
+
function isFailureWithError(t) {
|
|
19
|
+
return ((t.status === 'failed' || t.status === 'flaky') && Boolean(t.error_text));
|
|
20
|
+
}
|
|
21
|
+
function toCommonFailureRow(record, t) {
|
|
22
|
+
return {
|
|
23
|
+
test_name: t.test_name,
|
|
24
|
+
suite: def(record.suite, ''),
|
|
25
|
+
failure_category: def(t.failure_category, 'other'),
|
|
26
|
+
error_text: t.error_text,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
export class AnalysisEngine {
|
|
30
|
+
store;
|
|
31
|
+
constructor(store) {
|
|
32
|
+
this.store = store;
|
|
33
|
+
}
|
|
34
|
+
run(opts = {}) {
|
|
35
|
+
const window = opts.window ?? 30;
|
|
36
|
+
const delta = opts.delta ?? 20.0;
|
|
37
|
+
const weeks = opts.weeks ?? 4;
|
|
38
|
+
const minSuites = opts.minSuites ?? 2;
|
|
39
|
+
const minFlakeRate = opts.minFlakeRate ?? 10.0;
|
|
40
|
+
const minGreen = opts.minGreen ?? 5;
|
|
41
|
+
const recentFailures = opts.recentFailures ?? 3;
|
|
42
|
+
const suite = opts.suite ?? null;
|
|
43
|
+
const flaky = this.store.queryFlaky(window, suite, minFlakeRate);
|
|
44
|
+
const suitesToQuery = suite ? [suite] : this.discoverSuites();
|
|
45
|
+
const spikesRows = [];
|
|
46
|
+
for (const s of suitesToQuery) {
|
|
47
|
+
const summary = this.store.querySummary(s, window * 2);
|
|
48
|
+
for (const row of def(summary.runs, [])) {
|
|
49
|
+
// query_summary rows omit suite; the spikes builder groups by it, so
|
|
50
|
+
// tag each pooled row with the suite it came from (matches Python).
|
|
51
|
+
spikesRows.push({
|
|
52
|
+
suite: s,
|
|
53
|
+
timestamp: row.timestamp,
|
|
54
|
+
total: row.total,
|
|
55
|
+
failed: row.failed,
|
|
56
|
+
flaky: row.flaky,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
// Faithful to Python: area rows are never populated by run().
|
|
61
|
+
const areaRows = [];
|
|
62
|
+
const commonRows = this.queryCommonFailures(suite);
|
|
63
|
+
const regressionCandidates = this.detectRegressionCandidates(suite, minGreen, recentFailures);
|
|
64
|
+
const digest = buildDigest({
|
|
65
|
+
flaky,
|
|
66
|
+
spikes: spikesRows,
|
|
67
|
+
areaHealth: areaRows,
|
|
68
|
+
commonFailures: commonRows,
|
|
69
|
+
regressionCandidates,
|
|
70
|
+
window,
|
|
71
|
+
delta,
|
|
72
|
+
weeks,
|
|
73
|
+
minSuites,
|
|
74
|
+
});
|
|
75
|
+
const artifacts = {
|
|
76
|
+
'flaky.md': buildFlakyReport(flaky, window, minFlakeRate),
|
|
77
|
+
'spikes.md': buildSpikesReport(spikesRows, delta),
|
|
78
|
+
'area-health.md': buildAreaHealthReport(areaRows, weeks),
|
|
79
|
+
'common-failures.md': buildCommonFailuresReport(commonRows, minSuites),
|
|
80
|
+
'regression-candidates.md': buildRegressionCandidatesReport(regressionCandidates),
|
|
81
|
+
'digest.md': digest,
|
|
82
|
+
};
|
|
83
|
+
return {
|
|
84
|
+
flaky,
|
|
85
|
+
spikes: spikesRows,
|
|
86
|
+
areaHealth: areaRows,
|
|
87
|
+
commonFailures: commonRows,
|
|
88
|
+
regressionCandidates,
|
|
89
|
+
digestMd: digest,
|
|
90
|
+
artifacts,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
discoverSuites() {
|
|
94
|
+
if (!isReadable(this.store))
|
|
95
|
+
return [];
|
|
96
|
+
const suites = new Set();
|
|
97
|
+
for (const r of this.store.readAll()) {
|
|
98
|
+
if (r.suite)
|
|
99
|
+
suites.add(r.suite);
|
|
100
|
+
}
|
|
101
|
+
return [...suites];
|
|
102
|
+
}
|
|
103
|
+
queryCommonFailures(suite) {
|
|
104
|
+
if (!isReadable(this.store))
|
|
105
|
+
return [];
|
|
106
|
+
const rows = [];
|
|
107
|
+
for (const record of this.store.readAll()) {
|
|
108
|
+
if (suite && record.suite !== suite)
|
|
109
|
+
continue;
|
|
110
|
+
for (const t of def(record.tests, [])) {
|
|
111
|
+
if (isFailureWithError(t))
|
|
112
|
+
rows.push(toCommonFailureRow(record, t));
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return rows;
|
|
116
|
+
}
|
|
117
|
+
detectRegressionCandidates(suite, minGreen, recentFailures) {
|
|
118
|
+
if (!isReadable(this.store))
|
|
119
|
+
return [];
|
|
120
|
+
const testNames = new Set();
|
|
121
|
+
for (const record of this.store.readAll()) {
|
|
122
|
+
if (suite && record.suite !== suite)
|
|
123
|
+
continue;
|
|
124
|
+
for (const t of def(record.tests, []))
|
|
125
|
+
testNames.add(t.test_name);
|
|
126
|
+
}
|
|
127
|
+
const candidates = [];
|
|
128
|
+
for (const name of testNames) {
|
|
129
|
+
const timeline = this.store.queryTimeline(name);
|
|
130
|
+
const result = detectRegressions(timeline, minGreen, recentFailures);
|
|
131
|
+
if (result.is_regression) {
|
|
132
|
+
candidates.push({
|
|
133
|
+
test_name: name,
|
|
134
|
+
suite: timeline.length > 0 ? timeline[0].suite : '',
|
|
135
|
+
area: null,
|
|
136
|
+
green_streak: result.green_streak,
|
|
137
|
+
...(result.first_failure_commit
|
|
138
|
+
? { first_failure_commit: result.first_failure_commit }
|
|
139
|
+
: {}),
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return candidates;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
//# sourceMappingURL=engine.js.map
|
|
Binary file
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Row shapes consumed by the report builders.
|
|
3
|
+
*
|
|
4
|
+
* These mirror the pre-fetched query rows that the Python `reports.py` builders
|
|
5
|
+
* receive from the history store — one shape per report. Ported field-for-field
|
|
6
|
+
* so the TS builders produce identical output (see the parity harness).
|
|
7
|
+
*/
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=rows.js.map
|