canary-test-cli 6.4.0 → 6.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/doctor.d.ts +51 -3
- package/dist/doctor.js +76 -9
- package/dist/engine/analysis/cli.js +69 -6
- package/dist/engine/cli-commands.js +33 -0
- package/dist/engine/core/framework-registry.js +4 -1
- package/dist/engine/core/gate-result.js +9 -2
- package/dist/engine/core/scaffold-templates.js +123 -0
- package/dist/engine/core/scaffolder.js +4 -105
- package/dist/engine/core/static-linter.js +125 -13
- package/dist/engine/guardian/adjudication.js +364 -0
- package/dist/engine/guardian/analysis-emit.js +2 -0
- package/dist/engine/guardian/cli.js +282 -15
- package/dist/engine/guardian/hard-gate.js +15 -2
- package/dist/engine/guardian/pr-check.js +5 -12
- package/dist/engine/history/async-store.js +20 -0
- package/dist/engine/history/cli.js +67 -0
- package/dist/engine/history/ndjson-store.js +4 -0
- package/dist/engine/history/store.js +3 -0
- package/dist/gate-result.d.ts +67 -0
- package/dist/gate-result.js +73 -0
- package/dist/overlay-commands.js +17 -1
- package/package.json +2 -1
package/dist/doctor.d.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type { CommandDeps } from './overlay-commands.js';
|
|
2
|
+
import { type EngineCheckDeps } from './engine-checks.js';
|
|
2
3
|
import { type CommandRunner, type UrlProbe } from './doctor-manifest.js';
|
|
4
|
+
import { EXIT_ABSTAINED, type SkipEntry } from './gate-result.js';
|
|
5
|
+
export { EXIT_ABSTAINED };
|
|
3
6
|
/** Outcome of a single doctor check. */
|
|
4
7
|
export type CheckStatus = 'pass' | 'fail' | 'skip' | 'info';
|
|
5
8
|
/** A single doctor check result, rendered as one output line. */
|
|
@@ -21,6 +24,17 @@ export interface DoctorDeps extends CommandDeps {
|
|
|
21
24
|
probeUrl?: UrlProbe;
|
|
22
25
|
runCommand?: CommandRunner;
|
|
23
26
|
timeoutMs?: number;
|
|
27
|
+
/**
|
|
28
|
+
* Override the engine check set. A test seam only: it is the one input a
|
|
29
|
+
* hermetic run cannot control (engine checks read the real environment), and
|
|
30
|
+
* the abstention fixtures need a run whose denominator is exactly zero.
|
|
31
|
+
*/
|
|
32
|
+
runEngineChecks?: (deps: EngineCheckDeps) => Promise<CheckResult[]>;
|
|
33
|
+
}
|
|
34
|
+
/** One printed section: a header and its check results. */
|
|
35
|
+
export interface CheckGroup {
|
|
36
|
+
header: string;
|
|
37
|
+
results: CheckResult[];
|
|
24
38
|
}
|
|
25
39
|
/**
|
|
26
40
|
* The `canary doctor --json` machine contract (issue #318). Canary-owned and
|
|
@@ -46,6 +60,12 @@ export interface JsonReport {
|
|
|
46
60
|
allPassed: boolean;
|
|
47
61
|
/** Non-fatal advisories (e.g. an unknown `--audience`); empty when none. */
|
|
48
62
|
warnings: string[];
|
|
63
|
+
/** Checks that produced a verdict (`pass + fail`) -- the denominator (#508). */
|
|
64
|
+
checked: number;
|
|
65
|
+
/** Skipped checks, always reported; never folded into `allPassed` (D7). */
|
|
66
|
+
skipped: SkipEntry[];
|
|
67
|
+
/** True when nothing was verified: `allPassed` is false and the exit is 3. */
|
|
68
|
+
abstained: boolean;
|
|
49
69
|
}
|
|
50
70
|
/** `--json` requests machine output on stdout instead of the human report. */
|
|
51
71
|
export declare function parseJsonFlag(args: readonly string[]): boolean;
|
|
@@ -59,9 +79,37 @@ export declare function parseJsonFlag(args: readonly string[]): boolean;
|
|
|
59
79
|
* guess why their filter matched nothing.
|
|
60
80
|
*/
|
|
61
81
|
export declare function unknownAudienceHint(audience: string | null, known: readonly string[]): string | null;
|
|
82
|
+
/** Doctor's denominator, and the decision that follows from it (#508 D7). */
|
|
83
|
+
export interface DoctorSummary {
|
|
84
|
+
/** Checks that actually produced a verdict: `pass + fail`. */
|
|
85
|
+
checked: number;
|
|
86
|
+
passed: number;
|
|
87
|
+
failed: number;
|
|
88
|
+
/** Every skipped check, always visible, never folded into `passed`. */
|
|
89
|
+
skipped: SkipEntry[];
|
|
90
|
+
abstained: boolean;
|
|
91
|
+
exitCode: number;
|
|
92
|
+
summaryLine: string;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Compute doctor's denominator and summary line (#508 Wave 3, D7 / #505).
|
|
96
|
+
*
|
|
97
|
+
* Doctor used to count only failures, so a run in which EVERY check was
|
|
98
|
+
* skipped printed `All checks passed.` and exited 0 -- the doctrine violation
|
|
99
|
+
* that started #508. Here the denominator is explicit: `info` results are not
|
|
100
|
+
* verifications (they report context, not evidence), so a run with nothing but
|
|
101
|
+
* skips and info abstains.
|
|
102
|
+
*
|
|
103
|
+
* The decision (exit code + `abstained`) comes from `gateOutcome` and is never
|
|
104
|
+
* re-derived here; only the failure line keeps doctor's own vocabulary
|
|
105
|
+
* ("check(s) failed" rather than "finding(s)"), rendered with the helper's own
|
|
106
|
+
* skip suffix so the two can never drift.
|
|
107
|
+
*/
|
|
108
|
+
export declare function summarizeChecks(groups: readonly CheckGroup[]): DoctorSummary;
|
|
62
109
|
/**
|
|
63
|
-
* Run `canary doctor`. Returns a process exit code: 0 when
|
|
64
|
-
*
|
|
65
|
-
*
|
|
110
|
+
* Run `canary doctor`. Returns a process exit code: 0 when at least one check
|
|
111
|
+
* ran and none failed, 1 when any check failed, and `EXIT_ABSTAINED` (3) when
|
|
112
|
+
* nothing was actually verified (#508). A malformed manifest for one overlay
|
|
113
|
+
* never blocks engine checks or other overlays.
|
|
66
114
|
*/
|
|
67
115
|
export declare function runDoctor(args: readonly string[], deps?: DoctorDeps): Promise<number>;
|
package/dist/doctor.js
CHANGED
|
@@ -33,8 +33,10 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.EXIT_ABSTAINED = void 0;
|
|
36
37
|
exports.parseJsonFlag = parseJsonFlag;
|
|
37
38
|
exports.unknownAudienceHint = unknownAudienceHint;
|
|
39
|
+
exports.summarizeChecks = summarizeChecks;
|
|
38
40
|
exports.runDoctor = runDoctor;
|
|
39
41
|
/**
|
|
40
42
|
* `canary doctor` — environment self-check (Phase 2).
|
|
@@ -57,6 +59,8 @@ const os = __importStar(require("node:os"));
|
|
|
57
59
|
const engine_checks_js_1 = require("./engine-checks.js");
|
|
58
60
|
const doctor_manifest_js_1 = require("./doctor-manifest.js");
|
|
59
61
|
const registry = __importStar(require("./overlays-registry.js"));
|
|
62
|
+
const gate_result_js_1 = require("./gate-result.js");
|
|
63
|
+
Object.defineProperty(exports, "EXIT_ABSTAINED", { enumerable: true, get: function () { return gate_result_js_1.EXIT_ABSTAINED; } });
|
|
60
64
|
const SYMBOL = {
|
|
61
65
|
pass: '✓',
|
|
62
66
|
fail: '✗',
|
|
@@ -141,9 +145,60 @@ async function overlayResults(entry, deps, audience) {
|
|
|
141
145
|
return { group: { header, results }, loadedChecks: load.checks };
|
|
142
146
|
}
|
|
143
147
|
/**
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
148
|
+
* Compute doctor's denominator and summary line (#508 Wave 3, D7 / #505).
|
|
149
|
+
*
|
|
150
|
+
* Doctor used to count only failures, so a run in which EVERY check was
|
|
151
|
+
* skipped printed `All checks passed.` and exited 0 -- the doctrine violation
|
|
152
|
+
* that started #508. Here the denominator is explicit: `info` results are not
|
|
153
|
+
* verifications (they report context, not evidence), so a run with nothing but
|
|
154
|
+
* skips and info abstains.
|
|
155
|
+
*
|
|
156
|
+
* The decision (exit code + `abstained`) comes from `gateOutcome` and is never
|
|
157
|
+
* re-derived here; only the failure line keeps doctor's own vocabulary
|
|
158
|
+
* ("check(s) failed" rather than "finding(s)"), rendered with the helper's own
|
|
159
|
+
* skip suffix so the two can never drift.
|
|
160
|
+
*/
|
|
161
|
+
function summarizeChecks(groups) {
|
|
162
|
+
const results = groups.flatMap((g) => g.results);
|
|
163
|
+
const failures = results.filter((r) => r.status === 'fail');
|
|
164
|
+
const passed = results.filter((r) => r.status === 'pass').length;
|
|
165
|
+
const skipped = results
|
|
166
|
+
.filter((r) => r.status === 'skip')
|
|
167
|
+
.map((r) => ({ name: r.id, reason: r.remedy ?? r.label }));
|
|
168
|
+
const checked = passed + failures.length;
|
|
169
|
+
const outcome = (0, gate_result_js_1.gateOutcome)({ checked, findings: failures, skipped }, 'gate');
|
|
170
|
+
return {
|
|
171
|
+
checked,
|
|
172
|
+
passed,
|
|
173
|
+
failed: failures.length,
|
|
174
|
+
skipped,
|
|
175
|
+
abstained: outcome.abstained,
|
|
176
|
+
exitCode: outcome.exitCode,
|
|
177
|
+
summaryLine: failures.length > 0
|
|
178
|
+
? `${failures.length} check(s) failed${(0, gate_result_js_1.skippedSuffix)(skipped)}`
|
|
179
|
+
: outcome.summaryLine,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Remediation for an abstained run: why the denominator collapsed, and the
|
|
184
|
+
* first fix step. Required of every abstaining surface (spec: "an abstaining
|
|
185
|
+
* surface must say _why_ ... and the first fix step").
|
|
186
|
+
*/
|
|
187
|
+
function abstentionRemedy(summary) {
|
|
188
|
+
return summary.skipped.length > 0
|
|
189
|
+
? 'Every registered check was skipped or informational, so doctor verified ' +
|
|
190
|
+
'nothing. Grant command-check consent (re-run `canary overlay add ' +
|
|
191
|
+
'<name> --yes`) or install an overlay whose checks apply here, then ' +
|
|
192
|
+
're-run.'
|
|
193
|
+
: 'No check was registered, so doctor verified nothing. Install an ' +
|
|
194
|
+
'overlay that ships a `.canary/doctor.json` (`canary overlay add ' +
|
|
195
|
+
'<source>`), then re-run.';
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Run `canary doctor`. Returns a process exit code: 0 when at least one check
|
|
199
|
+
* ran and none failed, 1 when any check failed, and `EXIT_ABSTAINED` (3) when
|
|
200
|
+
* nothing was actually verified (#508). A malformed manifest for one overlay
|
|
201
|
+
* never blocks engine checks or other overlays.
|
|
147
202
|
*/
|
|
148
203
|
async function runDoctor(args, deps = {}) {
|
|
149
204
|
const out = deps.out ?? process.stdout;
|
|
@@ -157,8 +212,9 @@ async function runDoctor(args, deps = {}) {
|
|
|
157
212
|
getLatestVersion: deps.getLatestVersion,
|
|
158
213
|
timeoutMs: deps.timeoutMs,
|
|
159
214
|
};
|
|
215
|
+
const engineChecks = deps.runEngineChecks ?? engine_checks_js_1.runEngineChecks;
|
|
160
216
|
const groups = [
|
|
161
|
-
{ header: 'Engine', results: await (
|
|
217
|
+
{ header: 'Engine', results: await engineChecks(engineDeps) },
|
|
162
218
|
];
|
|
163
219
|
let reg;
|
|
164
220
|
try {
|
|
@@ -177,7 +233,10 @@ async function runDoctor(args, deps = {}) {
|
|
|
177
233
|
// tell them the valid vocabulary instead of silently filtering to only
|
|
178
234
|
// the audience-less checks and leaving them to wonder why.
|
|
179
235
|
const audienceHint = unknownAudienceHint(audience, (0, doctor_manifest_js_1.collectAudiences)(allChecks));
|
|
180
|
-
|
|
236
|
+
// #508: the denominator, not just the failure count. `summarizeChecks` is
|
|
237
|
+
// the only path to a summary line and an exit code, so doctor structurally
|
|
238
|
+
// cannot print a bare success over zero verified checks.
|
|
239
|
+
const summary = summarizeChecks(groups);
|
|
181
240
|
// Issue #318: `--json` emits the canary-owned machine contract instead of
|
|
182
241
|
// the human report — nothing else is written to stdout, so the whole stream
|
|
183
242
|
// parses as one JSON object.
|
|
@@ -191,11 +250,16 @@ async function runDoctor(args, deps = {}) {
|
|
|
191
250
|
...(r.remedy !== undefined ? { remedy: r.remedy } : {}),
|
|
192
251
|
group: g.header,
|
|
193
252
|
}))),
|
|
194
|
-
|
|
253
|
+
// #508: an abstained run is NOT "all passed" -- zero verified checks is
|
|
254
|
+
// an absent measurement, never a green.
|
|
255
|
+
allPassed: summary.failed === 0 && !summary.abstained,
|
|
195
256
|
warnings: audienceHint ? [audienceHint] : [],
|
|
257
|
+
checked: summary.checked,
|
|
258
|
+
skipped: summary.skipped,
|
|
259
|
+
abstained: summary.abstained,
|
|
196
260
|
};
|
|
197
261
|
out.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
198
|
-
return
|
|
262
|
+
return summary.exitCode;
|
|
199
263
|
}
|
|
200
264
|
out.write('canary doctor\n');
|
|
201
265
|
if (audienceHint) {
|
|
@@ -207,6 +271,9 @@ async function runDoctor(args, deps = {}) {
|
|
|
207
271
|
out.write(renderCheck(result));
|
|
208
272
|
}
|
|
209
273
|
}
|
|
210
|
-
out.write(`\n${
|
|
211
|
-
|
|
274
|
+
out.write(`\n${summary.summaryLine}\n`);
|
|
275
|
+
if (summary.abstained) {
|
|
276
|
+
out.write(` ${abstentionRemedy(summary)}\n`);
|
|
277
|
+
}
|
|
278
|
+
return summary.exitCode;
|
|
212
279
|
}
|
|
@@ -27,8 +27,9 @@ import { mkdirSync, writeFileSync } from 'node:fs';
|
|
|
27
27
|
import { join } from 'node:path';
|
|
28
28
|
import { Command, Option } from 'commander';
|
|
29
29
|
import { jsonIndent2, normalizeUsageExit } from '../cli-common.js';
|
|
30
|
+
import { gateOutcome } from '../core/gate-result.js';
|
|
30
31
|
import { AnalysisEngine } from './engine.js';
|
|
31
|
-
import {
|
|
32
|
+
import { buildCommonFailuresReport, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
|
|
32
33
|
import { NdjsonHistoryStore } from '../history/ndjson-store.js';
|
|
33
34
|
const DEFAULT_HISTORY_PATH = 'test-results/reports/history-v2.jsonl';
|
|
34
35
|
/** Process-backed defaults for production. */
|
|
@@ -53,6 +54,41 @@ export function defaultAnalyzeDeps() {
|
|
|
53
54
|
},
|
|
54
55
|
};
|
|
55
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* The shared denominator guard for every history-backed analyze report
|
|
59
|
+
* (#508 Wave 4a).
|
|
60
|
+
*
|
|
61
|
+
* The denominator is the number of RUNS in the store, never the number of
|
|
62
|
+
* result rows: zero flaky rows across 500 runs is a genuine clean fleet, while
|
|
63
|
+
* zero rows across zero runs is an absent measurement. Only `countRuns()`
|
|
64
|
+
* separates them -- keying off `rows.length` would abstain on every healthy
|
|
65
|
+
* fleet, which is the fastest way to get a doctrine muted (the katana lesson).
|
|
66
|
+
*
|
|
67
|
+
* Advisory (D3): the exit stays 0. On the human path the abstention line
|
|
68
|
+
* REPLACES the all-clear report; on `--json` the payload is a bare array with
|
|
69
|
+
* nowhere to put an `abstained` field, so stdout is left byte-identical and the
|
|
70
|
+
* notice rides stderr -- the same split `analyze` already uses for its
|
|
71
|
+
* `--db-url` note.
|
|
72
|
+
*
|
|
73
|
+
* Returns true when the caller should stop (the store was empty).
|
|
74
|
+
*/
|
|
75
|
+
function abstainOnEmptyHistory(store, deps, json, what) {
|
|
76
|
+
if (store.countRuns() > 0)
|
|
77
|
+
return false;
|
|
78
|
+
const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
|
|
79
|
+
const notice = `${outcome.summaryLine} No run history to analyze, so "${what}" is ` +
|
|
80
|
+
`unknown rather than clean. Record runs first ` +
|
|
81
|
+
`(\`canary history push\`, or a reporter that writes ` +
|
|
82
|
+
`${DEFAULT_HISTORY_PATH}), then re-run.`;
|
|
83
|
+
if (json) {
|
|
84
|
+
deps.out(jsonIndent2([]));
|
|
85
|
+
deps.err(notice);
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
deps.out(notice);
|
|
89
|
+
}
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
56
92
|
function writeArtifacts(artifacts, output) {
|
|
57
93
|
mkdirSync(output, { recursive: true });
|
|
58
94
|
for (const [name, content] of Object.entries(artifacts)) {
|
|
@@ -61,6 +97,9 @@ function writeArtifacts(artifacts, output) {
|
|
|
61
97
|
}
|
|
62
98
|
function flakyCmd(opts, deps) {
|
|
63
99
|
const store = deps.makeStore(opts.dbUrl);
|
|
100
|
+
if (abstainOnEmptyHistory(store, deps, opts.json === true, 'flake rate')) {
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
64
103
|
const rows = store.queryFlaky(opts.window, opts.suite ?? null, opts.minRate);
|
|
65
104
|
if (opts.json) {
|
|
66
105
|
deps.out(jsonIndent2(rows));
|
|
@@ -71,6 +110,9 @@ function flakyCmd(opts, deps) {
|
|
|
71
110
|
}
|
|
72
111
|
function spikesCmd(opts, deps) {
|
|
73
112
|
const store = deps.makeStore(opts.dbUrl);
|
|
113
|
+
if (abstainOnEmptyHistory(store, deps, opts.json === true, 'failure spikes')) {
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
74
116
|
const rows = [];
|
|
75
117
|
for (const r of store.readAll()) {
|
|
76
118
|
if (opts.since && (r.timestamp ?? '') < opts.since)
|
|
@@ -92,12 +134,25 @@ function spikesCmd(opts, deps) {
|
|
|
92
134
|
}
|
|
93
135
|
}
|
|
94
136
|
function areaHealthCmd(opts, deps) {
|
|
95
|
-
//
|
|
96
|
-
//
|
|
97
|
-
|
|
137
|
+
// #508 Wave 4a: this command builds its report from a HARDCODED empty row
|
|
138
|
+
// set (faithful to the Python original, which did the same and never branched
|
|
139
|
+
// on --json). Its denominator is therefore UNCONDITIONALLY zero -- with a
|
|
140
|
+
// thousand runs recorded it still renders "no area health data", which reads
|
|
141
|
+
// as a measured all-clear and is not one. So it always abstains, whatever the
|
|
142
|
+
// store holds. Wiring a real row set is a separate scope call: it changes the
|
|
143
|
+
// port's contract, and #515 deferred it for exactly that reason.
|
|
144
|
+
void opts;
|
|
145
|
+
const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
|
|
146
|
+
deps.out(`${outcome.summaryLine} \`analyze area-health\` computes no rows in this ` +
|
|
147
|
+
`build -- its report is a template, not a measurement, so a clean-looking ` +
|
|
148
|
+
`result here would be a fiction. Use \`analyze digest\` for the reports ` +
|
|
149
|
+
`that are wired, and track the area-health row set as unimplemented.`);
|
|
98
150
|
}
|
|
99
151
|
function commonFailuresCmd(opts, deps) {
|
|
100
152
|
const store = deps.makeStore(opts.dbUrl);
|
|
153
|
+
if (abstainOnEmptyHistory(store, deps, opts.json === true, 'common failures')) {
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
101
156
|
const rows = [];
|
|
102
157
|
for (const record of store.readAll()) {
|
|
103
158
|
if (opts.since && (record.timestamp ?? '') < opts.since)
|
|
@@ -122,7 +177,11 @@ function commonFailuresCmd(opts, deps) {
|
|
|
122
177
|
}
|
|
123
178
|
}
|
|
124
179
|
function regressionCandidatesCmd(opts, deps) {
|
|
125
|
-
const
|
|
180
|
+
const store = deps.makeStore(opts.dbUrl);
|
|
181
|
+
if (abstainOnEmptyHistory(store, deps, opts.json === true, 'regression candidates')) {
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
const engine = new AnalysisEngine(store);
|
|
126
185
|
const candidates = engine.detectRegressionCandidates(null, opts.minGreen, opts.recentFailures);
|
|
127
186
|
if (opts.json) {
|
|
128
187
|
deps.out(jsonIndent2(candidates));
|
|
@@ -132,7 +191,11 @@ function regressionCandidatesCmd(opts, deps) {
|
|
|
132
191
|
}
|
|
133
192
|
}
|
|
134
193
|
function digestCmd(opts, deps) {
|
|
135
|
-
const
|
|
194
|
+
const store = deps.makeStore(opts.dbUrl);
|
|
195
|
+
if (abstainOnEmptyHistory(store, deps, opts.json === true, 'fleet health')) {
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
const engine = new AnalysisEngine(store);
|
|
136
199
|
const result = engine.run({
|
|
137
200
|
window: opts.window,
|
|
138
201
|
delta: opts.delta,
|
|
@@ -14,6 +14,7 @@ import { existsSync, readdirSync, readFileSync, statSync, writeFileSync, } from
|
|
|
14
14
|
import { basename, join, resolve } from 'node:path';
|
|
15
15
|
import pc from 'picocolors';
|
|
16
16
|
import { CliExit, jsonIndent2 } from './cli-common.js';
|
|
17
|
+
import { gateOutcome } from './core/gate-result.js';
|
|
17
18
|
import { ckInitCmd } from './company-knowledge-cli.js';
|
|
18
19
|
import { extractFrameworkHint } from './core/classifier.js';
|
|
19
20
|
import { VALID_CATEGORIES, buildFeedback } from './core/feedback.js';
|
|
@@ -393,8 +394,39 @@ function findingPayload(f) {
|
|
|
393
394
|
suggestion: f.suggestion,
|
|
394
395
|
};
|
|
395
396
|
}
|
|
397
|
+
/**
|
|
398
|
+
* The denominator guard shared by the file-scanning gates (#508 Wave 4a).
|
|
399
|
+
*
|
|
400
|
+
* `review-test` and `flake-check` used to render a green all-clear whenever
|
|
401
|
+
* their finding list was empty -- indistinguishable from a run that scanned a
|
|
402
|
+
* directory matching zero test files. That is the #503 shape: a gate that
|
|
403
|
+
* verified nothing reporting a pass. Both are GATES (they carry an exit-code
|
|
404
|
+
* contract), so a collapsed denominator exits 3.
|
|
405
|
+
*
|
|
406
|
+
* Returns without throwing when at least one file was collected; the caller's
|
|
407
|
+
* normal rendering continues. `--json` keeps a parseable array on stdout -- only
|
|
408
|
+
* the exit code and the stderr notice carry the abstention.
|
|
409
|
+
*/
|
|
410
|
+
function abstainOnZeroFiles(files, path, deps, json) {
|
|
411
|
+
if (files.length > 0)
|
|
412
|
+
return;
|
|
413
|
+
const outcome = gateOutcome({ checked: 0, findings: [] }, 'gate');
|
|
414
|
+
const remedy = `No test file matched under ${path} (looked for test_*.py, ` +
|
|
415
|
+
`*.spec.ts/js, *.test.ts/js). Point at a directory that holds tests, ` +
|
|
416
|
+
`or pass a single file directly.`;
|
|
417
|
+
if (json) {
|
|
418
|
+
deps.out(jsonIndent2([]));
|
|
419
|
+
deps.err(`${outcome.summaryLine} ${remedy}`);
|
|
420
|
+
}
|
|
421
|
+
else {
|
|
422
|
+
deps.out(pc.bold(pc.yellow(outcome.summaryLine)));
|
|
423
|
+
deps.out(` ${remedy}`);
|
|
424
|
+
}
|
|
425
|
+
throw new CliExit(outcome.exitCode);
|
|
426
|
+
}
|
|
396
427
|
export function reviewTestCmd(path, opts, deps) {
|
|
397
428
|
const files = isDir(path) ? collectTestFiles(path) : [path];
|
|
429
|
+
abstainOnZeroFiles(files, path, deps, opts.json === true);
|
|
398
430
|
const linter = deps.makeLinter();
|
|
399
431
|
const allFindings = [];
|
|
400
432
|
for (const f of files)
|
|
@@ -431,6 +463,7 @@ export function reviewTestCmd(path, opts, deps) {
|
|
|
431
463
|
}
|
|
432
464
|
export function flakeCheckCmd(path, opts, deps) {
|
|
433
465
|
const files = isDir(path) ? collectTestFiles(path) : [path];
|
|
466
|
+
abstainOnZeroFiles(files, path, deps, opts.json === true);
|
|
434
467
|
const linter = deps.makeLinter();
|
|
435
468
|
const allFindings = [];
|
|
436
469
|
for (const f of files)
|
|
@@ -12,7 +12,10 @@ import { readFileSync } from 'node:fs';
|
|
|
12
12
|
import { dirname, resolve } from 'node:path';
|
|
13
13
|
import { fileURLToPath } from 'node:url';
|
|
14
14
|
import { def } from '../util/coalesce.js';
|
|
15
|
-
|
|
15
|
+
// The leaf data module, not `scaffolder.js`: the scaffolder imports this
|
|
16
|
+
// registry to degrade loudly on an unknown framework, so taking the set from
|
|
17
|
+
// there would close a cycle (#543).
|
|
18
|
+
import { scaffoldableFrameworks } from './scaffold-templates.js';
|
|
16
19
|
/** Default registry path: `<module dir>/../data/frameworks/registry.json`. */
|
|
17
20
|
export function defaultRegistryPath() {
|
|
18
21
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
@@ -25,8 +25,15 @@ const EMDASH = '\u{2014}'; // em dash
|
|
|
25
25
|
// C0 controls (incl. \n, ESC) and DEL: a skip name must never be able to
|
|
26
26
|
// forge output lines or smuggle ANSI sequences into the summary.
|
|
27
27
|
const CONTROL_CHARS = /[\u0000-\u001F\u007F]/g;
|
|
28
|
-
/**
|
|
29
|
-
|
|
28
|
+
/**
|
|
29
|
+
* D7: skipped entries render in EVERY summary line.
|
|
30
|
+
*
|
|
31
|
+
* Exported so a surface with its own failure vocabulary (doctor says
|
|
32
|
+
* "check(s) failed", not "finding(s)") can render the identical skip suffix
|
|
33
|
+
* instead of re-deriving the format -- the decision still comes from
|
|
34
|
+
* {@link gateOutcome}, only the noun differs.
|
|
35
|
+
*/
|
|
36
|
+
export function skippedSuffix(skipped) {
|
|
30
37
|
if (!skipped || skipped.length === 0)
|
|
31
38
|
return '';
|
|
32
39
|
const names = skipped
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scaffold templates - the config-file bodies canary writes for each supported
|
|
3
|
+
* framework, and the derived set of frameworks it can scaffold.
|
|
4
|
+
*
|
|
5
|
+
* Extracted from `scaffolder.ts` to break a circular dependency (#543):
|
|
6
|
+
* `framework-registry` needs {@link scaffoldableFrameworks} to answer the
|
|
7
|
+
* scaffold capability question, while `scaffolder` needs the registry to
|
|
8
|
+
* degrade loudly on an unknown framework. With the data here, both depend on a
|
|
9
|
+
* leaf module and neither depends on the other. `scaffolder.ts` re-exports
|
|
10
|
+
* both names, so existing importers (the migrator, the parity suite) are
|
|
11
|
+
* unaffected.
|
|
12
|
+
*
|
|
13
|
+
* The template strings remain a **byte-exact contract** with the Python
|
|
14
|
+
* originals - see the note in `scaffolder.ts`. They were moved by line slice,
|
|
15
|
+
* not retyped, and `core-parity.test.ts` compares TEMPLATES against the golden
|
|
16
|
+
* fixture.
|
|
17
|
+
*/
|
|
18
|
+
// Exported so the migrator port can compute would-create / already-present sets
|
|
19
|
+
// in its dry-run path (Python: `from agent.core.scaffolder import TEMPLATES`).
|
|
20
|
+
export const TEMPLATES = {
|
|
21
|
+
playwright: {
|
|
22
|
+
files: {
|
|
23
|
+
'playwright.config.ts': `import { defineConfig, devices } from '@playwright/test';
|
|
24
|
+
|
|
25
|
+
export default defineConfig({
|
|
26
|
+
testDir: './tests/e2e',
|
|
27
|
+
fullyParallel: true,
|
|
28
|
+
forbidOnly: !!process.env.CI,
|
|
29
|
+
retries: process.env.CI ? 2 : 0,
|
|
30
|
+
workers: process.env.CI ? 1 : undefined,
|
|
31
|
+
reporter: 'html',
|
|
32
|
+
use: {
|
|
33
|
+
trace: 'on-first-retry',
|
|
34
|
+
},
|
|
35
|
+
projects: [
|
|
36
|
+
{
|
|
37
|
+
name: 'chromium',
|
|
38
|
+
use: { ...devices['Desktop Chrome'] },
|
|
39
|
+
},
|
|
40
|
+
],
|
|
41
|
+
});
|
|
42
|
+
`,
|
|
43
|
+
},
|
|
44
|
+
dirs: ['tests/e2e'],
|
|
45
|
+
},
|
|
46
|
+
vitest: {
|
|
47
|
+
files: {
|
|
48
|
+
'vitest.config.ts': `import { defineConfig } from 'vitest/config';
|
|
49
|
+
|
|
50
|
+
export default defineConfig({
|
|
51
|
+
test: {
|
|
52
|
+
environment: 'node',
|
|
53
|
+
include: ['tests/unit/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
|
|
54
|
+
},
|
|
55
|
+
});
|
|
56
|
+
`,
|
|
57
|
+
},
|
|
58
|
+
dirs: ['tests/unit'],
|
|
59
|
+
},
|
|
60
|
+
pytest: {
|
|
61
|
+
files: {
|
|
62
|
+
'pytest.ini': `[pytest]
|
|
63
|
+
testpaths = tests
|
|
64
|
+
python_files = test_*.py *_test.py
|
|
65
|
+
python_classes = Test*
|
|
66
|
+
python_functions = test_*
|
|
67
|
+
`,
|
|
68
|
+
},
|
|
69
|
+
dirs: ['tests'],
|
|
70
|
+
},
|
|
71
|
+
k6: {
|
|
72
|
+
files: {
|
|
73
|
+
'k6.config.js': `export const options = {
|
|
74
|
+
vus: 10,
|
|
75
|
+
duration: '30s',
|
|
76
|
+
};
|
|
77
|
+
`,
|
|
78
|
+
},
|
|
79
|
+
dirs: ['tests/performance'],
|
|
80
|
+
},
|
|
81
|
+
wdio: {
|
|
82
|
+
files: {
|
|
83
|
+
'wdio.conf.ts': `import type { Options } from "@wdio/types";
|
|
84
|
+
|
|
85
|
+
// Appium + WebdriverIO config. Fill in the capabilities stub below for the
|
|
86
|
+
// device/platform under test (Android shown; add an iOS entry as needed).
|
|
87
|
+
export const config: Options.Testrunner = {
|
|
88
|
+
runner: "local",
|
|
89
|
+
specs: ["./tests/**/*.spec.ts"],
|
|
90
|
+
maxInstances: 1,
|
|
91
|
+
// Appium capabilities stub \u{2014} replace deviceName / app / versions to match
|
|
92
|
+
// your emulator or real device.
|
|
93
|
+
capabilities: [
|
|
94
|
+
{
|
|
95
|
+
platformName: "Android",
|
|
96
|
+
"appium:automationName": "UiAutomator2",
|
|
97
|
+
"appium:deviceName": "Android Emulator",
|
|
98
|
+
"appium:app": "./app/build/outputs/apk/debug/app-debug.apk",
|
|
99
|
+
},
|
|
100
|
+
],
|
|
101
|
+
framework: "mocha",
|
|
102
|
+
mochaOpts: {
|
|
103
|
+
ui: "bdd",
|
|
104
|
+
timeout: 60000,
|
|
105
|
+
},
|
|
106
|
+
reporters: ["spec"],
|
|
107
|
+
// Requires the Appium service: \`npm i -D @wdio/appium-service appium\`.
|
|
108
|
+
services: ["appium"],
|
|
109
|
+
};
|
|
110
|
+
`,
|
|
111
|
+
},
|
|
112
|
+
dirs: ['tests'],
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
/**
|
|
116
|
+
* Frameworks canary can scaffold - the single source of truth for the
|
|
117
|
+
* `scaffold` capability, derived from the templates that actually exist
|
|
118
|
+
* (Python: `scaffoldable_frameworks`).
|
|
119
|
+
*/
|
|
120
|
+
export function scaffoldableFrameworks() {
|
|
121
|
+
return new Set(Object.keys(TEMPLATES));
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=scaffold-templates.js.map
|
|
@@ -23,6 +23,10 @@
|
|
|
23
23
|
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
24
24
|
import { join, resolve } from 'node:path';
|
|
25
25
|
import { FrameworkRegistry } from './framework-registry.js';
|
|
26
|
+
import { TEMPLATES } from './scaffold-templates.js';
|
|
27
|
+
// Re-exported so existing importers keep one entry point for the scaffold
|
|
28
|
+
// surface (Python: `from agent.core.scaffolder import TEMPLATES`).
|
|
29
|
+
export { TEMPLATES, scaffoldableFrameworks } from './scaffold-templates.js';
|
|
26
30
|
// ---------------------------------------------------------------------------
|
|
27
31
|
// Python-compatibility helper (copied locally per-module, matching reporter.ts)
|
|
28
32
|
// ---------------------------------------------------------------------------
|
|
@@ -41,103 +45,6 @@ function pyTruthy(value) {
|
|
|
41
45
|
return Object.keys(value).length > 0;
|
|
42
46
|
return Boolean(value);
|
|
43
47
|
}
|
|
44
|
-
// Exported so the migrator port can compute would-create / already-present sets
|
|
45
|
-
// in its dry-run path (Python: `from agent.core.scaffolder import TEMPLATES`).
|
|
46
|
-
export const TEMPLATES = {
|
|
47
|
-
playwright: {
|
|
48
|
-
files: {
|
|
49
|
-
'playwright.config.ts': `import { defineConfig, devices } from '@playwright/test';
|
|
50
|
-
|
|
51
|
-
export default defineConfig({
|
|
52
|
-
testDir: './tests/e2e',
|
|
53
|
-
fullyParallel: true,
|
|
54
|
-
forbidOnly: !!process.env.CI,
|
|
55
|
-
retries: process.env.CI ? 2 : 0,
|
|
56
|
-
workers: process.env.CI ? 1 : undefined,
|
|
57
|
-
reporter: 'html',
|
|
58
|
-
use: {
|
|
59
|
-
trace: 'on-first-retry',
|
|
60
|
-
},
|
|
61
|
-
projects: [
|
|
62
|
-
{
|
|
63
|
-
name: 'chromium',
|
|
64
|
-
use: { ...devices['Desktop Chrome'] },
|
|
65
|
-
},
|
|
66
|
-
],
|
|
67
|
-
});
|
|
68
|
-
`,
|
|
69
|
-
},
|
|
70
|
-
dirs: ['tests/e2e'],
|
|
71
|
-
},
|
|
72
|
-
vitest: {
|
|
73
|
-
files: {
|
|
74
|
-
'vitest.config.ts': `import { defineConfig } from 'vitest/config';
|
|
75
|
-
|
|
76
|
-
export default defineConfig({
|
|
77
|
-
test: {
|
|
78
|
-
environment: 'node',
|
|
79
|
-
include: ['tests/unit/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
|
|
80
|
-
},
|
|
81
|
-
});
|
|
82
|
-
`,
|
|
83
|
-
},
|
|
84
|
-
dirs: ['tests/unit'],
|
|
85
|
-
},
|
|
86
|
-
pytest: {
|
|
87
|
-
files: {
|
|
88
|
-
'pytest.ini': `[pytest]
|
|
89
|
-
testpaths = tests
|
|
90
|
-
python_files = test_*.py *_test.py
|
|
91
|
-
python_classes = Test*
|
|
92
|
-
python_functions = test_*
|
|
93
|
-
`,
|
|
94
|
-
},
|
|
95
|
-
dirs: ['tests'],
|
|
96
|
-
},
|
|
97
|
-
k6: {
|
|
98
|
-
files: {
|
|
99
|
-
'k6.config.js': `export const options = {
|
|
100
|
-
vus: 10,
|
|
101
|
-
duration: '30s',
|
|
102
|
-
};
|
|
103
|
-
`,
|
|
104
|
-
},
|
|
105
|
-
dirs: ['tests/performance'],
|
|
106
|
-
},
|
|
107
|
-
wdio: {
|
|
108
|
-
files: {
|
|
109
|
-
'wdio.conf.ts': `import type { Options } from "@wdio/types";
|
|
110
|
-
|
|
111
|
-
// Appium + WebdriverIO config. Fill in the capabilities stub below for the
|
|
112
|
-
// device/platform under test (Android shown; add an iOS entry as needed).
|
|
113
|
-
export const config: Options.Testrunner = {
|
|
114
|
-
runner: "local",
|
|
115
|
-
specs: ["./tests/**/*.spec.ts"],
|
|
116
|
-
maxInstances: 1,
|
|
117
|
-
// Appium capabilities stub \u{2014} replace deviceName / app / versions to match
|
|
118
|
-
// your emulator or real device.
|
|
119
|
-
capabilities: [
|
|
120
|
-
{
|
|
121
|
-
platformName: "Android",
|
|
122
|
-
"appium:automationName": "UiAutomator2",
|
|
123
|
-
"appium:deviceName": "Android Emulator",
|
|
124
|
-
"appium:app": "./app/build/outputs/apk/debug/app-debug.apk",
|
|
125
|
-
},
|
|
126
|
-
],
|
|
127
|
-
framework: "mocha",
|
|
128
|
-
mochaOpts: {
|
|
129
|
-
ui: "bdd",
|
|
130
|
-
timeout: 60000,
|
|
131
|
-
},
|
|
132
|
-
reporters: ["spec"],
|
|
133
|
-
// Requires the Appium service: \`npm i -D @wdio/appium-service appium\`.
|
|
134
|
-
services: ["appium"],
|
|
135
|
-
};
|
|
136
|
-
`,
|
|
137
|
-
},
|
|
138
|
-
dirs: ['tests'],
|
|
139
|
-
},
|
|
140
|
-
};
|
|
141
48
|
/**
|
|
142
49
|
* Handles initialization and scaffolding of test suites.
|
|
143
50
|
*
|
|
@@ -225,12 +132,4 @@ export class Scaffolder {
|
|
|
225
132
|
};
|
|
226
133
|
}
|
|
227
134
|
}
|
|
228
|
-
/**
|
|
229
|
-
* Frameworks canary can scaffold - the single source of truth for the
|
|
230
|
-
* `scaffold` capability, derived from the templates that actually exist
|
|
231
|
-
* (Python: `scaffoldable_frameworks`).
|
|
232
|
-
*/
|
|
233
|
-
export function scaffoldableFrameworks() {
|
|
234
|
-
return new Set(Object.keys(TEMPLATES));
|
|
235
|
-
}
|
|
236
135
|
//# sourceMappingURL=scaffolder.js.map
|