canary-test-cli 6.3.0 → 6.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/canary-mcp.js +52 -0
- package/dist/doctor.d.ts +51 -3
- package/dist/doctor.js +76 -9
- package/dist/engine/analysis/cli.js +69 -6
- package/dist/engine/cli-commands.js +34 -1
- package/dist/engine/core/feedback.js +32 -18
- package/dist/engine/core/gate-result.js +80 -0
- package/dist/engine/core/migrator.js +83 -10
- package/dist/engine/core/skill-registry.js +95 -30
- package/dist/engine/guardian/adjudication.js +364 -0
- package/dist/engine/guardian/analysis-emit.js +2 -0
- package/dist/engine/guardian/cli.js +282 -15
- package/dist/engine/guardian/hard-gate.js +15 -2
- package/dist/engine/guardian/pr-check.js +5 -12
- package/dist/engine/history/cli.js +67 -0
- package/dist/engine/history/ndjson-store.js +4 -0
- package/dist/engine/history/store.js +3 -0
- package/dist/gate-result.d.ts +67 -0
- package/dist/gate-result.js +73 -0
- package/dist/overlay-commands.js +17 -1
- package/dist/overlay-lint.d.ts +6 -1
- package/dist/overlay-lint.js +53 -68
- package/dist/skill-frontmatter.d.ts +24 -0
- package/dist/skill-frontmatter.js +89 -0
- package/package.json +5 -2
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// Console-script entry for `canary-mcp` (#507): starts the Canary MCP server
|
|
5
|
+
// over stdio from the bundled TypeScript engine (dist/engine/mcp-server.js,
|
|
6
|
+
// staged by scripts/build-engine.mjs). The engine bundle is ESM while this
|
|
7
|
+
// package is CommonJS, so the server module is loaded via dynamic import().
|
|
8
|
+
// stdout carries the JSON-RPC stream and must never be polluted -- every
|
|
9
|
+
// failure path writes to stderr only.
|
|
10
|
+
|
|
11
|
+
const path = require('node:path');
|
|
12
|
+
const fs = require('node:fs');
|
|
13
|
+
const { pathToFileURL } = require('node:url');
|
|
14
|
+
|
|
15
|
+
/** Absolute path to the bundled engine's MCP server module. */
|
|
16
|
+
function getServerPath() {
|
|
17
|
+
return path.join(__dirname, '..', 'dist', 'engine', 'mcp-server.js');
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Start the server. Returns the exit code (0 when runStdio resolves, 1 when
|
|
22
|
+
* the bundle is missing). Dependencies are injectable for testing.
|
|
23
|
+
*/
|
|
24
|
+
async function main({
|
|
25
|
+
serverPath = getServerPath(),
|
|
26
|
+
existsSync = fs.existsSync,
|
|
27
|
+
stderr = process.stderr,
|
|
28
|
+
} = {}) {
|
|
29
|
+
if (!existsSync(serverPath)) {
|
|
30
|
+
stderr.write(
|
|
31
|
+
`canary MCP server not found at ${serverPath}.\n` +
|
|
32
|
+
`The package looks incomplete; try reinstalling: npm install -g canary-test-cli\n`,
|
|
33
|
+
);
|
|
34
|
+
return 1;
|
|
35
|
+
}
|
|
36
|
+
const { runStdio } = await import(pathToFileURL(serverPath).href);
|
|
37
|
+
await runStdio();
|
|
38
|
+
return 0;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
if (require.main === module) {
|
|
42
|
+
main()
|
|
43
|
+
.then((code) => {
|
|
44
|
+
if (code !== 0) process.exit(code);
|
|
45
|
+
})
|
|
46
|
+
.catch((err) => {
|
|
47
|
+
console.error(err);
|
|
48
|
+
process.exit(1);
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
module.exports = { getServerPath, main };
|
package/dist/doctor.d.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type { CommandDeps } from './overlay-commands.js';
|
|
2
|
+
import { type EngineCheckDeps } from './engine-checks.js';
|
|
2
3
|
import { type CommandRunner, type UrlProbe } from './doctor-manifest.js';
|
|
4
|
+
import { EXIT_ABSTAINED, type SkipEntry } from './gate-result.js';
|
|
5
|
+
export { EXIT_ABSTAINED };
|
|
3
6
|
/** Outcome of a single doctor check. */
|
|
4
7
|
export type CheckStatus = 'pass' | 'fail' | 'skip' | 'info';
|
|
5
8
|
/** A single doctor check result, rendered as one output line. */
|
|
@@ -21,6 +24,17 @@ export interface DoctorDeps extends CommandDeps {
|
|
|
21
24
|
probeUrl?: UrlProbe;
|
|
22
25
|
runCommand?: CommandRunner;
|
|
23
26
|
timeoutMs?: number;
|
|
27
|
+
/**
|
|
28
|
+
* Override the engine check set. A test seam only: it is the one input a
|
|
29
|
+
* hermetic run cannot control (engine checks read the real environment), and
|
|
30
|
+
* the abstention fixtures need a run whose denominator is exactly zero.
|
|
31
|
+
*/
|
|
32
|
+
runEngineChecks?: (deps: EngineCheckDeps) => Promise<CheckResult[]>;
|
|
33
|
+
}
|
|
34
|
+
/** One printed section: a header and its check results. */
|
|
35
|
+
export interface CheckGroup {
|
|
36
|
+
header: string;
|
|
37
|
+
results: CheckResult[];
|
|
24
38
|
}
|
|
25
39
|
/**
|
|
26
40
|
* The `canary doctor --json` machine contract (issue #318). Canary-owned and
|
|
@@ -46,6 +60,12 @@ export interface JsonReport {
|
|
|
46
60
|
allPassed: boolean;
|
|
47
61
|
/** Non-fatal advisories (e.g. an unknown `--audience`); empty when none. */
|
|
48
62
|
warnings: string[];
|
|
63
|
+
/** Checks that produced a verdict (`pass + fail`) -- the denominator (#508). */
|
|
64
|
+
checked: number;
|
|
65
|
+
/** Skipped checks, always reported; never folded into `allPassed` (D7). */
|
|
66
|
+
skipped: SkipEntry[];
|
|
67
|
+
/** True when nothing was verified: `allPassed` is false and the exit is 3. */
|
|
68
|
+
abstained: boolean;
|
|
49
69
|
}
|
|
50
70
|
/** `--json` requests machine output on stdout instead of the human report. */
|
|
51
71
|
export declare function parseJsonFlag(args: readonly string[]): boolean;
|
|
@@ -59,9 +79,37 @@ export declare function parseJsonFlag(args: readonly string[]): boolean;
|
|
|
59
79
|
* guess why their filter matched nothing.
|
|
60
80
|
*/
|
|
61
81
|
export declare function unknownAudienceHint(audience: string | null, known: readonly string[]): string | null;
|
|
82
|
+
/** Doctor's denominator, and the decision that follows from it (#508 D7). */
|
|
83
|
+
export interface DoctorSummary {
|
|
84
|
+
/** Checks that actually produced a verdict: `pass + fail`. */
|
|
85
|
+
checked: number;
|
|
86
|
+
passed: number;
|
|
87
|
+
failed: number;
|
|
88
|
+
/** Every skipped check, always visible, never folded into `passed`. */
|
|
89
|
+
skipped: SkipEntry[];
|
|
90
|
+
abstained: boolean;
|
|
91
|
+
exitCode: number;
|
|
92
|
+
summaryLine: string;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Compute doctor's denominator and summary line (#508 Wave 3, D7 / #505).
|
|
96
|
+
*
|
|
97
|
+
* Doctor used to count only failures, so a run in which EVERY check was
|
|
98
|
+
* skipped printed `All checks passed.` and exited 0 -- the doctrine violation
|
|
99
|
+
* that started #508. Here the denominator is explicit: `info` results are not
|
|
100
|
+
* verifications (they report context, not evidence), so a run with nothing but
|
|
101
|
+
* skips and info abstains.
|
|
102
|
+
*
|
|
103
|
+
* The decision (exit code + `abstained`) comes from `gateOutcome` and is never
|
|
104
|
+
* re-derived here; only the failure line keeps doctor's own vocabulary
|
|
105
|
+
* ("check(s) failed" rather than "finding(s)"), rendered with the helper's own
|
|
106
|
+
* skip suffix so the two can never drift.
|
|
107
|
+
*/
|
|
108
|
+
export declare function summarizeChecks(groups: readonly CheckGroup[]): DoctorSummary;
|
|
62
109
|
/**
|
|
63
|
-
* Run `canary doctor`. Returns a process exit code: 0 when
|
|
64
|
-
*
|
|
65
|
-
*
|
|
110
|
+
* Run `canary doctor`. Returns a process exit code: 0 when at least one check
|
|
111
|
+
* ran and none failed, 1 when any check failed, and `EXIT_ABSTAINED` (3) when
|
|
112
|
+
* nothing was actually verified (#508). A malformed manifest for one overlay
|
|
113
|
+
* never blocks engine checks or other overlays.
|
|
66
114
|
*/
|
|
67
115
|
export declare function runDoctor(args: readonly string[], deps?: DoctorDeps): Promise<number>;
|
package/dist/doctor.js
CHANGED
|
@@ -33,8 +33,10 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.EXIT_ABSTAINED = void 0;
|
|
36
37
|
exports.parseJsonFlag = parseJsonFlag;
|
|
37
38
|
exports.unknownAudienceHint = unknownAudienceHint;
|
|
39
|
+
exports.summarizeChecks = summarizeChecks;
|
|
38
40
|
exports.runDoctor = runDoctor;
|
|
39
41
|
/**
|
|
40
42
|
* `canary doctor` — environment self-check (Phase 2).
|
|
@@ -57,6 +59,8 @@ const os = __importStar(require("node:os"));
|
|
|
57
59
|
const engine_checks_js_1 = require("./engine-checks.js");
|
|
58
60
|
const doctor_manifest_js_1 = require("./doctor-manifest.js");
|
|
59
61
|
const registry = __importStar(require("./overlays-registry.js"));
|
|
62
|
+
const gate_result_js_1 = require("./gate-result.js");
|
|
63
|
+
Object.defineProperty(exports, "EXIT_ABSTAINED", { enumerable: true, get: function () { return gate_result_js_1.EXIT_ABSTAINED; } });
|
|
60
64
|
const SYMBOL = {
|
|
61
65
|
pass: '✓',
|
|
62
66
|
fail: '✗',
|
|
@@ -141,9 +145,60 @@ async function overlayResults(entry, deps, audience) {
|
|
|
141
145
|
return { group: { header, results }, loadedChecks: load.checks };
|
|
142
146
|
}
|
|
143
147
|
/**
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
148
|
+
* Compute doctor's denominator and summary line (#508 Wave 3, D7 / #505).
|
|
149
|
+
*
|
|
150
|
+
* Doctor used to count only failures, so a run in which EVERY check was
|
|
151
|
+
* skipped printed `All checks passed.` and exited 0 -- the doctrine violation
|
|
152
|
+
* that started #508. Here the denominator is explicit: `info` results are not
|
|
153
|
+
* verifications (they report context, not evidence), so a run with nothing but
|
|
154
|
+
* skips and info abstains.
|
|
155
|
+
*
|
|
156
|
+
* The decision (exit code + `abstained`) comes from `gateOutcome` and is never
|
|
157
|
+
* re-derived here; only the failure line keeps doctor's own vocabulary
|
|
158
|
+
* ("check(s) failed" rather than "finding(s)"), rendered with the helper's own
|
|
159
|
+
* skip suffix so the two can never drift.
|
|
160
|
+
*/
|
|
161
|
+
function summarizeChecks(groups) {
|
|
162
|
+
const results = groups.flatMap((g) => g.results);
|
|
163
|
+
const failures = results.filter((r) => r.status === 'fail');
|
|
164
|
+
const passed = results.filter((r) => r.status === 'pass').length;
|
|
165
|
+
const skipped = results
|
|
166
|
+
.filter((r) => r.status === 'skip')
|
|
167
|
+
.map((r) => ({ name: r.id, reason: r.remedy ?? r.label }));
|
|
168
|
+
const checked = passed + failures.length;
|
|
169
|
+
const outcome = (0, gate_result_js_1.gateOutcome)({ checked, findings: failures, skipped }, 'gate');
|
|
170
|
+
return {
|
|
171
|
+
checked,
|
|
172
|
+
passed,
|
|
173
|
+
failed: failures.length,
|
|
174
|
+
skipped,
|
|
175
|
+
abstained: outcome.abstained,
|
|
176
|
+
exitCode: outcome.exitCode,
|
|
177
|
+
summaryLine: failures.length > 0
|
|
178
|
+
? `${failures.length} check(s) failed${(0, gate_result_js_1.skippedSuffix)(skipped)}`
|
|
179
|
+
: outcome.summaryLine,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Remediation for an abstained run: why the denominator collapsed, and the
|
|
184
|
+
* first fix step. Required of every abstaining surface (spec: "an abstaining
|
|
185
|
+
* surface must say _why_ ... and the first fix step").
|
|
186
|
+
*/
|
|
187
|
+
function abstentionRemedy(summary) {
|
|
188
|
+
return summary.skipped.length > 0
|
|
189
|
+
? 'Every registered check was skipped or informational, so doctor verified ' +
|
|
190
|
+
'nothing. Grant command-check consent (re-run `canary overlay add ' +
|
|
191
|
+
'<name> --yes`) or install an overlay whose checks apply here, then ' +
|
|
192
|
+
're-run.'
|
|
193
|
+
: 'No check was registered, so doctor verified nothing. Install an ' +
|
|
194
|
+
'overlay that ships a `.canary/doctor.json` (`canary overlay add ' +
|
|
195
|
+
'<source>`), then re-run.';
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Run `canary doctor`. Returns a process exit code: 0 when at least one check
|
|
199
|
+
* ran and none failed, 1 when any check failed, and `EXIT_ABSTAINED` (3) when
|
|
200
|
+
* nothing was actually verified (#508). A malformed manifest for one overlay
|
|
201
|
+
* never blocks engine checks or other overlays.
|
|
147
202
|
*/
|
|
148
203
|
async function runDoctor(args, deps = {}) {
|
|
149
204
|
const out = deps.out ?? process.stdout;
|
|
@@ -157,8 +212,9 @@ async function runDoctor(args, deps = {}) {
|
|
|
157
212
|
getLatestVersion: deps.getLatestVersion,
|
|
158
213
|
timeoutMs: deps.timeoutMs,
|
|
159
214
|
};
|
|
215
|
+
const engineChecks = deps.runEngineChecks ?? engine_checks_js_1.runEngineChecks;
|
|
160
216
|
const groups = [
|
|
161
|
-
{ header: 'Engine', results: await (
|
|
217
|
+
{ header: 'Engine', results: await engineChecks(engineDeps) },
|
|
162
218
|
];
|
|
163
219
|
let reg;
|
|
164
220
|
try {
|
|
@@ -177,7 +233,10 @@ async function runDoctor(args, deps = {}) {
|
|
|
177
233
|
// tell them the valid vocabulary instead of silently filtering to only
|
|
178
234
|
// the audience-less checks and leaving them to wonder why.
|
|
179
235
|
const audienceHint = unknownAudienceHint(audience, (0, doctor_manifest_js_1.collectAudiences)(allChecks));
|
|
180
|
-
|
|
236
|
+
// #508: the denominator, not just the failure count. `summarizeChecks` is
|
|
237
|
+
// the only path to a summary line and an exit code, so doctor structurally
|
|
238
|
+
// cannot print a bare success over zero verified checks.
|
|
239
|
+
const summary = summarizeChecks(groups);
|
|
181
240
|
// Issue #318: `--json` emits the canary-owned machine contract instead of
|
|
182
241
|
// the human report — nothing else is written to stdout, so the whole stream
|
|
183
242
|
// parses as one JSON object.
|
|
@@ -191,11 +250,16 @@ async function runDoctor(args, deps = {}) {
|
|
|
191
250
|
...(r.remedy !== undefined ? { remedy: r.remedy } : {}),
|
|
192
251
|
group: g.header,
|
|
193
252
|
}))),
|
|
194
|
-
|
|
253
|
+
// #508: an abstained run is NOT "all passed" -- zero verified checks is
|
|
254
|
+
// an absent measurement, never a green.
|
|
255
|
+
allPassed: summary.failed === 0 && !summary.abstained,
|
|
195
256
|
warnings: audienceHint ? [audienceHint] : [],
|
|
257
|
+
checked: summary.checked,
|
|
258
|
+
skipped: summary.skipped,
|
|
259
|
+
abstained: summary.abstained,
|
|
196
260
|
};
|
|
197
261
|
out.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
198
|
-
return
|
|
262
|
+
return summary.exitCode;
|
|
199
263
|
}
|
|
200
264
|
out.write('canary doctor\n');
|
|
201
265
|
if (audienceHint) {
|
|
@@ -207,6 +271,9 @@ async function runDoctor(args, deps = {}) {
|
|
|
207
271
|
out.write(renderCheck(result));
|
|
208
272
|
}
|
|
209
273
|
}
|
|
210
|
-
out.write(`\n${
|
|
211
|
-
|
|
274
|
+
out.write(`\n${summary.summaryLine}\n`);
|
|
275
|
+
if (summary.abstained) {
|
|
276
|
+
out.write(` ${abstentionRemedy(summary)}\n`);
|
|
277
|
+
}
|
|
278
|
+
return summary.exitCode;
|
|
212
279
|
}
|
|
@@ -27,8 +27,9 @@ import { mkdirSync, writeFileSync } from 'node:fs';
|
|
|
27
27
|
import { join } from 'node:path';
|
|
28
28
|
import { Command, Option } from 'commander';
|
|
29
29
|
import { jsonIndent2, normalizeUsageExit } from '../cli-common.js';
|
|
30
|
+
import { gateOutcome } from '../core/gate-result.js';
|
|
30
31
|
import { AnalysisEngine } from './engine.js';
|
|
31
|
-
import {
|
|
32
|
+
import { buildCommonFailuresReport, buildFlakyReport, buildRegressionCandidatesReport, buildSpikesReport, } from './reports.js';
|
|
32
33
|
import { NdjsonHistoryStore } from '../history/ndjson-store.js';
|
|
33
34
|
const DEFAULT_HISTORY_PATH = 'test-results/reports/history-v2.jsonl';
|
|
34
35
|
/** Process-backed defaults for production. */
|
|
@@ -53,6 +54,41 @@ export function defaultAnalyzeDeps() {
|
|
|
53
54
|
},
|
|
54
55
|
};
|
|
55
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* The shared denominator guard for every history-backed analyze report
|
|
59
|
+
* (#508 Wave 4a).
|
|
60
|
+
*
|
|
61
|
+
* The denominator is the number of RUNS in the store, never the number of
|
|
62
|
+
* result rows: zero flaky rows across 500 runs is a genuine clean fleet, while
|
|
63
|
+
* zero rows across zero runs is an absent measurement. Only `countRuns()`
|
|
64
|
+
* separates them -- keying off `rows.length` would abstain on every healthy
|
|
65
|
+
* fleet, which is the fastest way to get a doctrine muted (the katana lesson).
|
|
66
|
+
*
|
|
67
|
+
* Advisory (D3): the exit stays 0. On the human path the abstention line
|
|
68
|
+
* REPLACES the all-clear report; on `--json` the payload is a bare array with
|
|
69
|
+
* nowhere to put an `abstained` field, so stdout is left byte-identical and the
|
|
70
|
+
* notice rides stderr -- the same split `analyze` already uses for its
|
|
71
|
+
* `--db-url` note.
|
|
72
|
+
*
|
|
73
|
+
* Returns true when the caller should stop (the store was empty).
|
|
74
|
+
*/
|
|
75
|
+
function abstainOnEmptyHistory(store, deps, json, what) {
|
|
76
|
+
if (store.countRuns() > 0)
|
|
77
|
+
return false;
|
|
78
|
+
const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
|
|
79
|
+
const notice = `${outcome.summaryLine} No run history to analyze, so "${what}" is ` +
|
|
80
|
+
`unknown rather than clean. Record runs first ` +
|
|
81
|
+
`(\`canary history push\`, or a reporter that writes ` +
|
|
82
|
+
`${DEFAULT_HISTORY_PATH}), then re-run.`;
|
|
83
|
+
if (json) {
|
|
84
|
+
deps.out(jsonIndent2([]));
|
|
85
|
+
deps.err(notice);
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
deps.out(notice);
|
|
89
|
+
}
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
56
92
|
function writeArtifacts(artifacts, output) {
|
|
57
93
|
mkdirSync(output, { recursive: true });
|
|
58
94
|
for (const [name, content] of Object.entries(artifacts)) {
|
|
@@ -61,6 +97,9 @@ function writeArtifacts(artifacts, output) {
|
|
|
61
97
|
}
|
|
62
98
|
function flakyCmd(opts, deps) {
|
|
63
99
|
const store = deps.makeStore(opts.dbUrl);
|
|
100
|
+
if (abstainOnEmptyHistory(store, deps, opts.json === true, 'flake rate')) {
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
64
103
|
const rows = store.queryFlaky(opts.window, opts.suite ?? null, opts.minRate);
|
|
65
104
|
if (opts.json) {
|
|
66
105
|
deps.out(jsonIndent2(rows));
|
|
@@ -71,6 +110,9 @@ function flakyCmd(opts, deps) {
|
|
|
71
110
|
}
|
|
72
111
|
function spikesCmd(opts, deps) {
|
|
73
112
|
const store = deps.makeStore(opts.dbUrl);
|
|
113
|
+
if (abstainOnEmptyHistory(store, deps, opts.json === true, 'failure spikes')) {
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
74
116
|
const rows = [];
|
|
75
117
|
for (const r of store.readAll()) {
|
|
76
118
|
if (opts.since && (r.timestamp ?? '') < opts.since)
|
|
@@ -92,12 +134,25 @@ function spikesCmd(opts, deps) {
|
|
|
92
134
|
}
|
|
93
135
|
}
|
|
94
136
|
function areaHealthCmd(opts, deps) {
|
|
95
|
-
//
|
|
96
|
-
//
|
|
97
|
-
|
|
137
|
+
// #508 Wave 4a: this command builds its report from a HARDCODED empty row
|
|
138
|
+
// set (faithful to the Python original, which did the same and never branched
|
|
139
|
+
// on --json). Its denominator is therefore UNCONDITIONALLY zero -- with a
|
|
140
|
+
// thousand runs recorded it still renders "no area health data", which reads
|
|
141
|
+
// as a measured all-clear and is not one. So it always abstains, whatever the
|
|
142
|
+
// store holds. Wiring a real row set is a separate scope call: it changes the
|
|
143
|
+
// port's contract, and #515 deferred it for exactly that reason.
|
|
144
|
+
void opts;
|
|
145
|
+
const outcome = gateOutcome({ checked: 0, findings: [] }, 'advisory');
|
|
146
|
+
deps.out(`${outcome.summaryLine} \`analyze area-health\` computes no rows in this ` +
|
|
147
|
+
`build -- its report is a template, not a measurement, so a clean-looking ` +
|
|
148
|
+
`result here would be a fiction. Use \`analyze digest\` for the reports ` +
|
|
149
|
+
`that are wired, and track the area-health row set as unimplemented.`);
|
|
98
150
|
}
|
|
99
151
|
function commonFailuresCmd(opts, deps) {
|
|
100
152
|
const store = deps.makeStore(opts.dbUrl);
|
|
153
|
+
if (abstainOnEmptyHistory(store, deps, opts.json === true, 'common failures')) {
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
101
156
|
const rows = [];
|
|
102
157
|
for (const record of store.readAll()) {
|
|
103
158
|
if (opts.since && (record.timestamp ?? '') < opts.since)
|
|
@@ -122,7 +177,11 @@ function commonFailuresCmd(opts, deps) {
|
|
|
122
177
|
}
|
|
123
178
|
}
|
|
124
179
|
function regressionCandidatesCmd(opts, deps) {
|
|
125
|
-
const
|
|
180
|
+
const store = deps.makeStore(opts.dbUrl);
|
|
181
|
+
if (abstainOnEmptyHistory(store, deps, opts.json === true, 'regression candidates')) {
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
const engine = new AnalysisEngine(store);
|
|
126
185
|
const candidates = engine.detectRegressionCandidates(null, opts.minGreen, opts.recentFailures);
|
|
127
186
|
if (opts.json) {
|
|
128
187
|
deps.out(jsonIndent2(candidates));
|
|
@@ -132,7 +191,11 @@ function regressionCandidatesCmd(opts, deps) {
|
|
|
132
191
|
}
|
|
133
192
|
}
|
|
134
193
|
function digestCmd(opts, deps) {
|
|
135
|
-
const
|
|
194
|
+
const store = deps.makeStore(opts.dbUrl);
|
|
195
|
+
if (abstainOnEmptyHistory(store, deps, opts.json === true, 'fleet health')) {
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
const engine = new AnalysisEngine(store);
|
|
136
199
|
const result = engine.run({
|
|
137
200
|
window: opts.window,
|
|
138
201
|
delta: opts.delta,
|
|
@@ -14,6 +14,7 @@ import { existsSync, readdirSync, readFileSync, statSync, writeFileSync, } from
|
|
|
14
14
|
import { basename, join, resolve } from 'node:path';
|
|
15
15
|
import pc from 'picocolors';
|
|
16
16
|
import { CliExit, jsonIndent2 } from './cli-common.js';
|
|
17
|
+
import { gateOutcome } from './core/gate-result.js';
|
|
17
18
|
import { ckInitCmd } from './company-knowledge-cli.js';
|
|
18
19
|
import { extractFrameworkHint } from './core/classifier.js';
|
|
19
20
|
import { VALID_CATEGORIES, buildFeedback } from './core/feedback.js';
|
|
@@ -155,7 +156,7 @@ export function feedbackCmd(message, opts, deps) {
|
|
|
155
156
|
deps.out(`${pc.bold(pc.red(CROSS))} A feedback message is required.\nUsage: ${pc.bold('canary feedback "<message>" [--category bug|ux|docs|idea]')}`);
|
|
156
157
|
throw new CliExit(1);
|
|
157
158
|
}
|
|
158
|
-
const fb = buildFeedback(message.trim(), opts.category);
|
|
159
|
+
const fb = buildFeedback(message.trim(), opts.category, resolveVersion(deps));
|
|
159
160
|
if (opts.json) {
|
|
160
161
|
deps.out(jsonIndent2(fb));
|
|
161
162
|
return;
|
|
@@ -393,8 +394,39 @@ function findingPayload(f) {
|
|
|
393
394
|
suggestion: f.suggestion,
|
|
394
395
|
};
|
|
395
396
|
}
|
|
397
|
+
/**
|
|
398
|
+
* The denominator guard shared by the file-scanning gates (#508 Wave 4a).
|
|
399
|
+
*
|
|
400
|
+
* `review-test` and `flake-check` used to render a green all-clear whenever
|
|
401
|
+
* their finding list was empty -- indistinguishable from a run that scanned a
|
|
402
|
+
* directory matching zero test files. That is the #503 shape: a gate that
|
|
403
|
+
* verified nothing reporting a pass. Both are GATES (they carry an exit-code
|
|
404
|
+
* contract), so a collapsed denominator exits 3.
|
|
405
|
+
*
|
|
406
|
+
* Returns without throwing when at least one file was collected; the caller's
|
|
407
|
+
* normal rendering continues. `--json` keeps a parseable array on stdout -- only
|
|
408
|
+
* the exit code and the stderr notice carry the abstention.
|
|
409
|
+
*/
|
|
410
|
+
function abstainOnZeroFiles(files, path, deps, json) {
|
|
411
|
+
if (files.length > 0)
|
|
412
|
+
return;
|
|
413
|
+
const outcome = gateOutcome({ checked: 0, findings: [] }, 'gate');
|
|
414
|
+
const remedy = `No test file matched under ${path} (looked for test_*.py, ` +
|
|
415
|
+
`*.spec.ts/js, *.test.ts/js). Point at a directory that holds tests, ` +
|
|
416
|
+
`or pass a single file directly.`;
|
|
417
|
+
if (json) {
|
|
418
|
+
deps.out(jsonIndent2([]));
|
|
419
|
+
deps.err(`${outcome.summaryLine} ${remedy}`);
|
|
420
|
+
}
|
|
421
|
+
else {
|
|
422
|
+
deps.out(pc.bold(pc.yellow(outcome.summaryLine)));
|
|
423
|
+
deps.out(` ${remedy}`);
|
|
424
|
+
}
|
|
425
|
+
throw new CliExit(outcome.exitCode);
|
|
426
|
+
}
|
|
396
427
|
export function reviewTestCmd(path, opts, deps) {
|
|
397
428
|
const files = isDir(path) ? collectTestFiles(path) : [path];
|
|
429
|
+
abstainOnZeroFiles(files, path, deps, opts.json === true);
|
|
398
430
|
const linter = deps.makeLinter();
|
|
399
431
|
const allFindings = [];
|
|
400
432
|
for (const f of files)
|
|
@@ -431,6 +463,7 @@ export function reviewTestCmd(path, opts, deps) {
|
|
|
431
463
|
}
|
|
432
464
|
export function flakeCheckCmd(path, opts, deps) {
|
|
433
465
|
const files = isDir(path) ? collectTestFiles(path) : [path];
|
|
466
|
+
abstainOnZeroFiles(files, path, deps, opts.json === true);
|
|
434
467
|
const linter = deps.makeLinter();
|
|
435
468
|
const allFindings = [];
|
|
436
469
|
for (const f of files)
|
|
@@ -12,10 +12,10 @@
|
|
|
12
12
|
* never reads environment variables or file contents.
|
|
13
13
|
*
|
|
14
14
|
* Python→TS nuances:
|
|
15
|
-
* -
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
15
|
+
* - The context keys are `{version, os, runtime, install}` (insertion order
|
|
16
|
+
* preserved). `runtime` carries `process.version`; the key was named
|
|
17
|
+
* `python` for golden-parity shape fidelity until #506 — in filed issues
|
|
18
|
+
* it misled triage ("user on python v22?") once the Python engine retired.
|
|
19
19
|
* - `urlencode(...)` (which uses `quote_plus`, space -> `+`) maps to
|
|
20
20
|
* `URLSearchParams`, which also form-encodes with space -> `+` and preserves
|
|
21
21
|
* insertion order. Exotic-character percent-encoding can differ byte-for-byte
|
|
@@ -26,12 +26,6 @@ import { release, type } from 'node:os';
|
|
|
26
26
|
/** The public issue tracker (from npm/package.json `repository`). */
|
|
27
27
|
export const TRACKER_URL = 'https://github.com/bop-clocktower/canary';
|
|
28
28
|
export const VALID_CATEGORIES = ['bug', 'ux', 'docs', 'idea'];
|
|
29
|
-
function canaryVersion() {
|
|
30
|
-
// Python reads `importlib.metadata.version("canary-test-ai")`, falling back to
|
|
31
|
-
// "unknown". The TS pilot has no equivalent package-metadata lookup wired in,
|
|
32
|
-
// so we return the same best-effort "unknown" sentinel.
|
|
33
|
-
return 'unknown';
|
|
34
|
-
}
|
|
35
29
|
/** Best-effort install-method label — never fails, never inspects secrets. */
|
|
36
30
|
function installMethod() {
|
|
37
31
|
const exe = (process.execPath || '').toLowerCase();
|
|
@@ -45,23 +39,43 @@ function installMethod() {
|
|
|
45
39
|
*
|
|
46
40
|
* Deliberately excludes environment variables and file contents — only the
|
|
47
41
|
* coarse runtime facts a maintainer needs to triage a CLI report.
|
|
42
|
+
*
|
|
43
|
+
* `version` comes from the caller (#506): this module is pure and cannot know
|
|
44
|
+
* which package it shipped in, but the CLI layer does (`deps.pkgVersion()`),
|
|
45
|
+
* and the version is the single most useful triage field.
|
|
48
46
|
*/
|
|
49
|
-
export function collectContext() {
|
|
47
|
+
export function collectContext(version = 'unknown') {
|
|
50
48
|
return {
|
|
51
|
-
version
|
|
49
|
+
version,
|
|
52
50
|
os: `${type()} ${release()}`.trim(),
|
|
53
|
-
|
|
51
|
+
runtime: process.version,
|
|
54
52
|
install: installMethod(),
|
|
55
53
|
};
|
|
56
54
|
}
|
|
55
|
+
// Horizontal ellipsis, kept as an escape so this source stays ASCII.
|
|
56
|
+
const ELLIPSIS = '\u{2026}';
|
|
57
|
+
/**
|
|
58
|
+
* Cap the title at 60 code points (`Array.from` slices by code point, not
|
|
59
|
+
* UTF-16 unit, so astral chars do not truncate early). When the cap bites,
|
|
60
|
+
* break on the last word boundary inside the budget (falling back to a hard
|
|
61
|
+
* cut for an unbreakable token) and append an ellipsis so the truncation is
|
|
62
|
+
* visible instead of ending mid-word (#506).
|
|
63
|
+
*/
|
|
64
|
+
function truncateTitle(message) {
|
|
65
|
+
const points = Array.from(message);
|
|
66
|
+
if (points.length <= 60)
|
|
67
|
+
return message;
|
|
68
|
+
const hard = points.slice(0, 60).join('');
|
|
69
|
+
const lastBreak = hard.search(/\s+\S*$/);
|
|
70
|
+
const cut = lastBreak > 0 ? hard.slice(0, lastBreak) : hard;
|
|
71
|
+
return `${cut}${ELLIPSIS}`;
|
|
72
|
+
}
|
|
57
73
|
/**
|
|
58
74
|
* A pre-filled GitHub 'new issue' URL: category in the title, message + context
|
|
59
75
|
* in the body, category as a label. All parts are URL-encoded.
|
|
60
76
|
*/
|
|
61
77
|
export function buildIssueUrl(category, message, context) {
|
|
62
|
-
|
|
63
|
-
// unit, so an astral char would truncate the title early. Match the oracle.
|
|
64
|
-
const title = `[${category}] ${Array.from(message).slice(0, 60).join('')}`.trim();
|
|
78
|
+
const title = `[${category}] ${truncateTitle(message)}`.trim();
|
|
65
79
|
const bodyLines = [
|
|
66
80
|
message,
|
|
67
81
|
'',
|
|
@@ -81,8 +95,8 @@ export function buildIssueUrl(category, message, context) {
|
|
|
81
95
|
return `${TRACKER_URL}/issues/new?${query.toString()}`;
|
|
82
96
|
}
|
|
83
97
|
/** Bundle a report: message, category, context, and the pre-filled URL. */
|
|
84
|
-
export function buildFeedback(message, category) {
|
|
85
|
-
const context = collectContext();
|
|
98
|
+
export function buildFeedback(message, category, version = 'unknown') {
|
|
99
|
+
const context = collectContext(version);
|
|
86
100
|
return {
|
|
87
101
|
message,
|
|
88
102
|
category,
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared gate-abstention helper (issue #508, no-silent-abstention spec).
|
|
3
|
+
*
|
|
4
|
+
* Doctrine: a check that verified zero items has ABSTAINED, not passed.
|
|
5
|
+
* Every gate reports its denominator (`checked`); zero is a distinct loud
|
|
6
|
+
* outcome. "Skipped" renders in every summary line and never aggregates
|
|
7
|
+
* into "passed" (D7).
|
|
8
|
+
*
|
|
9
|
+
* `gateOutcome` is the only path to a summary line for swept commands, so
|
|
10
|
+
* the refusal to print bare success on a zero denominator is structural.
|
|
11
|
+
* Surfaces append their own remediation text (why the denominator
|
|
12
|
+
* collapsed, first fix step) after the summary line.
|
|
13
|
+
*
|
|
14
|
+
* Output glyphs are written as `\u{...}` escapes so this source stays
|
|
15
|
+
* ASCII while the emitted bytes match the rest of the CLI (warning sign
|
|
16
|
+
* U+26A0, em dash U+2014).
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* Reserved CLI-wide (D4): exit 3 always means "abstained -- verified zero
|
|
20
|
+
* items", distinct from 0 (clean), 1 (findings), 2 (surface-specific).
|
|
21
|
+
*/
|
|
22
|
+
export const EXIT_ABSTAINED = 3;
|
|
23
|
+
const WARN = '\u{26A0}'; // warning sign
|
|
24
|
+
const EMDASH = '\u{2014}'; // em dash
|
|
25
|
+
// C0 controls (incl. \n, ESC) and DEL: a skip name must never be able to
|
|
26
|
+
// forge output lines or smuggle ANSI sequences into the summary.
|
|
27
|
+
const CONTROL_CHARS = /[\u0000-\u001F\u007F]/g;
|
|
28
|
+
/**
|
|
29
|
+
* D7: skipped entries render in EVERY summary line.
|
|
30
|
+
*
|
|
31
|
+
* Exported so a surface with its own failure vocabulary (doctor says
|
|
32
|
+
* "check(s) failed", not "finding(s)") can render the identical skip suffix
|
|
33
|
+
* instead of re-deriving the format -- the decision still comes from
|
|
34
|
+
* {@link gateOutcome}, only the noun differs.
|
|
35
|
+
*/
|
|
36
|
+
export function skippedSuffix(skipped) {
|
|
37
|
+
if (!skipped || skipped.length === 0)
|
|
38
|
+
return '';
|
|
39
|
+
const names = skipped
|
|
40
|
+
.map((s) => s.name.replace(CONTROL_CHARS, ''))
|
|
41
|
+
.join(', ');
|
|
42
|
+
return ` (${skipped.length} skipped: ${names})`;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* The single summary-line/exit-code path for swept commands.
|
|
46
|
+
*
|
|
47
|
+
* Non-abstained exit codes are helper defaults (findings -> 1 for gates);
|
|
48
|
+
* surfaces with richer contracts (e.g. freshness 2 = local edits) apply
|
|
49
|
+
* their own mapping AFTER checking `abstained`.
|
|
50
|
+
*/
|
|
51
|
+
export function gateOutcome(result, kind, opts = {}) {
|
|
52
|
+
const noun = opts.noun ?? 'check(s)';
|
|
53
|
+
const suffix = skippedSuffix(result.skipped);
|
|
54
|
+
// Findings outrank abstention: a finding proves something was checked,
|
|
55
|
+
// so it must never be masked by a collapsed/invalid denominator.
|
|
56
|
+
if (result.findings.length > 0) {
|
|
57
|
+
return {
|
|
58
|
+
exitCode: kind === 'gate' ? 1 : 0,
|
|
59
|
+
abstained: false,
|
|
60
|
+
summaryLine: `${result.findings.length} finding(s) across ` +
|
|
61
|
+
`${result.checked} checked${suffix}`,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
// Negated comparison so 0, negatives, and NaN all abstain: an invalid
|
|
65
|
+
// denominator must never render as success.
|
|
66
|
+
if (!(result.checked > 0)) {
|
|
67
|
+
return {
|
|
68
|
+
exitCode: kind === 'gate' ? EXIT_ABSTAINED : 0,
|
|
69
|
+
abstained: true,
|
|
70
|
+
summaryLine: `${WARN} Abstained ${EMDASH} verified zero items; ` +
|
|
71
|
+
`this is not a pass.${suffix}`,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
return {
|
|
75
|
+
exitCode: 0,
|
|
76
|
+
abstained: false,
|
|
77
|
+
summaryLine: `All ${result.checked} run ${noun} passed${suffix}`,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=gate-result.js.map
|