canary-test-cli 7.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/skills/README.md +308 -0
- package/agents/skills/canary:generate.md +49 -0
- package/agents/skills/canary:init.md +37 -0
- package/agents/skills/canary:migrate.md +66 -0
- package/agents/skills/claude-code/canary-add-framework/SKILL.md +248 -0
- package/agents/skills/claude-code/canary-blackhawk/SKILL.md +170 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/cli.mjs +188 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/rules.mjs +120 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/scanner.mjs +244 -0
- package/agents/skills/claude-code/canary-blackhawk/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-cassandra/SKILL.md +180 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +268 -0
- package/agents/skills/claude-code/canary-cassandra/scripts/engine.mjs +95 -0
- package/agents/skills/claude-code/canary-ci-ready/SKILL.md +161 -0
- package/agents/skills/claude-code/canary-ci-ready/skill.yaml +14 -0
- package/agents/skills/claude-code/canary-company-knowledge/SKILL.md +196 -0
- package/agents/skills/claude-code/canary-critical-areas/SKILL.md +142 -0
- package/agents/skills/claude-code/canary-critical-areas/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/SKILL.md +160 -0
- package/agents/skills/claude-code/canary-edge-case-discovery/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-fail-fast/SKILL.md +75 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/cli.mjs +118 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/digest.mjs +69 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/failures.mjs +60 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/fastfail_check.mjs +43 -0
- package/agents/skills/claude-code/canary-fail-fast/scripts/parse.mjs +149 -0
- package/agents/skills/claude-code/canary-failure-impact/SKILL.md +153 -0
- package/agents/skills/claude-code/canary-failure-impact/skill.yaml +15 -0
- package/agents/skills/claude-code/canary-fleet-health/SKILL.md +196 -0
- package/agents/skills/claude-code/canary-generate-test/SKILL.md +185 -0
- package/agents/skills/claude-code/canary-instrument/SKILL.md +157 -0
- package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs +178 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs +96 -0
- package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts +44 -0
- package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs +81 -0
- package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs +187 -0
- package/agents/skills/claude-code/canary-katana/SKILL.md +243 -0
- package/agents/skills/claude-code/canary-katana/scripts/alarm.mjs +296 -0
- package/agents/skills/claude-code/canary-katana/scripts/cli.mjs +247 -0
- package/agents/skills/claude-code/canary-katana/scripts/diffscan.mjs +0 -0
- package/agents/skills/claude-code/canary-katana/scripts/ledger.mjs +183 -0
- package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +129 -0
- package/agents/skills/claude-code/canary-pr-guardian/skill.yaml +17 -0
- package/agents/skills/claude-code/canary-promote-test/SKILL.md +228 -0
- package/agents/skills/claude-code/canary-savant/SKILL.md +233 -0
- package/agents/skills/claude-code/canary-savant/scripts/cli.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/restoration.mjs +274 -0
- package/agents/skills/claude-code/canary-savant/scripts/rules.mjs +168 -0
- package/agents/skills/claude-code/canary-savant/scripts/runner.mjs +572 -0
- package/agents/skills/claude-code/canary-savant/scripts/scanner.mjs +374 -0
- package/agents/skills/claude-code/canary-savant/scripts/string-literals.mjs +116 -0
- package/agents/skills/claude-code/canary-setup-harness/SKILL.md +263 -0
- package/agents/skills/claude-code/canary-shadow/SKILL.md +131 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cases.example.json +32 -0
- package/agents/skills/claude-code/canary-shadow/scripts/cli.mjs +195 -0
- package/agents/skills/claude-code/canary-ship/SKILL.md +177 -0
- package/agents/skills/claude-code/canary-ship/skill.yaml +16 -0
- package/agents/skills/claude-code/canary-strix/SKILL.md +130 -0
- package/agents/skills/claude-code/canary-strix/scripts/cli.mjs +255 -0
- package/agents/skills/claude-code/canary-strix/scripts/scanner.mjs +252 -0
- package/agents/skills/claude-code/canary-strix/scripts/terms.mjs +132 -0
- package/agents/skills/claude-code/canary-test-pipeline/SKILL.md +159 -0
- package/agents/skills/claude-code/canary-test-pipeline/skill.yaml +19 -0
- package/agents/skills/claude-code/canary-test-reporter/SKILL.md +138 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/cli.mjs +98 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/json_report.mjs +58 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/parse.mjs +216 -0
- package/agents/skills/claude-code/canary-test-reporter/scripts/render.mjs +114 -0
- package/agents/skills/lib/parse-args.mjs +214 -0
- package/dist/engine/analysis/cli.js +116 -54
- package/dist/engine/analysis/engine.js +34 -16
- package/dist/engine/analysis/reports.js +5 -4
- package/dist/engine/cli-commands.js +252 -106
- package/dist/engine/cli-common.js +15 -24
- package/dist/engine/cli.core.js +37 -11
- package/dist/engine/cli.js +2 -2
- package/dist/engine/company-knowledge-cli.js +2 -2
- package/dist/engine/core/adoption.js +408 -0
- package/dist/engine/core/framework-probes.js +7 -7
- package/dist/engine/core/fs-glob.js +2 -2
- package/dist/engine/core/gate-result.js +17 -0
- package/dist/engine/core/migrator.js +9 -17
- package/dist/engine/core/pattern-matcher.js +23 -5
- package/dist/engine/core/persona.js +421 -0
- package/dist/engine/core/promotion-verdict.js +261 -0
- package/dist/engine/core/quality-scorer.js +15 -2
- package/dist/engine/core/reporter.js +1 -9
- package/dist/engine/core/skill-dispatch.js +115 -0
- package/dist/engine/core/skill-examples.js +392 -0
- package/dist/engine/core/skill-registry.js +59 -4
- package/dist/engine/core/skill-surfaces.js +307 -0
- package/dist/engine/core/static-linter.js +310 -38
- package/dist/engine/core/test-files.js +77 -0
- package/dist/engine/core/ticket-updater.js +1 -7
- package/dist/engine/core/vacuity-scanner.js +726 -0
- package/dist/engine/core/workflow-discovery.js +2 -8
- package/dist/engine/core/workspace-detect.js +7 -6
- package/dist/engine/data/personas/registry.json +36 -0
- package/dist/engine/guardian/adjudication.js +5 -5
- package/dist/engine/guardian/analysis-emit.js +19 -28
- package/dist/engine/guardian/cli.js +150 -51
- package/dist/engine/guardian/coverage.js +1 -1
- package/dist/engine/guardian/diff-coverage/heuristic-tier.js +1 -1
- package/dist/engine/guardian/diff-coverage/orchestrator.js +2 -2
- package/dist/engine/guardian/pr-check.js +322 -33
- package/dist/engine/guardian/pr-comment.js +4 -3
- package/dist/engine/history/cli.js +210 -6
- package/dist/engine/history/ndjson-store.js +9 -5
- package/dist/engine/history/record.js +34 -5
- package/dist/engine/history/run-recorder.js +165 -0
- package/dist/engine/history/schema.js +25 -7
- package/dist/engine/history/store.js +9 -0
- package/dist/engine/mcp-server.js +35 -13
- package/dist/engine/skills-cli.js +249 -23
- package/dist/engine/util/ensure-ascii.js +37 -0
- package/dist/engine/workflow-cli.js +6 -6
- package/dist/gate-result.d.ts +11 -0
- package/dist/gate-result.js +18 -0
- package/dist/reporters/testtracker.d.ts +1 -1
- package/dist/reporters/testtracker.js +1 -1
- package/dist/uninstall.js +12 -5
- package/package.json +3 -2
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Follows the guardian CLI conventions (see `../cli-common.ts`): a
|
|
7
7
|
* {@link createHistoryCommand} factory wired to an injectable {@link HistoryDeps}
|
|
8
|
-
* (out/err sinks, env, a store factory), `
|
|
8
|
+
* (out/err sinks, env, a store factory), `CliExitError` for business exits, and
|
|
9
9
|
* `normalizeUsageExit` on every command so usage errors exit 2.
|
|
10
10
|
*
|
|
11
11
|
* Python->TS fidelity notes:
|
|
@@ -20,14 +20,22 @@
|
|
|
20
20
|
* no Python CLI test pinning them, so this port emits a simple aligned text
|
|
21
21
|
* table carrying the SAME cell content. `summary`/`push`/`migrate` are NOT
|
|
22
22
|
* tables and are reproduced byte-for-byte via picocolors stripping.
|
|
23
|
+
*
|
|
24
|
+
* `record` is NOT a port. It has no Python counterpart: nothing in either engine
|
|
25
|
+
* ever wrote the local store, which is why the whole `analyze` / `history`
|
|
26
|
+
* surface had only ever been exercised against synthetic fixtures (#538). Its
|
|
27
|
+
* conversion half lives in `run-recorder.ts`; it takes the async store contract
|
|
28
|
+
* per ADR 0013, so it writes local NDJSON or the remote store by configuration.
|
|
23
29
|
*/
|
|
24
30
|
import { existsSync, readFileSync } from 'node:fs';
|
|
25
31
|
import { Command, Option } from 'commander';
|
|
26
32
|
import pc from 'picocolors';
|
|
27
|
-
import {
|
|
33
|
+
import { CliExitError, jsonIndent2, normalizeUsageExit, } from '../cli-common.js';
|
|
28
34
|
import { gateOutcome } from '../core/gate-result.js';
|
|
29
35
|
import { makeRunId } from './schema.js';
|
|
30
36
|
import { makeStore as realMakeStore } from './store.js';
|
|
37
|
+
import { buildRunFromVitestReport, countReportResults, detectReportShape, RecordValidationError, } from './run-recorder.js';
|
|
38
|
+
import { def } from '../util/coalesce.js';
|
|
31
39
|
import { pyFloat } from '../util/round.js';
|
|
32
40
|
const EM_DASH = '\u{2014}';
|
|
33
41
|
const GEQ = '\u{2265}';
|
|
@@ -126,7 +134,7 @@ function renderTable(title, headers, rows, rightAlign) {
|
|
|
126
134
|
async function pushCmd(historyFile, opts, deps) {
|
|
127
135
|
if (!existsSync(historyFile)) {
|
|
128
136
|
deps.out(`${pc.red('Not found:')} ${historyFile}`);
|
|
129
|
-
throw new
|
|
137
|
+
throw new CliExitError(1);
|
|
130
138
|
}
|
|
131
139
|
const store = deps.makeStore(opts.dbUrl, historyFile);
|
|
132
140
|
const records = [];
|
|
@@ -137,7 +145,7 @@ async function pushCmd(historyFile, opts, deps) {
|
|
|
137
145
|
}
|
|
138
146
|
if (records.length === 0) {
|
|
139
147
|
deps.out(pc.yellow('No runs found in history file.'));
|
|
140
|
-
throw new
|
|
148
|
+
throw new CliExitError(0);
|
|
141
149
|
}
|
|
142
150
|
const latest = { ...records[records.length - 1] };
|
|
143
151
|
const testsRaw = latest['tests'] ?? [];
|
|
@@ -146,11 +154,188 @@ async function pushCmd(historyFile, opts, deps) {
|
|
|
146
154
|
const results = testsRaw.map((t) => pick(t, RESULT_FIELDS));
|
|
147
155
|
if (opts.dryRun) {
|
|
148
156
|
deps.out(`${pc.cyan('dry-run:')} would push run ${pc.bold(run.run_id)} (${results.length} tests)`);
|
|
149
|
-
throw new
|
|
157
|
+
throw new CliExitError(0);
|
|
150
158
|
}
|
|
151
159
|
await store.pushRun(run, results);
|
|
152
160
|
deps.out(`${pc.green('Pushed')} run ${pc.bold(run.run_id)} (${results.length} tests)`);
|
|
153
161
|
}
|
|
162
|
+
/** The `flaky: 0` caveat, printed rather than buried in a source comment. */
|
|
163
|
+
const FLAKY_VOCABULARY_NOTE = 'note: flaky=0 \u{2014} vitest reports no flaky status, so a test that was ' +
|
|
164
|
+
'retried and then passed is recorded as passed.';
|
|
165
|
+
/** Read + parse the results file, or exit 1 having said which and why. */
|
|
166
|
+
function readReport(resultsFile, deps) {
|
|
167
|
+
if (!existsSync(resultsFile)) {
|
|
168
|
+
deps.out(`${pc.red('Not found:')} ${resultsFile}`);
|
|
169
|
+
throw new CliExitError(1);
|
|
170
|
+
}
|
|
171
|
+
try {
|
|
172
|
+
return JSON.parse(readFileSync(resultsFile, 'utf-8'));
|
|
173
|
+
}
|
|
174
|
+
catch (err) {
|
|
175
|
+
// Loud, not silent: an unreadable report means this run recorded NOTHING,
|
|
176
|
+
// and a later `analyze` would abstain without ever saying why.
|
|
177
|
+
deps.out(`${pc.red('Could not be read:')} ${resultsFile} ` +
|
|
178
|
+
`(${err.message}) \u{2014} nothing was recorded.`);
|
|
179
|
+
throw new CliExitError(1);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
/** The repo slug, or exit 2 -- never a hardcoded default (see #538). */
|
|
183
|
+
function resolveRepo(opts, deps) {
|
|
184
|
+
const repo = def(opts.repo, deps.env['GITHUB_REPOSITORY']);
|
|
185
|
+
if (repo)
|
|
186
|
+
return repo;
|
|
187
|
+
// A hardcoded default repo slug is how a tool ends up filing one team's runs
|
|
188
|
+
// under another team's name. Usage error (exit 2), not a guess.
|
|
189
|
+
deps.out(`${pc.red('Missing --repo:')} the repo slug could not be inferred ` +
|
|
190
|
+
`(no GITHUB_REPOSITORY in the environment). Pass ` +
|
|
191
|
+
`--repo <owner>/<name>.`);
|
|
192
|
+
throw new CliExitError(2);
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Resolve the run-level facts a runner's report cannot carry.
|
|
196
|
+
*
|
|
197
|
+
* Fallbacks go through `def()` rather than `??` chains: as a call it is not a
|
|
198
|
+
* decision point, which keeps this mapper under the arch gate's per-function
|
|
199
|
+
* complexity threshold (the same reason the store's row mappers use it).
|
|
200
|
+
*/
|
|
201
|
+
function recordContext(opts, deps) {
|
|
202
|
+
const runId = opts.runId;
|
|
203
|
+
return {
|
|
204
|
+
suite: opts.suite,
|
|
205
|
+
repo: resolveRepo(opts, deps),
|
|
206
|
+
branch: def(opts.branch, def(deps.env['GITHUB_REF_NAME'], 'local')),
|
|
207
|
+
commitSha: def(opts.commit, def(deps.env['GITHUB_SHA'], 'local')),
|
|
208
|
+
// `exactOptionalPropertyTypes`: an absent --run-id omits the key rather
|
|
209
|
+
// than setting it to undefined.
|
|
210
|
+
...(runId === undefined ? {} : { runId }),
|
|
211
|
+
nowMs: Date.now(),
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* `record` is a GATE (ADR 0009): a zero-result report exits 3, because the
|
|
216
|
+
* caller is a CI step whose next command reads the store, and "the suite
|
|
217
|
+
* reported nothing" must not look like "the suite passed". #538.
|
|
218
|
+
*/
|
|
219
|
+
async function recordCmd(resultsFile, opts, deps) {
|
|
220
|
+
const parsed = readReport(resultsFile, deps);
|
|
221
|
+
if (detectReportShape(parsed) !== 'vitest') {
|
|
222
|
+
deps.out(`${pc.red('Unrecognized report:')} ${resultsFile} carries no ` +
|
|
223
|
+
`\`testResults\` array, so it is not a vitest --reporter=json report. ` +
|
|
224
|
+
`\`record\` reads vitest JSON today; Playwright JSON and JUnit XML ` +
|
|
225
|
+
`are not supported yet.`);
|
|
226
|
+
throw new CliExitError(1);
|
|
227
|
+
}
|
|
228
|
+
// The denominator first: a report with no results is an abstention whatever
|
|
229
|
+
// else is missing, and refusing it over an unresolvable --repo would report
|
|
230
|
+
// the wrong finding.
|
|
231
|
+
if (countReportResults(parsed) === 0) {
|
|
232
|
+
abstainOnEmptyReport(resultsFile, opts, deps);
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
const ctx = recordContext(opts, deps);
|
|
236
|
+
let built;
|
|
237
|
+
try {
|
|
238
|
+
built = buildRunFromVitestReport(parsed, ctx);
|
|
239
|
+
}
|
|
240
|
+
catch (err) {
|
|
241
|
+
if (!(err instanceof RecordValidationError))
|
|
242
|
+
throw err;
|
|
243
|
+
deps.out(`${pc.red('Invalid results:')} ${err.message}. Nothing was recorded ` +
|
|
244
|
+
`\u{2014} a malformed record degrades every later read of the store.`);
|
|
245
|
+
throw new CliExitError(1);
|
|
246
|
+
}
|
|
247
|
+
const remote = opts.dbUrl ?? deps.env['CANARY_HISTORY_DB_URL'];
|
|
248
|
+
const storePath = opts.path ?? DEFAULT_HISTORY_FILE;
|
|
249
|
+
const target = remote ? 'the configured remote store' : storePath;
|
|
250
|
+
if (opts.dryRun) {
|
|
251
|
+
reportDryRun(built, target, opts, deps);
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
if (remote && opts.path) {
|
|
255
|
+
deps.err(`note: --path is ignored while a db-url is configured; the run goes to ` +
|
|
256
|
+
`the remote store.`);
|
|
257
|
+
}
|
|
258
|
+
const store = deps.makeStore(opts.dbUrl, storePath);
|
|
259
|
+
const before = store.countRuns ? await store.countRuns() : null;
|
|
260
|
+
await store.pushRun(built.run, built.results);
|
|
261
|
+
const after = store.countRuns ? await store.countRuns() : null;
|
|
262
|
+
// `pushRun` skips a duplicate run_id SILENTLY (it is idempotent by design).
|
|
263
|
+
// For a CLI that is the wrong default: the caller believes it recorded a run.
|
|
264
|
+
if (before !== null && after === before) {
|
|
265
|
+
deps.out(`${pc.yellow('Already recorded:')} run_id ` +
|
|
266
|
+
`${pc.bold(built.run.run_id)} is already present in ${target}; ` +
|
|
267
|
+
`nothing was appended. Two runs of one suite at one commit inside the ` +
|
|
268
|
+
`same second collide \u{2014} pass --run-id to record a distinct run.`);
|
|
269
|
+
throw new CliExitError(1);
|
|
270
|
+
}
|
|
271
|
+
if (before === null) {
|
|
272
|
+
// Cannot verify is a finding, not a silence (#508).
|
|
273
|
+
deps.err(`note: this backend cannot report how many runs it holds, so a ` +
|
|
274
|
+
`duplicate run_id could not be verified \u{2014} the store skips ` +
|
|
275
|
+
`duplicates silently.`);
|
|
276
|
+
}
|
|
277
|
+
reportRecorded(built, target, opts, deps);
|
|
278
|
+
}
|
|
279
|
+
function abstainOnEmptyReport(resultsFile, opts, deps) {
|
|
280
|
+
const outcome = gateOutcome({ checked: 0, findings: [] }, 'gate');
|
|
281
|
+
const notice = `${outcome.summaryLine} ${resultsFile} carried zero test results, so ` +
|
|
282
|
+
`nothing was recorded \u{2014} an empty run is the denominator ` +
|
|
283
|
+
`collapsing, not a passing suite. Check that the runner wrote its report ` +
|
|
284
|
+
`(\`vitest --reporter=json --outputFile=<path>\`) and that the suite ran.`;
|
|
285
|
+
if (opts.json) {
|
|
286
|
+
deps.out(jsonIndent2({
|
|
287
|
+
suite: opts.suite,
|
|
288
|
+
checked: 0,
|
|
289
|
+
recorded: false,
|
|
290
|
+
abstained: true,
|
|
291
|
+
}));
|
|
292
|
+
deps.err(notice);
|
|
293
|
+
}
|
|
294
|
+
else {
|
|
295
|
+
deps.out(notice);
|
|
296
|
+
}
|
|
297
|
+
throw new CliExitError(outcome.exitCode);
|
|
298
|
+
}
|
|
299
|
+
/** The success payload/line. Shared shape so `--json` cannot drift from it. */
|
|
300
|
+
function recordPayload(built, target, extra) {
|
|
301
|
+
const { run } = built;
|
|
302
|
+
return {
|
|
303
|
+
run_id: run.run_id,
|
|
304
|
+
suite: run.suite,
|
|
305
|
+
repo: run.repo,
|
|
306
|
+
target,
|
|
307
|
+
checked: built.results.length,
|
|
308
|
+
passed: run.passed,
|
|
309
|
+
failed: run.failed,
|
|
310
|
+
flaky: run.flaky,
|
|
311
|
+
skipped: run.skipped,
|
|
312
|
+
abstained: false,
|
|
313
|
+
...extra,
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
function countsLine(built) {
|
|
317
|
+
const { run } = built;
|
|
318
|
+
return (`${built.results.length} result(s) for ${pc.bold(run.suite)} ` +
|
|
319
|
+
`(${run.passed} passed, ${run.failed} failed, ${run.skipped} skipped)`);
|
|
320
|
+
}
|
|
321
|
+
function reportDryRun(built, target, opts, deps) {
|
|
322
|
+
if (opts.json) {
|
|
323
|
+
deps.out(jsonIndent2(recordPayload(built, target, { recorded: false, dry_run: true })));
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
deps.out(`${pc.cyan('dry-run:')} would record ${countsLine(built)} as ` +
|
|
327
|
+
`${pc.bold(built.run.run_id)} \u{2192} ${target}`);
|
|
328
|
+
deps.out(FLAKY_VOCABULARY_NOTE);
|
|
329
|
+
}
|
|
330
|
+
function reportRecorded(built, target, opts, deps) {
|
|
331
|
+
if (opts.json) {
|
|
332
|
+
deps.out(jsonIndent2(recordPayload(built, target, { recorded: true })));
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
deps.out(`${pc.green('Recorded')} ${countsLine(built)} \u{2192} ${target}`);
|
|
336
|
+
deps.out(`run_id: ${built.run.run_id}`);
|
|
337
|
+
deps.out(FLAKY_VOCABULARY_NOTE);
|
|
338
|
+
}
|
|
154
339
|
async function flakyCmd(opts, deps) {
|
|
155
340
|
const store = deps.makeStore(opts.dbUrl);
|
|
156
341
|
if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'flake rate')) {
|
|
@@ -241,7 +426,7 @@ async function summaryCmd(suite, opts, deps) {
|
|
|
241
426
|
async function migrateCmd(file, opts, deps) {
|
|
242
427
|
if (!existsSync(file)) {
|
|
243
428
|
deps.out(`${pc.red('Not found:')} ${file}`);
|
|
244
|
-
throw new
|
|
429
|
+
throw new CliExitError(1);
|
|
245
430
|
}
|
|
246
431
|
const store = deps.makeStore(opts.dbUrl);
|
|
247
432
|
let migrated = 0;
|
|
@@ -313,6 +498,25 @@ export function createHistoryCommand(depsInit = {}) {
|
|
|
313
498
|
.action(async (historyFile, opts) => {
|
|
314
499
|
await pushCmd(historyFile, opts, deps);
|
|
315
500
|
});
|
|
501
|
+
program
|
|
502
|
+
.command('record')
|
|
503
|
+
.description('Record a finished test run into the history store (vitest JSON).')
|
|
504
|
+
.argument('<results_file>', "Path to the runner's JSON report.")
|
|
505
|
+
.requiredOption('--suite <suite>', 'Suite name for this run (e.g. e2e).')
|
|
506
|
+
.option('--repo <repo>', 'GitHub repo slug (default: $GITHUB_REPOSITORY).')
|
|
507
|
+
.option('--branch <branch>', 'Branch name (default: $GITHUB_REF_NAME).')
|
|
508
|
+
.option('--commit <sha>', 'Commit SHA (default: $GITHUB_SHA).')
|
|
509
|
+
// No commander default: an explicitly-passed --path has to stay
|
|
510
|
+
// distinguishable from the fallback, so a db-url + --path combination can
|
|
511
|
+
// say that --path is unused instead of silently dropping it.
|
|
512
|
+
.option('--path <store>', `Local NDJSON store to append to (default: ${DEFAULT_HISTORY_FILE}).`)
|
|
513
|
+
.option('--run-id <id>', 'Run id (default: <suite>-<commit[:8]>-<epoch seconds>).')
|
|
514
|
+
.addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
|
|
515
|
+
.option('--dry-run', 'Show what would be recorded without writing.')
|
|
516
|
+
.option('--json')
|
|
517
|
+
.action(async (resultsFile, opts) => {
|
|
518
|
+
await recordCmd(resultsFile, opts, deps);
|
|
519
|
+
});
|
|
316
520
|
program
|
|
317
521
|
.command('flaky')
|
|
318
522
|
.description('Show tests ranked by flake rate over the rolling window.')
|
|
@@ -7,14 +7,18 @@
|
|
|
7
7
|
*
|
|
8
8
|
* Deviation from the Python reader (deliberate): where Python's `_read_all`
|
|
9
9
|
* swallows any parse error and returns `[]`, this reader throws on malformed
|
|
10
|
-
* JSON and on an
|
|
11
|
-
*
|
|
10
|
+
* JSON and on an unrecognized `schema_version`, so a corrupt or future-version
|
|
11
|
+
* history fails loudly rather than silently analysing nothing.
|
|
12
|
+
*
|
|
13
|
+
* Version resolution lives in `resolveSchemaVersion` (#701): rows written by
|
|
14
|
+
* this store carry their version, and a legacy unstamped row is read as the
|
|
15
|
+
* version it was written at rather than as "current".
|
|
12
16
|
*/
|
|
13
17
|
import { appendFileSync, mkdirSync, readFileSync } from 'node:fs';
|
|
14
18
|
import { dirname } from 'node:path';
|
|
15
19
|
import { def } from '../util/coalesce.js';
|
|
16
20
|
import { round1 } from '../util/round.js';
|
|
17
|
-
import { SCHEMA_VERSION } from './record.js';
|
|
21
|
+
import { SCHEMA_VERSION, resolveSchemaVersion } from './record.js';
|
|
18
22
|
import { serializeLocalRecord } from './schema.js';
|
|
19
23
|
export class NdjsonHistoryStore {
|
|
20
24
|
path;
|
|
@@ -38,8 +42,8 @@ export class NdjsonHistoryStore {
|
|
|
38
42
|
if (!line)
|
|
39
43
|
continue;
|
|
40
44
|
const record = JSON.parse(line);
|
|
41
|
-
const version = record
|
|
42
|
-
if (version !==
|
|
45
|
+
const version = resolveSchemaVersion(record);
|
|
46
|
+
if (version !== SCHEMA_VERSION) {
|
|
43
47
|
throw new Error(`Unsupported history schema_version ${version} (expected ${SCHEMA_VERSION})`);
|
|
44
48
|
}
|
|
45
49
|
records.push(record);
|
|
@@ -4,11 +4,40 @@
|
|
|
4
4
|
* Mirrors `agent/history/schema.py` (RunRecord + TestResult) as persisted, one
|
|
5
5
|
* JSON object per line, in `test-results/reports/history-v2.jsonl`.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
* version
|
|
9
|
-
*
|
|
10
|
-
*
|
|
7
|
+
* Every row the store writes now carries its own `schema_version` (#701) — the
|
|
8
|
+
* version guard can only refuse data it can identify, and before the stamp the
|
|
9
|
+
* store's own history was the one thing it could never check.
|
|
10
|
+
*
|
|
11
|
+
* A row with no version is a **legacy row**, written before that stamp landed.
|
|
12
|
+
* It is resolved to `LEGACY_UNVERSIONED_SCHEMA_VERSION`, not to "current" — see
|
|
13
|
+
* `resolveSchemaVersion` for why the difference is load-bearing.
|
|
11
14
|
*/
|
|
12
|
-
/** The schema version this reader understands. */
|
|
15
|
+
/** The schema version this reader understands, and every writer stamps. */
|
|
13
16
|
export const SCHEMA_VERSION = 2;
|
|
17
|
+
/**
|
|
18
|
+
* The version an unstamped row was actually written at.
|
|
19
|
+
*
|
|
20
|
+
* **A frozen literal, never an alias of `SCHEMA_VERSION`.** Unversioned rows
|
|
21
|
+
* can only be v2: v2 is the sole version this store has ever had (the earlier
|
|
22
|
+
* "v1" lived in a differently-named file), and from #701 onward every row is
|
|
23
|
+
* stamped. So the value is a historical fact, not a default.
|
|
24
|
+
*
|
|
25
|
+
* Aliasing it to `SCHEMA_VERSION` would recreate the #701 bug on the day the
|
|
26
|
+
* version bumps: legacy v2 rows would be read as v3 and silently
|
|
27
|
+
* misinterpreted, which is precisely the guard's purpose to prevent.
|
|
28
|
+
*/
|
|
29
|
+
export const LEGACY_UNVERSIONED_SCHEMA_VERSION = 2;
|
|
30
|
+
/**
|
|
31
|
+
* The version a record should be read as.
|
|
32
|
+
*
|
|
33
|
+
* Deliberately conservative: an unversioned row resolves to the version it was
|
|
34
|
+
* written at, so the next `SCHEMA_VERSION` bump makes the reader **refuse**
|
|
35
|
+
* legacy rows loudly rather than reinterpret them under new semantics. That
|
|
36
|
+
* bump therefore has to ship a migration (rewrite the file with a stamp, or add
|
|
37
|
+
* an explicit upgrade path) — which is the point. Today the two constants are
|
|
38
|
+
* equal, so this changes nothing observable; it only decides what happens next.
|
|
39
|
+
*/
|
|
40
|
+
export function resolveSchemaVersion(record) {
|
|
41
|
+
return record.schema_version ?? LEGACY_UNVERSIONED_SCHEMA_VERSION;
|
|
42
|
+
}
|
|
14
43
|
//# sourceMappingURL=record.js.map
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turn a test runner's own report into a v2 history run (#538).
|
|
3
|
+
*
|
|
4
|
+
* `canary analyze` and `canary history` have always READ
|
|
5
|
+
* `test-results/reports/history-v2.jsonl`; until this module nothing in the
|
|
6
|
+
* product wrote it, so every consumer who wanted a flake report had to author
|
|
7
|
+
* the NDJSON by hand. This is the conversion half of `canary history record`;
|
|
8
|
+
* the CLI half lives in `cli.ts` and the append itself belongs to the store
|
|
9
|
+
* (`AsyncHistoryStore.pushRun`, per ADR 0013).
|
|
10
|
+
*
|
|
11
|
+
* Two deliberate limits, both of them data-honesty calls rather than laziness:
|
|
12
|
+
*
|
|
13
|
+
* - **vitest `--reporter=json` only.** The shape is detected rather than
|
|
14
|
+
* declared by flag, and an unrecognized shape is refused loudly. The
|
|
15
|
+
* alternative -- reading an unknown document and finding zero tests in it --
|
|
16
|
+
* is indistinguishable from a suite that genuinely ran nothing, which is
|
|
17
|
+
* exactly the denominator collapse #508 exists to make visible. Playwright
|
|
18
|
+
* JSON and JUnit XML are parsed elsewhere in the repo (canary-test-reporter,
|
|
19
|
+
* canary-savant) but by self-contained skill CLIs the engine cannot import.
|
|
20
|
+
* - **`flaky` is always 0.** Canary's status vocabulary is
|
|
21
|
+
* passed/failed/flaky/skipped; vitest has no flaky status, so a test that
|
|
22
|
+
* was retried and then passed arrives as `passed` and is invisible here.
|
|
23
|
+
* Recorded as zero and SAID OUT LOUD by the command, because a reader who
|
|
24
|
+
* takes `flaky: 0` for a clean fleet has been misled by the tool.
|
|
25
|
+
*/
|
|
26
|
+
import { makeRunId, serializeLocalRecord, } from './schema.js';
|
|
27
|
+
/** Canary's per-test status vocabulary (the store's read side keys on these). */
|
|
28
|
+
const RECORD_STATUSES = ['passed', 'failed', 'flaky', 'skipped'];
|
|
29
|
+
/**
|
|
30
|
+
* A report that cannot be recorded without corrupting the store.
|
|
31
|
+
*
|
|
32
|
+
* Thrown BEFORE any append: `NdjsonHistoryStore.pushRun` writes one line and
|
|
33
|
+
* every later report divides by it, so a malformed record is not a local
|
|
34
|
+
* failure -- it silently degrades every future read of the file.
|
|
35
|
+
*/
|
|
36
|
+
export class RecordValidationError extends Error {
|
|
37
|
+
constructor(message) {
|
|
38
|
+
super(message);
|
|
39
|
+
this.name = 'RecordValidationError';
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Detect by shape, not by flag -- callers should not have to know the format. */
|
|
43
|
+
export function detectReportShape(parsed) {
|
|
44
|
+
if (parsed === null || typeof parsed !== 'object')
|
|
45
|
+
return 'unknown';
|
|
46
|
+
const testResults = parsed.testResults;
|
|
47
|
+
return Array.isArray(testResults) ? 'vitest' : 'unknown';
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* How many per-test results the report carries -- the DENOMINATOR, available
|
|
51
|
+
* before any run-level fact is resolved.
|
|
52
|
+
*
|
|
53
|
+
* Separate from the conversion so `record` can abstain on an empty report
|
|
54
|
+
* before it starts complaining about a missing `--repo`: there is nothing to
|
|
55
|
+
* record either way, and the abstention is the finding worth reporting.
|
|
56
|
+
*/
|
|
57
|
+
export function countReportResults(parsed) {
|
|
58
|
+
const report = parsed;
|
|
59
|
+
let n = 0;
|
|
60
|
+
for (const file of report.testResults ?? []) {
|
|
61
|
+
n += file.assertionResults?.length ?? 0;
|
|
62
|
+
}
|
|
63
|
+
return n;
|
|
64
|
+
}
|
|
65
|
+
/** vitest status -> canary status. Anything not pass/fail is a skip. */
|
|
66
|
+
function toCanaryStatus(status) {
|
|
67
|
+
if (status === 'passed')
|
|
68
|
+
return 'passed';
|
|
69
|
+
if (status === 'failed')
|
|
70
|
+
return 'failed';
|
|
71
|
+
return 'skipped';
|
|
72
|
+
}
|
|
73
|
+
function toResultRow(assertion, file, ids) {
|
|
74
|
+
const first = assertion.failureMessages?.[0];
|
|
75
|
+
return {
|
|
76
|
+
run_id: ids.runId,
|
|
77
|
+
suite: ids.suite,
|
|
78
|
+
repo: ids.repo,
|
|
79
|
+
// No `(unnamed)` fallback: the test name is the join key every later query
|
|
80
|
+
// groups on, so inventing one would merge unrelated tests into a single
|
|
81
|
+
// history. A nameless row is a validation failure instead.
|
|
82
|
+
test_name: assertion.fullName ?? assertion.title ?? '',
|
|
83
|
+
test_file: file.name ?? '',
|
|
84
|
+
status: toCanaryStatus(assertion.status),
|
|
85
|
+
duration_ms: Math.round(assertion.duration ?? 0),
|
|
86
|
+
...(first === undefined
|
|
87
|
+
? {}
|
|
88
|
+
: { error_text: String(first).slice(0, 2000) }),
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
/** Convert a parsed vitest JSON report into a run + its per-test rows. */
|
|
92
|
+
export function buildRunFromVitestReport(parsed, ctx) {
|
|
93
|
+
const report = parsed;
|
|
94
|
+
const startedMs = typeof report.startTime === 'number' ? report.startTime : ctx.nowMs;
|
|
95
|
+
const epochSeconds = Math.floor(startedMs / 1000);
|
|
96
|
+
const runId = ctx.runId ?? makeRunId(ctx.suite, ctx.commitSha, epochSeconds);
|
|
97
|
+
const results = [];
|
|
98
|
+
for (const file of report.testResults ?? []) {
|
|
99
|
+
for (const assertion of file.assertionResults ?? []) {
|
|
100
|
+
results.push(toResultRow(assertion, file, {
|
|
101
|
+
runId,
|
|
102
|
+
suite: ctx.suite,
|
|
103
|
+
repo: ctx.repo,
|
|
104
|
+
}));
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
const count = (status) => results.filter((r) => r.status === status).length;
|
|
108
|
+
const run = {
|
|
109
|
+
run_id: runId,
|
|
110
|
+
suite: ctx.suite,
|
|
111
|
+
repo: ctx.repo,
|
|
112
|
+
branch: ctx.branch,
|
|
113
|
+
commit_sha: ctx.commitSha,
|
|
114
|
+
// The store's own timestamps use `+00:00` rather than `Z` (Python
|
|
115
|
+
// `datetime.isoformat()`), and `queryTimeline` sorts these as strings.
|
|
116
|
+
timestamp: new Date(startedMs).toISOString().replace('Z', '+00:00'),
|
|
117
|
+
total: results.length,
|
|
118
|
+
passed: count('passed'),
|
|
119
|
+
failed: count('failed'),
|
|
120
|
+
// See the module docstring: vitest has no flaky status to read.
|
|
121
|
+
flaky: 0,
|
|
122
|
+
skipped: count('skipped'),
|
|
123
|
+
duration_ms: results.reduce((n, r) => n + (r.duration_ms ?? 0), 0),
|
|
124
|
+
};
|
|
125
|
+
const built = { run, results };
|
|
126
|
+
validateBuiltRun(built);
|
|
127
|
+
return built;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Reject a run that would poison later reads, using the store's own serializer.
|
|
131
|
+
*
|
|
132
|
+
* Checked against `serializeLocalRecord` rather than against the input objects,
|
|
133
|
+
* because that is the exact shape the file receives -- a field lost or nulled
|
|
134
|
+
* during serialization is caught here rather than by whoever queries it next
|
|
135
|
+
* month.
|
|
136
|
+
*/
|
|
137
|
+
export function validateBuiltRun(built) {
|
|
138
|
+
const record = serializeLocalRecord(built.run, built.results);
|
|
139
|
+
for (const key of ['run_id', 'suite', 'repo', 'branch', 'commit_sha']) {
|
|
140
|
+
if (!record[key]) {
|
|
141
|
+
throw new RecordValidationError(`run field '${key}' is empty`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
const allowed = new Set(RECORD_STATUSES);
|
|
145
|
+
const rows = record['tests'];
|
|
146
|
+
for (const [index, row] of rows.entries()) {
|
|
147
|
+
if (!row['test_name']) {
|
|
148
|
+
throw new RecordValidationError(`result ${index} has no test name; the store has no key to join it on`);
|
|
149
|
+
}
|
|
150
|
+
if (!allowed.has(String(row['status']))) {
|
|
151
|
+
throw new RecordValidationError(`result ${index} ('${String(row['test_name'])}') has status ` +
|
|
152
|
+
`'${String(row['status'])}', which is outside ` +
|
|
153
|
+
`${RECORD_STATUSES.join('|')}`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
const { total, passed, failed, flaky, skipped } = built.run;
|
|
157
|
+
if (total !== rows.length) {
|
|
158
|
+
throw new RecordValidationError(`run total ${total} disagrees with ${rows.length} recorded result(s)`);
|
|
159
|
+
}
|
|
160
|
+
if (passed + failed + flaky + skipped !== total) {
|
|
161
|
+
throw new RecordValidationError(`status counts (${passed}/${failed}/${flaky}/${skipped}) do not sum to ` +
|
|
162
|
+
`the total of ${total}`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
//# sourceMappingURL=run-recorder.js.map
|
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Write-side schema for run-history records.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* shapes (loose, as parsed from disk); this module holds the *write*
|
|
6
|
-
* (the dataclass field sets) and the serializers that
|
|
7
|
-
*
|
|
8
|
-
*
|
|
4
|
+
* Originally a faithful TS port of `agent/history/schema.py`. `record.ts` holds
|
|
5
|
+
* the *read* shapes (loose, as parsed from disk); this module holds the *write*
|
|
6
|
+
* inputs (the dataclass field sets) and the serializers that produce the on-disk
|
|
7
|
+
* and remote-table shapes.
|
|
8
|
+
*
|
|
9
|
+
* The field sets still match what Python's `asdict()` produced, with one
|
|
10
|
+
* deliberate addition since the Python engine was retired at v6.0.0: the local
|
|
11
|
+
* record carries `schema_version` (#701). See `serializeLocalRecord`.
|
|
9
12
|
*/
|
|
10
13
|
import { def } from '../util/coalesce.js';
|
|
14
|
+
import { SCHEMA_VERSION } from './record.js';
|
|
11
15
|
/** `{suite}-{commit[:8]}-{epoch}` — identical to Python `make_run_id`. */
|
|
12
16
|
export function makeRunId(suite, commitSha, timestampEpoch) {
|
|
13
17
|
return `${suite}-${commitSha.slice(0, 8)}-${timestampEpoch}`;
|
|
@@ -51,9 +55,23 @@ export function serializeTestResult(t) {
|
|
|
51
55
|
}
|
|
52
56
|
/**
|
|
53
57
|
* The nested NDJSON line shape written by the local store: a serialized run
|
|
54
|
-
* with its `tests` embedded
|
|
58
|
+
* with its `tests` embedded.
|
|
59
|
+
*
|
|
60
|
+
* `schema_version` is stamped **here**, in the one serializer every local
|
|
61
|
+
* writer goes through, rather than at each call site (#701). A writer that has
|
|
62
|
+
* to remember the field is a writer that eventually forgets it — and an
|
|
63
|
+
* unstamped row is invisible to the reader's version guard, so the guard could
|
|
64
|
+
* never fire on the store's own history. Stamping by construction is what makes
|
|
65
|
+
* "every row is self-describing" a property instead of a convention.
|
|
66
|
+
*
|
|
67
|
+
* Deliberately absent from `serializeRun`/`serializeTestResult`: those map to
|
|
68
|
+
* the remote store's table columns, which have no such field.
|
|
55
69
|
*/
|
|
56
70
|
export function serializeLocalRecord(run, results) {
|
|
57
|
-
return {
|
|
71
|
+
return {
|
|
72
|
+
schema_version: SCHEMA_VERSION,
|
|
73
|
+
...serializeRun(run),
|
|
74
|
+
tests: results.map(serializeTestResult),
|
|
75
|
+
};
|
|
58
76
|
}
|
|
59
77
|
//# sourceMappingURL=schema.js.map
|
|
@@ -33,6 +33,15 @@ export class LocalAsyncAdapter {
|
|
|
33
33
|
async countRuns() {
|
|
34
34
|
return this.inner.countRuns();
|
|
35
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* Forward the local store's raw-record access (#711). This is the only
|
|
38
|
+
* implementation of the optional `readAll` capability, and the analysis
|
|
39
|
+
* engine's spikes / common-failures / regression-candidates sections are
|
|
40
|
+
* reachable through it alone — see `AsyncHistoryStore.readAll`.
|
|
41
|
+
*/
|
|
42
|
+
async readAll() {
|
|
43
|
+
return this.inner.readAll();
|
|
44
|
+
}
|
|
36
45
|
async querySummary(suite, runs) {
|
|
37
46
|
return this.inner.querySummary(suite, runs);
|
|
38
47
|
}
|