canary-test-cli 7.0.0 → 7.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/agents/skills/README.md +308 -0
  2. package/agents/skills/canary:generate.md +49 -0
  3. package/agents/skills/canary:init.md +37 -0
  4. package/agents/skills/canary:migrate.md +66 -0
  5. package/agents/skills/claude-code/canary-add-framework/SKILL.md +248 -0
  6. package/agents/skills/claude-code/canary-blackhawk/SKILL.md +170 -0
  7. package/agents/skills/claude-code/canary-blackhawk/scripts/cli.mjs +188 -0
  8. package/agents/skills/claude-code/canary-blackhawk/scripts/rules.mjs +120 -0
  9. package/agents/skills/claude-code/canary-blackhawk/scripts/scanner.mjs +244 -0
  10. package/agents/skills/claude-code/canary-blackhawk/scripts/string-literals.mjs +116 -0
  11. package/agents/skills/claude-code/canary-cassandra/SKILL.md +180 -0
  12. package/agents/skills/claude-code/canary-cassandra/scripts/cli.mjs +268 -0
  13. package/agents/skills/claude-code/canary-cassandra/scripts/engine.mjs +95 -0
  14. package/agents/skills/claude-code/canary-ci-ready/SKILL.md +161 -0
  15. package/agents/skills/claude-code/canary-ci-ready/skill.yaml +14 -0
  16. package/agents/skills/claude-code/canary-company-knowledge/SKILL.md +196 -0
  17. package/agents/skills/claude-code/canary-critical-areas/SKILL.md +142 -0
  18. package/agents/skills/claude-code/canary-critical-areas/skill.yaml +16 -0
  19. package/agents/skills/claude-code/canary-edge-case-discovery/SKILL.md +160 -0
  20. package/agents/skills/claude-code/canary-edge-case-discovery/skill.yaml +16 -0
  21. package/agents/skills/claude-code/canary-fail-fast/SKILL.md +75 -0
  22. package/agents/skills/claude-code/canary-fail-fast/scripts/cli.mjs +118 -0
  23. package/agents/skills/claude-code/canary-fail-fast/scripts/digest.mjs +69 -0
  24. package/agents/skills/claude-code/canary-fail-fast/scripts/failures.mjs +60 -0
  25. package/agents/skills/claude-code/canary-fail-fast/scripts/fastfail_check.mjs +43 -0
  26. package/agents/skills/claude-code/canary-fail-fast/scripts/parse.mjs +149 -0
  27. package/agents/skills/claude-code/canary-failure-impact/SKILL.md +153 -0
  28. package/agents/skills/claude-code/canary-failure-impact/skill.yaml +15 -0
  29. package/agents/skills/claude-code/canary-fleet-health/SKILL.md +196 -0
  30. package/agents/skills/claude-code/canary-generate-test/SKILL.md +185 -0
  31. package/agents/skills/claude-code/canary-instrument/SKILL.md +157 -0
  32. package/agents/skills/claude-code/canary-instrument/scripts/cli.mjs +178 -0
  33. package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/instrument.mjs +96 -0
  34. package/agents/skills/claude-code/canary-instrument/scripts/otel_bootstrap/playwright-fixture.ts +44 -0
  35. package/agents/skills/claude-code/canary-instrument/scripts/run_types.mjs +81 -0
  36. package/agents/skills/claude-code/canary-instrument/scripts/span_reader.mjs +187 -0
  37. package/agents/skills/claude-code/canary-katana/SKILL.md +243 -0
  38. package/agents/skills/claude-code/canary-katana/scripts/alarm.mjs +296 -0
  39. package/agents/skills/claude-code/canary-katana/scripts/cli.mjs +247 -0
  40. package/agents/skills/claude-code/canary-katana/scripts/diffscan.mjs +0 -0
  41. package/agents/skills/claude-code/canary-katana/scripts/ledger.mjs +183 -0
  42. package/agents/skills/claude-code/canary-pr-guardian/SKILL.md +129 -0
  43. package/agents/skills/claude-code/canary-pr-guardian/skill.yaml +17 -0
  44. package/agents/skills/claude-code/canary-promote-test/SKILL.md +228 -0
  45. package/agents/skills/claude-code/canary-savant/SKILL.md +233 -0
  46. package/agents/skills/claude-code/canary-savant/scripts/cli.mjs +274 -0
  47. package/agents/skills/claude-code/canary-savant/scripts/restoration.mjs +274 -0
  48. package/agents/skills/claude-code/canary-savant/scripts/rules.mjs +168 -0
  49. package/agents/skills/claude-code/canary-savant/scripts/runner.mjs +572 -0
  50. package/agents/skills/claude-code/canary-savant/scripts/scanner.mjs +374 -0
  51. package/agents/skills/claude-code/canary-savant/scripts/string-literals.mjs +116 -0
  52. package/agents/skills/claude-code/canary-setup-harness/SKILL.md +263 -0
  53. package/agents/skills/claude-code/canary-shadow/SKILL.md +131 -0
  54. package/agents/skills/claude-code/canary-shadow/scripts/cases.example.json +32 -0
  55. package/agents/skills/claude-code/canary-shadow/scripts/cli.mjs +195 -0
  56. package/agents/skills/claude-code/canary-ship/SKILL.md +177 -0
  57. package/agents/skills/claude-code/canary-ship/skill.yaml +16 -0
  58. package/agents/skills/claude-code/canary-strix/SKILL.md +130 -0
  59. package/agents/skills/claude-code/canary-strix/scripts/cli.mjs +255 -0
  60. package/agents/skills/claude-code/canary-strix/scripts/scanner.mjs +252 -0
  61. package/agents/skills/claude-code/canary-strix/scripts/terms.mjs +132 -0
  62. package/agents/skills/claude-code/canary-test-pipeline/SKILL.md +159 -0
  63. package/agents/skills/claude-code/canary-test-pipeline/skill.yaml +19 -0
  64. package/agents/skills/claude-code/canary-test-reporter/SKILL.md +138 -0
  65. package/agents/skills/claude-code/canary-test-reporter/scripts/cli.mjs +98 -0
  66. package/agents/skills/claude-code/canary-test-reporter/scripts/json_report.mjs +58 -0
  67. package/agents/skills/claude-code/canary-test-reporter/scripts/parse.mjs +216 -0
  68. package/agents/skills/claude-code/canary-test-reporter/scripts/render.mjs +114 -0
  69. package/agents/skills/lib/parse-args.mjs +214 -0
  70. package/dist/engine/analysis/cli.js +116 -54
  71. package/dist/engine/analysis/engine.js +34 -16
  72. package/dist/engine/analysis/reports.js +5 -4
  73. package/dist/engine/cli-commands.js +252 -106
  74. package/dist/engine/cli-common.js +15 -24
  75. package/dist/engine/cli.core.js +37 -11
  76. package/dist/engine/cli.js +2 -2
  77. package/dist/engine/company-knowledge-cli.js +2 -2
  78. package/dist/engine/core/adoption.js +408 -0
  79. package/dist/engine/core/framework-probes.js +7 -7
  80. package/dist/engine/core/fs-glob.js +2 -2
  81. package/dist/engine/core/gate-result.js +17 -0
  82. package/dist/engine/core/migrator.js +9 -17
  83. package/dist/engine/core/pattern-matcher.js +23 -5
  84. package/dist/engine/core/persona.js +421 -0
  85. package/dist/engine/core/promotion-verdict.js +261 -0
  86. package/dist/engine/core/quality-scorer.js +15 -2
  87. package/dist/engine/core/reporter.js +1 -9
  88. package/dist/engine/core/skill-dispatch.js +115 -0
  89. package/dist/engine/core/skill-examples.js +392 -0
  90. package/dist/engine/core/skill-registry.js +59 -4
  91. package/dist/engine/core/skill-surfaces.js +307 -0
  92. package/dist/engine/core/static-linter.js +310 -38
  93. package/dist/engine/core/test-files.js +77 -0
  94. package/dist/engine/core/ticket-updater.js +1 -7
  95. package/dist/engine/core/vacuity-scanner.js +726 -0
  96. package/dist/engine/core/workflow-discovery.js +2 -8
  97. package/dist/engine/core/workspace-detect.js +7 -6
  98. package/dist/engine/data/personas/registry.json +36 -0
  99. package/dist/engine/guardian/adjudication.js +5 -5
  100. package/dist/engine/guardian/analysis-emit.js +19 -28
  101. package/dist/engine/guardian/cli.js +150 -51
  102. package/dist/engine/guardian/coverage.js +1 -1
  103. package/dist/engine/guardian/diff-coverage/heuristic-tier.js +1 -1
  104. package/dist/engine/guardian/diff-coverage/orchestrator.js +2 -2
  105. package/dist/engine/guardian/pr-check.js +322 -33
  106. package/dist/engine/guardian/pr-comment.js +4 -3
  107. package/dist/engine/history/cli.js +210 -6
  108. package/dist/engine/history/ndjson-store.js +9 -5
  109. package/dist/engine/history/record.js +34 -5
  110. package/dist/engine/history/run-recorder.js +165 -0
  111. package/dist/engine/history/schema.js +25 -7
  112. package/dist/engine/history/store.js +9 -0
  113. package/dist/engine/mcp-server.js +35 -13
  114. package/dist/engine/skills-cli.js +249 -23
  115. package/dist/engine/util/ensure-ascii.js +37 -0
  116. package/dist/engine/workflow-cli.js +6 -6
  117. package/dist/gate-result.d.ts +11 -0
  118. package/dist/gate-result.js +18 -0
  119. package/dist/reporters/testtracker.d.ts +1 -1
  120. package/dist/reporters/testtracker.js +1 -1
  121. package/dist/uninstall.js +12 -5
  122. package/package.json +3 -2
@@ -5,7 +5,7 @@
5
5
  *
6
6
  * Follows the guardian CLI conventions (see `../cli-common.ts`): a
7
7
  * {@link createHistoryCommand} factory wired to an injectable {@link HistoryDeps}
8
- * (out/err sinks, env, a store factory), `CliExit` for business exits, and
8
+ * (out/err sinks, env, a store factory), `CliExitError` for business exits, and
9
9
  * `normalizeUsageExit` on every command so usage errors exit 2.
10
10
  *
11
11
  * Python->TS fidelity notes:
@@ -20,14 +20,22 @@
20
20
  * no Python CLI test pinning them, so this port emits a simple aligned text
21
21
  * table carrying the SAME cell content. `summary`/`push`/`migrate` are NOT
22
22
  * tables and are reproduced byte-for-byte via picocolors stripping.
23
+ *
24
+ * `record` is NOT a port. It has no Python counterpart: nothing in either engine
25
+ * ever wrote the local store, which is why the whole `analyze` / `history`
26
+ * surface had only ever been exercised against synthetic fixtures (#538). Its
27
+ * conversion half lives in `run-recorder.ts`; it takes the async store contract
28
+ * per ADR 0013, so it writes local NDJSON or the remote store by configuration.
23
29
  */
24
30
  import { existsSync, readFileSync } from 'node:fs';
25
31
  import { Command, Option } from 'commander';
26
32
  import pc from 'picocolors';
27
- import { CliExit, jsonIndent2, normalizeUsageExit } from '../cli-common.js';
33
+ import { CliExitError, jsonIndent2, normalizeUsageExit, } from '../cli-common.js';
28
34
  import { gateOutcome } from '../core/gate-result.js';
29
35
  import { makeRunId } from './schema.js';
30
36
  import { makeStore as realMakeStore } from './store.js';
37
+ import { buildRunFromVitestReport, countReportResults, detectReportShape, RecordValidationError, } from './run-recorder.js';
38
+ import { def } from '../util/coalesce.js';
31
39
  import { pyFloat } from '../util/round.js';
32
40
  const EM_DASH = '\u{2014}';
33
41
  const GEQ = '\u{2265}';
@@ -126,7 +134,7 @@ function renderTable(title, headers, rows, rightAlign) {
126
134
  async function pushCmd(historyFile, opts, deps) {
127
135
  if (!existsSync(historyFile)) {
128
136
  deps.out(`${pc.red('Not found:')} ${historyFile}`);
129
- throw new CliExit(1);
137
+ throw new CliExitError(1);
130
138
  }
131
139
  const store = deps.makeStore(opts.dbUrl, historyFile);
132
140
  const records = [];
@@ -137,7 +145,7 @@ async function pushCmd(historyFile, opts, deps) {
137
145
  }
138
146
  if (records.length === 0) {
139
147
  deps.out(pc.yellow('No runs found in history file.'));
140
- throw new CliExit(0);
148
+ throw new CliExitError(0);
141
149
  }
142
150
  const latest = { ...records[records.length - 1] };
143
151
  const testsRaw = latest['tests'] ?? [];
@@ -146,11 +154,188 @@ async function pushCmd(historyFile, opts, deps) {
146
154
  const results = testsRaw.map((t) => pick(t, RESULT_FIELDS));
147
155
  if (opts.dryRun) {
148
156
  deps.out(`${pc.cyan('dry-run:')} would push run ${pc.bold(run.run_id)} (${results.length} tests)`);
149
- throw new CliExit(0);
157
+ throw new CliExitError(0);
150
158
  }
151
159
  await store.pushRun(run, results);
152
160
  deps.out(`${pc.green('Pushed')} run ${pc.bold(run.run_id)} (${results.length} tests)`);
153
161
  }
162
+ /** The `flaky: 0` caveat, printed rather than buried in a source comment. */
163
+ const FLAKY_VOCABULARY_NOTE = 'note: flaky=0 \u{2014} vitest reports no flaky status, so a test that was ' +
164
+ 'retried and then passed is recorded as passed.';
165
+ /** Read + parse the results file, or exit 1 having said which and why. */
166
+ function readReport(resultsFile, deps) {
167
+ if (!existsSync(resultsFile)) {
168
+ deps.out(`${pc.red('Not found:')} ${resultsFile}`);
169
+ throw new CliExitError(1);
170
+ }
171
+ try {
172
+ return JSON.parse(readFileSync(resultsFile, 'utf-8'));
173
+ }
174
+ catch (err) {
175
+ // Loud, not silent: an unreadable report means this run recorded NOTHING,
176
+ // and a later `analyze` would abstain without ever saying why.
177
+ deps.out(`${pc.red('Could not be read:')} ${resultsFile} ` +
178
+ `(${err.message}) \u{2014} nothing was recorded.`);
179
+ throw new CliExitError(1);
180
+ }
181
+ }
182
+ /** The repo slug, or exit 2 -- never a hardcoded default (see #538). */
183
+ function resolveRepo(opts, deps) {
184
+ const repo = def(opts.repo, deps.env['GITHUB_REPOSITORY']);
185
+ if (repo)
186
+ return repo;
187
+ // A hardcoded default repo slug is how a tool ends up filing one team's runs
188
+ // under another team's name. Usage error (exit 2), not a guess.
189
+ deps.out(`${pc.red('Missing --repo:')} the repo slug could not be inferred ` +
190
+ `(no GITHUB_REPOSITORY in the environment). Pass ` +
191
+ `--repo <owner>/<name>.`);
192
+ throw new CliExitError(2);
193
+ }
194
+ /**
195
+ * Resolve the run-level facts a runner's report cannot carry.
196
+ *
197
+ * Fallbacks go through `def()` rather than `??` chains: as a call it is not a
198
+ * decision point, which keeps this mapper under the arch gate's per-function
199
+ * complexity threshold (the same reason the store's row mappers use it).
200
+ */
201
+ function recordContext(opts, deps) {
202
+ const runId = opts.runId;
203
+ return {
204
+ suite: opts.suite,
205
+ repo: resolveRepo(opts, deps),
206
+ branch: def(opts.branch, def(deps.env['GITHUB_REF_NAME'], 'local')),
207
+ commitSha: def(opts.commit, def(deps.env['GITHUB_SHA'], 'local')),
208
+ // `exactOptionalPropertyTypes`: an absent --run-id omits the key rather
209
+ // than setting it to undefined.
210
+ ...(runId === undefined ? {} : { runId }),
211
+ nowMs: Date.now(),
212
+ };
213
+ }
214
+ /**
215
+ * `record` is a GATE (ADR 0009): a zero-result report exits 3, because the
216
+ * caller is a CI step whose next command reads the store, and "the suite
217
+ * reported nothing" must not look like "the suite passed". #538.
218
+ */
219
+ async function recordCmd(resultsFile, opts, deps) {
220
+ const parsed = readReport(resultsFile, deps);
221
+ if (detectReportShape(parsed) !== 'vitest') {
222
+ deps.out(`${pc.red('Unrecognized report:')} ${resultsFile} carries no ` +
223
+ `\`testResults\` array, so it is not a vitest --reporter=json report. ` +
224
+ `\`record\` reads vitest JSON today; Playwright JSON and JUnit XML ` +
225
+ `are not supported yet.`);
226
+ throw new CliExitError(1);
227
+ }
228
+ // The denominator first: a report with no results is an abstention whatever
229
+ // else is missing, and refusing it over an unresolvable --repo would report
230
+ // the wrong finding.
231
+ if (countReportResults(parsed) === 0) {
232
+ abstainOnEmptyReport(resultsFile, opts, deps);
233
+ return;
234
+ }
235
+ const ctx = recordContext(opts, deps);
236
+ let built;
237
+ try {
238
+ built = buildRunFromVitestReport(parsed, ctx);
239
+ }
240
+ catch (err) {
241
+ if (!(err instanceof RecordValidationError))
242
+ throw err;
243
+ deps.out(`${pc.red('Invalid results:')} ${err.message}. Nothing was recorded ` +
244
+ `\u{2014} a malformed record degrades every later read of the store.`);
245
+ throw new CliExitError(1);
246
+ }
247
+ const remote = opts.dbUrl ?? deps.env['CANARY_HISTORY_DB_URL'];
248
+ const storePath = opts.path ?? DEFAULT_HISTORY_FILE;
249
+ const target = remote ? 'the configured remote store' : storePath;
250
+ if (opts.dryRun) {
251
+ reportDryRun(built, target, opts, deps);
252
+ return;
253
+ }
254
+ if (remote && opts.path) {
255
+ deps.err(`note: --path is ignored while a db-url is configured; the run goes to ` +
256
+ `the remote store.`);
257
+ }
258
+ const store = deps.makeStore(opts.dbUrl, storePath);
259
+ const before = store.countRuns ? await store.countRuns() : null;
260
+ await store.pushRun(built.run, built.results);
261
+ const after = store.countRuns ? await store.countRuns() : null;
262
+ // `pushRun` skips a duplicate run_id SILENTLY (it is idempotent by design).
263
+ // For a CLI that is the wrong default: the caller believes it recorded a run.
264
+ if (before !== null && after === before) {
265
+ deps.out(`${pc.yellow('Already recorded:')} run_id ` +
266
+ `${pc.bold(built.run.run_id)} is already present in ${target}; ` +
267
+ `nothing was appended. Two runs of one suite at one commit inside the ` +
268
+ `same second collide \u{2014} pass --run-id to record a distinct run.`);
269
+ throw new CliExitError(1);
270
+ }
271
+ if (before === null) {
272
+ // Cannot verify is a finding, not a silence (#508).
273
+ deps.err(`note: this backend cannot report how many runs it holds, so a ` +
274
+ `duplicate run_id could not be verified \u{2014} the store skips ` +
275
+ `duplicates silently.`);
276
+ }
277
+ reportRecorded(built, target, opts, deps);
278
+ }
279
+ function abstainOnEmptyReport(resultsFile, opts, deps) {
280
+ const outcome = gateOutcome({ checked: 0, findings: [] }, 'gate');
281
+ const notice = `${outcome.summaryLine} ${resultsFile} carried zero test results, so ` +
282
+ `nothing was recorded \u{2014} an empty run is the denominator ` +
283
+ `collapsing, not a passing suite. Check that the runner wrote its report ` +
284
+ `(\`vitest --reporter=json --outputFile=<path>\`) and that the suite ran.`;
285
+ if (opts.json) {
286
+ deps.out(jsonIndent2({
287
+ suite: opts.suite,
288
+ checked: 0,
289
+ recorded: false,
290
+ abstained: true,
291
+ }));
292
+ deps.err(notice);
293
+ }
294
+ else {
295
+ deps.out(notice);
296
+ }
297
+ throw new CliExitError(outcome.exitCode);
298
+ }
299
+ /** The success payload/line. Shared shape so `--json` cannot drift from it. */
300
+ function recordPayload(built, target, extra) {
301
+ const { run } = built;
302
+ return {
303
+ run_id: run.run_id,
304
+ suite: run.suite,
305
+ repo: run.repo,
306
+ target,
307
+ checked: built.results.length,
308
+ passed: run.passed,
309
+ failed: run.failed,
310
+ flaky: run.flaky,
311
+ skipped: run.skipped,
312
+ abstained: false,
313
+ ...extra,
314
+ };
315
+ }
316
+ function countsLine(built) {
317
+ const { run } = built;
318
+ return (`${built.results.length} result(s) for ${pc.bold(run.suite)} ` +
319
+ `(${run.passed} passed, ${run.failed} failed, ${run.skipped} skipped)`);
320
+ }
321
+ function reportDryRun(built, target, opts, deps) {
322
+ if (opts.json) {
323
+ deps.out(jsonIndent2(recordPayload(built, target, { recorded: false, dry_run: true })));
324
+ return;
325
+ }
326
+ deps.out(`${pc.cyan('dry-run:')} would record ${countsLine(built)} as ` +
327
+ `${pc.bold(built.run.run_id)} \u{2192} ${target}`);
328
+ deps.out(FLAKY_VOCABULARY_NOTE);
329
+ }
330
+ function reportRecorded(built, target, opts, deps) {
331
+ if (opts.json) {
332
+ deps.out(jsonIndent2(recordPayload(built, target, { recorded: true })));
333
+ return;
334
+ }
335
+ deps.out(`${pc.green('Recorded')} ${countsLine(built)} \u{2192} ${target}`);
336
+ deps.out(`run_id: ${built.run.run_id}`);
337
+ deps.out(FLAKY_VOCABULARY_NOTE);
338
+ }
154
339
  async function flakyCmd(opts, deps) {
155
340
  const store = deps.makeStore(opts.dbUrl);
156
341
  if (await abstainOnEmptyHistory(store, deps, opts.json === true, 'flake rate')) {
@@ -241,7 +426,7 @@ async function summaryCmd(suite, opts, deps) {
241
426
  async function migrateCmd(file, opts, deps) {
242
427
  if (!existsSync(file)) {
243
428
  deps.out(`${pc.red('Not found:')} ${file}`);
244
- throw new CliExit(1);
429
+ throw new CliExitError(1);
245
430
  }
246
431
  const store = deps.makeStore(opts.dbUrl);
247
432
  let migrated = 0;
@@ -313,6 +498,25 @@ export function createHistoryCommand(depsInit = {}) {
313
498
  .action(async (historyFile, opts) => {
314
499
  await pushCmd(historyFile, opts, deps);
315
500
  });
501
+ program
502
+ .command('record')
503
+ .description('Record a finished test run into the history store (vitest JSON).')
504
+ .argument('<results_file>', "Path to the runner's JSON report.")
505
+ .requiredOption('--suite <suite>', 'Suite name for this run (e.g. e2e).')
506
+ .option('--repo <repo>', 'GitHub repo slug (default: $GITHUB_REPOSITORY).')
507
+ .option('--branch <branch>', 'Branch name (default: $GITHUB_REF_NAME).')
508
+ .option('--commit <sha>', 'Commit SHA (default: $GITHUB_SHA).')
509
+ // No commander default: an explicitly-passed --path has to stay
510
+ // distinguishable from the fallback, so a db-url + --path combination can
511
+ // say that --path is unused instead of silently dropping it.
512
+ .option('--path <store>', `Local NDJSON store to append to (default: ${DEFAULT_HISTORY_FILE}).`)
513
+ .option('--run-id <id>', 'Run id (default: <suite>-<commit[:8]>-<epoch seconds>).')
514
+ .addOption(new Option('--db-url <url>').env('CANARY_HISTORY_DB_URL'))
515
+ .option('--dry-run', 'Show what would be recorded without writing.')
516
+ .option('--json')
517
+ .action(async (resultsFile, opts) => {
518
+ await recordCmd(resultsFile, opts, deps);
519
+ });
316
520
  program
317
521
  .command('flaky')
318
522
  .description('Show tests ranked by flake rate over the rolling window.')
@@ -7,14 +7,18 @@
7
7
  *
8
8
  * Deviation from the Python reader (deliberate): where Python's `_read_all`
9
9
  * swallows any parse error and returns `[]`, this reader throws on malformed
10
- * JSON and on an explicit unrecognized `schema_version`, so a corrupt or
11
- * future-version history fails loudly rather than silently analysing nothing.
10
+ * JSON and on an unrecognized `schema_version`, so a corrupt or future-version
11
+ * history fails loudly rather than silently analysing nothing.
12
+ *
13
+ * Version resolution lives in `resolveSchemaVersion` (#701): rows written by
14
+ * this store carry their version, and a legacy unstamped row is read as the
15
+ * version it was written at rather than as "current".
12
16
  */
13
17
  import { appendFileSync, mkdirSync, readFileSync } from 'node:fs';
14
18
  import { dirname } from 'node:path';
15
19
  import { def } from '../util/coalesce.js';
16
20
  import { round1 } from '../util/round.js';
17
- import { SCHEMA_VERSION } from './record.js';
21
+ import { SCHEMA_VERSION, resolveSchemaVersion } from './record.js';
18
22
  import { serializeLocalRecord } from './schema.js';
19
23
  export class NdjsonHistoryStore {
20
24
  path;
@@ -38,8 +42,8 @@ export class NdjsonHistoryStore {
38
42
  if (!line)
39
43
  continue;
40
44
  const record = JSON.parse(line);
41
- const version = record.schema_version;
42
- if (version !== undefined && version !== SCHEMA_VERSION) {
45
+ const version = resolveSchemaVersion(record);
46
+ if (version !== SCHEMA_VERSION) {
43
47
  throw new Error(`Unsupported history schema_version ${version} (expected ${SCHEMA_VERSION})`);
44
48
  }
45
49
  records.push(record);
@@ -4,11 +4,40 @@
4
4
  * Mirrors `agent/history/schema.py` (RunRecord + TestResult) as persisted, one
5
5
  * JSON object per line, in `test-results/reports/history-v2.jsonl`.
6
6
  *
7
- * The on-disk records written by the Python engine do NOT carry a per-record
8
- * version field (the "v2" lives in the filename). The reader therefore treats a
9
- * missing `schema_version` as the current version, but throws on an explicit
10
- * unrecognized one — a forward-compat guard, exercised by the store tests.
7
+ * Every row the store writes now carries its own `schema_version` (#701) the
8
+ * version guard can only refuse data it can identify, and before the stamp the
9
+ * store's own history was the one thing it could never check.
10
+ *
11
+ * A row with no version is a **legacy row**, written before that stamp landed.
12
+ * It is resolved to `LEGACY_UNVERSIONED_SCHEMA_VERSION`, not to "current" — see
13
+ * `resolveSchemaVersion` for why the difference is load-bearing.
11
14
  */
12
- /** The schema version this reader understands. */
15
+ /** The schema version this reader understands, and every writer stamps. */
13
16
  export const SCHEMA_VERSION = 2;
17
+ /**
18
+ * The version an unstamped row was actually written at.
19
+ *
20
+ * **A frozen literal, never an alias of `SCHEMA_VERSION`.** Unversioned rows
21
+ * can only be v2: v2 is the sole version this store has ever had (the earlier
22
+ * "v1" lived in a differently-named file), and from #701 onward every row is
23
+ * stamped. So the value is a historical fact, not a default.
24
+ *
25
+ * Aliasing it to `SCHEMA_VERSION` would recreate the #701 bug on the day the
26
+ * version bumps: legacy v2 rows would be read as v3 and silently
27
+ * misinterpreted, which is precisely the guard's purpose to prevent.
28
+ */
29
+ export const LEGACY_UNVERSIONED_SCHEMA_VERSION = 2;
30
+ /**
31
+ * The version a record should be read as.
32
+ *
33
+ * Deliberately conservative: an unversioned row resolves to the version it was
34
+ * written at, so the next `SCHEMA_VERSION` bump makes the reader **refuse**
35
+ * legacy rows loudly rather than reinterpret them under new semantics. That
36
+ * bump therefore has to ship a migration (rewrite the file with a stamp, or add
37
+ * an explicit upgrade path) — which is the point. Today the two constants are
38
+ * equal, so this changes nothing observable; it only decides what happens next.
39
+ */
40
+ export function resolveSchemaVersion(record) {
41
+ return record.schema_version ?? LEGACY_UNVERSIONED_SCHEMA_VERSION;
42
+ }
14
43
  //# sourceMappingURL=record.js.map
@@ -0,0 +1,165 @@
1
+ /**
2
+ * Turn a test runner's own report into a v2 history run (#538).
3
+ *
4
+ * `canary analyze` and `canary history` have always READ
5
+ * `test-results/reports/history-v2.jsonl`; until this module nothing in the
6
+ * product wrote it, so every consumer who wanted a flake report had to author
7
+ * the NDJSON by hand. This is the conversion half of `canary history record`;
8
+ * the CLI half lives in `cli.ts` and the append itself belongs to the store
9
+ * (`AsyncHistoryStore.pushRun`, per ADR 0013).
10
+ *
11
+ * Two deliberate limits, both of them data-honesty calls rather than laziness:
12
+ *
13
+ * - **vitest `--reporter=json` only.** The shape is detected rather than
14
+ * declared by flag, and an unrecognized shape is refused loudly. The
15
+ * alternative -- reading an unknown document and finding zero tests in it --
16
+ * is indistinguishable from a suite that genuinely ran nothing, which is
17
+ * exactly the denominator collapse #508 exists to make visible. Playwright
18
+ * JSON and JUnit XML are parsed elsewhere in the repo (canary-test-reporter,
19
+ * canary-savant) but by self-contained skill CLIs the engine cannot import.
20
+ * - **`flaky` is always 0.** Canary's status vocabulary is
21
+ * passed/failed/flaky/skipped; vitest has no flaky status, so a test that
22
+ * was retried and then passed arrives as `passed` and is invisible here.
23
+ * Recorded as zero and SAID OUT LOUD by the command, because a reader who
24
+ * takes `flaky: 0` for a clean fleet has been misled by the tool.
25
+ */
26
+ import { makeRunId, serializeLocalRecord, } from './schema.js';
27
+ /** Canary's per-test status vocabulary (the store's read side keys on these). */
28
+ const RECORD_STATUSES = ['passed', 'failed', 'flaky', 'skipped'];
29
+ /**
30
+ * A report that cannot be recorded without corrupting the store.
31
+ *
32
+ * Thrown BEFORE any append: `NdjsonHistoryStore.pushRun` writes one line and
33
+ * every later report divides by it, so a malformed record is not a local
34
+ * failure -- it silently degrades every future read of the file.
35
+ */
36
+ export class RecordValidationError extends Error {
37
+ constructor(message) {
38
+ super(message);
39
+ this.name = 'RecordValidationError';
40
+ }
41
+ }
42
+ /** Detect by shape, not by flag -- callers should not have to know the format. */
43
+ export function detectReportShape(parsed) {
44
+ if (parsed === null || typeof parsed !== 'object')
45
+ return 'unknown';
46
+ const testResults = parsed.testResults;
47
+ return Array.isArray(testResults) ? 'vitest' : 'unknown';
48
+ }
49
+ /**
50
+ * How many per-test results the report carries -- the DENOMINATOR, available
51
+ * before any run-level fact is resolved.
52
+ *
53
+ * Separate from the conversion so `record` can abstain on an empty report
54
+ * before it starts complaining about a missing `--repo`: there is nothing to
55
+ * record either way, and the abstention is the finding worth reporting.
56
+ */
57
+ export function countReportResults(parsed) {
58
+ const report = parsed;
59
+ let n = 0;
60
+ for (const file of report.testResults ?? []) {
61
+ n += file.assertionResults?.length ?? 0;
62
+ }
63
+ return n;
64
+ }
65
+ /** vitest status -> canary status. Anything not pass/fail is a skip. */
66
+ function toCanaryStatus(status) {
67
+ if (status === 'passed')
68
+ return 'passed';
69
+ if (status === 'failed')
70
+ return 'failed';
71
+ return 'skipped';
72
+ }
73
+ function toResultRow(assertion, file, ids) {
74
+ const first = assertion.failureMessages?.[0];
75
+ return {
76
+ run_id: ids.runId,
77
+ suite: ids.suite,
78
+ repo: ids.repo,
79
+ // No `(unnamed)` fallback: the test name is the join key every later query
80
+ // groups on, so inventing one would merge unrelated tests into a single
81
+ // history. A nameless row is a validation failure instead.
82
+ test_name: assertion.fullName ?? assertion.title ?? '',
83
+ test_file: file.name ?? '',
84
+ status: toCanaryStatus(assertion.status),
85
+ duration_ms: Math.round(assertion.duration ?? 0),
86
+ ...(first === undefined
87
+ ? {}
88
+ : { error_text: String(first).slice(0, 2000) }),
89
+ };
90
+ }
91
+ /** Convert a parsed vitest JSON report into a run + its per-test rows. */
92
+ export function buildRunFromVitestReport(parsed, ctx) {
93
+ const report = parsed;
94
+ const startedMs = typeof report.startTime === 'number' ? report.startTime : ctx.nowMs;
95
+ const epochSeconds = Math.floor(startedMs / 1000);
96
+ const runId = ctx.runId ?? makeRunId(ctx.suite, ctx.commitSha, epochSeconds);
97
+ const results = [];
98
+ for (const file of report.testResults ?? []) {
99
+ for (const assertion of file.assertionResults ?? []) {
100
+ results.push(toResultRow(assertion, file, {
101
+ runId,
102
+ suite: ctx.suite,
103
+ repo: ctx.repo,
104
+ }));
105
+ }
106
+ }
107
+ const count = (status) => results.filter((r) => r.status === status).length;
108
+ const run = {
109
+ run_id: runId,
110
+ suite: ctx.suite,
111
+ repo: ctx.repo,
112
+ branch: ctx.branch,
113
+ commit_sha: ctx.commitSha,
114
+ // The store's own timestamps use `+00:00` rather than `Z` (Python
115
+ // `datetime.isoformat()`), and `queryTimeline` sorts these as strings.
116
+ timestamp: new Date(startedMs).toISOString().replace('Z', '+00:00'),
117
+ total: results.length,
118
+ passed: count('passed'),
119
+ failed: count('failed'),
120
+ // See the module docstring: vitest has no flaky status to read.
121
+ flaky: 0,
122
+ skipped: count('skipped'),
123
+ duration_ms: results.reduce((n, r) => n + (r.duration_ms ?? 0), 0),
124
+ };
125
+ const built = { run, results };
126
+ validateBuiltRun(built);
127
+ return built;
128
+ }
129
+ /**
130
+ * Reject a run that would poison later reads, using the store's own serializer.
131
+ *
132
+ * Checked against `serializeLocalRecord` rather than against the input objects,
133
+ * because that is the exact shape the file receives -- a field lost or nulled
134
+ * during serialization is caught here rather than by whoever queries it next
135
+ * month.
136
+ */
137
+ export function validateBuiltRun(built) {
138
+ const record = serializeLocalRecord(built.run, built.results);
139
+ for (const key of ['run_id', 'suite', 'repo', 'branch', 'commit_sha']) {
140
+ if (!record[key]) {
141
+ throw new RecordValidationError(`run field '${key}' is empty`);
142
+ }
143
+ }
144
+ const allowed = new Set(RECORD_STATUSES);
145
+ const rows = record['tests'];
146
+ for (const [index, row] of rows.entries()) {
147
+ if (!row['test_name']) {
148
+ throw new RecordValidationError(`result ${index} has no test name; the store has no key to join it on`);
149
+ }
150
+ if (!allowed.has(String(row['status']))) {
151
+ throw new RecordValidationError(`result ${index} ('${String(row['test_name'])}') has status ` +
152
+ `'${String(row['status'])}', which is outside ` +
153
+ `${RECORD_STATUSES.join('|')}`);
154
+ }
155
+ }
156
+ const { total, passed, failed, flaky, skipped } = built.run;
157
+ if (total !== rows.length) {
158
+ throw new RecordValidationError(`run total ${total} disagrees with ${rows.length} recorded result(s)`);
159
+ }
160
+ if (passed + failed + flaky + skipped !== total) {
161
+ throw new RecordValidationError(`status counts (${passed}/${failed}/${flaky}/${skipped}) do not sum to ` +
162
+ `the total of ${total}`);
163
+ }
164
+ }
165
+ //# sourceMappingURL=run-recorder.js.map
@@ -1,13 +1,17 @@
1
1
  /**
2
2
  * Write-side schema for run-history records.
3
3
  *
4
- * Faithful TS port of `agent/history/schema.py`. `record.ts` holds the *read*
5
- * shapes (loose, as parsed from disk); this module holds the *write* inputs
6
- * (the dataclass field sets) and the serializers that turn them into the exact
7
- * dict shapes Python's `asdict()` produces — so a record written by TS is read
8
- * identically by the Python `LocalHistoryStore` (the write-seam proof).
4
+ * Originally a faithful TS port of `agent/history/schema.py`. `record.ts` holds
5
+ * the *read* shapes (loose, as parsed from disk); this module holds the *write*
6
+ * inputs (the dataclass field sets) and the serializers that produce the on-disk
7
+ * and remote-table shapes.
8
+ *
9
+ * The field sets still match what Python's `asdict()` produced, with one
10
+ * deliberate addition since the Python engine was retired at v6.0.0: the local
11
+ * record carries `schema_version` (#701). See `serializeLocalRecord`.
9
12
  */
10
13
  import { def } from '../util/coalesce.js';
14
+ import { SCHEMA_VERSION } from './record.js';
11
15
  /** `{suite}-{commit[:8]}-{epoch}` — identical to Python `make_run_id`. */
12
16
  export function makeRunId(suite, commitSha, timestampEpoch) {
13
17
  return `${suite}-${commitSha.slice(0, 8)}-${timestampEpoch}`;
@@ -51,9 +55,23 @@ export function serializeTestResult(t) {
51
55
  }
52
56
  /**
53
57
  * The nested NDJSON line shape written by the local store: a serialized run
54
- * with its `tests` embedded (matches Python `LocalHistoryStore.push_run`).
58
+ * with its `tests` embedded.
59
+ *
60
+ * `schema_version` is stamped **here**, in the one serializer every local
61
+ * writer goes through, rather than at each call site (#701). A writer that has
62
+ * to remember the field is a writer that eventually forgets it — and an
63
+ * unstamped row is invisible to the reader's version guard, so the guard could
64
+ * never fire on the store's own history. Stamping by construction is what makes
65
+ * "every row is self-describing" a property instead of a convention.
66
+ *
67
+ * Deliberately absent from `serializeRun`/`serializeTestResult`: those map to
68
+ * the remote store's table columns, which have no such field.
55
69
  */
56
70
  export function serializeLocalRecord(run, results) {
57
- return { ...serializeRun(run), tests: results.map(serializeTestResult) };
71
+ return {
72
+ schema_version: SCHEMA_VERSION,
73
+ ...serializeRun(run),
74
+ tests: results.map(serializeTestResult),
75
+ };
58
76
  }
59
77
  //# sourceMappingURL=schema.js.map
@@ -33,6 +33,15 @@ export class LocalAsyncAdapter {
33
33
  async countRuns() {
34
34
  return this.inner.countRuns();
35
35
  }
36
+ /**
37
+ * Forward the local store's raw-record access (#711). This is the only
38
+ * implementation of the optional `readAll` capability, and the analysis
39
+ * engine's spikes / common-failures / regression-candidates sections are
40
+ * reachable through it alone — see `AsyncHistoryStore.readAll`.
41
+ */
42
+ async readAll() {
43
+ return this.inner.readAll();
44
+ }
36
45
  async querySummary(suite, runs) {
37
46
  return this.inner.querySummary(suite, runs);
38
47
  }