@crvouga/atlas 0.15.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/reports/src/attachment-files.d.ts +3 -1
  3. package/dist/reports/src/attachment-files.js +14 -4
  4. package/dist/reports/src/evidence-detail.d.ts +19 -2
  5. package/dist/reports/src/evidence-detail.js +63 -14
  6. package/dist/reports/src/evidence-types.d.ts +53 -0
  7. package/dist/reports/src/layer-merge.js +7 -2
  8. package/dist/reports/src/layer-types.d.ts +1 -1
  9. package/dist/reports/src/publish-checks.js +11 -3
  10. package/dist/reports/src/recording-receipt.d.ts +48 -0
  11. package/dist/reports/src/recording-receipt.js +103 -0
  12. package/dist/reports/src/report-cli.d.ts +1 -1
  13. package/dist/reports/src/report-cli.js +20 -2
  14. package/dist/reports/src/report-status.d.ts +2 -0
  15. package/dist/reports/src/report-status.js +17 -1
  16. package/dist/reports/src/reports-explorer-route.d.ts +8 -0
  17. package/dist/reports/src/reports-route.d.ts +8 -0
  18. package/dist/reports/src/test-catalog.d.ts +24 -3
  19. package/dist/reports/src/test-catalog.js +37 -11
  20. package/dist/schema/src/api.d.ts +232 -0
  21. package/dist/schema/src/api.js +35 -6
  22. package/dist/schema/src/cli-results.d.ts +107 -2
  23. package/dist/schema/src/cli-results.js +11 -3
  24. package/dist/schema/src/layer.d.ts +165 -0
  25. package/dist/schema/src/layer.js +72 -1
  26. package/dist/server/src/app.js +3 -2
  27. package/dist/server/src/llms.js +2 -1
  28. package/dist/server/src/openapi.js +2 -1
  29. package/docs/reference/cli.md +5 -0
  30. package/docs/reference/http-api.md +4 -2
  31. package/docs/reference/reports.md +204 -5
  32. package/docs/reference/schema.md +75 -0
  33. package/docs/rich-evidence.md +29 -0
  34. package/docs/server.md +1 -1
  35. package/llms-full.txt +39 -3
  36. package/package.json +1 -1
  37. package/packages/schema/schemas/check-detail.schema.json +20 -0
  38. package/packages/schema/schemas/effective-delta.schema.json +143 -0
  39. package/packages/schema/schemas/effective-evidence.schema.json +143 -0
  40. package/packages/schema/schemas/evidence-page.schema.json +86 -0
  41. package/packages/schema/schemas/layer-rows.schema.json +57 -0
  42. package/packages/schema/schemas/layer.schema.json +153 -0
  43. package/packages/schema/schemas/publish-checks.schema.json +24 -0
  44. package/packages/schema/schemas/publish-result.schema.json +161 -1
  45. package/packages/schema/schemas/report-run.schema.json +71 -0
  46. package/packages/schema/schemas/report-runs.schema.json +38 -1
  47. package/packages/schema/schemas/report-status.schema.json +104 -0
  48. package/packages/schema/schemas/test-catalog.schema.json +41 -0
  49. package/packages/schema/schemas/test-history.schema.json +82 -0
  50. package/packages/server/openapi.json +830 -4
  51. package/packages/visualizer/src/evidence/marks.tsx +16 -0
  52. package/packages/visualizer/src/explorer/RunsView.tsx +128 -10
  53. package/packages/visualizer/src/explorer/catalog.tsx +16 -6
  54. package/packages/visualizer/src/explorer/search.ts +4 -1
@@ -43,6 +43,7 @@ export const REPORT_USAGE = `atlas report — layered evidence for any kind of s
43
43
  [--param key=value]... [--kind <k>] [--title <t>] [--recorded-at <iso|ms>]
44
44
  [--attach [role=]<file>]... [--label key=value]... [--link [name=]<url>]...
45
45
  [--attach-root <dir>]... [--max-attachment-mb <n>] [--max-total-attachment-mb <n>]
46
+ [--require-recordings]
46
47
  atlas report stream [--store <dir|s3://…|https://atlas.example.com>] --producer <name> [--environment <name>]
47
48
  [--branch <b>] [--sha <s>] [--run-url <u>] [--kind <k>] [--param key=value]...
48
49
  [--run-id <id>] [--run-title <t>]
@@ -79,6 +80,10 @@ export const REPORT_USAGE = `atlas report — layered evidence for any kind of s
79
80
  Files a report names (CTRF attachments, JUnit [[ATTACHMENT|path]] lines) come from the report's own
80
81
  folder and each --attach-root, and never from a hidden file or a link out: a report is data from a
81
82
  test run, not instructions. One that is missing or not allowed is skipped with a warning.
83
+ --require-recordings exit 1 when a check that declared recordingExpectation "required" has no
84
+ uploaded video (absent, left out as too large or not allowed, or refused). The layer is
85
+ still written and the checks keep their results; only this command fails. The receipt of
86
+ what was uploaded (counts, sizes, checksums) is in the JSON result under layer.recordings.
82
87
  From a checkout with uncommitted changes the layer is marked "modified".
83
88
  stream Send checks as they finish: one JSON object per line on standard input (the fields of one
84
89
  check: id, title, status, and any evidence). Each second's worth becomes one layer, so a server
@@ -310,7 +315,7 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
310
315
  const json = wantsJson(rest);
311
316
  const result = {
312
317
  ok: true, store: target.label,
313
- layer: { id: layer.id, producer, environment: layer.environment ?? null, checks: ingested.checks.length, passed: counts.passed, failed: counts.failed, notRun: counts.other, linkedToSpec: linkedCount, ...(freshness ? { freshness } : {}), attachmentFiles: attachedFiles, attachmentBytes: attachedBytes },
318
+ layer: { id: layer.id, producer, environment: layer.environment ?? null, checks: ingested.checks.length, passed: counts.passed, failed: counts.failed, notRun: counts.other, linkedToSpec: linkedCount, ...(freshness ? { freshness } : {}), attachmentFiles: attachedFiles, attachmentBytes: attachedBytes, ...(layer.recordings ? { recordings: layer.recordings } : {}) },
314
319
  warnings: warnings.map(warning => `attachment skipped: ${warning}`)
315
320
  };
316
321
  if (!json)
@@ -318,6 +323,19 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
318
323
  if (!json)
319
324
  for (const warning of warnings)
320
325
  io.out(`! attachment skipped: ${warning}\n`);
326
+ // Recording completeness is reported apart from the checks' results. Only --require-recordings turns a gap into a failing exit.
327
+ const gaps = (layer.recordings?.items ?? []).filter(item => item.state !== 'uploaded' && item.state !== 'referenced' && item.state !== 'not-applicable');
328
+ const strict = rest.includes('--require-recordings');
329
+ if (layer.recordings) {
330
+ result.recordings = { strict, complete: gaps.length === 0, expected: layer.recordings.expected, uploaded: layer.recordings.uploaded, gaps: gaps.map(({ check, state, reason }) => ({ check, state, ...(reason ? { reason } : {}) })) };
331
+ if (!json) {
332
+ io.out(`Recordings: ${layer.recordings.uploaded + layer.recordings.referenced} of ${layer.recordings.expected} expected published${layer.recordings.notApplicable ? `, ${layer.recordings.notApplicable} not applicable` : ''}\n`);
333
+ for (const gap of gaps)
334
+ io.out(`! recording ${gap.state}: ${gap.check}${gap.reason ? ` (${gap.reason})` : ''}\n`);
335
+ }
336
+ if (strict && gaps.length)
337
+ result.ok = false;
338
+ }
321
339
  if (specsDirectory) {
322
340
  const files = await collectSpecFiles(path.resolve(base, specsDirectory));
323
341
  // A spec named with --spec that the directory does not hold (or whose kind is not a spec file) is published too.
@@ -343,7 +361,7 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
343
361
  }
344
362
  if (json)
345
363
  io.out(`${JSON.stringify(result, null, 2)}\n`);
346
- return 0;
364
+ return result.ok ? 0 : 1;
347
365
  }
348
366
  if (command === 'stream') {
349
367
  const target = writeStoreOf(value(rest, 'store'), base, env);
@@ -26,6 +26,8 @@ export type StatusOptions = {
26
26
  * each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
27
27
  * report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
28
28
  * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
29
+ * `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
30
+ * completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
29
31
  *
30
32
  * @example
31
33
  * const status = reportStatus(await combineBackends(backends), { requireEnvironments: ['ios'] });
@@ -12,6 +12,7 @@ import { REPORT_STATUS_SCHEMA_VERSION } from '../../schema/src/layer.js';
12
12
  import { countResults } from './compose.js';
13
13
  import { normalizeEnvironment } from './layer-axes.js';
14
14
  import { isDiagnosticKind } from './layer-merge.js';
15
+ import { recordingCompleteness } from './recording-receipt.js';
15
16
  /** Total string order by code unit. */
16
17
  const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
17
18
  /**
@@ -23,6 +24,8 @@ const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
23
24
  * each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
24
25
  * report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
25
26
  * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
27
+ * `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
28
+ * completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
26
29
  *
27
30
  * @example
28
31
  * const status = reportStatus(await combineBackends(backends), { requireEnvironments: ['ios'] });
@@ -85,7 +88,12 @@ export function reportStatus(report, options = {}) {
85
88
  ...report.combined.specs.map(entry => specOutcome(entry.spec.path, entry.spec.title, entry.spec.kind, entry.results)),
86
89
  ...(report.combined.unassigned.length ? [specOutcome(null, 'No spec', 'unassigned', report.combined.unassigned)] : [])
87
90
  ],
88
- failing
91
+ failing,
92
+ // Apart from `ok`: whether the expected videos were published says nothing about whether the checks passed.
93
+ recordings: recordingCompleteness(outcomes.map(result => ({
94
+ spec: result.row.specPath, check: result.row.id, environment: result.layer.environment, layer: result.layer.id,
95
+ hasReceipt: result.layer.recordings !== undefined, ...(result.row.recording ? { recording: result.row.recording } : {})
96
+ })))
89
97
  };
90
98
  }
91
99
  /** `spec › check`, unless the check's id already starts with its spec. */
@@ -120,6 +128,14 @@ export function formatStatusText(status) {
120
128
  const where = [item.environment, item.branch, item.sha?.slice(0, 7)].filter(Boolean).join(' · ');
121
129
  lines.push(`✗ ${item.status === 'running' ? '[running] ' : ''}${label(item)}${where ? ` (${where})` : ''}${item.detail ? `\n ${item.detail.split('\n')[0]}` : ''}`);
122
130
  }
131
+ const recordings = status.recordings;
132
+ // Quiet for a report that never mentioned recordings; the JSON says `unknown` explicitly.
133
+ if (recordings && !(recordings.state === 'unknown' && recordings.expected === 0 && recordings.notApplicable === 0)) {
134
+ const published = recordings.uploaded + recordings.referenced;
135
+ lines.push(`· recordings: ${recordings.state} — ${published} of ${recordings.expected} expected published${recordings.notApplicable ? `, ${recordings.notApplicable} not applicable` : ''}${recordings.layersWithoutReceipt ? `, ${recordings.layersWithoutReceipt} layer${recordings.layersWithoutReceipt === 1 ? '' : 's'} unknown` : ''}`);
136
+ for (const gap of recordings.gaps)
137
+ lines.push(` ! recording ${gap.state}: ${label({ spec: gap.spec, check: gap.check })}${gap.reason ? ` (${gap.reason})` : ''}`);
138
+ }
123
139
  for (const notice of status.notices)
124
140
  lines.push(`! ${notice}`);
125
141
  for (const reason of status.reasons)
@@ -82,6 +82,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
82
82
  height?: number | undefined;
83
83
  durationMs?: number | undefined;
84
84
  description?: string | undefined;
85
+ omitted?: string | undefined;
85
86
  tags?: string[] | undefined;
86
87
  at?: {
87
88
  [x: string]: import("hono/utils/types").JSONValue;
@@ -126,6 +127,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
126
127
  height?: number | undefined;
127
128
  durationMs?: number | undefined;
128
129
  description?: string | undefined;
130
+ omitted?: string | undefined;
129
131
  tags?: string[] | undefined;
130
132
  at?: {
131
133
  [x: string]: import("hono/utils/types").JSONValue;
@@ -183,6 +185,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
183
185
  height?: number | undefined;
184
186
  durationMs?: number | undefined;
185
187
  description?: string | undefined;
188
+ omitted?: string | undefined;
186
189
  tags?: string[] | undefined;
187
190
  at?: {
188
191
  [x: string]: import("hono/utils/types").JSONValue;
@@ -250,6 +253,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
250
253
  height?: number | undefined;
251
254
  durationMs?: number | undefined;
252
255
  description?: string | undefined;
256
+ omitted?: string | undefined;
253
257
  tags?: string[] | undefined;
254
258
  at?: {
255
259
  [x: string]: import("hono/utils/types").JSONValue;
@@ -356,6 +360,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
356
360
  height?: number | undefined;
357
361
  durationMs?: number | undefined;
358
362
  description?: string | undefined;
363
+ omitted?: string | undefined;
359
364
  tags?: string[] | undefined;
360
365
  at?: {
361
366
  [x: string]: import("hono/utils/types").JSONValue;
@@ -400,6 +405,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
400
405
  height?: number | undefined;
401
406
  durationMs?: number | undefined;
402
407
  description?: string | undefined;
408
+ omitted?: string | undefined;
403
409
  tags?: string[] | undefined;
404
410
  at?: {
405
411
  [x: string]: import("hono/utils/types").JSONValue;
@@ -457,6 +463,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
457
463
  height?: number | undefined;
458
464
  durationMs?: number | undefined;
459
465
  description?: string | undefined;
466
+ omitted?: string | undefined;
460
467
  tags?: string[] | undefined;
461
468
  at?: {
462
469
  [x: string]: import("hono/utils/types").JSONValue;
@@ -524,6 +531,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
524
531
  height?: number | undefined;
525
532
  durationMs?: number | undefined;
526
533
  description?: string | undefined;
534
+ omitted?: string | undefined;
527
535
  tags?: string[] | undefined;
528
536
  at?: {
529
537
  [x: string]: import("hono/utils/types").JSONValue;
@@ -120,6 +120,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
120
120
  height?: number | undefined;
121
121
  durationMs?: number | undefined;
122
122
  description?: string | undefined;
123
+ omitted?: string | undefined;
123
124
  tags?: string[] | undefined;
124
125
  at?: {
125
126
  [x: string]: import("hono/utils/types").JSONValue;
@@ -164,6 +165,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
164
165
  height?: number | undefined;
165
166
  durationMs?: number | undefined;
166
167
  description?: string | undefined;
168
+ omitted?: string | undefined;
167
169
  tags?: string[] | undefined;
168
170
  at?: {
169
171
  [x: string]: import("hono/utils/types").JSONValue;
@@ -221,6 +223,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
221
223
  height?: number | undefined;
222
224
  durationMs?: number | undefined;
223
225
  description?: string | undefined;
226
+ omitted?: string | undefined;
224
227
  tags?: string[] | undefined;
225
228
  at?: {
226
229
  [x: string]: import("hono/utils/types").JSONValue;
@@ -288,6 +291,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
288
291
  height?: number | undefined;
289
292
  durationMs?: number | undefined;
290
293
  description?: string | undefined;
294
+ omitted?: string | undefined;
291
295
  tags?: string[] | undefined;
292
296
  at?: {
293
297
  [x: string]: import("hono/utils/types").JSONValue;
@@ -394,6 +398,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
394
398
  height?: number | undefined;
395
399
  durationMs?: number | undefined;
396
400
  description?: string | undefined;
401
+ omitted?: string | undefined;
397
402
  tags?: string[] | undefined;
398
403
  at?: {
399
404
  [x: string]: import("hono/utils/types").JSONValue;
@@ -438,6 +443,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
438
443
  height?: number | undefined;
439
444
  durationMs?: number | undefined;
440
445
  description?: string | undefined;
446
+ omitted?: string | undefined;
441
447
  tags?: string[] | undefined;
442
448
  at?: {
443
449
  [x: string]: import("hono/utils/types").JSONValue;
@@ -495,6 +501,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
495
501
  height?: number | undefined;
496
502
  durationMs?: number | undefined;
497
503
  description?: string | undefined;
504
+ omitted?: string | undefined;
498
505
  tags?: string[] | undefined;
499
506
  at?: {
500
507
  [x: string]: import("hono/utils/types").JSONValue;
@@ -562,6 +569,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
562
569
  height?: number | undefined;
563
570
  durationMs?: number | undefined;
564
571
  description?: string | undefined;
572
+ omitted?: string | undefined;
565
573
  tags?: string[] | undefined;
566
574
  at?: {
567
575
  [x: string]: import("hono/utils/types").JSONValue;
@@ -1,5 +1,5 @@
1
1
  import type { EvidenceStatus, SpecEntry } from './explorer-types.js';
2
- import type { EvidenceSummary, LayerEvidence, LayerHeader, LayerRow, StatusCounts } from './evidence-types.js';
2
+ import type { EvidenceSummary, LayerEvidence, LayerHeader, LayerRow, StatusCounts, VideoCounts, VideoRef } from './evidence-types.js';
3
3
  import { RecordedSpecs } from './recorded-specs.js';
4
4
  /** The version of the documents this module writes (`TestCatalog`, `TestHistoryDocument`, `RunList`, `RunDetail`). Only optional fields are added within it. */
5
5
  export declare const TEST_CATALOG_VERSION = 1;
@@ -258,13 +258,26 @@ export type RunSummary = {
258
258
  readonly environment: string | null;
259
259
  readonly detail: string;
260
260
  }[];
261
+ /**
262
+ * Its videos, from the run's layers and from the last result of each test: references, distinct playable recordings
263
+ * and references that cannot be played. Absent when it has none, and in a report that carries no video information.
264
+ */
265
+ readonly videos?: VideoCounts;
266
+ };
267
+ /** What `runList` can be asked. */
268
+ export type RunQuery = {
269
+ /** Only runs that have at least one playable video recording. `total` then counts only those. */
270
+ readonly video?: boolean;
261
271
  };
262
272
  /** `GET /api/reports/runs`: runs, newest first. */
263
273
  export type RunList = {
264
274
  readonly schemaVersion: typeof TEST_CATALOG_VERSION;
265
275
  readonly version: string;
276
+ /** Runs that matched, before `limit`. */
266
277
  readonly total: number;
267
278
  readonly runs: readonly RunSummary[];
279
+ /** How many of all runs (whatever was asked, before `limit`) have a playable video recording. */
280
+ readonly withVideo?: number;
268
281
  };
269
282
  /** One test's last result within one run. */
270
283
  export type RunResult = {
@@ -279,6 +292,8 @@ export type RunResult = {
279
292
  readonly at: number;
280
293
  readonly layer: string;
281
294
  readonly detail: string;
295
+ /** Every video its evidence references, with who holds it (check, retry attempt or step). Absent when there is none. */
296
+ readonly videos?: readonly VideoRef[];
282
297
  };
283
298
  /** `GET /api/reports/runs/detail`: one run and every result it recorded. */
284
299
  export type RunDetail = {
@@ -380,8 +395,14 @@ export declare function matchesQuery(model: CatalogModel, test: TestSummary, que
380
395
  export declare function testCatalog(model: CatalogModel, query?: TestQuery): TestCatalog;
381
396
  /** One test and its recorded results, or null when the report has no test with that key. */
382
397
  export declare function testHistory(model: CatalogModel, key: string): TestHistoryDocument | null;
383
- /** Every run the history knows of, newest first, `limit` at most (default 100). */
384
- export declare function runList(history: TestHistory, version: string, limit?: number): RunList;
398
+ /** Whether a run has at least one playable video recording. */
399
+ export declare function hasVideo(run: Pick<RunSummary, 'videos'>): boolean;
400
+ /**
401
+ * Every run the history knows of, newest first, `limit` at most (default 100). With `query.video` only the runs that
402
+ * have a playable video recording: the filter and `total` apply to every run before `limit`, and nothing is read from
403
+ * any recording's bytes. `withVideo` always counts every run that has one.
404
+ */
405
+ export declare function runList(history: TestHistory, version: string, limit?: number, query?: RunQuery): RunList;
385
406
  /** One run with every result it recorded, or null when no run has that key. */
386
407
  export declare function runDetail(history: TestHistory, version: string, key: string): RunDetail | null;
387
408
  /** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`, `artifactTag`, and `inherited=1`. */
@@ -21,6 +21,7 @@
21
21
  * @see docs/server.md#reading-the-report-as-an-agent
22
22
  */
23
23
  import { displayTag, normalizeTag, normalizeTags } from './artifact-tags.js';
24
+ import { videoCounts } from './evidence-detail.js';
24
25
  import { checkSlots } from './evidence-view.js';
25
26
  import { combineEvidence, emptyCounts, slotKey, specPath as normalSpecPath } from './layer-merge.js';
26
27
  import { RecordedSpecs } from './recorded-specs.js';
@@ -126,7 +127,7 @@ export class TestHistory {
126
127
  const key = runKey(layer);
127
128
  let run = this.#runs.get(key);
128
129
  if (!run) {
129
- run = { key, runUrl: layer.runUrl, branch: layer.branch, sha: layer.sha, start: layer.recordedAtMs, end: layer.recordedAtMs, firstAt: layer.recordedAtMs, first: layer.id, producers: new Set(), environments: new Set(), layers: 0, runId: layer.run?.id, title: undefined, titledAt: 0, titledBy: '', freshness: undefined };
130
+ run = { key, runUrl: layer.runUrl, branch: layer.branch, sha: layer.sha, start: layer.recordedAtMs, end: layer.recordedAtMs, firstAt: layer.recordedAtMs, first: layer.id, producers: new Set(), environments: new Set(), layers: 0, runId: layer.run?.id, title: undefined, titledAt: 0, titledBy: '', freshness: undefined, videos: [] };
130
131
  this.#runs.set(key, run);
131
132
  }
132
133
  // A run is told by its first layer (by time, then id), whatever order its layers arrive in.
@@ -153,6 +154,8 @@ export class TestHistory {
153
154
  if (layer.environment)
154
155
  run.environments.add(layer.environment);
155
156
  run.layers += 1;
157
+ if (layer.evidence?.videos)
158
+ run.videos.push(...layer.evidence.videos);
156
159
  const origin = originOf(layer, backend);
157
160
  for (const row of rows) {
158
161
  // "Still running" is progress, not history: a test running now shows it as its current outcome.
@@ -240,6 +243,7 @@ export class TestHistory {
240
243
  const results = cells.map(({ i, r }) => ({
241
244
  key: i.slot, id: i.id, title: i.title, specPath: i.specPath, environment: r.o.environment, producer: r.o.producer,
242
245
  status: r.s, durationMs: r.d, at: r.o.at, layer: r.o.layer, detail: shortDetail(r.t),
246
+ ...(r.e?.videos ? { videos: r.e.videos } : {}),
243
247
  }));
244
248
  results.sort((a, b) => FAILURE_RANK[b.status] - FAILURE_RANK[a.status] || compareText(a.specPath ?? '', b.specPath ?? '') || compareText(a.title, b.title) || compareText(a.environment ?? '', b.environment ?? '') || compareText(a.producer, b.producer) || compareText(a.key, b.key));
245
249
  return { run: runSummary(run, cells), results };
@@ -254,7 +258,10 @@ function runSummary(run, cells) {
254
258
  const tests = new Set();
255
259
  let total = 0, measured = false;
256
260
  const failed = [];
261
+ const videos = [...run.videos];
257
262
  for (const cell of cells) {
263
+ if (cell.r.e?.videos)
264
+ videos.push(...cell.r.e.videos);
258
265
  counts[cell.r.s] += 1;
259
266
  tests.add(cell.i.slot);
260
267
  if (cell.r.d !== null) {
@@ -265,6 +272,7 @@ function runSummary(run, cells) {
265
272
  failed.push(cell);
266
273
  }
267
274
  failed.sort((a, b) => compareText(a.i.slot, b.i.slot) || compareText(a.r.o.place, b.r.o.place) || compareText(a.r.k, b.r.k));
275
+ const video = videoCounts(videos);
268
276
  const failing = failed.slice(0, 20).map(cell => ({ key: cell.i.slot, title: cell.i.title, environment: cell.r.o.environment, detail: (cell.r.t.split('\n')[0] ?? '').slice(0, 500) }));
269
277
  return {
270
278
  key: run.key, ...(run.runId ? { runId: run.runId } : {}), ...(run.title ? { title: run.title } : {}), ...(run.freshness ? { freshness: run.freshness } : {}),
@@ -272,6 +280,7 @@ function runSummary(run, cells) {
272
280
  startedAtMs: run.start, endedAtMs: run.end, spanMs: run.end - run.start,
273
281
  producers: [...run.producers].sort(), environments: [...run.environments].sort(),
274
282
  layers: run.layers, tests: tests.size, counts, totalDurationMs: measured ? total : null, failing,
283
+ ...(video ? { videos: video } : {}),
275
284
  };
276
285
  }
277
286
  function compareText(a, b) {
@@ -571,10 +580,20 @@ export function testHistory(model, key) {
571
580
  return null;
572
581
  return { schemaVersion: TEST_CATALOG_VERSION, version: model.version, test, history: model.history?.of(key) ?? test.outcomes };
573
582
  }
574
- /** Every run the history knows of, newest first, `limit` at most (default 100). */
575
- export function runList(history, version, limit = 100) {
576
- const runs = history.runs();
577
- return { schemaVersion: TEST_CATALOG_VERSION, version, total: runs.length, runs: runs.slice(0, Math.max(0, limit)) };
583
+ /** Whether a run has at least one playable video recording. */
584
+ export function hasVideo(run) {
585
+ return (run.videos?.recordings ?? 0) > 0;
586
+ }
587
+ /**
588
+ * Every run the history knows of, newest first, `limit` at most (default 100). With `query.video` only the runs that
589
+ * have a playable video recording: the filter and `total` apply to every run before `limit`, and nothing is read from
590
+ * any recording's bytes. `withVideo` always counts every run that has one.
591
+ */
592
+ export function runList(history, version, limit = 100, query = {}) {
593
+ const all = history.runs();
594
+ const withVideo = all.filter(hasVideo).length;
595
+ const runs = query.video ? all.filter(hasVideo) : all;
596
+ return { schemaVersion: TEST_CATALOG_VERSION, version, total: runs.length, runs: runs.slice(0, Math.max(0, limit)), withVideo };
578
597
  }
579
598
  /** One run with every result it recorded, or null when no run has that key. */
580
599
  export function runDetail(history, version, key) {
@@ -687,6 +706,12 @@ export function historyMarkdown(document) {
687
706
  ...document.history.map(e => `| ${iso(e.at)} | ${e.environment ?? '—'} | ${e.status} | ${formatMs(e.durationMs)} | ${cell(e.branch ?? '—')} | ${e.sha?.slice(0, 10) ?? '—'} | ${cell(e.detail.split('\n')[0] ?? '')} |`),
688
707
  ].join('\n')}\n`;
689
708
  }
709
+ /** Video counts in a few words: `2 (3 references)`, with how many cannot be played, or `—` when there are none. */
710
+ function videosText(videos) {
711
+ if (!videos)
712
+ return '—';
713
+ return `${videos.recordings}${videos.references !== videos.recordings ? ` (${videos.references} references)` : ''}${videos.unavailable ? `, ${videos.unavailable} unavailable` : ''}`;
714
+ }
690
715
  /** What to say beside a commit that was not recorded from a clean checkout: nothing, ` (modified)` or ` (origin unknown)`. */
691
716
  function freshnessNote(freshness) {
692
717
  return freshness === 'modified' ? ' (modified: local changes)' : freshness === 'unknown' ? ' (origin unknown)' : '';
@@ -696,9 +721,9 @@ export function runsMarkdown(list) {
696
721
  return `${[
697
722
  `# Runs (${list.runs.length} of ${list.total})`,
698
723
  '',
699
- '| Ended | Branch | Commit | Environments | Tests | Passed | Failed | Not run | Span | Run |',
700
- '|---|---|---|---|---:|---:|---:|---:|---:|---|',
701
- ...list.runs.map(r => `| ${iso(r.endedAtMs)} | ${cell(r.branch ?? '—')} | ${r.sha?.slice(0, 10) ?? '—'}${freshnessNote(r.freshness)} | ${r.environments.join(', ') || '—'} | ${r.tests} | ${r.counts.passed} | ${r.counts.failed} | ${r.counts['not-run']} | ${formatMs(r.spanMs)} | ${cell(r.title ? `${r.title} (${r.key})` : r.key)} |`),
724
+ '| Ended | Branch | Commit | Environments | Tests | Passed | Failed | Not run | Span | Videos | Run |',
725
+ '|---|---|---|---|---:|---:|---:|---:|---:|---|---|',
726
+ ...list.runs.map(r => `| ${iso(r.endedAtMs)} | ${cell(r.branch ?? '—')} | ${r.sha?.slice(0, 10) ?? '—'}${freshnessNote(r.freshness)} | ${r.environments.join(', ') || '—'} | ${r.tests} | ${r.counts.passed} | ${r.counts.failed} | ${r.counts['not-run']} | ${formatMs(r.spanMs)} | ${videosText(r.videos)} | ${cell(r.title ? `${r.title} (${r.key})` : r.key)} |`),
702
727
  ].join('\n')}\n`;
703
728
  }
704
729
  /** One run as Markdown. */
@@ -712,9 +737,10 @@ export function runMarkdown(detail) {
712
737
  `- ${iso(run.startedAtMs)} to ${iso(run.endedAtMs)} (${formatMs(run.spanMs)}); ${run.layers} layers from ${run.producers.join(', ')}`,
713
738
  `- Environments: ${run.environments.join(', ') || '—'}`,
714
739
  `- ${run.tests} tests: ${counted(run.counts)}${run.totalDurationMs === null ? '' : `; ${formatMs(run.totalDurationMs)} of test time`}`,
740
+ ...(run.videos ? [`- Videos: ${videosText(run.videos)}`] : []),
715
741
  '',
716
- '| Status | Test | Spec | Environment | Duration | Detail |',
717
- '|---|---|---|---|---:|---|',
718
- ...detail.results.map(r => `| ${r.status} | ${cell(r.title)} | ${cell(r.specPath ?? '—')} | ${r.environment ?? '—'} | ${formatMs(r.durationMs)} | ${cell(r.detail.split('\n')[0] ?? '')} |`),
742
+ '| Status | Test | Spec | Environment | Duration | Videos | Detail |',
743
+ '|---|---|---|---|---:|---|---|',
744
+ ...detail.results.map(r => `| ${r.status} | ${cell(r.title)} | ${cell(r.specPath ?? '—')} | ${r.environment ?? '—'} | ${formatMs(r.durationMs)} | ${videosText(videoCounts(r.videos))} | ${cell(r.detail.split('\n')[0] ?? '')} |`),
719
745
  ].join('\n')}\n`;
720
746
  }