@crvouga/atlas 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/reports/src/attachment-files.d.ts +3 -1
- package/dist/reports/src/attachment-files.js +14 -4
- package/dist/reports/src/evidence-detail.d.ts +19 -2
- package/dist/reports/src/evidence-detail.js +63 -14
- package/dist/reports/src/evidence-types.d.ts +53 -0
- package/dist/reports/src/layer-merge.js +7 -2
- package/dist/reports/src/layer-types.d.ts +1 -1
- package/dist/reports/src/publish-checks.js +11 -3
- package/dist/reports/src/recording-receipt.d.ts +48 -0
- package/dist/reports/src/recording-receipt.js +103 -0
- package/dist/reports/src/report-cli.d.ts +1 -1
- package/dist/reports/src/report-cli.js +20 -2
- package/dist/reports/src/report-status.d.ts +2 -0
- package/dist/reports/src/report-status.js +17 -1
- package/dist/reports/src/reports-explorer-route.d.ts +8 -0
- package/dist/reports/src/reports-route.d.ts +8 -0
- package/dist/reports/src/test-catalog.d.ts +24 -3
- package/dist/reports/src/test-catalog.js +37 -11
- package/dist/schema/src/api.d.ts +232 -0
- package/dist/schema/src/api.js +35 -6
- package/dist/schema/src/cli-results.d.ts +107 -2
- package/dist/schema/src/cli-results.js +11 -3
- package/dist/schema/src/layer.d.ts +165 -0
- package/dist/schema/src/layer.js +72 -1
- package/dist/server/src/app.js +3 -2
- package/dist/server/src/llms.js +2 -1
- package/dist/server/src/openapi.js +2 -1
- package/docs/reference/cli.md +5 -0
- package/docs/reference/http-api.md +4 -2
- package/docs/reference/reports.md +204 -5
- package/docs/reference/schema.md +75 -0
- package/docs/rich-evidence.md +29 -0
- package/docs/server.md +1 -1
- package/llms-full.txt +39 -3
- package/package.json +1 -1
- package/packages/schema/schemas/check-detail.schema.json +20 -0
- package/packages/schema/schemas/effective-delta.schema.json +143 -0
- package/packages/schema/schemas/effective-evidence.schema.json +143 -0
- package/packages/schema/schemas/evidence-page.schema.json +86 -0
- package/packages/schema/schemas/layer-rows.schema.json +57 -0
- package/packages/schema/schemas/layer.schema.json +153 -0
- package/packages/schema/schemas/publish-checks.schema.json +24 -0
- package/packages/schema/schemas/publish-result.schema.json +161 -1
- package/packages/schema/schemas/report-run.schema.json +71 -0
- package/packages/schema/schemas/report-runs.schema.json +38 -1
- package/packages/schema/schemas/report-status.schema.json +104 -0
- package/packages/schema/schemas/test-catalog.schema.json +41 -0
- package/packages/schema/schemas/test-history.schema.json +82 -0
- package/packages/server/openapi.json +830 -4
- package/packages/visualizer/src/evidence/marks.tsx +16 -0
- package/packages/visualizer/src/explorer/RunsView.tsx +128 -10
- package/packages/visualizer/src/explorer/catalog.tsx +16 -6
- package/packages/visualizer/src/explorer/search.ts +4 -1
|
@@ -43,6 +43,7 @@ export const REPORT_USAGE = `atlas report — layered evidence for any kind of s
|
|
|
43
43
|
[--param key=value]... [--kind <k>] [--title <t>] [--recorded-at <iso|ms>]
|
|
44
44
|
[--attach [role=]<file>]... [--label key=value]... [--link [name=]<url>]...
|
|
45
45
|
[--attach-root <dir>]... [--max-attachment-mb <n>] [--max-total-attachment-mb <n>]
|
|
46
|
+
[--require-recordings]
|
|
46
47
|
atlas report stream [--store <dir|s3://…|https://atlas.example.com>] --producer <name> [--environment <name>]
|
|
47
48
|
[--branch <b>] [--sha <s>] [--run-url <u>] [--kind <k>] [--param key=value]...
|
|
48
49
|
[--run-id <id>] [--run-title <t>]
|
|
@@ -79,6 +80,10 @@ export const REPORT_USAGE = `atlas report — layered evidence for any kind of s
|
|
|
79
80
|
Files a report names (CTRF attachments, JUnit [[ATTACHMENT|path]] lines) come from the report's own
|
|
80
81
|
folder and each --attach-root, and never from a hidden file or a link out: a report is data from a
|
|
81
82
|
test run, not instructions. One that is missing or not allowed is skipped with a warning.
|
|
83
|
+
--require-recordings exit 1 when a check that declared recordingExpectation "required" has no
|
|
84
|
+
uploaded video (absent, left out as too large or not allowed, or refused). The layer is
|
|
85
|
+
still written and the checks keep their results; only this command fails. The receipt of
|
|
86
|
+
what was uploaded (counts, sizes, checksums) is in the JSON result under layer.recordings.
|
|
82
87
|
From a checkout with uncommitted changes the layer is marked "modified".
|
|
83
88
|
stream Send checks as they finish: one JSON object per line on standard input (the fields of one
|
|
84
89
|
check: id, title, status, and any evidence). Each second's worth becomes one layer, so a server
|
|
@@ -310,7 +315,7 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
|
|
|
310
315
|
const json = wantsJson(rest);
|
|
311
316
|
const result = {
|
|
312
317
|
ok: true, store: target.label,
|
|
313
|
-
layer: { id: layer.id, producer, environment: layer.environment ?? null, checks: ingested.checks.length, passed: counts.passed, failed: counts.failed, notRun: counts.other, linkedToSpec: linkedCount, ...(freshness ? { freshness } : {}), attachmentFiles: attachedFiles, attachmentBytes: attachedBytes },
|
|
318
|
+
layer: { id: layer.id, producer, environment: layer.environment ?? null, checks: ingested.checks.length, passed: counts.passed, failed: counts.failed, notRun: counts.other, linkedToSpec: linkedCount, ...(freshness ? { freshness } : {}), attachmentFiles: attachedFiles, attachmentBytes: attachedBytes, ...(layer.recordings ? { recordings: layer.recordings } : {}) },
|
|
314
319
|
warnings: warnings.map(warning => `attachment skipped: ${warning}`)
|
|
315
320
|
};
|
|
316
321
|
if (!json)
|
|
@@ -318,6 +323,19 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
|
|
|
318
323
|
if (!json)
|
|
319
324
|
for (const warning of warnings)
|
|
320
325
|
io.out(`! attachment skipped: ${warning}\n`);
|
|
326
|
+
// Recording completeness is reported apart from the checks' results. Only --require-recordings turns a gap into a failing exit.
|
|
327
|
+
const gaps = (layer.recordings?.items ?? []).filter(item => item.state !== 'uploaded' && item.state !== 'referenced' && item.state !== 'not-applicable');
|
|
328
|
+
const strict = rest.includes('--require-recordings');
|
|
329
|
+
if (layer.recordings) {
|
|
330
|
+
result.recordings = { strict, complete: gaps.length === 0, expected: layer.recordings.expected, uploaded: layer.recordings.uploaded, gaps: gaps.map(({ check, state, reason }) => ({ check, state, ...(reason ? { reason } : {}) })) };
|
|
331
|
+
if (!json) {
|
|
332
|
+
io.out(`Recordings: ${layer.recordings.uploaded + layer.recordings.referenced} of ${layer.recordings.expected} expected published${layer.recordings.notApplicable ? `, ${layer.recordings.notApplicable} not applicable` : ''}\n`);
|
|
333
|
+
for (const gap of gaps)
|
|
334
|
+
io.out(`! recording ${gap.state}: ${gap.check}${gap.reason ? ` (${gap.reason})` : ''}\n`);
|
|
335
|
+
}
|
|
336
|
+
if (strict && gaps.length)
|
|
337
|
+
result.ok = false;
|
|
338
|
+
}
|
|
321
339
|
if (specsDirectory) {
|
|
322
340
|
const files = await collectSpecFiles(path.resolve(base, specsDirectory));
|
|
323
341
|
// A spec named with --spec that the directory does not hold (or whose kind is not a spec file) is published too.
|
|
@@ -343,7 +361,7 @@ export async function reportMain(argv, io = { out: text => process.stdout.write(
|
|
|
343
361
|
}
|
|
344
362
|
if (json)
|
|
345
363
|
io.out(`${JSON.stringify(result, null, 2)}\n`);
|
|
346
|
-
return 0;
|
|
364
|
+
return result.ok ? 0 : 1;
|
|
347
365
|
}
|
|
348
366
|
if (command === 'stream') {
|
|
349
367
|
const target = writeStoreOf(value(rest, 'store'), base, env);
|
|
@@ -26,6 +26,8 @@ export type StatusOptions = {
|
|
|
26
26
|
* each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
|
|
27
27
|
* report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
|
|
28
28
|
* not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
|
|
29
|
+
* `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
|
|
30
|
+
* completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
|
|
29
31
|
*
|
|
30
32
|
* @example
|
|
31
33
|
* const status = reportStatus(await combineBackends(backends), { requireEnvironments: ['ios'] });
|
|
@@ -12,6 +12,7 @@ import { REPORT_STATUS_SCHEMA_VERSION } from '../../schema/src/layer.js';
|
|
|
12
12
|
import { countResults } from './compose.js';
|
|
13
13
|
import { normalizeEnvironment } from './layer-axes.js';
|
|
14
14
|
import { isDiagnosticKind } from './layer-merge.js';
|
|
15
|
+
import { recordingCompleteness } from './recording-receipt.js';
|
|
15
16
|
/** Total string order by code unit. */
|
|
16
17
|
const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
17
18
|
/**
|
|
@@ -23,6 +24,8 @@ const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
|
23
24
|
* each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
|
|
24
25
|
* report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
|
|
25
26
|
* not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
|
|
27
|
+
* `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
|
|
28
|
+
* completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
|
|
26
29
|
*
|
|
27
30
|
* @example
|
|
28
31
|
* const status = reportStatus(await combineBackends(backends), { requireEnvironments: ['ios'] });
|
|
@@ -85,7 +88,12 @@ export function reportStatus(report, options = {}) {
|
|
|
85
88
|
...report.combined.specs.map(entry => specOutcome(entry.spec.path, entry.spec.title, entry.spec.kind, entry.results)),
|
|
86
89
|
...(report.combined.unassigned.length ? [specOutcome(null, 'No spec', 'unassigned', report.combined.unassigned)] : [])
|
|
87
90
|
],
|
|
88
|
-
failing
|
|
91
|
+
failing,
|
|
92
|
+
// Apart from `ok`: whether the expected videos were published says nothing about whether the checks passed.
|
|
93
|
+
recordings: recordingCompleteness(outcomes.map(result => ({
|
|
94
|
+
spec: result.row.specPath, check: result.row.id, environment: result.layer.environment, layer: result.layer.id,
|
|
95
|
+
hasReceipt: result.layer.recordings !== undefined, ...(result.row.recording ? { recording: result.row.recording } : {})
|
|
96
|
+
})))
|
|
89
97
|
};
|
|
90
98
|
}
|
|
91
99
|
/** `spec › check`, unless the check's id already starts with its spec. */
|
|
@@ -120,6 +128,14 @@ export function formatStatusText(status) {
|
|
|
120
128
|
const where = [item.environment, item.branch, item.sha?.slice(0, 7)].filter(Boolean).join(' · ');
|
|
121
129
|
lines.push(`✗ ${item.status === 'running' ? '[running] ' : ''}${label(item)}${where ? ` (${where})` : ''}${item.detail ? `\n ${item.detail.split('\n')[0]}` : ''}`);
|
|
122
130
|
}
|
|
131
|
+
const recordings = status.recordings;
|
|
132
|
+
// Quiet for a report that never mentioned recordings; the JSON says `unknown` explicitly.
|
|
133
|
+
if (recordings && !(recordings.state === 'unknown' && recordings.expected === 0 && recordings.notApplicable === 0)) {
|
|
134
|
+
const published = recordings.uploaded + recordings.referenced;
|
|
135
|
+
lines.push(`· recordings: ${recordings.state} — ${published} of ${recordings.expected} expected published${recordings.notApplicable ? `, ${recordings.notApplicable} not applicable` : ''}${recordings.layersWithoutReceipt ? `, ${recordings.layersWithoutReceipt} layer${recordings.layersWithoutReceipt === 1 ? '' : 's'} unknown` : ''}`);
|
|
136
|
+
for (const gap of recordings.gaps)
|
|
137
|
+
lines.push(` ! recording ${gap.state}: ${label({ spec: gap.spec, check: gap.check })}${gap.reason ? ` (${gap.reason})` : ''}`);
|
|
138
|
+
}
|
|
123
139
|
for (const notice of status.notices)
|
|
124
140
|
lines.push(`! ${notice}`);
|
|
125
141
|
for (const reason of status.reasons)
|
|
@@ -82,6 +82,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
82
82
|
height?: number | undefined;
|
|
83
83
|
durationMs?: number | undefined;
|
|
84
84
|
description?: string | undefined;
|
|
85
|
+
omitted?: string | undefined;
|
|
85
86
|
tags?: string[] | undefined;
|
|
86
87
|
at?: {
|
|
87
88
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -126,6 +127,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
126
127
|
height?: number | undefined;
|
|
127
128
|
durationMs?: number | undefined;
|
|
128
129
|
description?: string | undefined;
|
|
130
|
+
omitted?: string | undefined;
|
|
129
131
|
tags?: string[] | undefined;
|
|
130
132
|
at?: {
|
|
131
133
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -183,6 +185,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
183
185
|
height?: number | undefined;
|
|
184
186
|
durationMs?: number | undefined;
|
|
185
187
|
description?: string | undefined;
|
|
188
|
+
omitted?: string | undefined;
|
|
186
189
|
tags?: string[] | undefined;
|
|
187
190
|
at?: {
|
|
188
191
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -250,6 +253,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
250
253
|
height?: number | undefined;
|
|
251
254
|
durationMs?: number | undefined;
|
|
252
255
|
description?: string | undefined;
|
|
256
|
+
omitted?: string | undefined;
|
|
253
257
|
tags?: string[] | undefined;
|
|
254
258
|
at?: {
|
|
255
259
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -356,6 +360,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
356
360
|
height?: number | undefined;
|
|
357
361
|
durationMs?: number | undefined;
|
|
358
362
|
description?: string | undefined;
|
|
363
|
+
omitted?: string | undefined;
|
|
359
364
|
tags?: string[] | undefined;
|
|
360
365
|
at?: {
|
|
361
366
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -400,6 +405,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
400
405
|
height?: number | undefined;
|
|
401
406
|
durationMs?: number | undefined;
|
|
402
407
|
description?: string | undefined;
|
|
408
|
+
omitted?: string | undefined;
|
|
403
409
|
tags?: string[] | undefined;
|
|
404
410
|
at?: {
|
|
405
411
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -457,6 +463,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
457
463
|
height?: number | undefined;
|
|
458
464
|
durationMs?: number | undefined;
|
|
459
465
|
description?: string | undefined;
|
|
466
|
+
omitted?: string | undefined;
|
|
460
467
|
tags?: string[] | undefined;
|
|
461
468
|
at?: {
|
|
462
469
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -524,6 +531,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
524
531
|
height?: number | undefined;
|
|
525
532
|
durationMs?: number | undefined;
|
|
526
533
|
description?: string | undefined;
|
|
534
|
+
omitted?: string | undefined;
|
|
527
535
|
tags?: string[] | undefined;
|
|
528
536
|
at?: {
|
|
529
537
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -120,6 +120,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
120
120
|
height?: number | undefined;
|
|
121
121
|
durationMs?: number | undefined;
|
|
122
122
|
description?: string | undefined;
|
|
123
|
+
omitted?: string | undefined;
|
|
123
124
|
tags?: string[] | undefined;
|
|
124
125
|
at?: {
|
|
125
126
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -164,6 +165,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
164
165
|
height?: number | undefined;
|
|
165
166
|
durationMs?: number | undefined;
|
|
166
167
|
description?: string | undefined;
|
|
168
|
+
omitted?: string | undefined;
|
|
167
169
|
tags?: string[] | undefined;
|
|
168
170
|
at?: {
|
|
169
171
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -221,6 +223,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
221
223
|
height?: number | undefined;
|
|
222
224
|
durationMs?: number | undefined;
|
|
223
225
|
description?: string | undefined;
|
|
226
|
+
omitted?: string | undefined;
|
|
224
227
|
tags?: string[] | undefined;
|
|
225
228
|
at?: {
|
|
226
229
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -288,6 +291,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
288
291
|
height?: number | undefined;
|
|
289
292
|
durationMs?: number | undefined;
|
|
290
293
|
description?: string | undefined;
|
|
294
|
+
omitted?: string | undefined;
|
|
291
295
|
tags?: string[] | undefined;
|
|
292
296
|
at?: {
|
|
293
297
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -394,6 +398,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
394
398
|
height?: number | undefined;
|
|
395
399
|
durationMs?: number | undefined;
|
|
396
400
|
description?: string | undefined;
|
|
401
|
+
omitted?: string | undefined;
|
|
397
402
|
tags?: string[] | undefined;
|
|
398
403
|
at?: {
|
|
399
404
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -438,6 +443,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
438
443
|
height?: number | undefined;
|
|
439
444
|
durationMs?: number | undefined;
|
|
440
445
|
description?: string | undefined;
|
|
446
|
+
omitted?: string | undefined;
|
|
441
447
|
tags?: string[] | undefined;
|
|
442
448
|
at?: {
|
|
443
449
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -495,6 +501,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
495
501
|
height?: number | undefined;
|
|
496
502
|
durationMs?: number | undefined;
|
|
497
503
|
description?: string | undefined;
|
|
504
|
+
omitted?: string | undefined;
|
|
498
505
|
tags?: string[] | undefined;
|
|
499
506
|
at?: {
|
|
500
507
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -562,6 +569,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
562
569
|
height?: number | undefined;
|
|
563
570
|
durationMs?: number | undefined;
|
|
564
571
|
description?: string | undefined;
|
|
572
|
+
omitted?: string | undefined;
|
|
565
573
|
tags?: string[] | undefined;
|
|
566
574
|
at?: {
|
|
567
575
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { EvidenceStatus, SpecEntry } from './explorer-types.js';
|
|
2
|
-
import type { EvidenceSummary, LayerEvidence, LayerHeader, LayerRow, StatusCounts } from './evidence-types.js';
|
|
2
|
+
import type { EvidenceSummary, LayerEvidence, LayerHeader, LayerRow, StatusCounts, VideoCounts, VideoRef } from './evidence-types.js';
|
|
3
3
|
import { RecordedSpecs } from './recorded-specs.js';
|
|
4
4
|
/** The version of the documents this module writes (`TestCatalog`, `TestHistoryDocument`, `RunList`, `RunDetail`). Only optional fields are added within it. */
|
|
5
5
|
export declare const TEST_CATALOG_VERSION = 1;
|
|
@@ -258,13 +258,26 @@ export type RunSummary = {
|
|
|
258
258
|
readonly environment: string | null;
|
|
259
259
|
readonly detail: string;
|
|
260
260
|
}[];
|
|
261
|
+
/**
|
|
262
|
+
* Its videos, from the run's layers and from the last result of each test: references, distinct playable recordings
|
|
263
|
+
* and references that cannot be played. Absent when it has none, and in a report that carries no video information.
|
|
264
|
+
*/
|
|
265
|
+
readonly videos?: VideoCounts;
|
|
266
|
+
};
|
|
267
|
+
/** What `runList` can be asked. */
|
|
268
|
+
export type RunQuery = {
|
|
269
|
+
/** Only runs that have at least one playable video recording. `total` then counts only those. */
|
|
270
|
+
readonly video?: boolean;
|
|
261
271
|
};
|
|
262
272
|
/** `GET /api/reports/runs`: runs, newest first. */
|
|
263
273
|
export type RunList = {
|
|
264
274
|
readonly schemaVersion: typeof TEST_CATALOG_VERSION;
|
|
265
275
|
readonly version: string;
|
|
276
|
+
/** Runs that matched, before `limit`. */
|
|
266
277
|
readonly total: number;
|
|
267
278
|
readonly runs: readonly RunSummary[];
|
|
279
|
+
/** How many of all runs (whatever was asked, before `limit`) have a playable video recording. */
|
|
280
|
+
readonly withVideo?: number;
|
|
268
281
|
};
|
|
269
282
|
/** One test's last result within one run. */
|
|
270
283
|
export type RunResult = {
|
|
@@ -279,6 +292,8 @@ export type RunResult = {
|
|
|
279
292
|
readonly at: number;
|
|
280
293
|
readonly layer: string;
|
|
281
294
|
readonly detail: string;
|
|
295
|
+
/** Every video its evidence references, with who holds it (check, retry attempt or step). Absent when there is none. */
|
|
296
|
+
readonly videos?: readonly VideoRef[];
|
|
282
297
|
};
|
|
283
298
|
/** `GET /api/reports/runs/detail`: one run and every result it recorded. */
|
|
284
299
|
export type RunDetail = {
|
|
@@ -380,8 +395,14 @@ export declare function matchesQuery(model: CatalogModel, test: TestSummary, que
|
|
|
380
395
|
export declare function testCatalog(model: CatalogModel, query?: TestQuery): TestCatalog;
|
|
381
396
|
/** One test and its recorded results, or null when the report has no test with that key. */
|
|
382
397
|
export declare function testHistory(model: CatalogModel, key: string): TestHistoryDocument | null;
|
|
383
|
-
/**
|
|
384
|
-
export declare function
|
|
398
|
+
/** Whether a run has at least one playable video recording. */
|
|
399
|
+
export declare function hasVideo(run: Pick<RunSummary, 'videos'>): boolean;
|
|
400
|
+
/**
|
|
401
|
+
* Every run the history knows of, newest first, `limit` at most (default 100). With `query.video` only the runs that
|
|
402
|
+
* have a playable video recording: the filter and `total` apply to every run before `limit`, and nothing is read from
|
|
403
|
+
* any recording's bytes. `withVideo` always counts every run that has one.
|
|
404
|
+
*/
|
|
405
|
+
export declare function runList(history: TestHistory, version: string, limit?: number, query?: RunQuery): RunList;
|
|
385
406
|
/** One run with every result it recorded, or null when no run has that key. */
|
|
386
407
|
export declare function runDetail(history: TestHistory, version: string, key: string): RunDetail | null;
|
|
387
408
|
/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`, `artifactTag`, and `inherited=1`. */
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
* @see docs/server.md#reading-the-report-as-an-agent
|
|
22
22
|
*/
|
|
23
23
|
import { displayTag, normalizeTag, normalizeTags } from './artifact-tags.js';
|
|
24
|
+
import { videoCounts } from './evidence-detail.js';
|
|
24
25
|
import { checkSlots } from './evidence-view.js';
|
|
25
26
|
import { combineEvidence, emptyCounts, slotKey, specPath as normalSpecPath } from './layer-merge.js';
|
|
26
27
|
import { RecordedSpecs } from './recorded-specs.js';
|
|
@@ -126,7 +127,7 @@ export class TestHistory {
|
|
|
126
127
|
const key = runKey(layer);
|
|
127
128
|
let run = this.#runs.get(key);
|
|
128
129
|
if (!run) {
|
|
129
|
-
run = { key, runUrl: layer.runUrl, branch: layer.branch, sha: layer.sha, start: layer.recordedAtMs, end: layer.recordedAtMs, firstAt: layer.recordedAtMs, first: layer.id, producers: new Set(), environments: new Set(), layers: 0, runId: layer.run?.id, title: undefined, titledAt: 0, titledBy: '', freshness: undefined };
|
|
130
|
+
run = { key, runUrl: layer.runUrl, branch: layer.branch, sha: layer.sha, start: layer.recordedAtMs, end: layer.recordedAtMs, firstAt: layer.recordedAtMs, first: layer.id, producers: new Set(), environments: new Set(), layers: 0, runId: layer.run?.id, title: undefined, titledAt: 0, titledBy: '', freshness: undefined, videos: [] };
|
|
130
131
|
this.#runs.set(key, run);
|
|
131
132
|
}
|
|
132
133
|
// A run is told by its first layer (by time, then id), whatever order its layers arrive in.
|
|
@@ -153,6 +154,8 @@ export class TestHistory {
|
|
|
153
154
|
if (layer.environment)
|
|
154
155
|
run.environments.add(layer.environment);
|
|
155
156
|
run.layers += 1;
|
|
157
|
+
if (layer.evidence?.videos)
|
|
158
|
+
run.videos.push(...layer.evidence.videos);
|
|
156
159
|
const origin = originOf(layer, backend);
|
|
157
160
|
for (const row of rows) {
|
|
158
161
|
// "Still running" is progress, not history: a test running now shows it as its current outcome.
|
|
@@ -240,6 +243,7 @@ export class TestHistory {
|
|
|
240
243
|
const results = cells.map(({ i, r }) => ({
|
|
241
244
|
key: i.slot, id: i.id, title: i.title, specPath: i.specPath, environment: r.o.environment, producer: r.o.producer,
|
|
242
245
|
status: r.s, durationMs: r.d, at: r.o.at, layer: r.o.layer, detail: shortDetail(r.t),
|
|
246
|
+
...(r.e?.videos ? { videos: r.e.videos } : {}),
|
|
243
247
|
}));
|
|
244
248
|
results.sort((a, b) => FAILURE_RANK[b.status] - FAILURE_RANK[a.status] || compareText(a.specPath ?? '', b.specPath ?? '') || compareText(a.title, b.title) || compareText(a.environment ?? '', b.environment ?? '') || compareText(a.producer, b.producer) || compareText(a.key, b.key));
|
|
245
249
|
return { run: runSummary(run, cells), results };
|
|
@@ -254,7 +258,10 @@ function runSummary(run, cells) {
|
|
|
254
258
|
const tests = new Set();
|
|
255
259
|
let total = 0, measured = false;
|
|
256
260
|
const failed = [];
|
|
261
|
+
const videos = [...run.videos];
|
|
257
262
|
for (const cell of cells) {
|
|
263
|
+
if (cell.r.e?.videos)
|
|
264
|
+
videos.push(...cell.r.e.videos);
|
|
258
265
|
counts[cell.r.s] += 1;
|
|
259
266
|
tests.add(cell.i.slot);
|
|
260
267
|
if (cell.r.d !== null) {
|
|
@@ -265,6 +272,7 @@ function runSummary(run, cells) {
|
|
|
265
272
|
failed.push(cell);
|
|
266
273
|
}
|
|
267
274
|
failed.sort((a, b) => compareText(a.i.slot, b.i.slot) || compareText(a.r.o.place, b.r.o.place) || compareText(a.r.k, b.r.k));
|
|
275
|
+
const video = videoCounts(videos);
|
|
268
276
|
const failing = failed.slice(0, 20).map(cell => ({ key: cell.i.slot, title: cell.i.title, environment: cell.r.o.environment, detail: (cell.r.t.split('\n')[0] ?? '').slice(0, 500) }));
|
|
269
277
|
return {
|
|
270
278
|
key: run.key, ...(run.runId ? { runId: run.runId } : {}), ...(run.title ? { title: run.title } : {}), ...(run.freshness ? { freshness: run.freshness } : {}),
|
|
@@ -272,6 +280,7 @@ function runSummary(run, cells) {
|
|
|
272
280
|
startedAtMs: run.start, endedAtMs: run.end, spanMs: run.end - run.start,
|
|
273
281
|
producers: [...run.producers].sort(), environments: [...run.environments].sort(),
|
|
274
282
|
layers: run.layers, tests: tests.size, counts, totalDurationMs: measured ? total : null, failing,
|
|
283
|
+
...(video ? { videos: video } : {}),
|
|
275
284
|
};
|
|
276
285
|
}
|
|
277
286
|
function compareText(a, b) {
|
|
@@ -571,10 +580,20 @@ export function testHistory(model, key) {
|
|
|
571
580
|
return null;
|
|
572
581
|
return { schemaVersion: TEST_CATALOG_VERSION, version: model.version, test, history: model.history?.of(key) ?? test.outcomes };
|
|
573
582
|
}
|
|
574
|
-
/**
|
|
575
|
-
export function
|
|
576
|
-
|
|
577
|
-
|
|
583
|
+
/** Whether a run has at least one playable video recording. */
|
|
584
|
+
export function hasVideo(run) {
|
|
585
|
+
return (run.videos?.recordings ?? 0) > 0;
|
|
586
|
+
}
|
|
587
|
+
/**
|
|
588
|
+
* Every run the history knows of, newest first, `limit` at most (default 100). With `query.video` only the runs that
|
|
589
|
+
* have a playable video recording: the filter and `total` apply to every run before `limit`, and nothing is read from
|
|
590
|
+
* any recording's bytes. `withVideo` always counts every run that has one.
|
|
591
|
+
*/
|
|
592
|
+
export function runList(history, version, limit = 100, query = {}) {
|
|
593
|
+
const all = history.runs();
|
|
594
|
+
const withVideo = all.filter(hasVideo).length;
|
|
595
|
+
const runs = query.video ? all.filter(hasVideo) : all;
|
|
596
|
+
return { schemaVersion: TEST_CATALOG_VERSION, version, total: runs.length, runs: runs.slice(0, Math.max(0, limit)), withVideo };
|
|
578
597
|
}
|
|
579
598
|
/** One run with every result it recorded, or null when no run has that key. */
|
|
580
599
|
export function runDetail(history, version, key) {
|
|
@@ -687,6 +706,12 @@ export function historyMarkdown(document) {
|
|
|
687
706
|
...document.history.map(e => `| ${iso(e.at)} | ${e.environment ?? '—'} | ${e.status} | ${formatMs(e.durationMs)} | ${cell(e.branch ?? '—')} | ${e.sha?.slice(0, 10) ?? '—'} | ${cell(e.detail.split('\n')[0] ?? '')} |`),
|
|
688
707
|
].join('\n')}\n`;
|
|
689
708
|
}
|
|
709
|
+
/** Video counts in a few words: `2 (3 references)`, with how many cannot be played, or `—` when there are none. */
|
|
710
|
+
function videosText(videos) {
|
|
711
|
+
if (!videos)
|
|
712
|
+
return '—';
|
|
713
|
+
return `${videos.recordings}${videos.references !== videos.recordings ? ` (${videos.references} references)` : ''}${videos.unavailable ? `, ${videos.unavailable} unavailable` : ''}`;
|
|
714
|
+
}
|
|
690
715
|
/** What to say beside a commit that was not recorded from a clean checkout: nothing, ` (modified)` or ` (origin unknown)`. */
|
|
691
716
|
function freshnessNote(freshness) {
|
|
692
717
|
return freshness === 'modified' ? ' (modified: local changes)' : freshness === 'unknown' ? ' (origin unknown)' : '';
|
|
@@ -696,9 +721,9 @@ export function runsMarkdown(list) {
|
|
|
696
721
|
return `${[
|
|
697
722
|
`# Runs (${list.runs.length} of ${list.total})`,
|
|
698
723
|
'',
|
|
699
|
-
'| Ended | Branch | Commit | Environments | Tests | Passed | Failed | Not run | Span | Run |',
|
|
700
|
-
'
|
|
701
|
-
...list.runs.map(r => `| ${iso(r.endedAtMs)} | ${cell(r.branch ?? '—')} | ${r.sha?.slice(0, 10) ?? '—'}${freshnessNote(r.freshness)} | ${r.environments.join(', ') || '—'} | ${r.tests} | ${r.counts.passed} | ${r.counts.failed} | ${r.counts['not-run']} | ${formatMs(r.spanMs)} | ${cell(r.title ? `${r.title} (${r.key})` : r.key)} |`),
|
|
724
|
+
'| Ended | Branch | Commit | Environments | Tests | Passed | Failed | Not run | Span | Videos | Run |',
|
|
725
|
+
'|---|---|---|---|---:|---:|---:|---:|---:|---|---|',
|
|
726
|
+
...list.runs.map(r => `| ${iso(r.endedAtMs)} | ${cell(r.branch ?? '—')} | ${r.sha?.slice(0, 10) ?? '—'}${freshnessNote(r.freshness)} | ${r.environments.join(', ') || '—'} | ${r.tests} | ${r.counts.passed} | ${r.counts.failed} | ${r.counts['not-run']} | ${formatMs(r.spanMs)} | ${videosText(r.videos)} | ${cell(r.title ? `${r.title} (${r.key})` : r.key)} |`),
|
|
702
727
|
].join('\n')}\n`;
|
|
703
728
|
}
|
|
704
729
|
/** One run as Markdown. */
|
|
@@ -712,9 +737,10 @@ export function runMarkdown(detail) {
|
|
|
712
737
|
`- ${iso(run.startedAtMs)} to ${iso(run.endedAtMs)} (${formatMs(run.spanMs)}); ${run.layers} layers from ${run.producers.join(', ')}`,
|
|
713
738
|
`- Environments: ${run.environments.join(', ') || '—'}`,
|
|
714
739
|
`- ${run.tests} tests: ${counted(run.counts)}${run.totalDurationMs === null ? '' : `; ${formatMs(run.totalDurationMs)} of test time`}`,
|
|
740
|
+
...(run.videos ? [`- Videos: ${videosText(run.videos)}`] : []),
|
|
715
741
|
'',
|
|
716
|
-
'| Status | Test | Spec | Environment | Duration | Detail |',
|
|
717
|
-
'
|
|
718
|
-
...detail.results.map(r => `| ${r.status} | ${cell(r.title)} | ${cell(r.specPath ?? '—')} | ${r.environment ?? '—'} | ${formatMs(r.durationMs)} | ${cell(r.detail.split('\n')[0] ?? '')} |`),
|
|
742
|
+
'| Status | Test | Spec | Environment | Duration | Videos | Detail |',
|
|
743
|
+
'|---|---|---|---|---:|---|---|',
|
|
744
|
+
...detail.results.map(r => `| ${r.status} | ${cell(r.title)} | ${cell(r.specPath ?? '—')} | ${r.environment ?? '—'} | ${formatMs(r.durationMs)} | ${videosText(videoCounts(r.videos))} | ${cell(r.detail.split('\n')[0] ?? '')} |`),
|
|
719
745
|
].join('\n')}\n`;
|
|
720
746
|
}
|