@crvouga/atlas 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/playwright-reporter/src/index.d.ts +11 -0
- package/dist/playwright-reporter/src/index.js +4 -2
- package/dist/reports/src/artifact-tag-store.d.ts +20 -0
- package/dist/reports/src/artifact-tag-store.js +70 -0
- package/dist/reports/src/artifact-tags.d.ts +96 -0
- package/dist/reports/src/artifact-tags.js +96 -0
- package/dist/reports/src/evidence-detail.d.ts +1 -1
- package/dist/reports/src/evidence-detail.js +6 -2
- package/dist/reports/src/evidence-types.d.ts +5 -0
- package/dist/reports/src/layer-merge.d.ts +1 -1
- package/dist/reports/src/layer-merge.js +3 -2
- package/dist/reports/src/layer-types.d.ts +3 -0
- package/dist/reports/src/publish-checks.js +1 -0
- package/dist/reports/src/report-cli.d.ts +1 -1
- package/dist/reports/src/report-cli.js +30 -2
- package/dist/reports/src/reporter.d.ts +1 -1
- package/dist/reports/src/reports-explorer-route.d.ts +8 -0
- package/dist/reports/src/reports-route.d.ts +8 -0
- package/dist/reports/src/test-catalog.d.ts +31 -27
- package/dist/reports/src/test-catalog.js +73 -12
- package/dist/schema/src/api.d.ts +128 -1
- package/dist/schema/src/api.js +38 -4
- package/dist/schema/src/layer.d.ts +59 -0
- package/dist/schema/src/layer.js +28 -1
- package/dist/server/src/app.js +32 -1
- package/dist/server/src/openapi.d.ts +1 -1
- package/dist/server/src/openapi.js +18 -2
- package/docs/extending.md +3 -0
- package/docs/integrating.md +8 -0
- package/docs/reference/cli.md +7 -0
- package/docs/reference/http-api.md +48 -0
- package/docs/reference/playwright-reporter.md +6 -6
- package/docs/reference/reports.md +235 -8
- package/docs/reference/schema.md +52 -1
- package/docs/report-contract.md +1 -0
- package/docs/rich-evidence.md +28 -0
- package/docs/server.md +24 -0
- package/llms-full.txt +119 -0
- package/package.json +1 -1
- package/packages/schema/schemas/check-detail.schema.json +28 -0
- package/packages/schema/schemas/effective-delta.schema.json +44 -0
- package/packages/schema/schemas/effective-evidence.schema.json +44 -0
- package/packages/schema/schemas/evidence-page.schema.json +37 -0
- package/packages/schema/schemas/layer-rows.schema.json +7 -0
- package/packages/schema/schemas/layer.schema.json +100 -0
- package/packages/schema/schemas/publish-checks.schema.json +65 -0
- package/packages/schema/schemas/report-run.schema.json +17 -1
- package/packages/schema/schemas/report-runs.schema.json +17 -1
- package/packages/schema/schemas/test-catalog.schema.json +42 -0
- package/packages/schema/schemas/test-history.schema.json +28 -0
- package/packages/server/openapi.json +886 -3
- package/packages/visualizer/src/evidence/artifact-tags.tsx +285 -0
- package/packages/visualizer/src/evidence/artifacts.tsx +18 -12
- package/packages/visualizer/src/explorer/RunsView.tsx +23 -4
- package/packages/visualizer/src/explorer/parts.tsx +2 -1
|
@@ -82,6 +82,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
82
82
|
height?: number | undefined;
|
|
83
83
|
durationMs?: number | undefined;
|
|
84
84
|
description?: string | undefined;
|
|
85
|
+
tags?: string[] | undefined;
|
|
85
86
|
at?: {
|
|
86
87
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
87
88
|
step?: string | undefined;
|
|
@@ -125,6 +126,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
125
126
|
height?: number | undefined;
|
|
126
127
|
durationMs?: number | undefined;
|
|
127
128
|
description?: string | undefined;
|
|
129
|
+
tags?: string[] | undefined;
|
|
128
130
|
at?: {
|
|
129
131
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
130
132
|
step?: string | undefined;
|
|
@@ -181,6 +183,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
181
183
|
height?: number | undefined;
|
|
182
184
|
durationMs?: number | undefined;
|
|
183
185
|
description?: string | undefined;
|
|
186
|
+
tags?: string[] | undefined;
|
|
184
187
|
at?: {
|
|
185
188
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
186
189
|
step?: string | undefined;
|
|
@@ -247,6 +250,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
247
250
|
height?: number | undefined;
|
|
248
251
|
durationMs?: number | undefined;
|
|
249
252
|
description?: string | undefined;
|
|
253
|
+
tags?: string[] | undefined;
|
|
250
254
|
at?: {
|
|
251
255
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
252
256
|
step?: string | undefined;
|
|
@@ -352,6 +356,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
352
356
|
height?: number | undefined;
|
|
353
357
|
durationMs?: number | undefined;
|
|
354
358
|
description?: string | undefined;
|
|
359
|
+
tags?: string[] | undefined;
|
|
355
360
|
at?: {
|
|
356
361
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
357
362
|
step?: string | undefined;
|
|
@@ -395,6 +400,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
395
400
|
height?: number | undefined;
|
|
396
401
|
durationMs?: number | undefined;
|
|
397
402
|
description?: string | undefined;
|
|
403
|
+
tags?: string[] | undefined;
|
|
398
404
|
at?: {
|
|
399
405
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
400
406
|
step?: string | undefined;
|
|
@@ -451,6 +457,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
451
457
|
height?: number | undefined;
|
|
452
458
|
durationMs?: number | undefined;
|
|
453
459
|
description?: string | undefined;
|
|
460
|
+
tags?: string[] | undefined;
|
|
454
461
|
at?: {
|
|
455
462
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
456
463
|
step?: string | undefined;
|
|
@@ -517,6 +524,7 @@ export declare function createReportsExplorerApp(deps: ReportsExplorerDeps): imp
|
|
|
517
524
|
height?: number | undefined;
|
|
518
525
|
durationMs?: number | undefined;
|
|
519
526
|
description?: string | undefined;
|
|
527
|
+
tags?: string[] | undefined;
|
|
520
528
|
at?: {
|
|
521
529
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
522
530
|
step?: string | undefined;
|
|
@@ -120,6 +120,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
120
120
|
height?: number | undefined;
|
|
121
121
|
durationMs?: number | undefined;
|
|
122
122
|
description?: string | undefined;
|
|
123
|
+
tags?: string[] | undefined;
|
|
123
124
|
at?: {
|
|
124
125
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
125
126
|
step?: string | undefined;
|
|
@@ -163,6 +164,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
163
164
|
height?: number | undefined;
|
|
164
165
|
durationMs?: number | undefined;
|
|
165
166
|
description?: string | undefined;
|
|
167
|
+
tags?: string[] | undefined;
|
|
166
168
|
at?: {
|
|
167
169
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
168
170
|
step?: string | undefined;
|
|
@@ -219,6 +221,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
219
221
|
height?: number | undefined;
|
|
220
222
|
durationMs?: number | undefined;
|
|
221
223
|
description?: string | undefined;
|
|
224
|
+
tags?: string[] | undefined;
|
|
222
225
|
at?: {
|
|
223
226
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
224
227
|
step?: string | undefined;
|
|
@@ -285,6 +288,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
285
288
|
height?: number | undefined;
|
|
286
289
|
durationMs?: number | undefined;
|
|
287
290
|
description?: string | undefined;
|
|
291
|
+
tags?: string[] | undefined;
|
|
288
292
|
at?: {
|
|
289
293
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
290
294
|
step?: string | undefined;
|
|
@@ -390,6 +394,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
390
394
|
height?: number | undefined;
|
|
391
395
|
durationMs?: number | undefined;
|
|
392
396
|
description?: string | undefined;
|
|
397
|
+
tags?: string[] | undefined;
|
|
393
398
|
at?: {
|
|
394
399
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
395
400
|
step?: string | undefined;
|
|
@@ -433,6 +438,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
433
438
|
height?: number | undefined;
|
|
434
439
|
durationMs?: number | undefined;
|
|
435
440
|
description?: string | undefined;
|
|
441
|
+
tags?: string[] | undefined;
|
|
436
442
|
at?: {
|
|
437
443
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
438
444
|
step?: string | undefined;
|
|
@@ -489,6 +495,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
489
495
|
height?: number | undefined;
|
|
490
496
|
durationMs?: number | undefined;
|
|
491
497
|
description?: string | undefined;
|
|
498
|
+
tags?: string[] | undefined;
|
|
492
499
|
at?: {
|
|
493
500
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
494
501
|
step?: string | undefined;
|
|
@@ -555,6 +562,7 @@ export declare function createReportsApp(deps: ReportsAppDeps): import("hono/hon
|
|
|
555
562
|
height?: number | undefined;
|
|
556
563
|
durationMs?: number | undefined;
|
|
557
564
|
description?: string | undefined;
|
|
565
|
+
tags?: string[] | undefined;
|
|
558
566
|
at?: {
|
|
559
567
|
[x: string]: import("hono/utils/types").JSONValue;
|
|
560
568
|
step?: string | undefined;
|
|
@@ -1,25 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @module Test catalog, history and runs
|
|
3
|
-
*
|
|
4
|
-
* What a person or an agent asks of a report once it holds more than a handful of checks: every test with where it ran,
|
|
5
|
-
* how it went over time and how long it takes; the specs, domains, environments and producers they fall under; the CI
|
|
6
|
-
* runs that produced them and which tests each one ran.
|
|
7
|
-
*
|
|
8
|
-
* - `TestHistory` keeps every recorded result of every test (bounded per test) and every run's results, folded in one
|
|
9
|
-
* layer at a time, in any order, with the same outcome. It never decides a status: the current status of a test is
|
|
10
|
-
* always the report's winner (`combineEvidence`), and history only describes how it got there.
|
|
11
|
-
* - `testCatalog` turns the report's winners (plus a history, when there is one) into a `TestCatalog`: a summary with
|
|
12
|
-
* insights (slowest, flaky, failing), facets to filter by, and the tests that match a `TestQuery`.
|
|
13
|
-
* - `testHistory`, `runList` and `runDetail` answer for one test, every run, and one run.
|
|
14
|
-
* - `catalogMarkdown`, `historyMarkdown`, `runsMarkdown` and `runMarkdown` say the same in Markdown, for an agent or a
|
|
15
|
-
* terminal.
|
|
16
|
-
*
|
|
17
|
-
* A test is a check identity (its spec path and id) regardless of producer, environment and parameters; each of those
|
|
18
|
-
* combinations is one of its results. A result that is not `passed` is never counted as a pass, and a test with no
|
|
19
|
-
* completed result has no pass rate.
|
|
20
|
-
*
|
|
21
|
-
* @see docs/server.md#reading-the-report-as-an-agent
|
|
22
|
-
*/
|
|
23
1
|
import type { EvidenceStatus, SpecEntry } from './explorer-types.js';
|
|
24
2
|
import type { EvidenceSummary, LayerEvidence, LayerHeader, LayerRow, StatusCounts } from './evidence-types.js';
|
|
25
3
|
import { RecordedSpecs } from './recorded-specs.js';
|
|
@@ -116,6 +94,10 @@ export type TestSummary = {
|
|
|
116
94
|
readonly outcomes: readonly TestOutcome[];
|
|
117
95
|
readonly stats: TestStats;
|
|
118
96
|
readonly tags: readonly string[];
|
|
97
|
+
/** The normalized tags of the attachments of its current results (`demo`, shown as `#demo`). Never inherited from a check or run; the `tags` above are. Absent in documents of older servers. */
|
|
98
|
+
readonly artifactTags?: readonly string[];
|
|
99
|
+
/** The normalized tags its attachments inherit from their check and run (those of the results that have attachments), apart from `artifactTags`. What an inherited artifact query also matches. */
|
|
100
|
+
readonly inheritedArtifactTags?: readonly string[];
|
|
119
101
|
readonly labels: Readonly<Record<string, string>>;
|
|
120
102
|
/** What its current results carry: attachments, steps, retries, and the kinds of attachment. */
|
|
121
103
|
readonly evidence: {
|
|
@@ -187,6 +169,8 @@ export type CatalogFacets = {
|
|
|
187
169
|
readonly counts: StatusCounts;
|
|
188
170
|
})[];
|
|
189
171
|
readonly tags: readonly FacetValue[];
|
|
172
|
+
/** The tags of attachments, normalized. Absent in documents of older servers. */
|
|
173
|
+
readonly artifactTags?: readonly FacetValue[];
|
|
190
174
|
readonly branches: readonly FacetValue[];
|
|
191
175
|
readonly labels: Readonly<Record<string, readonly FacetValue[]>>;
|
|
192
176
|
};
|
|
@@ -206,6 +190,10 @@ export type TestQuery = {
|
|
|
206
190
|
/** Spec paths (`specs/...`). */
|
|
207
191
|
readonly spec?: readonly string[];
|
|
208
192
|
readonly tag?: readonly string[];
|
|
193
|
+
/** Only tests with an attachment carrying this tag (`demo` and `#demo` are the same), not counting tags inherited from a check or run: the exact artifact query. `tag` is the inherited one. */
|
|
194
|
+
readonly artifactTag?: readonly string[];
|
|
195
|
+
/** With `artifactTag`: also match the tags an attachment inherits from its check and run (a recording without a tag of its own under a run tagged `demo`). Default false: the exact query. */
|
|
196
|
+
readonly inherited?: boolean;
|
|
209
197
|
/** Only tests recorded in this run (a key from `runList`). */
|
|
210
198
|
readonly run?: string;
|
|
211
199
|
readonly sort?: TestSort;
|
|
@@ -234,10 +222,19 @@ export type TestHistoryDocument = {
|
|
|
234
222
|
readonly test: TestSummary;
|
|
235
223
|
readonly history: readonly TestResultEntry[];
|
|
236
224
|
};
|
|
237
|
-
/**
|
|
225
|
+
/**
|
|
226
|
+
* A run: the layers that name the same execution (`run.id`), else share a CI run URL, else a branch and commit. When,
|
|
227
|
+
* where, and how its tests went.
|
|
228
|
+
*/
|
|
238
229
|
export type RunSummary = {
|
|
239
|
-
/** Stable key: the run URL,
|
|
230
|
+
/** Stable key: `run:<id>` for an execution a producer named, else the run URL, else `branch@sha`. */
|
|
240
231
|
readonly key: string;
|
|
232
|
+
/** The execution id a producer named for it, when it did. Layers that share it are one run, however many uploads they took; another invocation at the same commit has another. */
|
|
233
|
+
readonly runId?: string;
|
|
234
|
+
/** The human title the producer gave the execution (the newest layer's). Presentation only: the key, the commit and `freshness` are unaffected. */
|
|
235
|
+
readonly title?: string;
|
|
236
|
+
/** `modified` when any of its layers was recorded from a checkout with uncommitted changes, else `unknown` when any origin was not captured; absent when every layer is from a clean commit. Advisory; never changes a status. */
|
|
237
|
+
readonly freshness?: 'unknown' | 'modified';
|
|
241
238
|
readonly runUrl: string | null;
|
|
242
239
|
readonly branch: string | null;
|
|
243
240
|
readonly sha: string | null;
|
|
@@ -290,8 +287,15 @@ export type RunDetail = {
|
|
|
290
287
|
readonly run: RunSummary;
|
|
291
288
|
readonly results: readonly RunResult[];
|
|
292
289
|
};
|
|
293
|
-
/**
|
|
294
|
-
|
|
290
|
+
/**
|
|
291
|
+
* The run a layer belongs to: the execution its producer named (`run:<id>`), else its run URL, else `branch@sha`, else
|
|
292
|
+
* `local`. Layers with the same key are one run. Only grouping: it is never part of the slot a result fills.
|
|
293
|
+
*/
|
|
294
|
+
export declare function runKey(layer: Pick<LayerHeader, 'runUrl' | 'branch' | 'sha'> & {
|
|
295
|
+
readonly run?: {
|
|
296
|
+
readonly id: string;
|
|
297
|
+
};
|
|
298
|
+
}): string;
|
|
295
299
|
/**
|
|
296
300
|
* Every recorded result of every test (the newest `HISTORY_PER_TEST` per test, leaving out `running` progress) and the
|
|
297
301
|
* runs they came from. A run's results are read from that history, so memory follows the number of tests and not the age
|
|
@@ -380,7 +384,7 @@ export declare function testHistory(model: CatalogModel, key: string): TestHisto
|
|
|
380
384
|
export declare function runList(history: TestHistory, version: string, limit?: number): RunList;
|
|
381
385
|
/** One run with every result it recorded, or null when no run has that key. */
|
|
382
386
|
export declare function runDetail(history: TestHistory, version: string, key: string): RunDetail | null;
|
|
383
|
-
/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`. */
|
|
387
|
+
/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`, `artifactTag`, and `inherited=1`. */
|
|
384
388
|
export declare function parseTestQuery(params: URLSearchParams): TestQuery;
|
|
385
389
|
/** A duration for a person: `850 ms`, `12.3 s`, `4m 05s`. */
|
|
386
390
|
export declare function formatMs(ms: number | null | undefined): string;
|
|
@@ -1,3 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module Test catalog, history and runs
|
|
3
|
+
*
|
|
4
|
+
* What a person or an agent asks of a report once it holds more than a handful of checks: every test with where it ran,
|
|
5
|
+
* how it went over time and how long it takes; the specs, domains, environments and producers they fall under; the CI
|
|
6
|
+
* runs that produced them and which tests each one ran.
|
|
7
|
+
*
|
|
8
|
+
* - `TestHistory` keeps every recorded result of every test (bounded per test) and every run's results, folded in one
|
|
9
|
+
* layer at a time, in any order, with the same outcome. It never decides a status: the current status of a test is
|
|
10
|
+
* always the report's winner (`combineEvidence`), and history only describes how it got there.
|
|
11
|
+
* - `testCatalog` turns the report's winners (plus a history, when there is one) into a `TestCatalog`: a summary with
|
|
12
|
+
* insights (slowest, flaky, failing), facets to filter by, and the tests that match a `TestQuery`.
|
|
13
|
+
* - `testHistory`, `runList` and `runDetail` answer for one test, every run, and one run.
|
|
14
|
+
* - `catalogMarkdown`, `historyMarkdown`, `runsMarkdown` and `runMarkdown` say the same in Markdown, for an agent or a
|
|
15
|
+
* terminal.
|
|
16
|
+
*
|
|
17
|
+
* A test is a check identity (its spec path and id) regardless of producer, environment and parameters; each of those
|
|
18
|
+
* combinations is one of its results. A result that is not `passed` is never counted as a pass, and a test with no
|
|
19
|
+
* completed result has no pass rate.
|
|
20
|
+
*
|
|
21
|
+
* @see docs/server.md#reading-the-report-as-an-agent
|
|
22
|
+
*/
|
|
23
|
+
import { displayTag, normalizeTag, normalizeTags } from './artifact-tags.js';
|
|
1
24
|
import { checkSlots } from './evidence-view.js';
|
|
2
25
|
import { combineEvidence, emptyCounts, slotKey, specPath as normalSpecPath } from './layer-merge.js';
|
|
3
26
|
import { RecordedSpecs } from './recorded-specs.js';
|
|
@@ -8,8 +31,13 @@ export const HISTORY_PER_TEST = 100;
|
|
|
8
31
|
const FAILURE_RANK = { passed: 0, 'not-run': 1, running: 2, failed: 3 };
|
|
9
32
|
/** How tests are ordered. */
|
|
10
33
|
export const TEST_SORTS = ['spec', 'title', 'recent', 'slowest', 'failures', 'flaky', 'runs'];
|
|
11
|
-
/**
|
|
34
|
+
/**
|
|
35
|
+
* The run a layer belongs to: the execution its producer named (`run:<id>`), else its run URL, else `branch@sha`, else
|
|
36
|
+
* `local`. Layers with the same key are one run. Only grouping: it is never part of the slot a result fills.
|
|
37
|
+
*/
|
|
12
38
|
export function runKey(layer) {
|
|
39
|
+
if (layer.run?.id)
|
|
40
|
+
return `run:${layer.run.id}`;
|
|
13
41
|
if (layer.runUrl)
|
|
14
42
|
return layer.runUrl;
|
|
15
43
|
if (layer.branch || layer.sha)
|
|
@@ -98,17 +126,29 @@ export class TestHistory {
|
|
|
98
126
|
const key = runKey(layer);
|
|
99
127
|
let run = this.#runs.get(key);
|
|
100
128
|
if (!run) {
|
|
101
|
-
run = { key, runUrl: layer.runUrl, branch: layer.branch, sha: layer.sha, start: layer.recordedAtMs, end: layer.recordedAtMs, first: layer.id, producers: new Set(), environments: new Set(), layers: 0 };
|
|
129
|
+
run = { key, runUrl: layer.runUrl, branch: layer.branch, sha: layer.sha, start: layer.recordedAtMs, end: layer.recordedAtMs, firstAt: layer.recordedAtMs, first: layer.id, producers: new Set(), environments: new Set(), layers: 0, runId: layer.run?.id, title: undefined, titledAt: 0, titledBy: '', freshness: undefined };
|
|
102
130
|
this.#runs.set(key, run);
|
|
103
131
|
}
|
|
104
132
|
// A run is told by its first layer (by time, then id), whatever order its layers arrive in.
|
|
105
|
-
if (layer.recordedAtMs < run.
|
|
106
|
-
run.
|
|
133
|
+
if (layer.recordedAtMs < run.firstAt || (layer.recordedAtMs === run.firstAt && layer.id < run.first)) {
|
|
134
|
+
run.firstAt = layer.recordedAtMs;
|
|
107
135
|
run.first = layer.id;
|
|
108
136
|
run.branch = layer.branch;
|
|
109
137
|
run.sha = layer.sha;
|
|
138
|
+
run.runUrl = layer.runUrl;
|
|
139
|
+
}
|
|
140
|
+
run.start = Math.min(run.start, layer.recordedAtMs, layer.run?.startedAtMs ?? Infinity);
|
|
141
|
+
run.end = Math.max(run.end, layer.recordedAtMs, layer.run?.endedAtMs ?? -Infinity);
|
|
142
|
+
// The newest title wins, so a person can rename a run by sending its id again with another title.
|
|
143
|
+
const title = layer.run?.title;
|
|
144
|
+
if (title && (layer.recordedAtMs > run.titledAt || (layer.recordedAtMs === run.titledAt && layer.id > run.titledBy))) {
|
|
145
|
+
run.title = title;
|
|
146
|
+
run.titledAt = layer.recordedAtMs;
|
|
147
|
+
run.titledBy = layer.id;
|
|
110
148
|
}
|
|
111
|
-
|
|
149
|
+
// Local changes are never lost to a clean sibling: the run is as modified as its most modified layer.
|
|
150
|
+
if (layer.freshness === 'modified' || (layer.freshness === 'unknown' && run.freshness !== 'modified'))
|
|
151
|
+
run.freshness = layer.freshness;
|
|
112
152
|
run.producers.add(layer.producer);
|
|
113
153
|
if (layer.environment)
|
|
114
154
|
run.environments.add(layer.environment);
|
|
@@ -227,7 +267,8 @@ function runSummary(run, cells) {
|
|
|
227
267
|
failed.sort((a, b) => compareText(a.i.slot, b.i.slot) || compareText(a.r.o.place, b.r.o.place) || compareText(a.r.k, b.r.k));
|
|
228
268
|
const failing = failed.slice(0, 20).map(cell => ({ key: cell.i.slot, title: cell.i.title, environment: cell.r.o.environment, detail: (cell.r.t.split('\n')[0] ?? '').slice(0, 500) }));
|
|
229
269
|
return {
|
|
230
|
-
key: run.key,
|
|
270
|
+
key: run.key, ...(run.runId ? { runId: run.runId } : {}), ...(run.title ? { title: run.title } : {}), ...(run.freshness ? { freshness: run.freshness } : {}),
|
|
271
|
+
runUrl: run.runUrl, branch: run.branch, sha: run.sha,
|
|
231
272
|
startedAtMs: run.start, endedAtMs: run.end, spanMs: run.end - run.start,
|
|
232
273
|
producers: [...run.producers].sort(), environments: [...run.environments].sort(),
|
|
233
274
|
layers: run.layers, tests: tests.size, counts, totalDurationMs: measured ? total : null, failing,
|
|
@@ -319,15 +360,22 @@ function outcomeOf(result) {
|
|
|
319
360
|
function summaryOf(slot, history, known) {
|
|
320
361
|
const outcomes = slot.outcomes.map(outcomeOf);
|
|
321
362
|
const tags = new Set();
|
|
363
|
+
const artifactTags = new Set();
|
|
364
|
+
const inheritedTags = new Set();
|
|
322
365
|
const labels = {};
|
|
323
366
|
let attachments = 0, steps = 0, attempts = 0;
|
|
324
367
|
const roles = new Set();
|
|
325
|
-
for (const outcome of outcomes) {
|
|
368
|
+
for (const [at, outcome] of outcomes.entries()) {
|
|
326
369
|
const evidence = outcome.evidence;
|
|
327
370
|
if (!evidence)
|
|
328
371
|
continue;
|
|
372
|
+
if (evidence.attachments > 0)
|
|
373
|
+
for (const tag of normalizeTags([...evidence.tags, ...(slot.outcomes[at]?.layer.evidence?.tags ?? [])]))
|
|
374
|
+
inheritedTags.add(tag);
|
|
329
375
|
for (const tag of evidence.tags)
|
|
330
376
|
tags.add(tag);
|
|
377
|
+
for (const tag of evidence.artifactTags ?? [])
|
|
378
|
+
artifactTags.add(tag);
|
|
331
379
|
Object.assign(labels, evidence.labels);
|
|
332
380
|
attachments += evidence.attachments;
|
|
333
381
|
steps += evidence.steps;
|
|
@@ -348,6 +396,8 @@ function summaryOf(slot, history, known) {
|
|
|
348
396
|
outcomes,
|
|
349
397
|
stats: statsOf(history?.of(slot.key) ?? [], outcomes, tagList, labels),
|
|
350
398
|
tags: tagList,
|
|
399
|
+
...(artifactTags.size ? { artifactTags: [...artifactTags].sort() } : {}),
|
|
400
|
+
...([...inheritedTags].some(tag => !artifactTags.has(tag)) ? { inheritedArtifactTags: [...inheritedTags].filter(tag => !artifactTags.has(tag)).sort() } : {}),
|
|
351
401
|
labels,
|
|
352
402
|
evidence: { attachments, steps, attempts, roles: [...roles].sort() },
|
|
353
403
|
lastRecordedAtMs: slot.latestAtMs,
|
|
@@ -357,7 +407,7 @@ function summaryOf(slot, history, known) {
|
|
|
357
407
|
/** The lowercase text a search matches a test against: title, id, spec, domain, tags, labels, environments and producers. */
|
|
358
408
|
export function haystackOf(test) {
|
|
359
409
|
return [
|
|
360
|
-
test.title, test.id, test.specPath ?? '', test.spec?.title ?? '', test.spec?.domain ?? '', ...test.tags,
|
|
410
|
+
test.title, test.id, test.specPath ?? '', test.spec?.title ?? '', test.spec?.domain ?? '', ...test.tags, ...(test.artifactTags ?? []).map(displayTag),
|
|
361
411
|
...Object.entries(test.labels).flatMap(([key, value]) => [key, value]), ...test.environments, ...test.producers,
|
|
362
412
|
].join('\n').toLowerCase();
|
|
363
413
|
}
|
|
@@ -463,6 +513,7 @@ export function catalogModel(layers, sources = {}) {
|
|
|
463
513
|
domains: facet(tests, test => [test.spec?.domain || '(no spec)']),
|
|
464
514
|
specs: [...specGroups].map(([, list]) => { const g = group(list[0].spec.path, list); return { ...list[0].spec, tests: g.tests, counts: g.counts }; }).sort((a, b) => compareText(a.domain, b.domain) || compareText(a.title, b.title)),
|
|
465
515
|
tags: facet(tests, test => test.tags),
|
|
516
|
+
artifactTags: facet(tests, test => test.artifactTags ?? []),
|
|
466
517
|
branches: facet(tests, test => test.stats.branches),
|
|
467
518
|
labels: Object.fromEntries([...labelKeys].sort().map(key => [key, facet(tests, test => (test.labels[key] === undefined ? [] : [test.labels[key]]))])),
|
|
468
519
|
};
|
|
@@ -497,6 +548,8 @@ export function matchesQuery(model, test, query, inRun) {
|
|
|
497
548
|
return false;
|
|
498
549
|
if (query.tag?.length && !query.tag.some(tag => test.tags.includes(tag)))
|
|
499
550
|
return false;
|
|
551
|
+
if (query.artifactTag?.length && !query.artifactTag.some(tag => { const wanted = normalizeTag(tag); return wanted !== null && ((test.artifactTags?.includes(wanted) ?? false) || (query.inherited === true && (test.inheritedArtifactTags?.includes(wanted) ?? false))); }))
|
|
552
|
+
return false;
|
|
500
553
|
if (inRun && !inRun.has(test.key))
|
|
501
554
|
return false;
|
|
502
555
|
return true;
|
|
@@ -528,7 +581,7 @@ export function runDetail(history, version, key) {
|
|
|
528
581
|
const found = history.run(key);
|
|
529
582
|
return found ? { schemaVersion: TEST_CATALOG_VERSION, version, ...found } : null;
|
|
530
583
|
}
|
|
531
|
-
/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`. */
|
|
584
|
+
/** Reads a `TestQuery` from URL search parameters: `q`, `sort`, `offset`, `limit`, `run`, and repeatable or comma-separated `status`, `environment`, `producer`, `domain`, `spec`, `tag`, `artifactTag`, and `inherited=1`. */
|
|
532
585
|
export function parseTestQuery(params) {
|
|
533
586
|
const many = (name) => params.getAll(name).flatMap(value => value.split(',')).map(value => value.trim()).filter(Boolean);
|
|
534
587
|
const number = (name) => { const value = params.get(name); return value !== null && /^\d+$/.test(value) ? Number(value) : undefined; };
|
|
@@ -537,11 +590,13 @@ export function parseTestQuery(params) {
|
|
|
537
590
|
const q = params.get('q');
|
|
538
591
|
if (q)
|
|
539
592
|
query.q = q;
|
|
540
|
-
for (const name of ['status', 'environment', 'producer', 'domain', 'spec', 'tag']) {
|
|
593
|
+
for (const name of ['status', 'environment', 'producer', 'domain', 'spec', 'tag', 'artifactTag']) {
|
|
541
594
|
const values = many(name);
|
|
542
595
|
if (values.length)
|
|
543
596
|
query[name] = values;
|
|
544
597
|
}
|
|
598
|
+
if (['1', 'true'].includes(params.get('inherited') ?? ''))
|
|
599
|
+
query.inherited = true;
|
|
545
600
|
const run = params.get('run');
|
|
546
601
|
if (run)
|
|
547
602
|
query.run = run;
|
|
@@ -615,6 +670,7 @@ export function historyMarkdown(document) {
|
|
|
615
670
|
`- Results: ${test.stats.results} recorded, ${test.stats.passed} passed, ${test.stats.failed} failed${test.stats.passRate === null ? '' : `, pass rate ${Math.round(test.stats.passRate * 100)}%`}, in ${test.stats.runs} runs`,
|
|
616
671
|
`- Duration: ${d ? `avg ${formatMs(d.avgMs)}, p50 ${formatMs(d.p50Ms)}, p95 ${formatMs(d.p95Ms)}, min ${formatMs(d.minMs)}, max ${formatMs(d.maxMs)}, last ${formatMs(d.lastMs)}` : 'not measured'}`,
|
|
617
672
|
...(test.tags.length ? [`- Tags: ${test.tags.join(', ')}`] : []),
|
|
673
|
+
...(test.artifactTags?.length ? [`- Artifact tags: ${test.artifactTags.map(displayTag).join(', ')}`] : []),
|
|
618
674
|
...(Object.keys(test.labels).length ? [`- Labels: ${Object.entries(test.labels).map(([k, v]) => `${k}=${v}`).join(', ')}`] : []),
|
|
619
675
|
`- Evidence: ${test.evidence.attachments} attachments${test.evidence.roles.length ? ` (${test.evidence.roles.join(', ')})` : ''}, ${test.evidence.steps} steps`,
|
|
620
676
|
'',
|
|
@@ -631,6 +687,10 @@ export function historyMarkdown(document) {
|
|
|
631
687
|
...document.history.map(e => `| ${iso(e.at)} | ${e.environment ?? '—'} | ${e.status} | ${formatMs(e.durationMs)} | ${cell(e.branch ?? '—')} | ${e.sha?.slice(0, 10) ?? '—'} | ${cell(e.detail.split('\n')[0] ?? '')} |`),
|
|
632
688
|
].join('\n')}\n`;
|
|
633
689
|
}
|
|
690
|
+
/** What to say beside a commit that was not recorded from a clean checkout: nothing, ` (modified)` or ` (origin unknown)`. */
|
|
691
|
+
function freshnessNote(freshness) {
|
|
692
|
+
return freshness === 'modified' ? ' (modified: local changes)' : freshness === 'unknown' ? ' (origin unknown)' : '';
|
|
693
|
+
}
|
|
634
694
|
/** The run list as Markdown. */
|
|
635
695
|
export function runsMarkdown(list) {
|
|
636
696
|
return `${[
|
|
@@ -638,7 +698,7 @@ export function runsMarkdown(list) {
|
|
|
638
698
|
'',
|
|
639
699
|
'| Ended | Branch | Commit | Environments | Tests | Passed | Failed | Not run | Span | Run |',
|
|
640
700
|
'|---|---|---|---|---:|---:|---:|---:|---:|---|',
|
|
641
|
-
...list.runs.map(r => `| ${iso(r.endedAtMs)} | ${cell(r.branch ?? '—')} | ${r.sha?.slice(0, 10) ?? '—'} | ${r.environments.join(', ') || '—'} | ${r.tests} | ${r.counts.passed} | ${r.counts.failed} | ${r.counts['not-run']} | ${formatMs(r.spanMs)} | ${cell(r.key)} |`),
|
|
701
|
+
...list.runs.map(r => `| ${iso(r.endedAtMs)} | ${cell(r.branch ?? '—')} | ${r.sha?.slice(0, 10) ?? '—'}${freshnessNote(r.freshness)} | ${r.environments.join(', ') || '—'} | ${r.tests} | ${r.counts.passed} | ${r.counts.failed} | ${r.counts['not-run']} | ${formatMs(r.spanMs)} | ${cell(r.title ? `${r.title} (${r.key})` : r.key)} |`),
|
|
642
702
|
].join('\n')}\n`;
|
|
643
703
|
}
|
|
644
704
|
/** One run as Markdown. */
|
|
@@ -647,7 +707,8 @@ export function runMarkdown(detail) {
|
|
|
647
707
|
return `${[
|
|
648
708
|
`# Run ${run.key}`,
|
|
649
709
|
'',
|
|
650
|
-
|
|
710
|
+
...(run.title ? [`- Title: ${run.title}`] : []),
|
|
711
|
+
`- Branch ${run.branch ?? '—'}, commit ${run.sha ?? '—'}${freshnessNote(run.freshness)}`,
|
|
651
712
|
`- ${iso(run.startedAtMs)} to ${iso(run.endedAtMs)} (${formatMs(run.spanMs)}); ${run.layers} layers from ${run.producers.join(', ')}`,
|
|
652
713
|
`- Environments: ${run.environments.join(', ') || '—'}`,
|
|
653
714
|
`- ${run.tests} tests: ${counted(run.counts)}${run.totalDurationMs === null ? '' : `; ${formatMs(run.totalDurationMs)} of test time`}`,
|