@crvouga/atlas 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/playwright-reporter/src/index.d.ts +11 -0
- package/dist/playwright-reporter/src/index.js +4 -2
- package/dist/reports/src/evidence-types.d.ts +3 -0
- package/dist/reports/src/layer-merge.d.ts +1 -1
- package/dist/reports/src/layer-merge.js +3 -2
- package/dist/reports/src/layer-types.d.ts +3 -0
- package/dist/reports/src/publish-checks.js +1 -0
- package/dist/reports/src/report-cli.d.ts +1 -1
- package/dist/reports/src/report-cli.js +30 -2
- package/dist/reports/src/reporter.d.ts +1 -1
- package/dist/reports/src/test-catalog.d.ts +20 -4
- package/dist/reports/src/test-catalog.js +31 -8
- package/dist/schema/src/api.d.ts +49 -1
- package/dist/schema/src/api.js +7 -3
- package/dist/schema/src/layer.d.ts +26 -0
- package/dist/schema/src/layer.js +17 -0
- package/docs/integrating.md +8 -0
- package/docs/reference/cli.md +7 -0
- package/docs/reference/playwright-reporter.md +6 -6
- package/docs/reference/reports.md +18 -6
- package/docs/reference/schema.md +20 -1
- package/docs/report-contract.md +1 -0
- package/docs/ui.md +1 -1
- package/llms-full.txt +17 -1
- package/package.json +1 -1
- package/packages/schema/schemas/effective-delta.schema.json +30 -0
- package/packages/schema/schemas/effective-evidence.schema.json +30 -0
- package/packages/schema/schemas/evidence-page.schema.json +30 -0
- package/packages/schema/schemas/layer.schema.json +30 -0
- package/packages/schema/schemas/publish-checks.schema.json +30 -0
- package/packages/schema/schemas/report-run.schema.json +17 -1
- package/packages/schema/schemas/report-runs.schema.json +17 -1
- package/packages/server/openapi.json +154 -2
- package/packages/visualizer/src/evidence/check-detail.tsx +10 -4
- package/packages/visualizer/src/evidence/recording-page.tsx +5 -2
- package/packages/visualizer/src/explorer/Explorer.tsx +6 -6
- package/packages/visualizer/src/explorer/RunsView.tsx +25 -6
- package/packages/visualizer/src/explorer/TestDetail.tsx +142 -14
- package/packages/visualizer/src/explorer/parts.tsx +2 -1
- package/packages/visualizer/src/explorer/pinned.ts +91 -0
- package/packages/visualizer/src/explorer/search.ts +5 -2
package/dist/schema/src/layer.js
CHANGED
|
@@ -50,6 +50,18 @@ export const CheckStatusSchema = z.enum(CHECK_STATUSES);
|
|
|
50
50
|
* changes only with a new `schemaVersion`.
|
|
51
51
|
*/
|
|
52
52
|
export const FRESHNESS = ['unknown', 'modified'];
|
|
53
|
+
/**
|
|
54
|
+
* The execution a layer belongs to, named by the producer: one invocation of a local or manual run, which several
|
|
55
|
+
* layers (shards, batches, producers) may share. Presentation and history grouping only: it never takes part in the
|
|
56
|
+
* slot a result fills, in which result wins, or in a status. Absent, a run is told by its CI run URL, else `branch@sha`.
|
|
57
|
+
* Within one project, layers that carry the same `id` are one run; a later invocation gets another `id`.
|
|
58
|
+
*/
|
|
59
|
+
export const LayerRunSchema = z.looseObject({
|
|
60
|
+
id: z.string().min(1).max(256).describe('The execution\'s identity within a project, chosen by the producer: a UUID, a timestamped name. Layers with the same id are one run; it never changes after it is recorded'),
|
|
61
|
+
title: z.string().optional().describe('A human name for the run. Presentation only: it is not part of the identity and does not hide the commit or `freshness`'),
|
|
62
|
+
startedAtMs: z.number().nonnegative().optional().describe('When the execution started, epoch milliseconds'),
|
|
63
|
+
endedAtMs: z.number().nonnegative().optional().describe('When the execution ended, epoch milliseconds')
|
|
64
|
+
});
|
|
53
65
|
/**
|
|
54
66
|
* A number a check measured, such as a latency or a size, with its unit and an optional allowance. A measurement over
|
|
55
67
|
* its `limit` is reported and never changes the check's status.
|
|
@@ -250,6 +262,7 @@ export const LayerDocumentSchema = z.looseObject({
|
|
|
250
262
|
environment: z.string().nullable().optional().describe('The client that was checked: desktop-web, mobile-web, ios, android, or any other lowercase name'),
|
|
251
263
|
parameters: z.record(z.string(), z.string()).default({}).describe('Other choices that make a result its own slot, such as a device size'),
|
|
252
264
|
freshness: z.enum(FRESHNESS).optional().describe('`modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured'),
|
|
265
|
+
run: LayerRunSchema.optional().describe('The execution this layer belongs to, when the producer names one (a local or manual run without a CI run URL). Grouping and display only; it never partitions a report or decides a result'),
|
|
253
266
|
specs: z.array(SpecEntrySchema).default([]),
|
|
254
267
|
parts: z.array(ReportPartSchema),
|
|
255
268
|
source: LayerSourceSchema.nullable().default(null),
|
|
@@ -298,6 +311,9 @@ function lenientEvidence(holder, nested = []) {
|
|
|
298
311
|
}
|
|
299
312
|
function lenientLayer(raw) {
|
|
300
313
|
const layer = lenientEvidence(raw);
|
|
314
|
+
// The execution only groups and labels: one that cannot be read leaves the layer to the documented grouping, never refuses it.
|
|
315
|
+
if ('run' in layer && !LayerRunSchema.safeParse(layer.run).success)
|
|
316
|
+
delete layer.run;
|
|
301
317
|
if (Array.isArray(layer.parts)) {
|
|
302
318
|
layer.parts = layer.parts.map((part) => {
|
|
303
319
|
const found = object(part);
|
|
@@ -384,6 +400,7 @@ export const PublishChecksSchema = z.looseObject({
|
|
|
384
400
|
environment: z.string().nullable().optional(),
|
|
385
401
|
parameters: z.record(z.string(), z.string()).default({}),
|
|
386
402
|
freshness: z.enum(FRESHNESS).optional(),
|
|
403
|
+
run: LayerRunSchema.optional().describe('Name the execution these checks belong to, so repeated local runs at one commit stay separate runs and shards of one execution stay together'),
|
|
387
404
|
specs: z.array(SpecEntrySchema).default([]),
|
|
388
405
|
tags: evidenceShape.tags,
|
|
389
406
|
labels: evidenceShape.labels,
|
package/docs/integrating.md
CHANGED
|
@@ -95,6 +95,14 @@ first commit, they are `null`: that is provenance missing, not an error, and nev
|
|
|
95
95
|
result wins. `--param key=value` (repeatable) adds a parameter that makes a result its own slot, such
|
|
96
96
|
as `--param device=iphone-15`.
|
|
97
97
|
|
|
98
|
+
Without a CI run URL, layers recorded at one branch and commit group into one run, so repeating a local demo
|
|
99
|
+
looks like one execution. To keep invocations apart, name each one: `--run-id <id>` (or `ATLAS_RUN_ID`), with
|
|
100
|
+
`--run-title <title>` (or `ATLAS_RUN_TITLE`) for a human name and `--run-started-at` / `--run-ended-at` for when
|
|
101
|
+
it ran. Every shard or batch of one invocation passes the same id and is one run; the next invocation passes a new
|
|
102
|
+
one. A run recorded from a checkout with uncommitted changes keeps saying so beside its commit. The id only
|
|
103
|
+
groups: it is not part of which result a check fills or which wins, and a producer that never passes it keeps
|
|
104
|
+
the CI-run-URL, then `branch@sha`, grouping.
|
|
105
|
+
|
|
98
106
|
The time the checks ran comes from the report (CTRF `summary.stop`, a JUnit `timestamp`, a manifest's
|
|
99
107
|
`finishedAt`), or failing that from the report file's modification time, in milliseconds. Publish in
|
|
100
108
|
the same job that wrote the report and that is the time it ran. If you copy the report first, pass
|
package/docs/reference/cli.md
CHANGED
|
@@ -86,12 +86,14 @@ atlas report — layered evidence for any kind of spec
|
|
|
86
86
|
atlas report publish [--store <dir|s3://bucket/prefix>] [--producer <name>]
|
|
87
87
|
( --ctrf <file> | --junit <file> | --manifest <file> | --checks <file.json> )
|
|
88
88
|
[--environment <name>] [--branch <b>] [--sha <s>] [--run-url <u>]
|
|
89
|
+
[--run-id <id>] [--run-title <t>] [--run-started-at <iso|ms>] [--run-ended-at <iso|ms>]
|
|
89
90
|
[--spec <path> | --specs <dir>] [--upload-run]
|
|
90
91
|
[--param key=value]... [--kind <k>] [--title <t>] [--recorded-at <iso|ms>]
|
|
91
92
|
[--attach [role=]<file>]... [--label key=value]... [--link [name=]<url>]...
|
|
92
93
|
[--attach-root <dir>]... [--max-attachment-mb <n>] [--max-total-attachment-mb <n>]
|
|
93
94
|
atlas report stream [--store <dir|s3://…|https://atlas.example.com>] --producer <name> [--environment <name>]
|
|
94
95
|
[--branch <b>] [--sha <s>] [--run-url <u>] [--kind <k>] [--param key=value]...
|
|
96
|
+
[--run-id <id>] [--run-title <t>]
|
|
95
97
|
[--flush-ms 1000] [--batch 100] (checks as JSON lines on standard input)
|
|
96
98
|
atlas report status [--store <dir|s3://…|id=url>]... [--backends <module>]
|
|
97
99
|
[--specs <dir>] [--format text|json|markdown]
|
|
@@ -112,6 +114,11 @@ atlas report — layered evidence for any kind of spec
|
|
|
112
114
|
--specs <dir> link checks to the spec files in <dir> by path, or by the suite, class or
|
|
113
115
|
file name the report gives; also publishes the specs themselves, so the
|
|
114
116
|
node can show them without a checkout
|
|
117
|
+
--run-id <id> name this execution (a local or manual run with no CI run URL; also ATLAS_RUN_ID).
|
|
118
|
+
Layers that share an id are one run, however many shards or batches uploaded them;
|
|
119
|
+
another invocation at the same commit gets another id. Never part of what a check is
|
|
120
|
+
--run-title <t> a human name for the run (also ATLAS_RUN_TITLE); --run-started-at and --run-ended-at
|
|
121
|
+
record when it ran. Without --run-id a run is its CI run URL, else branch@sha
|
|
115
122
|
--upload-run with --manifest: also upload the run's screenshots and clips for the viewer
|
|
116
123
|
--attach <file> attach a file to the whole layer: a CI log, a coverage report, a trace archive
|
|
117
124
|
(screenshots, videos and traces of a single test go in its CTRF or checks file)
|
|
@@ -281,6 +281,12 @@ export type AtlasPlaywrightReporterOptions = {
|
|
|
281
281
|
readonly branch?: string;
|
|
282
282
|
/** Where to read the CI run. Default the GitHub Actions run (or the CI's own run URL variable). */
|
|
283
283
|
readonly runUrl?: string;
|
|
284
|
+
/**
|
|
285
|
+
* Names this execution, for a local or manual run with no CI run URL: every shard and project that gets the same `id`
|
|
286
|
+
* is one run, and the next invocation gets another. Default: `ATLAS_RUN_ID` (and `ATLAS_RUN_TITLE`) when set. Only
|
|
287
|
+
* groups and labels runs; it never changes what a check is or which result wins.
|
|
288
|
+
*/
|
|
289
|
+
readonly run?: { readonly id: string; readonly title?: string; readonly startedAtMs?: number; readonly endedAtMs?: number };
|
|
284
290
|
/** The spec a test belongs to, relative to the repository. Default: the `.feature` of a playwright-bdd test, else the test file. */
|
|
285
291
|
readonly specPath?: (test: TestCase) => string | undefined;
|
|
286
292
|
/** Paths are relative to this directory. Default the git root above Playwright's `rootDir`, else `rootDir`. */
|
|
@@ -290,12 +296,6 @@ export type AtlasPlaywrightReporterOptions = {
|
|
|
290
296
|
/** A larger file is not uploaded; the check says so in an annotation. Default 50 MiB. */
|
|
291
297
|
readonly maxAttachmentBytes?: number;
|
|
292
298
|
/** The most timeline steps recorded per result. Default 500. */
|
|
293
|
-
readonly maxSteps?: number;
|
|
294
|
-
/** Record the project's spec files (path, domain, kind, title) in every layer, so a server can group checks by spec without a published spec set. Default true. */
|
|
295
|
-
readonly specs?: boolean;
|
|
296
|
-
/** Send what has finished this often, in ms. Default 5000 (every layer repeats the spec list). */
|
|
297
|
-
readonly flushMs?: number;
|
|
298
|
-
/** Send at once when this many checks are waiting. Default 50 (their attachments are held in memory until sent). */
|
|
299
299
|
…
|
|
300
300
|
```
|
|
301
301
|
|
|
@@ -1454,6 +1454,8 @@ export type LayerHeader = {
|
|
|
1454
1454
|
readonly parameters: LayerParameters;
|
|
1455
1455
|
/** `modified`: recorded from a checkout with uncommitted changes; `unknown`: origin not captured; absent: clean. Advisory; never changes a status. */
|
|
1456
1456
|
readonly freshness?: 'unknown' | 'modified';
|
|
1457
|
+
/** The execution the producer named for it (one local or manual invocation), or absent. Grouping and display only; never part of the slot. */
|
|
1458
|
+
readonly run?: LayerRun;
|
|
1457
1459
|
/** Where this layer's recorded timelines and captured files are read from, or null when it has none. */
|
|
1458
1460
|
readonly recording: RecordingAddress | null;
|
|
1459
1461
|
/** From a spec run that has not been collected into a layer yet. */
|
|
@@ -3560,7 +3562,7 @@ Drops each mutable layer whose execution is already archived as an immutable one
|
|
|
3560
3562
|
export function layerExecutionKey(layer: LayerSummary): string
|
|
3561
3563
|
```
|
|
3562
3564
|
|
|
3563
|
-
A string equal for the mutable and the archived address of one execution: its producer, sha, time and
|
|
3565
|
+
A string equal for the mutable and the archived address of one execution: its producer, sha, time, parts and, when the producer named one, its execution id.
|
|
3564
3566
|
Object key order is ignored at every depth, including check evidence and extensions; array order is preserved.
|
|
3565
3567
|
This is an advisory execution identity, not the content digest that addresses an immutable layer in storage.
|
|
3566
3568
|
|
|
@@ -3725,6 +3727,8 @@ export type LayerSummary = LayerExtras & {
|
|
|
3725
3727
|
readonly environment?: string | null;
|
|
3726
3728
|
/** Provenance: the CI run that produced the layer, or `null`. Never part of the slot. */
|
|
3727
3729
|
readonly runUrl: string | null;
|
|
3730
|
+
/** The execution (one invocation of a local or manual run) the layer belongs to, when the producer named one. Grouping and display only; never part of the slot. Absent: the run is told by `runUrl`, else `branch@sha`. */
|
|
3731
|
+
readonly run?: LayerRun;
|
|
3728
3732
|
/** Free choices that make a result its own slot. The `environment` parameter is read as the environment (see `layerPlace`). */
|
|
3729
3733
|
readonly parameters: LayerParameters;
|
|
3730
3734
|
/** The specs the layer was recorded against, with fingerprints of their text. */
|
|
@@ -5935,7 +5939,7 @@ See docs/server.md#streaming-a-run-as-it-happens
|
|
|
5935
5939
|
#### `ReporterIdentity`
|
|
5936
5940
|
|
|
5937
5941
|
```ts
|
|
5938
|
-
export type ReporterIdentity = Pick<PublishChecksInput, 'producer' | 'kind' | 'title' | 'sha' | 'branch' | 'runUrl' | 'environment' | 'parameters' | 'freshness' | 'specs' | 'tags' | 'labels' | 'links' | 'attachments' | 'annotations' | 'extensions'>;
|
|
5942
|
+
export type ReporterIdentity = Pick<PublishChecksInput, 'producer' | 'kind' | 'title' | 'sha' | 'branch' | 'runUrl' | 'run' | 'environment' | 'parameters' | 'freshness' | 'specs' | 'tags' | 'labels' | 'links' | 'attachments' | 'annotations' | 'extensions'>;
|
|
5939
5943
|
```
|
|
5940
5944
|
|
|
5941
5945
|
What every layer of one stream shares: who is reporting, from where. Whatever `publishChecks` takes, except the checks and the time.
|
|
@@ -7685,8 +7689,14 @@ export type TestHistoryDocument = {
|
|
|
7685
7689
|
|
|
7686
7690
|
```ts
|
|
7687
7691
|
export type RunSummary = {
|
|
7688
|
-
/** Stable key: the run URL,
|
|
7692
|
+
/** Stable key: `run:<id>` for an execution a producer named, else the run URL, else `branch@sha`. */
|
|
7689
7693
|
readonly key: string;
|
|
7694
|
+
/** The execution id a producer named for it, when it did. Layers that share it are one run, however many uploads they took; another invocation at the same commit has another. */
|
|
7695
|
+
readonly runId?: string;
|
|
7696
|
+
/** The human title the producer gave the execution (the newest layer's). Presentation only: the key, the commit and `freshness` are unaffected. */
|
|
7697
|
+
readonly title?: string;
|
|
7698
|
+
/** `modified` when any of its layers was recorded from a checkout with uncommitted changes, else `unknown` when any origin was not captured; absent when every layer is from a clean commit. Advisory; never changes a status. */
|
|
7699
|
+
readonly freshness?: 'unknown' | 'modified';
|
|
7690
7700
|
readonly runUrl: string | null;
|
|
7691
7701
|
readonly branch: string | null;
|
|
7692
7702
|
readonly sha: string | null;
|
|
@@ -7708,7 +7718,8 @@ export type RunSummary = {
|
|
|
7708
7718
|
};
|
|
7709
7719
|
```
|
|
7710
7720
|
|
|
7711
|
-
A
|
|
7721
|
+
A run: the layers that name the same execution (`run.id`), else share a CI run URL, else a branch and commit. When,
|
|
7722
|
+
where, and how its tests went.
|
|
7712
7723
|
|
|
7713
7724
|
#### `RunList`
|
|
7714
7725
|
|
|
@@ -7759,10 +7770,11 @@ export type RunDetail = {
|
|
|
7759
7770
|
#### `runKey`
|
|
7760
7771
|
|
|
7761
7772
|
```ts
|
|
7762
|
-
export function runKey(layer: Pick<LayerHeader, 'runUrl' | 'branch' | 'sha'>): string
|
|
7773
|
+
export function runKey(layer: Pick<LayerHeader, 'runUrl' | 'branch' | 'sha'> & { readonly run?: { readonly id: string } }): string
|
|
7763
7774
|
```
|
|
7764
7775
|
|
|
7765
|
-
The run a layer belongs to: its run URL, else `branch@sha`, else
|
|
7776
|
+
The run a layer belongs to: the execution its producer named (`run:<id>`), else its run URL, else `branch@sha`, else
|
|
7777
|
+
`local`. Layers with the same key are one run. Only grouping: it is never part of the slot a result fills.
|
|
7766
7778
|
|
|
7767
7779
|
#### `TestHistory`
|
|
7768
7780
|
|
package/docs/reference/schema.md
CHANGED
|
@@ -258,7 +258,7 @@ export const TestHistorySchema = …
|
|
|
258
258
|
export const ReportRunSchema = …
|
|
259
259
|
```
|
|
260
260
|
|
|
261
|
-
A run: the layers that share a run URL
|
|
261
|
+
A run: the layers that name the same execution (else share a run URL, else a branch and commit), with what they recorded.
|
|
262
262
|
|
|
263
263
|
#### `RunListSchema`
|
|
264
264
|
|
|
@@ -643,6 +643,25 @@ Why a layer's origin is advisory: `modified` (recorded from a checkout with unco
|
|
|
643
643
|
(origin not captured). Absent on a layer from a clean commit. It never changes a status. A closed vocabulary: it
|
|
644
644
|
changes only with a new `schemaVersion`.
|
|
645
645
|
|
|
646
|
+
#### `LayerRunSchema`
|
|
647
|
+
|
|
648
|
+
```ts
|
|
649
|
+
export const LayerRunSchema = …
|
|
650
|
+
```
|
|
651
|
+
|
|
652
|
+
The execution a layer belongs to, named by the producer: one invocation of a local or manual run, which several
|
|
653
|
+
layers (shards, batches, producers) may share. Presentation and history grouping only: it never takes part in the
|
|
654
|
+
slot a result fills, in which result wins, or in a status. Absent, a run is told by its CI run URL, else `branch@sha`.
|
|
655
|
+
Within one project, layers that carry the same `id` are one run; a later invocation gets another `id`.
|
|
656
|
+
|
|
657
|
+
#### `LayerRun`
|
|
658
|
+
|
|
659
|
+
```ts
|
|
660
|
+
export type LayerRun = z.output<typeof LayerRunSchema>;
|
|
661
|
+
```
|
|
662
|
+
|
|
663
|
+
The execution of a layer, parsed from `LayerRunSchema`.
|
|
664
|
+
|
|
646
665
|
#### `MeasurementSchema`
|
|
647
666
|
|
|
648
667
|
```ts
|
package/docs/report-contract.md
CHANGED
|
@@ -82,6 +82,7 @@ Zod schema as `LayerDocumentSchema` from `@crvouga/atlas/schema`.
|
|
|
82
82
|
| `environment` | The client checked, or absent. Normalised to a lowercase slug: `iOS` → `ios`, `Desktop Web` → `desktop-web`, `iphone` → `ios`, `mobile browser` → `mobile-web`. Any other name is kept. Part of the slot. |
|
|
83
83
|
| `parameters` | Other string choices that make a result its own slot, such as `{ "device": "iphone-15" }`. A parameter named `environment` is read as the environment. Part of the slot. |
|
|
84
84
|
| `freshness` | `modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Advisory only; see [Statuses](#statuses-and-the-gate). |
|
|
85
|
+
| `run` | Optional `{ id, title?, startedAtMs?, endedAtMs? }`: the execution a local or manual producer names. Layers with the same `id` in one project are one run in the run list, however many shards or batches uploaded them; a later invocation at the same branch and commit gets another `id`. Without it a run is its `runUrl`, else `branch@sha`, as before. `title` is for people. Grouping and display only: it is **never part of the slot**, never decides a winner or a status, and never hides `sha`, `branch` or `freshness`. A `run` a reader cannot read is treated as absent. |
|
|
85
86
|
| `specs` | The specs the layer was recorded against, with a `fingerprint` of their text, so a reader can warn when a spec changed since. Optional. |
|
|
86
87
|
| `parts[].kind` | An open category (`behavior`, `performance`, `accessibility`, …). A new kind needs no change to any reader. Part of the slot. |
|
|
87
88
|
| `parts[].checks[]` | `id` (required, stable), `title`, `status`, `detail`, `specPath`, `measurements`, `durationMs`. |
|
package/docs/ui.md
CHANGED
|
@@ -119,7 +119,7 @@ The narrowly scoped exceptions are:
|
|
|
119
119
|
| --- | --- | --- |
|
|
120
120
|
| Shared shell | Project switching, spec navigation, run selection, connection status, diagnostics, mobile index | AppShell, Burger, NavLink, Menu, Popover, NativeSelect, Drawer |
|
|
121
121
|
| Project home | Tests/runs tabs, metrics, search, status/environment/domain filters, grouping, sorting, show more | Container, Tabs, SimpleGrid, Paper, TextInput, Button, Accordion |
|
|
122
|
-
| Test detail | Current outcomes, statistics, duration graph, history, artifacts, recording | Drawer, Table, Badge, Alert, Stage |
|
|
122
|
+
| Test detail | Current outcomes, statistics, duration graph, history, artifacts, recording; the `layer` address parameter pins it to one run's result, labelled "This run" against "Latest combined evidence" | Drawer, Table, Badge, Alert, Stage |
|
|
123
123
|
| Specs | All six existing kinds, overview/index, Gherkin rules/scenarios/examples, Quint declarations/traces, DMN tables, source text | SpecHeader, Accordion, Table, CodeHighlightTabs, ResultRow |
|
|
124
124
|
| Statecharts | Graph, screen/event/journey details, developer metadata, chart test results | Mantine cards/navigation and product compositions; graph exception above |
|
|
125
125
|
| Evidence | Steps, retries, annotations, facts, unknown extensions, safe downloads and previews | List, Badge, Accordion, Code, Image, native media |
|
package/llms-full.txt
CHANGED
|
@@ -505,6 +505,14 @@ first commit, they are `null`: that is provenance missing, not an error, and nev
|
|
|
505
505
|
result wins. `--param key=value` (repeatable) adds a parameter that makes a result its own slot, such
|
|
506
506
|
as `--param device=iphone-15`.
|
|
507
507
|
|
|
508
|
+
Without a CI run URL, layers recorded at one branch and commit group into one run, so repeating a local demo
|
|
509
|
+
looks like one execution. To keep invocations apart, name each one: `--run-id <id>` (or `ATLAS_RUN_ID`), with
|
|
510
|
+
`--run-title <title>` (or `ATLAS_RUN_TITLE`) for a human name and `--run-started-at` / `--run-ended-at` for when
|
|
511
|
+
it ran. Every shard or batch of one invocation passes the same id and is one run; the next invocation passes a new
|
|
512
|
+
one. A run recorded from a checkout with uncommitted changes keeps saying so beside its commit. The id only
|
|
513
|
+
groups: it is not part of which result a check fills or which wins, and a producer that never passes it keeps
|
|
514
|
+
the CI-run-URL, then `branch@sha`, grouping.
|
|
515
|
+
|
|
508
516
|
The time the checks ran comes from the report (CTRF `summary.stop`, a JUnit `timestamp`, a manifest's
|
|
509
517
|
`finishedAt`), or failing that from the report file's modification time, in milliseconds. Publish in
|
|
510
518
|
the same job that wrote the report and that is the time it ran. If you copy the report first, pass
|
|
@@ -2174,6 +2182,7 @@ Zod schema as `LayerDocumentSchema` from `@crvouga/atlas/schema`.
|
|
|
2174
2182
|
| `environment` | The client checked, or absent. Normalised to a lowercase slug: `iOS` → `ios`, `Desktop Web` → `desktop-web`, `iphone` → `ios`, `mobile browser` → `mobile-web`. Any other name is kept. Part of the slot. |
|
|
2175
2183
|
| `parameters` | Other string choices that make a result its own slot, such as `{ "device": "iphone-15" }`. A parameter named `environment` is read as the environment. Part of the slot. |
|
|
2176
2184
|
| `freshness` | `modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Advisory only; see [Statuses](#statuses-and-the-gate). |
|
|
2185
|
+
| `run` | Optional `{ id, title?, startedAtMs?, endedAtMs? }`: the execution a local or manual producer names. Layers with the same `id` in one project are one run in the run list, however many shards or batches uploaded them; a later invocation at the same branch and commit gets another `id`. Without it a run is its `runUrl`, else `branch@sha`, as before. `title` is for people. Grouping and display only: it is **never part of the slot**, never decides a winner or a status, and never hides `sha`, `branch` or `freshness`. A `run` a reader cannot read is treated as absent. |
|
|
2177
2186
|
| `specs` | The specs the layer was recorded against, with a `fingerprint` of their text, so a reader can warn when a spec changed since. Optional. |
|
|
2178
2187
|
| `parts[].kind` | An open category (`behavior`, `performance`, `accessibility`, …). A new kind needs no change to any reader. Part of the slot. |
|
|
2179
2188
|
| `parts[].checks[]` | `id` (required, stable), `title`, `status`, `detail`, `specPath`, `measurements`, `durationMs`. |
|
|
@@ -3789,7 +3798,7 @@ The narrowly scoped exceptions are:
|
|
|
3789
3798
|
| --- | --- | --- |
|
|
3790
3799
|
| Shared shell | Project switching, spec navigation, run selection, connection status, diagnostics, mobile index | AppShell, Burger, NavLink, Menu, Popover, NativeSelect, Drawer |
|
|
3791
3800
|
| Project home | Tests/runs tabs, metrics, search, status/environment/domain filters, grouping, sorting, show more | Container, Tabs, SimpleGrid, Paper, TextInput, Button, Accordion |
|
|
3792
|
-
| Test detail | Current outcomes, statistics, duration graph, history, artifacts, recording | Drawer, Table, Badge, Alert, Stage |
|
|
3801
|
+
| Test detail | Current outcomes, statistics, duration graph, history, artifacts, recording; the `layer` address parameter pins it to one run's result, labelled "This run" against "Latest combined evidence" | Drawer, Table, Badge, Alert, Stage |
|
|
3793
3802
|
| Specs | All six existing kinds, overview/index, Gherkin rules/scenarios/examples, Quint declarations/traces, DMN tables, source text | SpecHeader, Accordion, Table, CodeHighlightTabs, ResultRow |
|
|
3794
3803
|
| Statecharts | Graph, screen/event/journey details, developer metadata, chart test results | Mantine cards/navigation and product compositions; graph exception above |
|
|
3795
3804
|
| Evidence | Steps, retries, annotations, facts, unknown extensions, safe downloads and previews | List, Badge, Accordion, Code, Image, native media |
|
|
@@ -4246,12 +4255,14 @@ atlas report — layered evidence for any kind of spec
|
|
|
4246
4255
|
atlas report publish [--store <dir|s3://bucket/prefix>] [--producer <name>]
|
|
4247
4256
|
( --ctrf <file> | --junit <file> | --manifest <file> | --checks <file.json> )
|
|
4248
4257
|
[--environment <name>] [--branch <b>] [--sha <s>] [--run-url <u>]
|
|
4258
|
+
[--run-id <id>] [--run-title <t>] [--run-started-at <iso|ms>] [--run-ended-at <iso|ms>]
|
|
4249
4259
|
[--spec <path> | --specs <dir>] [--upload-run]
|
|
4250
4260
|
[--param key=value]... [--kind <k>] [--title <t>] [--recorded-at <iso|ms>]
|
|
4251
4261
|
[--attach [role=]<file>]... [--label key=value]... [--link [name=]<url>]...
|
|
4252
4262
|
[--attach-root <dir>]... [--max-attachment-mb <n>] [--max-total-attachment-mb <n>]
|
|
4253
4263
|
atlas report stream [--store <dir|s3://…|https://atlas.example.com>] --producer <name> [--environment <name>]
|
|
4254
4264
|
[--branch <b>] [--sha <s>] [--run-url <u>] [--kind <k>] [--param key=value]...
|
|
4265
|
+
[--run-id <id>] [--run-title <t>]
|
|
4255
4266
|
[--flush-ms 1000] [--batch 100] (checks as JSON lines on standard input)
|
|
4256
4267
|
atlas report status [--store <dir|s3://…|id=url>]... [--backends <module>]
|
|
4257
4268
|
[--specs <dir>] [--format text|json|markdown]
|
|
@@ -4272,6 +4283,11 @@ atlas report — layered evidence for any kind of spec
|
|
|
4272
4283
|
--specs <dir> link checks to the spec files in <dir> by path, or by the suite, class or
|
|
4273
4284
|
file name the report gives; also publishes the specs themselves, so the
|
|
4274
4285
|
node can show them without a checkout
|
|
4286
|
+
--run-id <id> name this execution (a local or manual run with no CI run URL; also ATLAS_RUN_ID).
|
|
4287
|
+
Layers that share an id are one run, however many shards or batches uploaded them;
|
|
4288
|
+
another invocation at the same commit gets another id. Never part of what a check is
|
|
4289
|
+
--run-title <t> a human name for the run (also ATLAS_RUN_TITLE); --run-started-at and --run-ended-at
|
|
4290
|
+
record when it ran. Without --run-id a run is its CI run URL, else branch@sha
|
|
4275
4291
|
--upload-run with --manifest: also upload the run's screenshots and clips for the viewer
|
|
4276
4292
|
--attach <file> attach a file to the whole layer: a CI log, a coverage report, a trace archive
|
|
4277
4293
|
(screenshots, videos and traces of a single test go in its CTRF or checks file)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crvouga/atlas",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Atlas: specs of any kind, reported as layered evidence. Publish checks from any runner (JUnit, CTRF, a script), combine them across nodes, branches, environments and backends, gate CI on the result, and run statecharts against the real app.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -256,6 +256,36 @@
|
|
|
256
256
|
"modified"
|
|
257
257
|
]
|
|
258
258
|
},
|
|
259
|
+
"run": {
|
|
260
|
+
"description": "The execution the producer named for it (one local or manual invocation). Grouping and display only; never part of the slot",
|
|
261
|
+
"type": "object",
|
|
262
|
+
"properties": {
|
|
263
|
+
"id": {
|
|
264
|
+
"type": "string",
|
|
265
|
+
"minLength": 1,
|
|
266
|
+
"maxLength": 256,
|
|
267
|
+
"description": "The execution's identity within a project, chosen by the producer: a UUID, a timestamped name. Layers with the same id are one run; it never changes after it is recorded"
|
|
268
|
+
},
|
|
269
|
+
"title": {
|
|
270
|
+
"description": "A human name for the run. Presentation only: it is not part of the identity and does not hide the commit or `freshness`",
|
|
271
|
+
"type": "string"
|
|
272
|
+
},
|
|
273
|
+
"startedAtMs": {
|
|
274
|
+
"description": "When the execution started, epoch milliseconds",
|
|
275
|
+
"type": "number",
|
|
276
|
+
"minimum": 0
|
|
277
|
+
},
|
|
278
|
+
"endedAtMs": {
|
|
279
|
+
"description": "When the execution ended, epoch milliseconds",
|
|
280
|
+
"type": "number",
|
|
281
|
+
"minimum": 0
|
|
282
|
+
}
|
|
283
|
+
},
|
|
284
|
+
"required": [
|
|
285
|
+
"id"
|
|
286
|
+
],
|
|
287
|
+
"additionalProperties": {}
|
|
288
|
+
},
|
|
259
289
|
"recording": {
|
|
260
290
|
"anyOf": [
|
|
261
291
|
{},
|
|
@@ -256,6 +256,36 @@
|
|
|
256
256
|
"modified"
|
|
257
257
|
]
|
|
258
258
|
},
|
|
259
|
+
"run": {
|
|
260
|
+
"description": "The execution the producer named for it (one local or manual invocation). Grouping and display only; never part of the slot",
|
|
261
|
+
"type": "object",
|
|
262
|
+
"properties": {
|
|
263
|
+
"id": {
|
|
264
|
+
"type": "string",
|
|
265
|
+
"minLength": 1,
|
|
266
|
+
"maxLength": 256,
|
|
267
|
+
"description": "The execution's identity within a project, chosen by the producer: a UUID, a timestamped name. Layers with the same id are one run; it never changes after it is recorded"
|
|
268
|
+
},
|
|
269
|
+
"title": {
|
|
270
|
+
"description": "A human name for the run. Presentation only: it is not part of the identity and does not hide the commit or `freshness`",
|
|
271
|
+
"type": "string"
|
|
272
|
+
},
|
|
273
|
+
"startedAtMs": {
|
|
274
|
+
"description": "When the execution started, epoch milliseconds",
|
|
275
|
+
"type": "number",
|
|
276
|
+
"minimum": 0
|
|
277
|
+
},
|
|
278
|
+
"endedAtMs": {
|
|
279
|
+
"description": "When the execution ended, epoch milliseconds",
|
|
280
|
+
"type": "number",
|
|
281
|
+
"minimum": 0
|
|
282
|
+
}
|
|
283
|
+
},
|
|
284
|
+
"required": [
|
|
285
|
+
"id"
|
|
286
|
+
],
|
|
287
|
+
"additionalProperties": {}
|
|
288
|
+
},
|
|
259
289
|
"recording": {
|
|
260
290
|
"anyOf": [
|
|
261
291
|
{},
|
|
@@ -75,6 +75,36 @@
|
|
|
75
75
|
"modified"
|
|
76
76
|
]
|
|
77
77
|
},
|
|
78
|
+
"run": {
|
|
79
|
+
"description": "The execution the producer named for it (one local or manual invocation). Grouping and display only; never part of the slot",
|
|
80
|
+
"type": "object",
|
|
81
|
+
"properties": {
|
|
82
|
+
"id": {
|
|
83
|
+
"type": "string",
|
|
84
|
+
"minLength": 1,
|
|
85
|
+
"maxLength": 256,
|
|
86
|
+
"description": "The execution's identity within a project, chosen by the producer: a UUID, a timestamped name. Layers with the same id are one run; it never changes after it is recorded"
|
|
87
|
+
},
|
|
88
|
+
"title": {
|
|
89
|
+
"description": "A human name for the run. Presentation only: it is not part of the identity and does not hide the commit or `freshness`",
|
|
90
|
+
"type": "string"
|
|
91
|
+
},
|
|
92
|
+
"startedAtMs": {
|
|
93
|
+
"description": "When the execution started, epoch milliseconds",
|
|
94
|
+
"type": "number",
|
|
95
|
+
"minimum": 0
|
|
96
|
+
},
|
|
97
|
+
"endedAtMs": {
|
|
98
|
+
"description": "When the execution ended, epoch milliseconds",
|
|
99
|
+
"type": "number",
|
|
100
|
+
"minimum": 0
|
|
101
|
+
}
|
|
102
|
+
},
|
|
103
|
+
"required": [
|
|
104
|
+
"id"
|
|
105
|
+
],
|
|
106
|
+
"additionalProperties": {}
|
|
107
|
+
},
|
|
78
108
|
"recording": {
|
|
79
109
|
"anyOf": [
|
|
80
110
|
{},
|
|
@@ -70,6 +70,36 @@
|
|
|
70
70
|
"modified"
|
|
71
71
|
]
|
|
72
72
|
},
|
|
73
|
+
"run": {
|
|
74
|
+
"description": "The execution this layer belongs to, when the producer names one (a local or manual run without a CI run URL). Grouping and display only; it never partitions a report or decides a result",
|
|
75
|
+
"type": "object",
|
|
76
|
+
"properties": {
|
|
77
|
+
"id": {
|
|
78
|
+
"type": "string",
|
|
79
|
+
"minLength": 1,
|
|
80
|
+
"maxLength": 256,
|
|
81
|
+
"description": "The execution's identity within a project, chosen by the producer: a UUID, a timestamped name. Layers with the same id are one run; it never changes after it is recorded"
|
|
82
|
+
},
|
|
83
|
+
"title": {
|
|
84
|
+
"description": "A human name for the run. Presentation only: it is not part of the identity and does not hide the commit or `freshness`",
|
|
85
|
+
"type": "string"
|
|
86
|
+
},
|
|
87
|
+
"startedAtMs": {
|
|
88
|
+
"description": "When the execution started, epoch milliseconds",
|
|
89
|
+
"type": "number",
|
|
90
|
+
"minimum": 0
|
|
91
|
+
},
|
|
92
|
+
"endedAtMs": {
|
|
93
|
+
"description": "When the execution ended, epoch milliseconds",
|
|
94
|
+
"type": "number",
|
|
95
|
+
"minimum": 0
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
"required": [
|
|
99
|
+
"id"
|
|
100
|
+
],
|
|
101
|
+
"additionalProperties": {}
|
|
102
|
+
},
|
|
73
103
|
"specs": {
|
|
74
104
|
"default": [],
|
|
75
105
|
"type": "array",
|
|
@@ -1110,6 +1110,36 @@
|
|
|
1110
1110
|
"modified"
|
|
1111
1111
|
]
|
|
1112
1112
|
},
|
|
1113
|
+
"run": {
|
|
1114
|
+
"description": "Name the execution these checks belong to, so repeated local runs at one commit stay separate runs and shards of one execution stay together",
|
|
1115
|
+
"type": "object",
|
|
1116
|
+
"properties": {
|
|
1117
|
+
"id": {
|
|
1118
|
+
"type": "string",
|
|
1119
|
+
"minLength": 1,
|
|
1120
|
+
"maxLength": 256,
|
|
1121
|
+
"description": "The execution's identity within a project, chosen by the producer: a UUID, a timestamped name. Layers with the same id are one run; it never changes after it is recorded"
|
|
1122
|
+
},
|
|
1123
|
+
"title": {
|
|
1124
|
+
"description": "A human name for the run. Presentation only: it is not part of the identity and does not hide the commit or `freshness`",
|
|
1125
|
+
"type": "string"
|
|
1126
|
+
},
|
|
1127
|
+
"startedAtMs": {
|
|
1128
|
+
"description": "When the execution started, epoch milliseconds",
|
|
1129
|
+
"type": "number",
|
|
1130
|
+
"minimum": 0
|
|
1131
|
+
},
|
|
1132
|
+
"endedAtMs": {
|
|
1133
|
+
"description": "When the execution ended, epoch milliseconds",
|
|
1134
|
+
"type": "number",
|
|
1135
|
+
"minimum": 0
|
|
1136
|
+
}
|
|
1137
|
+
},
|
|
1138
|
+
"required": [
|
|
1139
|
+
"id"
|
|
1140
|
+
],
|
|
1141
|
+
"additionalProperties": {}
|
|
1142
|
+
},
|
|
1113
1143
|
"specs": {
|
|
1114
1144
|
"default": [],
|
|
1115
1145
|
"type": "array",
|
|
@@ -15,7 +15,23 @@
|
|
|
15
15
|
"properties": {
|
|
16
16
|
"key": {
|
|
17
17
|
"type": "string",
|
|
18
|
-
"description": "
|
|
18
|
+
"description": "`run:<id>` for an execution a producer named, else the run URL, else `branch@sha`. Pass it to `/api/reports/runs/detail?key=` or `/api/reports/tests?run=`"
|
|
19
|
+
},
|
|
20
|
+
"runId": {
|
|
21
|
+
"description": "The execution id the producer named, when it did. Layers that share it are one run however many uploads they took",
|
|
22
|
+
"type": "string"
|
|
23
|
+
},
|
|
24
|
+
"title": {
|
|
25
|
+
"description": "The human title of the execution (the newest layer's). Presentation only: the key, commit and `freshness` are unaffected",
|
|
26
|
+
"type": "string"
|
|
27
|
+
},
|
|
28
|
+
"freshness": {
|
|
29
|
+
"description": "`modified` when any layer of the run was recorded from a checkout with uncommitted changes; `unknown` when an origin was not captured; absent when all are clean. Shown beside the commit, never replaced by the title",
|
|
30
|
+
"type": "string",
|
|
31
|
+
"enum": [
|
|
32
|
+
"unknown",
|
|
33
|
+
"modified"
|
|
34
|
+
]
|
|
19
35
|
},
|
|
20
36
|
"runUrl": {
|
|
21
37
|
"type": [
|
|
@@ -22,7 +22,23 @@
|
|
|
22
22
|
"properties": {
|
|
23
23
|
"key": {
|
|
24
24
|
"type": "string",
|
|
25
|
-
"description": "
|
|
25
|
+
"description": "`run:<id>` for an execution a producer named, else the run URL, else `branch@sha`. Pass it to `/api/reports/runs/detail?key=` or `/api/reports/tests?run=`"
|
|
26
|
+
},
|
|
27
|
+
"runId": {
|
|
28
|
+
"description": "The execution id the producer named, when it did. Layers that share it are one run however many uploads they took",
|
|
29
|
+
"type": "string"
|
|
30
|
+
},
|
|
31
|
+
"title": {
|
|
32
|
+
"description": "The human title of the execution (the newest layer's). Presentation only: the key, commit and `freshness` are unaffected",
|
|
33
|
+
"type": "string"
|
|
34
|
+
},
|
|
35
|
+
"freshness": {
|
|
36
|
+
"description": "`modified` when any layer of the run was recorded from a checkout with uncommitted changes; `unknown` when an origin was not captured; absent when all are clean. Shown beside the commit, never replaced by the title",
|
|
37
|
+
"type": "string",
|
|
38
|
+
"enum": [
|
|
39
|
+
"unknown",
|
|
40
|
+
"modified"
|
|
41
|
+
]
|
|
26
42
|
},
|
|
27
43
|
"runUrl": {
|
|
28
44
|
"type": [
|