@crvouga/atlas 1.12.0 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +13 -1
  2. package/dist/critics/src/contract.d.ts +113 -0
  3. package/dist/critics/src/contract.js +48 -0
  4. package/dist/critics/src/credentials.d.ts +72 -0
  5. package/dist/critics/src/credentials.js +136 -0
  6. package/dist/critics/src/critique.d.ts +128 -0
  7. package/dist/critics/src/critique.js +163 -0
  8. package/dist/critics/src/index.d.ts +21 -0
  9. package/dist/critics/src/index.js +21 -0
  10. package/dist/critics/src/judging.d.ts +153 -0
  11. package/dist/critics/src/judging.js +145 -0
  12. package/dist/critics/src/publish.d.ts +23 -0
  13. package/dist/critics/src/publish.js +21 -0
  14. package/dist/critics/src/runner.d.ts +104 -0
  15. package/dist/critics/src/runner.js +202 -0
  16. package/dist/critics/src/scope.d.ts +8 -0
  17. package/dist/critics/src/scope.js +26 -0
  18. package/dist/playwright/src/index.d.ts +4 -0
  19. package/dist/playwright/src/index.js +4 -0
  20. package/dist/playwright/src/ui-catalog.d.ts +51 -0
  21. package/dist/playwright/src/ui-catalog.js +44 -0
  22. package/dist/reports/src/compose.d.ts +2 -2
  23. package/dist/reports/src/compose.js +6 -5
  24. package/dist/reports/src/evidence-types.d.ts +3 -0
  25. package/dist/reports/src/evidence-view.d.ts +2 -1
  26. package/dist/reports/src/evidence-view.js +5 -2
  27. package/dist/reports/src/layer-merge.d.ts +16 -4
  28. package/dist/reports/src/layer-merge.js +23 -5
  29. package/dist/reports/src/report-index.js +9 -5
  30. package/dist/reports/src/report-status.d.ts +1 -1
  31. package/dist/reports/src/report-status.js +6 -4
  32. package/dist/reports/src/ui-catalog-publish.d.ts +92 -0
  33. package/dist/reports/src/ui-catalog-publish.js +68 -0
  34. package/dist/reports/src/ui-catalog.d.ts +229 -0
  35. package/dist/reports/src/ui-catalog.js +273 -0
  36. package/dist/reports/src/ui-state.d.ts +70 -0
  37. package/dist/reports/src/ui-state.js +66 -0
  38. package/dist/schema/src/api.d.ts +351 -0
  39. package/dist/schema/src/api.js +72 -1
  40. package/dist/schema/src/critique.d.ts +144 -0
  41. package/dist/schema/src/critique.js +96 -0
  42. package/dist/schema/src/index.d.ts +2 -1
  43. package/dist/schema/src/index.js +2 -1
  44. package/dist/schema/src/server-config.d.ts +7 -1
  45. package/dist/schema/src/server-config.js +6 -3
  46. package/dist/server/src/app.js +15 -0
  47. package/dist/server/src/auth.d.ts +1 -1
  48. package/dist/server/src/auth.js +2 -2
  49. package/dist/server/src/config.js +2 -2
  50. package/dist/server/src/openapi.d.ts +3 -3
  51. package/dist/server/src/openapi.js +29 -6
  52. package/dist/server/src/store-api.js +39 -1
  53. package/docs/agents.md +1 -0
  54. package/docs/critics.md +190 -0
  55. package/docs/extending.md +1 -0
  56. package/docs/reference/critics.md +840 -0
  57. package/docs/reference/http-api.md +52 -2
  58. package/docs/reference/playwright.md +63 -0
  59. package/docs/reference/reports.md +608 -8
  60. package/docs/reference/schema.md +152 -2
  61. package/docs/reference/server.md +3 -3
  62. package/docs/report-contract.md +6 -2
  63. package/docs/rich-evidence.md +5 -0
  64. package/docs/server.md +8 -4
  65. package/docs/ui-catalog.md +173 -0
  66. package/docs/ui.md +15 -0
  67. package/llms-full.txt +457 -8
  68. package/llms.txt +3 -0
  69. package/package.json +8 -2
  70. package/packages/critics/README.md +18 -0
  71. package/packages/schema/schemas/critique.schema.json +254 -0
  72. package/packages/schema/schemas/effective-delta.schema.json +73 -0
  73. package/packages/schema/schemas/effective-evidence.schema.json +73 -0
  74. package/packages/schema/schemas/layer-rows.schema.json +73 -0
  75. package/packages/schema/schemas/server-config.schema.json +2 -1
  76. package/packages/schema/schemas/server-info.schema.json +1 -0
  77. package/packages/server/openapi.json +1443 -80
  78. package/packages/visualizer/src/app/AppShell.tsx +36 -30
  79. package/packages/visualizer/src/app/router.tsx +10 -1
  80. package/packages/visualizer/src/catalog/UiCatalogFilters.tsx +65 -0
  81. package/packages/visualizer/src/catalog/UiCatalogPage.tsx +342 -0
  82. package/packages/visualizer/src/catalog/provider.tsx +25 -0
  83. package/packages/visualizer/src/catalog/search.ts +68 -0
  84. package/packages/visualizer/src/catalog/words.ts +26 -0
  85. package/packages/visualizer/src/components/Header.tsx +17 -4
@@ -498,6 +498,39 @@ export const RunDetailSchema = …
498
498
 
499
499
  `GET /api/reports/runs/detail`: one run and its last result for every test, environment and producer it recorded.
500
500
 
501
+ #### `UiProfileSchema`
502
+
503
+ ```ts
504
+ export const UiProfileSchema = …
505
+ ```
506
+
507
+ A state's profile: the client and the choices it was captured under.
508
+
509
+ #### `UiImageSchema`
510
+
511
+ ```ts
512
+ export const UiImageSchema = …
513
+ ```
514
+
515
+ The picture of a capture.
516
+
517
+ #### `UiCellSchema`
518
+
519
+ ```ts
520
+ export const UiCellSchema = …
521
+ ```
522
+
523
+ One state in one profile.
524
+
525
+ #### `UiCatalogSchema`
526
+
527
+ ```ts
528
+ export const UiCatalogSchema = …
529
+ ```
530
+
531
+ `GET /api/reports/ui-catalog`: the UI catalog, computed from the winning results of the `ui-state` and `ui-plan` parts
532
+ of every layer. States are in id order and cells in profile order, so the same layers always give the same document.
533
+
501
534
  ### Command results (`--format json`)
502
535
 
503
536
  Source: `packages/schema/src/cli-results.ts`
@@ -775,12 +808,128 @@ export function transitionId(source: string, event: string)
775
808
  Builds the id of a transition, the key of `transitions` in a manifest: the source state's name,
776
809
  `TRANSITION_ID_SEPARATOR`, then the event name.
777
810
 
811
+ ### The `atlas.critique` evidence schema
812
+
813
+ Source: `packages/schema/src/critique.ts`
814
+
815
+ What an automated reviewer (a critic) says about one captured UI. It is not a new layer format: a critique is a
816
+ check inside a part of kind `critique`, and the opinion rides in the check's `extensions["atlas.critique"]`, so by the
817
+ rules of docs/extending.md it can never decide a result. The check's `status` says only whether the critic ran
818
+ (`passed` a verdict was produced, `failed` the critic could not produce one, `running`, `not-run`).
819
+
820
+ Everything except `schema`, `subject` and `critic` is optional, so a single score (a model-based critic) and rich
821
+ findings (an LLM critic) are one shape. Every object is open, and `kind`, `category`, `severity`, `label` and `reason`
822
+ are open strings: a value nobody has heard of is kept and shown generically, never rejected. A reader that cannot read
823
+ the whole extension drops it and keeps the check and its status (`readCritique` in `@crvouga/atlas/critics`).
824
+
825
+ See docs/critics.md
826
+ See docs/extending.md
827
+
828
+ #### `CRITIQUE_EXTENSION`
829
+
830
+ ```ts
831
+ export const CRITIQUE_EXTENSION = …
832
+ ```
833
+
834
+ The key under which a critique is stored in a check's `extensions`.
835
+
836
+ #### `CRITIQUE_SCHEMA_VERSION`
837
+
838
+ ```ts
839
+ export const CRITIQUE_SCHEMA_VERSION = …
840
+ ```
841
+
842
+ The version of the `atlas.critique` extension a reader of this release understands. A critique of another version is unknown, not wrong.
843
+
844
+ #### `CRITIQUE_PART_KIND`
845
+
846
+ ```ts
847
+ export const CRITIQUE_PART_KIND = …
848
+ ```
849
+
850
+ The `kind` of the report part that holds critiques. Parts of this kind are opinions: never counted, never part of the gate.
851
+
852
+ #### `CRITIC_PRODUCER_PREFIX`
853
+
854
+ ```ts
855
+ export const CRITIC_PRODUCER_PREFIX = …
856
+ ```
857
+
858
+ The prefix of the `producer` of a layer of critiques: `critic.<critic name>`.
859
+
860
+ #### `CritiqueRegionSchema`
861
+
862
+ ```ts
863
+ export const CritiqueRegionSchema = …
864
+ ```
865
+
866
+ The region of the image a finding is about, in the stated unit (`px` of the stored image, or `normalized` 0..1).
867
+
868
+ #### `CritiqueFindingSchema`
869
+
870
+ ```ts
871
+ export const CritiqueFindingSchema = …
872
+ ```
873
+
874
+ One thing a critic found: where, what kind, how bad, why, and what to do. Nothing is required.
875
+
876
+ #### `CritiqueRelativeSchema`
877
+
878
+ ```ts
879
+ export const CritiqueRelativeSchema = …
880
+ ```
881
+
882
+ How a score reads in context. A raw score is not absolute: a loading overlay is compared with other loading overlays,
883
+ not with full screens, and a change against the same entry's previous image is more telling than the level.
884
+
885
+ #### `CritiqueVerdictSchema`
886
+
887
+ ```ts
888
+ export const CritiqueVerdictSchema = …
889
+ ```
890
+
891
+ The judgement: a score and how to read it, a label, a confidence. A critic that cannot score may give only a label.
892
+
893
+ #### `CritiqueFailureSchema`
894
+
895
+ ```ts
896
+ export const CritiqueFailureSchema = …
897
+ ```
898
+
899
+ Why a critic produced no verdict: a reason code in an open vocabulary (`error`, `timeout`, `rate-limited`, `invalid-output`) and a message that holds no credential.
900
+
901
+ #### `CritiqueSchema`
902
+
903
+ ```ts
904
+ export const CritiqueSchema = …
905
+ ```
906
+
907
+ The `atlas.critique` extension, version 1: the image judged (`subject`), who judged (`critic`, `model`,
908
+ `parameters`), the opinion (`verdict`, `findings`) or why there is none (`failure`), and what it cost (`usage`).
909
+ The prompt is recorded only as a hash (`parameters.rubric`). Emitted as `critique.schema.json`.
910
+
911
+ #### `Critique`
912
+
913
+ ```ts
914
+ export type Critique = z.output<typeof CritiqueSchema>;
915
+ ```
916
+
917
+ A critique as stored in a check's `extensions["atlas.critique"]`.
918
+
919
+ #### `CritiqueFinding`
920
+
921
+ ```ts
922
+ export type CritiqueFinding = z.output<typeof CritiqueFindingSchema>;
923
+ ```
924
+
925
+ One finding of a critique.
926
+
778
927
  ### Schema package entry point (`@crvouga/atlas/schema`)
779
928
 
780
929
  Source: `packages/schema/src/index.ts`. Import: `@crvouga/atlas/schema`
781
930
 
782
931
  Re-exports every Zod schema, constant and inferred type of the Atlas contract: the layer format and report status
783
- (`layer`), the statechart run manifest (`manifest`, `common`), the spec file formats (`spec`), the run index and
932
+ (`layer`), the `atlas.critique` evidence of automated UI reviewers (`critique`), the statechart run manifest (`manifest`, `common`), the spec file formats (`spec`), the run index and
784
933
  report sources the viewer reads (`runs-index`, `report-sources`) and the server configuration (`server-config`).
785
934
  The JSON Schemas shipped in `schemas/` are generated from these by `bun run schemas`.
786
935
 
@@ -1869,7 +2018,8 @@ See docs/server.md
1869
2018
  export const ServerRoleSchema = …
1870
2019
  ```
1871
2020
 
1872
- What a bearer token may do: `read` (look), `publish` (also upload) or `admin` (also remove scratch objects).
2021
+ What a bearer token may do: `read` (look), `critic` (also publish critiques, and nothing else), `publish` (also upload
2022
+ anything) or `admin` (also remove scratch objects).
1873
2023
 
1874
2024
  #### `ServerRole`
1875
2025
 
@@ -93,7 +93,7 @@ What it knows lives in its backends (by default a folder on its own disk), so re
93
93
 
94
94
  Source: `packages/server/src/auth.ts`
95
95
 
96
- Who is asking, and may they. Tokens have a role (`read` < `publish` < `admin`) and are compared in constant time. Reading
96
+ Who is asking, and may they. Tokens have a role (`read` < `critic` < `publish` < `admin`) and are compared in constant time. Reading
97
97
  accepts a Bearer token or the browser's sign-in box (any user name, the token as the password); changing anything needs
98
98
  the Bearer header, because a browser sends the sign-in box's credentials to any page that asks, which would let another
99
99
  site upload with them. A server with no tokens is open, and refuses to listen beyond this machine unless told it may.
@@ -530,10 +530,10 @@ The version of the API described, `major.minor.patch`: the minor grows when an o
530
530
  #### `Role`
531
531
 
532
532
  ```ts
533
- export type Role = 'none' | 'read' | 'publish' | 'admin';
533
+ export type Role = 'none' | 'read' | 'critic' | 'publish' | 'admin';
534
534
  ```
535
535
 
536
- Who may call an operation: `none` (anyone), or the least role that may (`read` < `publish` < `admin`).
536
+ Who may call an operation: `none` (anyone), or the least role that may (`read` < `critic` < `publish` < `admin`).
537
537
 
538
538
  #### `Operation`
539
539
 
@@ -84,7 +84,7 @@ Zod schema as `LayerDocumentSchema` from `@crvouga/atlas/schema`.
84
84
  | `freshness` | `modified`: recorded from a checkout with uncommitted changes. `unknown`: origin not captured. Advisory only; see [Statuses](#statuses-and-the-gate). |
85
85
  | `run` | Optional `{ id, title?, startedAtMs?, endedAtMs? }`: the execution a local or manual producer names. Layers with the same `id` in one project are one run in the run list, however many shards or batches uploaded them; a later invocation at the same branch and commit gets another `id`. Without it a run is its `runUrl`, else `branch@sha`, as before. `title` is for people. Grouping and display only: it is **never part of the slot**, never decides a winner or a status, and never hides `sha`, `branch` or `freshness`. A `run` a reader cannot read is treated as absent. |
86
86
  | `specs` | The specs the layer was recorded against, with a `fingerprint` of their text, so a reader can warn when a spec changed since. Optional. |
87
- | `parts[].kind` | An open category (`behavior`, `performance`, `accessibility`, …). A new kind needs no change to any reader. Part of the slot. |
87
+ | `parts[].kind` | An open category (`behavior`, `performance`, `accessibility`, …). A new kind needs no change to any reader. Part of the slot. `ui-state` and `ui-plan` are the [UI catalog](ui-catalog.md)'s: a captured UI state, and the states that are expected. |
88
88
  | `parts[].checks[]` | `id` (required, stable), `title`, `status`, `detail`, `specPath`, `measurements`, `durationMs`. |
89
89
  | `tags`, `labels`, `links`, `attachments`, `annotations`, `extensions` | Evidence, on the layer, a part or a check: what a result carries beyond its outcome. All optional and open. See [evidence](#evidence) and [the guide](extending.md). |
90
90
  | `checks[].steps`, `checks[].attempts` | A check's timeline, and every try when it was retried. Optional. |
@@ -227,9 +227,13 @@ are part of the contract:
227
227
  Tools map to these four. CTRF `skipped`, `pending` and `other`, and JUnit `<skipped>`, are `not-run`.
228
228
  An Atlas path that passed on retry (`flaky`) is `passed`, with the detail "Passed on retry (flaky)".
229
229
 
230
- Timings that only qualify a check (parts of kind `performance`) are not counted as checks when
230
+ Timings that only qualify a check (parts of kind `performance`), and what a UI catalog expects (`ui-plan`), are not counted as checks when
231
231
  anything else is recorded. A measurement over its allowance is reported and never fails a check.
232
232
 
233
+ Opinions (parts of kind `critique`, the verdicts of [automated UI reviewers](critics.md)) are never counted as checks,
234
+ even when they are all that was recorded, and are never part of the gate: a low score, or a critic that crashed, leaves
235
+ `atlas report status` as it was. The `status` of a critique check says only whether the critic ran.
236
+
233
237
  **Freshness never changes a status.** A layer whose spec fingerprint no longer matches, whose
234
238
  checkout was modified, or whose related code changed still shows its recorded outcome, with a
235
239
  warning ("Potentially outdated: spec changed"). Atlas favours quick, possibly stale evidence over
@@ -191,6 +191,11 @@ time. Every field after `unit` is optional and none of them decides a status, a
191
191
  - **Reading it from the API.** `GET /api/reports/tests/history` entries carry `latency` (their product-latency
192
192
  measurements) next to `durationMs`. The pure functions are in `@crvouga/atlas/reports/latency`.
193
193
 
194
+ ## A catalog of UI states
195
+
196
+ A screenshot per screen, per viewport and color scheme, is a *UI catalog*: check ids are the states, the viewport and scheme are
197
+ `parameters`, and the viewer's `/catalog` shows them with what is still to be captured. See [the UI catalog](ui-catalog.md).
198
+
194
199
  ## Finding runs and artifacts
195
200
 
196
201
  The viewer's **Runs** and **Artifacts** tabs and the API answer one query, so a link means the same thing everywhere.
package/docs/server.md CHANGED
@@ -205,7 +205,7 @@ is said in words and the server does not start. The file has a published
205
205
  | How often (seconds) the backends are looked at for layers written by something else | `refreshSeconds` | `--refresh <seconds>` | `ATLAS_REFRESH_SECONDS` | `10` |
206
206
  | Let anyone look, not upload | `auth.anonymousRead` | `--anonymous-read` | `ATLAS_ANONYMOUS_READ` | no |
207
207
  | Listen publicly with no tokens (something in front signs people in) | `auth.insecure` | `--insecure` | `ATLAS_INSECURE` | no |
208
- | The admin token (also `ATLAS_PUBLISH_TOKEN` and `ATLAS_READ_TOKEN` for the other roles, or `auth.tokens` in the file) | `auth.tokens` | | `ATLAS_TOKEN` | none: the server is open and listens on this machine only |
208
+ | The admin token (also `ATLAS_PUBLISH_TOKEN`, `ATLAS_CRITIC_TOKEN` and `ATLAS_READ_TOKEN` for the other roles, or `auth.tokens` in the file) | `auth.tokens` | | `ATLAS_TOKEN` | none: the server is open and listens on this machine only |
209
209
  | The configuration file | `(the file itself)` | `--config <file>` | `ATLAS_CONFIG` | `./atlas.server.json` when it exists |
210
210
  | Largest uploaded object in MiB | `limits.maxObjectMiB` | | `ATLAS_MAX_OBJECT_MIB` | `256` |
211
211
  | Largest JSON upload in MiB | `limits.maxJsonMiB` | | `ATLAS_MAX_JSON_MIB` | `32` |
@@ -238,16 +238,18 @@ backend in the client and `/readyz`, and recovers by itself.
238
238
  | Role | May |
239
239
  | --- | --- |
240
240
  | `read` | look: the client, the report, `/api/status` |
241
+ | `critic` | also publish [critiques](critics.md#authentication-and-authorization) and nothing else: blobs and layers whose producer is `critic.<name>` and whose parts are all of kind `critique`. It cannot post test results, start live runs or change settings |
241
242
  | `publish` | also upload layers, specs and runs, and remove the scratch objects `atlas report doctor` writes |
242
243
  | `admin` | all server operations |
243
244
 
244
245
  Tokens are at least 12 characters (`openssl rand -hex 24`) and different from one another. Give them as
245
- `ATLAS_TOKEN` (admin), `ATLAS_PUBLISH_TOKEN`, `ATLAS_READ_TOKEN`, or under `auth.tokens` in the file, preferably
246
+ `ATLAS_TOKEN` (admin), `ATLAS_PUBLISH_TOKEN`, `ATLAS_CRITIC_TOKEN`, `ATLAS_READ_TOKEN`, or under `auth.tokens` in the file, preferably
246
247
  with `tokenEnv` so the file holds no secret.
247
248
 
248
249
  - **Sending a token:** `Authorization: Bearer <token>`. A browser's sign-in box also works for *looking* (any
249
250
  user name, the token as the password). **Uploading needs the Bearer header, never the sign-in box**: a browser
250
251
  sends those credentials by itself to any page that asks, which would let another site upload with them.
252
+ - **Run critics against a server with tokens.** With no tokens every caller is an administrator, a critic included.
251
253
  - **No tokens** means open, and then the server refuses to listen beyond this machine unless told
252
254
  `--insecure` (for when a proxy in front does the signing in).
253
255
  - `/healthz` and `/readyz` never need a token, so a load balancer can use them.
@@ -490,8 +492,8 @@ curl "$ATLAS_SERVER_URL/api/projects/$ATLAS_PROJECT/reports/tests?format=markdow
490
492
  | `GET /api/projects/{project}/store/{key}` | read | Read an object. |
491
493
  | `HEAD /api/store/{key}` | read | Does an object exist. |
492
494
  | `HEAD /api/projects/{project}/store/{key}` | read | Does an object exist. |
493
- | `PUT /api/store/{key}` | publish | Upload an object. |
494
- | `PUT /api/projects/{project}/store/{key}` | publish | Upload an object. |
495
+ | `PUT /api/store/{key}` | critic | Upload an object. |
496
+ | `PUT /api/projects/{project}/store/{key}` | critic | Upload an object. |
495
497
  | `DELETE /api/store/{key}` | publish | Remove a scratch object. |
496
498
  | `DELETE /api/projects/{project}/store/{key}` | publish | Remove a scratch object. |
497
499
  | `GET /api/reports/tests` | read | Every test, with insights, facets and history statistics. |
@@ -520,6 +522,8 @@ curl "$ATLAS_SERVER_URL/api/projects/$ATLAS_PROJECT/reports/tests?format=markdow
520
522
  | `GET /api/projects/{project}/reports/runs` | read | The runs that recorded the report. |
521
523
  | `GET /api/reports/artifacts` | read | Recorded results that carry evidence, across the whole history. |
522
524
  | `GET /api/projects/{project}/reports/artifacts` | read | Recorded results that carry evidence, across the whole history. |
525
+ | `GET /api/reports/ui-catalog` | read | The UI catalog: captured UI states per profile, and what is still expected. |
526
+ | `GET /api/projects/{project}/reports/ui-catalog` | read | The UI catalog: captured UI states per profile, and what is still expected. |
523
527
  | `GET /api/reports/runs/detail` | read | Every test one run recorded. |
524
528
  | `GET /api/projects/{project}/reports/runs/detail` | read | Every test one run recorded. |
525
529
  | `GET /api/reports/outline` | read | The specs a report is laid over. |
@@ -0,0 +1,173 @@
1
+ # The UI catalog
2
+
3
+ A browsable inventory of a product's user interface: each **state** (a screen, or one state of a screen) captured once per
4
+ **profile** (a viewport, a color scheme, a device), with the facts needed to find and trust it, and what should exist but has not
5
+ been captured yet. Any program that can write a layer can feed it, and the catalog fills in as results stream in, so a run of
6
+ thousands of captures can be watched while it happens and split across as many machines as needed.
7
+
8
+ Nothing about it is new to the report contract. A state is an ordinary check with a screenshot, the expected set is an ordinary
9
+ layer, and the stream is the one every viewer already holds. This page is the convention that makes those usable as a catalog, the
10
+ decisions behind it, and what is not built yet.
11
+
12
+ - [How a state is written](#how-a-state-is-written)
13
+ - [What is expected](#what-is-expected)
14
+ - [Publishing from any collector](#publishing-from-any-collector)
15
+ - [Sharding](#sharding)
16
+ - [Streaming and loading](#streaming-and-loading)
17
+ - [Exploring it](#exploring-it)
18
+ - [Decisions](#decisions)
19
+ - [What stays in your project, and what is not built yet](#what-stays-in-your-project-and-what-is-not-built-yet)
20
+
21
+ ## How a state is written
22
+
23
+ A state is one **check** in a layer whose part has `kind` **`ui-state`**.
24
+
25
+ | Part of the idea | Where it goes |
26
+ | --- | --- |
27
+ | The state's identity | The check `id`. Make it stable (`checkout/payment`, `home?cohort=new`): a new capture replaces an old one only if the id is the same. |
28
+ | How the capture went | `status`: `passed` is **captured**, `failed` is attempted and failed (say why in `detail`), `running` is being taken, `not-run` is skipped. |
29
+ | The picture | An attachment with `role: "screenshot"` (else the first image), stored in the layer by content or given as an `https` `uri`. `width` and `height` help the viewer lay it out. |
30
+ | The profile | The layer's `environment` (`desktop-web`, `mobile-web`, `ios`, `android`) and `parameters` (`{ "viewport": "390x844", "scheme": "dark" }`). Parameters split a slot, so the same state at another viewport is **another result, side by side**, never a replacement. |
31
+ | Facts the producer declares | The check's `extensions["atlas.ui-state"]`: `route`, `fixture`, `cohort`, `area`, `flags` (a list), `covers` (the product surfaces the state shows). All optional. |
32
+ | Facts that describe without splitting | The check's `labels` (`shard`, `browser`) and `tags`. Labels never make a different slot. |
33
+ | Where it came from | The layer's `producer`, `sha`, `branch`, `runUrl`, `freshness`, `recordedAtMs`: shown with every capture, with the layer id and the backend that holds it. |
34
+
35
+ ```json
36
+ {
37
+ "producer": "device-farm", "kind": "ui-state", "environment": "mobile-web",
38
+ "parameters": { "viewport": "390x844", "scheme": "dark" }, "recordedAtMs": 1760000000000,
39
+ "checks": [{
40
+ "id": "checkout/payment", "title": "Payment, card declined", "status": "passed",
41
+ "labels": { "shard": "2/8" },
42
+ "extensions": { "atlas.ui-state": { "route": "/checkout", "fixture": "declined-card", "cohort": "new-customer", "flags": ["new-nav"], "covers": ["payment-form"] } },
43
+ "attachments": [{ "name": "payment.png", "role": "screenshot", "contentType": "image/png", "path": "payment.png" }]
44
+ }]
45
+ }
46
+ ```
47
+
48
+ `atlas report publish --checks file.json --store …` publishes that file, reading `payment.png` from beside it. Nothing in it
49
+ mentions how the page was driven. Because a state is a check, the gate counts it (a `failed` capture fails `atlas report status`),
50
+ `atlas report status` lists it, and a reader that does not know the catalog still shows a check with a screenshot. Evidence never
51
+ decides a result: a picture that cannot be read leaves the check, its status and its slot as they were.
52
+
53
+ ## What is expected
54
+
55
+ What should exist is a **layer too**, of kind **`ui-plan`**: the same ids, the same profiles (environment and parameters), every
56
+ check `not-run`, published by a planner, by each shard for the states it was given, or both. A plan fills a slot of its own (the
57
+ kind is part of the slot), so a capture never erases it and it never hides a capture; and `not-run` is never a pass.
58
+
59
+ | A cell (one state in one profile) is | When |
60
+ | --- | --- |
61
+ | **Captured** | the winning capture passed |
62
+ | **Capture failed** | the winning capture failed (its `detail` says why, and a diagnostic picture may be attached) |
63
+ | **Capturing** | the winning result is `running` |
64
+ | **Skipped** | a producer recorded it as `not-run` |
65
+ | **Not captured yet** | a plan expects it and nothing has been recorded |
66
+
67
+ A captured cell that no plan names is shown as **not in the expected set**. **Without any plan, nothing is "not captured yet" and
68
+ nothing is "not in the expected set"**: what should exist is then unknown, and the catalog says so instead of guessing. "Not yet
69
+ captured" is therefore different from "never declared".
70
+
71
+ Coverage against an **inventory of the product's surfaces** is a plan entry with `"inventory": true` (the id is the surface;
72
+ nothing is captured for it). A state says which surfaces it shows in `covers`. The catalog lists every surface as *covered* (a
73
+ state that shows it is captured), *states cover it, none captured yet*, or *no state covers it*.
74
+
75
+ ## Publishing from any collector
76
+
77
+ Any program that writes the layer format feeds the catalog: a shell script, a Go test, a Storybook crawler, a device farm.
78
+
79
+ ```ts
80
+ import { publishChecks } from '@crvouga/atlas/reports/publish-checks';
81
+ import { uiPlanLayer, uiStateLayer } from '@crvouga/atlas/reports/ui-catalog-publish';
82
+
83
+ // The planner, once: what the product should have, in every profile.
84
+ await publishChecks(store, uiPlanLayer({
85
+ producer: 'catalog-plan', recordedAtMs: Date.now(), environment: 'mobile-web', profile: { viewport: '390x844', scheme: 'dark' },
86
+ states: [{ id: 'checkout/payment', covers: ['payment-form'] }, { id: 'settings', inventory: true }]
87
+ }));
88
+
89
+ // Each collector or shard, with its own captures and no knowledge of the others.
90
+ await publishChecks(store, uiStateLayer({
91
+ producer: 'web-capture', recordedAtMs: Date.now(), environment: 'mobile-web', profile: { viewport: '390x844', scheme: 'dark' }, shard: '2/8',
92
+ states: [{ id: 'checkout/payment', route: '/checkout', covers: ['payment-form'], screenshot: { data: png, width: 390, height: 844 } }]
93
+ }));
94
+ ```
95
+
96
+ `uiStateLayer` and `uiPlanLayer` only build the input of `publishChecks`; they use no browser and no test framework. `store` can be
97
+ a directory, a bucket or the server (`httpStore`), exactly as for any layer.
98
+
99
+ **Playwright is one adapter and not a dependency.** `captureUiState(page, { id, ready })` from `@crvouga/atlas/playwright` waits
100
+ for a readiness check (bounded), photographs the page and returns the capture for `uiStateLayer`. A page that never became ready is
101
+ a `failed` capture that says why (with a picture of what it showed), never an exception that ends the run. Nothing that reads,
102
+ composes or shows the catalog imports a collection framework, and a test keeps it that way.
103
+
104
+ ## Sharding
105
+
106
+ A shard is **any publisher that does not know the others**: by case, by profile, by area or by machine; Atlas does not care which.
107
+ Each publishes its own layers (a `shard` label records which, and is shown with each capture). The guarantees are those of every
108
+ layer ([the contract](report-contract.md#guarantees)), which is what the tests prove for the catalog:
109
+
110
+ - The captures of N shards (N is 1 or more), combined, equal the catalog one unsharded run produces.
111
+ - Running a shard again replaces only the cells it produces (a newer `recordedAtMs` wins its slot); every other cell is untouched.
112
+ - A shard that never publishes leaves its cells **not captured yet** when a plan names them, and absent otherwise. They are never
113
+ shown as captured, and the other shards' cells are unaffected.
114
+ - Atlas does not need the shard count. Completeness is the plan: a cell is expected because a plan says so.
115
+
116
+ ## Streaming and loading
117
+
118
+ - **Streaming.** There is no second stream. A viewer holds the report over `/api/events` and reads what changed with
119
+ `/api/reports/evidence/effective?since=<version>`; the catalog is computed from the report it holds, so a capture appears when
120
+ its layer arrives, with no reload and no file rebuilt. Events carry metadata only; the picture is fetched when a card is on screen.
121
+ - **Reconnecting.** A viewer that was offline or whose stream dropped reads the delta since the version it holds (or the whole report
122
+ when the server no longer remembers it). Cells are keyed by state and profile, so nothing is missing and nothing is duplicated.
123
+ - **No stream.** A viewer without the stream (or a static export) polls, as the rest of the viewer does, and shows everything that exists.
124
+ - **Slow and failing backends.** Each backend is listed and read on its own, so a slow one never holds back the others: their
125
+ captures are shown, and the slow backend's appear when it answers. A backend that fails, or a layer that cannot be read, is named
126
+ on the page and in the document (`sources`); a capture that depended on it is not shown as captured.
127
+ - **Images.** A picture is a file in the layer, addressed by content (`/layers/<layer>/blobs/attachments/<sha256>`), loaded lazily
128
+ for the cards on screen, and served under the same untrusted-bytes rules as every attachment. A picture given as a `uri` is
129
+ followed only if it is `http` or `https`. A producer may leave the bytes out (`omitted`), which is told apart from never taken.
130
+
131
+ ## Exploring it
132
+
133
+ The viewer's **UI catalog** (`/catalog`, in the menu beside the spec types once the report holds a state or a plan) shows a card per
134
+ state with its picture and a coverage badge per profile, and filters by words, coverage, profile, area, route, cohort, feature flag,
135
+ surface and producer. The address holds the filters and the open state, so a link reproduces the view. Opening a state shows its
136
+ profiles side by side with the declared facts, the failure reason, and for each picture the layer, backend, producer, shard, commit
137
+ and time. *Download the manifest* saves the filtered catalog as JSON.
138
+
139
+ ```sh
140
+ curl "$ATLAS_SERVER_URL/api/reports/ui-catalog?coverage=failed,missing&profile=…" # JSON; add format=markdown for text
141
+ ```
142
+
143
+ `GET /api/reports/ui-catalog` answers the same document the viewer computes (`uiCatalog` from
144
+ `@crvouga/atlas/reports/ui-catalog`), filtered by `q`, `coverage`, `profile`, `area`, `route`, `cohort`, `flag`, `surface` and
145
+ `producer`, paged with `limit` and `offset`, with an ETag. Its picture `path` is relative to `/api/reports`.
146
+
147
+ ## Decisions
148
+
149
+ The questions the original proposal left open, and how they are answered here. Each is the simplest answer that stays inside the
150
+ [stable contracts](report-contract.md#versioning-and-stability): optional fields only, no change to a slot or a winner.
151
+
152
+ | Question | Decision |
153
+ | --- | --- |
154
+ | Layer and event schema | An existing check in a `ui-state` part, a `screenshot` attachment, facts under the `atlas.ui-state` extension. No new part type, no new schema version. The report row gained one optional field, `state`, only for `ui-state` and `ui-plan` rows. |
155
+ | Identity and slot | The check id is the state; the slot is the usual (producer, kind, spec, id, environment, parameters). Profile dimensions are `parameters` because they must split the slot; labels never do. |
156
+ | Expected versus captured | A `ui-plan` layer. `ui-plan` is a diagnostic kind (like `performance`): it qualifies a check and is not counted as one when the check has any other result. |
157
+ | Coverage against an inventory | `inventory: true` plan entries plus each state's `covers`. |
158
+ | Carrying images | By content in the layer (`bundleKey`) or by `uri`; loaded lazily; the existing size limits of an upload apply. |
159
+ | The event stream | No new stream or events: the existing `report` event and `?since=` deltas. |
160
+ | Sharding | No shard concept in the format; a shard is any publisher. A `shard` label is provenance. Completeness comes from the plan, so Atlas needs no shard count. |
161
+ | What moves | Reading, composing, the viewer, the document and the API, the publish builders, a Playwright capture adapter. Staying with a product: its state declarations, its inventory, its fixtures and its runner. |
162
+ | Where it lives | `@crvouga/atlas/reports/ui-catalog` (model), `…/ui-catalog-publish` (builders), `…/ui-state` (convention); the viewer's `/catalog`; the server's `/api/reports/ui-catalog`; `@crvouga/atlas/playwright`'s `captureUiState`. |
163
+ | Exploration capabilities | Search, filters, profile comparison, coverage and surface coverage, failure diagnostics, deep links, manifest export. |
164
+ | Static export | The catalog is computed from the layers a viewer holds, so a static export (`atlas client build`) shows it; it just does not stream. |
165
+
166
+ ## What stays in your project, and what is not built yet
167
+
168
+ **Stays in a project** (it is specific to the product): the list of states and profiles to capture, the inventory of surfaces,
169
+ fixtures and test data, the runner that drives the product to each state, and the readiness conditions of each screen.
170
+
171
+ **Not built yet:** a command line to list and publish a catalog without a script (use `atlas report publish --checks` today), a
172
+ dedicated static single-file export, diff views that compare two commits of a state, and a hosting-platform publish step beyond
173
+ `atlas serve`. The first slice makes the catalog explorable, streaming, shardable and independent of how it was collected.
package/docs/ui.md CHANGED
@@ -96,6 +96,20 @@ when Watch is on screen.
96
96
  - The spec index searches titles, paths and declared domains, and can group by the domain from spec metadata (specs with
97
97
  none are listed under "No domain declared"; the Tests view reports domain problems).
98
98
 
99
+ ## UI catalog
100
+
101
+ `/catalog` (`catalog/`) is the viewer's inventory of captured UI states ([the guide](ui-catalog.md)). It is computed in the page from
102
+ the layers the viewer already holds (`catalog/provider.tsx`, only while the page is open), so states appear as results stream in and
103
+ it works without a server. Cards are Mantine `Card`s in a responsive `SimpleGrid`; the filters live in the shell's navigation column
104
+ (`UiCatalogFilters`) and the open state in a `Drawer`, and both are the address (`catalog/search.ts`).
105
+
106
+ - Only `captured` is called captured. Every other coverage (`Capture failed`, `Capturing`, `Skipped`, `Not captured yet`) has its own
107
+ words and mark, and a cell without a picture says so in words instead of showing an empty frame. Never show a plan as a result.
108
+ - A backend that is not available, a notice, and layers that could not be read are shown above the grid; the catalog says it may be
109
+ incomplete. Waiting follows the loading policy below, and layers still arriving are counted.
110
+ - Pictures are `loading="lazy"` and only the cards on screen are fetched. The grid is bounded with *Show more*.
111
+ - The open state shows every profile side by side with the layer, backend, producer, shard, commit and time of each picture.
112
+
99
113
  ## Reliability
100
114
 
101
115
  An open test shows its reliability (`explorer/Reliability.tsx`, worked out by `reports/reliability` from the history the page
@@ -276,6 +290,7 @@ The narrowly scoped exceptions are:
276
290
  | Area | Screens/interactions | Mantine composition |
277
291
  | --- | --- | --- |
278
292
  | Shared shell | Project switching, spec navigation, run selection, connection status, diagnostics, mobile index | AppShell, Burger, NavLink, Menu, Popover, NativeSelect, Drawer |
293
+ | UI catalog | Captured UI states per profile as cards with coverage badges, filters in the index column, surface coverage, an open state with its profiles side by side and where each picture came from, manifest download | AppShell navbar, SimpleGrid, Card, Image, MultiSelect, TextInput, Drawer, Accordion, Table, Badge, Alert |
279
294
  | Project home | Tests/runs/artifacts tabs, a run's Artifacts (files of its layers, shared recordings, direct links), Watch playlist of a filtered set, metrics, search, status/environment/domain/artifact-tag filters, run and artifact facets with saved views, grouping, sorting, show more | Container, Tabs, SimpleGrid, Paper, TextInput, MultiSelect, Popover, Button, Accordion, Modal, Grid, Switch |
280
295
  | Test detail | Current outcomes, statistics, duration graph, product latency with budgets and baselines, history, artifacts, recording; the `layer` address parameter pins it to one run's result, labelled "This run" against "Latest combined evidence" | Drawer, Table, Badge, Alert, Stage |
281
296
  | Specs | All six existing kinds, overview/index with title search and folder or declared-domain grouping, Gherkin rules/scenarios/examples with a coverage summary and Read / Watch / Inspect per behavior, Quint declarations/traces, DMN tables, source text | SpecHeader, Accordion, Tabs, Table, CodeHighlightTabs, ResultRow |