@crvouga/atlas 1.12.0 → 1.14.0

package/dist/reports/src/report-status.js

@@ -11,7 +11,7 @@
 import { REPORT_STATUS_SCHEMA_VERSION } from '../../schema/src/layer.js';
 import { countResults } from './compose.js';
 import { normalizeEnvironment } from './layer-axes.js';
-import { isDiagnosticKind } from './layer-merge.js';
+import { isDiagnosticKind, isOpinionKind } from './layer-merge.js';
 import { recordingCompleteness } from './recording-receipt.js';
 /** Total string order by code unit. */
 const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
@@ -23,7 +23,7 @@ const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
  * checks (or none completed), failed checks, running checks, specs without results (only with `requireAllSpecs`),
  * each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
  * report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
- * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
+ * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded, and `critique` opinions are always left out.
  * `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
  * completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
  *
@@ -34,7 +34,8 @@ const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
  * @see docs/report-contract.md#atlas-report-status
  */
 export function reportStatus(report, options = {}) {
-    const results = [...report.combined.specs.flatMap(entry => entry.results), ...report.combined.unassigned];
+    // A critique is an opinion: a low score or a crashed critic never fails the gate (see `isOpinionKind`).
+    const results = [...report.combined.specs.flatMap(entry => entry.results), ...report.combined.unassigned].filter(result => !isOpinionKind(result.row.kind));
     const outcomes = results.some(result => !isDiagnosticKind(result.row.kind)) ? results.filter(result => !isDiagnosticKind(result.row.kind)) : results;
     const failing = outcomes
         .filter(result => result.row.status === 'failed' || result.row.status === 'running')
@@ -98,7 +99,8 @@ export function reportStatus(report, options = {}) {
 }
 /** `spec › check`, unless the check's id already starts with its spec. */
 const label = (item) => (item.spec && !item.check.startsWith(item.spec) ? `${item.spec} › ${item.check}` : item.check);
-function specOutcome(path, title, kind, results) {
+function specOutcome(path, title, kind, all) {
+    const results = all.filter(result => !isOpinionKind(result.row.kind));
     const environments = new Map();
     for (const result of results) {
         const key = result.layer.environment ?? 'unspecified';

package/dist/reports/src/ui-catalog-publish.d.ts

@@ -0,0 +1,92 @@
+/**
+ * @module Writing a UI catalog from any collector
+ *
+ * Two small builders that turn what a collector captured, or what a planner expects, into the input of `publishChecks`
+ * (and of `atlas report publish --checks`), following the convention in {@link ./ui-state}. They are plain data in and plain
+ * data out: no browser, no test framework, no file system. A Playwright test, a Cypress run, a mobile device lab, a Storybook
+ * crawler or a shell script that writes the same JSON all produce the same catalog.
+ *
+ * Shards need no coordination: each shard calls {@link uiStateLayer} for the states it captured (and optionally
+ * {@link uiPlanLayer} for the states it was given), with its own `shard` label, and publishes. A planner that knows the whole
+ * product publishes the plan once.
+ *
+ * @see docs/ui-catalog.md
+ */
+import type { PublishChecksInput } from '../../schema/src/layer.js';
+import { type UiStateFacts } from './ui-state.js';
+/** A picture to store with a capture: its bytes (`data`, `base64`), or where it already is (`uri`). */
+export type UiPicture = {
+    /** The file name; default `<id>.png`. */
+    readonly name?: string;
+    readonly data?: Uint8Array;
+    readonly base64?: string;
+    readonly uri?: string;
+    /** Default `image/png`. */
+    readonly contentType?: string;
+    readonly width?: number;
+    readonly height?: number;
+};
+/** One captured state. `id` must be stable across runs: it is how a new capture replaces an old one. */
+export type UiStateCapture = UiStateFacts & {
+    readonly id: string;
+    /** For a person; default the id. */
+    readonly title?: string;
+    /** `passed` (default): captured. `failed`: the capture was attempted and did not work (say why in `detail`). `running`: being taken. `not-run`: skipped. */
+    readonly status?: 'passed' | 'failed' | 'running' | 'not-run';
+    /** Why it failed or was skipped: a readiness check that never became true, a console error. */
+    readonly detail?: string;
+    readonly screenshot?: UiPicture;
+    readonly durationMs?: number;
+    /** More facts about the capture that never split it (`browser`, `os`). */
+    readonly labels?: Readonly<Record<string, string>>;
+    readonly tags?: readonly string[];
+};
+/** One expected state, or (with `inventory`) one surface of the product's inventory. */
+export type UiStatePlan = UiStateFacts & {
+    readonly id: string;
+    readonly title?: string;
+};
+/** What every layer of a catalog says about where and when it was recorded. */
+export type UiLayerOrigin = {
+    /** The suite or tool that makes the layer. Part of the slot; a planner and a collector normally differ. */
+    readonly producer: string;
+    /** When the captures were taken (or the plan was made), epoch milliseconds. Take it from the run, not from the moment of publishing. */
+    readonly recordedAtMs: number;
+    /** The client: `desktop-web`, `mobile-web`, `ios`, `android`… Part of the slot. */
+    readonly environment?: string;
+    /** The rest of the profile, such as `{ viewport: '390x844', scheme: 'dark' }`. Each combination is its own result. Part of the slot. */
+    readonly profile?: Readonly<Record<string, string>>;
+    readonly sha?: string | null;
+    readonly branch?: string | null;
+    readonly runUrl?: string | null;
+    /** Which shard this is (`2/4`, `checkout`, a machine name). Shown with each capture; never part of the slot. */
+    readonly shard?: string;
+    /** The execution these layers belong to, so shards of one run stay one run. */
+    readonly run?: {
+        readonly id: string;
+        readonly title?: string;
+    };
+};
+/**
+ * The input of `publishChecks` for the states one collector (or one shard) captured under one profile. Throws when a state id
+ * repeats, because the second would silently replace the first. A state without a `status` is `passed`: it was captured.
+ *
+ * @example
+ * ```ts
+ * await publishChecks(store, uiStateLayer({
+ *   producer: 'web-capture', recordedAtMs: Date.now(), environment: 'mobile-web', profile: { viewport: '390x844', scheme: 'dark' }, shard: '1/4',
+ *   states: [{ id: 'checkout/payment', title: 'Payment', route: '/checkout', covers: ['payment-form'], screenshot: { data: png } }]
+ * }));
+ * ```
+ */
+export declare function uiStateLayer(input: UiLayerOrigin & {
+    readonly states: readonly UiStateCapture[];
+}): PublishChecksInput;
+/**
+ * The input of `publishChecks` for what is expected: states that should be captured under this profile and, with `inventory`,
+ * surfaces the product is known to have. Every entry is `not-run`, so a plan never counts as a pass, and it fills a slot of
+ * its own (kind `ui-plan`), so a capture never erases it and it never hides a capture. Throws when an id repeats.
+ */
+export declare function uiPlanLayer(input: UiLayerOrigin & {
+    readonly states: readonly UiStatePlan[];
+}): PublishChecksInput;

package/dist/reports/src/ui-catalog-publish.js

@@ -0,0 +1,68 @@
+import { UI_PLAN_KIND, UI_STATE_EXTENSION, UI_STATE_KIND } from './ui-state.js';
+function common(origin, kind, title) {
+    return {
+        producer: origin.producer, kind, title, recordedAtMs: origin.recordedAtMs,
+        ...(origin.environment ? { environment: origin.environment } : {}),
+        parameters: { ...origin.profile },
+        ...(origin.sha !== undefined ? { sha: origin.sha } : {}), ...(origin.branch !== undefined ? { branch: origin.branch } : {}), ...(origin.runUrl !== undefined ? { runUrl: origin.runUrl } : {}),
+        ...(origin.run ? { run: origin.run } : {})
+    };
+}
+function facts(source) {
+    const { route, fixture, cohort, area, flags, covers, inventory } = source;
+    return { ...(route ? { route } : {}), ...(fixture ? { fixture } : {}), ...(cohort ? { cohort } : {}), ...(area ? { area } : {}), ...(flags?.length ? { flags } : {}), ...(covers?.length ? { covers } : {}), ...(inventory ? { inventory } : {}) };
+}
+/**
+ * The input of `publishChecks` for the states one collector (or one shard) captured under one profile. Throws when a state id
+ * repeats, because the second would silently replace the first. A state without a `status` is `passed`: it was captured.
+ *
+ * @example
+ * ```ts
+ * await publishChecks(store, uiStateLayer({
+ *   producer: 'web-capture', recordedAtMs: Date.now(), environment: 'mobile-web', profile: { viewport: '390x844', scheme: 'dark' }, shard: '1/4',
+ *   states: [{ id: 'checkout/payment', title: 'Payment', route: '/checkout', covers: ['payment-form'], screenshot: { data: png } }]
+ * }));
+ * ```
+ */
+export function uiStateLayer(input) {
+    const seen = new Set();
+    const checks = input.states.map(state => {
+        if (seen.has(state.id))
+            throw new Error(`uiStateLayer: the state "${state.id}" appears twice; give each state of one profile its own id`);
+        seen.add(state.id);
+        const shot = state.screenshot;
+        const attachments = shot ? [{
+                name: shot.name ?? `${state.id.replace(/[^\w.-]+/g, '-')}.png`, role: 'screenshot', contentType: shot.contentType ?? 'image/png',
+                ...(shot.data ? { data: shot.data } : {}), ...(shot.base64 ? { base64: shot.base64 } : {}), ...(shot.uri ? { uri: shot.uri } : {}),
+                ...(shot.width !== undefined ? { width: shot.width } : {}), ...(shot.height !== undefined ? { height: shot.height } : {})
+            }] : undefined;
+        const labels = { ...state.labels, ...(input.shard ? { shard: input.shard } : {}) };
+        return {
+            id: state.id, title: state.title ?? state.id, status: state.status ?? 'passed', detail: state.detail ?? '',
+            ...(state.durationMs !== undefined ? { durationMs: state.durationMs } : {}),
+            ...(attachments ? { attachments } : {}),
+            ...(Object.keys(labels).length ? { labels } : {}), ...(state.tags?.length ? { tags: [...state.tags] } : {}),
+            extensions: { [UI_STATE_EXTENSION]: facts(state) }
+        };
+    });
+    return { ...common(input, UI_STATE_KIND, 'UI states'), checks };
+}
+/**
+ * The input of `publishChecks` for what is expected: states that should be captured under this profile and, with `inventory`,
+ * surfaces the product is known to have. Every entry is `not-run`, so a plan never counts as a pass, and it fills a slot of
+ * its own (kind `ui-plan`), so a capture never erases it and it never hides a capture. Throws when an id repeats.
+ */
+export function uiPlanLayer(input) {
+    const seen = new Set();
+    const checks = input.states.map(state => {
+        if (seen.has(state.id))
+            throw new Error(`uiPlanLayer: "${state.id}" appears twice`);
+        seen.add(state.id);
+        return {
+            id: state.id, title: state.title ?? state.id, status: 'not-run', detail: state.inventory ? 'In the product inventory' : 'Expected, not captured yet',
+            ...(input.shard ? { labels: { shard: input.shard } } : {}),
+            extensions: { [UI_STATE_EXTENSION]: facts(state) }
+        };
+    });
+    return { ...common(input, UI_PLAN_KIND, 'UI states expected'), checks };
+}

package/dist/reports/src/ui-catalog.d.ts

@@ -0,0 +1,229 @@
+/**
+ * @module UI catalog
+ *
+ * A browsable inventory of a product's user interface, computed from layers and nothing else: each **state** (a screen, or
+ * one state of a screen) captured once per **profile** (viewport, color scheme, device), with the facts needed to find and
+ * trust it, and what should exist but has not been captured yet. `uiCatalog` is a pure function of the winning results of
+ * the layers it is given, so it inherits the report contract's laws: the same layers in any order, split any way across
+ * backends, shards and machines, give the same catalog; a layer changes only the cells it fills; an unreadable layer is
+ * absent, never captured. It imports no collection framework and knows nothing about how a picture was taken.
+ *
+ * How the convention maps onto the layer format is in {@link ./ui-state}: a state is a check of a `ui-state` part, a profile
+ * is the layer's `environment` and `parameters`, the expected set is a `ui-plan` part. The viewer computes the catalog from
+ * the report it already streams (so it fills in as results arrive), and the server answers the same document at
+ * `GET /api/reports/ui-catalog`.
+ *
+ * @see docs/ui-catalog.md
+ * @see docs/report-contract.md#how-layers-compose
+ */
+import type { BackendInfo, LayerEvidence, LayerHeader } from './evidence-types.js';
+import type { EvidenceStatus } from './explorer-types.js';
+import { type UiScreenshot } from './ui-state.js';
+/** The `schemaVersion` of the catalog document. */
+export declare const UI_CATALOG_VERSION = 1;
+/** One profile: the client and the choices a state was captured under. Its `key` identifies it across layers. */
+export type UiProfile = {
+    /** Stable identity: the environment and the sorted parameters. */
+    readonly key: string;
+    /** The client (`desktop-web`, `ios`…), or null when the layer named none. */
+    readonly environment: string | null;
+    /** The layer's other choices, such as `{ viewport: '390x844', scheme: 'dark' }`. */
+    readonly parameters: Readonly<Record<string, string>>;
+    /** For a person: the client and each parameter's value. */
+    readonly label: string;
+};
+/**
+ * How one state stands in one profile. `captured`: a capture passed. `failed`: it was attempted and failed (its `detail`
+ * says why). `capturing`: it is being taken now. `skipped`: a producer recorded it as not run. `missing`: expected, and
+ * nothing has been recorded. Only `captured` is ever shown as captured; every other value means there is no picture to trust.
+ */
+export type UiCoverage = 'captured' | 'failed' | 'capturing' | 'skipped' | 'missing';
+/** The values of {@link UiCoverage}, in the order a person reads them. */
+export declare const UI_COVERAGE: readonly ["captured", "failed", "capturing", "skipped", "missing"];
+/** A picture, with the address a browser loads it from. */
+export type UiImage = UiScreenshot & {
+    /** Where the stored bytes are, relative to the reports API root (`/layers/<layer>/blobs/attachments/<sha256>`). Absent for an address-only or omitted picture. */
+    readonly path?: string;
+};
+/** Where a capture came from: the layer, the backend that holds it, and the provenance the layer recorded. */
+export type UiOrigin = {
+    readonly layer: string;
+    readonly backend: string;
+    readonly producer: string;
+    readonly recordedAtMs: number;
+    readonly sha: string | null;
+    readonly branch: string | null;
+    readonly runUrl: string | null;
+    readonly freshness?: 'unknown' | 'modified';
+};
+/** The winning capture of one cell. */
+export type UiCapture = {
+    readonly status: EvidenceStatus;
+    /** Why it failed or was skipped; empty otherwise. */
+    readonly detail: string;
+    readonly durationMs: number | null;
+    /** The picture, or null when the capture has none (or it was left out: see `omitted`). */
+    readonly image: UiImage | null;
+    /** The check's labels: what describes the capture without splitting it (a `shard`, a `browser`). */
+    readonly labels: Readonly<Record<string, string>>;
+    readonly origin: UiOrigin;
+    /** How many other captures of this cell exist from other producers. The winner is the one the report contract picks. */
+    readonly alternatives: number;
+};
+/** One state in one profile. */
+export type UiCell = {
+    readonly profile: UiProfile;
+    readonly coverage: UiCoverage;
+    /** Whether a plan expects this state in this profile: `declared`; `undeclared` when plans exist and none names it; `unknown` when no plan was published at all. */
+    readonly expected: 'declared' | 'undeclared' | 'unknown';
+    readonly capture: UiCapture | null;
+    /** The layer of the plan that expects it, when one does. */
+    readonly planned: {
+        readonly layer: string;
+        readonly backend: string;
+    } | null;
+};
+/** One state of the product's interface, across its profiles. */
+export type UiCatalogState = {
+    /** The check id the producers gave it. */
+    readonly id: string;
+    readonly title: string;
+    readonly route: string | null;
+    readonly fixture: string | null;
+    readonly cohort: string | null;
+    readonly area: string | null;
+    readonly flags: readonly string[];
+    /** The surfaces it covers. */
+    readonly covers: readonly string[];
+    /** The producers that captured or planned it. */
+    readonly producers: readonly string[];
+    /** One cell per profile, ordered by the profile's label. */
+    readonly cells: readonly UiCell[];
+};
+/** One surface of the product: an inventory entry, or something a state says it covers. */
+export type UiSurface = {
+    readonly id: string;
+    /** True when a `ui-plan` inventory entry names it: the product is known to have it. */
+    readonly inventoried: boolean;
+    /** States that cover it. */
+    readonly states: number;
+    /** Of those, states with at least one captured profile. */
+    readonly captured: number;
+    /** `covered`: some covering state is captured. `uncaptured`: states cover it and none is captured yet. `uncovered`: no state covers it. */
+    readonly status: 'covered' | 'uncaptured' | 'uncovered';
+};
+/** A value of a facet and how many states have it. */
+export type UiFacetValue = {
+    readonly value: string;
+    readonly count: number;
+    readonly label?: string;
+};
+/** The values each filter offers, over every state of the catalog. */
+export type UiCatalogFacets = {
+    readonly profiles: readonly UiFacetValue[];
+    readonly coverage: readonly UiFacetValue[];
+    readonly areas: readonly UiFacetValue[];
+    readonly routes: readonly UiFacetValue[];
+    readonly cohorts: readonly UiFacetValue[];
+    readonly flags: readonly UiFacetValue[];
+    readonly surfaces: readonly UiFacetValue[];
+    readonly producers: readonly UiFacetValue[];
+};
+/** The catalog in numbers. Cells count a state once per profile. */
+export type UiCatalogSummary = {
+    readonly states: number;
+    readonly profiles: number;
+    readonly cells: number;
+    readonly coverage: Readonly<Record<UiCoverage, number>>;
+    /** Captured or failed cells no plan expects. Zero when no plan was published, because then nothing is known to be unexpected. */
+    readonly undeclared: number;
+    /** Whether any plan (an expected set) was published. Without one, `missing` can only ever be zero. */
+    readonly planned: boolean;
+    readonly surfaces: number;
+    readonly uncoveredSurfaces: number;
+};
+/** The catalog, as `GET /api/reports/ui-catalog` answers it (`version` is the report's, when the caller knows it). */
+export type UiCatalog = {
+    readonly schemaVersion: typeof UI_CATALOG_VERSION;
+    readonly version?: string;
+    readonly summary: UiCatalogSummary;
+    readonly profiles: readonly UiProfile[];
+    readonly states: readonly UiCatalogState[];
+    readonly surfaces: readonly UiSurface[];
+    readonly facets: UiCatalogFacets;
+    /** What could not be read. A state that depended on it is not in the catalog as captured. Filled by the caller who read the backends. */
+    readonly sources: {
+        readonly layers: number;
+        readonly backends: readonly BackendInfo[];
+        readonly notices: readonly string[];
+    };
+};
+/** What a caller may say about how the layers were read. */
+export type UiCatalogOptions = {
+    /** The report's version, repeated in the document. */
+    readonly version?: string;
+    /** The backends, with whether each could be read. */
+    readonly backends?: readonly BackendInfo[];
+    /** What could not be read. */
+    readonly notices?: readonly string[];
+};
+/** The profile of a layer: its environment and its other parameters. */
+export declare function uiProfile(layer: Pick<LayerHeader, 'environment' | 'parameters'>): UiProfile;
+/**
+ * The UI catalog of some layers (as the effective report holds them, or any set of {@link LayerEvidence}).
+ *
+ * Only rows of `ui-state` and `ui-plan` parts take part. The winner of every slot is decided by the report contract
+ * (`combineEvidence`), so the result does not depend on the order of `layers` or on how they are divided among backends. A
+ * state is the set of cells that share a check `id`; a cell is one profile of it. A cell is `captured` only when the winning
+ * capture passed; a plan that nothing has captured is `missing`. When no plan exists nothing is `missing` and nothing is
+ * `undeclared`: what is expected is then unknown, and the catalog says so (`summary.planned`).
+ *
+ * @example
+ * ```ts
+ * const catalog = uiCatalog(effective.layers, { version: effective.version, backends: effective.backends });
+ * catalog.states.map(state => [state.id, state.cells.map(cell => cell.coverage)]);
+ * ```
+ */
+export declare function uiCatalog(layers: readonly LayerEvidence[], options?: UiCatalogOptions): UiCatalog;
+/** What a person narrows the catalog by. Every field is optional; nothing asked is everything. Values within a field are alternatives, fields combine with AND. */
+export type UiCatalogQuery = {
+    /** Words (all of them) found in the id, title, route, fixture, cohort, area, flags, surfaces or producers. */
+    readonly q?: string;
+    readonly coverage?: readonly UiCoverage[];
+    /** Profile keys, as {@link UiProfile.key}. */
+    readonly profile?: readonly string[];
+    readonly area?: readonly string[];
+    readonly route?: readonly string[];
+    readonly cohort?: readonly string[];
+    readonly flag?: readonly string[];
+    /** Surfaces a state covers. */
+    readonly surface?: readonly string[];
+    readonly producer?: readonly string[];
+};
+/** Reads a query from URL search parameters (`q`, and the comma-separated {@link UiCatalogQuery} lists). Unknown coverage values are left out, never an error. */
+export declare function parseUiCatalogQuery(params: URLSearchParams): UiCatalogQuery;
+/** A query as URL search parameters, the form {@link parseUiCatalogQuery} reads back. */
+export declare function uiCatalogParams(query: UiCatalogQuery): URLSearchParams;
+/** Whether a query narrows anything. */
+export declare function isUiCatalogFiltered(query: UiCatalogQuery): boolean;
+/**
+ * The part of a catalog a query asks for. A state is kept when it matches the words and the fact filters; its cells are
+ * kept when they match the `profile` and `coverage` filters, and a state left with no cell is dropped. The summary is
+ * recomputed for what is kept; the profiles, surfaces, facets and sources stay those of the whole catalog, so the
+ * controls keep offering every choice.
+ */
+export declare function filterUiCatalog(catalog: UiCatalog, query: UiCatalogQuery): UiCatalog;
+/** A page of a catalog's states, for a document that would otherwise be large. The summary, facets and sources are the whole catalog's. */
+export declare function pageUiCatalog(catalog: UiCatalog, page: {
+    readonly limit?: number;
+    readonly offset?: number;
+}): UiCatalog & {
+    readonly total: number;
+    readonly offset: number;
+    readonly limit: number;
+};
+/**
+ * The catalog as Markdown for a person or an agent reading text: the summary in words, the surfaces that are not covered, and
+ * one line per state with its coverage in each profile. Only `captured` is called captured.
+ */
+export declare function uiCatalogMarkdown(catalog: UiCatalog): string;