@crvouga/atlas 1.12.0 → 1.14.0

package/dist/critics/src/runner.js

@@ -0,0 +1,202 @@
+/**
+ * @module The critic runner
+ *
+ * Runs one critic over entries of the UI catalog and returns the critiques to publish. The runner, not the critic, owns
+ * everything around the judgement: skipping what was already judged (same critic, version, model, parameters and image
+ * hash), fetching images only when needed, a bounded number of concurrent calls, a per-call timeout, retries with
+ * jittered backoff that honour a provider's `Retry-After`, budgets (images, tokens, money) that stop the run cleanly,
+ * and credential redaction of every message it records. Failures are data: a crash, a timeout, a rate limit that never
+ * cleared and an unparseable answer each become a critique with a `failure` and a reason code, which the next run
+ * retries, and never a score.
+ *
+ * Results are returned in the order of the entries, so the same inputs publish the same layer. Nothing here reads a
+ * clock, the environment or the network on its own: images, time, sleeping and randomness are passed in.
+ *
+ * @see docs/critics.md#running-critics
+ */
+import { z } from 'zod/v4';
+import { sha256Hex } from '../../reports/src/sigv4.js';
+import { CriticError, InvalidOutputError, RateLimitedError } from './contract.js';
+import { buildCritique, critiqueIdentity, identityOfCritique } from './critique.js';
+import { createRedactor } from './credentials.js';
+/** The shape a critic's answer must have. A score that is not a finite number, or a finding with a bad region, is `invalid-output`. */
+const AnswerSchema = z.object({
+    verdict: z.looseObject({ score: z.number().finite().optional(), confidence: z.number().min(0).max(1).optional() }).optional(),
+    findings: z.array(z.looseObject({ confidence: z.number().min(0).max(1).optional() })).optional(),
+    usage: z.looseObject({ inputTokens: z.number().nonnegative().optional(), outputTokens: z.number().nonnegative().optional(), latencyMs: z.number().nonnegative().optional(), costUsd: z.number().nonnegative().optional() }).optional()
+}).refine(answer => answer.verdict !== undefined || (answer.findings !== undefined && answer.findings.length > 0), 'the answer has neither a verdict nor findings');
+const defaultSleep = (ms, signal) => new Promise(resolve => {
+    const timer = setTimeout(done, ms);
+    function done() { clearTimeout(timer); signal?.removeEventListener('abort', done); resolve(); }
+    signal?.addEventListener('abort', done, { once: true });
+});
+const sha256Bytes = (bytes) => sha256Hex(bytes);
+/** Races a critic's call against a timeout and the run's cancellation. Rejects with `CriticError('timeout')` on timeout. */
+async function withTimeout(work, timeoutMs, outer) {
+    const controller = new AbortController();
+    const onOuter = () => controller.abort(outer?.reason);
+    if (outer?.aborted)
+        controller.abort(outer.reason);
+    else
+        outer?.addEventListener('abort', onOuter, { once: true });
+    let timer;
+    try {
+        return await Promise.race([
+            work(controller.signal),
+            new Promise((_, reject) => {
+                // Settle first: aborting may make the critic's own call reject at once, and the reason must be the timeout.
+                timer = setTimeout(() => { reject(new CriticError('timeout', `The critic did not answer within ${timeoutMs} ms`)); controller.abort(new Error('timeout')); }, timeoutMs);
+            })
+        ]);
+    }
+    finally {
+        if (timer !== undefined)
+            clearTimeout(timer);
+        outer?.removeEventListener('abort', onOuter);
+    }
+}
+/** Collects a sync or async iterable, keeping the first of each key. */
+async function collect(entries) {
+    const seen = new Set();
+    const out = [];
+    for await (const entry of entries) {
+        if (seen.has(entry.key))
+            continue;
+        seen.add(entry.key);
+        out.push(entry);
+    }
+    return out;
+}
+/**
+ * Runs a critic over entries. Entries with a current critique of the same identity are skipped without fetching their
+ * image; the rest are judged with bounded concurrency, a timeout and retries. The run never throws for a critic's
+ * failure: it records it. It throws only for a wrong call (a non-positive `concurrency`, `maxAttempts` or `timeoutMs`).
+ *
+ * A run stops cleanly when a budget is reached or the signal aborts: calls in flight finish (or are aborted), what was
+ * finished is returned in `outcomes` to publish, and `stoppedBy` says why. With concurrency above 1 a budget can be
+ * overshot by the calls already in flight.
+ *
+ * @example
+ * const result = await runCritic({ critic, entries, loadImage: entry => fetchImage(entry), existing, budget: { maxImages: 200 } });
+ * await publishChecks(store, critiqueLayer({ critic, outcomes: result.outcomes, recordedAtMs: Date.now() }));
+ */
+export async function runCritic(options) {
+    const concurrency = options.concurrency ?? 4;
+    const maxAttempts = options.maxAttempts ?? 4;
+    const timeoutMs = options.timeoutMs ?? 60_000;
+    for (const [name, value] of [['concurrency', concurrency], ['maxAttempts', maxAttempts], ['timeoutMs', timeoutMs]]) {
+        if (!Number.isFinite(value) || value < 1)
+            throw new Error(`Invalid ${name}: ${value}`);
+    }
+    const sleep = options.sleep ?? defaultSleep;
+    const random = options.random ?? Math.random;
+    const redact = createRedactor(options.secrets ?? []);
+    const { critic } = options;
+    const description = critic.describe();
+    const entries = await collect(options.entries);
+    const known = new Set();
+    if (!options.force)
+        for (const critique of options.existing ?? [])
+            if (critique.verdict !== undefined && !critique.failure)
+                known.add(identityOfCritique(critique));
+    const isKnown = (entry) => known.has(critiqueIdentity({ critic, model: description.model, parameters: description.parameters, imageSha256: entry.imageSha256 }));
+    const slots = entries.map(entry => (isKnown(entry) ? 'skipped' : undefined));
+    const identity = { name: critic.name, version: critic.version };
+    const toJudge = slots.filter(slot => slot === undefined).length;
+    if (options.dryRun) {
+        return { critic: identity, sendsImagesTo: description.sendsImagesTo, outcomes: [], skipped: entries.length - toJudge, judged: 0, failed: 0, waiting: toJudge, spent: { images: 0, tokens: 0, costUsd: 0 }, dryRun: true };
+    }
+    const budget = options.budget ?? {};
+    const spent = { images: 0, tokens: 0, costUsd: 0 };
+    let judged = 0, failed = 0, skipped = entries.length - toJudge, stoppedBy;
+    const exhausted = () => (budget.maxImages !== undefined && spent.images >= budget.maxImages) ||
+        (budget.maxTokens !== undefined && spent.tokens >= budget.maxTokens) ||
+        (budget.maxCostUsd !== undefined && spent.costUsd >= budget.maxCostUsd);
+    const progress = (entry, result) => options.onProgress?.({ entryKey: entry.key, result, judged, skipped, failed, waiting: slots.filter(slot => slot === undefined).length - inFlight });
+    let inFlight = 0;
+    const fail = (entry, reason, message) => ({
+        entry,
+        critique: buildCritique(critic, description, entry, { failure: { reason, message: redact(message).slice(0, 500) } })
+    });
+    /** Judges one entry; `null` when the run was cancelled meanwhile, because a cancelled call is not a failure of the critic. */
+    async function judge(entry) {
+        let image;
+        try {
+            image = await options.loadImage(entry);
+        }
+        catch (error) {
+            return fail(entry, 'image-unavailable', error instanceof Error ? error.message : String(error));
+        }
+        if ((await sha256Bytes(image)) !== entry.imageSha256)
+            return fail(entry, 'image-mismatch', 'The image bytes are not the ones the catalog hashed; nothing was judged');
+        for (let attempt = 1;; attempt++) {
+            if (options.signal?.aborted)
+                return null;
+            try {
+                const answer = await withTimeout(signal => critic.evaluate({ image, imageSha256: entry.imageSha256, entry: { key: entry.key, title: entry.title, state: entry.state, ...(entry.route === undefined ? {} : { route: entry.route }), ...(entry.profile === undefined ? {} : { profile: entry.profile }) }, signal }), timeoutMs, options.signal);
+                const checked = AnswerSchema.safeParse(answer);
+                if (!checked.success)
+                    throw new InvalidOutputError(`The answer is not a critique: ${checked.error.issues[0]?.message ?? 'unknown'}`);
+                const usage = answer.usage;
+                spent.tokens += (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0);
+                spent.costUsd += usage?.costUsd ?? 0;
+                return { entry, critique: buildCritique(critic, description, entry, answer) };
+            }
+            catch (error) {
+                if (options.signal?.aborted)
+                    return null;
+                if (error instanceof RateLimitedError && attempt < maxAttempts && !options.signal?.aborted) {
+                    const base = options.backoff?.baseMs ?? 500, cap = options.backoff?.maxMs ?? 30_000;
+                    const jittered = Math.floor(random() * Math.min(cap, base * 2 ** (attempt - 1)));
+                    await sleep(Math.max(error.retryAfterMs ?? 0, jittered), options.signal);
+                    continue;
+                }
+                if (error instanceof CriticError)
+                    return fail(entry, error.reason, error.message);
+                return fail(entry, 'error', error instanceof Error ? error.message : String(error));
+            }
+        }
+    }
+    let next = 0;
+    async function worker() {
+        for (;;) {
+            if (options.signal?.aborted) {
+                stoppedBy ??= 'aborted';
+                return;
+            }
+            if (exhausted()) {
+                stoppedBy ??= 'budget';
+                return;
+            }
+            let index = next;
+            while (index < entries.length && slots[index] !== undefined)
+                index++;
+            if (index >= entries.length)
+                return;
+            next = index + 1;
+            const entry = entries[index];
+            inFlight += 1;
+            spent.images += 1;
+            const outcome = await judge(entry);
+            inFlight -= 1;
+            if (!outcome) {
+                stoppedBy ??= 'aborted';
+                return;
+            }
+            slots[index] = outcome;
+            if (outcome.critique.failure)
+                failed += 1;
+            else
+                judged += 1;
+            progress(entry, outcome.critique.failure ? 'failed' : 'judged');
+        }
+    }
+    for (let index = 0; index < entries.length; index++)
+        if (slots[index] === 'skipped')
+            progress(entries[index], 'skipped');
+    await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(toJudge, 1)) }, worker));
+    // A run that handled every entry did not stop early, whatever the budget now says.
+    const waiting = slots.filter(slot => slot === undefined).length;
+    const outcomes = slots.filter((slot) => slot !== undefined && slot !== 'skipped');
+    return { critic: identity, sendsImagesTo: description.sendsImagesTo, outcomes, skipped, judged, failed, waiting, ...(waiting > 0 && stoppedBy ? { stoppedBy } : {}), spent, dryRun: false };
+}

package/dist/critics/src/scope.d.ts

@@ -0,0 +1,8 @@
+/**
+ * The rule a layer breaks for a critic-scoped credential, or `null` when it is within scope. The message says which rule
+ * and what was found, so the critic can fix it.
+ */
+export declare function criticScopeViolation(layer: {
+    readonly producer?: unknown;
+    readonly parts?: unknown;
+}): string | null;

package/dist/critics/src/scope.js

@@ -0,0 +1,26 @@
+/**
+ * @module The scope of a critic's credential
+ *
+ * A critic only needs to add opinions, so its credential may read and publish layers whose producer is `critic.<name>`
+ * and whose parts are all of kind `critique`, and nothing else. `criticScopeViolation` names the rule a layer breaks;
+ * the server applies it to every layer a critic-role token uploads, so a stolen critic token cannot forge test results.
+ *
+ * @see docs/critics.md#authentication-and-authorization
+ */
+import { CRITIC_PRODUCER_PREFIX, CRITIQUE_PART_KIND } from '../../schema/src/critique.js';
+/**
+ * The rule a layer breaks for a critic-scoped credential, or `null` when it is within scope. The message says which rule
+ * and what was found, so the critic can fix it.
+ */
+export function criticScopeViolation(layer) {
+    if (typeof layer.producer !== 'string' || !layer.producer.startsWith(CRITIC_PRODUCER_PREFIX) || layer.producer.length === CRITIC_PRODUCER_PREFIX.length) {
+        return `A critic token may publish only layers whose producer is "${CRITIC_PRODUCER_PREFIX}<name>"; this layer's producer is ${JSON.stringify(layer.producer ?? null)}`;
+    }
+    const parts = Array.isArray(layer.parts) ? layer.parts : [];
+    if (!parts.length)
+        return `A critic token may publish only layers of parts of kind "${CRITIQUE_PART_KIND}"; this layer has no parts`;
+    const other = parts.find(part => part?.kind !== CRITIQUE_PART_KIND);
+    if (other)
+        return `A critic token may publish only parts of kind "${CRITIQUE_PART_KIND}"; this layer has a part of kind ${JSON.stringify(other?.kind ?? null)}`;
+    return null;
+}

package/dist/playwright/src/index.d.ts

@@ -6,8 +6,12 @@
  * the media tools (`captureScreen`, `ScreencastRecorder`, `ffmpeg`, `PHONE` and the pacing presets). Needs the
  * `playwright` package; clips and thumbnails need FFmpeg on the PATH.
  *
+ * Also `captureUiState`, the optional Playwright adapter of the UI catalog (it needs no Playwright import itself).
+ *
  * @see docs/statecharts.md
+ * @see docs/ui-catalog.md
  */
 export { playwrightDriver, type PlaywrightContext, type PlaywrightDriverOptions } from './driver.js';
 export { captureScreen, ScreencastRecorder, UserActions, PHONE, SHOWCASE_PACING, FAST_PACING, ffmpeg, type Pacing } from './media.js';
 export { recognize, checkSignals, screen, locate, describeSignal, testId, text, role, label, css, type Signal, type SignatureSpec } from './signals.js';
+export { captureUiState, type CapturePage, type CaptureUiStateOptions } from './ui-catalog.js';

package/dist/playwright/src/index.js

@@ -6,8 +6,12 @@
  * the media tools (`captureScreen`, `ScreencastRecorder`, `ffmpeg`, `PHONE` and the pacing presets). Needs the
  * `playwright` package; clips and thumbnails need FFmpeg on the PATH.
  *
+ * Also `captureUiState`, the optional Playwright adapter of the UI catalog (it needs no Playwright import itself).
+ *
  * @see docs/statecharts.md
+ * @see docs/ui-catalog.md
  */
 export { playwrightDriver } from './driver.js';
 export { captureScreen, ScreencastRecorder, UserActions, PHONE, SHOWCASE_PACING, FAST_PACING, ffmpeg } from './media.js';
 export { recognize, checkSignals, screen, locate, describeSignal, testId, text, role, label, css } from './signals.js';
+export { captureUiState } from './ui-catalog.js';

package/dist/playwright/src/ui-catalog.d.ts

@@ -0,0 +1,51 @@
+/**
+ * @module Playwright adapter for the UI catalog
+ *
+ * The optional Playwright side of the UI catalog: take a page to a state, wait until it is ready, photograph it, and hand the
+ * result to `uiStateLayer` (`@crvouga/atlas/reports/ui-catalog-publish`) as a capture. This is the only Playwright-specific code
+ * of the feature; reading, composing and exploring the catalog know nothing about it, and any other program that writes the
+ * same layer produces the same catalog. The page is typed by the three methods used, so no `playwright` import is needed here
+ * and a Playwright Test `page`, a `Page` of the driver, or a stub all fit.
+ *
+ * @see docs/ui-catalog.md
+ */
+import type { UiStateCapture } from '../../reports/src/ui-catalog-publish.js';
+import type { UiStateFacts } from '../../reports/src/ui-state.js';
+/** The part of a Playwright `Page` that capturing uses. */
+export type CapturePage = {
+    screenshot(options?: {
+        readonly fullPage?: boolean;
+        readonly animations?: 'disabled' | 'allow';
+        readonly type?: 'png';
+    }): Promise<Uint8Array>;
+    viewportSize(): {
+        readonly width: number;
+        readonly height: number;
+    } | null;
+};
+/** What to capture and how to know the page is ready. */
+export type CaptureUiStateOptions = UiStateFacts & {
+    /** The state's stable id: how a later capture replaces this one. */
+    readonly id: string;
+    readonly title?: string;
+    /** Resolves when the page shows the state (a locator is visible, the network is idle). A rejection or a timeout makes the capture `failed`, with the reason as its detail. */
+    readonly ready?: (page: CapturePage) => Promise<void>;
+    /** How long to wait for `ready`; default 15 seconds. */
+    readonly readyTimeoutMs?: number;
+    /** Photograph the whole scrollable page rather than the viewport. */
+    readonly fullPage?: boolean;
+    /** Extra labels for the capture (`browser`, `os`), which never split it. */
+    readonly labels?: Readonly<Record<string, string>>;
+};
+/**
+ * Wait for the page to be ready, photograph it, and return the capture to publish. Never throws for the page's sake: a page
+ * that never became ready, or could not be photographed, is a `failed` capture that says why, so a broken state shows in the
+ * catalog instead of ending the run. A readiness failure still attaches a picture of what the page showed, for diagnosis.
+ *
+ * @example
+ * ```ts
+ * const shot = await captureUiState(page, { id: 'checkout/payment', route: '/checkout', ready: p => expectVisible(p, '#total') });
+ * await publishChecks(store, uiStateLayer({ producer: 'playwright', recordedAtMs: Date.now(), environment: 'desktop-web', profile: { viewport: '1280x800' }, states: [shot] }));
+ * ```
+ */
+export declare function captureUiState(page: CapturePage, options: CaptureUiStateOptions): Promise<UiStateCapture>;

package/dist/playwright/src/ui-catalog.js

@@ -0,0 +1,44 @@
+async function within(work, ms, what) {
+    let timer;
+    try {
+        return await Promise.race([work, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`${what} did not become true within ${ms} ms`)), ms); })]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+/**
+ * Wait for the page to be ready, photograph it, and return the capture to publish. Never throws for the page's sake: a page
+ * that never became ready, or could not be photographed, is a `failed` capture that says why, so a broken state shows in the
+ * catalog instead of ending the run. A readiness failure still attaches a picture of what the page showed, for diagnosis.
+ *
+ * @example
+ * ```ts
+ * const shot = await captureUiState(page, { id: 'checkout/payment', route: '/checkout', ready: p => expectVisible(p, '#total') });
+ * await publishChecks(store, uiStateLayer({ producer: 'playwright', recordedAtMs: Date.now(), environment: 'desktop-web', profile: { viewport: '1280x800' }, states: [shot] }));
+ * ```
+ */
+export async function captureUiState(page, options) {
+    const { ready, readyTimeoutMs = 15_000, fullPage = false, labels, ...rest } = options;
+    const started = Date.now();
+    let status = 'passed';
+    let detail = '';
+    if (ready) {
+        try {
+            await within(ready(page), readyTimeoutMs, 'The ready check');
+        }
+        catch (error) {
+            status = 'failed';
+            detail = error instanceof Error ? error.message : String(error);
+        }
+    }
+    const size = page.viewportSize();
+    try {
+        const data = await page.screenshot({ fullPage, animations: 'disabled', type: 'png' });
+        return { ...rest, status, detail, durationMs: Date.now() - started, ...(labels ? { labels } : {}), screenshot: { data, contentType: 'image/png', ...(size && !fullPage ? { width: size.width, height: size.height } : {}) } };
+    }
+    catch (error) {
+        return { ...rest, status: 'failed', detail: [detail, `The picture could not be taken: ${error instanceof Error ? error.message : String(error)}`].filter(Boolean).join('. '), durationMs: Date.now() - started, ...(labels ? { labels } : {}) };
+    }
+}

package/dist/reports/src/compose.d.ts

@@ -50,10 +50,10 @@ export type CombinedReport = {
     readonly summary: ReportSummary;
 };
 /**
- * Counts results by status. Diagnostic results (parts of kind `performance`, which only qualify a check) are left out
+ * Counts results by status. Opinion results (parts of kind `critique`) are never counted. Diagnostic results (parts of kind `performance`, which only qualify a check) are left out
  * whenever any other result is present; if only diagnostics exist, they are counted so the report is not empty.
  */
-export declare function countResults(results: readonly EvidenceResult[]): StatusCounts & {
+export declare function countResults(all: readonly EvidenceResult[]): StatusCounts & {
     total: number;
 };
 /** Counts a combined report's winning results in total and per environment, and lists the specs that have no result at all. */

package/dist/reports/src/compose.js

@@ -11,14 +11,15 @@
  */
 import { eachLimited } from './completion-order.js';
 import { catalogEntries, catalogLayers } from './layer-catalog.js';
-import { combineEvidence, distinctHeaders, emptyCounts, isDiagnosticKind, layerEvidence } from './layer-merge.js';
+import { combineEvidence, distinctHeaders, emptyCounts, isDiagnosticKind, isOpinionKind, layerEvidence } from './layer-merge.js';
 import { RecordedSpecs } from './recorded-specs.js';
 const BACKEND_ID = /^[a-z0-9-]+$/;
 /**
- * Counts results by status. Diagnostic results (parts of kind `performance`, which only qualify a check) are left out
+ * Counts results by status. Opinion results (parts of kind `critique`) are never counted. Diagnostic results (parts of kind `performance`, which only qualify a check) are left out
  * whenever any other result is present; if only diagnostics exist, they are counted so the report is not empty.
  */
-export function countResults(results) {
+export function countResults(all) {
+    const results = all.filter(result => !isOpinionKind(result.row.kind));
     const outcomes = results.some(result => !isDiagnosticKind(result.row.kind)) ? results.filter(result => !isDiagnosticKind(result.row.kind)) : results;
     const counts = emptyCounts();
     for (const result of outcomes)
@@ -27,7 +28,7 @@ export function countResults(results) {
 }
 /** Counts a combined report's winning results in total and per environment, and lists the specs that have no result at all. */
 export function summarize(combined) {
-    const results = [...combined.specs.flatMap(entry => entry.results), ...combined.unassigned];
+    const results = [...combined.specs.flatMap(entry => entry.results), ...combined.unassigned].filter(result => !isOpinionKind(result.row.kind));
     const environments = new Map();
     for (const result of results) {
         const key = result.layer.environment ?? 'unspecified';
@@ -40,7 +41,7 @@ export function summarize(combined) {
     return {
         checks: countResults(results),
         byEnvironment: Object.fromEntries([...environments].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([key, value]) => [key, countResults(value)])),
-        specsWithoutEvidence: combined.specs.filter(entry => entry.results.length === 0).map(entry => entry.spec.path)
+        specsWithoutEvidence: combined.specs.filter(entry => entry.results.every(result => isOpinionKind(result.row.kind))).map(entry => entry.spec.path)
     };
 }
 /**

package/dist/reports/src/evidence-types.d.ts

@@ -11,6 +11,7 @@
 import type { EvidenceMeasurement, EvidenceStatus } from './explorer-types.js';
 import type { EvidenceScope, LayerRun } from '../../schema/src/layer.js';
 import type { LayerParameters } from './layer-types.js';
+import type { UiStateFact } from './ui-state.js';
 /** Where a recorded test's timeline and captured files are read from. */
 export type RecordingAddress = {
     readonly type: 'layer';
@@ -168,6 +169,8 @@ export type LayerRow = {
     readonly durationMs: number | null;
     /** What this result carries beyond its outcome. Absent when nothing. */
     readonly evidence?: EvidenceSummary;
+    /** For a check of a `ui-state` or `ui-plan` part: its picture and declared facts, so a catalog is listed without another request (see `ui-state.ts`). Absent for every other kind. */
+    readonly state?: UiStateFact;
     /** What became of this check's expected recording: `uploaded`, `referenced`, `skipped`, `unresolved` or `not-applicable`, with a reason when it is not uploaded. Absent when the check declared no expectation. Independent of `status`. */
     readonly recording?: {
         readonly state: string;

package/dist/reports/src/evidence-view.d.ts

@@ -32,7 +32,8 @@ export type CheckSlot = {
 /**
  * Groups the combined results (the winners of the slots) by check, in the order the checks were first seen.
  * A check's status is the worst of its non-diagnostic results; if it has only diagnostics (timings), those are its
- * outcomes. The title and id shown are the newest result's.
+ * outcomes. The title and id shown are the newest result's. Opinions (parts of kind `critique`, see `isOpinionKind`) are
+ * not checks and are left out.
  */
 export declare function checkSlots(results: readonly EvidenceResult[]): CheckSlot[];
 /** A spec's rolled-up status: the worst of its checks, or `none` when nothing was recorded for it. */

package/dist/reports/src/evidence-view.js

@@ -1,4 +1,4 @@
-import { emptyCounts, isDiagnosticKind, slotKey } from './layer-merge.js';
+import { emptyCounts, isDiagnosticKind, isOpinionKind, slotKey } from './layer-merge.js';
 const WORST = { passed: 0, 'not-run': 1, running: 2, failed: 3 };
 function worst(statuses) {
     let status = 'passed';
@@ -13,11 +13,14 @@ function newestFirst(a, b) {
 /**
  * Groups the combined results (the winners of the slots) by check, in the order the checks were first seen.
  * A check's status is the worst of its non-diagnostic results; if it has only diagnostics (timings), those are its
- * outcomes. The title and id shown are the newest result's.
+ * outcomes. The title and id shown are the newest result's. Opinions (parts of kind `critique`, see `isOpinionKind`) are
+ * not checks and are left out.
  */
 export function checkSlots(results) {
     const groups = new Map();
     for (const result of results) {
+        if (isOpinionKind(result.row.kind))
+            continue;
         const group = groups.get(result.slot);
         if (group)
             group.push(result);

package/dist/reports/src/layer-merge.d.ts

@@ -34,15 +34,27 @@ export declare function textFingerprint(text: string): string;
 export declare function specPath(path: string): string;
 /** Serialises parameters as JSON key/value pairs sorted by key, so equal parameter sets compare equal as strings. */
 export declare function parameterKey(parameters: LayerParameters): string;
-/** Whether a part `kind` only qualifies an outcome (today only `performance`) and so is not counted as a check when anything else is recorded. */
+/**
+ * Whether a part `kind` only qualifies an outcome and so is not counted as a check when anything else is recorded: a
+ * `performance` timing, or a `ui-plan` entry (what a UI catalog expects, which a capture of the same state replaces in the
+ * tests' view). It never decides a slot: both still fill their own.
+ */
 export declare function isDiagnosticKind(kind: string): boolean;
+/**
+ * Whether a part `kind` records an opinion rather than an outcome (today only `critique`, an automated reviewer's
+ * judgement of a captured UI). An opinion is never counted as a check, never gates, and is not folded into a spec's
+ * rollup, even when it is the only thing recorded: a report of opinions alone has no test results. The check's `status`
+ * of such a part says whether the reviewer ran, not whether anything passed.
+ */
+export declare function isOpinionKind(kind: string): boolean;
 /** A fresh status tally with every status at zero. */
 export declare function emptyCounts(): Record<EvidenceStatus, number>;
 /**
- * Counts rows by status, leaving out the diagnostic rows (see `isDiagnosticKind`) that only qualify an outcome.
- * When every row is diagnostic they are counted, so a layer of timings alone is not empty.
+ * Counts rows by status, leaving out the opinion rows (see `isOpinionKind`) always, and the diagnostic rows (see
+ * `isDiagnosticKind`) that only qualify an outcome. When every remaining row is diagnostic they are counted, so a
+ * layer of timings alone is not empty.
  */
-export declare function outcomeCounts(rows: readonly LayerRow[]): StatusCounts;
+export declare function outcomeCounts(allRows: readonly LayerRow[]): StatusCounts;
 /**
  * Flattens a layer into what composition reads: a header (origin, normalised environment and parameters, counts,
  * evidence summary), one row per check of every part, and the spec fingerprints it recorded.

package/dist/reports/src/layer-merge.js

@@ -1,6 +1,7 @@
 import { summarizeEvidenceIn, summarizeLayerEvidence } from './evidence-detail.js';
 import { effectiveScope } from './evidence-scope.js';
 import { layerPlace } from './layer-axes.js';
+import { UI_PLAN_KIND, uiStateOf } from './ui-state.js';
 /**
  * A short, deterministic fingerprint of a spec's text, shaped `<length>:<hash>:<hash>` (the length in UTF-16 code
  * units and two 32-bit hashes in hex). A layer records it per spec so a reader can warn when the spec changed
@@ -34,19 +35,34 @@ export function specPath(path) {
 export function parameterKey(parameters) {
     return JSON.stringify(Object.entries(parameters).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0));
 }
-/** Whether a part `kind` only qualifies an outcome (today only `performance`) and so is not counted as a check when anything else is recorded. */
+/**
+ * Whether a part `kind` only qualifies an outcome and so is not counted as a check when anything else is recorded: a
+ * `performance` timing, or a `ui-plan` entry (what a UI catalog expects, which a capture of the same state replaces in the
+ * tests' view). It never decides a slot: both still fill their own.
+ */
 export function isDiagnosticKind(kind) {
-    return kind === 'performance';
+    return kind === 'performance' || kind === UI_PLAN_KIND;
+}
+/**
+ * Whether a part `kind` records an opinion rather than an outcome (today only `critique`, an automated reviewer's
+ * judgement of a captured UI). An opinion is never counted as a check, never gates, and is not folded into a spec's
+ * rollup, even when it is the only thing recorded: a report of opinions alone has no test results. The check's `status`
+ * of such a part says whether the reviewer ran, not whether anything passed.
+ */
+export function isOpinionKind(kind) {
+    return kind === 'critique';
 }
 /** A fresh status tally with every status at zero. */
 export function emptyCounts() {
     return { passed: 0, failed: 0, running: 0, 'not-run': 0 };
 }
 /**
- * Counts rows by status, leaving out the diagnostic rows (see `isDiagnosticKind`) that only qualify an outcome.
- * When every row is diagnostic they are counted, so a layer of timings alone is not empty.
+ * Counts rows by status, leaving out the opinion rows (see `isOpinionKind`) always, and the diagnostic rows (see
+ * `isDiagnosticKind`) that only qualify an outcome. When every remaining row is diagnostic they are counted, so a
+ * layer of timings alone is not empty.
  */
-export function outcomeCounts(rows) {
+export function outcomeCounts(allRows) {
+    const rows = allRows.filter(row => !isOpinionKind(row.kind));
     const outcomes = rows.some(row => !isDiagnosticKind(row.kind)) ? rows.filter(row => !isDiagnosticKind(row.kind)) : rows;
     const counts = emptyCounts();
     for (const row of outcomes)
@@ -82,6 +98,7 @@ export function layerEvidence(summary, backend) {
             const test = tests.get(check.id);
             const evidence = summarizeEvidenceIn(effectiveScope(part.scope, summary.scope), check);
             const item = expectations?.get(check.id);
+            const state = uiStateOf(part.kind, check);
             rows.push({
                 kind: part.kind,
                 id: check.id,
@@ -94,6 +111,7 @@ export function layerEvidence(summary, backend) {
                 variant: test ? { system: test.system, runtime: test.runtime, seed: test.seed } : {},
                 durationMs: test?.durationMs ?? check.durationMs ?? null,
                 ...(evidence ? { evidence } : {}),
+                ...(state ? { state } : {}),
                 ...(item ? { recording: { state: item.state, ...(item.reason ? { reason: item.reason } : {}) } } : {}),
             });
         }

package/dist/reports/src/report-index.js

@@ -142,13 +142,15 @@ export class ReportIndex {
     }
     async #refresh() {
         const before = this.#rev;
-        const fresh = [];
         // The whole catalog on the first pass and now and then; otherwise only its newest end, where new layers sort.
         const full = this.#passes++ % this.#fullListEvery === 0;
+        // Each backend is listed and read on its own, so a slow or unreachable one never holds back the layers of the others: the
+        // report fills in as each answers. Oldest first or newest first gives the same report; newest first shows recent work soonest.
         await Promise.all(this.#backends.map(async (backend) => {
+            const fresh = [];
             try {
                 const consider = (key) => { if (!this.#folded.has(`${backend.id}|${key}`) && !this.#recentlyFailed(`${backend.id}|${key}`))
-                    fresh.push({ backend, key }); };
+                    fresh.push(key); };
                 if (!full && isPaged(backend.store)) {
                     // Newest first: pages of keys we already know mean nothing newer is behind them.
                     for (let after;;) {
@@ -175,10 +177,12 @@ export class ReportIndex {
                 this.#available.set(backend.id, false);
                 this.#failed.set(`${backend.id}|`, { at: this.#now(), message: `${backend.label ?? backend.id} is unavailable: ${error instanceof Error ? error.message : String(error)}` });
             }
+            fresh.sort();
+            await eachLimited(fresh, this.#concurrency, key => this.#load(backend, key));
+            // This backend has answered (or been named unavailable): the report is a report, if a partial one, and viewers are told now, not when the slowest backend has answered.
+            this.#refreshedAt ||= this.#now();
+            this.#emit();
         }));
-        // Oldest first or newest first gives the same report; newest first shows recent work soonest.
-        fresh.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : a.backend.id < b.backend.id ? -1 : 1));
-        await eachLimited(fresh, this.#concurrency, ({ backend, key }) => this.#load(backend, key));
         this.#refreshedAt = this.#now();
         this.#emit();
         return this.#rev !== before;

package/dist/reports/src/report-status.d.ts

@@ -25,7 +25,7 @@ export type StatusOptions = {
  * checks (or none completed), failed checks, running checks, specs without results (only with `requireAllSpecs`),
  * each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
  * report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
- * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
+ * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded, and `critique` opinions are always left out.
  * `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
  * completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
  *