@crvouga/atlas 1.12.0 → 1.13.0

package/dist/critics/src/runner.js

@@ -0,0 +1,202 @@
+/**
+ * @module The critic runner
+ *
+ * Runs one critic over entries of the UI catalog and returns the critiques to publish. The runner, not the critic, owns
+ * everything around the judgement: skipping what was already judged (same critic, version, model, parameters and image
+ * hash), fetching images only when needed, a bounded number of concurrent calls, a per-call timeout, retries with
+ * jittered backoff that honour a provider's `Retry-After`, budgets (images, tokens, money) that stop the run cleanly,
+ * and credential redaction of every message it records. Failures are data: a crash, a timeout, a rate limit that never
+ * cleared and an unparseable answer each become a critique with a `failure` and a reason code, which the next run
+ * retries, and never a score.
+ *
+ * Results are returned in the order of the entries, so the same inputs publish the same layer. Nothing here reads a
+ * clock, the environment or the network on its own: images, time, sleeping and randomness are passed in.
+ *
+ * @see docs/critics.md#running-critics
+ */
+import { z } from 'zod/v4';
+import { sha256Hex } from '../../reports/src/sigv4.js';
+import { CriticError, InvalidOutputError, RateLimitedError } from './contract.js';
+import { buildCritique, critiqueIdentity, identityOfCritique } from './critique.js';
+import { createRedactor } from './credentials.js';
+/** The shape a critic's answer must have. A score that is not a finite number, or a finding with a bad region, is `invalid-output`. */
+const AnswerSchema = z.object({
+    verdict: z.looseObject({ score: z.number().finite().optional(), confidence: z.number().min(0).max(1).optional() }).optional(),
+    findings: z.array(z.looseObject({ confidence: z.number().min(0).max(1).optional() })).optional(),
+    usage: z.looseObject({ inputTokens: z.number().nonnegative().optional(), outputTokens: z.number().nonnegative().optional(), latencyMs: z.number().nonnegative().optional(), costUsd: z.number().nonnegative().optional() }).optional()
+}).refine(answer => answer.verdict !== undefined || (answer.findings !== undefined && answer.findings.length > 0), 'the answer has neither a verdict nor findings');
+const defaultSleep = (ms, signal) => new Promise(resolve => {
+    const timer = setTimeout(done, ms);
+    function done() { clearTimeout(timer); signal?.removeEventListener('abort', done); resolve(); }
+    signal?.addEventListener('abort', done, { once: true });
+});
+const sha256Bytes = (bytes) => sha256Hex(bytes);
+/** Races a critic's call against a timeout and the run's cancellation. Rejects with `CriticError('timeout')` on timeout. */
+async function withTimeout(work, timeoutMs, outer) {
+    const controller = new AbortController();
+    const onOuter = () => controller.abort(outer?.reason);
+    if (outer?.aborted)
+        controller.abort(outer.reason);
+    else
+        outer?.addEventListener('abort', onOuter, { once: true });
+    let timer;
+    try {
+        return await Promise.race([
+            work(controller.signal),
+            new Promise((_, reject) => {
+                // Settle first: aborting may make the critic's own call reject at once, and the reason must be the timeout.
+                timer = setTimeout(() => { reject(new CriticError('timeout', `The critic did not answer within ${timeoutMs} ms`)); controller.abort(new Error('timeout')); }, timeoutMs);
+            })
+        ]);
+    }
+    finally {
+        if (timer !== undefined)
+            clearTimeout(timer);
+        outer?.removeEventListener('abort', onOuter);
+    }
+}
+/** Collects a sync or async iterable, keeping the first of each key. */
+async function collect(entries) {
+    const seen = new Set();
+    const out = [];
+    for await (const entry of entries) {
+        if (seen.has(entry.key))
+            continue;
+        seen.add(entry.key);
+        out.push(entry);
+    }
+    return out;
+}
+/**
+ * Runs a critic over entries. Entries with a current critique of the same identity are skipped without fetching their
+ * image; the rest are judged with bounded concurrency, a timeout and retries. The run never throws for a critic's
+ * failure: it records it. It throws only for a wrong call (a non-positive `concurrency`, `maxAttempts` or `timeoutMs`).
+ *
+ * A run stops cleanly when a budget is reached or the signal aborts: calls in flight finish (or are aborted), what was
+ * finished is returned in `outcomes` to publish, and `stoppedBy` says why. With concurrency above 1 a budget can be
+ * overshot by the calls already in flight.
+ *
+ * @example
+ * const result = await runCritic({ critic, entries, loadImage: entry => fetchImage(entry), existing, budget: { maxImages: 200 } });
+ * await publishChecks(store, critiqueLayer({ critic, outcomes: result.outcomes, recordedAtMs: Date.now() }));
+ */
+export async function runCritic(options) {
+    const concurrency = options.concurrency ?? 4;
+    const maxAttempts = options.maxAttempts ?? 4;
+    const timeoutMs = options.timeoutMs ?? 60_000;
+    for (const [name, value] of [['concurrency', concurrency], ['maxAttempts', maxAttempts], ['timeoutMs', timeoutMs]]) {
+        if (!Number.isFinite(value) || value < 1)
+            throw new Error(`Invalid ${name}: ${value}`);
+    }
+    const sleep = options.sleep ?? defaultSleep;
+    const random = options.random ?? Math.random;
+    const redact = createRedactor(options.secrets ?? []);
+    const { critic } = options;
+    const description = critic.describe();
+    const entries = await collect(options.entries);
+    const known = new Set();
+    if (!options.force)
+        for (const critique of options.existing ?? [])
+            if (critique.verdict !== undefined && !critique.failure)
+                known.add(identityOfCritique(critique));
+    const isKnown = (entry) => known.has(critiqueIdentity({ critic, model: description.model, parameters: description.parameters, imageSha256: entry.imageSha256 }));
+    const slots = entries.map(entry => (isKnown(entry) ? 'skipped' : undefined));
+    const identity = { name: critic.name, version: critic.version };
+    const toJudge = slots.filter(slot => slot === undefined).length;
+    if (options.dryRun) {
+        return { critic: identity, sendsImagesTo: description.sendsImagesTo, outcomes: [], skipped: entries.length - toJudge, judged: 0, failed: 0, waiting: toJudge, spent: { images: 0, tokens: 0, costUsd: 0 }, dryRun: true };
+    }
+    const budget = options.budget ?? {};
+    const spent = { images: 0, tokens: 0, costUsd: 0 };
+    let judged = 0, failed = 0, skipped = entries.length - toJudge, stoppedBy;
+    const exhausted = () => (budget.maxImages !== undefined && spent.images >= budget.maxImages) ||
+        (budget.maxTokens !== undefined && spent.tokens >= budget.maxTokens) ||
+        (budget.maxCostUsd !== undefined && spent.costUsd >= budget.maxCostUsd);
+    const progress = (entry, result) => options.onProgress?.({ entryKey: entry.key, result, judged, skipped, failed, waiting: slots.filter(slot => slot === undefined).length - inFlight });
+    let inFlight = 0;
+    const fail = (entry, reason, message) => ({
+        entry,
+        critique: buildCritique(critic, description, entry, { failure: { reason, message: redact(message).slice(0, 500) } })
+    });
+    /** Judges one entry; `null` when the run was cancelled meanwhile, because a cancelled call is not a failure of the critic. */
+    async function judge(entry) {
+        let image;
+        try {
+            image = await options.loadImage(entry);
+        }
+        catch (error) {
+            return fail(entry, 'image-unavailable', error instanceof Error ? error.message : String(error));
+        }
+        if ((await sha256Bytes(image)) !== entry.imageSha256)
+            return fail(entry, 'image-mismatch', 'The image bytes are not the ones the catalog hashed; nothing was judged');
+        for (let attempt = 1;; attempt++) {
+            if (options.signal?.aborted)
+                return null;
+            try {
+                const answer = await withTimeout(signal => critic.evaluate({ image, imageSha256: entry.imageSha256, entry: { key: entry.key, title: entry.title, state: entry.state, ...(entry.route === undefined ? {} : { route: entry.route }), ...(entry.profile === undefined ? {} : { profile: entry.profile }) }, signal }), timeoutMs, options.signal);
+                const checked = AnswerSchema.safeParse(answer);
+                if (!checked.success)
+                    throw new InvalidOutputError(`The answer is not a critique: ${checked.error.issues[0]?.message ?? 'unknown'}`);
+                const usage = answer.usage;
+                spent.tokens += (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0);
+                spent.costUsd += usage?.costUsd ?? 0;
+                return { entry, critique: buildCritique(critic, description, entry, answer) };
+            }
+            catch (error) {
+                if (options.signal?.aborted)
+                    return null;
+                if (error instanceof RateLimitedError && attempt < maxAttempts && !options.signal?.aborted) {
+                    const base = options.backoff?.baseMs ?? 500, cap = options.backoff?.maxMs ?? 30_000;
+                    const jittered = Math.floor(random() * Math.min(cap, base * 2 ** (attempt - 1)));
+                    await sleep(Math.max(error.retryAfterMs ?? 0, jittered), options.signal);
+                    continue;
+                }
+                if (error instanceof CriticError)
+                    return fail(entry, error.reason, error.message);
+                return fail(entry, 'error', error instanceof Error ? error.message : String(error));
+            }
+        }
+    }
+    let next = 0;
+    async function worker() {
+        for (;;) {
+            if (options.signal?.aborted) {
+                stoppedBy ??= 'aborted';
+                return;
+            }
+            if (exhausted()) {
+                stoppedBy ??= 'budget';
+                return;
+            }
+            let index = next;
+            while (index < entries.length && slots[index] !== undefined)
+                index++;
+            if (index >= entries.length)
+                return;
+            next = index + 1;
+            const entry = entries[index];
+            inFlight += 1;
+            spent.images += 1;
+            const outcome = await judge(entry);
+            inFlight -= 1;
+            if (!outcome) {
+                stoppedBy ??= 'aborted';
+                return;
+            }
+            slots[index] = outcome;
+            if (outcome.critique.failure)
+                failed += 1;
+            else
+                judged += 1;
+            progress(entry, outcome.critique.failure ? 'failed' : 'judged');
+        }
+    }
+    for (let index = 0; index < entries.length; index++)
+        if (slots[index] === 'skipped')
+            progress(entries[index], 'skipped');
+    await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(toJudge, 1)) }, worker));
+    // A run that handled every entry did not stop early, whatever the budget now says.
+    const waiting = slots.filter(slot => slot === undefined).length;
+    const outcomes = slots.filter((slot) => slot !== undefined && slot !== 'skipped');
+    return { critic: identity, sendsImagesTo: description.sendsImagesTo, outcomes, skipped, judged, failed, waiting, ...(waiting > 0 && stoppedBy ? { stoppedBy } : {}), spent, dryRun: false };
+}

package/dist/critics/src/scope.d.ts

@@ -0,0 +1,8 @@
+/**
+ * The rule a layer breaks for a critic-scoped credential, or `null` when it is within scope. The message says which rule
+ * and what was found, so the critic can fix it.
+ */
+export declare function criticScopeViolation(layer: {
+    readonly producer?: unknown;
+    readonly parts?: unknown;
+}): string | null;

package/dist/critics/src/scope.js

@@ -0,0 +1,26 @@
+/**
+ * @module The scope of a critic's credential
+ *
+ * A critic only needs to add opinions, so its credential may read and publish layers whose producer is `critic.<name>`
+ * and whose parts are all of kind `critique`, and nothing else. `criticScopeViolation` names the rule a layer breaks;
+ * the server applies it to every layer a critic-role token uploads, so a stolen critic token cannot forge test results.
+ *
+ * @see docs/critics.md#authentication-and-authorization
+ */
+import { CRITIC_PRODUCER_PREFIX, CRITIQUE_PART_KIND } from '../../schema/src/critique.js';
+/**
+ * The rule a layer breaks for a critic-scoped credential, or `null` when it is within scope. The message says which rule
+ * and what was found, so the critic can fix it.
+ */
+export function criticScopeViolation(layer) {
+    if (typeof layer.producer !== 'string' || !layer.producer.startsWith(CRITIC_PRODUCER_PREFIX) || layer.producer.length === CRITIC_PRODUCER_PREFIX.length) {
+        return `A critic token may publish only layers whose producer is "${CRITIC_PRODUCER_PREFIX}<name>"; this layer's producer is ${JSON.stringify(layer.producer ?? null)}`;
+    }
+    const parts = Array.isArray(layer.parts) ? layer.parts : [];
+    if (!parts.length)
+        return `A critic token may publish only layers of parts of kind "${CRITIQUE_PART_KIND}"; this layer has no parts`;
+    const other = parts.find(part => part?.kind !== CRITIQUE_PART_KIND);
+    if (other)
+        return `A critic token may publish only parts of kind "${CRITIQUE_PART_KIND}"; this layer has a part of kind ${JSON.stringify(other?.kind ?? null)}`;
+    return null;
+}

package/dist/reports/src/compose.d.ts

@@ -50,10 +50,10 @@ export type CombinedReport = {
     readonly summary: ReportSummary;
 };
 /**
- * Counts results by status. Diagnostic results (parts of kind `performance`, which only qualify a check) are left out
+ * Counts results by status. Opinion results (parts of kind `critique`) are never counted. Diagnostic results (parts of kind `performance`, which only qualify a check) are left out
  * whenever any other result is present; if only diagnostics exist, they are counted so the report is not empty.
  */
-export declare function countResults(results: readonly EvidenceResult[]): StatusCounts & {
+export declare function countResults(all: readonly EvidenceResult[]): StatusCounts & {
     total: number;
 };
 /** Counts a combined report's winning results in total and per environment, and lists the specs that have no result at all. */

package/dist/reports/src/compose.js

@@ -11,14 +11,15 @@
  */
 import { eachLimited } from './completion-order.js';
 import { catalogEntries, catalogLayers } from './layer-catalog.js';
-import { combineEvidence, distinctHeaders, emptyCounts, isDiagnosticKind, layerEvidence } from './layer-merge.js';
+import { combineEvidence, distinctHeaders, emptyCounts, isDiagnosticKind, isOpinionKind, layerEvidence } from './layer-merge.js';
 import { RecordedSpecs } from './recorded-specs.js';
 const BACKEND_ID = /^[a-z0-9-]+$/;
 /**
- * Counts results by status. Diagnostic results (parts of kind `performance`, which only qualify a check) are left out
+ * Counts results by status. Opinion results (parts of kind `critique`) are never counted. Diagnostic results (parts of kind `performance`, which only qualify a check) are left out
  * whenever any other result is present; if only diagnostics exist, they are counted so the report is not empty.
  */
-export function countResults(results) {
+export function countResults(all) {
+    const results = all.filter(result => !isOpinionKind(result.row.kind));
     const outcomes = results.some(result => !isDiagnosticKind(result.row.kind)) ? results.filter(result => !isDiagnosticKind(result.row.kind)) : results;
     const counts = emptyCounts();
     for (const result of outcomes)
@@ -27,7 +28,7 @@ export function countResults(results) {
 }
 /** Counts a combined report's winning results in total and per environment, and lists the specs that have no result at all. */
 export function summarize(combined) {
-    const results = [...combined.specs.flatMap(entry => entry.results), ...combined.unassigned];
+    const results = [...combined.specs.flatMap(entry => entry.results), ...combined.unassigned].filter(result => !isOpinionKind(result.row.kind));
     const environments = new Map();
     for (const result of results) {
         const key = result.layer.environment ?? 'unspecified';
@@ -40,7 +41,7 @@ export function summarize(combined) {
     return {
         checks: countResults(results),
         byEnvironment: Object.fromEntries([...environments].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([key, value]) => [key, countResults(value)])),
-        specsWithoutEvidence: combined.specs.filter(entry => entry.results.length === 0).map(entry => entry.spec.path)
+        specsWithoutEvidence: combined.specs.filter(entry => entry.results.every(result => isOpinionKind(result.row.kind))).map(entry => entry.spec.path)
     };
 }
 /**

package/dist/reports/src/evidence-view.d.ts

@@ -32,7 +32,8 @@ export type CheckSlot = {
 /**
  * Groups the combined results (the winners of the slots) by check, in the order the checks were first seen.
  * A check's status is the worst of its non-diagnostic results; if it has only diagnostics (timings), those are its
- * outcomes. The title and id shown are the newest result's.
+ * outcomes. The title and id shown are the newest result's. Opinions (parts of kind `critique`, see `isOpinionKind`) are
+ * not checks and are left out.
  */
 export declare function checkSlots(results: readonly EvidenceResult[]): CheckSlot[];
 /** A spec's rolled-up status: the worst of its checks, or `none` when nothing was recorded for it. */

package/dist/reports/src/evidence-view.js

@@ -1,4 +1,4 @@
-import { emptyCounts, isDiagnosticKind, slotKey } from './layer-merge.js';
+import { emptyCounts, isDiagnosticKind, isOpinionKind, slotKey } from './layer-merge.js';
 const WORST = { passed: 0, 'not-run': 1, running: 2, failed: 3 };
 function worst(statuses) {
     let status = 'passed';
@@ -13,11 +13,14 @@ function newestFirst(a, b) {
 /**
  * Groups the combined results (the winners of the slots) by check, in the order the checks were first seen.
  * A check's status is the worst of its non-diagnostic results; if it has only diagnostics (timings), those are its
- * outcomes. The title and id shown are the newest result's.
+ * outcomes. The title and id shown are the newest result's. Opinions (parts of kind `critique`, see `isOpinionKind`) are
+ * not checks and are left out.
  */
 export function checkSlots(results) {
     const groups = new Map();
     for (const result of results) {
+        if (isOpinionKind(result.row.kind))
+            continue;
         const group = groups.get(result.slot);
         if (group)
             group.push(result);

package/dist/reports/src/layer-merge.d.ts

@@ -36,13 +36,21 @@ export declare function specPath(path: string): string;
 export declare function parameterKey(parameters: LayerParameters): string;
 /** Whether a part `kind` only qualifies an outcome (today only `performance`) and so is not counted as a check when anything else is recorded. */
 export declare function isDiagnosticKind(kind: string): boolean;
+/**
+ * Whether a part `kind` records an opinion rather than an outcome (today only `critique`, an automated reviewer's
+ * judgement of a captured UI). An opinion is never counted as a check, never gates, and is not folded into a spec's
+ * rollup, even when it is the only thing recorded: a report of opinions alone has no test results. The check's `status`
+ * of such a part says whether the reviewer ran, not whether anything passed.
+ */
+export declare function isOpinionKind(kind: string): boolean;
 /** A fresh status tally with every status at zero. */
 export declare function emptyCounts(): Record<EvidenceStatus, number>;
 /**
- * Counts rows by status, leaving out the diagnostic rows (see `isDiagnosticKind`) that only qualify an outcome.
- * When every row is diagnostic they are counted, so a layer of timings alone is not empty.
+ * Counts rows by status, leaving out the opinion rows (see `isOpinionKind`) always, and the diagnostic rows (see
+ * `isDiagnosticKind`) that only qualify an outcome. When every remaining row is diagnostic they are counted, so a
+ * layer of timings alone is not empty.
  */
-export declare function outcomeCounts(rows: readonly LayerRow[]): StatusCounts;
+export declare function outcomeCounts(allRows: readonly LayerRow[]): StatusCounts;
 /**
  * Flattens a layer into what composition reads: a header (origin, normalised environment and parameters, counts,
  * evidence summary), one row per check of every part, and the spec fingerprints it recorded.

package/dist/reports/src/layer-merge.js

@@ -38,15 +38,26 @@ export function parameterKey(parameters) {
 export function isDiagnosticKind(kind) {
     return kind === 'performance';
 }
+/**
+ * Whether a part `kind` records an opinion rather than an outcome (today only `critique`, an automated reviewer's
+ * judgement of a captured UI). An opinion is never counted as a check, never gates, and is not folded into a spec's
+ * rollup, even when it is the only thing recorded: a report of opinions alone has no test results. The check's `status`
+ * of such a part says whether the reviewer ran, not whether anything passed.
+ */
+export function isOpinionKind(kind) {
+    return kind === 'critique';
+}
 /** A fresh status tally with every status at zero. */
 export function emptyCounts() {
     return { passed: 0, failed: 0, running: 0, 'not-run': 0 };
 }
 /**
- * Counts rows by status, leaving out the diagnostic rows (see `isDiagnosticKind`) that only qualify an outcome.
- * When every row is diagnostic they are counted, so a layer of timings alone is not empty.
+ * Counts rows by status, leaving out the opinion rows (see `isOpinionKind`) always, and the diagnostic rows (see
+ * `isDiagnosticKind`) that only qualify an outcome. When every remaining row is diagnostic they are counted, so a
+ * layer of timings alone is not empty.
  */
-export function outcomeCounts(rows) {
+export function outcomeCounts(allRows) {
+    const rows = allRows.filter(row => !isOpinionKind(row.kind));
     const outcomes = rows.some(row => !isDiagnosticKind(row.kind)) ? rows.filter(row => !isDiagnosticKind(row.kind)) : rows;
     const counts = emptyCounts();
     for (const row of outcomes)

package/dist/reports/src/report-status.d.ts

@@ -25,7 +25,7 @@ export type StatusOptions = {
  * checks (or none completed), failed checks, running checks, specs without results (only with `requireAllSpecs`),
  * each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
  * report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
- * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
+ * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded, and `critique` opinions are always left out.
  * `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
  * completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
  *

package/dist/reports/src/report-status.js

@@ -11,7 +11,7 @@
 import { REPORT_STATUS_SCHEMA_VERSION } from '../../schema/src/layer.js';
 import { countResults } from './compose.js';
 import { normalizeEnvironment } from './layer-axes.js';
-import { isDiagnosticKind } from './layer-merge.js';
+import { isDiagnosticKind, isOpinionKind } from './layer-merge.js';
 import { recordingCompleteness } from './recording-receipt.js';
 /** Total string order by code unit. */
 const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
@@ -23,7 +23,7 @@ const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
  * checks (or none completed), failed checks, running checks, specs without results (only with `requireAllSpecs`),
  * each required environment with no checks (the name is normalised, so `iOS` means `ios`), then every notice of the
  * report (an unreadable layer or backend). `failing` lists each failed or running winner, in a fixed order that does
- * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded.
+ * not depend on read order; diagnostic `performance` checks are left out unless nothing else was recorded, and `critique` opinions are always left out.
  * `recordings` reports recording completeness of the same results apart from `ok`: an incomplete or unknown
  * completeness never changes the gate (use `atlas report publish --require-recordings` to fail on a missing video).
  *
@@ -34,7 +34,8 @@ const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
  * @see docs/report-contract.md#atlas-report-status
  */
 export function reportStatus(report, options = {}) {
-    const results = [...report.combined.specs.flatMap(entry => entry.results), ...report.combined.unassigned];
+    // A critique is an opinion: a low score or a crashed critic never fails the gate (see `isOpinionKind`).
+    const results = [...report.combined.specs.flatMap(entry => entry.results), ...report.combined.unassigned].filter(result => !isOpinionKind(result.row.kind));
     const outcomes = results.some(result => !isDiagnosticKind(result.row.kind)) ? results.filter(result => !isDiagnosticKind(result.row.kind)) : results;
     const failing = outcomes
         .filter(result => result.row.status === 'failed' || result.row.status === 'running')
@@ -98,7 +99,8 @@ export function reportStatus(report, options = {}) {
 }
 /** `spec › check`, unless the check's id already starts with its spec. */
 const label = (item) => (item.spec && !item.check.startsWith(item.spec) ? `${item.spec} › ${item.check}` : item.check);
-function specOutcome(path, title, kind, results) {
+function specOutcome(path, title, kind, all) {
+    const results = all.filter(result => !isOpinionKind(result.row.kind));
     const environments = new Map();
     for (const result of results) {
         const key = result.layer.environment ?? 'unspecified';

package/dist/schema/src/api.d.ts

@@ -55,6 +55,7 @@ export declare const ServerInfoSchema: z.ZodObject<{
     version: z.ZodNullable<z.ZodString>;
     role: z.ZodEnum<{
         read: "read";
+        critic: "critic";
         publish: "publish";
         admin: "admin";
     }>;

package/dist/schema/src/api.js

@@ -44,7 +44,7 @@ export const ReadySchema = z.looseObject({
 export const ServerInfoSchema = z.looseObject({
     name: z.literal('atlas'),
     version: z.string().nullable().describe('The @crvouga/atlas version the server runs, or null when it does not know'),
-    role: z.enum(['read', 'publish', 'admin']).describe('What the token of this request may do. A server with no tokens treats everyone as admin'),
+    role: z.enum(['read', 'critic', 'publish', 'admin']).describe('What the token of this request may do. A server with no tokens treats everyone as admin'),
     open: z.boolean().describe('True when the server has no tokens at all'),
     write: z.string().describe('The id of the backend uploads go to'),
     ready: z.boolean(),

package/dist/schema/src/critique.d.ts

@@ -0,0 +1,144 @@
+/**
+ * @module The `atlas.critique` evidence schema
+ *
+ * What an automated reviewer (a critic) says about one captured UI. It is not a new layer format: a critique is a
+ * check inside a part of kind `critique`, and the opinion rides in the check's `extensions["atlas.critique"]`, so by the
+ * rules of docs/extending.md it can never decide a result. The check's `status` says only whether the critic ran
+ * (`passed` a verdict was produced, `failed` the critic could not produce one, `running`, `not-run`).
+ *
+ * Everything except `schema`, `subject` and `critic` is optional, so a single score (a model-based critic) and rich
+ * findings (an LLM critic) are one shape. Every object is open, and `kind`, `category`, `severity`, `label` and `reason`
+ * are open strings: a value nobody has heard of is kept and shown generically, never rejected. A reader that cannot read
+ * the whole extension drops it and keeps the check and its status (`readCritique` in `@crvouga/atlas/critics`).
+ *
+ * @see docs/critics.md
+ * @see docs/extending.md
+ */
+import { z } from 'zod/v4';
+/** The key under which a critique is stored in a check's `extensions`. */
+export declare const CRITIQUE_EXTENSION = "atlas.critique";
+/** The version of the `atlas.critique` extension a reader of this release understands. A critique of another version is unknown, not wrong. */
+export declare const CRITIQUE_SCHEMA_VERSION = 1;
+/** The `kind` of the report part that holds critiques. Parts of this kind are opinions: never counted, never part of the gate. */
+export declare const CRITIQUE_PART_KIND = "critique";
+/** The prefix of the `producer` of a layer of critiques: `critic.<critic name>`. */
+export declare const CRITIC_PRODUCER_PREFIX = "critic.";
+/** The region of the image a finding is about, in the stated unit (`px` of the stored image, or `normalized` 0..1). */
+export declare const CritiqueRegionSchema: z.ZodObject<{
+    unit: z.ZodString;
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    w: z.ZodNumber;
+    h: z.ZodNumber;
+}, z.core.$loose>;
+/** One thing a critic found: where, what kind, how bad, why, and what to do. Nothing is required. */
+export declare const CritiqueFindingSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodString>;
+    category: z.ZodOptional<z.ZodString>;
+    severity: z.ZodOptional<z.ZodString>;
+    confidence: z.ZodOptional<z.ZodNumber>;
+    region: z.ZodOptional<z.ZodObject<{
+        unit: z.ZodString;
+        x: z.ZodNumber;
+        y: z.ZodNumber;
+        w: z.ZodNumber;
+        h: z.ZodNumber;
+    }, z.core.$loose>>;
+    rationale: z.ZodOptional<z.ZodString>;
+    suggestion: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>;
+/**
+ * How a score reads in context. A raw score is not absolute: a loading overlay is compared with other loading overlays,
+ * not with full screens, and a change against the same entry's previous image is more telling than the level.
+ */
+export declare const CritiqueRelativeSchema: z.ZodObject<{
+    stateKind: z.ZodString;
+    percentile: z.ZodOptional<z.ZodNumber>;
+    sampleSize: z.ZodOptional<z.ZodNumber>;
+    previousScore: z.ZodOptional<z.ZodNumber>;
+    delta: z.ZodOptional<z.ZodNumber>;
+}, z.core.$loose>;
+/** The judgement: a score and how to read it, a label, a confidence. A critic that cannot score may give only a label. */
+export declare const CritiqueVerdictSchema: z.ZodObject<{
+    score: z.ZodOptional<z.ZodNumber>;
+    scale: z.ZodOptional<z.ZodString>;
+    label: z.ZodOptional<z.ZodString>;
+    confidence: z.ZodOptional<z.ZodNumber>;
+    relative: z.ZodOptional<z.ZodObject<{
+        stateKind: z.ZodString;
+        percentile: z.ZodOptional<z.ZodNumber>;
+        sampleSize: z.ZodOptional<z.ZodNumber>;
+        previousScore: z.ZodOptional<z.ZodNumber>;
+        delta: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$loose>>;
+}, z.core.$loose>;
+/** Why a critic produced no verdict: a reason code in an open vocabulary (`error`, `timeout`, `rate-limited`, `invalid-output`) and a message that holds no credential. */
+export declare const CritiqueFailureSchema: z.ZodObject<{
+    reason: z.ZodString;
+    message: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>;
+/**
+ * The `atlas.critique` extension, version 1: the image judged (`subject`), who judged (`critic`, `model`,
+ * `parameters`), the opinion (`verdict`, `findings`) or why there is none (`failure`), and what it cost (`usage`).
+ * The prompt is recorded only as a hash (`parameters.rubric`). Emitted as `critique.schema.json`.
+ */
+export declare const CritiqueSchema: z.ZodObject<{
+    schema: z.ZodNumber;
+    subject: z.ZodObject<{
+        entryKey: z.ZodString;
+        imageSha256: z.ZodString;
+        state: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>;
+    critic: z.ZodObject<{
+        name: z.ZodString;
+        version: z.ZodString;
+        kind: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>;
+    model: z.ZodOptional<z.ZodObject<{
+        id: z.ZodString;
+        provider: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>>;
+    parameters: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    verdict: z.ZodOptional<z.ZodObject<{
+        score: z.ZodOptional<z.ZodNumber>;
+        scale: z.ZodOptional<z.ZodString>;
+        label: z.ZodOptional<z.ZodString>;
+        confidence: z.ZodOptional<z.ZodNumber>;
+        relative: z.ZodOptional<z.ZodObject<{
+            stateKind: z.ZodString;
+            percentile: z.ZodOptional<z.ZodNumber>;
+            sampleSize: z.ZodOptional<z.ZodNumber>;
+            previousScore: z.ZodOptional<z.ZodNumber>;
+            delta: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$loose>>;
+    }, z.core.$loose>>;
+    findings: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        id: z.ZodOptional<z.ZodString>;
+        category: z.ZodOptional<z.ZodString>;
+        severity: z.ZodOptional<z.ZodString>;
+        confidence: z.ZodOptional<z.ZodNumber>;
+        region: z.ZodOptional<z.ZodObject<{
+            unit: z.ZodString;
+            x: z.ZodNumber;
+            y: z.ZodNumber;
+            w: z.ZodNumber;
+            h: z.ZodNumber;
+        }, z.core.$loose>>;
+        rationale: z.ZodOptional<z.ZodString>;
+        suggestion: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>>>;
+    failure: z.ZodOptional<z.ZodObject<{
+        reason: z.ZodString;
+        message: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>>;
+    usage: z.ZodOptional<z.ZodObject<{
+        inputTokens: z.ZodOptional<z.ZodNumber>;
+        outputTokens: z.ZodOptional<z.ZodNumber>;
+        latencyMs: z.ZodOptional<z.ZodNumber>;
+        costUsd: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$loose>>;
+}, z.core.$loose>;
+/** A critique as stored in a check's `extensions["atlas.critique"]`. */
+export type Critique = z.output<typeof CritiqueSchema>;
+/** One finding of a critique. */
+export type CritiqueFinding = z.output<typeof CritiqueFindingSchema>;