@crvouga/atlas 1.11.0 → 1.13.0

package/dist/critics/src/critique.js

@@ -0,0 +1,163 @@
+/**
+ * @module Critiques as layers
+ *
+ * Turns a critic's judgements into the checks of a layer (`critiqueChecks`, `critiqueLayer`) and reads them back
+ * (`readCritique`, `critiquesOfLayer`). A critique is a check in a part of kind `critique` whose `status` says whether
+ * the critic ran; the opinion is in `extensions["atlas.critique"]`, and is repeated in standard evidence (a
+ * measurement for the score, an annotation for each finding) so a viewer that has never heard of critics still shows
+ * it. Critiques are anchored to the image hash they judged: when an entry's image changes, an older critique is still
+ * shown, as judging an earlier image, and the entry counts as not yet judged (`critiqueCurrency`).
+ *
+ * @see docs/critics.md
+ * @see docs/extending.md
+ */
+import { CRITIC_PRODUCER_PREFIX, CRITIQUE_EXTENSION, CRITIQUE_PART_KIND, CRITIQUE_SCHEMA_VERSION, CritiqueSchema } from '../../schema/src/critique.js';
+/** The prefix of the id of every critique check: `critique/<entry key>`. It keeps a critique from ever sharing a slot with a test. */
+export const CRITIQUE_CHECK_PREFIX = 'critique/';
+/** The check ids and labels one critique is published under. */
+export function critiqueCheckId(entryKey) {
+    return `${CRITIQUE_CHECK_PREFIX}${entryKey}`;
+}
+/** `critic.<name>`: the producer of every layer of the critic's critiques. A critic-scoped token may publish only such layers. */
+export function criticProducer(criticName) {
+    return `${CRITIC_PRODUCER_PREFIX}${criticName}`;
+}
+const LEVELS = { info: 'info', minor: 'warning', warning: 'warning', major: 'error', critical: 'error', error: 'error' };
+function findingMessage(finding) {
+    const head = [finding.category, finding.rationale].filter(Boolean).join(': ');
+    const text = head || 'Finding';
+    return finding.suggestion ? `${text} (suggestion: ${finding.suggestion})` : text;
+}
+/**
+ * Builds the critique a critic produced for an entry. It fills in `schema`, `subject`, `critic`, `model` and
+ * `parameters` from the critic and the entry (so they cannot disagree with what was run) and keeps the verdict,
+ * findings and usage as given.
+ *
+ * @param critic the critic that ran.
+ * @param description the critic's `describe()` at the time of the run.
+ * @param entry the entry and the image hash that was judged.
+ * @param answer the critic's answer, or a `failure` instead of one.
+ */
+export function buildCritique(critic, description, entry, answer) {
+    return {
+        schema: CRITIQUE_SCHEMA_VERSION,
+        subject: { entryKey: entry.key, imageSha256: entry.imageSha256, state: entry.state },
+        critic: { name: critic.name, version: critic.version, ...(critic.kind ? { kind: critic.kind } : {}) },
+        ...(description.model ? { model: { ...description.model } } : {}),
+        ...(description.parameters ? { parameters: { ...description.parameters } } : {}),
+        ...(answer.verdict ? { verdict: answer.verdict } : {}),
+        ...(answer.findings ? { findings: answer.findings } : {}),
+        ...(answer.failure ? { failure: answer.failure } : {}),
+        ...(answer.usage ? { usage: answer.usage } : {})
+    };
+}
+/**
+ * The checks of one layer for a set of outcomes, in the order given. A verdict is a `passed` check (the critic ran), a
+ * failure a `failed` one with its reason code in `detail`. The score is a measurement named `polish`; each finding is
+ * also an annotation of type `critique`. Nothing here decides a result: a low score is `passed`.
+ */
+export function critiqueChecks(outcomes, options = {}) {
+    return outcomes.map(({ entry, critique }) => {
+        const verdict = critique.verdict;
+        const failure = critique.failure;
+        const score = verdict?.score;
+        return {
+            id: critiqueCheckId(entry.key),
+            title: `Critique of ${entry.title}`,
+            status: failure ? 'failed' : 'passed',
+            detail: failure ? `${failure.reason}${failure.message ? `: ${failure.message}` : ''}` : [verdict?.label, score === undefined ? undefined : `score ${score}`].filter(Boolean).join(', '),
+            measurements: score !== undefined && Number.isFinite(score) ? [{ name: 'polish', value: score, unit: 'score', ...(options.threshold === undefined ? {} : { limit: options.threshold }) }] : [],
+            annotations: (critique.findings ?? []).map(finding => ({
+                type: 'critique',
+                level: LEVELS[finding.severity ?? 'info'] ?? 'info',
+                message: findingMessage(finding),
+                ...(finding.id ? { extensions: { [CRITIQUE_EXTENSION]: { finding: finding.id } } } : {})
+            })),
+            labels: { critic: critique.critic.name, 'critic-version': critique.critic.version, ...(critique.model ? { model: critique.model.id } : {}) },
+            extensions: { [CRITIQUE_EXTENSION]: critique }
+        };
+    });
+}
+/**
+ * The input of `publishChecks` for one critic's outcomes: producer `critic.<name>`, part kind `critique`. The model and
+ * critic versions are labels and not part of the slot, so a newer version replaces an older one's critique of the same
+ * entry. Pass `parameters` to move something (such as a model version) into the slot instead, to compare side by side.
+ *
+ * @param args.critic the critic that ran.
+ * @param args.outcomes what it produced, successes and failures.
+ * @param args.recordedAtMs when, epoch milliseconds. Nothing here reads a clock, so the same input is the same layer.
+ */
+export function critiqueLayer(args) {
+    return {
+        producer: criticProducer(args.critic.name),
+        kind: CRITIQUE_PART_KIND,
+        title: `Critiques by ${args.critic.name}`,
+        recordedAtMs: args.recordedAtMs,
+        sha: args.sha ?? null,
+        branch: args.branch ?? null,
+        runUrl: args.runUrl ?? null,
+        ...(args.environment === undefined ? {} : { environment: args.environment }),
+        parameters: { ...(args.parameters ?? {}) },
+        labels: { critic: args.critic.name, 'critic-version': args.critic.version },
+        checks: critiqueChecks(args.outcomes, args.threshold === undefined ? {} : { threshold: args.threshold })
+    };
+}
+/**
+ * Reads the critique of a check, or `null` when there is none that this reader can read: no `atlas.critique` extension,
+ * one that is malformed (a missing `subject`, a score that is not a number), or one of a version this release does not
+ * know. The check and its status are never affected: an unreadable opinion is unknown, not a pass and not a failure.
+ * Fields and values it does not know (a new category, severity or label) are kept.
+ */
+export function readCritique(check) {
+    const raw = check.extensions?.[CRITIQUE_EXTENSION];
+    if (raw === undefined)
+        return null;
+    const parsed = CritiqueSchema.safeParse(raw);
+    if (!parsed.success || parsed.data.schema !== CRITIQUE_SCHEMA_VERSION)
+        return null;
+    return parsed.data;
+}
+/**
+ * Every readable critique in a layer's parts of kind `critique`. Checks whose critique cannot be read are left out, and
+ * so is any `atlas.critique` extension in a part of another kind: a test result is never mistaken for an opinion.
+ */
+export function critiquesOfLayer(layer) {
+    const found = [];
+    for (const part of layer.parts) {
+        if (part.kind !== CRITIQUE_PART_KIND)
+            continue;
+        for (const check of part.checks) {
+            const critique = readCritique(check);
+            if (critique)
+                found.push({ layerId: layer.id, producer: layer.producer, recordedAtMs: part.recordedAtMs, checkId: check.id, status: check.status, critique });
+        }
+    }
+    return found;
+}
+/**
+ * Whether a critique judged the image an entry has now. `current`: yes. `earlier-image`: the entry's image has changed
+ * since, so the critique is shown as judging an earlier image and the entry counts as not yet judged by that critic.
+ */
+export function critiqueCurrency(critique, currentImageSha256) {
+    return critique.subject.imageSha256 === currentImageSha256 ? 'current' : 'earlier-image';
+}
+const canonical = (value) => {
+    if (value === null || typeof value !== 'object')
+        return JSON.stringify(value) ?? 'null';
+    if (Array.isArray(value))
+        return `[${value.map(canonical).join(',')}]`;
+    const object = value;
+    return `{${Object.keys(object).sort().filter(key => object[key] !== undefined).map(key => `${JSON.stringify(key)}:${canonical(object[key])}`).join(',')}}`;
+};
+/**
+ * The identity of a critique for caching: the critic, its version, the model, the fixed parameters (the rubric hash
+ * among them) and the image hash. Two critiques with the same identity are the same judgement, and the second is not
+ * made. Key order in `parameters` does not matter.
+ */
+export function critiqueIdentity(args) {
+    return canonical([args.critic.name, args.critic.version, args.model?.id ?? null, args.model?.provider ?? null, args.parameters ?? {}, args.imageSha256]);
+}
+/** The identity of an existing critique, as {@link critiqueIdentity} computes it for a run. */
+export function identityOfCritique(critique) {
+    return critiqueIdentity({ critic: critique.critic, model: critique.model, parameters: critique.parameters, imageSha256: critique.subject.imageSha256 });
+}

package/dist/critics/src/index.d.ts

@@ -0,0 +1,21 @@
+/**
+ * @module Critics (`@crvouga/atlas/critics`)
+ *
+ * Automated reviewers of the UI catalog that mark which UIs need polish. A critic is a producer: it judges one captured
+ * UI at a time through one small contract (`defineCritic`), the runner (`runCritic`) owns concurrency, retries, rate
+ * limits, budgets and caching, and the opinions are stored as ordinary layers of parts of kind `critique` with the
+ * `atlas.critique` evidence (`critiqueLayer`, `readCritique`). A critique never changes whether a report passes: it is
+ * not a test result, is not counted, and only a project that opts in to `gate` mode can fail a command on a score, and
+ * only from calibrated, held-out evidence. Model-based, LLM-based and heuristic critics are the same on every
+ * path. Nothing here needs a model, a provider SDK or a network; critics bring their own.
+ *
+ * @see docs/critics.md
+ */
+export * from './contract.js';
+export * from './credentials.js';
+export * from './critique.js';
+export * from './judging.js';
+export * from './publish.js';
+export * from './runner.js';
+export * from './scope.js';
+export { CRITIC_PRODUCER_PREFIX, CRITIQUE_EXTENSION, CRITIQUE_PART_KIND, CRITIQUE_SCHEMA_VERSION, CritiqueSchema, type Critique, type CritiqueFinding } from '../../schema/src/critique.js';

package/dist/critics/src/index.js

@@ -0,0 +1,21 @@
+/**
+ * @module Critics (`@crvouga/atlas/critics`)
+ *
+ * Automated reviewers of the UI catalog that mark which UIs need polish. A critic is a producer: it judges one captured
+ * UI at a time through one small contract (`defineCritic`), the runner (`runCritic`) owns concurrency, retries, rate
+ * limits, budgets and caching, and the opinions are stored as ordinary layers of parts of kind `critique` with the
+ * `atlas.critique` evidence (`critiqueLayer`, `readCritique`). A critique never changes whether a report passes: it is
+ * not a test result, is not counted, and only a project that opts in to `gate` mode can fail a command on a score, and
+ * only from calibrated, held-out evidence. Model-based, LLM-based and heuristic critics are the same on every
+ * path. Nothing here needs a model, a provider SDK or a network; critics bring their own.
+ *
+ * @see docs/critics.md
+ */
+export * from './contract.js';
+export * from './credentials.js';
+export * from './critique.js';
+export * from './judging.js';
+export * from './publish.js';
+export * from './runner.js';
+export * from './scope.js';
+export { CRITIC_PRODUCER_PREFIX, CRITIQUE_EXTENSION, CRITIQUE_PART_KIND, CRITIQUE_SCHEMA_VERSION, CritiqueSchema } from '../../schema/src/critique.js';

package/dist/critics/src/judging.d.ts

@@ -0,0 +1,153 @@
+/**
+ * @module State-aware judging, calibration and modes
+ *
+ * A raw score is not absolute. A loading overlay is not a full screen, so a score is read against the project's scores
+ * of the same state kind (`relativeVerdict`), and a threshold is measured per critic and per state kind against labelled
+ * examples (`calibrate`), recording sample sizes, precision and recall and whether the figures are in-sample or
+ * held-out. What happens when a score is low is the project's choice (`gateDecision`): `report` (shown, never fails
+ * anything), `warn` (surfaced on the command line) or `gate` (exit code 1 when a threshold is crossed, but only on a
+ * calibration with enough held-out evidence). A gate is computed here, by the reader, from critique data. It never
+ * changes a check's status, a slot or a winner.
+ *
+ * All scores are "higher is better". Everything here is pure.
+ *
+ * @see docs/critics.md#state-aware-judging-calibration-and-modes
+ */
+import type { Critique } from '../../schema/src/critique.js';
+/** The relative reading of a score, as stored in `verdict.relative`. */
+export type Relative = NonNullable<NonNullable<Critique['verdict']>['relative']>;
+/**
+ * Reads a score against the scores of the same state kind, and against the same entry's previous image.
+ *
+ * `percentile` is where the score falls among the population's scores of `stateKind` (ties count half), 0..100, higher
+ * is better; it is absent when no score of that state kind is known. Scores of other state kinds are ignored, so a
+ * loading overlay is compared with other loading overlays and never with full screens. `delta` is the score minus
+ * `previousScore` when there was one.
+ *
+ * @param args.score the entry's raw score.
+ * @param args.stateKind the kind of UI state of the entry.
+ * @param args.population every known score of the critic, with its state kind. May include the entry's own.
+ * @param args.previousScore the critic's score for the entry's previous image, when there was one.
+ */
+export declare function relativeVerdict(args: {
+    readonly score: number;
+    readonly stateKind: string;
+    readonly population: readonly {
+        readonly stateKind: string;
+        readonly score: number;
+    }[];
+    readonly previousScore?: number;
+}): Relative;
+/** A labelled example for calibration: the critic's score for a UI, its state kind, and whether people judged it bad (needs polish). */
+export type LabelledExample = {
+    readonly score: number;
+    readonly stateKind: string;
+    /** True when the UI needs polish: a positive for the critic to find. */
+    readonly bad: boolean;
+    /** True for an example the threshold was not chosen on. */
+    readonly heldOut?: boolean;
+};
+/** The calibration of one state kind. `threshold` flags a score below it; `null` when the examples cannot tell (one class missing). */
+export type KindCalibration = {
+    readonly stateKind: string;
+    /** Scores below this are flagged as needing polish. `null`: none could be chosen. */
+    readonly threshold: number | null;
+    /** Examples of this state kind, all together. */
+    readonly samples: number;
+    /** How many of them are bad, and how many good. */
+    readonly positives: number;
+    readonly negatives: number;
+    /** Of what was flagged, the share that was bad; `null` when nothing was flagged. Measured on the examples named by `evaluatedOn`. */
+    readonly precision: number | null;
+    /** Of what was bad, the share that was flagged; `null` when nothing was bad. */
+    readonly recall: number | null;
+    /** True when the figures are measured on the examples the threshold was chosen on, which flatters them. */
+    readonly inSample: boolean;
+    /** Which examples the precision and recall are over: the `fit` examples (in-sample) or the `held-out` ones. */
+    readonly evaluatedOn: 'fit' | 'held-out';
+};
+/** The result of `calibrate`: per state kind, with the critic's provenance. */
+export type Calibration = {
+    readonly schema: 1;
+    readonly critic: {
+        readonly name: string;
+        readonly version: string;
+    };
+    readonly kinds: readonly KindCalibration[];
+};
+/**
+ * Measures a critic against labelled examples, per state kind, and proposes a threshold for each.
+ *
+ * The threshold of a state kind is chosen on its examples that are not `heldOut` (best F1, then best precision, then
+ * the lower threshold), among the midpoints between neighbouring scores. If there are held-out examples of that kind,
+ * precision and recall are measured on them and `inSample` is false; otherwise they are measured on the same examples
+ * the threshold was chosen on and `inSample` is true, so nobody mistakes them for evidence. A state kind without both
+ * a good and a bad example gets `threshold: null`. State kinds come out sorted.
+ *
+ * @param critic the critic whose scores these are.
+ * @param examples labelled examples; scores must be finite.
+ */
+export declare function calibrate(critic: {
+    readonly name: string;
+    readonly version: string;
+}, examples: readonly LabelledExample[]): Calibration;
+/** What happens when a score is below its threshold. `report` shows it and never fails anything. `warn` surfaces it on the command line. `gate` exits 1. */
+export type GateMode = 'report' | 'warn' | 'gate';
+/** The least evidence a calibration must have before `gate` is allowed. */
+export type MinimumEvidence = {
+    /** Bad examples and good examples of the state kind, each. Default 10. */
+    readonly perClass: number;
+    /** Whether precision and recall must be measured on held-out examples. Default true. */
+    readonly heldOut: boolean;
+};
+/** The default minimum evidence for `gate`: 10 bad and 10 good examples per state kind, measured on held-out examples. */
+export declare const DEFAULT_MINIMUM_EVIDENCE: MinimumEvidence;
+/** A score below its state kind's threshold. */
+export type BelowThreshold = {
+    readonly entryKey: string;
+    readonly stateKind: string;
+    readonly score: number;
+    readonly threshold: number;
+};
+/** The decision: the exit code and why. `refusal` is set when `gate` was asked for without enough evidence. */
+export type GateDecision = {
+    readonly mode: GateMode;
+    /** 0 it passes (always, in `report` and `warn`), 1 a threshold was crossed in `gate`, 2 `gate` was refused for lack of evidence. */
+    readonly exitCode: 0 | 1 | 2;
+    /** Scores below their threshold, in the order given. */
+    readonly below: readonly BelowThreshold[];
+    /** Entries with no score to judge (the critic failed or gave none). Never counted as below, never as fine. */
+    readonly unjudged: readonly string[];
+    /** For `warn`: one line per score below its threshold. */
+    readonly warnings: readonly string[];
+    /** For a refused `gate`: what is missing, per state kind. */
+    readonly refusal: readonly string[];
+};
+/**
+ * Decides what a set of critic scores means for the exit code of a command.
+ *
+ * `report` and `warn` always give exit code 0 (`warn` also lists a warning for each score below its threshold).
+ * `gate` gives 1 when any score is below its state kind's threshold, but first requires, for every state kind present, a
+ * calibration of this critic and version with a threshold and at least `minimum.perClass` bad and good examples,
+ * measured on held-out examples when `minimum.heldOut`. Without that it refuses (exit 2) and says what is missing.
+ * An explicit `thresholds` entry replaces the calibrated threshold of its state kind but never waives the evidence.
+ *
+ * @param args.scores the critic's scores, one per entry; `score: null` for an entry it could not judge.
+ * @param args.calibration the critic's calibration, when there is one.
+ * @param args.thresholds thresholds by state kind that replace the calibrated ones.
+ */
+export declare function gateDecision(args: {
+    readonly mode: GateMode;
+    readonly critic: {
+        readonly name: string;
+        readonly version: string;
+    };
+    readonly scores: readonly {
+        readonly entryKey: string;
+        readonly stateKind: string;
+        readonly score: number | null;
+    }[];
+    readonly calibration?: Calibration;
+    readonly thresholds?: Readonly<Record<string, number>>;
+    readonly minimum?: MinimumEvidence;
+}): GateDecision;

package/dist/critics/src/judging.js

@@ -0,0 +1,145 @@
+const round = (value) => Math.round(value * 100) / 100;
+/**
+ * Reads a score against the scores of the same state kind, and against the same entry's previous image.
+ *
+ * `percentile` is where the score falls among the population's scores of `stateKind` (ties count half), 0..100, higher
+ * is better; it is absent when no score of that state kind is known. Scores of other state kinds are ignored, so a
+ * loading overlay is compared with other loading overlays and never with full screens. `delta` is the score minus
+ * `previousScore` when there was one.
+ *
+ * @param args.score the entry's raw score.
+ * @param args.stateKind the kind of UI state of the entry.
+ * @param args.population every known score of the critic, with its state kind. May include the entry's own.
+ * @param args.previousScore the critic's score for the entry's previous image, when there was one.
+ */
+export function relativeVerdict(args) {
+    const peers = args.population.filter(item => item.stateKind === args.stateKind && Number.isFinite(item.score));
+    const below = peers.filter(item => item.score < args.score).length;
+    const equal = peers.filter(item => item.score === args.score).length;
+    return {
+        stateKind: args.stateKind,
+        sampleSize: peers.length,
+        ...(peers.length ? { percentile: round(((below + equal / 2) / peers.length) * 100) } : {}),
+        ...(args.previousScore !== undefined && Number.isFinite(args.previousScore) ? { previousScore: args.previousScore, delta: round(args.score - args.previousScore) } : {})
+    };
+}
+function measure(examples, threshold) {
+    let tp = 0, fp = 0, fn = 0;
+    for (const example of examples) {
+        const flagged = example.score < threshold;
+        if (flagged && example.bad)
+            tp++;
+        else if (flagged)
+            fp++;
+        else if (example.bad)
+            fn++;
+    }
+    const precision = tp + fp ? tp / (tp + fp) : null;
+    const recall = tp + fn ? tp / (tp + fn) : null;
+    return { precision, recall, f1: precision && recall ? (2 * precision * recall) / (precision + recall) : 0 };
+}
+/**
+ * Measures a critic against labelled examples, per state kind, and proposes a threshold for each.
+ *
+ * The threshold of a state kind is chosen on its examples that are not `heldOut` (best F1, then best precision, then
+ * the lower threshold), among the midpoints between neighbouring scores. If there are held-out examples of that kind,
+ * precision and recall are measured on them and `inSample` is false; otherwise they are measured on the same examples
+ * the threshold was chosen on and `inSample` is true, so nobody mistakes them for evidence. A state kind without both
+ * a good and a bad example gets `threshold: null`. State kinds come out sorted.
+ *
+ * @param critic the critic whose scores these are.
+ * @param examples labelled examples; scores must be finite.
+ */
+export function calibrate(critic, examples) {
+    const kinds = [...new Set(examples.map(example => example.stateKind))].sort();
+    return {
+        schema: 1,
+        critic: { name: critic.name, version: critic.version },
+        kinds: kinds.map(stateKind => {
+            const own = examples.filter(example => example.stateKind === stateKind && Number.isFinite(example.score));
+            const fit = own.filter(example => !example.heldOut);
+            const held = own.filter(example => example.heldOut);
+            const scores = [...new Set(fit.map(example => example.score))].sort((a, b) => a - b);
+            const usable = fit.some(example => example.bad) && fit.some(example => !example.bad);
+            let threshold = null;
+            if (usable) {
+                // Midpoints between neighbours, and one above the highest score (flag everything).
+                const candidates = [...scores.slice(1).map((score, index) => (score + scores[index]) / 2), scores[scores.length - 1] + 1];
+                let best = { f1: -1, precision: -1, threshold: 0 };
+                for (const candidate of candidates) {
+                    const { f1, precision } = measure(fit, candidate);
+                    if (f1 > best.f1 || (f1 === best.f1 && (precision ?? 0) > best.precision))
+                        best = { f1, precision: precision ?? 0, threshold: candidate };
+                }
+                threshold = round(best.threshold);
+            }
+            const evaluated = held.length ? held : fit;
+            const figures = threshold === null ? { precision: null, recall: null } : measure(evaluated, threshold);
+            return {
+                stateKind, threshold,
+                samples: own.length,
+                positives: own.filter(example => example.bad).length,
+                negatives: own.filter(example => !example.bad).length,
+                precision: figures.precision === null ? null : round(figures.precision),
+                recall: figures.recall === null ? null : round(figures.recall),
+                inSample: held.length === 0,
+                evaluatedOn: held.length ? 'held-out' : 'fit'
+            };
+        })
+    };
+}
+/** The default minimum evidence for `gate`: 10 bad and 10 good examples per state kind, measured on held-out examples. */
+export const DEFAULT_MINIMUM_EVIDENCE = { perClass: 10, heldOut: true };
+/**
+ * Decides what a set of critic scores means for the exit code of a command.
+ *
+ * `report` and `warn` always give exit code 0 (`warn` also lists a warning for each score below its threshold).
+ * `gate` gives 1 when any score is below its state kind's threshold, but first requires, for every state kind present, a
+ * calibration of this critic and version with a threshold and at least `minimum.perClass` bad and good examples,
+ * measured on held-out examples when `minimum.heldOut`. Without that it refuses (exit 2) and says what is missing.
+ * An explicit `thresholds` entry replaces the calibrated threshold of its state kind but never waives the evidence.
+ *
+ * @param args.scores the critic's scores, one per entry; `score: null` for an entry it could not judge.
+ * @param args.calibration the critic's calibration, when there is one.
+ * @param args.thresholds thresholds by state kind that replace the calibrated ones.
+ */
+export function gateDecision(args) {
+    const minimum = args.minimum ?? DEFAULT_MINIMUM_EVIDENCE;
+    const calibrated = args.calibration && args.calibration.critic.name === args.critic.name && args.calibration.critic.version === args.critic.version ? args.calibration : undefined;
+    const thresholdOf = (stateKind) => args.thresholds?.[stateKind] ?? calibrated?.kinds.find(kind => kind.stateKind === stateKind)?.threshold ?? null;
+    const unjudged = args.scores.filter(item => item.score === null).map(item => item.entryKey);
+    const below = [];
+    for (const item of args.scores) {
+        const threshold = thresholdOf(item.stateKind);
+        if (item.score !== null && threshold !== null && item.score < threshold)
+            below.push({ entryKey: item.entryKey, stateKind: item.stateKind, score: item.score, threshold });
+    }
+    const refusal = [];
+    if (args.mode === 'gate') {
+        if (args.calibration && !calibrated)
+            refusal.push(`The calibration is for ${args.calibration.critic.name}@${args.calibration.critic.version}, not ${args.critic.name}@${args.critic.version}: calibrate this version`);
+        for (const stateKind of [...new Set(args.scores.map(item => item.stateKind))].sort()) {
+            const kind = calibrated?.kinds.find(candidate => candidate.stateKind === stateKind);
+            if (!kind) {
+                if (!args.calibration || calibrated)
+                    refusal.push(`No calibration for state kind "${stateKind}": run \`atlas critic calibrate ${args.critic.name}\` with labelled examples of it`);
+                continue;
+            }
+            if (kind.threshold === null)
+                refusal.push(`Calibration for "${stateKind}" has no threshold: it needs both good and bad examples`);
+            if (kind.positives < minimum.perClass || kind.negatives < minimum.perClass)
+                refusal.push(`Calibration for "${stateKind}" has ${kind.positives} bad and ${kind.negatives} good examples; gate needs at least ${minimum.perClass} of each`);
+            if (minimum.heldOut && kind.inSample)
+                refusal.push(`Calibration for "${stateKind}" is in-sample: gate needs precision and recall measured on held-out examples`);
+        }
+    }
+    const refused = refusal.length > 0;
+    return {
+        mode: args.mode,
+        exitCode: refused ? 2 : args.mode === 'gate' && below.length ? 1 : 0,
+        below: refused ? [] : below,
+        unjudged,
+        warnings: args.mode === 'warn' ? below.map(item => `${item.entryKey} (${item.stateKind}) scored ${item.score}, below ${item.threshold}`) : [],
+        refusal
+    };
+}

package/dist/critics/src/publish.d.ts

@@ -0,0 +1,23 @@
+/**
+ * @module Publishing critiques
+ *
+ * `publishCritiques` publishes a critic's outcomes as one layer through any store, after checking that the layer is
+ * within what a critic-scoped credential may publish. The layer is content-addressed, so publishing the same outcomes
+ * again writes the same layer and changes nothing.
+ *
+ * @see docs/critics.md
+ */
+import type { ReportLayer } from '../../reports/src/layer-types.js';
+import type { ObjectStore } from '../../reports/src/object-store.js';
+import { type PublishChecksOptions } from '../../reports/src/publish-checks.js';
+import { critiqueLayer } from './critique.js';
+/**
+ * Publishes the layer {@link critiqueLayer} builds for `args`. Returns the stored layer, or `null` when there is nothing
+ * to publish (no outcomes). Throws when the layer would be outside a critic's scope, which cannot happen for a layer
+ * built here and is checked so that a change cannot make it happen quietly.
+ *
+ * @param store where to publish.
+ * @param args see {@link critiqueLayer}.
+ * @param options attachment limits, as for `publishChecks`.
+ */
+export declare function publishCritiques(store: ObjectStore, args: Parameters<typeof critiqueLayer>[0], options?: PublishChecksOptions): Promise<ReportLayer | null>;

package/dist/critics/src/publish.js

@@ -0,0 +1,21 @@
+import { publishChecks } from '../../reports/src/publish-checks.js';
+import { critiqueLayer } from './critique.js';
+import { criticScopeViolation } from './scope.js';
+/**
+ * Publishes the layer {@link critiqueLayer} builds for `args`. Returns the stored layer, or `null` when there is nothing
+ * to publish (no outcomes). Throws when the layer would be outside a critic's scope, which cannot happen for a layer
+ * built here and is checked so that a change cannot make it happen quietly.
+ *
+ * @param store where to publish.
+ * @param args see {@link critiqueLayer}.
+ * @param options attachment limits, as for `publishChecks`.
+ */
+export async function publishCritiques(store, args, options = {}) {
+    if (!args.outcomes.length)
+        return null;
+    const input = critiqueLayer(args);
+    const violation = criticScopeViolation({ producer: input.producer, parts: [{ kind: input.kind }] });
+    if (violation)
+        throw new Error(violation);
+    return publishChecks(store, input, options);
+}

package/dist/critics/src/runner.d.ts

@@ -0,0 +1,104 @@
+import type { Critique } from '../../schema/src/critique.js';
+import { type Critic, type JudgeableEntry } from './contract.js';
+import { type CritiqueOutcome } from './critique.js';
+/** The most a run may spend before it stops cleanly. Any limit left out is unlimited. */
+export type Budget = {
+    /** The most images to start judging in one run (retries of one image count once). */
+    readonly maxImages?: number;
+    /** The most tokens (input and output) to spend, summed from what critics report as `usage`. */
+    readonly maxTokens?: number;
+    /** The most money to spend, in US dollars, summed from what critics report as `usage.costUsd`. */
+    readonly maxCostUsd?: number;
+};
+/** What a run was told to do and how. Everything but the critic, the entries and `loadImage` has a default. */
+export type RunOptions = {
+    /** The critic to run. */
+    readonly critic: Critic;
+    /** The entries to judge, each with the hash of its stored image. Judged in this order; a repeated key is judged once. */
+    readonly entries: Iterable<JudgeableEntry> | AsyncIterable<JudgeableEntry>;
+    /** Fetches an entry's image bytes. Called only for entries that will be judged. The runner checks the bytes against `imageSha256`. */
+    readonly loadImage: (entry: JudgeableEntry) => Promise<Uint8Array>;
+    /** Critiques that already exist (read with `critiquesOfLayer`). One with a verdict and the same identity as this run's makes the image skipped; a failed one does not. */
+    readonly existing?: readonly Critique[];
+    /** Judge again even when a current critique exists. */
+    readonly force?: boolean;
+    /** Plan only: count what would be judged and skipped and say where images would go. Nothing is loaded, sent or returned to publish. */
+    readonly dryRun?: boolean;
+    /** Most calls in flight at once. Default 4. */
+    readonly concurrency?: number;
+    /** Most attempts for one image when the provider is rate limited. Default 4. */
+    readonly maxAttempts?: number;
+    /** A call that takes longer than this is recorded as `timeout`, and the run goes on. Default 60 000. */
+    readonly timeoutMs?: number;
+    /** Backoff between attempts: `baseMs` doubles each attempt up to `maxMs`, with full jitter, and never less than `Retry-After`. Defaults 500 and 30 000. */
+    readonly backoff?: {
+        readonly baseMs?: number;
+        readonly maxMs?: number;
+    };
+    /** Stop cleanly when any limit is reached. */
+    readonly budget?: Budget;
+    /** Credentials this run holds. Every recorded message has them removed. */
+    readonly secrets?: readonly string[];
+    /** Cancels the run: in-flight calls are aborted, what finished is returned. */
+    readonly signal?: AbortSignal;
+    /** Waits. Default `setTimeout`. Tests pass a fake. */
+    readonly sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+    /** A number in [0, 1) for jitter. Default `Math.random`. */
+    readonly random?: () => number;
+    /** Called after each entry is finished or skipped, for progress. Receives no image bytes and no credentials. */
+    readonly onProgress?: (event: ProgressEvent) => void;
+};
+/** One step of progress. `waiting` counts entries not yet started. */
+export type ProgressEvent = {
+    readonly entryKey: string;
+    readonly result: 'judged' | 'skipped' | 'failed';
+    readonly judged: number;
+    readonly skipped: number;
+    readonly failed: number;
+    readonly waiting: number;
+};
+/** Why a run ended before every entry was handled: it `budget`ed out or was `aborted`. */
+export type StoppedBy = 'budget' | 'aborted';
+/** What a run did. `outcomes` are what to publish (verdicts and failures); counts and `spent` are for the summary. */
+export type RunResult = {
+    readonly critic: {
+        readonly name: string;
+        readonly version: string;
+    };
+    /** Where images are sent, as the critic described it. */
+    readonly sendsImagesTo: string | null;
+    /** The critiques to publish, in entry order: verdicts and failures. */
+    readonly outcomes: readonly CritiqueOutcome[];
+    /** Entries with a current critique that were not judged again. */
+    readonly skipped: number;
+    /** Entries judged with a verdict. */
+    readonly judged: number;
+    /** Entries recorded as failed. */
+    readonly failed: number;
+    /** Entries left unhandled because the run stopped. */
+    readonly waiting: number;
+    /** Set when the run stopped early, and why. */
+    readonly stoppedBy?: StoppedBy;
+    /** What the run spent, from the usage critics reported. */
+    readonly spent: {
+        readonly images: number;
+        readonly tokens: number;
+        readonly costUsd: number;
+    };
+    /** Whether this was a dry run: nothing was loaded, sent or to publish. */
+    readonly dryRun: boolean;
+};
+/**
+ * Runs a critic over entries. Entries with a current critique of the same identity are skipped without fetching their
+ * image; the rest are judged with bounded concurrency, a timeout and retries. The run never throws for a critic's
+ * failure: it records it. It throws only for a wrong call (a non-positive `concurrency`, `maxAttempts` or `timeoutMs`).
+ *
+ * A run stops cleanly when a budget is reached or the signal aborts: calls in flight finish (or are aborted), what was
+ * finished is returned in `outcomes` to publish, and `stoppedBy` says why. With concurrency above 1 a budget can be
+ * overshot by the calls already in flight.
+ *
+ * @example
+ * const result = await runCritic({ critic, entries, loadImage: entry => fetchImage(entry), existing, budget: { maxImages: 200 } });
+ * await publishChecks(store, critiqueLayer({ critic, outcomes: result.outcomes, recordedAtMs: Date.now() }));
+ */
+export declare function runCritic(options: RunOptions): Promise<RunResult>;