@crvouga/atlas 1.21.0 → 1.22.0

package/CHANGELOG.md

@@ -1,6 +1,14 @@
 # Changelog: @crvouga/atlas
 
-## 1.21.0 (2026-10-29)
+## 1.22.0 (2026-10-29)
+
+### Features
+
+- watch, review and calibrate commands, finding overlays, agreement view and reviewer marks ([e871be2](https://github.com/crvouga/atlas/commit/e871be24e00b2faa97421c5e405e3790d60a10e9))
+- connect Read, Watch and Inspect across behaviors, runs and recordings ([b986ee5](https://github.com/crvouga/atlas/commit/b986ee56413fda0451cd020d0c251fd83d63c92b))
+- read the catalog, run critics from atlas critic, and list what needs polish ([91f1fb8](https://github.com/crvouga/atlas/commit/91f1fb8bca3e3f04c132e3114aa0fc3ef1b2f2db))
+
+## 1.21.0 (2026-10-28)
 
 ### Features
 

package/dist/core/src/cli.d.ts

@@ -2,13 +2,13 @@
  * @module The `atlas` command
  *
  * Dispatches `atlas <command>`: the statechart commands (`lint`, `plan`, `run`, `merge`, `export`) and, loaded on demand so
- * they cost nothing otherwise, `serve`, `dev`, `init` and `report`. Statechart configuration is `atlas.config.ts` (see
+ * they cost nothing otherwise, `serve`, `dev`, `init`, `report` and `critic`. Statechart configuration is `atlas.config.ts` (see
  * `defineConfig`). A command that is wrong prints a message and exits 2; a plan, lint or run that finds problems exits 1.
  *
  * @see docs/statecharts.md
  * @see docs/reference/cli.md
  */
 /** The text `atlas --help` prints: every command, and the flags of the statechart commands. */
-export declare const USAGE = "atlas \u2014 executable statecharts\n\n  atlas lint   [--config atlas.config.ts | --specs <dir>] [--require-implementations]\n  atlas plan   [--config atlas.config.ts] [--state <name>]\n  atlas run    [--config atlas.config.ts] [--mode fast|showcase] [--workers n] [--state <name>]\n               [--seed <name>]... [--paths id,id] [--shard i/n] [--rerun-failed | --reuse <run-dir>]\n               [--no-retry] [--output <dir>]\n  atlas merge  [--config atlas.config.ts] [--output <dir>] <run-dir> <run-dir>...\n  atlas export --specs <dir> --format scxml|mermaid|markdown|json [--out <file>]\n  atlas server [--port 8080]                                     one Atlas node: API, projects and client; needs ATLAS_S3_* (atlas serve --help)\n  atlas client build [--no-data] [--out <dir>]                    build the project-agnostic Atlas client\n  atlas dev    [--port 5180]                                     develop the Atlas server and client on one host\n  atlas init   [--server <url>] [--write]                         integrate this project: detect its suites, print the plan (atlas init --help)\n  atlas report publish|stream|status|merge|doctor|policy \u2026       layered evidence for any kind of spec (atlas report --help)\n  atlas ui-catalog plan|publish|status|diff|export \u2026              the UI catalog of captured states (atlas ui-catalog --help)\n\n  --workers       path attempts that run at once (default: one per CPU, capped by the driver)\n  --seed          only paths that start from this seed (\"start\" for paths setup starts); repeatable\n  --shard 2/4     run the second quarter of the plan; merge the shards' runs afterwards\n  --rerun-failed  keep the newest run's passed paths and run only the rest\n";
+export declare const USAGE = "atlas \u2014 executable statecharts\n\n  atlas lint   [--config atlas.config.ts | --specs <dir>] [--require-implementations]\n  atlas plan   [--config atlas.config.ts] [--state <name>]\n  atlas run    [--config atlas.config.ts] [--mode fast|showcase] [--workers n] [--state <name>]\n               [--seed <name>]... [--paths id,id] [--shard i/n] [--rerun-failed | --reuse <run-dir>]\n               [--no-retry] [--output <dir>]\n  atlas merge  [--config atlas.config.ts] [--output <dir>] <run-dir> <run-dir>...\n  atlas export --specs <dir> --format scxml|mermaid|markdown|json [--out <file>]\n  atlas server [--port 8080]                                     one Atlas node: API, projects and client; needs ATLAS_S3_* (atlas serve --help)\n  atlas client build [--no-data] [--out <dir>]                    build the project-agnostic Atlas client\n  atlas dev    [--port 5180]                                     develop the Atlas server and client on one host\n  atlas init   [--server <url>] [--write]                         integrate this project: detect its suites, print the plan (atlas init --help)\n  atlas report publish|stream|status|merge|doctor|policy \u2026       layered evidence for any kind of spec (atlas report --help)\n  atlas critic list|setup|doctor|run \u2026                           automated reviewers that mark which UIs need polish (atlas critic --help)\n  atlas ui-catalog plan|publish|status|diff|export \u2026              the UI catalog of captured states (atlas ui-catalog --help)\n\n  --workers       path attempts that run at once (default: one per CPU, capped by the driver)\n  --seed          only paths that start from this seed (\"start\" for paths setup starts); repeatable\n  --shard 2/4     run the second quarter of the plan; merge the shards' runs afterwards\n  --rerun-failed  keep the newest run's passed paths and run only the rest\n";
 /** Runs `atlas` with `argv` (default: the process arguments) and returns the exit code. Throws for a command that is wrong, which the `atlas` binary reports as a message and exit code 2. */
 export declare function main(argv?: string[]): Promise<number>;

package/dist/core/src/cli.js

@@ -2,7 +2,7 @@
  * @module The `atlas` command
  *
  * Dispatches `atlas <command>`: the statechart commands (`lint`, `plan`, `run`, `merge`, `export`) and, loaded on demand so
- * they cost nothing otherwise, `serve`, `dev`, `init` and `report`. Statechart configuration is `atlas.config.ts` (see
+ * they cost nothing otherwise, `serve`, `dev`, `init`, `report` and `critic`. Statechart configuration is `atlas.config.ts` (see
  * `defineConfig`). A command that is wrong prints a message and exits 2; a plan, lint or run that finds problems exits 1.
  *
  * @see docs/statecharts.md
@@ -33,6 +33,7 @@ export const USAGE = `atlas — executable statecharts
   atlas dev    [--port 5180]                                     develop the Atlas server and client on one host
   atlas init   [--server <url>] [--write]                         integrate this project: detect its suites, print the plan (atlas init --help)
   atlas report publish|stream|status|merge|doctor|policy …       layered evidence for any kind of spec (atlas report --help)
+  atlas critic list|setup|doctor|run …                           automated reviewers that mark which UIs need polish (atlas critic --help)
   atlas ui-catalog plan|publish|status|diff|export …              the UI catalog of captured states (atlas ui-catalog --help)
 
   --workers       path attempts that run at once (default: one per CPU, capped by the driver)
@@ -104,6 +105,8 @@ export async function main(argv = process.argv.slice(2)) {
     }
     if (command === 'init')
         return (await import('../../reports/src/init.js')).initMain(argv.slice(1), { out: text => process.stdout.write(text) });
+    if (command === 'critic')
+        return (await import('../../critics/src/cli.js')).runCriticCommand(argv.slice(1));
     if (command === 'report')
         return (await import('../../reports/src/report-cli.js')).runReportCommand(argv.slice(1));
     if (command === 'ui-catalog')

package/dist/critics/src/catalog-client.d.ts

@@ -0,0 +1,91 @@
+import { type CatalogEntriesQuery, type CatalogEntry } from '../../reports/src/ui-catalog-entries.js';
+/** How to reach a server. `token` is a `critic`, `publish` or `admin` token for reading the catalog; any token that may read works. */
+export type CatalogClientOptions = {
+    /** The Atlas server, such as `https://atlas.example.com`. */
+    readonly url: string;
+    /** Sent as `Authorization: Bearer`. Never put in the URL, never in an error. */
+    readonly token?: string;
+    /** A project of a shared node. Omit for the default project. */
+    readonly project?: string;
+    /** For tests and special runtimes. Default `fetch`. */
+    readonly fetch?: typeof fetch;
+    /** Retries after the first attempt for network errors, 5xx and 429. Default 3. */
+    readonly retries?: number;
+    /** Waits. Default `setTimeout`. */
+    readonly sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+};
+/** One page of the catalog as the server answered it. */
+export type CatalogPage = {
+    /** The report version this was read at. */
+    readonly version: string;
+    readonly entries: readonly CatalogEntry[];
+    /** Entries that match, across all pages. */
+    readonly total: number;
+    /** Pass as `cursor` to read the next page; absent on the last. */
+    readonly nextCursor?: string;
+    /** Present when `since` was asked: whether the server remembered that version. When it did not, `entries` is the whole catalog. */
+    readonly since?: {
+        readonly requested: string;
+        readonly honored: boolean;
+    };
+    /** Captured cells that could not be offered for judging because no stored picture of a known hash backs them. */
+    readonly unjudgeable: number;
+    /** What the server could not read. */
+    readonly notices: readonly string[];
+};
+/** Everything a pass over the catalog saw, and the version to resume from. */
+export type CatalogRead = {
+    /** The version of the first page. Resume with `since` this: it finds whatever changed while the rest was read. */
+    readonly version: string;
+    readonly entries: readonly CatalogEntry[];
+    readonly unjudgeable: number;
+    readonly notices: readonly string[];
+    /** False when a `since` was asked and the server did not remember it, so `entries` is the whole catalog. */
+    readonly sinceHonored: boolean;
+};
+/** Something new in the catalog, as {@link CatalogClient.follow} hands it over. */
+export type CatalogBatch = {
+    /** The report version this batch brings the follower up to. */
+    readonly version: string;
+    /** Entries not seen before, or whose picture changed; none twice. */
+    readonly entries: readonly CatalogEntry[];
+    /** True for the first batch (the catch-up on connecting), false for what arrived afterwards. */
+    readonly initial: boolean;
+};
+/** An Atlas server refused or could not answer, with what to do about it. Its message never holds a token. */
+export declare class CatalogClientError extends Error {
+    readonly status: number | null;
+    /** @param status the HTTP status, or null when the server could not be reached. */
+    constructor(message: string, status: number | null);
+}
+/** A connection to the catalog of one Atlas server. */
+export type CatalogClient = {
+    /** The server address, with no secrets. */
+    readonly url: string;
+    /** One page of entries for a query (`cursor`, `limit`, `since`, `unjudgedBy`, filters). */
+    page(query?: CatalogEntriesQuery): Promise<CatalogPage>;
+    /** Every page of a query, in key order, each entry once. `query.cursor` is ignored. */
+    read(query?: Omit<CatalogEntriesQuery, 'cursor'>): Promise<CatalogRead>;
+    /** The picture of an entry: its stored bytes, checked against `entry.image.sha256`. Throws if they differ. */
+    image(entry: Pick<CatalogEntry, 'image'>): Promise<Uint8Array>;
+    /**
+     * Catches up with the catalog and then follows it: a batch on connecting (everything, or what changed since `since`), then
+     * a batch each time the server announces a change. A dropped connection is reconnected after a pause, and the next batch
+     * is a `since` read from the version held, so nothing is missed and nothing is handed over twice. Ends when `signal` aborts.
+     */
+    follow(options?: {
+        readonly since?: string;
+        readonly query?: Omit<CatalogEntriesQuery, 'cursor' | 'since'>;
+        readonly signal?: AbortSignal;
+        readonly reconnectMs?: number;
+    }): AsyncGenerator<CatalogBatch, void, void>;
+};
+/**
+ * A client for the catalog of an Atlas server. Nothing is read until a method is called.
+ *
+ * @example
+ * const catalog = createCatalogClient({ url: 'https://atlas.example.com', token: process.env.ATLAS_CRITIC_TOKEN });
+ * const { version, entries } = await catalog.read({ unjudgedBy: { name: 'uiclip', version: '1.0.0' } });
+ * for (const entry of entries) judge(entry, await catalog.image(entry));
+ */
+export declare function createCatalogClient(options: CatalogClientOptions): CatalogClient;

package/dist/critics/src/catalog-client.js

@@ -0,0 +1,194 @@
+/**
+ * @module The catalog client
+ *
+ * How a critic process reads the UI catalog from an Atlas server: list entries a page at a time with a stable cursor
+ * (`page`, `entries`), read only what changed since a report version it holds (`since`), follow the server's event stream and
+ * catch up after every announcement or reconnection without missing an entry or handling one twice (`follow`), and fetch an
+ * entry's picture by the hash it was stored under, checking the bytes against it (`image`). It needs only `fetch`: no
+ * browser, no model, no SDK. The token goes in `Authorization: Bearer`, never in a URL, and is never put in an error.
+ *
+ * A missed event costs nothing: every wake-up is followed by a `since` read, and a version the server does not remember is
+ * answered with the whole catalog, which the client de-duplicates by entry key and picture hash.
+ *
+ * @see docs/critics.md#reading-the-catalog
+ */
+import { CatalogEntriesSchema } from '../../schema/src/api.js';
+import { catalogEntriesParams } from '../../reports/src/ui-catalog-entries.js';
+import { sha256Hex } from '../../reports/src/sigv4.js';
+/** An Atlas server refused or could not answer, with what to do about it. Its message never holds a token. */
+export class CatalogClientError extends Error {
+    status;
+    /** @param status the HTTP status, or null when the server could not be reached. */
+    constructor(message, status) {
+        super(message);
+        this.status = status;
+        this.name = 'CatalogClientError';
+    }
+}
+const defaultSleep = (ms, signal) => new Promise(resolve => {
+    const timer = setTimeout(done, ms);
+    function done() { clearTimeout(timer); signal?.removeEventListener('abort', done); resolve(); }
+    signal?.addEventListener('abort', done, { once: true });
+});
+const encode = (path) => path.split('/').map(encodeURIComponent).join('/');
+/**
+ * A client for the catalog of an Atlas server. Nothing is read until a method is called.
+ *
+ * @example
+ * const catalog = createCatalogClient({ url: 'https://atlas.example.com', token: process.env.ATLAS_CRITIC_TOKEN });
+ * const { version, entries } = await catalog.read({ unjudgedBy: { name: 'uiclip', version: '1.0.0' } });
+ * for (const entry of entries) judge(entry, await catalog.image(entry));
+ */
+export function createCatalogClient(options) {
+    const base = options.url.replace(/\/+$/, '');
+    const api = options.project ? `${base}/api/projects/${encodeURIComponent(options.project)}` : `${base}/api`;
+    const doFetch = options.fetch ?? fetch;
+    const sleep = options.sleep ?? defaultSleep;
+    const retries = options.retries ?? 3;
+    const headers = options.token ? { authorization: `Bearer ${options.token}` } : {};
+    function explain(status, said) {
+        const tail = said ? `: ${said.replace(/[.\s]+$/, '')}` : '';
+        if (status === 401)
+            return new CatalogClientError(`${base} wants a token${tail}. Set ATLAS_CRITIC_TOKEN (or ATLAS_TOKEN): a critic, publish or admin token.`, status);
+        if (status === 403)
+            return new CatalogClientError(`${base} knows this token but it may not do that${tail}.`, status);
+        if (status === 404)
+            return new CatalogClientError(`${base} has no such thing${tail}. An older server may not have the catalog list: upgrade it.`, status);
+        return new CatalogClientError(`${base} answered ${status}${tail}`, status);
+    }
+    async function request(url, init = {}) {
+        for (let attempt = 0;; attempt++) {
+            let failure;
+            try {
+                const response = await doFetch(url, { headers: { ...headers, ...(init.accept ? { accept: init.accept } : {}) }, ...(init.signal ? { signal: init.signal } : {}) });
+                if (response.ok)
+                    return response;
+                const said = await response.json().then((body) => (body && typeof body === 'object' && 'error' in body && typeof body.error === 'string' ? body.error : null), () => null);
+                failure = explain(response.status, said);
+                if (response.status < 500 && response.status !== 429)
+                    throw failure;
+            }
+            catch (error) {
+                if (init.signal?.aborted)
+                    throw error;
+                if (error instanceof CatalogClientError && error.status !== null && error.status < 500 && error.status !== 429)
+                    throw error;
+                failure = error instanceof CatalogClientError ? error : new CatalogClientError(`cannot reach ${base} (${error instanceof Error ? error.message : String(error)})`, null);
+            }
+            if (attempt >= retries)
+                throw failure;
+            await sleep(Math.min(8_000, 100 * 2 ** attempt) * (0.5 + Math.random() / 2), init.signal);
+        }
+    }
+    async function page(query = {}) {
+        const search = catalogEntriesParams(query).toString();
+        const response = await request(`${api}/reports/ui-catalog/entries${search ? `?${search}` : ''}`, { accept: 'application/json' });
+        const parsed = CatalogEntriesSchema.safeParse(await response.json().catch(() => null));
+        // A document this reader cannot read is unknown, never an empty catalog.
+        if (!parsed.success)
+            throw new CatalogClientError(`${base} answered something that is not a catalog list. Upgrade @crvouga/atlas, or the server.`, response.status);
+        const document = parsed.data;
+        return {
+            version: document.version, entries: document.entries.map(entry => ({ ...entry, marks: entry.marks ?? [] })), total: document.total, unjudgeable: document.unjudgeable, notices: document.sources.notices,
+            ...(document.nextCursor ? { nextCursor: document.nextCursor } : {}),
+            ...(document.since ? { since: document.since } : {})
+        };
+    }
+    async function read(query = {}) {
+        const entries = [];
+        let version, unjudgeable = 0, notices = [], sinceHonored = true;
+        for (let cursor;;) {
+            const found = await page({ ...query, ...(cursor === undefined ? {} : { cursor }) });
+            if (version === undefined) {
+                version = found.version;
+                sinceHonored = found.since?.honored ?? true;
+            }
+            unjudgeable = found.unjudgeable;
+            notices = found.notices;
+            entries.push(...found.entries);
+            if (!found.nextCursor)
+                break;
+            cursor = found.nextCursor;
+        }
+        return { version: version, entries, unjudgeable, notices, sinceHonored };
+    }
+    async function image(entry) {
+        const response = await request(`${api}/reports${encode(entry.image.path)}`);
+        const bytes = new Uint8Array(await response.arrayBuffer());
+        if ((await sha256Hex(bytes)) !== entry.image.sha256)
+            throw new CatalogClientError(`The picture ${entry.image.path} is not the bytes the catalog hashed (${entry.image.sha256.slice(0, 12)}…); nothing was judged.`, response.status);
+        return bytes;
+    }
+    async function* follow(followOptions = {}) {
+        const signal = followOptions.signal;
+        let held = followOptions.since;
+        let initial = true;
+        // What was handed over: an entry is new when its key is unseen or its picture changed.
+        const seen = new Map();
+        async function* catchUp() {
+            const found = await read({ ...(followOptions.query ?? {}), ...(held === undefined ? {} : { since: held }) });
+            const fresh = found.entries.filter(entry => seen.get(entry.key) !== entry.image.sha256);
+            for (const entry of fresh)
+                seen.set(entry.key, entry.image.sha256);
+            held = found.version;
+            if (fresh.length || initial) {
+                const batch = { version: found.version, entries: fresh, initial };
+                initial = false;
+                yield batch;
+            }
+        }
+        while (!signal?.aborted) {
+            let response;
+            try {
+                response = await request(`${api}/events`, { accept: 'text/event-stream', ...(signal ? { signal } : {}) });
+            }
+            catch (error) {
+                if (signal?.aborted)
+                    return;
+                // A refusal will not pass by waiting; the server being away will.
+                if (error instanceof CatalogClientError && error.status !== null && error.status !== 503)
+                    throw error;
+            }
+            if (response?.body) {
+                // Whatever happened while disconnected is found by this read, not by the events that were missed.
+                try {
+                    yield* catchUp();
+                    const reader = response.body.getReader();
+                    const decoder = new TextDecoder();
+                    let buffered = '';
+                    const onAbort = () => { reader.cancel().catch(() => { }); };
+                    signal?.addEventListener('abort', onAbort, { once: true });
+                    try {
+                        for (;;) {
+                            const chunk = await reader.read();
+                            if (chunk.done)
+                                break;
+                            buffered += decoder.decode(chunk.value, { stream: true });
+                            let end;
+                            while ((end = buffered.indexOf('\n\n')) >= 0) {
+                                const frame = buffered.slice(0, end);
+                                buffered = buffered.slice(end + 2);
+                                const name = /^event: (.+)$/m.exec(frame)?.[1];
+                                if (name === 'report')
+                                    yield* catchUp();
+                            }
+                        }
+                    }
+                    finally {
+                        signal?.removeEventListener('abort', onAbort);
+                    }
+                }
+                catch (error) {
+                    if (signal?.aborted)
+                        return;
+                    if (error instanceof CatalogClientError && error.status !== null && error.status !== 503)
+                        throw error;
+                }
+            }
+            if (signal?.aborted)
+                return;
+            await sleep(followOptions.reconnectMs ?? 2_000, signal);
+        }
+    }
+    return { url: base, page, read, image, follow };
+}

package/dist/critics/src/cli-calibrate.d.ts

@@ -0,0 +1,4 @@
+import { type CriticIo } from './cli-common.js';
+import { type LoadedConfig } from './registry.js';
+/** `atlas critic calibrate <name>`. */
+export declare function calibrateCommand(argv: readonly string[], io: CriticIo, loaded: LoadedConfig): Promise<number>;

package/dist/critics/src/cli-calibrate.js

@@ -0,0 +1,211 @@
+/**
+ * @module `atlas critic calibrate`
+ *
+ * Measures a critic against labelled examples, per state kind, and proposes a threshold for each: the library function
+ * `calibrate` behind a command. The examples are pictures in directories (`--good`, `--bad`, and `--heldout-good`, `--heldout-bad` for
+ * pictures the threshold must not be chosen on; a subdirectory's name is the state kind of the pictures in it, else `--state`, else
+ * `unknown`), and/or the whole-critique marks reviewers made (`--from-marks`: accepted means the UI needs polish, dismissed means it
+ * does not). The critic judges the pictures here, with the same consent, timeouts and budgets as `run`. Nothing is published: the
+ * calibration is a document (`--out`) that `atlas critic run --calibration` reads. Figures measured on the examples the threshold was
+ * chosen on are labelled in-sample; only held-out figures can open `--mode gate`.
+ *
+ * @see docs/critics.md#state-aware-judging-calibration-and-modes
+ */
+import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { markStanding } from '../../reports/src/critique-review.js';
+import { sha256Hex } from '../../reports/src/sigv4.js';
+import { createCatalogClient } from './catalog-client.js';
+import { connection, defaultAsk, list, numberFlag, positionals, print, redactIn, value, wantsJson } from './cli-common.js';
+import { calibrate, gateDecision } from './judging.js';
+import { buildCritic } from './registry.js';
+import { runCritic } from './runner.js';
+const IMAGE = /\.(png|jpe?g|webp|gif)$/i;
+const GROUPS = ['good', 'bad', 'heldout-good', 'heldout-bad'];
+async function filesOf(directory) {
+    const found = [];
+    const walk = async (current) => {
+        for (const entry of (await readdir(current, { withFileTypes: true })).sort((a, b) => (a.name < b.name ? -1 : 1))) {
+            const full = path.join(current, entry.name);
+            if (entry.isDirectory())
+                await walk(full);
+            else if (IMAGE.test(entry.name))
+                found.push(full);
+        }
+    };
+    await walk(directory);
+    return found;
+}
+/** `atlas critic calibrate <name>`. */
+export async function calibrateCommand(argv, io, loaded) {
+    const env = io.env ?? process.env;
+    const json = wantsJson(argv);
+    const interactive = !argv.includes('--non-interactive') && !json && (io.interactive ?? Boolean(process.stdin.isTTY && process.stdout.isTTY));
+    const ask = io.ask ?? defaultAsk;
+    const dryRun = argv.includes('--dry-run');
+    const cwd = io.cwd ?? process.cwd();
+    const [name] = positionals(argv);
+    if (!name)
+        throw new Error(`calibrate needs the name of a critic: ${Object.keys(loaded.config.critics).join(', ') || 'none are configured in atlas.critics.json'}`);
+    const dirs = Object.fromEntries(GROUPS.map(group => [group, list(argv, group)]));
+    const fromMarks = argv.includes('--from-marks');
+    const haveDirs = GROUPS.some(group => dirs[group].length > 0);
+    if (!haveDirs && !fromMarks)
+        throw new Error('calibrate needs labelled examples: --good <dir> and --bad <dir> (and --heldout-good / --heldout-bad for pictures the threshold must not be chosen on), or --from-marks to use the marks reviewers made on a server');
+    const defaultState = value(argv, 'state') ?? 'unknown';
+    const server = fromMarks ? await connection(argv, loaded, env) : null;
+    if (fromMarks && !server)
+        throw new Error('--from-marks needs --server <url> (or "server.url" in atlas.critics.json, or ATLAS_SERVER_URL)');
+    const sources = [];
+    for (const group of GROUPS)
+        for (const directory of dirs[group]) {
+            const root = path.resolve(cwd, directory);
+            let files;
+            try {
+                files = await filesOf(root);
+            }
+            catch {
+                throw new Error(`--${group} ${directory} cannot be read as a directory of pictures`);
+            }
+            if (!files.length)
+                throw new Error(`--${group} ${directory} holds no pictures (png, jpg, webp or gif)`);
+            for (const file of files) {
+                const relative = path.relative(root, file).split(path.sep);
+                sources.push({ key: `${group}/${path.basename(directory)}/${relative.join('/')}`, group, stateKind: relative.length > 1 ? relative[0] : defaultState, file });
+            }
+        }
+    const skipped = [];
+    const examples = [];
+    let version = value(argv, 'critic-version') ?? null;
+    let model;
+    let parameters;
+    let sendsImagesTo = null;
+    let fileExamples = 0;
+    if (haveDirs) {
+        const built = await buildCritic(loaded, name, io);
+        const secrets = built.secrets;
+        const description = built.critic.describe();
+        version = built.critic.version;
+        model = description.model ? { ...description.model } : undefined;
+        parameters = description.parameters ? { ...description.parameters } : undefined;
+        sendsImagesTo = description.sendsImagesTo;
+        if (!dryRun && sendsImagesTo !== null && !argv.includes('--yes')) {
+            if (!interactive)
+                throw new Error(`${name} sends images to ${sendsImagesTo}: pass --yes to allow it (nothing was sent)`);
+            const answer = await ask(`${name} will send ${sources.length} image${sources.length === 1 ? '' : 's'} to ${sendsImagesTo}. Continue? [y/N] `);
+            if (!/^y(es)?$/i.test(answer.trim()))
+                throw new Error(`Stopped: ${name} would send images to ${sendsImagesTo} and that was not confirmed (nothing was sent)`);
+        }
+        if (!dryRun)
+            await built.ready();
+        const bytes = new Map();
+        const entries = [];
+        for (const source of sources) {
+            const data = new Uint8Array(await readFile(source.file));
+            bytes.set(source.key, data);
+            entries.push({ key: source.key, title: path.basename(source.file), state: source.stateKind, imageSha256: await sha256Hex(data) });
+        }
+        const maxImages = numberFlag(argv, 'max-images', true), concurrency = numberFlag(argv, 'concurrency', true), timeoutMs = numberFlag(argv, 'timeout-ms', true);
+        const result = await runCritic({
+            critic: built.critic, entries, force: true, dryRun, loadImage: entry => Promise.resolve(bytes.get(entry.key)), secrets,
+            ...(maxImages ? { budget: { maxImages } } : {}), ...(concurrency ? { concurrency } : {}), ...(timeoutMs ? { timeoutMs } : {})
+        });
+        if (dryRun) {
+            const document = { ok: true, dryRun, critic: { name, version }, sendsImagesTo, examples: { files: sources.length, marks: 0, used: 0, skipped }, calibration: null, written: null, gate: null, warnings: [] };
+            if (json)
+                print(io, document);
+            else
+                io.out(`Would judge ${sources.length} ${sources.length === 1 ? 'picture' : 'pictures'} with ${name}@${version}. ${sendsImagesTo ? `Images would go to ${sendsImagesTo}` : 'Images stay on this machine'}. Nothing was sent or written.\n`);
+            return 0;
+        }
+        for (const outcome of result.outcomes) {
+            const source = sources.find(candidate => candidate.key === outcome.entry.key);
+            const score = outcome.critique.verdict?.score;
+            if (outcome.critique.failure || score === undefined) {
+                skipped.push({ source: source.key, reason: outcome.critique.failure ? `the critic could not judge it (${outcome.critique.failure.reason})` : 'the critic gave no score' });
+                continue;
+            }
+            examples.push({ score, stateKind: source.stateKind, bad: source.group.endsWith('bad'), ...(source.group.startsWith('heldout') ? { heldOut: true } : {}) });
+            fileExamples += 1;
+        }
+        for (const source of sources)
+            if (!result.outcomes.some(outcome => outcome.entry.key === source.key))
+                skipped.push({ source: source.key, reason: result.stoppedBy === 'budget' ? 'the budget was reached before it was judged' : 'it was not judged' });
+    }
+    // The marks: a whole-critique accept says the UI needs polish, a dismissal says it does not; findings are not examples.
+    let markExamples = 0;
+    if (fromMarks && server) {
+        const client = createCatalogClient({ url: server.url, ...(server.token ? { token: server.token } : {}), ...(server.project ? { project: server.project } : {}), ...(io.fetch ? { fetch: io.fetch } : {}), ...(io.sleep ? { sleep: io.sleep } : {}) });
+        const read = await client.read({});
+        for (const entry of read.entries) {
+            const found = entry.critiques.filter(item => item.currency === 'current' && item.critique.critic.name === name && !item.critique.failure && item.critique.verdict?.score !== undefined);
+            const standing = markStanding(entry.marks, { critic: name });
+            if (standing.state === 'unmarked' && !standing.earlier)
+                continue;
+            if (!found.length) {
+                skipped.push({ source: entry.key, reason: `${name} has no current score for it` });
+                continue;
+            }
+            if (version === null)
+                version = found[0].critique.critic.version;
+            const critique = found.find(item => item.critique.critic.version === version);
+            if (!critique) {
+                skipped.push({ source: entry.key, reason: `${name} judged it as version ${found[0].critique.critic.version}, not ${version}` });
+                continue;
+            }
+            if (standing.state === 'unmarked') {
+                skipped.push({ source: entry.key, reason: 'its marks were made about an earlier picture' });
+                continue;
+            }
+            if (standing.state === 'disputed') {
+                skipped.push({ source: entry.key, reason: 'reviewers disagree' });
+                continue;
+            }
+            examples.push({ score: critique.critique.verdict.score, stateKind: entry.state, bad: standing.state === 'accepted' });
+            markExamples += 1;
+        }
+        if (version === null)
+            version = read.entries.flatMap(entry => entry.critiques).find(item => item.critique.critic.name === name)?.critique.critic.version ?? null;
+    }
+    if (version === null)
+        throw new Error(`${name} has no critique on the server to calibrate (give --critic-version, or run it first)`);
+    const measured = calibrate({ name, version }, examples);
+    const calibration = {
+        ...measured,
+        critic: { ...measured.critic, ...(model ? { model } : {}), ...(parameters ? { parameters } : {}) },
+        createdAtMs: (io.clock ?? Date.now)()
+    };
+    const refusal = gateDecision({ mode: 'gate', critic: { name, version }, scores: calibration.kinds.map(kind => ({ entryKey: kind.stateKind, stateKind: kind.stateKind, score: null })), calibration }).refusal;
+    const warnings = [];
+    if (!examples.length)
+        warnings.push('No example had a score, so there is nothing to measure');
+    let written = null;
+    const out = value(argv, 'out');
+    if (out) {
+        const target = path.resolve(cwd, out);
+        await mkdir(path.dirname(target), { recursive: true });
+        await writeFile(target, `${JSON.stringify(calibration, null, 2)}\n`);
+        written = path.relative(cwd, target) || target;
+    }
+    const document = {
+        ok: true, dryRun, critic: { name, version }, sendsImagesTo,
+        examples: { files: fileExamples, marks: markExamples, used: examples.length, skipped },
+        calibration, written, gate: { eligible: refusal.length === 0 && examples.length > 0, reasons: refusal },
+        warnings: warnings.map(warning => redactIn(warning, []))
+    };
+    if (json) {
+        print(io, document);
+        return 0;
+    }
+    io.out(`Calibration of ${name}@${version}: ${examples.length} ${examples.length === 1 ? 'example' : 'examples'} (${fileExamples} from pictures, ${markExamples} from marks)${skipped.length ? `, ${skipped.length} left out` : ''}\n`);
+    for (const kind of calibration.kinds) {
+        io.out(`  ${kind.stateKind.padEnd(12)} ${kind.samples} samples (${kind.positives} bad, ${kind.negatives} good)  threshold ${kind.threshold ?? 'none'}  precision ${kind.precision ?? 'n/a'}  recall ${kind.recall ?? 'n/a'}  ${kind.inSample ? 'IN-SAMPLE: measured on the examples the threshold was chosen on' : 'held-out'}\n`);
+    }
+    for (const item of skipped.slice(0, 10))
+        io.out(`  left out ${item.source}: ${item.reason}\n`);
+    io.out(document.gate.eligible ? 'Gate mode would accept this calibration.\n' : `Gate mode would refuse this calibration:\n${refusal.map(line => `  - ${line}`).join('\n')}\n`);
+    io.out(written ? `Written to ${written}. Use it with \`atlas critic run --mode gate --calibration ${written}\`.\n` : 'Not written: pass --out <file> to keep it.\n');
+    for (const warning of document.warnings)
+        io.out(`! ${warning}\n`);
+    return 0;
+}