@crvouga/atlas 1.20.0 → 1.22.0

package/dist/critics/src/cli.d.ts

@@ -0,0 +1,13 @@
+import { type CriticIo } from './cli-common.js';
+export type { CriticIo } from './cli-common.js';
+/** The text `atlas critic --help` prints. Also the source of the command reference in `docs/reference/cli.md`. */
+export declare const CRITIC_USAGE = "atlas critic \u2014 automated reviewers of the UI catalog\n\n  atlas critic list    [--config atlas.critics.json] [--format text|json]\n  atlas critic setup   <name> [--config atlas.critics.json] [--format text|json]\n  atlas critic doctor  [--server <url>] [--critic <name>]... [--config atlas.critics.json] [--format text|json]\n  atlas critic run     --critic <name>... ( --all | --unjudged | --since <version> | filters ) [--server <url>]\n                       [--area a,b] [--profile key] [--state screen,overlay] [--route r] [--q words]\n                       [--dry-run] [--yes] [--non-interactive] [--force]\n                       [--max-images n] [--max-tokens n] [--max-cost usd] [--concurrency n] [--timeout-ms n]\n                       [--mode report|warn|gate] [--threshold state=score]... [--calibration file.json]\n                       [--spool dir] [--project slug] [--config atlas.critics.json] [--format text|json]\n  atlas critic watch   --critic <name>... [--server <url>] [--since <version>] [filters] [--yes] [--dry-run]\n                       [--max-batches n] [--max-images n] [--max-tokens n] [--max-cost usd] [--concurrency n] [--timeout-ms n]\n                       [--spool dir] [--project slug] [--config atlas.critics.json] [--format text|json]\n  atlas critic review  [--server <url>] [--critic <name>] [--state kind] [--limit n] [--list] [--non-interactive]\n                       [--entry <key> ( --accept | --dismiss ) [--finding <id>]... [--note text] [--reviewer name] [--dry-run]]\n                       [--project slug] [--config atlas.critics.json] [--format text|json]\n  atlas critic calibrate <name> ( --good <dir>... --bad <dir>... [--heldout-good <dir>...] [--heldout-bad <dir>...] [--state kind]\n                       | --from-marks [--server <url>] [--critic-version v] ) [--out file.json] [--dry-run] [--yes]\n                       [--max-images n] [--concurrency n] [--timeout-ms n] [--config atlas.critics.json] [--format text|json]\n\n  A critic is a producer: it runs here (on a laptop, in CI or in a worker), reads the catalog from an Atlas server, and publishes\n  what it thinks as layers of critiques. The server never runs a model and never holds a provider key. A critique is an opinion,\n  not a test result: it never changes whether a report passes.\n\n  Configuration is atlas.critics.json, which may be committed: it names credentials by reference (env:NAME, file:PATH) and a\n  literal credential in it is refused. The server is --server, \"server.url\" there, ATLAS_SERVER_URL or an https ATLAS_STORE;\n  the token is ATLAS_CRITIC_TOKEN (or ATLAS_TOKEN), never a flag, and is never sent over plain http beyond this machine.\n\n  list      The critics available and configured: type, whether each is ready or needs setup, and where it sends images.\n  setup     Provision a critic: fetch and verify a model checkpoint against its pinned SHA-256 (a bad checksum is an error), or\n            check that a credential reference resolves. This is the only place anything is downloaded; judging never downloads.\n  doctor    Check the server is reachable, the token is accepted and has the scope a critic needs (and warn when the server is\n            open, so every caller is an administrator), that the server lists the catalog, and for each configured critic its\n            model file, dependencies, credential and provider. Each problem is reported separately, with how to fix it.\n  run       Judge the entries in the chosen scope. An entry that already has a current critique of the same critic, version,\n            model, parameters and image is skipped without a call; a changed image is judged again. Failures are recorded\n            with a reason code and retried by the next run. Critiques that cannot be sent (the server went away) are kept in\n            the spool directory (default .atlas/critic-spool) and sent, byte for byte, by the next invocation.\n            --dry-run        send and publish nothing: say how many entries would be judged or skipped, and where images would go\n            --yes            allow sending images to a third party without asking (required without a terminal)\n            --max-images n   stop cleanly after n images; --max-tokens and --max-cost likewise, from what critics report\n            --mode           report (default): scores are shown and the exit code is 0; warn: also list scores below their threshold;\n                             gate: exit 1 when a score is below its state kind's threshold, refused (exit 2) unless a calibration\n                             with held-out evidence of this critic and version is given with --calibration\n\n  watch     A worker: run in a loop over the catalog. It catches up on connecting, judges new or changed entries as they arrive (a picture\n            already judged is skipped without a call), publishes as it goes, keeps what the server cannot take and sends it next time.\n            It stops on Ctrl-C, after --max-batches batches, or when a budget (for the whole watch) is spent, and prints the version to\n            resume from (--since). It reports; it never gates. --dry-run reads the catalog once and sends and publishes nothing.\n  review    The ranked \"needs polish\" list of one critic (lowest first) with its findings and the marks already made. With a terminal it\n            walks through the list and asks: accept (the opinion is right: the UI needs polish), dismiss (it is not), skip. With\n            --entry and --accept or --dismiss it records one decision, for the whole opinion or for --finding ids. A mark is a layer\n            under critic.review.<reviewer> (--reviewer, ATLAS_REVIEWER); it needs no more power than a critic token and never changes\n            a result, a count or the gate.\n  calibrate Judge labelled pictures (--good and --bad directories; a subdirectory's name is the state kind of the pictures in it) and/or\n            read the whole-opinion marks reviewers made (--from-marks), and measure the critic per state kind: sample sizes, precision,\n            recall, the threshold, and whether the figures are in-sample. --heldout-* pictures are never used to choose the threshold.\n            --out writes the calibration that run --calibration reads. Nothing is published.\n\n  --format json   prints one document (schemas: critic-list-result, critic-setup-result, critic-doctor-result, critic-run-result,\n                  critic-watch-result, critic-review-result, critic-calibrate-result).\n                  A command that is wrong is {\"ok\":false,\"error\":{\"message\":\u2026,\"code\":\"usage\"}} on standard error.\n  Exit codes: 0 ok, 1 the answer is no (a critic failed, a gate crossed, a problem found), 2 the command was wrong.\n";
+/**
+ * `atlas critic …`. Returns the exit code: 0 worked, 1 the answer is no. Throws on a mistake in the command (a missing flag, a
+ * gate refused, a configuration that holds a credential), which {@link runCriticCommand} reports and turns into exit code 2.
+ *
+ * @param argv the arguments after `critic`.
+ */
+export declare function criticMain(argv: readonly string[], io: CriticIo): Promise<number>;
+/** `atlas critic …` for a terminal: a mistake in the command is a message on stderr (a JSON document with `--format json`) and exit code 2. */
+export declare function runCriticCommand(argv: readonly string[], io?: Partial<CriticIo>): Promise<number>;

package/dist/critics/src/cli.js

@@ -0,0 +1,490 @@
+/**
+ * @module The `atlas critic` command
+ *
+ * `atlas critic list | setup | doctor | run | watch | review | calibrate`: run automated reviewers of the UI catalog on this machine and publish what they
+ * think as critiques. The server stores and shows; it never runs a model and never holds a provider key. `criticMain` takes its
+ * arguments, environment, working directory, clock and a prompt as parameters and returns the exit code, so it is driven the same
+ * from a terminal, CI and tests; `runCriticCommand` is the terminal wrapper.
+ *
+ * Interactive when there is a terminal: missing choices are asked for, a run that sends images to a third party names the
+ * destination and asks for confirmation. Without a terminal (or with `--non-interactive`, or `--format json`) nothing is ever asked:
+ * every choice is a flag, a missing one exits 2 and names the flags, and sending images to a third party needs `--yes`.
+ *
+ * Exit codes: 0 it worked, 1 it ran and the answer is no (a critic failed on an entry, a gate was crossed, a check of `doctor`
+ * found a problem), 2 the command itself was wrong (a missing flag, a gate refused for lack of evidence).
+ *
+ * @see docs/critics.md#the-atlas-critic-command
+ */
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { httpStore } from '../../reports/src/http-store.js';
+import { ServerInfoSchema } from '../../schema/src/api.js';
+import { createCatalogClient, CatalogClientError } from './catalog-client.js';
+import { calibrateCommand } from './cli-calibrate.js';
+import { connection, defaultAsk, judgeEntries, list, numberFlag, populationOf, positionals, print, redactIn, scopeOf, value, values, wantsJson } from './cli-common.js';
+import { reviewCommand } from './cli-review.js';
+import { watchCommand } from './cli-watch.js';
+import { gateDecision } from './judging.js';
+import { buildCritic, loadCriticsConfig } from './registry.js';
+import { flushSpool, pendingInSpool, spoolingStore } from './spool.js';
+/** The text `atlas critic --help` prints. Also the source of the command reference in `docs/reference/cli.md`. */
+export const CRITIC_USAGE = `atlas critic — automated reviewers of the UI catalog
+
+  atlas critic list    [--config atlas.critics.json] [--format text|json]
+  atlas critic setup   <name> [--config atlas.critics.json] [--format text|json]
+  atlas critic doctor  [--server <url>] [--critic <name>]... [--config atlas.critics.json] [--format text|json]
+  atlas critic run     --critic <name>... ( --all | --unjudged | --since <version> | filters ) [--server <url>]
+                       [--area a,b] [--profile key] [--state screen,overlay] [--route r] [--q words]
+                       [--dry-run] [--yes] [--non-interactive] [--force]
+                       [--max-images n] [--max-tokens n] [--max-cost usd] [--concurrency n] [--timeout-ms n]
+                       [--mode report|warn|gate] [--threshold state=score]... [--calibration file.json]
+                       [--spool dir] [--project slug] [--config atlas.critics.json] [--format text|json]
+  atlas critic watch   --critic <name>... [--server <url>] [--since <version>] [filters] [--yes] [--dry-run]
+                       [--max-batches n] [--max-images n] [--max-tokens n] [--max-cost usd] [--concurrency n] [--timeout-ms n]
+                       [--spool dir] [--project slug] [--config atlas.critics.json] [--format text|json]
+  atlas critic review  [--server <url>] [--critic <name>] [--state kind] [--limit n] [--list] [--non-interactive]
+                       [--entry <key> ( --accept | --dismiss ) [--finding <id>]... [--note text] [--reviewer name] [--dry-run]]
+                       [--project slug] [--config atlas.critics.json] [--format text|json]
+  atlas critic calibrate <name> ( --good <dir>... --bad <dir>... [--heldout-good <dir>...] [--heldout-bad <dir>...] [--state kind]
+                       | --from-marks [--server <url>] [--critic-version v] ) [--out file.json] [--dry-run] [--yes]
+                       [--max-images n] [--concurrency n] [--timeout-ms n] [--config atlas.critics.json] [--format text|json]
+
+  A critic is a producer: it runs here (on a laptop, in CI or in a worker), reads the catalog from an Atlas server, and publishes
+  what it thinks as layers of critiques. The server never runs a model and never holds a provider key. A critique is an opinion,
+  not a test result: it never changes whether a report passes.
+
+  Configuration is atlas.critics.json, which may be committed: it names credentials by reference (env:NAME, file:PATH) and a
+  literal credential in it is refused. The server is --server, "server.url" there, ATLAS_SERVER_URL or an https ATLAS_STORE;
+  the token is ATLAS_CRITIC_TOKEN (or ATLAS_TOKEN), never a flag, and is never sent over plain http beyond this machine.
+
+  list      The critics available and configured: type, whether each is ready or needs setup, and where it sends images.
+  setup     Provision a critic: fetch and verify a model checkpoint against its pinned SHA-256 (a bad checksum is an error), or
+            check that a credential reference resolves. This is the only place anything is downloaded; judging never downloads.
+  doctor    Check the server is reachable, the token is accepted and has the scope a critic needs (and warn when the server is
+            open, so every caller is an administrator), that the server lists the catalog, and for each configured critic its
+            model file, dependencies, credential and provider. Each problem is reported separately, with how to fix it.
+  run       Judge the entries in the chosen scope. An entry that already has a current critique of the same critic, version,
+            model, parameters and image is skipped without a call; a changed image is judged again. Failures are recorded
+            with a reason code and retried by the next run. Critiques that cannot be sent (the server went away) are kept in
+            the spool directory (default .atlas/critic-spool) and sent, byte for byte, by the next invocation.
+            --dry-run        send and publish nothing: say how many entries would be judged or skipped, and where images would go
+            --yes            allow sending images to a third party without asking (required without a terminal)
+            --max-images n   stop cleanly after n images; --max-tokens and --max-cost likewise, from what critics report
+            --mode           report (default): scores are shown and the exit code is 0; warn: also list scores below their threshold;
+                             gate: exit 1 when a score is below its state kind's threshold, refused (exit 2) unless a calibration
+                             with held-out evidence of this critic and version is given with --calibration
+
+  watch     A worker: run in a loop over the catalog. It catches up on connecting, judges new or changed entries as they arrive (a picture
+            already judged is skipped without a call), publishes as it goes, keeps what the server cannot take and sends it next time.
+            It stops on Ctrl-C, after --max-batches batches, or when a budget (for the whole watch) is spent, and prints the version to
+            resume from (--since). It reports; it never gates. --dry-run reads the catalog once and sends and publishes nothing.
+  review    The ranked "needs polish" list of one critic (lowest first) with its findings and the marks already made. With a terminal it
+            walks through the list and asks: accept (the opinion is right: the UI needs polish), dismiss (it is not), skip. With
+            --entry and --accept or --dismiss it records one decision, for the whole opinion or for --finding ids. A mark is a layer
+            under critic.review.<reviewer> (--reviewer, ATLAS_REVIEWER); it needs no more power than a critic token and never changes
+            a result, a count or the gate.
+  calibrate Judge labelled pictures (--good and --bad directories; a subdirectory's name is the state kind of the pictures in it) and/or
+            read the whole-opinion marks reviewers made (--from-marks), and measure the critic per state kind: sample sizes, precision,
+            recall, the threshold, and whether the figures are in-sample. --heldout-* pictures are never used to choose the threshold.
+            --out writes the calibration that run --calibration reads. Nothing is published.
+
+  --format json   prints one document (schemas: critic-list-result, critic-setup-result, critic-doctor-result, critic-run-result,
+                  critic-watch-result, critic-review-result, critic-calibrate-result).
+                  A command that is wrong is {"ok":false,"error":{"message":…,"code":"usage"}} on standard error.
+  Exit codes: 0 ok, 1 the answer is no (a critic failed, a gate crossed, a problem found), 2 the command was wrong.
+`;
+function parseThresholds(argv) {
+    const out = {};
+    for (const pair of values(argv, 'threshold')) {
+        const at = pair.indexOf('=');
+        const score = Number(pair.slice(at + 1));
+        if (at < 1 || !Number.isFinite(score))
+            throw new Error(`--threshold takes state=score, such as screen=0.6 (got "${pair}")`);
+        out[pair.slice(0, at)] = score;
+    }
+    return out;
+}
+/** `atlas critic list`. */
+async function list_(argv, io, loaded) {
+    const rows = [];
+    for (const name of Object.keys(loaded.config.critics).sort()) {
+        const entry = loaded.config.critics[name];
+        try {
+            const built = await buildCritic(loaded, name, io);
+            const description = built.critic.describe();
+            const problems = (await built.inspect()).filter(item => item.status === 'problem');
+            rows.push({
+                name, type: entry.type, kind: built.critic.kind ?? null, version: built.critic.version,
+                status: problems.length ? (problems.some(item => item.aspect === 'model-file') ? 'needs-setup' : 'misconfigured') : 'ready',
+                sendsImagesTo: description.sendsImagesTo, model: description.model?.id ?? null,
+                ...(problems.length ? { detail: redactIn(problems.map(item => `${item.detail}${item.fix ? ` (${item.fix})` : ''}`).join('; '), built.secrets) } : {})
+            });
+        }
+        catch (error) {
+            rows.push({ name, type: entry.type, kind: null, version: null, status: 'misconfigured', sendsImagesTo: null, model: null, detail: error instanceof Error ? error.message.split('\n')[0] : String(error) });
+        }
+    }
+    for (const type of ['uiclip', 'llm']) {
+        if (!Object.values(loaded.config.critics).some(entry => entry.type === type)) {
+            rows.push({ name: type, type: 'not-configured', kind: null, version: null, status: 'not-configured', sendsImagesTo: null, model: null, detail: `Built in. Add a "${type}" entry under "critics" in atlas.critics.json to use it` });
+        }
+    }
+    if (wantsJson(argv)) {
+        print(io, { ok: true, config: loaded.file ? path.relative(io.cwd ?? process.cwd(), loaded.file) || path.basename(loaded.file) : null, critics: rows });
+        return 0;
+    }
+    io.out(`${loaded.file ? `Critics in ${path.basename(loaded.file)}` : 'No atlas.critics.json here'}\n`);
+    for (const row of rows)
+        io.out(`  ${row.status === 'ready' ? '✓' : row.status === 'not-configured' ? '·' : '✗'} ${row.name.padEnd(14)} ${String(row.type).padEnd(14)} ${row.status.padEnd(14)} ${row.sendsImagesTo ? `images go to ${row.sendsImagesTo}` : row.status === 'not-configured' ? '' : 'images stay on this machine'}${row.detail ? `\n      → ${row.detail}` : ''}\n`);
+    return 0;
+}
+/** `atlas critic setup <name>`. */
+async function setup_(argv, io, loaded) {
+    const [name] = positionals(argv);
+    if (!name)
+        throw new Error(`setup needs the name of a critic: ${Object.keys(loaded.config.critics).join(', ') || 'none are configured in atlas.critics.json'}`);
+    const built = await buildCritic(loaded, name, io);
+    const type = loaded.config.critics[name].type;
+    try {
+        const done = await built.setup();
+        if (wantsJson(argv))
+            print(io, { ok: true, critic: name, type, details: done.details });
+        else
+            for (const line of done.details)
+                io.out(`✓ ${line}\n`);
+        return 0;
+    }
+    catch (error) {
+        const message = redactIn(error instanceof Error ? error.message : String(error), built.secrets);
+        if (wantsJson(argv))
+            print(io, { ok: false, critic: name, type, details: [], error: message });
+        else
+            io.out(`✗ ${message}\n`);
+        return 1;
+    }
+}
+/** `atlas critic doctor`. */
+async function doctor_(argv, io, loaded) {
+    const env = io.env ?? process.env;
+    const checks = [];
+    const add = (check) => checks.push(check);
+    let role = null, open = null, serverLabel = null;
+    const wanted = list(argv, 'critic');
+    const names = wanted.length ? wanted : Object.keys(loaded.config.critics);
+    let target = null;
+    try {
+        target = await connection(argv, loaded, env);
+    }
+    catch (error) {
+        add({ name: 'server', status: 'problem', detail: error instanceof Error ? error.message : String(error), fix: 'Use an https address, or put the token in the environment for a server on this machine' });
+    }
+    if (!target && !checks.length)
+        add({ name: 'server', status: 'problem', detail: 'No server is configured', fix: 'Pass --server <url>, set "server.url" in atlas.critics.json, or export ATLAS_SERVER_URL' });
+    if (target) {
+        serverLabel = target.url;
+        const doFetch = io.fetch ?? fetch;
+        const headers = target.token ? { authorization: `Bearer ${target.token}` } : {};
+        let response = null;
+        try {
+            response = await doFetch(`${target.url}/api/server`, { headers, signal: AbortSignal.timeout(10_000) });
+        }
+        catch (error) {
+            add({ name: 'server', status: 'problem', detail: `${target.url} cannot be reached (${error instanceof Error ? error.message : 'error'})`, fix: 'Check the address, that the server is running, and the network' });
+        }
+        let authorized = false;
+        if (response) {
+            if (response.status === 401 || response.status === 403) {
+                add({ name: 'server', status: 'ok', detail: `${target.url} is reachable` });
+                add({ name: 'token', status: 'problem', detail: target.token ? 'The server did not accept this token' : 'The server wants a token and none was given', fix: 'Set ATLAS_CRITIC_TOKEN to a token with role critic (the server operator creates one with ATLAS_CRITIC_TOKEN or auth.tokens)' });
+                await response.body?.cancel();
+            }
+            else if (!response.ok) {
+                add({ name: 'server', status: 'problem', detail: `${target.url} answered ${response.status}`, fix: 'Check that this is an Atlas server, and that it is running' });
+                await response.body?.cancel();
+            }
+            else {
+                const info = ServerInfoSchema.safeParse(await response.json().catch(() => null));
+                if (!info.success)
+                    add({ name: 'server', status: 'problem', detail: `${target.url} did not answer like an Atlas server`, fix: 'Check the address' });
+                else {
+                    authorized = true;
+                    role = info.data.role;
+                    open = info.data.open;
+                    add({ name: 'server', status: 'ok', detail: `${target.url} is reachable${info.data.version ? ` (Atlas ${info.data.version})` : ''}` });
+                    if (open)
+                        add({ name: 'scope', status: 'warning', detail: 'The server has no tokens, so every caller is an administrator, critics included', fix: 'Give the server tokens (ATLAS_CRITIC_TOKEN for critics, ATLAS_PUBLISH_TOKEN for CI) so a critic can publish critiques and nothing else' });
+                    else if (info.data.role === 'critic')
+                        add({ name: 'scope', status: 'ok', detail: 'The token may read the catalog and publish critiques, and nothing else' });
+                    else if (info.data.role === 'read')
+                        add({ name: 'scope', status: 'problem', detail: 'The token may read but not publish critiques', fix: 'Use a token with role critic' });
+                    else
+                        add({ name: 'scope', status: 'warning', detail: `The token has role ${info.data.role}, which can do more than a critic needs (it could publish test results or change settings)`, fix: 'Use a token with role critic' });
+                    if (!info.data.capabilities.catalogEntries)
+                        add({ name: 'catalog', status: 'problem', detail: 'This server cannot list the catalog for critics', fix: 'Upgrade the server to a release with GET /api/reports/ui-catalog/entries' });
+                }
+            }
+        }
+        if (authorized && checks.every(check => check.name !== 'catalog')) {
+            try {
+                const page = await createCatalogClient({ url: target.url, ...(target.token ? { token: target.token } : {}), ...(target.project ? { project: target.project } : {}), ...(io.fetch ? { fetch: io.fetch } : {}), retries: 0 }).page({ limit: 1 });
+                add({ name: 'catalog', status: 'ok', detail: `${page.total} captured ${page.total === 1 ? 'entry' : 'entries'} can be judged${page.unjudgeable ? `; ${page.unjudgeable} captured without a stored picture cannot` : ''}` });
+            }
+            catch (error) {
+                add({ name: 'catalog', status: 'problem', detail: error instanceof CatalogClientError ? error.message : 'The catalog could not be read', fix: 'Check the token and the server version' });
+            }
+        }
+    }
+    const spool = path.resolve(loaded.directory, value(argv, 'spool') ?? loaded.config.spool ?? '.atlas/critic-spool');
+    const waiting = await pendingInSpool(spool);
+    add(waiting ? { name: 'spool', status: 'warning', detail: `${waiting} object${waiting === 1 ? '' : 's'} of critiques that could not be sent wait in ${path.relative(io.cwd ?? process.cwd(), spool) || spool}`, fix: 'The next `atlas critic run` sends them' } : { name: 'spool', status: 'ok', detail: 'Nothing is waiting to be sent' });
+    for (const name of names) {
+        let built;
+        try {
+            built = await buildCritic(loaded, name, io);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message.split('\n')[0] : String(error);
+            add({ name: `${name}:${/credential/i.test(message) ? 'credential' : 'configuration'}`, status: 'problem', detail: message, fix: /not set/.test(message) ? `Export the variable that ${name} names` : 'Fix atlas.critics.json' });
+            continue;
+        }
+        let found;
+        try {
+            found = await built.inspect();
+        }
+        catch (error) {
+            found = [{ aspect: 'inspect', status: 'problem', detail: error instanceof Error ? error.message : String(error) }];
+        }
+        for (const item of found)
+            add({ name: `${name}:${item.aspect}`, status: item.status, detail: redactIn(item.detail, built.secrets), ...(item.fix ? { fix: item.fix } : {}) });
+    }
+    if (wanted.length)
+        for (const name of wanted)
+            if (!loaded.config.critics[name])
+                add({ name: `${name}:configuration`, status: 'problem', detail: `No critic named "${name}" is configured`, fix: 'Add it under "critics" in atlas.critics.json' });
+    const ok = checks.every(check => check.status !== 'problem');
+    if (wantsJson(argv)) {
+        print(io, { ok, server: serverLabel, role, open, checks });
+        return ok ? 0 : 1;
+    }
+    for (const check of checks) {
+        io.out(`  ${check.status === 'ok' ? '✓' : check.status === 'warning' ? '!' : '✗'} ${check.name.padEnd(22)} ${check.detail}\n`);
+        if (check.status !== 'ok' && check.fix)
+            io.out(`      → ${check.fix}\n`);
+    }
+    io.out(ok ? 'No problems found.\n' : 'Something needs attention (see →).\n');
+    return ok ? 0 : 1;
+}
+/** `atlas critic run`. */
+async function run_(argv, io, loaded) {
+    const env = io.env ?? process.env;
+    const json = wantsJson(argv);
+    const interactive = !argv.includes('--non-interactive') && !json && (io.interactive ?? Boolean(process.stdin.isTTY && process.stdout.isTTY));
+    const ask = io.ask ?? defaultAsk;
+    const dryRun = argv.includes('--dry-run');
+    const clock = io.clock ?? Date.now;
+    let names = list(argv, 'critic');
+    let { query, chosen } = scopeOf(argv);
+    let target = await connection(argv, loaded, env);
+    const hasScope = () => Object.keys(chosen).length > 0;
+    const missing = () => [...(names.length ? [] : ['--critic <name>']), ...(target ? [] : ['--server <url>']), ...(hasScope() ? [] : ['a scope: --all, --unjudged, --since <version>, or a filter such as --area or --state'])];
+    if (missing().length && interactive) {
+        if (!names.length) {
+            const configured = Object.keys(loaded.config.critics);
+            names = (await ask(`Which critic${configured.length ? ` (${configured.join(', ')})` : ''}? `)).split(/[,\s]+/).filter(Boolean);
+        }
+        if (!target) {
+            const url = (await ask('Atlas server address? ')).trim();
+            if (url)
+                target = { url: url.replace(/\/+$/, ''), ...(env.ATLAS_CRITIC_TOKEN || env.ATLAS_TOKEN ? { token: (env.ATLAS_CRITIC_TOKEN || env.ATLAS_TOKEN) } : {}) };
+        }
+        if (!hasScope()) {
+            const answer = (await ask('Scope: judge (a)ll entries, or only those (u)njudged by this critic? ')).trim().toLowerCase();
+            if (answer.startsWith('a'))
+                chosen = { ...chosen, all: 'true' };
+            else if (answer.startsWith('u'))
+                chosen = { ...chosen, unjudged: 'true' };
+        }
+    }
+    if (missing().length)
+        throw new Error(`run needs ${missing().join(', ')}. Nothing was asked: ${interactive ? 'an answer was empty' : 'there is no terminal (or --non-interactive / --format json was given)'}`);
+    const server = target;
+    const mode = (value(argv, 'mode') ?? 'report');
+    if (!['report', 'warn', 'gate'].includes(mode))
+        throw new Error(`--mode takes report|warn|gate (got "${mode}")`);
+    const thresholds = parseThresholds(argv);
+    let calibration;
+    if (value(argv, 'calibration')) {
+        try {
+            calibration = JSON.parse(await readFile(path.resolve(io.cwd ?? process.cwd(), value(argv, 'calibration')), 'utf8'));
+        }
+        catch {
+            throw new Error(`--calibration ${value(argv, 'calibration')} cannot be read as a calibration document`);
+        }
+    }
+    const budget = { ...(numberFlag(argv, 'max-images', true) ? { maxImages: numberFlag(argv, 'max-images', true) } : {}), ...(numberFlag(argv, 'max-tokens', true) ? { maxTokens: numberFlag(argv, 'max-tokens', true) } : {}), ...(numberFlag(argv, 'max-cost') ? { maxCostUsd: numberFlag(argv, 'max-cost') } : {}) };
+    const concurrency = numberFlag(argv, 'concurrency', true), timeoutMs = numberFlag(argv, 'timeout-ms', true);
+    const builts = [];
+    for (const name of names)
+        builts.push(await buildCritic(loaded, name, io));
+    const secrets = builts.flatMap(built => built.secrets);
+    const client = createCatalogClient({ url: server.url, ...(server.token ? { token: server.token } : {}), ...(server.project ? { project: server.project } : {}), ...(io.fetch ? { fetch: io.fetch } : {}), ...(io.sleep ? { sleep: io.sleep } : {}) });
+    const spoolDir = path.resolve(loaded.directory, value(argv, 'spool') ?? loaded.config.spool ?? '.atlas/critic-spool');
+    const base = httpStore({ url: server.url, ...(server.token ? { token: server.token } : {}), ...(server.project ? { project: server.project } : {}), ...(io.fetch ? { fetch: io.fetch } : {}), ...(io.sleep ? { sleep: io.sleep } : {}) });
+    const warnings = [];
+    // What an earlier run could not send goes first, byte for byte; a dry run publishes nothing, so it sends nothing either.
+    let flushed = 0;
+    if (!dryRun) {
+        const sent = await flushSpool(base, spoolDir);
+        flushed = sent.sent;
+        if (sent.error)
+            warnings.push(`${sent.remaining} spooled object${sent.remaining === 1 ? '' : 's'} could not be sent yet: ${redactIn(sent.error, secrets)}`);
+    }
+    const store = spoolingStore(base, spoolDir);
+    const scopeQuery = { ...query };
+    const results = [];
+    for (const built of builts) {
+        const name = built.critic.name;
+        const wholeQuery = chosen.unjudged ? { ...scopeQuery, unjudgedBy: { name, version: built.critic.version } } : scopeQuery;
+        const read = await client.read(wholeQuery);
+        if (!read.sinceHonored)
+            warnings.push(`The server does not remember version ${chosen.since}: the whole catalog was read`);
+        for (const notice of read.notices)
+            warnings.push(notice);
+        // `gate` is refused before anything is spent when the evidence is not there.
+        if (mode === 'gate') {
+            const early = gateDecision({ mode, critic: built.critic, scores: read.entries.map(entry => ({ entryKey: entry.key, stateKind: entry.state, score: null })), ...(calibration ? { calibration } : {}), thresholds });
+            if (early.exitCode === 2)
+                throw new Error(`gate was refused for ${name}@${built.critic.version}:\n${early.refusal.map(line => `  ${line}`).join('\n')}`);
+        }
+        const description = built.critic.describe();
+        if (!dryRun) {
+            await built.ready();
+            if (description.sendsImagesTo !== null) {
+                const planned = read.entries.length;
+                if (!argv.includes('--yes')) {
+                    if (!interactive)
+                        throw new Error(`${name} sends images to ${description.sendsImagesTo}: pass --yes to allow it (nothing was sent)`);
+                    const answer = await ask(`${name} will send up to ${planned} image${planned === 1 ? '' : 's'} to ${description.sendsImagesTo}. Continue? [y/N] `);
+                    if (!/^y(es)?$/i.test(answer.trim()))
+                        throw new Error(`Stopped: ${name} would send images to ${description.sendsImagesTo} and that was not confirmed (nothing was sent)`);
+                }
+            }
+        }
+        // A raw score is read against the project's scores of the same state kind, and against the entry's previous image.
+        const whole = Object.keys(query).length === 0 && !chosen.unjudged ? read : await client.read({});
+        const population = populationOf(whole.entries, name);
+        const { result, outcomes, layer } = await judgeEntries({
+            built, entries: read.entries, client, store, population, dryRun, ...(argv.includes('--force') ? { force: true } : {}),
+            budget, secrets, clock, ...(concurrency ? { concurrency } : {}), ...(timeoutMs ? { timeoutMs } : {}),
+            ...(json || dryRun ? {} : { onProgress: (event) => io.out(`  ${event.result === 'judged' ? '✓' : event.result === 'skipped' ? '·' : '✗'} ${event.entryKey} (${event.judged} judged, ${event.skipped} skipped, ${event.failed} failed, ${event.waiting} waiting)\n`) })
+        });
+        const current = new Map();
+        for (const entry of read.entries) {
+            const found = entry.critiques.find(c => c.currency === 'current' && c.critique.critic.name === name && c.critique.verdict?.score !== undefined);
+            current.set(entry.key, found?.critique.verdict?.score ?? null);
+        }
+        for (const outcome of outcomes)
+            current.set(outcome.entry.key, outcome.critique.verdict?.score ?? null);
+        const decision = gateDecision({ mode, critic: built.critic, scores: read.entries.map(entry => ({ entryKey: entry.key, stateKind: entry.state, score: current.get(entry.key) ?? null })), ...(calibration ? { calibration } : {}), thresholds });
+        results.push({ built, result, entries: read.entries.length, decision, layer, critiques: outcomes.length, failures: outcomes.filter(o => o.critique.failure).map(o => ({ entry: o.entry.key, reason: o.critique.failure.reason })) });
+    }
+    const pending = await pendingInSpool(spoolDir);
+    if (store.spooled() > 0)
+        warnings.push(`The server could not be reached: ${store.spooled()} object${store.spooled() === 1 ? '' : 's'} of critiques were kept in ${path.relative(io.cwd ?? process.cwd(), spoolDir) || spoolDir} and will be sent by the next invocation`);
+    const failed = results.some(item => item.result.failed > 0);
+    const crossed = results.some(item => item.decision?.exitCode === 1);
+    const ok = !failed && !crossed;
+    const document = {
+        ok, dryRun, server: server.url, scope: chosen,
+        critics: results.map(({ built, result, entries, decision, layer, critiques, failures }) => ({
+            critic: built.critic.name, version: built.critic.version, sendsImagesTo: result.sendsImagesTo, entries,
+            judged: result.judged, skipped: result.skipped, failed: result.failed, waiting: result.waiting, ...(result.stoppedBy ? { stoppedBy: result.stoppedBy } : {}),
+            failures, spent: result.spent, published: { layer, critiques },
+            gate: { mode, exitCode: decision?.exitCode ?? 0, below: decision?.below.length ?? 0, warnings: decision?.warnings ?? [], refusal: decision?.refusal ?? [] }
+        })),
+        spool: { flushed, spooled: store.spooled(), pending },
+        warnings: warnings.map(warning => redactIn(warning, secrets))
+    };
+    if (json) {
+        print(io, document);
+        return ok ? 0 : 1;
+    }
+    for (const item of document.critics) {
+        io.out(`${dryRun ? 'Would judge' : 'Judged'} with ${item.critic}@${item.version}: ${item.entries} entries in scope, ${dryRun ? `${item.waiting} to judge` : `${item.judged} judged`}, ${item.skipped} skipped (already judged), ${item.failed} failed${item.waiting && !dryRun ? `, ${item.waiting} left` : ''}\n`);
+        io.out(item.sendsImagesTo ? `  Images ${dryRun ? 'would go' : 'went'} to ${item.sendsImagesTo}\n` : '  Images stay on this machine\n');
+        for (const failure of item.failures)
+            io.out(`  ✗ ${failure.entry}: ${failure.reason}\n`);
+        if (item.stoppedBy === 'budget')
+            io.out('  The budget was reached: the run stopped and published what it had finished.\n');
+        for (const line of item.gate.warnings)
+            io.out(`  ! ${line}\n`);
+        if (item.published.layer)
+            io.out(`  Published ${item.published.critiques} critique${item.published.critiques === 1 ? '' : 's'} as layer ${item.published.layer.slice(0, 12)}\n`);
+        if (mode === 'gate')
+            io.out(item.gate.exitCode === 1 ? `  ✗ gate: ${item.gate.below} score${item.gate.below === 1 ? '' : 's'} below threshold\n` : '  ✓ gate: no score below threshold\n');
+    }
+    if (flushed)
+        io.out(`Sent ${flushed} object${flushed === 1 ? '' : 's'} left by an earlier run\n`);
+    for (const warning of document.warnings)
+        io.out(`! ${warning}\n`);
+    return ok ? 0 : 1;
+}
+/**
+ * `atlas critic …`. Returns the exit code: 0 worked, 1 the answer is no. Throws on a mistake in the command (a missing flag, a
+ * gate refused, a configuration that holds a credential), which {@link runCriticCommand} reports and turns into exit code 2.
+ *
+ * @param argv the arguments after `critic`.
+ */
+export async function criticMain(argv, io) {
+    const [command, ...rest] = argv;
+    if (!command || command === '--help' || command === '-h' || rest.includes('--help')) {
+        io.out(CRITIC_USAGE);
+        return 0;
+    }
+    if (!['list', 'setup', 'doctor', 'run', 'watch', 'review', 'calibrate'].includes(command)) {
+        io.out(CRITIC_USAGE);
+        return 2;
+    }
+    const format = value(rest, 'format');
+    if (format !== undefined && !['text', 'json'].includes(format))
+        throw new Error(`--format takes text|json (got "${format}")`);
+    const loaded = await loadCriticsConfig(io.cwd ?? process.cwd(), value(rest, 'config'));
+    if (command === 'list')
+        return list_(rest, io, loaded);
+    if (command === 'setup')
+        return setup_(rest, io, loaded);
+    if (command === 'doctor')
+        return doctor_(rest, io, loaded);
+    if (command === 'watch')
+        return watchCommand(rest, io, loaded);
+    if (command === 'review')
+        return reviewCommand(rest, io, loaded);
+    if (command === 'calibrate')
+        return calibrateCommand(rest, io, loaded);
+    return run_(rest, io, loaded);
+}
+/** `atlas critic …` for a terminal: a mistake in the command is a message on stderr (a JSON document with `--format json`) and exit code 2. */
+export async function runCriticCommand(argv, io = {}) {
+    // An interrupt ends a long-running `watch` cleanly: what it finished is published and its summary printed. Other commands keep the default.
+    const controller = new AbortController();
+    const interrupt = () => controller.abort();
+    const watching = argv[0] === 'watch' && !io.signal;
+    if (watching) {
+        process.once('SIGINT', interrupt);
+        process.once('SIGTERM', interrupt);
+    }
+    try {
+        return await criticMain(argv, { out: text => process.stdout.write(text), ...(watching ? { signal: controller.signal } : {}), ...io });
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        process.stderr.write(wantsJson(argv) ? `${JSON.stringify({ ok: false, error: { message, code: 'usage' } })}\n` : `atlas critic: ${message}\n`);
+        return 2;
+    }
+    finally {
+        if (watching) {
+            process.off('SIGINT', interrupt);
+            process.off('SIGTERM', interrupt);
+        }
+    }
+}

package/dist/critics/src/index.d.ts

@@ -11,11 +11,14 @@
  *
  * @see docs/critics.md
  */
+export * from './catalog-client.js';
 export * from './contract.js';
 export * from './credentials.js';
 export * from './critique.js';
 export * from './judging.js';
 export * from './publish.js';
+export * from './registry.js';
 export * from './runner.js';
 export * from './scope.js';
+export * from './spool.js';
 export { CRITIC_PRODUCER_PREFIX, CRITIQUE_EXTENSION, CRITIQUE_PART_KIND, CRITIQUE_SCHEMA_VERSION, CritiqueSchema, type Critique, type CritiqueFinding } from '../../schema/src/critique.js';

package/dist/critics/src/index.js

@@ -11,11 +11,14 @@
  *
  * @see docs/critics.md
  */
+export * from './catalog-client.js';
 export * from './contract.js';
 export * from './credentials.js';
 export * from './critique.js';
 export * from './judging.js';
 export * from './publish.js';
+export * from './registry.js';
 export * from './runner.js';
 export * from './scope.js';
+export * from './spool.js';
 export { CRITIC_PRODUCER_PREFIX, CRITIQUE_EXTENSION, CRITIQUE_PART_KIND, CRITIQUE_SCHEMA_VERSION, CritiqueSchema } from '../../schema/src/critique.js';

package/dist/critics/src/judging.d.ts

@@ -69,11 +69,19 @@ export type KindCalibration = {
 /** The result of `calibrate`: per state kind, with the critic's provenance. */
 export type Calibration = {
     readonly schema: 1;
+    /** The critic and its version, and (when `atlas critic calibrate` wrote it) the model and fixed parameters the scores came from. */
     readonly critic: {
         readonly name: string;
         readonly version: string;
+        readonly model?: {
+            readonly id: string;
+            readonly provider?: string;
+        };
+        readonly parameters?: Readonly<Record<string, unknown>>;
     };
     readonly kinds: readonly KindCalibration[];
+    /** When it was measured, epoch milliseconds, when `atlas critic calibrate` wrote it. */
+    readonly createdAtMs?: number;
 };
 /**
  * Measures a critic against labelled examples, per state kind, and proposes a threshold for each.