@crvouga/atlas 1.20.0 → 1.22.0

package/dist/critics/src/cli-common.d.ts

@@ -0,0 +1,108 @@
+/**
+ * @module Shared parts of the `atlas critic` commands
+ *
+ * What every `atlas critic` subcommand reads the same way: flags, the server and token (a token is never sent over plain http beyond
+ * this machine), the scope that narrows the catalog, the way output is printed, and `judgeEntries`, the one place where entries are
+ * judged, read against their state kind and published, so `run` and `watch` cannot disagree about it.
+ *
+ * @see docs/critics.md#the-atlas-critic-command
+ */
+import type { ObjectStore } from '../../reports/src/object-store.js';
+import type { CatalogEntry, CatalogEntriesQuery } from '../../reports/src/ui-catalog-entries.js';
+import type { CatalogClient } from './catalog-client.js';
+import type { Budget, ProgressEvent, RunResult } from './runner.js';
+import type { JudgeableEntry } from './contract.js';
+import type { CritiqueOutcome } from './critique.js';
+import type { BuiltCritic, LoadedConfig } from './registry.js';
+/** The environment a command reads: a map of names to values. */
+export type Env = Readonly<Record<string, string | undefined>>;
+/** What a command is given besides arguments: where output goes, and, for tests and embedding, the environment, working directory, clock, `fetch`, module loading, a prompt and a signal that ends a long-running command. */
+export type CriticIo = {
+    readonly out: (text: string) => void;
+    readonly env?: Env;
+    readonly cwd?: string;
+    readonly clock?: () => number;
+    readonly fetch?: typeof fetch;
+    readonly importModule?: (name: string) => Promise<unknown>;
+    /** Waits between retries of a request to the server. Default `setTimeout`. Tests pass a fake. */
+    readonly sleep?: (ms: number) => Promise<void>;
+    /** Whether a person is at a terminal. Default: standard input and output are terminals. Nothing is asked when this is false. */
+    readonly interactive?: boolean;
+    /** Asks a question and returns the answer. Default: a line from standard input. */
+    readonly ask?: (question: string) => Promise<string>;
+    /** Ends `atlas critic watch` cleanly (what it finished is kept and published). The terminal wrapper aborts it on interrupt. */
+    readonly signal?: AbortSignal;
+};
+/** The flags that take a value, so a value is not mistaken for a positional argument. */
+export declare const VALUE_FLAGS: string[];
+/** Every value given for `--name`, in order. */
+export declare const values: (argv: readonly string[], name: string) => string[];
+/** The first value given for `--name`. */
+export declare const value: (argv: readonly string[], name: string) => string | undefined;
+/** Every value of `--name`, comma-separated values split and trimmed. */
+export declare const list: (argv: readonly string[], name: string) => string[];
+/** Whether `--format json` (or `--json`) was given. */
+export declare const wantsJson: (argv: readonly string[]) => boolean;
+/** The arguments that are not flags or their values. */
+export declare function positionals(argv: readonly string[]): string[];
+/** A positive number from `--name` (a whole number when `whole`), or undefined when it was not given. Throws for anything else. */
+export declare function numberFlag(argv: readonly string[], name: string, whole?: boolean): number | undefined;
+/** Where the server is and how to reach it. */
+export type Connection = {
+    url: string;
+    token?: string;
+    project?: string;
+};
+/** Where the server is and how to reach it, from flags, configuration and the environment; `null` when no server is named. Throws for a token that would travel over plain http beyond this machine. */
+export declare function connection(argv: readonly string[], loaded: LoadedConfig, env: Env): Promise<Connection | null>;
+/** `text` with every secret of six characters or more replaced by `[redacted]`. */
+export declare const redactIn: (text: string, secrets: readonly string[]) => string;
+/** Asks a question on the terminal and returns the line typed. */
+export declare function defaultAsk(question: string): Promise<string>;
+/** Prints one JSON document, indented, with a final newline. */
+export declare const print: (io: CriticIo, document: unknown) => void;
+/** The catalog query and the readable summary of the choices that narrowed it (`--area`, `--profile`, `--state`, `--route`, `--q`, `--since`, `--all`, `--unjudged`). */
+export declare function scopeOf(argv: readonly string[]): {
+    query: CatalogEntriesQuery;
+    chosen: Record<string, string>;
+};
+/** A catalog entry as a critic is told about it. */
+export declare const toJudgeable: (entry: CatalogEntry) => JudgeableEntry;
+/** The scores a critic's current critiques already give, by entry key with the entry's state kind: the population a new score is read against. */
+export declare function populationOf(entries: readonly CatalogEntry[], critic: string): Map<string, {
+    stateKind: string;
+    score: number;
+}>;
+/** What judging a set of entries did: the run, the outcomes read against their state kind, and the layer they were published as. */
+export type Judged = {
+    readonly result: RunResult;
+    readonly outcomes: readonly CritiqueOutcome[];
+    readonly layer: string | null;
+};
+/**
+ * Judges `entries` with one critic and publishes the outcomes as one layer. A raw score is read against the project's scores of the
+ * same state kind (`population`, which is updated with the new scores) and against the entry's previous image. Nothing is published
+ * on a dry run or when there is nothing to publish.
+ *
+ * @param args.population the critic's known scores by entry key (see {@link populationOf}); updated.
+ * @param args.store where critiques are published (usually a spooling store).
+ */
+export declare function judgeEntries(args: {
+    readonly built: BuiltCritic;
+    readonly entries: readonly CatalogEntry[];
+    readonly client: Pick<CatalogClient, 'image'>;
+    readonly store: ObjectStore;
+    readonly population: Map<string, {
+        stateKind: string;
+        score: number;
+    }>;
+    readonly dryRun: boolean;
+    readonly force?: boolean;
+    readonly budget?: Budget;
+    readonly secrets: readonly string[];
+    readonly concurrency?: number;
+    readonly timeoutMs?: number;
+    readonly signal?: AbortSignal;
+    readonly clock: () => number;
+    readonly onProgress?: (event: ProgressEvent) => void;
+}): Promise<Judged>;

package/dist/critics/src/cli-common.js

@@ -0,0 +1,155 @@
+import { resolveCredential } from './credentials.js';
+import { relativeVerdict } from './judging.js';
+import { publishCritiques } from './publish.js';
+import { runCritic } from './runner.js';
+/** The flags that take a value, so a value is not mistaken for a positional argument. */
+export const VALUE_FLAGS = [
+    'config', 'server', 'project', 'critic', 'area', 'profile', 'state', 'route', 'q', 'since', 'format', 'max-images', 'max-tokens', 'max-cost', 'concurrency', 'timeout-ms', 'mode', 'threshold', 'calibration', 'spool',
+    'max-batches', 'entry', 'finding', 'note', 'reviewer', 'limit', 'good', 'bad', 'heldout-good', 'heldout-bad', 'out', 'critic-version'
+];
+/** Every value given for `--name`, in order. */
+export const values = (argv, name) => argv.flatMap((token, i) => (token === `--${name}` && argv[i + 1] !== undefined ? [argv[i + 1]] : []));
+/** The first value given for `--name`. */
+export const value = (argv, name) => values(argv, name)[0];
+/** Every value of `--name`, comma-separated values split and trimmed. */
+export const list = (argv, name) => values(argv, name).flatMap(item => item.split(',')).map(item => item.trim()).filter(Boolean);
+/** Whether `--format json` (or `--json`) was given. */
+export const wantsJson = (argv) => value(argv, 'format') === 'json' || argv.includes('--json');
+/** The arguments that are not flags or their values. */
+export function positionals(argv) {
+    const out = [];
+    for (let i = 0; i < argv.length; i++) {
+        const token = argv[i];
+        if (token.startsWith('--')) {
+            if (VALUE_FLAGS.includes(token.slice(2)))
+                i++;
+            continue;
+        }
+        out.push(token);
+    }
+    return out;
+}
+/** A positive number from `--name` (a whole number when `whole`), or undefined when it was not given. Throws for anything else. */
+export function numberFlag(argv, name, whole = false) {
+    const text = value(argv, name);
+    if (text === undefined)
+        return undefined;
+    const n = Number(text);
+    if (!Number.isFinite(n) || n <= 0 || (whole && !Number.isInteger(n)))
+        throw new Error(`--${name} takes a positive ${whole ? 'whole ' : ''}number (got "${text}")`);
+    return n;
+}
+const isLoopback = (url) => ['localhost', '127.0.0.1', '[::1]', '::1'].includes(url.hostname);
+/** Where the server is and how to reach it, from flags, configuration and the environment; `null` when no server is named. Throws for a token that would travel over plain http beyond this machine. */
+export async function connection(argv, loaded, env) {
+    const store = env.ATLAS_STORE && /^https?:\/\//.test(env.ATLAS_STORE) ? env.ATLAS_STORE : undefined;
+    const url = value(argv, 'server') ?? loaded.config.server?.url ?? env.ATLAS_SERVER_URL ?? store;
+    if (!url)
+        return null;
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        throw new Error(`The server address "${url}" is not a URL`);
+    }
+    const reference = loaded.config.server?.token;
+    const token = reference ? await resolveCredential(reference, { env }) : env.ATLAS_CRITIC_TOKEN || env.ATLAS_TOKEN || undefined;
+    // A token is a credential: it goes over https, or to this machine.
+    if (token && parsed.protocol !== 'https:' && !isLoopback(parsed))
+        throw new Error(`Refusing to send a token to ${parsed.host} over plain ${parsed.protocol.replace(':', '')}: use an https address (plain http is allowed only for this machine)`);
+    const project = value(argv, 'project') ?? loaded.config.server?.project ?? env.ATLAS_PROJECT;
+    return { url: url.replace(/\/+$/, ''), ...(token ? { token } : {}), ...(project ? { project } : {}) };
+}
+/** `text` with every secret of six characters or more replaced by `[redacted]`. */
+export const redactIn = (text, secrets) => secrets.reduce((acc, secret) => (secret.length >= 6 ? acc.split(secret).join('[redacted]') : acc), text);
+/** Asks a question on the terminal and returns the line typed. */
+export async function defaultAsk(question) {
+    const readline = await import('node:readline/promises');
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    try {
+        return await rl.question(question);
+    }
+    finally {
+        rl.close();
+    }
+}
+/** Prints one JSON document, indented, with a final newline. */
+export const print = (io, document) => io.out(`${JSON.stringify(document, null, 2)}\n`);
+/** The catalog query and the readable summary of the choices that narrowed it (`--area`, `--profile`, `--state`, `--route`, `--q`, `--since`, `--all`, `--unjudged`). */
+export function scopeOf(argv) {
+    const chosen = {};
+    const query = {};
+    for (const name of ['area', 'profile', 'state', 'route']) {
+        const found = list(argv, name);
+        if (found.length) {
+            query[name] = found;
+            chosen[name] = found.join(',');
+        }
+    }
+    const q = value(argv, 'q');
+    if (q) {
+        query.q = q;
+        chosen.q = q;
+    }
+    const since = value(argv, 'since');
+    if (since) {
+        query.since = since;
+        chosen.since = since;
+    }
+    if (argv.includes('--all'))
+        chosen.all = 'true';
+    if (argv.includes('--unjudged'))
+        chosen.unjudged = 'true';
+    return { query: query, chosen };
+}
+/** A catalog entry as a critic is told about it. */
+export const toJudgeable = (entry) => ({
+    key: entry.key, title: entry.title, state: entry.state, imageSha256: entry.image.sha256,
+    ...(entry.route ? { route: entry.route } : {}), profile: { key: entry.profile.key, environment: entry.profile.environment, parameters: entry.profile.parameters }
+});
+/** The scores a critic's current critiques already give, by entry key with the entry's state kind: the population a new score is read against. */
+export function populationOf(entries, critic) {
+    const population = new Map();
+    for (const entry of entries)
+        for (const found of entry.critiques)
+            if (found.currency === 'current' && found.critique.critic.name === critic && found.critique.verdict?.score !== undefined)
+                population.set(entry.key, { stateKind: entry.state, score: found.critique.verdict.score });
+    return population;
+}
+/**
+ * Judges `entries` with one critic and publishes the outcomes as one layer. A raw score is read against the project's scores of the
+ * same state kind (`population`, which is updated with the new scores) and against the entry's previous image. Nothing is published
+ * on a dry run or when there is nothing to publish.
+ *
+ * @param args.population the critic's known scores by entry key (see {@link populationOf}); updated.
+ * @param args.store where critiques are published (usually a spooling store).
+ */
+export async function judgeEntries(args) {
+    const { built } = args;
+    const name = built.critic.name;
+    const existing = args.entries.flatMap(entry => entry.critiques.map(found => found.critique));
+    const result = await runCritic({
+        critic: built.critic, entries: args.entries.map(toJudgeable), existing, dryRun: args.dryRun, ...(args.force ? { force: true } : {}),
+        loadImage: judgeable => args.client.image(args.entries.find(entry => entry.key === judgeable.key)),
+        ...(args.budget ? { budget: args.budget } : {}), secrets: args.secrets,
+        ...(args.concurrency ? { concurrency: args.concurrency } : {}), ...(args.timeoutMs ? { timeoutMs: args.timeoutMs } : {}),
+        ...(args.signal ? { signal: args.signal } : {}), ...(args.onProgress ? { onProgress: args.onProgress } : {})
+    });
+    for (const outcome of result.outcomes)
+        if (outcome.critique.verdict?.score !== undefined)
+            args.population.set(outcome.entry.key, { stateKind: outcome.entry.state, score: outcome.critique.verdict.score });
+    const outcomes = result.outcomes.map(outcome => {
+        const score = outcome.critique.verdict?.score;
+        if (score === undefined || outcome.critique.verdict?.relative)
+            return outcome;
+        const entry = args.entries.find(candidate => candidate.key === outcome.entry.key);
+        const previous = entry?.critiques.find(found => found.currency === 'earlier-image' && found.critique.critic.name === name && found.critique.verdict?.score !== undefined)?.critique.verdict?.score;
+        const relative = relativeVerdict({ score, stateKind: outcome.entry.state, population: [...args.population.values()], ...(previous === undefined ? {} : { previousScore: previous }) });
+        return { ...outcome, critique: { ...outcome.critique, verdict: { ...outcome.critique.verdict, relative } } };
+    });
+    let layer = null;
+    if (!args.dryRun && outcomes.length)
+        layer = (await publishCritiques(args.store, { critic: built.critic, outcomes, recordedAtMs: args.clock() }))?.id ?? null;
+    return { result, outcomes, layer };
+}

package/dist/critics/src/cli-review.d.ts

@@ -0,0 +1,4 @@
+import { type CriticIo } from './cli-common.js';
+import type { LoadedConfig } from './registry.js';
+/** `atlas critic review`. */
+export declare function reviewCommand(argv: readonly string[], io: CriticIo, loaded: LoadedConfig): Promise<number>;

package/dist/critics/src/cli-review.js

@@ -0,0 +1,174 @@
+/**
+ * @module `atlas critic review`
+ *
+ * Reads the ranked "needs polish" list from the server and lets a person mark what critics found: accept an opinion (it is right) or
+ * dismiss it (it is not), as a whole or finding by finding. Without `--entry` it lists the entries one critic judged, lowest first, with
+ * their findings and the marks already made; at a terminal it then walks through them and asks. With `--entry` and `--accept` or
+ * `--dismiss` it records one decision, with no terminal needed. A mark is published as a layer under `critic.review.<reviewer>` (see
+ * {@link ./marks}); it needs no more power than a critic's token has, is never counted, and never changes a status, a slot or the gate.
+ * Nothing is marked when a flag names something that is not there: the command says what is.
+ *
+ * @see docs/critics.md#reviewing-and-marking
+ */
+import { httpStore } from '../../reports/src/http-store.js';
+import { markStanding, rankCritiques } from '../../reports/src/critique-review.js';
+import { createCatalogClient } from './catalog-client.js';
+import { connection, defaultAsk, list, numberFlag, print, scopeOf, value, values, wantsJson } from './cli-common.js';
+import { publishMarks, reviewerName } from './marks.js';
+const DEFAULT_LIMIT = 20;
+const standingJson = (standing) => ({ state: standing.state, accepted: [...standing.accepted], dismissed: [...standing.dismissed], earlier: standing.earlier, notes: [...standing.notes] });
+/** The critics that have a critique of any of the entries, most critiques first. */
+function criticsOf(entries) {
+    const tally = new Map();
+    for (const entry of entries)
+        for (const found of entry.critiques)
+            tally.set(found.critique.critic.name, (tally.get(found.critique.critic.name) ?? 0) + 1);
+    return [...tally].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1)).map(([name]) => name);
+}
+function describeEntry(entry, found, rank) {
+    const { critique } = found;
+    return {
+        rank, key: entry.key, title: entry.title, state: entry.state, imageSha256: entry.image.sha256,
+        score: critique.verdict?.score ?? null, percentile: critique.verdict?.relative?.percentile ?? null, label: critique.verdict?.label ?? null,
+        currency: found.currency,
+        ...(critique.failure ? { failure: critique.failure.reason } : {}),
+        marks: standingJson(markStanding(entry.marks, { critic: critique.critic.name })),
+        findings: (critique.findings ?? []).map(finding => ({
+            id: finding.id ?? null, category: finding.category ?? null, severity: finding.severity ?? null,
+            ...(finding.region ? { region: { unit: finding.region.unit, x: finding.region.x, y: finding.region.y, w: finding.region.w, h: finding.region.h } } : {}),
+            rationale: finding.rationale ?? null, suggestion: finding.suggestion ?? null,
+            marks: finding.id ? standingJson(markStanding(entry.marks, { critic: critique.critic.name, finding: finding.id })) : null
+        }))
+    };
+}
+/** Looks up what a `--entry` flag names and which critique of it is being marked. Throws, saying what there is, when it is not there. */
+function targetOf(entries, key, criticFlag) {
+    const entry = entries.find(candidate => candidate.key === key);
+    if (!entry)
+        throw new Error(`No entry "${key}" in the catalog. List them with \`atlas critic review\` (or \`atlas critic run --dry-run\`)`);
+    const named = [...new Set(entry.critiques.map(found => found.critique.critic.name))];
+    const criticName = criticFlag ?? (named.length === 1 ? named[0] : undefined);
+    if (!criticName)
+        throw new Error(named.length ? `${key} has critiques from ${named.join(', ')}: say whose opinion with --critic <name>` : `${key} has no critique to mark yet: run a critic on it first`);
+    const mine = entry.critiques.filter(found => found.critique.critic.name === criticName);
+    if (!mine.length)
+        throw new Error(`${criticName} has not judged ${key}${named.length ? ` (judged by ${named.join(', ')})` : ''}`);
+    const found = mine.find(candidate => candidate.currency === 'current');
+    if (!found)
+        throw new Error(`${criticName}'s critique of ${key} judged an earlier picture; the picture has changed. Run the critic again (\`atlas critic run --critic ${criticName} --all\`) and mark the new opinion`);
+    if (found.critique.failure)
+        throw new Error(`${criticName} could not judge ${key} (${found.critique.failure.reason}); there is no opinion to mark`);
+    return { entry, found };
+}
+/** `atlas critic review`. */
+export async function reviewCommand(argv, io, loaded) {
+    const env = io.env ?? process.env;
+    const json = wantsJson(argv);
+    const interactive = !argv.includes('--non-interactive') && !argv.includes('--list') && !json && (io.interactive ?? Boolean(process.stdin.isTTY && process.stdout.isTTY));
+    const ask = io.ask ?? defaultAsk;
+    const dryRun = argv.includes('--dry-run');
+    const server = await connection(argv, loaded, env);
+    if (!server)
+        throw new Error('review needs --server <url> (or "server.url" in atlas.critics.json, or ATLAS_SERVER_URL)');
+    const accept = argv.includes('--accept'), dismiss = argv.includes('--dismiss');
+    if (accept && dismiss)
+        throw new Error('--accept and --dismiss are opposites: give one');
+    const entryKey = value(argv, 'entry');
+    if ((accept || dismiss) && !entryKey)
+        throw new Error(`--${accept ? 'accept' : 'dismiss'} needs --entry <key>, the entry whose opinion you are marking (list them with \`atlas critic review\`)`);
+    if (!accept && !dismiss && (values(argv, 'finding').length || value(argv, 'note')))
+        throw new Error('--finding and --note go with --accept or --dismiss');
+    const reviewer = reviewerName(value(argv, 'reviewer') ?? env.ATLAS_REVIEWER ?? 'reviewer');
+    const { query } = scopeOf(argv);
+    const client = createCatalogClient({ url: server.url, ...(server.token ? { token: server.token } : {}), ...(server.project ? { project: server.project } : {}), ...(io.fetch ? { fetch: io.fetch } : {}), ...(io.sleep ? { sleep: io.sleep } : {}) });
+    const read = await client.read(query);
+    const clock = io.clock ?? Date.now;
+    const store = httpStore({ url: server.url, ...(server.token ? { token: server.token } : {}), ...(server.project ? { project: server.project } : {}), ...(io.fetch ? { fetch: io.fetch } : {}), ...(io.sleep ? { sleep: io.sleep } : {}) });
+    const marked = [];
+    let layer = null;
+    const record = async (requests) => {
+        for (const request of requests)
+            marked.push({ entry: request.entry.key, critic: request.critic.name, finding: request.finding ?? null, decision: request.decision });
+        if (!dryRun && requests.length)
+            layer = (await publishMarks(store, { reviewer, marks: requests, recordedAtMs: clock() }))?.id ?? null;
+    };
+    const criticFlag = value(argv, 'critic');
+    let targetCritic;
+    if (criticFlag && !criticsOf(read.entries).includes(criticFlag) && !entryKey)
+        throw new Error(`No entry has a critique from "${criticFlag}". Critics with opinions: ${criticsOf(read.entries).join(', ') || 'none yet'}`);
+    // One decision, by flag: no terminal needed.
+    if (entryKey && (accept || dismiss)) {
+        const { entry, found } = targetOf(read.entries, entryKey, criticFlag);
+        targetCritic = found.critique.critic.name;
+        const decision = accept ? 'accept' : 'dismiss';
+        const ids = list(argv, 'finding');
+        const known = (found.critique.findings ?? []).map(finding => finding.id).filter((id) => Boolean(id));
+        for (const id of ids)
+            if (!known.includes(id))
+                throw new Error(`${found.critique.critic.name}'s critique of ${entryKey} has no finding "${id}". ${known.length ? `Its findings: ${known.join(', ')}` : 'It has no findings with an id: mark the whole opinion by leaving out --finding'}`);
+        const note = value(argv, 'note');
+        const base = { entry: { key: entry.key, title: entry.title, imageSha256: entry.image.sha256 }, critic: { name: found.critique.critic.name, version: found.critique.critic.version }, decision, layer: found.layer, ...(note ? { note } : {}) };
+        await record(ids.length ? ids.map(id => ({ ...base, finding: id })) : [base]);
+    }
+    const critics = criticsOf(read.entries);
+    const critic = criticFlag ?? targetCritic ?? (entryKey ? read.entries.find(entry => entry.key === entryKey)?.critiques[0]?.critique.critic.name : undefined) ?? critics[0] ?? null;
+    const ranking = rankCritiques(read.entries, critic, value(argv, 'state'));
+    const limit = numberFlag(argv, 'limit', true) ?? DEFAULT_LIMIT;
+    const everything = [...ranking.ranked.map((item, index) => describeEntry(item.entry, item.found, index + 1)), ...ranking.unranked.map(item => describeEntry(item.entry, item.found, null))];
+    // One entry when one was named (so its findings and marks can be read before or after marking), else the top of the list.
+    const shown = entryKey ? everything.filter(item => item.key === entryKey) : everything.slice(0, limit);
+    // Walk the list and ask, at a terminal.
+    if (!entryKey && interactive && critic) {
+        const requests = [];
+        walk: for (const item of ranking.ranked.slice(0, limit)) {
+            const { critique } = item.found;
+            io.out(`\n${item.entry.title} (${item.entry.key}) — ${item.entry.state}, score ${critique.verdict?.score ?? 'none'}${critique.verdict?.label ? `, ${critique.verdict.label}` : ''}\n`);
+            const base = { entry: { key: item.entry.key, title: item.entry.title, imageSha256: item.entry.image.sha256 }, critic: { name: critique.critic.name, version: critique.critic.version }, layer: item.found.layer };
+            const targets = (critique.findings ?? []).length
+                ? (critique.findings ?? []).filter(finding => finding.id).map(finding => ({ finding: finding.id, text: `${finding.category ?? 'finding'} (${finding.severity ?? 'no severity'}): ${finding.rationale ?? ''}` }))
+                : [{ finding: undefined, text: 'the opinion as a whole' }];
+            for (const target of targets) {
+                const answer = (await ask(`  ${target.text}\n  [a]ccept, [d]ismiss, [s]kip, [q]uit? `)).trim().toLowerCase();
+                if (answer.startsWith('q'))
+                    break walk;
+                if (answer.startsWith('a') || answer.startsWith('d'))
+                    requests.push({ ...base, ...(target.finding ? { finding: target.finding } : {}), decision: answer.startsWith('a') ? 'accept' : 'dismiss' });
+            }
+        }
+        await record(requests);
+    }
+    const document = {
+        ok: true, server: server.url, critic, critics, reviewer,
+        rankedBy: ranking.rankedBy,
+        summary: { ranked: ranking.ranked.length, unranked: ranking.unranked.length, unjudged: ranking.unjudged },
+        entries: shown,
+        ...(marked.length || accept || dismiss ? { marked: { dryRun, reviewer, layer, marks: marked } } : {})
+    };
+    if (json) {
+        print(io, document);
+        return 0;
+    }
+    if (marked.length) {
+        io.out(`${dryRun ? 'Would mark' : 'Marked'} ${marked.length} ${marked.length === 1 ? 'opinion' : 'opinions'} as ${reviewer}:\n`);
+        for (const mark of marked)
+            io.out(`  ${mark.decision === 'accept' ? 'accepted' : 'dismissed'} ${mark.critic} on ${mark.entry}${mark.finding ? ` (${mark.finding})` : ''}\n`);
+        if (layer)
+            io.out(`Published as layer ${layer.slice(0, 12)}\n`);
+        return 0;
+    }
+    if (!critic) {
+        io.out('No critic has judged a captured state yet. Run `atlas critic run --critic <name> --all` first.\n');
+        return 0;
+    }
+    io.out(`Needs polish, by ${critic} (${ranking.rankedBy === 'standing-within-kind' ? 'lowest standing within its state kind first' : 'lowest score first'}):\n`);
+    for (const entry of shown) {
+        const marksText = entry.marks.state === 'unmarked' ? '' : `  [${entry.marks.state}]`;
+        io.out(`  ${entry.rank === null ? '-' : String(entry.rank).padStart(2)}. ${entry.title} (${entry.key}) ${entry.state} score ${entry.score ?? 'none'}${entry.percentile === null ? '' : `, ${Math.round(entry.percentile)}th percentile in kind`}${entry.currency === 'earlier-image' ? ' — judged an earlier picture' : ''}${entry.failure ? ` — could not judge (${entry.failure})` : ''}${marksText}\n`);
+        for (const finding of entry.findings)
+            io.out(`        ${finding.id ?? '?'}: ${finding.category ?? 'finding'} (${finding.severity ?? 'no severity'}) ${finding.rationale ?? ''}${finding.marks && finding.marks.state !== 'unmarked' ? `  [${finding.marks.state}]` : ''}\n`);
+    }
+    if (ranking.unjudged)
+        io.out(`${ranking.unjudged} not judged yet by ${critic}.\n`);
+    io.out('Mark one with `atlas critic review --entry <key> --accept|--dismiss [--finding <id>] [--note <why>]`.\n');
+    return 0;
+}

package/dist/critics/src/cli-watch.d.ts

@@ -0,0 +1,4 @@
+import { type CriticIo } from './cli-common.js';
+import type { LoadedConfig } from './registry.js';
+/** `atlas critic watch`. */
+export declare function watchCommand(argv: readonly string[], io: CriticIo, loaded: LoadedConfig): Promise<number>;

package/dist/critics/src/cli-watch.js

@@ -0,0 +1,191 @@
+/**
+ * @module `atlas critic watch`
+ *
+ * A worker: follows the catalog of an Atlas server and judges new or changed entries as they arrive, until it is stopped. It is
+ * `atlas critic run` in a loop over `createCatalogClient(…).follow`, so it catches up on connecting, reads again after every
+ * announcement and every reconnection, misses no entry and judges none twice (a picture a critic has judged is skipped by the cache
+ * without a call). What it finished is published as it goes, kept in the spool when the server goes away, and sent by the next
+ * invocation. Budgets are for the whole watch. It reports; it never gates.
+ *
+ * It ends when interrupted, when `--max-batches` batches have been handled, or when a budget is spent, and then prints what it did
+ * and the report version to resume from (`--since`). The server never runs a model: the critic runs here.
+ *
+ * @see docs/critics.md#watching-the-catalog
+ */
+import path from 'node:path';
+import { httpStore } from '../../reports/src/http-store.js';
+import { createCatalogClient } from './catalog-client.js';
+import { connection, defaultAsk, judgeEntries, list, numberFlag, populationOf, print, redactIn, scopeOf, value, wantsJson } from './cli-common.js';
+import { buildCritic } from './registry.js';
+import { flushSpool, pendingInSpool, spoolingStore } from './spool.js';
+/** `atlas critic watch`. */
+export async function watchCommand(argv, io, loaded) {
+    const env = io.env ?? process.env;
+    const json = wantsJson(argv);
+    const interactive = !argv.includes('--non-interactive') && !json && (io.interactive ?? Boolean(process.stdin.isTTY && process.stdout.isTTY));
+    const ask = io.ask ?? defaultAsk;
+    const dryRun = argv.includes('--dry-run');
+    const clock = io.clock ?? Date.now;
+    let names = list(argv, 'critic');
+    let target = await connection(argv, loaded, env);
+    const missing = () => [...(names.length ? [] : ['--critic <name>']), ...(target ? [] : ['--server <url>'])];
+    if (missing().length && interactive) {
+        if (!names.length) {
+            const configured = Object.keys(loaded.config.critics);
+            names = (await ask(`Which critic${configured.length ? ` (${configured.join(', ')})` : ''}? `)).split(/[,\s]+/).filter(Boolean);
+        }
+        if (!target) {
+            const url = (await ask('Atlas server address? ')).trim();
+            if (url)
+                target = { url: url.replace(/\/+$/, ''), ...(env.ATLAS_CRITIC_TOKEN || env.ATLAS_TOKEN ? { token: (env.ATLAS_CRITIC_TOKEN || env.ATLAS_TOKEN) } : {}) };
+        }
+    }
+    if (missing().length)
+        throw new Error(`watch needs ${missing().join(', ')}. Nothing was asked: ${interactive ? 'an answer was empty' : 'there is no terminal (or --non-interactive / --format json was given)'}`);
+    const server = target;
+    const { query: scope, chosen } = scopeOf(argv);
+    const { since: _since, ...filters } = scope;
+    const query = filters;
+    const since = value(argv, 'since');
+    const maxBatches = numberFlag(argv, 'max-batches', true);
+    const total = { images: numberFlag(argv, 'max-images', true), tokens: numberFlag(argv, 'max-tokens', true), costUsd: numberFlag(argv, 'max-cost') };
+    const concurrency = numberFlag(argv, 'concurrency', true), timeoutMs = numberFlag(argv, 'timeout-ms', true);
+    const builts = [];
+    for (const name of names)
+        builts.push(await buildCritic(loaded, name, io));
+    const secrets = builts.flatMap(built => built.secrets);
+    // Images that leave the machine need a yes, once, before the worker starts: it cannot ask later, and it never sends silently.
+    for (const built of builts) {
+        const destination = built.critic.describe().sendsImagesTo;
+        if (destination === null || dryRun || argv.includes('--yes'))
+            continue;
+        if (!interactive)
+            throw new Error(`${built.critic.name} sends images to ${destination}: pass --yes to allow it (nothing was sent)`);
+        const answer = await ask(`${built.critic.name} will send the picture of every new or changed entry to ${destination}, for as long as this runs. Continue? [y/N] `);
+        if (!/^y(es)?$/i.test(answer.trim()))
+            throw new Error(`Stopped: ${built.critic.name} would send images to ${destination} and that was not confirmed (nothing was sent)`);
+    }
+    if (!dryRun)
+        for (const built of builts)
+            await built.ready();
+    const transport = { url: server.url, ...(server.token ? { token: server.token } : {}), ...(server.project ? { project: server.project } : {}), ...(io.fetch ? { fetch: io.fetch } : {}), ...(io.sleep ? { sleep: io.sleep } : {}) };
+    const client = createCatalogClient(transport);
+    const spoolDir = path.resolve(loaded.directory, value(argv, 'spool') ?? loaded.config.spool ?? '.atlas/critic-spool');
+    const base = httpStore(transport);
+    const warnings = [];
+    let flushed = 0;
+    if (!dryRun) {
+        const sent = await flushSpool(base, spoolDir);
+        flushed = sent.sent;
+        if (sent.error)
+            warnings.push(`${sent.remaining} spooled object${sent.remaining === 1 ? '' : 's'} could not be sent yet: ${redactIn(sent.error, secrets)}`);
+    }
+    const store = spoolingStore(base, spoolDir);
+    // A score is read against the project's scores of its state kind: the whole catalog is read once when the watch is narrowed or resumed.
+    const populations = new Map(builts.map(built => [built.critic.name, new Map()]));
+    const seed = async () => {
+        const whole = (await client.read({})).entries;
+        for (const built of builts)
+            for (const [key, item] of populationOf(whole, built.critic.name))
+                populations.get(built.critic.name).set(key, item);
+    };
+    if (Object.keys(filters).length > 0 || since)
+        await seed();
+    const tallies = new Map(builts.map(built => [built.critic.name, { entries: 0, judged: 0, skipped: 0, failed: 0, spent: { images: 0, tokens: 0, costUsd: 0 }, layers: 0, critiques: 0, failures: [] }]));
+    const controller = new AbortController();
+    if (io.signal?.aborted)
+        controller.abort();
+    else
+        io.signal?.addEventListener('abort', () => controller.abort(), { once: true });
+    let stoppedBy = 'interrupted';
+    let version = since ?? null;
+    let batches = 0;
+    const spentTotal = () => [...tallies.values()].reduce((sum, tally) => ({ images: sum.images + tally.spent.images, tokens: sum.tokens + tally.spent.tokens, costUsd: sum.costUsd + tally.spent.costUsd }), { images: 0, tokens: 0, costUsd: 0 });
+    const exhausted = () => { const spent = spentTotal(); return (total.images !== undefined && spent.images >= total.images) || (total.tokens !== undefined && spent.tokens >= total.tokens) || (total.costUsd !== undefined && spent.costUsd >= total.costUsd); };
+    if (!json)
+        io.out(dryRun ? 'Reading the catalog once (dry run: nothing is sent or published)\n' : `Watching ${server.url} with ${builts.map(built => `${built.critic.name}@${built.critic.version}`).join(', ')}. Press Ctrl-C to stop.\n`);
+    for await (const batch of client.follow({ ...(since ? { since } : {}), query, signal: controller.signal })) {
+        version = batch.version;
+        batches += 1;
+        const lines = [];
+        for (const built of builts) {
+            const name = built.critic.name;
+            const tally = tallies.get(name);
+            const population = populations.get(name);
+            for (const [key, item] of populationOf(batch.entries, name))
+                population.set(key, item);
+            const spent = spentTotal();
+            const budget = {
+                ...(total.images === undefined ? {} : { maxImages: Math.max(0, total.images - spent.images) }),
+                ...(total.tokens === undefined ? {} : { maxTokens: Math.max(0, total.tokens - spent.tokens) }),
+                ...(total.costUsd === undefined ? {} : { maxCostUsd: Math.max(0, total.costUsd - spent.costUsd) })
+            };
+            if (!batch.entries.length || (Object.keys(budget).length && Object.values(budget).some(left => left <= 0)))
+                continue;
+            const { result, outcomes, layer } = await judgeEntries({
+                built, entries: batch.entries, client, store, population, dryRun, budget, secrets, clock, signal: controller.signal,
+                ...(concurrency ? { concurrency } : {}), ...(timeoutMs ? { timeoutMs } : {})
+            });
+            const run = result;
+            tally.entries += batch.entries.length;
+            tally.judged += run.judged;
+            tally.skipped += run.skipped;
+            tally.failed += run.failed;
+            tally.spent.images += run.spent.images;
+            tally.spent.tokens += run.spent.tokens;
+            tally.spent.costUsd += run.spent.costUsd;
+            if (layer) {
+                tally.layers += 1;
+                tally.critiques += outcomes.length;
+            }
+            for (const outcome of outcomes)
+                if (outcome.critique.failure)
+                    tally.failures.push({ entry: outcome.entry.key, reason: outcome.critique.failure.reason });
+            lines.push(`${name}: ${batch.entries.length} ${batch.entries.length === 1 ? 'entry' : 'entries'}, ${run.judged} judged, ${run.skipped} skipped, ${run.failed} failed`);
+        }
+        if (!json && lines.length)
+            io.out(`${dryRun ? 'Would handle' : 'Handled'} ${batch.initial ? 'the catalog' : 'what arrived'} at version ${batch.version.slice(0, 12)}: ${lines.join('; ')}\n`);
+        if (dryRun) {
+            stoppedBy = 'dry-run';
+            controller.abort();
+        }
+        else if (exhausted()) {
+            stoppedBy = 'budget';
+            controller.abort();
+        }
+        else if (maxBatches !== undefined && batches >= maxBatches) {
+            stoppedBy = 'max-batches';
+            controller.abort();
+        }
+    }
+    const pending = await pendingInSpool(spoolDir);
+    if (store.spooled() > 0)
+        warnings.push(`The server could not be reached: ${store.spooled()} object${store.spooled() === 1 ? '' : 's'} of critiques were kept in ${path.relative(io.cwd ?? process.cwd(), spoolDir) || spoolDir} and will be sent by the next invocation`);
+    const failed = [...tallies.values()].some(tally => tally.failed > 0);
+    const ok = !failed;
+    const document = {
+        ok, dryRun, server: server.url, scope: chosen, stoppedBy, batches, version,
+        critics: builts.map(built => {
+            const tally = tallies.get(built.critic.name);
+            return {
+                critic: built.critic.name, version: built.critic.version, sendsImagesTo: built.critic.describe().sendsImagesTo,
+                entries: tally.entries, judged: tally.judged, skipped: tally.skipped, failed: tally.failed, failures: tally.failures, spent: tally.spent,
+                published: { layers: tally.layers, critiques: tally.critiques }
+            };
+        }),
+        spool: { flushed, spooled: store.spooled(), pending },
+        warnings: warnings.map(warning => redactIn(warning, secrets))
+    };
+    if (json) {
+        print(io, document);
+        return ok ? 0 : 1;
+    }
+    io.out(`Stopped (${stoppedBy === 'interrupted' ? 'interrupted' : stoppedBy === 'budget' ? 'the budget was reached' : stoppedBy === 'max-batches' ? `${batches} ${batches === 1 ? 'batch' : 'batches'} handled` : 'dry run'}) after ${batches} ${batches === 1 ? 'batch' : 'batches'}.\n`);
+    for (const item of document.critics)
+        io.out(`  ${item.critic}@${item.version}: ${item.judged} judged, ${item.skipped} skipped, ${item.failed} failed; published ${item.published.critiques} ${item.published.critiques === 1 ? 'critique' : 'critiques'} in ${item.published.layers} ${item.published.layers === 1 ? 'layer' : 'layers'}\n`);
+    if (version && !dryRun)
+        io.out(`Resume with --since ${version}\n`);
+    for (const warning of document.warnings)
+        io.out(`! ${warning}\n`);
+    return ok ? 0 : 1;
+}