@crvouga/atlas 1.20.0 → 1.22.0

package/dist/schema/src/cli-results.js

@@ -1,7 +1,7 @@
 /**
  * @module Command results (`--format json`)
  *
- * What `atlas report publish`, `stream`, `merge`, `doctor` and `verify-recordings` print on standard output with `--format json` (alias
+ * What `atlas report publish`, `stream`, `merge`, `doctor` and `verify-recordings`, `atlas ui-catalog plan`, `publish`, `status`, `diff` and `export`, and `atlas critic list`, `setup`, `doctor` and `run`, print on standard output with `--format json` (alias
  * `--json`), and what any `atlas` command prints on standard error for a command that was wrong: one JSON document,
  * so an agent or a script never has to parse prose. (`status` and `policy` already print documents:
  * `ReportStatusSchema`, and an IAM policy.) Each is also published as a JSON Schema.
@@ -154,6 +154,258 @@ export const InitPlanSchema = z.looseObject({
     nextSteps: z.array(z.string()).describe('What to do next, in order, as plain instructions an agent can follow'),
     warnings: z.array(z.string())
 });
+/** One critic as `atlas critic list` shows it. */
+export const CriticListEntrySchema = z.looseObject({
+    name: z.string().describe('The name in the configuration; also the producer suffix (`critic.<name>`)'),
+    type: z.string().describe('`uiclip`, `llm`, `module`, or `not-configured` for a built-in critic that has no configuration yet'),
+    kind: z.string().nullable().describe('`model`, `llm`, `heuristic`… as the critic says; null when it is not configured'),
+    version: z.string().nullable(),
+    status: z.enum(['ready', 'needs-setup', 'misconfigured', 'not-configured']).describe('`needs-setup`: run `atlas critic setup <name>`. `misconfigured`: the configuration or a credential reference cannot be used (see `detail`)'),
+    sendsImagesTo: z.string().nullable().describe('Where images are sent: a host, or null when they never leave this machine. A run that sends images to a third party needs confirmation'),
+    model: z.string().nullable(),
+    detail: z.string().optional().describe('Why it is not ready, and what to do. Never holds a credential')
+});
+/** What `atlas critic list --format json` prints. */
+export const CriticListResultSchema = z.looseObject({
+    ok: z.literal(true),
+    config: z.string().nullable().describe('The configuration file that was read, or null when there is none'),
+    critics: z.array(CriticListEntrySchema)
+});
+/** What `atlas critic setup --format json` prints. */
+export const CriticSetupResultSchema = z.looseObject({
+    ok: z.boolean(),
+    critic: z.string(),
+    type: z.string(),
+    details: z.array(z.string()).describe('What was provisioned or checked. A checkpoint is verified against its pinned SHA-256; inference never downloads anything'),
+    error: z.string().optional().describe('Why setup failed, with no credential in it. The exit code is then 1')
+});
+/** One thing `atlas critic doctor` checked. */
+export const CriticDoctorCheckSchema = z.looseObject({
+    name: z.string().describe('`server`, `token`, `scope`, `catalog`, `spool`, or `<critic>:<aspect>` such as `uiclip:model-file`'),
+    status: z.enum(['ok', 'warning', 'problem']).describe('`problem` makes the exit code 1; `warning` does not'),
+    detail: z.string(),
+    fix: z.string().optional().describe('What to change, when the status is not `ok`')
+});
+/** What `atlas critic doctor --format json` prints. */
+export const CriticDoctorResultSchema = z.looseObject({
+    ok: z.boolean().describe('False when any check is a `problem` (the exit code is then 1)'),
+    server: z.string().nullable().describe('The server that was checked, with no secrets'),
+    role: z.string().nullable().describe('What the token may do: `read`, `critic`, `publish` or `admin`; null when the server could not be asked'),
+    open: z.boolean().nullable().describe('True when the server has no tokens, so every caller is an administrator; null when unknown'),
+    checks: z.array(CriticDoctorCheckSchema)
+});
+/** What `atlas critic run` did for one critic. */
+export const CriticRunCriticSchema = z.looseObject({
+    critic: z.string(),
+    version: z.string(),
+    sendsImagesTo: z.string().nullable(),
+    entries: z.number().int().nonnegative().describe('Entries in the chosen scope'),
+    judged: z.number().int().nonnegative(),
+    skipped: z.number().int().nonnegative().describe('Entries that already had a current critique of the same identity (critic, version, model, parameters, image hash): no call was made'),
+    failed: z.number().int().nonnegative().describe('Entries recorded as failed with a reason code; the next run retries them'),
+    waiting: z.number().int().nonnegative().describe('Entries left unjudged because the run stopped'),
+    stoppedBy: z.enum(['budget', 'aborted']).optional(),
+    failures: z.array(z.looseObject({ entry: z.string(), reason: z.string() })).describe('Each failed entry and its reason code'),
+    spent: z.looseObject({ images: z.number(), tokens: z.number(), costUsd: z.number() }),
+    published: z.looseObject({ layer: z.string().nullable().describe('The layer id of the critiques, or null when nothing was published (a dry run, or nothing to publish)'), critiques: z.number().int().nonnegative() }),
+    gate: z.looseObject({
+        mode: z.enum(['report', 'warn', 'gate']),
+        exitCode: z.number().int(),
+        below: z.number().int().nonnegative().describe('Scores below their state kind\'s threshold'),
+        warnings: z.array(z.string()),
+        refusal: z.array(z.string()).describe('Why `gate` was refused, when it was')
+    }).optional()
+});
+/** What `atlas critic run --format json` prints. A dry run publishes and sends nothing. */
+export const CriticRunResultSchema = z.looseObject({
+    ok: z.boolean().describe('False when a critic failed on any entry or a gate was crossed (the exit code is then 1)'),
+    dryRun: z.boolean(),
+    server: z.string(),
+    scope: z.record(z.string(), z.string()).describe('The choices that narrowed the catalog, as given'),
+    critics: z.array(CriticRunCriticSchema),
+    spool: z.looseObject({
+        flushed: z.number().int().nonnegative().describe('Objects from an earlier run that were sent now'),
+        spooled: z.number().int().nonnegative().describe('Objects of this run kept locally because the server could not be reached; the next invocation sends them'),
+        pending: z.number().int().nonnegative().describe('Objects still waiting in the spool')
+    }),
+    warnings: z.array(z.string())
+});
+/** What `atlas critic watch` did for one critic. */
+export const CriticWatchCriticSchema = z.looseObject({
+    critic: z.string(),
+    version: z.string(),
+    sendsImagesTo: z.string().nullable(),
+    entries: z.number().int().nonnegative().describe('Entries handed over by the catalog, counting a changed picture again'),
+    judged: z.number().int().nonnegative(),
+    skipped: z.number().int().nonnegative().describe('Entries that already had a current critique of the same identity: no call was made'),
+    failed: z.number().int().nonnegative().describe('Entries recorded as failed with a reason code; the next batch that brings them, or the next run, retries them'),
+    failures: z.array(z.looseObject({ entry: z.string(), reason: z.string() })),
+    spent: z.looseObject({ images: z.number(), tokens: z.number(), costUsd: z.number() }),
+    published: z.looseObject({ layers: z.number().int().nonnegative(), critiques: z.number().int().nonnegative() })
+});
+/** What `atlas critic watch --format json` prints, once, when the watch ends. */
+export const CriticWatchResultSchema = z.looseObject({
+    ok: z.boolean().describe('False when a critic failed on any entry (the exit code is then 1)'),
+    dryRun: z.boolean(),
+    server: z.string(),
+    scope: z.record(z.string(), z.string()).describe('The choices that narrowed the catalog, as given'),
+    stoppedBy: z.enum(['interrupted', 'max-batches', 'budget', 'dry-run']).describe('Why the watch ended: Ctrl-C or a signal, `--max-batches`, a budget for the whole watch, or a dry run (one read)'),
+    batches: z.number().int().nonnegative().describe('How many times the catalog handed over something new, the catch-up on connecting included'),
+    version: z.string().nullable().describe('The report version the watch had reached: resume with `--since` this. Null when nothing was read'),
+    critics: z.array(CriticWatchCriticSchema),
+    spool: z.looseObject({ flushed: z.number().int().nonnegative(), spooled: z.number().int().nonnegative(), pending: z.number().int().nonnegative() }).describe('As for `critic-run-result`'),
+    warnings: z.array(z.string())
+});
+/** What reviewers decided about one opinion, as `atlas critic review` reports it. */
+export const CriticMarkStandingSchema = z.looseObject({
+    state: z.enum(['accepted', 'dismissed', 'disputed', 'unmarked']).describe('`disputed`: reviewers disagree. `unmarked`: nobody has marked it at the current picture'),
+    accepted: z.array(z.string()).describe('Reviewers who accepted it'),
+    dismissed: z.array(z.string()).describe('Reviewers who dismissed it'),
+    earlier: z.number().int().nonnegative().describe('Marks made about an earlier picture, which count for nothing now'),
+    notes: z.array(z.string())
+});
+/** What `atlas critic review --format json` prints. */
+export const CriticReviewResultSchema = z.looseObject({
+    ok: z.literal(true),
+    server: z.string(),
+    critic: z.string().nullable().describe('The critic whose opinions are listed; null when none has judged anything'),
+    critics: z.array(z.string()).describe('Every critic with an opinion, most opinions first'),
+    reviewer: z.string().describe('The name marks are recorded under'),
+    rankedBy: z.enum(['standing-within-kind', 'score']),
+    summary: z.looseObject({ ranked: z.number().int().nonnegative(), unranked: z.number().int().nonnegative().describe('Judged at an earlier picture, or the critic could not judge'), unjudged: z.number().int().nonnegative() }),
+    entries: z.array(z.looseObject({
+        rank: z.number().int().positive().nullable().describe('Null for an entry that is not ranked'),
+        key: z.string(), title: z.string(), state: z.string(), imageSha256: z.string(),
+        score: z.number().nullable(), percentile: z.number().nullable().describe('Standing among the scores of the same state kind, 0..100, higher is better'), label: z.string().nullable(),
+        currency: z.enum(['current', 'earlier-image']),
+        failure: z.string().optional().describe('The reason code, when the critic could not judge it'),
+        marks: CriticMarkStandingSchema.describe('The marks of the whole opinion'),
+        findings: z.array(z.looseObject({
+            id: z.string().nullable().describe('A finding without an id cannot be marked on its own'),
+            category: z.string().nullable(), severity: z.string().nullable(),
+            region: z.looseObject({ unit: z.string(), x: z.number(), y: z.number(), w: z.number(), h: z.number() }).optional(),
+            rationale: z.string().nullable(), suggestion: z.string().nullable(),
+            marks: CriticMarkStandingSchema.nullable()
+        }))
+    })),
+    marked: z.looseObject({
+        dryRun: z.boolean(), reviewer: z.string(),
+        layer: z.string().nullable().describe('The layer the marks were published as; null on a dry run or when nothing was marked'),
+        marks: z.array(z.looseObject({ entry: z.string(), critic: z.string(), finding: z.string().nullable().describe('Null: the whole opinion'), decision: z.enum(['accept', 'dismiss']) }))
+    }).optional().describe('Present when this call recorded (or, with --dry-run, would record) marks')
+});
+/** What `atlas critic calibrate --format json` prints. A dry run measures nothing and writes nothing. */
+export const CriticCalibrateResultSchema = z.looseObject({
+    ok: z.literal(true),
+    dryRun: z.boolean(),
+    critic: z.looseObject({ name: z.string(), version: z.string().nullable() }),
+    sendsImagesTo: z.string().nullable().describe('Where the labelled pictures were (or, on a dry run, would be) sent; null when they stay on this machine or only marks were used'),
+    examples: z.looseObject({
+        files: z.number().int().nonnegative().describe('Examples from labelled pictures (on a dry run, the pictures found)'),
+        marks: z.number().int().nonnegative().describe('Examples from reviewers\' whole-opinion marks'),
+        used: z.number().int().nonnegative(),
+        skipped: z.array(z.looseObject({ source: z.string(), reason: z.string() })).describe('Examples left out, and why: no score, a critic failure, disputed marks, a budget')
+    }),
+    calibration: z.looseObject({
+        schema: z.literal(1),
+        critic: z.looseObject({ name: z.string(), version: z.string(), model: z.looseObject({ id: z.string(), provider: z.string().optional() }).optional(), parameters: z.record(z.string(), z.unknown()).optional() }),
+        createdAtMs: z.number().optional(),
+        kinds: z.array(z.looseObject({
+            stateKind: z.string(),
+            threshold: z.number().nullable().describe('Scores below this are flagged as needing polish; null when the examples cannot tell (one class is missing)'),
+            samples: z.number().int().nonnegative(), positives: z.number().int().nonnegative(), negatives: z.number().int().nonnegative(),
+            precision: z.number().nullable(), recall: z.number().nullable(),
+            inSample: z.boolean().describe('True when precision and recall are measured on the examples the threshold was chosen on, which flatters them'),
+            evaluatedOn: z.enum(['fit', 'held-out'])
+        }))
+    }).nullable().describe('The calibration `atlas critic run --calibration` reads; null on a dry run'),
+    written: z.string().nullable().describe('The file it was written to (`--out`), or null'),
+    gate: z.looseObject({ eligible: z.boolean().describe('Whether `--mode gate` would accept this calibration'), reasons: z.array(z.string()).describe('What is missing, per state kind, when it would not') }).nullable(),
+    warnings: z.array(z.string())
+});
+/** The coverage of one state in one profile, as the UI catalog words it. */
+const UiCoverageSchema = z.enum(['captured', 'failed', 'capturing', 'skipped', 'missing']);
+/** What `atlas ui-catalog plan --format json` and `atlas ui-catalog publish --format json` print: the one layer that was written. */
+export const UiCatalogPublishResultSchema = z.looseObject({
+    ok: z.literal(true).describe('True: the layer was written. A state whose picture could not be read is published as `failed` and named in `warnings`; it does not fail the command'),
+    store: z.string().describe('Where it was written: a folder, `s3://…` or a server URL'),
+    layer: z.looseObject({
+        id: z.string().describe('The layer id: 64 hex digits, the SHA-256 of its content. Publishing the same input again gives the same id'),
+        kind: z.enum(['ui-state', 'ui-plan']).describe('`ui-state`: captures (`publish`). `ui-plan`: the expected set (`plan`)'),
+        producer: z.string(),
+        environment: z.string().nullable(),
+        parameters: z.record(z.string(), z.string()).describe('The rest of the profile, such as `{ "viewport": "390x844" }`'),
+        states: z.number().int().nonnegative().describe('States in the layer'),
+        captured: z.number().int().nonnegative().describe('For `publish`: states that were captured (`passed`). Zero for a plan'),
+        failed: z.number().int().nonnegative().describe('For `publish`: states recorded as failed, including those whose picture could not be read'),
+        pictures: z.number().int().nonnegative().describe('Pictures stored in the layer'),
+        pictureBytes: z.number().int().nonnegative(),
+        shard: z.string().nullable().describe('The `--shard` label, which describes the layer and never splits its slot')
+    }),
+    warnings: z.array(z.string()).describe('What was skipped and why, such as a picture file that could not be read. Never fatal')
+});
+/** One thing that keeps a UI catalog from being complete and good. */
+export const UiCatalogProblemSchema = z.looseObject({
+    kind: z.enum(['failed', 'missing', 'capturing', 'unavailable', 'unreadable', 'empty']).describe('`failed`: a capture was attempted and failed. `missing`: a plan expects it and nothing was captured. `capturing`: still being taken, so not known. `unavailable`: a backend could not be read. `unreadable`: a layer could not be read. `empty`: the catalog holds no state at all'),
+    state: z.string().optional().describe('The state id, for `failed`, `missing` and `capturing`'),
+    profile: z.string().optional().describe('The profile label, for `failed`, `missing` and `capturing`'),
+    detail: z.string().optional().describe('Why, when the producer said')
+});
+/** What `atlas ui-catalog status --format json` prints. */
+export const UiCatalogStatusSchema = z.looseObject({
+    ok: z.boolean().describe('True only when the catalog holds a state and has no problem: nothing failed, nothing expected is missing, nothing is still being captured, and every backend and layer was read (otherwise the exit code is 1). Skipped cells are counted, not problems'),
+    summary: z.looseObject({
+        states: z.number().int().nonnegative(),
+        profiles: z.number().int().nonnegative(),
+        cells: z.number().int().nonnegative().describe('States counted once per profile'),
+        coverage: z.record(UiCoverageSchema, z.number().int().nonnegative()).describe('Cells by coverage'),
+        undeclared: z.number().int().nonnegative().describe('Captured cells no plan expects; zero when no plan exists'),
+        planned: z.boolean().describe('Whether any plan was published. Without one, `missing` is always zero because what should exist is unknown'),
+        surfaces: z.number().int().nonnegative(),
+        uncoveredSurfaces: z.number().int().nonnegative()
+    }),
+    profiles: z.array(z.looseObject({ key: z.string(), label: z.string() })),
+    problems: z.array(UiCatalogProblemSchema),
+    sources: z.looseObject({ layers: z.number().int().nonnegative(), backends: z.array(z.looseObject({ id: z.string(), label: z.string(), available: z.boolean() })), notices: z.array(z.string()) })
+});
+/** One side of `atlas ui-catalog diff`. */
+const UiCatalogCommitSchema = z.looseObject({ sha: z.string(), states: z.number().int().nonnegative(), layers: z.number().int().nonnegative() });
+/** What `atlas ui-catalog diff --format json` prints. */
+export const UiCatalogDiffResultSchema = z.looseObject({
+    ok: z.boolean().describe('False when a layer or backend could not be read (the comparison may be wrong), or, with `--fail-on-change`, when anything differs. Otherwise true: a difference is not a failure by itself'),
+    from: UiCatalogCommitSchema,
+    to: UiCatalogCommitSchema,
+    summary: z.looseObject({
+        coverage: z.number().int().nonnegative().describe('Pairs that stand differently (captured against failed, for example)'),
+        picture: z.number().int().nonnegative().describe('Both captured, and the stored bytes differ. Not a measure of how many pixels'),
+        unverified: z.number().int().nonnegative().describe('Both captured, but a side has no content hash, so whether the picture changed is unknown'),
+        added: z.number().int().nonnegative().describe('Only `to` has it'),
+        removed: z.number().int().nonnegative().describe('Only `from` has it. Not a regression: the other side may not have run it'),
+        unchanged: z.number().int().nonnegative(),
+        total: z.number().int().nonnegative()
+    }),
+    changes: z.array(z.looseObject({
+        state: z.string(), title: z.string(), profile: z.string(),
+        change: z.enum(['coverage', 'picture', 'unverified', 'added', 'removed']),
+        from: UiCoverageSchema.nullable(), to: UiCoverageSchema.nullable()
+    })).describe('Every pair that is not unchanged'),
+    notices: z.array(z.string()).describe('What could not be read. Any makes `ok` false: an unreadable layer is unknown, never unchanged')
+});
+/** What `atlas ui-catalog export --format json` prints. */
+export const UiCatalogExportResultSchema = z.looseObject({
+    ok: z.literal(true),
+    out: z.string().describe('The HTML file that was written'),
+    bytes: z.number().int().nonnegative().describe('Its size'),
+    states: z.number().int().nonnegative(),
+    cells: z.number().int().nonnegative(),
+    pictures: z.looseObject({
+        embedded: z.number().int().nonnegative(),
+        embeddedBytes: z.number().int().nonnegative(),
+        left: z.number().int().nonnegative().describe('Pictures not embedded (over the size limit, not stored, not an image, or `--images none`); each is listed on the page with the reason')
+    }),
+    manifest: z.string().nullable().describe('The JSON manifest written beside it with `--manifest`, or null'),
+    warnings: z.array(z.string()).describe('Backends and layers that could not be read, and pictures left out. The page says so too')
+});
 /**
  * JSON command contracts, shared by schema publication, agent documentation and executable command checks.
  * A new result needs an entry here and a runtime scenario in the documentation contract tests.
@@ -166,5 +418,17 @@ export const CLI_RESULTS = {
     doctor: { command: 'atlas report doctor', schemaName: 'doctor-result', schema: DoctorResultSchema },
     status: { command: 'atlas report status', schemaName: 'report-status', schema: ReportStatusSchema },
     verifyRecordings: { command: 'atlas report verify-recordings', schemaName: 'verify-recordings-result', schema: VerifyRecordingsResultSchema },
+    criticList: { command: 'atlas critic list', schemaName: 'critic-list-result', schema: CriticListResultSchema },
+    criticSetup: { command: 'atlas critic setup', schemaName: 'critic-setup-result', schema: CriticSetupResultSchema },
+    criticDoctor: { command: 'atlas critic doctor', schemaName: 'critic-doctor-result', schema: CriticDoctorResultSchema },
+    criticRun: { command: 'atlas critic run', schemaName: 'critic-run-result', schema: CriticRunResultSchema },
+    criticWatch: { command: 'atlas critic watch', schemaName: 'critic-watch-result', schema: CriticWatchResultSchema },
+    criticReview: { command: 'atlas critic review', schemaName: 'critic-review-result', schema: CriticReviewResultSchema },
+    criticCalibrate: { command: 'atlas critic calibrate', schemaName: 'critic-calibrate-result', schema: CriticCalibrateResultSchema },
+    uiCatalogPlan: { command: 'atlas ui-catalog plan', schemaName: 'ui-catalog-publish-result', schema: UiCatalogPublishResultSchema },
+    uiCatalogPublish: { command: 'atlas ui-catalog publish', schemaName: 'ui-catalog-publish-result', schema: UiCatalogPublishResultSchema },
+    uiCatalogStatus: { command: 'atlas ui-catalog status', schemaName: 'ui-catalog-status', schema: UiCatalogStatusSchema },
+    uiCatalogDiff: { command: 'atlas ui-catalog diff', schemaName: 'ui-catalog-diff-result', schema: UiCatalogDiffResultSchema },
+    uiCatalogExport: { command: 'atlas ui-catalog export', schemaName: 'ui-catalog-export-result', schema: UiCatalogExportResultSchema },
     error: { command: 'a command that was wrong (standard error)', schemaName: 'command-error', schema: CommandErrorSchema }
 };

package/dist/schema/src/critique.d.ts

@@ -142,3 +142,34 @@ export declare const CritiqueSchema: z.ZodObject<{
 export type Critique = z.output<typeof CritiqueSchema>;
 /** One finding of a critique. */
 export type CritiqueFinding = z.output<typeof CritiqueFindingSchema>;
+/** The key under which a reviewer's mark of a critique is stored in a check's `extensions`. */
+export declare const CRITIQUE_MARK_EXTENSION = "atlas.critique-mark";
+/** The version of the `atlas.critique-mark` extension a reader of this release understands. A mark of another version is unknown, not wrong. */
+export declare const CRITIQUE_MARK_SCHEMA_VERSION = 1;
+/** The prefix of the `producer` of a layer of marks: `critic.review.<reviewer>`. It is within what a critic-role token may publish. */
+export declare const CRITIQUE_MARK_PRODUCER_PREFIX = "critic.review.";
+/**
+ * The `atlas.critique-mark` extension, version 1: a person's verdict on a critic's opinion. `accept` says the opinion was right
+ * (the finding is valid, the UI needs polish); `dismiss` says it was not. A mark names the entry and the exact picture it was made
+ * about, the critic whose opinion it marks, and optionally one finding of it; without `finding` it marks the critique as a whole.
+ * Like the critique it is evidence in a part of kind `critique`: it is never counted and never changes a status, a slot or the gate.
+ * A newer mark of the same reviewer for the same subject replaces an older one. Emitted as `critique-mark.schema.json`.
+ */
+export declare const CritiqueMarkSchema: z.ZodObject<{
+    schema: z.ZodNumber;
+    subject: z.ZodObject<{
+        entryKey: z.ZodString;
+        imageSha256: z.ZodString;
+    }, z.core.$loose>;
+    critic: z.ZodObject<{
+        name: z.ZodString;
+        version: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>;
+    finding: z.ZodOptional<z.ZodString>;
+    decision: z.ZodString;
+    reviewer: z.ZodOptional<z.ZodString>;
+    note: z.ZodOptional<z.ZodString>;
+    layer: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>;
+/** A mark as stored in a check's `extensions["atlas.critique-mark"]`. */
+export type CritiqueMark = z.output<typeof CritiqueMarkSchema>;

package/dist/schema/src/critique.js

@@ -94,3 +94,29 @@ export const CritiqueSchema = z.looseObject({
         costUsd: z.number().nonnegative().optional()
     }).optional()
 });
+/** The key under which a reviewer's mark of a critique is stored in a check's `extensions`. */
+export const CRITIQUE_MARK_EXTENSION = 'atlas.critique-mark';
+/** The version of the `atlas.critique-mark` extension a reader of this release understands. A mark of another version is unknown, not wrong. */
+export const CRITIQUE_MARK_SCHEMA_VERSION = 1;
+/** The prefix of the `producer` of a layer of marks: `critic.review.<reviewer>`. It is within what a critic-role token may publish. */
+export const CRITIQUE_MARK_PRODUCER_PREFIX = 'critic.review.';
+/**
+ * The `atlas.critique-mark` extension, version 1: a person's verdict on a critic's opinion. `accept` says the opinion was right
+ * (the finding is valid, the UI needs polish); `dismiss` says it was not. A mark names the entry and the exact picture it was made
+ * about, the critic whose opinion it marks, and optionally one finding of it; without `finding` it marks the critique as a whole.
+ * Like the critique it is evidence in a part of kind `critique`: it is never counted and never changes a status, a slot or the gate.
+ * A newer mark of the same reviewer for the same subject replaces an older one. Emitted as `critique-mark.schema.json`.
+ */
+export const CritiqueMarkSchema = z.looseObject({
+    schema: z.number().int().positive().describe('The version of this extension. 1'),
+    subject: z.looseObject({
+        entryKey: z.string().min(1).describe('The catalog entry the marked critique is about'),
+        imageSha256: z.string().min(1).describe('The hash of the picture the marked critique judged, so a mark about an earlier picture is shown as that')
+    }),
+    critic: z.looseObject({ name: z.string().min(1), version: z.string().min(1).optional() }).describe('The critic whose opinion is marked'),
+    finding: z.string().min(1).optional().describe('The `id` of the marked finding. Absent: the critique as a whole'),
+    decision: z.string().min(1).describe('Open vocabulary; this release understands `accept` (the opinion is right) and `dismiss` (it is not). Anything else is shown as written and counts as neither'),
+    reviewer: z.string().min(1).optional().describe('Who marked it: a name the reviewer chose, never an account'),
+    note: z.string().optional().describe('Why, in the reviewer\'s words'),
+    layer: z.string().optional().describe('The layer of the critique that was marked, for tracing')
+});

package/dist/schema/src/server-config.d.ts

@@ -16,8 +16,8 @@ import { z } from 'zod/v4';
  * anything) or `admin` (also remove scratch objects).
  */
 export declare const ServerRoleSchema: z.ZodEnum<{
-    read: "read";
     critic: "critic";
+    read: "read";
     publish: "publish";
     admin: "admin";
 }>;
@@ -30,8 +30,8 @@ export type ServerRole = z.infer<typeof ServerRoleSchema>;
 export declare const ServerTokenSchema: z.ZodObject<{
     name: z.ZodString;
     role: z.ZodEnum<{
-        read: "read";
         critic: "critic";
+        read: "read";
         publish: "publish";
         admin: "admin";
     }>;
@@ -49,8 +49,8 @@ export declare const ServerConfigSchema: z.ZodObject<{
         tokens: z.ZodDefault<z.ZodArray<z.ZodObject<{
             name: z.ZodString;
             role: z.ZodEnum<{
-                read: "read";
                 critic: "critic";
+                read: "read";
                 publish: "publish";
                 admin: "admin";
             }>;

package/dist/server/src/app.js

@@ -29,6 +29,7 @@ import { latestSpecSet, readSpecBody } from '../../reports/src/spec-set.js';
 import { catalogHealth, healthMarkdown, parseRootMapping, parseRootMappings } from '../../reports/src/catalog-health.js';
 import { parseArtifactFilter } from '../../reports/src/artifact-query.js';
 import { filterUiCatalog, pageUiCatalog, parseUiCatalogQuery, uiCatalog, uiCatalogMarkdown } from '../../reports/src/ui-catalog.js';
+import { catalogEntries, pageCatalogEntries, parseCatalogEntriesQuery, CATALOG_ENTRIES_VERSION } from '../../reports/src/ui-catalog-entries.js';
 import { parseReliabilityScope, reliability, reliabilityMarkdown } from '../../reports/src/reliability.js';
 import { artifactList, artifactsMarkdown, catalogMarkdown, catalogModel, historyMarkdown, parseTestQuery, runDetail, runList, runMarkdown, runsMarkdown, testCatalog, testHistory } from '../../reports/src/test-catalog.js';
 import { llmsTxt } from './llms.js';
@@ -261,7 +262,7 @@ function createProjectApplication(options) {
             name: 'atlas', version: options.version ?? null, role: identity?.role ?? 'read', open: auth.open,
             write: options.write, ready: index.ready, viewers: events.count, time: now(),
             backends: snapshot.backends.map(({ id, label, available }) => ({ id, label, available, write: id === options.write })),
-            ...(({ backends: _unused, ...rest }) => rest)(index.stats()), capabilities: { events: true, ingest: true, store: true, checks: true, status: true, effective: true, live: true, tests: true, artifactTags: true, artifactSearch: true, reviewCollections: true, recordingVerification: true }
+            ...(({ backends: _unused, ...rest }) => rest)(index.stats()), capabilities: { events: true, ingest: true, store: true, checks: true, status: true, effective: true, live: true, tests: true, artifactTags: true, artifactSearch: true, reviewCollections: true, recordingVerification: true, catalogEntries: true }
         });
     });
     app.get('/api/live', async (c) => c.json({ runs: await live.list() }, 200, { 'cache-control': 'no-store' }));
@@ -420,6 +421,7 @@ function createProjectApplication(options) {
         }
         return model.value;
     };
+    let entriesModel;
     let uiModel;
     const wantsMarkdown = (c) => {
         const format = c.req.query('format');
@@ -522,6 +524,31 @@ function createProjectApplication(options) {
         const document = () => pageUiCatalog(filterUiCatalog(whole, parseUiCatalogQuery(search)), { limit: number('limit', 200), offset: number('offset', 0) });
         return answer(c, snapshot.version, document, () => uiCatalogMarkdown(document()), `ui-catalog|${search.toString()}|${wantsMarkdown(c)}`);
     });
+    app.get('/api/reports/ui-catalog/entries', async (c) => {
+        if (!index.ready)
+            return notReady(c);
+        const snapshot = index.snapshot();
+        const search = new URL(c.req.url).searchParams;
+        const query = parseCatalogEntriesQuery(search);
+        // Entries are a function of the report, so they are listed once per version and narrowed per question.
+        if (entriesModel?.version !== snapshot.version)
+            entriesModel = { version: snapshot.version, value: catalogEntries(snapshot.layers) };
+        const whole = entriesModel.value;
+        const changed = query.since === undefined ? null : index.changedSince(query.since);
+        let page;
+        try {
+            page = pageCatalogEntries(whole.entries, query, changed);
+        }
+        catch (error) {
+            return problemResponse(400, error instanceof Error ? error.message : 'Bad cursor.');
+        }
+        const document = () => ({
+            schemaVersion: CATALOG_ENTRIES_VERSION, version: snapshot.version, ...page,
+            ...(query.since === undefined ? {} : { since: { requested: query.since, honored: changed !== null } }),
+            unjudgeable: whole.unjudgeable, sources: { backends: snapshot.backends, notices: snapshot.notices }
+        });
+        return answer(c, snapshot.version, document, () => `${page.entries.map(entry => `- ${entry.key} (${entry.state}): ${entry.title}`).join('\n')}\n`, `ui-entries|${search.toString()}|${wantsMarkdown(c)}`);
+    });
     app.get('/api/reports/runs/detail', async (c) => {
         if (!index.ready)
             return notReady(c);

package/dist/server/src/openapi.js

@@ -14,7 +14,7 @@
  */
 import { z } from 'zod/v4';
 import { ProjectsSchema } from '../../schema/src/project.js';
-import { EffectiveDeltaSchema, EffectiveEvidenceSchema, EvidencePageSchema, HealthSchema, LayerRowsSchema, CheckDetailSchema, OutlineSchema, ProblemSchema, ReadySchema, ServerInfoSchema, StoreListingSchema, StoreWriteResultSchema, ChecksResultSchema, HelloEventSchema, ReportEventSchema, PingEventSchema, TestCatalogSchema, CatalogHealthSchema, TestHistorySchema, TestReliabilitySchema, RunListSchema, RunDetailSchema, ArtifactListSchema, UiCatalogSchema, ArtifactTagEditSchema, ArtifactTagEntrySchema, ArtifactTagsSchema, ReviewCollectionRequestSchema, ReviewCollectionSchema, ReviewCollectionListSchema, ReviewCollectionDetailSchema, ReviewNoteInputSchema, ReviewNoteSchema, ReviewManifestSchema, RecordingVerificationSchema, RecordingVerifyRequestSchema } from '../../schema/src/api.js';
+import { EffectiveDeltaSchema, EffectiveEvidenceSchema, EvidencePageSchema, HealthSchema, LayerRowsSchema, CheckDetailSchema, OutlineSchema, ProblemSchema, ReadySchema, ServerInfoSchema, StoreListingSchema, StoreWriteResultSchema, ChecksResultSchema, HelloEventSchema, ReportEventSchema, PingEventSchema, TestCatalogSchema, CatalogHealthSchema, TestHistorySchema, TestReliabilitySchema, RunListSchema, RunDetailSchema, ArtifactListSchema, UiCatalogSchema, CatalogEntriesSchema, ArtifactTagEditSchema, ArtifactTagEntrySchema, ArtifactTagsSchema, ReviewCollectionRequestSchema, ReviewCollectionSchema, ReviewCollectionListSchema, ReviewCollectionDetailSchema, ReviewNoteInputSchema, ReviewNoteSchema, ReviewManifestSchema, RecordingVerificationSchema, RecordingVerifyRequestSchema } from '../../schema/src/api.js';
 import { LiveRunSchema, LiveRunStartSchema, LiveRunUpdateSchema, ServerEditableSettingsSchema, ServerSettingsSnapshotSchema, GcPolicySchema, GcSnapshotSchema, GcRunResultSchema, AtlasChangeSchema, PublishChecksSchema, ReportSourcesSchema, ReportStatusSchema, RunsIndexSchema, SpecIndexSchema } from '../../schema/src/index.js';
 /** The version of the API described, `major.minor.patch`: the minor grows when an operation or field is added, the major never within `/api`. */
 export const API_VERSION = '1.7.0';
@@ -39,7 +39,7 @@ const COMPONENTS = {
     ArtifactTagEdit: [ArtifactTagEditSchema, 'input'], ArtifactTagEntry: [ArtifactTagEntrySchema, 'output'], ArtifactTags: [ArtifactTagsSchema, 'output'],
     ReviewCollectionRequest: [ReviewCollectionRequestSchema, 'input'], ReviewCollection: [ReviewCollectionSchema, 'output'], ReviewCollectionList: [ReviewCollectionListSchema, 'output'], ReviewCollectionDetail: [ReviewCollectionDetailSchema, 'output'], ReviewNoteInput: [ReviewNoteInputSchema, 'input'], ReviewNote: [ReviewNoteSchema, 'output'], ReviewManifest: [ReviewManifestSchema, 'output'],
     RecordingVerification: [RecordingVerificationSchema, 'output'], RecordingVerifyRequest: [RecordingVerifyRequestSchema, 'input'],
-    TestCatalog: [TestCatalogSchema, 'output'], CatalogHealth: [CatalogHealthSchema, 'output'], TestHistory: [TestHistorySchema, 'output'], TestReliability: [TestReliabilitySchema, 'output'], RunList: [RunListSchema, 'output'], ArtifactList: [ArtifactListSchema, 'output'], UiCatalog: [UiCatalogSchema, 'output'], RunDetail: [RunDetailSchema, 'output'],
+    TestCatalog: [TestCatalogSchema, 'output'], CatalogHealth: [CatalogHealthSchema, 'output'], TestHistory: [TestHistorySchema, 'output'], TestReliability: [TestReliabilitySchema, 'output'], RunList: [RunListSchema, 'output'], ArtifactList: [ArtifactListSchema, 'output'], UiCatalog: [UiCatalogSchema, 'output'], CatalogEntries: [CatalogEntriesSchema, 'output'], RunDetail: [RunDetailSchema, 'output'],
     SpecIndex: [SpecIndexSchema, 'output'], ReportSources: [ReportSourcesSchema, 'output'], RunsIndex: [RunsIndexSchema, 'output'],
     RunEvent: [LiveRunSchema, 'output'], HelloEvent: [HelloEventSchema, 'output'], ReportEvent: [ReportEventSchema, 'output'], AtlasChangeEvent: [AtlasChangeSchema, 'output'], PingEvent: [PingEventSchema, 'output']
 };
@@ -503,6 +503,23 @@ const BASE_OPERATIONS = [
         ],
         responses: { '200': json('The catalog.', 'UiCatalog'), '304': { description: 'Unchanged since the `If-None-Match` ETag.' }, '401': UNAUTHORIZED, '503': NOT_READY }
     },
+    {
+        route: 'GET /api/reports/ui-catalog/entries', path: '/api/reports/ui-catalog/entries', operationId: 'getUiCatalogEntries', tag: 'Report', role: 'read',
+        summary: 'The UI catalog as a list a critic iterates: captured states with their picture hash and critiques',
+        description: 'The captured states of the UI catalog, one entry per state per profile that has a picture stored in its layer, in key order, for a process that reviews them (a critic: see the critics guide). Page with `limit` and the opaque `cursor` of the previous page (`nextCursor`): the cursor names the last key, so each entry is seen exactly once and in the same order. An entry carries `image.sha256` (the hash a critique is anchored to) and `image.path` (fetch the bytes from `/api/reports` + path, through the same safe attachment path as every file), its `state` kind, and `critiques`: those already made, newest first, each marked `current` or `earlier-image`. Keep the `version` of the first page; later ask `since=<that version>` to read only the entries whose layer was added or changed since. A `since` the server does not remember is answered with the whole catalog and `since.honored: false`. `unjudgedBy=<critic>@<version>` keeps entries with no current critique with a verdict from that critic (a failed critique is not a judgement). To be told that something changed, hold `/api/events`: a `report` event carries the new version; then read `since`. A critic-role token may read this.',
+        parameters: [
+            { name: 'limit', in: 'query', description: 'At most this many entries per page (default 100, at most 1000).', schema: { type: 'integer', minimum: 1, maximum: 1000 } },
+            { name: 'cursor', in: 'query', description: 'The `nextCursor` of the previous page. Anything else is `400`.', schema: str },
+            { name: 'since', in: 'query', description: 'A report version (`version` of an earlier answer): only entries whose layer was added or changed since. An unknown version is answered with everything.', schema: str },
+            { name: 'unjudgedBy', in: 'query', description: '`name` or `name@version` of a critic: only entries it has no current verdict for.', schema: str },
+            { name: 'q', in: 'query', description: 'Words that must all appear in an entry\'s key, title, route or area.', schema: str },
+            { name: 'area', in: 'query', description: 'Areas. Comma-separated.', schema: str },
+            { name: 'profile', in: 'query', description: 'Profile keys, as listed in `profile.key`. Comma-separated.', schema: str },
+            { name: 'state', in: 'query', description: 'State kinds: `screen`, `overlay`, `loading`, `error`, `empty`, `unknown`… Comma-separated.', schema: str },
+            { name: 'route', in: 'query', description: 'Routes. Comma-separated.', schema: str }
+        ],
+        responses: { '200': json('One page of entries.', 'CatalogEntries'), '304': { description: 'Unchanged since the `If-None-Match` ETag.' }, '400': problem('The cursor is not one this server made.'), '401': UNAUTHORIZED, '503': NOT_READY }
+    },
     {
         route: 'GET /api/reports/runs/detail', path: '/api/reports/runs/detail', operationId: 'getReportRun', tag: 'Report', role: 'read',
         summary: 'Every test one run recorded',

package/docs/agents.md

@@ -104,6 +104,8 @@ The complete public export map is generated from the package manifest (wildcards
 @crvouga/atlas/schema
 @crvouga/atlas/server
 @crvouga/atlas/critics
+@crvouga/atlas/critics/uiclip
+@crvouga/atlas/critics/llm
 @crvouga/atlas/reports/*
 @crvouga/atlas/schemas/*
 @crvouga/atlas/openapi.json
@@ -151,7 +153,7 @@ and authentication. Keep synthetic failure checks in a scratch store rather than
 ## What an agent can rely on
 
 **Machine-readable output.** Add `--format json` (alias `--json`) to `atlas init`, `atlas report publish`, `stream`,
-`merge`, `doctor` and `status`. Each prints one JSON document on standard output that validates against a published
+`merge`, `doctor`, `status` and `verify-recordings`, to `atlas ui-catalog`, and to `atlas critic list`, `setup`, `doctor`, `run`, `watch`, `review` and `calibrate`. Each prints one JSON document on standard output that validates against a published
 JSON Schema (`@crvouga/atlas/schemas/<name>.schema.json`, draft 2020-12):
 
 <!-- generated:json-command-schemas -->
@@ -164,6 +166,18 @@ JSON Schema (`@crvouga/atlas/schemas/<name>.schema.json`, draft 2020-12):
 | `atlas report doctor` | `doctor-result` |
 | `atlas report status` | `report-status` |
 | `atlas report verify-recordings` | `verify-recordings-result` |
+| `atlas critic list` | `critic-list-result` |
+| `atlas critic setup` | `critic-setup-result` |
+| `atlas critic doctor` | `critic-doctor-result` |
+| `atlas critic run` | `critic-run-result` |
+| `atlas critic watch` | `critic-watch-result` |
+| `atlas critic review` | `critic-review-result` |
+| `atlas critic calibrate` | `critic-calibrate-result` |
+| `atlas ui-catalog plan` | `ui-catalog-publish-result` |
+| `atlas ui-catalog publish` | `ui-catalog-publish-result` |
+| `atlas ui-catalog status` | `ui-catalog-status` |
+| `atlas ui-catalog diff` | `ui-catalog-diff-result` |
+| `atlas ui-catalog export` | `ui-catalog-export-result` |
 | `a command that was wrong (standard error)` | `command-error` |
 <!-- /generated:json-command-schemas -->
 
@@ -172,7 +186,7 @@ pass, a store that needs attention, checks that could not be sent, a token that
 itself was wrong (nothing was written). With `--format json` a wrong command is `{"ok":false,"error":{"message":…,"code":"usage"}}`
 on standard error.
 
-**No prompts, no interactive steps, no hidden state.** Configuration is flags and environment variables:
+**No prompts, no interactive steps, no hidden state.** (`atlas critic` asks questions only when a person is at a terminal and never with `--non-interactive` or `--format json`.) Configuration is flags and environment variables:
 `ATLAS_STORE` (a folder, `s3://…`, or the server's `https://…`), `ATLAS_TOKEN` (or `ATLAS_TOKEN_FILE`). A token is
 a secret of one server and is never sent over plain `http://` beyond this machine.
 For HTTP stores, `ATLAS_PROJECT` (or `--project <slug>` on report commands) selects the project. Native runner