@crvouga/atlas 1.12.0 → 1.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crvouga/atlas",
-  "version": "1.12.0",
+  "version": "1.13.0",
   "description": "Atlas: specs of any kind, reported as layered evidence. Publish checks from any runner (JUnit, CTRF, a script), combine them across nodes, branches, environments and backends, gate CI on the result, and run statecharts against the real app.",
   "license": "MIT",
   "type": "module",
@@ -85,6 +85,10 @@
       "types": "./dist/server/src/index.d.ts",
       "default": "./dist/server/src/index.js"
     },
+    "./critics": {
+      "types": "./dist/critics/src/index.d.ts",
+      "default": "./dist/critics/src/index.js"
+    },
     "./reports/*": {
       "types": "./dist/reports/src/*.d.ts",
       "default": "./dist/reports/src/*.js"
@@ -155,7 +159,8 @@
     "docs/runners.md",
     "packages/runners/compat",
     "docs/ui.md",
-    "docs/workspaces.md"
+    "docs/workspaces.md",
+    "docs/critics.md"
   ],
   "scripts": {
     "build": "bun scripts/build-package.ts",

package/packages/critics/README.md

@@ -0,0 +1,18 @@
+# @crvouga/atlas/critics
+
+Automated reviewers of the UI catalog that mark which UIs need polish. A critic judges one captured UI through one small
+contract; the runner owns caching, retries, rate limits, timeouts and budgets; opinions are stored as ordinary layers of
+parts of kind `critique` and never change whether a report passes.
+
+Read [the critics guide](../../docs/critics.md). The function-level reference is generated from the code:
+[critics](../../docs/reference/critics.md).
+
+| Module | |
+| --- | --- |
+| `contract` | `defineCritic`, the `Critic` type, `CriticError`, `RateLimitedError`, `InvalidOutputError` |
+| `runner` | `runCritic`: skip what was judged, bounded concurrency, timeout, backoff, budgets, dry run |
+| `critique` | build, read and key critiques; `critiqueLayer`, `readCritique`, `critiqueCurrency` |
+| `publish` | `publishCritiques` |
+| `judging` | `relativeVerdict`, `calibrate`, `gateDecision` (report, warn, gate) |
+| `credentials` | references (`env:NAME`), literal-credential refusal, redaction |
+| `scope` | the rule a critic-role token is held to |

package/packages/schema/schemas/critique.schema.json

@@ -0,0 +1,254 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/critique.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "schema": {
+      "type": "integer",
+      "exclusiveMinimum": 0,
+      "maximum": 9007199254740991,
+      "description": "The version of this extension. 1"
+    },
+    "subject": {
+      "type": "object",
+      "properties": {
+        "entryKey": {
+          "type": "string",
+          "minLength": 1,
+          "description": "The catalog entry that was judged"
+        },
+        "imageSha256": {
+          "type": "string",
+          "minLength": 1,
+          "description": "The hash of the exact image bytes that were judged. When the entry's image changes, this critique judged an earlier image"
+        },
+        "state": {
+          "description": "The kind of UI state: `screen`, `overlay`, `loading`, `error`, `empty`…",
+          "type": "string",
+          "minLength": 1
+        }
+      },
+      "required": [
+        "entryKey",
+        "imageSha256"
+      ],
+      "additionalProperties": {}
+    },
+    "critic": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string",
+          "minLength": 1
+        },
+        "version": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Of the critic's own logic. A newer version replaces an older one"
+        },
+        "kind": {
+          "description": "Open vocabulary: `model`, `llm`, `ml`, `heuristic`…",
+          "type": "string",
+          "minLength": 1
+        }
+      },
+      "required": [
+        "name",
+        "version"
+      ],
+      "additionalProperties": {}
+    },
+    "model": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "provider": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "id"
+      ],
+      "additionalProperties": {}
+    },
+    "parameters": {
+      "description": "Everything else that explains the result: temperature, seed, the SHA-256 of the rubric",
+      "type": "object",
+      "propertyNames": {
+        "type": "string"
+      },
+      "additionalProperties": {}
+    },
+    "verdict": {
+      "type": "object",
+      "properties": {
+        "score": {
+          "description": "A finite number. `scale` says what it means",
+          "type": "number"
+        },
+        "scale": {
+          "description": "For example `0..1, higher is better`",
+          "type": "string"
+        },
+        "label": {
+          "description": "Open vocabulary, conventionally `ok` or `needs-polish`",
+          "type": "string"
+        },
+        "confidence": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "relative": {
+          "type": "object",
+          "properties": {
+            "stateKind": {
+              "type": "string",
+              "minLength": 1,
+              "description": "The kind of UI state the score is relative to: `screen`, `overlay`, `loading`, `error`, `empty`…"
+            },
+            "percentile": {
+              "description": "Where the score falls among the project's scores of the same state kind, 0..100, higher is better",
+              "type": "number",
+              "minimum": 0,
+              "maximum": 100
+            },
+            "sampleSize": {
+              "description": "How many scores of that state kind the percentile is over",
+              "type": "integer",
+              "minimum": 0,
+              "maximum": 9007199254740991
+            },
+            "previousScore": {
+              "description": "The same entry's score for its previous image, from the same critic",
+              "type": "number"
+            },
+            "delta": {
+              "description": "`score` minus `previousScore`",
+              "type": "number"
+            }
+          },
+          "required": [
+            "stateKind"
+          ],
+          "additionalProperties": {}
+        }
+      },
+      "additionalProperties": {}
+    },
+    "findings": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "description": "Stable within the critique, so a reviewer can mark this finding valid or not",
+            "type": "string",
+            "minLength": 1
+          },
+          "category": {
+            "description": "Open vocabulary: `spacing`, `contrast`, `alignment`, `overflow`, `hierarchy`…",
+            "type": "string",
+            "minLength": 1
+          },
+          "severity": {
+            "description": "Open vocabulary, conventionally `info`, `minor`, `major`",
+            "type": "string",
+            "minLength": 1
+          },
+          "confidence": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 1
+          },
+          "region": {
+            "type": "object",
+            "properties": {
+              "unit": {
+                "type": "string",
+                "minLength": 1,
+                "description": "`px` (pixels of the stored image) or `normalized` (0..1 of its width and height). Open vocabulary"
+              },
+              "x": {
+                "type": "number"
+              },
+              "y": {
+                "type": "number"
+              },
+              "w": {
+                "type": "number",
+                "minimum": 0
+              },
+              "h": {
+                "type": "number",
+                "minimum": 0
+              }
+            },
+            "required": [
+              "unit",
+              "x",
+              "y",
+              "w",
+              "h"
+            ],
+            "additionalProperties": {}
+          },
+          "rationale": {
+            "type": "string"
+          },
+          "suggestion": {
+            "type": "string"
+          }
+        },
+        "additionalProperties": {}
+      }
+    },
+    "failure": {
+      "type": "object",
+      "properties": {
+        "reason": {
+          "type": "string",
+          "minLength": 1
+        },
+        "message": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "reason"
+      ],
+      "additionalProperties": {}
+    },
+    "usage": {
+      "type": "object",
+      "properties": {
+        "inputTokens": {
+          "type": "number",
+          "minimum": 0
+        },
+        "outputTokens": {
+          "type": "number",
+          "minimum": 0
+        },
+        "latencyMs": {
+          "type": "number",
+          "minimum": 0
+        },
+        "costUsd": {
+          "type": "number",
+          "minimum": 0
+        }
+      },
+      "additionalProperties": {}
+    }
+  },
+  "required": [
+    "schema",
+    "subject",
+    "critic"
+  ],
+  "additionalProperties": {}
+}

package/packages/schema/schemas/server-config.schema.json

@@ -40,10 +40,11 @@
                 "type": "string",
                 "enum": [
                   "read",
+                  "critic",
                   "publish",
                   "admin"
                 ],
-                "description": "`read` may look. `publish` may also upload. `admin` may also remove scratch objects"
+                "description": "`read` may look. `critic` may also publish critiques (layers of producer `critic.*` whose parts are all of kind `critique`). `publish` may also upload. `admin` may also remove scratch objects"
               },
               "token": {
                 "description": "The token. Prefer `tokenEnv`, so the file holds no secret",

package/packages/schema/schemas/server-info.schema.json

@@ -18,6 +18,7 @@
       "type": "string",
       "enum": [
         "read",
+        "critic",
         "publish",
         "admin"
       ],

package/packages/server/openapi.json

@@ -2,9 +2,9 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Atlas server API",
-    "version": "1.5.0",
+    "version": "1.6.0",
     "summary": "Take in test results from every environment, keep them, and show everyone the combined report, live.",
-    "description": "One Atlas node serves multiple projects. Clients select a project using ?project=<slug>, X-Atlas-Project, or /api/projects/<slug>/… . Each project has independent specs, layers, recordings, status, and event streams. Within each project, the newest completed result of every check wins its slot. Unqualified requests address the legacy default project.\n\n**Authentication.** With tokens configured, send `Authorization: Bearer <token>`. Roles: `read` < `publish` < `admin`. Uploading needs a Bearer token; a browser's HTTP Basic sign-in (any user name, the token as the password) is accepted for reading only. A server with no tokens is open and listens on this machine only unless told otherwise.\n\n**Errors** are [RFC 9457](https://www.rfc-editor.org/rfc/rfc9457) problem documents (`application/problem+json`) with a `detail` that says what to do. `401` send a token, `403` that token may not, `409` upload what this depends on first or you are changing something immutable, `413` too large, `422` not valid, `503` still starting (retry after `Retry-After`).\n\n**Compatibility.** Within `/api` the API only grows: optional parameters, optional response fields, new events. Ignore what you do not know.\n\nGuides: `/llms.txt`. Source: https://github.com/crvouga/atlas.",
+    "description": "One Atlas node serves multiple projects. Clients select a project using ?project=<slug>, X-Atlas-Project, or /api/projects/<slug>/… . Each project has independent specs, layers, recordings, status, and event streams. Within each project, the newest completed result of every check wins its slot. Unqualified requests address the legacy default project.\n\n**Authentication.** With tokens configured, send `Authorization: Bearer <token>`. Roles: `read` < `critic` < `publish` < `admin`. A `critic` token may read, and publish critiques (layers of automated reviewers) and nothing else. Uploading needs a Bearer token; a browser's HTTP Basic sign-in (any user name, the token as the password) is accepted for reading only. A server with no tokens is open and listens on this machine only unless told otherwise.\n\n**Errors** are [RFC 9457](https://www.rfc-editor.org/rfc/rfc9457) problem documents (`application/problem+json`) with a `detail` that says what to do. `401` send a token, `403` that token may not, `409` upload what this depends on first or you are changing something immutable, `413` too large, `422` not valid, `503` still starting (retry after `Retry-After`).\n\n**Compatibility.** Within `/api` the API only grows: optional parameters, optional response fields, new events. Ignore what you do not know.\n\nGuides: `/llms.txt`. Source: https://github.com/crvouga/atlas.",
     "license": {
       "name": "MIT",
       "identifier": "MIT"
@@ -2683,8 +2683,8 @@
           "Store protocol"
         ],
         "summary": "Upload an object",
-        "description": "The write half of the `ObjectStore` interface. The server checks what it can from the bytes in hand and says no to what would corrupt the report for everyone:\n\n- a **layer manifest** (`layers/manifests/<id>.json`) must be a valid layer whose content hashes to `<id>`, and its blobs must already be there (`409` otherwise);\n- a **blob** (`layers/blobs/<sha256>.gz`) must be gzip of content whose SHA-256 is its name; a **spec file** likewise;\n- a **catalog entry** (`…/index/…`) must name something already uploaded (`409` otherwise): it is what makes a layer visible, so it is written last;\n- a **run manifest** must have passed its privacy scan;\n- keys are immutable: the same bytes again are `200` (a retry), other bytes under the same key are `409`.\n\nUpload in dependency order: blobs, then the manifest, then the catalog entry. `atlas report publish`, `atlas report stream` and `httpStore` do. Needs a `Bearer` token of role `publish` or `admin`; a browser's sign-in box is refused.\n\nRole needed: `publish`.",
-        "x-atlas-role": "publish",
+        "description": "The write half of the `ObjectStore` interface. The server checks what it can from the bytes in hand and says no to what would corrupt the report for everyone:\n\n- a **layer manifest** (`layers/manifests/<id>.json`) must be a valid layer whose content hashes to `<id>`, and its blobs must already be there (`409` otherwise);\n- a **blob** (`layers/blobs/<sha256>.gz`) must be gzip of content whose SHA-256 is its name; a **spec file** likewise;\n- a **catalog entry** (`…/index/…`) must name something already uploaded (`409` otherwise): it is what makes a layer visible, so it is written last;\n- a **run manifest** must have passed its privacy scan;\n- keys are immutable: the same bytes again are `200` (a retry), other bytes under the same key are `409`.\n\nUpload in dependency order: blobs, then the manifest, then the catalog entry. `atlas report publish`, `atlas report stream` and `httpStore` do. Needs a `Bearer` token of role `publish` or `admin`; a browser's sign-in box is refused.\n\nA token of role `critic` may upload only what a critic publishes: blobs, and layers (and their catalog entries) whose `producer` is `critic.<name>` and whose parts are all of kind `critique`. Anything else is `403` with the rule it broke, and nothing is stored.\n\nRole needed: `critic`.",
+        "x-atlas-role": "critic",
         "security": [
           {
             "bearer": []
@@ -3029,8 +3029,8 @@
           "Store protocol"
         ],
         "summary": "Upload an object",
-        "description": "The write half of the `ObjectStore` interface. The server checks what it can from the bytes in hand and says no to what would corrupt the report for everyone:\n\n- a **layer manifest** (`layers/manifests/<id>.json`) must be a valid layer whose content hashes to `<id>`, and its blobs must already be there (`409` otherwise);\n- a **blob** (`layers/blobs/<sha256>.gz`) must be gzip of content whose SHA-256 is its name; a **spec file** likewise;\n- a **catalog entry** (`…/index/…`) must name something already uploaded (`409` otherwise): it is what makes a layer visible, so it is written last;\n- a **run manifest** must have passed its privacy scan;\n- keys are immutable: the same bytes again are `200` (a retry), other bytes under the same key are `409`.\n\nUpload in dependency order: blobs, then the manifest, then the catalog entry. `atlas report publish`, `atlas report stream` and `httpStore` do. Needs a `Bearer` token of role `publish` or `admin`; a browser's sign-in box is refused.\n\nScoped to one project. The same project may be selected on the unprefixed API using ?project= or X-Atlas-Project. Unknown projects return 404 on reads and are created by an authenticated upload.\n\nRole needed: `publish`.",
-        "x-atlas-role": "publish",
+        "description": "The write half of the `ObjectStore` interface. The server checks what it can from the bytes in hand and says no to what would corrupt the report for everyone:\n\n- a **layer manifest** (`layers/manifests/<id>.json`) must be a valid layer whose content hashes to `<id>`, and its blobs must already be there (`409` otherwise);\n- a **blob** (`layers/blobs/<sha256>.gz`) must be gzip of content whose SHA-256 is its name; a **spec file** likewise;\n- a **catalog entry** (`…/index/…`) must name something already uploaded (`409` otherwise): it is what makes a layer visible, so it is written last;\n- a **run manifest** must have passed its privacy scan;\n- keys are immutable: the same bytes again are `200` (a retry), other bytes under the same key are `409`.\n\nUpload in dependency order: blobs, then the manifest, then the catalog entry. `atlas report publish`, `atlas report stream` and `httpStore` do. Needs a `Bearer` token of role `publish` or `admin`; a browser's sign-in box is refused.\n\nA token of role `critic` may upload only what a critic publishes: blobs, and layers (and their catalog entries) whose `producer` is `critic.<name>` and whose parts are all of kind `critique`. Anything else is `403` with the rule it broke, and nothing is stored.\n\nScoped to one project. The same project may be selected on the unprefixed API using ?project= or X-Atlas-Project. Unknown projects return 404 on reads and are created by an authenticated upload.\n\nRole needed: `critic`.",
+        "x-atlas-role": "critic",
         "security": [
           {
             "bearer": []
@@ -10011,6 +10011,7 @@
             "type": "string",
             "enum": [
               "read",
+              "critic",
               "publish",
               "admin"
             ],