@crvouga/atlas 1.20.0 → 1.22.0

package/llms.txt

@@ -48,7 +48,7 @@ Given only the repository link, start at docs/agents.md: install `@crvouga/atlas
 - [TypeScript API: detox](https://raw.githubusercontent.com/crvouga/atlas/main/docs/reference/detox.md): The Detox driver for statechart runs of React Native apps.
 - [TypeScript API: runners](https://raw.githubusercontent.com/crvouga/atlas/main/docs/reference/runners.md): Native reporters that upload to the project server.
 - [TypeScript API: vitest](https://raw.githubusercontent.com/crvouga/atlas/main/docs/reference/vitest.md): Run a statechart plan as Vitest tests.
-- [TypeScript API: critics](https://raw.githubusercontent.com/crvouga/atlas/main/docs/reference/critics.md): Automated reviewers of the UI catalog: the critic contract, the runner (caching, retries, rate limits, budgets), credential references and redaction, the `atlas.critique` evidence as layers, calibration and report, warn and gate modes.
+- [TypeScript API: critics](https://raw.githubusercontent.com/crvouga/atlas/main/docs/reference/critics.md): Automated reviewers of the UI catalog: the critic contract, the runner (caching, retries, rate limits, budgets), credential references and redaction, the `atlas.critique` evidence as layers, the catalog client, spooling, calibration and report, warn and gate modes, accept and dismiss marks by reviewers, the `atlas critic` command (list, setup, doctor, run, watch, review, calibrate), and the UIClip and LLM critic adapters (OpenAI-compatible, Anthropic, Gemini).
 - [TypeScript API: playwright-reporter](https://raw.githubusercontent.com/crvouga/atlas/main/docs/reference/playwright-reporter.md): A Playwright Test reporter that streams every test to Atlas with its duration, device labels, steps, retries, videos, screenshots and traces.
 - [JSON Schemas](https://unpkg.com/@crvouga/atlas/packages/schema/schemas/): the layer format, checks file, report status, command results, server responses, server configuration.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crvouga/atlas",
-  "version": "1.20.0",
+  "version": "1.22.0",
   "description": "Atlas: specs of any kind, reported as layered evidence. Publish checks from any runner (JUnit, CTRF, a script), combine them across nodes, branches, environments and backends, gate CI on the result, and run statecharts against the real app.",
   "license": "MIT",
   "type": "module",
@@ -89,6 +89,14 @@
       "types": "./dist/critics/src/index.d.ts",
       "default": "./dist/critics/src/index.js"
     },
+    "./critics/uiclip": {
+      "types": "./dist/critics/src/uiclip.d.ts",
+      "default": "./dist/critics/src/uiclip.js"
+    },
+    "./critics/llm": {
+      "types": "./dist/critics/src/llm.d.ts",
+      "default": "./dist/critics/src/llm.js"
+    },
     "./reports/*": {
       "types": "./dist/reports/src/*.d.ts",
       "default": "./dist/reports/src/*.js"

package/packages/critics/README.md

@@ -13,6 +13,7 @@ Read [the critics guide](../../docs/critics.md). The function-level reference is
 | `runner` | `runCritic`: skip what was judged, bounded concurrency, timeout, backoff, budgets, dry run |
 | `critique` | build, read and key critiques; `critiqueLayer`, `readCritique`, `critiqueCurrency` |
 | `publish` | `publishCritiques` |
+| `marks` | `markLayer`, `publishMarks`: a reviewer's accept or dismiss of an opinion, as a layer |
 | `judging` | `relativeVerdict`, `calibrate`, `gateDecision` (report, warn, gate) |
 | `credentials` | references (`env:NAME`), literal-credential refusal, redaction |
 | `scope` | the rule a critic-role token is held to |

package/packages/schema/schemas/critic-calibrate-result.schema.json

@@ -0,0 +1,274 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/critic-calibrate-result.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "ok": {
+      "type": "boolean",
+      "const": true
+    },
+    "dryRun": {
+      "type": "boolean"
+    },
+    "critic": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": [
+            "string",
+            "null"
+          ]
+        }
+      },
+      "required": [
+        "name",
+        "version"
+      ],
+      "additionalProperties": {}
+    },
+    "sendsImagesTo": {
+      "description": "Where the labelled pictures were (or, on a dry run, would be) sent; null when they stay on this machine or only marks were used",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "examples": {
+      "type": "object",
+      "properties": {
+        "files": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991,
+          "description": "Examples from labelled pictures (on a dry run, the pictures found)"
+        },
+        "marks": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991,
+          "description": "Examples from reviewers' whole-opinion marks"
+        },
+        "used": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "skipped": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "source": {
+                "type": "string"
+              },
+              "reason": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "source",
+              "reason"
+            ],
+            "additionalProperties": {}
+          },
+          "description": "Examples left out, and why: no score, a critic failure, disputed marks, a budget"
+        }
+      },
+      "required": [
+        "files",
+        "marks",
+        "used",
+        "skipped"
+      ],
+      "additionalProperties": {}
+    },
+    "calibration": {
+      "anyOf": [
+        {
+          "type": "object",
+          "properties": {
+            "schema": {
+              "type": "number",
+              "const": 1
+            },
+            "critic": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "version": {
+                  "type": "string"
+                },
+                "model": {
+                  "type": "object",
+                  "properties": {
+                    "id": {
+                      "type": "string"
+                    },
+                    "provider": {
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "id"
+                  ],
+                  "additionalProperties": {}
+                },
+                "parameters": {
+                  "type": "object",
+                  "propertyNames": {
+                    "type": "string"
+                  },
+                  "additionalProperties": {}
+                }
+              },
+              "required": [
+                "name",
+                "version"
+              ],
+              "additionalProperties": {}
+            },
+            "createdAtMs": {
+              "type": "number"
+            },
+            "kinds": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "stateKind": {
+                    "type": "string"
+                  },
+                  "threshold": {
+                    "description": "Scores below this are flagged as needing polish; null when the examples cannot tell (one class is missing)",
+                    "type": [
+                      "number",
+                      "null"
+                    ]
+                  },
+                  "samples": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "positives": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "negatives": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  "precision": {
+                    "type": [
+                      "number",
+                      "null"
+                    ]
+                  },
+                  "recall": {
+                    "type": [
+                      "number",
+                      "null"
+                    ]
+                  },
+                  "inSample": {
+                    "type": "boolean",
+                    "description": "True when precision and recall are measured on the examples the threshold was chosen on, which flatters them"
+                  },
+                  "evaluatedOn": {
+                    "type": "string",
+                    "enum": [
+                      "fit",
+                      "held-out"
+                    ]
+                  }
+                },
+                "required": [
+                  "stateKind",
+                  "threshold",
+                  "samples",
+                  "positives",
+                  "negatives",
+                  "precision",
+                  "recall",
+                  "inSample",
+                  "evaluatedOn"
+                ],
+                "additionalProperties": {}
+              }
+            }
+          },
+          "required": [
+            "schema",
+            "critic",
+            "kinds"
+          ],
+          "additionalProperties": {}
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "The calibration `atlas critic run --calibration` reads; null on a dry run"
+    },
+    "written": {
+      "description": "The file it was written to (`--out`), or null",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "gate": {
+      "anyOf": [
+        {
+          "type": "object",
+          "properties": {
+            "eligible": {
+              "type": "boolean",
+              "description": "Whether `--mode gate` would accept this calibration"
+            },
+            "reasons": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "What is missing, per state kind, when it would not"
+            }
+          },
+          "required": [
+            "eligible",
+            "reasons"
+          ],
+          "additionalProperties": {}
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "warnings": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    }
+  },
+  "required": [
+    "ok",
+    "dryRun",
+    "critic",
+    "sendsImagesTo",
+    "examples",
+    "calibration",
+    "written",
+    "gate",
+    "warnings"
+  ],
+  "additionalProperties": {}
+}

package/packages/schema/schemas/critic-doctor-result.schema.json

@@ -0,0 +1,74 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/critic-doctor-result.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "ok": {
+      "type": "boolean",
+      "description": "False when any check is a `problem` (the exit code is then 1)"
+    },
+    "server": {
+      "description": "The server that was checked, with no secrets",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "role": {
+      "description": "What the token may do: `read`, `critic`, `publish` or `admin`; null when the server could not be asked",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "open": {
+      "description": "True when the server has no tokens, so every caller is an administrator; null when unknown",
+      "type": [
+        "boolean",
+        "null"
+      ]
+    },
+    "checks": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "`server`, `token`, `scope`, `catalog`, `spool`, or `<critic>:<aspect>` such as `uiclip:model-file`"
+          },
+          "status": {
+            "type": "string",
+            "enum": [
+              "ok",
+              "warning",
+              "problem"
+            ],
+            "description": "`problem` makes the exit code 1; `warning` does not"
+          },
+          "detail": {
+            "type": "string"
+          },
+          "fix": {
+            "description": "What to change, when the status is not `ok`",
+            "type": "string"
+          }
+        },
+        "required": [
+          "name",
+          "status",
+          "detail"
+        ],
+        "additionalProperties": {}
+      }
+    }
+  },
+  "required": [
+    "ok",
+    "server",
+    "role",
+    "open",
+    "checks"
+  ],
+  "additionalProperties": {}
+}

package/packages/schema/schemas/critic-list-result.schema.json

@@ -0,0 +1,90 @@
+{
+  "$id": "https://github.com/crvouga/atlas/schemas/critic-list-result.schema.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "ok": {
+      "type": "boolean",
+      "const": true
+    },
+    "config": {
+      "description": "The configuration file that was read, or null when there is none",
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "critics": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "The name in the configuration; also the producer suffix (`critic.<name>`)"
+          },
+          "type": {
+            "type": "string",
+            "description": "`uiclip`, `llm`, `module`, or `not-configured` for a built-in critic that has no configuration yet"
+          },
+          "kind": {
+            "description": "`model`, `llm`, `heuristic`… as the critic says; null when it is not configured",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "version": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "status": {
+            "type": "string",
+            "enum": [
+              "ready",
+              "needs-setup",
+              "misconfigured",
+              "not-configured"
+            ],
+            "description": "`needs-setup`: run `atlas critic setup <name>`. `misconfigured`: the configuration or a credential reference cannot be used (see `detail`)"
+          },
+          "sendsImagesTo": {
+            "description": "Where images are sent: a host, or null when they never leave this machine. A run that sends images to a third party needs confirmation",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "model": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "detail": {
+            "description": "Why it is not ready, and what to do. Never holds a credential",
+            "type": "string"
+          }
+        },
+        "required": [
+          "name",
+          "type",
+          "kind",
+          "version",
+          "status",
+          "sendsImagesTo",
+          "model"
+        ],
+        "additionalProperties": {}
+      }
+    }
+  },
+  "required": [
+    "ok",
+    "config",
+    "critics"
+  ],
+  "additionalProperties": {}
+}