@cef-ai/eval 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,203 @@
-# Temporary Holding Version
+# @cef-ai/eval
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Evaluations for CEF workflows: build datasets of cases from scratch or from real
+runs, run them as experiments against a workflow version, and compare results and
+run metrics against a baseline. Warn-only — nothing here blocks a deploy.
+
+Browser-safe (no Node built-ins). ROC and the `cef eval` CLI use the same
+library over the same storage layout, so either reads what the other wrote.
+
+## Vocabulary
+
+| Term | What it is |
+| --- | --- |
+| **Dataset** | The set of cases for one workflow; versioned. Names are unique per workflow. |
+| **Case** | One `workflow.start` payload (`input`) and what the run's Result should be (`expected`). |
+| **Version** | A frozen cut of a dataset: every live case pinned at one revision by sha256. |
+| **Experiment** | One run of a dataset version on a workflow version. |
+| **Repeats** | How many times each case runs in an experiment (default 1). |
+| **Run** | One case × one repeat inside an experiment — a `RunRecord` with output, scores and metrics. |
+| **Baseline** | The experiment others are compared to (`Header.baselineId`, `setBaseline`). |
+| **Result** | What a workflow declares as its output: an `output` node's `params.result`. A run's `workflow.completed` carries exactly those fields; that is what cases are scored against. |
+
+## Declaring a Result
+
+```jsonc
+{ "id": "result", "kind": "output", "params": {
+  "result": { "category": "={{ $json.category }}", "confidence": "={{ $json.score }}" },
+  "resultTypes": { "category": "string", "confidence": "number" },
+  "outcome": "={{ $json.category }}"
+} }
+```
+
+## Cases and scoring
+
+```json
+{
+  "id": "refund-angry",
+  "input": { "text": "I want my money back NOW" },
+  "expected": {
+    "category": "billing",
+    "confidence": ">= 0.8",
+    "tags": { "$contains": "refund" },
+    "reply.language": { "$oneOf": ["en", "en-GB"] }
+  },
+  "limits": { "maxDurationMs": 20000 }
+}
+```
+
+One score per expected field (`eval/field@1`), addressed by dot path. A plain
+value is deep equality (an object is a subset match), a string like `">= 0.8"`
+is a numeric band, and matchers are `$eq $oneOf $contains $regex $between $gte
+$lte $exists $approx/$tol`. A matcher naming several operators needs all of them
+to hold (`{ "$exists": true, "$gte": 1 }`); `$between` is `[min, max]` with
+min ≤ max. A field the output does not have fails. A case passes
+when every judged field score passes; with no expectations it is unscored. A run that
+failed or timed out fails. The summary's `passRate` (shown as **Accuracy**) and
+`fields` (per expected field: passed / failed / notApplicable / rate) roll the
+scores up. Limits (`maxDurationMs maxTokens maxSteps maxCost`,
+header-wide or per case) are scored as `eval/limit@1`, counted as breaches, and
+never fail a case.
+
+**`$regex` and trust.** Patterns run in JavaScript's backtracking engine,
+synchronously, inside scoring — nothing can time one out, so a catastrophic
+pattern freezes the CLI or the ROC tab. Cases come from the shared bucket, so
+scoring refuses (fails the field with `unsafe $regex refused: …`) a pattern
+longer than 256 characters or one that quantifies a group which itself contains
+a quantifier (`(a+)+`, `(\w+\s?)*`). That is a heuristic, not a proof — an
+overlapping alternation such as `(a|a)*` passes it. Treat write access to a
+dataset like write access to the workflow it tests: anyone who can write cases
+can make scoring slow.
+
+## Scoring against the version's Result
+
+An experiment records the Result the workflow version declares
+(`Experiment.resultFields`, from `resultFieldsFromGraph(graph)` — the graph
+object, its JSON text, or a manifest's `params.graph`; several `output` nodes
+are unioned). An expected field that Result does not declare is scored n/a
+(`pass: null`, "not in this version's Result"), and one whose expectation cannot
+match the declared type is n/a too ("type changed: expected number, Result
+declares string") — the dataset is older than the version, which is not a
+regression. `checkResultCompatibility(resultFields, cases)` reports the same up
+front: `missing`, `typeChanged`, and `unexpectedNew` (declared, never expected).
+
+## Storage
+
+On an injected `EvalStore` (`list`, `get`, `put`, `putIfAbsent`; run
+`storeConformance` against an implementation). In the agent service's bucket:
+
+```
+workflows/<wf>/datasets/<dataset>/header.json                     Header (+ baselineId)
+workflows/<wf>/datasets/<dataset>/cases/<caseId>/r<n>.json        write-once revisions
+workflows/<wf>/datasets/<dataset>/cases/<caseId>/deleted.json     tombstone
+workflows/<wf>/datasets/<dataset>/archived/<caseId>.json          archive flag
+workflows/<wf>/datasets/<dataset>/versions/v<n>.json              {version, header, cases:[{id,rev,sha256}], createdAt, note?}
+workflows/<wf>/experiments/<expId>/meta.json                      Experiment (dataset@version, workflowVersion, resultFields, status, summary)
+workflows/<wf>/experiments/<expId>/runs/<caseId>.<repeat>.json    RunRecord
+workflows/<wf>/experiments/<expId>/attempts/a<n>.json             attempt claim (write-once)
+```
+
+Experiments are siblings of the datasets: an experiment names the
+`dataset@version` it ran. Every dataset function takes `(store, workflowId,
+dataset, …)`, every experiment function `(store, workflowId, expId, …)`.
+
+Storage written by the first release (`evals/datasets/<dataset>/…`, experiments
+inside the dataset) is copied into this layout by `migrateLegacyLayout(store)`
+(`cef eval migrate`): `putIfAbsent` per key, legacy keys left in place, a
+`evals/MIGRATED.json` marker written once every dataset was copied.
+`needsLegacyMigration(store)` says whether to run it.
+
+## Dataset versions
+
+Versions are cut automatically: `ensureDatasetVersion(store, wf, dataset)`
+returns the latest version when the working set (live cases at their latest
+revisions) is what it pins, and otherwise cuts the next one with the note
+`auto: <n> added, <m> edited, <k> removed`. `runExperiment` calls it when no
+`datasetVersion` is given. `cutVersion` remains for explicit cuts (`cef eval
+push --cut`); `versionHistory` lists each version with what changed and how
+many experiments ran it.
+
+`compareExperiments` compares only the cases present on both sides at the same
+revision (`comparedOn`), and reports the rest as `datasetDiff: {added, removed,
+edited}`.
+
+## Running experiments
+
+```ts
+import { runExperiment, resultFieldsFromGraph } from "@cef-ai/eval";
+
+const exp = await runExperiment({
+  store, workflowId: "ticket-triage", dataset: "triage", // datasetVersion: 3 — default: ensureDatasetVersion
+  workflowVersion: "1.4.0", live: false,
+  resultFields: resultFieldsFromGraph(graph),
+  repeats: 3, concurrency: 4, timeoutMs: 120_000,
+  target,                      // WorkflowTarget: start / waitForEnd / metrics / pinVersion?
+  onProgress: ({ done, total }) => console.log(done, total),
+});
+```
+
+Each run publishes into its own context, `eval-<expId>-<caseId>-<repeat>-a<attempt>`.
+A non-live version is run through `target.pinVersion`, which routes only this
+experiment's contexts (prefix `eval-<expId>-`) to it. The meta is written first,
+each run record as it finishes, the summary last; pass `experimentId` to resume.
+A resume is a new attempt (`attempt` on the meta and on each run it writes), so
+a run the dead attempt had already started never shares a context with its
+replacement. The attempt is claimed write-once (`putIfAbsent` on
+`experiments/<expId>/attempts/a<n>.json`, n upward until the claim sticks)
+before anything is published, so two concurrent resumes of one experiment get
+different attempts and never publish into the same contexts.
+
+## CLI walkthrough
+
+```sh
+export CEF_AS_PUBKEY=0x…            # the agent service; names the bucket
+export CEF_KEYSTORE=~/owner.json    # owner wallet (or CEF_SECRET_PHRASE)
+export CEF_KEYSTORE_PASSWORD=…
+export CEF_ACCESS_TOKEN=…           # CLI access token from ROC, for the orchestrator
+
+# In the workflow's folder: --workflow defaults to the workflow cef.config.ts
+# (or dist/) declares, --dir to ./datasets.
+cef eval create triage --max-duration-ms 30000
+cef eval pull triage                # → datasets/triage/{header.json,cases/*.json,.versions.json}
+$EDITOR datasets/triage/cases/refund-angry.json
+cef eval push triage                # new revisions; --cut "note" cuts a version explicitly
+cef eval run triage                 # latest dataset version (cut if the cases changed) × the live workflow version
+cef eval run triage --workflow-version 1.5.0 --repeats 3 --baseline exp-mgfz8c1k-a1b2
+cef eval run triage --dataset-version 2   # an older dataset version
+cef eval experiments triage
+cef eval compare exp-mgfz8c1k-a1b2 exp-mgg0p2xq-9zt4
+cef eval migrate                    # once, for a bucket written by the first release
+```
+
+The wallet's S3 credential is minted once, kept for its hour in
+`~/.cef/eval-credentials.json` (owner-only; `CEF_EVAL_CREDENTIAL_CACHE=<file>`
+moves it, `=off` disables it) so each command does not mint another against the
+gateway's per-wallet cap, re-minted before it expires during a long run, and
+re-minted once if the gateway refuses it.
+
+Store credentials are taken in this order: explicit `--access-key`/`--secret`
+(`$CEF_EVAL_KEY`/`$CEF_EVAL_SECRET`, an S3 key issued for the owner's wallet);
+then the owner's wallet, minting as above. `$CEF_DDC_ACCESS_TOKEN`, the token
+`cef push` uses, cannot open dataset storage: it is limited to the agent
+service's registry bucket, while datasets live in the gateway bucket
+`as-<pubkey>`, a separate DDC bucket, and it cannot be delegated to the gateway.
+With only that token set, `cef eval` says so and names the two options above.
+
+`run` records the version's Result — the live deployment's graph when the
+version is live, else the project's own graph of exactly that version — and
+warns before starting about cases expecting fields it no longer returns. The
+baseline stays on the platform: `pull` leaves it out of `header.json`, `push`
+keeps the bucket's, and `run` compares against it when `--baseline` is not given.
+
+Every command takes `--json`. `push` writes a new revision only for a case that
+changed and refuses one edited in the bucket since the last pull (`--force` to
+write over it). A repo case that is deleted or archived in the bucket is refused
+before anything is written; `--restore` undeletes/unarchives it and writes the
+repo's version (`--force` does not). `pull` and `push --prune` only remove what
+the last pull wrote (`.versions.json`): a case file the pull never wrote is
+reported as local-only and kept, and a case added in the bucket after the pull
+is never pruned. `pull` records each case's content hash in `.versions.json` and
+refuses, before writing anything, to overwrite or remove a case file edited since
+the last pull/push — one matching neither that hash nor the bucket's current
+content — unless `--force`. Both `pull` and `push` refuse a dataset dir whose
+`header.json` names another workflow.

package/dist/capture.d.ts

@@ -0,0 +1,37 @@
+import type { Case, Json, Matcher } from "./types.js";
+export interface CapturedRun {
+    /** The `workflow.start` payload. An inline `graph` is dropped. */
+    input: Record<string, Json>;
+    /** `workflow.completed`'s `output`; undefined or null when the run did not complete. */
+    output?: Json | null;
+    /** The run's context. */
+    context: string;
+    workflowVersion?: string;
+    /** Case id; derived from the context when absent. */
+    id?: string;
+    capturedAt?: string;
+    tags?: string[];
+    notes?: string;
+}
+/** A case id from free text: `[A-Za-z0-9._-]`, at most 80 characters, never empty. */
+export declare function slugCaseId(text: string): string;
+/** `base`, or `base-2`, `base-3`… — the first one not in `taken`. */
+export declare function uniqueCaseId(base: string, taken: Iterable<string>): string;
+/**
+ * A case from a captured run. `expected` is the run's Result, field by field,
+ * when it is an object; a non-object output cannot be keyed by field, so it is
+ * left out and the person states the expectation themselves.
+ */
+/**
+ * Whether a captured value is free text a model wrote — a sentence, not a label.
+ *
+ * Pinning prose with an exact match makes every later experiment fail on a
+ * harmless rewording, which buries the regressions the dataset exists to show.
+ * So a captured run expects such a field to be PRESENT; a person tightens it
+ * (contains, regex, or a judge) when the wording itself matters.
+ */
+export declare function isFreeText(v: unknown): v is string;
+/** What a captured output should be expected to be: labels exactly, prose present. */
+export declare function expectationFrom(output: Record<string, unknown>): Record<string, Json | Matcher>;
+export declare function caseFromRun(run: CapturedRun): Case;
+//# sourceMappingURL=capture.d.ts.map

package/dist/capture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"capture.d.ts","sourceRoot":"","sources":["../src/capture.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAEtD,MAAM,WAAW,WAAW;IAC1B,kEAAkE;IAClE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC5B,wFAAwF;IACxF,MAAM,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IACrB,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qDAAqD;IACrD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAKD,sFAAsF;AACtF,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAG/C;AAED,qEAAqE;AACrE,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,MAAM,CAS1E;AAED;;;;GAIG;AACH;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,CAIlD;AAED,sFAAsF;AACtF,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC,CAI/F;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI,CAiBlD"}

package/dist/capture.js

@@ -0,0 +1,70 @@
+/**
+ * Building cases from real runs: a run's trigger payload becomes the input,
+ * its declared Result becomes the expectation (to be edited — what the
+ * workflow produced is not necessarily what it should have).
+ */
+import { isCaseId } from "./layout.js";
+const isPlainObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+/** A case id from free text: `[A-Za-z0-9._-]`, at most 80 characters, never empty. */
+export function slugCaseId(text) {
+    const slug = text.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^[-.]+|[-.]+$/g, "").slice(0, 80);
+    return slug && isCaseId(slug) ? slug : "case";
+}
+/** `base`, or `base-2`, `base-3`… — the first one not in `taken`. */
+export function uniqueCaseId(base, taken) {
+    const used = new Set(taken);
+    const root = slugCaseId(base);
+    if (!used.has(root))
+        return root;
+    for (let i = 2;; i++) {
+        const suffix = `-${i}`;
+        const id = `${root.slice(0, 80 - suffix.length)}${suffix}`;
+        if (!used.has(id))
+            return id;
+    }
+}
+/**
+ * A case from a captured run. `expected` is the run's Result, field by field,
+ * when it is an object; a non-object output cannot be keyed by field, so it is
+ * left out and the person states the expectation themselves.
+ */
+/**
+ * Whether a captured value is free text a model wrote — a sentence, not a label.
+ *
+ * Pinning prose with an exact match makes every later experiment fail on a
+ * harmless rewording, which buries the regressions the dataset exists to show.
+ * So a captured run expects such a field to be PRESENT; a person tightens it
+ * (contains, regex, or a judge) when the wording itself matters.
+ */
+export function isFreeText(v) {
+    if (typeof v !== "string")
+        return false;
+    const t = v.trim();
+    return t.length > 60 || t.split(/\s+/).length > 8;
+}
+/** What a captured output should be expected to be: labels exactly, prose present. */
+export function expectationFrom(output) {
+    const out = {};
+    for (const [k, v] of Object.entries(output))
+        out[k] = isFreeText(v) ? { $exists: true } : v;
+    return out;
+}
+export function caseFromRun(run) {
+    const { graph: _inline, ...input } = run.input;
+    const expected = isPlainObject(run.output) && Object.keys(run.output).length > 0
+        ? expectationFrom(run.output)
+        : undefined;
+    return {
+        id: run.id ?? slugCaseId(run.context),
+        input,
+        ...(expected ? { expected } : {}),
+        ...(run.tags?.length ? { tags: run.tags } : {}),
+        ...(run.notes ? { notes: run.notes } : {}),
+        source: {
+            context: run.context,
+            ...(run.workflowVersion ? { workflowVersion: run.workflowVersion } : {}),
+            capturedAt: run.capturedAt ?? new Date().toISOString(),
+        },
+    };
+}
+//# sourceMappingURL=capture.js.map

package/dist/capture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"capture.js","sourceRoot":"","sources":["../src/capture.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAkBvC,MAAM,aAAa,GAAG,CAAC,CAAU,EAA6B,EAAE,CAC9D,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAE3D,sFAAsF;AACtF,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/F,OAAO,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AAChD,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,KAAuB;IAChE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAI,CAAC,EAAE,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;QACvB,MAAM,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC;QAC3D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,OAAO,EAAE,CAAC;IAC/B,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH;;;;;;;GAOG;AACH,MAAM,UAAU,UAAU,CAAC,CAAU;IACnC,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACxC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACnB,OAAO,CAAC,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AACpD,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,eAAe,CAAC,MAA+B;IAC7D,MAAM,GAAG,GAAmC,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,GAAG,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAU,CAAC;IACtG,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAgB;IAC1C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,KAAK,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC;IAC/C,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC;QAC9E,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,MAAiC,CAAC;QACxD,CAAC,CAAC,SAAS,CAAC;IACd,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE,IAAI,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC;QACrC,KAAK;QACL,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1C,MAAM,EAAE;YACN,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACvD;KACF,CAAC;AACJ,CAAC"}

package/dist/conformance.d.ts

@@ -0,0 +1,6 @@
+import type { EvalStore } from "./storage.js";
+export declare function storeConformance(makeStore: () => Promise<EvalStore>): Array<{
+    name: string;
+    run: () => Promise<void>;
+}>;
+//# sourceMappingURL=conformance.d.ts.map

package/dist/conformance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../src/conformance.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,SAAS,EAAkB,MAAM,cAAc,CAAC;AAwC9D,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,MAAM,OAAO,CAAC,SAAS,CAAC,GAClC,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAAC,CAyPnD"}

package/dist/conformance.js

@@ -0,0 +1,257 @@
+/**
+ * The `EvalStore` contract as runnable checks.
+ *
+ * Exported rather than kept in this package's own tests because a second
+ * consumer (ROC) implements the same interface over a different client, and prose is
+ * not a contract. Each check throws on violation; the caller wraps them in
+ * whatever test runner it has.
+ */
+import { pool } from "./pool.js";
+const assert = (ok, what) => {
+    if (!ok)
+        throw new Error(`store conformance: ${what}`);
+};
+/** Every mechanism this suite knows how to be told about. */
+const MECHANISMS = ["conditional-write", "single-threaded"];
+/**
+ * The declaration gate.
+ *
+ * A store that says nothing is REFUSED rather than quietly run, because the
+ * concurrent check below cannot tell a real compare-and-set from a
+ * write-then-verify and silently accepting the second is the bug this exists
+ * to close (see `StoreAtomicity`). An unrecognised word counts as saying
+ * nothing: ROC reaches this through the built package, where TypeScript's
+ * union is gone by the time the object is constructed.
+ */
+function declaredMechanism(store) {
+    const declared = store.atomicity;
+    if (declared === undefined) {
+        throw new Error("store conformance: this store declares no `atomicity`, so the suite refuses it. " +
+            "No in-process check can tell a real compare-and-set from a write-then-verify " +
+            "(both leave exactly one winner under Promise.all), so an implementation must " +
+            `state which it is — one of ${MECHANISMS.join(", ")} — and pin that claim in its ` +
+            "own test against what the transport actually sends.");
+    }
+    if (!MECHANISMS.includes(declared)) {
+        throw new Error(`store conformance: unknown \`atomicity\` ${JSON.stringify(declared)}; the suite ` +
+            `only knows ${MECHANISMS.join(", ")}. An unrecognised word is treated as no ` +
+            "declaration at all, not as a pass.");
+    }
+    return declared;
+}
+export function storeConformance(makeStore) {
+    const k = () => `conformance/${crypto.randomUUID()}`;
+    // Bodies receive the store and the mechanism it declared, so the gate below
+    // can be applied to all of them in one place. A body that does not care
+    // about the mechanism just omits the second parameter.
+    const checks = [
+        {
+            name: "get answers null for a key that was never written",
+            body: async (s) => {
+                assert((await s.get(k())) === null, "absent key must read null, not throw");
+            },
+        },
+        {
+            name: "put then get round-trips the exact bytes",
+            body: async (s) => {
+                const key = k();
+                await s.put(key, '{"a":1}\n');
+                assert((await s.get(key)) === '{"a":1}\n', "bytes must round-trip unchanged");
+            },
+        },
+        {
+            // Headers (baselineId), experiment metas (status) and flags are
+            // rewritten in place. A store that treats every key as write-once would
+            // keep the first baseline forever and every experiment `running`.
+            name: "put replaces the bytes of a key that exists",
+            body: async (s) => {
+                const key = k();
+                await s.put(key, "first");
+                await s.put(key, "second");
+                assert((await s.get(key)) === "second", "a second put must replace the first");
+            },
+        },
+        {
+            // The v2 layout nests datasets and experiments two levels under
+            // `workflows/<id>/`; a delimiter listing must answer at that depth too.
+            name: "listPrefixes, when present, answers below the first level",
+            body: async (s) => {
+                if (!s.listPrefixes)
+                    return;
+                const root = `${k()}/workflows/wf`;
+                await s.put(`${root}/datasets/a/header.json`, "a");
+                await s.put(`${root}/datasets/b/cases/x/r1.json`, "b");
+                await s.put(`${root}/experiments/e/meta.json`, "e");
+                const got = await s.listPrefixes(`${root}/datasets/`);
+                assert(JSON.stringify([...got].sort()) === JSON.stringify([`${root}/datasets/a/`, `${root}/datasets/b/`]), `listPrefixes must return a nested prefix's children, got ${JSON.stringify(got)}`);
+            },
+        },
+        {
+            name: "putIfAbsent writes once and refuses the second writer",
+            body: async (s) => {
+                const key = k();
+                assert(await s.putIfAbsent(key, "first"), "first putIfAbsent must write");
+                assert(!(await s.putIfAbsent(key, "second")), "second must answer false");
+                assert((await s.get(key)) === "first", "a refused putIfAbsent must leave the stored bytes untouched");
+            },
+        },
+        {
+            // Two halves, because neither alone is enough.
+            //
+            // The DECLARATION is the half that matters most, and it is checked
+            // first. The check above is strictly sequential — write, write again,
+            // read — so a read-then-write `putIfAbsent` with no conditional header
+            // passes it. Eight at once catches the crudest shape of that (a store
+            // that reads, then writes, then answers `true` lets all eight see an
+            // absent key and all eight win) but it CANNOT catch a write-then-verify:
+            // that one clobbers seven writes and still leaves exactly one caller
+            // answering `true` over its own bytes, so it is green here and wrong in
+            // production, and wrong across two processes where both writers win.
+            // Nothing in this process can separate the two, so the store is required
+            // to say which it is and its own test pins the word to the wire (see
+            // `StoreAtomicity`).
+            //
+            // What the write-once layout rests on, and why this is not fussiness:
+            // the version number and the revision number are both DERIVED, so two
+            // writers compute the same one and only the conditional write decides
+            // who holds it. `memoryStore` declares `single-threaded` and passes by
+            // construction (no `await` between its check and its write); `s3Store`
+            // and ROC's store declare `conditional-write` and the dev gateway backs
+            // it with verified 412 behaviour.
+            name: "putIfAbsent declares its mechanism, and 8 concurrent calls leave one winner",
+            body: async (s, mechanism) => {
+                // Half of this union IS falsifiable in this process, and that half
+                // gets tested rather than trusted.
+                //
+                // `conditional-write` cannot be: a server adjudicating and a store
+                // writing-then-verifying are indistinguishable from in here, which is
+                // the whole reason the word is declared at all. `single-threaded` can
+                // be, because it is a much narrower claim than it sounds — it says the
+                // check and the write sit in ONE synchronous run with nothing
+                // suspending between them. So issue two `putIfAbsent`s on one key in
+                // the same turn, without awaiting in between: an honest
+                // single-threaded store finishes the first outright and the second
+                // finds the key taken, `[true, false]`. A write-then-verify wearing
+                // the label suspends on its first `get`, both calls read absent, both
+                // write, and it answers `[false, true]` — the first writer's bytes are
+                // already gone. Either way the label was false, and it fails here
+                // rather than in production.
+                if (mechanism === "single-threaded") {
+                    const key = k();
+                    const sameTurn = await Promise.all([
+                        s.putIfAbsent(key, "same-turn-a"),
+                        s.putIfAbsent(key, "same-turn-b"),
+                    ]);
+                    assert(sameTurn[0] === true && sameTurn[1] === false, "a store declaring single-threaded must settle putIfAbsent without suspending: " +
+                        "two calls issued in ONE synchronous turn must answer [true, false], got " +
+                        `${JSON.stringify(sameTurn)} — this store suspends between its check and its ` +
+                        "write, so the declaration is wrong");
+                    assert((await s.get(key)) === "same-turn-a", "the same-turn winner's bytes must be the ones stored");
+                }
+                const key = k();
+                const bodies = Array.from({ length: 8 }, (_, i) => `writer-${i}`);
+                const won = await Promise.all(bodies.map((body) => s.putIfAbsent(key, body)));
+                const winners = won.filter(Boolean).length;
+                // Zero winners is not the same failure as two: if the gateway refused
+                // every writer (e.g. answered 409/412 to all eight, including the one
+                // that would otherwise have won), the write-once contract was never
+                // actually exercised — that's contention exhausting the race, not a
+                // broken guarantee. Reporting it as a contract violation would send
+                // someone hunting a storage bug that isn't there.
+                if (winners === 0) {
+                    throw new Error("store conformance: contention, not a broken guarantee — all 8 concurrent " +
+                        "putIfAbsent calls on one key answered false (the gateway refused every " +
+                        "writer, e.g. 409/412 to all eight, including whichever would have won); " +
+                        "nobody actually exercised write-once here, so this is not a storage bug");
+                }
+                assert(winners === 1, `exactly one of 8 concurrent putIfAbsent calls may answer true, ${winners} did`);
+                const winner = bodies[won.indexOf(true)];
+                const stored = await s.get(key);
+                assert(stored === winner, `the stored bytes must be the winner's (${winner}), got ${JSON.stringify(stored)}`);
+            },
+        },
+        {
+            name: "list returns every key under a prefix and nothing above it",
+            body: async (s) => {
+                const root = k();
+                await s.put(`${root}/a.json`, "a");
+                await s.put(`${root}/nested/b.json`, "b");
+                await s.put(`${root}-sibling/c.json`, "c");
+                const keys = (await s.list(`${root}/`)).sort();
+                assert(keys.length === 2 && keys[0] === `${root}/a.json` && keys[1] === `${root}/nested/b.json`, `list must return exactly the prefix's keys, got ${JSON.stringify(keys)}`);
+            },
+        },
+        {
+            // Optional, so a store without it passes trivially — the library then
+            // derives prefixes from `list`. A store that HAS it is held to it: a
+            // wrong answer here hides datasets and experiments, quietly.
+            name: "listPrefixes, when present, returns each immediate child prefix once",
+            body: async (s) => {
+                if (!s.listPrefixes)
+                    return;
+                const root = k();
+                await s.put(`${root}/top.json`, "t");
+                await s.put(`${root}/x/1.json`, "1");
+                await s.put(`${root}/x/2.json`, "2");
+                await s.put(`${root}/y/deep/3.json`, "3");
+                await s.put(`${root}-sibling/z/4.json`, "4");
+                const got = await s.listPrefixes(`${root}/`);
+                assert(JSON.stringify([...got].sort()) === JSON.stringify([`${root}/x/`, `${root}/y/`]), `listPrefixes must return exactly the child prefixes, each once, got ${JSON.stringify(got)}`);
+            },
+        },
+        {
+            name: "list pages past any server limit",
+            body: async (s) => {
+                const root = k();
+                // The gateway caps a page at 1000 and does NOT signal truncation to a
+                // naive caller. A set that silently stops at 1000 cuts a version
+                // missing the rest, and the experiment built from it looks complete.
+                // Fixed at 1050 on purpose — a consumer whose backend makes this too
+                // slow to run every commit should filter storeConformance's returned
+                // array by name and skip this one check in their own test file, not
+                // shrink the count here: a smaller count can pass against a `list`
+                // that duplicates or drops keys just as easily as against a correct
+                // one, which would make the check stop proving anything and no test
+                // output would say so.
+                const n = 1050;
+                const expected = new Set();
+                // Bounded concurrency: 1050 sequential round trips over HTTP is the
+                // slow part of this check, not `list` itself — 32 in flight is fast
+                // against a real gateway without opening 1050 sockets at once.
+                await pool(Array.from({ length: n }, (_, i) => i), 32, async (i) => {
+                    const key = `${root}/${i}.json`;
+                    await s.put(key, String(i));
+                    expected.add(key);
+                });
+                const keys = await s.list(`${root}/`);
+                assert(keys.length === n, `expected ${n} keys, got ${keys.length}`);
+                assert(new Set(keys).size === n, "list must not return duplicate keys across pages");
+                assert(keys.every((key) => expected.has(key)), "list must return exactly the keys written, not just the right count");
+            },
+        },
+    ];
+    /**
+     * EVERY check is gated, not one of them.
+     *
+     * The gate first lived inside the concurrent check, reasoning that a
+     * standalone entry could be filtered away. But filtering by NAME is exactly
+     * what the paging check above documents as the supported way to skip a check
+     * a consumer's backend makes too slow — and that same filter drops a gate
+     * hidden in any one entry. Wrapping instead costs one string comparison per
+     * check and cannot be skipped by any subset a consumer chooses to run:
+     * whichever checks you keep, the store had to say how `putIfAbsent` is
+     * atomic before any of them ran.
+     *
+     * `makeStore()` is still called exactly once per check. Consumers depend on
+     * that — `eval-store.test.ts` mints one gateway credential in `beforeAll`
+     * precisely because this is per-entry and not per-operation.
+     */
+    return checks.map(({ name, body }) => ({
+        name,
+        run: async () => {
+            const s = await makeStore();
+            await body(s, declaredMechanism(s));
+        },
+    }));
+}
+//# sourceMappingURL=conformance.js.map

package/dist/conformance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"conformance.js","sourceRoot":"","sources":["../src/conformance.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,MAAM,MAAM,GAAG,CAAC,EAAW,EAAE,IAAY,EAAQ,EAAE;IACjD,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,CAAC;AACzD,CAAC,CAAC;AAEF,6DAA6D;AAC7D,MAAM,UAAU,GAA8B,CAAC,mBAAmB,EAAE,iBAAiB,CAAC,CAAC;AAEvF;;;;;;;;;GASG;AACH,SAAS,iBAAiB,CAAC,KAAgB;IACzC,MAAM,QAAQ,GAAY,KAAK,CAAC,SAAS,CAAC;IAC1C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,kFAAkF;YAChF,+EAA+E;YAC/E,+EAA+E;YAC/E,8BAA8B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B;YAClF,qDAAqD,CACxD,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,QAA0B,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CACb,4CAA4C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,cAAc;YAChF,cAAc,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,0CAA0C;YAC7E,oCAAoC,CACvC,CAAC;IACJ,CAAC;IACD,OAAO,QAA0B,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,SAAmC;IAEnC,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,eAAe,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC;IACrD,4EAA4E;IAC5E,wEAAwE;IACxE,uDAAuD;IACvD,MAAM,MAAM,GAGP;QACH;YACE,IAAI,EAAE,mDAAmD;YACzD,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;gBAChB,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,sCAAsC,CAAC,CAAC;YAC9E,CAAC;SACF;QACD;YACE,IAAI,EAAE,0CAA0C;YAChD,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;gBAChB,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;gBAC9B,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,WAAW,EAAE,iCAAiC,CAAC,CAAC;YAChF,CAAC;SACF;QACD;YACE,gEAAgE;YAChE,wEAAwE;YACxE,kEAAkE;YAClE,IAAI,EAAE,6CAA6C;YACnD,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;gBAChB,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAC1B,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;gBAC3B,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,QAAQ,EAAE,qCAAqC,CAAC,CAAC;YACjF,CAAC;SACF;QACD;YACE,gEAAgE;YAChE,wEAAwE;YACxE,IAAI,EAAE,2DAA2D;YACjE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;gBAChB,IAAI,CAAC,CAAC,CAAC,YAAY;oBAAE,OAAO;gBAC5B,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,eAAe,CAAC;gBACnC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,yBAAyB,EAAE,GAAG,CAAC,CAAC;gBACnD,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,6BAA6B,EAAE,GAAG,CAAC,CAAC;gBACvD,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,0BAA0B,EAAE,GAAG,CAAC,CAAC;gBACpD,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,YAAY,CAAC,GAAG,IAAI,YAAY,CAAC,CAAC;gBACtD,MAAM,CACJ,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,IAAI,cAAc,EAAE,GAAG,IAAI,cAAc,CAAC,CAAC,EAClG,4DAA4D,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAClF,CAAC;YACJ,CAAC;SACF;QACD;YACE,IAAI,EAAE,uDAAuD;YAC7D,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;gBAChB,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,8BAA8B,CAAC,CAAC;gBAC1E,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC;gBAC1E,MAAM,CACJ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,OAAO,EAC9B,6DAA6D,CAC9D,CAAC;YACJ,CAAC;SACF;QACD;YACE,+CAA+C;YAC/C,EAAE;YACF,mEAAmE;YACnE,sEAAsE;YACtE,uEAAuE;YACvE,sEAAsE;YACtE,qEAAqE;YACrE,yEAAyE;YACzE,qEAAqE;YACrE,wEAAwE;YACxE,qEAAqE;YACrE,yEAAyE;YACzE,qEAAqE;YACrE,qBAAqB;YACrB,EAAE;YACF,sEAAsE;YACtE,sEAAsE;YACtE,sEAAsE;YACtE,uEAAuE;YACvE,uEAAuE;YACvE,wEAAwE;YACxE,kCAAkC;YAClC,IAAI,EAAE,6EAA6E;YACnF,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE;gBAC3B,mEAAmE;gBACnE,mCAAmC;gBACnC,EAAE;gBACF,mEAAmE;gBACnE,sEAAsE;gBACtE,sEAAsE;gBACtE,uEAAuE;gBACvE,8DAA8D;gBAC9D,qEAAqE;gBACrE,wDAAwD;gBACxD,mEAAmE;gBACnE,oEAAoE;gBACpE,sEAAsE;gBACtE,uEAAuE;gBACvE,kEAAkE;gBAClE,6BAA6B;gBAC7B,IAAI,SAAS,KAAK,iBAAiB,EAAE,CAAC;oBACpC,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;oBAChB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;wBACjC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,aAAa,CAAC;wBACjC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,aAAa,CAAC;qBAClC,CAAC,CAAC;oBACH,MAAM,CACJ,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,KAAK,EAC7C,gFAAgF;wBAC9E,0EAA0E;wBAC1E,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,mDAAmD;wBAC9E,oCAAoC,CACvC,CAAC;oBACF,MAAM,CACJ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,aAAa,EACpC,sDAAsD,CACvD,CAAC;gBACJ,CAAC;gBACD,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;gBAClE,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC9E,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;gBAC3C,sEAAsE;gBACtE,sEAAsE;gBACtE,oEAAoE;gBACpE,oEAAoE;gBACpE,oEAAoE;gBACpE,kDAAkD;gBAClD,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CACb,2EAA2E;wBACzE,yEAAyE;wBACzE,0EAA0E;wBAC1E,yEAAyE,CAC5E,CAAC;gBACJ,CAAC;gBACD,MAAM,CACJ,OAAO,KAAK,CAAC,EACb,kEAAkE,OAAO,MAAM,CAChF,CAAC;gBACF,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAE,CAAC;gBAC1C,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAChC,MAAM,CACJ,MAAM,KAAK,MAAM,EACjB,0CAA0C,MAAM,UAAU,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CACnF,CAAC;YACJ,CAAC;SACF;QACD;YACE,IAAI,EAAE,4DAA4D;YAClE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;gBAChB,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC;gBACjB,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,SAAS,EAAE,GAAG,CAAC,CAAC;gBACnC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,gBAAgB,EAAE,GAAG,CAAC,CAAC;gBAC1C,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,iBAAiB,EAAE,GAAG,CAAC,CAAC;gBAC3C,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC/C,MAAM,CACJ,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,gBAAgB,EACxF,mDAAmD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAC1E,CAAC;YACJ,CAAC;SACF;QACD;YACE,sEAAsE;YACtE,qEAAqE;YACrE,6DAA6D;YAC7D,IAAI,EAAE,sEAAsE;YAC5E,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;gBAChB,IAAI,CAAC,CAAC,CAAC,YAAY;oBAAE,OAAO;gBAC5B,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC;gBACjB,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,WAAW,EAAE,GAAG,CAAC,CAAC;gBACrC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,WAAW,EAAE,GAAG,CAAC,CAAC;gBACrC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,WAAW,EAAE,GAAG,CAAC,CAAC;gBACrC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,gBAAgB,EAAE,GAAG,CAAC,CAAC;gBAC1C,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,mBAAmB,EAAE,GAAG,CAAC,CAAC;gBAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,YAAY,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC;gBAC7C,MAAM,CACJ,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,IAAI,KAAK,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC,EAChF,uEAAuE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAC7F,CAAC;YACJ,CAAC;SACF;QACD;YACE,IAAI,EAAE,kCAAkC;YACxC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;gBAChB,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC;gBACjB,sEAAsE;gBACtE,iEAAiE;gBACjE,qEAAqE;gBACrE,qEAAqE;gBACrE,qEAAqE;gBACrE,oEAAoE;gBACpE,mEAAmE;gBACnE,oEAAoE;gBACpE,oEAAoE;gBACpE,uBAAuB;gBACvB,MAAM,CAAC,GAAG,IAAI,CAAC;gBACf,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;gBACnC,oEAAoE;gBACpE,oEAAoE;gBACpE,+DAA+D;gBAC/D,MAAM,IAAI,CACR,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EACtC,EAAE,EACF,KAAK,EAAE,CAAC,EAAE,EAAE;oBACV,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC;oBAChC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC5B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACpB,CAAC,CACF,CAAC;gBACF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC;gBACtC,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,YAAY,CAAC,cAAc,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBACpE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,kDAAkD,CAAC,CAAC;gBACrF,MAAM,CACJ,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EACtC,qEAAqE,CACtE,CAAC;YACJ,CAAC;SACF;KACF,CAAC;IAEF;;;;;;;;;;;;;;;OAeG;IACH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI;QACJ,GAAG,EAAE,KAAK,IAAI,EAAE;YACd,MAAM,CAAC,GAAG,MAAM,SAAS,EAAE,CAAC;YAC5B,MAAM,IAAI,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;KACF,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/contract.d.ts

@@ -0,0 +1,18 @@
+import type { Case, Experiment, Header, Limits, RunRecord, VersionManifest } from "./types.js";
+export declare class EvalContractError extends Error {
+    constructor(message: string);
+}
+/** A plain JSON object: not null, not an array. */
+export declare const isObject: (v: unknown) => v is Record<string, unknown>;
+export declare function parseLimits(raw: unknown, where: string): Limits | undefined;
+export declare function parseHeader(raw: unknown): Header;
+export declare function parseCase(raw: unknown): Case;
+export declare function parseVersionManifest(raw: unknown, dataset: string, version: number): VersionManifest;
+/**
+ * A run record read back. Lenient: the library wrote it. A record written
+ * before v2 numbers its repeat `trial`; it reads as `repeat`.
+ */
+export declare function parseRunRecord(raw: unknown, where: string): RunRecord;
+/** An experiment's meta read back. Lenient: the library wrote it. */
+export declare function parseExperiment(raw: unknown, where: string): Experiment;
+//# sourceMappingURL=contract.d.ts.map

package/dist/contract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../src/contract.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAQ,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAErG,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,OAAO,EAAE,MAAM;CAI5B;AAED,mDAAmD;AACnD,eAAO,MAAM,QAAQ,GAAI,GAAG,OAAO,KAAG,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CACP,CAAC;AAiB3D,wBAAgB,WAAW,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAU3E;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAwBhD;AAED,wBAAgB,SAAS,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CA0C5C;AAED,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,eAAe,CAoBpG;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAQrE;AAED,qEAAqE;AACrE,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,UAAU,CAQvE"}