@openwop/openwop-conformance 2.46.30 → 2.46.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,3 +1,11 @@
1
+ ## [2.46.32] — 2026-10-26 — RFC 0252 Active
2
+
3
+ - **New scenario `v2-eval-recorded-fixtures` (RFC 0252)** over three fixtures and the counter tool, run in order source, replay (`fixtureRuns: {record: source}`), drift, then the source again as a probe: `0252.shape` (server-free), `source-replayed`, `drift-fails-closed`, `source-refused`, `source-uniform`, `fields-refused`. Judges in `lib/eval-recorded-fixtures-witness.ts` with self-tests.
4
+ - **Host impact:** opt-in. The legs need `agents.evalSuite.recordedFixtures`, `fixtureMatching` and the three fixtures listed; no host does yet, so they record `inapplicable`. Listing a recorded fixture opts the host in to the counter-tool contract.
5
+ ## [2.46.31] — 2026-10-26 — RFC 0251 Accepted
6
+
7
+ - **No assertion changes.** RFC 0251's G2 closes on openwop-app's certified cut `24b6a05ff` (the five `0251.*` legs and both `0244.regression-*` legs `executed-pass`), so the packed `spec/v1/gaps.json` changes.
8
+
1
9
  ## [2.46.30] — 2026-10-26 — RFC 0252 Draft (no assertion changes)
2
10
 
3
11
  - **No scenario or assertion changes.** RFC 0252 (`Draft`) adds its gap register and RFC 0244's G5 row moves to it, so the packed `spec/v1/gaps.json` changes.
package/README.md CHANGED
@@ -137,7 +137,7 @@ Each scenario file records its disposition and assertion count in `<report-dir>/
137
137
 
138
138
  ## What's Covered
139
139
 
140
- The current suite has 663 scenario files under `src/scenarios/`. By target major ([`scenario-majors.json`](./scenario-majors.json)): 223 files run against a v2 host and 455 against a v1 host; 15 of them run at both.
140
+ The current suite has 664 scenario files under `src/scenarios/`. By target major ([`scenario-majors.json`](./scenario-majors.json)): 224 files run against a v2 host and 455 against a v1 host; 15 of them run at both.
141
141
 
142
142
  The v2 scenarios (`src/scenarios/v2-*.test.ts`) cover, by area:
143
143
 
@@ -166,7 +166,7 @@ Three kinds of scenario share the tree:
166
166
  - **Host** checks drive a live host over HTTP.
167
167
  - **Gated** checks run only when the host advertises the family, profile or fixture they need, and otherwise record `inapplicable` or `blocked` with a reason. `--require-behavior` turns an advertised-but-unwitnessable behaviour into a failure.
168
168
 
169
- Current source tree: 663 scenario files. [`coverage.md`](./coverage.md) maps each spec document and every OpenAPI operation to its scenarios. [`CHANGELOG.md`](./CHANGELOG.md) records when each scenario landed.
169
+ Current source tree: 664 scenario files. [`coverage.md`](./coverage.md) maps each spec document and every OpenAPI operation to its scenarios. [`CHANGELOG.md`](./CHANGELOG.md) records when each scenario landed.
170
170
 
171
171
  ## How the 2.x suite is built
172
172
 
@@ -1,5 +1,5 @@
1
1
  {
2
2
  "package": "@openwop/spec-artifacts",
3
- "version": "2.46.30",
4
- "stampSha256": "ab7c7f2bc8efddb11e0c565b046f69538a7ab3145416ee4227e8d96265b7d9b9"
3
+ "version": "2.46.32",
4
+ "stampSha256": "e6decdfd7ff3c1d06ccb5b895967821962de134e122765fa012698d3e1fa5535"
5
5
  }
@@ -0,0 +1,46 @@
1
+ {
2
+ "suiteId": "core.openwop.evals.conformance-eval-recorded-drift",
3
+ "version": "1.0.0",
4
+ "targetAgentId": "core.conformance.agents.scripted",
5
+ "modes": [
6
+ "golden"
7
+ ],
8
+ "tasks": [
9
+ {
10
+ "taskId": "record",
11
+ "input": {
12
+ "script": [
13
+ {
14
+ "tool": "conformance.eval:counter",
15
+ "arguments": {
16
+ "key": "P",
17
+ "op": "read"
18
+ }
19
+ },
20
+ {
21
+ "tool": "conformance.eval:counter",
22
+ "arguments": {
23
+ "key": "P",
24
+ "op": "inc"
25
+ }
26
+ },
27
+ {
28
+ "tool": "conformance.eval:counter",
29
+ "arguments": {
30
+ "key": "R",
31
+ "op": "read"
32
+ }
33
+ }
34
+ ],
35
+ "output": "done"
36
+ },
37
+ "expected": {
38
+ "kind": "golden",
39
+ "match": {
40
+ "strategy": "contains",
41
+ "value": "done"
42
+ }
43
+ }
44
+ }
45
+ ]
46
+ }
@@ -0,0 +1,46 @@
1
+ {
2
+ "suiteId": "core.openwop.evals.conformance-eval-recorded-replay",
3
+ "version": "1.0.0",
4
+ "targetAgentId": "core.conformance.agents.scripted",
5
+ "modes": [
6
+ "golden"
7
+ ],
8
+ "tasks": [
9
+ {
10
+ "taskId": "record",
11
+ "input": {
12
+ "script": [
13
+ {
14
+ "tool": "conformance.eval:counter",
15
+ "arguments": {
16
+ "key": "P",
17
+ "op": "read"
18
+ }
19
+ },
20
+ {
21
+ "tool": "conformance.eval:counter",
22
+ "arguments": {
23
+ "key": "R",
24
+ "op": "inc"
25
+ }
26
+ },
27
+ {
28
+ "tool": "conformance.eval:counter",
29
+ "arguments": {
30
+ "key": "R",
31
+ "op": "read"
32
+ }
33
+ }
34
+ ],
35
+ "output": "done"
36
+ },
37
+ "expected": {
38
+ "kind": "golden",
39
+ "match": {
40
+ "strategy": "contains",
41
+ "value": "done"
42
+ }
43
+ }
44
+ }
45
+ ]
46
+ }
@@ -0,0 +1,61 @@
1
+ {
2
+ "suiteId": "core.openwop.evals.conformance-eval-recorded-source",
3
+ "version": "1.0.0",
4
+ "targetAgentId": "core.conformance.agents.scripted",
5
+ "modes": [
6
+ "golden"
7
+ ],
8
+ "tasks": [
9
+ {
10
+ "taskId": "record",
11
+ "input": {
12
+ "script": [
13
+ {
14
+ "tool": "conformance.eval:counter",
15
+ "arguments": {
16
+ "key": "P",
17
+ "op": "read"
18
+ }
19
+ },
20
+ {
21
+ "tool": "conformance.eval:counter",
22
+ "arguments": {
23
+ "key": "R",
24
+ "op": "inc"
25
+ }
26
+ },
27
+ {
28
+ "tool": "conformance.eval:counter",
29
+ "arguments": {
30
+ "key": "R",
31
+ "op": "read"
32
+ }
33
+ }
34
+ ],
35
+ "output": "done"
36
+ },
37
+ "expected": {
38
+ "kind": "golden",
39
+ "match": {
40
+ "strategy": "contains",
41
+ "value": "done"
42
+ }
43
+ },
44
+ "fixtures": {
45
+ "onMiss": "live",
46
+ "toolResponses": [
47
+ {
48
+ "tool": "conformance.eval:lookup",
49
+ "args": {
50
+ "orderId": "o-1"
51
+ },
52
+ "argsMatch": "exact",
53
+ "response": {
54
+ "status": "shipped"
55
+ }
56
+ }
57
+ ]
58
+ }
59
+ }
60
+ ]
61
+ }
package/fixtures.md CHANGED
@@ -80,6 +80,7 @@ All fixtures MUST advertise:
80
80
  | Eval — Efficiency Ungated | `conformance-eval-efficiency-ungated` | RFC 0244 §C — two golden tasks with one output, `lean` (1 call) and `busy` (8 calls), at `passScore: 1` (`v2-eval-trajectory-trials`) | eval run `completed`; both tasks and the suite pass | ≤ 30s |
81
81
  | Eval — Memory Isolation | `conformance-eval-memory-isolation` | RFC 0244 §B, G3 — `trials.count: 2`; one task seeds `seeded`, then reads it, reads `trial-note`, writes it and reads it back through `conformance.eval:memory` (`v2-eval-trajectory-trials`) | eval run `completed`; each trial reads the seed and finds no `trial-note` before its own write | ≤ 30s |
82
82
  | Eval — Gate Failing | `conformance-eval-gate-failing` | RFC 0082 §E — one task whose output never contains the golden `done`, so the summary is `passed: false`; its run is then cited by a `promote` of the scripted agent (`v2-deployment-eval-gate`) | eval run `completed`, `passed: false`; the promote is refused `422 eval_gate_unmet` | ≤ 30s |
83
+ | Eval — Recorded Source / Replay / Drift | `conformance-eval-recorded-source`, `conformance-eval-recorded-replay`, `conformance-eval-recorded-drift` | RFC 0252 — task `record` through `conformance.eval:counter`: source `read P`, `inc R`, `read R` live; replay the same script with `fixtureRuns: {record: <source>}`; drift `read P`, `inc P`, `read R` with the same source (`v2-eval-recorded-fixtures`) | source and replay `completed`, passing; drift `completed` with `passed: false` | ≤ 30s each |
83
84
  | Eval — Regression (base) | `conformance-eval-regression-base` | RFC 0251 — the baseline run: suite 1.0.0, `base-only` fails, `shared-a` and `shared-b` pass (`v2-eval-regression-baseline`) | eval run `completed`; `aggregateScore` 2/3 | ≤ 30s |
84
85
  | Eval — Regression (candidate) | `conformance-eval-regression-candidate` | RFC 0251, RFC 0244 §F — the same `suiteId` at 1.1.0: the two shared tasks unchanged, plus `candidate-only` (passes), run against the base run (`v2-eval-regression-baseline`) | eval run `completed`; `aggregateScore` 1; `scoreDelta` 1/3; `pairedTaskCount` 2; verdict neither `improved` nor `regressed` | ≤ 30s |
85
86
  | Dispatch Fan-Out Waves | `conformance-dispatch-fanout-waves` | RFC 0243 §A.3 — a parallel `core.dispatch` (`maxConcurrency: 2`, `wait-all`/`collect`) whose supervisor's `mockDispatchPlan` names three `conformance-delay` children. At most `min(2, maxFanOut)` run at once and none is dropped. Gated on `agents.fanOut` admitting `wait-all` with `collect`. | `completed` | ≤ 30s |
@@ -1071,8 +1072,9 @@ Every task targets **the scripted agent `core.conformance.agents.scripted`**. It
1071
1072
  | `conformance-eval-gate-failing` | `agents.evalSuite` + `agents.deployment` | RFC 0082 §E: a `promote` of a `draft`, `test` or `staged` record of the scripted agent citing this run as `evalRunId` is refused `422 eval_gate_unmet`, and the record keeps its `state`, `channels` and `canaryPercent`. A summary that does not say `passed: false` records `blocked` |
1072
1073
  | `conformance-eval-regression-base` | `agents.evalSuite.baselineSelection` + `regression` in `modes` | run first with `evalModes: [golden]`; the baseline of the candidate run |
1073
1074
  | `conformance-eval-regression-candidate` | `agents.evalSuite.baselineSelection` + `regression` in `modes` (+ `regressionVerdict` for the §F legs) | run with `evalModes: [golden, regression]` and the base run as `baselineRunId`: both carriers name it, `scoreDelta` 1/3, `pairedTaskCount` 2, verdict neither `improved` nor `regressed`. Both fixtures share one `suiteId`, so a host's suite store resolves them by fixture id |
1075
+ | `conformance-eval-recorded-source` / `conformance-eval-recorded-replay` / `conformance-eval-recorded-drift` | `agents.evalSuite.recordedFixtures` + `fixtureMatching` | RFC 0252, run in order source, replay, drift, then source again as a probe: the replay's `eval.started.fixtureRuns` echoes the map and its three counter outcomes equal the source's with `fixtureMissCount` 0; the drift counts one miss and fails; the probe's `inc R` is the source's plus one and its `read P` equals the source's. A counter that did not persist, or moved by more than two, records `blocked` |
1074
1076
 
1075
- **The counter tool `conformance.eval:counter` is conformance-RESERVED** (the pattern of `core.conformance.side-effect`). A host that lists either counter fixture in discovery `fixtures` MUST, at its eval tool boundary:
1077
+ **The counter tool `conformance.eval:counter` is conformance-RESERVED** (the pattern of `core.conformance.side-effect`). A host that lists either counter fixture, or any of `conformance-eval-recorded-source`, `conformance-eval-recorded-replay` and `conformance-eval-recorded-drift`, in discovery `fixtures` MUST, at its eval tool boundary:
1076
1078
 
1077
1079
  - answer a call that reaches the live tool with inputs `{key, op}` as follows. `inc` adds one to a counter scoped to the caller's owner triple and `key`, then returns `{key, count}`. `read` returns `{key, count}` without changing it. A key never incremented reads `0`.
1078
1080
  - keep that state for at least an hour, across eval runs;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openwop/openwop-conformance",
3
- "version": "2.46.30",
3
+ "version": "2.46.32",
4
4
  "description": "Black-box conformance suite for OpenWOP hosts (v2 current; v1 scenarios retained for the frozen v1 tree).",
5
5
  "repository": {
6
6
  "type": "git",
@@ -56,6 +56,6 @@
56
56
  "@openwop/spec-artifacts": "file:../spec-artifacts"
57
57
  },
58
58
  "peerDependencies": {
59
- "@openwop/spec-artifacts": "2.46.30"
59
+ "@openwop/spec-artifacts": "2.46.32"
60
60
  }
61
61
  }
package/requirements.json CHANGED
@@ -2,11 +2,11 @@
2
2
  "$comment": "GENERATED by conformance/scripts/generate-requirement-registry.mjs — do not edit. One record per it()/test() in src/scenarios. Ids: openwop.it.<file-stem>.<title-slug>[~n] (src/lib/requirement-ids.ts). A record with id null has an interpolated title; its run-time row is keyed by the rendered title and maps here by file+line only. Renamed ids need a row in requirement-aliases.json.",
3
3
  "generatedFrom": "src/scenarios/*.test.ts",
4
4
  "counts": {
5
- "files": 751,
6
- "tests": 2857,
7
- "withStableId": 2857,
5
+ "files": 752,
6
+ "tests": 2863,
7
+ "withStableId": 2863,
8
8
  "interpolatedTitles": 0,
9
- "explicitIds": 2730
9
+ "explicitIds": 2736
10
10
  },
11
11
  "records": [
12
12
  {
@@ -31105,6 +31105,80 @@
31105
31105
  }
31106
31106
  ]
31107
31107
  },
31108
+ {
31109
+ "id": "openwop.it.v2-eval-recorded-fixtures.the-recordedfixtures-member-eval-started-fixtureruns-and-the-createrun-property",
31110
+ "file": "v2-eval-recorded-fixtures.test.ts",
31111
+ "line": 130,
31112
+ "title": "the recordedFixtures member, eval.started.fixtureRuns and the createRun property are in the derived contract",
31113
+ "explicitId": "openwop.requirement.0252.shape",
31114
+ "citations": [
31115
+ {
31116
+ "section": "agents.schema.json §evalSuite.recordedFixtures, createRun fixtureRuns, eval.started.fixtureRuns (RFC 0252)",
31117
+ "requirement": "recordedFixtures: true MUST validate"
31118
+ },
31119
+ {
31120
+ "section": "agents.schema.json §evalSuite.recordedFixtures, createRun fixtureRuns, eval.started.fixtureRuns (RFC 0252)",
31121
+ "requirement": "recordedFixtures MUST be a boolean"
31122
+ },
31123
+ {
31124
+ "section": "agents.schema.json §evalSuite.recordedFixtures, createRun fixtureRuns, eval.started.fixtureRuns (RFC 0252)",
31125
+ "requirement": null,
31126
+ "interpolated": true
31127
+ },
31128
+ {
31129
+ "section": "agents.schema.json §evalSuite.recordedFixtures, createRun fixtureRuns, eval.started.fixtureRuns (RFC 0252)",
31130
+ "requirement": "an empty fixtureRuns MUST be refused"
31131
+ },
31132
+ {
31133
+ "section": "agents.schema.json §evalSuite.recordedFixtures, createRun fixtureRuns, eval.started.fixtureRuns (RFC 0252)",
31134
+ "requirement": "a fixtureRuns value MUST be a runId"
31135
+ },
31136
+ {
31137
+ "section": "agents.schema.json §evalSuite.recordedFixtures, createRun fixtureRuns, eval.started.fixtureRuns (RFC 0252)",
31138
+ "requirement": "createRun MUST carry fixtureRuns, a non-empty map onto runIds"
31139
+ }
31140
+ ]
31141
+ },
31142
+ {
31143
+ "id": "openwop.it.v2-eval-recorded-fixtures.the-replay-is-served-the-source-run-s-recorded-answers-and-never-reaches-the-liv",
31144
+ "file": "v2-eval-recorded-fixtures.test.ts",
31145
+ "line": 149,
31146
+ "title": "the replay is served the source run's recorded answers and never reaches the live tool",
31147
+ "explicitId": "openwop.requirement.0252.source-replayed",
31148
+ "citations": []
31149
+ },
31150
+ {
31151
+ "id": "openwop.it.v2-eval-recorded-fixtures.a-call-the-source-never-made-misses-fails-the-task-and-never-runs-live",
31152
+ "file": "v2-eval-recorded-fixtures.test.ts",
31153
+ "line": 158,
31154
+ "title": "a call the source never made misses, fails the task, and never runs live",
31155
+ "explicitId": "openwop.requirement.0252.drift-fails-closed",
31156
+ "citations": []
31157
+ },
31158
+ {
31159
+ "id": "openwop.it.v2-eval-recorded-fixtures.an-unknown-source-and-a-run-that-never-invoked-the-agent-are-refused-without-the",
31160
+ "file": "v2-eval-recorded-fixtures.test.ts",
31161
+ "line": 167,
31162
+ "title": "an unknown source, and a run that never invoked the agent, are refused without the id",
31163
+ "explicitId": "openwop.requirement.0252.source-refused",
31164
+ "citations": []
31165
+ },
31166
+ {
31167
+ "id": "openwop.it.v2-eval-recorded-fixtures.another-tenant-s-run-is-refused-exactly-as-an-id-that-never-existed",
31168
+ "file": "v2-eval-recorded-fixtures.test.ts",
31169
+ "line": 186,
31170
+ "title": "another tenant's run is refused exactly as an id that never existed",
31171
+ "explicitId": "openwop.requirement.0252.source-uniform",
31172
+ "citations": []
31173
+ },
31174
+ {
31175
+ "id": "openwop.it.v2-eval-recorded-fixtures.a-key-naming-no-task-a-target-with-its-own-fixtures-and-fixtureruns-without-mode",
31176
+ "file": "v2-eval-recorded-fixtures.test.ts",
31177
+ "line": 203,
31178
+ "title": "a key naming no task, a target with its own fixtures, and fixtureRuns without mode are refused",
31179
+ "explicitId": "openwop.requirement.0252.fields-refused",
31180
+ "citations": []
31181
+ },
31108
31182
  {
31109
31183
  "id": "openwop.it.v2-eval-regression-baseline.the-baselineselection-member-and-the-createrun-eval-properties-are-in-the-derive",
31110
31184
  "file": "v2-eval-regression-baseline.test.ts",
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "$comment": "GENERATED by conformance/scripts/generate-scenario-majors.mjs (RFC 0168 §D.3). Do not edit; add a file to BOTH_MAJORS in the generator to target both majors.",
3
3
  "counts": {
4
- "files": 663,
4
+ "files": 664,
5
5
  "v1": 455,
6
- "v2": 223
6
+ "v2": 224
7
7
  },
8
8
  "majors": {
9
9
  "a2a-1-0-agent-card.test.ts": [
@@ -1485,6 +1485,9 @@
1485
1485
  "v2-eval-mode-unadvertised-refused.test.ts": [
1486
1486
  2
1487
1487
  ],
1488
+ "v2-eval-recorded-fixtures.test.ts": [
1489
+ 2
1490
+ ],
1488
1491
  "v2-eval-regression-baseline.test.ts": [
1489
1492
  2
1490
1493
  ],
@@ -1,17 +1,17 @@
1
1
  {
2
2
  "_comment": "Provenance of @openwop/spec-artifacts (RFC 0168 §D.2). files: SHA-256 per file; the conformance suite compares the installed peer against dist/spec-artifacts.lock.json at start.",
3
3
  "package": "@openwop/spec-artifacts",
4
- "version": "2.46.30",
5
- "corpusTag": "v2.46.30",
4
+ "version": "2.46.32",
5
+ "corpusTag": "v2.46.32",
6
6
  "files": {
7
7
  "api/.redocly.lint-ignore.yaml": "bf5a8350b88a72fa43f59605ed8d903ed24b6cfccda5e45509c9f6ed9ee4e712",
8
8
  "api/asyncapi.yaml": "d5ecb9ee6114582be3b1f662c84bfac9ae96dae7bacb853e461168f70a8e1c7d",
9
9
  "api/grpc/openwop.proto": "c3e72bb17cba514ee98feb6434e6c9b6ea6795bfd086489ec69fd882dd1ad977",
10
10
  "api/openapi.yaml": "ac6de5303872664b78ff419c035915593e01599e02835e777a0123de1baeb398",
11
11
  "api/redocly.yaml": "b0604c89b2ca6d5076ec25725c539dad44a741a811fe524439ee6daef8baa09f",
12
- "api/seams-v2.yaml": "08c68b844a0298d1f7232e5956713ed5235414eca608782f8c9cc607cee06f32",
13
- "api/v2/asyncapi.yaml": "edfda007ec4a3ef1ceafa8125c87d02dbf77367e4816f0b41455c9827ae3f8fe",
14
- "api/v2/openapi.yaml": "2e8450ca565d0fe75c27dae848122891ba80e5fa40f0876796c51979572efd44",
12
+ "api/seams-v2.yaml": "2a890e944caa3925d4e5ba1f2f6a74b0a20f5449d1cefddf27fb625b763432f7",
13
+ "api/v2/asyncapi.yaml": "6498030118946490e6eeeb96c9c16b95a6334a55b6e5e50f203c94b197a083b3",
14
+ "api/v2/openapi.yaml": "2e7da662960ecf05088eeb24b2eb2996f481c16ba5b8fc867e7869ad3f9ab20e",
15
15
  "api/v2/redocly.yaml": "1e66b60e6118ad11a823bb620678be464d99dfe50a40e3e6f93ec9429b88b34c",
16
16
  "schemas/README.md": "276fc1d51eb08e14e9127b9fd27761d7046ee33c12000f3a64a801322db8bf58",
17
17
  "schemas/a2a-task-state.schema.json": "75d5049dea8bd873ff0e7546f1c60c8d36c219bec7084264360be54a8be30ae1",
@@ -117,7 +117,7 @@
117
117
  "schemas/v2/artifact.schema.json": "85b9dc284d39b99d2a66fa8cbf47f2dd3d8526d3bd6cc411ad315b372de38975",
118
118
  "schemas/v2/audit-verify-result.schema.json": "14af8e9283510f0a40e48a2059c929ee44e0fc9fa3cc66a95bfbe90b17c48297",
119
119
  "schemas/v2/budget-policy.schema.json": "1de68095db0beb5c7a4be629de26e79f62ebbe748a88d46f36e768edfd1f0d68",
120
- "schemas/v2/capabilities.schema.json": "b06b8142d288e839fe871bdddde35ca08d38d00da2a510fb9cb4ab201f37aa99",
120
+ "schemas/v2/capabilities.schema.json": "e580833bd80ce894ce610290de8766f15b9007642456a7404470ca81e0bf3170",
121
121
  "schemas/v2/certification-bundle.schema.json": "5cfa7fb5247160333e33a58b60d42bcd4e5269d8391d6a112d301e14828892ed",
122
122
  "schemas/v2/channel-presence-payload.schema.json": "fc03df0c20cc12a25f7ee6a47503a4c5931d79fe3d5ff24815e8fd7c1bcb9a87",
123
123
  "schemas/v2/channel-written-payload.schema.json": "ccecff3c71a3275ad8db035ac0e04db25e6f8d23cf091ddbcd5a60d1be1cbb6b",
@@ -178,7 +178,7 @@
178
178
  "schemas/v2/residency.schema.json": "3fd23c9732beb140504e81560fc426eb6764d547e56da5fb92c5a5c7bf04db3a",
179
179
  "schemas/v2/run-ancestry-response.schema.json": "334c17f2062dea62726ca1d07b8be3f169cdc422af843c1def5fb317039814c9",
180
180
  "schemas/v2/run-diff-response.schema.json": "8bdb2045c274580038bd74781f7d062f5049839797c164a89de4ed89edc53a74",
181
- "schemas/v2/run-event-payloads.schema.json": "e21d0cb64cb969be32e49e87f3af418bacb40e8fcb515a848dfe7d9609d8d0b6",
181
+ "schemas/v2/run-event-payloads.schema.json": "8b4907cfd123d0b875a60899cd3c46c24a6b3af5a6a7e904070355477fa84814",
182
182
  "schemas/v2/run-event.schema.json": "2f0d7416bd376c56affe97b6b2b49981532e8e9ff45c0a162cf806c78b3e7fb7",
183
183
  "schemas/v2/run-list-response.schema.json": "c2dbc34768dcf64528914a213aa3cb55952e8e3e90a64b04a9423b2a19749d38",
184
184
  "schemas/v2/run-options.schema.json": "f0e9f494d80086c2ed2cf9fe87df54e5b4e66601699c4a1089f4edbc7ba58867",
@@ -212,14 +212,14 @@
212
212
  "schemas/workspace-file.schema.json": "464de85c2a068243084ee9c1d969bc7cd5d8f7948574e58450d6493c38a0e1e4",
213
213
  "spec/v1/alias-detectors.json": "22444c22bb88c568339ef512553f9fc38146e22b566d11ab8608f655d688d19c",
214
214
  "spec/v1/capability-declaration-classes.json": "e7729aed5c4b4e1dd02abab0530f14cc95f5d4070fe51fb139e7f5cccefa00c6",
215
- "spec/v1/core-standard-manifest.json": "e341bdbad39a25a8c04f535b1dc44d581b5473069c5ebbe786f5f8031f61a679",
215
+ "spec/v1/core-standard-manifest.json": "c8cca62eac811cb159b789202137f01c658ea8a88b2a4d7c3b450f28c7cb997f",
216
216
  "spec/v1/deprecations.json": "e723bc11d7272c71789ad1303bb7f21809a8a024abf57594f9d5c35dc199ece9",
217
217
  "spec/v1/deprecations.schema.json": "3e393c405d2a41b467d8c5e3c468549078df1ce6a6d2588fc488097b95d9b55b",
218
218
  "spec/v1/end-of-support.json": "35f0bf3b70fe78b16e2a2fd81a153fc27e703f6c16376c6fe7cdf1f4f2c4e6d4",
219
219
  "spec/v1/event-codemap.json": "3da60d884157793a360da532a9fcbbfb5285636db325a74cec94b34622186d97",
220
220
  "spec/v1/event-codemap.schema.json": "d05933b2e88103aff51a2f774df97b9bda065e0fc88dbd9b114cc47a32189174",
221
221
  "spec/v1/extensions.json": "79a60754aa16cbdbdb604f8e5c00038af535a1690a4d07bd5ff7c38f0256a7bc",
222
- "spec/v1/gaps.json": "16ab681ea051c41104d53161bcf179704f80659861f7dd3ba8bdbfe3015f71b8",
222
+ "spec/v1/gaps.json": "c2f6e7de84e8f59bc35e59e5afc39ac1359b78c5b857d6c8b45a46803240362c",
223
223
  "spec/v1/gaps.schema.json": "8fd83259f556553c9df0f53e7a82ca8c2a4771d471197f69b4e2ae0ceeacfd66",
224
224
  "spec/v1/migrations.json": "4c50760be70cb644d02427bcaf5749606700edf4513f423903cf3b1d12f5c300",
225
225
  "spec/v1/migrations.schema.json": "886779aa6c22e646db097f5df210adb018a4dd14a7b815465a18c8a7056c8f72",
@@ -285,7 +285,7 @@
285
285
  "spec/v2/ext/sandbox-runtime-notes/README.md": "0d1290fb1e20b2e35c0b15051541ab6546e4e2a6f693b4f912cb3f7d6a65e92c",
286
286
  "spec/v2/ext/webResearch/README.md": "b675e6f6646128d781b661f754c2393eed86e5935d16c3a22df582a5ce025b9b",
287
287
  "spec/v2/facets/a2a.schema.json": "b68bf5e29e455e46c2ad7cfcccf5963730ec2310ff9db81884550d8df30fd6ef",
288
- "spec/v2/facets/agents.schema.json": "198c01fdbb09deeb84f8f45f9f47976e696ac35ef087a2bc5c6f3c4fbb2ac370",
288
+ "spec/v2/facets/agents.schema.json": "cd6bf5cd1d3d9b61646e0232027f078113e44dd4d0beec3283318fb6c5a5e296",
289
289
  "spec/v2/facets/aiProviders.schema.json": "6e4327bcd48110686cc1adfd26ddd6fbbf4ff529c476dcc74eeff8c5ae9965ce",
290
290
  "spec/v2/facets/auditLogIntegrity.schema.json": "9a7f81320c8f940c6f4129bc7dd895a4063231c5458a717a08c2cbf9bb886c46",
291
291
  "spec/v2/facets/auth.schema.json": "7d98bc81a0f7d307460d039eebfebd21388091bfe1d538762583940f94c0bd2a",
@@ -318,9 +318,9 @@
318
318
  "spec/v2/path-manifest.json": "20f598f56df460ad3ed50aba3d6bc2933ea16a3c96ce125da1c5dd6989d4155c",
319
319
  "spec/v2/peer-dependency-aliases.json": "d10299280abee08258502925bc327293ee413e0108cd6e6ec75ff6110653308d",
320
320
  "spec/v2/profiles.json": "b382b261c7844798575f7c1b37c934f0da19846016a2337d883d4f5ef6e05479",
321
- "spec/v2/release.json": "5bb12627a01a0f732dc8ab1f6c7f8d23ed4f583782b83591a1d72803be9592bc",
321
+ "spec/v2/release.json": "8ce37c12db023b43d7a517a703130cc8baaafcf46dc49ddf9af8339c80321b59",
322
322
  "spec/v2/retention-floors.json": "eaf3722d95c79947af1d4269ef85117e126518c588cfcf1a2b21b97269f51624",
323
- "spec/v2/surface-baseline.json": "da513617337bdcf1073907d88662f3c853110bb0107e789ea3cfd245e86a0933"
323
+ "spec/v2/surface-baseline.json": "920c6c86508e70e620595d76a45b6cd428c9bd665d0208b375c33ace60556fae"
324
324
  },
325
- "corpusCommit": "2c254f70680834a1ed087ba9369dcdbfd1c0a40d"
325
+ "corpusCommit": "2c0733837695218c74a0875305c704eb500a1665"
326
326
  }
@@ -0,0 +1,88 @@
1
+ /**
2
+ * RFC 0252 — an eval task takes its fixtures from a recorded run. Pure judges
3
+ * over four sequential eval runs, for `v2-eval-recorded-fixtures.test.ts`.
4
+ *
5
+ * The runs go through the suite-owned counter tool `conformance.eval:counter`,
6
+ * whose live `inc` changes persistent tenant-scoped state (contract in
7
+ * `fixtures.md`):
8
+ * 1. source `conformance-eval-recorded-source` (live): read P, inc R, read R.
9
+ * 2. replay `conformance-eval-recorded-replay`, `fixtureRuns: {record: source}`:
10
+ * the same script, served the source's recorded answers.
11
+ * 3. drift `conformance-eval-recorded-drift`, the same source: read P,
12
+ * inc P, read R. The `inc P` was never recorded, so it misses.
13
+ * 4. probe the source suite again, live.
14
+ * The probe's `inc R` is the source's plus one when neither the replay nor
15
+ * the drift reached the live counter, and its `read P` equals the source's
16
+ * when the drift's missed `inc P` never ran. Persistence not shown, or a
17
+ * counter that moved by more than the replay could explain (a concurrent cut
18
+ * on the tenant), is `blocked`, never a failure.
19
+ *
20
+ * @see RFCS/0252-eval-fixtures-from-a-recorded-run.md
21
+ */
22
+
23
+ import type { CounterCall } from './eval-counter-witness.js';
24
+
25
+ export interface RecordedFinding { readonly ok: boolean; readonly message: string }
26
+ export type RecordedVerdict =
27
+ | { readonly kind: 'blocked'; readonly reason: string }
28
+ | { readonly kind: 'judged'; readonly findings: readonly RecordedFinding[] };
29
+
30
+ type Rec = Record<string, unknown>;
31
+ const isRec = (v: unknown): v is Rec => typeof v === 'object' && v !== null && !Array.isArray(v);
32
+ const count = (c: CounterCall | undefined): number | undefined => {
33
+ const o = c?.outcome;
34
+ return isRec(o) && typeof o['count'] === 'number' && Number.isInteger(o['count']) ? o['count'] : undefined;
35
+ };
36
+
37
+ export const TASK = 'record';
38
+
39
+ /** The summary entry for task `record`. */
40
+ export function recordTask(summary: Rec): Rec | undefined {
41
+ const tasks = summary['tasks'];
42
+ return Array.isArray(tasks) ? tasks.find((t): t is Rec => isRec(t) && t['taskId'] === TASK) : undefined;
43
+ }
44
+
45
+ /** A source or probe run: read P, inc R, read R, all answered live with `{key, count}`. */
46
+ function liveCounts(calls: readonly CounterCall[], label: string): { P: number; R: number } | { reason: string } {
47
+ if (calls.length !== 3) return { reason: `the ${label} run MUST log its three scripted counter calls (got ${calls.length})` };
48
+ const [p, r] = [count(calls[0]), count(calls[1])];
49
+ if (p === undefined || r === undefined) return { reason: `the ${label} run's counter calls did not all return {key, count} live` };
50
+ return { P: p, R: r };
51
+ }
52
+
53
+ /** §E: `eval.started.fixtureRuns` is the accepted map verbatim. */
54
+ export function judgeEchoed(events: readonly { type: string; payload: Rec }[], sent: Rec): RecordedFinding[] {
55
+ const started = events.filter((e) => e.type === 'eval.started');
56
+ const got = started[0]?.payload['fixtureRuns'];
57
+ return [{ ok: started.length === 1 && JSON.stringify(got) === JSON.stringify(sent), message: `eval.started MUST echo fixtureRuns ${JSON.stringify(sent)} (got ${JSON.stringify(got)})` }];
58
+ }
59
+
60
+ /** §D: the replay was served the source's answers, missed nothing, and the probe shows no live inc R. */
61
+ export function judgeReplayed(source: readonly CounterCall[], replay: readonly CounterCall[], replayTask: Rec | undefined, probe: readonly CounterCall[]): RecordedVerdict {
62
+ const s = liveCounts(source, 'source');
63
+ if ('reason' in s) return { kind: 'blocked', reason: s.reason };
64
+ const p = liveCounts(probe, 'probe');
65
+ if ('reason' in p) return { kind: 'blocked', reason: p.reason };
66
+ if (p.R > s.R + 2) return { kind: 'blocked', reason: `counter R moved from ${s.R} to ${p.R} across the replay: more than this scenario can cause, so another run on the tenant is using it` };
67
+ if (p.R <= s.R) return { kind: 'blocked', reason: `counter R did not persist across runs (source ${s.R}, probe ${p.R})` };
68
+ const same = replay.length === source.length && replay.every((c, i) => c.returned && JSON.stringify(c.outcome) === JSON.stringify(source[i]!.outcome));
69
+ return { kind: 'judged', findings: [
70
+ { ok: same, message: `each replayed counter call MUST return the source run's recorded outcome (source ${JSON.stringify(source.map((c) => c.outcome))}, replay ${JSON.stringify(replay.map((c) => c.outcome))})` },
71
+ { ok: replayTask?.['fixtureMissCount'] === 0, message: `a replay of the same script MUST miss nothing (fixtureMissCount ${JSON.stringify(replayTask?.['fixtureMissCount'])})` },
72
+ { ok: p.R === s.R + 1, message: `the replay MUST NOT reach the live counter: the probe's inc R MUST be the source's plus one (source ${s.R}, probe ${p.R})` },
73
+ ] };
74
+ }
75
+
76
+ /** §D: an unrecorded call misses, the task fails, and the missed inc P never ran. */
77
+ export function judgeDriftFailsClosed(source: readonly CounterCall[], driftTask: Rec | undefined, driftPassed: unknown, probe: readonly CounterCall[]): RecordedVerdict {
78
+ const s = liveCounts(source, 'source');
79
+ if ('reason' in s) return { kind: 'blocked', reason: s.reason };
80
+ const p = liveCounts(probe, 'probe');
81
+ if ('reason' in p) return { kind: 'blocked', reason: p.reason };
82
+ if (p.R <= s.R) return { kind: 'blocked', reason: `counter R did not persist across runs (source ${s.R}, probe ${p.R}), so an unchanged P proves nothing` };
83
+ return { kind: 'judged', findings: [
84
+ { ok: driftTask?.['fixtureMissCount'] === 1, message: `the one unrecorded call MUST be counted as a miss (fixtureMissCount ${JSON.stringify(driftTask?.['fixtureMissCount'])})` },
85
+ { ok: driftPassed === false, message: `a task with a miss under the derived onMiss "fail" MUST NOT pass (passed ${JSON.stringify(driftPassed)})` },
86
+ { ok: p.P === s.P, message: `the missed inc P MUST NOT reach the live counter: the probe MUST read P as the source did (source ${s.P}, probe ${p.P})` },
87
+ ] };
88
+ }
@@ -63,7 +63,7 @@ export function judgeRefused(a: Answer, suppliedId: string | undefined, what: st
63
63
  { ok: a.status === 400, message: `${what} MUST be refused 400 before the run starts (got ${a.status})` },
64
64
  { ok: readErrorCode(a.json) === 'validation_error', message: `${what} MUST be refused with validation_error (got ${JSON.stringify(readErrorCode(a.json))})` },
65
65
  ];
66
- if (suppliedId !== undefined) out.push({ ok: !a.text.includes(suppliedId), message: `the refusal of ${what} MUST NOT echo the supplied baselineRunId` });
66
+ if (suppliedId !== undefined) out.push({ ok: !a.text.includes(suppliedId), message: `the refusal of ${what} MUST NOT echo the supplied run id` });
67
67
  return out;
68
68
  }
69
69
 
@@ -0,0 +1,218 @@
1
+ /**
2
+ * v2 — an eval task takes its fixtures from a recorded run (RFC 0252; the
3
+ * rules are the `agents.evalSuite.recordedFixtures` facet description in
4
+ * `spec/v2/facets/agents.schema.json`, the `createRun` `fixtureRuns`
5
+ * description and `eval.started.fixtureRuns`).
6
+ *
7
+ * Server-free leg:
8
+ * 1. `0252.shape` — the facet member is a boolean, `eval.started` carries
9
+ * `fixtureRuns` (taskId → runId, non-empty), and the derived `createRun`
10
+ * eval body carries `fixtureRuns`.
11
+ *
12
+ * Host legs — gated on `recordedFixtures`, `fixtureMatching`, and the three
13
+ * `conformance-eval-recorded-*` fixtures in discovery `fixtures`. Four runs
14
+ * through the reserved counter tool, in order, once per worker: source,
15
+ * replay (`fixtureRuns: {record: source}`), drift (the same source), and the
16
+ * source suite again as a probe. Judges in `lib/eval-recorded-fixtures-witness.ts`.
17
+ * 2. `0252.source-replayed` — `eval.started` echoes the map; each replayed
18
+ * call returns the source's outcome; no miss; the probe shows the replay
19
+ * never reached the live counter.
20
+ * 3. `0252.drift-fails-closed` — the unrecorded `inc P` misses, the task
21
+ * fails, and the probe reads P unchanged.
22
+ * 4. `0252.source-refused` — an unknown id, and a `conformance-noop`
23
+ * workflow run (when listed) that never invoked the agent, are refused
24
+ * `400 validation_error` without the id.
25
+ * 5. `0252.source-uniform` — tenant B's source run
26
+ * (`OPENWOP_TEST_TENANT_B_API_KEY`) is refused exactly as the unknown id.
27
+ * 6. `0252.fields-refused` — a key naming no task, a target task that
28
+ * carries its own fixtures, and `fixtureRuns` without `mode`.
29
+ *
30
+ * @see RFCS/0252-eval-fixtures-from-a-recorded-run.md
31
+ */
32
+
33
+ import { describe, it, expect } from 'vitest';
34
+ import { readFileSync } from 'node:fs';
35
+ import { join } from 'node:path';
36
+ import { randomBytes } from 'node:crypto';
37
+ import { driver, type OpenWOPResponse } from '../lib/driver.js';
38
+ import { v2Discovery, v2RefValidator } from '../lib/v2.js';
39
+ import { softSkip } from '../lib/soft-skip.js';
40
+ import { req } from '../lib/requirement-ids.js';
41
+ import { readErrorCode } from '../lib/error-envelope.js';
42
+ import { majorProfile } from '../lib/major-profile.js';
43
+ import { isFixtureAdvertised } from '../lib/fixtures.js';
44
+ import { API_DIR } from '../lib/paths.js';
45
+ import { pollTerminal, readEvents, isRecord, type Ev } from '../lib/fixture-run-observer.js';
46
+ import { counterCalls, type CounterCall } from '../lib/eval-counter-witness.js';
47
+ import { judgeRefused, judgeUniform, type Answer } from '../lib/eval-regression-witness.js';
48
+ import { judgeDriftFailsClosed, judgeEchoed, judgeReplayed, recordTask, TASK, type RecordedFinding, type RecordedVerdict } from '../lib/eval-recorded-fixtures-witness.js';
49
+
50
+ const PROFILE = majorProfile(2);
51
+ const UNREADABLE = 'v2 discovery unreachable — /.well-known/openwop did not answer 200 JSON under OpenWOP-Version: 2.0';
52
+ const DOC = 'agents.schema.json §evalSuite.recordedFixtures, createRun fixtureRuns, eval.started.fixtureRuns (RFC 0252)';
53
+ const AGENT = 'core.conformance.agents.scripted';
54
+ const SUITE_URI = (id: string) => `https://conformance.openwop.dev/eval-suites/${id}`;
55
+ const SOURCE = 'conformance-eval-recorded-source';
56
+ const REPLAY = 'conformance-eval-recorded-replay';
57
+ const DRIFT = 'conformance-eval-recorded-drift';
58
+ const ID_SHAPE = 'openwop.requirement.0252.shape';
59
+ const ID_REPLAYED = 'openwop.requirement.0252.source-replayed';
60
+ const ID_DRIFT = 'openwop.requirement.0252.drift-fails-closed';
61
+ const ID_REFUSED = 'openwop.requirement.0252.source-refused';
62
+ const ID_UNIFORM = 'openwop.requirement.0252.source-uniform';
63
+ const ID_FIELDS = 'openwop.requirement.0252.fields-refused';
64
+
65
+ type Skip = { skip: [Parameters<typeof softSkip>[0], string] };
66
+ type Done = { runId: string; events: Ev[]; summary: Record<string, unknown>; calls: CounterCall[] };
67
+ type Runs = { source: Done; replay: Done; drift: Done; probe: Done };
68
+
69
+ async function http(fn: () => Promise<OpenWOPResponse>): Promise<OpenWOPResponse | null> {
70
+ try { return await fn(); } catch { return null; }
71
+ }
72
+ const answer = (r: OpenWOPResponse): Answer => ({ status: r.status, json: r.json, text: r.text });
73
+ const expectAll = (id: string, findings: readonly RecordedFinding[]): void => {
74
+ for (const x of findings) expect(x.ok, req(id, DOC, x.message)).toBe(true);
75
+ };
76
+ const expectVerdict = (id: string, v: RecordedVerdict): void => {
77
+ if (v.kind === 'blocked') return softSkip('blocked', v.reason);
78
+ expectAll(id, v.findings);
79
+ };
80
+
81
+ async function gate(): Promise<Skip | null> {
82
+ let doc: Record<string, unknown> | null;
83
+ try { doc = await v2Discovery(); } catch { return { skip: ['blocked', UNREADABLE] }; }
84
+ const agents = doc?.['agents'];
85
+ const facet = isRecord(agents) && isRecord(agents['evalSuite']) ? agents['evalSuite'] : null;
86
+ if (facet === null || facet['recordedFixtures'] !== true) return { skip: ['inapplicable', 'the host does not advertise agents.evalSuite.recordedFixtures'] };
87
+ if (facet['fixtureMatching'] !== true) return { skip: ['inapplicable', 'the host advertises recordedFixtures without fixtureMatching'] };
88
+ for (const id of [SOURCE, REPLAY, DRIFT]) if (!isFixtureAdvertised(id)) return { skip: ['inapplicable', `the host does not advertise fixture ${id}`] };
89
+ return null;
90
+ }
91
+
92
+ const create = (id: string, extra: Record<string, unknown>, headers?: Record<string, string>) =>
93
+ http(() => driver.post(PROFILE.runsPath, { mode: 'eval', evalSuiteRef: SUITE_URI(id), agentId: AGENT, ...extra }, headers ? { headers } : {}));
94
+
95
+ async function complete(id: string, extra: Record<string, unknown>): Promise<Skip | Done> {
96
+ const res = await create(id, extra);
97
+ if (res === null) return { skip: ['blocked', 'POST /runs unreachable (fetch failed)'] };
98
+ if (res.status === 429) return { skip: ['blocked', 'POST /runs answered 429 — the run budget, not the wire'] };
99
+ const runId = isRecord(res.json) && typeof res.json['runId'] === 'string' ? res.json['runId'] : undefined;
100
+ if ((res.status !== 201 && res.status !== 202) || runId === undefined) return { skip: ['blocked', `createRun of ${id} answered ${res.status} ${readErrorCode(res.json) ?? ''}`.trim()] };
101
+ const terminal = await pollTerminal(PROFILE, runId, { timeoutMs: 60_000 });
102
+ if (terminal.snapshot['status'] !== 'completed') return { skip: ['blocked', `eval run ${runId} ended ${String(terminal.snapshot['status'])}, not completed`] };
103
+ const s = await http(() => driver.get(`${PROFILE.runsPath}/${PROFILE.idSegment(runId)}/eval-summary`));
104
+ if (s === null || s.status !== 200 || !isRecord(s.json)) return { skip: ['blocked', `GET …/eval-summary answered ${s?.status ?? 'nothing'}`] };
105
+ const events = await readEvents(PROFILE, runId);
106
+ return { runId, events, summary: s.json, calls: counterCalls(events, AGENT) };
107
+ }
108
+
109
+ let seq: Promise<Skip | Runs> | undefined;
110
+ /** Source → replay → drift → probe, strictly in order, once per worker: the host legs read the same runs. */
111
+ function runs(): Promise<Skip | Runs> {
112
+ seq ??= (async (): Promise<Skip | Runs> => {
113
+ const source = await complete(SOURCE, {});
114
+ if ('skip' in source) return source;
115
+ const replay = await complete(REPLAY, { fixtureRuns: { [TASK]: source.runId } });
116
+ if ('skip' in replay) return replay;
117
+ const drift = await complete(DRIFT, { fixtureRuns: { [TASK]: source.runId } });
118
+ if ('skip' in drift) return drift;
119
+ const probe = await complete(SOURCE, {});
120
+ if ('skip' in probe) return probe;
121
+ return { source, replay, drift, probe };
122
+ })();
123
+ return seq;
124
+ }
125
+
126
+ /** An id in the caller's own tenant, with the runId grammar, that the host never minted. */
127
+ const unknownBeside = (runId: string): string => `${runId.split('/')[0]}/${randomBytes(16).toString('hex')}`;
128
+
129
+ describe('v2 RFC 0252 — wire shape (server-free)', () => {
130
+ it('the recordedFixtures member, eval.started.fixtureRuns and the createRun property are in the derived contract', () => {
131
+ const facet = v2RefValidator('capabilities.schema.json#/properties/agents/properties/evalSuite/properties/recordedFixtures');
132
+ expect(facet(true).ok, req(ID_SHAPE, DOC, 'recordedFixtures: true MUST validate')).toBe(true);
133
+ expect(facet('yes').ok, req(ID_SHAPE, DOC, 'recordedFixtures MUST be a boolean')).toBe(false);
134
+ const started = v2RefValidator('run-event-payloads.schema.json#/$defs/evalStarted');
135
+ const base = { suiteId: 'core.openwop.evals.x', suiteVersion: '1.0.0', taskCount: 1, modes: ['golden'] };
136
+ const r = started({ ...base, fixtureRuns: { record: 'conformance-tenant/0123456789abcdef0123456789abcdef' } });
137
+ expect(r.ok, req(ID_SHAPE, DOC, `eval.started with fixtureRuns MUST validate: ${r.errors}`)).toBe(true);
138
+ expect(started({ ...base, fixtureRuns: {} }).ok, req(ID_SHAPE, DOC, 'an empty fixtureRuns MUST be refused')).toBe(false);
139
+ expect(started({ ...base, fixtureRuns: { record: 42 } }).ok, req(ID_SHAPE, DOC, 'a fixtureRuns value MUST be a runId')).toBe(false);
140
+ const api = readFileSync(join(API_DIR, 'v2', 'openapi.yaml'), 'utf8');
141
+ const at = api.indexOf('operationId: createRun');
142
+ const next = api.indexOf('operationId:', at + 1);
143
+ const block = at < 0 ? '' : api.slice(at, next < 0 ? undefined : next);
144
+ expect(block, req(ID_SHAPE, DOC, 'createRun MUST carry fixtureRuns, a non-empty map onto runIds')).toMatch(/fixtureRuns:\s*\n\s*type: object\s*\n\s*minProperties: 1[\s\S]*?additionalProperties:\s*\n\s*\$ref: \.\.\/\.\.\/schemas\/v2\/ids\.schema\.json#\/\$defs\/runId/);
145
+ });
146
+ });
147
+
148
+ describe('v2 RFC 0252 — an eval task takes its fixtures from a recorded run (host)', () => {
149
+ it('the replay is served the source run\'s recorded answers and never reaches the live tool', async () => {
150
+ const g = await gate();
151
+ if (g) return softSkip(...g.skip);
152
+ const r = await runs();
153
+ if ('skip' in r) return softSkip(...r.skip);
154
+ expectAll(ID_REPLAYED, judgeEchoed(r.replay.events, { [TASK]: r.source.runId }));
155
+ expectVerdict(ID_REPLAYED, judgeReplayed(r.source.calls, r.replay.calls, recordTask(r.replay.summary), r.probe.calls));
156
+ });
157
+
158
+ it('a call the source never made misses, fails the task, and never runs live', async () => {
159
+ const g = await gate();
160
+ if (g) return softSkip(...g.skip);
161
+ const r = await runs();
162
+ if ('skip' in r) return softSkip(...r.skip);
163
+ const task = recordTask(r.drift.summary);
164
+ expectVerdict(ID_DRIFT, judgeDriftFailsClosed(r.source.calls, task, task?.['passed'], r.probe.calls));
165
+ });
166
+
167
+ it('an unknown source, and a run that never invoked the agent, are refused without the id', async () => {
168
+ const g = await gate();
169
+ if (g) return softSkip(...g.skip);
170
+ const r = await runs();
171
+ if ('skip' in r) return softSkip(...r.skip);
172
+ const unknownId = unknownBeside(r.source.runId);
173
+ const unknown = await create(REPLAY, { fixtureRuns: { [TASK]: unknownId } });
174
+ if (unknown === null) return softSkip('blocked', 'POST /runs unreachable (fetch failed)');
175
+ expectAll(ID_REFUSED, judgeRefused(answer(unknown), unknownId, 'an unknown source run'));
176
+ // The second case needs a workflow run of the scripted fixture; without conformance-noop only the unknown id is judged.
177
+ const wf = isFixtureAdvertised('conformance-noop') ? await http(() => driver.post(PROFILE.runsPath, { workflowId: 'conformance-noop' })) : null;
178
+ const wfId = wf && isRecord(wf.json) && typeof wf.json['runId'] === 'string' ? wf.json['runId'] : undefined;
179
+ if (wfId !== undefined) {
180
+ await pollTerminal(PROFILE, wfId, { timeoutMs: 30_000 });
181
+ const noAgent = await create(REPLAY, { fixtureRuns: { [TASK]: wfId } });
182
+ if (noAgent !== null) expectAll(ID_REFUSED, judgeRefused(answer(noAgent), wfId, 'a source run that never invoked the agent'));
183
+ }
184
+ });
185
+
186
+ it("another tenant's run is refused exactly as an id that never existed", async () => {
187
+ const g = await gate();
188
+ if (g) return softSkip(...g.skip);
189
+ const tenantB = process.env['OPENWOP_TEST_TENANT_B_API_KEY']?.trim();
190
+ if (!tenantB) return softSkip('blocked', 'OPENWOP_TEST_TENANT_B_API_KEY (a credential bound to a second tenant) is not set');
191
+ if (tenantB === process.env['OPENWOP_API_KEY']) return softSkip('blocked', 'OPENWOP_TEST_TENANT_B_API_KEY equals OPENWOP_API_KEY — the leg needs a second tenant');
192
+ const foreign = await create(SOURCE, {}, { Authorization: `Bearer ${tenantB}` });
193
+ const foreignId = foreign && isRecord(foreign.json) && typeof foreign.json['runId'] === 'string' ? foreign.json['runId'] : undefined;
194
+ if (foreignId === undefined) return softSkip('blocked', `tenant B's createRun of ${SOURCE} answered ${foreign?.status ?? 'nothing'}`);
195
+ const r = await runs();
196
+ if ('skip' in r) return softSkip(...r.skip);
197
+ const unknown = await create(REPLAY, { fixtureRuns: { [TASK]: unknownBeside(r.source.runId) } });
198
+ const viaForeign = await create(REPLAY, { fixtureRuns: { [TASK]: foreignId } });
199
+ if (unknown === null || viaForeign === null) return softSkip('blocked', 'POST /runs unreachable (fetch failed)');
200
+ expectAll(ID_UNIFORM, judgeUniform(answer(unknown), answer(viaForeign), foreignId));
201
+ });
202
+
203
+ it('a key naming no task, a target with its own fixtures, and fixtureRuns without mode are refused', async () => {
204
+ const g = await gate();
205
+ if (g) return softSkip(...g.skip);
206
+ const r = await runs();
207
+ if ('skip' in r) return softSkip(...r.skip);
208
+ const map = { [TASK]: r.source.runId };
209
+ const noTask = await create(REPLAY, { fixtureRuns: { nope: r.source.runId } });
210
+ const ownFixtures = await create(SOURCE, { fixtureRuns: map });
211
+ if (noTask === null || ownFixtures === null) return softSkip('blocked', 'POST /runs unreachable (fetch failed)');
212
+ expectAll(ID_FIELDS, judgeRefused(answer(noTask), undefined, 'a fixtureRuns key naming no task of the suite'));
213
+ expectAll(ID_FIELDS, judgeRefused(answer(ownFixtures), undefined, 'a named task whose suite fixtures carry toolResponses or onMiss'));
214
+ // A workflow create needs a workflow the host holds, or its 404 would answer first.
215
+ const noMode = isFixtureAdvertised('conformance-noop') ? await http(() => driver.post(PROFILE.runsPath, { workflowId: 'conformance-noop', fixtureRuns: map })) : null;
216
+ if (noMode !== null) expectAll(ID_FIELDS, judgeRefused(answer(noMode), undefined, 'fixtureRuns without mode eval'));
217
+ });
218
+ });
@@ -239,7 +239,7 @@ describe('v2 RFC 0244 — wire shape (server-free)', () => {
239
239
  });
240
240
 
241
241
  it('the eval-suite fixtures validate against the v2 suite schema', () => {
242
- for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss', 'conformance-eval-trajectory-gate', 'conformance-eval-trajectory-gate-mean', 'conformance-eval-fixture-error', 'conformance-eval-counter-live', 'conformance-eval-counter-miss', 'conformance-eval-efficiency-ungated', 'conformance-eval-memory-isolation', 'conformance-eval-gate-failing', 'conformance-eval-regression-base', 'conformance-eval-regression-candidate']) {
242
+ for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss', 'conformance-eval-trajectory-gate', 'conformance-eval-trajectory-gate-mean', 'conformance-eval-fixture-error', 'conformance-eval-counter-live', 'conformance-eval-counter-miss', 'conformance-eval-efficiency-ungated', 'conformance-eval-memory-isolation', 'conformance-eval-gate-failing', 'conformance-eval-regression-base', 'conformance-eval-regression-candidate', 'conformance-eval-recorded-source', 'conformance-eval-recorded-replay', 'conformance-eval-recorded-drift']) {
243
243
  const r = suiteSchema(fixture(id));
244
244
  expect(r.ok, req(ID_FIXTURES, 'fixtures.md §eval-suites', `${id} MUST validate: ${r.errors}`)).toBe(true);
245
245
  }