@openwop/openwop-conformance 2.46.15 → 2.46.17

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [2.46.17] — 2026-10-18 — RFC 0245: a recorded tool error replays as an error
+
+- **New fixture `fixtures/eval-suites/conformance-eval-fixture-error`.** One task calls `conformance.eval:lookup` twice. Its fixtures serve an `isError: true` entry, then a success.
+- **New legs in `v2-eval-trajectory-trials`:**
+  - `0245.fixture-error-shape` (server-free): an `isError` entry validates, and a string is refused.
+  - `0245.fixture-error-replayed`, gated on `fixtureErrors` + `fixtureMatching` and the fixture: the first call's `agent.tool-returned` carries `error {code, message}` and no `outcome` (`status: "error"` under tool hooks), the second carries `outcome`, and `fixtureMissCount` is 0.
+  - `0245.unadvertised-refused`: without `fixtureErrors`, the suite is refused with `400 validation_error`.
+  The judge is `lib/eval-fixture-error-witness.ts`; its self-tests fail each wrong host answer.
+- **Host impact: none today.** No host advertises `fixtureErrors`, so the replay leg records `inapplicable`. The refusal leg binds on a host that lists the fixture and advertises `evalSuite` without the member.
+
+## [2.46.16] — 2026-10-18 — RFC 0228 Accepted: gap G4 closes
+
+- **No assertion changes.** The packed `spec/v1/gaps.json` changes: RFC 0228 G4 (a host witness of `v2-table-schema-enforcement`) closes on openwop-app's certified 2.46.14 production cut (`ddcf7f944`), and RFC 0228 goes `Accepted`.
+
 ## [2.46.15] — 2026-10-17 — RFC 0228: gap G7 closes
 
 - **No assertion changes.** The packed `spec/v1/gaps.json` changes: RFC 0228 G7 (MyndHyve's run-creation `validation_error` leg) closes on MyndHyve's certified 2.46.13 cut (`c8d511d84`), which contains myndhyve#597, with a live probe of that build. Every RFC 0228 rename row is now closed.

package/dist/spec-artifacts.lock.json

@@ -1,5 +1,5 @@
 {
   "package": "@openwop/spec-artifacts",
-  "version": "2.46.15",
-  "stampSha256": "6208e336d886b7202d3cf537bf09c85e89c4384e0d31ed133722abc4c6314fdc"
+  "version": "2.46.17",
+  "stampSha256": "d8df0298a5a6405eca16d0a24cb7c48123d7e3e0675b337eca37ad36585e02f6"
 }

package/fixtures/eval-suites/conformance-eval-fixture-error.json

@@ -0,0 +1,68 @@
+{
+  "suiteId": "core.openwop.evals.conformance-eval-fixture-error",
+  "version": "1.0.0",
+  "targetAgentId": "core.conformance.agents.scripted",
+  "modes": [
+    "golden"
+  ],
+  "tasks": [
+    {
+      "taskId": "recovers",
+      "input": {
+        "script": [
+          {
+            "tool": "conformance.eval:lookup",
+            "arguments": {
+              "orderId": "o-1"
+            }
+          },
+          {
+            "tool": "conformance.eval:lookup",
+            "arguments": {
+              "orderId": "o-1"
+            }
+          }
+        ],
+        "output": "done"
+      },
+      "expected": {
+        "kind": "golden",
+        "match": {
+          "strategy": "contains",
+          "value": "done"
+        }
+      },
+      "fixtures": {
+        "onMiss": "fail",
+        "toolResponses": [
+          {
+            "tool": "conformance.eval:lookup",
+            "args": {
+              "orderId": "o-1"
+            },
+            "argsMatch": "exact",
+            "isError": true,
+            "response": {
+              "content": [
+                {
+                  "type": "text",
+                  "text": "{\"error\":\"upstream_down\"}"
+                }
+              ]
+            }
+          },
+          {
+            "tool": "conformance.eval:lookup",
+            "args": {
+              "orderId": "o-1"
+            },
+            "argsMatch": "exact",
+            "response": {
+              "status": "shipped"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

package/fixtures.md

@@ -74,6 +74,7 @@ All fixtures MUST advertise:
 | Eval — Fixture Miss | `conformance-eval-fixture-miss` | RFC 0244 §D — `onMiss: "fail"`; one task served by an args-matched fixture, one that misses it (`v2-eval-trajectory-trials`) | eval run `completed`; the missed task fails | ≤ 30s |
 | Eval — Trajectory Gate | `conformance-eval-trajectory-gate` | RFC 0246 — two golden tasks scoring 1 at `passScore: 0.6`; `path-matched` keeps its trajectory, `path-wrong` calls a forbidden tool (`v2-eval-trajectory-trials`) | eval run `completed`; `aggregateScore` 1, `gatedAggregateScore` 0.5, suite `passed: false` | ≤ 30s |
 | Eval — Trajectory Gate (mean) | `conformance-eval-trajectory-gate-mean` | RFC 0246 — `trials: {count: 2, passRule: "mean"}`, one golden task scoring 1 that calls a forbidden tool in both trials (`v2-eval-trajectory-trials`) | eval run `completed`; `meanScore` 1, `gatedScore` 0, task `passed: false` | ≤ 30s |
+| Eval — Fixture Error | `conformance-eval-fixture-error` | RFC 0245 — `onMiss: "fail"`; one task calls the same tool twice, served first by an `isError: true` entry, then by a success (`v2-eval-trajectory-trials`) | eval run `completed`; the first call is recorded as a tool error | ≤ 30s |
 | Dispatch Fan-Out Waves | `conformance-dispatch-fanout-waves` | RFC 0243 §A.3 — a parallel `core.dispatch` (`maxConcurrency: 2`, `wait-all`/`collect`) whose supervisor's `mockDispatchPlan` names three `conformance-delay` children. At most `min(2, maxFanOut)` run at once and none is dropped. Gated on `agents.fanOut` admitting `wait-all` with `collect`. | `completed` | ≤ 30s |
 | Dispatch Per-Item Input | `conformance-dispatch-per-item` | RFC 0243 §A.4 — a sequential `core.dispatch` whose decision dispatches `conformance-dispatch-input-mapping-child` twice with `nextWorkerInputs` `item-a`, `item-b`. At `agents.fanOut.perItemInput: true` each child receives its item; otherwise the dispatch node fails `validation_error` and no child is dispatched. | `completed`, or `failed` without `perItemInput` | ≤ 30s |
 | Dispatch Per-Item Length Mismatch | `conformance-dispatch-per-item-mismatch` | `orchestrator-decision.schema.json` `nextWorkerInputs` — one item for two children. Every dispatch host fails the dispatch node `validation_error` and dispatches no child. | `failed` | ≤ 30s |
@@ -1034,6 +1035,7 @@ Every task targets **the scripted agent `core.conformance.agents.scripted`**. It
 | `conformance-eval-fixture-miss` | `agents.evalSuite.fixtureMatching` | `missed` reports `fixtureMissCount: 1`, `passed: false` and `score: 0` (summary and every `eval.scored`); `served` passes with no miss. That no live tool ran is not observable here (RFC 0244 G2) |
 | `conformance-eval-trajectory-gate` | `agents.evalSuite.trajectory` | RFC 0246: both tasks keep `score` 1 and `aggregateScore` 1; `path-matched` carries `gatedScore` 1 and `path-wrong` `gatedScore` 0; `gatedAggregateScore` 0.5; suite `passed: false` although `aggregateScore` clears `passScore` 0.6 |
 | `conformance-eval-trajectory-gate-mean` | `agents.evalSuite.trajectory` + `agents.evalSuite.trials` | RFC 0246: two `eval.scored` with `score` 1 and `passed: false`; `trials.meanScore` 1, `gatedScore` 0, task `passed: false`; the summary carries `gatedAggregateScore` |
+| `conformance-eval-fixture-error` | `agents.evalSuite.fixtureErrors` + `agents.evalSuite.fixtureMatching` | RFC 0245: two `agent.tool-called` and two `agent.tool-returned` for `conformance.eval:lookup`, paired by `callId`; the first return carries `error` with non-empty `code` and `message` and no `outcome` (`status: "error"` under `toolHooks.prePostEvents`), the second carries `outcome`; `recovers` reports `fixtureMissCount: 0`. Without `fixtureErrors`: `400 validation_error`. That the agent saw an error and no live tool ran is not observable here (RFC 0245 G1, RFC 0244 G2) |
 
 ## OAuth provider fixtures
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openwop/openwop-conformance",
-  "version": "2.46.15",
+  "version": "2.46.17",
   "description": "Black-box conformance suite for OpenWOP hosts (v2 current; v1 scenarios retained for the frozen v1 tree).",
   "repository": {
     "type": "git",
@@ -56,6 +56,6 @@
     "@openwop/spec-artifacts": "file:../spec-artifacts"
   },
   "peerDependencies": {
-    "@openwop/spec-artifacts": "2.46.15"
+    "@openwop/spec-artifacts": "2.46.17"
   }
 }

package/requirements.json

@@ -3,10 +3,10 @@
   "generatedFrom": "src/scenarios/*.test.ts",
   "counts": {
     "files": 746,
-    "tests": 2811,
-    "withStableId": 2811,
+    "tests": 2814,
+    "withStableId": 2814,
     "interpolatedTitles": 0,
-    "explicitIds": 2684
+    "explicitIds": 2687
   },
   "records": [
     {
@@ -31028,7 +31028,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-suite-summary-and-event-fields-validate-and-the-negatives-are-refus",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 132,
+      "line": 150,
       "title": "the RFC 0244 suite, summary and event fields validate and the negatives are refused",
       "explicitId": "openwop.requirement.0244.wire-shape",
       "citations": [
@@ -31067,7 +31067,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-additions-are-content-free-eval-summary-no-content-leak",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 155,
+      "line": 173,
       "title": "the RFC 0244 additions are content-free (eval-summary-no-content-leak)",
       "explicitId": "openwop.requirement.0244.content-free",
       "citations": [
@@ -31084,7 +31084,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-eval-suite-fixtures-validate-against-the-v2-suite-schema",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 162,
+      "line": 180,
       "title": "the eval-suite fixtures validate against the v2 suite schema",
       "explicitId": "openwop.requirement.0244.fixtures-valid",
       "citations": [
@@ -31098,7 +31098,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0246-gating-fields-validate-and-stay-in-0-1",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 169,
+      "line": 187,
       "title": "the RFC 0246 gating fields validate and stay in 0–1",
       "explicitId": "openwop.requirement.0246.gated-fields-shape",
       "citations": [
@@ -31117,10 +31117,32 @@
         }
       ]
     },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-entry-may-be-marked-iserror-and-only-as-a-boolean-rfc-0245-a",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 195,
+      "title": "a fixture entry may be marked isError, and only as a boolean (RFC 0245 §A)",
+      "explicitId": "openwop.requirement.0245.fixture-error-shape",
+      "citations": [
+        {
+          "section": "agents.schema.json §evalSuite.fixtureErrors (RFC 0245)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite.fixtureErrors (RFC 0245)",
+          "requirement": "an isError: false entry MUST validate"
+        },
+        {
+          "section": "agents.schema.json §evalSuite.fixtureErrors (RFC 0245)",
+          "requirement": "a string isError MUST be refused"
+        }
+      ]
+    },
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-trajectory-suite-is-graded-per-the-reference-grader",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 179,
+      "line": 205,
       "title": "a trajectory suite is graded per the reference grader",
       "explicitId": "openwop.requirement.0244.trajectory-graded",
       "citations": [
@@ -31154,7 +31176,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.repeated-trials-emit-one-eval-scored-per-task-per-trial",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 215,
+      "line": 241,
       "title": "repeated trials emit one eval.scored per task per trial",
       "explicitId": "openwop.requirement.0244.trials-per-task",
       "citations": [
@@ -31172,7 +31194,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-per-task-trials-block-and-trialstats-follow-the-b-formulas",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 226,
+      "line": 252,
       "title": "the per-task trials block and trialStats follow the §B formulas",
       "explicitId": "openwop.requirement.0244.trial-stats-formula",
       "citations": [
@@ -31211,7 +31233,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-miss-under-onmiss-fail-fails-the-trial-and-is-counted",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 249,
+      "line": 275,
       "title": "a fixture miss under onMiss \"fail\" fails the trial and is counted",
       "explicitId": "openwop.requirement.0244.fixture-miss-fails",
       "citations": [
@@ -31248,7 +31270,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-an-unadvertised-rfc-0244-feature-is-refused-at-createrun",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 269,
+      "line": 295,
       "title": "a suite using an unadvertised RFC 0244 feature is refused at createRun",
       "explicitId": "openwop.requirement.0244.unadvertised-feature-refused",
       "citations": [
@@ -31266,7 +31288,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.under-passrule-mean-a-task-whose-trials-failed-their-trajectory-does-not-pass-rf",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 285,
+      "line": 311,
       "title": "under passRule \"mean\" a task whose trials failed their trajectory does not pass (RFC 0246)",
       "explicitId": "openwop.requirement.0246.mean-gated",
       "citations": [
@@ -31280,7 +31302,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-suite-whose-tasks-failed-their-trajectory-does-not-pass-on-the-output-scores-r",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 298,
+      "line": 324,
       "title": "a suite whose tasks failed their trajectory does not pass on the output scores (RFC 0246)",
       "explicitId": "openwop.requirement.0246.suite-gated",
       "citations": [
@@ -31291,6 +31313,38 @@
         }
       ]
     },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-entry-marked-iserror-is-replayed-as-a-tool-error-not-a-success-rfc-024",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 335,
+      "title": "a fixture entry marked isError is replayed as a tool error, not a success (RFC 0245)",
+      "explicitId": "openwop.requirement.0245.fixture-error-replayed",
+      "citations": [
+        {
+          "section": null,
+          "requirement": null,
+          "interpolated": true
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-iserror-is-refused-at-createrun-without-fixtureerrors-rfc-0245-c",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 349,
+      "title": "a suite using isError is refused at createRun without fixtureErrors (RFC 0245 §C)",
+      "explicitId": "openwop.requirement.0245.unadvertised-refused",
+      "citations": [
+        {
+          "section": "agents.schema.json §evalSuite.fixtureErrors (RFC 0245)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite.fixtureErrors (RFC 0245)",
+          "requirement": "the refusal MUST carry validation_error"
+        }
+      ]
+    },
     {
       "id": "openwop.it.v2-event-type-closed.every-emitted-type-is-a-registered-protocol-type-or-a-vendor-type-under-an-adver",
       "file": "v2-event-type-closed.test.ts",

package/schemas/CORPUS-STAMP.json

@@ -1,17 +1,17 @@
 {
   "_comment": "Provenance of @openwop/spec-artifacts (RFC 0168 §D.2). files: SHA-256 per file; the conformance suite compares the installed peer against dist/spec-artifacts.lock.json at start.",
   "package": "@openwop/spec-artifacts",
-  "version": "2.46.15",
-  "corpusTag": "v2.46.15",
+  "version": "2.46.17",
+  "corpusTag": "v2.46.17",
   "files": {
     "api/.redocly.lint-ignore.yaml": "bf5a8350b88a72fa43f59605ed8d903ed24b6cfccda5e45509c9f6ed9ee4e712",
     "api/asyncapi.yaml": "d5ecb9ee6114582be3b1f662c84bfac9ae96dae7bacb853e461168f70a8e1c7d",
     "api/grpc/openwop.proto": "c3e72bb17cba514ee98feb6434e6c9b6ea6795bfd086489ec69fd882dd1ad977",
     "api/openapi.yaml": "ac6de5303872664b78ff419c035915593e01599e02835e777a0123de1baeb398",
     "api/redocly.yaml": "b0604c89b2ca6d5076ec25725c539dad44a741a811fe524439ee6daef8baa09f",
-    "api/seams-v2.yaml": "b65af486352308cc3fddb032c8318e8bf58757e5bf6111c3ae70906944c0018b",
-    "api/v2/asyncapi.yaml": "706f386c6db7665d69468694af6f0de452f51052b811b34bd3140accfd7b72f3",
-    "api/v2/openapi.yaml": "5c0e15d6db838252435b61411a03d283bb3a42e53b6a9929825d1c2666756865",
+    "api/seams-v2.yaml": "08c520d1ccc23d43054ee5499276d73e18f3350d0d8825d5caa3934286e7af19",
+    "api/v2/asyncapi.yaml": "4625ddc33db126fd959e48084edca58121340042a5e1e170f87a9b4851b8b2d8",
+    "api/v2/openapi.yaml": "993bda2b91c365f8b3751fd3c827de538e39ae8e4ff3fefe2452b7b261b3c110",
     "api/v2/redocly.yaml": "1e66b60e6118ad11a823bb620678be464d99dfe50a40e3e6f93ec9429b88b34c",
     "schemas/README.md": "276fc1d51eb08e14e9127b9fd27761d7046ee33c12000f3a64a801322db8bf58",
     "schemas/a2a-task-state.schema.json": "75d5049dea8bd873ff0e7546f1c60c8d36c219bec7084264360be54a8be30ae1",
@@ -208,14 +208,14 @@
     "schemas/workspace-file.schema.json": "464de85c2a068243084ee9c1d969bc7cd5d8f7948574e58450d6493c38a0e1e4",
     "spec/v1/alias-detectors.json": "22444c22bb88c568339ef512553f9fc38146e22b566d11ab8608f655d688d19c",
     "spec/v1/capability-declaration-classes.json": "e7729aed5c4b4e1dd02abab0530f14cc95f5d4070fe51fb139e7f5cccefa00c6",
-    "spec/v1/core-standard-manifest.json": "b4c09ddeab1c5bca702340cc28e7ec938fe26ed1f9537e7c24b4658fc719101d",
+    "spec/v1/core-standard-manifest.json": "d8a41123025c45515092970fae945b4cfa238d550cdfb9549cb649e740e8b7e3",
     "spec/v1/deprecations.json": "e723bc11d7272c71789ad1303bb7f21809a8a024abf57594f9d5c35dc199ece9",
     "spec/v1/deprecations.schema.json": "3e393c405d2a41b467d8c5e3c468549078df1ce6a6d2588fc488097b95d9b55b",
     "spec/v1/end-of-support.json": "35f0bf3b70fe78b16e2a2fd81a153fc27e703f6c16376c6fe7cdf1f4f2c4e6d4",
     "spec/v1/event-codemap.json": "3da60d884157793a360da532a9fcbbfb5285636db325a74cec94b34622186d97",
     "spec/v1/event-codemap.schema.json": "d05933b2e88103aff51a2f774df97b9bda065e0fc88dbd9b114cc47a32189174",
     "spec/v1/extensions.json": "79a60754aa16cbdbdb604f8e5c00038af535a1690a4d07bd5ff7c38f0256a7bc",
-    "spec/v1/gaps.json": "518cdd55e5e9fcf56157ab549f00e9990be2a416698d09310c113f26b2b676f9",
+    "spec/v1/gaps.json": "572b68c8a4926339fb94017fcd8f9c78adfaa4900dc809fb8e76b094a7260ba0",
     "spec/v1/gaps.schema.json": "8fd83259f556553c9df0f53e7a82ca8c2a4771d471197f69b4e2ae0ceeacfd66",
     "spec/v1/migrations.json": "4c50760be70cb644d02427bcaf5749606700edf4513f423903cf3b1d12f5c300",
     "spec/v1/migrations.schema.json": "886779aa6c22e646db097f5df210adb018a4dd14a7b815465a18c8a7056c8f72",
@@ -314,9 +314,9 @@
     "spec/v2/path-manifest.json": "7b5679a4dad98c80d855361d88a98c8a4e045af15e310862a05b285b815dc9fc",
     "spec/v2/peer-dependency-aliases.json": "d10299280abee08258502925bc327293ee413e0108cd6e6ec75ff6110653308d",
     "spec/v2/profiles.json": "b382b261c7844798575f7c1b37c934f0da19846016a2337d883d4f5ef6e05479",
-    "spec/v2/release.json": "e198402d77d27c62f19c9c7c0a66d18e64df2616a5c16243d6005650522ff8a5",
+    "spec/v2/release.json": "b6e1227e823697bad80029172057d535b866ca0d8faac6a1254ecfa9074b3654",
     "spec/v2/retention-floors.json": "eaf3722d95c79947af1d4269ef85117e126518c588cfcf1a2b21b97269f51624",
-    "spec/v2/surface-baseline.json": "9931a6fce192bb1eb427ddf75b53fcea6f5d3b775e40e4c9f1778d4370ba09c1"
+    "spec/v2/surface-baseline.json": "60061090d890cf22d91de7a4515e95814bde5690f9f9f843ee7efbdf0cdb1712"
   },
-  "corpusCommit": "e34b1c09bf6878c8be3a61ce4db06c4b2ddf8664"
+  "corpusCommit": "3d19f1af09cc0499a1bdfd35dac28707b16806a9"
 }

package/src/lib/eval-fixture-error-witness.ts

@@ -0,0 +1,79 @@
+/**
+ * RFC 0245 — an eval fixture entry marked `isError: true` is replayed as a
+ * tool error, never as a success. A pure judge over one eval run of the
+ * `conformance-eval-fixture-error` fixture, for
+ * `v2-eval-trajectory-trials.test.ts`.
+ *
+ * The fixture's one task scripts the same call twice, and its fixtures serve
+ * an `isError: true` entry and then a success (RFC 0244 §D: first unconsumed
+ * match). So the expected log is known in advance: two `agent.tool-called`
+ * from the scripted agent, and two `agent.tool-returned` paired to them by
+ * `callId` — the first with `error` and no `outcome`, the second with
+ * `outcome`. A served error is not a miss, so `fixtureMissCount` stays 0.
+ *
+ * That the agent saw a tool error, and that no live tool ran, are seam-gated
+ * (RFC 0245 G1, RFC 0244 G2) and not judged here.
+ *
+ * @see RFCS/0245-eval-fixtures-replay-tool-errors.md
+ */
+
+export interface FixtureErrorFinding { readonly ok: boolean; readonly doc: string; readonly message: string }
+
+type Rec = Record<string, unknown>;
+interface Event { readonly type: string; readonly sequence: number; readonly payload: Rec }
+
+const isRec = (v: unknown): v is Rec => typeof v === 'object' && v !== null && !Array.isArray(v);
+const nonEmpty = (v: unknown): boolean => typeof v === 'string' && v.length > 0;
+const f = (ok: boolean, doc: string, message: string): FixtureErrorFinding => ({ ok, doc, message });
+
+const B = 'RFC 0245 §B';
+export const FIXTURE_ERROR_TOOL = 'conformance.eval:lookup';
+export const FIXTURE_ERROR_TASK = 'recovers';
+
+export interface FixtureErrorOptions {
+  /** The scripted agent's id; tool events from any other agent are ignored. */
+  readonly agentId: string;
+  /** Discovery `toolHooks.prePostEvents === true`: the error return also carries `status: "error"`. */
+  readonly prePostEvents: boolean;
+}
+
+export function judgeFixtureErrorReplayed(events: readonly Event[], summary: Rec, opts: FixtureErrorOptions): FixtureErrorFinding[] {
+  const mine = (e: Event): boolean => e.payload['agentId'] === opts.agentId && e.payload['toolName'] === FIXTURE_ERROR_TOOL;
+  const ordered = [...events].sort((a, b) => a.sequence - b.sequence);
+  const called = ordered.filter((e) => e.type === 'agent.tool-called' && mine(e));
+  const returned = ordered.filter((e) => e.type === 'agent.tool-returned' && mine(e));
+  const out: FixtureErrorFinding[] = [];
+
+  out.push(f(called.length === 2, B, `the log MUST carry two agent.tool-called for ${FIXTURE_ERROR_TOOL} (got ${called.length})`));
+  out.push(f(returned.length === 2, B, `the log MUST carry two agent.tool-returned for ${FIXTURE_ERROR_TOOL} (got ${returned.length})`));
+
+  const byCall = (c: Event | undefined): Rec | undefined => {
+    const id = c?.payload['callId'];
+    if (!nonEmpty(id)) return undefined;
+    return returned.find((r) => r.payload['callId'] === id)?.payload;
+  };
+  const first = byCall(called[0]);
+  const second = byCall(called[1]);
+
+  if (first === undefined) {
+    out.push(f(false, B, 'the first scripted call MUST have an agent.tool-returned with the same callId'));
+  } else {
+    const err = first['error'];
+    out.push(f(isRec(err) && nonEmpty(err['code']) && nonEmpty(err['message']), B, 'the call served by the isError entry MUST be recorded with error {code, message}, both non-empty'));
+    out.push(f(!('outcome' in first) || first['outcome'] === undefined, B, 'the call served by the isError entry MUST NOT carry outcome — that replays a failure as a success'));
+    if (opts.prePostEvents) out.push(f(first['status'] === 'error', B, `under toolHooks.prePostEvents the error return MUST carry status "error" (got ${JSON.stringify(first['status'])})`));
+  }
+
+  if (second === undefined) {
+    out.push(f(false, B, 'the second scripted call MUST have an agent.tool-returned with the same callId'));
+  } else {
+    out.push(f('outcome' in second && second['outcome'] !== undefined, B, 'the call served by the following success entry MUST carry outcome'));
+    out.push(f(second['error'] === undefined, B, 'the call served by the following success entry MUST NOT carry error'));
+  }
+
+  const tasks = Array.isArray(summary['tasks']) ? (summary['tasks'] as unknown[]).filter(isRec) : [];
+  const task = tasks.find((t) => t['taskId'] === FIXTURE_ERROR_TASK);
+  out.push(f(task !== undefined, B, `the summary MUST carry task ${FIXTURE_ERROR_TASK}`));
+  if (task !== undefined) out.push(f((task['fixtureMissCount'] ?? 0) === 0, B, `a served error entry is not a miss: fixtureMissCount MUST be 0 (got ${JSON.stringify(task['fixtureMissCount'])})`));
+  return out;
+}

package/src/scenarios/v2-eval-trajectory-trials.test.ts

@@ -44,8 +44,21 @@
  *      clears `passScore` 0.6, the gating aggregate 0.5 does not, so the suite
  *      does not pass. An RFC 0244-only host reports `passed: true` on both.
  *
+ * RFC 0245: a fixture entry marked `isError: true` is replayed as a tool
+ * error, never as a success. Judge in `lib/eval-fixture-error-witness.ts`.
+ *  10. Server-free — an `isError: true` entry validates; a string `"true"` is
+ *      refused; `conformance-eval-fixture-error` validates (leg 2).
+ *  11. `conformance-eval-fixture-error` (fixtureErrors + fixtureMatching) —
+ *      the call served by the error entry is recorded with `error` and no
+ *      `outcome` (`status: "error"` under tool hooks), the next with
+ *      `outcome`, and `fixtureMissCount` is 0. The two-call sequence needs
+ *      RFC 0244 §D's first-unconsumed selection, hence `fixtureMatching`.
+ *  12. Refusal — a host advertising `agents.evalSuite` without `fixtureErrors`
+ *      that lists the fixture refuses it with `400 validation_error`.
+ *
  * @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md
  * @see RFCS/0246-trajectory-failure-gates-the-pass.md
+ * @see RFCS/0245-eval-fixtures-replay-tool-errors.md
  */
 
 import { describe, it, expect } from 'vitest';
@@ -62,6 +75,7 @@ import { FIXTURES_DIR } from '../lib/paths.js';
 import { pollTerminal, readEvents, isRecord, type Ev } from '../lib/fixture-run-observer.js';
 import { gradeTrajectory, passAtK, passHatK, scoreStderr, withinTolerance, type TrajectorySpec } from '../lib/eval-trajectory.js';
 import { judgeMeanGated, judgeSuiteGated } from '../lib/eval-gate-witness.js';
+import { judgeFixtureErrorReplayed } from '../lib/eval-fixture-error-witness.js';
 
 const PROFILE = majorProfile(2);
 const UNREADABLE = 'v2 discovery unreachable — /.well-known/openwop did not answer 200 JSON under OpenWOP-Version: 2.0';
@@ -80,8 +94,12 @@ const ID_GATED_SHAPE = 'openwop.requirement.0246.gated-fields-shape';
 const ID_MEAN_GATED = 'openwop.requirement.0246.mean-gated';
 const ID_SUITE_GATED = 'openwop.requirement.0246.suite-gated';
 const DOC_0246 = 'eval-summary.schema.json gatedScore / gatedAggregateScore (RFC 0246)';
+const ID_FIXTURE_ERROR_SHAPE = 'openwop.requirement.0245.fixture-error-shape';
+const ID_FIXTURE_ERROR = 'openwop.requirement.0245.fixture-error-replayed';
+const ID_FIXTURE_ERROR_REFUSED = 'openwop.requirement.0245.unadvertised-refused';
+const DOC_0245 = 'agents.schema.json §evalSuite.fixtureErrors (RFC 0245)';
 
-interface Task { taskId: string; input: { script: Array<{ tool: string; arguments?: unknown }>; output: string }; expected: { kind: string; trajectory?: TrajectorySpec } }
+interface Task { taskId: string; input: { script: Array<{ tool: string; arguments?: unknown }>; output: string }; expected: { kind: string; trajectory?: TrajectorySpec }; fixtures?: { toolResponses?: Array<Record<string, unknown>> } }
 interface Suite { tasks: Task[]; trials?: { count: number; k?: number; passRule?: string } }
 const fixture = (id: string): Suite => JSON.parse(readFileSync(join(FIXTURES_DIR, 'eval-suites', `${id}.json`), 'utf8')) as Suite;
 
@@ -160,7 +178,7 @@ describe('v2 RFC 0244 — wire shape (server-free)', () => {
   });
 
   it('the eval-suite fixtures validate against the v2 suite schema', () => {
-    for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss', 'conformance-eval-trajectory-gate', 'conformance-eval-trajectory-gate-mean']) {
+    for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss', 'conformance-eval-trajectory-gate', 'conformance-eval-trajectory-gate-mean', 'conformance-eval-fixture-error']) {
       const r = suiteSchema(fixture(id));
       expect(r.ok, req(ID_FIXTURES, 'fixtures.md §eval-suites', `${id} MUST validate: ${r.errors}`)).toBe(true);
     }
@@ -173,6 +191,14 @@ describe('v2 RFC 0244 — wire shape (server-free)', () => {
     expect(summarySchema({ ...ok, gatedAggregateScore: 1.5 }).ok, req(ID_GATED_SHAPE, DOC_0246, 'gatedAggregateScore above 1 MUST be refused')).toBe(false);
     expect(summarySchema({ ...ok, tasks: [{ taskId: 't', score: 1, gatedScore: -0.1, passed: false }] }).ok, req(ID_GATED_SHAPE, DOC_0246, 'a negative gatedScore MUST be refused')).toBe(false);
   });
+
+  it('a fixture entry may be marked isError, and only as a boolean (RFC 0245 §A)', () => {
+    const base = fixture('conformance-eval-fixture-error');
+    const withIsError = (v: unknown): boolean => { const s = structuredClone(base) as Suite; s.tasks[0]!.fixtures!.toolResponses![0]!['isError'] = v; return suiteSchema(s).ok; };
+    expect(withIsError(true), req(ID_FIXTURE_ERROR_SHAPE, DOC_0245, `an isError: true entry MUST validate: ${suiteSchema(base).errors}`)).toBe(true);
+    expect(withIsError(false), req(ID_FIXTURE_ERROR_SHAPE, DOC_0245, 'an isError: false entry MUST validate')).toBe(true);
+    expect(withIsError('true'), req(ID_FIXTURE_ERROR_SHAPE, DOC_0245, 'a string isError MUST be refused')).toBe(false);
+  });
 });
 
 describe('v2 RFC 0244 — host behaviour', () => {
@@ -305,4 +331,34 @@ describe('v2 RFC 0244 — host behaviour', () => {
     if (o.kind === 'skip') return softSkip(o.disposition, o.reason);
     for (const x of judgeSuiteGated(o.summary, 0.6)) expect(x.ok, req(ID_SUITE_GATED, x.doc, x.message)).toBe(true);
   });
+
+  it('a fixture entry marked isError is replayed as a tool error, not a success (RFC 0245)', async () => {
+    const { doc, facet } = await evalSuiteFacet();
+    if (!doc) return softSkip('blocked', UNREADABLE);
+    if (facet?.['fixtureErrors'] !== true) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite.fixtureErrors');
+    if (facet['fixtureMatching'] !== true) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite.fixtureMatching, which the error-then-success sequence needs (RFC 0244 §D first unconsumed match)');
+    const id = 'conformance-eval-fixture-error';
+    if (!isFixtureAdvertised(id)) return softSkip('inapplicable', `the host does not advertise fixture ${id}`);
+    const o = await runSuite(id);
+    if (o.kind === 'skip') return softSkip(o.disposition, o.reason);
+    const hooks = doc['toolHooks'];
+    const prePostEvents = isRecord(hooks) && hooks['prePostEvents'] === true;
+    for (const x of judgeFixtureErrorReplayed(o.events, o.summary, { agentId: AGENT, prePostEvents })) expect(x.ok, req(ID_FIXTURE_ERROR, x.doc, x.message)).toBe(true);
+  });
+
+  it('a suite using isError is refused at createRun without fixtureErrors (RFC 0245 §C)', async () => {
+    const { doc, facet } = await evalSuiteFacet();
+    if (!doc) return softSkip('blocked', UNREADABLE);
+    if (facet === null) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite (v2-eval-mode-unadvertised-refused covers it)');
+    if (facet['fixtureErrors'] === true) return softSkip('inapplicable', 'the host advertises agents.evalSuite.fixtureErrors, so the isError suite is served');
+    const id = 'conformance-eval-fixture-error';
+    if (!isFixtureAdvertised(id)) return softSkip('inapplicable', `the host does not advertise fixture ${id}`);
+    const res = await http(() => driver.post(PROFILE.runsPath, { mode: 'eval', evalSuiteRef: SUITE_URI(id), agentId: AGENT }));
+    if (res === null) return softSkip('blocked', 'POST /runs unreachable (fetch failed)');
+    if (res.status === 429) return softSkip('blocked', 'POST /runs answered 429 — the run budget, not the wire');
+    const runId = isRecord(res.json) && typeof res.json['runId'] === 'string' ? res.json['runId'] : undefined;
+    if (res.status === 201 && runId !== undefined) await http(() => driver.post(`${PROFILE.runsPath}/${PROFILE.idSegment(runId)}/cancel`, {}));
+    expect(res.status, req(ID_FIXTURE_ERROR_REFUSED, DOC_0245, `an isError suite on a host without agents.evalSuite.fixtureErrors MUST be refused with 400 (got ${res.status})`)).toBe(400);
+    expect(readErrorCode(res.json), req(ID_FIXTURE_ERROR_REFUSED, DOC_0245, 'the refusal MUST carry validation_error')).toBe('validation_error');
+  });
 });