@openwop/openwop-conformance 2.46.26 → 2.46.28

package/requirements.json

@@ -2,11 +2,11 @@
   "$comment": "GENERATED by conformance/scripts/generate-requirement-registry.mjs — do not edit. One record per it()/test() in src/scenarios. Ids: openwop.it.<file-stem>.<title-slug>[~n] (src/lib/requirement-ids.ts). A record with id null has an interpolated title; its run-time row is keyed by the rendered title and maps here by file+line only. Renamed ids need a row in requirement-aliases.json.",
   "generatedFrom": "src/scenarios/*.test.ts",
   "counts": {
-    "files": 749,
-    "tests": 2846,
-    "withStableId": 2846,
+    "files": 750,
+    "tests": 2855,
+    "withStableId": 2855,
     "interpolatedTitles": 0,
-    "explicitIds": 2719
+    "explicitIds": 2728
   },
   "records": [
     {
@@ -31089,10 +31089,87 @@
         }
       ]
     },
+    {
+      "id": "openwop.it.v2-eval-regression-baseline.the-baselineselection-member-and-the-createrun-eval-properties-are-in-the-derive",
+      "file": "v2-eval-regression-baseline.test.ts",
+      "line": 131,
+      "title": "the baselineSelection member and the createRun eval properties are in the derived contract",
+      "explicitId": "openwop.requirement.0251.shape",
+      "citations": [
+        {
+          "section": "agents.schema.json §evalSuite.baselineSelection, createRun evalModes / baselineRunId (RFC 0251)",
+          "requirement": "baselineSelection: true MUST validate"
+        },
+        {
+          "section": "agents.schema.json §evalSuite.baselineSelection, createRun evalModes / baselineRunId (RFC 0251)",
+          "requirement": "baselineSelection MUST be a boolean"
+        },
+        {
+          "section": "agents.schema.json §evalSuite.baselineSelection, createRun evalModes / baselineRunId (RFC 0251)",
+          "requirement": "createRun MUST carry evalModes over the RFC 0081 §D modes"
+        },
+        {
+          "section": "agents.schema.json §evalSuite.baselineSelection, createRun evalModes / baselineRunId (RFC 0251)",
+          "requirement": "evalModes MUST be non-empty and distinct"
+        },
+        {
+          "section": "agents.schema.json §evalSuite.baselineSelection, createRun evalModes / baselineRunId (RFC 0251)",
+          "requirement": "createRun MUST carry baselineRunId as a runId"
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-regression-baseline.the-candidate-run-names-the-baseline-it-was-given-and-scoredelta-is-the-aggregat",
+      "file": "v2-eval-regression-baseline.test.ts",
+      "line": 146,
+      "title": "the candidate run names the baseline it was given, and scoreDelta is the aggregate difference",
+      "explicitId": "openwop.requirement.0251.baseline-echoed",
+      "citations": []
+    },
+    {
+      "id": "openwop.it.v2-eval-regression-baseline.only-the-tasks-in-both-runs-are-paired-rfc-0244-f",
+      "file": "v2-eval-regression-baseline.test.ts",
+      "line": 154,
+      "title": "only the tasks in both runs are paired (RFC 0244 §F)",
+      "explicitId": "openwop.requirement.0244.regression-paired",
+      "citations": []
+    },
+    {
+      "id": "openwop.it.v2-eval-regression-baseline.equal-paired-scores-never-yield-improved-or-regressed-rfc-0244-f",
+      "file": "v2-eval-regression-baseline.test.ts",
+      "line": 163,
+      "title": "equal paired scores never yield improved or regressed (RFC 0244 §F)",
+      "explicitId": "openwop.requirement.0244.regression-no-false-verdict",
+      "citations": []
+    },
+    {
+      "id": "openwop.it.v2-eval-regression-baseline.an-unknown-baseline-and-a-run-of-another-suite-are-refused-without-the-id",
+      "file": "v2-eval-regression-baseline.test.ts",
+      "line": 172,
+      "title": "an unknown baseline, and a run of another suite, are refused without the id",
+      "explicitId": "openwop.requirement.0251.baseline-refused",
+      "citations": []
+    },
+    {
+      "id": "openwop.it.v2-eval-regression-baseline.another-tenant-s-run-is-refused-exactly-as-an-id-that-never-existed",
+      "file": "v2-eval-regression-baseline.test.ts",
+      "line": 186,
+      "title": "another tenant's run is refused exactly as an id that never existed",
+      "explicitId": "openwop.requirement.0251.baseline-uniform",
+      "citations": []
+    },
+    {
+      "id": "openwop.it.v2-eval-regression-baseline.a-baseline-without-regression-and-regression-without-a-baseline-are-refused",
+      "file": "v2-eval-regression-baseline.test.ts",
+      "line": 203,
+      "title": "a baseline without regression, and regression without a baseline, are refused",
+      "explicitId": "openwop.requirement.0251.fields-refused",
+      "citations": []
+    },
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-suite-summary-and-event-fields-validate-and-the-negatives-are-refus",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 196,
+      "line": 211,
       "title": "the RFC 0244 suite, summary and event fields validate and the negatives are refused",
       "explicitId": "openwop.requirement.0244.wire-shape",
       "citations": [
@@ -31131,7 +31208,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-additions-are-content-free-eval-summary-no-content-leak",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 219,
+      "line": 234,
       "title": "the RFC 0244 additions are content-free (eval-summary-no-content-leak)",
       "explicitId": "openwop.requirement.0244.content-free",
       "citations": [
@@ -31148,7 +31225,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-eval-suite-fixtures-validate-against-the-v2-suite-schema",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 226,
+      "line": 241,
       "title": "the eval-suite fixtures validate against the v2 suite schema",
       "explicitId": "openwop.requirement.0244.fixtures-valid",
       "citations": [
@@ -31162,7 +31239,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0246-gating-fields-validate-and-stay-in-0-1",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 233,
+      "line": 248,
       "title": "the RFC 0246 gating fields validate and stay in 0–1",
       "explicitId": "openwop.requirement.0246.gated-fields-shape",
       "citations": [
@@ -31184,7 +31261,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-entry-may-be-marked-iserror-and-only-as-a-boolean-rfc-0245-a",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 241,
+      "line": 256,
       "title": "a fixture entry may be marked isError, and only as a boolean (RFC 0245 §A)",
       "explicitId": "openwop.requirement.0245.fixture-error-shape",
       "citations": [
@@ -31206,7 +31283,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-trajectory-suite-is-graded-per-the-reference-grader",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 251,
+      "line": 266,
       "title": "a trajectory suite is graded per the reference grader",
       "explicitId": "openwop.requirement.0244.trajectory-graded",
       "citations": [
@@ -31240,7 +31317,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.an-eval-run-brackets-its-scores-with-one-eval-started-and-one-eval-completed-rfc",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 285,
+      "line": 300,
       "title": "an eval run brackets its scores with one eval.started and one eval.completed (RFC 0081 §C)",
       "explicitId": "openwop.requirement.0081.eval-envelope",
       "citations": [
@@ -31254,7 +31331,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-served-evalsummary-validates-and-its-counts-match-its-rows-rfc-0081-c",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 293,
+      "line": 308,
       "title": "the served EvalSummary validates and its counts match its rows (RFC 0081 §C)",
       "explicitId": "openwop.requirement.0081.summary-served",
       "citations": [
@@ -31268,7 +31345,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.repeated-trials-emit-one-eval-scored-per-task-per-trial",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 314,
+      "line": 329,
       "title": "repeated trials emit one eval.scored per task per trial",
       "explicitId": "openwop.requirement.0244.trials-per-task",
       "citations": [
@@ -31286,7 +31363,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-per-task-trials-block-and-trialstats-follow-the-b-formulas",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 325,
+      "line": 340,
       "title": "the per-task trials block and trialStats follow the §B formulas",
       "explicitId": "openwop.requirement.0244.trial-stats-formula",
       "citations": [
@@ -31325,7 +31402,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.efficiency-toolcallcount-counts-the-scripted-agent-tool-called-events-c",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 348,
+      "line": 363,
       "title": "efficiency.toolCallCount counts the scripted agent.tool-called events (§C)",
       "explicitId": "openwop.requirement.0244.efficiency-tool-calls",
       "citations": [
@@ -31339,7 +31416,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.efficiency-model-and-token-counts-match-the-agent-s-provider-usage-c",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 360,
+      "line": 375,
       "title": "efficiency model and token counts match the agent's provider.usage (§C)",
       "explicitId": "openwop.requirement.0244.efficiency-model-usage",
       "citations": [
@@ -31353,7 +31430,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.totalcostusd-counts-every-trial-b",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 371,
+      "line": 386,
       "title": "totalCostUsd counts every trial (§B)",
       "explicitId": "openwop.requirement.0244.cost-every-trial",
       "citations": [
@@ -31364,10 +31441,38 @@
         }
       ]
     },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.a-host-does-not-fail-a-task-or-a-suite-on-efficiency-c",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 394,
+      "title": "a host does not fail a task or a suite on efficiency (§C)",
+      "explicitId": "openwop.requirement.0244.efficiency-never-gated",
+      "citations": [
+        {
+          "section": "eval-summary.schema.json efficiency (RFC 0244 §C)",
+          "requirement": null,
+          "interpolated": true
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.memory-one-trial-writes-is-not-visible-to-another-trial-b",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 409,
+      "title": "memory one trial writes is not visible to another trial (§B)",
+      "explicitId": "openwop.requirement.0244.trial-memory-isolated",
+      "citations": [
+        {
+          "section": "agent-eval-suite.schema.json trials, fixtures.md memory contract (RFC 0244 §B)",
+          "requirement": null,
+          "interpolated": true
+        }
+      ]
+    },
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-miss-under-onmiss-live-reaches-the-live-counter-and-is-counted-d",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 401,
+      "line": 447,
       "title": "a miss under onMiss \"live\" reaches the live counter and is counted (§D)",
       "explicitId": "openwop.requirement.0244.live-miss-counted",
       "citations": [
@@ -31381,7 +31486,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.repeated-calls-are-served-the-first-unconsumed-fixture-entry-once-each-d",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 409,
+      "line": 455,
       "title": "repeated calls are served the first unconsumed fixture entry, once each (§D)",
       "explicitId": "openwop.requirement.0244.first-unconsumed",
       "citations": [
@@ -31395,7 +31500,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-miss-under-onmiss-fail-never-reaches-the-live-counter-d",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 417,
+      "line": 463,
       "title": "a miss under onMiss \"fail\" never reaches the live counter (§D)",
       "explicitId": "openwop.requirement.0244.miss-no-live-call",
       "citations": [
@@ -31409,7 +31514,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-miss-under-onmiss-fail-fails-the-trial-and-is-counted",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 425,
+      "line": 471,
       "title": "a fixture miss under onMiss \"fail\" fails the trial and is counted",
       "explicitId": "openwop.requirement.0244.fixture-miss-fails",
       "citations": [
@@ -31446,7 +31551,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-an-unadvertised-rfc-0244-feature-is-refused-at-createrun",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 445,
+      "line": 491,
       "title": "a suite using an unadvertised RFC 0244 feature is refused at createRun",
       "explicitId": "openwop.requirement.0244.unadvertised-feature-refused",
       "citations": [
@@ -31464,7 +31569,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.under-passrule-mean-a-task-whose-trials-failed-their-trajectory-does-not-pass-rf",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 461,
+      "line": 507,
       "title": "under passRule \"mean\" a task whose trials failed their trajectory does not pass (RFC 0246)",
       "explicitId": "openwop.requirement.0246.mean-gated",
       "citations": [
@@ -31478,7 +31583,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-suite-whose-tasks-failed-their-trajectory-does-not-pass-on-the-output-scores-r",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 474,
+      "line": 520,
       "title": "a suite whose tasks failed their trajectory does not pass on the output scores (RFC 0246)",
       "explicitId": "openwop.requirement.0246.suite-gated",
       "citations": [
@@ -31492,7 +31597,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-entry-marked-iserror-is-replayed-as-a-tool-error-not-a-success-rfc-024",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 485,
+      "line": 531,
       "title": "a fixture entry marked isError is replayed as a tool error, not a success (RFC 0245)",
       "explicitId": "openwop.requirement.0245.fixture-error-replayed",
       "citations": [
@@ -31506,7 +31611,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-iserror-is-refused-at-createrun-without-fixtureerrors-rfc-0245-c",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 499,
+      "line": 545,
       "title": "a suite using isError is refused at createRun without fixtureErrors (RFC 0245 §C)",
       "explicitId": "openwop.requirement.0245.unadvertised-refused",
       "citations": [

package/scenario-majors.json

@@ -1,9 +1,9 @@
 {
   "$comment": "GENERATED by conformance/scripts/generate-scenario-majors.mjs (RFC 0168 §D.3). Do not edit; add a file to BOTH_MAJORS in the generator to target both majors.",
   "counts": {
-    "files": 661,
+    "files": 662,
     "v1": 455,
-    "v2": 221
+    "v2": 222
   },
   "majors": {
     "a2a-1-0-agent-card.test.ts": [
@@ -1482,6 +1482,9 @@
     "v2-eval-mode-unadvertised-refused.test.ts": [
       2
     ],
+    "v2-eval-regression-baseline.test.ts": [
+      2
+    ],
     "v2-eval-trajectory-trials.test.ts": [
       2
     ],

package/schemas/CORPUS-STAMP.json

@@ -1,17 +1,17 @@
 {
   "_comment": "Provenance of @openwop/spec-artifacts (RFC 0168 §D.2). files: SHA-256 per file; the conformance suite compares the installed peer against dist/spec-artifacts.lock.json at start.",
   "package": "@openwop/spec-artifacts",
-  "version": "2.46.26",
-  "corpusTag": "v2.46.26",
+  "version": "2.46.28",
+  "corpusTag": "v2.46.28",
   "files": {
     "api/.redocly.lint-ignore.yaml": "bf5a8350b88a72fa43f59605ed8d903ed24b6cfccda5e45509c9f6ed9ee4e712",
     "api/asyncapi.yaml": "d5ecb9ee6114582be3b1f662c84bfac9ae96dae7bacb853e461168f70a8e1c7d",
     "api/grpc/openwop.proto": "c3e72bb17cba514ee98feb6434e6c9b6ea6795bfd086489ec69fd882dd1ad977",
     "api/openapi.yaml": "ac6de5303872664b78ff419c035915593e01599e02835e777a0123de1baeb398",
     "api/redocly.yaml": "b0604c89b2ca6d5076ec25725c539dad44a741a811fe524439ee6daef8baa09f",
-    "api/seams-v2.yaml": "fc5aaff321913531e04b13522de92a9a8fa3f61be836696e9ab83fbc563caa87",
-    "api/v2/asyncapi.yaml": "0ea85f3cb0af276cd7ed82e14810c1093917f0d1daa80bc0226b78e127c4e17d",
-    "api/v2/openapi.yaml": "7c7cd37cc0cabcd37285502594977f6ddc5a59f30938d7aede36f69e05322244",
+    "api/seams-v2.yaml": "1281524c0152c77a87db9bfdc01440aed092421619b377cdf85f862af41a3ae5",
+    "api/v2/asyncapi.yaml": "58ded9e1117f35837dc0eea8a44438a1def186e8fad0046cb8edacd07cb4906e",
+    "api/v2/openapi.yaml": "0fb51374a913d05366ddc721ef7dc872ba5489f8eabdd587c13b28561089c170",
     "api/v2/redocly.yaml": "1e66b60e6118ad11a823bb620678be464d99dfe50a40e3e6f93ec9429b88b34c",
     "schemas/README.md": "276fc1d51eb08e14e9127b9fd27761d7046ee33c12000f3a64a801322db8bf58",
     "schemas/a2a-task-state.schema.json": "75d5049dea8bd873ff0e7546f1c60c8d36c219bec7084264360be54a8be30ae1",
@@ -117,7 +117,7 @@
     "schemas/v2/artifact.schema.json": "85b9dc284d39b99d2a66fa8cbf47f2dd3d8526d3bd6cc411ad315b372de38975",
     "schemas/v2/audit-verify-result.schema.json": "14af8e9283510f0a40e48a2059c929ee44e0fc9fa3cc66a95bfbe90b17c48297",
     "schemas/v2/budget-policy.schema.json": "1de68095db0beb5c7a4be629de26e79f62ebbe748a88d46f36e768edfd1f0d68",
-    "schemas/v2/capabilities.schema.json": "6ca940cc38ec7f4b582def52c18337fd1e9ea03d627a5446766230a19af1a8ab",
+    "schemas/v2/capabilities.schema.json": "b06b8142d288e839fe871bdddde35ca08d38d00da2a510fb9cb4ab201f37aa99",
     "schemas/v2/certification-bundle.schema.json": "5cfa7fb5247160333e33a58b60d42bcd4e5269d8391d6a112d301e14828892ed",
     "schemas/v2/channel-presence-payload.schema.json": "fc03df0c20cc12a25f7ee6a47503a4c5931d79fe3d5ff24815e8fd7c1bcb9a87",
     "schemas/v2/channel-written-payload.schema.json": "ccecff3c71a3275ad8db035ac0e04db25e6f8d23cf091ddbcd5a60d1be1cbb6b",
@@ -212,14 +212,14 @@
     "schemas/workspace-file.schema.json": "464de85c2a068243084ee9c1d969bc7cd5d8f7948574e58450d6493c38a0e1e4",
     "spec/v1/alias-detectors.json": "22444c22bb88c568339ef512553f9fc38146e22b566d11ab8608f655d688d19c",
     "spec/v1/capability-declaration-classes.json": "e7729aed5c4b4e1dd02abab0530f14cc95f5d4070fe51fb139e7f5cccefa00c6",
-    "spec/v1/core-standard-manifest.json": "580b853bb37383d5acea488fba6747193d1db4e6e63025255433dababf008d60",
+    "spec/v1/core-standard-manifest.json": "ec3b1a7322067f80dd5562428fd2963a210d73da347eeb29661637745f2f7825",
     "spec/v1/deprecations.json": "e723bc11d7272c71789ad1303bb7f21809a8a024abf57594f9d5c35dc199ece9",
     "spec/v1/deprecations.schema.json": "3e393c405d2a41b467d8c5e3c468549078df1ce6a6d2588fc488097b95d9b55b",
     "spec/v1/end-of-support.json": "35f0bf3b70fe78b16e2a2fd81a153fc27e703f6c16376c6fe7cdf1f4f2c4e6d4",
     "spec/v1/event-codemap.json": "3da60d884157793a360da532a9fcbbfb5285636db325a74cec94b34622186d97",
     "spec/v1/event-codemap.schema.json": "d05933b2e88103aff51a2f774df97b9bda065e0fc88dbd9b114cc47a32189174",
     "spec/v1/extensions.json": "79a60754aa16cbdbdb604f8e5c00038af535a1690a4d07bd5ff7c38f0256a7bc",
-    "spec/v1/gaps.json": "21d53eee36f47b3cd44ba54802acc5cdc810d242d1fd929b1e76167da8566957",
+    "spec/v1/gaps.json": "23f953428b6609c68917d2201f0b289fe855033452e895028dd1b6f9aae45e62",
     "spec/v1/gaps.schema.json": "8fd83259f556553c9df0f53e7a82ca8c2a4771d471197f69b4e2ae0ceeacfd66",
     "spec/v1/migrations.json": "4c50760be70cb644d02427bcaf5749606700edf4513f423903cf3b1d12f5c300",
     "spec/v1/migrations.schema.json": "886779aa6c22e646db097f5df210adb018a4dd14a7b815465a18c8a7056c8f72",
@@ -285,7 +285,7 @@
     "spec/v2/ext/sandbox-runtime-notes/README.md": "0d1290fb1e20b2e35c0b15051541ab6546e4e2a6f693b4f912cb3f7d6a65e92c",
     "spec/v2/ext/webResearch/README.md": "b675e6f6646128d781b661f754c2393eed86e5935d16c3a22df582a5ce025b9b",
     "spec/v2/facets/a2a.schema.json": "b68bf5e29e455e46c2ad7cfcccf5963730ec2310ff9db81884550d8df30fd6ef",
-    "spec/v2/facets/agents.schema.json": "0eb1172ea140607cb46d164af8bf405b32f5b47aa54baac7844e2c4161290025",
+    "spec/v2/facets/agents.schema.json": "198c01fdbb09deeb84f8f45f9f47976e696ac35ef087a2bc5c6f3c4fbb2ac370",
     "spec/v2/facets/aiProviders.schema.json": "6e4327bcd48110686cc1adfd26ddd6fbbf4ff529c476dcc74eeff8c5ae9965ce",
     "spec/v2/facets/auditLogIntegrity.schema.json": "9a7f81320c8f940c6f4129bc7dd895a4063231c5458a717a08c2cbf9bb886c46",
     "spec/v2/facets/auth.schema.json": "7d98bc81a0f7d307460d039eebfebd21388091bfe1d538762583940f94c0bd2a",
@@ -318,9 +318,9 @@
     "spec/v2/path-manifest.json": "20f598f56df460ad3ed50aba3d6bc2933ea16a3c96ce125da1c5dd6989d4155c",
     "spec/v2/peer-dependency-aliases.json": "d10299280abee08258502925bc327293ee413e0108cd6e6ec75ff6110653308d",
     "spec/v2/profiles.json": "b382b261c7844798575f7c1b37c934f0da19846016a2337d883d4f5ef6e05479",
-    "spec/v2/release.json": "e2a9a7d13015eb3ca85bdbcd6ec422b6e91780fb9019b0e75e2855b826728b1d",
+    "spec/v2/release.json": "4bc3da43c14c5c1fa31fc3f7fda0a342d073dfbd5d053ff8557300d3bb67f067",
     "spec/v2/retention-floors.json": "eaf3722d95c79947af1d4269ef85117e126518c588cfcf1a2b21b97269f51624",
-    "spec/v2/surface-baseline.json": "4a1e5ab2349349eeeb9fa173e392627d49c2bee77ac8a42897aeacf2c3a4d784"
+    "spec/v2/surface-baseline.json": "7d7a3073bfa01345b7560caffa0b6e382b0e82a205d53119c92fe98d4233d5b8"
   },
-  "corpusCommit": "e5e8306a4f9c8c41394cfbe63fb58a08a74fe5b9"
+  "corpusCommit": "cb64bb3fe9bf7c37a6432da5207026e0d556cb32"
 }

package/src/lib/eval-efficiency-witness.ts

@@ -12,7 +12,8 @@
  * agent calls a model, which the fixture contract leaves to the host.
  *
  * `efficiency` is optional on a summary entry, so a run whose entries carry
- * none is not evidence either way (`applicable: false`).
+ * none is not evidence either way (`applicable: false`). The never-gated rule
+ * (`judgeNeverGated`) holds whether or not a host reports `efficiency`.
  *
  * @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md §C
  */
@@ -137,3 +138,30 @@ export function judgeCostEveryTrial(summary: Rec, log: { bracketed: boolean; usa
   const sum = (costs as number[]).reduce((a, b) => a + b, 0);
   return { applicable: true, findings: [{ ok: total >= sum || close(total, sum), message: `totalCostUsd MUST count every trial: it MUST be at least the ${sum} costEstimateUsd of the agent's ${log.usage.length} provider.usage events across all trials (got ${total})` }] };
 }
+
+/**
+ * §C "MUST NOT fail a task or a suite on" efficiency, on
+ * `conformance-eval-efficiency-ungated`: `lean` and `busy` share an output and
+ * an expectation and differ only in their scripted call counts, so efficiency
+ * is the only thing that can tell them apart. `busy` MUST pass and score what
+ * `lean` scores, and the suite (`passScore` 1) MUST pass. The verdict is
+ * `blocked` unless the log carries every scripted call (otherwise the
+ * difference was never produced) and `lean` passes (otherwise a failing `busy`
+ * could have another cause).
+ */
+export function judgeNeverGated(summary: Rec, lean: ScriptedTask, busy: ScriptedTask, toolCallsInLog: number): { blocked: string } | EfficiencyVerdict {
+  const want = lean.scriptLength + busy.scriptLength;
+  if (toolCallsInLog !== want) return { blocked: `the log carries ${toolCallsInLog} agent.tool-called events, not the ${want} scripted calls, so the two tasks' efficiency never differed` };
+  const entries = (Array.isArray(summary['tasks']) ? (summary['tasks'] as unknown[]) : []).filter(isRec);
+  const l = entries.find((e) => e['taskId'] === lean.taskId);
+  const b = entries.find((e) => e['taskId'] === busy.taskId);
+  if (l?.['passed'] !== true) return { blocked: `the control task ${lean.taskId} did not pass, so a failing ${busy.taskId} could have a cause other than efficiency` };
+  return {
+    applicable: true,
+    findings: [
+      { ok: b?.['passed'] === true, message: `task ${busy.taskId} (${busy.scriptLength} tool calls) MUST pass like ${lean.taskId} (${lean.scriptLength}): a host MUST NOT fail a task on efficiency (got passed ${String(b?.['passed'])})` },
+      { ok: typeof b?.['score'] === 'number' && b['score'] === l['score'], message: `task ${busy.taskId} MUST score what ${lean.taskId} scores (${String(l['score'])}): efficiency is informational (got ${String(b?.['score'])})` },
+      { ok: summary['passed'] === true, message: `the suite MUST pass at passScore 1 when both tasks pass: a host MUST NOT fail a suite on efficiency (got passed ${String(summary['passed'])})` },
+    ],
+  };
+}

package/src/lib/eval-memory-witness.ts

@@ -0,0 +1,82 @@
+/**
+ * RFC 0244 §B, G3 — trial isolation: each trial starts from the same
+ * `memorySeed` snapshot, and memory one trial writes is not visible to another
+ * trial of the same eval run. The suite-owned memory tool
+ * `conformance.eval:memory` makes both observable from outside. A pure judge
+ * over one eval run of `conformance-eval-memory-isolation`, for
+ * `v2-eval-trajectory-trials.test.ts`.
+ *
+ * The tool reads and writes the trial's own agent memory, the store
+ * `memorySeed` seeds (contract in `fixtures.md` §Eval-suite fixtures):
+ * `write {id, content}` stores an entry, `read {id}` returns `{id, content}`,
+ * with `content: null` for an entry the trial's memory does not hold. Every
+ * trial runs the same script:
+ *   1. read `seeded`      — the seed, in every trial;
+ *   2. read `trial-note`  — nothing yet, unless another trial's write leaked;
+ *   3. write `trial-note` = `written`;
+ *   4. read `trial-note`  — `written`, the positive control.
+ *
+ * Trials may run concurrently (scheduling is host-internal), so the judge
+ * never groups calls by trial. Over n trials a conforming host returns n
+ * `null` and n `written` reads of `trial-note`. A leak turns a step-2 `null`
+ * into `written`, so the run reads fewer than n `null`. A host whose writes
+ * never read back (fewer than n `written`) shows nothing about isolation, and
+ * the verdict is `blocked`, never a pass.
+ *
+ * @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md §B
+ */
+
+export const MEMORY_TOOL = 'conformance.eval:memory';
+export const SEEDED = { id: 'seeded', content: 'seed' } as const;
+export const NOTE = { id: 'trial-note', content: 'written' } as const;
+
+export interface MemoryFinding { readonly ok: boolean; readonly message: string }
+export type MemoryVerdict =
+  | { readonly kind: 'blocked'; readonly reason: string }
+  | { readonly kind: 'judged'; readonly findings: readonly MemoryFinding[] };
+
+type Rec = Record<string, unknown>;
+export interface MemoryEvent { readonly type: string; readonly payload: Rec }
+export interface MemoryCall { readonly op: unknown; readonly id: unknown; readonly outcome: unknown; readonly returned: boolean }
+
+const isRec = (v: unknown): v is Rec => typeof v === 'object' && v !== null && !Array.isArray(v);
+
+/** The memory-tool calls of one run, in log order, each paired with its return by `callId`. */
+export function memoryCalls(events: readonly MemoryEvent[], agentId: string): MemoryCall[] {
+  const returns = new Map<string, Rec>();
+  for (const e of events) {
+    if (e.type === 'agent.tool-returned' && e.payload['toolName'] === MEMORY_TOOL && typeof e.payload['callId'] === 'string') returns.set(e.payload['callId'], e.payload);
+  }
+  const out: MemoryCall[] = [];
+  for (const e of events) {
+    if (e.type !== 'agent.tool-called' || e.payload['toolName'] !== MEMORY_TOOL || e.payload['agentId'] !== agentId) continue;
+    const inputs = isRec(e.payload['inputs']) ? e.payload['inputs'] : {};
+    const r = typeof e.payload['callId'] === 'string' ? returns.get(e.payload['callId']) : undefined;
+    out.push({ op: inputs['op'], id: inputs['id'], outcome: r?.['outcome'], returned: r !== undefined });
+  }
+  return out;
+}
+
+const content = (c: MemoryCall): unknown => (isRec(c.outcome) ? c.outcome['content'] : undefined);
+
+/** §B isolation over `trials` trials of the four-step script. */
+export function judgeTrialMemoryIsolated(calls: readonly MemoryCall[], trials: number): MemoryVerdict {
+  if (calls.length !== 4 * trials || !calls.every((c) => c.returned)) {
+    return { kind: 'blocked', reason: `the run MUST log the ${4 * trials} scripted memory calls (four per trial), each with an agent.tool-returned (got ${calls.length}, ${calls.filter((c) => c.returned).length} returned)` };
+  }
+  const reads = calls.filter((c) => c.op === 'read');
+  const seeded = reads.filter((c) => c.id === SEEDED.id);
+  const note = reads.filter((c) => c.id === NOTE.id);
+  const written = note.filter((c) => content(c) === NOTE.content).length;
+  if (written < trials) {
+    return { kind: 'blocked', reason: `only ${written} of ${trials} trials read back their own write of ${NOTE.id}, so the memory tool does not reach the trial's memory (fixtures.md memory contract) and isolation is not observable` };
+  }
+  const empty = note.filter((c) => content(c) === null).length;
+  return {
+    kind: 'judged',
+    findings: [
+      { ok: seeded.length === trials && seeded.every((c) => content(c) === SEEDED.content), message: `every trial MUST start from the memorySeed snapshot: each of the ${trials} reads of ${SEEDED.id} MUST return content "${SEEDED.content}" (got ${JSON.stringify(seeded.map(content))})` },
+      { ok: empty === trials && note.length === 2 * trials, message: `memory one trial writes MUST NOT be visible to another trial: each trial's first read of ${NOTE.id} MUST find nothing, so ${trials} of the ${note.length} reads return content null (got ${empty})` },
+    ],
+  };
+}

package/src/lib/eval-regression-witness.ts

@@ -0,0 +1,77 @@
+/**
+ * RFC 0251 — an eval run names its modes and its regression baseline — and
+ * RFC 0244 §F pairing, which RFC 0251 makes causable. Pure judges over two
+ * eval runs and the refusals, for `v2-eval-regression-baseline.test.ts`.
+ *
+ * The fixtures share one `suiteId`. The base run (1.0.0) holds `base-only`
+ * (fails), `shared-a` and `shared-b` (pass); the candidate (1.1.0) holds the
+ * two shared tasks, unchanged, and `candidate-only` (passes). So the
+ * aggregates differ by 1/3 while every paired score is equal: a host that
+ * compares aggregates, not paired tasks, would call the candidate `improved`.
+ *
+ * @see RFCS/0251-eval-run-mode-and-baseline-selection.md
+ * @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md §F
+ */
+
+import { readErrorCode } from './error-envelope.js';
+
+export interface RegressionFinding { readonly ok: boolean; readonly message: string }
+type Rec = Record<string, unknown>;
+const isRec = (v: unknown): v is Rec => typeof v === 'object' && v !== null && !Array.isArray(v);
+const close = (a: number, b: number): boolean => Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(b));
+
+export const SHARED_TASKS = ['shared-a', 'shared-b'] as const;
+
+/** §D: both carriers name the requested baseline, and `scoreDelta` is the aggregate difference. */
+export function judgeBaselineEchoed(baselineRunId: string, events: readonly { type: string; payload: Rec }[], candidate: Rec, base: Rec): RegressionFinding[] {
+  const started = events.filter((e) => e.type === 'eval.started');
+  const reg = isRec(candidate['regression']) ? candidate['regression'] : {};
+  const want = typeof candidate['aggregateScore'] === 'number' && typeof base['aggregateScore'] === 'number' ? candidate['aggregateScore'] - base['aggregateScore'] : undefined;
+  const delta = reg['scoreDelta'];
+  return [
+    { ok: started.length === 1 && started[0]!.payload['baselineRunId'] === baselineRunId, message: `eval.started MUST carry baselineRunId ${baselineRunId} (got ${JSON.stringify(started.map((e) => e.payload['baselineRunId']))})` },
+    { ok: reg['baselineRunId'] === baselineRunId, message: `EvalSummary.regression.baselineRunId MUST be ${baselineRunId} (got ${JSON.stringify(reg['baselineRunId'])})` },
+    { ok: want !== undefined && typeof delta === 'number' && close(delta, want), message: `regression.scoreDelta MUST be this run's aggregateScore minus the baseline's, ${String(want)} (got ${JSON.stringify(delta)})` },
+  ];
+}
+
+/** RFC 0244 §F: only the tasks in both runs are paired, and the host says how many. */
+export function judgePaired(candidate: Rec): RegressionFinding[] {
+  const reg = isRec(candidate['regression']) ? candidate['regression'] : {};
+  return [{ ok: reg['pairedTaskCount'] === SHARED_TASKS.length, message: `regression.pairedTaskCount MUST count the ${SHARED_TASKS.length} tasks present in both runs, paired by taskId (got ${JSON.stringify(reg['pairedTaskCount'])})` }];
+}
+
+/**
+ * RFC 0244 §F: every paired score is equal, so no method can distinguish the
+ * runs. The verdict MUST be present (the host advertises `regressionVerdict`)
+ * and MUST NOT be `improved` or `regressed`.
+ */
+export function judgeNoFalseVerdict(candidate: Rec): RegressionFinding[] {
+  const reg = isRec(candidate['regression']) ? candidate['regression'] : {};
+  const v = reg['verdict'];
+  return [
+    { ok: typeof v === 'string', message: `a host advertising regressionVerdict MUST report regression.verdict (got ${JSON.stringify(v)})` },
+    { ok: v !== 'improved' && v !== 'regressed', message: `with every paired task scoring the same, the verdict MUST NOT be improved or regressed: comparing aggregates is not pairing (got ${JSON.stringify(v)})` },
+  ];
+}
+
+export interface Answer { readonly status: number; readonly json: unknown; readonly text: string }
+
+/** §B / §C: a refused create answers `400 validation_error` and does not echo the supplied id. */
+export function judgeRefused(a: Answer, suppliedId: string | undefined, what: string): RegressionFinding[] {
+  const out: RegressionFinding[] = [
+    { ok: a.status === 400, message: `${what} MUST be refused 400 before the run starts (got ${a.status})` },
+    { ok: readErrorCode(a.json) === 'validation_error', message: `${what} MUST be refused with validation_error (got ${JSON.stringify(readErrorCode(a.json))})` },
+  ];
+  if (suppliedId !== undefined) out.push({ ok: !a.text.includes(suppliedId), message: `the refusal of ${what} MUST NOT echo the supplied baselineRunId` });
+  return out;
+}
+
+/** §C: another tenant's real run is refused exactly as an id that never existed. */
+export function judgeUniform(unknown: Answer, foreign: Answer, foreignId: string): RegressionFinding[] {
+  return [
+    { ok: foreign.status === unknown.status, message: `another tenant's run MUST be refused with the same status as an unknown id (${foreign.status} vs ${unknown.status})` },
+    { ok: readErrorCode(foreign.json) === readErrorCode(unknown.json), message: `another tenant's run MUST be refused with the same code as an unknown id (${JSON.stringify(readErrorCode(foreign.json))} vs ${JSON.stringify(readErrorCode(unknown.json))})` },
+    { ok: !foreign.text.includes(foreignId), message: "the refusal MUST NOT echo another tenant's run id" },
+  ];
+}