@openwop/openwop-conformance 2.46.7 → 2.46.9

package/requirements.json

@@ -2,11 +2,11 @@
   "$comment": "GENERATED by conformance/scripts/generate-requirement-registry.mjs — do not edit. One record per it()/test() in src/scenarios. Ids: openwop.it.<file-stem>.<title-slug>[~n] (src/lib/requirement-ids.ts). A record with id null has an interpolated title; its run-time row is keyed by the rendered title and maps here by file+line only. Renamed ids need a row in requirement-aliases.json.",
   "generatedFrom": "src/scenarios/*.test.ts",
   "counts": {
-    "files": 745,
-    "tests": 2800,
-    "withStableId": 2800,
+    "files": 746,
+    "tests": 2808,
+    "withStableId": 2808,
     "interpolatedTitles": 0,
-    "explicitIds": 2673
+    "explicitIds": 2681
   },
   "records": [
     {
@@ -31025,6 +31025,222 @@
         }
       ]
     },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-suite-summary-and-event-fields-validate-and-the-negatives-are-refus",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 114,
+      "title": "the RFC 0244 suite, summary and event fields validate and the negatives are refused",
+      "explicitId": "openwop.requirement.0244.wire-shape",
+      "citations": [
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "trajectory.match \"superset\" MUST be refused (use any-order)"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "trials.count 0 MUST be refused"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "an unknown trajectory member MUST be refused"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "eval.started.trialCount MUST validate"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "an unknown evaluatorType MUST be refused"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-additions-are-content-free-eval-summary-no-content-leak",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 137,
+      "title": "the RFC 0244 additions are content-free (eval-summary-no-content-leak)",
+      "explicitId": "openwop.requirement.0244.content-free",
+      "citations": [
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "a trajectory block naming tools MUST be refused"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "a prose scoreLabel MUST be refused"
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.the-eval-suite-fixtures-validate-against-the-v2-suite-schema",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 144,
+      "title": "the eval-suite fixtures validate against the v2 suite schema",
+      "explicitId": "openwop.requirement.0244.fixtures-valid",
+      "citations": [
+        {
+          "section": "fixtures.md §eval-suites",
+          "requirement": null,
+          "interpolated": true
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.a-trajectory-suite-is-graded-per-the-reference-grader",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 153,
+      "title": "a trajectory suite is graded per the reference grader",
+      "explicitId": "openwop.requirement.0244.trajectory-graded",
+      "citations": [
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.repeated-trials-emit-one-eval-scored-per-task-per-trial",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 189,
+      "title": "repeated trials emit one eval.scored per task per trial",
+      "explicitId": "openwop.requirement.0244.trials-per-task",
+      "citations": [
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "eval.started MUST carry trialCount"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.the-per-task-trials-block-and-trialstats-follow-the-b-formulas",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 200,
+      "title": "the per-task trials block and trialStats follow the §B formulas",
+      "explicitId": "openwop.requirement.0244.trial-stats-formula",
+      "citations": [
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-miss-under-onmiss-fail-fails-the-trial-and-is-counted",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 223,
+      "title": "a fixture miss under onMiss \"fail\" fails the trial and is counted",
+      "explicitId": "openwop.requirement.0244.fixture-miss-fails",
+      "citations": [
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "the missed task MUST report fixtureMissCount 1"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "a miss under onMiss \"fail\" MUST fail the trial"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "a trial that fails closed MUST report score 0 in the summary"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "the missed task MUST emit eval.scored"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "eval.scored for a trial that fails closed MUST carry score 0"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "a served task MUST still pass"
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "a served task MUST NOT report a miss"
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-an-unadvertised-rfc-0244-feature-is-refused-at-createrun",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 243,
+      "title": "a suite using an unadvertised RFC 0244 feature is refused at createRun",
+      "explicitId": "openwop.requirement.0244.unadvertised-feature-refused",
+      "citations": [
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": null,
+          "interpolated": true
+        },
+        {
+          "section": "agents.schema.json §evalSuite (RFC 0244)",
+          "requirement": "the refusal MUST carry validation_error"
+        }
+      ]
+    },
     {
       "id": "openwop.it.v2-event-type-closed.every-emitted-type-is-a-registered-protocol-type-or-a-vendor-type-under-an-adver",
       "file": "v2-event-type-closed.test.ts",
@@ -45856,7 +46072,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.every-fixture-id-mentioned-in-fixtures-md-has-a-corresponding-json",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1449,
+      "line": 1453,
       "title": "every fixture id mentioned in fixtures.md has a corresponding JSON",
       "explicitId": "openwop.it.spec-corpus-validity.every-fixture-id-mentioned-in-fixtures-md-has-a-corresponding-json",
       "citations": [
@@ -45870,7 +46086,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.every-fixture-json-file-is-referenced-by-fixtures-md",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1491,
+      "line": 1495,
       "title": "every fixture JSON file is referenced by fixtures.md",
       "explicitId": "openwop.it.spec-corpus-validity.every-fixture-json-file-is-referenced-by-fixtures-md",
       "citations": [
@@ -45884,7 +46100,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.a-sample-bundle-validates-against-conformance-certification-bundle-schema-json",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1544,
+      "line": 1548,
       "title": "a sample bundle validates against conformance-certification-bundle.schema.json",
       "explicitId": "openwop.it.spec-corpus-validity.a-sample-bundle-validates-against-conformance-certification-bundle-schema-json",
       "citations": [
@@ -45898,7 +46114,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.verifybundle-accepts-a-claim-that-is-discovery-derivable-and-floor-proven-b",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1550,
+      "line": 1554,
       "title": "verifyBundle ACCEPTS a claim that is discovery-derivable AND floor-proven (§B)",
       "explicitId": "openwop.it.spec-corpus-validity.verifybundle-accepts-a-claim-that-is-discovery-derivable-and-floor-proven-b",
       "citations": [
@@ -45911,7 +46127,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.verifybundle-rejects-a-profile-its-discovery-document-does-not-derive-b-1",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1557,
+      "line": 1561,
       "title": "verifyBundle REJECTS a profile its discovery document does not derive (§B(1))",
       "explicitId": "openwop.it.spec-corpus-validity.verifybundle-rejects-a-profile-its-discovery-document-does-not-derive-b-1",
       "citations": [
@@ -45924,7 +46140,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.verifybundle-rejects-a-bundle-missing-a-floor-scenario-b-2",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1567,
+      "line": 1571,
       "title": "verifyBundle REJECTS a bundle missing a floor scenario (§B(2))",
       "explicitId": "openwop.it.spec-corpus-validity.verifybundle-rejects-a-bundle-missing-a-floor-scenario-b-2",
       "citations": [
@@ -45937,7 +46153,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.the-committed-bundle-file-exists-generated-by-certify",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1604,
+      "line": 1608,
       "title": "the committed bundle file exists (generated by --certify)",
       "explicitId": "openwop.it.spec-corpus-validity.the-committed-bundle-file-exists-generated-by-certify",
       "citations": [
@@ -45951,7 +46167,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.the-committed-reference-bundle-validates-against-the-bundle-schema-a",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1608,
+      "line": 1612,
       "title": "the committed reference bundle validates against the bundle schema (§A)",
       "explicitId": "openwop.it.spec-corpus-validity.the-committed-reference-bundle-validates-against-the-bundle-schema-a",
       "citations": [
@@ -45965,7 +46181,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.verifybundle-rejects-the-committed-reference-bundle-it-is-invalidated-rfc-0148-d",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1615,
+      "line": 1619,
       "title": "verifyBundle REJECTS the committed reference bundle — it is `invalidated` (RFC 0148 §D)",
       "explicitId": "openwop.it.spec-corpus-validity.verifybundle-rejects-the-committed-reference-bundle-it-is-invalidated-rfc-0148-d",
       "citations": [
@@ -45986,7 +46202,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.discovery-sha256-is-the-canonical-json-sha-256-of-the-captured-discovery-documen",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1641,
+      "line": 1645,
       "title": "discovery.sha256 is the canonical-JSON SHA-256 of the captured discovery.document",
       "explicitId": "openwop.it.spec-corpus-validity.discovery-sha256-is-the-canonical-json-sha-256-of-the-captured-discovery-documen",
       "citations": [
@@ -45999,7 +46215,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.the-requestbody-closes-at-the-composition-unevaluatedproperties-not-inside-a-bra",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1702,
+      "line": 1706,
       "title": "the requestBody closes at the composition (unevaluatedProperties), not inside a branch",
       "explicitId": "openwop.it.spec-corpus-validity.the-requestbody-closes-at-the-composition-unevaluatedproperties-not-inside-a-bra",
       "citations": [
@@ -46012,7 +46228,7 @@
     {
       "id": "openwop.it.spec-corpus-validity.canonical-createrun-bodies-pass-the-composed-schema-an-undeclared-property-fails",
       "file": "spec-corpus-validity.test.ts",
-      "line": 1717,
+      "line": 1721,
       "title": "canonical createRun bodies PASS the composed schema; an undeclared property FAILS",
       "explicitId": "openwop.it.spec-corpus-validity.canonical-createrun-bodies-pass-the-composed-schema-an-undeclared-property-fails",
       "citations": [

package/scenario-majors.json

@@ -1,9 +1,9 @@
 {
   "$comment": "GENERATED by conformance/scripts/generate-scenario-majors.mjs (RFC 0168 §D.3). Do not edit; add a file to BOTH_MAJORS in the generator to target both majors.",
   "counts": {
-    "files": 657,
+    "files": 658,
     "v1": 455,
-    "v2": 217
+    "v2": 218
   },
   "majors": {
     "a2a-1-0-agent-card.test.ts": [
@@ -1479,6 +1479,9 @@
     "v2-eval-mode-unadvertised-refused.test.ts": [
       2
     ],
+    "v2-eval-trajectory-trials.test.ts": [
+      2
+    ],
     "v2-event-type-closed.test.ts": [
       2
     ],

package/schemas/CORPUS-STAMP.json

@@ -1,17 +1,17 @@
 {
   "_comment": "Provenance of @openwop/spec-artifacts (RFC 0168 §D.2). files: SHA-256 per file; the conformance suite compares the installed peer against dist/spec-artifacts.lock.json at start.",
   "package": "@openwop/spec-artifacts",
-  "version": "2.46.7",
-  "corpusTag": "v2.46.7",
+  "version": "2.46.9",
+  "corpusTag": "v2.46.9",
   "files": {
     "api/.redocly.lint-ignore.yaml": "bf5a8350b88a72fa43f59605ed8d903ed24b6cfccda5e45509c9f6ed9ee4e712",
     "api/asyncapi.yaml": "d5ecb9ee6114582be3b1f662c84bfac9ae96dae7bacb853e461168f70a8e1c7d",
     "api/grpc/openwop.proto": "c3e72bb17cba514ee98feb6434e6c9b6ea6795bfd086489ec69fd882dd1ad977",
     "api/openapi.yaml": "ac6de5303872664b78ff419c035915593e01599e02835e777a0123de1baeb398",
     "api/redocly.yaml": "b0604c89b2ca6d5076ec25725c539dad44a741a811fe524439ee6daef8baa09f",
-    "api/seams-v2.yaml": "1f51bb8566237c46f9df4d262f218667143e72664a56403838787360f0e543fb",
-    "api/v2/asyncapi.yaml": "a4f623300eee3948f8debdf07cdf82c26ae7e6ceb9a6a15bd72a31adcd103690",
-    "api/v2/openapi.yaml": "f1229644f8d1fa6d9e9b7e3ed83595e45daaf20ba4c8cad6b52fbdfa335b3575",
+    "api/seams-v2.yaml": "24976e53171c747270890e0f2591ccc2630477824c56edc003d00135bb89daea",
+    "api/v2/asyncapi.yaml": "0da749b02294f7e8c9ded99858bb15296b250b0f1a58d28e5a3f9f289e477d34",
+    "api/v2/openapi.yaml": "b05a3b2e80b7348f0f02647dbd117dcda1604010b59da62a9871139f79726311",
     "api/v2/redocly.yaml": "1e66b60e6118ad11a823bb620678be464d99dfe50a40e3e6f93ec9429b88b34c",
     "schemas/README.md": "276fc1d51eb08e14e9127b9fd27761d7046ee33c12000f3a64a801322db8bf58",
     "schemas/a2a-task-state.schema.json": "75d5049dea8bd873ff0e7546f1c60c8d36c219bec7084264360be54a8be30ae1",
@@ -103,7 +103,7 @@
     "schemas/v2/a2ui-surface-delta-frame.schema.json": "6889c5da3fe7e76cb614038c0eea6774776fa640448558b258eb87fb50cb9b64",
     "schemas/v2/agent-deployment-transition.schema.json": "93a334ff042d51378c89fdd7134664271bf9592419bb5e3e00821ed06bc1904e",
     "schemas/v2/agent-deployment.schema.json": "d59b43fb62b522005ee22bc3bc53775a6777493e5e42d71812abc347625fa7fe",
-    "schemas/v2/agent-eval-suite.schema.json": "b565c9af1fd665e11884e57525a581914deaa80a86e7a186a6c6623b207a55d0",
+    "schemas/v2/agent-eval-suite.schema.json": "e80439bc6873a5d9a14122f81d16f7650104018d814bd316aafd3fc8b8225604",
     "schemas/v2/agent-inventory-response.schema.json": "5f74c3da946dfc3114624567a989717748081ab947b4d2af3f49034d67f4306b",
     "schemas/v2/agent-manifest.schema.json": "004ff150e955deebb9e64273de5093f5144b9b3895b09953e65aa74b61140778",
     "schemas/v2/agent-org-chart.schema.json": "113b079e1dce95908467cbef7f83e076fdb82aab5b0678fc1a43e32bfc9cbadf",
@@ -117,7 +117,7 @@
     "schemas/v2/artifact.schema.json": "85b9dc284d39b99d2a66fa8cbf47f2dd3d8526d3bd6cc411ad315b372de38975",
     "schemas/v2/audit-verify-result.schema.json": "14af8e9283510f0a40e48a2059c929ee44e0fc9fa3cc66a95bfbe90b17c48297",
     "schemas/v2/budget-policy.schema.json": "1de68095db0beb5c7a4be629de26e79f62ebbe748a88d46f36e768edfd1f0d68",
-    "schemas/v2/capabilities.schema.json": "ac237b887fa6e3117eddf5568bb7c2a2664f491c140968b060a173e12e79d643",
+    "schemas/v2/capabilities.schema.json": "a717d34d98201cefe1769421c468d22e124d1a46881f60683cb6e103843dbc9a",
     "schemas/v2/certification-bundle.schema.json": "5cfa7fb5247160333e33a58b60d42bcd4e5269d8391d6a112d301e14828892ed",
     "schemas/v2/channel-presence-payload.schema.json": "fc03df0c20cc12a25f7ee6a47503a4c5931d79fe3d5ff24815e8fd7c1bcb9a87",
     "schemas/v2/channel-written-payload.schema.json": "ccecff3c71a3275ad8db035ac0e04db25e6f8d23cf091ddbcd5a60d1be1cbb6b",
@@ -145,7 +145,7 @@
     "schemas/v2/envelopes/schema.response.schema.json": "d049a081267c06c28879c331b147f3ed821b14ccfd93005c67aff64db0a8c68e",
     "schemas/v2/envelopes/ui.a2ui-surface.schema.json": "e92a2548ba65454c9e0a34f72ea6768d18f59e61fa5da7668d0b2d5baddaddc8",
     "schemas/v2/error-envelope.schema.json": "8ddb1d80330dfef17572a74dfe3a53e1a06411e4b0af6a6f1dd6e7e7461da987",
-    "schemas/v2/eval-summary.schema.json": "2f384daf05b8f3b4f604688cae504d6d9971ba41b8c771e55fe9a17e09e86b7a",
+    "schemas/v2/eval-summary.schema.json": "e670fad6c4c487f80edf3832ccd427b534b2066fbbc1cc2c158e205d6e7e3f6b",
     "schemas/v2/export-bundle.schema.json": "bd28905a7729437ab7f2ad8bb33e13a371b2bf5605a7eb2962a0d246de312d69",
     "schemas/v2/form-content-pack-manifest.schema.json": "a1942a124e250a6e9fbdb6b811c5a125e78fbcbcaa46dfc71d0636f24710a339",
     "schemas/v2/frontend-plugin-manifest.schema.json": "b279da288d542906a72b8cb4e872e001de632eaff539e0259b9cce287f82bf8f",
@@ -174,7 +174,7 @@
     "schemas/v2/residency.schema.json": "3fd23c9732beb140504e81560fc426eb6764d547e56da5fb92c5a5c7bf04db3a",
     "schemas/v2/run-ancestry-response.schema.json": "334c17f2062dea62726ca1d07b8be3f169cdc422af843c1def5fb317039814c9",
     "schemas/v2/run-diff-response.schema.json": "8bdb2045c274580038bd74781f7d062f5049839797c164a89de4ed89edc53a74",
-    "schemas/v2/run-event-payloads.schema.json": "ff80204f4a9a55da482d8c7d54d6dbb8682e13fff155d4fc26fa5b7fa0790b83",
+    "schemas/v2/run-event-payloads.schema.json": "e21d0cb64cb969be32e49e87f3af418bacb40e8fcb515a848dfe7d9609d8d0b6",
     "schemas/v2/run-event.schema.json": "2f0d7416bd376c56affe97b6b2b49981532e8e9ff45c0a162cf806c78b3e7fb7",
     "schemas/v2/run-list-response.schema.json": "c2dbc34768dcf64528914a213aa3cb55952e8e3e90a64b04a9423b2a19749d38",
     "schemas/v2/run-options.schema.json": "f0e9f494d80086c2ed2cf9fe87df54e5b4e66601699c4a1089f4edbc7ba58867",
@@ -208,14 +208,14 @@
     "schemas/workspace-file.schema.json": "464de85c2a068243084ee9c1d969bc7cd5d8f7948574e58450d6493c38a0e1e4",
     "spec/v1/alias-detectors.json": "22444c22bb88c568339ef512553f9fc38146e22b566d11ab8608f655d688d19c",
     "spec/v1/capability-declaration-classes.json": "e7729aed5c4b4e1dd02abab0530f14cc95f5d4070fe51fb139e7f5cccefa00c6",
-    "spec/v1/core-standard-manifest.json": "5c3946728d3bb64d4cf290e93156788445da37272d4e44a4da803be695f9b543",
+    "spec/v1/core-standard-manifest.json": "baa24251d00109b8fa47b5dfd8c6cecab312b75240ff5519e71f704eb7fd03f0",
     "spec/v1/deprecations.json": "e723bc11d7272c71789ad1303bb7f21809a8a024abf57594f9d5c35dc199ece9",
     "spec/v1/deprecations.schema.json": "3e393c405d2a41b467d8c5e3c468549078df1ce6a6d2588fc488097b95d9b55b",
     "spec/v1/end-of-support.json": "35f0bf3b70fe78b16e2a2fd81a153fc27e703f6c16376c6fe7cdf1f4f2c4e6d4",
     "spec/v1/event-codemap.json": "3da60d884157793a360da532a9fcbbfb5285636db325a74cec94b34622186d97",
     "spec/v1/event-codemap.schema.json": "d05933b2e88103aff51a2f774df97b9bda065e0fc88dbd9b114cc47a32189174",
     "spec/v1/extensions.json": "79a60754aa16cbdbdb604f8e5c00038af535a1690a4d07bd5ff7c38f0256a7bc",
-    "spec/v1/gaps.json": "e31dd0a2c440ff38469a8033e3324f372278ee94d209e0f84207d2111123f506",
+    "spec/v1/gaps.json": "a3ab3af1f48fe9f36283d296b06970a4dd759d4e3beb731828831f09e8e8f53b",
     "spec/v1/gaps.schema.json": "8fd83259f556553c9df0f53e7a82ca8c2a4771d471197f69b4e2ae0ceeacfd66",
     "spec/v1/migrations.json": "4c50760be70cb644d02427bcaf5749606700edf4513f423903cf3b1d12f5c300",
     "spec/v1/migrations.schema.json": "886779aa6c22e646db097f5df210adb018a4dd14a7b815465a18c8a7056c8f72",
@@ -281,7 +281,7 @@
     "spec/v2/ext/sandbox-runtime-notes/README.md": "0d1290fb1e20b2e35c0b15051541ab6546e4e2a6f693b4f912cb3f7d6a65e92c",
     "spec/v2/ext/webResearch/README.md": "b675e6f6646128d781b661f754c2393eed86e5935d16c3a22df582a5ce025b9b",
     "spec/v2/facets/a2a.schema.json": "b68bf5e29e455e46c2ad7cfcccf5963730ec2310ff9db81884550d8df30fd6ef",
-    "spec/v2/facets/agents.schema.json": "3be9326f7db9c7b9b4268aec731c0c2b087eba33c9bb31669ec75a18961bc40e",
+    "spec/v2/facets/agents.schema.json": "62a979d23219ca684bc16f91f81a4c92c2cb56e0f084d2e4f694c03e0c7125df",
     "spec/v2/facets/aiProviders.schema.json": "4eb911c2403b8f618325194855a1326680fbda2549dfbfe86a70dd2743f8fb12",
     "spec/v2/facets/auditLogIntegrity.schema.json": "9a7f81320c8f940c6f4129bc7dd895a4063231c5458a717a08c2cbf9bb886c46",
     "spec/v2/facets/auth.schema.json": "7d98bc81a0f7d307460d039eebfebd21388091bfe1d538762583940f94c0bd2a",
@@ -313,9 +313,9 @@
     "spec/v2/path-manifest.json": "7b5679a4dad98c80d855361d88a98c8a4e045af15e310862a05b285b815dc9fc",
     "spec/v2/peer-dependency-aliases.json": "d10299280abee08258502925bc327293ee413e0108cd6e6ec75ff6110653308d",
     "spec/v2/profiles.json": "b382b261c7844798575f7c1b37c934f0da19846016a2337d883d4f5ef6e05479",
-    "spec/v2/release.json": "1348d062c28c19897650cecb1f968c61adece3a61fdc4ecb8b7d8675468b7f65",
+    "spec/v2/release.json": "3d39622c2a2f6e53ad2bc4fc45439006a0877a3b4b4000de5ea4c5fc90863b64",
     "spec/v2/retention-floors.json": "eaf3722d95c79947af1d4269ef85117e126518c588cfcf1a2b21b97269f51624",
-    "spec/v2/surface-baseline.json": "185c226ce4868f6d435275b1005890c9328c18f2f86163f4622cce8b0dec63e3"
+    "spec/v2/surface-baseline.json": "8782769b454e3f781b889d06fe11978bc6da5eb1993935c205bf05c1b09e5d67"
   },
-  "corpusCommit": "b7f55b6d06df56983be5a9e9d518dd5a8fae8b29"
+  "corpusCommit": "30aa92326225c92edeea084e2c3c484ca18e10e8"
 }

package/src/lib/eval-trajectory.ts

@@ -0,0 +1,146 @@
+/**
+ * RFC 0244 — the suite's reference grader for eval trajectories and trials.
+ *
+ * Pure functions, no I/O. `v2-eval-trajectory-trials.test.ts` uses them to
+ * recompute what a host must report, so a host's grade is checked against the
+ * normative definition rather than against itself:
+ *
+ *   gradeTrajectory   §A — the four match modes, `forbidden`, `argsMatch`,
+ *                     and the mode-independent `matchedCount` (a maximum
+ *                     one-to-one matching, ignoring order).
+ *   passAtK / passHatK / scoreStderr   §B — the normative formulas.
+ *
+ * @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md
+ * @see schemas/v2/agent-eval-suite.schema.json, schemas/v2/eval-summary.schema.json
+ */
+
+export type ArgsMatch = 'ignore' | 'exact' | 'subset';
+export type MatchMode = 'exact' | 'in-order' | 'any-order' | 'subset';
+export interface ExpectedCall { readonly tool: string; readonly args?: unknown; readonly argsMatch?: ArgsMatch }
+export interface ObservedCall { readonly tool: string; readonly inputs?: unknown }
+export interface TrajectorySpec { readonly match: MatchMode; readonly calls: readonly ExpectedCall[]; readonly forbidden?: readonly string[] }
+export interface TrajectoryGrade {
+  readonly mode: MatchMode;
+  readonly matched: boolean;
+  readonly expectedCount: number;
+  readonly observedCount: number;
+  readonly matchedCount: number;
+  readonly forbiddenCount: number;
+}
+
+const isRecord = (v: unknown): v is Record<string, unknown> => v !== null && typeof v === 'object' && !Array.isArray(v);
+
+/** JSON deep equality, key-order insensitive. */
+export function jsonEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (Array.isArray(a) && Array.isArray(b)) return a.length === b.length && a.every((x, i) => jsonEqual(x, b[i]));
+  if (isRecord(a) && isRecord(b)) {
+    const ka = Object.keys(a);
+    return ka.length === Object.keys(b).length && ka.every((k) => k in b && jsonEqual(a[k], b[k]));
+  }
+  return false;
+}
+
+/** `subset`: every key path in `expected` is present in `actual` with a JSON-deep-equal value. */
+export function jsonSubset(expected: unknown, actual: unknown): boolean {
+  if (isRecord(expected)) return isRecord(actual) && Object.keys(expected).every((k) => k in actual && jsonSubset(expected[k], actual[k]));
+  return jsonEqual(expected, actual);
+}
+
+export function callMatches(e: ExpectedCall, o: ObservedCall): boolean {
+  if (e.tool !== o.tool) return false;
+  const mode = e.argsMatch ?? 'ignore';
+  if (mode === 'ignore' || e.args === undefined) return true;
+  return mode === 'exact' ? jsonEqual(e.args, o.inputs) : jsonSubset(e.args, o.inputs);
+}
+
+/** Size of a maximum one-to-one matching (Kuhn's augmenting paths). */
+function maxMatching(expected: readonly ExpectedCall[], observed: readonly ObservedCall[]): number {
+  const owner: (number | undefined)[] = new Array(observed.length).fill(undefined);
+  const augment = (i: number, seen: boolean[]): boolean => {
+    for (let j = 0; j < observed.length; j++) {
+      if (seen[j] || !callMatches(expected[i]!, observed[j]!)) continue;
+      seen[j] = true;
+      if (owner[j] === undefined || augment(owner[j]!, seen)) { owner[j] = i; return true; }
+    }
+    return false;
+  };
+  let n = 0;
+  for (let i = 0; i < expected.length; i++) if (augment(i, new Array(observed.length).fill(false))) n++;
+  return n;
+}
+
+function inOrder(expected: readonly ExpectedCall[], observed: readonly ObservedCall[]): boolean {
+  let j = 0;
+  for (const e of expected) {
+    while (j < observed.length && !callMatches(e, observed[j]!)) j++;
+    if (j === observed.length) return false;
+    j++;
+  }
+  return true;
+}
+
+export function gradeTrajectory(spec: TrajectorySpec, observed: readonly ObservedCall[]): TrajectoryGrade {
+  const forbidden = new Set(spec.forbidden ?? []);
+  const forbiddenCount = observed.filter((o) => forbidden.has(o.tool)).length;
+  const matchedCount = maxMatching(spec.calls, observed);
+  let modeOk: boolean;
+  switch (spec.match) {
+    case 'exact': modeOk = observed.length === spec.calls.length && spec.calls.every((e, i) => callMatches(e, observed[i]!)); break;
+    case 'in-order': modeOk = inOrder(spec.calls, observed); break;
+    case 'any-order': modeOk = matchedCount === spec.calls.length; break;
+    case 'subset': modeOk = observed.every((o) => spec.calls.some((e) => callMatches(e, o))); break;
+  }
+  return { mode: spec.match, matched: modeOk && forbiddenCount === 0, expectedCount: spec.calls.length, observedCount: observed.length, matchedCount, forbiddenCount };
+}
+
+/**
+ * C(a, k) / C(b, k) as an exact rational, converted to a double once. The
+ * ratio telescopes to Π (a−i)/(b−i) for i < k, so no factorial is formed and
+ * the result is stable for large n (pass^k(200, 100, 50) ≈ 2.2e-19).
+ */
+function binomRatio(a: number, b: number, k: number): number {
+  if (k < 0 || k > b) throw new RangeError(`k=${k} outside 0..${b}`);
+  if (k > a) return 0;
+  let num = 1n;
+  let den = 1n;
+  for (let i = 0; i < k; i++) { num *= BigInt(a - i); den *= BigInt(b - i); }
+  return ratioToNumber(num, den);
+}
+
+/** num/den (num ≤ den, both > 0) to the nearest double, without overflowing either. */
+function ratioToNumber(num: bigint, den: bigint): number {
+  if (num === 0n) return 0;
+  const shift = den.toString(2).length - num.toString(2).length + 64;
+  const q = (num << BigInt(shift)) / den;
+  let r = Number(q);
+  let e = shift;
+  while (e > 1000) { r *= 2 ** -1000; e -= 1000; }
+  return r * 2 ** -e;
+}
+
+/** pass@k = 1 − C(n−c, k) / C(n, k); 0 when c = 0. */
+export function passAtK(n: number, c: number, k: number): number {
+  if (c === 0) return 0;
+  return 1 - binomRatio(n - c, n, k);
+}
+
+/** pass^k = C(c, k) / C(n, k); 0 when c < k. */
+export function passHatK(n: number, c: number, k: number): number {
+  if (c < k) return 0;
+  return binomRatio(c, n, k);
+}
+
+/** RFC 0244 §B tolerance: a host value agrees with the formula when |host − ref| ≤ 1e-9 · max(1, |ref|). */
+export function withinTolerance(host: number, ref: number): boolean {
+  return Number.isFinite(host) && Math.abs(host - ref) <= 1e-9 * Math.max(1, Math.abs(ref));
+}
+
+/** Sample sd of the per-task mean scores over √T; undefined when T < 2. */
+export function scoreStderr(taskMeans: readonly number[]): number | undefined {
+  const t = taskMeans.length;
+  if (t < 2) return undefined;
+  const mean = taskMeans.reduce((a, b) => a + b, 0) / t;
+  const variance = taskMeans.reduce((a, b) => a + (b - mean) ** 2, 0) / (t - 1);
+  return Math.sqrt(variance) / Math.sqrt(t);
+}