@openwop/spec-artifacts 2.46.7 → 2.46.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. package/CORPUS-STAMP.json +15 -15
  2. package/api/seams-v2.yaml +1 -1
  3. package/api/v2/asyncapi.yaml +1 -1
  4. package/api/v2/openapi.yaml +1 -1
  5. package/package.json +1 -1
  6. package/schemas/v2/agent-eval-suite.schema.json +118 -5
  7. package/schemas/v2/capabilities.schema.json +28 -1
  8. package/schemas/v2/eval-summary.schema.json +166 -0
  9. package/schemas/v2/run-event-payloads.schema.json +25 -1
  10. package/spec/v1/core-standard-manifest.json +2 -2
  11. package/spec/v1/gaps.json +187 -0
  12. package/spec/v2/facets/agents.schema.json +28 -1
  13. package/spec/v2/release.json +2 -2
  14. package/spec/v2/surface-baseline.json +148 -2
package/CORPUS-STAMP.json CHANGED
@@ -1,17 +1,17 @@
1
1
  {
2
2
  "_comment": "Provenance of @openwop/spec-artifacts (RFC 0168 §D.2). files: SHA-256 per file; the conformance suite compares the installed peer against dist/spec-artifacts.lock.json at start.",
3
3
  "package": "@openwop/spec-artifacts",
4
- "version": "2.46.7",
5
- "corpusTag": "v2.46.7",
4
+ "version": "2.46.8",
5
+ "corpusTag": "v2.46.8",
6
6
  "files": {
7
7
  "api/.redocly.lint-ignore.yaml": "bf5a8350b88a72fa43f59605ed8d903ed24b6cfccda5e45509c9f6ed9ee4e712",
8
8
  "api/asyncapi.yaml": "d5ecb9ee6114582be3b1f662c84bfac9ae96dae7bacb853e461168f70a8e1c7d",
9
9
  "api/grpc/openwop.proto": "c3e72bb17cba514ee98feb6434e6c9b6ea6795bfd086489ec69fd882dd1ad977",
10
10
  "api/openapi.yaml": "ac6de5303872664b78ff419c035915593e01599e02835e777a0123de1baeb398",
11
11
  "api/redocly.yaml": "b0604c89b2ca6d5076ec25725c539dad44a741a811fe524439ee6daef8baa09f",
12
- "api/seams-v2.yaml": "1f51bb8566237c46f9df4d262f218667143e72664a56403838787360f0e543fb",
13
- "api/v2/asyncapi.yaml": "a4f623300eee3948f8debdf07cdf82c26ae7e6ceb9a6a15bd72a31adcd103690",
14
- "api/v2/openapi.yaml": "f1229644f8d1fa6d9e9b7e3ed83595e45daaf20ba4c8cad6b52fbdfa335b3575",
12
+ "api/seams-v2.yaml": "f9a0c7da940a9f31a689bea929c33b4e70b0293f955e2a91329ec46d86eced3f",
13
+ "api/v2/asyncapi.yaml": "77259d3d85bbc4cd83daa54382c7cfa08a84fb253cb0b1f7b8005a9a38d94ff7",
14
+ "api/v2/openapi.yaml": "139ef732f96270dc64e2365a42cdaa23f3177a80db63c9f16ee36f16935446f7",
15
15
  "api/v2/redocly.yaml": "1e66b60e6118ad11a823bb620678be464d99dfe50a40e3e6f93ec9429b88b34c",
16
16
  "schemas/README.md": "276fc1d51eb08e14e9127b9fd27761d7046ee33c12000f3a64a801322db8bf58",
17
17
  "schemas/a2a-task-state.schema.json": "75d5049dea8bd873ff0e7546f1c60c8d36c219bec7084264360be54a8be30ae1",
@@ -103,7 +103,7 @@
103
103
  "schemas/v2/a2ui-surface-delta-frame.schema.json": "6889c5da3fe7e76cb614038c0eea6774776fa640448558b258eb87fb50cb9b64",
104
104
  "schemas/v2/agent-deployment-transition.schema.json": "93a334ff042d51378c89fdd7134664271bf9592419bb5e3e00821ed06bc1904e",
105
105
  "schemas/v2/agent-deployment.schema.json": "d59b43fb62b522005ee22bc3bc53775a6777493e5e42d71812abc347625fa7fe",
106
- "schemas/v2/agent-eval-suite.schema.json": "b565c9af1fd665e11884e57525a581914deaa80a86e7a186a6c6623b207a55d0",
106
+ "schemas/v2/agent-eval-suite.schema.json": "e6925711c478f35c03dcc7d1c8b7c8ec0814d468ad1ed2beb3e54405d8eb0752",
107
107
  "schemas/v2/agent-inventory-response.schema.json": "5f74c3da946dfc3114624567a989717748081ab947b4d2af3f49034d67f4306b",
108
108
  "schemas/v2/agent-manifest.schema.json": "004ff150e955deebb9e64273de5093f5144b9b3895b09953e65aa74b61140778",
109
109
  "schemas/v2/agent-org-chart.schema.json": "113b079e1dce95908467cbef7f83e076fdb82aab5b0678fc1a43e32bfc9cbadf",
@@ -117,7 +117,7 @@
117
117
  "schemas/v2/artifact.schema.json": "85b9dc284d39b99d2a66fa8cbf47f2dd3d8526d3bd6cc411ad315b372de38975",
118
118
  "schemas/v2/audit-verify-result.schema.json": "14af8e9283510f0a40e48a2059c929ee44e0fc9fa3cc66a95bfbe90b17c48297",
119
119
  "schemas/v2/budget-policy.schema.json": "1de68095db0beb5c7a4be629de26e79f62ebbe748a88d46f36e768edfd1f0d68",
120
- "schemas/v2/capabilities.schema.json": "ac237b887fa6e3117eddf5568bb7c2a2664f491c140968b060a173e12e79d643",
120
+ "schemas/v2/capabilities.schema.json": "a717d34d98201cefe1769421c468d22e124d1a46881f60683cb6e103843dbc9a",
121
121
  "schemas/v2/certification-bundle.schema.json": "5cfa7fb5247160333e33a58b60d42bcd4e5269d8391d6a112d301e14828892ed",
122
122
  "schemas/v2/channel-presence-payload.schema.json": "fc03df0c20cc12a25f7ee6a47503a4c5931d79fe3d5ff24815e8fd7c1bcb9a87",
123
123
  "schemas/v2/channel-written-payload.schema.json": "ccecff3c71a3275ad8db035ac0e04db25e6f8d23cf091ddbcd5a60d1be1cbb6b",
@@ -145,7 +145,7 @@
145
145
  "schemas/v2/envelopes/schema.response.schema.json": "d049a081267c06c28879c331b147f3ed821b14ccfd93005c67aff64db0a8c68e",
146
146
  "schemas/v2/envelopes/ui.a2ui-surface.schema.json": "e92a2548ba65454c9e0a34f72ea6768d18f59e61fa5da7668d0b2d5baddaddc8",
147
147
  "schemas/v2/error-envelope.schema.json": "8ddb1d80330dfef17572a74dfe3a53e1a06411e4b0af6a6f1dd6e7e7461da987",
148
- "schemas/v2/eval-summary.schema.json": "2f384daf05b8f3b4f604688cae504d6d9971ba41b8c771e55fe9a17e09e86b7a",
148
+ "schemas/v2/eval-summary.schema.json": "fc8ea719a8241eacd12e90d36934369386673bd50edebe6814782a19605a7776",
149
149
  "schemas/v2/export-bundle.schema.json": "bd28905a7729437ab7f2ad8bb33e13a371b2bf5605a7eb2962a0d246de312d69",
150
150
  "schemas/v2/form-content-pack-manifest.schema.json": "a1942a124e250a6e9fbdb6b811c5a125e78fbcbcaa46dfc71d0636f24710a339",
151
151
  "schemas/v2/frontend-plugin-manifest.schema.json": "b279da288d542906a72b8cb4e872e001de632eaff539e0259b9cce287f82bf8f",
@@ -174,7 +174,7 @@
174
174
  "schemas/v2/residency.schema.json": "3fd23c9732beb140504e81560fc426eb6764d547e56da5fb92c5a5c7bf04db3a",
175
175
  "schemas/v2/run-ancestry-response.schema.json": "334c17f2062dea62726ca1d07b8be3f169cdc422af843c1def5fb317039814c9",
176
176
  "schemas/v2/run-diff-response.schema.json": "8bdb2045c274580038bd74781f7d062f5049839797c164a89de4ed89edc53a74",
177
- "schemas/v2/run-event-payloads.schema.json": "ff80204f4a9a55da482d8c7d54d6dbb8682e13fff155d4fc26fa5b7fa0790b83",
177
+ "schemas/v2/run-event-payloads.schema.json": "24992db063ccab827710d462e834cc6299bf6f7aa651b118e30ef33506cc7aae",
178
178
  "schemas/v2/run-event.schema.json": "2f0d7416bd376c56affe97b6b2b49981532e8e9ff45c0a162cf806c78b3e7fb7",
179
179
  "schemas/v2/run-list-response.schema.json": "c2dbc34768dcf64528914a213aa3cb55952e8e3e90a64b04a9423b2a19749d38",
180
180
  "schemas/v2/run-options.schema.json": "f0e9f494d80086c2ed2cf9fe87df54e5b4e66601699c4a1089f4edbc7ba58867",
@@ -208,14 +208,14 @@
208
208
  "schemas/workspace-file.schema.json": "464de85c2a068243084ee9c1d969bc7cd5d8f7948574e58450d6493c38a0e1e4",
209
209
  "spec/v1/alias-detectors.json": "22444c22bb88c568339ef512553f9fc38146e22b566d11ab8608f655d688d19c",
210
210
  "spec/v1/capability-declaration-classes.json": "e7729aed5c4b4e1dd02abab0530f14cc95f5d4070fe51fb139e7f5cccefa00c6",
211
- "spec/v1/core-standard-manifest.json": "5c3946728d3bb64d4cf290e93156788445da37272d4e44a4da803be695f9b543",
211
+ "spec/v1/core-standard-manifest.json": "031419b7f2fa27bb1d550d5f8caf92cea889de73998aa723859c8a3e29b4458e",
212
212
  "spec/v1/deprecations.json": "e723bc11d7272c71789ad1303bb7f21809a8a024abf57594f9d5c35dc199ece9",
213
213
  "spec/v1/deprecations.schema.json": "3e393c405d2a41b467d8c5e3c468549078df1ce6a6d2588fc488097b95d9b55b",
214
214
  "spec/v1/end-of-support.json": "35f0bf3b70fe78b16e2a2fd81a153fc27e703f6c16376c6fe7cdf1f4f2c4e6d4",
215
215
  "spec/v1/event-codemap.json": "3da60d884157793a360da532a9fcbbfb5285636db325a74cec94b34622186d97",
216
216
  "spec/v1/event-codemap.schema.json": "d05933b2e88103aff51a2f774df97b9bda065e0fc88dbd9b114cc47a32189174",
217
217
  "spec/v1/extensions.json": "79a60754aa16cbdbdb604f8e5c00038af535a1690a4d07bd5ff7c38f0256a7bc",
218
- "spec/v1/gaps.json": "e31dd0a2c440ff38469a8033e3324f372278ee94d209e0f84207d2111123f506",
218
+ "spec/v1/gaps.json": "a3ab3af1f48fe9f36283d296b06970a4dd759d4e3beb731828831f09e8e8f53b",
219
219
  "spec/v1/gaps.schema.json": "8fd83259f556553c9df0f53e7a82ca8c2a4771d471197f69b4e2ae0ceeacfd66",
220
220
  "spec/v1/migrations.json": "4c50760be70cb644d02427bcaf5749606700edf4513f423903cf3b1d12f5c300",
221
221
  "spec/v1/migrations.schema.json": "886779aa6c22e646db097f5df210adb018a4dd14a7b815465a18c8a7056c8f72",
@@ -281,7 +281,7 @@
281
281
  "spec/v2/ext/sandbox-runtime-notes/README.md": "0d1290fb1e20b2e35c0b15051541ab6546e4e2a6f693b4f912cb3f7d6a65e92c",
282
282
  "spec/v2/ext/webResearch/README.md": "b675e6f6646128d781b661f754c2393eed86e5935d16c3a22df582a5ce025b9b",
283
283
  "spec/v2/facets/a2a.schema.json": "b68bf5e29e455e46c2ad7cfcccf5963730ec2310ff9db81884550d8df30fd6ef",
284
- "spec/v2/facets/agents.schema.json": "3be9326f7db9c7b9b4268aec731c0c2b087eba33c9bb31669ec75a18961bc40e",
284
+ "spec/v2/facets/agents.schema.json": "62a979d23219ca684bc16f91f81a4c92c2cb56e0f084d2e4f694c03e0c7125df",
285
285
  "spec/v2/facets/aiProviders.schema.json": "4eb911c2403b8f618325194855a1326680fbda2549dfbfe86a70dd2743f8fb12",
286
286
  "spec/v2/facets/auditLogIntegrity.schema.json": "9a7f81320c8f940c6f4129bc7dd895a4063231c5458a717a08c2cbf9bb886c46",
287
287
  "spec/v2/facets/auth.schema.json": "7d98bc81a0f7d307460d039eebfebd21388091bfe1d538762583940f94c0bd2a",
@@ -313,9 +313,9 @@
313
313
  "spec/v2/path-manifest.json": "7b5679a4dad98c80d855361d88a98c8a4e045af15e310862a05b285b815dc9fc",
314
314
  "spec/v2/peer-dependency-aliases.json": "d10299280abee08258502925bc327293ee413e0108cd6e6ec75ff6110653308d",
315
315
  "spec/v2/profiles.json": "b382b261c7844798575f7c1b37c934f0da19846016a2337d883d4f5ef6e05479",
316
- "spec/v2/release.json": "1348d062c28c19897650cecb1f968c61adece3a61fdc4ecb8b7d8675468b7f65",
316
+ "spec/v2/release.json": "c291a28051bf090d15ad928487c427566b37bcded475b7475312200d91af022d",
317
317
  "spec/v2/retention-floors.json": "eaf3722d95c79947af1d4269ef85117e126518c588cfcf1a2b21b97269f51624",
318
- "spec/v2/surface-baseline.json": "185c226ce4868f6d435275b1005890c9328c18f2f86163f4622cce8b0dec63e3"
318
+ "spec/v2/surface-baseline.json": "15f53de22504577387afe746758f4bf5c6e76f1d6190cc1e5c9dd18ebd77e0b9"
319
319
  },
320
- "corpusCommit": "b7f55b6d06df56983be5a9e9d518dd5a8fae8b29"
320
+ "corpusCommit": "2793cd29bd41ca40727d919971b3ebc8b5e61dcf"
321
321
  }
package/api/seams-v2.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  openapi: 3.1.0
2
2
  info:
3
3
  title: OpenWOP conformance seams profile (openwop-conformance-seams-v2)
4
- version: 2.46.7
4
+ version: 2.46.8
5
5
  description: 'The OpenWOP conformance seams profile (`openwop-conformance-seams-v2`): test-only operations in their own
6
6
  path space, `/conformance/seams/…`. The profile is versioned and validated against the canonical v2 schemas with no tolerance
7
7
  path. It is not a capability: a host advertises the profile, never a `testSeams` flag.
package/api/v2/asyncapi.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  asyncapi: 3.1.0
2
2
  info:
3
3
  title: OpenWOP v2 event streams
4
- version: 2.46.7
4
+ version: 2.46.8
5
5
  description: 'OpenWOP v2 event streams over SSE. The run-events channel''s address is the OpenAPI path key, and the server
6
6
  pathname is empty (bare origin). `streamMode` is a stream mode from the closed set, or a comma-separated combination of
7
7
  them; `values` never combines. `hostEvents` has a default address, `/host/events`; a host MAY declare another under `heartbeat.deliveryChannel`.
package/api/v2/openapi.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  openapi: 3.1.0
2
2
  info:
3
3
  title: OpenWOP v2 API
4
- version: 2.46.7
4
+ version: 2.46.8
5
5
  summary: REST surface for declaring, executing, suspending, resuming, and observing multi-step workflows.
6
6
  description: 'The OpenWOP v2 REST API. Paths are unversioned and served at the bare origin; a client selects a version with
7
7
  the `OpenWOP-Version` header from the host''s `protocolVersions[]`. Conformance seams and test-mode operations are not
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openwop/spec-artifacts",
3
- "version": "2.46.7",
3
+ "version": "2.46.8",
4
4
  "description": "The OpenWOP machine-readable contract: api/ (OpenAPI, AsyncAPI, the seams profile), schemas/ (v1 and v2 JSON Schemas), the spec/v1 and spec/v2 registries (errors, event codemap, declaration, deprecations, migrations, gaps) and CORPUS-STAMP.json — published from the openwop/openwop corpus tag; the peer dependency @openwop/openwop-conformance digest-checks at start (RFC 0168 §D.2).",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
package/schemas/v2/agent-eval-suite.schema.json CHANGED
@@ -81,6 +81,34 @@
81
81
  }
82
82
  }
83
83
  },
84
+ "trials": {
85
+ "type": "object",
86
+ "additionalProperties": false,
87
+ "description": "RFC 0244 §B. MAY. Repeated trials. A host MUST run each task `count` times, each as its own child agent invocation started from the same fixtures and `memorySeed` snapshot; memory one trial writes MUST NOT be visible to another trial of the run, and a fixture cursor resets per trial. `k` MUST NOT exceed `count` (a suite rule, not a schema conditional). `count` > 1 requires `agents.evalSuite.trials` with `count` <= `maxTrials`, else the suite is refused at run-create with `400 validation_error`. Cost ceilings count every trial.",
88
+ "properties": {
89
+ "count": {
90
+ "type": "integer",
91
+ "minimum": 1,
92
+ "default": 1,
93
+ "description": "Trials per task."
94
+ },
95
+ "k": {
96
+ "type": "integer",
97
+ "minimum": 1,
98
+ "description": "The k of pass@k and pass^k. Absent: `count`."
99
+ },
100
+ "passRule": {
101
+ "type": "string",
102
+ "enum": [
103
+ "all",
104
+ "any",
105
+ "mean"
106
+ ],
107
+ "default": "all",
108
+ "description": "A task passes when: `all`, every trial passed; `any`, at least one did; `mean`, the mean trial score >= `thresholds.passScore`."
109
+ }
110
+ }
111
+ },
84
112
  "tasks": {
85
113
  "type": "array",
86
114
  "minItems": 1,
@@ -114,9 +142,10 @@
114
142
  "type": "string",
115
143
  "enum": [
116
144
  "golden",
117
- "rubric"
145
+ "rubric",
146
+ "trajectory"
118
147
  ],
119
- "description": "Scoring mode for this task. A suite declaring a non-`golden`/`rubric` `modes` entry (e.g. `adversarial`) still scores each task via one of these two `kind`s."
148
+ "description": "Scoring mode for this task. A suite declaring a non-`golden`/`rubric` `modes` entry (e.g. `adversarial`) still scores each task via one of these two `kind`s. `trajectory` (RFC 0244 §A): the task is graded on its tool path alone; its score is 1 when `trajectory` matched and 0 otherwise, and it carries `trajectory` and neither `match` nor `rubric` (a suite rule, not a schema conditional: COMPATIBILITY.md §2.4)."
120
149
  },
121
150
  "match": {
122
151
  "type": "object",
@@ -166,6 +195,71 @@
166
195
  }
167
196
  }
168
197
  }
198
+ },
199
+ "trajectory": {
200
+ "type": "object",
201
+ "additionalProperties": false,
202
+ "required": [
203
+ "match",
204
+ "calls"
205
+ ],
206
+ "description": "RFC 0244 §A. MAY. The tool path the task is held to, graded from the `agent.tool-called` events in each trial's `agent.invocation-*` bracket whose `agentId` is the task's target agent (every call in the bracket when `includeDelegates` is true). On a `golden` or `rubric` task a trial MUST NOT pass unless the trajectory matched, and its score stays the output score. Requires `agents.evalSuite.trajectory`; a host that does not advertise it MUST refuse the suite at run-create with `400 validation_error`.",
207
+ "properties": {
208
+ "match": {
209
+ "type": "string",
210
+ "enum": [
211
+ "exact",
212
+ "in-order",
213
+ "any-order",
214
+ "subset"
215
+ ],
216
+ "description": "`exact`: same length, and the i-th observed call matches the i-th expected call. `in-order`: the expected calls match a subsequence of the observed calls, in order; extra calls allowed. `any-order`: each expected call matches a distinct observed call, any order; extra calls allowed. `subset`: every observed call matches some expected call; no tool outside `calls` was called."
217
+ },
218
+ "calls": {
219
+ "type": "array",
220
+ "items": {
221
+ "type": "object",
222
+ "additionalProperties": false,
223
+ "required": [
224
+ "tool"
225
+ ],
226
+ "properties": {
227
+ "tool": {
228
+ "type": "string",
229
+ "minLength": 1,
230
+ "description": "The `<scope>:<tool-id>` compared with the event's `toolName`."
231
+ },
232
+ "args": {
233
+ "description": "MAY. The expected arguments, compared under `argsMatch`. Suite content: never echoed onto the wire."
234
+ },
235
+ "argsMatch": {
236
+ "type": "string",
237
+ "enum": [
238
+ "ignore",
239
+ "exact",
240
+ "subset"
241
+ ],
242
+ "description": "RFC 0244 §A. How `args` is compared with the call's `inputs`. `ignore` (default): not compared. `exact`: `inputs` JSON-deep-equals `args`. `subset`: every key path in `args` is present in `inputs` with a JSON-deep-equal value. The host compares the call as executed, before any wire redaction."
243
+ }
244
+ }
245
+ },
246
+ "description": "The expected calls, in order where the mode reads order."
247
+ },
248
+ "forbidden": {
249
+ "type": "array",
250
+ "uniqueItems": true,
251
+ "items": {
252
+ "type": "string",
253
+ "minLength": 1
254
+ },
255
+ "description": "MAY. Tools that MUST NOT be called. One call to any of them makes the trajectory unmatched, whatever the mode."
256
+ },
257
+ "includeDelegates": {
258
+ "type": "boolean",
259
+ "default": false,
260
+ "description": "MAY. `true` grades every `agent.tool-called` in the bracket, delegates and fan-out children included, in log order. Default: the target agent's own calls only."
261
+ }
262
+ }
169
263
  }
170
264
  }
171
265
  },
@@ -176,7 +270,7 @@
176
270
  "properties": {
177
271
  "toolResponses": {
178
272
  "type": "array",
179
- "description": "Canned tool results keyed by tool invocation, injected in place of live tool calls.",
273
+ "description": "Canned tool results keyed by tool invocation, injected in place of live tool calls. RFC 0244 §D: each call is served the first unconsumed entry whose `tool` and `args` match; an entry is consumed once, so repeated calls walk a scripted sequence.",
180
274
  "items": {
181
275
  "type": "object",
182
276
  "additionalProperties": false,
@@ -191,6 +285,18 @@
191
285
  },
192
286
  "response": {
193
287
  "description": "The canned result the host returns for that tool. Opaque."
288
+ },
289
+ "args": {
290
+ "description": "RFC 0244 §D. MAY. The arguments this response answers, compared under `argsMatch`. Absent with `argsMatch` absent: the entry answers any call to `tool`."
291
+ },
292
+ "argsMatch": {
293
+ "type": "string",
294
+ "enum": [
295
+ "ignore",
296
+ "exact",
297
+ "subset"
298
+ ],
299
+ "description": "RFC 0244 §A. How `args` is compared with the call's `inputs`. `ignore` (default): not compared. `exact`: `inputs` JSON-deep-equals `args`. `subset`: every key path in `args` is present in `inputs` with a JSON-deep-equal value. The host compares the call as executed, before any wire redaction."
194
300
  }
195
301
  }
196
302
  }
@@ -202,12 +308,19 @@
202
308
  "type": "object",
203
309
  "additionalProperties": true
204
310
  }
311
+ },
312
+ "onMiss": {
313
+ "type": "string",
314
+ "enum": [
315
+ "fail",
316
+ "live"
317
+ ],
318
+ "description": "RFC 0244 §D. MAY. `fail`: a call matching no unconsumed entry MUST NOT reach a live tool or any external side effect; the host returns a tool error to the agent, counts it in the task's `fixtureMissCount`, and reports the trial `passed: false`. `live`: a miss MAY call the live tool, and the task's `fixtureMissCount` MUST then be > 0. Absent: the RFC 0081 behaviour is unchanged (a host SHOULD treat it as `fail`). Requires `agents.evalSuite.fixtureMatching`, as does `toolResponses[].args`."
205
319
  }
206
320
  }
207
321
  }
208
322
  }
209
323
  }
210
324
  }
211
- },
212
- "x-openwop-seeded-from": "v1"
325
+ }
213
326
  }
package/schemas/v2/capabilities.schema.json CHANGED
@@ -3277,7 +3277,7 @@
3277
3277
  },
3278
3278
  "evalSuite": {
3279
3279
  "type": "object",
3280
- "description": "RFC 0081. The host runs portable `agent-eval-suite.schema.json` suites as eval runs (`mode: eval` on `createRun`, runs.md) and terminates each with an `eval-summary.schema.json` scorecard as the run's output. Hosts that omit this block reject `mode: eval` (runs.md). A host advertising this MUST execute each task as a child agent invocation, reusing the `agent.invocation-*` bracket and the `agent.*` and `provider.usage` events, score it, and MUST emit exactly one `eval.started`, one `eval.scored` per task after that task's terminal `agent.decided`, and one `eval.completed` before `run.completed`. When a task carries `fixtures`, the host MUST inject `fixtures.toolResponses` in place of live tool calls and seed `fixtures.memorySeed` into the agent's read snapshot before the invocation; `live-shadow` alone ignores fixtures. Seeded memory is tenant-scoped and redacted like any memory write. The `eval.*` events and the summary are content-free (SECURITY invariant `eval-summary-no-content-leak`) and are recorded facts: a replay re-reads a score, never re-judges. An eval run, its suite, fixtures and summary are scoped to the caller's owner triple; a suite or summary MUST NOT be readable across tenants. A human MAY override an auto-score with an annotation on its `eval.scored` event. Composes per-task cost (`provider.usage`) and the regression baseline diff.",
3280
+ "description": "RFC 0081. The host runs portable `agent-eval-suite.schema.json` suites as eval runs (`mode: eval` on `createRun`, runs.md) and terminates each with an `eval-summary.schema.json` scorecard as the run's output. Hosts that omit this block reject `mode: eval` (runs.md). A host advertising this MUST execute each task as a child agent invocation, reusing the `agent.invocation-*` bracket and the `agent.*` and `provider.usage` events, score it, and MUST emit exactly one `eval.started`, one `eval.scored` per task after that task's terminal `agent.decided`, and one `eval.completed` before `run.completed`. When a task carries `fixtures`, the host MUST inject `fixtures.toolResponses` in place of live tool calls and seed `fixtures.memorySeed` into the agent's read snapshot before the invocation; `live-shadow` alone ignores fixtures. Seeded memory is tenant-scoped and redacted like any memory write. The `eval.*` events and the summary are content-free (SECURITY invariant `eval-summary-no-content-leak`) and are recorded facts: a replay re-reads a score, never re-judges. An eval run, its suite, fixtures and summary are scoped to the caller's owner triple; a suite or summary MUST NOT be readable across tenants. A human MAY override an auto-score with an annotation on its `eval.scored` event. Composes per-task cost (`provider.usage`) and the regression baseline diff. RFC 0244: a suite that uses `expected.trajectory` or `kind: \"trajectory\"`, `trials.count` > 1 (or above `trials.maxTrials`), `fixtures.onMiss` or `toolResponses[].args` on a host that does not advertise the matching member (`trajectory`, `trials`, `fixtureMatching`) MUST be refused at run-create with `400 validation_error` naming the field; the host MUST NOT grade it as if the field were absent. Each trial is a recorded fact; per-task efficiency counts (`toolCallCount`, `modelCallCount`, tokens) are reported, never gated.",
3281
3281
  "additionalProperties": false,
3282
3282
  "properties": {
3283
3283
  "modes": {
@@ -3304,6 +3304,33 @@
3304
3304
  "type": "number",
3305
3305
  "minimum": 0,
3306
3306
  "description": "MAY. Host ceiling on total eval-run cost; composes RFC 0084 budget enforcement when advertised."
3307
+ },
3308
+ "trajectory": {
3309
+ "type": "boolean",
3310
+ "description": "RFC 0244 §A. `true`: the host grades `expected.trajectory` and `kind: \"trajectory\"` with the four match modes, `forbidden`, `argsMatch` and `includeDelegates`, and reports the per-task `trajectory` counts."
3311
+ },
3312
+ "trials": {
3313
+ "type": "object",
3314
+ "additionalProperties": false,
3315
+ "required": [
3316
+ "maxTrials"
3317
+ ],
3318
+ "description": "RFC 0244 §B. Presence: the host runs `trials.count` > 1, emits `eval.scored` once per task per trial with `trialIndex` and `eval.started.trialCount`, and computes the per-task `trials` block and `trialStats` by the normative formulas. A host value agrees with a formula when |host − formula| ≤ 1e-9 · max(1, |formula|); hosts MUST agree to within that tolerance, and need not reproduce any particular float evaluation order.",
3319
+ "properties": {
3320
+ "maxTrials": {
3321
+ "type": "integer",
3322
+ "minimum": 2,
3323
+ "description": "The largest `trials.count` the host runs; a suite above it is refused at run-create with `400 validation_error`."
3324
+ }
3325
+ }
3326
+ },
3327
+ "fixtureMatching": {
3328
+ "type": "boolean",
3329
+ "description": "RFC 0244 §D. `true`: the host honours `toolResponses[].args` / `argsMatch` (first unconsumed match) and `fixtures.onMiss`, and reports `fixtureMissCount`."
3330
+ },
3331
+ "regressionVerdict": {
3332
+ "type": "boolean",
3333
+ "description": "RFC 0244 §F. `true`: in `regression` mode the host reports `regression.verdict`, `method`, `alpha` and `pairedTaskCount`, pairing tasks by `taskId`."
3307
3334
  }
3308
3335
  }
3309
3336
  },
package/schemas/v2/eval-summary.schema.json CHANGED
@@ -131,6 +131,116 @@
131
131
  }
132
132
  }
133
133
  }
134
+ },
135
+ "trajectory": {
136
+ "type": "object",
137
+ "additionalProperties": false,
138
+ "required": [
139
+ "mode",
140
+ "matched",
141
+ "expectedCount",
142
+ "observedCount",
143
+ "matchedCount",
144
+ "forbiddenCount"
145
+ ],
146
+ "description": "RFC 0244 \u00a7A. The trajectory grade of the task's last trial, as counts only: never tool names, arguments or results (`eval-summary-no-content-leak`). `expectedCount` is the number of `calls`, `observedCount` the number of graded `agent.tool-called` events, `forbiddenCount` the observed calls to a `forbidden` tool. `matchedCount` is mode-independent: the size of a maximum one-to-one matching between expected and observed calls under each call's `argsMatch`, ignoring order. Precision is `matchedCount/observedCount`, recall `matchedCount/expectedCount`.",
147
+ "properties": {
148
+ "mode": {
149
+ "type": "string",
150
+ "enum": [
151
+ "exact",
152
+ "in-order",
153
+ "any-order",
154
+ "subset"
155
+ ]
156
+ },
157
+ "matched": {
158
+ "type": "boolean"
159
+ },
160
+ "expectedCount": {
161
+ "type": "integer",
162
+ "minimum": 0
163
+ },
164
+ "observedCount": {
165
+ "type": "integer",
166
+ "minimum": 0
167
+ },
168
+ "matchedCount": {
169
+ "type": "integer",
170
+ "minimum": 0
171
+ },
172
+ "forbiddenCount": {
173
+ "type": "integer",
174
+ "minimum": 0
175
+ }
176
+ }
177
+ },
178
+ "trials": {
179
+ "type": "object",
180
+ "additionalProperties": false,
181
+ "required": [
182
+ "n",
183
+ "passedCount",
184
+ "meanScore",
185
+ "passAtK",
186
+ "passHatK"
187
+ ],
188
+ "description": "RFC 0244 \u00a7B. With n trials of which c passed, and the suite's k: `passAtK` = 1 - C(n-c,k)/C(n,k), `passHatK` = C(c,k)/C(n,k) (0 when c < k), `meanScore` the mean trial score. The task's `score` is `meanScore`. A host value agrees with a formula when |host \u2212 formula| \u2264 1e-9 \u00b7 max(1, |formula|); hosts MUST agree to within that tolerance, and need not reproduce any particular float evaluation order.",
189
+ "properties": {
190
+ "n": {
191
+ "type": "integer",
192
+ "minimum": 1
193
+ },
194
+ "passedCount": {
195
+ "type": "integer",
196
+ "minimum": 0
197
+ },
198
+ "meanScore": {
199
+ "type": "number",
200
+ "minimum": 0,
201
+ "maximum": 1
202
+ },
203
+ "passAtK": {
204
+ "type": "number",
205
+ "minimum": 0,
206
+ "maximum": 1
207
+ },
208
+ "passHatK": {
209
+ "type": "number",
210
+ "minimum": 0,
211
+ "maximum": 1
212
+ }
213
+ }
214
+ },
215
+ "efficiency": {
216
+ "type": "object",
217
+ "additionalProperties": false,
218
+ "description": "RFC 0244 \u00a7C. Reported, never gated: a host MUST NOT fail a task or a suite on these. Means over the task's trials, counted inside its invocation brackets; judge calls excluded.",
219
+ "properties": {
220
+ "toolCallCount": {
221
+ "type": "number",
222
+ "minimum": 0,
223
+ "description": "`agent.tool-called` events."
224
+ },
225
+ "modelCallCount": {
226
+ "type": "number",
227
+ "minimum": 0,
228
+ "description": "`provider.usage` events."
229
+ },
230
+ "inputTokens": {
231
+ "type": "number",
232
+ "minimum": 0
233
+ },
234
+ "outputTokens": {
235
+ "type": "number",
236
+ "minimum": 0
237
+ }
238
+ }
239
+ },
240
+ "fixtureMissCount": {
241
+ "type": "integer",
242
+ "minimum": 0,
243
+ "description": "RFC 0244 \u00a7D. Tool calls that matched no unconsumed fixture entry, summed over the task's trials."
134
244
  }
135
245
  }
136
246
  }
@@ -165,6 +275,62 @@
165
275
  "diffRef": {
166
276
  "type": "string",
167
277
  "description": "MAY. A pointer to `GET /runs/{runId}:diff?against={baselineRunId}` (RFC 0054) for the structural delta."
278
+ },
279
+ "verdict": {
280
+ "type": "string",
281
+ "enum": [
282
+ "improved",
283
+ "regressed",
284
+ "unchanged",
285
+ "inconclusive"
286
+ ],
287
+ "description": "RFC 0244 \u00a7F. MAY. Computed over tasks present in both runs, paired by `taskId`. A host MUST report `inconclusive` rather than `improved` or `regressed` when its method cannot distinguish the runs at `alpha`. Requires `agents.evalSuite.regressionVerdict`."
288
+ },
289
+ "method": {
290
+ "type": "string",
291
+ "pattern": "^[a-z0-9][a-z0-9-]{0,63}$",
292
+ "description": "RFC 0244 \u00a7F. The statistical method behind `verdict`. RECOMMENDED: `paired-bootstrap`, `mcnemar`, `sign-test`, `paired-t`."
293
+ },
294
+ "alpha": {
295
+ "type": "number",
296
+ "exclusiveMinimum": 0,
297
+ "maximum": 1,
298
+ "description": "RFC 0244 \u00a7F. The significance level the method used."
299
+ },
300
+ "pairedTaskCount": {
301
+ "type": "integer",
302
+ "minimum": 0,
303
+ "description": "RFC 0244 \u00a7F. Tasks present in both runs."
304
+ }
305
+ }
306
+ },
307
+ "trialStats": {
308
+ "type": "object",
309
+ "additionalProperties": false,
310
+ "required": [
311
+ "k",
312
+ "passAtK",
313
+ "passHatK"
314
+ ],
315
+ "description": "RFC 0244 \u00a7B. Present when the suite ran more than one trial per task. `passAtK` / `passHatK` are the means of the per-task values; `scoreStderr` is the sample standard deviation of the per-task `meanScore` values divided by the square root of `taskCount` (clustered by task), omitted when `taskCount` < 2. A host value agrees with a formula when |host \u2212 formula| \u2264 1e-9 \u00b7 max(1, |formula|); hosts MUST agree to within that tolerance, and need not reproduce any particular float evaluation order.",
316
+ "properties": {
317
+ "k": {
318
+ "type": "integer",
319
+ "minimum": 1
320
+ },
321
+ "passAtK": {
322
+ "type": "number",
323
+ "minimum": 0,
324
+ "maximum": 1
325
+ },
326
+ "passHatK": {
327
+ "type": "number",
328
+ "minimum": 0,
329
+ "maximum": 1
330
+ },
331
+ "scoreStderr": {
332
+ "type": "number",
333
+ "minimum": 0
168
334
  }
169
335
  }
170
336
  }
package/schemas/v2/run-event-payloads.schema.json CHANGED
@@ -3429,6 +3429,11 @@
3429
3429
  "baselineRunId": {
3430
3430
  "$ref": "ids.schema.json#/$defs/runId",
3431
3431
  "description": "MAY \u2014 for `regression` mode, the prior eval run this run is scored against."
3432
+ },
3433
+ "trialCount": {
3434
+ "type": "integer",
3435
+ "minimum": 2,
3436
+ "description": "RFC 0244 \u00a7B. Present when the suite runs more than one trial per task: each task then emits `trialCount` `eval.scored` events."
3432
3437
  }
3433
3438
  },
3434
3439
  "patternProperties": { "^(openwop-|x-|vendor\\.)": { "description": "Vendor / host extension escape hatch (RFC 0185 \u00a7B). This def was `additionalProperties: true` in v1 and closed in v2 with no hatch and no migration row, so every extra key a host LEGITIMATELY recorded under v1 became invalid at the cut with nowhere to put it. Two production hosts independently lost data to that: one drops 32 distinct keys at the major-2 read (the conversation content among them), the other projects era-3 rows ON WRITE and lost a variable's value AT REST. A property matching this pattern is carried opaquely \u2014 a host MUST NOT interpret it, and MUST NOT silently drop it (RFC 0185 \u00a7C)." } },
@@ -3436,7 +3441,7 @@
3436
3441
  },
3437
3442
  "evalScored": {
3438
3443
  "type": "object",
3439
- "description": "RFC 0081 \u00a7C. Emitted ONCE PER TASK, after that task's terminal `agent.decided`, so a streaming consumer sees results land incrementally. Content-free: the task id + scalars + counts ONLY \u2014 NEVER the task output, rubric prose, or model completion (SECURITY invariant `eval-summary-no-content-leak`). A recorded-fact event.",
3444
+ "description": "RFC 0081 \u00a7C. Emitted ONCE PER TASK (once per task per trial under RFC 0244 \u00a7B), after that task's terminal `agent.decided`, so a streaming consumer sees results land incrementally. Content-free: the task id + scalars + counts ONLY \u2014 NEVER the task output, rubric prose, or model completion (SECURITY invariant `eval-summary-no-content-leak`). A recorded-fact event.",
3440
3445
  "required": [
3441
3446
  "taskId",
3442
3447
  "score",
@@ -3476,6 +3481,25 @@
3476
3481
  "type": "integer",
3477
3482
  "minimum": 0,
3478
3483
  "description": "MAY \u2014 count of safety findings for the task (the redaction-safe descriptors live on `EvalSummary`; the event carries the count only)."
3484
+ },
3485
+ "trialIndex": {
3486
+ "type": "integer",
3487
+ "minimum": 0,
3488
+ "description": "RFC 0244 \u00a7B. The 0-based trial this score is for; contiguous and unique per `(taskId, trialIndex)`. MAY be omitted when the suite runs one trial."
3489
+ },
3490
+ "scoreLabel": {
3491
+ "type": "string",
3492
+ "pattern": "^[a-z0-9][a-z0-9_-]{0,63}$",
3493
+ "description": "RFC 0244 \u00a7E. MAY. A low-cardinality label (e.g. `pass`, `fail`, `partial`); maps onto OpenTelemetry `gen_ai.evaluation.score.label`. Pattern-bound so it cannot carry prose."
3494
+ },
3495
+ "evaluatorType": {
3496
+ "type": "string",
3497
+ "enum": [
3498
+ "deterministic",
3499
+ "llm_judge",
3500
+ "human"
3501
+ ],
3502
+ "description": "RFC 0244 \u00a7E. MAY. Who produced the score."
3479
3503
  }
3480
3504
  },
3481
3505
  "patternProperties": { "^(openwop-|x-|vendor\\.)": { "description": "Vendor / host extension escape hatch (RFC 0185 \u00a7B). This def was `additionalProperties: true` in v1 and closed in v2 with no hatch and no migration row, so every extra key a host LEGITIMATELY recorded under v1 became invalid at the cut with nowhere to put it. Two production hosts independently lost data to that: one drops 32 distinct keys at the major-2 read (the conversation content among them), the other projects era-3 rows ON WRITE and lost a variable's value AT REST. A property matching this pattern is carried opaquely \u2014 a host MUST NOT interpret it, and MUST NOT silently drop it (RFC 0185 \u00a7C)." } },
package/spec/v1/core-standard-manifest.json CHANGED
@@ -4,7 +4,7 @@
4
4
  "provenance": {
5
5
  "suite": {
6
6
  "package": "@openwop/openwop-conformance",
7
- "version": "2.46.7"
7
+ "version": "2.46.8"
8
8
  },
9
9
  "note": "Derived from the corpus at generation time. Regenerate with --write; verify with --check."
10
10
  },
@@ -426,5 +426,5 @@
426
426
  "$id": "https://openwop.dev/spec/v1/workspace-file.schema.json"
427
427
  }
428
428
  ],
429
- "digest": "4173d63e7ac045c20e9e57fc285ac341643c0a78b21b16870a77b137bf0d8ce9"
429
+ "digest": "75d418bd54138b86712944bfe1c1a6a1081689e4a81eb66a34778e1dbcb90427"
430
430
  }
package/spec/v1/gaps.json CHANGED
@@ -14530,6 +14530,193 @@
14530
14530
  }
14531
14531
  ]
14532
14532
  },
14533
+ {
14534
+ "id": "openwop.gap.0244.1",
14535
+ "rfc": "0244",
14536
+ "local": "G1",
14537
+ "section": "§Conformance",
14538
+ "surface": "Whether an eval task's `agentId` can target `core.conformance.mock-agent` (a workflow node type carrying an `agent` block), or whether the fixture needs a conformance manifest agent whose dispatch replays `mockToolCalls`.",
14539
+ "witness": "unclassified",
14540
+ "requirementId": null,
14541
+ "disposition": "closed",
14542
+ "target": null,
14543
+ "sources": [
14544
+ {
14545
+ "file": "RFCS/registers/0244-eval-trajectories-trials-and-recorded-fixtures.gaps.md",
14546
+ "token": "G1"
14547
+ }
14548
+ ]
14549
+ },
14550
+ {
14551
+ "id": "openwop.gap.0244.2",
14552
+ "rfc": "0244",
14553
+ "local": "G2",
14554
+ "section": "§D, §Falsifiability",
14555
+ "surface": "There is no conformance *tool* with an observable effect counter. `mock-agent` scripts tool results and invokes nothing, and `core.conformance.side-effect` is a node. So \"a miss never reaches the live tool\" and \"first-unconsumed selection\" cannot be caused from outside.",
14556
+ "witness": "unclassified",
14557
+ "requirementId": null,
14558
+ "disposition": "transferred",
14559
+ "target": "TODO.md",
14560
+ "sources": [
14561
+ {
14562
+ "file": "RFCS/registers/0244-eval-trajectories-trials-and-recorded-fixtures.gaps.md",
14563
+ "token": "G2"
14564
+ }
14565
+ ]
14566
+ },
14567
+ {
14568
+ "id": "openwop.gap.0244.3",
14569
+ "rfc": "0244",
14570
+ "local": "G3",
14571
+ "section": "§B",
14572
+ "surface": "Trial memory isolation has no causable probe. It needs a scripted agent that writes memory in trial 0 and reads it in trial 1.",
14573
+ "witness": "unclassified",
14574
+ "requirementId": null,
14575
+ "disposition": "transferred",
14576
+ "target": "TODO.md",
14577
+ "sources": [
14578
+ {
14579
+ "file": "RFCS/registers/0244-eval-trajectories-trials-and-recorded-fixtures.gaps.md",
14580
+ "token": "G3"
14581
+ }
14582
+ ]
14583
+ },
14584
+ {
14585
+ "id": "openwop.gap.0244.4",
14586
+ "rfc": "0244",
14587
+ "local": "G4",
14588
+ "section": "§E",
14589
+ "surface": "`evaluatorType` copies OpenTelemetry PR #359, which is unmerged as of 2026-10-12.",
14590
+ "witness": "unclassified",
14591
+ "requirementId": null,
14592
+ "disposition": "closed",
14593
+ "target": null,
14594
+ "sources": [
14595
+ {
14596
+ "file": "RFCS/registers/0244-eval-trajectories-trials-and-recorded-fixtures.gaps.md",
14597
+ "token": "G4"
14598
+ }
14599
+ ]
14600
+ },
14601
+ {
14602
+ "id": "openwop.gap.0244.5",
14603
+ "rfc": "0244",
14604
+ "local": "G5",
14605
+ "section": "Implementation notes, UQ 7",
14606
+ "surface": "There is no portable way to say \"fixtures = the recorded tool boundary of run X\". It is the highest-value practice found, and it cannot ride the suite, because suites carry no host ids.",
14607
+ "witness": "unclassified",
14608
+ "requirementId": null,
14609
+ "disposition": "transferred",
14610
+ "target": "TODO.md",
14611
+ "sources": [
14612
+ {
14613
+ "file": "RFCS/registers/0244-eval-trajectories-trials-and-recorded-fixtures.gaps.md",
14614
+ "token": "G5"
14615
+ }
14616
+ ]
14617
+ },
14618
+ {
14619
+ "id": "openwop.gap.0244.6",
14620
+ "rfc": "0244",
14621
+ "local": "G6",
14622
+ "section": "§A",
14623
+ "surface": "Whether `agent.tool-called.inputs` is ever redacted on the v2 wire, which would make `argsMatch` unrecomputable from outside (UQ 6).",
14624
+ "witness": "unclassified",
14625
+ "requirementId": null,
14626
+ "disposition": "closed",
14627
+ "target": null,
14628
+ "sources": [
14629
+ {
14630
+ "file": "RFCS/registers/0244-eval-trajectories-trials-and-recorded-fixtures.gaps.md",
14631
+ "token": "G6"
14632
+ }
14633
+ ]
14634
+ },
14635
+ {
14636
+ "id": "openwop.gap.0244.7",
14637
+ "rfc": "0244",
14638
+ "local": "G7",
14639
+ "section": "§F",
14640
+ "surface": "Method vocabulary: open string vs closed enum (UQ 5).",
14641
+ "witness": "unclassified",
14642
+ "requirementId": null,
14643
+ "disposition": "closed",
14644
+ "target": null,
14645
+ "sources": [
14646
+ {
14647
+ "file": "RFCS/registers/0244-eval-trajectories-trials-and-recorded-fixtures.gaps.md",
14648
+ "token": "G7"
14649
+ }
14650
+ ]
14651
+ },
14652
+ {
14653
+ "id": "openwop.gap.0244.8",
14654
+ "rfc": "0244",
14655
+ "local": "G8",
14656
+ "section": "§A",
14657
+ "surface": "Sub-agent calls inside a task's bracket: included in the trajectory or not (UQ 2). This interacts with RFC 0243 fan-out.",
14658
+ "witness": "unclassified",
14659
+ "requirementId": null,
14660
+ "disposition": "closed",
14661
+ "target": null,
14662
+ "sources": [
14663
+ {
14664
+ "file": "RFCS/registers/0244-eval-trajectories-trials-and-recorded-fixtures.gaps.md",
14665
+ "token": "G8"
14666
+ }
14667
+ ]
14668
+ },
14669
+ {
14670
+ "id": "openwop.gap.0244.9",
14671
+ "rfc": "0244",
14672
+ "local": "G9",
14673
+ "section": "Implementation notes",
14674
+ "surface": "No host advertises even RFC 0081 `agents.evalSuite` in the v2 production cut. openwop-app answers `422`. The RFC cannot reach `Accepted` without one.",
14675
+ "witness": "unclassified",
14676
+ "requirementId": null,
14677
+ "disposition": "externally-gated",
14678
+ "target": "host-advertises-evalSuite-trajectory-and-trials",
14679
+ "sources": [
14680
+ {
14681
+ "file": "RFCS/registers/0244-eval-trajectories-trials-and-recorded-fixtures.gaps.md",
14682
+ "token": "G9"
14683
+ }
14684
+ ]
14685
+ },
14686
+ {
14687
+ "id": "openwop.gap.0244.10",
14688
+ "rfc": "0244",
14689
+ "local": "G10",
14690
+ "section": "§Conformance",
14691
+ "surface": "The `testing.mockProviders` gate. The behavioural legs need test keys, so a production-only bundle reads them as `inapplicable`.",
14692
+ "witness": "unclassified",
14693
+ "requirementId": null,
14694
+ "disposition": "externally-gated",
14695
+ "target": "host-seeds-eval-suite-fixtures",
14696
+ "sources": [
14697
+ {
14698
+ "file": "RFCS/registers/0244-eval-trajectories-trials-and-recorded-fixtures.gaps.md",
14699
+ "token": "G10"
14700
+ }
14701
+ ]
14702
+ },
14703
+ {
14704
+ "id": "openwop.gap.0244.11",
14705
+ "rfc": "0244",
14706
+ "local": "G11",
14707
+ "section": "Implementation notes, G9",
14708
+ "surface": "MyndHyve de-advertised `agents.evalSuite` on 2026-10-02 (myndhyve#586). Its RFC 0081 witness was a sample seam with no real agent run, so **no host serves a real `mode: \"eval\"` run today**. RFC 0081's `Accepted` evidence is weaker than that RFC's header says. That is flagged in `TODO.md`, not fixed here.",
14709
+ "witness": "unclassified",
14710
+ "requirementId": null,
14711
+ "disposition": "externally-gated",
14712
+ "target": "a-host-serves-a-real-mode-eval-run",
14713
+ "sources": [
14714
+ {
14715
+ "file": "RFCS/registers/0244-eval-trajectories-trials-and-recorded-fixtures.gaps.md",
14716
+ "token": "G11"
14717
+ }
14718
+ ]
14719
+ },
14533
14720
  {
14534
14721
  "id": "openwop.gap.spec.a2a-integration.g1",
14535
14722
  "rfc": "0152",
package/spec/v2/facets/agents.schema.json CHANGED
@@ -133,7 +133,7 @@
133
133
  },
134
134
  "evalSuite": {
135
135
  "type": "object",
136
- "description": "RFC 0081. The host runs portable `agent-eval-suite.schema.json` suites as eval runs (`mode: eval` on `createRun`, runs.md) and terminates each with an `eval-summary.schema.json` scorecard as the run's output. Hosts that omit this block reject `mode: eval` (runs.md). A host advertising this MUST execute each task as a child agent invocation, reusing the `agent.invocation-*` bracket and the `agent.*` and `provider.usage` events, score it, and MUST emit exactly one `eval.started`, one `eval.scored` per task after that task's terminal `agent.decided`, and one `eval.completed` before `run.completed`. When a task carries `fixtures`, the host MUST inject `fixtures.toolResponses` in place of live tool calls and seed `fixtures.memorySeed` into the agent's read snapshot before the invocation; `live-shadow` alone ignores fixtures. Seeded memory is tenant-scoped and redacted like any memory write. The `eval.*` events and the summary are content-free (SECURITY invariant `eval-summary-no-content-leak`) and are recorded facts: a replay re-reads a score, never re-judges. An eval run, its suite, fixtures and summary are scoped to the caller's owner triple; a suite or summary MUST NOT be readable across tenants. A human MAY override an auto-score with an annotation on its `eval.scored` event. Composes per-task cost (`provider.usage`) and the regression baseline diff.",
136
+ "description": "RFC 0081. The host runs portable `agent-eval-suite.schema.json` suites as eval runs (`mode: eval` on `createRun`, runs.md) and terminates each with an `eval-summary.schema.json` scorecard as the run's output. Hosts that omit this block reject `mode: eval` (runs.md). A host advertising this MUST execute each task as a child agent invocation, reusing the `agent.invocation-*` bracket and the `agent.*` and `provider.usage` events, score it, and MUST emit exactly one `eval.started`, one `eval.scored` per task after that task's terminal `agent.decided`, and one `eval.completed` before `run.completed`. When a task carries `fixtures`, the host MUST inject `fixtures.toolResponses` in place of live tool calls and seed `fixtures.memorySeed` into the agent's read snapshot before the invocation; `live-shadow` alone ignores fixtures. Seeded memory is tenant-scoped and redacted like any memory write. The `eval.*` events and the summary are content-free (SECURITY invariant `eval-summary-no-content-leak`) and are recorded facts: a replay re-reads a score, never re-judges. An eval run, its suite, fixtures and summary are scoped to the caller's owner triple; a suite or summary MUST NOT be readable across tenants. A human MAY override an auto-score with an annotation on its `eval.scored` event. Composes per-task cost (`provider.usage`) and the regression baseline diff. RFC 0244: a suite that uses `expected.trajectory` or `kind: \"trajectory\"`, `trials.count` > 1 (or above `trials.maxTrials`), `fixtures.onMiss` or `toolResponses[].args` on a host that does not advertise the matching member (`trajectory`, `trials`, `fixtureMatching`) MUST be refused at run-create with `400 validation_error` naming the field; the host MUST NOT grade it as if the field were absent. Each trial is a recorded fact; per-task efficiency counts (`toolCallCount`, `modelCallCount`, tokens) are reported, never gated.",
137
137
  "additionalProperties": false,
138
138
  "properties": {
139
139
  "modes": {
@@ -160,6 +160,33 @@
160
160
  "type": "number",
161
161
  "minimum": 0,
162
162
  "description": "MAY. Host ceiling on total eval-run cost; composes RFC 0084 budget enforcement when advertised."
163
+ },
164
+ "trajectory": {
165
+ "type": "boolean",
166
+ "description": "RFC 0244 §A. `true`: the host grades `expected.trajectory` and `kind: \"trajectory\"` with the four match modes, `forbidden`, `argsMatch` and `includeDelegates`, and reports the per-task `trajectory` counts."
167
+ },
168
+ "trials": {
169
+ "type": "object",
170
+ "additionalProperties": false,
171
+ "required": [
172
+ "maxTrials"
173
+ ],
174
+ "description": "RFC 0244 §B. Presence: the host runs `trials.count` > 1, emits `eval.scored` once per task per trial with `trialIndex` and `eval.started.trialCount`, and computes the per-task `trials` block and `trialStats` by the normative formulas. A host value agrees with a formula when |host − formula| ≤ 1e-9 · max(1, |formula|); hosts MUST agree to within that tolerance, and need not reproduce any particular float evaluation order.",
175
+ "properties": {
176
+ "maxTrials": {
177
+ "type": "integer",
178
+ "minimum": 2,
179
+ "description": "The largest `trials.count` the host runs; a suite above it is refused at run-create with `400 validation_error`."
180
+ }
181
+ }
182
+ },
183
+ "fixtureMatching": {
184
+ "type": "boolean",
185
+ "description": "RFC 0244 §D. `true`: the host honours `toolResponses[].args` / `argsMatch` (first unconsumed match) and `fixtures.onMiss`, and reports `fixtureMissCount`."
186
+ },
187
+ "regressionVerdict": {
188
+ "type": "boolean",
189
+ "description": "RFC 0244 §F. `true`: in `regression` mode the host reports `regression.verdict`, `method`, `alpha` and `pairedTaskCount`, pairing tasks by `taskId`."
163
190
  }
164
191
  }
165
192
  },
package/spec/v2/release.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "$comment": "RFC 0172 §D.1 — the one release identity the v2 artifacts derive from. `version` is the next corpus tag `v<version>` (the publish workflow's coordinated-release tag pattern `v*`; RFC 0172's `openwop/v2.<minor>.<patch>` spelling is amended to this at its flip). api/v2/*.yaml info.version, the suite's 2.x version and @openwop/spec-artifacts read it. Bumped by the release PR that cuts the tag, never by hand elsewhere.",
3
- "version": "2.46.7",
4
- "corpusTag": "v2.46.7",
3
+ "version": "2.46.8",
4
+ "corpusTag": "v2.46.8",
5
5
  "updated": "2026-10-14"
6
6
  }
package/spec/v2/surface-baseline.json CHANGED
@@ -4,7 +4,7 @@
4
4
  "schemas/v2/**/*.json",
5
5
  "spec/v2/path-manifest.json"
6
6
  ],
7
- "release": "2.46.7",
7
+ "release": "2.46.8",
8
8
  "kinds": {
9
9
  "union": [
10
10
  "channel",
@@ -18,7 +18,7 @@
18
18
  "type"
19
19
  ]
20
20
  },
21
- "count": 8781,
21
+ "count": 8927,
22
22
  "surfaces": [
23
23
  "schemas/v2/a2a-task-state.schema.json#/contextId|type|string",
24
24
  "schemas/v2/a2a-task-state.schema.json#/interruptKind|enum-member|\"approval\"",
@@ -156,6 +156,7 @@
156
156
  "schemas/v2/agent-eval-suite.schema.json#/suiteId|type|string",
157
157
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/kind|enum-member|\"golden\"",
158
158
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/kind|enum-member|\"rubric\"",
159
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/kind|enum-member|\"trajectory\"",
159
160
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/kind|type|string",
160
161
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/match/strategy|enum-member|\"contains\"",
161
162
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/match/strategy|enum-member|\"exact\"",
@@ -176,16 +177,54 @@
176
177
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/rubric/[]|required|weight",
177
178
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/rubric/[]|type|object",
178
179
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/rubric|type|array",
180
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls/[]/argsMatch|enum-member|\"exact\"",
181
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls/[]/argsMatch|enum-member|\"ignore\"",
182
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls/[]/argsMatch|enum-member|\"subset\"",
183
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls/[]/argsMatch|type|string",
184
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls/[]/tool|type|string",
185
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls/[]|closed|true",
186
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls/[]|property|args",
187
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls/[]|property|argsMatch",
188
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls/[]|property|tool",
189
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls/[]|required|tool",
190
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls/[]|type|object",
191
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/calls|type|array",
192
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/forbidden/[]|type|string",
193
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/forbidden|type|array",
194
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/includeDelegates|type|boolean",
195
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/match|enum-member|\"any-order\"",
196
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/match|enum-member|\"exact\"",
197
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/match|enum-member|\"in-order\"",
198
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/match|enum-member|\"subset\"",
199
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory/match|type|string",
200
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory|closed|true",
201
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory|property|calls",
202
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory|property|forbidden",
203
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory|property|includeDelegates",
204
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory|property|match",
205
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory|required|calls",
206
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory|required|match",
207
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected/trajectory|type|object",
179
208
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected|closed|true",
180
209
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected|property|kind",
181
210
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected|property|match",
182
211
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected|property|rubric",
212
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected|property|trajectory",
183
213
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected|required|kind",
184
214
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/expected|type|object",
185
215
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/memorySeed/[]|type|object",
186
216
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/memorySeed|type|array",
217
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/onMiss|enum-member|\"fail\"",
218
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/onMiss|enum-member|\"live\"",
219
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/onMiss|type|string",
220
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses/[]/argsMatch|enum-member|\"exact\"",
221
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses/[]/argsMatch|enum-member|\"ignore\"",
222
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses/[]/argsMatch|enum-member|\"subset\"",
223
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses/[]/argsMatch|type|string",
187
224
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses/[]/tool|type|string",
188
225
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses/[]|closed|true",
226
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses/[]|property|args",
227
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses/[]|property|argsMatch",
189
228
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses/[]|property|response",
190
229
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses/[]|property|tool",
191
230
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses/[]|required|tool",
@@ -193,6 +232,7 @@
193
232
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures/toolResponses|type|array",
194
233
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures|closed|true",
195
234
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures|property|memorySeed",
235
+ "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures|property|onMiss",
196
236
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures|property|toolResponses",
197
237
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/fixtures|type|object",
198
238
  "schemas/v2/agent-eval-suite.schema.json#/tasks/[]/taskId|type|string",
@@ -214,6 +254,17 @@
214
254
  "schemas/v2/agent-eval-suite.schema.json#/thresholds|property|maxP95LatencyMs",
215
255
  "schemas/v2/agent-eval-suite.schema.json#/thresholds|property|passScore",
216
256
  "schemas/v2/agent-eval-suite.schema.json#/thresholds|type|object",
257
+ "schemas/v2/agent-eval-suite.schema.json#/trials/count|type|integer",
258
+ "schemas/v2/agent-eval-suite.schema.json#/trials/k|type|integer",
259
+ "schemas/v2/agent-eval-suite.schema.json#/trials/passRule|enum-member|\"all\"",
260
+ "schemas/v2/agent-eval-suite.schema.json#/trials/passRule|enum-member|\"any\"",
261
+ "schemas/v2/agent-eval-suite.schema.json#/trials/passRule|enum-member|\"mean\"",
262
+ "schemas/v2/agent-eval-suite.schema.json#/trials/passRule|type|string",
263
+ "schemas/v2/agent-eval-suite.schema.json#/trials|closed|true",
264
+ "schemas/v2/agent-eval-suite.schema.json#/trials|property|count",
265
+ "schemas/v2/agent-eval-suite.schema.json#/trials|property|k",
266
+ "schemas/v2/agent-eval-suite.schema.json#/trials|property|passRule",
267
+ "schemas/v2/agent-eval-suite.schema.json#/trials|type|object",
217
268
  "schemas/v2/agent-eval-suite.schema.json#/version|type|string",
218
269
  "schemas/v2/agent-eval-suite.schema.json#|closed|true",
219
270
  "schemas/v2/agent-eval-suite.schema.json#|property|allowedModels",
@@ -222,6 +273,7 @@
222
273
  "schemas/v2/agent-eval-suite.schema.json#|property|targetAgentId",
223
274
  "schemas/v2/agent-eval-suite.schema.json#|property|tasks",
224
275
  "schemas/v2/agent-eval-suite.schema.json#|property|thresholds",
276
+ "schemas/v2/agent-eval-suite.schema.json#|property|trials",
225
277
  "schemas/v2/agent-eval-suite.schema.json#|property|version",
226
278
  "schemas/v2/agent-eval-suite.schema.json#|required|modes",
227
279
  "schemas/v2/agent-eval-suite.schema.json#|required|suiteId",
@@ -929,6 +981,7 @@
929
981
  "schemas/v2/capabilities.schema.json#/agents/deployment|type|object",
930
982
  "schemas/v2/capabilities.schema.json#/agents/dispatchMapping|type|boolean",
931
983
  "schemas/v2/capabilities.schema.json#/agents/dispatch|type|boolean",
984
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite/fixtureMatching|type|boolean",
932
985
  "schemas/v2/capabilities.schema.json#/agents/evalSuite/maxCostUsdPerSuite|type|number",
933
986
  "schemas/v2/capabilities.schema.json#/agents/evalSuite/maxTasksPerSuite|type|integer",
934
987
  "schemas/v2/capabilities.schema.json#/agents/evalSuite/modes/[]|enum-member|\"adversarial\"",
@@ -938,10 +991,21 @@
938
991
  "schemas/v2/capabilities.schema.json#/agents/evalSuite/modes/[]|enum-member|\"rubric\"",
939
992
  "schemas/v2/capabilities.schema.json#/agents/evalSuite/modes/[]|type|string",
940
993
  "schemas/v2/capabilities.schema.json#/agents/evalSuite/modes|type|array",
994
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite/regressionVerdict|type|boolean",
995
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite/trajectory|type|boolean",
996
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite/trials/maxTrials|type|integer",
997
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite/trials|closed|true",
998
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite/trials|property|maxTrials",
999
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite/trials|required|maxTrials",
1000
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite/trials|type|object",
941
1001
  "schemas/v2/capabilities.schema.json#/agents/evalSuite|closed|true",
1002
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite|property|fixtureMatching",
942
1003
  "schemas/v2/capabilities.schema.json#/agents/evalSuite|property|maxCostUsdPerSuite",
943
1004
  "schemas/v2/capabilities.schema.json#/agents/evalSuite|property|maxTasksPerSuite",
944
1005
  "schemas/v2/capabilities.schema.json#/agents/evalSuite|property|modes",
1006
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite|property|regressionVerdict",
1007
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite|property|trajectory",
1008
+ "schemas/v2/capabilities.schema.json#/agents/evalSuite|property|trials",
945
1009
  "schemas/v2/capabilities.schema.json#/agents/evalSuite|type|object",
946
1010
  "schemas/v2/capabilities.schema.json#/agents/fanOut/joinModes/[]|enum-member|\"first\"",
947
1011
  "schemas/v2/capabilities.schema.json#/agents/fanOut/joinModes/[]|enum-member|\"quorum\"",
@@ -5634,12 +5698,24 @@
5634
5698
  "schemas/v2/eval-summary.schema.json#/passedCount|type|integer",
5635
5699
  "schemas/v2/eval-summary.schema.json#/passed|type|boolean",
5636
5700
  "schemas/v2/eval-summary.schema.json#/pinnedNodeIds|type|array",
5701
+ "schemas/v2/eval-summary.schema.json#/regression/alpha|type|number",
5637
5702
  "schemas/v2/eval-summary.schema.json#/regression/diffRef|type|string",
5703
+ "schemas/v2/eval-summary.schema.json#/regression/method|type|string",
5704
+ "schemas/v2/eval-summary.schema.json#/regression/pairedTaskCount|type|integer",
5638
5705
  "schemas/v2/eval-summary.schema.json#/regression/scoreDelta|type|number",
5706
+ "schemas/v2/eval-summary.schema.json#/regression/verdict|enum-member|\"improved\"",
5707
+ "schemas/v2/eval-summary.schema.json#/regression/verdict|enum-member|\"inconclusive\"",
5708
+ "schemas/v2/eval-summary.schema.json#/regression/verdict|enum-member|\"regressed\"",
5709
+ "schemas/v2/eval-summary.schema.json#/regression/verdict|enum-member|\"unchanged\"",
5710
+ "schemas/v2/eval-summary.schema.json#/regression/verdict|type|string",
5639
5711
  "schemas/v2/eval-summary.schema.json#/regression|closed|true",
5712
+ "schemas/v2/eval-summary.schema.json#/regression|property|alpha",
5640
5713
  "schemas/v2/eval-summary.schema.json#/regression|property|baselineRunId",
5641
5714
  "schemas/v2/eval-summary.schema.json#/regression|property|diffRef",
5715
+ "schemas/v2/eval-summary.schema.json#/regression|property|method",
5716
+ "schemas/v2/eval-summary.schema.json#/regression|property|pairedTaskCount",
5642
5717
  "schemas/v2/eval-summary.schema.json#/regression|property|scoreDelta",
5718
+ "schemas/v2/eval-summary.schema.json#/regression|property|verdict",
5643
5719
  "schemas/v2/eval-summary.schema.json#/regression|required|baselineRunId",
5644
5720
  "schemas/v2/eval-summary.schema.json#/regression|required|scoreDelta",
5645
5721
  "schemas/v2/eval-summary.schema.json#/regression|type|object",
@@ -5647,6 +5723,17 @@
5647
5723
  "schemas/v2/eval-summary.schema.json#/suiteVersion|type|string",
5648
5724
  "schemas/v2/eval-summary.schema.json#/taskCount|type|integer",
5649
5725
  "schemas/v2/eval-summary.schema.json#/tasks/[]/costUsd|type|number",
5726
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/efficiency/inputTokens|type|number",
5727
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/efficiency/modelCallCount|type|number",
5728
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/efficiency/outputTokens|type|number",
5729
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/efficiency/toolCallCount|type|number",
5730
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/efficiency|closed|true",
5731
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/efficiency|property|inputTokens",
5732
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/efficiency|property|modelCallCount",
5733
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/efficiency|property|outputTokens",
5734
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/efficiency|property|toolCallCount",
5735
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/efficiency|type|object",
5736
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/fixtureMissCount|type|integer",
5650
5737
  "schemas/v2/eval-summary.schema.json#/tasks/[]/latencyMs|type|integer",
5651
5738
  "schemas/v2/eval-summary.schema.json#/tasks/[]/passed|type|boolean",
5652
5739
  "schemas/v2/eval-summary.schema.json#/tasks/[]/safetyFindings/[]/kind|type|string",
@@ -5665,20 +5752,78 @@
5665
5752
  "schemas/v2/eval-summary.schema.json#/tasks/[]/schemaValid|type|boolean",
5666
5753
  "schemas/v2/eval-summary.schema.json#/tasks/[]/score|type|number",
5667
5754
  "schemas/v2/eval-summary.schema.json#/tasks/[]/taskId|type|string",
5755
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory/expectedCount|type|integer",
5756
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory/forbiddenCount|type|integer",
5757
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory/matchedCount|type|integer",
5758
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory/matched|type|boolean",
5759
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory/mode|enum-member|\"any-order\"",
5760
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory/mode|enum-member|\"exact\"",
5761
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory/mode|enum-member|\"in-order\"",
5762
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory/mode|enum-member|\"subset\"",
5763
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory/mode|type|string",
5764
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory/observedCount|type|integer",
5765
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|closed|true",
5766
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|property|expectedCount",
5767
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|property|forbiddenCount",
5768
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|property|matched",
5769
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|property|matchedCount",
5770
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|property|mode",
5771
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|property|observedCount",
5772
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|required|expectedCount",
5773
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|required|forbiddenCount",
5774
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|required|matched",
5775
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|required|matchedCount",
5776
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|required|mode",
5777
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|required|observedCount",
5778
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trajectory|type|object",
5779
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials/meanScore|type|number",
5780
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials/n|type|integer",
5781
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials/passAtK|type|number",
5782
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials/passHatK|type|number",
5783
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials/passedCount|type|integer",
5784
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|closed|true",
5785
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|property|meanScore",
5786
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|property|n",
5787
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|property|passAtK",
5788
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|property|passHatK",
5789
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|property|passedCount",
5790
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|required|meanScore",
5791
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|required|n",
5792
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|required|passAtK",
5793
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|required|passHatK",
5794
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|required|passedCount",
5795
+ "schemas/v2/eval-summary.schema.json#/tasks/[]/trials|type|object",
5668
5796
  "schemas/v2/eval-summary.schema.json#/tasks/[]|closed|true",
5669
5797
  "schemas/v2/eval-summary.schema.json#/tasks/[]|property|costUsd",
5798
+ "schemas/v2/eval-summary.schema.json#/tasks/[]|property|efficiency",
5799
+ "schemas/v2/eval-summary.schema.json#/tasks/[]|property|fixtureMissCount",
5670
5800
  "schemas/v2/eval-summary.schema.json#/tasks/[]|property|latencyMs",
5671
5801
  "schemas/v2/eval-summary.schema.json#/tasks/[]|property|passed",
5672
5802
  "schemas/v2/eval-summary.schema.json#/tasks/[]|property|safetyFindings",
5673
5803
  "schemas/v2/eval-summary.schema.json#/tasks/[]|property|schemaValid",
5674
5804
  "schemas/v2/eval-summary.schema.json#/tasks/[]|property|score",
5675
5805
  "schemas/v2/eval-summary.schema.json#/tasks/[]|property|taskId",
5806
+ "schemas/v2/eval-summary.schema.json#/tasks/[]|property|trajectory",
5807
+ "schemas/v2/eval-summary.schema.json#/tasks/[]|property|trials",
5676
5808
  "schemas/v2/eval-summary.schema.json#/tasks/[]|required|passed",
5677
5809
  "schemas/v2/eval-summary.schema.json#/tasks/[]|required|score",
5678
5810
  "schemas/v2/eval-summary.schema.json#/tasks/[]|required|taskId",
5679
5811
  "schemas/v2/eval-summary.schema.json#/tasks/[]|type|object",
5680
5812
  "schemas/v2/eval-summary.schema.json#/tasks|type|array",
5681
5813
  "schemas/v2/eval-summary.schema.json#/totalCostUsd|type|number",
5814
+ "schemas/v2/eval-summary.schema.json#/trialStats/k|type|integer",
5815
+ "schemas/v2/eval-summary.schema.json#/trialStats/passAtK|type|number",
5816
+ "schemas/v2/eval-summary.schema.json#/trialStats/passHatK|type|number",
5817
+ "schemas/v2/eval-summary.schema.json#/trialStats/scoreStderr|type|number",
5818
+ "schemas/v2/eval-summary.schema.json#/trialStats|closed|true",
5819
+ "schemas/v2/eval-summary.schema.json#/trialStats|property|k",
5820
+ "schemas/v2/eval-summary.schema.json#/trialStats|property|passAtK",
5821
+ "schemas/v2/eval-summary.schema.json#/trialStats|property|passHatK",
5822
+ "schemas/v2/eval-summary.schema.json#/trialStats|property|scoreStderr",
5823
+ "schemas/v2/eval-summary.schema.json#/trialStats|required|k",
5824
+ "schemas/v2/eval-summary.schema.json#/trialStats|required|passAtK",
5825
+ "schemas/v2/eval-summary.schema.json#/trialStats|required|passHatK",
5826
+ "schemas/v2/eval-summary.schema.json#/trialStats|type|object",
5682
5827
  "schemas/v2/eval-summary.schema.json#|closed|true",
5683
5828
  "schemas/v2/eval-summary.schema.json#|property|aggregateScore",
5684
5829
  "schemas/v2/eval-summary.schema.json#|property|evaluatedModelClass",
@@ -5691,6 +5836,7 @@
5691
5836
  "schemas/v2/eval-summary.schema.json#|property|taskCount",
5692
5837
  "schemas/v2/eval-summary.schema.json#|property|tasks",
5693
5838
  "schemas/v2/eval-summary.schema.json#|property|totalCostUsd",
5839
+ "schemas/v2/eval-summary.schema.json#|property|trialStats",
5694
5840
  "schemas/v2/eval-summary.schema.json#|required|aggregateScore",
5695
5841
  "schemas/v2/eval-summary.schema.json#|required|passed",
5696
5842
  "schemas/v2/eval-summary.schema.json#|required|passedCount",