@openwop/openwop-conformance 2.46.23 → 2.46.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,3 +1,13 @@
1
+ ## [2.46.25] — 2026-10-25 — RFC 0244 §C efficiency leg
2
+
3
+ - **New leg `0244.efficiency-tool-calls` in `v2-eval-trajectory-trials`**, on the existing `conformance-eval-trajectory` run (and the `conformance-eval-trials` run when served), so no extra run. Each task's `efficiency.toolCallCount` must equal its script length, and on the one-trial run the counts must sum to the log's `agent.tool-called` events. Judge in `lib/eval-efficiency-witness.ts`.
4
+ - **Host impact:** openwop-app and MyndHyve run it. A host that reports no `efficiency` records `inapplicable`; one that counts judge or other-agent calls fails.
5
+ - **Three new §D legs through the reserved counter tool `conformance.eval:counter` (RFC 0244 G2):** `0244.live-miss-counted`, `0244.first-unconsumed` and `0244.miss-no-live-call`. Two new fixtures (`conformance-eval-counter-live`, `-miss`) run as three sequential eval runs. Judges in `lib/eval-counter-witness.ts`. A host opts in by listing both fixtures (contract in `fixtures.md`); no host does yet, so the legs record `inapplicable`.
6
+
7
+ ## [2.46.24] — 2026-10-25 — RFC 0250 Accepted
8
+
9
+ - **No assertion changes.** RFC 0250's G6 closes on openwop-app's certified cut `f94471e98` (all six `0250.*` legs `executed-pass`), so the packed `spec/v1/gaps.json` changes.
10
+
1
11
  ## [2.46.23] — 2026-10-24 — RFC 0250 Active: opt-in host-event replay
2
12
 
3
13
  - **New scenario `v2-host-event-strict-replay`.** The legs are `0250.shape` (server-free), `resume-backlog`, `ephemeral-not-replayed`, `cursor-refused`, `refusal-uniform` and `legacy-unchanged`. The suite emits durable D1, D2, ephemeral E and durable D3 through the RFC 0241 trigger, then reconnects with `replay=required` from D1 and emits D4 live. Judges are in `lib/host-event-replay-witness.ts`; their self-tests fail each wrong host answer, including a foreign cursor answered differently (an oracle).
@@ -1,5 +1,5 @@
1
1
  {
2
2
  "package": "@openwop/spec-artifacts",
3
- "version": "2.46.23",
4
- "stampSha256": "1b49ee96c0cbd693f4446f8be5372a386ac55a776780987853b5e8b1a0af0c44"
3
+ "version": "2.46.25",
4
+ "stampSha256": "d50d0faa638afab3defdf1ab0771218528dfddbcf4867d112cb2c3cc68dfb120"
5
5
  }
@@ -0,0 +1,68 @@
1
+ {
2
+ "suiteId": "core.openwop.evals.conformance-eval-counter-live",
3
+ "version": "1.0.0",
4
+ "targetAgentId": "core.conformance.agents.scripted",
5
+ "modes": [
6
+ "golden"
7
+ ],
8
+ "tasks": [
9
+ {
10
+ "taskId": "live-counter",
11
+ "input": {
12
+ "script": [
13
+ {
14
+ "tool": "conformance.eval:counter",
15
+ "arguments": {
16
+ "key": "L",
17
+ "op": "inc"
18
+ }
19
+ },
20
+ {
21
+ "tool": "conformance.eval:counter",
22
+ "arguments": {
23
+ "key": "L",
24
+ "op": "inc"
25
+ }
26
+ },
27
+ {
28
+ "tool": "conformance.eval:counter",
29
+ "arguments": {
30
+ "key": "F",
31
+ "op": "read"
32
+ }
33
+ },
34
+ {
35
+ "tool": "conformance.eval:counter",
36
+ "arguments": {
37
+ "key": "S",
38
+ "op": "read"
39
+ }
40
+ }
41
+ ],
42
+ "output": "done"
43
+ },
44
+ "expected": {
45
+ "kind": "golden",
46
+ "match": {
47
+ "strategy": "contains",
48
+ "value": "done"
49
+ }
50
+ },
51
+ "fixtures": {
52
+ "onMiss": "live",
53
+ "toolResponses": [
54
+ {
55
+ "tool": "conformance.eval:lookup",
56
+ "args": {
57
+ "orderId": "o-1"
58
+ },
59
+ "argsMatch": "exact",
60
+ "response": {
61
+ "status": "shipped"
62
+ }
63
+ }
64
+ ]
65
+ }
66
+ }
67
+ ]
68
+ }
@@ -0,0 +1,80 @@
1
+ {
2
+ "suiteId": "core.openwop.evals.conformance-eval-counter-miss",
3
+ "version": "1.0.0",
4
+ "targetAgentId": "core.conformance.agents.scripted",
5
+ "modes": [
6
+ "golden"
7
+ ],
8
+ "tasks": [
9
+ {
10
+ "taskId": "served-then-miss",
11
+ "input": {
12
+ "script": [
13
+ {
14
+ "tool": "conformance.eval:counter",
15
+ "arguments": {
16
+ "key": "S",
17
+ "op": "inc"
18
+ }
19
+ },
20
+ {
21
+ "tool": "conformance.eval:counter",
22
+ "arguments": {
23
+ "key": "S",
24
+ "op": "inc"
25
+ }
26
+ },
27
+ {
28
+ "tool": "conformance.eval:counter",
29
+ "arguments": {
30
+ "key": "S",
31
+ "op": "inc"
32
+ }
33
+ },
34
+ {
35
+ "tool": "conformance.eval:counter",
36
+ "arguments": {
37
+ "key": "F",
38
+ "op": "inc"
39
+ }
40
+ }
41
+ ],
42
+ "output": "done"
43
+ },
44
+ "expected": {
45
+ "kind": "golden",
46
+ "match": {
47
+ "strategy": "contains",
48
+ "value": "done"
49
+ }
50
+ },
51
+ "fixtures": {
52
+ "onMiss": "fail",
53
+ "toolResponses": [
54
+ {
55
+ "tool": "conformance.eval:counter",
56
+ "args": {
57
+ "key": "S",
58
+ "op": "inc"
59
+ },
60
+ "argsMatch": "exact",
61
+ "response": {
62
+ "served": 1
63
+ }
64
+ },
65
+ {
66
+ "tool": "conformance.eval:counter",
67
+ "args": {
68
+ "key": "S",
69
+ "op": "inc"
70
+ },
71
+ "argsMatch": "exact",
72
+ "response": {
73
+ "served": 2
74
+ }
75
+ }
76
+ ]
77
+ }
78
+ }
79
+ ]
80
+ }
package/fixtures.md CHANGED
@@ -75,6 +75,8 @@ All fixtures MUST advertise:
75
75
  | Eval — Trajectory Gate | `conformance-eval-trajectory-gate` | RFC 0246 — two golden tasks scoring 1 at `passScore: 0.6`; `path-matched` keeps its trajectory, `path-wrong` calls a forbidden tool (`v2-eval-trajectory-trials`) | eval run `completed`; `aggregateScore` 1, `gatedAggregateScore` 0.5, suite `passed: false` | ≤ 30s |
76
76
  | Eval — Trajectory Gate (mean) | `conformance-eval-trajectory-gate-mean` | RFC 0246 — `trials: {count: 2, passRule: "mean"}`, one golden task scoring 1 that calls a forbidden tool in both trials (`v2-eval-trajectory-trials`) | eval run `completed`; `meanScore` 1, `gatedScore` 0, task `passed: false` | ≤ 30s |
77
77
  | Eval — Fixture Error | `conformance-eval-fixture-error` | RFC 0245 — `onMiss: "fail"`; one task calls the same tool twice, served first by an `isError: true` entry, then by a success (`v2-eval-trajectory-trials`) | eval run `completed`; the first call is recorded as a tool error | ≤ 30s |
78
+ | Eval — Counter (live) | `conformance-eval-counter-live` | RFC 0244 §D, G2 — `onMiss: "live"`; one task makes four `conformance.eval:counter` calls that match no entry, so each runs the live counter (`v2-eval-trajectory-trials`) | eval run `completed`; four live `{key, count}` outcomes | ≤ 30s |
79
+ | Eval — Counter (miss) | `conformance-eval-counter-miss` | RFC 0244 §D, G2 — `onMiss: "fail"`; three `inc S` calls against two exact entries, then an unmatched `inc F` (`v2-eval-trajectory-trials`) | eval run `completed`; `{served: 1}`, `{served: 2}`, then two tool errors | ≤ 30s |
78
80
  | Dispatch Fan-Out Waves | `conformance-dispatch-fanout-waves` | RFC 0243 §A.3 — a parallel `core.dispatch` (`maxConcurrency: 2`, `wait-all`/`collect`) whose supervisor's `mockDispatchPlan` names three `conformance-delay` children. At most `min(2, maxFanOut)` run at once and none is dropped. Gated on `agents.fanOut` admitting `wait-all` with `collect`. | `completed` | ≤ 30s |
79
81
  | Dispatch Per-Item Input | `conformance-dispatch-per-item` | RFC 0243 §A.4 — a sequential `core.dispatch` whose decision dispatches `conformance-dispatch-input-mapping-child` twice with `nextWorkerInputs` `item-a`, `item-b`. At `agents.fanOut.perItemInput: true` each child receives its item; otherwise the dispatch node fails `validation_error` and no child is dispatched. | `completed`, or `failed` without `perItemInput` | ≤ 30s |
80
82
  | Dispatch Per-Item Length Mismatch | `conformance-dispatch-per-item-mismatch` | `orchestrator-decision.schema.json` `nextWorkerInputs` — one item for two children. Every dispatch host fails the dispatch node `validation_error` and dispatches no child. | `failed` | ≤ 30s |
@@ -1046,17 +1048,27 @@ The `fixtures/eval-suites/` sub-directory holds `AgentEvalSuite` documents (`../
1046
1048
  Every task targets **the scripted agent `core.conformance.agents.scripted`**. Its contract:
1047
1049
 
1048
1050
  - `input.script` is an ordered list of `{ tool, arguments? }`. The agent makes exactly those tool calls, in order, each emitting `agent.tool-called` with `agentId: "core.conformance.agents.scripted"`, `toolName: tool` and `inputs: arguments`.
1049
- - Each call goes through the host's eval tool boundary. In a task with `fixtures.toolResponses`, the RFC 0244 §D selection and `onMiss` rules apply. Otherwise the host answers the call with its own `arguments`. No scripted call reaches a live tool or an external effect.
1051
+ - Each call goes through the host's eval tool boundary. In a task with `fixtures.toolResponses`, the RFC 0244 §D selection and `onMiss` rules apply. Otherwise the host answers the call with its own `arguments`. No scripted call reaches a live tool or an external effect, except a miss on the counter tool under `onMiss: "live"` (below).
1050
1052
  - After the last call the agent decides, and its output is `input.output`.
1051
1053
 
1052
1054
  | Fixture | Facet | What the suite asserts |
1053
1055
  | --- | --- | --- |
1054
1056
  | `conformance-eval-trajectory` | `agents.evalSuite.trajectory` | each task's `trajectory` counts equal `lib/eval-trajectory.ts`'s grade of the script; an unmatched trajectory fails; a `kind: "trajectory"` task scores 1 or 0; the log carries one `agent.tool-called` per scripted call |
1055
1057
  | `conformance-eval-trials` | `agents.evalSuite.trials` | `eval.started.trialCount: 3`; three `eval.scored` per task with `trialIndex` 0..2; the per-task `trials` block and `trialStats` equal the §B formulas. Without the facet: `400 validation_error` at `createRun` |
1056
- | `conformance-eval-fixture-miss` | `agents.evalSuite.fixtureMatching` | `missed` reports `fixtureMissCount: 1`, `passed: false` and `score: 0` (summary and every `eval.scored`); `served` passes with no miss. That no live tool ran is not observable here (RFC 0244 G2) |
1058
+ | `conformance-eval-fixture-miss` | `agents.evalSuite.fixtureMatching` | `missed` reports `fixtureMissCount: 1`, `passed: false` and `score: 0` (summary and every `eval.scored`); `served` passes with no miss. That no live tool ran is shown by the counter fixtures below (RFC 0244 G2) |
1057
1059
  | `conformance-eval-trajectory-gate` | `agents.evalSuite.trajectory` | RFC 0246: both tasks keep `score` 1 and `aggregateScore` 1; `path-matched` carries `gatedScore` 1 and `path-wrong` `gatedScore` 0; `gatedAggregateScore` 0.5; suite `passed: false` although `aggregateScore` clears `passScore` 0.6 |
1058
1060
  | `conformance-eval-trajectory-gate-mean` | `agents.evalSuite.trajectory` + `agents.evalSuite.trials` | RFC 0246: two `eval.scored` with `score` 1 and `passed: false`; `trials.meanScore` 1, `gatedScore` 0, task `passed: false`; the summary carries `gatedAggregateScore` |
1059
1061
  | `conformance-eval-fixture-error` | `agents.evalSuite.fixtureErrors` + `agents.evalSuite.fixtureMatching` | RFC 0245: two `agent.tool-called` and two `agent.tool-returned` for `conformance.eval:lookup`, paired by `callId`; the first return carries `error` with non-empty `code` and `message` and no `outcome` (`status: "error"` under `toolHooks.prePostEvents`), the second carries no `error` (`outcome` is optional; not `status: "error"` under tool hooks); `recovers` reports `fixtureMissCount: 0`. Without `fixtureErrors`: `400 validation_error`. That the agent saw an error and no live tool ran is not observable here (RFC 0245 G1, RFC 0244 G2) |
1062
+ | `conformance-eval-counter-live` | `agents.evalSuite.fixtureMatching` | run twice, before and after `conformance-eval-counter-miss`: `fixtureMissCount` > 0; the two live `inc L` counts are consecutive; the second run's first `L` is the first run's last plus one (else `blocked`), and its `F` and `S` reads equal the first run's |
1063
+ | `conformance-eval-counter-miss` | `agents.evalSuite.fixtureMatching` | the three `inc S` calls return `{served: 1}`, `{served: 2}`, then a tool error; `inc F` returns a tool error. Neither miss moves the live counter |
1064
+
1065
+ **The counter tool `conformance.eval:counter` is conformance-RESERVED** (the pattern of `core.conformance.side-effect`). A host that lists either counter fixture in discovery `fixtures` MUST, at its eval tool boundary:
1066
+
1067
+ - answer a call that reaches the live tool with inputs `{key, op}` as follows. `inc` adds one to a counter scoped to the caller's owner triple and `key`, then returns `{key, count}`. `read` returns `{key, count}` without changing it. A key never incremented reads `0`.
1068
+ - keep that state for at least an hour, across eval runs;
1069
+ - record `outcome` on `agent.tool-returned` for every counter call: the live `{key, count}`, or a served entry's `response` verbatim. A miss under `onMiss: "fail"` carries `error` instead.
1070
+
1071
+ The counter has no external effect, so serving it is safe on a production tenant. Two concurrent cuts on one tenant make the persistence check fail; the legs then record `blocked`, not a failure.
1060
1072
 
1061
1073
  ## OAuth provider fixtures
1062
1074
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openwop/openwop-conformance",
3
- "version": "2.46.23",
3
+ "version": "2.46.25",
4
4
  "description": "Black-box conformance suite for OpenWOP hosts (v2 current; v1 scenarios retained for the frozen v1 tree).",
5
5
  "repository": {
6
6
  "type": "git",
@@ -56,6 +56,6 @@
56
56
  "@openwop/spec-artifacts": "file:../spec-artifacts"
57
57
  },
58
58
  "peerDependencies": {
59
- "@openwop/spec-artifacts": "2.46.23"
59
+ "@openwop/spec-artifacts": "2.46.25"
60
60
  }
61
61
  }
package/requirements.json CHANGED
@@ -3,10 +3,10 @@
3
3
  "generatedFrom": "src/scenarios/*.test.ts",
4
4
  "counts": {
5
5
  "files": 749,
6
- "tests": 2840,
7
- "withStableId": 2840,
6
+ "tests": 2844,
7
+ "withStableId": 2844,
8
8
  "interpolatedTitles": 0,
9
- "explicitIds": 2713
9
+ "explicitIds": 2717
10
10
  },
11
11
  "records": [
12
12
  {
@@ -31092,7 +31092,7 @@
31092
31092
  {
31093
31093
  "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-suite-summary-and-event-fields-validate-and-the-negatives-are-refus",
31094
31094
  "file": "v2-eval-trajectory-trials.test.ts",
31095
- "line": 162,
31095
+ "line": 185,
31096
31096
  "title": "the RFC 0244 suite, summary and event fields validate and the negatives are refused",
31097
31097
  "explicitId": "openwop.requirement.0244.wire-shape",
31098
31098
  "citations": [
@@ -31131,7 +31131,7 @@
31131
31131
  {
31132
31132
  "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-additions-are-content-free-eval-summary-no-content-leak",
31133
31133
  "file": "v2-eval-trajectory-trials.test.ts",
31134
- "line": 185,
31134
+ "line": 208,
31135
31135
  "title": "the RFC 0244 additions are content-free (eval-summary-no-content-leak)",
31136
31136
  "explicitId": "openwop.requirement.0244.content-free",
31137
31137
  "citations": [
@@ -31148,7 +31148,7 @@
31148
31148
  {
31149
31149
  "id": "openwop.it.v2-eval-trajectory-trials.the-eval-suite-fixtures-validate-against-the-v2-suite-schema",
31150
31150
  "file": "v2-eval-trajectory-trials.test.ts",
31151
- "line": 192,
31151
+ "line": 215,
31152
31152
  "title": "the eval-suite fixtures validate against the v2 suite schema",
31153
31153
  "explicitId": "openwop.requirement.0244.fixtures-valid",
31154
31154
  "citations": [
@@ -31162,7 +31162,7 @@
31162
31162
  {
31163
31163
  "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0246-gating-fields-validate-and-stay-in-0-1",
31164
31164
  "file": "v2-eval-trajectory-trials.test.ts",
31165
- "line": 199,
31165
+ "line": 222,
31166
31166
  "title": "the RFC 0246 gating fields validate and stay in 0–1",
31167
31167
  "explicitId": "openwop.requirement.0246.gated-fields-shape",
31168
31168
  "citations": [
@@ -31184,7 +31184,7 @@
31184
31184
  {
31185
31185
  "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-entry-may-be-marked-iserror-and-only-as-a-boolean-rfc-0245-a",
31186
31186
  "file": "v2-eval-trajectory-trials.test.ts",
31187
- "line": 207,
31187
+ "line": 230,
31188
31188
  "title": "a fixture entry may be marked isError, and only as a boolean (RFC 0245 §A)",
31189
31189
  "explicitId": "openwop.requirement.0245.fixture-error-shape",
31190
31190
  "citations": [
@@ -31206,7 +31206,7 @@
31206
31206
  {
31207
31207
  "id": "openwop.it.v2-eval-trajectory-trials.a-trajectory-suite-is-graded-per-the-reference-grader",
31208
31208
  "file": "v2-eval-trajectory-trials.test.ts",
31209
- "line": 217,
31209
+ "line": 240,
31210
31210
  "title": "a trajectory suite is graded per the reference grader",
31211
31211
  "explicitId": "openwop.requirement.0244.trajectory-graded",
31212
31212
  "citations": [
@@ -31240,7 +31240,7 @@
31240
31240
  {
31241
31241
  "id": "openwop.it.v2-eval-trajectory-trials.an-eval-run-brackets-its-scores-with-one-eval-started-and-one-eval-completed-rfc",
31242
31242
  "file": "v2-eval-trajectory-trials.test.ts",
31243
- "line": 251,
31243
+ "line": 274,
31244
31244
  "title": "an eval run brackets its scores with one eval.started and one eval.completed (RFC 0081 §C)",
31245
31245
  "explicitId": "openwop.requirement.0081.eval-envelope",
31246
31246
  "citations": [
@@ -31254,7 +31254,7 @@
31254
31254
  {
31255
31255
  "id": "openwop.it.v2-eval-trajectory-trials.the-served-evalsummary-validates-and-its-counts-match-its-rows-rfc-0081-c",
31256
31256
  "file": "v2-eval-trajectory-trials.test.ts",
31257
- "line": 259,
31257
+ "line": 282,
31258
31258
  "title": "the served EvalSummary validates and its counts match its rows (RFC 0081 §C)",
31259
31259
  "explicitId": "openwop.requirement.0081.summary-served",
31260
31260
  "citations": [
@@ -31268,7 +31268,7 @@
31268
31268
  {
31269
31269
  "id": "openwop.it.v2-eval-trajectory-trials.repeated-trials-emit-one-eval-scored-per-task-per-trial",
31270
31270
  "file": "v2-eval-trajectory-trials.test.ts",
31271
- "line": 280,
31271
+ "line": 303,
31272
31272
  "title": "repeated trials emit one eval.scored per task per trial",
31273
31273
  "explicitId": "openwop.requirement.0244.trials-per-task",
31274
31274
  "citations": [
@@ -31286,7 +31286,7 @@
31286
31286
  {
31287
31287
  "id": "openwop.it.v2-eval-trajectory-trials.the-per-task-trials-block-and-trialstats-follow-the-b-formulas",
31288
31288
  "file": "v2-eval-trajectory-trials.test.ts",
31289
- "line": 291,
31289
+ "line": 314,
31290
31290
  "title": "the per-task trials block and trialStats follow the §B formulas",
31291
31291
  "explicitId": "openwop.requirement.0244.trial-stats-formula",
31292
31292
  "citations": [
@@ -31322,10 +31322,66 @@
31322
31322
  }
31323
31323
  ]
31324
31324
  },
31325
+ {
31326
+ "id": "openwop.it.v2-eval-trajectory-trials.efficiency-toolcallcount-counts-the-scripted-agent-tool-called-events-c",
31327
+ "file": "v2-eval-trajectory-trials.test.ts",
31328
+ "line": 337,
31329
+ "title": "efficiency.toolCallCount counts the scripted agent.tool-called events (§C)",
31330
+ "explicitId": "openwop.requirement.0244.efficiency-tool-calls",
31331
+ "citations": [
31332
+ {
31333
+ "section": "eval-summary.schema.json efficiency (RFC 0244 §C)",
31334
+ "requirement": null,
31335
+ "interpolated": true
31336
+ }
31337
+ ]
31338
+ },
31339
+ {
31340
+ "id": "openwop.it.v2-eval-trajectory-trials.a-miss-under-onmiss-live-reaches-the-live-counter-and-is-counted-d",
31341
+ "file": "v2-eval-trajectory-trials.test.ts",
31342
+ "line": 371,
31343
+ "title": "a miss under onMiss \"live\" reaches the live counter and is counted (§D)",
31344
+ "explicitId": "openwop.requirement.0244.live-miss-counted",
31345
+ "citations": [
31346
+ {
31347
+ "section": "RFC 0244 §D, fixtures.md counter contract",
31348
+ "requirement": null,
31349
+ "interpolated": true
31350
+ }
31351
+ ]
31352
+ },
31353
+ {
31354
+ "id": "openwop.it.v2-eval-trajectory-trials.repeated-calls-are-served-the-first-unconsumed-fixture-entry-once-each-d",
31355
+ "file": "v2-eval-trajectory-trials.test.ts",
31356
+ "line": 379,
31357
+ "title": "repeated calls are served the first unconsumed fixture entry, once each (§D)",
31358
+ "explicitId": "openwop.requirement.0244.first-unconsumed",
31359
+ "citations": [
31360
+ {
31361
+ "section": "RFC 0244 §D, fixtures.md counter contract",
31362
+ "requirement": null,
31363
+ "interpolated": true
31364
+ }
31365
+ ]
31366
+ },
31367
+ {
31368
+ "id": "openwop.it.v2-eval-trajectory-trials.a-miss-under-onmiss-fail-never-reaches-the-live-counter-d",
31369
+ "file": "v2-eval-trajectory-trials.test.ts",
31370
+ "line": 387,
31371
+ "title": "a miss under onMiss \"fail\" never reaches the live counter (§D)",
31372
+ "explicitId": "openwop.requirement.0244.miss-no-live-call",
31373
+ "citations": [
31374
+ {
31375
+ "section": "RFC 0244 §D, fixtures.md counter contract",
31376
+ "requirement": null,
31377
+ "interpolated": true
31378
+ }
31379
+ ]
31380
+ },
31325
31381
  {
31326
31382
  "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-miss-under-onmiss-fail-fails-the-trial-and-is-counted",
31327
31383
  "file": "v2-eval-trajectory-trials.test.ts",
31328
- "line": 314,
31384
+ "line": 395,
31329
31385
  "title": "a fixture miss under onMiss \"fail\" fails the trial and is counted",
31330
31386
  "explicitId": "openwop.requirement.0244.fixture-miss-fails",
31331
31387
  "citations": [
@@ -31362,7 +31418,7 @@
31362
31418
  {
31363
31419
  "id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-an-unadvertised-rfc-0244-feature-is-refused-at-createrun",
31364
31420
  "file": "v2-eval-trajectory-trials.test.ts",
31365
- "line": 334,
31421
+ "line": 415,
31366
31422
  "title": "a suite using an unadvertised RFC 0244 feature is refused at createRun",
31367
31423
  "explicitId": "openwop.requirement.0244.unadvertised-feature-refused",
31368
31424
  "citations": [
@@ -31380,7 +31436,7 @@
31380
31436
  {
31381
31437
  "id": "openwop.it.v2-eval-trajectory-trials.under-passrule-mean-a-task-whose-trials-failed-their-trajectory-does-not-pass-rf",
31382
31438
  "file": "v2-eval-trajectory-trials.test.ts",
31383
- "line": 350,
31439
+ "line": 431,
31384
31440
  "title": "under passRule \"mean\" a task whose trials failed their trajectory does not pass (RFC 0246)",
31385
31441
  "explicitId": "openwop.requirement.0246.mean-gated",
31386
31442
  "citations": [
@@ -31394,7 +31450,7 @@
31394
31450
  {
31395
31451
  "id": "openwop.it.v2-eval-trajectory-trials.a-suite-whose-tasks-failed-their-trajectory-does-not-pass-on-the-output-scores-r",
31396
31452
  "file": "v2-eval-trajectory-trials.test.ts",
31397
- "line": 363,
31453
+ "line": 444,
31398
31454
  "title": "a suite whose tasks failed their trajectory does not pass on the output scores (RFC 0246)",
31399
31455
  "explicitId": "openwop.requirement.0246.suite-gated",
31400
31456
  "citations": [
@@ -31408,7 +31464,7 @@
31408
31464
  {
31409
31465
  "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-entry-marked-iserror-is-replayed-as-a-tool-error-not-a-success-rfc-024",
31410
31466
  "file": "v2-eval-trajectory-trials.test.ts",
31411
- "line": 374,
31467
+ "line": 455,
31412
31468
  "title": "a fixture entry marked isError is replayed as a tool error, not a success (RFC 0245)",
31413
31469
  "explicitId": "openwop.requirement.0245.fixture-error-replayed",
31414
31470
  "citations": [
@@ -31422,7 +31478,7 @@
31422
31478
  {
31423
31479
  "id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-iserror-is-refused-at-createrun-without-fixtureerrors-rfc-0245-c",
31424
31480
  "file": "v2-eval-trajectory-trials.test.ts",
31425
- "line": 388,
31481
+ "line": 469,
31426
31482
  "title": "a suite using isError is refused at createRun without fixtureErrors (RFC 0245 §C)",
31427
31483
  "explicitId": "openwop.requirement.0245.unadvertised-refused",
31428
31484
  "citations": [
@@ -1,17 +1,17 @@
1
1
  {
2
2
  "_comment": "Provenance of @openwop/spec-artifacts (RFC 0168 §D.2). files: SHA-256 per file; the conformance suite compares the installed peer against dist/spec-artifacts.lock.json at start.",
3
3
  "package": "@openwop/spec-artifacts",
4
- "version": "2.46.23",
5
- "corpusTag": "v2.46.23",
4
+ "version": "2.46.25",
5
+ "corpusTag": "v2.46.25",
6
6
  "files": {
7
7
  "api/.redocly.lint-ignore.yaml": "bf5a8350b88a72fa43f59605ed8d903ed24b6cfccda5e45509c9f6ed9ee4e712",
8
8
  "api/asyncapi.yaml": "d5ecb9ee6114582be3b1f662c84bfac9ae96dae7bacb853e461168f70a8e1c7d",
9
9
  "api/grpc/openwop.proto": "c3e72bb17cba514ee98feb6434e6c9b6ea6795bfd086489ec69fd882dd1ad977",
10
10
  "api/openapi.yaml": "ac6de5303872664b78ff419c035915593e01599e02835e777a0123de1baeb398",
11
11
  "api/redocly.yaml": "b0604c89b2ca6d5076ec25725c539dad44a741a811fe524439ee6daef8baa09f",
12
- "api/seams-v2.yaml": "52382aca8c66e0682ecd314bfc3eb5696481ea5ae82f4707f7d15ae971667832",
13
- "api/v2/asyncapi.yaml": "e5f747b10c1b518f5db8b26158bbe09b006f4854742fefc5b7d3f5be1e2cd71d",
14
- "api/v2/openapi.yaml": "e35c8fd7e1b5aaad265b2d6a3ca7378661139930895cb8b931a0e7d365db4a3f",
12
+ "api/seams-v2.yaml": "8fd581fc6b17706f32f6793d5cf00fd6de9c1d4791645d37efe90df2d430d9d5",
13
+ "api/v2/asyncapi.yaml": "f1a40cc22ed7c3c7069e49ff50da59b893f8cac43474fed8f48b4dbed4bdf5a2",
14
+ "api/v2/openapi.yaml": "8287abaade3983fc4546cb2a7c7d5e4e50595377b4fdf53ca65738d7284ce1a4",
15
15
  "api/v2/redocly.yaml": "1e66b60e6118ad11a823bb620678be464d99dfe50a40e3e6f93ec9429b88b34c",
16
16
  "schemas/README.md": "276fc1d51eb08e14e9127b9fd27761d7046ee33c12000f3a64a801322db8bf58",
17
17
  "schemas/a2a-task-state.schema.json": "75d5049dea8bd873ff0e7546f1c60c8d36c219bec7084264360be54a8be30ae1",
@@ -212,14 +212,14 @@
212
212
  "schemas/workspace-file.schema.json": "464de85c2a068243084ee9c1d969bc7cd5d8f7948574e58450d6493c38a0e1e4",
213
213
  "spec/v1/alias-detectors.json": "22444c22bb88c568339ef512553f9fc38146e22b566d11ab8608f655d688d19c",
214
214
  "spec/v1/capability-declaration-classes.json": "e7729aed5c4b4e1dd02abab0530f14cc95f5d4070fe51fb139e7f5cccefa00c6",
215
- "spec/v1/core-standard-manifest.json": "9e67cef74ab08d445a80e5066a47c04f0cc86a6e426ed3d2a407ace31fc8dfe5",
215
+ "spec/v1/core-standard-manifest.json": "c3705885e787498f604ffd256c4d6e26afcc929a79a548b7ecd46bf87e9e8a0f",
216
216
  "spec/v1/deprecations.json": "e723bc11d7272c71789ad1303bb7f21809a8a024abf57594f9d5c35dc199ece9",
217
217
  "spec/v1/deprecations.schema.json": "3e393c405d2a41b467d8c5e3c468549078df1ce6a6d2588fc488097b95d9b55b",
218
218
  "spec/v1/end-of-support.json": "35f0bf3b70fe78b16e2a2fd81a153fc27e703f6c16376c6fe7cdf1f4f2c4e6d4",
219
219
  "spec/v1/event-codemap.json": "3da60d884157793a360da532a9fcbbfb5285636db325a74cec94b34622186d97",
220
220
  "spec/v1/event-codemap.schema.json": "d05933b2e88103aff51a2f774df97b9bda065e0fc88dbd9b114cc47a32189174",
221
221
  "spec/v1/extensions.json": "79a60754aa16cbdbdb604f8e5c00038af535a1690a4d07bd5ff7c38f0256a7bc",
222
- "spec/v1/gaps.json": "622cc6c1d2f789f54f7ace1c754a06cb3a70f9f39e5b623c73d0730b3af42f0e",
222
+ "spec/v1/gaps.json": "21d53eee36f47b3cd44ba54802acc5cdc810d242d1fd929b1e76167da8566957",
223
223
  "spec/v1/gaps.schema.json": "8fd83259f556553c9df0f53e7a82ca8c2a4771d471197f69b4e2ae0ceeacfd66",
224
224
  "spec/v1/migrations.json": "4c50760be70cb644d02427bcaf5749606700edf4513f423903cf3b1d12f5c300",
225
225
  "spec/v1/migrations.schema.json": "886779aa6c22e646db097f5df210adb018a4dd14a7b815465a18c8a7056c8f72",
@@ -318,9 +318,9 @@
318
318
  "spec/v2/path-manifest.json": "20f598f56df460ad3ed50aba3d6bc2933ea16a3c96ce125da1c5dd6989d4155c",
319
319
  "spec/v2/peer-dependency-aliases.json": "d10299280abee08258502925bc327293ee413e0108cd6e6ec75ff6110653308d",
320
320
  "spec/v2/profiles.json": "b382b261c7844798575f7c1b37c934f0da19846016a2337d883d4f5ef6e05479",
321
- "spec/v2/release.json": "144826ee9c8d37e14bbe13262748b0dc6109d84a9d457a93ef2f7cc6f7675396",
321
+ "spec/v2/release.json": "0d42cf69cb062a05c6b7d7d09dea7bbc44e918ffb056112303459883ddd032f0",
322
322
  "spec/v2/retention-floors.json": "eaf3722d95c79947af1d4269ef85117e126518c588cfcf1a2b21b97269f51624",
323
- "spec/v2/surface-baseline.json": "4c847211a7eb32a7b76dd38d15ebb5fa80e78f3e82cd2c4ee000a74f708c6341"
323
+ "spec/v2/surface-baseline.json": "585d61f9dae285fdb9e302568fefc2d91faaa166bd6a8c20e3bac1a82940f8af"
324
324
  },
325
- "corpusCommit": "2db5b3ce931febf665ae21cea429e4d8632aeeb9"
325
+ "corpusCommit": "77aeb0cacb91551dab835cf0e985be8423e06542"
326
326
  }
@@ -0,0 +1,112 @@
1
+ /**
2
+ * RFC 0244 §D, G2 — the suite-owned counter tool `conformance.eval:counter`
3
+ * makes "a miss under onMiss: \"fail\" never reaches a live tool" and "the
4
+ * first unconsumed entry is served" observable from outside. Pure judges over
5
+ * three sequential eval runs, for `v2-eval-trajectory-trials.test.ts`.
6
+ *
7
+ * The counter's live effect is persistent, tenant-scoped state: `inc` adds one
8
+ * to `(owner, key)` and returns `{key, count}`; `read` returns it unchanged
9
+ * (contract in `fixtures.md` §Eval-suite fixtures). The runs are:
10
+ * A. `conformance-eval-counter-live` (`onMiss: "live"`): inc L, inc L, read F, read S.
11
+ * B. `conformance-eval-counter-miss` (`onMiss: "fail"`): inc S three times
12
+ * against two exact entries serving `{served: 1}` then `{served: 2}`, then inc F.
13
+ * C. `conformance-eval-counter-live` again.
14
+ * C's first L is A's second L plus one, which proves the state persisted
15
+ * between runs; only then do unchanged F and S counts prove B's two misses
16
+ * never ran the live tool. A host that leaked a miss reads one higher. When
17
+ * persistence is not shown (a concurrent cut on the tenant, or no durable
18
+ * state) the verdict is `blocked`, never a failure.
19
+ *
20
+ * @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md §D
21
+ */
22
+
23
+ export const COUNTER_TOOL = 'conformance.eval:counter';
24
+
25
+ export interface CounterFinding { readonly ok: boolean; readonly message: string }
26
+ export type CounterVerdict =
27
+ | { readonly kind: 'blocked'; readonly reason: string }
28
+ | { readonly kind: 'judged'; readonly findings: readonly CounterFinding[] };
29
+
30
+ type Rec = Record<string, unknown>;
31
+ export interface CounterEvent { readonly type: string; readonly payload: Rec }
32
+ export interface CounterCall { readonly key: unknown; readonly op: unknown; readonly outcome: unknown; readonly error: unknown; readonly returned: boolean }
33
+
34
+ const isRec = (v: unknown): v is Rec => typeof v === 'object' && v !== null && !Array.isArray(v);
35
+ const count = (c: CounterCall | undefined): number | undefined => {
36
+ const o = c?.outcome;
37
+ return isRec(o) && typeof o['count'] === 'number' && Number.isInteger(o['count']) ? o['count'] : undefined;
38
+ };
39
+
40
+ /** The counter calls of one run, in log order, each paired with its return by `callId`. */
41
+ export function counterCalls(events: readonly CounterEvent[], agentId: string): CounterCall[] {
42
+ const returns = new Map<string, Rec>();
43
+ for (const e of events) {
44
+ if (e.type === 'agent.tool-returned' && e.payload['toolName'] === COUNTER_TOOL && typeof e.payload['callId'] === 'string') returns.set(e.payload['callId'], e.payload);
45
+ }
46
+ const out: CounterCall[] = [];
47
+ for (const e of events) {
48
+ if (e.type !== 'agent.tool-called' || e.payload['toolName'] !== COUNTER_TOOL || e.payload['agentId'] !== agentId) continue;
49
+ const inputs = isRec(e.payload['inputs']) ? e.payload['inputs'] : {};
50
+ const r = typeof e.payload['callId'] === 'string' ? returns.get(e.payload['callId']) : undefined;
51
+ out.push({ key: inputs['key'], op: inputs['op'], outcome: r?.['outcome'], error: r?.['error'], returned: r !== undefined });
52
+ }
53
+ return out;
54
+ }
55
+
56
+ /** Run A or C: inc L, inc L, read F, read S, every one answered live with `{key, count}`. */
57
+ function liveCounts(calls: readonly CounterCall[], label: string): { counts: { L1: number; L2: number; F: number; S: number } } | { reason: string } {
58
+ if (calls.length !== 4) return { reason: `run ${label} MUST log the four scripted counter calls (got ${calls.length})` };
59
+ const [l1, l2, f, s] = calls.map(count);
60
+ if (l1 === undefined || l2 === undefined || f === undefined || s === undefined) {
61
+ return { reason: `run ${label}: every live counter call MUST record outcome {key, count} on agent.tool-returned (fixtures.md counter contract)` };
62
+ }
63
+ return { counts: { L1: l1, L2: l2, F: f, S: s } };
64
+ }
65
+
66
+ /** §D `onMiss: "live"` on run A: the misses reached the live tool, and the summary reports them. */
67
+ export function judgeLiveMiss(a: readonly CounterCall[], fixtureMissCount: unknown): CounterVerdict {
68
+ const live = liveCounts(a, 'A');
69
+ if ('reason' in live) return { kind: 'blocked', reason: live.reason };
70
+ return {
71
+ kind: 'judged',
72
+ findings: [
73
+ { ok: typeof fixtureMissCount === 'number' && fixtureMissCount > 0, message: `a task whose misses ran live under onMiss "live" MUST report fixtureMissCount > 0 (got ${String(fixtureMissCount)})` },
74
+ { ok: live.counts.L2 === live.counts.L1 + 1, message: `two live inc calls MUST advance the counter by one each (got ${live.counts.L1} then ${live.counts.L2})` },
75
+ ],
76
+ };
77
+ }
78
+
79
+ /** §D selection on run B: the two exact entries are served in order, once each, and the third call is a miss. */
80
+ export function judgeFirstUnconsumed(b: readonly CounterCall[]): CounterVerdict {
81
+ const s = b.filter((c) => c.key === 'S');
82
+ if (s.length !== 3 || !s.every((c) => c.returned)) return { kind: 'blocked', reason: `run B MUST log three paired counter calls for key S (got ${s.length})` };
83
+ const served = (c: CounterCall): unknown => (isRec(c.outcome) ? c.outcome['served'] : undefined);
84
+ return {
85
+ kind: 'judged',
86
+ findings: [
87
+ { ok: served(s[0]!) === 1 && s[0]!.error === undefined, message: `the first call MUST be served the first unconsumed entry {served: 1} (got ${JSON.stringify(s[0]!.outcome ?? s[0]!.error)})` },
88
+ { ok: served(s[1]!) === 2 && s[1]!.error === undefined, message: `the second call MUST be served the next unconsumed entry {served: 2}, each entry consumed once (got ${JSON.stringify(s[1]!.outcome ?? s[1]!.error)})` },
89
+ { ok: s[2]!.error !== undefined && count(s[2]) === undefined && served(s[2]!) === undefined, message: `the third call matches no unconsumed entry, so under onMiss "fail" it MUST return a tool error (got ${JSON.stringify(s[2]!.outcome ?? s[2]!.error)})` },
90
+ ],
91
+ };
92
+ }
93
+
94
+ /** §D `onMiss: "fail"` across A → B → C: neither miss in B ran the live tool. */
95
+ export function judgeMissNoLiveCall(a: readonly CounterCall[], b: readonly CounterCall[], c: readonly CounterCall[]): CounterVerdict {
96
+ const ra = liveCounts(a, 'A');
97
+ if ('reason' in ra) return { kind: 'blocked', reason: ra.reason };
98
+ const rc = liveCounts(c, 'C');
99
+ if ('reason' in rc) return { kind: 'blocked', reason: rc.reason };
100
+ if (rc.counts.L1 !== ra.counts.L2 + 1) {
101
+ return { kind: 'blocked', reason: `the counter state did not persist from run A to run C on its own (L went ${ra.counts.L2} → ${rc.counts.L1}, want ${ra.counts.L2 + 1}): another run on the tenant, or no durable state, so unchanged counts prove nothing` };
102
+ }
103
+ const f = b.filter((x) => x.key === 'F');
104
+ return {
105
+ kind: 'judged',
106
+ findings: [
107
+ { ok: f.length === 1 && f[0]!.error !== undefined && count(f[0]) === undefined, message: `run B's unmatched inc F MUST return a tool error, not a live count (got ${JSON.stringify(f[0]?.outcome ?? f[0]?.error)})` },
108
+ { ok: rc.counts.F === ra.counts.F, message: `a miss under onMiss "fail" MUST NOT reach the live tool: counter F MUST read the same after run B (${ra.counts.F} → ${rc.counts.F})` },
109
+ { ok: rc.counts.S === ra.counts.S, message: `fixture-served calls and the unmatched third inc S MUST NOT reach the live tool: counter S MUST read the same after run B (${ra.counts.S} → ${rc.counts.S})` },
110
+ ],
111
+ };
112
+ }
@@ -0,0 +1,58 @@
1
+ /**
2
+ * RFC 0244 §C — the per-task `efficiency.toolCallCount` is the count of
3
+ * `agent.tool-called` events inside the task's invocation brackets, as a mean
4
+ * over its trials. A pure judge over an eval run's `EvalSummary` and event log,
5
+ * for `v2-eval-trajectory-trials.test.ts`.
6
+ *
7
+ * The expected count is known in advance: the scripted agent replays each
8
+ * task's `input.script` as tool calls on every trial, so a task's mean is its
9
+ * script length, whatever the host. The judge never reads the host's own count
10
+ * to decide what the count should be. Only `toolCallCount` is judged:
11
+ * `modelCallCount` and the token sums depend on whether the host's scripted
12
+ * agent calls a model, which the fixture contract leaves to the host.
13
+ *
14
+ * `efficiency` is optional on a summary entry, so a run whose entries carry
15
+ * none is not evidence either way (`applicable: false`).
16
+ *
17
+ * @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md §C
18
+ */
19
+
20
+ export interface EfficiencyFinding { readonly ok: boolean; readonly message: string }
21
+ export interface EfficiencyVerdict { readonly applicable: boolean; readonly findings: readonly EfficiencyFinding[] }
22
+ export interface ScriptedTask { readonly taskId: string; readonly scriptLength: number }
23
+
24
+ type Rec = Record<string, unknown>;
25
+ const isRec = (v: unknown): v is Rec => typeof v === 'object' && v !== null && !Array.isArray(v);
26
+
27
+ /**
28
+ * `toolCallsInLog` is the number of `agent.tool-called` events the run's log
29
+ * carries for the agent under test. On a run of one trial per task the
30
+ * per-task counts sum to it; on a multi-trial run pass `undefined`, since the
31
+ * sum of means is not an event count.
32
+ */
33
+ export function judgeToolCallCount(summary: Rec, tasks: readonly ScriptedTask[], toolCallsInLog: number | undefined): EfficiencyVerdict {
34
+ const entries = (Array.isArray(summary['tasks']) ? (summary['tasks'] as unknown[]) : []).filter(isRec);
35
+ const findings: EfficiencyFinding[] = [];
36
+ let reported = 0;
37
+ let sum = 0;
38
+ for (const t of tasks) {
39
+ const entry = entries.find((e) => e['taskId'] === t.taskId);
40
+ const eff = entry?.['efficiency'];
41
+ if (!isRec(eff)) continue;
42
+ reported += 1;
43
+ const got = eff['toolCallCount'];
44
+ findings.push({
45
+ ok: typeof got === 'number' && got === t.scriptLength,
46
+ message: `task ${t.taskId}: efficiency.toolCallCount MUST count the agent.tool-called events in its brackets, the ${t.scriptLength} scripted calls per trial (got ${String(got)})`,
47
+ });
48
+ if (typeof got === 'number') sum += got;
49
+ }
50
+ if (reported === 0) return { applicable: false, findings: [] };
51
+ if (toolCallsInLog !== undefined && reported === tasks.length) {
52
+ findings.push({
53
+ ok: sum === toolCallsInLog,
54
+ message: `the per-task toolCallCount values MUST sum to the run's ${toolCallsInLog} agent.tool-called events on a one-trial run (got ${sum})`,
55
+ });
56
+ }
57
+ return { applicable: true, findings };
58
+ }
@@ -31,6 +31,22 @@
31
31
  * 6. Refusal — a host advertising `agents.evalSuite` without `trials` that
32
32
  * lists `conformance-eval-trials` refuses it at createRun with
33
33
  * `400 validation_error`.
34
+ * 6a. §C efficiency (2.46.25) — on the trajectory run, and on the trials run
35
+ * when served, each task's `efficiency.toolCallCount` equals its script
36
+ * length (the mean over identical trials), and on the one-trial run the
37
+ * counts sum to the log's `agent.tool-called` events. `efficiency` is
38
+ * optional, so a host that reports none records `inapplicable`. Judge in
39
+ * `lib/eval-efficiency-witness.ts`. The never-gated rule is not judged:
40
+ * no fixture makes efficiency the only difference between two verdicts.
41
+ * 6b. §D through the counter tool `conformance.eval:counter` (2.46.25, G2),
42
+ * gated on `fixtureMatching` and both counter fixtures in discovery.
43
+ * Three sequential runs: `conformance-eval-counter-live` (A), then
44
+ * `conformance-eval-counter-miss` (B), then the live suite again (C).
45
+ * `0244.live-miss-counted` (A's misses ran live and are counted),
46
+ * `0244.first-unconsumed` (B serves `{served: 1}`, `{served: 2}`, then a
47
+ * tool error), `0244.miss-no-live-call` (the live counters B's misses
48
+ * touched read the same in C as in A, once C proves the state persisted).
49
+ * Judges in `lib/eval-counter-witness.ts`.
34
50
  *
35
51
  * RFC 0081 §C, on the `conformance-eval-trajectory` run (2.46.20). Judges in
36
52
  * `lib/eval-envelope-witness.ts`.
@@ -84,6 +100,8 @@ import { judgeEvalEnvelope, judgeSummaryServed } from '../lib/eval-envelope-witn
84
100
  import { gradeTrajectory, passAtK, passHatK, scoreStderr, withinTolerance, type TrajectorySpec } from '../lib/eval-trajectory.js';
85
101
  import { judgeMeanGated, judgeSuiteGated } from '../lib/eval-gate-witness.js';
86
102
  import { judgeFixtureErrorReplayed } from '../lib/eval-fixture-error-witness.js';
103
+ import { judgeToolCallCount } from '../lib/eval-efficiency-witness.js';
104
+ import { counterCalls, judgeFirstUnconsumed, judgeLiveMiss, judgeMissNoLiveCall, type CounterCall } from '../lib/eval-counter-witness.js';
87
105
 
88
106
  const PROFILE = majorProfile(2);
89
107
  const UNREADABLE = 'v2 discovery unreachable — /.well-known/openwop did not answer 200 JSON under OpenWOP-Version: 2.0';
@@ -98,6 +116,11 @@ const ID_TRIALS = 'openwop.requirement.0244.trials-per-task';
98
116
  const ID_TRIAL_STATS = 'openwop.requirement.0244.trial-stats-formula';
99
117
  const ID_MISS = 'openwop.requirement.0244.fixture-miss-fails';
100
118
  const ID_REFUSED = 'openwop.requirement.0244.unadvertised-feature-refused';
119
+ const ID_EFFICIENCY = 'openwop.requirement.0244.efficiency-tool-calls';
120
+ const ID_LIVE_MISS = 'openwop.requirement.0244.live-miss-counted';
121
+ const ID_FIRST_UNCONSUMED = 'openwop.requirement.0244.first-unconsumed';
122
+ const ID_NO_LIVE_CALL = 'openwop.requirement.0244.miss-no-live-call';
123
+ const DOC_COUNTER = 'RFC 0244 §D, fixtures.md counter contract';
101
124
  const ID_GATED_SHAPE = 'openwop.requirement.0246.gated-fields-shape';
102
125
  const ID_MEAN_GATED = 'openwop.requirement.0246.mean-gated';
103
126
  const ID_SUITE_GATED = 'openwop.requirement.0246.suite-gated';
@@ -190,7 +213,7 @@ describe('v2 RFC 0244 — wire shape (server-free)', () => {
190
213
  });
191
214
 
192
215
  it('the eval-suite fixtures validate against the v2 suite schema', () => {
193
- for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss', 'conformance-eval-trajectory-gate', 'conformance-eval-trajectory-gate-mean', 'conformance-eval-fixture-error']) {
216
+ for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss', 'conformance-eval-trajectory-gate', 'conformance-eval-trajectory-gate-mean', 'conformance-eval-fixture-error', 'conformance-eval-counter-live', 'conformance-eval-counter-miss']) {
194
217
  const r = suiteSchema(fixture(id));
195
218
  expect(r.ok, req(ID_FIXTURES, 'fixtures.md §eval-suites', `${id} MUST validate: ${r.errors}`)).toBe(true);
196
219
  }
@@ -311,6 +334,64 @@ describe('v2 RFC 0244 — host behaviour', () => {
311
334
  if (se !== undefined) expect(withinTolerance(Number(stats['scoreStderr']), se), req(ID_TRIAL_STATS, DOC, `trialStats.scoreStderr MUST equal sd(per-task means)/√T within 1e-9 relative (got ${String(stats['scoreStderr'])}, want ${se})`)).toBe(true);
312
335
  });
313
336
 
337
+ it('efficiency.toolCallCount counts the scripted agent.tool-called events (§C)', async () => {
338
+ const r = await trajectoryRun();
339
+ if ('skip' in r) return softSkip(...r.skip);
340
+ const scriptedTasks = (s: Suite) => s.tasks.map((t) => ({ taskId: t.taskId, scriptLength: t.input.script.length }));
341
+ const logCalls = r.o.events.filter((e) => e.type === 'agent.tool-called' && e.payload['agentId'] === AGENT).length;
342
+ const verdicts = [judgeToolCallCount(r.o.summary, scriptedTasks(fixture('conformance-eval-trajectory')), logCalls)];
343
+ const t = await trialsRun();
344
+ if (!('skip' in t)) verdicts.push(judgeToolCallCount(t.o.summary, scriptedTasks(t.suite), undefined));
345
+ if (!verdicts.some((v) => v.applicable)) return softSkip('inapplicable', 'no task entry carries efficiency, which RFC 0244 §C makes optional');
346
+ for (const v of verdicts) for (const x of v.findings) expect(x.ok, req(ID_EFFICIENCY, 'eval-summary.schema.json efficiency (RFC 0244 §C)', x.message)).toBe(true);
347
+ });
348
+
349
+ type CounterRuns = { skip: [Parameters<typeof softSkip>[0], string] } | { a: CounterCall[]; b: CounterCall[]; c: CounterCall[]; aMisses: unknown };
350
+ let counterSeq: Promise<CounterRuns> | undefined;
351
+ /** A → B → C, strictly in order, once per worker: the three counter legs read the same runs. */
352
+ function counterRuns(): Promise<CounterRuns> {
353
+ counterSeq ??= (async (): Promise<CounterRuns> => {
354
+ const { doc, facet } = await evalSuiteFacet();
355
+ if (!doc) return { skip: ['blocked', UNREADABLE] };
356
+ if (facet?.['fixtureMatching'] !== true) return { skip: ['inapplicable', 'the host does not advertise agents.evalSuite.fixtureMatching'] };
357
+ for (const id of ['conformance-eval-counter-live', 'conformance-eval-counter-miss']) {
358
+ if (!isFixtureAdvertised(id)) return { skip: ['inapplicable', `the host does not advertise fixture ${id}, its opt-in to the counter-tool contract`] };
359
+ }
360
+ const a = await runSuiteOnce('conformance-eval-counter-live');
361
+ if (a.kind === 'skip') return { skip: [a.disposition, `run A: ${a.reason}`] };
362
+ const b = await runSuiteOnce('conformance-eval-counter-miss');
363
+ if (b.kind === 'skip') return { skip: [b.disposition, `run B: ${b.reason}`] };
364
+ const c = await runSuiteOnce('conformance-eval-counter-live');
365
+ if (c.kind === 'skip') return { skip: [c.disposition, `run C: ${c.reason}`] };
366
+ return { a: counterCalls(a.events, AGENT), b: counterCalls(b.events, AGENT), c: counterCalls(c.events, AGENT), aMisses: taskEntry(a.summary, 'live-counter')?.['fixtureMissCount'] };
367
+ })();
368
+ return counterSeq;
369
+ }
370
+
371
+ it('a miss under onMiss "live" reaches the live counter and is counted (§D)', async () => {
372
+ const r = await counterRuns();
373
+ if ('skip' in r) return softSkip(...r.skip);
374
+ const v = judgeLiveMiss(r.a, r.aMisses);
375
+ if (v.kind === 'blocked') return softSkip('blocked', v.reason);
376
+ for (const x of v.findings) expect(x.ok, req(ID_LIVE_MISS, DOC_COUNTER, x.message)).toBe(true);
377
+ });
378
+
379
+ it('repeated calls are served the first unconsumed fixture entry, once each (§D)', async () => {
380
+ const r = await counterRuns();
381
+ if ('skip' in r) return softSkip(...r.skip);
382
+ const v = judgeFirstUnconsumed(r.b);
383
+ if (v.kind === 'blocked') return softSkip('blocked', v.reason);
384
+ for (const x of v.findings) expect(x.ok, req(ID_FIRST_UNCONSUMED, DOC_COUNTER, x.message)).toBe(true);
385
+ });
386
+
387
+ it('a miss under onMiss "fail" never reaches the live counter (§D)', async () => {
388
+ const r = await counterRuns();
389
+ if ('skip' in r) return softSkip(...r.skip);
390
+ const v = judgeMissNoLiveCall(r.a, r.b, r.c);
391
+ if (v.kind === 'blocked') return softSkip('blocked', v.reason);
392
+ for (const x of v.findings) expect(x.ok, req(ID_NO_LIVE_CALL, DOC_COUNTER, x.message)).toBe(true);
393
+ });
394
+
314
395
  it('a fixture miss under onMiss "fail" fails the trial and is counted', async () => {
315
396
  const { doc, facet } = await evalSuiteFacet();
316
397
  if (!doc) return softSkip('blocked', UNREADABLE);