@openwop/openwop-conformance 2.46.25 → 2.46.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/spec-artifacts.lock.json +2 -2
- package/fixtures/eval-suites/conformance-eval-efficiency-ungated.json +97 -0
- package/fixtures/eval-suites/conformance-eval-memory-isolation.json +71 -0
- package/fixtures.md +14 -2
- package/package.json +2 -2
- package/requirements.json +79 -23
- package/schemas/CORPUS-STAMP.json +10 -10
- package/src/lib/eval-efficiency-witness.ts +110 -1
- package/src/lib/eval-memory-witness.ts +82 -0
- package/src/scenarios/v2-eval-trajectory-trials.test.ts +80 -4
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
## [2.46.27] — 2026-10-25 — RFC 0244 never-gated and trial-isolation legs
|
|
2
|
+
|
|
3
|
+
- **New leg `0244.efficiency-never-gated` in `v2-eval-trajectory-trials`**, on the new fixture `conformance-eval-efficiency-ungated`: tasks `lean` (one scripted call) and `busy` (eight) share an output and an expectation, so `busy` must pass with `lean`'s score and the suite must pass at `passScore` 1. `blocked` when the calls never ran or `lean` fails. Judge `judgeNeverGated` in `lib/eval-efficiency-witness.ts`.
|
|
4
|
+
- **New leg `0244.trial-memory-isolated` (RFC 0244 G3)**, on the new fixture `conformance-eval-memory-isolation` (`trials.count: 2`, gated on `agents.evalSuite.trials`), through the conformance-reserved memory tool `conformance.eval:memory` (contract in `fixtures.md`). Each trial must read the `memorySeed` entry, and its first read of `trial-note` must find nothing before its own write. Judge in `lib/eval-memory-witness.ts`; a tool whose writes never read back records `blocked`.
|
|
5
|
+
- **Host impact:** none until a host lists a new fixture in discovery `fixtures`. Listing the memory fixture opts the host in to the memory-tool contract.
|
|
6
|
+
|
|
7
|
+
## [2.46.26] — 2026-10-25 — RFC 0244 model-usage and cost legs
|
|
8
|
+
|
|
9
|
+
- **New leg `0244.efficiency-model-usage` in `v2-eval-trajectory-trials`**, on the existing trajectory and trials runs. Per-task `modelCallCount`, `inputTokens` and `outputTokens`, times the trial count, must sum to the agent's `provider.usage` inside its `agent.invocation.*` brackets (judge and delegate calls excluded); without brackets they must not exceed the run's usage.
|
|
10
|
+
- **New leg `0244.cost-every-trial`**, on the trials run: `totalCostUsd` must be at least the agent's priced usage over all trials. Judges in `lib/eval-efficiency-witness.ts`.
|
|
11
|
+
- **Host impact:** a run with no model usage, no brackets or unpriced usage records `inapplicable`; one that counts judge calls as the agent's, or prices one trial, fails.
|
|
12
|
+
|
|
1
13
|
## [2.46.25] — 2026-10-25 — RFC 0244 §C efficiency leg
|
|
2
14
|
|
|
3
15
|
- **New leg `0244.efficiency-tool-calls` in `v2-eval-trajectory-trials`**, on the existing `conformance-eval-trajectory` run (and the `conformance-eval-trials` run when served), so no extra run. Each task's `efficiency.toolCallCount` must equal its script length, and on the one-trial run the counts must sum to the log's `agent.tool-called` events. Judge in `lib/eval-efficiency-witness.ts`.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
{
|
|
2
|
+
"suiteId": "core.openwop.evals.conformance-eval-efficiency-ungated",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"targetAgentId": "core.conformance.agents.scripted",
|
|
5
|
+
"modes": [
|
|
6
|
+
"golden"
|
|
7
|
+
],
|
|
8
|
+
"thresholds": {
|
|
9
|
+
"passScore": 1
|
|
10
|
+
},
|
|
11
|
+
"tasks": [
|
|
12
|
+
{
|
|
13
|
+
"taskId": "lean",
|
|
14
|
+
"input": {
|
|
15
|
+
"script": [
|
|
16
|
+
{
|
|
17
|
+
"tool": "conformance.eval:lookup",
|
|
18
|
+
"arguments": {
|
|
19
|
+
"orderId": "o-1"
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
],
|
|
23
|
+
"output": "done"
|
|
24
|
+
},
|
|
25
|
+
"expected": {
|
|
26
|
+
"kind": "golden",
|
|
27
|
+
"match": {
|
|
28
|
+
"strategy": "contains",
|
|
29
|
+
"value": "done"
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"taskId": "busy",
|
|
35
|
+
"input": {
|
|
36
|
+
"script": [
|
|
37
|
+
{
|
|
38
|
+
"tool": "conformance.eval:lookup",
|
|
39
|
+
"arguments": {
|
|
40
|
+
"orderId": "o-1"
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"tool": "conformance.eval:lookup",
|
|
45
|
+
"arguments": {
|
|
46
|
+
"orderId": "o-2"
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"tool": "conformance.eval:lookup",
|
|
51
|
+
"arguments": {
|
|
52
|
+
"orderId": "o-3"
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"tool": "conformance.eval:lookup",
|
|
57
|
+
"arguments": {
|
|
58
|
+
"orderId": "o-4"
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"tool": "conformance.eval:lookup",
|
|
63
|
+
"arguments": {
|
|
64
|
+
"orderId": "o-5"
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"tool": "conformance.eval:lookup",
|
|
69
|
+
"arguments": {
|
|
70
|
+
"orderId": "o-6"
|
|
71
|
+
}
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
"tool": "conformance.eval:lookup",
|
|
75
|
+
"arguments": {
|
|
76
|
+
"orderId": "o-7"
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"tool": "conformance.eval:lookup",
|
|
81
|
+
"arguments": {
|
|
82
|
+
"orderId": "o-8"
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
],
|
|
86
|
+
"output": "done"
|
|
87
|
+
},
|
|
88
|
+
"expected": {
|
|
89
|
+
"kind": "golden",
|
|
90
|
+
"match": {
|
|
91
|
+
"strategy": "contains",
|
|
92
|
+
"value": "done"
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
]
|
|
97
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
{
|
|
2
|
+
"suiteId": "core.openwop.evals.conformance-eval-memory-isolation",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"targetAgentId": "core.conformance.agents.scripted",
|
|
5
|
+
"modes": [
|
|
6
|
+
"golden"
|
|
7
|
+
],
|
|
8
|
+
"trials": {
|
|
9
|
+
"count": 2,
|
|
10
|
+
"k": 1,
|
|
11
|
+
"passRule": "all"
|
|
12
|
+
},
|
|
13
|
+
"tasks": [
|
|
14
|
+
{
|
|
15
|
+
"taskId": "write-then-read",
|
|
16
|
+
"input": {
|
|
17
|
+
"script": [
|
|
18
|
+
{
|
|
19
|
+
"tool": "conformance.eval:memory",
|
|
20
|
+
"arguments": {
|
|
21
|
+
"op": "read",
|
|
22
|
+
"id": "seeded"
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"tool": "conformance.eval:memory",
|
|
27
|
+
"arguments": {
|
|
28
|
+
"op": "read",
|
|
29
|
+
"id": "trial-note"
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"tool": "conformance.eval:memory",
|
|
34
|
+
"arguments": {
|
|
35
|
+
"op": "write",
|
|
36
|
+
"id": "trial-note",
|
|
37
|
+
"content": "written"
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"tool": "conformance.eval:memory",
|
|
42
|
+
"arguments": {
|
|
43
|
+
"op": "read",
|
|
44
|
+
"id": "trial-note"
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
],
|
|
48
|
+
"output": "done"
|
|
49
|
+
},
|
|
50
|
+
"expected": {
|
|
51
|
+
"kind": "golden",
|
|
52
|
+
"match": {
|
|
53
|
+
"strategy": "contains",
|
|
54
|
+
"value": "done"
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
"fixtures": {
|
|
58
|
+
"memorySeed": [
|
|
59
|
+
{
|
|
60
|
+
"id": "seeded",
|
|
61
|
+
"content": "seed",
|
|
62
|
+
"tags": [
|
|
63
|
+
"conformance"
|
|
64
|
+
],
|
|
65
|
+
"createdAt": "2026-01-01T00:00:00Z"
|
|
66
|
+
}
|
|
67
|
+
]
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
]
|
|
71
|
+
}
|
package/fixtures.md
CHANGED
|
@@ -77,6 +77,8 @@ All fixtures MUST advertise:
|
|
|
77
77
|
| Eval — Fixture Error | `conformance-eval-fixture-error` | RFC 0245 — `onMiss: "fail"`; one task calls the same tool twice, served first by an `isError: true` entry, then by a success (`v2-eval-trajectory-trials`) | eval run `completed`; the first call is recorded as a tool error | ≤ 30s |
|
|
78
78
|
| Eval — Counter (live) | `conformance-eval-counter-live` | RFC 0244 §D, G2 — `onMiss: "live"`; one task makes four `conformance.eval:counter` calls that match no entry, so each runs the live counter (`v2-eval-trajectory-trials`) | eval run `completed`; four live `{key, count}` outcomes | ≤ 30s |
|
|
79
79
|
| Eval — Counter (miss) | `conformance-eval-counter-miss` | RFC 0244 §D, G2 — `onMiss: "fail"`; three `inc S` calls against two exact entries, then an unmatched `inc F` (`v2-eval-trajectory-trials`) | eval run `completed`; `{served: 1}`, `{served: 2}`, then two tool errors | ≤ 30s |
|
|
80
|
+
| Eval — Efficiency Ungated | `conformance-eval-efficiency-ungated` | RFC 0244 §C — two golden tasks with one output, `lean` (1 call) and `busy` (8 calls), at `passScore: 1` (`v2-eval-trajectory-trials`) | eval run `completed`; both tasks and the suite pass | ≤ 30s |
|
|
81
|
+
| Eval — Memory Isolation | `conformance-eval-memory-isolation` | RFC 0244 §B, G3 — `trials.count: 2`; one task seeds `seeded`, then reads it, reads `trial-note`, writes it and reads it back through `conformance.eval:memory` (`v2-eval-trajectory-trials`) | eval run `completed`; each trial reads the seed and finds no `trial-note` before its own write | ≤ 30s |
|
|
80
82
|
| Dispatch Fan-Out Waves | `conformance-dispatch-fanout-waves` | RFC 0243 §A.3 — a parallel `core.dispatch` (`maxConcurrency: 2`, `wait-all`/`collect`) whose supervisor's `mockDispatchPlan` names three `conformance-delay` children. At most `min(2, maxFanOut)` run at once and none is dropped. Gated on `agents.fanOut` admitting `wait-all` with `collect`. | `completed` | ≤ 30s |
|
|
81
83
|
| Dispatch Per-Item Input | `conformance-dispatch-per-item` | RFC 0243 §A.4 — a sequential `core.dispatch` whose decision dispatches `conformance-dispatch-input-mapping-child` twice with `nextWorkerInputs` `item-a`, `item-b`. At `agents.fanOut.perItemInput: true` each child receives its item; otherwise the dispatch node fails `validation_error` and no child is dispatched. | `completed`, or `failed` without `perItemInput` | ≤ 30s |
|
|
82
84
|
| Dispatch Per-Item Length Mismatch | `conformance-dispatch-per-item-mismatch` | `orchestrator-decision.schema.json` `nextWorkerInputs` — one item for two children. Every dispatch host fails the dispatch node `validation_error` and dispatches no child. | `failed` | ≤ 30s |
|
|
@@ -1048,19 +1050,21 @@ The `fixtures/eval-suites/` sub-directory holds `AgentEvalSuite` documents (`../
|
|
|
1048
1050
|
Every task targets **the scripted agent `core.conformance.agents.scripted`**. Its contract:
|
|
1049
1051
|
|
|
1050
1052
|
- `input.script` is an ordered list of `{ tool, arguments? }`. The agent makes exactly those tool calls, in order, each emitting `agent.tool-called` with `agentId: "core.conformance.agents.scripted"`, `toolName: tool` and `inputs: arguments`.
|
|
1051
|
-
- Each call goes through the host's eval tool boundary. In a task with `fixtures.toolResponses`, the RFC 0244 §D selection and `onMiss` rules apply. Otherwise the host answers the call with its own `arguments`. No scripted call reaches a live tool or an external effect, except a miss on the counter tool under `onMiss: "live"` (below).
|
|
1053
|
+
- Each call goes through the host's eval tool boundary. In a task with `fixtures.toolResponses`, the RFC 0244 §D selection and `onMiss` rules apply. Otherwise the host answers the call with its own `arguments`. No scripted call reaches a live tool or an external effect, except a miss on the counter tool under `onMiss: "live"` and a call to the memory tool (both below).
|
|
1052
1054
|
- After the last call the agent decides, and its output is `input.output`.
|
|
1053
1055
|
|
|
1054
1056
|
| Fixture | Facet | What the suite asserts |
|
|
1055
1057
|
| --- | --- | --- |
|
|
1056
1058
|
| `conformance-eval-trajectory` | `agents.evalSuite.trajectory` | each task's `trajectory` counts equal `lib/eval-trajectory.ts`'s grade of the script; an unmatched trajectory fails; a `kind: "trajectory"` task scores 1 or 0; the log carries one `agent.tool-called` per scripted call |
|
|
1057
|
-
| `conformance-eval-trials` | `agents.evalSuite.trials` | `eval.started.trialCount: 3`; three `eval.scored` per task with `trialIndex` 0..2; the per-task `trials` block and `trialStats` equal the §B formulas. Without the facet: `400 validation_error` at `createRun` |
|
|
1059
|
+
| `conformance-eval-trials` | `agents.evalSuite.trials` | `eval.started.trialCount: 3`; three `eval.scored` per task with `trialIndex` 0..2; the per-task `trials` block and `trialStats` equal the §B formulas; `totalCostUsd`, when served with bracketed priced usage, is at least the agent's cost over all three trials. Without the facet: `400 validation_error` at `createRun` |
|
|
1058
1060
|
| `conformance-eval-fixture-miss` | `agents.evalSuite.fixtureMatching` | `missed` reports `fixtureMissCount: 1`, `passed: false` and `score: 0` (summary and every `eval.scored`); `served` passes with no miss. That no live tool ran is shown by the counter fixtures below (RFC 0244 G2) |
|
|
1059
1061
|
| `conformance-eval-trajectory-gate` | `agents.evalSuite.trajectory` | RFC 0246: both tasks keep `score` 1 and `aggregateScore` 1; `path-matched` carries `gatedScore` 1 and `path-wrong` `gatedScore` 0; `gatedAggregateScore` 0.5; suite `passed: false` although `aggregateScore` clears `passScore` 0.6 |
|
|
1060
1062
|
| `conformance-eval-trajectory-gate-mean` | `agents.evalSuite.trajectory` + `agents.evalSuite.trials` | RFC 0246: two `eval.scored` with `score` 1 and `passed: false`; `trials.meanScore` 1, `gatedScore` 0, task `passed: false`; the summary carries `gatedAggregateScore` |
|
|
1061
1063
|
| `conformance-eval-fixture-error` | `agents.evalSuite.fixtureErrors` + `agents.evalSuite.fixtureMatching` | RFC 0245: two `agent.tool-called` and two `agent.tool-returned` for `conformance.eval:lookup`, paired by `callId`; the first return carries `error` with non-empty `code` and `message` and no `outcome` (`status: "error"` under `toolHooks.prePostEvents`), the second carries no `error` (`outcome` is optional; not `status: "error"` under tool hooks); `recovers` reports `fixtureMissCount: 0`. Without `fixtureErrors`: `400 validation_error`. That the agent saw an error and no live tool ran is not observable here (RFC 0245 G1, RFC 0244 G2) |
|
|
1062
1064
|
| `conformance-eval-counter-live` | `agents.evalSuite.fixtureMatching` | run twice, before and after `conformance-eval-counter-miss`: `fixtureMissCount` > 0; the two live `inc L` counts are consecutive; the second run's first `L` is the first run's last plus one (else `blocked`), and its `F` and `S` reads equal the first run's |
|
|
1063
1065
|
| `conformance-eval-counter-miss` | `agents.evalSuite.fixtureMatching` | the three `inc S` calls return `{served: 1}`, `{served: 2}`, then a tool error; `inc F` returns a tool error. Neither miss moves the live counter |
|
|
1066
|
+
| `conformance-eval-efficiency-ungated` | `agents.evalSuite` | RFC 0244 §C never-gated: with all nine scripted calls in the log and `lean` passing, `busy` passes with `lean`'s score and the suite passes. A host MUST NOT fail either on `efficiency`, reported or not |
|
|
1067
|
+
| `conformance-eval-memory-isolation` | `agents.evalSuite.trials` | RFC 0244 §B isolation: both reads of `seeded` return `seed`; of the four `trial-note` reads, two return `written` (each trial's read-back) and two return `null` (each trial's first read). A later trial reading `written` first saw an earlier trial's write. Fewer than two read-backs record `blocked` |
|
|
1064
1068
|
|
|
1065
1069
|
**The counter tool `conformance.eval:counter` is conformance-RESERVED** (the pattern of `core.conformance.side-effect`). A host that lists either counter fixture in discovery `fixtures` MUST, at its eval tool boundary:
|
|
1066
1070
|
|
|
@@ -1070,6 +1074,14 @@ Every task targets **the scripted agent `core.conformance.agents.scripted`**. It
|
|
|
1070
1074
|
|
|
1071
1075
|
The counter has no external effect, so serving it is safe on a production tenant. Two concurrent cuts on one tenant make the persistence check fail; the legs then record `blocked`, not a failure.
|
|
1072
1076
|
|
|
1077
|
+
**The memory tool `conformance.eval:memory` is conformance-RESERVED.** A host that lists `conformance-eval-memory-isolation` in discovery `fixtures` MUST, at its eval tool boundary:
|
|
1078
|
+
|
|
1079
|
+
- answer every call to it from the agent's memory for the current trial: the store the task's `fixtures.memorySeed` seeds (RFC 0004 `MemoryAdapter` entries) and the agent's own memory writes land in. It is never served from `toolResponses` and never counted as a fixture miss.
|
|
1080
|
+
- answer `{op: "write", id, content}` by storing the entry and returning `{id}`, and `{op: "read", id}` by returning `{id, content}`, with `content: null` when the trial's memory holds no entry `id`;
|
|
1081
|
+
- record `outcome` on `agent.tool-returned` for every memory call.
|
|
1082
|
+
|
|
1083
|
+
The tool reaches only the trial's eval memory, so it has no external effect. A host that answers it from a store other than the one its trials really use proves nothing about §B isolation, and the contract forbids it.
|
|
1084
|
+
|
|
1073
1085
|
## OAuth provider fixtures
|
|
1074
1086
|
|
|
1075
1087
|
RFC 0199 adds `provider.issuer`, `provider.pkce` and the two issuer-less providers (`issuerlessProviders[]`) the distinct-redirect-URI leg needs; the endpoints a host actually talks to are supplied at run time by the `oauth/authorize-start` seam, which points them at the suite's authorization-server double (`src/lib/oauth-as-double.ts`).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openwop/openwop-conformance",
|
|
3
|
-
"version": "2.46.
|
|
3
|
+
"version": "2.46.27",
|
|
4
4
|
"description": "Black-box conformance suite for OpenWOP hosts (v2 current; v1 scenarios retained for the frozen v1 tree).",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -56,6 +56,6 @@
|
|
|
56
56
|
"@openwop/spec-artifacts": "file:../spec-artifacts"
|
|
57
57
|
},
|
|
58
58
|
"peerDependencies": {
|
|
59
|
-
"@openwop/spec-artifacts": "2.46.
|
|
59
|
+
"@openwop/spec-artifacts": "2.46.27"
|
|
60
60
|
}
|
|
61
61
|
}
|
package/requirements.json
CHANGED
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
"generatedFrom": "src/scenarios/*.test.ts",
|
|
4
4
|
"counts": {
|
|
5
5
|
"files": 749,
|
|
6
|
-
"tests":
|
|
7
|
-
"withStableId":
|
|
6
|
+
"tests": 2848,
|
|
7
|
+
"withStableId": 2848,
|
|
8
8
|
"interpolatedTitles": 0,
|
|
9
|
-
"explicitIds":
|
|
9
|
+
"explicitIds": 2721
|
|
10
10
|
},
|
|
11
11
|
"records": [
|
|
12
12
|
{
|
|
@@ -31092,7 +31092,7 @@
|
|
|
31092
31092
|
{
|
|
31093
31093
|
"id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-suite-summary-and-event-fields-validate-and-the-negatives-are-refus",
|
|
31094
31094
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31095
|
-
"line":
|
|
31095
|
+
"line": 211,
|
|
31096
31096
|
"title": "the RFC 0244 suite, summary and event fields validate and the negatives are refused",
|
|
31097
31097
|
"explicitId": "openwop.requirement.0244.wire-shape",
|
|
31098
31098
|
"citations": [
|
|
@@ -31131,7 +31131,7 @@
|
|
|
31131
31131
|
{
|
|
31132
31132
|
"id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-additions-are-content-free-eval-summary-no-content-leak",
|
|
31133
31133
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31134
|
-
"line":
|
|
31134
|
+
"line": 234,
|
|
31135
31135
|
"title": "the RFC 0244 additions are content-free (eval-summary-no-content-leak)",
|
|
31136
31136
|
"explicitId": "openwop.requirement.0244.content-free",
|
|
31137
31137
|
"citations": [
|
|
@@ -31148,7 +31148,7 @@
|
|
|
31148
31148
|
{
|
|
31149
31149
|
"id": "openwop.it.v2-eval-trajectory-trials.the-eval-suite-fixtures-validate-against-the-v2-suite-schema",
|
|
31150
31150
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31151
|
-
"line":
|
|
31151
|
+
"line": 241,
|
|
31152
31152
|
"title": "the eval-suite fixtures validate against the v2 suite schema",
|
|
31153
31153
|
"explicitId": "openwop.requirement.0244.fixtures-valid",
|
|
31154
31154
|
"citations": [
|
|
@@ -31162,7 +31162,7 @@
|
|
|
31162
31162
|
{
|
|
31163
31163
|
"id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0246-gating-fields-validate-and-stay-in-0-1",
|
|
31164
31164
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31165
|
-
"line":
|
|
31165
|
+
"line": 248,
|
|
31166
31166
|
"title": "the RFC 0246 gating fields validate and stay in 0–1",
|
|
31167
31167
|
"explicitId": "openwop.requirement.0246.gated-fields-shape",
|
|
31168
31168
|
"citations": [
|
|
@@ -31184,7 +31184,7 @@
|
|
|
31184
31184
|
{
|
|
31185
31185
|
"id": "openwop.it.v2-eval-trajectory-trials.a-fixture-entry-may-be-marked-iserror-and-only-as-a-boolean-rfc-0245-a",
|
|
31186
31186
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31187
|
-
"line":
|
|
31187
|
+
"line": 256,
|
|
31188
31188
|
"title": "a fixture entry may be marked isError, and only as a boolean (RFC 0245 §A)",
|
|
31189
31189
|
"explicitId": "openwop.requirement.0245.fixture-error-shape",
|
|
31190
31190
|
"citations": [
|
|
@@ -31206,7 +31206,7 @@
|
|
|
31206
31206
|
{
|
|
31207
31207
|
"id": "openwop.it.v2-eval-trajectory-trials.a-trajectory-suite-is-graded-per-the-reference-grader",
|
|
31208
31208
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31209
|
-
"line":
|
|
31209
|
+
"line": 266,
|
|
31210
31210
|
"title": "a trajectory suite is graded per the reference grader",
|
|
31211
31211
|
"explicitId": "openwop.requirement.0244.trajectory-graded",
|
|
31212
31212
|
"citations": [
|
|
@@ -31240,7 +31240,7 @@
|
|
|
31240
31240
|
{
|
|
31241
31241
|
"id": "openwop.it.v2-eval-trajectory-trials.an-eval-run-brackets-its-scores-with-one-eval-started-and-one-eval-completed-rfc",
|
|
31242
31242
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31243
|
-
"line":
|
|
31243
|
+
"line": 300,
|
|
31244
31244
|
"title": "an eval run brackets its scores with one eval.started and one eval.completed (RFC 0081 §C)",
|
|
31245
31245
|
"explicitId": "openwop.requirement.0081.eval-envelope",
|
|
31246
31246
|
"citations": [
|
|
@@ -31254,7 +31254,7 @@
|
|
|
31254
31254
|
{
|
|
31255
31255
|
"id": "openwop.it.v2-eval-trajectory-trials.the-served-evalsummary-validates-and-its-counts-match-its-rows-rfc-0081-c",
|
|
31256
31256
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31257
|
-
"line":
|
|
31257
|
+
"line": 308,
|
|
31258
31258
|
"title": "the served EvalSummary validates and its counts match its rows (RFC 0081 §C)",
|
|
31259
31259
|
"explicitId": "openwop.requirement.0081.summary-served",
|
|
31260
31260
|
"citations": [
|
|
@@ -31268,7 +31268,7 @@
|
|
|
31268
31268
|
{
|
|
31269
31269
|
"id": "openwop.it.v2-eval-trajectory-trials.repeated-trials-emit-one-eval-scored-per-task-per-trial",
|
|
31270
31270
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31271
|
-
"line":
|
|
31271
|
+
"line": 329,
|
|
31272
31272
|
"title": "repeated trials emit one eval.scored per task per trial",
|
|
31273
31273
|
"explicitId": "openwop.requirement.0244.trials-per-task",
|
|
31274
31274
|
"citations": [
|
|
@@ -31286,7 +31286,7 @@
|
|
|
31286
31286
|
{
|
|
31287
31287
|
"id": "openwop.it.v2-eval-trajectory-trials.the-per-task-trials-block-and-trialstats-follow-the-b-formulas",
|
|
31288
31288
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31289
|
-
"line":
|
|
31289
|
+
"line": 340,
|
|
31290
31290
|
"title": "the per-task trials block and trialStats follow the §B formulas",
|
|
31291
31291
|
"explicitId": "openwop.requirement.0244.trial-stats-formula",
|
|
31292
31292
|
"citations": [
|
|
@@ -31325,7 +31325,7 @@
|
|
|
31325
31325
|
{
|
|
31326
31326
|
"id": "openwop.it.v2-eval-trajectory-trials.efficiency-toolcallcount-counts-the-scripted-agent-tool-called-events-c",
|
|
31327
31327
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31328
|
-
"line":
|
|
31328
|
+
"line": 363,
|
|
31329
31329
|
"title": "efficiency.toolCallCount counts the scripted agent.tool-called events (§C)",
|
|
31330
31330
|
"explicitId": "openwop.requirement.0244.efficiency-tool-calls",
|
|
31331
31331
|
"citations": [
|
|
@@ -31336,10 +31336,66 @@
|
|
|
31336
31336
|
}
|
|
31337
31337
|
]
|
|
31338
31338
|
},
|
|
31339
|
+
{
|
|
31340
|
+
"id": "openwop.it.v2-eval-trajectory-trials.efficiency-model-and-token-counts-match-the-agent-s-provider-usage-c",
|
|
31341
|
+
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31342
|
+
"line": 375,
|
|
31343
|
+
"title": "efficiency model and token counts match the agent's provider.usage (§C)",
|
|
31344
|
+
"explicitId": "openwop.requirement.0244.efficiency-model-usage",
|
|
31345
|
+
"citations": [
|
|
31346
|
+
{
|
|
31347
|
+
"section": "eval-summary.schema.json efficiency (RFC 0244 §C)",
|
|
31348
|
+
"requirement": null,
|
|
31349
|
+
"interpolated": true
|
|
31350
|
+
}
|
|
31351
|
+
]
|
|
31352
|
+
},
|
|
31353
|
+
{
|
|
31354
|
+
"id": "openwop.it.v2-eval-trajectory-trials.totalcostusd-counts-every-trial-b",
|
|
31355
|
+
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31356
|
+
"line": 386,
|
|
31357
|
+
"title": "totalCostUsd counts every trial (§B)",
|
|
31358
|
+
"explicitId": "openwop.requirement.0244.cost-every-trial",
|
|
31359
|
+
"citations": [
|
|
31360
|
+
{
|
|
31361
|
+
"section": "agent-eval-suite.schema.json thresholds.maxCostUsd, eval-summary.schema.json totalCostUsd (RFC 0244 §B)",
|
|
31362
|
+
"requirement": null,
|
|
31363
|
+
"interpolated": true
|
|
31364
|
+
}
|
|
31365
|
+
]
|
|
31366
|
+
},
|
|
31367
|
+
{
|
|
31368
|
+
"id": "openwop.it.v2-eval-trajectory-trials.a-host-does-not-fail-a-task-or-a-suite-on-efficiency-c",
|
|
31369
|
+
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31370
|
+
"line": 394,
|
|
31371
|
+
"title": "a host does not fail a task or a suite on efficiency (§C)",
|
|
31372
|
+
"explicitId": "openwop.requirement.0244.efficiency-never-gated",
|
|
31373
|
+
"citations": [
|
|
31374
|
+
{
|
|
31375
|
+
"section": "eval-summary.schema.json efficiency (RFC 0244 §C)",
|
|
31376
|
+
"requirement": null,
|
|
31377
|
+
"interpolated": true
|
|
31378
|
+
}
|
|
31379
|
+
]
|
|
31380
|
+
},
|
|
31381
|
+
{
|
|
31382
|
+
"id": "openwop.it.v2-eval-trajectory-trials.memory-one-trial-writes-is-not-visible-to-another-trial-b",
|
|
31383
|
+
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31384
|
+
"line": 409,
|
|
31385
|
+
"title": "memory one trial writes is not visible to another trial (§B)",
|
|
31386
|
+
"explicitId": "openwop.requirement.0244.trial-memory-isolated",
|
|
31387
|
+
"citations": [
|
|
31388
|
+
{
|
|
31389
|
+
"section": "agent-eval-suite.schema.json trials, fixtures.md memory contract (RFC 0244 §B)",
|
|
31390
|
+
"requirement": null,
|
|
31391
|
+
"interpolated": true
|
|
31392
|
+
}
|
|
31393
|
+
]
|
|
31394
|
+
},
|
|
31339
31395
|
{
|
|
31340
31396
|
"id": "openwop.it.v2-eval-trajectory-trials.a-miss-under-onmiss-live-reaches-the-live-counter-and-is-counted-d",
|
|
31341
31397
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31342
|
-
"line":
|
|
31398
|
+
"line": 447,
|
|
31343
31399
|
"title": "a miss under onMiss \"live\" reaches the live counter and is counted (§D)",
|
|
31344
31400
|
"explicitId": "openwop.requirement.0244.live-miss-counted",
|
|
31345
31401
|
"citations": [
|
|
@@ -31353,7 +31409,7 @@
|
|
|
31353
31409
|
{
|
|
31354
31410
|
"id": "openwop.it.v2-eval-trajectory-trials.repeated-calls-are-served-the-first-unconsumed-fixture-entry-once-each-d",
|
|
31355
31411
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31356
|
-
"line":
|
|
31412
|
+
"line": 455,
|
|
31357
31413
|
"title": "repeated calls are served the first unconsumed fixture entry, once each (§D)",
|
|
31358
31414
|
"explicitId": "openwop.requirement.0244.first-unconsumed",
|
|
31359
31415
|
"citations": [
|
|
@@ -31367,7 +31423,7 @@
|
|
|
31367
31423
|
{
|
|
31368
31424
|
"id": "openwop.it.v2-eval-trajectory-trials.a-miss-under-onmiss-fail-never-reaches-the-live-counter-d",
|
|
31369
31425
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31370
|
-
"line":
|
|
31426
|
+
"line": 463,
|
|
31371
31427
|
"title": "a miss under onMiss \"fail\" never reaches the live counter (§D)",
|
|
31372
31428
|
"explicitId": "openwop.requirement.0244.miss-no-live-call",
|
|
31373
31429
|
"citations": [
|
|
@@ -31381,7 +31437,7 @@
|
|
|
31381
31437
|
{
|
|
31382
31438
|
"id": "openwop.it.v2-eval-trajectory-trials.a-fixture-miss-under-onmiss-fail-fails-the-trial-and-is-counted",
|
|
31383
31439
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31384
|
-
"line":
|
|
31440
|
+
"line": 471,
|
|
31385
31441
|
"title": "a fixture miss under onMiss \"fail\" fails the trial and is counted",
|
|
31386
31442
|
"explicitId": "openwop.requirement.0244.fixture-miss-fails",
|
|
31387
31443
|
"citations": [
|
|
@@ -31418,7 +31474,7 @@
|
|
|
31418
31474
|
{
|
|
31419
31475
|
"id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-an-unadvertised-rfc-0244-feature-is-refused-at-createrun",
|
|
31420
31476
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31421
|
-
"line":
|
|
31477
|
+
"line": 491,
|
|
31422
31478
|
"title": "a suite using an unadvertised RFC 0244 feature is refused at createRun",
|
|
31423
31479
|
"explicitId": "openwop.requirement.0244.unadvertised-feature-refused",
|
|
31424
31480
|
"citations": [
|
|
@@ -31436,7 +31492,7 @@
|
|
|
31436
31492
|
{
|
|
31437
31493
|
"id": "openwop.it.v2-eval-trajectory-trials.under-passrule-mean-a-task-whose-trials-failed-their-trajectory-does-not-pass-rf",
|
|
31438
31494
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31439
|
-
"line":
|
|
31495
|
+
"line": 507,
|
|
31440
31496
|
"title": "under passRule \"mean\" a task whose trials failed their trajectory does not pass (RFC 0246)",
|
|
31441
31497
|
"explicitId": "openwop.requirement.0246.mean-gated",
|
|
31442
31498
|
"citations": [
|
|
@@ -31450,7 +31506,7 @@
|
|
|
31450
31506
|
{
|
|
31451
31507
|
"id": "openwop.it.v2-eval-trajectory-trials.a-suite-whose-tasks-failed-their-trajectory-does-not-pass-on-the-output-scores-r",
|
|
31452
31508
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31453
|
-
"line":
|
|
31509
|
+
"line": 520,
|
|
31454
31510
|
"title": "a suite whose tasks failed their trajectory does not pass on the output scores (RFC 0246)",
|
|
31455
31511
|
"explicitId": "openwop.requirement.0246.suite-gated",
|
|
31456
31512
|
"citations": [
|
|
@@ -31464,7 +31520,7 @@
|
|
|
31464
31520
|
{
|
|
31465
31521
|
"id": "openwop.it.v2-eval-trajectory-trials.a-fixture-entry-marked-iserror-is-replayed-as-a-tool-error-not-a-success-rfc-024",
|
|
31466
31522
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31467
|
-
"line":
|
|
31523
|
+
"line": 531,
|
|
31468
31524
|
"title": "a fixture entry marked isError is replayed as a tool error, not a success (RFC 0245)",
|
|
31469
31525
|
"explicitId": "openwop.requirement.0245.fixture-error-replayed",
|
|
31470
31526
|
"citations": [
|
|
@@ -31478,7 +31534,7 @@
|
|
|
31478
31534
|
{
|
|
31479
31535
|
"id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-iserror-is-refused-at-createrun-without-fixtureerrors-rfc-0245-c",
|
|
31480
31536
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31481
|
-
"line":
|
|
31537
|
+
"line": 545,
|
|
31482
31538
|
"title": "a suite using isError is refused at createRun without fixtureErrors (RFC 0245 §C)",
|
|
31483
31539
|
"explicitId": "openwop.requirement.0245.unadvertised-refused",
|
|
31484
31540
|
"citations": [
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_comment": "Provenance of @openwop/spec-artifacts (RFC 0168 §D.2). files: SHA-256 per file; the conformance suite compares the installed peer against dist/spec-artifacts.lock.json at start.",
|
|
3
3
|
"package": "@openwop/spec-artifacts",
|
|
4
|
-
"version": "2.46.
|
|
5
|
-
"corpusTag": "v2.46.
|
|
4
|
+
"version": "2.46.27",
|
|
5
|
+
"corpusTag": "v2.46.27",
|
|
6
6
|
"files": {
|
|
7
7
|
"api/.redocly.lint-ignore.yaml": "bf5a8350b88a72fa43f59605ed8d903ed24b6cfccda5e45509c9f6ed9ee4e712",
|
|
8
8
|
"api/asyncapi.yaml": "d5ecb9ee6114582be3b1f662c84bfac9ae96dae7bacb853e461168f70a8e1c7d",
|
|
9
9
|
"api/grpc/openwop.proto": "c3e72bb17cba514ee98feb6434e6c9b6ea6795bfd086489ec69fd882dd1ad977",
|
|
10
10
|
"api/openapi.yaml": "ac6de5303872664b78ff419c035915593e01599e02835e777a0123de1baeb398",
|
|
11
11
|
"api/redocly.yaml": "b0604c89b2ca6d5076ec25725c539dad44a741a811fe524439ee6daef8baa09f",
|
|
12
|
-
"api/seams-v2.yaml": "
|
|
13
|
-
"api/v2/asyncapi.yaml": "
|
|
14
|
-
"api/v2/openapi.yaml": "
|
|
12
|
+
"api/seams-v2.yaml": "fe9b3e83272ce29a2d0d13a09e12907cbf0e38e7de36d48bbb9343b6d88aa649",
|
|
13
|
+
"api/v2/asyncapi.yaml": "b647ccc7a8b7b6d62385d754bc597ef95725cf123655a9622510518ee2e64516",
|
|
14
|
+
"api/v2/openapi.yaml": "bfc7c189810490162495eb7e0f744b53274f6b72236004996cec951bb08af494",
|
|
15
15
|
"api/v2/redocly.yaml": "1e66b60e6118ad11a823bb620678be464d99dfe50a40e3e6f93ec9429b88b34c",
|
|
16
16
|
"schemas/README.md": "276fc1d51eb08e14e9127b9fd27761d7046ee33c12000f3a64a801322db8bf58",
|
|
17
17
|
"schemas/a2a-task-state.schema.json": "75d5049dea8bd873ff0e7546f1c60c8d36c219bec7084264360be54a8be30ae1",
|
|
@@ -212,14 +212,14 @@
|
|
|
212
212
|
"schemas/workspace-file.schema.json": "464de85c2a068243084ee9c1d969bc7cd5d8f7948574e58450d6493c38a0e1e4",
|
|
213
213
|
"spec/v1/alias-detectors.json": "22444c22bb88c568339ef512553f9fc38146e22b566d11ab8608f655d688d19c",
|
|
214
214
|
"spec/v1/capability-declaration-classes.json": "e7729aed5c4b4e1dd02abab0530f14cc95f5d4070fe51fb139e7f5cccefa00c6",
|
|
215
|
-
"spec/v1/core-standard-manifest.json": "
|
|
215
|
+
"spec/v1/core-standard-manifest.json": "0511812bdd86bbbe79f7a1dddd60878b01c66b1d4a43122822f74f5cd6a12bf9",
|
|
216
216
|
"spec/v1/deprecations.json": "e723bc11d7272c71789ad1303bb7f21809a8a024abf57594f9d5c35dc199ece9",
|
|
217
217
|
"spec/v1/deprecations.schema.json": "3e393c405d2a41b467d8c5e3c468549078df1ce6a6d2588fc488097b95d9b55b",
|
|
218
218
|
"spec/v1/end-of-support.json": "35f0bf3b70fe78b16e2a2fd81a153fc27e703f6c16376c6fe7cdf1f4f2c4e6d4",
|
|
219
219
|
"spec/v1/event-codemap.json": "3da60d884157793a360da532a9fcbbfb5285636db325a74cec94b34622186d97",
|
|
220
220
|
"spec/v1/event-codemap.schema.json": "d05933b2e88103aff51a2f774df97b9bda065e0fc88dbd9b114cc47a32189174",
|
|
221
221
|
"spec/v1/extensions.json": "79a60754aa16cbdbdb604f8e5c00038af535a1690a4d07bd5ff7c38f0256a7bc",
|
|
222
|
-
"spec/v1/gaps.json": "
|
|
222
|
+
"spec/v1/gaps.json": "c590927affc3d909b5096faf341995100e17717d9e302bc9f00dd1729353de45",
|
|
223
223
|
"spec/v1/gaps.schema.json": "8fd83259f556553c9df0f53e7a82ca8c2a4771d471197f69b4e2ae0ceeacfd66",
|
|
224
224
|
"spec/v1/migrations.json": "4c50760be70cb644d02427bcaf5749606700edf4513f423903cf3b1d12f5c300",
|
|
225
225
|
"spec/v1/migrations.schema.json": "886779aa6c22e646db097f5df210adb018a4dd14a7b815465a18c8a7056c8f72",
|
|
@@ -318,9 +318,9 @@
|
|
|
318
318
|
"spec/v2/path-manifest.json": "20f598f56df460ad3ed50aba3d6bc2933ea16a3c96ce125da1c5dd6989d4155c",
|
|
319
319
|
"spec/v2/peer-dependency-aliases.json": "d10299280abee08258502925bc327293ee413e0108cd6e6ec75ff6110653308d",
|
|
320
320
|
"spec/v2/profiles.json": "b382b261c7844798575f7c1b37c934f0da19846016a2337d883d4f5ef6e05479",
|
|
321
|
-
"spec/v2/release.json": "
|
|
321
|
+
"spec/v2/release.json": "6452f97c5ee21eddfce6f69085a8c4828d39d8337f0a6b4830025ecb3c6abc98",
|
|
322
322
|
"spec/v2/retention-floors.json": "eaf3722d95c79947af1d4269ef85117e126518c588cfcf1a2b21b97269f51624",
|
|
323
|
-
"spec/v2/surface-baseline.json": "
|
|
323
|
+
"spec/v2/surface-baseline.json": "aaef2430679e649dc80e1eb961a0e6900cea84ce2d8999a45ef58ca9911fcb3b"
|
|
324
324
|
},
|
|
325
|
-
"corpusCommit": "
|
|
325
|
+
"corpusCommit": "7d8354c0ba855f4d721cdaa6ae34ace611650353"
|
|
326
326
|
}
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
* agent calls a model, which the fixture contract leaves to the host.
|
|
13
13
|
*
|
|
14
14
|
* `efficiency` is optional on a summary entry, so a run whose entries carry
|
|
15
|
-
* none is not evidence either way (`applicable: false`).
|
|
15
|
+
* none is not evidence either way (`applicable: false`). The never-gated rule
|
|
16
|
+
* (`judgeNeverGated`) holds whether or not a host reports `efficiency`.
|
|
16
17
|
*
|
|
17
18
|
* @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md §C
|
|
18
19
|
*/
|
|
@@ -56,3 +57,111 @@ export function judgeToolCallCount(summary: Rec, tasks: readonly ScriptedTask[],
|
|
|
56
57
|
}
|
|
57
58
|
return { applicable: true, findings };
|
|
58
59
|
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* The `provider.usage` payloads of the agent under test, in log order.
|
|
63
|
+
* `provider.usage` carries no `agentId`, so attribution is positional: a usage
|
|
64
|
+
* event counts while an `agent.invocation.started` of `agentId` is open and no
|
|
65
|
+
* other agent's invocation (a judge, a delegate) is open inside it. Brackets
|
|
66
|
+
* are emitted only by an `agents.liveRuntime` host; `bracketed: false` means
|
|
67
|
+
* the log has none, and the usage list is then every `provider.usage` of the run.
|
|
68
|
+
*/
|
|
69
|
+
export function agentUsage(events: readonly { readonly type: string; readonly payload: Rec }[], agentId: string): { bracketed: boolean; usage: Rec[] } {
|
|
70
|
+
const open = new Map<string, boolean>();
|
|
71
|
+
let mine = 0;
|
|
72
|
+
let other = 0;
|
|
73
|
+
let bracketed = false;
|
|
74
|
+
const inside: Rec[] = [];
|
|
75
|
+
const all: Rec[] = [];
|
|
76
|
+
for (const e of events) {
|
|
77
|
+
const id = e.payload['invocationId'];
|
|
78
|
+
if (e.type === 'agent.invocation.started' && typeof id === 'string') {
|
|
79
|
+
bracketed = true;
|
|
80
|
+
const isMine = e.payload['agentId'] === agentId;
|
|
81
|
+
open.set(id, isMine);
|
|
82
|
+
if (isMine) mine += 1; else other += 1;
|
|
83
|
+
} else if (e.type === 'agent.invocation.completed' && typeof id === 'string' && open.has(id)) {
|
|
84
|
+
if (open.get(id)) mine -= 1; else other -= 1;
|
|
85
|
+
open.delete(id);
|
|
86
|
+
} else if (e.type === 'provider.usage') {
|
|
87
|
+
all.push(e.payload);
|
|
88
|
+
if (mine > 0 && other === 0) inside.push(e.payload);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return bracketed ? { bracketed, usage: inside } : { bracketed, usage: all };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const FIELDS = [['modelCallCount', () => 1], ['inputTokens', (u: Rec) => u['inputTokens']], ['outputTokens', (u: Rec) => u['outputTokens']]] as const;
|
|
95
|
+
const close = (a: number, b: number): boolean => Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(b));
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* §C `modelCallCount`, `inputTokens` and `outputTokens`. Each is a mean over
|
|
99
|
+
* `trials` identical trials, so the per-task values times `trials` sum to the
|
|
100
|
+
* agent's usage in the log. With brackets the sums MUST be equal; without them
|
|
101
|
+
* the run's usage also holds any host-internal call, so the sums MUST NOT
|
|
102
|
+
* exceed it. A field is judged only when every task reports it. A run with no
|
|
103
|
+
* usage and zero counts witnesses nothing (`applicable: false`).
|
|
104
|
+
*/
|
|
105
|
+
export function judgeModelUsage(summary: Rec, taskIds: readonly string[], log: { bracketed: boolean; usage: readonly Rec[] }, trials: number): EfficiencyVerdict {
|
|
106
|
+
const entries = (Array.isArray(summary['tasks']) ? (summary['tasks'] as unknown[]) : []).filter(isRec);
|
|
107
|
+
const effs = taskIds.map((id) => entries.find((e) => e['taskId'] === id)?.['efficiency']);
|
|
108
|
+
const findings: EfficiencyFinding[] = [];
|
|
109
|
+
let witnessed = false;
|
|
110
|
+
for (const [field, of] of FIELDS) {
|
|
111
|
+
const vals = effs.map((x) => (isRec(x) ? x[field] : undefined));
|
|
112
|
+
if (!vals.every((v) => typeof v === 'number')) continue;
|
|
113
|
+
const reported = (vals as number[]).reduce((a, b) => a + b, 0) * trials;
|
|
114
|
+
const inLog = log.usage.reduce((a, u) => a + (typeof of(u) === 'number' ? (of(u) as number) : 0), 0);
|
|
115
|
+
if (reported === 0 && inLog === 0) continue;
|
|
116
|
+
witnessed = true;
|
|
117
|
+
findings.push(log.bracketed
|
|
118
|
+
? { ok: close(reported, inLog), message: `the per-task efficiency.${field} values times ${trials} trial(s) MUST sum to the ${inLog} the agent's provider.usage events inside its invocation brackets carry (got ${reported})` }
|
|
119
|
+
: { ok: reported <= inLog + 1e-9 * Math.max(1, inLog), message: `the per-task efficiency.${field} values times ${trials} trial(s) MUST NOT exceed the ${inLog} the run's provider.usage events carry (got ${reported})` });
|
|
120
|
+
}
|
|
121
|
+
return witnessed ? { applicable: true, findings } : { applicable: false, findings: [] };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* §B "cost counts every trial": on a multi-trial run the summary's
|
|
126
|
+
* `totalCostUsd` holds the `costEstimateUsd` of the agent's usage in every
|
|
127
|
+
* trial, so a host that priced only one trial reads short. Only that floor is
|
|
128
|
+
* judged: whether a judge's or the host's own calls are priced in is not
|
|
129
|
+
* pinned, so a higher total is not a failure. Judged only
|
|
130
|
+
* with brackets (otherwise the agent's usage cannot be told from the host's
|
|
131
|
+
* own) and only when every attributed usage event carries a cost.
|
|
132
|
+
*/
|
|
133
|
+
export function judgeCostEveryTrial(summary: Rec, log: { bracketed: boolean; usage: readonly Rec[] }): EfficiencyVerdict {
|
|
134
|
+
const total = summary['totalCostUsd'];
|
|
135
|
+
if (typeof total !== 'number' || !log.bracketed || log.usage.length === 0) return { applicable: false, findings: [] };
|
|
136
|
+
const costs = log.usage.map((u) => u['costEstimateUsd']);
|
|
137
|
+
if (!costs.every((c) => typeof c === 'number')) return { applicable: false, findings: [] };
|
|
138
|
+
const sum = (costs as number[]).reduce((a, b) => a + b, 0);
|
|
139
|
+
return { applicable: true, findings: [{ ok: total >= sum || close(total, sum), message: `totalCostUsd MUST count every trial: it MUST be at least the ${sum} costEstimateUsd of the agent's ${log.usage.length} provider.usage events across all trials (got ${total})` }] };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* §C "MUST NOT fail a task or a suite on" efficiency, on
|
|
144
|
+
* `conformance-eval-efficiency-ungated`: `lean` and `busy` share an output and
|
|
145
|
+
* an expectation and differ only in their scripted call counts, so efficiency
|
|
146
|
+
* is the only thing that can tell them apart. `busy` MUST pass and score what
|
|
147
|
+
* `lean` scores, and the suite (`passScore` 1) MUST pass. The verdict is
|
|
148
|
+
* `blocked` unless the log carries every scripted call (otherwise the
|
|
149
|
+
* difference was never produced) and `lean` passes (otherwise a failing `busy`
|
|
150
|
+
* could have another cause).
|
|
151
|
+
*/
|
|
152
|
+
export function judgeNeverGated(summary: Rec, lean: ScriptedTask, busy: ScriptedTask, toolCallsInLog: number): { blocked: string } | EfficiencyVerdict {
|
|
153
|
+
const want = lean.scriptLength + busy.scriptLength;
|
|
154
|
+
if (toolCallsInLog !== want) return { blocked: `the log carries ${toolCallsInLog} agent.tool-called events, not the ${want} scripted calls, so the two tasks' efficiency never differed` };
|
|
155
|
+
const entries = (Array.isArray(summary['tasks']) ? (summary['tasks'] as unknown[]) : []).filter(isRec);
|
|
156
|
+
const l = entries.find((e) => e['taskId'] === lean.taskId);
|
|
157
|
+
const b = entries.find((e) => e['taskId'] === busy.taskId);
|
|
158
|
+
if (l?.['passed'] !== true) return { blocked: `the control task ${lean.taskId} did not pass, so a failing ${busy.taskId} could have a cause other than efficiency` };
|
|
159
|
+
return {
|
|
160
|
+
applicable: true,
|
|
161
|
+
findings: [
|
|
162
|
+
{ ok: b?.['passed'] === true, message: `task ${busy.taskId} (${busy.scriptLength} tool calls) MUST pass like ${lean.taskId} (${lean.scriptLength}): a host MUST NOT fail a task on efficiency (got passed ${String(b?.['passed'])})` },
|
|
163
|
+
{ ok: typeof b?.['score'] === 'number' && b['score'] === l['score'], message: `task ${busy.taskId} MUST score what ${lean.taskId} scores (${String(l['score'])}): efficiency is informational (got ${String(b?.['score'])})` },
|
|
164
|
+
{ ok: summary['passed'] === true, message: `the suite MUST pass at passScore 1 when both tasks pass: a host MUST NOT fail a suite on efficiency (got passed ${String(summary['passed'])})` },
|
|
165
|
+
],
|
|
166
|
+
};
|
|
167
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0244 §B, G3 — trial isolation: each trial starts from the same
|
|
3
|
+
* `memorySeed` snapshot, and memory one trial writes is not visible to another
|
|
4
|
+
* trial of the same eval run. The suite-owned memory tool
|
|
5
|
+
* `conformance.eval:memory` makes both observable from outside. A pure judge
|
|
6
|
+
* over one eval run of `conformance-eval-memory-isolation`, for
|
|
7
|
+
* `v2-eval-trajectory-trials.test.ts`.
|
|
8
|
+
*
|
|
9
|
+
* The tool reads and writes the trial's own agent memory, the store
|
|
10
|
+
* `memorySeed` seeds (contract in `fixtures.md` §Eval-suite fixtures):
|
|
11
|
+
* `write {id, content}` stores an entry, `read {id}` returns `{id, content}`,
|
|
12
|
+
* with `content: null` for an entry the trial's memory does not hold. Every
|
|
13
|
+
* trial runs the same script:
|
|
14
|
+
* 1. read `seeded` — the seed, in every trial;
|
|
15
|
+
* 2. read `trial-note` — nothing yet, unless another trial's write leaked;
|
|
16
|
+
* 3. write `trial-note` = `written`;
|
|
17
|
+
* 4. read `trial-note` — `written`, the positive control.
|
|
18
|
+
*
|
|
19
|
+
* Trials may run concurrently (scheduling is host-internal), so the judge
|
|
20
|
+
* never groups calls by trial. Over n trials a conforming host returns n
|
|
21
|
+
* `null` and n `written` reads of `trial-note`. A leak turns a step-2 `null`
|
|
22
|
+
* into `written`, so the run reads fewer than n `null`. A host whose writes
|
|
23
|
+
* never read back (fewer than n `written`) shows nothing about isolation, and
|
|
24
|
+
* the verdict is `blocked`, never a pass.
|
|
25
|
+
*
|
|
26
|
+
* @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md §B
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
export const MEMORY_TOOL = 'conformance.eval:memory';
|
|
30
|
+
export const SEEDED = { id: 'seeded', content: 'seed' } as const;
|
|
31
|
+
export const NOTE = { id: 'trial-note', content: 'written' } as const;
|
|
32
|
+
|
|
33
|
+
export interface MemoryFinding { readonly ok: boolean; readonly message: string }
|
|
34
|
+
export type MemoryVerdict =
|
|
35
|
+
| { readonly kind: 'blocked'; readonly reason: string }
|
|
36
|
+
| { readonly kind: 'judged'; readonly findings: readonly MemoryFinding[] };
|
|
37
|
+
|
|
38
|
+
type Rec = Record<string, unknown>;
|
|
39
|
+
export interface MemoryEvent { readonly type: string; readonly payload: Rec }
|
|
40
|
+
export interface MemoryCall { readonly op: unknown; readonly id: unknown; readonly outcome: unknown; readonly returned: boolean }
|
|
41
|
+
|
|
42
|
+
const isRec = (v: unknown): v is Rec => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
43
|
+
|
|
44
|
+
/** The memory-tool calls of one run, in log order, each paired with its return by `callId`. */
|
|
45
|
+
export function memoryCalls(events: readonly MemoryEvent[], agentId: string): MemoryCall[] {
|
|
46
|
+
const returns = new Map<string, Rec>();
|
|
47
|
+
for (const e of events) {
|
|
48
|
+
if (e.type === 'agent.tool-returned' && e.payload['toolName'] === MEMORY_TOOL && typeof e.payload['callId'] === 'string') returns.set(e.payload['callId'], e.payload);
|
|
49
|
+
}
|
|
50
|
+
const out: MemoryCall[] = [];
|
|
51
|
+
for (const e of events) {
|
|
52
|
+
if (e.type !== 'agent.tool-called' || e.payload['toolName'] !== MEMORY_TOOL || e.payload['agentId'] !== agentId) continue;
|
|
53
|
+
const inputs = isRec(e.payload['inputs']) ? e.payload['inputs'] : {};
|
|
54
|
+
const r = typeof e.payload['callId'] === 'string' ? returns.get(e.payload['callId']) : undefined;
|
|
55
|
+
out.push({ op: inputs['op'], id: inputs['id'], outcome: r?.['outcome'], returned: r !== undefined });
|
|
56
|
+
}
|
|
57
|
+
return out;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const content = (c: MemoryCall): unknown => (isRec(c.outcome) ? c.outcome['content'] : undefined);
|
|
61
|
+
|
|
62
|
+
/** §B isolation over `trials` trials of the four-step script. */
|
|
63
|
+
export function judgeTrialMemoryIsolated(calls: readonly MemoryCall[], trials: number): MemoryVerdict {
|
|
64
|
+
if (calls.length !== 4 * trials || !calls.every((c) => c.returned)) {
|
|
65
|
+
return { kind: 'blocked', reason: `the run MUST log the ${4 * trials} scripted memory calls (four per trial), each with an agent.tool-returned (got ${calls.length}, ${calls.filter((c) => c.returned).length} returned)` };
|
|
66
|
+
}
|
|
67
|
+
const reads = calls.filter((c) => c.op === 'read');
|
|
68
|
+
const seeded = reads.filter((c) => c.id === SEEDED.id);
|
|
69
|
+
const note = reads.filter((c) => c.id === NOTE.id);
|
|
70
|
+
const written = note.filter((c) => content(c) === NOTE.content).length;
|
|
71
|
+
if (written < trials) {
|
|
72
|
+
return { kind: 'blocked', reason: `only ${written} of ${trials} trials read back their own write of ${NOTE.id}, so the memory tool does not reach the trial's memory (fixtures.md memory contract) and isolation is not observable` };
|
|
73
|
+
}
|
|
74
|
+
const empty = note.filter((c) => content(c) === null).length;
|
|
75
|
+
return {
|
|
76
|
+
kind: 'judged',
|
|
77
|
+
findings: [
|
|
78
|
+
{ ok: seeded.length === trials && seeded.every((c) => content(c) === SEEDED.content), message: `every trial MUST start from the memorySeed snapshot: each of the ${trials} reads of ${SEEDED.id} MUST return content "${SEEDED.content}" (got ${JSON.stringify(seeded.map(content))})` },
|
|
79
|
+
{ ok: empty === trials && note.length === 2 * trials, message: `memory one trial writes MUST NOT be visible to another trial: each trial's first read of ${NOTE.id} MUST find nothing, so ${trials} of the ${note.length} reads return content null (got ${empty})` },
|
|
80
|
+
],
|
|
81
|
+
};
|
|
82
|
+
}
|
|
@@ -36,8 +36,29 @@
|
|
|
36
36
|
* length (the mean over identical trials), and on the one-trial run the
|
|
37
37
|
* counts sum to the log's `agent.tool-called` events. `efficiency` is
|
|
38
38
|
* optional, so a host that reports none records `inapplicable`. Judge in
|
|
39
|
-
* `lib/eval-efficiency-witness.ts`.
|
|
40
|
-
*
|
|
39
|
+
* `lib/eval-efficiency-witness.ts`.
|
|
40
|
+
* 6d. §C never-gated (2.46.27), `conformance-eval-efficiency-ungated`: two
|
|
41
|
+
* tasks with one output and one expectation, `lean` (1 scripted call) and
|
|
42
|
+
* `busy` (8). `busy` passes and scores what `lean` scores, and the suite
|
|
43
|
+
* passes at `passScore` 1 (`0244.efficiency-never-gated`). `blocked` when
|
|
44
|
+
* the calls never ran or `lean` itself fails. Gated on `agents.evalSuite`
|
|
45
|
+
* and the fixture only: the rule binds whether or not a host reports
|
|
46
|
+
* `efficiency`.
|
|
47
|
+
* 6e. §B trial isolation (2.46.27, G3), `conformance-eval-memory-isolation`,
|
|
48
|
+
* gated on `trials`: two trials of read seed, read note, write note, read
|
|
49
|
+
* note through the reserved memory tool `conformance.eval:memory`. Every
|
|
50
|
+
* trial reads the seed, and each trial's first note read finds nothing
|
|
51
|
+
* (`0244.trial-memory-isolated`); a tool whose writes never read back is
|
|
52
|
+
* `blocked`. Judge in `lib/eval-memory-witness.ts`.
|
|
53
|
+
* 6c. §C model usage and §B cost (2.46.26), on the same runs. The per-task
|
|
54
|
+
* `modelCallCount`, `inputTokens` and `outputTokens`, times the trial
|
|
55
|
+
* count, sum to the agent's `provider.usage` inside its
|
|
56
|
+
* `agent.invocation.*` brackets (judge and delegate calls excluded), or do
|
|
57
|
+
* not exceed the run's usage when the host emits no brackets
|
|
58
|
+
* (`0244.efficiency-model-usage`). On the trials run, `totalCostUsd` is
|
|
59
|
+
* at least the agent's priced usage over all three trials
|
|
60
|
+
* (`0244.cost-every-trial`). The scripted agent need not call a model, so
|
|
61
|
+
* a run with no usage, no brackets or no prices records `inapplicable`.
|
|
41
62
|
* 6b. §D through the counter tool `conformance.eval:counter` (2.46.25, G2),
|
|
42
63
|
* gated on `fixtureMatching` and both counter fixtures in discovery.
|
|
43
64
|
* Three sequential runs: `conformance-eval-counter-live` (A), then
|
|
@@ -100,7 +121,8 @@ import { judgeEvalEnvelope, judgeSummaryServed } from '../lib/eval-envelope-witn
|
|
|
100
121
|
import { gradeTrajectory, passAtK, passHatK, scoreStderr, withinTolerance, type TrajectorySpec } from '../lib/eval-trajectory.js';
|
|
101
122
|
import { judgeMeanGated, judgeSuiteGated } from '../lib/eval-gate-witness.js';
|
|
102
123
|
import { judgeFixtureErrorReplayed } from '../lib/eval-fixture-error-witness.js';
|
|
103
|
-
import { judgeToolCallCount } from '../lib/eval-efficiency-witness.js';
|
|
124
|
+
import { agentUsage, judgeCostEveryTrial, judgeModelUsage, judgeNeverGated, judgeToolCallCount } from '../lib/eval-efficiency-witness.js';
|
|
125
|
+
import { judgeTrialMemoryIsolated, memoryCalls } from '../lib/eval-memory-witness.js';
|
|
104
126
|
import { counterCalls, judgeFirstUnconsumed, judgeLiveMiss, judgeMissNoLiveCall, type CounterCall } from '../lib/eval-counter-witness.js';
|
|
105
127
|
|
|
106
128
|
const PROFILE = majorProfile(2);
|
|
@@ -117,6 +139,10 @@ const ID_TRIAL_STATS = 'openwop.requirement.0244.trial-stats-formula';
|
|
|
117
139
|
const ID_MISS = 'openwop.requirement.0244.fixture-miss-fails';
|
|
118
140
|
const ID_REFUSED = 'openwop.requirement.0244.unadvertised-feature-refused';
|
|
119
141
|
const ID_EFFICIENCY = 'openwop.requirement.0244.efficiency-tool-calls';
|
|
142
|
+
const ID_MODEL_USAGE = 'openwop.requirement.0244.efficiency-model-usage';
|
|
143
|
+
const ID_COST_TRIALS = 'openwop.requirement.0244.cost-every-trial';
|
|
144
|
+
const ID_NEVER_GATED = 'openwop.requirement.0244.efficiency-never-gated';
|
|
145
|
+
const ID_MEMORY_ISOLATED = 'openwop.requirement.0244.trial-memory-isolated';
|
|
120
146
|
const ID_LIVE_MISS = 'openwop.requirement.0244.live-miss-counted';
|
|
121
147
|
const ID_FIRST_UNCONSUMED = 'openwop.requirement.0244.first-unconsumed';
|
|
122
148
|
const ID_NO_LIVE_CALL = 'openwop.requirement.0244.miss-no-live-call';
|
|
@@ -213,7 +239,7 @@ describe('v2 RFC 0244 — wire shape (server-free)', () => {
|
|
|
213
239
|
});
|
|
214
240
|
|
|
215
241
|
it('the eval-suite fixtures validate against the v2 suite schema', () => {
|
|
216
|
-
for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss', 'conformance-eval-trajectory-gate', 'conformance-eval-trajectory-gate-mean', 'conformance-eval-fixture-error', 'conformance-eval-counter-live', 'conformance-eval-counter-miss']) {
|
|
242
|
+
for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss', 'conformance-eval-trajectory-gate', 'conformance-eval-trajectory-gate-mean', 'conformance-eval-fixture-error', 'conformance-eval-counter-live', 'conformance-eval-counter-miss', 'conformance-eval-efficiency-ungated', 'conformance-eval-memory-isolation']) {
|
|
217
243
|
const r = suiteSchema(fixture(id));
|
|
218
244
|
expect(r.ok, req(ID_FIXTURES, 'fixtures.md §eval-suites', `${id} MUST validate: ${r.errors}`)).toBe(true);
|
|
219
245
|
}
|
|
@@ -346,6 +372,56 @@ describe('v2 RFC 0244 — host behaviour', () => {
|
|
|
346
372
|
for (const v of verdicts) for (const x of v.findings) expect(x.ok, req(ID_EFFICIENCY, 'eval-summary.schema.json efficiency (RFC 0244 §C)', x.message)).toBe(true);
|
|
347
373
|
});
|
|
348
374
|
|
|
375
|
+
it('efficiency model and token counts match the agent\'s provider.usage (§C)', async () => {
|
|
376
|
+
const r = await trajectoryRun();
|
|
377
|
+
if ('skip' in r) return softSkip(...r.skip);
|
|
378
|
+
const ids = (s: Suite) => s.tasks.map((t) => t.taskId);
|
|
379
|
+
const verdicts = [judgeModelUsage(r.o.summary, ids(fixture('conformance-eval-trajectory')), agentUsage(r.o.events, AGENT), 1)];
|
|
380
|
+
const t = await trialsRun();
|
|
381
|
+
if (!('skip' in t)) verdicts.push(judgeModelUsage(t.o.summary, ids(t.suite), agentUsage(t.o.events, AGENT), t.n));
|
|
382
|
+
if (!verdicts.some((v) => v.applicable)) return softSkip('inapplicable', 'no run carries both a reported model count and provider.usage to check it against (the scripted agent need not call a model; RFC 0244 §C fields are optional)');
|
|
383
|
+
for (const v of verdicts) for (const x of v.findings) expect(x.ok, req(ID_MODEL_USAGE, 'eval-summary.schema.json efficiency (RFC 0244 §C)', x.message)).toBe(true);
|
|
384
|
+
});
|
|
385
|
+
|
|
386
|
+
it('totalCostUsd counts every trial (§B)', async () => {
|
|
387
|
+
const t = await trialsRun();
|
|
388
|
+
if ('skip' in t) return softSkip(...t.skip);
|
|
389
|
+
const v = judgeCostEveryTrial(t.o.summary, agentUsage(t.o.events, AGENT));
|
|
390
|
+
if (!v.applicable) return softSkip('inapplicable', 'the trials run carries no totalCostUsd, no agent.invocation.* brackets, or an unpriced provider.usage, so the per-trial cost is not observable');
|
|
391
|
+
for (const x of v.findings) expect(x.ok, req(ID_COST_TRIALS, 'agent-eval-suite.schema.json thresholds.maxCostUsd, eval-summary.schema.json totalCostUsd (RFC 0244 §B)', x.message)).toBe(true);
|
|
392
|
+
});
|
|
393
|
+
|
|
394
|
+
it('a host does not fail a task or a suite on efficiency (§C)', async () => {
|
|
395
|
+
const { doc, facet } = await evalSuiteFacet();
|
|
396
|
+
if (!doc) return softSkip('blocked', UNREADABLE);
|
|
397
|
+
if (facet === null) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite');
|
|
398
|
+
const id = 'conformance-eval-efficiency-ungated';
|
|
399
|
+
if (!isFixtureAdvertised(id)) return softSkip('inapplicable', `the host does not advertise fixture ${id}`);
|
|
400
|
+
const o = await runSuite(id);
|
|
401
|
+
if (o.kind === 'skip') return softSkip(o.disposition, o.reason);
|
|
402
|
+
const task = (taskId: string) => ({ taskId, scriptLength: fixture(id).tasks.find((t) => t.taskId === taskId)!.input.script.length });
|
|
403
|
+
const logCalls = o.events.filter((e) => e.type === 'agent.tool-called' && e.payload['agentId'] === AGENT).length;
|
|
404
|
+
const v = judgeNeverGated(o.summary, task('lean'), task('busy'), logCalls);
|
|
405
|
+
if ('blocked' in v) return softSkip('blocked', v.blocked);
|
|
406
|
+
for (const x of v.findings) expect(x.ok, req(ID_NEVER_GATED, 'eval-summary.schema.json efficiency (RFC 0244 §C)', x.message)).toBe(true);
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
it('memory one trial writes is not visible to another trial (§B)', async () => {
|
|
410
|
+
const { doc, facet } = await evalSuiteFacet();
|
|
411
|
+
if (!doc) return softSkip('blocked', UNREADABLE);
|
|
412
|
+
const trials = facet?.['trials'];
|
|
413
|
+
if (!isRecord(trials)) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite.trials');
|
|
414
|
+
const id = 'conformance-eval-memory-isolation';
|
|
415
|
+
const n = fixture(id).trials!.count;
|
|
416
|
+
if (typeof trials['maxTrials'] === 'number' && trials['maxTrials'] < n) return softSkip('inapplicable', `the host advertises trials.maxTrials ${trials['maxTrials']}, below the fixture's ${n}`);
|
|
417
|
+
if (!isFixtureAdvertised(id)) return softSkip('inapplicable', `the host does not advertise fixture ${id}, its opt-in to the memory-tool contract`);
|
|
418
|
+
const o = await runSuite(id);
|
|
419
|
+
if (o.kind === 'skip') return softSkip(o.disposition, o.reason);
|
|
420
|
+
const v = judgeTrialMemoryIsolated(memoryCalls(o.events, AGENT), n);
|
|
421
|
+
if (v.kind === 'blocked') return softSkip('blocked', v.reason);
|
|
422
|
+
for (const x of v.findings) expect(x.ok, req(ID_MEMORY_ISOLATED, 'agent-eval-suite.schema.json trials, fixtures.md memory contract (RFC 0244 §B)', x.message)).toBe(true);
|
|
423
|
+
});
|
|
424
|
+
|
|
349
425
|
type CounterRuns = { skip: [Parameters<typeof softSkip>[0], string] } | { a: CounterCall[]; b: CounterCall[]; c: CounterCall[]; aMisses: unknown };
|
|
350
426
|
let counterSeq: Promise<CounterRuns> | undefined;
|
|
351
427
|
/** A → B → C, strictly in order, once per worker: the three counter legs read the same runs. */
|