@openwop/openwop-conformance 2.46.11 → 2.46.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/spec-artifacts.lock.json +2 -2
- package/fixtures/eval-suites/conformance-eval-trajectory-gate-mean.json +54 -0
- package/fixtures/eval-suites/conformance-eval-trajectory-gate.json +81 -0
- package/fixtures.md +4 -0
- package/package.json +2 -2
- package/requirements.json +61 -11
- package/schemas/CORPUS-STAMP.json +17 -16
- package/src/lib/eval-gate-witness.ts +102 -0
- package/src/scenarios/v2-eval-trajectory-trials.test.ts +51 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,19 @@
|
|
|
1
|
+
## [2.46.13] — 2026-10-16 — RFC 0246: a trajectory-failed trial cannot carry a pass (safety fix)
|
|
2
|
+
|
|
3
|
+
### Security
|
|
4
|
+
|
|
5
|
+
- **RFC 0246 (`GHSA-ffx7-pm5h-v6pq`): the eval pass decisions read a gating score.** Three new legs in `v2-eval-trajectory-trials`, judged by the new `lib/eval-gate-witness.ts`:
|
|
6
|
+
- `0246.gated-fields-shape` (server-free): `gatedScore` and `gatedAggregateScore` validate in 0–1.
|
|
7
|
+
- `0246.mean-gated`, on the new `conformance-eval-trajectory-gate-mean` (trajectory + trials): under `passRule: "mean"` a task whose two trials called a forbidden tool keeps `meanScore: 1`, carries `gatedScore: 0` and does not pass.
|
|
8
|
+
- `0246.suite-gated`, on the new `conformance-eval-trajectory-gate` (trajectory): `aggregateScore` 1 clears `passScore` 0.6, `gatedAggregateScore` 0.5 does not, and the suite does not pass.
|
|
9
|
+
- Both host legs also fail a host that zeroes the reported `score` instead (RFC 0246 §C).
|
|
10
|
+
- **What a host's next bundle shows:** the fixtures are opt-in through discovery `fixtures`, so a host that does not list them records both host legs `inapplicable`. A host implementing RFC 0244 trajectories that lists them and has not adopted the gate records both `executed-fail`.
|
|
11
|
+
|
|
12
|
+
## [2.46.12] — 2026-10-16 — RFC 0245 and RFC 0247 Active
|
|
13
|
+
|
|
14
|
+
- **No assertion changes yet.** RFC 0245 (eval fixtures replay a recorded tool error as an error) is `Active`: the packed v2 `agent-eval-suite` schema gains optional `toolResponses[].isError`, the capability schema gains `agents.evalSuite.fixtureErrors`, and three gap rows change the packed `spec/v1/gaps.json`. The fixture and the two gated legs follow.
|
|
15
|
+
- **RFC 0247 `Active`: bounded stream lifetime.** The packed v2 surfaces gain the `resumable` query parameter on `streamRunEvents` and `streamHostEvents`, and the limits facet gains `maxStreamAgeSec`. No new leg (gap G1: a bounded close needs a long-held live run). A host that advertises nothing new, and a consumer that never sends `resumable=true`, see no change.
|
|
16
|
+
|
|
1
17
|
## [2.46.11] — 2026-10-15 — a shared id reads the same on every run
|
|
2
18
|
|
|
3
19
|
- **Ledger reader: one requirement id recorded by two scenario files.** Each file runs in its own worker, and `readLedgerFile` kept whichever line came last, so the row's `assertions` and `scenario` belonged to the worker that finished last. openwop-app's `prompts.resolution-precedes-composition` read 9 then 5 assertions on identical results.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
{
|
|
2
|
+
"suiteId": "core.openwop.evals.conformance-eval-trajectory-gate-mean",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"targetAgentId": "core.conformance.agents.scripted",
|
|
5
|
+
"modes": [
|
|
6
|
+
"golden"
|
|
7
|
+
],
|
|
8
|
+
"thresholds": {
|
|
9
|
+
"passScore": 0.6
|
|
10
|
+
},
|
|
11
|
+
"trials": {
|
|
12
|
+
"count": 2,
|
|
13
|
+
"k": 2,
|
|
14
|
+
"passRule": "mean"
|
|
15
|
+
},
|
|
16
|
+
"tasks": [
|
|
17
|
+
{
|
|
18
|
+
"taskId": "path-wrong",
|
|
19
|
+
"input": {
|
|
20
|
+
"script": [
|
|
21
|
+
{
|
|
22
|
+
"tool": "conformance.eval:lookup",
|
|
23
|
+
"arguments": {
|
|
24
|
+
"orderId": "o-1"
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"tool": "conformance.eval:refund-all",
|
|
29
|
+
"arguments": {}
|
|
30
|
+
}
|
|
31
|
+
],
|
|
32
|
+
"output": "done"
|
|
33
|
+
},
|
|
34
|
+
"expected": {
|
|
35
|
+
"kind": "golden",
|
|
36
|
+
"match": {
|
|
37
|
+
"strategy": "contains",
|
|
38
|
+
"value": "done"
|
|
39
|
+
},
|
|
40
|
+
"trajectory": {
|
|
41
|
+
"match": "any-order",
|
|
42
|
+
"calls": [
|
|
43
|
+
{
|
|
44
|
+
"tool": "conformance.eval:lookup"
|
|
45
|
+
}
|
|
46
|
+
],
|
|
47
|
+
"forbidden": [
|
|
48
|
+
"conformance.eval:refund-all"
|
|
49
|
+
]
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
]
|
|
54
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
{
|
|
2
|
+
"suiteId": "core.openwop.evals.conformance-eval-trajectory-gate",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"targetAgentId": "core.conformance.agents.scripted",
|
|
5
|
+
"modes": [
|
|
6
|
+
"golden"
|
|
7
|
+
],
|
|
8
|
+
"thresholds": {
|
|
9
|
+
"passScore": 0.6
|
|
10
|
+
},
|
|
11
|
+
"tasks": [
|
|
12
|
+
{
|
|
13
|
+
"taskId": "path-matched",
|
|
14
|
+
"input": {
|
|
15
|
+
"script": [
|
|
16
|
+
{
|
|
17
|
+
"tool": "conformance.eval:lookup",
|
|
18
|
+
"arguments": {
|
|
19
|
+
"orderId": "o-1"
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
],
|
|
23
|
+
"output": "done"
|
|
24
|
+
},
|
|
25
|
+
"expected": {
|
|
26
|
+
"kind": "golden",
|
|
27
|
+
"match": {
|
|
28
|
+
"strategy": "contains",
|
|
29
|
+
"value": "done"
|
|
30
|
+
},
|
|
31
|
+
"trajectory": {
|
|
32
|
+
"match": "any-order",
|
|
33
|
+
"calls": [
|
|
34
|
+
{
|
|
35
|
+
"tool": "conformance.eval:lookup"
|
|
36
|
+
}
|
|
37
|
+
],
|
|
38
|
+
"forbidden": [
|
|
39
|
+
"conformance.eval:refund-all"
|
|
40
|
+
]
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"taskId": "path-wrong",
|
|
46
|
+
"input": {
|
|
47
|
+
"script": [
|
|
48
|
+
{
|
|
49
|
+
"tool": "conformance.eval:lookup",
|
|
50
|
+
"arguments": {
|
|
51
|
+
"orderId": "o-1"
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"tool": "conformance.eval:refund-all",
|
|
56
|
+
"arguments": {}
|
|
57
|
+
}
|
|
58
|
+
],
|
|
59
|
+
"output": "done"
|
|
60
|
+
},
|
|
61
|
+
"expected": {
|
|
62
|
+
"kind": "golden",
|
|
63
|
+
"match": {
|
|
64
|
+
"strategy": "contains",
|
|
65
|
+
"value": "done"
|
|
66
|
+
},
|
|
67
|
+
"trajectory": {
|
|
68
|
+
"match": "any-order",
|
|
69
|
+
"calls": [
|
|
70
|
+
{
|
|
71
|
+
"tool": "conformance.eval:lookup"
|
|
72
|
+
}
|
|
73
|
+
],
|
|
74
|
+
"forbidden": [
|
|
75
|
+
"conformance.eval:refund-all"
|
|
76
|
+
]
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
]
|
|
81
|
+
}
|
package/fixtures.md
CHANGED
|
@@ -72,6 +72,8 @@ All fixtures MUST advertise:
|
|
|
72
72
|
| Eval — Trajectory | `conformance-eval-trajectory` | RFC 0244 §A — an `AgentEvalSuite` under `eval-suites/` (not a workflow), run with `mode: eval` against the scripted agent; five tasks whose tool paths match / miss each mode and a forbidden call (`v2-eval-trajectory-trials`) | eval run `completed`; per-task grade as the reference grader | ≤ 30s |
|
|
73
73
|
| Eval — Trials | `conformance-eval-trials` | RFC 0244 §B — two tasks × `trials.count: 3`, `k: 2`; one always passes, one always fails (`v2-eval-trajectory-trials`) | eval run `completed`, or `400` where `trials` is unadvertised | ≤ 30s |
|
|
74
74
|
| Eval — Fixture Miss | `conformance-eval-fixture-miss` | RFC 0244 §D — `onMiss: "fail"`; one task served by an args-matched fixture, one that misses it (`v2-eval-trajectory-trials`) | eval run `completed`; the missed task fails | ≤ 30s |
|
|
75
|
+
| Eval — Trajectory Gate | `conformance-eval-trajectory-gate` | RFC 0246 — two golden tasks scoring 1 at `passScore: 0.6`; `path-matched` keeps its trajectory, `path-wrong` calls a forbidden tool (`v2-eval-trajectory-trials`) | eval run `completed`; `aggregateScore` 1, `gatedAggregateScore` 0.5, suite `passed: false` | ≤ 30s |
|
|
76
|
+
| Eval — Trajectory Gate (mean) | `conformance-eval-trajectory-gate-mean` | RFC 0246 — `trials: {count: 2, passRule: "mean"}`, one golden task scoring 1 that calls a forbidden tool in both trials (`v2-eval-trajectory-trials`) | eval run `completed`; `meanScore` 1, `gatedScore` 0, task `passed: false` | ≤ 30s |
|
|
75
77
|
| Dispatch Fan-Out Waves | `conformance-dispatch-fanout-waves` | RFC 0243 §A.3 — a parallel `core.dispatch` (`maxConcurrency: 2`, `wait-all`/`collect`) whose supervisor's `mockDispatchPlan` names three `conformance-delay` children. At most `min(2, maxFanOut)` run at once and none is dropped. Gated on `agents.fanOut` admitting `wait-all` with `collect`. | `completed` | ≤ 30s |
|
|
76
78
|
| Dispatch Per-Item Input | `conformance-dispatch-per-item` | RFC 0243 §A.4 — a sequential `core.dispatch` whose decision dispatches `conformance-dispatch-input-mapping-child` twice with `nextWorkerInputs` `item-a`, `item-b`. At `agents.fanOut.perItemInput: true` each child receives its item; otherwise the dispatch node fails `validation_error` and no child is dispatched. | `completed`, or `failed` without `perItemInput` | ≤ 30s |
|
|
77
79
|
| Dispatch Per-Item Length Mismatch | `conformance-dispatch-per-item-mismatch` | `orchestrator-decision.schema.json` `nextWorkerInputs` — one item for two children. Every dispatch host fails the dispatch node `validation_error` and dispatches no child. | `failed` | ≤ 30s |
|
|
@@ -1030,6 +1032,8 @@ Every task targets **the scripted agent `core.conformance.agents.scripted`**. It
|
|
|
1030
1032
|
| `conformance-eval-trajectory` | `agents.evalSuite.trajectory` | each task's `trajectory` counts equal `lib/eval-trajectory.ts`'s grade of the script; an unmatched trajectory fails; a `kind: "trajectory"` task scores 1 or 0; the log carries one `agent.tool-called` per scripted call |
|
|
1031
1033
|
| `conformance-eval-trials` | `agents.evalSuite.trials` | `eval.started.trialCount: 3`; three `eval.scored` per task with `trialIndex` 0..2; the per-task `trials` block and `trialStats` equal the §B formulas. Without the facet: `400 validation_error` at `createRun` |
|
|
1032
1034
|
| `conformance-eval-fixture-miss` | `agents.evalSuite.fixtureMatching` | `missed` reports `fixtureMissCount: 1`, `passed: false` and `score: 0` (summary and every `eval.scored`); `served` passes with no miss. That no live tool ran is not observable here (RFC 0244 G2) |
|
|
1035
|
+
| `conformance-eval-trajectory-gate` | `agents.evalSuite.trajectory` | RFC 0246: both tasks keep `score` 1 and `aggregateScore` 1; `path-matched` carries `gatedScore` 1 and `path-wrong` `gatedScore` 0; `gatedAggregateScore` 0.5; suite `passed: false` although `aggregateScore` clears `passScore` 0.6 |
|
|
1036
|
+
| `conformance-eval-trajectory-gate-mean` | `agents.evalSuite.trajectory` + `agents.evalSuite.trials` | RFC 0246: two `eval.scored` with `score` 1 and `passed: false`; `trials.meanScore` 1, `gatedScore` 0, task `passed: false`; the summary carries `gatedAggregateScore` |
|
|
1033
1037
|
|
|
1034
1038
|
## OAuth provider fixtures
|
|
1035
1039
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openwop/openwop-conformance",
|
|
3
|
-
"version": "2.46.
|
|
3
|
+
"version": "2.46.13",
|
|
4
4
|
"description": "Black-box conformance suite for OpenWOP hosts (v2 current; v1 scenarios retained for the frozen v1 tree).",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -56,6 +56,6 @@
|
|
|
56
56
|
"@openwop/spec-artifacts": "file:../spec-artifacts"
|
|
57
57
|
},
|
|
58
58
|
"peerDependencies": {
|
|
59
|
-
"@openwop/spec-artifacts": "2.46.
|
|
59
|
+
"@openwop/spec-artifacts": "2.46.13"
|
|
60
60
|
}
|
|
61
61
|
}
|
package/requirements.json
CHANGED
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
"generatedFrom": "src/scenarios/*.test.ts",
|
|
4
4
|
"counts": {
|
|
5
5
|
"files": 746,
|
|
6
|
-
"tests":
|
|
7
|
-
"withStableId":
|
|
6
|
+
"tests": 2811,
|
|
7
|
+
"withStableId": 2811,
|
|
8
8
|
"interpolatedTitles": 0,
|
|
9
|
-
"explicitIds":
|
|
9
|
+
"explicitIds": 2684
|
|
10
10
|
},
|
|
11
11
|
"records": [
|
|
12
12
|
{
|
|
@@ -31028,7 +31028,7 @@
|
|
|
31028
31028
|
{
|
|
31029
31029
|
"id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-suite-summary-and-event-fields-validate-and-the-negatives-are-refus",
|
|
31030
31030
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31031
|
-
"line":
|
|
31031
|
+
"line": 132,
|
|
31032
31032
|
"title": "the RFC 0244 suite, summary and event fields validate and the negatives are refused",
|
|
31033
31033
|
"explicitId": "openwop.requirement.0244.wire-shape",
|
|
31034
31034
|
"citations": [
|
|
@@ -31067,7 +31067,7 @@
|
|
|
31067
31067
|
{
|
|
31068
31068
|
"id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-additions-are-content-free-eval-summary-no-content-leak",
|
|
31069
31069
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31070
|
-
"line":
|
|
31070
|
+
"line": 155,
|
|
31071
31071
|
"title": "the RFC 0244 additions are content-free (eval-summary-no-content-leak)",
|
|
31072
31072
|
"explicitId": "openwop.requirement.0244.content-free",
|
|
31073
31073
|
"citations": [
|
|
@@ -31084,7 +31084,7 @@
|
|
|
31084
31084
|
{
|
|
31085
31085
|
"id": "openwop.it.v2-eval-trajectory-trials.the-eval-suite-fixtures-validate-against-the-v2-suite-schema",
|
|
31086
31086
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31087
|
-
"line":
|
|
31087
|
+
"line": 162,
|
|
31088
31088
|
"title": "the eval-suite fixtures validate against the v2 suite schema",
|
|
31089
31089
|
"explicitId": "openwop.requirement.0244.fixtures-valid",
|
|
31090
31090
|
"citations": [
|
|
@@ -31095,10 +31095,32 @@
|
|
|
31095
31095
|
}
|
|
31096
31096
|
]
|
|
31097
31097
|
},
|
|
31098
|
+
{
|
|
31099
|
+
"id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0246-gating-fields-validate-and-stay-in-0-1",
|
|
31100
|
+
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31101
|
+
"line": 169,
|
|
31102
|
+
"title": "the RFC 0246 gating fields validate and stay in 0–1",
|
|
31103
|
+
"explicitId": "openwop.requirement.0246.gated-fields-shape",
|
|
31104
|
+
"citations": [
|
|
31105
|
+
{
|
|
31106
|
+
"section": "eval-summary.schema.json gatedScore / gatedAggregateScore (RFC 0246)",
|
|
31107
|
+
"requirement": null,
|
|
31108
|
+
"interpolated": true
|
|
31109
|
+
},
|
|
31110
|
+
{
|
|
31111
|
+
"section": "eval-summary.schema.json gatedScore / gatedAggregateScore (RFC 0246)",
|
|
31112
|
+
"requirement": "gatedAggregateScore above 1 MUST be refused"
|
|
31113
|
+
},
|
|
31114
|
+
{
|
|
31115
|
+
"section": "eval-summary.schema.json gatedScore / gatedAggregateScore (RFC 0246)",
|
|
31116
|
+
"requirement": "a negative gatedScore MUST be refused"
|
|
31117
|
+
}
|
|
31118
|
+
]
|
|
31119
|
+
},
|
|
31098
31120
|
{
|
|
31099
31121
|
"id": "openwop.it.v2-eval-trajectory-trials.a-trajectory-suite-is-graded-per-the-reference-grader",
|
|
31100
31122
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31101
|
-
"line":
|
|
31123
|
+
"line": 179,
|
|
31102
31124
|
"title": "a trajectory suite is graded per the reference grader",
|
|
31103
31125
|
"explicitId": "openwop.requirement.0244.trajectory-graded",
|
|
31104
31126
|
"citations": [
|
|
@@ -31132,7 +31154,7 @@
|
|
|
31132
31154
|
{
|
|
31133
31155
|
"id": "openwop.it.v2-eval-trajectory-trials.repeated-trials-emit-one-eval-scored-per-task-per-trial",
|
|
31134
31156
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31135
|
-
"line":
|
|
31157
|
+
"line": 215,
|
|
31136
31158
|
"title": "repeated trials emit one eval.scored per task per trial",
|
|
31137
31159
|
"explicitId": "openwop.requirement.0244.trials-per-task",
|
|
31138
31160
|
"citations": [
|
|
@@ -31150,7 +31172,7 @@
|
|
|
31150
31172
|
{
|
|
31151
31173
|
"id": "openwop.it.v2-eval-trajectory-trials.the-per-task-trials-block-and-trialstats-follow-the-b-formulas",
|
|
31152
31174
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31153
|
-
"line":
|
|
31175
|
+
"line": 226,
|
|
31154
31176
|
"title": "the per-task trials block and trialStats follow the §B formulas",
|
|
31155
31177
|
"explicitId": "openwop.requirement.0244.trial-stats-formula",
|
|
31156
31178
|
"citations": [
|
|
@@ -31189,7 +31211,7 @@
|
|
|
31189
31211
|
{
|
|
31190
31212
|
"id": "openwop.it.v2-eval-trajectory-trials.a-fixture-miss-under-onmiss-fail-fails-the-trial-and-is-counted",
|
|
31191
31213
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31192
|
-
"line":
|
|
31214
|
+
"line": 249,
|
|
31193
31215
|
"title": "a fixture miss under onMiss \"fail\" fails the trial and is counted",
|
|
31194
31216
|
"explicitId": "openwop.requirement.0244.fixture-miss-fails",
|
|
31195
31217
|
"citations": [
|
|
@@ -31226,7 +31248,7 @@
|
|
|
31226
31248
|
{
|
|
31227
31249
|
"id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-an-unadvertised-rfc-0244-feature-is-refused-at-createrun",
|
|
31228
31250
|
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31229
|
-
"line":
|
|
31251
|
+
"line": 269,
|
|
31230
31252
|
"title": "a suite using an unadvertised RFC 0244 feature is refused at createRun",
|
|
31231
31253
|
"explicitId": "openwop.requirement.0244.unadvertised-feature-refused",
|
|
31232
31254
|
"citations": [
|
|
@@ -31241,6 +31263,34 @@
|
|
|
31241
31263
|
}
|
|
31242
31264
|
]
|
|
31243
31265
|
},
|
|
31266
|
+
{
|
|
31267
|
+
"id": "openwop.it.v2-eval-trajectory-trials.under-passrule-mean-a-task-whose-trials-failed-their-trajectory-does-not-pass-rf",
|
|
31268
|
+
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31269
|
+
"line": 285,
|
|
31270
|
+
"title": "under passRule \"mean\" a task whose trials failed their trajectory does not pass (RFC 0246)",
|
|
31271
|
+
"explicitId": "openwop.requirement.0246.mean-gated",
|
|
31272
|
+
"citations": [
|
|
31273
|
+
{
|
|
31274
|
+
"section": null,
|
|
31275
|
+
"requirement": null,
|
|
31276
|
+
"interpolated": true
|
|
31277
|
+
}
|
|
31278
|
+
]
|
|
31279
|
+
},
|
|
31280
|
+
{
|
|
31281
|
+
"id": "openwop.it.v2-eval-trajectory-trials.a-suite-whose-tasks-failed-their-trajectory-does-not-pass-on-the-output-scores-r",
|
|
31282
|
+
"file": "v2-eval-trajectory-trials.test.ts",
|
|
31283
|
+
"line": 298,
|
|
31284
|
+
"title": "a suite whose tasks failed their trajectory does not pass on the output scores (RFC 0246)",
|
|
31285
|
+
"explicitId": "openwop.requirement.0246.suite-gated",
|
|
31286
|
+
"citations": [
|
|
31287
|
+
{
|
|
31288
|
+
"section": null,
|
|
31289
|
+
"requirement": null,
|
|
31290
|
+
"interpolated": true
|
|
31291
|
+
}
|
|
31292
|
+
]
|
|
31293
|
+
},
|
|
31244
31294
|
{
|
|
31245
31295
|
"id": "openwop.it.v2-event-type-closed.every-emitted-type-is-a-registered-protocol-type-or-a-vendor-type-under-an-adver",
|
|
31246
31296
|
"file": "v2-event-type-closed.test.ts",
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_comment": "Provenance of @openwop/spec-artifacts (RFC 0168 §D.2). files: SHA-256 per file; the conformance suite compares the installed peer against dist/spec-artifacts.lock.json at start.",
|
|
3
3
|
"package": "@openwop/spec-artifacts",
|
|
4
|
-
"version": "2.46.
|
|
5
|
-
"corpusTag": "v2.46.
|
|
4
|
+
"version": "2.46.13",
|
|
5
|
+
"corpusTag": "v2.46.13",
|
|
6
6
|
"files": {
|
|
7
7
|
"api/.redocly.lint-ignore.yaml": "bf5a8350b88a72fa43f59605ed8d903ed24b6cfccda5e45509c9f6ed9ee4e712",
|
|
8
8
|
"api/asyncapi.yaml": "d5ecb9ee6114582be3b1f662c84bfac9ae96dae7bacb853e461168f70a8e1c7d",
|
|
9
9
|
"api/grpc/openwop.proto": "c3e72bb17cba514ee98feb6434e6c9b6ea6795bfd086489ec69fd882dd1ad977",
|
|
10
10
|
"api/openapi.yaml": "ac6de5303872664b78ff419c035915593e01599e02835e777a0123de1baeb398",
|
|
11
11
|
"api/redocly.yaml": "b0604c89b2ca6d5076ec25725c539dad44a741a811fe524439ee6daef8baa09f",
|
|
12
|
-
"api/seams-v2.yaml": "
|
|
13
|
-
"api/v2/asyncapi.yaml": "
|
|
14
|
-
"api/v2/openapi.yaml": "
|
|
12
|
+
"api/seams-v2.yaml": "eaec2c50979e9b55ac575d2be5c490a3048ae86613b1c018a59c1e8e0a26f24b",
|
|
13
|
+
"api/v2/asyncapi.yaml": "656c8c4031b8ebd96ad1bd536b64456f9eaf46249311e3332d540e4fbfcb63d7",
|
|
14
|
+
"api/v2/openapi.yaml": "473b40600965a068ab54e77e7efc09e9be4ed7eff670f015158b4b68aaad1e83",
|
|
15
15
|
"api/v2/redocly.yaml": "1e66b60e6118ad11a823bb620678be464d99dfe50a40e3e6f93ec9429b88b34c",
|
|
16
16
|
"schemas/README.md": "276fc1d51eb08e14e9127b9fd27761d7046ee33c12000f3a64a801322db8bf58",
|
|
17
17
|
"schemas/a2a-task-state.schema.json": "75d5049dea8bd873ff0e7546f1c60c8d36c219bec7084264360be54a8be30ae1",
|
|
@@ -103,7 +103,7 @@
|
|
|
103
103
|
"schemas/v2/a2ui-surface-delta-frame.schema.json": "6889c5da3fe7e76cb614038c0eea6774776fa640448558b258eb87fb50cb9b64",
|
|
104
104
|
"schemas/v2/agent-deployment-transition.schema.json": "93a334ff042d51378c89fdd7134664271bf9592419bb5e3e00821ed06bc1904e",
|
|
105
105
|
"schemas/v2/agent-deployment.schema.json": "d59b43fb62b522005ee22bc3bc53775a6777493e5e42d71812abc347625fa7fe",
|
|
106
|
-
"schemas/v2/agent-eval-suite.schema.json": "
|
|
106
|
+
"schemas/v2/agent-eval-suite.schema.json": "a49ad7a232ec781512375001f8d8c70ab073b69ca094f5251dbb30a0bbd972a7",
|
|
107
107
|
"schemas/v2/agent-inventory-response.schema.json": "5f74c3da946dfc3114624567a989717748081ab947b4d2af3f49034d67f4306b",
|
|
108
108
|
"schemas/v2/agent-manifest.schema.json": "004ff150e955deebb9e64273de5093f5144b9b3895b09953e65aa74b61140778",
|
|
109
109
|
"schemas/v2/agent-org-chart.schema.json": "113b079e1dce95908467cbef7f83e076fdb82aab5b0678fc1a43e32bfc9cbadf",
|
|
@@ -117,7 +117,7 @@
|
|
|
117
117
|
"schemas/v2/artifact.schema.json": "85b9dc284d39b99d2a66fa8cbf47f2dd3d8526d3bd6cc411ad315b372de38975",
|
|
118
118
|
"schemas/v2/audit-verify-result.schema.json": "14af8e9283510f0a40e48a2059c929ee44e0fc9fa3cc66a95bfbe90b17c48297",
|
|
119
119
|
"schemas/v2/budget-policy.schema.json": "1de68095db0beb5c7a4be629de26e79f62ebbe748a88d46f36e768edfd1f0d68",
|
|
120
|
-
"schemas/v2/capabilities.schema.json": "
|
|
120
|
+
"schemas/v2/capabilities.schema.json": "87faea080e8043abd16cba19dece777a810731fc3e4455c96c02d0bfd0997d84",
|
|
121
121
|
"schemas/v2/certification-bundle.schema.json": "5cfa7fb5247160333e33a58b60d42bcd4e5269d8391d6a112d301e14828892ed",
|
|
122
122
|
"schemas/v2/channel-presence-payload.schema.json": "fc03df0c20cc12a25f7ee6a47503a4c5931d79fe3d5ff24815e8fd7c1bcb9a87",
|
|
123
123
|
"schemas/v2/channel-written-payload.schema.json": "ccecff3c71a3275ad8db035ac0e04db25e6f8d23cf091ddbcd5a60d1be1cbb6b",
|
|
@@ -145,7 +145,7 @@
|
|
|
145
145
|
"schemas/v2/envelopes/schema.response.schema.json": "d049a081267c06c28879c331b147f3ed821b14ccfd93005c67aff64db0a8c68e",
|
|
146
146
|
"schemas/v2/envelopes/ui.a2ui-surface.schema.json": "e92a2548ba65454c9e0a34f72ea6768d18f59e61fa5da7668d0b2d5baddaddc8",
|
|
147
147
|
"schemas/v2/error-envelope.schema.json": "8ddb1d80330dfef17572a74dfe3a53e1a06411e4b0af6a6f1dd6e7e7461da987",
|
|
148
|
-
"schemas/v2/eval-summary.schema.json": "
|
|
148
|
+
"schemas/v2/eval-summary.schema.json": "d07f4d116c5410997a81d0bc5387545fb61933a6333c360a26ab8a71c749336b",
|
|
149
149
|
"schemas/v2/export-bundle.schema.json": "bd28905a7729437ab7f2ad8bb33e13a371b2bf5605a7eb2962a0d246de312d69",
|
|
150
150
|
"schemas/v2/form-content-pack-manifest.schema.json": "a1942a124e250a6e9fbdb6b811c5a125e78fbcbcaa46dfc71d0636f24710a339",
|
|
151
151
|
"schemas/v2/frontend-plugin-manifest.schema.json": "b279da288d542906a72b8cb4e872e001de632eaff539e0259b9cce287f82bf8f",
|
|
@@ -208,14 +208,14 @@
|
|
|
208
208
|
"schemas/workspace-file.schema.json": "464de85c2a068243084ee9c1d969bc7cd5d8f7948574e58450d6493c38a0e1e4",
|
|
209
209
|
"spec/v1/alias-detectors.json": "22444c22bb88c568339ef512553f9fc38146e22b566d11ab8608f655d688d19c",
|
|
210
210
|
"spec/v1/capability-declaration-classes.json": "e7729aed5c4b4e1dd02abab0530f14cc95f5d4070fe51fb139e7f5cccefa00c6",
|
|
211
|
-
"spec/v1/core-standard-manifest.json": "
|
|
211
|
+
"spec/v1/core-standard-manifest.json": "1070771f6480ead748c5bf02f0cfbc397ce0009dc4dd237c58d1178360c91179",
|
|
212
212
|
"spec/v1/deprecations.json": "e723bc11d7272c71789ad1303bb7f21809a8a024abf57594f9d5c35dc199ece9",
|
|
213
213
|
"spec/v1/deprecations.schema.json": "3e393c405d2a41b467d8c5e3c468549078df1ce6a6d2588fc488097b95d9b55b",
|
|
214
214
|
"spec/v1/end-of-support.json": "35f0bf3b70fe78b16e2a2fd81a153fc27e703f6c16376c6fe7cdf1f4f2c4e6d4",
|
|
215
215
|
"spec/v1/event-codemap.json": "3da60d884157793a360da532a9fcbbfb5285636db325a74cec94b34622186d97",
|
|
216
216
|
"spec/v1/event-codemap.schema.json": "d05933b2e88103aff51a2f774df97b9bda065e0fc88dbd9b114cc47a32189174",
|
|
217
217
|
"spec/v1/extensions.json": "79a60754aa16cbdbdb604f8e5c00038af535a1690a4d07bd5ff7c38f0256a7bc",
|
|
218
|
-
"spec/v1/gaps.json": "
|
|
218
|
+
"spec/v1/gaps.json": "ced525510fbb3405dae0cdff4f0943a6d2aaa1bbff39312faac2679a55952116",
|
|
219
219
|
"spec/v1/gaps.schema.json": "8fd83259f556553c9df0f53e7a82ca8c2a4771d471197f69b4e2ae0ceeacfd66",
|
|
220
220
|
"spec/v1/migrations.json": "4c50760be70cb644d02427bcaf5749606700edf4513f423903cf3b1d12f5c300",
|
|
221
221
|
"spec/v1/migrations.schema.json": "886779aa6c22e646db097f5df210adb018a4dd14a7b815465a18c8a7056c8f72",
|
|
@@ -228,7 +228,7 @@
|
|
|
228
228
|
"spec/v2/core/connection-packs.md": "c80f95bd6a03698f35c72cdb2b46bd973f49203bbd786c26760e35fa86e6f23d",
|
|
229
229
|
"spec/v2/core/conversation.md": "8d6abedf6af408a74577753bfca24719d318e2632e10b45ff5d63bdcf31a8aaa",
|
|
230
230
|
"spec/v2/core/errors.md": "e48838701b86bbfe0ffee77b18a8ffc4e5f04724b4318507d9771da653f8da0f",
|
|
231
|
-
"spec/v2/core/events.md": "
|
|
231
|
+
"spec/v2/core/events.md": "c3dcece6f1b1aa43e1e01c5ece23b2df075a505d5f6c2f37de0801aeaa486b62",
|
|
232
232
|
"spec/v2/core/execution.md": "4218dbb397534940f83ca5d46561f02bf8f3fdf68d2ae7fefe7f2eb2ed771103",
|
|
233
233
|
"spec/v2/core/form-content-packs.md": "02931227899676fd4581a27017be2a6c4953904717be9b81c29d12f87ab3917d",
|
|
234
234
|
"spec/v2/core/headers.md": "f69fe81e4a3ad21879bb85948660c16860eb05bf152928c31aa4898f6d7d1887",
|
|
@@ -254,7 +254,7 @@
|
|
|
254
254
|
"spec/v2/core/workflow-chain-packs.md": "fdce7a8477e269950a2bac05ee6eddbd66eb0b670b464d00054f7196ec05d29c",
|
|
255
255
|
"spec/v2/corrections.json": "3a8d93e33865b692d26a3b6bfd3819766e206b2e0c2a69669baa4bf833d05a49",
|
|
256
256
|
"spec/v2/corrections.schema.json": "46bbffb954d13bd9995aebc912ceeb2544f7064331abf25fd6ab85c81a5cec94",
|
|
257
|
-
"spec/v2/declaration.json": "
|
|
257
|
+
"spec/v2/declaration.json": "de8558d76a03d07101c5e26a7311d3e90b449f065c9ba418f4abd9c91864a58b",
|
|
258
258
|
"spec/v2/declaration.schema.json": "ef951633e4070899f64828f5cf1b5f4870270a0360f81dbb49b5b4d6cd407550",
|
|
259
259
|
"spec/v2/eos-override.json": "4c72d2f6a658943b01c1de1643cf294642091933af7832f38ee5495df684cb2d",
|
|
260
260
|
"spec/v2/errors.json": "1f0e29455edbd22e28f67f221766209c3c55ac4987beb77993b5702dd7cae131",
|
|
@@ -281,7 +281,7 @@
|
|
|
281
281
|
"spec/v2/ext/sandbox-runtime-notes/README.md": "0d1290fb1e20b2e35c0b15051541ab6546e4e2a6f693b4f912cb3f7d6a65e92c",
|
|
282
282
|
"spec/v2/ext/webResearch/README.md": "b675e6f6646128d781b661f754c2393eed86e5935d16c3a22df582a5ce025b9b",
|
|
283
283
|
"spec/v2/facets/a2a.schema.json": "b68bf5e29e455e46c2ad7cfcccf5963730ec2310ff9db81884550d8df30fd6ef",
|
|
284
|
-
"spec/v2/facets/agents.schema.json": "
|
|
284
|
+
"spec/v2/facets/agents.schema.json": "88fe144572c25e1b09b99fd83e2ad12b2611d4d7366b5d53c3a50d0310faad93",
|
|
285
285
|
"spec/v2/facets/aiProviders.schema.json": "4eb911c2403b8f618325194855a1326680fbda2549dfbfe86a70dd2743f8fb12",
|
|
286
286
|
"spec/v2/facets/auditLogIntegrity.schema.json": "9a7f81320c8f940c6f4129bc7dd895a4063231c5458a717a08c2cbf9bb886c46",
|
|
287
287
|
"spec/v2/facets/auth.schema.json": "7d98bc81a0f7d307460d039eebfebd21388091bfe1d538762583940f94c0bd2a",
|
|
@@ -290,6 +290,7 @@
|
|
|
290
290
|
"spec/v2/facets/heartbeat.schema.json": "81d7fa1895a8b0241bd07cf00a7e7905bef0fd14eb3427325261c62eee5746a6",
|
|
291
291
|
"spec/v2/facets/hostEvents.schema.json": "973ff452cb91356b4c625d0b06c8d2ef335e18183de09e1afde8acead22290b3",
|
|
292
292
|
"spec/v2/facets/interrupt.schema.json": "2110ef32c3f880bf706475ef4cf8ddf97ff06871b8e6003a5edbcf4439f1fb70",
|
|
293
|
+
"spec/v2/facets/limits.schema.json": "218fcbd34dd2cdf7548c29c8c76e1a6aea2b7d7df1ba32fbe201bd40917bc66f",
|
|
293
294
|
"spec/v2/facets/mcp.schema.json": "7060657dfcff170f079beed160e3db7c1d3cc4410cd3cf64863baf6459f781cd",
|
|
294
295
|
"spec/v2/facets/nondeterminismPolicy.schema.json": "e9dfaa32e0778a1a837fdad3b8a0673fccda9579ab08c8bfe9777362dd15e989",
|
|
295
296
|
"spec/v2/facets/oauth.schema.json": "3a0cbe249d3bd5a5982349ae7ba2534776cc958e4a57e0fbeabf2bdab6e99e69",
|
|
@@ -313,9 +314,9 @@
|
|
|
313
314
|
"spec/v2/path-manifest.json": "7b5679a4dad98c80d855361d88a98c8a4e045af15e310862a05b285b815dc9fc",
|
|
314
315
|
"spec/v2/peer-dependency-aliases.json": "d10299280abee08258502925bc327293ee413e0108cd6e6ec75ff6110653308d",
|
|
315
316
|
"spec/v2/profiles.json": "b382b261c7844798575f7c1b37c934f0da19846016a2337d883d4f5ef6e05479",
|
|
316
|
-
"spec/v2/release.json": "
|
|
317
|
+
"spec/v2/release.json": "e8fe4796dd85264ee4877b203da962d8c593c996e428289b165827cd86bfd2cc",
|
|
317
318
|
"spec/v2/retention-floors.json": "eaf3722d95c79947af1d4269ef85117e126518c588cfcf1a2b21b97269f51624",
|
|
318
|
-
"spec/v2/surface-baseline.json": "
|
|
319
|
+
"spec/v2/surface-baseline.json": "094912462a9b1bdd8bc3d7c9f10b4dc342d0db41b2802c3a1c60d8a8b27d2860"
|
|
319
320
|
},
|
|
320
|
-
"corpusCommit": "
|
|
321
|
+
"corpusCommit": "234f6c57b1e01099827dcfabeaad542c2604b17a"
|
|
321
322
|
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0246 — a trial that fails its trajectory cannot carry a task or a suite
|
|
3
|
+
* to a pass. Pure judges over an eval run's `EvalSummary` and `eval.scored`
|
|
4
|
+
* events, for `v2-eval-trajectory-trials.test.ts`.
|
|
5
|
+
*
|
|
6
|
+
* The rule, in one line: every reported score stays the RFC 0244 measurement
|
|
7
|
+
* (§C), and the two pass decisions read a GATING value in which a
|
|
8
|
+
* trajectory-failed trial counts 0 (§A, §B). The summary carries that value as
|
|
9
|
+
* `gatedScore` on every trajectory task and `gatedAggregateScore` on any suite
|
|
10
|
+
* with one (§D).
|
|
11
|
+
*
|
|
12
|
+
* Each judge takes what the host served and returns findings; it never reads
|
|
13
|
+
* the host's own `passed` to decide what `passed` should be. The two fixtures
|
|
14
|
+
* make the expected answer known in advance: the scripted agent produces the
|
|
15
|
+
* golden output (score 1) on every task, and calls a `forbidden` tool on the
|
|
16
|
+
* `path-wrong` task.
|
|
17
|
+
*
|
|
18
|
+
* @see RFCS/0246-trajectory-failure-gates-the-pass.md
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { withinTolerance } from './eval-trajectory.js';
|
|
22
|
+
|
|
23
|
+
export interface GateFinding { readonly ok: boolean; readonly doc: string; readonly message: string }
|
|
24
|
+
|
|
25
|
+
type Rec = Record<string, unknown>;
|
|
26
|
+
const isRec = (v: unknown): v is Rec => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
27
|
+
const num = (v: unknown): number | undefined => (typeof v === 'number' && Number.isFinite(v) ? v : undefined);
|
|
28
|
+
const f = (ok: boolean, doc: string, message: string): GateFinding => ({ ok, doc, message });
|
|
29
|
+
const close = (a: number | undefined, b: number): boolean => a !== undefined && withinTolerance(a, b);
|
|
30
|
+
|
|
31
|
+
const A = 'RFC 0246 §A';
|
|
32
|
+
const B = 'RFC 0246 §B';
|
|
33
|
+
const C = 'RFC 0246 §C';
|
|
34
|
+
const D = 'RFC 0246 §D';
|
|
35
|
+
|
|
36
|
+
function task(summary: Rec, taskId: string): Rec | undefined {
|
|
37
|
+
const tasks = Array.isArray(summary['tasks']) ? (summary['tasks'] as unknown[]) : [];
|
|
38
|
+
return tasks.filter(isRec).find((t) => t['taskId'] === taskId);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* `conformance-eval-trajectory-gate-mean`: one task, `passRule: "mean"`, two
|
|
43
|
+
* trials, each the golden output with a forbidden call. Under RFC 0244 alone a
|
|
44
|
+
* host reports `meanScore: 1` and `passed: true`; RFC 0246 requires
|
|
45
|
+
* `gatedScore: 0` and `passed: false` with `meanScore` unchanged.
|
|
46
|
+
*/
|
|
47
|
+
export function judgeMeanGated(summary: Rec, scored: readonly Rec[], trialCount: number): GateFinding[] {
|
|
48
|
+
const out: GateFinding[] = [];
|
|
49
|
+
const rows = scored.filter((e) => e['taskId'] === 'path-wrong');
|
|
50
|
+
out.push(f(rows.length === trialCount, 'RFC 0244 §B', `path-wrong MUST emit one eval.scored per trial (${trialCount}; got ${rows.length})`));
|
|
51
|
+
for (const r of rows) {
|
|
52
|
+
out.push(
|
|
53
|
+
f(num(r['score']) === 1, C, `eval.scored.score MUST stay the output score 1 for trial ${String(r['trialIndex'])} (got ${String(r['score'])})`),
|
|
54
|
+
f(r['passed'] === false, 'RFC 0244 §A', `a trial that called a forbidden tool MUST NOT pass (trial ${String(r['trialIndex'])})`),
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
const t = task(summary, 'path-wrong');
|
|
58
|
+
out.push(f(t !== undefined, D, 'the summary MUST carry task path-wrong'));
|
|
59
|
+
if (t === undefined) return out;
|
|
60
|
+
const trials = isRec(t['trials']) ? t['trials'] : {};
|
|
61
|
+
out.push(
|
|
62
|
+
f(close(num(trials['meanScore']), 1), C, `trials.meanScore MUST stay the mean output score 1 (got ${String(trials['meanScore'])})`),
|
|
63
|
+
f(num(t['gatedScore']) !== undefined, D, 'a task with expected.trajectory MUST carry gatedScore'),
|
|
64
|
+
f(close(num(t['gatedScore']), 0), A, `gatedScore MUST be 0 when every trial failed its trajectory (got ${String(t['gatedScore'])})`),
|
|
65
|
+
f(t['passed'] === false, B, `under passRule "mean" the task MUST NOT pass on the gating score 0, whatever meanScore is (got passed ${String(t['passed'])})`),
|
|
66
|
+
f(num(summary['gatedAggregateScore']) !== undefined, D, 'a suite with a trajectory task MUST carry gatedAggregateScore'),
|
|
67
|
+
);
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* `conformance-eval-trajectory-gate`: two tasks, one trial each, both the
|
|
73
|
+
* golden output (score 1); `path-matched` keeps to its trajectory and
|
|
74
|
+
* `path-wrong` calls a forbidden tool. `passScore` is 0.6. Under RFC 0244 alone
|
|
75
|
+
* `aggregateScore` 1 clears it and suite `passed` is true; RFC 0246 requires the
|
|
76
|
+
* gating aggregate 0.5 to decide, so `passed` is false.
|
|
77
|
+
*/
|
|
78
|
+
export function judgeSuiteGated(summary: Rec, passScore: number): GateFinding[] {
|
|
79
|
+
const out: GateFinding[] = [];
|
|
80
|
+
const matched = task(summary, 'path-matched');
|
|
81
|
+
const wrong = task(summary, 'path-wrong');
|
|
82
|
+
out.push(f(matched !== undefined && wrong !== undefined, D, 'the summary MUST carry tasks path-matched and path-wrong'));
|
|
83
|
+
if (matched === undefined || wrong === undefined) return out;
|
|
84
|
+
const ms = num(matched['score']);
|
|
85
|
+
const ws = num(wrong['score']);
|
|
86
|
+
const agg = num(summary['aggregateScore']);
|
|
87
|
+
const gAgg = num(summary['gatedAggregateScore']);
|
|
88
|
+
const mg = num(matched['gatedScore']);
|
|
89
|
+
const wg = num(wrong['gatedScore']);
|
|
90
|
+
out.push(
|
|
91
|
+
f(ms === 1 && ws === 1, C, `both tasks MUST keep the output score 1 (got ${String(ms)}, ${String(ws)})`),
|
|
92
|
+
f(wrong['passed'] === false, 'RFC 0244 §A', 'path-wrong called a forbidden tool and MUST NOT pass'),
|
|
93
|
+
f(close(agg, 1), C, `aggregateScore MUST stay the aggregate of the task scores, 1 (got ${String(agg)})`),
|
|
94
|
+
f(mg !== undefined && wg !== undefined, D, 'every task with expected.trajectory MUST carry gatedScore, matched or not'),
|
|
95
|
+
f(ms !== undefined && close(mg, ms), A, `path-matched MUST carry gatedScore equal to its score (got ${String(mg)})`),
|
|
96
|
+
f(close(wg, 0), A, `path-wrong MUST carry gatedScore 0 (got ${String(wg)})`),
|
|
97
|
+
f(gAgg !== undefined, D, 'a suite with a trajectory task MUST carry gatedAggregateScore'),
|
|
98
|
+
f(close(gAgg, 0.5), A, `gatedAggregateScore MUST aggregate the gating scores [1, 0] to 0.5 (got ${String(gAgg)})`),
|
|
99
|
+
f(summary['passed'] === false, B, `suite passed MUST read the gating aggregate (0.5 < passScore ${passScore}) even though aggregateScore (${String(agg)}) clears it (got passed ${String(summary['passed'])})`),
|
|
100
|
+
);
|
|
101
|
+
return out;
|
|
102
|
+
}
|
|
@@ -32,7 +32,20 @@
|
|
|
32
32
|
* lists `conformance-eval-trials` refuses it at createRun with
|
|
33
33
|
* `400 validation_error`.
|
|
34
34
|
*
|
|
35
|
+
* RFC 0246 (safety fix): a trial that failed its trajectory counts 0 in the
|
|
36
|
+
* two pass decisions, while every reported score is unchanged. Judges in
|
|
37
|
+
* `lib/eval-gate-witness.ts`.
|
|
38
|
+
* 7. Server-free — `gatedScore` and `gatedAggregateScore` validate; a value
|
|
39
|
+
* outside 0–1 is refused.
|
|
40
|
+
* 8. `conformance-eval-trajectory-gate-mean` (trajectory + trials) — under
|
|
41
|
+
* `passRule: "mean"` a task whose two trials took a forbidden path keeps
|
|
42
|
+
* `meanScore: 1` and does not pass (`gatedScore: 0`).
|
|
43
|
+
* 9. `conformance-eval-trajectory-gate` (trajectory) — `aggregateScore` 1
|
|
44
|
+
* clears `passScore` 0.6, the gating aggregate 0.5 does not, so the suite
|
|
45
|
+
* does not pass. An RFC 0244-only host reports `passed: true` on both.
|
|
46
|
+
*
|
|
35
47
|
* @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md
|
|
48
|
+
* @see RFCS/0246-trajectory-failure-gates-the-pass.md
|
|
36
49
|
*/
|
|
37
50
|
|
|
38
51
|
import { describe, it, expect } from 'vitest';
|
|
@@ -48,6 +61,7 @@ import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
|
48
61
|
import { FIXTURES_DIR } from '../lib/paths.js';
|
|
49
62
|
import { pollTerminal, readEvents, isRecord, type Ev } from '../lib/fixture-run-observer.js';
|
|
50
63
|
import { gradeTrajectory, passAtK, passHatK, scoreStderr, withinTolerance, type TrajectorySpec } from '../lib/eval-trajectory.js';
|
|
64
|
+
import { judgeMeanGated, judgeSuiteGated } from '../lib/eval-gate-witness.js';
|
|
51
65
|
|
|
52
66
|
const PROFILE = majorProfile(2);
|
|
53
67
|
const UNREADABLE = 'v2 discovery unreachable — /.well-known/openwop did not answer 200 JSON under OpenWOP-Version: 2.0';
|
|
@@ -62,6 +76,10 @@ const ID_TRIALS = 'openwop.requirement.0244.trials-per-task';
|
|
|
62
76
|
const ID_TRIAL_STATS = 'openwop.requirement.0244.trial-stats-formula';
|
|
63
77
|
const ID_MISS = 'openwop.requirement.0244.fixture-miss-fails';
|
|
64
78
|
const ID_REFUSED = 'openwop.requirement.0244.unadvertised-feature-refused';
|
|
79
|
+
const ID_GATED_SHAPE = 'openwop.requirement.0246.gated-fields-shape';
|
|
80
|
+
const ID_MEAN_GATED = 'openwop.requirement.0246.mean-gated';
|
|
81
|
+
const ID_SUITE_GATED = 'openwop.requirement.0246.suite-gated';
|
|
82
|
+
const DOC_0246 = 'eval-summary.schema.json gatedScore / gatedAggregateScore (RFC 0246)';
|
|
65
83
|
|
|
66
84
|
interface Task { taskId: string; input: { script: Array<{ tool: string; arguments?: unknown }>; output: string }; expected: { kind: string; trajectory?: TrajectorySpec } }
|
|
67
85
|
interface Suite { tasks: Task[]; trials?: { count: number; k?: number; passRule?: string } }
|
|
@@ -142,11 +160,19 @@ describe('v2 RFC 0244 — wire shape (server-free)', () => {
|
|
|
142
160
|
});
|
|
143
161
|
|
|
144
162
|
it('the eval-suite fixtures validate against the v2 suite schema', () => {
|
|
145
|
-
for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss']) {
|
|
163
|
+
for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss', 'conformance-eval-trajectory-gate', 'conformance-eval-trajectory-gate-mean']) {
|
|
146
164
|
const r = suiteSchema(fixture(id));
|
|
147
165
|
expect(r.ok, req(ID_FIXTURES, 'fixtures.md §eval-suites', `${id} MUST validate: ${r.errors}`)).toBe(true);
|
|
148
166
|
}
|
|
149
167
|
});
|
|
168
|
+
|
|
169
|
+
it('the RFC 0246 gating fields validate and stay in 0–1', () => {
|
|
170
|
+
const base = { suiteId: 'core.openwop.evals.x', suiteVersion: '1.0.0', aggregateScore: 1, passed: false, taskCount: 1, passedCount: 0 };
|
|
171
|
+
const ok = { ...base, gatedAggregateScore: 0, tasks: [{ taskId: 't', score: 1, gatedScore: 0, passed: false }] };
|
|
172
|
+
expect(summarySchema(ok).ok, req(ID_GATED_SHAPE, DOC_0246, `gatedScore and gatedAggregateScore MUST validate: ${summarySchema(ok).errors}`)).toBe(true);
|
|
173
|
+
expect(summarySchema({ ...ok, gatedAggregateScore: 1.5 }).ok, req(ID_GATED_SHAPE, DOC_0246, 'gatedAggregateScore above 1 MUST be refused')).toBe(false);
|
|
174
|
+
expect(summarySchema({ ...ok, tasks: [{ taskId: 't', score: 1, gatedScore: -0.1, passed: false }] }).ok, req(ID_GATED_SHAPE, DOC_0246, 'a negative gatedScore MUST be refused')).toBe(false);
|
|
175
|
+
});
|
|
150
176
|
});
|
|
151
177
|
|
|
152
178
|
describe('v2 RFC 0244 — host behaviour', () => {
|
|
@@ -255,4 +281,28 @@ describe('v2 RFC 0244 — host behaviour', () => {
|
|
|
255
281
|
expect(res.status, req(ID_REFUSED, DOC, `a trials suite on a host without agents.evalSuite.trials MUST be refused with 400 (got ${res.status})`)).toBe(400);
|
|
256
282
|
expect(readErrorCode(res.json), req(ID_REFUSED, DOC, 'the refusal MUST carry validation_error')).toBe('validation_error');
|
|
257
283
|
});
|
|
284
|
+
|
|
285
|
+
it('under passRule "mean" a task whose trials failed their trajectory does not pass (RFC 0246)', async () => {
|
|
286
|
+
const { doc, facet } = await evalSuiteFacet();
|
|
287
|
+
if (!doc) return softSkip('blocked', UNREADABLE);
|
|
288
|
+
if (facet?.['trajectory'] !== true) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite.trajectory');
|
|
289
|
+
if (!isRecord(facet['trials'])) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite.trials, so passRule "mean" over trials does not apply');
|
|
290
|
+
const id = 'conformance-eval-trajectory-gate-mean';
|
|
291
|
+
if (!isFixtureAdvertised(id)) return softSkip('inapplicable', `the host does not advertise fixture ${id}`);
|
|
292
|
+
const o = await runSuite(id);
|
|
293
|
+
if (o.kind === 'skip') return softSkip(o.disposition, o.reason);
|
|
294
|
+
const scoredRows = o.events.filter((e) => e.type === 'eval.scored').map((e) => e.payload);
|
|
295
|
+
for (const x of judgeMeanGated(o.summary, scoredRows, fixture(id).trials!.count)) expect(x.ok, req(ID_MEAN_GATED, x.doc, x.message)).toBe(true);
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
it('a suite whose tasks failed their trajectory does not pass on the output scores (RFC 0246)', async () => {
|
|
299
|
+
const { doc, facet } = await evalSuiteFacet();
|
|
300
|
+
if (!doc) return softSkip('blocked', UNREADABLE);
|
|
301
|
+
if (facet?.['trajectory'] !== true) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite.trajectory');
|
|
302
|
+
const id = 'conformance-eval-trajectory-gate';
|
|
303
|
+
if (!isFixtureAdvertised(id)) return softSkip('inapplicable', `the host does not advertise fixture ${id}`);
|
|
304
|
+
const o = await runSuite(id);
|
|
305
|
+
if (o.kind === 'skip') return softSkip(o.disposition, o.reason);
|
|
306
|
+
for (const x of judgeSuiteGated(o.summary, 0.6)) expect(x.ok, req(ID_SUITE_GATED, x.doc, x.message)).toBe(true);
|
|
307
|
+
});
|
|
258
308
|
});
|