@openwop/openwop-conformance 2.46.27 → 2.46.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +2 -2
- package/dist/spec-artifacts.lock.json +2 -2
- package/fixtures/eval-suites/conformance-eval-regression-base.json +74 -0
- package/fixtures/eval-suites/conformance-eval-regression-candidate.json +74 -0
- package/fixtures.md +4 -0
- package/package.json +2 -2
- package/requirements.json +81 -4
- package/scenario-majors.json +5 -2
- package/schemas/CORPUS-STAMP.json +12 -12
- package/src/lib/eval-regression-witness.ts +77 -0
- package/src/scenarios/v2-eval-regression-baseline.test.ts +214 -0
- package/src/scenarios/v2-eval-trajectory-trials.test.ts +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
## [2.46.28] — 2026-10-26 — RFC 0244 G2 closed; RFC 0251 and the §F legs
|
|
2
|
+
|
|
3
|
+
- **New scenario `v2-eval-regression-baseline` (RFC 0251)** over two fixtures sharing one `suiteId` (`conformance-eval-regression-base` / `-candidate`), whose paired scores are equal while the aggregates differ by 1/3: `0251.shape` (server-free), `baseline-echoed`, `baseline-refused`, `baseline-uniform` (needs `OPENWOP_TEST_TENANT_B_API_KEY`, else `blocked`), `fields-refused`, and RFC 0244 §F `0244.regression-paired` and `regression-no-false-verdict`. Judges in `lib/eval-regression-witness.ts`.
|
|
4
|
+
- **Host impact:** opt-in. The legs need `agents.evalSuite.baselineSelection` and both fixtures listed; no host does yet, so they record `inapplicable`.
|
|
5
|
+
- **RFC 0244's G2 closes** on openwop-app's certified cut `fc9f666ed` (the three counter legs and `0244.efficiency-tool-calls` `executed-pass`), so the packed `spec/v1/gaps.json` changes.
|
|
6
|
+
|
|
1
7
|
## [2.46.27] — 2026-10-25 — RFC 0244 never-gated and trial-isolation legs
|
|
2
8
|
|
|
3
9
|
- **New leg `0244.efficiency-never-gated` in `v2-eval-trajectory-trials`**, on the new fixture `conformance-eval-efficiency-ungated`: tasks `lean` (one scripted call) and `busy` (eight) share an output and an expectation, so `busy` must pass with `lean`'s score and the suite must pass at `passScore` 1. `blocked` when the calls never ran or `lean` fails. Judge `judgeNeverGated` in `lib/eval-efficiency-witness.ts`.
|
package/README.md
CHANGED
|
@@ -137,7 +137,7 @@ Each scenario file records its disposition and assertion count in `<report-dir>/
|
|
|
137
137
|
|
|
138
138
|
## What's Covered
|
|
139
139
|
|
|
140
|
-
The current suite has
|
|
140
|
+
The current suite has 662 scenario files under `src/scenarios/`. By target major ([`scenario-majors.json`](./scenario-majors.json)): 222 files run against a v2 host and 455 against a v1 host; 15 of them run at both.
|
|
141
141
|
|
|
142
142
|
The v2 scenarios (`src/scenarios/v2-*.test.ts`) cover, by area:
|
|
143
143
|
|
|
@@ -166,7 +166,7 @@ Three kinds of scenario share the tree:
|
|
|
166
166
|
- **Host** checks drive a live host over HTTP.
|
|
167
167
|
- **Gated** checks run only when the host advertises the family, profile or fixture they need, and otherwise record `inapplicable` or `blocked` with a reason. `--require-behavior` turns an advertised-but-unwitnessable behaviour into a failure.
|
|
168
168
|
|
|
169
|
-
Current source tree:
|
|
169
|
+
Current source tree: 662 scenario files. [`coverage.md`](./coverage.md) maps each spec document and every OpenAPI operation to its scenarios. [`CHANGELOG.md`](./CHANGELOG.md) records when each scenario landed.
|
|
170
170
|
|
|
171
171
|
## How the 2.x suite is built
|
|
172
172
|
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
{
|
|
2
|
+
"suiteId": "core.openwop.evals.conformance-eval-regression",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"targetAgentId": "core.conformance.agents.scripted",
|
|
5
|
+
"modes": [
|
|
6
|
+
"golden",
|
|
7
|
+
"regression"
|
|
8
|
+
],
|
|
9
|
+
"tasks": [
|
|
10
|
+
{
|
|
11
|
+
"taskId": "base-only",
|
|
12
|
+
"input": {
|
|
13
|
+
"script": [
|
|
14
|
+
{
|
|
15
|
+
"tool": "conformance.eval:lookup",
|
|
16
|
+
"arguments": {
|
|
17
|
+
"orderId": "o-1"
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
],
|
|
21
|
+
"output": "done"
|
|
22
|
+
},
|
|
23
|
+
"expected": {
|
|
24
|
+
"kind": "golden",
|
|
25
|
+
"match": {
|
|
26
|
+
"strategy": "contains",
|
|
27
|
+
"value": "never-in-output"
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"taskId": "shared-a",
|
|
33
|
+
"input": {
|
|
34
|
+
"script": [
|
|
35
|
+
{
|
|
36
|
+
"tool": "conformance.eval:lookup",
|
|
37
|
+
"arguments": {
|
|
38
|
+
"orderId": "o-1"
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
],
|
|
42
|
+
"output": "done"
|
|
43
|
+
},
|
|
44
|
+
"expected": {
|
|
45
|
+
"kind": "golden",
|
|
46
|
+
"match": {
|
|
47
|
+
"strategy": "contains",
|
|
48
|
+
"value": "done"
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"taskId": "shared-b",
|
|
54
|
+
"input": {
|
|
55
|
+
"script": [
|
|
56
|
+
{
|
|
57
|
+
"tool": "conformance.eval:lookup",
|
|
58
|
+
"arguments": {
|
|
59
|
+
"orderId": "o-1"
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
],
|
|
63
|
+
"output": "done"
|
|
64
|
+
},
|
|
65
|
+
"expected": {
|
|
66
|
+
"kind": "golden",
|
|
67
|
+
"match": {
|
|
68
|
+
"strategy": "contains",
|
|
69
|
+
"value": "done"
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
]
|
|
74
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
{
|
|
2
|
+
"suiteId": "core.openwop.evals.conformance-eval-regression",
|
|
3
|
+
"version": "1.1.0",
|
|
4
|
+
"targetAgentId": "core.conformance.agents.scripted",
|
|
5
|
+
"modes": [
|
|
6
|
+
"golden",
|
|
7
|
+
"regression"
|
|
8
|
+
],
|
|
9
|
+
"tasks": [
|
|
10
|
+
{
|
|
11
|
+
"taskId": "shared-a",
|
|
12
|
+
"input": {
|
|
13
|
+
"script": [
|
|
14
|
+
{
|
|
15
|
+
"tool": "conformance.eval:lookup",
|
|
16
|
+
"arguments": {
|
|
17
|
+
"orderId": "o-1"
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
],
|
|
21
|
+
"output": "done"
|
|
22
|
+
},
|
|
23
|
+
"expected": {
|
|
24
|
+
"kind": "golden",
|
|
25
|
+
"match": {
|
|
26
|
+
"strategy": "contains",
|
|
27
|
+
"value": "done"
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"taskId": "shared-b",
|
|
33
|
+
"input": {
|
|
34
|
+
"script": [
|
|
35
|
+
{
|
|
36
|
+
"tool": "conformance.eval:lookup",
|
|
37
|
+
"arguments": {
|
|
38
|
+
"orderId": "o-1"
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
],
|
|
42
|
+
"output": "done"
|
|
43
|
+
},
|
|
44
|
+
"expected": {
|
|
45
|
+
"kind": "golden",
|
|
46
|
+
"match": {
|
|
47
|
+
"strategy": "contains",
|
|
48
|
+
"value": "done"
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"taskId": "candidate-only",
|
|
54
|
+
"input": {
|
|
55
|
+
"script": [
|
|
56
|
+
{
|
|
57
|
+
"tool": "conformance.eval:lookup",
|
|
58
|
+
"arguments": {
|
|
59
|
+
"orderId": "o-1"
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
],
|
|
63
|
+
"output": "done"
|
|
64
|
+
},
|
|
65
|
+
"expected": {
|
|
66
|
+
"kind": "golden",
|
|
67
|
+
"match": {
|
|
68
|
+
"strategy": "contains",
|
|
69
|
+
"value": "done"
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
]
|
|
74
|
+
}
|
package/fixtures.md
CHANGED
|
@@ -79,6 +79,8 @@ All fixtures MUST advertise:
|
|
|
79
79
|
| Eval — Counter (miss) | `conformance-eval-counter-miss` | RFC 0244 §D, G2 — `onMiss: "fail"`; three `inc S` calls against two exact entries, then an unmatched `inc F` (`v2-eval-trajectory-trials`) | eval run `completed`; `{served: 1}`, `{served: 2}`, then two tool errors | ≤ 30s |
|
|
80
80
|
| Eval — Efficiency Ungated | `conformance-eval-efficiency-ungated` | RFC 0244 §C — two golden tasks with one output, `lean` (1 call) and `busy` (8 calls), at `passScore: 1` (`v2-eval-trajectory-trials`) | eval run `completed`; both tasks and the suite pass | ≤ 30s |
|
|
81
81
|
| Eval — Memory Isolation | `conformance-eval-memory-isolation` | RFC 0244 §B, G3 — `trials.count: 2`; one task seeds `seeded`, then reads it, reads `trial-note`, writes it and reads it back through `conformance.eval:memory` (`v2-eval-trajectory-trials`) | eval run `completed`; each trial reads the seed and finds no `trial-note` before its own write | ≤ 30s |
|
|
82
|
+
| Eval — Regression (base) | `conformance-eval-regression-base` | RFC 0251 — the baseline run: suite 1.0.0, `base-only` fails, `shared-a` and `shared-b` pass (`v2-eval-regression-baseline`) | eval run `completed`; `aggregateScore` 2/3 | ≤ 30s |
|
|
83
|
+
| Eval — Regression (candidate) | `conformance-eval-regression-candidate` | RFC 0251, RFC 0244 §F — the same `suiteId` at 1.1.0: the two shared tasks unchanged, plus `candidate-only` (passes), run against the base run (`v2-eval-regression-baseline`) | eval run `completed`; `aggregateScore` 1; `scoreDelta` 1/3; `pairedTaskCount` 2; verdict neither `improved` nor `regressed` | ≤ 30s |
|
|
82
84
|
| Dispatch Fan-Out Waves | `conformance-dispatch-fanout-waves` | RFC 0243 §A.3 — a parallel `core.dispatch` (`maxConcurrency: 2`, `wait-all`/`collect`) whose supervisor's `mockDispatchPlan` names three `conformance-delay` children. At most `min(2, maxFanOut)` run at once and none is dropped. Gated on `agents.fanOut` admitting `wait-all` with `collect`. | `completed` | ≤ 30s |
|
|
83
85
|
| Dispatch Per-Item Input | `conformance-dispatch-per-item` | RFC 0243 §A.4 — a sequential `core.dispatch` whose decision dispatches `conformance-dispatch-input-mapping-child` twice with `nextWorkerInputs` `item-a`, `item-b`. At `agents.fanOut.perItemInput: true` each child receives its item; otherwise the dispatch node fails `validation_error` and no child is dispatched. | `completed`, or `failed` without `perItemInput` | ≤ 30s |
|
|
84
86
|
| Dispatch Per-Item Length Mismatch | `conformance-dispatch-per-item-mismatch` | `orchestrator-decision.schema.json` `nextWorkerInputs` — one item for two children. Every dispatch host fails the dispatch node `validation_error` and dispatches no child. | `failed` | ≤ 30s |
|
|
@@ -1065,6 +1067,8 @@ Every task targets **the scripted agent `core.conformance.agents.scripted`**. It
|
|
|
1065
1067
|
| `conformance-eval-counter-miss` | `agents.evalSuite.fixtureMatching` | the three `inc S` calls return `{served: 1}`, `{served: 2}`, then a tool error; `inc F` returns a tool error. Neither miss moves the live counter |
|
|
1066
1068
|
| `conformance-eval-efficiency-ungated` | `agents.evalSuite` | RFC 0244 §C never-gated: with all nine scripted calls in the log and `lean` passing, `busy` passes with `lean`'s score and the suite passes. A host MUST NOT fail either on `efficiency`, reported or not |
|
|
1067
1069
|
| `conformance-eval-memory-isolation` | `agents.evalSuite.trials` | RFC 0244 §B isolation: both reads of `seeded` return `seed`; of the four `trial-note` reads, two return `written` (each trial's read-back) and two return `null` (each trial's first read). A later trial reading `written` first saw an earlier trial's write. Fewer than two read-backs record `blocked` |
|
|
1070
|
+
| `conformance-eval-regression-base` | `agents.evalSuite.baselineSelection` + `regression` in `modes` | run first with `evalModes: [golden]`; the baseline of the candidate run |
|
|
1071
|
+
| `conformance-eval-regression-candidate` | `agents.evalSuite.baselineSelection` + `regression` in `modes` (+ `regressionVerdict` for the §F legs) | run with `evalModes: [golden, regression]` and the base run as `baselineRunId`: both carriers name it, `scoreDelta` 1/3, `pairedTaskCount` 2, verdict neither `improved` nor `regressed`. Both fixtures share one `suiteId`, so a host's suite store resolves them by fixture id |
|
|
1068
1072
|
|
|
1069
1073
|
**The counter tool `conformance.eval:counter` is conformance-RESERVED** (the pattern of `core.conformance.side-effect`). A host that lists either counter fixture in discovery `fixtures` MUST, at its eval tool boundary:
|
|
1070
1074
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openwop/openwop-conformance",
|
|
3
|
-
"version": "2.46.
|
|
3
|
+
"version": "2.46.28",
|
|
4
4
|
"description": "Black-box conformance suite for OpenWOP hosts (v2 current; v1 scenarios retained for the frozen v1 tree).",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -56,6 +56,6 @@
|
|
|
56
56
|
"@openwop/spec-artifacts": "file:../spec-artifacts"
|
|
57
57
|
},
|
|
58
58
|
"peerDependencies": {
|
|
59
|
-
"@openwop/spec-artifacts": "2.46.
|
|
59
|
+
"@openwop/spec-artifacts": "2.46.28"
|
|
60
60
|
}
|
|
61
61
|
}
|
package/requirements.json
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
"$comment": "GENERATED by conformance/scripts/generate-requirement-registry.mjs — do not edit. One record per it()/test() in src/scenarios. Ids: openwop.it.<file-stem>.<title-slug>[~n] (src/lib/requirement-ids.ts). A record with id null has an interpolated title; its run-time row is keyed by the rendered title and maps here by file+line only. Renamed ids need a row in requirement-aliases.json.",
|
|
3
3
|
"generatedFrom": "src/scenarios/*.test.ts",
|
|
4
4
|
"counts": {
|
|
5
|
-
"files":
|
|
6
|
-
"tests":
|
|
7
|
-
"withStableId":
|
|
5
|
+
"files": 750,
|
|
6
|
+
"tests": 2855,
|
|
7
|
+
"withStableId": 2855,
|
|
8
8
|
"interpolatedTitles": 0,
|
|
9
|
-
"explicitIds":
|
|
9
|
+
"explicitIds": 2728
|
|
10
10
|
},
|
|
11
11
|
"records": [
|
|
12
12
|
{
|
|
@@ -31089,6 +31089,83 @@
|
|
|
31089
31089
|
}
|
|
31090
31090
|
]
|
|
31091
31091
|
},
|
|
31092
|
+
{
|
|
31093
|
+
"id": "openwop.it.v2-eval-regression-baseline.the-baselineselection-member-and-the-createrun-eval-properties-are-in-the-derive",
|
|
31094
|
+
"file": "v2-eval-regression-baseline.test.ts",
|
|
31095
|
+
"line": 131,
|
|
31096
|
+
"title": "the baselineSelection member and the createRun eval properties are in the derived contract",
|
|
31097
|
+
"explicitId": "openwop.requirement.0251.shape",
|
|
31098
|
+
"citations": [
|
|
31099
|
+
{
|
|
31100
|
+
"section": "agents.schema.json §evalSuite.baselineSelection, createRun evalModes / baselineRunId (RFC 0251)",
|
|
31101
|
+
"requirement": "baselineSelection: true MUST validate"
|
|
31102
|
+
},
|
|
31103
|
+
{
|
|
31104
|
+
"section": "agents.schema.json §evalSuite.baselineSelection, createRun evalModes / baselineRunId (RFC 0251)",
|
|
31105
|
+
"requirement": "baselineSelection MUST be a boolean"
|
|
31106
|
+
},
|
|
31107
|
+
{
|
|
31108
|
+
"section": "agents.schema.json §evalSuite.baselineSelection, createRun evalModes / baselineRunId (RFC 0251)",
|
|
31109
|
+
"requirement": "createRun MUST carry evalModes over the RFC 0081 §D modes"
|
|
31110
|
+
},
|
|
31111
|
+
{
|
|
31112
|
+
"section": "agents.schema.json §evalSuite.baselineSelection, createRun evalModes / baselineRunId (RFC 0251)",
|
|
31113
|
+
"requirement": "evalModes MUST be non-empty and distinct"
|
|
31114
|
+
},
|
|
31115
|
+
{
|
|
31116
|
+
"section": "agents.schema.json §evalSuite.baselineSelection, createRun evalModes / baselineRunId (RFC 0251)",
|
|
31117
|
+
"requirement": "createRun MUST carry baselineRunId as a runId"
|
|
31118
|
+
}
|
|
31119
|
+
]
|
|
31120
|
+
},
|
|
31121
|
+
{
|
|
31122
|
+
"id": "openwop.it.v2-eval-regression-baseline.the-candidate-run-names-the-baseline-it-was-given-and-scoredelta-is-the-aggregat",
|
|
31123
|
+
"file": "v2-eval-regression-baseline.test.ts",
|
|
31124
|
+
"line": 146,
|
|
31125
|
+
"title": "the candidate run names the baseline it was given, and scoreDelta is the aggregate difference",
|
|
31126
|
+
"explicitId": "openwop.requirement.0251.baseline-echoed",
|
|
31127
|
+
"citations": []
|
|
31128
|
+
},
|
|
31129
|
+
{
|
|
31130
|
+
"id": "openwop.it.v2-eval-regression-baseline.only-the-tasks-in-both-runs-are-paired-rfc-0244-f",
|
|
31131
|
+
"file": "v2-eval-regression-baseline.test.ts",
|
|
31132
|
+
"line": 154,
|
|
31133
|
+
"title": "only the tasks in both runs are paired (RFC 0244 §F)",
|
|
31134
|
+
"explicitId": "openwop.requirement.0244.regression-paired",
|
|
31135
|
+
"citations": []
|
|
31136
|
+
},
|
|
31137
|
+
{
|
|
31138
|
+
"id": "openwop.it.v2-eval-regression-baseline.equal-paired-scores-never-yield-improved-or-regressed-rfc-0244-f",
|
|
31139
|
+
"file": "v2-eval-regression-baseline.test.ts",
|
|
31140
|
+
"line": 163,
|
|
31141
|
+
"title": "equal paired scores never yield improved or regressed (RFC 0244 §F)",
|
|
31142
|
+
"explicitId": "openwop.requirement.0244.regression-no-false-verdict",
|
|
31143
|
+
"citations": []
|
|
31144
|
+
},
|
|
31145
|
+
{
|
|
31146
|
+
"id": "openwop.it.v2-eval-regression-baseline.an-unknown-baseline-and-a-run-of-another-suite-are-refused-without-the-id",
|
|
31147
|
+
"file": "v2-eval-regression-baseline.test.ts",
|
|
31148
|
+
"line": 172,
|
|
31149
|
+
"title": "an unknown baseline, and a run of another suite, are refused without the id",
|
|
31150
|
+
"explicitId": "openwop.requirement.0251.baseline-refused",
|
|
31151
|
+
"citations": []
|
|
31152
|
+
},
|
|
31153
|
+
{
|
|
31154
|
+
"id": "openwop.it.v2-eval-regression-baseline.another-tenant-s-run-is-refused-exactly-as-an-id-that-never-existed",
|
|
31155
|
+
"file": "v2-eval-regression-baseline.test.ts",
|
|
31156
|
+
"line": 186,
|
|
31157
|
+
"title": "another tenant's run is refused exactly as an id that never existed",
|
|
31158
|
+
"explicitId": "openwop.requirement.0251.baseline-uniform",
|
|
31159
|
+
"citations": []
|
|
31160
|
+
},
|
|
31161
|
+
{
|
|
31162
|
+
"id": "openwop.it.v2-eval-regression-baseline.a-baseline-without-regression-and-regression-without-a-baseline-are-refused",
|
|
31163
|
+
"file": "v2-eval-regression-baseline.test.ts",
|
|
31164
|
+
"line": 203,
|
|
31165
|
+
"title": "a baseline without regression, and regression without a baseline, are refused",
|
|
31166
|
+
"explicitId": "openwop.requirement.0251.fields-refused",
|
|
31167
|
+
"citations": []
|
|
31168
|
+
},
|
|
31092
31169
|
{
|
|
31093
31170
|
"id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-suite-summary-and-event-fields-validate-and-the-negatives-are-refus",
|
|
31094
31171
|
"file": "v2-eval-trajectory-trials.test.ts",
|
package/scenario-majors.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$comment": "GENERATED by conformance/scripts/generate-scenario-majors.mjs (RFC 0168 §D.3). Do not edit; add a file to BOTH_MAJORS in the generator to target both majors.",
|
|
3
3
|
"counts": {
|
|
4
|
-
"files":
|
|
4
|
+
"files": 662,
|
|
5
5
|
"v1": 455,
|
|
6
|
-
"v2":
|
|
6
|
+
"v2": 222
|
|
7
7
|
},
|
|
8
8
|
"majors": {
|
|
9
9
|
"a2a-1-0-agent-card.test.ts": [
|
|
@@ -1482,6 +1482,9 @@
|
|
|
1482
1482
|
"v2-eval-mode-unadvertised-refused.test.ts": [
|
|
1483
1483
|
2
|
|
1484
1484
|
],
|
|
1485
|
+
"v2-eval-regression-baseline.test.ts": [
|
|
1486
|
+
2
|
|
1487
|
+
],
|
|
1485
1488
|
"v2-eval-trajectory-trials.test.ts": [
|
|
1486
1489
|
2
|
|
1487
1490
|
],
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_comment": "Provenance of @openwop/spec-artifacts (RFC 0168 §D.2). files: SHA-256 per file; the conformance suite compares the installed peer against dist/spec-artifacts.lock.json at start.",
|
|
3
3
|
"package": "@openwop/spec-artifacts",
|
|
4
|
-
"version": "2.46.
|
|
5
|
-
"corpusTag": "v2.46.
|
|
4
|
+
"version": "2.46.28",
|
|
5
|
+
"corpusTag": "v2.46.28",
|
|
6
6
|
"files": {
|
|
7
7
|
"api/.redocly.lint-ignore.yaml": "bf5a8350b88a72fa43f59605ed8d903ed24b6cfccda5e45509c9f6ed9ee4e712",
|
|
8
8
|
"api/asyncapi.yaml": "d5ecb9ee6114582be3b1f662c84bfac9ae96dae7bacb853e461168f70a8e1c7d",
|
|
9
9
|
"api/grpc/openwop.proto": "c3e72bb17cba514ee98feb6434e6c9b6ea6795bfd086489ec69fd882dd1ad977",
|
|
10
10
|
"api/openapi.yaml": "ac6de5303872664b78ff419c035915593e01599e02835e777a0123de1baeb398",
|
|
11
11
|
"api/redocly.yaml": "b0604c89b2ca6d5076ec25725c539dad44a741a811fe524439ee6daef8baa09f",
|
|
12
|
-
"api/seams-v2.yaml": "
|
|
13
|
-
"api/v2/asyncapi.yaml": "
|
|
14
|
-
"api/v2/openapi.yaml": "
|
|
12
|
+
"api/seams-v2.yaml": "1281524c0152c77a87db9bfdc01440aed092421619b377cdf85f862af41a3ae5",
|
|
13
|
+
"api/v2/asyncapi.yaml": "58ded9e1117f35837dc0eea8a44438a1def186e8fad0046cb8edacd07cb4906e",
|
|
14
|
+
"api/v2/openapi.yaml": "0fb51374a913d05366ddc721ef7dc872ba5489f8eabdd587c13b28561089c170",
|
|
15
15
|
"api/v2/redocly.yaml": "1e66b60e6118ad11a823bb620678be464d99dfe50a40e3e6f93ec9429b88b34c",
|
|
16
16
|
"schemas/README.md": "276fc1d51eb08e14e9127b9fd27761d7046ee33c12000f3a64a801322db8bf58",
|
|
17
17
|
"schemas/a2a-task-state.schema.json": "75d5049dea8bd873ff0e7546f1c60c8d36c219bec7084264360be54a8be30ae1",
|
|
@@ -117,7 +117,7 @@
|
|
|
117
117
|
"schemas/v2/artifact.schema.json": "85b9dc284d39b99d2a66fa8cbf47f2dd3d8526d3bd6cc411ad315b372de38975",
|
|
118
118
|
"schemas/v2/audit-verify-result.schema.json": "14af8e9283510f0a40e48a2059c929ee44e0fc9fa3cc66a95bfbe90b17c48297",
|
|
119
119
|
"schemas/v2/budget-policy.schema.json": "1de68095db0beb5c7a4be629de26e79f62ebbe748a88d46f36e768edfd1f0d68",
|
|
120
|
-
"schemas/v2/capabilities.schema.json": "
|
|
120
|
+
"schemas/v2/capabilities.schema.json": "b06b8142d288e839fe871bdddde35ca08d38d00da2a510fb9cb4ab201f37aa99",
|
|
121
121
|
"schemas/v2/certification-bundle.schema.json": "5cfa7fb5247160333e33a58b60d42bcd4e5269d8391d6a112d301e14828892ed",
|
|
122
122
|
"schemas/v2/channel-presence-payload.schema.json": "fc03df0c20cc12a25f7ee6a47503a4c5931d79fe3d5ff24815e8fd7c1bcb9a87",
|
|
123
123
|
"schemas/v2/channel-written-payload.schema.json": "ccecff3c71a3275ad8db035ac0e04db25e6f8d23cf091ddbcd5a60d1be1cbb6b",
|
|
@@ -212,14 +212,14 @@
|
|
|
212
212
|
"schemas/workspace-file.schema.json": "464de85c2a068243084ee9c1d969bc7cd5d8f7948574e58450d6493c38a0e1e4",
|
|
213
213
|
"spec/v1/alias-detectors.json": "22444c22bb88c568339ef512553f9fc38146e22b566d11ab8608f655d688d19c",
|
|
214
214
|
"spec/v1/capability-declaration-classes.json": "e7729aed5c4b4e1dd02abab0530f14cc95f5d4070fe51fb139e7f5cccefa00c6",
|
|
215
|
-
"spec/v1/core-standard-manifest.json": "
|
|
215
|
+
"spec/v1/core-standard-manifest.json": "ec3b1a7322067f80dd5562428fd2963a210d73da347eeb29661637745f2f7825",
|
|
216
216
|
"spec/v1/deprecations.json": "e723bc11d7272c71789ad1303bb7f21809a8a024abf57594f9d5c35dc199ece9",
|
|
217
217
|
"spec/v1/deprecations.schema.json": "3e393c405d2a41b467d8c5e3c468549078df1ce6a6d2588fc488097b95d9b55b",
|
|
218
218
|
"spec/v1/end-of-support.json": "35f0bf3b70fe78b16e2a2fd81a153fc27e703f6c16376c6fe7cdf1f4f2c4e6d4",
|
|
219
219
|
"spec/v1/event-codemap.json": "3da60d884157793a360da532a9fcbbfb5285636db325a74cec94b34622186d97",
|
|
220
220
|
"spec/v1/event-codemap.schema.json": "d05933b2e88103aff51a2f774df97b9bda065e0fc88dbd9b114cc47a32189174",
|
|
221
221
|
"spec/v1/extensions.json": "79a60754aa16cbdbdb604f8e5c00038af535a1690a4d07bd5ff7c38f0256a7bc",
|
|
222
|
-
"spec/v1/gaps.json": "
|
|
222
|
+
"spec/v1/gaps.json": "23f953428b6609c68917d2201f0b289fe855033452e895028dd1b6f9aae45e62",
|
|
223
223
|
"spec/v1/gaps.schema.json": "8fd83259f556553c9df0f53e7a82ca8c2a4771d471197f69b4e2ae0ceeacfd66",
|
|
224
224
|
"spec/v1/migrations.json": "4c50760be70cb644d02427bcaf5749606700edf4513f423903cf3b1d12f5c300",
|
|
225
225
|
"spec/v1/migrations.schema.json": "886779aa6c22e646db097f5df210adb018a4dd14a7b815465a18c8a7056c8f72",
|
|
@@ -285,7 +285,7 @@
|
|
|
285
285
|
"spec/v2/ext/sandbox-runtime-notes/README.md": "0d1290fb1e20b2e35c0b15051541ab6546e4e2a6f693b4f912cb3f7d6a65e92c",
|
|
286
286
|
"spec/v2/ext/webResearch/README.md": "b675e6f6646128d781b661f754c2393eed86e5935d16c3a22df582a5ce025b9b",
|
|
287
287
|
"spec/v2/facets/a2a.schema.json": "b68bf5e29e455e46c2ad7cfcccf5963730ec2310ff9db81884550d8df30fd6ef",
|
|
288
|
-
"spec/v2/facets/agents.schema.json": "
|
|
288
|
+
"spec/v2/facets/agents.schema.json": "198c01fdbb09deeb84f8f45f9f47976e696ac35ef087a2bc5c6f3c4fbb2ac370",
|
|
289
289
|
"spec/v2/facets/aiProviders.schema.json": "6e4327bcd48110686cc1adfd26ddd6fbbf4ff529c476dcc74eeff8c5ae9965ce",
|
|
290
290
|
"spec/v2/facets/auditLogIntegrity.schema.json": "9a7f81320c8f940c6f4129bc7dd895a4063231c5458a717a08c2cbf9bb886c46",
|
|
291
291
|
"spec/v2/facets/auth.schema.json": "7d98bc81a0f7d307460d039eebfebd21388091bfe1d538762583940f94c0bd2a",
|
|
@@ -318,9 +318,9 @@
|
|
|
318
318
|
"spec/v2/path-manifest.json": "20f598f56df460ad3ed50aba3d6bc2933ea16a3c96ce125da1c5dd6989d4155c",
|
|
319
319
|
"spec/v2/peer-dependency-aliases.json": "d10299280abee08258502925bc327293ee413e0108cd6e6ec75ff6110653308d",
|
|
320
320
|
"spec/v2/profiles.json": "b382b261c7844798575f7c1b37c934f0da19846016a2337d883d4f5ef6e05479",
|
|
321
|
-
"spec/v2/release.json": "
|
|
321
|
+
"spec/v2/release.json": "4bc3da43c14c5c1fa31fc3f7fda0a342d073dfbd5d053ff8557300d3bb67f067",
|
|
322
322
|
"spec/v2/retention-floors.json": "eaf3722d95c79947af1d4269ef85117e126518c588cfcf1a2b21b97269f51624",
|
|
323
|
-
"spec/v2/surface-baseline.json": "
|
|
323
|
+
"spec/v2/surface-baseline.json": "7d7a3073bfa01345b7560caffa0b6e382b0e82a205d53119c92fe98d4233d5b8"
|
|
324
324
|
},
|
|
325
|
-
"corpusCommit": "
|
|
325
|
+
"corpusCommit": "cb64bb3fe9bf7c37a6432da5207026e0d556cb32"
|
|
326
326
|
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0251 — an eval run names its modes and its regression baseline — and
|
|
3
|
+
* RFC 0244 §F pairing, which RFC 0251 makes causable. Pure judges over two
|
|
4
|
+
* eval runs and the refusals, for `v2-eval-regression-baseline.test.ts`.
|
|
5
|
+
*
|
|
6
|
+
* The fixtures share one `suiteId`. The base run (1.0.0) holds `base-only`
|
|
7
|
+
* (fails), `shared-a` and `shared-b` (pass); the candidate (1.1.0) holds the
|
|
8
|
+
* two shared tasks, unchanged, and `candidate-only` (passes). So the
|
|
9
|
+
* aggregates differ by 1/3 while every paired score is equal: a host that
|
|
10
|
+
* compares aggregates, not paired tasks, would call the candidate `improved`.
|
|
11
|
+
*
|
|
12
|
+
* @see RFCS/0251-eval-run-mode-and-baseline-selection.md
|
|
13
|
+
* @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md §F
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { readErrorCode } from './error-envelope.js';
|
|
17
|
+
|
|
18
|
+
export interface RegressionFinding { readonly ok: boolean; readonly message: string }
|
|
19
|
+
type Rec = Record<string, unknown>;
|
|
20
|
+
const isRec = (v: unknown): v is Rec => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
21
|
+
const close = (a: number, b: number): boolean => Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(b));
|
|
22
|
+
|
|
23
|
+
export const SHARED_TASKS = ['shared-a', 'shared-b'] as const;
|
|
24
|
+
|
|
25
|
+
/** §D: both carriers name the requested baseline, and `scoreDelta` is the aggregate difference. */
|
|
26
|
+
export function judgeBaselineEchoed(baselineRunId: string, events: readonly { type: string; payload: Rec }[], candidate: Rec, base: Rec): RegressionFinding[] {
|
|
27
|
+
const started = events.filter((e) => e.type === 'eval.started');
|
|
28
|
+
const reg = isRec(candidate['regression']) ? candidate['regression'] : {};
|
|
29
|
+
const want = typeof candidate['aggregateScore'] === 'number' && typeof base['aggregateScore'] === 'number' ? candidate['aggregateScore'] - base['aggregateScore'] : undefined;
|
|
30
|
+
const delta = reg['scoreDelta'];
|
|
31
|
+
return [
|
|
32
|
+
{ ok: started.length === 1 && started[0]!.payload['baselineRunId'] === baselineRunId, message: `eval.started MUST carry baselineRunId ${baselineRunId} (got ${JSON.stringify(started.map((e) => e.payload['baselineRunId']))})` },
|
|
33
|
+
{ ok: reg['baselineRunId'] === baselineRunId, message: `EvalSummary.regression.baselineRunId MUST be ${baselineRunId} (got ${JSON.stringify(reg['baselineRunId'])})` },
|
|
34
|
+
{ ok: want !== undefined && typeof delta === 'number' && close(delta, want), message: `regression.scoreDelta MUST be this run's aggregateScore minus the baseline's, ${String(want)} (got ${JSON.stringify(delta)})` },
|
|
35
|
+
];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** RFC 0244 §F: only the tasks in both runs are paired, and the host says how many. */
|
|
39
|
+
export function judgePaired(candidate: Rec): RegressionFinding[] {
|
|
40
|
+
const reg = isRec(candidate['regression']) ? candidate['regression'] : {};
|
|
41
|
+
return [{ ok: reg['pairedTaskCount'] === SHARED_TASKS.length, message: `regression.pairedTaskCount MUST count the ${SHARED_TASKS.length} tasks present in both runs, paired by taskId (got ${JSON.stringify(reg['pairedTaskCount'])})` }];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* RFC 0244 §F: every paired score is equal, so no method can distinguish the
|
|
46
|
+
* runs. The verdict MUST be present (the host advertises `regressionVerdict`)
|
|
47
|
+
* and MUST NOT be `improved` or `regressed`.
|
|
48
|
+
*/
|
|
49
|
+
export function judgeNoFalseVerdict(candidate: Rec): RegressionFinding[] {
|
|
50
|
+
const reg = isRec(candidate['regression']) ? candidate['regression'] : {};
|
|
51
|
+
const v = reg['verdict'];
|
|
52
|
+
return [
|
|
53
|
+
{ ok: typeof v === 'string', message: `a host advertising regressionVerdict MUST report regression.verdict (got ${JSON.stringify(v)})` },
|
|
54
|
+
{ ok: v !== 'improved' && v !== 'regressed', message: `with every paired task scoring the same, the verdict MUST NOT be improved or regressed: comparing aggregates is not pairing (got ${JSON.stringify(v)})` },
|
|
55
|
+
];
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface Answer { readonly status: number; readonly json: unknown; readonly text: string }
|
|
59
|
+
|
|
60
|
+
/** §B / §C: a refused create answers `400 validation_error` and does not echo the supplied id. */
|
|
61
|
+
export function judgeRefused(a: Answer, suppliedId: string | undefined, what: string): RegressionFinding[] {
|
|
62
|
+
const out: RegressionFinding[] = [
|
|
63
|
+
{ ok: a.status === 400, message: `${what} MUST be refused 400 before the run starts (got ${a.status})` },
|
|
64
|
+
{ ok: readErrorCode(a.json) === 'validation_error', message: `${what} MUST be refused with validation_error (got ${JSON.stringify(readErrorCode(a.json))})` },
|
|
65
|
+
];
|
|
66
|
+
if (suppliedId !== undefined) out.push({ ok: !a.text.includes(suppliedId), message: `the refusal of ${what} MUST NOT echo the supplied baselineRunId` });
|
|
67
|
+
return out;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** §C: another tenant's real run is refused exactly as an id that never existed. */
|
|
71
|
+
export function judgeUniform(unknown: Answer, foreign: Answer, foreignId: string): RegressionFinding[] {
|
|
72
|
+
return [
|
|
73
|
+
{ ok: foreign.status === unknown.status, message: `another tenant's run MUST be refused with the same status as an unknown id (${foreign.status} vs ${unknown.status})` },
|
|
74
|
+
{ ok: readErrorCode(foreign.json) === readErrorCode(unknown.json), message: `another tenant's run MUST be refused with the same code as an unknown id (${JSON.stringify(readErrorCode(foreign.json))} vs ${JSON.stringify(readErrorCode(unknown.json))})` },
|
|
75
|
+
{ ok: !foreign.text.includes(foreignId), message: "the refusal MUST NOT echo another tenant's run id" },
|
|
76
|
+
];
|
|
77
|
+
}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v2 — an eval run names its modes and its regression baseline (RFC 0251;
|
|
3
|
+
* the rules are the `agents.evalSuite.baselineSelection` facet description in
|
|
4
|
+
* `spec/v2/facets/agents.schema.json` and the `createRun` `evalModes` /
|
|
5
|
+
* `baselineRunId` descriptions), and RFC 0244 §F pairing, which a
|
|
6
|
+
* client-named baseline makes causable.
|
|
7
|
+
*
|
|
8
|
+
* Server-free leg:
|
|
9
|
+
* 1. `0251.shape` — the facet member is a boolean, and the derived
|
|
10
|
+
* `createRun` eval body carries `evalModes` (closed mode enum, non-empty,
|
|
11
|
+
* distinct) and `baselineRunId`.
|
|
12
|
+
*
|
|
13
|
+
* Host legs — gated on `baselineSelection`, `regression` in `modes`, and both
|
|
14
|
+
* `conformance-eval-regression-base` and `-candidate` in discovery `fixtures`.
|
|
15
|
+
* One base run (`evalModes: [golden]`), then one candidate run
|
|
16
|
+
* (`evalModes: [golden, regression]`, the base run as baseline). The two
|
|
17
|
+
* fixtures share a `suiteId`; every paired score is equal while the
|
|
18
|
+
* aggregates differ by 1/3. Judges in `lib/eval-regression-witness.ts`.
|
|
19
|
+
* 2. `0251.baseline-echoed` — `eval.started.baselineRunId` and
|
|
20
|
+
* `regression.baselineRunId` name the base run; `scoreDelta` is the
|
|
21
|
+
* aggregate difference.
|
|
22
|
+
* 3. `0244.regression-paired` — `pairedTaskCount` is 2 (needs
|
|
23
|
+
* `regressionVerdict`).
|
|
24
|
+
* 4. `0244.regression-no-false-verdict` — the verdict is present and is
|
|
25
|
+
* neither `improved` nor `regressed` (needs `regressionVerdict`).
|
|
26
|
+
* 5. `0251.baseline-refused` — an unknown id, and a completed run of
|
|
27
|
+
* another suite (`conformance-eval-trajectory`, when listed), are refused
|
|
28
|
+
* `400 validation_error` without the id.
|
|
29
|
+
* 6. `0251.baseline-uniform` — tenant B's run
|
|
30
|
+
* (`OPENWOP_TEST_TENANT_B_API_KEY`) is refused exactly as the unknown id.
|
|
31
|
+
* 7. `0251.fields-refused` — `baselineRunId` without `regression` in
|
|
32
|
+
* `evalModes`, and `regression` without a baseline, are refused.
|
|
33
|
+
*
|
|
34
|
+
* @see RFCS/0251-eval-run-mode-and-baseline-selection.md
|
|
35
|
+
* @see RFCS/0244-eval-trajectories-trials-and-recorded-fixtures.md §F
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
import { describe, it, expect } from 'vitest';
|
|
39
|
+
import { readFileSync } from 'node:fs';
|
|
40
|
+
import { join } from 'node:path';
|
|
41
|
+
import { randomBytes } from 'node:crypto';
|
|
42
|
+
import { driver, type OpenWOPResponse } from '../lib/driver.js';
|
|
43
|
+
import { v2Discovery, v2RefValidator } from '../lib/v2.js';
|
|
44
|
+
import { softSkip } from '../lib/soft-skip.js';
|
|
45
|
+
import { req } from '../lib/requirement-ids.js';
|
|
46
|
+
import { readErrorCode } from '../lib/error-envelope.js';
|
|
47
|
+
import { majorProfile } from '../lib/major-profile.js';
|
|
48
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
49
|
+
import { API_DIR } from '../lib/paths.js';
|
|
50
|
+
import { pollTerminal, readEvents, isRecord, type Ev } from '../lib/fixture-run-observer.js';
|
|
51
|
+
import { judgeBaselineEchoed, judgeNoFalseVerdict, judgePaired, judgeRefused, judgeUniform, type Answer, type RegressionFinding } from '../lib/eval-regression-witness.js';
|
|
52
|
+
|
|
53
|
+
const PROFILE = majorProfile(2);
|
|
54
|
+
const UNREADABLE = 'v2 discovery unreachable — /.well-known/openwop did not answer 200 JSON under OpenWOP-Version: 2.0';
|
|
55
|
+
const DOC = 'agents.schema.json §evalSuite.baselineSelection, createRun evalModes / baselineRunId (RFC 0251)';
|
|
56
|
+
const DOC_F = 'eval-summary.schema.json regression (RFC 0244 §F)';
|
|
57
|
+
const AGENT = 'core.conformance.agents.scripted';
|
|
58
|
+
const SUITE_URI = (id: string) => `https://conformance.openwop.dev/eval-suites/${id}`;
|
|
59
|
+
const BASE = 'conformance-eval-regression-base';
|
|
60
|
+
const CANDIDATE = 'conformance-eval-regression-candidate';
|
|
61
|
+
const OTHER = 'conformance-eval-trajectory';
|
|
62
|
+
const ID_SHAPE = 'openwop.requirement.0251.shape';
|
|
63
|
+
const ID_ECHOED = 'openwop.requirement.0251.baseline-echoed';
|
|
64
|
+
const ID_PAIRED = 'openwop.requirement.0244.regression-paired';
|
|
65
|
+
const ID_NO_FALSE = 'openwop.requirement.0244.regression-no-false-verdict';
|
|
66
|
+
const ID_REFUSED = 'openwop.requirement.0251.baseline-refused';
|
|
67
|
+
const ID_UNIFORM = 'openwop.requirement.0251.baseline-uniform';
|
|
68
|
+
const ID_FIELDS = 'openwop.requirement.0251.fields-refused';
|
|
69
|
+
|
|
70
|
+
type Skip = { skip: [Parameters<typeof softSkip>[0], string] };
|
|
71
|
+
type Done = { runId: string; events: Ev[]; summary: Record<string, unknown> };
|
|
72
|
+
|
|
73
|
+
async function http(fn: () => Promise<OpenWOPResponse>): Promise<OpenWOPResponse | null> {
|
|
74
|
+
try { return await fn(); } catch { return null; }
|
|
75
|
+
}
|
|
76
|
+
const answer = (r: OpenWOPResponse): Answer => ({ status: r.status, json: r.json, text: r.text });
|
|
77
|
+
const expectAll = (id: string, doc: string, findings: readonly RegressionFinding[]): void => {
|
|
78
|
+
for (const x of findings) expect(x.ok, req(id, doc, x.message)).toBe(true);
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
async function evalSuiteFacet(): Promise<Record<string, unknown> | null | undefined> {
|
|
82
|
+
let doc: Record<string, unknown> | null;
|
|
83
|
+
try { doc = await v2Discovery(); } catch { return undefined; }
|
|
84
|
+
const agents = doc?.['agents'];
|
|
85
|
+
return isRecord(agents) && isRecord(agents['evalSuite']) ? agents['evalSuite'] : null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** The gate every host leg shares. */
|
|
89
|
+
async function gate(): Promise<Skip | { facet: Record<string, unknown> }> {
|
|
90
|
+
const facet = await evalSuiteFacet();
|
|
91
|
+
if (facet === undefined) return { skip: ['blocked', UNREADABLE] };
|
|
92
|
+
if (facet === null || facet['baselineSelection'] !== true) return { skip: ['inapplicable', 'the host does not advertise agents.evalSuite.baselineSelection'] };
|
|
93
|
+
if (!Array.isArray(facet['modes']) || !facet['modes'].includes('regression')) return { skip: ['inapplicable', 'the host does not advertise regression in agents.evalSuite.modes'] };
|
|
94
|
+
for (const id of [BASE, CANDIDATE]) if (!isFixtureAdvertised(id)) return { skip: ['inapplicable', `the host does not advertise fixture ${id}`] };
|
|
95
|
+
return { facet };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const create = (id: string, extra: Record<string, unknown>, headers?: Record<string, string>) =>
|
|
99
|
+
http(() => driver.post(PROFILE.runsPath, { mode: 'eval', evalSuiteRef: SUITE_URI(id), agentId: AGENT, ...extra }, headers ? { headers } : {}));
|
|
100
|
+
|
|
101
|
+
async function complete(id: string, extra: Record<string, unknown>): Promise<Skip | Done> {
|
|
102
|
+
const res = await create(id, extra);
|
|
103
|
+
if (res === null) return { skip: ['blocked', 'POST /runs unreachable (fetch failed)'] };
|
|
104
|
+
if (res.status === 429) return { skip: ['blocked', 'POST /runs answered 429 — the run budget, not the wire'] };
|
|
105
|
+
const runId = isRecord(res.json) && typeof res.json['runId'] === 'string' ? res.json['runId'] : undefined;
|
|
106
|
+
if ((res.status !== 201 && res.status !== 202) || runId === undefined) return { skip: ['blocked', `createRun of ${id} answered ${res.status} ${readErrorCode(res.json) ?? ''}`.trim()] };
|
|
107
|
+
const terminal = await pollTerminal(PROFILE, runId, { timeoutMs: 60_000 });
|
|
108
|
+
if (terminal.snapshot['status'] !== 'completed') return { skip: ['blocked', `eval run ${runId} ended ${String(terminal.snapshot['status'])}, not completed`] };
|
|
109
|
+
const s = await http(() => driver.get(`${PROFILE.runsPath}/${PROFILE.idSegment(runId)}/eval-summary`));
|
|
110
|
+
if (s === null || s.status !== 200 || !isRecord(s.json)) return { skip: ['blocked', `GET …/eval-summary answered ${s?.status ?? 'nothing'}`] };
|
|
111
|
+
return { runId, events: await readEvents(PROFILE, runId), summary: s.json };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
let pair: Promise<Skip | { base: Done; cand: Done }> | undefined;
|
|
115
|
+
/** Base, then candidate against it, once per worker: the host legs read the same two runs. */
|
|
116
|
+
function runs(): Promise<Skip | { base: Done; cand: Done }> {
|
|
117
|
+
pair ??= (async () => {
|
|
118
|
+
const base = await complete(BASE, { evalModes: ['golden'] });
|
|
119
|
+
if ('skip' in base) return base;
|
|
120
|
+
const cand = await complete(CANDIDATE, { evalModes: ['golden', 'regression'], baselineRunId: base.runId });
|
|
121
|
+
if ('skip' in cand) return cand;
|
|
122
|
+
return { base, cand };
|
|
123
|
+
})();
|
|
124
|
+
return pair;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** An id in the caller's own tenant, with the runId grammar, that the host never minted. */
|
|
128
|
+
const unknownBeside = (runId: string): string => `${runId.split('/')[0]}/${randomBytes(16).toString('hex')}`;
|
|
129
|
+
|
|
130
|
+
describe('v2 RFC 0251 — wire shape (server-free)', () => {
|
|
131
|
+
it('the baselineSelection member and the createRun eval properties are in the derived contract', () => {
|
|
132
|
+
const v = v2RefValidator('capabilities.schema.json#/properties/agents/properties/evalSuite/properties/baselineSelection');
|
|
133
|
+
expect(v(true).ok, req(ID_SHAPE, DOC, 'baselineSelection: true MUST validate')).toBe(true);
|
|
134
|
+
expect(v('yes').ok, req(ID_SHAPE, DOC, 'baselineSelection MUST be a boolean')).toBe(false);
|
|
135
|
+
const api = readFileSync(join(API_DIR, 'v2', 'openapi.yaml'), 'utf8');
|
|
136
|
+
const at = api.indexOf('operationId: createRun');
|
|
137
|
+
const next = api.indexOf('operationId:', at + 1);
|
|
138
|
+
const block = at < 0 ? '' : api.slice(at, next < 0 ? undefined : next);
|
|
139
|
+
expect(block, req(ID_SHAPE, DOC, 'createRun MUST carry evalModes over the RFC 0081 §D modes')).toMatch(/evalModes:[\s\S]*?enum:\s*\n?\s*-?\s*\[?\s*golden/);
|
|
140
|
+
expect(block, req(ID_SHAPE, DOC, 'evalModes MUST be non-empty and distinct')).toMatch(/evalModes:\s*\n\s*type: array\s*\n\s*minItems: 1\s*\n\s*uniqueItems: true/);
|
|
141
|
+
expect(block, req(ID_SHAPE, DOC, 'createRun MUST carry baselineRunId as a runId')).toMatch(/baselineRunId:\s*\n\s*\$ref: \.\.\/\.\.\/schemas\/v2\/ids\.schema\.json#\/\$defs\/runId/);
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
describe('v2 RFC 0251 — a client-named regression baseline (host)', () => {
|
|
146
|
+
it('the candidate run names the baseline it was given, and scoreDelta is the aggregate difference', async () => {
|
|
147
|
+
const g = await gate();
|
|
148
|
+
if ('skip' in g) return softSkip(...g.skip);
|
|
149
|
+
const r = await runs();
|
|
150
|
+
if ('skip' in r) return softSkip(...r.skip);
|
|
151
|
+
expectAll(ID_ECHOED, DOC, judgeBaselineEchoed(r.base.runId, r.cand.events, r.cand.summary, r.base.summary));
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
it('only the tasks in both runs are paired (RFC 0244 §F)', async () => {
|
|
155
|
+
const g = await gate();
|
|
156
|
+
if ('skip' in g) return softSkip(...g.skip);
|
|
157
|
+
if (g.facet['regressionVerdict'] !== true) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite.regressionVerdict');
|
|
158
|
+
const r = await runs();
|
|
159
|
+
if ('skip' in r) return softSkip(...r.skip);
|
|
160
|
+
expectAll(ID_PAIRED, DOC_F, judgePaired(r.cand.summary));
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it('equal paired scores never yield improved or regressed (RFC 0244 §F)', async () => {
|
|
164
|
+
const g = await gate();
|
|
165
|
+
if ('skip' in g) return softSkip(...g.skip);
|
|
166
|
+
if (g.facet['regressionVerdict'] !== true) return softSkip('inapplicable', 'the host does not advertise agents.evalSuite.regressionVerdict');
|
|
167
|
+
const r = await runs();
|
|
168
|
+
if ('skip' in r) return softSkip(...r.skip);
|
|
169
|
+
expectAll(ID_NO_FALSE, DOC_F, judgeNoFalseVerdict(r.cand.summary));
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it('an unknown baseline, and a run of another suite, are refused without the id', async () => {
|
|
173
|
+
const g = await gate();
|
|
174
|
+
if ('skip' in g) return softSkip(...g.skip);
|
|
175
|
+
const r = await runs();
|
|
176
|
+
if ('skip' in r) return softSkip(...r.skip);
|
|
177
|
+
const unknownId = unknownBeside(r.base.runId);
|
|
178
|
+
const unknown = await create(CANDIDATE, { evalModes: ['golden', 'regression'], baselineRunId: unknownId });
|
|
179
|
+
if (unknown === null) return softSkip('blocked', 'POST /runs unreachable (fetch failed)');
|
|
180
|
+
expectAll(ID_REFUSED, DOC, judgeRefused(answer(unknown), unknownId, 'an unknown baselineRunId'));
|
|
181
|
+
const other = isFixtureAdvertised(OTHER) ? await complete(OTHER, {}) : undefined;
|
|
182
|
+
const res = other !== undefined && !('skip' in other) ? await create(CANDIDATE, { evalModes: ['golden', 'regression'], baselineRunId: other.runId }) : null;
|
|
183
|
+
if (other !== undefined && !('skip' in other) && res !== null) expectAll(ID_REFUSED, DOC, judgeRefused(answer(res), other.runId, 'a baseline that ran another suite'));
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it("another tenant's run is refused exactly as an id that never existed", async () => {
|
|
187
|
+
const g = await gate();
|
|
188
|
+
if ('skip' in g) return softSkip(...g.skip);
|
|
189
|
+
const tenantB = process.env['OPENWOP_TEST_TENANT_B_API_KEY']?.trim();
|
|
190
|
+
if (!tenantB) return softSkip('blocked', 'OPENWOP_TEST_TENANT_B_API_KEY (a credential bound to a second tenant) is not set');
|
|
191
|
+
if (tenantB === process.env['OPENWOP_API_KEY']) return softSkip('blocked', 'OPENWOP_TEST_TENANT_B_API_KEY equals OPENWOP_API_KEY — the leg needs a second tenant');
|
|
192
|
+
const foreign = await create(BASE, { evalModes: ['golden'] }, { Authorization: `Bearer ${tenantB}` });
|
|
193
|
+
const foreignId = foreign && isRecord(foreign.json) && typeof foreign.json['runId'] === 'string' ? foreign.json['runId'] : undefined;
|
|
194
|
+
if (foreignId === undefined) return softSkip('blocked', `tenant B's createRun of ${BASE} answered ${foreign?.status ?? 'nothing'}`);
|
|
195
|
+
const r = await runs();
|
|
196
|
+
if ('skip' in r) return softSkip(...r.skip);
|
|
197
|
+
const unknown = await create(CANDIDATE, { evalModes: ['golden', 'regression'], baselineRunId: unknownBeside(r.base.runId) });
|
|
198
|
+
const viaForeign = await create(CANDIDATE, { evalModes: ['golden', 'regression'], baselineRunId: foreignId });
|
|
199
|
+
if (unknown === null || viaForeign === null) return softSkip('blocked', 'POST /runs unreachable (fetch failed)');
|
|
200
|
+
expectAll(ID_UNIFORM, DOC, judgeUniform(answer(unknown), answer(viaForeign), foreignId));
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
it('a baseline without regression, and regression without a baseline, are refused', async () => {
|
|
204
|
+
const g = await gate();
|
|
205
|
+
if ('skip' in g) return softSkip(...g.skip);
|
|
206
|
+
const r = await runs();
|
|
207
|
+
if ('skip' in r) return softSkip(...r.skip);
|
|
208
|
+
const noMode = await create(CANDIDATE, { evalModes: ['golden'], baselineRunId: r.base.runId });
|
|
209
|
+
const noBase = await create(CANDIDATE, { evalModes: ['golden', 'regression'] });
|
|
210
|
+
if (noMode === null || noBase === null) return softSkip('blocked', 'POST /runs unreachable (fetch failed)');
|
|
211
|
+
expectAll(ID_FIELDS, DOC, judgeRefused(answer(noMode), undefined, 'baselineRunId without regression in evalModes'));
|
|
212
|
+
expectAll(ID_FIELDS, DOC, judgeRefused(answer(noBase), undefined, 'regression in evalModes without baselineRunId'));
|
|
213
|
+
});
|
|
214
|
+
});
|
|
@@ -239,7 +239,7 @@ describe('v2 RFC 0244 — wire shape (server-free)', () => {
|
|
|
239
239
|
});
|
|
240
240
|
|
|
241
241
|
it('the eval-suite fixtures validate against the v2 suite schema', () => {
|
|
242
|
-
for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss', 'conformance-eval-trajectory-gate', 'conformance-eval-trajectory-gate-mean', 'conformance-eval-fixture-error', 'conformance-eval-counter-live', 'conformance-eval-counter-miss', 'conformance-eval-efficiency-ungated', 'conformance-eval-memory-isolation']) {
|
|
242
|
+
for (const id of ['conformance-eval-trajectory', 'conformance-eval-trials', 'conformance-eval-fixture-miss', 'conformance-eval-trajectory-gate', 'conformance-eval-trajectory-gate-mean', 'conformance-eval-fixture-error', 'conformance-eval-counter-live', 'conformance-eval-counter-miss', 'conformance-eval-efficiency-ungated', 'conformance-eval-memory-isolation', 'conformance-eval-regression-base', 'conformance-eval-regression-candidate']) {
|
|
243
243
|
const r = suiteSchema(fixture(id));
|
|
244
244
|
expect(r.ok, req(ID_FIXTURES, 'fixtures.md §eval-suites', `${id} MUST validate: ${r.errors}`)).toBe(true);
|
|
245
245
|
}
|