@openwop/openwop-conformance 2.0.0-rc.55 → 2.0.0-rc.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/lib/scenario-disposition.js +32 -0
- package/dist/lib/soft-skip.js +39 -8
- package/dist/spec-artifacts.lock.json +2 -2
- package/package.json +2 -2
- package/requirements.json +1 -1
- package/schemas/CORPUS-STAMP.json +7 -7
- package/src/lib/scenario-disposition.ts +34 -0
- package/src/lib/soft-skip.ts +42 -7
- package/src/scenarios/v2-coherence-not-in-bundle.test.ts +11 -2
- package/src/setup.ts +14 -17
package/README.md
CHANGED
|
@@ -3,7 +3,12 @@
|
|
|
3
3
|
**openwop is an open, wire-level protocol for multi-agent workflow orchestration** — a single contract for runs in which LLM agents, deterministic tools, sub-workflows, and human reviewers collaborate, with durable suspend / resume, replay, version negotiation, and observability owned by the protocol itself. This package is the black-box conformance suite: point it at any OpenWOP-compliant server (your own or a third party's) and it issues real HTTP requests against the spec'd endpoints and asserts that responses match.
|
|
4
4
|
|
|
5
5
|
```bash
|
|
6
|
-
|
|
6
|
+
# Install BOTH packages at the SAME explicit version — the suite declares
|
|
7
|
+
# @openwop/spec-artifacts as an exact-pinned peer, and `npm i --legacy-peer-deps`
|
|
8
|
+
# on the suite alone does not pull it (a host measured "corpus stamp MISMATCH —
|
|
9
|
+
# missing @openwop/spec-artifacts" on 2026-09-05). Pre-release 2.x is on the
|
|
10
|
+
# `next` dist-tag; pin the version, never the tag (runbook §0.2b).
|
|
11
|
+
npm install @openwop/openwop-conformance@2.0.0-rc.57 @openwop/spec-artifacts@2.0.0-rc.57
|
|
7
12
|
# or run without install:
|
|
8
13
|
npx @openwop/openwop-conformance --base-url https://api.example.com --api-key hk_test_...
|
|
9
14
|
```
|
|
@@ -77,6 +77,38 @@ export function requirementIdForFile(basename) {
|
|
|
77
77
|
* the thing that passed.
|
|
78
78
|
*/
|
|
79
79
|
export const PARTIAL_WITNESS_PREFIX = 'partial-witness: ';
|
|
80
|
+
/**
|
|
81
|
+
* The per-`it` record (RFC 0148 §A at test granularity), as `setup.ts`
|
|
82
|
+
* computes it in `afterEach`. Pure so a lib test can pin it:
|
|
83
|
+
* - fail ⇒ executed-fail (detail = the first error message)
|
|
84
|
+
* - pass with ≥ 1 assertion ⇒ executed-pass
|
|
85
|
+
* - a behaviorGate entry journaled during the test ⇒ that gate's disposition
|
|
86
|
+
* - pass with 0 assertions ⇒ the softSkip note written DURING THIS TEST
|
|
87
|
+
* (`inapplicable` / `skipped` / `blocked`, worst-first),
|
|
88
|
+
* else `blocked` + the unclassified-return marker
|
|
89
|
+
* - vitest skip (ctx.skip / it.skip) ⇒ the note written before the skip, else `skipped`
|
|
90
|
+
*
|
|
91
|
+
* rc.56: the fourth line is new. Until then a zero-assertion pass consulted the
|
|
92
|
+
* journal only, so a leg that returned `softSkip('inapplicable', …)` was
|
|
93
|
+
* recorded `blocked / unclassified return` at `it` granularity while its file
|
|
94
|
+
* row (which does read the notes — `resolveFileRecord`) was `inapplicable`.
|
|
95
|
+
* A bundle with any `blocked` row does not certify (RFC 0168 §E.1), so the
|
|
96
|
+
* dishonest per-`it` rows denied certification to every profile on a host that
|
|
97
|
+
* simply did not advertise the gated surface.
|
|
98
|
+
*/
|
|
99
|
+
export function resolveItRecord(state, assertionCalls, gate, noted, firstError) {
|
|
100
|
+
if (state === 'fail')
|
|
101
|
+
return { disposition: 'executed-fail', detail: `the test executed and failed: ${(firstError ?? 'no message').slice(0, 300)}` };
|
|
102
|
+
if (state === 'pass' && assertionCalls > 0)
|
|
103
|
+
return { disposition: 'executed-pass' };
|
|
104
|
+
if (gate !== undefined)
|
|
105
|
+
return { disposition: gate.disposition, detail: gate.detail ?? `${gate.disposition} (gate recorded no reason)` };
|
|
106
|
+
if (noted !== null)
|
|
107
|
+
return { disposition: noted.kind, detail: noted.reason };
|
|
108
|
+
if (state === 'pass')
|
|
109
|
+
return { disposition: 'blocked', detail: 'unclassified return: the test passed with zero assertions and recorded no reason — RFC 0148 §A resolves it to blocked, never to a pass' };
|
|
110
|
+
return { disposition: 'skipped', detail: 'vitest skipped the test (ctx.skip / it.skip) without a recorded gate reason' };
|
|
111
|
+
}
|
|
80
112
|
/** Worker half: fold a file's per-test states (+ any gate-recorded reason) into
|
|
81
113
|
* the ONE disposition the file records. */
|
|
82
114
|
export function fileDisposition(states, gateReason, assertionCount) {
|
package/dist/lib/soft-skip.js
CHANGED
|
@@ -22,12 +22,24 @@
|
|
|
22
22
|
* resolves it to blocked" — and stays UNCLASSIFIED for certification (a floor
|
|
23
23
|
* row with that disposition still rejects), so the honest bundle row and the
|
|
24
24
|
* pressure to say why both survive.
|
|
25
|
+
*
|
|
26
|
+
* rc.56: every note also carries a sequence number, so the per-`it` row can
|
|
27
|
+
* read the notes written DURING ITS OWN TEST (`softSkipMark()` at test start,
|
|
28
|
+
* `softSkipDispositionSince(file, mark)` at test end). Until rc.56 the
|
|
29
|
+
* per-`it` row consulted only the journal's `behaviorGate` entries, never a
|
|
30
|
+
* softSkip note, so a leg that returned `softSkip('inapplicable', 'a2a facet
|
|
31
|
+
* not advertised')` was recorded `blocked / unclassified return` at `it`
|
|
32
|
+
* granularity while its file row was honestly `inapplicable` — and a bundle
|
|
33
|
+
* with any `blocked` row does not certify (RFC 0168 §E.1). Forty-five such
|
|
34
|
+
* rows on a host that simply does not advertise A2A/MCP denied certification
|
|
35
|
+
* to every profile it claimed.
|
|
25
36
|
*/
|
|
26
37
|
import { expect } from 'vitest';
|
|
27
38
|
import { basename } from 'node:path';
|
|
28
39
|
/** Detail marker the runner writes for a zero-assertion file that noted nothing. */
|
|
29
40
|
export const UNCLASSIFIED_RETURN_DETAIL = 'every test returned early with zero assertions and no recorded reason — unclassified return; RFC 0148 §A resolves it to blocked (add softSkip(kind, reason) at the early return)';
|
|
30
41
|
const notes = new Map();
|
|
42
|
+
let seq = 0;
|
|
31
43
|
function currentFile() {
|
|
32
44
|
try {
|
|
33
45
|
const p = expect.getState().testPath;
|
|
@@ -43,8 +55,9 @@ export function softSkip(kind, reason) {
|
|
|
43
55
|
if (file === null)
|
|
44
56
|
return undefined;
|
|
45
57
|
const arr = notes.get(file) ?? [];
|
|
46
|
-
|
|
47
|
-
|
|
58
|
+
// Every call is recorded with its own sequence number so a per-test window
|
|
59
|
+
// sees it; the file-level join de-duplicates identical (kind, reason) pairs.
|
|
60
|
+
arr.push({ kind, reason, seq: ++seq });
|
|
48
61
|
notes.set(file, arr);
|
|
49
62
|
return undefined;
|
|
50
63
|
}
|
|
@@ -62,6 +75,17 @@ export function seamAbsent(reason) {
|
|
|
62
75
|
return softSkip('blocked', reason);
|
|
63
76
|
}
|
|
64
77
|
const RANK = { blocked: 0, skipped: 1, inapplicable: 2 };
|
|
78
|
+
function fold(arr) {
|
|
79
|
+
if (arr.length === 0)
|
|
80
|
+
return null;
|
|
81
|
+
const uniq = [];
|
|
82
|
+
for (const n of arr)
|
|
83
|
+
if (!uniq.some((u) => u.kind === n.kind && u.reason === n.reason))
|
|
84
|
+
uniq.push(n);
|
|
85
|
+
const kind = [...uniq].sort((a, b) => RANK[a.kind] - RANK[b.kind])[0].kind;
|
|
86
|
+
const reason = uniq.map((n) => (uniq.length > 1 ? `[${n.kind}] ${n.reason}` : n.reason)).join('; ');
|
|
87
|
+
return { kind, reason };
|
|
88
|
+
}
|
|
65
89
|
/**
|
|
66
90
|
* The noted disposition for a file, worst-first when mixed (`blocked` beats
|
|
67
91
|
* `skipped` beats `inapplicable` — a file that could not check one thing is
|
|
@@ -69,14 +93,21 @@ const RANK = { blocked: 0, skipped: 1, inapplicable: 2 };
|
|
|
69
93
|
* the reasons joined. `null` when nothing was noted.
|
|
70
94
|
*/
|
|
71
95
|
export function softSkipDisposition(file) {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
96
|
+
return fold(notes.get(file) ?? []);
|
|
97
|
+
}
|
|
98
|
+
/** A position in the note sequence; pass it to `softSkipDispositionSince`. */
|
|
99
|
+
export function softSkipMark() {
|
|
100
|
+
return seq;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* The noted disposition for a file counting only notes written AFTER `mark`
|
|
104
|
+
* — the notes of the test that is ending. Same fold as the file rule.
|
|
105
|
+
*/
|
|
106
|
+
export function softSkipDispositionSince(file, mark) {
|
|
107
|
+
return fold((notes.get(file) ?? []).filter((n) => n.seq > mark));
|
|
78
108
|
}
|
|
79
109
|
/** Test hook. */
|
|
80
110
|
export function resetSoftSkips() {
|
|
81
111
|
notes.clear();
|
|
112
|
+
seq = 0;
|
|
82
113
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
2
|
"package": "@openwop/spec-artifacts",
|
|
3
|
-
"version": "2.0.0-rc.
|
|
4
|
-
"stampSha256": "
|
|
3
|
+
"version": "2.0.0-rc.57",
|
|
4
|
+
"stampSha256": "cc3f5bc494e60e39d45ae5c1b09839c60b7ec5974b1fb6f59d7452cd68782a28"
|
|
5
5
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openwop/openwop-conformance",
|
|
3
|
-
"version": "2.0.0-rc.
|
|
3
|
+
"version": "2.0.0-rc.57",
|
|
4
4
|
"description": "Production-ready black-box conformance suite for OpenWOP v1.0 compliant servers.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -56,6 +56,6 @@
|
|
|
56
56
|
"@openwop/spec-artifacts": "file:../spec-artifacts"
|
|
57
57
|
},
|
|
58
58
|
"peerDependencies": {
|
|
59
|
-
"@openwop/spec-artifacts": "2.0.0-rc.
|
|
59
|
+
"@openwop/spec-artifacts": "2.0.0-rc.57"
|
|
60
60
|
}
|
|
61
61
|
}
|
package/requirements.json
CHANGED
|
@@ -26228,7 +26228,7 @@
|
|
|
26228
26228
|
{
|
|
26229
26229
|
"id": "openwop.it.v2-coherence-not-in-bundle.the-bundle-schema-rejects-a-v3-bundle-carrying-a-corpus-coherence-id-and-accepts",
|
|
26230
26230
|
"file": "v2-coherence-not-in-bundle.test.ts",
|
|
26231
|
-
"line":
|
|
26231
|
+
"line": 98,
|
|
26232
26232
|
"title": "the bundle schema rejects a v3 bundle carrying a corpus-coherence id and accepts one under openwop.requirement.",
|
|
26233
26233
|
"explicitId": "openwop.requirement.0168.coherence-not-in-bundle.schema-forbids-coherence-ids",
|
|
26234
26234
|
"citations": [
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_comment": "Provenance of @openwop/spec-artifacts (RFC 0168 §D.2). files: SHA-256 per file; the conformance suite compares the installed peer against dist/spec-artifacts.lock.json at start.",
|
|
3
3
|
"package": "@openwop/spec-artifacts",
|
|
4
|
-
"version": "2.0.0-rc.
|
|
4
|
+
"version": "2.0.0-rc.57",
|
|
5
5
|
"corpusTag": null,
|
|
6
6
|
"files": {
|
|
7
7
|
"api/.redocly.lint-ignore.yaml": "bf5a8350b88a72fa43f59605ed8d903ed24b6cfccda5e45509c9f6ed9ee4e712",
|
|
@@ -9,9 +9,9 @@
|
|
|
9
9
|
"api/grpc/openwop.proto": "c3e72bb17cba514ee98feb6434e6c9b6ea6795bfd086489ec69fd882dd1ad977",
|
|
10
10
|
"api/openapi.yaml": "39081c59fb696159806b0f2f9a42e7e9ff830d622fcf2ad4159b21357580a955",
|
|
11
11
|
"api/redocly.yaml": "b0604c89b2ca6d5076ec25725c539dad44a741a811fe524439ee6daef8baa09f",
|
|
12
|
-
"api/seams-v2.yaml": "
|
|
13
|
-
"api/v2/asyncapi.yaml": "
|
|
14
|
-
"api/v2/openapi.yaml": "
|
|
12
|
+
"api/seams-v2.yaml": "275c0837bfbde0e6a809455243a408b72634ebf08cc5be9d8edc4a88ec2b1dfd",
|
|
13
|
+
"api/v2/asyncapi.yaml": "2c8959b0e89d7a818be8913927d8d6eb48ad1c1fade78e156134d094f8ed84d0",
|
|
14
|
+
"api/v2/openapi.yaml": "347ccba185bd6f6db33bbed3915831474773e1e26300ebc6af8988e3916fae5a",
|
|
15
15
|
"api/v2/redocly.yaml": "1e66b60e6118ad11a823bb620678be464d99dfe50a40e3e6f93ec9429b88b34c",
|
|
16
16
|
"schemas/README.md": "0c0b737ffcf8f30e7d2809cec8a498232de710f41443212922ad8337cdde0b51",
|
|
17
17
|
"schemas/a2a-task-state.schema.json": "c9365918f993f943b4b619d42551eb066a1ed33a08d895d51b395432a5b1f1bc",
|
|
@@ -199,7 +199,7 @@
|
|
|
199
199
|
"schemas/workspace-file.schema.json": "464de85c2a068243084ee9c1d969bc7cd5d8f7948574e58450d6493c38a0e1e4",
|
|
200
200
|
"spec/v1/alias-detectors.json": "fee4594ef49953953ffcd0b3813300067d16b3e65ebff2aac722034ac9b3f545",
|
|
201
201
|
"spec/v1/capability-declaration-classes.json": "e7729aed5c4b4e1dd02abab0530f14cc95f5d4070fe51fb139e7f5cccefa00c6",
|
|
202
|
-
"spec/v1/core-standard-manifest.json": "
|
|
202
|
+
"spec/v1/core-standard-manifest.json": "bd2cb4609ed1319c02899f26a8e78fc6f95aa1a7abd966e576a5b38d2b47c957",
|
|
203
203
|
"spec/v1/deprecations.json": "1d5acb69a9b8ccb57275a95605f74aef1d920685f8407c9d382a46b59dc803bb",
|
|
204
204
|
"spec/v1/deprecations.schema.json": "18c87e78bedc210431f795ae44c5b5d202f2f3317850d5cf86867d4f1fa1cdfb",
|
|
205
205
|
"spec/v1/event-codemap.json": "3da60d884157793a360da532a9fcbbfb5285636db325a74cec94b34622186d97",
|
|
@@ -230,8 +230,8 @@
|
|
|
230
230
|
"spec/v2/path-manifest.json": "034152e09b1458c66810d4050e20a273b2b9b8fe2d92b66e8b819477de58a1be",
|
|
231
231
|
"spec/v2/peer-dependency-aliases.json": "d10299280abee08258502925bc327293ee413e0108cd6e6ec75ff6110653308d",
|
|
232
232
|
"spec/v2/profiles.json": "5019ac8209540bfbeee9c3232530d4e599be0ab926d817ff48edcec8419eb209",
|
|
233
|
-
"spec/v2/release.json": "
|
|
233
|
+
"spec/v2/release.json": "0e87e35dacdd91562d515ea4c3074cabbc064e022c98ff22c645419f452d4088",
|
|
234
234
|
"spec/v2/retention-floors.json": "eaf3722d95c79947af1d4269ef85117e126518c588cfcf1a2b21b97269f51624"
|
|
235
235
|
},
|
|
236
|
-
"corpusCommit": "
|
|
236
|
+
"corpusCommit": "2c21c36ab32a313d6d170175d80a572ec14ba3d7"
|
|
237
237
|
}
|
|
@@ -79,6 +79,40 @@ export const PARTIAL_WITNESS_PREFIX = 'partial-witness: ';
|
|
|
79
79
|
|
|
80
80
|
export type FileTestState = 'pass' | 'fail' | 'skip';
|
|
81
81
|
|
|
82
|
+
/**
|
|
83
|
+
* The per-`it` record (RFC 0148 §A at test granularity), as `setup.ts`
|
|
84
|
+
* computes it in `afterEach`. Pure so a lib test can pin it:
|
|
85
|
+
* - fail ⇒ executed-fail (detail = the first error message)
|
|
86
|
+
* - pass with ≥ 1 assertion ⇒ executed-pass
|
|
87
|
+
* - a behaviorGate entry journaled during the test ⇒ that gate's disposition
|
|
88
|
+
* - pass with 0 assertions ⇒ the softSkip note written DURING THIS TEST
|
|
89
|
+
* (`inapplicable` / `skipped` / `blocked`, worst-first),
|
|
90
|
+
* else `blocked` + the unclassified-return marker
|
|
91
|
+
* - vitest skip (ctx.skip / it.skip) ⇒ the note written before the skip, else `skipped`
|
|
92
|
+
*
|
|
93
|
+
* rc.56: the fourth line is new. Until then a zero-assertion pass consulted the
|
|
94
|
+
* journal only, so a leg that returned `softSkip('inapplicable', …)` was
|
|
95
|
+
* recorded `blocked / unclassified return` at `it` granularity while its file
|
|
96
|
+
* row (which does read the notes — `resolveFileRecord`) was `inapplicable`.
|
|
97
|
+
* A bundle with any `blocked` row does not certify (RFC 0168 §E.1), so the
|
|
98
|
+
* dishonest per-`it` rows denied certification to every profile on a host that
|
|
99
|
+
* simply did not advertise the gated surface.
|
|
100
|
+
*/
|
|
101
|
+
export function resolveItRecord(
|
|
102
|
+
state: FileTestState,
|
|
103
|
+
assertionCalls: number,
|
|
104
|
+
gate: { disposition: 'inapplicable' | 'skipped'; detail?: string } | undefined,
|
|
105
|
+
noted: { kind: 'inapplicable' | 'skipped' | 'blocked'; reason: string } | null,
|
|
106
|
+
firstError?: string,
|
|
107
|
+
): { disposition: Disposition; detail?: string } {
|
|
108
|
+
if (state === 'fail') return { disposition: 'executed-fail', detail: `the test executed and failed: ${(firstError ?? 'no message').slice(0, 300)}` };
|
|
109
|
+
if (state === 'pass' && assertionCalls > 0) return { disposition: 'executed-pass' };
|
|
110
|
+
if (gate !== undefined) return { disposition: gate.disposition, detail: gate.detail ?? `${gate.disposition} (gate recorded no reason)` };
|
|
111
|
+
if (noted !== null) return { disposition: noted.kind, detail: noted.reason };
|
|
112
|
+
if (state === 'pass') return { disposition: 'blocked', detail: 'unclassified return: the test passed with zero assertions and recorded no reason — RFC 0148 §A resolves it to blocked, never to a pass' };
|
|
113
|
+
return { disposition: 'skipped', detail: 'vitest skipped the test (ctx.skip / it.skip) without a recorded gate reason' };
|
|
114
|
+
}
|
|
115
|
+
|
|
82
116
|
/** Worker half: fold a file's per-test states (+ any gate-recorded reason) into
|
|
83
117
|
* the ONE disposition the file records. */
|
|
84
118
|
export function fileDisposition(
|
package/src/lib/soft-skip.ts
CHANGED
|
@@ -22,6 +22,17 @@
|
|
|
22
22
|
* resolves it to blocked" — and stays UNCLASSIFIED for certification (a floor
|
|
23
23
|
* row with that disposition still rejects), so the honest bundle row and the
|
|
24
24
|
* pressure to say why both survive.
|
|
25
|
+
*
|
|
26
|
+
* rc.56: every note also carries a sequence number, so the per-`it` row can
|
|
27
|
+
* read the notes written DURING ITS OWN TEST (`softSkipMark()` at test start,
|
|
28
|
+
* `softSkipDispositionSince(file, mark)` at test end). Until rc.56 the
|
|
29
|
+
* per-`it` row consulted only the journal's `behaviorGate` entries, never a
|
|
30
|
+
* softSkip note, so a leg that returned `softSkip('inapplicable', 'a2a facet
|
|
31
|
+
* not advertised')` was recorded `blocked / unclassified return` at `it`
|
|
32
|
+
* granularity while its file row was honestly `inapplicable` — and a bundle
|
|
33
|
+
* with any `blocked` row does not certify (RFC 0168 §E.1). Forty-five such
|
|
34
|
+
* rows on a host that simply does not advertise A2A/MCP denied certification
|
|
35
|
+
* to every profile it claimed.
|
|
25
36
|
*/
|
|
26
37
|
|
|
27
38
|
import { expect } from 'vitest';
|
|
@@ -32,7 +43,10 @@ export type SoftSkipKind = 'inapplicable' | 'skipped' | 'blocked';
|
|
|
32
43
|
/** Detail marker the runner writes for a zero-assertion file that noted nothing. */
|
|
33
44
|
export const UNCLASSIFIED_RETURN_DETAIL = 'every test returned early with zero assertions and no recorded reason — unclassified return; RFC 0148 §A resolves it to blocked (add softSkip(kind, reason) at the early return)';
|
|
34
45
|
|
|
35
|
-
|
|
46
|
+
interface Note { readonly kind: SoftSkipKind; readonly reason: string; readonly seq: number }
|
|
47
|
+
|
|
48
|
+
const notes = new Map<string, Note[]>();
|
|
49
|
+
let seq = 0;
|
|
36
50
|
|
|
37
51
|
function currentFile(): string | null {
|
|
38
52
|
try {
|
|
@@ -48,7 +62,9 @@ export function softSkip(kind: SoftSkipKind, reason: string): undefined {
|
|
|
48
62
|
const file = currentFile();
|
|
49
63
|
if (file === null) return undefined;
|
|
50
64
|
const arr = notes.get(file) ?? [];
|
|
51
|
-
|
|
65
|
+
// Every call is recorded with its own sequence number so a per-test window
|
|
66
|
+
// sees it; the file-level join de-duplicates identical (kind, reason) pairs.
|
|
67
|
+
arr.push({ kind, reason, seq: ++seq });
|
|
52
68
|
notes.set(file, arr);
|
|
53
69
|
return undefined;
|
|
54
70
|
}
|
|
@@ -69,6 +85,15 @@ export function seamAbsent(reason: string): undefined {
|
|
|
69
85
|
|
|
70
86
|
const RANK: Record<SoftSkipKind, number> = { blocked: 0, skipped: 1, inapplicable: 2 };
|
|
71
87
|
|
|
88
|
+
function fold(arr: readonly Note[]): { kind: SoftSkipKind; reason: string } | null {
|
|
89
|
+
if (arr.length === 0) return null;
|
|
90
|
+
const uniq: Note[] = [];
|
|
91
|
+
for (const n of arr) if (!uniq.some((u) => u.kind === n.kind && u.reason === n.reason)) uniq.push(n);
|
|
92
|
+
const kind = [...uniq].sort((a, b) => RANK[a.kind] - RANK[b.kind])[0]!.kind;
|
|
93
|
+
const reason = uniq.map((n) => (uniq.length > 1 ? `[${n.kind}] ${n.reason}` : n.reason)).join('; ');
|
|
94
|
+
return { kind, reason };
|
|
95
|
+
}
|
|
96
|
+
|
|
72
97
|
/**
|
|
73
98
|
* The noted disposition for a file, worst-first when mixed (`blocked` beats
|
|
74
99
|
* `skipped` beats `inapplicable` — a file that could not check one thing is
|
|
@@ -76,14 +101,24 @@ const RANK: Record<SoftSkipKind, number> = { blocked: 0, skipped: 1, inapplicabl
|
|
|
76
101
|
* the reasons joined. `null` when nothing was noted.
|
|
77
102
|
*/
|
|
78
103
|
export function softSkipDisposition(file: string): { kind: SoftSkipKind; reason: string } | null {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
104
|
+
return fold(notes.get(file) ?? []);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** A position in the note sequence; pass it to `softSkipDispositionSince`. */
|
|
108
|
+
export function softSkipMark(): number {
|
|
109
|
+
return seq;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* The noted disposition for a file counting only notes written AFTER `mark`
|
|
114
|
+
* — the notes of the test that is ending. Same fold as the file rule.
|
|
115
|
+
*/
|
|
116
|
+
export function softSkipDispositionSince(file: string, mark: number): { kind: SoftSkipKind; reason: string } | null {
|
|
117
|
+
return fold((notes.get(file) ?? []).filter((n) => n.seq > mark));
|
|
84
118
|
}
|
|
85
119
|
|
|
86
120
|
/** Test hook. */
|
|
87
121
|
export function resetSoftSkips(): void {
|
|
88
122
|
notes.clear();
|
|
123
|
+
seq = 0;
|
|
89
124
|
}
|
|
@@ -58,8 +58,17 @@ function bundleWith(rows: BundleV3Requirement[]): BundleV3 {
|
|
|
58
58
|
|
|
59
59
|
describe('v2-coherence-not-in-bundle (RFC 0168 §D.1)', () => {
|
|
60
60
|
it('every corpus-ledger id has a scenario in src/coherence and none in src/scenarios — the two id sets are disjoint by construction', () => {
|
|
61
|
-
|
|
62
|
-
|
|
61
|
+
// rc.57: `inapplicable`, not `blocked`. The subject of this scenario is the
|
|
62
|
+
// CORPUS — it reads evidence/corpus-ledger.json and the two source
|
|
63
|
+
// directories and asserts nothing about a host. RFC 0148 §A defines
|
|
64
|
+
// `blocked` over ADVERTISED behaviour a missing dependency prevented
|
|
65
|
+
// exercising; there is none here, and `blocked` denied certification
|
|
66
|
+
// bundle-wide (RFC 0168 §E.1) to every host running the published tarball,
|
|
67
|
+
// which ships neither the ledger nor src/coherence. Same reasoning as
|
|
68
|
+
// lib/spec-coherence.ts for the v1 corpus scenarios; the reference host
|
|
69
|
+
// carried this row as one of its "16 blocked" since rc.16.
|
|
70
|
+
if (!existsSync(LEDGER)) return softSkip('inapplicable', 'inapplicable to any host: evidence/corpus-ledger.json is absent from this layout — the subject of this scenario is the corpus, which the published tarball does not ship; it runs in a spec checkout (scripts/check-spec-coherence.mjs) and needs no host');
|
|
71
|
+
if (SCENARIOS_DIR === null || COHERENCE_DIR === null || !existsSync(COHERENCE_DIR)) return softSkip('inapplicable', 'inapplicable to any host: src/scenarios or src/coherence is absent from this layout — the subject of this scenario is the corpus, which the published tarball does not ship; it runs in a spec checkout and needs no host');
|
|
63
72
|
const ledger = JSON.parse(readFileSync(LEDGER, 'utf8')) as { requirements: Record<string, unknown> };
|
|
64
73
|
const ids = Object.keys(ledger.requirements);
|
|
65
74
|
expect(ids.length, req('openwop.requirement.0168.coherence-not-in-bundle.disjoint-by-construction', SECTION, 'the corpus ledger MUST carry at least one requirement id')).toBeGreaterThan(0);
|
package/src/setup.ts
CHANGED
|
@@ -35,8 +35,8 @@ import { basename, join } from 'node:path';
|
|
|
35
35
|
import { existsSync, readFileSync } from 'node:fs';
|
|
36
36
|
import { PKG_ROOT_PATH } from './lib/paths.js';
|
|
37
37
|
import { recordRequirement, hasRequirement, journalLength, journalSince } from './lib/requirement-ledger.js';
|
|
38
|
-
import { requirementIdForFile, resolveFileRecord, type FileTestState } from './lib/scenario-disposition.js';
|
|
39
|
-
import { softSkipDisposition } from './lib/soft-skip.js';
|
|
38
|
+
import { requirementIdForFile, resolveFileRecord, resolveItRecord, type FileTestState } from './lib/scenario-disposition.js';
|
|
39
|
+
import { softSkipDisposition, softSkipDispositionSince, softSkipMark } from './lib/soft-skip.js';
|
|
40
40
|
import { ItIdAllocator, takeExplicitRequirementId } from './lib/requirement-ids.js';
|
|
41
41
|
import { SPEC_COHERENCE_SCENARIOS, SPEC_COHERENCE_DETAIL } from './lib/spec-coherence.js';
|
|
42
42
|
import type { DiscoveryPayload } from './lib/profiles.js';
|
|
@@ -239,6 +239,7 @@ const _ledgerMarks = new Map<string, number>();
|
|
|
239
239
|
const _itAllocators = new Map<string, ItIdAllocator>();
|
|
240
240
|
const _itMarks = new Map<string, number>();
|
|
241
241
|
const _itAssertionsBefore = new Map<string, number>();
|
|
242
|
+
const _itSoftSkipMarks = new Map<string, number>();
|
|
242
243
|
function _assertionCalls(): number {
|
|
243
244
|
try {
|
|
244
245
|
return (expect.getState() as { assertionCalls?: number }).assertionCalls ?? 0;
|
|
@@ -347,6 +348,7 @@ beforeEach(({ task }) => {
|
|
|
347
348
|
// Window for this test's own gate decisions and its own assertion count.
|
|
348
349
|
_itMarks.set(file, journalLength());
|
|
349
350
|
_itAssertionsBefore.set(file, _assertionCalls());
|
|
351
|
+
_itSoftSkipMarks.set(file, softSkipMark()); // rc.56: this test's own softSkip window
|
|
350
352
|
takeExplicitRequirementId(); // clear any override left by a test that threw before afterEach
|
|
351
353
|
});
|
|
352
354
|
afterEach(({ task }) => {
|
|
@@ -395,7 +397,8 @@ afterEach(({ task }) => {
|
|
|
395
397
|
_itAllocators.set(file, alloc);
|
|
396
398
|
const itId = explicit ?? alloc.allocate(file, task.name);
|
|
397
399
|
const since = journalSince(_itMarks.get(file) ?? 0);
|
|
398
|
-
const
|
|
400
|
+
const gateEntry = since.find((e) => e.disposition === 'inapplicable') ?? since.find((e) => e.disposition === 'skipped');
|
|
401
|
+
const gate = gateEntry === undefined ? undefined : { disposition: gateEntry.disposition as 'inapplicable' | 'skipped', ...(gateEntry.detail === undefined ? {} : { detail: gateEntry.detail }) };
|
|
399
402
|
let disposition: 'executed-pass' | 'executed-fail' | 'skipped' | 'inapplicable' | 'blocked';
|
|
400
403
|
let detail: string | undefined;
|
|
401
404
|
// Suite 2.0.0: under the corpus gate (scripts/check-spec-coherence.mjs sets OPENWOP_CORPUS_GATE) a coherence scenario IS the subject; its rows are real dispositions for evidence/corpus-ledger.json.
|
|
@@ -405,21 +408,15 @@ afterEach(({ task }) => {
|
|
|
405
408
|
// rule `resolveFileRecord` applies to the file in the published layout.
|
|
406
409
|
disposition = 'inapplicable';
|
|
407
410
|
detail = SPEC_COHERENCE_DETAIL;
|
|
408
|
-
} else if (state === 'fail') {
|
|
409
|
-
disposition = 'executed-fail';
|
|
410
|
-
const err = (task.result?.errors ?? [])[0] as { message?: string } | undefined;
|
|
411
|
-
detail = `the test executed and failed: ${(err?.message ?? 'no message').slice(0, 300)}`;
|
|
412
|
-
} else if (state === 'pass' && calls > 0) {
|
|
413
|
-
disposition = 'executed-pass';
|
|
414
|
-
} else if (gate !== undefined) {
|
|
415
|
-
disposition = gate.disposition as 'skipped' | 'inapplicable';
|
|
416
|
-
detail = gate.detail ?? `${gate.disposition} (gate recorded no reason)`;
|
|
417
|
-
} else if (state === 'pass') {
|
|
418
|
-
disposition = 'blocked';
|
|
419
|
-
detail = 'unclassified return: the test passed with zero assertions and recorded no reason — RFC 0148 §A resolves it to blocked, never to a pass';
|
|
420
411
|
} else {
|
|
421
|
-
|
|
422
|
-
|
|
412
|
+
// rc.56: the softSkip notes THIS test wrote are its reason (the file row
|
|
413
|
+
// already read them; the per-`it` row did not, and a leg that said
|
|
414
|
+
// `inapplicable` came out `blocked` — which denies certification bundle-wide).
|
|
415
|
+
const noted = softSkipDispositionSince(file, _itSoftSkipMarks.get(file) ?? 0);
|
|
416
|
+
const err = (task.result?.errors ?? [])[0] as { message?: string } | undefined;
|
|
417
|
+
const rec = resolveItRecord(state === 'pass' ? 'pass' : state === 'fail' ? 'fail' : 'skip', calls, gate, noted, err?.message);
|
|
418
|
+
disposition = rec.disposition;
|
|
419
|
+
detail = rec.detail;
|
|
423
420
|
}
|
|
424
421
|
try {
|
|
425
422
|
recordRequirement(itId, disposition, detail, { assertionCount: calls, scenarioFile: file });
|