@openwop/openwop-conformance 2.0.0-rc.8 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/README.md +8 -3
- package/dist/cli.js +24 -34
- package/dist/lib/requirement-registry.js +69 -0
- package/dist/lib/scenario-disposition.js +84 -11
- package/dist/lib/seams.js +72 -0
- package/dist/lib/soft-skip.js +39 -8
- package/dist/spec-artifacts.lock.json +2 -2
- package/package.json +2 -2
- package/requirement-aliases.json +4 -1
- package/requirements.json +1087 -52
- package/scenario-majors.json +57 -3
- package/schemas/CORPUS-STAMP.json +41 -38
- package/src/cli.ts +23 -30
- package/src/global-setup.ts +11 -0
- package/src/lib/corpus-stamp.ts +24 -2
- package/src/lib/era2-seed.ts +12 -1
- package/src/lib/fixtures.ts +31 -0
- package/src/lib/requirement-registry.ts +68 -0
- package/src/lib/scenario-disposition.ts +88 -9
- package/src/lib/seams.ts +31 -1
- package/src/lib/soft-skip.ts +42 -7
- package/src/lib/sse.ts +8 -0
- package/src/scenarios/era-key-stamped-v1.test.ts +156 -0
- package/src/scenarios/pause-resume.test.ts +159 -75
- package/src/scenarios/v2-advertised-fixtures-exist.test.ts +25 -43
- package/src/scenarios/v2-advertised-path-space-served.test.ts +165 -0
- package/src/scenarios/v2-assurance-downgrade-audited.test.ts +1 -1
- package/src/scenarios/v2-chain-pin-exact.test.ts +1 -1
- package/src/scenarios/v2-coherence-not-in-bundle.test.ts +11 -2
- package/src/scenarios/v2-created-run-readable.test.ts +98 -0
- package/src/scenarios/v2-dual-stack-negotiation.test.ts +29 -1
- package/src/scenarios/v2-effect-identity-business-key.test.ts +1 -1
- package/src/scenarios/v2-effect-seam-manifest.test.ts +21 -55
- package/src/scenarios/v2-effect-seam-no-refire.test.ts +104 -0
- package/src/scenarios/v2-era-2-append-vocabulary.test.ts +16 -2
- package/src/scenarios/v2-interrupt-token-scheme.test.ts +1 -1
- package/src/scenarios/v2-malformed-body-envelope.test.ts +79 -0
- package/src/scenarios/v2-manifest-ceiling-refused.test.ts +1 -1
- package/src/scenarios/v2-manifest-hatch-carried.test.ts +1 -1
- package/src/scenarios/v2-minimum-version-refused.test.ts +1 -1
- package/src/scenarios/v2-mrtr-rounds-ceiling.test.ts +1 -1
- package/src/scenarios/v2-negotiation-authenticated.test.ts +1 -1
- package/src/scenarios/v2-negotiation-decided-emitted.test.ts +1 -1
- package/src/scenarios/v2-pack-isolation.test.ts +1 -1
- package/src/scenarios/v2-peer-dependency-declared.test.ts +1 -1
- package/src/scenarios/v2-poll-cursor-v2.test.ts +18 -0
- package/src/scenarios/v2-revocation-honored.test.ts +1 -1
- package/src/scenarios/v2-run-annotation-not-event.test.ts +71 -0
- package/src/scenarios/v2-run-bulk-cancel.test.ts +91 -0
- package/src/scenarios/v2-run-cancel.test.ts +97 -0
- package/src/scenarios/v2-run-completed-outputs.test.ts +93 -0
- package/src/scenarios/v2-run-diff-identical.test.ts +75 -0
- package/src/scenarios/v2-run-fork-prefix.test.ts +160 -0
- package/src/scenarios/v2-run-fork-refusals.test.ts +70 -0
- package/src/scenarios/v2-run-options-limits.test.ts +69 -0
- package/src/scenarios/v2-run-pause-resume.test.ts +117 -0
- package/src/scenarios/v2-run-snapshot-etag.test.ts +57 -0
- package/src/scenarios/v2-sse-last-event-id.test.ts +93 -0
- package/src/scenarios/v2-stream-mode-refusal.test.ts +118 -0
- package/src/scenarios/v2-stream-sse-projection.test.ts +79 -0
- package/src/scenarios/v2-subject-link-record.test.ts +1 -1
- package/src/scenarios/v2-v1-signed-webhook-accepted.test.ts +1 -1
- package/src/scenarios/version-negotiation.test.ts +25 -3
- package/src/setup.ts +104 -50
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `spec/v2/core/events.md` §The events channel — `streamMode` is one pattern,
|
|
3
|
+
* refused with `400 unsupported_stream_mode`, validated before content
|
|
4
|
+
* negotiation; `bufferMs` batches (suite 2.0.0, target major 2; unaided; one
|
|
5
|
+
* run created).
|
|
6
|
+
*
|
|
7
|
+
* Legs:
|
|
8
|
+
* 1. `?streamMode=bogus` → 400 `unsupported_stream_mode`, `details.supported`
|
|
9
|
+
* a non-empty array of individual modes; the same request with
|
|
10
|
+
* `Accept: application/json` → still 400, never 406 (validation runs
|
|
11
|
+
* before content negotiation; the only v2 406 is a version mismatch and
|
|
12
|
+
* this request names a listed major); `?streamMode=updates,values` → 400
|
|
13
|
+
* (`values` never combines); a mode absent from `supported`, if any → 400.
|
|
14
|
+
* 2. control: `?streamMode=updates` → 200 `text/event-stream` with ≥1 frame
|
|
15
|
+
* (a host that answers 400 to every mode fails here; `updates` is a MUST).
|
|
16
|
+
* 3. `?bufferMs=200` → at least one `event: batch` frame whose `data:` is an
|
|
17
|
+
* array of RunEventDoc, and the flattened frames equal the log in order;
|
|
18
|
+
* without `bufferMs` no frame is `batch` (the control). "Every frame is
|
|
19
|
+
* batch" is deliberately NOT asserted: a consumer MUST tolerate an
|
|
20
|
+
* unbatched frame beside a one-element batch (§SSE frames).
|
|
21
|
+
*
|
|
22
|
+
* @see spec/v2/core/events.md §The events channel, §SSE frames
|
|
23
|
+
* @see spec/v2/errors.json unsupported_stream_mode
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { describe, it, expect } from 'vitest';
|
|
27
|
+
import { driver, type OpenWOPResponse } from '../lib/driver.js';
|
|
28
|
+
import { subscribe, type SseEvent } from '../lib/sse.js';
|
|
29
|
+
import { v2Discovery } from '../lib/v2.js';
|
|
30
|
+
import { readErrorCode } from '../lib/error-envelope.js';
|
|
31
|
+
import { softSkip } from '../lib/soft-skip.js';
|
|
32
|
+
import { req } from '../lib/requirement-ids.js';
|
|
33
|
+
|
|
34
|
+
const ID = 'openwop.requirement.0171.stream-mode-refusal';
|
|
35
|
+
const DOC = 'spec/v2/core/events.md §The events channel';
|
|
36
|
+
const NOOP = 'conformance-noop';
|
|
37
|
+
const MODES = ['values', 'updates', 'messages', 'debug'];
|
|
38
|
+
const V2 = { 'OpenWOP-Version': '2.0' };
|
|
39
|
+
|
|
40
|
+
async function discovery(): Promise<Record<string, unknown> | null> { try { return await v2Discovery(); } catch { return null; } }
|
|
41
|
+
async function http(fn: () => Promise<OpenWOPResponse>): Promise<OpenWOPResponse | null> { try { return await fn(); } catch { return null; } }
|
|
42
|
+
const enc = (id: string): string => encodeURIComponent(id);
|
|
43
|
+
|
|
44
|
+
async function createSettled(): Promise<{ runId: string } | { reason: string }> {
|
|
45
|
+
const res = await http(() => driver.post('/runs', { workflowId: NOOP }));
|
|
46
|
+
if (res === null) return { reason: 'POST /runs unreachable (fetch failed)' };
|
|
47
|
+
const runId = (res.json as { runId?: unknown } | null)?.runId;
|
|
48
|
+
if (res.status !== 201 || typeof runId !== 'string') return { reason: `POST /runs answered ${res.status} ${readErrorCode(res.json) ?? ''}`.trim() };
|
|
49
|
+
const t0 = Date.now();
|
|
50
|
+
while (Date.now() - t0 < 10_000) {
|
|
51
|
+
const s = await http(() => driver.get(`/runs/${enc(runId)}`));
|
|
52
|
+
if (s?.status === 200 && ['completed', 'failed', 'cancelled'].includes(String((s.json as { status?: unknown }).status))) return { runId };
|
|
53
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
54
|
+
}
|
|
55
|
+
return { reason: 'the noop run did not settle within 10 s' };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The events a frame carries: a batch frame's array flattened, a plain frame's one document. */
|
|
59
|
+
function docsOf(frame: SseEvent): Array<Record<string, unknown>> {
|
|
60
|
+
try { const p = JSON.parse(frame.data) as unknown; return (Array.isArray(p) ? p : [p]).filter((x): x is Record<string, unknown> => !!x && typeof x === 'object'); } catch { return []; }
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
describe('v2 stream-mode-refusal (events.md §The events channel)', () => {
|
|
64
|
+
it('a value outside the pattern, a forbidden combination, and an unimplemented mode are refused 400 unsupported_stream_mode with details.supported — before content negotiation', async () => {
|
|
65
|
+
if (!(await discovery())) return softSkip('blocked', 'v2 discovery unreachable');
|
|
66
|
+
const c = await createSettled(); if ('reason' in c) return softSkip('blocked', c.reason);
|
|
67
|
+
const path = `/runs/${enc(c.runId)}/events`;
|
|
68
|
+
const bogus = await http(() => driver.get(`${path}?streamMode=bogus`, { headers: { Accept: 'text/event-stream' } }));
|
|
69
|
+
if (bogus === null) return softSkip('blocked', 'GET /runs/{runId}/events unreachable (fetch failed)');
|
|
70
|
+
if (bogus.status === 404) return softSkip('blocked', 'GET /runs/{runId}/events answered 404 — streamRunEvents is a core operation (runs.md §Surface) and is not mounted');
|
|
71
|
+
expect(bogus.status, req(ID, DOC, `a streamMode outside the pattern MUST be refused 400 — got ${bogus.status}`)).toBe(400);
|
|
72
|
+
expect(readErrorCode(bogus.json), req(ID, DOC, 'the refusal MUST be unsupported_stream_mode')).toBe('unsupported_stream_mode');
|
|
73
|
+
const supported = (bogus.json as { details?: { supported?: unknown } } | null)?.details?.supported;
|
|
74
|
+
expect(Array.isArray(supported) && supported.length > 0 && supported.every((m) => MODES.includes(String(m))), req(ID, DOC, `details.supported MUST list each individual mode the host serves (got ${JSON.stringify(supported)})`)).toBe(true);
|
|
75
|
+
expect((supported as string[]).includes('updates'), req(ID, DOC, 'a host MUST implement updates, so supported MUST list it')).toBe(true);
|
|
76
|
+
|
|
77
|
+
const negotiated = await http(() => driver.get(`${path}?streamMode=bogus`, { headers: { Accept: 'application/json' } }));
|
|
78
|
+
expect(negotiated?.status ?? null, req(ID, DOC, `validation MUST run before any content negotiation: a bogus streamMode with Accept: application/json is still 400 unsupported_stream_mode, never 406 (the only v2 406 is a version mismatch and this request names a listed major) — got ${negotiated?.status ?? 'no response'}`)).toBe(400);
|
|
79
|
+
|
|
80
|
+
const combo = await http(() => driver.get(`${path}?streamMode=updates,values`, { headers: { Accept: 'text/event-stream' } }));
|
|
81
|
+
expect(combo?.status ?? null, req(ID, DOC, `values never combines: streamMode=updates,values is outside the pattern and MUST be refused 400 — got ${combo?.status ?? 'no response'}`)).toBe(400);
|
|
82
|
+
expect(readErrorCode(combo?.json), req(ID, DOC, 'the refusal MUST be unsupported_stream_mode')).toBe('unsupported_stream_mode');
|
|
83
|
+
|
|
84
|
+
// The unimplemented-mode leg has something to request only when a mode is
|
|
85
|
+
// absent from details.supported; a host serving all four has no such mode,
|
|
86
|
+
// and the three refusals above are its witness.
|
|
87
|
+
const missing = MODES.find((m) => !(supported as string[]).includes(m));
|
|
88
|
+
if (missing !== undefined) {
|
|
89
|
+
const unimpl = await http(() => driver.get(`${path}?streamMode=${missing}`, { headers: { Accept: 'text/event-stream' } }));
|
|
90
|
+
expect(unimpl?.status ?? null, req(ID, DOC, `a mode the host does not implement (${missing}, absent from details.supported) MUST be refused 400 — got ${unimpl?.status ?? 'no response'}`)).toBe(400);
|
|
91
|
+
expect(readErrorCode(unimpl?.json), req(ID, DOC, 'the refusal MUST be unsupported_stream_mode')).toBe('unsupported_stream_mode');
|
|
92
|
+
}
|
|
93
|
+
}, 30_000);
|
|
94
|
+
|
|
95
|
+
it('updates streams 200 text/event-stream with frames; bufferMs yields at least one batch frame whose data is an array and loses nothing; without bufferMs no frame is batch', async () => {
|
|
96
|
+
if (!(await discovery())) return softSkip('blocked', 'v2 discovery unreachable');
|
|
97
|
+
const c = await createSettled(); if ('reason' in c) return softSkip('blocked', c.reason);
|
|
98
|
+
const path = `/runs/${enc(c.runId)}/events`;
|
|
99
|
+
const plain = await subscribe(`${path}?streamMode=updates`, { timeoutMs: 8_000, extraHeaders: V2 });
|
|
100
|
+
if (plain.status === 404) return softSkip('blocked', 'GET /runs/{runId}/events answered 404 — not mounted');
|
|
101
|
+
expect(plain.status, req(ID, DOC, `updates is a MUST: streamMode=updates MUST answer 200 — got ${plain.status} (the control for the refusals: a host answering 400 to every mode fails here)`)).toBe(200);
|
|
102
|
+
expect(plain.events.length, req(ID, 'spec/v2/core/events.md §SSE frames', 'the completed run\'s log MUST stream as at least one frame before the server closes')).toBeGreaterThan(0);
|
|
103
|
+
expect(plain.closedBy, req(ID, 'spec/v2/core/events.md §SSE frames', `the host MUST close after the terminal event (closed by ${plain.closedBy})`)).toBe('server');
|
|
104
|
+
expect(plain.events.some((f) => f.event === 'batch'), req(ID, 'spec/v2/core/events.md §SSE frames', 'without bufferMs no frame is a batch (the control for the batch leg)')).toBe(false);
|
|
105
|
+
const plainDocs = plain.events.flatMap(docsOf).map((d) => d['sequence']);
|
|
106
|
+
|
|
107
|
+
const buffered = await subscribe(`${path}?streamMode=updates&bufferMs=200`, { timeoutMs: 8_000, extraHeaders: V2 });
|
|
108
|
+
expect(buffered.status, req(ID, 'spec/v2/core/events.md §SSE frames', `streamMode=updates&bufferMs=200 MUST answer 200 — got ${buffered.status}`)).toBe(200);
|
|
109
|
+
const batches = buffered.events.filter((f) => f.event === 'batch');
|
|
110
|
+
expect(batches.length, req(ID, 'spec/v2/core/events.md §SSE frames', `with bufferMs the host accumulates events into event: batch frames — none seen among ${buffered.events.length} frame(s) (${[...new Set(buffered.events.map((f) => f.event))].join(', ')})`)).toBeGreaterThan(0);
|
|
111
|
+
for (const b of batches) {
|
|
112
|
+
let parsed: unknown; try { parsed = JSON.parse(b.data); } catch { parsed = undefined; }
|
|
113
|
+
expect(Array.isArray(parsed), req(ID, 'spec/v2/core/events.md §SSE frames', 'a batch frame\'s data MUST be an array of RunEventDoc')).toBe(true);
|
|
114
|
+
}
|
|
115
|
+
const bufferedDocs = buffered.events.flatMap(docsOf).map((d) => d['sequence']);
|
|
116
|
+
expect(bufferedDocs, req(ID, 'spec/v2/core/events.md §SSE frames', 'batching MUST NOT lose or reorder events: the flattened buffered stream equals the unbuffered log (a one-element batch and an unbatched frame are both tolerated)')).toEqual(plainDocs);
|
|
117
|
+
}, 30_000);
|
|
118
|
+
});
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `spec/v2/core/events.md` §SSE frames + `identity.md` §5 — every `data:` frame
|
|
3
|
+
* on the major-2 run stream names the run by its tenant-bound id (suite 2.0.0,
|
|
4
|
+
* target major 2; unaided).
|
|
5
|
+
*
|
|
6
|
+
* This scenario exists because both production hosts shipped the same defect
|
|
7
|
+
* and the suite could not see it. Each built one projection seam for the
|
|
8
|
+
* tenant-bound `runId` and each found an emitter outside it on the stream
|
|
9
|
+
* path: one host's SSE route had three `res.write` sites and no per-frame
|
|
10
|
+
* projector at all; the other's per-frame path projected while its `batch`
|
|
11
|
+
* flush wrote the raw array. Sixteen green mount tests on one host all used
|
|
12
|
+
* `res.json` handlers. Of the 56 `v2-*` files, none read a stream frame —
|
|
13
|
+
* `grep -E '^v2-.*(sse|stream)'` was empty — so both defects were invisible to
|
|
14
|
+
* the suite by construction and were found by a live witness and by a peer's
|
|
15
|
+
* report. A tier-1 host asked for this file by name.
|
|
16
|
+
*
|
|
17
|
+
* The assertion is one line: every frame's `runId` is the bound id the create
|
|
18
|
+
* returned. It is asserted per frame rather than on the first, because the
|
|
19
|
+
* batch-flush variant projects the first frame and not the rest.
|
|
20
|
+
*
|
|
21
|
+
* @see spec/v2/core/events.md §SSE frames
|
|
22
|
+
* @see spec/v2/core/identity.md §5
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { describe, it, expect } from 'vitest';
|
|
26
|
+
import { driver, type OpenWOPResponse } from '../lib/driver.js';
|
|
27
|
+
import { v2Discovery, v2Validator } from '../lib/v2.js';
|
|
28
|
+
import { streamEvents } from '../lib/era2-seed.js';
|
|
29
|
+
import { readErrorCode } from '../lib/error-envelope.js';
|
|
30
|
+
import { softSkip } from '../lib/soft-skip.js';
|
|
31
|
+
import { req } from '../lib/requirement-ids.js';
|
|
32
|
+
|
|
33
|
+
const ID = 'openwop.requirement.0171.stream-sse-projection';
|
|
34
|
+
const DOC = 'spec/v2/core/events.md §SSE frames';
|
|
35
|
+
const NOOP_WORKFLOW_ID = 'conformance-noop';
|
|
36
|
+
const RUN_ID = /^[A-Za-z0-9._~-]{1,128}\/[A-Za-z0-9._~-]{16,128}$/;
|
|
37
|
+
|
|
38
|
+
async function http(fn: () => Promise<OpenWOPResponse>): Promise<OpenWOPResponse | null> {
|
|
39
|
+
try { return await fn(); } catch { return null; }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function createRun(): Promise<{ runId: string } | { reason: string }> {
|
|
43
|
+
try { if (!(await v2Discovery())) return { reason: 'v2 discovery unreachable' }; } catch { return { reason: 'v2 discovery unreachable' }; }
|
|
44
|
+
const res = await http(() => driver.post('/runs', { workflowId: NOOP_WORKFLOW_ID }));
|
|
45
|
+
if (res === null) return { reason: 'POST /runs unreachable (fetch failed)' };
|
|
46
|
+
const runId = (res.json as { runId?: unknown } | undefined)?.runId;
|
|
47
|
+
if (res.status !== 201 || typeof runId !== 'string') return { reason: `POST /runs {workflowId: ${NOOP_WORKFLOW_ID}} answered ${res.status} ${readErrorCode(res.json) ?? ''} — the smallest valid create was refused (fixture not seeded?)`.trim() };
|
|
48
|
+
return { runId };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
describe('v2 stream-sse-projection (events.md §SSE frames)', () => {
|
|
52
|
+
it('every data: frame on the major-2 stream carries the tenant-bound runId of the run it belongs to', async () => {
|
|
53
|
+
const c = await createRun();
|
|
54
|
+
if ('reason' in c) return softSkip('blocked', c.reason);
|
|
55
|
+
expect(RUN_ID.test(c.runId), req(ID, 'spec/v2/core/identity.md §5', `the created runId MUST be tenant-bound before the stream can be held to it (got ${c.runId})`)).toBe(true);
|
|
56
|
+
|
|
57
|
+
const s = await streamEvents(c.runId);
|
|
58
|
+
if (s === null) return softSkip('blocked', 'GET /runs/{runId}/events (SSE, OpenWOP-Version: 2.0) unreachable (fetch failed)');
|
|
59
|
+
if (s.status !== 200) return softSkip('blocked', `GET /runs/{runId}/events (SSE) answered ${s.status} for the run just created`);
|
|
60
|
+
if (s.events.length === 0) return softSkip('blocked', 'the stream delivered no data: frames within the window — nothing to hold to the grammar');
|
|
61
|
+
|
|
62
|
+
const validate = v2Validator('run-event');
|
|
63
|
+
let i = 0;
|
|
64
|
+
for (const ev of s.events) {
|
|
65
|
+
i += 1;
|
|
66
|
+
const rid = (ev as { runId?: unknown }).runId;
|
|
67
|
+
expect(
|
|
68
|
+
typeof rid === 'string' && RUN_ID.test(rid),
|
|
69
|
+
req(ID, DOC, `frame ${i} (${String((ev as { type?: unknown }).type)}): data.runId MUST be tenant-bound <tenantId>/<opaque> (identity.md §5) — a stream frame is a rendering of the run event and every rendering of a v2 runId uses the same grammar; a bare storage id here is the projection seam missing the stream path (got ${JSON.stringify(rid)})`),
|
|
70
|
+
).toBe(true);
|
|
71
|
+
expect(
|
|
72
|
+
rid,
|
|
73
|
+
req(ID, DOC, `frame ${i}: data.runId MUST be the run's own bound id — asserted on EVERY frame, not the first, because a batch flush that skips the per-frame projector projects frame 1 and not the rest`),
|
|
74
|
+
).toBe(c.runId);
|
|
75
|
+
const r = validate(ev);
|
|
76
|
+
expect(r.ok, req(ID, DOC, `frame ${i}: data MUST be a valid RunEventDoc (run-event.schema.json) — ${r.errors}`)).toBe(true);
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
});
|
|
@@ -56,7 +56,7 @@ async function readLink(externalId: string): Promise<Record<string, unknown> | n
|
|
|
56
56
|
async function form(): Promise<Formed> {
|
|
57
57
|
const doc = await discovery();
|
|
58
58
|
if (!doc) return { kind: 'blocked', reason: 'v2 discovery unreachable — /.well-known/openwop did not answer 200 with a JSON body under OpenWOP-Version: 2.0' };
|
|
59
|
-
if (!seamsProfileAdvertised(doc)) return { kind: '
|
|
59
|
+
if (!seamsProfileAdvertised(doc)) return { kind: 'inapplicable', reason: 'seams profile not advertised (conformance.seamsProfile !== openwop-conformance-seams-v2) — the link record is seam-gated' };
|
|
60
60
|
const auth = await familyAdvertised('auth');
|
|
61
61
|
const lanes = new Set((Array.isArray(auth?.['lanes']) ? (auth['lanes'] as Array<Record<string, unknown>>) : []).map((l) => String(l['lane'])));
|
|
62
62
|
if (!(lanes.has('saml') && lanes.has('scim'))) return { kind: 'inapplicable', reason: 'the host does not advertise both the saml and scim lanes — advertising both is what implies the linking contract (row C2.5)' };
|
|
@@ -90,7 +90,7 @@ describe('RFC 0176 §D.2 — v1-signed-webhook-accepted (gated on webhooks + sea
|
|
|
90
90
|
const doc = await discovery();
|
|
91
91
|
if (!doc) return softSkip('blocked', 'discovery unreachable');
|
|
92
92
|
if (!(await gateFamily('webhooks'))) return softSkip('inapplicable', 'webhooks family not advertised (gate recorded under openwop.family.webhooks)');
|
|
93
|
-
if (!seamsProfileAdvertised(doc)) return softSkip('
|
|
93
|
+
if (!seamsProfileAdvertised(doc)) return softSkip('inapplicable', `seams profile not advertised (conformance.seamsProfile !== openwop-conformance-seams-v2) — the host's inbound receiver is reachable only through ${RECEIVE}`);
|
|
94
94
|
const secret = `conformance-secret-${Date.now().toString(36)}`;
|
|
95
95
|
const body = JSON.stringify({ runId: 'run-conformance-v1-signed', workspaceId: 'ws-conformance', event: { type: 'run.completed', sequence: 3, payload: { durationMs: 1 } } });
|
|
96
96
|
const good = await deliver(secret, v1Delivery(secret, body));
|
|
@@ -7,9 +7,31 @@
|
|
|
7
7
|
*
|
|
8
8
|
* What we CAN test cheaply:
|
|
9
9
|
* 1. Server advertises a `protocolVersion` in `Capabilities`.
|
|
10
|
-
* 2.
|
|
11
|
-
* `
|
|
12
|
-
*
|
|
10
|
+
* 2. `protocolVersion` is advertised, and every event carries the six
|
|
11
|
+
* required `RunEventDoc` fields.
|
|
12
|
+
*
|
|
13
|
+
* This file previously claimed to check "the four version axes
|
|
14
|
+
* (`engineVersion`, `eventLogSchemaVersion`, per-event `schemaVersion`,
|
|
15
|
+
* `pinnedVersions`)". IT DID NOT. `protocolVersion` was the only axis
|
|
16
|
+
* asserted, and across all 444 v1 scenario files the sole occurrence of the
|
|
17
|
+
* identifier `eventLogSchemaVersion` was that sentence — a docstring
|
|
18
|
+
* describing a check that did not exist. A comment claiming coverage is
|
|
19
|
+
* worse than no comment: it answers "is this tested?" for anyone who greps,
|
|
20
|
+
* and answers it wrongly.
|
|
21
|
+
*
|
|
22
|
+
* Current state of the four, stated so this comment can be checked rather
|
|
23
|
+
* than trusted: `eventLogSchemaVersion` and `engineVersion` are witnessed by
|
|
24
|
+
* `era-key-stamped-v1.test.ts` (both are run-document `MUST`s in
|
|
25
|
+
* `version-negotiation.md` §Stamping, and both were unasserted until
|
|
26
|
+
* 2026-09-04). Per-event `schemaVersion` and `pinnedVersions` are **not
|
|
27
|
+
* asserted here and carry no `MUST` in that document** — checked, rather
|
|
28
|
+
* than assumed to be a gap.
|
|
29
|
+
*
|
|
30
|
+
* This paragraph was itself wrong for one release candidate: it said
|
|
31
|
+
* `engineVersion` "remains UNASSERTED" after the leg asserting it had
|
|
32
|
+
* landed. A docstring that describes coverage goes stale the moment
|
|
33
|
+
* coverage changes, which is the argument for stating what can be
|
|
34
|
+
* re-derived rather than what was true once.
|
|
13
35
|
* 3. Forward-compat read: events carrying an UNKNOWN
|
|
14
36
|
* `schemaVersion` SHOULD still be readable via the events/poll
|
|
15
37
|
* endpoint without 5xx (best-effort fold per
|
package/src/setup.ts
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
* `it` here; vitest treats setupFiles differently from scenario files.
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
|
-
import { setAdvertisedFixtures } from './lib/fixtures.js';
|
|
28
|
+
import { setAdvertisedFixtures, setDiscoveryUnreadable } from './lib/fixtures.js';
|
|
29
29
|
import { setMultiAgentCapabilities } from './lib/multi-agent-capabilities.js';
|
|
30
30
|
import { OtelCollector, setCollector } from './lib/otel-collector.js';
|
|
31
31
|
import { McpFakeServer, setMcpFakeServer } from './lib/mcp-fake-server.js';
|
|
@@ -35,19 +35,23 @@ import { basename, join } from 'node:path';
|
|
|
35
35
|
import { existsSync, readFileSync } from 'node:fs';
|
|
36
36
|
import { PKG_ROOT_PATH } from './lib/paths.js';
|
|
37
37
|
import { recordRequirement, hasRequirement, journalLength, journalSince } from './lib/requirement-ledger.js';
|
|
38
|
-
import { requirementIdForFile, resolveFileRecord, type FileTestState } from './lib/scenario-disposition.js';
|
|
39
|
-
import { softSkipDisposition } from './lib/soft-skip.js';
|
|
38
|
+
import { requirementIdForFile, resolveFileRecord, resolveItRecord, type FileTestState } from './lib/scenario-disposition.js';
|
|
39
|
+
import { softSkipDisposition, softSkipDispositionSince, softSkipMark } from './lib/soft-skip.js';
|
|
40
40
|
import { ItIdAllocator, takeExplicitRequirementId } from './lib/requirement-ids.js';
|
|
41
41
|
import { SPEC_COHERENCE_SCENARIOS, SPEC_COHERENCE_DETAIL } from './lib/spec-coherence.js';
|
|
42
42
|
import type { DiscoveryPayload } from './lib/profiles.js';
|
|
43
|
+
import { targetMajor } from './lib/seams.js';
|
|
44
|
+
import { softSkip } from './lib/soft-skip.js';
|
|
43
45
|
|
|
44
|
-
|
|
46
|
+
// 20 s, not 5: a Cloud Run cold start routinely exceeds 5 s, and a discovery
|
|
47
|
+
// fetch that aborted at init used to turn every fixture-gated scenario into a
|
|
48
|
+
// vacuous `inapplicable`. Measured 2026-09-05 on a host answering in 200 ms.
|
|
49
|
+
const SUITE_INIT_TIMEOUT_MS = 20_000;
|
|
50
|
+
const SUITE_INIT_ATTEMPTS = 2;
|
|
45
51
|
|
|
46
52
|
async function loadHostFixtures(): Promise<void> {
|
|
47
53
|
const baseUrl = process.env.OPENWOP_BASE_URL?.trim();
|
|
48
54
|
if (!baseUrl) {
|
|
49
|
-
// Offline / fixture-stub-only run. No host to ask; treat as "host
|
|
50
|
-
// advertises no fixtures" so all fixture-dependent scenarios skip.
|
|
51
55
|
setAdvertisedFixtures(null);
|
|
52
56
|
setMultiAgentCapabilities(null);
|
|
53
57
|
return;
|
|
@@ -55,39 +59,36 @@ async function loadHostFixtures(): Promise<void> {
|
|
|
55
59
|
|
|
56
60
|
const normalizedBase = baseUrl.replace(/\/$/, '');
|
|
57
61
|
const url = `${normalizedBase}/.well-known/openwop`;
|
|
62
|
+
// The representation the header selects is the one whose fixtures[] this
|
|
63
|
+
// lane is held to (versioning.md §5): read it under the lane's major.
|
|
64
|
+
const headers: Record<string, string> = { Accept: 'application/json' };
|
|
65
|
+
if (targetMajor() === 2) headers['OpenWOP-Version'] = '2.0';
|
|
58
66
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
);
|
|
73
|
-
setAdvertisedFixtures(null);
|
|
74
|
-
setMultiAgentCapabilities(null);
|
|
67
|
+
let lastFailure = 'unknown';
|
|
68
|
+
for (let attempt = 1; attempt <= SUITE_INIT_ATTEMPTS; attempt += 1) {
|
|
69
|
+
const controller = new AbortController();
|
|
70
|
+
const timer = setTimeout(() => controller.abort(), SUITE_INIT_TIMEOUT_MS);
|
|
71
|
+
try {
|
|
72
|
+
const res = await fetch(url, { method: 'GET', headers, signal: controller.signal });
|
|
73
|
+
if (!res.ok) {
|
|
74
|
+
lastFailure = `HTTP ${res.status} on attempt ${attempt}`;
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
const body = (await res.json()) as DiscoveryPayload;
|
|
78
|
+
setAdvertisedFixtures(body);
|
|
79
|
+
setMultiAgentCapabilities(body);
|
|
75
80
|
return;
|
|
81
|
+
} catch (err) {
|
|
82
|
+
lastFailure = `${(err as Error).message ?? 'unknown'} on attempt ${attempt}`;
|
|
83
|
+
} finally {
|
|
84
|
+
clearTimeout(timer);
|
|
76
85
|
}
|
|
77
|
-
const body = (await res.json()) as DiscoveryPayload;
|
|
78
|
-
setAdvertisedFixtures(body);
|
|
79
|
-
setMultiAgentCapabilities(body);
|
|
80
|
-
} catch (err) {
|
|
81
|
-
// eslint-disable-next-line no-console
|
|
82
|
-
console.warn(
|
|
83
|
-
`[openwop-conformance setup] discovery fetch failed (${(err as Error).message ?? 'unknown'}); ` +
|
|
84
|
-
`treating host as advertising no fixtures. Fixture-dependent scenarios will skip.`,
|
|
85
|
-
);
|
|
86
|
-
setAdvertisedFixtures(null);
|
|
87
|
-
setMultiAgentCapabilities(null);
|
|
88
|
-
} finally {
|
|
89
|
-
clearTimeout(timer);
|
|
90
86
|
}
|
|
87
|
+
// Not "the host advertises no fixtures". The document went UNREAD, and every
|
|
88
|
+
// fixture gate will say so as `blocked` rather than `inapplicable`.
|
|
89
|
+
console.warn(`[openwop-conformance setup] discovery unreadable after ${SUITE_INIT_ATTEMPTS} attempt(s) (${lastFailure}); fixture-gated scenarios will record blocked, not inapplicable.`);
|
|
90
|
+
setDiscoveryUnreadable(lastFailure);
|
|
91
|
+
setMultiAgentCapabilities(null);
|
|
91
92
|
}
|
|
92
93
|
|
|
93
94
|
/**
|
|
@@ -238,6 +239,7 @@ const _ledgerMarks = new Map<string, number>();
|
|
|
238
239
|
const _itAllocators = new Map<string, ItIdAllocator>();
|
|
239
240
|
const _itMarks = new Map<string, number>();
|
|
240
241
|
const _itAssertionsBefore = new Map<string, number>();
|
|
242
|
+
const _itSoftSkipMarks = new Map<string, number>();
|
|
241
243
|
function _assertionCalls(): number {
|
|
242
244
|
try {
|
|
243
245
|
return (expect.getState() as { assertionCalls?: number }).assertionCalls ?? 0;
|
|
@@ -277,6 +279,62 @@ function _fileOf(task: { file?: { filepath?: string; name?: string } } | undefin
|
|
|
277
279
|
const f = task?.file?.filepath ?? task?.file?.name;
|
|
278
280
|
return typeof f === 'string' && f.length > 0 ? basename(f) : null;
|
|
279
281
|
}
|
|
282
|
+
// ---------------------------------------------------------------------------
|
|
283
|
+
// The applicability check and the assertion MUST run under the same contract.
|
|
284
|
+
//
|
|
285
|
+
// A scenario's registration in scenario-majors.json says which target majors it
|
|
286
|
+
// is written for. The driver reads OPENWOP_TARGET_MAJOR to decide which header
|
|
287
|
+
// and path space every probe uses. Nothing connected the two: a lane that ran
|
|
288
|
+
// vitest directly over all 501 files at the default (major 1) executed every
|
|
289
|
+
// major-2 scenario with major-1 requests. The scenarios' own gates call
|
|
290
|
+
// v2Discovery(), which sets the header EXPLICITLY, so the gate passed and the
|
|
291
|
+
// probe went out as v1 — a check that proved the host speaks v2 with one
|
|
292
|
+
// request, then tested a v2 requirement with a request that did not.
|
|
293
|
+
//
|
|
294
|
+
// Measured on a tier-1 host 2026-09-04: three "host defects" reported from that
|
|
295
|
+
// lane, all of which evaporated at major 2; the host was one edit from "fixing"
|
|
296
|
+
// correct behaviour. And in the other direction: 4 of 56 v2 files fail on every
|
|
297
|
+
// host forever under a major-1 driver, so a gate that runs them that way is red
|
|
298
|
+
// by construction and gets reasoned past. Both are the same defect: a scoped
|
|
299
|
+
// signal read as unscoped, with the scope nowhere in the output.
|
|
300
|
+
//
|
|
301
|
+
// So: a file whose registered majors do not include the driver's target major
|
|
302
|
+
// is INAPPLICABLE to this lane, recorded as such with the reason, and never
|
|
303
|
+
// probes. Files not in the registry (coherence checks, lib tests) are untouched.
|
|
304
|
+
// ---------------------------------------------------------------------------
|
|
305
|
+
const SCENARIO_MAJORS: Record<string, number[]> = (() => {
|
|
306
|
+
try {
|
|
307
|
+
const p = join(PKG_ROOT_PATH, 'scenario-majors.json');
|
|
308
|
+
if (!existsSync(p)) return {};
|
|
309
|
+
return (JSON.parse(readFileSync(p, 'utf8')) as { majors?: Record<string, number[]> }).majors ?? {};
|
|
310
|
+
} catch {
|
|
311
|
+
return {};
|
|
312
|
+
}
|
|
313
|
+
})();
|
|
314
|
+
|
|
315
|
+
beforeEach((ctx) => {
|
|
316
|
+
const p = (expect.getState() as { testPath?: string }).testPath;
|
|
317
|
+
if (!p) return;
|
|
318
|
+
const file = basename(p);
|
|
319
|
+
const majors = SCENARIO_MAJORS[file];
|
|
320
|
+
if (!majors) return;
|
|
321
|
+
const lane = targetMajor();
|
|
322
|
+
if (majors.includes(lane)) return;
|
|
323
|
+
const detail = `registered for target major ${majors.join('/')} and this lane runs at major ${lane} (OPENWOP_TARGET_MAJOR) — the probe would go out under a contract the scenario's gate does not use; select files with --target-major or set the variable`;
|
|
324
|
+
// Two records, one per resolver. The per-TEST disposition is read from the
|
|
325
|
+
// requirement journal (the same entry behaviorGate writes), so the row lands
|
|
326
|
+
// as `inapplicable` with this reason rather than `skipped`, which under RFC
|
|
327
|
+
// 0148 would claim the operator opted out. The per-FILE note covers the
|
|
328
|
+
// all-skipped fallback in resolveFileRecord.
|
|
329
|
+
try {
|
|
330
|
+
recordRequirement('openwop.family.lane-target-major', 'inapplicable', detail, { scenarioFile: file });
|
|
331
|
+
} catch {
|
|
332
|
+
/* never fail a test for bookkeeping */
|
|
333
|
+
}
|
|
334
|
+
softSkip('inapplicable', detail);
|
|
335
|
+
ctx.skip();
|
|
336
|
+
});
|
|
337
|
+
|
|
280
338
|
beforeAll(({}, suite) => {
|
|
281
339
|
// Mark the ledger journal BEFORE the file's tests run, so a behaviorGate
|
|
282
340
|
// decision made by the very first test is inside the file's window.
|
|
@@ -290,6 +348,7 @@ beforeEach(({ task }) => {
|
|
|
290
348
|
// Window for this test's own gate decisions and its own assertion count.
|
|
291
349
|
_itMarks.set(file, journalLength());
|
|
292
350
|
_itAssertionsBefore.set(file, _assertionCalls());
|
|
351
|
+
_itSoftSkipMarks.set(file, softSkipMark()); // rc.56: this test's own softSkip window
|
|
293
352
|
takeExplicitRequirementId(); // clear any override left by a test that threw before afterEach
|
|
294
353
|
});
|
|
295
354
|
afterEach(({ task }) => {
|
|
@@ -338,7 +397,8 @@ afterEach(({ task }) => {
|
|
|
338
397
|
_itAllocators.set(file, alloc);
|
|
339
398
|
const itId = explicit ?? alloc.allocate(file, task.name);
|
|
340
399
|
const since = journalSince(_itMarks.get(file) ?? 0);
|
|
341
|
-
const
|
|
400
|
+
const gateEntry = since.find((e) => e.disposition === 'inapplicable') ?? since.find((e) => e.disposition === 'skipped');
|
|
401
|
+
const gate = gateEntry === undefined ? undefined : { disposition: gateEntry.disposition as 'inapplicable' | 'skipped', ...(gateEntry.detail === undefined ? {} : { detail: gateEntry.detail }) };
|
|
342
402
|
let disposition: 'executed-pass' | 'executed-fail' | 'skipped' | 'inapplicable' | 'blocked';
|
|
343
403
|
let detail: string | undefined;
|
|
344
404
|
// Suite 2.0.0: under the corpus gate (scripts/check-spec-coherence.mjs sets OPENWOP_CORPUS_GATE) a coherence scenario IS the subject; its rows are real dispositions for evidence/corpus-ledger.json.
|
|
@@ -348,21 +408,15 @@ afterEach(({ task }) => {
|
|
|
348
408
|
// rule `resolveFileRecord` applies to the file in the published layout.
|
|
349
409
|
disposition = 'inapplicable';
|
|
350
410
|
detail = SPEC_COHERENCE_DETAIL;
|
|
351
|
-
} else if (state === 'fail') {
|
|
352
|
-
disposition = 'executed-fail';
|
|
353
|
-
const err = (task.result?.errors ?? [])[0] as { message?: string } | undefined;
|
|
354
|
-
detail = `the test executed and failed: ${(err?.message ?? 'no message').slice(0, 300)}`;
|
|
355
|
-
} else if (state === 'pass' && calls > 0) {
|
|
356
|
-
disposition = 'executed-pass';
|
|
357
|
-
} else if (gate !== undefined) {
|
|
358
|
-
disposition = gate.disposition as 'skipped' | 'inapplicable';
|
|
359
|
-
detail = gate.detail ?? `${gate.disposition} (gate recorded no reason)`;
|
|
360
|
-
} else if (state === 'pass') {
|
|
361
|
-
disposition = 'blocked';
|
|
362
|
-
detail = 'unclassified return: the test passed with zero assertions and recorded no reason — RFC 0148 §A resolves it to blocked, never to a pass';
|
|
363
411
|
} else {
|
|
364
|
-
|
|
365
|
-
|
|
412
|
+
// rc.56: the softSkip notes THIS test wrote are its reason (the file row
|
|
413
|
+
// already read them; the per-`it` row did not, and a leg that said
|
|
414
|
+
// `inapplicable` came out `blocked` — which denies certification bundle-wide).
|
|
415
|
+
const noted = softSkipDispositionSince(file, _itSoftSkipMarks.get(file) ?? 0);
|
|
416
|
+
const err = (task.result?.errors ?? [])[0] as { message?: string } | undefined;
|
|
417
|
+
const rec = resolveItRecord(state === 'pass' ? 'pass' : state === 'fail' ? 'fail' : 'skip', calls, gate, noted, err?.message);
|
|
418
|
+
disposition = rec.disposition;
|
|
419
|
+
detail = rec.detail;
|
|
366
420
|
}
|
|
367
421
|
try {
|
|
368
422
|
recordRequirement(itId, disposition, detail, { assertionCount: calls, scenarioFile: file });
|