@openwop/openwop-conformance 1.136.11 → 1.139.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +268 -3
- package/coverage.md +2 -2
- package/dist/cli.js +8 -1
- package/dist/lib/requirement-ledger.js +8 -0
- package/dist/lib/scenario-disposition.js +30 -0
- package/package.json +1 -1
- package/schemas/CORPUS-STAMP.json +2 -2
- package/schemas/capabilities.schema.json +1 -1
- package/schemas/certification-bundle-v2.schema.json +11 -1
- package/src/cli.ts +8 -1
- package/src/global-setup.ts +169 -0
- package/src/lib/fork-availability.test.ts +69 -0
- package/src/lib/fork-availability.ts +61 -0
- package/src/lib/global-setup.test.ts +76 -0
- package/src/lib/polling.test.ts +80 -0
- package/src/lib/polling.ts +39 -2
- package/src/lib/requirement-ledger.test.ts +75 -0
- package/src/lib/requirement-ledger.ts +9 -0
- package/src/lib/risk-disposition.test.ts +91 -0
- package/src/lib/scenario-disposition.ts +30 -0
- package/src/lib/webhook-receiver.test.ts +76 -0
- package/src/lib/webhook-receiver.ts +24 -3
- package/src/scenarios/a2a-task-roundtrip.test.ts +66 -10
- package/src/scenarios/agent-channel-dispatch.test.ts +3 -3
- package/src/scenarios/conformance-execution-witness.test.ts +29 -0
- package/src/scenarios/cross-host-traceparent-propagation.test.ts +10 -6
- package/src/scenarios/durability-poison-exhaustion.test.ts +154 -0
- package/src/scenarios/replay-fanout-suppression.test.ts +328 -0
- package/src/scenarios/replay-fork-arbitrary.test.ts +9 -3
- package/src/scenarios/replay-fork.test.ts +10 -2
- package/src/scenarios/replay-side-effect-suppression.test.ts +3 -2
- package/src/scenarios/replayDeterminism.test.ts +8 -6
- package/src/scenarios/webhook-receiver-adversarial.test.ts +9 -3
- package/src/scenarios/webhook-signed-delivery.test.ts +76 -13
- package/src/setup.ts +24 -5
- package/vitest.config.ts +7 -0
package/coverage.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# OpenWOP Conformance Coverage Map
|
|
2
2
|
|
|
3
|
-
> **Status: Living document. Updated 2026-
|
|
3
|
+
> **Status: Living document. Updated 2026-08-25.** This map connects the current scenario files to the protocol surfaces they protect and records the remaining gaps from the protocol deep dive. Scenario names are source-of-truth file names under `conformance/src/scenarios/`.
|
|
4
4
|
|
|
5
5
|
> **Shape grade vs behavior grade.** Some optional-profile scenarios validate **capability shape** (the host's discovery advertisement is well-formed) without yet exercising **behavior** (the host actually implements the profile end-to-end). The "Current grade" column reflects shape; see §"Capability-gated scenarios: shape vs behavior" below for the dual-grade view and the `OPENWOP_REQUIRE_BEHAVIOR=true` strict-mode runner flag.
|
|
6
6
|
|
|
@@ -200,7 +200,7 @@ Every OpenAPI operation should have:
|
|
|
200
200
|
4. A cited spec section in each assertion message.
|
|
201
201
|
|
|
202
202
|
| Operation ID | Positive coverage | Negative / auth / validation coverage | Gap |
|
|
203
|
-
| ------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
203
|
+
| ------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
204
204
|
| `getCapabilities` | `discovery.test.ts`, `runtime-capabilities.test.ts`, `profileDerivation.test.ts`, `mcp-discoverability.test.ts` | `discovery.test.ts` covers optional `Capabilities-Etag`; `spec-corpus-validity.test.ts` validates schema shape | Add scoped discovery scenario when a host advertises it. |
|
|
205
205
|
| `getOpenApiSpec` | `discovery.test.ts` | `spec-corpus-validity.test.ts` validates OpenAPI refs | Add unavailable/transient error scenario only if host can simulate it. |
|
|
206
206
|
| `getWorkflow` | `route-coverage.test.ts`; fixture-dependent lifecycle tests indirectly require seeded workflow IDs | `route-coverage.test.ts` covers unknown workflow `404`/`403` envelope | Good. |
|
package/dist/cli.js
CHANGED
|
@@ -17,7 +17,14 @@
|
|
|
17
17
|
* Environment variables override flags (per the conformance harness's
|
|
18
18
|
* existing convention):
|
|
19
19
|
* OPENWOP_BASE_URL, OPENWOP_API_KEY, OPENWOP_IMPLEMENTATION_NAME,
|
|
20
|
-
* OPENWOP_IMPLEMENTATION_VERSION, OPENWOP_LIFECYCLE_TIMEOUT_MS
|
|
20
|
+
* OPENWOP_IMPLEMENTATION_VERSION, OPENWOP_LIFECYCLE_TIMEOUT_MS,
|
|
21
|
+
* OPENWOP_POLL_TIMEOUT_SCALE, OPENWOP_MAX_WORKERS
|
|
22
|
+
*
|
|
23
|
+
* OPENWOP_LIFECYCLE_TIMEOUT_MS sets the DEFAULT poll bound only — it does
|
|
24
|
+
* not reach the many scenarios that pass an explicit `timeoutMs`.
|
|
25
|
+
* OPENWOP_POLL_TIMEOUT_SCALE (default 1) multiplies EVERY bound and is the
|
|
26
|
+
* knob to reach for when a run's failures are the endpoint (cold start,
|
|
27
|
+
* contention) rather than the host. See src/lib/polling.ts.
|
|
21
28
|
*
|
|
22
29
|
* Exit codes:
|
|
23
30
|
* 0 all scenarios pass
|
|
@@ -123,6 +123,14 @@ export function readLedgerFile(path) {
|
|
|
123
123
|
* returned early, threw and swallowed, or was never written leaves no entry, and
|
|
124
124
|
* the honest reading of no entry is "this was not exercised".
|
|
125
125
|
*/
|
|
126
|
+
/**
|
|
127
|
+
* Has this requirement already been recorded in THIS run? Distinguishes "the
|
|
128
|
+
* scenario classified itself" from "nothing has been recorded yet" — which
|
|
129
|
+
* `dispositionOf` cannot, since it folds the absent case to `blocked`.
|
|
130
|
+
*/
|
|
131
|
+
export function hasRequirement(requirementId) {
|
|
132
|
+
return ledger.has(requirementId);
|
|
133
|
+
}
|
|
126
134
|
export function dispositionOf(requirementId) {
|
|
127
135
|
return ledger.get(requirementId)?.disposition ?? 'blocked';
|
|
128
136
|
}
|
|
@@ -48,6 +48,14 @@ export function requirementIdForFile(basename) {
|
|
|
48
48
|
? requirementIdForScenario(basename)
|
|
49
49
|
: `openwop.scenario.${basename.replace(/\.test\.ts$/, '')}`;
|
|
50
50
|
}
|
|
51
|
+
/**
|
|
52
|
+
* Marks an `executed-pass` row whose file ALSO recorded a soft-skip note — the
|
|
53
|
+
* file asserted something, then stopped short (gap G8). Greppable on purpose: a
|
|
54
|
+
* bundle reader filters `disposition === 'executed-pass' && detail?.startsWith(
|
|
55
|
+
* PARTIAL_WITNESS_PREFIX)` to find rows where the requirement may not have been
|
|
56
|
+
* the thing that passed.
|
|
57
|
+
*/
|
|
58
|
+
export const PARTIAL_WITNESS_PREFIX = 'partial-witness: ';
|
|
51
59
|
/** Worker half: fold a file's per-test states (+ any gate-recorded reason) into
|
|
52
60
|
* the ONE disposition the file records. */
|
|
53
61
|
export function fileDisposition(states, gateReason, assertionCount) {
|
|
@@ -115,6 +123,28 @@ export function resolveFileRecord(states, gateReason, assertionCount, noted) {
|
|
|
115
123
|
disposition = noted.kind;
|
|
116
124
|
detail = noted.reason;
|
|
117
125
|
}
|
|
126
|
+
else if (noted !== null && disposition === 'executed-pass') {
|
|
127
|
+
// PARTIAL WITNESS (2026-08-19, gap G8). The file asserted something and then
|
|
128
|
+
// soft-skipped: `return softSkip(...)` yields a PASS state, not a skip, so
|
|
129
|
+
// neither branch above fires and the note used to be discarded outright. The
|
|
130
|
+
// row then read `executed-pass` for a requirement the run may never have
|
|
131
|
+
// reached — e.g. a file asserting a `201` setup precondition before
|
|
132
|
+
// returning `inapplicable`.
|
|
133
|
+
//
|
|
134
|
+
// Same defect as the note-after-`ctx.skip()` case the comment above records;
|
|
135
|
+
// note-after-ASSERTION was the half that stayed. Both hid because nothing
|
|
136
|
+
// goes red.
|
|
137
|
+
//
|
|
138
|
+
// The disposition is deliberately NOT changed. Honouring the note here would
|
|
139
|
+
// downgrade a file that legitimately completed its requirement AND
|
|
140
|
+
// soft-skipped an optional extra leg — trading a false positive for a false
|
|
141
|
+
// negative, on a per-FILE note that cannot say which leg it came from. The
|
|
142
|
+
// durable fix is per-`it` recording; this makes the affected rows
|
|
143
|
+
// self-identifying first, so that change follows measurement instead of
|
|
144
|
+
// preceding it. `detail` is permitted on `executed-pass` (RFC 0148 §A only
|
|
145
|
+
// REQUIRES it for other dispositions), so this is additive on the wire.
|
|
146
|
+
detail = `${PARTIAL_WITNESS_PREFIX}${noted.kind}: ${noted.reason}`;
|
|
147
|
+
}
|
|
118
148
|
return detail === undefined ? { disposition } : { disposition, detail };
|
|
119
149
|
}
|
|
120
150
|
/**
|
package/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_comment": "Provenance of this vendored schemas/ copy. See conformance/README.md \u00a7\"Resolving the contract\". Compare against the stamp in your installed @openwop/openwop-conformance to detect a stale hand-copied contract.",
|
|
3
|
-
"suiteVersion": "1.
|
|
4
|
-
"corpusCommit": "
|
|
3
|
+
"suiteVersion": "1.139.0",
|
|
4
|
+
"corpusCommit": "f34a41ac4c1737ca8af34d0f7f17876268f58e69"
|
|
5
5
|
}
|
|
@@ -3495,7 +3495,7 @@
|
|
|
3495
3495
|
},
|
|
3496
3496
|
"idempotency": {
|
|
3497
3497
|
"type": "object",
|
|
3498
|
-
"description": "RFC 0036 \u2014 Multi-region idempotency contract. Optional v1 advertisement. The existing `crossRegion: 'single-region'|'
|
|
3498
|
+
"description": "RFC 0036 \u2014 Multi-region idempotency contract. Optional v1 advertisement. The existing `crossRegion: 'single-region'|'reconciled-records'|'fenced-effects'` categorical claim lives under `capabilities.idempotency.crossRegion` (this description named the pre-RFC-0150-\u00a7D vocabulary `'best-effort'|'strict'` until 2026-08-18 \u2014 both values were RETIRED by \u00a7D and the sibling `crossRegion` enum has been correct throughout, so only this prose was wrong; a reader of it would have believed two non-values were advertisable) per spec/v1/idempotency.md \u00a7'Multi-region idempotency (annex)'. The `multiRegion` sub-block here gives a granular advertisement that hosts SHOULD pair with the categorical `crossRegion` claim.",
|
|
3499
3499
|
"additionalProperties": true,
|
|
3500
3500
|
"properties": {
|
|
3501
3501
|
"supported": {
|
|
@@ -24,7 +24,17 @@
|
|
|
24
24
|
"properties": {
|
|
25
25
|
"name": { "type": "string", "minLength": 1 },
|
|
26
26
|
"version": { "type": "string", "minLength": 1 },
|
|
27
|
-
"vendor": { "type": "string" }
|
|
27
|
+
"vendor": { "type": "string" },
|
|
28
|
+
"commit": {
|
|
29
|
+
"type": "string",
|
|
30
|
+
"minLength": 1,
|
|
31
|
+
"description": "OPTIONAL, self-reported source revision the host was built from. NOT sufficient to attribute the bundle: a commit does not determine the artifact unless the build is deterministic — a tier-2 host measured 283/22 vs 303/2 from two builds of one commit whose container image ran `npm install` rather than `npm ci` (conformance-certification.md §\"What a bundle attributes to\")."
|
|
32
|
+
},
|
|
33
|
+
"build": {
|
|
34
|
+
"type": "string",
|
|
35
|
+
"minLength": 1,
|
|
36
|
+
"description": "OPTIONAL, self-reported identifier of what ACTUALLY EXECUTED — this, not `commit`, is what a verifier needs to re-run the same thing. Which identifier is reachable depends on when the bundle is written: a container image digest is strongest but does not exist yet for a bundle baked INTO the image at certify time; a builder-assigned build id is available during the build; a digest of the resolved dependency manifest (lockfile or resolved tree) is available at certify time and pins the closure that actually varied in the measurement behind this field. State which kind it is rather than reaching for the strongest-sounding one. See conformance-certification.md §\"A bundle attributes to a BUILD\"."
|
|
37
|
+
}
|
|
28
38
|
}
|
|
29
39
|
},
|
|
30
40
|
"discovery": {
|
package/src/cli.ts
CHANGED
|
@@ -17,7 +17,14 @@
|
|
|
17
17
|
* Environment variables override flags (per the conformance harness's
|
|
18
18
|
* existing convention):
|
|
19
19
|
* OPENWOP_BASE_URL, OPENWOP_API_KEY, OPENWOP_IMPLEMENTATION_NAME,
|
|
20
|
-
* OPENWOP_IMPLEMENTATION_VERSION, OPENWOP_LIFECYCLE_TIMEOUT_MS
|
|
20
|
+
* OPENWOP_IMPLEMENTATION_VERSION, OPENWOP_LIFECYCLE_TIMEOUT_MS,
|
|
21
|
+
* OPENWOP_POLL_TIMEOUT_SCALE, OPENWOP_MAX_WORKERS
|
|
22
|
+
*
|
|
23
|
+
* OPENWOP_LIFECYCLE_TIMEOUT_MS sets the DEFAULT poll bound only — it does
|
|
24
|
+
* not reach the many scenarios that pass an explicit `timeoutMs`.
|
|
25
|
+
* OPENWOP_POLL_TIMEOUT_SCALE (default 1) multiplies EVERY bound and is the
|
|
26
|
+
* knob to reach for when a run's failures are the endpoint (cold start,
|
|
27
|
+
* contention) rather than the host. See src/lib/polling.ts.
|
|
21
28
|
*
|
|
22
29
|
* Exit codes:
|
|
23
30
|
* 0 all scenarios pass
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run-end disposition summary (RFC 0148 §A, 2026-08-25).
|
|
3
|
+
*
|
|
4
|
+
* ## The gap this closes
|
|
5
|
+
*
|
|
6
|
+
* Every scenario file already records ONE honest disposition when it
|
|
7
|
+
* finishes (`setup.ts`'s `afterAll` → `resolveFileRecord`): a file that
|
|
8
|
+
* returned early with zero assertions is `blocked` / `skipped` /
|
|
9
|
+
* `inapplicable` with a reason, never a pass. That machinery is correct.
|
|
10
|
+
*
|
|
11
|
+
* It was **published on one invocation out of two**. The worker appends
|
|
12
|
+
* each recording to `OPENWOP_LEDGER_PATH`, and the ONLY writer of that
|
|
13
|
+
* variable was `cli.ts` inside `runCertify`. Under a plain `vitest run`
|
|
14
|
+
* — what `README.md` documents for host operators, and what host
|
|
15
|
+
* implementers actually run — the worker computed `blocked`, held it in
|
|
16
|
+
* memory, and discarded it at process exit. The single surviving artifact
|
|
17
|
+
* was vitest's `1 passed`.
|
|
18
|
+
*
|
|
19
|
+
* So the suite computed an honest disposition on every run and published
|
|
20
|
+
* it on one. Two host implementers independently read the console line as
|
|
21
|
+
* coverage and reported the suite as claiming a pass it had internally
|
|
22
|
+
* classified `blocked` — correctly, given the only artifact they had.
|
|
23
|
+
* RFC 0158 §"Operator preconditions are declared, not hidden" requires a
|
|
24
|
+
* blocked row be surfaced "never silently skipped or reported as a pass";
|
|
25
|
+
* the bundle honored that and the console contradicted it.
|
|
26
|
+
*
|
|
27
|
+
* ## What this does
|
|
28
|
+
*
|
|
29
|
+
* When `OPENWOP_LEDGER_PATH` is unset, point it at a temp file so the
|
|
30
|
+
* workers record there anyway, then read it at run end and print a
|
|
31
|
+
* summary. `--certify` sets the variable itself, so this defers to it
|
|
32
|
+
* completely and never interferes with bundle generation.
|
|
33
|
+
*
|
|
34
|
+
* The summary is deliberately NOT a pass/fail signal — it does not change
|
|
35
|
+
* the exit code. `vitest`'s exit code answers "did any assertion fail";
|
|
36
|
+
* this answers "what did the run actually witness", and those are
|
|
37
|
+
* different questions. Conflating them is what produced the gap.
|
|
38
|
+
*
|
|
39
|
+
* Set `OPENWOP_DISPOSITION_SUMMARY=false` to silence it.
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
43
|
+
import { join } from 'node:path';
|
|
44
|
+
import { tmpdir } from 'node:os';
|
|
45
|
+
|
|
46
|
+
/** Dispositions that are NOT a witnessed execution — the rows worth printing. */
|
|
47
|
+
const NON_EXECUTING = new Set(['blocked', 'skipped', 'inapplicable']);
|
|
48
|
+
|
|
49
|
+
/** Cap on printed rows; the overflow is always announced, never silent. */
|
|
50
|
+
const MAX_ROWS = 40;
|
|
51
|
+
|
|
52
|
+
// Import the REAL entry type rather than restating its shape. A hand-written
|
|
53
|
+
// structural guess here read `entry.id` — a key the ledger has never written
|
|
54
|
+
// (it is `requirementId`) — and the `(unnamed)` fallback made the mismatch
|
|
55
|
+
// look like missing data instead of a wrong reader. A type-only import erases
|
|
56
|
+
// at runtime, so this costs nothing and makes the compiler the oracle: rename
|
|
57
|
+
// a ledger field and this file stops building instead of silently degrading.
|
|
58
|
+
import type { LedgerEntry } from './lib/requirement-ledger.js';
|
|
59
|
+
|
|
60
|
+
/** What a JSONL line parses to before validation — every field may be absent. */
|
|
61
|
+
type LedgerLine = Partial<Record<keyof LedgerEntry, unknown>>;
|
|
62
|
+
|
|
63
|
+
let ownedDir: string | null = null;
|
|
64
|
+
let ledgerPath: string | null = null;
|
|
65
|
+
|
|
66
|
+
export function setup(): void {
|
|
67
|
+
if (process.env['OPENWOP_DISPOSITION_SUMMARY'] === 'false') return;
|
|
68
|
+
// `--certify` already routes the ledger to its own report dir and reads it
|
|
69
|
+
// there. Never take it over — this exists only for the path that had no
|
|
70
|
+
// reader at all.
|
|
71
|
+
if ((process.env['OPENWOP_LEDGER_PATH'] ?? '') !== '') return;
|
|
72
|
+
ownedDir = mkdtempSync(join(tmpdir(), 'owp-dispositions-'));
|
|
73
|
+
ledgerPath = join(ownedDir, 'requirement-ledger.jsonl');
|
|
74
|
+
process.env['OPENWOP_LEDGER_PATH'] = ledgerPath;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Fold raw JSONL into the printable summary. PURE — no I/O, no env — so
|
|
79
|
+
* `global-setup.test.ts` can pin it without a vitest subprocess or a host.
|
|
80
|
+
* Returns `null` when there is nothing to say.
|
|
81
|
+
*/
|
|
82
|
+
export function summarise(raw: string): string | null {
|
|
83
|
+
const counts = new Map<string, number>();
|
|
84
|
+
const rows: Array<{ id: string; disposition: string; detail: string }> = [];
|
|
85
|
+
for (const line of raw.split('\n')) {
|
|
86
|
+
if (line.trim() === '') continue;
|
|
87
|
+
let entry: LedgerLine;
|
|
88
|
+
try {
|
|
89
|
+
entry = JSON.parse(line) as LedgerLine;
|
|
90
|
+
} catch {
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
const disposition = typeof entry.disposition === 'string' ? entry.disposition : 'unknown';
|
|
94
|
+
const id = typeof entry.requirementId === 'string' ? entry.requirementId : '(unnamed)';
|
|
95
|
+
const detail = typeof entry.detail === 'string' ? entry.detail : '';
|
|
96
|
+
counts.set(disposition, (counts.get(disposition) ?? 0) + 1);
|
|
97
|
+
if (NON_EXECUTING.has(disposition)) {
|
|
98
|
+
rows.push({ id, disposition, detail });
|
|
99
|
+
} else if (disposition === 'executed-pass' && entry.assertionCount === 0) {
|
|
100
|
+
// The other shape RFC 0148 §A rejects: a pass that asserted nothing.
|
|
101
|
+
rows.push({ id, disposition: 'executed-pass (0 assertions)', detail });
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (counts.size === 0) return null;
|
|
105
|
+
const total = [...counts.values()].reduce((a, b) => a + b, 0);
|
|
106
|
+
const order = ['executed-pass', 'executed-fail', 'blocked', 'skipped', 'inapplicable'];
|
|
107
|
+
const summary = [...order, ...[...counts.keys()].filter((k) => !order.includes(k))]
|
|
108
|
+
.filter((k) => counts.has(k))
|
|
109
|
+
.map((k) => `${k} ${counts.get(k)}`)
|
|
110
|
+
.join(' \u00b7 ');
|
|
111
|
+
const out: string[] = [
|
|
112
|
+
'',
|
|
113
|
+
`[openwop-conformance] RFC 0148 \u00a7A dispositions \u2014 ${total} requirement(s) recorded`,
|
|
114
|
+
` ${summary}`,
|
|
115
|
+
];
|
|
116
|
+
if (rows.length > 0) {
|
|
117
|
+
out.push(
|
|
118
|
+
` ${rows.length} requirement(s) did NOT witness. vitest reports these as passes because`,
|
|
119
|
+
' no assertion failed; that is a test outcome, not a conformance disposition.',
|
|
120
|
+
);
|
|
121
|
+
for (const r of rows.slice(0, MAX_ROWS)) {
|
|
122
|
+
const detail = r.detail.length > 120 ? `${r.detail.slice(0, 117)}...` : r.detail;
|
|
123
|
+
out.push(` ${r.disposition.padEnd(28)} ${r.id}${detail === '' ? '' : `\n ${detail}`}`);
|
|
124
|
+
}
|
|
125
|
+
if (rows.length > MAX_ROWS) {
|
|
126
|
+
// Never truncate silently: a capped list that does not say it was capped
|
|
127
|
+
// reads as a complete one, which is the same defect this whole summary
|
|
128
|
+
// exists to remove.
|
|
129
|
+
out.push(` ... and ${rows.length - MAX_ROWS} more not listed (run with --certify for the full bundle)`);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
out.push('');
|
|
133
|
+
return out.join('\n');
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export function teardown(): void {
|
|
137
|
+
if (ledgerPath === null || ownedDir === null) return;
|
|
138
|
+
// We set the variable; unset it so a subsequent in-process run re-derives.
|
|
139
|
+
delete process.env['OPENWOP_LEDGER_PATH'];
|
|
140
|
+
let raw = '';
|
|
141
|
+
try {
|
|
142
|
+
raw = readFileSync(ledgerPath, 'utf8');
|
|
143
|
+
} catch {
|
|
144
|
+
// No file means no worker ever recorded \u2014 nothing to summarise, and a
|
|
145
|
+
// missing summary must never be mistaken for "nothing was blocked", so
|
|
146
|
+
// say so rather than printing an empty clean bill.
|
|
147
|
+
process.stderr.write(
|
|
148
|
+
'\n[openwop-conformance] no disposition ledger was written \u2014 the run recorded nothing, ' +
|
|
149
|
+
'which is NOT the same as every scenario having witnessed its requirement.\n',
|
|
150
|
+
);
|
|
151
|
+
cleanup();
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
const text = summarise(raw);
|
|
155
|
+
if (text !== null) process.stderr.write(text);
|
|
156
|
+
cleanup();
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function cleanup(): void {
|
|
160
|
+
if (ownedDir !== null) {
|
|
161
|
+
try {
|
|
162
|
+
rmSync(ownedDir, { recursive: true, force: true });
|
|
163
|
+
} catch {
|
|
164
|
+
/* best effort */
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
ownedDir = null;
|
|
168
|
+
ledgerPath = null;
|
|
169
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pins `forkDeclined` (RFC 0148 §A).
|
|
3
|
+
*
|
|
4
|
+
* The risk in a helper like this is not that it misses a status — that shows up
|
|
5
|
+
* as a red test somebody investigates. It is that it swallows one: every status
|
|
6
|
+
* it accepts turns an assertion into a skip, silently, and a suite that skips
|
|
7
|
+
* is indistinguishable from a suite that passes unless something records why.
|
|
8
|
+
* So the load-bearing cases here are the NEGATIVE ones.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { describe, expect, it, beforeEach } from 'vitest';
|
|
12
|
+
import { forkDeclined } from './fork-availability.js';
|
|
13
|
+
import { resetSoftSkips, softSkipDisposition } from './soft-skip.js';
|
|
14
|
+
|
|
15
|
+
describe('forkDeclined', () => {
|
|
16
|
+
beforeEach(() => resetSoftSkips());
|
|
17
|
+
|
|
18
|
+
it('treats 404 and 403 the same as 501 — route absent is not weaker than route-declines', () => {
|
|
19
|
+
// The asymmetry this fixes: every replay scenario handled 501 and none
|
|
20
|
+
// handled 404. `501` means "I know this route and decline"; `404` means
|
|
21
|
+
// "there is no such route" — strictly less implemented. The suite treated
|
|
22
|
+
// the weaker signal as a skip and the stronger one as a defect, so a host
|
|
23
|
+
// had to implement the route in order to say it had not implemented it.
|
|
24
|
+
for (const status of [404, 403, 501]) {
|
|
25
|
+
expect(forkDeclined(status, 'leg'), `status ${status} means the fork did not happen`).toBe(true);
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
it('does NOT swallow any other status — a 500 is still a defect, not a skip', () => {
|
|
30
|
+
// If this ever goes green for one of these, a real failure has become a
|
|
31
|
+
// silent skip and no assertion downstream will ever run again.
|
|
32
|
+
for (const status of [200, 201, 202, 400, 409, 422, 500, 502, 503]) {
|
|
33
|
+
expect(forkDeclined(status, 'leg'), `status ${status} must reach the assertion`).toBe(false);
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it('records WHY, so the ledger row is not an unclassified return', () => {
|
|
38
|
+
// Two of the eleven original call sites were a bare `return`. A bare return
|
|
39
|
+
// in a file whose OTHER tests assert is invisible: the file records
|
|
40
|
+
// `executed-pass` and the leg that never ran leaves no trace.
|
|
41
|
+
forkDeclined(404, 'fanout-suppression replay fork');
|
|
42
|
+
const noted = softSkipDisposition('fork-availability.test.ts');
|
|
43
|
+
expect(noted?.kind).toBe('blocked');
|
|
44
|
+
expect(noted?.reason).toContain('404');
|
|
45
|
+
expect(noted?.reason).toContain('fanout-suppression replay fork');
|
|
46
|
+
expect(noted?.reason).toContain('not mounted');
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it('records nothing when the status is not a decline', () => {
|
|
50
|
+
forkDeclined(201, 'leg');
|
|
51
|
+
expect(softSkipDisposition('fork-availability.test.ts')).toBeNull();
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it('names the leg, so one file with several forks stays attributable', () => {
|
|
55
|
+
forkDeclined(501, 'branch fork');
|
|
56
|
+
expect(softSkipDisposition('fork-availability.test.ts')?.reason).toContain('branch fork');
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('distinguishes the three declines in its reason text', () => {
|
|
60
|
+
forkDeclined(404, 'a');
|
|
61
|
+
const a = softSkipDisposition('fork-availability.test.ts')?.reason ?? '';
|
|
62
|
+
resetSoftSkips();
|
|
63
|
+
forkDeclined(501, 'a');
|
|
64
|
+
const b = softSkipDisposition('fork-availability.test.ts')?.reason ?? '';
|
|
65
|
+
// `blocked` alone does not tell a bundle reader whether the host lacks the
|
|
66
|
+
// route or declines the range; the reason must.
|
|
67
|
+
expect(a).not.toBe(b);
|
|
68
|
+
});
|
|
69
|
+
});
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* "The fork seam did not run" — one rule, one place (RFC 0148 §A).
|
|
3
|
+
*
|
|
4
|
+
* `POST /v1/runs/{runId}:fork` can decline in more than one way, and eleven
|
|
5
|
+
* call sites across six scenarios each decided for themselves which ways
|
|
6
|
+
* counted:
|
|
7
|
+
*
|
|
8
|
+
* - **501** — every site handled this. "Advertised but not implemented for
|
|
9
|
+
* this range", per the suite convention their comments cite.
|
|
10
|
+
* - **404 / 403** — *no* site in the replay family handled it. A host that
|
|
11
|
+
* never mounts the route hard-failed `expect(status).toBe(201)`.
|
|
12
|
+
*
|
|
13
|
+
* That asymmetry is backwards. `501` means "I know this route and decline";
|
|
14
|
+
* `404` means "there is no such route" — strictly less implemented, and the
|
|
15
|
+
* suite treated the weaker signal as a skip and the stronger one as a defect.
|
|
16
|
+
* The postgres reference host 404s, so `replay-fanout-suppression` was red in
|
|
17
|
+
* CI on every run while the other four replay scenarios skipped past the same
|
|
18
|
+
* seam for unrelated fixture reasons. **The suite required a host to implement
|
|
19
|
+
* the route in order to say it had not implemented the route.**
|
|
20
|
+
*
|
|
21
|
+
* Recording matters as much as the predicate. Two of those eleven sites were a
|
|
22
|
+
* bare `return` with no note: `replay-fork.test.ts` reported "6 tests | 6
|
|
23
|
+
* skipped" against a host whose fork surface is entirely unmeasured, and the
|
|
24
|
+
* ledger had nothing to say about why. §A resolves an unclassified return to
|
|
25
|
+
* `blocked`, but only when the file records nothing at all — a bare return in a
|
|
26
|
+
* file whose *other* tests assert is invisible.
|
|
27
|
+
*
|
|
28
|
+
* Disposition is `blocked`, not `inapplicable`: the obligation applies to this
|
|
29
|
+
* host, it simply could not be witnessed. `inapplicable` would claim the host
|
|
30
|
+
* is outside the requirement's scope, which nothing here establishes — these
|
|
31
|
+
* scenarios carry no capability gate, so the suite does not actually know
|
|
32
|
+
* whether the host advertises fork. Saying `blocked` is the claim the evidence
|
|
33
|
+
* supports.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
import { softSkip } from './soft-skip.js';
|
|
37
|
+
|
|
38
|
+
/** Statuses that mean "the fork did not happen", with what each one tells us. */
|
|
39
|
+
const DECLINED: ReadonlyMap<number, string> = new Map([
|
|
40
|
+
[404, 'the route is not mounted on this host'],
|
|
41
|
+
[403, 'the caller is not permitted to fork (route present, access refused)'],
|
|
42
|
+
[501, 'the host knows the route and declines this range as not implemented'],
|
|
43
|
+
]);
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* True when `status` means the fork seam did not run — and records WHY for the
|
|
47
|
+
* RFC 0148 §A ledger as a side effect, so a caller can `if (forkDeclined(...))
|
|
48
|
+
* return;` without leaving an unclassified return behind.
|
|
49
|
+
*
|
|
50
|
+
* `where` names the leg, so a bundle reader can tell which of several forks in
|
|
51
|
+
* one file declined.
|
|
52
|
+
*/
|
|
53
|
+
export function forkDeclined(status: number, where: string): boolean {
|
|
54
|
+
const why = DECLINED.get(status);
|
|
55
|
+
if (why === undefined) return false;
|
|
56
|
+
softSkip(
|
|
57
|
+
'blocked',
|
|
58
|
+
`${where}: POST /v1/runs/{runId}:fork returned ${status} — ${why}, so the behaviour this requirement is stated over never occurred and its absence below would prove nothing`,
|
|
59
|
+
);
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pins the run-end disposition summary (`src/global-setup.ts`, RFC 0148 §A).
|
|
3
|
+
*
|
|
4
|
+
* The defect this guards is not "the summary is ugly" — it is that the suite
|
|
5
|
+
* computed an honest disposition on every run and PUBLISHED it on one. Two
|
|
6
|
+
* host implementers read vitest's `1 passed` as coverage for a requirement the
|
|
7
|
+
* suite had internally classified `blocked`, because the classification had no
|
|
8
|
+
* reader outside `--certify`.
|
|
9
|
+
*
|
|
10
|
+
* Server-free and pure: `summarise` takes the raw JSONL and returns the text.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { describe, expect, it } from 'vitest';
|
|
14
|
+
import { summarise } from '../global-setup.js';
|
|
15
|
+
|
|
16
|
+
const line = (o: Record<string, unknown>): string => JSON.stringify(o);
|
|
17
|
+
|
|
18
|
+
describe('global-setup: RFC 0148 §A disposition summary', () => {
|
|
19
|
+
it('reads the ledger key the ledger actually writes (`requirementId`, not `id`)', () => {
|
|
20
|
+
// Regression: the first draft of this reader keyed on `entry.id`, a field
|
|
21
|
+
// `recordRequirement` has never written, and fell back to `(unnamed)`.
|
|
22
|
+
// The fallback made a WRONG READER look like MISSING DATA — the same
|
|
23
|
+
// failure shape the summary exists to expose, reproduced in the exposer.
|
|
24
|
+
const out = summarise(
|
|
25
|
+
line({ requirementId: 'openwop.scenario.webhook-signed-delivery', disposition: 'blocked', detail: 'guard rejected loopback' }),
|
|
26
|
+
);
|
|
27
|
+
expect(out).toContain('openwop.scenario.webhook-signed-delivery');
|
|
28
|
+
expect(out).not.toContain('(unnamed)');
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
it('lists blocked / skipped / inapplicable as NOT witnessed, with the reason', () => {
|
|
32
|
+
const out = summarise(
|
|
33
|
+
[
|
|
34
|
+
line({ requirementId: 'a', disposition: 'blocked', detail: 'seam absent' }),
|
|
35
|
+
line({ requirementId: 'b', disposition: 'skipped', detail: 'operator opted out' }),
|
|
36
|
+
line({ requirementId: 'c', disposition: 'inapplicable', detail: 'profile not advertised' }),
|
|
37
|
+
].join('\n'),
|
|
38
|
+
);
|
|
39
|
+
expect(out).toContain('did NOT witness');
|
|
40
|
+
for (const s of ['seam absent', 'operator opted out', 'profile not advertised']) {
|
|
41
|
+
expect(out).toContain(s);
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it('flags an executed-pass that asserted nothing — the other RFC 0148 §A vacuity shape', () => {
|
|
46
|
+
const out = summarise(line({ requirementId: 'vacuous', disposition: 'executed-pass', assertionCount: 0 }));
|
|
47
|
+
expect(out).toContain('executed-pass (0 assertions)');
|
|
48
|
+
expect(out).toContain('vacuous');
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it('does NOT flag a real executed-pass', () => {
|
|
52
|
+
const out = summarise(line({ requirementId: 'real', disposition: 'executed-pass', assertionCount: 12 }));
|
|
53
|
+
expect(out).toContain('executed-pass 1');
|
|
54
|
+
expect(out).not.toContain('did NOT witness');
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it('announces truncation rather than capping silently', () => {
|
|
58
|
+
// A capped list that does not say it was capped reads as a complete one.
|
|
59
|
+
const many = Array.from({ length: 45 }, (_, i) =>
|
|
60
|
+
line({ requirementId: `r${i}`, disposition: 'blocked', detail: 'why' }),
|
|
61
|
+
).join('\n');
|
|
62
|
+
const out = summarise(many);
|
|
63
|
+
expect(out).toContain('5 more not listed');
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('returns null on an empty ledger so teardown prints nothing', () => {
|
|
67
|
+
expect(summarise('')).toBeNull();
|
|
68
|
+
expect(summarise('\n\n')).toBeNull();
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it('skips unparseable lines without discarding the rest of the run', () => {
|
|
72
|
+
const out = summarise(['{ not json', line({ requirementId: 'ok', disposition: 'blocked', detail: 'r' })].join('\n'));
|
|
73
|
+
expect(out).toContain('ok');
|
|
74
|
+
expect(out).toContain('blocked 1');
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for `polling.ts` — the poll-bound scaling knob.
|
|
3
|
+
*
|
|
4
|
+
* These exist because the defect being fixed was a knob that did not reach
|
|
5
|
+
* what its documentation said it reached: `OPENWOP_LIFECYCLE_TIMEOUT_MS` was
|
|
6
|
+
* documented as the way to bound long polls, but it supplies only the DEFAULT,
|
|
7
|
+
* so every call site passing an explicit `timeoutMs` ignored it. Shipping the
|
|
8
|
+
* replacement untested would repeat the defect one layer up, so the scale is
|
|
9
|
+
* asserted here rather than assumed.
|
|
10
|
+
*
|
|
11
|
+
* @see polling.ts
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { describe, it, expect, afterEach } from 'vitest';
|
|
15
|
+
import { scaledTimeoutMs } from './polling.js';
|
|
16
|
+
|
|
17
|
+
const KEY = 'OPENWOP_POLL_TIMEOUT_SCALE';
|
|
18
|
+
|
|
19
|
+
function withScale(value: string | undefined, fn: () => void): void {
|
|
20
|
+
if (value === undefined) delete process.env[KEY];
|
|
21
|
+
else process.env[KEY] = value;
|
|
22
|
+
fn();
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
afterEach(() => {
|
|
26
|
+
delete process.env[KEY];
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
describe('polling: scaledTimeoutMs', () => {
|
|
30
|
+
it('is an exact no-op when the knob is unset — no existing measurement moves', () => {
|
|
31
|
+
withScale(undefined, () => {
|
|
32
|
+
for (const ms of [100, 1_000, 5_000, 10_000, 15_000, 30_000, 60_000]) {
|
|
33
|
+
expect(scaledTimeoutMs(ms)).toBe(ms);
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it('is an exact no-op at scale 1, including the string form', () => {
|
|
39
|
+
withScale('1', () => expect(scaledTimeoutMs(10_000)).toBe(10_000));
|
|
40
|
+
withScale('1.0', () => expect(scaledTimeoutMs(10_000)).toBe(10_000));
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it('scales explicit bounds, which the documented env var could never reach', () => {
|
|
44
|
+
withScale('3', () => {
|
|
45
|
+
expect(scaledTimeoutMs(10_000)).toBe(30_000);
|
|
46
|
+
expect(scaledTimeoutMs(15_000)).toBe(45_000);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it('preserves the ORDERING of deliberately-short bounds — the reason this scales rather than floors', () => {
|
|
51
|
+
withScale('4', () => {
|
|
52
|
+
const short = scaledTimeoutMs(100);
|
|
53
|
+
const long = scaledTimeoutMs(10_000);
|
|
54
|
+
expect(short).toBe(400);
|
|
55
|
+
expect(long).toBe(40_000);
|
|
56
|
+
// A floor would have collapsed these two to the same value, silently
|
|
57
|
+
// rewriting every negative assertion that depends on a short bound.
|
|
58
|
+
expect(short).toBeLessThan(long);
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('rounds up, so a fractional scale never shortens a bound', () => {
|
|
63
|
+
withScale('1.5', () => expect(scaledTimeoutMs(1_001)).toBe(1_502));
|
|
64
|
+
withScale('1.0001', () => expect(scaledTimeoutMs(100)).toBeGreaterThanOrEqual(100));
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('falls back to 1 for a mis-set knob rather than producing an instant failure', () => {
|
|
68
|
+
// A zero, negative, or unparseable scale would turn every poll into an
|
|
69
|
+
// immediate timeout — which reads as a catastrophic host defect, the exact
|
|
70
|
+
// misattribution this knob exists to prevent.
|
|
71
|
+
for (const bad of ['0', '-2', 'abc', 'NaN', 'Infinity', '', ' ']) {
|
|
72
|
+
withScale(bad, () => expect(scaledTimeoutMs(10_000)).toBe(10_000));
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('reads the environment at CALL time, so a harness may set it after import', () => {
|
|
77
|
+
withScale('2', () => expect(scaledTimeoutMs(1_000)).toBe(2_000));
|
|
78
|
+
withScale('5', () => expect(scaledTimeoutMs(1_000)).toBe(5_000));
|
|
79
|
+
});
|
|
80
|
+
});
|