@openwop/openwop-conformance 1.136.10 → 1.138.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +267 -2
- package/coverage.md +1 -1
- package/dist/cli.js +8 -1
- package/dist/lib/requirement-ledger.js +8 -0
- package/dist/lib/scenario-disposition.js +30 -0
- package/package.json +1 -1
- package/schemas/CORPUS-STAMP.json +2 -2
- package/schemas/capabilities.schema.json +1 -1
- package/schemas/certification-bundle-v2.schema.json +11 -1
- package/src/cli.ts +8 -1
- package/src/lib/polling.test.ts +80 -0
- package/src/lib/polling.ts +39 -2
- package/src/lib/requirement-ledger.test.ts +75 -0
- package/src/lib/requirement-ledger.ts +9 -0
- package/src/lib/risk-disposition.test.ts +91 -0
- package/src/lib/scenario-disposition.ts +30 -0
- package/src/lib/webhook-receiver.test.ts +76 -0
- package/src/lib/webhook-receiver.ts +24 -3
- package/src/scenarios/certification-floor-enforcement.test.ts +65 -0
- package/src/scenarios/conformance-execution-witness.test.ts +29 -0
- package/src/scenarios/cross-host-traceparent-propagation.test.ts +10 -6
- package/src/scenarios/durability-poison-exhaustion.test.ts +154 -0
- package/src/scenarios/replay-fanout-suppression.test.ts +295 -0
- package/src/scenarios/webhook-receiver-adversarial.test.ts +9 -3
- package/src/scenarios/webhook-signed-delivery.test.ts +23 -6
- package/src/setup.ts +24 -5
|
@@ -17,6 +17,13 @@
|
|
|
17
17
|
|
|
18
18
|
import { createHmac, timingSafeEqual } from 'node:crypto';
|
|
19
19
|
|
|
20
|
+
/**
|
|
21
|
+
* The `X-openwop-Signature` value prefix, per `webhooks.md` §"Delivery headers"
|
|
22
|
+
* (`sha256={hex}`). Distinct from the ALGORITHM header's `v1`, which names the
|
|
23
|
+
* signing scheme, not the encoding — see `verifyWebhookDelivery`.
|
|
24
|
+
*/
|
|
25
|
+
export const SIGNATURE_PREFIX = 'sha256=';
|
|
26
|
+
|
|
20
27
|
export const DEFAULT_FRESHNESS_WINDOW_SECONDS = 300;
|
|
21
28
|
|
|
22
29
|
export type WebhookRejectionReason =
|
|
@@ -75,10 +82,24 @@ export function verifyWebhookDelivery(
|
|
|
75
82
|
}
|
|
76
83
|
|
|
77
84
|
// 2. Signature header parse.
|
|
78
|
-
|
|
85
|
+
//
|
|
86
|
+
// `sha256=`, NOT `v1=` (corrected 2026-08-19). `webhooks.md` §"Delivery
|
|
87
|
+
// headers" specifies `X-openwop-Signature: sha256={hex}` and its verification
|
|
88
|
+
// recipe says "Strip the `sha256=` prefix". This verifier required `v1=` and
|
|
89
|
+
// rejected the spec's own header as malformed — so the reference verifier a
|
|
90
|
+
// subscriber implementer would copy refused every conforming delivery.
|
|
91
|
+
//
|
|
92
|
+
// The confusion is visible one comment above: `v1` is the value of the
|
|
93
|
+
// ALGORITHM header (`X-openwop-Signature-Algorithm: v1`), a different field.
|
|
94
|
+
// One value, two fields, conflated. It survived because
|
|
95
|
+
// `webhook-receiver-adversarial.test.ts` signs with `signPayload` and verifies
|
|
96
|
+
// with this function — a closed loop that is self-consistent and wrong, and
|
|
97
|
+
// therefore green on every host. Reported by a tier-2 host that could not
|
|
98
|
+
// adjudicate which of the suite's three signature shapes was canonical.
|
|
99
|
+
if (!signatureHeader.startsWith(SIGNATURE_PREFIX)) {
|
|
79
100
|
return { accepted: false, reason: 'malformed_signature_header' };
|
|
80
101
|
}
|
|
81
|
-
const providedHex = signatureHeader.slice(
|
|
102
|
+
const providedHex = signatureHeader.slice(SIGNATURE_PREFIX.length);
|
|
82
103
|
if (!/^[0-9a-f]+$/i.test(providedHex)) {
|
|
83
104
|
return { accepted: false, reason: 'malformed_signature_header' };
|
|
84
105
|
}
|
|
@@ -130,7 +151,7 @@ export function signPayload(
|
|
|
130
151
|
const bodyStr = typeof rawBody === 'string' ? rawBody : rawBody.toString('utf8');
|
|
131
152
|
const hex = createHmac('sha256', secret).update(`${timestamp}.${bodyStr}`, 'utf8').digest('hex');
|
|
132
153
|
return {
|
|
133
|
-
signatureHeader:
|
|
154
|
+
signatureHeader: `${SIGNATURE_PREFIX}${hex}`,
|
|
134
155
|
timestampHeader: String(timestamp),
|
|
135
156
|
algorithmHeader: 'v1',
|
|
136
157
|
};
|
|
@@ -20,7 +20,10 @@
|
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
22
|
import { describe, it, expect } from 'vitest';
|
|
23
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
24
|
+
import { join } from 'node:path';
|
|
23
25
|
import { verifyBundle, verifyBundleProfile, PROFILE_FLOOR_SCENARIOS } from '../lib/profiles.js';
|
|
26
|
+
import { SCENARIOS_DIR } from '../lib/paths.js';
|
|
24
27
|
|
|
25
28
|
/** Derives `openwop-core` and `openwop-stream-sse` (transports omitted ⇒ all). */
|
|
26
29
|
const streamingDiscovery = {
|
|
@@ -135,5 +138,67 @@ describe('RFC 0148 §C — floor enforcement is not vacuous', () => {
|
|
|
135
138
|
for (const { profile, scenario } of all) {
|
|
136
139
|
expect(scenario, `${profile} floor cites a non-scenario filename`).toMatch(/\.test\.ts$/);
|
|
137
140
|
}
|
|
141
|
+
|
|
142
|
+
// 2026-08-18: this leg's NAME promised "files that exist" and it only
|
|
143
|
+
// matched the `.test.ts` suffix — a string check wearing an existence
|
|
144
|
+
// check's name. The phantom `audit-log-verification.test.ts` floor row sat
|
|
145
|
+
// in `openwop-core-standard` until `--certify` hit it against a live host,
|
|
146
|
+
// because nothing here opened the directory.
|
|
147
|
+
if (SCENARIOS_DIR === null) return; // published layout ships src/, but stay honest if it ever does not
|
|
148
|
+
const dir = SCENARIOS_DIR as string;
|
|
149
|
+
const missing = all
|
|
150
|
+
.filter(({ scenario }) => !existsSync(join(dir, scenario)))
|
|
151
|
+
.map(({ profile, scenario }) => `${profile} → ${scenario}`);
|
|
152
|
+
expect(
|
|
153
|
+
missing,
|
|
154
|
+
'a floor cites a scenario file that does not exist — that requirement can never be satisfied, ' +
|
|
155
|
+
'so the profile can never certify, for a reason unrelated to any host',
|
|
156
|
+
).toEqual([]);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('no floor scenario is corpus-only — a floor must be provable from the published package', () => {
|
|
160
|
+
// openwop-app's suggestion, and it is right that this be a red test rather
|
|
161
|
+
// than a paragraph: the failure mode is SOMEONE LATER adding a corpus
|
|
162
|
+
// self-check to a floor, and prose in a PR body will not be in front of
|
|
163
|
+
// them.
|
|
164
|
+
//
|
|
165
|
+
// A scenario that can only run in a repo checkout (it reads `spec/` or
|
|
166
|
+
// `RFCS/` through `V1_DIR`) records `blocked` in the published layout,
|
|
167
|
+
// where the package ships no corpus by design. `blocked` is correct there
|
|
168
|
+
// — RFC 0148 §A defines it as a missing dependency — but a `blocked`
|
|
169
|
+
// requirement in a claimed profile invalidates that profile. So a
|
|
170
|
+
// corpus-only scenario in a floor makes the profile UNCERTIFIABLE from the
|
|
171
|
+
// npm tarball, permanently, for a reason no host can fix. It is the mirror
|
|
172
|
+
// of the phantom-row trap above: that one named a file that does not
|
|
173
|
+
// exist, this one names a file that cannot execute where certification is
|
|
174
|
+
// measured.
|
|
175
|
+
if (SCENARIOS_DIR === null) return;
|
|
176
|
+
const dir = SCENARIOS_DIR as string;
|
|
177
|
+
|
|
178
|
+
/** Every `it`/`test` in the file is V1_DIR-guarded, or every `describe` is. */
|
|
179
|
+
const isCorpusOnly = (source: string): boolean => {
|
|
180
|
+
if (!source.includes('V1_DIR')) return false;
|
|
181
|
+
const guard = /\(V1_DIR === null\)/;
|
|
182
|
+
const its = [...source.matchAll(/\b(?:it|test)(\.skipIf\([^)]*\))?\s*\(/g)];
|
|
183
|
+
const describes = [...source.matchAll(/\bdescribe(\.skipIf\([^)]*\))?\s*\(/g)];
|
|
184
|
+
const allItsGuarded = its.length > 0 && its.every((m) => guard.test(m[1] ?? ''));
|
|
185
|
+
const allDescribesGuarded = describes.length > 0 && describes.every((m) => guard.test(m[1] ?? ''));
|
|
186
|
+
return allItsGuarded || allDescribesGuarded;
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
const offenders = Object.entries(PROFILE_FLOOR_SCENARIOS)
|
|
190
|
+
.flatMap(([profile, floor]) => floor.required.map((scenario) => ({ profile, scenario })))
|
|
191
|
+
.filter(({ scenario }) => {
|
|
192
|
+
const path = join(dir, scenario);
|
|
193
|
+
return existsSync(path) && isCorpusOnly(readFileSync(path, 'utf8'));
|
|
194
|
+
})
|
|
195
|
+
.map(({ profile, scenario }) => `${profile} → ${scenario}`);
|
|
196
|
+
|
|
197
|
+
expect(
|
|
198
|
+
offenders,
|
|
199
|
+
'a profile floor cites a CORPUS-ONLY scenario (all of its tests are guarded on `V1_DIR === null`). ' +
|
|
200
|
+
'It records `blocked` in the published layout, so that profile can never certify from the npm ' +
|
|
201
|
+
'tarball — no host can fix it. Keep corpus self-checks out of floors.',
|
|
202
|
+
).toEqual([]);
|
|
138
203
|
});
|
|
139
204
|
});
|
|
@@ -51,6 +51,35 @@ describe('RFC 0148 §A — conformance-execution-witness: the runner record', ()
|
|
|
51
51
|
expect(resolveFileRecord(['pass'], undefined, 0, { kind: 'skipped', reason: 'operator opt-out' })).toEqual({ disposition: 'skipped', detail: 'operator opt-out' });
|
|
52
52
|
});
|
|
53
53
|
|
|
54
|
+
it('gap G8 — a file that asserted then soft-skipped keeps the note as a partial-witness detail, and its disposition does NOT move', () => {
|
|
55
|
+
// `return softSkip(...)` produces a PASS state, not a skip, so neither the
|
|
56
|
+
// zero-assertion branch nor the all-skipped branch fires. The note used to be
|
|
57
|
+
// discarded outright and the row read a bare `executed-pass` — for a
|
|
58
|
+
// requirement the run may never have reached, because the assertions came
|
|
59
|
+
// from a positive control or a setup precondition. Measured on this exact
|
|
60
|
+
// function by a tier-1 host writing a scenario whose first leg is a control.
|
|
61
|
+
const one = resolveFileRecord(['pass'], undefined, 1, { kind: 'blocked', reason: 'fork returned 501' });
|
|
62
|
+
expect(one.disposition).toBe('executed-pass');
|
|
63
|
+
expect(one.detail).toBe('partial-witness: blocked: fork returned 501');
|
|
64
|
+
|
|
65
|
+
const many = resolveFileRecord(['pass', 'pass'], undefined, 4, { kind: 'inapplicable', reason: 'fork unsupported' });
|
|
66
|
+
expect(many.disposition).toBe('executed-pass');
|
|
67
|
+
expect(many.detail).toBe('partial-witness: inapplicable: fork unsupported');
|
|
68
|
+
|
|
69
|
+
// The disposition deliberately does not move: honouring a per-FILE note here
|
|
70
|
+
// would downgrade a file that legitimately finished its requirement and also
|
|
71
|
+
// soft-skipped an optional extra leg. The marker makes the rows findable so
|
|
72
|
+
// the per-`it` fix can follow measurement instead of preceding it.
|
|
73
|
+
expect(many.disposition).not.toBe('inapplicable');
|
|
74
|
+
|
|
75
|
+
// A clean pass is untouched — no note, no marker, no detail at all.
|
|
76
|
+
expect(resolveFileRecord(['pass', 'pass'], undefined, 7, null)).toEqual({ disposition: 'executed-pass' });
|
|
77
|
+
|
|
78
|
+
// A FAILING file keeps its failure; the note must not dress a red as a pass.
|
|
79
|
+
const red = resolveFileRecord(['pass', 'fail'], undefined, 3, { kind: 'blocked', reason: 'seam 404' });
|
|
80
|
+
expect(red.disposition).toBe('executed-fail');
|
|
81
|
+
});
|
|
82
|
+
|
|
54
83
|
it('every test ctx.skip()ped takes the noted reason when one was written before the skip, else stays the blocked marker', () => {
|
|
55
84
|
// `ctx.skip()` throws — a `softSkip(...)` AFTER it is dead code. Seven files
|
|
56
85
|
// carried exactly that dead note and reported as unclassified for a suite minor.
|
|
@@ -51,17 +51,21 @@ describe('cross-host-traceparent-propagation: behavioral (RFC 0040 §B)', () =>
|
|
|
51
51
|
// Until the peer harness lands, the assertion is surfaced as `it.skip` so
|
|
52
52
|
// test reporters track the gap rather than reporting a vacuous PASS.
|
|
53
53
|
// Marked out of stable profile via RFC 0042 §B (experimental tier):
|
|
54
|
-
// RFC 0040
|
|
55
|
-
//
|
|
54
|
+
// RFC 0040 is Accepted, but the cross-host behavioral scenario stays
|
|
55
|
+
// experimental until a non-steward host produces the behavioral
|
|
56
|
+
// traceparent evidence — RFC status (Accepted) and conformance-profile
|
|
57
|
+
// tier (experimental) are separate axes per RFC 0042 §B. Hosts that wire
|
|
58
|
+
// Phase 3 cross-host causation SHOULD advertise
|
|
56
59
|
// `multiAgent.executionModel.tier: 'experimental'` per RFC 0042 §A
|
|
57
|
-
// until
|
|
60
|
+
// until that behavioral evidence lands. Path-to-runnable
|
|
58
61
|
// requires the MCP peer harness (OPENWOP_MCP_REAL_SERVER_URL) +
|
|
59
62
|
// inbound-header recorder; flips to a real `it()` on first non-steward
|
|
60
63
|
// Phase 3 host advertising matching capabilities.
|
|
61
64
|
it.skip('Phase 3 host MUST inject parent run\'s traceparent into outbound MCP requests — out of stable profile via RFC 0042');
|
|
62
65
|
|
|
63
|
-
// Same routing — out of stable profile via RFC 0042 §B until
|
|
64
|
-
//
|
|
65
|
-
// designed alongside the
|
|
66
|
+
// Same routing — out of stable profile via RFC 0042 §B until behavioral
|
|
67
|
+
// A2A cross-host evidence lands (RFC 0040 itself is already Accepted); the
|
|
68
|
+
// A2A test seam contract is still to be designed alongside the
|
|
69
|
+
// corresponding peer harness.
|
|
66
70
|
it.skip('Phase 3 host MUST inject parent run\'s traceparent into outbound A2A messages — out of stable profile via RFC 0042');
|
|
67
71
|
});
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Poison work reaches a bounded, operator-visible terminal state
|
|
3
|
+
* (RFC 0158 §C.8, conformance row `durability/poison-exhaustion`).
|
|
4
|
+
*
|
|
5
|
+
* THE REQUIREMENT: work that fails deterministically **MUST** reach a terminal,
|
|
6
|
+
* operator-visible state within a bounded number of attempts, and **MUST NOT**
|
|
7
|
+
* be redelivered indefinitely.
|
|
8
|
+
*
|
|
9
|
+
* ── Why the obvious assertion proves only half of it ─────────────────────────
|
|
10
|
+
* "The run reached `failed`" is the half everyone writes, and `failure-path.
|
|
11
|
+
* test.ts` already covers it. It says nothing about the second clause: a host
|
|
12
|
+
* that redelivers forever ALSO reports a terminal status at some point, or
|
|
13
|
+
* reports one while the work keeps being re-dispatched behind it. An assertion
|
|
14
|
+
* that stops at the terminal status cannot distinguish "terminated after N
|
|
15
|
+
* attempts" from "still going".
|
|
16
|
+
*
|
|
17
|
+
* So the load-bearing leg here is STABILITY AFTER TERMINAL: count the attempts
|
|
18
|
+
* on the run's own log, wait a scaled quiet window, count again, and assert the
|
|
19
|
+
* number did not move. A host still redelivering shows new attempts; a host that
|
|
20
|
+
* stopped shows the same count. That is the falsifiable form of "MUST NOT be
|
|
21
|
+
* redelivered indefinitely" — and, unlike a ceiling, it needs no number the wire
|
|
22
|
+
* does not carry (see below).
|
|
23
|
+
*
|
|
24
|
+
* ── What this scenario deliberately does NOT assert ──────────────────────────
|
|
25
|
+
* Conformance to a SPECIFIC declared attempt bound. RFC 0158 §E mints no
|
|
26
|
+
* advertised capability field, so a host's declared bound is not on the wire and
|
|
27
|
+
* the suite cannot read it. Asserting against a number the suite invented would
|
|
28
|
+
* be a bound of the suite's own making, which is the inverse of the §B.5
|
|
29
|
+
* discipline ("derived from the mechanism that enforces it"). When
|
|
30
|
+
* `OPENWOP_DECLARED_ATTEMPT_BOUND` is supplied by the operator — the same
|
|
31
|
+
* operator-supplied-input shape as `OPENWOP_OPTED_OUT_PROFILES` — the ceiling leg
|
|
32
|
+
* runs too. Without it, boundedness is still asserted; only the specific bound is
|
|
33
|
+
* not.
|
|
34
|
+
*
|
|
35
|
+
* ── Honest about its gating ──────────────────────────────────────────────────
|
|
36
|
+
* This is NOT a pure black-box scenario. Attempt counts live on the run's event
|
|
37
|
+
* log, which is read through the EXISTING `/v1/host/sample/test/runs/{runId}/
|
|
38
|
+
* events` seam — no new seam, but a seam. A host that has not wired it records
|
|
39
|
+
* `blocked`: unobservable, not unmet. Outside every profile floor, for the same
|
|
40
|
+
* reason.
|
|
41
|
+
*
|
|
42
|
+
* @see RFCS/0158-durable-execution-and-disaster-recovery-qualification.md §C.8
|
|
43
|
+
* @see spec/v1/host-sample-test-seams.md
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
import { describe, expect, it } from 'vitest';
|
|
47
|
+
import { driver } from '../lib/driver.js';
|
|
48
|
+
import { pollUntilTerminal, scaledTimeoutMs } from '../lib/polling.js';
|
|
49
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
50
|
+
import { softSkip } from '../lib/soft-skip.js';
|
|
51
|
+
import { queryTestEvents, requireEvents } from '../lib/event-log-query.js';
|
|
52
|
+
import { recordRequirement } from '../lib/requirement-ledger.js';
|
|
53
|
+
import { requirementIdForFile } from '../lib/scenario-disposition.js';
|
|
54
|
+
|
|
55
|
+
const WORKFLOW_ID = 'conformance-failure';
|
|
56
|
+
const FILE = 'durability-poison-exhaustion.test.ts';
|
|
57
|
+
const REQ = requirementIdForFile(FILE);
|
|
58
|
+
|
|
59
|
+
/** Attempt-bearing event types. A `node.started` is attempt 1; each
|
|
60
|
+
* `node.retried` is one more. Counting BOTH means a host that re-dispatches
|
|
61
|
+
* without emitting `node.retried` is still caught. */
|
|
62
|
+
const ATTEMPT_TYPES = new Set(['node.started', 'node.retried']);
|
|
63
|
+
|
|
64
|
+
/** How long to watch for further attempts after the run reports terminal.
|
|
65
|
+
* Scaled: a longer wait is a STRONGER claim here, because it is a wait for
|
|
66
|
+
* something that must not happen. */
|
|
67
|
+
const QUIET_WINDOW_MS = 4_000;
|
|
68
|
+
|
|
69
|
+
function declaredAttemptBound(): number | null {
|
|
70
|
+
const raw = process.env['OPENWOP_DECLARED_ATTEMPT_BOUND'];
|
|
71
|
+
if (raw === undefined || raw.trim() === '') return null;
|
|
72
|
+
const n = Number(raw);
|
|
73
|
+
return Number.isFinite(n) && n > 0 ? Math.floor(n) : null;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
|
|
77
|
+
|
|
78
|
+
// Recorded at MODULE scope, not inside the `it`. `describe.skipIf` never runs the
|
|
79
|
+
// callback, so a `recordRequirement` in the test body is dead code on exactly the
|
|
80
|
+
// path it exists to classify — the same shape as the note written after
|
|
81
|
+
// `ctx.skip()` threw, which left seven files carrying notes the ledger never saw.
|
|
82
|
+
// Caught by reading a CI log: this file and `failure-path.test.ts` both skipped
|
|
83
|
+
// against the postgres host, which does not advertise the fixture, and the row
|
|
84
|
+
// would have been recorded `blocked` with the unclassified-return marker instead
|
|
85
|
+
// of `inapplicable` with a reason.
|
|
86
|
+
if (SKIP_NO_FIXTURE) {
|
|
87
|
+
recordRequirement(REQ, 'inapplicable', `fixture ${WORKFLOW_ID} not advertised`);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
describe.skipIf(SKIP_NO_FIXTURE)('RFC 0158 §C.8 — poison work terminates within a bounded number of attempts', () => {
|
|
91
|
+
it('reaches terminal AND stops being retried, asserted on the log rather than on the status alone', async () => {
|
|
92
|
+
const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
|
|
93
|
+
expect(create.status, driver.describe(
|
|
94
|
+
'rest-endpoints.md',
|
|
95
|
+
'POST /v1/runs MUST return 201 even for work that fails at runtime',
|
|
96
|
+
)).toBe(201);
|
|
97
|
+
const runId = (create.json as { runId: string }).runId;
|
|
98
|
+
|
|
99
|
+
// ── First clause: a terminal, OPERATOR-VISIBLE state ─────────────────────
|
|
100
|
+
const terminal = await pollUntilTerminal(runId);
|
|
101
|
+
expect(terminal.status, driver.describe(
|
|
102
|
+
'RFC 0158 §C.8',
|
|
103
|
+
'deterministically failing work MUST reach a terminal, operator-visible state',
|
|
104
|
+
)).toBe('failed');
|
|
105
|
+
|
|
106
|
+
// ── The seam gate comes AFTER the control above, so a host that wired the
|
|
107
|
+
// fixture but not the log still proves the fixture ran. ───────────────
|
|
108
|
+
const first = await queryTestEvents(runId);
|
|
109
|
+
if (!first.ok) {
|
|
110
|
+
const why = first.reason === 'seam_unavailable'
|
|
111
|
+
? 'event-log seam /v1/host/sample/test/runs/{runId}/events not wired — attempts are unobservable'
|
|
112
|
+
: `event-log seam returned HTTP ${first.status}`;
|
|
113
|
+
recordRequirement(REQ, 'blocked', why);
|
|
114
|
+
return softSkip('blocked', why);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const events = requireEvents(first, 'RFC 0158 §C.8 attempt counting');
|
|
118
|
+
// Non-vacuity: the failure must actually be ON the log. Without this, a host
|
|
119
|
+
// returning an empty array would sail through every count comparison below,
|
|
120
|
+
// since 0 === 0 after any wait.
|
|
121
|
+
expect(events.some((e) => e.type === 'node.failed'), driver.describe(
|
|
122
|
+
'RFC 0158 §C.8',
|
|
123
|
+
'the run log MUST record the deterministic failure — an empty log makes every attempt count vacuous',
|
|
124
|
+
)).toBe(true);
|
|
125
|
+
|
|
126
|
+
const attemptsBefore = events.filter((e) => ATTEMPT_TYPES.has(e.type)).length;
|
|
127
|
+
expect(attemptsBefore, driver.describe(
|
|
128
|
+
'RFC 0158 §C.8',
|
|
129
|
+
'at least one attempt MUST be recorded — zero attempts means nothing was delivered',
|
|
130
|
+
)).toBeGreaterThan(0);
|
|
131
|
+
|
|
132
|
+
// ── Second clause, the load-bearing one: NOT redelivered indefinitely ────
|
|
133
|
+
await new Promise((r) => setTimeout(r, scaledTimeoutMs(QUIET_WINDOW_MS)));
|
|
134
|
+
const second = await queryTestEvents(runId);
|
|
135
|
+
const after = requireEvents(second, 'RFC 0158 §C.8 attempt counting (post-terminal)');
|
|
136
|
+
const attemptsAfter = after.filter((e) => ATTEMPT_TYPES.has(e.type)).length;
|
|
137
|
+
|
|
138
|
+
expect(attemptsAfter, driver.describe(
|
|
139
|
+
'RFC 0158 §C.8',
|
|
140
|
+
'attempts MUST NOT continue after the run reports terminal — a host still redelivering records more',
|
|
141
|
+
)).toBe(attemptsBefore);
|
|
142
|
+
|
|
143
|
+
// ── Optional ceiling, only when the operator supplies the declared bound ─
|
|
144
|
+
const bound = declaredAttemptBound();
|
|
145
|
+
if (bound !== null) {
|
|
146
|
+
expect(attemptsAfter, driver.describe(
|
|
147
|
+
'RFC 0158 §C.8 + §B.5',
|
|
148
|
+
'total attempts MUST NOT exceed the operator-declared attempt bound',
|
|
149
|
+
)).toBeLessThanOrEqual(bound);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
recordRequirement(REQ, 'executed-pass', undefined, { assertionCount: bound === null ? 5 : 6 });
|
|
153
|
+
});
|
|
154
|
+
});
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host-initiated fan-out is an external effect (`replay.md` §"Host-initiated
|
|
3
|
+
* fan-out is an external effect").
|
|
4
|
+
*
|
|
5
|
+
* THE MUST NOT: a host that projects its event log outward — webhook delivery,
|
|
6
|
+
* outbound streams, analytics or audit sinks — MUST NOT emit those outbound
|
|
7
|
+
* deliveries for events a `mode: "replay"` fork re-emits as fixed history.
|
|
8
|
+
*
|
|
9
|
+
* WHY THIS SCENARIO EXISTS. Until now this was the largest normative MUST NOT on
|
|
10
|
+
* the replay surface with **no conformance scenario and no SECURITY invariant**.
|
|
11
|
+
* Two hosts had it deployed and the interop matrix still had to record it as
|
|
12
|
+
* *not suite-witnessed*, because both rows rested on the hosts' own tests. A
|
|
13
|
+
* host's own test is evidence about that host; it is not a witness of the wire.
|
|
14
|
+
*
|
|
15
|
+
* WHY IT IS OBSERVABLE AT ALL, unlike some other unmeasured invariants: the
|
|
16
|
+
* outbound side has a real probe surface. The suite can BE the subscriber — boot
|
|
17
|
+
* a local receiver, register it via `POST /v1/webhooks`, and read what actually
|
|
18
|
+
* arrives. The property is stated over exactly what a subscriber sees, so this
|
|
19
|
+
* asserts the thing itself rather than an adjacent proxy.
|
|
20
|
+
*
|
|
21
|
+
* ── The positive control is not decoration here; it is the whole test ────────
|
|
22
|
+
* "No delivery arrived" passes identically when delivery never worked at all: a
|
|
23
|
+
* mistyped URL, a subscription that was never created, a host that rejected the
|
|
24
|
+
* receiver, a fixture that emitted nothing. That is a vacuous witness in its
|
|
25
|
+
* purest form. So all three legs run in ONE test against ONE receiver and ONE
|
|
26
|
+
* subscription, and the absence is only ever asserted AFTER presence has been
|
|
27
|
+
* proven on that exact wiring. Do not split this into separate `it` blocks —
|
|
28
|
+
* separate blocks re-register, and a re-registration that silently fails turns
|
|
29
|
+
* the negative leg back into a vacuous pass.
|
|
30
|
+
*
|
|
31
|
+
* ── Why the timeout knob makes this leg STRICTER, not looser ─────────────────
|
|
32
|
+
* Absence has nothing to poll for, so the negative is "nothing arrived within
|
|
33
|
+
* N". `OPENWOP_POLL_TIMEOUT_SCALE` multiplies N. The reflex on seeing a timeout
|
|
34
|
+
* multiplier is that it weakens assertions — here it is the reverse: waiting
|
|
35
|
+
* longer for a delivery that must never come is a STRONGER claim, and an
|
|
36
|
+
* operator raising the scale on a slow host makes this scenario harder to pass,
|
|
37
|
+
* not easier.
|
|
38
|
+
*
|
|
39
|
+
* ── Deliberately NOT in the `openwop-replay-fork` floor ──────────────────────
|
|
40
|
+
* This scenario needs the host to accept a loopback receiver. A host with an
|
|
41
|
+
* SSRF guard on `POST /v1/webhooks` correctly refuses one, and the suite's
|
|
42
|
+
* standing operator contract asks such hosts for an opt-in
|
|
43
|
+
* (`OPENWOP_WEBHOOK_ALLOW_PRIVATE=true` on the SQLite reference). At least one
|
|
44
|
+
* certifying host rejects it today. Putting a receiver-gated row in that
|
|
45
|
+
* profile's floor would make the profile un-certifiable for a host whose
|
|
46
|
+
* SECURITY CONTROL IS CORRECT, and would read in the matrix as a regression that
|
|
47
|
+
* is not one. Capability-gated, outside the floor. Moving it into a floor is a
|
|
48
|
+
* separate decision needing an RFC 0148 §C argument.
|
|
49
|
+
*
|
|
50
|
+
* ── One interpretive call, stated rather than buried ─────────────────────────
|
|
51
|
+
* The subscription below is for `run.completed`. That is a LIFECYCLE event, and
|
|
52
|
+
* `replay.md` notes lifecycle events are "ambiguous noise" next to a
|
|
53
|
+
* recorded-fact event like `memory.written`, which is "a false statement". The
|
|
54
|
+
* reading applied here is that a replay fork reproduces the source's log, so the
|
|
55
|
+
* `run.completed` in the fork's log is RE-EMITTED HISTORY and therefore squarely
|
|
56
|
+
* inside "any event re-emitted as fixed history is in scope". A host that
|
|
57
|
+
* suppressed only recorded-fact events while delivering re-emitted lifecycle
|
|
58
|
+
* events would fail this scenario.
|
|
59
|
+
*
|
|
60
|
+
* That reading is no longer an inference: `replay.md` was clarified on
|
|
61
|
+
* 2026-08-19 to say the contrast RANKS THE HARM and does not narrow the scope,
|
|
62
|
+
* because the requirement above it already decides the question — replay-ness is
|
|
63
|
+
* read from the run, never from the event type, so suppressing by event type is
|
|
64
|
+
* selecting by event type. The paragraph stays here because a host that reads
|
|
65
|
+
* the older text should meet a visible claim rather than a surprise red.
|
|
66
|
+
*
|
|
67
|
+
* @see spec/v1/replay.md §"Host-initiated fan-out is an external effect"
|
|
68
|
+
* @see spec/v1/webhooks.md §"Register"
|
|
69
|
+
*/
|
|
70
|
+
|
|
71
|
+
import { afterEach, describe, expect, it } from 'vitest';
|
|
72
|
+
import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
|
|
73
|
+
import { softSkip } from '../lib/soft-skip.js';
|
|
74
|
+
import { driver } from '../lib/driver.js';
|
|
75
|
+
import { discoveryFamilies } from '../lib/discovery-capabilities.js';
|
|
76
|
+
import { pollUntilTerminal, scaledTimeoutMs } from '../lib/polling.js';
|
|
77
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
78
|
+
import { discoverOwnedTenant } from '../lib/webhook-receiver.js';
|
|
79
|
+
import { recordRequirement } from '../lib/requirement-ledger.js';
|
|
80
|
+
import { requirementIdForFile } from '../lib/scenario-disposition.js';
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* RFC 0148 §A — the MUST NOT gets its OWN disposition, separate from the file's.
|
|
84
|
+
*
|
|
85
|
+
* Without this the file would be recorded `executed-pass` off leg 1's assertions
|
|
86
|
+
* even on a run where the fork returned 501 and the MUST NOT was never
|
|
87
|
+
* exercised. That is a partial witness reading as a full one: the positive
|
|
88
|
+
* control genuinely passed, so the file is not lying, but the matrix cell for
|
|
89
|
+
* this requirement would claim coverage the run did not produce. Recording the
|
|
90
|
+
* requirement explicitly at every exit keeps the cell honest whichever path is
|
|
91
|
+
* taken. `setup.ts` records the file's aggregate under the SAME id and catches
|
|
92
|
+
* the resulting conflict — "a scenario that recorded its own file id first
|
|
93
|
+
* wins" — so this is the sanctioned override, not a second row.
|
|
94
|
+
*
|
|
95
|
+
* It MUST be `requirementIdForFile`, not `requirementIdForScenario`. Those two
|
|
96
|
+
* agree only for files in a certification FLOOR; for a non-floor scenario like
|
|
97
|
+
* this one they differ (`openwop.scenario.<name>` vs the registry id). Recording
|
|
98
|
+
* the registry id here would mint an ORPHAN requirement nothing reads, while the
|
|
99
|
+
* row the bundle actually carries kept the file-level `executed-pass` — a fix
|
|
100
|
+
* that changes nothing and looks like it did.
|
|
101
|
+
*/
|
|
102
|
+
const REQUIREMENT_ID = requirementIdForFile('replay-fanout-suppression.test.ts');
|
|
103
|
+
|
|
104
|
+
interface Delivered {
|
|
105
|
+
readonly body: string;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async function startReceiver(): Promise<{ server: Server; url: string; received: Delivered[] }> {
|
|
109
|
+
const received: Delivered[] = [];
|
|
110
|
+
const server = createServer((req: IncomingMessage, res: ServerResponse) => {
|
|
111
|
+
const chunks: Buffer[] = [];
|
|
112
|
+
req.on('data', (c: Buffer) => chunks.push(c));
|
|
113
|
+
req.on('end', () => {
|
|
114
|
+
received.push({ body: Buffer.concat(chunks).toString('utf8') });
|
|
115
|
+
res.writeHead(204);
|
|
116
|
+
res.end();
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', () => resolve()));
|
|
120
|
+
const addr = server.address();
|
|
121
|
+
if (typeof addr !== 'object' || addr === null) throw new Error('receiver address unavailable');
|
|
122
|
+
return { server, url: `http://127.0.0.1:${addr.port}/`, received };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
let activeServer: Server | null = null;
|
|
126
|
+
afterEach(async () => {
|
|
127
|
+
if (activeServer) {
|
|
128
|
+
await new Promise<void>((resolve) => activeServer!.close(() => resolve()));
|
|
129
|
+
activeServer = null;
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
/** Deliveries whose body mentions this run id — the fan-out attributable to it. */
|
|
134
|
+
function forRun(received: readonly Delivered[], runId: string): Delivered[] {
|
|
135
|
+
return received.filter((d) => d.body.includes(runId));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Wait `ms`, then report what arrived. Used for the NEGATIVE leg, where there is
|
|
140
|
+
* no event to poll for. Scaled, per the note in the file header.
|
|
141
|
+
*/
|
|
142
|
+
async function quietWindow(ms: number): Promise<void> {
|
|
143
|
+
await new Promise((resolve) => setTimeout(resolve, scaledTimeoutMs(ms)));
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** Grace for fire-and-forget delivery on the POSITIVE legs. */
|
|
147
|
+
const DELIVERY_GRACE_MS = 1_500;
|
|
148
|
+
/**
|
|
149
|
+
* How long the replay fork is watched for a delivery that must never arrive.
|
|
150
|
+
* Deliberately several times the positive grace: a host that merely DELAYS
|
|
151
|
+
* fan-out rather than suppressing it must not pass by being slow.
|
|
152
|
+
*/
|
|
153
|
+
const SUPPRESSION_WINDOW_MS = 6_000;
|
|
154
|
+
|
|
155
|
+
describe('replay-fanout-suppression: a replay fork MUST NOT fan out re-emitted events', () => {
|
|
156
|
+
it('delivers for a live run, suppresses for a replay fork, and delivers again for a branch fork', async (ctx) => {
|
|
157
|
+
const disco = await driver.get('/.well-known/openwop');
|
|
158
|
+
const caps = discoveryFamilies(disco.json) as {
|
|
159
|
+
webhooks?: { supported?: boolean };
|
|
160
|
+
replay?: { fork?: boolean; supported?: boolean };
|
|
161
|
+
};
|
|
162
|
+
if (caps.webhooks?.supported !== true) {
|
|
163
|
+
recordRequirement(REQUIREMENT_ID, 'inapplicable', 'host does not advertise webhooks.supported — outbound fan-out has no probe surface on this host');
|
|
164
|
+
return softSkip('inapplicable', '[replay-fanout-suppression] host does not advertise webhook support');
|
|
165
|
+
}
|
|
166
|
+
if (!isFixtureAdvertised('conformance-noop')) {
|
|
167
|
+
recordRequirement(REQUIREMENT_ID, 'inapplicable', 'conformance-noop fixture not advertised — no run to fan out');
|
|
168
|
+
return softSkip('inapplicable', '[replay-fanout-suppression] conformance-noop not advertised');
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const receiver = await startReceiver();
|
|
172
|
+
activeServer = receiver.server;
|
|
173
|
+
|
|
174
|
+
const ownedTenant = await discoverOwnedTenant(driver);
|
|
175
|
+
const reg = await driver.post('/v1/webhooks', {
|
|
176
|
+
url: receiver.url,
|
|
177
|
+
events: ['run.completed'],
|
|
178
|
+
...(ownedTenant ? { tenantId: ownedTenant } : {}),
|
|
179
|
+
});
|
|
180
|
+
if (reg.status === 400 && (reg.json as { error?: string }).error === 'webhook_url_rejected') {
|
|
181
|
+
// The host's SSRF guard refused a loopback destination. That is CORRECT
|
|
182
|
+
// host behaviour, not a failure — see the floor note in the file header.
|
|
183
|
+
recordRequirement(
|
|
184
|
+
REQUIREMENT_ID,
|
|
185
|
+
'blocked',
|
|
186
|
+
"precondition not met — body.error === 'webhook_url_rejected'; the host's SSRF guard refused the loopback "
|
|
187
|
+
+ 'receiver, which is correct host behaviour, so this requirement is unobservable rather than unmet',
|
|
188
|
+
);
|
|
189
|
+
return softSkip(
|
|
190
|
+
'blocked',
|
|
191
|
+
'[replay-fanout-suppression] host SSRF guard rejected the loopback receiver; '
|
|
192
|
+
+ 'set OPENWOP_WEBHOOK_ALLOW_PRIVATE=true on the host (or equivalent) to run',
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
expect(reg.status, driver.describe(
|
|
196
|
+
'webhooks.md §"Register"',
|
|
197
|
+
'POST /v1/webhooks MUST return 201 on success',
|
|
198
|
+
)).toBe(201);
|
|
199
|
+
|
|
200
|
+
// ── LEG 1 — POSITIVE CONTROL. Prove this wiring delivers at all. ─────────
|
|
201
|
+
const create = await driver.post('/v1/runs', { workflowId: 'conformance-noop' });
|
|
202
|
+
expect(create.status, 'failed to start conformance-noop').toBe(201);
|
|
203
|
+
const sourceRunId = (create.json as { runId: string }).runId;
|
|
204
|
+
await pollUntilTerminal(sourceRunId, { timeoutMs: 10_000 });
|
|
205
|
+
await quietWindow(DELIVERY_GRACE_MS);
|
|
206
|
+
|
|
207
|
+
expect(
|
|
208
|
+
forRun(receiver.received, sourceRunId).length,
|
|
209
|
+
driver.describe(
|
|
210
|
+
'webhooks.md §"Register"',
|
|
211
|
+
'POSITIVE CONTROL: the host must deliver the source run\'s events to this receiver — '
|
|
212
|
+
+ 'without this, every "no delivery" assertion below is vacuous',
|
|
213
|
+
),
|
|
214
|
+
).toBeGreaterThan(0);
|
|
215
|
+
|
|
216
|
+
// ── LEG 2 — THE MUST NOT. A replay fork re-emits; it must not deliver. ───
|
|
217
|
+
const replay = await driver.post(`/v1/runs/${encodeURIComponent(sourceRunId)}:fork`, {
|
|
218
|
+
mode: 'replay',
|
|
219
|
+
});
|
|
220
|
+
if (replay.status === 501) {
|
|
221
|
+
// Advertised but not implemented for this range. Leg 1 already asserted,
|
|
222
|
+
// so the FILE is `executed-pass` — but the MUST NOT was never exercised,
|
|
223
|
+
// and that is what this row must say.
|
|
224
|
+
recordRequirement(
|
|
225
|
+
REQUIREMENT_ID,
|
|
226
|
+
'blocked',
|
|
227
|
+
'replay fork returned 501 — the re-emission this requirement is stated over never happened, '
|
|
228
|
+
+ 'so the absence of deliveries below would prove nothing',
|
|
229
|
+
);
|
|
230
|
+
ctx.skip();
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
expect(replay.status, 'replay fork should be accepted').toBe(201);
|
|
234
|
+
const replayRunId = (replay.json as { runId: string }).runId;
|
|
235
|
+
await pollUntilTerminal(replayRunId, { timeoutMs: 30_000 });
|
|
236
|
+
await quietWindow(SUPPRESSION_WINDOW_MS);
|
|
237
|
+
|
|
238
|
+
expect(
|
|
239
|
+
forRun(receiver.received, replayRunId).map((d) => d.body.slice(0, 200)),
|
|
240
|
+
driver.describe(
|
|
241
|
+
'replay.md §"Host-initiated fan-out is an external effect"',
|
|
242
|
+
'a mode:"replay" fork MUST NOT emit outbound deliveries for the events it re-emits as fixed history — '
|
|
243
|
+
+ 'delivering one asserts to a subscriber that something happened in this run which did not',
|
|
244
|
+
),
|
|
245
|
+
).toEqual([]);
|
|
246
|
+
|
|
247
|
+
// Guard against the OTHER vacuity: a fork that emitted nothing at all would
|
|
248
|
+
// also deliver nothing. The fork's own log MUST still carry the re-emitted
|
|
249
|
+
// events (`replay.md`: suppression is outbound-only, the fork's event log
|
|
250
|
+
// still carries them). Without this, a host that simply failed the fork
|
|
251
|
+
// would pass leg 2.
|
|
252
|
+
const forkEvents = await driver.get(`/v1/runs/${encodeURIComponent(replayRunId)}/events`);
|
|
253
|
+
expect(forkEvents.status, 'fork events must be readable').toBe(200);
|
|
254
|
+
const forkEventList = (forkEvents.json as { events?: { type?: string }[] }).events ?? [];
|
|
255
|
+
expect(
|
|
256
|
+
forkEventList.length,
|
|
257
|
+
driver.describe(
|
|
258
|
+
'replay.md §"Host-initiated fan-out is an external effect"',
|
|
259
|
+
'suppression is OUTBOUND ONLY — the fork\'s own event log MUST still carry the re-emitted events, '
|
|
260
|
+
+ 'so an empty fork log means leg 2 proved nothing',
|
|
261
|
+
),
|
|
262
|
+
).toBeGreaterThan(0);
|
|
263
|
+
|
|
264
|
+
// The MUST NOT has now been exercised against wiring proven to deliver, on a
|
|
265
|
+
// fork proven to have re-emitted. Recorded here rather than after leg 3,
|
|
266
|
+
// because leg 3 pins the boundary and may legitimately not run.
|
|
267
|
+
recordRequirement(REQUIREMENT_ID, 'executed-pass', undefined, { assertionCount: 4 });
|
|
268
|
+
|
|
269
|
+
// ── LEG 3 — the boundary. `branch` is explicitly OUT of scope. ───────────
|
|
270
|
+
// This is what distinguishes "reads replay-ness from the RUN" from "silences
|
|
271
|
+
// anything that is a fork". A host that suppressed both would pass legs 1-2
|
|
272
|
+
// and be wrong: a branch fork's events are new facts and its effects are the
|
|
273
|
+
// ones the operator asked for.
|
|
274
|
+
const branch = await driver.post(`/v1/runs/${encodeURIComponent(sourceRunId)}:fork`, {
|
|
275
|
+
mode: 'branch',
|
|
276
|
+
});
|
|
277
|
+
if (branch.status === 501 || branch.status === 400) {
|
|
278
|
+
// branch not offered on this range — leg 2 still stands on its own.
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
expect(branch.status, 'branch fork should be accepted').toBe(201);
|
|
282
|
+
const branchRunId = (branch.json as { runId: string }).runId;
|
|
283
|
+
await pollUntilTerminal(branchRunId, { timeoutMs: 30_000 });
|
|
284
|
+
await quietWindow(DELIVERY_GRACE_MS);
|
|
285
|
+
|
|
286
|
+
expect(
|
|
287
|
+
forRun(receiver.received, branchRunId).length,
|
|
288
|
+
driver.describe(
|
|
289
|
+
'replay.md §"Host-initiated fan-out is an external effect"',
|
|
290
|
+
'a branch fork is deliberately OUT of scope — its events are new facts, so suppressing them '
|
|
291
|
+
+ 'means the host keyed on "is a fork" rather than on replay-ness read from the run',
|
|
292
|
+
),
|
|
293
|
+
).toBeGreaterThan(0);
|
|
294
|
+
});
|
|
295
|
+
});
|