@openwop/openwop-conformance 1.136.11 → 1.139.0

package/src/scenarios/replayDeterminism.test.ts

@@ -18,6 +18,8 @@
 
 import { describe, it, expect } from 'vitest';
 import { driver } from '../lib/driver.js';
+import { softSkip } from '../lib/soft-skip.js';
+import { forkDeclined } from '../lib/fork-availability.js';
 import { pollUntilTerminal } from '../lib/polling.js';
 import { isFixtureAdvertised } from '../lib/fixtures.js';
 
@@ -69,7 +71,7 @@ function structuralShape(events: readonly RawEvent[]): Array<{ type: unknown; no
 describe('replay-determinism: openwop-replay-fork profile gate', () => {
   it('host advertising replay.supported MUST also advertise replay.modes', async () => {
     const replay = await fetchReplayCapability();
-    if (replay === null || replay.supported !== true) return; // skip-equivalent
+    if (replay === null || replay.supported !== true) return softSkip('inapplicable', 'host does not advertise `replay.supported: true` — the replay contract does not apply to it');
 
     expect(Array.isArray(replay.modes), driver.describe(
       'spec/v1/replay.md',
@@ -91,8 +93,8 @@ describe.skipIf(SKIP_NO_NOOP)('replay-determinism: same fromSeq + same workflow
     'two replay forks of the same point produce structurally-identical event lists',
     async () => {
       const replay = await fetchReplayCapability();
-      if (replay === null || replay.supported !== true) return; // host doesn't claim replay
-      if (!Array.isArray(replay.modes) || !replay.modes.includes('replay')) return; // mode not supported
+      if (replay === null || replay.supported !== true) return softSkip('inapplicable', 'host does not advertise `replay.supported: true` — the replay contract does not apply to it');
+      if (!Array.isArray(replay.modes) || !replay.modes.includes('replay')) return softSkip('inapplicable', 'host advertises replay but not the `replay` mode — this leg is out of scope for it');
 
       // Phase 1: complete an original run.
       const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
@@ -105,7 +107,7 @@ describe.skipIf(SKIP_NO_NOOP)('replay-determinism: same fromSeq + same workflow
         mode: 'replay',
         fromSeq: 0,
       });
-      if (fork1.status === 501) return; // mode not implemented; skip-equivalent
+      if (forkDeclined(fork1.status, 'determinism fork 1')) return;
       expect(fork1.status, driver.describe(
         'spec/v1/replay.md',
         'POST /v1/runs/{runId}:fork with mode=replay MUST return 201',
@@ -118,7 +120,7 @@ describe.skipIf(SKIP_NO_NOOP)('replay-determinism: same fromSeq + same workflow
         mode: 'replay',
         fromSeq: 0,
       });
-      if (fork2.status === 501) return;
+      if (forkDeclined(fork2.status, 'determinism fork 2')) return;
       expect(fork2.status).toBe(201);
       const fork2Id = (fork2.json as { runId: string }).runId;
       await pollUntilTerminal(fork2Id, { timeoutMs: 10_000 });
@@ -152,7 +154,7 @@ describe.skipIf(SKIP_NO_NOOP)('replay-determinism: same fromSeq + same workflow
 describe.skipIf(SKIP_NO_NOOP)('replay-determinism: branch-mode is permitted to diverge', () => {
   it('branch mode does NOT need to produce identical event sequences (negative-control)', async () => {
     const replay = await fetchReplayCapability();
-    if (replay === null || replay.supported !== true) return;
+    if (replay === null || replay.supported !== true) return softSkip('inapplicable', 'host does not advertise `replay.supported: true` — the replay contract does not apply to it');
     if (!Array.isArray(replay.modes) || !replay.modes.includes('branch')) return;
 
     // Self-test on the spec interpretation: branch and replay are

package/src/scenarios/webhook-receiver-adversarial.test.ts

@@ -17,7 +17,7 @@
  *      delivers body B'. Receiver MUST reject with
  *      `signature_mismatch`.
  *   2. Tampered HMAC — body is valid; adversary flips a byte of the
- *      v1=<hex> signature. Receiver MUST reject with
+ *      sha256=<hex> signature. Receiver MUST reject with
  *      `signature_mismatch`.
  *   3. Stale timestamp — body + HMAC are valid but timestamp is
  *      older than the default 5-minute window. Receiver MUST reject
@@ -42,6 +42,7 @@ import {
   createReceiverState,
   signPayload,
   verifyWebhookDelivery,
+  SIGNATURE_PREFIX,
 } from '../lib/webhook-receiver.js';
 
 describe('webhook-receiver-adversarial: receiver rejects five canonical attacks', () => {
@@ -93,8 +94,13 @@ describe('webhook-receiver-adversarial: receiver rejects five canonical attacks'
   it('case 2: tampered HMAC → signature_mismatch', () => {
     const state = createReceiverState();
     const { signatureHeader, timestampHeader, algorithmHeader } = signPayload(secret, ts, body);
-    // Flip one hex character of the v1=<hex> signature.
-    const flipIndex = 5;
+    // Flip one hex character of the `sha256=<hex>` signature. Derived from
+    // SIGNATURE_PREFIX rather than hard-coded: the literal `5` was inside the
+    // hex only while the prefix was the (non-spec) `v1=`, and silently moved
+    // INTO the prefix when it was corrected to `sha256=` — turning a
+    // tampered-HMAC case into a malformed-header one. The test caught it; the
+    // constant is what makes it stay caught.
+    const flipIndex = SIGNATURE_PREFIX.length + 2;
     const orig = signatureHeader[flipIndex]!;
     const replacement = orig === '0' ? '1' : '0';
     const tampered = signatureHeader.slice(0, flipIndex) + replacement + signatureHeader.slice(flipIndex + 1);

package/src/scenarios/webhook-signed-delivery.test.ts

@@ -7,18 +7,68 @@
  *   2. `X-openwop-Signature-Algorithm: v1` header is present.
  *   3. `X-openwop-Signature` is a valid HMAC-SHA256 of
  *      `${timestamp}.${rawBody}` under the subscription secret.
- *   4. `X-openwop-Subscription-Id` matches the returned subscription id.
+ *   4. `X-openwop-Webhook-Id` matches the returned `webhookId`.
+ *      (This line said `X-openwop-Subscription-Id` until 2026-08-19, two years
+ *      after the assertion below stopped checking that header — a docblock
+ *      asserting more than the code did, in the file that documents a
+ *      security-relevant contract.)
  *
  * Capability-gated: skips when the host does not advertise
  * `capabilities.webhooks.supported = true`.
  *
- * Operator contract: hosts that implement a SSRF guard on
- * `POST /v1/webhooks` (rejecting loopback / RFC1918 / link-local
- * destinations to protect deployer infrastructure) MUST allow the test
- * receiver. The SQLite reference host bypasses the guard when the
- * `OPENWOP_WEBHOOK_ALLOW_PRIVATE=true` env var is set at boot. Test-only
- * hosts SHOULD provide an equivalent opt-in. When the host rejects with
- * `400 webhook_url_rejected`, this scenario skips with a warning.
+ * Operator contract — THREE gates, not one (clarified 2026-08-25).
+ *
+ * The test receiver is `http://127.0.0.1:{port}/`, and `webhooks.md`
+ * forbids it three separate times. A host honoring
+ * `OPENWOP_WEBHOOK_ALLOW_PRIVATE=true` (or an equivalent opt-in) MUST
+ * relax ALL THREE for this scenario to be witnessable:
+ *
+ *   1. **Scheme.** §"SSRF protection" bullet 1 rejects non-`https://`
+ *      protocols, and §"Register" says `url` MUST be `https://`. The
+ *      receiver is plain `http`. This gate is the one the flag's
+ *      description omitted until 2026-08-25, and it fires FIRST on a
+ *      host that validates scheme before address.
+ *   2. **Registration-time address check.** §"SSRF protection": the
+ *      server MUST validate subscription URLs at registration time and
+ *      reject loopback / RFC1918 / link-local / ULA / metadata.
+ *   3. **Delivery-time re-resolution.** §"Delivery-time egress
+ *      validation (RFC 0093)": the dispatcher MUST re-resolve at
+ *      delivery time and validate every resolved address against the
+ *      same ranges.
+ *
+ * Gates 2 and 3 are INDEPENDENT MUSTs at different layers, so "which
+ * layer must the opt-in reach" is not a matter of taste: a relaxation
+ * reaching only one layer cannot produce a witness. Delivery-only leaves
+ * registration returning `400 webhook_url_rejected` (observed on a tier-2
+ * host, 2026-08-25); registration-only leaves the dispatcher re-resolving
+ * `127.0.0.1` and refusing to connect. A host whose opt-in reaches one
+ * layer is not non-conformant — it simply cannot witness this scenario,
+ * and that is a property of the test posture, not of its webhook signing.
+ *
+ * What that costs is more than a checkmark, and it is worth stating plainly
+ * because the cost is invisible from inside such a host. This scenario is
+ * typically the ONLY automated oracle a host has on its delivery header
+ * NAMES: a host-local delivery test almost always compares a hand-written
+ * literal in the test to a hand-written literal in the implementation, which
+ * is a mirror — typo both and it stays green — while the HMAC assertion
+ * beside it recomputes from `node:crypto` and is real. So the names go
+ * unchecked exactly where the bytes are checked well. All three reference
+ * hosts emitted invented header names for 16 days (openwop-examples#22) and
+ * the runs that would have caught it were the ones this scenario could not
+ * reach. A host that runs it is not thereby careful; it is thereby measured.
+ *
+ * Relaxing gates 2 and 3 is test-only posture. See
+ * `SECURITY/threat-model-secret-leakage.md` §4.9 for why a
+ * registration-time relaxation is the more dangerous of the two: it
+ * writes a durable subscription row that survives the flag being turned
+ * back off.
+ *
+ * When the host rejects with `400 webhook_url_rejected`, this scenario
+ * records `blocked` (RFC 0148 §A) — NOT a pass. Under a plain
+ * `vitest run` the console still prints "1 passed", because that is
+ * vitest reporting test outcomes rather than conformance dispositions;
+ * the run-end disposition summary (`src/global-setup.ts`) is where the
+ * `blocked` becomes visible without `--certify`.
  *
  * @see spec/v1/webhooks.md §"Signature scheme"
  */
@@ -122,10 +172,23 @@ describe('webhook-signed-delivery: end-to-end HMAC v1', () => {
 
     expect(reg.status, driver.describe(
       'webhooks.md §"Register"',
-      'POST /v1/webhooks MUST return 201 with subscriptionId + secret on success',
+      'POST /v1/webhooks MUST return 201 with webhookId + secret on success',
     )).toBe(201);
-    const sub = reg.json as { subscriptionId: string; secret: string };
-    expect(typeof sub.subscriptionId).toBe('string');
+    // `webhookId`, NOT `subscriptionId` (corrected 2026-08-19). `webhooks.md`
+    // §"Register" shows `{"webhookId": "wh_a3b9c2", ...}`, `api/openapi.yaml`
+    // declares the 201 body `required: [webhookId]` with no `subscriptionId`
+    // property at all, and the sibling `webhook-tenant-isolation.test.ts` reads
+    // `webhookId`. This file required a field the contract does not define, so a
+    // SPEC-CONFORMING host failed at the first assertion — the suite being
+    // different from the spec, which is worse than the stricter-than-spec case
+    // COMPATIBILITY.md §2.3 forbids. Reported by a tier-2 host emitting exactly
+    // what the spec shows. The postgres reference host returns both names, which
+    // is why nothing went red here.
+    const sub = reg.json as { webhookId: string; secret: string };
+    expect(typeof sub.webhookId, driver.describe(
+      'api/openapi.yaml registerWebhook 201',
+      'the 201 body MUST carry `webhookId` (required) — `subscriptionId` is not in the contract',
+    )).toBe('string');
     expect(typeof sub.secret).toBe('string');
     expect(sub.secret.length).toBeGreaterThan(0);
 
@@ -174,7 +237,7 @@ describe('webhook-signed-delivery: end-to-end HMAC v1', () => {
     expect(
       first.headers['x-openwop-webhook-id'],
       driver.describe('webhooks.md §"Delivery headers"', 'X-openwop-Webhook-Id MUST carry the subscription id'),
-    ).toBe(sub.subscriptionId);
+    ).toBe(sub.webhookId);
 
     const timestamp = first.headers['x-openwop-timestamp'];
     expect(
@@ -201,7 +264,7 @@ describe('webhook-signed-delivery: end-to-end HMAC v1', () => {
     expect(event.runId).toBe(runId);
 
     // Cleanup: unregister.
-    const del = await driver.delete(`/v1/webhooks/${encodeURIComponent(sub.subscriptionId)}`);
+    const del = await driver.delete(`/v1/webhooks/${encodeURIComponent(sub.webhookId)}`);
     expect(del.status).toBeGreaterThanOrEqual(200);
     expect(del.status).toBeLessThan(300);
   });

package/src/setup.ts

@@ -32,7 +32,7 @@ import { McpFakeServer, setMcpFakeServer } from './lib/mcp-fake-server.js';
 import { A2AFakePeer, setA2AFakePeer } from './lib/a2a-fake-peer.js';
 import { afterAll, afterEach, beforeAll, expect } from 'vitest';
 import { basename } from 'node:path';
-import { recordRequirement, journalLength, journalSince } from './lib/requirement-ledger.js';
+import { recordRequirement, hasRequirement, journalLength, journalSince } from './lib/requirement-ledger.js';
 import { requirementIdForFile, resolveFileRecord, type FileTestState } from './lib/scenario-disposition.js';
 import { softSkipDisposition } from './lib/soft-skip.js';
 import type { DiscoveryPayload } from './lib/profiles.js';
@@ -279,10 +279,29 @@ afterAll(({}, suite) => {
   // honest bundle row and the pressure to say why both survive. The rule is
   // `resolveFileRecord` (pinned by conformance-execution-witness.test.ts).
   const { disposition, detail } = resolveFileRecord(states, gateReason, assertionCount, softSkipDisposition(file));
-  try {
-    recordRequirement(requirementIdForFile(file), disposition, detail, { assertionCount });
-  } catch {
-    /* a scenario that recorded its own file id first wins; never fail the file for bookkeeping */
+  const fileRequirementId = requirementIdForFile(file);
+  // A scenario that classified ITSELF wins outright — including its `detail` and
+  // its `assertionCount`.
+  //
+  // The `catch` below has always made the explicit record win when the two
+  // DISAGREE (`recordRequirement` throws on a conflicting disposition). It did
+  // not when they AGREE: the automatic call then reached `ledger.set` and
+  // silently replaced the scenario's own detail and count with the file-level
+  // ones. Invisible until 2026-08-19, when `resolveFileRecord` started attaching
+  // a `partial-witness:` marker — a scenario that recorded `executed-pass` for a
+  // requirement it really did exercise, in a file whose LAST leg soft-skipped,
+  // would have been stamped "may not have witnessed this" over its own explicit
+  // finding. That would inject false positives into exactly the measurement the
+  // marker exists to produce.
+  //
+  // So the comment describing this line was true of half the cases. It is true
+  // of both now.
+  if (!hasRequirement(fileRequirementId)) {
+    try {
+      recordRequirement(fileRequirementId, disposition, detail, { assertionCount });
+    } catch {
+      /* never fail a file for bookkeeping */
+    }
   }
   _fileStates.delete(file);
   _fileAssertions.delete(file);

package/vitest.config.ts

@@ -17,5 +17,12 @@ export default defineConfig({
     // populates `lib/fixtures.ts` so `describe.skipIf(...)` predicates
     // see the cached set when scenarios register their tests.
     setupFiles: ['src/setup.ts'],
+    // RFC 0148 §A run-end disposition summary. `setup.ts` records an honest
+    // disposition per scenario file on EVERY run; before this, only
+    // `--certify` set OPENWOP_LEDGER_PATH, so on a plain `vitest run` the
+    // recording was computed and then discarded, leaving "N passed" as the
+    // sole artifact for rows the suite had classified `blocked`. This routes
+    // the ledger to a temp file and prints what did not witness.
+    globalSetup: ['src/global-setup.ts'],
   },
 });