@openwop/openwop-conformance 1.106.0 → 1.123.0

package/src/scenarios/subrun-approval-gate.test.ts

@@ -10,23 +10,24 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { readSubRunAttestationCap, invokeSubRunAttest } from '../lib/subRunAttestation.js';
 
 describe('subrun-approval-gate (RFC 0063 §C)', () => {
   it('accept merges the child outputs; reject does not', async () => {
-    if ((await readSubRunAttestationCap()) !== true) return;
+    if ((await readSubRunAttestationCap()) !== true) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `(await readSubRunAttestationCap()) !== true` returned early');
     const base = { childOutputs: { artifact: 'x' }, outputAttestation: { requireApproval: true } };
 
     const accepted = await invokeSubRunAttest({ ...base, approvalAction: 'accept' });
-    if (accepted === null) return; // seam absent — soft-skip
+    if (accepted === null) return softSkip('blocked', 'seam absent — soft-skip');
     expect(
       accepted.merged,
       driver.describe('RFC 0063 §C', 'an `accept` approval MUST merge the child outputs'),
     ).toBe(true);
 
     const rejected = await invokeSubRunAttest({ ...base, approvalAction: 'reject' });
-    if (rejected === null) return;
+    if (rejected === null) return softSkip('blocked', 'precondition not met — `rejected === null` returned early (seam, prior step, or fixture unavailable)');
     expect(
       rejected.merged,
       driver.describe('RFC 0063 §C', 'a `reject` approval MUST NOT merge the child outputs'),

package/src/scenarios/subrun-attestation-shape.test.ts

@@ -11,6 +11,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { readSubRunAttestationCap } from '../lib/subRunAttestation.js';
 
@@ -18,7 +19,7 @@ describe('subrun-attestation-shape: advertisement (RFC 0063 §A)', () => {
   it('capabilities.agents.subRunAttestation is absent or a boolean', async () => {
     const cap = await readSubRunAttestationCap();
     // null = unadvertised (no agents block OR flag omitted) — valid.
-    if (cap === null) return;
+    if (cap === null) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `cap === null` returned early');
     expect(
       typeof cap,
       driver.describe(

package/src/scenarios/subrun-checksum-stable.test.ts

@@ -11,21 +11,22 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { readSubRunAttestationCap, invokeSubRunAttest } from '../lib/subRunAttestation.js';
 
 describe('subrun-checksum-stable (RFC 0063 §B)', () => {
   it('identical child outputs produce an identical sha256 attestation checksum', async () => {
-    if ((await readSubRunAttestationCap()) !== true) return;
+    if ((await readSubRunAttestationCap()) !== true) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `(await readSubRunAttestationCap()) !== true` returned early');
     const childOutputs = { report: 'done', score: 0.9, tags: ['a', 'b'] };
     const a = await invokeSubRunAttest({ childOutputs, outputAttestation: { checksum: true } });
-    if (a === null) return; // seam absent — soft-skip
+    if (a === null) return softSkip('blocked', 'seam absent — soft-skip');
     // Key-reordered but value-identical: JCS canonicalization MUST yield the same hash.
     const b = await invokeSubRunAttest({
       childOutputs: { tags: ['a', 'b'], score: 0.9, report: 'done' },
       outputAttestation: { checksum: true },
     });
-    if (b === null) return;
+    if (b === null) return softSkip('blocked', 'precondition not met — `b === null` returned early (seam, prior step, or fixture unavailable)');
     const att = a.attestation ?? {};
     expect(
       typeof att.checksum === 'string' && (att.checksum as string).length > 0,

package/src/scenarios/test-seam-unauthenticated.test.ts

@@ -67,11 +67,41 @@ const SEAM_PROBES: readonly string[] = [
 /** 200 is the only unambiguous failure: the seam answered a stranger. */
 const ACCEPTABLE = new Set([401, 403, 404, 405, 501]);
 
+/**
+ * A `200` can arrive from two very different places, and the difference decides
+ * what you should go fix.
+ *
+ * Every seam in `host-sample-test-seams.md` answers with a JSON body — the doc
+ * specifies each one's response shape as an object (`{ released, event }`,
+ * `{ ok: true, packId }`, `{ entries: … }`). **No seam answers with an HTML
+ * document.** So a `200 text/html` is not a seam that talked to a stranger; it
+ * is a static-hosting catch-all — an SPA rewrite in front of the API — matching
+ * a path the backend never saw.
+ *
+ * This was found by probing `app.openwop.dev` (Firebase Hosting, `**` rewrite
+ * to the SPA shell) instead of the backend's own origin. Every probe returned
+ * `200 text/html` with the app's `index.html`, which under the previous message
+ * read as *"an open control surface on a public origin"* — alarming, and wrong.
+ * The same paths on `…-backend-….run.app` return `404 application/json`.
+ *
+ * **Both cases still fail.** Nothing here relaxes the assertion, because the
+ * cheap way to silence a false alarm is to stop failing on `200`, and that would
+ * delete the check. What changes is the diagnosis: an HTML catch-all means the
+ * suite is pointed at the CDN rather than the API origin, which is its own
+ * defect — a run against the wrong origin cannot witness anything about the
+ * host, and every other leg in the suite is equally blind at that base URL.
+ */
+function classify200(contentType: string | null): 'seam' | 'catchall' {
+  return (contentType ?? '').toLowerCase().includes('text/html') ? 'catchall' : 'seam';
+}
+
 describe('test-seam-unauthenticated: an enabled seam still authenticates', () => {
   it('no /v1/host/sample/* seam answers an unauthenticated request with 200', async () => {
     const env = loadEnv();
     const answered: string[] = [];
     let probed = 0;
+    let seamAnswers = 0;
+    let catchAllAnswers = 0;
 
     for (const path of SEAM_PROBES) {
       // node:fetch directly with NO Authorization header — the driver's
@@ -83,8 +113,18 @@ describe('test-seam-unauthenticated: an enabled seam still authenticates', () =>
         continue; // connection-level refusal is a stronger answer than 404
       }
       probed += 1;
-      if (res.status === 200) answered.push(`${path} -> 200`);
-      else if (!ACCEPTABLE.has(res.status)) {
+      if (res.status === 200) {
+        const contentType = res.headers.get('content-type');
+        const kind = classify200(contentType);
+        if (kind === 'seam') seamAnswers += 1;
+        else catchAllAnswers += 1;
+        answered.push(
+          `${path} -> 200 (${contentType ?? 'no content-type'}) — ` +
+            (kind === 'seam'
+              ? 'SEAM ANSWERED A CREDENTIAL-LESS CALLER'
+              : 'HTML body: a static-hosting catch-all, not the API origin'),
+        );
+      } else if (!ACCEPTABLE.has(res.status)) {
         // Not a pass and not the known failure — record it rather than
         // silently tolerating a status nobody reasoned about.
         answered.push(`${path} -> ${res.status} (unexpected; expected one of ${[...ACCEPTABLE].join('/')})`);
@@ -100,17 +140,27 @@ describe('test-seam-unauthenticated: an enabled seam still authenticates', () =>
         'not looking, which is the failure mode it exists to prevent',
     ).toBeGreaterThan(0);
 
+    // The diagnosis is chosen from what the answers actually were. An HTML
+    // catch-all and an open seam are both failures and they are not the same
+    // bug, so the message must not name the wrong one — a check that reports a
+    // security finding for a misrouted base URL trains its reader to distrust it.
+    const diagnosis =
+      seamAnswers === 0 && catchAllAnswers > 0
+        ? 'WRONG ORIGIN, not an open seam. Every 200 above carried an HTML body, and no seam in ' +
+          'host-sample-test-seams.md answers with HTML — so these are a static-hosting rewrite ' +
+          '(an SPA `**` catch-all) matching paths the backend never received. Point ' +
+          'OPENWOP_BASE_URL at the API origin itself. This is still a failure: at this base URL ' +
+          'no leg in the suite is witnessing the host, so a green run here would mean nothing.'
+        : 'An ENABLED seam MUST require an authenticated, NON-ANONYMOUS principal. A host that ' +
+          'mints an anonymous identity for credential-less callers MUST NOT treat it as ' +
+          'satisfying that. The env-gate governs whether a seam EXISTS; it does not govern who ' +
+          'may call it. A seam answering a credential-less request with 200 is an open control ' +
+          'surface on a public origin — and staging keys such as `nodeId` are not secrets, they ' +
+          'ship inside chain packs.';
+
     expect(
       answered,
-      driver.describe(
-        'host-sample-test-seams.md §"Production safety (normative)"',
-        'An ENABLED seam MUST require an authenticated, NON-ANONYMOUS principal. A host that mints ' +
-          'an anonymous identity for credential-less callers MUST NOT treat it as satisfying that. ' +
-          'The env-gate governs whether a seam EXISTS; it does not govern who may call it. A seam ' +
-          'answering a credential-less request with 200 is an open control surface on a public ' +
-          'origin — and staging keys such as `nodeId` are not secrets, they ship inside chain ' +
-          'packs.\n  ' + answered.join('\n  '),
-      ),
+      driver.describe('host-sample-test-seams.md §"Production safety (normative)"', `${diagnosis}\n  ${answered.join('\n  ')}`),
     ).toEqual([]);
   });
 });

package/src/scenarios/tool-hooks-authorization-fail-closed.test.ts

@@ -12,13 +12,14 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { readToolHooksCap, invokeToolHook } from '../lib/toolHooks.js';
 
 describe('tool-hooks-authorization-fail-closed (RFC 0064 §C)', () => {
   it('a principal lacking a tool scope is denied and the tool is not invoked', async () => {
     const cap = await readToolHooksCap();
-    if (cap?.perToolAuthorization !== true) return;
+    if (cap?.perToolAuthorization !== true) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `cap?.perToolAuthorization !== true` returned early');
     // A principal with no scopes against a tool requiring one MUST be denied.
     const res = await invokeToolHook({
       principal: 'conformance-unprivileged',
@@ -26,7 +27,7 @@ describe('tool-hooks-authorization-fail-closed (RFC 0064 §C)', () => {
       requiredScopes: ['db:write'],
       args: {},
     });
-    if (res === null) return; // seam absent — soft-skip
+    if (res === null) return softSkip('blocked', 'seam absent — soft-skip');
     expect(
       (res.toolReturned ?? {}).status,
       driver.describe('RFC 0064 §C', 'a missing/unevaluable tool scope MUST fail closed → status:"forbidden"'),

package/src/scenarios/tool-hooks-content-free.test.ts

@@ -11,15 +11,16 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { readToolHooksCap, invokeToolHook } from '../lib/toolHooks.js';
 
 describe('tool-hooks-content-free (RFC 0064 §B)', () => {
   it('toolCalled carries argsHash; toolReturned carries status + durationMs', async () => {
     const cap = await readToolHooksCap();
-    if (cap?.prePostEvents !== true) return;
+    if (cap?.prePostEvents !== true) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `cap?.prePostEvents !== true` returned early');
     const res = await invokeToolHook({ principal: 'core.system', toolName: 'web.search', args: { q: 'openwop' } });
-    if (res === null) return; // seam absent — soft-skip
+    if (res === null) return softSkip('blocked', 'seam absent — soft-skip');
     const called = res.toolCalled ?? {};
     const returned = res.toolReturned ?? {};
     expect(

package/src/scenarios/tool-hooks-rate-limit.test.ts

@@ -10,20 +10,21 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { readToolHooksCap, invokeToolHook } from '../lib/toolHooks.js';
 
 describe('tool-hooks-rate-limit (RFC 0064 §D)', () => {
   it('an exhausted (principal, tool) bucket yields status:"rate_limited"', async () => {
     const cap = await readToolHooksCap();
-    if (cap?.perToolRateLimit !== true) return;
+    if (cap?.perToolRateLimit !== true) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `cap?.perToolRateLimit !== true` returned early');
     const res = await invokeToolHook({
       principal: 'core.system',
       toolName: 'web.search',
       args: { q: 'x' },
       simulateRateLimitExhausted: true,
     });
-    if (res === null) return; // seam absent — soft-skip
+    if (res === null) return softSkip('blocked', 'seam absent — soft-skip');
     expect(
       (res.toolReturned ?? {}).status,
       driver.describe('RFC 0064 §D', 'an exhausted token bucket MUST yield status:"rate_limited" without invoking the tool'),

package/src/scenarios/tool-hooks-secret-redaction.test.ts

@@ -11,6 +11,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { readToolHooksCap, invokeToolHook } from '../lib/toolHooks.js';
 
@@ -19,13 +20,13 @@ const CANARY = 'sk-canary-rfc0064-do-not-leak-xyz789';
 describe('tool-hooks-secret-redaction (RFC 0064 §E)', () => {
   it('a secret-shaped tool arg never appears in the emitted events', async () => {
     const cap = await readToolHooksCap();
-    if (cap?.prePostEvents !== true) return;
+    if (cap?.prePostEvents !== true) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `cap?.prePostEvents !== true` returned early');
     const res = await invokeToolHook({
       principal: 'core.system',
       toolName: 'web.search',
       args: { apiKey: CANARY, q: 'openwop' },
     });
-    if (res === null) return; // seam absent — soft-skip
+    if (res === null) return softSkip('blocked', 'seam absent — soft-skip');
     expect(
       JSON.stringify(res).includes(CANARY),
       driver.describe('RFC 0064 §B', 'a resolved secret MUST be redacted before hashing; the raw value MUST NOT appear in argsHash or any emitted field (SR-1)'),

package/src/scenarios/tool-hooks-shape.test.ts

@@ -11,13 +11,14 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { readToolHooksCap } from '../lib/toolHooks.js';
 
 describe('tool-hooks-shape: advertisement (RFC 0064 §A)', () => {
   it('capabilities.toolHooks is absent or a well-formed object', async () => {
     const cap = await readToolHooksCap();
-    if (cap === null) return; // not advertised — valid
+    if (cap === null) return softSkip('inapplicable', 'not advertised — valid');
     expect(
       typeof cap.supported,
       driver.describe('capabilities.schema.json §toolHooks', 'toolHooks.supported MUST be a boolean when the block is present'),

package/src/scenarios/trigger-bridge-delivery.test.ts

@@ -28,6 +28,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { seamAbsent } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { behaviorGate } from '../lib/behavior-gate.js';
 import {
@@ -52,11 +53,11 @@ function expectContentFree(payload: Record<string, unknown>, where: string): voi
 describe('trigger-bridge-delivery (RFC 0083 §C)', () => {
   it('de-dups by dedupKey, retries to dead-letter, and links delivery→run causation', async () => {
     if (!behaviorGate('openwop-trigger-bridge', await isTriggerBridgeProfileAdvertised())) return;
-    if (!(await isEventLogSeamAvailable())) return; // event-log seam absent — soft-skip
+    if (!(await isEventLogSeamAvailable())) return seamAbsent('host advertises openwop-trigger-bridge but the event-log seam is absent');
 
     // ---- Leg 1: dedup → effectively-once (§C-1) ---------------------------
     const dedup = await driveDelivery({ scenario: 'dedup', dedupKey: 'conformance-dedup-key', source: 'queue' });
-    if (dedup === null) return; // delivery seam unwired — soft-skip the whole behavioral suite
+    if (dedup === null) return seamAbsent('host advertises openwop-trigger-bridge but the delivery seam is unwired');
 
     // The profile is derived AND the seam is wired — missing evidence is a
     // FAILURE, not a soft-skip. A repeated dedupKey MUST be effectively-once:

package/src/scenarios/version-fold.test.ts

@@ -34,6 +34,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver, type OpenWOPResponse } from '../lib/driver.js';
 import { pollUntilTerminal } from '../lib/polling.js';
 import { isFixtureAdvertised } from '../lib/fixtures.js';
@@ -84,7 +85,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('version-fold: forced engine versions fold-best
         '[version-fold] host does not advertise Capabilities.testing.forceEngineVersionRange; ' +
           'skipping (the X-Force-Engine-Version seam is test-keys-only and optional)',
       );
-      return;
+      return softSkip('blocked', 'precondition not met — `seam === null` returned early (seam, prior step, or fixture unavailable)');
     }
 
     const candidates = [seam.range.min, ...(seam.current === null ? [] : [seam.current]), seam.range.max];
@@ -104,7 +105,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('version-fold: forced engine versions fold-best
       if (create.status === 403 && errCode(create.json) === 'force_engine_version_forbidden') {
         // eslint-disable-next-line no-console
         console.warn('[version-fold] API key is production-scoped (403 force_engine_version_forbidden); skipping');
-        return;
+        return softSkip('blocked', '[version-fold] API key is production-scoped (403 force_engine_version_forbidden); skipping (create.status === 403 && errCode(create.json) === \'force_engine_version_forbidden\')');
       }
       expect(
         create.status,
@@ -165,7 +166,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('version-fold: forced engine versions fold-best
     if (seam === null) {
       // eslint-disable-next-line no-console
       console.warn('[version-fold] forceEngineVersionRange not advertised; skipping negative leg');
-      return;
+      return softSkip('inapplicable', '[version-fold] forceEngineVersionRange not advertised; skipping negative leg');
     }
 
     const outOfRange = seam.range.max + 1;
@@ -173,7 +174,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('version-fold: forced engine versions fold-best
     if (create.status === 403 && errCode(create.json) === 'force_engine_version_forbidden') {
       // eslint-disable-next-line no-console
       console.warn('[version-fold] API key is production-scoped; skipping negative leg');
-      return;
+      return softSkip('blocked', '[version-fold] API key is production-scoped; skipping negative leg (create.status === 403 && errCode(create.json) === \'force_engine_version_forbidden\')');
     }
     expect(
       create.status,

package/src/scenarios/wasm-pack-abi-version-rejection.test.ts

@@ -25,6 +25,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { capabilityFamily } from '../lib/discovery-capabilities.js';
 
@@ -37,7 +38,7 @@ describe('wasm-pack-abi-version-rejection: host advertises supported ABI version
     const wasm =
       capabilityFamily<{ wasm?: Record<string, unknown> }>(disco.json, 'nodePackRuntimes')?.wasm;
 
-    if (!wasm?.supported) return;
+    if (!wasm?.supported) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!wasm?.supported` returned early');
 
     expect(Array.isArray(wasm.abiVersions), driver.describe(
       'RFCS/0008-wasm-abi.md §H',
@@ -63,7 +64,7 @@ describe('wasm-pack-abi-version-rejection: positive path via misbehaving pack',
     const wasm =
       capabilityFamily<{ wasm?: Record<string, unknown> }>(disco.json, 'nodePackRuntimes')?.wasm;
 
-    if (!wasm?.supported) return;
+    if (!wasm?.supported) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!wasm?.supported` returned early');
 
     if (!Array.isArray(wasm.loadedPacks)) {
       // eslint-disable-next-line no-console
@@ -71,7 +72,7 @@ describe('wasm-pack-abi-version-rejection: positive path via misbehaving pack',
         '[wasm-pack-abi-version-rejection] host does not advertise capabilities.nodePackRuntimes.wasm.loadedPacks[]; skipping positive path. ' +
           'Hosts MAY advertise loadedPacks[] to surface ABI rejection observably (RFC 0008 §H + Track 7).',
       );
-      return;
+      return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!Array.isArray(wasm.loadedPacks)` returned early');
     }
 
     // Soft-skip when the well-behaved reference pack isn't loaded. If
@@ -84,7 +85,7 @@ describe('wasm-pack-abi-version-rejection: positive path via misbehaving pack',
         `[wasm-pack-abi-version-rejection] reference pack ${WELL_BEHAVED_PACK_NAME} not in loadedPacks[]; ` +
           'skipping positive path (build examples/packs/rust-hello first or run against a host that ships it).',
       );
-      return;
+      return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!wasm.loadedPacks.includes(WELL_BEHAVED_PACK_NAME)` returned early');
     }
 
     // Core assertion: ABI 999 pack MUST NOT be in loadedPacks[]

package/src/scenarios/wasm-pack-invoke-completed.test.ts

@@ -13,6 +13,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { pollUntilTerminal } from '../lib/polling.js';
 import { isFixtureAdvertised } from '../lib/fixtures.js';
@@ -29,11 +30,11 @@ async function isWasmSupported(): Promise<boolean> {
 
 describe('wasm-pack-invoke-completed: round-trip output', () => {
   it('greet node returns Hello, <name>! and run reaches completed', async () => {
-    if (!isFixtureAdvertised(FIXTURE)) return;
+    if (!isFixtureAdvertised(FIXTURE)) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!isFixtureAdvertised(FIXTURE)` returned early');
     if (!(await isWasmSupported())) {
       // eslint-disable-next-line no-console
       console.warn('[wasm-pack-invoke-completed] WASM not advertised; skipping');
-      return;
+      return softSkip('inapplicable', '[wasm-pack-invoke-completed] WASM not advertised; skipping');
     }
 
     const create = await driver.post('/v1/runs', {

package/src/scenarios/wasm-pack-invoke-suspended.test.ts

@@ -17,6 +17,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { pollUntilTerminal } from '../lib/polling.js';
 import { isFixtureAdvertised } from '../lib/fixtures.js';
@@ -33,8 +34,8 @@ async function isWasmSupported(): Promise<boolean> {
 
 describe('wasm-pack-invoke-suspended: suspend → resume round-trip', () => {
   it('host either suspends the run or explicitly reports wasm_suspend_not_implemented', async () => {
-    if (!isFixtureAdvertised(FIXTURE)) return;
-    if (!(await isWasmSupported())) return;
+    if (!isFixtureAdvertised(FIXTURE)) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!isFixtureAdvertised(FIXTURE)` returned early');
+    if (!(await isWasmSupported())) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isWasmSupported())` returned early');
 
     // The reference rust-hello pack does NOT itself suspend (it always
     // returns `completed`), so against that pack this scenario can only

package/src/scenarios/wasm-pack-load.test.ts

@@ -13,6 +13,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { isFixtureAdvertised } from '../lib/fixtures.js';
 import { capabilityFamily } from '../lib/discovery-capabilities.js';
@@ -39,7 +40,7 @@ describe('wasm-pack-load: discovery surfaces WASM runtime support', () => {
     if (!wasm?.supported) {
       // eslint-disable-next-line no-console
       console.warn('[wasm-pack-load] host does not advertise WASM support; skipping');
-      return;
+      return softSkip('inapplicable', '[wasm-pack-load] host does not advertise WASM support; skipping');
     }
     expect(Array.isArray(wasm.abiVersions), driver.describe(
       'RFCS/0008-wasm-abi.md §H',
@@ -55,11 +56,11 @@ describe('wasm-pack-load: discovery surfaces WASM runtime support', () => {
 describe('wasm-pack-load: loaded pack typeIds are dispatchable', () => {
   it('host accepts a workflow whose node typeId is provided by a loaded WASM pack', async () => {
     const wasm = await getWasmCaps();
-    if (!wasm?.supported) return;
+    if (!wasm?.supported) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!wasm?.supported` returned early');
     if (!isFixtureAdvertised(FIXTURE)) {
       // eslint-disable-next-line no-console
       console.warn(`[wasm-pack-load] fixture ${FIXTURE} not advertised; skipping`);
-      return;
+      return softSkip('inapplicable', '[wasm-pack-load] fixture … not advertised; skipping');
     }
     // Creating a run against the fixture proves the host knows about the
     // WASM-provided typeId. A host that loaded the pack accepts the

package/src/scenarios/wasm-pack-memory-cap.test.ts

@@ -23,6 +23,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { pollUntilTerminal } from '../lib/polling.js';
 import { isFixtureAdvertised } from '../lib/fixtures.js';
@@ -36,7 +37,7 @@ describe('wasm-pack-memory-cap: host advertises maxMemoryBytes', () => {
     const wasm =
       capabilityFamily<{ wasm?: Record<string, unknown> }>(disco.json, 'nodePackRuntimes')?.wasm;
 
-    if (!wasm?.supported) return;
+    if (!wasm?.supported) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!wasm?.supported` returned early');
 
     // The cap is REQUIRED to enforce §K. Hosts MUST surface the configured
     // ceiling so clients can size their packs accordingly.
@@ -59,12 +60,12 @@ describe('wasm-pack-memory-cap: positive path via misbehaving pack', () => {
         `[wasm-pack-memory-cap] fixture ${CAP_BREACH_FIXTURE} not advertised; skipping positive path. ` +
           'Build the misbehaving pack at examples/packs/rust-misbehaving-memory/ and ensure the host serves the fixture.',
       );
-      return;
+      return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!isFixtureAdvertised(CAP_BREACH_FIXTURE)` returned early');
     }
     const disco = await driver.get('/.well-known/openwop');
     const wasm =
       capabilityFamily<{ wasm?: Record<string, unknown> }>(disco.json, 'nodePackRuntimes')?.wasm;
-    if (!wasm?.supported) return;
+    if (!wasm?.supported) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!wasm?.supported` returned early');
 
     const create = await driver.post('/v1/runs', {
       workflowId: CAP_BREACH_FIXTURE,

package/src/scenarios/wasm-pack-replay-determinism.test.ts

@@ -11,6 +11,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { pollUntilTerminal } from '../lib/polling.js';
 import { isFixtureAdvertised } from '../lib/fixtures.js';
@@ -33,8 +34,8 @@ function extractGreeting(events: Array<{ type: string; data?: unknown }>): strin
 
 describe('wasm-pack-replay-determinism: same inputs → same output', () => {
   it('two independent runs with same inputs produce same WASM output', async () => {
-    if (!isFixtureAdvertised(FIXTURE)) return;
-    if (!(await isWasmSupported())) return;
+    if (!isFixtureAdvertised(FIXTURE)) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!isFixtureAdvertised(FIXTURE)` returned early');
+    if (!(await isWasmSupported())) return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isWasmSupported())` returned early');
 
     const inputs = { name: 'determinism-probe' };
 

package/src/scenarios/webhook-signed-delivery.test.ts

@@ -24,6 +24,7 @@
  */
 
 import { afterEach, describe, expect, it } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { createHmac } from 'node:crypto';
 import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
 import { driver } from '../lib/driver.js';
@@ -78,12 +79,12 @@ describe('webhook-signed-delivery: end-to-end HMAC v1', () => {
     if (!(await isWebhookSupported())) {
       // eslint-disable-next-line no-console
       console.warn('[webhook-signed-delivery] host does not advertise webhook support; skipping');
-      return;
+      return softSkip('inapplicable', '[webhook-signed-delivery] host does not advertise webhook support; skipping');
     }
     if (!isFixtureAdvertised('conformance-noop')) {
       // eslint-disable-next-line no-console
       console.warn('[webhook-signed-delivery] conformance-noop not advertised; skipping');
-      return;
+      return softSkip('inapplicable', '[webhook-signed-delivery] conformance-noop not advertised; skipping');
     }
 
     const receiver = await startReceiver();
@@ -114,7 +115,7 @@ describe('webhook-signed-delivery: end-to-end HMAC v1', () => {
           '[webhook-signed-delivery] host SSRF guard rejected the loopback receiver; ' +
             'set OPENWOP_WEBHOOK_ALLOW_PRIVATE=true on the host (or equivalent) to run',
         );
-        return;
+        return softSkip('blocked', 'precondition not met — `body.error === \'webhook_url_rejected\'` returned early (seam, prior step, or fixture unavailable)');
       }
     }
 

package/src/scenarios/workflow-primary-output-annotation.test.ts

@@ -20,35 +20,25 @@
 import { describe, it, expect } from 'vitest';
 import Ajv2020 from 'ajv/dist/2020.js';
 import addFormats from 'ajv-formats';
-import { readFileSync } from 'node:fs';
+import { readFileSync, readdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { SCHEMAS_DIR } from '../lib/paths.js';
 
 function compileWorkflowDefinition(): ReturnType<Ajv2020['compile']> {
   const ajv = new Ajv2020({ strict: false, allErrors: true });
   addFormats(ajv);
-  // Register cross-file `$ref` targets — same pattern as
-  // `fixtures-valid.test.ts`. Without these, Ajv throws
-  // `missingRef` when compiling `workflow-definition.schema.json`
-  // because it references agent-ref + prompt-ref by URL.
-  const agentRefSchema = JSON.parse(
-    readFileSync(join(SCHEMAS_DIR, 'agent-ref.schema.json'), 'utf8'),
-  ) as Record<string, unknown>;
-  const promptRefSchema = JSON.parse(
-    readFileSync(join(SCHEMAS_DIR, 'prompt-ref.schema.json'), 'utf8'),
-  ) as Record<string, unknown>;
-  const promptKindSchema = JSON.parse(
-    readFileSync(join(SCHEMAS_DIR, 'prompt-kind.schema.json'), 'utf8'),
-  ) as Record<string, unknown>;
-  ajv.addSchema(agentRefSchema, 'agent-ref.schema.json');
-  ajv.addSchema(promptRefSchema, 'prompt-ref.schema.json');
-  ajv.addSchema(promptRefSchema, './prompt-ref.schema.json');
-  ajv.addSchema(promptKindSchema, 'prompt-kind.schema.json');
-  ajv.addSchema(promptKindSchema, './prompt-kind.schema.json');
-  const schema = JSON.parse(
-    readFileSync(join(SCHEMAS_DIR, 'workflow-definition.schema.json'), 'utf8'),
-  ) as Record<string, unknown>;
-  return ajv.compile(schema);
+  // Register EVERY schema in the corpus (S14, 2026-08-16) — a fixed ref list is
+  // a claim about the schema graph that nothing keeps true; the fourth cross-file
+  // `$ref` (#1009) broke the previous list at describe time.
+  for (const file of readdirSync(SCHEMAS_DIR).filter((f) => f.endsWith('.schema.json'))) {
+    const s = JSON.parse(readFileSync(join(SCHEMAS_DIR, file), 'utf8')) as Record<string, unknown>;
+    ajv.addSchema(s, file);
+    ajv.addSchema(s, `./${file}`);
+  }
+  return (
+    ajv.getSchema('workflow-definition.schema.json') ??
+    ajv.compile(JSON.parse(readFileSync(join(SCHEMAS_DIR, 'workflow-definition.schema.json'), 'utf8')) as Record<string, unknown>)
+  );
 }
 
 /** Build the minimal-required shape of a WorkflowDefinition. Tests

package/src/scenarios/workspace-cross-tenant-isolation-blackbox.test.ts

@@ -24,6 +24,7 @@
  * @see SECURITY/invariants.yaml workspace-cross-tenant-isolation
  */
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { randomUUID } from 'node:crypto';
 import { driver } from '../lib/driver.js';
 import { capabilityFamily } from '../lib/discovery-capabilities.js';
@@ -42,11 +43,11 @@ const asTenantB = { headers: { Authorization: `Bearer ${tenantBKey ?? ''}` } };
 
 describe('workspace-cross-tenant-isolation (black-box): a §C file MUST NOT leak across owners (RFC 0059 §E WCT-1)', () => {
   it('a file written by owner A is unreadable + un-enumerable by a second-tenant credential', async () => {
-    if (!(await workspaceSupported())) return; // capability not advertised — skip
+    if (!(await workspaceSupported())) return softSkip('inapplicable', 'capability not advertised — skip');
     if (!tenantBKey) {
       // eslint-disable-next-line no-console
       console.warn('[workspace-cross-tenant-isolation-blackbox] OPENWOP_TEST_TENANT_B_API_KEY not supplied; skipping the production-path cross-tenant assertion');
-      return;
+      return softSkip('blocked', '[workspace-cross-tenant-isolation-blackbox] OPENWOP_TEST_TENANT_B_API_KEY not supplied; skipping the production-path cross-tenant assertion');
     }
 
     const path = `wct-blackbox-${randomUUID()}.md`;