@openwop/openwop-conformance 2.42.6 → 2.42.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +3 -3
- package/coverage.md +1 -1
- package/dist/lib/soft-skip.js +13 -0
- package/dist/spec-artifacts.lock.json +2 -2
- package/package.json +2 -2
- package/requirements.json +423 -160
- package/scenario-majors.json +16 -3
- package/schemas/CORPUS-STAMP.json +64 -63
- package/src/lib/audit-checkpoint.ts +72 -0
- package/src/lib/ext-claims.ts +96 -0
- package/src/lib/soft-skip.ts +14 -0
- package/src/scenarios/a2a-peer-authority.test.ts +6 -13
- package/src/scenarios/a2a-task-roundtrip.test.ts +6 -34
- package/src/scenarios/a2a-version-negotiation.test.ts +6 -12
- package/src/scenarios/a2ui-surface-delta-transport.test.ts +23 -12
- package/src/scenarios/agent-channel-dispatch.test.ts +13 -4
- package/src/scenarios/agent-live-invocation-bracket.test.ts +1 -1
- package/src/scenarios/agent-manifest-runtime.test.ts +3 -0
- package/src/scenarios/aiEnvelope.capBreached.test.ts +2 -2
- package/src/scenarios/aiEnvelope.contractRefusal.test.ts +2 -2
- package/src/scenarios/aiEnvelope.schemaDrift.test.ts +2 -2
- package/src/scenarios/aiEnvelope.universalKinds.test.ts +2 -2
- package/src/scenarios/aiproviders-selfhosted-honesty.test.ts +2 -2
- package/src/scenarios/append-ordering.test.ts +26 -21
- package/src/scenarios/artifact-auth.test.ts +3 -1
- package/src/scenarios/artifact-type-registration-source.test.ts +2 -2
- package/src/scenarios/audit-checkpoint-signature.test.ts +63 -0
- package/src/scenarios/audit-checkpoint-vectors.test.ts +93 -0
- package/src/scenarios/auth-mtls.test.ts +7 -6
- package/src/scenarios/auth-subject-link-key-class.test.ts +3 -3
- package/src/scenarios/auth-subject-link.test.ts +19 -18
- package/src/scenarios/blob-presign-expiry.test.ts +2 -2
- package/src/scenarios/byok-roundtrip.test.ts +4 -1
- package/src/scenarios/compensation-behavior.test.ts +10 -5
- package/src/scenarios/conformance-advertised-seam-required.test.ts +6 -0
- package/src/scenarios/connection-pack-apihosts.test.ts +2 -2
- package/src/scenarios/connection-provider-resolution.test.ts +2 -2
- package/src/scenarios/context-budget-transcript-bound.test.ts +7 -3
- package/src/scenarios/context-summarization-replay.test.ts +2 -0
- package/src/scenarios/cost-attribution.test.ts +3 -1
- package/src/scenarios/debugBundle.test.ts +12 -4
- package/src/scenarios/discovery.test.ts +6 -2
- package/src/scenarios/dispatch-cross-worker-handoff.test.ts +2 -2
- package/src/scenarios/dispatch-input-mapping.test.ts +2 -2
- package/src/scenarios/dispatch-output-mapping.test.ts +15 -2
- package/src/scenarios/distillation-token-budget.test.ts +2 -2
- package/src/scenarios/durability-poison-exhaustion.test.ts +2 -2
- package/src/scenarios/envelope-nl-to-format-engaged.test.ts +2 -2
- package/src/scenarios/envelope-recovery-applied.test.ts +2 -2
- package/src/scenarios/envelope-refusal-shape.test.ts +2 -2
- package/src/scenarios/envelope-retry-attempted.test.ts +2 -2
- package/src/scenarios/envelope-retry-exhausted.test.ts +2 -2
- package/src/scenarios/envelope-tier-one-subset-static.test.ts +1 -1
- package/src/scenarios/envelope-truncation-cap-exhaustion.test.ts +2 -2
- package/src/scenarios/eventOrdering.test.ts +17 -10
- package/src/scenarios/form-content-instantiation.test.ts +2 -2
- package/src/scenarios/heartbeat-idempotent-no-spam.test.ts +2 -2
- package/src/scenarios/highConcurrency.test.ts +9 -9
- package/src/scenarios/idempotency-concurrent-claim.test.ts +5 -5
- package/src/scenarios/idempotency-key-determinism.test.ts +8 -8
- package/src/scenarios/idempotency.test.ts +6 -6
- package/src/scenarios/idempotencyRetry.test.ts +16 -11
- package/src/scenarios/interrupt-token-matrix.test.ts +24 -11
- package/src/scenarios/mcp-cache-tenant-scope.test.ts +3 -3
- package/src/scenarios/mcp-current-auth-boundary.test.ts +9 -6
- package/src/scenarios/mcp-extension-opacity.test.ts +6 -5
- package/src/scenarios/mcp-mrtr-roundtrip.test.ts +9 -8
- package/src/scenarios/mcp-version-negotiation.test.ts +5 -10
- package/src/scenarios/media-url-inline-cap.test.ts +3 -2
- package/src/scenarios/memory-compaction-provenance-tag.test.ts +2 -0
- package/src/scenarios/memory-compaction-sr1-carry-forward.test.ts +2 -2
- package/src/scenarios/memory-consolidation-idempotent.test.ts +2 -2
- package/src/scenarios/memory-injection-budget.test.ts +2 -2
- package/src/scenarios/multi-party-conversation-behavioral.test.ts +2 -2
- package/src/scenarios/multi-region-idempotency-behavior.test.ts +9 -9
- package/src/scenarios/multi-region-idempotency.test.ts +2 -2
- package/src/scenarios/pack-registry-isolation.test.ts +13 -12
- package/src/scenarios/pack-registry.test.ts +3 -2
- package/src/scenarios/pause-resume.test.ts +30 -11
- package/src/scenarios/prompt-read-workspace-membership-enforced.test.ts +43 -34
- package/src/scenarios/provider-usage.test.ts +2 -2
- package/src/scenarios/redaction.test.ts +7 -5
- package/src/scenarios/redactionAdversarial.test.ts +5 -4
- package/src/scenarios/registry-public.test.ts +5 -1
- package/src/scenarios/replay-divergence-at-refusal.test.ts +10 -7
- package/src/scenarios/replay-fanout-suppression.test.ts +7 -4
- package/src/scenarios/replay-fork-arbitrary.test.ts +2 -2
- package/src/scenarios/replay-side-effect-suppression.test.ts +2 -2
- package/src/scenarios/replayDeterminism.test.ts +9 -4
- package/src/scenarios/runner-ledger.test.ts +4 -0
- package/src/scenarios/runtime-capabilities.test.ts +2 -0
- package/src/scenarios/speech-synthesis-roundtrip.test.ts +1 -1
- package/src/scenarios/streamReconnect.test.ts +2 -2
- package/src/scenarios/subrun-approval-gate.test.ts +2 -2
- package/src/scenarios/subworkflow-input-mapping.test.ts +2 -2
- package/src/scenarios/tool-catalog-projection.test.ts +10 -7
- package/src/scenarios/tool-session-lifecycle.test.ts +1 -1
- package/src/scenarios/trigger-ingestion.test.ts +2 -2
- package/src/scenarios/v2-a2ui-v09-surface.test.ts +15 -0
- package/src/scenarios/v2-effect-identity-business-key.test.ts +12 -0
- package/src/scenarios/v2-ext-family-claims.test.ts +62 -0
- package/src/scenarios/v2-ext-rest-transport.test.ts +151 -0
- package/src/scenarios/v2-idempotency-in-flight.test.ts +3 -0
- package/src/scenarios/v2-legacy-profiles-absent.test.ts +4 -6
- package/src/scenarios/v2-mcp-tasks.test.ts +1 -1
- package/src/scenarios/v2-min-client-version.test.ts +2 -0
- package/src/scenarios/v2-refresh-sla.test.ts +6 -5
- package/src/scenarios/v2-sse-last-event-id-cursor.test.ts +3 -3
- package/src/scenarios/v2-sse-last-event-id.test.ts +2 -0
- package/src/scenarios/v2-version-header-honored.test.ts +3 -0
- package/src/scenarios/v2-webhook-durable-delivery.test.ts +2 -0
- package/src/scenarios/v2-webhook-secret-rotation.test.ts +8 -1
- package/src/scenarios/verifier-gating.test.ts +2 -2
- package/src/scenarios/version-fold.test.ts +2 -2
- package/src/scenarios/wasm-pack-invoke-suspended.test.ts +7 -6
- package/src/scenarios/webhook-signed-delivery.test.ts +2 -0
- package/src/scenarios/workflow-chain-host-expansion.test.ts +10 -1
- package/src/scenarios/workflow-variable-format.test.ts +2 -2
- package/src/scenarios/workload-identity-chain-bounds.test.ts +6 -3
- package/vectors/audit-checkpoint-v1.json +260 -0
|
@@ -48,7 +48,7 @@ import { CANARIES, CANARY_MARKER, getCanary } from '../lib/canaries.js';
|
|
|
48
48
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
49
49
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
50
50
|
import { req } from '../lib/requirement-ids.js';
|
|
51
|
-
import {
|
|
51
|
+
import { blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
52
52
|
|
|
53
53
|
const NOOP_WORKFLOW_ID = 'conformance-noop';
|
|
54
54
|
const SKIP_NO_NOOP = !isFixtureAdvertised(NOOP_WORKFLOW_ID);
|
|
@@ -77,8 +77,9 @@ describe.skipIf(SKIP_NO_NOOP)('redaction-adversarial: workflow input canaries',
|
|
|
77
77
|
`/v1/runs/${encodeURIComponent(runId)}/events/poll`,
|
|
78
78
|
);
|
|
79
79
|
if (events.status !== 200) {
|
|
80
|
-
//
|
|
81
|
-
|
|
80
|
+
// events/poll is a core endpoint (rest-endpoints.md); without it the
|
|
81
|
+
// canary check below never runs, so the leg is blocked, not passed.
|
|
82
|
+
return blockedDespiteAssertions('GET /v1/runs/{runId}/events/poll did not answer 200 — the canary non-disclosure is unobserved');
|
|
82
83
|
}
|
|
83
84
|
|
|
84
85
|
const eventsText = events.text;
|
|
@@ -139,7 +140,7 @@ describe.skipIf(SKIP_NO_NOOP)('redaction-adversarial: bearer-shaped strings roun
|
|
|
139
140
|
const events = await driver.get(
|
|
140
141
|
`/v1/runs/${encodeURIComponent(runId)}/events/poll`,
|
|
141
142
|
);
|
|
142
|
-
if (events.status !== 200) return
|
|
143
|
+
if (events.status !== 200) return blockedDespiteAssertions('GET /v1/runs/{runId}/events/poll did not answer 200 — the canary non-disclosure is unobserved');
|
|
143
144
|
|
|
144
145
|
expect(events.text.includes(canary), req('openwop.it.redactionAdversarial.anthropic-shaped-canary-embedded-in-non-credential-input-must-not-appear-verbati',
|
|
145
146
|
'capabilities.md §"Secrets" + NFR-7',
|
|
@@ -215,7 +215,11 @@ describe('registry-public: tarball + signature + Ed25519 verify roundtrip', () =
|
|
|
215
215
|
// `method=ed25519` — the simpler path. Extending to `manual` would
|
|
216
216
|
// require the tarball extractor from registry/scripts/verify-
|
|
217
217
|
// signatures.mjs which is intentionally out of scope here.
|
|
218
|
-
|
|
218
|
+
// partial-witness-ok: tarball, signature, public key and SRI were asserted
|
|
219
|
+
// above. `manual` is a conforming signing method whose pack.json-in-tarball
|
|
220
|
+
// verification this scenario does not implement, a suite limitation, not a
|
|
221
|
+
// host shortfall.
|
|
222
|
+
if (method !== 'ed25519') return softSkip('inapplicable', `${PACK_NAME} is signed method=manual; this scenario verifies only method=ed25519 signatures`);
|
|
219
223
|
|
|
220
224
|
const verified = cryptoVerify(null, tarball.bytes, publicKey, sig.bytes);
|
|
221
225
|
expect(verified, req('openwop.it.registry-public.tarball-sig-public-key-all-retrievable-sri-matches-ed25519-verifies-for', 'spec/v1/registry-operations.md', `Ed25519 signature over ${PACK_NAME}@${PACK_VERSION}.tgz MUST verify against ${manifest.signing!.keyId}`))
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
*/
|
|
46
46
|
|
|
47
47
|
import { describe, it, expect } from 'vitest';
|
|
48
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
48
|
+
import { blockedDespiteAssertions, softSkip } from '../lib/soft-skip.js';
|
|
49
49
|
import { driver } from '../lib/driver.js';
|
|
50
50
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
51
51
|
import { req } from '../lib/requirement-ids.js';
|
|
@@ -246,9 +246,12 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
246
246
|
workflowId: 'conformance-phase4-replay-divergence',
|
|
247
247
|
});
|
|
248
248
|
if (createRes.status === 404 || createRes.status === 422) {
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
249
|
+
// The host advertises refusalDivergenceEmission and its mock seam took the
|
|
250
|
+
// program (asserted above), but the fixture run would not start: the
|
|
251
|
+
// requirement was never observed. ctx.skip() throws, so this note is the
|
|
252
|
+
// row's disposition (`blocked`), exactly as before 2026-09-27.
|
|
253
|
+
blockedDespiteAssertions('the mock-AI seam accepted the program but POST /v1/runs of conformance-phase4-replay-divergence answered 404/422 — the fixture is not seeded, so no divergence was driven');
|
|
254
|
+
return ctx.skip();
|
|
252
255
|
}
|
|
253
256
|
expect(createRes.status).toBe(201);
|
|
254
257
|
const sourceRunId = (createRes.json as { runId: string }).runId;
|
|
@@ -336,9 +339,9 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
336
339
|
workflowId: 'conformance-phase4-replay-divergence',
|
|
337
340
|
});
|
|
338
341
|
if (createRes.status === 404 || createRes.status === 422) {
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
return
|
|
342
|
+
// As in the first case: ctx.skip() throws, so the note is the disposition.
|
|
343
|
+
blockedDespiteAssertions('the mock-AI seam accepted the program but POST /v1/runs of conformance-phase4-replay-divergence answered 404/422 — the fixture is not seeded, so no divergence was driven');
|
|
344
|
+
return ctx.skip();
|
|
342
345
|
}
|
|
343
346
|
expect(createRes.status).toBe(201);
|
|
344
347
|
const sourceRunId = (createRes.json as { runId: string }).runId;
|
|
@@ -283,8 +283,8 @@ describe('replay-fanout-suppression: a replay fork MUST NOT fan out re-emitted e
|
|
|
283
283
|
'host does not advertise `replay.supported: true`, so a replay fork cannot occur and this MUST NOT '
|
|
284
284
|
+ 'has nothing to constrain on this host',
|
|
285
285
|
);
|
|
286
|
-
ctx.skip()
|
|
287
|
-
return
|
|
286
|
+
// recordRequirement above carries the disposition; ctx.skip() throws.
|
|
287
|
+
return ctx.skip();
|
|
288
288
|
}
|
|
289
289
|
|
|
290
290
|
// ── LEG 2 — THE MUST NOT. A replay fork re-emits; it must not deliver. ───
|
|
@@ -308,8 +308,8 @@ describe('replay-fanout-suppression: a replay fork MUST NOT fan out re-emitted e
|
|
|
308
308
|
`replay fork returned ${replay.status} — the re-emission this requirement is stated over never happened, `
|
|
309
309
|
+ 'so the absence of deliveries below would prove nothing',
|
|
310
310
|
);
|
|
311
|
-
ctx.skip()
|
|
312
|
-
return
|
|
311
|
+
// recordRequirement above carries the disposition; ctx.skip() throws.
|
|
312
|
+
return ctx.skip();
|
|
313
313
|
}
|
|
314
314
|
expect(replay.status, req('openwop.it.replay-fanout-suppression.delivers-for-a-live-run-suppresses-for-a-replay-fork-and-delivers-again-for-a-br', 'webhooks.md §"Register"', 'replay fork should be accepted')).toBe(201);
|
|
315
315
|
const replayRunId = (replay.json as { runId: string }).runId;
|
|
@@ -373,6 +373,9 @@ describe('replay-fanout-suppression: a replay fork MUST NOT fan out re-emitted e
|
|
|
373
373
|
});
|
|
374
374
|
if (branch.status === 501 || branch.status === 400) {
|
|
375
375
|
// branch not offered on this range — leg 2 still stands on its own.
|
|
376
|
+
// partial-witness-ok: the MUST NOT (leg 2) was observed and recorded
|
|
377
|
+
// executed-pass above; leg 3 only pins the out-of-scope boundary, on a
|
|
378
|
+
// fork mode the host may not offer.
|
|
376
379
|
return softSkip('blocked', 'precondition not met — `branch.status === 501 || branch.status === 400` returned early (branch not offered on this range — leg 2 still stands on its own.) (seam, prior step, or fixture unavailable)');
|
|
377
380
|
}
|
|
378
381
|
expect(branch.status, req('openwop.it.replay-fanout-suppression.delivers-for-a-live-run-suppresses-for-a-replay-fork-and-delivers-again-for-a-br', 'replay.md §"Host-initiated fan-out is an external effect"', 'branch fork should be accepted')).toBe(201);
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
|
|
31
31
|
import { describe, it, expect } from 'vitest';
|
|
32
32
|
import { driver } from '../lib/driver.js';
|
|
33
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
33
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
34
34
|
import { forkDeclined } from '../lib/fork-availability.js';
|
|
35
35
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
36
36
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
@@ -246,7 +246,7 @@ describe.skipIf(SKIP_NO_MULTI)(
|
|
|
246
246
|
);
|
|
247
247
|
if (forkDeclined(fork2.status, 'arbitrary-event fork 2')) {
|
|
248
248
|
ctx.skip();
|
|
249
|
-
return
|
|
249
|
+
return blockedDespiteAssertions('arbitrary-event fork 2 was declined — fork determinism is unobserved');
|
|
250
250
|
}
|
|
251
251
|
expect(fork2.status).toBe(201);
|
|
252
252
|
const fork2Id = (fork2.json as { runId: string }).runId;
|
|
@@ -55,7 +55,7 @@ import { readFileSync } from 'node:fs';
|
|
|
55
55
|
import { join } from 'node:path';
|
|
56
56
|
import { V1_DIR } from '../lib/paths.js';
|
|
57
57
|
import { req } from '../lib/requirement-ids.js';
|
|
58
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
58
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
59
59
|
|
|
60
60
|
const WORKFLOW_ID = 'conformance-replay-side-effect';
|
|
61
61
|
const EFFECT_NODE = 'effect';
|
|
@@ -178,7 +178,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('replay-side-effect-suppression: a replay does
|
|
|
178
178
|
});
|
|
179
179
|
if (forkDeclined(fork.status, 'side-effect-suppression replay fork')) {
|
|
180
180
|
ctx.skip();
|
|
181
|
-
return
|
|
181
|
+
return blockedDespiteAssertions('the replay fork was declined — side-effect suppression is unobserved');
|
|
182
182
|
}
|
|
183
183
|
expect(fork.status, req('openwop.it.replay-side-effect-suppression.fails-a-side-effecting-node-closed-with-replay-source-missing-when-the-source-ne', 'replay.md §"Side-effect suppression in replay" requirement 1', 'fork should be accepted')).toBe(201);
|
|
184
184
|
const forkRunId = (fork.json as { runId: string }).runId;
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
|
|
19
19
|
import { describe, it, expect } from 'vitest';
|
|
20
20
|
import { driver } from '../lib/driver.js';
|
|
21
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
21
|
+
import { blockedDespiteAssertions, softSkip } from '../lib/soft-skip.js';
|
|
22
22
|
import { forkDeclined } from '../lib/fork-availability.js';
|
|
23
23
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
24
24
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
@@ -121,7 +121,10 @@ describe.skipIf(SKIP_NO_NOOP)('replay-determinism: same fromSeq + same workflow
|
|
|
121
121
|
mode: 'replay',
|
|
122
122
|
fromSeq: 0,
|
|
123
123
|
});
|
|
124
|
-
|
|
124
|
+
// Fork 1 of the same run was accepted (asserted above), so a decline here
|
|
125
|
+
// leaves the determinism comparison unobserved on a host that can fork:
|
|
126
|
+
// `blocked`, not the partial-witness pass a plain softSkip records.
|
|
127
|
+
if (forkDeclined(fork2.status, 'determinism fork 2')) return blockedDespiteAssertions(`the second replay fork of ${originalRunId} was declined (${fork2.status}) after the first was accepted — there is no second event list to compare`);
|
|
125
128
|
expect(fork2.status).toBe(201);
|
|
126
129
|
const fork2Id = (fork2.json as { runId: string }).runId;
|
|
127
130
|
await pollUntilTerminal(fork2Id, { timeoutMs: 10_000 });
|
|
@@ -129,11 +132,13 @@ describe.skipIf(SKIP_NO_NOOP)('replay-determinism: same fromSeq + same workflow
|
|
|
129
132
|
// Phase 4: fetch both fork event streams.
|
|
130
133
|
const fork1Events = await driver.get(`/v1/runs/${encodeURIComponent(fork1Id)}/events/poll`);
|
|
131
134
|
const fork2Events = await driver.get(`/v1/runs/${encodeURIComponent(fork2Id)}/events/poll`);
|
|
132
|
-
|
|
135
|
+
// rest-endpoints.md §"Required endpoints": /events/poll is a MUST, so a
|
|
136
|
+
// conforming host always makes the two event lists readable.
|
|
137
|
+
if (fork1Events.status !== 200 || fork2Events.status !== 200) return blockedDespiteAssertions(`GET /events/poll for the forks answered ${fork1Events.status} / ${fork2Events.status} — the event lists were never compared`);
|
|
133
138
|
|
|
134
139
|
const fork1Body = fork1Events.json as { events?: RawEvent[] };
|
|
135
140
|
const fork2Body = fork2Events.json as { events?: RawEvent[] };
|
|
136
|
-
if (!fork1Body.events || !fork2Body.events) return
|
|
141
|
+
if (!fork1Body.events || !fork2Body.events) return blockedDespiteAssertions('a fork /events/poll body carried no events[] — the event lists were never compared');
|
|
137
142
|
|
|
138
143
|
// Phase 5: assert structural identity (modulo timestamps + IDs).
|
|
139
144
|
expect(fork1Body.events.length, req('openwop.it.replayDeterminism.two-replay-forks-of-the-same-point-produce-structurally-identical-event-lists',
|
|
@@ -251,10 +251,14 @@ describe('RFC 0148 §A (S6) — the runner derivation', () => {
|
|
|
251
251
|
describe('RFC 0148 §A — softSkip notes the reason for an early return, per file', () => {
|
|
252
252
|
it('notes are keyed to the current test file, worst-first when mixed, and read back with joined reasons', () => {
|
|
253
253
|
resetSoftSkips();
|
|
254
|
+
// partial-witness-ok: not a skip — this leg unit-tests softSkip itself, and
|
|
255
|
+
// resetSoftSkips() below clears every note it writes before the leg ends.
|
|
254
256
|
expect(softSkip('inapplicable', 'host does not advertise X'), req('openwop.it.runner-ledger.notes-are-keyed-to-the-current-test-file-worst-first-when-mixed-and-read-back-wi', 'RFC 0148 §A', 'notes are keyed to the current test file, worst-first when mixed, and read back with joined reasons')).toBeUndefined();
|
|
257
|
+
// partial-witness-ok: not a skip — the subject under test (de-duplication).
|
|
255
258
|
softSkip('inapplicable', 'host does not advertise X'); // de-duplicated
|
|
256
259
|
let d = softSkipDisposition('runner-ledger.test.ts');
|
|
257
260
|
expect(d).toEqual({ kind: 'inapplicable', reason: 'host does not advertise X' });
|
|
261
|
+
// partial-witness-ok: not a skip — the subject under test (worst-first fold).
|
|
258
262
|
softSkip('blocked', 'seam not mounted');
|
|
259
263
|
d = softSkipDisposition('runner-ledger.test.ts');
|
|
260
264
|
expect(d?.kind).toBe('blocked');
|
|
@@ -52,6 +52,8 @@ describe('runtime-capabilities: /.well-known/openwop forward-compat shape', () =
|
|
|
52
52
|
// of v1 hosts will omit it. Assertion passes trivially; don't
|
|
53
53
|
// force a value on a host that doesn't advertise any.
|
|
54
54
|
expect(caps).toBeUndefined();
|
|
55
|
+
// partial-witness-ok: discovery answered 200 and runtimeCapabilities is OPTIONAL; when
|
|
56
|
+
// absent, the IF-present MUSTs below bind nothing.
|
|
55
57
|
return softSkip('inapplicable', 'capability or profile not advertised by this host — gate `caps === undefined` returned early');
|
|
56
58
|
}
|
|
57
59
|
|
|
@@ -58,7 +58,7 @@ describe('speech-synthesis-roundtrip (RFC 0105 §A)', () => {
|
|
|
58
58
|
audio !== undefined,
|
|
59
59
|
req('openwop.it.speech-synthesis-roundtrip.synthesizes-an-audio-asset-with-exactly-one-of-url-base64-and-echoes-the-voiceid', 'RFC 0105 §A', 'the response MUST carry an `audio` object'),
|
|
60
60
|
).toBe(true);
|
|
61
|
-
if (!audio)
|
|
61
|
+
if (!audio) throw new Error('unreachable: the assertion above has already failed this leg');
|
|
62
62
|
|
|
63
63
|
const hasUrl = typeof audio.url === 'string' && (audio.url as string).length > 0;
|
|
64
64
|
const hasBase64 = typeof audio.base64 === 'string' && (audio.base64 as string).length > 0;
|
|
@@ -22,7 +22,7 @@ import { subscribe, type SseEvent } from '../lib/sse.js';
|
|
|
22
22
|
import { pollUntilTerminal } from '../lib/polling.js';
|
|
23
23
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
24
24
|
import { req } from '../lib/requirement-ids.js';
|
|
25
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
25
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
26
26
|
|
|
27
27
|
const WORKFLOW_ID = 'conformance-cancellable';
|
|
28
28
|
const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
|
|
@@ -84,7 +84,7 @@ describe.skipIf(SKIP_NO_FIXTURE)('stream-reconnect: Last-Event-ID resume per spe
|
|
|
84
84
|
// because the run already completed and the server closed before we
|
|
85
85
|
// got events. Skip the rest of this scenario; the host is fast enough
|
|
86
86
|
// that the reconnect path doesn't apply.
|
|
87
|
-
return
|
|
87
|
+
return blockedDespiteAssertions('the first SSE connection yielded no event with a parseable sequence within 1s — the Last-Event-ID resume is unobserved');
|
|
88
88
|
}
|
|
89
89
|
|
|
90
90
|
// Phase 3: reconnect with Last-Event-ID set to the last seq we saw.
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import { describe, it, expect } from 'vitest';
|
|
13
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
13
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
14
14
|
import { readSubRunAttestationCap, invokeSubRunAttest } from '../lib/subRunAttestation.js';
|
|
15
15
|
import { req } from '../lib/requirement-ids.js';
|
|
16
16
|
|
|
@@ -27,7 +27,7 @@ describe('subrun-approval-gate (RFC 0063 §C)', () => {
|
|
|
27
27
|
).toBe(true);
|
|
28
28
|
|
|
29
29
|
const rejected = await invokeSubRunAttest({ ...base, approvalAction: 'reject' });
|
|
30
|
-
if (rejected === null) return
|
|
30
|
+
if (rejected === null) return blockedDespiteAssertions('the sub-run attestation seam answered accept but not reject — the no-merge-on-reject rule is unobserved');
|
|
31
31
|
expect(
|
|
32
32
|
rejected.merged,
|
|
33
33
|
req('openwop.it.subrun-approval-gate.accept-merges-the-child-outputs-reject-does-not', 'RFC 0063 §C', 'a `reject` approval MUST NOT merge the child outputs'),
|
|
@@ -24,7 +24,7 @@ import { pollUntilTerminal, pollUntilStatus } from '../lib/polling.js';
|
|
|
24
24
|
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
25
25
|
import { setHostCapability, resetHostCapabilities, isToggleAvailable } from '../lib/host-toggle.js';
|
|
26
26
|
import { req } from '../lib/requirement-ids.js';
|
|
27
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
27
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
28
28
|
|
|
29
29
|
const PARENT = 'conformance-subworkflow-input-mapping';
|
|
30
30
|
const CHILD = 'conformance-subworkflow-input-mapping-child';
|
|
@@ -102,7 +102,7 @@ describe.skipIf(SKIP)('subworkflow-input-mapping: parent → child variable seed
|
|
|
102
102
|
const subwfCompleted = events.find(
|
|
103
103
|
(e) => e.type === 'node.completed' && e.nodeId === 'subwf-call',
|
|
104
104
|
);
|
|
105
|
-
if (!subwfCompleted) return
|
|
105
|
+
if (!subwfCompleted) return blockedDespiteAssertions('the advertised fixture\'s parent run has no node.completed for subwf-call — the unset-variable projection is unobserved');
|
|
106
106
|
const childRunId = subwfCompleted.payload?.outputs?.childRunId;
|
|
107
107
|
|
|
108
108
|
const childRes = await driver.get(`/v1/runs/${encodeURIComponent(childRunId!)}`);
|
|
@@ -84,13 +84,6 @@ describe('tool-catalog-projection (RFC 0078 §B/§F)', () => {
|
|
|
84
84
|
if (!(await toolCatalogGate('openwop-tool-catalog'))) return;
|
|
85
85
|
const validate = descriptorValidator();
|
|
86
86
|
|
|
87
|
-
// ---- Leg 3: auth-gated (unauthenticated list MUST be 401) -------------
|
|
88
|
-
const unauth = await driver.get(toolsPath(), { authenticated: false });
|
|
89
|
-
expect(
|
|
90
|
-
unauth.status === 401,
|
|
91
|
-
req('openwop.it.tool-catalog-projection.lists-schema-valid-tooldescriptors-serves-by-id-404s-is-auth-gated-and-never-dis', 'tool-catalog.md §B', `GET ${toolsPath()} MUST require authentication (401 unauthenticated)`),
|
|
92
|
-
).toBe(true);
|
|
93
|
-
|
|
94
87
|
// ---- Leg 1: the list (§B) -------------------------------------------
|
|
95
88
|
// unfailable-leg audit wave 2, 2026-09-27: a host that ADVERTISES the
|
|
96
89
|
// catalog but 404s the list (or answers 500 / a non-list 200, which the
|
|
@@ -99,7 +92,17 @@ describe('tool-catalog-projection (RFC 0078 §B/§F)', () => {
|
|
|
99
92
|
// fails under OPENWOP_REQUIRE_BEHAVIOR=true), and any other non-200 or
|
|
100
93
|
// non-list body FAILS.
|
|
101
94
|
const read = await listTools();
|
|
95
|
+
// The unserved check runs before any assertion: after the 401 assertion below,
|
|
96
|
+
// seamAbsent's `blocked` note recorded a partial-witness pass at major 1.
|
|
102
97
|
if (read.unserved) return seamAbsent(`host advertises toolCatalog but ${toolsPath()} answered ${read.status}`);
|
|
98
|
+
|
|
99
|
+
// ---- Leg 3: auth-gated (unauthenticated list MUST be 401) -------------
|
|
100
|
+
const unauth = await driver.get(toolsPath(), { authenticated: false });
|
|
101
|
+
expect(
|
|
102
|
+
unauth.status === 401,
|
|
103
|
+
req('openwop.it.tool-catalog-projection.lists-schema-valid-tooldescriptors-serves-by-id-404s-is-auth-gated-and-never-dis', 'tool-catalog.md §B', `GET ${toolsPath()} MUST require authentication (401 unauthenticated)`),
|
|
104
|
+
).toBe(true);
|
|
105
|
+
|
|
103
106
|
expect(
|
|
104
107
|
read.status,
|
|
105
108
|
req('openwop.it.tool-catalog-projection.lists-schema-valid-tooldescriptors-serves-by-id-404s-is-auth-gated-and-never-dis', 'tool-catalog.md §B', `GET ${toolsPath()} MUST return 200 to an authenticated caller on a host advertising toolCatalog`),
|
|
@@ -50,7 +50,7 @@ describe('tool-session-lifecycle (RFC 0078 §D)', () => {
|
|
|
50
50
|
opened.length >= 1 && closed.length >= 1,
|
|
51
51
|
req('openwop.it.tool-session-lifecycle.brackets-the-call-events-with-tool-session-opened-first-closed-last-one-sessioni', 'tool-catalog.md §D', 'a tool session MUST emit tool.session.opened + tool.session.closed'),
|
|
52
52
|
).toBe(true);
|
|
53
|
-
if (opened.length === 0 || closed.length === 0)
|
|
53
|
+
if (opened.length === 0 || closed.length === 0) throw new Error('unreachable: the assertion above has already failed this leg');
|
|
54
54
|
|
|
55
55
|
const open = opened[0]!;
|
|
56
56
|
const close = closed[closed.length - 1]!;
|
|
@@ -48,7 +48,7 @@ import { driver } from '../lib/driver.js';
|
|
|
48
48
|
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
49
49
|
import { readCapabilityFamily } from '../lib/discovery-capabilities.js';
|
|
50
50
|
import { req } from '../lib/requirement-ids.js';
|
|
51
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
51
|
+
import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
52
52
|
|
|
53
53
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
54
54
|
|
|
@@ -213,7 +213,7 @@ describe.skipIf(HTTP_SKIP)('trigger-ingestion: behavioral ingestion + SSRF (capa
|
|
|
213
213
|
verification: { mode: 'none' },
|
|
214
214
|
webhook: { method: 'POST', headers: { Authorization: 'Bearer canary' }, body: { x: 1 } },
|
|
215
215
|
});
|
|
216
|
-
if (del.status === 404 || del.status === 403) return
|
|
216
|
+
if (del.status === 404 || del.status === 403) return blockedDespiteAssertions('the ingest seam answered the SSRF probe but not the webhook delivery — ingestion content redaction is unobserved');
|
|
217
217
|
const delBody = del.json as
|
|
218
218
|
| { deliveryEvent?: Record<string, unknown>; triggerEvent?: { webhook?: { headers?: Record<string, string> } } }
|
|
219
219
|
| undefined;
|
|
@@ -306,3 +306,18 @@ describe('RFC 0209 §C.9 — no render before root (declared unwitnessable here;
|
|
|
306
306
|
return softSkip('inapplicable', 'unwitnessable by this suite (RFC 0209 §C.9 Falsifiability verdict): a render-side guarantee, and a server-oriented suite cannot observe a renderer. The fold guard the server CAN observe is openwop.requirement.0209.fold-guarded, above. The render half is witnessed as a reference-app client probe (tier: reference-impl, the a2ui-surface-no-code-exec precedent): openwop/openwop-app frontend/react/src/chat/a2ui/__tests__/a2ui-v09-render-needs-root.test.tsx (openwop-app#4121, ADR 0749). Removing the `if (!state.renderable)` guard in frontend/react/src/chat/a2ui/v09/A2uiV09Surface.tsx turns 2 of its 4 cases red. This row records that the suite reached the requirement, and it is not a pass.');
|
|
307
307
|
});
|
|
308
308
|
});
|
|
309
|
+
|
|
310
|
+
describe('RFC 0220 §C — the a2uiSurface family witness (seam-gated)', () => {
|
|
311
|
+
it('a version-2 surface is admitted and a version-1 body under schema version 2 is refused, in the same run', async () => {
|
|
312
|
+
const g = await gate((f) => f >= 2, 'a floor of at least 2');
|
|
313
|
+
if (!g.ok) return softSkip(g.kind, g.reason);
|
|
314
|
+
const { runId } = g.ctx;
|
|
315
|
+
const R = (why: string) => req('openwop.family.a2uiSurface', 'spec/v2/ext/a2uiSurface/README.md §Conformance', why);
|
|
316
|
+
try {
|
|
317
|
+
const good = await emit(runId, envelope(2, v2(sid('fam-good'), (POSITIVE['messages'] as Json[]))));
|
|
318
|
+
expect(good.status, R(`the positive version-2 surface MUST be admitted (201), got ${code(good)}`)).toBe(201);
|
|
319
|
+
const mixed = await emit(runId, envelope(2, V1_SURFACE));
|
|
320
|
+
expect(code(mixed), R('a version-1 body under schema version 2 MUST be refused with envelope_invalid (control: a host that admits everything fails here)')).toBe('422 envelope_invalid');
|
|
321
|
+
} finally { await done(runId); }
|
|
322
|
+
});
|
|
323
|
+
});
|
|
@@ -29,6 +29,15 @@ import { softSkip, blockedDespiteAssertions } from '../lib/soft-skip.js';
|
|
|
29
29
|
import { startEffectReceiver } from '../lib/effect-receiver.js';
|
|
30
30
|
import { req } from '../lib/requirement-ids.js';
|
|
31
31
|
|
|
32
|
+
/**
|
|
33
|
+
* The retry leg hands the host the address of the suite's OWN receiver
|
|
34
|
+
* (`lib/effect-receiver.ts`, since 2.42.4) and the host calls it — a connection
|
|
35
|
+
* the host originates back to the harness. `host-callback-declaration` requires
|
|
36
|
+
* that to be declared; 2.42.4 added the receiver and not the declaration, so
|
|
37
|
+
* the self-check failed on every host that runs the full catalog.
|
|
38
|
+
*/
|
|
39
|
+
export const REQUIRES_HOST_CALLBACK = 'the host makes the retried outbound effect call to the suite-owned effect receiver (OPENWOP_WEBHOOK_RECEIVER_PORT)';
|
|
40
|
+
|
|
32
41
|
const FIXTURE = 'conformance-noop';
|
|
33
42
|
const KEYING = ['business-identity', 'activity-recipe'];
|
|
34
43
|
const TERMINAL = new Set(['completed', 'failed', 'cancelled']);
|
|
@@ -73,6 +82,9 @@ describe('RFC 0173 §B — effect-identity-business-key (gated on idempotency)',
|
|
|
73
82
|
const body = res.json as { runId?: unknown; effects?: Array<{ effectId?: unknown; keying?: unknown; providerKey?: unknown }> } | null;
|
|
74
83
|
const effects = body?.effects ?? [];
|
|
75
84
|
if (res.status === 200 && check.ok && body?.runId === runId && effects.length === 0) {
|
|
85
|
+
// partial-witness-ok: no assertion has passed at runtime. The only earlier
|
|
86
|
+
// expect runs solely when the create is not 201, and then it throws, so this
|
|
87
|
+
// skip is a zero-assertion `inapplicable` (the gate reads source order).
|
|
76
88
|
return softSkip('inapplicable', 'the noop fixture issued no external effect — the ledger read is well-formed but the per-row keying leg had no rows (an effect-issuing fixture would exercise it)');
|
|
77
89
|
}
|
|
78
90
|
expect(
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v2-ext-family-claims — RFC 0220 §B, the `claims-check` witness for the 11
|
|
3
|
+
* discovery-only extension families (suite 2.42.7, target major 2).
|
|
4
|
+
*
|
|
5
|
+
* Each family is a reservation: v2 defines no portable operation for it, so the
|
|
6
|
+
* only thing a server-oriented suite can check is the claim itself. A host that
|
|
7
|
+
* advertises `extensions["<org>.<extensionName>"]` records `executed-pass` under
|
|
8
|
+
* `openwop.family.<key>` when the claim is well formed (lib/ext-claims.ts), and
|
|
9
|
+
* `executed-fail` when it is not. A host that does not advertise the family
|
|
10
|
+
* records `inapplicable`.
|
|
11
|
+
*
|
|
12
|
+
* Why not behaviorGate: an ext family is outside every profile
|
|
13
|
+
* (spec/v2/ext/README.md), so strict mode has nothing to demand here — a host
|
|
14
|
+
* that serves no `kanban` has made no promise about one. The gate is the claim.
|
|
15
|
+
*
|
|
16
|
+
* Every req() id is a literal so the requirement registry collects it. That a
|
|
17
|
+
* leg exists for every claims-check family in the declaration is a corpus fact,
|
|
18
|
+
* checked by the self-test in src/lib/ext-claims.test.ts, not by a host run.
|
|
19
|
+
*
|
|
20
|
+
* @see RFCS/0220-ext-families-graduate-on-evidence.md
|
|
21
|
+
* @see spec/v2/ext/README.md
|
|
22
|
+
* @see spec/v2/core/capabilities.md §3.2
|
|
23
|
+
*/
|
|
24
|
+
import { describe, it, expect } from 'vitest';
|
|
25
|
+
import { v2Discovery } from '../lib/v2.js';
|
|
26
|
+
import { extDeclaration, classifyExtClaim } from '../lib/ext-claims.js';
|
|
27
|
+
import { softSkip } from '../lib/soft-skip.js';
|
|
28
|
+
import { req } from '../lib/requirement-ids.js';
|
|
29
|
+
|
|
30
|
+
const LEGS = ['brand', 'canvas', 'chat', 'coordination', 'dataIntegration', 'entities', 'kanban', 'knowledge', 'launchStudio', 'messaging', 'webResearch'] as const;
|
|
31
|
+
|
|
32
|
+
async function leg(key: (typeof LEGS)[number], cite: (why: string) => string): Promise<undefined> {
|
|
33
|
+
const decl = extDeclaration();
|
|
34
|
+
if (!decl) return softSkip('blocked', 'spec/v2/declaration.json is not resolvable in this layout — the extension name and the org registry come from it, and guessing either would make the suite the registry (conformance.md §Whose fact is the reason?)');
|
|
35
|
+
const family = decl.families.find((f) => f.key === key);
|
|
36
|
+
if (!family) return softSkip('blocked', `the declaration has no ext row for ${key} — this leg is stale against the corpus`);
|
|
37
|
+
const doc = await v2Discovery();
|
|
38
|
+
if (!doc) return softSkip('blocked', 'v2 discovery unreachable');
|
|
39
|
+
const claims = classifyExtClaim(doc, family, decl);
|
|
40
|
+
if (claims.length === 1 && claims[0].state === 'absent') return softSkip('inapplicable', `no extensions["<org>.${family.extensionName}"] record under a registered org — the host does not advertise ${key}`);
|
|
41
|
+
for (const c of claims) {
|
|
42
|
+
if (c.state === 'absent') continue;
|
|
43
|
+
expect(c.state, cite(c.state === 'malformed' ? `extensions["${c.key}"]: ${c.why}` : `extensions["${c.key}"] is a well-formed claim`)).toBe('well-formed');
|
|
44
|
+
}
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const S = (key: string): string => `spec/v2/ext/${key}/README.md §Conformance`;
|
|
49
|
+
|
|
50
|
+
describe('RFC 0220 §B — an extension family is claimed in a well-formed extensions record (claims-check)', () => {
|
|
51
|
+
it('brand', () => leg('brand', (why) => req('openwop.family.brand', S('brand'), why)));
|
|
52
|
+
it('canvas', () => leg('canvas', (why) => req('openwop.family.canvas', S('canvas'), why)));
|
|
53
|
+
it('chat', () => leg('chat', (why) => req('openwop.family.chat', S('chat'), why)));
|
|
54
|
+
it('coordination', () => leg('coordination', (why) => req('openwop.family.coordination', S('coordination'), why)));
|
|
55
|
+
it('dataIntegration', () => leg('dataIntegration', (why) => req('openwop.family.dataIntegration', S('dataIntegration'), why)));
|
|
56
|
+
it('entities', () => leg('entities', (why) => req('openwop.family.entities', S('entities'), why)));
|
|
57
|
+
it('kanban', () => leg('kanban', (why) => req('openwop.family.kanban', S('kanban'), why)));
|
|
58
|
+
it('knowledge', () => leg('knowledge', (why) => req('openwop.family.knowledge', S('knowledge'), why)));
|
|
59
|
+
it('launchStudio', () => leg('launchStudio', (why) => req('openwop.family.launchStudio', S('launchStudio'), why)));
|
|
60
|
+
it('messaging', () => leg('messaging', (why) => req('openwop.family.messaging', S('messaging'), why)));
|
|
61
|
+
it('webResearch', () => leg('webResearch', (why) => req('openwop.family.webResearch', S('webResearch'), why)));
|
|
62
|
+
});
|