@openwop/openwop-conformance 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +241 -0
  3. package/api/asyncapi.yaml +481 -0
  4. package/api/openapi.yaml +830 -0
  5. package/api/redocly.yaml +8 -0
  6. package/coverage.md +80 -0
  7. package/dist/cli.js +161 -0
  8. package/fixtures/conformance-a2a-task-roundtrip.json +27 -0
  9. package/fixtures/conformance-agent-identity.json +27 -0
  10. package/fixtures/conformance-agent-low-confidence.json +29 -0
  11. package/fixtures/conformance-agent-memory-cross-tenant.json +28 -0
  12. package/fixtures/conformance-agent-memory-redaction.json +32 -0
  13. package/fixtures/conformance-agent-memory-roundtrip.json +32 -0
  14. package/fixtures/conformance-agent-memory-ttl.json +31 -0
  15. package/fixtures/conformance-agent-pack-export.json +26 -0
  16. package/fixtures/conformance-agent-pack-install.json +26 -0
  17. package/fixtures/conformance-agent-pack-provenance.json +31 -0
  18. package/fixtures/conformance-agent-reasoning.json +29 -0
  19. package/fixtures/conformance-approval.json +27 -0
  20. package/fixtures/conformance-cancellable.json +33 -0
  21. package/fixtures/conformance-cap-breach.json +27 -0
  22. package/fixtures/conformance-capability-missing.json +23 -0
  23. package/fixtures/conformance-channel-ttl.json +60 -0
  24. package/fixtures/conformance-clarification.json +30 -0
  25. package/fixtures/conformance-conversation-capability-negotiation.json +23 -0
  26. package/fixtures/conformance-conversation-lifecycle.json +32 -0
  27. package/fixtures/conformance-conversation-replay.json +33 -0
  28. package/fixtures/conformance-conversation-vs-clarification.json +26 -0
  29. package/fixtures/conformance-delay.json +33 -0
  30. package/fixtures/conformance-dispatch-loop.json +38 -0
  31. package/fixtures/conformance-failure.json +23 -0
  32. package/fixtures/conformance-idempotent.json +30 -0
  33. package/fixtures/conformance-identity.json +32 -0
  34. package/fixtures/conformance-interrupt-auth-required.json +28 -0
  35. package/fixtures/conformance-interrupt-external-event.json +33 -0
  36. package/fixtures/conformance-interrupt-parent-child-cancel-child.json +27 -0
  37. package/fixtures/conformance-interrupt-parent-child-cancel.json +26 -0
  38. package/fixtures/conformance-interrupt-quorum.json +30 -0
  39. package/fixtures/conformance-mcp-tool-roundtrip.json +32 -0
  40. package/fixtures/conformance-message-reducer.json +31 -0
  41. package/fixtures/conformance-multi-node.json +21 -0
  42. package/fixtures/conformance-noop.json +23 -0
  43. package/fixtures/conformance-orchestrator-dispatch.json +47 -0
  44. package/fixtures/conformance-orchestrator-low-confidence.json +41 -0
  45. package/fixtures/conformance-orchestrator-terminate.json +44 -0
  46. package/fixtures/conformance-stream-text.json +26 -0
  47. package/fixtures/conformance-subworkflow-child.json +21 -0
  48. package/fixtures/conformance-subworkflow-parent.json +49 -0
  49. package/fixtures/conformance-version-fold.json +23 -0
  50. package/fixtures/conformance-wasm-pack-roundtrip.json +25 -0
  51. package/fixtures/pack-manifests/pack-private-example.json +26 -0
  52. package/fixtures.md +404 -0
  53. package/package.json +48 -0
  54. package/schemas/README.md +75 -0
  55. package/schemas/agent-manifest.schema.json +107 -0
  56. package/schemas/agent-ref.schema.json +53 -0
  57. package/schemas/capabilities.schema.json +287 -0
  58. package/schemas/channel-written-payload.schema.json +55 -0
  59. package/schemas/conversation-event.schema.json +120 -0
  60. package/schemas/conversation-turn.schema.json +72 -0
  61. package/schemas/debug-bundle.schema.json +196 -0
  62. package/schemas/dispatch-config.schema.json +46 -0
  63. package/schemas/error-envelope.schema.json +25 -0
  64. package/schemas/memory-entry.schema.json +36 -0
  65. package/schemas/memory-list-options.schema.json +21 -0
  66. package/schemas/node-pack-manifest.schema.json +235 -0
  67. package/schemas/orchestrator-decision.schema.json +60 -0
  68. package/schemas/run-event-payloads.schema.json +663 -0
  69. package/schemas/run-event.schema.json +116 -0
  70. package/schemas/run-options.schema.json +81 -0
  71. package/schemas/run-orchestrator-decided-event.schema.json +20 -0
  72. package/schemas/run-snapshot.schema.json +121 -0
  73. package/schemas/suspend-request.schema.json +182 -0
  74. package/schemas/workflow-definition.schema.json +430 -0
  75. package/src/cli.ts +187 -0
  76. package/src/lib/a2a-fake-peer.ts +233 -0
  77. package/src/lib/canaries.ts +186 -0
  78. package/src/lib/driver.ts +96 -0
  79. package/src/lib/env.ts +49 -0
  80. package/src/lib/fixtures.ts +93 -0
  81. package/src/lib/mcp-fake-server.ts +185 -0
  82. package/src/lib/multi-agent-capabilities.ts +155 -0
  83. package/src/lib/multiProcess.ts +141 -0
  84. package/src/lib/otel-collector.ts +312 -0
  85. package/src/lib/paths.ts +198 -0
  86. package/src/lib/polling.ts +81 -0
  87. package/src/lib/profiles.ts +258 -0
  88. package/src/lib/sse.ts +172 -0
  89. package/src/scenarios/a2a-task-roundtrip.test.ts +149 -0
  90. package/src/scenarios/agentConfidenceEscalation.test.ts +61 -0
  91. package/src/scenarios/agentMemoryCrossTenantIsolation.test.ts +54 -0
  92. package/src/scenarios/agentMemoryRedactionContract.test.ts +46 -0
  93. package/src/scenarios/agentMemoryRoundTrip.test.ts +52 -0
  94. package/src/scenarios/agentMemoryTtlExpiry.test.ts +47 -0
  95. package/src/scenarios/agentMessageReducer.test.ts +57 -0
  96. package/src/scenarios/agentMetadata.test.ts +56 -0
  97. package/src/scenarios/agentPackExport.test.ts +45 -0
  98. package/src/scenarios/agentPackInstall.test.ts +50 -0
  99. package/src/scenarios/agentPackProvenance.test.ts +53 -0
  100. package/src/scenarios/agentReasoningEvents.test.ts +72 -0
  101. package/src/scenarios/append-ordering.test.ts +91 -0
  102. package/src/scenarios/approval-payload.test.ts +120 -0
  103. package/src/scenarios/audit-log-integrity.test.ts +106 -0
  104. package/src/scenarios/auth.test.ts +55 -0
  105. package/src/scenarios/byok-roundtrip.test.ts +166 -0
  106. package/src/scenarios/cancellation.test.ts +68 -0
  107. package/src/scenarios/cap-breach.test.ts +149 -0
  108. package/src/scenarios/channel-ttl.test.ts +70 -0
  109. package/src/scenarios/configurable-schema.test.ts +76 -0
  110. package/src/scenarios/conversationCapabilityNegotiation.test.ts +39 -0
  111. package/src/scenarios/conversationLifecycle.test.ts +64 -0
  112. package/src/scenarios/conversationReplayDeterminism.test.ts +52 -0
  113. package/src/scenarios/conversationVsLegacySuspend.test.ts +46 -0
  114. package/src/scenarios/cost-attribution.test.ts +207 -0
  115. package/src/scenarios/debugBundle.test.ts +222 -0
  116. package/src/scenarios/discovery.test.ts +147 -0
  117. package/src/scenarios/dispatchLoop.test.ts +52 -0
  118. package/src/scenarios/errors.test.ts +144 -0
  119. package/src/scenarios/eventOrdering.test.ts +144 -0
  120. package/src/scenarios/failure-path.test.ts +46 -0
  121. package/src/scenarios/fixtures-gating.test.ts +137 -0
  122. package/src/scenarios/fixtures-valid.test.ts +140 -0
  123. package/src/scenarios/highConcurrency.test.ts +263 -0
  124. package/src/scenarios/idempotency.test.ts +83 -0
  125. package/src/scenarios/idempotencyRetry.test.ts +130 -0
  126. package/src/scenarios/identity-passthrough.test.ts +54 -0
  127. package/src/scenarios/interrupt-approval.test.ts +97 -0
  128. package/src/scenarios/interrupt-auth-required-resume.test.ts +88 -0
  129. package/src/scenarios/interrupt-clarification.test.ts +45 -0
  130. package/src/scenarios/interrupt-external-event-correlation.test.ts +113 -0
  131. package/src/scenarios/interrupt-parent-child-cascade.test.ts +102 -0
  132. package/src/scenarios/interrupt-quorum-resolution.test.ts +97 -0
  133. package/src/scenarios/interruptRace.test.ts +176 -0
  134. package/src/scenarios/maliciousManifest.test.ts +154 -0
  135. package/src/scenarios/mcp-discoverability.test.ts +129 -0
  136. package/src/scenarios/mcp-tool-roundtrip.test.ts +149 -0
  137. package/src/scenarios/multi-node-ordering.test.ts +60 -0
  138. package/src/scenarios/multi-region-idempotency.test.ts +52 -0
  139. package/src/scenarios/orchestratorConservativePath.test.ts +63 -0
  140. package/src/scenarios/orchestratorDispatch.test.ts +66 -0
  141. package/src/scenarios/orchestratorTermination.test.ts +54 -0
  142. package/src/scenarios/otel-emission.test.ts +113 -0
  143. package/src/scenarios/otel-trace-propagation.test.ts +90 -0
  144. package/src/scenarios/pack-registry-publish.test.ts +93 -0
  145. package/src/scenarios/pack-registry.test.ts +328 -0
  146. package/src/scenarios/pause-resume.test.ts +109 -0
  147. package/src/scenarios/policies.test.ts +162 -0
  148. package/src/scenarios/profileDerivation.test.ts +335 -0
  149. package/src/scenarios/providerPolicyEnforcement.test.ts +132 -0
  150. package/src/scenarios/rate-limit-envelope.test.ts +97 -0
  151. package/src/scenarios/redaction.test.ts +254 -0
  152. package/src/scenarios/redactionAdversarial.test.ts +162 -0
  153. package/src/scenarios/replay-fork-arbitrary.test.ts +347 -0
  154. package/src/scenarios/replay-fork.test.ts +216 -0
  155. package/src/scenarios/replayDeterminism.test.ts +171 -0
  156. package/src/scenarios/route-coverage.test.ts +129 -0
  157. package/src/scenarios/runs-lifecycle.test.ts +65 -0
  158. package/src/scenarios/runtime-capabilities.test.ts +118 -0
  159. package/src/scenarios/spec-corpus-validity.test.ts +1257 -0
  160. package/src/scenarios/staleClaim.test.ts +223 -0
  161. package/src/scenarios/stream-modes-buffer.test.ts +148 -0
  162. package/src/scenarios/stream-modes-mixed.test.ts +149 -0
  163. package/src/scenarios/stream-modes.test.ts +139 -0
  164. package/src/scenarios/streamReconnect.test.ts +162 -0
  165. package/src/scenarios/subworkflow.test.ts +126 -0
  166. package/src/scenarios/version-negotiation.test.ts +157 -0
  167. package/src/scenarios/wasm-pack-abi-version-rejection.test.ts +47 -0
  168. package/src/scenarios/wasm-pack-invoke-completed.test.ts +69 -0
  169. package/src/scenarios/wasm-pack-invoke-suspended.test.ts +74 -0
  170. package/src/scenarios/wasm-pack-load.test.ts +75 -0
  171. package/src/scenarios/wasm-pack-memory-cap.test.ts +43 -0
  172. package/src/scenarios/wasm-pack-replay-determinism.test.ts +61 -0
  173. package/src/scenarios/webhook-sig-algorithm.test.ts +61 -0
  174. package/src/setup.ts +173 -0
  175. package/vitest.config.ts +17 -0
@@ -0,0 +1,120 @@
1
+ /**
2
+ * Approval-payload scenarios — `capabilities.md` §"interrupt" +
3
+ * `interrupt.md` §"`ApprovalResume`" + `schemas/run-event-payloads
4
+ * .schema.json#$defs/approvalReceived`.
5
+ *
6
+ * Vendor-neutral DISCOVERY-SHAPE contracts for the approval payload
7
+ * vocabulary. These run against every host's `/.well-known/openwop`
8
+ * surface — they don't drive an actual approval flow, which would
9
+ * require a configured workflow + RBAC + interactive interrupt
10
+ * resolution (outside the black-box contract surface this suite
11
+ * asserts).
12
+ *
13
+ * Why discovery-shape only:
14
+ *
15
+ * The wire vocabulary (action enum, refineFeedback object shape,
16
+ * decidedBy contract) is the cross-implementation contract. The
17
+ * round-trip path (configure → trigger → resolve → assert event
18
+ * shape) needs server fixtures the conformance suite doesn't
19
+ * currently provide. Hosts MUST run their own integration tests
20
+ * against their resolution endpoints.
21
+ *
22
+ * Per-action required-fields scenarios (`refine` MUST carry
23
+ * `refineFeedback.scope`; `edit-accept` MUST carry
24
+ * `editedArtifactData`) are deferred pending a future test-mode
25
+ * capability that lets conformance suites trigger an
26
+ * `awaiting_approval` state without going through the full
27
+ * workflow registration + run-create flow.
28
+ *
29
+ * Scenario gating:
30
+ *
31
+ * - **Vocabulary advertisement** runs against every host. Asserts
32
+ * that any approval-related capability the host advertises uses
33
+ * the spec-documented action vocabulary, not the legacy
34
+ * pre-correction `'edit'` form.
35
+ *
36
+ * - **Interrupt-payload retrieval** is a future scenario gated
37
+ * on test-mode capability (see CHANGELOG entry).
38
+ *
39
+ * @see interrupt.md §"`ApprovalResume`"
40
+ * @see schemas/run-event-payloads.schema.json#$defs/approvalReceived
41
+ * @see schemas/suspend-request.schema.json (actions[] enum)
42
+ */
43
+
44
+ import { describe, it, expect } from 'vitest';
45
+ import { driver } from '../lib/driver.js';
46
+
47
+ const CANONICAL_ACTIONS = ['accept', 'reject', 'refine', 'edit-accept', 'ask'] as const;
48
+ const CANONICAL_EVENT_ACTIONS = ['accept', 'reject', 'refine', 'edit-accept', 'timeout'] as const;
49
+ const CANONICAL_REFINE_SCOPES = ['whole', 'section', 'items'] as const;
50
+
51
+ describe('approval-payload: vocabulary discovery contract', () => {
52
+ it('host capability declaration does not regress on the legacy `edit` form (§7 drift pin)', async () => {
53
+ // The spec briefly used `'edit'` for the edit-accept action in
54
+ // commit 0e0171b (2026-04-30) before being corrected to
55
+ // `'edit-accept'`. Any host that captured the spec during that
56
+ // ~30-min window MAY have surfaced `'edit'` somewhere observable
57
+ // in their capability declaration.
58
+ //
59
+ // This scenario walks the discovery payload looking for any
60
+ // string-array field containing the legacy `'edit'` (without the
61
+ // `-accept` suffix). Findings are an indicator the host needs to
62
+ // re-derive its capability declaration from the corrected spec.
63
+ //
64
+ // Most hosts won't surface action vocabularies in /.well-known/openwop
65
+ // at all — that's a `runtimeCapabilities` extension, not a v1
66
+ // mandate. Pass-through (no occurrences) is the expected result.
67
+ const res = await driver.get('/.well-known/openwop', { authenticated: false });
68
+ expect(res.status).toBe(200);
69
+
70
+ const text = JSON.stringify(res.json ?? {});
71
+ // We look for `"edit"` (quoted) to avoid false positives on
72
+ // `"edit-accept"`. The trailing `-accept` ensures the legacy form
73
+ // is distinguishable from the canonical form.
74
+ const legacyHits = text.match(/"edit"/g) ?? [];
75
+
76
+ expect(legacyHits.length, driver.describe(
77
+ 'interrupt.md §"`ApprovalResume`"',
78
+ 'capability declaration MUST NOT contain the legacy `"edit"` action token (use `"edit-accept"` per spec)',
79
+ )).toBe(0);
80
+ });
81
+
82
+ it('canonical action vocabulary is documented in spec (assertion-free reference)', () => {
83
+ // Self-documenting test. The canonical resume actions per spec are
84
+ // accept/reject/refine/edit-accept/ask. Per-host advertisement is
85
+ // optional; this test pins the vocabulary itself for future
86
+ // scenarios that gate on it.
87
+ expect(CANONICAL_ACTIONS).toHaveLength(5);
88
+ expect(new Set(CANONICAL_ACTIONS)).toEqual(
89
+ new Set(['accept', 'reject', 'refine', 'edit-accept', 'ask']),
90
+ );
91
+ });
92
+
93
+ it('canonical event action vocabulary differs from resume (timeout instead of ask)', () => {
94
+ // Subtle: `'ask'` is a resume action that does NOT exit the
95
+ // suspend (per interrupt.md), so it doesn't appear in the
96
+ // approval.received event vocabulary. `'timeout'` IS an event-
97
+ // emitted action (host emits when the suspend window elapses)
98
+ // but isn't a resume action (clients can't submit a timeout).
99
+ //
100
+ // Pin this asymmetry so it doesn't drift.
101
+ expect(CANONICAL_EVENT_ACTIONS).toHaveLength(5);
102
+ expect(new Set(CANONICAL_EVENT_ACTIONS)).toEqual(
103
+ new Set(['accept', 'reject', 'refine', 'edit-accept', 'timeout']),
104
+ );
105
+ // Resume-only token (ask) MUST NOT appear in event vocabulary.
106
+ expect(CANONICAL_EVENT_ACTIONS as readonly string[]).not.toContain('ask');
107
+ // Event-only token (timeout) MUST NOT appear in resume vocabulary.
108
+ expect(CANONICAL_ACTIONS as readonly string[]).not.toContain('timeout');
109
+ });
110
+
111
+ it('refineFeedback scope vocabulary pin (§7 audit, A.5 prereq)', () => {
112
+ // The 3 documented scopes `whole/section/items` MUST be a stable
113
+ // set in v1.x. Adding a scope is additive (clients tolerating
114
+ // unknown values) but semantic changes need a spec discussion.
115
+ expect(CANONICAL_REFINE_SCOPES).toHaveLength(3);
116
+ expect(new Set(CANONICAL_REFINE_SCOPES)).toEqual(
117
+ new Set(['whole', 'section', 'items']),
118
+ );
119
+ });
120
+ });
@@ -0,0 +1,106 @@
1
+ /**
2
+ * Track 13: audit-log integrity profile (auth-profiles.md v1.1).
3
+ *
4
+ * Verifies that hosts claiming the `openwop-audit-log-integrity` profile:
5
+ * 1. Surface `capabilities.auth.auditLogIntegrity.hashChain: true`.
6
+ * 2. Expose `GET /v1/audit/verify` which returns `{chainValid, checkpoints, anomalies}`.
7
+ * 3. Report `chainValid: true` for an unmodified range.
8
+ * 4. Surface at least one signed checkpoint with a non-empty `signature`.
9
+ *
10
+ * Tamper detection (mutating an entry then asserting `chainValid: false`)
11
+ * requires admin access to the host's audit store and is NOT exercised
12
+ * by this black-box suite. Hosts SHOULD implement a separate internal
13
+ * test for tamper detection — see auth-profiles.md.
14
+ *
15
+ * @see spec/v1/auth-profiles.md §"Audit-log integrity"
16
+ */
17
+
18
+ import { describe, it, expect } from 'vitest';
19
+ import { driver } from '../lib/driver.js';
20
+
21
+ interface AuditIntegrityCaps {
22
+ hashChain?: boolean;
23
+ checkpointSignatureAlgorithm?: string;
24
+ checkpointPublicKey?: string;
25
+ checkpointIntervalEntries?: number;
26
+ checkpointIntervalSeconds?: number;
27
+ }
28
+
29
+ interface AuthCaps {
30
+ profiles?: string[];
31
+ auditLogIntegrity?: AuditIntegrityCaps;
32
+ }
33
+
34
+ async function isProfileAdvertised(): Promise<boolean> {
35
+ const disco = await driver.get('/.well-known/openwop');
36
+ const auth = (disco.json as { capabilities?: { auth?: AuthCaps } }).capabilities?.auth ?? {};
37
+ return Array.isArray(auth.profiles) && auth.profiles.includes('openwop-audit-log-integrity');
38
+ }
39
+
40
+ describe('audit-log-integrity: profile shape', () => {
41
+ it('host that claims the profile advertises required capability fields', async () => {
42
+ if (!(await isProfileAdvertised())) {
43
+ // eslint-disable-next-line no-console
44
+ console.warn('[audit-log-integrity] profile not advertised; skipping');
45
+ return;
46
+ }
47
+
48
+ const disco = await driver.get('/.well-known/openwop');
49
+ const integrity =
50
+ (disco.json as { capabilities?: { auth?: AuthCaps } }).capabilities?.auth
51
+ ?.auditLogIntegrity ?? {};
52
+
53
+ expect(integrity.hashChain, driver.describe(
54
+ 'auth-profiles.md §"Audit-log integrity"',
55
+ "openwop-audit-log-integrity profile MUST advertise auditLogIntegrity.hashChain: true",
56
+ )).toBe(true);
57
+ expect(integrity.checkpointSignatureAlgorithm, driver.describe(
58
+ 'auth-profiles.md §"Audit-log integrity" §"Key management"',
59
+ 'checkpointSignatureAlgorithm MUST be present (canonical: ed25519)',
60
+ )).toBeDefined();
61
+ expect(typeof integrity.checkpointPublicKey).toBe('string');
62
+ });
63
+ });
64
+
65
+ describe('audit-log-integrity: verify endpoint returns chainValid', () => {
66
+ it('GET /v1/audit/verify on a recent range reports chainValid: true', async () => {
67
+ if (!(await isProfileAdvertised())) {
68
+ // eslint-disable-next-line no-console
69
+ console.warn('[audit-log-integrity] profile not advertised; skipping');
70
+ return;
71
+ }
72
+
73
+ const verify = await driver.get('/v1/audit/verify?fromSeq=0&toSeq=100');
74
+ if (verify.status === 404) {
75
+ // Host claims the profile but doesn't expose the endpoint — that's
76
+ // a profile-claim violation. Fail explicitly.
77
+ expect(verify.status, driver.describe(
78
+ 'auth-profiles.md §"Audit-log integrity" §"Verification endpoint"',
79
+ 'claiming openwop-audit-log-integrity profile REQUIRES exposing GET /v1/audit/verify',
80
+ )).not.toBe(404);
81
+ return;
82
+ }
83
+ expect(verify.status).toBe(200);
84
+
85
+ const body = verify.json as {
86
+ fromSeq?: number;
87
+ toSeq?: number;
88
+ chainValid?: boolean;
89
+ checkpoints?: Array<{ checkpoint?: string; merkleRoot?: string; signature?: string }>;
90
+ anomalies?: unknown[];
91
+ };
92
+
93
+ expect(body.chainValid, driver.describe(
94
+ 'auth-profiles.md §"Audit-log integrity"',
95
+ 'unmodified audit range MUST report chainValid: true',
96
+ )).toBe(true);
97
+ expect(Array.isArray(body.anomalies)).toBe(true);
98
+ expect(body.anomalies?.length ?? -1).toBe(0);
99
+
100
+ if (Array.isArray(body.checkpoints) && body.checkpoints.length > 0) {
101
+ const cp = body.checkpoints[0];
102
+ expect(typeof cp.signature, 'checkpoint signature MUST be a non-empty string').toBe('string');
103
+ expect((cp.signature ?? '').length).toBeGreaterThan(0);
104
+ }
105
+ });
106
+ });
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Auth scenarios — credential rejection contracts.
3
+ *
4
+ * Tests that authenticated endpoints (manifest read, run create) return
5
+ * the canonical 401 envelope when called with no credential or an
6
+ * invalid credential. Per auth.md §3.
7
+ */
8
+
9
+ import { describe, it, expect } from 'vitest';
10
+ import { driver } from '../lib/driver.js';
11
+
12
+ const KNOWN_AUTHED_PATH = '/v1/runs';
13
+
14
+ describe('auth: missing credential', () => {
15
+ it('returns 401 with canonical error envelope per auth.md §3', async () => {
16
+ const res = await driver.post(
17
+ KNOWN_AUTHED_PATH,
18
+ { workflowId: 'conformance-noop' },
19
+ { authenticated: false },
20
+ );
21
+
22
+ expect(res.status, driver.describe(
23
+ 'auth.md §3',
24
+ 'request without Authorization header MUST return 401',
25
+ )).toBe(401);
26
+
27
+ const body = res.json as { error?: unknown; message?: unknown } | undefined;
28
+ expect(typeof body?.error, driver.describe(
29
+ 'auth.md §3 + rest-endpoints.md error envelope',
30
+ 'response body MUST include `error` (machine code) string',
31
+ )).toBe('string');
32
+ expect(typeof body?.message, driver.describe(
33
+ 'auth.md §3 + rest-endpoints.md error envelope',
34
+ 'response body MUST include `message` (human description) string',
35
+ )).toBe('string');
36
+ });
37
+ });
38
+
39
+ describe('auth: invalid credential', () => {
40
+ it('returns 401 (not 200, not 403) per auth.md §3', async () => {
41
+ const res = await driver.post(
42
+ KNOWN_AUTHED_PATH,
43
+ { workflowId: 'conformance-noop' },
44
+ {
45
+ authenticated: false,
46
+ headers: { Authorization: 'Bearer hk_definitely_not_a_real_key_12345' },
47
+ },
48
+ );
49
+
50
+ expect(res.status, driver.describe(
51
+ 'auth.md §3',
52
+ 'request with invalid Authorization MUST return 401, not 403',
53
+ )).toBe(401);
54
+ });
55
+ });
@@ -0,0 +1,166 @@
1
+ /**
2
+ * BYOK end-to-end roundtrip scenarios (`openwop-byok` profile).
3
+ *
4
+ * Companion to `redaction.test.ts` + `redactionAdversarial.test.ts`,
5
+ * which assert credentialRefs DON'T leak (negative tests). This file
6
+ * asserts credentialRefs DO resolve and DO get used (positive test) —
7
+ * with redaction-safe verification via SHA-256 hashing.
8
+ *
9
+ * Both scenarios skip trivially-pass when the host returns 404/422 from
10
+ * the start-run call (production deployments don't advertise the
11
+ * fixture surface). Hosts that opt into `OPENWOP_CONFORMANCE_FIXTURES=1`
12
+ * AND pre-provision a secret under `openwop-conformance-canary-secret`
13
+ * expose the surface and the scenarios run end-to-end.
14
+ *
15
+ * The scenarios assert shape + non-empty + redaction, not exact value
16
+ * equality — any host-defined canary value works as long as it's
17
+ * non-empty and the host's `secrets.resolve` returns it intact.
18
+ *
19
+ * Spec references:
20
+ * - https://github.com/openwop/openwop/blob/main/spec/v1/run-options.md §"Credential references"
21
+ * - https://github.com/openwop/openwop/blob/main/spec/v1/auth.md §"Secret resolution"
22
+ * - https://github.com/openwop/openwop/blob/main/spec/v1/observability.md §"Redaction"
23
+ */
24
+
25
+ import { describe, it, expect } from 'vitest';
26
+ import { driver } from '../lib/driver.js';
27
+ import { pollUntilTerminal } from '../lib/polling.js';
28
+ import { isFixtureAdvertised } from '../lib/fixtures.js';
29
+
30
+ /** SHA-256 hex regex — 64 lowercase hex chars exactly. */
31
+ const SHA256_HEX_RE = /^[0-9a-f]{64}$/;
32
+
33
+ const BYOK_WORKFLOW_ID = 'openwop-smoke-byok-roundtrip';
34
+ const SKIP_NO_FIXTURE = !isFixtureAdvertised(BYOK_WORKFLOW_ID);
35
+
36
+ describe.skipIf(SKIP_NO_FIXTURE)('byok: end-to-end credentialRef resolution roundtrip (openwop-byok profile)', () => {
37
+ it('the canary fixture run MUST resolve a host-provisioned secret and emit SHA-256 hex', async () => {
38
+ const create = await driver.post('/v1/runs', {
39
+ workflowId: 'openwop-smoke-byok-roundtrip',
40
+ });
41
+
42
+ // Fixture absent OR canary not provisioned — host doesn't opt in.
43
+ // Scenario passes trivially.
44
+ if (create.status === 404 || create.status === 422) {
45
+ return;
46
+ }
47
+
48
+ expect(create.status, driver.describe(
49
+ 'rest-endpoints.md POST /v1/runs',
50
+ 'starting openwop-smoke-byok-roundtrip MUST succeed when OPENWOP_CONFORMANCE_FIXTURES=1 is advertised AND openwop-conformance-canary-secret is provisioned',
51
+ )).toBe(201);
52
+ const runId = (create.json as { runId: string }).runId;
53
+
54
+ const terminal = await pollUntilTerminal(runId);
55
+ expect(terminal.status, driver.describe(
56
+ 'auth.md §"Secret resolution"',
57
+ 'BYOK fixture run MUST reach terminal completed when canary is provisioned',
58
+ )).toBe('completed');
59
+
60
+ // The fixture writes secretSha256 + secretLength as node outputs
61
+ // (which surface on the run via the engine's variables/outputs map
62
+ // depending on host implementation). At minimum, the run terminates
63
+ // completed — that's enough to know secrets.resolve returned a
64
+ // non-empty value. The shape assertion below is the additional
65
+ // proof that the canary value reached the NodeModule intact.
66
+ const variables = terminal.variables ?? {};
67
+ const outputs =
68
+ (terminal as { outputs?: Record<string, unknown> }).outputs ?? {};
69
+
70
+ // The host MAY surface fixture outputs via variables OR via a
71
+ // host-specific outputs map. Look in both.
72
+ const candidate =
73
+ (outputs['resolve-secret'] as Record<string, unknown> | undefined) ??
74
+ (variables['resolve-secret'] as Record<string, unknown> | undefined) ??
75
+ (outputs as Record<string, unknown>) ??
76
+ (variables as Record<string, unknown>);
77
+
78
+ if (!candidate || typeof candidate !== 'object') {
79
+ // Host doesn't expose node outputs in variables/outputs map —
80
+ // some hosts only expose them on the events stream. Skip the
81
+ // shape check; the run-completed assertion above is sufficient.
82
+ return;
83
+ }
84
+
85
+ if ('secretSha256' in candidate && typeof candidate.secretSha256 === 'string') {
86
+ expect(candidate.secretSha256, driver.describe(
87
+ 'auth.md §"Secret resolution"',
88
+ 'fixture-emitted SHA-256 hex MUST be 64 lowercase hex chars',
89
+ )).toMatch(SHA256_HEX_RE);
90
+ }
91
+ if ('secretLength' in candidate && typeof candidate.secretLength === 'number') {
92
+ expect(candidate.secretLength, driver.describe(
93
+ 'auth.md §"Secret resolution"',
94
+ 'resolved canary length MUST be > 0 (non-empty)',
95
+ )).toBeGreaterThan(0);
96
+ }
97
+ });
98
+
99
+ it('BYOK fixture run MUST emit a node.completed event for the resolve step', async () => {
100
+ const create = await driver.post('/v1/runs', {
101
+ workflowId: 'openwop-smoke-byok-roundtrip',
102
+ });
103
+ if (create.status === 404 || create.status === 422) {
104
+ return;
105
+ }
106
+ expect(create.status).toBe(201);
107
+ const runId = (create.json as { runId: string }).runId;
108
+
109
+ await pollUntilTerminal(runId);
110
+
111
+ const eventsResp = await driver.get(`/v1/runs/${runId}/events`);
112
+ expect(eventsResp.status).toBe(200);
113
+ const events = (eventsResp.json as { events: Array<{ type: string; nodeId?: string }> })
114
+ .events;
115
+
116
+ const completed = events.filter(
117
+ (e) => e.type === 'node.completed' && e.nodeId === 'resolve-secret',
118
+ );
119
+ expect(completed.length, driver.describe(
120
+ 'event-log.md §node.completed',
121
+ 'BYOK fixture node MUST emit exactly one node.completed event when secrets.resolve succeeds',
122
+ )).toBe(1);
123
+ });
124
+
125
+ it('BYOK fixture run event log MUST NOT echo the resolved secret value (redaction)', async () => {
126
+ const create = await driver.post('/v1/runs', {
127
+ workflowId: 'openwop-smoke-byok-roundtrip',
128
+ });
129
+ if (create.status === 404 || create.status === 422) {
130
+ return;
131
+ }
132
+ expect(create.status).toBe(201);
133
+ const runId = (create.json as { runId: string }).runId;
134
+
135
+ await pollUntilTerminal(runId);
136
+
137
+ const eventsResp = await driver.get(`/v1/runs/${runId}/events`);
138
+ expect(eventsResp.status).toBe(200);
139
+
140
+ // Universal redaction marker — same pattern as redaction.test.ts.
141
+ // The test cannot know the exact canary value (host-defined), but
142
+ // it MUST NOT contain a SHA-256-shaped or base64-shaped substring
143
+ // adjacent to a `value:`/`secret:`/`password:` key. This is a
144
+ // defense-in-depth check; the existing redaction.test.ts has the
145
+ // canonical assertion.
146
+ const dump = JSON.stringify(eventsResp.json);
147
+
148
+ // The fixture emits `secretSha256: <hash>` and `secretLength: <n>`.
149
+ // These ARE allowed in the event log (they're hash + length, not
150
+ // the raw value). What MUST NOT appear: a key named `value` or
151
+ // `password` carrying string-typed content alongside a
152
+ // `secretSha256` field — that would suggest the raw value leaked.
153
+ const suspiciousPatterns = [
154
+ /"value"\s*:\s*"[^"]{8,}".*"secretSha256"/,
155
+ /"password"\s*:\s*"[^"]{8,}"/,
156
+ /"plaintext"\s*:\s*"[^"]{8,}"/,
157
+ /"raw_secret"\s*:\s*"[^"]{8,}"/,
158
+ ];
159
+ for (const pat of suspiciousPatterns) {
160
+ expect(dump, driver.describe(
161
+ 'observability.md §"Redaction"',
162
+ `event log MUST NOT contain a payload matching ${pat} — secret.echo fixture only emits hash + length`,
163
+ )).not.toMatch(pat);
164
+ }
165
+ });
166
+ });
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Cancellation scenarios — exercises `POST /v1/runs/{runId}/cancel`
3
+ * mid-flight using the `conformance-cancellable` fixture.
4
+ *
5
+ * The fixture sleeps `delayMs` (caller-supplied). The test starts a
6
+ * run with delayMs=10s, polls until `running`, issues cancel, and
7
+ * verifies terminal `cancelled` within 5s.
8
+ */
9
+
10
+ import { describe, it, expect } from 'vitest';
11
+ import { driver } from '../lib/driver.js';
12
+ import { pollUntilStatus } from '../lib/polling.js';
13
+ import { isFixtureAdvertised } from '../lib/fixtures.js';
14
+
15
+ const WORKFLOW_ID = 'conformance-cancellable';
16
+ const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
17
+
18
+ describe.skipIf(SKIP_NO_FIXTURE)('cancellation: in-flight :cancel reaches terminal `cancelled`', () => {
19
+ it('POST /v1/runs/{runId}/cancel returns 200 and run terminates as cancelled', async () => {
20
+ const create = await driver.post('/v1/runs', {
21
+ workflowId: WORKFLOW_ID,
22
+ inputs: { delayMs: 10_000 },
23
+ });
24
+ expect(create.status, driver.describe(
25
+ 'rest-endpoints.md',
26
+ 'POST /v1/runs MUST return 201 on accepted run',
27
+ )).toBe(201);
28
+ const runId = (create.json as { runId: string }).runId;
29
+
30
+ // Wait for run to reach `running` so the cancel hits a live executor,
31
+ // not the dispatch queue. Allow up to 5s for boot.
32
+ await pollUntilStatus(runId, 'running', { timeoutMs: 5_000 });
33
+
34
+ const cancel = await driver.post(
35
+ `/v1/runs/${encodeURIComponent(runId)}/cancel`,
36
+ { reason: 'conformance-cancellation-test' },
37
+ );
38
+ expect(cancel.status, driver.describe(
39
+ 'rest-endpoints.md POST /v1/runs/{runId}/cancel',
40
+ 'cancel MUST return 200 on accepted cancellation',
41
+ )).toBe(200);
42
+
43
+ const cancelBody = cancel.json as { status?: string };
44
+ expect(
45
+ ['cancelled', 'cancelling'].includes(cancelBody.status ?? ''),
46
+ driver.describe(
47
+ 'rest-endpoints.md POST /v1/runs/{runId}/cancel',
48
+ 'cancel response status MUST be one of `cancelled` or `cancelling`',
49
+ ),
50
+ ).toBe(true);
51
+
52
+ const terminal = await pollUntilStatus(runId, 'cancelled', { timeoutMs: 5_000 });
53
+ expect(terminal.status, driver.describe(
54
+ 'fixtures.md conformance-cancellable §Terminal status',
55
+ 'fixture MUST reach terminal `cancelled` within 5s of cancel',
56
+ )).toBe('cancelled');
57
+ });
58
+ });
59
+
60
+ describe('cancellation: cancelling an unknown run returns 404', () => {
61
+ it('POST /v1/runs/{nonexistentId}/cancel returns 404', async () => {
62
+ const res = await driver.post('/v1/runs/openwop-conformance-no-such-run/cancel', {});
63
+ expect(
64
+ [403, 404].includes(res.status),
65
+ driver.describe('rest-endpoints.md', 'cancel on unknown run MUST return 404 or 403'),
66
+ ).toBe(true);
67
+ });
68
+ });
@@ -0,0 +1,149 @@
1
+ /**
2
+ * Cap-breach scenarios (G4 / CC-1) — exercises `conformance-cap-breach`
3
+ * fixture with `RunOptions.configurable.recursionLimit: 3` to trigger the
4
+ * per-run nodeExecutionCount cap.
5
+ *
6
+ * Verifies:
7
+ * 1. Run reaches terminal `failed` with `error.code = "recursion_limit_exceeded"`.
8
+ * 2. `cap.breached` event is emitted with `kind: "node-executions"` payload
9
+ * containing `limit`, `observed`, and `nodeId`.
10
+ * 3. `cap.breached` precedes `run.failed` in the event log (the breach is
11
+ * detected BEFORE the over-limit node fires, so `node.started` for the
12
+ * over-limit node MUST NOT appear).
13
+ *
14
+ * Spec references:
15
+ * - run-options.md §recursionLimit
16
+ * - observability.md §cap.breached
17
+ * - schemas/run-event-payloads.schema.json §capBreached
18
+ * - docs/spec gap G4
19
+ */
20
+
21
+ import { describe, it, expect } from 'vitest';
22
+ import { driver } from '../lib/driver.js';
23
+ import { pollUntilTerminal } from '../lib/polling.js';
24
+ import { isFixtureAdvertised } from '../lib/fixtures.js';
25
+
26
+ const WORKFLOW_ID = 'conformance-cap-breach';
27
+ const RECURSION_LIMIT = 3;
28
+ const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
29
+
30
+ interface RunEvent {
31
+ readonly eventId: string;
32
+ readonly runId: string;
33
+ readonly nodeId?: string;
34
+ readonly type: string;
35
+ readonly sequence: number;
36
+ readonly payload?: unknown;
37
+ }
38
+
39
+ describe.skipIf(SKIP_NO_FIXTURE)('cap-breach: conformance-cap-breach fixture fails with recursion_limit_exceeded', () => {
40
+ it('emits cap.breached + transitions to terminal failed when configurable.recursionLimit is exceeded', async () => {
41
+ const create = await driver.post('/v1/runs', {
42
+ workflowId: WORKFLOW_ID,
43
+ configurable: { recursionLimit: RECURSION_LIMIT },
44
+ });
45
+ expect(create.status, driver.describe(
46
+ 'rest-endpoints.md POST /v1/runs',
47
+ 'run creation MUST accept the request even when configurable.recursionLimit is below the workflow size',
48
+ )).toBe(201);
49
+ const runId = (create.json as { runId: string }).runId;
50
+
51
+ const terminal = await pollUntilTerminal(runId);
52
+
53
+ expect(terminal.status, driver.describe(
54
+ 'fixtures.md conformance-cap-breach §Terminal status',
55
+ 'fixture MUST reach terminal `failed` when recursion limit is exceeded',
56
+ )).toBe('failed');
57
+
58
+ expect(terminal.error?.code, driver.describe(
59
+ 'run-options.md §recursionLimit',
60
+ 'RunSnapshot.error.code MUST equal "recursion_limit_exceeded"',
61
+ )).toBe('recursion_limit_exceeded');
62
+
63
+ expect(typeof terminal.error?.message, driver.describe(
64
+ 'rest-endpoints.md RunSnapshot.error.message',
65
+ 'RunSnapshot.error.message MUST be a string',
66
+ )).toBe('string');
67
+
68
+ const eventsRes = await driver.get(
69
+ `/v1/runs/${encodeURIComponent(runId)}/events/poll?lastSequence=0&timeout=1`,
70
+ );
71
+ expect(eventsRes.status).toBe(200);
72
+ const events = (eventsRes.json as { events?: RunEvent[] } | undefined)?.events ?? [];
73
+
74
+ const capBreachEvents = events.filter((e) => e.type === 'cap.breached');
75
+ expect(capBreachEvents.length, driver.describe(
76
+ 'observability.md §cap.breached',
77
+ 'exactly one cap.breached event MUST be emitted on recursion-limit exceedance',
78
+ )).toBe(1);
79
+
80
+ const breach = capBreachEvents[0];
81
+ const payload = breach.payload as
82
+ | { kind?: string; limit?: number; observed?: number; nodeId?: string }
83
+ | undefined;
84
+
85
+ expect(payload?.kind, driver.describe(
86
+ 'run-event-payloads.schema.json §capBreached.kind',
87
+ 'cap.breached payload MUST carry kind="node-executions"',
88
+ )).toBe('node-executions');
89
+
90
+ expect(payload?.limit, driver.describe(
91
+ 'run-event-payloads.schema.json §capBreached.limit',
92
+ 'cap.breached payload MUST carry the resolved limit (3 from configurable.recursionLimit)',
93
+ )).toBe(RECURSION_LIMIT);
94
+
95
+ expect(typeof payload?.observed, driver.describe(
96
+ 'run-event-payloads.schema.json §capBreached.observed',
97
+ 'cap.breached payload MUST carry the observed count as a number',
98
+ )).toBe('number');
99
+ expect(payload?.observed).toBeGreaterThan(RECURSION_LIMIT);
100
+
101
+ expect(typeof payload?.nodeId, driver.describe(
102
+ 'run-event-payloads.schema.json §capBreached.nodeId',
103
+ 'cap.breached payload MUST carry the offending nodeId for node-executions kind',
104
+ )).toBe('string');
105
+ });
106
+
107
+ it('cap.breached precedes run.failed in the event sequence (breach detected before over-limit node fires)', async () => {
108
+ const create = await driver.post('/v1/runs', {
109
+ workflowId: WORKFLOW_ID,
110
+ configurable: { recursionLimit: RECURSION_LIMIT },
111
+ });
112
+ const runId = (create.json as { runId: string }).runId;
113
+
114
+ await pollUntilTerminal(runId);
115
+
116
+ const eventsRes = await driver.get(
117
+ `/v1/runs/${encodeURIComponent(runId)}/events/poll?lastSequence=0&timeout=1`,
118
+ );
119
+ const events = (eventsRes.json as { events?: RunEvent[] } | undefined)?.events ?? [];
120
+
121
+ const capBreach = events.find((e) => e.type === 'cap.breached');
122
+ const runFailed = events.find((e) => e.type === 'run.failed');
123
+
124
+ expect(capBreach, 'cap.breached MUST be emitted').toBeDefined();
125
+ expect(runFailed, 'run.failed MUST be emitted').toBeDefined();
126
+
127
+ expect(capBreach!.sequence, driver.describe(
128
+ 'observability.md §event ordering',
129
+ 'cap.breached MUST precede run.failed in sequence (breach detected BEFORE over-limit node fires)',
130
+ )).toBeLessThan(runFailed!.sequence);
131
+
132
+ // Count node.started events. With recursionLimit=3 and the breach
133
+ // detected BEFORE the 4th node fires, AT MOST 3 node.started events
134
+ // SHOULD appear (the over-limit node MUST NOT receive node.started).
135
+ // We assert a range rather than equality to tolerate transient pre-
136
+ // breach node failures (e.g. a `node.failed` cutting the chain
137
+ // short) — those would emit fewer than `RECURSION_LIMIT` started
138
+ // events while still satisfying the invariant.
139
+ const nodeStarted = events.filter((e) => e.type === 'node.started');
140
+ expect(nodeStarted.length, driver.describe(
141
+ 'run-options.md §recursionLimit',
142
+ 'at most `limit` node.started events MUST be emitted; the over-limit node MUST NOT receive node.started',
143
+ )).toBeLessThanOrEqual(RECURSION_LIMIT);
144
+ expect(nodeStarted.length, driver.describe(
145
+ 'run-options.md §recursionLimit',
146
+ 'at least one node MUST start before the breach (otherwise the workflow never executed)',
147
+ )).toBeGreaterThan(0);
148
+ });
149
+ });