@openwop/openwop-conformance 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +241 -0
  3. package/api/asyncapi.yaml +481 -0
  4. package/api/openapi.yaml +830 -0
  5. package/api/redocly.yaml +8 -0
  6. package/coverage.md +80 -0
  7. package/dist/cli.js +161 -0
  8. package/fixtures/conformance-a2a-task-roundtrip.json +27 -0
  9. package/fixtures/conformance-agent-identity.json +27 -0
  10. package/fixtures/conformance-agent-low-confidence.json +29 -0
  11. package/fixtures/conformance-agent-memory-cross-tenant.json +28 -0
  12. package/fixtures/conformance-agent-memory-redaction.json +32 -0
  13. package/fixtures/conformance-agent-memory-roundtrip.json +32 -0
  14. package/fixtures/conformance-agent-memory-ttl.json +31 -0
  15. package/fixtures/conformance-agent-pack-export.json +26 -0
  16. package/fixtures/conformance-agent-pack-install.json +26 -0
  17. package/fixtures/conformance-agent-pack-provenance.json +31 -0
  18. package/fixtures/conformance-agent-reasoning.json +29 -0
  19. package/fixtures/conformance-approval.json +27 -0
  20. package/fixtures/conformance-cancellable.json +33 -0
  21. package/fixtures/conformance-cap-breach.json +27 -0
  22. package/fixtures/conformance-capability-missing.json +23 -0
  23. package/fixtures/conformance-channel-ttl.json +60 -0
  24. package/fixtures/conformance-clarification.json +30 -0
  25. package/fixtures/conformance-conversation-capability-negotiation.json +23 -0
  26. package/fixtures/conformance-conversation-lifecycle.json +32 -0
  27. package/fixtures/conformance-conversation-replay.json +33 -0
  28. package/fixtures/conformance-conversation-vs-clarification.json +26 -0
  29. package/fixtures/conformance-delay.json +33 -0
  30. package/fixtures/conformance-dispatch-loop.json +38 -0
  31. package/fixtures/conformance-failure.json +23 -0
  32. package/fixtures/conformance-idempotent.json +30 -0
  33. package/fixtures/conformance-identity.json +32 -0
  34. package/fixtures/conformance-interrupt-auth-required.json +28 -0
  35. package/fixtures/conformance-interrupt-external-event.json +33 -0
  36. package/fixtures/conformance-interrupt-parent-child-cancel-child.json +27 -0
  37. package/fixtures/conformance-interrupt-parent-child-cancel.json +26 -0
  38. package/fixtures/conformance-interrupt-quorum.json +30 -0
  39. package/fixtures/conformance-mcp-tool-roundtrip.json +32 -0
  40. package/fixtures/conformance-message-reducer.json +31 -0
  41. package/fixtures/conformance-multi-node.json +21 -0
  42. package/fixtures/conformance-noop.json +23 -0
  43. package/fixtures/conformance-orchestrator-dispatch.json +47 -0
  44. package/fixtures/conformance-orchestrator-low-confidence.json +41 -0
  45. package/fixtures/conformance-orchestrator-terminate.json +44 -0
  46. package/fixtures/conformance-stream-text.json +26 -0
  47. package/fixtures/conformance-subworkflow-child.json +21 -0
  48. package/fixtures/conformance-subworkflow-parent.json +49 -0
  49. package/fixtures/conformance-version-fold.json +23 -0
  50. package/fixtures/conformance-wasm-pack-roundtrip.json +25 -0
  51. package/fixtures/pack-manifests/pack-private-example.json +26 -0
  52. package/fixtures.md +404 -0
  53. package/package.json +48 -0
  54. package/schemas/README.md +75 -0
  55. package/schemas/agent-manifest.schema.json +107 -0
  56. package/schemas/agent-ref.schema.json +53 -0
  57. package/schemas/capabilities.schema.json +287 -0
  58. package/schemas/channel-written-payload.schema.json +55 -0
  59. package/schemas/conversation-event.schema.json +120 -0
  60. package/schemas/conversation-turn.schema.json +72 -0
  61. package/schemas/debug-bundle.schema.json +196 -0
  62. package/schemas/dispatch-config.schema.json +46 -0
  63. package/schemas/error-envelope.schema.json +25 -0
  64. package/schemas/memory-entry.schema.json +36 -0
  65. package/schemas/memory-list-options.schema.json +21 -0
  66. package/schemas/node-pack-manifest.schema.json +235 -0
  67. package/schemas/orchestrator-decision.schema.json +60 -0
  68. package/schemas/run-event-payloads.schema.json +663 -0
  69. package/schemas/run-event.schema.json +116 -0
  70. package/schemas/run-options.schema.json +81 -0
  71. package/schemas/run-orchestrator-decided-event.schema.json +20 -0
  72. package/schemas/run-snapshot.schema.json +121 -0
  73. package/schemas/suspend-request.schema.json +182 -0
  74. package/schemas/workflow-definition.schema.json +430 -0
  75. package/src/cli.ts +187 -0
  76. package/src/lib/a2a-fake-peer.ts +233 -0
  77. package/src/lib/canaries.ts +186 -0
  78. package/src/lib/driver.ts +96 -0
  79. package/src/lib/env.ts +49 -0
  80. package/src/lib/fixtures.ts +93 -0
  81. package/src/lib/mcp-fake-server.ts +185 -0
  82. package/src/lib/multi-agent-capabilities.ts +155 -0
  83. package/src/lib/multiProcess.ts +141 -0
  84. package/src/lib/otel-collector.ts +312 -0
  85. package/src/lib/paths.ts +198 -0
  86. package/src/lib/polling.ts +81 -0
  87. package/src/lib/profiles.ts +258 -0
  88. package/src/lib/sse.ts +172 -0
  89. package/src/scenarios/a2a-task-roundtrip.test.ts +149 -0
  90. package/src/scenarios/agentConfidenceEscalation.test.ts +61 -0
  91. package/src/scenarios/agentMemoryCrossTenantIsolation.test.ts +54 -0
  92. package/src/scenarios/agentMemoryRedactionContract.test.ts +46 -0
  93. package/src/scenarios/agentMemoryRoundTrip.test.ts +52 -0
  94. package/src/scenarios/agentMemoryTtlExpiry.test.ts +47 -0
  95. package/src/scenarios/agentMessageReducer.test.ts +57 -0
  96. package/src/scenarios/agentMetadata.test.ts +56 -0
  97. package/src/scenarios/agentPackExport.test.ts +45 -0
  98. package/src/scenarios/agentPackInstall.test.ts +50 -0
  99. package/src/scenarios/agentPackProvenance.test.ts +53 -0
  100. package/src/scenarios/agentReasoningEvents.test.ts +72 -0
  101. package/src/scenarios/append-ordering.test.ts +91 -0
  102. package/src/scenarios/approval-payload.test.ts +120 -0
  103. package/src/scenarios/audit-log-integrity.test.ts +106 -0
  104. package/src/scenarios/auth.test.ts +55 -0
  105. package/src/scenarios/byok-roundtrip.test.ts +166 -0
  106. package/src/scenarios/cancellation.test.ts +68 -0
  107. package/src/scenarios/cap-breach.test.ts +149 -0
  108. package/src/scenarios/channel-ttl.test.ts +70 -0
  109. package/src/scenarios/configurable-schema.test.ts +76 -0
  110. package/src/scenarios/conversationCapabilityNegotiation.test.ts +39 -0
  111. package/src/scenarios/conversationLifecycle.test.ts +64 -0
  112. package/src/scenarios/conversationReplayDeterminism.test.ts +52 -0
  113. package/src/scenarios/conversationVsLegacySuspend.test.ts +46 -0
  114. package/src/scenarios/cost-attribution.test.ts +207 -0
  115. package/src/scenarios/debugBundle.test.ts +222 -0
  116. package/src/scenarios/discovery.test.ts +147 -0
  117. package/src/scenarios/dispatchLoop.test.ts +52 -0
  118. package/src/scenarios/errors.test.ts +144 -0
  119. package/src/scenarios/eventOrdering.test.ts +144 -0
  120. package/src/scenarios/failure-path.test.ts +46 -0
  121. package/src/scenarios/fixtures-gating.test.ts +137 -0
  122. package/src/scenarios/fixtures-valid.test.ts +140 -0
  123. package/src/scenarios/highConcurrency.test.ts +263 -0
  124. package/src/scenarios/idempotency.test.ts +83 -0
  125. package/src/scenarios/idempotencyRetry.test.ts +130 -0
  126. package/src/scenarios/identity-passthrough.test.ts +54 -0
  127. package/src/scenarios/interrupt-approval.test.ts +97 -0
  128. package/src/scenarios/interrupt-auth-required-resume.test.ts +88 -0
  129. package/src/scenarios/interrupt-clarification.test.ts +45 -0
  130. package/src/scenarios/interrupt-external-event-correlation.test.ts +113 -0
  131. package/src/scenarios/interrupt-parent-child-cascade.test.ts +102 -0
  132. package/src/scenarios/interrupt-quorum-resolution.test.ts +97 -0
  133. package/src/scenarios/interruptRace.test.ts +176 -0
  134. package/src/scenarios/maliciousManifest.test.ts +154 -0
  135. package/src/scenarios/mcp-discoverability.test.ts +129 -0
  136. package/src/scenarios/mcp-tool-roundtrip.test.ts +149 -0
  137. package/src/scenarios/multi-node-ordering.test.ts +60 -0
  138. package/src/scenarios/multi-region-idempotency.test.ts +52 -0
  139. package/src/scenarios/orchestratorConservativePath.test.ts +63 -0
  140. package/src/scenarios/orchestratorDispatch.test.ts +66 -0
  141. package/src/scenarios/orchestratorTermination.test.ts +54 -0
  142. package/src/scenarios/otel-emission.test.ts +113 -0
  143. package/src/scenarios/otel-trace-propagation.test.ts +90 -0
  144. package/src/scenarios/pack-registry-publish.test.ts +93 -0
  145. package/src/scenarios/pack-registry.test.ts +328 -0
  146. package/src/scenarios/pause-resume.test.ts +109 -0
  147. package/src/scenarios/policies.test.ts +162 -0
  148. package/src/scenarios/profileDerivation.test.ts +335 -0
  149. package/src/scenarios/providerPolicyEnforcement.test.ts +132 -0
  150. package/src/scenarios/rate-limit-envelope.test.ts +97 -0
  151. package/src/scenarios/redaction.test.ts +254 -0
  152. package/src/scenarios/redactionAdversarial.test.ts +162 -0
  153. package/src/scenarios/replay-fork-arbitrary.test.ts +347 -0
  154. package/src/scenarios/replay-fork.test.ts +216 -0
  155. package/src/scenarios/replayDeterminism.test.ts +171 -0
  156. package/src/scenarios/route-coverage.test.ts +129 -0
  157. package/src/scenarios/runs-lifecycle.test.ts +65 -0
  158. package/src/scenarios/runtime-capabilities.test.ts +118 -0
  159. package/src/scenarios/spec-corpus-validity.test.ts +1257 -0
  160. package/src/scenarios/staleClaim.test.ts +223 -0
  161. package/src/scenarios/stream-modes-buffer.test.ts +148 -0
  162. package/src/scenarios/stream-modes-mixed.test.ts +149 -0
  163. package/src/scenarios/stream-modes.test.ts +139 -0
  164. package/src/scenarios/streamReconnect.test.ts +162 -0
  165. package/src/scenarios/subworkflow.test.ts +126 -0
  166. package/src/scenarios/version-negotiation.test.ts +157 -0
  167. package/src/scenarios/wasm-pack-abi-version-rejection.test.ts +47 -0
  168. package/src/scenarios/wasm-pack-invoke-completed.test.ts +69 -0
  169. package/src/scenarios/wasm-pack-invoke-suspended.test.ts +74 -0
  170. package/src/scenarios/wasm-pack-load.test.ts +75 -0
  171. package/src/scenarios/wasm-pack-memory-cap.test.ts +43 -0
  172. package/src/scenarios/wasm-pack-replay-determinism.test.ts +61 -0
  173. package/src/scenarios/webhook-sig-algorithm.test.ts +61 -0
  174. package/src/setup.ts +173 -0
  175. package/vitest.config.ts +17 -0
@@ -0,0 +1,223 @@
1
+ /**
2
+ * Stale-claim recovery scenario per spec/v1/scale-profiles.md
3
+ * §"Replay semantics" + spec/v1/storage-adapters.md §"Claim acquisition."
4
+ *
5
+ * When a process holding a run claim dies without releasing the claim,
6
+ * another process that boots later (after the claim TTL has expired)
7
+ * MUST pick up the run and resume execution. The conformance contract:
8
+ *
9
+ * - Process A starts a long-running run, writes some events,
10
+ * SIGKILLs (claim left as held + expires_at populated).
11
+ * - After CLAIM_TTL_MS elapses, claim is "stale" by definition.
12
+ * - Process B boots pointing at the same DB; resume-on-startup
13
+ * re-acquires the claim and finishes the run.
14
+ * - The run's terminal status is observable through process B's
15
+ * HTTP surface.
16
+ *
17
+ * **`@multi-process`** — needs `child_process.spawn` to drive two host
18
+ * processes against a shared SQLite file. Skipped against hosts that
19
+ * aren't the SQLite reference (no shared-storage contract).
20
+ *
21
+ * **`@timing-sensitive`** — relies on a configurable claim TTL.
22
+ * Skipped automatically against hosts that don't expose the TTL via
23
+ * env (the test reads `OPENWOP_STALE_CLAIM_HOST_DIR`; if unset, the
24
+ * scenario skip-equivalents).
25
+ *
26
+ * @see lib/multiProcess.ts — spawnHost helper
27
+ * @see examples/hosts/sqlite/src/server.ts — heartbeat + resume
28
+ */
29
+
30
+ import { describe, it, expect, afterEach } from 'vitest';
31
+ import { mkdtempSync, rmSync } from 'node:fs';
32
+ import { tmpdir } from 'node:os';
33
+ import { join } from 'node:path';
34
+ import { spawnHost, type SpawnedHost } from '../lib/multiProcess.js';
35
+
36
+ // Default off: scenario must be opted in via env. The opt-in lists
37
+ // the host package dir relative to repo root that exposes the
38
+ // OPENWOP_CLAIM_TTL_MS / OPENWOP_HEARTBEAT_INTERVAL_MS / OPENWOP_SQLITE_PATH env
39
+ // vars. The reference SQLite host satisfies this contract.
40
+ const HOST_PACKAGE_DIR = process.env.OPENWOP_STALE_CLAIM_HOST_DIR ?? 'examples/hosts/sqlite';
41
+ const RUN_THIS_SCENARIO = process.env.OPENWOP_RUN_STALE_CLAIM === '1';
42
+
43
+ const APIKEY_A = 'openwop-stale-claim-A';
44
+ const APIKEY_B = 'openwop-stale-claim-B';
45
+ const PORT_A = 4801;
46
+ const PORT_B = 4802;
47
+ const CLAIM_TTL_MS = 2000;
48
+ const HEARTBEAT_INTERVAL_MS = 500;
49
+
50
+ interface RunSnapshot {
51
+ status?: string;
52
+ runId?: string;
53
+ }
54
+
55
+ interface PollResponse {
56
+ events?: Array<{ type?: string; nodeId?: string | null; data?: unknown }>;
57
+ isComplete?: boolean;
58
+ }
59
+
60
+ async function fetchSnapshot(baseUrl: string, apiKey: string, runId: string): Promise<RunSnapshot> {
61
+ const res = await fetch(`${baseUrl}/v1/runs/${encodeURIComponent(runId)}`, {
62
+ headers: { Authorization: `Bearer ${apiKey}` },
63
+ });
64
+ if (!res.ok) throw new Error(`GET /v1/runs/${runId} failed: ${res.status}`);
65
+ return (await res.json()) as RunSnapshot;
66
+ }
67
+
68
+ async function fetchEvents(
69
+ baseUrl: string,
70
+ apiKey: string,
71
+ runId: string,
72
+ ): Promise<PollResponse> {
73
+ const res = await fetch(
74
+ `${baseUrl}/v1/runs/${encodeURIComponent(runId)}/events/poll`,
75
+ { headers: { Authorization: `Bearer ${apiKey}` } },
76
+ );
77
+ if (!res.ok) throw new Error(`poll failed: ${res.status}`);
78
+ return (await res.json()) as PollResponse;
79
+ }
80
+
81
+ async function pollUntilStatus(
82
+ baseUrl: string,
83
+ apiKey: string,
84
+ runId: string,
85
+ predicate: (s: string) => boolean,
86
+ timeoutMs: number,
87
+ ): Promise<RunSnapshot> {
88
+ const deadline = Date.now() + timeoutMs;
89
+ let last: RunSnapshot = {};
90
+ while (Date.now() < deadline) {
91
+ last = await fetchSnapshot(baseUrl, apiKey, runId);
92
+ if (typeof last.status === 'string' && predicate(last.status)) return last;
93
+ await new Promise((r) => setTimeout(r, 200));
94
+ }
95
+ throw new Error(
96
+ `pollUntilStatus did not match predicate within ${timeoutMs}ms; last status: ${last.status}`,
97
+ );
98
+ }
99
+
100
+ describe.skipIf(!RUN_THIS_SCENARIO)(
101
+ 'staleClaim: orphaned run resumes on a second host process per spec/v1/storage-adapters.md',
102
+ () => {
103
+ let dbDir: string | null = null;
104
+ let hostA: SpawnedHost | null = null;
105
+ let hostB: SpawnedHost | null = null;
106
+
107
+ afterEach(async () => {
108
+ if (hostA) {
109
+ await hostA.kill().catch(() => {});
110
+ hostA = null;
111
+ }
112
+ if (hostB) {
113
+ await hostB.shutdown().catch(() => {});
114
+ hostB = null;
115
+ }
116
+ if (dbDir !== null) {
117
+ try {
118
+ rmSync(dbDir, { recursive: true, force: true });
119
+ } catch {
120
+ // best-effort cleanup
121
+ }
122
+ dbDir = null;
123
+ }
124
+ });
125
+
126
+ it(
127
+ 'process B picks up the orphaned run after process A dies + claim expires',
128
+ async () => {
129
+ // Phase 1: shared DB file in a temp dir.
130
+ dbDir = mkdtempSync(join(tmpdir(), 'openwop-stale-claim-'));
131
+ const dbPath = join(dbDir, 'host.sqlite');
132
+
133
+ // Phase 2: spawn host A and start a long-running cancellable run.
134
+ hostA = await spawnHost({
135
+ packageDir: HOST_PACKAGE_DIR,
136
+ port: PORT_A,
137
+ apiKey: APIKEY_A,
138
+ dbPath,
139
+ claimTtlMs: CLAIM_TTL_MS,
140
+ heartbeatIntervalMs: HEARTBEAT_INTERVAL_MS,
141
+ });
142
+ await hostA.ready();
143
+
144
+ const createRes = await fetch(`${hostA.baseUrl}/v1/runs`, {
145
+ method: 'POST',
146
+ headers: {
147
+ 'Content-Type': 'application/json',
148
+ Authorization: `Bearer ${APIKEY_A}`,
149
+ },
150
+ body: JSON.stringify({
151
+ workflowId: 'conformance-cancellable',
152
+ inputs: { delayMs: 5000 },
153
+ }),
154
+ });
155
+ expect(createRes.status).toBe(201);
156
+ const { runId } = (await createRes.json()) as { runId: string };
157
+
158
+ // Phase 3: wait until A reports the run as `running`.
159
+ await pollUntilStatus(hostA.baseUrl, APIKEY_A, runId, (s) => s === 'running', 5000);
160
+
161
+ // Phase 4: SIGKILL A. The kill MUST NOT release the claim —
162
+ // graceful shutdown is the OPPOSITE behavior.
163
+ await hostA.kill();
164
+ hostA = null;
165
+
166
+ // Phase 5: wait for the claim TTL to lapse. With CLAIM_TTL_MS=2000
167
+ // we wait ~3s to be safely past expiry.
168
+ await new Promise((r) => setTimeout(r, CLAIM_TTL_MS + 1000));
169
+
170
+ // Phase 6: spawn host B at the SAME DB. Its resume-on-startup
171
+ // MUST find the orphaned run, claim it, and dispatch.
172
+ hostB = await spawnHost({
173
+ packageDir: HOST_PACKAGE_DIR,
174
+ port: PORT_B,
175
+ apiKey: APIKEY_B,
176
+ dbPath,
177
+ claimTtlMs: CLAIM_TTL_MS,
178
+ heartbeatIntervalMs: HEARTBEAT_INTERVAL_MS,
179
+ });
180
+ await hostB.ready();
181
+
182
+ // Phase 7: poll until B reports the run as terminal. The run
183
+ // restarts from the beginning of the delay node (5s) on B,
184
+ // plus a small slack window — generous timeout is fine.
185
+ const terminal = await pollUntilStatus(
186
+ hostB.baseUrl,
187
+ APIKEY_B,
188
+ runId,
189
+ (s) => s === 'completed' || s === 'failed' || s === 'cancelled',
190
+ 15_000,
191
+ );
192
+
193
+ expect(terminal.status, 'orphaned run MUST resume to a terminal status under host B').toBe(
194
+ 'completed',
195
+ );
196
+
197
+ // Phase 8: verify the event log records the resume. A
198
+ // `run.resumed` event MUST be present (per the SQLite host's
199
+ // implementation; other hosts MAY use a different marker but
200
+ // SOMETHING that distinguishes resume from fresh start MUST
201
+ // exist in the event log).
202
+ const events = await fetchEvents(hostB.baseUrl, APIKEY_B, runId);
203
+ expect(Array.isArray(events.events), 'events poll MUST return an events array').toBe(true);
204
+ if (events.events && events.events.length > 0) {
205
+ const types = events.events.map((e) => e.type);
206
+ expect(
207
+ types.includes('run.resumed') || types.includes('run.started'),
208
+ 'event log MUST contain at least run.started; resume hosts SHOULD also emit run.resumed',
209
+ ).toBe(true);
210
+ }
211
+ },
212
+ 60_000,
213
+ );
214
+ },
215
+ );
216
+
217
+ // Always-on smoke test for the multiProcess library shape — runs even
218
+ // when the scenario is gated off.
219
+ describe('staleClaim lib: spawnHost surface contract', () => {
220
+ it('spawnHost is exported and has the expected shape', async () => {
221
+ expect(typeof spawnHost).toBe('function');
222
+ });
223
+ });
@@ -0,0 +1,148 @@
1
+ /**
2
+ * SSE buffering scenarios (G1 / S3) — exercises `?bufferMs=` aggregation
3
+ * hint against the existing `conformance-delay` fixture.
4
+ *
5
+ * Verifies:
6
+ * 1. Server accepts `bufferMs` in [0..5000] without error.
7
+ * 2. Out-of-range `bufferMs` returns 400 with `validation_error`.
8
+ * 3. Buffered mode emits at least one `event: batch` SSE frame whose
9
+ * data is a JSON array of `RunEventDoc`.
10
+ * 4. Force-flush on terminal: the run.completed event arrives bundled
11
+ * in a batch, not held back to the next interval.
12
+ * 5. Total event count in buffered mode equals the unbuffered mode
13
+ * count (no events dropped).
14
+ *
15
+ * Spec references:
16
+ * - stream-modes.md §Aggregation hint
17
+ * - spec gap G1
18
+ */
19
+
20
+ import { describe, it, expect } from 'vitest';
21
+ import { driver } from '../lib/driver.js';
22
+ import { subscribe } from '../lib/sse.js';
23
+ import { pollUntilTerminal } from '../lib/polling.js';
24
+ import { isFixtureAdvertised } from '../lib/fixtures.js';
25
+
26
+ const WORKFLOW_ID = 'conformance-delay';
27
+ const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
28
+
29
+ interface RunEventDoc {
30
+ readonly type: string;
31
+ readonly sequence: number;
32
+ }
33
+
34
+ describe.skipIf(SKIP_NO_FIXTURE)('stream-modes-buffer: ?bufferMs= aggregation hint', () => {
35
+ it('accepts bufferMs in range and emits at least one event: batch frame', async () => {
36
+ const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
37
+ expect(create.status).toBe(201);
38
+ const runId = (create.json as { runId: string }).runId;
39
+
40
+ const result = await subscribe(
41
+ `/v1/runs/${encodeURIComponent(runId)}/events?streamMode=updates&bufferMs=200`,
42
+ { timeoutMs: 30_000 },
43
+ );
44
+
45
+ expect(result.status, driver.describe(
46
+ 'stream-modes.md §Aggregation hint',
47
+ 'GET /v1/runs/{runId}/events with valid bufferMs MUST return 200 SSE',
48
+ )).toBe(200);
49
+
50
+ const batchEvents = result.events.filter((e) => e.event === 'batch');
51
+ expect(batchEvents.length, driver.describe(
52
+ 'stream-modes.md §Aggregation hint',
53
+ 'buffered mode MUST emit at least one `event: batch` SSE frame',
54
+ )).toBeGreaterThan(0);
55
+
56
+ // Each batch's data is a JSON array of RunEventDoc.
57
+ for (const batch of batchEvents) {
58
+ const parsed = JSON.parse(batch.data);
59
+ expect(Array.isArray(parsed), driver.describe(
60
+ 'stream-modes.md §batch data shape',
61
+ 'event: batch data MUST parse to a JSON array',
62
+ )).toBe(true);
63
+ expect(parsed.length).toBeGreaterThan(0);
64
+ for (const event of parsed) {
65
+ expect(typeof event.sequence).toBe('number');
66
+ expect(typeof event.type).toBe('string');
67
+ }
68
+ }
69
+ });
70
+
71
+ it('rejects out-of-range bufferMs with 400 validation_error', async () => {
72
+ const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
73
+ const runId = (create.json as { runId: string }).runId;
74
+
75
+ const result = await subscribe(
76
+ `/v1/runs/${encodeURIComponent(runId)}/events?bufferMs=99999`,
77
+ { timeoutMs: 5_000 },
78
+ );
79
+
80
+ expect(result.status, driver.describe(
81
+ 'stream-modes.md §Aggregation hint range',
82
+ 'bufferMs > 5000 MUST return 400',
83
+ )).toBe(400);
84
+
85
+ // Drain the run so it doesn't stall the test runner.
86
+ await pollUntilTerminal(runId);
87
+ });
88
+
89
+ it('forces flush on terminal — run.completed arrives bundled in a batch BEFORE the timer fires', async () => {
90
+ const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
91
+ const runId = (create.json as { runId: string }).runId;
92
+
93
+ // Use a long bufferMs (4000ms) so the only flush before terminal
94
+ // would come from the force-flush rule. We measure elapsed time
95
+ // from subscribe-start to terminal-arrival; if force-flush works,
96
+ // it arrives in well under bufferMs/2 (i.e., the run completes +
97
+ // force-flush fires + we observe it before any timer-based flush
98
+ // could have happened). Without force-flush, terminal would either
99
+ // arrive AFTER bufferMs (timer-based delivery) OR not at all
100
+ // (stream closed before the timer fired).
101
+ const BUFFER_MS = 4000;
102
+ const startedAt = Date.now();
103
+ const result = await subscribe(
104
+ `/v1/runs/${encodeURIComponent(runId)}/events?streamMode=updates&bufferMs=${BUFFER_MS}`,
105
+ { timeoutMs: 30_000 },
106
+ );
107
+ const elapsedMs = Date.now() - startedAt;
108
+
109
+ const batchEvents = result.events.filter((e) => e.event === 'batch');
110
+ const allFlattened: RunEventDoc[] = batchEvents.flatMap(
111
+ (b) => JSON.parse(b.data) as RunEventDoc[],
112
+ );
113
+ const hasTerminal = allFlattened.some(
114
+ (e) => e.type === 'run.completed' || e.type === 'run.failed' || e.type === 'run.cancelled',
115
+ );
116
+
117
+ expect(hasTerminal, driver.describe(
118
+ 'stream-modes.md §Aggregation hint — force-flush triggers',
119
+ 'terminal events MUST be force-flushed; the stream MUST NOT close before delivering run.completed',
120
+ )).toBe(true);
121
+
122
+ // Force-flush fires immediately on terminal; without it, terminal
123
+ // would arrive ~bufferMs after the run actually completed. We allow
124
+ // bufferMs/2 as headroom for cold-start latency on the conformance
125
+ // server, but failing here proves the timer fired before terminal
126
+ // arrived (i.e., force-flush is broken).
127
+ expect(elapsedMs, driver.describe(
128
+ 'stream-modes.md §Aggregation hint — force-flush is immediate',
129
+ `terminal SHOULD arrive in well under bufferMs (${BUFFER_MS}ms); observed ${elapsedMs}ms — if elapsed is close to bufferMs, force-flush is not firing`,
130
+ )).toBeLessThan(BUFFER_MS / 2);
131
+ });
132
+
133
+ it('bufferMs=0 behaves identically to omitting (per-event mode)', async () => {
134
+ const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
135
+ const runId = (create.json as { runId: string }).runId;
136
+
137
+ const result = await subscribe(
138
+ `/v1/runs/${encodeURIComponent(runId)}/events?streamMode=updates&bufferMs=0`,
139
+ { timeoutMs: 30_000 },
140
+ );
141
+
142
+ const batchEvents = result.events.filter((e) => e.event === 'batch');
143
+ expect(batchEvents.length, driver.describe(
144
+ 'stream-modes.md §Aggregation hint — bufferMs=0 sentinel',
145
+ 'bufferMs=0 MUST behave identically to omitting (no batch frames)',
146
+ )).toBe(0);
147
+ });
148
+ });
@@ -0,0 +1,149 @@
1
+ /**
2
+ * Mixed-mode SSE scenarios (G2 / S4) — exercises comma-separated
3
+ * `?streamMode=` against the existing `conformance-delay` fixture.
4
+ *
5
+ * Verifies:
6
+ * 1. Server accepts `streamMode=updates,messages` (mixed subset).
7
+ * 2. Server rejects `streamMode=values,updates` with 400 +
8
+ * `unsupported_stream_mode` error envelope (values is exclusive).
9
+ * 3. Server rejects `streamMode=updates,bogus` (one bad mode → whole
10
+ * list fails).
11
+ * 4. Mixed mode sees AT LEAST every event the corresponding single
12
+ * mode would see (union semantics).
13
+ *
14
+ * Spec references:
15
+ * - stream-modes.md §Mixed mode (closes S4)
16
+ * - spec gap G2
17
+ */
18
+
19
+ import { describe, it, expect } from 'vitest';
20
+ import { driver } from '../lib/driver.js';
21
+ import { subscribe, type SseEvent } from '../lib/sse.js';
22
+ import { pollUntilTerminal } from '../lib/polling.js';
23
+ import { isFixtureAdvertised } from '../lib/fixtures.js';
24
+
25
+ const WORKFLOW_ID = 'conformance-delay';
26
+ const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
27
+
28
+ function eventTypes(events: readonly SseEvent[]): string[] {
29
+ return events.map((e) => e.event);
30
+ }
31
+
32
+ describe.skipIf(SKIP_NO_FIXTURE)('stream-modes-mixed: comma-separated subsets', () => {
33
+ it('accepts streamMode=updates,messages and emits a server-closed stream', async () => {
34
+ const create = await driver.post('/v1/runs', {
35
+ workflowId: WORKFLOW_ID,
36
+ inputs: { delayMs: 500 },
37
+ });
38
+ expect(create.status).toBe(201);
39
+ const runId = (create.json as { runId: string }).runId;
40
+
41
+ const result = await subscribe(
42
+ `/v1/runs/${encodeURIComponent(runId)}/events?streamMode=updates,messages`,
43
+ { timeoutMs: 15_000 },
44
+ );
45
+
46
+ expect(result.status, driver.describe(
47
+ 'stream-modes.md §Mixed mode',
48
+ 'streamMode=updates,messages MUST return 200',
49
+ )).toBe(200);
50
+
51
+ expect(result.closedBy, driver.describe(
52
+ 'stream-modes.md §Mixed mode + §updates',
53
+ 'server MUST close the stream on terminal run event',
54
+ )).toBe('server');
55
+
56
+ const types = eventTypes(result.events);
57
+ expect(types, driver.describe(
58
+ 'stream-modes.md §Mixed mode (union semantics)',
59
+ 'mixed updates,messages MUST include run.completed (admitted by updates)',
60
+ )).toContain('run.completed');
61
+ });
62
+
63
+ it('rejects streamMode=values,updates with 400 + unsupported_stream_mode', async () => {
64
+ const create = await driver.post('/v1/runs', {
65
+ workflowId: WORKFLOW_ID,
66
+ inputs: { delayMs: 100 },
67
+ });
68
+ const runId = (create.json as { runId: string }).runId;
69
+
70
+ const res = await driver.get(
71
+ `/v1/runs/${encodeURIComponent(runId)}/events?streamMode=values,updates`,
72
+ );
73
+ expect(res.status, driver.describe(
74
+ 'stream-modes.md §Mixed mode',
75
+ 'values combined with another mode MUST return 400',
76
+ )).toBe(400);
77
+
78
+ const body = res.json as
79
+ | { error?: string; message?: string; details?: { supported?: string[] } }
80
+ | undefined;
81
+ expect(body?.error, driver.describe(
82
+ 'stream-modes.md §Mode selection error envelope + error-envelope.schema.json',
83
+ 'unsupported_stream_mode error envelope MUST carry an `error` string discriminator',
84
+ )).toBe('unsupported_stream_mode');
85
+ expect(typeof body?.message, driver.describe(
86
+ 'error-envelope.schema.json',
87
+ 'error envelope MUST carry a human-readable `message` string',
88
+ )).toBe('string');
89
+ expect(Array.isArray(body?.details?.supported), driver.describe(
90
+ 'stream-modes.md §Mode selection error envelope',
91
+ 'error body MUST carry `details.supported` array (NOT top-level — `details` is the canonical contextual-data slot per error-envelope.schema.json)',
92
+ )).toBe(true);
93
+
94
+ await pollUntilTerminal(runId);
95
+ });
96
+
97
+ it('rejects streamMode=updates,bogus (one bad mode fails the whole list)', async () => {
98
+ const create = await driver.post('/v1/runs', {
99
+ workflowId: WORKFLOW_ID,
100
+ inputs: { delayMs: 100 },
101
+ });
102
+ const runId = (create.json as { runId: string }).runId;
103
+
104
+ const res = await driver.get(
105
+ `/v1/runs/${encodeURIComponent(runId)}/events?streamMode=updates,bogus`,
106
+ );
107
+ expect(res.status, driver.describe(
108
+ 'stream-modes.md §Mixed mode + §Mode selection',
109
+ 'partial-unknown lists MUST return 400',
110
+ )).toBe(400);
111
+
112
+ await pollUntilTerminal(runId);
113
+ });
114
+
115
+ it('mixed mode union: updates,debug sees every event updates sees', async () => {
116
+ // Run twice — once with updates only, once with updates,debug.
117
+ // The mixed-mode response MUST be a superset of the updates-only
118
+ // response (union semantics).
119
+ const r1 = await driver.post('/v1/runs', {
120
+ workflowId: WORKFLOW_ID,
121
+ inputs: { delayMs: 500 },
122
+ });
123
+ const runId1 = (r1.json as { runId: string }).runId;
124
+ const updatesOnly = await subscribe(
125
+ `/v1/runs/${encodeURIComponent(runId1)}/events?streamMode=updates`,
126
+ { timeoutMs: 15_000 },
127
+ );
128
+
129
+ const r2 = await driver.post('/v1/runs', {
130
+ workflowId: WORKFLOW_ID,
131
+ inputs: { delayMs: 500 },
132
+ });
133
+ const runId2 = (r2.json as { runId: string }).runId;
134
+ const mixed = await subscribe(
135
+ `/v1/runs/${encodeURIComponent(runId2)}/events?streamMode=updates,debug`,
136
+ { timeoutMs: 15_000 },
137
+ );
138
+
139
+ const updatesTypes = new Set(eventTypes(updatesOnly.events));
140
+ const mixedTypes = new Set(eventTypes(mixed.events));
141
+
142
+ for (const t of updatesTypes) {
143
+ expect(mixedTypes.has(t), driver.describe(
144
+ 'stream-modes.md §Mixed mode (union)',
145
+ `updates,debug MUST include every event type updates produces (missing: ${t})`,
146
+ )).toBe(true);
147
+ }
148
+ });
149
+ });
@@ -0,0 +1,139 @@
1
+ /**
2
+ * Stream-mode scenarios — exercises `GET /v1/runs/{runId}/events` SSE
3
+ * with different `streamMode` query parameters per stream-modes.md.
4
+ *
5
+ * Uses the `conformance-delay` fixture with a short delay (1s) so the
6
+ * stream has well-defined start + completion bounds without making
7
+ * tests slow.
8
+ */
9
+
10
+ import { describe, it, expect } from 'vitest';
11
+ import { driver } from '../lib/driver.js';
12
+ import { subscribe, type SseEvent } from '../lib/sse.js';
13
+ import { isFixtureAdvertised } from '../lib/fixtures.js';
14
+
15
+ const WORKFLOW_ID = 'conformance-delay';
16
+ const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
17
+
18
+ async function startDelayRun(delayMs: number): Promise<string> {
19
+ const create = await driver.post('/v1/runs', {
20
+ workflowId: WORKFLOW_ID,
21
+ inputs: { delayMs },
22
+ });
23
+ if (create.status !== 201) {
24
+ throw new Error(`Failed to start ${WORKFLOW_ID} run: ${create.status} ${create.text}`);
25
+ }
26
+ return (create.json as { runId: string }).runId;
27
+ }
28
+
29
+ function eventTypes(events: readonly SseEvent[]): string[] {
30
+ return events.map((e) => e.event);
31
+ }
32
+
33
+ describe.skipIf(SKIP_NO_FIXTURE)('stream-modes: updates (default) closes on terminal event', () => {
34
+ it('emits at least run.started + run.completed and server closes the stream', async () => {
35
+ const runId = await startDelayRun(1_000);
36
+ const { events, closedBy } = await subscribe(
37
+ `/v1/runs/${encodeURIComponent(runId)}/events?streamMode=updates`,
38
+ { timeoutMs: 15_000 },
39
+ );
40
+
41
+ expect(closedBy, driver.describe(
42
+ 'stream-modes.md §updates',
43
+ 'server MUST close the connection on terminal run event',
44
+ )).toBe('server');
45
+
46
+ const types = eventTypes(events);
47
+ expect(types, driver.describe(
48
+ 'stream-modes.md §updates',
49
+ 'updates stream MUST include run.started',
50
+ )).toContain('run.started');
51
+ expect(types, driver.describe(
52
+ 'stream-modes.md §updates',
53
+ 'updates stream MUST include run.completed for a successful run',
54
+ )).toContain('run.completed');
55
+ });
56
+ });
57
+
58
+ describe.skipIf(SKIP_NO_FIXTURE)('stream-modes: invalid streamMode is rejected', () => {
59
+ it('returns 400 and a structured error body', async () => {
60
+ const runId = await startDelayRun(1_000);
61
+ const res = await driver.get(
62
+ `/v1/runs/${encodeURIComponent(runId)}/events?streamMode=does-not-exist`,
63
+ );
64
+
65
+ expect(res.status, driver.describe(
66
+ 'stream-modes.md §Mode selection',
67
+ 'unsupported streamMode MUST return 400',
68
+ )).toBe(400);
69
+
70
+ const body = res.json as
71
+ | { error?: unknown; message?: unknown; details?: { supported?: unknown } }
72
+ | undefined;
73
+ expect(typeof body?.error, driver.describe(
74
+ 'stream-modes.md §Mode selection + error-envelope.schema.json',
75
+ 'unsupported_stream_mode error body MUST include `error` string discriminator',
76
+ )).toBe('string');
77
+ expect(typeof body?.message, driver.describe(
78
+ 'error-envelope.schema.json',
79
+ 'error envelope MUST include a human-readable `message` string',
80
+ )).toBe('string');
81
+ expect(Array.isArray(body?.details?.supported), driver.describe(
82
+ 'stream-modes.md §Mode selection',
83
+ 'error body MUST include `details.supported` array of mode names (under `details` per error-envelope.schema.json)',
84
+ )).toBe(true);
85
+ });
86
+ });
87
+
88
+ describe.skipIf(SKIP_NO_FIXTURE)('stream-modes: values mode is reachable + closes on terminal', () => {
89
+ it('returns 200 + emits at least one event + server-closes per stream-modes.md §values', async () => {
90
+ const runId = await startDelayRun(1_000);
91
+ const result = await subscribe(
92
+ `/v1/runs/${encodeURIComponent(runId)}/events?streamMode=values`,
93
+ { timeoutMs: 15_000 },
94
+ );
95
+
96
+ // The state.snapshot payload schema is implementation-shaped per
97
+ // spec gap S1, so we don't assert payload shape here. What's
98
+ // canonical: the connection MUST be reachable, MUST emit at least
99
+ // one event before terminal, AND the server MUST close on terminal.
100
+ expect(result.closedBy, driver.describe(
101
+ 'stream-modes.md §values',
102
+ 'server MUST close the connection on terminal run event',
103
+ )).toBe('server');
104
+
105
+ expect(result.events.length, driver.describe(
106
+ 'stream-modes.md §values',
107
+ 'values mode MUST emit at least one event before terminal',
108
+ )).toBeGreaterThan(0);
109
+ });
110
+ });
111
+
112
+ describe.skipIf(SKIP_NO_FIXTURE)('stream-modes: debug emits at least as many events as updates', () => {
113
+ it('debug stream is a superset of updates per stream-modes.md mode-mapping', async () => {
114
+ const runIdUpdates = await startDelayRun(1_000);
115
+ const updatesResult = await subscribe(
116
+ `/v1/runs/${encodeURIComponent(runIdUpdates)}/events?streamMode=updates`,
117
+ { timeoutMs: 15_000 },
118
+ );
119
+
120
+ const runIdDebug = await startDelayRun(1_000);
121
+ const debugResult = await subscribe(
122
+ `/v1/runs/${encodeURIComponent(runIdDebug)}/events?streamMode=debug`,
123
+ { timeoutMs: 15_000 },
124
+ );
125
+
126
+ // Both runs are conformance-delay with the same input, so updates
127
+ // events (run.started, node.started not in updates per spec, node.completed,
128
+ // run.completed) should be a subset of debug events.
129
+ expect(debugResult.events.length, driver.describe(
130
+ 'stream-modes.md mode-to-event mapping',
131
+ 'debug stream event count MUST be >= updates stream event count',
132
+ )).toBeGreaterThanOrEqual(updatesResult.events.length);
133
+
134
+ expect(debugResult.closedBy, driver.describe(
135
+ 'stream-modes.md §debug',
136
+ 'debug stream MUST close on terminal event',
137
+ )).toBe('server');
138
+ });
139
+ });