@durable-streams/client-conformance-tests 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1400 @@
1
+ "use strict";
2
+ const require_chunk = require('./chunk-BCwAaXi7.cjs');
3
+ const require_protocol = require('./protocol-XeAOKBD-.cjs');
4
+ const node_child_process = require_chunk.__toESM(require("node:child_process"));
5
+ const node_readline = require_chunk.__toESM(require("node:readline"));
6
+ const node_crypto = require_chunk.__toESM(require("node:crypto"));
7
+ const __durable_streams_server = require_chunk.__toESM(require("@durable-streams/server"));
8
+ const node_fs = require_chunk.__toESM(require("node:fs"));
9
+ const node_path = require_chunk.__toESM(require("node:path"));
10
+ const yaml = require_chunk.__toESM(require("yaml"));
11
+ const __durable_streams_client = require_chunk.__toESM(require("@durable-streams/client"));
12
+
13
+ //#region src/test-cases.ts
14
+ /**
15
+ * Load all test suites from a directory.
16
+ */
17
+ function loadTestSuites(dir) {
18
+ const suites = [];
19
+ function walkDir(currentDir) {
20
+ const entries = node_fs.readdirSync(currentDir, { withFileTypes: true });
21
+ for (const entry of entries) {
22
+ const fullPath = node_path.join(currentDir, entry.name);
23
+ if (entry.isDirectory()) walkDir(fullPath);
24
+ else if (entry.isFile() && (entry.name.endsWith(`.yaml`) || entry.name.endsWith(`.yml`))) {
25
+ const content = node_fs.readFileSync(fullPath, `utf-8`);
26
+ const suite = yaml.default.parse(content);
27
+ suites.push(suite);
28
+ }
29
+ }
30
+ }
31
+ walkDir(dir);
32
+ return suites;
33
+ }
34
+ /**
35
+ * Load test suites from the embedded test-cases directory.
36
+ */
37
+ function loadEmbeddedTestSuites() {
38
+ const testCasesDir = node_path.join(__dirname, `..`, `test-cases`);
39
+ return loadTestSuites(testCasesDir);
40
+ }
41
+ /**
42
+ * Filter test suites by category.
43
+ */
44
+ function filterByCategory(suites, category) {
45
+ return suites.filter((s) => s.category === category);
46
+ }
47
+ /**
48
+ * Get total test count.
49
+ */
50
+ function countTests(suites) {
51
+ return suites.reduce((sum, suite) => sum + suite.tests.length, 0);
52
+ }
53
+
54
+ //#endregion
55
+ //#region src/runner.ts
56
+ var ClientAdapter = class {
57
+ process;
58
+ readline;
59
+ pendingResponse = null;
60
+ initialized = false;
61
+ constructor(executable, args = []) {
62
+ this.process = (0, node_child_process.spawn)(executable, args, { stdio: [
63
+ `pipe`,
64
+ `pipe`,
65
+ `pipe`
66
+ ] });
67
+ if (!this.process.stdout || !this.process.stdin) throw new Error(`Failed to create client adapter process`);
68
+ this.readline = (0, node_readline.createInterface)({
69
+ input: this.process.stdout,
70
+ crlfDelay: Infinity
71
+ });
72
+ this.readline.on(`line`, (line) => {
73
+ if (this.pendingResponse) {
74
+ try {
75
+ const result = require_protocol.parseResult(line);
76
+ this.pendingResponse.resolve(result);
77
+ } catch {
78
+ this.pendingResponse.reject(new Error(`Failed to parse client response: ${line}`));
79
+ }
80
+ this.pendingResponse = null;
81
+ }
82
+ });
83
+ this.process.stderr?.on(`data`, (data) => {
84
+ console.error(`[client stderr] ${data.toString().trim()}`);
85
+ });
86
+ this.process.on(`error`, (err) => {
87
+ if (this.pendingResponse) {
88
+ this.pendingResponse.reject(err);
89
+ this.pendingResponse = null;
90
+ }
91
+ });
92
+ this.process.on(`exit`, (code) => {
93
+ if (this.pendingResponse) {
94
+ this.pendingResponse.reject(new Error(`Client adapter exited with code ${code}`));
95
+ this.pendingResponse = null;
96
+ }
97
+ });
98
+ }
99
+ async send(command, timeoutMs = 3e4) {
100
+ if (!this.process.stdin) throw new Error(`Client adapter stdin not available`);
101
+ return new Promise((resolve, reject) => {
102
+ const timeout = setTimeout(() => {
103
+ this.pendingResponse = null;
104
+ reject(new Error(`Command timed out after ${timeoutMs}ms: ${command.type}`));
105
+ }, timeoutMs);
106
+ this.pendingResponse = {
107
+ resolve: (result) => {
108
+ clearTimeout(timeout);
109
+ resolve(result);
110
+ },
111
+ reject: (error) => {
112
+ clearTimeout(timeout);
113
+ reject(error);
114
+ }
115
+ };
116
+ const line = require_protocol.serializeCommand(command) + `\n`;
117
+ this.process.stdin.write(line);
118
+ });
119
+ }
120
+ async init(serverUrl) {
121
+ const result = await this.send({
122
+ type: `init`,
123
+ serverUrl
124
+ });
125
+ if (result.success) this.initialized = true;
126
+ return result;
127
+ }
128
+ async shutdown() {
129
+ if (this.initialized) try {
130
+ await this.send({ type: `shutdown` }, 5e3);
131
+ } catch {}
132
+ this.process.kill();
133
+ this.readline.close();
134
+ }
135
+ isInitialized() {
136
+ return this.initialized;
137
+ }
138
+ };
139
+ function resolveVariables(value, variables) {
140
+ return value.replace(/\$\{([^}]+)\}/g, (match, expr) => {
141
+ if (expr === `randomUUID`) return (0, node_crypto.randomUUID)();
142
+ const parts = expr.split(`.`);
143
+ let current = variables.get(parts[0]);
144
+ if (current === void 0) throw new Error(`Undefined variable: ${parts[0]} (in ${match})`);
145
+ for (let i = 1; i < parts.length && current != null; i++) current = current[parts[i]];
146
+ return String(current ?? ``);
147
+ });
148
+ }
149
+ function generateStreamPath() {
150
+ return `/test-stream-${(0, node_crypto.randomUUID)()}`;
151
+ }
152
+ async function executeOperation(op, ctx) {
153
+ const { client, variables, verbose, commandTimeout } = ctx;
154
+ switch (op.action) {
155
+ case `create`: {
156
+ const path = op.path ? resolveVariables(op.path, variables) : generateStreamPath();
157
+ if (op.as) variables.set(op.as, path);
158
+ const result = await client.send({
159
+ type: `create`,
160
+ path,
161
+ contentType: op.contentType,
162
+ ttlSeconds: op.ttlSeconds,
163
+ expiresAt: op.expiresAt,
164
+ headers: op.headers
165
+ }, commandTimeout);
166
+ if (verbose) console.log(` create ${path}: ${result.success ? `ok` : `failed`}`);
167
+ return { result };
168
+ }
169
+ case `connect`: {
170
+ const path = resolveVariables(op.path, variables);
171
+ const result = await client.send({
172
+ type: `connect`,
173
+ path,
174
+ headers: op.headers
175
+ }, commandTimeout);
176
+ if (verbose) console.log(` connect ${path}: ${result.success ? `ok` : `failed`}`);
177
+ return { result };
178
+ }
179
+ case `append`: {
180
+ const path = resolveVariables(op.path, variables);
181
+ const data = op.data ? resolveVariables(op.data, variables) : ``;
182
+ const result = await client.send({
183
+ type: `append`,
184
+ path,
185
+ data: op.binaryData ?? data,
186
+ binary: !!op.binaryData,
187
+ seq: op.seq,
188
+ headers: op.headers
189
+ }, commandTimeout);
190
+ if (verbose) console.log(` append ${path}: ${result.success ? `ok` : `failed`}`);
191
+ if (result.success && result.type === `append` && op.expect?.storeOffsetAs) variables.set(op.expect.storeOffsetAs, result.offset);
192
+ return { result };
193
+ }
194
+ case `append-batch`: {
195
+ const path = resolveVariables(op.path, variables);
196
+ const results = [];
197
+ for (const item of op.items) {
198
+ const result = await client.send({
199
+ type: `append`,
200
+ path,
201
+ data: item.binaryData ?? item.data ?? ``,
202
+ binary: !!item.binaryData,
203
+ seq: item.seq,
204
+ headers: op.headers
205
+ }, commandTimeout);
206
+ results.push(result);
207
+ }
208
+ if (verbose) {
209
+ const succeeded = results.filter((r) => r.success).length;
210
+ console.log(` append-batch ${path}: ${succeeded}/${results.length} succeeded`);
211
+ }
212
+ const allSucceeded = results.every((r) => r.success);
213
+ return { result: {
214
+ type: `append`,
215
+ success: allSucceeded,
216
+ status: allSucceeded ? 200 : 207
217
+ } };
218
+ }
219
+ case `read`: {
220
+ const path = resolveVariables(op.path, variables);
221
+ const offset = op.offset ? resolveVariables(op.offset, variables) : void 0;
222
+ if (op.background && op.as) {
223
+ const resultPromise = client.send({
224
+ type: `read`,
225
+ path,
226
+ offset,
227
+ live: op.live,
228
+ timeoutMs: op.timeoutMs,
229
+ maxChunks: op.maxChunks,
230
+ waitForUpToDate: op.waitForUpToDate,
231
+ headers: op.headers
232
+ }, commandTimeout);
233
+ ctx.backgroundOps.set(op.as, resultPromise);
234
+ if (verbose) console.log(` read ${path}: started in background as ${op.as}`);
235
+ return {};
236
+ }
237
+ const result = await client.send({
238
+ type: `read`,
239
+ path,
240
+ offset,
241
+ live: op.live,
242
+ timeoutMs: op.timeoutMs,
243
+ maxChunks: op.maxChunks,
244
+ waitForUpToDate: op.waitForUpToDate,
245
+ headers: op.headers
246
+ }, commandTimeout);
247
+ if (verbose) console.log(` read ${path}: ${result.success ? `ok` : `failed`}`);
248
+ if (result.success && result.type === `read`) {
249
+ if (op.expect?.storeOffsetAs) variables.set(op.expect.storeOffsetAs, result.offset);
250
+ if (op.expect?.storeDataAs) {
251
+ const data = result.chunks.map((c) => c.data).join(``);
252
+ variables.set(op.expect.storeDataAs, data);
253
+ }
254
+ }
255
+ return { result };
256
+ }
257
+ case `head`: {
258
+ const path = resolveVariables(op.path, variables);
259
+ const result = await client.send({
260
+ type: `head`,
261
+ path,
262
+ headers: op.headers
263
+ }, commandTimeout);
264
+ if (verbose) console.log(` head ${path}: ${result.success ? `ok` : `failed`}`);
265
+ if (result.success && op.expect?.storeAs) variables.set(op.expect.storeAs, result);
266
+ return { result };
267
+ }
268
+ case `delete`: {
269
+ const path = resolveVariables(op.path, variables);
270
+ const result = await client.send({
271
+ type: `delete`,
272
+ path,
273
+ headers: op.headers
274
+ }, commandTimeout);
275
+ if (verbose) console.log(` delete ${path}: ${result.success ? `ok` : `failed`}`);
276
+ return { result };
277
+ }
278
+ case `wait`: {
279
+ await new Promise((resolve) => setTimeout(resolve, op.ms));
280
+ return {};
281
+ }
282
+ case `set`: {
283
+ const value = resolveVariables(op.value, variables);
284
+ variables.set(op.name, value);
285
+ return {};
286
+ }
287
+ case `assert`: {
288
+ if (op.equals) {
289
+ const left = resolveVariables(op.equals.left, variables);
290
+ const right = resolveVariables(op.equals.right, variables);
291
+ if (left !== right) return { error: op.message ?? `Assertion failed: expected "${left}" to equal "${right}"` };
292
+ }
293
+ if (op.notEquals) {
294
+ const left = resolveVariables(op.notEquals.left, variables);
295
+ const right = resolveVariables(op.notEquals.right, variables);
296
+ if (left === right) return { error: op.message ?? `Assertion failed: expected "${left}" to not equal "${right}"` };
297
+ }
298
+ if (op.contains) {
299
+ const value = resolveVariables(op.contains.value, variables);
300
+ const substring = resolveVariables(op.contains.substring, variables);
301
+ if (!value.includes(substring)) return { error: op.message ?? `Assertion failed: expected "${value}" to contain "${substring}"` };
302
+ }
303
+ if (op.matches) {
304
+ const value = resolveVariables(op.matches.value, variables);
305
+ const pattern = op.matches.pattern;
306
+ try {
307
+ const regex = new RegExp(pattern);
308
+ if (!regex.test(value)) return { error: op.message ?? `Assertion failed: expected "${value}" to match /${pattern}/` };
309
+ } catch {
310
+ return { error: `Invalid regex pattern: ${pattern}` };
311
+ }
312
+ }
313
+ return {};
314
+ }
315
+ case `server-append`: {
316
+ const path = resolveVariables(op.path, variables);
317
+ const data = resolveVariables(op.data, variables);
318
+ try {
319
+ const headResponse = await fetch(`${ctx.serverUrl}${path}`, { method: `HEAD` });
320
+ const contentType = headResponse.headers.get(`content-type`) ?? `application/octet-stream`;
321
+ const response = await fetch(`${ctx.serverUrl}${path}`, {
322
+ method: `POST`,
323
+ body: data,
324
+ headers: {
325
+ "content-type": contentType,
326
+ ...op.headers
327
+ }
328
+ });
329
+ if (verbose) console.log(` server-append ${path}: ${response.ok ? `ok` : `failed (${response.status})`}`);
330
+ if (!response.ok) return { error: `Server append failed with status ${response.status}` };
331
+ return {};
332
+ } catch (err) {
333
+ return { error: `Server append failed: ${err instanceof Error ? err.message : String(err)}` };
334
+ }
335
+ }
336
+ case `await`: {
337
+ const ref = op.ref;
338
+ const promise = ctx.backgroundOps.get(ref);
339
+ if (!promise) return { error: `No background operation found with ref: ${ref}` };
340
+ const result = await promise;
341
+ ctx.backgroundOps.delete(ref);
342
+ if (verbose) console.log(` await ${ref}: ${result.success ? `ok` : `failed`}`);
343
+ return { result };
344
+ }
345
+ case `inject-error`: {
346
+ const path = resolveVariables(op.path, variables);
347
+ try {
348
+ const response = await fetch(`${ctx.serverUrl}/_test/inject-error`, {
349
+ method: `POST`,
350
+ headers: { "content-type": `application/json` },
351
+ body: JSON.stringify({
352
+ path,
353
+ status: op.status,
354
+ count: op.count ?? 1,
355
+ retryAfter: op.retryAfter
356
+ })
357
+ });
358
+ if (verbose) console.log(` inject-error ${path} ${op.status}x${op.count ?? 1}: ${response.ok ? `ok` : `failed`}`);
359
+ if (!response.ok) return { error: `Failed to inject error: ${response.status}` };
360
+ return {};
361
+ } catch (err) {
362
+ return { error: `Failed to inject error: ${err instanceof Error ? err.message : String(err)}` };
363
+ }
364
+ }
365
+ case `clear-errors`: try {
366
+ const response = await fetch(`${ctx.serverUrl}/_test/inject-error`, { method: `DELETE` });
367
+ if (verbose) console.log(` clear-errors: ${response.ok ? `ok` : `failed`}`);
368
+ return {};
369
+ } catch (err) {
370
+ return { error: `Failed to clear errors: ${err instanceof Error ? err.message : String(err)}` };
371
+ }
372
+ case `set-dynamic-header`: {
373
+ const result = await client.send({
374
+ type: `set-dynamic-header`,
375
+ name: op.name,
376
+ valueType: op.valueType,
377
+ initialValue: op.initialValue
378
+ }, commandTimeout);
379
+ if (verbose) console.log(` set-dynamic-header ${op.name}: ${result.success ? `ok` : `failed`}`);
380
+ return { result };
381
+ }
382
+ case `set-dynamic-param`: {
383
+ const result = await client.send({
384
+ type: `set-dynamic-param`,
385
+ name: op.name,
386
+ valueType: op.valueType
387
+ }, commandTimeout);
388
+ if (verbose) console.log(` set-dynamic-param ${op.name}: ${result.success ? `ok` : `failed`}`);
389
+ return { result };
390
+ }
391
+ case `clear-dynamic`: {
392
+ const result = await client.send({ type: `clear-dynamic` }, commandTimeout);
393
+ if (verbose) console.log(` clear-dynamic: ${result.success ? `ok` : `failed`}`);
394
+ return { result };
395
+ }
396
+ default: return { error: `Unknown operation: ${op.action}` };
397
+ }
398
+ }
399
+ function isReadResult(result) {
400
+ return result.type === `read` && result.success;
401
+ }
402
+ function isAppendResult(result) {
403
+ return result.type === `append` && result.success;
404
+ }
405
+ function isHeadResult(result) {
406
+ return result.type === `head` && result.success;
407
+ }
408
+ function isErrorResult(result) {
409
+ return result.type === `error` && !result.success;
410
+ }
411
+ function validateExpectation(result, expect) {
412
+ if (!expect) return null;
413
+ if (expect.status !== void 0 && `status` in result) {
414
+ if (result.status !== expect.status) return `Expected status ${expect.status}, got ${result.status}`;
415
+ }
416
+ if (expect.errorCode !== void 0) {
417
+ if (result.success) return `Expected error ${expect.errorCode}, but operation succeeded`;
418
+ if (isErrorResult(result) && result.errorCode !== expect.errorCode) return `Expected error code ${expect.errorCode}, got ${result.errorCode}`;
419
+ }
420
+ if (expect.data !== void 0 && isReadResult(result)) {
421
+ const actualData = result.chunks.map((c) => c.data).join(``);
422
+ if (actualData !== expect.data) return `Expected data "${expect.data}", got "${actualData}"`;
423
+ }
424
+ if (expect.dataContains !== void 0 && isReadResult(result)) {
425
+ const actualData = result.chunks.map((c) => c.data).join(``);
426
+ if (!actualData.includes(expect.dataContains)) return `Expected data to contain "${expect.dataContains}", got "${actualData}"`;
427
+ }
428
+ if (expect.dataContainsAll !== void 0 && isReadResult(result)) {
429
+ const actualData = result.chunks.map((c) => c.data).join(``);
430
+ const missing = expect.dataContainsAll.filter((s) => !actualData.includes(s));
431
+ if (missing.length > 0) return `Expected data to contain all of [${expect.dataContainsAll.join(`, `)}], missing: [${missing.join(`, `)}]`;
432
+ }
433
+ if (expect.upToDate !== void 0 && isReadResult(result)) {
434
+ if (result.upToDate !== expect.upToDate) return `Expected upToDate=${expect.upToDate}, got ${result.upToDate}`;
435
+ }
436
+ if (expect.chunkCount !== void 0 && isReadResult(result)) {
437
+ if (result.chunks.length !== expect.chunkCount) return `Expected ${expect.chunkCount} chunks, got ${result.chunks.length}`;
438
+ }
439
+ if (expect.minChunks !== void 0 && isReadResult(result)) {
440
+ if (result.chunks.length < expect.minChunks) return `Expected at least ${expect.minChunks} chunks, got ${result.chunks.length}`;
441
+ }
442
+ if (expect.contentType !== void 0 && isHeadResult(result)) {
443
+ if (result.contentType !== expect.contentType) return `Expected contentType "${expect.contentType}", got "${result.contentType}"`;
444
+ }
445
+ if (expect.hasOffset !== void 0 && isHeadResult(result)) {
446
+ const hasOffset = result.offset !== void 0 && result.offset !== ``;
447
+ if (hasOffset !== expect.hasOffset) return `Expected hasOffset=${expect.hasOffset}, got ${hasOffset}`;
448
+ }
449
+ if (expect.headersSent !== void 0) {
450
+ const expectedHeaders = expect.headersSent;
451
+ let actualHeaders;
452
+ if (isAppendResult(result)) actualHeaders = result.headersSent;
453
+ else if (isReadResult(result)) actualHeaders = result.headersSent;
454
+ if (!actualHeaders) return `Expected headersSent but result does not contain headersSent`;
455
+ for (const [key, expectedValue] of Object.entries(expectedHeaders)) {
456
+ const actualValue = actualHeaders[key];
457
+ if (actualValue !== expectedValue) return `Expected headersSent[${key}]="${expectedValue}", got "${actualValue ?? `undefined`}"`;
458
+ }
459
+ }
460
+ if (expect.paramsSent !== void 0) {
461
+ const expectedParams = expect.paramsSent;
462
+ let actualParams;
463
+ if (isAppendResult(result)) actualParams = result.paramsSent;
464
+ else if (isReadResult(result)) actualParams = result.paramsSent;
465
+ if (!actualParams) return `Expected paramsSent but result does not contain paramsSent`;
466
+ for (const [key, expectedValue] of Object.entries(expectedParams)) {
467
+ const actualValue = actualParams[key];
468
+ if (actualValue !== expectedValue) return `Expected paramsSent[${key}]="${expectedValue}", got "${actualValue ?? `undefined`}"`;
469
+ }
470
+ }
471
+ return null;
472
+ }
473
+ /**
474
+ * Map feature names from YAML (kebab-case) to client feature property names (camelCase).
475
+ */
476
+ function featureToProperty(feature) {
477
+ const map = {
478
+ batching: `batching`,
479
+ sse: `sse`,
480
+ "long-poll": `longPoll`,
481
+ longPoll: `longPoll`,
482
+ streaming: `streaming`,
483
+ dynamicHeaders: `dynamicHeaders`,
484
+ "dynamic-headers": `dynamicHeaders`
485
+ };
486
+ return map[feature];
487
+ }
488
+ /**
489
+ * Check if client supports all required features.
490
+ * Returns list of missing features, or empty array if all satisfied.
491
+ */
492
+ function getMissingFeatures(requires, clientFeatures) {
493
+ if (!requires || requires.length === 0) return [];
494
+ return requires.filter((feature) => {
495
+ const prop = featureToProperty(feature);
496
+ return !prop || !clientFeatures[prop];
497
+ });
498
+ }
499
+ async function runTestCase(test, ctx) {
500
+ const startTime = Date.now();
501
+ if (test.skip) return {
502
+ suite: ``,
503
+ test: test.id,
504
+ passed: true,
505
+ duration: 0,
506
+ skipped: true,
507
+ skipReason: typeof test.skip === `string` ? test.skip : void 0
508
+ };
509
+ const missingFeatures = getMissingFeatures(test.requires, ctx.clientFeatures);
510
+ if (missingFeatures.length > 0) return {
511
+ suite: ``,
512
+ test: test.id,
513
+ passed: true,
514
+ duration: 0,
515
+ skipped: true,
516
+ skipReason: `missing features: ${missingFeatures.join(`, `)}`
517
+ };
518
+ ctx.variables.clear();
519
+ ctx.backgroundOps.clear();
520
+ try {
521
+ if (test.setup) for (const op of test.setup) {
522
+ const { error } = await executeOperation(op, ctx);
523
+ if (error) return {
524
+ suite: ``,
525
+ test: test.id,
526
+ passed: false,
527
+ duration: Date.now() - startTime,
528
+ error: `Setup failed: ${error}`
529
+ };
530
+ }
531
+ for (const op of test.operations) {
532
+ const { result, error } = await executeOperation(op, ctx);
533
+ if (error) return {
534
+ suite: ``,
535
+ test: test.id,
536
+ passed: false,
537
+ duration: Date.now() - startTime,
538
+ error
539
+ };
540
+ if (result && `expect` in op && op.expect) {
541
+ const validationError = validateExpectation(result, op.expect);
542
+ if (validationError) return {
543
+ suite: ``,
544
+ test: test.id,
545
+ passed: false,
546
+ duration: Date.now() - startTime,
547
+ error: validationError
548
+ };
549
+ }
550
+ }
551
+ if (test.cleanup) for (const op of test.cleanup) try {
552
+ await executeOperation(op, ctx);
553
+ } catch {}
554
+ return {
555
+ suite: ``,
556
+ test: test.id,
557
+ passed: true,
558
+ duration: Date.now() - startTime
559
+ };
560
+ } catch (err) {
561
+ return {
562
+ suite: ``,
563
+ test: test.id,
564
+ passed: false,
565
+ duration: Date.now() - startTime,
566
+ error: err instanceof Error ? err.message : String(err)
567
+ };
568
+ }
569
+ }
570
+ async function runConformanceTests(options) {
571
+ const startTime = Date.now();
572
+ const results = [];
573
+ let suites = loadEmbeddedTestSuites();
574
+ if (options.suites) suites = suites.filter((s) => options.suites.includes(s.category));
575
+ if (options.tags) suites = suites.map((suite) => ({
576
+ ...suite,
577
+ tests: suite.tests.filter((test) => test.tags?.some((t) => options.tags.includes(t)) || suite.tags?.some((t) => options.tags.includes(t)))
578
+ })).filter((suite) => suite.tests.length > 0);
579
+ const totalTests = countTests(suites);
580
+ console.log(`\nRunning ${totalTests} client conformance tests...\n`);
581
+ const server = new __durable_streams_server.DurableStreamTestServer({ port: options.serverPort ?? 0 });
582
+ await server.start();
583
+ const serverUrl = server.url;
584
+ console.log(`Reference server started at ${serverUrl}\n`);
585
+ let adapterPath = options.clientAdapter;
586
+ let adapterArgs = options.clientArgs ?? [];
587
+ if (adapterPath === `ts` || adapterPath === `typescript`) {
588
+ adapterPath = `npx`;
589
+ adapterArgs = [`tsx`, new URL(
590
+ `./adapters/typescript-adapter.ts`,
591
+ // Multi-status
592
+ // No result yet - will be retrieved via await
593
+ // Clean up
594
+ require("url").pathToFileURL(__filename).href
595
+ ).pathname];
596
+ }
597
+ const client = new ClientAdapter(adapterPath, adapterArgs);
598
+ try {
599
+ const initResult = await client.init(serverUrl);
600
+ if (!initResult.success) throw new Error(`Failed to initialize client adapter: ${initResult.message}`);
601
+ let clientFeatures = {};
602
+ if (initResult.type === `init`) {
603
+ console.log(`Client: ${initResult.clientName} v${initResult.clientVersion}`);
604
+ if (initResult.features) {
605
+ clientFeatures = initResult.features;
606
+ const featureList = Object.entries(initResult.features).filter(([, v]) => v).map(([k]) => k);
607
+ console.log(`Features: ${featureList.join(`, `) || `none`}\n`);
608
+ }
609
+ }
610
+ const ctx = {
611
+ serverUrl,
612
+ variables: new Map(),
613
+ client,
614
+ verbose: options.verbose ?? false,
615
+ clientFeatures,
616
+ backgroundOps: new Map(),
617
+ commandTimeout: options.testTimeout ?? 3e4
618
+ };
619
+ for (const suite of suites) {
620
+ console.log(`\n${suite.name}`);
621
+ console.log(`─`.repeat(suite.name.length));
622
+ const suiteMissingFeatures = getMissingFeatures(suite.requires, clientFeatures);
623
+ for (const test of suite.tests) {
624
+ if (suiteMissingFeatures.length > 0) {
625
+ const result$1 = {
626
+ suite: suite.id,
627
+ test: test.id,
628
+ passed: true,
629
+ duration: 0,
630
+ skipped: true,
631
+ skipReason: `missing features: ${suiteMissingFeatures.join(`, `)}`
632
+ };
633
+ results.push(result$1);
634
+ console.log(` ○ ${test.name} (skipped: missing features: ${suiteMissingFeatures.join(`, `)})`);
635
+ continue;
636
+ }
637
+ const result = await runTestCase(test, ctx);
638
+ result.suite = suite.id;
639
+ results.push(result);
640
+ const icon = result.passed ? result.skipped ? `○` : `✓` : `✗`;
641
+ const status = result.skipped ? `skipped${result.skipReason ? `: ${result.skipReason}` : ``}` : result.passed ? `${result.duration}ms` : result.error;
642
+ console.log(` ${icon} ${test.name} (${status})`);
643
+ if (options.failFast && !result.passed && !result.skipped) break;
644
+ }
645
+ if (options.failFast && results.some((r) => !r.passed && !r.skipped)) break;
646
+ }
647
+ } finally {
648
+ await client.shutdown();
649
+ await server.stop();
650
+ }
651
+ const passed = results.filter((r) => r.passed && !r.skipped).length;
652
+ const failed = results.filter((r) => !r.passed).length;
653
+ const skipped = results.filter((r) => r.skipped).length;
654
+ const summary = {
655
+ total: results.length,
656
+ passed,
657
+ failed,
658
+ skipped,
659
+ duration: Date.now() - startTime,
660
+ results
661
+ };
662
+ console.log(`\n` + `═`.repeat(40));
663
+ console.log(`Total: ${summary.total} tests`);
664
+ console.log(`Passed: ${summary.passed}`);
665
+ console.log(`Failed: ${summary.failed}`);
666
+ console.log(`Skipped: ${summary.skipped}`);
667
+ console.log(`Duration: ${(summary.duration / 1e3).toFixed(2)}s`);
668
+ console.log(`═`.repeat(40) + `\n`);
669
+ if (failed > 0) {
670
+ console.log(`Failed tests:`);
671
+ for (const result of results.filter((r) => !r.passed)) console.log(` - ${result.suite}/${result.test}: ${result.error}`);
672
+ console.log();
673
+ }
674
+ return summary;
675
+ }
676
+
677
+ //#endregion
678
+ //#region src/benchmark-scenarios.ts
679
+ const appendLatencyScenario = {
680
+ id: `latency-append`,
681
+ name: `Append Latency`,
682
+ description: `Measure time to complete a single append operation`,
683
+ category: `latency`,
684
+ config: {
685
+ warmupIterations: 10,
686
+ measureIterations: 100,
687
+ messageSize: 100
688
+ },
689
+ criteria: {
690
+ maxP50Ms: 20,
691
+ maxP99Ms: 100
692
+ },
693
+ createOperation: (ctx) => ({
694
+ op: `append`,
695
+ path: `${ctx.basePath}/stream`,
696
+ size: 100
697
+ }),
698
+ setup: (ctx) => {
699
+ ctx.setupData.streamPath = `${ctx.basePath}/stream`;
700
+ return Promise.resolve({});
701
+ }
702
+ };
703
+ const readLatencyScenario = {
704
+ id: `latency-read`,
705
+ name: `Read Latency`,
706
+ description: `Measure time to complete a single read operation`,
707
+ category: `latency`,
708
+ config: {
709
+ warmupIterations: 10,
710
+ measureIterations: 100,
711
+ messageSize: 100
712
+ },
713
+ criteria: {
714
+ maxP50Ms: 20,
715
+ maxP99Ms: 100
716
+ },
717
+ createOperation: (ctx) => ({
718
+ op: `read`,
719
+ path: `${ctx.basePath}/stream`,
720
+ offset: ctx.setupData.offset
721
+ }),
722
+ setup: (ctx) => {
723
+ ctx.setupData.streamPath = `${ctx.basePath}/stream`;
724
+ return Promise.resolve({});
725
+ }
726
+ };
727
+ const roundtripLatencyScenario = {
728
+ id: `latency-roundtrip`,
729
+ name: `Roundtrip Latency`,
730
+ description: `Measure time to append and immediately read back via long-poll`,
731
+ category: `latency`,
732
+ requires: [`longPoll`],
733
+ config: {
734
+ warmupIterations: 5,
735
+ measureIterations: 50,
736
+ messageSize: 100
737
+ },
738
+ criteria: {
739
+ maxP50Ms: 50,
740
+ maxP99Ms: 200
741
+ },
742
+ createOperation: (ctx) => ({
743
+ op: `roundtrip`,
744
+ path: `${ctx.basePath}/roundtrip-${ctx.iteration}`,
745
+ size: 100,
746
+ live: `long-poll`
747
+ })
748
+ };
749
+ const createLatencyScenario = {
750
+ id: `latency-create`,
751
+ name: `Create Latency`,
752
+ description: `Measure time to create a new stream`,
753
+ category: `latency`,
754
+ config: {
755
+ warmupIterations: 5,
756
+ measureIterations: 50,
757
+ messageSize: 0
758
+ },
759
+ criteria: {
760
+ maxP50Ms: 30,
761
+ maxP99Ms: 150
762
+ },
763
+ createOperation: (ctx) => ({
764
+ op: `create`,
765
+ path: `${ctx.basePath}/create-${ctx.iteration}`,
766
+ contentType: `application/octet-stream`
767
+ })
768
+ };
769
+ const smallMessageThroughputScenario = {
770
+ id: `throughput-small-messages`,
771
+ name: `Small Message Throughput`,
772
+ description: `Measure throughput for 100-byte messages at high concurrency`,
773
+ category: `throughput`,
774
+ requires: [`batching`],
775
+ config: {
776
+ warmupIterations: 2,
777
+ measureIterations: 10,
778
+ messageSize: 100,
779
+ concurrency: 200
780
+ },
781
+ criteria: { minOpsPerSecond: 1e3 },
782
+ createOperation: (ctx) => ({
783
+ op: `throughput_append`,
784
+ path: `${ctx.basePath}/throughput-small`,
785
+ count: 1e4,
786
+ size: 100,
787
+ concurrency: 200
788
+ })
789
+ };
790
+ const largeMessageThroughputScenario = {
791
+ id: `throughput-large-messages`,
792
+ name: `Large Message Throughput`,
793
+ description: `Measure throughput for 1MB messages`,
794
+ category: `throughput`,
795
+ requires: [`batching`],
796
+ config: {
797
+ warmupIterations: 1,
798
+ measureIterations: 5,
799
+ messageSize: 1024 * 1024,
800
+ concurrency: 10
801
+ },
802
+ criteria: { minOpsPerSecond: 20 },
803
+ createOperation: (ctx) => ({
804
+ op: `throughput_append`,
805
+ path: `${ctx.basePath}/throughput-large`,
806
+ count: 50,
807
+ size: 1024 * 1024,
808
+ concurrency: 10
809
+ })
810
+ };
811
+ const readThroughputScenario = {
812
+ id: `throughput-read`,
813
+ name: `Read Throughput`,
814
+ description: `Measure JSON parsing and iteration speed reading back messages`,
815
+ category: `throughput`,
816
+ config: {
817
+ warmupIterations: 1,
818
+ measureIterations: 5,
819
+ messageSize: 100
820
+ },
821
+ criteria: { minMBPerSecond: 3 },
822
+ createOperation: (ctx) => ({
823
+ op: `throughput_read`,
824
+ path: `${ctx.basePath}/throughput-read`,
825
+ expectedCount: ctx.setupData.expectedCount
826
+ }),
827
+ setup: (ctx) => {
828
+ ctx.setupData.expectedCount = 1e4;
829
+ return Promise.resolve({ data: { expectedCount: 1e4 } });
830
+ }
831
+ };
832
+ const sseLatencyScenario = {
833
+ id: `streaming-sse-latency`,
834
+ name: `SSE First Event Latency`,
835
+ description: `Measure time to receive first event via SSE`,
836
+ category: `streaming`,
837
+ requires: [`sse`],
838
+ config: {
839
+ warmupIterations: 3,
840
+ measureIterations: 20,
841
+ messageSize: 100
842
+ },
843
+ criteria: {
844
+ maxP50Ms: 100,
845
+ maxP99Ms: 500
846
+ },
847
+ createOperation: (ctx) => ({
848
+ op: `roundtrip`,
849
+ path: `${ctx.basePath}/sse-latency-${ctx.iteration}`,
850
+ size: 100,
851
+ live: `sse`,
852
+ contentType: `application/json`
853
+ })
854
+ };
855
+ const allScenarios = [
856
+ appendLatencyScenario,
857
+ readLatencyScenario,
858
+ roundtripLatencyScenario,
859
+ createLatencyScenario,
860
+ smallMessageThroughputScenario,
861
+ largeMessageThroughputScenario,
862
+ readThroughputScenario,
863
+ sseLatencyScenario
864
+ ];
865
+ const scenariosByCategory = {
866
+ latency: allScenarios.filter((s) => s.category === `latency`),
867
+ throughput: allScenarios.filter((s) => s.category === `throughput`),
868
+ streaming: allScenarios.filter((s) => s.category === `streaming`)
869
+ };
870
+ function getScenarioById(id) {
871
+ return allScenarios.find((s) => s.id === id);
872
+ }
873
+ function getScenariosByCategory(category) {
874
+ return scenariosByCategory[category];
875
+ }
876
+
877
+ //#endregion
878
+ //#region src/benchmark-runner.ts
879
+ var BenchmarkClientAdapter = class {
880
+ process;
881
+ readline;
882
+ pendingResponse = null;
883
+ initialized = false;
884
+ constructor(executable, args = []) {
885
+ this.process = (0, node_child_process.spawn)(executable, args, { stdio: [
886
+ `pipe`,
887
+ `pipe`,
888
+ `pipe`
889
+ ] });
890
+ if (!this.process.stdout || !this.process.stdin) throw new Error(`Failed to create client adapter process`);
891
+ this.readline = (0, node_readline.createInterface)({
892
+ input: this.process.stdout,
893
+ crlfDelay: Infinity
894
+ });
895
+ this.readline.on(`line`, (line) => {
896
+ if (this.pendingResponse) {
897
+ try {
898
+ const result = require_protocol.parseResult(line);
899
+ this.pendingResponse.resolve(result);
900
+ } catch {
901
+ this.pendingResponse.reject(new Error(`Failed to parse client response: ${line}`));
902
+ }
903
+ this.pendingResponse = null;
904
+ }
905
+ });
906
+ this.process.stderr?.on(`data`, (data) => {
907
+ console.error(`[client stderr] ${data.toString().trim()}`);
908
+ });
909
+ this.process.on(`error`, (err) => {
910
+ if (this.pendingResponse) {
911
+ this.pendingResponse.reject(err);
912
+ this.pendingResponse = null;
913
+ }
914
+ });
915
+ this.process.on(`exit`, (code) => {
916
+ if (this.pendingResponse) {
917
+ this.pendingResponse.reject(new Error(`Client adapter exited with code ${code}`));
918
+ this.pendingResponse = null;
919
+ }
920
+ });
921
+ }
922
+ async send(command, timeoutMs = 6e4) {
923
+ if (!this.process.stdin) throw new Error(`Client adapter stdin not available`);
924
+ return new Promise((resolve, reject) => {
925
+ const timeout = setTimeout(() => {
926
+ this.pendingResponse = null;
927
+ reject(new Error(`Command timed out after ${timeoutMs}ms: ${command.type}`));
928
+ }, timeoutMs);
929
+ this.pendingResponse = {
930
+ resolve: (result) => {
931
+ clearTimeout(timeout);
932
+ resolve(result);
933
+ },
934
+ reject: (error) => {
935
+ clearTimeout(timeout);
936
+ reject(error);
937
+ }
938
+ };
939
+ const line = require_protocol.serializeCommand(command) + `\n`;
940
+ this.process.stdin.write(line);
941
+ });
942
+ }
943
+ async init(serverUrl) {
944
+ const result = await this.send({
945
+ type: `init`,
946
+ serverUrl
947
+ });
948
+ if (result.success) this.initialized = true;
949
+ return result;
950
+ }
951
+ async benchmark(iterationId, operation) {
952
+ const result = await this.send({
953
+ type: `benchmark`,
954
+ iterationId,
955
+ operation
956
+ });
957
+ if (result.type === `benchmark`) return result;
958
+ return null;
959
+ }
960
+ async shutdown() {
961
+ if (this.initialized) try {
962
+ await this.send({ type: `shutdown` }, 5e3);
963
+ } catch {}
964
+ this.process.kill();
965
+ this.readline.close();
966
+ }
967
+ isInitialized() {
968
+ return this.initialized;
969
+ }
970
+ };
971
+ async function runScenario(scenario, client, serverUrl, clientFeatures, verbose, log) {
972
+ if (scenario.requires) {
973
+ const missing = scenario.requires.filter((f) => !clientFeatures[f]);
974
+ if (missing.length > 0) return {
975
+ scenario,
976
+ stats: require_protocol.calculateStats([]),
977
+ criteriaMet: true,
978
+ criteriaDetails: [],
979
+ skipped: true,
980
+ skipReason: `missing features: ${missing.join(`, `)}`
981
+ };
982
+ }
983
+ const basePath = `/bench-${(0, node_crypto.randomUUID)()}`;
984
+ const durations = [];
985
+ try {
986
+ const setupCtx = {
987
+ basePath,
988
+ iteration: 0,
989
+ setupData: {}
990
+ };
991
+ if (scenario.setup) await scenario.setup(setupCtx);
992
+ if (scenario.id === `latency-append` || scenario.id === `latency-read` || scenario.id.startsWith(`throughput-`)) {
993
+ const streamUrl = `${serverUrl}${basePath}/stream`;
994
+ await __durable_streams_client.DurableStream.create({
995
+ url: streamUrl,
996
+ contentType: `application/octet-stream`
997
+ });
998
+ if (scenario.id === `latency-read`) {
999
+ const stream = new __durable_streams_client.DurableStream({
1000
+ url: streamUrl,
1001
+ contentType: `application/octet-stream`
1002
+ });
1003
+ const chunk = new Uint8Array(1024).fill(42);
1004
+ for (let i = 0; i < 10; i++) await stream.append(chunk);
1005
+ }
1006
+ if (scenario.id === `throughput-read`) {
1007
+ const url = `${serverUrl}${basePath}/throughput-read`;
1008
+ await __durable_streams_client.DurableStream.create({
1009
+ url,
1010
+ contentType: `application/json`
1011
+ });
1012
+ const ds = new __durable_streams_client.DurableStream({
1013
+ url,
1014
+ contentType: `application/json`
1015
+ });
1016
+ const messages = [];
1017
+ for (let i = 0; i < 1e4; i++) messages.push({
1018
+ n: i,
1019
+ data: `message-${i}-padding-for-size`
1020
+ });
1021
+ await Promise.all(messages.map((msg) => ds.append(msg)));
1022
+ }
1023
+ }
1024
+ if (verbose) log(` Warmup: ${scenario.config.warmupIterations} iterations...`);
1025
+ for (let i = 0; i < scenario.config.warmupIterations; i++) {
1026
+ const ctx = {
1027
+ basePath,
1028
+ iteration: i,
1029
+ setupData: setupCtx.setupData
1030
+ };
1031
+ const operation = scenario.createOperation(ctx);
1032
+ await client.benchmark(`warmup-${i}`, operation);
1033
+ }
1034
+ if (verbose) log(` Measuring: ${scenario.config.measureIterations} iterations...`);
1035
+ let totalMessagesProcessed = 0;
1036
+ let totalBytesTransferred = 0;
1037
+ for (let i = 0; i < scenario.config.measureIterations; i++) {
1038
+ const ctx = {
1039
+ basePath,
1040
+ iteration: i,
1041
+ setupData: setupCtx.setupData
1042
+ };
1043
+ const operation = scenario.createOperation(ctx);
1044
+ const result = await client.benchmark(`measure-${i}`, operation);
1045
+ if (result) {
1046
+ durations.push(BigInt(result.durationNs));
1047
+ if (result.metrics) {
1048
+ totalMessagesProcessed += result.metrics.messagesProcessed ?? 0;
1049
+ totalBytesTransferred += result.metrics.bytesTransferred ?? 0;
1050
+ }
1051
+ }
1052
+ }
1053
+ if (scenario.cleanup) await scenario.cleanup(setupCtx);
1054
+ if (durations.length === 0) return {
1055
+ scenario,
1056
+ stats: require_protocol.calculateStats([]),
1057
+ criteriaMet: false,
1058
+ criteriaDetails: [],
1059
+ skipped: false,
1060
+ error: `No benchmark samples collected (adapter returned no results)`
1061
+ };
1062
+ const stats = require_protocol.calculateStats(durations);
1063
+ const totalTimeMs = durations.reduce((sum, ns) => sum + Number(ns) / 1e6, 0);
1064
+ const totalTimeSec = totalTimeMs / 1e3;
1065
+ let computedOpsPerSec;
1066
+ let computedMbPerSec;
1067
+ if (scenario.category === `throughput`) {
1068
+ if (totalMessagesProcessed > 0 && totalTimeSec > 0) computedOpsPerSec = totalMessagesProcessed / totalTimeSec;
1069
+ else if (stats.mean > 0) computedOpsPerSec = 1e3 / stats.mean;
1070
+ if (totalBytesTransferred > 0 && totalTimeSec > 0) computedMbPerSec = totalBytesTransferred / 1024 / 1024 / totalTimeSec;
1071
+ }
1072
+ const criteriaDetails = [];
1073
+ let criteriaMet = true;
1074
+ if (scenario.criteria) {
1075
+ if (scenario.criteria.maxP50Ms !== void 0) {
1076
+ const met = stats.median <= scenario.criteria.maxP50Ms;
1077
+ criteriaDetails.push({
1078
+ criterion: `p50 <= ${scenario.criteria.maxP50Ms}ms`,
1079
+ met,
1080
+ actual: stats.median,
1081
+ expected: scenario.criteria.maxP50Ms
1082
+ });
1083
+ if (!met) criteriaMet = false;
1084
+ }
1085
+ if (scenario.criteria.maxP99Ms !== void 0) {
1086
+ const met = stats.p99 <= scenario.criteria.maxP99Ms;
1087
+ criteriaDetails.push({
1088
+ criterion: `p99 <= ${scenario.criteria.maxP99Ms}ms`,
1089
+ met,
1090
+ actual: stats.p99,
1091
+ expected: scenario.criteria.maxP99Ms
1092
+ });
1093
+ if (!met) criteriaMet = false;
1094
+ }
1095
+ if (scenario.criteria.minOpsPerSecond !== void 0) {
1096
+ const opsPerSec = computedOpsPerSec ?? 0;
1097
+ const met = opsPerSec >= scenario.criteria.minOpsPerSecond;
1098
+ criteriaDetails.push({
1099
+ criterion: `ops/sec >= ${scenario.criteria.minOpsPerSecond}`,
1100
+ met,
1101
+ actual: opsPerSec,
1102
+ expected: scenario.criteria.minOpsPerSecond
1103
+ });
1104
+ if (!met) criteriaMet = false;
1105
+ }
1106
+ if (scenario.criteria.minMBPerSecond !== void 0) {
1107
+ const mbPerSec = totalTimeSec > 0 ? totalBytesTransferred / 1024 / 1024 / totalTimeSec : 0;
1108
+ const met = mbPerSec >= scenario.criteria.minMBPerSecond;
1109
+ criteriaDetails.push({
1110
+ criterion: `MB/sec >= ${scenario.criteria.minMBPerSecond}`,
1111
+ met,
1112
+ actual: mbPerSec,
1113
+ expected: scenario.criteria.minMBPerSecond
1114
+ });
1115
+ if (!met) criteriaMet = false;
1116
+ }
1117
+ }
1118
+ return {
1119
+ scenario,
1120
+ stats,
1121
+ criteriaMet,
1122
+ criteriaDetails,
1123
+ skipped: false,
1124
+ opsPerSec: computedOpsPerSec,
1125
+ mbPerSec: computedMbPerSec
1126
+ };
1127
+ } catch (err) {
1128
+ return {
1129
+ scenario,
1130
+ stats: require_protocol.calculateStats([]),
1131
+ criteriaMet: false,
1132
+ criteriaDetails: [],
1133
+ skipped: false,
1134
+ error: err instanceof Error ? err.message : String(err)
1135
+ };
1136
+ }
1137
+ }
1138
+ function printConsoleResults(summary) {
1139
+ console.log(`\n${`=`.repeat(60)}`);
1140
+ console.log(`CLIENT BENCHMARK RESULTS`);
1141
+ console.log(`${`=`.repeat(60)}`);
1142
+ console.log(`Adapter: ${summary.adapter} v${summary.adapterVersion}`);
1143
+ console.log(`Server: ${summary.serverUrl}`);
1144
+ console.log(`Timestamp: ${summary.timestamp}`);
1145
+ console.log(`Duration: ${(summary.duration / 1e3).toFixed(2)}s`);
1146
+ console.log();
1147
+ const byCategory = {
1148
+ latency: summary.results.filter((r) => r.scenario.category === `latency`),
1149
+ throughput: summary.results.filter((r) => r.scenario.category === `throughput`),
1150
+ streaming: summary.results.filter((r) => r.scenario.category === `streaming`)
1151
+ };
1152
+ for (const [category, results] of Object.entries(byCategory)) {
1153
+ if (results.length === 0) continue;
1154
+ console.log(`\n${category.toUpperCase()}`);
1155
+ console.log(`${`-`.repeat(40)}`);
1156
+ for (const result of results) {
1157
+ const icon = result.skipped ? `○` : result.criteriaMet ? `✓` : `✗`;
1158
+ const status = result.skipped ? `skipped: ${result.skipReason}` : result.error ? `error: ${result.error}` : result.criteriaMet ? `passed` : `failed`;
1159
+ console.log(`${icon} ${result.scenario.name} (${status})`);
1160
+ if (!result.skipped && !result.error) {
1161
+ const formatted = require_protocol.formatStats(result.stats);
1162
+ if (result.scenario.category === `throughput`) {
1163
+ const mbStr = result.mbPerSec ? result.mbPerSec.toLocaleString(`en-US`, {
1164
+ minimumFractionDigits: 1,
1165
+ maximumFractionDigits: 1
1166
+ }) : `N/A`;
1167
+ if (result.scenario.id === `throughput-read`) console.log(` MB/sec: ${mbStr}`);
1168
+ else {
1169
+ const opsStr = result.opsPerSec ? result.opsPerSec.toLocaleString(`en-US`, { maximumFractionDigits: 0 }) : `N/A`;
1170
+ console.log(` Ops/sec: ${opsStr} MB/sec: ${mbStr}`);
1171
+ }
1172
+ } else console.log(` Median: ${formatted.Median} P99: ${formatted.P99}`);
1173
+ if (!result.criteriaMet) for (const c of result.criteriaDetails.filter((d) => !d.met)) console.log(` ✗ ${c.criterion}: got ${c.actual.toFixed(2)}, expected ${c.expected}`);
1174
+ }
1175
+ }
1176
+ }
1177
+ console.log(`\n${`=`.repeat(60)}`);
1178
+ console.log(`Summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.skipped} skipped`);
1179
+ console.log(`${`=`.repeat(60)}\n`);
1180
+ }
1181
+ function generateMarkdownReport(summary) {
1182
+ const lines = [];
1183
+ const statusIcon = summary.failed === 0 ? `✓` : `✗`;
1184
+ const statusText = summary.failed === 0 ? `${summary.passed} passed` : `${summary.passed} passed, ${summary.failed} failed`;
1185
+ lines.push(`<details>`);
1186
+ lines.push(`<summary><strong>${summary.adapter}</strong>: ${statusText} ${statusIcon}</summary>`);
1187
+ lines.push(``);
1188
+ lines.push(`### ${summary.adapter} v${summary.adapterVersion}`);
1189
+ lines.push(``);
1190
+ lines.push(`**Server**: ${summary.serverUrl}`);
1191
+ lines.push(`**Date**: ${summary.timestamp}`);
1192
+ lines.push(`**Duration**: ${(summary.duration / 1e3).toFixed(2)}s`);
1193
+ lines.push(``);
1194
+ const latencyResults = summary.results.filter((r) => r.scenario.category === `latency` && !r.skipped && !r.error);
1195
+ if (latencyResults.length > 0) {
1196
+ lines.push(`#### Latency`);
1197
+ lines.push(``);
1198
+ lines.push(`Single-operation latency tests measure the time for individual operations to complete.`);
1199
+ lines.push(``);
1200
+ lines.push(`| Scenario | Description | Min | Median | P95 | P99 | Max | Status |`);
1201
+ lines.push(`|----------|-------------|-----|--------|-----|-----|-----|--------|`);
1202
+ for (const r of latencyResults) {
1203
+ const s = r.stats;
1204
+ const status = r.criteriaMet ? `Pass` : `Fail`;
1205
+ lines.push(`| ${r.scenario.name} | ${r.scenario.description} | ${s.min.toFixed(2)}ms | ${s.median.toFixed(2)}ms | ${s.p95.toFixed(2)}ms | ${s.p99.toFixed(2)}ms | ${s.max.toFixed(2)}ms | ${status} |`);
1206
+ }
1207
+ lines.push(``);
1208
+ }
1209
+ const throughputResults = summary.results.filter((r) => r.scenario.category === `throughput` && !r.skipped && !r.error);
1210
+ if (throughputResults.length > 0) {
1211
+ lines.push(`#### Throughput`);
1212
+ lines.push(``);
1213
+ lines.push(`Throughput tests measure how quickly the client can batch and send/receive data.`);
1214
+ lines.push(`Writes use automatic batching to maximize operations per second.`);
1215
+ lines.push(`Reads measure JSON parsing and iteration speed.`);
1216
+ lines.push(``);
1217
+ lines.push(`| Scenario | Description | Ops/sec | MB/sec | Status |`);
1218
+ lines.push(`|----------|-------------|---------|--------|--------|`);
1219
+ for (const r of throughputResults) {
1220
+ const opsPerSec = r.scenario.id === `throughput-read` ? `-` : r.opsPerSec !== void 0 ? r.opsPerSec.toLocaleString(`en-US`, { maximumFractionDigits: 0 }) : `N/A`;
1221
+ const mbPerSec = r.mbPerSec !== void 0 ? r.mbPerSec.toLocaleString(`en-US`, {
1222
+ minimumFractionDigits: 1,
1223
+ maximumFractionDigits: 1
1224
+ }) : `N/A`;
1225
+ const status = r.criteriaMet ? `Pass` : `Fail`;
1226
+ lines.push(`| ${r.scenario.name} | ${r.scenario.description} | ${opsPerSec} | ${mbPerSec} | ${status} |`);
1227
+ }
1228
+ lines.push(``);
1229
+ }
1230
+ const streamingResults = summary.results.filter((r) => r.scenario.category === `streaming` && !r.skipped && !r.error);
1231
+ if (streamingResults.length > 0) {
1232
+ lines.push(`#### Streaming`);
1233
+ lines.push(``);
1234
+ lines.push(`Streaming tests measure real-time event delivery via SSE (Server-Sent Events).`);
1235
+ lines.push(``);
1236
+ lines.push(`| Scenario | Description | Min | Median | P95 | P99 | Max | Status |`);
1237
+ lines.push(`|----------|-------------|-----|--------|-----|-----|-----|--------|`);
1238
+ for (const r of streamingResults) {
1239
+ const s = r.stats;
1240
+ const status = r.criteriaMet ? `Pass` : `Fail`;
1241
+ lines.push(`| ${r.scenario.name} | ${r.scenario.description} | ${s.min.toFixed(2)}ms | ${s.median.toFixed(2)}ms | ${s.p95.toFixed(2)}ms | ${s.p99.toFixed(2)}ms | ${s.max.toFixed(2)}ms | ${status} |`);
1242
+ }
1243
+ lines.push(``);
1244
+ }
1245
+ lines.push(`#### Summary`);
1246
+ lines.push(``);
1247
+ lines.push(`- **Passed**: ${summary.passed}`);
1248
+ lines.push(`- **Failed**: ${summary.failed}`);
1249
+ lines.push(`- **Skipped**: ${summary.skipped}`);
1250
+ lines.push(``);
1251
+ lines.push(`</details>`);
1252
+ return lines.join(`\n`);
1253
+ }
1254
+ async function runBenchmarks(options) {
1255
+ const startTime = Date.now();
1256
+ const results = [];
1257
+ const log = (message) => {
1258
+ if (options.format === `json` || options.format === `markdown`) process.stderr.write(message + `\n`);
1259
+ else console.log(message);
1260
+ };
1261
+ let scenarios = allScenarios;
1262
+ if (options.scenarios && options.scenarios.length > 0) scenarios = options.scenarios.map((id) => getScenarioById(id)).filter((s) => s !== void 0);
1263
+ if (options.categories && options.categories.length > 0) scenarios = scenarios.filter((s) => options.categories.includes(s.category));
1264
+ log(`\nRunning ${scenarios.length} benchmark scenarios...\n`);
1265
+ const server = new __durable_streams_server.DurableStreamTestServer({ port: options.serverPort ?? 0 });
1266
+ await server.start();
1267
+ const serverUrl = server.url;
1268
+ log(`Reference server started at ${serverUrl}\n`);
1269
+ let adapterPath = options.clientAdapter;
1270
+ let adapterArgs = options.clientArgs ?? [];
1271
+ if (adapterPath === `ts` || adapterPath === `typescript`) {
1272
+ adapterPath = `npx`;
1273
+ adapterArgs = [`tsx`, new URL(`./adapters/typescript-adapter.ts`, require("url").pathToFileURL(__filename).href).pathname];
1274
+ }
1275
+ const client = new BenchmarkClientAdapter(adapterPath, adapterArgs);
1276
+ let adapterName = `unknown`;
1277
+ let adapterVersion = `0.0.0`;
1278
+ let clientFeatures = {};
1279
+ try {
1280
+ const initResult = await client.init(serverUrl);
1281
+ if (!initResult.success) throw new Error(`Failed to initialize client adapter`);
1282
+ if (initResult.type === `init`) {
1283
+ adapterName = initResult.clientName;
1284
+ adapterVersion = initResult.clientVersion;
1285
+ clientFeatures = initResult.features ?? {};
1286
+ log(`Client: ${adapterName} v${adapterVersion}`);
1287
+ const featureList = Object.entries(clientFeatures).filter(([, v]) => v).map(([k]) => k);
1288
+ log(`Features: ${featureList.join(`, `) || `none`}\n`);
1289
+ }
1290
+ for (const scenario of scenarios) {
1291
+ log(`\n${scenario.name}`);
1292
+ log(`${`─`.repeat(scenario.name.length)}`);
1293
+ log(`${scenario.description}`);
1294
+ const result = await runScenario(scenario, client, serverUrl, clientFeatures, options.verbose ?? false, log);
1295
+ results.push(result);
1296
+ if (result.skipped) log(` Skipped: ${result.skipReason}`);
1297
+ else if (result.error) log(` Error: ${result.error}`);
1298
+ else {
1299
+ const icon = result.criteriaMet ? `✓` : `✗`;
1300
+ if (result.scenario.category === `throughput`) {
1301
+ const mbStr = result.mbPerSec ? result.mbPerSec.toLocaleString(`en-US`, {
1302
+ minimumFractionDigits: 1,
1303
+ maximumFractionDigits: 1
1304
+ }) : `N/A`;
1305
+ if (result.scenario.id === `throughput-read`) log(` ${icon} MB/sec: ${mbStr}`);
1306
+ else {
1307
+ const opsStr = result.opsPerSec ? result.opsPerSec.toLocaleString(`en-US`, { maximumFractionDigits: 0 }) : `N/A`;
1308
+ log(` ${icon} Ops/sec: ${opsStr}, MB/sec: ${mbStr}`);
1309
+ }
1310
+ } else log(` ${icon} Median: ${result.stats.median.toFixed(2)}ms, P99: ${result.stats.p99.toFixed(2)}ms`);
1311
+ }
1312
+ }
1313
+ } finally {
1314
+ await client.shutdown();
1315
+ await server.stop();
1316
+ }
1317
+ const summary = {
1318
+ adapter: adapterName,
1319
+ adapterVersion,
1320
+ serverUrl,
1321
+ timestamp: new Date().toISOString(),
1322
+ duration: Date.now() - startTime,
1323
+ results,
1324
+ passed: results.filter((r) => !r.skipped && !r.error && r.criteriaMet).length,
1325
+ failed: results.filter((r) => !r.skipped && (!r.criteriaMet || r.error)).length,
1326
+ skipped: results.filter((r) => r.skipped).length
1327
+ };
1328
+ switch (options.format) {
1329
+ case `json`:
1330
+ console.log(JSON.stringify(summary, null, 2));
1331
+ break;
1332
+ case `markdown`:
1333
+ console.log(generateMarkdownReport(summary));
1334
+ break;
1335
+ default: printConsoleResults(summary);
1336
+ }
1337
+ return summary;
1338
+ }
1339
+
1340
+ //#endregion
1341
+ Object.defineProperty(exports, 'allScenarios', {
1342
+ enumerable: true,
1343
+ get: function () {
1344
+ return allScenarios;
1345
+ }
1346
+ });
1347
+ Object.defineProperty(exports, 'countTests', {
1348
+ enumerable: true,
1349
+ get: function () {
1350
+ return countTests;
1351
+ }
1352
+ });
1353
+ Object.defineProperty(exports, 'filterByCategory', {
1354
+ enumerable: true,
1355
+ get: function () {
1356
+ return filterByCategory;
1357
+ }
1358
+ });
1359
+ Object.defineProperty(exports, 'getScenarioById', {
1360
+ enumerable: true,
1361
+ get: function () {
1362
+ return getScenarioById;
1363
+ }
1364
+ });
1365
+ Object.defineProperty(exports, 'getScenariosByCategory', {
1366
+ enumerable: true,
1367
+ get: function () {
1368
+ return getScenariosByCategory;
1369
+ }
1370
+ });
1371
+ Object.defineProperty(exports, 'loadEmbeddedTestSuites', {
1372
+ enumerable: true,
1373
+ get: function () {
1374
+ return loadEmbeddedTestSuites;
1375
+ }
1376
+ });
1377
+ Object.defineProperty(exports, 'loadTestSuites', {
1378
+ enumerable: true,
1379
+ get: function () {
1380
+ return loadTestSuites;
1381
+ }
1382
+ });
1383
+ Object.defineProperty(exports, 'runBenchmarks', {
1384
+ enumerable: true,
1385
+ get: function () {
1386
+ return runBenchmarks;
1387
+ }
1388
+ });
1389
+ Object.defineProperty(exports, 'runConformanceTests', {
1390
+ enumerable: true,
1391
+ get: function () {
1392
+ return runConformanceTests;
1393
+ }
1394
+ });
1395
+ Object.defineProperty(exports, 'scenariosByCategory', {
1396
+ enumerable: true,
1397
+ get: function () {
1398
+ return scenariosByCategory;
1399
+ }
1400
+ });