@durable-streams/client-conformance-tests 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/typescript-adapter.cjs +588 -0
- package/dist/adapters/typescript-adapter.d.cts +1 -0
- package/dist/benchmark-runner-CLAR9oLd.cjs +1400 -0
- package/dist/chunk-BCwAaXi7.cjs +31 -0
- package/dist/cli.cjs +266 -0
- package/dist/cli.d.cts +1 -0
- package/dist/index.cjs +22 -0
- package/dist/index.d.cts +508 -0
- package/dist/protocol-3cf94Xyb.d.cts +472 -0
- package/dist/protocol-XeAOKBD-.cjs +175 -0
- package/dist/protocol.cjs +11 -0
- package/dist/protocol.d.cts +2 -0
- package/package.json +51 -36
|
@@ -0,0 +1,1400 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
const require_chunk = require('./chunk-BCwAaXi7.cjs');
|
|
3
|
+
const require_protocol = require('./protocol-XeAOKBD-.cjs');
|
|
4
|
+
const node_child_process = require_chunk.__toESM(require("node:child_process"));
|
|
5
|
+
const node_readline = require_chunk.__toESM(require("node:readline"));
|
|
6
|
+
const node_crypto = require_chunk.__toESM(require("node:crypto"));
|
|
7
|
+
const __durable_streams_server = require_chunk.__toESM(require("@durable-streams/server"));
|
|
8
|
+
const node_fs = require_chunk.__toESM(require("node:fs"));
|
|
9
|
+
const node_path = require_chunk.__toESM(require("node:path"));
|
|
10
|
+
const yaml = require_chunk.__toESM(require("yaml"));
|
|
11
|
+
const __durable_streams_client = require_chunk.__toESM(require("@durable-streams/client"));
|
|
12
|
+
|
|
13
|
+
//#region src/test-cases.ts
|
|
14
|
+
/**
|
|
15
|
+
* Load all test suites from a directory.
|
|
16
|
+
*/
|
|
17
|
+
function loadTestSuites(dir) {
|
|
18
|
+
const suites = [];
|
|
19
|
+
function walkDir(currentDir) {
|
|
20
|
+
const entries = node_fs.readdirSync(currentDir, { withFileTypes: true });
|
|
21
|
+
for (const entry of entries) {
|
|
22
|
+
const fullPath = node_path.join(currentDir, entry.name);
|
|
23
|
+
if (entry.isDirectory()) walkDir(fullPath);
|
|
24
|
+
else if (entry.isFile() && (entry.name.endsWith(`.yaml`) || entry.name.endsWith(`.yml`))) {
|
|
25
|
+
const content = node_fs.readFileSync(fullPath, `utf-8`);
|
|
26
|
+
const suite = yaml.default.parse(content);
|
|
27
|
+
suites.push(suite);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
walkDir(dir);
|
|
32
|
+
return suites;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Load test suites from the embedded test-cases directory.
|
|
36
|
+
*/
|
|
37
|
+
function loadEmbeddedTestSuites() {
|
|
38
|
+
const testCasesDir = node_path.join(__dirname, `..`, `test-cases`);
|
|
39
|
+
return loadTestSuites(testCasesDir);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Filter test suites by category.
|
|
43
|
+
*/
|
|
44
|
+
function filterByCategory(suites, category) {
|
|
45
|
+
return suites.filter((s) => s.category === category);
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Get total test count.
|
|
49
|
+
*/
|
|
50
|
+
function countTests(suites) {
|
|
51
|
+
return suites.reduce((sum, suite) => sum + suite.tests.length, 0);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
//#endregion
|
|
55
|
+
//#region src/runner.ts
|
|
56
|
+
var ClientAdapter = class {
|
|
57
|
+
process;
|
|
58
|
+
readline;
|
|
59
|
+
pendingResponse = null;
|
|
60
|
+
initialized = false;
|
|
61
|
+
constructor(executable, args = []) {
|
|
62
|
+
this.process = (0, node_child_process.spawn)(executable, args, { stdio: [
|
|
63
|
+
`pipe`,
|
|
64
|
+
`pipe`,
|
|
65
|
+
`pipe`
|
|
66
|
+
] });
|
|
67
|
+
if (!this.process.stdout || !this.process.stdin) throw new Error(`Failed to create client adapter process`);
|
|
68
|
+
this.readline = (0, node_readline.createInterface)({
|
|
69
|
+
input: this.process.stdout,
|
|
70
|
+
crlfDelay: Infinity
|
|
71
|
+
});
|
|
72
|
+
this.readline.on(`line`, (line) => {
|
|
73
|
+
if (this.pendingResponse) {
|
|
74
|
+
try {
|
|
75
|
+
const result = require_protocol.parseResult(line);
|
|
76
|
+
this.pendingResponse.resolve(result);
|
|
77
|
+
} catch {
|
|
78
|
+
this.pendingResponse.reject(new Error(`Failed to parse client response: ${line}`));
|
|
79
|
+
}
|
|
80
|
+
this.pendingResponse = null;
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
this.process.stderr?.on(`data`, (data) => {
|
|
84
|
+
console.error(`[client stderr] ${data.toString().trim()}`);
|
|
85
|
+
});
|
|
86
|
+
this.process.on(`error`, (err) => {
|
|
87
|
+
if (this.pendingResponse) {
|
|
88
|
+
this.pendingResponse.reject(err);
|
|
89
|
+
this.pendingResponse = null;
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
this.process.on(`exit`, (code) => {
|
|
93
|
+
if (this.pendingResponse) {
|
|
94
|
+
this.pendingResponse.reject(new Error(`Client adapter exited with code ${code}`));
|
|
95
|
+
this.pendingResponse = null;
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
async send(command, timeoutMs = 3e4) {
|
|
100
|
+
if (!this.process.stdin) throw new Error(`Client adapter stdin not available`);
|
|
101
|
+
return new Promise((resolve, reject) => {
|
|
102
|
+
const timeout = setTimeout(() => {
|
|
103
|
+
this.pendingResponse = null;
|
|
104
|
+
reject(new Error(`Command timed out after ${timeoutMs}ms: ${command.type}`));
|
|
105
|
+
}, timeoutMs);
|
|
106
|
+
this.pendingResponse = {
|
|
107
|
+
resolve: (result) => {
|
|
108
|
+
clearTimeout(timeout);
|
|
109
|
+
resolve(result);
|
|
110
|
+
},
|
|
111
|
+
reject: (error) => {
|
|
112
|
+
clearTimeout(timeout);
|
|
113
|
+
reject(error);
|
|
114
|
+
}
|
|
115
|
+
};
|
|
116
|
+
const line = require_protocol.serializeCommand(command) + `\n`;
|
|
117
|
+
this.process.stdin.write(line);
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
async init(serverUrl) {
|
|
121
|
+
const result = await this.send({
|
|
122
|
+
type: `init`,
|
|
123
|
+
serverUrl
|
|
124
|
+
});
|
|
125
|
+
if (result.success) this.initialized = true;
|
|
126
|
+
return result;
|
|
127
|
+
}
|
|
128
|
+
async shutdown() {
|
|
129
|
+
if (this.initialized) try {
|
|
130
|
+
await this.send({ type: `shutdown` }, 5e3);
|
|
131
|
+
} catch {}
|
|
132
|
+
this.process.kill();
|
|
133
|
+
this.readline.close();
|
|
134
|
+
}
|
|
135
|
+
isInitialized() {
|
|
136
|
+
return this.initialized;
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
function resolveVariables(value, variables) {
|
|
140
|
+
return value.replace(/\$\{([^}]+)\}/g, (match, expr) => {
|
|
141
|
+
if (expr === `randomUUID`) return (0, node_crypto.randomUUID)();
|
|
142
|
+
const parts = expr.split(`.`);
|
|
143
|
+
let current = variables.get(parts[0]);
|
|
144
|
+
if (current === void 0) throw new Error(`Undefined variable: ${parts[0]} (in ${match})`);
|
|
145
|
+
for (let i = 1; i < parts.length && current != null; i++) current = current[parts[i]];
|
|
146
|
+
return String(current ?? ``);
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
function generateStreamPath() {
|
|
150
|
+
return `/test-stream-${(0, node_crypto.randomUUID)()}`;
|
|
151
|
+
}
|
|
152
|
+
async function executeOperation(op, ctx) {
|
|
153
|
+
const { client, variables, verbose, commandTimeout } = ctx;
|
|
154
|
+
switch (op.action) {
|
|
155
|
+
case `create`: {
|
|
156
|
+
const path = op.path ? resolveVariables(op.path, variables) : generateStreamPath();
|
|
157
|
+
if (op.as) variables.set(op.as, path);
|
|
158
|
+
const result = await client.send({
|
|
159
|
+
type: `create`,
|
|
160
|
+
path,
|
|
161
|
+
contentType: op.contentType,
|
|
162
|
+
ttlSeconds: op.ttlSeconds,
|
|
163
|
+
expiresAt: op.expiresAt,
|
|
164
|
+
headers: op.headers
|
|
165
|
+
}, commandTimeout);
|
|
166
|
+
if (verbose) console.log(` create ${path}: ${result.success ? `ok` : `failed`}`);
|
|
167
|
+
return { result };
|
|
168
|
+
}
|
|
169
|
+
case `connect`: {
|
|
170
|
+
const path = resolveVariables(op.path, variables);
|
|
171
|
+
const result = await client.send({
|
|
172
|
+
type: `connect`,
|
|
173
|
+
path,
|
|
174
|
+
headers: op.headers
|
|
175
|
+
}, commandTimeout);
|
|
176
|
+
if (verbose) console.log(` connect ${path}: ${result.success ? `ok` : `failed`}`);
|
|
177
|
+
return { result };
|
|
178
|
+
}
|
|
179
|
+
case `append`: {
|
|
180
|
+
const path = resolveVariables(op.path, variables);
|
|
181
|
+
const data = op.data ? resolveVariables(op.data, variables) : ``;
|
|
182
|
+
const result = await client.send({
|
|
183
|
+
type: `append`,
|
|
184
|
+
path,
|
|
185
|
+
data: op.binaryData ?? data,
|
|
186
|
+
binary: !!op.binaryData,
|
|
187
|
+
seq: op.seq,
|
|
188
|
+
headers: op.headers
|
|
189
|
+
}, commandTimeout);
|
|
190
|
+
if (verbose) console.log(` append ${path}: ${result.success ? `ok` : `failed`}`);
|
|
191
|
+
if (result.success && result.type === `append` && op.expect?.storeOffsetAs) variables.set(op.expect.storeOffsetAs, result.offset);
|
|
192
|
+
return { result };
|
|
193
|
+
}
|
|
194
|
+
case `append-batch`: {
|
|
195
|
+
const path = resolveVariables(op.path, variables);
|
|
196
|
+
const results = [];
|
|
197
|
+
for (const item of op.items) {
|
|
198
|
+
const result = await client.send({
|
|
199
|
+
type: `append`,
|
|
200
|
+
path,
|
|
201
|
+
data: item.binaryData ?? item.data ?? ``,
|
|
202
|
+
binary: !!item.binaryData,
|
|
203
|
+
seq: item.seq,
|
|
204
|
+
headers: op.headers
|
|
205
|
+
}, commandTimeout);
|
|
206
|
+
results.push(result);
|
|
207
|
+
}
|
|
208
|
+
if (verbose) {
|
|
209
|
+
const succeeded = results.filter((r) => r.success).length;
|
|
210
|
+
console.log(` append-batch ${path}: ${succeeded}/${results.length} succeeded`);
|
|
211
|
+
}
|
|
212
|
+
const allSucceeded = results.every((r) => r.success);
|
|
213
|
+
return { result: {
|
|
214
|
+
type: `append`,
|
|
215
|
+
success: allSucceeded,
|
|
216
|
+
status: allSucceeded ? 200 : 207
|
|
217
|
+
} };
|
|
218
|
+
}
|
|
219
|
+
case `read`: {
|
|
220
|
+
const path = resolveVariables(op.path, variables);
|
|
221
|
+
const offset = op.offset ? resolveVariables(op.offset, variables) : void 0;
|
|
222
|
+
if (op.background && op.as) {
|
|
223
|
+
const resultPromise = client.send({
|
|
224
|
+
type: `read`,
|
|
225
|
+
path,
|
|
226
|
+
offset,
|
|
227
|
+
live: op.live,
|
|
228
|
+
timeoutMs: op.timeoutMs,
|
|
229
|
+
maxChunks: op.maxChunks,
|
|
230
|
+
waitForUpToDate: op.waitForUpToDate,
|
|
231
|
+
headers: op.headers
|
|
232
|
+
}, commandTimeout);
|
|
233
|
+
ctx.backgroundOps.set(op.as, resultPromise);
|
|
234
|
+
if (verbose) console.log(` read ${path}: started in background as ${op.as}`);
|
|
235
|
+
return {};
|
|
236
|
+
}
|
|
237
|
+
const result = await client.send({
|
|
238
|
+
type: `read`,
|
|
239
|
+
path,
|
|
240
|
+
offset,
|
|
241
|
+
live: op.live,
|
|
242
|
+
timeoutMs: op.timeoutMs,
|
|
243
|
+
maxChunks: op.maxChunks,
|
|
244
|
+
waitForUpToDate: op.waitForUpToDate,
|
|
245
|
+
headers: op.headers
|
|
246
|
+
}, commandTimeout);
|
|
247
|
+
if (verbose) console.log(` read ${path}: ${result.success ? `ok` : `failed`}`);
|
|
248
|
+
if (result.success && result.type === `read`) {
|
|
249
|
+
if (op.expect?.storeOffsetAs) variables.set(op.expect.storeOffsetAs, result.offset);
|
|
250
|
+
if (op.expect?.storeDataAs) {
|
|
251
|
+
const data = result.chunks.map((c) => c.data).join(``);
|
|
252
|
+
variables.set(op.expect.storeDataAs, data);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
return { result };
|
|
256
|
+
}
|
|
257
|
+
case `head`: {
|
|
258
|
+
const path = resolveVariables(op.path, variables);
|
|
259
|
+
const result = await client.send({
|
|
260
|
+
type: `head`,
|
|
261
|
+
path,
|
|
262
|
+
headers: op.headers
|
|
263
|
+
}, commandTimeout);
|
|
264
|
+
if (verbose) console.log(` head ${path}: ${result.success ? `ok` : `failed`}`);
|
|
265
|
+
if (result.success && op.expect?.storeAs) variables.set(op.expect.storeAs, result);
|
|
266
|
+
return { result };
|
|
267
|
+
}
|
|
268
|
+
case `delete`: {
|
|
269
|
+
const path = resolveVariables(op.path, variables);
|
|
270
|
+
const result = await client.send({
|
|
271
|
+
type: `delete`,
|
|
272
|
+
path,
|
|
273
|
+
headers: op.headers
|
|
274
|
+
}, commandTimeout);
|
|
275
|
+
if (verbose) console.log(` delete ${path}: ${result.success ? `ok` : `failed`}`);
|
|
276
|
+
return { result };
|
|
277
|
+
}
|
|
278
|
+
case `wait`: {
|
|
279
|
+
await new Promise((resolve) => setTimeout(resolve, op.ms));
|
|
280
|
+
return {};
|
|
281
|
+
}
|
|
282
|
+
case `set`: {
|
|
283
|
+
const value = resolveVariables(op.value, variables);
|
|
284
|
+
variables.set(op.name, value);
|
|
285
|
+
return {};
|
|
286
|
+
}
|
|
287
|
+
case `assert`: {
|
|
288
|
+
if (op.equals) {
|
|
289
|
+
const left = resolveVariables(op.equals.left, variables);
|
|
290
|
+
const right = resolveVariables(op.equals.right, variables);
|
|
291
|
+
if (left !== right) return { error: op.message ?? `Assertion failed: expected "${left}" to equal "${right}"` };
|
|
292
|
+
}
|
|
293
|
+
if (op.notEquals) {
|
|
294
|
+
const left = resolveVariables(op.notEquals.left, variables);
|
|
295
|
+
const right = resolveVariables(op.notEquals.right, variables);
|
|
296
|
+
if (left === right) return { error: op.message ?? `Assertion failed: expected "${left}" to not equal "${right}"` };
|
|
297
|
+
}
|
|
298
|
+
if (op.contains) {
|
|
299
|
+
const value = resolveVariables(op.contains.value, variables);
|
|
300
|
+
const substring = resolveVariables(op.contains.substring, variables);
|
|
301
|
+
if (!value.includes(substring)) return { error: op.message ?? `Assertion failed: expected "${value}" to contain "${substring}"` };
|
|
302
|
+
}
|
|
303
|
+
if (op.matches) {
|
|
304
|
+
const value = resolveVariables(op.matches.value, variables);
|
|
305
|
+
const pattern = op.matches.pattern;
|
|
306
|
+
try {
|
|
307
|
+
const regex = new RegExp(pattern);
|
|
308
|
+
if (!regex.test(value)) return { error: op.message ?? `Assertion failed: expected "${value}" to match /${pattern}/` };
|
|
309
|
+
} catch {
|
|
310
|
+
return { error: `Invalid regex pattern: ${pattern}` };
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
return {};
|
|
314
|
+
}
|
|
315
|
+
case `server-append`: {
|
|
316
|
+
const path = resolveVariables(op.path, variables);
|
|
317
|
+
const data = resolveVariables(op.data, variables);
|
|
318
|
+
try {
|
|
319
|
+
const headResponse = await fetch(`${ctx.serverUrl}${path}`, { method: `HEAD` });
|
|
320
|
+
const contentType = headResponse.headers.get(`content-type`) ?? `application/octet-stream`;
|
|
321
|
+
const response = await fetch(`${ctx.serverUrl}${path}`, {
|
|
322
|
+
method: `POST`,
|
|
323
|
+
body: data,
|
|
324
|
+
headers: {
|
|
325
|
+
"content-type": contentType,
|
|
326
|
+
...op.headers
|
|
327
|
+
}
|
|
328
|
+
});
|
|
329
|
+
if (verbose) console.log(` server-append ${path}: ${response.ok ? `ok` : `failed (${response.status})`}`);
|
|
330
|
+
if (!response.ok) return { error: `Server append failed with status ${response.status}` };
|
|
331
|
+
return {};
|
|
332
|
+
} catch (err) {
|
|
333
|
+
return { error: `Server append failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
case `await`: {
|
|
337
|
+
const ref = op.ref;
|
|
338
|
+
const promise = ctx.backgroundOps.get(ref);
|
|
339
|
+
if (!promise) return { error: `No background operation found with ref: ${ref}` };
|
|
340
|
+
const result = await promise;
|
|
341
|
+
ctx.backgroundOps.delete(ref);
|
|
342
|
+
if (verbose) console.log(` await ${ref}: ${result.success ? `ok` : `failed`}`);
|
|
343
|
+
return { result };
|
|
344
|
+
}
|
|
345
|
+
case `inject-error`: {
|
|
346
|
+
const path = resolveVariables(op.path, variables);
|
|
347
|
+
try {
|
|
348
|
+
const response = await fetch(`${ctx.serverUrl}/_test/inject-error`, {
|
|
349
|
+
method: `POST`,
|
|
350
|
+
headers: { "content-type": `application/json` },
|
|
351
|
+
body: JSON.stringify({
|
|
352
|
+
path,
|
|
353
|
+
status: op.status,
|
|
354
|
+
count: op.count ?? 1,
|
|
355
|
+
retryAfter: op.retryAfter
|
|
356
|
+
})
|
|
357
|
+
});
|
|
358
|
+
if (verbose) console.log(` inject-error ${path} ${op.status}x${op.count ?? 1}: ${response.ok ? `ok` : `failed`}`);
|
|
359
|
+
if (!response.ok) return { error: `Failed to inject error: ${response.status}` };
|
|
360
|
+
return {};
|
|
361
|
+
} catch (err) {
|
|
362
|
+
return { error: `Failed to inject error: ${err instanceof Error ? err.message : String(err)}` };
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
case `clear-errors`: try {
|
|
366
|
+
const response = await fetch(`${ctx.serverUrl}/_test/inject-error`, { method: `DELETE` });
|
|
367
|
+
if (verbose) console.log(` clear-errors: ${response.ok ? `ok` : `failed`}`);
|
|
368
|
+
return {};
|
|
369
|
+
} catch (err) {
|
|
370
|
+
return { error: `Failed to clear errors: ${err instanceof Error ? err.message : String(err)}` };
|
|
371
|
+
}
|
|
372
|
+
case `set-dynamic-header`: {
|
|
373
|
+
const result = await client.send({
|
|
374
|
+
type: `set-dynamic-header`,
|
|
375
|
+
name: op.name,
|
|
376
|
+
valueType: op.valueType,
|
|
377
|
+
initialValue: op.initialValue
|
|
378
|
+
}, commandTimeout);
|
|
379
|
+
if (verbose) console.log(` set-dynamic-header ${op.name}: ${result.success ? `ok` : `failed`}`);
|
|
380
|
+
return { result };
|
|
381
|
+
}
|
|
382
|
+
case `set-dynamic-param`: {
|
|
383
|
+
const result = await client.send({
|
|
384
|
+
type: `set-dynamic-param`,
|
|
385
|
+
name: op.name,
|
|
386
|
+
valueType: op.valueType
|
|
387
|
+
}, commandTimeout);
|
|
388
|
+
if (verbose) console.log(` set-dynamic-param ${op.name}: ${result.success ? `ok` : `failed`}`);
|
|
389
|
+
return { result };
|
|
390
|
+
}
|
|
391
|
+
case `clear-dynamic`: {
|
|
392
|
+
const result = await client.send({ type: `clear-dynamic` }, commandTimeout);
|
|
393
|
+
if (verbose) console.log(` clear-dynamic: ${result.success ? `ok` : `failed`}`);
|
|
394
|
+
return { result };
|
|
395
|
+
}
|
|
396
|
+
default: return { error: `Unknown operation: ${op.action}` };
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
function isReadResult(result) {
|
|
400
|
+
return result.type === `read` && result.success;
|
|
401
|
+
}
|
|
402
|
+
function isAppendResult(result) {
|
|
403
|
+
return result.type === `append` && result.success;
|
|
404
|
+
}
|
|
405
|
+
function isHeadResult(result) {
|
|
406
|
+
return result.type === `head` && result.success;
|
|
407
|
+
}
|
|
408
|
+
function isErrorResult(result) {
|
|
409
|
+
return result.type === `error` && !result.success;
|
|
410
|
+
}
|
|
411
|
+
function validateExpectation(result, expect) {
|
|
412
|
+
if (!expect) return null;
|
|
413
|
+
if (expect.status !== void 0 && `status` in result) {
|
|
414
|
+
if (result.status !== expect.status) return `Expected status ${expect.status}, got ${result.status}`;
|
|
415
|
+
}
|
|
416
|
+
if (expect.errorCode !== void 0) {
|
|
417
|
+
if (result.success) return `Expected error ${expect.errorCode}, but operation succeeded`;
|
|
418
|
+
if (isErrorResult(result) && result.errorCode !== expect.errorCode) return `Expected error code ${expect.errorCode}, got ${result.errorCode}`;
|
|
419
|
+
}
|
|
420
|
+
if (expect.data !== void 0 && isReadResult(result)) {
|
|
421
|
+
const actualData = result.chunks.map((c) => c.data).join(``);
|
|
422
|
+
if (actualData !== expect.data) return `Expected data "${expect.data}", got "${actualData}"`;
|
|
423
|
+
}
|
|
424
|
+
if (expect.dataContains !== void 0 && isReadResult(result)) {
|
|
425
|
+
const actualData = result.chunks.map((c) => c.data).join(``);
|
|
426
|
+
if (!actualData.includes(expect.dataContains)) return `Expected data to contain "${expect.dataContains}", got "${actualData}"`;
|
|
427
|
+
}
|
|
428
|
+
if (expect.dataContainsAll !== void 0 && isReadResult(result)) {
|
|
429
|
+
const actualData = result.chunks.map((c) => c.data).join(``);
|
|
430
|
+
const missing = expect.dataContainsAll.filter((s) => !actualData.includes(s));
|
|
431
|
+
if (missing.length > 0) return `Expected data to contain all of [${expect.dataContainsAll.join(`, `)}], missing: [${missing.join(`, `)}]`;
|
|
432
|
+
}
|
|
433
|
+
if (expect.upToDate !== void 0 && isReadResult(result)) {
|
|
434
|
+
if (result.upToDate !== expect.upToDate) return `Expected upToDate=${expect.upToDate}, got ${result.upToDate}`;
|
|
435
|
+
}
|
|
436
|
+
if (expect.chunkCount !== void 0 && isReadResult(result)) {
|
|
437
|
+
if (result.chunks.length !== expect.chunkCount) return `Expected ${expect.chunkCount} chunks, got ${result.chunks.length}`;
|
|
438
|
+
}
|
|
439
|
+
if (expect.minChunks !== void 0 && isReadResult(result)) {
|
|
440
|
+
if (result.chunks.length < expect.minChunks) return `Expected at least ${expect.minChunks} chunks, got ${result.chunks.length}`;
|
|
441
|
+
}
|
|
442
|
+
if (expect.contentType !== void 0 && isHeadResult(result)) {
|
|
443
|
+
if (result.contentType !== expect.contentType) return `Expected contentType "${expect.contentType}", got "${result.contentType}"`;
|
|
444
|
+
}
|
|
445
|
+
if (expect.hasOffset !== void 0 && isHeadResult(result)) {
|
|
446
|
+
const hasOffset = result.offset !== void 0 && result.offset !== ``;
|
|
447
|
+
if (hasOffset !== expect.hasOffset) return `Expected hasOffset=${expect.hasOffset}, got ${hasOffset}`;
|
|
448
|
+
}
|
|
449
|
+
if (expect.headersSent !== void 0) {
|
|
450
|
+
const expectedHeaders = expect.headersSent;
|
|
451
|
+
let actualHeaders;
|
|
452
|
+
if (isAppendResult(result)) actualHeaders = result.headersSent;
|
|
453
|
+
else if (isReadResult(result)) actualHeaders = result.headersSent;
|
|
454
|
+
if (!actualHeaders) return `Expected headersSent but result does not contain headersSent`;
|
|
455
|
+
for (const [key, expectedValue] of Object.entries(expectedHeaders)) {
|
|
456
|
+
const actualValue = actualHeaders[key];
|
|
457
|
+
if (actualValue !== expectedValue) return `Expected headersSent[${key}]="${expectedValue}", got "${actualValue ?? `undefined`}"`;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
if (expect.paramsSent !== void 0) {
|
|
461
|
+
const expectedParams = expect.paramsSent;
|
|
462
|
+
let actualParams;
|
|
463
|
+
if (isAppendResult(result)) actualParams = result.paramsSent;
|
|
464
|
+
else if (isReadResult(result)) actualParams = result.paramsSent;
|
|
465
|
+
if (!actualParams) return `Expected paramsSent but result does not contain paramsSent`;
|
|
466
|
+
for (const [key, expectedValue] of Object.entries(expectedParams)) {
|
|
467
|
+
const actualValue = actualParams[key];
|
|
468
|
+
if (actualValue !== expectedValue) return `Expected paramsSent[${key}]="${expectedValue}", got "${actualValue ?? `undefined`}"`;
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
return null;
|
|
472
|
+
}
|
|
473
|
+
/**
|
|
474
|
+
* Map feature names from YAML (kebab-case) to client feature property names (camelCase).
|
|
475
|
+
*/
|
|
476
|
+
function featureToProperty(feature) {
|
|
477
|
+
const map = {
|
|
478
|
+
batching: `batching`,
|
|
479
|
+
sse: `sse`,
|
|
480
|
+
"long-poll": `longPoll`,
|
|
481
|
+
longPoll: `longPoll`,
|
|
482
|
+
streaming: `streaming`,
|
|
483
|
+
dynamicHeaders: `dynamicHeaders`,
|
|
484
|
+
"dynamic-headers": `dynamicHeaders`
|
|
485
|
+
};
|
|
486
|
+
return map[feature];
|
|
487
|
+
}
|
|
488
|
+
/**
|
|
489
|
+
* Check if client supports all required features.
|
|
490
|
+
* Returns list of missing features, or empty array if all satisfied.
|
|
491
|
+
*/
|
|
492
|
+
function getMissingFeatures(requires, clientFeatures) {
|
|
493
|
+
if (!requires || requires.length === 0) return [];
|
|
494
|
+
return requires.filter((feature) => {
|
|
495
|
+
const prop = featureToProperty(feature);
|
|
496
|
+
return !prop || !clientFeatures[prop];
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
async function runTestCase(test, ctx) {
|
|
500
|
+
const startTime = Date.now();
|
|
501
|
+
if (test.skip) return {
|
|
502
|
+
suite: ``,
|
|
503
|
+
test: test.id,
|
|
504
|
+
passed: true,
|
|
505
|
+
duration: 0,
|
|
506
|
+
skipped: true,
|
|
507
|
+
skipReason: typeof test.skip === `string` ? test.skip : void 0
|
|
508
|
+
};
|
|
509
|
+
const missingFeatures = getMissingFeatures(test.requires, ctx.clientFeatures);
|
|
510
|
+
if (missingFeatures.length > 0) return {
|
|
511
|
+
suite: ``,
|
|
512
|
+
test: test.id,
|
|
513
|
+
passed: true,
|
|
514
|
+
duration: 0,
|
|
515
|
+
skipped: true,
|
|
516
|
+
skipReason: `missing features: ${missingFeatures.join(`, `)}`
|
|
517
|
+
};
|
|
518
|
+
ctx.variables.clear();
|
|
519
|
+
ctx.backgroundOps.clear();
|
|
520
|
+
try {
|
|
521
|
+
if (test.setup) for (const op of test.setup) {
|
|
522
|
+
const { error } = await executeOperation(op, ctx);
|
|
523
|
+
if (error) return {
|
|
524
|
+
suite: ``,
|
|
525
|
+
test: test.id,
|
|
526
|
+
passed: false,
|
|
527
|
+
duration: Date.now() - startTime,
|
|
528
|
+
error: `Setup failed: ${error}`
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
for (const op of test.operations) {
|
|
532
|
+
const { result, error } = await executeOperation(op, ctx);
|
|
533
|
+
if (error) return {
|
|
534
|
+
suite: ``,
|
|
535
|
+
test: test.id,
|
|
536
|
+
passed: false,
|
|
537
|
+
duration: Date.now() - startTime,
|
|
538
|
+
error
|
|
539
|
+
};
|
|
540
|
+
if (result && `expect` in op && op.expect) {
|
|
541
|
+
const validationError = validateExpectation(result, op.expect);
|
|
542
|
+
if (validationError) return {
|
|
543
|
+
suite: ``,
|
|
544
|
+
test: test.id,
|
|
545
|
+
passed: false,
|
|
546
|
+
duration: Date.now() - startTime,
|
|
547
|
+
error: validationError
|
|
548
|
+
};
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
if (test.cleanup) for (const op of test.cleanup) try {
|
|
552
|
+
await executeOperation(op, ctx);
|
|
553
|
+
} catch {}
|
|
554
|
+
return {
|
|
555
|
+
suite: ``,
|
|
556
|
+
test: test.id,
|
|
557
|
+
passed: true,
|
|
558
|
+
duration: Date.now() - startTime
|
|
559
|
+
};
|
|
560
|
+
} catch (err) {
|
|
561
|
+
return {
|
|
562
|
+
suite: ``,
|
|
563
|
+
test: test.id,
|
|
564
|
+
passed: false,
|
|
565
|
+
duration: Date.now() - startTime,
|
|
566
|
+
error: err instanceof Error ? err.message : String(err)
|
|
567
|
+
};
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
async function runConformanceTests(options) {
|
|
571
|
+
const startTime = Date.now();
|
|
572
|
+
const results = [];
|
|
573
|
+
let suites = loadEmbeddedTestSuites();
|
|
574
|
+
if (options.suites) suites = suites.filter((s) => options.suites.includes(s.category));
|
|
575
|
+
if (options.tags) suites = suites.map((suite) => ({
|
|
576
|
+
...suite,
|
|
577
|
+
tests: suite.tests.filter((test) => test.tags?.some((t) => options.tags.includes(t)) || suite.tags?.some((t) => options.tags.includes(t)))
|
|
578
|
+
})).filter((suite) => suite.tests.length > 0);
|
|
579
|
+
const totalTests = countTests(suites);
|
|
580
|
+
console.log(`\nRunning ${totalTests} client conformance tests...\n`);
|
|
581
|
+
const server = new __durable_streams_server.DurableStreamTestServer({ port: options.serverPort ?? 0 });
|
|
582
|
+
await server.start();
|
|
583
|
+
const serverUrl = server.url;
|
|
584
|
+
console.log(`Reference server started at ${serverUrl}\n`);
|
|
585
|
+
let adapterPath = options.clientAdapter;
|
|
586
|
+
let adapterArgs = options.clientArgs ?? [];
|
|
587
|
+
if (adapterPath === `ts` || adapterPath === `typescript`) {
|
|
588
|
+
adapterPath = `npx`;
|
|
589
|
+
adapterArgs = [`tsx`, new URL(
|
|
590
|
+
`./adapters/typescript-adapter.ts`,
|
|
591
|
+
// Multi-status
|
|
592
|
+
// No result yet - will be retrieved via await
|
|
593
|
+
// Clean up
|
|
594
|
+
require("url").pathToFileURL(__filename).href
|
|
595
|
+
).pathname];
|
|
596
|
+
}
|
|
597
|
+
const client = new ClientAdapter(adapterPath, adapterArgs);
|
|
598
|
+
try {
|
|
599
|
+
const initResult = await client.init(serverUrl);
|
|
600
|
+
if (!initResult.success) throw new Error(`Failed to initialize client adapter: ${initResult.message}`);
|
|
601
|
+
let clientFeatures = {};
|
|
602
|
+
if (initResult.type === `init`) {
|
|
603
|
+
console.log(`Client: ${initResult.clientName} v${initResult.clientVersion}`);
|
|
604
|
+
if (initResult.features) {
|
|
605
|
+
clientFeatures = initResult.features;
|
|
606
|
+
const featureList = Object.entries(initResult.features).filter(([, v]) => v).map(([k]) => k);
|
|
607
|
+
console.log(`Features: ${featureList.join(`, `) || `none`}\n`);
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
const ctx = {
|
|
611
|
+
serverUrl,
|
|
612
|
+
variables: new Map(),
|
|
613
|
+
client,
|
|
614
|
+
verbose: options.verbose ?? false,
|
|
615
|
+
clientFeatures,
|
|
616
|
+
backgroundOps: new Map(),
|
|
617
|
+
commandTimeout: options.testTimeout ?? 3e4
|
|
618
|
+
};
|
|
619
|
+
for (const suite of suites) {
|
|
620
|
+
console.log(`\n${suite.name}`);
|
|
621
|
+
console.log(`─`.repeat(suite.name.length));
|
|
622
|
+
const suiteMissingFeatures = getMissingFeatures(suite.requires, clientFeatures);
|
|
623
|
+
for (const test of suite.tests) {
|
|
624
|
+
if (suiteMissingFeatures.length > 0) {
|
|
625
|
+
const result$1 = {
|
|
626
|
+
suite: suite.id,
|
|
627
|
+
test: test.id,
|
|
628
|
+
passed: true,
|
|
629
|
+
duration: 0,
|
|
630
|
+
skipped: true,
|
|
631
|
+
skipReason: `missing features: ${suiteMissingFeatures.join(`, `)}`
|
|
632
|
+
};
|
|
633
|
+
results.push(result$1);
|
|
634
|
+
console.log(` ○ ${test.name} (skipped: missing features: ${suiteMissingFeatures.join(`, `)})`);
|
|
635
|
+
continue;
|
|
636
|
+
}
|
|
637
|
+
const result = await runTestCase(test, ctx);
|
|
638
|
+
result.suite = suite.id;
|
|
639
|
+
results.push(result);
|
|
640
|
+
const icon = result.passed ? result.skipped ? `○` : `✓` : `✗`;
|
|
641
|
+
const status = result.skipped ? `skipped${result.skipReason ? `: ${result.skipReason}` : ``}` : result.passed ? `${result.duration}ms` : result.error;
|
|
642
|
+
console.log(` ${icon} ${test.name} (${status})`);
|
|
643
|
+
if (options.failFast && !result.passed && !result.skipped) break;
|
|
644
|
+
}
|
|
645
|
+
if (options.failFast && results.some((r) => !r.passed && !r.skipped)) break;
|
|
646
|
+
}
|
|
647
|
+
} finally {
|
|
648
|
+
await client.shutdown();
|
|
649
|
+
await server.stop();
|
|
650
|
+
}
|
|
651
|
+
const passed = results.filter((r) => r.passed && !r.skipped).length;
|
|
652
|
+
const failed = results.filter((r) => !r.passed).length;
|
|
653
|
+
const skipped = results.filter((r) => r.skipped).length;
|
|
654
|
+
const summary = {
|
|
655
|
+
total: results.length,
|
|
656
|
+
passed,
|
|
657
|
+
failed,
|
|
658
|
+
skipped,
|
|
659
|
+
duration: Date.now() - startTime,
|
|
660
|
+
results
|
|
661
|
+
};
|
|
662
|
+
console.log(`\n` + `═`.repeat(40));
|
|
663
|
+
console.log(`Total: ${summary.total} tests`);
|
|
664
|
+
console.log(`Passed: ${summary.passed}`);
|
|
665
|
+
console.log(`Failed: ${summary.failed}`);
|
|
666
|
+
console.log(`Skipped: ${summary.skipped}`);
|
|
667
|
+
console.log(`Duration: ${(summary.duration / 1e3).toFixed(2)}s`);
|
|
668
|
+
console.log(`═`.repeat(40) + `\n`);
|
|
669
|
+
if (failed > 0) {
|
|
670
|
+
console.log(`Failed tests:`);
|
|
671
|
+
for (const result of results.filter((r) => !r.passed)) console.log(` - ${result.suite}/${result.test}: ${result.error}`);
|
|
672
|
+
console.log();
|
|
673
|
+
}
|
|
674
|
+
return summary;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
//#endregion
|
|
678
|
+
//#region src/benchmark-scenarios.ts
|
|
679
|
+
const appendLatencyScenario = {
|
|
680
|
+
id: `latency-append`,
|
|
681
|
+
name: `Append Latency`,
|
|
682
|
+
description: `Measure time to complete a single append operation`,
|
|
683
|
+
category: `latency`,
|
|
684
|
+
config: {
|
|
685
|
+
warmupIterations: 10,
|
|
686
|
+
measureIterations: 100,
|
|
687
|
+
messageSize: 100
|
|
688
|
+
},
|
|
689
|
+
criteria: {
|
|
690
|
+
maxP50Ms: 20,
|
|
691
|
+
maxP99Ms: 100
|
|
692
|
+
},
|
|
693
|
+
createOperation: (ctx) => ({
|
|
694
|
+
op: `append`,
|
|
695
|
+
path: `${ctx.basePath}/stream`,
|
|
696
|
+
size: 100
|
|
697
|
+
}),
|
|
698
|
+
setup: (ctx) => {
|
|
699
|
+
ctx.setupData.streamPath = `${ctx.basePath}/stream`;
|
|
700
|
+
return Promise.resolve({});
|
|
701
|
+
}
|
|
702
|
+
};
|
|
703
|
+
const readLatencyScenario = {
|
|
704
|
+
id: `latency-read`,
|
|
705
|
+
name: `Read Latency`,
|
|
706
|
+
description: `Measure time to complete a single read operation`,
|
|
707
|
+
category: `latency`,
|
|
708
|
+
config: {
|
|
709
|
+
warmupIterations: 10,
|
|
710
|
+
measureIterations: 100,
|
|
711
|
+
messageSize: 100
|
|
712
|
+
},
|
|
713
|
+
criteria: {
|
|
714
|
+
maxP50Ms: 20,
|
|
715
|
+
maxP99Ms: 100
|
|
716
|
+
},
|
|
717
|
+
createOperation: (ctx) => ({
|
|
718
|
+
op: `read`,
|
|
719
|
+
path: `${ctx.basePath}/stream`,
|
|
720
|
+
offset: ctx.setupData.offset
|
|
721
|
+
}),
|
|
722
|
+
setup: (ctx) => {
|
|
723
|
+
ctx.setupData.streamPath = `${ctx.basePath}/stream`;
|
|
724
|
+
return Promise.resolve({});
|
|
725
|
+
}
|
|
726
|
+
};
|
|
727
|
+
const roundtripLatencyScenario = {
|
|
728
|
+
id: `latency-roundtrip`,
|
|
729
|
+
name: `Roundtrip Latency`,
|
|
730
|
+
description: `Measure time to append and immediately read back via long-poll`,
|
|
731
|
+
category: `latency`,
|
|
732
|
+
requires: [`longPoll`],
|
|
733
|
+
config: {
|
|
734
|
+
warmupIterations: 5,
|
|
735
|
+
measureIterations: 50,
|
|
736
|
+
messageSize: 100
|
|
737
|
+
},
|
|
738
|
+
criteria: {
|
|
739
|
+
maxP50Ms: 50,
|
|
740
|
+
maxP99Ms: 200
|
|
741
|
+
},
|
|
742
|
+
createOperation: (ctx) => ({
|
|
743
|
+
op: `roundtrip`,
|
|
744
|
+
path: `${ctx.basePath}/roundtrip-${ctx.iteration}`,
|
|
745
|
+
size: 100,
|
|
746
|
+
live: `long-poll`
|
|
747
|
+
})
|
|
748
|
+
};
|
|
749
|
+
const createLatencyScenario = {
|
|
750
|
+
id: `latency-create`,
|
|
751
|
+
name: `Create Latency`,
|
|
752
|
+
description: `Measure time to create a new stream`,
|
|
753
|
+
category: `latency`,
|
|
754
|
+
config: {
|
|
755
|
+
warmupIterations: 5,
|
|
756
|
+
measureIterations: 50,
|
|
757
|
+
messageSize: 0
|
|
758
|
+
},
|
|
759
|
+
criteria: {
|
|
760
|
+
maxP50Ms: 30,
|
|
761
|
+
maxP99Ms: 150
|
|
762
|
+
},
|
|
763
|
+
createOperation: (ctx) => ({
|
|
764
|
+
op: `create`,
|
|
765
|
+
path: `${ctx.basePath}/create-${ctx.iteration}`,
|
|
766
|
+
contentType: `application/octet-stream`
|
|
767
|
+
})
|
|
768
|
+
};
|
|
769
|
+
const smallMessageThroughputScenario = {
|
|
770
|
+
id: `throughput-small-messages`,
|
|
771
|
+
name: `Small Message Throughput`,
|
|
772
|
+
description: `Measure throughput for 100-byte messages at high concurrency`,
|
|
773
|
+
category: `throughput`,
|
|
774
|
+
requires: [`batching`],
|
|
775
|
+
config: {
|
|
776
|
+
warmupIterations: 2,
|
|
777
|
+
measureIterations: 10,
|
|
778
|
+
messageSize: 100,
|
|
779
|
+
concurrency: 200
|
|
780
|
+
},
|
|
781
|
+
criteria: { minOpsPerSecond: 1e3 },
|
|
782
|
+
createOperation: (ctx) => ({
|
|
783
|
+
op: `throughput_append`,
|
|
784
|
+
path: `${ctx.basePath}/throughput-small`,
|
|
785
|
+
count: 1e4,
|
|
786
|
+
size: 100,
|
|
787
|
+
concurrency: 200
|
|
788
|
+
})
|
|
789
|
+
};
|
|
790
|
+
const largeMessageThroughputScenario = {
|
|
791
|
+
id: `throughput-large-messages`,
|
|
792
|
+
name: `Large Message Throughput`,
|
|
793
|
+
description: `Measure throughput for 1MB messages`,
|
|
794
|
+
category: `throughput`,
|
|
795
|
+
requires: [`batching`],
|
|
796
|
+
config: {
|
|
797
|
+
warmupIterations: 1,
|
|
798
|
+
measureIterations: 5,
|
|
799
|
+
messageSize: 1024 * 1024,
|
|
800
|
+
concurrency: 10
|
|
801
|
+
},
|
|
802
|
+
criteria: { minOpsPerSecond: 20 },
|
|
803
|
+
createOperation: (ctx) => ({
|
|
804
|
+
op: `throughput_append`,
|
|
805
|
+
path: `${ctx.basePath}/throughput-large`,
|
|
806
|
+
count: 50,
|
|
807
|
+
size: 1024 * 1024,
|
|
808
|
+
concurrency: 10
|
|
809
|
+
})
|
|
810
|
+
};
|
|
811
|
+
const readThroughputScenario = {
|
|
812
|
+
id: `throughput-read`,
|
|
813
|
+
name: `Read Throughput`,
|
|
814
|
+
description: `Measure JSON parsing and iteration speed reading back messages`,
|
|
815
|
+
category: `throughput`,
|
|
816
|
+
config: {
|
|
817
|
+
warmupIterations: 1,
|
|
818
|
+
measureIterations: 5,
|
|
819
|
+
messageSize: 100
|
|
820
|
+
},
|
|
821
|
+
criteria: { minMBPerSecond: 3 },
|
|
822
|
+
createOperation: (ctx) => ({
|
|
823
|
+
op: `throughput_read`,
|
|
824
|
+
path: `${ctx.basePath}/throughput-read`,
|
|
825
|
+
expectedCount: ctx.setupData.expectedCount
|
|
826
|
+
}),
|
|
827
|
+
setup: (ctx) => {
|
|
828
|
+
ctx.setupData.expectedCount = 1e4;
|
|
829
|
+
return Promise.resolve({ data: { expectedCount: 1e4 } });
|
|
830
|
+
}
|
|
831
|
+
};
|
|
832
|
+
const sseLatencyScenario = {
|
|
833
|
+
id: `streaming-sse-latency`,
|
|
834
|
+
name: `SSE First Event Latency`,
|
|
835
|
+
description: `Measure time to receive first event via SSE`,
|
|
836
|
+
category: `streaming`,
|
|
837
|
+
requires: [`sse`],
|
|
838
|
+
config: {
|
|
839
|
+
warmupIterations: 3,
|
|
840
|
+
measureIterations: 20,
|
|
841
|
+
messageSize: 100
|
|
842
|
+
},
|
|
843
|
+
criteria: {
|
|
844
|
+
maxP50Ms: 100,
|
|
845
|
+
maxP99Ms: 500
|
|
846
|
+
},
|
|
847
|
+
createOperation: (ctx) => ({
|
|
848
|
+
op: `roundtrip`,
|
|
849
|
+
path: `${ctx.basePath}/sse-latency-${ctx.iteration}`,
|
|
850
|
+
size: 100,
|
|
851
|
+
live: `sse`,
|
|
852
|
+
contentType: `application/json`
|
|
853
|
+
})
|
|
854
|
+
};
|
|
855
|
+
const allScenarios = [
|
|
856
|
+
appendLatencyScenario,
|
|
857
|
+
readLatencyScenario,
|
|
858
|
+
roundtripLatencyScenario,
|
|
859
|
+
createLatencyScenario,
|
|
860
|
+
smallMessageThroughputScenario,
|
|
861
|
+
largeMessageThroughputScenario,
|
|
862
|
+
readThroughputScenario,
|
|
863
|
+
sseLatencyScenario
|
|
864
|
+
];
|
|
865
|
+
const scenariosByCategory = {
|
|
866
|
+
latency: allScenarios.filter((s) => s.category === `latency`),
|
|
867
|
+
throughput: allScenarios.filter((s) => s.category === `throughput`),
|
|
868
|
+
streaming: allScenarios.filter((s) => s.category === `streaming`)
|
|
869
|
+
};
|
|
870
|
+
function getScenarioById(id) {
|
|
871
|
+
return allScenarios.find((s) => s.id === id);
|
|
872
|
+
}
|
|
873
|
+
function getScenariosByCategory(category) {
|
|
874
|
+
return scenariosByCategory[category];
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
//#endregion
|
|
878
|
+
//#region src/benchmark-runner.ts
|
|
879
|
+
var BenchmarkClientAdapter = class {
|
|
880
|
+
process;
|
|
881
|
+
readline;
|
|
882
|
+
pendingResponse = null;
|
|
883
|
+
initialized = false;
|
|
884
|
+
constructor(executable, args = []) {
|
|
885
|
+
this.process = (0, node_child_process.spawn)(executable, args, { stdio: [
|
|
886
|
+
`pipe`,
|
|
887
|
+
`pipe`,
|
|
888
|
+
`pipe`
|
|
889
|
+
] });
|
|
890
|
+
if (!this.process.stdout || !this.process.stdin) throw new Error(`Failed to create client adapter process`);
|
|
891
|
+
this.readline = (0, node_readline.createInterface)({
|
|
892
|
+
input: this.process.stdout,
|
|
893
|
+
crlfDelay: Infinity
|
|
894
|
+
});
|
|
895
|
+
this.readline.on(`line`, (line) => {
|
|
896
|
+
if (this.pendingResponse) {
|
|
897
|
+
try {
|
|
898
|
+
const result = require_protocol.parseResult(line);
|
|
899
|
+
this.pendingResponse.resolve(result);
|
|
900
|
+
} catch {
|
|
901
|
+
this.pendingResponse.reject(new Error(`Failed to parse client response: ${line}`));
|
|
902
|
+
}
|
|
903
|
+
this.pendingResponse = null;
|
|
904
|
+
}
|
|
905
|
+
});
|
|
906
|
+
this.process.stderr?.on(`data`, (data) => {
|
|
907
|
+
console.error(`[client stderr] ${data.toString().trim()}`);
|
|
908
|
+
});
|
|
909
|
+
this.process.on(`error`, (err) => {
|
|
910
|
+
if (this.pendingResponse) {
|
|
911
|
+
this.pendingResponse.reject(err);
|
|
912
|
+
this.pendingResponse = null;
|
|
913
|
+
}
|
|
914
|
+
});
|
|
915
|
+
this.process.on(`exit`, (code) => {
|
|
916
|
+
if (this.pendingResponse) {
|
|
917
|
+
this.pendingResponse.reject(new Error(`Client adapter exited with code ${code}`));
|
|
918
|
+
this.pendingResponse = null;
|
|
919
|
+
}
|
|
920
|
+
});
|
|
921
|
+
}
|
|
922
|
+
async send(command, timeoutMs = 6e4) {
|
|
923
|
+
if (!this.process.stdin) throw new Error(`Client adapter stdin not available`);
|
|
924
|
+
return new Promise((resolve, reject) => {
|
|
925
|
+
const timeout = setTimeout(() => {
|
|
926
|
+
this.pendingResponse = null;
|
|
927
|
+
reject(new Error(`Command timed out after ${timeoutMs}ms: ${command.type}`));
|
|
928
|
+
}, timeoutMs);
|
|
929
|
+
this.pendingResponse = {
|
|
930
|
+
resolve: (result) => {
|
|
931
|
+
clearTimeout(timeout);
|
|
932
|
+
resolve(result);
|
|
933
|
+
},
|
|
934
|
+
reject: (error) => {
|
|
935
|
+
clearTimeout(timeout);
|
|
936
|
+
reject(error);
|
|
937
|
+
}
|
|
938
|
+
};
|
|
939
|
+
const line = require_protocol.serializeCommand(command) + `\n`;
|
|
940
|
+
this.process.stdin.write(line);
|
|
941
|
+
});
|
|
942
|
+
}
|
|
943
|
+
async init(serverUrl) {
|
|
944
|
+
const result = await this.send({
|
|
945
|
+
type: `init`,
|
|
946
|
+
serverUrl
|
|
947
|
+
});
|
|
948
|
+
if (result.success) this.initialized = true;
|
|
949
|
+
return result;
|
|
950
|
+
}
|
|
951
|
+
async benchmark(iterationId, operation) {
|
|
952
|
+
const result = await this.send({
|
|
953
|
+
type: `benchmark`,
|
|
954
|
+
iterationId,
|
|
955
|
+
operation
|
|
956
|
+
});
|
|
957
|
+
if (result.type === `benchmark`) return result;
|
|
958
|
+
return null;
|
|
959
|
+
}
|
|
960
|
+
async shutdown() {
|
|
961
|
+
if (this.initialized) try {
|
|
962
|
+
await this.send({ type: `shutdown` }, 5e3);
|
|
963
|
+
} catch {}
|
|
964
|
+
this.process.kill();
|
|
965
|
+
this.readline.close();
|
|
966
|
+
}
|
|
967
|
+
isInitialized() {
|
|
968
|
+
return this.initialized;
|
|
969
|
+
}
|
|
970
|
+
};
|
|
971
|
+
async function runScenario(scenario, client, serverUrl, clientFeatures, verbose, log) {
|
|
972
|
+
if (scenario.requires) {
|
|
973
|
+
const missing = scenario.requires.filter((f) => !clientFeatures[f]);
|
|
974
|
+
if (missing.length > 0) return {
|
|
975
|
+
scenario,
|
|
976
|
+
stats: require_protocol.calculateStats([]),
|
|
977
|
+
criteriaMet: true,
|
|
978
|
+
criteriaDetails: [],
|
|
979
|
+
skipped: true,
|
|
980
|
+
skipReason: `missing features: ${missing.join(`, `)}`
|
|
981
|
+
};
|
|
982
|
+
}
|
|
983
|
+
const basePath = `/bench-${(0, node_crypto.randomUUID)()}`;
|
|
984
|
+
const durations = [];
|
|
985
|
+
try {
|
|
986
|
+
const setupCtx = {
|
|
987
|
+
basePath,
|
|
988
|
+
iteration: 0,
|
|
989
|
+
setupData: {}
|
|
990
|
+
};
|
|
991
|
+
if (scenario.setup) await scenario.setup(setupCtx);
|
|
992
|
+
if (scenario.id === `latency-append` || scenario.id === `latency-read` || scenario.id.startsWith(`throughput-`)) {
|
|
993
|
+
const streamUrl = `${serverUrl}${basePath}/stream`;
|
|
994
|
+
await __durable_streams_client.DurableStream.create({
|
|
995
|
+
url: streamUrl,
|
|
996
|
+
contentType: `application/octet-stream`
|
|
997
|
+
});
|
|
998
|
+
if (scenario.id === `latency-read`) {
|
|
999
|
+
const stream = new __durable_streams_client.DurableStream({
|
|
1000
|
+
url: streamUrl,
|
|
1001
|
+
contentType: `application/octet-stream`
|
|
1002
|
+
});
|
|
1003
|
+
const chunk = new Uint8Array(1024).fill(42);
|
|
1004
|
+
for (let i = 0; i < 10; i++) await stream.append(chunk);
|
|
1005
|
+
}
|
|
1006
|
+
if (scenario.id === `throughput-read`) {
|
|
1007
|
+
const url = `${serverUrl}${basePath}/throughput-read`;
|
|
1008
|
+
await __durable_streams_client.DurableStream.create({
|
|
1009
|
+
url,
|
|
1010
|
+
contentType: `application/json`
|
|
1011
|
+
});
|
|
1012
|
+
const ds = new __durable_streams_client.DurableStream({
|
|
1013
|
+
url,
|
|
1014
|
+
contentType: `application/json`
|
|
1015
|
+
});
|
|
1016
|
+
const messages = [];
|
|
1017
|
+
for (let i = 0; i < 1e4; i++) messages.push({
|
|
1018
|
+
n: i,
|
|
1019
|
+
data: `message-${i}-padding-for-size`
|
|
1020
|
+
});
|
|
1021
|
+
await Promise.all(messages.map((msg) => ds.append(msg)));
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
if (verbose) log(` Warmup: ${scenario.config.warmupIterations} iterations...`);
|
|
1025
|
+
for (let i = 0; i < scenario.config.warmupIterations; i++) {
|
|
1026
|
+
const ctx = {
|
|
1027
|
+
basePath,
|
|
1028
|
+
iteration: i,
|
|
1029
|
+
setupData: setupCtx.setupData
|
|
1030
|
+
};
|
|
1031
|
+
const operation = scenario.createOperation(ctx);
|
|
1032
|
+
await client.benchmark(`warmup-${i}`, operation);
|
|
1033
|
+
}
|
|
1034
|
+
if (verbose) log(` Measuring: ${scenario.config.measureIterations} iterations...`);
|
|
1035
|
+
let totalMessagesProcessed = 0;
|
|
1036
|
+
let totalBytesTransferred = 0;
|
|
1037
|
+
for (let i = 0; i < scenario.config.measureIterations; i++) {
|
|
1038
|
+
const ctx = {
|
|
1039
|
+
basePath,
|
|
1040
|
+
iteration: i,
|
|
1041
|
+
setupData: setupCtx.setupData
|
|
1042
|
+
};
|
|
1043
|
+
const operation = scenario.createOperation(ctx);
|
|
1044
|
+
const result = await client.benchmark(`measure-${i}`, operation);
|
|
1045
|
+
if (result) {
|
|
1046
|
+
durations.push(BigInt(result.durationNs));
|
|
1047
|
+
if (result.metrics) {
|
|
1048
|
+
totalMessagesProcessed += result.metrics.messagesProcessed ?? 0;
|
|
1049
|
+
totalBytesTransferred += result.metrics.bytesTransferred ?? 0;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
if (scenario.cleanup) await scenario.cleanup(setupCtx);
|
|
1054
|
+
if (durations.length === 0) return {
|
|
1055
|
+
scenario,
|
|
1056
|
+
stats: require_protocol.calculateStats([]),
|
|
1057
|
+
criteriaMet: false,
|
|
1058
|
+
criteriaDetails: [],
|
|
1059
|
+
skipped: false,
|
|
1060
|
+
error: `No benchmark samples collected (adapter returned no results)`
|
|
1061
|
+
};
|
|
1062
|
+
const stats = require_protocol.calculateStats(durations);
|
|
1063
|
+
const totalTimeMs = durations.reduce((sum, ns) => sum + Number(ns) / 1e6, 0);
|
|
1064
|
+
const totalTimeSec = totalTimeMs / 1e3;
|
|
1065
|
+
let computedOpsPerSec;
|
|
1066
|
+
let computedMbPerSec;
|
|
1067
|
+
if (scenario.category === `throughput`) {
|
|
1068
|
+
if (totalMessagesProcessed > 0 && totalTimeSec > 0) computedOpsPerSec = totalMessagesProcessed / totalTimeSec;
|
|
1069
|
+
else if (stats.mean > 0) computedOpsPerSec = 1e3 / stats.mean;
|
|
1070
|
+
if (totalBytesTransferred > 0 && totalTimeSec > 0) computedMbPerSec = totalBytesTransferred / 1024 / 1024 / totalTimeSec;
|
|
1071
|
+
}
|
|
1072
|
+
const criteriaDetails = [];
|
|
1073
|
+
let criteriaMet = true;
|
|
1074
|
+
if (scenario.criteria) {
|
|
1075
|
+
if (scenario.criteria.maxP50Ms !== void 0) {
|
|
1076
|
+
const met = stats.median <= scenario.criteria.maxP50Ms;
|
|
1077
|
+
criteriaDetails.push({
|
|
1078
|
+
criterion: `p50 <= ${scenario.criteria.maxP50Ms}ms`,
|
|
1079
|
+
met,
|
|
1080
|
+
actual: stats.median,
|
|
1081
|
+
expected: scenario.criteria.maxP50Ms
|
|
1082
|
+
});
|
|
1083
|
+
if (!met) criteriaMet = false;
|
|
1084
|
+
}
|
|
1085
|
+
if (scenario.criteria.maxP99Ms !== void 0) {
|
|
1086
|
+
const met = stats.p99 <= scenario.criteria.maxP99Ms;
|
|
1087
|
+
criteriaDetails.push({
|
|
1088
|
+
criterion: `p99 <= ${scenario.criteria.maxP99Ms}ms`,
|
|
1089
|
+
met,
|
|
1090
|
+
actual: stats.p99,
|
|
1091
|
+
expected: scenario.criteria.maxP99Ms
|
|
1092
|
+
});
|
|
1093
|
+
if (!met) criteriaMet = false;
|
|
1094
|
+
}
|
|
1095
|
+
if (scenario.criteria.minOpsPerSecond !== void 0) {
|
|
1096
|
+
const opsPerSec = computedOpsPerSec ?? 0;
|
|
1097
|
+
const met = opsPerSec >= scenario.criteria.minOpsPerSecond;
|
|
1098
|
+
criteriaDetails.push({
|
|
1099
|
+
criterion: `ops/sec >= ${scenario.criteria.minOpsPerSecond}`,
|
|
1100
|
+
met,
|
|
1101
|
+
actual: opsPerSec,
|
|
1102
|
+
expected: scenario.criteria.minOpsPerSecond
|
|
1103
|
+
});
|
|
1104
|
+
if (!met) criteriaMet = false;
|
|
1105
|
+
}
|
|
1106
|
+
if (scenario.criteria.minMBPerSecond !== void 0) {
|
|
1107
|
+
const mbPerSec = totalTimeSec > 0 ? totalBytesTransferred / 1024 / 1024 / totalTimeSec : 0;
|
|
1108
|
+
const met = mbPerSec >= scenario.criteria.minMBPerSecond;
|
|
1109
|
+
criteriaDetails.push({
|
|
1110
|
+
criterion: `MB/sec >= ${scenario.criteria.minMBPerSecond}`,
|
|
1111
|
+
met,
|
|
1112
|
+
actual: mbPerSec,
|
|
1113
|
+
expected: scenario.criteria.minMBPerSecond
|
|
1114
|
+
});
|
|
1115
|
+
if (!met) criteriaMet = false;
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
return {
|
|
1119
|
+
scenario,
|
|
1120
|
+
stats,
|
|
1121
|
+
criteriaMet,
|
|
1122
|
+
criteriaDetails,
|
|
1123
|
+
skipped: false,
|
|
1124
|
+
opsPerSec: computedOpsPerSec,
|
|
1125
|
+
mbPerSec: computedMbPerSec
|
|
1126
|
+
};
|
|
1127
|
+
} catch (err) {
|
|
1128
|
+
return {
|
|
1129
|
+
scenario,
|
|
1130
|
+
stats: require_protocol.calculateStats([]),
|
|
1131
|
+
criteriaMet: false,
|
|
1132
|
+
criteriaDetails: [],
|
|
1133
|
+
skipped: false,
|
|
1134
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1135
|
+
};
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
function printConsoleResults(summary) {
|
|
1139
|
+
console.log(`\n${`=`.repeat(60)}`);
|
|
1140
|
+
console.log(`CLIENT BENCHMARK RESULTS`);
|
|
1141
|
+
console.log(`${`=`.repeat(60)}`);
|
|
1142
|
+
console.log(`Adapter: ${summary.adapter} v${summary.adapterVersion}`);
|
|
1143
|
+
console.log(`Server: ${summary.serverUrl}`);
|
|
1144
|
+
console.log(`Timestamp: ${summary.timestamp}`);
|
|
1145
|
+
console.log(`Duration: ${(summary.duration / 1e3).toFixed(2)}s`);
|
|
1146
|
+
console.log();
|
|
1147
|
+
const byCategory = {
|
|
1148
|
+
latency: summary.results.filter((r) => r.scenario.category === `latency`),
|
|
1149
|
+
throughput: summary.results.filter((r) => r.scenario.category === `throughput`),
|
|
1150
|
+
streaming: summary.results.filter((r) => r.scenario.category === `streaming`)
|
|
1151
|
+
};
|
|
1152
|
+
for (const [category, results] of Object.entries(byCategory)) {
|
|
1153
|
+
if (results.length === 0) continue;
|
|
1154
|
+
console.log(`\n${category.toUpperCase()}`);
|
|
1155
|
+
console.log(`${`-`.repeat(40)}`);
|
|
1156
|
+
for (const result of results) {
|
|
1157
|
+
const icon = result.skipped ? `○` : result.criteriaMet ? `✓` : `✗`;
|
|
1158
|
+
const status = result.skipped ? `skipped: ${result.skipReason}` : result.error ? `error: ${result.error}` : result.criteriaMet ? `passed` : `failed`;
|
|
1159
|
+
console.log(`${icon} ${result.scenario.name} (${status})`);
|
|
1160
|
+
if (!result.skipped && !result.error) {
|
|
1161
|
+
const formatted = require_protocol.formatStats(result.stats);
|
|
1162
|
+
if (result.scenario.category === `throughput`) {
|
|
1163
|
+
const mbStr = result.mbPerSec ? result.mbPerSec.toLocaleString(`en-US`, {
|
|
1164
|
+
minimumFractionDigits: 1,
|
|
1165
|
+
maximumFractionDigits: 1
|
|
1166
|
+
}) : `N/A`;
|
|
1167
|
+
if (result.scenario.id === `throughput-read`) console.log(` MB/sec: ${mbStr}`);
|
|
1168
|
+
else {
|
|
1169
|
+
const opsStr = result.opsPerSec ? result.opsPerSec.toLocaleString(`en-US`, { maximumFractionDigits: 0 }) : `N/A`;
|
|
1170
|
+
console.log(` Ops/sec: ${opsStr} MB/sec: ${mbStr}`);
|
|
1171
|
+
}
|
|
1172
|
+
} else console.log(` Median: ${formatted.Median} P99: ${formatted.P99}`);
|
|
1173
|
+
if (!result.criteriaMet) for (const c of result.criteriaDetails.filter((d) => !d.met)) console.log(` ✗ ${c.criterion}: got ${c.actual.toFixed(2)}, expected ${c.expected}`);
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
console.log(`\n${`=`.repeat(60)}`);
|
|
1178
|
+
console.log(`Summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.skipped} skipped`);
|
|
1179
|
+
console.log(`${`=`.repeat(60)}\n`);
|
|
1180
|
+
}
|
|
1181
|
+
function generateMarkdownReport(summary) {
|
|
1182
|
+
const lines = [];
|
|
1183
|
+
const statusIcon = summary.failed === 0 ? `✓` : `✗`;
|
|
1184
|
+
const statusText = summary.failed === 0 ? `${summary.passed} passed` : `${summary.passed} passed, ${summary.failed} failed`;
|
|
1185
|
+
lines.push(`<details>`);
|
|
1186
|
+
lines.push(`<summary><strong>${summary.adapter}</strong>: ${statusText} ${statusIcon}</summary>`);
|
|
1187
|
+
lines.push(``);
|
|
1188
|
+
lines.push(`### ${summary.adapter} v${summary.adapterVersion}`);
|
|
1189
|
+
lines.push(``);
|
|
1190
|
+
lines.push(`**Server**: ${summary.serverUrl}`);
|
|
1191
|
+
lines.push(`**Date**: ${summary.timestamp}`);
|
|
1192
|
+
lines.push(`**Duration**: ${(summary.duration / 1e3).toFixed(2)}s`);
|
|
1193
|
+
lines.push(``);
|
|
1194
|
+
const latencyResults = summary.results.filter((r) => r.scenario.category === `latency` && !r.skipped && !r.error);
|
|
1195
|
+
if (latencyResults.length > 0) {
|
|
1196
|
+
lines.push(`#### Latency`);
|
|
1197
|
+
lines.push(``);
|
|
1198
|
+
lines.push(`Single-operation latency tests measure the time for individual operations to complete.`);
|
|
1199
|
+
lines.push(``);
|
|
1200
|
+
lines.push(`| Scenario | Description | Min | Median | P95 | P99 | Max | Status |`);
|
|
1201
|
+
lines.push(`|----------|-------------|-----|--------|-----|-----|-----|--------|`);
|
|
1202
|
+
for (const r of latencyResults) {
|
|
1203
|
+
const s = r.stats;
|
|
1204
|
+
const status = r.criteriaMet ? `Pass` : `Fail`;
|
|
1205
|
+
lines.push(`| ${r.scenario.name} | ${r.scenario.description} | ${s.min.toFixed(2)}ms | ${s.median.toFixed(2)}ms | ${s.p95.toFixed(2)}ms | ${s.p99.toFixed(2)}ms | ${s.max.toFixed(2)}ms | ${status} |`);
|
|
1206
|
+
}
|
|
1207
|
+
lines.push(``);
|
|
1208
|
+
}
|
|
1209
|
+
const throughputResults = summary.results.filter((r) => r.scenario.category === `throughput` && !r.skipped && !r.error);
|
|
1210
|
+
if (throughputResults.length > 0) {
|
|
1211
|
+
lines.push(`#### Throughput`);
|
|
1212
|
+
lines.push(``);
|
|
1213
|
+
lines.push(`Throughput tests measure how quickly the client can batch and send/receive data.`);
|
|
1214
|
+
lines.push(`Writes use automatic batching to maximize operations per second.`);
|
|
1215
|
+
lines.push(`Reads measure JSON parsing and iteration speed.`);
|
|
1216
|
+
lines.push(``);
|
|
1217
|
+
lines.push(`| Scenario | Description | Ops/sec | MB/sec | Status |`);
|
|
1218
|
+
lines.push(`|----------|-------------|---------|--------|--------|`);
|
|
1219
|
+
for (const r of throughputResults) {
|
|
1220
|
+
const opsPerSec = r.scenario.id === `throughput-read` ? `-` : r.opsPerSec !== void 0 ? r.opsPerSec.toLocaleString(`en-US`, { maximumFractionDigits: 0 }) : `N/A`;
|
|
1221
|
+
const mbPerSec = r.mbPerSec !== void 0 ? r.mbPerSec.toLocaleString(`en-US`, {
|
|
1222
|
+
minimumFractionDigits: 1,
|
|
1223
|
+
maximumFractionDigits: 1
|
|
1224
|
+
}) : `N/A`;
|
|
1225
|
+
const status = r.criteriaMet ? `Pass` : `Fail`;
|
|
1226
|
+
lines.push(`| ${r.scenario.name} | ${r.scenario.description} | ${opsPerSec} | ${mbPerSec} | ${status} |`);
|
|
1227
|
+
}
|
|
1228
|
+
lines.push(``);
|
|
1229
|
+
}
|
|
1230
|
+
const streamingResults = summary.results.filter((r) => r.scenario.category === `streaming` && !r.skipped && !r.error);
|
|
1231
|
+
if (streamingResults.length > 0) {
|
|
1232
|
+
lines.push(`#### Streaming`);
|
|
1233
|
+
lines.push(``);
|
|
1234
|
+
lines.push(`Streaming tests measure real-time event delivery via SSE (Server-Sent Events).`);
|
|
1235
|
+
lines.push(``);
|
|
1236
|
+
lines.push(`| Scenario | Description | Min | Median | P95 | P99 | Max | Status |`);
|
|
1237
|
+
lines.push(`|----------|-------------|-----|--------|-----|-----|-----|--------|`);
|
|
1238
|
+
for (const r of streamingResults) {
|
|
1239
|
+
const s = r.stats;
|
|
1240
|
+
const status = r.criteriaMet ? `Pass` : `Fail`;
|
|
1241
|
+
lines.push(`| ${r.scenario.name} | ${r.scenario.description} | ${s.min.toFixed(2)}ms | ${s.median.toFixed(2)}ms | ${s.p95.toFixed(2)}ms | ${s.p99.toFixed(2)}ms | ${s.max.toFixed(2)}ms | ${status} |`);
|
|
1242
|
+
}
|
|
1243
|
+
lines.push(``);
|
|
1244
|
+
}
|
|
1245
|
+
lines.push(`#### Summary`);
|
|
1246
|
+
lines.push(``);
|
|
1247
|
+
lines.push(`- **Passed**: ${summary.passed}`);
|
|
1248
|
+
lines.push(`- **Failed**: ${summary.failed}`);
|
|
1249
|
+
lines.push(`- **Skipped**: ${summary.skipped}`);
|
|
1250
|
+
lines.push(``);
|
|
1251
|
+
lines.push(`</details>`);
|
|
1252
|
+
return lines.join(`\n`);
|
|
1253
|
+
}
|
|
1254
|
+
async function runBenchmarks(options) {
|
|
1255
|
+
const startTime = Date.now();
|
|
1256
|
+
const results = [];
|
|
1257
|
+
const log = (message) => {
|
|
1258
|
+
if (options.format === `json` || options.format === `markdown`) process.stderr.write(message + `\n`);
|
|
1259
|
+
else console.log(message);
|
|
1260
|
+
};
|
|
1261
|
+
let scenarios = allScenarios;
|
|
1262
|
+
if (options.scenarios && options.scenarios.length > 0) scenarios = options.scenarios.map((id) => getScenarioById(id)).filter((s) => s !== void 0);
|
|
1263
|
+
if (options.categories && options.categories.length > 0) scenarios = scenarios.filter((s) => options.categories.includes(s.category));
|
|
1264
|
+
log(`\nRunning ${scenarios.length} benchmark scenarios...\n`);
|
|
1265
|
+
const server = new __durable_streams_server.DurableStreamTestServer({ port: options.serverPort ?? 0 });
|
|
1266
|
+
await server.start();
|
|
1267
|
+
const serverUrl = server.url;
|
|
1268
|
+
log(`Reference server started at ${serverUrl}\n`);
|
|
1269
|
+
let adapterPath = options.clientAdapter;
|
|
1270
|
+
let adapterArgs = options.clientArgs ?? [];
|
|
1271
|
+
if (adapterPath === `ts` || adapterPath === `typescript`) {
|
|
1272
|
+
adapterPath = `npx`;
|
|
1273
|
+
adapterArgs = [`tsx`, new URL(`./adapters/typescript-adapter.ts`, require("url").pathToFileURL(__filename).href).pathname];
|
|
1274
|
+
}
|
|
1275
|
+
const client = new BenchmarkClientAdapter(adapterPath, adapterArgs);
|
|
1276
|
+
let adapterName = `unknown`;
|
|
1277
|
+
let adapterVersion = `0.0.0`;
|
|
1278
|
+
let clientFeatures = {};
|
|
1279
|
+
try {
|
|
1280
|
+
const initResult = await client.init(serverUrl);
|
|
1281
|
+
if (!initResult.success) throw new Error(`Failed to initialize client adapter`);
|
|
1282
|
+
if (initResult.type === `init`) {
|
|
1283
|
+
adapterName = initResult.clientName;
|
|
1284
|
+
adapterVersion = initResult.clientVersion;
|
|
1285
|
+
clientFeatures = initResult.features ?? {};
|
|
1286
|
+
log(`Client: ${adapterName} v${adapterVersion}`);
|
|
1287
|
+
const featureList = Object.entries(clientFeatures).filter(([, v]) => v).map(([k]) => k);
|
|
1288
|
+
log(`Features: ${featureList.join(`, `) || `none`}\n`);
|
|
1289
|
+
}
|
|
1290
|
+
for (const scenario of scenarios) {
|
|
1291
|
+
log(`\n${scenario.name}`);
|
|
1292
|
+
log(`${`─`.repeat(scenario.name.length)}`);
|
|
1293
|
+
log(`${scenario.description}`);
|
|
1294
|
+
const result = await runScenario(scenario, client, serverUrl, clientFeatures, options.verbose ?? false, log);
|
|
1295
|
+
results.push(result);
|
|
1296
|
+
if (result.skipped) log(` Skipped: ${result.skipReason}`);
|
|
1297
|
+
else if (result.error) log(` Error: ${result.error}`);
|
|
1298
|
+
else {
|
|
1299
|
+
const icon = result.criteriaMet ? `✓` : `✗`;
|
|
1300
|
+
if (result.scenario.category === `throughput`) {
|
|
1301
|
+
const mbStr = result.mbPerSec ? result.mbPerSec.toLocaleString(`en-US`, {
|
|
1302
|
+
minimumFractionDigits: 1,
|
|
1303
|
+
maximumFractionDigits: 1
|
|
1304
|
+
}) : `N/A`;
|
|
1305
|
+
if (result.scenario.id === `throughput-read`) log(` ${icon} MB/sec: ${mbStr}`);
|
|
1306
|
+
else {
|
|
1307
|
+
const opsStr = result.opsPerSec ? result.opsPerSec.toLocaleString(`en-US`, { maximumFractionDigits: 0 }) : `N/A`;
|
|
1308
|
+
log(` ${icon} Ops/sec: ${opsStr}, MB/sec: ${mbStr}`);
|
|
1309
|
+
}
|
|
1310
|
+
} else log(` ${icon} Median: ${result.stats.median.toFixed(2)}ms, P99: ${result.stats.p99.toFixed(2)}ms`);
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
} finally {
|
|
1314
|
+
await client.shutdown();
|
|
1315
|
+
await server.stop();
|
|
1316
|
+
}
|
|
1317
|
+
const summary = {
|
|
1318
|
+
adapter: adapterName,
|
|
1319
|
+
adapterVersion,
|
|
1320
|
+
serverUrl,
|
|
1321
|
+
timestamp: new Date().toISOString(),
|
|
1322
|
+
duration: Date.now() - startTime,
|
|
1323
|
+
results,
|
|
1324
|
+
passed: results.filter((r) => !r.skipped && !r.error && r.criteriaMet).length,
|
|
1325
|
+
failed: results.filter((r) => !r.skipped && (!r.criteriaMet || r.error)).length,
|
|
1326
|
+
skipped: results.filter((r) => r.skipped).length
|
|
1327
|
+
};
|
|
1328
|
+
switch (options.format) {
|
|
1329
|
+
case `json`:
|
|
1330
|
+
console.log(JSON.stringify(summary, null, 2));
|
|
1331
|
+
break;
|
|
1332
|
+
case `markdown`:
|
|
1333
|
+
console.log(generateMarkdownReport(summary));
|
|
1334
|
+
break;
|
|
1335
|
+
default: printConsoleResults(summary);
|
|
1336
|
+
}
|
|
1337
|
+
return summary;
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
//#endregion
|
|
1341
|
+
Object.defineProperty(exports, 'allScenarios', {
|
|
1342
|
+
enumerable: true,
|
|
1343
|
+
get: function () {
|
|
1344
|
+
return allScenarios;
|
|
1345
|
+
}
|
|
1346
|
+
});
|
|
1347
|
+
Object.defineProperty(exports, 'countTests', {
|
|
1348
|
+
enumerable: true,
|
|
1349
|
+
get: function () {
|
|
1350
|
+
return countTests;
|
|
1351
|
+
}
|
|
1352
|
+
});
|
|
1353
|
+
Object.defineProperty(exports, 'filterByCategory', {
|
|
1354
|
+
enumerable: true,
|
|
1355
|
+
get: function () {
|
|
1356
|
+
return filterByCategory;
|
|
1357
|
+
}
|
|
1358
|
+
});
|
|
1359
|
+
Object.defineProperty(exports, 'getScenarioById', {
|
|
1360
|
+
enumerable: true,
|
|
1361
|
+
get: function () {
|
|
1362
|
+
return getScenarioById;
|
|
1363
|
+
}
|
|
1364
|
+
});
|
|
1365
|
+
Object.defineProperty(exports, 'getScenariosByCategory', {
|
|
1366
|
+
enumerable: true,
|
|
1367
|
+
get: function () {
|
|
1368
|
+
return getScenariosByCategory;
|
|
1369
|
+
}
|
|
1370
|
+
});
|
|
1371
|
+
Object.defineProperty(exports, 'loadEmbeddedTestSuites', {
|
|
1372
|
+
enumerable: true,
|
|
1373
|
+
get: function () {
|
|
1374
|
+
return loadEmbeddedTestSuites;
|
|
1375
|
+
}
|
|
1376
|
+
});
|
|
1377
|
+
Object.defineProperty(exports, 'loadTestSuites', {
|
|
1378
|
+
enumerable: true,
|
|
1379
|
+
get: function () {
|
|
1380
|
+
return loadTestSuites;
|
|
1381
|
+
}
|
|
1382
|
+
});
|
|
1383
|
+
Object.defineProperty(exports, 'runBenchmarks', {
|
|
1384
|
+
enumerable: true,
|
|
1385
|
+
get: function () {
|
|
1386
|
+
return runBenchmarks;
|
|
1387
|
+
}
|
|
1388
|
+
});
|
|
1389
|
+
Object.defineProperty(exports, 'runConformanceTests', {
|
|
1390
|
+
enumerable: true,
|
|
1391
|
+
get: function () {
|
|
1392
|
+
return runConformanceTests;
|
|
1393
|
+
}
|
|
1394
|
+
});
|
|
1395
|
+
Object.defineProperty(exports, 'scenariosByCategory', {
|
|
1396
|
+
enumerable: true,
|
|
1397
|
+
get: function () {
|
|
1398
|
+
return scenariosByCategory;
|
|
1399
|
+
}
|
|
1400
|
+
});
|