@durable-streams/client-conformance-tests 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/typescript-adapter.cjs +75 -2
- package/dist/adapters/typescript-adapter.js +76 -3
- package/dist/{benchmark-runner-C_Yghc8f.js → benchmark-runner-CrE6JkbX.js} +106 -12
- package/dist/{benchmark-runner-CLAR9oLd.cjs → benchmark-runner-Db4he452.cjs} +107 -12
- package/dist/cli.cjs +1 -1
- package/dist/cli.js +1 -1
- package/dist/index.cjs +1 -1
- package/dist/index.d.cts +126 -11
- package/dist/index.d.ts +126 -11
- package/dist/index.js +1 -1
- package/dist/{protocol-3cf94Xyb.d.cts → protocol-D37G3c4e.d.cts} +80 -4
- package/dist/{protocol-DyEvTHPF.d.ts → protocol-Mcbiq3nQ.d.ts} +80 -4
- package/dist/protocol.d.cts +2 -2
- package/dist/protocol.d.ts +2 -2
- package/package.json +3 -3
- package/src/adapters/typescript-adapter.ts +127 -5
- package/src/protocol.ts +85 -1
- package/src/runner.ts +202 -17
- package/src/test-cases.ts +130 -8
- package/test-cases/consumer/error-handling.yaml +42 -0
- package/test-cases/consumer/fault-injection.yaml +202 -0
- package/test-cases/consumer/offset-handling.yaml +209 -0
- package/test-cases/producer/idempotent/autoclaim.yaml +214 -0
- package/test-cases/producer/idempotent/batching.yaml +98 -0
- package/test-cases/producer/idempotent/concurrent-requests.yaml +100 -0
- package/test-cases/producer/idempotent/epoch-management.yaml +333 -0
- package/test-cases/producer/idempotent/error-handling.yaml +194 -0
- package/test-cases/producer/idempotent/multi-producer.yaml +322 -0
- package/test-cases/producer/idempotent/sequence-validation.yaml +339 -0
- package/test-cases/producer/idempotent-json-batching.yaml +134 -0
|
@@ -55,10 +55,52 @@ interface AppendCommand {
|
|
|
55
55
|
data: string;
|
|
56
56
|
/** Whether data is base64 encoded binary */
|
|
57
57
|
binary?: boolean;
|
|
58
|
-
/** Optional sequence number for ordering */
|
|
58
|
+
/** Optional sequence number for ordering (Stream-Seq header) */
|
|
59
59
|
seq?: number;
|
|
60
60
|
/** Custom headers to include */
|
|
61
61
|
headers?: Record<string, string>;
|
|
62
|
+
/** Producer ID for idempotent producers */
|
|
63
|
+
producerId?: string;
|
|
64
|
+
/** Producer epoch for idempotent producers */
|
|
65
|
+
producerEpoch?: number;
|
|
66
|
+
/** Producer sequence for idempotent producers */
|
|
67
|
+
producerSeq?: number;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Append via IdempotentProducer client (tests client-side exactly-once semantics).
|
|
71
|
+
*/
|
|
72
|
+
interface IdempotentAppendCommand {
|
|
73
|
+
type: `idempotent-append`;
|
|
74
|
+
path: string;
|
|
75
|
+
/** Data to append (string - will be JSON parsed for JSON streams) */
|
|
76
|
+
data: string;
|
|
77
|
+
/** Producer ID */
|
|
78
|
+
producerId: string;
|
|
79
|
+
/** Producer epoch */
|
|
80
|
+
epoch: number;
|
|
81
|
+
/** Auto-claim epoch on 403 */
|
|
82
|
+
autoClaim: boolean;
|
|
83
|
+
/** Custom headers to include */
|
|
84
|
+
headers?: Record<string, string>;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Batch append via IdempotentProducer client (tests client-side JSON batching).
|
|
88
|
+
*/
|
|
89
|
+
interface IdempotentAppendBatchCommand {
|
|
90
|
+
type: `idempotent-append-batch`;
|
|
91
|
+
path: string;
|
|
92
|
+
/** Items to append - will be batched by the client */
|
|
93
|
+
items: Array<string>;
|
|
94
|
+
/** Producer ID */
|
|
95
|
+
producerId: string;
|
|
96
|
+
/** Producer epoch */
|
|
97
|
+
epoch: number;
|
|
98
|
+
/** Auto-claim epoch on 403 */
|
|
99
|
+
autoClaim: boolean;
|
|
100
|
+
/** Max concurrent batches in flight (default 1, set higher to test 409 retry) */
|
|
101
|
+
maxInFlight?: number;
|
|
102
|
+
/** Custom headers to include */
|
|
103
|
+
headers?: Record<string, string>;
|
|
62
104
|
}
|
|
63
105
|
/**
|
|
64
106
|
* Read from a stream (GET request).
|
|
@@ -193,7 +235,7 @@ interface BenchmarkThroughputReadOp {
|
|
|
193
235
|
/**
|
|
194
236
|
* All possible commands from test runner to client.
|
|
195
237
|
*/
|
|
196
|
-
type TestCommand = InitCommand | CreateCommand | ConnectCommand | AppendCommand | ReadCommand | HeadCommand | DeleteCommand | ShutdownCommand | SetDynamicHeaderCommand | SetDynamicParamCommand | ClearDynamicCommand | BenchmarkCommand;
|
|
238
|
+
type TestCommand = InitCommand | CreateCommand | ConnectCommand | AppendCommand | IdempotentAppendCommand | IdempotentAppendBatchCommand | ReadCommand | HeadCommand | DeleteCommand | ShutdownCommand | SetDynamicHeaderCommand | SetDynamicParamCommand | ClearDynamicCommand | BenchmarkCommand;
|
|
197
239
|
/**
|
|
198
240
|
* Successful initialization result.
|
|
199
241
|
*/
|
|
@@ -256,6 +298,40 @@ interface AppendResult {
|
|
|
256
298
|
headersSent?: Record<string, string>;
|
|
257
299
|
/** Params that were sent in the request (for dynamic param testing) */
|
|
258
300
|
paramsSent?: Record<string, string>;
|
|
301
|
+
/** Whether this was a duplicate (204 response) - for idempotent producers */
|
|
302
|
+
duplicate?: boolean;
|
|
303
|
+
/** Current producer epoch from server (on 200 or 403) */
|
|
304
|
+
producerEpoch?: number;
|
|
305
|
+
/** Server's highest accepted sequence for this (stream, producerId, epoch) - returned in Producer-Seq header on 200/204 */
|
|
306
|
+
producerSeq?: number;
|
|
307
|
+
/** Expected producer sequence (on 409 sequence gap) */
|
|
308
|
+
producerExpectedSeq?: number;
|
|
309
|
+
/** Received producer sequence (on 409 sequence gap) */
|
|
310
|
+
producerReceivedSeq?: number;
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Successful idempotent-append result.
|
|
314
|
+
*/
|
|
315
|
+
interface IdempotentAppendResult {
|
|
316
|
+
type: `idempotent-append`;
|
|
317
|
+
success: true;
|
|
318
|
+
status: number;
|
|
319
|
+
/** New offset after append */
|
|
320
|
+
offset?: string;
|
|
321
|
+
/** Whether this was a duplicate */
|
|
322
|
+
duplicate?: boolean;
|
|
323
|
+
/** Server's highest accepted sequence for this (stream, producerId, epoch) - returned in Producer-Seq header */
|
|
324
|
+
producerSeq?: number;
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Successful idempotent-append-batch result.
|
|
328
|
+
*/
|
|
329
|
+
interface IdempotentAppendBatchResult {
|
|
330
|
+
type: `idempotent-append-batch`;
|
|
331
|
+
success: true;
|
|
332
|
+
status: number;
|
|
333
|
+
/** Server's highest accepted sequence for this (stream, producerId, epoch) - returned in Producer-Seq header */
|
|
334
|
+
producerSeq?: number;
|
|
259
335
|
}
|
|
260
336
|
/**
|
|
261
337
|
* A chunk of data read from the stream.
|
|
@@ -385,7 +461,7 @@ interface ErrorResult {
|
|
|
385
461
|
/**
|
|
386
462
|
* All possible results from client to test runner.
|
|
387
463
|
*/
|
|
388
|
-
type TestResult = InitResult | CreateResult | ConnectResult | AppendResult | ReadResult | HeadResult | DeleteResult | ShutdownResult | SetDynamicHeaderResult | SetDynamicParamResult | ClearDynamicResult | BenchmarkResult | ErrorResult;
|
|
464
|
+
type TestResult = InitResult | CreateResult | ConnectResult | AppendResult | IdempotentAppendResult | IdempotentAppendBatchResult | ReadResult | HeadResult | DeleteResult | ShutdownResult | SetDynamicHeaderResult | SetDynamicParamResult | ClearDynamicResult | BenchmarkResult | ErrorResult;
|
|
389
465
|
/**
|
|
390
466
|
* Parse a JSON line into a TestCommand.
|
|
391
467
|
*/
|
|
@@ -469,4 +545,4 @@ declare function calculateStats(durationsNs: Array<bigint>): BenchmarkStats;
|
|
|
469
545
|
* Format a BenchmarkStats object for display.
|
|
470
546
|
*/
|
|
471
547
|
declare function formatStats(stats: BenchmarkStats, unit?: string): Record<string, string>; //#endregion
|
|
472
|
-
export { AppendCommand, AppendResult, BenchmarkAppendOp, BenchmarkCommand, BenchmarkCreateOp, BenchmarkOperation, BenchmarkReadOp, BenchmarkResult, BenchmarkRoundtripOp, BenchmarkStats, BenchmarkThroughputAppendOp, BenchmarkThroughputReadOp, ClearDynamicCommand, ClearDynamicResult, ConnectCommand, ConnectResult, CreateCommand, CreateResult, DeleteCommand, DeleteResult, ErrorCode, ErrorCodes as ErrorCodes$1, ErrorResult, HeadCommand, HeadResult, InitCommand, InitResult, ReadChunk, ReadCommand, ReadResult, SetDynamicHeaderCommand, SetDynamicHeaderResult, SetDynamicParamCommand, SetDynamicParamResult, ShutdownCommand, ShutdownResult, TestCommand, TestResult, calculateStats as calculateStats$1, decodeBase64 as decodeBase64$1, encodeBase64 as encodeBase64$1, formatStats as formatStats$1, parseCommand as parseCommand$1, parseResult as parseResult$1, serializeCommand as serializeCommand$1, serializeResult as serializeResult$1 };
|
|
548
|
+
export { AppendCommand, AppendResult, BenchmarkAppendOp, BenchmarkCommand, BenchmarkCreateOp, BenchmarkOperation, BenchmarkReadOp, BenchmarkResult, BenchmarkRoundtripOp, BenchmarkStats, BenchmarkThroughputAppendOp, BenchmarkThroughputReadOp, ClearDynamicCommand, ClearDynamicResult, ConnectCommand, ConnectResult, CreateCommand, CreateResult, DeleteCommand, DeleteResult, ErrorCode, ErrorCodes as ErrorCodes$1, ErrorResult, HeadCommand, HeadResult, IdempotentAppendBatchCommand, IdempotentAppendBatchResult, IdempotentAppendCommand, IdempotentAppendResult, InitCommand, InitResult, ReadChunk, ReadCommand, ReadResult, SetDynamicHeaderCommand, SetDynamicHeaderResult, SetDynamicParamCommand, SetDynamicParamResult, ShutdownCommand, ShutdownResult, TestCommand, TestResult, calculateStats as calculateStats$1, decodeBase64 as decodeBase64$1, encodeBase64 as encodeBase64$1, formatStats as formatStats$1, parseCommand as parseCommand$1, parseResult as parseResult$1, serializeCommand as serializeCommand$1, serializeResult as serializeResult$1 };
|
package/dist/protocol.d.cts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { AppendCommand, AppendResult, BenchmarkAppendOp, BenchmarkCommand, BenchmarkCreateOp, BenchmarkOperation, BenchmarkReadOp, BenchmarkResult, BenchmarkRoundtripOp, BenchmarkStats, BenchmarkThroughputAppendOp, BenchmarkThroughputReadOp, ClearDynamicCommand, ClearDynamicResult, ConnectCommand, ConnectResult, CreateCommand, CreateResult, DeleteCommand, DeleteResult, ErrorCode, ErrorCodes, ErrorResult, HeadCommand, HeadResult, InitCommand, InitResult, ReadChunk, ReadCommand, ReadResult, SetDynamicHeaderCommand, SetDynamicHeaderResult, SetDynamicParamCommand, SetDynamicParamResult, ShutdownCommand, ShutdownResult, TestCommand, TestResult, calculateStats, decodeBase64, encodeBase64, formatStats, parseCommand, parseResult, serializeCommand, serializeResult } from "./protocol-
|
|
2
|
-
export { AppendCommand, AppendResult, BenchmarkAppendOp, BenchmarkCommand, BenchmarkCreateOp, BenchmarkOperation, BenchmarkReadOp, BenchmarkResult, BenchmarkRoundtripOp, BenchmarkStats, BenchmarkThroughputAppendOp, BenchmarkThroughputReadOp, ClearDynamicCommand, ClearDynamicResult, ConnectCommand, ConnectResult, CreateCommand, CreateResult, DeleteCommand, DeleteResult, ErrorCode, ErrorCodes, ErrorResult, HeadCommand, HeadResult, InitCommand, InitResult, ReadChunk, ReadCommand, ReadResult, SetDynamicHeaderCommand, SetDynamicHeaderResult, SetDynamicParamCommand, SetDynamicParamResult, ShutdownCommand, ShutdownResult, TestCommand, TestResult, calculateStats, decodeBase64, encodeBase64, formatStats, parseCommand, parseResult, serializeCommand, serializeResult };
|
|
1
|
+
import { AppendCommand, AppendResult, BenchmarkAppendOp, BenchmarkCommand, BenchmarkCreateOp, BenchmarkOperation, BenchmarkReadOp, BenchmarkResult, BenchmarkRoundtripOp, BenchmarkStats, BenchmarkThroughputAppendOp, BenchmarkThroughputReadOp, ClearDynamicCommand, ClearDynamicResult, ConnectCommand, ConnectResult, CreateCommand, CreateResult, DeleteCommand, DeleteResult, ErrorCode, ErrorCodes, ErrorResult, HeadCommand, HeadResult, IdempotentAppendBatchCommand, IdempotentAppendBatchResult, IdempotentAppendCommand, IdempotentAppendResult, InitCommand, InitResult, ReadChunk, ReadCommand, ReadResult, SetDynamicHeaderCommand, SetDynamicHeaderResult, SetDynamicParamCommand, SetDynamicParamResult, ShutdownCommand, ShutdownResult, TestCommand, TestResult, calculateStats, decodeBase64, encodeBase64, formatStats, parseCommand, parseResult, serializeCommand, serializeResult } from "./protocol-D37G3c4e.cjs";
|
|
2
|
+
export { AppendCommand, AppendResult, BenchmarkAppendOp, BenchmarkCommand, BenchmarkCreateOp, BenchmarkOperation, BenchmarkReadOp, BenchmarkResult, BenchmarkRoundtripOp, BenchmarkStats, BenchmarkThroughputAppendOp, BenchmarkThroughputReadOp, ClearDynamicCommand, ClearDynamicResult, ConnectCommand, ConnectResult, CreateCommand, CreateResult, DeleteCommand, DeleteResult, ErrorCode, ErrorCodes, ErrorResult, HeadCommand, HeadResult, IdempotentAppendBatchCommand, IdempotentAppendBatchResult, IdempotentAppendCommand, IdempotentAppendResult, InitCommand, InitResult, ReadChunk, ReadCommand, ReadResult, SetDynamicHeaderCommand, SetDynamicHeaderResult, SetDynamicParamCommand, SetDynamicParamResult, ShutdownCommand, ShutdownResult, TestCommand, TestResult, calculateStats, decodeBase64, encodeBase64, formatStats, parseCommand, parseResult, serializeCommand, serializeResult };
|
package/dist/protocol.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { AppendCommand, AppendResult, BenchmarkAppendOp, BenchmarkCommand, BenchmarkCreateOp, BenchmarkOperation, BenchmarkReadOp, BenchmarkResult, BenchmarkRoundtripOp, BenchmarkStats, BenchmarkThroughputAppendOp, BenchmarkThroughputReadOp, ClearDynamicCommand, ClearDynamicResult, ConnectCommand, ConnectResult, CreateCommand, CreateResult, DeleteCommand, DeleteResult, ErrorCode, ErrorCodes$1 as ErrorCodes, ErrorResult, HeadCommand, HeadResult, InitCommand, InitResult, ReadChunk, ReadCommand, ReadResult, SetDynamicHeaderCommand, SetDynamicHeaderResult, SetDynamicParamCommand, SetDynamicParamResult, ShutdownCommand, ShutdownResult, TestCommand, TestResult, calculateStats$1 as calculateStats, decodeBase64$1 as decodeBase64, encodeBase64$1 as encodeBase64, formatStats$1 as formatStats, parseCommand$1 as parseCommand, parseResult$1 as parseResult, serializeCommand$1 as serializeCommand, serializeResult$1 as serializeResult } from "./protocol-
|
|
2
|
-
export { AppendCommand, AppendResult, BenchmarkAppendOp, BenchmarkCommand, BenchmarkCreateOp, BenchmarkOperation, BenchmarkReadOp, BenchmarkResult, BenchmarkRoundtripOp, BenchmarkStats, BenchmarkThroughputAppendOp, BenchmarkThroughputReadOp, ClearDynamicCommand, ClearDynamicResult, ConnectCommand, ConnectResult, CreateCommand, CreateResult, DeleteCommand, DeleteResult, ErrorCode, ErrorCodes, ErrorResult, HeadCommand, HeadResult, InitCommand, InitResult, ReadChunk, ReadCommand, ReadResult, SetDynamicHeaderCommand, SetDynamicHeaderResult, SetDynamicParamCommand, SetDynamicParamResult, ShutdownCommand, ShutdownResult, TestCommand, TestResult, calculateStats, decodeBase64, encodeBase64, formatStats, parseCommand, parseResult, serializeCommand, serializeResult };
|
|
1
|
+
import { AppendCommand, AppendResult, BenchmarkAppendOp, BenchmarkCommand, BenchmarkCreateOp, BenchmarkOperation, BenchmarkReadOp, BenchmarkResult, BenchmarkRoundtripOp, BenchmarkStats, BenchmarkThroughputAppendOp, BenchmarkThroughputReadOp, ClearDynamicCommand, ClearDynamicResult, ConnectCommand, ConnectResult, CreateCommand, CreateResult, DeleteCommand, DeleteResult, ErrorCode, ErrorCodes$1 as ErrorCodes, ErrorResult, HeadCommand, HeadResult, IdempotentAppendBatchCommand, IdempotentAppendBatchResult, IdempotentAppendCommand, IdempotentAppendResult, InitCommand, InitResult, ReadChunk, ReadCommand, ReadResult, SetDynamicHeaderCommand, SetDynamicHeaderResult, SetDynamicParamCommand, SetDynamicParamResult, ShutdownCommand, ShutdownResult, TestCommand, TestResult, calculateStats$1 as calculateStats, decodeBase64$1 as decodeBase64, encodeBase64$1 as encodeBase64, formatStats$1 as formatStats, parseCommand$1 as parseCommand, parseResult$1 as parseResult, serializeCommand$1 as serializeCommand, serializeResult$1 as serializeResult } from "./protocol-Mcbiq3nQ.js";
|
|
2
|
+
export { AppendCommand, AppendResult, BenchmarkAppendOp, BenchmarkCommand, BenchmarkCreateOp, BenchmarkOperation, BenchmarkReadOp, BenchmarkResult, BenchmarkRoundtripOp, BenchmarkStats, BenchmarkThroughputAppendOp, BenchmarkThroughputReadOp, ClearDynamicCommand, ClearDynamicResult, ConnectCommand, ConnectResult, CreateCommand, CreateResult, DeleteCommand, DeleteResult, ErrorCode, ErrorCodes, ErrorResult, HeadCommand, HeadResult, IdempotentAppendBatchCommand, IdempotentAppendBatchResult, IdempotentAppendCommand, IdempotentAppendResult, InitCommand, InitResult, ReadChunk, ReadCommand, ReadResult, SetDynamicHeaderCommand, SetDynamicHeaderResult, SetDynamicParamCommand, SetDynamicParamResult, ShutdownCommand, ShutdownResult, TestCommand, TestResult, calculateStats, decodeBase64, encodeBase64, formatStats, parseCommand, parseResult, serializeCommand, serializeResult };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@durable-streams/client-conformance-tests",
|
|
3
3
|
"description": "Conformance test suite for Durable Streams client implementations (producer and consumer)",
|
|
4
|
-
"version": "0.1.
|
|
4
|
+
"version": "0.1.7",
|
|
5
5
|
"author": "Durable Stream contributors",
|
|
6
6
|
"bin": {
|
|
7
7
|
"client-conformance-tests": "./dist/cli.js",
|
|
@@ -14,8 +14,8 @@
|
|
|
14
14
|
"fast-check": "^4.4.0",
|
|
15
15
|
"tsx": "^4.19.2",
|
|
16
16
|
"yaml": "^2.7.1",
|
|
17
|
-
"@durable-streams/client": "0.1.
|
|
18
|
-
"@durable-streams/server": "0.1.
|
|
17
|
+
"@durable-streams/client": "0.1.4",
|
|
18
|
+
"@durable-streams/server": "0.1.5"
|
|
19
19
|
},
|
|
20
20
|
"devDependencies": {
|
|
21
21
|
"tsdown": "^0.9.0",
|
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
DurableStream,
|
|
15
15
|
DurableStreamError,
|
|
16
16
|
FetchError,
|
|
17
|
+
IdempotentProducer,
|
|
17
18
|
stream,
|
|
18
19
|
} from "@durable-streams/client"
|
|
19
20
|
import {
|
|
@@ -439,7 +440,7 @@ async function handleCommand(command: TestCommand): Promise<TestResult> {
|
|
|
439
440
|
return {
|
|
440
441
|
type: `read`,
|
|
441
442
|
success: true,
|
|
442
|
-
status:
|
|
443
|
+
status: response.status,
|
|
443
444
|
chunks,
|
|
444
445
|
offset: finalOffset,
|
|
445
446
|
upToDate,
|
|
@@ -543,6 +544,117 @@ async function handleCommand(command: TestCommand): Promise<TestResult> {
|
|
|
543
544
|
}
|
|
544
545
|
}
|
|
545
546
|
|
|
547
|
+
case `idempotent-append`: {
|
|
548
|
+
try {
|
|
549
|
+
const url = `${serverUrl}${command.path}`
|
|
550
|
+
|
|
551
|
+
// Get content-type from cache or use default
|
|
552
|
+
const contentType =
|
|
553
|
+
streamContentTypes.get(command.path) ?? `application/octet-stream`
|
|
554
|
+
|
|
555
|
+
const ds = new DurableStream({
|
|
556
|
+
url,
|
|
557
|
+
contentType,
|
|
558
|
+
})
|
|
559
|
+
|
|
560
|
+
const producer = new IdempotentProducer(ds, command.producerId, {
|
|
561
|
+
epoch: command.epoch,
|
|
562
|
+
autoClaim: command.autoClaim,
|
|
563
|
+
maxInFlight: 1,
|
|
564
|
+
lingerMs: 0, // Send immediately for testing
|
|
565
|
+
})
|
|
566
|
+
|
|
567
|
+
// For JSON streams, parse the string data into a native object
|
|
568
|
+
// (IdempotentProducer expects native objects for JSON streams)
|
|
569
|
+
const normalizedContentType = contentType
|
|
570
|
+
.split(`;`)[0]
|
|
571
|
+
?.trim()
|
|
572
|
+
.toLowerCase()
|
|
573
|
+
const isJson = normalizedContentType === `application/json`
|
|
574
|
+
const data = isJson ? JSON.parse(command.data) : command.data
|
|
575
|
+
|
|
576
|
+
try {
|
|
577
|
+
// append() is fire-and-forget (synchronous), then flush() sends the batch
|
|
578
|
+
producer.append(data)
|
|
579
|
+
await producer.flush()
|
|
580
|
+
await producer.close()
|
|
581
|
+
|
|
582
|
+
return {
|
|
583
|
+
type: `idempotent-append`,
|
|
584
|
+
success: true,
|
|
585
|
+
status: 200,
|
|
586
|
+
}
|
|
587
|
+
} catch (err) {
|
|
588
|
+
await producer.close()
|
|
589
|
+
throw err
|
|
590
|
+
}
|
|
591
|
+
} catch (err) {
|
|
592
|
+
return errorResult(`idempotent-append`, err)
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
case `idempotent-append-batch`: {
|
|
597
|
+
try {
|
|
598
|
+
const url = `${serverUrl}${command.path}`
|
|
599
|
+
|
|
600
|
+
// Get content-type from cache or use default
|
|
601
|
+
const contentType =
|
|
602
|
+
streamContentTypes.get(command.path) ?? `application/octet-stream`
|
|
603
|
+
|
|
604
|
+
const ds = new DurableStream({
|
|
605
|
+
url,
|
|
606
|
+
contentType,
|
|
607
|
+
})
|
|
608
|
+
|
|
609
|
+
// Use provided maxInFlight or default to 1 for compatibility
|
|
610
|
+
const maxInFlight = command.maxInFlight ?? 1
|
|
611
|
+
|
|
612
|
+
// When testing concurrency (maxInFlight > 1), use small batches to force
|
|
613
|
+
// multiple concurrent requests. Otherwise batch all items together.
|
|
614
|
+
const testingConcurrency = maxInFlight > 1
|
|
615
|
+
const producer = new IdempotentProducer(ds, command.producerId, {
|
|
616
|
+
epoch: command.epoch,
|
|
617
|
+
autoClaim: command.autoClaim,
|
|
618
|
+
maxInFlight,
|
|
619
|
+
lingerMs: testingConcurrency ? 0 : 1000,
|
|
620
|
+
maxBatchBytes: testingConcurrency ? 1 : 1024 * 1024,
|
|
621
|
+
})
|
|
622
|
+
|
|
623
|
+
// For JSON streams, parse string items into native objects
|
|
624
|
+
// (IdempotentProducer expects native objects for JSON streams)
|
|
625
|
+
const normalizedContentType = contentType
|
|
626
|
+
.split(`;`)[0]
|
|
627
|
+
?.trim()
|
|
628
|
+
.toLowerCase()
|
|
629
|
+
const isJson = normalizedContentType === `application/json`
|
|
630
|
+
const items = isJson
|
|
631
|
+
? command.items.map((item: string) => JSON.parse(item))
|
|
632
|
+
: command.items
|
|
633
|
+
|
|
634
|
+
try {
|
|
635
|
+
// append() is fire-and-forget (synchronous), adds to pending batch
|
|
636
|
+
for (const item of items) {
|
|
637
|
+
producer.append(item)
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
// flush() sends the batch and waits for completion
|
|
641
|
+
await producer.flush()
|
|
642
|
+
await producer.close()
|
|
643
|
+
|
|
644
|
+
return {
|
|
645
|
+
type: `idempotent-append-batch`,
|
|
646
|
+
success: true,
|
|
647
|
+
status: 200,
|
|
648
|
+
}
|
|
649
|
+
} catch (err) {
|
|
650
|
+
await producer.close()
|
|
651
|
+
throw err
|
|
652
|
+
}
|
|
653
|
+
} catch (err) {
|
|
654
|
+
return errorResult(`idempotent-append-batch`, err)
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
|
|
546
658
|
default:
|
|
547
659
|
return {
|
|
548
660
|
type: `error`,
|
|
@@ -708,6 +820,7 @@ async function handleBenchmark(command: BenchmarkCommand): Promise<TestResult> {
|
|
|
708
820
|
res.cancel()
|
|
709
821
|
resolve(chunk.data)
|
|
710
822
|
}
|
|
823
|
+
return Promise.resolve()
|
|
711
824
|
})
|
|
712
825
|
})
|
|
713
826
|
})()
|
|
@@ -748,10 +861,19 @@ async function handleBenchmark(command: BenchmarkCommand): Promise<TestResult> {
|
|
|
748
861
|
// Generate payload (using fill for speed - don't want to measure PRNG)
|
|
749
862
|
const payload = new Uint8Array(operation.size).fill(42)
|
|
750
863
|
|
|
751
|
-
//
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
864
|
+
// Use IdempotentProducer for automatic batching and pipelining
|
|
865
|
+
const producer = new IdempotentProducer(ds, `bench-producer`, {
|
|
866
|
+
lingerMs: 0, // No linger - send batches immediately when ready
|
|
867
|
+
onError: (err) => console.error(`Batch failed:`, err),
|
|
868
|
+
})
|
|
869
|
+
|
|
870
|
+
// Fire-and-forget: don't await individual appends, producer batches in background
|
|
871
|
+
for (let i = 0; i < operation.count; i++) {
|
|
872
|
+
producer.append(payload)
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
// Wait for all messages to be delivered
|
|
876
|
+
await producer.flush()
|
|
755
877
|
|
|
756
878
|
metrics.bytesTransferred = operation.count * operation.size
|
|
757
879
|
metrics.messagesProcessed = operation.count
|
package/src/protocol.ts
CHANGED
|
@@ -62,10 +62,54 @@ export interface AppendCommand {
|
|
|
62
62
|
data: string
|
|
63
63
|
/** Whether data is base64 encoded binary */
|
|
64
64
|
binary?: boolean
|
|
65
|
-
/** Optional sequence number for ordering */
|
|
65
|
+
/** Optional sequence number for ordering (Stream-Seq header) */
|
|
66
66
|
seq?: number
|
|
67
67
|
/** Custom headers to include */
|
|
68
68
|
headers?: Record<string, string>
|
|
69
|
+
/** Producer ID for idempotent producers */
|
|
70
|
+
producerId?: string
|
|
71
|
+
/** Producer epoch for idempotent producers */
|
|
72
|
+
producerEpoch?: number
|
|
73
|
+
/** Producer sequence for idempotent producers */
|
|
74
|
+
producerSeq?: number
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Append via IdempotentProducer client (tests client-side exactly-once semantics).
|
|
79
|
+
*/
|
|
80
|
+
export interface IdempotentAppendCommand {
|
|
81
|
+
type: `idempotent-append`
|
|
82
|
+
path: string
|
|
83
|
+
/** Data to append (string - will be JSON parsed for JSON streams) */
|
|
84
|
+
data: string
|
|
85
|
+
/** Producer ID */
|
|
86
|
+
producerId: string
|
|
87
|
+
/** Producer epoch */
|
|
88
|
+
epoch: number
|
|
89
|
+
/** Auto-claim epoch on 403 */
|
|
90
|
+
autoClaim: boolean
|
|
91
|
+
/** Custom headers to include */
|
|
92
|
+
headers?: Record<string, string>
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Batch append via IdempotentProducer client (tests client-side JSON batching).
|
|
97
|
+
*/
|
|
98
|
+
export interface IdempotentAppendBatchCommand {
|
|
99
|
+
type: `idempotent-append-batch`
|
|
100
|
+
path: string
|
|
101
|
+
/** Items to append - will be batched by the client */
|
|
102
|
+
items: Array<string>
|
|
103
|
+
/** Producer ID */
|
|
104
|
+
producerId: string
|
|
105
|
+
/** Producer epoch */
|
|
106
|
+
epoch: number
|
|
107
|
+
/** Auto-claim epoch on 403 */
|
|
108
|
+
autoClaim: boolean
|
|
109
|
+
/** Max concurrent batches in flight (default 1, set higher to test 409 retry) */
|
|
110
|
+
maxInFlight?: number
|
|
111
|
+
/** Custom headers to include */
|
|
112
|
+
headers?: Record<string, string>
|
|
69
113
|
}
|
|
70
114
|
|
|
71
115
|
/**
|
|
@@ -235,6 +279,8 @@ export type TestCommand =
|
|
|
235
279
|
| CreateCommand
|
|
236
280
|
| ConnectCommand
|
|
237
281
|
| AppendCommand
|
|
282
|
+
| IdempotentAppendCommand
|
|
283
|
+
| IdempotentAppendBatchCommand
|
|
238
284
|
| ReadCommand
|
|
239
285
|
| HeadCommand
|
|
240
286
|
| DeleteCommand
|
|
@@ -313,6 +359,42 @@ export interface AppendResult {
|
|
|
313
359
|
headersSent?: Record<string, string>
|
|
314
360
|
/** Params that were sent in the request (for dynamic param testing) */
|
|
315
361
|
paramsSent?: Record<string, string>
|
|
362
|
+
/** Whether this was a duplicate (204 response) - for idempotent producers */
|
|
363
|
+
duplicate?: boolean
|
|
364
|
+
/** Current producer epoch from server (on 200 or 403) */
|
|
365
|
+
producerEpoch?: number
|
|
366
|
+
/** Server's highest accepted sequence for this (stream, producerId, epoch) - returned in Producer-Seq header on 200/204 */
|
|
367
|
+
producerSeq?: number
|
|
368
|
+
/** Expected producer sequence (on 409 sequence gap) */
|
|
369
|
+
producerExpectedSeq?: number
|
|
370
|
+
/** Received producer sequence (on 409 sequence gap) */
|
|
371
|
+
producerReceivedSeq?: number
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* Successful idempotent-append result.
|
|
376
|
+
*/
|
|
377
|
+
export interface IdempotentAppendResult {
|
|
378
|
+
type: `idempotent-append`
|
|
379
|
+
success: true
|
|
380
|
+
status: number
|
|
381
|
+
/** New offset after append */
|
|
382
|
+
offset?: string
|
|
383
|
+
/** Whether this was a duplicate */
|
|
384
|
+
duplicate?: boolean
|
|
385
|
+
/** Server's highest accepted sequence for this (stream, producerId, epoch) - returned in Producer-Seq header */
|
|
386
|
+
producerSeq?: number
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/**
|
|
390
|
+
* Successful idempotent-append-batch result.
|
|
391
|
+
*/
|
|
392
|
+
export interface IdempotentAppendBatchResult {
|
|
393
|
+
type: `idempotent-append-batch`
|
|
394
|
+
success: true
|
|
395
|
+
status: number
|
|
396
|
+
/** Server's highest accepted sequence for this (stream, producerId, epoch) - returned in Producer-Seq header */
|
|
397
|
+
producerSeq?: number
|
|
316
398
|
}
|
|
317
399
|
|
|
318
400
|
/**
|
|
@@ -458,6 +540,8 @@ export type TestResult =
|
|
|
458
540
|
| CreateResult
|
|
459
541
|
| ConnectResult
|
|
460
542
|
| AppendResult
|
|
543
|
+
| IdempotentAppendResult
|
|
544
|
+
| IdempotentAppendBatchResult
|
|
461
545
|
| ReadResult
|
|
462
546
|
| HeadResult
|
|
463
547
|
| DeleteResult
|