@durable-streams/client 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +168 -4
- package/dist/index.cjs +431 -3
- package/dist/index.d.cts +218 -1
- package/dist/index.d.ts +218 -1
- package/dist/index.js +424 -4
- package/package.json +2 -2
- package/src/constants.ts +31 -0
- package/src/idempotent-producer.ts +642 -0
- package/src/index.ts +21 -0
- package/src/stream.ts +5 -0
- package/src/types.ts +93 -0
package/README.md
CHANGED
|
@@ -10,14 +10,17 @@ npm install @durable-streams/client
|
|
|
10
10
|
|
|
11
11
|
## Overview
|
|
12
12
|
|
|
13
|
-
The Durable Streams client provides
|
|
13
|
+
The Durable Streams client provides three main APIs:
|
|
14
14
|
|
|
15
15
|
1. **`stream()` function** - A fetch-like read-only API for consuming streams
|
|
16
16
|
2. **`DurableStream` class** - A handle for read/write operations on a stream
|
|
17
|
+
3. **`IdempotentProducer` class** - High-throughput producer with exactly-once write semantics (recommended for writes)
|
|
17
18
|
|
|
18
19
|
## Key Features
|
|
19
20
|
|
|
20
|
-
- **
|
|
21
|
+
- **Exactly-Once Writes**: `IdempotentProducer` provides Kafka-style exactly-once semantics with automatic deduplication
|
|
22
|
+
- **Automatic Batching**: Multiple writes are automatically batched together for high throughput
|
|
23
|
+
- **Pipelining**: Up to 5 concurrent batches in flight by default for maximum throughput
|
|
21
24
|
- **Streaming Reads**: `stream()` and `DurableStream.stream()` provide rich consumption options (promises, ReadableStreams, subscribers)
|
|
22
25
|
- **Resumable**: Offset-based reads let you resume from any point
|
|
23
26
|
- **Real-time**: Long-poll and SSE modes for live tailing with catch-up from any offset
|
|
@@ -81,9 +84,56 @@ const unsubscribe3 = res.subscribeText(async (chunk) => {
|
|
|
81
84
|
})
|
|
82
85
|
```
|
|
83
86
|
|
|
87
|
+
### High-Throughput Writes: Using `IdempotentProducer` (Recommended)
|
|
88
|
+
|
|
89
|
+
For reliable, high-throughput writes with exactly-once semantics, use `IdempotentProducer`:
|
|
90
|
+
|
|
91
|
+
```typescript
|
|
92
|
+
import { DurableStream, IdempotentProducer } from "@durable-streams/client"
|
|
93
|
+
|
|
94
|
+
const stream = await DurableStream.create({
|
|
95
|
+
url: "https://streams.example.com/events",
|
|
96
|
+
contentType: "application/json",
|
|
97
|
+
})
|
|
98
|
+
|
|
99
|
+
const producer = new IdempotentProducer(stream, "event-processor-1", {
|
|
100
|
+
autoClaim: true,
|
|
101
|
+
onError: (err) => console.error("Batch failed:", err), // Errors reported here
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
// Fire-and-forget - don't await, errors go to onError callback
|
|
105
|
+
for (const event of events) {
|
|
106
|
+
producer.append(event) // Objects serialized automatically for JSON streams
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// IMPORTANT: Always flush before shutdown to ensure delivery
|
|
110
|
+
await producer.flush()
|
|
111
|
+
await producer.close()
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
For high-throughput scenarios, `append()` is fire-and-forget (returns immediately):
|
|
115
|
+
|
|
116
|
+
```typescript
|
|
117
|
+
// Fire-and-forget - errors reported via onError callback
|
|
118
|
+
for (const event of events) {
|
|
119
|
+
producer.append(event) // Returns void, adds to batch
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Always flush before shutdown to ensure delivery
|
|
123
|
+
await producer.flush()
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
**Why use IdempotentProducer?**
|
|
127
|
+
|
|
128
|
+
- **Exactly-once delivery**: Server deduplicates using `(producerId, epoch, seq)` tuple
|
|
129
|
+
- **Automatic batching**: Multiple writes batched into single HTTP requests
|
|
130
|
+
- **Pipelining**: Multiple batches in flight concurrently
|
|
131
|
+
- **Zombie fencing**: Stale producers are rejected, preventing split-brain scenarios
|
|
132
|
+
- **Network resilience**: Safe to retry on network errors (server deduplicates)
|
|
133
|
+
|
|
84
134
|
### Read/Write: Using `DurableStream`
|
|
85
135
|
|
|
86
|
-
For write operations or when you need a persistent handle:
|
|
136
|
+
For simple write operations or when you need a persistent handle:
|
|
87
137
|
|
|
88
138
|
```typescript
|
|
89
139
|
import { DurableStream } from "@durable-streams/client"
|
|
@@ -98,7 +148,7 @@ const handle = await DurableStream.create({
|
|
|
98
148
|
ttlSeconds: 3600,
|
|
99
149
|
})
|
|
100
150
|
|
|
101
|
-
// Append data
|
|
151
|
+
// Append data (simple API without exactly-once guarantees)
|
|
102
152
|
await handle.append(JSON.stringify({ type: "message", text: "Hello" }), {
|
|
103
153
|
seq: "writer-1-000001",
|
|
104
154
|
})
|
|
@@ -787,6 +837,117 @@ res.subscribeJson(async (batch) => {
|
|
|
787
837
|
|
|
788
838
|
---
|
|
789
839
|
|
|
840
|
+
## IdempotentProducer
|
|
841
|
+
|
|
842
|
+
The `IdempotentProducer` class provides Kafka-style exactly-once write semantics with automatic batching and pipelining.
|
|
843
|
+
|
|
844
|
+
### Constructor
|
|
845
|
+
|
|
846
|
+
```typescript
|
|
847
|
+
new IdempotentProducer(stream: DurableStream, producerId: string, opts?: IdempotentProducerOptions)
|
|
848
|
+
```
|
|
849
|
+
|
|
850
|
+
**Parameters:**
|
|
851
|
+
|
|
852
|
+
- `stream` - The DurableStream to write to
|
|
853
|
+
- `producerId` - Stable identifier for this producer (e.g., "order-service-1")
|
|
854
|
+
- `opts` - Optional configuration
|
|
855
|
+
|
|
856
|
+
**Options:**
|
|
857
|
+
|
|
858
|
+
```typescript
|
|
859
|
+
interface IdempotentProducerOptions {
|
|
860
|
+
epoch?: number // Starting epoch (default: 0)
|
|
861
|
+
autoClaim?: boolean // On 403, retry with epoch+1 (default: false)
|
|
862
|
+
maxBatchBytes?: number // Max bytes before sending batch (default: 1MB)
|
|
863
|
+
lingerMs?: number // Max time to wait for more messages (default: 5ms)
|
|
864
|
+
maxInFlight?: number // Concurrent batches in flight (default: 5)
|
|
865
|
+
signal?: AbortSignal // Cancellation signal
|
|
866
|
+
fetch?: typeof fetch // Custom fetch implementation
|
|
867
|
+
onError?: (error: Error) => void // Error callback for batch failures
|
|
868
|
+
}
|
|
869
|
+
```
|
|
870
|
+
|
|
871
|
+
### Methods
|
|
872
|
+
|
|
873
|
+
#### `append(body): void`
|
|
874
|
+
|
|
875
|
+
Append data to the stream (fire-and-forget). For JSON streams, you can pass objects directly.
|
|
876
|
+
Returns immediately after adding to the internal batch. Errors are reported via `onError` callback.
|
|
877
|
+
|
|
878
|
+
```typescript
|
|
879
|
+
// For JSON streams - pass objects directly
|
|
880
|
+
producer.append({ event: "click", x: 100 })
|
|
881
|
+
|
|
882
|
+
// Or strings/bytes
|
|
883
|
+
producer.append("message data")
|
|
884
|
+
producer.append(new Uint8Array([1, 2, 3]))
|
|
885
|
+
|
|
886
|
+
// All appends are fire-and-forget - use flush() to wait for delivery
|
|
887
|
+
await producer.flush()
|
|
888
|
+
```
|
|
889
|
+
|
|
890
|
+
#### `flush(): Promise<void>`
|
|
891
|
+
|
|
892
|
+
Send any pending batch immediately and wait for all in-flight batches to complete.
|
|
893
|
+
|
|
894
|
+
```typescript
|
|
895
|
+
// Always call before shutdown
|
|
896
|
+
await producer.flush()
|
|
897
|
+
```
|
|
898
|
+
|
|
899
|
+
#### `close(): Promise<void>`
|
|
900
|
+
|
|
901
|
+
Flush pending messages and close the producer. Further `append()` calls will throw.
|
|
902
|
+
|
|
903
|
+
```typescript
|
|
904
|
+
await producer.close()
|
|
905
|
+
```
|
|
906
|
+
|
|
907
|
+
#### `restart(): Promise<void>`
|
|
908
|
+
|
|
909
|
+
Increment epoch and reset sequence. Call this when restarting the producer to establish a new session.
|
|
910
|
+
|
|
911
|
+
```typescript
|
|
912
|
+
await producer.restart()
|
|
913
|
+
```
|
|
914
|
+
|
|
915
|
+
### Properties
|
|
916
|
+
|
|
917
|
+
- `epoch: number` - Current epoch for this producer
|
|
918
|
+
- `nextSeq: number` - Next sequence number to be assigned
|
|
919
|
+
- `pendingCount: number` - Messages in the current pending batch
|
|
920
|
+
- `inFlightCount: number` - Batches currently in flight
|
|
921
|
+
|
|
922
|
+
### Error Handling
|
|
923
|
+
|
|
924
|
+
Errors are delivered via the `onError` callback since `append()` is fire-and-forget:
|
|
925
|
+
|
|
926
|
+
```typescript
|
|
927
|
+
import {
|
|
928
|
+
IdempotentProducer,
|
|
929
|
+
StaleEpochError,
|
|
930
|
+
SequenceGapError,
|
|
931
|
+
} from "@durable-streams/client"
|
|
932
|
+
|
|
933
|
+
const producer = new IdempotentProducer(stream, "my-producer", {
|
|
934
|
+
onError: (error) => {
|
|
935
|
+
if (error instanceof StaleEpochError) {
|
|
936
|
+
// Another producer has a higher epoch - this producer is "fenced"
|
|
937
|
+
console.log(`Fenced by epoch ${error.currentEpoch}`)
|
|
938
|
+
} else if (error instanceof SequenceGapError) {
|
|
939
|
+
// Sequence gap detected (should not happen with proper usage)
|
|
940
|
+
console.log(`Expected seq ${error.expectedSeq}, got ${error.receivedSeq}`)
|
|
941
|
+
}
|
|
942
|
+
},
|
|
943
|
+
})
|
|
944
|
+
|
|
945
|
+
producer.append("data") // Fire-and-forget, errors go to onError
|
|
946
|
+
await producer.flush() // Wait for all batches to complete
|
|
947
|
+
```
|
|
948
|
+
|
|
949
|
+
---
|
|
950
|
+
|
|
790
951
|
## Types
|
|
791
952
|
|
|
792
953
|
Key types exported from the package:
|
|
@@ -797,6 +958,9 @@ Key types exported from the package:
|
|
|
797
958
|
- `JsonBatch<T>` - `{ items: T[], offset: Offset, upToDate: boolean, cursor?: string }`
|
|
798
959
|
- `TextChunk` - `{ text: string, offset: Offset, upToDate: boolean, cursor?: string }`
|
|
799
960
|
- `HeadResult` - Metadata from HEAD requests
|
|
961
|
+
- `IdempotentProducer` - Exactly-once producer class
|
|
962
|
+
- `StaleEpochError` - Thrown when producer epoch is stale (zombie fencing)
|
|
963
|
+
- `SequenceGapError` - Thrown when sequence numbers are out of order
|
|
800
964
|
- `DurableStreamError` - Protocol-level errors with codes
|
|
801
965
|
- `FetchError` - Transport/network errors
|
|
802
966
|
|
package/dist/index.cjs
CHANGED
|
@@ -59,6 +59,28 @@ const STREAM_TTL_HEADER = `Stream-TTL`;
|
|
|
59
59
|
*/
|
|
60
60
|
const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`;
|
|
61
61
|
/**
|
|
62
|
+
* Request header for producer ID (client-supplied stable identifier).
|
|
63
|
+
*/
|
|
64
|
+
const PRODUCER_ID_HEADER = `Producer-Id`;
|
|
65
|
+
/**
|
|
66
|
+
* Request/response header for producer epoch.
|
|
67
|
+
* Client-declared, server-validated monotonically increasing.
|
|
68
|
+
*/
|
|
69
|
+
const PRODUCER_EPOCH_HEADER = `Producer-Epoch`;
|
|
70
|
+
/**
|
|
71
|
+
* Request header for producer sequence number.
|
|
72
|
+
* Monotonically increasing per epoch, per-batch (not per-message).
|
|
73
|
+
*/
|
|
74
|
+
const PRODUCER_SEQ_HEADER = `Producer-Seq`;
|
|
75
|
+
/**
|
|
76
|
+
* Response header indicating expected sequence number on 409 Conflict.
|
|
77
|
+
*/
|
|
78
|
+
const PRODUCER_EXPECTED_SEQ_HEADER = `Producer-Expected-Seq`;
|
|
79
|
+
/**
|
|
80
|
+
* Response header indicating received sequence number on 409 Conflict.
|
|
81
|
+
*/
|
|
82
|
+
const PRODUCER_RECEIVED_SEQ_HEADER = `Producer-Received-Seq`;
|
|
83
|
+
/**
|
|
62
84
|
* Query parameter for starting offset.
|
|
63
85
|
*/
|
|
64
86
|
const OFFSET_QUERY_PARAM = `offset`;
|
|
@@ -1406,7 +1428,7 @@ async function streamInternal(options) {
|
|
|
1406
1428
|
* Normalize content-type by extracting the media type (before any semicolon).
|
|
1407
1429
|
* Handles cases like "application/json; charset=utf-8".
|
|
1408
1430
|
*/
|
|
1409
|
-
function normalizeContentType(contentType) {
|
|
1431
|
+
function normalizeContentType$1(contentType) {
|
|
1410
1432
|
if (!contentType) return ``;
|
|
1411
1433
|
return contentType.split(`;`)[0].trim().toLowerCase();
|
|
1412
1434
|
}
|
|
@@ -1472,6 +1494,7 @@ var DurableStream = class DurableStream {
|
|
|
1472
1494
|
url: urlStr
|
|
1473
1495
|
};
|
|
1474
1496
|
this.#onError = opts.onError;
|
|
1497
|
+
if (opts.contentType) this.contentType = opts.contentType;
|
|
1475
1498
|
this.#batchingEnabled = opts.batching !== false;
|
|
1476
1499
|
if (this.#batchingEnabled) this.#queue = fastq.default.promise(this.#batchWorker.bind(this), 1);
|
|
1477
1500
|
const baseFetchClient = opts.fetch ?? ((...args) => fetch(...args));
|
|
@@ -1620,7 +1643,7 @@ var DurableStream = class DurableStream {
|
|
|
1620
1643
|
const contentType = opts?.contentType ?? this.#options.contentType ?? this.contentType;
|
|
1621
1644
|
if (contentType) requestHeaders[`content-type`] = contentType;
|
|
1622
1645
|
if (opts?.seq) requestHeaders[STREAM_SEQ_HEADER] = opts.seq;
|
|
1623
|
-
const isJson = normalizeContentType(contentType) === `application/json`;
|
|
1646
|
+
const isJson = normalizeContentType$1(contentType) === `application/json`;
|
|
1624
1647
|
const bodyToEncode = isJson ? [body] : body;
|
|
1625
1648
|
const encodedBody = encodeBody(bodyToEncode);
|
|
1626
1649
|
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
@@ -1686,7 +1709,7 @@ var DurableStream = class DurableStream {
|
|
|
1686
1709
|
break;
|
|
1687
1710
|
}
|
|
1688
1711
|
if (highestSeq) requestHeaders[STREAM_SEQ_HEADER] = highestSeq;
|
|
1689
|
-
const isJson = normalizeContentType(contentType) === `application/json`;
|
|
1712
|
+
const isJson = normalizeContentType$1(contentType) === `application/json`;
|
|
1690
1713
|
let batchedBody;
|
|
1691
1714
|
if (isJson) {
|
|
1692
1715
|
const values = batch.map((m) => m.data);
|
|
@@ -1925,6 +1948,403 @@ function validateOptions(options) {
|
|
|
1925
1948
|
warnIfUsingHttpInBrowser(options.url, options.warnOnHttp);
|
|
1926
1949
|
}
|
|
1927
1950
|
|
|
1951
|
+
//#endregion
|
|
1952
|
+
//#region src/idempotent-producer.ts
|
|
1953
|
+
/**
|
|
1954
|
+
* Error thrown when a producer's epoch is stale (zombie fencing).
|
|
1955
|
+
*/
|
|
1956
|
+
var StaleEpochError = class extends Error {
|
|
1957
|
+
/**
|
|
1958
|
+
* The current epoch on the server.
|
|
1959
|
+
*/
|
|
1960
|
+
currentEpoch;
|
|
1961
|
+
constructor(currentEpoch) {
|
|
1962
|
+
super(`Producer epoch is stale. Current server epoch: ${currentEpoch}. Call restart() or create a new producer with a higher epoch.`);
|
|
1963
|
+
this.name = `StaleEpochError`;
|
|
1964
|
+
this.currentEpoch = currentEpoch;
|
|
1965
|
+
}
|
|
1966
|
+
};
|
|
1967
|
+
/**
|
|
1968
|
+
* Error thrown when an unrecoverable sequence gap is detected.
|
|
1969
|
+
*
|
|
1970
|
+
* With maxInFlight > 1, HTTP requests can arrive out of order at the server,
|
|
1971
|
+
* causing temporary 409 responses. The client automatically handles these
|
|
1972
|
+
* by waiting for earlier sequences to complete, then retrying.
|
|
1973
|
+
*
|
|
1974
|
+
* This error is only thrown when the gap cannot be resolved (e.g., the
|
|
1975
|
+
* expected sequence is >= our sequence, indicating a true protocol violation).
|
|
1976
|
+
*/
|
|
1977
|
+
var SequenceGapError = class extends Error {
|
|
1978
|
+
expectedSeq;
|
|
1979
|
+
receivedSeq;
|
|
1980
|
+
constructor(expectedSeq, receivedSeq) {
|
|
1981
|
+
super(`Producer sequence gap: expected ${expectedSeq}, received ${receivedSeq}`);
|
|
1982
|
+
this.name = `SequenceGapError`;
|
|
1983
|
+
this.expectedSeq = expectedSeq;
|
|
1984
|
+
this.receivedSeq = receivedSeq;
|
|
1985
|
+
}
|
|
1986
|
+
};
|
|
1987
|
+
/**
|
|
1988
|
+
* Normalize content-type by extracting the media type (before any semicolon).
|
|
1989
|
+
*/
|
|
1990
|
+
function normalizeContentType(contentType) {
|
|
1991
|
+
if (!contentType) return ``;
|
|
1992
|
+
return contentType.split(`;`)[0].trim().toLowerCase();
|
|
1993
|
+
}
|
|
1994
|
+
/**
|
|
1995
|
+
* An idempotent producer for exactly-once writes to a durable stream.
|
|
1996
|
+
*
|
|
1997
|
+
* Features:
|
|
1998
|
+
* - Fire-and-forget: append() returns immediately, batches in background
|
|
1999
|
+
* - Exactly-once: server deduplicates using (producerId, epoch, seq)
|
|
2000
|
+
* - Batching: multiple appends batched into single HTTP request
|
|
2001
|
+
* - Pipelining: up to maxInFlight concurrent batches
|
|
2002
|
+
* - Zombie fencing: stale producers rejected via epoch validation
|
|
2003
|
+
*
|
|
2004
|
+
* @example
|
|
2005
|
+
* ```typescript
|
|
2006
|
+
* const stream = new DurableStream({ url: "https://..." });
|
|
2007
|
+
* const producer = new IdempotentProducer(stream, "order-service-1", {
|
|
2008
|
+
* epoch: 0,
|
|
2009
|
+
* autoClaim: true,
|
|
2010
|
+
* });
|
|
2011
|
+
*
|
|
2012
|
+
* // Fire-and-forget writes (synchronous, returns immediately)
|
|
2013
|
+
* producer.append("message 1");
|
|
2014
|
+
* producer.append("message 2");
|
|
2015
|
+
*
|
|
2016
|
+
* // Ensure all messages are delivered before shutdown
|
|
2017
|
+
* await producer.flush();
|
|
2018
|
+
* await producer.close();
|
|
2019
|
+
* ```
|
|
2020
|
+
*/
|
|
2021
|
+
var IdempotentProducer = class {
|
|
2022
|
+
#stream;
|
|
2023
|
+
#producerId;
|
|
2024
|
+
#epoch;
|
|
2025
|
+
#nextSeq = 0;
|
|
2026
|
+
#autoClaim;
|
|
2027
|
+
#maxBatchBytes;
|
|
2028
|
+
#lingerMs;
|
|
2029
|
+
#fetchClient;
|
|
2030
|
+
#signal;
|
|
2031
|
+
#onError;
|
|
2032
|
+
#pendingBatch = [];
|
|
2033
|
+
#batchBytes = 0;
|
|
2034
|
+
#lingerTimeout = null;
|
|
2035
|
+
#queue;
|
|
2036
|
+
#maxInFlight;
|
|
2037
|
+
#closed = false;
|
|
2038
|
+
#epochClaimed;
|
|
2039
|
+
#seqState = new Map();
|
|
2040
|
+
/**
|
|
2041
|
+
* Create an idempotent producer for a stream.
|
|
2042
|
+
*
|
|
2043
|
+
* @param stream - The DurableStream to write to
|
|
2044
|
+
* @param producerId - Stable identifier for this producer (e.g., "order-service-1")
|
|
2045
|
+
* @param opts - Producer options
|
|
2046
|
+
*/
|
|
2047
|
+
constructor(stream$1, producerId, opts) {
|
|
2048
|
+
this.#stream = stream$1;
|
|
2049
|
+
this.#producerId = producerId;
|
|
2050
|
+
this.#epoch = opts?.epoch ?? 0;
|
|
2051
|
+
this.#autoClaim = opts?.autoClaim ?? false;
|
|
2052
|
+
this.#maxBatchBytes = opts?.maxBatchBytes ?? 1024 * 1024;
|
|
2053
|
+
this.#lingerMs = opts?.lingerMs ?? 5;
|
|
2054
|
+
this.#signal = opts?.signal;
|
|
2055
|
+
this.#onError = opts?.onError;
|
|
2056
|
+
this.#fetchClient = opts?.fetch ?? ((...args) => fetch(...args));
|
|
2057
|
+
this.#maxInFlight = opts?.maxInFlight ?? 5;
|
|
2058
|
+
this.#epochClaimed = !this.#autoClaim;
|
|
2059
|
+
this.#queue = fastq.default.promise(this.#batchWorker.bind(this), this.#maxInFlight);
|
|
2060
|
+
if (this.#signal) this.#signal.addEventListener(`abort`, () => {
|
|
2061
|
+
this.#rejectPendingBatch(new DurableStreamError(`Producer aborted`, `ALREADY_CLOSED`, void 0, void 0));
|
|
2062
|
+
}, { once: true });
|
|
2063
|
+
}
|
|
2064
|
+
/**
|
|
2065
|
+
* Append data to the stream.
|
|
2066
|
+
*
|
|
2067
|
+
* This is fire-and-forget: returns immediately after adding to the batch.
|
|
2068
|
+
* The message is batched and sent when:
|
|
2069
|
+
* - maxBatchBytes is reached
|
|
2070
|
+
* - lingerMs elapses
|
|
2071
|
+
* - flush() is called
|
|
2072
|
+
*
|
|
2073
|
+
* Errors are reported via onError callback if configured. Use flush() to
|
|
2074
|
+
* wait for all pending messages to be sent.
|
|
2075
|
+
*
|
|
2076
|
+
* For JSON streams, pass native objects (which will be serialized internally).
|
|
2077
|
+
* For byte streams, pass string or Uint8Array.
|
|
2078
|
+
*
|
|
2079
|
+
* @param body - Data to append (object for JSON streams, string or Uint8Array for byte streams)
|
|
2080
|
+
*/
|
|
2081
|
+
append(body) {
|
|
2082
|
+
if (this.#closed) throw new DurableStreamError(`Producer is closed`, `ALREADY_CLOSED`, void 0, void 0);
|
|
2083
|
+
const isJson = normalizeContentType(this.#stream.contentType) === `application/json`;
|
|
2084
|
+
let bytes;
|
|
2085
|
+
let data;
|
|
2086
|
+
if (isJson) {
|
|
2087
|
+
const json = JSON.stringify(body);
|
|
2088
|
+
bytes = new TextEncoder().encode(json);
|
|
2089
|
+
data = body;
|
|
2090
|
+
} else {
|
|
2091
|
+
if (typeof body === `string`) bytes = new TextEncoder().encode(body);
|
|
2092
|
+
else if (body instanceof Uint8Array) bytes = body;
|
|
2093
|
+
else throw new DurableStreamError(`Non-JSON streams require string or Uint8Array`, `BAD_REQUEST`, 400, void 0);
|
|
2094
|
+
data = bytes;
|
|
2095
|
+
}
|
|
2096
|
+
this.#pendingBatch.push({
|
|
2097
|
+
data,
|
|
2098
|
+
body: bytes
|
|
2099
|
+
});
|
|
2100
|
+
this.#batchBytes += bytes.length;
|
|
2101
|
+
if (this.#batchBytes >= this.#maxBatchBytes) this.#enqueuePendingBatch();
|
|
2102
|
+
else if (!this.#lingerTimeout) this.#lingerTimeout = setTimeout(() => {
|
|
2103
|
+
this.#lingerTimeout = null;
|
|
2104
|
+
if (this.#pendingBatch.length > 0) this.#enqueuePendingBatch();
|
|
2105
|
+
}, this.#lingerMs);
|
|
2106
|
+
}
|
|
2107
|
+
/**
|
|
2108
|
+
* Send any pending batch immediately and wait for all in-flight batches.
|
|
2109
|
+
*
|
|
2110
|
+
* Call this before shutdown to ensure all messages are delivered.
|
|
2111
|
+
*/
|
|
2112
|
+
async flush() {
|
|
2113
|
+
if (this.#lingerTimeout) {
|
|
2114
|
+
clearTimeout(this.#lingerTimeout);
|
|
2115
|
+
this.#lingerTimeout = null;
|
|
2116
|
+
}
|
|
2117
|
+
if (this.#pendingBatch.length > 0) this.#enqueuePendingBatch();
|
|
2118
|
+
await this.#queue.drained();
|
|
2119
|
+
}
|
|
2120
|
+
/**
|
|
2121
|
+
* Flush pending messages and close the producer.
|
|
2122
|
+
*
|
|
2123
|
+
* After calling close(), further append() calls will throw.
|
|
2124
|
+
*/
|
|
2125
|
+
async close() {
|
|
2126
|
+
if (this.#closed) return;
|
|
2127
|
+
this.#closed = true;
|
|
2128
|
+
try {
|
|
2129
|
+
await this.flush();
|
|
2130
|
+
} catch {}
|
|
2131
|
+
}
|
|
2132
|
+
/**
|
|
2133
|
+
* Increment epoch and reset sequence.
|
|
2134
|
+
*
|
|
2135
|
+
* Call this when restarting the producer to establish a new session.
|
|
2136
|
+
* Flushes any pending messages first.
|
|
2137
|
+
*/
|
|
2138
|
+
async restart() {
|
|
2139
|
+
await this.flush();
|
|
2140
|
+
this.#epoch++;
|
|
2141
|
+
this.#nextSeq = 0;
|
|
2142
|
+
}
|
|
2143
|
+
/**
|
|
2144
|
+
* Current epoch for this producer.
|
|
2145
|
+
*/
|
|
2146
|
+
get epoch() {
|
|
2147
|
+
return this.#epoch;
|
|
2148
|
+
}
|
|
2149
|
+
/**
|
|
2150
|
+
* Next sequence number to be assigned.
|
|
2151
|
+
*/
|
|
2152
|
+
get nextSeq() {
|
|
2153
|
+
return this.#nextSeq;
|
|
2154
|
+
}
|
|
2155
|
+
/**
|
|
2156
|
+
* Number of messages in the current pending batch.
|
|
2157
|
+
*/
|
|
2158
|
+
get pendingCount() {
|
|
2159
|
+
return this.#pendingBatch.length;
|
|
2160
|
+
}
|
|
2161
|
+
/**
|
|
2162
|
+
* Number of batches currently in flight.
|
|
2163
|
+
*/
|
|
2164
|
+
get inFlightCount() {
|
|
2165
|
+
return this.#queue.length();
|
|
2166
|
+
}
|
|
2167
|
+
/**
|
|
2168
|
+
* Enqueue the current pending batch for processing.
|
|
2169
|
+
*/
|
|
2170
|
+
#enqueuePendingBatch() {
|
|
2171
|
+
if (this.#pendingBatch.length === 0) return;
|
|
2172
|
+
const batch = this.#pendingBatch;
|
|
2173
|
+
const seq = this.#nextSeq;
|
|
2174
|
+
this.#pendingBatch = [];
|
|
2175
|
+
this.#batchBytes = 0;
|
|
2176
|
+
this.#nextSeq++;
|
|
2177
|
+
if (this.#autoClaim && !this.#epochClaimed && this.#queue.length() > 0) this.#queue.drained().then(() => {
|
|
2178
|
+
this.#queue.push({
|
|
2179
|
+
batch,
|
|
2180
|
+
seq
|
|
2181
|
+
}).catch(() => {});
|
|
2182
|
+
});
|
|
2183
|
+
else this.#queue.push({
|
|
2184
|
+
batch,
|
|
2185
|
+
seq
|
|
2186
|
+
}).catch(() => {});
|
|
2187
|
+
}
|
|
2188
|
+
/**
|
|
2189
|
+
* Batch worker - processes batches via fastq.
|
|
2190
|
+
*/
|
|
2191
|
+
async #batchWorker(task) {
|
|
2192
|
+
const { batch, seq } = task;
|
|
2193
|
+
const epoch = this.#epoch;
|
|
2194
|
+
try {
|
|
2195
|
+
await this.#doSendBatch(batch, seq, epoch);
|
|
2196
|
+
if (!this.#epochClaimed) this.#epochClaimed = true;
|
|
2197
|
+
this.#signalSeqComplete(epoch, seq, void 0);
|
|
2198
|
+
} catch (error) {
|
|
2199
|
+
this.#signalSeqComplete(epoch, seq, error);
|
|
2200
|
+
if (this.#onError) this.#onError(error);
|
|
2201
|
+
throw error;
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
/**
|
|
2205
|
+
* Signal that a sequence has completed (success or failure).
|
|
2206
|
+
*/
|
|
2207
|
+
#signalSeqComplete(epoch, seq, error) {
|
|
2208
|
+
let epochMap = this.#seqState.get(epoch);
|
|
2209
|
+
if (!epochMap) {
|
|
2210
|
+
epochMap = new Map();
|
|
2211
|
+
this.#seqState.set(epoch, epochMap);
|
|
2212
|
+
}
|
|
2213
|
+
const state = epochMap.get(seq);
|
|
2214
|
+
if (state) {
|
|
2215
|
+
state.resolved = true;
|
|
2216
|
+
state.error = error;
|
|
2217
|
+
for (const waiter of state.waiters) waiter(error);
|
|
2218
|
+
state.waiters = [];
|
|
2219
|
+
} else epochMap.set(seq, {
|
|
2220
|
+
resolved: true,
|
|
2221
|
+
error,
|
|
2222
|
+
waiters: []
|
|
2223
|
+
});
|
|
2224
|
+
const cleanupThreshold = seq - this.#maxInFlight * 3;
|
|
2225
|
+
if (cleanupThreshold > 0) {
|
|
2226
|
+
for (const oldSeq of epochMap.keys()) if (oldSeq < cleanupThreshold) epochMap.delete(oldSeq);
|
|
2227
|
+
}
|
|
2228
|
+
}
|
|
2229
|
+
/**
|
|
2230
|
+
* Wait for a specific sequence to complete.
|
|
2231
|
+
* Returns immediately if already completed.
|
|
2232
|
+
* Throws if the sequence failed.
|
|
2233
|
+
*/
|
|
2234
|
+
#waitForSeq(epoch, seq) {
|
|
2235
|
+
let epochMap = this.#seqState.get(epoch);
|
|
2236
|
+
if (!epochMap) {
|
|
2237
|
+
epochMap = new Map();
|
|
2238
|
+
this.#seqState.set(epoch, epochMap);
|
|
2239
|
+
}
|
|
2240
|
+
const state = epochMap.get(seq);
|
|
2241
|
+
if (state?.resolved) {
|
|
2242
|
+
if (state.error) return Promise.reject(state.error);
|
|
2243
|
+
return Promise.resolve();
|
|
2244
|
+
}
|
|
2245
|
+
return new Promise((resolve, reject) => {
|
|
2246
|
+
const waiter = (err) => {
|
|
2247
|
+
if (err) reject(err);
|
|
2248
|
+
else resolve();
|
|
2249
|
+
};
|
|
2250
|
+
if (state) state.waiters.push(waiter);
|
|
2251
|
+
else epochMap.set(seq, {
|
|
2252
|
+
resolved: false,
|
|
2253
|
+
waiters: [waiter]
|
|
2254
|
+
});
|
|
2255
|
+
});
|
|
2256
|
+
}
|
|
2257
|
+
/**
|
|
2258
|
+
* Actually send the batch to the server.
|
|
2259
|
+
* Handles auto-claim retry on 403 (stale epoch) if autoClaim is enabled.
|
|
2260
|
+
* Does NOT implement general retry/backoff for network errors or 5xx responses.
|
|
2261
|
+
*/
|
|
2262
|
+
async #doSendBatch(batch, seq, epoch) {
|
|
2263
|
+
const contentType = this.#stream.contentType ?? `application/octet-stream`;
|
|
2264
|
+
const isJson = normalizeContentType(contentType) === `application/json`;
|
|
2265
|
+
let batchedBody;
|
|
2266
|
+
if (isJson) {
|
|
2267
|
+
const values = batch.map((e) => e.data);
|
|
2268
|
+
batchedBody = JSON.stringify(values);
|
|
2269
|
+
} else {
|
|
2270
|
+
const totalSize = batch.reduce((sum, e) => sum + e.body.length, 0);
|
|
2271
|
+
const concatenated = new Uint8Array(totalSize);
|
|
2272
|
+
let offset = 0;
|
|
2273
|
+
for (const entry of batch) {
|
|
2274
|
+
concatenated.set(entry.body, offset);
|
|
2275
|
+
offset += entry.body.length;
|
|
2276
|
+
}
|
|
2277
|
+
batchedBody = concatenated;
|
|
2278
|
+
}
|
|
2279
|
+
const url = this.#stream.url;
|
|
2280
|
+
const headers = {
|
|
2281
|
+
"content-type": contentType,
|
|
2282
|
+
[PRODUCER_ID_HEADER]: this.#producerId,
|
|
2283
|
+
[PRODUCER_EPOCH_HEADER]: epoch.toString(),
|
|
2284
|
+
[PRODUCER_SEQ_HEADER]: seq.toString()
|
|
2285
|
+
};
|
|
2286
|
+
const response = await this.#fetchClient(url, {
|
|
2287
|
+
method: `POST`,
|
|
2288
|
+
headers,
|
|
2289
|
+
body: batchedBody,
|
|
2290
|
+
signal: this.#signal
|
|
2291
|
+
});
|
|
2292
|
+
if (response.status === 204) return {
|
|
2293
|
+
offset: ``,
|
|
2294
|
+
duplicate: true
|
|
2295
|
+
};
|
|
2296
|
+
if (response.status === 200) {
|
|
2297
|
+
const resultOffset = response.headers.get(STREAM_OFFSET_HEADER) ?? ``;
|
|
2298
|
+
return {
|
|
2299
|
+
offset: resultOffset,
|
|
2300
|
+
duplicate: false
|
|
2301
|
+
};
|
|
2302
|
+
}
|
|
2303
|
+
if (response.status === 403) {
|
|
2304
|
+
const currentEpochStr = response.headers.get(PRODUCER_EPOCH_HEADER);
|
|
2305
|
+
const currentEpoch = currentEpochStr ? parseInt(currentEpochStr, 10) : epoch;
|
|
2306
|
+
if (this.#autoClaim) {
|
|
2307
|
+
const newEpoch = currentEpoch + 1;
|
|
2308
|
+
this.#epoch = newEpoch;
|
|
2309
|
+
this.#nextSeq = 1;
|
|
2310
|
+
return this.#doSendBatch(batch, 0, newEpoch);
|
|
2311
|
+
}
|
|
2312
|
+
throw new StaleEpochError(currentEpoch);
|
|
2313
|
+
}
|
|
2314
|
+
if (response.status === 409) {
|
|
2315
|
+
const expectedSeqStr = response.headers.get(PRODUCER_EXPECTED_SEQ_HEADER);
|
|
2316
|
+
const expectedSeq = expectedSeqStr ? parseInt(expectedSeqStr, 10) : 0;
|
|
2317
|
+
if (expectedSeq < seq) {
|
|
2318
|
+
const waitPromises = [];
|
|
2319
|
+
for (let s = expectedSeq; s < seq; s++) waitPromises.push(this.#waitForSeq(epoch, s));
|
|
2320
|
+
await Promise.all(waitPromises);
|
|
2321
|
+
return this.#doSendBatch(batch, seq, epoch);
|
|
2322
|
+
}
|
|
2323
|
+
const receivedSeqStr = response.headers.get(PRODUCER_RECEIVED_SEQ_HEADER);
|
|
2324
|
+
const receivedSeq = receivedSeqStr ? parseInt(receivedSeqStr, 10) : seq;
|
|
2325
|
+
throw new SequenceGapError(expectedSeq, receivedSeq);
|
|
2326
|
+
}
|
|
2327
|
+
if (response.status === 400) {
|
|
2328
|
+
const error$1 = await DurableStreamError.fromResponse(response, url);
|
|
2329
|
+
throw error$1;
|
|
2330
|
+
}
|
|
2331
|
+
const error = await FetchError.fromResponse(response, url);
|
|
2332
|
+
throw error;
|
|
2333
|
+
}
|
|
2334
|
+
/**
|
|
2335
|
+
* Clear pending batch and report error.
|
|
2336
|
+
*/
|
|
2337
|
+
#rejectPendingBatch(error) {
|
|
2338
|
+
if (this.#onError && this.#pendingBatch.length > 0) this.#onError(error);
|
|
2339
|
+
this.#pendingBatch = [];
|
|
2340
|
+
this.#batchBytes = 0;
|
|
2341
|
+
if (this.#lingerTimeout) {
|
|
2342
|
+
clearTimeout(this.#lingerTimeout);
|
|
2343
|
+
this.#lingerTimeout = null;
|
|
2344
|
+
}
|
|
2345
|
+
}
|
|
2346
|
+
};
|
|
2347
|
+
|
|
1928
2348
|
//#endregion
|
|
1929
2349
|
exports.BackoffDefaults = BackoffDefaults
|
|
1930
2350
|
exports.CURSOR_QUERY_PARAM = CURSOR_QUERY_PARAM
|
|
@@ -1933,10 +2353,16 @@ exports.DurableStream = DurableStream
|
|
|
1933
2353
|
exports.DurableStreamError = DurableStreamError
|
|
1934
2354
|
exports.FetchBackoffAbortError = FetchBackoffAbortError
|
|
1935
2355
|
exports.FetchError = FetchError
|
|
2356
|
+
exports.IdempotentProducer = IdempotentProducer
|
|
1936
2357
|
exports.InvalidSignalError = InvalidSignalError
|
|
1937
2358
|
exports.LIVE_QUERY_PARAM = LIVE_QUERY_PARAM
|
|
1938
2359
|
exports.MissingStreamUrlError = MissingStreamUrlError
|
|
1939
2360
|
exports.OFFSET_QUERY_PARAM = OFFSET_QUERY_PARAM
|
|
2361
|
+
exports.PRODUCER_EPOCH_HEADER = PRODUCER_EPOCH_HEADER
|
|
2362
|
+
exports.PRODUCER_EXPECTED_SEQ_HEADER = PRODUCER_EXPECTED_SEQ_HEADER
|
|
2363
|
+
exports.PRODUCER_ID_HEADER = PRODUCER_ID_HEADER
|
|
2364
|
+
exports.PRODUCER_RECEIVED_SEQ_HEADER = PRODUCER_RECEIVED_SEQ_HEADER
|
|
2365
|
+
exports.PRODUCER_SEQ_HEADER = PRODUCER_SEQ_HEADER
|
|
1940
2366
|
exports.SSE_COMPATIBLE_CONTENT_TYPES = SSE_COMPATIBLE_CONTENT_TYPES
|
|
1941
2367
|
exports.STREAM_CURSOR_HEADER = STREAM_CURSOR_HEADER
|
|
1942
2368
|
exports.STREAM_EXPIRES_AT_HEADER = STREAM_EXPIRES_AT_HEADER
|
|
@@ -1944,6 +2370,8 @@ exports.STREAM_OFFSET_HEADER = STREAM_OFFSET_HEADER
|
|
|
1944
2370
|
exports.STREAM_SEQ_HEADER = STREAM_SEQ_HEADER
|
|
1945
2371
|
exports.STREAM_TTL_HEADER = STREAM_TTL_HEADER
|
|
1946
2372
|
exports.STREAM_UP_TO_DATE_HEADER = STREAM_UP_TO_DATE_HEADER
|
|
2373
|
+
exports.SequenceGapError = SequenceGapError
|
|
2374
|
+
exports.StaleEpochError = StaleEpochError
|
|
1947
2375
|
exports._resetHttpWarningForTesting = _resetHttpWarningForTesting
|
|
1948
2376
|
exports.asAsyncIterableReadableStream = asAsyncIterableReadableStream
|
|
1949
2377
|
exports.createFetchWithBackoff = createFetchWithBackoff
|