@durable-streams/client 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -10
- package/dist/index.cjs +772 -718
- package/dist/index.d.cts +63 -25
- package/dist/index.d.ts +63 -25
- package/dist/index.js +772 -718
- package/package.json +2 -2
- package/src/idempotent-producer.ts +51 -38
- package/src/response.ts +69 -18
- package/src/sse.ts +17 -4
- package/src/stream-api.ts +17 -10
- package/src/stream.ts +77 -56
- package/src/types.ts +24 -12
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@durable-streams/client",
|
|
3
3
|
"description": "TypeScript client for the Durable Streams protocol",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.2.0",
|
|
5
5
|
"author": "Durable Stream contributors",
|
|
6
6
|
"license": "Apache-2.0",
|
|
7
7
|
"repository": {
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"devDependencies": {
|
|
48
48
|
"fast-check": "^4.4.0",
|
|
49
49
|
"tsdown": "^0.9.0",
|
|
50
|
-
"@durable-streams/server": "0.1.
|
|
50
|
+
"@durable-streams/server": "0.1.7"
|
|
51
51
|
},
|
|
52
52
|
"engines": {
|
|
53
53
|
"node": ">=18.0.0"
|
|
@@ -76,12 +76,9 @@ function normalizeContentType(contentType: string | undefined): string {
|
|
|
76
76
|
|
|
77
77
|
/**
|
|
78
78
|
* Internal type for pending batch entries.
|
|
79
|
-
* Stores original data for proper JSON batching.
|
|
80
79
|
*/
|
|
81
80
|
interface PendingEntry {
|
|
82
|
-
/**
|
|
83
|
-
data: unknown
|
|
84
|
-
/** Encoded bytes for byte-stream mode */
|
|
81
|
+
/** Encoded bytes */
|
|
85
82
|
body: Uint8Array
|
|
86
83
|
}
|
|
87
84
|
|
|
@@ -173,18 +170,37 @@ export class IdempotentProducer {
|
|
|
173
170
|
producerId: string,
|
|
174
171
|
opts?: IdempotentProducerOptions
|
|
175
172
|
) {
|
|
173
|
+
// Validate inputs
|
|
174
|
+
const epoch = opts?.epoch ?? 0
|
|
175
|
+
const maxBatchBytes = opts?.maxBatchBytes ?? 1024 * 1024 // 1MB
|
|
176
|
+
const maxInFlight = opts?.maxInFlight ?? 5
|
|
177
|
+
const lingerMs = opts?.lingerMs ?? 5
|
|
178
|
+
|
|
179
|
+
if (epoch < 0) {
|
|
180
|
+
throw new Error(`epoch must be >= 0`)
|
|
181
|
+
}
|
|
182
|
+
if (maxBatchBytes <= 0) {
|
|
183
|
+
throw new Error(`maxBatchBytes must be > 0`)
|
|
184
|
+
}
|
|
185
|
+
if (maxInFlight <= 0) {
|
|
186
|
+
throw new Error(`maxInFlight must be > 0`)
|
|
187
|
+
}
|
|
188
|
+
if (lingerMs < 0) {
|
|
189
|
+
throw new Error(`lingerMs must be >= 0`)
|
|
190
|
+
}
|
|
191
|
+
|
|
176
192
|
this.#stream = stream
|
|
177
193
|
this.#producerId = producerId
|
|
178
|
-
this.#epoch =
|
|
194
|
+
this.#epoch = epoch
|
|
179
195
|
this.#autoClaim = opts?.autoClaim ?? false
|
|
180
|
-
this.#maxBatchBytes =
|
|
181
|
-
this.#lingerMs =
|
|
196
|
+
this.#maxBatchBytes = maxBatchBytes
|
|
197
|
+
this.#lingerMs = lingerMs
|
|
182
198
|
this.#signal = opts?.signal
|
|
183
199
|
this.#onError = opts?.onError
|
|
184
200
|
this.#fetchClient =
|
|
185
201
|
opts?.fetch ?? ((...args: Parameters<typeof fetch>) => fetch(...args))
|
|
186
202
|
|
|
187
|
-
this.#maxInFlight =
|
|
203
|
+
this.#maxInFlight = maxInFlight
|
|
188
204
|
|
|
189
205
|
// When autoClaim is true, epoch is not yet known until first batch completes
|
|
190
206
|
// We block pipelining until then to avoid racing with the claim
|
|
@@ -224,12 +240,22 @@ export class IdempotentProducer {
|
|
|
224
240
|
* Errors are reported via onError callback if configured. Use flush() to
|
|
225
241
|
* wait for all pending messages to be sent.
|
|
226
242
|
*
|
|
227
|
-
* For JSON streams, pass
|
|
243
|
+
* For JSON streams, pass pre-serialized JSON strings.
|
|
228
244
|
* For byte streams, pass string or Uint8Array.
|
|
229
245
|
*
|
|
230
|
-
* @param body - Data to append (
|
|
246
|
+
* @param body - Data to append (string or Uint8Array)
|
|
247
|
+
*
|
|
248
|
+
* @example
|
|
249
|
+
* ```typescript
|
|
250
|
+
* // JSON stream
|
|
251
|
+
* producer.append(JSON.stringify({ message: "hello" }));
|
|
252
|
+
*
|
|
253
|
+
* // Byte stream
|
|
254
|
+
* producer.append("raw text data");
|
|
255
|
+
* producer.append(new Uint8Array([1, 2, 3]));
|
|
256
|
+
* ```
|
|
231
257
|
*/
|
|
232
|
-
append(body: Uint8Array | string
|
|
258
|
+
append(body: Uint8Array | string): void {
|
|
233
259
|
if (this.#closed) {
|
|
234
260
|
throw new DurableStreamError(
|
|
235
261
|
`Producer is closed`,
|
|
@@ -239,35 +265,21 @@ export class IdempotentProducer {
|
|
|
239
265
|
)
|
|
240
266
|
}
|
|
241
267
|
|
|
242
|
-
const isJson =
|
|
243
|
-
normalizeContentType(this.#stream.contentType) === `application/json`
|
|
244
|
-
|
|
245
268
|
let bytes: Uint8Array
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
if (
|
|
249
|
-
|
|
250
|
-
const json = JSON.stringify(body)
|
|
251
|
-
bytes = new TextEncoder().encode(json)
|
|
252
|
-
data = body
|
|
269
|
+
if (typeof body === `string`) {
|
|
270
|
+
bytes = new TextEncoder().encode(body)
|
|
271
|
+
} else if (body instanceof Uint8Array) {
|
|
272
|
+
bytes = body
|
|
253
273
|
} else {
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
throw new DurableStreamError(
|
|
261
|
-
`Non-JSON streams require string or Uint8Array`,
|
|
262
|
-
`BAD_REQUEST`,
|
|
263
|
-
400,
|
|
264
|
-
undefined
|
|
265
|
-
)
|
|
266
|
-
}
|
|
267
|
-
data = bytes
|
|
274
|
+
throw new DurableStreamError(
|
|
275
|
+
`append() requires string or Uint8Array. For objects, use JSON.stringify().`,
|
|
276
|
+
`BAD_REQUEST`,
|
|
277
|
+
400,
|
|
278
|
+
undefined
|
|
279
|
+
)
|
|
268
280
|
}
|
|
269
281
|
|
|
270
|
-
this.#pendingBatch.push({
|
|
282
|
+
this.#pendingBatch.push({ body: bytes })
|
|
271
283
|
this.#batchBytes += bytes.length
|
|
272
284
|
|
|
273
285
|
// Check if batch should be sent immediately
|
|
@@ -523,8 +535,9 @@ export class IdempotentProducer {
|
|
|
523
535
|
// For JSON mode: always send as array (server flattens one level)
|
|
524
536
|
// Single append: [value] → server stores value
|
|
525
537
|
// Multiple appends: [val1, val2] → server stores val1, val2
|
|
526
|
-
|
|
527
|
-
|
|
538
|
+
// Input is pre-serialized JSON strings, join them into an array
|
|
539
|
+
const jsonStrings = batch.map((e) => new TextDecoder().decode(e.body))
|
|
540
|
+
batchedBody = `[${jsonStrings.join(`,`)}]`
|
|
528
541
|
} else {
|
|
529
542
|
// For byte mode: concatenate all chunks
|
|
530
543
|
const totalSize = batch.reduce((sum, e) => sum + e.body.length, 0)
|
package/src/response.ts
CHANGED
|
@@ -96,9 +96,9 @@ export class StreamResponseImpl<
|
|
|
96
96
|
#isLoading: boolean
|
|
97
97
|
|
|
98
98
|
// --- Evolving state ---
|
|
99
|
-
offset: Offset
|
|
100
|
-
cursor?: string
|
|
101
|
-
upToDate: boolean
|
|
99
|
+
#offset: Offset
|
|
100
|
+
#cursor?: string
|
|
101
|
+
#upToDate: boolean
|
|
102
102
|
|
|
103
103
|
// --- Internal state ---
|
|
104
104
|
#isJsonMode: boolean
|
|
@@ -133,9 +133,9 @@ export class StreamResponseImpl<
|
|
|
133
133
|
this.contentType = config.contentType
|
|
134
134
|
this.live = config.live
|
|
135
135
|
this.startOffset = config.startOffset
|
|
136
|
-
this
|
|
137
|
-
this
|
|
138
|
-
this
|
|
136
|
+
this.#offset = config.initialOffset
|
|
137
|
+
this.#cursor = config.initialCursor
|
|
138
|
+
this.#upToDate = config.initialUpToDate
|
|
139
139
|
|
|
140
140
|
// Initialize response metadata from first response
|
|
141
141
|
this.#headers = config.firstResponse.headers
|
|
@@ -284,6 +284,20 @@ export class StreamResponseImpl<
|
|
|
284
284
|
return this.#isLoading
|
|
285
285
|
}
|
|
286
286
|
|
|
287
|
+
// --- Evolving state getters ---
|
|
288
|
+
|
|
289
|
+
get offset(): Offset {
|
|
290
|
+
return this.#offset
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
get cursor(): string | undefined {
|
|
294
|
+
return this.#cursor
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
get upToDate(): boolean {
|
|
298
|
+
return this.#upToDate
|
|
299
|
+
}
|
|
300
|
+
|
|
287
301
|
// =================================
|
|
288
302
|
// Internal helpers
|
|
289
303
|
// =================================
|
|
@@ -340,10 +354,10 @@ export class StreamResponseImpl<
|
|
|
340
354
|
#updateStateFromResponse(response: Response): void {
|
|
341
355
|
// Update stream-specific state
|
|
342
356
|
const offset = response.headers.get(STREAM_OFFSET_HEADER)
|
|
343
|
-
if (offset) this
|
|
357
|
+
if (offset) this.#offset = offset
|
|
344
358
|
const cursor = response.headers.get(STREAM_CURSOR_HEADER)
|
|
345
|
-
if (cursor) this
|
|
346
|
-
this
|
|
359
|
+
if (cursor) this.#cursor = cursor
|
|
360
|
+
this.#upToDate = response.headers.has(STREAM_UP_TO_DATE_HEADER)
|
|
347
361
|
|
|
348
362
|
// Update response metadata to reflect latest server response
|
|
349
363
|
this.#headers = response.headers
|
|
@@ -400,12 +414,12 @@ export class StreamResponseImpl<
|
|
|
400
414
|
* Update instance state from an SSE control event.
|
|
401
415
|
*/
|
|
402
416
|
#updateStateFromSSEControl(controlEvent: SSEControlEvent): void {
|
|
403
|
-
this
|
|
417
|
+
this.#offset = controlEvent.streamNextOffset
|
|
404
418
|
if (controlEvent.streamCursor) {
|
|
405
|
-
this
|
|
419
|
+
this.#cursor = controlEvent.streamCursor
|
|
406
420
|
}
|
|
407
421
|
if (controlEvent.upToDate !== undefined) {
|
|
408
|
-
this
|
|
422
|
+
this.#upToDate = controlEvent.upToDate
|
|
409
423
|
}
|
|
410
424
|
}
|
|
411
425
|
|
|
@@ -887,7 +901,17 @@ export class StreamResponseImpl<
|
|
|
887
901
|
// Get response text first (handles empty responses gracefully)
|
|
888
902
|
const text = await result.value.text()
|
|
889
903
|
const content = text.trim() || `[]` // Default to empty array if no content or whitespace
|
|
890
|
-
|
|
904
|
+
let parsed: T | Array<T>
|
|
905
|
+
try {
|
|
906
|
+
parsed = JSON.parse(content) as T | Array<T>
|
|
907
|
+
} catch (err) {
|
|
908
|
+
const preview =
|
|
909
|
+
content.length > 100 ? content.slice(0, 100) + `...` : content
|
|
910
|
+
throw new DurableStreamError(
|
|
911
|
+
`Failed to parse JSON response: ${err instanceof Error ? err.message : String(err)}. Data: ${preview}`,
|
|
912
|
+
`PARSE_ERROR`
|
|
913
|
+
)
|
|
914
|
+
}
|
|
891
915
|
if (Array.isArray(parsed)) {
|
|
892
916
|
items.push(...parsed)
|
|
893
917
|
} else {
|
|
@@ -1021,7 +1045,17 @@ export class StreamResponseImpl<
|
|
|
1021
1045
|
// Parse JSON and flatten arrays (handle empty responses gracefully)
|
|
1022
1046
|
const text = await response.text()
|
|
1023
1047
|
const content = text.trim() || `[]` // Default to empty array if no content or whitespace
|
|
1024
|
-
|
|
1048
|
+
let parsed: TJson | Array<TJson>
|
|
1049
|
+
try {
|
|
1050
|
+
parsed = JSON.parse(content) as TJson | Array<TJson>
|
|
1051
|
+
} catch (err) {
|
|
1052
|
+
const preview =
|
|
1053
|
+
content.length > 100 ? content.slice(0, 100) + `...` : content
|
|
1054
|
+
throw new DurableStreamError(
|
|
1055
|
+
`Failed to parse JSON response: ${err instanceof Error ? err.message : String(err)}. Data: ${preview}`,
|
|
1056
|
+
`PARSE_ERROR`
|
|
1057
|
+
)
|
|
1058
|
+
}
|
|
1025
1059
|
pendingItems = Array.isArray(parsed) ? parsed : [parsed]
|
|
1026
1060
|
|
|
1027
1061
|
// Enqueue first item
|
|
@@ -1065,7 +1099,7 @@ export class StreamResponseImpl<
|
|
|
1065
1099
|
// =====================
|
|
1066
1100
|
|
|
1067
1101
|
subscribeJson<T = TJson>(
|
|
1068
|
-
subscriber: (batch: JsonBatch<T>) => Promise<void>
|
|
1102
|
+
subscriber: (batch: JsonBatch<T>) => void | Promise<void>
|
|
1069
1103
|
): () => void {
|
|
1070
1104
|
this.#ensureNoConsumption(`subscribeJson`)
|
|
1071
1105
|
this.#ensureJsonMode()
|
|
@@ -1086,9 +1120,20 @@ export class StreamResponseImpl<
|
|
|
1086
1120
|
// Get response text first (handles empty responses gracefully)
|
|
1087
1121
|
const text = await response.text()
|
|
1088
1122
|
const content = text.trim() || `[]` // Default to empty array if no content or whitespace
|
|
1089
|
-
|
|
1123
|
+
let parsed: T | Array<T>
|
|
1124
|
+
try {
|
|
1125
|
+
parsed = JSON.parse(content) as T | Array<T>
|
|
1126
|
+
} catch (err) {
|
|
1127
|
+
const preview =
|
|
1128
|
+
content.length > 100 ? content.slice(0, 100) + `...` : content
|
|
1129
|
+
throw new DurableStreamError(
|
|
1130
|
+
`Failed to parse JSON response: ${err instanceof Error ? err.message : String(err)}. Data: ${preview}`,
|
|
1131
|
+
`PARSE_ERROR`
|
|
1132
|
+
)
|
|
1133
|
+
}
|
|
1090
1134
|
const items = Array.isArray(parsed) ? parsed : [parsed]
|
|
1091
1135
|
|
|
1136
|
+
// Await callback (handles both sync and async)
|
|
1092
1137
|
await subscriber({
|
|
1093
1138
|
items,
|
|
1094
1139
|
offset,
|
|
@@ -1121,7 +1166,9 @@ export class StreamResponseImpl<
|
|
|
1121
1166
|
}
|
|
1122
1167
|
}
|
|
1123
1168
|
|
|
1124
|
-
subscribeBytes(
|
|
1169
|
+
subscribeBytes(
|
|
1170
|
+
subscriber: (chunk: ByteChunk) => void | Promise<void>
|
|
1171
|
+
): () => void {
|
|
1125
1172
|
this.#ensureNoConsumption(`subscribeBytes`)
|
|
1126
1173
|
const abortController = new AbortController()
|
|
1127
1174
|
const reader = this.#getResponseReader()
|
|
@@ -1139,6 +1186,7 @@ export class StreamResponseImpl<
|
|
|
1139
1186
|
|
|
1140
1187
|
const buffer = await response.arrayBuffer()
|
|
1141
1188
|
|
|
1189
|
+
// Await callback (handles both sync and async)
|
|
1142
1190
|
await subscriber({
|
|
1143
1191
|
data: new Uint8Array(buffer),
|
|
1144
1192
|
offset,
|
|
@@ -1171,7 +1219,9 @@ export class StreamResponseImpl<
|
|
|
1171
1219
|
}
|
|
1172
1220
|
}
|
|
1173
1221
|
|
|
1174
|
-
subscribeText(
|
|
1222
|
+
subscribeText(
|
|
1223
|
+
subscriber: (chunk: TextChunk) => void | Promise<void>
|
|
1224
|
+
): () => void {
|
|
1175
1225
|
this.#ensureNoConsumption(`subscribeText`)
|
|
1176
1226
|
const abortController = new AbortController()
|
|
1177
1227
|
const reader = this.#getResponseReader()
|
|
@@ -1189,6 +1239,7 @@ export class StreamResponseImpl<
|
|
|
1189
1239
|
|
|
1190
1240
|
const text = await response.text()
|
|
1191
1241
|
|
|
1242
|
+
// Await callback (handles both sync and async)
|
|
1192
1243
|
await subscriber({
|
|
1193
1244
|
text,
|
|
1194
1245
|
offset,
|
package/src/sse.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* - `event: control` events contain `streamNextOffset` and optional `streamCursor` and `upToDate`
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
+
import { DurableStreamError } from "./error"
|
|
9
10
|
import type { Offset } from "./types"
|
|
10
11
|
|
|
11
12
|
/**
|
|
@@ -79,10 +80,17 @@ export async function* parseSSEStream(
|
|
|
79
80
|
streamCursor: control.streamCursor,
|
|
80
81
|
upToDate: control.upToDate,
|
|
81
82
|
}
|
|
82
|
-
} catch {
|
|
83
|
-
//
|
|
83
|
+
} catch (err) {
|
|
84
|
+
// Control events contain critical offset data - don't silently ignore
|
|
85
|
+
const preview =
|
|
86
|
+
dataStr.length > 100 ? dataStr.slice(0, 100) + `...` : dataStr
|
|
87
|
+
throw new DurableStreamError(
|
|
88
|
+
`Failed to parse SSE control event: ${err instanceof Error ? err.message : String(err)}. Data: ${preview}`,
|
|
89
|
+
`PARSE_ERROR`
|
|
90
|
+
)
|
|
84
91
|
}
|
|
85
92
|
}
|
|
93
|
+
// Unknown event types are silently skipped per protocol
|
|
86
94
|
}
|
|
87
95
|
currentEvent = { data: [] }
|
|
88
96
|
} else if (line.startsWith(`event:`)) {
|
|
@@ -122,8 +130,13 @@ export async function* parseSSEStream(
|
|
|
122
130
|
streamCursor: control.streamCursor,
|
|
123
131
|
upToDate: control.upToDate,
|
|
124
132
|
}
|
|
125
|
-
} catch {
|
|
126
|
-
|
|
133
|
+
} catch (err) {
|
|
134
|
+
const preview =
|
|
135
|
+
dataStr.length > 100 ? dataStr.slice(0, 100) + `...` : dataStr
|
|
136
|
+
throw new DurableStreamError(
|
|
137
|
+
`Failed to parse SSE control event: ${err instanceof Error ? err.message : String(err)}. Data: ${preview}`,
|
|
138
|
+
`PARSE_ERROR`
|
|
139
|
+
)
|
|
127
140
|
}
|
|
128
141
|
}
|
|
129
142
|
}
|
package/src/stream-api.ts
CHANGED
|
@@ -14,7 +14,12 @@ import {
|
|
|
14
14
|
import { DurableStreamError, FetchBackoffAbortError } from "./error"
|
|
15
15
|
import { BackoffDefaults, createFetchWithBackoff } from "./fetch"
|
|
16
16
|
import { StreamResponseImpl } from "./response"
|
|
17
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
handleErrorResponse,
|
|
19
|
+
resolveHeaders,
|
|
20
|
+
resolveParams,
|
|
21
|
+
warnIfUsingHttpInBrowser,
|
|
22
|
+
} from "./utils"
|
|
18
23
|
import type { LiveMode, Offset, StreamOptions, StreamResponse } from "./types"
|
|
19
24
|
|
|
20
25
|
/**
|
|
@@ -41,7 +46,7 @@ import type { LiveMode, Offset, StreamOptions, StreamResponse } from "./types"
|
|
|
41
46
|
* url,
|
|
42
47
|
* auth,
|
|
43
48
|
* offset: savedOffset,
|
|
44
|
-
* live:
|
|
49
|
+
* live: true,
|
|
45
50
|
* })
|
|
46
51
|
* live.subscribeJson(async (batch) => {
|
|
47
52
|
* for (const item of batch.items) {
|
|
@@ -119,6 +124,9 @@ async function streamInternal<TJson = unknown>(
|
|
|
119
124
|
// Normalize URL
|
|
120
125
|
const url = options.url instanceof URL ? options.url.toString() : options.url
|
|
121
126
|
|
|
127
|
+
// Warn if using HTTP in browser (can cause connection limit issues)
|
|
128
|
+
warnIfUsingHttpInBrowser(url, options.warnOnHttp)
|
|
129
|
+
|
|
122
130
|
// Build the first request
|
|
123
131
|
const fetchUrl = new URL(url)
|
|
124
132
|
|
|
@@ -127,7 +135,8 @@ async function streamInternal<TJson = unknown>(
|
|
|
127
135
|
fetchUrl.searchParams.set(OFFSET_QUERY_PARAM, startOffset)
|
|
128
136
|
|
|
129
137
|
// Set live query param for explicit modes
|
|
130
|
-
|
|
138
|
+
// true means auto-select (no query param, handled by consumption method)
|
|
139
|
+
const live: LiveMode = options.live ?? true
|
|
131
140
|
if (live === `long-poll` || live === `sse`) {
|
|
132
141
|
fetchUrl.searchParams.set(LIVE_QUERY_PARAM, live)
|
|
133
142
|
}
|
|
@@ -197,15 +206,13 @@ async function streamInternal<TJson = unknown>(
|
|
|
197
206
|
const nextUrl = new URL(url)
|
|
198
207
|
nextUrl.searchParams.set(OFFSET_QUERY_PARAM, offset)
|
|
199
208
|
|
|
200
|
-
// For subsequent requests
|
|
201
|
-
//
|
|
202
|
-
// to avoid a long-poll that holds for 20sec - we want an immediate response
|
|
203
|
-
// so the UI can show "connected" status quickly
|
|
209
|
+
// For subsequent requests, set live mode unless resuming from pause
|
|
210
|
+
// (resuming from pause needs immediate response for UI status)
|
|
204
211
|
if (!resumingFromPause) {
|
|
205
|
-
if (live === `
|
|
206
|
-
nextUrl.searchParams.set(LIVE_QUERY_PARAM, `long-poll`)
|
|
207
|
-
} else if (live === `sse`) {
|
|
212
|
+
if (live === `sse`) {
|
|
208
213
|
nextUrl.searchParams.set(LIVE_QUERY_PARAM, `sse`)
|
|
214
|
+
} else if (live === true || live === `long-poll`) {
|
|
215
|
+
nextUrl.searchParams.set(LIVE_QUERY_PARAM, `long-poll`)
|
|
209
216
|
}
|
|
210
217
|
}
|
|
211
218
|
|
package/src/stream.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
InvalidSignalError,
|
|
12
12
|
MissingStreamUrlError,
|
|
13
13
|
} from "./error"
|
|
14
|
+
import { IdempotentProducer } from "./idempotent-producer"
|
|
14
15
|
import {
|
|
15
16
|
SSE_COMPATIBLE_CONTENT_TYPES,
|
|
16
17
|
STREAM_EXPIRES_AT_HEADER,
|
|
@@ -37,6 +38,7 @@ import type {
|
|
|
37
38
|
CreateOptions,
|
|
38
39
|
HeadResult,
|
|
39
40
|
HeadersRecord,
|
|
41
|
+
IdempotentProducerOptions,
|
|
40
42
|
MaybePromise,
|
|
41
43
|
ParamsRecord,
|
|
42
44
|
StreamErrorHandler,
|
|
@@ -49,7 +51,7 @@ import type {
|
|
|
49
51
|
* Queued message for batching.
|
|
50
52
|
*/
|
|
51
53
|
interface QueuedMessage {
|
|
52
|
-
data:
|
|
54
|
+
data: Uint8Array | string
|
|
53
55
|
seq?: string
|
|
54
56
|
contentType?: string
|
|
55
57
|
signal?: AbortSignal
|
|
@@ -71,10 +73,7 @@ function normalizeContentType(contentType: string | undefined): string {
|
|
|
71
73
|
*/
|
|
72
74
|
function isPromiseLike(value: unknown): value is PromiseLike<unknown> {
|
|
73
75
|
return (
|
|
74
|
-
value
|
|
75
|
-
typeof value === `object` &&
|
|
76
|
-
`then` in value &&
|
|
77
|
-
typeof (value as PromiseLike<unknown>).then === `function`
|
|
76
|
+
value != null && typeof (value as PromiseLike<unknown>).then === `function`
|
|
78
77
|
)
|
|
79
78
|
}
|
|
80
79
|
|
|
@@ -121,7 +120,7 @@ export interface DurableStreamOptions extends StreamHandleOptions {
|
|
|
121
120
|
* });
|
|
122
121
|
*
|
|
123
122
|
* // Write data
|
|
124
|
-
* await stream.append({ message: "hello" });
|
|
123
|
+
* await stream.append(JSON.stringify({ message: "hello" }));
|
|
125
124
|
*
|
|
126
125
|
* // Read with the new API
|
|
127
126
|
* const res = await stream.stream<{ message: string }>();
|
|
@@ -350,23 +349,27 @@ export class DurableStream {
|
|
|
350
349
|
* a POST is in-flight will be batched together into a single request.
|
|
351
350
|
* This significantly improves throughput for high-frequency writes.
|
|
352
351
|
*
|
|
353
|
-
* - `body`
|
|
354
|
-
* -
|
|
352
|
+
* - `body` must be string or Uint8Array.
|
|
353
|
+
* - For JSON streams, pass pre-serialized JSON strings.
|
|
354
|
+
* - `body` may also be a Promise that resolves to string or Uint8Array.
|
|
355
355
|
* - Strings are encoded as UTF-8.
|
|
356
356
|
* - `seq` (if provided) is sent as stream-seq (writer coordination).
|
|
357
357
|
*
|
|
358
358
|
* @example
|
|
359
359
|
* ```typescript
|
|
360
|
-
* //
|
|
361
|
-
* await stream.append({ message: "hello" });
|
|
360
|
+
* // JSON stream - pass pre-serialized JSON
|
|
361
|
+
* await stream.append(JSON.stringify({ message: "hello" }));
|
|
362
|
+
*
|
|
363
|
+
* // Byte stream
|
|
364
|
+
* await stream.append("raw text data");
|
|
365
|
+
* await stream.append(new Uint8Array([1, 2, 3]));
|
|
362
366
|
*
|
|
363
367
|
* // Promise value - awaited before buffering
|
|
364
368
|
* await stream.append(fetchData());
|
|
365
|
-
* await stream.append(Promise.all([a, b, c]));
|
|
366
369
|
* ```
|
|
367
370
|
*/
|
|
368
371
|
async append(
|
|
369
|
-
body:
|
|
372
|
+
body: Uint8Array | string | Promise<Uint8Array | string>,
|
|
370
373
|
opts?: AppendOptions
|
|
371
374
|
): Promise<void> {
|
|
372
375
|
// Await promises before buffering
|
|
@@ -382,7 +385,7 @@ export class DurableStream {
|
|
|
382
385
|
* Direct append without batching (used when batching is disabled).
|
|
383
386
|
*/
|
|
384
387
|
async #appendDirect(
|
|
385
|
-
body:
|
|
388
|
+
body: Uint8Array | string,
|
|
386
389
|
opts?: AppendOptions
|
|
387
390
|
): Promise<void> {
|
|
388
391
|
const { requestHeaders, fetchUrl } = await this.#buildRequest()
|
|
@@ -398,9 +401,11 @@ export class DurableStream {
|
|
|
398
401
|
}
|
|
399
402
|
|
|
400
403
|
// For JSON mode, wrap body in array to match protocol (server flattens one level)
|
|
404
|
+
// Input is pre-serialized JSON string
|
|
401
405
|
const isJson = normalizeContentType(contentType) === `application/json`
|
|
402
|
-
const
|
|
403
|
-
|
|
406
|
+
const bodyStr =
|
|
407
|
+
typeof body === `string` ? body : new TextDecoder().decode(body)
|
|
408
|
+
const encodedBody: BodyInit = isJson ? `[${bodyStr}]` : bodyStr
|
|
404
409
|
|
|
405
410
|
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
406
411
|
method: `POST`,
|
|
@@ -418,7 +423,7 @@ export class DurableStream {
|
|
|
418
423
|
* Append with batching - buffers messages and sends them in batches.
|
|
419
424
|
*/
|
|
420
425
|
async #appendWithBatching(
|
|
421
|
-
body:
|
|
426
|
+
body: Uint8Array | string,
|
|
422
427
|
opts?: AppendOptions
|
|
423
428
|
): Promise<void> {
|
|
424
429
|
return new Promise<void>((resolve, reject) => {
|
|
@@ -511,29 +516,17 @@ export class DurableStream {
|
|
|
511
516
|
// For JSON mode: always send as array (server flattens one level)
|
|
512
517
|
// Single append: [value] → server stores value
|
|
513
518
|
// Multiple appends: [val1, val2] → server stores val1, val2
|
|
514
|
-
|
|
515
|
-
|
|
519
|
+
// Input is pre-serialized JSON strings, join them into an array
|
|
520
|
+
const jsonStrings = batch.map((m) =>
|
|
521
|
+
typeof m.data === `string` ? m.data : new TextDecoder().decode(m.data)
|
|
522
|
+
)
|
|
523
|
+
batchedBody = `[${jsonStrings.join(`,`)}]`
|
|
516
524
|
} else {
|
|
517
|
-
// For byte mode: concatenate all chunks
|
|
518
|
-
const
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
: (m.data as Uint8Array).length
|
|
523
|
-
return sum + size
|
|
524
|
-
}, 0)
|
|
525
|
-
|
|
526
|
-
const concatenated = new Uint8Array(totalSize)
|
|
527
|
-
let offset = 0
|
|
528
|
-
for (const msg of batch) {
|
|
529
|
-
const bytes =
|
|
530
|
-
typeof msg.data === `string`
|
|
531
|
-
? new TextEncoder().encode(msg.data)
|
|
532
|
-
: (msg.data as Uint8Array)
|
|
533
|
-
concatenated.set(bytes, offset)
|
|
534
|
-
offset += bytes.length
|
|
535
|
-
}
|
|
536
|
-
batchedBody = concatenated
|
|
525
|
+
// For byte mode: concatenate all chunks as a string
|
|
526
|
+
const strings = batch.map((m) =>
|
|
527
|
+
typeof m.data === `string` ? m.data : new TextDecoder().decode(m.data)
|
|
528
|
+
)
|
|
529
|
+
batchedBody = strings.join(``)
|
|
537
530
|
}
|
|
538
531
|
|
|
539
532
|
// Combine signals: stream-level signal + any per-message signals
|
|
@@ -634,6 +627,11 @@ export class DurableStream {
|
|
|
634
627
|
* Returns a WritableStream that can be used with `pipeTo()` or
|
|
635
628
|
* `pipeThrough()` from any ReadableStream source.
|
|
636
629
|
*
|
|
630
|
+
* Uses IdempotentProducer internally for:
|
|
631
|
+
* - Automatic batching (controlled by lingerMs, maxBatchBytes)
|
|
632
|
+
* - Exactly-once delivery semantics
|
|
633
|
+
* - Streaming writes (doesn't buffer entire content in memory)
|
|
634
|
+
*
|
|
637
635
|
* @example
|
|
638
636
|
* ```typescript
|
|
639
637
|
* // Pipe from fetch response
|
|
@@ -643,32 +641,55 @@ export class DurableStream {
|
|
|
643
641
|
* // Pipe through a transform
|
|
644
642
|
* const readable = someStream.pipeThrough(new TextEncoderStream());
|
|
645
643
|
* await readable.pipeTo(stream.writable());
|
|
644
|
+
*
|
|
645
|
+
* // With custom producer options
|
|
646
|
+
* await source.pipeTo(stream.writable({
|
|
647
|
+
* producerId: "my-producer",
|
|
648
|
+
* lingerMs: 10,
|
|
649
|
+
* maxBatchBytes: 64 * 1024,
|
|
650
|
+
* }));
|
|
646
651
|
* ```
|
|
647
652
|
*/
|
|
648
|
-
writable(
|
|
649
|
-
|
|
650
|
-
|
|
653
|
+
writable(
|
|
654
|
+
opts?: Pick<
|
|
655
|
+
IdempotentProducerOptions,
|
|
656
|
+
`lingerMs` | `maxBatchBytes` | `onError`
|
|
657
|
+
> & {
|
|
658
|
+
producerId?: string
|
|
659
|
+
signal?: AbortSignal
|
|
660
|
+
}
|
|
661
|
+
): WritableStream<Uint8Array | string> {
|
|
662
|
+
// Generate a random producer ID if not provided
|
|
663
|
+
const producerId =
|
|
664
|
+
opts?.producerId ?? `writable-${crypto.randomUUID().slice(0, 8)}`
|
|
665
|
+
|
|
666
|
+
// Track async errors to surface in close() so pipeTo() rejects on failure
|
|
667
|
+
let writeError: Error | null = null
|
|
668
|
+
|
|
669
|
+
const producer = new IdempotentProducer(this, producerId, {
|
|
670
|
+
autoClaim: true, // Ephemeral producer, auto-claim epoch
|
|
671
|
+
lingerMs: opts?.lingerMs,
|
|
672
|
+
maxBatchBytes: opts?.maxBatchBytes,
|
|
673
|
+
onError: (error) => {
|
|
674
|
+
if (!writeError) writeError = error // Capture first error
|
|
675
|
+
opts?.onError?.(error) // Still call user's handler
|
|
676
|
+
},
|
|
677
|
+
signal: opts?.signal ?? this.#options.signal,
|
|
678
|
+
})
|
|
651
679
|
|
|
652
680
|
return new WritableStream<Uint8Array | string>({
|
|
653
681
|
write(chunk) {
|
|
654
|
-
|
|
682
|
+
producer.append(chunk)
|
|
655
683
|
},
|
|
656
684
|
async close() {
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
start(controller) {
|
|
661
|
-
for (const chunk of chunks) {
|
|
662
|
-
controller.enqueue(chunk)
|
|
663
|
-
}
|
|
664
|
-
controller.close()
|
|
665
|
-
},
|
|
666
|
-
})
|
|
667
|
-
await stream.appendStream(readable, opts)
|
|
668
|
-
}
|
|
685
|
+
await producer.flush()
|
|
686
|
+
await producer.close()
|
|
687
|
+
if (writeError) throw writeError // Causes pipeTo() to reject
|
|
669
688
|
},
|
|
670
|
-
abort(
|
|
671
|
-
|
|
689
|
+
abort(_reason) {
|
|
690
|
+
producer.close().catch((err) => {
|
|
691
|
+
opts?.onError?.(err) // Report instead of swallowing
|
|
692
|
+
})
|
|
672
693
|
},
|
|
673
694
|
})
|
|
674
695
|
}
|