@durable-streams/server 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +494 -0
- package/dist/index.js +1592 -0
- package/dist/index.js.map +1 -0
- package/package.json +41 -0
- package/src/cursor.ts +156 -0
- package/src/file-manager.ts +89 -0
- package/src/file-store.ts +863 -0
- package/src/index.ts +27 -0
- package/src/path-encoding.ts +61 -0
- package/src/registry-hook.ts +118 -0
- package/src/server.ts +946 -0
- package/src/store.ts +433 -0
- package/src/types.ts +183 -0
package/src/server.ts
ADDED
|
@@ -0,0 +1,946 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HTTP server for durable streams testing.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { createServer } from "node:http"
|
|
6
|
+
import { deflateSync, gzipSync } from "node:zlib"
|
|
7
|
+
import { StreamStore } from "./store"
|
|
8
|
+
import { FileBackedStreamStore } from "./file-store"
|
|
9
|
+
import { generateResponseCursor } from "./cursor"
|
|
10
|
+
import type { CursorOptions } from "./cursor"
|
|
11
|
+
import type { IncomingMessage, Server, ServerResponse } from "node:http"
|
|
12
|
+
import type { StreamLifecycleEvent, TestServerOptions } from "./types"
|
|
13
|
+
|
|
14
|
+
// Protocol headers (aligned with PROTOCOL.md)
|
|
15
|
+
const STREAM_OFFSET_HEADER = `Stream-Next-Offset`
|
|
16
|
+
const STREAM_CURSOR_HEADER = `Stream-Cursor`
|
|
17
|
+
const STREAM_UP_TO_DATE_HEADER = `Stream-Up-To-Date`
|
|
18
|
+
const STREAM_SEQ_HEADER = `Stream-Seq`
|
|
19
|
+
const STREAM_TTL_HEADER = `Stream-TTL`
|
|
20
|
+
const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`
|
|
21
|
+
|
|
22
|
+
// SSE control event fields (Protocol Section 5.7)
|
|
23
|
+
const SSE_OFFSET_FIELD = `streamNextOffset`
|
|
24
|
+
const SSE_CURSOR_FIELD = `streamCursor`
|
|
25
|
+
const SSE_UP_TO_DATE_FIELD = `upToDate`
|
|
26
|
+
|
|
27
|
+
// Query params
|
|
28
|
+
const OFFSET_QUERY_PARAM = `offset`
|
|
29
|
+
const LIVE_QUERY_PARAM = `live`
|
|
30
|
+
const CURSOR_QUERY_PARAM = `cursor`
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Encode data for SSE format.
|
|
34
|
+
* Per SSE spec, each line in the payload needs its own "data:" prefix.
|
|
35
|
+
* Newlines in the payload become separate data: lines.
|
|
36
|
+
*/
|
|
37
|
+
function encodeSSEData(payload: string): string {
|
|
38
|
+
const lines = payload.split(`\n`)
|
|
39
|
+
return lines.map((line) => `data: ${line}`).join(`\n`) + `\n\n`
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Minimum response size to consider for compression.
|
|
44
|
+
* Responses smaller than this won't benefit from compression.
|
|
45
|
+
*/
|
|
46
|
+
const COMPRESSION_THRESHOLD = 1024
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Determine the best compression encoding from Accept-Encoding header.
|
|
50
|
+
* Returns 'gzip', 'deflate', or null if no compression should be used.
|
|
51
|
+
*/
|
|
52
|
+
function getCompressionEncoding(
|
|
53
|
+
acceptEncoding: string | undefined
|
|
54
|
+
): `gzip` | `deflate` | null {
|
|
55
|
+
if (!acceptEncoding) return null
|
|
56
|
+
|
|
57
|
+
// Parse Accept-Encoding header (e.g., "gzip, deflate, br" or "gzip;q=1.0, deflate;q=0.5")
|
|
58
|
+
const encodings = acceptEncoding
|
|
59
|
+
.toLowerCase()
|
|
60
|
+
.split(`,`)
|
|
61
|
+
.map((e) => e.trim())
|
|
62
|
+
|
|
63
|
+
// Prefer gzip over deflate (better compression, wider support)
|
|
64
|
+
for (const encoding of encodings) {
|
|
65
|
+
const name = encoding.split(`;`)[0]?.trim()
|
|
66
|
+
if (name === `gzip`) return `gzip`
|
|
67
|
+
}
|
|
68
|
+
for (const encoding of encodings) {
|
|
69
|
+
const name = encoding.split(`;`)[0]?.trim()
|
|
70
|
+
if (name === `deflate`) return `deflate`
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return null
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Compress data using the specified encoding.
|
|
78
|
+
*/
|
|
79
|
+
function compressData(
|
|
80
|
+
data: Uint8Array,
|
|
81
|
+
encoding: `gzip` | `deflate`
|
|
82
|
+
): Uint8Array {
|
|
83
|
+
if (encoding === `gzip`) {
|
|
84
|
+
return gzipSync(data)
|
|
85
|
+
} else {
|
|
86
|
+
return deflateSync(data)
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* HTTP server for testing durable streams.
|
|
92
|
+
* Supports both in-memory and file-backed storage modes.
|
|
93
|
+
*/
|
|
94
|
+
/**
|
|
95
|
+
* Configuration for injected errors (for testing retry/resilience).
|
|
96
|
+
*/
|
|
97
|
+
interface InjectedError {
|
|
98
|
+
/** HTTP status code to return */
|
|
99
|
+
status: number
|
|
100
|
+
/** Number of times to return this error (decremented on each use) */
|
|
101
|
+
count: number
|
|
102
|
+
/** Optional Retry-After header value (seconds) */
|
|
103
|
+
retryAfter?: number
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export class DurableStreamTestServer {
|
|
107
|
+
readonly store: StreamStore | FileBackedStreamStore
|
|
108
|
+
private server: Server | null = null
|
|
109
|
+
private options: Required<
|
|
110
|
+
Omit<
|
|
111
|
+
TestServerOptions,
|
|
112
|
+
| `dataDir`
|
|
113
|
+
| `onStreamCreated`
|
|
114
|
+
| `onStreamDeleted`
|
|
115
|
+
| `compression`
|
|
116
|
+
| `cursorIntervalSeconds`
|
|
117
|
+
| `cursorEpoch`
|
|
118
|
+
>
|
|
119
|
+
> & {
|
|
120
|
+
dataDir?: string
|
|
121
|
+
onStreamCreated?: (event: StreamLifecycleEvent) => void | Promise<void>
|
|
122
|
+
onStreamDeleted?: (event: StreamLifecycleEvent) => void | Promise<void>
|
|
123
|
+
compression: boolean
|
|
124
|
+
cursorOptions: CursorOptions
|
|
125
|
+
}
|
|
126
|
+
private _url: string | null = null
|
|
127
|
+
private activeSSEResponses = new Set<ServerResponse>()
|
|
128
|
+
private isShuttingDown = false
|
|
129
|
+
/** Injected errors for testing retry/resilience */
|
|
130
|
+
private injectedErrors = new Map<string, InjectedError>()
|
|
131
|
+
|
|
132
|
+
constructor(options: TestServerOptions = {}) {
|
|
133
|
+
// Choose store based on dataDir option
|
|
134
|
+
if (options.dataDir) {
|
|
135
|
+
this.store = new FileBackedStreamStore({
|
|
136
|
+
dataDir: options.dataDir,
|
|
137
|
+
})
|
|
138
|
+
} else {
|
|
139
|
+
this.store = new StreamStore()
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
this.options = {
|
|
143
|
+
port: options.port ?? 4437,
|
|
144
|
+
host: options.host ?? `127.0.0.1`,
|
|
145
|
+
longPollTimeout: options.longPollTimeout ?? 30_000,
|
|
146
|
+
dataDir: options.dataDir,
|
|
147
|
+
onStreamCreated: options.onStreamCreated,
|
|
148
|
+
onStreamDeleted: options.onStreamDeleted,
|
|
149
|
+
compression: options.compression ?? true,
|
|
150
|
+
cursorOptions: {
|
|
151
|
+
intervalSeconds: options.cursorIntervalSeconds,
|
|
152
|
+
epoch: options.cursorEpoch,
|
|
153
|
+
},
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Start the server.
|
|
159
|
+
*/
|
|
160
|
+
async start(): Promise<string> {
|
|
161
|
+
if (this.server) {
|
|
162
|
+
throw new Error(`Server already started`)
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
return new Promise((resolve, reject) => {
|
|
166
|
+
this.server = createServer((req, res) => {
|
|
167
|
+
this.handleRequest(req, res).catch((err) => {
|
|
168
|
+
console.error(`Request error:`, err)
|
|
169
|
+
if (!res.headersSent) {
|
|
170
|
+
res.writeHead(500, { "content-type": `text/plain` })
|
|
171
|
+
res.end(`Internal server error`)
|
|
172
|
+
}
|
|
173
|
+
})
|
|
174
|
+
})
|
|
175
|
+
|
|
176
|
+
this.server.on(`error`, reject)
|
|
177
|
+
|
|
178
|
+
this.server.listen(this.options.port, this.options.host, () => {
|
|
179
|
+
const addr = this.server!.address()
|
|
180
|
+
if (typeof addr === `string`) {
|
|
181
|
+
this._url = addr
|
|
182
|
+
} else if (addr) {
|
|
183
|
+
this._url = `http://${this.options.host}:${addr.port}`
|
|
184
|
+
}
|
|
185
|
+
resolve(this._url!)
|
|
186
|
+
})
|
|
187
|
+
})
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Stop the server.
|
|
192
|
+
*/
|
|
193
|
+
async stop(): Promise<void> {
|
|
194
|
+
if (!this.server) {
|
|
195
|
+
return
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// Mark as shutting down to stop SSE handlers
|
|
199
|
+
this.isShuttingDown = true
|
|
200
|
+
|
|
201
|
+
// Cancel all pending long-polls and SSE waits to unblock connection handlers
|
|
202
|
+
if (`cancelAllWaits` in this.store) {
|
|
203
|
+
;(this.store as { cancelAllWaits: () => void }).cancelAllWaits()
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Force-close all active SSE connections
|
|
207
|
+
for (const res of this.activeSSEResponses) {
|
|
208
|
+
res.end()
|
|
209
|
+
}
|
|
210
|
+
this.activeSSEResponses.clear()
|
|
211
|
+
|
|
212
|
+
return new Promise((resolve, reject) => {
|
|
213
|
+
this.server!.close(async (err) => {
|
|
214
|
+
if (err) {
|
|
215
|
+
reject(err)
|
|
216
|
+
return
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
try {
|
|
220
|
+
// Close file-backed store if used
|
|
221
|
+
if (this.store instanceof FileBackedStreamStore) {
|
|
222
|
+
await this.store.close()
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
this.server = null
|
|
226
|
+
this._url = null
|
|
227
|
+
this.isShuttingDown = false
|
|
228
|
+
resolve()
|
|
229
|
+
} catch (closeErr) {
|
|
230
|
+
reject(closeErr)
|
|
231
|
+
}
|
|
232
|
+
})
|
|
233
|
+
})
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Get the server URL.
|
|
238
|
+
*/
|
|
239
|
+
get url(): string {
|
|
240
|
+
if (!this._url) {
|
|
241
|
+
throw new Error(`Server not started`)
|
|
242
|
+
}
|
|
243
|
+
return this._url
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Clear all streams.
|
|
248
|
+
*/
|
|
249
|
+
clear(): void {
|
|
250
|
+
this.store.clear()
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Inject an error to be returned on the next N requests to a path.
|
|
255
|
+
* Used for testing retry/resilience behavior.
|
|
256
|
+
*/
|
|
257
|
+
injectError(
|
|
258
|
+
path: string,
|
|
259
|
+
status: number,
|
|
260
|
+
count: number = 1,
|
|
261
|
+
retryAfter?: number
|
|
262
|
+
): void {
|
|
263
|
+
this.injectedErrors.set(path, { status, count, retryAfter })
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Clear all injected errors.
|
|
268
|
+
*/
|
|
269
|
+
clearInjectedErrors(): void {
|
|
270
|
+
this.injectedErrors.clear()
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Check if there's an injected error for this path and consume it.
|
|
275
|
+
* Returns the error config if one should be returned, null otherwise.
|
|
276
|
+
*/
|
|
277
|
+
private consumeInjectedError(path: string): InjectedError | null {
|
|
278
|
+
const error = this.injectedErrors.get(path)
|
|
279
|
+
if (!error) return null
|
|
280
|
+
|
|
281
|
+
error.count--
|
|
282
|
+
if (error.count <= 0) {
|
|
283
|
+
this.injectedErrors.delete(path)
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
return error
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// ============================================================================
|
|
290
|
+
// Request handling
|
|
291
|
+
// ============================================================================
|
|
292
|
+
|
|
293
|
+
private async handleRequest(
|
|
294
|
+
req: IncomingMessage,
|
|
295
|
+
res: ServerResponse
|
|
296
|
+
): Promise<void> {
|
|
297
|
+
const url = new URL(req.url ?? `/`, `http://${req.headers.host}`)
|
|
298
|
+
const path = url.pathname
|
|
299
|
+
const method = req.method?.toUpperCase()
|
|
300
|
+
|
|
301
|
+
// CORS headers for browser testing
|
|
302
|
+
res.setHeader(`access-control-allow-origin`, `*`)
|
|
303
|
+
res.setHeader(
|
|
304
|
+
`access-control-allow-methods`,
|
|
305
|
+
`GET, POST, PUT, DELETE, HEAD, OPTIONS`
|
|
306
|
+
)
|
|
307
|
+
res.setHeader(
|
|
308
|
+
`access-control-allow-headers`,
|
|
309
|
+
`content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At`
|
|
310
|
+
)
|
|
311
|
+
res.setHeader(
|
|
312
|
+
`access-control-expose-headers`,
|
|
313
|
+
`Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Date, etag, content-type, content-encoding, vary`
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
// Handle CORS preflight
|
|
317
|
+
if (method === `OPTIONS`) {
|
|
318
|
+
res.writeHead(204)
|
|
319
|
+
res.end()
|
|
320
|
+
return
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// Handle test control endpoints (for error injection)
|
|
324
|
+
if (path === `/_test/inject-error`) {
|
|
325
|
+
await this.handleTestInjectError(method, req, res)
|
|
326
|
+
return
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// Check for injected errors (for testing retry/resilience)
|
|
330
|
+
const injectedError = this.consumeInjectedError(path)
|
|
331
|
+
if (injectedError) {
|
|
332
|
+
const headers: Record<string, string> = {
|
|
333
|
+
"content-type": `text/plain`,
|
|
334
|
+
}
|
|
335
|
+
if (injectedError.retryAfter !== undefined) {
|
|
336
|
+
headers[`retry-after`] = injectedError.retryAfter.toString()
|
|
337
|
+
}
|
|
338
|
+
res.writeHead(injectedError.status, headers)
|
|
339
|
+
res.end(`Injected error for testing`)
|
|
340
|
+
return
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
try {
|
|
344
|
+
switch (method) {
|
|
345
|
+
case `PUT`:
|
|
346
|
+
await this.handleCreate(path, req, res)
|
|
347
|
+
break
|
|
348
|
+
case `HEAD`:
|
|
349
|
+
this.handleHead(path, res)
|
|
350
|
+
break
|
|
351
|
+
case `GET`:
|
|
352
|
+
await this.handleRead(path, url, req, res)
|
|
353
|
+
break
|
|
354
|
+
case `POST`:
|
|
355
|
+
await this.handleAppend(path, req, res)
|
|
356
|
+
break
|
|
357
|
+
case `DELETE`:
|
|
358
|
+
await this.handleDelete(path, res)
|
|
359
|
+
break
|
|
360
|
+
default:
|
|
361
|
+
res.writeHead(405, { "content-type": `text/plain` })
|
|
362
|
+
res.end(`Method not allowed`)
|
|
363
|
+
}
|
|
364
|
+
} catch (err) {
|
|
365
|
+
if (err instanceof Error) {
|
|
366
|
+
if (err.message.includes(`not found`)) {
|
|
367
|
+
res.writeHead(404, { "content-type": `text/plain` })
|
|
368
|
+
res.end(`Stream not found`)
|
|
369
|
+
} else if (
|
|
370
|
+
err.message.includes(`already exists with different configuration`)
|
|
371
|
+
) {
|
|
372
|
+
res.writeHead(409, { "content-type": `text/plain` })
|
|
373
|
+
res.end(`Stream already exists with different configuration`)
|
|
374
|
+
} else if (err.message.includes(`Sequence conflict`)) {
|
|
375
|
+
res.writeHead(409, { "content-type": `text/plain` })
|
|
376
|
+
res.end(`Sequence conflict`)
|
|
377
|
+
} else if (err.message.includes(`Content-type mismatch`)) {
|
|
378
|
+
res.writeHead(409, { "content-type": `text/plain` })
|
|
379
|
+
res.end(`Content-type mismatch`)
|
|
380
|
+
} else if (err.message.includes(`Invalid JSON`)) {
|
|
381
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
382
|
+
res.end(`Invalid JSON`)
|
|
383
|
+
} else if (err.message.includes(`Empty arrays are not allowed`)) {
|
|
384
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
385
|
+
res.end(`Empty arrays are not allowed`)
|
|
386
|
+
} else {
|
|
387
|
+
throw err
|
|
388
|
+
}
|
|
389
|
+
} else {
|
|
390
|
+
throw err
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* Handle PUT - create stream
|
|
397
|
+
*/
|
|
398
|
+
private async handleCreate(
|
|
399
|
+
path: string,
|
|
400
|
+
req: IncomingMessage,
|
|
401
|
+
res: ServerResponse
|
|
402
|
+
): Promise<void> {
|
|
403
|
+
let contentType = req.headers[`content-type`]
|
|
404
|
+
|
|
405
|
+
// Sanitize content-type: if empty or invalid, use default
|
|
406
|
+
if (
|
|
407
|
+
!contentType ||
|
|
408
|
+
contentType.trim() === `` ||
|
|
409
|
+
!/^[\w-]+\/[\w-]+/.test(contentType)
|
|
410
|
+
) {
|
|
411
|
+
contentType = `application/octet-stream`
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
const ttlHeader = req.headers[STREAM_TTL_HEADER.toLowerCase()] as
|
|
415
|
+
| string
|
|
416
|
+
| undefined
|
|
417
|
+
const expiresAtHeader = req.headers[
|
|
418
|
+
STREAM_EXPIRES_AT_HEADER.toLowerCase()
|
|
419
|
+
] as string | undefined
|
|
420
|
+
|
|
421
|
+
// Validate TTL and Expires-At headers
|
|
422
|
+
if (ttlHeader && expiresAtHeader) {
|
|
423
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
424
|
+
res.end(`Cannot specify both Stream-TTL and Stream-Expires-At`)
|
|
425
|
+
return
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
let ttlSeconds: number | undefined
|
|
429
|
+
if (ttlHeader) {
|
|
430
|
+
// Strict TTL validation: must be a positive integer without leading zeros,
|
|
431
|
+
// plus signs, decimals, whitespace, or non-decimal notation
|
|
432
|
+
const ttlPattern = /^(0|[1-9]\d*)$/
|
|
433
|
+
if (!ttlPattern.test(ttlHeader)) {
|
|
434
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
435
|
+
res.end(`Invalid Stream-TTL value`)
|
|
436
|
+
return
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
ttlSeconds = parseInt(ttlHeader, 10)
|
|
440
|
+
if (isNaN(ttlSeconds) || ttlSeconds < 0) {
|
|
441
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
442
|
+
res.end(`Invalid Stream-TTL value`)
|
|
443
|
+
return
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
// Validate Expires-At timestamp format (ISO 8601)
|
|
448
|
+
if (expiresAtHeader) {
|
|
449
|
+
const timestamp = new Date(expiresAtHeader)
|
|
450
|
+
if (isNaN(timestamp.getTime())) {
|
|
451
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
452
|
+
res.end(`Invalid Stream-Expires-At timestamp`)
|
|
453
|
+
return
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// Read body if present
|
|
458
|
+
const body = await this.readBody(req)
|
|
459
|
+
|
|
460
|
+
const isNew = !this.store.has(path)
|
|
461
|
+
|
|
462
|
+
// Support both sync (StreamStore) and async (FileBackedStreamStore) create
|
|
463
|
+
await Promise.resolve(
|
|
464
|
+
this.store.create(path, {
|
|
465
|
+
contentType,
|
|
466
|
+
ttlSeconds,
|
|
467
|
+
expiresAt: expiresAtHeader,
|
|
468
|
+
initialData: body.length > 0 ? body : undefined,
|
|
469
|
+
})
|
|
470
|
+
)
|
|
471
|
+
|
|
472
|
+
const stream = this.store.get(path)!
|
|
473
|
+
|
|
474
|
+
// Call lifecycle hook for new streams
|
|
475
|
+
if (isNew && this.options.onStreamCreated) {
|
|
476
|
+
await Promise.resolve(
|
|
477
|
+
this.options.onStreamCreated({
|
|
478
|
+
type: `created`,
|
|
479
|
+
path,
|
|
480
|
+
contentType,
|
|
481
|
+
timestamp: Date.now(),
|
|
482
|
+
})
|
|
483
|
+
)
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
// Return 201 for new streams, 200 for idempotent creates
|
|
487
|
+
const headers: Record<string, string> = {
|
|
488
|
+
"content-type": contentType,
|
|
489
|
+
[STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
// Add Location header for 201 Created responses
|
|
493
|
+
if (isNew) {
|
|
494
|
+
headers[`location`] = `${this._url}${path}`
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
res.writeHead(isNew ? 201 : 200, headers)
|
|
498
|
+
res.end()
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/**
|
|
502
|
+
* Handle HEAD - get metadata
|
|
503
|
+
*/
|
|
504
|
+
private handleHead(path: string, res: ServerResponse): void {
|
|
505
|
+
const stream = this.store.get(path)
|
|
506
|
+
if (!stream) {
|
|
507
|
+
res.writeHead(404, { "content-type": `text/plain` })
|
|
508
|
+
res.end()
|
|
509
|
+
return
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
const headers: Record<string, string> = {
|
|
513
|
+
[STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
if (stream.contentType) {
|
|
517
|
+
headers[`content-type`] = stream.contentType
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
// Generate ETag: {path}:-1:{offset} (consistent with GET format)
|
|
521
|
+
headers[`etag`] =
|
|
522
|
+
`"${Buffer.from(path).toString(`base64`)}:-1:${stream.currentOffset}"`
|
|
523
|
+
|
|
524
|
+
res.writeHead(200, headers)
|
|
525
|
+
res.end()
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* Handle GET - read data
|
|
530
|
+
*/
|
|
531
|
+
private async handleRead(
|
|
532
|
+
path: string,
|
|
533
|
+
url: URL,
|
|
534
|
+
req: IncomingMessage,
|
|
535
|
+
res: ServerResponse
|
|
536
|
+
): Promise<void> {
|
|
537
|
+
const stream = this.store.get(path)
|
|
538
|
+
if (!stream) {
|
|
539
|
+
res.writeHead(404, { "content-type": `text/plain` })
|
|
540
|
+
res.end(`Stream not found`)
|
|
541
|
+
return
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
const offset = url.searchParams.get(OFFSET_QUERY_PARAM) ?? undefined
|
|
545
|
+
const live = url.searchParams.get(LIVE_QUERY_PARAM)
|
|
546
|
+
const cursor = url.searchParams.get(CURSOR_QUERY_PARAM) ?? undefined
|
|
547
|
+
|
|
548
|
+
// Validate offset parameter
|
|
549
|
+
if (offset !== undefined) {
|
|
550
|
+
// Reject empty offset
|
|
551
|
+
if (offset === ``) {
|
|
552
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
553
|
+
res.end(`Empty offset parameter`)
|
|
554
|
+
return
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
// Reject multiple offset parameters
|
|
558
|
+
const allOffsets = url.searchParams.getAll(OFFSET_QUERY_PARAM)
|
|
559
|
+
if (allOffsets.length > 1) {
|
|
560
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
561
|
+
res.end(`Multiple offset parameters not allowed`)
|
|
562
|
+
return
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// Validate offset format: must be "-1" or match our offset format (digits_digits)
|
|
566
|
+
// This prevents path traversal, injection attacks, and invalid characters
|
|
567
|
+
const validOffsetPattern = /^(-1|\d+_\d+)$/
|
|
568
|
+
if (!validOffsetPattern.test(offset)) {
|
|
569
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
570
|
+
res.end(`Invalid offset format`)
|
|
571
|
+
return
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// Require offset parameter for long-poll and SSE per protocol spec
|
|
576
|
+
if ((live === `long-poll` || live === `sse`) && !offset) {
|
|
577
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
578
|
+
res.end(
|
|
579
|
+
`${live === `sse` ? `SSE` : `Long-poll`} requires offset parameter`
|
|
580
|
+
)
|
|
581
|
+
return
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
// Handle SSE mode
|
|
585
|
+
if (live === `sse`) {
|
|
586
|
+
await this.handleSSE(path, stream, offset!, cursor, res)
|
|
587
|
+
return
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
// Read current messages
|
|
591
|
+
let { messages, upToDate } = this.store.read(path, offset)
|
|
592
|
+
|
|
593
|
+
// Only wait in long-poll if:
|
|
594
|
+
// 1. long-poll mode is enabled
|
|
595
|
+
// 2. Client provided an offset (not first request)
|
|
596
|
+
// 3. Client's offset matches current offset (already caught up)
|
|
597
|
+
// 4. No new messages
|
|
598
|
+
const clientIsCaughtUp = offset && offset === stream.currentOffset
|
|
599
|
+
if (live === `long-poll` && clientIsCaughtUp && messages.length === 0) {
|
|
600
|
+
const result = await this.store.waitForMessages(
|
|
601
|
+
path,
|
|
602
|
+
offset,
|
|
603
|
+
this.options.longPollTimeout
|
|
604
|
+
)
|
|
605
|
+
|
|
606
|
+
if (result.timedOut) {
|
|
607
|
+
// Return 204 No Content on timeout (per Protocol Section 5.6)
|
|
608
|
+
// Generate cursor for CDN cache collapsing (Protocol Section 8.1)
|
|
609
|
+
const responseCursor = generateResponseCursor(
|
|
610
|
+
cursor,
|
|
611
|
+
this.options.cursorOptions
|
|
612
|
+
)
|
|
613
|
+
res.writeHead(204, {
|
|
614
|
+
[STREAM_OFFSET_HEADER]: offset,
|
|
615
|
+
[STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
616
|
+
[STREAM_CURSOR_HEADER]: responseCursor,
|
|
617
|
+
})
|
|
618
|
+
res.end()
|
|
619
|
+
return
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
messages = result.messages
|
|
623
|
+
upToDate = true
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// Build response
|
|
627
|
+
const headers: Record<string, string> = {}
|
|
628
|
+
|
|
629
|
+
if (stream.contentType) {
|
|
630
|
+
headers[`content-type`] = stream.contentType
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
// Set offset header to the last message's offset, or current if no messages
|
|
634
|
+
const lastMessage = messages[messages.length - 1]
|
|
635
|
+
const responseOffset = lastMessage?.offset ?? stream.currentOffset
|
|
636
|
+
headers[STREAM_OFFSET_HEADER] = responseOffset
|
|
637
|
+
|
|
638
|
+
// Generate cursor for live mode responses (Protocol Section 8.1)
|
|
639
|
+
if (live === `long-poll`) {
|
|
640
|
+
headers[STREAM_CURSOR_HEADER] = generateResponseCursor(
|
|
641
|
+
cursor,
|
|
642
|
+
this.options.cursorOptions
|
|
643
|
+
)
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// Set up-to-date header
|
|
647
|
+
if (upToDate) {
|
|
648
|
+
headers[STREAM_UP_TO_DATE_HEADER] = `true`
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
// Generate ETag: based on path, start offset, and end offset
|
|
652
|
+
const startOffset = offset ?? `-1`
|
|
653
|
+
const etag = `"${Buffer.from(path).toString(`base64`)}:${startOffset}:${responseOffset}"`
|
|
654
|
+
headers[`etag`] = etag
|
|
655
|
+
|
|
656
|
+
// Check If-None-Match for conditional GET (Protocol Section 8.1)
|
|
657
|
+
const ifNoneMatch = req.headers[`if-none-match`]
|
|
658
|
+
if (ifNoneMatch && ifNoneMatch === etag) {
|
|
659
|
+
res.writeHead(304, { etag })
|
|
660
|
+
res.end()
|
|
661
|
+
return
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
// Format response (wraps JSON in array brackets)
|
|
665
|
+
const responseData = this.store.formatResponse(path, messages)
|
|
666
|
+
|
|
667
|
+
// Apply compression if enabled and response is large enough
|
|
668
|
+
let finalData: Uint8Array = responseData
|
|
669
|
+
if (
|
|
670
|
+
this.options.compression &&
|
|
671
|
+
responseData.length >= COMPRESSION_THRESHOLD
|
|
672
|
+
) {
|
|
673
|
+
const acceptEncoding = req.headers[`accept-encoding`]
|
|
674
|
+
const encoding = getCompressionEncoding(acceptEncoding)
|
|
675
|
+
if (encoding) {
|
|
676
|
+
finalData = compressData(responseData, encoding)
|
|
677
|
+
headers[`content-encoding`] = encoding
|
|
678
|
+
// Add Vary header to indicate response varies by Accept-Encoding
|
|
679
|
+
headers[`vary`] = `accept-encoding`
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
res.writeHead(200, headers)
|
|
684
|
+
res.end(Buffer.from(finalData))
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* Handle SSE (Server-Sent Events) mode
|
|
689
|
+
*/
|
|
690
|
+
private async handleSSE(
|
|
691
|
+
path: string,
|
|
692
|
+
stream: ReturnType<StreamStore[`get`]>,
|
|
693
|
+
initialOffset: string,
|
|
694
|
+
cursor: string | undefined,
|
|
695
|
+
res: ServerResponse
|
|
696
|
+
): Promise<void> {
|
|
697
|
+
// Track this SSE connection
|
|
698
|
+
this.activeSSEResponses.add(res)
|
|
699
|
+
|
|
700
|
+
// Set SSE headers
|
|
701
|
+
res.writeHead(200, {
|
|
702
|
+
"content-type": `text/event-stream`,
|
|
703
|
+
"cache-control": `no-cache`,
|
|
704
|
+
connection: `keep-alive`,
|
|
705
|
+
"access-control-allow-origin": `*`,
|
|
706
|
+
})
|
|
707
|
+
|
|
708
|
+
let currentOffset = initialOffset
|
|
709
|
+
let isConnected = true
|
|
710
|
+
const decoder = new TextDecoder()
|
|
711
|
+
|
|
712
|
+
// Handle client disconnect
|
|
713
|
+
res.on(`close`, () => {
|
|
714
|
+
isConnected = false
|
|
715
|
+
this.activeSSEResponses.delete(res)
|
|
716
|
+
})
|
|
717
|
+
|
|
718
|
+
// Get content type for formatting
|
|
719
|
+
const isJsonStream = stream?.contentType?.includes(`application/json`)
|
|
720
|
+
|
|
721
|
+
// Send initial data and then wait for more
|
|
722
|
+
// Note: isConnected and isShuttingDown can change asynchronously
|
|
723
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
724
|
+
while (isConnected && !this.isShuttingDown) {
|
|
725
|
+
// Read current messages from offset
|
|
726
|
+
const { messages, upToDate } = this.store.read(path, currentOffset)
|
|
727
|
+
|
|
728
|
+
// Send data events for each message
|
|
729
|
+
for (const message of messages) {
|
|
730
|
+
// Format data based on content type
|
|
731
|
+
let dataPayload: string
|
|
732
|
+
if (isJsonStream) {
|
|
733
|
+
// Use formatResponse to get properly formatted JSON (strips trailing commas)
|
|
734
|
+
const jsonBytes = this.store.formatResponse(path, [message])
|
|
735
|
+
dataPayload = decoder.decode(jsonBytes)
|
|
736
|
+
} else {
|
|
737
|
+
dataPayload = decoder.decode(message.data)
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
// Send data event - encode multiline payloads per SSE spec
|
|
741
|
+
// Each line in the payload needs its own "data:" prefix
|
|
742
|
+
res.write(`event: data\n`)
|
|
743
|
+
res.write(encodeSSEData(dataPayload))
|
|
744
|
+
|
|
745
|
+
currentOffset = message.offset
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
// Compute offset the same way as HTTP GET: last message's offset, or stream's current offset
|
|
749
|
+
const controlOffset =
|
|
750
|
+
messages[messages.length - 1]?.offset ?? stream!.currentOffset
|
|
751
|
+
|
|
752
|
+
// Send control event with current offset/cursor (Protocol Section 5.7)
|
|
753
|
+
// Generate cursor for CDN cache collapsing (Protocol Section 8.1)
|
|
754
|
+
const responseCursor = generateResponseCursor(
|
|
755
|
+
cursor,
|
|
756
|
+
this.options.cursorOptions
|
|
757
|
+
)
|
|
758
|
+
const controlData: Record<string, string | boolean> = {
|
|
759
|
+
[SSE_OFFSET_FIELD]: controlOffset,
|
|
760
|
+
[SSE_CURSOR_FIELD]: responseCursor,
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
// Include upToDate flag when client has caught up to head
|
|
764
|
+
if (upToDate) {
|
|
765
|
+
controlData[SSE_UP_TO_DATE_FIELD] = true
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
res.write(`event: control\n`)
|
|
769
|
+
res.write(encodeSSEData(JSON.stringify(controlData)))
|
|
770
|
+
|
|
771
|
+
// Update currentOffset for next iteration (use controlOffset for consistency)
|
|
772
|
+
currentOffset = controlOffset
|
|
773
|
+
|
|
774
|
+
// If caught up, wait for new messages
|
|
775
|
+
if (upToDate) {
|
|
776
|
+
const result = await this.store.waitForMessages(
|
|
777
|
+
path,
|
|
778
|
+
currentOffset,
|
|
779
|
+
this.options.longPollTimeout
|
|
780
|
+
)
|
|
781
|
+
|
|
782
|
+
// Check if we should exit after wait returns (values can change during await)
|
|
783
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
784
|
+
if (this.isShuttingDown || !isConnected) break
|
|
785
|
+
|
|
786
|
+
if (result.timedOut) {
|
|
787
|
+
// Send keep-alive control event on timeout (Protocol Section 5.7)
|
|
788
|
+
// Generate cursor for CDN cache collapsing (Protocol Section 8.1)
|
|
789
|
+
const keepAliveCursor = generateResponseCursor(
|
|
790
|
+
cursor,
|
|
791
|
+
this.options.cursorOptions
|
|
792
|
+
)
|
|
793
|
+
const keepAliveData: Record<string, string | boolean> = {
|
|
794
|
+
[SSE_OFFSET_FIELD]: currentOffset,
|
|
795
|
+
[SSE_CURSOR_FIELD]: keepAliveCursor,
|
|
796
|
+
[SSE_UP_TO_DATE_FIELD]: true, // Still caught up after timeout
|
|
797
|
+
}
|
|
798
|
+
res.write(`event: control\n`)
|
|
799
|
+
res.write(encodeSSEData(JSON.stringify(keepAliveData)))
|
|
800
|
+
}
|
|
801
|
+
// Loop will continue and read new messages
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
this.activeSSEResponses.delete(res)
|
|
806
|
+
res.end()
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
/**
|
|
810
|
+
* Handle POST - append data
|
|
811
|
+
*/
|
|
812
|
+
private async handleAppend(
|
|
813
|
+
path: string,
|
|
814
|
+
req: IncomingMessage,
|
|
815
|
+
res: ServerResponse
|
|
816
|
+
): Promise<void> {
|
|
817
|
+
const contentType = req.headers[`content-type`]
|
|
818
|
+
const seq = req.headers[STREAM_SEQ_HEADER.toLowerCase()] as
|
|
819
|
+
| string
|
|
820
|
+
| undefined
|
|
821
|
+
|
|
822
|
+
const body = await this.readBody(req)
|
|
823
|
+
|
|
824
|
+
if (body.length === 0) {
|
|
825
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
826
|
+
res.end(`Empty body`)
|
|
827
|
+
return
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
// Content-Type is required per protocol
|
|
831
|
+
if (!contentType) {
|
|
832
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
833
|
+
res.end(`Content-Type header is required`)
|
|
834
|
+
return
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
// Support both sync (StreamStore) and async (FileBackedStreamStore) append
|
|
838
|
+
// Note: append returns null only for empty arrays with isInitialCreate=true,
|
|
839
|
+
// which doesn't apply to POST requests (those throw on empty arrays)
|
|
840
|
+
const message = await Promise.resolve(
|
|
841
|
+
this.store.append(path, body, { seq, contentType })
|
|
842
|
+
)
|
|
843
|
+
|
|
844
|
+
res.writeHead(200, {
|
|
845
|
+
[STREAM_OFFSET_HEADER]: message!.offset,
|
|
846
|
+
})
|
|
847
|
+
res.end()
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* Handle DELETE - delete stream
|
|
852
|
+
*/
|
|
853
|
+
private async handleDelete(path: string, res: ServerResponse): Promise<void> {
|
|
854
|
+
if (!this.store.has(path)) {
|
|
855
|
+
res.writeHead(404, { "content-type": `text/plain` })
|
|
856
|
+
res.end(`Stream not found`)
|
|
857
|
+
return
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
this.store.delete(path)
|
|
861
|
+
|
|
862
|
+
// Call lifecycle hook
|
|
863
|
+
if (this.options.onStreamDeleted) {
|
|
864
|
+
await Promise.resolve(
|
|
865
|
+
this.options.onStreamDeleted({
|
|
866
|
+
type: `deleted`,
|
|
867
|
+
path,
|
|
868
|
+
timestamp: Date.now(),
|
|
869
|
+
})
|
|
870
|
+
)
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
res.writeHead(204)
|
|
874
|
+
res.end()
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
/**
|
|
878
|
+
* Handle test control endpoints for error injection.
|
|
879
|
+
* POST /_test/inject-error - inject an error
|
|
880
|
+
* DELETE /_test/inject-error - clear all injected errors
|
|
881
|
+
*/
|
|
882
|
+
private async handleTestInjectError(
|
|
883
|
+
method: string | undefined,
|
|
884
|
+
req: IncomingMessage,
|
|
885
|
+
res: ServerResponse
|
|
886
|
+
): Promise<void> {
|
|
887
|
+
if (method === `POST`) {
|
|
888
|
+
const body = await this.readBody(req)
|
|
889
|
+
try {
|
|
890
|
+
const config = JSON.parse(new TextDecoder().decode(body)) as {
|
|
891
|
+
path: string
|
|
892
|
+
status: number
|
|
893
|
+
count?: number
|
|
894
|
+
retryAfter?: number
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
if (!config.path || !config.status) {
|
|
898
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
899
|
+
res.end(`Missing required fields: path, status`)
|
|
900
|
+
return
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
this.injectError(
|
|
904
|
+
config.path,
|
|
905
|
+
config.status,
|
|
906
|
+
config.count ?? 1,
|
|
907
|
+
config.retryAfter
|
|
908
|
+
)
|
|
909
|
+
|
|
910
|
+
res.writeHead(200, { "content-type": `application/json` })
|
|
911
|
+
res.end(JSON.stringify({ ok: true }))
|
|
912
|
+
} catch {
|
|
913
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
914
|
+
res.end(`Invalid JSON body`)
|
|
915
|
+
}
|
|
916
|
+
} else if (method === `DELETE`) {
|
|
917
|
+
this.clearInjectedErrors()
|
|
918
|
+
res.writeHead(200, { "content-type": `application/json` })
|
|
919
|
+
res.end(JSON.stringify({ ok: true }))
|
|
920
|
+
} else {
|
|
921
|
+
res.writeHead(405, { "content-type": `text/plain` })
|
|
922
|
+
res.end(`Method not allowed`)
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
// ============================================================================
|
|
927
|
+
// Helpers
|
|
928
|
+
// ============================================================================
|
|
929
|
+
|
|
930
|
+
private readBody(req: IncomingMessage): Promise<Uint8Array> {
|
|
931
|
+
return new Promise((resolve, reject) => {
|
|
932
|
+
const chunks: Array<Buffer> = []
|
|
933
|
+
|
|
934
|
+
req.on(`data`, (chunk: Buffer) => {
|
|
935
|
+
chunks.push(chunk)
|
|
936
|
+
})
|
|
937
|
+
|
|
938
|
+
req.on(`end`, () => {
|
|
939
|
+
const body = Buffer.concat(chunks)
|
|
940
|
+
resolve(new Uint8Array(body))
|
|
941
|
+
})
|
|
942
|
+
|
|
943
|
+
req.on(`error`, reject)
|
|
944
|
+
})
|
|
945
|
+
}
|
|
946
|
+
}
|