@durable-streams/server 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1297 -260
- package/dist/index.d.cts +236 -2
- package/dist/index.d.ts +236 -2
- package/dist/index.js +1344 -312
- package/package.json +3 -3
- package/src/crypto.ts +217 -0
- package/src/file-store.ts +187 -144
- package/src/glob.ts +70 -0
- package/src/index.ts +14 -0
- package/src/log.ts +56 -0
- package/src/server.ts +75 -26
- package/src/store.ts +59 -7
- package/src/subscription-manager.ts +882 -0
- package/src/subscription-routes.ts +504 -0
- package/src/subscription-types.ts +80 -0
- package/src/types.ts +8 -0
package/dist/index.js
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { createServer } from "node:http";
|
|
2
2
|
import { deflateSync, gzipSync } from "node:zlib";
|
|
3
|
+
import { CURSOR_QUERY_PARAM, DurableStream, LIVE_QUERY_PARAM, OFFSET_QUERY_PARAM, PRODUCER_EPOCH_HEADER, PRODUCER_EXPECTED_SEQ_HEADER, PRODUCER_ID_HEADER, PRODUCER_RECEIVED_SEQ_HEADER, PRODUCER_SEQ_HEADER, SSE_CLOSED_FIELD, SSE_CURSOR_FIELD, SSE_OFFSET_FIELD, STREAM_CLOSED_HEADER, STREAM_CURSOR_HEADER, STREAM_EXPIRES_AT_HEADER, STREAM_OFFSET_HEADER, STREAM_SEQ_HEADER, STREAM_TTL_HEADER, STREAM_UP_TO_DATE_HEADER } from "@durable-streams/client";
|
|
3
4
|
import * as fs from "node:fs";
|
|
4
|
-
import * as path$1 from "node:path";
|
|
5
5
|
import * as path from "node:path";
|
|
6
|
-
import { createHash, randomBytes } from "node:crypto";
|
|
6
|
+
import { createHash, createHmac, generateKeyPairSync, randomBytes, sign, timingSafeEqual } from "node:crypto";
|
|
7
7
|
import { open } from "lmdb";
|
|
8
8
|
import { SieveCache } from "@neophi/sieve-cache";
|
|
9
|
-
import
|
|
10
|
-
import { DurableStream } from "@durable-streams/client";
|
|
9
|
+
import { isIP } from "node:net";
|
|
11
10
|
import { createStateSchema } from "@durable-streams/state";
|
|
12
11
|
|
|
13
12
|
//#region src/store.ts
|
|
@@ -50,17 +49,35 @@ function processJsonAppend(data, isInitialCreate = false) {
|
|
|
50
49
|
} else result = JSON.stringify(parsed) + `,`;
|
|
51
50
|
return new TextEncoder().encode(result);
|
|
52
51
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
* Strips trailing comma before wrapping.
|
|
56
|
-
*/
|
|
57
|
-
function formatJsonResponse(data) {
|
|
58
|
-
if (data.length === 0) return new TextEncoder().encode(`[]`);
|
|
59
|
-
let text = new TextDecoder().decode(data);
|
|
60
|
-
text = text.trimEnd();
|
|
52
|
+
function decodeStoredJsonMessage(data) {
|
|
53
|
+
let text = new TextDecoder().decode(data).trimEnd();
|
|
61
54
|
if (text.endsWith(`,`)) text = text.slice(0, -1);
|
|
62
|
-
|
|
63
|
-
|
|
55
|
+
return text;
|
|
56
|
+
}
|
|
57
|
+
function enrichJsonValueWithOffset(parsed, offset) {
|
|
58
|
+
if (!parsed || typeof parsed !== `object` || Array.isArray(parsed)) return JSON.stringify(parsed);
|
|
59
|
+
const candidate = parsed;
|
|
60
|
+
const headers = candidate.headers;
|
|
61
|
+
if (!headers || typeof headers !== `object`) return JSON.stringify(parsed);
|
|
62
|
+
const isStateChange = typeof headers.operation === `string`;
|
|
63
|
+
const isStateControl = typeof headers.control === `string`;
|
|
64
|
+
if (!isStateChange && !isStateControl) return JSON.stringify(parsed);
|
|
65
|
+
return JSON.stringify({
|
|
66
|
+
...candidate,
|
|
67
|
+
headers: {
|
|
68
|
+
...headers,
|
|
69
|
+
offset
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
function formatJsonMessages(messages) {
|
|
74
|
+
if (messages.length === 0) return new TextEncoder().encode(`[]`);
|
|
75
|
+
const items = messages.flatMap((message) => {
|
|
76
|
+
const rawFragment = decodeStoredJsonMessage(message.data);
|
|
77
|
+
const parsed = JSON.parse(`[${rawFragment}]`);
|
|
78
|
+
return parsed.map((value) => enrichJsonValueWithOffset(value, message.offset));
|
|
79
|
+
});
|
|
80
|
+
return new TextEncoder().encode(`[${items.join(`,`)}]`);
|
|
64
81
|
}
|
|
65
82
|
var StreamStore = class {
|
|
66
83
|
streams = new Map();
|
|
@@ -90,15 +107,15 @@ var StreamStore = class {
|
|
|
90
107
|
* Returns undefined if stream doesn't exist or is expired (and has no refs).
|
|
91
108
|
* Expired streams with refCount > 0 are soft-deleted instead of fully deleted.
|
|
92
109
|
*/
|
|
93
|
-
getIfNotExpired(path$
|
|
94
|
-
const stream = this.streams.get(path$
|
|
110
|
+
getIfNotExpired(path$1) {
|
|
111
|
+
const stream = this.streams.get(path$1);
|
|
95
112
|
if (!stream) return void 0;
|
|
96
113
|
if (this.isExpired(stream)) {
|
|
97
114
|
if (stream.refCount > 0) {
|
|
98
115
|
stream.softDeleted = true;
|
|
99
116
|
return stream;
|
|
100
117
|
}
|
|
101
|
-
this.delete(path$
|
|
118
|
+
this.delete(path$1);
|
|
102
119
|
return void 0;
|
|
103
120
|
}
|
|
104
121
|
return stream;
|
|
@@ -106,8 +123,8 @@ var StreamStore = class {
|
|
|
106
123
|
/**
|
|
107
124
|
* Update lastAccessedAt to now. Called on reads and appends (not HEAD).
|
|
108
125
|
*/
|
|
109
|
-
touchAccess(path$
|
|
110
|
-
const stream = this.streams.get(path$
|
|
126
|
+
touchAccess(path$1) {
|
|
127
|
+
const stream = this.streams.get(path$1);
|
|
111
128
|
if (stream) stream.lastAccessedAt = Date.now();
|
|
112
129
|
}
|
|
113
130
|
/**
|
|
@@ -116,12 +133,12 @@ var StreamStore = class {
|
|
|
116
133
|
* @throws Error if fork source not found, soft-deleted, or offset invalid
|
|
117
134
|
* @returns existing stream if config matches (idempotent)
|
|
118
135
|
*/
|
|
119
|
-
create(path$
|
|
120
|
-
const existingRaw = this.streams.get(path$
|
|
136
|
+
create(path$1, options = {}) {
|
|
137
|
+
const existingRaw = this.streams.get(path$1);
|
|
121
138
|
if (existingRaw) if (this.isExpired(existingRaw)) {
|
|
122
|
-
this.streams.delete(path$
|
|
123
|
-
this.cancelLongPollsForStream(path$
|
|
124
|
-
} else if (existingRaw.softDeleted) throw new Error(`Stream has active forks — path cannot be reused until all forks are removed: ${path$
|
|
139
|
+
this.streams.delete(path$1);
|
|
140
|
+
this.cancelLongPollsForStream(path$1);
|
|
141
|
+
} else if (existingRaw.softDeleted) throw new Error(`Stream has active forks — path cannot be reused until all forks are removed: ${path$1}`);
|
|
125
142
|
else {
|
|
126
143
|
const contentTypeMatches = (normalizeContentType(options.contentType) || `application/octet-stream`) === (normalizeContentType(existingRaw.contentType) || `application/octet-stream`);
|
|
127
144
|
const ttlMatches = options.ttlSeconds === existingRaw.ttlSeconds;
|
|
@@ -130,7 +147,7 @@ var StreamStore = class {
|
|
|
130
147
|
const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
|
|
131
148
|
const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
|
|
132
149
|
if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches) return existingRaw;
|
|
133
|
-
else throw new Error(`Stream already exists with different configuration: ${path$
|
|
150
|
+
else throw new Error(`Stream already exists with different configuration: ${path$1}`);
|
|
134
151
|
}
|
|
135
152
|
const isFork = !!options.forkedFrom;
|
|
136
153
|
let forkOffset = `0000000000000000_0000000000000000`;
|
|
@@ -160,7 +177,7 @@ var StreamStore = class {
|
|
|
160
177
|
effectiveTtlSeconds = resolved.ttlSeconds;
|
|
161
178
|
}
|
|
162
179
|
const stream = {
|
|
163
|
-
path: path$
|
|
180
|
+
path: path$1,
|
|
164
181
|
contentType,
|
|
165
182
|
messages: [],
|
|
166
183
|
currentOffset: isFork ? forkOffset : `0000000000000000_0000000000000000`,
|
|
@@ -179,7 +196,7 @@ var StreamStore = class {
|
|
|
179
196
|
if (isFork && sourceStream) sourceStream.refCount--;
|
|
180
197
|
throw err;
|
|
181
198
|
}
|
|
182
|
-
this.streams.set(path$
|
|
199
|
+
this.streams.set(path$1, stream);
|
|
183
200
|
return stream;
|
|
184
201
|
}
|
|
185
202
|
/**
|
|
@@ -198,15 +215,15 @@ var StreamStore = class {
|
|
|
198
215
|
* Returns undefined if stream doesn't exist or is expired.
|
|
199
216
|
* Returns soft-deleted streams (caller should check stream.softDeleted).
|
|
200
217
|
*/
|
|
201
|
-
get(path$
|
|
202
|
-
const stream = this.streams.get(path$
|
|
218
|
+
get(path$1) {
|
|
219
|
+
const stream = this.streams.get(path$1);
|
|
203
220
|
if (!stream) return void 0;
|
|
204
221
|
if (this.isExpired(stream)) {
|
|
205
222
|
if (stream.refCount > 0) {
|
|
206
223
|
stream.softDeleted = true;
|
|
207
224
|
return stream;
|
|
208
225
|
}
|
|
209
|
-
this.delete(path$
|
|
226
|
+
this.delete(path$1);
|
|
210
227
|
return void 0;
|
|
211
228
|
}
|
|
212
229
|
return stream;
|
|
@@ -214,8 +231,8 @@ var StreamStore = class {
|
|
|
214
231
|
/**
|
|
215
232
|
* Check if a stream exists, is not expired, and is not soft-deleted.
|
|
216
233
|
*/
|
|
217
|
-
has(path$
|
|
218
|
-
const stream = this.get(path$
|
|
234
|
+
has(path$1) {
|
|
235
|
+
const stream = this.get(path$1);
|
|
219
236
|
if (!stream) return false;
|
|
220
237
|
if (stream.softDeleted) return false;
|
|
221
238
|
return true;
|
|
@@ -225,27 +242,27 @@ var StreamStore = class {
|
|
|
225
242
|
* If the stream has forks (refCount > 0), it is soft-deleted instead of fully removed.
|
|
226
243
|
* Returns true if the stream was found and deleted (or soft-deleted).
|
|
227
244
|
*/
|
|
228
|
-
delete(path$
|
|
229
|
-
const stream = this.streams.get(path$
|
|
245
|
+
delete(path$1) {
|
|
246
|
+
const stream = this.streams.get(path$1);
|
|
230
247
|
if (!stream) return false;
|
|
231
248
|
if (stream.softDeleted) return true;
|
|
232
249
|
if (stream.refCount > 0) {
|
|
233
250
|
stream.softDeleted = true;
|
|
234
251
|
return true;
|
|
235
252
|
}
|
|
236
|
-
this.deleteWithCascade(path$
|
|
253
|
+
this.deleteWithCascade(path$1);
|
|
237
254
|
return true;
|
|
238
255
|
}
|
|
239
256
|
/**
|
|
240
257
|
* Fully delete a stream and cascade to soft-deleted parents
|
|
241
258
|
* whose refcount drops to zero.
|
|
242
259
|
*/
|
|
243
|
-
deleteWithCascade(path$
|
|
244
|
-
const stream = this.streams.get(path$
|
|
260
|
+
deleteWithCascade(path$1) {
|
|
261
|
+
const stream = this.streams.get(path$1);
|
|
245
262
|
if (!stream) return;
|
|
246
263
|
const forkedFrom = stream.forkedFrom;
|
|
247
|
-
this.streams.delete(path$
|
|
248
|
-
this.cancelLongPollsForStream(path$
|
|
264
|
+
this.streams.delete(path$1);
|
|
265
|
+
this.cancelLongPollsForStream(path$1);
|
|
249
266
|
if (forkedFrom) {
|
|
250
267
|
const parent = this.streams.get(forkedFrom);
|
|
251
268
|
if (parent) {
|
|
@@ -344,8 +361,8 @@ var StreamStore = class {
|
|
|
344
361
|
* Acquire a lock for serialized producer operations.
|
|
345
362
|
* Returns a release function.
|
|
346
363
|
*/
|
|
347
|
-
async acquireProducerLock(path$
|
|
348
|
-
const lockKey = `${path$
|
|
364
|
+
async acquireProducerLock(path$1, producerId) {
|
|
365
|
+
const lockKey = `${path$1}:${producerId}`;
|
|
349
366
|
while (this.producerLocks.has(lockKey)) await this.producerLocks.get(lockKey);
|
|
350
367
|
let releaseLock;
|
|
351
368
|
const lockPromise = new Promise((resolve) => {
|
|
@@ -363,10 +380,10 @@ var StreamStore = class {
|
|
|
363
380
|
* @throws Error if seq is lower than lastSeq
|
|
364
381
|
* @throws Error if JSON mode and array is empty
|
|
365
382
|
*/
|
|
366
|
-
append(path$
|
|
367
|
-
const stream = this.getIfNotExpired(path$
|
|
368
|
-
if (!stream) throw new Error(`Stream not found: ${path$
|
|
369
|
-
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path$
|
|
383
|
+
append(path$1, data, options = {}) {
|
|
384
|
+
const stream = this.getIfNotExpired(path$1);
|
|
385
|
+
if (!stream) throw new Error(`Stream not found: ${path$1}`);
|
|
386
|
+
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path$1}`);
|
|
370
387
|
if (stream.closed) {
|
|
371
388
|
if (options.producerId && stream.closedBy && stream.closedBy.producerId === options.producerId && stream.closedBy.epoch === options.producerEpoch && stream.closedBy.seq === options.producerSeq) return {
|
|
372
389
|
message: null,
|
|
@@ -408,8 +425,8 @@ var StreamStore = class {
|
|
|
408
425
|
seq: options.producerSeq
|
|
409
426
|
};
|
|
410
427
|
}
|
|
411
|
-
this.notifyLongPolls(path$
|
|
412
|
-
if (options.close) this.notifyLongPollsClosed(path$
|
|
428
|
+
this.notifyLongPolls(path$1);
|
|
429
|
+
if (options.close) this.notifyLongPollsClosed(path$1);
|
|
413
430
|
if (producerResult || options.close) return {
|
|
414
431
|
message,
|
|
415
432
|
producerResult,
|
|
@@ -421,15 +438,15 @@ var StreamStore = class {
|
|
|
421
438
|
* Append with producer serialization for concurrent request handling.
|
|
422
439
|
* This ensures that validation+append is atomic per producer.
|
|
423
440
|
*/
|
|
424
|
-
async appendWithProducer(path$
|
|
441
|
+
async appendWithProducer(path$1, data, options) {
|
|
425
442
|
if (!options.producerId) {
|
|
426
|
-
const result = this.append(path$
|
|
443
|
+
const result = this.append(path$1, data, options);
|
|
427
444
|
if (`message` in result) return result;
|
|
428
445
|
return { message: result };
|
|
429
446
|
}
|
|
430
|
-
const releaseLock = await this.acquireProducerLock(path$
|
|
447
|
+
const releaseLock = await this.acquireProducerLock(path$1, options.producerId);
|
|
431
448
|
try {
|
|
432
|
-
const result = this.append(path$
|
|
449
|
+
const result = this.append(path$1, data, options);
|
|
433
450
|
if (`message` in result) return result;
|
|
434
451
|
return { message: result };
|
|
435
452
|
} finally {
|
|
@@ -440,13 +457,13 @@ var StreamStore = class {
|
|
|
440
457
|
* Close a stream without appending data.
|
|
441
458
|
* @returns The final offset, or null if stream doesn't exist
|
|
442
459
|
*/
|
|
443
|
-
closeStream(path$
|
|
444
|
-
const stream = this.getIfNotExpired(path$
|
|
460
|
+
closeStream(path$1) {
|
|
461
|
+
const stream = this.getIfNotExpired(path$1);
|
|
445
462
|
if (!stream) return null;
|
|
446
|
-
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path$
|
|
463
|
+
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path$1}`);
|
|
447
464
|
const alreadyClosed = stream.closed ?? false;
|
|
448
465
|
stream.closed = true;
|
|
449
|
-
this.notifyLongPollsClosed(path$
|
|
466
|
+
this.notifyLongPollsClosed(path$1);
|
|
450
467
|
return {
|
|
451
468
|
finalOffset: stream.currentOffset,
|
|
452
469
|
alreadyClosed
|
|
@@ -457,10 +474,10 @@ var StreamStore = class {
|
|
|
457
474
|
* Participates in producer sequencing for deduplication.
|
|
458
475
|
* @returns The final offset and producer result, or null if stream doesn't exist
|
|
459
476
|
*/
|
|
460
|
-
async closeStreamWithProducer(path$
|
|
461
|
-
const releaseLock = await this.acquireProducerLock(path$
|
|
477
|
+
async closeStreamWithProducer(path$1, options) {
|
|
478
|
+
const releaseLock = await this.acquireProducerLock(path$1, options.producerId);
|
|
462
479
|
try {
|
|
463
|
-
const stream = this.getIfNotExpired(path$
|
|
480
|
+
const stream = this.getIfNotExpired(path$1);
|
|
464
481
|
if (!stream) return null;
|
|
465
482
|
if (stream.closed) {
|
|
466
483
|
if (stream.closedBy && stream.closedBy.producerId === options.producerId && stream.closedBy.epoch === options.producerEpoch && stream.closedBy.seq === options.producerSeq) return {
|
|
@@ -490,7 +507,7 @@ var StreamStore = class {
|
|
|
490
507
|
epoch: options.producerEpoch,
|
|
491
508
|
seq: options.producerSeq
|
|
492
509
|
};
|
|
493
|
-
this.notifyLongPollsClosed(path$
|
|
510
|
+
this.notifyLongPollsClosed(path$1);
|
|
494
511
|
return {
|
|
495
512
|
finalOffset: stream.currentOffset,
|
|
496
513
|
alreadyClosed: false,
|
|
@@ -504,8 +521,8 @@ var StreamStore = class {
|
|
|
504
521
|
* Get the current epoch for a producer on a stream.
|
|
505
522
|
* Returns undefined if the producer doesn't exist or stream not found.
|
|
506
523
|
*/
|
|
507
|
-
getProducerEpoch(path$
|
|
508
|
-
const stream = this.getIfNotExpired(path$
|
|
524
|
+
getProducerEpoch(path$1, producerId) {
|
|
525
|
+
const stream = this.getIfNotExpired(path$1);
|
|
509
526
|
if (!stream?.producers) return void 0;
|
|
510
527
|
return stream.producers.get(producerId)?.epoch;
|
|
511
528
|
}
|
|
@@ -514,9 +531,9 @@ var StreamStore = class {
|
|
|
514
531
|
* For forked streams, stitches messages from the source chain and the fork's own messages.
|
|
515
532
|
* @throws Error if stream doesn't exist or is expired
|
|
516
533
|
*/
|
|
517
|
-
read(path$
|
|
518
|
-
const stream = this.getIfNotExpired(path$
|
|
519
|
-
if (!stream) throw new Error(`Stream not found: ${path$
|
|
534
|
+
read(path$1, offset) {
|
|
535
|
+
const stream = this.getIfNotExpired(path$1);
|
|
536
|
+
if (!stream) throw new Error(`Stream not found: ${path$1}`);
|
|
520
537
|
if (!offset || offset === `-1`) {
|
|
521
538
|
if (stream.forkedFrom) {
|
|
522
539
|
const inherited = this.readForkedMessages(stream.forkedFrom, void 0, stream.forkOffset);
|
|
@@ -595,9 +612,10 @@ var StreamStore = class {
|
|
|
595
612
|
* For JSON mode, wraps concatenated data in array brackets.
|
|
596
613
|
* @throws Error if stream doesn't exist or is expired
|
|
597
614
|
*/
|
|
598
|
-
formatResponse(path$
|
|
599
|
-
const stream = this.getIfNotExpired(path$
|
|
600
|
-
if (!stream) throw new Error(`Stream not found: ${path$
|
|
615
|
+
formatResponse(path$1, messages) {
|
|
616
|
+
const stream = this.getIfNotExpired(path$1);
|
|
617
|
+
if (!stream) throw new Error(`Stream not found: ${path$1}`);
|
|
618
|
+
if (normalizeContentType(stream.contentType) === `application/json`) return formatJsonMessages(messages);
|
|
601
619
|
const totalSize = messages.reduce((sum, m) => sum + m.data.length, 0);
|
|
602
620
|
const concatenated = new Uint8Array(totalSize);
|
|
603
621
|
let offset = 0;
|
|
@@ -605,24 +623,23 @@ var StreamStore = class {
|
|
|
605
623
|
concatenated.set(msg.data, offset);
|
|
606
624
|
offset += msg.data.length;
|
|
607
625
|
}
|
|
608
|
-
if (normalizeContentType(stream.contentType) === `application/json`) return formatJsonResponse(concatenated);
|
|
609
626
|
return concatenated;
|
|
610
627
|
}
|
|
611
628
|
/**
|
|
612
629
|
* Wait for new messages (long-poll).
|
|
613
630
|
* @throws Error if stream doesn't exist or is expired
|
|
614
631
|
*/
|
|
615
|
-
async waitForMessages(path$
|
|
616
|
-
const stream = this.getIfNotExpired(path$
|
|
617
|
-
if (!stream) throw new Error(`Stream not found: ${path$
|
|
632
|
+
async waitForMessages(path$1, offset, timeoutMs) {
|
|
633
|
+
const stream = this.getIfNotExpired(path$1);
|
|
634
|
+
if (!stream) throw new Error(`Stream not found: ${path$1}`);
|
|
618
635
|
if (stream.forkedFrom && offset < stream.forkOffset) {
|
|
619
|
-
const { messages: messages$1 } = this.read(path$
|
|
636
|
+
const { messages: messages$1 } = this.read(path$1, offset);
|
|
620
637
|
return {
|
|
621
638
|
messages: messages$1,
|
|
622
639
|
timedOut: false
|
|
623
640
|
};
|
|
624
641
|
}
|
|
625
|
-
const { messages } = this.read(path$
|
|
642
|
+
const { messages } = this.read(path$1, offset);
|
|
626
643
|
if (messages.length > 0) return {
|
|
627
644
|
messages,
|
|
628
645
|
timedOut: false
|
|
@@ -635,7 +652,7 @@ var StreamStore = class {
|
|
|
635
652
|
return new Promise((resolve) => {
|
|
636
653
|
const timeoutId = setTimeout(() => {
|
|
637
654
|
this.removePendingLongPoll(pending);
|
|
638
|
-
const currentStream = this.getIfNotExpired(path$
|
|
655
|
+
const currentStream = this.getIfNotExpired(path$1);
|
|
639
656
|
const streamClosed = currentStream?.closed ?? false;
|
|
640
657
|
resolve({
|
|
641
658
|
messages: [],
|
|
@@ -644,12 +661,12 @@ var StreamStore = class {
|
|
|
644
661
|
});
|
|
645
662
|
}, timeoutMs);
|
|
646
663
|
const pending = {
|
|
647
|
-
path: path$
|
|
664
|
+
path: path$1,
|
|
648
665
|
offset,
|
|
649
666
|
resolve: (msgs) => {
|
|
650
667
|
clearTimeout(timeoutId);
|
|
651
668
|
this.removePendingLongPoll(pending);
|
|
652
|
-
const currentStream = this.getIfNotExpired(path$
|
|
669
|
+
const currentStream = this.getIfNotExpired(path$1);
|
|
653
670
|
const streamClosed = currentStream?.closed && msgs.length === 0 ? true : void 0;
|
|
654
671
|
resolve({
|
|
655
672
|
messages: msgs,
|
|
@@ -666,8 +683,8 @@ var StreamStore = class {
|
|
|
666
683
|
* Get the current offset for a stream.
|
|
667
684
|
* Returns undefined if stream doesn't exist or is expired.
|
|
668
685
|
*/
|
|
669
|
-
getCurrentOffset(path$
|
|
670
|
-
return this.getIfNotExpired(path$
|
|
686
|
+
getCurrentOffset(path$1) {
|
|
687
|
+
return this.getIfNotExpired(path$1)?.currentOffset;
|
|
671
688
|
}
|
|
672
689
|
/**
|
|
673
690
|
* Clear all streams.
|
|
@@ -705,7 +722,8 @@ var StreamStore = class {
|
|
|
705
722
|
const parts = stream.currentOffset.split(`_`).map(Number);
|
|
706
723
|
const readSeq = parts[0];
|
|
707
724
|
const byteOffset = parts[1];
|
|
708
|
-
const
|
|
725
|
+
const FRAME_OVERHEAD = 5;
|
|
726
|
+
const newByteOffset = byteOffset + FRAME_OVERHEAD + processedData.length;
|
|
709
727
|
const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
710
728
|
const message = {
|
|
711
729
|
data: processedData,
|
|
@@ -720,10 +738,10 @@ var StreamStore = class {
|
|
|
720
738
|
for (let i = 0; i < stream.messages.length; i++) if (stream.messages[i].offset > offset) return i;
|
|
721
739
|
return -1;
|
|
722
740
|
}
|
|
723
|
-
notifyLongPolls(path$
|
|
724
|
-
const toNotify = this.pendingLongPolls.filter((p) => p.path === path$
|
|
741
|
+
notifyLongPolls(path$1) {
|
|
742
|
+
const toNotify = this.pendingLongPolls.filter((p) => p.path === path$1);
|
|
725
743
|
for (const pending of toNotify) {
|
|
726
|
-
const { messages } = this.read(path$
|
|
744
|
+
const { messages } = this.read(path$1, pending.offset);
|
|
727
745
|
if (messages.length > 0) pending.resolve(messages);
|
|
728
746
|
}
|
|
729
747
|
}
|
|
@@ -731,17 +749,17 @@ var StreamStore = class {
|
|
|
731
749
|
* Notify pending long-polls that a stream has been closed.
|
|
732
750
|
* They should wake up immediately and return Stream-Closed: true.
|
|
733
751
|
*/
|
|
734
|
-
notifyLongPollsClosed(path$
|
|
735
|
-
const toNotify = this.pendingLongPolls.filter((p) => p.path === path$
|
|
752
|
+
notifyLongPollsClosed(path$1) {
|
|
753
|
+
const toNotify = this.pendingLongPolls.filter((p) => p.path === path$1);
|
|
736
754
|
for (const pending of toNotify) pending.resolve([]);
|
|
737
755
|
}
|
|
738
|
-
cancelLongPollsForStream(path$
|
|
739
|
-
const toCancel = this.pendingLongPolls.filter((p) => p.path === path$
|
|
756
|
+
cancelLongPollsForStream(path$1) {
|
|
757
|
+
const toCancel = this.pendingLongPolls.filter((p) => p.path === path$1);
|
|
740
758
|
for (const pending of toCancel) {
|
|
741
759
|
clearTimeout(pending.timeoutId);
|
|
742
760
|
pending.resolve([]);
|
|
743
761
|
}
|
|
744
|
-
this.pendingLongPolls = this.pendingLongPolls.filter((p) => p.path !== path$
|
|
762
|
+
this.pendingLongPolls = this.pendingLongPolls.filter((p) => p.path !== path$1);
|
|
745
763
|
}
|
|
746
764
|
removePendingLongPoll(pending) {
|
|
747
765
|
const index = this.pendingLongPolls.indexOf(pending);
|
|
@@ -749,6 +767,48 @@ var StreamStore = class {
|
|
|
749
767
|
}
|
|
750
768
|
};
|
|
751
769
|
|
|
770
|
+
//#endregion
|
|
771
|
+
//#region src/log.ts
|
|
772
|
+
const streamsLogFile = process.env.STREAMS_LOG_FILE;
|
|
773
|
+
async function appendLogLine(line) {
|
|
774
|
+
if (!streamsLogFile) return;
|
|
775
|
+
const fs$1 = await import(`node:fs/promises`);
|
|
776
|
+
const path$1 = await import(`node:path`);
|
|
777
|
+
await fs$1.mkdir(path$1.dirname(streamsLogFile), { recursive: true });
|
|
778
|
+
await fs$1.appendFile(streamsLogFile, `${line}\n`);
|
|
779
|
+
}
|
|
780
|
+
function serializeArg(arg) {
|
|
781
|
+
if (arg instanceof Error) return arg.stack ?? arg.message;
|
|
782
|
+
if (typeof arg === `string`) return arg;
|
|
783
|
+
try {
|
|
784
|
+
return JSON.stringify(arg);
|
|
785
|
+
} catch {
|
|
786
|
+
return String(arg);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
function write(level, args) {
|
|
790
|
+
const line = args.map(serializeArg).join(` `);
|
|
791
|
+
const formatted = `[${level}] ${line}`;
|
|
792
|
+
if (level === `error`) console.error(formatted);
|
|
793
|
+
else if (level === `warn`) console.warn(formatted);
|
|
794
|
+
else console.info(formatted);
|
|
795
|
+
appendLogLine(formatted).catch(() => void 0);
|
|
796
|
+
}
|
|
797
|
+
const serverLog = {
|
|
798
|
+
info(...args) {
|
|
799
|
+
write(`info`, args);
|
|
800
|
+
},
|
|
801
|
+
warn(...args) {
|
|
802
|
+
write(`warn`, args);
|
|
803
|
+
},
|
|
804
|
+
error(...args) {
|
|
805
|
+
write(`error`, args);
|
|
806
|
+
},
|
|
807
|
+
event(obj, msg) {
|
|
808
|
+
write(`info`, [msg, obj]);
|
|
809
|
+
}
|
|
810
|
+
};
|
|
811
|
+
|
|
752
812
|
//#endregion
|
|
753
813
|
//#region src/path-encoding.ts
|
|
754
814
|
const MAX_ENCODED_LENGTH = 200;
|
|
@@ -759,10 +819,10 @@ const MAX_ENCODED_LENGTH = 200;
|
|
|
759
819
|
* @example
|
|
760
820
|
* encodeStreamPath("/stream/users:created") → "L3N0cmVhbS91c2VyczpjcmVhdGVk"
|
|
761
821
|
*/
|
|
762
|
-
function encodeStreamPath(path$
|
|
763
|
-
const base64 = Buffer.from(path$
|
|
822
|
+
function encodeStreamPath(path$1) {
|
|
823
|
+
const base64 = Buffer.from(path$1, `utf-8`).toString(`base64`).replace(/\+/g, `-`).replace(/\//g, `_`).replace(/=/g, ``);
|
|
764
824
|
if (base64.length > MAX_ENCODED_LENGTH) {
|
|
765
|
-
const hash = createHash(`sha256`).update(path$
|
|
825
|
+
const hash = createHash(`sha256`).update(path$1).digest(`hex`).slice(0, 16);
|
|
766
826
|
return `${base64.slice(0, 180)}~${hash}`;
|
|
767
827
|
}
|
|
768
828
|
return base64;
|
|
@@ -785,82 +845,6 @@ function decodeStreamPath(encoded) {
|
|
|
785
845
|
return Buffer.from(padded, `base64`).toString(`utf-8`);
|
|
786
846
|
}
|
|
787
847
|
|
|
788
|
-
//#endregion
|
|
789
|
-
//#region src/file-manager.ts
|
|
790
|
-
var StreamFileManager = class {
|
|
791
|
-
constructor(streamsDir) {
|
|
792
|
-
this.streamsDir = streamsDir;
|
|
793
|
-
}
|
|
794
|
-
/**
|
|
795
|
-
* Create a directory for a new stream and initialize the first segment file.
|
|
796
|
-
* Returns the absolute path to the stream directory.
|
|
797
|
-
*/
|
|
798
|
-
async createStreamDirectory(streamPath) {
|
|
799
|
-
const encoded = encodeStreamPath(streamPath);
|
|
800
|
-
const dir = path$1.join(this.streamsDir, encoded);
|
|
801
|
-
await fs$1.mkdir(dir, { recursive: true });
|
|
802
|
-
const segmentPath = path$1.join(dir, `segment_00000.log`);
|
|
803
|
-
await fs$1.writeFile(segmentPath, ``);
|
|
804
|
-
return dir;
|
|
805
|
-
}
|
|
806
|
-
/**
|
|
807
|
-
* Delete a stream directory and all its contents.
|
|
808
|
-
*/
|
|
809
|
-
async deleteStreamDirectory(streamPath) {
|
|
810
|
-
const encoded = encodeStreamPath(streamPath);
|
|
811
|
-
const dir = path$1.join(this.streamsDir, encoded);
|
|
812
|
-
await fs$1.rm(dir, {
|
|
813
|
-
recursive: true,
|
|
814
|
-
force: true
|
|
815
|
-
});
|
|
816
|
-
}
|
|
817
|
-
/**
|
|
818
|
-
* Delete a directory by its exact name (used for unique directory names).
|
|
819
|
-
*/
|
|
820
|
-
async deleteDirectoryByName(directoryName) {
|
|
821
|
-
const dir = path$1.join(this.streamsDir, directoryName);
|
|
822
|
-
await fs$1.rm(dir, {
|
|
823
|
-
recursive: true,
|
|
824
|
-
force: true
|
|
825
|
-
});
|
|
826
|
-
}
|
|
827
|
-
/**
|
|
828
|
-
* Get the absolute path to a stream's directory.
|
|
829
|
-
* Returns null if the directory doesn't exist.
|
|
830
|
-
*/
|
|
831
|
-
async getStreamDirectory(streamPath) {
|
|
832
|
-
const encoded = encodeStreamPath(streamPath);
|
|
833
|
-
const dir = path$1.join(this.streamsDir, encoded);
|
|
834
|
-
try {
|
|
835
|
-
await fs$1.access(dir);
|
|
836
|
-
return dir;
|
|
837
|
-
} catch {
|
|
838
|
-
return null;
|
|
839
|
-
}
|
|
840
|
-
}
|
|
841
|
-
/**
|
|
842
|
-
* List all stream paths by scanning the streams directory.
|
|
843
|
-
*/
|
|
844
|
-
async listStreamPaths() {
|
|
845
|
-
try {
|
|
846
|
-
const entries = await fs$1.readdir(this.streamsDir, { withFileTypes: true });
|
|
847
|
-
return entries.filter((e) => e.isDirectory()).map((e) => decodeStreamPath(e.name));
|
|
848
|
-
} catch {
|
|
849
|
-
return [];
|
|
850
|
-
}
|
|
851
|
-
}
|
|
852
|
-
/**
|
|
853
|
-
* Get the path to a segment file within a stream directory.
|
|
854
|
-
*
|
|
855
|
-
* @param streamDir - Absolute path to the stream directory
|
|
856
|
-
* @param index - Segment index (0-based)
|
|
857
|
-
*/
|
|
858
|
-
getSegmentPath(streamDir, index) {
|
|
859
|
-
const paddedIndex = String(index).padStart(5, `0`);
|
|
860
|
-
return path$1.join(streamDir, `segment_${paddedIndex}.log`);
|
|
861
|
-
}
|
|
862
|
-
};
|
|
863
|
-
|
|
864
848
|
//#endregion
|
|
865
849
|
//#region src/file-store.ts
|
|
866
850
|
var FileHandlePool = class {
|
|
@@ -868,7 +852,7 @@ var FileHandlePool = class {
|
|
|
868
852
|
constructor(maxSize) {
|
|
869
853
|
this.cache = new SieveCache(maxSize, { evictHook: (_key, handle) => {
|
|
870
854
|
this.closeHandle(handle).catch((err) => {
|
|
871
|
-
|
|
855
|
+
serverLog.error(`[FileHandlePool] Error closing evicted handle:`, err);
|
|
872
856
|
});
|
|
873
857
|
} });
|
|
874
858
|
}
|
|
@@ -876,41 +860,70 @@ var FileHandlePool = class {
|
|
|
876
860
|
let handle = this.cache.get(filePath);
|
|
877
861
|
if (!handle) {
|
|
878
862
|
const stream = fs.createWriteStream(filePath, { flags: `a` });
|
|
879
|
-
handle = {
|
|
863
|
+
handle = {
|
|
864
|
+
stream,
|
|
865
|
+
syncLeader: null
|
|
866
|
+
};
|
|
880
867
|
this.cache.set(filePath, handle);
|
|
881
868
|
}
|
|
882
869
|
return handle.stream;
|
|
883
870
|
}
|
|
884
871
|
/**
|
|
872
|
+
* Open a write stream eagerly so the first write does not pay the lazy
|
|
873
|
+
* `open()` stall. Resolves once the underlying fd is ready.
|
|
874
|
+
*/
|
|
875
|
+
async openWriteStream(filePath) {
|
|
876
|
+
const stream = this.getWriteStream(filePath);
|
|
877
|
+
const fd = stream.fd;
|
|
878
|
+
if (typeof fd === `number`) return stream;
|
|
879
|
+
await new Promise((resolve, reject) => {
|
|
880
|
+
stream.once(`open`, () => resolve());
|
|
881
|
+
stream.once(`error`, (err) => reject(err));
|
|
882
|
+
});
|
|
883
|
+
return stream;
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
885
886
|
* Flush a specific file to disk immediately.
|
|
886
|
-
*
|
|
887
|
+
* Concurrent callers on the same fd share one in-flight fdatasync: the
|
|
888
|
+
* first caller issues the syscall, later arrivals during that window wait
|
|
889
|
+
* for it to finish and then issue a fresh syscall (because their writes
|
|
890
|
+
* may have landed after the in-flight syscall started). This preserves
|
|
891
|
+
* durability without adding scheduling latency.
|
|
887
892
|
*/
|
|
888
|
-
|
|
893
|
+
fsyncFile(filePath) {
|
|
889
894
|
const handle = this.cache.get(filePath);
|
|
890
|
-
if (!handle) return;
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
else resolve();
|
|
899
|
-
});
|
|
900
|
-
};
|
|
901
|
-
const onError = (err) => {
|
|
902
|
-
handle.stream.off(`open`, onOpen);
|
|
903
|
-
reject(err);
|
|
904
|
-
};
|
|
905
|
-
handle.stream.once(`open`, onOpen);
|
|
906
|
-
handle.stream.once(`error`, onError);
|
|
907
|
-
return;
|
|
908
|
-
}
|
|
909
|
-
fs.fdatasync(fd, (err) => {
|
|
910
|
-
if (err) reject(err);
|
|
911
|
-
else resolve();
|
|
912
|
-
});
|
|
895
|
+
if (!handle) return Promise.reject(new Error(`[FileHandlePool] Cannot fsync: handle not found for ${filePath}`));
|
|
896
|
+
const existing = handle.syncLeader;
|
|
897
|
+
if (existing && existing.scheduled) return existing.promise;
|
|
898
|
+
let resolveFn;
|
|
899
|
+
let rejectFn;
|
|
900
|
+
const promise = new Promise((res, rej) => {
|
|
901
|
+
resolveFn = res;
|
|
902
|
+
rejectFn = rej;
|
|
913
903
|
});
|
|
904
|
+
const leader = {
|
|
905
|
+
promise,
|
|
906
|
+
scheduled: true
|
|
907
|
+
};
|
|
908
|
+
handle.syncLeader = leader;
|
|
909
|
+
const runSyscall = (fd$1) => {
|
|
910
|
+
leader.scheduled = false;
|
|
911
|
+
fs.fdatasync(fd$1, (err) => {
|
|
912
|
+
if (handle.syncLeader === leader) handle.syncLeader = null;
|
|
913
|
+
if (err) rejectFn(err);
|
|
914
|
+
else resolveFn();
|
|
915
|
+
});
|
|
916
|
+
};
|
|
917
|
+
const fd = handle.stream.fd;
|
|
918
|
+
if (typeof fd === `number`) runSyscall(fd);
|
|
919
|
+
else {
|
|
920
|
+
handle.stream.once(`open`, (openedFd) => runSyscall(openedFd));
|
|
921
|
+
handle.stream.once(`error`, (err) => {
|
|
922
|
+
if (handle.syncLeader === leader) handle.syncLeader = null;
|
|
923
|
+
rejectFn(err);
|
|
924
|
+
});
|
|
925
|
+
}
|
|
926
|
+
return promise;
|
|
914
927
|
}
|
|
915
928
|
async closeAll() {
|
|
916
929
|
const promises = [];
|
|
@@ -946,13 +959,15 @@ function generateUniqueDirectoryName(streamPath) {
|
|
|
946
959
|
const random = randomBytes(4).toString(`hex`);
|
|
947
960
|
return `${encoded}~${timestamp}~${random}`;
|
|
948
961
|
}
|
|
962
|
+
function segmentFile(dataDir, dirName) {
|
|
963
|
+
return path.join(dataDir, `streams`, `${dirName}.log`);
|
|
964
|
+
}
|
|
949
965
|
/**
|
|
950
966
|
* File-backed implementation of StreamStore.
|
|
951
967
|
* Maintains the same interface as the in-memory StreamStore for drop-in compatibility.
|
|
952
968
|
*/
|
|
953
969
|
var FileBackedStreamStore = class {
|
|
954
970
|
db;
|
|
955
|
-
fileManager;
|
|
956
971
|
fileHandlePool;
|
|
957
972
|
pendingLongPolls = [];
|
|
958
973
|
dataDir;
|
|
@@ -972,9 +987,12 @@ var FileBackedStreamStore = class {
|
|
|
972
987
|
this.dataDir = options.dataDir;
|
|
973
988
|
this.db = open({
|
|
974
989
|
path: path.join(this.dataDir, `metadata.lmdb`),
|
|
975
|
-
compression: true
|
|
990
|
+
compression: true,
|
|
991
|
+
noMemInit: true,
|
|
992
|
+
cache: true,
|
|
993
|
+
sharedStructuresKey: Symbol.for(`structures`)
|
|
976
994
|
});
|
|
977
|
-
|
|
995
|
+
fs.mkdirSync(path.join(this.dataDir, `streams`), { recursive: true });
|
|
978
996
|
const maxFileHandles = options.maxFileHandles ?? 100;
|
|
979
997
|
this.fileHandlePool = new FileHandlePool(maxFileHandles);
|
|
980
998
|
this.recover();
|
|
@@ -984,7 +1002,7 @@ var FileBackedStreamStore = class {
|
|
|
984
1002
|
* Validates that LMDB metadata matches actual file contents and reconciles any mismatches.
|
|
985
1003
|
*/
|
|
986
1004
|
recover() {
|
|
987
|
-
|
|
1005
|
+
serverLog.info(`[FileBackedStreamStore] Starting recovery...`);
|
|
988
1006
|
let recovered = 0;
|
|
989
1007
|
let reconciled = 0;
|
|
990
1008
|
let errors = 0;
|
|
@@ -997,9 +1015,9 @@ var FileBackedStreamStore = class {
|
|
|
997
1015
|
if (typeof key !== `string`) continue;
|
|
998
1016
|
const streamMeta = value;
|
|
999
1017
|
const streamPath = key.replace(`stream:`, ``);
|
|
1000
|
-
const segmentPath =
|
|
1018
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1001
1019
|
if (!fs.existsSync(segmentPath)) {
|
|
1002
|
-
|
|
1020
|
+
serverLog.warn(`[FileBackedStreamStore] Recovery: Stream file missing for ${streamPath}, removing from LMDB`);
|
|
1003
1021
|
this.db.removeSync(key);
|
|
1004
1022
|
errors++;
|
|
1005
1023
|
continue;
|
|
@@ -1013,7 +1031,7 @@ var FileBackedStreamStore = class {
|
|
|
1013
1031
|
trueOffset = `${String(0).padStart(16, `0`)}_${String(logicalBytes).padStart(16, `0`)}`;
|
|
1014
1032
|
} else trueOffset = physicalOffset;
|
|
1015
1033
|
if (trueOffset !== streamMeta.currentOffset) {
|
|
1016
|
-
|
|
1034
|
+
serverLog.warn(`[FileBackedStreamStore] Recovery: Offset mismatch for ${streamPath}: LMDB says ${streamMeta.currentOffset}, file says ${trueOffset}. Reconciling to file.`);
|
|
1017
1035
|
const reconciledMeta = {
|
|
1018
1036
|
...streamMeta,
|
|
1019
1037
|
currentOffset: trueOffset
|
|
@@ -1023,10 +1041,10 @@ var FileBackedStreamStore = class {
|
|
|
1023
1041
|
}
|
|
1024
1042
|
recovered++;
|
|
1025
1043
|
} catch (err) {
|
|
1026
|
-
|
|
1044
|
+
serverLog.error(`[FileBackedStreamStore] Error recovering stream:`, err);
|
|
1027
1045
|
errors++;
|
|
1028
1046
|
}
|
|
1029
|
-
|
|
1047
|
+
serverLog.info(`[FileBackedStreamStore] Recovery complete: ${recovered} streams, ${reconciled} reconciled, ${errors} errors`);
|
|
1030
1048
|
}
|
|
1031
1049
|
/**
|
|
1032
1050
|
* Scan a segment file to compute the true last offset.
|
|
@@ -1036,19 +1054,16 @@ var FileBackedStreamStore = class {
|
|
|
1036
1054
|
try {
|
|
1037
1055
|
const fileContent = fs.readFileSync(segmentPath);
|
|
1038
1056
|
let filePos = 0;
|
|
1039
|
-
let currentDataOffset = 0;
|
|
1040
1057
|
while (filePos < fileContent.length) {
|
|
1041
1058
|
if (filePos + 4 > fileContent.length) break;
|
|
1042
1059
|
const messageLength = fileContent.readUInt32BE(filePos);
|
|
1043
|
-
filePos
|
|
1044
|
-
if (
|
|
1045
|
-
filePos
|
|
1046
|
-
if (filePos < fileContent.length) filePos += 1;
|
|
1047
|
-
currentDataOffset += messageLength;
|
|
1060
|
+
const frameEnd = filePos + 4 + messageLength + 1;
|
|
1061
|
+
if (frameEnd > fileContent.length) break;
|
|
1062
|
+
filePos = frameEnd;
|
|
1048
1063
|
}
|
|
1049
|
-
return `0000000000000000_${String(
|
|
1064
|
+
return `0000000000000000_${String(filePos).padStart(16, `0`)}`;
|
|
1050
1065
|
} catch (err) {
|
|
1051
|
-
|
|
1066
|
+
serverLog.error(`[FileBackedStreamStore] Error scanning file ${segmentPath}:`, err);
|
|
1052
1067
|
return `0000000000000000_0000000000000000`;
|
|
1053
1068
|
}
|
|
1054
1069
|
}
|
|
@@ -1316,6 +1331,7 @@ var FileBackedStreamStore = class {
|
|
|
1316
1331
|
effectiveTtlSeconds = resolved.ttlSeconds;
|
|
1317
1332
|
}
|
|
1318
1333
|
const key = `stream:${streamPath}`;
|
|
1334
|
+
const t0 = performance.now();
|
|
1319
1335
|
const streamMeta = {
|
|
1320
1336
|
path: streamPath,
|
|
1321
1337
|
contentType,
|
|
@@ -1333,11 +1349,10 @@ var FileBackedStreamStore = class {
|
|
|
1333
1349
|
forkOffset: isFork ? forkOffset : void 0,
|
|
1334
1350
|
refCount: 0
|
|
1335
1351
|
};
|
|
1336
|
-
const
|
|
1352
|
+
const tAfterMeta = performance.now();
|
|
1353
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1337
1354
|
try {
|
|
1338
|
-
|
|
1339
|
-
const segmentPath = path.join(streamDir, `segment_00000.log`);
|
|
1340
|
-
fs.writeFileSync(segmentPath, ``);
|
|
1355
|
+
await this.db.put(key, streamMeta);
|
|
1341
1356
|
} catch (err) {
|
|
1342
1357
|
if (isFork && sourceMeta) {
|
|
1343
1358
|
const sourceKey = `stream:${options.forkedFrom}`;
|
|
@@ -1350,10 +1365,18 @@ var FileBackedStreamStore = class {
|
|
|
1350
1365
|
this.db.putSync(sourceKey, updatedSource);
|
|
1351
1366
|
}
|
|
1352
1367
|
}
|
|
1353
|
-
|
|
1368
|
+
serverLog.error(`[FileBackedStreamStore] Error creating stream (LMDB put):`, err);
|
|
1369
|
+
throw err;
|
|
1370
|
+
}
|
|
1371
|
+
const tAfterLmdb = performance.now();
|
|
1372
|
+
try {
|
|
1373
|
+
await this.fileHandlePool.openWriteStream(segmentPath);
|
|
1374
|
+
} catch (err) {
|
|
1375
|
+
this.db.removeSync(key);
|
|
1376
|
+
serverLog.error(`[FileBackedStreamStore] Error creating stream (file open):`, err);
|
|
1354
1377
|
throw err;
|
|
1355
1378
|
}
|
|
1356
|
-
|
|
1379
|
+
const tAfterOpen = performance.now();
|
|
1357
1380
|
if (options.initialData && options.initialData.length > 0) try {
|
|
1358
1381
|
await this.append(streamPath, options.initialData, {
|
|
1359
1382
|
contentType: options.contentType,
|
|
@@ -1373,12 +1396,24 @@ var FileBackedStreamStore = class {
|
|
|
1373
1396
|
}
|
|
1374
1397
|
throw err;
|
|
1375
1398
|
}
|
|
1399
|
+
const tAfterAppend = performance.now();
|
|
1376
1400
|
if (options.closed) {
|
|
1377
1401
|
const updatedMeta = this.db.get(key);
|
|
1378
1402
|
updatedMeta.closed = true;
|
|
1379
|
-
this.db.
|
|
1403
|
+
await this.db.put(key, updatedMeta);
|
|
1380
1404
|
}
|
|
1381
1405
|
const updated = this.db.get(key);
|
|
1406
|
+
const totalMs = performance.now() - t0;
|
|
1407
|
+
if (totalMs > 50) serverLog.event({
|
|
1408
|
+
event: `store.create`,
|
|
1409
|
+
path: streamPath,
|
|
1410
|
+
totalMs: +totalMs.toFixed(2),
|
|
1411
|
+
metaMs: +(tAfterMeta - t0).toFixed(2),
|
|
1412
|
+
lmdbMs: +(tAfterLmdb - tAfterMeta).toFixed(2),
|
|
1413
|
+
openMs: +(tAfterOpen - tAfterLmdb).toFixed(2),
|
|
1414
|
+
appendMs: +(tAfterAppend - tAfterOpen).toFixed(2),
|
|
1415
|
+
initBytes: options.initialData?.length ?? 0
|
|
1416
|
+
}, `store.create slow`);
|
|
1382
1417
|
return this.streamMetaToStream(updated);
|
|
1383
1418
|
}
|
|
1384
1419
|
get(streamPath) {
|
|
@@ -1419,13 +1454,10 @@ var FileBackedStreamStore = class {
|
|
|
1419
1454
|
if (!streamMeta) return;
|
|
1420
1455
|
const forkedFrom = streamMeta.forkedFrom;
|
|
1421
1456
|
this.cancelLongPollsForStream(streamPath);
|
|
1422
|
-
const segmentPath =
|
|
1423
|
-
this.fileHandlePool.closeFileHandle(segmentPath).catch((err) => {
|
|
1424
|
-
console.error(`[FileBackedStreamStore] Error closing file handle:`, err);
|
|
1425
|
-
});
|
|
1457
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1426
1458
|
this.db.removeSync(key);
|
|
1427
|
-
this.
|
|
1428
|
-
|
|
1459
|
+
this.fileHandlePool.closeFileHandle(segmentPath).then(() => fs.promises.unlink(segmentPath)).catch((err) => {
|
|
1460
|
+
serverLog.error(`[FileBackedStreamStore] Error cleaning up stream file:`, err);
|
|
1429
1461
|
});
|
|
1430
1462
|
if (forkedFrom) {
|
|
1431
1463
|
const parentKey = `stream:${forkedFrom}`;
|
|
@@ -1496,10 +1528,11 @@ var FileBackedStreamStore = class {
|
|
|
1496
1528
|
const parts = streamMeta.currentOffset.split(`_`).map(Number);
|
|
1497
1529
|
const readSeq = parts[0];
|
|
1498
1530
|
const byteOffset = parts[1];
|
|
1499
|
-
const
|
|
1531
|
+
const FRAME_OVERHEAD = 5;
|
|
1532
|
+
const newByteOffset = byteOffset + FRAME_OVERHEAD + processedData.length;
|
|
1500
1533
|
const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
1501
|
-
const
|
|
1502
|
-
const
|
|
1534
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1535
|
+
const tAppendStart = performance.now();
|
|
1503
1536
|
const stream = this.fileHandlePool.getWriteStream(segmentPath);
|
|
1504
1537
|
const lengthBuf = Buffer.allocUnsafe(4);
|
|
1505
1538
|
lengthBuf.writeUInt32BE(processedData.length, 0);
|
|
@@ -1514,12 +1547,14 @@ var FileBackedStreamStore = class {
|
|
|
1514
1547
|
else resolve();
|
|
1515
1548
|
});
|
|
1516
1549
|
});
|
|
1550
|
+
const tAfterWrite = performance.now();
|
|
1517
1551
|
const message = {
|
|
1518
1552
|
data: processedData,
|
|
1519
1553
|
offset: newOffset,
|
|
1520
1554
|
timestamp: Date.now()
|
|
1521
1555
|
};
|
|
1522
1556
|
await this.fileHandlePool.fsyncFile(segmentPath);
|
|
1557
|
+
const tAfterFsync = performance.now();
|
|
1523
1558
|
const updatedProducers = { ...streamMeta.producers };
|
|
1524
1559
|
if (producerResult && producerResult.status === `accepted`) updatedProducers[producerResult.producerId] = producerResult.proposedState;
|
|
1525
1560
|
let closedBy = void 0;
|
|
@@ -1538,7 +1573,19 @@ var FileBackedStreamStore = class {
|
|
|
1538
1573
|
closedBy: closedBy ?? streamMeta.closedBy
|
|
1539
1574
|
};
|
|
1540
1575
|
const key = `stream:${streamPath}`;
|
|
1541
|
-
this.db.
|
|
1576
|
+
await this.db.put(key, updatedMeta);
|
|
1577
|
+
const tAfterLmdb = performance.now();
|
|
1578
|
+
const appendTotal = tAfterLmdb - tAppendStart;
|
|
1579
|
+
if (appendTotal > 50) serverLog.event({
|
|
1580
|
+
event: `store.append`,
|
|
1581
|
+
path: streamPath,
|
|
1582
|
+
totalMs: +appendTotal.toFixed(2),
|
|
1583
|
+
writeMs: +(tAfterWrite - tAppendStart).toFixed(2),
|
|
1584
|
+
fsyncMs: +(tAfterFsync - tAfterWrite).toFixed(2),
|
|
1585
|
+
lmdbMs: +(tAfterLmdb - tAfterFsync).toFixed(2),
|
|
1586
|
+
bytes: processedData.length,
|
|
1587
|
+
isInitial: options.isInitialCreate ?? false
|
|
1588
|
+
}, `store.append slow`);
|
|
1542
1589
|
this.notifyLongPolls(streamPath);
|
|
1543
1590
|
if (options.close) this.notifyLongPollsClosed(streamPath);
|
|
1544
1591
|
if (producerResult || options.close) return {
|
|
@@ -1631,7 +1678,7 @@ var FileBackedStreamStore = class {
|
|
|
1631
1678
|
},
|
|
1632
1679
|
producers: updatedProducers
|
|
1633
1680
|
};
|
|
1634
|
-
this.db.
|
|
1681
|
+
await this.db.put(key, updatedMeta);
|
|
1635
1682
|
this.notifyLongPollsClosed(streamPath);
|
|
1636
1683
|
return {
|
|
1637
1684
|
finalOffset: streamMeta.currentOffset,
|
|
@@ -1665,7 +1712,7 @@ var FileBackedStreamStore = class {
|
|
|
1665
1712
|
const messageData = fileContent.subarray(filePos, filePos + messageLength);
|
|
1666
1713
|
filePos += messageLength;
|
|
1667
1714
|
filePos += 1;
|
|
1668
|
-
physicalDataOffset += messageLength;
|
|
1715
|
+
physicalDataOffset += messageLength + 5;
|
|
1669
1716
|
const logicalOffset = baseByteOffset + physicalDataOffset;
|
|
1670
1717
|
if (capByte !== void 0 && logicalOffset > capByte) break;
|
|
1671
1718
|
if (logicalOffset > startByte) messages.push({
|
|
@@ -1675,7 +1722,7 @@ var FileBackedStreamStore = class {
|
|
|
1675
1722
|
});
|
|
1676
1723
|
}
|
|
1677
1724
|
} catch (err) {
|
|
1678
|
-
|
|
1725
|
+
serverLog.error(`[FileBackedStreamStore] Error reading segment file:`, err);
|
|
1679
1726
|
}
|
|
1680
1727
|
return messages;
|
|
1681
1728
|
}
|
|
@@ -1697,7 +1744,7 @@ var FileBackedStreamStore = class {
|
|
|
1697
1744
|
messages.push(...inherited);
|
|
1698
1745
|
}
|
|
1699
1746
|
}
|
|
1700
|
-
const segmentPath =
|
|
1747
|
+
const segmentPath = segmentFile(this.dataDir, sourceMeta.directoryName);
|
|
1701
1748
|
const sourceBaseByte = sourceMeta.forkOffset ? Number(sourceMeta.forkOffset.split(`_`)[1] ?? 0) : 0;
|
|
1702
1749
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, sourceBaseByte, capByte);
|
|
1703
1750
|
messages.push(...ownMessages);
|
|
@@ -1724,11 +1771,11 @@ var FileBackedStreamStore = class {
|
|
|
1724
1771
|
const inherited = this.readForkedMessages(streamMeta.forkedFrom, startByte, forkByte);
|
|
1725
1772
|
messages.push(...inherited);
|
|
1726
1773
|
}
|
|
1727
|
-
const segmentPath =
|
|
1774
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1728
1775
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, forkByte);
|
|
1729
1776
|
messages.push(...ownMessages);
|
|
1730
1777
|
} else {
|
|
1731
|
-
const segmentPath =
|
|
1778
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1732
1779
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, 0);
|
|
1733
1780
|
messages.push(...ownMessages);
|
|
1734
1781
|
}
|
|
@@ -1799,6 +1846,7 @@ var FileBackedStreamStore = class {
|
|
|
1799
1846
|
formatResponse(streamPath, messages) {
|
|
1800
1847
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1801
1848
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1849
|
+
if (normalizeContentType(streamMeta.contentType) === `application/json`) return formatJsonMessages(messages);
|
|
1802
1850
|
const totalSize = messages.reduce((sum, m) => sum + m.data.length, 0);
|
|
1803
1851
|
const concatenated = new Uint8Array(totalSize);
|
|
1804
1852
|
let offset = 0;
|
|
@@ -1806,7 +1854,6 @@ var FileBackedStreamStore = class {
|
|
|
1806
1854
|
concatenated.set(msg.data, offset);
|
|
1807
1855
|
offset += msg.data.length;
|
|
1808
1856
|
}
|
|
1809
|
-
if (normalizeContentType(streamMeta.contentType) === `application/json`) return formatJsonResponse(concatenated);
|
|
1810
1857
|
return concatenated;
|
|
1811
1858
|
}
|
|
1812
1859
|
getCurrentOffset(streamPath) {
|
|
@@ -1826,7 +1873,7 @@ var FileBackedStreamStore = class {
|
|
|
1826
1873
|
const entries = Array.from(range);
|
|
1827
1874
|
for (const { key } of entries) this.db.removeSync(key);
|
|
1828
1875
|
this.fileHandlePool.closeAll().catch((err) => {
|
|
1829
|
-
|
|
1876
|
+
serverLog.error(`[FileBackedStreamStore] Error closing handles:`, err);
|
|
1830
1877
|
});
|
|
1831
1878
|
}
|
|
1832
1879
|
/**
|
|
@@ -1980,30 +2027,985 @@ function handleCursorCollision(currentCursor, previousCursor, options = {}) {
|
|
|
1980
2027
|
return generateResponseCursor(previousCursor, options);
|
|
1981
2028
|
}
|
|
1982
2029
|
|
|
2030
|
+
//#endregion
|
|
2031
|
+
//#region src/crypto.ts
|
|
2032
|
+
/**
|
|
2033
|
+
* Generate a unique wake ID.
|
|
2034
|
+
*/
|
|
2035
|
+
function generateWakeId() {
|
|
2036
|
+
return `w_${randomBytes(12).toString(`hex`)}`;
|
|
2037
|
+
}
|
|
2038
|
+
const WEBHOOK_KEYPAIR = generateKeyPairSync(`ed25519`);
|
|
2039
|
+
const WEBHOOK_PUBLIC_JWK = buildWebhookPublicJwk();
|
|
2040
|
+
function buildWebhookPublicJwk() {
|
|
2041
|
+
const exported = WEBHOOK_KEYPAIR.publicKey.export({ format: `jwk` });
|
|
2042
|
+
if (exported.kty !== `OKP` || exported.crv !== `Ed25519` || !exported.x) throw new Error(`Failed to export Ed25519 webhook signing key`);
|
|
2043
|
+
const thumbprintInput = JSON.stringify({
|
|
2044
|
+
crv: exported.crv,
|
|
2045
|
+
kty: exported.kty,
|
|
2046
|
+
x: exported.x
|
|
2047
|
+
});
|
|
2048
|
+
const kid = `ds_${createHash(`sha256`).update(thumbprintInput).digest(`base64url`)}`;
|
|
2049
|
+
return {
|
|
2050
|
+
kty: `OKP`,
|
|
2051
|
+
crv: `Ed25519`,
|
|
2052
|
+
x: exported.x,
|
|
2053
|
+
kid,
|
|
2054
|
+
use: `sig`,
|
|
2055
|
+
alg: `EdDSA`
|
|
2056
|
+
};
|
|
2057
|
+
}
|
|
2058
|
+
function getWebhookSigningKeyId() {
|
|
2059
|
+
return WEBHOOK_PUBLIC_JWK.kid;
|
|
2060
|
+
}
|
|
2061
|
+
function getWebhookJwks() {
|
|
2062
|
+
return { keys: [{ ...WEBHOOK_PUBLIC_JWK }] };
|
|
2063
|
+
}
|
|
2064
|
+
/**
|
|
2065
|
+
* Sign a webhook payload for the Webhook-Signature header.
|
|
2066
|
+
* Format: t=<timestamp>,kid=<key_id>,ed25519=<base64url_signature>
|
|
2067
|
+
*/
|
|
2068
|
+
function signWebhookPayload(body) {
|
|
2069
|
+
const timestamp = Math.floor(Date.now() / 1e3);
|
|
2070
|
+
const payload = `${timestamp}.${body}`;
|
|
2071
|
+
const signature = sign(null, Buffer.from(payload), WEBHOOK_KEYPAIR.privateKey).toString(`base64url`);
|
|
2072
|
+
return `t=${timestamp},kid=${WEBHOOK_PUBLIC_JWK.kid},ed25519=${signature}`;
|
|
2073
|
+
}
|
|
2074
|
+
const TOKEN_KEY = randomBytes(32);
|
|
2075
|
+
/**
|
|
2076
|
+
* Generate a signed callback token.
|
|
2077
|
+
* Token format: base64url(json_payload).base64url(hmac_signature)
|
|
2078
|
+
* Payload: { consumer_id, epoch, exp }
|
|
2079
|
+
*/
|
|
2080
|
+
function generateCallbackToken(consumerId, epoch) {
|
|
2081
|
+
const payload = {
|
|
2082
|
+
sub: consumerId,
|
|
2083
|
+
epoch,
|
|
2084
|
+
exp: Math.floor(Date.now() / 1e3) + 3600,
|
|
2085
|
+
jti: randomBytes(8).toString(`hex`)
|
|
2086
|
+
};
|
|
2087
|
+
const payloadStr = Buffer.from(JSON.stringify(payload)).toString(`base64url`);
|
|
2088
|
+
const sig = createHmac(`sha256`, TOKEN_KEY).update(payloadStr).digest(`base64url`);
|
|
2089
|
+
return `${payloadStr}.${sig}`;
|
|
2090
|
+
}
|
|
2091
|
+
/**
|
|
2092
|
+
* Validate a callback token. Returns the decoded payload or null.
|
|
2093
|
+
* On success, includes `exp` (unix seconds) so callers can decide
|
|
2094
|
+
* whether the token needs refreshing.
|
|
2095
|
+
*/
|
|
2096
|
+
function validateCallbackToken(token, consumerId) {
|
|
2097
|
+
const parts = token.split(`.`);
|
|
2098
|
+
if (parts.length !== 2) return {
|
|
2099
|
+
valid: false,
|
|
2100
|
+
code: `TOKEN_INVALID`
|
|
2101
|
+
};
|
|
2102
|
+
const [payloadStr, sig] = parts;
|
|
2103
|
+
const expectedSig = createHmac(`sha256`, TOKEN_KEY).update(payloadStr).digest(`base64url`);
|
|
2104
|
+
try {
|
|
2105
|
+
if (!timingSafeEqual(Buffer.from(sig), Buffer.from(expectedSig))) return {
|
|
2106
|
+
valid: false,
|
|
2107
|
+
code: `TOKEN_INVALID`
|
|
2108
|
+
};
|
|
2109
|
+
} catch {
|
|
2110
|
+
return {
|
|
2111
|
+
valid: false,
|
|
2112
|
+
code: `TOKEN_INVALID`
|
|
2113
|
+
};
|
|
2114
|
+
}
|
|
2115
|
+
let payload;
|
|
2116
|
+
try {
|
|
2117
|
+
payload = JSON.parse(Buffer.from(payloadStr, `base64url`).toString());
|
|
2118
|
+
} catch {
|
|
2119
|
+
return {
|
|
2120
|
+
valid: false,
|
|
2121
|
+
code: `TOKEN_INVALID`
|
|
2122
|
+
};
|
|
2123
|
+
}
|
|
2124
|
+
if (payload.sub !== consumerId) return {
|
|
2125
|
+
valid: false,
|
|
2126
|
+
code: `TOKEN_INVALID`
|
|
2127
|
+
};
|
|
2128
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
2129
|
+
if (now > payload.exp) return {
|
|
2130
|
+
valid: false,
|
|
2131
|
+
code: `TOKEN_EXPIRED`
|
|
2132
|
+
};
|
|
2133
|
+
return {
|
|
2134
|
+
valid: true,
|
|
2135
|
+
exp: payload.exp,
|
|
2136
|
+
epoch: payload.epoch
|
|
2137
|
+
};
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
//#endregion
|
|
2141
|
+
//#region src/glob.ts
|
|
2142
|
+
/**
|
|
2143
|
+
* Glob pattern matching for webhook subscription patterns.
|
|
2144
|
+
*
|
|
2145
|
+
* Supports:
|
|
2146
|
+
* - `*` matches exactly one path segment
|
|
2147
|
+
* - `**` matches zero or more path segments (recursive)
|
|
2148
|
+
* - Literal segments match exactly
|
|
2149
|
+
*/
|
|
2150
|
+
/**
|
|
2151
|
+
* Match a stream path against a glob pattern.
|
|
2152
|
+
*/
|
|
2153
|
+
function globMatch(pattern, path$1) {
|
|
2154
|
+
const patternParts = splitPath(pattern);
|
|
2155
|
+
const pathParts = splitPath(path$1);
|
|
2156
|
+
return matchParts(patternParts, 0, pathParts, 0);
|
|
2157
|
+
}
|
|
2158
|
+
function splitPath(p) {
|
|
2159
|
+
return p.replace(/^\/+/, ``).replace(/\/+$/, ``).split(`/`).filter((s) => s.length > 0);
|
|
2160
|
+
}
|
|
2161
|
+
function matchParts(pattern, pi, path$1, si) {
|
|
2162
|
+
while (pi < pattern.length && si < path$1.length) {
|
|
2163
|
+
const seg = pattern[pi];
|
|
2164
|
+
if (seg === `**`) {
|
|
2165
|
+
for (let i = si; i <= path$1.length; i++) if (matchParts(pattern, pi + 1, path$1, i)) return true;
|
|
2166
|
+
return false;
|
|
2167
|
+
}
|
|
2168
|
+
if (seg === `*`) {
|
|
2169
|
+
pi++;
|
|
2170
|
+
si++;
|
|
2171
|
+
continue;
|
|
2172
|
+
}
|
|
2173
|
+
const decodedSeg = seg.replace(/%2[Aa]/g, `*`);
|
|
2174
|
+
if (decodedSeg !== path$1[si]) return false;
|
|
2175
|
+
pi++;
|
|
2176
|
+
si++;
|
|
2177
|
+
}
|
|
2178
|
+
while (pi < pattern.length && pattern[pi] === `**`) pi++;
|
|
2179
|
+
return pi === pattern.length && si === path$1.length;
|
|
2180
|
+
}
|
|
2181
|
+
|
|
2182
|
+
//#endregion
|
|
2183
|
+
//#region src/subscription-manager.ts
|
|
2184
|
+
const DEFAULT_LEASE_TTL_MS = 3e4;
|
|
2185
|
+
const MIN_LEASE_TTL_MS = 1e3;
|
|
2186
|
+
const MAX_LEASE_TTL_MS = 10 * 6e4;
|
|
2187
|
+
const ZERO_OFFSET = `0000000000000000_0000000000000000`;
|
|
2188
|
+
const BEFORE_FIRST_OFFSET = `-1`;
|
|
2189
|
+
const MAX_RETRY_DELAY_MS = 6e4;
|
|
2190
|
+
function compareOffsets(a, b) {
|
|
2191
|
+
if (a < b) return -1;
|
|
2192
|
+
if (a > b) return 1;
|
|
2193
|
+
return 0;
|
|
2194
|
+
}
|
|
2195
|
+
function normalizeRelativePath(path$1) {
|
|
2196
|
+
return path$1.replace(/^\/+/, ``).replace(/\/+$/, ``);
|
|
2197
|
+
}
|
|
2198
|
+
function toAbsoluteStreamPath(streamPath) {
|
|
2199
|
+
return `/v1/stream/${normalizeRelativePath(streamPath)}`;
|
|
2200
|
+
}
|
|
2201
|
+
function toStreamRelativePath(absolutePath) {
|
|
2202
|
+
const streamRoot = `/v1/stream/`;
|
|
2203
|
+
if (!absolutePath.startsWith(streamRoot)) return null;
|
|
2204
|
+
const path$1 = absolutePath.slice(streamRoot.length);
|
|
2205
|
+
if (path$1 === `__ds` || path$1.startsWith(`__ds/`)) return null;
|
|
2206
|
+
return path$1.length > 0 ? path$1 : null;
|
|
2207
|
+
}
|
|
2208
|
+
function stableConfigHash(input) {
|
|
2209
|
+
const canonical = {
|
|
2210
|
+
type: input.type,
|
|
2211
|
+
pattern: input.pattern,
|
|
2212
|
+
streams: [...new Set(input.streams)].sort(),
|
|
2213
|
+
webhook: input.webhook ? { url: input.webhook.url } : void 0,
|
|
2214
|
+
wake_stream: input.wake_stream,
|
|
2215
|
+
lease_ttl_ms: input.lease_ttl_ms,
|
|
2216
|
+
description: input.description
|
|
2217
|
+
};
|
|
2218
|
+
return createHash(`sha256`).update(JSON.stringify(canonical)).digest(`hex`);
|
|
2219
|
+
}
|
|
2220
|
+
function isPrivateOrLinkLocalIpv4(host) {
|
|
2221
|
+
const parts = host.split(`.`).map((part) => Number(part));
|
|
2222
|
+
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part))) return false;
|
|
2223
|
+
const [a, b] = parts;
|
|
2224
|
+
return a === 10 || a === 127 || a === 0 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a === 169 && b === 254;
|
|
2225
|
+
}
|
|
2226
|
+
function isLocalDevHost(host) {
|
|
2227
|
+
return host === `localhost` || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host);
|
|
2228
|
+
}
|
|
2229
|
+
function validateWebhookUrl(rawUrl) {
|
|
2230
|
+
let url;
|
|
2231
|
+
try {
|
|
2232
|
+
url = new URL(rawUrl);
|
|
2233
|
+
} catch {
|
|
2234
|
+
return {
|
|
2235
|
+
ok: false,
|
|
2236
|
+
message: `webhook.url must be a valid URL`
|
|
2237
|
+
};
|
|
2238
|
+
}
|
|
2239
|
+
const host = url.hostname.toLowerCase();
|
|
2240
|
+
if (url.protocol === `http:`) {
|
|
2241
|
+
if (isLocalDevHost(host)) return { ok: true };
|
|
2242
|
+
return {
|
|
2243
|
+
ok: false,
|
|
2244
|
+
message: `http webhook URLs are only allowed for localhost or 127.0.0.x`
|
|
2245
|
+
};
|
|
2246
|
+
}
|
|
2247
|
+
if (url.protocol !== `https:`) return {
|
|
2248
|
+
ok: false,
|
|
2249
|
+
message: `webhook.url must use https`
|
|
2250
|
+
};
|
|
2251
|
+
if (host === `localhost`) return {
|
|
2252
|
+
ok: false,
|
|
2253
|
+
message: `localhost webhook URLs must use http for dev`
|
|
2254
|
+
};
|
|
2255
|
+
if (isIP(host) === 4 && isPrivateOrLinkLocalIpv4(host)) return {
|
|
2256
|
+
ok: false,
|
|
2257
|
+
message: `webhook.url must not target private or link-local hosts`
|
|
2258
|
+
};
|
|
2259
|
+
if (isIP(host) === 6) return {
|
|
2260
|
+
ok: false,
|
|
2261
|
+
message: `IPv6 webhook hosts are not accepted by the reference server`
|
|
2262
|
+
};
|
|
2263
|
+
return { ok: true };
|
|
2264
|
+
}
|
|
2265
|
+
var SubscriptionManager = class {
|
|
2266
|
+
subscriptions = new Map();
|
|
2267
|
+
streamStore;
|
|
2268
|
+
callbackBaseUrl;
|
|
2269
|
+
webhooksEnabled;
|
|
2270
|
+
isShuttingDown = false;
|
|
2271
|
+
constructor(opts) {
|
|
2272
|
+
this.callbackBaseUrl = opts.callbackBaseUrl;
|
|
2273
|
+
this.streamStore = opts.streamStore;
|
|
2274
|
+
this.webhooksEnabled = opts.webhooksEnabled ?? true;
|
|
2275
|
+
}
|
|
2276
|
+
createOrConfirm(id, input) {
|
|
2277
|
+
const configHash = stableConfigHash(input);
|
|
2278
|
+
const existing = this.subscriptions.get(id);
|
|
2279
|
+
if (existing) {
|
|
2280
|
+
if (existing.config_hash !== configHash) return { error: {
|
|
2281
|
+
code: `SUBSCRIPTION_ALREADY_EXISTS`,
|
|
2282
|
+
message: `Subscription already exists with different configuration`
|
|
2283
|
+
} };
|
|
2284
|
+
return {
|
|
2285
|
+
subscription: existing,
|
|
2286
|
+
created: false
|
|
2287
|
+
};
|
|
2288
|
+
}
|
|
2289
|
+
if (input.type === `webhook`) {
|
|
2290
|
+
if (!this.webhooksEnabled) return { error: {
|
|
2291
|
+
code: `INVALID_REQUEST`,
|
|
2292
|
+
message: `webhook subscriptions are not enabled on this server`
|
|
2293
|
+
} };
|
|
2294
|
+
if (!input.webhook) return { error: {
|
|
2295
|
+
code: `INVALID_REQUEST`,
|
|
2296
|
+
message: `webhook subscriptions require webhook.url`
|
|
2297
|
+
} };
|
|
2298
|
+
const validation = validateWebhookUrl(input.webhook.url);
|
|
2299
|
+
if (!validation.ok) return { error: {
|
|
2300
|
+
code: `WEBHOOK_URL_REJECTED`,
|
|
2301
|
+
message: validation.message
|
|
2302
|
+
} };
|
|
2303
|
+
}
|
|
2304
|
+
if (input.type === `pull-wake` && !input.wake_stream) return { error: {
|
|
2305
|
+
code: `INVALID_REQUEST`,
|
|
2306
|
+
message: `pull-wake subscriptions require wake_stream`
|
|
2307
|
+
} };
|
|
2308
|
+
const subscription = {
|
|
2309
|
+
id,
|
|
2310
|
+
type: input.type,
|
|
2311
|
+
pattern: input.pattern,
|
|
2312
|
+
webhook: input.webhook ? { url: input.webhook.url } : void 0,
|
|
2313
|
+
wake_stream: input.wake_stream,
|
|
2314
|
+
lease_ttl_ms: input.lease_ttl_ms,
|
|
2315
|
+
description: input.description,
|
|
2316
|
+
created_at: new Date().toISOString(),
|
|
2317
|
+
status: `active`,
|
|
2318
|
+
config_hash: configHash,
|
|
2319
|
+
streams: new Map(),
|
|
2320
|
+
generation: 0,
|
|
2321
|
+
wake_id: null,
|
|
2322
|
+
wake_snapshot: new Map(),
|
|
2323
|
+
token: null,
|
|
2324
|
+
holder: null,
|
|
2325
|
+
lease_timer: null,
|
|
2326
|
+
retry_count: 0,
|
|
2327
|
+
retry_timer: null,
|
|
2328
|
+
next_attempt_at: null
|
|
2329
|
+
};
|
|
2330
|
+
for (const stream of input.streams) this.linkStream(subscription, stream, `explicit`, this.getTailOffset(stream));
|
|
2331
|
+
if (input.pattern) {
|
|
2332
|
+
for (const stream of this.listStreams()) if (globMatch(input.pattern, stream)) this.linkStream(subscription, stream, `glob`, this.getTailOffset(stream));
|
|
2333
|
+
}
|
|
2334
|
+
this.subscriptions.set(id, subscription);
|
|
2335
|
+
return {
|
|
2336
|
+
subscription,
|
|
2337
|
+
created: true
|
|
2338
|
+
};
|
|
2339
|
+
}
|
|
2340
|
+
get(id) {
|
|
2341
|
+
return this.subscriptions.get(id);
|
|
2342
|
+
}
|
|
2343
|
+
delete(id) {
|
|
2344
|
+
const subscription = this.subscriptions.get(id);
|
|
2345
|
+
if (!subscription) return false;
|
|
2346
|
+
this.clearLease(subscription);
|
|
2347
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2348
|
+
this.subscriptions.delete(id);
|
|
2349
|
+
return true;
|
|
2350
|
+
}
|
|
2351
|
+
addExplicitStreams(id, streams) {
|
|
2352
|
+
const subscription = this.get(id);
|
|
2353
|
+
if (!subscription) return false;
|
|
2354
|
+
for (const stream of streams) this.linkStream(subscription, stream, `explicit`, this.getTailOffset(stream));
|
|
2355
|
+
return true;
|
|
2356
|
+
}
|
|
2357
|
+
removeExplicitStream(id, streamPath) {
|
|
2358
|
+
const subscription = this.get(id);
|
|
2359
|
+
if (!subscription) return false;
|
|
2360
|
+
const normalized = normalizeRelativePath(streamPath);
|
|
2361
|
+
const link = subscription.streams.get(normalized);
|
|
2362
|
+
if (!link) return true;
|
|
2363
|
+
link.link_types.delete(`explicit`);
|
|
2364
|
+
if (link.link_types.size === 0) subscription.streams.delete(normalized);
|
|
2365
|
+
return true;
|
|
2366
|
+
}
|
|
2367
|
+
async onStreamAppend(absolutePath) {
|
|
2368
|
+
if (this.isShuttingDown) return;
|
|
2369
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2370
|
+
const relative = toStreamRelativePath(absolutePath);
|
|
2371
|
+
if (!relative) continue;
|
|
2372
|
+
if (subscription.pattern && globMatch(subscription.pattern, relative)) {
|
|
2373
|
+
const existing = subscription.streams.get(relative);
|
|
2374
|
+
this.linkStream(subscription, relative, `glob`, existing?.acked_offset ?? BEFORE_FIRST_OFFSET);
|
|
2375
|
+
}
|
|
2376
|
+
if (subscription.streams.has(relative)) await this.maybeWake(subscription, relative);
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
onStreamDeleted(absolutePath) {
|
|
2380
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2381
|
+
const relative = toStreamRelativePath(absolutePath);
|
|
2382
|
+
if (relative) subscription.streams.delete(relative);
|
|
2383
|
+
}
|
|
2384
|
+
}
|
|
2385
|
+
async handleWebhookCallback(id, token, request) {
|
|
2386
|
+
const subscription = this.get(id);
|
|
2387
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2388
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2389
|
+
if (fenced) return fenced;
|
|
2390
|
+
const ackError = this.applyAcks(subscription, request);
|
|
2391
|
+
if (ackError) return ackError;
|
|
2392
|
+
this.extendLease(subscription);
|
|
2393
|
+
let nextWake = false;
|
|
2394
|
+
if (request.done === true) {
|
|
2395
|
+
this.clearLease(subscription);
|
|
2396
|
+
subscription.token = null;
|
|
2397
|
+
subscription.holder = null;
|
|
2398
|
+
subscription.wake_id = null;
|
|
2399
|
+
subscription.wake_snapshot.clear();
|
|
2400
|
+
nextWake = await this.triggerNextWakeIfPending(subscription);
|
|
2401
|
+
}
|
|
2402
|
+
return {
|
|
2403
|
+
status: 200,
|
|
2404
|
+
body: {
|
|
2405
|
+
ok: true,
|
|
2406
|
+
next_wake: nextWake
|
|
2407
|
+
}
|
|
2408
|
+
};
|
|
2409
|
+
}
|
|
2410
|
+
async claim(id, worker) {
|
|
2411
|
+
const subscription = this.get(id);
|
|
2412
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2413
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2414
|
+
if (subscription.holder) return {
|
|
2415
|
+
status: 409,
|
|
2416
|
+
body: { error: {
|
|
2417
|
+
code: `ALREADY_CLAIMED`,
|
|
2418
|
+
current_holder: subscription.holder,
|
|
2419
|
+
generation: subscription.generation
|
|
2420
|
+
} }
|
|
2421
|
+
};
|
|
2422
|
+
if (!this.hasPendingWork(subscription)) return this.errorResponse(409, `NO_PENDING_WORK`, `Subscription has no pending work`);
|
|
2423
|
+
if (!subscription.wake_id) await this.createWake(subscription, this.firstPendingStream(subscription));
|
|
2424
|
+
subscription.holder = worker;
|
|
2425
|
+
subscription.token = generateCallbackToken(this.tokenSubject(subscription), subscription.generation);
|
|
2426
|
+
this.extendLease(subscription);
|
|
2427
|
+
return {
|
|
2428
|
+
status: 200,
|
|
2429
|
+
body: {
|
|
2430
|
+
wake_id: subscription.wake_id,
|
|
2431
|
+
generation: subscription.generation,
|
|
2432
|
+
token: subscription.token,
|
|
2433
|
+
streams: this.streamInfos(subscription),
|
|
2434
|
+
lease_ttl_ms: subscription.lease_ttl_ms
|
|
2435
|
+
}
|
|
2436
|
+
};
|
|
2437
|
+
}
|
|
2438
|
+
async ack(id, token, request) {
|
|
2439
|
+
const subscription = this.get(id);
|
|
2440
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2441
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2442
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2443
|
+
if (fenced) return fenced;
|
|
2444
|
+
const ackError = this.applyAcks(subscription, request);
|
|
2445
|
+
if (ackError) return ackError;
|
|
2446
|
+
this.extendLease(subscription);
|
|
2447
|
+
let nextWake = false;
|
|
2448
|
+
if (request.done === true) {
|
|
2449
|
+
this.clearLease(subscription);
|
|
2450
|
+
subscription.token = null;
|
|
2451
|
+
subscription.holder = null;
|
|
2452
|
+
subscription.wake_id = null;
|
|
2453
|
+
subscription.wake_snapshot.clear();
|
|
2454
|
+
nextWake = await this.triggerNextWakeIfPending(subscription);
|
|
2455
|
+
}
|
|
2456
|
+
return {
|
|
2457
|
+
status: 200,
|
|
2458
|
+
body: {
|
|
2459
|
+
ok: true,
|
|
2460
|
+
next_wake: nextWake
|
|
2461
|
+
}
|
|
2462
|
+
};
|
|
2463
|
+
}
|
|
2464
|
+
async release(id, token, request) {
|
|
2465
|
+
const subscription = this.get(id);
|
|
2466
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2467
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2468
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2469
|
+
if (fenced) return fenced;
|
|
2470
|
+
this.clearLease(subscription);
|
|
2471
|
+
subscription.token = null;
|
|
2472
|
+
subscription.holder = null;
|
|
2473
|
+
subscription.wake_id = null;
|
|
2474
|
+
subscription.wake_snapshot.clear();
|
|
2475
|
+
await this.triggerNextWakeIfPending(subscription);
|
|
2476
|
+
return { status: 204 };
|
|
2477
|
+
}
|
|
2478
|
+
serialize(subscription) {
|
|
2479
|
+
return {
|
|
2480
|
+
id: subscription.id,
|
|
2481
|
+
subscription_id: subscription.id,
|
|
2482
|
+
type: subscription.type,
|
|
2483
|
+
pattern: subscription.pattern,
|
|
2484
|
+
streams: this.streamInfos(subscription).map((stream) => ({
|
|
2485
|
+
path: stream.path,
|
|
2486
|
+
link_type: stream.link_type,
|
|
2487
|
+
acked_offset: stream.acked_offset
|
|
2488
|
+
})),
|
|
2489
|
+
webhook: subscription.webhook ? {
|
|
2490
|
+
url: subscription.webhook.url,
|
|
2491
|
+
signing: this.webhookSigningMetadata()
|
|
2492
|
+
} : void 0,
|
|
2493
|
+
wake_stream: subscription.wake_stream,
|
|
2494
|
+
lease_ttl_ms: subscription.lease_ttl_ms,
|
|
2495
|
+
created_at: subscription.created_at,
|
|
2496
|
+
status: subscription.status,
|
|
2497
|
+
description: subscription.description
|
|
2498
|
+
};
|
|
2499
|
+
}
|
|
2500
|
+
getWebhookJwks() {
|
|
2501
|
+
return getWebhookJwks();
|
|
2502
|
+
}
|
|
2503
|
+
shutdown() {
|
|
2504
|
+
this.isShuttingDown = true;
|
|
2505
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2506
|
+
this.clearLease(subscription);
|
|
2507
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2508
|
+
}
|
|
2509
|
+
this.subscriptions.clear();
|
|
2510
|
+
}
|
|
2511
|
+
async maybeWake(subscription, triggeredBy) {
|
|
2512
|
+
if (subscription.wake_id || subscription.holder) return;
|
|
2513
|
+
if (!this.hasPendingWork(subscription)) return;
|
|
2514
|
+
await this.createWake(subscription, triggeredBy);
|
|
2515
|
+
}
|
|
2516
|
+
async createWake(subscription, triggeredBy) {
|
|
2517
|
+
subscription.generation++;
|
|
2518
|
+
subscription.wake_id = generateWakeId();
|
|
2519
|
+
subscription.wake_snapshot = new Map(this.streamInfos(subscription).map((stream) => [stream.path, stream.tail_offset]));
|
|
2520
|
+
if (subscription.type === `webhook`) {
|
|
2521
|
+
subscription.token = generateCallbackToken(this.tokenSubject(subscription), subscription.generation);
|
|
2522
|
+
this.extendLease(subscription);
|
|
2523
|
+
this.deliverWebhook(subscription, [triggeredBy]);
|
|
2524
|
+
return;
|
|
2525
|
+
}
|
|
2526
|
+
await this.writePullWakeEvent(subscription, triggeredBy);
|
|
2527
|
+
}
|
|
2528
|
+
async deliverWebhook(subscription, triggeredBy) {
|
|
2529
|
+
if (!subscription.webhook || !subscription.wake_id || !subscription.token) return;
|
|
2530
|
+
const body = JSON.stringify({
|
|
2531
|
+
subscription_id: subscription.id,
|
|
2532
|
+
wake_id: subscription.wake_id,
|
|
2533
|
+
generation: subscription.generation,
|
|
2534
|
+
streams: this.streamInfos(subscription),
|
|
2535
|
+
callback_url: this.subscriptionActionUrl(subscription, `callback`),
|
|
2536
|
+
callback_token: subscription.token
|
|
2537
|
+
});
|
|
2538
|
+
const headers = {
|
|
2539
|
+
"content-type": `application/json`,
|
|
2540
|
+
"webhook-signature": signWebhookPayload(body)
|
|
2541
|
+
};
|
|
2542
|
+
try {
|
|
2543
|
+
const response = await fetch(subscription.webhook.url, {
|
|
2544
|
+
method: `POST`,
|
|
2545
|
+
headers,
|
|
2546
|
+
body
|
|
2547
|
+
});
|
|
2548
|
+
if (!response.ok) {
|
|
2549
|
+
this.scheduleWebhookRetry(subscription, triggeredBy);
|
|
2550
|
+
return;
|
|
2551
|
+
}
|
|
2552
|
+
subscription.status = `active`;
|
|
2553
|
+
subscription.retry_count = 0;
|
|
2554
|
+
subscription.next_attempt_at = null;
|
|
2555
|
+
let parsed = null;
|
|
2556
|
+
try {
|
|
2557
|
+
parsed = await response.json();
|
|
2558
|
+
} catch {
|
|
2559
|
+
parsed = null;
|
|
2560
|
+
}
|
|
2561
|
+
if (parsed?.done === true) {
|
|
2562
|
+
this.autoAckWakeSnapshot(subscription);
|
|
2563
|
+
this.clearLease(subscription);
|
|
2564
|
+
subscription.token = null;
|
|
2565
|
+
subscription.holder = null;
|
|
2566
|
+
subscription.wake_id = null;
|
|
2567
|
+
subscription.wake_snapshot.clear();
|
|
2568
|
+
await this.triggerNextWakeIfPending(subscription);
|
|
2569
|
+
}
|
|
2570
|
+
} catch (err) {
|
|
2571
|
+
serverLog.warn(`[subscriptions] webhook delivery failed:`, err);
|
|
2572
|
+
this.scheduleWebhookRetry(subscription, triggeredBy);
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
scheduleWebhookRetry(subscription, triggeredBy) {
|
|
2576
|
+
if (this.isShuttingDown) return;
|
|
2577
|
+
subscription.retry_count++;
|
|
2578
|
+
const baseDelay = Math.min(1e3 * Math.pow(2, Math.max(0, subscription.retry_count - 1)), MAX_RETRY_DELAY_MS);
|
|
2579
|
+
const jitter = baseDelay * .2 * (Math.random() * 2 - 1);
|
|
2580
|
+
const delay = Math.max(0, Math.round(baseDelay + jitter));
|
|
2581
|
+
subscription.status = `failed`;
|
|
2582
|
+
subscription.next_attempt_at = Date.now() + delay;
|
|
2583
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2584
|
+
subscription.retry_timer = setTimeout(() => {
|
|
2585
|
+
subscription.retry_timer = null;
|
|
2586
|
+
this.deliverWebhook(subscription, triggeredBy);
|
|
2587
|
+
}, delay);
|
|
2588
|
+
}
|
|
2589
|
+
async writePullWakeEvent(subscription, streamPath) {
|
|
2590
|
+
if (!subscription.wake_stream) return;
|
|
2591
|
+
const wakeStream = toAbsoluteStreamPath(subscription.wake_stream);
|
|
2592
|
+
if (!this.streamStore.has(wakeStream)) {
|
|
2593
|
+
serverLog.warn(`[subscriptions] wake stream does not exist: ${wakeStream}`);
|
|
2594
|
+
return;
|
|
2595
|
+
}
|
|
2596
|
+
const event = {
|
|
2597
|
+
type: `wake`,
|
|
2598
|
+
subscription_id: subscription.id,
|
|
2599
|
+
stream: streamPath,
|
|
2600
|
+
generation: subscription.generation,
|
|
2601
|
+
ts: Date.now()
|
|
2602
|
+
};
|
|
2603
|
+
await Promise.resolve(this.streamStore.append(wakeStream, new TextEncoder().encode(JSON.stringify(event))));
|
|
2604
|
+
}
|
|
2605
|
+
autoAckWakeSnapshot(subscription) {
|
|
2606
|
+
for (const [stream, tail] of subscription.wake_snapshot) {
|
|
2607
|
+
const link = subscription.streams.get(stream);
|
|
2608
|
+
if (link) link.acked_offset = tail;
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
applyAcks(subscription, request) {
|
|
2612
|
+
if (!request.acks) return null;
|
|
2613
|
+
for (const ack of request.acks) {
|
|
2614
|
+
const stream = normalizeRelativePath(ack.stream ?? ack.path ?? ``);
|
|
2615
|
+
const link = subscription.streams.get(stream);
|
|
2616
|
+
if (!stream || !link) return this.errorResponse(409, `INVALID_OFFSET`, `Ack references an unknown subscription stream`);
|
|
2617
|
+
if (ack.offset === BEFORE_FIRST_OFFSET) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset must not be -1`);
|
|
2618
|
+
if (compareOffsets(ack.offset, link.acked_offset) < 0) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset regresses the committed cursor`);
|
|
2619
|
+
if (compareOffsets(ack.offset, this.getTailOffset(stream)) > 0) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset is beyond stream tail`);
|
|
2620
|
+
}
|
|
2621
|
+
for (const ack of request.acks) {
|
|
2622
|
+
const stream = normalizeRelativePath(ack.stream ?? ack.path ?? ``);
|
|
2623
|
+
subscription.streams.get(stream).acked_offset = ack.offset;
|
|
2624
|
+
}
|
|
2625
|
+
return null;
|
|
2626
|
+
}
|
|
2627
|
+
validateWakeToken(subscription, token, request) {
|
|
2628
|
+
const tokenResult = validateCallbackToken(token, this.tokenSubject(subscription));
|
|
2629
|
+
if (!tokenResult.valid) return this.errorResponse(401, tokenResult.code, tokenResult.code === `TOKEN_EXPIRED` ? `Token expired` : `Token invalid`);
|
|
2630
|
+
if (tokenResult.epoch !== subscription.generation || request.generation !== subscription.generation || request.wake_id !== subscription.wake_id) return this.errorResponse(409, `FENCED`, `Wake generation is stale`);
|
|
2631
|
+
return null;
|
|
2632
|
+
}
|
|
2633
|
+
async triggerNextWakeIfPending(subscription) {
|
|
2634
|
+
if (!this.hasPendingWork(subscription)) return false;
|
|
2635
|
+
await this.createWake(subscription, this.firstPendingStream(subscription));
|
|
2636
|
+
return true;
|
|
2637
|
+
}
|
|
2638
|
+
hasPendingWork(subscription) {
|
|
2639
|
+
return this.streamInfos(subscription).some((stream) => stream.has_pending);
|
|
2640
|
+
}
|
|
2641
|
+
firstPendingStream(subscription) {
|
|
2642
|
+
return this.streamInfos(subscription).find((stream) => stream.has_pending)?.path ?? ``;
|
|
2643
|
+
}
|
|
2644
|
+
streamInfos(subscription) {
|
|
2645
|
+
return Array.from(subscription.streams.values()).map((link) => {
|
|
2646
|
+
const tail = this.getTailOffset(link.path);
|
|
2647
|
+
return {
|
|
2648
|
+
path: link.path,
|
|
2649
|
+
link_type: link.link_types.has(`explicit`) ? `explicit` : `glob`,
|
|
2650
|
+
acked_offset: link.acked_offset,
|
|
2651
|
+
tail_offset: tail,
|
|
2652
|
+
has_pending: compareOffsets(tail, link.acked_offset) > 0
|
|
2653
|
+
};
|
|
2654
|
+
});
|
|
2655
|
+
}
|
|
2656
|
+
linkStream(subscription, streamPath, linkType, ackedOffset) {
|
|
2657
|
+
const normalized = normalizeRelativePath(streamPath);
|
|
2658
|
+
const existing = subscription.streams.get(normalized);
|
|
2659
|
+
if (existing) {
|
|
2660
|
+
existing.link_types.add(linkType);
|
|
2661
|
+
return existing;
|
|
2662
|
+
}
|
|
2663
|
+
const link = {
|
|
2664
|
+
path: normalized,
|
|
2665
|
+
link_types: new Set([linkType]),
|
|
2666
|
+
acked_offset: ackedOffset
|
|
2667
|
+
};
|
|
2668
|
+
subscription.streams.set(normalized, link);
|
|
2669
|
+
return link;
|
|
2670
|
+
}
|
|
2671
|
+
listStreams() {
|
|
2672
|
+
return this.streamStore.list().map((path$1) => toStreamRelativePath(path$1)).filter((path$1) => path$1 !== null);
|
|
2673
|
+
}
|
|
2674
|
+
getTailOffset(streamPath) {
|
|
2675
|
+
return this.streamStore.get(toAbsoluteStreamPath(streamPath))?.currentOffset ?? ZERO_OFFSET;
|
|
2676
|
+
}
|
|
2677
|
+
subscriptionActionUrl(subscription, action) {
|
|
2678
|
+
const url = new URL(`/v1/stream/__ds/subscriptions/${encodeURIComponent(subscription.id)}/${action}`, this.callbackBaseUrl);
|
|
2679
|
+
return url.toString();
|
|
2680
|
+
}
|
|
2681
|
+
webhookJwksUrl() {
|
|
2682
|
+
const url = new URL(`/v1/stream/__ds/jwks.json`, this.callbackBaseUrl);
|
|
2683
|
+
return url.toString();
|
|
2684
|
+
}
|
|
2685
|
+
webhookSigningMetadata() {
|
|
2686
|
+
return {
|
|
2687
|
+
alg: `ed25519`,
|
|
2688
|
+
kid: getWebhookSigningKeyId(),
|
|
2689
|
+
jwks_url: this.webhookJwksUrl()
|
|
2690
|
+
};
|
|
2691
|
+
}
|
|
2692
|
+
extendLease(subscription) {
|
|
2693
|
+
this.clearLease(subscription);
|
|
2694
|
+
subscription.lease_timer = setTimeout(() => {
|
|
2695
|
+
subscription.lease_timer = null;
|
|
2696
|
+
subscription.holder = null;
|
|
2697
|
+
subscription.token = null;
|
|
2698
|
+
subscription.wake_id = null;
|
|
2699
|
+
subscription.wake_snapshot.clear();
|
|
2700
|
+
this.triggerNextWakeIfPending(subscription);
|
|
2701
|
+
}, subscription.lease_ttl_ms);
|
|
2702
|
+
}
|
|
2703
|
+
clearLease(subscription) {
|
|
2704
|
+
if (subscription.lease_timer) {
|
|
2705
|
+
clearTimeout(subscription.lease_timer);
|
|
2706
|
+
subscription.lease_timer = null;
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
tokenSubject(subscription) {
|
|
2710
|
+
return `subscription:${subscription.id}`;
|
|
2711
|
+
}
|
|
2712
|
+
errorResponse(status, code, message) {
|
|
2713
|
+
return {
|
|
2714
|
+
status,
|
|
2715
|
+
body: { error: {
|
|
2716
|
+
code,
|
|
2717
|
+
message
|
|
2718
|
+
} }
|
|
2719
|
+
};
|
|
2720
|
+
}
|
|
2721
|
+
};
|
|
2722
|
+
|
|
2723
|
+
//#endregion
|
|
2724
|
+
//#region src/subscription-routes.ts
|
|
2725
|
+
const RESERVED_CONTROL_PREFIX = `/v1/stream/__ds`;
|
|
2726
|
+
const SUBSCRIPTION_PREFIX = `${RESERVED_CONTROL_PREFIX}/subscriptions/`;
|
|
2727
|
+
const JWKS_PATH = `${RESERVED_CONTROL_PREFIX}/jwks.json`;
|
|
2728
|
+
const ERROR_STATUS = {
|
|
2729
|
+
INVALID_REQUEST: 400,
|
|
2730
|
+
SUBSCRIPTION_NOT_FOUND: 404,
|
|
2731
|
+
SUBSCRIPTION_ALREADY_EXISTS: 409,
|
|
2732
|
+
WEBHOOK_URL_REJECTED: 400,
|
|
2733
|
+
TOKEN_INVALID: 401,
|
|
2734
|
+
TOKEN_EXPIRED: 401,
|
|
2735
|
+
FENCED: 409,
|
|
2736
|
+
ALREADY_CLAIMED: 409,
|
|
2737
|
+
NO_PENDING_WORK: 409,
|
|
2738
|
+
INVALID_OFFSET: 409
|
|
2739
|
+
};
|
|
2740
|
+
var SubscriptionRoutes = class {
|
|
2741
|
+
manager;
|
|
2742
|
+
constructor(manager) {
|
|
2743
|
+
this.manager = manager;
|
|
2744
|
+
}
|
|
2745
|
+
async handleRequest(method, path$1, req, res) {
|
|
2746
|
+
if (path$1 === JWKS_PATH) {
|
|
2747
|
+
this.handleJwks(method, res);
|
|
2748
|
+
return true;
|
|
2749
|
+
}
|
|
2750
|
+
const route = this.parseRoute(path$1);
|
|
2751
|
+
if (!route) {
|
|
2752
|
+
if (path$1 === RESERVED_CONTROL_PREFIX || path$1.startsWith(`${RESERVED_CONTROL_PREFIX}/`)) {
|
|
2753
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Durable Streams control route not found`);
|
|
2754
|
+
return true;
|
|
2755
|
+
}
|
|
2756
|
+
return false;
|
|
2757
|
+
}
|
|
2758
|
+
try {
|
|
2759
|
+
switch (route.action) {
|
|
2760
|
+
case `base`:
|
|
2761
|
+
await this.handleBase(route, method, req, res);
|
|
2762
|
+
return true;
|
|
2763
|
+
case `streams`:
|
|
2764
|
+
await this.handleStreams(route, method, req, res);
|
|
2765
|
+
return true;
|
|
2766
|
+
case `stream`:
|
|
2767
|
+
this.handleStream(route, method, res);
|
|
2768
|
+
return true;
|
|
2769
|
+
case `callback`:
|
|
2770
|
+
await this.handleCallback(route, req, res);
|
|
2771
|
+
return true;
|
|
2772
|
+
case `claim`:
|
|
2773
|
+
await this.handleClaim(route, req, res);
|
|
2774
|
+
return true;
|
|
2775
|
+
case `ack`:
|
|
2776
|
+
await this.handleAck(route, req, res);
|
|
2777
|
+
return true;
|
|
2778
|
+
case `release`:
|
|
2779
|
+
await this.handleRelease(route, req, res);
|
|
2780
|
+
return true;
|
|
2781
|
+
}
|
|
2782
|
+
} catch (err) {
|
|
2783
|
+
if (err instanceof SyntaxError) {
|
|
2784
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `Invalid JSON body`);
|
|
2785
|
+
return true;
|
|
2786
|
+
}
|
|
2787
|
+
throw err;
|
|
2788
|
+
}
|
|
2789
|
+
}
|
|
2790
|
+
async handleBase(route, method, req, res) {
|
|
2791
|
+
if (method === `PUT`) {
|
|
2792
|
+
const parsed = await this.readJson(req);
|
|
2793
|
+
const input = this.parseCreateInput(parsed);
|
|
2794
|
+
if (`error` in input) {
|
|
2795
|
+
this.writeError(res, 400, `INVALID_REQUEST`, input.error);
|
|
2796
|
+
return;
|
|
2797
|
+
}
|
|
2798
|
+
const result = this.manager.createOrConfirm(route.subscriptionId, input.value);
|
|
2799
|
+
if (`error` in result) {
|
|
2800
|
+
this.writeError(res, ERROR_STATUS[result.error.code], result.error.code, result.error.message);
|
|
2801
|
+
return;
|
|
2802
|
+
}
|
|
2803
|
+
this.writeJson(res, result.created ? 201 : 200, this.manager.serialize(result.subscription));
|
|
2804
|
+
return;
|
|
2805
|
+
}
|
|
2806
|
+
if (method === `GET`) {
|
|
2807
|
+
const subscription = this.manager.get(route.subscriptionId);
|
|
2808
|
+
if (!subscription) {
|
|
2809
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2810
|
+
return;
|
|
2811
|
+
}
|
|
2812
|
+
this.writeJson(res, 200, this.manager.serialize(subscription));
|
|
2813
|
+
return;
|
|
2814
|
+
}
|
|
2815
|
+
if (method === `DELETE`) {
|
|
2816
|
+
this.manager.delete(route.subscriptionId);
|
|
2817
|
+
res.writeHead(204);
|
|
2818
|
+
res.end();
|
|
2819
|
+
return;
|
|
2820
|
+
}
|
|
2821
|
+
this.methodNotAllowed(res);
|
|
2822
|
+
}
|
|
2823
|
+
handleJwks(method, res) {
|
|
2824
|
+
if (method !== `GET`) {
|
|
2825
|
+
this.methodNotAllowed(res);
|
|
2826
|
+
return;
|
|
2827
|
+
}
|
|
2828
|
+
res.writeHead(200, {
|
|
2829
|
+
"content-type": `application/jwk-set+json`,
|
|
2830
|
+
"cache-control": `public, max-age=300`
|
|
2831
|
+
});
|
|
2832
|
+
res.end(JSON.stringify(this.manager.getWebhookJwks()));
|
|
2833
|
+
}
|
|
2834
|
+
async handleStreams(route, method, req, res) {
|
|
2835
|
+
if (method !== `POST`) {
|
|
2836
|
+
this.methodNotAllowed(res);
|
|
2837
|
+
return;
|
|
2838
|
+
}
|
|
2839
|
+
const parsed = await this.readJson(req);
|
|
2840
|
+
const streams = parsed.streams;
|
|
2841
|
+
if (!Array.isArray(streams) || streams.some((stream) => typeof stream !== `string` || stream.length === 0)) {
|
|
2842
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `streams must be a non-empty string array`);
|
|
2843
|
+
return;
|
|
2844
|
+
}
|
|
2845
|
+
const ok = this.manager.addExplicitStreams(route.subscriptionId, streams.map(normalizeRelativePath));
|
|
2846
|
+
if (!ok) {
|
|
2847
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2848
|
+
return;
|
|
2849
|
+
}
|
|
2850
|
+
res.writeHead(204);
|
|
2851
|
+
res.end();
|
|
2852
|
+
}
|
|
2853
|
+
handleStream(route, method, res) {
|
|
2854
|
+
if (method !== `DELETE`) {
|
|
2855
|
+
this.methodNotAllowed(res);
|
|
2856
|
+
return;
|
|
2857
|
+
}
|
|
2858
|
+
const ok = this.manager.removeExplicitStream(route.subscriptionId, route.streamPath ?? ``);
|
|
2859
|
+
if (!ok) {
|
|
2860
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2861
|
+
return;
|
|
2862
|
+
}
|
|
2863
|
+
res.writeHead(204);
|
|
2864
|
+
res.end();
|
|
2865
|
+
}
|
|
2866
|
+
async handleCallback(route, req, res) {
|
|
2867
|
+
const token = this.readBearerToken(req);
|
|
2868
|
+
if (!token) {
|
|
2869
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2870
|
+
return;
|
|
2871
|
+
}
|
|
2872
|
+
const body = await this.readJson(req);
|
|
2873
|
+
const result = await this.manager.handleWebhookCallback(route.subscriptionId, token, body);
|
|
2874
|
+
this.writeManagerResult(res, result);
|
|
2875
|
+
}
|
|
2876
|
+
async handleClaim(route, req, res) {
|
|
2877
|
+
const parsed = await this.readJson(req);
|
|
2878
|
+
const worker = parsed.worker;
|
|
2879
|
+
if (typeof worker !== `string` || worker.length === 0) {
|
|
2880
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `worker must be a non-empty string`);
|
|
2881
|
+
return;
|
|
2882
|
+
}
|
|
2883
|
+
const result = await this.manager.claim(route.subscriptionId, worker);
|
|
2884
|
+
this.writeManagerResult(res, result);
|
|
2885
|
+
}
|
|
2886
|
+
async handleAck(route, req, res) {
|
|
2887
|
+
const token = this.readBearerToken(req);
|
|
2888
|
+
if (!token) {
|
|
2889
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2890
|
+
return;
|
|
2891
|
+
}
|
|
2892
|
+
const body = await this.readJson(req);
|
|
2893
|
+
const result = await this.manager.ack(route.subscriptionId, token, body);
|
|
2894
|
+
this.writeManagerResult(res, result);
|
|
2895
|
+
}
|
|
2896
|
+
async handleRelease(route, req, res) {
|
|
2897
|
+
const token = this.readBearerToken(req);
|
|
2898
|
+
if (!token) {
|
|
2899
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2900
|
+
return;
|
|
2901
|
+
}
|
|
2902
|
+
const body = await this.readJson(req);
|
|
2903
|
+
const result = await this.manager.release(route.subscriptionId, token, body);
|
|
2904
|
+
this.writeManagerResult(res, result);
|
|
2905
|
+
}
|
|
2906
|
+
parseCreateInput(value) {
|
|
2907
|
+
if (!value || typeof value !== `object`) return { error: `Request body must be a JSON object` };
|
|
2908
|
+
const payload = value;
|
|
2909
|
+
if (payload.type !== `webhook` && payload.type !== `pull-wake`) return { error: `type must be "webhook" or "pull-wake"` };
|
|
2910
|
+
const type = payload.type;
|
|
2911
|
+
const pattern = typeof payload.pattern === `string` && payload.pattern.length > 0 ? normalizeRelativePath(payload.pattern) : void 0;
|
|
2912
|
+
const streams = Array.isArray(payload.streams) && payload.streams.length > 0 ? payload.streams.map((stream) => typeof stream === `string` ? normalizeRelativePath(stream) : null) : [];
|
|
2913
|
+
if (streams.some((stream) => stream === null)) return { error: `streams must contain only strings` };
|
|
2914
|
+
if (!pattern && streams.length === 0) return { error: `At least one of pattern or streams is required` };
|
|
2915
|
+
const leaseTtl = payload.lease_ttl_ms === void 0 ? DEFAULT_LEASE_TTL_MS : payload.lease_ttl_ms;
|
|
2916
|
+
if (typeof leaseTtl !== `number` || !Number.isInteger(leaseTtl) || leaseTtl < MIN_LEASE_TTL_MS || leaseTtl > MAX_LEASE_TTL_MS) return { error: `lease_ttl_ms must be an integer from 1000 to 600000` };
|
|
2917
|
+
let webhook;
|
|
2918
|
+
if (type === `webhook`) {
|
|
2919
|
+
const rawWebhook = payload.webhook;
|
|
2920
|
+
if (!rawWebhook || typeof rawWebhook !== `object`) return { error: `webhook subscriptions require webhook.url` };
|
|
2921
|
+
const url = rawWebhook.url;
|
|
2922
|
+
if (typeof url !== `string` || url.length === 0) return { error: `webhook subscriptions require webhook.url` };
|
|
2923
|
+
webhook = { url };
|
|
2924
|
+
}
|
|
2925
|
+
const wakeStream = typeof payload.wake_stream === `string` && payload.wake_stream.length > 0 ? normalizeRelativePath(payload.wake_stream) : void 0;
|
|
2926
|
+
if (type === `pull-wake` && !wakeStream) return { error: `pull-wake subscriptions require wake_stream` };
|
|
2927
|
+
return { value: {
|
|
2928
|
+
type,
|
|
2929
|
+
pattern,
|
|
2930
|
+
streams,
|
|
2931
|
+
webhook,
|
|
2932
|
+
wake_stream: wakeStream,
|
|
2933
|
+
lease_ttl_ms: leaseTtl,
|
|
2934
|
+
description: typeof payload.description === `string` ? payload.description : void 0
|
|
2935
|
+
} };
|
|
2936
|
+
}
|
|
2937
|
+
parseRoute(path$1) {
|
|
2938
|
+
if (!path$1.startsWith(SUBSCRIPTION_PREFIX)) return null;
|
|
2939
|
+
const rest = path$1.slice(SUBSCRIPTION_PREFIX.length);
|
|
2940
|
+
const parts = rest.split(`/`);
|
|
2941
|
+
const subscriptionId = parts[0] ? decodeURIComponent(parts[0]) : ``;
|
|
2942
|
+
if (!subscriptionId) return null;
|
|
2943
|
+
const tail = parts.slice(1);
|
|
2944
|
+
if (tail.length === 0) return {
|
|
2945
|
+
subscriptionId,
|
|
2946
|
+
action: `base`
|
|
2947
|
+
};
|
|
2948
|
+
if (tail[0] === `streams` && tail.length === 1) return {
|
|
2949
|
+
subscriptionId,
|
|
2950
|
+
action: `streams`
|
|
2951
|
+
};
|
|
2952
|
+
if (tail[0] === `streams` && tail.length > 1) return {
|
|
2953
|
+
subscriptionId,
|
|
2954
|
+
action: `stream`,
|
|
2955
|
+
streamPath: normalizeRelativePath(decodeURIComponent(tail.slice(1).join(`/`)))
|
|
2956
|
+
};
|
|
2957
|
+
if (tail.length === 1 && [
|
|
2958
|
+
`callback`,
|
|
2959
|
+
`claim`,
|
|
2960
|
+
`ack`,
|
|
2961
|
+
`release`
|
|
2962
|
+
].includes(tail[0])) return {
|
|
2963
|
+
subscriptionId,
|
|
2964
|
+
action: tail[0]
|
|
2965
|
+
};
|
|
2966
|
+
return null;
|
|
2967
|
+
}
|
|
2968
|
+
readBearerToken(req) {
|
|
2969
|
+
const authHeader = req.headers.authorization;
|
|
2970
|
+
if (!authHeader || !authHeader.startsWith(`Bearer `)) return null;
|
|
2971
|
+
return authHeader.slice(`Bearer `.length);
|
|
2972
|
+
}
|
|
2973
|
+
async readJson(req) {
|
|
2974
|
+
const chunks = [];
|
|
2975
|
+
for await (const chunk of req) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
2976
|
+
const raw = Buffer.concat(chunks).toString(`utf8`);
|
|
2977
|
+
return raw.length > 0 ? JSON.parse(raw) : {};
|
|
2978
|
+
}
|
|
2979
|
+
writeManagerResult(res, result) {
|
|
2980
|
+
if (result.status === 204) {
|
|
2981
|
+
res.writeHead(204);
|
|
2982
|
+
res.end();
|
|
2983
|
+
return;
|
|
2984
|
+
}
|
|
2985
|
+
this.writeJson(res, result.status, result.body ?? {});
|
|
2986
|
+
}
|
|
2987
|
+
writeJson(res, status, body) {
|
|
2988
|
+
res.writeHead(status, { "content-type": `application/json` });
|
|
2989
|
+
res.end(JSON.stringify(body));
|
|
2990
|
+
}
|
|
2991
|
+
writeError(res, status, code, message) {
|
|
2992
|
+
this.writeJson(res, status, { error: {
|
|
2993
|
+
code,
|
|
2994
|
+
message
|
|
2995
|
+
} });
|
|
2996
|
+
}
|
|
2997
|
+
methodNotAllowed(res) {
|
|
2998
|
+
res.writeHead(405, { "content-type": `text/plain` });
|
|
2999
|
+
res.end(`Method not allowed`);
|
|
3000
|
+
}
|
|
3001
|
+
};
|
|
3002
|
+
|
|
1983
3003
|
//#endregion
|
|
1984
3004
|
//#region src/server.ts
|
|
1985
|
-
const STREAM_OFFSET_HEADER = `Stream-Next-Offset`;
|
|
1986
|
-
const STREAM_CURSOR_HEADER = `Stream-Cursor`;
|
|
1987
|
-
const STREAM_UP_TO_DATE_HEADER = `Stream-Up-To-Date`;
|
|
1988
|
-
const STREAM_SEQ_HEADER = `Stream-Seq`;
|
|
1989
|
-
const STREAM_TTL_HEADER = `Stream-TTL`;
|
|
1990
|
-
const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`;
|
|
1991
3005
|
const STREAM_SSE_DATA_ENCODING_HEADER = `Stream-SSE-Data-Encoding`;
|
|
1992
|
-
const PRODUCER_ID_HEADER = `Producer-Id`;
|
|
1993
|
-
const PRODUCER_EPOCH_HEADER = `Producer-Epoch`;
|
|
1994
|
-
const PRODUCER_SEQ_HEADER = `Producer-Seq`;
|
|
1995
|
-
const PRODUCER_EXPECTED_SEQ_HEADER = `Producer-Expected-Seq`;
|
|
1996
|
-
const PRODUCER_RECEIVED_SEQ_HEADER = `Producer-Received-Seq`;
|
|
1997
|
-
const SSE_OFFSET_FIELD = `streamNextOffset`;
|
|
1998
|
-
const SSE_CURSOR_FIELD = `streamCursor`;
|
|
1999
3006
|
const SSE_UP_TO_DATE_FIELD = `upToDate`;
|
|
2000
|
-
const SSE_CLOSED_FIELD = `streamClosed`;
|
|
2001
|
-
const STREAM_CLOSED_HEADER = `Stream-Closed`;
|
|
2002
3007
|
const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`;
|
|
2003
3008
|
const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`;
|
|
2004
|
-
const OFFSET_QUERY_PARAM = `offset`;
|
|
2005
|
-
const LIVE_QUERY_PARAM = `live`;
|
|
2006
|
-
const CURSOR_QUERY_PARAM = `cursor`;
|
|
2007
3009
|
/**
|
|
2008
3010
|
* Encode data for SSE format.
|
|
2009
3011
|
* Per SSE spec, each line in the payload needs its own "data:" prefix.
|
|
@@ -2057,6 +3059,8 @@ var DurableStreamTestServer = class {
|
|
|
2057
3059
|
isShuttingDown = false;
|
|
2058
3060
|
/** Injected faults for testing retry/resilience */
|
|
2059
3061
|
injectedFaults = new Map();
|
|
3062
|
+
subscriptionManager = null;
|
|
3063
|
+
subscriptionRoutes = null;
|
|
2060
3064
|
constructor(options = {}) {
|
|
2061
3065
|
if (options.dataDir) this.store = new FileBackedStreamStore({ dataDir: options.dataDir });
|
|
2062
3066
|
else this.store = new StreamStore();
|
|
@@ -2071,7 +3075,8 @@ var DurableStreamTestServer = class {
|
|
|
2071
3075
|
cursorOptions: {
|
|
2072
3076
|
intervalSeconds: options.cursorIntervalSeconds,
|
|
2073
3077
|
epoch: options.cursorEpoch
|
|
2074
|
-
}
|
|
3078
|
+
},
|
|
3079
|
+
webhooks: options.webhooks ?? false
|
|
2075
3080
|
};
|
|
2076
3081
|
}
|
|
2077
3082
|
/**
|
|
@@ -2082,7 +3087,7 @@ var DurableStreamTestServer = class {
|
|
|
2082
3087
|
return new Promise((resolve, reject) => {
|
|
2083
3088
|
this.server = createServer((req, res) => {
|
|
2084
3089
|
this.handleRequest(req, res).catch((err) => {
|
|
2085
|
-
|
|
3090
|
+
serverLog.error(`Request error:`, err);
|
|
2086
3091
|
if (!res.headersSent) {
|
|
2087
3092
|
res.writeHead(500, { "content-type": `text/plain` });
|
|
2088
3093
|
res.end(`Internal server error`);
|
|
@@ -2094,6 +3099,12 @@ var DurableStreamTestServer = class {
|
|
|
2094
3099
|
const addr = this.server.address();
|
|
2095
3100
|
if (typeof addr === `string`) this._url = addr;
|
|
2096
3101
|
else if (addr) this._url = `http://${this.options.host}:${addr.port}`;
|
|
3102
|
+
this.subscriptionManager = new SubscriptionManager({
|
|
3103
|
+
callbackBaseUrl: this._url,
|
|
3104
|
+
streamStore: this.store,
|
|
3105
|
+
webhooksEnabled: this.options.webhooks
|
|
3106
|
+
});
|
|
3107
|
+
this.subscriptionRoutes = new SubscriptionRoutes(this.subscriptionManager);
|
|
2097
3108
|
resolve(this._url);
|
|
2098
3109
|
});
|
|
2099
3110
|
});
|
|
@@ -2104,6 +3115,11 @@ var DurableStreamTestServer = class {
|
|
|
2104
3115
|
async stop() {
|
|
2105
3116
|
if (!this.server) return;
|
|
2106
3117
|
this.isShuttingDown = true;
|
|
3118
|
+
if (this.subscriptionManager) {
|
|
3119
|
+
this.subscriptionManager.shutdown();
|
|
3120
|
+
this.subscriptionManager = null;
|
|
3121
|
+
this.subscriptionRoutes = null;
|
|
3122
|
+
}
|
|
2107
3123
|
if (`cancelAllWaits` in this.store) this.store.cancelAllWaits();
|
|
2108
3124
|
for (const res of this.activeSSEResponses) res.end();
|
|
2109
3125
|
this.activeSSEResponses.clear();
|
|
@@ -2143,8 +3159,8 @@ var DurableStreamTestServer = class {
|
|
|
2143
3159
|
* Used for testing retry/resilience behavior.
|
|
2144
3160
|
* @deprecated Use injectFault for full fault injection capabilities
|
|
2145
3161
|
*/
|
|
2146
|
-
injectError(path$
|
|
2147
|
-
this.injectedFaults.set(path$
|
|
3162
|
+
injectError(path$1, status, count = 1, retryAfter) {
|
|
3163
|
+
this.injectedFaults.set(path$1, {
|
|
2148
3164
|
status,
|
|
2149
3165
|
count,
|
|
2150
3166
|
retryAfter
|
|
@@ -2154,8 +3170,8 @@ var DurableStreamTestServer = class {
|
|
|
2154
3170
|
* Inject a fault to be triggered on the next N requests to a path.
|
|
2155
3171
|
* Supports various fault types: delays, connection drops, body corruption, etc.
|
|
2156
3172
|
*/
|
|
2157
|
-
injectFault(path$
|
|
2158
|
-
this.injectedFaults.set(path$
|
|
3173
|
+
injectFault(path$1, fault) {
|
|
3174
|
+
this.injectedFaults.set(path$1, {
|
|
2159
3175
|
count: 1,
|
|
2160
3176
|
...fault
|
|
2161
3177
|
});
|
|
@@ -2170,13 +3186,13 @@ var DurableStreamTestServer = class {
|
|
|
2170
3186
|
* Check if there's an injected fault for this path/method and consume it.
|
|
2171
3187
|
* Returns the fault config if one should be triggered, null otherwise.
|
|
2172
3188
|
*/
|
|
2173
|
-
consumeInjectedFault(path$
|
|
2174
|
-
const fault = this.injectedFaults.get(path$
|
|
3189
|
+
consumeInjectedFault(path$1, method) {
|
|
3190
|
+
const fault = this.injectedFaults.get(path$1);
|
|
2175
3191
|
if (!fault) return null;
|
|
2176
3192
|
if (fault.method && fault.method.toUpperCase() !== method.toUpperCase()) return null;
|
|
2177
3193
|
if (fault.probability !== void 0 && Math.random() > fault.probability) return null;
|
|
2178
3194
|
fault.count--;
|
|
2179
|
-
if (fault.count <= 0) this.injectedFaults.delete(path$
|
|
3195
|
+
if (fault.count <= 0) this.injectedFaults.delete(path$1);
|
|
2180
3196
|
return fault;
|
|
2181
3197
|
}
|
|
2182
3198
|
/**
|
|
@@ -2211,7 +3227,7 @@ var DurableStreamTestServer = class {
|
|
|
2211
3227
|
}
|
|
2212
3228
|
async handleRequest(req, res) {
|
|
2213
3229
|
const url = new URL(req.url ?? `/`, `http://${req.headers.host}`);
|
|
2214
|
-
const path$
|
|
3230
|
+
const path$1 = url.pathname;
|
|
2215
3231
|
const method = req.method?.toUpperCase();
|
|
2216
3232
|
res.setHeader(`access-control-allow-origin`, `*`);
|
|
2217
3233
|
res.setHeader(`access-control-allow-methods`, `GET, POST, PUT, DELETE, HEAD, OPTIONS`);
|
|
@@ -2224,11 +3240,11 @@ var DurableStreamTestServer = class {
|
|
|
2224
3240
|
res.end();
|
|
2225
3241
|
return;
|
|
2226
3242
|
}
|
|
2227
|
-
if (path$
|
|
3243
|
+
if (path$1 === `/_test/inject-error`) {
|
|
2228
3244
|
await this.handleTestInjectError(method, req, res);
|
|
2229
3245
|
return;
|
|
2230
3246
|
}
|
|
2231
|
-
const fault = this.consumeInjectedFault(path$
|
|
3247
|
+
const fault = this.consumeInjectedFault(path$1, method ?? `GET`);
|
|
2232
3248
|
if (fault) {
|
|
2233
3249
|
await this.applyFaultDelay(fault);
|
|
2234
3250
|
if (fault.dropConnection) {
|
|
@@ -2244,22 +3260,26 @@ var DurableStreamTestServer = class {
|
|
|
2244
3260
|
}
|
|
2245
3261
|
if (fault.truncateBodyBytes !== void 0 || fault.corruptBody || fault.injectSseEvent) res._injectedFault = fault;
|
|
2246
3262
|
}
|
|
3263
|
+
if (this.subscriptionRoutes && method) {
|
|
3264
|
+
const handled = await this.subscriptionRoutes.handleRequest(method, path$1, req, res);
|
|
3265
|
+
if (handled) return;
|
|
3266
|
+
}
|
|
2247
3267
|
try {
|
|
2248
3268
|
switch (method) {
|
|
2249
3269
|
case `PUT`:
|
|
2250
|
-
await this.handleCreate(path$
|
|
3270
|
+
await this.handleCreate(path$1, req, res);
|
|
2251
3271
|
break;
|
|
2252
3272
|
case `HEAD`:
|
|
2253
|
-
this.handleHead(path$
|
|
3273
|
+
this.handleHead(path$1, res);
|
|
2254
3274
|
break;
|
|
2255
3275
|
case `GET`:
|
|
2256
|
-
await this.handleRead(path$
|
|
3276
|
+
await this.handleRead(path$1, url, req, res);
|
|
2257
3277
|
break;
|
|
2258
3278
|
case `POST`:
|
|
2259
|
-
await this.handleAppend(path$
|
|
3279
|
+
await this.handleAppend(path$1, req, res);
|
|
2260
3280
|
break;
|
|
2261
3281
|
case `DELETE`:
|
|
2262
|
-
await this.handleDelete(path$
|
|
3282
|
+
await this.handleDelete(path$1, res);
|
|
2263
3283
|
break;
|
|
2264
3284
|
default:
|
|
2265
3285
|
res.writeHead(405, { "content-type": `text/plain` });
|
|
@@ -2297,7 +3317,7 @@ var DurableStreamTestServer = class {
|
|
|
2297
3317
|
/**
|
|
2298
3318
|
* Handle PUT - create stream
|
|
2299
3319
|
*/
|
|
2300
|
-
async handleCreate(path$
|
|
3320
|
+
async handleCreate(path$1, req, res) {
|
|
2301
3321
|
let contentType = req.headers[`content-type`];
|
|
2302
3322
|
const forkedFromHeader = req.headers[STREAM_FORKED_FROM_HEADER.toLowerCase()];
|
|
2303
3323
|
const forkOffsetHeader = req.headers[STREAM_FORK_OFFSET_HEADER.toLowerCase()];
|
|
@@ -2343,9 +3363,9 @@ var DurableStreamTestServer = class {
|
|
|
2343
3363
|
}
|
|
2344
3364
|
}
|
|
2345
3365
|
const body = await this.readBody(req);
|
|
2346
|
-
const isNew = !this.store.has(path$
|
|
3366
|
+
const isNew = !this.store.has(path$1);
|
|
2347
3367
|
try {
|
|
2348
|
-
await Promise.resolve(this.store.create(path$
|
|
3368
|
+
await Promise.resolve(this.store.create(path$1, {
|
|
2349
3369
|
contentType,
|
|
2350
3370
|
ttlSeconds,
|
|
2351
3371
|
expiresAt: expiresAtHeader,
|
|
@@ -2379,19 +3399,20 @@ var DurableStreamTestServer = class {
|
|
|
2379
3399
|
}
|
|
2380
3400
|
throw err;
|
|
2381
3401
|
}
|
|
2382
|
-
const stream = this.store.get(path$
|
|
3402
|
+
const stream = this.store.get(path$1);
|
|
2383
3403
|
const resolvedContentType = stream.contentType ?? contentType ?? `application/octet-stream`;
|
|
2384
3404
|
if (isNew && this.options.onStreamCreated) await Promise.resolve(this.options.onStreamCreated({
|
|
2385
3405
|
type: `created`,
|
|
2386
|
-
path: path$
|
|
3406
|
+
path: path$1,
|
|
2387
3407
|
contentType: resolvedContentType,
|
|
2388
3408
|
timestamp: Date.now()
|
|
2389
3409
|
}));
|
|
3410
|
+
if (isNew && body.length > 0) await this.notifyStreamAppend(path$1);
|
|
2390
3411
|
const headers = {
|
|
2391
3412
|
"content-type": resolvedContentType,
|
|
2392
3413
|
[STREAM_OFFSET_HEADER]: stream.currentOffset
|
|
2393
3414
|
};
|
|
2394
|
-
if (isNew) headers[`location`] = `${this._url}${path$
|
|
3415
|
+
if (isNew) headers[`location`] = `${this._url}${path$1}`;
|
|
2395
3416
|
if (stream.closed) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
2396
3417
|
res.writeHead(isNew ? 201 : 200, headers);
|
|
2397
3418
|
res.end();
|
|
@@ -2399,8 +3420,8 @@ var DurableStreamTestServer = class {
|
|
|
2399
3420
|
/**
|
|
2400
3421
|
* Handle HEAD - get metadata
|
|
2401
3422
|
*/
|
|
2402
|
-
handleHead(path$
|
|
2403
|
-
const stream = this.store.get(path$
|
|
3423
|
+
handleHead(path$1, res) {
|
|
3424
|
+
const stream = this.store.get(path$1);
|
|
2404
3425
|
if (!stream) {
|
|
2405
3426
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
2406
3427
|
res.end();
|
|
@@ -2420,15 +3441,15 @@ var DurableStreamTestServer = class {
|
|
|
2420
3441
|
if (stream.ttlSeconds !== void 0) headers[STREAM_TTL_HEADER] = String(stream.ttlSeconds);
|
|
2421
3442
|
if (stream.expiresAt) headers[STREAM_EXPIRES_AT_HEADER] = stream.expiresAt;
|
|
2422
3443
|
const closedSuffix = stream.closed ? `:c` : ``;
|
|
2423
|
-
headers[`etag`] = `"${Buffer.from(path$
|
|
3444
|
+
headers[`etag`] = `"${Buffer.from(path$1).toString(`base64`)}:-1:${stream.currentOffset}${closedSuffix}"`;
|
|
2424
3445
|
res.writeHead(200, headers);
|
|
2425
3446
|
res.end();
|
|
2426
3447
|
}
|
|
2427
3448
|
/**
|
|
2428
3449
|
* Handle GET - read data
|
|
2429
3450
|
*/
|
|
2430
|
-
async handleRead(path$
|
|
2431
|
-
const stream = this.store.get(path$
|
|
3451
|
+
async handleRead(path$1, url, req, res) {
|
|
3452
|
+
const stream = this.store.get(path$1);
|
|
2432
3453
|
if (!stream) {
|
|
2433
3454
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
2434
3455
|
res.end(`Stream not found`);
|
|
@@ -2474,7 +3495,7 @@ var DurableStreamTestServer = class {
|
|
|
2474
3495
|
}
|
|
2475
3496
|
if (live === `sse`) {
|
|
2476
3497
|
const sseOffset = offset === `now` ? stream.currentOffset : offset;
|
|
2477
|
-
await this.handleSSE(path$
|
|
3498
|
+
await this.handleSSE(path$1, stream, sseOffset, cursor, useBase64, res);
|
|
2478
3499
|
return;
|
|
2479
3500
|
}
|
|
2480
3501
|
const effectiveOffset = offset === `now` ? stream.currentOffset : offset;
|
|
@@ -2492,8 +3513,8 @@ var DurableStreamTestServer = class {
|
|
|
2492
3513
|
res.end(responseBody);
|
|
2493
3514
|
return;
|
|
2494
3515
|
}
|
|
2495
|
-
let { messages, upToDate } = this.store.read(path$
|
|
2496
|
-
this.store.touchAccess(path$
|
|
3516
|
+
let { messages, upToDate } = this.store.read(path$1, effectiveOffset);
|
|
3517
|
+
this.store.touchAccess(path$1);
|
|
2497
3518
|
const clientIsCaughtUp = effectiveOffset && effectiveOffset === stream.currentOffset || offset === `now`;
|
|
2498
3519
|
if (live === `long-poll` && clientIsCaughtUp && messages.length === 0) {
|
|
2499
3520
|
if (stream.closed) {
|
|
@@ -2505,8 +3526,8 @@ var DurableStreamTestServer = class {
|
|
|
2505
3526
|
res.end();
|
|
2506
3527
|
return;
|
|
2507
3528
|
}
|
|
2508
|
-
const result = await this.store.waitForMessages(path$
|
|
2509
|
-
this.store.touchAccess(path$
|
|
3529
|
+
const result = await this.store.waitForMessages(path$1, effectiveOffset ?? stream.currentOffset, this.options.longPollTimeout);
|
|
3530
|
+
this.store.touchAccess(path$1);
|
|
2510
3531
|
if (result.streamClosed) {
|
|
2511
3532
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2512
3533
|
res.writeHead(204, {
|
|
@@ -2520,7 +3541,7 @@ var DurableStreamTestServer = class {
|
|
|
2520
3541
|
}
|
|
2521
3542
|
if (result.timedOut) {
|
|
2522
3543
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2523
|
-
const currentStream$1 = this.store.get(path$
|
|
3544
|
+
const currentStream$1 = this.store.get(path$1);
|
|
2524
3545
|
const timeoutHeaders = {
|
|
2525
3546
|
[STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
|
|
2526
3547
|
[STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
@@ -2541,12 +3562,12 @@ var DurableStreamTestServer = class {
|
|
|
2541
3562
|
headers[STREAM_OFFSET_HEADER] = responseOffset;
|
|
2542
3563
|
if (live === `long-poll`) headers[STREAM_CURSOR_HEADER] = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2543
3564
|
if (upToDate) headers[STREAM_UP_TO_DATE_HEADER] = `true`;
|
|
2544
|
-
const currentStream = this.store.get(path$
|
|
3565
|
+
const currentStream = this.store.get(path$1);
|
|
2545
3566
|
const clientAtTail = responseOffset === currentStream?.currentOffset;
|
|
2546
3567
|
if (currentStream?.closed && clientAtTail && upToDate) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
2547
3568
|
const startOffset = offset ?? `-1`;
|
|
2548
3569
|
const closedSuffix = currentStream?.closed && clientAtTail && upToDate ? `:c` : ``;
|
|
2549
|
-
const etag = `"${Buffer.from(path$
|
|
3570
|
+
const etag = `"${Buffer.from(path$1).toString(`base64`)}:${startOffset}:${responseOffset}${closedSuffix}"`;
|
|
2550
3571
|
headers[`etag`] = etag;
|
|
2551
3572
|
const ifNoneMatch = req.headers[`if-none-match`];
|
|
2552
3573
|
if (ifNoneMatch && ifNoneMatch === etag) {
|
|
@@ -2554,7 +3575,7 @@ var DurableStreamTestServer = class {
|
|
|
2554
3575
|
res.end();
|
|
2555
3576
|
return;
|
|
2556
3577
|
}
|
|
2557
|
-
const responseData = this.store.formatResponse(path$
|
|
3578
|
+
const responseData = this.store.formatResponse(path$1, messages);
|
|
2558
3579
|
let finalData = responseData;
|
|
2559
3580
|
if (this.options.compression && responseData.length >= COMPRESSION_THRESHOLD) {
|
|
2560
3581
|
const acceptEncoding = req.headers[`accept-encoding`];
|
|
@@ -2572,7 +3593,7 @@ var DurableStreamTestServer = class {
|
|
|
2572
3593
|
/**
|
|
2573
3594
|
* Handle SSE (Server-Sent Events) mode
|
|
2574
3595
|
*/
|
|
2575
|
-
async handleSSE(path$
|
|
3596
|
+
async handleSSE(path$1, stream, initialOffset, cursor, useBase64, res) {
|
|
2576
3597
|
this.activeSSEResponses.add(res);
|
|
2577
3598
|
const sseHeaders = {
|
|
2578
3599
|
"content-type": `text/event-stream`,
|
|
@@ -2598,20 +3619,20 @@ var DurableStreamTestServer = class {
|
|
|
2598
3619
|
});
|
|
2599
3620
|
const isJsonStream = stream?.contentType?.includes(`application/json`);
|
|
2600
3621
|
while (isConnected && !this.isShuttingDown) {
|
|
2601
|
-
const { messages, upToDate } = this.store.read(path$
|
|
2602
|
-
this.store.touchAccess(path$
|
|
3622
|
+
const { messages, upToDate } = this.store.read(path$1, currentOffset);
|
|
3623
|
+
this.store.touchAccess(path$1);
|
|
2603
3624
|
for (const message of messages) {
|
|
2604
3625
|
let dataPayload;
|
|
2605
3626
|
if (useBase64) dataPayload = Buffer.from(message.data).toString(`base64`);
|
|
2606
3627
|
else if (isJsonStream) {
|
|
2607
|
-
const jsonBytes = this.store.formatResponse(path$
|
|
3628
|
+
const jsonBytes = this.store.formatResponse(path$1, [message]);
|
|
2608
3629
|
dataPayload = decoder.decode(jsonBytes);
|
|
2609
3630
|
} else dataPayload = decoder.decode(message.data);
|
|
2610
3631
|
res.write(`event: data\n`);
|
|
2611
3632
|
res.write(encodeSSEData(dataPayload));
|
|
2612
3633
|
currentOffset = message.offset;
|
|
2613
3634
|
}
|
|
2614
|
-
const currentStream = this.store.get(path$
|
|
3635
|
+
const currentStream = this.store.get(path$1);
|
|
2615
3636
|
const controlOffset = messages[messages.length - 1]?.offset ?? currentStream.currentOffset;
|
|
2616
3637
|
const streamIsClosed = currentStream?.closed ?? false;
|
|
2617
3638
|
const clientAtTail = controlOffset === currentStream.currentOffset;
|
|
@@ -2636,8 +3657,8 @@ var DurableStreamTestServer = class {
|
|
|
2636
3657
|
res.write(encodeSSEData(JSON.stringify(finalControlData)));
|
|
2637
3658
|
break;
|
|
2638
3659
|
}
|
|
2639
|
-
const result = await this.store.waitForMessages(path$
|
|
2640
|
-
this.store.touchAccess(path$
|
|
3660
|
+
const result = await this.store.waitForMessages(path$1, currentOffset, this.options.longPollTimeout);
|
|
3661
|
+
this.store.touchAccess(path$1);
|
|
2641
3662
|
if (this.isShuttingDown || !isConnected) break;
|
|
2642
3663
|
if (result.streamClosed && result.messages.length === 0) {
|
|
2643
3664
|
const finalControlData = {
|
|
@@ -2650,7 +3671,7 @@ var DurableStreamTestServer = class {
|
|
|
2650
3671
|
}
|
|
2651
3672
|
if (result.timedOut) {
|
|
2652
3673
|
const keepAliveCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2653
|
-
const streamAfterWait = this.store.get(path$
|
|
3674
|
+
const streamAfterWait = this.store.get(path$1);
|
|
2654
3675
|
if (streamAfterWait?.closed) {
|
|
2655
3676
|
const closedControlData = {
|
|
2656
3677
|
[SSE_OFFSET_FIELD]: currentOffset,
|
|
@@ -2675,7 +3696,7 @@ var DurableStreamTestServer = class {
|
|
|
2675
3696
|
/**
|
|
2676
3697
|
* Handle POST - append data
|
|
2677
3698
|
*/
|
|
2678
|
-
async handleAppend(path$
|
|
3699
|
+
async handleAppend(path$1, req, res) {
|
|
2679
3700
|
const contentType = req.headers[`content-type`];
|
|
2680
3701
|
const seq = req.headers[STREAM_SEQ_HEADER.toLowerCase()];
|
|
2681
3702
|
const closedHeader = req.headers[STREAM_CLOSED_HEADER.toLowerCase()];
|
|
@@ -2725,7 +3746,7 @@ var DurableStreamTestServer = class {
|
|
|
2725
3746
|
const body = await this.readBody(req);
|
|
2726
3747
|
if (body.length === 0 && closeStream) {
|
|
2727
3748
|
if (hasAllProducerHeaders) {
|
|
2728
|
-
const closeResult$1 = await this.store.closeStreamWithProducer(path$
|
|
3749
|
+
const closeResult$1 = await this.store.closeStreamWithProducer(path$1, {
|
|
2729
3750
|
producerId,
|
|
2730
3751
|
producerEpoch,
|
|
2731
3752
|
producerSeq
|
|
@@ -2768,7 +3789,7 @@ var DurableStreamTestServer = class {
|
|
|
2768
3789
|
return;
|
|
2769
3790
|
}
|
|
2770
3791
|
if (closeResult$1.producerResult?.status === `stream_closed`) {
|
|
2771
|
-
const stream = this.store.get(path$
|
|
3792
|
+
const stream = this.store.get(path$1);
|
|
2772
3793
|
res.writeHead(409, {
|
|
2773
3794
|
"content-type": `text/plain`,
|
|
2774
3795
|
[STREAM_CLOSED_HEADER]: `true`,
|
|
@@ -2786,7 +3807,7 @@ var DurableStreamTestServer = class {
|
|
|
2786
3807
|
res.end();
|
|
2787
3808
|
return;
|
|
2788
3809
|
}
|
|
2789
|
-
const closeResult = this.store.closeStream(path$
|
|
3810
|
+
const closeResult = this.store.closeStream(path$1);
|
|
2790
3811
|
if (!closeResult) {
|
|
2791
3812
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
2792
3813
|
res.end(`Stream not found`);
|
|
@@ -2818,14 +3839,14 @@ var DurableStreamTestServer = class {
|
|
|
2818
3839
|
close: closeStream
|
|
2819
3840
|
};
|
|
2820
3841
|
let result;
|
|
2821
|
-
if (producerId !== void 0) result = await this.store.appendWithProducer(path$
|
|
2822
|
-
else result = await Promise.resolve(this.store.append(path$
|
|
2823
|
-
this.store.touchAccess(path$
|
|
3842
|
+
if (producerId !== void 0) result = await this.store.appendWithProducer(path$1, body, appendOptions);
|
|
3843
|
+
else result = await Promise.resolve(this.store.append(path$1, body, appendOptions));
|
|
3844
|
+
this.store.touchAccess(path$1);
|
|
2824
3845
|
if (result && typeof result === `object` && `message` in result) {
|
|
2825
3846
|
const { message: message$1, producerResult, streamClosed } = result;
|
|
2826
3847
|
if (streamClosed && !message$1) {
|
|
2827
3848
|
if (producerResult?.status === `duplicate`) {
|
|
2828
|
-
const stream = this.store.get(path$
|
|
3849
|
+
const stream = this.store.get(path$1);
|
|
2829
3850
|
res.writeHead(204, {
|
|
2830
3851
|
[STREAM_OFFSET_HEADER]: stream?.currentOffset ?? ``,
|
|
2831
3852
|
[STREAM_CLOSED_HEADER]: `true`,
|
|
@@ -2835,7 +3856,7 @@ var DurableStreamTestServer = class {
|
|
|
2835
3856
|
res.end();
|
|
2836
3857
|
return;
|
|
2837
3858
|
}
|
|
2838
|
-
const closedStream = this.store.get(path$
|
|
3859
|
+
const closedStream = this.store.get(path$1);
|
|
2839
3860
|
res.writeHead(409, {
|
|
2840
3861
|
"content-type": `text/plain`,
|
|
2841
3862
|
[STREAM_CLOSED_HEADER]: `true`,
|
|
@@ -2852,6 +3873,7 @@ var DurableStreamTestServer = class {
|
|
|
2852
3873
|
const statusCode = producerId !== void 0 ? 200 : 204;
|
|
2853
3874
|
res.writeHead(statusCode, responseHeaders$1);
|
|
2854
3875
|
res.end();
|
|
3876
|
+
await this.notifyStreamAppend(path$1);
|
|
2855
3877
|
return;
|
|
2856
3878
|
}
|
|
2857
3879
|
switch (producerResult.status) {
|
|
@@ -2892,18 +3914,27 @@ var DurableStreamTestServer = class {
|
|
|
2892
3914
|
if (closeStream) responseHeaders[STREAM_CLOSED_HEADER] = `true`;
|
|
2893
3915
|
res.writeHead(204, responseHeaders);
|
|
2894
3916
|
res.end();
|
|
3917
|
+
await this.notifyStreamAppend(path$1);
|
|
3918
|
+
}
|
|
3919
|
+
async notifyStreamAppend(path$1) {
|
|
3920
|
+
if (!this.subscriptionManager) return;
|
|
3921
|
+
try {
|
|
3922
|
+
await this.subscriptionManager.onStreamAppend(path$1);
|
|
3923
|
+
} catch (err) {
|
|
3924
|
+
serverLog.error(`[server] subscription append hook failed:`, err);
|
|
3925
|
+
}
|
|
2895
3926
|
}
|
|
2896
3927
|
/**
|
|
2897
3928
|
* Handle DELETE - delete stream
|
|
2898
3929
|
*/
|
|
2899
|
-
async handleDelete(path$
|
|
2900
|
-
const existing = this.store.get(path$
|
|
3930
|
+
async handleDelete(path$1, res) {
|
|
3931
|
+
const existing = this.store.get(path$1);
|
|
2901
3932
|
if (existing?.softDeleted) {
|
|
2902
3933
|
res.writeHead(410, { "content-type": `text/plain` });
|
|
2903
3934
|
res.end(`Stream is gone`);
|
|
2904
3935
|
return;
|
|
2905
3936
|
}
|
|
2906
|
-
const deleted = this.store.delete(path$
|
|
3937
|
+
const deleted = this.store.delete(path$1);
|
|
2907
3938
|
if (!deleted) {
|
|
2908
3939
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
2909
3940
|
res.end(`Stream not found`);
|
|
@@ -2911,9 +3942,10 @@ var DurableStreamTestServer = class {
|
|
|
2911
3942
|
}
|
|
2912
3943
|
if (this.options.onStreamDeleted) await Promise.resolve(this.options.onStreamDeleted({
|
|
2913
3944
|
type: `deleted`,
|
|
2914
|
-
path: path$
|
|
3945
|
+
path: path$1,
|
|
2915
3946
|
timestamp: Date.now()
|
|
2916
3947
|
}));
|
|
3948
|
+
if (this.subscriptionManager) this.subscriptionManager.onStreamDeleted(path$1);
|
|
2917
3949
|
res.writeHead(204);
|
|
2918
3950
|
res.end();
|
|
2919
3951
|
}
|
|
@@ -3043,4 +4075,4 @@ function createRegistryHooks(store, serverUrl) {
|
|
|
3043
4075
|
}
|
|
3044
4076
|
|
|
3045
4077
|
//#endregion
|
|
3046
|
-
export { DEFAULT_CURSOR_EPOCH, DEFAULT_CURSOR_INTERVAL_SECONDS, DurableStreamTestServer, FileBackedStreamStore, StreamStore, calculateCursor, createRegistryHooks, decodeStreamPath, encodeStreamPath, generateResponseCursor, handleCursorCollision };
|
|
4078
|
+
export { DEFAULT_CURSOR_EPOCH, DEFAULT_CURSOR_INTERVAL_SECONDS, DurableStreamTestServer, FileBackedStreamStore, StreamStore, SubscriptionManager, SubscriptionRoutes, calculateCursor, createRegistryHooks, decodeStreamPath, encodeStreamPath, generateResponseCursor, globMatch, handleCursorCollision, validateWebhookUrl };
|