@durable-streams/server 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1344 -266
- package/dist/index.d.cts +258 -2
- package/dist/index.d.ts +258 -2
- package/dist/index.js +1391 -318
- package/package.json +4 -4
- package/src/crypto.ts +217 -0
- package/src/file-store.ts +239 -144
- package/src/glob.ts +70 -0
- package/src/index.ts +14 -0
- package/src/log.ts +56 -0
- package/src/server.ts +96 -40
- package/src/store.ts +66 -10
- package/src/subscription-manager.ts +882 -0
- package/src/subscription-routes.ts +504 -0
- package/src/subscription-types.ts +80 -0
- package/src/types.ts +8 -0
package/dist/index.js
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { createServer } from "node:http";
|
|
2
2
|
import { deflateSync, gzipSync } from "node:zlib";
|
|
3
|
+
import { CURSOR_QUERY_PARAM, DurableStream, LIVE_QUERY_PARAM, OFFSET_QUERY_PARAM, PRODUCER_EPOCH_HEADER, PRODUCER_EXPECTED_SEQ_HEADER, PRODUCER_ID_HEADER, PRODUCER_RECEIVED_SEQ_HEADER, PRODUCER_SEQ_HEADER, SSE_CLOSED_FIELD, SSE_CURSOR_FIELD, SSE_OFFSET_FIELD, STREAM_CLOSED_HEADER, STREAM_CURSOR_HEADER, STREAM_EXPIRES_AT_HEADER, STREAM_OFFSET_HEADER, STREAM_SEQ_HEADER, STREAM_TTL_HEADER, STREAM_UP_TO_DATE_HEADER } from "@durable-streams/client";
|
|
3
4
|
import * as fs from "node:fs";
|
|
4
|
-
import * as path$1 from "node:path";
|
|
5
5
|
import * as path from "node:path";
|
|
6
|
-
import { createHash, randomBytes } from "node:crypto";
|
|
6
|
+
import { createHash, createHmac, generateKeyPairSync, randomBytes, sign, timingSafeEqual } from "node:crypto";
|
|
7
7
|
import { open } from "lmdb";
|
|
8
8
|
import { SieveCache } from "@neophi/sieve-cache";
|
|
9
|
-
import
|
|
10
|
-
import { DurableStream } from "@durable-streams/client";
|
|
9
|
+
import { isIP } from "node:net";
|
|
11
10
|
import { createStateSchema } from "@durable-streams/state";
|
|
12
11
|
|
|
13
12
|
//#region src/store.ts
|
|
@@ -50,17 +49,35 @@ function processJsonAppend(data, isInitialCreate = false) {
|
|
|
50
49
|
} else result = JSON.stringify(parsed) + `,`;
|
|
51
50
|
return new TextEncoder().encode(result);
|
|
52
51
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
* Strips trailing comma before wrapping.
|
|
56
|
-
*/
|
|
57
|
-
function formatJsonResponse(data) {
|
|
58
|
-
if (data.length === 0) return new TextEncoder().encode(`[]`);
|
|
59
|
-
let text = new TextDecoder().decode(data);
|
|
60
|
-
text = text.trimEnd();
|
|
52
|
+
function decodeStoredJsonMessage(data) {
|
|
53
|
+
let text = new TextDecoder().decode(data).trimEnd();
|
|
61
54
|
if (text.endsWith(`,`)) text = text.slice(0, -1);
|
|
62
|
-
|
|
63
|
-
|
|
55
|
+
return text;
|
|
56
|
+
}
|
|
57
|
+
function enrichJsonValueWithOffset(parsed, offset) {
|
|
58
|
+
if (!parsed || typeof parsed !== `object` || Array.isArray(parsed)) return JSON.stringify(parsed);
|
|
59
|
+
const candidate = parsed;
|
|
60
|
+
const headers = candidate.headers;
|
|
61
|
+
if (!headers || typeof headers !== `object`) return JSON.stringify(parsed);
|
|
62
|
+
const isStateChange = typeof headers.operation === `string`;
|
|
63
|
+
const isStateControl = typeof headers.control === `string`;
|
|
64
|
+
if (!isStateChange && !isStateControl) return JSON.stringify(parsed);
|
|
65
|
+
return JSON.stringify({
|
|
66
|
+
...candidate,
|
|
67
|
+
headers: {
|
|
68
|
+
...headers,
|
|
69
|
+
offset
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
function formatJsonMessages(messages) {
|
|
74
|
+
if (messages.length === 0) return new TextEncoder().encode(`[]`);
|
|
75
|
+
const items = messages.flatMap((message) => {
|
|
76
|
+
const rawFragment = decodeStoredJsonMessage(message.data);
|
|
77
|
+
const parsed = JSON.parse(`[${rawFragment}]`);
|
|
78
|
+
return parsed.map((value) => enrichJsonValueWithOffset(value, message.offset));
|
|
79
|
+
});
|
|
80
|
+
return new TextEncoder().encode(`[${items.join(`,`)}]`);
|
|
64
81
|
}
|
|
65
82
|
var StreamStore = class {
|
|
66
83
|
streams = new Map();
|
|
@@ -90,15 +107,15 @@ var StreamStore = class {
|
|
|
90
107
|
* Returns undefined if stream doesn't exist or is expired (and has no refs).
|
|
91
108
|
* Expired streams with refCount > 0 are soft-deleted instead of fully deleted.
|
|
92
109
|
*/
|
|
93
|
-
getIfNotExpired(path$
|
|
94
|
-
const stream = this.streams.get(path$
|
|
110
|
+
getIfNotExpired(path$1) {
|
|
111
|
+
const stream = this.streams.get(path$1);
|
|
95
112
|
if (!stream) return void 0;
|
|
96
113
|
if (this.isExpired(stream)) {
|
|
97
114
|
if (stream.refCount > 0) {
|
|
98
115
|
stream.softDeleted = true;
|
|
99
116
|
return stream;
|
|
100
117
|
}
|
|
101
|
-
this.delete(path$
|
|
118
|
+
this.delete(path$1);
|
|
102
119
|
return void 0;
|
|
103
120
|
}
|
|
104
121
|
return stream;
|
|
@@ -106,8 +123,8 @@ var StreamStore = class {
|
|
|
106
123
|
/**
|
|
107
124
|
* Update lastAccessedAt to now. Called on reads and appends (not HEAD).
|
|
108
125
|
*/
|
|
109
|
-
touchAccess(path$
|
|
110
|
-
const stream = this.streams.get(path$
|
|
126
|
+
touchAccess(path$1) {
|
|
127
|
+
const stream = this.streams.get(path$1);
|
|
111
128
|
if (stream) stream.lastAccessedAt = Date.now();
|
|
112
129
|
}
|
|
113
130
|
/**
|
|
@@ -116,12 +133,12 @@ var StreamStore = class {
|
|
|
116
133
|
* @throws Error if fork source not found, soft-deleted, or offset invalid
|
|
117
134
|
* @returns existing stream if config matches (idempotent)
|
|
118
135
|
*/
|
|
119
|
-
create(path$
|
|
120
|
-
const existingRaw = this.streams.get(path$
|
|
136
|
+
create(path$1, options = {}) {
|
|
137
|
+
const existingRaw = this.streams.get(path$1);
|
|
121
138
|
if (existingRaw) if (this.isExpired(existingRaw)) {
|
|
122
|
-
this.streams.delete(path$
|
|
123
|
-
this.cancelLongPollsForStream(path$
|
|
124
|
-
} else if (existingRaw.softDeleted) throw new Error(`Stream has active forks — path cannot be reused until all forks are removed: ${path$
|
|
139
|
+
this.streams.delete(path$1);
|
|
140
|
+
this.cancelLongPollsForStream(path$1);
|
|
141
|
+
} else if (existingRaw.softDeleted) throw new Error(`Stream has active forks — path cannot be reused until all forks are removed: ${path$1}`);
|
|
125
142
|
else {
|
|
126
143
|
const contentTypeMatches = (normalizeContentType(options.contentType) || `application/octet-stream`) === (normalizeContentType(existingRaw.contentType) || `application/octet-stream`);
|
|
127
144
|
const ttlMatches = options.ttlSeconds === existingRaw.ttlSeconds;
|
|
@@ -130,7 +147,7 @@ var StreamStore = class {
|
|
|
130
147
|
const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
|
|
131
148
|
const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
|
|
132
149
|
if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches) return existingRaw;
|
|
133
|
-
else throw new Error(`Stream already exists with different configuration: ${path$
|
|
150
|
+
else throw new Error(`Stream already exists with different configuration: ${path$1}`);
|
|
134
151
|
}
|
|
135
152
|
const isFork = !!options.forkedFrom;
|
|
136
153
|
let forkOffset = `0000000000000000_0000000000000000`;
|
|
@@ -160,7 +177,7 @@ var StreamStore = class {
|
|
|
160
177
|
effectiveTtlSeconds = resolved.ttlSeconds;
|
|
161
178
|
}
|
|
162
179
|
const stream = {
|
|
163
|
-
path: path$
|
|
180
|
+
path: path$1,
|
|
164
181
|
contentType,
|
|
165
182
|
messages: [],
|
|
166
183
|
currentOffset: isFork ? forkOffset : `0000000000000000_0000000000000000`,
|
|
@@ -179,7 +196,7 @@ var StreamStore = class {
|
|
|
179
196
|
if (isFork && sourceStream) sourceStream.refCount--;
|
|
180
197
|
throw err;
|
|
181
198
|
}
|
|
182
|
-
this.streams.set(path$
|
|
199
|
+
this.streams.set(path$1, stream);
|
|
183
200
|
return stream;
|
|
184
201
|
}
|
|
185
202
|
/**
|
|
@@ -198,15 +215,15 @@ var StreamStore = class {
|
|
|
198
215
|
* Returns undefined if stream doesn't exist or is expired.
|
|
199
216
|
* Returns soft-deleted streams (caller should check stream.softDeleted).
|
|
200
217
|
*/
|
|
201
|
-
get(path$
|
|
202
|
-
const stream = this.streams.get(path$
|
|
218
|
+
get(path$1) {
|
|
219
|
+
const stream = this.streams.get(path$1);
|
|
203
220
|
if (!stream) return void 0;
|
|
204
221
|
if (this.isExpired(stream)) {
|
|
205
222
|
if (stream.refCount > 0) {
|
|
206
223
|
stream.softDeleted = true;
|
|
207
224
|
return stream;
|
|
208
225
|
}
|
|
209
|
-
this.delete(path$
|
|
226
|
+
this.delete(path$1);
|
|
210
227
|
return void 0;
|
|
211
228
|
}
|
|
212
229
|
return stream;
|
|
@@ -214,8 +231,8 @@ var StreamStore = class {
|
|
|
214
231
|
/**
|
|
215
232
|
* Check if a stream exists, is not expired, and is not soft-deleted.
|
|
216
233
|
*/
|
|
217
|
-
has(path$
|
|
218
|
-
const stream = this.get(path$
|
|
234
|
+
has(path$1) {
|
|
235
|
+
const stream = this.get(path$1);
|
|
219
236
|
if (!stream) return false;
|
|
220
237
|
if (stream.softDeleted) return false;
|
|
221
238
|
return true;
|
|
@@ -225,27 +242,27 @@ var StreamStore = class {
|
|
|
225
242
|
* If the stream has forks (refCount > 0), it is soft-deleted instead of fully removed.
|
|
226
243
|
* Returns true if the stream was found and deleted (or soft-deleted).
|
|
227
244
|
*/
|
|
228
|
-
delete(path$
|
|
229
|
-
const stream = this.streams.get(path$
|
|
245
|
+
delete(path$1) {
|
|
246
|
+
const stream = this.streams.get(path$1);
|
|
230
247
|
if (!stream) return false;
|
|
231
248
|
if (stream.softDeleted) return true;
|
|
232
249
|
if (stream.refCount > 0) {
|
|
233
250
|
stream.softDeleted = true;
|
|
234
251
|
return true;
|
|
235
252
|
}
|
|
236
|
-
this.deleteWithCascade(path$
|
|
253
|
+
this.deleteWithCascade(path$1);
|
|
237
254
|
return true;
|
|
238
255
|
}
|
|
239
256
|
/**
|
|
240
257
|
* Fully delete a stream and cascade to soft-deleted parents
|
|
241
258
|
* whose refcount drops to zero.
|
|
242
259
|
*/
|
|
243
|
-
deleteWithCascade(path$
|
|
244
|
-
const stream = this.streams.get(path$
|
|
260
|
+
deleteWithCascade(path$1) {
|
|
261
|
+
const stream = this.streams.get(path$1);
|
|
245
262
|
if (!stream) return;
|
|
246
263
|
const forkedFrom = stream.forkedFrom;
|
|
247
|
-
this.streams.delete(path$
|
|
248
|
-
this.cancelLongPollsForStream(path$
|
|
264
|
+
this.streams.delete(path$1);
|
|
265
|
+
this.cancelLongPollsForStream(path$1);
|
|
249
266
|
if (forkedFrom) {
|
|
250
267
|
const parent = this.streams.get(forkedFrom);
|
|
251
268
|
if (parent) {
|
|
@@ -344,8 +361,8 @@ var StreamStore = class {
|
|
|
344
361
|
* Acquire a lock for serialized producer operations.
|
|
345
362
|
* Returns a release function.
|
|
346
363
|
*/
|
|
347
|
-
async acquireProducerLock(path$
|
|
348
|
-
const lockKey = `${path$
|
|
364
|
+
async acquireProducerLock(path$1, producerId) {
|
|
365
|
+
const lockKey = `${path$1}:${producerId}`;
|
|
349
366
|
while (this.producerLocks.has(lockKey)) await this.producerLocks.get(lockKey);
|
|
350
367
|
let releaseLock;
|
|
351
368
|
const lockPromise = new Promise((resolve) => {
|
|
@@ -363,10 +380,10 @@ var StreamStore = class {
|
|
|
363
380
|
* @throws Error if seq is lower than lastSeq
|
|
364
381
|
* @throws Error if JSON mode and array is empty
|
|
365
382
|
*/
|
|
366
|
-
append(path$
|
|
367
|
-
const stream = this.getIfNotExpired(path$
|
|
368
|
-
if (!stream) throw new Error(`Stream not found: ${path$
|
|
369
|
-
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path$
|
|
383
|
+
append(path$1, data, options = {}) {
|
|
384
|
+
const stream = this.getIfNotExpired(path$1);
|
|
385
|
+
if (!stream) throw new Error(`Stream not found: ${path$1}`);
|
|
386
|
+
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path$1}`);
|
|
370
387
|
if (stream.closed) {
|
|
371
388
|
if (options.producerId && stream.closedBy && stream.closedBy.producerId === options.producerId && stream.closedBy.epoch === options.producerEpoch && stream.closedBy.seq === options.producerSeq) return {
|
|
372
389
|
message: null,
|
|
@@ -407,9 +424,9 @@ var StreamStore = class {
|
|
|
407
424
|
epoch: options.producerEpoch,
|
|
408
425
|
seq: options.producerSeq
|
|
409
426
|
};
|
|
410
|
-
this.notifyLongPollsClosed(path$2);
|
|
411
427
|
}
|
|
412
|
-
this.notifyLongPolls(path$
|
|
428
|
+
this.notifyLongPolls(path$1);
|
|
429
|
+
if (options.close) this.notifyLongPollsClosed(path$1);
|
|
413
430
|
if (producerResult || options.close) return {
|
|
414
431
|
message,
|
|
415
432
|
producerResult,
|
|
@@ -421,15 +438,15 @@ var StreamStore = class {
|
|
|
421
438
|
* Append with producer serialization for concurrent request handling.
|
|
422
439
|
* This ensures that validation+append is atomic per producer.
|
|
423
440
|
*/
|
|
424
|
-
async appendWithProducer(path$
|
|
441
|
+
async appendWithProducer(path$1, data, options) {
|
|
425
442
|
if (!options.producerId) {
|
|
426
|
-
const result = this.append(path$
|
|
443
|
+
const result = this.append(path$1, data, options);
|
|
427
444
|
if (`message` in result) return result;
|
|
428
445
|
return { message: result };
|
|
429
446
|
}
|
|
430
|
-
const releaseLock = await this.acquireProducerLock(path$
|
|
447
|
+
const releaseLock = await this.acquireProducerLock(path$1, options.producerId);
|
|
431
448
|
try {
|
|
432
|
-
const result = this.append(path$
|
|
449
|
+
const result = this.append(path$1, data, options);
|
|
433
450
|
if (`message` in result) return result;
|
|
434
451
|
return { message: result };
|
|
435
452
|
} finally {
|
|
@@ -440,13 +457,13 @@ var StreamStore = class {
|
|
|
440
457
|
* Close a stream without appending data.
|
|
441
458
|
* @returns The final offset, or null if stream doesn't exist
|
|
442
459
|
*/
|
|
443
|
-
closeStream(path$
|
|
444
|
-
const stream = this.getIfNotExpired(path$
|
|
460
|
+
closeStream(path$1) {
|
|
461
|
+
const stream = this.getIfNotExpired(path$1);
|
|
445
462
|
if (!stream) return null;
|
|
446
|
-
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path$
|
|
463
|
+
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path$1}`);
|
|
447
464
|
const alreadyClosed = stream.closed ?? false;
|
|
448
465
|
stream.closed = true;
|
|
449
|
-
this.notifyLongPollsClosed(path$
|
|
466
|
+
this.notifyLongPollsClosed(path$1);
|
|
450
467
|
return {
|
|
451
468
|
finalOffset: stream.currentOffset,
|
|
452
469
|
alreadyClosed
|
|
@@ -457,10 +474,10 @@ var StreamStore = class {
|
|
|
457
474
|
* Participates in producer sequencing for deduplication.
|
|
458
475
|
* @returns The final offset and producer result, or null if stream doesn't exist
|
|
459
476
|
*/
|
|
460
|
-
async closeStreamWithProducer(path$
|
|
461
|
-
const releaseLock = await this.acquireProducerLock(path$
|
|
477
|
+
async closeStreamWithProducer(path$1, options) {
|
|
478
|
+
const releaseLock = await this.acquireProducerLock(path$1, options.producerId);
|
|
462
479
|
try {
|
|
463
|
-
const stream = this.getIfNotExpired(path$
|
|
480
|
+
const stream = this.getIfNotExpired(path$1);
|
|
464
481
|
if (!stream) return null;
|
|
465
482
|
if (stream.closed) {
|
|
466
483
|
if (stream.closedBy && stream.closedBy.producerId === options.producerId && stream.closedBy.epoch === options.producerEpoch && stream.closedBy.seq === options.producerSeq) return {
|
|
@@ -490,7 +507,7 @@ var StreamStore = class {
|
|
|
490
507
|
epoch: options.producerEpoch,
|
|
491
508
|
seq: options.producerSeq
|
|
492
509
|
};
|
|
493
|
-
this.notifyLongPollsClosed(path$
|
|
510
|
+
this.notifyLongPollsClosed(path$1);
|
|
494
511
|
return {
|
|
495
512
|
finalOffset: stream.currentOffset,
|
|
496
513
|
alreadyClosed: false,
|
|
@@ -504,8 +521,8 @@ var StreamStore = class {
|
|
|
504
521
|
* Get the current epoch for a producer on a stream.
|
|
505
522
|
* Returns undefined if the producer doesn't exist or stream not found.
|
|
506
523
|
*/
|
|
507
|
-
getProducerEpoch(path$
|
|
508
|
-
const stream = this.getIfNotExpired(path$
|
|
524
|
+
getProducerEpoch(path$1, producerId) {
|
|
525
|
+
const stream = this.getIfNotExpired(path$1);
|
|
509
526
|
if (!stream?.producers) return void 0;
|
|
510
527
|
return stream.producers.get(producerId)?.epoch;
|
|
511
528
|
}
|
|
@@ -514,9 +531,9 @@ var StreamStore = class {
|
|
|
514
531
|
* For forked streams, stitches messages from the source chain and the fork's own messages.
|
|
515
532
|
* @throws Error if stream doesn't exist or is expired
|
|
516
533
|
*/
|
|
517
|
-
read(path$
|
|
518
|
-
const stream = this.getIfNotExpired(path$
|
|
519
|
-
if (!stream) throw new Error(`Stream not found: ${path$
|
|
534
|
+
read(path$1, offset) {
|
|
535
|
+
const stream = this.getIfNotExpired(path$1);
|
|
536
|
+
if (!stream) throw new Error(`Stream not found: ${path$1}`);
|
|
520
537
|
if (!offset || offset === `-1`) {
|
|
521
538
|
if (stream.forkedFrom) {
|
|
522
539
|
const inherited = this.readForkedMessages(stream.forkedFrom, void 0, stream.forkOffset);
|
|
@@ -595,9 +612,10 @@ var StreamStore = class {
|
|
|
595
612
|
* For JSON mode, wraps concatenated data in array brackets.
|
|
596
613
|
* @throws Error if stream doesn't exist or is expired
|
|
597
614
|
*/
|
|
598
|
-
formatResponse(path$
|
|
599
|
-
const stream = this.getIfNotExpired(path$
|
|
600
|
-
if (!stream) throw new Error(`Stream not found: ${path$
|
|
615
|
+
formatResponse(path$1, messages) {
|
|
616
|
+
const stream = this.getIfNotExpired(path$1);
|
|
617
|
+
if (!stream) throw new Error(`Stream not found: ${path$1}`);
|
|
618
|
+
if (normalizeContentType(stream.contentType) === `application/json`) return formatJsonMessages(messages);
|
|
601
619
|
const totalSize = messages.reduce((sum, m) => sum + m.data.length, 0);
|
|
602
620
|
const concatenated = new Uint8Array(totalSize);
|
|
603
621
|
let offset = 0;
|
|
@@ -605,24 +623,23 @@ var StreamStore = class {
|
|
|
605
623
|
concatenated.set(msg.data, offset);
|
|
606
624
|
offset += msg.data.length;
|
|
607
625
|
}
|
|
608
|
-
if (normalizeContentType(stream.contentType) === `application/json`) return formatJsonResponse(concatenated);
|
|
609
626
|
return concatenated;
|
|
610
627
|
}
|
|
611
628
|
/**
|
|
612
629
|
* Wait for new messages (long-poll).
|
|
613
630
|
* @throws Error if stream doesn't exist or is expired
|
|
614
631
|
*/
|
|
615
|
-
async waitForMessages(path$
|
|
616
|
-
const stream = this.getIfNotExpired(path$
|
|
617
|
-
if (!stream) throw new Error(`Stream not found: ${path$
|
|
632
|
+
async waitForMessages(path$1, offset, timeoutMs) {
|
|
633
|
+
const stream = this.getIfNotExpired(path$1);
|
|
634
|
+
if (!stream) throw new Error(`Stream not found: ${path$1}`);
|
|
618
635
|
if (stream.forkedFrom && offset < stream.forkOffset) {
|
|
619
|
-
const { messages: messages$1 } = this.read(path$
|
|
636
|
+
const { messages: messages$1 } = this.read(path$1, offset);
|
|
620
637
|
return {
|
|
621
638
|
messages: messages$1,
|
|
622
639
|
timedOut: false
|
|
623
640
|
};
|
|
624
641
|
}
|
|
625
|
-
const { messages } = this.read(path$
|
|
642
|
+
const { messages } = this.read(path$1, offset);
|
|
626
643
|
if (messages.length > 0) return {
|
|
627
644
|
messages,
|
|
628
645
|
timedOut: false
|
|
@@ -635,7 +652,7 @@ var StreamStore = class {
|
|
|
635
652
|
return new Promise((resolve) => {
|
|
636
653
|
const timeoutId = setTimeout(() => {
|
|
637
654
|
this.removePendingLongPoll(pending);
|
|
638
|
-
const currentStream = this.getIfNotExpired(path$
|
|
655
|
+
const currentStream = this.getIfNotExpired(path$1);
|
|
639
656
|
const streamClosed = currentStream?.closed ?? false;
|
|
640
657
|
resolve({
|
|
641
658
|
messages: [],
|
|
@@ -644,12 +661,12 @@ var StreamStore = class {
|
|
|
644
661
|
});
|
|
645
662
|
}, timeoutMs);
|
|
646
663
|
const pending = {
|
|
647
|
-
path: path$
|
|
664
|
+
path: path$1,
|
|
648
665
|
offset,
|
|
649
666
|
resolve: (msgs) => {
|
|
650
667
|
clearTimeout(timeoutId);
|
|
651
668
|
this.removePendingLongPoll(pending);
|
|
652
|
-
const currentStream = this.getIfNotExpired(path$
|
|
669
|
+
const currentStream = this.getIfNotExpired(path$1);
|
|
653
670
|
const streamClosed = currentStream?.closed && msgs.length === 0 ? true : void 0;
|
|
654
671
|
resolve({
|
|
655
672
|
messages: msgs,
|
|
@@ -666,8 +683,8 @@ var StreamStore = class {
|
|
|
666
683
|
* Get the current offset for a stream.
|
|
667
684
|
* Returns undefined if stream doesn't exist or is expired.
|
|
668
685
|
*/
|
|
669
|
-
getCurrentOffset(path$
|
|
670
|
-
return this.getIfNotExpired(path$
|
|
686
|
+
getCurrentOffset(path$1) {
|
|
687
|
+
return this.getIfNotExpired(path$1)?.currentOffset;
|
|
671
688
|
}
|
|
672
689
|
/**
|
|
673
690
|
* Clear all streams.
|
|
@@ -705,7 +722,8 @@ var StreamStore = class {
|
|
|
705
722
|
const parts = stream.currentOffset.split(`_`).map(Number);
|
|
706
723
|
const readSeq = parts[0];
|
|
707
724
|
const byteOffset = parts[1];
|
|
708
|
-
const
|
|
725
|
+
const FRAME_OVERHEAD = 5;
|
|
726
|
+
const newByteOffset = byteOffset + FRAME_OVERHEAD + processedData.length;
|
|
709
727
|
const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
710
728
|
const message = {
|
|
711
729
|
data: processedData,
|
|
@@ -720,10 +738,10 @@ var StreamStore = class {
|
|
|
720
738
|
for (let i = 0; i < stream.messages.length; i++) if (stream.messages[i].offset > offset) return i;
|
|
721
739
|
return -1;
|
|
722
740
|
}
|
|
723
|
-
notifyLongPolls(path$
|
|
724
|
-
const toNotify = this.pendingLongPolls.filter((p) => p.path === path$
|
|
741
|
+
notifyLongPolls(path$1) {
|
|
742
|
+
const toNotify = this.pendingLongPolls.filter((p) => p.path === path$1);
|
|
725
743
|
for (const pending of toNotify) {
|
|
726
|
-
const { messages } = this.read(path$
|
|
744
|
+
const { messages } = this.read(path$1, pending.offset);
|
|
727
745
|
if (messages.length > 0) pending.resolve(messages);
|
|
728
746
|
}
|
|
729
747
|
}
|
|
@@ -731,17 +749,17 @@ var StreamStore = class {
|
|
|
731
749
|
* Notify pending long-polls that a stream has been closed.
|
|
732
750
|
* They should wake up immediately and return Stream-Closed: true.
|
|
733
751
|
*/
|
|
734
|
-
notifyLongPollsClosed(path$
|
|
735
|
-
const toNotify = this.pendingLongPolls.filter((p) => p.path === path$
|
|
752
|
+
notifyLongPollsClosed(path$1) {
|
|
753
|
+
const toNotify = this.pendingLongPolls.filter((p) => p.path === path$1);
|
|
736
754
|
for (const pending of toNotify) pending.resolve([]);
|
|
737
755
|
}
|
|
738
|
-
cancelLongPollsForStream(path$
|
|
739
|
-
const toCancel = this.pendingLongPolls.filter((p) => p.path === path$
|
|
756
|
+
cancelLongPollsForStream(path$1) {
|
|
757
|
+
const toCancel = this.pendingLongPolls.filter((p) => p.path === path$1);
|
|
740
758
|
for (const pending of toCancel) {
|
|
741
759
|
clearTimeout(pending.timeoutId);
|
|
742
760
|
pending.resolve([]);
|
|
743
761
|
}
|
|
744
|
-
this.pendingLongPolls = this.pendingLongPolls.filter((p) => p.path !== path$
|
|
762
|
+
this.pendingLongPolls = this.pendingLongPolls.filter((p) => p.path !== path$1);
|
|
745
763
|
}
|
|
746
764
|
removePendingLongPoll(pending) {
|
|
747
765
|
const index = this.pendingLongPolls.indexOf(pending);
|
|
@@ -749,6 +767,48 @@ var StreamStore = class {
|
|
|
749
767
|
}
|
|
750
768
|
};
|
|
751
769
|
|
|
770
|
+
//#endregion
|
|
771
|
+
//#region src/log.ts
|
|
772
|
+
const streamsLogFile = process.env.STREAMS_LOG_FILE;
|
|
773
|
+
async function appendLogLine(line) {
|
|
774
|
+
if (!streamsLogFile) return;
|
|
775
|
+
const fs$1 = await import(`node:fs/promises`);
|
|
776
|
+
const path$1 = await import(`node:path`);
|
|
777
|
+
await fs$1.mkdir(path$1.dirname(streamsLogFile), { recursive: true });
|
|
778
|
+
await fs$1.appendFile(streamsLogFile, `${line}\n`);
|
|
779
|
+
}
|
|
780
|
+
function serializeArg(arg) {
|
|
781
|
+
if (arg instanceof Error) return arg.stack ?? arg.message;
|
|
782
|
+
if (typeof arg === `string`) return arg;
|
|
783
|
+
try {
|
|
784
|
+
return JSON.stringify(arg);
|
|
785
|
+
} catch {
|
|
786
|
+
return String(arg);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
function write(level, args) {
|
|
790
|
+
const line = args.map(serializeArg).join(` `);
|
|
791
|
+
const formatted = `[${level}] ${line}`;
|
|
792
|
+
if (level === `error`) console.error(formatted);
|
|
793
|
+
else if (level === `warn`) console.warn(formatted);
|
|
794
|
+
else console.info(formatted);
|
|
795
|
+
appendLogLine(formatted).catch(() => void 0);
|
|
796
|
+
}
|
|
797
|
+
const serverLog = {
|
|
798
|
+
info(...args) {
|
|
799
|
+
write(`info`, args);
|
|
800
|
+
},
|
|
801
|
+
warn(...args) {
|
|
802
|
+
write(`warn`, args);
|
|
803
|
+
},
|
|
804
|
+
error(...args) {
|
|
805
|
+
write(`error`, args);
|
|
806
|
+
},
|
|
807
|
+
event(obj, msg) {
|
|
808
|
+
write(`info`, [msg, obj]);
|
|
809
|
+
}
|
|
810
|
+
};
|
|
811
|
+
|
|
752
812
|
//#endregion
|
|
753
813
|
//#region src/path-encoding.ts
|
|
754
814
|
const MAX_ENCODED_LENGTH = 200;
|
|
@@ -759,10 +819,10 @@ const MAX_ENCODED_LENGTH = 200;
|
|
|
759
819
|
* @example
|
|
760
820
|
* encodeStreamPath("/stream/users:created") → "L3N0cmVhbS91c2VyczpjcmVhdGVk"
|
|
761
821
|
*/
|
|
762
|
-
function encodeStreamPath(path$
|
|
763
|
-
const base64 = Buffer.from(path$
|
|
822
|
+
function encodeStreamPath(path$1) {
|
|
823
|
+
const base64 = Buffer.from(path$1, `utf-8`).toString(`base64`).replace(/\+/g, `-`).replace(/\//g, `_`).replace(/=/g, ``);
|
|
764
824
|
if (base64.length > MAX_ENCODED_LENGTH) {
|
|
765
|
-
const hash = createHash(`sha256`).update(path$
|
|
825
|
+
const hash = createHash(`sha256`).update(path$1).digest(`hex`).slice(0, 16);
|
|
766
826
|
return `${base64.slice(0, 180)}~${hash}`;
|
|
767
827
|
}
|
|
768
828
|
return base64;
|
|
@@ -785,82 +845,6 @@ function decodeStreamPath(encoded) {
|
|
|
785
845
|
return Buffer.from(padded, `base64`).toString(`utf-8`);
|
|
786
846
|
}
|
|
787
847
|
|
|
788
|
-
//#endregion
|
|
789
|
-
//#region src/file-manager.ts
|
|
790
|
-
var StreamFileManager = class {
|
|
791
|
-
constructor(streamsDir) {
|
|
792
|
-
this.streamsDir = streamsDir;
|
|
793
|
-
}
|
|
794
|
-
/**
|
|
795
|
-
* Create a directory for a new stream and initialize the first segment file.
|
|
796
|
-
* Returns the absolute path to the stream directory.
|
|
797
|
-
*/
|
|
798
|
-
async createStreamDirectory(streamPath) {
|
|
799
|
-
const encoded = encodeStreamPath(streamPath);
|
|
800
|
-
const dir = path$1.join(this.streamsDir, encoded);
|
|
801
|
-
await fs$1.mkdir(dir, { recursive: true });
|
|
802
|
-
const segmentPath = path$1.join(dir, `segment_00000.log`);
|
|
803
|
-
await fs$1.writeFile(segmentPath, ``);
|
|
804
|
-
return dir;
|
|
805
|
-
}
|
|
806
|
-
/**
|
|
807
|
-
* Delete a stream directory and all its contents.
|
|
808
|
-
*/
|
|
809
|
-
async deleteStreamDirectory(streamPath) {
|
|
810
|
-
const encoded = encodeStreamPath(streamPath);
|
|
811
|
-
const dir = path$1.join(this.streamsDir, encoded);
|
|
812
|
-
await fs$1.rm(dir, {
|
|
813
|
-
recursive: true,
|
|
814
|
-
force: true
|
|
815
|
-
});
|
|
816
|
-
}
|
|
817
|
-
/**
|
|
818
|
-
* Delete a directory by its exact name (used for unique directory names).
|
|
819
|
-
*/
|
|
820
|
-
async deleteDirectoryByName(directoryName) {
|
|
821
|
-
const dir = path$1.join(this.streamsDir, directoryName);
|
|
822
|
-
await fs$1.rm(dir, {
|
|
823
|
-
recursive: true,
|
|
824
|
-
force: true
|
|
825
|
-
});
|
|
826
|
-
}
|
|
827
|
-
/**
|
|
828
|
-
* Get the absolute path to a stream's directory.
|
|
829
|
-
* Returns null if the directory doesn't exist.
|
|
830
|
-
*/
|
|
831
|
-
async getStreamDirectory(streamPath) {
|
|
832
|
-
const encoded = encodeStreamPath(streamPath);
|
|
833
|
-
const dir = path$1.join(this.streamsDir, encoded);
|
|
834
|
-
try {
|
|
835
|
-
await fs$1.access(dir);
|
|
836
|
-
return dir;
|
|
837
|
-
} catch {
|
|
838
|
-
return null;
|
|
839
|
-
}
|
|
840
|
-
}
|
|
841
|
-
/**
|
|
842
|
-
* List all stream paths by scanning the streams directory.
|
|
843
|
-
*/
|
|
844
|
-
async listStreamPaths() {
|
|
845
|
-
try {
|
|
846
|
-
const entries = await fs$1.readdir(this.streamsDir, { withFileTypes: true });
|
|
847
|
-
return entries.filter((e) => e.isDirectory()).map((e) => decodeStreamPath(e.name));
|
|
848
|
-
} catch {
|
|
849
|
-
return [];
|
|
850
|
-
}
|
|
851
|
-
}
|
|
852
|
-
/**
|
|
853
|
-
* Get the path to a segment file within a stream directory.
|
|
854
|
-
*
|
|
855
|
-
* @param streamDir - Absolute path to the stream directory
|
|
856
|
-
* @param index - Segment index (0-based)
|
|
857
|
-
*/
|
|
858
|
-
getSegmentPath(streamDir, index) {
|
|
859
|
-
const paddedIndex = String(index).padStart(5, `0`);
|
|
860
|
-
return path$1.join(streamDir, `segment_${paddedIndex}.log`);
|
|
861
|
-
}
|
|
862
|
-
};
|
|
863
|
-
|
|
864
848
|
//#endregion
|
|
865
849
|
//#region src/file-store.ts
|
|
866
850
|
var FileHandlePool = class {
|
|
@@ -868,7 +852,7 @@ var FileHandlePool = class {
|
|
|
868
852
|
constructor(maxSize) {
|
|
869
853
|
this.cache = new SieveCache(maxSize, { evictHook: (_key, handle) => {
|
|
870
854
|
this.closeHandle(handle).catch((err) => {
|
|
871
|
-
|
|
855
|
+
serverLog.error(`[FileHandlePool] Error closing evicted handle:`, err);
|
|
872
856
|
});
|
|
873
857
|
} });
|
|
874
858
|
}
|
|
@@ -876,41 +860,70 @@ var FileHandlePool = class {
|
|
|
876
860
|
let handle = this.cache.get(filePath);
|
|
877
861
|
if (!handle) {
|
|
878
862
|
const stream = fs.createWriteStream(filePath, { flags: `a` });
|
|
879
|
-
handle = {
|
|
863
|
+
handle = {
|
|
864
|
+
stream,
|
|
865
|
+
syncLeader: null
|
|
866
|
+
};
|
|
880
867
|
this.cache.set(filePath, handle);
|
|
881
868
|
}
|
|
882
869
|
return handle.stream;
|
|
883
870
|
}
|
|
884
871
|
/**
|
|
872
|
+
* Open a write stream eagerly so the first write does not pay the lazy
|
|
873
|
+
* `open()` stall. Resolves once the underlying fd is ready.
|
|
874
|
+
*/
|
|
875
|
+
async openWriteStream(filePath) {
|
|
876
|
+
const stream = this.getWriteStream(filePath);
|
|
877
|
+
const fd = stream.fd;
|
|
878
|
+
if (typeof fd === `number`) return stream;
|
|
879
|
+
await new Promise((resolve, reject) => {
|
|
880
|
+
stream.once(`open`, () => resolve());
|
|
881
|
+
stream.once(`error`, (err) => reject(err));
|
|
882
|
+
});
|
|
883
|
+
return stream;
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
885
886
|
* Flush a specific file to disk immediately.
|
|
886
|
-
*
|
|
887
|
+
* Concurrent callers on the same fd share one in-flight fdatasync: the
|
|
888
|
+
* first caller issues the syscall, later arrivals during that window wait
|
|
889
|
+
* for it to finish and then issue a fresh syscall (because their writes
|
|
890
|
+
* may have landed after the in-flight syscall started). This preserves
|
|
891
|
+
* durability without adding scheduling latency.
|
|
887
892
|
*/
|
|
888
|
-
|
|
893
|
+
fsyncFile(filePath) {
|
|
889
894
|
const handle = this.cache.get(filePath);
|
|
890
|
-
if (!handle) return;
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
else resolve();
|
|
899
|
-
});
|
|
900
|
-
};
|
|
901
|
-
const onError = (err) => {
|
|
902
|
-
handle.stream.off(`open`, onOpen);
|
|
903
|
-
reject(err);
|
|
904
|
-
};
|
|
905
|
-
handle.stream.once(`open`, onOpen);
|
|
906
|
-
handle.stream.once(`error`, onError);
|
|
907
|
-
return;
|
|
908
|
-
}
|
|
909
|
-
fs.fdatasync(fd, (err) => {
|
|
910
|
-
if (err) reject(err);
|
|
911
|
-
else resolve();
|
|
912
|
-
});
|
|
895
|
+
if (!handle) return Promise.reject(new Error(`[FileHandlePool] Cannot fsync: handle not found for ${filePath}`));
|
|
896
|
+
const existing = handle.syncLeader;
|
|
897
|
+
if (existing && existing.scheduled) return existing.promise;
|
|
898
|
+
let resolveFn;
|
|
899
|
+
let rejectFn;
|
|
900
|
+
const promise = new Promise((res, rej) => {
|
|
901
|
+
resolveFn = res;
|
|
902
|
+
rejectFn = rej;
|
|
913
903
|
});
|
|
904
|
+
const leader = {
|
|
905
|
+
promise,
|
|
906
|
+
scheduled: true
|
|
907
|
+
};
|
|
908
|
+
handle.syncLeader = leader;
|
|
909
|
+
const runSyscall = (fd$1) => {
|
|
910
|
+
leader.scheduled = false;
|
|
911
|
+
fs.fdatasync(fd$1, (err) => {
|
|
912
|
+
if (handle.syncLeader === leader) handle.syncLeader = null;
|
|
913
|
+
if (err) rejectFn(err);
|
|
914
|
+
else resolveFn();
|
|
915
|
+
});
|
|
916
|
+
};
|
|
917
|
+
const fd = handle.stream.fd;
|
|
918
|
+
if (typeof fd === `number`) runSyscall(fd);
|
|
919
|
+
else {
|
|
920
|
+
handle.stream.once(`open`, (openedFd) => runSyscall(openedFd));
|
|
921
|
+
handle.stream.once(`error`, (err) => {
|
|
922
|
+
if (handle.syncLeader === leader) handle.syncLeader = null;
|
|
923
|
+
rejectFn(err);
|
|
924
|
+
});
|
|
925
|
+
}
|
|
926
|
+
return promise;
|
|
914
927
|
}
|
|
915
928
|
async closeAll() {
|
|
916
929
|
const promises = [];
|
|
@@ -946,13 +959,15 @@ function generateUniqueDirectoryName(streamPath) {
|
|
|
946
959
|
const random = randomBytes(4).toString(`hex`);
|
|
947
960
|
return `${encoded}~${timestamp}~${random}`;
|
|
948
961
|
}
|
|
962
|
+
function segmentFile(dataDir, dirName) {
|
|
963
|
+
return path.join(dataDir, `streams`, `${dirName}.log`);
|
|
964
|
+
}
|
|
949
965
|
/**
|
|
950
966
|
* File-backed implementation of StreamStore.
|
|
951
967
|
* Maintains the same interface as the in-memory StreamStore for drop-in compatibility.
|
|
952
968
|
*/
|
|
953
969
|
var FileBackedStreamStore = class {
|
|
954
970
|
db;
|
|
955
|
-
fileManager;
|
|
956
971
|
fileHandlePool;
|
|
957
972
|
pendingLongPolls = [];
|
|
958
973
|
dataDir;
|
|
@@ -961,13 +976,23 @@ var FileBackedStreamStore = class {
|
|
|
961
976
|
* Key: "{streamPath}:{producerId}"
|
|
962
977
|
*/
|
|
963
978
|
producerLocks = new Map();
|
|
979
|
+
/**
|
|
980
|
+
* Per-stream append locks. Serializes the read-modify-write of currentOffset
|
|
981
|
+
* across all concurrent appenders on the same stream so the LMDB-tracked
|
|
982
|
+
* offset cannot drift behind the file's actual byte position.
|
|
983
|
+
* Key: streamPath
|
|
984
|
+
*/
|
|
985
|
+
streamAppendLocks = new Map();
|
|
964
986
|
constructor(options) {
|
|
965
987
|
this.dataDir = options.dataDir;
|
|
966
988
|
this.db = open({
|
|
967
989
|
path: path.join(this.dataDir, `metadata.lmdb`),
|
|
968
|
-
compression: true
|
|
990
|
+
compression: true,
|
|
991
|
+
noMemInit: true,
|
|
992
|
+
cache: true,
|
|
993
|
+
sharedStructuresKey: Symbol.for(`structures`)
|
|
969
994
|
});
|
|
970
|
-
|
|
995
|
+
fs.mkdirSync(path.join(this.dataDir, `streams`), { recursive: true });
|
|
971
996
|
const maxFileHandles = options.maxFileHandles ?? 100;
|
|
972
997
|
this.fileHandlePool = new FileHandlePool(maxFileHandles);
|
|
973
998
|
this.recover();
|
|
@@ -977,7 +1002,7 @@ var FileBackedStreamStore = class {
|
|
|
977
1002
|
* Validates that LMDB metadata matches actual file contents and reconciles any mismatches.
|
|
978
1003
|
*/
|
|
979
1004
|
recover() {
|
|
980
|
-
|
|
1005
|
+
serverLog.info(`[FileBackedStreamStore] Starting recovery...`);
|
|
981
1006
|
let recovered = 0;
|
|
982
1007
|
let reconciled = 0;
|
|
983
1008
|
let errors = 0;
|
|
@@ -990,9 +1015,9 @@ var FileBackedStreamStore = class {
|
|
|
990
1015
|
if (typeof key !== `string`) continue;
|
|
991
1016
|
const streamMeta = value;
|
|
992
1017
|
const streamPath = key.replace(`stream:`, ``);
|
|
993
|
-
const segmentPath =
|
|
1018
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
994
1019
|
if (!fs.existsSync(segmentPath)) {
|
|
995
|
-
|
|
1020
|
+
serverLog.warn(`[FileBackedStreamStore] Recovery: Stream file missing for ${streamPath}, removing from LMDB`);
|
|
996
1021
|
this.db.removeSync(key);
|
|
997
1022
|
errors++;
|
|
998
1023
|
continue;
|
|
@@ -1006,7 +1031,7 @@ var FileBackedStreamStore = class {
|
|
|
1006
1031
|
trueOffset = `${String(0).padStart(16, `0`)}_${String(logicalBytes).padStart(16, `0`)}`;
|
|
1007
1032
|
} else trueOffset = physicalOffset;
|
|
1008
1033
|
if (trueOffset !== streamMeta.currentOffset) {
|
|
1009
|
-
|
|
1034
|
+
serverLog.warn(`[FileBackedStreamStore] Recovery: Offset mismatch for ${streamPath}: LMDB says ${streamMeta.currentOffset}, file says ${trueOffset}. Reconciling to file.`);
|
|
1010
1035
|
const reconciledMeta = {
|
|
1011
1036
|
...streamMeta,
|
|
1012
1037
|
currentOffset: trueOffset
|
|
@@ -1016,10 +1041,10 @@ var FileBackedStreamStore = class {
|
|
|
1016
1041
|
}
|
|
1017
1042
|
recovered++;
|
|
1018
1043
|
} catch (err) {
|
|
1019
|
-
|
|
1044
|
+
serverLog.error(`[FileBackedStreamStore] Error recovering stream:`, err);
|
|
1020
1045
|
errors++;
|
|
1021
1046
|
}
|
|
1022
|
-
|
|
1047
|
+
serverLog.info(`[FileBackedStreamStore] Recovery complete: ${recovered} streams, ${reconciled} reconciled, ${errors} errors`);
|
|
1023
1048
|
}
|
|
1024
1049
|
/**
|
|
1025
1050
|
* Scan a segment file to compute the true last offset.
|
|
@@ -1029,19 +1054,16 @@ var FileBackedStreamStore = class {
|
|
|
1029
1054
|
try {
|
|
1030
1055
|
const fileContent = fs.readFileSync(segmentPath);
|
|
1031
1056
|
let filePos = 0;
|
|
1032
|
-
let currentDataOffset = 0;
|
|
1033
1057
|
while (filePos < fileContent.length) {
|
|
1034
1058
|
if (filePos + 4 > fileContent.length) break;
|
|
1035
1059
|
const messageLength = fileContent.readUInt32BE(filePos);
|
|
1036
|
-
filePos
|
|
1037
|
-
if (
|
|
1038
|
-
filePos
|
|
1039
|
-
if (filePos < fileContent.length) filePos += 1;
|
|
1040
|
-
currentDataOffset += messageLength;
|
|
1060
|
+
const frameEnd = filePos + 4 + messageLength + 1;
|
|
1061
|
+
if (frameEnd > fileContent.length) break;
|
|
1062
|
+
filePos = frameEnd;
|
|
1041
1063
|
}
|
|
1042
|
-
return `0000000000000000_${String(
|
|
1064
|
+
return `0000000000000000_${String(filePos).padStart(16, `0`)}`;
|
|
1043
1065
|
} catch (err) {
|
|
1044
|
-
|
|
1066
|
+
serverLog.error(`[FileBackedStreamStore] Error scanning file ${segmentPath}:`, err);
|
|
1045
1067
|
return `0000000000000000_0000000000000000`;
|
|
1046
1068
|
}
|
|
1047
1069
|
}
|
|
@@ -1157,6 +1179,26 @@ var FileBackedStreamStore = class {
|
|
|
1157
1179
|
};
|
|
1158
1180
|
}
|
|
1159
1181
|
/**
|
|
1182
|
+
* Acquire a per-stream append lock that serializes the read-modify-write
|
|
1183
|
+
* of currentOffset across all concurrent appenders on the same stream.
|
|
1184
|
+
* Without this, two concurrent appends can read the same starting
|
|
1185
|
+
* currentOffset, both compute their newOffset, both write a frame to the
|
|
1186
|
+
* file, but only one of their LMDB updates wins — leaving currentOffset
|
|
1187
|
+
* lagging the file's actual byte position. Returns a release function.
|
|
1188
|
+
*/
|
|
1189
|
+
async acquireStreamAppendLock(streamPath) {
|
|
1190
|
+
while (this.streamAppendLocks.has(streamPath)) await this.streamAppendLocks.get(streamPath);
|
|
1191
|
+
let releaseLock;
|
|
1192
|
+
const lockPromise = new Promise((resolve) => {
|
|
1193
|
+
releaseLock = resolve;
|
|
1194
|
+
});
|
|
1195
|
+
this.streamAppendLocks.set(streamPath, lockPromise);
|
|
1196
|
+
return () => {
|
|
1197
|
+
this.streamAppendLocks.delete(streamPath);
|
|
1198
|
+
releaseLock();
|
|
1199
|
+
};
|
|
1200
|
+
}
|
|
1201
|
+
/**
|
|
1160
1202
|
* Get the current epoch for a producer on a stream.
|
|
1161
1203
|
* Returns undefined if the producer doesn't exist or stream not found.
|
|
1162
1204
|
*/
|
|
@@ -1289,6 +1331,7 @@ var FileBackedStreamStore = class {
|
|
|
1289
1331
|
effectiveTtlSeconds = resolved.ttlSeconds;
|
|
1290
1332
|
}
|
|
1291
1333
|
const key = `stream:${streamPath}`;
|
|
1334
|
+
const t0 = performance.now();
|
|
1292
1335
|
const streamMeta = {
|
|
1293
1336
|
path: streamPath,
|
|
1294
1337
|
contentType,
|
|
@@ -1306,11 +1349,10 @@ var FileBackedStreamStore = class {
|
|
|
1306
1349
|
forkOffset: isFork ? forkOffset : void 0,
|
|
1307
1350
|
refCount: 0
|
|
1308
1351
|
};
|
|
1309
|
-
const
|
|
1352
|
+
const tAfterMeta = performance.now();
|
|
1353
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1310
1354
|
try {
|
|
1311
|
-
|
|
1312
|
-
const segmentPath = path.join(streamDir, `segment_00000.log`);
|
|
1313
|
-
fs.writeFileSync(segmentPath, ``);
|
|
1355
|
+
await this.db.put(key, streamMeta);
|
|
1314
1356
|
} catch (err) {
|
|
1315
1357
|
if (isFork && sourceMeta) {
|
|
1316
1358
|
const sourceKey = `stream:${options.forkedFrom}`;
|
|
@@ -1323,10 +1365,18 @@ var FileBackedStreamStore = class {
|
|
|
1323
1365
|
this.db.putSync(sourceKey, updatedSource);
|
|
1324
1366
|
}
|
|
1325
1367
|
}
|
|
1326
|
-
|
|
1368
|
+
serverLog.error(`[FileBackedStreamStore] Error creating stream (LMDB put):`, err);
|
|
1327
1369
|
throw err;
|
|
1328
1370
|
}
|
|
1329
|
-
|
|
1371
|
+
const tAfterLmdb = performance.now();
|
|
1372
|
+
try {
|
|
1373
|
+
await this.fileHandlePool.openWriteStream(segmentPath);
|
|
1374
|
+
} catch (err) {
|
|
1375
|
+
this.db.removeSync(key);
|
|
1376
|
+
serverLog.error(`[FileBackedStreamStore] Error creating stream (file open):`, err);
|
|
1377
|
+
throw err;
|
|
1378
|
+
}
|
|
1379
|
+
const tAfterOpen = performance.now();
|
|
1330
1380
|
if (options.initialData && options.initialData.length > 0) try {
|
|
1331
1381
|
await this.append(streamPath, options.initialData, {
|
|
1332
1382
|
contentType: options.contentType,
|
|
@@ -1346,12 +1396,24 @@ var FileBackedStreamStore = class {
|
|
|
1346
1396
|
}
|
|
1347
1397
|
throw err;
|
|
1348
1398
|
}
|
|
1399
|
+
const tAfterAppend = performance.now();
|
|
1349
1400
|
if (options.closed) {
|
|
1350
1401
|
const updatedMeta = this.db.get(key);
|
|
1351
1402
|
updatedMeta.closed = true;
|
|
1352
|
-
this.db.
|
|
1403
|
+
await this.db.put(key, updatedMeta);
|
|
1353
1404
|
}
|
|
1354
1405
|
const updated = this.db.get(key);
|
|
1406
|
+
const totalMs = performance.now() - t0;
|
|
1407
|
+
if (totalMs > 50) serverLog.event({
|
|
1408
|
+
event: `store.create`,
|
|
1409
|
+
path: streamPath,
|
|
1410
|
+
totalMs: +totalMs.toFixed(2),
|
|
1411
|
+
metaMs: +(tAfterMeta - t0).toFixed(2),
|
|
1412
|
+
lmdbMs: +(tAfterLmdb - tAfterMeta).toFixed(2),
|
|
1413
|
+
openMs: +(tAfterOpen - tAfterLmdb).toFixed(2),
|
|
1414
|
+
appendMs: +(tAfterAppend - tAfterOpen).toFixed(2),
|
|
1415
|
+
initBytes: options.initialData?.length ?? 0
|
|
1416
|
+
}, `store.create slow`);
|
|
1355
1417
|
return this.streamMetaToStream(updated);
|
|
1356
1418
|
}
|
|
1357
1419
|
get(streamPath) {
|
|
@@ -1392,13 +1454,10 @@ var FileBackedStreamStore = class {
|
|
|
1392
1454
|
if (!streamMeta) return;
|
|
1393
1455
|
const forkedFrom = streamMeta.forkedFrom;
|
|
1394
1456
|
this.cancelLongPollsForStream(streamPath);
|
|
1395
|
-
const segmentPath =
|
|
1396
|
-
this.fileHandlePool.closeFileHandle(segmentPath).catch((err) => {
|
|
1397
|
-
console.error(`[FileBackedStreamStore] Error closing file handle:`, err);
|
|
1398
|
-
});
|
|
1457
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1399
1458
|
this.db.removeSync(key);
|
|
1400
|
-
this.
|
|
1401
|
-
|
|
1459
|
+
this.fileHandlePool.closeFileHandle(segmentPath).then(() => fs.promises.unlink(segmentPath)).catch((err) => {
|
|
1460
|
+
serverLog.error(`[FileBackedStreamStore] Error cleaning up stream file:`, err);
|
|
1402
1461
|
});
|
|
1403
1462
|
if (forkedFrom) {
|
|
1404
1463
|
const parentKey = `stream:${forkedFrom}`;
|
|
@@ -1414,7 +1473,20 @@ var FileBackedStreamStore = class {
|
|
|
1414
1473
|
}
|
|
1415
1474
|
}
|
|
1416
1475
|
}
|
|
1476
|
+
/**
|
|
1477
|
+
* Public append entry point. Serializes concurrent appends to the same
|
|
1478
|
+
* stream so the read-modify-write of currentOffset cannot interleave —
|
|
1479
|
+
* see acquireStreamAppendLock for the underlying race.
|
|
1480
|
+
*/
|
|
1417
1481
|
async append(streamPath, data, options = {}) {
|
|
1482
|
+
const releaseLock = await this.acquireStreamAppendLock(streamPath);
|
|
1483
|
+
try {
|
|
1484
|
+
return await this.appendInner(streamPath, data, options);
|
|
1485
|
+
} finally {
|
|
1486
|
+
releaseLock();
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
async appendInner(streamPath, data, options = {}) {
|
|
1418
1490
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1419
1491
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1420
1492
|
if (streamMeta.softDeleted) throw new Error(`Stream is soft-deleted: ${streamPath}`);
|
|
@@ -1456,10 +1528,11 @@ var FileBackedStreamStore = class {
|
|
|
1456
1528
|
const parts = streamMeta.currentOffset.split(`_`).map(Number);
|
|
1457
1529
|
const readSeq = parts[0];
|
|
1458
1530
|
const byteOffset = parts[1];
|
|
1459
|
-
const
|
|
1531
|
+
const FRAME_OVERHEAD = 5;
|
|
1532
|
+
const newByteOffset = byteOffset + FRAME_OVERHEAD + processedData.length;
|
|
1460
1533
|
const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
1461
|
-
const
|
|
1462
|
-
const
|
|
1534
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1535
|
+
const tAppendStart = performance.now();
|
|
1463
1536
|
const stream = this.fileHandlePool.getWriteStream(segmentPath);
|
|
1464
1537
|
const lengthBuf = Buffer.allocUnsafe(4);
|
|
1465
1538
|
lengthBuf.writeUInt32BE(processedData.length, 0);
|
|
@@ -1474,12 +1547,14 @@ var FileBackedStreamStore = class {
|
|
|
1474
1547
|
else resolve();
|
|
1475
1548
|
});
|
|
1476
1549
|
});
|
|
1550
|
+
const tAfterWrite = performance.now();
|
|
1477
1551
|
const message = {
|
|
1478
1552
|
data: processedData,
|
|
1479
1553
|
offset: newOffset,
|
|
1480
1554
|
timestamp: Date.now()
|
|
1481
1555
|
};
|
|
1482
1556
|
await this.fileHandlePool.fsyncFile(segmentPath);
|
|
1557
|
+
const tAfterFsync = performance.now();
|
|
1483
1558
|
const updatedProducers = { ...streamMeta.producers };
|
|
1484
1559
|
if (producerResult && producerResult.status === `accepted`) updatedProducers[producerResult.producerId] = producerResult.proposedState;
|
|
1485
1560
|
let closedBy = void 0;
|
|
@@ -1498,7 +1573,19 @@ var FileBackedStreamStore = class {
|
|
|
1498
1573
|
closedBy: closedBy ?? streamMeta.closedBy
|
|
1499
1574
|
};
|
|
1500
1575
|
const key = `stream:${streamPath}`;
|
|
1501
|
-
this.db.
|
|
1576
|
+
await this.db.put(key, updatedMeta);
|
|
1577
|
+
const tAfterLmdb = performance.now();
|
|
1578
|
+
const appendTotal = tAfterLmdb - tAppendStart;
|
|
1579
|
+
if (appendTotal > 50) serverLog.event({
|
|
1580
|
+
event: `store.append`,
|
|
1581
|
+
path: streamPath,
|
|
1582
|
+
totalMs: +appendTotal.toFixed(2),
|
|
1583
|
+
writeMs: +(tAfterWrite - tAppendStart).toFixed(2),
|
|
1584
|
+
fsyncMs: +(tAfterFsync - tAfterWrite).toFixed(2),
|
|
1585
|
+
lmdbMs: +(tAfterLmdb - tAfterFsync).toFixed(2),
|
|
1586
|
+
bytes: processedData.length,
|
|
1587
|
+
isInitial: options.isInitialCreate ?? false
|
|
1588
|
+
}, `store.append slow`);
|
|
1502
1589
|
this.notifyLongPolls(streamPath);
|
|
1503
1590
|
if (options.close) this.notifyLongPollsClosed(streamPath);
|
|
1504
1591
|
if (producerResult || options.close) return {
|
|
@@ -1591,7 +1678,7 @@ var FileBackedStreamStore = class {
|
|
|
1591
1678
|
},
|
|
1592
1679
|
producers: updatedProducers
|
|
1593
1680
|
};
|
|
1594
|
-
this.db.
|
|
1681
|
+
await this.db.put(key, updatedMeta);
|
|
1595
1682
|
this.notifyLongPollsClosed(streamPath);
|
|
1596
1683
|
return {
|
|
1597
1684
|
finalOffset: streamMeta.currentOffset,
|
|
@@ -1625,7 +1712,7 @@ var FileBackedStreamStore = class {
|
|
|
1625
1712
|
const messageData = fileContent.subarray(filePos, filePos + messageLength);
|
|
1626
1713
|
filePos += messageLength;
|
|
1627
1714
|
filePos += 1;
|
|
1628
|
-
physicalDataOffset += messageLength;
|
|
1715
|
+
physicalDataOffset += messageLength + 5;
|
|
1629
1716
|
const logicalOffset = baseByteOffset + physicalDataOffset;
|
|
1630
1717
|
if (capByte !== void 0 && logicalOffset > capByte) break;
|
|
1631
1718
|
if (logicalOffset > startByte) messages.push({
|
|
@@ -1635,7 +1722,7 @@ var FileBackedStreamStore = class {
|
|
|
1635
1722
|
});
|
|
1636
1723
|
}
|
|
1637
1724
|
} catch (err) {
|
|
1638
|
-
|
|
1725
|
+
serverLog.error(`[FileBackedStreamStore] Error reading segment file:`, err);
|
|
1639
1726
|
}
|
|
1640
1727
|
return messages;
|
|
1641
1728
|
}
|
|
@@ -1657,7 +1744,7 @@ var FileBackedStreamStore = class {
|
|
|
1657
1744
|
messages.push(...inherited);
|
|
1658
1745
|
}
|
|
1659
1746
|
}
|
|
1660
|
-
const segmentPath =
|
|
1747
|
+
const segmentPath = segmentFile(this.dataDir, sourceMeta.directoryName);
|
|
1661
1748
|
const sourceBaseByte = sourceMeta.forkOffset ? Number(sourceMeta.forkOffset.split(`_`)[1] ?? 0) : 0;
|
|
1662
1749
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, sourceBaseByte, capByte);
|
|
1663
1750
|
messages.push(...ownMessages);
|
|
@@ -1684,11 +1771,11 @@ var FileBackedStreamStore = class {
|
|
|
1684
1771
|
const inherited = this.readForkedMessages(streamMeta.forkedFrom, startByte, forkByte);
|
|
1685
1772
|
messages.push(...inherited);
|
|
1686
1773
|
}
|
|
1687
|
-
const segmentPath =
|
|
1774
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1688
1775
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, forkByte);
|
|
1689
1776
|
messages.push(...ownMessages);
|
|
1690
1777
|
} else {
|
|
1691
|
-
const segmentPath =
|
|
1778
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1692
1779
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, 0);
|
|
1693
1780
|
messages.push(...ownMessages);
|
|
1694
1781
|
}
|
|
@@ -1759,6 +1846,7 @@ var FileBackedStreamStore = class {
|
|
|
1759
1846
|
formatResponse(streamPath, messages) {
|
|
1760
1847
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1761
1848
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1849
|
+
if (normalizeContentType(streamMeta.contentType) === `application/json`) return formatJsonMessages(messages);
|
|
1762
1850
|
const totalSize = messages.reduce((sum, m) => sum + m.data.length, 0);
|
|
1763
1851
|
const concatenated = new Uint8Array(totalSize);
|
|
1764
1852
|
let offset = 0;
|
|
@@ -1766,7 +1854,6 @@ var FileBackedStreamStore = class {
|
|
|
1766
1854
|
concatenated.set(msg.data, offset);
|
|
1767
1855
|
offset += msg.data.length;
|
|
1768
1856
|
}
|
|
1769
|
-
if (normalizeContentType(streamMeta.contentType) === `application/json`) return formatJsonResponse(concatenated);
|
|
1770
1857
|
return concatenated;
|
|
1771
1858
|
}
|
|
1772
1859
|
getCurrentOffset(streamPath) {
|
|
@@ -1786,7 +1873,7 @@ var FileBackedStreamStore = class {
|
|
|
1786
1873
|
const entries = Array.from(range);
|
|
1787
1874
|
for (const { key } of entries) this.db.removeSync(key);
|
|
1788
1875
|
this.fileHandlePool.closeAll().catch((err) => {
|
|
1789
|
-
|
|
1876
|
+
serverLog.error(`[FileBackedStreamStore] Error closing handles:`, err);
|
|
1790
1877
|
});
|
|
1791
1878
|
}
|
|
1792
1879
|
/**
|
|
@@ -1940,30 +2027,985 @@ function handleCursorCollision(currentCursor, previousCursor, options = {}) {
|
|
|
1940
2027
|
return generateResponseCursor(previousCursor, options);
|
|
1941
2028
|
}
|
|
1942
2029
|
|
|
2030
|
+
//#endregion
|
|
2031
|
+
//#region src/crypto.ts
|
|
2032
|
+
/**
|
|
2033
|
+
* Generate a unique wake ID.
|
|
2034
|
+
*/
|
|
2035
|
+
function generateWakeId() {
|
|
2036
|
+
return `w_${randomBytes(12).toString(`hex`)}`;
|
|
2037
|
+
}
|
|
2038
|
+
const WEBHOOK_KEYPAIR = generateKeyPairSync(`ed25519`);
|
|
2039
|
+
const WEBHOOK_PUBLIC_JWK = buildWebhookPublicJwk();
|
|
2040
|
+
function buildWebhookPublicJwk() {
|
|
2041
|
+
const exported = WEBHOOK_KEYPAIR.publicKey.export({ format: `jwk` });
|
|
2042
|
+
if (exported.kty !== `OKP` || exported.crv !== `Ed25519` || !exported.x) throw new Error(`Failed to export Ed25519 webhook signing key`);
|
|
2043
|
+
const thumbprintInput = JSON.stringify({
|
|
2044
|
+
crv: exported.crv,
|
|
2045
|
+
kty: exported.kty,
|
|
2046
|
+
x: exported.x
|
|
2047
|
+
});
|
|
2048
|
+
const kid = `ds_${createHash(`sha256`).update(thumbprintInput).digest(`base64url`)}`;
|
|
2049
|
+
return {
|
|
2050
|
+
kty: `OKP`,
|
|
2051
|
+
crv: `Ed25519`,
|
|
2052
|
+
x: exported.x,
|
|
2053
|
+
kid,
|
|
2054
|
+
use: `sig`,
|
|
2055
|
+
alg: `EdDSA`
|
|
2056
|
+
};
|
|
2057
|
+
}
|
|
2058
|
+
function getWebhookSigningKeyId() {
|
|
2059
|
+
return WEBHOOK_PUBLIC_JWK.kid;
|
|
2060
|
+
}
|
|
2061
|
+
function getWebhookJwks() {
|
|
2062
|
+
return { keys: [{ ...WEBHOOK_PUBLIC_JWK }] };
|
|
2063
|
+
}
|
|
2064
|
+
/**
|
|
2065
|
+
* Sign a webhook payload for the Webhook-Signature header.
|
|
2066
|
+
* Format: t=<timestamp>,kid=<key_id>,ed25519=<base64url_signature>
|
|
2067
|
+
*/
|
|
2068
|
+
function signWebhookPayload(body) {
|
|
2069
|
+
const timestamp = Math.floor(Date.now() / 1e3);
|
|
2070
|
+
const payload = `${timestamp}.${body}`;
|
|
2071
|
+
const signature = sign(null, Buffer.from(payload), WEBHOOK_KEYPAIR.privateKey).toString(`base64url`);
|
|
2072
|
+
return `t=${timestamp},kid=${WEBHOOK_PUBLIC_JWK.kid},ed25519=${signature}`;
|
|
2073
|
+
}
|
|
2074
|
+
const TOKEN_KEY = randomBytes(32);
|
|
2075
|
+
/**
|
|
2076
|
+
* Generate a signed callback token.
|
|
2077
|
+
* Token format: base64url(json_payload).base64url(hmac_signature)
|
|
2078
|
+
* Payload: { consumer_id, epoch, exp }
|
|
2079
|
+
*/
|
|
2080
|
+
function generateCallbackToken(consumerId, epoch) {
|
|
2081
|
+
const payload = {
|
|
2082
|
+
sub: consumerId,
|
|
2083
|
+
epoch,
|
|
2084
|
+
exp: Math.floor(Date.now() / 1e3) + 3600,
|
|
2085
|
+
jti: randomBytes(8).toString(`hex`)
|
|
2086
|
+
};
|
|
2087
|
+
const payloadStr = Buffer.from(JSON.stringify(payload)).toString(`base64url`);
|
|
2088
|
+
const sig = createHmac(`sha256`, TOKEN_KEY).update(payloadStr).digest(`base64url`);
|
|
2089
|
+
return `${payloadStr}.${sig}`;
|
|
2090
|
+
}
|
|
2091
|
+
/**
|
|
2092
|
+
* Validate a callback token. Returns the decoded payload or null.
|
|
2093
|
+
* On success, includes `exp` (unix seconds) so callers can decide
|
|
2094
|
+
* whether the token needs refreshing.
|
|
2095
|
+
*/
|
|
2096
|
+
function validateCallbackToken(token, consumerId) {
|
|
2097
|
+
const parts = token.split(`.`);
|
|
2098
|
+
if (parts.length !== 2) return {
|
|
2099
|
+
valid: false,
|
|
2100
|
+
code: `TOKEN_INVALID`
|
|
2101
|
+
};
|
|
2102
|
+
const [payloadStr, sig] = parts;
|
|
2103
|
+
const expectedSig = createHmac(`sha256`, TOKEN_KEY).update(payloadStr).digest(`base64url`);
|
|
2104
|
+
try {
|
|
2105
|
+
if (!timingSafeEqual(Buffer.from(sig), Buffer.from(expectedSig))) return {
|
|
2106
|
+
valid: false,
|
|
2107
|
+
code: `TOKEN_INVALID`
|
|
2108
|
+
};
|
|
2109
|
+
} catch {
|
|
2110
|
+
return {
|
|
2111
|
+
valid: false,
|
|
2112
|
+
code: `TOKEN_INVALID`
|
|
2113
|
+
};
|
|
2114
|
+
}
|
|
2115
|
+
let payload;
|
|
2116
|
+
try {
|
|
2117
|
+
payload = JSON.parse(Buffer.from(payloadStr, `base64url`).toString());
|
|
2118
|
+
} catch {
|
|
2119
|
+
return {
|
|
2120
|
+
valid: false,
|
|
2121
|
+
code: `TOKEN_INVALID`
|
|
2122
|
+
};
|
|
2123
|
+
}
|
|
2124
|
+
if (payload.sub !== consumerId) return {
|
|
2125
|
+
valid: false,
|
|
2126
|
+
code: `TOKEN_INVALID`
|
|
2127
|
+
};
|
|
2128
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
2129
|
+
if (now > payload.exp) return {
|
|
2130
|
+
valid: false,
|
|
2131
|
+
code: `TOKEN_EXPIRED`
|
|
2132
|
+
};
|
|
2133
|
+
return {
|
|
2134
|
+
valid: true,
|
|
2135
|
+
exp: payload.exp,
|
|
2136
|
+
epoch: payload.epoch
|
|
2137
|
+
};
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
//#endregion
|
|
2141
|
+
//#region src/glob.ts
|
|
2142
|
+
/**
|
|
2143
|
+
* Glob pattern matching for webhook subscription patterns.
|
|
2144
|
+
*
|
|
2145
|
+
* Supports:
|
|
2146
|
+
* - `*` matches exactly one path segment
|
|
2147
|
+
* - `**` matches zero or more path segments (recursive)
|
|
2148
|
+
* - Literal segments match exactly
|
|
2149
|
+
*/
|
|
2150
|
+
/**
|
|
2151
|
+
* Match a stream path against a glob pattern.
|
|
2152
|
+
*/
|
|
2153
|
+
function globMatch(pattern, path$1) {
|
|
2154
|
+
const patternParts = splitPath(pattern);
|
|
2155
|
+
const pathParts = splitPath(path$1);
|
|
2156
|
+
return matchParts(patternParts, 0, pathParts, 0);
|
|
2157
|
+
}
|
|
2158
|
+
function splitPath(p) {
|
|
2159
|
+
return p.replace(/^\/+/, ``).replace(/\/+$/, ``).split(`/`).filter((s) => s.length > 0);
|
|
2160
|
+
}
|
|
2161
|
+
function matchParts(pattern, pi, path$1, si) {
|
|
2162
|
+
while (pi < pattern.length && si < path$1.length) {
|
|
2163
|
+
const seg = pattern[pi];
|
|
2164
|
+
if (seg === `**`) {
|
|
2165
|
+
for (let i = si; i <= path$1.length; i++) if (matchParts(pattern, pi + 1, path$1, i)) return true;
|
|
2166
|
+
return false;
|
|
2167
|
+
}
|
|
2168
|
+
if (seg === `*`) {
|
|
2169
|
+
pi++;
|
|
2170
|
+
si++;
|
|
2171
|
+
continue;
|
|
2172
|
+
}
|
|
2173
|
+
const decodedSeg = seg.replace(/%2[Aa]/g, `*`);
|
|
2174
|
+
if (decodedSeg !== path$1[si]) return false;
|
|
2175
|
+
pi++;
|
|
2176
|
+
si++;
|
|
2177
|
+
}
|
|
2178
|
+
while (pi < pattern.length && pattern[pi] === `**`) pi++;
|
|
2179
|
+
return pi === pattern.length && si === path$1.length;
|
|
2180
|
+
}
|
|
2181
|
+
|
|
2182
|
+
//#endregion
|
|
2183
|
+
//#region src/subscription-manager.ts
|
|
2184
|
+
const DEFAULT_LEASE_TTL_MS = 3e4;
|
|
2185
|
+
const MIN_LEASE_TTL_MS = 1e3;
|
|
2186
|
+
const MAX_LEASE_TTL_MS = 10 * 6e4;
|
|
2187
|
+
const ZERO_OFFSET = `0000000000000000_0000000000000000`;
|
|
2188
|
+
const BEFORE_FIRST_OFFSET = `-1`;
|
|
2189
|
+
const MAX_RETRY_DELAY_MS = 6e4;
|
|
2190
|
+
function compareOffsets(a, b) {
|
|
2191
|
+
if (a < b) return -1;
|
|
2192
|
+
if (a > b) return 1;
|
|
2193
|
+
return 0;
|
|
2194
|
+
}
|
|
2195
|
+
function normalizeRelativePath(path$1) {
|
|
2196
|
+
return path$1.replace(/^\/+/, ``).replace(/\/+$/, ``);
|
|
2197
|
+
}
|
|
2198
|
+
function toAbsoluteStreamPath(streamPath) {
|
|
2199
|
+
return `/v1/stream/${normalizeRelativePath(streamPath)}`;
|
|
2200
|
+
}
|
|
2201
|
+
function toStreamRelativePath(absolutePath) {
|
|
2202
|
+
const streamRoot = `/v1/stream/`;
|
|
2203
|
+
if (!absolutePath.startsWith(streamRoot)) return null;
|
|
2204
|
+
const path$1 = absolutePath.slice(streamRoot.length);
|
|
2205
|
+
if (path$1 === `__ds` || path$1.startsWith(`__ds/`)) return null;
|
|
2206
|
+
return path$1.length > 0 ? path$1 : null;
|
|
2207
|
+
}
|
|
2208
|
+
function stableConfigHash(input) {
|
|
2209
|
+
const canonical = {
|
|
2210
|
+
type: input.type,
|
|
2211
|
+
pattern: input.pattern,
|
|
2212
|
+
streams: [...new Set(input.streams)].sort(),
|
|
2213
|
+
webhook: input.webhook ? { url: input.webhook.url } : void 0,
|
|
2214
|
+
wake_stream: input.wake_stream,
|
|
2215
|
+
lease_ttl_ms: input.lease_ttl_ms,
|
|
2216
|
+
description: input.description
|
|
2217
|
+
};
|
|
2218
|
+
return createHash(`sha256`).update(JSON.stringify(canonical)).digest(`hex`);
|
|
2219
|
+
}
|
|
2220
|
+
function isPrivateOrLinkLocalIpv4(host) {
|
|
2221
|
+
const parts = host.split(`.`).map((part) => Number(part));
|
|
2222
|
+
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part))) return false;
|
|
2223
|
+
const [a, b] = parts;
|
|
2224
|
+
return a === 10 || a === 127 || a === 0 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a === 169 && b === 254;
|
|
2225
|
+
}
|
|
2226
|
+
function isLocalDevHost(host) {
|
|
2227
|
+
return host === `localhost` || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host);
|
|
2228
|
+
}
|
|
2229
|
+
function validateWebhookUrl(rawUrl) {
|
|
2230
|
+
let url;
|
|
2231
|
+
try {
|
|
2232
|
+
url = new URL(rawUrl);
|
|
2233
|
+
} catch {
|
|
2234
|
+
return {
|
|
2235
|
+
ok: false,
|
|
2236
|
+
message: `webhook.url must be a valid URL`
|
|
2237
|
+
};
|
|
2238
|
+
}
|
|
2239
|
+
const host = url.hostname.toLowerCase();
|
|
2240
|
+
if (url.protocol === `http:`) {
|
|
2241
|
+
if (isLocalDevHost(host)) return { ok: true };
|
|
2242
|
+
return {
|
|
2243
|
+
ok: false,
|
|
2244
|
+
message: `http webhook URLs are only allowed for localhost or 127.0.0.x`
|
|
2245
|
+
};
|
|
2246
|
+
}
|
|
2247
|
+
if (url.protocol !== `https:`) return {
|
|
2248
|
+
ok: false,
|
|
2249
|
+
message: `webhook.url must use https`
|
|
2250
|
+
};
|
|
2251
|
+
if (host === `localhost`) return {
|
|
2252
|
+
ok: false,
|
|
2253
|
+
message: `localhost webhook URLs must use http for dev`
|
|
2254
|
+
};
|
|
2255
|
+
if (isIP(host) === 4 && isPrivateOrLinkLocalIpv4(host)) return {
|
|
2256
|
+
ok: false,
|
|
2257
|
+
message: `webhook.url must not target private or link-local hosts`
|
|
2258
|
+
};
|
|
2259
|
+
if (isIP(host) === 6) return {
|
|
2260
|
+
ok: false,
|
|
2261
|
+
message: `IPv6 webhook hosts are not accepted by the reference server`
|
|
2262
|
+
};
|
|
2263
|
+
return { ok: true };
|
|
2264
|
+
}
|
|
2265
|
+
var SubscriptionManager = class {
|
|
2266
|
+
subscriptions = new Map();
|
|
2267
|
+
streamStore;
|
|
2268
|
+
callbackBaseUrl;
|
|
2269
|
+
webhooksEnabled;
|
|
2270
|
+
isShuttingDown = false;
|
|
2271
|
+
constructor(opts) {
|
|
2272
|
+
this.callbackBaseUrl = opts.callbackBaseUrl;
|
|
2273
|
+
this.streamStore = opts.streamStore;
|
|
2274
|
+
this.webhooksEnabled = opts.webhooksEnabled ?? true;
|
|
2275
|
+
}
|
|
2276
|
+
createOrConfirm(id, input) {
|
|
2277
|
+
const configHash = stableConfigHash(input);
|
|
2278
|
+
const existing = this.subscriptions.get(id);
|
|
2279
|
+
if (existing) {
|
|
2280
|
+
if (existing.config_hash !== configHash) return { error: {
|
|
2281
|
+
code: `SUBSCRIPTION_ALREADY_EXISTS`,
|
|
2282
|
+
message: `Subscription already exists with different configuration`
|
|
2283
|
+
} };
|
|
2284
|
+
return {
|
|
2285
|
+
subscription: existing,
|
|
2286
|
+
created: false
|
|
2287
|
+
};
|
|
2288
|
+
}
|
|
2289
|
+
if (input.type === `webhook`) {
|
|
2290
|
+
if (!this.webhooksEnabled) return { error: {
|
|
2291
|
+
code: `INVALID_REQUEST`,
|
|
2292
|
+
message: `webhook subscriptions are not enabled on this server`
|
|
2293
|
+
} };
|
|
2294
|
+
if (!input.webhook) return { error: {
|
|
2295
|
+
code: `INVALID_REQUEST`,
|
|
2296
|
+
message: `webhook subscriptions require webhook.url`
|
|
2297
|
+
} };
|
|
2298
|
+
const validation = validateWebhookUrl(input.webhook.url);
|
|
2299
|
+
if (!validation.ok) return { error: {
|
|
2300
|
+
code: `WEBHOOK_URL_REJECTED`,
|
|
2301
|
+
message: validation.message
|
|
2302
|
+
} };
|
|
2303
|
+
}
|
|
2304
|
+
if (input.type === `pull-wake` && !input.wake_stream) return { error: {
|
|
2305
|
+
code: `INVALID_REQUEST`,
|
|
2306
|
+
message: `pull-wake subscriptions require wake_stream`
|
|
2307
|
+
} };
|
|
2308
|
+
const subscription = {
|
|
2309
|
+
id,
|
|
2310
|
+
type: input.type,
|
|
2311
|
+
pattern: input.pattern,
|
|
2312
|
+
webhook: input.webhook ? { url: input.webhook.url } : void 0,
|
|
2313
|
+
wake_stream: input.wake_stream,
|
|
2314
|
+
lease_ttl_ms: input.lease_ttl_ms,
|
|
2315
|
+
description: input.description,
|
|
2316
|
+
created_at: new Date().toISOString(),
|
|
2317
|
+
status: `active`,
|
|
2318
|
+
config_hash: configHash,
|
|
2319
|
+
streams: new Map(),
|
|
2320
|
+
generation: 0,
|
|
2321
|
+
wake_id: null,
|
|
2322
|
+
wake_snapshot: new Map(),
|
|
2323
|
+
token: null,
|
|
2324
|
+
holder: null,
|
|
2325
|
+
lease_timer: null,
|
|
2326
|
+
retry_count: 0,
|
|
2327
|
+
retry_timer: null,
|
|
2328
|
+
next_attempt_at: null
|
|
2329
|
+
};
|
|
2330
|
+
for (const stream of input.streams) this.linkStream(subscription, stream, `explicit`, this.getTailOffset(stream));
|
|
2331
|
+
if (input.pattern) {
|
|
2332
|
+
for (const stream of this.listStreams()) if (globMatch(input.pattern, stream)) this.linkStream(subscription, stream, `glob`, this.getTailOffset(stream));
|
|
2333
|
+
}
|
|
2334
|
+
this.subscriptions.set(id, subscription);
|
|
2335
|
+
return {
|
|
2336
|
+
subscription,
|
|
2337
|
+
created: true
|
|
2338
|
+
};
|
|
2339
|
+
}
|
|
2340
|
+
get(id) {
|
|
2341
|
+
return this.subscriptions.get(id);
|
|
2342
|
+
}
|
|
2343
|
+
delete(id) {
|
|
2344
|
+
const subscription = this.subscriptions.get(id);
|
|
2345
|
+
if (!subscription) return false;
|
|
2346
|
+
this.clearLease(subscription);
|
|
2347
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2348
|
+
this.subscriptions.delete(id);
|
|
2349
|
+
return true;
|
|
2350
|
+
}
|
|
2351
|
+
addExplicitStreams(id, streams) {
|
|
2352
|
+
const subscription = this.get(id);
|
|
2353
|
+
if (!subscription) return false;
|
|
2354
|
+
for (const stream of streams) this.linkStream(subscription, stream, `explicit`, this.getTailOffset(stream));
|
|
2355
|
+
return true;
|
|
2356
|
+
}
|
|
2357
|
+
removeExplicitStream(id, streamPath) {
|
|
2358
|
+
const subscription = this.get(id);
|
|
2359
|
+
if (!subscription) return false;
|
|
2360
|
+
const normalized = normalizeRelativePath(streamPath);
|
|
2361
|
+
const link = subscription.streams.get(normalized);
|
|
2362
|
+
if (!link) return true;
|
|
2363
|
+
link.link_types.delete(`explicit`);
|
|
2364
|
+
if (link.link_types.size === 0) subscription.streams.delete(normalized);
|
|
2365
|
+
return true;
|
|
2366
|
+
}
|
|
2367
|
+
async onStreamAppend(absolutePath) {
|
|
2368
|
+
if (this.isShuttingDown) return;
|
|
2369
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2370
|
+
const relative = toStreamRelativePath(absolutePath);
|
|
2371
|
+
if (!relative) continue;
|
|
2372
|
+
if (subscription.pattern && globMatch(subscription.pattern, relative)) {
|
|
2373
|
+
const existing = subscription.streams.get(relative);
|
|
2374
|
+
this.linkStream(subscription, relative, `glob`, existing?.acked_offset ?? BEFORE_FIRST_OFFSET);
|
|
2375
|
+
}
|
|
2376
|
+
if (subscription.streams.has(relative)) await this.maybeWake(subscription, relative);
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
onStreamDeleted(absolutePath) {
|
|
2380
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2381
|
+
const relative = toStreamRelativePath(absolutePath);
|
|
2382
|
+
if (relative) subscription.streams.delete(relative);
|
|
2383
|
+
}
|
|
2384
|
+
}
|
|
2385
|
+
async handleWebhookCallback(id, token, request) {
|
|
2386
|
+
const subscription = this.get(id);
|
|
2387
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2388
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2389
|
+
if (fenced) return fenced;
|
|
2390
|
+
const ackError = this.applyAcks(subscription, request);
|
|
2391
|
+
if (ackError) return ackError;
|
|
2392
|
+
this.extendLease(subscription);
|
|
2393
|
+
let nextWake = false;
|
|
2394
|
+
if (request.done === true) {
|
|
2395
|
+
this.clearLease(subscription);
|
|
2396
|
+
subscription.token = null;
|
|
2397
|
+
subscription.holder = null;
|
|
2398
|
+
subscription.wake_id = null;
|
|
2399
|
+
subscription.wake_snapshot.clear();
|
|
2400
|
+
nextWake = await this.triggerNextWakeIfPending(subscription);
|
|
2401
|
+
}
|
|
2402
|
+
return {
|
|
2403
|
+
status: 200,
|
|
2404
|
+
body: {
|
|
2405
|
+
ok: true,
|
|
2406
|
+
next_wake: nextWake
|
|
2407
|
+
}
|
|
2408
|
+
};
|
|
2409
|
+
}
|
|
2410
|
+
async claim(id, worker) {
|
|
2411
|
+
const subscription = this.get(id);
|
|
2412
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2413
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2414
|
+
if (subscription.holder) return {
|
|
2415
|
+
status: 409,
|
|
2416
|
+
body: { error: {
|
|
2417
|
+
code: `ALREADY_CLAIMED`,
|
|
2418
|
+
current_holder: subscription.holder,
|
|
2419
|
+
generation: subscription.generation
|
|
2420
|
+
} }
|
|
2421
|
+
};
|
|
2422
|
+
if (!this.hasPendingWork(subscription)) return this.errorResponse(409, `NO_PENDING_WORK`, `Subscription has no pending work`);
|
|
2423
|
+
if (!subscription.wake_id) await this.createWake(subscription, this.firstPendingStream(subscription));
|
|
2424
|
+
subscription.holder = worker;
|
|
2425
|
+
subscription.token = generateCallbackToken(this.tokenSubject(subscription), subscription.generation);
|
|
2426
|
+
this.extendLease(subscription);
|
|
2427
|
+
return {
|
|
2428
|
+
status: 200,
|
|
2429
|
+
body: {
|
|
2430
|
+
wake_id: subscription.wake_id,
|
|
2431
|
+
generation: subscription.generation,
|
|
2432
|
+
token: subscription.token,
|
|
2433
|
+
streams: this.streamInfos(subscription),
|
|
2434
|
+
lease_ttl_ms: subscription.lease_ttl_ms
|
|
2435
|
+
}
|
|
2436
|
+
};
|
|
2437
|
+
}
|
|
2438
|
+
async ack(id, token, request) {
|
|
2439
|
+
const subscription = this.get(id);
|
|
2440
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2441
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2442
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2443
|
+
if (fenced) return fenced;
|
|
2444
|
+
const ackError = this.applyAcks(subscription, request);
|
|
2445
|
+
if (ackError) return ackError;
|
|
2446
|
+
this.extendLease(subscription);
|
|
2447
|
+
let nextWake = false;
|
|
2448
|
+
if (request.done === true) {
|
|
2449
|
+
this.clearLease(subscription);
|
|
2450
|
+
subscription.token = null;
|
|
2451
|
+
subscription.holder = null;
|
|
2452
|
+
subscription.wake_id = null;
|
|
2453
|
+
subscription.wake_snapshot.clear();
|
|
2454
|
+
nextWake = await this.triggerNextWakeIfPending(subscription);
|
|
2455
|
+
}
|
|
2456
|
+
return {
|
|
2457
|
+
status: 200,
|
|
2458
|
+
body: {
|
|
2459
|
+
ok: true,
|
|
2460
|
+
next_wake: nextWake
|
|
2461
|
+
}
|
|
2462
|
+
};
|
|
2463
|
+
}
|
|
2464
|
+
async release(id, token, request) {
|
|
2465
|
+
const subscription = this.get(id);
|
|
2466
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2467
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2468
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2469
|
+
if (fenced) return fenced;
|
|
2470
|
+
this.clearLease(subscription);
|
|
2471
|
+
subscription.token = null;
|
|
2472
|
+
subscription.holder = null;
|
|
2473
|
+
subscription.wake_id = null;
|
|
2474
|
+
subscription.wake_snapshot.clear();
|
|
2475
|
+
await this.triggerNextWakeIfPending(subscription);
|
|
2476
|
+
return { status: 204 };
|
|
2477
|
+
}
|
|
2478
|
+
serialize(subscription) {
|
|
2479
|
+
return {
|
|
2480
|
+
id: subscription.id,
|
|
2481
|
+
subscription_id: subscription.id,
|
|
2482
|
+
type: subscription.type,
|
|
2483
|
+
pattern: subscription.pattern,
|
|
2484
|
+
streams: this.streamInfos(subscription).map((stream) => ({
|
|
2485
|
+
path: stream.path,
|
|
2486
|
+
link_type: stream.link_type,
|
|
2487
|
+
acked_offset: stream.acked_offset
|
|
2488
|
+
})),
|
|
2489
|
+
webhook: subscription.webhook ? {
|
|
2490
|
+
url: subscription.webhook.url,
|
|
2491
|
+
signing: this.webhookSigningMetadata()
|
|
2492
|
+
} : void 0,
|
|
2493
|
+
wake_stream: subscription.wake_stream,
|
|
2494
|
+
lease_ttl_ms: subscription.lease_ttl_ms,
|
|
2495
|
+
created_at: subscription.created_at,
|
|
2496
|
+
status: subscription.status,
|
|
2497
|
+
description: subscription.description
|
|
2498
|
+
};
|
|
2499
|
+
}
|
|
2500
|
+
getWebhookJwks() {
|
|
2501
|
+
return getWebhookJwks();
|
|
2502
|
+
}
|
|
2503
|
+
shutdown() {
|
|
2504
|
+
this.isShuttingDown = true;
|
|
2505
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2506
|
+
this.clearLease(subscription);
|
|
2507
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2508
|
+
}
|
|
2509
|
+
this.subscriptions.clear();
|
|
2510
|
+
}
|
|
2511
|
+
async maybeWake(subscription, triggeredBy) {
|
|
2512
|
+
if (subscription.wake_id || subscription.holder) return;
|
|
2513
|
+
if (!this.hasPendingWork(subscription)) return;
|
|
2514
|
+
await this.createWake(subscription, triggeredBy);
|
|
2515
|
+
}
|
|
2516
|
+
async createWake(subscription, triggeredBy) {
|
|
2517
|
+
subscription.generation++;
|
|
2518
|
+
subscription.wake_id = generateWakeId();
|
|
2519
|
+
subscription.wake_snapshot = new Map(this.streamInfos(subscription).map((stream) => [stream.path, stream.tail_offset]));
|
|
2520
|
+
if (subscription.type === `webhook`) {
|
|
2521
|
+
subscription.token = generateCallbackToken(this.tokenSubject(subscription), subscription.generation);
|
|
2522
|
+
this.extendLease(subscription);
|
|
2523
|
+
this.deliverWebhook(subscription, [triggeredBy]);
|
|
2524
|
+
return;
|
|
2525
|
+
}
|
|
2526
|
+
await this.writePullWakeEvent(subscription, triggeredBy);
|
|
2527
|
+
}
|
|
2528
|
+
async deliverWebhook(subscription, triggeredBy) {
|
|
2529
|
+
if (!subscription.webhook || !subscription.wake_id || !subscription.token) return;
|
|
2530
|
+
const body = JSON.stringify({
|
|
2531
|
+
subscription_id: subscription.id,
|
|
2532
|
+
wake_id: subscription.wake_id,
|
|
2533
|
+
generation: subscription.generation,
|
|
2534
|
+
streams: this.streamInfos(subscription),
|
|
2535
|
+
callback_url: this.subscriptionActionUrl(subscription, `callback`),
|
|
2536
|
+
callback_token: subscription.token
|
|
2537
|
+
});
|
|
2538
|
+
const headers = {
|
|
2539
|
+
"content-type": `application/json`,
|
|
2540
|
+
"webhook-signature": signWebhookPayload(body)
|
|
2541
|
+
};
|
|
2542
|
+
try {
|
|
2543
|
+
const response = await fetch(subscription.webhook.url, {
|
|
2544
|
+
method: `POST`,
|
|
2545
|
+
headers,
|
|
2546
|
+
body
|
|
2547
|
+
});
|
|
2548
|
+
if (!response.ok) {
|
|
2549
|
+
this.scheduleWebhookRetry(subscription, triggeredBy);
|
|
2550
|
+
return;
|
|
2551
|
+
}
|
|
2552
|
+
subscription.status = `active`;
|
|
2553
|
+
subscription.retry_count = 0;
|
|
2554
|
+
subscription.next_attempt_at = null;
|
|
2555
|
+
let parsed = null;
|
|
2556
|
+
try {
|
|
2557
|
+
parsed = await response.json();
|
|
2558
|
+
} catch {
|
|
2559
|
+
parsed = null;
|
|
2560
|
+
}
|
|
2561
|
+
if (parsed?.done === true) {
|
|
2562
|
+
this.autoAckWakeSnapshot(subscription);
|
|
2563
|
+
this.clearLease(subscription);
|
|
2564
|
+
subscription.token = null;
|
|
2565
|
+
subscription.holder = null;
|
|
2566
|
+
subscription.wake_id = null;
|
|
2567
|
+
subscription.wake_snapshot.clear();
|
|
2568
|
+
await this.triggerNextWakeIfPending(subscription);
|
|
2569
|
+
}
|
|
2570
|
+
} catch (err) {
|
|
2571
|
+
serverLog.warn(`[subscriptions] webhook delivery failed:`, err);
|
|
2572
|
+
this.scheduleWebhookRetry(subscription, triggeredBy);
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
scheduleWebhookRetry(subscription, triggeredBy) {
|
|
2576
|
+
if (this.isShuttingDown) return;
|
|
2577
|
+
subscription.retry_count++;
|
|
2578
|
+
const baseDelay = Math.min(1e3 * Math.pow(2, Math.max(0, subscription.retry_count - 1)), MAX_RETRY_DELAY_MS);
|
|
2579
|
+
const jitter = baseDelay * .2 * (Math.random() * 2 - 1);
|
|
2580
|
+
const delay = Math.max(0, Math.round(baseDelay + jitter));
|
|
2581
|
+
subscription.status = `failed`;
|
|
2582
|
+
subscription.next_attempt_at = Date.now() + delay;
|
|
2583
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2584
|
+
subscription.retry_timer = setTimeout(() => {
|
|
2585
|
+
subscription.retry_timer = null;
|
|
2586
|
+
this.deliverWebhook(subscription, triggeredBy);
|
|
2587
|
+
}, delay);
|
|
2588
|
+
}
|
|
2589
|
+
async writePullWakeEvent(subscription, streamPath) {
|
|
2590
|
+
if (!subscription.wake_stream) return;
|
|
2591
|
+
const wakeStream = toAbsoluteStreamPath(subscription.wake_stream);
|
|
2592
|
+
if (!this.streamStore.has(wakeStream)) {
|
|
2593
|
+
serverLog.warn(`[subscriptions] wake stream does not exist: ${wakeStream}`);
|
|
2594
|
+
return;
|
|
2595
|
+
}
|
|
2596
|
+
const event = {
|
|
2597
|
+
type: `wake`,
|
|
2598
|
+
subscription_id: subscription.id,
|
|
2599
|
+
stream: streamPath,
|
|
2600
|
+
generation: subscription.generation,
|
|
2601
|
+
ts: Date.now()
|
|
2602
|
+
};
|
|
2603
|
+
await Promise.resolve(this.streamStore.append(wakeStream, new TextEncoder().encode(JSON.stringify(event))));
|
|
2604
|
+
}
|
|
2605
|
+
autoAckWakeSnapshot(subscription) {
|
|
2606
|
+
for (const [stream, tail] of subscription.wake_snapshot) {
|
|
2607
|
+
const link = subscription.streams.get(stream);
|
|
2608
|
+
if (link) link.acked_offset = tail;
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
applyAcks(subscription, request) {
|
|
2612
|
+
if (!request.acks) return null;
|
|
2613
|
+
for (const ack of request.acks) {
|
|
2614
|
+
const stream = normalizeRelativePath(ack.stream ?? ack.path ?? ``);
|
|
2615
|
+
const link = subscription.streams.get(stream);
|
|
2616
|
+
if (!stream || !link) return this.errorResponse(409, `INVALID_OFFSET`, `Ack references an unknown subscription stream`);
|
|
2617
|
+
if (ack.offset === BEFORE_FIRST_OFFSET) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset must not be -1`);
|
|
2618
|
+
if (compareOffsets(ack.offset, link.acked_offset) < 0) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset regresses the committed cursor`);
|
|
2619
|
+
if (compareOffsets(ack.offset, this.getTailOffset(stream)) > 0) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset is beyond stream tail`);
|
|
2620
|
+
}
|
|
2621
|
+
for (const ack of request.acks) {
|
|
2622
|
+
const stream = normalizeRelativePath(ack.stream ?? ack.path ?? ``);
|
|
2623
|
+
subscription.streams.get(stream).acked_offset = ack.offset;
|
|
2624
|
+
}
|
|
2625
|
+
return null;
|
|
2626
|
+
}
|
|
2627
|
+
validateWakeToken(subscription, token, request) {
|
|
2628
|
+
const tokenResult = validateCallbackToken(token, this.tokenSubject(subscription));
|
|
2629
|
+
if (!tokenResult.valid) return this.errorResponse(401, tokenResult.code, tokenResult.code === `TOKEN_EXPIRED` ? `Token expired` : `Token invalid`);
|
|
2630
|
+
if (tokenResult.epoch !== subscription.generation || request.generation !== subscription.generation || request.wake_id !== subscription.wake_id) return this.errorResponse(409, `FENCED`, `Wake generation is stale`);
|
|
2631
|
+
return null;
|
|
2632
|
+
}
|
|
2633
|
+
async triggerNextWakeIfPending(subscription) {
|
|
2634
|
+
if (!this.hasPendingWork(subscription)) return false;
|
|
2635
|
+
await this.createWake(subscription, this.firstPendingStream(subscription));
|
|
2636
|
+
return true;
|
|
2637
|
+
}
|
|
2638
|
+
hasPendingWork(subscription) {
|
|
2639
|
+
return this.streamInfos(subscription).some((stream) => stream.has_pending);
|
|
2640
|
+
}
|
|
2641
|
+
firstPendingStream(subscription) {
|
|
2642
|
+
return this.streamInfos(subscription).find((stream) => stream.has_pending)?.path ?? ``;
|
|
2643
|
+
}
|
|
2644
|
+
streamInfos(subscription) {
|
|
2645
|
+
return Array.from(subscription.streams.values()).map((link) => {
|
|
2646
|
+
const tail = this.getTailOffset(link.path);
|
|
2647
|
+
return {
|
|
2648
|
+
path: link.path,
|
|
2649
|
+
link_type: link.link_types.has(`explicit`) ? `explicit` : `glob`,
|
|
2650
|
+
acked_offset: link.acked_offset,
|
|
2651
|
+
tail_offset: tail,
|
|
2652
|
+
has_pending: compareOffsets(tail, link.acked_offset) > 0
|
|
2653
|
+
};
|
|
2654
|
+
});
|
|
2655
|
+
}
|
|
2656
|
+
linkStream(subscription, streamPath, linkType, ackedOffset) {
|
|
2657
|
+
const normalized = normalizeRelativePath(streamPath);
|
|
2658
|
+
const existing = subscription.streams.get(normalized);
|
|
2659
|
+
if (existing) {
|
|
2660
|
+
existing.link_types.add(linkType);
|
|
2661
|
+
return existing;
|
|
2662
|
+
}
|
|
2663
|
+
const link = {
|
|
2664
|
+
path: normalized,
|
|
2665
|
+
link_types: new Set([linkType]),
|
|
2666
|
+
acked_offset: ackedOffset
|
|
2667
|
+
};
|
|
2668
|
+
subscription.streams.set(normalized, link);
|
|
2669
|
+
return link;
|
|
2670
|
+
}
|
|
2671
|
+
listStreams() {
|
|
2672
|
+
return this.streamStore.list().map((path$1) => toStreamRelativePath(path$1)).filter((path$1) => path$1 !== null);
|
|
2673
|
+
}
|
|
2674
|
+
getTailOffset(streamPath) {
|
|
2675
|
+
return this.streamStore.get(toAbsoluteStreamPath(streamPath))?.currentOffset ?? ZERO_OFFSET;
|
|
2676
|
+
}
|
|
2677
|
+
subscriptionActionUrl(subscription, action) {
|
|
2678
|
+
const url = new URL(`/v1/stream/__ds/subscriptions/${encodeURIComponent(subscription.id)}/${action}`, this.callbackBaseUrl);
|
|
2679
|
+
return url.toString();
|
|
2680
|
+
}
|
|
2681
|
+
webhookJwksUrl() {
|
|
2682
|
+
const url = new URL(`/v1/stream/__ds/jwks.json`, this.callbackBaseUrl);
|
|
2683
|
+
return url.toString();
|
|
2684
|
+
}
|
|
2685
|
+
webhookSigningMetadata() {
|
|
2686
|
+
return {
|
|
2687
|
+
alg: `ed25519`,
|
|
2688
|
+
kid: getWebhookSigningKeyId(),
|
|
2689
|
+
jwks_url: this.webhookJwksUrl()
|
|
2690
|
+
};
|
|
2691
|
+
}
|
|
2692
|
+
extendLease(subscription) {
|
|
2693
|
+
this.clearLease(subscription);
|
|
2694
|
+
subscription.lease_timer = setTimeout(() => {
|
|
2695
|
+
subscription.lease_timer = null;
|
|
2696
|
+
subscription.holder = null;
|
|
2697
|
+
subscription.token = null;
|
|
2698
|
+
subscription.wake_id = null;
|
|
2699
|
+
subscription.wake_snapshot.clear();
|
|
2700
|
+
this.triggerNextWakeIfPending(subscription);
|
|
2701
|
+
}, subscription.lease_ttl_ms);
|
|
2702
|
+
}
|
|
2703
|
+
clearLease(subscription) {
|
|
2704
|
+
if (subscription.lease_timer) {
|
|
2705
|
+
clearTimeout(subscription.lease_timer);
|
|
2706
|
+
subscription.lease_timer = null;
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
tokenSubject(subscription) {
|
|
2710
|
+
return `subscription:${subscription.id}`;
|
|
2711
|
+
}
|
|
2712
|
+
errorResponse(status, code, message) {
|
|
2713
|
+
return {
|
|
2714
|
+
status,
|
|
2715
|
+
body: { error: {
|
|
2716
|
+
code,
|
|
2717
|
+
message
|
|
2718
|
+
} }
|
|
2719
|
+
};
|
|
2720
|
+
}
|
|
2721
|
+
};
|
|
2722
|
+
|
|
2723
|
+
//#endregion
|
|
2724
|
+
//#region src/subscription-routes.ts
|
|
2725
|
+
const RESERVED_CONTROL_PREFIX = `/v1/stream/__ds`;
|
|
2726
|
+
const SUBSCRIPTION_PREFIX = `${RESERVED_CONTROL_PREFIX}/subscriptions/`;
|
|
2727
|
+
const JWKS_PATH = `${RESERVED_CONTROL_PREFIX}/jwks.json`;
|
|
2728
|
+
const ERROR_STATUS = {
|
|
2729
|
+
INVALID_REQUEST: 400,
|
|
2730
|
+
SUBSCRIPTION_NOT_FOUND: 404,
|
|
2731
|
+
SUBSCRIPTION_ALREADY_EXISTS: 409,
|
|
2732
|
+
WEBHOOK_URL_REJECTED: 400,
|
|
2733
|
+
TOKEN_INVALID: 401,
|
|
2734
|
+
TOKEN_EXPIRED: 401,
|
|
2735
|
+
FENCED: 409,
|
|
2736
|
+
ALREADY_CLAIMED: 409,
|
|
2737
|
+
NO_PENDING_WORK: 409,
|
|
2738
|
+
INVALID_OFFSET: 409
|
|
2739
|
+
};
|
|
2740
|
+
var SubscriptionRoutes = class {
|
|
2741
|
+
manager;
|
|
2742
|
+
constructor(manager) {
|
|
2743
|
+
this.manager = manager;
|
|
2744
|
+
}
|
|
2745
|
+
async handleRequest(method, path$1, req, res) {
|
|
2746
|
+
if (path$1 === JWKS_PATH) {
|
|
2747
|
+
this.handleJwks(method, res);
|
|
2748
|
+
return true;
|
|
2749
|
+
}
|
|
2750
|
+
const route = this.parseRoute(path$1);
|
|
2751
|
+
if (!route) {
|
|
2752
|
+
if (path$1 === RESERVED_CONTROL_PREFIX || path$1.startsWith(`${RESERVED_CONTROL_PREFIX}/`)) {
|
|
2753
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Durable Streams control route not found`);
|
|
2754
|
+
return true;
|
|
2755
|
+
}
|
|
2756
|
+
return false;
|
|
2757
|
+
}
|
|
2758
|
+
try {
|
|
2759
|
+
switch (route.action) {
|
|
2760
|
+
case `base`:
|
|
2761
|
+
await this.handleBase(route, method, req, res);
|
|
2762
|
+
return true;
|
|
2763
|
+
case `streams`:
|
|
2764
|
+
await this.handleStreams(route, method, req, res);
|
|
2765
|
+
return true;
|
|
2766
|
+
case `stream`:
|
|
2767
|
+
this.handleStream(route, method, res);
|
|
2768
|
+
return true;
|
|
2769
|
+
case `callback`:
|
|
2770
|
+
await this.handleCallback(route, req, res);
|
|
2771
|
+
return true;
|
|
2772
|
+
case `claim`:
|
|
2773
|
+
await this.handleClaim(route, req, res);
|
|
2774
|
+
return true;
|
|
2775
|
+
case `ack`:
|
|
2776
|
+
await this.handleAck(route, req, res);
|
|
2777
|
+
return true;
|
|
2778
|
+
case `release`:
|
|
2779
|
+
await this.handleRelease(route, req, res);
|
|
2780
|
+
return true;
|
|
2781
|
+
}
|
|
2782
|
+
} catch (err) {
|
|
2783
|
+
if (err instanceof SyntaxError) {
|
|
2784
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `Invalid JSON body`);
|
|
2785
|
+
return true;
|
|
2786
|
+
}
|
|
2787
|
+
throw err;
|
|
2788
|
+
}
|
|
2789
|
+
}
|
|
2790
|
+
async handleBase(route, method, req, res) {
|
|
2791
|
+
if (method === `PUT`) {
|
|
2792
|
+
const parsed = await this.readJson(req);
|
|
2793
|
+
const input = this.parseCreateInput(parsed);
|
|
2794
|
+
if (`error` in input) {
|
|
2795
|
+
this.writeError(res, 400, `INVALID_REQUEST`, input.error);
|
|
2796
|
+
return;
|
|
2797
|
+
}
|
|
2798
|
+
const result = this.manager.createOrConfirm(route.subscriptionId, input.value);
|
|
2799
|
+
if (`error` in result) {
|
|
2800
|
+
this.writeError(res, ERROR_STATUS[result.error.code], result.error.code, result.error.message);
|
|
2801
|
+
return;
|
|
2802
|
+
}
|
|
2803
|
+
this.writeJson(res, result.created ? 201 : 200, this.manager.serialize(result.subscription));
|
|
2804
|
+
return;
|
|
2805
|
+
}
|
|
2806
|
+
if (method === `GET`) {
|
|
2807
|
+
const subscription = this.manager.get(route.subscriptionId);
|
|
2808
|
+
if (!subscription) {
|
|
2809
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2810
|
+
return;
|
|
2811
|
+
}
|
|
2812
|
+
this.writeJson(res, 200, this.manager.serialize(subscription));
|
|
2813
|
+
return;
|
|
2814
|
+
}
|
|
2815
|
+
if (method === `DELETE`) {
|
|
2816
|
+
this.manager.delete(route.subscriptionId);
|
|
2817
|
+
res.writeHead(204);
|
|
2818
|
+
res.end();
|
|
2819
|
+
return;
|
|
2820
|
+
}
|
|
2821
|
+
this.methodNotAllowed(res);
|
|
2822
|
+
}
|
|
2823
|
+
handleJwks(method, res) {
|
|
2824
|
+
if (method !== `GET`) {
|
|
2825
|
+
this.methodNotAllowed(res);
|
|
2826
|
+
return;
|
|
2827
|
+
}
|
|
2828
|
+
res.writeHead(200, {
|
|
2829
|
+
"content-type": `application/jwk-set+json`,
|
|
2830
|
+
"cache-control": `public, max-age=300`
|
|
2831
|
+
});
|
|
2832
|
+
res.end(JSON.stringify(this.manager.getWebhookJwks()));
|
|
2833
|
+
}
|
|
2834
|
+
async handleStreams(route, method, req, res) {
|
|
2835
|
+
if (method !== `POST`) {
|
|
2836
|
+
this.methodNotAllowed(res);
|
|
2837
|
+
return;
|
|
2838
|
+
}
|
|
2839
|
+
const parsed = await this.readJson(req);
|
|
2840
|
+
const streams = parsed.streams;
|
|
2841
|
+
if (!Array.isArray(streams) || streams.some((stream) => typeof stream !== `string` || stream.length === 0)) {
|
|
2842
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `streams must be a non-empty string array`);
|
|
2843
|
+
return;
|
|
2844
|
+
}
|
|
2845
|
+
const ok = this.manager.addExplicitStreams(route.subscriptionId, streams.map(normalizeRelativePath));
|
|
2846
|
+
if (!ok) {
|
|
2847
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2848
|
+
return;
|
|
2849
|
+
}
|
|
2850
|
+
res.writeHead(204);
|
|
2851
|
+
res.end();
|
|
2852
|
+
}
|
|
2853
|
+
handleStream(route, method, res) {
|
|
2854
|
+
if (method !== `DELETE`) {
|
|
2855
|
+
this.methodNotAllowed(res);
|
|
2856
|
+
return;
|
|
2857
|
+
}
|
|
2858
|
+
const ok = this.manager.removeExplicitStream(route.subscriptionId, route.streamPath ?? ``);
|
|
2859
|
+
if (!ok) {
|
|
2860
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2861
|
+
return;
|
|
2862
|
+
}
|
|
2863
|
+
res.writeHead(204);
|
|
2864
|
+
res.end();
|
|
2865
|
+
}
|
|
2866
|
+
async handleCallback(route, req, res) {
|
|
2867
|
+
const token = this.readBearerToken(req);
|
|
2868
|
+
if (!token) {
|
|
2869
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2870
|
+
return;
|
|
2871
|
+
}
|
|
2872
|
+
const body = await this.readJson(req);
|
|
2873
|
+
const result = await this.manager.handleWebhookCallback(route.subscriptionId, token, body);
|
|
2874
|
+
this.writeManagerResult(res, result);
|
|
2875
|
+
}
|
|
2876
|
+
async handleClaim(route, req, res) {
|
|
2877
|
+
const parsed = await this.readJson(req);
|
|
2878
|
+
const worker = parsed.worker;
|
|
2879
|
+
if (typeof worker !== `string` || worker.length === 0) {
|
|
2880
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `worker must be a non-empty string`);
|
|
2881
|
+
return;
|
|
2882
|
+
}
|
|
2883
|
+
const result = await this.manager.claim(route.subscriptionId, worker);
|
|
2884
|
+
this.writeManagerResult(res, result);
|
|
2885
|
+
}
|
|
2886
|
+
async handleAck(route, req, res) {
|
|
2887
|
+
const token = this.readBearerToken(req);
|
|
2888
|
+
if (!token) {
|
|
2889
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2890
|
+
return;
|
|
2891
|
+
}
|
|
2892
|
+
const body = await this.readJson(req);
|
|
2893
|
+
const result = await this.manager.ack(route.subscriptionId, token, body);
|
|
2894
|
+
this.writeManagerResult(res, result);
|
|
2895
|
+
}
|
|
2896
|
+
async handleRelease(route, req, res) {
|
|
2897
|
+
const token = this.readBearerToken(req);
|
|
2898
|
+
if (!token) {
|
|
2899
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2900
|
+
return;
|
|
2901
|
+
}
|
|
2902
|
+
const body = await this.readJson(req);
|
|
2903
|
+
const result = await this.manager.release(route.subscriptionId, token, body);
|
|
2904
|
+
this.writeManagerResult(res, result);
|
|
2905
|
+
}
|
|
2906
|
+
parseCreateInput(value) {
|
|
2907
|
+
if (!value || typeof value !== `object`) return { error: `Request body must be a JSON object` };
|
|
2908
|
+
const payload = value;
|
|
2909
|
+
if (payload.type !== `webhook` && payload.type !== `pull-wake`) return { error: `type must be "webhook" or "pull-wake"` };
|
|
2910
|
+
const type = payload.type;
|
|
2911
|
+
const pattern = typeof payload.pattern === `string` && payload.pattern.length > 0 ? normalizeRelativePath(payload.pattern) : void 0;
|
|
2912
|
+
const streams = Array.isArray(payload.streams) && payload.streams.length > 0 ? payload.streams.map((stream) => typeof stream === `string` ? normalizeRelativePath(stream) : null) : [];
|
|
2913
|
+
if (streams.some((stream) => stream === null)) return { error: `streams must contain only strings` };
|
|
2914
|
+
if (!pattern && streams.length === 0) return { error: `At least one of pattern or streams is required` };
|
|
2915
|
+
const leaseTtl = payload.lease_ttl_ms === void 0 ? DEFAULT_LEASE_TTL_MS : payload.lease_ttl_ms;
|
|
2916
|
+
if (typeof leaseTtl !== `number` || !Number.isInteger(leaseTtl) || leaseTtl < MIN_LEASE_TTL_MS || leaseTtl > MAX_LEASE_TTL_MS) return { error: `lease_ttl_ms must be an integer from 1000 to 600000` };
|
|
2917
|
+
let webhook;
|
|
2918
|
+
if (type === `webhook`) {
|
|
2919
|
+
const rawWebhook = payload.webhook;
|
|
2920
|
+
if (!rawWebhook || typeof rawWebhook !== `object`) return { error: `webhook subscriptions require webhook.url` };
|
|
2921
|
+
const url = rawWebhook.url;
|
|
2922
|
+
if (typeof url !== `string` || url.length === 0) return { error: `webhook subscriptions require webhook.url` };
|
|
2923
|
+
webhook = { url };
|
|
2924
|
+
}
|
|
2925
|
+
const wakeStream = typeof payload.wake_stream === `string` && payload.wake_stream.length > 0 ? normalizeRelativePath(payload.wake_stream) : void 0;
|
|
2926
|
+
if (type === `pull-wake` && !wakeStream) return { error: `pull-wake subscriptions require wake_stream` };
|
|
2927
|
+
return { value: {
|
|
2928
|
+
type,
|
|
2929
|
+
pattern,
|
|
2930
|
+
streams,
|
|
2931
|
+
webhook,
|
|
2932
|
+
wake_stream: wakeStream,
|
|
2933
|
+
lease_ttl_ms: leaseTtl,
|
|
2934
|
+
description: typeof payload.description === `string` ? payload.description : void 0
|
|
2935
|
+
} };
|
|
2936
|
+
}
|
|
2937
|
+
parseRoute(path$1) {
|
|
2938
|
+
if (!path$1.startsWith(SUBSCRIPTION_PREFIX)) return null;
|
|
2939
|
+
const rest = path$1.slice(SUBSCRIPTION_PREFIX.length);
|
|
2940
|
+
const parts = rest.split(`/`);
|
|
2941
|
+
const subscriptionId = parts[0] ? decodeURIComponent(parts[0]) : ``;
|
|
2942
|
+
if (!subscriptionId) return null;
|
|
2943
|
+
const tail = parts.slice(1);
|
|
2944
|
+
if (tail.length === 0) return {
|
|
2945
|
+
subscriptionId,
|
|
2946
|
+
action: `base`
|
|
2947
|
+
};
|
|
2948
|
+
if (tail[0] === `streams` && tail.length === 1) return {
|
|
2949
|
+
subscriptionId,
|
|
2950
|
+
action: `streams`
|
|
2951
|
+
};
|
|
2952
|
+
if (tail[0] === `streams` && tail.length > 1) return {
|
|
2953
|
+
subscriptionId,
|
|
2954
|
+
action: `stream`,
|
|
2955
|
+
streamPath: normalizeRelativePath(decodeURIComponent(tail.slice(1).join(`/`)))
|
|
2956
|
+
};
|
|
2957
|
+
if (tail.length === 1 && [
|
|
2958
|
+
`callback`,
|
|
2959
|
+
`claim`,
|
|
2960
|
+
`ack`,
|
|
2961
|
+
`release`
|
|
2962
|
+
].includes(tail[0])) return {
|
|
2963
|
+
subscriptionId,
|
|
2964
|
+
action: tail[0]
|
|
2965
|
+
};
|
|
2966
|
+
return null;
|
|
2967
|
+
}
|
|
2968
|
+
readBearerToken(req) {
|
|
2969
|
+
const authHeader = req.headers.authorization;
|
|
2970
|
+
if (!authHeader || !authHeader.startsWith(`Bearer `)) return null;
|
|
2971
|
+
return authHeader.slice(`Bearer `.length);
|
|
2972
|
+
}
|
|
2973
|
+
async readJson(req) {
|
|
2974
|
+
const chunks = [];
|
|
2975
|
+
for await (const chunk of req) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
2976
|
+
const raw = Buffer.concat(chunks).toString(`utf8`);
|
|
2977
|
+
return raw.length > 0 ? JSON.parse(raw) : {};
|
|
2978
|
+
}
|
|
2979
|
+
writeManagerResult(res, result) {
|
|
2980
|
+
if (result.status === 204) {
|
|
2981
|
+
res.writeHead(204);
|
|
2982
|
+
res.end();
|
|
2983
|
+
return;
|
|
2984
|
+
}
|
|
2985
|
+
this.writeJson(res, result.status, result.body ?? {});
|
|
2986
|
+
}
|
|
2987
|
+
writeJson(res, status, body) {
|
|
2988
|
+
res.writeHead(status, { "content-type": `application/json` });
|
|
2989
|
+
res.end(JSON.stringify(body));
|
|
2990
|
+
}
|
|
2991
|
+
writeError(res, status, code, message) {
|
|
2992
|
+
this.writeJson(res, status, { error: {
|
|
2993
|
+
code,
|
|
2994
|
+
message
|
|
2995
|
+
} });
|
|
2996
|
+
}
|
|
2997
|
+
methodNotAllowed(res) {
|
|
2998
|
+
res.writeHead(405, { "content-type": `text/plain` });
|
|
2999
|
+
res.end(`Method not allowed`);
|
|
3000
|
+
}
|
|
3001
|
+
};
|
|
3002
|
+
|
|
1943
3003
|
//#endregion
|
|
1944
3004
|
//#region src/server.ts
|
|
1945
|
-
const STREAM_OFFSET_HEADER = `Stream-Next-Offset`;
|
|
1946
|
-
const STREAM_CURSOR_HEADER = `Stream-Cursor`;
|
|
1947
|
-
const STREAM_UP_TO_DATE_HEADER = `Stream-Up-To-Date`;
|
|
1948
|
-
const STREAM_SEQ_HEADER = `Stream-Seq`;
|
|
1949
|
-
const STREAM_TTL_HEADER = `Stream-TTL`;
|
|
1950
|
-
const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`;
|
|
1951
3005
|
const STREAM_SSE_DATA_ENCODING_HEADER = `Stream-SSE-Data-Encoding`;
|
|
1952
|
-
const PRODUCER_ID_HEADER = `Producer-Id`;
|
|
1953
|
-
const PRODUCER_EPOCH_HEADER = `Producer-Epoch`;
|
|
1954
|
-
const PRODUCER_SEQ_HEADER = `Producer-Seq`;
|
|
1955
|
-
const PRODUCER_EXPECTED_SEQ_HEADER = `Producer-Expected-Seq`;
|
|
1956
|
-
const PRODUCER_RECEIVED_SEQ_HEADER = `Producer-Received-Seq`;
|
|
1957
|
-
const SSE_OFFSET_FIELD = `streamNextOffset`;
|
|
1958
|
-
const SSE_CURSOR_FIELD = `streamCursor`;
|
|
1959
3006
|
const SSE_UP_TO_DATE_FIELD = `upToDate`;
|
|
1960
|
-
const SSE_CLOSED_FIELD = `streamClosed`;
|
|
1961
|
-
const STREAM_CLOSED_HEADER = `Stream-Closed`;
|
|
1962
3007
|
const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`;
|
|
1963
3008
|
const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`;
|
|
1964
|
-
const OFFSET_QUERY_PARAM = `offset`;
|
|
1965
|
-
const LIVE_QUERY_PARAM = `live`;
|
|
1966
|
-
const CURSOR_QUERY_PARAM = `cursor`;
|
|
1967
3009
|
/**
|
|
1968
3010
|
* Encode data for SSE format.
|
|
1969
3011
|
* Per SSE spec, each line in the payload needs its own "data:" prefix.
|
|
@@ -2017,6 +3059,8 @@ var DurableStreamTestServer = class {
|
|
|
2017
3059
|
isShuttingDown = false;
|
|
2018
3060
|
/** Injected faults for testing retry/resilience */
|
|
2019
3061
|
injectedFaults = new Map();
|
|
3062
|
+
subscriptionManager = null;
|
|
3063
|
+
subscriptionRoutes = null;
|
|
2020
3064
|
constructor(options = {}) {
|
|
2021
3065
|
if (options.dataDir) this.store = new FileBackedStreamStore({ dataDir: options.dataDir });
|
|
2022
3066
|
else this.store = new StreamStore();
|
|
@@ -2031,7 +3075,8 @@ var DurableStreamTestServer = class {
|
|
|
2031
3075
|
cursorOptions: {
|
|
2032
3076
|
intervalSeconds: options.cursorIntervalSeconds,
|
|
2033
3077
|
epoch: options.cursorEpoch
|
|
2034
|
-
}
|
|
3078
|
+
},
|
|
3079
|
+
webhooks: options.webhooks ?? false
|
|
2035
3080
|
};
|
|
2036
3081
|
}
|
|
2037
3082
|
/**
|
|
@@ -2042,7 +3087,7 @@ var DurableStreamTestServer = class {
|
|
|
2042
3087
|
return new Promise((resolve, reject) => {
|
|
2043
3088
|
this.server = createServer((req, res) => {
|
|
2044
3089
|
this.handleRequest(req, res).catch((err) => {
|
|
2045
|
-
|
|
3090
|
+
serverLog.error(`Request error:`, err);
|
|
2046
3091
|
if (!res.headersSent) {
|
|
2047
3092
|
res.writeHead(500, { "content-type": `text/plain` });
|
|
2048
3093
|
res.end(`Internal server error`);
|
|
@@ -2054,6 +3099,12 @@ var DurableStreamTestServer = class {
|
|
|
2054
3099
|
const addr = this.server.address();
|
|
2055
3100
|
if (typeof addr === `string`) this._url = addr;
|
|
2056
3101
|
else if (addr) this._url = `http://${this.options.host}:${addr.port}`;
|
|
3102
|
+
this.subscriptionManager = new SubscriptionManager({
|
|
3103
|
+
callbackBaseUrl: this._url,
|
|
3104
|
+
streamStore: this.store,
|
|
3105
|
+
webhooksEnabled: this.options.webhooks
|
|
3106
|
+
});
|
|
3107
|
+
this.subscriptionRoutes = new SubscriptionRoutes(this.subscriptionManager);
|
|
2057
3108
|
resolve(this._url);
|
|
2058
3109
|
});
|
|
2059
3110
|
});
|
|
@@ -2064,6 +3115,11 @@ var DurableStreamTestServer = class {
|
|
|
2064
3115
|
async stop() {
|
|
2065
3116
|
if (!this.server) return;
|
|
2066
3117
|
this.isShuttingDown = true;
|
|
3118
|
+
if (this.subscriptionManager) {
|
|
3119
|
+
this.subscriptionManager.shutdown();
|
|
3120
|
+
this.subscriptionManager = null;
|
|
3121
|
+
this.subscriptionRoutes = null;
|
|
3122
|
+
}
|
|
2067
3123
|
if (`cancelAllWaits` in this.store) this.store.cancelAllWaits();
|
|
2068
3124
|
for (const res of this.activeSSEResponses) res.end();
|
|
2069
3125
|
this.activeSSEResponses.clear();
|
|
@@ -2103,8 +3159,8 @@ var DurableStreamTestServer = class {
|
|
|
2103
3159
|
* Used for testing retry/resilience behavior.
|
|
2104
3160
|
* @deprecated Use injectFault for full fault injection capabilities
|
|
2105
3161
|
*/
|
|
2106
|
-
injectError(path$
|
|
2107
|
-
this.injectedFaults.set(path$
|
|
3162
|
+
injectError(path$1, status, count = 1, retryAfter) {
|
|
3163
|
+
this.injectedFaults.set(path$1, {
|
|
2108
3164
|
status,
|
|
2109
3165
|
count,
|
|
2110
3166
|
retryAfter
|
|
@@ -2114,8 +3170,8 @@ var DurableStreamTestServer = class {
|
|
|
2114
3170
|
* Inject a fault to be triggered on the next N requests to a path.
|
|
2115
3171
|
* Supports various fault types: delays, connection drops, body corruption, etc.
|
|
2116
3172
|
*/
|
|
2117
|
-
injectFault(path$
|
|
2118
|
-
this.injectedFaults.set(path$
|
|
3173
|
+
injectFault(path$1, fault) {
|
|
3174
|
+
this.injectedFaults.set(path$1, {
|
|
2119
3175
|
count: 1,
|
|
2120
3176
|
...fault
|
|
2121
3177
|
});
|
|
@@ -2130,13 +3186,13 @@ var DurableStreamTestServer = class {
|
|
|
2130
3186
|
* Check if there's an injected fault for this path/method and consume it.
|
|
2131
3187
|
* Returns the fault config if one should be triggered, null otherwise.
|
|
2132
3188
|
*/
|
|
2133
|
-
consumeInjectedFault(path$
|
|
2134
|
-
const fault = this.injectedFaults.get(path$
|
|
3189
|
+
consumeInjectedFault(path$1, method) {
|
|
3190
|
+
const fault = this.injectedFaults.get(path$1);
|
|
2135
3191
|
if (!fault) return null;
|
|
2136
3192
|
if (fault.method && fault.method.toUpperCase() !== method.toUpperCase()) return null;
|
|
2137
3193
|
if (fault.probability !== void 0 && Math.random() > fault.probability) return null;
|
|
2138
3194
|
fault.count--;
|
|
2139
|
-
if (fault.count <= 0) this.injectedFaults.delete(path$
|
|
3195
|
+
if (fault.count <= 0) this.injectedFaults.delete(path$1);
|
|
2140
3196
|
return fault;
|
|
2141
3197
|
}
|
|
2142
3198
|
/**
|
|
@@ -2171,7 +3227,7 @@ var DurableStreamTestServer = class {
|
|
|
2171
3227
|
}
|
|
2172
3228
|
async handleRequest(req, res) {
|
|
2173
3229
|
const url = new URL(req.url ?? `/`, `http://${req.headers.host}`);
|
|
2174
|
-
const path$
|
|
3230
|
+
const path$1 = url.pathname;
|
|
2175
3231
|
const method = req.method?.toUpperCase();
|
|
2176
3232
|
res.setHeader(`access-control-allow-origin`, `*`);
|
|
2177
3233
|
res.setHeader(`access-control-allow-methods`, `GET, POST, PUT, DELETE, HEAD, OPTIONS`);
|
|
@@ -2184,11 +3240,11 @@ var DurableStreamTestServer = class {
|
|
|
2184
3240
|
res.end();
|
|
2185
3241
|
return;
|
|
2186
3242
|
}
|
|
2187
|
-
if (path$
|
|
3243
|
+
if (path$1 === `/_test/inject-error`) {
|
|
2188
3244
|
await this.handleTestInjectError(method, req, res);
|
|
2189
3245
|
return;
|
|
2190
3246
|
}
|
|
2191
|
-
const fault = this.consumeInjectedFault(path$
|
|
3247
|
+
const fault = this.consumeInjectedFault(path$1, method ?? `GET`);
|
|
2192
3248
|
if (fault) {
|
|
2193
3249
|
await this.applyFaultDelay(fault);
|
|
2194
3250
|
if (fault.dropConnection) {
|
|
@@ -2204,22 +3260,26 @@ var DurableStreamTestServer = class {
|
|
|
2204
3260
|
}
|
|
2205
3261
|
if (fault.truncateBodyBytes !== void 0 || fault.corruptBody || fault.injectSseEvent) res._injectedFault = fault;
|
|
2206
3262
|
}
|
|
3263
|
+
if (this.subscriptionRoutes && method) {
|
|
3264
|
+
const handled = await this.subscriptionRoutes.handleRequest(method, path$1, req, res);
|
|
3265
|
+
if (handled) return;
|
|
3266
|
+
}
|
|
2207
3267
|
try {
|
|
2208
3268
|
switch (method) {
|
|
2209
3269
|
case `PUT`:
|
|
2210
|
-
await this.handleCreate(path$
|
|
3270
|
+
await this.handleCreate(path$1, req, res);
|
|
2211
3271
|
break;
|
|
2212
3272
|
case `HEAD`:
|
|
2213
|
-
this.handleHead(path$
|
|
3273
|
+
this.handleHead(path$1, res);
|
|
2214
3274
|
break;
|
|
2215
3275
|
case `GET`:
|
|
2216
|
-
await this.handleRead(path$
|
|
3276
|
+
await this.handleRead(path$1, url, req, res);
|
|
2217
3277
|
break;
|
|
2218
3278
|
case `POST`:
|
|
2219
|
-
await this.handleAppend(path$
|
|
3279
|
+
await this.handleAppend(path$1, req, res);
|
|
2220
3280
|
break;
|
|
2221
3281
|
case `DELETE`:
|
|
2222
|
-
await this.handleDelete(path$
|
|
3282
|
+
await this.handleDelete(path$1, res);
|
|
2223
3283
|
break;
|
|
2224
3284
|
default:
|
|
2225
3285
|
res.writeHead(405, { "content-type": `text/plain` });
|
|
@@ -2257,15 +3317,15 @@ var DurableStreamTestServer = class {
|
|
|
2257
3317
|
/**
|
|
2258
3318
|
* Handle PUT - create stream
|
|
2259
3319
|
*/
|
|
2260
|
-
async handleCreate(path$
|
|
3320
|
+
async handleCreate(path$1, req, res) {
|
|
2261
3321
|
let contentType = req.headers[`content-type`];
|
|
2262
|
-
|
|
3322
|
+
const forkedFromHeader = req.headers[STREAM_FORKED_FROM_HEADER.toLowerCase()];
|
|
3323
|
+
const forkOffsetHeader = req.headers[STREAM_FORK_OFFSET_HEADER.toLowerCase()];
|
|
3324
|
+
if (!contentType || contentType.trim() === `` || !/^[\w-]+\/[\w-]+/.test(contentType)) contentType = forkedFromHeader ? void 0 : `application/octet-stream`;
|
|
2263
3325
|
const ttlHeader = req.headers[STREAM_TTL_HEADER.toLowerCase()];
|
|
2264
3326
|
const expiresAtHeader = req.headers[STREAM_EXPIRES_AT_HEADER.toLowerCase()];
|
|
2265
3327
|
const closedHeader = req.headers[STREAM_CLOSED_HEADER.toLowerCase()];
|
|
2266
3328
|
const createClosed = closedHeader === `true`;
|
|
2267
|
-
const forkedFromHeader = req.headers[STREAM_FORKED_FROM_HEADER.toLowerCase()];
|
|
2268
|
-
const forkOffsetHeader = req.headers[STREAM_FORK_OFFSET_HEADER.toLowerCase()];
|
|
2269
3329
|
if (ttlHeader && expiresAtHeader) {
|
|
2270
3330
|
res.writeHead(400, { "content-type": `text/plain` });
|
|
2271
3331
|
res.end(`Cannot specify both Stream-TTL and Stream-Expires-At`);
|
|
@@ -2303,9 +3363,9 @@ var DurableStreamTestServer = class {
|
|
|
2303
3363
|
}
|
|
2304
3364
|
}
|
|
2305
3365
|
const body = await this.readBody(req);
|
|
2306
|
-
const isNew = !this.store.has(path$
|
|
3366
|
+
const isNew = !this.store.has(path$1);
|
|
2307
3367
|
try {
|
|
2308
|
-
await Promise.resolve(this.store.create(path$
|
|
3368
|
+
await Promise.resolve(this.store.create(path$1, {
|
|
2309
3369
|
contentType,
|
|
2310
3370
|
ttlSeconds,
|
|
2311
3371
|
expiresAt: expiresAtHeader,
|
|
@@ -2339,18 +3399,20 @@ var DurableStreamTestServer = class {
|
|
|
2339
3399
|
}
|
|
2340
3400
|
throw err;
|
|
2341
3401
|
}
|
|
2342
|
-
const stream = this.store.get(path$
|
|
3402
|
+
const stream = this.store.get(path$1);
|
|
3403
|
+
const resolvedContentType = stream.contentType ?? contentType ?? `application/octet-stream`;
|
|
2343
3404
|
if (isNew && this.options.onStreamCreated) await Promise.resolve(this.options.onStreamCreated({
|
|
2344
3405
|
type: `created`,
|
|
2345
|
-
path: path$
|
|
2346
|
-
contentType:
|
|
3406
|
+
path: path$1,
|
|
3407
|
+
contentType: resolvedContentType,
|
|
2347
3408
|
timestamp: Date.now()
|
|
2348
3409
|
}));
|
|
3410
|
+
if (isNew && body.length > 0) await this.notifyStreamAppend(path$1);
|
|
2349
3411
|
const headers = {
|
|
2350
|
-
"content-type":
|
|
3412
|
+
"content-type": resolvedContentType,
|
|
2351
3413
|
[STREAM_OFFSET_HEADER]: stream.currentOffset
|
|
2352
3414
|
};
|
|
2353
|
-
if (isNew) headers[`location`] = `${this._url}${path$
|
|
3415
|
+
if (isNew) headers[`location`] = `${this._url}${path$1}`;
|
|
2354
3416
|
if (stream.closed) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
2355
3417
|
res.writeHead(isNew ? 201 : 200, headers);
|
|
2356
3418
|
res.end();
|
|
@@ -2358,8 +3420,8 @@ var DurableStreamTestServer = class {
|
|
|
2358
3420
|
/**
|
|
2359
3421
|
* Handle HEAD - get metadata
|
|
2360
3422
|
*/
|
|
2361
|
-
handleHead(path$
|
|
2362
|
-
const stream = this.store.get(path$
|
|
3423
|
+
handleHead(path$1, res) {
|
|
3424
|
+
const stream = this.store.get(path$1);
|
|
2363
3425
|
if (!stream) {
|
|
2364
3426
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
2365
3427
|
res.end();
|
|
@@ -2379,15 +3441,15 @@ var DurableStreamTestServer = class {
|
|
|
2379
3441
|
if (stream.ttlSeconds !== void 0) headers[STREAM_TTL_HEADER] = String(stream.ttlSeconds);
|
|
2380
3442
|
if (stream.expiresAt) headers[STREAM_EXPIRES_AT_HEADER] = stream.expiresAt;
|
|
2381
3443
|
const closedSuffix = stream.closed ? `:c` : ``;
|
|
2382
|
-
headers[`etag`] = `"${Buffer.from(path$
|
|
3444
|
+
headers[`etag`] = `"${Buffer.from(path$1).toString(`base64`)}:-1:${stream.currentOffset}${closedSuffix}"`;
|
|
2383
3445
|
res.writeHead(200, headers);
|
|
2384
3446
|
res.end();
|
|
2385
3447
|
}
|
|
2386
3448
|
/**
|
|
2387
3449
|
* Handle GET - read data
|
|
2388
3450
|
*/
|
|
2389
|
-
async handleRead(path$
|
|
2390
|
-
const stream = this.store.get(path$
|
|
3451
|
+
async handleRead(path$1, url, req, res) {
|
|
3452
|
+
const stream = this.store.get(path$1);
|
|
2391
3453
|
if (!stream) {
|
|
2392
3454
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
2393
3455
|
res.end(`Stream not found`);
|
|
@@ -2433,7 +3495,7 @@ var DurableStreamTestServer = class {
|
|
|
2433
3495
|
}
|
|
2434
3496
|
if (live === `sse`) {
|
|
2435
3497
|
const sseOffset = offset === `now` ? stream.currentOffset : offset;
|
|
2436
|
-
await this.handleSSE(path$
|
|
3498
|
+
await this.handleSSE(path$1, stream, sseOffset, cursor, useBase64, res);
|
|
2437
3499
|
return;
|
|
2438
3500
|
}
|
|
2439
3501
|
const effectiveOffset = offset === `now` ? stream.currentOffset : offset;
|
|
@@ -2451,8 +3513,8 @@ var DurableStreamTestServer = class {
|
|
|
2451
3513
|
res.end(responseBody);
|
|
2452
3514
|
return;
|
|
2453
3515
|
}
|
|
2454
|
-
let { messages, upToDate } = this.store.read(path$
|
|
2455
|
-
this.store.touchAccess(path$
|
|
3516
|
+
let { messages, upToDate } = this.store.read(path$1, effectiveOffset);
|
|
3517
|
+
this.store.touchAccess(path$1);
|
|
2456
3518
|
const clientIsCaughtUp = effectiveOffset && effectiveOffset === stream.currentOffset || offset === `now`;
|
|
2457
3519
|
if (live === `long-poll` && clientIsCaughtUp && messages.length === 0) {
|
|
2458
3520
|
if (stream.closed) {
|
|
@@ -2464,8 +3526,8 @@ var DurableStreamTestServer = class {
|
|
|
2464
3526
|
res.end();
|
|
2465
3527
|
return;
|
|
2466
3528
|
}
|
|
2467
|
-
const result = await this.store.waitForMessages(path$
|
|
2468
|
-
this.store.touchAccess(path$
|
|
3529
|
+
const result = await this.store.waitForMessages(path$1, effectiveOffset ?? stream.currentOffset, this.options.longPollTimeout);
|
|
3530
|
+
this.store.touchAccess(path$1);
|
|
2469
3531
|
if (result.streamClosed) {
|
|
2470
3532
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2471
3533
|
res.writeHead(204, {
|
|
@@ -2479,7 +3541,7 @@ var DurableStreamTestServer = class {
|
|
|
2479
3541
|
}
|
|
2480
3542
|
if (result.timedOut) {
|
|
2481
3543
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2482
|
-
const currentStream$1 = this.store.get(path$
|
|
3544
|
+
const currentStream$1 = this.store.get(path$1);
|
|
2483
3545
|
const timeoutHeaders = {
|
|
2484
3546
|
[STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
|
|
2485
3547
|
[STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
@@ -2500,12 +3562,12 @@ var DurableStreamTestServer = class {
|
|
|
2500
3562
|
headers[STREAM_OFFSET_HEADER] = responseOffset;
|
|
2501
3563
|
if (live === `long-poll`) headers[STREAM_CURSOR_HEADER] = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2502
3564
|
if (upToDate) headers[STREAM_UP_TO_DATE_HEADER] = `true`;
|
|
2503
|
-
const currentStream = this.store.get(path$
|
|
3565
|
+
const currentStream = this.store.get(path$1);
|
|
2504
3566
|
const clientAtTail = responseOffset === currentStream?.currentOffset;
|
|
2505
3567
|
if (currentStream?.closed && clientAtTail && upToDate) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
2506
3568
|
const startOffset = offset ?? `-1`;
|
|
2507
3569
|
const closedSuffix = currentStream?.closed && clientAtTail && upToDate ? `:c` : ``;
|
|
2508
|
-
const etag = `"${Buffer.from(path$
|
|
3570
|
+
const etag = `"${Buffer.from(path$1).toString(`base64`)}:${startOffset}:${responseOffset}${closedSuffix}"`;
|
|
2509
3571
|
headers[`etag`] = etag;
|
|
2510
3572
|
const ifNoneMatch = req.headers[`if-none-match`];
|
|
2511
3573
|
if (ifNoneMatch && ifNoneMatch === etag) {
|
|
@@ -2513,7 +3575,7 @@ var DurableStreamTestServer = class {
|
|
|
2513
3575
|
res.end();
|
|
2514
3576
|
return;
|
|
2515
3577
|
}
|
|
2516
|
-
const responseData = this.store.formatResponse(path$
|
|
3578
|
+
const responseData = this.store.formatResponse(path$1, messages);
|
|
2517
3579
|
let finalData = responseData;
|
|
2518
3580
|
if (this.options.compression && responseData.length >= COMPRESSION_THRESHOLD) {
|
|
2519
3581
|
const acceptEncoding = req.headers[`accept-encoding`];
|
|
@@ -2531,7 +3593,7 @@ var DurableStreamTestServer = class {
|
|
|
2531
3593
|
/**
|
|
2532
3594
|
* Handle SSE (Server-Sent Events) mode
|
|
2533
3595
|
*/
|
|
2534
|
-
async handleSSE(path$
|
|
3596
|
+
async handleSSE(path$1, stream, initialOffset, cursor, useBase64, res) {
|
|
2535
3597
|
this.activeSSEResponses.add(res);
|
|
2536
3598
|
const sseHeaders = {
|
|
2537
3599
|
"content-type": `text/event-stream`,
|
|
@@ -2557,20 +3619,20 @@ var DurableStreamTestServer = class {
|
|
|
2557
3619
|
});
|
|
2558
3620
|
const isJsonStream = stream?.contentType?.includes(`application/json`);
|
|
2559
3621
|
while (isConnected && !this.isShuttingDown) {
|
|
2560
|
-
const { messages, upToDate } = this.store.read(path$
|
|
2561
|
-
this.store.touchAccess(path$
|
|
3622
|
+
const { messages, upToDate } = this.store.read(path$1, currentOffset);
|
|
3623
|
+
this.store.touchAccess(path$1);
|
|
2562
3624
|
for (const message of messages) {
|
|
2563
3625
|
let dataPayload;
|
|
2564
3626
|
if (useBase64) dataPayload = Buffer.from(message.data).toString(`base64`);
|
|
2565
3627
|
else if (isJsonStream) {
|
|
2566
|
-
const jsonBytes = this.store.formatResponse(path$
|
|
3628
|
+
const jsonBytes = this.store.formatResponse(path$1, [message]);
|
|
2567
3629
|
dataPayload = decoder.decode(jsonBytes);
|
|
2568
3630
|
} else dataPayload = decoder.decode(message.data);
|
|
2569
3631
|
res.write(`event: data\n`);
|
|
2570
3632
|
res.write(encodeSSEData(dataPayload));
|
|
2571
3633
|
currentOffset = message.offset;
|
|
2572
3634
|
}
|
|
2573
|
-
const currentStream = this.store.get(path$
|
|
3635
|
+
const currentStream = this.store.get(path$1);
|
|
2574
3636
|
const controlOffset = messages[messages.length - 1]?.offset ?? currentStream.currentOffset;
|
|
2575
3637
|
const streamIsClosed = currentStream?.closed ?? false;
|
|
2576
3638
|
const clientAtTail = controlOffset === currentStream.currentOffset;
|
|
@@ -2595,10 +3657,10 @@ var DurableStreamTestServer = class {
|
|
|
2595
3657
|
res.write(encodeSSEData(JSON.stringify(finalControlData)));
|
|
2596
3658
|
break;
|
|
2597
3659
|
}
|
|
2598
|
-
const result = await this.store.waitForMessages(path$
|
|
2599
|
-
this.store.touchAccess(path$
|
|
3660
|
+
const result = await this.store.waitForMessages(path$1, currentOffset, this.options.longPollTimeout);
|
|
3661
|
+
this.store.touchAccess(path$1);
|
|
2600
3662
|
if (this.isShuttingDown || !isConnected) break;
|
|
2601
|
-
if (result.streamClosed) {
|
|
3663
|
+
if (result.streamClosed && result.messages.length === 0) {
|
|
2602
3664
|
const finalControlData = {
|
|
2603
3665
|
[SSE_OFFSET_FIELD]: currentOffset,
|
|
2604
3666
|
[SSE_CLOSED_FIELD]: true
|
|
@@ -2609,7 +3671,7 @@ var DurableStreamTestServer = class {
|
|
|
2609
3671
|
}
|
|
2610
3672
|
if (result.timedOut) {
|
|
2611
3673
|
const keepAliveCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2612
|
-
const streamAfterWait = this.store.get(path$
|
|
3674
|
+
const streamAfterWait = this.store.get(path$1);
|
|
2613
3675
|
if (streamAfterWait?.closed) {
|
|
2614
3676
|
const closedControlData = {
|
|
2615
3677
|
[SSE_OFFSET_FIELD]: currentOffset,
|
|
@@ -2634,7 +3696,7 @@ var DurableStreamTestServer = class {
|
|
|
2634
3696
|
/**
|
|
2635
3697
|
* Handle POST - append data
|
|
2636
3698
|
*/
|
|
2637
|
-
async handleAppend(path$
|
|
3699
|
+
async handleAppend(path$1, req, res) {
|
|
2638
3700
|
const contentType = req.headers[`content-type`];
|
|
2639
3701
|
const seq = req.headers[STREAM_SEQ_HEADER.toLowerCase()];
|
|
2640
3702
|
const closedHeader = req.headers[STREAM_CLOSED_HEADER.toLowerCase()];
|
|
@@ -2684,7 +3746,7 @@ var DurableStreamTestServer = class {
|
|
|
2684
3746
|
const body = await this.readBody(req);
|
|
2685
3747
|
if (body.length === 0 && closeStream) {
|
|
2686
3748
|
if (hasAllProducerHeaders) {
|
|
2687
|
-
const closeResult$1 = await this.store.closeStreamWithProducer(path$
|
|
3749
|
+
const closeResult$1 = await this.store.closeStreamWithProducer(path$1, {
|
|
2688
3750
|
producerId,
|
|
2689
3751
|
producerEpoch,
|
|
2690
3752
|
producerSeq
|
|
@@ -2727,7 +3789,7 @@ var DurableStreamTestServer = class {
|
|
|
2727
3789
|
return;
|
|
2728
3790
|
}
|
|
2729
3791
|
if (closeResult$1.producerResult?.status === `stream_closed`) {
|
|
2730
|
-
const stream = this.store.get(path$
|
|
3792
|
+
const stream = this.store.get(path$1);
|
|
2731
3793
|
res.writeHead(409, {
|
|
2732
3794
|
"content-type": `text/plain`,
|
|
2733
3795
|
[STREAM_CLOSED_HEADER]: `true`,
|
|
@@ -2745,7 +3807,7 @@ var DurableStreamTestServer = class {
|
|
|
2745
3807
|
res.end();
|
|
2746
3808
|
return;
|
|
2747
3809
|
}
|
|
2748
|
-
const closeResult = this.store.closeStream(path$
|
|
3810
|
+
const closeResult = this.store.closeStream(path$1);
|
|
2749
3811
|
if (!closeResult) {
|
|
2750
3812
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
2751
3813
|
res.end(`Stream not found`);
|
|
@@ -2777,14 +3839,14 @@ var DurableStreamTestServer = class {
|
|
|
2777
3839
|
close: closeStream
|
|
2778
3840
|
};
|
|
2779
3841
|
let result;
|
|
2780
|
-
if (producerId !== void 0) result = await this.store.appendWithProducer(path$
|
|
2781
|
-
else result = await Promise.resolve(this.store.append(path$
|
|
2782
|
-
this.store.touchAccess(path$
|
|
3842
|
+
if (producerId !== void 0) result = await this.store.appendWithProducer(path$1, body, appendOptions);
|
|
3843
|
+
else result = await Promise.resolve(this.store.append(path$1, body, appendOptions));
|
|
3844
|
+
this.store.touchAccess(path$1);
|
|
2783
3845
|
if (result && typeof result === `object` && `message` in result) {
|
|
2784
3846
|
const { message: message$1, producerResult, streamClosed } = result;
|
|
2785
3847
|
if (streamClosed && !message$1) {
|
|
2786
3848
|
if (producerResult?.status === `duplicate`) {
|
|
2787
|
-
const stream = this.store.get(path$
|
|
3849
|
+
const stream = this.store.get(path$1);
|
|
2788
3850
|
res.writeHead(204, {
|
|
2789
3851
|
[STREAM_OFFSET_HEADER]: stream?.currentOffset ?? ``,
|
|
2790
3852
|
[STREAM_CLOSED_HEADER]: `true`,
|
|
@@ -2794,7 +3856,7 @@ var DurableStreamTestServer = class {
|
|
|
2794
3856
|
res.end();
|
|
2795
3857
|
return;
|
|
2796
3858
|
}
|
|
2797
|
-
const closedStream = this.store.get(path$
|
|
3859
|
+
const closedStream = this.store.get(path$1);
|
|
2798
3860
|
res.writeHead(409, {
|
|
2799
3861
|
"content-type": `text/plain`,
|
|
2800
3862
|
[STREAM_CLOSED_HEADER]: `true`,
|
|
@@ -2811,6 +3873,7 @@ var DurableStreamTestServer = class {
|
|
|
2811
3873
|
const statusCode = producerId !== void 0 ? 200 : 204;
|
|
2812
3874
|
res.writeHead(statusCode, responseHeaders$1);
|
|
2813
3875
|
res.end();
|
|
3876
|
+
await this.notifyStreamAppend(path$1);
|
|
2814
3877
|
return;
|
|
2815
3878
|
}
|
|
2816
3879
|
switch (producerResult.status) {
|
|
@@ -2851,18 +3914,27 @@ var DurableStreamTestServer = class {
|
|
|
2851
3914
|
if (closeStream) responseHeaders[STREAM_CLOSED_HEADER] = `true`;
|
|
2852
3915
|
res.writeHead(204, responseHeaders);
|
|
2853
3916
|
res.end();
|
|
3917
|
+
await this.notifyStreamAppend(path$1);
|
|
3918
|
+
}
|
|
3919
|
+
async notifyStreamAppend(path$1) {
|
|
3920
|
+
if (!this.subscriptionManager) return;
|
|
3921
|
+
try {
|
|
3922
|
+
await this.subscriptionManager.onStreamAppend(path$1);
|
|
3923
|
+
} catch (err) {
|
|
3924
|
+
serverLog.error(`[server] subscription append hook failed:`, err);
|
|
3925
|
+
}
|
|
2854
3926
|
}
|
|
2855
3927
|
/**
|
|
2856
3928
|
* Handle DELETE - delete stream
|
|
2857
3929
|
*/
|
|
2858
|
-
async handleDelete(path$
|
|
2859
|
-
const existing = this.store.get(path$
|
|
3930
|
+
async handleDelete(path$1, res) {
|
|
3931
|
+
const existing = this.store.get(path$1);
|
|
2860
3932
|
if (existing?.softDeleted) {
|
|
2861
3933
|
res.writeHead(410, { "content-type": `text/plain` });
|
|
2862
3934
|
res.end(`Stream is gone`);
|
|
2863
3935
|
return;
|
|
2864
3936
|
}
|
|
2865
|
-
const deleted = this.store.delete(path$
|
|
3937
|
+
const deleted = this.store.delete(path$1);
|
|
2866
3938
|
if (!deleted) {
|
|
2867
3939
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
2868
3940
|
res.end(`Stream not found`);
|
|
@@ -2870,9 +3942,10 @@ var DurableStreamTestServer = class {
|
|
|
2870
3942
|
}
|
|
2871
3943
|
if (this.options.onStreamDeleted) await Promise.resolve(this.options.onStreamDeleted({
|
|
2872
3944
|
type: `deleted`,
|
|
2873
|
-
path: path$
|
|
3945
|
+
path: path$1,
|
|
2874
3946
|
timestamp: Date.now()
|
|
2875
3947
|
}));
|
|
3948
|
+
if (this.subscriptionManager) this.subscriptionManager.onStreamDeleted(path$1);
|
|
2876
3949
|
res.writeHead(204);
|
|
2877
3950
|
res.end();
|
|
2878
3951
|
}
|
|
@@ -3002,4 +4075,4 @@ function createRegistryHooks(store, serverUrl) {
|
|
|
3002
4075
|
}
|
|
3003
4076
|
|
|
3004
4077
|
//#endregion
|
|
3005
|
-
export { DEFAULT_CURSOR_EPOCH, DEFAULT_CURSOR_INTERVAL_SECONDS, DurableStreamTestServer, FileBackedStreamStore, StreamStore, calculateCursor, createRegistryHooks, decodeStreamPath, encodeStreamPath, generateResponseCursor, handleCursorCollision };
|
|
4078
|
+
export { DEFAULT_CURSOR_EPOCH, DEFAULT_CURSOR_INTERVAL_SECONDS, DurableStreamTestServer, FileBackedStreamStore, StreamStore, SubscriptionManager, SubscriptionRoutes, calculateCursor, createRegistryHooks, decodeStreamPath, encodeStreamPath, generateResponseCursor, globMatch, handleCursorCollision, validateWebhookUrl };
|