@durable-streams/server 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1344 -266
- package/dist/index.d.cts +258 -2
- package/dist/index.d.ts +258 -2
- package/dist/index.js +1391 -318
- package/package.json +4 -4
- package/src/crypto.ts +217 -0
- package/src/file-store.ts +239 -144
- package/src/glob.ts +70 -0
- package/src/index.ts +14 -0
- package/src/log.ts +56 -0
- package/src/server.ts +96 -40
- package/src/store.ts +66 -10
- package/src/subscription-manager.ts +882 -0
- package/src/subscription-routes.ts +504 -0
- package/src/subscription-types.ts +80 -0
- package/src/types.ts +8 -0
package/dist/index.cjs
CHANGED
|
@@ -24,13 +24,13 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
24
24
|
//#endregion
|
|
25
25
|
const node_http = __toESM(require("node:http"));
|
|
26
26
|
const node_zlib = __toESM(require("node:zlib"));
|
|
27
|
+
const __durable_streams_client = __toESM(require("@durable-streams/client"));
|
|
27
28
|
const node_fs = __toESM(require("node:fs"));
|
|
28
29
|
const node_path = __toESM(require("node:path"));
|
|
29
30
|
const node_crypto = __toESM(require("node:crypto"));
|
|
30
31
|
const lmdb = __toESM(require("lmdb"));
|
|
31
32
|
const __neophi_sieve_cache = __toESM(require("@neophi/sieve-cache"));
|
|
32
|
-
const
|
|
33
|
-
const __durable_streams_client = __toESM(require("@durable-streams/client"));
|
|
33
|
+
const node_net = __toESM(require("node:net"));
|
|
34
34
|
const __durable_streams_state = __toESM(require("@durable-streams/state"));
|
|
35
35
|
|
|
36
36
|
//#region src/store.ts
|
|
@@ -73,17 +73,35 @@ function processJsonAppend(data, isInitialCreate = false) {
|
|
|
73
73
|
} else result = JSON.stringify(parsed) + `,`;
|
|
74
74
|
return new TextEncoder().encode(result);
|
|
75
75
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
* Strips trailing comma before wrapping.
|
|
79
|
-
*/
|
|
80
|
-
function formatJsonResponse(data) {
|
|
81
|
-
if (data.length === 0) return new TextEncoder().encode(`[]`);
|
|
82
|
-
let text = new TextDecoder().decode(data);
|
|
83
|
-
text = text.trimEnd();
|
|
76
|
+
function decodeStoredJsonMessage(data) {
|
|
77
|
+
let text = new TextDecoder().decode(data).trimEnd();
|
|
84
78
|
if (text.endsWith(`,`)) text = text.slice(0, -1);
|
|
85
|
-
|
|
86
|
-
|
|
79
|
+
return text;
|
|
80
|
+
}
|
|
81
|
+
function enrichJsonValueWithOffset(parsed, offset) {
|
|
82
|
+
if (!parsed || typeof parsed !== `object` || Array.isArray(parsed)) return JSON.stringify(parsed);
|
|
83
|
+
const candidate = parsed;
|
|
84
|
+
const headers = candidate.headers;
|
|
85
|
+
if (!headers || typeof headers !== `object`) return JSON.stringify(parsed);
|
|
86
|
+
const isStateChange = typeof headers.operation === `string`;
|
|
87
|
+
const isStateControl = typeof headers.control === `string`;
|
|
88
|
+
if (!isStateChange && !isStateControl) return JSON.stringify(parsed);
|
|
89
|
+
return JSON.stringify({
|
|
90
|
+
...candidate,
|
|
91
|
+
headers: {
|
|
92
|
+
...headers,
|
|
93
|
+
offset
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
function formatJsonMessages(messages) {
|
|
98
|
+
if (messages.length === 0) return new TextEncoder().encode(`[]`);
|
|
99
|
+
const items = messages.flatMap((message) => {
|
|
100
|
+
const rawFragment = decodeStoredJsonMessage(message.data);
|
|
101
|
+
const parsed = JSON.parse(`[${rawFragment}]`);
|
|
102
|
+
return parsed.map((value) => enrichJsonValueWithOffset(value, message.offset));
|
|
103
|
+
});
|
|
104
|
+
return new TextEncoder().encode(`[${items.join(`,`)}]`);
|
|
87
105
|
}
|
|
88
106
|
var StreamStore = class {
|
|
89
107
|
streams = new Map();
|
|
@@ -430,9 +448,9 @@ var StreamStore = class {
|
|
|
430
448
|
epoch: options.producerEpoch,
|
|
431
449
|
seq: options.producerSeq
|
|
432
450
|
};
|
|
433
|
-
this.notifyLongPollsClosed(path);
|
|
434
451
|
}
|
|
435
452
|
this.notifyLongPolls(path);
|
|
453
|
+
if (options.close) this.notifyLongPollsClosed(path);
|
|
436
454
|
if (producerResult || options.close) return {
|
|
437
455
|
message,
|
|
438
456
|
producerResult,
|
|
@@ -621,6 +639,7 @@ var StreamStore = class {
|
|
|
621
639
|
formatResponse(path, messages) {
|
|
622
640
|
const stream = this.getIfNotExpired(path);
|
|
623
641
|
if (!stream) throw new Error(`Stream not found: ${path}`);
|
|
642
|
+
if (normalizeContentType(stream.contentType) === `application/json`) return formatJsonMessages(messages);
|
|
624
643
|
const totalSize = messages.reduce((sum, m) => sum + m.data.length, 0);
|
|
625
644
|
const concatenated = new Uint8Array(totalSize);
|
|
626
645
|
let offset = 0;
|
|
@@ -628,7 +647,6 @@ var StreamStore = class {
|
|
|
628
647
|
concatenated.set(msg.data, offset);
|
|
629
648
|
offset += msg.data.length;
|
|
630
649
|
}
|
|
631
|
-
if (normalizeContentType(stream.contentType) === `application/json`) return formatJsonResponse(concatenated);
|
|
632
650
|
return concatenated;
|
|
633
651
|
}
|
|
634
652
|
/**
|
|
@@ -728,7 +746,8 @@ var StreamStore = class {
|
|
|
728
746
|
const parts = stream.currentOffset.split(`_`).map(Number);
|
|
729
747
|
const readSeq = parts[0];
|
|
730
748
|
const byteOffset = parts[1];
|
|
731
|
-
const
|
|
749
|
+
const FRAME_OVERHEAD = 5;
|
|
750
|
+
const newByteOffset = byteOffset + FRAME_OVERHEAD + processedData.length;
|
|
732
751
|
const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
733
752
|
const message = {
|
|
734
753
|
data: processedData,
|
|
@@ -772,6 +791,48 @@ var StreamStore = class {
|
|
|
772
791
|
}
|
|
773
792
|
};
|
|
774
793
|
|
|
794
|
+
//#endregion
|
|
795
|
+
//#region src/log.ts
|
|
796
|
+
const streamsLogFile = process.env.STREAMS_LOG_FILE;
|
|
797
|
+
async function appendLogLine(line) {
|
|
798
|
+
if (!streamsLogFile) return;
|
|
799
|
+
const fs = await import(`node:fs/promises`);
|
|
800
|
+
const path = await import(`node:path`);
|
|
801
|
+
await fs.mkdir(path.dirname(streamsLogFile), { recursive: true });
|
|
802
|
+
await fs.appendFile(streamsLogFile, `${line}\n`);
|
|
803
|
+
}
|
|
804
|
+
function serializeArg(arg) {
|
|
805
|
+
if (arg instanceof Error) return arg.stack ?? arg.message;
|
|
806
|
+
if (typeof arg === `string`) return arg;
|
|
807
|
+
try {
|
|
808
|
+
return JSON.stringify(arg);
|
|
809
|
+
} catch {
|
|
810
|
+
return String(arg);
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
function write(level, args) {
|
|
814
|
+
const line = args.map(serializeArg).join(` `);
|
|
815
|
+
const formatted = `[${level}] ${line}`;
|
|
816
|
+
if (level === `error`) console.error(formatted);
|
|
817
|
+
else if (level === `warn`) console.warn(formatted);
|
|
818
|
+
else console.info(formatted);
|
|
819
|
+
appendLogLine(formatted).catch(() => void 0);
|
|
820
|
+
}
|
|
821
|
+
const serverLog = {
|
|
822
|
+
info(...args) {
|
|
823
|
+
write(`info`, args);
|
|
824
|
+
},
|
|
825
|
+
warn(...args) {
|
|
826
|
+
write(`warn`, args);
|
|
827
|
+
},
|
|
828
|
+
error(...args) {
|
|
829
|
+
write(`error`, args);
|
|
830
|
+
},
|
|
831
|
+
event(obj, msg) {
|
|
832
|
+
write(`info`, [msg, obj]);
|
|
833
|
+
}
|
|
834
|
+
};
|
|
835
|
+
|
|
775
836
|
//#endregion
|
|
776
837
|
//#region src/path-encoding.ts
|
|
777
838
|
const MAX_ENCODED_LENGTH = 200;
|
|
@@ -808,82 +869,6 @@ function decodeStreamPath(encoded) {
|
|
|
808
869
|
return Buffer.from(padded, `base64`).toString(`utf-8`);
|
|
809
870
|
}
|
|
810
871
|
|
|
811
|
-
//#endregion
|
|
812
|
-
//#region src/file-manager.ts
|
|
813
|
-
var StreamFileManager = class {
|
|
814
|
-
constructor(streamsDir) {
|
|
815
|
-
this.streamsDir = streamsDir;
|
|
816
|
-
}
|
|
817
|
-
/**
|
|
818
|
-
* Create a directory for a new stream and initialize the first segment file.
|
|
819
|
-
* Returns the absolute path to the stream directory.
|
|
820
|
-
*/
|
|
821
|
-
async createStreamDirectory(streamPath) {
|
|
822
|
-
const encoded = encodeStreamPath(streamPath);
|
|
823
|
-
const dir = node_path.join(this.streamsDir, encoded);
|
|
824
|
-
await node_fs_promises.mkdir(dir, { recursive: true });
|
|
825
|
-
const segmentPath = node_path.join(dir, `segment_00000.log`);
|
|
826
|
-
await node_fs_promises.writeFile(segmentPath, ``);
|
|
827
|
-
return dir;
|
|
828
|
-
}
|
|
829
|
-
/**
|
|
830
|
-
* Delete a stream directory and all its contents.
|
|
831
|
-
*/
|
|
832
|
-
async deleteStreamDirectory(streamPath) {
|
|
833
|
-
const encoded = encodeStreamPath(streamPath);
|
|
834
|
-
const dir = node_path.join(this.streamsDir, encoded);
|
|
835
|
-
await node_fs_promises.rm(dir, {
|
|
836
|
-
recursive: true,
|
|
837
|
-
force: true
|
|
838
|
-
});
|
|
839
|
-
}
|
|
840
|
-
/**
|
|
841
|
-
* Delete a directory by its exact name (used for unique directory names).
|
|
842
|
-
*/
|
|
843
|
-
async deleteDirectoryByName(directoryName) {
|
|
844
|
-
const dir = node_path.join(this.streamsDir, directoryName);
|
|
845
|
-
await node_fs_promises.rm(dir, {
|
|
846
|
-
recursive: true,
|
|
847
|
-
force: true
|
|
848
|
-
});
|
|
849
|
-
}
|
|
850
|
-
/**
|
|
851
|
-
* Get the absolute path to a stream's directory.
|
|
852
|
-
* Returns null if the directory doesn't exist.
|
|
853
|
-
*/
|
|
854
|
-
async getStreamDirectory(streamPath) {
|
|
855
|
-
const encoded = encodeStreamPath(streamPath);
|
|
856
|
-
const dir = node_path.join(this.streamsDir, encoded);
|
|
857
|
-
try {
|
|
858
|
-
await node_fs_promises.access(dir);
|
|
859
|
-
return dir;
|
|
860
|
-
} catch {
|
|
861
|
-
return null;
|
|
862
|
-
}
|
|
863
|
-
}
|
|
864
|
-
/**
|
|
865
|
-
* List all stream paths by scanning the streams directory.
|
|
866
|
-
*/
|
|
867
|
-
async listStreamPaths() {
|
|
868
|
-
try {
|
|
869
|
-
const entries = await node_fs_promises.readdir(this.streamsDir, { withFileTypes: true });
|
|
870
|
-
return entries.filter((e) => e.isDirectory()).map((e) => decodeStreamPath(e.name));
|
|
871
|
-
} catch {
|
|
872
|
-
return [];
|
|
873
|
-
}
|
|
874
|
-
}
|
|
875
|
-
/**
|
|
876
|
-
* Get the path to a segment file within a stream directory.
|
|
877
|
-
*
|
|
878
|
-
* @param streamDir - Absolute path to the stream directory
|
|
879
|
-
* @param index - Segment index (0-based)
|
|
880
|
-
*/
|
|
881
|
-
getSegmentPath(streamDir, index) {
|
|
882
|
-
const paddedIndex = String(index).padStart(5, `0`);
|
|
883
|
-
return node_path.join(streamDir, `segment_${paddedIndex}.log`);
|
|
884
|
-
}
|
|
885
|
-
};
|
|
886
|
-
|
|
887
872
|
//#endregion
|
|
888
873
|
//#region src/file-store.ts
|
|
889
874
|
var FileHandlePool = class {
|
|
@@ -891,7 +876,7 @@ var FileHandlePool = class {
|
|
|
891
876
|
constructor(maxSize) {
|
|
892
877
|
this.cache = new __neophi_sieve_cache.SieveCache(maxSize, { evictHook: (_key, handle) => {
|
|
893
878
|
this.closeHandle(handle).catch((err) => {
|
|
894
|
-
|
|
879
|
+
serverLog.error(`[FileHandlePool] Error closing evicted handle:`, err);
|
|
895
880
|
});
|
|
896
881
|
} });
|
|
897
882
|
}
|
|
@@ -899,41 +884,70 @@ var FileHandlePool = class {
|
|
|
899
884
|
let handle = this.cache.get(filePath);
|
|
900
885
|
if (!handle) {
|
|
901
886
|
const stream = node_fs.createWriteStream(filePath, { flags: `a` });
|
|
902
|
-
handle = {
|
|
887
|
+
handle = {
|
|
888
|
+
stream,
|
|
889
|
+
syncLeader: null
|
|
890
|
+
};
|
|
903
891
|
this.cache.set(filePath, handle);
|
|
904
892
|
}
|
|
905
893
|
return handle.stream;
|
|
906
894
|
}
|
|
907
895
|
/**
|
|
896
|
+
* Open a write stream eagerly so the first write does not pay the lazy
|
|
897
|
+
* `open()` stall. Resolves once the underlying fd is ready.
|
|
898
|
+
*/
|
|
899
|
+
async openWriteStream(filePath) {
|
|
900
|
+
const stream = this.getWriteStream(filePath);
|
|
901
|
+
const fd = stream.fd;
|
|
902
|
+
if (typeof fd === `number`) return stream;
|
|
903
|
+
await new Promise((resolve, reject) => {
|
|
904
|
+
stream.once(`open`, () => resolve());
|
|
905
|
+
stream.once(`error`, (err) => reject(err));
|
|
906
|
+
});
|
|
907
|
+
return stream;
|
|
908
|
+
}
|
|
909
|
+
/**
|
|
908
910
|
* Flush a specific file to disk immediately.
|
|
909
|
-
*
|
|
911
|
+
* Concurrent callers on the same fd share one in-flight fdatasync: the
|
|
912
|
+
* first caller issues the syscall, later arrivals during that window wait
|
|
913
|
+
* for it to finish and then issue a fresh syscall (because their writes
|
|
914
|
+
* may have landed after the in-flight syscall started). This preserves
|
|
915
|
+
* durability without adding scheduling latency.
|
|
910
916
|
*/
|
|
911
|
-
|
|
917
|
+
fsyncFile(filePath) {
|
|
912
918
|
const handle = this.cache.get(filePath);
|
|
913
|
-
if (!handle) return;
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
else resolve();
|
|
922
|
-
});
|
|
923
|
-
};
|
|
924
|
-
const onError = (err) => {
|
|
925
|
-
handle.stream.off(`open`, onOpen);
|
|
926
|
-
reject(err);
|
|
927
|
-
};
|
|
928
|
-
handle.stream.once(`open`, onOpen);
|
|
929
|
-
handle.stream.once(`error`, onError);
|
|
930
|
-
return;
|
|
931
|
-
}
|
|
932
|
-
node_fs.fdatasync(fd, (err) => {
|
|
933
|
-
if (err) reject(err);
|
|
934
|
-
else resolve();
|
|
935
|
-
});
|
|
919
|
+
if (!handle) return Promise.reject(new Error(`[FileHandlePool] Cannot fsync: handle not found for ${filePath}`));
|
|
920
|
+
const existing = handle.syncLeader;
|
|
921
|
+
if (existing && existing.scheduled) return existing.promise;
|
|
922
|
+
let resolveFn;
|
|
923
|
+
let rejectFn;
|
|
924
|
+
const promise = new Promise((res, rej) => {
|
|
925
|
+
resolveFn = res;
|
|
926
|
+
rejectFn = rej;
|
|
936
927
|
});
|
|
928
|
+
const leader = {
|
|
929
|
+
promise,
|
|
930
|
+
scheduled: true
|
|
931
|
+
};
|
|
932
|
+
handle.syncLeader = leader;
|
|
933
|
+
const runSyscall = (fd$1) => {
|
|
934
|
+
leader.scheduled = false;
|
|
935
|
+
node_fs.fdatasync(fd$1, (err) => {
|
|
936
|
+
if (handle.syncLeader === leader) handle.syncLeader = null;
|
|
937
|
+
if (err) rejectFn(err);
|
|
938
|
+
else resolveFn();
|
|
939
|
+
});
|
|
940
|
+
};
|
|
941
|
+
const fd = handle.stream.fd;
|
|
942
|
+
if (typeof fd === `number`) runSyscall(fd);
|
|
943
|
+
else {
|
|
944
|
+
handle.stream.once(`open`, (openedFd) => runSyscall(openedFd));
|
|
945
|
+
handle.stream.once(`error`, (err) => {
|
|
946
|
+
if (handle.syncLeader === leader) handle.syncLeader = null;
|
|
947
|
+
rejectFn(err);
|
|
948
|
+
});
|
|
949
|
+
}
|
|
950
|
+
return promise;
|
|
937
951
|
}
|
|
938
952
|
async closeAll() {
|
|
939
953
|
const promises = [];
|
|
@@ -969,13 +983,15 @@ function generateUniqueDirectoryName(streamPath) {
|
|
|
969
983
|
const random = (0, node_crypto.randomBytes)(4).toString(`hex`);
|
|
970
984
|
return `${encoded}~${timestamp}~${random}`;
|
|
971
985
|
}
|
|
986
|
+
function segmentFile(dataDir, dirName) {
|
|
987
|
+
return node_path.join(dataDir, `streams`, `${dirName}.log`);
|
|
988
|
+
}
|
|
972
989
|
/**
|
|
973
990
|
* File-backed implementation of StreamStore.
|
|
974
991
|
* Maintains the same interface as the in-memory StreamStore for drop-in compatibility.
|
|
975
992
|
*/
|
|
976
993
|
var FileBackedStreamStore = class {
|
|
977
994
|
db;
|
|
978
|
-
fileManager;
|
|
979
995
|
fileHandlePool;
|
|
980
996
|
pendingLongPolls = [];
|
|
981
997
|
dataDir;
|
|
@@ -984,13 +1000,23 @@ var FileBackedStreamStore = class {
|
|
|
984
1000
|
* Key: "{streamPath}:{producerId}"
|
|
985
1001
|
*/
|
|
986
1002
|
producerLocks = new Map();
|
|
1003
|
+
/**
|
|
1004
|
+
* Per-stream append locks. Serializes the read-modify-write of currentOffset
|
|
1005
|
+
* across all concurrent appenders on the same stream so the LMDB-tracked
|
|
1006
|
+
* offset cannot drift behind the file's actual byte position.
|
|
1007
|
+
* Key: streamPath
|
|
1008
|
+
*/
|
|
1009
|
+
streamAppendLocks = new Map();
|
|
987
1010
|
constructor(options) {
|
|
988
1011
|
this.dataDir = options.dataDir;
|
|
989
1012
|
this.db = (0, lmdb.open)({
|
|
990
1013
|
path: node_path.join(this.dataDir, `metadata.lmdb`),
|
|
991
|
-
compression: true
|
|
1014
|
+
compression: true,
|
|
1015
|
+
noMemInit: true,
|
|
1016
|
+
cache: true,
|
|
1017
|
+
sharedStructuresKey: Symbol.for(`structures`)
|
|
992
1018
|
});
|
|
993
|
-
|
|
1019
|
+
node_fs.mkdirSync(node_path.join(this.dataDir, `streams`), { recursive: true });
|
|
994
1020
|
const maxFileHandles = options.maxFileHandles ?? 100;
|
|
995
1021
|
this.fileHandlePool = new FileHandlePool(maxFileHandles);
|
|
996
1022
|
this.recover();
|
|
@@ -1000,7 +1026,7 @@ var FileBackedStreamStore = class {
|
|
|
1000
1026
|
* Validates that LMDB metadata matches actual file contents and reconciles any mismatches.
|
|
1001
1027
|
*/
|
|
1002
1028
|
recover() {
|
|
1003
|
-
|
|
1029
|
+
serverLog.info(`[FileBackedStreamStore] Starting recovery...`);
|
|
1004
1030
|
let recovered = 0;
|
|
1005
1031
|
let reconciled = 0;
|
|
1006
1032
|
let errors = 0;
|
|
@@ -1013,9 +1039,9 @@ var FileBackedStreamStore = class {
|
|
|
1013
1039
|
if (typeof key !== `string`) continue;
|
|
1014
1040
|
const streamMeta = value;
|
|
1015
1041
|
const streamPath = key.replace(`stream:`, ``);
|
|
1016
|
-
const segmentPath =
|
|
1042
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1017
1043
|
if (!node_fs.existsSync(segmentPath)) {
|
|
1018
|
-
|
|
1044
|
+
serverLog.warn(`[FileBackedStreamStore] Recovery: Stream file missing for ${streamPath}, removing from LMDB`);
|
|
1019
1045
|
this.db.removeSync(key);
|
|
1020
1046
|
errors++;
|
|
1021
1047
|
continue;
|
|
@@ -1029,7 +1055,7 @@ var FileBackedStreamStore = class {
|
|
|
1029
1055
|
trueOffset = `${String(0).padStart(16, `0`)}_${String(logicalBytes).padStart(16, `0`)}`;
|
|
1030
1056
|
} else trueOffset = physicalOffset;
|
|
1031
1057
|
if (trueOffset !== streamMeta.currentOffset) {
|
|
1032
|
-
|
|
1058
|
+
serverLog.warn(`[FileBackedStreamStore] Recovery: Offset mismatch for ${streamPath}: LMDB says ${streamMeta.currentOffset}, file says ${trueOffset}. Reconciling to file.`);
|
|
1033
1059
|
const reconciledMeta = {
|
|
1034
1060
|
...streamMeta,
|
|
1035
1061
|
currentOffset: trueOffset
|
|
@@ -1039,10 +1065,10 @@ var FileBackedStreamStore = class {
|
|
|
1039
1065
|
}
|
|
1040
1066
|
recovered++;
|
|
1041
1067
|
} catch (err) {
|
|
1042
|
-
|
|
1068
|
+
serverLog.error(`[FileBackedStreamStore] Error recovering stream:`, err);
|
|
1043
1069
|
errors++;
|
|
1044
1070
|
}
|
|
1045
|
-
|
|
1071
|
+
serverLog.info(`[FileBackedStreamStore] Recovery complete: ${recovered} streams, ${reconciled} reconciled, ${errors} errors`);
|
|
1046
1072
|
}
|
|
1047
1073
|
/**
|
|
1048
1074
|
* Scan a segment file to compute the true last offset.
|
|
@@ -1052,19 +1078,16 @@ var FileBackedStreamStore = class {
|
|
|
1052
1078
|
try {
|
|
1053
1079
|
const fileContent = node_fs.readFileSync(segmentPath);
|
|
1054
1080
|
let filePos = 0;
|
|
1055
|
-
let currentDataOffset = 0;
|
|
1056
1081
|
while (filePos < fileContent.length) {
|
|
1057
1082
|
if (filePos + 4 > fileContent.length) break;
|
|
1058
1083
|
const messageLength = fileContent.readUInt32BE(filePos);
|
|
1059
|
-
filePos
|
|
1060
|
-
if (
|
|
1061
|
-
filePos
|
|
1062
|
-
if (filePos < fileContent.length) filePos += 1;
|
|
1063
|
-
currentDataOffset += messageLength;
|
|
1084
|
+
const frameEnd = filePos + 4 + messageLength + 1;
|
|
1085
|
+
if (frameEnd > fileContent.length) break;
|
|
1086
|
+
filePos = frameEnd;
|
|
1064
1087
|
}
|
|
1065
|
-
return `0000000000000000_${String(
|
|
1088
|
+
return `0000000000000000_${String(filePos).padStart(16, `0`)}`;
|
|
1066
1089
|
} catch (err) {
|
|
1067
|
-
|
|
1090
|
+
serverLog.error(`[FileBackedStreamStore] Error scanning file ${segmentPath}:`, err);
|
|
1068
1091
|
return `0000000000000000_0000000000000000`;
|
|
1069
1092
|
}
|
|
1070
1093
|
}
|
|
@@ -1180,6 +1203,26 @@ var FileBackedStreamStore = class {
|
|
|
1180
1203
|
};
|
|
1181
1204
|
}
|
|
1182
1205
|
/**
|
|
1206
|
+
* Acquire a per-stream append lock that serializes the read-modify-write
|
|
1207
|
+
* of currentOffset across all concurrent appenders on the same stream.
|
|
1208
|
+
* Without this, two concurrent appends can read the same starting
|
|
1209
|
+
* currentOffset, both compute their newOffset, both write a frame to the
|
|
1210
|
+
* file, but only one of their LMDB updates wins — leaving currentOffset
|
|
1211
|
+
* lagging the file's actual byte position. Returns a release function.
|
|
1212
|
+
*/
|
|
1213
|
+
async acquireStreamAppendLock(streamPath) {
|
|
1214
|
+
while (this.streamAppendLocks.has(streamPath)) await this.streamAppendLocks.get(streamPath);
|
|
1215
|
+
let releaseLock;
|
|
1216
|
+
const lockPromise = new Promise((resolve) => {
|
|
1217
|
+
releaseLock = resolve;
|
|
1218
|
+
});
|
|
1219
|
+
this.streamAppendLocks.set(streamPath, lockPromise);
|
|
1220
|
+
return () => {
|
|
1221
|
+
this.streamAppendLocks.delete(streamPath);
|
|
1222
|
+
releaseLock();
|
|
1223
|
+
};
|
|
1224
|
+
}
|
|
1225
|
+
/**
|
|
1183
1226
|
* Get the current epoch for a producer on a stream.
|
|
1184
1227
|
* Returns undefined if the producer doesn't exist or stream not found.
|
|
1185
1228
|
*/
|
|
@@ -1312,6 +1355,7 @@ var FileBackedStreamStore = class {
|
|
|
1312
1355
|
effectiveTtlSeconds = resolved.ttlSeconds;
|
|
1313
1356
|
}
|
|
1314
1357
|
const key = `stream:${streamPath}`;
|
|
1358
|
+
const t0 = performance.now();
|
|
1315
1359
|
const streamMeta = {
|
|
1316
1360
|
path: streamPath,
|
|
1317
1361
|
contentType,
|
|
@@ -1329,11 +1373,10 @@ var FileBackedStreamStore = class {
|
|
|
1329
1373
|
forkOffset: isFork ? forkOffset : void 0,
|
|
1330
1374
|
refCount: 0
|
|
1331
1375
|
};
|
|
1332
|
-
const
|
|
1376
|
+
const tAfterMeta = performance.now();
|
|
1377
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1333
1378
|
try {
|
|
1334
|
-
|
|
1335
|
-
const segmentPath = node_path.join(streamDir, `segment_00000.log`);
|
|
1336
|
-
node_fs.writeFileSync(segmentPath, ``);
|
|
1379
|
+
await this.db.put(key, streamMeta);
|
|
1337
1380
|
} catch (err) {
|
|
1338
1381
|
if (isFork && sourceMeta) {
|
|
1339
1382
|
const sourceKey = `stream:${options.forkedFrom}`;
|
|
@@ -1346,10 +1389,18 @@ var FileBackedStreamStore = class {
|
|
|
1346
1389
|
this.db.putSync(sourceKey, updatedSource);
|
|
1347
1390
|
}
|
|
1348
1391
|
}
|
|
1349
|
-
|
|
1392
|
+
serverLog.error(`[FileBackedStreamStore] Error creating stream (LMDB put):`, err);
|
|
1350
1393
|
throw err;
|
|
1351
1394
|
}
|
|
1352
|
-
|
|
1395
|
+
const tAfterLmdb = performance.now();
|
|
1396
|
+
try {
|
|
1397
|
+
await this.fileHandlePool.openWriteStream(segmentPath);
|
|
1398
|
+
} catch (err) {
|
|
1399
|
+
this.db.removeSync(key);
|
|
1400
|
+
serverLog.error(`[FileBackedStreamStore] Error creating stream (file open):`, err);
|
|
1401
|
+
throw err;
|
|
1402
|
+
}
|
|
1403
|
+
const tAfterOpen = performance.now();
|
|
1353
1404
|
if (options.initialData && options.initialData.length > 0) try {
|
|
1354
1405
|
await this.append(streamPath, options.initialData, {
|
|
1355
1406
|
contentType: options.contentType,
|
|
@@ -1369,12 +1420,24 @@ var FileBackedStreamStore = class {
|
|
|
1369
1420
|
}
|
|
1370
1421
|
throw err;
|
|
1371
1422
|
}
|
|
1423
|
+
const tAfterAppend = performance.now();
|
|
1372
1424
|
if (options.closed) {
|
|
1373
1425
|
const updatedMeta = this.db.get(key);
|
|
1374
1426
|
updatedMeta.closed = true;
|
|
1375
|
-
this.db.
|
|
1427
|
+
await this.db.put(key, updatedMeta);
|
|
1376
1428
|
}
|
|
1377
1429
|
const updated = this.db.get(key);
|
|
1430
|
+
const totalMs = performance.now() - t0;
|
|
1431
|
+
if (totalMs > 50) serverLog.event({
|
|
1432
|
+
event: `store.create`,
|
|
1433
|
+
path: streamPath,
|
|
1434
|
+
totalMs: +totalMs.toFixed(2),
|
|
1435
|
+
metaMs: +(tAfterMeta - t0).toFixed(2),
|
|
1436
|
+
lmdbMs: +(tAfterLmdb - tAfterMeta).toFixed(2),
|
|
1437
|
+
openMs: +(tAfterOpen - tAfterLmdb).toFixed(2),
|
|
1438
|
+
appendMs: +(tAfterAppend - tAfterOpen).toFixed(2),
|
|
1439
|
+
initBytes: options.initialData?.length ?? 0
|
|
1440
|
+
}, `store.create slow`);
|
|
1378
1441
|
return this.streamMetaToStream(updated);
|
|
1379
1442
|
}
|
|
1380
1443
|
get(streamPath) {
|
|
@@ -1415,13 +1478,10 @@ var FileBackedStreamStore = class {
|
|
|
1415
1478
|
if (!streamMeta) return;
|
|
1416
1479
|
const forkedFrom = streamMeta.forkedFrom;
|
|
1417
1480
|
this.cancelLongPollsForStream(streamPath);
|
|
1418
|
-
const segmentPath =
|
|
1419
|
-
this.fileHandlePool.closeFileHandle(segmentPath).catch((err) => {
|
|
1420
|
-
console.error(`[FileBackedStreamStore] Error closing file handle:`, err);
|
|
1421
|
-
});
|
|
1481
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1422
1482
|
this.db.removeSync(key);
|
|
1423
|
-
this.
|
|
1424
|
-
|
|
1483
|
+
this.fileHandlePool.closeFileHandle(segmentPath).then(() => node_fs.promises.unlink(segmentPath)).catch((err) => {
|
|
1484
|
+
serverLog.error(`[FileBackedStreamStore] Error cleaning up stream file:`, err);
|
|
1425
1485
|
});
|
|
1426
1486
|
if (forkedFrom) {
|
|
1427
1487
|
const parentKey = `stream:${forkedFrom}`;
|
|
@@ -1437,7 +1497,20 @@ var FileBackedStreamStore = class {
|
|
|
1437
1497
|
}
|
|
1438
1498
|
}
|
|
1439
1499
|
}
|
|
1500
|
+
/**
|
|
1501
|
+
* Public append entry point. Serializes concurrent appends to the same
|
|
1502
|
+
* stream so the read-modify-write of currentOffset cannot interleave —
|
|
1503
|
+
* see acquireStreamAppendLock for the underlying race.
|
|
1504
|
+
*/
|
|
1440
1505
|
async append(streamPath, data, options = {}) {
|
|
1506
|
+
const releaseLock = await this.acquireStreamAppendLock(streamPath);
|
|
1507
|
+
try {
|
|
1508
|
+
return await this.appendInner(streamPath, data, options);
|
|
1509
|
+
} finally {
|
|
1510
|
+
releaseLock();
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
async appendInner(streamPath, data, options = {}) {
|
|
1441
1514
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1442
1515
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1443
1516
|
if (streamMeta.softDeleted) throw new Error(`Stream is soft-deleted: ${streamPath}`);
|
|
@@ -1479,10 +1552,11 @@ var FileBackedStreamStore = class {
|
|
|
1479
1552
|
const parts = streamMeta.currentOffset.split(`_`).map(Number);
|
|
1480
1553
|
const readSeq = parts[0];
|
|
1481
1554
|
const byteOffset = parts[1];
|
|
1482
|
-
const
|
|
1555
|
+
const FRAME_OVERHEAD = 5;
|
|
1556
|
+
const newByteOffset = byteOffset + FRAME_OVERHEAD + processedData.length;
|
|
1483
1557
|
const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
1484
|
-
const
|
|
1485
|
-
const
|
|
1558
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1559
|
+
const tAppendStart = performance.now();
|
|
1486
1560
|
const stream = this.fileHandlePool.getWriteStream(segmentPath);
|
|
1487
1561
|
const lengthBuf = Buffer.allocUnsafe(4);
|
|
1488
1562
|
lengthBuf.writeUInt32BE(processedData.length, 0);
|
|
@@ -1497,12 +1571,14 @@ var FileBackedStreamStore = class {
|
|
|
1497
1571
|
else resolve();
|
|
1498
1572
|
});
|
|
1499
1573
|
});
|
|
1574
|
+
const tAfterWrite = performance.now();
|
|
1500
1575
|
const message = {
|
|
1501
1576
|
data: processedData,
|
|
1502
1577
|
offset: newOffset,
|
|
1503
1578
|
timestamp: Date.now()
|
|
1504
1579
|
};
|
|
1505
1580
|
await this.fileHandlePool.fsyncFile(segmentPath);
|
|
1581
|
+
const tAfterFsync = performance.now();
|
|
1506
1582
|
const updatedProducers = { ...streamMeta.producers };
|
|
1507
1583
|
if (producerResult && producerResult.status === `accepted`) updatedProducers[producerResult.producerId] = producerResult.proposedState;
|
|
1508
1584
|
let closedBy = void 0;
|
|
@@ -1521,7 +1597,19 @@ var FileBackedStreamStore = class {
|
|
|
1521
1597
|
closedBy: closedBy ?? streamMeta.closedBy
|
|
1522
1598
|
};
|
|
1523
1599
|
const key = `stream:${streamPath}`;
|
|
1524
|
-
this.db.
|
|
1600
|
+
await this.db.put(key, updatedMeta);
|
|
1601
|
+
const tAfterLmdb = performance.now();
|
|
1602
|
+
const appendTotal = tAfterLmdb - tAppendStart;
|
|
1603
|
+
if (appendTotal > 50) serverLog.event({
|
|
1604
|
+
event: `store.append`,
|
|
1605
|
+
path: streamPath,
|
|
1606
|
+
totalMs: +appendTotal.toFixed(2),
|
|
1607
|
+
writeMs: +(tAfterWrite - tAppendStart).toFixed(2),
|
|
1608
|
+
fsyncMs: +(tAfterFsync - tAfterWrite).toFixed(2),
|
|
1609
|
+
lmdbMs: +(tAfterLmdb - tAfterFsync).toFixed(2),
|
|
1610
|
+
bytes: processedData.length,
|
|
1611
|
+
isInitial: options.isInitialCreate ?? false
|
|
1612
|
+
}, `store.append slow`);
|
|
1525
1613
|
this.notifyLongPolls(streamPath);
|
|
1526
1614
|
if (options.close) this.notifyLongPollsClosed(streamPath);
|
|
1527
1615
|
if (producerResult || options.close) return {
|
|
@@ -1614,7 +1702,7 @@ var FileBackedStreamStore = class {
|
|
|
1614
1702
|
},
|
|
1615
1703
|
producers: updatedProducers
|
|
1616
1704
|
};
|
|
1617
|
-
this.db.
|
|
1705
|
+
await this.db.put(key, updatedMeta);
|
|
1618
1706
|
this.notifyLongPollsClosed(streamPath);
|
|
1619
1707
|
return {
|
|
1620
1708
|
finalOffset: streamMeta.currentOffset,
|
|
@@ -1648,7 +1736,7 @@ var FileBackedStreamStore = class {
|
|
|
1648
1736
|
const messageData = fileContent.subarray(filePos, filePos + messageLength);
|
|
1649
1737
|
filePos += messageLength;
|
|
1650
1738
|
filePos += 1;
|
|
1651
|
-
physicalDataOffset += messageLength;
|
|
1739
|
+
physicalDataOffset += messageLength + 5;
|
|
1652
1740
|
const logicalOffset = baseByteOffset + physicalDataOffset;
|
|
1653
1741
|
if (capByte !== void 0 && logicalOffset > capByte) break;
|
|
1654
1742
|
if (logicalOffset > startByte) messages.push({
|
|
@@ -1658,7 +1746,7 @@ var FileBackedStreamStore = class {
|
|
|
1658
1746
|
});
|
|
1659
1747
|
}
|
|
1660
1748
|
} catch (err) {
|
|
1661
|
-
|
|
1749
|
+
serverLog.error(`[FileBackedStreamStore] Error reading segment file:`, err);
|
|
1662
1750
|
}
|
|
1663
1751
|
return messages;
|
|
1664
1752
|
}
|
|
@@ -1680,7 +1768,7 @@ var FileBackedStreamStore = class {
|
|
|
1680
1768
|
messages.push(...inherited);
|
|
1681
1769
|
}
|
|
1682
1770
|
}
|
|
1683
|
-
const segmentPath =
|
|
1771
|
+
const segmentPath = segmentFile(this.dataDir, sourceMeta.directoryName);
|
|
1684
1772
|
const sourceBaseByte = sourceMeta.forkOffset ? Number(sourceMeta.forkOffset.split(`_`)[1] ?? 0) : 0;
|
|
1685
1773
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, sourceBaseByte, capByte);
|
|
1686
1774
|
messages.push(...ownMessages);
|
|
@@ -1707,11 +1795,11 @@ var FileBackedStreamStore = class {
|
|
|
1707
1795
|
const inherited = this.readForkedMessages(streamMeta.forkedFrom, startByte, forkByte);
|
|
1708
1796
|
messages.push(...inherited);
|
|
1709
1797
|
}
|
|
1710
|
-
const segmentPath =
|
|
1798
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1711
1799
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, forkByte);
|
|
1712
1800
|
messages.push(...ownMessages);
|
|
1713
1801
|
} else {
|
|
1714
|
-
const segmentPath =
|
|
1802
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1715
1803
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, 0);
|
|
1716
1804
|
messages.push(...ownMessages);
|
|
1717
1805
|
}
|
|
@@ -1782,6 +1870,7 @@ var FileBackedStreamStore = class {
|
|
|
1782
1870
|
formatResponse(streamPath, messages) {
|
|
1783
1871
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1784
1872
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1873
|
+
if (normalizeContentType(streamMeta.contentType) === `application/json`) return formatJsonMessages(messages);
|
|
1785
1874
|
const totalSize = messages.reduce((sum, m) => sum + m.data.length, 0);
|
|
1786
1875
|
const concatenated = new Uint8Array(totalSize);
|
|
1787
1876
|
let offset = 0;
|
|
@@ -1789,7 +1878,6 @@ var FileBackedStreamStore = class {
|
|
|
1789
1878
|
concatenated.set(msg.data, offset);
|
|
1790
1879
|
offset += msg.data.length;
|
|
1791
1880
|
}
|
|
1792
|
-
if (normalizeContentType(streamMeta.contentType) === `application/json`) return formatJsonResponse(concatenated);
|
|
1793
1881
|
return concatenated;
|
|
1794
1882
|
}
|
|
1795
1883
|
getCurrentOffset(streamPath) {
|
|
@@ -1809,7 +1897,7 @@ var FileBackedStreamStore = class {
|
|
|
1809
1897
|
const entries = Array.from(range);
|
|
1810
1898
|
for (const { key } of entries) this.db.removeSync(key);
|
|
1811
1899
|
this.fileHandlePool.closeAll().catch((err) => {
|
|
1812
|
-
|
|
1900
|
+
serverLog.error(`[FileBackedStreamStore] Error closing handles:`, err);
|
|
1813
1901
|
});
|
|
1814
1902
|
}
|
|
1815
1903
|
/**
|
|
@@ -1963,30 +2051,985 @@ function handleCursorCollision(currentCursor, previousCursor, options = {}) {
|
|
|
1963
2051
|
return generateResponseCursor(previousCursor, options);
|
|
1964
2052
|
}
|
|
1965
2053
|
|
|
2054
|
+
//#endregion
|
|
2055
|
+
//#region src/crypto.ts
|
|
2056
|
+
/**
|
|
2057
|
+
* Generate a unique wake ID.
|
|
2058
|
+
*/
|
|
2059
|
+
function generateWakeId() {
|
|
2060
|
+
return `w_${(0, node_crypto.randomBytes)(12).toString(`hex`)}`;
|
|
2061
|
+
}
|
|
2062
|
+
const WEBHOOK_KEYPAIR = (0, node_crypto.generateKeyPairSync)(`ed25519`);
|
|
2063
|
+
const WEBHOOK_PUBLIC_JWK = buildWebhookPublicJwk();
|
|
2064
|
+
function buildWebhookPublicJwk() {
|
|
2065
|
+
const exported = WEBHOOK_KEYPAIR.publicKey.export({ format: `jwk` });
|
|
2066
|
+
if (exported.kty !== `OKP` || exported.crv !== `Ed25519` || !exported.x) throw new Error(`Failed to export Ed25519 webhook signing key`);
|
|
2067
|
+
const thumbprintInput = JSON.stringify({
|
|
2068
|
+
crv: exported.crv,
|
|
2069
|
+
kty: exported.kty,
|
|
2070
|
+
x: exported.x
|
|
2071
|
+
});
|
|
2072
|
+
const kid = `ds_${(0, node_crypto.createHash)(`sha256`).update(thumbprintInput).digest(`base64url`)}`;
|
|
2073
|
+
return {
|
|
2074
|
+
kty: `OKP`,
|
|
2075
|
+
crv: `Ed25519`,
|
|
2076
|
+
x: exported.x,
|
|
2077
|
+
kid,
|
|
2078
|
+
use: `sig`,
|
|
2079
|
+
alg: `EdDSA`
|
|
2080
|
+
};
|
|
2081
|
+
}
|
|
2082
|
+
function getWebhookSigningKeyId() {
|
|
2083
|
+
return WEBHOOK_PUBLIC_JWK.kid;
|
|
2084
|
+
}
|
|
2085
|
+
function getWebhookJwks() {
|
|
2086
|
+
return { keys: [{ ...WEBHOOK_PUBLIC_JWK }] };
|
|
2087
|
+
}
|
|
2088
|
+
/**
|
|
2089
|
+
* Sign a webhook payload for the Webhook-Signature header.
|
|
2090
|
+
* Format: t=<timestamp>,kid=<key_id>,ed25519=<base64url_signature>
|
|
2091
|
+
*/
|
|
2092
|
+
function signWebhookPayload(body) {
|
|
2093
|
+
const timestamp = Math.floor(Date.now() / 1e3);
|
|
2094
|
+
const payload = `${timestamp}.${body}`;
|
|
2095
|
+
const signature = (0, node_crypto.sign)(null, Buffer.from(payload), WEBHOOK_KEYPAIR.privateKey).toString(`base64url`);
|
|
2096
|
+
return `t=${timestamp},kid=${WEBHOOK_PUBLIC_JWK.kid},ed25519=${signature}`;
|
|
2097
|
+
}
|
|
2098
|
+
const TOKEN_KEY = (0, node_crypto.randomBytes)(32);
|
|
2099
|
+
/**
|
|
2100
|
+
* Generate a signed callback token.
|
|
2101
|
+
* Token format: base64url(json_payload).base64url(hmac_signature)
|
|
2102
|
+
* Payload: { consumer_id, epoch, exp }
|
|
2103
|
+
*/
|
|
2104
|
+
function generateCallbackToken(consumerId, epoch) {
|
|
2105
|
+
const payload = {
|
|
2106
|
+
sub: consumerId,
|
|
2107
|
+
epoch,
|
|
2108
|
+
exp: Math.floor(Date.now() / 1e3) + 3600,
|
|
2109
|
+
jti: (0, node_crypto.randomBytes)(8).toString(`hex`)
|
|
2110
|
+
};
|
|
2111
|
+
const payloadStr = Buffer.from(JSON.stringify(payload)).toString(`base64url`);
|
|
2112
|
+
const sig = (0, node_crypto.createHmac)(`sha256`, TOKEN_KEY).update(payloadStr).digest(`base64url`);
|
|
2113
|
+
return `${payloadStr}.${sig}`;
|
|
2114
|
+
}
|
|
2115
|
+
/**
|
|
2116
|
+
* Validate a callback token. Returns the decoded payload or null.
|
|
2117
|
+
* On success, includes `exp` (unix seconds) so callers can decide
|
|
2118
|
+
* whether the token needs refreshing.
|
|
2119
|
+
*/
|
|
2120
|
+
function validateCallbackToken(token, consumerId) {
|
|
2121
|
+
const parts = token.split(`.`);
|
|
2122
|
+
if (parts.length !== 2) return {
|
|
2123
|
+
valid: false,
|
|
2124
|
+
code: `TOKEN_INVALID`
|
|
2125
|
+
};
|
|
2126
|
+
const [payloadStr, sig] = parts;
|
|
2127
|
+
const expectedSig = (0, node_crypto.createHmac)(`sha256`, TOKEN_KEY).update(payloadStr).digest(`base64url`);
|
|
2128
|
+
try {
|
|
2129
|
+
if (!(0, node_crypto.timingSafeEqual)(Buffer.from(sig), Buffer.from(expectedSig))) return {
|
|
2130
|
+
valid: false,
|
|
2131
|
+
code: `TOKEN_INVALID`
|
|
2132
|
+
};
|
|
2133
|
+
} catch {
|
|
2134
|
+
return {
|
|
2135
|
+
valid: false,
|
|
2136
|
+
code: `TOKEN_INVALID`
|
|
2137
|
+
};
|
|
2138
|
+
}
|
|
2139
|
+
let payload;
|
|
2140
|
+
try {
|
|
2141
|
+
payload = JSON.parse(Buffer.from(payloadStr, `base64url`).toString());
|
|
2142
|
+
} catch {
|
|
2143
|
+
return {
|
|
2144
|
+
valid: false,
|
|
2145
|
+
code: `TOKEN_INVALID`
|
|
2146
|
+
};
|
|
2147
|
+
}
|
|
2148
|
+
if (payload.sub !== consumerId) return {
|
|
2149
|
+
valid: false,
|
|
2150
|
+
code: `TOKEN_INVALID`
|
|
2151
|
+
};
|
|
2152
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
2153
|
+
if (now > payload.exp) return {
|
|
2154
|
+
valid: false,
|
|
2155
|
+
code: `TOKEN_EXPIRED`
|
|
2156
|
+
};
|
|
2157
|
+
return {
|
|
2158
|
+
valid: true,
|
|
2159
|
+
exp: payload.exp,
|
|
2160
|
+
epoch: payload.epoch
|
|
2161
|
+
};
|
|
2162
|
+
}
|
|
2163
|
+
|
|
2164
|
+
//#endregion
|
|
2165
|
+
//#region src/glob.ts
|
|
2166
|
+
/**
|
|
2167
|
+
* Glob pattern matching for webhook subscription patterns.
|
|
2168
|
+
*
|
|
2169
|
+
* Supports:
|
|
2170
|
+
* - `*` matches exactly one path segment
|
|
2171
|
+
* - `**` matches zero or more path segments (recursive)
|
|
2172
|
+
* - Literal segments match exactly
|
|
2173
|
+
*/
|
|
2174
|
+
/**
|
|
2175
|
+
* Match a stream path against a glob pattern.
|
|
2176
|
+
*/
|
|
2177
|
+
function globMatch(pattern, path) {
|
|
2178
|
+
const patternParts = splitPath(pattern);
|
|
2179
|
+
const pathParts = splitPath(path);
|
|
2180
|
+
return matchParts(patternParts, 0, pathParts, 0);
|
|
2181
|
+
}
|
|
2182
|
+
function splitPath(p) {
|
|
2183
|
+
return p.replace(/^\/+/, ``).replace(/\/+$/, ``).split(`/`).filter((s) => s.length > 0);
|
|
2184
|
+
}
|
|
2185
|
+
function matchParts(pattern, pi, path, si) {
|
|
2186
|
+
while (pi < pattern.length && si < path.length) {
|
|
2187
|
+
const seg = pattern[pi];
|
|
2188
|
+
if (seg === `**`) {
|
|
2189
|
+
for (let i = si; i <= path.length; i++) if (matchParts(pattern, pi + 1, path, i)) return true;
|
|
2190
|
+
return false;
|
|
2191
|
+
}
|
|
2192
|
+
if (seg === `*`) {
|
|
2193
|
+
pi++;
|
|
2194
|
+
si++;
|
|
2195
|
+
continue;
|
|
2196
|
+
}
|
|
2197
|
+
const decodedSeg = seg.replace(/%2[Aa]/g, `*`);
|
|
2198
|
+
if (decodedSeg !== path[si]) return false;
|
|
2199
|
+
pi++;
|
|
2200
|
+
si++;
|
|
2201
|
+
}
|
|
2202
|
+
while (pi < pattern.length && pattern[pi] === `**`) pi++;
|
|
2203
|
+
return pi === pattern.length && si === path.length;
|
|
2204
|
+
}
|
|
2205
|
+
|
|
2206
|
+
//#endregion
|
|
2207
|
+
//#region src/subscription-manager.ts
|
|
2208
|
+
const DEFAULT_LEASE_TTL_MS = 3e4;
|
|
2209
|
+
const MIN_LEASE_TTL_MS = 1e3;
|
|
2210
|
+
const MAX_LEASE_TTL_MS = 10 * 6e4;
|
|
2211
|
+
const ZERO_OFFSET = `0000000000000000_0000000000000000`;
|
|
2212
|
+
const BEFORE_FIRST_OFFSET = `-1`;
|
|
2213
|
+
const MAX_RETRY_DELAY_MS = 6e4;
|
|
2214
|
+
function compareOffsets(a, b) {
|
|
2215
|
+
if (a < b) return -1;
|
|
2216
|
+
if (a > b) return 1;
|
|
2217
|
+
return 0;
|
|
2218
|
+
}
|
|
2219
|
+
function normalizeRelativePath(path) {
|
|
2220
|
+
return path.replace(/^\/+/, ``).replace(/\/+$/, ``);
|
|
2221
|
+
}
|
|
2222
|
+
function toAbsoluteStreamPath(streamPath) {
|
|
2223
|
+
return `/v1/stream/${normalizeRelativePath(streamPath)}`;
|
|
2224
|
+
}
|
|
2225
|
+
function toStreamRelativePath(absolutePath) {
|
|
2226
|
+
const streamRoot = `/v1/stream/`;
|
|
2227
|
+
if (!absolutePath.startsWith(streamRoot)) return null;
|
|
2228
|
+
const path = absolutePath.slice(streamRoot.length);
|
|
2229
|
+
if (path === `__ds` || path.startsWith(`__ds/`)) return null;
|
|
2230
|
+
return path.length > 0 ? path : null;
|
|
2231
|
+
}
|
|
2232
|
+
function stableConfigHash(input) {
|
|
2233
|
+
const canonical = {
|
|
2234
|
+
type: input.type,
|
|
2235
|
+
pattern: input.pattern,
|
|
2236
|
+
streams: [...new Set(input.streams)].sort(),
|
|
2237
|
+
webhook: input.webhook ? { url: input.webhook.url } : void 0,
|
|
2238
|
+
wake_stream: input.wake_stream,
|
|
2239
|
+
lease_ttl_ms: input.lease_ttl_ms,
|
|
2240
|
+
description: input.description
|
|
2241
|
+
};
|
|
2242
|
+
return (0, node_crypto.createHash)(`sha256`).update(JSON.stringify(canonical)).digest(`hex`);
|
|
2243
|
+
}
|
|
2244
|
+
function isPrivateOrLinkLocalIpv4(host) {
|
|
2245
|
+
const parts = host.split(`.`).map((part) => Number(part));
|
|
2246
|
+
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part))) return false;
|
|
2247
|
+
const [a, b] = parts;
|
|
2248
|
+
return a === 10 || a === 127 || a === 0 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a === 169 && b === 254;
|
|
2249
|
+
}
|
|
2250
|
+
function isLocalDevHost(host) {
|
|
2251
|
+
return host === `localhost` || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host);
|
|
2252
|
+
}
|
|
2253
|
+
function validateWebhookUrl(rawUrl) {
|
|
2254
|
+
let url;
|
|
2255
|
+
try {
|
|
2256
|
+
url = new URL(rawUrl);
|
|
2257
|
+
} catch {
|
|
2258
|
+
return {
|
|
2259
|
+
ok: false,
|
|
2260
|
+
message: `webhook.url must be a valid URL`
|
|
2261
|
+
};
|
|
2262
|
+
}
|
|
2263
|
+
const host = url.hostname.toLowerCase();
|
|
2264
|
+
if (url.protocol === `http:`) {
|
|
2265
|
+
if (isLocalDevHost(host)) return { ok: true };
|
|
2266
|
+
return {
|
|
2267
|
+
ok: false,
|
|
2268
|
+
message: `http webhook URLs are only allowed for localhost or 127.0.0.x`
|
|
2269
|
+
};
|
|
2270
|
+
}
|
|
2271
|
+
if (url.protocol !== `https:`) return {
|
|
2272
|
+
ok: false,
|
|
2273
|
+
message: `webhook.url must use https`
|
|
2274
|
+
};
|
|
2275
|
+
if (host === `localhost`) return {
|
|
2276
|
+
ok: false,
|
|
2277
|
+
message: `localhost webhook URLs must use http for dev`
|
|
2278
|
+
};
|
|
2279
|
+
if ((0, node_net.isIP)(host) === 4 && isPrivateOrLinkLocalIpv4(host)) return {
|
|
2280
|
+
ok: false,
|
|
2281
|
+
message: `webhook.url must not target private or link-local hosts`
|
|
2282
|
+
};
|
|
2283
|
+
if ((0, node_net.isIP)(host) === 6) return {
|
|
2284
|
+
ok: false,
|
|
2285
|
+
message: `IPv6 webhook hosts are not accepted by the reference server`
|
|
2286
|
+
};
|
|
2287
|
+
return { ok: true };
|
|
2288
|
+
}
|
|
2289
|
+
var SubscriptionManager = class {
|
|
2290
|
+
subscriptions = new Map();
|
|
2291
|
+
streamStore;
|
|
2292
|
+
callbackBaseUrl;
|
|
2293
|
+
webhooksEnabled;
|
|
2294
|
+
isShuttingDown = false;
|
|
2295
|
+
constructor(opts) {
|
|
2296
|
+
this.callbackBaseUrl = opts.callbackBaseUrl;
|
|
2297
|
+
this.streamStore = opts.streamStore;
|
|
2298
|
+
this.webhooksEnabled = opts.webhooksEnabled ?? true;
|
|
2299
|
+
}
|
|
2300
|
+
createOrConfirm(id, input) {
|
|
2301
|
+
const configHash = stableConfigHash(input);
|
|
2302
|
+
const existing = this.subscriptions.get(id);
|
|
2303
|
+
if (existing) {
|
|
2304
|
+
if (existing.config_hash !== configHash) return { error: {
|
|
2305
|
+
code: `SUBSCRIPTION_ALREADY_EXISTS`,
|
|
2306
|
+
message: `Subscription already exists with different configuration`
|
|
2307
|
+
} };
|
|
2308
|
+
return {
|
|
2309
|
+
subscription: existing,
|
|
2310
|
+
created: false
|
|
2311
|
+
};
|
|
2312
|
+
}
|
|
2313
|
+
if (input.type === `webhook`) {
|
|
2314
|
+
if (!this.webhooksEnabled) return { error: {
|
|
2315
|
+
code: `INVALID_REQUEST`,
|
|
2316
|
+
message: `webhook subscriptions are not enabled on this server`
|
|
2317
|
+
} };
|
|
2318
|
+
if (!input.webhook) return { error: {
|
|
2319
|
+
code: `INVALID_REQUEST`,
|
|
2320
|
+
message: `webhook subscriptions require webhook.url`
|
|
2321
|
+
} };
|
|
2322
|
+
const validation = validateWebhookUrl(input.webhook.url);
|
|
2323
|
+
if (!validation.ok) return { error: {
|
|
2324
|
+
code: `WEBHOOK_URL_REJECTED`,
|
|
2325
|
+
message: validation.message
|
|
2326
|
+
} };
|
|
2327
|
+
}
|
|
2328
|
+
if (input.type === `pull-wake` && !input.wake_stream) return { error: {
|
|
2329
|
+
code: `INVALID_REQUEST`,
|
|
2330
|
+
message: `pull-wake subscriptions require wake_stream`
|
|
2331
|
+
} };
|
|
2332
|
+
const subscription = {
|
|
2333
|
+
id,
|
|
2334
|
+
type: input.type,
|
|
2335
|
+
pattern: input.pattern,
|
|
2336
|
+
webhook: input.webhook ? { url: input.webhook.url } : void 0,
|
|
2337
|
+
wake_stream: input.wake_stream,
|
|
2338
|
+
lease_ttl_ms: input.lease_ttl_ms,
|
|
2339
|
+
description: input.description,
|
|
2340
|
+
created_at: new Date().toISOString(),
|
|
2341
|
+
status: `active`,
|
|
2342
|
+
config_hash: configHash,
|
|
2343
|
+
streams: new Map(),
|
|
2344
|
+
generation: 0,
|
|
2345
|
+
wake_id: null,
|
|
2346
|
+
wake_snapshot: new Map(),
|
|
2347
|
+
token: null,
|
|
2348
|
+
holder: null,
|
|
2349
|
+
lease_timer: null,
|
|
2350
|
+
retry_count: 0,
|
|
2351
|
+
retry_timer: null,
|
|
2352
|
+
next_attempt_at: null
|
|
2353
|
+
};
|
|
2354
|
+
for (const stream of input.streams) this.linkStream(subscription, stream, `explicit`, this.getTailOffset(stream));
|
|
2355
|
+
if (input.pattern) {
|
|
2356
|
+
for (const stream of this.listStreams()) if (globMatch(input.pattern, stream)) this.linkStream(subscription, stream, `glob`, this.getTailOffset(stream));
|
|
2357
|
+
}
|
|
2358
|
+
this.subscriptions.set(id, subscription);
|
|
2359
|
+
return {
|
|
2360
|
+
subscription,
|
|
2361
|
+
created: true
|
|
2362
|
+
};
|
|
2363
|
+
}
|
|
2364
|
+
get(id) {
|
|
2365
|
+
return this.subscriptions.get(id);
|
|
2366
|
+
}
|
|
2367
|
+
delete(id) {
|
|
2368
|
+
const subscription = this.subscriptions.get(id);
|
|
2369
|
+
if (!subscription) return false;
|
|
2370
|
+
this.clearLease(subscription);
|
|
2371
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2372
|
+
this.subscriptions.delete(id);
|
|
2373
|
+
return true;
|
|
2374
|
+
}
|
|
2375
|
+
addExplicitStreams(id, streams) {
|
|
2376
|
+
const subscription = this.get(id);
|
|
2377
|
+
if (!subscription) return false;
|
|
2378
|
+
for (const stream of streams) this.linkStream(subscription, stream, `explicit`, this.getTailOffset(stream));
|
|
2379
|
+
return true;
|
|
2380
|
+
}
|
|
2381
|
+
removeExplicitStream(id, streamPath) {
|
|
2382
|
+
const subscription = this.get(id);
|
|
2383
|
+
if (!subscription) return false;
|
|
2384
|
+
const normalized = normalizeRelativePath(streamPath);
|
|
2385
|
+
const link = subscription.streams.get(normalized);
|
|
2386
|
+
if (!link) return true;
|
|
2387
|
+
link.link_types.delete(`explicit`);
|
|
2388
|
+
if (link.link_types.size === 0) subscription.streams.delete(normalized);
|
|
2389
|
+
return true;
|
|
2390
|
+
}
|
|
2391
|
+
async onStreamAppend(absolutePath) {
|
|
2392
|
+
if (this.isShuttingDown) return;
|
|
2393
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2394
|
+
const relative = toStreamRelativePath(absolutePath);
|
|
2395
|
+
if (!relative) continue;
|
|
2396
|
+
if (subscription.pattern && globMatch(subscription.pattern, relative)) {
|
|
2397
|
+
const existing = subscription.streams.get(relative);
|
|
2398
|
+
this.linkStream(subscription, relative, `glob`, existing?.acked_offset ?? BEFORE_FIRST_OFFSET);
|
|
2399
|
+
}
|
|
2400
|
+
if (subscription.streams.has(relative)) await this.maybeWake(subscription, relative);
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
onStreamDeleted(absolutePath) {
|
|
2404
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2405
|
+
const relative = toStreamRelativePath(absolutePath);
|
|
2406
|
+
if (relative) subscription.streams.delete(relative);
|
|
2407
|
+
}
|
|
2408
|
+
}
|
|
2409
|
+
async handleWebhookCallback(id, token, request) {
|
|
2410
|
+
const subscription = this.get(id);
|
|
2411
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2412
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2413
|
+
if (fenced) return fenced;
|
|
2414
|
+
const ackError = this.applyAcks(subscription, request);
|
|
2415
|
+
if (ackError) return ackError;
|
|
2416
|
+
this.extendLease(subscription);
|
|
2417
|
+
let nextWake = false;
|
|
2418
|
+
if (request.done === true) {
|
|
2419
|
+
this.clearLease(subscription);
|
|
2420
|
+
subscription.token = null;
|
|
2421
|
+
subscription.holder = null;
|
|
2422
|
+
subscription.wake_id = null;
|
|
2423
|
+
subscription.wake_snapshot.clear();
|
|
2424
|
+
nextWake = await this.triggerNextWakeIfPending(subscription);
|
|
2425
|
+
}
|
|
2426
|
+
return {
|
|
2427
|
+
status: 200,
|
|
2428
|
+
body: {
|
|
2429
|
+
ok: true,
|
|
2430
|
+
next_wake: nextWake
|
|
2431
|
+
}
|
|
2432
|
+
};
|
|
2433
|
+
}
|
|
2434
|
+
async claim(id, worker) {
|
|
2435
|
+
const subscription = this.get(id);
|
|
2436
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2437
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2438
|
+
if (subscription.holder) return {
|
|
2439
|
+
status: 409,
|
|
2440
|
+
body: { error: {
|
|
2441
|
+
code: `ALREADY_CLAIMED`,
|
|
2442
|
+
current_holder: subscription.holder,
|
|
2443
|
+
generation: subscription.generation
|
|
2444
|
+
} }
|
|
2445
|
+
};
|
|
2446
|
+
if (!this.hasPendingWork(subscription)) return this.errorResponse(409, `NO_PENDING_WORK`, `Subscription has no pending work`);
|
|
2447
|
+
if (!subscription.wake_id) await this.createWake(subscription, this.firstPendingStream(subscription));
|
|
2448
|
+
subscription.holder = worker;
|
|
2449
|
+
subscription.token = generateCallbackToken(this.tokenSubject(subscription), subscription.generation);
|
|
2450
|
+
this.extendLease(subscription);
|
|
2451
|
+
return {
|
|
2452
|
+
status: 200,
|
|
2453
|
+
body: {
|
|
2454
|
+
wake_id: subscription.wake_id,
|
|
2455
|
+
generation: subscription.generation,
|
|
2456
|
+
token: subscription.token,
|
|
2457
|
+
streams: this.streamInfos(subscription),
|
|
2458
|
+
lease_ttl_ms: subscription.lease_ttl_ms
|
|
2459
|
+
}
|
|
2460
|
+
};
|
|
2461
|
+
}
|
|
2462
|
+
async ack(id, token, request) {
|
|
2463
|
+
const subscription = this.get(id);
|
|
2464
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2465
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2466
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2467
|
+
if (fenced) return fenced;
|
|
2468
|
+
const ackError = this.applyAcks(subscription, request);
|
|
2469
|
+
if (ackError) return ackError;
|
|
2470
|
+
this.extendLease(subscription);
|
|
2471
|
+
let nextWake = false;
|
|
2472
|
+
if (request.done === true) {
|
|
2473
|
+
this.clearLease(subscription);
|
|
2474
|
+
subscription.token = null;
|
|
2475
|
+
subscription.holder = null;
|
|
2476
|
+
subscription.wake_id = null;
|
|
2477
|
+
subscription.wake_snapshot.clear();
|
|
2478
|
+
nextWake = await this.triggerNextWakeIfPending(subscription);
|
|
2479
|
+
}
|
|
2480
|
+
return {
|
|
2481
|
+
status: 200,
|
|
2482
|
+
body: {
|
|
2483
|
+
ok: true,
|
|
2484
|
+
next_wake: nextWake
|
|
2485
|
+
}
|
|
2486
|
+
};
|
|
2487
|
+
}
|
|
2488
|
+
async release(id, token, request) {
|
|
2489
|
+
const subscription = this.get(id);
|
|
2490
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2491
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2492
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2493
|
+
if (fenced) return fenced;
|
|
2494
|
+
this.clearLease(subscription);
|
|
2495
|
+
subscription.token = null;
|
|
2496
|
+
subscription.holder = null;
|
|
2497
|
+
subscription.wake_id = null;
|
|
2498
|
+
subscription.wake_snapshot.clear();
|
|
2499
|
+
await this.triggerNextWakeIfPending(subscription);
|
|
2500
|
+
return { status: 204 };
|
|
2501
|
+
}
|
|
2502
|
+
serialize(subscription) {
|
|
2503
|
+
return {
|
|
2504
|
+
id: subscription.id,
|
|
2505
|
+
subscription_id: subscription.id,
|
|
2506
|
+
type: subscription.type,
|
|
2507
|
+
pattern: subscription.pattern,
|
|
2508
|
+
streams: this.streamInfos(subscription).map((stream) => ({
|
|
2509
|
+
path: stream.path,
|
|
2510
|
+
link_type: stream.link_type,
|
|
2511
|
+
acked_offset: stream.acked_offset
|
|
2512
|
+
})),
|
|
2513
|
+
webhook: subscription.webhook ? {
|
|
2514
|
+
url: subscription.webhook.url,
|
|
2515
|
+
signing: this.webhookSigningMetadata()
|
|
2516
|
+
} : void 0,
|
|
2517
|
+
wake_stream: subscription.wake_stream,
|
|
2518
|
+
lease_ttl_ms: subscription.lease_ttl_ms,
|
|
2519
|
+
created_at: subscription.created_at,
|
|
2520
|
+
status: subscription.status,
|
|
2521
|
+
description: subscription.description
|
|
2522
|
+
};
|
|
2523
|
+
}
|
|
2524
|
+
getWebhookJwks() {
|
|
2525
|
+
return getWebhookJwks();
|
|
2526
|
+
}
|
|
2527
|
+
shutdown() {
|
|
2528
|
+
this.isShuttingDown = true;
|
|
2529
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2530
|
+
this.clearLease(subscription);
|
|
2531
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2532
|
+
}
|
|
2533
|
+
this.subscriptions.clear();
|
|
2534
|
+
}
|
|
2535
|
+
async maybeWake(subscription, triggeredBy) {
|
|
2536
|
+
if (subscription.wake_id || subscription.holder) return;
|
|
2537
|
+
if (!this.hasPendingWork(subscription)) return;
|
|
2538
|
+
await this.createWake(subscription, triggeredBy);
|
|
2539
|
+
}
|
|
2540
|
+
async createWake(subscription, triggeredBy) {
|
|
2541
|
+
subscription.generation++;
|
|
2542
|
+
subscription.wake_id = generateWakeId();
|
|
2543
|
+
subscription.wake_snapshot = new Map(this.streamInfos(subscription).map((stream) => [stream.path, stream.tail_offset]));
|
|
2544
|
+
if (subscription.type === `webhook`) {
|
|
2545
|
+
subscription.token = generateCallbackToken(this.tokenSubject(subscription), subscription.generation);
|
|
2546
|
+
this.extendLease(subscription);
|
|
2547
|
+
this.deliverWebhook(subscription, [triggeredBy]);
|
|
2548
|
+
return;
|
|
2549
|
+
}
|
|
2550
|
+
await this.writePullWakeEvent(subscription, triggeredBy);
|
|
2551
|
+
}
|
|
2552
|
+
async deliverWebhook(subscription, triggeredBy) {
|
|
2553
|
+
if (!subscription.webhook || !subscription.wake_id || !subscription.token) return;
|
|
2554
|
+
const body = JSON.stringify({
|
|
2555
|
+
subscription_id: subscription.id,
|
|
2556
|
+
wake_id: subscription.wake_id,
|
|
2557
|
+
generation: subscription.generation,
|
|
2558
|
+
streams: this.streamInfos(subscription),
|
|
2559
|
+
callback_url: this.subscriptionActionUrl(subscription, `callback`),
|
|
2560
|
+
callback_token: subscription.token
|
|
2561
|
+
});
|
|
2562
|
+
const headers = {
|
|
2563
|
+
"content-type": `application/json`,
|
|
2564
|
+
"webhook-signature": signWebhookPayload(body)
|
|
2565
|
+
};
|
|
2566
|
+
try {
|
|
2567
|
+
const response = await fetch(subscription.webhook.url, {
|
|
2568
|
+
method: `POST`,
|
|
2569
|
+
headers,
|
|
2570
|
+
body
|
|
2571
|
+
});
|
|
2572
|
+
if (!response.ok) {
|
|
2573
|
+
this.scheduleWebhookRetry(subscription, triggeredBy);
|
|
2574
|
+
return;
|
|
2575
|
+
}
|
|
2576
|
+
subscription.status = `active`;
|
|
2577
|
+
subscription.retry_count = 0;
|
|
2578
|
+
subscription.next_attempt_at = null;
|
|
2579
|
+
let parsed = null;
|
|
2580
|
+
try {
|
|
2581
|
+
parsed = await response.json();
|
|
2582
|
+
} catch {
|
|
2583
|
+
parsed = null;
|
|
2584
|
+
}
|
|
2585
|
+
if (parsed?.done === true) {
|
|
2586
|
+
this.autoAckWakeSnapshot(subscription);
|
|
2587
|
+
this.clearLease(subscription);
|
|
2588
|
+
subscription.token = null;
|
|
2589
|
+
subscription.holder = null;
|
|
2590
|
+
subscription.wake_id = null;
|
|
2591
|
+
subscription.wake_snapshot.clear();
|
|
2592
|
+
await this.triggerNextWakeIfPending(subscription);
|
|
2593
|
+
}
|
|
2594
|
+
} catch (err) {
|
|
2595
|
+
serverLog.warn(`[subscriptions] webhook delivery failed:`, err);
|
|
2596
|
+
this.scheduleWebhookRetry(subscription, triggeredBy);
|
|
2597
|
+
}
|
|
2598
|
+
}
|
|
2599
|
+
scheduleWebhookRetry(subscription, triggeredBy) {
|
|
2600
|
+
if (this.isShuttingDown) return;
|
|
2601
|
+
subscription.retry_count++;
|
|
2602
|
+
const baseDelay = Math.min(1e3 * Math.pow(2, Math.max(0, subscription.retry_count - 1)), MAX_RETRY_DELAY_MS);
|
|
2603
|
+
const jitter = baseDelay * .2 * (Math.random() * 2 - 1);
|
|
2604
|
+
const delay = Math.max(0, Math.round(baseDelay + jitter));
|
|
2605
|
+
subscription.status = `failed`;
|
|
2606
|
+
subscription.next_attempt_at = Date.now() + delay;
|
|
2607
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2608
|
+
subscription.retry_timer = setTimeout(() => {
|
|
2609
|
+
subscription.retry_timer = null;
|
|
2610
|
+
this.deliverWebhook(subscription, triggeredBy);
|
|
2611
|
+
}, delay);
|
|
2612
|
+
}
|
|
2613
|
+
async writePullWakeEvent(subscription, streamPath) {
|
|
2614
|
+
if (!subscription.wake_stream) return;
|
|
2615
|
+
const wakeStream = toAbsoluteStreamPath(subscription.wake_stream);
|
|
2616
|
+
if (!this.streamStore.has(wakeStream)) {
|
|
2617
|
+
serverLog.warn(`[subscriptions] wake stream does not exist: ${wakeStream}`);
|
|
2618
|
+
return;
|
|
2619
|
+
}
|
|
2620
|
+
const event = {
|
|
2621
|
+
type: `wake`,
|
|
2622
|
+
subscription_id: subscription.id,
|
|
2623
|
+
stream: streamPath,
|
|
2624
|
+
generation: subscription.generation,
|
|
2625
|
+
ts: Date.now()
|
|
2626
|
+
};
|
|
2627
|
+
await Promise.resolve(this.streamStore.append(wakeStream, new TextEncoder().encode(JSON.stringify(event))));
|
|
2628
|
+
}
|
|
2629
|
+
autoAckWakeSnapshot(subscription) {
|
|
2630
|
+
for (const [stream, tail] of subscription.wake_snapshot) {
|
|
2631
|
+
const link = subscription.streams.get(stream);
|
|
2632
|
+
if (link) link.acked_offset = tail;
|
|
2633
|
+
}
|
|
2634
|
+
}
|
|
2635
|
+
applyAcks(subscription, request) {
|
|
2636
|
+
if (!request.acks) return null;
|
|
2637
|
+
for (const ack of request.acks) {
|
|
2638
|
+
const stream = normalizeRelativePath(ack.stream ?? ack.path ?? ``);
|
|
2639
|
+
const link = subscription.streams.get(stream);
|
|
2640
|
+
if (!stream || !link) return this.errorResponse(409, `INVALID_OFFSET`, `Ack references an unknown subscription stream`);
|
|
2641
|
+
if (ack.offset === BEFORE_FIRST_OFFSET) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset must not be -1`);
|
|
2642
|
+
if (compareOffsets(ack.offset, link.acked_offset) < 0) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset regresses the committed cursor`);
|
|
2643
|
+
if (compareOffsets(ack.offset, this.getTailOffset(stream)) > 0) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset is beyond stream tail`);
|
|
2644
|
+
}
|
|
2645
|
+
for (const ack of request.acks) {
|
|
2646
|
+
const stream = normalizeRelativePath(ack.stream ?? ack.path ?? ``);
|
|
2647
|
+
subscription.streams.get(stream).acked_offset = ack.offset;
|
|
2648
|
+
}
|
|
2649
|
+
return null;
|
|
2650
|
+
}
|
|
2651
|
+
validateWakeToken(subscription, token, request) {
|
|
2652
|
+
const tokenResult = validateCallbackToken(token, this.tokenSubject(subscription));
|
|
2653
|
+
if (!tokenResult.valid) return this.errorResponse(401, tokenResult.code, tokenResult.code === `TOKEN_EXPIRED` ? `Token expired` : `Token invalid`);
|
|
2654
|
+
if (tokenResult.epoch !== subscription.generation || request.generation !== subscription.generation || request.wake_id !== subscription.wake_id) return this.errorResponse(409, `FENCED`, `Wake generation is stale`);
|
|
2655
|
+
return null;
|
|
2656
|
+
}
|
|
2657
|
+
async triggerNextWakeIfPending(subscription) {
|
|
2658
|
+
if (!this.hasPendingWork(subscription)) return false;
|
|
2659
|
+
await this.createWake(subscription, this.firstPendingStream(subscription));
|
|
2660
|
+
return true;
|
|
2661
|
+
}
|
|
2662
|
+
hasPendingWork(subscription) {
|
|
2663
|
+
return this.streamInfos(subscription).some((stream) => stream.has_pending);
|
|
2664
|
+
}
|
|
2665
|
+
firstPendingStream(subscription) {
|
|
2666
|
+
return this.streamInfos(subscription).find((stream) => stream.has_pending)?.path ?? ``;
|
|
2667
|
+
}
|
|
2668
|
+
streamInfos(subscription) {
|
|
2669
|
+
return Array.from(subscription.streams.values()).map((link) => {
|
|
2670
|
+
const tail = this.getTailOffset(link.path);
|
|
2671
|
+
return {
|
|
2672
|
+
path: link.path,
|
|
2673
|
+
link_type: link.link_types.has(`explicit`) ? `explicit` : `glob`,
|
|
2674
|
+
acked_offset: link.acked_offset,
|
|
2675
|
+
tail_offset: tail,
|
|
2676
|
+
has_pending: compareOffsets(tail, link.acked_offset) > 0
|
|
2677
|
+
};
|
|
2678
|
+
});
|
|
2679
|
+
}
|
|
2680
|
+
linkStream(subscription, streamPath, linkType, ackedOffset) {
|
|
2681
|
+
const normalized = normalizeRelativePath(streamPath);
|
|
2682
|
+
const existing = subscription.streams.get(normalized);
|
|
2683
|
+
if (existing) {
|
|
2684
|
+
existing.link_types.add(linkType);
|
|
2685
|
+
return existing;
|
|
2686
|
+
}
|
|
2687
|
+
const link = {
|
|
2688
|
+
path: normalized,
|
|
2689
|
+
link_types: new Set([linkType]),
|
|
2690
|
+
acked_offset: ackedOffset
|
|
2691
|
+
};
|
|
2692
|
+
subscription.streams.set(normalized, link);
|
|
2693
|
+
return link;
|
|
2694
|
+
}
|
|
2695
|
+
listStreams() {
|
|
2696
|
+
return this.streamStore.list().map((path) => toStreamRelativePath(path)).filter((path) => path !== null);
|
|
2697
|
+
}
|
|
2698
|
+
getTailOffset(streamPath) {
|
|
2699
|
+
return this.streamStore.get(toAbsoluteStreamPath(streamPath))?.currentOffset ?? ZERO_OFFSET;
|
|
2700
|
+
}
|
|
2701
|
+
subscriptionActionUrl(subscription, action) {
|
|
2702
|
+
const url = new URL(`/v1/stream/__ds/subscriptions/${encodeURIComponent(subscription.id)}/${action}`, this.callbackBaseUrl);
|
|
2703
|
+
return url.toString();
|
|
2704
|
+
}
|
|
2705
|
+
webhookJwksUrl() {
|
|
2706
|
+
const url = new URL(`/v1/stream/__ds/jwks.json`, this.callbackBaseUrl);
|
|
2707
|
+
return url.toString();
|
|
2708
|
+
}
|
|
2709
|
+
webhookSigningMetadata() {
|
|
2710
|
+
return {
|
|
2711
|
+
alg: `ed25519`,
|
|
2712
|
+
kid: getWebhookSigningKeyId(),
|
|
2713
|
+
jwks_url: this.webhookJwksUrl()
|
|
2714
|
+
};
|
|
2715
|
+
}
|
|
2716
|
+
extendLease(subscription) {
|
|
2717
|
+
this.clearLease(subscription);
|
|
2718
|
+
subscription.lease_timer = setTimeout(() => {
|
|
2719
|
+
subscription.lease_timer = null;
|
|
2720
|
+
subscription.holder = null;
|
|
2721
|
+
subscription.token = null;
|
|
2722
|
+
subscription.wake_id = null;
|
|
2723
|
+
subscription.wake_snapshot.clear();
|
|
2724
|
+
this.triggerNextWakeIfPending(subscription);
|
|
2725
|
+
}, subscription.lease_ttl_ms);
|
|
2726
|
+
}
|
|
2727
|
+
clearLease(subscription) {
|
|
2728
|
+
if (subscription.lease_timer) {
|
|
2729
|
+
clearTimeout(subscription.lease_timer);
|
|
2730
|
+
subscription.lease_timer = null;
|
|
2731
|
+
}
|
|
2732
|
+
}
|
|
2733
|
+
tokenSubject(subscription) {
|
|
2734
|
+
return `subscription:${subscription.id}`;
|
|
2735
|
+
}
|
|
2736
|
+
errorResponse(status, code, message) {
|
|
2737
|
+
return {
|
|
2738
|
+
status,
|
|
2739
|
+
body: { error: {
|
|
2740
|
+
code,
|
|
2741
|
+
message
|
|
2742
|
+
} }
|
|
2743
|
+
};
|
|
2744
|
+
}
|
|
2745
|
+
};
|
|
2746
|
+
|
|
2747
|
+
//#endregion
|
|
2748
|
+
//#region src/subscription-routes.ts
|
|
2749
|
+
const RESERVED_CONTROL_PREFIX = `/v1/stream/__ds`;
|
|
2750
|
+
const SUBSCRIPTION_PREFIX = `${RESERVED_CONTROL_PREFIX}/subscriptions/`;
|
|
2751
|
+
const JWKS_PATH = `${RESERVED_CONTROL_PREFIX}/jwks.json`;
|
|
2752
|
+
const ERROR_STATUS = {
|
|
2753
|
+
INVALID_REQUEST: 400,
|
|
2754
|
+
SUBSCRIPTION_NOT_FOUND: 404,
|
|
2755
|
+
SUBSCRIPTION_ALREADY_EXISTS: 409,
|
|
2756
|
+
WEBHOOK_URL_REJECTED: 400,
|
|
2757
|
+
TOKEN_INVALID: 401,
|
|
2758
|
+
TOKEN_EXPIRED: 401,
|
|
2759
|
+
FENCED: 409,
|
|
2760
|
+
ALREADY_CLAIMED: 409,
|
|
2761
|
+
NO_PENDING_WORK: 409,
|
|
2762
|
+
INVALID_OFFSET: 409
|
|
2763
|
+
};
|
|
2764
|
+
var SubscriptionRoutes = class {
|
|
2765
|
+
manager;
|
|
2766
|
+
constructor(manager) {
|
|
2767
|
+
this.manager = manager;
|
|
2768
|
+
}
|
|
2769
|
+
async handleRequest(method, path, req, res) {
|
|
2770
|
+
if (path === JWKS_PATH) {
|
|
2771
|
+
this.handleJwks(method, res);
|
|
2772
|
+
return true;
|
|
2773
|
+
}
|
|
2774
|
+
const route = this.parseRoute(path);
|
|
2775
|
+
if (!route) {
|
|
2776
|
+
if (path === RESERVED_CONTROL_PREFIX || path.startsWith(`${RESERVED_CONTROL_PREFIX}/`)) {
|
|
2777
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Durable Streams control route not found`);
|
|
2778
|
+
return true;
|
|
2779
|
+
}
|
|
2780
|
+
return false;
|
|
2781
|
+
}
|
|
2782
|
+
try {
|
|
2783
|
+
switch (route.action) {
|
|
2784
|
+
case `base`:
|
|
2785
|
+
await this.handleBase(route, method, req, res);
|
|
2786
|
+
return true;
|
|
2787
|
+
case `streams`:
|
|
2788
|
+
await this.handleStreams(route, method, req, res);
|
|
2789
|
+
return true;
|
|
2790
|
+
case `stream`:
|
|
2791
|
+
this.handleStream(route, method, res);
|
|
2792
|
+
return true;
|
|
2793
|
+
case `callback`:
|
|
2794
|
+
await this.handleCallback(route, req, res);
|
|
2795
|
+
return true;
|
|
2796
|
+
case `claim`:
|
|
2797
|
+
await this.handleClaim(route, req, res);
|
|
2798
|
+
return true;
|
|
2799
|
+
case `ack`:
|
|
2800
|
+
await this.handleAck(route, req, res);
|
|
2801
|
+
return true;
|
|
2802
|
+
case `release`:
|
|
2803
|
+
await this.handleRelease(route, req, res);
|
|
2804
|
+
return true;
|
|
2805
|
+
}
|
|
2806
|
+
} catch (err) {
|
|
2807
|
+
if (err instanceof SyntaxError) {
|
|
2808
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `Invalid JSON body`);
|
|
2809
|
+
return true;
|
|
2810
|
+
}
|
|
2811
|
+
throw err;
|
|
2812
|
+
}
|
|
2813
|
+
}
|
|
2814
|
+
async handleBase(route, method, req, res) {
|
|
2815
|
+
if (method === `PUT`) {
|
|
2816
|
+
const parsed = await this.readJson(req);
|
|
2817
|
+
const input = this.parseCreateInput(parsed);
|
|
2818
|
+
if (`error` in input) {
|
|
2819
|
+
this.writeError(res, 400, `INVALID_REQUEST`, input.error);
|
|
2820
|
+
return;
|
|
2821
|
+
}
|
|
2822
|
+
const result = this.manager.createOrConfirm(route.subscriptionId, input.value);
|
|
2823
|
+
if (`error` in result) {
|
|
2824
|
+
this.writeError(res, ERROR_STATUS[result.error.code], result.error.code, result.error.message);
|
|
2825
|
+
return;
|
|
2826
|
+
}
|
|
2827
|
+
this.writeJson(res, result.created ? 201 : 200, this.manager.serialize(result.subscription));
|
|
2828
|
+
return;
|
|
2829
|
+
}
|
|
2830
|
+
if (method === `GET`) {
|
|
2831
|
+
const subscription = this.manager.get(route.subscriptionId);
|
|
2832
|
+
if (!subscription) {
|
|
2833
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2834
|
+
return;
|
|
2835
|
+
}
|
|
2836
|
+
this.writeJson(res, 200, this.manager.serialize(subscription));
|
|
2837
|
+
return;
|
|
2838
|
+
}
|
|
2839
|
+
if (method === `DELETE`) {
|
|
2840
|
+
this.manager.delete(route.subscriptionId);
|
|
2841
|
+
res.writeHead(204);
|
|
2842
|
+
res.end();
|
|
2843
|
+
return;
|
|
2844
|
+
}
|
|
2845
|
+
this.methodNotAllowed(res);
|
|
2846
|
+
}
|
|
2847
|
+
handleJwks(method, res) {
|
|
2848
|
+
if (method !== `GET`) {
|
|
2849
|
+
this.methodNotAllowed(res);
|
|
2850
|
+
return;
|
|
2851
|
+
}
|
|
2852
|
+
res.writeHead(200, {
|
|
2853
|
+
"content-type": `application/jwk-set+json`,
|
|
2854
|
+
"cache-control": `public, max-age=300`
|
|
2855
|
+
});
|
|
2856
|
+
res.end(JSON.stringify(this.manager.getWebhookJwks()));
|
|
2857
|
+
}
|
|
2858
|
+
async handleStreams(route, method, req, res) {
|
|
2859
|
+
if (method !== `POST`) {
|
|
2860
|
+
this.methodNotAllowed(res);
|
|
2861
|
+
return;
|
|
2862
|
+
}
|
|
2863
|
+
const parsed = await this.readJson(req);
|
|
2864
|
+
const streams = parsed.streams;
|
|
2865
|
+
if (!Array.isArray(streams) || streams.some((stream) => typeof stream !== `string` || stream.length === 0)) {
|
|
2866
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `streams must be a non-empty string array`);
|
|
2867
|
+
return;
|
|
2868
|
+
}
|
|
2869
|
+
const ok = this.manager.addExplicitStreams(route.subscriptionId, streams.map(normalizeRelativePath));
|
|
2870
|
+
if (!ok) {
|
|
2871
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2872
|
+
return;
|
|
2873
|
+
}
|
|
2874
|
+
res.writeHead(204);
|
|
2875
|
+
res.end();
|
|
2876
|
+
}
|
|
2877
|
+
handleStream(route, method, res) {
|
|
2878
|
+
if (method !== `DELETE`) {
|
|
2879
|
+
this.methodNotAllowed(res);
|
|
2880
|
+
return;
|
|
2881
|
+
}
|
|
2882
|
+
const ok = this.manager.removeExplicitStream(route.subscriptionId, route.streamPath ?? ``);
|
|
2883
|
+
if (!ok) {
|
|
2884
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2885
|
+
return;
|
|
2886
|
+
}
|
|
2887
|
+
res.writeHead(204);
|
|
2888
|
+
res.end();
|
|
2889
|
+
}
|
|
2890
|
+
async handleCallback(route, req, res) {
|
|
2891
|
+
const token = this.readBearerToken(req);
|
|
2892
|
+
if (!token) {
|
|
2893
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2894
|
+
return;
|
|
2895
|
+
}
|
|
2896
|
+
const body = await this.readJson(req);
|
|
2897
|
+
const result = await this.manager.handleWebhookCallback(route.subscriptionId, token, body);
|
|
2898
|
+
this.writeManagerResult(res, result);
|
|
2899
|
+
}
|
|
2900
|
+
async handleClaim(route, req, res) {
|
|
2901
|
+
const parsed = await this.readJson(req);
|
|
2902
|
+
const worker = parsed.worker;
|
|
2903
|
+
if (typeof worker !== `string` || worker.length === 0) {
|
|
2904
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `worker must be a non-empty string`);
|
|
2905
|
+
return;
|
|
2906
|
+
}
|
|
2907
|
+
const result = await this.manager.claim(route.subscriptionId, worker);
|
|
2908
|
+
this.writeManagerResult(res, result);
|
|
2909
|
+
}
|
|
2910
|
+
async handleAck(route, req, res) {
|
|
2911
|
+
const token = this.readBearerToken(req);
|
|
2912
|
+
if (!token) {
|
|
2913
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2914
|
+
return;
|
|
2915
|
+
}
|
|
2916
|
+
const body = await this.readJson(req);
|
|
2917
|
+
const result = await this.manager.ack(route.subscriptionId, token, body);
|
|
2918
|
+
this.writeManagerResult(res, result);
|
|
2919
|
+
}
|
|
2920
|
+
async handleRelease(route, req, res) {
|
|
2921
|
+
const token = this.readBearerToken(req);
|
|
2922
|
+
if (!token) {
|
|
2923
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2924
|
+
return;
|
|
2925
|
+
}
|
|
2926
|
+
const body = await this.readJson(req);
|
|
2927
|
+
const result = await this.manager.release(route.subscriptionId, token, body);
|
|
2928
|
+
this.writeManagerResult(res, result);
|
|
2929
|
+
}
|
|
2930
|
+
parseCreateInput(value) {
|
|
2931
|
+
if (!value || typeof value !== `object`) return { error: `Request body must be a JSON object` };
|
|
2932
|
+
const payload = value;
|
|
2933
|
+
if (payload.type !== `webhook` && payload.type !== `pull-wake`) return { error: `type must be "webhook" or "pull-wake"` };
|
|
2934
|
+
const type = payload.type;
|
|
2935
|
+
const pattern = typeof payload.pattern === `string` && payload.pattern.length > 0 ? normalizeRelativePath(payload.pattern) : void 0;
|
|
2936
|
+
const streams = Array.isArray(payload.streams) && payload.streams.length > 0 ? payload.streams.map((stream) => typeof stream === `string` ? normalizeRelativePath(stream) : null) : [];
|
|
2937
|
+
if (streams.some((stream) => stream === null)) return { error: `streams must contain only strings` };
|
|
2938
|
+
if (!pattern && streams.length === 0) return { error: `At least one of pattern or streams is required` };
|
|
2939
|
+
const leaseTtl = payload.lease_ttl_ms === void 0 ? DEFAULT_LEASE_TTL_MS : payload.lease_ttl_ms;
|
|
2940
|
+
if (typeof leaseTtl !== `number` || !Number.isInteger(leaseTtl) || leaseTtl < MIN_LEASE_TTL_MS || leaseTtl > MAX_LEASE_TTL_MS) return { error: `lease_ttl_ms must be an integer from 1000 to 600000` };
|
|
2941
|
+
let webhook;
|
|
2942
|
+
if (type === `webhook`) {
|
|
2943
|
+
const rawWebhook = payload.webhook;
|
|
2944
|
+
if (!rawWebhook || typeof rawWebhook !== `object`) return { error: `webhook subscriptions require webhook.url` };
|
|
2945
|
+
const url = rawWebhook.url;
|
|
2946
|
+
if (typeof url !== `string` || url.length === 0) return { error: `webhook subscriptions require webhook.url` };
|
|
2947
|
+
webhook = { url };
|
|
2948
|
+
}
|
|
2949
|
+
const wakeStream = typeof payload.wake_stream === `string` && payload.wake_stream.length > 0 ? normalizeRelativePath(payload.wake_stream) : void 0;
|
|
2950
|
+
if (type === `pull-wake` && !wakeStream) return { error: `pull-wake subscriptions require wake_stream` };
|
|
2951
|
+
return { value: {
|
|
2952
|
+
type,
|
|
2953
|
+
pattern,
|
|
2954
|
+
streams,
|
|
2955
|
+
webhook,
|
|
2956
|
+
wake_stream: wakeStream,
|
|
2957
|
+
lease_ttl_ms: leaseTtl,
|
|
2958
|
+
description: typeof payload.description === `string` ? payload.description : void 0
|
|
2959
|
+
} };
|
|
2960
|
+
}
|
|
2961
|
+
parseRoute(path) {
|
|
2962
|
+
if (!path.startsWith(SUBSCRIPTION_PREFIX)) return null;
|
|
2963
|
+
const rest = path.slice(SUBSCRIPTION_PREFIX.length);
|
|
2964
|
+
const parts = rest.split(`/`);
|
|
2965
|
+
const subscriptionId = parts[0] ? decodeURIComponent(parts[0]) : ``;
|
|
2966
|
+
if (!subscriptionId) return null;
|
|
2967
|
+
const tail = parts.slice(1);
|
|
2968
|
+
if (tail.length === 0) return {
|
|
2969
|
+
subscriptionId,
|
|
2970
|
+
action: `base`
|
|
2971
|
+
};
|
|
2972
|
+
if (tail[0] === `streams` && tail.length === 1) return {
|
|
2973
|
+
subscriptionId,
|
|
2974
|
+
action: `streams`
|
|
2975
|
+
};
|
|
2976
|
+
if (tail[0] === `streams` && tail.length > 1) return {
|
|
2977
|
+
subscriptionId,
|
|
2978
|
+
action: `stream`,
|
|
2979
|
+
streamPath: normalizeRelativePath(decodeURIComponent(tail.slice(1).join(`/`)))
|
|
2980
|
+
};
|
|
2981
|
+
if (tail.length === 1 && [
|
|
2982
|
+
`callback`,
|
|
2983
|
+
`claim`,
|
|
2984
|
+
`ack`,
|
|
2985
|
+
`release`
|
|
2986
|
+
].includes(tail[0])) return {
|
|
2987
|
+
subscriptionId,
|
|
2988
|
+
action: tail[0]
|
|
2989
|
+
};
|
|
2990
|
+
return null;
|
|
2991
|
+
}
|
|
2992
|
+
readBearerToken(req) {
|
|
2993
|
+
const authHeader = req.headers.authorization;
|
|
2994
|
+
if (!authHeader || !authHeader.startsWith(`Bearer `)) return null;
|
|
2995
|
+
return authHeader.slice(`Bearer `.length);
|
|
2996
|
+
}
|
|
2997
|
+
async readJson(req) {
|
|
2998
|
+
const chunks = [];
|
|
2999
|
+
for await (const chunk of req) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
3000
|
+
const raw = Buffer.concat(chunks).toString(`utf8`);
|
|
3001
|
+
return raw.length > 0 ? JSON.parse(raw) : {};
|
|
3002
|
+
}
|
|
3003
|
+
writeManagerResult(res, result) {
|
|
3004
|
+
if (result.status === 204) {
|
|
3005
|
+
res.writeHead(204);
|
|
3006
|
+
res.end();
|
|
3007
|
+
return;
|
|
3008
|
+
}
|
|
3009
|
+
this.writeJson(res, result.status, result.body ?? {});
|
|
3010
|
+
}
|
|
3011
|
+
writeJson(res, status, body) {
|
|
3012
|
+
res.writeHead(status, { "content-type": `application/json` });
|
|
3013
|
+
res.end(JSON.stringify(body));
|
|
3014
|
+
}
|
|
3015
|
+
writeError(res, status, code, message) {
|
|
3016
|
+
this.writeJson(res, status, { error: {
|
|
3017
|
+
code,
|
|
3018
|
+
message
|
|
3019
|
+
} });
|
|
3020
|
+
}
|
|
3021
|
+
methodNotAllowed(res) {
|
|
3022
|
+
res.writeHead(405, { "content-type": `text/plain` });
|
|
3023
|
+
res.end(`Method not allowed`);
|
|
3024
|
+
}
|
|
3025
|
+
};
|
|
3026
|
+
|
|
1966
3027
|
//#endregion
|
|
1967
3028
|
//#region src/server.ts
|
|
1968
|
-
const STREAM_OFFSET_HEADER = `Stream-Next-Offset`;
|
|
1969
|
-
const STREAM_CURSOR_HEADER = `Stream-Cursor`;
|
|
1970
|
-
const STREAM_UP_TO_DATE_HEADER = `Stream-Up-To-Date`;
|
|
1971
|
-
const STREAM_SEQ_HEADER = `Stream-Seq`;
|
|
1972
|
-
const STREAM_TTL_HEADER = `Stream-TTL`;
|
|
1973
|
-
const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`;
|
|
1974
3029
|
const STREAM_SSE_DATA_ENCODING_HEADER = `Stream-SSE-Data-Encoding`;
|
|
1975
|
-
const PRODUCER_ID_HEADER = `Producer-Id`;
|
|
1976
|
-
const PRODUCER_EPOCH_HEADER = `Producer-Epoch`;
|
|
1977
|
-
const PRODUCER_SEQ_HEADER = `Producer-Seq`;
|
|
1978
|
-
const PRODUCER_EXPECTED_SEQ_HEADER = `Producer-Expected-Seq`;
|
|
1979
|
-
const PRODUCER_RECEIVED_SEQ_HEADER = `Producer-Received-Seq`;
|
|
1980
|
-
const SSE_OFFSET_FIELD = `streamNextOffset`;
|
|
1981
|
-
const SSE_CURSOR_FIELD = `streamCursor`;
|
|
1982
3030
|
const SSE_UP_TO_DATE_FIELD = `upToDate`;
|
|
1983
|
-
const SSE_CLOSED_FIELD = `streamClosed`;
|
|
1984
|
-
const STREAM_CLOSED_HEADER = `Stream-Closed`;
|
|
1985
3031
|
const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`;
|
|
1986
3032
|
const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`;
|
|
1987
|
-
const OFFSET_QUERY_PARAM = `offset`;
|
|
1988
|
-
const LIVE_QUERY_PARAM = `live`;
|
|
1989
|
-
const CURSOR_QUERY_PARAM = `cursor`;
|
|
1990
3033
|
/**
|
|
1991
3034
|
* Encode data for SSE format.
|
|
1992
3035
|
* Per SSE spec, each line in the payload needs its own "data:" prefix.
|
|
@@ -2040,6 +3083,8 @@ var DurableStreamTestServer = class {
|
|
|
2040
3083
|
isShuttingDown = false;
|
|
2041
3084
|
/** Injected faults for testing retry/resilience */
|
|
2042
3085
|
injectedFaults = new Map();
|
|
3086
|
+
subscriptionManager = null;
|
|
3087
|
+
subscriptionRoutes = null;
|
|
2043
3088
|
constructor(options = {}) {
|
|
2044
3089
|
if (options.dataDir) this.store = new FileBackedStreamStore({ dataDir: options.dataDir });
|
|
2045
3090
|
else this.store = new StreamStore();
|
|
@@ -2054,7 +3099,8 @@ var DurableStreamTestServer = class {
|
|
|
2054
3099
|
cursorOptions: {
|
|
2055
3100
|
intervalSeconds: options.cursorIntervalSeconds,
|
|
2056
3101
|
epoch: options.cursorEpoch
|
|
2057
|
-
}
|
|
3102
|
+
},
|
|
3103
|
+
webhooks: options.webhooks ?? false
|
|
2058
3104
|
};
|
|
2059
3105
|
}
|
|
2060
3106
|
/**
|
|
@@ -2065,7 +3111,7 @@ var DurableStreamTestServer = class {
|
|
|
2065
3111
|
return new Promise((resolve, reject) => {
|
|
2066
3112
|
this.server = (0, node_http.createServer)((req, res) => {
|
|
2067
3113
|
this.handleRequest(req, res).catch((err) => {
|
|
2068
|
-
|
|
3114
|
+
serverLog.error(`Request error:`, err);
|
|
2069
3115
|
if (!res.headersSent) {
|
|
2070
3116
|
res.writeHead(500, { "content-type": `text/plain` });
|
|
2071
3117
|
res.end(`Internal server error`);
|
|
@@ -2077,6 +3123,12 @@ var DurableStreamTestServer = class {
|
|
|
2077
3123
|
const addr = this.server.address();
|
|
2078
3124
|
if (typeof addr === `string`) this._url = addr;
|
|
2079
3125
|
else if (addr) this._url = `http://${this.options.host}:${addr.port}`;
|
|
3126
|
+
this.subscriptionManager = new SubscriptionManager({
|
|
3127
|
+
callbackBaseUrl: this._url,
|
|
3128
|
+
streamStore: this.store,
|
|
3129
|
+
webhooksEnabled: this.options.webhooks
|
|
3130
|
+
});
|
|
3131
|
+
this.subscriptionRoutes = new SubscriptionRoutes(this.subscriptionManager);
|
|
2080
3132
|
resolve(this._url);
|
|
2081
3133
|
});
|
|
2082
3134
|
});
|
|
@@ -2087,6 +3139,11 @@ var DurableStreamTestServer = class {
|
|
|
2087
3139
|
async stop() {
|
|
2088
3140
|
if (!this.server) return;
|
|
2089
3141
|
this.isShuttingDown = true;
|
|
3142
|
+
if (this.subscriptionManager) {
|
|
3143
|
+
this.subscriptionManager.shutdown();
|
|
3144
|
+
this.subscriptionManager = null;
|
|
3145
|
+
this.subscriptionRoutes = null;
|
|
3146
|
+
}
|
|
2090
3147
|
if (`cancelAllWaits` in this.store) this.store.cancelAllWaits();
|
|
2091
3148
|
for (const res of this.activeSSEResponses) res.end();
|
|
2092
3149
|
this.activeSSEResponses.clear();
|
|
@@ -2227,6 +3284,10 @@ var DurableStreamTestServer = class {
|
|
|
2227
3284
|
}
|
|
2228
3285
|
if (fault.truncateBodyBytes !== void 0 || fault.corruptBody || fault.injectSseEvent) res._injectedFault = fault;
|
|
2229
3286
|
}
|
|
3287
|
+
if (this.subscriptionRoutes && method) {
|
|
3288
|
+
const handled = await this.subscriptionRoutes.handleRequest(method, path, req, res);
|
|
3289
|
+
if (handled) return;
|
|
3290
|
+
}
|
|
2230
3291
|
try {
|
|
2231
3292
|
switch (method) {
|
|
2232
3293
|
case `PUT`:
|
|
@@ -2282,13 +3343,13 @@ var DurableStreamTestServer = class {
|
|
|
2282
3343
|
*/
|
|
2283
3344
|
async handleCreate(path, req, res) {
|
|
2284
3345
|
let contentType = req.headers[`content-type`];
|
|
2285
|
-
if (!contentType || contentType.trim() === `` || !/^[\w-]+\/[\w-]+/.test(contentType)) contentType = `application/octet-stream`;
|
|
2286
|
-
const ttlHeader = req.headers[STREAM_TTL_HEADER.toLowerCase()];
|
|
2287
|
-
const expiresAtHeader = req.headers[STREAM_EXPIRES_AT_HEADER.toLowerCase()];
|
|
2288
|
-
const closedHeader = req.headers[STREAM_CLOSED_HEADER.toLowerCase()];
|
|
2289
|
-
const createClosed = closedHeader === `true`;
|
|
2290
3346
|
const forkedFromHeader = req.headers[STREAM_FORKED_FROM_HEADER.toLowerCase()];
|
|
2291
3347
|
const forkOffsetHeader = req.headers[STREAM_FORK_OFFSET_HEADER.toLowerCase()];
|
|
3348
|
+
if (!contentType || contentType.trim() === `` || !/^[\w-]+\/[\w-]+/.test(contentType)) contentType = forkedFromHeader ? void 0 : `application/octet-stream`;
|
|
3349
|
+
const ttlHeader = req.headers[__durable_streams_client.STREAM_TTL_HEADER.toLowerCase()];
|
|
3350
|
+
const expiresAtHeader = req.headers[__durable_streams_client.STREAM_EXPIRES_AT_HEADER.toLowerCase()];
|
|
3351
|
+
const closedHeader = req.headers[__durable_streams_client.STREAM_CLOSED_HEADER.toLowerCase()];
|
|
3352
|
+
const createClosed = closedHeader === `true`;
|
|
2292
3353
|
if (ttlHeader && expiresAtHeader) {
|
|
2293
3354
|
res.writeHead(400, { "content-type": `text/plain` });
|
|
2294
3355
|
res.end(`Cannot specify both Stream-TTL and Stream-Expires-At`);
|
|
@@ -2363,18 +3424,20 @@ var DurableStreamTestServer = class {
|
|
|
2363
3424
|
throw err;
|
|
2364
3425
|
}
|
|
2365
3426
|
const stream = this.store.get(path);
|
|
3427
|
+
const resolvedContentType = stream.contentType ?? contentType ?? `application/octet-stream`;
|
|
2366
3428
|
if (isNew && this.options.onStreamCreated) await Promise.resolve(this.options.onStreamCreated({
|
|
2367
3429
|
type: `created`,
|
|
2368
3430
|
path,
|
|
2369
|
-
contentType:
|
|
3431
|
+
contentType: resolvedContentType,
|
|
2370
3432
|
timestamp: Date.now()
|
|
2371
3433
|
}));
|
|
3434
|
+
if (isNew && body.length > 0) await this.notifyStreamAppend(path);
|
|
2372
3435
|
const headers = {
|
|
2373
|
-
"content-type":
|
|
2374
|
-
[STREAM_OFFSET_HEADER]: stream.currentOffset
|
|
3436
|
+
"content-type": resolvedContentType,
|
|
3437
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream.currentOffset
|
|
2375
3438
|
};
|
|
2376
3439
|
if (isNew) headers[`location`] = `${this._url}${path}`;
|
|
2377
|
-
if (stream.closed) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
3440
|
+
if (stream.closed) headers[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2378
3441
|
res.writeHead(isNew ? 201 : 200, headers);
|
|
2379
3442
|
res.end();
|
|
2380
3443
|
}
|
|
@@ -2394,13 +3457,13 @@ var DurableStreamTestServer = class {
|
|
|
2394
3457
|
return;
|
|
2395
3458
|
}
|
|
2396
3459
|
const headers = {
|
|
2397
|
-
[STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
3460
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
2398
3461
|
"cache-control": `no-store`
|
|
2399
3462
|
};
|
|
2400
3463
|
if (stream.contentType) headers[`content-type`] = stream.contentType;
|
|
2401
|
-
if (stream.closed) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
2402
|
-
if (stream.ttlSeconds !== void 0) headers[STREAM_TTL_HEADER] = String(stream.ttlSeconds);
|
|
2403
|
-
if (stream.expiresAt) headers[STREAM_EXPIRES_AT_HEADER] = stream.expiresAt;
|
|
3464
|
+
if (stream.closed) headers[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
3465
|
+
if (stream.ttlSeconds !== void 0) headers[__durable_streams_client.STREAM_TTL_HEADER] = String(stream.ttlSeconds);
|
|
3466
|
+
if (stream.expiresAt) headers[__durable_streams_client.STREAM_EXPIRES_AT_HEADER] = stream.expiresAt;
|
|
2404
3467
|
const closedSuffix = stream.closed ? `:c` : ``;
|
|
2405
3468
|
headers[`etag`] = `"${Buffer.from(path).toString(`base64`)}:-1:${stream.currentOffset}${closedSuffix}"`;
|
|
2406
3469
|
res.writeHead(200, headers);
|
|
@@ -2421,16 +3484,16 @@ var DurableStreamTestServer = class {
|
|
|
2421
3484
|
res.end(`Stream is gone`);
|
|
2422
3485
|
return;
|
|
2423
3486
|
}
|
|
2424
|
-
const offset = url.searchParams.get(OFFSET_QUERY_PARAM) ?? void 0;
|
|
2425
|
-
const live = url.searchParams.get(LIVE_QUERY_PARAM);
|
|
2426
|
-
const cursor = url.searchParams.get(CURSOR_QUERY_PARAM) ?? void 0;
|
|
3487
|
+
const offset = url.searchParams.get(__durable_streams_client.OFFSET_QUERY_PARAM) ?? void 0;
|
|
3488
|
+
const live = url.searchParams.get(__durable_streams_client.LIVE_QUERY_PARAM);
|
|
3489
|
+
const cursor = url.searchParams.get(__durable_streams_client.CURSOR_QUERY_PARAM) ?? void 0;
|
|
2427
3490
|
if (offset !== void 0) {
|
|
2428
3491
|
if (offset === ``) {
|
|
2429
3492
|
res.writeHead(400, { "content-type": `text/plain` });
|
|
2430
3493
|
res.end(`Empty offset parameter`);
|
|
2431
3494
|
return;
|
|
2432
3495
|
}
|
|
2433
|
-
const allOffsets = url.searchParams.getAll(OFFSET_QUERY_PARAM);
|
|
3496
|
+
const allOffsets = url.searchParams.getAll(__durable_streams_client.OFFSET_QUERY_PARAM);
|
|
2434
3497
|
if (allOffsets.length > 1) {
|
|
2435
3498
|
res.writeHead(400, { "content-type": `text/plain` });
|
|
2436
3499
|
res.end(`Multiple offset parameters not allowed`);
|
|
@@ -2462,12 +3525,12 @@ var DurableStreamTestServer = class {
|
|
|
2462
3525
|
const effectiveOffset = offset === `now` ? stream.currentOffset : offset;
|
|
2463
3526
|
if (offset === `now` && live !== `long-poll`) {
|
|
2464
3527
|
const headers$1 = {
|
|
2465
|
-
[STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
2466
|
-
[STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
3528
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
3529
|
+
[__durable_streams_client.STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
2467
3530
|
[`cache-control`]: `no-store`
|
|
2468
3531
|
};
|
|
2469
3532
|
if (stream.contentType) headers$1[`content-type`] = stream.contentType;
|
|
2470
|
-
if (stream.closed) headers$1[STREAM_CLOSED_HEADER] = `true`;
|
|
3533
|
+
if (stream.closed) headers$1[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2471
3534
|
const isJsonMode = stream.contentType?.includes(`application/json`);
|
|
2472
3535
|
const responseBody = isJsonMode ? `[]` : ``;
|
|
2473
3536
|
res.writeHead(200, headers$1);
|
|
@@ -2480,9 +3543,9 @@ var DurableStreamTestServer = class {
|
|
|
2480
3543
|
if (live === `long-poll` && clientIsCaughtUp && messages.length === 0) {
|
|
2481
3544
|
if (stream.closed) {
|
|
2482
3545
|
res.writeHead(204, {
|
|
2483
|
-
[STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
2484
|
-
[STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
2485
|
-
[STREAM_CLOSED_HEADER]: `true`
|
|
3546
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
3547
|
+
[__durable_streams_client.STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
3548
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`
|
|
2486
3549
|
});
|
|
2487
3550
|
res.end();
|
|
2488
3551
|
return;
|
|
@@ -2492,10 +3555,10 @@ var DurableStreamTestServer = class {
|
|
|
2492
3555
|
if (result.streamClosed) {
|
|
2493
3556
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2494
3557
|
res.writeHead(204, {
|
|
2495
|
-
[STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
|
|
2496
|
-
[STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
2497
|
-
[STREAM_CURSOR_HEADER]: responseCursor,
|
|
2498
|
-
[STREAM_CLOSED_HEADER]: `true`
|
|
3558
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
|
|
3559
|
+
[__durable_streams_client.STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
3560
|
+
[__durable_streams_client.STREAM_CURSOR_HEADER]: responseCursor,
|
|
3561
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`
|
|
2499
3562
|
});
|
|
2500
3563
|
res.end();
|
|
2501
3564
|
return;
|
|
@@ -2504,11 +3567,11 @@ var DurableStreamTestServer = class {
|
|
|
2504
3567
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2505
3568
|
const currentStream$1 = this.store.get(path);
|
|
2506
3569
|
const timeoutHeaders = {
|
|
2507
|
-
[STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
|
|
2508
|
-
[STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
2509
|
-
[STREAM_CURSOR_HEADER]: responseCursor
|
|
3570
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
|
|
3571
|
+
[__durable_streams_client.STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
3572
|
+
[__durable_streams_client.STREAM_CURSOR_HEADER]: responseCursor
|
|
2510
3573
|
};
|
|
2511
|
-
if (currentStream$1?.closed) timeoutHeaders[STREAM_CLOSED_HEADER] = `true`;
|
|
3574
|
+
if (currentStream$1?.closed) timeoutHeaders[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2512
3575
|
res.writeHead(204, timeoutHeaders);
|
|
2513
3576
|
res.end();
|
|
2514
3577
|
return;
|
|
@@ -2520,12 +3583,12 @@ var DurableStreamTestServer = class {
|
|
|
2520
3583
|
if (stream.contentType) headers[`content-type`] = stream.contentType;
|
|
2521
3584
|
const lastMessage = messages[messages.length - 1];
|
|
2522
3585
|
const responseOffset = lastMessage?.offset ?? stream.currentOffset;
|
|
2523
|
-
headers[STREAM_OFFSET_HEADER] = responseOffset;
|
|
2524
|
-
if (live === `long-poll`) headers[STREAM_CURSOR_HEADER] = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2525
|
-
if (upToDate) headers[STREAM_UP_TO_DATE_HEADER] = `true`;
|
|
3586
|
+
headers[__durable_streams_client.STREAM_OFFSET_HEADER] = responseOffset;
|
|
3587
|
+
if (live === `long-poll`) headers[__durable_streams_client.STREAM_CURSOR_HEADER] = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
3588
|
+
if (upToDate) headers[__durable_streams_client.STREAM_UP_TO_DATE_HEADER] = `true`;
|
|
2526
3589
|
const currentStream = this.store.get(path);
|
|
2527
3590
|
const clientAtTail = responseOffset === currentStream?.currentOffset;
|
|
2528
|
-
if (currentStream?.closed && clientAtTail && upToDate) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
3591
|
+
if (currentStream?.closed && clientAtTail && upToDate) headers[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2529
3592
|
const startOffset = offset ?? `-1`;
|
|
2530
3593
|
const closedSuffix = currentStream?.closed && clientAtTail && upToDate ? `:c` : ``;
|
|
2531
3594
|
const etag = `"${Buffer.from(path).toString(`base64`)}:${startOffset}:${responseOffset}${closedSuffix}"`;
|
|
@@ -2598,10 +3661,10 @@ var DurableStreamTestServer = class {
|
|
|
2598
3661
|
const streamIsClosed = currentStream?.closed ?? false;
|
|
2599
3662
|
const clientAtTail = controlOffset === currentStream.currentOffset;
|
|
2600
3663
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2601
|
-
const controlData = { [SSE_OFFSET_FIELD]: controlOffset };
|
|
2602
|
-
if (streamIsClosed && clientAtTail) controlData[SSE_CLOSED_FIELD] = true;
|
|
3664
|
+
const controlData = { [__durable_streams_client.SSE_OFFSET_FIELD]: controlOffset };
|
|
3665
|
+
if (streamIsClosed && clientAtTail) controlData[__durable_streams_client.SSE_CLOSED_FIELD] = true;
|
|
2603
3666
|
else {
|
|
2604
|
-
controlData[SSE_CURSOR_FIELD] = responseCursor;
|
|
3667
|
+
controlData[__durable_streams_client.SSE_CURSOR_FIELD] = responseCursor;
|
|
2605
3668
|
if (upToDate) controlData[SSE_UP_TO_DATE_FIELD] = true;
|
|
2606
3669
|
}
|
|
2607
3670
|
res.write(`event: control\n`);
|
|
@@ -2611,8 +3674,8 @@ var DurableStreamTestServer = class {
|
|
|
2611
3674
|
if (upToDate) {
|
|
2612
3675
|
if (currentStream?.closed) {
|
|
2613
3676
|
const finalControlData = {
|
|
2614
|
-
[SSE_OFFSET_FIELD]: currentOffset,
|
|
2615
|
-
[SSE_CLOSED_FIELD]: true
|
|
3677
|
+
[__durable_streams_client.SSE_OFFSET_FIELD]: currentOffset,
|
|
3678
|
+
[__durable_streams_client.SSE_CLOSED_FIELD]: true
|
|
2616
3679
|
};
|
|
2617
3680
|
res.write(`event: control\n`);
|
|
2618
3681
|
res.write(encodeSSEData(JSON.stringify(finalControlData)));
|
|
@@ -2621,10 +3684,10 @@ var DurableStreamTestServer = class {
|
|
|
2621
3684
|
const result = await this.store.waitForMessages(path, currentOffset, this.options.longPollTimeout);
|
|
2622
3685
|
this.store.touchAccess(path);
|
|
2623
3686
|
if (this.isShuttingDown || !isConnected) break;
|
|
2624
|
-
if (result.streamClosed) {
|
|
3687
|
+
if (result.streamClosed && result.messages.length === 0) {
|
|
2625
3688
|
const finalControlData = {
|
|
2626
|
-
[SSE_OFFSET_FIELD]: currentOffset,
|
|
2627
|
-
[SSE_CLOSED_FIELD]: true
|
|
3689
|
+
[__durable_streams_client.SSE_OFFSET_FIELD]: currentOffset,
|
|
3690
|
+
[__durable_streams_client.SSE_CLOSED_FIELD]: true
|
|
2628
3691
|
};
|
|
2629
3692
|
res.write(`event: control\n`);
|
|
2630
3693
|
res.write(encodeSSEData(JSON.stringify(finalControlData)));
|
|
@@ -2635,16 +3698,16 @@ var DurableStreamTestServer = class {
|
|
|
2635
3698
|
const streamAfterWait = this.store.get(path);
|
|
2636
3699
|
if (streamAfterWait?.closed) {
|
|
2637
3700
|
const closedControlData = {
|
|
2638
|
-
[SSE_OFFSET_FIELD]: currentOffset,
|
|
2639
|
-
[SSE_CLOSED_FIELD]: true
|
|
3701
|
+
[__durable_streams_client.SSE_OFFSET_FIELD]: currentOffset,
|
|
3702
|
+
[__durable_streams_client.SSE_CLOSED_FIELD]: true
|
|
2640
3703
|
};
|
|
2641
3704
|
res.write(`event: control\n`);
|
|
2642
3705
|
res.write(encodeSSEData(JSON.stringify(closedControlData)));
|
|
2643
3706
|
break;
|
|
2644
3707
|
}
|
|
2645
3708
|
const keepAliveData = {
|
|
2646
|
-
[SSE_OFFSET_FIELD]: currentOffset,
|
|
2647
|
-
[SSE_CURSOR_FIELD]: keepAliveCursor,
|
|
3709
|
+
[__durable_streams_client.SSE_OFFSET_FIELD]: currentOffset,
|
|
3710
|
+
[__durable_streams_client.SSE_CURSOR_FIELD]: keepAliveCursor,
|
|
2648
3711
|
[SSE_UP_TO_DATE_FIELD]: true
|
|
2649
3712
|
};
|
|
2650
3713
|
res.write(`event: control\n` + encodeSSEData(JSON.stringify(keepAliveData)));
|
|
@@ -2659,12 +3722,12 @@ var DurableStreamTestServer = class {
|
|
|
2659
3722
|
*/
|
|
2660
3723
|
async handleAppend(path, req, res) {
|
|
2661
3724
|
const contentType = req.headers[`content-type`];
|
|
2662
|
-
const seq = req.headers[STREAM_SEQ_HEADER.toLowerCase()];
|
|
2663
|
-
const closedHeader = req.headers[STREAM_CLOSED_HEADER.toLowerCase()];
|
|
3725
|
+
const seq = req.headers[__durable_streams_client.STREAM_SEQ_HEADER.toLowerCase()];
|
|
3726
|
+
const closedHeader = req.headers[__durable_streams_client.STREAM_CLOSED_HEADER.toLowerCase()];
|
|
2664
3727
|
const closeStream = closedHeader === `true`;
|
|
2665
|
-
const producerId = req.headers[PRODUCER_ID_HEADER.toLowerCase()];
|
|
2666
|
-
const producerEpochStr = req.headers[PRODUCER_EPOCH_HEADER.toLowerCase()];
|
|
2667
|
-
const producerSeqStr = req.headers[PRODUCER_SEQ_HEADER.toLowerCase()];
|
|
3728
|
+
const producerId = req.headers[__durable_streams_client.PRODUCER_ID_HEADER.toLowerCase()];
|
|
3729
|
+
const producerEpochStr = req.headers[__durable_streams_client.PRODUCER_EPOCH_HEADER.toLowerCase()];
|
|
3730
|
+
const producerSeqStr = req.headers[__durable_streams_client.PRODUCER_SEQ_HEADER.toLowerCase()];
|
|
2668
3731
|
const hasProducerHeaders = producerId !== void 0 || producerEpochStr !== void 0 || producerSeqStr !== void 0;
|
|
2669
3732
|
const hasAllProducerHeaders = producerId !== void 0 && producerEpochStr !== void 0 && producerSeqStr !== void 0;
|
|
2670
3733
|
if (hasProducerHeaders && !hasAllProducerHeaders) {
|
|
@@ -2719,10 +3782,10 @@ var DurableStreamTestServer = class {
|
|
|
2719
3782
|
}
|
|
2720
3783
|
if (closeResult$1.producerResult?.status === `duplicate`) {
|
|
2721
3784
|
res.writeHead(204, {
|
|
2722
|
-
[STREAM_OFFSET_HEADER]: closeResult$1.finalOffset,
|
|
2723
|
-
[STREAM_CLOSED_HEADER]: `true`,
|
|
2724
|
-
[PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
2725
|
-
[PRODUCER_SEQ_HEADER]: closeResult$1.producerResult.lastSeq.toString()
|
|
3785
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: closeResult$1.finalOffset,
|
|
3786
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`,
|
|
3787
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
3788
|
+
[__durable_streams_client.PRODUCER_SEQ_HEADER]: closeResult$1.producerResult.lastSeq.toString()
|
|
2726
3789
|
});
|
|
2727
3790
|
res.end();
|
|
2728
3791
|
return;
|
|
@@ -2730,7 +3793,7 @@ var DurableStreamTestServer = class {
|
|
|
2730
3793
|
if (closeResult$1.producerResult?.status === `stale_epoch`) {
|
|
2731
3794
|
res.writeHead(403, {
|
|
2732
3795
|
"content-type": `text/plain`,
|
|
2733
|
-
[PRODUCER_EPOCH_HEADER]: closeResult$1.producerResult.currentEpoch.toString()
|
|
3796
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: closeResult$1.producerResult.currentEpoch.toString()
|
|
2734
3797
|
});
|
|
2735
3798
|
res.end(`Stale producer epoch`);
|
|
2736
3799
|
return;
|
|
@@ -2743,8 +3806,8 @@ var DurableStreamTestServer = class {
|
|
|
2743
3806
|
if (closeResult$1.producerResult?.status === `sequence_gap`) {
|
|
2744
3807
|
res.writeHead(409, {
|
|
2745
3808
|
"content-type": `text/plain`,
|
|
2746
|
-
[PRODUCER_EXPECTED_SEQ_HEADER]: closeResult$1.producerResult.expectedSeq.toString(),
|
|
2747
|
-
[PRODUCER_RECEIVED_SEQ_HEADER]: closeResult$1.producerResult.receivedSeq.toString()
|
|
3809
|
+
[__durable_streams_client.PRODUCER_EXPECTED_SEQ_HEADER]: closeResult$1.producerResult.expectedSeq.toString(),
|
|
3810
|
+
[__durable_streams_client.PRODUCER_RECEIVED_SEQ_HEADER]: closeResult$1.producerResult.receivedSeq.toString()
|
|
2748
3811
|
});
|
|
2749
3812
|
res.end(`Producer sequence gap`);
|
|
2750
3813
|
return;
|
|
@@ -2753,17 +3816,17 @@ var DurableStreamTestServer = class {
|
|
|
2753
3816
|
const stream = this.store.get(path);
|
|
2754
3817
|
res.writeHead(409, {
|
|
2755
3818
|
"content-type": `text/plain`,
|
|
2756
|
-
[STREAM_CLOSED_HEADER]: `true`,
|
|
2757
|
-
[STREAM_OFFSET_HEADER]: stream?.currentOffset ?? ``
|
|
3819
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`,
|
|
3820
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream?.currentOffset ?? ``
|
|
2758
3821
|
});
|
|
2759
3822
|
res.end(`Stream is closed`);
|
|
2760
3823
|
return;
|
|
2761
3824
|
}
|
|
2762
3825
|
res.writeHead(204, {
|
|
2763
|
-
[STREAM_OFFSET_HEADER]: closeResult$1.finalOffset,
|
|
2764
|
-
[STREAM_CLOSED_HEADER]: `true`,
|
|
2765
|
-
[PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
2766
|
-
[PRODUCER_SEQ_HEADER]: producerSeq.toString()
|
|
3826
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: closeResult$1.finalOffset,
|
|
3827
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`,
|
|
3828
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
3829
|
+
[__durable_streams_client.PRODUCER_SEQ_HEADER]: producerSeq.toString()
|
|
2767
3830
|
});
|
|
2768
3831
|
res.end();
|
|
2769
3832
|
return;
|
|
@@ -2775,8 +3838,8 @@ var DurableStreamTestServer = class {
|
|
|
2775
3838
|
return;
|
|
2776
3839
|
}
|
|
2777
3840
|
res.writeHead(204, {
|
|
2778
|
-
[STREAM_OFFSET_HEADER]: closeResult.finalOffset,
|
|
2779
|
-
[STREAM_CLOSED_HEADER]: `true`
|
|
3841
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: closeResult.finalOffset,
|
|
3842
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`
|
|
2780
3843
|
});
|
|
2781
3844
|
res.end();
|
|
2782
3845
|
return;
|
|
@@ -2809,10 +3872,10 @@ var DurableStreamTestServer = class {
|
|
|
2809
3872
|
if (producerResult?.status === `duplicate`) {
|
|
2810
3873
|
const stream = this.store.get(path);
|
|
2811
3874
|
res.writeHead(204, {
|
|
2812
|
-
[STREAM_OFFSET_HEADER]: stream?.currentOffset ?? ``,
|
|
2813
|
-
[STREAM_CLOSED_HEADER]: `true`,
|
|
2814
|
-
[PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
2815
|
-
[PRODUCER_SEQ_HEADER]: producerResult.lastSeq.toString()
|
|
3875
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream?.currentOffset ?? ``,
|
|
3876
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`,
|
|
3877
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
3878
|
+
[__durable_streams_client.PRODUCER_SEQ_HEADER]: producerResult.lastSeq.toString()
|
|
2816
3879
|
});
|
|
2817
3880
|
res.end();
|
|
2818
3881
|
return;
|
|
@@ -2820,29 +3883,30 @@ var DurableStreamTestServer = class {
|
|
|
2820
3883
|
const closedStream = this.store.get(path);
|
|
2821
3884
|
res.writeHead(409, {
|
|
2822
3885
|
"content-type": `text/plain`,
|
|
2823
|
-
[STREAM_CLOSED_HEADER]: `true`,
|
|
2824
|
-
[STREAM_OFFSET_HEADER]: closedStream?.currentOffset ?? ``
|
|
3886
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`,
|
|
3887
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: closedStream?.currentOffset ?? ``
|
|
2825
3888
|
});
|
|
2826
3889
|
res.end(`Stream is closed`);
|
|
2827
3890
|
return;
|
|
2828
3891
|
}
|
|
2829
3892
|
if (!producerResult || producerResult.status === `accepted`) {
|
|
2830
|
-
const responseHeaders$1 = { [STREAM_OFFSET_HEADER]: message$1.offset };
|
|
2831
|
-
if (producerEpoch !== void 0) responseHeaders$1[PRODUCER_EPOCH_HEADER] = producerEpoch.toString();
|
|
2832
|
-
if (producerSeq !== void 0) responseHeaders$1[PRODUCER_SEQ_HEADER] = producerSeq.toString();
|
|
2833
|
-
if (streamClosed) responseHeaders$1[STREAM_CLOSED_HEADER] = `true`;
|
|
3893
|
+
const responseHeaders$1 = { [__durable_streams_client.STREAM_OFFSET_HEADER]: message$1.offset };
|
|
3894
|
+
if (producerEpoch !== void 0) responseHeaders$1[__durable_streams_client.PRODUCER_EPOCH_HEADER] = producerEpoch.toString();
|
|
3895
|
+
if (producerSeq !== void 0) responseHeaders$1[__durable_streams_client.PRODUCER_SEQ_HEADER] = producerSeq.toString();
|
|
3896
|
+
if (streamClosed) responseHeaders$1[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2834
3897
|
const statusCode = producerId !== void 0 ? 200 : 204;
|
|
2835
3898
|
res.writeHead(statusCode, responseHeaders$1);
|
|
2836
3899
|
res.end();
|
|
3900
|
+
await this.notifyStreamAppend(path);
|
|
2837
3901
|
return;
|
|
2838
3902
|
}
|
|
2839
3903
|
switch (producerResult.status) {
|
|
2840
3904
|
case `duplicate`: {
|
|
2841
3905
|
const dupHeaders = {
|
|
2842
|
-
[PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
2843
|
-
[PRODUCER_SEQ_HEADER]: producerResult.lastSeq.toString()
|
|
3906
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
3907
|
+
[__durable_streams_client.PRODUCER_SEQ_HEADER]: producerResult.lastSeq.toString()
|
|
2844
3908
|
};
|
|
2845
|
-
if (streamClosed) dupHeaders[STREAM_CLOSED_HEADER] = `true`;
|
|
3909
|
+
if (streamClosed) dupHeaders[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2846
3910
|
res.writeHead(204, dupHeaders);
|
|
2847
3911
|
res.end();
|
|
2848
3912
|
return;
|
|
@@ -2850,7 +3914,7 @@ var DurableStreamTestServer = class {
|
|
|
2850
3914
|
case `stale_epoch`: {
|
|
2851
3915
|
res.writeHead(403, {
|
|
2852
3916
|
"content-type": `text/plain`,
|
|
2853
|
-
[PRODUCER_EPOCH_HEADER]: producerResult.currentEpoch.toString()
|
|
3917
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: producerResult.currentEpoch.toString()
|
|
2854
3918
|
});
|
|
2855
3919
|
res.end(`Stale producer epoch`);
|
|
2856
3920
|
return;
|
|
@@ -2862,18 +3926,27 @@ var DurableStreamTestServer = class {
|
|
|
2862
3926
|
case `sequence_gap`:
|
|
2863
3927
|
res.writeHead(409, {
|
|
2864
3928
|
"content-type": `text/plain`,
|
|
2865
|
-
[PRODUCER_EXPECTED_SEQ_HEADER]: producerResult.expectedSeq.toString(),
|
|
2866
|
-
[PRODUCER_RECEIVED_SEQ_HEADER]: producerResult.receivedSeq.toString()
|
|
3929
|
+
[__durable_streams_client.PRODUCER_EXPECTED_SEQ_HEADER]: producerResult.expectedSeq.toString(),
|
|
3930
|
+
[__durable_streams_client.PRODUCER_RECEIVED_SEQ_HEADER]: producerResult.receivedSeq.toString()
|
|
2867
3931
|
});
|
|
2868
3932
|
res.end(`Producer sequence gap`);
|
|
2869
3933
|
return;
|
|
2870
3934
|
}
|
|
2871
3935
|
}
|
|
2872
3936
|
const message = result;
|
|
2873
|
-
const responseHeaders = { [STREAM_OFFSET_HEADER]: message.offset };
|
|
2874
|
-
if (closeStream) responseHeaders[STREAM_CLOSED_HEADER] = `true`;
|
|
3937
|
+
const responseHeaders = { [__durable_streams_client.STREAM_OFFSET_HEADER]: message.offset };
|
|
3938
|
+
if (closeStream) responseHeaders[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2875
3939
|
res.writeHead(204, responseHeaders);
|
|
2876
3940
|
res.end();
|
|
3941
|
+
await this.notifyStreamAppend(path);
|
|
3942
|
+
}
|
|
3943
|
+
async notifyStreamAppend(path) {
|
|
3944
|
+
if (!this.subscriptionManager) return;
|
|
3945
|
+
try {
|
|
3946
|
+
await this.subscriptionManager.onStreamAppend(path);
|
|
3947
|
+
} catch (err) {
|
|
3948
|
+
serverLog.error(`[server] subscription append hook failed:`, err);
|
|
3949
|
+
}
|
|
2877
3950
|
}
|
|
2878
3951
|
/**
|
|
2879
3952
|
* Handle DELETE - delete stream
|
|
@@ -2896,6 +3969,7 @@ var DurableStreamTestServer = class {
|
|
|
2896
3969
|
path,
|
|
2897
3970
|
timestamp: Date.now()
|
|
2898
3971
|
}));
|
|
3972
|
+
if (this.subscriptionManager) this.subscriptionManager.onStreamDeleted(path);
|
|
2899
3973
|
res.writeHead(204);
|
|
2900
3974
|
res.end();
|
|
2901
3975
|
}
|
|
@@ -3030,9 +4104,13 @@ exports.DEFAULT_CURSOR_INTERVAL_SECONDS = DEFAULT_CURSOR_INTERVAL_SECONDS
|
|
|
3030
4104
|
exports.DurableStreamTestServer = DurableStreamTestServer
|
|
3031
4105
|
exports.FileBackedStreamStore = FileBackedStreamStore
|
|
3032
4106
|
exports.StreamStore = StreamStore
|
|
4107
|
+
exports.SubscriptionManager = SubscriptionManager
|
|
4108
|
+
exports.SubscriptionRoutes = SubscriptionRoutes
|
|
3033
4109
|
exports.calculateCursor = calculateCursor
|
|
3034
4110
|
exports.createRegistryHooks = createRegistryHooks
|
|
3035
4111
|
exports.decodeStreamPath = decodeStreamPath
|
|
3036
4112
|
exports.encodeStreamPath = encodeStreamPath
|
|
3037
4113
|
exports.generateResponseCursor = generateResponseCursor
|
|
3038
|
-
exports.
|
|
4114
|
+
exports.globMatch = globMatch
|
|
4115
|
+
exports.handleCursorCollision = handleCursorCollision
|
|
4116
|
+
exports.validateWebhookUrl = validateWebhookUrl
|