@durable-streams/server 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1297 -260
- package/dist/index.d.cts +236 -2
- package/dist/index.d.ts +236 -2
- package/dist/index.js +1344 -312
- package/package.json +4 -4
- package/src/crypto.ts +217 -0
- package/src/file-store.ts +187 -144
- package/src/glob.ts +70 -0
- package/src/index.ts +14 -0
- package/src/log.ts +56 -0
- package/src/server.ts +75 -26
- package/src/store.ts +59 -7
- package/src/subscription-manager.ts +882 -0
- package/src/subscription-routes.ts +504 -0
- package/src/subscription-types.ts +80 -0
- package/src/types.ts +8 -0
package/dist/index.cjs
CHANGED
|
@@ -24,13 +24,13 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
24
24
|
//#endregion
|
|
25
25
|
const node_http = __toESM(require("node:http"));
|
|
26
26
|
const node_zlib = __toESM(require("node:zlib"));
|
|
27
|
+
const __durable_streams_client = __toESM(require("@durable-streams/client"));
|
|
27
28
|
const node_fs = __toESM(require("node:fs"));
|
|
28
29
|
const node_path = __toESM(require("node:path"));
|
|
29
30
|
const node_crypto = __toESM(require("node:crypto"));
|
|
30
31
|
const lmdb = __toESM(require("lmdb"));
|
|
31
32
|
const __neophi_sieve_cache = __toESM(require("@neophi/sieve-cache"));
|
|
32
|
-
const
|
|
33
|
-
const __durable_streams_client = __toESM(require("@durable-streams/client"));
|
|
33
|
+
const node_net = __toESM(require("node:net"));
|
|
34
34
|
const __durable_streams_state = __toESM(require("@durable-streams/state"));
|
|
35
35
|
|
|
36
36
|
//#region src/store.ts
|
|
@@ -73,17 +73,35 @@ function processJsonAppend(data, isInitialCreate = false) {
|
|
|
73
73
|
} else result = JSON.stringify(parsed) + `,`;
|
|
74
74
|
return new TextEncoder().encode(result);
|
|
75
75
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
* Strips trailing comma before wrapping.
|
|
79
|
-
*/
|
|
80
|
-
function formatJsonResponse(data) {
|
|
81
|
-
if (data.length === 0) return new TextEncoder().encode(`[]`);
|
|
82
|
-
let text = new TextDecoder().decode(data);
|
|
83
|
-
text = text.trimEnd();
|
|
76
|
+
function decodeStoredJsonMessage(data) {
|
|
77
|
+
let text = new TextDecoder().decode(data).trimEnd();
|
|
84
78
|
if (text.endsWith(`,`)) text = text.slice(0, -1);
|
|
85
|
-
|
|
86
|
-
|
|
79
|
+
return text;
|
|
80
|
+
}
|
|
81
|
+
function enrichJsonValueWithOffset(parsed, offset) {
|
|
82
|
+
if (!parsed || typeof parsed !== `object` || Array.isArray(parsed)) return JSON.stringify(parsed);
|
|
83
|
+
const candidate = parsed;
|
|
84
|
+
const headers = candidate.headers;
|
|
85
|
+
if (!headers || typeof headers !== `object`) return JSON.stringify(parsed);
|
|
86
|
+
const isStateChange = typeof headers.operation === `string`;
|
|
87
|
+
const isStateControl = typeof headers.control === `string`;
|
|
88
|
+
if (!isStateChange && !isStateControl) return JSON.stringify(parsed);
|
|
89
|
+
return JSON.stringify({
|
|
90
|
+
...candidate,
|
|
91
|
+
headers: {
|
|
92
|
+
...headers,
|
|
93
|
+
offset
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
function formatJsonMessages(messages) {
|
|
98
|
+
if (messages.length === 0) return new TextEncoder().encode(`[]`);
|
|
99
|
+
const items = messages.flatMap((message) => {
|
|
100
|
+
const rawFragment = decodeStoredJsonMessage(message.data);
|
|
101
|
+
const parsed = JSON.parse(`[${rawFragment}]`);
|
|
102
|
+
return parsed.map((value) => enrichJsonValueWithOffset(value, message.offset));
|
|
103
|
+
});
|
|
104
|
+
return new TextEncoder().encode(`[${items.join(`,`)}]`);
|
|
87
105
|
}
|
|
88
106
|
var StreamStore = class {
|
|
89
107
|
streams = new Map();
|
|
@@ -621,6 +639,7 @@ var StreamStore = class {
|
|
|
621
639
|
formatResponse(path, messages) {
|
|
622
640
|
const stream = this.getIfNotExpired(path);
|
|
623
641
|
if (!stream) throw new Error(`Stream not found: ${path}`);
|
|
642
|
+
if (normalizeContentType(stream.contentType) === `application/json`) return formatJsonMessages(messages);
|
|
624
643
|
const totalSize = messages.reduce((sum, m) => sum + m.data.length, 0);
|
|
625
644
|
const concatenated = new Uint8Array(totalSize);
|
|
626
645
|
let offset = 0;
|
|
@@ -628,7 +647,6 @@ var StreamStore = class {
|
|
|
628
647
|
concatenated.set(msg.data, offset);
|
|
629
648
|
offset += msg.data.length;
|
|
630
649
|
}
|
|
631
|
-
if (normalizeContentType(stream.contentType) === `application/json`) return formatJsonResponse(concatenated);
|
|
632
650
|
return concatenated;
|
|
633
651
|
}
|
|
634
652
|
/**
|
|
@@ -728,7 +746,8 @@ var StreamStore = class {
|
|
|
728
746
|
const parts = stream.currentOffset.split(`_`).map(Number);
|
|
729
747
|
const readSeq = parts[0];
|
|
730
748
|
const byteOffset = parts[1];
|
|
731
|
-
const
|
|
749
|
+
const FRAME_OVERHEAD = 5;
|
|
750
|
+
const newByteOffset = byteOffset + FRAME_OVERHEAD + processedData.length;
|
|
732
751
|
const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
733
752
|
const message = {
|
|
734
753
|
data: processedData,
|
|
@@ -772,6 +791,48 @@ var StreamStore = class {
|
|
|
772
791
|
}
|
|
773
792
|
};
|
|
774
793
|
|
|
794
|
+
//#endregion
|
|
795
|
+
//#region src/log.ts
|
|
796
|
+
const streamsLogFile = process.env.STREAMS_LOG_FILE;
|
|
797
|
+
async function appendLogLine(line) {
|
|
798
|
+
if (!streamsLogFile) return;
|
|
799
|
+
const fs = await import(`node:fs/promises`);
|
|
800
|
+
const path = await import(`node:path`);
|
|
801
|
+
await fs.mkdir(path.dirname(streamsLogFile), { recursive: true });
|
|
802
|
+
await fs.appendFile(streamsLogFile, `${line}\n`);
|
|
803
|
+
}
|
|
804
|
+
function serializeArg(arg) {
|
|
805
|
+
if (arg instanceof Error) return arg.stack ?? arg.message;
|
|
806
|
+
if (typeof arg === `string`) return arg;
|
|
807
|
+
try {
|
|
808
|
+
return JSON.stringify(arg);
|
|
809
|
+
} catch {
|
|
810
|
+
return String(arg);
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
function write(level, args) {
|
|
814
|
+
const line = args.map(serializeArg).join(` `);
|
|
815
|
+
const formatted = `[${level}] ${line}`;
|
|
816
|
+
if (level === `error`) console.error(formatted);
|
|
817
|
+
else if (level === `warn`) console.warn(formatted);
|
|
818
|
+
else console.info(formatted);
|
|
819
|
+
appendLogLine(formatted).catch(() => void 0);
|
|
820
|
+
}
|
|
821
|
+
const serverLog = {
|
|
822
|
+
info(...args) {
|
|
823
|
+
write(`info`, args);
|
|
824
|
+
},
|
|
825
|
+
warn(...args) {
|
|
826
|
+
write(`warn`, args);
|
|
827
|
+
},
|
|
828
|
+
error(...args) {
|
|
829
|
+
write(`error`, args);
|
|
830
|
+
},
|
|
831
|
+
event(obj, msg) {
|
|
832
|
+
write(`info`, [msg, obj]);
|
|
833
|
+
}
|
|
834
|
+
};
|
|
835
|
+
|
|
775
836
|
//#endregion
|
|
776
837
|
//#region src/path-encoding.ts
|
|
777
838
|
const MAX_ENCODED_LENGTH = 200;
|
|
@@ -808,82 +869,6 @@ function decodeStreamPath(encoded) {
|
|
|
808
869
|
return Buffer.from(padded, `base64`).toString(`utf-8`);
|
|
809
870
|
}
|
|
810
871
|
|
|
811
|
-
//#endregion
|
|
812
|
-
//#region src/file-manager.ts
|
|
813
|
-
var StreamFileManager = class {
|
|
814
|
-
constructor(streamsDir) {
|
|
815
|
-
this.streamsDir = streamsDir;
|
|
816
|
-
}
|
|
817
|
-
/**
|
|
818
|
-
* Create a directory for a new stream and initialize the first segment file.
|
|
819
|
-
* Returns the absolute path to the stream directory.
|
|
820
|
-
*/
|
|
821
|
-
async createStreamDirectory(streamPath) {
|
|
822
|
-
const encoded = encodeStreamPath(streamPath);
|
|
823
|
-
const dir = node_path.join(this.streamsDir, encoded);
|
|
824
|
-
await node_fs_promises.mkdir(dir, { recursive: true });
|
|
825
|
-
const segmentPath = node_path.join(dir, `segment_00000.log`);
|
|
826
|
-
await node_fs_promises.writeFile(segmentPath, ``);
|
|
827
|
-
return dir;
|
|
828
|
-
}
|
|
829
|
-
/**
|
|
830
|
-
* Delete a stream directory and all its contents.
|
|
831
|
-
*/
|
|
832
|
-
async deleteStreamDirectory(streamPath) {
|
|
833
|
-
const encoded = encodeStreamPath(streamPath);
|
|
834
|
-
const dir = node_path.join(this.streamsDir, encoded);
|
|
835
|
-
await node_fs_promises.rm(dir, {
|
|
836
|
-
recursive: true,
|
|
837
|
-
force: true
|
|
838
|
-
});
|
|
839
|
-
}
|
|
840
|
-
/**
|
|
841
|
-
* Delete a directory by its exact name (used for unique directory names).
|
|
842
|
-
*/
|
|
843
|
-
async deleteDirectoryByName(directoryName) {
|
|
844
|
-
const dir = node_path.join(this.streamsDir, directoryName);
|
|
845
|
-
await node_fs_promises.rm(dir, {
|
|
846
|
-
recursive: true,
|
|
847
|
-
force: true
|
|
848
|
-
});
|
|
849
|
-
}
|
|
850
|
-
/**
|
|
851
|
-
* Get the absolute path to a stream's directory.
|
|
852
|
-
* Returns null if the directory doesn't exist.
|
|
853
|
-
*/
|
|
854
|
-
async getStreamDirectory(streamPath) {
|
|
855
|
-
const encoded = encodeStreamPath(streamPath);
|
|
856
|
-
const dir = node_path.join(this.streamsDir, encoded);
|
|
857
|
-
try {
|
|
858
|
-
await node_fs_promises.access(dir);
|
|
859
|
-
return dir;
|
|
860
|
-
} catch {
|
|
861
|
-
return null;
|
|
862
|
-
}
|
|
863
|
-
}
|
|
864
|
-
/**
|
|
865
|
-
* List all stream paths by scanning the streams directory.
|
|
866
|
-
*/
|
|
867
|
-
async listStreamPaths() {
|
|
868
|
-
try {
|
|
869
|
-
const entries = await node_fs_promises.readdir(this.streamsDir, { withFileTypes: true });
|
|
870
|
-
return entries.filter((e) => e.isDirectory()).map((e) => decodeStreamPath(e.name));
|
|
871
|
-
} catch {
|
|
872
|
-
return [];
|
|
873
|
-
}
|
|
874
|
-
}
|
|
875
|
-
/**
|
|
876
|
-
* Get the path to a segment file within a stream directory.
|
|
877
|
-
*
|
|
878
|
-
* @param streamDir - Absolute path to the stream directory
|
|
879
|
-
* @param index - Segment index (0-based)
|
|
880
|
-
*/
|
|
881
|
-
getSegmentPath(streamDir, index) {
|
|
882
|
-
const paddedIndex = String(index).padStart(5, `0`);
|
|
883
|
-
return node_path.join(streamDir, `segment_${paddedIndex}.log`);
|
|
884
|
-
}
|
|
885
|
-
};
|
|
886
|
-
|
|
887
872
|
//#endregion
|
|
888
873
|
//#region src/file-store.ts
|
|
889
874
|
var FileHandlePool = class {
|
|
@@ -891,7 +876,7 @@ var FileHandlePool = class {
|
|
|
891
876
|
constructor(maxSize) {
|
|
892
877
|
this.cache = new __neophi_sieve_cache.SieveCache(maxSize, { evictHook: (_key, handle) => {
|
|
893
878
|
this.closeHandle(handle).catch((err) => {
|
|
894
|
-
|
|
879
|
+
serverLog.error(`[FileHandlePool] Error closing evicted handle:`, err);
|
|
895
880
|
});
|
|
896
881
|
} });
|
|
897
882
|
}
|
|
@@ -899,41 +884,70 @@ var FileHandlePool = class {
|
|
|
899
884
|
let handle = this.cache.get(filePath);
|
|
900
885
|
if (!handle) {
|
|
901
886
|
const stream = node_fs.createWriteStream(filePath, { flags: `a` });
|
|
902
|
-
handle = {
|
|
887
|
+
handle = {
|
|
888
|
+
stream,
|
|
889
|
+
syncLeader: null
|
|
890
|
+
};
|
|
903
891
|
this.cache.set(filePath, handle);
|
|
904
892
|
}
|
|
905
893
|
return handle.stream;
|
|
906
894
|
}
|
|
907
895
|
/**
|
|
896
|
+
* Open a write stream eagerly so the first write does not pay the lazy
|
|
897
|
+
* `open()` stall. Resolves once the underlying fd is ready.
|
|
898
|
+
*/
|
|
899
|
+
async openWriteStream(filePath) {
|
|
900
|
+
const stream = this.getWriteStream(filePath);
|
|
901
|
+
const fd = stream.fd;
|
|
902
|
+
if (typeof fd === `number`) return stream;
|
|
903
|
+
await new Promise((resolve, reject) => {
|
|
904
|
+
stream.once(`open`, () => resolve());
|
|
905
|
+
stream.once(`error`, (err) => reject(err));
|
|
906
|
+
});
|
|
907
|
+
return stream;
|
|
908
|
+
}
|
|
909
|
+
/**
|
|
908
910
|
* Flush a specific file to disk immediately.
|
|
909
|
-
*
|
|
911
|
+
* Concurrent callers on the same fd share one in-flight fdatasync: the
|
|
912
|
+
* first caller issues the syscall, later arrivals during that window wait
|
|
913
|
+
* for it to finish and then issue a fresh syscall (because their writes
|
|
914
|
+
* may have landed after the in-flight syscall started). This preserves
|
|
915
|
+
* durability without adding scheduling latency.
|
|
910
916
|
*/
|
|
911
|
-
|
|
917
|
+
fsyncFile(filePath) {
|
|
912
918
|
const handle = this.cache.get(filePath);
|
|
913
|
-
if (!handle) return;
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
else resolve();
|
|
922
|
-
});
|
|
923
|
-
};
|
|
924
|
-
const onError = (err) => {
|
|
925
|
-
handle.stream.off(`open`, onOpen);
|
|
926
|
-
reject(err);
|
|
927
|
-
};
|
|
928
|
-
handle.stream.once(`open`, onOpen);
|
|
929
|
-
handle.stream.once(`error`, onError);
|
|
930
|
-
return;
|
|
931
|
-
}
|
|
932
|
-
node_fs.fdatasync(fd, (err) => {
|
|
933
|
-
if (err) reject(err);
|
|
934
|
-
else resolve();
|
|
935
|
-
});
|
|
919
|
+
if (!handle) return Promise.reject(new Error(`[FileHandlePool] Cannot fsync: handle not found for ${filePath}`));
|
|
920
|
+
const existing = handle.syncLeader;
|
|
921
|
+
if (existing && existing.scheduled) return existing.promise;
|
|
922
|
+
let resolveFn;
|
|
923
|
+
let rejectFn;
|
|
924
|
+
const promise = new Promise((res, rej) => {
|
|
925
|
+
resolveFn = res;
|
|
926
|
+
rejectFn = rej;
|
|
936
927
|
});
|
|
928
|
+
const leader = {
|
|
929
|
+
promise,
|
|
930
|
+
scheduled: true
|
|
931
|
+
};
|
|
932
|
+
handle.syncLeader = leader;
|
|
933
|
+
const runSyscall = (fd$1) => {
|
|
934
|
+
leader.scheduled = false;
|
|
935
|
+
node_fs.fdatasync(fd$1, (err) => {
|
|
936
|
+
if (handle.syncLeader === leader) handle.syncLeader = null;
|
|
937
|
+
if (err) rejectFn(err);
|
|
938
|
+
else resolveFn();
|
|
939
|
+
});
|
|
940
|
+
};
|
|
941
|
+
const fd = handle.stream.fd;
|
|
942
|
+
if (typeof fd === `number`) runSyscall(fd);
|
|
943
|
+
else {
|
|
944
|
+
handle.stream.once(`open`, (openedFd) => runSyscall(openedFd));
|
|
945
|
+
handle.stream.once(`error`, (err) => {
|
|
946
|
+
if (handle.syncLeader === leader) handle.syncLeader = null;
|
|
947
|
+
rejectFn(err);
|
|
948
|
+
});
|
|
949
|
+
}
|
|
950
|
+
return promise;
|
|
937
951
|
}
|
|
938
952
|
async closeAll() {
|
|
939
953
|
const promises = [];
|
|
@@ -969,13 +983,15 @@ function generateUniqueDirectoryName(streamPath) {
|
|
|
969
983
|
const random = (0, node_crypto.randomBytes)(4).toString(`hex`);
|
|
970
984
|
return `${encoded}~${timestamp}~${random}`;
|
|
971
985
|
}
|
|
986
|
+
function segmentFile(dataDir, dirName) {
|
|
987
|
+
return node_path.join(dataDir, `streams`, `${dirName}.log`);
|
|
988
|
+
}
|
|
972
989
|
/**
|
|
973
990
|
* File-backed implementation of StreamStore.
|
|
974
991
|
* Maintains the same interface as the in-memory StreamStore for drop-in compatibility.
|
|
975
992
|
*/
|
|
976
993
|
var FileBackedStreamStore = class {
|
|
977
994
|
db;
|
|
978
|
-
fileManager;
|
|
979
995
|
fileHandlePool;
|
|
980
996
|
pendingLongPolls = [];
|
|
981
997
|
dataDir;
|
|
@@ -995,9 +1011,12 @@ var FileBackedStreamStore = class {
|
|
|
995
1011
|
this.dataDir = options.dataDir;
|
|
996
1012
|
this.db = (0, lmdb.open)({
|
|
997
1013
|
path: node_path.join(this.dataDir, `metadata.lmdb`),
|
|
998
|
-
compression: true
|
|
1014
|
+
compression: true,
|
|
1015
|
+
noMemInit: true,
|
|
1016
|
+
cache: true,
|
|
1017
|
+
sharedStructuresKey: Symbol.for(`structures`)
|
|
999
1018
|
});
|
|
1000
|
-
|
|
1019
|
+
node_fs.mkdirSync(node_path.join(this.dataDir, `streams`), { recursive: true });
|
|
1001
1020
|
const maxFileHandles = options.maxFileHandles ?? 100;
|
|
1002
1021
|
this.fileHandlePool = new FileHandlePool(maxFileHandles);
|
|
1003
1022
|
this.recover();
|
|
@@ -1007,7 +1026,7 @@ var FileBackedStreamStore = class {
|
|
|
1007
1026
|
* Validates that LMDB metadata matches actual file contents and reconciles any mismatches.
|
|
1008
1027
|
*/
|
|
1009
1028
|
recover() {
|
|
1010
|
-
|
|
1029
|
+
serverLog.info(`[FileBackedStreamStore] Starting recovery...`);
|
|
1011
1030
|
let recovered = 0;
|
|
1012
1031
|
let reconciled = 0;
|
|
1013
1032
|
let errors = 0;
|
|
@@ -1020,9 +1039,9 @@ var FileBackedStreamStore = class {
|
|
|
1020
1039
|
if (typeof key !== `string`) continue;
|
|
1021
1040
|
const streamMeta = value;
|
|
1022
1041
|
const streamPath = key.replace(`stream:`, ``);
|
|
1023
|
-
const segmentPath =
|
|
1042
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1024
1043
|
if (!node_fs.existsSync(segmentPath)) {
|
|
1025
|
-
|
|
1044
|
+
serverLog.warn(`[FileBackedStreamStore] Recovery: Stream file missing for ${streamPath}, removing from LMDB`);
|
|
1026
1045
|
this.db.removeSync(key);
|
|
1027
1046
|
errors++;
|
|
1028
1047
|
continue;
|
|
@@ -1036,7 +1055,7 @@ var FileBackedStreamStore = class {
|
|
|
1036
1055
|
trueOffset = `${String(0).padStart(16, `0`)}_${String(logicalBytes).padStart(16, `0`)}`;
|
|
1037
1056
|
} else trueOffset = physicalOffset;
|
|
1038
1057
|
if (trueOffset !== streamMeta.currentOffset) {
|
|
1039
|
-
|
|
1058
|
+
serverLog.warn(`[FileBackedStreamStore] Recovery: Offset mismatch for ${streamPath}: LMDB says ${streamMeta.currentOffset}, file says ${trueOffset}. Reconciling to file.`);
|
|
1040
1059
|
const reconciledMeta = {
|
|
1041
1060
|
...streamMeta,
|
|
1042
1061
|
currentOffset: trueOffset
|
|
@@ -1046,10 +1065,10 @@ var FileBackedStreamStore = class {
|
|
|
1046
1065
|
}
|
|
1047
1066
|
recovered++;
|
|
1048
1067
|
} catch (err) {
|
|
1049
|
-
|
|
1068
|
+
serverLog.error(`[FileBackedStreamStore] Error recovering stream:`, err);
|
|
1050
1069
|
errors++;
|
|
1051
1070
|
}
|
|
1052
|
-
|
|
1071
|
+
serverLog.info(`[FileBackedStreamStore] Recovery complete: ${recovered} streams, ${reconciled} reconciled, ${errors} errors`);
|
|
1053
1072
|
}
|
|
1054
1073
|
/**
|
|
1055
1074
|
* Scan a segment file to compute the true last offset.
|
|
@@ -1059,19 +1078,16 @@ var FileBackedStreamStore = class {
|
|
|
1059
1078
|
try {
|
|
1060
1079
|
const fileContent = node_fs.readFileSync(segmentPath);
|
|
1061
1080
|
let filePos = 0;
|
|
1062
|
-
let currentDataOffset = 0;
|
|
1063
1081
|
while (filePos < fileContent.length) {
|
|
1064
1082
|
if (filePos + 4 > fileContent.length) break;
|
|
1065
1083
|
const messageLength = fileContent.readUInt32BE(filePos);
|
|
1066
|
-
filePos
|
|
1067
|
-
if (
|
|
1068
|
-
filePos
|
|
1069
|
-
if (filePos < fileContent.length) filePos += 1;
|
|
1070
|
-
currentDataOffset += messageLength;
|
|
1084
|
+
const frameEnd = filePos + 4 + messageLength + 1;
|
|
1085
|
+
if (frameEnd > fileContent.length) break;
|
|
1086
|
+
filePos = frameEnd;
|
|
1071
1087
|
}
|
|
1072
|
-
return `0000000000000000_${String(
|
|
1088
|
+
return `0000000000000000_${String(filePos).padStart(16, `0`)}`;
|
|
1073
1089
|
} catch (err) {
|
|
1074
|
-
|
|
1090
|
+
serverLog.error(`[FileBackedStreamStore] Error scanning file ${segmentPath}:`, err);
|
|
1075
1091
|
return `0000000000000000_0000000000000000`;
|
|
1076
1092
|
}
|
|
1077
1093
|
}
|
|
@@ -1339,6 +1355,7 @@ var FileBackedStreamStore = class {
|
|
|
1339
1355
|
effectiveTtlSeconds = resolved.ttlSeconds;
|
|
1340
1356
|
}
|
|
1341
1357
|
const key = `stream:${streamPath}`;
|
|
1358
|
+
const t0 = performance.now();
|
|
1342
1359
|
const streamMeta = {
|
|
1343
1360
|
path: streamPath,
|
|
1344
1361
|
contentType,
|
|
@@ -1356,11 +1373,10 @@ var FileBackedStreamStore = class {
|
|
|
1356
1373
|
forkOffset: isFork ? forkOffset : void 0,
|
|
1357
1374
|
refCount: 0
|
|
1358
1375
|
};
|
|
1359
|
-
const
|
|
1376
|
+
const tAfterMeta = performance.now();
|
|
1377
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1360
1378
|
try {
|
|
1361
|
-
|
|
1362
|
-
const segmentPath = node_path.join(streamDir, `segment_00000.log`);
|
|
1363
|
-
node_fs.writeFileSync(segmentPath, ``);
|
|
1379
|
+
await this.db.put(key, streamMeta);
|
|
1364
1380
|
} catch (err) {
|
|
1365
1381
|
if (isFork && sourceMeta) {
|
|
1366
1382
|
const sourceKey = `stream:${options.forkedFrom}`;
|
|
@@ -1373,10 +1389,18 @@ var FileBackedStreamStore = class {
|
|
|
1373
1389
|
this.db.putSync(sourceKey, updatedSource);
|
|
1374
1390
|
}
|
|
1375
1391
|
}
|
|
1376
|
-
|
|
1392
|
+
serverLog.error(`[FileBackedStreamStore] Error creating stream (LMDB put):`, err);
|
|
1393
|
+
throw err;
|
|
1394
|
+
}
|
|
1395
|
+
const tAfterLmdb = performance.now();
|
|
1396
|
+
try {
|
|
1397
|
+
await this.fileHandlePool.openWriteStream(segmentPath);
|
|
1398
|
+
} catch (err) {
|
|
1399
|
+
this.db.removeSync(key);
|
|
1400
|
+
serverLog.error(`[FileBackedStreamStore] Error creating stream (file open):`, err);
|
|
1377
1401
|
throw err;
|
|
1378
1402
|
}
|
|
1379
|
-
|
|
1403
|
+
const tAfterOpen = performance.now();
|
|
1380
1404
|
if (options.initialData && options.initialData.length > 0) try {
|
|
1381
1405
|
await this.append(streamPath, options.initialData, {
|
|
1382
1406
|
contentType: options.contentType,
|
|
@@ -1396,12 +1420,24 @@ var FileBackedStreamStore = class {
|
|
|
1396
1420
|
}
|
|
1397
1421
|
throw err;
|
|
1398
1422
|
}
|
|
1423
|
+
const tAfterAppend = performance.now();
|
|
1399
1424
|
if (options.closed) {
|
|
1400
1425
|
const updatedMeta = this.db.get(key);
|
|
1401
1426
|
updatedMeta.closed = true;
|
|
1402
|
-
this.db.
|
|
1427
|
+
await this.db.put(key, updatedMeta);
|
|
1403
1428
|
}
|
|
1404
1429
|
const updated = this.db.get(key);
|
|
1430
|
+
const totalMs = performance.now() - t0;
|
|
1431
|
+
if (totalMs > 50) serverLog.event({
|
|
1432
|
+
event: `store.create`,
|
|
1433
|
+
path: streamPath,
|
|
1434
|
+
totalMs: +totalMs.toFixed(2),
|
|
1435
|
+
metaMs: +(tAfterMeta - t0).toFixed(2),
|
|
1436
|
+
lmdbMs: +(tAfterLmdb - tAfterMeta).toFixed(2),
|
|
1437
|
+
openMs: +(tAfterOpen - tAfterLmdb).toFixed(2),
|
|
1438
|
+
appendMs: +(tAfterAppend - tAfterOpen).toFixed(2),
|
|
1439
|
+
initBytes: options.initialData?.length ?? 0
|
|
1440
|
+
}, `store.create slow`);
|
|
1405
1441
|
return this.streamMetaToStream(updated);
|
|
1406
1442
|
}
|
|
1407
1443
|
get(streamPath) {
|
|
@@ -1442,13 +1478,10 @@ var FileBackedStreamStore = class {
|
|
|
1442
1478
|
if (!streamMeta) return;
|
|
1443
1479
|
const forkedFrom = streamMeta.forkedFrom;
|
|
1444
1480
|
this.cancelLongPollsForStream(streamPath);
|
|
1445
|
-
const segmentPath =
|
|
1446
|
-
this.fileHandlePool.closeFileHandle(segmentPath).catch((err) => {
|
|
1447
|
-
console.error(`[FileBackedStreamStore] Error closing file handle:`, err);
|
|
1448
|
-
});
|
|
1481
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1449
1482
|
this.db.removeSync(key);
|
|
1450
|
-
this.
|
|
1451
|
-
|
|
1483
|
+
this.fileHandlePool.closeFileHandle(segmentPath).then(() => node_fs.promises.unlink(segmentPath)).catch((err) => {
|
|
1484
|
+
serverLog.error(`[FileBackedStreamStore] Error cleaning up stream file:`, err);
|
|
1452
1485
|
});
|
|
1453
1486
|
if (forkedFrom) {
|
|
1454
1487
|
const parentKey = `stream:${forkedFrom}`;
|
|
@@ -1519,10 +1552,11 @@ var FileBackedStreamStore = class {
|
|
|
1519
1552
|
const parts = streamMeta.currentOffset.split(`_`).map(Number);
|
|
1520
1553
|
const readSeq = parts[0];
|
|
1521
1554
|
const byteOffset = parts[1];
|
|
1522
|
-
const
|
|
1555
|
+
const FRAME_OVERHEAD = 5;
|
|
1556
|
+
const newByteOffset = byteOffset + FRAME_OVERHEAD + processedData.length;
|
|
1523
1557
|
const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
1524
|
-
const
|
|
1525
|
-
const
|
|
1558
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1559
|
+
const tAppendStart = performance.now();
|
|
1526
1560
|
const stream = this.fileHandlePool.getWriteStream(segmentPath);
|
|
1527
1561
|
const lengthBuf = Buffer.allocUnsafe(4);
|
|
1528
1562
|
lengthBuf.writeUInt32BE(processedData.length, 0);
|
|
@@ -1537,12 +1571,14 @@ var FileBackedStreamStore = class {
|
|
|
1537
1571
|
else resolve();
|
|
1538
1572
|
});
|
|
1539
1573
|
});
|
|
1574
|
+
const tAfterWrite = performance.now();
|
|
1540
1575
|
const message = {
|
|
1541
1576
|
data: processedData,
|
|
1542
1577
|
offset: newOffset,
|
|
1543
1578
|
timestamp: Date.now()
|
|
1544
1579
|
};
|
|
1545
1580
|
await this.fileHandlePool.fsyncFile(segmentPath);
|
|
1581
|
+
const tAfterFsync = performance.now();
|
|
1546
1582
|
const updatedProducers = { ...streamMeta.producers };
|
|
1547
1583
|
if (producerResult && producerResult.status === `accepted`) updatedProducers[producerResult.producerId] = producerResult.proposedState;
|
|
1548
1584
|
let closedBy = void 0;
|
|
@@ -1561,7 +1597,19 @@ var FileBackedStreamStore = class {
|
|
|
1561
1597
|
closedBy: closedBy ?? streamMeta.closedBy
|
|
1562
1598
|
};
|
|
1563
1599
|
const key = `stream:${streamPath}`;
|
|
1564
|
-
this.db.
|
|
1600
|
+
await this.db.put(key, updatedMeta);
|
|
1601
|
+
const tAfterLmdb = performance.now();
|
|
1602
|
+
const appendTotal = tAfterLmdb - tAppendStart;
|
|
1603
|
+
if (appendTotal > 50) serverLog.event({
|
|
1604
|
+
event: `store.append`,
|
|
1605
|
+
path: streamPath,
|
|
1606
|
+
totalMs: +appendTotal.toFixed(2),
|
|
1607
|
+
writeMs: +(tAfterWrite - tAppendStart).toFixed(2),
|
|
1608
|
+
fsyncMs: +(tAfterFsync - tAfterWrite).toFixed(2),
|
|
1609
|
+
lmdbMs: +(tAfterLmdb - tAfterFsync).toFixed(2),
|
|
1610
|
+
bytes: processedData.length,
|
|
1611
|
+
isInitial: options.isInitialCreate ?? false
|
|
1612
|
+
}, `store.append slow`);
|
|
1565
1613
|
this.notifyLongPolls(streamPath);
|
|
1566
1614
|
if (options.close) this.notifyLongPollsClosed(streamPath);
|
|
1567
1615
|
if (producerResult || options.close) return {
|
|
@@ -1654,7 +1702,7 @@ var FileBackedStreamStore = class {
|
|
|
1654
1702
|
},
|
|
1655
1703
|
producers: updatedProducers
|
|
1656
1704
|
};
|
|
1657
|
-
this.db.
|
|
1705
|
+
await this.db.put(key, updatedMeta);
|
|
1658
1706
|
this.notifyLongPollsClosed(streamPath);
|
|
1659
1707
|
return {
|
|
1660
1708
|
finalOffset: streamMeta.currentOffset,
|
|
@@ -1688,7 +1736,7 @@ var FileBackedStreamStore = class {
|
|
|
1688
1736
|
const messageData = fileContent.subarray(filePos, filePos + messageLength);
|
|
1689
1737
|
filePos += messageLength;
|
|
1690
1738
|
filePos += 1;
|
|
1691
|
-
physicalDataOffset += messageLength;
|
|
1739
|
+
physicalDataOffset += messageLength + 5;
|
|
1692
1740
|
const logicalOffset = baseByteOffset + physicalDataOffset;
|
|
1693
1741
|
if (capByte !== void 0 && logicalOffset > capByte) break;
|
|
1694
1742
|
if (logicalOffset > startByte) messages.push({
|
|
@@ -1698,7 +1746,7 @@ var FileBackedStreamStore = class {
|
|
|
1698
1746
|
});
|
|
1699
1747
|
}
|
|
1700
1748
|
} catch (err) {
|
|
1701
|
-
|
|
1749
|
+
serverLog.error(`[FileBackedStreamStore] Error reading segment file:`, err);
|
|
1702
1750
|
}
|
|
1703
1751
|
return messages;
|
|
1704
1752
|
}
|
|
@@ -1720,7 +1768,7 @@ var FileBackedStreamStore = class {
|
|
|
1720
1768
|
messages.push(...inherited);
|
|
1721
1769
|
}
|
|
1722
1770
|
}
|
|
1723
|
-
const segmentPath =
|
|
1771
|
+
const segmentPath = segmentFile(this.dataDir, sourceMeta.directoryName);
|
|
1724
1772
|
const sourceBaseByte = sourceMeta.forkOffset ? Number(sourceMeta.forkOffset.split(`_`)[1] ?? 0) : 0;
|
|
1725
1773
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, sourceBaseByte, capByte);
|
|
1726
1774
|
messages.push(...ownMessages);
|
|
@@ -1747,11 +1795,11 @@ var FileBackedStreamStore = class {
|
|
|
1747
1795
|
const inherited = this.readForkedMessages(streamMeta.forkedFrom, startByte, forkByte);
|
|
1748
1796
|
messages.push(...inherited);
|
|
1749
1797
|
}
|
|
1750
|
-
const segmentPath =
|
|
1798
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1751
1799
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, forkByte);
|
|
1752
1800
|
messages.push(...ownMessages);
|
|
1753
1801
|
} else {
|
|
1754
|
-
const segmentPath =
|
|
1802
|
+
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1755
1803
|
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, 0);
|
|
1756
1804
|
messages.push(...ownMessages);
|
|
1757
1805
|
}
|
|
@@ -1822,6 +1870,7 @@ var FileBackedStreamStore = class {
|
|
|
1822
1870
|
formatResponse(streamPath, messages) {
|
|
1823
1871
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1824
1872
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1873
|
+
if (normalizeContentType(streamMeta.contentType) === `application/json`) return formatJsonMessages(messages);
|
|
1825
1874
|
const totalSize = messages.reduce((sum, m) => sum + m.data.length, 0);
|
|
1826
1875
|
const concatenated = new Uint8Array(totalSize);
|
|
1827
1876
|
let offset = 0;
|
|
@@ -1829,7 +1878,6 @@ var FileBackedStreamStore = class {
|
|
|
1829
1878
|
concatenated.set(msg.data, offset);
|
|
1830
1879
|
offset += msg.data.length;
|
|
1831
1880
|
}
|
|
1832
|
-
if (normalizeContentType(streamMeta.contentType) === `application/json`) return formatJsonResponse(concatenated);
|
|
1833
1881
|
return concatenated;
|
|
1834
1882
|
}
|
|
1835
1883
|
getCurrentOffset(streamPath) {
|
|
@@ -1849,7 +1897,7 @@ var FileBackedStreamStore = class {
|
|
|
1849
1897
|
const entries = Array.from(range);
|
|
1850
1898
|
for (const { key } of entries) this.db.removeSync(key);
|
|
1851
1899
|
this.fileHandlePool.closeAll().catch((err) => {
|
|
1852
|
-
|
|
1900
|
+
serverLog.error(`[FileBackedStreamStore] Error closing handles:`, err);
|
|
1853
1901
|
});
|
|
1854
1902
|
}
|
|
1855
1903
|
/**
|
|
@@ -2003,30 +2051,985 @@ function handleCursorCollision(currentCursor, previousCursor, options = {}) {
|
|
|
2003
2051
|
return generateResponseCursor(previousCursor, options);
|
|
2004
2052
|
}
|
|
2005
2053
|
|
|
2054
|
+
//#endregion
|
|
2055
|
+
//#region src/crypto.ts
|
|
2056
|
+
/**
|
|
2057
|
+
* Generate a unique wake ID.
|
|
2058
|
+
*/
|
|
2059
|
+
function generateWakeId() {
|
|
2060
|
+
return `w_${(0, node_crypto.randomBytes)(12).toString(`hex`)}`;
|
|
2061
|
+
}
|
|
2062
|
+
const WEBHOOK_KEYPAIR = (0, node_crypto.generateKeyPairSync)(`ed25519`);
|
|
2063
|
+
const WEBHOOK_PUBLIC_JWK = buildWebhookPublicJwk();
|
|
2064
|
+
function buildWebhookPublicJwk() {
|
|
2065
|
+
const exported = WEBHOOK_KEYPAIR.publicKey.export({ format: `jwk` });
|
|
2066
|
+
if (exported.kty !== `OKP` || exported.crv !== `Ed25519` || !exported.x) throw new Error(`Failed to export Ed25519 webhook signing key`);
|
|
2067
|
+
const thumbprintInput = JSON.stringify({
|
|
2068
|
+
crv: exported.crv,
|
|
2069
|
+
kty: exported.kty,
|
|
2070
|
+
x: exported.x
|
|
2071
|
+
});
|
|
2072
|
+
const kid = `ds_${(0, node_crypto.createHash)(`sha256`).update(thumbprintInput).digest(`base64url`)}`;
|
|
2073
|
+
return {
|
|
2074
|
+
kty: `OKP`,
|
|
2075
|
+
crv: `Ed25519`,
|
|
2076
|
+
x: exported.x,
|
|
2077
|
+
kid,
|
|
2078
|
+
use: `sig`,
|
|
2079
|
+
alg: `EdDSA`
|
|
2080
|
+
};
|
|
2081
|
+
}
|
|
2082
|
+
function getWebhookSigningKeyId() {
|
|
2083
|
+
return WEBHOOK_PUBLIC_JWK.kid;
|
|
2084
|
+
}
|
|
2085
|
+
function getWebhookJwks() {
|
|
2086
|
+
return { keys: [{ ...WEBHOOK_PUBLIC_JWK }] };
|
|
2087
|
+
}
|
|
2088
|
+
/**
|
|
2089
|
+
* Sign a webhook payload for the Webhook-Signature header.
|
|
2090
|
+
* Format: t=<timestamp>,kid=<key_id>,ed25519=<base64url_signature>
|
|
2091
|
+
*/
|
|
2092
|
+
function signWebhookPayload(body) {
|
|
2093
|
+
const timestamp = Math.floor(Date.now() / 1e3);
|
|
2094
|
+
const payload = `${timestamp}.${body}`;
|
|
2095
|
+
const signature = (0, node_crypto.sign)(null, Buffer.from(payload), WEBHOOK_KEYPAIR.privateKey).toString(`base64url`);
|
|
2096
|
+
return `t=${timestamp},kid=${WEBHOOK_PUBLIC_JWK.kid},ed25519=${signature}`;
|
|
2097
|
+
}
|
|
2098
|
+
const TOKEN_KEY = (0, node_crypto.randomBytes)(32);
|
|
2099
|
+
/**
|
|
2100
|
+
* Generate a signed callback token.
|
|
2101
|
+
* Token format: base64url(json_payload).base64url(hmac_signature)
|
|
2102
|
+
* Payload: { consumer_id, epoch, exp }
|
|
2103
|
+
*/
|
|
2104
|
+
function generateCallbackToken(consumerId, epoch) {
|
|
2105
|
+
const payload = {
|
|
2106
|
+
sub: consumerId,
|
|
2107
|
+
epoch,
|
|
2108
|
+
exp: Math.floor(Date.now() / 1e3) + 3600,
|
|
2109
|
+
jti: (0, node_crypto.randomBytes)(8).toString(`hex`)
|
|
2110
|
+
};
|
|
2111
|
+
const payloadStr = Buffer.from(JSON.stringify(payload)).toString(`base64url`);
|
|
2112
|
+
const sig = (0, node_crypto.createHmac)(`sha256`, TOKEN_KEY).update(payloadStr).digest(`base64url`);
|
|
2113
|
+
return `${payloadStr}.${sig}`;
|
|
2114
|
+
}
|
|
2115
|
+
/**
|
|
2116
|
+
* Validate a callback token. Returns the decoded payload or null.
|
|
2117
|
+
* On success, includes `exp` (unix seconds) so callers can decide
|
|
2118
|
+
* whether the token needs refreshing.
|
|
2119
|
+
*/
|
|
2120
|
+
function validateCallbackToken(token, consumerId) {
|
|
2121
|
+
const parts = token.split(`.`);
|
|
2122
|
+
if (parts.length !== 2) return {
|
|
2123
|
+
valid: false,
|
|
2124
|
+
code: `TOKEN_INVALID`
|
|
2125
|
+
};
|
|
2126
|
+
const [payloadStr, sig] = parts;
|
|
2127
|
+
const expectedSig = (0, node_crypto.createHmac)(`sha256`, TOKEN_KEY).update(payloadStr).digest(`base64url`);
|
|
2128
|
+
try {
|
|
2129
|
+
if (!(0, node_crypto.timingSafeEqual)(Buffer.from(sig), Buffer.from(expectedSig))) return {
|
|
2130
|
+
valid: false,
|
|
2131
|
+
code: `TOKEN_INVALID`
|
|
2132
|
+
};
|
|
2133
|
+
} catch {
|
|
2134
|
+
return {
|
|
2135
|
+
valid: false,
|
|
2136
|
+
code: `TOKEN_INVALID`
|
|
2137
|
+
};
|
|
2138
|
+
}
|
|
2139
|
+
let payload;
|
|
2140
|
+
try {
|
|
2141
|
+
payload = JSON.parse(Buffer.from(payloadStr, `base64url`).toString());
|
|
2142
|
+
} catch {
|
|
2143
|
+
return {
|
|
2144
|
+
valid: false,
|
|
2145
|
+
code: `TOKEN_INVALID`
|
|
2146
|
+
};
|
|
2147
|
+
}
|
|
2148
|
+
if (payload.sub !== consumerId) return {
|
|
2149
|
+
valid: false,
|
|
2150
|
+
code: `TOKEN_INVALID`
|
|
2151
|
+
};
|
|
2152
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
2153
|
+
if (now > payload.exp) return {
|
|
2154
|
+
valid: false,
|
|
2155
|
+
code: `TOKEN_EXPIRED`
|
|
2156
|
+
};
|
|
2157
|
+
return {
|
|
2158
|
+
valid: true,
|
|
2159
|
+
exp: payload.exp,
|
|
2160
|
+
epoch: payload.epoch
|
|
2161
|
+
};
|
|
2162
|
+
}
|
|
2163
|
+
|
|
2164
|
+
//#endregion
|
|
2165
|
+
//#region src/glob.ts
|
|
2166
|
+
/**
|
|
2167
|
+
* Glob pattern matching for webhook subscription patterns.
|
|
2168
|
+
*
|
|
2169
|
+
* Supports:
|
|
2170
|
+
* - `*` matches exactly one path segment
|
|
2171
|
+
* - `**` matches zero or more path segments (recursive)
|
|
2172
|
+
* - Literal segments match exactly
|
|
2173
|
+
*/
|
|
2174
|
+
/**
|
|
2175
|
+
* Match a stream path against a glob pattern.
|
|
2176
|
+
*/
|
|
2177
|
+
function globMatch(pattern, path) {
|
|
2178
|
+
const patternParts = splitPath(pattern);
|
|
2179
|
+
const pathParts = splitPath(path);
|
|
2180
|
+
return matchParts(patternParts, 0, pathParts, 0);
|
|
2181
|
+
}
|
|
2182
|
+
function splitPath(p) {
|
|
2183
|
+
return p.replace(/^\/+/, ``).replace(/\/+$/, ``).split(`/`).filter((s) => s.length > 0);
|
|
2184
|
+
}
|
|
2185
|
+
function matchParts(pattern, pi, path, si) {
|
|
2186
|
+
while (pi < pattern.length && si < path.length) {
|
|
2187
|
+
const seg = pattern[pi];
|
|
2188
|
+
if (seg === `**`) {
|
|
2189
|
+
for (let i = si; i <= path.length; i++) if (matchParts(pattern, pi + 1, path, i)) return true;
|
|
2190
|
+
return false;
|
|
2191
|
+
}
|
|
2192
|
+
if (seg === `*`) {
|
|
2193
|
+
pi++;
|
|
2194
|
+
si++;
|
|
2195
|
+
continue;
|
|
2196
|
+
}
|
|
2197
|
+
const decodedSeg = seg.replace(/%2[Aa]/g, `*`);
|
|
2198
|
+
if (decodedSeg !== path[si]) return false;
|
|
2199
|
+
pi++;
|
|
2200
|
+
si++;
|
|
2201
|
+
}
|
|
2202
|
+
while (pi < pattern.length && pattern[pi] === `**`) pi++;
|
|
2203
|
+
return pi === pattern.length && si === path.length;
|
|
2204
|
+
}
|
|
2205
|
+
|
|
2206
|
+
//#endregion
|
|
2207
|
+
//#region src/subscription-manager.ts
|
|
2208
|
+
const DEFAULT_LEASE_TTL_MS = 3e4;
|
|
2209
|
+
const MIN_LEASE_TTL_MS = 1e3;
|
|
2210
|
+
const MAX_LEASE_TTL_MS = 10 * 6e4;
|
|
2211
|
+
const ZERO_OFFSET = `0000000000000000_0000000000000000`;
|
|
2212
|
+
const BEFORE_FIRST_OFFSET = `-1`;
|
|
2213
|
+
const MAX_RETRY_DELAY_MS = 6e4;
|
|
2214
|
+
function compareOffsets(a, b) {
|
|
2215
|
+
if (a < b) return -1;
|
|
2216
|
+
if (a > b) return 1;
|
|
2217
|
+
return 0;
|
|
2218
|
+
}
|
|
2219
|
+
function normalizeRelativePath(path) {
|
|
2220
|
+
return path.replace(/^\/+/, ``).replace(/\/+$/, ``);
|
|
2221
|
+
}
|
|
2222
|
+
function toAbsoluteStreamPath(streamPath) {
|
|
2223
|
+
return `/v1/stream/${normalizeRelativePath(streamPath)}`;
|
|
2224
|
+
}
|
|
2225
|
+
function toStreamRelativePath(absolutePath) {
|
|
2226
|
+
const streamRoot = `/v1/stream/`;
|
|
2227
|
+
if (!absolutePath.startsWith(streamRoot)) return null;
|
|
2228
|
+
const path = absolutePath.slice(streamRoot.length);
|
|
2229
|
+
if (path === `__ds` || path.startsWith(`__ds/`)) return null;
|
|
2230
|
+
return path.length > 0 ? path : null;
|
|
2231
|
+
}
|
|
2232
|
+
function stableConfigHash(input) {
|
|
2233
|
+
const canonical = {
|
|
2234
|
+
type: input.type,
|
|
2235
|
+
pattern: input.pattern,
|
|
2236
|
+
streams: [...new Set(input.streams)].sort(),
|
|
2237
|
+
webhook: input.webhook ? { url: input.webhook.url } : void 0,
|
|
2238
|
+
wake_stream: input.wake_stream,
|
|
2239
|
+
lease_ttl_ms: input.lease_ttl_ms,
|
|
2240
|
+
description: input.description
|
|
2241
|
+
};
|
|
2242
|
+
return (0, node_crypto.createHash)(`sha256`).update(JSON.stringify(canonical)).digest(`hex`);
|
|
2243
|
+
}
|
|
2244
|
+
function isPrivateOrLinkLocalIpv4(host) {
|
|
2245
|
+
const parts = host.split(`.`).map((part) => Number(part));
|
|
2246
|
+
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part))) return false;
|
|
2247
|
+
const [a, b] = parts;
|
|
2248
|
+
return a === 10 || a === 127 || a === 0 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a === 169 && b === 254;
|
|
2249
|
+
}
|
|
2250
|
+
function isLocalDevHost(host) {
|
|
2251
|
+
return host === `localhost` || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host);
|
|
2252
|
+
}
|
|
2253
|
+
function validateWebhookUrl(rawUrl) {
|
|
2254
|
+
let url;
|
|
2255
|
+
try {
|
|
2256
|
+
url = new URL(rawUrl);
|
|
2257
|
+
} catch {
|
|
2258
|
+
return {
|
|
2259
|
+
ok: false,
|
|
2260
|
+
message: `webhook.url must be a valid URL`
|
|
2261
|
+
};
|
|
2262
|
+
}
|
|
2263
|
+
const host = url.hostname.toLowerCase();
|
|
2264
|
+
if (url.protocol === `http:`) {
|
|
2265
|
+
if (isLocalDevHost(host)) return { ok: true };
|
|
2266
|
+
return {
|
|
2267
|
+
ok: false,
|
|
2268
|
+
message: `http webhook URLs are only allowed for localhost or 127.0.0.x`
|
|
2269
|
+
};
|
|
2270
|
+
}
|
|
2271
|
+
if (url.protocol !== `https:`) return {
|
|
2272
|
+
ok: false,
|
|
2273
|
+
message: `webhook.url must use https`
|
|
2274
|
+
};
|
|
2275
|
+
if (host === `localhost`) return {
|
|
2276
|
+
ok: false,
|
|
2277
|
+
message: `localhost webhook URLs must use http for dev`
|
|
2278
|
+
};
|
|
2279
|
+
if ((0, node_net.isIP)(host) === 4 && isPrivateOrLinkLocalIpv4(host)) return {
|
|
2280
|
+
ok: false,
|
|
2281
|
+
message: `webhook.url must not target private or link-local hosts`
|
|
2282
|
+
};
|
|
2283
|
+
if ((0, node_net.isIP)(host) === 6) return {
|
|
2284
|
+
ok: false,
|
|
2285
|
+
message: `IPv6 webhook hosts are not accepted by the reference server`
|
|
2286
|
+
};
|
|
2287
|
+
return { ok: true };
|
|
2288
|
+
}
|
|
2289
|
+
var SubscriptionManager = class {
|
|
2290
|
+
subscriptions = new Map();
|
|
2291
|
+
streamStore;
|
|
2292
|
+
callbackBaseUrl;
|
|
2293
|
+
webhooksEnabled;
|
|
2294
|
+
isShuttingDown = false;
|
|
2295
|
+
constructor(opts) {
|
|
2296
|
+
this.callbackBaseUrl = opts.callbackBaseUrl;
|
|
2297
|
+
this.streamStore = opts.streamStore;
|
|
2298
|
+
this.webhooksEnabled = opts.webhooksEnabled ?? true;
|
|
2299
|
+
}
|
|
2300
|
+
createOrConfirm(id, input) {
|
|
2301
|
+
const configHash = stableConfigHash(input);
|
|
2302
|
+
const existing = this.subscriptions.get(id);
|
|
2303
|
+
if (existing) {
|
|
2304
|
+
if (existing.config_hash !== configHash) return { error: {
|
|
2305
|
+
code: `SUBSCRIPTION_ALREADY_EXISTS`,
|
|
2306
|
+
message: `Subscription already exists with different configuration`
|
|
2307
|
+
} };
|
|
2308
|
+
return {
|
|
2309
|
+
subscription: existing,
|
|
2310
|
+
created: false
|
|
2311
|
+
};
|
|
2312
|
+
}
|
|
2313
|
+
if (input.type === `webhook`) {
|
|
2314
|
+
if (!this.webhooksEnabled) return { error: {
|
|
2315
|
+
code: `INVALID_REQUEST`,
|
|
2316
|
+
message: `webhook subscriptions are not enabled on this server`
|
|
2317
|
+
} };
|
|
2318
|
+
if (!input.webhook) return { error: {
|
|
2319
|
+
code: `INVALID_REQUEST`,
|
|
2320
|
+
message: `webhook subscriptions require webhook.url`
|
|
2321
|
+
} };
|
|
2322
|
+
const validation = validateWebhookUrl(input.webhook.url);
|
|
2323
|
+
if (!validation.ok) return { error: {
|
|
2324
|
+
code: `WEBHOOK_URL_REJECTED`,
|
|
2325
|
+
message: validation.message
|
|
2326
|
+
} };
|
|
2327
|
+
}
|
|
2328
|
+
if (input.type === `pull-wake` && !input.wake_stream) return { error: {
|
|
2329
|
+
code: `INVALID_REQUEST`,
|
|
2330
|
+
message: `pull-wake subscriptions require wake_stream`
|
|
2331
|
+
} };
|
|
2332
|
+
const subscription = {
|
|
2333
|
+
id,
|
|
2334
|
+
type: input.type,
|
|
2335
|
+
pattern: input.pattern,
|
|
2336
|
+
webhook: input.webhook ? { url: input.webhook.url } : void 0,
|
|
2337
|
+
wake_stream: input.wake_stream,
|
|
2338
|
+
lease_ttl_ms: input.lease_ttl_ms,
|
|
2339
|
+
description: input.description,
|
|
2340
|
+
created_at: new Date().toISOString(),
|
|
2341
|
+
status: `active`,
|
|
2342
|
+
config_hash: configHash,
|
|
2343
|
+
streams: new Map(),
|
|
2344
|
+
generation: 0,
|
|
2345
|
+
wake_id: null,
|
|
2346
|
+
wake_snapshot: new Map(),
|
|
2347
|
+
token: null,
|
|
2348
|
+
holder: null,
|
|
2349
|
+
lease_timer: null,
|
|
2350
|
+
retry_count: 0,
|
|
2351
|
+
retry_timer: null,
|
|
2352
|
+
next_attempt_at: null
|
|
2353
|
+
};
|
|
2354
|
+
for (const stream of input.streams) this.linkStream(subscription, stream, `explicit`, this.getTailOffset(stream));
|
|
2355
|
+
if (input.pattern) {
|
|
2356
|
+
for (const stream of this.listStreams()) if (globMatch(input.pattern, stream)) this.linkStream(subscription, stream, `glob`, this.getTailOffset(stream));
|
|
2357
|
+
}
|
|
2358
|
+
this.subscriptions.set(id, subscription);
|
|
2359
|
+
return {
|
|
2360
|
+
subscription,
|
|
2361
|
+
created: true
|
|
2362
|
+
};
|
|
2363
|
+
}
|
|
2364
|
+
get(id) {
|
|
2365
|
+
return this.subscriptions.get(id);
|
|
2366
|
+
}
|
|
2367
|
+
delete(id) {
|
|
2368
|
+
const subscription = this.subscriptions.get(id);
|
|
2369
|
+
if (!subscription) return false;
|
|
2370
|
+
this.clearLease(subscription);
|
|
2371
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2372
|
+
this.subscriptions.delete(id);
|
|
2373
|
+
return true;
|
|
2374
|
+
}
|
|
2375
|
+
addExplicitStreams(id, streams) {
|
|
2376
|
+
const subscription = this.get(id);
|
|
2377
|
+
if (!subscription) return false;
|
|
2378
|
+
for (const stream of streams) this.linkStream(subscription, stream, `explicit`, this.getTailOffset(stream));
|
|
2379
|
+
return true;
|
|
2380
|
+
}
|
|
2381
|
+
removeExplicitStream(id, streamPath) {
|
|
2382
|
+
const subscription = this.get(id);
|
|
2383
|
+
if (!subscription) return false;
|
|
2384
|
+
const normalized = normalizeRelativePath(streamPath);
|
|
2385
|
+
const link = subscription.streams.get(normalized);
|
|
2386
|
+
if (!link) return true;
|
|
2387
|
+
link.link_types.delete(`explicit`);
|
|
2388
|
+
if (link.link_types.size === 0) subscription.streams.delete(normalized);
|
|
2389
|
+
return true;
|
|
2390
|
+
}
|
|
2391
|
+
async onStreamAppend(absolutePath) {
|
|
2392
|
+
if (this.isShuttingDown) return;
|
|
2393
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2394
|
+
const relative = toStreamRelativePath(absolutePath);
|
|
2395
|
+
if (!relative) continue;
|
|
2396
|
+
if (subscription.pattern && globMatch(subscription.pattern, relative)) {
|
|
2397
|
+
const existing = subscription.streams.get(relative);
|
|
2398
|
+
this.linkStream(subscription, relative, `glob`, existing?.acked_offset ?? BEFORE_FIRST_OFFSET);
|
|
2399
|
+
}
|
|
2400
|
+
if (subscription.streams.has(relative)) await this.maybeWake(subscription, relative);
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
onStreamDeleted(absolutePath) {
|
|
2404
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2405
|
+
const relative = toStreamRelativePath(absolutePath);
|
|
2406
|
+
if (relative) subscription.streams.delete(relative);
|
|
2407
|
+
}
|
|
2408
|
+
}
|
|
2409
|
+
async handleWebhookCallback(id, token, request) {
|
|
2410
|
+
const subscription = this.get(id);
|
|
2411
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2412
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2413
|
+
if (fenced) return fenced;
|
|
2414
|
+
const ackError = this.applyAcks(subscription, request);
|
|
2415
|
+
if (ackError) return ackError;
|
|
2416
|
+
this.extendLease(subscription);
|
|
2417
|
+
let nextWake = false;
|
|
2418
|
+
if (request.done === true) {
|
|
2419
|
+
this.clearLease(subscription);
|
|
2420
|
+
subscription.token = null;
|
|
2421
|
+
subscription.holder = null;
|
|
2422
|
+
subscription.wake_id = null;
|
|
2423
|
+
subscription.wake_snapshot.clear();
|
|
2424
|
+
nextWake = await this.triggerNextWakeIfPending(subscription);
|
|
2425
|
+
}
|
|
2426
|
+
return {
|
|
2427
|
+
status: 200,
|
|
2428
|
+
body: {
|
|
2429
|
+
ok: true,
|
|
2430
|
+
next_wake: nextWake
|
|
2431
|
+
}
|
|
2432
|
+
};
|
|
2433
|
+
}
|
|
2434
|
+
async claim(id, worker) {
|
|
2435
|
+
const subscription = this.get(id);
|
|
2436
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2437
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2438
|
+
if (subscription.holder) return {
|
|
2439
|
+
status: 409,
|
|
2440
|
+
body: { error: {
|
|
2441
|
+
code: `ALREADY_CLAIMED`,
|
|
2442
|
+
current_holder: subscription.holder,
|
|
2443
|
+
generation: subscription.generation
|
|
2444
|
+
} }
|
|
2445
|
+
};
|
|
2446
|
+
if (!this.hasPendingWork(subscription)) return this.errorResponse(409, `NO_PENDING_WORK`, `Subscription has no pending work`);
|
|
2447
|
+
if (!subscription.wake_id) await this.createWake(subscription, this.firstPendingStream(subscription));
|
|
2448
|
+
subscription.holder = worker;
|
|
2449
|
+
subscription.token = generateCallbackToken(this.tokenSubject(subscription), subscription.generation);
|
|
2450
|
+
this.extendLease(subscription);
|
|
2451
|
+
return {
|
|
2452
|
+
status: 200,
|
|
2453
|
+
body: {
|
|
2454
|
+
wake_id: subscription.wake_id,
|
|
2455
|
+
generation: subscription.generation,
|
|
2456
|
+
token: subscription.token,
|
|
2457
|
+
streams: this.streamInfos(subscription),
|
|
2458
|
+
lease_ttl_ms: subscription.lease_ttl_ms
|
|
2459
|
+
}
|
|
2460
|
+
};
|
|
2461
|
+
}
|
|
2462
|
+
async ack(id, token, request) {
|
|
2463
|
+
const subscription = this.get(id);
|
|
2464
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2465
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2466
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2467
|
+
if (fenced) return fenced;
|
|
2468
|
+
const ackError = this.applyAcks(subscription, request);
|
|
2469
|
+
if (ackError) return ackError;
|
|
2470
|
+
this.extendLease(subscription);
|
|
2471
|
+
let nextWake = false;
|
|
2472
|
+
if (request.done === true) {
|
|
2473
|
+
this.clearLease(subscription);
|
|
2474
|
+
subscription.token = null;
|
|
2475
|
+
subscription.holder = null;
|
|
2476
|
+
subscription.wake_id = null;
|
|
2477
|
+
subscription.wake_snapshot.clear();
|
|
2478
|
+
nextWake = await this.triggerNextWakeIfPending(subscription);
|
|
2479
|
+
}
|
|
2480
|
+
return {
|
|
2481
|
+
status: 200,
|
|
2482
|
+
body: {
|
|
2483
|
+
ok: true,
|
|
2484
|
+
next_wake: nextWake
|
|
2485
|
+
}
|
|
2486
|
+
};
|
|
2487
|
+
}
|
|
2488
|
+
async release(id, token, request) {
|
|
2489
|
+
const subscription = this.get(id);
|
|
2490
|
+
if (!subscription) return this.errorResponse(404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2491
|
+
if (subscription.type !== `pull-wake`) return this.errorResponse(400, `INVALID_REQUEST`, `Subscription is not pull-wake`);
|
|
2492
|
+
const fenced = this.validateWakeToken(subscription, token, request);
|
|
2493
|
+
if (fenced) return fenced;
|
|
2494
|
+
this.clearLease(subscription);
|
|
2495
|
+
subscription.token = null;
|
|
2496
|
+
subscription.holder = null;
|
|
2497
|
+
subscription.wake_id = null;
|
|
2498
|
+
subscription.wake_snapshot.clear();
|
|
2499
|
+
await this.triggerNextWakeIfPending(subscription);
|
|
2500
|
+
return { status: 204 };
|
|
2501
|
+
}
|
|
2502
|
+
serialize(subscription) {
|
|
2503
|
+
return {
|
|
2504
|
+
id: subscription.id,
|
|
2505
|
+
subscription_id: subscription.id,
|
|
2506
|
+
type: subscription.type,
|
|
2507
|
+
pattern: subscription.pattern,
|
|
2508
|
+
streams: this.streamInfos(subscription).map((stream) => ({
|
|
2509
|
+
path: stream.path,
|
|
2510
|
+
link_type: stream.link_type,
|
|
2511
|
+
acked_offset: stream.acked_offset
|
|
2512
|
+
})),
|
|
2513
|
+
webhook: subscription.webhook ? {
|
|
2514
|
+
url: subscription.webhook.url,
|
|
2515
|
+
signing: this.webhookSigningMetadata()
|
|
2516
|
+
} : void 0,
|
|
2517
|
+
wake_stream: subscription.wake_stream,
|
|
2518
|
+
lease_ttl_ms: subscription.lease_ttl_ms,
|
|
2519
|
+
created_at: subscription.created_at,
|
|
2520
|
+
status: subscription.status,
|
|
2521
|
+
description: subscription.description
|
|
2522
|
+
};
|
|
2523
|
+
}
|
|
2524
|
+
getWebhookJwks() {
|
|
2525
|
+
return getWebhookJwks();
|
|
2526
|
+
}
|
|
2527
|
+
shutdown() {
|
|
2528
|
+
this.isShuttingDown = true;
|
|
2529
|
+
for (const subscription of this.subscriptions.values()) {
|
|
2530
|
+
this.clearLease(subscription);
|
|
2531
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2532
|
+
}
|
|
2533
|
+
this.subscriptions.clear();
|
|
2534
|
+
}
|
|
2535
|
+
async maybeWake(subscription, triggeredBy) {
|
|
2536
|
+
if (subscription.wake_id || subscription.holder) return;
|
|
2537
|
+
if (!this.hasPendingWork(subscription)) return;
|
|
2538
|
+
await this.createWake(subscription, triggeredBy);
|
|
2539
|
+
}
|
|
2540
|
+
async createWake(subscription, triggeredBy) {
|
|
2541
|
+
subscription.generation++;
|
|
2542
|
+
subscription.wake_id = generateWakeId();
|
|
2543
|
+
subscription.wake_snapshot = new Map(this.streamInfos(subscription).map((stream) => [stream.path, stream.tail_offset]));
|
|
2544
|
+
if (subscription.type === `webhook`) {
|
|
2545
|
+
subscription.token = generateCallbackToken(this.tokenSubject(subscription), subscription.generation);
|
|
2546
|
+
this.extendLease(subscription);
|
|
2547
|
+
this.deliverWebhook(subscription, [triggeredBy]);
|
|
2548
|
+
return;
|
|
2549
|
+
}
|
|
2550
|
+
await this.writePullWakeEvent(subscription, triggeredBy);
|
|
2551
|
+
}
|
|
2552
|
+
async deliverWebhook(subscription, triggeredBy) {
|
|
2553
|
+
if (!subscription.webhook || !subscription.wake_id || !subscription.token) return;
|
|
2554
|
+
const body = JSON.stringify({
|
|
2555
|
+
subscription_id: subscription.id,
|
|
2556
|
+
wake_id: subscription.wake_id,
|
|
2557
|
+
generation: subscription.generation,
|
|
2558
|
+
streams: this.streamInfos(subscription),
|
|
2559
|
+
callback_url: this.subscriptionActionUrl(subscription, `callback`),
|
|
2560
|
+
callback_token: subscription.token
|
|
2561
|
+
});
|
|
2562
|
+
const headers = {
|
|
2563
|
+
"content-type": `application/json`,
|
|
2564
|
+
"webhook-signature": signWebhookPayload(body)
|
|
2565
|
+
};
|
|
2566
|
+
try {
|
|
2567
|
+
const response = await fetch(subscription.webhook.url, {
|
|
2568
|
+
method: `POST`,
|
|
2569
|
+
headers,
|
|
2570
|
+
body
|
|
2571
|
+
});
|
|
2572
|
+
if (!response.ok) {
|
|
2573
|
+
this.scheduleWebhookRetry(subscription, triggeredBy);
|
|
2574
|
+
return;
|
|
2575
|
+
}
|
|
2576
|
+
subscription.status = `active`;
|
|
2577
|
+
subscription.retry_count = 0;
|
|
2578
|
+
subscription.next_attempt_at = null;
|
|
2579
|
+
let parsed = null;
|
|
2580
|
+
try {
|
|
2581
|
+
parsed = await response.json();
|
|
2582
|
+
} catch {
|
|
2583
|
+
parsed = null;
|
|
2584
|
+
}
|
|
2585
|
+
if (parsed?.done === true) {
|
|
2586
|
+
this.autoAckWakeSnapshot(subscription);
|
|
2587
|
+
this.clearLease(subscription);
|
|
2588
|
+
subscription.token = null;
|
|
2589
|
+
subscription.holder = null;
|
|
2590
|
+
subscription.wake_id = null;
|
|
2591
|
+
subscription.wake_snapshot.clear();
|
|
2592
|
+
await this.triggerNextWakeIfPending(subscription);
|
|
2593
|
+
}
|
|
2594
|
+
} catch (err) {
|
|
2595
|
+
serverLog.warn(`[subscriptions] webhook delivery failed:`, err);
|
|
2596
|
+
this.scheduleWebhookRetry(subscription, triggeredBy);
|
|
2597
|
+
}
|
|
2598
|
+
}
|
|
2599
|
+
scheduleWebhookRetry(subscription, triggeredBy) {
|
|
2600
|
+
if (this.isShuttingDown) return;
|
|
2601
|
+
subscription.retry_count++;
|
|
2602
|
+
const baseDelay = Math.min(1e3 * Math.pow(2, Math.max(0, subscription.retry_count - 1)), MAX_RETRY_DELAY_MS);
|
|
2603
|
+
const jitter = baseDelay * .2 * (Math.random() * 2 - 1);
|
|
2604
|
+
const delay = Math.max(0, Math.round(baseDelay + jitter));
|
|
2605
|
+
subscription.status = `failed`;
|
|
2606
|
+
subscription.next_attempt_at = Date.now() + delay;
|
|
2607
|
+
if (subscription.retry_timer) clearTimeout(subscription.retry_timer);
|
|
2608
|
+
subscription.retry_timer = setTimeout(() => {
|
|
2609
|
+
subscription.retry_timer = null;
|
|
2610
|
+
this.deliverWebhook(subscription, triggeredBy);
|
|
2611
|
+
}, delay);
|
|
2612
|
+
}
|
|
2613
|
+
async writePullWakeEvent(subscription, streamPath) {
|
|
2614
|
+
if (!subscription.wake_stream) return;
|
|
2615
|
+
const wakeStream = toAbsoluteStreamPath(subscription.wake_stream);
|
|
2616
|
+
if (!this.streamStore.has(wakeStream)) {
|
|
2617
|
+
serverLog.warn(`[subscriptions] wake stream does not exist: ${wakeStream}`);
|
|
2618
|
+
return;
|
|
2619
|
+
}
|
|
2620
|
+
const event = {
|
|
2621
|
+
type: `wake`,
|
|
2622
|
+
subscription_id: subscription.id,
|
|
2623
|
+
stream: streamPath,
|
|
2624
|
+
generation: subscription.generation,
|
|
2625
|
+
ts: Date.now()
|
|
2626
|
+
};
|
|
2627
|
+
await Promise.resolve(this.streamStore.append(wakeStream, new TextEncoder().encode(JSON.stringify(event))));
|
|
2628
|
+
}
|
|
2629
|
+
autoAckWakeSnapshot(subscription) {
|
|
2630
|
+
for (const [stream, tail] of subscription.wake_snapshot) {
|
|
2631
|
+
const link = subscription.streams.get(stream);
|
|
2632
|
+
if (link) link.acked_offset = tail;
|
|
2633
|
+
}
|
|
2634
|
+
}
|
|
2635
|
+
applyAcks(subscription, request) {
|
|
2636
|
+
if (!request.acks) return null;
|
|
2637
|
+
for (const ack of request.acks) {
|
|
2638
|
+
const stream = normalizeRelativePath(ack.stream ?? ack.path ?? ``);
|
|
2639
|
+
const link = subscription.streams.get(stream);
|
|
2640
|
+
if (!stream || !link) return this.errorResponse(409, `INVALID_OFFSET`, `Ack references an unknown subscription stream`);
|
|
2641
|
+
if (ack.offset === BEFORE_FIRST_OFFSET) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset must not be -1`);
|
|
2642
|
+
if (compareOffsets(ack.offset, link.acked_offset) < 0) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset regresses the committed cursor`);
|
|
2643
|
+
if (compareOffsets(ack.offset, this.getTailOffset(stream)) > 0) return this.errorResponse(409, `INVALID_OFFSET`, `Ack offset is beyond stream tail`);
|
|
2644
|
+
}
|
|
2645
|
+
for (const ack of request.acks) {
|
|
2646
|
+
const stream = normalizeRelativePath(ack.stream ?? ack.path ?? ``);
|
|
2647
|
+
subscription.streams.get(stream).acked_offset = ack.offset;
|
|
2648
|
+
}
|
|
2649
|
+
return null;
|
|
2650
|
+
}
|
|
2651
|
+
validateWakeToken(subscription, token, request) {
|
|
2652
|
+
const tokenResult = validateCallbackToken(token, this.tokenSubject(subscription));
|
|
2653
|
+
if (!tokenResult.valid) return this.errorResponse(401, tokenResult.code, tokenResult.code === `TOKEN_EXPIRED` ? `Token expired` : `Token invalid`);
|
|
2654
|
+
if (tokenResult.epoch !== subscription.generation || request.generation !== subscription.generation || request.wake_id !== subscription.wake_id) return this.errorResponse(409, `FENCED`, `Wake generation is stale`);
|
|
2655
|
+
return null;
|
|
2656
|
+
}
|
|
2657
|
+
async triggerNextWakeIfPending(subscription) {
|
|
2658
|
+
if (!this.hasPendingWork(subscription)) return false;
|
|
2659
|
+
await this.createWake(subscription, this.firstPendingStream(subscription));
|
|
2660
|
+
return true;
|
|
2661
|
+
}
|
|
2662
|
+
hasPendingWork(subscription) {
|
|
2663
|
+
return this.streamInfos(subscription).some((stream) => stream.has_pending);
|
|
2664
|
+
}
|
|
2665
|
+
firstPendingStream(subscription) {
|
|
2666
|
+
return this.streamInfos(subscription).find((stream) => stream.has_pending)?.path ?? ``;
|
|
2667
|
+
}
|
|
2668
|
+
streamInfos(subscription) {
|
|
2669
|
+
return Array.from(subscription.streams.values()).map((link) => {
|
|
2670
|
+
const tail = this.getTailOffset(link.path);
|
|
2671
|
+
return {
|
|
2672
|
+
path: link.path,
|
|
2673
|
+
link_type: link.link_types.has(`explicit`) ? `explicit` : `glob`,
|
|
2674
|
+
acked_offset: link.acked_offset,
|
|
2675
|
+
tail_offset: tail,
|
|
2676
|
+
has_pending: compareOffsets(tail, link.acked_offset) > 0
|
|
2677
|
+
};
|
|
2678
|
+
});
|
|
2679
|
+
}
|
|
2680
|
+
linkStream(subscription, streamPath, linkType, ackedOffset) {
|
|
2681
|
+
const normalized = normalizeRelativePath(streamPath);
|
|
2682
|
+
const existing = subscription.streams.get(normalized);
|
|
2683
|
+
if (existing) {
|
|
2684
|
+
existing.link_types.add(linkType);
|
|
2685
|
+
return existing;
|
|
2686
|
+
}
|
|
2687
|
+
const link = {
|
|
2688
|
+
path: normalized,
|
|
2689
|
+
link_types: new Set([linkType]),
|
|
2690
|
+
acked_offset: ackedOffset
|
|
2691
|
+
};
|
|
2692
|
+
subscription.streams.set(normalized, link);
|
|
2693
|
+
return link;
|
|
2694
|
+
}
|
|
2695
|
+
listStreams() {
|
|
2696
|
+
return this.streamStore.list().map((path) => toStreamRelativePath(path)).filter((path) => path !== null);
|
|
2697
|
+
}
|
|
2698
|
+
getTailOffset(streamPath) {
|
|
2699
|
+
return this.streamStore.get(toAbsoluteStreamPath(streamPath))?.currentOffset ?? ZERO_OFFSET;
|
|
2700
|
+
}
|
|
2701
|
+
subscriptionActionUrl(subscription, action) {
|
|
2702
|
+
const url = new URL(`/v1/stream/__ds/subscriptions/${encodeURIComponent(subscription.id)}/${action}`, this.callbackBaseUrl);
|
|
2703
|
+
return url.toString();
|
|
2704
|
+
}
|
|
2705
|
+
webhookJwksUrl() {
|
|
2706
|
+
const url = new URL(`/v1/stream/__ds/jwks.json`, this.callbackBaseUrl);
|
|
2707
|
+
return url.toString();
|
|
2708
|
+
}
|
|
2709
|
+
webhookSigningMetadata() {
|
|
2710
|
+
return {
|
|
2711
|
+
alg: `ed25519`,
|
|
2712
|
+
kid: getWebhookSigningKeyId(),
|
|
2713
|
+
jwks_url: this.webhookJwksUrl()
|
|
2714
|
+
};
|
|
2715
|
+
}
|
|
2716
|
+
extendLease(subscription) {
|
|
2717
|
+
this.clearLease(subscription);
|
|
2718
|
+
subscription.lease_timer = setTimeout(() => {
|
|
2719
|
+
subscription.lease_timer = null;
|
|
2720
|
+
subscription.holder = null;
|
|
2721
|
+
subscription.token = null;
|
|
2722
|
+
subscription.wake_id = null;
|
|
2723
|
+
subscription.wake_snapshot.clear();
|
|
2724
|
+
this.triggerNextWakeIfPending(subscription);
|
|
2725
|
+
}, subscription.lease_ttl_ms);
|
|
2726
|
+
}
|
|
2727
|
+
clearLease(subscription) {
|
|
2728
|
+
if (subscription.lease_timer) {
|
|
2729
|
+
clearTimeout(subscription.lease_timer);
|
|
2730
|
+
subscription.lease_timer = null;
|
|
2731
|
+
}
|
|
2732
|
+
}
|
|
2733
|
+
tokenSubject(subscription) {
|
|
2734
|
+
return `subscription:${subscription.id}`;
|
|
2735
|
+
}
|
|
2736
|
+
errorResponse(status, code, message) {
|
|
2737
|
+
return {
|
|
2738
|
+
status,
|
|
2739
|
+
body: { error: {
|
|
2740
|
+
code,
|
|
2741
|
+
message
|
|
2742
|
+
} }
|
|
2743
|
+
};
|
|
2744
|
+
}
|
|
2745
|
+
};
|
|
2746
|
+
|
|
2747
|
+
//#endregion
|
|
2748
|
+
//#region src/subscription-routes.ts
|
|
2749
|
+
const RESERVED_CONTROL_PREFIX = `/v1/stream/__ds`;
|
|
2750
|
+
const SUBSCRIPTION_PREFIX = `${RESERVED_CONTROL_PREFIX}/subscriptions/`;
|
|
2751
|
+
const JWKS_PATH = `${RESERVED_CONTROL_PREFIX}/jwks.json`;
|
|
2752
|
+
const ERROR_STATUS = {
|
|
2753
|
+
INVALID_REQUEST: 400,
|
|
2754
|
+
SUBSCRIPTION_NOT_FOUND: 404,
|
|
2755
|
+
SUBSCRIPTION_ALREADY_EXISTS: 409,
|
|
2756
|
+
WEBHOOK_URL_REJECTED: 400,
|
|
2757
|
+
TOKEN_INVALID: 401,
|
|
2758
|
+
TOKEN_EXPIRED: 401,
|
|
2759
|
+
FENCED: 409,
|
|
2760
|
+
ALREADY_CLAIMED: 409,
|
|
2761
|
+
NO_PENDING_WORK: 409,
|
|
2762
|
+
INVALID_OFFSET: 409
|
|
2763
|
+
};
|
|
2764
|
+
var SubscriptionRoutes = class {
|
|
2765
|
+
manager;
|
|
2766
|
+
constructor(manager) {
|
|
2767
|
+
this.manager = manager;
|
|
2768
|
+
}
|
|
2769
|
+
async handleRequest(method, path, req, res) {
|
|
2770
|
+
if (path === JWKS_PATH) {
|
|
2771
|
+
this.handleJwks(method, res);
|
|
2772
|
+
return true;
|
|
2773
|
+
}
|
|
2774
|
+
const route = this.parseRoute(path);
|
|
2775
|
+
if (!route) {
|
|
2776
|
+
if (path === RESERVED_CONTROL_PREFIX || path.startsWith(`${RESERVED_CONTROL_PREFIX}/`)) {
|
|
2777
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Durable Streams control route not found`);
|
|
2778
|
+
return true;
|
|
2779
|
+
}
|
|
2780
|
+
return false;
|
|
2781
|
+
}
|
|
2782
|
+
try {
|
|
2783
|
+
switch (route.action) {
|
|
2784
|
+
case `base`:
|
|
2785
|
+
await this.handleBase(route, method, req, res);
|
|
2786
|
+
return true;
|
|
2787
|
+
case `streams`:
|
|
2788
|
+
await this.handleStreams(route, method, req, res);
|
|
2789
|
+
return true;
|
|
2790
|
+
case `stream`:
|
|
2791
|
+
this.handleStream(route, method, res);
|
|
2792
|
+
return true;
|
|
2793
|
+
case `callback`:
|
|
2794
|
+
await this.handleCallback(route, req, res);
|
|
2795
|
+
return true;
|
|
2796
|
+
case `claim`:
|
|
2797
|
+
await this.handleClaim(route, req, res);
|
|
2798
|
+
return true;
|
|
2799
|
+
case `ack`:
|
|
2800
|
+
await this.handleAck(route, req, res);
|
|
2801
|
+
return true;
|
|
2802
|
+
case `release`:
|
|
2803
|
+
await this.handleRelease(route, req, res);
|
|
2804
|
+
return true;
|
|
2805
|
+
}
|
|
2806
|
+
} catch (err) {
|
|
2807
|
+
if (err instanceof SyntaxError) {
|
|
2808
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `Invalid JSON body`);
|
|
2809
|
+
return true;
|
|
2810
|
+
}
|
|
2811
|
+
throw err;
|
|
2812
|
+
}
|
|
2813
|
+
}
|
|
2814
|
+
async handleBase(route, method, req, res) {
|
|
2815
|
+
if (method === `PUT`) {
|
|
2816
|
+
const parsed = await this.readJson(req);
|
|
2817
|
+
const input = this.parseCreateInput(parsed);
|
|
2818
|
+
if (`error` in input) {
|
|
2819
|
+
this.writeError(res, 400, `INVALID_REQUEST`, input.error);
|
|
2820
|
+
return;
|
|
2821
|
+
}
|
|
2822
|
+
const result = this.manager.createOrConfirm(route.subscriptionId, input.value);
|
|
2823
|
+
if (`error` in result) {
|
|
2824
|
+
this.writeError(res, ERROR_STATUS[result.error.code], result.error.code, result.error.message);
|
|
2825
|
+
return;
|
|
2826
|
+
}
|
|
2827
|
+
this.writeJson(res, result.created ? 201 : 200, this.manager.serialize(result.subscription));
|
|
2828
|
+
return;
|
|
2829
|
+
}
|
|
2830
|
+
if (method === `GET`) {
|
|
2831
|
+
const subscription = this.manager.get(route.subscriptionId);
|
|
2832
|
+
if (!subscription) {
|
|
2833
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2834
|
+
return;
|
|
2835
|
+
}
|
|
2836
|
+
this.writeJson(res, 200, this.manager.serialize(subscription));
|
|
2837
|
+
return;
|
|
2838
|
+
}
|
|
2839
|
+
if (method === `DELETE`) {
|
|
2840
|
+
this.manager.delete(route.subscriptionId);
|
|
2841
|
+
res.writeHead(204);
|
|
2842
|
+
res.end();
|
|
2843
|
+
return;
|
|
2844
|
+
}
|
|
2845
|
+
this.methodNotAllowed(res);
|
|
2846
|
+
}
|
|
2847
|
+
handleJwks(method, res) {
|
|
2848
|
+
if (method !== `GET`) {
|
|
2849
|
+
this.methodNotAllowed(res);
|
|
2850
|
+
return;
|
|
2851
|
+
}
|
|
2852
|
+
res.writeHead(200, {
|
|
2853
|
+
"content-type": `application/jwk-set+json`,
|
|
2854
|
+
"cache-control": `public, max-age=300`
|
|
2855
|
+
});
|
|
2856
|
+
res.end(JSON.stringify(this.manager.getWebhookJwks()));
|
|
2857
|
+
}
|
|
2858
|
+
async handleStreams(route, method, req, res) {
|
|
2859
|
+
if (method !== `POST`) {
|
|
2860
|
+
this.methodNotAllowed(res);
|
|
2861
|
+
return;
|
|
2862
|
+
}
|
|
2863
|
+
const parsed = await this.readJson(req);
|
|
2864
|
+
const streams = parsed.streams;
|
|
2865
|
+
if (!Array.isArray(streams) || streams.some((stream) => typeof stream !== `string` || stream.length === 0)) {
|
|
2866
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `streams must be a non-empty string array`);
|
|
2867
|
+
return;
|
|
2868
|
+
}
|
|
2869
|
+
const ok = this.manager.addExplicitStreams(route.subscriptionId, streams.map(normalizeRelativePath));
|
|
2870
|
+
if (!ok) {
|
|
2871
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2872
|
+
return;
|
|
2873
|
+
}
|
|
2874
|
+
res.writeHead(204);
|
|
2875
|
+
res.end();
|
|
2876
|
+
}
|
|
2877
|
+
handleStream(route, method, res) {
|
|
2878
|
+
if (method !== `DELETE`) {
|
|
2879
|
+
this.methodNotAllowed(res);
|
|
2880
|
+
return;
|
|
2881
|
+
}
|
|
2882
|
+
const ok = this.manager.removeExplicitStream(route.subscriptionId, route.streamPath ?? ``);
|
|
2883
|
+
if (!ok) {
|
|
2884
|
+
this.writeError(res, 404, `SUBSCRIPTION_NOT_FOUND`, `Subscription not found`);
|
|
2885
|
+
return;
|
|
2886
|
+
}
|
|
2887
|
+
res.writeHead(204);
|
|
2888
|
+
res.end();
|
|
2889
|
+
}
|
|
2890
|
+
async handleCallback(route, req, res) {
|
|
2891
|
+
const token = this.readBearerToken(req);
|
|
2892
|
+
if (!token) {
|
|
2893
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2894
|
+
return;
|
|
2895
|
+
}
|
|
2896
|
+
const body = await this.readJson(req);
|
|
2897
|
+
const result = await this.manager.handleWebhookCallback(route.subscriptionId, token, body);
|
|
2898
|
+
this.writeManagerResult(res, result);
|
|
2899
|
+
}
|
|
2900
|
+
async handleClaim(route, req, res) {
|
|
2901
|
+
const parsed = await this.readJson(req);
|
|
2902
|
+
const worker = parsed.worker;
|
|
2903
|
+
if (typeof worker !== `string` || worker.length === 0) {
|
|
2904
|
+
this.writeError(res, 400, `INVALID_REQUEST`, `worker must be a non-empty string`);
|
|
2905
|
+
return;
|
|
2906
|
+
}
|
|
2907
|
+
const result = await this.manager.claim(route.subscriptionId, worker);
|
|
2908
|
+
this.writeManagerResult(res, result);
|
|
2909
|
+
}
|
|
2910
|
+
async handleAck(route, req, res) {
|
|
2911
|
+
const token = this.readBearerToken(req);
|
|
2912
|
+
if (!token) {
|
|
2913
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2914
|
+
return;
|
|
2915
|
+
}
|
|
2916
|
+
const body = await this.readJson(req);
|
|
2917
|
+
const result = await this.manager.ack(route.subscriptionId, token, body);
|
|
2918
|
+
this.writeManagerResult(res, result);
|
|
2919
|
+
}
|
|
2920
|
+
async handleRelease(route, req, res) {
|
|
2921
|
+
const token = this.readBearerToken(req);
|
|
2922
|
+
if (!token) {
|
|
2923
|
+
this.writeError(res, 401, `TOKEN_INVALID`, `Missing or malformed Authorization header`);
|
|
2924
|
+
return;
|
|
2925
|
+
}
|
|
2926
|
+
const body = await this.readJson(req);
|
|
2927
|
+
const result = await this.manager.release(route.subscriptionId, token, body);
|
|
2928
|
+
this.writeManagerResult(res, result);
|
|
2929
|
+
}
|
|
2930
|
+
parseCreateInput(value) {
|
|
2931
|
+
if (!value || typeof value !== `object`) return { error: `Request body must be a JSON object` };
|
|
2932
|
+
const payload = value;
|
|
2933
|
+
if (payload.type !== `webhook` && payload.type !== `pull-wake`) return { error: `type must be "webhook" or "pull-wake"` };
|
|
2934
|
+
const type = payload.type;
|
|
2935
|
+
const pattern = typeof payload.pattern === `string` && payload.pattern.length > 0 ? normalizeRelativePath(payload.pattern) : void 0;
|
|
2936
|
+
const streams = Array.isArray(payload.streams) && payload.streams.length > 0 ? payload.streams.map((stream) => typeof stream === `string` ? normalizeRelativePath(stream) : null) : [];
|
|
2937
|
+
if (streams.some((stream) => stream === null)) return { error: `streams must contain only strings` };
|
|
2938
|
+
if (!pattern && streams.length === 0) return { error: `At least one of pattern or streams is required` };
|
|
2939
|
+
const leaseTtl = payload.lease_ttl_ms === void 0 ? DEFAULT_LEASE_TTL_MS : payload.lease_ttl_ms;
|
|
2940
|
+
if (typeof leaseTtl !== `number` || !Number.isInteger(leaseTtl) || leaseTtl < MIN_LEASE_TTL_MS || leaseTtl > MAX_LEASE_TTL_MS) return { error: `lease_ttl_ms must be an integer from 1000 to 600000` };
|
|
2941
|
+
let webhook;
|
|
2942
|
+
if (type === `webhook`) {
|
|
2943
|
+
const rawWebhook = payload.webhook;
|
|
2944
|
+
if (!rawWebhook || typeof rawWebhook !== `object`) return { error: `webhook subscriptions require webhook.url` };
|
|
2945
|
+
const url = rawWebhook.url;
|
|
2946
|
+
if (typeof url !== `string` || url.length === 0) return { error: `webhook subscriptions require webhook.url` };
|
|
2947
|
+
webhook = { url };
|
|
2948
|
+
}
|
|
2949
|
+
const wakeStream = typeof payload.wake_stream === `string` && payload.wake_stream.length > 0 ? normalizeRelativePath(payload.wake_stream) : void 0;
|
|
2950
|
+
if (type === `pull-wake` && !wakeStream) return { error: `pull-wake subscriptions require wake_stream` };
|
|
2951
|
+
return { value: {
|
|
2952
|
+
type,
|
|
2953
|
+
pattern,
|
|
2954
|
+
streams,
|
|
2955
|
+
webhook,
|
|
2956
|
+
wake_stream: wakeStream,
|
|
2957
|
+
lease_ttl_ms: leaseTtl,
|
|
2958
|
+
description: typeof payload.description === `string` ? payload.description : void 0
|
|
2959
|
+
} };
|
|
2960
|
+
}
|
|
2961
|
+
parseRoute(path) {
|
|
2962
|
+
if (!path.startsWith(SUBSCRIPTION_PREFIX)) return null;
|
|
2963
|
+
const rest = path.slice(SUBSCRIPTION_PREFIX.length);
|
|
2964
|
+
const parts = rest.split(`/`);
|
|
2965
|
+
const subscriptionId = parts[0] ? decodeURIComponent(parts[0]) : ``;
|
|
2966
|
+
if (!subscriptionId) return null;
|
|
2967
|
+
const tail = parts.slice(1);
|
|
2968
|
+
if (tail.length === 0) return {
|
|
2969
|
+
subscriptionId,
|
|
2970
|
+
action: `base`
|
|
2971
|
+
};
|
|
2972
|
+
if (tail[0] === `streams` && tail.length === 1) return {
|
|
2973
|
+
subscriptionId,
|
|
2974
|
+
action: `streams`
|
|
2975
|
+
};
|
|
2976
|
+
if (tail[0] === `streams` && tail.length > 1) return {
|
|
2977
|
+
subscriptionId,
|
|
2978
|
+
action: `stream`,
|
|
2979
|
+
streamPath: normalizeRelativePath(decodeURIComponent(tail.slice(1).join(`/`)))
|
|
2980
|
+
};
|
|
2981
|
+
if (tail.length === 1 && [
|
|
2982
|
+
`callback`,
|
|
2983
|
+
`claim`,
|
|
2984
|
+
`ack`,
|
|
2985
|
+
`release`
|
|
2986
|
+
].includes(tail[0])) return {
|
|
2987
|
+
subscriptionId,
|
|
2988
|
+
action: tail[0]
|
|
2989
|
+
};
|
|
2990
|
+
return null;
|
|
2991
|
+
}
|
|
2992
|
+
readBearerToken(req) {
|
|
2993
|
+
const authHeader = req.headers.authorization;
|
|
2994
|
+
if (!authHeader || !authHeader.startsWith(`Bearer `)) return null;
|
|
2995
|
+
return authHeader.slice(`Bearer `.length);
|
|
2996
|
+
}
|
|
2997
|
+
async readJson(req) {
|
|
2998
|
+
const chunks = [];
|
|
2999
|
+
for await (const chunk of req) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
3000
|
+
const raw = Buffer.concat(chunks).toString(`utf8`);
|
|
3001
|
+
return raw.length > 0 ? JSON.parse(raw) : {};
|
|
3002
|
+
}
|
|
3003
|
+
writeManagerResult(res, result) {
|
|
3004
|
+
if (result.status === 204) {
|
|
3005
|
+
res.writeHead(204);
|
|
3006
|
+
res.end();
|
|
3007
|
+
return;
|
|
3008
|
+
}
|
|
3009
|
+
this.writeJson(res, result.status, result.body ?? {});
|
|
3010
|
+
}
|
|
3011
|
+
writeJson(res, status, body) {
|
|
3012
|
+
res.writeHead(status, { "content-type": `application/json` });
|
|
3013
|
+
res.end(JSON.stringify(body));
|
|
3014
|
+
}
|
|
3015
|
+
writeError(res, status, code, message) {
|
|
3016
|
+
this.writeJson(res, status, { error: {
|
|
3017
|
+
code,
|
|
3018
|
+
message
|
|
3019
|
+
} });
|
|
3020
|
+
}
|
|
3021
|
+
methodNotAllowed(res) {
|
|
3022
|
+
res.writeHead(405, { "content-type": `text/plain` });
|
|
3023
|
+
res.end(`Method not allowed`);
|
|
3024
|
+
}
|
|
3025
|
+
};
|
|
3026
|
+
|
|
2006
3027
|
//#endregion
|
|
2007
3028
|
//#region src/server.ts
|
|
2008
|
-
const STREAM_OFFSET_HEADER = `Stream-Next-Offset`;
|
|
2009
|
-
const STREAM_CURSOR_HEADER = `Stream-Cursor`;
|
|
2010
|
-
const STREAM_UP_TO_DATE_HEADER = `Stream-Up-To-Date`;
|
|
2011
|
-
const STREAM_SEQ_HEADER = `Stream-Seq`;
|
|
2012
|
-
const STREAM_TTL_HEADER = `Stream-TTL`;
|
|
2013
|
-
const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`;
|
|
2014
3029
|
const STREAM_SSE_DATA_ENCODING_HEADER = `Stream-SSE-Data-Encoding`;
|
|
2015
|
-
const PRODUCER_ID_HEADER = `Producer-Id`;
|
|
2016
|
-
const PRODUCER_EPOCH_HEADER = `Producer-Epoch`;
|
|
2017
|
-
const PRODUCER_SEQ_HEADER = `Producer-Seq`;
|
|
2018
|
-
const PRODUCER_EXPECTED_SEQ_HEADER = `Producer-Expected-Seq`;
|
|
2019
|
-
const PRODUCER_RECEIVED_SEQ_HEADER = `Producer-Received-Seq`;
|
|
2020
|
-
const SSE_OFFSET_FIELD = `streamNextOffset`;
|
|
2021
|
-
const SSE_CURSOR_FIELD = `streamCursor`;
|
|
2022
3030
|
const SSE_UP_TO_DATE_FIELD = `upToDate`;
|
|
2023
|
-
const SSE_CLOSED_FIELD = `streamClosed`;
|
|
2024
|
-
const STREAM_CLOSED_HEADER = `Stream-Closed`;
|
|
2025
3031
|
const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`;
|
|
2026
3032
|
const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`;
|
|
2027
|
-
const OFFSET_QUERY_PARAM = `offset`;
|
|
2028
|
-
const LIVE_QUERY_PARAM = `live`;
|
|
2029
|
-
const CURSOR_QUERY_PARAM = `cursor`;
|
|
2030
3033
|
/**
|
|
2031
3034
|
* Encode data for SSE format.
|
|
2032
3035
|
* Per SSE spec, each line in the payload needs its own "data:" prefix.
|
|
@@ -2080,6 +3083,8 @@ var DurableStreamTestServer = class {
|
|
|
2080
3083
|
isShuttingDown = false;
|
|
2081
3084
|
/** Injected faults for testing retry/resilience */
|
|
2082
3085
|
injectedFaults = new Map();
|
|
3086
|
+
subscriptionManager = null;
|
|
3087
|
+
subscriptionRoutes = null;
|
|
2083
3088
|
constructor(options = {}) {
|
|
2084
3089
|
if (options.dataDir) this.store = new FileBackedStreamStore({ dataDir: options.dataDir });
|
|
2085
3090
|
else this.store = new StreamStore();
|
|
@@ -2094,7 +3099,8 @@ var DurableStreamTestServer = class {
|
|
|
2094
3099
|
cursorOptions: {
|
|
2095
3100
|
intervalSeconds: options.cursorIntervalSeconds,
|
|
2096
3101
|
epoch: options.cursorEpoch
|
|
2097
|
-
}
|
|
3102
|
+
},
|
|
3103
|
+
webhooks: options.webhooks ?? false
|
|
2098
3104
|
};
|
|
2099
3105
|
}
|
|
2100
3106
|
/**
|
|
@@ -2105,7 +3111,7 @@ var DurableStreamTestServer = class {
|
|
|
2105
3111
|
return new Promise((resolve, reject) => {
|
|
2106
3112
|
this.server = (0, node_http.createServer)((req, res) => {
|
|
2107
3113
|
this.handleRequest(req, res).catch((err) => {
|
|
2108
|
-
|
|
3114
|
+
serverLog.error(`Request error:`, err);
|
|
2109
3115
|
if (!res.headersSent) {
|
|
2110
3116
|
res.writeHead(500, { "content-type": `text/plain` });
|
|
2111
3117
|
res.end(`Internal server error`);
|
|
@@ -2117,6 +3123,12 @@ var DurableStreamTestServer = class {
|
|
|
2117
3123
|
const addr = this.server.address();
|
|
2118
3124
|
if (typeof addr === `string`) this._url = addr;
|
|
2119
3125
|
else if (addr) this._url = `http://${this.options.host}:${addr.port}`;
|
|
3126
|
+
this.subscriptionManager = new SubscriptionManager({
|
|
3127
|
+
callbackBaseUrl: this._url,
|
|
3128
|
+
streamStore: this.store,
|
|
3129
|
+
webhooksEnabled: this.options.webhooks
|
|
3130
|
+
});
|
|
3131
|
+
this.subscriptionRoutes = new SubscriptionRoutes(this.subscriptionManager);
|
|
2120
3132
|
resolve(this._url);
|
|
2121
3133
|
});
|
|
2122
3134
|
});
|
|
@@ -2127,6 +3139,11 @@ var DurableStreamTestServer = class {
|
|
|
2127
3139
|
async stop() {
|
|
2128
3140
|
if (!this.server) return;
|
|
2129
3141
|
this.isShuttingDown = true;
|
|
3142
|
+
if (this.subscriptionManager) {
|
|
3143
|
+
this.subscriptionManager.shutdown();
|
|
3144
|
+
this.subscriptionManager = null;
|
|
3145
|
+
this.subscriptionRoutes = null;
|
|
3146
|
+
}
|
|
2130
3147
|
if (`cancelAllWaits` in this.store) this.store.cancelAllWaits();
|
|
2131
3148
|
for (const res of this.activeSSEResponses) res.end();
|
|
2132
3149
|
this.activeSSEResponses.clear();
|
|
@@ -2267,6 +3284,10 @@ var DurableStreamTestServer = class {
|
|
|
2267
3284
|
}
|
|
2268
3285
|
if (fault.truncateBodyBytes !== void 0 || fault.corruptBody || fault.injectSseEvent) res._injectedFault = fault;
|
|
2269
3286
|
}
|
|
3287
|
+
if (this.subscriptionRoutes && method) {
|
|
3288
|
+
const handled = await this.subscriptionRoutes.handleRequest(method, path, req, res);
|
|
3289
|
+
if (handled) return;
|
|
3290
|
+
}
|
|
2270
3291
|
try {
|
|
2271
3292
|
switch (method) {
|
|
2272
3293
|
case `PUT`:
|
|
@@ -2325,9 +3346,9 @@ var DurableStreamTestServer = class {
|
|
|
2325
3346
|
const forkedFromHeader = req.headers[STREAM_FORKED_FROM_HEADER.toLowerCase()];
|
|
2326
3347
|
const forkOffsetHeader = req.headers[STREAM_FORK_OFFSET_HEADER.toLowerCase()];
|
|
2327
3348
|
if (!contentType || contentType.trim() === `` || !/^[\w-]+\/[\w-]+/.test(contentType)) contentType = forkedFromHeader ? void 0 : `application/octet-stream`;
|
|
2328
|
-
const ttlHeader = req.headers[STREAM_TTL_HEADER.toLowerCase()];
|
|
2329
|
-
const expiresAtHeader = req.headers[STREAM_EXPIRES_AT_HEADER.toLowerCase()];
|
|
2330
|
-
const closedHeader = req.headers[STREAM_CLOSED_HEADER.toLowerCase()];
|
|
3349
|
+
const ttlHeader = req.headers[__durable_streams_client.STREAM_TTL_HEADER.toLowerCase()];
|
|
3350
|
+
const expiresAtHeader = req.headers[__durable_streams_client.STREAM_EXPIRES_AT_HEADER.toLowerCase()];
|
|
3351
|
+
const closedHeader = req.headers[__durable_streams_client.STREAM_CLOSED_HEADER.toLowerCase()];
|
|
2331
3352
|
const createClosed = closedHeader === `true`;
|
|
2332
3353
|
if (ttlHeader && expiresAtHeader) {
|
|
2333
3354
|
res.writeHead(400, { "content-type": `text/plain` });
|
|
@@ -2410,12 +3431,13 @@ var DurableStreamTestServer = class {
|
|
|
2410
3431
|
contentType: resolvedContentType,
|
|
2411
3432
|
timestamp: Date.now()
|
|
2412
3433
|
}));
|
|
3434
|
+
if (isNew && body.length > 0) await this.notifyStreamAppend(path);
|
|
2413
3435
|
const headers = {
|
|
2414
3436
|
"content-type": resolvedContentType,
|
|
2415
|
-
[STREAM_OFFSET_HEADER]: stream.currentOffset
|
|
3437
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream.currentOffset
|
|
2416
3438
|
};
|
|
2417
3439
|
if (isNew) headers[`location`] = `${this._url}${path}`;
|
|
2418
|
-
if (stream.closed) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
3440
|
+
if (stream.closed) headers[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2419
3441
|
res.writeHead(isNew ? 201 : 200, headers);
|
|
2420
3442
|
res.end();
|
|
2421
3443
|
}
|
|
@@ -2435,13 +3457,13 @@ var DurableStreamTestServer = class {
|
|
|
2435
3457
|
return;
|
|
2436
3458
|
}
|
|
2437
3459
|
const headers = {
|
|
2438
|
-
[STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
3460
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
2439
3461
|
"cache-control": `no-store`
|
|
2440
3462
|
};
|
|
2441
3463
|
if (stream.contentType) headers[`content-type`] = stream.contentType;
|
|
2442
|
-
if (stream.closed) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
2443
|
-
if (stream.ttlSeconds !== void 0) headers[STREAM_TTL_HEADER] = String(stream.ttlSeconds);
|
|
2444
|
-
if (stream.expiresAt) headers[STREAM_EXPIRES_AT_HEADER] = stream.expiresAt;
|
|
3464
|
+
if (stream.closed) headers[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
3465
|
+
if (stream.ttlSeconds !== void 0) headers[__durable_streams_client.STREAM_TTL_HEADER] = String(stream.ttlSeconds);
|
|
3466
|
+
if (stream.expiresAt) headers[__durable_streams_client.STREAM_EXPIRES_AT_HEADER] = stream.expiresAt;
|
|
2445
3467
|
const closedSuffix = stream.closed ? `:c` : ``;
|
|
2446
3468
|
headers[`etag`] = `"${Buffer.from(path).toString(`base64`)}:-1:${stream.currentOffset}${closedSuffix}"`;
|
|
2447
3469
|
res.writeHead(200, headers);
|
|
@@ -2462,16 +3484,16 @@ var DurableStreamTestServer = class {
|
|
|
2462
3484
|
res.end(`Stream is gone`);
|
|
2463
3485
|
return;
|
|
2464
3486
|
}
|
|
2465
|
-
const offset = url.searchParams.get(OFFSET_QUERY_PARAM) ?? void 0;
|
|
2466
|
-
const live = url.searchParams.get(LIVE_QUERY_PARAM);
|
|
2467
|
-
const cursor = url.searchParams.get(CURSOR_QUERY_PARAM) ?? void 0;
|
|
3487
|
+
const offset = url.searchParams.get(__durable_streams_client.OFFSET_QUERY_PARAM) ?? void 0;
|
|
3488
|
+
const live = url.searchParams.get(__durable_streams_client.LIVE_QUERY_PARAM);
|
|
3489
|
+
const cursor = url.searchParams.get(__durable_streams_client.CURSOR_QUERY_PARAM) ?? void 0;
|
|
2468
3490
|
if (offset !== void 0) {
|
|
2469
3491
|
if (offset === ``) {
|
|
2470
3492
|
res.writeHead(400, { "content-type": `text/plain` });
|
|
2471
3493
|
res.end(`Empty offset parameter`);
|
|
2472
3494
|
return;
|
|
2473
3495
|
}
|
|
2474
|
-
const allOffsets = url.searchParams.getAll(OFFSET_QUERY_PARAM);
|
|
3496
|
+
const allOffsets = url.searchParams.getAll(__durable_streams_client.OFFSET_QUERY_PARAM);
|
|
2475
3497
|
if (allOffsets.length > 1) {
|
|
2476
3498
|
res.writeHead(400, { "content-type": `text/plain` });
|
|
2477
3499
|
res.end(`Multiple offset parameters not allowed`);
|
|
@@ -2503,12 +3525,12 @@ var DurableStreamTestServer = class {
|
|
|
2503
3525
|
const effectiveOffset = offset === `now` ? stream.currentOffset : offset;
|
|
2504
3526
|
if (offset === `now` && live !== `long-poll`) {
|
|
2505
3527
|
const headers$1 = {
|
|
2506
|
-
[STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
2507
|
-
[STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
3528
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
3529
|
+
[__durable_streams_client.STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
2508
3530
|
[`cache-control`]: `no-store`
|
|
2509
3531
|
};
|
|
2510
3532
|
if (stream.contentType) headers$1[`content-type`] = stream.contentType;
|
|
2511
|
-
if (stream.closed) headers$1[STREAM_CLOSED_HEADER] = `true`;
|
|
3533
|
+
if (stream.closed) headers$1[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2512
3534
|
const isJsonMode = stream.contentType?.includes(`application/json`);
|
|
2513
3535
|
const responseBody = isJsonMode ? `[]` : ``;
|
|
2514
3536
|
res.writeHead(200, headers$1);
|
|
@@ -2521,9 +3543,9 @@ var DurableStreamTestServer = class {
|
|
|
2521
3543
|
if (live === `long-poll` && clientIsCaughtUp && messages.length === 0) {
|
|
2522
3544
|
if (stream.closed) {
|
|
2523
3545
|
res.writeHead(204, {
|
|
2524
|
-
[STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
2525
|
-
[STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
2526
|
-
[STREAM_CLOSED_HEADER]: `true`
|
|
3546
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
3547
|
+
[__durable_streams_client.STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
3548
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`
|
|
2527
3549
|
});
|
|
2528
3550
|
res.end();
|
|
2529
3551
|
return;
|
|
@@ -2533,10 +3555,10 @@ var DurableStreamTestServer = class {
|
|
|
2533
3555
|
if (result.streamClosed) {
|
|
2534
3556
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2535
3557
|
res.writeHead(204, {
|
|
2536
|
-
[STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
|
|
2537
|
-
[STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
2538
|
-
[STREAM_CURSOR_HEADER]: responseCursor,
|
|
2539
|
-
[STREAM_CLOSED_HEADER]: `true`
|
|
3558
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
|
|
3559
|
+
[__durable_streams_client.STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
3560
|
+
[__durable_streams_client.STREAM_CURSOR_HEADER]: responseCursor,
|
|
3561
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`
|
|
2540
3562
|
});
|
|
2541
3563
|
res.end();
|
|
2542
3564
|
return;
|
|
@@ -2545,11 +3567,11 @@ var DurableStreamTestServer = class {
|
|
|
2545
3567
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2546
3568
|
const currentStream$1 = this.store.get(path);
|
|
2547
3569
|
const timeoutHeaders = {
|
|
2548
|
-
[STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
|
|
2549
|
-
[STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
2550
|
-
[STREAM_CURSOR_HEADER]: responseCursor
|
|
3570
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
|
|
3571
|
+
[__durable_streams_client.STREAM_UP_TO_DATE_HEADER]: `true`,
|
|
3572
|
+
[__durable_streams_client.STREAM_CURSOR_HEADER]: responseCursor
|
|
2551
3573
|
};
|
|
2552
|
-
if (currentStream$1?.closed) timeoutHeaders[STREAM_CLOSED_HEADER] = `true`;
|
|
3574
|
+
if (currentStream$1?.closed) timeoutHeaders[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2553
3575
|
res.writeHead(204, timeoutHeaders);
|
|
2554
3576
|
res.end();
|
|
2555
3577
|
return;
|
|
@@ -2561,12 +3583,12 @@ var DurableStreamTestServer = class {
|
|
|
2561
3583
|
if (stream.contentType) headers[`content-type`] = stream.contentType;
|
|
2562
3584
|
const lastMessage = messages[messages.length - 1];
|
|
2563
3585
|
const responseOffset = lastMessage?.offset ?? stream.currentOffset;
|
|
2564
|
-
headers[STREAM_OFFSET_HEADER] = responseOffset;
|
|
2565
|
-
if (live === `long-poll`) headers[STREAM_CURSOR_HEADER] = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2566
|
-
if (upToDate) headers[STREAM_UP_TO_DATE_HEADER] = `true`;
|
|
3586
|
+
headers[__durable_streams_client.STREAM_OFFSET_HEADER] = responseOffset;
|
|
3587
|
+
if (live === `long-poll`) headers[__durable_streams_client.STREAM_CURSOR_HEADER] = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
3588
|
+
if (upToDate) headers[__durable_streams_client.STREAM_UP_TO_DATE_HEADER] = `true`;
|
|
2567
3589
|
const currentStream = this.store.get(path);
|
|
2568
3590
|
const clientAtTail = responseOffset === currentStream?.currentOffset;
|
|
2569
|
-
if (currentStream?.closed && clientAtTail && upToDate) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
3591
|
+
if (currentStream?.closed && clientAtTail && upToDate) headers[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2570
3592
|
const startOffset = offset ?? `-1`;
|
|
2571
3593
|
const closedSuffix = currentStream?.closed && clientAtTail && upToDate ? `:c` : ``;
|
|
2572
3594
|
const etag = `"${Buffer.from(path).toString(`base64`)}:${startOffset}:${responseOffset}${closedSuffix}"`;
|
|
@@ -2639,10 +3661,10 @@ var DurableStreamTestServer = class {
|
|
|
2639
3661
|
const streamIsClosed = currentStream?.closed ?? false;
|
|
2640
3662
|
const clientAtTail = controlOffset === currentStream.currentOffset;
|
|
2641
3663
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2642
|
-
const controlData = { [SSE_OFFSET_FIELD]: controlOffset };
|
|
2643
|
-
if (streamIsClosed && clientAtTail) controlData[SSE_CLOSED_FIELD] = true;
|
|
3664
|
+
const controlData = { [__durable_streams_client.SSE_OFFSET_FIELD]: controlOffset };
|
|
3665
|
+
if (streamIsClosed && clientAtTail) controlData[__durable_streams_client.SSE_CLOSED_FIELD] = true;
|
|
2644
3666
|
else {
|
|
2645
|
-
controlData[SSE_CURSOR_FIELD] = responseCursor;
|
|
3667
|
+
controlData[__durable_streams_client.SSE_CURSOR_FIELD] = responseCursor;
|
|
2646
3668
|
if (upToDate) controlData[SSE_UP_TO_DATE_FIELD] = true;
|
|
2647
3669
|
}
|
|
2648
3670
|
res.write(`event: control\n`);
|
|
@@ -2652,8 +3674,8 @@ var DurableStreamTestServer = class {
|
|
|
2652
3674
|
if (upToDate) {
|
|
2653
3675
|
if (currentStream?.closed) {
|
|
2654
3676
|
const finalControlData = {
|
|
2655
|
-
[SSE_OFFSET_FIELD]: currentOffset,
|
|
2656
|
-
[SSE_CLOSED_FIELD]: true
|
|
3677
|
+
[__durable_streams_client.SSE_OFFSET_FIELD]: currentOffset,
|
|
3678
|
+
[__durable_streams_client.SSE_CLOSED_FIELD]: true
|
|
2657
3679
|
};
|
|
2658
3680
|
res.write(`event: control\n`);
|
|
2659
3681
|
res.write(encodeSSEData(JSON.stringify(finalControlData)));
|
|
@@ -2664,8 +3686,8 @@ var DurableStreamTestServer = class {
|
|
|
2664
3686
|
if (this.isShuttingDown || !isConnected) break;
|
|
2665
3687
|
if (result.streamClosed && result.messages.length === 0) {
|
|
2666
3688
|
const finalControlData = {
|
|
2667
|
-
[SSE_OFFSET_FIELD]: currentOffset,
|
|
2668
|
-
[SSE_CLOSED_FIELD]: true
|
|
3689
|
+
[__durable_streams_client.SSE_OFFSET_FIELD]: currentOffset,
|
|
3690
|
+
[__durable_streams_client.SSE_CLOSED_FIELD]: true
|
|
2669
3691
|
};
|
|
2670
3692
|
res.write(`event: control\n`);
|
|
2671
3693
|
res.write(encodeSSEData(JSON.stringify(finalControlData)));
|
|
@@ -2676,16 +3698,16 @@ var DurableStreamTestServer = class {
|
|
|
2676
3698
|
const streamAfterWait = this.store.get(path);
|
|
2677
3699
|
if (streamAfterWait?.closed) {
|
|
2678
3700
|
const closedControlData = {
|
|
2679
|
-
[SSE_OFFSET_FIELD]: currentOffset,
|
|
2680
|
-
[SSE_CLOSED_FIELD]: true
|
|
3701
|
+
[__durable_streams_client.SSE_OFFSET_FIELD]: currentOffset,
|
|
3702
|
+
[__durable_streams_client.SSE_CLOSED_FIELD]: true
|
|
2681
3703
|
};
|
|
2682
3704
|
res.write(`event: control\n`);
|
|
2683
3705
|
res.write(encodeSSEData(JSON.stringify(closedControlData)));
|
|
2684
3706
|
break;
|
|
2685
3707
|
}
|
|
2686
3708
|
const keepAliveData = {
|
|
2687
|
-
[SSE_OFFSET_FIELD]: currentOffset,
|
|
2688
|
-
[SSE_CURSOR_FIELD]: keepAliveCursor,
|
|
3709
|
+
[__durable_streams_client.SSE_OFFSET_FIELD]: currentOffset,
|
|
3710
|
+
[__durable_streams_client.SSE_CURSOR_FIELD]: keepAliveCursor,
|
|
2689
3711
|
[SSE_UP_TO_DATE_FIELD]: true
|
|
2690
3712
|
};
|
|
2691
3713
|
res.write(`event: control\n` + encodeSSEData(JSON.stringify(keepAliveData)));
|
|
@@ -2700,12 +3722,12 @@ var DurableStreamTestServer = class {
|
|
|
2700
3722
|
*/
|
|
2701
3723
|
async handleAppend(path, req, res) {
|
|
2702
3724
|
const contentType = req.headers[`content-type`];
|
|
2703
|
-
const seq = req.headers[STREAM_SEQ_HEADER.toLowerCase()];
|
|
2704
|
-
const closedHeader = req.headers[STREAM_CLOSED_HEADER.toLowerCase()];
|
|
3725
|
+
const seq = req.headers[__durable_streams_client.STREAM_SEQ_HEADER.toLowerCase()];
|
|
3726
|
+
const closedHeader = req.headers[__durable_streams_client.STREAM_CLOSED_HEADER.toLowerCase()];
|
|
2705
3727
|
const closeStream = closedHeader === `true`;
|
|
2706
|
-
const producerId = req.headers[PRODUCER_ID_HEADER.toLowerCase()];
|
|
2707
|
-
const producerEpochStr = req.headers[PRODUCER_EPOCH_HEADER.toLowerCase()];
|
|
2708
|
-
const producerSeqStr = req.headers[PRODUCER_SEQ_HEADER.toLowerCase()];
|
|
3728
|
+
const producerId = req.headers[__durable_streams_client.PRODUCER_ID_HEADER.toLowerCase()];
|
|
3729
|
+
const producerEpochStr = req.headers[__durable_streams_client.PRODUCER_EPOCH_HEADER.toLowerCase()];
|
|
3730
|
+
const producerSeqStr = req.headers[__durable_streams_client.PRODUCER_SEQ_HEADER.toLowerCase()];
|
|
2709
3731
|
const hasProducerHeaders = producerId !== void 0 || producerEpochStr !== void 0 || producerSeqStr !== void 0;
|
|
2710
3732
|
const hasAllProducerHeaders = producerId !== void 0 && producerEpochStr !== void 0 && producerSeqStr !== void 0;
|
|
2711
3733
|
if (hasProducerHeaders && !hasAllProducerHeaders) {
|
|
@@ -2760,10 +3782,10 @@ var DurableStreamTestServer = class {
|
|
|
2760
3782
|
}
|
|
2761
3783
|
if (closeResult$1.producerResult?.status === `duplicate`) {
|
|
2762
3784
|
res.writeHead(204, {
|
|
2763
|
-
[STREAM_OFFSET_HEADER]: closeResult$1.finalOffset,
|
|
2764
|
-
[STREAM_CLOSED_HEADER]: `true`,
|
|
2765
|
-
[PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
2766
|
-
[PRODUCER_SEQ_HEADER]: closeResult$1.producerResult.lastSeq.toString()
|
|
3785
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: closeResult$1.finalOffset,
|
|
3786
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`,
|
|
3787
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
3788
|
+
[__durable_streams_client.PRODUCER_SEQ_HEADER]: closeResult$1.producerResult.lastSeq.toString()
|
|
2767
3789
|
});
|
|
2768
3790
|
res.end();
|
|
2769
3791
|
return;
|
|
@@ -2771,7 +3793,7 @@ var DurableStreamTestServer = class {
|
|
|
2771
3793
|
if (closeResult$1.producerResult?.status === `stale_epoch`) {
|
|
2772
3794
|
res.writeHead(403, {
|
|
2773
3795
|
"content-type": `text/plain`,
|
|
2774
|
-
[PRODUCER_EPOCH_HEADER]: closeResult$1.producerResult.currentEpoch.toString()
|
|
3796
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: closeResult$1.producerResult.currentEpoch.toString()
|
|
2775
3797
|
});
|
|
2776
3798
|
res.end(`Stale producer epoch`);
|
|
2777
3799
|
return;
|
|
@@ -2784,8 +3806,8 @@ var DurableStreamTestServer = class {
|
|
|
2784
3806
|
if (closeResult$1.producerResult?.status === `sequence_gap`) {
|
|
2785
3807
|
res.writeHead(409, {
|
|
2786
3808
|
"content-type": `text/plain`,
|
|
2787
|
-
[PRODUCER_EXPECTED_SEQ_HEADER]: closeResult$1.producerResult.expectedSeq.toString(),
|
|
2788
|
-
[PRODUCER_RECEIVED_SEQ_HEADER]: closeResult$1.producerResult.receivedSeq.toString()
|
|
3809
|
+
[__durable_streams_client.PRODUCER_EXPECTED_SEQ_HEADER]: closeResult$1.producerResult.expectedSeq.toString(),
|
|
3810
|
+
[__durable_streams_client.PRODUCER_RECEIVED_SEQ_HEADER]: closeResult$1.producerResult.receivedSeq.toString()
|
|
2789
3811
|
});
|
|
2790
3812
|
res.end(`Producer sequence gap`);
|
|
2791
3813
|
return;
|
|
@@ -2794,17 +3816,17 @@ var DurableStreamTestServer = class {
|
|
|
2794
3816
|
const stream = this.store.get(path);
|
|
2795
3817
|
res.writeHead(409, {
|
|
2796
3818
|
"content-type": `text/plain`,
|
|
2797
|
-
[STREAM_CLOSED_HEADER]: `true`,
|
|
2798
|
-
[STREAM_OFFSET_HEADER]: stream?.currentOffset ?? ``
|
|
3819
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`,
|
|
3820
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream?.currentOffset ?? ``
|
|
2799
3821
|
});
|
|
2800
3822
|
res.end(`Stream is closed`);
|
|
2801
3823
|
return;
|
|
2802
3824
|
}
|
|
2803
3825
|
res.writeHead(204, {
|
|
2804
|
-
[STREAM_OFFSET_HEADER]: closeResult$1.finalOffset,
|
|
2805
|
-
[STREAM_CLOSED_HEADER]: `true`,
|
|
2806
|
-
[PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
2807
|
-
[PRODUCER_SEQ_HEADER]: producerSeq.toString()
|
|
3826
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: closeResult$1.finalOffset,
|
|
3827
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`,
|
|
3828
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
3829
|
+
[__durable_streams_client.PRODUCER_SEQ_HEADER]: producerSeq.toString()
|
|
2808
3830
|
});
|
|
2809
3831
|
res.end();
|
|
2810
3832
|
return;
|
|
@@ -2816,8 +3838,8 @@ var DurableStreamTestServer = class {
|
|
|
2816
3838
|
return;
|
|
2817
3839
|
}
|
|
2818
3840
|
res.writeHead(204, {
|
|
2819
|
-
[STREAM_OFFSET_HEADER]: closeResult.finalOffset,
|
|
2820
|
-
[STREAM_CLOSED_HEADER]: `true`
|
|
3841
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: closeResult.finalOffset,
|
|
3842
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`
|
|
2821
3843
|
});
|
|
2822
3844
|
res.end();
|
|
2823
3845
|
return;
|
|
@@ -2850,10 +3872,10 @@ var DurableStreamTestServer = class {
|
|
|
2850
3872
|
if (producerResult?.status === `duplicate`) {
|
|
2851
3873
|
const stream = this.store.get(path);
|
|
2852
3874
|
res.writeHead(204, {
|
|
2853
|
-
[STREAM_OFFSET_HEADER]: stream?.currentOffset ?? ``,
|
|
2854
|
-
[STREAM_CLOSED_HEADER]: `true`,
|
|
2855
|
-
[PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
2856
|
-
[PRODUCER_SEQ_HEADER]: producerResult.lastSeq.toString()
|
|
3875
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: stream?.currentOffset ?? ``,
|
|
3876
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`,
|
|
3877
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
3878
|
+
[__durable_streams_client.PRODUCER_SEQ_HEADER]: producerResult.lastSeq.toString()
|
|
2857
3879
|
});
|
|
2858
3880
|
res.end();
|
|
2859
3881
|
return;
|
|
@@ -2861,29 +3883,30 @@ var DurableStreamTestServer = class {
|
|
|
2861
3883
|
const closedStream = this.store.get(path);
|
|
2862
3884
|
res.writeHead(409, {
|
|
2863
3885
|
"content-type": `text/plain`,
|
|
2864
|
-
[STREAM_CLOSED_HEADER]: `true`,
|
|
2865
|
-
[STREAM_OFFSET_HEADER]: closedStream?.currentOffset ?? ``
|
|
3886
|
+
[__durable_streams_client.STREAM_CLOSED_HEADER]: `true`,
|
|
3887
|
+
[__durable_streams_client.STREAM_OFFSET_HEADER]: closedStream?.currentOffset ?? ``
|
|
2866
3888
|
});
|
|
2867
3889
|
res.end(`Stream is closed`);
|
|
2868
3890
|
return;
|
|
2869
3891
|
}
|
|
2870
3892
|
if (!producerResult || producerResult.status === `accepted`) {
|
|
2871
|
-
const responseHeaders$1 = { [STREAM_OFFSET_HEADER]: message$1.offset };
|
|
2872
|
-
if (producerEpoch !== void 0) responseHeaders$1[PRODUCER_EPOCH_HEADER] = producerEpoch.toString();
|
|
2873
|
-
if (producerSeq !== void 0) responseHeaders$1[PRODUCER_SEQ_HEADER] = producerSeq.toString();
|
|
2874
|
-
if (streamClosed) responseHeaders$1[STREAM_CLOSED_HEADER] = `true`;
|
|
3893
|
+
const responseHeaders$1 = { [__durable_streams_client.STREAM_OFFSET_HEADER]: message$1.offset };
|
|
3894
|
+
if (producerEpoch !== void 0) responseHeaders$1[__durable_streams_client.PRODUCER_EPOCH_HEADER] = producerEpoch.toString();
|
|
3895
|
+
if (producerSeq !== void 0) responseHeaders$1[__durable_streams_client.PRODUCER_SEQ_HEADER] = producerSeq.toString();
|
|
3896
|
+
if (streamClosed) responseHeaders$1[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2875
3897
|
const statusCode = producerId !== void 0 ? 200 : 204;
|
|
2876
3898
|
res.writeHead(statusCode, responseHeaders$1);
|
|
2877
3899
|
res.end();
|
|
3900
|
+
await this.notifyStreamAppend(path);
|
|
2878
3901
|
return;
|
|
2879
3902
|
}
|
|
2880
3903
|
switch (producerResult.status) {
|
|
2881
3904
|
case `duplicate`: {
|
|
2882
3905
|
const dupHeaders = {
|
|
2883
|
-
[PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
2884
|
-
[PRODUCER_SEQ_HEADER]: producerResult.lastSeq.toString()
|
|
3906
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
|
|
3907
|
+
[__durable_streams_client.PRODUCER_SEQ_HEADER]: producerResult.lastSeq.toString()
|
|
2885
3908
|
};
|
|
2886
|
-
if (streamClosed) dupHeaders[STREAM_CLOSED_HEADER] = `true`;
|
|
3909
|
+
if (streamClosed) dupHeaders[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2887
3910
|
res.writeHead(204, dupHeaders);
|
|
2888
3911
|
res.end();
|
|
2889
3912
|
return;
|
|
@@ -2891,7 +3914,7 @@ var DurableStreamTestServer = class {
|
|
|
2891
3914
|
case `stale_epoch`: {
|
|
2892
3915
|
res.writeHead(403, {
|
|
2893
3916
|
"content-type": `text/plain`,
|
|
2894
|
-
[PRODUCER_EPOCH_HEADER]: producerResult.currentEpoch.toString()
|
|
3917
|
+
[__durable_streams_client.PRODUCER_EPOCH_HEADER]: producerResult.currentEpoch.toString()
|
|
2895
3918
|
});
|
|
2896
3919
|
res.end(`Stale producer epoch`);
|
|
2897
3920
|
return;
|
|
@@ -2903,18 +3926,27 @@ var DurableStreamTestServer = class {
|
|
|
2903
3926
|
case `sequence_gap`:
|
|
2904
3927
|
res.writeHead(409, {
|
|
2905
3928
|
"content-type": `text/plain`,
|
|
2906
|
-
[PRODUCER_EXPECTED_SEQ_HEADER]: producerResult.expectedSeq.toString(),
|
|
2907
|
-
[PRODUCER_RECEIVED_SEQ_HEADER]: producerResult.receivedSeq.toString()
|
|
3929
|
+
[__durable_streams_client.PRODUCER_EXPECTED_SEQ_HEADER]: producerResult.expectedSeq.toString(),
|
|
3930
|
+
[__durable_streams_client.PRODUCER_RECEIVED_SEQ_HEADER]: producerResult.receivedSeq.toString()
|
|
2908
3931
|
});
|
|
2909
3932
|
res.end(`Producer sequence gap`);
|
|
2910
3933
|
return;
|
|
2911
3934
|
}
|
|
2912
3935
|
}
|
|
2913
3936
|
const message = result;
|
|
2914
|
-
const responseHeaders = { [STREAM_OFFSET_HEADER]: message.offset };
|
|
2915
|
-
if (closeStream) responseHeaders[STREAM_CLOSED_HEADER] = `true`;
|
|
3937
|
+
const responseHeaders = { [__durable_streams_client.STREAM_OFFSET_HEADER]: message.offset };
|
|
3938
|
+
if (closeStream) responseHeaders[__durable_streams_client.STREAM_CLOSED_HEADER] = `true`;
|
|
2916
3939
|
res.writeHead(204, responseHeaders);
|
|
2917
3940
|
res.end();
|
|
3941
|
+
await this.notifyStreamAppend(path);
|
|
3942
|
+
}
|
|
3943
|
+
async notifyStreamAppend(path) {
|
|
3944
|
+
if (!this.subscriptionManager) return;
|
|
3945
|
+
try {
|
|
3946
|
+
await this.subscriptionManager.onStreamAppend(path);
|
|
3947
|
+
} catch (err) {
|
|
3948
|
+
serverLog.error(`[server] subscription append hook failed:`, err);
|
|
3949
|
+
}
|
|
2918
3950
|
}
|
|
2919
3951
|
/**
|
|
2920
3952
|
* Handle DELETE - delete stream
|
|
@@ -2937,6 +3969,7 @@ var DurableStreamTestServer = class {
|
|
|
2937
3969
|
path,
|
|
2938
3970
|
timestamp: Date.now()
|
|
2939
3971
|
}));
|
|
3972
|
+
if (this.subscriptionManager) this.subscriptionManager.onStreamDeleted(path);
|
|
2940
3973
|
res.writeHead(204);
|
|
2941
3974
|
res.end();
|
|
2942
3975
|
}
|
|
@@ -3071,9 +4104,13 @@ exports.DEFAULT_CURSOR_INTERVAL_SECONDS = DEFAULT_CURSOR_INTERVAL_SECONDS
|
|
|
3071
4104
|
exports.DurableStreamTestServer = DurableStreamTestServer
|
|
3072
4105
|
exports.FileBackedStreamStore = FileBackedStreamStore
|
|
3073
4106
|
exports.StreamStore = StreamStore
|
|
4107
|
+
exports.SubscriptionManager = SubscriptionManager
|
|
4108
|
+
exports.SubscriptionRoutes = SubscriptionRoutes
|
|
3074
4109
|
exports.calculateCursor = calculateCursor
|
|
3075
4110
|
exports.createRegistryHooks = createRegistryHooks
|
|
3076
4111
|
exports.decodeStreamPath = decodeStreamPath
|
|
3077
4112
|
exports.encodeStreamPath = encodeStreamPath
|
|
3078
4113
|
exports.generateResponseCursor = generateResponseCursor
|
|
3079
|
-
exports.
|
|
4114
|
+
exports.globMatch = globMatch
|
|
4115
|
+
exports.handleCursorCollision = handleCursorCollision
|
|
4116
|
+
exports.validateWebhookUrl = validateWebhookUrl
|