@durable-streams/server 0.2.3 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +541 -89
- package/dist/index.d.cts +97 -6
- package/dist/index.d.ts +97 -6
- package/dist/index.js +541 -89
- package/package.json +3 -3
- package/src/file-store.ts +491 -90
- package/src/server.ts +108 -16
- package/src/store.ts +363 -36
- package/src/types.ts +29 -0
package/dist/index.js
CHANGED
|
@@ -80,72 +80,180 @@ var StreamStore = class {
|
|
|
80
80
|
if (!Number.isFinite(expiryTime) || now >= expiryTime) return true;
|
|
81
81
|
}
|
|
82
82
|
if (stream.ttlSeconds !== void 0) {
|
|
83
|
-
const expiryTime = stream.
|
|
83
|
+
const expiryTime = stream.lastAccessedAt + stream.ttlSeconds * 1e3;
|
|
84
84
|
if (now >= expiryTime) return true;
|
|
85
85
|
}
|
|
86
86
|
return false;
|
|
87
87
|
}
|
|
88
88
|
/**
|
|
89
|
-
* Get a stream,
|
|
90
|
-
* Returns undefined if stream doesn't exist or is expired.
|
|
89
|
+
* Get a stream, handling expiry.
|
|
90
|
+
* Returns undefined if stream doesn't exist or is expired (and has no refs).
|
|
91
|
+
* Expired streams with refCount > 0 are soft-deleted instead of fully deleted.
|
|
91
92
|
*/
|
|
92
93
|
getIfNotExpired(path$2) {
|
|
93
94
|
const stream = this.streams.get(path$2);
|
|
94
95
|
if (!stream) return void 0;
|
|
95
96
|
if (this.isExpired(stream)) {
|
|
97
|
+
if (stream.refCount > 0) {
|
|
98
|
+
stream.softDeleted = true;
|
|
99
|
+
return stream;
|
|
100
|
+
}
|
|
96
101
|
this.delete(path$2);
|
|
97
102
|
return void 0;
|
|
98
103
|
}
|
|
99
104
|
return stream;
|
|
100
105
|
}
|
|
101
106
|
/**
|
|
107
|
+
* Update lastAccessedAt to now. Called on reads and appends (not HEAD).
|
|
108
|
+
*/
|
|
109
|
+
touchAccess(path$2) {
|
|
110
|
+
const stream = this.streams.get(path$2);
|
|
111
|
+
if (stream) stream.lastAccessedAt = Date.now();
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
102
114
|
* Create a new stream.
|
|
103
115
|
* @throws Error if stream already exists with different config
|
|
116
|
+
* @throws Error if fork source not found, soft-deleted, or offset invalid
|
|
104
117
|
* @returns existing stream if config matches (idempotent)
|
|
105
118
|
*/
|
|
106
119
|
create(path$2, options = {}) {
|
|
107
|
-
const
|
|
108
|
-
if (
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
120
|
+
const existingRaw = this.streams.get(path$2);
|
|
121
|
+
if (existingRaw) if (this.isExpired(existingRaw)) {
|
|
122
|
+
this.streams.delete(path$2);
|
|
123
|
+
this.cancelLongPollsForStream(path$2);
|
|
124
|
+
} else if (existingRaw.softDeleted) throw new Error(`Stream has active forks — path cannot be reused until all forks are removed: ${path$2}`);
|
|
125
|
+
else {
|
|
126
|
+
const contentTypeMatches = (normalizeContentType(options.contentType) || `application/octet-stream`) === (normalizeContentType(existingRaw.contentType) || `application/octet-stream`);
|
|
127
|
+
const ttlMatches = options.ttlSeconds === existingRaw.ttlSeconds;
|
|
128
|
+
const expiresMatches = options.expiresAt === existingRaw.expiresAt;
|
|
129
|
+
const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
|
|
130
|
+
const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
|
|
131
|
+
const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
|
|
132
|
+
if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches) return existingRaw;
|
|
114
133
|
else throw new Error(`Stream already exists with different configuration: ${path$2}`);
|
|
115
134
|
}
|
|
135
|
+
const isFork = !!options.forkedFrom;
|
|
136
|
+
let forkOffset = `0000000000000000_0000000000000000`;
|
|
137
|
+
let sourceContentType;
|
|
138
|
+
let sourceStream;
|
|
139
|
+
if (isFork) {
|
|
140
|
+
sourceStream = this.streams.get(options.forkedFrom);
|
|
141
|
+
if (!sourceStream) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
142
|
+
if (sourceStream.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
|
|
143
|
+
if (this.isExpired(sourceStream)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
144
|
+
sourceContentType = sourceStream.contentType;
|
|
145
|
+
if (options.forkOffset) forkOffset = options.forkOffset;
|
|
146
|
+
else forkOffset = sourceStream.currentOffset;
|
|
147
|
+
const zeroOffset = `0000000000000000_0000000000000000`;
|
|
148
|
+
if (forkOffset < zeroOffset || sourceStream.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
|
|
149
|
+
sourceStream.refCount++;
|
|
150
|
+
}
|
|
151
|
+
let contentType = options.contentType;
|
|
152
|
+
if (!contentType || contentType.trim() === ``) {
|
|
153
|
+
if (isFork) contentType = sourceContentType;
|
|
154
|
+
} else if (isFork && normalizeContentType(contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
|
|
155
|
+
let effectiveExpiresAt = options.expiresAt;
|
|
156
|
+
let effectiveTtlSeconds = options.ttlSeconds;
|
|
157
|
+
if (isFork) {
|
|
158
|
+
const resolved = this.resolveForkExpiry(options, sourceStream);
|
|
159
|
+
effectiveExpiresAt = resolved.expiresAt;
|
|
160
|
+
effectiveTtlSeconds = resolved.ttlSeconds;
|
|
161
|
+
}
|
|
116
162
|
const stream = {
|
|
117
163
|
path: path$2,
|
|
118
|
-
contentType
|
|
164
|
+
contentType,
|
|
119
165
|
messages: [],
|
|
120
|
-
currentOffset: `0000000000000000_0000000000000000`,
|
|
121
|
-
ttlSeconds:
|
|
122
|
-
expiresAt:
|
|
166
|
+
currentOffset: isFork ? forkOffset : `0000000000000000_0000000000000000`,
|
|
167
|
+
ttlSeconds: effectiveTtlSeconds,
|
|
168
|
+
expiresAt: effectiveExpiresAt,
|
|
123
169
|
createdAt: Date.now(),
|
|
124
|
-
|
|
170
|
+
lastAccessedAt: Date.now(),
|
|
171
|
+
closed: options.closed ?? false,
|
|
172
|
+
refCount: 0,
|
|
173
|
+
forkedFrom: isFork ? options.forkedFrom : void 0,
|
|
174
|
+
forkOffset: isFork ? forkOffset : void 0
|
|
125
175
|
};
|
|
126
|
-
if (options.initialData && options.initialData.length > 0)
|
|
176
|
+
if (options.initialData && options.initialData.length > 0) try {
|
|
177
|
+
this.appendToStream(stream, options.initialData, true);
|
|
178
|
+
} catch (err) {
|
|
179
|
+
if (isFork && sourceStream) sourceStream.refCount--;
|
|
180
|
+
throw err;
|
|
181
|
+
}
|
|
127
182
|
this.streams.set(path$2, stream);
|
|
128
183
|
return stream;
|
|
129
184
|
}
|
|
130
185
|
/**
|
|
186
|
+
* Resolve fork expiry per the decision table.
|
|
187
|
+
* Forks have independent lifetimes — no capping at source expiry.
|
|
188
|
+
*/
|
|
189
|
+
resolveForkExpiry(opts, sourceMeta) {
|
|
190
|
+
if (opts.ttlSeconds !== void 0) return { ttlSeconds: opts.ttlSeconds };
|
|
191
|
+
if (opts.expiresAt) return { expiresAt: opts.expiresAt };
|
|
192
|
+
if (sourceMeta.ttlSeconds !== void 0) return { ttlSeconds: sourceMeta.ttlSeconds };
|
|
193
|
+
if (sourceMeta.expiresAt) return { expiresAt: sourceMeta.expiresAt };
|
|
194
|
+
return {};
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
131
197
|
* Get a stream by path.
|
|
132
198
|
* Returns undefined if stream doesn't exist or is expired.
|
|
199
|
+
* Returns soft-deleted streams (caller should check stream.softDeleted).
|
|
133
200
|
*/
|
|
134
201
|
get(path$2) {
|
|
135
|
-
|
|
202
|
+
const stream = this.streams.get(path$2);
|
|
203
|
+
if (!stream) return void 0;
|
|
204
|
+
if (this.isExpired(stream)) {
|
|
205
|
+
if (stream.refCount > 0) {
|
|
206
|
+
stream.softDeleted = true;
|
|
207
|
+
return stream;
|
|
208
|
+
}
|
|
209
|
+
this.delete(path$2);
|
|
210
|
+
return void 0;
|
|
211
|
+
}
|
|
212
|
+
return stream;
|
|
136
213
|
}
|
|
137
214
|
/**
|
|
138
|
-
* Check if a stream exists
|
|
215
|
+
* Check if a stream exists, is not expired, and is not soft-deleted.
|
|
139
216
|
*/
|
|
140
217
|
has(path$2) {
|
|
141
|
-
|
|
218
|
+
const stream = this.get(path$2);
|
|
219
|
+
if (!stream) return false;
|
|
220
|
+
if (stream.softDeleted) return false;
|
|
221
|
+
return true;
|
|
142
222
|
}
|
|
143
223
|
/**
|
|
144
224
|
* Delete a stream.
|
|
225
|
+
* If the stream has forks (refCount > 0), it is soft-deleted instead of fully removed.
|
|
226
|
+
* Returns true if the stream was found and deleted (or soft-deleted).
|
|
145
227
|
*/
|
|
146
228
|
delete(path$2) {
|
|
229
|
+
const stream = this.streams.get(path$2);
|
|
230
|
+
if (!stream) return false;
|
|
231
|
+
if (stream.softDeleted) return true;
|
|
232
|
+
if (stream.refCount > 0) {
|
|
233
|
+
stream.softDeleted = true;
|
|
234
|
+
return true;
|
|
235
|
+
}
|
|
236
|
+
this.deleteWithCascade(path$2);
|
|
237
|
+
return true;
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* Fully delete a stream and cascade to soft-deleted parents
|
|
241
|
+
* whose refcount drops to zero.
|
|
242
|
+
*/
|
|
243
|
+
deleteWithCascade(path$2) {
|
|
244
|
+
const stream = this.streams.get(path$2);
|
|
245
|
+
if (!stream) return;
|
|
246
|
+
const forkedFrom = stream.forkedFrom;
|
|
247
|
+
this.streams.delete(path$2);
|
|
147
248
|
this.cancelLongPollsForStream(path$2);
|
|
148
|
-
|
|
249
|
+
if (forkedFrom) {
|
|
250
|
+
const parent = this.streams.get(forkedFrom);
|
|
251
|
+
if (parent) {
|
|
252
|
+
parent.refCount--;
|
|
253
|
+
if (parent.refCount < 0) parent.refCount = 0;
|
|
254
|
+
if (parent.refCount === 0 && parent.softDeleted) this.deleteWithCascade(forkedFrom);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
149
257
|
}
|
|
150
258
|
/**
|
|
151
259
|
* Validate producer state WITHOUT mutating.
|
|
@@ -258,6 +366,7 @@ var StreamStore = class {
|
|
|
258
366
|
append(path$2, data, options = {}) {
|
|
259
367
|
const stream = this.getIfNotExpired(path$2);
|
|
260
368
|
if (!stream) throw new Error(`Stream not found: ${path$2}`);
|
|
369
|
+
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path$2}`);
|
|
261
370
|
if (stream.closed) {
|
|
262
371
|
if (options.producerId && stream.closedBy && stream.closedBy.producerId === options.producerId && stream.closedBy.epoch === options.producerEpoch && stream.closedBy.seq === options.producerSeq) return {
|
|
263
372
|
message: null,
|
|
@@ -334,6 +443,7 @@ var StreamStore = class {
|
|
|
334
443
|
closeStream(path$2) {
|
|
335
444
|
const stream = this.getIfNotExpired(path$2);
|
|
336
445
|
if (!stream) return null;
|
|
446
|
+
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path$2}`);
|
|
337
447
|
const alreadyClosed = stream.closed ?? false;
|
|
338
448
|
stream.closed = true;
|
|
339
449
|
this.notifyLongPollsClosed(path$2);
|
|
@@ -401,15 +511,26 @@ var StreamStore = class {
|
|
|
401
511
|
}
|
|
402
512
|
/**
|
|
403
513
|
* Read messages from a stream starting at the given offset.
|
|
514
|
+
* For forked streams, stitches messages from the source chain and the fork's own messages.
|
|
404
515
|
* @throws Error if stream doesn't exist or is expired
|
|
405
516
|
*/
|
|
406
517
|
read(path$2, offset) {
|
|
407
518
|
const stream = this.getIfNotExpired(path$2);
|
|
408
519
|
if (!stream) throw new Error(`Stream not found: ${path$2}`);
|
|
409
|
-
if (!offset || offset === `-1`)
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
520
|
+
if (!offset || offset === `-1`) {
|
|
521
|
+
if (stream.forkedFrom) {
|
|
522
|
+
const inherited = this.readForkedMessages(stream.forkedFrom, void 0, stream.forkOffset);
|
|
523
|
+
return {
|
|
524
|
+
messages: [...inherited, ...stream.messages],
|
|
525
|
+
upToDate: true
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
return {
|
|
529
|
+
messages: [...stream.messages],
|
|
530
|
+
upToDate: true
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
if (stream.forkedFrom) return this.readFromFork(stream, offset);
|
|
413
534
|
const offsetIndex = this.findOffsetIndex(stream, offset);
|
|
414
535
|
if (offsetIndex === -1) return {
|
|
415
536
|
messages: [],
|
|
@@ -421,6 +542,55 @@ var StreamStore = class {
|
|
|
421
542
|
};
|
|
422
543
|
}
|
|
423
544
|
/**
|
|
545
|
+
* Read from a forked stream, stitching inherited and own messages.
|
|
546
|
+
*/
|
|
547
|
+
readFromFork(stream, offset) {
|
|
548
|
+
const messages = [];
|
|
549
|
+
if (offset < stream.forkOffset) {
|
|
550
|
+
const inherited = this.readForkedMessages(stream.forkedFrom, offset, stream.forkOffset);
|
|
551
|
+
messages.push(...inherited);
|
|
552
|
+
}
|
|
553
|
+
const ownMessages = this.readOwnMessages(stream, offset);
|
|
554
|
+
messages.push(...ownMessages);
|
|
555
|
+
return {
|
|
556
|
+
messages,
|
|
557
|
+
upToDate: true
|
|
558
|
+
};
|
|
559
|
+
}
|
|
560
|
+
/**
|
|
561
|
+
* Read a stream's own messages starting after the given offset.
|
|
562
|
+
*/
|
|
563
|
+
readOwnMessages(stream, offset) {
|
|
564
|
+
const offsetIndex = this.findOffsetIndex(stream, offset);
|
|
565
|
+
if (offsetIndex === -1) return [];
|
|
566
|
+
return stream.messages.slice(offsetIndex);
|
|
567
|
+
}
|
|
568
|
+
/**
|
|
569
|
+
* Recursively read messages from a fork's source chain.
|
|
570
|
+
* Reads from source (and its sources if also forked), capped at forkOffset.
|
|
571
|
+
* Does NOT check softDeleted — forks must read through soft-deleted sources.
|
|
572
|
+
*/
|
|
573
|
+
readForkedMessages(sourcePath, offset, capOffset) {
|
|
574
|
+
const source = this.streams.get(sourcePath);
|
|
575
|
+
if (!source) return [];
|
|
576
|
+
const messages = [];
|
|
577
|
+
if (source.forkedFrom && (!offset || offset < source.forkOffset)) {
|
|
578
|
+
const inherited = this.readForkedMessages(
|
|
579
|
+
source.forkedFrom,
|
|
580
|
+
offset,
|
|
581
|
+
// Cap at the minimum of source's forkOffset and our capOffset
|
|
582
|
+
source.forkOffset < capOffset ? source.forkOffset : capOffset
|
|
583
|
+
);
|
|
584
|
+
messages.push(...inherited);
|
|
585
|
+
}
|
|
586
|
+
for (const msg of source.messages) {
|
|
587
|
+
if (offset && msg.offset <= offset) continue;
|
|
588
|
+
if (msg.offset > capOffset) break;
|
|
589
|
+
messages.push(msg);
|
|
590
|
+
}
|
|
591
|
+
return messages;
|
|
592
|
+
}
|
|
593
|
+
/**
|
|
424
594
|
* Format messages for response.
|
|
425
595
|
* For JSON mode, wraps concatenated data in array brackets.
|
|
426
596
|
* @throws Error if stream doesn't exist or is expired
|
|
@@ -445,6 +615,13 @@ var StreamStore = class {
|
|
|
445
615
|
async waitForMessages(path$2, offset, timeoutMs) {
|
|
446
616
|
const stream = this.getIfNotExpired(path$2);
|
|
447
617
|
if (!stream) throw new Error(`Stream not found: ${path$2}`);
|
|
618
|
+
if (stream.forkedFrom && offset < stream.forkOffset) {
|
|
619
|
+
const { messages: messages$1 } = this.read(path$2, offset);
|
|
620
|
+
return {
|
|
621
|
+
messages: messages$1,
|
|
622
|
+
timedOut: false
|
|
623
|
+
};
|
|
624
|
+
}
|
|
448
625
|
const { messages } = this.read(path$2, offset);
|
|
449
626
|
if (messages.length > 0) return {
|
|
450
627
|
messages,
|
|
@@ -820,7 +997,14 @@ var FileBackedStreamStore = class {
|
|
|
820
997
|
errors++;
|
|
821
998
|
continue;
|
|
822
999
|
}
|
|
823
|
-
const
|
|
1000
|
+
const physicalOffset = this.scanFileForTrueOffset(segmentPath);
|
|
1001
|
+
const physicalBytes = Number(physicalOffset.split(`_`)[1] ?? 0);
|
|
1002
|
+
let trueOffset;
|
|
1003
|
+
if (streamMeta.forkOffset) {
|
|
1004
|
+
const forkBaseByte = Number(streamMeta.forkOffset.split(`_`)[1] ?? 0);
|
|
1005
|
+
const logicalBytes = forkBaseByte + physicalBytes;
|
|
1006
|
+
trueOffset = `${String(0).padStart(16, `0`)}_${String(logicalBytes).padStart(16, `0`)}`;
|
|
1007
|
+
} else trueOffset = physicalOffset;
|
|
824
1008
|
if (trueOffset !== streamMeta.currentOffset) {
|
|
825
1009
|
console.warn(`[FileBackedStreamStore] Recovery: Offset mismatch for ${streamPath}: LMDB says ${streamMeta.currentOffset}, file says ${trueOffset}. Reconciling to file.`);
|
|
826
1010
|
const reconciledMeta = {
|
|
@@ -879,9 +1063,14 @@ var FileBackedStreamStore = class {
|
|
|
879
1063
|
ttlSeconds: meta.ttlSeconds,
|
|
880
1064
|
expiresAt: meta.expiresAt,
|
|
881
1065
|
createdAt: meta.createdAt,
|
|
1066
|
+
lastAccessedAt: meta.lastAccessedAt ?? meta.createdAt,
|
|
882
1067
|
producers,
|
|
883
1068
|
closed: meta.closed,
|
|
884
|
-
closedBy: meta.closedBy
|
|
1069
|
+
closedBy: meta.closedBy,
|
|
1070
|
+
forkedFrom: meta.forkedFrom,
|
|
1071
|
+
forkOffset: meta.forkOffset,
|
|
1072
|
+
refCount: meta.refCount ?? 0,
|
|
1073
|
+
softDeleted: meta.softDeleted
|
|
885
1074
|
};
|
|
886
1075
|
}
|
|
887
1076
|
/**
|
|
@@ -977,6 +1166,20 @@ var FileBackedStreamStore = class {
|
|
|
977
1166
|
return meta.producers[producerId]?.epoch;
|
|
978
1167
|
}
|
|
979
1168
|
/**
|
|
1169
|
+
* Update lastAccessedAt to now. Called on reads and appends (not HEAD).
|
|
1170
|
+
*/
|
|
1171
|
+
touchAccess(streamPath) {
|
|
1172
|
+
const key = `stream:${streamPath}`;
|
|
1173
|
+
const meta = this.db.get(key);
|
|
1174
|
+
if (meta) {
|
|
1175
|
+
const updatedMeta = {
|
|
1176
|
+
...meta,
|
|
1177
|
+
lastAccessedAt: Date.now()
|
|
1178
|
+
};
|
|
1179
|
+
this.db.putSync(key, updatedMeta);
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
/**
|
|
980
1183
|
* Check if a stream is expired based on TTL or Expires-At.
|
|
981
1184
|
*/
|
|
982
1185
|
isExpired(meta) {
|
|
@@ -986,26 +1189,50 @@ var FileBackedStreamStore = class {
|
|
|
986
1189
|
if (!Number.isFinite(expiryTime) || now >= expiryTime) return true;
|
|
987
1190
|
}
|
|
988
1191
|
if (meta.ttlSeconds !== void 0) {
|
|
989
|
-
const
|
|
1192
|
+
const lastAccessed = meta.lastAccessedAt ?? meta.createdAt;
|
|
1193
|
+
const expiryTime = lastAccessed + meta.ttlSeconds * 1e3;
|
|
990
1194
|
if (now >= expiryTime) return true;
|
|
991
1195
|
}
|
|
992
1196
|
return false;
|
|
993
1197
|
}
|
|
994
1198
|
/**
|
|
995
1199
|
* Get stream metadata, deleting it if expired.
|
|
996
|
-
* Returns undefined if stream doesn't exist or is expired.
|
|
1200
|
+
* Returns undefined if stream doesn't exist or is expired (and has no refs).
|
|
1201
|
+
* Expired streams with refCount > 0 are soft-deleted instead of fully deleted.
|
|
997
1202
|
*/
|
|
998
1203
|
getMetaIfNotExpired(streamPath) {
|
|
999
1204
|
const key = `stream:${streamPath}`;
|
|
1000
1205
|
const meta = this.db.get(key);
|
|
1001
1206
|
if (!meta) return void 0;
|
|
1002
1207
|
if (this.isExpired(meta)) {
|
|
1208
|
+
if ((meta.refCount ?? 0) > 0) {
|
|
1209
|
+
if (!meta.softDeleted) {
|
|
1210
|
+
const updatedMeta = {
|
|
1211
|
+
...meta,
|
|
1212
|
+
softDeleted: true
|
|
1213
|
+
};
|
|
1214
|
+
this.db.putSync(key, updatedMeta);
|
|
1215
|
+
return updatedMeta;
|
|
1216
|
+
}
|
|
1217
|
+
return meta;
|
|
1218
|
+
}
|
|
1003
1219
|
this.delete(streamPath);
|
|
1004
1220
|
return void 0;
|
|
1005
1221
|
}
|
|
1006
1222
|
return meta;
|
|
1007
1223
|
}
|
|
1008
1224
|
/**
|
|
1225
|
+
* Resolve fork expiry per the decision table.
|
|
1226
|
+
* Forks have independent lifetimes — no capping at source expiry.
|
|
1227
|
+
*/
|
|
1228
|
+
resolveForkExpiry(opts, sourceMeta) {
|
|
1229
|
+
if (opts.ttlSeconds !== void 0) return { ttlSeconds: opts.ttlSeconds };
|
|
1230
|
+
if (opts.expiresAt) return { expiresAt: opts.expiresAt };
|
|
1231
|
+
if (sourceMeta.ttlSeconds !== void 0) return { ttlSeconds: sourceMeta.ttlSeconds };
|
|
1232
|
+
if (sourceMeta.expiresAt) return { expiresAt: sourceMeta.expiresAt };
|
|
1233
|
+
return {};
|
|
1234
|
+
}
|
|
1235
|
+
/**
|
|
1009
1236
|
* Close the store, closing all file handles and database.
|
|
1010
1237
|
* All data is already fsynced on each append, so no final flush needed.
|
|
1011
1238
|
*/
|
|
@@ -1014,29 +1241,70 @@ var FileBackedStreamStore = class {
|
|
|
1014
1241
|
await this.db.close();
|
|
1015
1242
|
}
|
|
1016
1243
|
async create(streamPath, options = {}) {
|
|
1017
|
-
const
|
|
1018
|
-
if (
|
|
1244
|
+
const existingRaw = this.db.get(`stream:${streamPath}`);
|
|
1245
|
+
if (existingRaw) if (this.isExpired(existingRaw)) this.delete(streamPath);
|
|
1246
|
+
else if (existingRaw.softDeleted) throw new Error(`Stream has active forks — path cannot be reused until all forks are removed: ${streamPath}`);
|
|
1247
|
+
else {
|
|
1019
1248
|
const normalizeMimeType = (ct) => (ct ?? `application/octet-stream`).toLowerCase();
|
|
1020
|
-
const contentTypeMatches = normalizeMimeType(options.contentType) === normalizeMimeType(
|
|
1021
|
-
const ttlMatches = options.ttlSeconds ===
|
|
1022
|
-
const expiresMatches = options.expiresAt ===
|
|
1023
|
-
const closedMatches = (options.closed ?? false) === (
|
|
1024
|
-
|
|
1249
|
+
const contentTypeMatches = normalizeMimeType(options.contentType) === normalizeMimeType(existingRaw.contentType);
|
|
1250
|
+
const ttlMatches = options.ttlSeconds === existingRaw.ttlSeconds;
|
|
1251
|
+
const expiresMatches = options.expiresAt === existingRaw.expiresAt;
|
|
1252
|
+
const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
|
|
1253
|
+
const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
|
|
1254
|
+
const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
|
|
1255
|
+
if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches) return this.streamMetaToStream(existingRaw);
|
|
1025
1256
|
else throw new Error(`Stream already exists with different configuration: ${streamPath}`);
|
|
1026
1257
|
}
|
|
1258
|
+
const isFork = !!options.forkedFrom;
|
|
1259
|
+
let forkOffset = `0000000000000000_0000000000000000`;
|
|
1260
|
+
let sourceContentType;
|
|
1261
|
+
let sourceMeta;
|
|
1262
|
+
if (isFork) {
|
|
1263
|
+
const sourceKey = `stream:${options.forkedFrom}`;
|
|
1264
|
+
sourceMeta = this.db.get(sourceKey);
|
|
1265
|
+
if (!sourceMeta) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
1266
|
+
if (sourceMeta.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
|
|
1267
|
+
if (this.isExpired(sourceMeta)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
1268
|
+
sourceContentType = sourceMeta.contentType;
|
|
1269
|
+
if (options.forkOffset) forkOffset = options.forkOffset;
|
|
1270
|
+
else forkOffset = sourceMeta.currentOffset;
|
|
1271
|
+
const zeroOffset = `0000000000000000_0000000000000000`;
|
|
1272
|
+
if (forkOffset < zeroOffset || sourceMeta.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
|
|
1273
|
+
const freshSource = this.db.get(sourceKey);
|
|
1274
|
+
const updatedSource = {
|
|
1275
|
+
...freshSource,
|
|
1276
|
+
refCount: (freshSource.refCount ?? 0) + 1
|
|
1277
|
+
};
|
|
1278
|
+
this.db.putSync(sourceKey, updatedSource);
|
|
1279
|
+
}
|
|
1280
|
+
let contentType = options.contentType;
|
|
1281
|
+
if (!contentType || contentType.trim() === ``) {
|
|
1282
|
+
if (isFork) contentType = sourceContentType;
|
|
1283
|
+
} else if (isFork && normalizeContentType(contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
|
|
1284
|
+
let effectiveExpiresAt = options.expiresAt;
|
|
1285
|
+
let effectiveTtlSeconds = options.ttlSeconds;
|
|
1286
|
+
if (isFork) {
|
|
1287
|
+
const resolved = this.resolveForkExpiry(options, sourceMeta);
|
|
1288
|
+
effectiveExpiresAt = resolved.expiresAt;
|
|
1289
|
+
effectiveTtlSeconds = resolved.ttlSeconds;
|
|
1290
|
+
}
|
|
1027
1291
|
const key = `stream:${streamPath}`;
|
|
1028
1292
|
const streamMeta = {
|
|
1029
1293
|
path: streamPath,
|
|
1030
|
-
contentType
|
|
1031
|
-
currentOffset: `0000000000000000_0000000000000000`,
|
|
1294
|
+
contentType,
|
|
1295
|
+
currentOffset: isFork ? forkOffset : `0000000000000000_0000000000000000`,
|
|
1032
1296
|
lastSeq: void 0,
|
|
1033
|
-
ttlSeconds:
|
|
1034
|
-
expiresAt:
|
|
1297
|
+
ttlSeconds: effectiveTtlSeconds,
|
|
1298
|
+
expiresAt: effectiveExpiresAt,
|
|
1035
1299
|
createdAt: Date.now(),
|
|
1300
|
+
lastAccessedAt: Date.now(),
|
|
1036
1301
|
segmentCount: 1,
|
|
1037
1302
|
totalBytes: 0,
|
|
1038
1303
|
directoryName: generateUniqueDirectoryName(streamPath),
|
|
1039
|
-
closed: false
|
|
1304
|
+
closed: false,
|
|
1305
|
+
forkedFrom: isFork ? options.forkedFrom : void 0,
|
|
1306
|
+
forkOffset: isFork ? forkOffset : void 0,
|
|
1307
|
+
refCount: 0
|
|
1040
1308
|
};
|
|
1041
1309
|
const streamDir = path.join(this.dataDir, `streams`, streamMeta.directoryName);
|
|
1042
1310
|
try {
|
|
@@ -1044,14 +1312,40 @@ var FileBackedStreamStore = class {
|
|
|
1044
1312
|
const segmentPath = path.join(streamDir, `segment_00000.log`);
|
|
1045
1313
|
fs.writeFileSync(segmentPath, ``);
|
|
1046
1314
|
} catch (err) {
|
|
1315
|
+
if (isFork && sourceMeta) {
|
|
1316
|
+
const sourceKey = `stream:${options.forkedFrom}`;
|
|
1317
|
+
const freshSource = this.db.get(sourceKey);
|
|
1318
|
+
if (freshSource) {
|
|
1319
|
+
const updatedSource = {
|
|
1320
|
+
...freshSource,
|
|
1321
|
+
refCount: Math.max(0, (freshSource.refCount ?? 0) - 1)
|
|
1322
|
+
};
|
|
1323
|
+
this.db.putSync(sourceKey, updatedSource);
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1047
1326
|
console.error(`[FileBackedStreamStore] Error creating stream directory:`, err);
|
|
1048
1327
|
throw err;
|
|
1049
1328
|
}
|
|
1050
1329
|
this.db.putSync(key, streamMeta);
|
|
1051
|
-
if (options.initialData && options.initialData.length > 0)
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1330
|
+
if (options.initialData && options.initialData.length > 0) try {
|
|
1331
|
+
await this.append(streamPath, options.initialData, {
|
|
1332
|
+
contentType: options.contentType,
|
|
1333
|
+
isInitialCreate: true
|
|
1334
|
+
});
|
|
1335
|
+
} catch (err) {
|
|
1336
|
+
if (isFork && sourceMeta) {
|
|
1337
|
+
const sourceKey = `stream:${options.forkedFrom}`;
|
|
1338
|
+
const freshSource = this.db.get(sourceKey);
|
|
1339
|
+
if (freshSource) {
|
|
1340
|
+
const updatedSource = {
|
|
1341
|
+
...freshSource,
|
|
1342
|
+
refCount: Math.max(0, (freshSource.refCount ?? 0) - 1)
|
|
1343
|
+
};
|
|
1344
|
+
this.db.putSync(sourceKey, updatedSource);
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
throw err;
|
|
1348
|
+
}
|
|
1055
1349
|
if (options.closed) {
|
|
1056
1350
|
const updatedMeta = this.db.get(key);
|
|
1057
1351
|
updatedMeta.closed = true;
|
|
@@ -1062,15 +1356,41 @@ var FileBackedStreamStore = class {
|
|
|
1062
1356
|
}
|
|
1063
1357
|
get(streamPath) {
|
|
1064
1358
|
const meta = this.getMetaIfNotExpired(streamPath);
|
|
1065
|
-
|
|
1359
|
+
if (!meta) return void 0;
|
|
1360
|
+
return this.streamMetaToStream(meta);
|
|
1066
1361
|
}
|
|
1067
1362
|
has(streamPath) {
|
|
1068
|
-
|
|
1363
|
+
const meta = this.getMetaIfNotExpired(streamPath);
|
|
1364
|
+
if (!meta) return false;
|
|
1365
|
+
if (meta.softDeleted) return false;
|
|
1366
|
+
return true;
|
|
1069
1367
|
}
|
|
1070
1368
|
delete(streamPath) {
|
|
1071
1369
|
const key = `stream:${streamPath}`;
|
|
1072
1370
|
const streamMeta = this.db.get(key);
|
|
1073
1371
|
if (!streamMeta) return false;
|
|
1372
|
+
if (streamMeta.softDeleted) return true;
|
|
1373
|
+
if ((streamMeta.refCount ?? 0) > 0) {
|
|
1374
|
+
const updatedMeta = {
|
|
1375
|
+
...streamMeta,
|
|
1376
|
+
softDeleted: true
|
|
1377
|
+
};
|
|
1378
|
+
this.db.putSync(key, updatedMeta);
|
|
1379
|
+
this.cancelLongPollsForStream(streamPath);
|
|
1380
|
+
return true;
|
|
1381
|
+
}
|
|
1382
|
+
this.deleteWithCascade(streamPath);
|
|
1383
|
+
return true;
|
|
1384
|
+
}
|
|
1385
|
+
/**
|
|
1386
|
+
* Fully delete a stream and cascade to soft-deleted parents
|
|
1387
|
+
* whose refcount drops to zero.
|
|
1388
|
+
*/
|
|
1389
|
+
deleteWithCascade(streamPath) {
|
|
1390
|
+
const key = `stream:${streamPath}`;
|
|
1391
|
+
const streamMeta = this.db.get(key);
|
|
1392
|
+
if (!streamMeta) return;
|
|
1393
|
+
const forkedFrom = streamMeta.forkedFrom;
|
|
1074
1394
|
this.cancelLongPollsForStream(streamPath);
|
|
1075
1395
|
const segmentPath = path.join(this.dataDir, `streams`, streamMeta.directoryName, `segment_00000.log`);
|
|
1076
1396
|
this.fileHandlePool.closeFileHandle(segmentPath).catch((err) => {
|
|
@@ -1080,11 +1400,24 @@ var FileBackedStreamStore = class {
|
|
|
1080
1400
|
this.fileManager.deleteDirectoryByName(streamMeta.directoryName).catch((err) => {
|
|
1081
1401
|
console.error(`[FileBackedStreamStore] Error deleting stream directory:`, err);
|
|
1082
1402
|
});
|
|
1083
|
-
|
|
1403
|
+
if (forkedFrom) {
|
|
1404
|
+
const parentKey = `stream:${forkedFrom}`;
|
|
1405
|
+
const parentMeta = this.db.get(parentKey);
|
|
1406
|
+
if (parentMeta) {
|
|
1407
|
+
const newRefCount = Math.max(0, (parentMeta.refCount ?? 0) - 1);
|
|
1408
|
+
const updatedParent = {
|
|
1409
|
+
...parentMeta,
|
|
1410
|
+
refCount: newRefCount
|
|
1411
|
+
};
|
|
1412
|
+
this.db.putSync(parentKey, updatedParent);
|
|
1413
|
+
if (newRefCount === 0 && updatedParent.softDeleted) this.deleteWithCascade(forkedFrom);
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1084
1416
|
}
|
|
1085
1417
|
async append(streamPath, data, options = {}) {
|
|
1086
1418
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1087
1419
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1420
|
+
if (streamMeta.softDeleted) throw new Error(`Stream is soft-deleted: ${streamPath}`);
|
|
1088
1421
|
if (streamMeta.closed) {
|
|
1089
1422
|
if (options.producerId && streamMeta.closedBy && streamMeta.closedBy.producerId === options.producerId && streamMeta.closedBy.epoch === options.producerEpoch && streamMeta.closedBy.seq === options.producerSeq) return {
|
|
1090
1423
|
message: null,
|
|
@@ -1269,34 +1602,21 @@ var FileBackedStreamStore = class {
|
|
|
1269
1602
|
releaseLock();
|
|
1270
1603
|
}
|
|
1271
1604
|
}
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
if (streamMeta.currentOffset === `0000000000000000_0000000000000000`) return {
|
|
1282
|
-
messages: [],
|
|
1283
|
-
upToDate: true
|
|
1284
|
-
};
|
|
1285
|
-
if (startByte >= currentByte) return {
|
|
1286
|
-
messages: [],
|
|
1287
|
-
upToDate: true
|
|
1288
|
-
};
|
|
1289
|
-
const streamDir = path.join(this.dataDir, `streams`, streamMeta.directoryName);
|
|
1290
|
-
const segmentPath = path.join(streamDir, `segment_00000.log`);
|
|
1291
|
-
if (!fs.existsSync(segmentPath)) return {
|
|
1292
|
-
messages: [],
|
|
1293
|
-
upToDate: true
|
|
1294
|
-
};
|
|
1605
|
+
/**
|
|
1606
|
+
* Read messages from a specific segment file.
|
|
1607
|
+
* @param segmentPath - Path to the segment file
|
|
1608
|
+
* @param startByte - Start byte offset (skip messages at or before this offset)
|
|
1609
|
+
* @param baseByteOffset - Base byte offset to add to physical offsets (for fork stitching)
|
|
1610
|
+
* @param capByte - Optional cap: stop reading when logical offset exceeds this value
|
|
1611
|
+
* @returns Array of messages with properly computed offsets
|
|
1612
|
+
*/
|
|
1613
|
+
readMessagesFromSegmentFile(segmentPath, startByte, baseByteOffset, capByte) {
|
|
1295
1614
|
const messages = [];
|
|
1615
|
+
if (!fs.existsSync(segmentPath)) return messages;
|
|
1296
1616
|
try {
|
|
1297
1617
|
const fileContent = fs.readFileSync(segmentPath);
|
|
1298
1618
|
let filePos = 0;
|
|
1299
|
-
let
|
|
1619
|
+
let physicalDataOffset = 0;
|
|
1300
1620
|
while (filePos < fileContent.length) {
|
|
1301
1621
|
if (filePos + 4 > fileContent.length) break;
|
|
1302
1622
|
const messageLength = fileContent.readUInt32BE(filePos);
|
|
@@ -1305,16 +1625,72 @@ var FileBackedStreamStore = class {
|
|
|
1305
1625
|
const messageData = fileContent.subarray(filePos, filePos + messageLength);
|
|
1306
1626
|
filePos += messageLength;
|
|
1307
1627
|
filePos += 1;
|
|
1308
|
-
|
|
1309
|
-
|
|
1628
|
+
physicalDataOffset += messageLength;
|
|
1629
|
+
const logicalOffset = baseByteOffset + physicalDataOffset;
|
|
1630
|
+
if (capByte !== void 0 && logicalOffset > capByte) break;
|
|
1631
|
+
if (logicalOffset > startByte) messages.push({
|
|
1310
1632
|
data: new Uint8Array(messageData),
|
|
1311
|
-
offset: `${String(
|
|
1633
|
+
offset: `${String(0).padStart(16, `0`)}_${String(logicalOffset).padStart(16, `0`)}`,
|
|
1312
1634
|
timestamp: 0
|
|
1313
1635
|
});
|
|
1314
|
-
currentDataOffset = messageOffset;
|
|
1315
1636
|
}
|
|
1316
1637
|
} catch (err) {
|
|
1317
|
-
console.error(`[FileBackedStreamStore] Error reading file:`, err);
|
|
1638
|
+
console.error(`[FileBackedStreamStore] Error reading segment file:`, err);
|
|
1639
|
+
}
|
|
1640
|
+
return messages;
|
|
1641
|
+
}
|
|
1642
|
+
/**
|
|
1643
|
+
* Recursively read messages from a fork's source chain.
|
|
1644
|
+
* Reads from source (and its sources if also forked), capped at capByte.
|
|
1645
|
+
* Does NOT check softDeleted -- forks must read through soft-deleted sources.
|
|
1646
|
+
*/
|
|
1647
|
+
readForkedMessages(sourcePath, startByte, capByte) {
|
|
1648
|
+
const sourceKey = `stream:${sourcePath}`;
|
|
1649
|
+
const sourceMeta = this.db.get(sourceKey);
|
|
1650
|
+
if (!sourceMeta) return [];
|
|
1651
|
+
const messages = [];
|
|
1652
|
+
if (sourceMeta.forkedFrom && sourceMeta.forkOffset) {
|
|
1653
|
+
const sourceForkByte = Number(sourceMeta.forkOffset.split(`_`)[1] ?? 0);
|
|
1654
|
+
if (startByte < sourceForkByte) {
|
|
1655
|
+
const inheritedCap = Math.min(sourceForkByte, capByte);
|
|
1656
|
+
const inherited = this.readForkedMessages(sourceMeta.forkedFrom, startByte, inheritedCap);
|
|
1657
|
+
messages.push(...inherited);
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
const segmentPath = path.join(this.dataDir, `streams`, sourceMeta.directoryName, `segment_00000.log`);
|
|
1661
|
+
const sourceBaseByte = sourceMeta.forkOffset ? Number(sourceMeta.forkOffset.split(`_`)[1] ?? 0) : 0;
|
|
1662
|
+
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, sourceBaseByte, capByte);
|
|
1663
|
+
messages.push(...ownMessages);
|
|
1664
|
+
return messages;
|
|
1665
|
+
}
|
|
1666
|
+
read(streamPath, offset) {
|
|
1667
|
+
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1668
|
+
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1669
|
+
const startOffset = offset ?? `0000000000000000_0000000000000000`;
|
|
1670
|
+
const startByte = Number(startOffset.split(`_`)[1] ?? 0);
|
|
1671
|
+
const currentByte = Number(streamMeta.currentOffset.split(`_`)[1] ?? 0);
|
|
1672
|
+
if (streamMeta.currentOffset === `0000000000000000_0000000000000000`) return {
|
|
1673
|
+
messages: [],
|
|
1674
|
+
upToDate: true
|
|
1675
|
+
};
|
|
1676
|
+
if (startByte >= currentByte) return {
|
|
1677
|
+
messages: [],
|
|
1678
|
+
upToDate: true
|
|
1679
|
+
};
|
|
1680
|
+
const messages = [];
|
|
1681
|
+
if (streamMeta.forkedFrom && streamMeta.forkOffset) {
|
|
1682
|
+
const forkByte = Number(streamMeta.forkOffset.split(`_`)[1] ?? 0);
|
|
1683
|
+
if (startByte < forkByte) {
|
|
1684
|
+
const inherited = this.readForkedMessages(streamMeta.forkedFrom, startByte, forkByte);
|
|
1685
|
+
messages.push(...inherited);
|
|
1686
|
+
}
|
|
1687
|
+
const segmentPath = path.join(this.dataDir, `streams`, streamMeta.directoryName, `segment_00000.log`);
|
|
1688
|
+
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, forkByte);
|
|
1689
|
+
messages.push(...ownMessages);
|
|
1690
|
+
} else {
|
|
1691
|
+
const segmentPath = path.join(this.dataDir, `streams`, streamMeta.directoryName, `segment_00000.log`);
|
|
1692
|
+
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, 0);
|
|
1693
|
+
messages.push(...ownMessages);
|
|
1318
1694
|
}
|
|
1319
1695
|
return {
|
|
1320
1696
|
messages,
|
|
@@ -1324,6 +1700,13 @@ var FileBackedStreamStore = class {
|
|
|
1324
1700
|
async waitForMessages(streamPath, offset, timeoutMs) {
|
|
1325
1701
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1326
1702
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1703
|
+
if (streamMeta.forkedFrom && streamMeta.forkOffset && offset < streamMeta.forkOffset) {
|
|
1704
|
+
const { messages: messages$1 } = this.read(streamPath, offset);
|
|
1705
|
+
return {
|
|
1706
|
+
messages: messages$1,
|
|
1707
|
+
timedOut: false
|
|
1708
|
+
};
|
|
1709
|
+
}
|
|
1327
1710
|
if (streamMeta.closed && offset === streamMeta.currentOffset) return {
|
|
1328
1711
|
messages: [],
|
|
1329
1712
|
timedOut: false,
|
|
@@ -1576,6 +1959,8 @@ const SSE_CURSOR_FIELD = `streamCursor`;
|
|
|
1576
1959
|
const SSE_UP_TO_DATE_FIELD = `upToDate`;
|
|
1577
1960
|
const SSE_CLOSED_FIELD = `streamClosed`;
|
|
1578
1961
|
const STREAM_CLOSED_HEADER = `Stream-Closed`;
|
|
1962
|
+
const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`;
|
|
1963
|
+
const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`;
|
|
1579
1964
|
const OFFSET_QUERY_PARAM = `offset`;
|
|
1580
1965
|
const LIVE_QUERY_PARAM = `live`;
|
|
1581
1966
|
const CURSOR_QUERY_PARAM = `cursor`;
|
|
@@ -1790,7 +2175,7 @@ var DurableStreamTestServer = class {
|
|
|
1790
2175
|
const method = req.method?.toUpperCase();
|
|
1791
2176
|
res.setHeader(`access-control-allow-origin`, `*`);
|
|
1792
2177
|
res.setHeader(`access-control-allow-methods`, `GET, POST, PUT, DELETE, HEAD, OPTIONS`);
|
|
1793
|
-
res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq`);
|
|
2178
|
+
res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset`);
|
|
1794
2179
|
res.setHeader(`access-control-expose-headers`, `Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Date, Stream-Closed, Producer-Epoch, Producer-Seq, Producer-Expected-Seq, Producer-Received-Seq, etag, content-type, content-encoding, vary`);
|
|
1795
2180
|
res.setHeader(`x-content-type-options`, `nosniff`);
|
|
1796
2181
|
res.setHeader(`cross-origin-resource-policy`, `cross-origin`);
|
|
@@ -1841,7 +2226,13 @@ var DurableStreamTestServer = class {
|
|
|
1841
2226
|
res.end(`Method not allowed`);
|
|
1842
2227
|
}
|
|
1843
2228
|
} catch (err) {
|
|
1844
|
-
if (err instanceof Error) if (err.message.includes(`
|
|
2229
|
+
if (err instanceof Error) if (err.message.includes(`active forks`)) {
|
|
2230
|
+
res.writeHead(409, { "content-type": `text/plain` });
|
|
2231
|
+
res.end(`stream was deleted but still has active forks — path cannot be reused until all forks are removed`);
|
|
2232
|
+
} else if (err.message.includes(`soft-deleted`)) {
|
|
2233
|
+
res.writeHead(410, { "content-type": `text/plain` });
|
|
2234
|
+
res.end(`Stream is gone`);
|
|
2235
|
+
} else if (err.message.includes(`not found`)) {
|
|
1845
2236
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
1846
2237
|
res.end(`Stream not found`);
|
|
1847
2238
|
} else if (err.message.includes(`already exists with different configuration`)) {
|
|
@@ -1873,6 +2264,8 @@ var DurableStreamTestServer = class {
|
|
|
1873
2264
|
const expiresAtHeader = req.headers[STREAM_EXPIRES_AT_HEADER.toLowerCase()];
|
|
1874
2265
|
const closedHeader = req.headers[STREAM_CLOSED_HEADER.toLowerCase()];
|
|
1875
2266
|
const createClosed = closedHeader === `true`;
|
|
2267
|
+
const forkedFromHeader = req.headers[STREAM_FORKED_FROM_HEADER.toLowerCase()];
|
|
2268
|
+
const forkOffsetHeader = req.headers[STREAM_FORK_OFFSET_HEADER.toLowerCase()];
|
|
1876
2269
|
if (ttlHeader && expiresAtHeader) {
|
|
1877
2270
|
res.writeHead(400, { "content-type": `text/plain` });
|
|
1878
2271
|
res.end(`Cannot specify both Stream-TTL and Stream-Expires-At`);
|
|
@@ -1901,24 +2294,60 @@ var DurableStreamTestServer = class {
|
|
|
1901
2294
|
return;
|
|
1902
2295
|
}
|
|
1903
2296
|
}
|
|
2297
|
+
if (forkOffsetHeader) {
|
|
2298
|
+
const validOffsetPattern = /^\d+_\d+$/;
|
|
2299
|
+
if (!validOffsetPattern.test(forkOffsetHeader)) {
|
|
2300
|
+
res.writeHead(400, { "content-type": `text/plain` });
|
|
2301
|
+
res.end(`Invalid Stream-Fork-Offset format`);
|
|
2302
|
+
return;
|
|
2303
|
+
}
|
|
2304
|
+
}
|
|
1904
2305
|
const body = await this.readBody(req);
|
|
1905
2306
|
const isNew = !this.store.has(path$2);
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
2307
|
+
try {
|
|
2308
|
+
await Promise.resolve(this.store.create(path$2, {
|
|
2309
|
+
contentType,
|
|
2310
|
+
ttlSeconds,
|
|
2311
|
+
expiresAt: expiresAtHeader,
|
|
2312
|
+
initialData: body.length > 0 ? body : void 0,
|
|
2313
|
+
closed: createClosed,
|
|
2314
|
+
forkedFrom: forkedFromHeader,
|
|
2315
|
+
forkOffset: forkOffsetHeader
|
|
2316
|
+
}));
|
|
2317
|
+
} catch (err) {
|
|
2318
|
+
if (err instanceof Error) {
|
|
2319
|
+
if (err.message.includes(`Source stream not found`)) {
|
|
2320
|
+
res.writeHead(404, { "content-type": `text/plain` });
|
|
2321
|
+
res.end(`Source stream not found`);
|
|
2322
|
+
return;
|
|
2323
|
+
}
|
|
2324
|
+
if (err.message.includes(`Invalid fork offset`)) {
|
|
2325
|
+
res.writeHead(400, { "content-type": `text/plain` });
|
|
2326
|
+
res.end(`Fork offset beyond source stream length`);
|
|
2327
|
+
return;
|
|
2328
|
+
}
|
|
2329
|
+
if (err.message.includes(`soft-deleted`)) {
|
|
2330
|
+
res.writeHead(409, { "content-type": `text/plain` });
|
|
2331
|
+
res.end(`source stream was deleted but still has active forks`);
|
|
2332
|
+
return;
|
|
2333
|
+
}
|
|
2334
|
+
if (err.message.includes(`Content type mismatch`)) {
|
|
2335
|
+
res.writeHead(409, { "content-type": `text/plain` });
|
|
2336
|
+
res.end(`Content type mismatch with source stream`);
|
|
2337
|
+
return;
|
|
2338
|
+
}
|
|
2339
|
+
}
|
|
2340
|
+
throw err;
|
|
2341
|
+
}
|
|
1913
2342
|
const stream = this.store.get(path$2);
|
|
1914
2343
|
if (isNew && this.options.onStreamCreated) await Promise.resolve(this.options.onStreamCreated({
|
|
1915
2344
|
type: `created`,
|
|
1916
2345
|
path: path$2,
|
|
1917
|
-
contentType,
|
|
2346
|
+
contentType: stream.contentType ?? contentType,
|
|
1918
2347
|
timestamp: Date.now()
|
|
1919
2348
|
}));
|
|
1920
2349
|
const headers = {
|
|
1921
|
-
"content-type": contentType,
|
|
2350
|
+
"content-type": stream.contentType ?? contentType,
|
|
1922
2351
|
[STREAM_OFFSET_HEADER]: stream.currentOffset
|
|
1923
2352
|
};
|
|
1924
2353
|
if (isNew) headers[`location`] = `${this._url}${path$2}`;
|
|
@@ -1936,12 +2365,19 @@ var DurableStreamTestServer = class {
|
|
|
1936
2365
|
res.end();
|
|
1937
2366
|
return;
|
|
1938
2367
|
}
|
|
2368
|
+
if (stream.softDeleted) {
|
|
2369
|
+
res.writeHead(410, { "content-type": `text/plain` });
|
|
2370
|
+
res.end();
|
|
2371
|
+
return;
|
|
2372
|
+
}
|
|
1939
2373
|
const headers = {
|
|
1940
2374
|
[STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
1941
2375
|
"cache-control": `no-store`
|
|
1942
2376
|
};
|
|
1943
2377
|
if (stream.contentType) headers[`content-type`] = stream.contentType;
|
|
1944
2378
|
if (stream.closed) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
2379
|
+
if (stream.ttlSeconds !== void 0) headers[STREAM_TTL_HEADER] = String(stream.ttlSeconds);
|
|
2380
|
+
if (stream.expiresAt) headers[STREAM_EXPIRES_AT_HEADER] = stream.expiresAt;
|
|
1945
2381
|
const closedSuffix = stream.closed ? `:c` : ``;
|
|
1946
2382
|
headers[`etag`] = `"${Buffer.from(path$2).toString(`base64`)}:-1:${stream.currentOffset}${closedSuffix}"`;
|
|
1947
2383
|
res.writeHead(200, headers);
|
|
@@ -1957,6 +2393,11 @@ var DurableStreamTestServer = class {
|
|
|
1957
2393
|
res.end(`Stream not found`);
|
|
1958
2394
|
return;
|
|
1959
2395
|
}
|
|
2396
|
+
if (stream.softDeleted) {
|
|
2397
|
+
res.writeHead(410, { "content-type": `text/plain` });
|
|
2398
|
+
res.end(`Stream is gone`);
|
|
2399
|
+
return;
|
|
2400
|
+
}
|
|
1960
2401
|
const offset = url.searchParams.get(OFFSET_QUERY_PARAM) ?? void 0;
|
|
1961
2402
|
const live = url.searchParams.get(LIVE_QUERY_PARAM);
|
|
1962
2403
|
const cursor = url.searchParams.get(CURSOR_QUERY_PARAM) ?? void 0;
|
|
@@ -2011,6 +2452,7 @@ var DurableStreamTestServer = class {
|
|
|
2011
2452
|
return;
|
|
2012
2453
|
}
|
|
2013
2454
|
let { messages, upToDate } = this.store.read(path$2, effectiveOffset);
|
|
2455
|
+
this.store.touchAccess(path$2);
|
|
2014
2456
|
const clientIsCaughtUp = effectiveOffset && effectiveOffset === stream.currentOffset || offset === `now`;
|
|
2015
2457
|
if (live === `long-poll` && clientIsCaughtUp && messages.length === 0) {
|
|
2016
2458
|
if (stream.closed) {
|
|
@@ -2023,6 +2465,7 @@ var DurableStreamTestServer = class {
|
|
|
2023
2465
|
return;
|
|
2024
2466
|
}
|
|
2025
2467
|
const result = await this.store.waitForMessages(path$2, effectiveOffset ?? stream.currentOffset, this.options.longPollTimeout);
|
|
2468
|
+
this.store.touchAccess(path$2);
|
|
2026
2469
|
if (result.streamClosed) {
|
|
2027
2470
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2028
2471
|
res.writeHead(204, {
|
|
@@ -2115,6 +2558,7 @@ var DurableStreamTestServer = class {
|
|
|
2115
2558
|
const isJsonStream = stream?.contentType?.includes(`application/json`);
|
|
2116
2559
|
while (isConnected && !this.isShuttingDown) {
|
|
2117
2560
|
const { messages, upToDate } = this.store.read(path$2, currentOffset);
|
|
2561
|
+
this.store.touchAccess(path$2);
|
|
2118
2562
|
for (const message of messages) {
|
|
2119
2563
|
let dataPayload;
|
|
2120
2564
|
if (useBase64) dataPayload = Buffer.from(message.data).toString(`base64`);
|
|
@@ -2152,6 +2596,7 @@ var DurableStreamTestServer = class {
|
|
|
2152
2596
|
break;
|
|
2153
2597
|
}
|
|
2154
2598
|
const result = await this.store.waitForMessages(path$2, currentOffset, this.options.longPollTimeout);
|
|
2599
|
+
this.store.touchAccess(path$2);
|
|
2155
2600
|
if (this.isShuttingDown || !isConnected) break;
|
|
2156
2601
|
if (result.streamClosed) {
|
|
2157
2602
|
const finalControlData = {
|
|
@@ -2334,6 +2779,7 @@ var DurableStreamTestServer = class {
|
|
|
2334
2779
|
let result;
|
|
2335
2780
|
if (producerId !== void 0) result = await this.store.appendWithProducer(path$2, body, appendOptions);
|
|
2336
2781
|
else result = await Promise.resolve(this.store.append(path$2, body, appendOptions));
|
|
2782
|
+
this.store.touchAccess(path$2);
|
|
2337
2783
|
if (result && typeof result === `object` && `message` in result) {
|
|
2338
2784
|
const { message: message$1, producerResult, streamClosed } = result;
|
|
2339
2785
|
if (streamClosed && !message$1) {
|
|
@@ -2410,12 +2856,18 @@ var DurableStreamTestServer = class {
|
|
|
2410
2856
|
* Handle DELETE - delete stream
|
|
2411
2857
|
*/
|
|
2412
2858
|
async handleDelete(path$2, res) {
|
|
2413
|
-
|
|
2859
|
+
const existing = this.store.get(path$2);
|
|
2860
|
+
if (existing?.softDeleted) {
|
|
2861
|
+
res.writeHead(410, { "content-type": `text/plain` });
|
|
2862
|
+
res.end(`Stream is gone`);
|
|
2863
|
+
return;
|
|
2864
|
+
}
|
|
2865
|
+
const deleted = this.store.delete(path$2);
|
|
2866
|
+
if (!deleted) {
|
|
2414
2867
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
2415
2868
|
res.end(`Stream not found`);
|
|
2416
2869
|
return;
|
|
2417
2870
|
}
|
|
2418
|
-
this.store.delete(path$2);
|
|
2419
2871
|
if (this.options.onStreamDeleted) await Promise.resolve(this.options.onStreamDeleted({
|
|
2420
2872
|
type: `deleted`,
|
|
2421
2873
|
path: path$2,
|