@durable-streams/server 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +541 -89
- package/dist/index.d.cts +97 -6
- package/dist/index.d.ts +97 -6
- package/dist/index.js +541 -89
- package/package.json +3 -3
- package/src/file-store.ts +491 -90
- package/src/server.ts +108 -16
- package/src/store.ts +363 -36
- package/src/types.ts +29 -0
package/dist/index.cjs
CHANGED
|
@@ -103,72 +103,180 @@ var StreamStore = class {
|
|
|
103
103
|
if (!Number.isFinite(expiryTime) || now >= expiryTime) return true;
|
|
104
104
|
}
|
|
105
105
|
if (stream.ttlSeconds !== void 0) {
|
|
106
|
-
const expiryTime = stream.
|
|
106
|
+
const expiryTime = stream.lastAccessedAt + stream.ttlSeconds * 1e3;
|
|
107
107
|
if (now >= expiryTime) return true;
|
|
108
108
|
}
|
|
109
109
|
return false;
|
|
110
110
|
}
|
|
111
111
|
/**
|
|
112
|
-
* Get a stream,
|
|
113
|
-
* Returns undefined if stream doesn't exist or is expired.
|
|
112
|
+
* Get a stream, handling expiry.
|
|
113
|
+
* Returns undefined if stream doesn't exist or is expired (and has no refs).
|
|
114
|
+
* Expired streams with refCount > 0 are soft-deleted instead of fully deleted.
|
|
114
115
|
*/
|
|
115
116
|
getIfNotExpired(path) {
|
|
116
117
|
const stream = this.streams.get(path);
|
|
117
118
|
if (!stream) return void 0;
|
|
118
119
|
if (this.isExpired(stream)) {
|
|
120
|
+
if (stream.refCount > 0) {
|
|
121
|
+
stream.softDeleted = true;
|
|
122
|
+
return stream;
|
|
123
|
+
}
|
|
119
124
|
this.delete(path);
|
|
120
125
|
return void 0;
|
|
121
126
|
}
|
|
122
127
|
return stream;
|
|
123
128
|
}
|
|
124
129
|
/**
|
|
130
|
+
* Update lastAccessedAt to now. Called on reads and appends (not HEAD).
|
|
131
|
+
*/
|
|
132
|
+
touchAccess(path) {
|
|
133
|
+
const stream = this.streams.get(path);
|
|
134
|
+
if (stream) stream.lastAccessedAt = Date.now();
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
125
137
|
* Create a new stream.
|
|
126
138
|
* @throws Error if stream already exists with different config
|
|
139
|
+
* @throws Error if fork source not found, soft-deleted, or offset invalid
|
|
127
140
|
* @returns existing stream if config matches (idempotent)
|
|
128
141
|
*/
|
|
129
142
|
create(path, options = {}) {
|
|
130
|
-
const
|
|
131
|
-
if (
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
143
|
+
const existingRaw = this.streams.get(path);
|
|
144
|
+
if (existingRaw) if (this.isExpired(existingRaw)) {
|
|
145
|
+
this.streams.delete(path);
|
|
146
|
+
this.cancelLongPollsForStream(path);
|
|
147
|
+
} else if (existingRaw.softDeleted) throw new Error(`Stream has active forks — path cannot be reused until all forks are removed: ${path}`);
|
|
148
|
+
else {
|
|
149
|
+
const contentTypeMatches = (normalizeContentType(options.contentType) || `application/octet-stream`) === (normalizeContentType(existingRaw.contentType) || `application/octet-stream`);
|
|
150
|
+
const ttlMatches = options.ttlSeconds === existingRaw.ttlSeconds;
|
|
151
|
+
const expiresMatches = options.expiresAt === existingRaw.expiresAt;
|
|
152
|
+
const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
|
|
153
|
+
const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
|
|
154
|
+
const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
|
|
155
|
+
if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches) return existingRaw;
|
|
137
156
|
else throw new Error(`Stream already exists with different configuration: ${path}`);
|
|
138
157
|
}
|
|
158
|
+
const isFork = !!options.forkedFrom;
|
|
159
|
+
let forkOffset = `0000000000000000_0000000000000000`;
|
|
160
|
+
let sourceContentType;
|
|
161
|
+
let sourceStream;
|
|
162
|
+
if (isFork) {
|
|
163
|
+
sourceStream = this.streams.get(options.forkedFrom);
|
|
164
|
+
if (!sourceStream) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
165
|
+
if (sourceStream.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
|
|
166
|
+
if (this.isExpired(sourceStream)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
167
|
+
sourceContentType = sourceStream.contentType;
|
|
168
|
+
if (options.forkOffset) forkOffset = options.forkOffset;
|
|
169
|
+
else forkOffset = sourceStream.currentOffset;
|
|
170
|
+
const zeroOffset = `0000000000000000_0000000000000000`;
|
|
171
|
+
if (forkOffset < zeroOffset || sourceStream.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
|
|
172
|
+
sourceStream.refCount++;
|
|
173
|
+
}
|
|
174
|
+
let contentType = options.contentType;
|
|
175
|
+
if (!contentType || contentType.trim() === ``) {
|
|
176
|
+
if (isFork) contentType = sourceContentType;
|
|
177
|
+
} else if (isFork && normalizeContentType(contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
|
|
178
|
+
let effectiveExpiresAt = options.expiresAt;
|
|
179
|
+
let effectiveTtlSeconds = options.ttlSeconds;
|
|
180
|
+
if (isFork) {
|
|
181
|
+
const resolved = this.resolveForkExpiry(options, sourceStream);
|
|
182
|
+
effectiveExpiresAt = resolved.expiresAt;
|
|
183
|
+
effectiveTtlSeconds = resolved.ttlSeconds;
|
|
184
|
+
}
|
|
139
185
|
const stream = {
|
|
140
186
|
path,
|
|
141
|
-
contentType
|
|
187
|
+
contentType,
|
|
142
188
|
messages: [],
|
|
143
|
-
currentOffset: `0000000000000000_0000000000000000`,
|
|
144
|
-
ttlSeconds:
|
|
145
|
-
expiresAt:
|
|
189
|
+
currentOffset: isFork ? forkOffset : `0000000000000000_0000000000000000`,
|
|
190
|
+
ttlSeconds: effectiveTtlSeconds,
|
|
191
|
+
expiresAt: effectiveExpiresAt,
|
|
146
192
|
createdAt: Date.now(),
|
|
147
|
-
|
|
193
|
+
lastAccessedAt: Date.now(),
|
|
194
|
+
closed: options.closed ?? false,
|
|
195
|
+
refCount: 0,
|
|
196
|
+
forkedFrom: isFork ? options.forkedFrom : void 0,
|
|
197
|
+
forkOffset: isFork ? forkOffset : void 0
|
|
148
198
|
};
|
|
149
|
-
if (options.initialData && options.initialData.length > 0)
|
|
199
|
+
if (options.initialData && options.initialData.length > 0) try {
|
|
200
|
+
this.appendToStream(stream, options.initialData, true);
|
|
201
|
+
} catch (err) {
|
|
202
|
+
if (isFork && sourceStream) sourceStream.refCount--;
|
|
203
|
+
throw err;
|
|
204
|
+
}
|
|
150
205
|
this.streams.set(path, stream);
|
|
151
206
|
return stream;
|
|
152
207
|
}
|
|
153
208
|
/**
|
|
209
|
+
* Resolve fork expiry per the decision table.
|
|
210
|
+
* Forks have independent lifetimes — no capping at source expiry.
|
|
211
|
+
*/
|
|
212
|
+
resolveForkExpiry(opts, sourceMeta) {
|
|
213
|
+
if (opts.ttlSeconds !== void 0) return { ttlSeconds: opts.ttlSeconds };
|
|
214
|
+
if (opts.expiresAt) return { expiresAt: opts.expiresAt };
|
|
215
|
+
if (sourceMeta.ttlSeconds !== void 0) return { ttlSeconds: sourceMeta.ttlSeconds };
|
|
216
|
+
if (sourceMeta.expiresAt) return { expiresAt: sourceMeta.expiresAt };
|
|
217
|
+
return {};
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
154
220
|
* Get a stream by path.
|
|
155
221
|
* Returns undefined if stream doesn't exist or is expired.
|
|
222
|
+
* Returns soft-deleted streams (caller should check stream.softDeleted).
|
|
156
223
|
*/
|
|
157
224
|
get(path) {
|
|
158
|
-
|
|
225
|
+
const stream = this.streams.get(path);
|
|
226
|
+
if (!stream) return void 0;
|
|
227
|
+
if (this.isExpired(stream)) {
|
|
228
|
+
if (stream.refCount > 0) {
|
|
229
|
+
stream.softDeleted = true;
|
|
230
|
+
return stream;
|
|
231
|
+
}
|
|
232
|
+
this.delete(path);
|
|
233
|
+
return void 0;
|
|
234
|
+
}
|
|
235
|
+
return stream;
|
|
159
236
|
}
|
|
160
237
|
/**
|
|
161
|
-
* Check if a stream exists
|
|
238
|
+
* Check if a stream exists, is not expired, and is not soft-deleted.
|
|
162
239
|
*/
|
|
163
240
|
has(path) {
|
|
164
|
-
|
|
241
|
+
const stream = this.get(path);
|
|
242
|
+
if (!stream) return false;
|
|
243
|
+
if (stream.softDeleted) return false;
|
|
244
|
+
return true;
|
|
165
245
|
}
|
|
166
246
|
/**
|
|
167
247
|
* Delete a stream.
|
|
248
|
+
* If the stream has forks (refCount > 0), it is soft-deleted instead of fully removed.
|
|
249
|
+
* Returns true if the stream was found and deleted (or soft-deleted).
|
|
168
250
|
*/
|
|
169
251
|
delete(path) {
|
|
252
|
+
const stream = this.streams.get(path);
|
|
253
|
+
if (!stream) return false;
|
|
254
|
+
if (stream.softDeleted) return true;
|
|
255
|
+
if (stream.refCount > 0) {
|
|
256
|
+
stream.softDeleted = true;
|
|
257
|
+
return true;
|
|
258
|
+
}
|
|
259
|
+
this.deleteWithCascade(path);
|
|
260
|
+
return true;
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Fully delete a stream and cascade to soft-deleted parents
|
|
264
|
+
* whose refcount drops to zero.
|
|
265
|
+
*/
|
|
266
|
+
deleteWithCascade(path) {
|
|
267
|
+
const stream = this.streams.get(path);
|
|
268
|
+
if (!stream) return;
|
|
269
|
+
const forkedFrom = stream.forkedFrom;
|
|
270
|
+
this.streams.delete(path);
|
|
170
271
|
this.cancelLongPollsForStream(path);
|
|
171
|
-
|
|
272
|
+
if (forkedFrom) {
|
|
273
|
+
const parent = this.streams.get(forkedFrom);
|
|
274
|
+
if (parent) {
|
|
275
|
+
parent.refCount--;
|
|
276
|
+
if (parent.refCount < 0) parent.refCount = 0;
|
|
277
|
+
if (parent.refCount === 0 && parent.softDeleted) this.deleteWithCascade(forkedFrom);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
172
280
|
}
|
|
173
281
|
/**
|
|
174
282
|
* Validate producer state WITHOUT mutating.
|
|
@@ -281,6 +389,7 @@ var StreamStore = class {
|
|
|
281
389
|
append(path, data, options = {}) {
|
|
282
390
|
const stream = this.getIfNotExpired(path);
|
|
283
391
|
if (!stream) throw new Error(`Stream not found: ${path}`);
|
|
392
|
+
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path}`);
|
|
284
393
|
if (stream.closed) {
|
|
285
394
|
if (options.producerId && stream.closedBy && stream.closedBy.producerId === options.producerId && stream.closedBy.epoch === options.producerEpoch && stream.closedBy.seq === options.producerSeq) return {
|
|
286
395
|
message: null,
|
|
@@ -357,6 +466,7 @@ var StreamStore = class {
|
|
|
357
466
|
closeStream(path) {
|
|
358
467
|
const stream = this.getIfNotExpired(path);
|
|
359
468
|
if (!stream) return null;
|
|
469
|
+
if (stream.softDeleted) throw new Error(`Stream is soft-deleted: ${path}`);
|
|
360
470
|
const alreadyClosed = stream.closed ?? false;
|
|
361
471
|
stream.closed = true;
|
|
362
472
|
this.notifyLongPollsClosed(path);
|
|
@@ -424,15 +534,26 @@ var StreamStore = class {
|
|
|
424
534
|
}
|
|
425
535
|
/**
|
|
426
536
|
* Read messages from a stream starting at the given offset.
|
|
537
|
+
* For forked streams, stitches messages from the source chain and the fork's own messages.
|
|
427
538
|
* @throws Error if stream doesn't exist or is expired
|
|
428
539
|
*/
|
|
429
540
|
read(path, offset) {
|
|
430
541
|
const stream = this.getIfNotExpired(path);
|
|
431
542
|
if (!stream) throw new Error(`Stream not found: ${path}`);
|
|
432
|
-
if (!offset || offset === `-1`)
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
543
|
+
if (!offset || offset === `-1`) {
|
|
544
|
+
if (stream.forkedFrom) {
|
|
545
|
+
const inherited = this.readForkedMessages(stream.forkedFrom, void 0, stream.forkOffset);
|
|
546
|
+
return {
|
|
547
|
+
messages: [...inherited, ...stream.messages],
|
|
548
|
+
upToDate: true
|
|
549
|
+
};
|
|
550
|
+
}
|
|
551
|
+
return {
|
|
552
|
+
messages: [...stream.messages],
|
|
553
|
+
upToDate: true
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
if (stream.forkedFrom) return this.readFromFork(stream, offset);
|
|
436
557
|
const offsetIndex = this.findOffsetIndex(stream, offset);
|
|
437
558
|
if (offsetIndex === -1) return {
|
|
438
559
|
messages: [],
|
|
@@ -444,6 +565,55 @@ var StreamStore = class {
|
|
|
444
565
|
};
|
|
445
566
|
}
|
|
446
567
|
/**
|
|
568
|
+
* Read from a forked stream, stitching inherited and own messages.
|
|
569
|
+
*/
|
|
570
|
+
readFromFork(stream, offset) {
|
|
571
|
+
const messages = [];
|
|
572
|
+
if (offset < stream.forkOffset) {
|
|
573
|
+
const inherited = this.readForkedMessages(stream.forkedFrom, offset, stream.forkOffset);
|
|
574
|
+
messages.push(...inherited);
|
|
575
|
+
}
|
|
576
|
+
const ownMessages = this.readOwnMessages(stream, offset);
|
|
577
|
+
messages.push(...ownMessages);
|
|
578
|
+
return {
|
|
579
|
+
messages,
|
|
580
|
+
upToDate: true
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
/**
|
|
584
|
+
* Read a stream's own messages starting after the given offset.
|
|
585
|
+
*/
|
|
586
|
+
readOwnMessages(stream, offset) {
|
|
587
|
+
const offsetIndex = this.findOffsetIndex(stream, offset);
|
|
588
|
+
if (offsetIndex === -1) return [];
|
|
589
|
+
return stream.messages.slice(offsetIndex);
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* Recursively read messages from a fork's source chain.
|
|
593
|
+
* Reads from source (and its sources if also forked), capped at forkOffset.
|
|
594
|
+
* Does NOT check softDeleted — forks must read through soft-deleted sources.
|
|
595
|
+
*/
|
|
596
|
+
readForkedMessages(sourcePath, offset, capOffset) {
|
|
597
|
+
const source = this.streams.get(sourcePath);
|
|
598
|
+
if (!source) return [];
|
|
599
|
+
const messages = [];
|
|
600
|
+
if (source.forkedFrom && (!offset || offset < source.forkOffset)) {
|
|
601
|
+
const inherited = this.readForkedMessages(
|
|
602
|
+
source.forkedFrom,
|
|
603
|
+
offset,
|
|
604
|
+
// Cap at the minimum of source's forkOffset and our capOffset
|
|
605
|
+
source.forkOffset < capOffset ? source.forkOffset : capOffset
|
|
606
|
+
);
|
|
607
|
+
messages.push(...inherited);
|
|
608
|
+
}
|
|
609
|
+
for (const msg of source.messages) {
|
|
610
|
+
if (offset && msg.offset <= offset) continue;
|
|
611
|
+
if (msg.offset > capOffset) break;
|
|
612
|
+
messages.push(msg);
|
|
613
|
+
}
|
|
614
|
+
return messages;
|
|
615
|
+
}
|
|
616
|
+
/**
|
|
447
617
|
* Format messages for response.
|
|
448
618
|
* For JSON mode, wraps concatenated data in array brackets.
|
|
449
619
|
* @throws Error if stream doesn't exist or is expired
|
|
@@ -468,6 +638,13 @@ var StreamStore = class {
|
|
|
468
638
|
async waitForMessages(path, offset, timeoutMs) {
|
|
469
639
|
const stream = this.getIfNotExpired(path);
|
|
470
640
|
if (!stream) throw new Error(`Stream not found: ${path}`);
|
|
641
|
+
if (stream.forkedFrom && offset < stream.forkOffset) {
|
|
642
|
+
const { messages: messages$1 } = this.read(path, offset);
|
|
643
|
+
return {
|
|
644
|
+
messages: messages$1,
|
|
645
|
+
timedOut: false
|
|
646
|
+
};
|
|
647
|
+
}
|
|
471
648
|
const { messages } = this.read(path, offset);
|
|
472
649
|
if (messages.length > 0) return {
|
|
473
650
|
messages,
|
|
@@ -843,7 +1020,14 @@ var FileBackedStreamStore = class {
|
|
|
843
1020
|
errors++;
|
|
844
1021
|
continue;
|
|
845
1022
|
}
|
|
846
|
-
const
|
|
1023
|
+
const physicalOffset = this.scanFileForTrueOffset(segmentPath);
|
|
1024
|
+
const physicalBytes = Number(physicalOffset.split(`_`)[1] ?? 0);
|
|
1025
|
+
let trueOffset;
|
|
1026
|
+
if (streamMeta.forkOffset) {
|
|
1027
|
+
const forkBaseByte = Number(streamMeta.forkOffset.split(`_`)[1] ?? 0);
|
|
1028
|
+
const logicalBytes = forkBaseByte + physicalBytes;
|
|
1029
|
+
trueOffset = `${String(0).padStart(16, `0`)}_${String(logicalBytes).padStart(16, `0`)}`;
|
|
1030
|
+
} else trueOffset = physicalOffset;
|
|
847
1031
|
if (trueOffset !== streamMeta.currentOffset) {
|
|
848
1032
|
console.warn(`[FileBackedStreamStore] Recovery: Offset mismatch for ${streamPath}: LMDB says ${streamMeta.currentOffset}, file says ${trueOffset}. Reconciling to file.`);
|
|
849
1033
|
const reconciledMeta = {
|
|
@@ -902,9 +1086,14 @@ var FileBackedStreamStore = class {
|
|
|
902
1086
|
ttlSeconds: meta.ttlSeconds,
|
|
903
1087
|
expiresAt: meta.expiresAt,
|
|
904
1088
|
createdAt: meta.createdAt,
|
|
1089
|
+
lastAccessedAt: meta.lastAccessedAt ?? meta.createdAt,
|
|
905
1090
|
producers,
|
|
906
1091
|
closed: meta.closed,
|
|
907
|
-
closedBy: meta.closedBy
|
|
1092
|
+
closedBy: meta.closedBy,
|
|
1093
|
+
forkedFrom: meta.forkedFrom,
|
|
1094
|
+
forkOffset: meta.forkOffset,
|
|
1095
|
+
refCount: meta.refCount ?? 0,
|
|
1096
|
+
softDeleted: meta.softDeleted
|
|
908
1097
|
};
|
|
909
1098
|
}
|
|
910
1099
|
/**
|
|
@@ -1000,6 +1189,20 @@ var FileBackedStreamStore = class {
|
|
|
1000
1189
|
return meta.producers[producerId]?.epoch;
|
|
1001
1190
|
}
|
|
1002
1191
|
/**
|
|
1192
|
+
* Update lastAccessedAt to now. Called on reads and appends (not HEAD).
|
|
1193
|
+
*/
|
|
1194
|
+
touchAccess(streamPath) {
|
|
1195
|
+
const key = `stream:${streamPath}`;
|
|
1196
|
+
const meta = this.db.get(key);
|
|
1197
|
+
if (meta) {
|
|
1198
|
+
const updatedMeta = {
|
|
1199
|
+
...meta,
|
|
1200
|
+
lastAccessedAt: Date.now()
|
|
1201
|
+
};
|
|
1202
|
+
this.db.putSync(key, updatedMeta);
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
/**
|
|
1003
1206
|
* Check if a stream is expired based on TTL or Expires-At.
|
|
1004
1207
|
*/
|
|
1005
1208
|
isExpired(meta) {
|
|
@@ -1009,26 +1212,50 @@ var FileBackedStreamStore = class {
|
|
|
1009
1212
|
if (!Number.isFinite(expiryTime) || now >= expiryTime) return true;
|
|
1010
1213
|
}
|
|
1011
1214
|
if (meta.ttlSeconds !== void 0) {
|
|
1012
|
-
const
|
|
1215
|
+
const lastAccessed = meta.lastAccessedAt ?? meta.createdAt;
|
|
1216
|
+
const expiryTime = lastAccessed + meta.ttlSeconds * 1e3;
|
|
1013
1217
|
if (now >= expiryTime) return true;
|
|
1014
1218
|
}
|
|
1015
1219
|
return false;
|
|
1016
1220
|
}
|
|
1017
1221
|
/**
|
|
1018
1222
|
* Get stream metadata, deleting it if expired.
|
|
1019
|
-
* Returns undefined if stream doesn't exist or is expired.
|
|
1223
|
+
* Returns undefined if stream doesn't exist or is expired (and has no refs).
|
|
1224
|
+
* Expired streams with refCount > 0 are soft-deleted instead of fully deleted.
|
|
1020
1225
|
*/
|
|
1021
1226
|
getMetaIfNotExpired(streamPath) {
|
|
1022
1227
|
const key = `stream:${streamPath}`;
|
|
1023
1228
|
const meta = this.db.get(key);
|
|
1024
1229
|
if (!meta) return void 0;
|
|
1025
1230
|
if (this.isExpired(meta)) {
|
|
1231
|
+
if ((meta.refCount ?? 0) > 0) {
|
|
1232
|
+
if (!meta.softDeleted) {
|
|
1233
|
+
const updatedMeta = {
|
|
1234
|
+
...meta,
|
|
1235
|
+
softDeleted: true
|
|
1236
|
+
};
|
|
1237
|
+
this.db.putSync(key, updatedMeta);
|
|
1238
|
+
return updatedMeta;
|
|
1239
|
+
}
|
|
1240
|
+
return meta;
|
|
1241
|
+
}
|
|
1026
1242
|
this.delete(streamPath);
|
|
1027
1243
|
return void 0;
|
|
1028
1244
|
}
|
|
1029
1245
|
return meta;
|
|
1030
1246
|
}
|
|
1031
1247
|
/**
|
|
1248
|
+
* Resolve fork expiry per the decision table.
|
|
1249
|
+
* Forks have independent lifetimes — no capping at source expiry.
|
|
1250
|
+
*/
|
|
1251
|
+
resolveForkExpiry(opts, sourceMeta) {
|
|
1252
|
+
if (opts.ttlSeconds !== void 0) return { ttlSeconds: opts.ttlSeconds };
|
|
1253
|
+
if (opts.expiresAt) return { expiresAt: opts.expiresAt };
|
|
1254
|
+
if (sourceMeta.ttlSeconds !== void 0) return { ttlSeconds: sourceMeta.ttlSeconds };
|
|
1255
|
+
if (sourceMeta.expiresAt) return { expiresAt: sourceMeta.expiresAt };
|
|
1256
|
+
return {};
|
|
1257
|
+
}
|
|
1258
|
+
/**
|
|
1032
1259
|
* Close the store, closing all file handles and database.
|
|
1033
1260
|
* All data is already fsynced on each append, so no final flush needed.
|
|
1034
1261
|
*/
|
|
@@ -1037,29 +1264,70 @@ var FileBackedStreamStore = class {
|
|
|
1037
1264
|
await this.db.close();
|
|
1038
1265
|
}
|
|
1039
1266
|
async create(streamPath, options = {}) {
|
|
1040
|
-
const
|
|
1041
|
-
if (
|
|
1267
|
+
const existingRaw = this.db.get(`stream:${streamPath}`);
|
|
1268
|
+
if (existingRaw) if (this.isExpired(existingRaw)) this.delete(streamPath);
|
|
1269
|
+
else if (existingRaw.softDeleted) throw new Error(`Stream has active forks — path cannot be reused until all forks are removed: ${streamPath}`);
|
|
1270
|
+
else {
|
|
1042
1271
|
const normalizeMimeType = (ct) => (ct ?? `application/octet-stream`).toLowerCase();
|
|
1043
|
-
const contentTypeMatches = normalizeMimeType(options.contentType) === normalizeMimeType(
|
|
1044
|
-
const ttlMatches = options.ttlSeconds ===
|
|
1045
|
-
const expiresMatches = options.expiresAt ===
|
|
1046
|
-
const closedMatches = (options.closed ?? false) === (
|
|
1047
|
-
|
|
1272
|
+
const contentTypeMatches = normalizeMimeType(options.contentType) === normalizeMimeType(existingRaw.contentType);
|
|
1273
|
+
const ttlMatches = options.ttlSeconds === existingRaw.ttlSeconds;
|
|
1274
|
+
const expiresMatches = options.expiresAt === existingRaw.expiresAt;
|
|
1275
|
+
const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
|
|
1276
|
+
const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
|
|
1277
|
+
const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
|
|
1278
|
+
if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches) return this.streamMetaToStream(existingRaw);
|
|
1048
1279
|
else throw new Error(`Stream already exists with different configuration: ${streamPath}`);
|
|
1049
1280
|
}
|
|
1281
|
+
const isFork = !!options.forkedFrom;
|
|
1282
|
+
let forkOffset = `0000000000000000_0000000000000000`;
|
|
1283
|
+
let sourceContentType;
|
|
1284
|
+
let sourceMeta;
|
|
1285
|
+
if (isFork) {
|
|
1286
|
+
const sourceKey = `stream:${options.forkedFrom}`;
|
|
1287
|
+
sourceMeta = this.db.get(sourceKey);
|
|
1288
|
+
if (!sourceMeta) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
1289
|
+
if (sourceMeta.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
|
|
1290
|
+
if (this.isExpired(sourceMeta)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
1291
|
+
sourceContentType = sourceMeta.contentType;
|
|
1292
|
+
if (options.forkOffset) forkOffset = options.forkOffset;
|
|
1293
|
+
else forkOffset = sourceMeta.currentOffset;
|
|
1294
|
+
const zeroOffset = `0000000000000000_0000000000000000`;
|
|
1295
|
+
if (forkOffset < zeroOffset || sourceMeta.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
|
|
1296
|
+
const freshSource = this.db.get(sourceKey);
|
|
1297
|
+
const updatedSource = {
|
|
1298
|
+
...freshSource,
|
|
1299
|
+
refCount: (freshSource.refCount ?? 0) + 1
|
|
1300
|
+
};
|
|
1301
|
+
this.db.putSync(sourceKey, updatedSource);
|
|
1302
|
+
}
|
|
1303
|
+
let contentType = options.contentType;
|
|
1304
|
+
if (!contentType || contentType.trim() === ``) {
|
|
1305
|
+
if (isFork) contentType = sourceContentType;
|
|
1306
|
+
} else if (isFork && normalizeContentType(contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
|
|
1307
|
+
let effectiveExpiresAt = options.expiresAt;
|
|
1308
|
+
let effectiveTtlSeconds = options.ttlSeconds;
|
|
1309
|
+
if (isFork) {
|
|
1310
|
+
const resolved = this.resolveForkExpiry(options, sourceMeta);
|
|
1311
|
+
effectiveExpiresAt = resolved.expiresAt;
|
|
1312
|
+
effectiveTtlSeconds = resolved.ttlSeconds;
|
|
1313
|
+
}
|
|
1050
1314
|
const key = `stream:${streamPath}`;
|
|
1051
1315
|
const streamMeta = {
|
|
1052
1316
|
path: streamPath,
|
|
1053
|
-
contentType
|
|
1054
|
-
currentOffset: `0000000000000000_0000000000000000`,
|
|
1317
|
+
contentType,
|
|
1318
|
+
currentOffset: isFork ? forkOffset : `0000000000000000_0000000000000000`,
|
|
1055
1319
|
lastSeq: void 0,
|
|
1056
|
-
ttlSeconds:
|
|
1057
|
-
expiresAt:
|
|
1320
|
+
ttlSeconds: effectiveTtlSeconds,
|
|
1321
|
+
expiresAt: effectiveExpiresAt,
|
|
1058
1322
|
createdAt: Date.now(),
|
|
1323
|
+
lastAccessedAt: Date.now(),
|
|
1059
1324
|
segmentCount: 1,
|
|
1060
1325
|
totalBytes: 0,
|
|
1061
1326
|
directoryName: generateUniqueDirectoryName(streamPath),
|
|
1062
|
-
closed: false
|
|
1327
|
+
closed: false,
|
|
1328
|
+
forkedFrom: isFork ? options.forkedFrom : void 0,
|
|
1329
|
+
forkOffset: isFork ? forkOffset : void 0,
|
|
1330
|
+
refCount: 0
|
|
1063
1331
|
};
|
|
1064
1332
|
const streamDir = node_path.join(this.dataDir, `streams`, streamMeta.directoryName);
|
|
1065
1333
|
try {
|
|
@@ -1067,14 +1335,40 @@ var FileBackedStreamStore = class {
|
|
|
1067
1335
|
const segmentPath = node_path.join(streamDir, `segment_00000.log`);
|
|
1068
1336
|
node_fs.writeFileSync(segmentPath, ``);
|
|
1069
1337
|
} catch (err) {
|
|
1338
|
+
if (isFork && sourceMeta) {
|
|
1339
|
+
const sourceKey = `stream:${options.forkedFrom}`;
|
|
1340
|
+
const freshSource = this.db.get(sourceKey);
|
|
1341
|
+
if (freshSource) {
|
|
1342
|
+
const updatedSource = {
|
|
1343
|
+
...freshSource,
|
|
1344
|
+
refCount: Math.max(0, (freshSource.refCount ?? 0) - 1)
|
|
1345
|
+
};
|
|
1346
|
+
this.db.putSync(sourceKey, updatedSource);
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1070
1349
|
console.error(`[FileBackedStreamStore] Error creating stream directory:`, err);
|
|
1071
1350
|
throw err;
|
|
1072
1351
|
}
|
|
1073
1352
|
this.db.putSync(key, streamMeta);
|
|
1074
|
-
if (options.initialData && options.initialData.length > 0)
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1353
|
+
if (options.initialData && options.initialData.length > 0) try {
|
|
1354
|
+
await this.append(streamPath, options.initialData, {
|
|
1355
|
+
contentType: options.contentType,
|
|
1356
|
+
isInitialCreate: true
|
|
1357
|
+
});
|
|
1358
|
+
} catch (err) {
|
|
1359
|
+
if (isFork && sourceMeta) {
|
|
1360
|
+
const sourceKey = `stream:${options.forkedFrom}`;
|
|
1361
|
+
const freshSource = this.db.get(sourceKey);
|
|
1362
|
+
if (freshSource) {
|
|
1363
|
+
const updatedSource = {
|
|
1364
|
+
...freshSource,
|
|
1365
|
+
refCount: Math.max(0, (freshSource.refCount ?? 0) - 1)
|
|
1366
|
+
};
|
|
1367
|
+
this.db.putSync(sourceKey, updatedSource);
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
throw err;
|
|
1371
|
+
}
|
|
1078
1372
|
if (options.closed) {
|
|
1079
1373
|
const updatedMeta = this.db.get(key);
|
|
1080
1374
|
updatedMeta.closed = true;
|
|
@@ -1085,15 +1379,41 @@ var FileBackedStreamStore = class {
|
|
|
1085
1379
|
}
|
|
1086
1380
|
get(streamPath) {
|
|
1087
1381
|
const meta = this.getMetaIfNotExpired(streamPath);
|
|
1088
|
-
|
|
1382
|
+
if (!meta) return void 0;
|
|
1383
|
+
return this.streamMetaToStream(meta);
|
|
1089
1384
|
}
|
|
1090
1385
|
has(streamPath) {
|
|
1091
|
-
|
|
1386
|
+
const meta = this.getMetaIfNotExpired(streamPath);
|
|
1387
|
+
if (!meta) return false;
|
|
1388
|
+
if (meta.softDeleted) return false;
|
|
1389
|
+
return true;
|
|
1092
1390
|
}
|
|
1093
1391
|
delete(streamPath) {
|
|
1094
1392
|
const key = `stream:${streamPath}`;
|
|
1095
1393
|
const streamMeta = this.db.get(key);
|
|
1096
1394
|
if (!streamMeta) return false;
|
|
1395
|
+
if (streamMeta.softDeleted) return true;
|
|
1396
|
+
if ((streamMeta.refCount ?? 0) > 0) {
|
|
1397
|
+
const updatedMeta = {
|
|
1398
|
+
...streamMeta,
|
|
1399
|
+
softDeleted: true
|
|
1400
|
+
};
|
|
1401
|
+
this.db.putSync(key, updatedMeta);
|
|
1402
|
+
this.cancelLongPollsForStream(streamPath);
|
|
1403
|
+
return true;
|
|
1404
|
+
}
|
|
1405
|
+
this.deleteWithCascade(streamPath);
|
|
1406
|
+
return true;
|
|
1407
|
+
}
|
|
1408
|
+
/**
|
|
1409
|
+
* Fully delete a stream and cascade to soft-deleted parents
|
|
1410
|
+
* whose refcount drops to zero.
|
|
1411
|
+
*/
|
|
1412
|
+
deleteWithCascade(streamPath) {
|
|
1413
|
+
const key = `stream:${streamPath}`;
|
|
1414
|
+
const streamMeta = this.db.get(key);
|
|
1415
|
+
if (!streamMeta) return;
|
|
1416
|
+
const forkedFrom = streamMeta.forkedFrom;
|
|
1097
1417
|
this.cancelLongPollsForStream(streamPath);
|
|
1098
1418
|
const segmentPath = node_path.join(this.dataDir, `streams`, streamMeta.directoryName, `segment_00000.log`);
|
|
1099
1419
|
this.fileHandlePool.closeFileHandle(segmentPath).catch((err) => {
|
|
@@ -1103,11 +1423,24 @@ var FileBackedStreamStore = class {
|
|
|
1103
1423
|
this.fileManager.deleteDirectoryByName(streamMeta.directoryName).catch((err) => {
|
|
1104
1424
|
console.error(`[FileBackedStreamStore] Error deleting stream directory:`, err);
|
|
1105
1425
|
});
|
|
1106
|
-
|
|
1426
|
+
if (forkedFrom) {
|
|
1427
|
+
const parentKey = `stream:${forkedFrom}`;
|
|
1428
|
+
const parentMeta = this.db.get(parentKey);
|
|
1429
|
+
if (parentMeta) {
|
|
1430
|
+
const newRefCount = Math.max(0, (parentMeta.refCount ?? 0) - 1);
|
|
1431
|
+
const updatedParent = {
|
|
1432
|
+
...parentMeta,
|
|
1433
|
+
refCount: newRefCount
|
|
1434
|
+
};
|
|
1435
|
+
this.db.putSync(parentKey, updatedParent);
|
|
1436
|
+
if (newRefCount === 0 && updatedParent.softDeleted) this.deleteWithCascade(forkedFrom);
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1107
1439
|
}
|
|
1108
1440
|
async append(streamPath, data, options = {}) {
|
|
1109
1441
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1110
1442
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1443
|
+
if (streamMeta.softDeleted) throw new Error(`Stream is soft-deleted: ${streamPath}`);
|
|
1111
1444
|
if (streamMeta.closed) {
|
|
1112
1445
|
if (options.producerId && streamMeta.closedBy && streamMeta.closedBy.producerId === options.producerId && streamMeta.closedBy.epoch === options.producerEpoch && streamMeta.closedBy.seq === options.producerSeq) return {
|
|
1113
1446
|
message: null,
|
|
@@ -1292,34 +1625,21 @@ var FileBackedStreamStore = class {
|
|
|
1292
1625
|
releaseLock();
|
|
1293
1626
|
}
|
|
1294
1627
|
}
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
if (streamMeta.currentOffset === `0000000000000000_0000000000000000`) return {
|
|
1305
|
-
messages: [],
|
|
1306
|
-
upToDate: true
|
|
1307
|
-
};
|
|
1308
|
-
if (startByte >= currentByte) return {
|
|
1309
|
-
messages: [],
|
|
1310
|
-
upToDate: true
|
|
1311
|
-
};
|
|
1312
|
-
const streamDir = node_path.join(this.dataDir, `streams`, streamMeta.directoryName);
|
|
1313
|
-
const segmentPath = node_path.join(streamDir, `segment_00000.log`);
|
|
1314
|
-
if (!node_fs.existsSync(segmentPath)) return {
|
|
1315
|
-
messages: [],
|
|
1316
|
-
upToDate: true
|
|
1317
|
-
};
|
|
1628
|
+
/**
|
|
1629
|
+
* Read messages from a specific segment file.
|
|
1630
|
+
* @param segmentPath - Path to the segment file
|
|
1631
|
+
* @param startByte - Start byte offset (skip messages at or before this offset)
|
|
1632
|
+
* @param baseByteOffset - Base byte offset to add to physical offsets (for fork stitching)
|
|
1633
|
+
* @param capByte - Optional cap: stop reading when logical offset exceeds this value
|
|
1634
|
+
* @returns Array of messages with properly computed offsets
|
|
1635
|
+
*/
|
|
1636
|
+
readMessagesFromSegmentFile(segmentPath, startByte, baseByteOffset, capByte) {
|
|
1318
1637
|
const messages = [];
|
|
1638
|
+
if (!node_fs.existsSync(segmentPath)) return messages;
|
|
1319
1639
|
try {
|
|
1320
1640
|
const fileContent = node_fs.readFileSync(segmentPath);
|
|
1321
1641
|
let filePos = 0;
|
|
1322
|
-
let
|
|
1642
|
+
let physicalDataOffset = 0;
|
|
1323
1643
|
while (filePos < fileContent.length) {
|
|
1324
1644
|
if (filePos + 4 > fileContent.length) break;
|
|
1325
1645
|
const messageLength = fileContent.readUInt32BE(filePos);
|
|
@@ -1328,16 +1648,72 @@ var FileBackedStreamStore = class {
|
|
|
1328
1648
|
const messageData = fileContent.subarray(filePos, filePos + messageLength);
|
|
1329
1649
|
filePos += messageLength;
|
|
1330
1650
|
filePos += 1;
|
|
1331
|
-
|
|
1332
|
-
|
|
1651
|
+
physicalDataOffset += messageLength;
|
|
1652
|
+
const logicalOffset = baseByteOffset + physicalDataOffset;
|
|
1653
|
+
if (capByte !== void 0 && logicalOffset > capByte) break;
|
|
1654
|
+
if (logicalOffset > startByte) messages.push({
|
|
1333
1655
|
data: new Uint8Array(messageData),
|
|
1334
|
-
offset: `${String(
|
|
1656
|
+
offset: `${String(0).padStart(16, `0`)}_${String(logicalOffset).padStart(16, `0`)}`,
|
|
1335
1657
|
timestamp: 0
|
|
1336
1658
|
});
|
|
1337
|
-
currentDataOffset = messageOffset;
|
|
1338
1659
|
}
|
|
1339
1660
|
} catch (err) {
|
|
1340
|
-
console.error(`[FileBackedStreamStore] Error reading file:`, err);
|
|
1661
|
+
console.error(`[FileBackedStreamStore] Error reading segment file:`, err);
|
|
1662
|
+
}
|
|
1663
|
+
return messages;
|
|
1664
|
+
}
|
|
1665
|
+
/**
|
|
1666
|
+
* Recursively read messages from a fork's source chain.
|
|
1667
|
+
* Reads from source (and its sources if also forked), capped at capByte.
|
|
1668
|
+
* Does NOT check softDeleted -- forks must read through soft-deleted sources.
|
|
1669
|
+
*/
|
|
1670
|
+
readForkedMessages(sourcePath, startByte, capByte) {
|
|
1671
|
+
const sourceKey = `stream:${sourcePath}`;
|
|
1672
|
+
const sourceMeta = this.db.get(sourceKey);
|
|
1673
|
+
if (!sourceMeta) return [];
|
|
1674
|
+
const messages = [];
|
|
1675
|
+
if (sourceMeta.forkedFrom && sourceMeta.forkOffset) {
|
|
1676
|
+
const sourceForkByte = Number(sourceMeta.forkOffset.split(`_`)[1] ?? 0);
|
|
1677
|
+
if (startByte < sourceForkByte) {
|
|
1678
|
+
const inheritedCap = Math.min(sourceForkByte, capByte);
|
|
1679
|
+
const inherited = this.readForkedMessages(sourceMeta.forkedFrom, startByte, inheritedCap);
|
|
1680
|
+
messages.push(...inherited);
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
const segmentPath = node_path.join(this.dataDir, `streams`, sourceMeta.directoryName, `segment_00000.log`);
|
|
1684
|
+
const sourceBaseByte = sourceMeta.forkOffset ? Number(sourceMeta.forkOffset.split(`_`)[1] ?? 0) : 0;
|
|
1685
|
+
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, sourceBaseByte, capByte);
|
|
1686
|
+
messages.push(...ownMessages);
|
|
1687
|
+
return messages;
|
|
1688
|
+
}
|
|
1689
|
+
read(streamPath, offset) {
|
|
1690
|
+
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1691
|
+
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1692
|
+
const startOffset = offset ?? `0000000000000000_0000000000000000`;
|
|
1693
|
+
const startByte = Number(startOffset.split(`_`)[1] ?? 0);
|
|
1694
|
+
const currentByte = Number(streamMeta.currentOffset.split(`_`)[1] ?? 0);
|
|
1695
|
+
if (streamMeta.currentOffset === `0000000000000000_0000000000000000`) return {
|
|
1696
|
+
messages: [],
|
|
1697
|
+
upToDate: true
|
|
1698
|
+
};
|
|
1699
|
+
if (startByte >= currentByte) return {
|
|
1700
|
+
messages: [],
|
|
1701
|
+
upToDate: true
|
|
1702
|
+
};
|
|
1703
|
+
const messages = [];
|
|
1704
|
+
if (streamMeta.forkedFrom && streamMeta.forkOffset) {
|
|
1705
|
+
const forkByte = Number(streamMeta.forkOffset.split(`_`)[1] ?? 0);
|
|
1706
|
+
if (startByte < forkByte) {
|
|
1707
|
+
const inherited = this.readForkedMessages(streamMeta.forkedFrom, startByte, forkByte);
|
|
1708
|
+
messages.push(...inherited);
|
|
1709
|
+
}
|
|
1710
|
+
const segmentPath = node_path.join(this.dataDir, `streams`, streamMeta.directoryName, `segment_00000.log`);
|
|
1711
|
+
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, forkByte);
|
|
1712
|
+
messages.push(...ownMessages);
|
|
1713
|
+
} else {
|
|
1714
|
+
const segmentPath = node_path.join(this.dataDir, `streams`, streamMeta.directoryName, `segment_00000.log`);
|
|
1715
|
+
const ownMessages = this.readMessagesFromSegmentFile(segmentPath, startByte, 0);
|
|
1716
|
+
messages.push(...ownMessages);
|
|
1341
1717
|
}
|
|
1342
1718
|
return {
|
|
1343
1719
|
messages,
|
|
@@ -1347,6 +1723,13 @@ var FileBackedStreamStore = class {
|
|
|
1347
1723
|
async waitForMessages(streamPath, offset, timeoutMs) {
|
|
1348
1724
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1349
1725
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
1726
|
+
if (streamMeta.forkedFrom && streamMeta.forkOffset && offset < streamMeta.forkOffset) {
|
|
1727
|
+
const { messages: messages$1 } = this.read(streamPath, offset);
|
|
1728
|
+
return {
|
|
1729
|
+
messages: messages$1,
|
|
1730
|
+
timedOut: false
|
|
1731
|
+
};
|
|
1732
|
+
}
|
|
1350
1733
|
if (streamMeta.closed && offset === streamMeta.currentOffset) return {
|
|
1351
1734
|
messages: [],
|
|
1352
1735
|
timedOut: false,
|
|
@@ -1599,6 +1982,8 @@ const SSE_CURSOR_FIELD = `streamCursor`;
|
|
|
1599
1982
|
const SSE_UP_TO_DATE_FIELD = `upToDate`;
|
|
1600
1983
|
const SSE_CLOSED_FIELD = `streamClosed`;
|
|
1601
1984
|
const STREAM_CLOSED_HEADER = `Stream-Closed`;
|
|
1985
|
+
const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`;
|
|
1986
|
+
const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`;
|
|
1602
1987
|
const OFFSET_QUERY_PARAM = `offset`;
|
|
1603
1988
|
const LIVE_QUERY_PARAM = `live`;
|
|
1604
1989
|
const CURSOR_QUERY_PARAM = `cursor`;
|
|
@@ -1813,7 +2198,7 @@ var DurableStreamTestServer = class {
|
|
|
1813
2198
|
const method = req.method?.toUpperCase();
|
|
1814
2199
|
res.setHeader(`access-control-allow-origin`, `*`);
|
|
1815
2200
|
res.setHeader(`access-control-allow-methods`, `GET, POST, PUT, DELETE, HEAD, OPTIONS`);
|
|
1816
|
-
res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq`);
|
|
2201
|
+
res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset`);
|
|
1817
2202
|
res.setHeader(`access-control-expose-headers`, `Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Date, Stream-Closed, Producer-Epoch, Producer-Seq, Producer-Expected-Seq, Producer-Received-Seq, etag, content-type, content-encoding, vary`);
|
|
1818
2203
|
res.setHeader(`x-content-type-options`, `nosniff`);
|
|
1819
2204
|
res.setHeader(`cross-origin-resource-policy`, `cross-origin`);
|
|
@@ -1864,7 +2249,13 @@ var DurableStreamTestServer = class {
|
|
|
1864
2249
|
res.end(`Method not allowed`);
|
|
1865
2250
|
}
|
|
1866
2251
|
} catch (err) {
|
|
1867
|
-
if (err instanceof Error) if (err.message.includes(`
|
|
2252
|
+
if (err instanceof Error) if (err.message.includes(`active forks`)) {
|
|
2253
|
+
res.writeHead(409, { "content-type": `text/plain` });
|
|
2254
|
+
res.end(`stream was deleted but still has active forks — path cannot be reused until all forks are removed`);
|
|
2255
|
+
} else if (err.message.includes(`soft-deleted`)) {
|
|
2256
|
+
res.writeHead(410, { "content-type": `text/plain` });
|
|
2257
|
+
res.end(`Stream is gone`);
|
|
2258
|
+
} else if (err.message.includes(`not found`)) {
|
|
1868
2259
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
1869
2260
|
res.end(`Stream not found`);
|
|
1870
2261
|
} else if (err.message.includes(`already exists with different configuration`)) {
|
|
@@ -1896,6 +2287,8 @@ var DurableStreamTestServer = class {
|
|
|
1896
2287
|
const expiresAtHeader = req.headers[STREAM_EXPIRES_AT_HEADER.toLowerCase()];
|
|
1897
2288
|
const closedHeader = req.headers[STREAM_CLOSED_HEADER.toLowerCase()];
|
|
1898
2289
|
const createClosed = closedHeader === `true`;
|
|
2290
|
+
const forkedFromHeader = req.headers[STREAM_FORKED_FROM_HEADER.toLowerCase()];
|
|
2291
|
+
const forkOffsetHeader = req.headers[STREAM_FORK_OFFSET_HEADER.toLowerCase()];
|
|
1899
2292
|
if (ttlHeader && expiresAtHeader) {
|
|
1900
2293
|
res.writeHead(400, { "content-type": `text/plain` });
|
|
1901
2294
|
res.end(`Cannot specify both Stream-TTL and Stream-Expires-At`);
|
|
@@ -1924,24 +2317,60 @@ var DurableStreamTestServer = class {
|
|
|
1924
2317
|
return;
|
|
1925
2318
|
}
|
|
1926
2319
|
}
|
|
2320
|
+
if (forkOffsetHeader) {
|
|
2321
|
+
const validOffsetPattern = /^\d+_\d+$/;
|
|
2322
|
+
if (!validOffsetPattern.test(forkOffsetHeader)) {
|
|
2323
|
+
res.writeHead(400, { "content-type": `text/plain` });
|
|
2324
|
+
res.end(`Invalid Stream-Fork-Offset format`);
|
|
2325
|
+
return;
|
|
2326
|
+
}
|
|
2327
|
+
}
|
|
1927
2328
|
const body = await this.readBody(req);
|
|
1928
2329
|
const isNew = !this.store.has(path);
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
2330
|
+
try {
|
|
2331
|
+
await Promise.resolve(this.store.create(path, {
|
|
2332
|
+
contentType,
|
|
2333
|
+
ttlSeconds,
|
|
2334
|
+
expiresAt: expiresAtHeader,
|
|
2335
|
+
initialData: body.length > 0 ? body : void 0,
|
|
2336
|
+
closed: createClosed,
|
|
2337
|
+
forkedFrom: forkedFromHeader,
|
|
2338
|
+
forkOffset: forkOffsetHeader
|
|
2339
|
+
}));
|
|
2340
|
+
} catch (err) {
|
|
2341
|
+
if (err instanceof Error) {
|
|
2342
|
+
if (err.message.includes(`Source stream not found`)) {
|
|
2343
|
+
res.writeHead(404, { "content-type": `text/plain` });
|
|
2344
|
+
res.end(`Source stream not found`);
|
|
2345
|
+
return;
|
|
2346
|
+
}
|
|
2347
|
+
if (err.message.includes(`Invalid fork offset`)) {
|
|
2348
|
+
res.writeHead(400, { "content-type": `text/plain` });
|
|
2349
|
+
res.end(`Fork offset beyond source stream length`);
|
|
2350
|
+
return;
|
|
2351
|
+
}
|
|
2352
|
+
if (err.message.includes(`soft-deleted`)) {
|
|
2353
|
+
res.writeHead(409, { "content-type": `text/plain` });
|
|
2354
|
+
res.end(`source stream was deleted but still has active forks`);
|
|
2355
|
+
return;
|
|
2356
|
+
}
|
|
2357
|
+
if (err.message.includes(`Content type mismatch`)) {
|
|
2358
|
+
res.writeHead(409, { "content-type": `text/plain` });
|
|
2359
|
+
res.end(`Content type mismatch with source stream`);
|
|
2360
|
+
return;
|
|
2361
|
+
}
|
|
2362
|
+
}
|
|
2363
|
+
throw err;
|
|
2364
|
+
}
|
|
1936
2365
|
const stream = this.store.get(path);
|
|
1937
2366
|
if (isNew && this.options.onStreamCreated) await Promise.resolve(this.options.onStreamCreated({
|
|
1938
2367
|
type: `created`,
|
|
1939
2368
|
path,
|
|
1940
|
-
contentType,
|
|
2369
|
+
contentType: stream.contentType ?? contentType,
|
|
1941
2370
|
timestamp: Date.now()
|
|
1942
2371
|
}));
|
|
1943
2372
|
const headers = {
|
|
1944
|
-
"content-type": contentType,
|
|
2373
|
+
"content-type": stream.contentType ?? contentType,
|
|
1945
2374
|
[STREAM_OFFSET_HEADER]: stream.currentOffset
|
|
1946
2375
|
};
|
|
1947
2376
|
if (isNew) headers[`location`] = `${this._url}${path}`;
|
|
@@ -1959,12 +2388,19 @@ var DurableStreamTestServer = class {
|
|
|
1959
2388
|
res.end();
|
|
1960
2389
|
return;
|
|
1961
2390
|
}
|
|
2391
|
+
if (stream.softDeleted) {
|
|
2392
|
+
res.writeHead(410, { "content-type": `text/plain` });
|
|
2393
|
+
res.end();
|
|
2394
|
+
return;
|
|
2395
|
+
}
|
|
1962
2396
|
const headers = {
|
|
1963
2397
|
[STREAM_OFFSET_HEADER]: stream.currentOffset,
|
|
1964
2398
|
"cache-control": `no-store`
|
|
1965
2399
|
};
|
|
1966
2400
|
if (stream.contentType) headers[`content-type`] = stream.contentType;
|
|
1967
2401
|
if (stream.closed) headers[STREAM_CLOSED_HEADER] = `true`;
|
|
2402
|
+
if (stream.ttlSeconds !== void 0) headers[STREAM_TTL_HEADER] = String(stream.ttlSeconds);
|
|
2403
|
+
if (stream.expiresAt) headers[STREAM_EXPIRES_AT_HEADER] = stream.expiresAt;
|
|
1968
2404
|
const closedSuffix = stream.closed ? `:c` : ``;
|
|
1969
2405
|
headers[`etag`] = `"${Buffer.from(path).toString(`base64`)}:-1:${stream.currentOffset}${closedSuffix}"`;
|
|
1970
2406
|
res.writeHead(200, headers);
|
|
@@ -1980,6 +2416,11 @@ var DurableStreamTestServer = class {
|
|
|
1980
2416
|
res.end(`Stream not found`);
|
|
1981
2417
|
return;
|
|
1982
2418
|
}
|
|
2419
|
+
if (stream.softDeleted) {
|
|
2420
|
+
res.writeHead(410, { "content-type": `text/plain` });
|
|
2421
|
+
res.end(`Stream is gone`);
|
|
2422
|
+
return;
|
|
2423
|
+
}
|
|
1983
2424
|
const offset = url.searchParams.get(OFFSET_QUERY_PARAM) ?? void 0;
|
|
1984
2425
|
const live = url.searchParams.get(LIVE_QUERY_PARAM);
|
|
1985
2426
|
const cursor = url.searchParams.get(CURSOR_QUERY_PARAM) ?? void 0;
|
|
@@ -2034,6 +2475,7 @@ var DurableStreamTestServer = class {
|
|
|
2034
2475
|
return;
|
|
2035
2476
|
}
|
|
2036
2477
|
let { messages, upToDate } = this.store.read(path, effectiveOffset);
|
|
2478
|
+
this.store.touchAccess(path);
|
|
2037
2479
|
const clientIsCaughtUp = effectiveOffset && effectiveOffset === stream.currentOffset || offset === `now`;
|
|
2038
2480
|
if (live === `long-poll` && clientIsCaughtUp && messages.length === 0) {
|
|
2039
2481
|
if (stream.closed) {
|
|
@@ -2046,6 +2488,7 @@ var DurableStreamTestServer = class {
|
|
|
2046
2488
|
return;
|
|
2047
2489
|
}
|
|
2048
2490
|
const result = await this.store.waitForMessages(path, effectiveOffset ?? stream.currentOffset, this.options.longPollTimeout);
|
|
2491
|
+
this.store.touchAccess(path);
|
|
2049
2492
|
if (result.streamClosed) {
|
|
2050
2493
|
const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
|
|
2051
2494
|
res.writeHead(204, {
|
|
@@ -2138,6 +2581,7 @@ var DurableStreamTestServer = class {
|
|
|
2138
2581
|
const isJsonStream = stream?.contentType?.includes(`application/json`);
|
|
2139
2582
|
while (isConnected && !this.isShuttingDown) {
|
|
2140
2583
|
const { messages, upToDate } = this.store.read(path, currentOffset);
|
|
2584
|
+
this.store.touchAccess(path);
|
|
2141
2585
|
for (const message of messages) {
|
|
2142
2586
|
let dataPayload;
|
|
2143
2587
|
if (useBase64) dataPayload = Buffer.from(message.data).toString(`base64`);
|
|
@@ -2175,6 +2619,7 @@ var DurableStreamTestServer = class {
|
|
|
2175
2619
|
break;
|
|
2176
2620
|
}
|
|
2177
2621
|
const result = await this.store.waitForMessages(path, currentOffset, this.options.longPollTimeout);
|
|
2622
|
+
this.store.touchAccess(path);
|
|
2178
2623
|
if (this.isShuttingDown || !isConnected) break;
|
|
2179
2624
|
if (result.streamClosed) {
|
|
2180
2625
|
const finalControlData = {
|
|
@@ -2357,6 +2802,7 @@ var DurableStreamTestServer = class {
|
|
|
2357
2802
|
let result;
|
|
2358
2803
|
if (producerId !== void 0) result = await this.store.appendWithProducer(path, body, appendOptions);
|
|
2359
2804
|
else result = await Promise.resolve(this.store.append(path, body, appendOptions));
|
|
2805
|
+
this.store.touchAccess(path);
|
|
2360
2806
|
if (result && typeof result === `object` && `message` in result) {
|
|
2361
2807
|
const { message: message$1, producerResult, streamClosed } = result;
|
|
2362
2808
|
if (streamClosed && !message$1) {
|
|
@@ -2433,12 +2879,18 @@ var DurableStreamTestServer = class {
|
|
|
2433
2879
|
* Handle DELETE - delete stream
|
|
2434
2880
|
*/
|
|
2435
2881
|
async handleDelete(path, res) {
|
|
2436
|
-
|
|
2882
|
+
const existing = this.store.get(path);
|
|
2883
|
+
if (existing?.softDeleted) {
|
|
2884
|
+
res.writeHead(410, { "content-type": `text/plain` });
|
|
2885
|
+
res.end(`Stream is gone`);
|
|
2886
|
+
return;
|
|
2887
|
+
}
|
|
2888
|
+
const deleted = this.store.delete(path);
|
|
2889
|
+
if (!deleted) {
|
|
2437
2890
|
res.writeHead(404, { "content-type": `text/plain` });
|
|
2438
2891
|
res.end(`Stream not found`);
|
|
2439
2892
|
return;
|
|
2440
2893
|
}
|
|
2441
|
-
this.store.delete(path);
|
|
2442
2894
|
if (this.options.onStreamDeleted) await Promise.resolve(this.options.onStreamDeleted({
|
|
2443
2895
|
type: `deleted`,
|
|
2444
2896
|
path,
|