@durable-streams/server 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +145 -7
- package/dist/index.d.cts +24 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +145 -7
- package/package.json +4 -4
- package/src/file-store.ts +141 -9
- package/src/server.ts +35 -1
- package/src/store.ts +119 -8
- package/src/types.ts +8 -0
package/dist/index.cjs
CHANGED
|
@@ -170,29 +170,35 @@ var StreamStore = class {
|
|
|
170
170
|
const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
|
|
171
171
|
const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
|
|
172
172
|
const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
|
|
173
|
-
|
|
173
|
+
const requestedSub = options.forkSubOffset ?? 0;
|
|
174
|
+
const existingSub = existingRaw.forkSubOffset ?? 0;
|
|
175
|
+
const forkSubOffsetMatches = requestedSub === existingSub;
|
|
176
|
+
if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches && forkSubOffsetMatches) return existingRaw;
|
|
174
177
|
else throw new Error(`Stream already exists with different configuration: ${path}`);
|
|
175
178
|
}
|
|
176
179
|
const isFork = !!options.forkedFrom;
|
|
177
180
|
let forkOffset = `0000000000000000_0000000000000000`;
|
|
178
181
|
let sourceContentType;
|
|
179
182
|
let sourceStream;
|
|
183
|
+
let forkSubOffsetPrefix;
|
|
180
184
|
if (isFork) {
|
|
181
185
|
sourceStream = this.streams.get(options.forkedFrom);
|
|
182
186
|
if (!sourceStream) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
183
187
|
if (sourceStream.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
|
|
184
188
|
if (this.isExpired(sourceStream)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
185
189
|
sourceContentType = sourceStream.contentType;
|
|
190
|
+
if (options.contentType && options.contentType.trim() !== `` && normalizeContentType(options.contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
|
|
186
191
|
if (options.forkOffset) forkOffset = options.forkOffset;
|
|
187
192
|
else forkOffset = sourceStream.currentOffset;
|
|
188
193
|
const zeroOffset = `0000000000000000_0000000000000000`;
|
|
189
194
|
if (forkOffset < zeroOffset || sourceStream.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
|
|
195
|
+
if (options.forkSubOffset && options.forkSubOffset > 0) forkSubOffsetPrefix = this.resolveForkSubOffset(sourceStream, forkOffset, options.forkSubOffset, normalizeContentType(sourceContentType) === `application/json`);
|
|
190
196
|
sourceStream.refCount++;
|
|
191
197
|
}
|
|
192
198
|
let contentType = options.contentType;
|
|
193
199
|
if (!contentType || contentType.trim() === ``) {
|
|
194
200
|
if (isFork) contentType = sourceContentType;
|
|
195
|
-
}
|
|
201
|
+
}
|
|
196
202
|
let effectiveExpiresAt = options.expiresAt;
|
|
197
203
|
let effectiveTtlSeconds = options.ttlSeconds;
|
|
198
204
|
if (isFork) {
|
|
@@ -214,6 +220,20 @@ var StreamStore = class {
|
|
|
214
220
|
forkedFrom: isFork ? options.forkedFrom : void 0,
|
|
215
221
|
forkOffset: isFork ? forkOffset : void 0
|
|
216
222
|
};
|
|
223
|
+
if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
|
|
224
|
+
const parts = stream.currentOffset.split(`_`).map(Number);
|
|
225
|
+
const readSeq = parts[0];
|
|
226
|
+
const byteOffset = parts[1];
|
|
227
|
+
const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5;
|
|
228
|
+
const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
229
|
+
stream.messages.push({
|
|
230
|
+
data: forkSubOffsetPrefix,
|
|
231
|
+
offset: newOffset,
|
|
232
|
+
timestamp: Date.now()
|
|
233
|
+
});
|
|
234
|
+
stream.currentOffset = newOffset;
|
|
235
|
+
stream.forkSubOffset = options.forkSubOffset;
|
|
236
|
+
}
|
|
217
237
|
if (options.initialData && options.initialData.length > 0) try {
|
|
218
238
|
this.appendToStream(stream, options.initialData, true);
|
|
219
239
|
} catch (err) {
|
|
@@ -737,6 +757,36 @@ var StreamStore = class {
|
|
|
737
757
|
list() {
|
|
738
758
|
return Array.from(this.streams.keys());
|
|
739
759
|
}
|
|
760
|
+
/**
|
|
761
|
+
* Resolve a sub-offset against a source stream and return the prefix bytes
|
|
762
|
+
* to materialize as the fork's first own message. Reads from the source
|
|
763
|
+
* (across its fork chain if any) starting at forkOffset; the first message
|
|
764
|
+
* returned is the one that starts at forkOffset. Throws if the sub-offset
|
|
765
|
+
* cannot be satisfied (no message past forkOffset, or overshoots its
|
|
766
|
+
* content extent).
|
|
767
|
+
*/
|
|
768
|
+
resolveForkSubOffset(sourceStream, forkOffset, subOffset, isJSON) {
|
|
769
|
+
let sourceMessages;
|
|
770
|
+
if (sourceStream.forkedFrom) sourceMessages = [...this.readForkedMessages(sourceStream.forkedFrom, forkOffset, sourceStream.forkOffset), ...this.readOwnMessages(sourceStream, forkOffset)];
|
|
771
|
+
else sourceMessages = this.readOwnMessages(sourceStream, forkOffset);
|
|
772
|
+
if (sourceMessages.length === 0) throw new Error(`Invalid fork sub-offset: no data past forkOffset`);
|
|
773
|
+
const first = sourceMessages[0];
|
|
774
|
+
if (isJSON) {
|
|
775
|
+
const text = new TextDecoder().decode(first.data);
|
|
776
|
+
const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text;
|
|
777
|
+
let values;
|
|
778
|
+
try {
|
|
779
|
+
values = JSON.parse(`[${trimmed}]`);
|
|
780
|
+
} catch {
|
|
781
|
+
throw new Error(`Invalid fork sub-offset: source JSON is unparseable`);
|
|
782
|
+
}
|
|
783
|
+
if (subOffset > values.length) throw new Error(`Invalid fork sub-offset: overshoots source message count`);
|
|
784
|
+
const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v));
|
|
785
|
+
return new TextEncoder().encode(prefix.join(`,`) + `,`);
|
|
786
|
+
}
|
|
787
|
+
if (subOffset > first.data.length) throw new Error(`Invalid fork sub-offset: overshoots source message length`);
|
|
788
|
+
return first.data.slice(0, subOffset);
|
|
789
|
+
}
|
|
740
790
|
appendToStream(stream, data, isInitialCreate = false) {
|
|
741
791
|
let processedData = data;
|
|
742
792
|
if (normalizeContentType(stream.contentType) === `application/json`) {
|
|
@@ -1318,13 +1368,17 @@ var FileBackedStreamStore = class {
|
|
|
1318
1368
|
const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
|
|
1319
1369
|
const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
|
|
1320
1370
|
const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
|
|
1321
|
-
|
|
1371
|
+
const requestedSub = options.forkSubOffset ?? 0;
|
|
1372
|
+
const existingSub = existingRaw.forkSubOffset ?? 0;
|
|
1373
|
+
const forkSubOffsetMatches = requestedSub === existingSub;
|
|
1374
|
+
if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches && forkSubOffsetMatches) return this.streamMetaToStream(existingRaw);
|
|
1322
1375
|
else throw new Error(`Stream already exists with different configuration: ${streamPath}`);
|
|
1323
1376
|
}
|
|
1324
1377
|
const isFork = !!options.forkedFrom;
|
|
1325
1378
|
let forkOffset = `0000000000000000_0000000000000000`;
|
|
1326
1379
|
let sourceContentType;
|
|
1327
1380
|
let sourceMeta;
|
|
1381
|
+
let forkSubOffsetPrefix;
|
|
1328
1382
|
if (isFork) {
|
|
1329
1383
|
const sourceKey = `stream:${options.forkedFrom}`;
|
|
1330
1384
|
sourceMeta = this.db.get(sourceKey);
|
|
@@ -1332,10 +1386,12 @@ var FileBackedStreamStore = class {
|
|
|
1332
1386
|
if (sourceMeta.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
|
|
1333
1387
|
if (this.isExpired(sourceMeta)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
1334
1388
|
sourceContentType = sourceMeta.contentType;
|
|
1389
|
+
if (options.contentType && options.contentType.trim() !== `` && normalizeContentType(options.contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
|
|
1335
1390
|
if (options.forkOffset) forkOffset = options.forkOffset;
|
|
1336
1391
|
else forkOffset = sourceMeta.currentOffset;
|
|
1337
1392
|
const zeroOffset = `0000000000000000_0000000000000000`;
|
|
1338
1393
|
if (forkOffset < zeroOffset || sourceMeta.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
|
|
1394
|
+
if (options.forkSubOffset && options.forkSubOffset > 0) forkSubOffsetPrefix = this.resolveForkSubOffset(options.forkedFrom, forkOffset, options.forkSubOffset, normalizeContentType(sourceContentType) === `application/json`);
|
|
1339
1395
|
const freshSource = this.db.get(sourceKey);
|
|
1340
1396
|
const updatedSource = {
|
|
1341
1397
|
...freshSource,
|
|
@@ -1346,7 +1402,7 @@ var FileBackedStreamStore = class {
|
|
|
1346
1402
|
let contentType = options.contentType;
|
|
1347
1403
|
if (!contentType || contentType.trim() === ``) {
|
|
1348
1404
|
if (isFork) contentType = sourceContentType;
|
|
1349
|
-
}
|
|
1405
|
+
}
|
|
1350
1406
|
let effectiveExpiresAt = options.expiresAt;
|
|
1351
1407
|
let effectiveTtlSeconds = options.ttlSeconds;
|
|
1352
1408
|
if (isFork) {
|
|
@@ -1371,11 +1427,39 @@ var FileBackedStreamStore = class {
|
|
|
1371
1427
|
closed: false,
|
|
1372
1428
|
forkedFrom: isFork ? options.forkedFrom : void 0,
|
|
1373
1429
|
forkOffset: isFork ? forkOffset : void 0,
|
|
1430
|
+
forkSubOffset: void 0,
|
|
1374
1431
|
refCount: 0
|
|
1375
1432
|
};
|
|
1376
1433
|
const tAfterMeta = performance.now();
|
|
1377
1434
|
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1378
1435
|
try {
|
|
1436
|
+
if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
|
|
1437
|
+
const lengthBuf = Buffer.allocUnsafe(4);
|
|
1438
|
+
lengthBuf.writeUInt32BE(forkSubOffsetPrefix.length, 0);
|
|
1439
|
+
const frameBuf = Buffer.concat([
|
|
1440
|
+
lengthBuf,
|
|
1441
|
+
Buffer.from(forkSubOffsetPrefix),
|
|
1442
|
+
Buffer.from(`\n`)
|
|
1443
|
+
]);
|
|
1444
|
+
const fd = node_fs.openSync(segmentPath, `wx`);
|
|
1445
|
+
try {
|
|
1446
|
+
let written = 0;
|
|
1447
|
+
while (written < frameBuf.length) {
|
|
1448
|
+
const bytesWritten = node_fs.writeSync(fd, frameBuf, written, frameBuf.length - written, written);
|
|
1449
|
+
if (bytesWritten === 0) throw new Error(`failed to write sub-offset prefix frame`);
|
|
1450
|
+
written += bytesWritten;
|
|
1451
|
+
}
|
|
1452
|
+
node_fs.fsyncSync(fd);
|
|
1453
|
+
} finally {
|
|
1454
|
+
node_fs.closeSync(fd);
|
|
1455
|
+
}
|
|
1456
|
+
const parts = streamMeta.currentOffset.split(`_`).map(Number);
|
|
1457
|
+
const readSeq = parts[0];
|
|
1458
|
+
const byteOffset = parts[1];
|
|
1459
|
+
const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5;
|
|
1460
|
+
streamMeta.currentOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
1461
|
+
streamMeta.forkSubOffset = options.forkSubOffset;
|
|
1462
|
+
}
|
|
1379
1463
|
await this.db.put(key, streamMeta);
|
|
1380
1464
|
} catch (err) {
|
|
1381
1465
|
if (isFork && sourceMeta) {
|
|
@@ -1389,7 +1473,7 @@ var FileBackedStreamStore = class {
|
|
|
1389
1473
|
this.db.putSync(sourceKey, updatedSource);
|
|
1390
1474
|
}
|
|
1391
1475
|
}
|
|
1392
|
-
serverLog.error(`[FileBackedStreamStore] Error creating stream
|
|
1476
|
+
serverLog.error(`[FileBackedStreamStore] Error creating stream before metadata commit:`, err);
|
|
1393
1477
|
throw err;
|
|
1394
1478
|
}
|
|
1395
1479
|
const tAfterLmdb = performance.now();
|
|
@@ -1774,6 +1858,35 @@ var FileBackedStreamStore = class {
|
|
|
1774
1858
|
messages.push(...ownMessages);
|
|
1775
1859
|
return messages;
|
|
1776
1860
|
}
|
|
1861
|
+
/**
|
|
1862
|
+
* Resolve a fork sub-offset against the source: read the message that
|
|
1863
|
+
* starts at forkOffset and return prefix bytes to materialize as the
|
|
1864
|
+
* fork's first own message. For JSON, parses comma-joined values.
|
|
1865
|
+
*/
|
|
1866
|
+
resolveForkSubOffset(sourcePath, forkOffset, subOffset, isJSON) {
|
|
1867
|
+
const forkByte = Number(forkOffset.split(`_`)[1] ?? 0);
|
|
1868
|
+
const sourceMeta = this.db.get(`stream:${sourcePath}`);
|
|
1869
|
+
if (!sourceMeta) throw new Error(`Source stream not found: ${sourcePath}`);
|
|
1870
|
+
const currentByte = Number(sourceMeta.currentOffset.split(`_`)[1] ?? 0);
|
|
1871
|
+
const messages = this.readForkedMessages(sourcePath, forkByte, currentByte);
|
|
1872
|
+
if (messages.length === 0) throw new Error(`Invalid fork sub-offset: no data past forkOffset`);
|
|
1873
|
+
const first = messages[0];
|
|
1874
|
+
if (isJSON) {
|
|
1875
|
+
const text = new TextDecoder().decode(first.data);
|
|
1876
|
+
const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text;
|
|
1877
|
+
let values;
|
|
1878
|
+
try {
|
|
1879
|
+
values = JSON.parse(`[${trimmed}]`);
|
|
1880
|
+
} catch {
|
|
1881
|
+
throw new Error(`Invalid fork sub-offset: source JSON is unparseable`);
|
|
1882
|
+
}
|
|
1883
|
+
if (subOffset > values.length) throw new Error(`Invalid fork sub-offset: overshoots source message count`);
|
|
1884
|
+
const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v));
|
|
1885
|
+
return new TextEncoder().encode(prefix.join(`,`) + `,`);
|
|
1886
|
+
}
|
|
1887
|
+
if (subOffset > first.data.length) throw new Error(`Invalid fork sub-offset: overshoots source message length`);
|
|
1888
|
+
return first.data.slice(0, subOffset);
|
|
1889
|
+
}
|
|
1777
1890
|
read(streamPath, offset) {
|
|
1778
1891
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1779
1892
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
@@ -3030,6 +3143,7 @@ const STREAM_SSE_DATA_ENCODING_HEADER = `Stream-SSE-Data-Encoding`;
|
|
|
3030
3143
|
const SSE_UP_TO_DATE_FIELD = `upToDate`;
|
|
3031
3144
|
const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`;
|
|
3032
3145
|
const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`;
|
|
3146
|
+
const STREAM_FORK_SUB_OFFSET_HEADER = `Stream-Fork-Sub-Offset`;
|
|
3033
3147
|
/**
|
|
3034
3148
|
* Encode data for SSE format.
|
|
3035
3149
|
* Per SSE spec, each line in the payload needs its own "data:" prefix.
|
|
@@ -3255,7 +3369,7 @@ var DurableStreamTestServer = class {
|
|
|
3255
3369
|
const method = req.method?.toUpperCase();
|
|
3256
3370
|
res.setHeader(`access-control-allow-origin`, `*`);
|
|
3257
3371
|
res.setHeader(`access-control-allow-methods`, `GET, POST, PUT, DELETE, HEAD, OPTIONS`);
|
|
3258
|
-
res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset`);
|
|
3372
|
+
res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset, Stream-Fork-Sub-Offset`);
|
|
3259
3373
|
res.setHeader(`access-control-expose-headers`, `Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Date, Stream-Closed, Producer-Epoch, Producer-Seq, Producer-Expected-Seq, Producer-Received-Seq, etag, content-type, content-encoding, vary`);
|
|
3260
3374
|
res.setHeader(`x-content-type-options`, `nosniff`);
|
|
3261
3375
|
res.setHeader(`cross-origin-resource-policy`, `cross-origin`);
|
|
@@ -3345,6 +3459,9 @@ var DurableStreamTestServer = class {
|
|
|
3345
3459
|
let contentType = req.headers[`content-type`];
|
|
3346
3460
|
const forkedFromHeader = req.headers[STREAM_FORKED_FROM_HEADER.toLowerCase()];
|
|
3347
3461
|
const forkOffsetHeader = req.headers[STREAM_FORK_OFFSET_HEADER.toLowerCase()];
|
|
3462
|
+
const forkSubOffsetHeaderRaw = req.headers[STREAM_FORK_SUB_OFFSET_HEADER.toLowerCase()];
|
|
3463
|
+
const forkSubOffsetHeaderPresent = forkSubOffsetHeaderRaw !== void 0;
|
|
3464
|
+
const forkSubOffsetHeader = Array.isArray(forkSubOffsetHeaderRaw) ? forkSubOffsetHeaderRaw[0] : forkSubOffsetHeaderRaw;
|
|
3348
3465
|
if (!contentType || contentType.trim() === `` || !/^[\w-]+\/[\w-]+/.test(contentType)) contentType = forkedFromHeader ? void 0 : `application/octet-stream`;
|
|
3349
3466
|
const ttlHeader = req.headers[__durable_streams_client.STREAM_TTL_HEADER.toLowerCase()];
|
|
3350
3467
|
const expiresAtHeader = req.headers[__durable_streams_client.STREAM_EXPIRES_AT_HEADER.toLowerCase()];
|
|
@@ -3386,6 +3503,21 @@ var DurableStreamTestServer = class {
|
|
|
3386
3503
|
return;
|
|
3387
3504
|
}
|
|
3388
3505
|
}
|
|
3506
|
+
let forkSubOffset;
|
|
3507
|
+
if (forkSubOffsetHeaderPresent) {
|
|
3508
|
+
if (!forkedFromHeader) {
|
|
3509
|
+
res.writeHead(400, { "content-type": `text/plain` });
|
|
3510
|
+
res.end(`Stream-Fork-Sub-Offset requires Stream-Forked-From`);
|
|
3511
|
+
return;
|
|
3512
|
+
}
|
|
3513
|
+
const subOffsetPattern = /^(0|[1-9]\d*)$/;
|
|
3514
|
+
if (forkSubOffsetHeader === void 0 || !subOffsetPattern.test(forkSubOffsetHeader)) {
|
|
3515
|
+
res.writeHead(400, { "content-type": `text/plain` });
|
|
3516
|
+
res.end(`Invalid Stream-Fork-Sub-Offset format`);
|
|
3517
|
+
return;
|
|
3518
|
+
}
|
|
3519
|
+
forkSubOffset = parseInt(forkSubOffsetHeader, 10);
|
|
3520
|
+
}
|
|
3389
3521
|
const body = await this.readBody(req);
|
|
3390
3522
|
const isNew = !this.store.has(path);
|
|
3391
3523
|
try {
|
|
@@ -3396,7 +3528,8 @@ var DurableStreamTestServer = class {
|
|
|
3396
3528
|
initialData: body.length > 0 ? body : void 0,
|
|
3397
3529
|
closed: createClosed,
|
|
3398
3530
|
forkedFrom: forkedFromHeader,
|
|
3399
|
-
forkOffset: forkOffsetHeader
|
|
3531
|
+
forkOffset: forkOffsetHeader,
|
|
3532
|
+
forkSubOffset
|
|
3400
3533
|
}));
|
|
3401
3534
|
} catch (err) {
|
|
3402
3535
|
if (err instanceof Error) {
|
|
@@ -3405,6 +3538,11 @@ var DurableStreamTestServer = class {
|
|
|
3405
3538
|
res.end(`Source stream not found`);
|
|
3406
3539
|
return;
|
|
3407
3540
|
}
|
|
3541
|
+
if (err.message.includes(`Invalid fork sub-offset`)) {
|
|
3542
|
+
res.writeHead(400, { "content-type": `text/plain` });
|
|
3543
|
+
res.end(`Invalid fork sub-offset`);
|
|
3544
|
+
return;
|
|
3545
|
+
}
|
|
3408
3546
|
if (err.message.includes(`Invalid fork offset`)) {
|
|
3409
3547
|
res.writeHead(400, { "content-type": `text/plain` });
|
|
3410
3548
|
res.end(`Fork offset beyond source stream length`);
|
package/dist/index.d.cts
CHANGED
|
@@ -99,6 +99,13 @@ interface Stream {
|
|
|
99
99
|
*/
|
|
100
100
|
forkOffset?: string;
|
|
101
101
|
/**
|
|
102
|
+
* User-supplied sub-offset value refining `forkOffset` (Section 4.2 of
|
|
103
|
+
* PROTOCOL.md). Stored verbatim for idempotent re-creation matching:
|
|
104
|
+
* bytes for non-JSON forks, flattened message count for JSON forks.
|
|
105
|
+
* `undefined` and `0` are equivalent.
|
|
106
|
+
*/
|
|
107
|
+
forkSubOffset?: number;
|
|
108
|
+
/**
|
|
102
109
|
* Number of forks referencing this stream.
|
|
103
110
|
* Defaults to 0.
|
|
104
111
|
*/
|
|
@@ -313,6 +320,7 @@ declare class StreamStore {
|
|
|
313
320
|
closed?: boolean;
|
|
314
321
|
forkedFrom?: string;
|
|
315
322
|
forkOffset?: string;
|
|
323
|
+
forkSubOffset?: number;
|
|
316
324
|
}): Stream;
|
|
317
325
|
/**
|
|
318
326
|
* Resolve fork expiry per the decision table.
|
|
@@ -459,6 +467,15 @@ declare class StreamStore {
|
|
|
459
467
|
* Get all stream paths.
|
|
460
468
|
*/
|
|
461
469
|
list(): Array<string>;
|
|
470
|
+
/**
|
|
471
|
+
* Resolve a sub-offset against a source stream and return the prefix bytes
|
|
472
|
+
* to materialize as the fork's first own message. Reads from the source
|
|
473
|
+
* (across its fork chain if any) starting at forkOffset; the first message
|
|
474
|
+
* returned is the one that starts at forkOffset. Throws if the sub-offset
|
|
475
|
+
* cannot be satisfied (no message past forkOffset, or overshoots its
|
|
476
|
+
* content extent).
|
|
477
|
+
*/
|
|
478
|
+
private resolveForkSubOffset;
|
|
462
479
|
private appendToStream;
|
|
463
480
|
private findOffsetIndex;
|
|
464
481
|
private notifyLongPolls;
|
|
@@ -571,6 +588,7 @@ declare class FileBackedStreamStore {
|
|
|
571
588
|
closed?: boolean;
|
|
572
589
|
forkedFrom?: string;
|
|
573
590
|
forkOffset?: string;
|
|
591
|
+
forkSubOffset?: number;
|
|
574
592
|
}): Promise<Stream>;
|
|
575
593
|
get(streamPath: string): Stream | undefined;
|
|
576
594
|
has(streamPath: string): boolean;
|
|
@@ -631,6 +649,12 @@ declare class FileBackedStreamStore {
|
|
|
631
649
|
* Does NOT check softDeleted -- forks must read through soft-deleted sources.
|
|
632
650
|
*/
|
|
633
651
|
private readForkedMessages;
|
|
652
|
+
/**
|
|
653
|
+
* Resolve a fork sub-offset against the source: read the message that
|
|
654
|
+
* starts at forkOffset and return prefix bytes to materialize as the
|
|
655
|
+
* fork's first own message. For JSON, parses comma-joined values.
|
|
656
|
+
*/
|
|
657
|
+
private resolveForkSubOffset;
|
|
634
658
|
read(streamPath: string, offset?: string): {
|
|
635
659
|
messages: Array<StreamMessage>;
|
|
636
660
|
upToDate: boolean;
|
package/dist/index.d.ts
CHANGED
|
@@ -99,6 +99,13 @@ interface Stream {
|
|
|
99
99
|
*/
|
|
100
100
|
forkOffset?: string;
|
|
101
101
|
/**
|
|
102
|
+
* User-supplied sub-offset value refining `forkOffset` (Section 4.2 of
|
|
103
|
+
* PROTOCOL.md). Stored verbatim for idempotent re-creation matching:
|
|
104
|
+
* bytes for non-JSON forks, flattened message count for JSON forks.
|
|
105
|
+
* `undefined` and `0` are equivalent.
|
|
106
|
+
*/
|
|
107
|
+
forkSubOffset?: number;
|
|
108
|
+
/**
|
|
102
109
|
* Number of forks referencing this stream.
|
|
103
110
|
* Defaults to 0.
|
|
104
111
|
*/
|
|
@@ -313,6 +320,7 @@ declare class StreamStore {
|
|
|
313
320
|
closed?: boolean;
|
|
314
321
|
forkedFrom?: string;
|
|
315
322
|
forkOffset?: string;
|
|
323
|
+
forkSubOffset?: number;
|
|
316
324
|
}): Stream;
|
|
317
325
|
/**
|
|
318
326
|
* Resolve fork expiry per the decision table.
|
|
@@ -459,6 +467,15 @@ declare class StreamStore {
|
|
|
459
467
|
* Get all stream paths.
|
|
460
468
|
*/
|
|
461
469
|
list(): Array<string>;
|
|
470
|
+
/**
|
|
471
|
+
* Resolve a sub-offset against a source stream and return the prefix bytes
|
|
472
|
+
* to materialize as the fork's first own message. Reads from the source
|
|
473
|
+
* (across its fork chain if any) starting at forkOffset; the first message
|
|
474
|
+
* returned is the one that starts at forkOffset. Throws if the sub-offset
|
|
475
|
+
* cannot be satisfied (no message past forkOffset, or overshoots its
|
|
476
|
+
* content extent).
|
|
477
|
+
*/
|
|
478
|
+
private resolveForkSubOffset;
|
|
462
479
|
private appendToStream;
|
|
463
480
|
private findOffsetIndex;
|
|
464
481
|
private notifyLongPolls;
|
|
@@ -571,6 +588,7 @@ declare class FileBackedStreamStore {
|
|
|
571
588
|
closed?: boolean;
|
|
572
589
|
forkedFrom?: string;
|
|
573
590
|
forkOffset?: string;
|
|
591
|
+
forkSubOffset?: number;
|
|
574
592
|
}): Promise<Stream>;
|
|
575
593
|
get(streamPath: string): Stream | undefined;
|
|
576
594
|
has(streamPath: string): boolean;
|
|
@@ -631,6 +649,12 @@ declare class FileBackedStreamStore {
|
|
|
631
649
|
* Does NOT check softDeleted -- forks must read through soft-deleted sources.
|
|
632
650
|
*/
|
|
633
651
|
private readForkedMessages;
|
|
652
|
+
/**
|
|
653
|
+
* Resolve a fork sub-offset against the source: read the message that
|
|
654
|
+
* starts at forkOffset and return prefix bytes to materialize as the
|
|
655
|
+
* fork's first own message. For JSON, parses comma-joined values.
|
|
656
|
+
*/
|
|
657
|
+
private resolveForkSubOffset;
|
|
634
658
|
read(streamPath: string, offset?: string): {
|
|
635
659
|
messages: Array<StreamMessage>;
|
|
636
660
|
upToDate: boolean;
|
package/dist/index.js
CHANGED
|
@@ -146,29 +146,35 @@ var StreamStore = class {
|
|
|
146
146
|
const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
|
|
147
147
|
const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
|
|
148
148
|
const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
|
|
149
|
-
|
|
149
|
+
const requestedSub = options.forkSubOffset ?? 0;
|
|
150
|
+
const existingSub = existingRaw.forkSubOffset ?? 0;
|
|
151
|
+
const forkSubOffsetMatches = requestedSub === existingSub;
|
|
152
|
+
if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches && forkSubOffsetMatches) return existingRaw;
|
|
150
153
|
else throw new Error(`Stream already exists with different configuration: ${path$1}`);
|
|
151
154
|
}
|
|
152
155
|
const isFork = !!options.forkedFrom;
|
|
153
156
|
let forkOffset = `0000000000000000_0000000000000000`;
|
|
154
157
|
let sourceContentType;
|
|
155
158
|
let sourceStream;
|
|
159
|
+
let forkSubOffsetPrefix;
|
|
156
160
|
if (isFork) {
|
|
157
161
|
sourceStream = this.streams.get(options.forkedFrom);
|
|
158
162
|
if (!sourceStream) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
159
163
|
if (sourceStream.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
|
|
160
164
|
if (this.isExpired(sourceStream)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
161
165
|
sourceContentType = sourceStream.contentType;
|
|
166
|
+
if (options.contentType && options.contentType.trim() !== `` && normalizeContentType(options.contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
|
|
162
167
|
if (options.forkOffset) forkOffset = options.forkOffset;
|
|
163
168
|
else forkOffset = sourceStream.currentOffset;
|
|
164
169
|
const zeroOffset = `0000000000000000_0000000000000000`;
|
|
165
170
|
if (forkOffset < zeroOffset || sourceStream.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
|
|
171
|
+
if (options.forkSubOffset && options.forkSubOffset > 0) forkSubOffsetPrefix = this.resolveForkSubOffset(sourceStream, forkOffset, options.forkSubOffset, normalizeContentType(sourceContentType) === `application/json`);
|
|
166
172
|
sourceStream.refCount++;
|
|
167
173
|
}
|
|
168
174
|
let contentType = options.contentType;
|
|
169
175
|
if (!contentType || contentType.trim() === ``) {
|
|
170
176
|
if (isFork) contentType = sourceContentType;
|
|
171
|
-
}
|
|
177
|
+
}
|
|
172
178
|
let effectiveExpiresAt = options.expiresAt;
|
|
173
179
|
let effectiveTtlSeconds = options.ttlSeconds;
|
|
174
180
|
if (isFork) {
|
|
@@ -190,6 +196,20 @@ var StreamStore = class {
|
|
|
190
196
|
forkedFrom: isFork ? options.forkedFrom : void 0,
|
|
191
197
|
forkOffset: isFork ? forkOffset : void 0
|
|
192
198
|
};
|
|
199
|
+
if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
|
|
200
|
+
const parts = stream.currentOffset.split(`_`).map(Number);
|
|
201
|
+
const readSeq = parts[0];
|
|
202
|
+
const byteOffset = parts[1];
|
|
203
|
+
const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5;
|
|
204
|
+
const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
205
|
+
stream.messages.push({
|
|
206
|
+
data: forkSubOffsetPrefix,
|
|
207
|
+
offset: newOffset,
|
|
208
|
+
timestamp: Date.now()
|
|
209
|
+
});
|
|
210
|
+
stream.currentOffset = newOffset;
|
|
211
|
+
stream.forkSubOffset = options.forkSubOffset;
|
|
212
|
+
}
|
|
193
213
|
if (options.initialData && options.initialData.length > 0) try {
|
|
194
214
|
this.appendToStream(stream, options.initialData, true);
|
|
195
215
|
} catch (err) {
|
|
@@ -713,6 +733,36 @@ var StreamStore = class {
|
|
|
713
733
|
list() {
|
|
714
734
|
return Array.from(this.streams.keys());
|
|
715
735
|
}
|
|
736
|
+
/**
|
|
737
|
+
* Resolve a sub-offset against a source stream and return the prefix bytes
|
|
738
|
+
* to materialize as the fork's first own message. Reads from the source
|
|
739
|
+
* (across its fork chain if any) starting at forkOffset; the first message
|
|
740
|
+
* returned is the one that starts at forkOffset. Throws if the sub-offset
|
|
741
|
+
* cannot be satisfied (no message past forkOffset, or overshoots its
|
|
742
|
+
* content extent).
|
|
743
|
+
*/
|
|
744
|
+
resolveForkSubOffset(sourceStream, forkOffset, subOffset, isJSON) {
|
|
745
|
+
let sourceMessages;
|
|
746
|
+
if (sourceStream.forkedFrom) sourceMessages = [...this.readForkedMessages(sourceStream.forkedFrom, forkOffset, sourceStream.forkOffset), ...this.readOwnMessages(sourceStream, forkOffset)];
|
|
747
|
+
else sourceMessages = this.readOwnMessages(sourceStream, forkOffset);
|
|
748
|
+
if (sourceMessages.length === 0) throw new Error(`Invalid fork sub-offset: no data past forkOffset`);
|
|
749
|
+
const first = sourceMessages[0];
|
|
750
|
+
if (isJSON) {
|
|
751
|
+
const text = new TextDecoder().decode(first.data);
|
|
752
|
+
const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text;
|
|
753
|
+
let values;
|
|
754
|
+
try {
|
|
755
|
+
values = JSON.parse(`[${trimmed}]`);
|
|
756
|
+
} catch {
|
|
757
|
+
throw new Error(`Invalid fork sub-offset: source JSON is unparseable`);
|
|
758
|
+
}
|
|
759
|
+
if (subOffset > values.length) throw new Error(`Invalid fork sub-offset: overshoots source message count`);
|
|
760
|
+
const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v));
|
|
761
|
+
return new TextEncoder().encode(prefix.join(`,`) + `,`);
|
|
762
|
+
}
|
|
763
|
+
if (subOffset > first.data.length) throw new Error(`Invalid fork sub-offset: overshoots source message length`);
|
|
764
|
+
return first.data.slice(0, subOffset);
|
|
765
|
+
}
|
|
716
766
|
appendToStream(stream, data, isInitialCreate = false) {
|
|
717
767
|
let processedData = data;
|
|
718
768
|
if (normalizeContentType(stream.contentType) === `application/json`) {
|
|
@@ -1294,13 +1344,17 @@ var FileBackedStreamStore = class {
|
|
|
1294
1344
|
const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
|
|
1295
1345
|
const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
|
|
1296
1346
|
const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
|
|
1297
|
-
|
|
1347
|
+
const requestedSub = options.forkSubOffset ?? 0;
|
|
1348
|
+
const existingSub = existingRaw.forkSubOffset ?? 0;
|
|
1349
|
+
const forkSubOffsetMatches = requestedSub === existingSub;
|
|
1350
|
+
if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches && forkSubOffsetMatches) return this.streamMetaToStream(existingRaw);
|
|
1298
1351
|
else throw new Error(`Stream already exists with different configuration: ${streamPath}`);
|
|
1299
1352
|
}
|
|
1300
1353
|
const isFork = !!options.forkedFrom;
|
|
1301
1354
|
let forkOffset = `0000000000000000_0000000000000000`;
|
|
1302
1355
|
let sourceContentType;
|
|
1303
1356
|
let sourceMeta;
|
|
1357
|
+
let forkSubOffsetPrefix;
|
|
1304
1358
|
if (isFork) {
|
|
1305
1359
|
const sourceKey = `stream:${options.forkedFrom}`;
|
|
1306
1360
|
sourceMeta = this.db.get(sourceKey);
|
|
@@ -1308,10 +1362,12 @@ var FileBackedStreamStore = class {
|
|
|
1308
1362
|
if (sourceMeta.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
|
|
1309
1363
|
if (this.isExpired(sourceMeta)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
|
|
1310
1364
|
sourceContentType = sourceMeta.contentType;
|
|
1365
|
+
if (options.contentType && options.contentType.trim() !== `` && normalizeContentType(options.contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
|
|
1311
1366
|
if (options.forkOffset) forkOffset = options.forkOffset;
|
|
1312
1367
|
else forkOffset = sourceMeta.currentOffset;
|
|
1313
1368
|
const zeroOffset = `0000000000000000_0000000000000000`;
|
|
1314
1369
|
if (forkOffset < zeroOffset || sourceMeta.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
|
|
1370
|
+
if (options.forkSubOffset && options.forkSubOffset > 0) forkSubOffsetPrefix = this.resolveForkSubOffset(options.forkedFrom, forkOffset, options.forkSubOffset, normalizeContentType(sourceContentType) === `application/json`);
|
|
1315
1371
|
const freshSource = this.db.get(sourceKey);
|
|
1316
1372
|
const updatedSource = {
|
|
1317
1373
|
...freshSource,
|
|
@@ -1322,7 +1378,7 @@ var FileBackedStreamStore = class {
|
|
|
1322
1378
|
let contentType = options.contentType;
|
|
1323
1379
|
if (!contentType || contentType.trim() === ``) {
|
|
1324
1380
|
if (isFork) contentType = sourceContentType;
|
|
1325
|
-
}
|
|
1381
|
+
}
|
|
1326
1382
|
let effectiveExpiresAt = options.expiresAt;
|
|
1327
1383
|
let effectiveTtlSeconds = options.ttlSeconds;
|
|
1328
1384
|
if (isFork) {
|
|
@@ -1347,11 +1403,39 @@ var FileBackedStreamStore = class {
|
|
|
1347
1403
|
closed: false,
|
|
1348
1404
|
forkedFrom: isFork ? options.forkedFrom : void 0,
|
|
1349
1405
|
forkOffset: isFork ? forkOffset : void 0,
|
|
1406
|
+
forkSubOffset: void 0,
|
|
1350
1407
|
refCount: 0
|
|
1351
1408
|
};
|
|
1352
1409
|
const tAfterMeta = performance.now();
|
|
1353
1410
|
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
|
|
1354
1411
|
try {
|
|
1412
|
+
if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
|
|
1413
|
+
const lengthBuf = Buffer.allocUnsafe(4);
|
|
1414
|
+
lengthBuf.writeUInt32BE(forkSubOffsetPrefix.length, 0);
|
|
1415
|
+
const frameBuf = Buffer.concat([
|
|
1416
|
+
lengthBuf,
|
|
1417
|
+
Buffer.from(forkSubOffsetPrefix),
|
|
1418
|
+
Buffer.from(`\n`)
|
|
1419
|
+
]);
|
|
1420
|
+
const fd = fs.openSync(segmentPath, `wx`);
|
|
1421
|
+
try {
|
|
1422
|
+
let written = 0;
|
|
1423
|
+
while (written < frameBuf.length) {
|
|
1424
|
+
const bytesWritten = fs.writeSync(fd, frameBuf, written, frameBuf.length - written, written);
|
|
1425
|
+
if (bytesWritten === 0) throw new Error(`failed to write sub-offset prefix frame`);
|
|
1426
|
+
written += bytesWritten;
|
|
1427
|
+
}
|
|
1428
|
+
fs.fsyncSync(fd);
|
|
1429
|
+
} finally {
|
|
1430
|
+
fs.closeSync(fd);
|
|
1431
|
+
}
|
|
1432
|
+
const parts = streamMeta.currentOffset.split(`_`).map(Number);
|
|
1433
|
+
const readSeq = parts[0];
|
|
1434
|
+
const byteOffset = parts[1];
|
|
1435
|
+
const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5;
|
|
1436
|
+
streamMeta.currentOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
|
|
1437
|
+
streamMeta.forkSubOffset = options.forkSubOffset;
|
|
1438
|
+
}
|
|
1355
1439
|
await this.db.put(key, streamMeta);
|
|
1356
1440
|
} catch (err) {
|
|
1357
1441
|
if (isFork && sourceMeta) {
|
|
@@ -1365,7 +1449,7 @@ var FileBackedStreamStore = class {
|
|
|
1365
1449
|
this.db.putSync(sourceKey, updatedSource);
|
|
1366
1450
|
}
|
|
1367
1451
|
}
|
|
1368
|
-
serverLog.error(`[FileBackedStreamStore] Error creating stream
|
|
1452
|
+
serverLog.error(`[FileBackedStreamStore] Error creating stream before metadata commit:`, err);
|
|
1369
1453
|
throw err;
|
|
1370
1454
|
}
|
|
1371
1455
|
const tAfterLmdb = performance.now();
|
|
@@ -1750,6 +1834,35 @@ var FileBackedStreamStore = class {
|
|
|
1750
1834
|
messages.push(...ownMessages);
|
|
1751
1835
|
return messages;
|
|
1752
1836
|
}
|
|
1837
|
+
/**
|
|
1838
|
+
* Resolve a fork sub-offset against the source: read the message that
|
|
1839
|
+
* starts at forkOffset and return prefix bytes to materialize as the
|
|
1840
|
+
* fork's first own message. For JSON, parses comma-joined values.
|
|
1841
|
+
*/
|
|
1842
|
+
resolveForkSubOffset(sourcePath, forkOffset, subOffset, isJSON) {
|
|
1843
|
+
const forkByte = Number(forkOffset.split(`_`)[1] ?? 0);
|
|
1844
|
+
const sourceMeta = this.db.get(`stream:${sourcePath}`);
|
|
1845
|
+
if (!sourceMeta) throw new Error(`Source stream not found: ${sourcePath}`);
|
|
1846
|
+
const currentByte = Number(sourceMeta.currentOffset.split(`_`)[1] ?? 0);
|
|
1847
|
+
const messages = this.readForkedMessages(sourcePath, forkByte, currentByte);
|
|
1848
|
+
if (messages.length === 0) throw new Error(`Invalid fork sub-offset: no data past forkOffset`);
|
|
1849
|
+
const first = messages[0];
|
|
1850
|
+
if (isJSON) {
|
|
1851
|
+
const text = new TextDecoder().decode(first.data);
|
|
1852
|
+
const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text;
|
|
1853
|
+
let values;
|
|
1854
|
+
try {
|
|
1855
|
+
values = JSON.parse(`[${trimmed}]`);
|
|
1856
|
+
} catch {
|
|
1857
|
+
throw new Error(`Invalid fork sub-offset: source JSON is unparseable`);
|
|
1858
|
+
}
|
|
1859
|
+
if (subOffset > values.length) throw new Error(`Invalid fork sub-offset: overshoots source message count`);
|
|
1860
|
+
const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v));
|
|
1861
|
+
return new TextEncoder().encode(prefix.join(`,`) + `,`);
|
|
1862
|
+
}
|
|
1863
|
+
if (subOffset > first.data.length) throw new Error(`Invalid fork sub-offset: overshoots source message length`);
|
|
1864
|
+
return first.data.slice(0, subOffset);
|
|
1865
|
+
}
|
|
1753
1866
|
read(streamPath, offset) {
|
|
1754
1867
|
const streamMeta = this.getMetaIfNotExpired(streamPath);
|
|
1755
1868
|
if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
|
|
@@ -3006,6 +3119,7 @@ const STREAM_SSE_DATA_ENCODING_HEADER = `Stream-SSE-Data-Encoding`;
|
|
|
3006
3119
|
const SSE_UP_TO_DATE_FIELD = `upToDate`;
|
|
3007
3120
|
const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`;
|
|
3008
3121
|
const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`;
|
|
3122
|
+
const STREAM_FORK_SUB_OFFSET_HEADER = `Stream-Fork-Sub-Offset`;
|
|
3009
3123
|
/**
|
|
3010
3124
|
* Encode data for SSE format.
|
|
3011
3125
|
* Per SSE spec, each line in the payload needs its own "data:" prefix.
|
|
@@ -3231,7 +3345,7 @@ var DurableStreamTestServer = class {
|
|
|
3231
3345
|
const method = req.method?.toUpperCase();
|
|
3232
3346
|
res.setHeader(`access-control-allow-origin`, `*`);
|
|
3233
3347
|
res.setHeader(`access-control-allow-methods`, `GET, POST, PUT, DELETE, HEAD, OPTIONS`);
|
|
3234
|
-
res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset`);
|
|
3348
|
+
res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset, Stream-Fork-Sub-Offset`);
|
|
3235
3349
|
res.setHeader(`access-control-expose-headers`, `Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Date, Stream-Closed, Producer-Epoch, Producer-Seq, Producer-Expected-Seq, Producer-Received-Seq, etag, content-type, content-encoding, vary`);
|
|
3236
3350
|
res.setHeader(`x-content-type-options`, `nosniff`);
|
|
3237
3351
|
res.setHeader(`cross-origin-resource-policy`, `cross-origin`);
|
|
@@ -3321,6 +3435,9 @@ var DurableStreamTestServer = class {
|
|
|
3321
3435
|
let contentType = req.headers[`content-type`];
|
|
3322
3436
|
const forkedFromHeader = req.headers[STREAM_FORKED_FROM_HEADER.toLowerCase()];
|
|
3323
3437
|
const forkOffsetHeader = req.headers[STREAM_FORK_OFFSET_HEADER.toLowerCase()];
|
|
3438
|
+
const forkSubOffsetHeaderRaw = req.headers[STREAM_FORK_SUB_OFFSET_HEADER.toLowerCase()];
|
|
3439
|
+
const forkSubOffsetHeaderPresent = forkSubOffsetHeaderRaw !== void 0;
|
|
3440
|
+
const forkSubOffsetHeader = Array.isArray(forkSubOffsetHeaderRaw) ? forkSubOffsetHeaderRaw[0] : forkSubOffsetHeaderRaw;
|
|
3324
3441
|
if (!contentType || contentType.trim() === `` || !/^[\w-]+\/[\w-]+/.test(contentType)) contentType = forkedFromHeader ? void 0 : `application/octet-stream`;
|
|
3325
3442
|
const ttlHeader = req.headers[STREAM_TTL_HEADER.toLowerCase()];
|
|
3326
3443
|
const expiresAtHeader = req.headers[STREAM_EXPIRES_AT_HEADER.toLowerCase()];
|
|
@@ -3362,6 +3479,21 @@ var DurableStreamTestServer = class {
|
|
|
3362
3479
|
return;
|
|
3363
3480
|
}
|
|
3364
3481
|
}
|
|
3482
|
+
let forkSubOffset;
|
|
3483
|
+
if (forkSubOffsetHeaderPresent) {
|
|
3484
|
+
if (!forkedFromHeader) {
|
|
3485
|
+
res.writeHead(400, { "content-type": `text/plain` });
|
|
3486
|
+
res.end(`Stream-Fork-Sub-Offset requires Stream-Forked-From`);
|
|
3487
|
+
return;
|
|
3488
|
+
}
|
|
3489
|
+
const subOffsetPattern = /^(0|[1-9]\d*)$/;
|
|
3490
|
+
if (forkSubOffsetHeader === void 0 || !subOffsetPattern.test(forkSubOffsetHeader)) {
|
|
3491
|
+
res.writeHead(400, { "content-type": `text/plain` });
|
|
3492
|
+
res.end(`Invalid Stream-Fork-Sub-Offset format`);
|
|
3493
|
+
return;
|
|
3494
|
+
}
|
|
3495
|
+
forkSubOffset = parseInt(forkSubOffsetHeader, 10);
|
|
3496
|
+
}
|
|
3365
3497
|
const body = await this.readBody(req);
|
|
3366
3498
|
const isNew = !this.store.has(path$1);
|
|
3367
3499
|
try {
|
|
@@ -3372,7 +3504,8 @@ var DurableStreamTestServer = class {
|
|
|
3372
3504
|
initialData: body.length > 0 ? body : void 0,
|
|
3373
3505
|
closed: createClosed,
|
|
3374
3506
|
forkedFrom: forkedFromHeader,
|
|
3375
|
-
forkOffset: forkOffsetHeader
|
|
3507
|
+
forkOffset: forkOffsetHeader,
|
|
3508
|
+
forkSubOffset
|
|
3376
3509
|
}));
|
|
3377
3510
|
} catch (err) {
|
|
3378
3511
|
if (err instanceof Error) {
|
|
@@ -3381,6 +3514,11 @@ var DurableStreamTestServer = class {
|
|
|
3381
3514
|
res.end(`Source stream not found`);
|
|
3382
3515
|
return;
|
|
3383
3516
|
}
|
|
3517
|
+
if (err.message.includes(`Invalid fork sub-offset`)) {
|
|
3518
|
+
res.writeHead(400, { "content-type": `text/plain` });
|
|
3519
|
+
res.end(`Invalid fork sub-offset`);
|
|
3520
|
+
return;
|
|
3521
|
+
}
|
|
3384
3522
|
if (err.message.includes(`Invalid fork offset`)) {
|
|
3385
3523
|
res.writeHead(400, { "content-type": `text/plain` });
|
|
3386
3524
|
res.end(`Fork offset beyond source stream length`);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@durable-streams/server",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.6",
|
|
4
4
|
"description": "Node.js reference server implementation for Durable Streams",
|
|
5
5
|
"author": "Durable Stream contributors",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -39,15 +39,15 @@
|
|
|
39
39
|
"dependencies": {
|
|
40
40
|
"@neophi/sieve-cache": "^1.0.0",
|
|
41
41
|
"lmdb": "^3.3.0",
|
|
42
|
-
"@durable-streams/client": "0.2.
|
|
43
|
-
"@durable-streams/state": "0.
|
|
42
|
+
"@durable-streams/client": "0.2.6",
|
|
43
|
+
"@durable-streams/state": "0.3.0"
|
|
44
44
|
},
|
|
45
45
|
"devDependencies": {
|
|
46
46
|
"@types/node": "^22.0.0",
|
|
47
47
|
"tsdown": "^0.9.0",
|
|
48
48
|
"typescript": "^5.0.0",
|
|
49
49
|
"vitest": "^4.0.0",
|
|
50
|
-
"@durable-streams/server-conformance-tests": "0.3.
|
|
50
|
+
"@durable-streams/server-conformance-tests": "0.3.5"
|
|
51
51
|
},
|
|
52
52
|
"files": [
|
|
53
53
|
"dist",
|
package/src/file-store.ts
CHANGED
|
@@ -87,6 +87,12 @@ interface StreamMetadata {
|
|
|
87
87
|
* Divergence offset from the source stream.
|
|
88
88
|
*/
|
|
89
89
|
forkOffset?: string
|
|
90
|
+
/**
|
|
91
|
+
* User-supplied sub-offset value refining `forkOffset` (Section 4.2 of
|
|
92
|
+
* PROTOCOL.md). Stored verbatim for idempotent re-creation matching:
|
|
93
|
+
* bytes for non-JSON forks, flattened message count for JSON forks.
|
|
94
|
+
*/
|
|
95
|
+
forkSubOffset?: number
|
|
90
96
|
/**
|
|
91
97
|
* Number of forks referencing this stream.
|
|
92
98
|
* Defaults to 0. Optional for backward-compatible deserialization from LMDB.
|
|
@@ -726,6 +732,7 @@ export class FileBackedStreamStore {
|
|
|
726
732
|
closed?: boolean
|
|
727
733
|
forkedFrom?: string
|
|
728
734
|
forkOffset?: string
|
|
735
|
+
forkSubOffset?: number
|
|
729
736
|
} = {}
|
|
730
737
|
): Promise<Stream> {
|
|
731
738
|
// Use getMetaIfNotExpired to treat expired streams as non-existent
|
|
@@ -762,6 +769,12 @@ export class FileBackedStreamStore {
|
|
|
762
769
|
const forkOffsetMatches =
|
|
763
770
|
options.forkOffset === undefined ||
|
|
764
771
|
options.forkOffset === existingRaw.forkOffset
|
|
772
|
+
// Sub-offset: undefined and 0 are equivalent. Compare the raw
|
|
773
|
+
// user-supplied integer (count for JSON, bytes for binary) so the
|
|
774
|
+
// comparison is independent of how it was resolved internally.
|
|
775
|
+
const requestedSub = options.forkSubOffset ?? 0
|
|
776
|
+
const existingSub = existingRaw.forkSubOffset ?? 0
|
|
777
|
+
const forkSubOffsetMatches = requestedSub === existingSub
|
|
765
778
|
|
|
766
779
|
if (
|
|
767
780
|
contentTypeMatches &&
|
|
@@ -769,7 +782,8 @@ export class FileBackedStreamStore {
|
|
|
769
782
|
expiresMatches &&
|
|
770
783
|
closedMatches &&
|
|
771
784
|
forkedFromMatches &&
|
|
772
|
-
forkOffsetMatches
|
|
785
|
+
forkOffsetMatches &&
|
|
786
|
+
forkSubOffsetMatches
|
|
773
787
|
) {
|
|
774
788
|
// Idempotent success - return existing stream
|
|
775
789
|
return this.streamMetaToStream(existingRaw)
|
|
@@ -787,6 +801,7 @@ export class FileBackedStreamStore {
|
|
|
787
801
|
let forkOffset = `0000000000000000_0000000000000000`
|
|
788
802
|
let sourceContentType: string | undefined
|
|
789
803
|
let sourceMeta: StreamMetadata | undefined
|
|
804
|
+
let forkSubOffsetPrefix: Uint8Array | undefined
|
|
790
805
|
|
|
791
806
|
if (isFork) {
|
|
792
807
|
const sourceKey = `stream:${options.forkedFrom!}`
|
|
@@ -803,6 +818,18 @@ export class FileBackedStreamStore {
|
|
|
803
818
|
|
|
804
819
|
sourceContentType = sourceMeta.contentType
|
|
805
820
|
|
|
821
|
+
// Reject a content-type mismatch up front, before taking a reference on
|
|
822
|
+
// the source. Doing this after the refCount increment below would leak a
|
|
823
|
+
// reference on the failed fork and pin the source in a soft-deleted state.
|
|
824
|
+
if (
|
|
825
|
+
options.contentType &&
|
|
826
|
+
options.contentType.trim() !== `` &&
|
|
827
|
+
normalizeContentType(options.contentType) !==
|
|
828
|
+
normalizeContentType(sourceContentType)
|
|
829
|
+
) {
|
|
830
|
+
throw new Error(`Content type mismatch with source stream`)
|
|
831
|
+
}
|
|
832
|
+
|
|
806
833
|
// Resolve fork offset: use provided or source's currentOffset
|
|
807
834
|
if (options.forkOffset) {
|
|
808
835
|
forkOffset = options.forkOffset
|
|
@@ -816,6 +843,17 @@ export class FileBackedStreamStore {
|
|
|
816
843
|
throw new Error(`Invalid fork offset: ${forkOffset}`)
|
|
817
844
|
}
|
|
818
845
|
|
|
846
|
+
// Resolve sub-offset against the source. Returns the prefix bytes to
|
|
847
|
+
// materialize as the fork's first own message.
|
|
848
|
+
if (options.forkSubOffset && options.forkSubOffset > 0) {
|
|
849
|
+
forkSubOffsetPrefix = this.resolveForkSubOffset(
|
|
850
|
+
options.forkedFrom!,
|
|
851
|
+
forkOffset,
|
|
852
|
+
options.forkSubOffset,
|
|
853
|
+
normalizeContentType(sourceContentType) === `application/json`
|
|
854
|
+
)
|
|
855
|
+
}
|
|
856
|
+
|
|
819
857
|
// Atomically increment source refcount in LMDB
|
|
820
858
|
const freshSource = this.db.get(sourceKey) as StreamMetadata
|
|
821
859
|
const updatedSource: StreamMetadata = {
|
|
@@ -825,18 +863,14 @@ export class FileBackedStreamStore {
|
|
|
825
863
|
this.db.putSync(sourceKey, updatedSource)
|
|
826
864
|
}
|
|
827
865
|
|
|
828
|
-
// Determine content type: use options, or inherit from source if fork
|
|
866
|
+
// Determine content type: use options, or inherit from source if fork. A
|
|
867
|
+
// fork content-type mismatch is already rejected above, before the source
|
|
868
|
+
// refCount is taken.
|
|
829
869
|
let contentType = options.contentType
|
|
830
870
|
if (!contentType || contentType.trim() === ``) {
|
|
831
871
|
if (isFork) {
|
|
832
872
|
contentType = sourceContentType
|
|
833
873
|
}
|
|
834
|
-
} else if (
|
|
835
|
-
isFork &&
|
|
836
|
-
normalizeContentType(contentType) !==
|
|
837
|
-
normalizeContentType(sourceContentType)
|
|
838
|
-
) {
|
|
839
|
-
throw new Error(`Content type mismatch with source stream`)
|
|
840
874
|
}
|
|
841
875
|
|
|
842
876
|
// Compute effective expiry for forks
|
|
@@ -871,6 +905,7 @@ export class FileBackedStreamStore {
|
|
|
871
905
|
closed: false, // Set to false initially, will be updated after initial append if needed
|
|
872
906
|
forkedFrom: isFork ? options.forkedFrom : undefined,
|
|
873
907
|
forkOffset: isFork ? forkOffset : undefined,
|
|
908
|
+
forkSubOffset: undefined,
|
|
874
909
|
refCount: 0,
|
|
875
910
|
}
|
|
876
911
|
|
|
@@ -878,6 +913,52 @@ export class FileBackedStreamStore {
|
|
|
878
913
|
|
|
879
914
|
const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName)
|
|
880
915
|
try {
|
|
916
|
+
// Materialize the sub-offset prefix as the first framed message
|
|
917
|
+
// in the new segment before persisting metadata or opening the
|
|
918
|
+
// write stream. The prefix must be fsynced before the LMDB commit
|
|
919
|
+
// so metadata never acknowledges bytes that are not durable.
|
|
920
|
+
if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
|
|
921
|
+
const lengthBuf = Buffer.allocUnsafe(4)
|
|
922
|
+
lengthBuf.writeUInt32BE(forkSubOffsetPrefix.length, 0)
|
|
923
|
+
const frameBuf = Buffer.concat([
|
|
924
|
+
lengthBuf,
|
|
925
|
+
Buffer.from(forkSubOffsetPrefix),
|
|
926
|
+
Buffer.from(`\n`),
|
|
927
|
+
])
|
|
928
|
+
|
|
929
|
+
const fd = fs.openSync(segmentPath, `wx`)
|
|
930
|
+
try {
|
|
931
|
+
let written = 0
|
|
932
|
+
while (written < frameBuf.length) {
|
|
933
|
+
const bytesWritten = fs.writeSync(
|
|
934
|
+
fd,
|
|
935
|
+
frameBuf,
|
|
936
|
+
written,
|
|
937
|
+
frameBuf.length - written,
|
|
938
|
+
written
|
|
939
|
+
)
|
|
940
|
+
if (bytesWritten === 0) {
|
|
941
|
+
throw new Error(`failed to write sub-offset prefix frame`)
|
|
942
|
+
}
|
|
943
|
+
written += bytesWritten
|
|
944
|
+
}
|
|
945
|
+
fs.fsyncSync(fd)
|
|
946
|
+
} finally {
|
|
947
|
+
fs.closeSync(fd)
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
const parts = streamMeta.currentOffset.split(`_`).map(Number)
|
|
951
|
+
const readSeq = parts[0]!
|
|
952
|
+
const byteOffset = parts[1]!
|
|
953
|
+
// Match append()'s frame-inclusive offset advance (4-byte length
|
|
954
|
+
// prefix + payload + 1-byte newline) so reads with a capByte don't
|
|
955
|
+
// truncate the materialized prefix when later chained-fork resolves.
|
|
956
|
+
const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5
|
|
957
|
+
streamMeta.currentOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`
|
|
958
|
+
// Persist the user-supplied sub-offset verbatim for idempotent
|
|
959
|
+
// re-creation matching, not the encoded byte length.
|
|
960
|
+
streamMeta.forkSubOffset = options.forkSubOffset
|
|
961
|
+
}
|
|
881
962
|
await this.db.put(key, streamMeta)
|
|
882
963
|
} catch (err) {
|
|
883
964
|
// Rollback source refcount on failure
|
|
@@ -893,7 +974,7 @@ export class FileBackedStreamStore {
|
|
|
893
974
|
}
|
|
894
975
|
}
|
|
895
976
|
serverLog.error(
|
|
896
|
-
`[FileBackedStreamStore] Error creating stream
|
|
977
|
+
`[FileBackedStreamStore] Error creating stream before metadata commit:`,
|
|
897
978
|
err
|
|
898
979
|
)
|
|
899
980
|
throw err
|
|
@@ -1589,6 +1670,57 @@ export class FileBackedStreamStore {
|
|
|
1589
1670
|
return messages
|
|
1590
1671
|
}
|
|
1591
1672
|
|
|
1673
|
+
/**
|
|
1674
|
+
* Resolve a fork sub-offset against the source: read the message that
|
|
1675
|
+
* starts at forkOffset and return prefix bytes to materialize as the
|
|
1676
|
+
* fork's first own message. For JSON, parses comma-joined values.
|
|
1677
|
+
*/
|
|
1678
|
+
private resolveForkSubOffset(
|
|
1679
|
+
sourcePath: string,
|
|
1680
|
+
forkOffset: string,
|
|
1681
|
+
subOffset: number,
|
|
1682
|
+
isJSON: boolean
|
|
1683
|
+
): Uint8Array {
|
|
1684
|
+
const forkByte = Number(forkOffset.split(`_`)[1] ?? 0)
|
|
1685
|
+
// Read source past forkOffset (cap at source's currentOffset by passing
|
|
1686
|
+
// a large cap; we only care about the first message past forkOffset).
|
|
1687
|
+
const sourceMeta = this.db.get(`stream:${sourcePath}`) as
|
|
1688
|
+
| StreamMetadata
|
|
1689
|
+
| undefined
|
|
1690
|
+
if (!sourceMeta) {
|
|
1691
|
+
throw new Error(`Source stream not found: ${sourcePath}`)
|
|
1692
|
+
}
|
|
1693
|
+
const currentByte = Number(sourceMeta.currentOffset.split(`_`)[1] ?? 0)
|
|
1694
|
+
const messages = this.readForkedMessages(sourcePath, forkByte, currentByte)
|
|
1695
|
+
if (messages.length === 0) {
|
|
1696
|
+
throw new Error(`Invalid fork sub-offset: no data past forkOffset`)
|
|
1697
|
+
}
|
|
1698
|
+
const first = messages[0]!
|
|
1699
|
+
if (isJSON) {
|
|
1700
|
+
const text = new TextDecoder().decode(first.data)
|
|
1701
|
+
const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text
|
|
1702
|
+
let values: Array<unknown>
|
|
1703
|
+
try {
|
|
1704
|
+
values = JSON.parse(`[${trimmed}]`)
|
|
1705
|
+
} catch {
|
|
1706
|
+
throw new Error(`Invalid fork sub-offset: source JSON is unparseable`)
|
|
1707
|
+
}
|
|
1708
|
+
if (subOffset > values.length) {
|
|
1709
|
+
throw new Error(
|
|
1710
|
+
`Invalid fork sub-offset: overshoots source message count`
|
|
1711
|
+
)
|
|
1712
|
+
}
|
|
1713
|
+
const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v))
|
|
1714
|
+
return new TextEncoder().encode(prefix.join(`,`) + `,`)
|
|
1715
|
+
}
|
|
1716
|
+
if (subOffset > first.data.length) {
|
|
1717
|
+
throw new Error(
|
|
1718
|
+
`Invalid fork sub-offset: overshoots source message length`
|
|
1719
|
+
)
|
|
1720
|
+
}
|
|
1721
|
+
return first.data.slice(0, subOffset)
|
|
1722
|
+
}
|
|
1723
|
+
|
|
1592
1724
|
read(
|
|
1593
1725
|
streamPath: string,
|
|
1594
1726
|
offset?: string
|
package/src/server.ts
CHANGED
|
@@ -42,6 +42,7 @@ const SSE_UP_TO_DATE_FIELD = `upToDate`
|
|
|
42
42
|
// Fork headers (request headers only — not set on responses)
|
|
43
43
|
const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`
|
|
44
44
|
const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`
|
|
45
|
+
const STREAM_FORK_SUB_OFFSET_HEADER = `Stream-Fork-Sub-Offset`
|
|
45
46
|
|
|
46
47
|
/**
|
|
47
48
|
* Encode data for SSE format.
|
|
@@ -449,7 +450,7 @@ export class DurableStreamTestServer {
|
|
|
449
450
|
)
|
|
450
451
|
res.setHeader(
|
|
451
452
|
`access-control-allow-headers`,
|
|
452
|
-
`content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset`
|
|
453
|
+
`content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset, Stream-Fork-Sub-Offset`
|
|
453
454
|
)
|
|
454
455
|
res.setHeader(
|
|
455
456
|
`access-control-expose-headers`,
|
|
@@ -598,6 +599,13 @@ export class DurableStreamTestServer {
|
|
|
598
599
|
const forkOffsetHeader = req.headers[
|
|
599
600
|
STREAM_FORK_OFFSET_HEADER.toLowerCase()
|
|
600
601
|
] as string | undefined
|
|
602
|
+
const forkSubOffsetHeaderRaw =
|
|
603
|
+
req.headers[STREAM_FORK_SUB_OFFSET_HEADER.toLowerCase()]
|
|
604
|
+
// Distinguish "header absent" from "header present but empty"
|
|
605
|
+
const forkSubOffsetHeaderPresent = forkSubOffsetHeaderRaw !== undefined
|
|
606
|
+
const forkSubOffsetHeader = Array.isArray(forkSubOffsetHeaderRaw)
|
|
607
|
+
? forkSubOffsetHeaderRaw[0]
|
|
608
|
+
: forkSubOffsetHeaderRaw
|
|
601
609
|
|
|
602
610
|
// Sanitize content-type: if empty or invalid, use default — but only
|
|
603
611
|
// for non-fork creates. For forks, an omitted Content-Type means "inherit
|
|
@@ -667,6 +675,26 @@ export class DurableStreamTestServer {
|
|
|
667
675
|
}
|
|
668
676
|
}
|
|
669
677
|
|
|
678
|
+
// Validate sub-offset if header was present (including empty value)
|
|
679
|
+
let forkSubOffset: number | undefined
|
|
680
|
+
if (forkSubOffsetHeaderPresent) {
|
|
681
|
+
if (!forkedFromHeader) {
|
|
682
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
683
|
+
res.end(`Stream-Fork-Sub-Offset requires Stream-Forked-From`)
|
|
684
|
+
return
|
|
685
|
+
}
|
|
686
|
+
const subOffsetPattern = /^(0|[1-9]\d*)$/
|
|
687
|
+
if (
|
|
688
|
+
forkSubOffsetHeader === undefined ||
|
|
689
|
+
!subOffsetPattern.test(forkSubOffsetHeader)
|
|
690
|
+
) {
|
|
691
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
692
|
+
res.end(`Invalid Stream-Fork-Sub-Offset format`)
|
|
693
|
+
return
|
|
694
|
+
}
|
|
695
|
+
forkSubOffset = parseInt(forkSubOffsetHeader, 10)
|
|
696
|
+
}
|
|
697
|
+
|
|
670
698
|
// Read body if present
|
|
671
699
|
const body = await this.readBody(req)
|
|
672
700
|
|
|
@@ -683,6 +711,7 @@ export class DurableStreamTestServer {
|
|
|
683
711
|
closed: createClosed,
|
|
684
712
|
forkedFrom: forkedFromHeader,
|
|
685
713
|
forkOffset: forkOffsetHeader,
|
|
714
|
+
forkSubOffset,
|
|
686
715
|
})
|
|
687
716
|
)
|
|
688
717
|
} catch (err) {
|
|
@@ -692,6 +721,11 @@ export class DurableStreamTestServer {
|
|
|
692
721
|
res.end(`Source stream not found`)
|
|
693
722
|
return
|
|
694
723
|
}
|
|
724
|
+
if (err.message.includes(`Invalid fork sub-offset`)) {
|
|
725
|
+
res.writeHead(400, { "content-type": `text/plain` })
|
|
726
|
+
res.end(`Invalid fork sub-offset`)
|
|
727
|
+
return
|
|
728
|
+
}
|
|
695
729
|
if (err.message.includes(`Invalid fork offset`)) {
|
|
696
730
|
res.writeHead(400, { "content-type": `text/plain` })
|
|
697
731
|
res.end(`Fork offset beyond source stream length`)
|
package/src/store.ts
CHANGED
|
@@ -247,6 +247,7 @@ export class StreamStore {
|
|
|
247
247
|
closed?: boolean
|
|
248
248
|
forkedFrom?: string
|
|
249
249
|
forkOffset?: string
|
|
250
|
+
forkSubOffset?: number
|
|
250
251
|
} = {}
|
|
251
252
|
): Stream {
|
|
252
253
|
// Check if stream already exists
|
|
@@ -280,6 +281,12 @@ export class StreamStore {
|
|
|
280
281
|
const forkOffsetMatches =
|
|
281
282
|
options.forkOffset === undefined ||
|
|
282
283
|
options.forkOffset === existingRaw.forkOffset
|
|
284
|
+
// Sub-offset: undefined and 0 are equivalent. Compare the raw
|
|
285
|
+
// user-supplied integer (count for JSON, bytes for binary) so the
|
|
286
|
+
// comparison is independent of how it was resolved internally.
|
|
287
|
+
const requestedSub = options.forkSubOffset ?? 0
|
|
288
|
+
const existingSub = existingRaw.forkSubOffset ?? 0
|
|
289
|
+
const forkSubOffsetMatches = requestedSub === existingSub
|
|
283
290
|
|
|
284
291
|
if (
|
|
285
292
|
contentTypeMatches &&
|
|
@@ -287,7 +294,8 @@ export class StreamStore {
|
|
|
287
294
|
expiresMatches &&
|
|
288
295
|
closedMatches &&
|
|
289
296
|
forkedFromMatches &&
|
|
290
|
-
forkOffsetMatches
|
|
297
|
+
forkOffsetMatches &&
|
|
298
|
+
forkSubOffsetMatches
|
|
291
299
|
) {
|
|
292
300
|
// Idempotent success - return existing stream
|
|
293
301
|
return existingRaw
|
|
@@ -305,6 +313,7 @@ export class StreamStore {
|
|
|
305
313
|
let forkOffset = `0000000000000000_0000000000000000`
|
|
306
314
|
let sourceContentType: string | undefined
|
|
307
315
|
let sourceStream: Stream | undefined
|
|
316
|
+
let forkSubOffsetPrefix: Uint8Array | undefined
|
|
308
317
|
|
|
309
318
|
if (isFork) {
|
|
310
319
|
sourceStream = this.streams.get(options.forkedFrom!)
|
|
@@ -320,6 +329,18 @@ export class StreamStore {
|
|
|
320
329
|
|
|
321
330
|
sourceContentType = sourceStream.contentType
|
|
322
331
|
|
|
332
|
+
// Reject a content-type mismatch up front, before taking a reference on
|
|
333
|
+
// the source. Doing this after the refCount increment below would leak a
|
|
334
|
+
// reference on the failed fork and pin the source in a soft-deleted state.
|
|
335
|
+
if (
|
|
336
|
+
options.contentType &&
|
|
337
|
+
options.contentType.trim() !== `` &&
|
|
338
|
+
normalizeContentType(options.contentType) !==
|
|
339
|
+
normalizeContentType(sourceContentType)
|
|
340
|
+
) {
|
|
341
|
+
throw new Error(`Content type mismatch with source stream`)
|
|
342
|
+
}
|
|
343
|
+
|
|
323
344
|
// Resolve fork offset: use provided or source's currentOffset
|
|
324
345
|
if (options.forkOffset) {
|
|
325
346
|
forkOffset = options.forkOffset
|
|
@@ -333,22 +354,30 @@ export class StreamStore {
|
|
|
333
354
|
throw new Error(`Invalid fork offset: ${forkOffset}`)
|
|
334
355
|
}
|
|
335
356
|
|
|
357
|
+
// Resolve sub-offset against the source. Both binary and JSON return
|
|
358
|
+
// a synthetic prefix to materialize as the fork's first own message,
|
|
359
|
+
// because in this store one POST = one message regardless of mode.
|
|
360
|
+
if (options.forkSubOffset && options.forkSubOffset > 0) {
|
|
361
|
+
forkSubOffsetPrefix = this.resolveForkSubOffset(
|
|
362
|
+
sourceStream,
|
|
363
|
+
forkOffset,
|
|
364
|
+
options.forkSubOffset,
|
|
365
|
+
normalizeContentType(sourceContentType) === `application/json`
|
|
366
|
+
)
|
|
367
|
+
}
|
|
368
|
+
|
|
336
369
|
// Increment source refcount
|
|
337
370
|
sourceStream.refCount++
|
|
338
371
|
}
|
|
339
372
|
|
|
340
|
-
// Determine content type: use options, or inherit from source if fork
|
|
373
|
+
// Determine content type: use options, or inherit from source if fork. A
|
|
374
|
+
// fork content-type mismatch is already rejected above, before the source
|
|
375
|
+
// refCount is taken.
|
|
341
376
|
let contentType = options.contentType
|
|
342
377
|
if (!contentType || contentType.trim() === ``) {
|
|
343
378
|
if (isFork) {
|
|
344
379
|
contentType = sourceContentType
|
|
345
380
|
}
|
|
346
|
-
} else if (
|
|
347
|
-
isFork &&
|
|
348
|
-
normalizeContentType(contentType) !==
|
|
349
|
-
normalizeContentType(sourceContentType)
|
|
350
|
-
) {
|
|
351
|
-
throw new Error(`Content type mismatch with source stream`)
|
|
352
381
|
}
|
|
353
382
|
|
|
354
383
|
// Compute effective expiry for forks
|
|
@@ -375,6 +404,27 @@ export class StreamStore {
|
|
|
375
404
|
forkOffset: isFork ? forkOffset : undefined,
|
|
376
405
|
}
|
|
377
406
|
|
|
407
|
+
// Materialize sub-offset prefix as the fork's first own message.
|
|
408
|
+
if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
|
|
409
|
+
const parts = stream.currentOffset.split(`_`).map(Number)
|
|
410
|
+
const readSeq = parts[0]!
|
|
411
|
+
const byteOffset = parts[1]!
|
|
412
|
+
// Match append()'s frame-inclusive offset advance (4-byte length
|
|
413
|
+
// prefix + payload + 1-byte newline) so reads with a capByte don't
|
|
414
|
+
// truncate the materialized prefix when later chained-fork resolves.
|
|
415
|
+
const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5
|
|
416
|
+
const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`
|
|
417
|
+
stream.messages.push({
|
|
418
|
+
data: forkSubOffsetPrefix,
|
|
419
|
+
offset: newOffset,
|
|
420
|
+
timestamp: Date.now(),
|
|
421
|
+
})
|
|
422
|
+
stream.currentOffset = newOffset
|
|
423
|
+
// Persist the user-supplied sub-offset verbatim for idempotent
|
|
424
|
+
// re-creation matching, not the encoded byte length.
|
|
425
|
+
stream.forkSubOffset = options.forkSubOffset
|
|
426
|
+
}
|
|
427
|
+
|
|
378
428
|
// If initial data is provided, append it
|
|
379
429
|
if (options.initialData && options.initialData.length > 0) {
|
|
380
430
|
try {
|
|
@@ -1236,6 +1286,67 @@ export class StreamStore {
|
|
|
1236
1286
|
// Private helpers
|
|
1237
1287
|
// ============================================================================
|
|
1238
1288
|
|
|
1289
|
+
/**
|
|
1290
|
+
* Resolve a sub-offset against a source stream and return the prefix bytes
|
|
1291
|
+
* to materialize as the fork's first own message. Reads from the source
|
|
1292
|
+
* (across its fork chain if any) starting at forkOffset; the first message
|
|
1293
|
+
* returned is the one that starts at forkOffset. Throws if the sub-offset
|
|
1294
|
+
* cannot be satisfied (no message past forkOffset, or overshoots its
|
|
1295
|
+
* content extent).
|
|
1296
|
+
*/
|
|
1297
|
+
private resolveForkSubOffset(
|
|
1298
|
+
sourceStream: Stream,
|
|
1299
|
+
forkOffset: string,
|
|
1300
|
+
subOffset: number,
|
|
1301
|
+
isJSON: boolean
|
|
1302
|
+
): Uint8Array {
|
|
1303
|
+
// Read source past forkOffset across its fork chain
|
|
1304
|
+
let sourceMessages: Array<StreamMessage>
|
|
1305
|
+
if (sourceStream.forkedFrom) {
|
|
1306
|
+
sourceMessages = [
|
|
1307
|
+
...this.readForkedMessages(
|
|
1308
|
+
sourceStream.forkedFrom,
|
|
1309
|
+
forkOffset,
|
|
1310
|
+
sourceStream.forkOffset!
|
|
1311
|
+
),
|
|
1312
|
+
...this.readOwnMessages(sourceStream, forkOffset),
|
|
1313
|
+
]
|
|
1314
|
+
} else {
|
|
1315
|
+
sourceMessages = this.readOwnMessages(sourceStream, forkOffset)
|
|
1316
|
+
}
|
|
1317
|
+
if (sourceMessages.length === 0) {
|
|
1318
|
+
throw new Error(`Invalid fork sub-offset: no data past forkOffset`)
|
|
1319
|
+
}
|
|
1320
|
+
const first = sourceMessages[0]!
|
|
1321
|
+
if (isJSON) {
|
|
1322
|
+
// The message data is comma-joined JSON values with a trailing comma
|
|
1323
|
+
// (e.g., `{"a":1},{"b":2},`). Wrap in [...] to parse, take first N
|
|
1324
|
+
// elements, re-encode in the same comma-joined format.
|
|
1325
|
+
const text = new TextDecoder().decode(first.data)
|
|
1326
|
+
const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text
|
|
1327
|
+
let values: Array<unknown>
|
|
1328
|
+
try {
|
|
1329
|
+
values = JSON.parse(`[${trimmed}]`)
|
|
1330
|
+
} catch {
|
|
1331
|
+
throw new Error(`Invalid fork sub-offset: source JSON is unparseable`)
|
|
1332
|
+
}
|
|
1333
|
+
if (subOffset > values.length) {
|
|
1334
|
+
throw new Error(
|
|
1335
|
+
`Invalid fork sub-offset: overshoots source message count`
|
|
1336
|
+
)
|
|
1337
|
+
}
|
|
1338
|
+
const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v))
|
|
1339
|
+
return new TextEncoder().encode(prefix.join(`,`) + `,`)
|
|
1340
|
+
}
|
|
1341
|
+
// Binary: take first subOffset bytes
|
|
1342
|
+
if (subOffset > first.data.length) {
|
|
1343
|
+
throw new Error(
|
|
1344
|
+
`Invalid fork sub-offset: overshoots source message length`
|
|
1345
|
+
)
|
|
1346
|
+
}
|
|
1347
|
+
return first.data.slice(0, subOffset)
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1239
1350
|
private appendToStream(
|
|
1240
1351
|
stream: Stream,
|
|
1241
1352
|
data: Uint8Array,
|
package/src/types.ts
CHANGED
|
@@ -107,6 +107,14 @@ export interface Stream {
|
|
|
107
107
|
*/
|
|
108
108
|
forkOffset?: string
|
|
109
109
|
|
|
110
|
+
/**
|
|
111
|
+
* User-supplied sub-offset value refining `forkOffset` (Section 4.2 of
|
|
112
|
+
* PROTOCOL.md). Stored verbatim for idempotent re-creation matching:
|
|
113
|
+
* bytes for non-JSON forks, flattened message count for JSON forks.
|
|
114
|
+
* `undefined` and `0` are equivalent.
|
|
115
|
+
*/
|
|
116
|
+
forkSubOffset?: number
|
|
117
|
+
|
|
110
118
|
/**
|
|
111
119
|
* Number of forks referencing this stream.
|
|
112
120
|
* Defaults to 0.
|