@durable-streams/server 0.3.4 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -170,29 +170,35 @@ var StreamStore = class {
170
170
  const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
171
171
  const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
172
172
  const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
173
- if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches) return existingRaw;
173
+ const requestedSub = options.forkSubOffset ?? 0;
174
+ const existingSub = existingRaw.forkSubOffset ?? 0;
175
+ const forkSubOffsetMatches = requestedSub === existingSub;
176
+ if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches && forkSubOffsetMatches) return existingRaw;
174
177
  else throw new Error(`Stream already exists with different configuration: ${path}`);
175
178
  }
176
179
  const isFork = !!options.forkedFrom;
177
180
  let forkOffset = `0000000000000000_0000000000000000`;
178
181
  let sourceContentType;
179
182
  let sourceStream;
183
+ let forkSubOffsetPrefix;
180
184
  if (isFork) {
181
185
  sourceStream = this.streams.get(options.forkedFrom);
182
186
  if (!sourceStream) throw new Error(`Source stream not found: ${options.forkedFrom}`);
183
187
  if (sourceStream.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
184
188
  if (this.isExpired(sourceStream)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
185
189
  sourceContentType = sourceStream.contentType;
190
+ if (options.contentType && options.contentType.trim() !== `` && normalizeContentType(options.contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
186
191
  if (options.forkOffset) forkOffset = options.forkOffset;
187
192
  else forkOffset = sourceStream.currentOffset;
188
193
  const zeroOffset = `0000000000000000_0000000000000000`;
189
194
  if (forkOffset < zeroOffset || sourceStream.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
195
+ if (options.forkSubOffset && options.forkSubOffset > 0) forkSubOffsetPrefix = this.resolveForkSubOffset(sourceStream, forkOffset, options.forkSubOffset, normalizeContentType(sourceContentType) === `application/json`);
190
196
  sourceStream.refCount++;
191
197
  }
192
198
  let contentType = options.contentType;
193
199
  if (!contentType || contentType.trim() === ``) {
194
200
  if (isFork) contentType = sourceContentType;
195
- } else if (isFork && normalizeContentType(contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
201
+ }
196
202
  let effectiveExpiresAt = options.expiresAt;
197
203
  let effectiveTtlSeconds = options.ttlSeconds;
198
204
  if (isFork) {
@@ -214,6 +220,20 @@ var StreamStore = class {
214
220
  forkedFrom: isFork ? options.forkedFrom : void 0,
215
221
  forkOffset: isFork ? forkOffset : void 0
216
222
  };
223
+ if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
224
+ const parts = stream.currentOffset.split(`_`).map(Number);
225
+ const readSeq = parts[0];
226
+ const byteOffset = parts[1];
227
+ const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5;
228
+ const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
229
+ stream.messages.push({
230
+ data: forkSubOffsetPrefix,
231
+ offset: newOffset,
232
+ timestamp: Date.now()
233
+ });
234
+ stream.currentOffset = newOffset;
235
+ stream.forkSubOffset = options.forkSubOffset;
236
+ }
217
237
  if (options.initialData && options.initialData.length > 0) try {
218
238
  this.appendToStream(stream, options.initialData, true);
219
239
  } catch (err) {
@@ -737,6 +757,36 @@ var StreamStore = class {
737
757
  list() {
738
758
  return Array.from(this.streams.keys());
739
759
  }
760
+ /**
761
+ * Resolve a sub-offset against a source stream and return the prefix bytes
762
+ * to materialize as the fork's first own message. Reads from the source
763
+ * (across its fork chain if any) starting at forkOffset; the first message
764
+ * returned is the one that starts at forkOffset. Throws if the sub-offset
765
+ * cannot be satisfied (no message past forkOffset, or overshoots its
766
+ * content extent).
767
+ */
768
+ resolveForkSubOffset(sourceStream, forkOffset, subOffset, isJSON) {
769
+ let sourceMessages;
770
+ if (sourceStream.forkedFrom) sourceMessages = [...this.readForkedMessages(sourceStream.forkedFrom, forkOffset, sourceStream.forkOffset), ...this.readOwnMessages(sourceStream, forkOffset)];
771
+ else sourceMessages = this.readOwnMessages(sourceStream, forkOffset);
772
+ if (sourceMessages.length === 0) throw new Error(`Invalid fork sub-offset: no data past forkOffset`);
773
+ const first = sourceMessages[0];
774
+ if (isJSON) {
775
+ const text = new TextDecoder().decode(first.data);
776
+ const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text;
777
+ let values;
778
+ try {
779
+ values = JSON.parse(`[${trimmed}]`);
780
+ } catch {
781
+ throw new Error(`Invalid fork sub-offset: source JSON is unparseable`);
782
+ }
783
+ if (subOffset > values.length) throw new Error(`Invalid fork sub-offset: overshoots source message count`);
784
+ const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v));
785
+ return new TextEncoder().encode(prefix.join(`,`) + `,`);
786
+ }
787
+ if (subOffset > first.data.length) throw new Error(`Invalid fork sub-offset: overshoots source message length`);
788
+ return first.data.slice(0, subOffset);
789
+ }
740
790
  appendToStream(stream, data, isInitialCreate = false) {
741
791
  let processedData = data;
742
792
  if (normalizeContentType(stream.contentType) === `application/json`) {
@@ -1318,13 +1368,17 @@ var FileBackedStreamStore = class {
1318
1368
  const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
1319
1369
  const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
1320
1370
  const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
1321
- if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches) return this.streamMetaToStream(existingRaw);
1371
+ const requestedSub = options.forkSubOffset ?? 0;
1372
+ const existingSub = existingRaw.forkSubOffset ?? 0;
1373
+ const forkSubOffsetMatches = requestedSub === existingSub;
1374
+ if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches && forkSubOffsetMatches) return this.streamMetaToStream(existingRaw);
1322
1375
  else throw new Error(`Stream already exists with different configuration: ${streamPath}`);
1323
1376
  }
1324
1377
  const isFork = !!options.forkedFrom;
1325
1378
  let forkOffset = `0000000000000000_0000000000000000`;
1326
1379
  let sourceContentType;
1327
1380
  let sourceMeta;
1381
+ let forkSubOffsetPrefix;
1328
1382
  if (isFork) {
1329
1383
  const sourceKey = `stream:${options.forkedFrom}`;
1330
1384
  sourceMeta = this.db.get(sourceKey);
@@ -1332,10 +1386,12 @@ var FileBackedStreamStore = class {
1332
1386
  if (sourceMeta.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
1333
1387
  if (this.isExpired(sourceMeta)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
1334
1388
  sourceContentType = sourceMeta.contentType;
1389
+ if (options.contentType && options.contentType.trim() !== `` && normalizeContentType(options.contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
1335
1390
  if (options.forkOffset) forkOffset = options.forkOffset;
1336
1391
  else forkOffset = sourceMeta.currentOffset;
1337
1392
  const zeroOffset = `0000000000000000_0000000000000000`;
1338
1393
  if (forkOffset < zeroOffset || sourceMeta.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
1394
+ if (options.forkSubOffset && options.forkSubOffset > 0) forkSubOffsetPrefix = this.resolveForkSubOffset(options.forkedFrom, forkOffset, options.forkSubOffset, normalizeContentType(sourceContentType) === `application/json`);
1339
1395
  const freshSource = this.db.get(sourceKey);
1340
1396
  const updatedSource = {
1341
1397
  ...freshSource,
@@ -1346,7 +1402,7 @@ var FileBackedStreamStore = class {
1346
1402
  let contentType = options.contentType;
1347
1403
  if (!contentType || contentType.trim() === ``) {
1348
1404
  if (isFork) contentType = sourceContentType;
1349
- } else if (isFork && normalizeContentType(contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
1405
+ }
1350
1406
  let effectiveExpiresAt = options.expiresAt;
1351
1407
  let effectiveTtlSeconds = options.ttlSeconds;
1352
1408
  if (isFork) {
@@ -1371,11 +1427,39 @@ var FileBackedStreamStore = class {
1371
1427
  closed: false,
1372
1428
  forkedFrom: isFork ? options.forkedFrom : void 0,
1373
1429
  forkOffset: isFork ? forkOffset : void 0,
1430
+ forkSubOffset: void 0,
1374
1431
  refCount: 0
1375
1432
  };
1376
1433
  const tAfterMeta = performance.now();
1377
1434
  const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
1378
1435
  try {
1436
+ if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
1437
+ const lengthBuf = Buffer.allocUnsafe(4);
1438
+ lengthBuf.writeUInt32BE(forkSubOffsetPrefix.length, 0);
1439
+ const frameBuf = Buffer.concat([
1440
+ lengthBuf,
1441
+ Buffer.from(forkSubOffsetPrefix),
1442
+ Buffer.from(`\n`)
1443
+ ]);
1444
+ const fd = node_fs.openSync(segmentPath, `wx`);
1445
+ try {
1446
+ let written = 0;
1447
+ while (written < frameBuf.length) {
1448
+ const bytesWritten = node_fs.writeSync(fd, frameBuf, written, frameBuf.length - written, written);
1449
+ if (bytesWritten === 0) throw new Error(`failed to write sub-offset prefix frame`);
1450
+ written += bytesWritten;
1451
+ }
1452
+ node_fs.fsyncSync(fd);
1453
+ } finally {
1454
+ node_fs.closeSync(fd);
1455
+ }
1456
+ const parts = streamMeta.currentOffset.split(`_`).map(Number);
1457
+ const readSeq = parts[0];
1458
+ const byteOffset = parts[1];
1459
+ const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5;
1460
+ streamMeta.currentOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
1461
+ streamMeta.forkSubOffset = options.forkSubOffset;
1462
+ }
1379
1463
  await this.db.put(key, streamMeta);
1380
1464
  } catch (err) {
1381
1465
  if (isFork && sourceMeta) {
@@ -1389,7 +1473,7 @@ var FileBackedStreamStore = class {
1389
1473
  this.db.putSync(sourceKey, updatedSource);
1390
1474
  }
1391
1475
  }
1392
- serverLog.error(`[FileBackedStreamStore] Error creating stream (LMDB put):`, err);
1476
+ serverLog.error(`[FileBackedStreamStore] Error creating stream before metadata commit:`, err);
1393
1477
  throw err;
1394
1478
  }
1395
1479
  const tAfterLmdb = performance.now();
@@ -1774,6 +1858,35 @@ var FileBackedStreamStore = class {
1774
1858
  messages.push(...ownMessages);
1775
1859
  return messages;
1776
1860
  }
1861
+ /**
1862
+ * Resolve a fork sub-offset against the source: read the message that
1863
+ * starts at forkOffset and return prefix bytes to materialize as the
1864
+ * fork's first own message. For JSON, parses comma-joined values.
1865
+ */
1866
+ resolveForkSubOffset(sourcePath, forkOffset, subOffset, isJSON) {
1867
+ const forkByte = Number(forkOffset.split(`_`)[1] ?? 0);
1868
+ const sourceMeta = this.db.get(`stream:${sourcePath}`);
1869
+ if (!sourceMeta) throw new Error(`Source stream not found: ${sourcePath}`);
1870
+ const currentByte = Number(sourceMeta.currentOffset.split(`_`)[1] ?? 0);
1871
+ const messages = this.readForkedMessages(sourcePath, forkByte, currentByte);
1872
+ if (messages.length === 0) throw new Error(`Invalid fork sub-offset: no data past forkOffset`);
1873
+ const first = messages[0];
1874
+ if (isJSON) {
1875
+ const text = new TextDecoder().decode(first.data);
1876
+ const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text;
1877
+ let values;
1878
+ try {
1879
+ values = JSON.parse(`[${trimmed}]`);
1880
+ } catch {
1881
+ throw new Error(`Invalid fork sub-offset: source JSON is unparseable`);
1882
+ }
1883
+ if (subOffset > values.length) throw new Error(`Invalid fork sub-offset: overshoots source message count`);
1884
+ const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v));
1885
+ return new TextEncoder().encode(prefix.join(`,`) + `,`);
1886
+ }
1887
+ if (subOffset > first.data.length) throw new Error(`Invalid fork sub-offset: overshoots source message length`);
1888
+ return first.data.slice(0, subOffset);
1889
+ }
1777
1890
  read(streamPath, offset) {
1778
1891
  const streamMeta = this.getMetaIfNotExpired(streamPath);
1779
1892
  if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
@@ -3030,6 +3143,7 @@ const STREAM_SSE_DATA_ENCODING_HEADER = `Stream-SSE-Data-Encoding`;
3030
3143
  const SSE_UP_TO_DATE_FIELD = `upToDate`;
3031
3144
  const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`;
3032
3145
  const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`;
3146
+ const STREAM_FORK_SUB_OFFSET_HEADER = `Stream-Fork-Sub-Offset`;
3033
3147
  /**
3034
3148
  * Encode data for SSE format.
3035
3149
  * Per SSE spec, each line in the payload needs its own "data:" prefix.
@@ -3255,7 +3369,7 @@ var DurableStreamTestServer = class {
3255
3369
  const method = req.method?.toUpperCase();
3256
3370
  res.setHeader(`access-control-allow-origin`, `*`);
3257
3371
  res.setHeader(`access-control-allow-methods`, `GET, POST, PUT, DELETE, HEAD, OPTIONS`);
3258
- res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset`);
3372
+ res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset, Stream-Fork-Sub-Offset`);
3259
3373
  res.setHeader(`access-control-expose-headers`, `Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Date, Stream-Closed, Producer-Epoch, Producer-Seq, Producer-Expected-Seq, Producer-Received-Seq, etag, content-type, content-encoding, vary`);
3260
3374
  res.setHeader(`x-content-type-options`, `nosniff`);
3261
3375
  res.setHeader(`cross-origin-resource-policy`, `cross-origin`);
@@ -3345,6 +3459,9 @@ var DurableStreamTestServer = class {
3345
3459
  let contentType = req.headers[`content-type`];
3346
3460
  const forkedFromHeader = req.headers[STREAM_FORKED_FROM_HEADER.toLowerCase()];
3347
3461
  const forkOffsetHeader = req.headers[STREAM_FORK_OFFSET_HEADER.toLowerCase()];
3462
+ const forkSubOffsetHeaderRaw = req.headers[STREAM_FORK_SUB_OFFSET_HEADER.toLowerCase()];
3463
+ const forkSubOffsetHeaderPresent = forkSubOffsetHeaderRaw !== void 0;
3464
+ const forkSubOffsetHeader = Array.isArray(forkSubOffsetHeaderRaw) ? forkSubOffsetHeaderRaw[0] : forkSubOffsetHeaderRaw;
3348
3465
  if (!contentType || contentType.trim() === `` || !/^[\w-]+\/[\w-]+/.test(contentType)) contentType = forkedFromHeader ? void 0 : `application/octet-stream`;
3349
3466
  const ttlHeader = req.headers[__durable_streams_client.STREAM_TTL_HEADER.toLowerCase()];
3350
3467
  const expiresAtHeader = req.headers[__durable_streams_client.STREAM_EXPIRES_AT_HEADER.toLowerCase()];
@@ -3386,6 +3503,21 @@ var DurableStreamTestServer = class {
3386
3503
  return;
3387
3504
  }
3388
3505
  }
3506
+ let forkSubOffset;
3507
+ if (forkSubOffsetHeaderPresent) {
3508
+ if (!forkedFromHeader) {
3509
+ res.writeHead(400, { "content-type": `text/plain` });
3510
+ res.end(`Stream-Fork-Sub-Offset requires Stream-Forked-From`);
3511
+ return;
3512
+ }
3513
+ const subOffsetPattern = /^(0|[1-9]\d*)$/;
3514
+ if (forkSubOffsetHeader === void 0 || !subOffsetPattern.test(forkSubOffsetHeader)) {
3515
+ res.writeHead(400, { "content-type": `text/plain` });
3516
+ res.end(`Invalid Stream-Fork-Sub-Offset format`);
3517
+ return;
3518
+ }
3519
+ forkSubOffset = parseInt(forkSubOffsetHeader, 10);
3520
+ }
3389
3521
  const body = await this.readBody(req);
3390
3522
  const isNew = !this.store.has(path);
3391
3523
  try {
@@ -3396,7 +3528,8 @@ var DurableStreamTestServer = class {
3396
3528
  initialData: body.length > 0 ? body : void 0,
3397
3529
  closed: createClosed,
3398
3530
  forkedFrom: forkedFromHeader,
3399
- forkOffset: forkOffsetHeader
3531
+ forkOffset: forkOffsetHeader,
3532
+ forkSubOffset
3400
3533
  }));
3401
3534
  } catch (err) {
3402
3535
  if (err instanceof Error) {
@@ -3405,6 +3538,11 @@ var DurableStreamTestServer = class {
3405
3538
  res.end(`Source stream not found`);
3406
3539
  return;
3407
3540
  }
3541
+ if (err.message.includes(`Invalid fork sub-offset`)) {
3542
+ res.writeHead(400, { "content-type": `text/plain` });
3543
+ res.end(`Invalid fork sub-offset`);
3544
+ return;
3545
+ }
3408
3546
  if (err.message.includes(`Invalid fork offset`)) {
3409
3547
  res.writeHead(400, { "content-type": `text/plain` });
3410
3548
  res.end(`Fork offset beyond source stream length`);
package/dist/index.d.cts CHANGED
@@ -99,6 +99,13 @@ interface Stream {
99
99
  */
100
100
  forkOffset?: string;
101
101
  /**
102
+ * User-supplied sub-offset value refining `forkOffset` (Section 4.2 of
103
+ * PROTOCOL.md). Stored verbatim for idempotent re-creation matching:
104
+ * bytes for non-JSON forks, flattened message count for JSON forks.
105
+ * `undefined` and `0` are equivalent.
106
+ */
107
+ forkSubOffset?: number;
108
+ /**
102
109
  * Number of forks referencing this stream.
103
110
  * Defaults to 0.
104
111
  */
@@ -313,6 +320,7 @@ declare class StreamStore {
313
320
  closed?: boolean;
314
321
  forkedFrom?: string;
315
322
  forkOffset?: string;
323
+ forkSubOffset?: number;
316
324
  }): Stream;
317
325
  /**
318
326
  * Resolve fork expiry per the decision table.
@@ -459,6 +467,15 @@ declare class StreamStore {
459
467
  * Get all stream paths.
460
468
  */
461
469
  list(): Array<string>;
470
+ /**
471
+ * Resolve a sub-offset against a source stream and return the prefix bytes
472
+ * to materialize as the fork's first own message. Reads from the source
473
+ * (across its fork chain if any) starting at forkOffset; the first message
474
+ * returned is the one that starts at forkOffset. Throws if the sub-offset
475
+ * cannot be satisfied (no message past forkOffset, or overshoots its
476
+ * content extent).
477
+ */
478
+ private resolveForkSubOffset;
462
479
  private appendToStream;
463
480
  private findOffsetIndex;
464
481
  private notifyLongPolls;
@@ -571,6 +588,7 @@ declare class FileBackedStreamStore {
571
588
  closed?: boolean;
572
589
  forkedFrom?: string;
573
590
  forkOffset?: string;
591
+ forkSubOffset?: number;
574
592
  }): Promise<Stream>;
575
593
  get(streamPath: string): Stream | undefined;
576
594
  has(streamPath: string): boolean;
@@ -631,6 +649,12 @@ declare class FileBackedStreamStore {
631
649
  * Does NOT check softDeleted -- forks must read through soft-deleted sources.
632
650
  */
633
651
  private readForkedMessages;
652
+ /**
653
+ * Resolve a fork sub-offset against the source: read the message that
654
+ * starts at forkOffset and return prefix bytes to materialize as the
655
+ * fork's first own message. For JSON, parses comma-joined values.
656
+ */
657
+ private resolveForkSubOffset;
634
658
  read(streamPath: string, offset?: string): {
635
659
  messages: Array<StreamMessage>;
636
660
  upToDate: boolean;
package/dist/index.d.ts CHANGED
@@ -99,6 +99,13 @@ interface Stream {
99
99
  */
100
100
  forkOffset?: string;
101
101
  /**
102
+ * User-supplied sub-offset value refining `forkOffset` (Section 4.2 of
103
+ * PROTOCOL.md). Stored verbatim for idempotent re-creation matching:
104
+ * bytes for non-JSON forks, flattened message count for JSON forks.
105
+ * `undefined` and `0` are equivalent.
106
+ */
107
+ forkSubOffset?: number;
108
+ /**
102
109
  * Number of forks referencing this stream.
103
110
  * Defaults to 0.
104
111
  */
@@ -313,6 +320,7 @@ declare class StreamStore {
313
320
  closed?: boolean;
314
321
  forkedFrom?: string;
315
322
  forkOffset?: string;
323
+ forkSubOffset?: number;
316
324
  }): Stream;
317
325
  /**
318
326
  * Resolve fork expiry per the decision table.
@@ -459,6 +467,15 @@ declare class StreamStore {
459
467
  * Get all stream paths.
460
468
  */
461
469
  list(): Array<string>;
470
+ /**
471
+ * Resolve a sub-offset against a source stream and return the prefix bytes
472
+ * to materialize as the fork's first own message. Reads from the source
473
+ * (across its fork chain if any) starting at forkOffset; the first message
474
+ * returned is the one that starts at forkOffset. Throws if the sub-offset
475
+ * cannot be satisfied (no message past forkOffset, or overshoots its
476
+ * content extent).
477
+ */
478
+ private resolveForkSubOffset;
462
479
  private appendToStream;
463
480
  private findOffsetIndex;
464
481
  private notifyLongPolls;
@@ -571,6 +588,7 @@ declare class FileBackedStreamStore {
571
588
  closed?: boolean;
572
589
  forkedFrom?: string;
573
590
  forkOffset?: string;
591
+ forkSubOffset?: number;
574
592
  }): Promise<Stream>;
575
593
  get(streamPath: string): Stream | undefined;
576
594
  has(streamPath: string): boolean;
@@ -631,6 +649,12 @@ declare class FileBackedStreamStore {
631
649
  * Does NOT check softDeleted -- forks must read through soft-deleted sources.
632
650
  */
633
651
  private readForkedMessages;
652
+ /**
653
+ * Resolve a fork sub-offset against the source: read the message that
654
+ * starts at forkOffset and return prefix bytes to materialize as the
655
+ * fork's first own message. For JSON, parses comma-joined values.
656
+ */
657
+ private resolveForkSubOffset;
634
658
  read(streamPath: string, offset?: string): {
635
659
  messages: Array<StreamMessage>;
636
660
  upToDate: boolean;
package/dist/index.js CHANGED
@@ -146,29 +146,35 @@ var StreamStore = class {
146
146
  const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
147
147
  const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
148
148
  const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
149
- if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches) return existingRaw;
149
+ const requestedSub = options.forkSubOffset ?? 0;
150
+ const existingSub = existingRaw.forkSubOffset ?? 0;
151
+ const forkSubOffsetMatches = requestedSub === existingSub;
152
+ if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches && forkSubOffsetMatches) return existingRaw;
150
153
  else throw new Error(`Stream already exists with different configuration: ${path$1}`);
151
154
  }
152
155
  const isFork = !!options.forkedFrom;
153
156
  let forkOffset = `0000000000000000_0000000000000000`;
154
157
  let sourceContentType;
155
158
  let sourceStream;
159
+ let forkSubOffsetPrefix;
156
160
  if (isFork) {
157
161
  sourceStream = this.streams.get(options.forkedFrom);
158
162
  if (!sourceStream) throw new Error(`Source stream not found: ${options.forkedFrom}`);
159
163
  if (sourceStream.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
160
164
  if (this.isExpired(sourceStream)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
161
165
  sourceContentType = sourceStream.contentType;
166
+ if (options.contentType && options.contentType.trim() !== `` && normalizeContentType(options.contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
162
167
  if (options.forkOffset) forkOffset = options.forkOffset;
163
168
  else forkOffset = sourceStream.currentOffset;
164
169
  const zeroOffset = `0000000000000000_0000000000000000`;
165
170
  if (forkOffset < zeroOffset || sourceStream.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
171
+ if (options.forkSubOffset && options.forkSubOffset > 0) forkSubOffsetPrefix = this.resolveForkSubOffset(sourceStream, forkOffset, options.forkSubOffset, normalizeContentType(sourceContentType) === `application/json`);
166
172
  sourceStream.refCount++;
167
173
  }
168
174
  let contentType = options.contentType;
169
175
  if (!contentType || contentType.trim() === ``) {
170
176
  if (isFork) contentType = sourceContentType;
171
- } else if (isFork && normalizeContentType(contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
177
+ }
172
178
  let effectiveExpiresAt = options.expiresAt;
173
179
  let effectiveTtlSeconds = options.ttlSeconds;
174
180
  if (isFork) {
@@ -190,6 +196,20 @@ var StreamStore = class {
190
196
  forkedFrom: isFork ? options.forkedFrom : void 0,
191
197
  forkOffset: isFork ? forkOffset : void 0
192
198
  };
199
+ if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
200
+ const parts = stream.currentOffset.split(`_`).map(Number);
201
+ const readSeq = parts[0];
202
+ const byteOffset = parts[1];
203
+ const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5;
204
+ const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
205
+ stream.messages.push({
206
+ data: forkSubOffsetPrefix,
207
+ offset: newOffset,
208
+ timestamp: Date.now()
209
+ });
210
+ stream.currentOffset = newOffset;
211
+ stream.forkSubOffset = options.forkSubOffset;
212
+ }
193
213
  if (options.initialData && options.initialData.length > 0) try {
194
214
  this.appendToStream(stream, options.initialData, true);
195
215
  } catch (err) {
@@ -713,6 +733,36 @@ var StreamStore = class {
713
733
  list() {
714
734
  return Array.from(this.streams.keys());
715
735
  }
736
+ /**
737
+ * Resolve a sub-offset against a source stream and return the prefix bytes
738
+ * to materialize as the fork's first own message. Reads from the source
739
+ * (across its fork chain if any) starting at forkOffset; the first message
740
+ * returned is the one that starts at forkOffset. Throws if the sub-offset
741
+ * cannot be satisfied (no message past forkOffset, or overshoots its
742
+ * content extent).
743
+ */
744
+ resolveForkSubOffset(sourceStream, forkOffset, subOffset, isJSON) {
745
+ let sourceMessages;
746
+ if (sourceStream.forkedFrom) sourceMessages = [...this.readForkedMessages(sourceStream.forkedFrom, forkOffset, sourceStream.forkOffset), ...this.readOwnMessages(sourceStream, forkOffset)];
747
+ else sourceMessages = this.readOwnMessages(sourceStream, forkOffset);
748
+ if (sourceMessages.length === 0) throw new Error(`Invalid fork sub-offset: no data past forkOffset`);
749
+ const first = sourceMessages[0];
750
+ if (isJSON) {
751
+ const text = new TextDecoder().decode(first.data);
752
+ const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text;
753
+ let values;
754
+ try {
755
+ values = JSON.parse(`[${trimmed}]`);
756
+ } catch {
757
+ throw new Error(`Invalid fork sub-offset: source JSON is unparseable`);
758
+ }
759
+ if (subOffset > values.length) throw new Error(`Invalid fork sub-offset: overshoots source message count`);
760
+ const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v));
761
+ return new TextEncoder().encode(prefix.join(`,`) + `,`);
762
+ }
763
+ if (subOffset > first.data.length) throw new Error(`Invalid fork sub-offset: overshoots source message length`);
764
+ return first.data.slice(0, subOffset);
765
+ }
716
766
  appendToStream(stream, data, isInitialCreate = false) {
717
767
  let processedData = data;
718
768
  if (normalizeContentType(stream.contentType) === `application/json`) {
@@ -1294,13 +1344,17 @@ var FileBackedStreamStore = class {
1294
1344
  const closedMatches = (options.closed ?? false) === (existingRaw.closed ?? false);
1295
1345
  const forkedFromMatches = (options.forkedFrom ?? void 0) === existingRaw.forkedFrom;
1296
1346
  const forkOffsetMatches = options.forkOffset === void 0 || options.forkOffset === existingRaw.forkOffset;
1297
- if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches) return this.streamMetaToStream(existingRaw);
1347
+ const requestedSub = options.forkSubOffset ?? 0;
1348
+ const existingSub = existingRaw.forkSubOffset ?? 0;
1349
+ const forkSubOffsetMatches = requestedSub === existingSub;
1350
+ if (contentTypeMatches && ttlMatches && expiresMatches && closedMatches && forkedFromMatches && forkOffsetMatches && forkSubOffsetMatches) return this.streamMetaToStream(existingRaw);
1298
1351
  else throw new Error(`Stream already exists with different configuration: ${streamPath}`);
1299
1352
  }
1300
1353
  const isFork = !!options.forkedFrom;
1301
1354
  let forkOffset = `0000000000000000_0000000000000000`;
1302
1355
  let sourceContentType;
1303
1356
  let sourceMeta;
1357
+ let forkSubOffsetPrefix;
1304
1358
  if (isFork) {
1305
1359
  const sourceKey = `stream:${options.forkedFrom}`;
1306
1360
  sourceMeta = this.db.get(sourceKey);
@@ -1308,10 +1362,12 @@ var FileBackedStreamStore = class {
1308
1362
  if (sourceMeta.softDeleted) throw new Error(`Source stream is soft-deleted: ${options.forkedFrom}`);
1309
1363
  if (this.isExpired(sourceMeta)) throw new Error(`Source stream not found: ${options.forkedFrom}`);
1310
1364
  sourceContentType = sourceMeta.contentType;
1365
+ if (options.contentType && options.contentType.trim() !== `` && normalizeContentType(options.contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
1311
1366
  if (options.forkOffset) forkOffset = options.forkOffset;
1312
1367
  else forkOffset = sourceMeta.currentOffset;
1313
1368
  const zeroOffset = `0000000000000000_0000000000000000`;
1314
1369
  if (forkOffset < zeroOffset || sourceMeta.currentOffset < forkOffset) throw new Error(`Invalid fork offset: ${forkOffset}`);
1370
+ if (options.forkSubOffset && options.forkSubOffset > 0) forkSubOffsetPrefix = this.resolveForkSubOffset(options.forkedFrom, forkOffset, options.forkSubOffset, normalizeContentType(sourceContentType) === `application/json`);
1315
1371
  const freshSource = this.db.get(sourceKey);
1316
1372
  const updatedSource = {
1317
1373
  ...freshSource,
@@ -1322,7 +1378,7 @@ var FileBackedStreamStore = class {
1322
1378
  let contentType = options.contentType;
1323
1379
  if (!contentType || contentType.trim() === ``) {
1324
1380
  if (isFork) contentType = sourceContentType;
1325
- } else if (isFork && normalizeContentType(contentType) !== normalizeContentType(sourceContentType)) throw new Error(`Content type mismatch with source stream`);
1381
+ }
1326
1382
  let effectiveExpiresAt = options.expiresAt;
1327
1383
  let effectiveTtlSeconds = options.ttlSeconds;
1328
1384
  if (isFork) {
@@ -1347,11 +1403,39 @@ var FileBackedStreamStore = class {
1347
1403
  closed: false,
1348
1404
  forkedFrom: isFork ? options.forkedFrom : void 0,
1349
1405
  forkOffset: isFork ? forkOffset : void 0,
1406
+ forkSubOffset: void 0,
1350
1407
  refCount: 0
1351
1408
  };
1352
1409
  const tAfterMeta = performance.now();
1353
1410
  const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName);
1354
1411
  try {
1412
+ if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
1413
+ const lengthBuf = Buffer.allocUnsafe(4);
1414
+ lengthBuf.writeUInt32BE(forkSubOffsetPrefix.length, 0);
1415
+ const frameBuf = Buffer.concat([
1416
+ lengthBuf,
1417
+ Buffer.from(forkSubOffsetPrefix),
1418
+ Buffer.from(`\n`)
1419
+ ]);
1420
+ const fd = fs.openSync(segmentPath, `wx`);
1421
+ try {
1422
+ let written = 0;
1423
+ while (written < frameBuf.length) {
1424
+ const bytesWritten = fs.writeSync(fd, frameBuf, written, frameBuf.length - written, written);
1425
+ if (bytesWritten === 0) throw new Error(`failed to write sub-offset prefix frame`);
1426
+ written += bytesWritten;
1427
+ }
1428
+ fs.fsyncSync(fd);
1429
+ } finally {
1430
+ fs.closeSync(fd);
1431
+ }
1432
+ const parts = streamMeta.currentOffset.split(`_`).map(Number);
1433
+ const readSeq = parts[0];
1434
+ const byteOffset = parts[1];
1435
+ const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5;
1436
+ streamMeta.currentOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`;
1437
+ streamMeta.forkSubOffset = options.forkSubOffset;
1438
+ }
1355
1439
  await this.db.put(key, streamMeta);
1356
1440
  } catch (err) {
1357
1441
  if (isFork && sourceMeta) {
@@ -1365,7 +1449,7 @@ var FileBackedStreamStore = class {
1365
1449
  this.db.putSync(sourceKey, updatedSource);
1366
1450
  }
1367
1451
  }
1368
- serverLog.error(`[FileBackedStreamStore] Error creating stream (LMDB put):`, err);
1452
+ serverLog.error(`[FileBackedStreamStore] Error creating stream before metadata commit:`, err);
1369
1453
  throw err;
1370
1454
  }
1371
1455
  const tAfterLmdb = performance.now();
@@ -1750,6 +1834,35 @@ var FileBackedStreamStore = class {
1750
1834
  messages.push(...ownMessages);
1751
1835
  return messages;
1752
1836
  }
1837
+ /**
1838
+ * Resolve a fork sub-offset against the source: read the message that
1839
+ * starts at forkOffset and return prefix bytes to materialize as the
1840
+ * fork's first own message. For JSON, parses comma-joined values.
1841
+ */
1842
+ resolveForkSubOffset(sourcePath, forkOffset, subOffset, isJSON) {
1843
+ const forkByte = Number(forkOffset.split(`_`)[1] ?? 0);
1844
+ const sourceMeta = this.db.get(`stream:${sourcePath}`);
1845
+ if (!sourceMeta) throw new Error(`Source stream not found: ${sourcePath}`);
1846
+ const currentByte = Number(sourceMeta.currentOffset.split(`_`)[1] ?? 0);
1847
+ const messages = this.readForkedMessages(sourcePath, forkByte, currentByte);
1848
+ if (messages.length === 0) throw new Error(`Invalid fork sub-offset: no data past forkOffset`);
1849
+ const first = messages[0];
1850
+ if (isJSON) {
1851
+ const text = new TextDecoder().decode(first.data);
1852
+ const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text;
1853
+ let values;
1854
+ try {
1855
+ values = JSON.parse(`[${trimmed}]`);
1856
+ } catch {
1857
+ throw new Error(`Invalid fork sub-offset: source JSON is unparseable`);
1858
+ }
1859
+ if (subOffset > values.length) throw new Error(`Invalid fork sub-offset: overshoots source message count`);
1860
+ const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v));
1861
+ return new TextEncoder().encode(prefix.join(`,`) + `,`);
1862
+ }
1863
+ if (subOffset > first.data.length) throw new Error(`Invalid fork sub-offset: overshoots source message length`);
1864
+ return first.data.slice(0, subOffset);
1865
+ }
1753
1866
  read(streamPath, offset) {
1754
1867
  const streamMeta = this.getMetaIfNotExpired(streamPath);
1755
1868
  if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
@@ -3006,6 +3119,7 @@ const STREAM_SSE_DATA_ENCODING_HEADER = `Stream-SSE-Data-Encoding`;
3006
3119
  const SSE_UP_TO_DATE_FIELD = `upToDate`;
3007
3120
  const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`;
3008
3121
  const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`;
3122
+ const STREAM_FORK_SUB_OFFSET_HEADER = `Stream-Fork-Sub-Offset`;
3009
3123
  /**
3010
3124
  * Encode data for SSE format.
3011
3125
  * Per SSE spec, each line in the payload needs its own "data:" prefix.
@@ -3231,7 +3345,7 @@ var DurableStreamTestServer = class {
3231
3345
  const method = req.method?.toUpperCase();
3232
3346
  res.setHeader(`access-control-allow-origin`, `*`);
3233
3347
  res.setHeader(`access-control-allow-methods`, `GET, POST, PUT, DELETE, HEAD, OPTIONS`);
3234
- res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset`);
3348
+ res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset, Stream-Fork-Sub-Offset`);
3235
3349
  res.setHeader(`access-control-expose-headers`, `Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Date, Stream-Closed, Producer-Epoch, Producer-Seq, Producer-Expected-Seq, Producer-Received-Seq, etag, content-type, content-encoding, vary`);
3236
3350
  res.setHeader(`x-content-type-options`, `nosniff`);
3237
3351
  res.setHeader(`cross-origin-resource-policy`, `cross-origin`);
@@ -3321,6 +3435,9 @@ var DurableStreamTestServer = class {
3321
3435
  let contentType = req.headers[`content-type`];
3322
3436
  const forkedFromHeader = req.headers[STREAM_FORKED_FROM_HEADER.toLowerCase()];
3323
3437
  const forkOffsetHeader = req.headers[STREAM_FORK_OFFSET_HEADER.toLowerCase()];
3438
+ const forkSubOffsetHeaderRaw = req.headers[STREAM_FORK_SUB_OFFSET_HEADER.toLowerCase()];
3439
+ const forkSubOffsetHeaderPresent = forkSubOffsetHeaderRaw !== void 0;
3440
+ const forkSubOffsetHeader = Array.isArray(forkSubOffsetHeaderRaw) ? forkSubOffsetHeaderRaw[0] : forkSubOffsetHeaderRaw;
3324
3441
  if (!contentType || contentType.trim() === `` || !/^[\w-]+\/[\w-]+/.test(contentType)) contentType = forkedFromHeader ? void 0 : `application/octet-stream`;
3325
3442
  const ttlHeader = req.headers[STREAM_TTL_HEADER.toLowerCase()];
3326
3443
  const expiresAtHeader = req.headers[STREAM_EXPIRES_AT_HEADER.toLowerCase()];
@@ -3362,6 +3479,21 @@ var DurableStreamTestServer = class {
3362
3479
  return;
3363
3480
  }
3364
3481
  }
3482
+ let forkSubOffset;
3483
+ if (forkSubOffsetHeaderPresent) {
3484
+ if (!forkedFromHeader) {
3485
+ res.writeHead(400, { "content-type": `text/plain` });
3486
+ res.end(`Stream-Fork-Sub-Offset requires Stream-Forked-From`);
3487
+ return;
3488
+ }
3489
+ const subOffsetPattern = /^(0|[1-9]\d*)$/;
3490
+ if (forkSubOffsetHeader === void 0 || !subOffsetPattern.test(forkSubOffsetHeader)) {
3491
+ res.writeHead(400, { "content-type": `text/plain` });
3492
+ res.end(`Invalid Stream-Fork-Sub-Offset format`);
3493
+ return;
3494
+ }
3495
+ forkSubOffset = parseInt(forkSubOffsetHeader, 10);
3496
+ }
3365
3497
  const body = await this.readBody(req);
3366
3498
  const isNew = !this.store.has(path$1);
3367
3499
  try {
@@ -3372,7 +3504,8 @@ var DurableStreamTestServer = class {
3372
3504
  initialData: body.length > 0 ? body : void 0,
3373
3505
  closed: createClosed,
3374
3506
  forkedFrom: forkedFromHeader,
3375
- forkOffset: forkOffsetHeader
3507
+ forkOffset: forkOffsetHeader,
3508
+ forkSubOffset
3376
3509
  }));
3377
3510
  } catch (err) {
3378
3511
  if (err instanceof Error) {
@@ -3381,6 +3514,11 @@ var DurableStreamTestServer = class {
3381
3514
  res.end(`Source stream not found`);
3382
3515
  return;
3383
3516
  }
3517
+ if (err.message.includes(`Invalid fork sub-offset`)) {
3518
+ res.writeHead(400, { "content-type": `text/plain` });
3519
+ res.end(`Invalid fork sub-offset`);
3520
+ return;
3521
+ }
3384
3522
  if (err.message.includes(`Invalid fork offset`)) {
3385
3523
  res.writeHead(400, { "content-type": `text/plain` });
3386
3524
  res.end(`Fork offset beyond source stream length`);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@durable-streams/server",
3
- "version": "0.3.4",
3
+ "version": "0.3.6",
4
4
  "description": "Node.js reference server implementation for Durable Streams",
5
5
  "author": "Durable Stream contributors",
6
6
  "license": "Apache-2.0",
@@ -39,15 +39,15 @@
39
39
  "dependencies": {
40
40
  "@neophi/sieve-cache": "^1.0.0",
41
41
  "lmdb": "^3.3.0",
42
- "@durable-streams/client": "0.2.5",
43
- "@durable-streams/state": "0.2.8"
42
+ "@durable-streams/client": "0.2.6",
43
+ "@durable-streams/state": "0.3.0"
44
44
  },
45
45
  "devDependencies": {
46
46
  "@types/node": "^22.0.0",
47
47
  "tsdown": "^0.9.0",
48
48
  "typescript": "^5.0.0",
49
49
  "vitest": "^4.0.0",
50
- "@durable-streams/server-conformance-tests": "0.3.3"
50
+ "@durable-streams/server-conformance-tests": "0.3.5"
51
51
  },
52
52
  "files": [
53
53
  "dist",
package/src/file-store.ts CHANGED
@@ -87,6 +87,12 @@ interface StreamMetadata {
87
87
  * Divergence offset from the source stream.
88
88
  */
89
89
  forkOffset?: string
90
+ /**
91
+ * User-supplied sub-offset value refining `forkOffset` (Section 4.2 of
92
+ * PROTOCOL.md). Stored verbatim for idempotent re-creation matching:
93
+ * bytes for non-JSON forks, flattened message count for JSON forks.
94
+ */
95
+ forkSubOffset?: number
90
96
  /**
91
97
  * Number of forks referencing this stream.
92
98
  * Defaults to 0. Optional for backward-compatible deserialization from LMDB.
@@ -726,6 +732,7 @@ export class FileBackedStreamStore {
726
732
  closed?: boolean
727
733
  forkedFrom?: string
728
734
  forkOffset?: string
735
+ forkSubOffset?: number
729
736
  } = {}
730
737
  ): Promise<Stream> {
731
738
  // Use getMetaIfNotExpired to treat expired streams as non-existent
@@ -762,6 +769,12 @@ export class FileBackedStreamStore {
762
769
  const forkOffsetMatches =
763
770
  options.forkOffset === undefined ||
764
771
  options.forkOffset === existingRaw.forkOffset
772
+ // Sub-offset: undefined and 0 are equivalent. Compare the raw
773
+ // user-supplied integer (count for JSON, bytes for binary) so the
774
+ // comparison is independent of how it was resolved internally.
775
+ const requestedSub = options.forkSubOffset ?? 0
776
+ const existingSub = existingRaw.forkSubOffset ?? 0
777
+ const forkSubOffsetMatches = requestedSub === existingSub
765
778
 
766
779
  if (
767
780
  contentTypeMatches &&
@@ -769,7 +782,8 @@ export class FileBackedStreamStore {
769
782
  expiresMatches &&
770
783
  closedMatches &&
771
784
  forkedFromMatches &&
772
- forkOffsetMatches
785
+ forkOffsetMatches &&
786
+ forkSubOffsetMatches
773
787
  ) {
774
788
  // Idempotent success - return existing stream
775
789
  return this.streamMetaToStream(existingRaw)
@@ -787,6 +801,7 @@ export class FileBackedStreamStore {
787
801
  let forkOffset = `0000000000000000_0000000000000000`
788
802
  let sourceContentType: string | undefined
789
803
  let sourceMeta: StreamMetadata | undefined
804
+ let forkSubOffsetPrefix: Uint8Array | undefined
790
805
 
791
806
  if (isFork) {
792
807
  const sourceKey = `stream:${options.forkedFrom!}`
@@ -803,6 +818,18 @@ export class FileBackedStreamStore {
803
818
 
804
819
  sourceContentType = sourceMeta.contentType
805
820
 
821
+ // Reject a content-type mismatch up front, before taking a reference on
822
+ // the source. Doing this after the refCount increment below would leak a
823
+ // reference on the failed fork and pin the source in a soft-deleted state.
824
+ if (
825
+ options.contentType &&
826
+ options.contentType.trim() !== `` &&
827
+ normalizeContentType(options.contentType) !==
828
+ normalizeContentType(sourceContentType)
829
+ ) {
830
+ throw new Error(`Content type mismatch with source stream`)
831
+ }
832
+
806
833
  // Resolve fork offset: use provided or source's currentOffset
807
834
  if (options.forkOffset) {
808
835
  forkOffset = options.forkOffset
@@ -816,6 +843,17 @@ export class FileBackedStreamStore {
816
843
  throw new Error(`Invalid fork offset: ${forkOffset}`)
817
844
  }
818
845
 
846
+ // Resolve sub-offset against the source. Returns the prefix bytes to
847
+ // materialize as the fork's first own message.
848
+ if (options.forkSubOffset && options.forkSubOffset > 0) {
849
+ forkSubOffsetPrefix = this.resolveForkSubOffset(
850
+ options.forkedFrom!,
851
+ forkOffset,
852
+ options.forkSubOffset,
853
+ normalizeContentType(sourceContentType) === `application/json`
854
+ )
855
+ }
856
+
819
857
  // Atomically increment source refcount in LMDB
820
858
  const freshSource = this.db.get(sourceKey) as StreamMetadata
821
859
  const updatedSource: StreamMetadata = {
@@ -825,18 +863,14 @@ export class FileBackedStreamStore {
825
863
  this.db.putSync(sourceKey, updatedSource)
826
864
  }
827
865
 
828
- // Determine content type: use options, or inherit from source if fork
866
+ // Determine content type: use options, or inherit from source if fork. A
867
+ // fork content-type mismatch is already rejected above, before the source
868
+ // refCount is taken.
829
869
  let contentType = options.contentType
830
870
  if (!contentType || contentType.trim() === ``) {
831
871
  if (isFork) {
832
872
  contentType = sourceContentType
833
873
  }
834
- } else if (
835
- isFork &&
836
- normalizeContentType(contentType) !==
837
- normalizeContentType(sourceContentType)
838
- ) {
839
- throw new Error(`Content type mismatch with source stream`)
840
874
  }
841
875
 
842
876
  // Compute effective expiry for forks
@@ -871,6 +905,7 @@ export class FileBackedStreamStore {
871
905
  closed: false, // Set to false initially, will be updated after initial append if needed
872
906
  forkedFrom: isFork ? options.forkedFrom : undefined,
873
907
  forkOffset: isFork ? forkOffset : undefined,
908
+ forkSubOffset: undefined,
874
909
  refCount: 0,
875
910
  }
876
911
 
@@ -878,6 +913,52 @@ export class FileBackedStreamStore {
878
913
 
879
914
  const segmentPath = segmentFile(this.dataDir, streamMeta.directoryName)
880
915
  try {
916
+ // Materialize the sub-offset prefix as the first framed message
917
+ // in the new segment before persisting metadata or opening the
918
+ // write stream. The prefix must be fsynced before the LMDB commit
919
+ // so metadata never acknowledges bytes that are not durable.
920
+ if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
921
+ const lengthBuf = Buffer.allocUnsafe(4)
922
+ lengthBuf.writeUInt32BE(forkSubOffsetPrefix.length, 0)
923
+ const frameBuf = Buffer.concat([
924
+ lengthBuf,
925
+ Buffer.from(forkSubOffsetPrefix),
926
+ Buffer.from(`\n`),
927
+ ])
928
+
929
+ const fd = fs.openSync(segmentPath, `wx`)
930
+ try {
931
+ let written = 0
932
+ while (written < frameBuf.length) {
933
+ const bytesWritten = fs.writeSync(
934
+ fd,
935
+ frameBuf,
936
+ written,
937
+ frameBuf.length - written,
938
+ written
939
+ )
940
+ if (bytesWritten === 0) {
941
+ throw new Error(`failed to write sub-offset prefix frame`)
942
+ }
943
+ written += bytesWritten
944
+ }
945
+ fs.fsyncSync(fd)
946
+ } finally {
947
+ fs.closeSync(fd)
948
+ }
949
+
950
+ const parts = streamMeta.currentOffset.split(`_`).map(Number)
951
+ const readSeq = parts[0]!
952
+ const byteOffset = parts[1]!
953
+ // Match append()'s frame-inclusive offset advance (4-byte length
954
+ // prefix + payload + 1-byte newline) so reads with a capByte don't
955
+ // truncate the materialized prefix when later chained-fork resolves.
956
+ const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5
957
+ streamMeta.currentOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`
958
+ // Persist the user-supplied sub-offset verbatim for idempotent
959
+ // re-creation matching, not the encoded byte length.
960
+ streamMeta.forkSubOffset = options.forkSubOffset
961
+ }
881
962
  await this.db.put(key, streamMeta)
882
963
  } catch (err) {
883
964
  // Rollback source refcount on failure
@@ -893,7 +974,7 @@ export class FileBackedStreamStore {
893
974
  }
894
975
  }
895
976
  serverLog.error(
896
- `[FileBackedStreamStore] Error creating stream (LMDB put):`,
977
+ `[FileBackedStreamStore] Error creating stream before metadata commit:`,
897
978
  err
898
979
  )
899
980
  throw err
@@ -1589,6 +1670,57 @@ export class FileBackedStreamStore {
1589
1670
  return messages
1590
1671
  }
1591
1672
 
1673
+ /**
1674
+ * Resolve a fork sub-offset against the source: read the message that
1675
+ * starts at forkOffset and return prefix bytes to materialize as the
1676
+ * fork's first own message. For JSON, parses comma-joined values.
1677
+ */
1678
+ private resolveForkSubOffset(
1679
+ sourcePath: string,
1680
+ forkOffset: string,
1681
+ subOffset: number,
1682
+ isJSON: boolean
1683
+ ): Uint8Array {
1684
+ const forkByte = Number(forkOffset.split(`_`)[1] ?? 0)
1685
+ // Read source past forkOffset (cap at source's currentOffset by passing
1686
+ // a large cap; we only care about the first message past forkOffset).
1687
+ const sourceMeta = this.db.get(`stream:${sourcePath}`) as
1688
+ | StreamMetadata
1689
+ | undefined
1690
+ if (!sourceMeta) {
1691
+ throw new Error(`Source stream not found: ${sourcePath}`)
1692
+ }
1693
+ const currentByte = Number(sourceMeta.currentOffset.split(`_`)[1] ?? 0)
1694
+ const messages = this.readForkedMessages(sourcePath, forkByte, currentByte)
1695
+ if (messages.length === 0) {
1696
+ throw new Error(`Invalid fork sub-offset: no data past forkOffset`)
1697
+ }
1698
+ const first = messages[0]!
1699
+ if (isJSON) {
1700
+ const text = new TextDecoder().decode(first.data)
1701
+ const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text
1702
+ let values: Array<unknown>
1703
+ try {
1704
+ values = JSON.parse(`[${trimmed}]`)
1705
+ } catch {
1706
+ throw new Error(`Invalid fork sub-offset: source JSON is unparseable`)
1707
+ }
1708
+ if (subOffset > values.length) {
1709
+ throw new Error(
1710
+ `Invalid fork sub-offset: overshoots source message count`
1711
+ )
1712
+ }
1713
+ const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v))
1714
+ return new TextEncoder().encode(prefix.join(`,`) + `,`)
1715
+ }
1716
+ if (subOffset > first.data.length) {
1717
+ throw new Error(
1718
+ `Invalid fork sub-offset: overshoots source message length`
1719
+ )
1720
+ }
1721
+ return first.data.slice(0, subOffset)
1722
+ }
1723
+
1592
1724
  read(
1593
1725
  streamPath: string,
1594
1726
  offset?: string
package/src/server.ts CHANGED
@@ -42,6 +42,7 @@ const SSE_UP_TO_DATE_FIELD = `upToDate`
42
42
  // Fork headers (request headers only — not set on responses)
43
43
  const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`
44
44
  const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`
45
+ const STREAM_FORK_SUB_OFFSET_HEADER = `Stream-Fork-Sub-Offset`
45
46
 
46
47
  /**
47
48
  * Encode data for SSE format.
@@ -449,7 +450,7 @@ export class DurableStreamTestServer {
449
450
  )
450
451
  res.setHeader(
451
452
  `access-control-allow-headers`,
452
- `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset`
453
+ `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Stream-Closed, Producer-Id, Producer-Epoch, Producer-Seq, Stream-Forked-From, Stream-Fork-Offset, Stream-Fork-Sub-Offset`
453
454
  )
454
455
  res.setHeader(
455
456
  `access-control-expose-headers`,
@@ -598,6 +599,13 @@ export class DurableStreamTestServer {
598
599
  const forkOffsetHeader = req.headers[
599
600
  STREAM_FORK_OFFSET_HEADER.toLowerCase()
600
601
  ] as string | undefined
602
+ const forkSubOffsetHeaderRaw =
603
+ req.headers[STREAM_FORK_SUB_OFFSET_HEADER.toLowerCase()]
604
+ // Distinguish "header absent" from "header present but empty"
605
+ const forkSubOffsetHeaderPresent = forkSubOffsetHeaderRaw !== undefined
606
+ const forkSubOffsetHeader = Array.isArray(forkSubOffsetHeaderRaw)
607
+ ? forkSubOffsetHeaderRaw[0]
608
+ : forkSubOffsetHeaderRaw
601
609
 
602
610
  // Sanitize content-type: if empty or invalid, use default — but only
603
611
  // for non-fork creates. For forks, an omitted Content-Type means "inherit
@@ -667,6 +675,26 @@ export class DurableStreamTestServer {
667
675
  }
668
676
  }
669
677
 
678
+ // Validate sub-offset if header was present (including empty value)
679
+ let forkSubOffset: number | undefined
680
+ if (forkSubOffsetHeaderPresent) {
681
+ if (!forkedFromHeader) {
682
+ res.writeHead(400, { "content-type": `text/plain` })
683
+ res.end(`Stream-Fork-Sub-Offset requires Stream-Forked-From`)
684
+ return
685
+ }
686
+ const subOffsetPattern = /^(0|[1-9]\d*)$/
687
+ if (
688
+ forkSubOffsetHeader === undefined ||
689
+ !subOffsetPattern.test(forkSubOffsetHeader)
690
+ ) {
691
+ res.writeHead(400, { "content-type": `text/plain` })
692
+ res.end(`Invalid Stream-Fork-Sub-Offset format`)
693
+ return
694
+ }
695
+ forkSubOffset = parseInt(forkSubOffsetHeader, 10)
696
+ }
697
+
670
698
  // Read body if present
671
699
  const body = await this.readBody(req)
672
700
 
@@ -683,6 +711,7 @@ export class DurableStreamTestServer {
683
711
  closed: createClosed,
684
712
  forkedFrom: forkedFromHeader,
685
713
  forkOffset: forkOffsetHeader,
714
+ forkSubOffset,
686
715
  })
687
716
  )
688
717
  } catch (err) {
@@ -692,6 +721,11 @@ export class DurableStreamTestServer {
692
721
  res.end(`Source stream not found`)
693
722
  return
694
723
  }
724
+ if (err.message.includes(`Invalid fork sub-offset`)) {
725
+ res.writeHead(400, { "content-type": `text/plain` })
726
+ res.end(`Invalid fork sub-offset`)
727
+ return
728
+ }
695
729
  if (err.message.includes(`Invalid fork offset`)) {
696
730
  res.writeHead(400, { "content-type": `text/plain` })
697
731
  res.end(`Fork offset beyond source stream length`)
package/src/store.ts CHANGED
@@ -247,6 +247,7 @@ export class StreamStore {
247
247
  closed?: boolean
248
248
  forkedFrom?: string
249
249
  forkOffset?: string
250
+ forkSubOffset?: number
250
251
  } = {}
251
252
  ): Stream {
252
253
  // Check if stream already exists
@@ -280,6 +281,12 @@ export class StreamStore {
280
281
  const forkOffsetMatches =
281
282
  options.forkOffset === undefined ||
282
283
  options.forkOffset === existingRaw.forkOffset
284
+ // Sub-offset: undefined and 0 are equivalent. Compare the raw
285
+ // user-supplied integer (count for JSON, bytes for binary) so the
286
+ // comparison is independent of how it was resolved internally.
287
+ const requestedSub = options.forkSubOffset ?? 0
288
+ const existingSub = existingRaw.forkSubOffset ?? 0
289
+ const forkSubOffsetMatches = requestedSub === existingSub
283
290
 
284
291
  if (
285
292
  contentTypeMatches &&
@@ -287,7 +294,8 @@ export class StreamStore {
287
294
  expiresMatches &&
288
295
  closedMatches &&
289
296
  forkedFromMatches &&
290
- forkOffsetMatches
297
+ forkOffsetMatches &&
298
+ forkSubOffsetMatches
291
299
  ) {
292
300
  // Idempotent success - return existing stream
293
301
  return existingRaw
@@ -305,6 +313,7 @@ export class StreamStore {
305
313
  let forkOffset = `0000000000000000_0000000000000000`
306
314
  let sourceContentType: string | undefined
307
315
  let sourceStream: Stream | undefined
316
+ let forkSubOffsetPrefix: Uint8Array | undefined
308
317
 
309
318
  if (isFork) {
310
319
  sourceStream = this.streams.get(options.forkedFrom!)
@@ -320,6 +329,18 @@ export class StreamStore {
320
329
 
321
330
  sourceContentType = sourceStream.contentType
322
331
 
332
+ // Reject a content-type mismatch up front, before taking a reference on
333
+ // the source. Doing this after the refCount increment below would leak a
334
+ // reference on the failed fork and pin the source in a soft-deleted state.
335
+ if (
336
+ options.contentType &&
337
+ options.contentType.trim() !== `` &&
338
+ normalizeContentType(options.contentType) !==
339
+ normalizeContentType(sourceContentType)
340
+ ) {
341
+ throw new Error(`Content type mismatch with source stream`)
342
+ }
343
+
323
344
  // Resolve fork offset: use provided or source's currentOffset
324
345
  if (options.forkOffset) {
325
346
  forkOffset = options.forkOffset
@@ -333,22 +354,30 @@ export class StreamStore {
333
354
  throw new Error(`Invalid fork offset: ${forkOffset}`)
334
355
  }
335
356
 
357
+ // Resolve sub-offset against the source. Both binary and JSON return
358
+ // a synthetic prefix to materialize as the fork's first own message,
359
+ // because in this store one POST = one message regardless of mode.
360
+ if (options.forkSubOffset && options.forkSubOffset > 0) {
361
+ forkSubOffsetPrefix = this.resolveForkSubOffset(
362
+ sourceStream,
363
+ forkOffset,
364
+ options.forkSubOffset,
365
+ normalizeContentType(sourceContentType) === `application/json`
366
+ )
367
+ }
368
+
336
369
  // Increment source refcount
337
370
  sourceStream.refCount++
338
371
  }
339
372
 
340
- // Determine content type: use options, or inherit from source if fork
373
+ // Determine content type: use options, or inherit from source if fork. A
374
+ // fork content-type mismatch is already rejected above, before the source
375
+ // refCount is taken.
341
376
  let contentType = options.contentType
342
377
  if (!contentType || contentType.trim() === ``) {
343
378
  if (isFork) {
344
379
  contentType = sourceContentType
345
380
  }
346
- } else if (
347
- isFork &&
348
- normalizeContentType(contentType) !==
349
- normalizeContentType(sourceContentType)
350
- ) {
351
- throw new Error(`Content type mismatch with source stream`)
352
381
  }
353
382
 
354
383
  // Compute effective expiry for forks
@@ -375,6 +404,27 @@ export class StreamStore {
375
404
  forkOffset: isFork ? forkOffset : undefined,
376
405
  }
377
406
 
407
+ // Materialize sub-offset prefix as the fork's first own message.
408
+ if (forkSubOffsetPrefix && forkSubOffsetPrefix.length > 0) {
409
+ const parts = stream.currentOffset.split(`_`).map(Number)
410
+ const readSeq = parts[0]!
411
+ const byteOffset = parts[1]!
412
+ // Match append()'s frame-inclusive offset advance (4-byte length
413
+ // prefix + payload + 1-byte newline) so reads with a capByte don't
414
+ // truncate the materialized prefix when later chained-fork resolves.
415
+ const newByteOffset = byteOffset + forkSubOffsetPrefix.length + 5
416
+ const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`
417
+ stream.messages.push({
418
+ data: forkSubOffsetPrefix,
419
+ offset: newOffset,
420
+ timestamp: Date.now(),
421
+ })
422
+ stream.currentOffset = newOffset
423
+ // Persist the user-supplied sub-offset verbatim for idempotent
424
+ // re-creation matching, not the encoded byte length.
425
+ stream.forkSubOffset = options.forkSubOffset
426
+ }
427
+
378
428
  // If initial data is provided, append it
379
429
  if (options.initialData && options.initialData.length > 0) {
380
430
  try {
@@ -1236,6 +1286,67 @@ export class StreamStore {
1236
1286
  // Private helpers
1237
1287
  // ============================================================================
1238
1288
 
1289
+ /**
1290
+ * Resolve a sub-offset against a source stream and return the prefix bytes
1291
+ * to materialize as the fork's first own message. Reads from the source
1292
+ * (across its fork chain if any) starting at forkOffset; the first message
1293
+ * returned is the one that starts at forkOffset. Throws if the sub-offset
1294
+ * cannot be satisfied (no message past forkOffset, or overshoots its
1295
+ * content extent).
1296
+ */
1297
+ private resolveForkSubOffset(
1298
+ sourceStream: Stream,
1299
+ forkOffset: string,
1300
+ subOffset: number,
1301
+ isJSON: boolean
1302
+ ): Uint8Array {
1303
+ // Read source past forkOffset across its fork chain
1304
+ let sourceMessages: Array<StreamMessage>
1305
+ if (sourceStream.forkedFrom) {
1306
+ sourceMessages = [
1307
+ ...this.readForkedMessages(
1308
+ sourceStream.forkedFrom,
1309
+ forkOffset,
1310
+ sourceStream.forkOffset!
1311
+ ),
1312
+ ...this.readOwnMessages(sourceStream, forkOffset),
1313
+ ]
1314
+ } else {
1315
+ sourceMessages = this.readOwnMessages(sourceStream, forkOffset)
1316
+ }
1317
+ if (sourceMessages.length === 0) {
1318
+ throw new Error(`Invalid fork sub-offset: no data past forkOffset`)
1319
+ }
1320
+ const first = sourceMessages[0]!
1321
+ if (isJSON) {
1322
+ // The message data is comma-joined JSON values with a trailing comma
1323
+ // (e.g., `{"a":1},{"b":2},`). Wrap in [...] to parse, take first N
1324
+ // elements, re-encode in the same comma-joined format.
1325
+ const text = new TextDecoder().decode(first.data)
1326
+ const trimmed = text.endsWith(`,`) ? text.slice(0, -1) : text
1327
+ let values: Array<unknown>
1328
+ try {
1329
+ values = JSON.parse(`[${trimmed}]`)
1330
+ } catch {
1331
+ throw new Error(`Invalid fork sub-offset: source JSON is unparseable`)
1332
+ }
1333
+ if (subOffset > values.length) {
1334
+ throw new Error(
1335
+ `Invalid fork sub-offset: overshoots source message count`
1336
+ )
1337
+ }
1338
+ const prefix = values.slice(0, subOffset).map((v) => JSON.stringify(v))
1339
+ return new TextEncoder().encode(prefix.join(`,`) + `,`)
1340
+ }
1341
+ // Binary: take first subOffset bytes
1342
+ if (subOffset > first.data.length) {
1343
+ throw new Error(
1344
+ `Invalid fork sub-offset: overshoots source message length`
1345
+ )
1346
+ }
1347
+ return first.data.slice(0, subOffset)
1348
+ }
1349
+
1239
1350
  private appendToStream(
1240
1351
  stream: Stream,
1241
1352
  data: Uint8Array,
package/src/types.ts CHANGED
@@ -107,6 +107,14 @@ export interface Stream {
107
107
  */
108
108
  forkOffset?: string
109
109
 
110
+ /**
111
+ * User-supplied sub-offset value refining `forkOffset` (Section 4.2 of
112
+ * PROTOCOL.md). Stored verbatim for idempotent re-creation matching:
113
+ * bytes for non-JSON forks, flattened message count for JSON forks.
114
+ * `undefined` and `0` are equivalent.
115
+ */
116
+ forkSubOffset?: number
117
+
110
118
  /**
111
119
  * Number of forks referencing this stream.
112
120
  * Defaults to 0.