@durable-streams/server 0.1.4 → 0.1.6

package/dist/index.cjs

@@ -35,6 +35,10 @@ const __durable_streams_state = __toESM(require("@durable-streams/state"));
 
 //#region src/store.ts
 /**
+* TTL for in-memory producer state cleanup (7 days).
+*/
+const PRODUCER_STATE_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
+/**
 * Normalize content-type by extracting the media type (before any semicolon).
 * Handles cases like "application/json; charset=utf-8".
 */
@@ -81,13 +85,15 @@ function formatJsonResponse(data) {
 	const wrapped = `[${text}]`;
 	return new TextEncoder().encode(wrapped);
 }
-/**
-* In-memory store for durable streams.
-*/
 var StreamStore = class {
 	streams = new Map();
 	pendingLongPolls = [];
 	/**
+	* Per-producer locks for serializing validation+append operations.
+	* Key: "{streamPath}:{producerId}"
+	*/
+	producerLocks = new Map();
+	/**
 	* Check if a stream is expired based on TTL or Expires-At.
 	*/
 	isExpired(stream) {
@@ -163,6 +169,108 @@ var StreamStore = class {
 		return this.streams.delete(path);
 	}
 	/**
+	* Validate producer state WITHOUT mutating.
+	* Returns proposed state to commit after successful append.
+	* Implements Kafka-style idempotent producer validation.
+	*
+	* IMPORTANT: This function does NOT mutate producer state. The caller must
+	* call commitProducerState() after successful append to apply the mutation.
+	* This ensures atomicity: if append fails (e.g., JSON validation), producer
+	* state is not incorrectly advanced.
+	*/
+	validateProducer(stream, producerId, epoch, seq) {
+		if (!stream.producers) stream.producers = new Map();
+		this.cleanupExpiredProducers(stream);
+		const state = stream.producers.get(producerId);
+		const now = Date.now();
+		if (!state) {
+			if (seq !== 0) return {
+				status: `sequence_gap`,
+				expectedSeq: 0,
+				receivedSeq: seq
+			};
+			return {
+				status: `accepted`,
+				isNew: true,
+				producerId,
+				proposedState: {
+					epoch,
+					lastSeq: 0,
+					lastUpdated: now
+				}
+			};
+		}
+		if (epoch < state.epoch) return {
+			status: `stale_epoch`,
+			currentEpoch: state.epoch
+		};
+		if (epoch > state.epoch) {
+			if (seq !== 0) return { status: `invalid_epoch_seq` };
+			return {
+				status: `accepted`,
+				isNew: true,
+				producerId,
+				proposedState: {
+					epoch,
+					lastSeq: 0,
+					lastUpdated: now
+				}
+			};
+		}
+		if (seq <= state.lastSeq) return {
+			status: `duplicate`,
+			lastSeq: state.lastSeq
+		};
+		if (seq === state.lastSeq + 1) return {
+			status: `accepted`,
+			isNew: false,
+			producerId,
+			proposedState: {
+				epoch,
+				lastSeq: seq,
+				lastUpdated: now
+			}
+		};
+		return {
+			status: `sequence_gap`,
+			expectedSeq: state.lastSeq + 1,
+			receivedSeq: seq
+		};
+	}
+	/**
+	* Commit producer state after successful append.
+	* This is the only place where producer state is mutated.
+	*/
+	commitProducerState(stream, result) {
+		if (result.status !== `accepted`) return;
+		stream.producers.set(result.producerId, result.proposedState);
+	}
+	/**
+	* Clean up expired producer states from a stream.
+	*/
+	cleanupExpiredProducers(stream) {
+		if (!stream.producers) return;
+		const now = Date.now();
+		for (const [id, state] of stream.producers) if (now - state.lastUpdated > PRODUCER_STATE_TTL_MS) stream.producers.delete(id);
+	}
+	/**
+	* Acquire a lock for serialized producer operations.
+	* Returns a release function.
+	*/
+	async acquireProducerLock(path, producerId) {
+		const lockKey = `${path}:${producerId}`;
+		while (this.producerLocks.has(lockKey)) await this.producerLocks.get(lockKey);
+		let releaseLock;
+		const lockPromise = new Promise((resolve) => {
+			releaseLock = resolve;
+		});
+		this.producerLocks.set(lockKey, lockPromise);
+		return () => {
+			this.producerLocks.delete(lockKey);
+			releaseLock();
+		};
+	}
+	/**
 	* Append data to a stream.
 	* @throws Error if stream doesn't exist or is expired
 	* @throws Error if seq is lower than lastSeq
@@ -176,15 +284,56 @@ var StreamStore = class {
 			const streamType = normalizeContentType(stream.contentType);
 			if (providedType !== streamType) throw new Error(`Content-type mismatch: expected ${stream.contentType}, got ${options.contentType}`);
 		}
+		let producerResult;
+		if (options.producerId !== void 0 && options.producerEpoch !== void 0 && options.producerSeq !== void 0) {
+			producerResult = this.validateProducer(stream, options.producerId, options.producerEpoch, options.producerSeq);
+			if (producerResult.status !== `accepted`) return {
+				message: null,
+				producerResult
+			};
+		}
 		if (options.seq !== void 0) {
 			if (stream.lastSeq !== void 0 && options.seq <= stream.lastSeq) throw new Error(`Sequence conflict: ${options.seq} <= ${stream.lastSeq}`);
-			stream.lastSeq = options.seq;
 		}
 		const message = this.appendToStream(stream, data);
+		if (producerResult) this.commitProducerState(stream, producerResult);
+		if (options.seq !== void 0) stream.lastSeq = options.seq;
 		this.notifyLongPolls(path);
+		if (producerResult) return {
+			message,
+			producerResult
+		};
 		return message;
 	}
 	/**
+	* Append with producer serialization for concurrent request handling.
+	* This ensures that validation+append is atomic per producer.
+	*/
+	async appendWithProducer(path, data, options) {
+		if (!options.producerId) {
+			const result = this.append(path, data, options);
+			if (`message` in result) return result;
+			return { message: result };
+		}
+		const releaseLock = await this.acquireProducerLock(path, options.producerId);
+		try {
+			const result = this.append(path, data, options);
+			if (`message` in result) return result;
+			return { message: result };
+		} finally {
+			releaseLock();
+		}
+	}
+	/**
+	* Get the current epoch for a producer on a stream.
+	* Returns undefined if the producer doesn't exist or stream not found.
+	*/
+	getProducerEpoch(path, producerId) {
+		const stream = this.getIfNotExpired(path);
+		if (!stream?.producers) return void 0;
+		return stream.producers.get(producerId)?.epoch;
+	}
+	/**
 	* Read messages from a stream starting at the given offset.
 	* @throws Error if stream doesn't exist or is expired
 	*/
@@ -545,6 +694,11 @@ var FileBackedStreamStore = class {
 	fileHandlePool;
 	pendingLongPolls = [];
 	dataDir;
+	/**
+	* Per-producer locks for serializing validation+append operations.
+	* Key: "{streamPath}:{producerId}"
+	*/
+	producerLocks = new Map();
 	constructor(options) {
 		this.dataDir = options.dataDir;
 		this.db = (0, lmdb.open)({
@@ -626,6 +780,11 @@ var FileBackedStreamStore = class {
 	* Convert LMDB metadata to Stream object.
 	*/
 	streamMetaToStream(meta) {
+		let producers;
+		if (meta.producers) {
+			producers = new Map();
+			for (const [id, state] of Object.entries(meta.producers)) producers.set(id, { ...state });
+		}
 		return {
 			path: meta.path,
 			contentType: meta.contentType,
@@ -634,10 +793,103 @@ var FileBackedStreamStore = class {
 			lastSeq: meta.lastSeq,
 			ttlSeconds: meta.ttlSeconds,
 			expiresAt: meta.expiresAt,
-			createdAt: meta.createdAt
+			createdAt: meta.createdAt,
+			producers
 		};
 	}
 	/**
+	* Validate producer state WITHOUT mutating.
+	* Returns proposed state to commit after successful append.
+	*
+	* IMPORTANT: This function does NOT mutate producer state. The caller must
+	* commit the proposedState after successful append (file write + fsync + LMDB).
+	* This ensures atomicity: if any step fails, producer state is not advanced.
+	*/
+	validateProducer(meta, producerId, epoch, seq) {
+		if (!meta.producers) meta.producers = {};
+		const state = meta.producers[producerId];
+		const now = Date.now();
+		if (!state) {
+			if (seq !== 0) return {
+				status: `sequence_gap`,
+				expectedSeq: 0,
+				receivedSeq: seq
+			};
+			return {
+				status: `accepted`,
+				isNew: true,
+				producerId,
+				proposedState: {
+					epoch,
+					lastSeq: 0,
+					lastUpdated: now
+				}
+			};
+		}
+		if (epoch < state.epoch) return {
+			status: `stale_epoch`,
+			currentEpoch: state.epoch
+		};
+		if (epoch > state.epoch) {
+			if (seq !== 0) return { status: `invalid_epoch_seq` };
+			return {
+				status: `accepted`,
+				isNew: true,
+				producerId,
+				proposedState: {
+					epoch,
+					lastSeq: 0,
+					lastUpdated: now
+				}
+			};
+		}
+		if (seq <= state.lastSeq) return {
+			status: `duplicate`,
+			lastSeq: state.lastSeq
+		};
+		if (seq === state.lastSeq + 1) return {
+			status: `accepted`,
+			isNew: false,
+			producerId,
+			proposedState: {
+				epoch,
+				lastSeq: seq,
+				lastUpdated: now
+			}
+		};
+		return {
+			status: `sequence_gap`,
+			expectedSeq: state.lastSeq + 1,
+			receivedSeq: seq
+		};
+	}
+	/**
+	* Acquire a lock for serialized producer operations.
+	* Returns a release function.
+	*/
+	async acquireProducerLock(streamPath, producerId) {
+		const lockKey = `${streamPath}:${producerId}`;
+		while (this.producerLocks.has(lockKey)) await this.producerLocks.get(lockKey);
+		let releaseLock;
+		const lockPromise = new Promise((resolve) => {
+			releaseLock = resolve;
+		});
+		this.producerLocks.set(lockKey, lockPromise);
+		return () => {
+			this.producerLocks.delete(lockKey);
+			releaseLock();
+		};
+	}
+	/**
+	* Get the current epoch for a producer on a stream.
+	* Returns undefined if the producer doesn't exist or stream not found.
+	*/
+	getProducerEpoch(streamPath, producerId) {
+		const meta = this.getMetaIfNotExpired(streamPath);
+		if (!meta?.producers) return void 0;
+		return meta.producers[producerId]?.epoch;
+	}
+	/**
 	* Check if a stream is expired based on TTL or Expires-At.
 	*/
 	isExpired(meta) {
@@ -747,6 +999,14 @@ var FileBackedStreamStore = class {
 			const streamType = normalizeContentType(streamMeta.contentType);
 			if (providedType !== streamType) throw new Error(`Content-type mismatch: expected ${streamMeta.contentType}, got ${options.contentType}`);
 		}
+		let producerResult;
+		if (options.producerId !== void 0 && options.producerEpoch !== void 0 && options.producerSeq !== void 0) {
+			producerResult = this.validateProducer(streamMeta, options.producerId, options.producerEpoch, options.producerSeq);
+			if (producerResult.status !== `accepted`) return {
+				message: null,
+				producerResult
+			};
+		}
 		if (options.seq !== void 0) {
 			if (streamMeta.lastSeq !== void 0 && options.seq <= streamMeta.lastSeq) throw new Error(`Sequence conflict: ${options.seq} <= ${streamMeta.lastSeq}`);
 		}
@@ -782,17 +1042,43 @@ var FileBackedStreamStore = class {
 			timestamp: Date.now()
 		};
 		await this.fileHandlePool.fsyncFile(segmentPath);
+		const updatedProducers = { ...streamMeta.producers };
+		if (producerResult && producerResult.status === `accepted`) updatedProducers[producerResult.producerId] = producerResult.proposedState;
 		const updatedMeta = {
 			...streamMeta,
 			currentOffset: newOffset,
 			lastSeq: options.seq ?? streamMeta.lastSeq,
-			totalBytes: streamMeta.totalBytes + processedData.length + 5
+			totalBytes: streamMeta.totalBytes + processedData.length + 5,
+			producers: updatedProducers
 		};
 		const key = `stream:${streamPath}`;
 		this.db.putSync(key, updatedMeta);
 		this.notifyLongPolls(streamPath);
+		if (producerResult) return {
+			message,
+			producerResult
+		};
 		return message;
 	}
+	/**
+	* Append with producer serialization for concurrent request handling.
+	* This ensures that validation+append is atomic per producer.
+	*/
+	async appendWithProducer(streamPath, data, options) {
+		if (!options.producerId) {
+			const result = await this.append(streamPath, data, options);
+			if (result && `message` in result) return result;
+			return { message: result };
+		}
+		const releaseLock = await this.acquireProducerLock(streamPath, options.producerId);
+		try {
+			const result = await this.append(streamPath, data, options);
+			if (result && `message` in result) return result;
+			return { message: result };
+		} finally {
+			releaseLock();
+		}
+	}
 	read(streamPath, offset) {
 		const streamMeta = this.getMetaIfNotExpired(streamPath);
 		if (!streamMeta) throw new Error(`Stream not found: ${streamPath}`);
@@ -1066,6 +1352,11 @@ const STREAM_UP_TO_DATE_HEADER = `Stream-Up-To-Date`;
 const STREAM_SEQ_HEADER = `Stream-Seq`;
 const STREAM_TTL_HEADER = `Stream-TTL`;
 const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`;
+const PRODUCER_ID_HEADER = `Producer-Id`;
+const PRODUCER_EPOCH_HEADER = `Producer-Epoch`;
+const PRODUCER_SEQ_HEADER = `Producer-Seq`;
+const PRODUCER_EXPECTED_SEQ_HEADER = `Producer-Expected-Seq`;
+const PRODUCER_RECEIVED_SEQ_HEADER = `Producer-Received-Seq`;
 const SSE_OFFSET_FIELD = `streamNextOffset`;
 const SSE_CURSOR_FIELD = `streamCursor`;
 const SSE_UP_TO_DATE_FIELD = `upToDate`;
@@ -1277,8 +1568,8 @@ var DurableStreamTestServer = class {
 		const method = req.method?.toUpperCase();
 		res.setHeader(`access-control-allow-origin`, `*`);
 		res.setHeader(`access-control-allow-methods`, `GET, POST, PUT, DELETE, HEAD, OPTIONS`);
-		res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At`);
-		res.setHeader(`access-control-expose-headers`, `Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Date, etag, content-type, content-encoding, vary`);
+		res.setHeader(`access-control-allow-headers`, `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Producer-Id, Producer-Epoch, Producer-Seq`);
+		res.setHeader(`access-control-expose-headers`, `Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Date, Producer-Epoch, Producer-Seq, Producer-Expected-Seq, Producer-Received-Seq, etag, content-type, content-encoding, vary`);
 		res.setHeader(`x-content-type-options`, `nosniff`);
 		res.setHeader(`cross-origin-resource-policy`, `cross-origin`);
 		if (method === `OPTIONS`) {
@@ -1453,7 +1744,7 @@ var DurableStreamTestServer = class {
 				res.end(`Multiple offset parameters not allowed`);
 				return;
 			}
-			const validOffsetPattern = /^(-1|\d+_\d+)$/;
+			const validOffsetPattern = /^(-1|now|\d+_\d+)$/;
 			if (!validOffsetPattern.test(offset)) {
 				res.writeHead(400, { "content-type": `text/plain` });
 				res.end(`Invalid offset format`);
@@ -1466,17 +1757,32 @@ var DurableStreamTestServer = class {
 			return;
 		}
 		if (live === `sse`) {
-			await this.handleSSE(path, stream, offset, cursor, res);
+			const sseOffset = offset === `now` ? stream.currentOffset : offset;
+			await this.handleSSE(path, stream, sseOffset, cursor, res);
 			return;
 		}
-		let { messages, upToDate } = this.store.read(path, offset);
-		const clientIsCaughtUp = offset && offset === stream.currentOffset;
+		const effectiveOffset = offset === `now` ? stream.currentOffset : offset;
+		if (offset === `now` && live !== `long-poll`) {
+			const headers$1 = {
+				[STREAM_OFFSET_HEADER]: stream.currentOffset,
+				[STREAM_UP_TO_DATE_HEADER]: `true`,
+				[`cache-control`]: `no-store`
+			};
+			if (stream.contentType) headers$1[`content-type`] = stream.contentType;
+			const isJsonMode = stream.contentType?.includes(`application/json`);
+			const responseBody = isJsonMode ? `[]` : ``;
+			res.writeHead(200, headers$1);
+			res.end(responseBody);
+			return;
+		}
+		let { messages, upToDate } = this.store.read(path, effectiveOffset);
+		const clientIsCaughtUp = effectiveOffset && effectiveOffset === stream.currentOffset || offset === `now`;
 		if (live === `long-poll` && clientIsCaughtUp && messages.length === 0) {
-			const result = await this.store.waitForMessages(path, offset, this.options.longPollTimeout);
+			const result = await this.store.waitForMessages(path, effectiveOffset ?? stream.currentOffset, this.options.longPollTimeout);
 			if (result.timedOut) {
 				const responseCursor = generateResponseCursor(cursor, this.options.cursorOptions);
 				res.writeHead(204, {
-					[STREAM_OFFSET_HEADER]: offset,
+					[STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
 					[STREAM_UP_TO_DATE_HEADER]: `true`,
 					[STREAM_CURSOR_HEADER]: responseCursor
 				});
@@ -1584,6 +1890,9 @@ var DurableStreamTestServer = class {
 	async handleAppend(path, req, res) {
 		const contentType = req.headers[`content-type`];
 		const seq = req.headers[STREAM_SEQ_HEADER.toLowerCase()];
+		const producerId = req.headers[PRODUCER_ID_HEADER.toLowerCase()];
+		const producerEpochStr = req.headers[PRODUCER_EPOCH_HEADER.toLowerCase()];
+		const producerSeqStr = req.headers[PRODUCER_SEQ_HEADER.toLowerCase()];
 		const body = await this.readBody(req);
 		if (body.length === 0) {
 			res.writeHead(400, { "content-type": `text/plain` });
@@ -1595,10 +1904,96 @@ var DurableStreamTestServer = class {
 			res.end(`Content-Type header is required`);
 			return;
 		}
-		const message = await Promise.resolve(this.store.append(path, body, {
+		const hasProducerHeaders = producerId !== void 0 || producerEpochStr !== void 0 || producerSeqStr !== void 0;
+		const hasAllProducerHeaders = producerId !== void 0 && producerEpochStr !== void 0 && producerSeqStr !== void 0;
+		if (hasProducerHeaders && !hasAllProducerHeaders) {
+			res.writeHead(400, { "content-type": `text/plain` });
+			res.end(`All producer headers (Producer-Id, Producer-Epoch, Producer-Seq) must be provided together`);
+			return;
+		}
+		if (hasAllProducerHeaders && producerId === ``) {
+			res.writeHead(400, { "content-type": `text/plain` });
+			res.end(`Invalid Producer-Id: must not be empty`);
+			return;
+		}
+		const STRICT_INTEGER_REGEX = /^\d+$/;
+		let producerEpoch;
+		let producerSeq;
+		if (hasAllProducerHeaders) {
+			if (!STRICT_INTEGER_REGEX.test(producerEpochStr)) {
+				res.writeHead(400, { "content-type": `text/plain` });
+				res.end(`Invalid Producer-Epoch: must be a non-negative integer`);
+				return;
+			}
+			producerEpoch = Number(producerEpochStr);
+			if (!Number.isSafeInteger(producerEpoch)) {
+				res.writeHead(400, { "content-type": `text/plain` });
+				res.end(`Invalid Producer-Epoch: must be a non-negative integer`);
+				return;
+			}
+			if (!STRICT_INTEGER_REGEX.test(producerSeqStr)) {
+				res.writeHead(400, { "content-type": `text/plain` });
+				res.end(`Invalid Producer-Seq: must be a non-negative integer`);
+				return;
+			}
+			producerSeq = Number(producerSeqStr);
+			if (!Number.isSafeInteger(producerSeq)) {
+				res.writeHead(400, { "content-type": `text/plain` });
+				res.end(`Invalid Producer-Seq: must be a non-negative integer`);
+				return;
+			}
+		}
+		const appendOptions = {
 			seq,
-			contentType
-		}));
+			contentType,
+			producerId,
+			producerEpoch,
+			producerSeq
+		};
+		let result;
+		if (producerId !== void 0) result = await this.store.appendWithProducer(path, body, appendOptions);
+		else result = await Promise.resolve(this.store.append(path, body, appendOptions));
+		if (result && typeof result === `object` && `producerResult` in result) {
+			const { message: message$1, producerResult } = result;
+			if (!producerResult || producerResult.status === `accepted`) {
+				const responseHeaders = { [STREAM_OFFSET_HEADER]: message$1.offset };
+				if (producerEpoch !== void 0) responseHeaders[PRODUCER_EPOCH_HEADER] = producerEpoch.toString();
+				if (producerSeq !== void 0) responseHeaders[PRODUCER_SEQ_HEADER] = producerSeq.toString();
+				res.writeHead(200, responseHeaders);
+				res.end();
+				return;
+			}
+			switch (producerResult.status) {
+				case `duplicate`:
+					res.writeHead(204, {
+						[PRODUCER_EPOCH_HEADER]: producerEpoch.toString(),
+						[PRODUCER_SEQ_HEADER]: producerResult.lastSeq.toString()
+					});
+					res.end();
+					return;
+				case `stale_epoch`: {
+					res.writeHead(403, {
+						"content-type": `text/plain`,
+						[PRODUCER_EPOCH_HEADER]: producerResult.currentEpoch.toString()
+					});
+					res.end(`Stale producer epoch`);
+					return;
+				}
+				case `invalid_epoch_seq`:
+					res.writeHead(400, { "content-type": `text/plain` });
+					res.end(`New epoch must start with sequence 0`);
+					return;
+				case `sequence_gap`:
+					res.writeHead(409, {
+						"content-type": `text/plain`,
+						[PRODUCER_EXPECTED_SEQ_HEADER]: producerResult.expectedSeq.toString(),
+						[PRODUCER_RECEIVED_SEQ_HEADER]: producerResult.receivedSeq.toString()
+					});
+					res.end(`Producer sequence gap`);
+					return;
+			}
+		}
+		const message = result;
 		res.writeHead(204, { [STREAM_OFFSET_HEADER]: message.offset });
 		res.end();
 	}