@durable-streams/client 0.1.3 → 0.1.4

package/dist/index.js

@@ -35,6 +35,28 @@ const STREAM_TTL_HEADER = `Stream-TTL`;
 */
 const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`;
 /**
+* Request header for producer ID (client-supplied stable identifier).
+*/
+const PRODUCER_ID_HEADER = `Producer-Id`;
+/**
+* Request/response header for producer epoch.
+* Client-declared, server-validated monotonically increasing.
+*/
+const PRODUCER_EPOCH_HEADER = `Producer-Epoch`;
+/**
+* Request header for producer sequence number.
+* Monotonically increasing per epoch, per-batch (not per-message).
+*/
+const PRODUCER_SEQ_HEADER = `Producer-Seq`;
+/**
+* Response header indicating expected sequence number on 409 Conflict.
+*/
+const PRODUCER_EXPECTED_SEQ_HEADER = `Producer-Expected-Seq`;
+/**
+* Response header indicating received sequence number on 409 Conflict.
+*/
+const PRODUCER_RECEIVED_SEQ_HEADER = `Producer-Received-Seq`;
+/**
 * Query parameter for starting offset.
 */
 const OFFSET_QUERY_PARAM = `offset`;
@@ -1382,7 +1404,7 @@ async function streamInternal(options) {
 * Normalize content-type by extracting the media type (before any semicolon).
 * Handles cases like "application/json; charset=utf-8".
 */
-function normalizeContentType(contentType) {
+function normalizeContentType$1(contentType) {
 	if (!contentType) return ``;
 	return contentType.split(`;`)[0].trim().toLowerCase();
 }
@@ -1448,6 +1470,7 @@ var DurableStream = class DurableStream {
 			url: urlStr
 		};
 		this.#onError = opts.onError;
+		if (opts.contentType) this.contentType = opts.contentType;
 		this.#batchingEnabled = opts.batching !== false;
 		if (this.#batchingEnabled) this.#queue = fastq.promise(this.#batchWorker.bind(this), 1);
 		const baseFetchClient = opts.fetch ?? ((...args) => fetch(...args));
@@ -1596,7 +1619,7 @@ var DurableStream = class DurableStream {
 		const contentType = opts?.contentType ?? this.#options.contentType ?? this.contentType;
 		if (contentType) requestHeaders[`content-type`] = contentType;
 		if (opts?.seq) requestHeaders[STREAM_SEQ_HEADER] = opts.seq;
-		const isJson = normalizeContentType(contentType) === `application/json`;
+		const isJson = normalizeContentType$1(contentType) === `application/json`;
 		const bodyToEncode = isJson ? [body] : body;
 		const encodedBody = encodeBody(bodyToEncode);
 		const response = await this.#fetchClient(fetchUrl.toString(), {
@@ -1662,7 +1685,7 @@ var DurableStream = class DurableStream {
 			break;
 		}
 		if (highestSeq) requestHeaders[STREAM_SEQ_HEADER] = highestSeq;
-		const isJson = normalizeContentType(contentType) === `application/json`;
+		const isJson = normalizeContentType$1(contentType) === `application/json`;
 		let batchedBody;
 		if (isJson) {
 			const values = batch.map((m) => m.data);
@@ -1902,4 +1925,401 @@ function validateOptions(options) {
 }
 
 //#endregion
-export { BackoffDefaults, CURSOR_QUERY_PARAM, DURABLE_STREAM_PROTOCOL_QUERY_PARAMS, DurableStream, DurableStreamError, FetchBackoffAbortError, FetchError, InvalidSignalError, LIVE_QUERY_PARAM, MissingStreamUrlError, OFFSET_QUERY_PARAM, SSE_COMPATIBLE_CONTENT_TYPES, STREAM_CURSOR_HEADER, STREAM_EXPIRES_AT_HEADER, STREAM_OFFSET_HEADER, STREAM_SEQ_HEADER, STREAM_TTL_HEADER, STREAM_UP_TO_DATE_HEADER, _resetHttpWarningForTesting, asAsyncIterableReadableStream, createFetchWithBackoff, createFetchWithConsumedBody, stream, warnIfUsingHttpInBrowser };
+//#region src/idempotent-producer.ts
+/**
+* Error thrown when a producer's epoch is stale (zombie fencing).
+*/
+var StaleEpochError = class extends Error {
+	/**
+	* The current epoch on the server.
+	*/
+	currentEpoch;
+	constructor(currentEpoch) {
+		super(`Producer epoch is stale. Current server epoch: ${currentEpoch}. Call restart() or create a new producer with a higher epoch.`);
+		this.name = `StaleEpochError`;
+		this.currentEpoch = currentEpoch;
+	}
+};
+/**
+* Error thrown when an unrecoverable sequence gap is detected.
+*
+* With maxInFlight > 1, HTTP requests can arrive out of order at the server,
+* causing temporary 409 responses. The client automatically handles these
+* by waiting for earlier sequences to complete, then retrying.
+*
+* This error is only thrown when the gap cannot be resolved (e.g., the
+* expected sequence is >= our sequence, indicating a true protocol violation).
+*/
+var SequenceGapError = class extends Error {
+	expectedSeq;
+	receivedSeq;
+	constructor(expectedSeq, receivedSeq) {
+		super(`Producer sequence gap: expected ${expectedSeq}, received ${receivedSeq}`);
+		this.name = `SequenceGapError`;
+		this.expectedSeq = expectedSeq;
+		this.receivedSeq = receivedSeq;
+	}
+};
+/**
+* Normalize content-type by extracting the media type (before any semicolon).
+*/
+function normalizeContentType(contentType) {
+	if (!contentType) return ``;
+	return contentType.split(`;`)[0].trim().toLowerCase();
+}
+/**
+* An idempotent producer for exactly-once writes to a durable stream.
+*
+* Features:
+* - Fire-and-forget: append() returns immediately, batches in background
+* - Exactly-once: server deduplicates using (producerId, epoch, seq)
+* - Batching: multiple appends batched into single HTTP request
+* - Pipelining: up to maxInFlight concurrent batches
+* - Zombie fencing: stale producers rejected via epoch validation
+*
+* @example
+* ```typescript
+* const stream = new DurableStream({ url: "https://..." });
+* const producer = new IdempotentProducer(stream, "order-service-1", {
+*   epoch: 0,
+*   autoClaim: true,
+* });
+*
+* // Fire-and-forget writes (synchronous, returns immediately)
+* producer.append("message 1");
+* producer.append("message 2");
+*
+* // Ensure all messages are delivered before shutdown
+* await producer.flush();
+* await producer.close();
+* ```
+*/
+var IdempotentProducer = class {
+	#stream;
+	#producerId;
+	#epoch;
+	#nextSeq = 0;
+	#autoClaim;
+	#maxBatchBytes;
+	#lingerMs;
+	#fetchClient;
+	#signal;
+	#onError;
+	#pendingBatch = [];
+	#batchBytes = 0;
+	#lingerTimeout = null;
+	#queue;
+	#maxInFlight;
+	#closed = false;
+	#epochClaimed;
+	#seqState = new Map();
+	/**
+	* Create an idempotent producer for a stream.
+	*
+	* @param stream - The DurableStream to write to
+	* @param producerId - Stable identifier for this producer (e.g., "order-service-1")
+	* @param opts - Producer options
+	*/
+	constructor(stream$1, producerId, opts) {
+		this.#stream = stream$1;
+		this.#producerId = producerId;
+		this.#epoch = opts?.epoch ?? 0;
+		this.#autoClaim = opts?.autoClaim ?? false;
+		this.#maxBatchBytes = opts?.maxBatchBytes ?? 1024 * 1024;
+		this.#lingerMs = opts?.lingerMs ?? 5;
+		this.#signal = opts?.signal;
+		this.#onError = opts?.onError;
+		this.#fetchClient = opts?.fetch ?? ((...args) => fetch(...args));
+		this.#maxInFlight = opts?.maxInFlight ?? 5;
+		this.#epochClaimed = !this.#autoClaim;
+		this.#queue = fastq.promise(this.#batchWorker.bind(this), this.#maxInFlight);
+		if (this.#signal) this.#signal.addEventListener(`abort`, () => {
+			this.#rejectPendingBatch(new DurableStreamError(`Producer aborted`, `ALREADY_CLOSED`, void 0, void 0));
+		}, { once: true });
+	}
+	/**
+	* Append data to the stream.
+	*
+	* This is fire-and-forget: returns immediately after adding to the batch.
+	* The message is batched and sent when:
+	* - maxBatchBytes is reached
+	* - lingerMs elapses
+	* - flush() is called
+	*
+	* Errors are reported via onError callback if configured. Use flush() to
+	* wait for all pending messages to be sent.
+	*
+	* For JSON streams, pass native objects (which will be serialized internally).
+	* For byte streams, pass string or Uint8Array.
+	*
+	* @param body - Data to append (object for JSON streams, string or Uint8Array for byte streams)
+	*/
+	append(body) {
+		if (this.#closed) throw new DurableStreamError(`Producer is closed`, `ALREADY_CLOSED`, void 0, void 0);
+		const isJson = normalizeContentType(this.#stream.contentType) === `application/json`;
+		let bytes;
+		let data;
+		if (isJson) {
+			const json = JSON.stringify(body);
+			bytes = new TextEncoder().encode(json);
+			data = body;
+		} else {
+			if (typeof body === `string`) bytes = new TextEncoder().encode(body);
+			else if (body instanceof Uint8Array) bytes = body;
+			else throw new DurableStreamError(`Non-JSON streams require string or Uint8Array`, `BAD_REQUEST`, 400, void 0);
+			data = bytes;
+		}
+		this.#pendingBatch.push({
+			data,
+			body: bytes
+		});
+		this.#batchBytes += bytes.length;
+		if (this.#batchBytes >= this.#maxBatchBytes) this.#enqueuePendingBatch();
+		else if (!this.#lingerTimeout) this.#lingerTimeout = setTimeout(() => {
+			this.#lingerTimeout = null;
+			if (this.#pendingBatch.length > 0) this.#enqueuePendingBatch();
+		}, this.#lingerMs);
+	}
+	/**
+	* Send any pending batch immediately and wait for all in-flight batches.
+	*
+	* Call this before shutdown to ensure all messages are delivered.
+	*/
+	async flush() {
+		if (this.#lingerTimeout) {
+			clearTimeout(this.#lingerTimeout);
+			this.#lingerTimeout = null;
+		}
+		if (this.#pendingBatch.length > 0) this.#enqueuePendingBatch();
+		await this.#queue.drained();
+	}
+	/**
+	* Flush pending messages and close the producer.
+	*
+	* After calling close(), further append() calls will throw.
+	*/
+	async close() {
+		if (this.#closed) return;
+		this.#closed = true;
+		try {
+			await this.flush();
+		} catch {}
+	}
+	/**
+	* Increment epoch and reset sequence.
+	*
+	* Call this when restarting the producer to establish a new session.
+	* Flushes any pending messages first.
+	*/
+	async restart() {
+		await this.flush();
+		this.#epoch++;
+		this.#nextSeq = 0;
+	}
+	/**
+	* Current epoch for this producer.
+	*/
+	get epoch() {
+		return this.#epoch;
+	}
+	/**
+	* Next sequence number to be assigned.
+	*/
+	get nextSeq() {
+		return this.#nextSeq;
+	}
+	/**
+	* Number of messages in the current pending batch.
+	*/
+	get pendingCount() {
+		return this.#pendingBatch.length;
+	}
+	/**
+	* Number of batches currently in flight.
+	*/
+	get inFlightCount() {
+		return this.#queue.length();
+	}
+	/**
+	* Enqueue the current pending batch for processing.
+	*/
+	#enqueuePendingBatch() {
+		if (this.#pendingBatch.length === 0) return;
+		const batch = this.#pendingBatch;
+		const seq = this.#nextSeq;
+		this.#pendingBatch = [];
+		this.#batchBytes = 0;
+		this.#nextSeq++;
+		if (this.#autoClaim && !this.#epochClaimed && this.#queue.length() > 0) this.#queue.drained().then(() => {
+			this.#queue.push({
+				batch,
+				seq
+			}).catch(() => {});
+		});
+		else this.#queue.push({
+			batch,
+			seq
+		}).catch(() => {});
+	}
+	/**
+	* Batch worker - processes batches via fastq.
+	*/
+	async #batchWorker(task) {
+		const { batch, seq } = task;
+		const epoch = this.#epoch;
+		try {
+			await this.#doSendBatch(batch, seq, epoch);
+			if (!this.#epochClaimed) this.#epochClaimed = true;
+			this.#signalSeqComplete(epoch, seq, void 0);
+		} catch (error) {
+			this.#signalSeqComplete(epoch, seq, error);
+			if (this.#onError) this.#onError(error);
+			throw error;
+		}
+	}
+	/**
+	* Signal that a sequence has completed (success or failure).
+	*/
+	#signalSeqComplete(epoch, seq, error) {
+		let epochMap = this.#seqState.get(epoch);
+		if (!epochMap) {
+			epochMap = new Map();
+			this.#seqState.set(epoch, epochMap);
+		}
+		const state = epochMap.get(seq);
+		if (state) {
+			state.resolved = true;
+			state.error = error;
+			for (const waiter of state.waiters) waiter(error);
+			state.waiters = [];
+		} else epochMap.set(seq, {
+			resolved: true,
+			error,
+			waiters: []
+		});
+		const cleanupThreshold = seq - this.#maxInFlight * 3;
+		if (cleanupThreshold > 0) {
+			for (const oldSeq of epochMap.keys()) if (oldSeq < cleanupThreshold) epochMap.delete(oldSeq);
+		}
+	}
+	/**
+	* Wait for a specific sequence to complete.
+	* Returns immediately if already completed.
+	* Throws if the sequence failed.
+	*/
+	#waitForSeq(epoch, seq) {
+		let epochMap = this.#seqState.get(epoch);
+		if (!epochMap) {
+			epochMap = new Map();
+			this.#seqState.set(epoch, epochMap);
+		}
+		const state = epochMap.get(seq);
+		if (state?.resolved) {
+			if (state.error) return Promise.reject(state.error);
+			return Promise.resolve();
+		}
+		return new Promise((resolve, reject) => {
+			const waiter = (err) => {
+				if (err) reject(err);
+				else resolve();
+			};
+			if (state) state.waiters.push(waiter);
+			else epochMap.set(seq, {
+				resolved: false,
+				waiters: [waiter]
+			});
+		});
+	}
+	/**
+	* Actually send the batch to the server.
+	* Handles auto-claim retry on 403 (stale epoch) if autoClaim is enabled.
+	* Does NOT implement general retry/backoff for network errors or 5xx responses.
+	*/
+	async #doSendBatch(batch, seq, epoch) {
+		const contentType = this.#stream.contentType ?? `application/octet-stream`;
+		const isJson = normalizeContentType(contentType) === `application/json`;
+		let batchedBody;
+		if (isJson) {
+			const values = batch.map((e) => e.data);
+			batchedBody = JSON.stringify(values);
+		} else {
+			const totalSize = batch.reduce((sum, e) => sum + e.body.length, 0);
+			const concatenated = new Uint8Array(totalSize);
+			let offset = 0;
+			for (const entry of batch) {
+				concatenated.set(entry.body, offset);
+				offset += entry.body.length;
+			}
+			batchedBody = concatenated;
+		}
+		const url = this.#stream.url;
+		const headers = {
+			"content-type": contentType,
+			[PRODUCER_ID_HEADER]: this.#producerId,
+			[PRODUCER_EPOCH_HEADER]: epoch.toString(),
+			[PRODUCER_SEQ_HEADER]: seq.toString()
+		};
+		const response = await this.#fetchClient(url, {
+			method: `POST`,
+			headers,
+			body: batchedBody,
+			signal: this.#signal
+		});
+		if (response.status === 204) return {
+			offset: ``,
+			duplicate: true
+		};
+		if (response.status === 200) {
+			const resultOffset = response.headers.get(STREAM_OFFSET_HEADER) ?? ``;
+			return {
+				offset: resultOffset,
+				duplicate: false
+			};
+		}
+		if (response.status === 403) {
+			const currentEpochStr = response.headers.get(PRODUCER_EPOCH_HEADER);
+			const currentEpoch = currentEpochStr ? parseInt(currentEpochStr, 10) : epoch;
+			if (this.#autoClaim) {
+				const newEpoch = currentEpoch + 1;
+				this.#epoch = newEpoch;
+				this.#nextSeq = 1;
+				return this.#doSendBatch(batch, 0, newEpoch);
+			}
+			throw new StaleEpochError(currentEpoch);
+		}
+		if (response.status === 409) {
+			const expectedSeqStr = response.headers.get(PRODUCER_EXPECTED_SEQ_HEADER);
+			const expectedSeq = expectedSeqStr ? parseInt(expectedSeqStr, 10) : 0;
+			if (expectedSeq < seq) {
+				const waitPromises = [];
+				for (let s = expectedSeq; s < seq; s++) waitPromises.push(this.#waitForSeq(epoch, s));
+				await Promise.all(waitPromises);
+				return this.#doSendBatch(batch, seq, epoch);
+			}
+			const receivedSeqStr = response.headers.get(PRODUCER_RECEIVED_SEQ_HEADER);
+			const receivedSeq = receivedSeqStr ? parseInt(receivedSeqStr, 10) : seq;
+			throw new SequenceGapError(expectedSeq, receivedSeq);
+		}
+		if (response.status === 400) {
+			const error$1 = await DurableStreamError.fromResponse(response, url);
+			throw error$1;
+		}
+		const error = await FetchError.fromResponse(response, url);
+		throw error;
+	}
+	/**
+	* Clear pending batch and report error.
+	*/
+	#rejectPendingBatch(error) {
+		if (this.#onError && this.#pendingBatch.length > 0) this.#onError(error);
+		this.#pendingBatch = [];
+		this.#batchBytes = 0;
+		if (this.#lingerTimeout) {
+			clearTimeout(this.#lingerTimeout);
+			this.#lingerTimeout = null;
+		}
+	}
+};
+
+//#endregion
+export { BackoffDefaults, CURSOR_QUERY_PARAM, DURABLE_STREAM_PROTOCOL_QUERY_PARAMS, DurableStream, DurableStreamError, FetchBackoffAbortError, FetchError, IdempotentProducer, InvalidSignalError, LIVE_QUERY_PARAM, MissingStreamUrlError, OFFSET_QUERY_PARAM, PRODUCER_EPOCH_HEADER, PRODUCER_EXPECTED_SEQ_HEADER, PRODUCER_ID_HEADER, PRODUCER_RECEIVED_SEQ_HEADER, PRODUCER_SEQ_HEADER, SSE_COMPATIBLE_CONTENT_TYPES, STREAM_CURSOR_HEADER, STREAM_EXPIRES_AT_HEADER, STREAM_OFFSET_HEADER, STREAM_SEQ_HEADER, STREAM_TTL_HEADER, STREAM_UP_TO_DATE_HEADER, SequenceGapError, StaleEpochError, _resetHttpWarningForTesting, asAsyncIterableReadableStream, createFetchWithBackoff, createFetchWithConsumedBody, stream, warnIfUsingHttpInBrowser };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@durable-streams/client",
   "description": "TypeScript client for the Durable Streams protocol",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "author": "Durable Stream contributors",
   "license": "Apache-2.0",
   "repository": {
@@ -47,7 +47,7 @@
   "devDependencies": {
     "fast-check": "^4.4.0",
     "tsdown": "^0.9.0",
-    "@durable-streams/server": "0.1.4"
+    "@durable-streams/server": "0.1.5"
   },
   "engines": {
     "node": ">=18.0.0"

package/src/constants.ts

@@ -46,6 +46,37 @@ export const STREAM_TTL_HEADER = `Stream-TTL`
  */
 export const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`
 
+// ============================================================================
+// Idempotent Producer Headers
+// ============================================================================
+
+/**
+ * Request header for producer ID (client-supplied stable identifier).
+ */
+export const PRODUCER_ID_HEADER = `Producer-Id`
+
+/**
+ * Request/response header for producer epoch.
+ * Client-declared, server-validated monotonically increasing.
+ */
+export const PRODUCER_EPOCH_HEADER = `Producer-Epoch`
+
+/**
+ * Request header for producer sequence number.
+ * Monotonically increasing per epoch, per-batch (not per-message).
+ */
+export const PRODUCER_SEQ_HEADER = `Producer-Seq`
+
+/**
+ * Response header indicating expected sequence number on 409 Conflict.
+ */
+export const PRODUCER_EXPECTED_SEQ_HEADER = `Producer-Expected-Seq`
+
+/**
+ * Response header indicating received sequence number on 409 Conflict.
+ */
+export const PRODUCER_RECEIVED_SEQ_HEADER = `Producer-Received-Seq`
+
 // ============================================================================
 // Query Parameters
 // ============================================================================