@durable-streams/server 0.1.4 → 0.1.6

package/dist/index.d.cts

@@ -56,6 +56,11 @@ interface Stream {
   * Timestamp when the stream was created.
   */
   createdAt: number;
+  /**
+  * Producer states for idempotent writes.
+  * Maps producer ID to their epoch and sequence state.
+  */
+  producers?: Map<string, ProducerState>;
 }
 /**
 * Event data for stream lifecycle hooks.
@@ -131,6 +136,49 @@ interface TestServerOptions {
   cursorEpoch?: Date;
 }
 /**
+* Producer state for idempotent writes.
+* Tracks epoch and sequence number per producer ID for deduplication.
+*/
+interface ProducerState {
+  /**
+  * Current epoch for this producer.
+  * Client-declared, server-validated monotonically increasing.
+  */
+  epoch: number;
+  /**
+  * Last sequence number received in this epoch.
+  */
+  lastSeq: number;
+  /**
+  * Timestamp when this producer state was last updated.
+  * Used for TTL-based cleanup.
+  */
+  lastUpdated: number;
+}
+/**
+* Result of producer validation for append operations.
+* For 'accepted' status, includes proposedState to commit after successful append.
+*/
+type ProducerValidationResult = {
+  status: `accepted`;
+  isNew: boolean;
+  /** State to commit after successful append (deferred mutation) */
+  proposedState: ProducerState;
+  producerId: string;
+} | {
+  status: `duplicate`;
+  lastSeq: number;
+} | {
+  status: `stale_epoch`;
+  currentEpoch: number;
+} | {
+  status: `invalid_epoch_seq`;
+} | {
+  status: `sequence_gap`;
+  expectedSeq: number;
+  receivedSeq: number;
+};
+/**
 * Pending long-poll request.
 */
 interface PendingLongPoll {
@@ -155,10 +203,32 @@ interface PendingLongPoll {
 /**
 * In-memory store for durable streams.
 */
+/**
+* Options for append operations.
+*/
+interface AppendOptions {
+  seq?: string;
+  contentType?: string;
+  producerId?: string;
+  producerEpoch?: number;
+  producerSeq?: number;
+}
+/**
+* Result of an append operation.
+*/
+interface AppendResult {
+  message: StreamMessage | null;
+  producerResult?: ProducerValidationResult;
+}
 declare class StreamStore {
   private streams;
   private pendingLongPolls;
   /**
+  * Per-producer locks for serializing validation+append operations.
+  * Key: "{streamPath}:{producerId}"
+  */
+  private producerLocks;
+  /**
   * Check if a stream is expired based on TTL or Expires-At.
   */
   private isExpired;
@@ -192,15 +262,47 @@ declare class StreamStore {
   */
   delete(path: string): boolean;
   /**
+  * Validate producer state WITHOUT mutating.
+  * Returns proposed state to commit after successful append.
+  * Implements Kafka-style idempotent producer validation.
+  *
+  * IMPORTANT: This function does NOT mutate producer state. The caller must
+  * call commitProducerState() after successful append to apply the mutation.
+  * This ensures atomicity: if append fails (e.g., JSON validation), producer
+  * state is not incorrectly advanced.
+  */
+  private validateProducer;
+  /**
+  * Commit producer state after successful append.
+  * This is the only place where producer state is mutated.
+  */
+  private commitProducerState;
+  /**
+  * Clean up expired producer states from a stream.
+  */
+  private cleanupExpiredProducers;
+  /**
+  * Acquire a lock for serialized producer operations.
+  * Returns a release function.
+  */
+  private acquireProducerLock;
+  /**
   * Append data to a stream.
   * @throws Error if stream doesn't exist or is expired
   * @throws Error if seq is lower than lastSeq
   * @throws Error if JSON mode and array is empty
   */
-  append(path: string, data: Uint8Array, options?: {
-    seq?: string;
-    contentType?: string;
-  }): StreamMessage;
+  append(path: string, data: Uint8Array, options?: AppendOptions): StreamMessage | AppendResult;
+  /**
+  * Append with producer serialization for concurrent request handling.
+  * This ensures that validation+append is atomic per producer.
+  */
+  appendWithProducer(path: string, data: Uint8Array, options: AppendOptions): Promise<AppendResult>;
+  /**
+  * Get the current epoch for a producer on a stream.
+  * Returns undefined if the producer doesn't exist or stream not found.
+  */
+  getProducerEpoch(path: string, producerId: string): number | undefined;
   /**
   * Read messages from a stream starting at the given offset.
   * @throws Error if stream doesn't exist or is expired
@@ -261,6 +363,11 @@ declare class FileBackedStreamStore {
   private fileHandlePool;
   private pendingLongPolls;
   private dataDir;
+  /**
+  * Per-producer locks for serializing validation+append operations.
+  * Key: "{streamPath}:{producerId}"
+  */
+  private producerLocks;
   constructor(options: FileBackedStreamStoreOptions);
   /**
   * Recover streams from disk on startup.
@@ -277,6 +384,25 @@ declare class FileBackedStreamStore {
   */
   private streamMetaToStream;
   /**
+  * Validate producer state WITHOUT mutating.
+  * Returns proposed state to commit after successful append.
+  *
+  * IMPORTANT: This function does NOT mutate producer state. The caller must
+  * commit the proposedState after successful append (file write + fsync + LMDB).
+  * This ensures atomicity: if any step fails, producer state is not advanced.
+  */
+  private validateProducer;
+  /**
+  * Acquire a lock for serialized producer operations.
+  * Returns a release function.
+  */
+  private acquireProducerLock;
+  /**
+  * Get the current epoch for a producer on a stream.
+  * Returns undefined if the producer doesn't exist or stream not found.
+  */
+  getProducerEpoch(streamPath: string, producerId: string): number | undefined;
+  /**
   * Check if a stream is expired based on TTL or Expires-At.
   */
   private isExpired;
@@ -299,11 +425,14 @@ declare class FileBackedStreamStore {
   get(streamPath: string): Stream | undefined;
   has(streamPath: string): boolean;
   delete(streamPath: string): boolean;
-  append(streamPath: string, data: Uint8Array, options?: {
-    seq?: string;
-    contentType?: string;
+  append(streamPath: string, data: Uint8Array, options?: AppendOptions & {
     isInitialCreate?: boolean;
-  }): Promise<StreamMessage | null>;
+  }): Promise<StreamMessage | AppendResult | null>;
+  /**
+  * Append with producer serialization for concurrent request handling.
+  * This ensures that validation+append is atomic per producer.
+  */
+  appendWithProducer(streamPath: string, data: Uint8Array, options: AppendOptions): Promise<AppendResult>;
   read(streamPath: string, offset?: string): {
     messages: Array<StreamMessage>;
     upToDate: boolean;
@@ -451,7 +580,9 @@ declare class DurableStreamTestServer {
   */
   private handleTestInjectError;
   private readBody;
-} //#endregion
+}
+
+//#endregion
 //#region src/path-encoding.d.ts
 /**
 * Encode a stream path to a filesystem-safe directory name using base64url encoding.

package/dist/index.d.ts

@@ -56,6 +56,11 @@ interface Stream {
   * Timestamp when the stream was created.
   */
   createdAt: number;
+  /**
+  * Producer states for idempotent writes.
+  * Maps producer ID to their epoch and sequence state.
+  */
+  producers?: Map<string, ProducerState>;
 }
 /**
 * Event data for stream lifecycle hooks.
@@ -131,6 +136,49 @@ interface TestServerOptions {
   cursorEpoch?: Date;
 }
 /**
+* Producer state for idempotent writes.
+* Tracks epoch and sequence number per producer ID for deduplication.
+*/
+interface ProducerState {
+  /**
+  * Current epoch for this producer.
+  * Client-declared, server-validated monotonically increasing.
+  */
+  epoch: number;
+  /**
+  * Last sequence number received in this epoch.
+  */
+  lastSeq: number;
+  /**
+  * Timestamp when this producer state was last updated.
+  * Used for TTL-based cleanup.
+  */
+  lastUpdated: number;
+}
+/**
+* Result of producer validation for append operations.
+* For 'accepted' status, includes proposedState to commit after successful append.
+*/
+type ProducerValidationResult = {
+  status: `accepted`;
+  isNew: boolean;
+  /** State to commit after successful append (deferred mutation) */
+  proposedState: ProducerState;
+  producerId: string;
+} | {
+  status: `duplicate`;
+  lastSeq: number;
+} | {
+  status: `stale_epoch`;
+  currentEpoch: number;
+} | {
+  status: `invalid_epoch_seq`;
+} | {
+  status: `sequence_gap`;
+  expectedSeq: number;
+  receivedSeq: number;
+};
+/**
 * Pending long-poll request.
 */
 interface PendingLongPoll {
@@ -155,10 +203,32 @@ interface PendingLongPoll {
 /**
 * In-memory store for durable streams.
 */
+/**
+* Options for append operations.
+*/
+interface AppendOptions {
+  seq?: string;
+  contentType?: string;
+  producerId?: string;
+  producerEpoch?: number;
+  producerSeq?: number;
+}
+/**
+* Result of an append operation.
+*/
+interface AppendResult {
+  message: StreamMessage | null;
+  producerResult?: ProducerValidationResult;
+}
 declare class StreamStore {
   private streams;
   private pendingLongPolls;
   /**
+  * Per-producer locks for serializing validation+append operations.
+  * Key: "{streamPath}:{producerId}"
+  */
+  private producerLocks;
+  /**
   * Check if a stream is expired based on TTL or Expires-At.
   */
   private isExpired;
@@ -192,15 +262,47 @@ declare class StreamStore {
   */
   delete(path: string): boolean;
   /**
+  * Validate producer state WITHOUT mutating.
+  * Returns proposed state to commit after successful append.
+  * Implements Kafka-style idempotent producer validation.
+  *
+  * IMPORTANT: This function does NOT mutate producer state. The caller must
+  * call commitProducerState() after successful append to apply the mutation.
+  * This ensures atomicity: if append fails (e.g., JSON validation), producer
+  * state is not incorrectly advanced.
+  */
+  private validateProducer;
+  /**
+  * Commit producer state after successful append.
+  * This is the only place where producer state is mutated.
+  */
+  private commitProducerState;
+  /**
+  * Clean up expired producer states from a stream.
+  */
+  private cleanupExpiredProducers;
+  /**
+  * Acquire a lock for serialized producer operations.
+  * Returns a release function.
+  */
+  private acquireProducerLock;
+  /**
   * Append data to a stream.
   * @throws Error if stream doesn't exist or is expired
   * @throws Error if seq is lower than lastSeq
   * @throws Error if JSON mode and array is empty
   */
-  append(path: string, data: Uint8Array, options?: {
-    seq?: string;
-    contentType?: string;
-  }): StreamMessage;
+  append(path: string, data: Uint8Array, options?: AppendOptions): StreamMessage | AppendResult;
+  /**
+  * Append with producer serialization for concurrent request handling.
+  * This ensures that validation+append is atomic per producer.
+  */
+  appendWithProducer(path: string, data: Uint8Array, options: AppendOptions): Promise<AppendResult>;
+  /**
+  * Get the current epoch for a producer on a stream.
+  * Returns undefined if the producer doesn't exist or stream not found.
+  */
+  getProducerEpoch(path: string, producerId: string): number | undefined;
   /**
   * Read messages from a stream starting at the given offset.
   * @throws Error if stream doesn't exist or is expired
@@ -261,6 +363,11 @@ declare class FileBackedStreamStore {
   private fileHandlePool;
   private pendingLongPolls;
   private dataDir;
+  /**
+  * Per-producer locks for serializing validation+append operations.
+  * Key: "{streamPath}:{producerId}"
+  */
+  private producerLocks;
   constructor(options: FileBackedStreamStoreOptions);
   /**
   * Recover streams from disk on startup.
@@ -277,6 +384,25 @@ declare class FileBackedStreamStore {
   */
   private streamMetaToStream;
   /**
+  * Validate producer state WITHOUT mutating.
+  * Returns proposed state to commit after successful append.
+  *
+  * IMPORTANT: This function does NOT mutate producer state. The caller must
+  * commit the proposedState after successful append (file write + fsync + LMDB).
+  * This ensures atomicity: if any step fails, producer state is not advanced.
+  */
+  private validateProducer;
+  /**
+  * Acquire a lock for serialized producer operations.
+  * Returns a release function.
+  */
+  private acquireProducerLock;
+  /**
+  * Get the current epoch for a producer on a stream.
+  * Returns undefined if the producer doesn't exist or stream not found.
+  */
+  getProducerEpoch(streamPath: string, producerId: string): number | undefined;
+  /**
   * Check if a stream is expired based on TTL or Expires-At.
   */
   private isExpired;
@@ -299,11 +425,14 @@ declare class FileBackedStreamStore {
   get(streamPath: string): Stream | undefined;
   has(streamPath: string): boolean;
   delete(streamPath: string): boolean;
-  append(streamPath: string, data: Uint8Array, options?: {
-    seq?: string;
-    contentType?: string;
+  append(streamPath: string, data: Uint8Array, options?: AppendOptions & {
     isInitialCreate?: boolean;
-  }): Promise<StreamMessage | null>;
+  }): Promise<StreamMessage | AppendResult | null>;
+  /**
+  * Append with producer serialization for concurrent request handling.
+  * This ensures that validation+append is atomic per producer.
+  */
+  appendWithProducer(streamPath: string, data: Uint8Array, options: AppendOptions): Promise<AppendResult>;
   read(streamPath: string, offset?: string): {
     messages: Array<StreamMessage>;
     upToDate: boolean;
@@ -451,7 +580,9 @@ declare class DurableStreamTestServer {
   */
   private handleTestInjectError;
   private readBody;
-} //#endregion
+}
+
+//#endregion
 //#region src/path-encoding.d.ts
 /**
 * Encode a stream path to a filesystem-safe directory name using base64url encoding.