@durable-streams/server 0.1.4 → 0.1.6

package/src/file-store.ts

@@ -15,8 +15,24 @@ import {
   normalizeContentType,
   processJsonAppend,
 } from "./store"
+import type { AppendOptions, AppendResult } from "./store"
 import type { Database } from "lmdb"
-import type { PendingLongPoll, Stream, StreamMessage } from "./types"
+import type {
+  PendingLongPoll,
+  ProducerState,
+  ProducerValidationResult,
+  Stream,
+  StreamMessage,
+} from "./types"
+
+/**
+ * Serializable producer state for LMDB storage.
+ */
+interface SerializableProducerState {
+  epoch: number
+  lastSeq: number
+  lastUpdated: number
+}
 
 /**
  * Stream metadata stored in LMDB.
@@ -37,6 +53,11 @@ interface StreamMetadata {
    * This allows safe async deletion and immediate reuse of stream paths.
    */
   directoryName: string
+  /**
+   * Producer states for idempotent writes.
+   * Stored as a plain object for LMDB serialization.
+   */
+  producers?: Record<string, SerializableProducerState>
 }
 
 /**
@@ -168,6 +189,11 @@ export class FileBackedStreamStore {
   private fileHandlePool: FileHandlePool
   private pendingLongPolls: Array<PendingLongPoll> = []
   private dataDir: string
+  /**
+   * Per-producer locks for serializing validation+append operations.
+   * Key: "{streamPath}:{producerId}"
+   */
+  private producerLocks = new Map<string, Promise<unknown>>()
 
   constructor(options: FileBackedStreamStoreOptions) {
     this.dataDir = options.dataDir
@@ -320,6 +346,15 @@ export class FileBackedStreamStore {
    * Convert LMDB metadata to Stream object.
    */
   private streamMetaToStream(meta: StreamMetadata): Stream {
+    // Convert producers from object to Map if present
+    let producers: Map<string, ProducerState> | undefined
+    if (meta.producers) {
+      producers = new Map()
+      for (const [id, state] of Object.entries(meta.producers)) {
+        producers.set(id, { ...state })
+      }
+    }
+
     return {
       path: meta.path,
       contentType: meta.contentType,
@@ -329,9 +364,132 @@ export class FileBackedStreamStore {
       ttlSeconds: meta.ttlSeconds,
       expiresAt: meta.expiresAt,
       createdAt: meta.createdAt,
+      producers,
     }
   }
 
+  /**
+   * Validate producer state WITHOUT mutating.
+   * Returns proposed state to commit after successful append.
+   *
+   * IMPORTANT: This function does NOT mutate producer state. The caller must
+   * commit the proposedState after successful append (file write + fsync + LMDB).
+   * This ensures atomicity: if any step fails, producer state is not advanced.
+   */
+  private validateProducer(
+    meta: StreamMetadata,
+    producerId: string,
+    epoch: number,
+    seq: number
+  ): ProducerValidationResult {
+    // Initialize producers map if needed (safe - just ensures map exists)
+    if (!meta.producers) {
+      meta.producers = {}
+    }
+
+    const state = meta.producers[producerId]
+    const now = Date.now()
+
+    // New producer - accept if seq is 0
+    if (!state) {
+      if (seq !== 0) {
+        return {
+          status: `sequence_gap`,
+          expectedSeq: 0,
+          receivedSeq: seq,
+        }
+      }
+      // Return proposed state, don't mutate yet
+      return {
+        status: `accepted`,
+        isNew: true,
+        producerId,
+        proposedState: { epoch, lastSeq: 0, lastUpdated: now },
+      }
+    }
+
+    // Epoch validation (client-declared, server-validated)
+    if (epoch < state.epoch) {
+      return { status: `stale_epoch`, currentEpoch: state.epoch }
+    }
+
+    if (epoch > state.epoch) {
+      // New epoch must start at seq=0
+      if (seq !== 0) {
+        return { status: `invalid_epoch_seq` }
+      }
+      // Return proposed state for new epoch, don't mutate yet
+      return {
+        status: `accepted`,
+        isNew: true,
+        producerId,
+        proposedState: { epoch, lastSeq: 0, lastUpdated: now },
+      }
+    }
+
+    // Same epoch: sequence validation
+    if (seq <= state.lastSeq) {
+      return { status: `duplicate`, lastSeq: state.lastSeq }
+    }
+
+    if (seq === state.lastSeq + 1) {
+      // Return proposed state, don't mutate yet
+      return {
+        status: `accepted`,
+        isNew: false,
+        producerId,
+        proposedState: { epoch, lastSeq: seq, lastUpdated: now },
+      }
+    }
+
+    // Sequence gap
+    return {
+      status: `sequence_gap`,
+      expectedSeq: state.lastSeq + 1,
+      receivedSeq: seq,
+    }
+  }
+
+  /**
+   * Acquire a lock for serialized producer operations.
+   * Returns a release function.
+   */
+  private async acquireProducerLock(
+    streamPath: string,
+    producerId: string
+  ): Promise<() => void> {
+    const lockKey = `${streamPath}:${producerId}`
+
+    // Wait for any existing lock
+    while (this.producerLocks.has(lockKey)) {
+      await this.producerLocks.get(lockKey)
+    }
+
+    // Create our lock
+    let releaseLock: () => void
+    const lockPromise = new Promise<void>((resolve) => {
+      releaseLock = resolve
+    })
+    this.producerLocks.set(lockKey, lockPromise)
+
+    return () => {
+      this.producerLocks.delete(lockKey)
+      releaseLock!()
+    }
+  }
+
+  /**
+   * Get the current epoch for a producer on a stream.
+   * Returns undefined if the producer doesn't exist or stream not found.
+   */
+  getProducerEpoch(streamPath: string, producerId: string): number | undefined {
+    const meta = this.getMetaIfNotExpired(streamPath)
+    if (!meta?.producers) {
+      return undefined
+    }
+    return meta.producers[producerId]?.epoch
+  }
+
   /**
    * Check if a stream is expired based on TTL or Expires-At.
    */
@@ -528,12 +686,8 @@ export class FileBackedStreamStore {
   async append(
     streamPath: string,
     data: Uint8Array,
-    options: {
-      seq?: string
-      contentType?: string
-      isInitialCreate?: boolean
-    } = {}
-  ): Promise<StreamMessage | null> {
+    options: AppendOptions & { isInitialCreate?: boolean } = {}
+  ): Promise<StreamMessage | AppendResult | null> {
     const streamMeta = this.getMetaIfNotExpired(streamPath)
 
     if (!streamMeta) {
@@ -551,7 +705,32 @@ export class FileBackedStreamStore {
       }
     }
 
-    // Check sequence for writer coordination
+    // Handle producer validation FIRST if producer headers are present
+    // This must happen before Stream-Seq check so that retries with both
+    // producer headers AND Stream-Seq can return 204 (duplicate) instead of
+    // failing the Stream-Seq conflict check.
+    let producerResult: ProducerValidationResult | undefined
+    if (
+      options.producerId !== undefined &&
+      options.producerEpoch !== undefined &&
+      options.producerSeq !== undefined
+    ) {
+      producerResult = this.validateProducer(
+        streamMeta,
+        options.producerId,
+        options.producerEpoch,
+        options.producerSeq
+      )
+
+      // Return early for non-accepted results (duplicate, stale epoch, gap)
+      // IMPORTANT: Return 204 for duplicate BEFORE Stream-Seq check
+      if (producerResult.status !== `accepted`) {
+        return { message: null, producerResult }
+      }
+    }
+
+    // Check sequence for writer coordination (Stream-Seq, separate from Producer-Seq)
+    // This happens AFTER producer validation so retries can be deduplicated
     if (options.seq !== undefined) {
       if (
         streamMeta.lastSeq !== undefined &&
@@ -620,12 +799,19 @@ export class FileBackedStreamStore {
     // 3. Flush to disk (blocks here until durable)
     await this.fileHandlePool.fsyncFile(segmentPath)
 
-    // 4. Update LMDB metadata (only after flush, so metadata reflects durability)
+    // 4. Update LMDB metadata atomically (only after flush, so metadata reflects durability)
+    //    This includes both the offset update and producer state update
+    //    Producer state is committed HERE (not in validateProducer) for atomicity
+    const updatedProducers = { ...streamMeta.producers }
+    if (producerResult && producerResult.status === `accepted`) {
+      updatedProducers[producerResult.producerId] = producerResult.proposedState
+    }
     const updatedMeta: StreamMetadata = {
       ...streamMeta,
       currentOffset: newOffset,
       lastSeq: options.seq ?? streamMeta.lastSeq,
       totalBytes: streamMeta.totalBytes + processedData.length + 5, // +4 for length, +1 for newline
+      producers: updatedProducers,
     }
     const key = `stream:${streamPath}`
     this.db.putSync(key, updatedMeta)
@@ -633,10 +819,52 @@ export class FileBackedStreamStore {
     // 5. Notify long-polls (data is now readable from disk)
     this.notifyLongPolls(streamPath)
 
-    // 6. Return (client knows data is durable)
+    // 6. Return AppendResult if producer headers were used
+    if (producerResult) {
+      return {
+        message,
+        producerResult,
+      }
+    }
+
     return message
   }
 
+  /**
+   * Append with producer serialization for concurrent request handling.
+   * This ensures that validation+append is atomic per producer.
+   */
+  async appendWithProducer(
+    streamPath: string,
+    data: Uint8Array,
+    options: AppendOptions
+  ): Promise<AppendResult> {
+    if (!options.producerId) {
+      // No producer - just do a normal append
+      const result = await this.append(streamPath, data, options)
+      if (result && `message` in result) {
+        return result
+      }
+      return { message: result }
+    }
+
+    // Acquire lock for this producer
+    const releaseLock = await this.acquireProducerLock(
+      streamPath,
+      options.producerId
+    )
+
+    try {
+      const result = await this.append(streamPath, data, options)
+      if (result && `message` in result) {
+        return result
+      }
+      return { message: result }
+    } finally {
+      releaseLock()
+    }
+  }
+
   read(
     streamPath: string,
     offset?: string

package/src/server.ts

@@ -19,6 +19,13 @@ const STREAM_SEQ_HEADER = `Stream-Seq`
 const STREAM_TTL_HEADER = `Stream-TTL`
 const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`
 
+// Idempotent producer headers
+const PRODUCER_ID_HEADER = `Producer-Id`
+const PRODUCER_EPOCH_HEADER = `Producer-Epoch`
+const PRODUCER_SEQ_HEADER = `Producer-Seq`
+const PRODUCER_EXPECTED_SEQ_HEADER = `Producer-Expected-Seq`
+const PRODUCER_RECEIVED_SEQ_HEADER = `Producer-Received-Seq`
+
 // SSE control event fields (Protocol Section 5.7)
 const SSE_OFFSET_FIELD = `streamNextOffset`
 const SSE_CURSOR_FIELD = `streamCursor`
@@ -398,11 +405,11 @@ export class DurableStreamTestServer {
     )
     res.setHeader(
       `access-control-allow-headers`,
-      `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At`
+      `content-type, authorization, Stream-Seq, Stream-TTL, Stream-Expires-At, Producer-Id, Producer-Epoch, Producer-Seq`
     )
     res.setHeader(
       `access-control-expose-headers`,
-      `Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Date, etag, content-type, content-encoding, vary`
+      `Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Date, Producer-Epoch, Producer-Seq, Producer-Expected-Seq, Producer-Received-Seq, etag, content-type, content-encoding, vary`
     )
 
     // Browser security headers (Protocol Section 10.7)
@@ -679,9 +686,9 @@ export class DurableStreamTestServer {
         return
       }
 
-      // Validate offset format: must be "-1" or match our offset format (digits_digits)
+      // Validate offset format: must be "-1", "now", or match our offset format (digits_digits)
       // This prevents path traversal, injection attacks, and invalid characters
-      const validOffsetPattern = /^(-1|\d+_\d+)$/
+      const validOffsetPattern = /^(-1|now|\d+_\d+)$/
       if (!validOffsetPattern.test(offset)) {
         res.writeHead(400, { "content-type": `text/plain` })
         res.end(`Invalid offset format`)
@@ -700,23 +707,57 @@ export class DurableStreamTestServer {
 
     // Handle SSE mode
     if (live === `sse`) {
-      await this.handleSSE(path, stream, offset!, cursor, res)
+      // For SSE with offset=now, convert to actual tail offset
+      const sseOffset = offset === `now` ? stream.currentOffset : offset!
+      await this.handleSSE(path, stream, sseOffset, cursor, res)
+      return
+    }
+
+    // For offset=now, convert to actual tail offset
+    // This allows long-poll to immediately start waiting for new data
+    const effectiveOffset = offset === `now` ? stream.currentOffset : offset
+
+    // Handle catch-up mode offset=now: return empty response with tail offset
+    // For long-poll mode, we fall through to wait for new data instead
+    if (offset === `now` && live !== `long-poll`) {
+      const headers: Record<string, string> = {
+        [STREAM_OFFSET_HEADER]: stream.currentOffset,
+        [STREAM_UP_TO_DATE_HEADER]: `true`,
+        // Prevent caching - tail offset changes with each append
+        [`cache-control`]: `no-store`,
+      }
+
+      if (stream.contentType) {
+        headers[`content-type`] = stream.contentType
+      }
+
+      // No ETag for offset=now responses - Cache-Control: no-store makes ETag unnecessary
+      // and some CDNs may behave unexpectedly with both headers
+
+      // For JSON mode, return empty array; otherwise empty body
+      const isJsonMode = stream.contentType?.includes(`application/json`)
+      const responseBody = isJsonMode ? `[]` : ``
+
+      res.writeHead(200, headers)
+      res.end(responseBody)
       return
     }
 
     // Read current messages
-    let { messages, upToDate } = this.store.read(path, offset)
+    let { messages, upToDate } = this.store.read(path, effectiveOffset)
 
     // Only wait in long-poll if:
     // 1. long-poll mode is enabled
-    // 2. Client provided an offset (not first request)
+    // 2. Client provided an offset (not first request) OR used offset=now
     // 3. Client's offset matches current offset (already caught up)
     // 4. No new messages
-    const clientIsCaughtUp = offset && offset === stream.currentOffset
+    const clientIsCaughtUp =
+      (effectiveOffset && effectiveOffset === stream.currentOffset) ||
+      offset === `now`
     if (live === `long-poll` && clientIsCaughtUp && messages.length === 0) {
       const result = await this.store.waitForMessages(
         path,
-        offset,
+        effectiveOffset ?? stream.currentOffset,
         this.options.longPollTimeout
       )
 
@@ -728,7 +769,7 @@ export class DurableStreamTestServer {
           this.options.cursorOptions
         )
         res.writeHead(204, {
-          [STREAM_OFFSET_HEADER]: offset,
+          [STREAM_OFFSET_HEADER]: effectiveOffset ?? stream.currentOffset,
           [STREAM_UP_TO_DATE_HEADER]: `true`,
           [STREAM_CURSOR_HEADER]: responseCursor,
         })
@@ -941,6 +982,17 @@ export class DurableStreamTestServer {
       | string
       | undefined
 
+    // Extract producer headers
+    const producerId = req.headers[PRODUCER_ID_HEADER.toLowerCase()] as
+      | string
+      | undefined
+    const producerEpochStr = req.headers[
+      PRODUCER_EPOCH_HEADER.toLowerCase()
+    ] as string | undefined
+    const producerSeqStr = req.headers[PRODUCER_SEQ_HEADER.toLowerCase()] as
+      | string
+      | undefined
+
     const body = await this.readBody(req)
 
     if (body.length === 0) {
@@ -956,15 +1008,148 @@ export class DurableStreamTestServer {
       return
     }
 
-    // Support both sync (StreamStore) and async (FileBackedStreamStore) append
-    // Note: append returns null only for empty arrays with isInitialCreate=true,
-    // which doesn't apply to POST requests (those throw on empty arrays)
-    const message = await Promise.resolve(
-      this.store.append(path, body, { seq, contentType })
-    )
+    // Validate producer headers - all three must be present together or none
+    // Also reject empty producer ID
+    const hasProducerHeaders =
+      producerId !== undefined ||
+      producerEpochStr !== undefined ||
+      producerSeqStr !== undefined
+    const hasAllProducerHeaders =
+      producerId !== undefined &&
+      producerEpochStr !== undefined &&
+      producerSeqStr !== undefined
+
+    if (hasProducerHeaders && !hasAllProducerHeaders) {
+      res.writeHead(400, { "content-type": `text/plain` })
+      res.end(
+        `All producer headers (Producer-Id, Producer-Epoch, Producer-Seq) must be provided together`
+      )
+      return
+    }
+
+    if (hasAllProducerHeaders && producerId === ``) {
+      res.writeHead(400, { "content-type": `text/plain` })
+      res.end(`Invalid Producer-Id: must not be empty`)
+      return
+    }
+
+    // Parse and validate producer epoch and seq as integers
+    // Use strict digit-only validation to reject values like "1abc" or "1e3"
+    const STRICT_INTEGER_REGEX = /^\d+$/
+    let producerEpoch: number | undefined
+    let producerSeq: number | undefined
+    if (hasAllProducerHeaders) {
+      if (!STRICT_INTEGER_REGEX.test(producerEpochStr)) {
+        res.writeHead(400, { "content-type": `text/plain` })
+        res.end(`Invalid Producer-Epoch: must be a non-negative integer`)
+        return
+      }
+      producerEpoch = Number(producerEpochStr)
+      if (!Number.isSafeInteger(producerEpoch)) {
+        res.writeHead(400, { "content-type": `text/plain` })
+        res.end(`Invalid Producer-Epoch: must be a non-negative integer`)
+        return
+      }
+
+      if (!STRICT_INTEGER_REGEX.test(producerSeqStr)) {
+        res.writeHead(400, { "content-type": `text/plain` })
+        res.end(`Invalid Producer-Seq: must be a non-negative integer`)
+        return
+      }
+      producerSeq = Number(producerSeqStr)
+      if (!Number.isSafeInteger(producerSeq)) {
+        res.writeHead(400, { "content-type": `text/plain` })
+        res.end(`Invalid Producer-Seq: must be a non-negative integer`)
+        return
+      }
+    }
+
+    // Build append options
+    const appendOptions = {
+      seq,
+      contentType,
+      producerId,
+      producerEpoch,
+      producerSeq,
+    }
+
+    // Use appendWithProducer for serialized producer operations
+    let result
+    if (producerId !== undefined) {
+      result = await this.store.appendWithProducer(path, body, appendOptions)
+    } else {
+      result = await Promise.resolve(
+        this.store.append(path, body, appendOptions)
+      )
+    }
+
+    // Handle AppendResult with producer validation
+    if (result && typeof result === `object` && `producerResult` in result) {
+      const { message, producerResult } = result
+
+      if (!producerResult || producerResult.status === `accepted`) {
+        // Success - return offset
+        const responseHeaders: Record<string, string> = {
+          [STREAM_OFFSET_HEADER]: message!.offset,
+        }
+        // Echo back the producer epoch and seq (highest accepted)
+        if (producerEpoch !== undefined) {
+          responseHeaders[PRODUCER_EPOCH_HEADER] = producerEpoch.toString()
+        }
+        if (producerSeq !== undefined) {
+          responseHeaders[PRODUCER_SEQ_HEADER] = producerSeq.toString()
+        }
+        res.writeHead(200, responseHeaders)
+        res.end()
+        return
+      }
+
+      // Handle producer validation failures
+      switch (producerResult.status) {
+        case `duplicate`:
+          // 204 No Content for duplicates (idempotent success)
+          // Return Producer-Seq as highest accepted (per PROTOCOL.md)
+          res.writeHead(204, {
+            [PRODUCER_EPOCH_HEADER]: producerEpoch!.toString(),
+            [PRODUCER_SEQ_HEADER]: producerResult.lastSeq.toString(),
+          })
+          res.end()
+          return
+
+        case `stale_epoch`: {
+          // 403 Forbidden for stale epochs (zombie fencing)
+          res.writeHead(403, {
+            "content-type": `text/plain`,
+            [PRODUCER_EPOCH_HEADER]: producerResult.currentEpoch.toString(),
+          })
+          res.end(`Stale producer epoch`)
+          return
+        }
+
+        case `invalid_epoch_seq`:
+          // 400 Bad Request for epoch increase with seq != 0
+          res.writeHead(400, { "content-type": `text/plain` })
+          res.end(`New epoch must start with sequence 0`)
+          return
+
+        case `sequence_gap`:
+          // 409 Conflict for sequence gaps
+          res.writeHead(409, {
+            "content-type": `text/plain`,
+            [PRODUCER_EXPECTED_SEQ_HEADER]:
+              producerResult.expectedSeq.toString(),
+            [PRODUCER_RECEIVED_SEQ_HEADER]:
+              producerResult.receivedSeq.toString(),
+          })
+          res.end(`Producer sequence gap`)
+          return
+      }
+    }
 
+    // Standard append (no producer) - result is StreamMessage
+    const message = result as { offset: string }
     res.writeHead(204, {
-      [STREAM_OFFSET_HEADER]: message!.offset,
+      [STREAM_OFFSET_HEADER]: message.offset,
     })
     res.end()
   }