@durable-streams/server 0.1.0

package/src/store.ts

@@ -0,0 +1,433 @@
+/**
+ * In-memory stream storage.
+ */
+
+import type { PendingLongPoll, Stream, StreamMessage } from "./types"
+
+/**
+ * Normalize content-type by extracting the media type (before any semicolon).
+ * Handles cases like "application/json; charset=utf-8".
+ */
+export function normalizeContentType(contentType: string | undefined): string {
+  if (!contentType) return ``
+  return contentType.split(`;`)[0]!.trim().toLowerCase()
+}
+
+/**
+ * Process JSON data for append in JSON mode.
+ * - Validates JSON
+ * - Extracts array elements if data is an array
+ * - Always appends trailing comma for easy concatenation
+ * @param isInitialCreate - If true, empty arrays are allowed (creates empty stream)
+ * @throws Error if JSON is invalid or array is empty (for non-create operations)
+ */
+export function processJsonAppend(
+  data: Uint8Array,
+  isInitialCreate = false
+): Uint8Array {
+  const text = new TextDecoder().decode(data)
+
+  // Validate JSON
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(text)
+  } catch {
+    throw new Error(`Invalid JSON`)
+  }
+
+  // If it's an array, extract elements and join with commas
+  let result: string
+  if (Array.isArray(parsed)) {
+    if (parsed.length === 0) {
+      // Empty arrays are valid for PUT (creates empty stream)
+      // but invalid for POST (no-op append, likely a bug)
+      if (isInitialCreate) {
+        return new Uint8Array(0) // Return empty data for empty stream
+      }
+      throw new Error(`Empty arrays are not allowed`)
+    }
+    const elements = parsed.map((item) => JSON.stringify(item))
+    result = elements.join(`,`) + `,`
+  } else {
+    // Single value - re-serialize to normalize whitespace (single-line JSON)
+    result = JSON.stringify(parsed) + `,`
+  }
+
+  return new TextEncoder().encode(result)
+}
+
+/**
+ * Format JSON mode response by wrapping in array brackets.
+ * Strips trailing comma before wrapping.
+ */
+export function formatJsonResponse(data: Uint8Array): Uint8Array {
+  if (data.length === 0) {
+    return new TextEncoder().encode(`[]`)
+  }
+
+  let text = new TextDecoder().decode(data)
+  // Strip trailing comma if present
+  text = text.trimEnd()
+  if (text.endsWith(`,`)) {
+    text = text.slice(0, -1)
+  }
+
+  const wrapped = `[${text}]`
+  return new TextEncoder().encode(wrapped)
+}
+
+/**
+ * In-memory store for durable streams.
+ */
+export class StreamStore {
+  private streams = new Map<string, Stream>()
+  private pendingLongPolls: Array<PendingLongPoll> = []
+
+  /**
+   * Create a new stream.
+   * @throws Error if stream already exists with different config
+   * @returns existing stream if config matches (idempotent)
+   */
+  create(
+    path: string,
+    options: {
+      contentType?: string
+      ttlSeconds?: number
+      expiresAt?: string
+      initialData?: Uint8Array
+    } = {}
+  ): Stream {
+    const existing = this.streams.get(path)
+    if (existing) {
+      // Check if config matches (idempotent create)
+      const contentTypeMatches =
+        (normalizeContentType(options.contentType) ||
+          `application/octet-stream`) ===
+        (normalizeContentType(existing.contentType) ||
+          `application/octet-stream`)
+      const ttlMatches = options.ttlSeconds === existing.ttlSeconds
+      const expiresMatches = options.expiresAt === existing.expiresAt
+
+      if (contentTypeMatches && ttlMatches && expiresMatches) {
+        // Idempotent success - return existing stream
+        return existing
+      } else {
+        // Config mismatch - conflict
+        throw new Error(
+          `Stream already exists with different configuration: ${path}`
+        )
+      }
+    }
+
+    const stream: Stream = {
+      path,
+      contentType: options.contentType,
+      messages: [],
+      currentOffset: `0000000000000000_0000000000000000`,
+      ttlSeconds: options.ttlSeconds,
+      expiresAt: options.expiresAt,
+      createdAt: Date.now(),
+    }
+
+    // If initial data is provided, append it
+    if (options.initialData && options.initialData.length > 0) {
+      this.appendToStream(stream, options.initialData, true) // isInitialCreate = true
+    }
+
+    this.streams.set(path, stream)
+    return stream
+  }
+
+  /**
+   * Get a stream by path.
+   */
+  get(path: string): Stream | undefined {
+    return this.streams.get(path)
+  }
+
+  /**
+   * Check if a stream exists.
+   */
+  has(path: string): boolean {
+    return this.streams.has(path)
+  }
+
+  /**
+   * Delete a stream.
+   */
+  delete(path: string): boolean {
+    // Cancel any pending long-polls for this stream
+    this.cancelLongPollsForStream(path)
+    return this.streams.delete(path)
+  }
+
+  /**
+   * Append data to a stream.
+   * @throws Error if stream doesn't exist
+   * @throws Error if seq is lower than lastSeq
+   * @throws Error if JSON mode and array is empty
+   */
+  append(
+    path: string,
+    data: Uint8Array,
+    options: { seq?: string; contentType?: string } = {}
+  ): StreamMessage {
+    const stream = this.streams.get(path)
+    if (!stream) {
+      throw new Error(`Stream not found: ${path}`)
+    }
+
+    // Check content type match using normalization (handles charset parameters)
+    if (options.contentType && stream.contentType) {
+      const providedType = normalizeContentType(options.contentType)
+      const streamType = normalizeContentType(stream.contentType)
+      if (providedType !== streamType) {
+        throw new Error(
+          `Content-type mismatch: expected ${stream.contentType}, got ${options.contentType}`
+        )
+      }
+    }
+
+    // Check sequence for writer coordination
+    if (options.seq !== undefined) {
+      if (stream.lastSeq !== undefined && options.seq <= stream.lastSeq) {
+        throw new Error(
+          `Sequence conflict: ${options.seq} <= ${stream.lastSeq}`
+        )
+      }
+      stream.lastSeq = options.seq
+    }
+
+    // appendToStream returns null only for empty arrays in create mode,
+    // but public append() never sets isInitialCreate, so empty arrays throw before this
+    const message = this.appendToStream(stream, data)!
+
+    // Notify any pending long-polls
+    this.notifyLongPolls(path)
+
+    return message
+  }
+
+  /**
+   * Read messages from a stream starting at the given offset.
+   */
+  read(
+    path: string,
+    offset?: string
+  ): { messages: Array<StreamMessage>; upToDate: boolean } {
+    const stream = this.streams.get(path)
+    if (!stream) {
+      throw new Error(`Stream not found: ${path}`)
+    }
+
+    // No offset or -1 means start from beginning
+    if (!offset || offset === `-1`) {
+      return {
+        messages: [...stream.messages],
+        upToDate: true,
+      }
+    }
+
+    // Find messages after the given offset
+    const offsetIndex = this.findOffsetIndex(stream, offset)
+    if (offsetIndex === -1) {
+      // Offset is at or past the end
+      return {
+        messages: [],
+        upToDate: true,
+      }
+    }
+
+    return {
+      messages: stream.messages.slice(offsetIndex),
+      upToDate: true,
+    }
+  }
+
+  /**
+   * Format messages for response.
+   * For JSON mode, wraps concatenated data in array brackets.
+   */
+  formatResponse(path: string, messages: Array<StreamMessage>): Uint8Array {
+    const stream = this.streams.get(path)
+    if (!stream) {
+      throw new Error(`Stream not found: ${path}`)
+    }
+
+    // Concatenate all message data
+    const totalSize = messages.reduce((sum, m) => sum + m.data.length, 0)
+    const concatenated = new Uint8Array(totalSize)
+    let offset = 0
+    for (const msg of messages) {
+      concatenated.set(msg.data, offset)
+      offset += msg.data.length
+    }
+
+    // For JSON mode, wrap in array brackets
+    if (normalizeContentType(stream.contentType) === `application/json`) {
+      return formatJsonResponse(concatenated)
+    }
+
+    return concatenated
+  }
+
+  /**
+   * Wait for new messages (long-poll).
+   */
+  async waitForMessages(
+    path: string,
+    offset: string,
+    timeoutMs: number
+  ): Promise<{ messages: Array<StreamMessage>; timedOut: boolean }> {
+    const stream = this.streams.get(path)
+    if (!stream) {
+      throw new Error(`Stream not found: ${path}`)
+    }
+
+    // Check if there are already new messages
+    const { messages } = this.read(path, offset)
+    if (messages.length > 0) {
+      return { messages, timedOut: false }
+    }
+
+    // Wait for new messages
+    return new Promise((resolve) => {
+      const timeoutId = setTimeout(() => {
+        // Remove from pending
+        this.removePendingLongPoll(pending)
+        resolve({ messages: [], timedOut: true })
+      }, timeoutMs)
+
+      const pending: PendingLongPoll = {
+        path,
+        offset,
+        resolve: (msgs) => {
+          clearTimeout(timeoutId)
+          this.removePendingLongPoll(pending)
+          resolve({ messages: msgs, timedOut: false })
+        },
+        timeoutId,
+      }
+
+      this.pendingLongPolls.push(pending)
+    })
+  }
+
+  /**
+   * Get the current offset for a stream.
+   */
+  getCurrentOffset(path: string): string | undefined {
+    return this.streams.get(path)?.currentOffset
+  }
+
+  /**
+   * Clear all streams.
+   */
+  clear(): void {
+    // Cancel all pending long-polls and resolve them with timeout
+    for (const pending of this.pendingLongPolls) {
+      clearTimeout(pending.timeoutId)
+      // Resolve with empty result to unblock waiting handlers
+      pending.resolve([])
+    }
+    this.pendingLongPolls = []
+    this.streams.clear()
+  }
+
+  /**
+   * Cancel all pending long-polls (used during shutdown).
+   */
+  cancelAllWaits(): void {
+    for (const pending of this.pendingLongPolls) {
+      clearTimeout(pending.timeoutId)
+      // Resolve with empty result to unblock waiting handlers
+      pending.resolve([])
+    }
+    this.pendingLongPolls = []
+  }
+
+  /**
+   * Get all stream paths.
+   */
+  list(): Array<string> {
+    return Array.from(this.streams.keys())
+  }
+
+  // ============================================================================
+  // Private helpers
+  // ============================================================================
+
+  private appendToStream(
+    stream: Stream,
+    data: Uint8Array,
+    isInitialCreate = false
+  ): StreamMessage | null {
+    // Process JSON mode data (throws on invalid JSON or empty arrays for appends)
+    let processedData = data
+    if (normalizeContentType(stream.contentType) === `application/json`) {
+      processedData = processJsonAppend(data, isInitialCreate)
+      // If empty array in create mode, return null (empty stream created successfully)
+      if (processedData.length === 0) {
+        return null
+      }
+    }
+
+    // Parse current offset
+    const parts = stream.currentOffset.split(`_`).map(Number)
+    const readSeq = parts[0]!
+    const byteOffset = parts[1]!
+
+    // Calculate new offset with zero-padding for lexicographic sorting
+    const newByteOffset = byteOffset + processedData.length
+    const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`
+
+    const message: StreamMessage = {
+      data: processedData,
+      offset: newOffset,
+      timestamp: Date.now(),
+    }
+
+    stream.messages.push(message)
+    stream.currentOffset = newOffset
+
+    return message
+  }
+
+  private findOffsetIndex(stream: Stream, offset: string): number {
+    // Find the first message with an offset greater than the given offset
+    // Use lexicographic comparison as required by protocol
+    for (let i = 0; i < stream.messages.length; i++) {
+      if (stream.messages[i]!.offset > offset) {
+        return i
+      }
+    }
+    return -1 // No messages after the offset
+  }
+
+  private notifyLongPolls(path: string): void {
+    const toNotify = this.pendingLongPolls.filter((p) => p.path === path)
+
+    for (const pending of toNotify) {
+      const { messages } = this.read(path, pending.offset)
+      if (messages.length > 0) {
+        pending.resolve(messages)
+      }
+    }
+  }
+
+  private cancelLongPollsForStream(path: string): void {
+    const toCancel = this.pendingLongPolls.filter((p) => p.path === path)
+    for (const pending of toCancel) {
+      clearTimeout(pending.timeoutId)
+      pending.resolve([])
+    }
+    this.pendingLongPolls = this.pendingLongPolls.filter((p) => p.path !== path)
+  }
+
+  private removePendingLongPoll(pending: PendingLongPoll): void {
+    const index = this.pendingLongPolls.indexOf(pending)
+    if (index !== -1) {
+      this.pendingLongPolls.splice(index, 1)
+    }
+  }
+}

package/src/types.ts

@@ -0,0 +1,183 @@
+/**
+ * Types for the in-memory durable streams test server.
+ */
+
+/**
+ * A single message in a stream.
+ */
+export interface StreamMessage {
+  /**
+   * The raw bytes of the message.
+   */
+  data: Uint8Array
+
+  /**
+   * The offset after this message.
+   * Format: "<read-seq>_<byte-offset>"
+   */
+  offset: string
+
+  /**
+   * Timestamp when the message was appended.
+   */
+  timestamp: number
+}
+
+/**
+ * Stream metadata and data.
+ */
+export interface Stream {
+  /**
+   * The stream URL path (key).
+   */
+  path: string
+
+  /**
+   * Content type of the stream.
+   */
+  contentType?: string
+
+  /**
+   * Messages in the stream.
+   */
+  messages: Array<StreamMessage>
+
+  /**
+   * Current offset (next offset to write to).
+   */
+  currentOffset: string
+
+  /**
+   * Last sequence number for writer coordination.
+   */
+  lastSeq?: string
+
+  /**
+   * TTL in seconds.
+   */
+  ttlSeconds?: number
+
+  /**
+   * Absolute expiry time (ISO 8601).
+   */
+  expiresAt?: string
+
+  /**
+   * Timestamp when the stream was created.
+   */
+  createdAt: number
+}
+
+/**
+ * Event data for stream lifecycle hooks.
+ */
+export interface StreamLifecycleEvent {
+  /**
+   * Type of event.
+   */
+  type: `created` | `deleted`
+
+  /**
+   * Stream path.
+   */
+  path: string
+
+  /**
+   * Content type (only for 'created' events).
+   */
+  contentType?: string
+
+  /**
+   * Timestamp of the event.
+   */
+  timestamp: number
+}
+
+/**
+ * Hook function called when a stream is created or deleted.
+ */
+export type StreamLifecycleHook = (
+  event: StreamLifecycleEvent
+) => void | Promise<void>
+
+/**
+ * Options for creating the test server.
+ */
+export interface TestServerOptions {
+  /**
+   * Port to listen on. Default: 0 (auto-assign).
+   */
+  port?: number
+
+  /**
+   * Host to bind to. Default: "127.0.0.1".
+   */
+  host?: string
+
+  /**
+   * Default long-poll timeout in milliseconds.
+   * Default: 30000 (30 seconds).
+   */
+  longPollTimeout?: number
+
+  /**
+   * Data directory for file-backed storage.
+   * If provided, enables file-backed mode using LMDB and append-only logs.
+   * If omitted, uses in-memory storage.
+   */
+  dataDir?: string
+
+  /**
+   * Hook called when a stream is created.
+   */
+  onStreamCreated?: StreamLifecycleHook
+
+  /**
+   * Hook called when a stream is deleted.
+   */
+  onStreamDeleted?: StreamLifecycleHook
+
+  /**
+   * Enable gzip/deflate compression for responses.
+   * Default: true.
+   */
+  compression?: boolean
+
+  /**
+   * Interval in seconds for cursor calculation.
+   * Used for CDN cache collapsing to prevent infinite cache loops.
+   * Default: 20 seconds.
+   */
+  cursorIntervalSeconds?: number
+
+  /**
+   * Epoch timestamp for cursor interval calculation.
+   * Default: October 9, 2024 00:00:00 UTC.
+   */
+  cursorEpoch?: Date
+}
+
+/**
+ * Pending long-poll request.
+ */
+export interface PendingLongPoll {
+  /**
+   * Stream path.
+   */
+  path: string
+
+  /**
+   * Offset to wait for.
+   */
+  offset: string
+
+  /**
+   * Resolve function.
+   */
+  resolve: (messages: Array<StreamMessage>) => void
+
+  /**
+   * Timeout ID.
+   */
+  timeoutId: ReturnType<typeof setTimeout>
+}