@durable-streams/server 0.3.1 → 0.3.3

package/src/glob.ts

@@ -0,0 +1,70 @@
+/**
+ * Glob pattern matching for webhook subscription patterns.
+ *
+ * Supports:
+ * - `*` matches exactly one path segment
+ * - `**` matches zero or more path segments (recursive)
+ * - Literal segments match exactly
+ */
+
+/**
+ * Match a stream path against a glob pattern.
+ */
+export function globMatch(pattern: string, path: string): boolean {
+  const patternParts = splitPath(pattern)
+  const pathParts = splitPath(path)
+  return matchParts(patternParts, 0, pathParts, 0)
+}
+
+function splitPath(p: string): Array<string> {
+  // Normalize: remove leading/trailing slashes, split on /
+  return p
+    .replace(/^\/+/, ``)
+    .replace(/\/+$/, ``)
+    .split(`/`)
+    .filter((s) => s.length > 0)
+}
+
+function matchParts(
+  pattern: Array<string>,
+  pi: number,
+  path: Array<string>,
+  si: number
+): boolean {
+  while (pi < pattern.length && si < path.length) {
+    const seg = pattern[pi]!
+
+    if (seg === `**`) {
+      // ** matches zero or more segments
+      // Try matching rest of pattern against every possible suffix of path
+      for (let i = si; i <= path.length; i++) {
+        if (matchParts(pattern, pi + 1, path, i)) {
+          return true
+        }
+      }
+      return false
+    }
+
+    if (seg === `*`) {
+      // * matches exactly one segment
+      pi++
+      si++
+      continue
+    }
+
+    // Literal match (also handle %2A as *)
+    const decodedSeg = seg.replace(/%2[Aa]/g, `*`)
+    if (decodedSeg !== path[si]) {
+      return false
+    }
+    pi++
+    si++
+  }
+
+  // Handle trailing ** which matches zero segments
+  while (pi < pattern.length && pattern[pi] === `**`) {
+    pi++
+  }
+
+  return pi === pattern.length && si === path.length
+}

package/src/index.ts

@@ -25,3 +25,17 @@ export type {
   StreamLifecycleEvent,
   StreamLifecycleHook,
 } from "./types"
+export { SubscriptionManager, validateWebhookUrl } from "./subscription-manager"
+export { SubscriptionRoutes } from "./subscription-routes"
+export type {
+  SubscriptionCallbackRequest,
+  SubscriptionCreateInput,
+  SubscriptionError,
+  SubscriptionErrorCode,
+  SubscriptionRecord,
+  SubscriptionStatus,
+  SubscriptionStreamInfo,
+  SubscriptionStreamLink,
+  SubscriptionType,
+} from "./subscription-types"
+export { globMatch } from "./glob"

package/src/log.ts

@@ -0,0 +1,56 @@
+const streamsLogFile = process.env.STREAMS_LOG_FILE
+
+async function appendLogLine(line: string): Promise<void> {
+  if (!streamsLogFile) return
+  const fs = await import(`node:fs/promises`)
+  const path = await import(`node:path`)
+  await fs.mkdir(path.dirname(streamsLogFile), { recursive: true })
+  await fs.appendFile(streamsLogFile, `${line}\n`)
+}
+
+function serializeArg(arg: unknown): string {
+  if (arg instanceof Error) {
+    return arg.stack ?? arg.message
+  }
+  if (typeof arg === `string`) {
+    return arg
+  }
+  try {
+    return JSON.stringify(arg)
+  } catch {
+    return String(arg)
+  }
+}
+
+function write(level: `info` | `warn` | `error`, args: Array<unknown>): void {
+  const line = args.map(serializeArg).join(` `)
+  const formatted = `[${level}] ${line}`
+
+  if (level === `error`) {
+    console.error(formatted)
+  } else if (level === `warn`) {
+    console.warn(formatted)
+  } else {
+    console.info(formatted)
+  }
+
+  void appendLogLine(formatted).catch(() => undefined)
+}
+
+export const serverLog = {
+  info(...args: Array<unknown>): void {
+    write(`info`, args)
+  },
+
+  warn(...args: Array<unknown>): void {
+    write(`warn`, args)
+  },
+
+  error(...args: Array<unknown>): void {
+    write(`error`, args)
+  },
+
+  event(obj: Record<string, unknown>, msg: string): void {
+    write(`info`, [msg, obj])
+  },
+}

package/src/server.ts

@@ -4,47 +4,45 @@
 
 import { createServer } from "node:http"
 import { deflateSync, gzipSync } from "node:zlib"
+import {
+  CURSOR_QUERY_PARAM,
+  LIVE_QUERY_PARAM,
+  OFFSET_QUERY_PARAM,
+  PRODUCER_EPOCH_HEADER,
+  PRODUCER_EXPECTED_SEQ_HEADER,
+  PRODUCER_ID_HEADER,
+  PRODUCER_RECEIVED_SEQ_HEADER,
+  PRODUCER_SEQ_HEADER,
+  SSE_CLOSED_FIELD,
+  SSE_CURSOR_FIELD,
+  SSE_OFFSET_FIELD,
+  STREAM_CLOSED_HEADER,
+  STREAM_CURSOR_HEADER,
+  STREAM_EXPIRES_AT_HEADER,
+  STREAM_OFFSET_HEADER,
+  STREAM_SEQ_HEADER,
+  STREAM_TTL_HEADER,
+  STREAM_UP_TO_DATE_HEADER,
+} from "@durable-streams/client"
 import { StreamStore } from "./store"
 import { FileBackedStreamStore } from "./file-store"
 import { generateResponseCursor } from "./cursor"
+import { SubscriptionManager } from "./subscription-manager"
+import { SubscriptionRoutes } from "./subscription-routes"
+import { serverLog } from "./log"
 import type { CursorOptions } from "./cursor"
 import type { IncomingMessage, Server, ServerResponse } from "node:http"
 import type { StreamLifecycleEvent, TestServerOptions } from "./types"
 
-// Protocol headers (aligned with PROTOCOL.md)
-const STREAM_OFFSET_HEADER = `Stream-Next-Offset`
-const STREAM_CURSOR_HEADER = `Stream-Cursor`
-const STREAM_UP_TO_DATE_HEADER = `Stream-Up-To-Date`
-const STREAM_SEQ_HEADER = `Stream-Seq`
-const STREAM_TTL_HEADER = `Stream-TTL`
-const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`
 const STREAM_SSE_DATA_ENCODING_HEADER = `Stream-SSE-Data-Encoding`
 
-// Idempotent producer headers
-const PRODUCER_ID_HEADER = `Producer-Id`
-const PRODUCER_EPOCH_HEADER = `Producer-Epoch`
-const PRODUCER_SEQ_HEADER = `Producer-Seq`
-const PRODUCER_EXPECTED_SEQ_HEADER = `Producer-Expected-Seq`
-const PRODUCER_RECEIVED_SEQ_HEADER = `Producer-Received-Seq`
-
 // SSE control event fields (Protocol Section 5.7)
-const SSE_OFFSET_FIELD = `streamNextOffset`
-const SSE_CURSOR_FIELD = `streamCursor`
 const SSE_UP_TO_DATE_FIELD = `upToDate`
-const SSE_CLOSED_FIELD = `streamClosed`
-
-// Stream closure header
-const STREAM_CLOSED_HEADER = `Stream-Closed`
 
 // Fork headers (request headers only — not set on responses)
 const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`
 const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`
 
-// Query params
-const OFFSET_QUERY_PARAM = `offset`
-const LIVE_QUERY_PARAM = `live`
-const CURSOR_QUERY_PARAM = `cursor`
-
 /**
  * Encode data for SSE format.
  * Per SSE spec, each line in the payload needs its own "data:" prefix.
@@ -161,6 +159,7 @@ export class DurableStreamTestServer {
       | `compression`
       | `cursorIntervalSeconds`
       | `cursorEpoch`
+      | `webhooks`
     >
   > & {
     dataDir?: string
@@ -168,12 +167,15 @@ export class DurableStreamTestServer {
     onStreamDeleted?: (event: StreamLifecycleEvent) => void | Promise<void>
     compression: boolean
     cursorOptions: CursorOptions
+    webhooks: boolean
   }
   private _url: string | null = null
   private activeSSEResponses = new Set<ServerResponse>()
   private isShuttingDown = false
   /** Injected faults for testing retry/resilience */
   private injectedFaults = new Map<string, InjectedFault>()
+  private subscriptionManager: SubscriptionManager | null = null
+  private subscriptionRoutes: SubscriptionRoutes | null = null
 
   constructor(options: TestServerOptions = {}) {
     // Choose store based on dataDir option
@@ -197,6 +199,7 @@ export class DurableStreamTestServer {
         intervalSeconds: options.cursorIntervalSeconds,
         epoch: options.cursorEpoch,
       },
+      webhooks: options.webhooks ?? false,
     }
   }
 
@@ -211,7 +214,7 @@ export class DurableStreamTestServer {
     return new Promise((resolve, reject) => {
       this.server = createServer((req, res) => {
         this.handleRequest(req, res).catch((err) => {
-          console.error(`Request error:`, err)
+          serverLog.error(`Request error:`, err)
           if (!res.headersSent) {
             res.writeHead(500, { "content-type": `text/plain` })
             res.end(`Internal server error`)
@@ -228,6 +231,15 @@ export class DurableStreamTestServer {
         } else if (addr) {
           this._url = `http://${this.options.host}:${addr.port}`
         }
+
+        this.subscriptionManager = new SubscriptionManager({
+          callbackBaseUrl: this._url!,
+          streamStore: this.store,
+          webhooksEnabled: this.options.webhooks,
+        })
+        this.subscriptionRoutes = new SubscriptionRoutes(
+          this.subscriptionManager
+        )
         resolve(this._url!)
       })
     })
@@ -244,6 +256,12 @@ export class DurableStreamTestServer {
     // Mark as shutting down to stop SSE handlers
     this.isShuttingDown = true
 
+    if (this.subscriptionManager) {
+      this.subscriptionManager.shutdown()
+      this.subscriptionManager = null
+      this.subscriptionRoutes = null
+    }
+
     // Cancel all pending long-polls and SSE waits to unblock connection handlers
     if (`cancelAllWaits` in this.store) {
       ;(this.store as { cancelAllWaits: () => void }).cancelAllWaits()
@@ -492,6 +510,16 @@ export class DurableStreamTestServer {
       }
     }
 
+    if (this.subscriptionRoutes && method) {
+      const handled = await this.subscriptionRoutes.handleRequest(
+        method,
+        path,
+        req,
+        res
+      )
+      if (handled) return
+    }
+
     try {
       switch (method) {
         case `PUT`:
@@ -562,13 +590,24 @@ export class DurableStreamTestServer {
   ): Promise<void> {
     let contentType = req.headers[`content-type`]
 
-    // Sanitize content-type: if empty or invalid, use default
+    // Parse fork headers (must come before content-type sanitization so
+    // forks can fall through to the store's content-type inheritance)
+    const forkedFromHeader = req.headers[
+      STREAM_FORKED_FROM_HEADER.toLowerCase()
+    ] as string | undefined
+    const forkOffsetHeader = req.headers[
+      STREAM_FORK_OFFSET_HEADER.toLowerCase()
+    ] as string | undefined
+
+    // Sanitize content-type: if empty or invalid, use default — but only
+    // for non-fork creates. For forks, an omitted Content-Type means "inherit
+    // from source", which is resolved by the store.
     if (
       !contentType ||
       contentType.trim() === `` ||
       !/^[\w-]+\/[\w-]+/.test(contentType)
     ) {
-      contentType = `application/octet-stream`
+      contentType = forkedFromHeader ? undefined : `application/octet-stream`
     }
 
     const ttlHeader = req.headers[STREAM_TTL_HEADER.toLowerCase()] as
@@ -582,14 +621,6 @@ export class DurableStreamTestServer {
     const closedHeader = req.headers[STREAM_CLOSED_HEADER.toLowerCase()]
     const createClosed = closedHeader === `true`
 
-    // Parse fork headers
-    const forkedFromHeader = req.headers[
-      STREAM_FORKED_FROM_HEADER.toLowerCase()
-    ] as string | undefined
-    const forkOffsetHeader = req.headers[
-      STREAM_FORK_OFFSET_HEADER.toLowerCase()
-    ] as string | undefined
-
     // Validate TTL and Expires-At headers
     if (ttlHeader && expiresAtHeader) {
       res.writeHead(400, { "content-type": `text/plain` })
@@ -681,6 +712,8 @@ export class DurableStreamTestServer {
     }
 
     const stream = this.store.get(path)!
+    const resolvedContentType =
+      stream.contentType ?? contentType ?? `application/octet-stream`
 
     // Call lifecycle hook for new streams
     if (isNew && this.options.onStreamCreated) {
@@ -688,15 +721,19 @@ export class DurableStreamTestServer {
         this.options.onStreamCreated({
           type: `created`,
           path,
-          contentType: stream.contentType ?? contentType,
+          contentType: resolvedContentType,
           timestamp: Date.now(),
         })
       )
     }
 
+    if (isNew && body.length > 0) {
+      await this.notifyStreamAppend(path)
+    }
+
     // Return 201 for new streams, 200 for idempotent creates
     const headers: Record<string, string> = {
-      "content-type": stream.contentType ?? contentType,
+      "content-type": resolvedContentType,
       [STREAM_OFFSET_HEADER]: stream.currentOffset,
     }
 
@@ -1182,8 +1219,10 @@ export class DurableStreamTestServer {
         // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
         if (this.isShuttingDown || !isConnected) break
 
-        // Check if stream was closed during wait
-        if (result.streamClosed) {
+        // Check if stream was closed during wait. If the close also appended
+        // final data, let the next loop iteration deliver those messages
+        // before emitting the streamClosed control event.
+        if (result.streamClosed && result.messages.length === 0) {
           const finalControlData: Record<string, string | boolean> = {
             [SSE_OFFSET_FIELD]: currentOffset,
             [SSE_CLOSED_FIELD]: true,
@@ -1508,6 +1547,8 @@ export class DurableStreamTestServer {
         const statusCode = producerId !== undefined ? 200 : 204
         res.writeHead(statusCode, responseHeaders)
         res.end()
+
+        await this.notifyStreamAppend(path)
         return
       }
 
@@ -1570,6 +1611,17 @@ export class DurableStreamTestServer {
     }
     res.writeHead(204, responseHeaders)
     res.end()
+
+    await this.notifyStreamAppend(path)
+  }
+
+  private async notifyStreamAppend(path: string): Promise<void> {
+    if (!this.subscriptionManager) return
+    try {
+      await this.subscriptionManager.onStreamAppend(path)
+    } catch (err) {
+      serverLog.error(`[server] subscription append hook failed:`, err)
+    }
   }
 
   /**
@@ -1602,6 +1654,10 @@ export class DurableStreamTestServer {
       )
     }
 
+    if (this.subscriptionManager) {
+      this.subscriptionManager.onStreamDeleted(path)
+    }
+
     res.writeHead(204)
     res.end()
   }

package/src/store.ts

@@ -86,6 +86,59 @@ export function formatJsonResponse(data: Uint8Array): Uint8Array {
   return new TextEncoder().encode(wrapped)
 }
 
+function decodeStoredJsonMessage(data: Uint8Array): string {
+  let text = new TextDecoder().decode(data).trimEnd()
+  if (text.endsWith(`,`)) {
+    text = text.slice(0, -1)
+  }
+  return text
+}
+
+function enrichJsonValueWithOffset(parsed: unknown, offset: string): string {
+  if (!parsed || typeof parsed !== `object` || Array.isArray(parsed)) {
+    return JSON.stringify(parsed)
+  }
+
+  const candidate = parsed as {
+    headers?: Record<string, unknown>
+  }
+  const headers = candidate.headers
+
+  if (!headers || typeof headers !== `object`) {
+    return JSON.stringify(parsed)
+  }
+
+  const isStateChange = typeof headers.operation === `string`
+  const isStateControl = typeof headers.control === `string`
+  if (!isStateChange && !isStateControl) {
+    return JSON.stringify(parsed)
+  }
+
+  return JSON.stringify({
+    ...candidate,
+    headers: {
+      ...headers,
+      offset,
+    },
+  })
+}
+
+export function formatJsonMessages(messages: Array<StreamMessage>): Uint8Array {
+  if (messages.length === 0) {
+    return new TextEncoder().encode(`[]`)
+  }
+
+  const items = messages.flatMap((message) => {
+    const rawFragment = decodeStoredJsonMessage(message.data)
+    const parsed = JSON.parse(`[${rawFragment}]`) as Array<unknown>
+    return parsed.map((value) =>
+      enrichJsonValueWithOffset(value, message.offset)
+    )
+  })
+
+  return new TextEncoder().encode(`[${items.join(`,`)}]`)
+}
+
 /**
  * In-memory store for durable streams.
  */
@@ -723,13 +776,17 @@ export class StreamStore {
           seq: options.producerSeq!,
         }
       }
-      // Notify pending long-polls that stream is closed
-      this.notifyLongPollsClosed(path)
     }
 
-    // Notify any pending long-polls of new messages
+    // Notify pending long-polls of new messages before empty close signals.
+    // Append-and-close must deliver the final message with streamClosed
+    // metadata instead of waking readers with an empty close event first.
     this.notifyLongPolls(path)
 
+    if (options.close) {
+      this.notifyLongPollsClosed(path)
+    }
+
     // Return AppendResult if producer headers were used or stream was closed
     if (producerResult || options.close) {
       return {
@@ -1052,6 +1109,10 @@ export class StreamStore {
       throw new Error(`Stream not found: ${path}`)
     }
 
+    if (normalizeContentType(stream.contentType) === `application/json`) {
+      return formatJsonMessages(messages)
+    }
+
     // Concatenate all message data
     const totalSize = messages.reduce((sum, m) => sum + m.data.length, 0)
     const concatenated = new Uint8Array(totalSize)
@@ -1061,11 +1122,6 @@ export class StreamStore {
       offset += msg.data.length
     }
 
-    // For JSON mode, wrap in array brackets
-    if (normalizeContentType(stream.contentType) === `application/json`) {
-      return formatJsonResponse(concatenated)
-    }
-
     return concatenated
   }
 
@@ -1200,8 +1256,8 @@ export class StreamStore {
     const readSeq = parts[0]!
     const byteOffset = parts[1]!
 
-    // Calculate new offset with zero-padding for lexicographic sorting
-    const newByteOffset = byteOffset + processedData.length
+    const FRAME_OVERHEAD = 5 // 4-byte length prefix + 1-byte newline
+    const newByteOffset = byteOffset + FRAME_OVERHEAD + processedData.length
     const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`
 
     const message: StreamMessage = {