@durable-streams/server 0.3.2 → 0.3.4

package/src/log.ts

@@ -0,0 +1,56 @@
+const streamsLogFile = process.env.STREAMS_LOG_FILE
+
+async function appendLogLine(line: string): Promise<void> {
+  if (!streamsLogFile) return
+  const fs = await import(`node:fs/promises`)
+  const path = await import(`node:path`)
+  await fs.mkdir(path.dirname(streamsLogFile), { recursive: true })
+  await fs.appendFile(streamsLogFile, `${line}\n`)
+}
+
+function serializeArg(arg: unknown): string {
+  if (arg instanceof Error) {
+    return arg.stack ?? arg.message
+  }
+  if (typeof arg === `string`) {
+    return arg
+  }
+  try {
+    return JSON.stringify(arg)
+  } catch {
+    return String(arg)
+  }
+}
+
+function write(level: `info` | `warn` | `error`, args: Array<unknown>): void {
+  const line = args.map(serializeArg).join(` `)
+  const formatted = `[${level}] ${line}`
+
+  if (level === `error`) {
+    console.error(formatted)
+  } else if (level === `warn`) {
+    console.warn(formatted)
+  } else {
+    console.info(formatted)
+  }
+
+  void appendLogLine(formatted).catch(() => undefined)
+}
+
+export const serverLog = {
+  info(...args: Array<unknown>): void {
+    write(`info`, args)
+  },
+
+  warn(...args: Array<unknown>): void {
+    write(`warn`, args)
+  },
+
+  error(...args: Array<unknown>): void {
+    write(`error`, args)
+  },
+
+  event(obj: Record<string, unknown>, msg: string): void {
+    write(`info`, [msg, obj])
+  },
+}

package/src/server.ts

@@ -4,47 +4,45 @@
 
 import { createServer } from "node:http"
 import { deflateSync, gzipSync } from "node:zlib"
+import {
+  CURSOR_QUERY_PARAM,
+  LIVE_QUERY_PARAM,
+  OFFSET_QUERY_PARAM,
+  PRODUCER_EPOCH_HEADER,
+  PRODUCER_EXPECTED_SEQ_HEADER,
+  PRODUCER_ID_HEADER,
+  PRODUCER_RECEIVED_SEQ_HEADER,
+  PRODUCER_SEQ_HEADER,
+  SSE_CLOSED_FIELD,
+  SSE_CURSOR_FIELD,
+  SSE_OFFSET_FIELD,
+  STREAM_CLOSED_HEADER,
+  STREAM_CURSOR_HEADER,
+  STREAM_EXPIRES_AT_HEADER,
+  STREAM_OFFSET_HEADER,
+  STREAM_SEQ_HEADER,
+  STREAM_TTL_HEADER,
+  STREAM_UP_TO_DATE_HEADER,
+} from "@durable-streams/client"
 import { StreamStore } from "./store"
 import { FileBackedStreamStore } from "./file-store"
 import { generateResponseCursor } from "./cursor"
+import { SubscriptionManager } from "./subscription-manager"
+import { SubscriptionRoutes } from "./subscription-routes"
+import { serverLog } from "./log"
 import type { CursorOptions } from "./cursor"
 import type { IncomingMessage, Server, ServerResponse } from "node:http"
 import type { StreamLifecycleEvent, TestServerOptions } from "./types"
 
-// Protocol headers (aligned with PROTOCOL.md)
-const STREAM_OFFSET_HEADER = `Stream-Next-Offset`
-const STREAM_CURSOR_HEADER = `Stream-Cursor`
-const STREAM_UP_TO_DATE_HEADER = `Stream-Up-To-Date`
-const STREAM_SEQ_HEADER = `Stream-Seq`
-const STREAM_TTL_HEADER = `Stream-TTL`
-const STREAM_EXPIRES_AT_HEADER = `Stream-Expires-At`
 const STREAM_SSE_DATA_ENCODING_HEADER = `Stream-SSE-Data-Encoding`
 
-// Idempotent producer headers
-const PRODUCER_ID_HEADER = `Producer-Id`
-const PRODUCER_EPOCH_HEADER = `Producer-Epoch`
-const PRODUCER_SEQ_HEADER = `Producer-Seq`
-const PRODUCER_EXPECTED_SEQ_HEADER = `Producer-Expected-Seq`
-const PRODUCER_RECEIVED_SEQ_HEADER = `Producer-Received-Seq`
-
 // SSE control event fields (Protocol Section 5.7)
-const SSE_OFFSET_FIELD = `streamNextOffset`
-const SSE_CURSOR_FIELD = `streamCursor`
 const SSE_UP_TO_DATE_FIELD = `upToDate`
-const SSE_CLOSED_FIELD = `streamClosed`
-
-// Stream closure header
-const STREAM_CLOSED_HEADER = `Stream-Closed`
 
 // Fork headers (request headers only — not set on responses)
 const STREAM_FORKED_FROM_HEADER = `Stream-Forked-From`
 const STREAM_FORK_OFFSET_HEADER = `Stream-Fork-Offset`
 
-// Query params
-const OFFSET_QUERY_PARAM = `offset`
-const LIVE_QUERY_PARAM = `live`
-const CURSOR_QUERY_PARAM = `cursor`
-
 /**
  * Encode data for SSE format.
  * Per SSE spec, each line in the payload needs its own "data:" prefix.
@@ -161,6 +159,7 @@ export class DurableStreamTestServer {
       | `compression`
       | `cursorIntervalSeconds`
       | `cursorEpoch`
+      | `webhooks`
     >
   > & {
     dataDir?: string
@@ -168,12 +167,15 @@ export class DurableStreamTestServer {
     onStreamDeleted?: (event: StreamLifecycleEvent) => void | Promise<void>
     compression: boolean
     cursorOptions: CursorOptions
+    webhooks: boolean
   }
   private _url: string | null = null
   private activeSSEResponses = new Set<ServerResponse>()
   private isShuttingDown = false
   /** Injected faults for testing retry/resilience */
   private injectedFaults = new Map<string, InjectedFault>()
+  private subscriptionManager: SubscriptionManager | null = null
+  private subscriptionRoutes: SubscriptionRoutes | null = null
 
   constructor(options: TestServerOptions = {}) {
     // Choose store based on dataDir option
@@ -197,6 +199,7 @@ export class DurableStreamTestServer {
         intervalSeconds: options.cursorIntervalSeconds,
         epoch: options.cursorEpoch,
       },
+      webhooks: options.webhooks ?? false,
     }
   }
 
@@ -211,7 +214,7 @@ export class DurableStreamTestServer {
     return new Promise((resolve, reject) => {
       this.server = createServer((req, res) => {
         this.handleRequest(req, res).catch((err) => {
-          console.error(`Request error:`, err)
+          serverLog.error(`Request error:`, err)
           if (!res.headersSent) {
             res.writeHead(500, { "content-type": `text/plain` })
             res.end(`Internal server error`)
@@ -228,6 +231,15 @@ export class DurableStreamTestServer {
         } else if (addr) {
           this._url = `http://${this.options.host}:${addr.port}`
         }
+
+        this.subscriptionManager = new SubscriptionManager({
+          callbackBaseUrl: this._url!,
+          streamStore: this.store,
+          webhooksEnabled: this.options.webhooks,
+        })
+        this.subscriptionRoutes = new SubscriptionRoutes(
+          this.subscriptionManager
+        )
         resolve(this._url!)
       })
     })
@@ -244,6 +256,12 @@ export class DurableStreamTestServer {
     // Mark as shutting down to stop SSE handlers
     this.isShuttingDown = true
 
+    if (this.subscriptionManager) {
+      this.subscriptionManager.shutdown()
+      this.subscriptionManager = null
+      this.subscriptionRoutes = null
+    }
+
     // Cancel all pending long-polls and SSE waits to unblock connection handlers
     if (`cancelAllWaits` in this.store) {
       ;(this.store as { cancelAllWaits: () => void }).cancelAllWaits()
@@ -492,6 +510,16 @@ export class DurableStreamTestServer {
       }
     }
 
+    if (this.subscriptionRoutes && method) {
+      const handled = await this.subscriptionRoutes.handleRequest(
+        method,
+        path,
+        req,
+        res
+      )
+      if (handled) return
+    }
+
     try {
       switch (method) {
         case `PUT`:
@@ -699,6 +727,10 @@ export class DurableStreamTestServer {
       )
     }
 
+    if (isNew && body.length > 0) {
+      await this.notifyStreamAppend(path)
+    }
+
     // Return 201 for new streams, 200 for idempotent creates
     const headers: Record<string, string> = {
       "content-type": resolvedContentType,
@@ -1515,6 +1547,8 @@ export class DurableStreamTestServer {
         const statusCode = producerId !== undefined ? 200 : 204
         res.writeHead(statusCode, responseHeaders)
         res.end()
+
+        await this.notifyStreamAppend(path)
         return
       }
 
@@ -1577,6 +1611,17 @@ export class DurableStreamTestServer {
     }
     res.writeHead(204, responseHeaders)
     res.end()
+
+    await this.notifyStreamAppend(path)
+  }
+
+  private async notifyStreamAppend(path: string): Promise<void> {
+    if (!this.subscriptionManager) return
+    try {
+      await this.subscriptionManager.onStreamAppend(path)
+    } catch (err) {
+      serverLog.error(`[server] subscription append hook failed:`, err)
+    }
   }
 
   /**
@@ -1609,6 +1654,10 @@ export class DurableStreamTestServer {
       )
     }
 
+    if (this.subscriptionManager) {
+      this.subscriptionManager.onStreamDeleted(path)
+    }
+
     res.writeHead(204)
     res.end()
   }

package/src/store.ts

@@ -86,6 +86,59 @@ export function formatJsonResponse(data: Uint8Array): Uint8Array {
   return new TextEncoder().encode(wrapped)
 }
 
+function decodeStoredJsonMessage(data: Uint8Array): string {
+  let text = new TextDecoder().decode(data).trimEnd()
+  if (text.endsWith(`,`)) {
+    text = text.slice(0, -1)
+  }
+  return text
+}
+
+function enrichJsonValueWithOffset(parsed: unknown, offset: string): string {
+  if (!parsed || typeof parsed !== `object` || Array.isArray(parsed)) {
+    return JSON.stringify(parsed)
+  }
+
+  const candidate = parsed as {
+    headers?: Record<string, unknown>
+  }
+  const headers = candidate.headers
+
+  if (!headers || typeof headers !== `object`) {
+    return JSON.stringify(parsed)
+  }
+
+  const isStateChange = typeof headers.operation === `string`
+  const isStateControl = typeof headers.control === `string`
+  if (!isStateChange && !isStateControl) {
+    return JSON.stringify(parsed)
+  }
+
+  return JSON.stringify({
+    ...candidate,
+    headers: {
+      ...headers,
+      offset,
+    },
+  })
+}
+
+export function formatJsonMessages(messages: Array<StreamMessage>): Uint8Array {
+  if (messages.length === 0) {
+    return new TextEncoder().encode(`[]`)
+  }
+
+  const items = messages.flatMap((message) => {
+    const rawFragment = decodeStoredJsonMessage(message.data)
+    const parsed = JSON.parse(`[${rawFragment}]`) as Array<unknown>
+    return parsed.map((value) =>
+      enrichJsonValueWithOffset(value, message.offset)
+    )
+  })
+
+  return new TextEncoder().encode(`[${items.join(`,`)}]`)
+}
+
 /**
  * In-memory store for durable streams.
  */
@@ -1056,6 +1109,10 @@ export class StreamStore {
       throw new Error(`Stream not found: ${path}`)
     }
 
+    if (normalizeContentType(stream.contentType) === `application/json`) {
+      return formatJsonMessages(messages)
+    }
+
     // Concatenate all message data
     const totalSize = messages.reduce((sum, m) => sum + m.data.length, 0)
     const concatenated = new Uint8Array(totalSize)
@@ -1065,11 +1122,6 @@ export class StreamStore {
       offset += msg.data.length
     }
 
-    // For JSON mode, wrap in array brackets
-    if (normalizeContentType(stream.contentType) === `application/json`) {
-      return formatJsonResponse(concatenated)
-    }
-
     return concatenated
   }
 
@@ -1204,8 +1256,8 @@ export class StreamStore {
     const readSeq = parts[0]!
     const byteOffset = parts[1]!
 
-    // Calculate new offset with zero-padding for lexicographic sorting
-    const newByteOffset = byteOffset + processedData.length
+    const FRAME_OVERHEAD = 5 // 4-byte length prefix + 1-byte newline
+    const newByteOffset = byteOffset + FRAME_OVERHEAD + processedData.length
     const newOffset = `${String(readSeq).padStart(16, `0`)}_${String(newByteOffset).padStart(16, `0`)}`
 
     const message: StreamMessage = {