@durable-streams/client 0.1.5 → 0.2.1

package/src/stream-api.ts

@@ -7,14 +7,21 @@
 import {
   LIVE_QUERY_PARAM,
   OFFSET_QUERY_PARAM,
+  STREAM_CLOSED_HEADER,
   STREAM_CURSOR_HEADER,
   STREAM_OFFSET_HEADER,
+  STREAM_SSE_DATA_ENCODING_HEADER,
   STREAM_UP_TO_DATE_HEADER,
 } from "./constants"
 import { DurableStreamError, FetchBackoffAbortError } from "./error"
 import { BackoffDefaults, createFetchWithBackoff } from "./fetch"
 import { StreamResponseImpl } from "./response"
-import { handleErrorResponse, resolveHeaders, resolveParams } from "./utils"
+import {
+  handleErrorResponse,
+  resolveHeaders,
+  resolveParams,
+  warnIfUsingHttpInBrowser,
+} from "./utils"
 import type { LiveMode, Offset, StreamOptions, StreamResponse } from "./types"
 
 /**
@@ -41,7 +48,7 @@ import type { LiveMode, Offset, StreamOptions, StreamResponse } from "./types"
  *   url,
  *   auth,
  *   offset: savedOffset,
- *   live: "auto",
+ *   live: true,
  * })
  * live.subscribeJson(async (batch) => {
  *   for (const item of batch.items) {
@@ -119,6 +126,9 @@ async function streamInternal<TJson = unknown>(
   // Normalize URL
   const url = options.url instanceof URL ? options.url.toString() : options.url
 
+  // Warn if using HTTP in browser (can cause connection limit issues)
+  warnIfUsingHttpInBrowser(url, options.warnOnHttp)
+
   // Build the first request
   const fetchUrl = new URL(url)
 
@@ -127,7 +137,8 @@ async function streamInternal<TJson = unknown>(
   fetchUrl.searchParams.set(OFFSET_QUERY_PARAM, startOffset)
 
   // Set live query param for explicit modes
-  const live: LiveMode = options.live ?? `auto`
+  // true means auto-select (no query param, handled by consumption method)
+  const live: LiveMode = options.live ?? true
   if (live === `long-poll` || live === `sse`) {
     fetchUrl.searchParams.set(LIVE_QUERY_PARAM, live)
   }
@@ -181,12 +192,21 @@ async function streamInternal<TJson = unknown>(
   const initialCursor =
     firstResponse.headers.get(STREAM_CURSOR_HEADER) ?? undefined
   const initialUpToDate = firstResponse.headers.has(STREAM_UP_TO_DATE_HEADER)
+  const initialStreamClosed =
+    firstResponse.headers.get(STREAM_CLOSED_HEADER)?.toLowerCase() === `true`
 
   // Determine if JSON mode
   const isJsonMode =
     options.json === true ||
     (contentType?.includes(`application/json`) ?? false)
 
+  // Detect SSE data encoding from response header (server auto-sets for binary streams)
+  const sseDataEncoding = firstResponse.headers.get(
+    STREAM_SSE_DATA_ENCODING_HEADER
+  )
+  const encoding =
+    sseDataEncoding === `base64` ? (`base64` as const) : undefined
+
   // Create the fetch function for subsequent requests
   const fetchNext = async (
     offset: Offset,
@@ -197,15 +217,13 @@ async function streamInternal<TJson = unknown>(
     const nextUrl = new URL(url)
     nextUrl.searchParams.set(OFFSET_QUERY_PARAM, offset)
 
-    // For subsequent requests in auto mode, use long-poll
-    // BUT: if we're resuming from a paused state, don't set live mode
-    // to avoid a long-poll that holds for 20sec - we want an immediate response
-    // so the UI can show "connected" status quickly
+    // For subsequent requests, set live mode unless resuming from pause
+    // (resuming from pause needs immediate response for UI status)
     if (!resumingFromPause) {
-      if (live === `auto` || live === `long-poll`) {
-        nextUrl.searchParams.set(LIVE_QUERY_PARAM, `long-poll`)
-      } else if (live === `sse`) {
+      if (live === `sse`) {
         nextUrl.searchParams.set(LIVE_QUERY_PARAM, `sse`)
+      } else if (live === true || live === `long-poll`) {
+        nextUrl.searchParams.set(LIVE_QUERY_PARAM, `long-poll`)
       }
     }
 
@@ -281,10 +299,12 @@ async function streamInternal<TJson = unknown>(
     initialOffset,
     initialCursor,
     initialUpToDate,
+    initialStreamClosed,
     firstResponse,
     abortController,
     fetchNext,
     startSSE,
     sseResilience: options.sseResilience,
+    encoding,
   })
 }

package/src/stream.ts

@@ -7,12 +7,13 @@
 import fastq from "fastq"
 
 import {
-  DurableStreamError,
   InvalidSignalError,
   MissingStreamUrlError,
+  StreamClosedError,
 } from "./error"
+import { IdempotentProducer } from "./idempotent-producer"
 import {
-  SSE_COMPATIBLE_CONTENT_TYPES,
+  STREAM_CLOSED_HEADER,
   STREAM_EXPIRES_AT_HEADER,
   STREAM_OFFSET_HEADER,
   STREAM_SEQ_HEADER,
@@ -34,9 +35,12 @@ import type { BackoffOptions } from "./fetch"
 import type { queueAsPromised } from "fastq"
 import type {
   AppendOptions,
+  CloseOptions,
+  CloseResult,
   CreateOptions,
   HeadResult,
   HeadersRecord,
+  IdempotentProducerOptions,
   MaybePromise,
   ParamsRecord,
   StreamErrorHandler,
@@ -49,7 +53,7 @@ import type {
  * Queued message for batching.
  */
 interface QueuedMessage {
-  data: unknown
+  data: Uint8Array | string
   seq?: string
   contentType?: string
   signal?: AbortSignal
@@ -71,10 +75,7 @@ function normalizeContentType(contentType: string | undefined): string {
  */
 function isPromiseLike(value: unknown): value is PromiseLike<unknown> {
   return (
-    value !== null &&
-    typeof value === `object` &&
-    `then` in value &&
-    typeof (value as PromiseLike<unknown>).then === `function`
+    value != null && typeof (value as PromiseLike<unknown>).then === `function`
   )
 }
 
@@ -121,7 +122,7 @@ export interface DurableStreamOptions extends StreamHandleOptions {
  * });
  *
  * // Write data
- * await stream.append({ message: "hello" });
+ * await stream.append(JSON.stringify({ message: "hello" }));
  *
  * // Read with the new API
  * const res = await stream.stream<{ message: string }>();
@@ -205,6 +206,7 @@ export class DurableStream {
       ttlSeconds: opts.ttlSeconds,
       expiresAt: opts.expiresAt,
       body: opts.body,
+      closed: opts.closed,
     })
     return stream
   }
@@ -270,6 +272,8 @@ export class DurableStream {
     const offset = response.headers.get(STREAM_OFFSET_HEADER) ?? undefined
     const etag = response.headers.get(`etag`) ?? undefined
     const cacheControl = response.headers.get(`cache-control`) ?? undefined
+    const streamClosed =
+      response.headers.get(STREAM_CLOSED_HEADER)?.toLowerCase() === `true`
 
     // Update instance contentType
     if (contentType) {
@@ -282,6 +286,7 @@ export class DurableStream {
       offset,
       etag,
       cacheControl,
+      streamClosed,
     }
   }
 
@@ -301,6 +306,9 @@ export class DurableStream {
     if (opts?.expiresAt) {
       requestHeaders[STREAM_EXPIRES_AT_HEADER] = opts.expiresAt
     }
+    if (opts?.closed) {
+      requestHeaders[STREAM_CLOSED_HEADER] = `true`
+    }
 
     const body = encodeBody(opts?.body)
 
@@ -343,6 +351,84 @@ export class DurableStream {
     }
   }
 
+  /**
+   * Close the stream, optionally with a final message.
+   *
+   * After closing:
+   * - No further appends are permitted (server returns 409)
+   * - Readers can observe the closed state and treat it as EOF
+   * - The stream's data remains fully readable
+   *
+   * Closing is:
+   * - **Durable**: The closed state is persisted
+   * - **Monotonic**: Once closed, a stream cannot be reopened
+   *
+   * **Idempotency:**
+   * - `close()` without body: Idempotent — safe to call multiple times
+   * - `close({ body })` with body: NOT idempotent — throws `StreamClosedError`
+   *   if stream is already closed (use `IdempotentProducer.close()` for
+   *   idempotent close-with-body semantics)
+   *
+   * @returns CloseResult with the final offset
+   * @throws StreamClosedError if called with body on an already-closed stream
+   */
+  async close(opts?: CloseOptions): Promise<CloseResult> {
+    const { requestHeaders, fetchUrl } = await this.#buildRequest()
+
+    const contentType =
+      opts?.contentType ?? this.#options.contentType ?? this.contentType
+    if (contentType) {
+      requestHeaders[`content-type`] = contentType
+    }
+
+    // Always send Stream-Closed: true header for close operation
+    requestHeaders[STREAM_CLOSED_HEADER] = `true`
+
+    // For JSON mode with body, wrap in array
+    let body: BodyInit | undefined
+    if (opts?.body !== undefined) {
+      const isJson = normalizeContentType(contentType) === `application/json`
+      if (isJson) {
+        const bodyStr =
+          typeof opts.body === `string`
+            ? opts.body
+            : new TextDecoder().decode(opts.body)
+        body = `[${bodyStr}]`
+      } else {
+        body =
+          typeof opts.body === `string`
+            ? opts.body
+            : (opts.body as unknown as BodyInit)
+      }
+    }
+
+    const response = await this.#fetchClient(fetchUrl.toString(), {
+      method: `POST`,
+      headers: requestHeaders,
+      body,
+      signal: opts?.signal ?? this.#options.signal,
+    })
+
+    // Check for 409 Conflict with Stream-Closed header
+    if (response.status === 409) {
+      const isClosed =
+        response.headers.get(STREAM_CLOSED_HEADER)?.toLowerCase() === `true`
+      if (isClosed) {
+        const finalOffset =
+          response.headers.get(STREAM_OFFSET_HEADER) ?? undefined
+        throw new StreamClosedError(this.url, finalOffset)
+      }
+    }
+
+    if (!response.ok) {
+      await handleErrorResponse(response, this.url)
+    }
+
+    const finalOffset = response.headers.get(STREAM_OFFSET_HEADER) ?? ``
+
+    return { finalOffset }
+  }
+
   /**
    * Append a single payload to the stream.
    *
@@ -350,23 +436,27 @@ export class DurableStream {
    * a POST is in-flight will be batched together into a single request.
    * This significantly improves throughput for high-frequency writes.
    *
-   * - `body` may be Uint8Array, string, or any JSON-serializable value (for JSON streams).
-   * - `body` may also be a Promise that resolves to any of the above types.
+   * - `body` must be string or Uint8Array.
+   * - For JSON streams, pass pre-serialized JSON strings.
+   * - `body` may also be a Promise that resolves to string or Uint8Array.
    * - Strings are encoded as UTF-8.
    * - `seq` (if provided) is sent as stream-seq (writer coordination).
    *
    * @example
    * ```typescript
-   * // Direct value
-   * await stream.append({ message: "hello" });
+   * // JSON stream - pass pre-serialized JSON
+   * await stream.append(JSON.stringify({ message: "hello" }));
+   *
+   * // Byte stream
+   * await stream.append("raw text data");
+   * await stream.append(new Uint8Array([1, 2, 3]));
    *
    * // Promise value - awaited before buffering
    * await stream.append(fetchData());
-   * await stream.append(Promise.all([a, b, c]));
    * ```
    */
   async append(
-    body: BodyInit | Uint8Array | string | unknown,
+    body: Uint8Array | string | Promise<Uint8Array | string>,
     opts?: AppendOptions
   ): Promise<void> {
     // Await promises before buffering
@@ -382,7 +472,7 @@ export class DurableStream {
    * Direct append without batching (used when batching is disabled).
    */
   async #appendDirect(
-    body: BodyInit | Uint8Array | string | unknown,
+    body: Uint8Array | string,
     opts?: AppendOptions
   ): Promise<void> {
     const { requestHeaders, fetchUrl } = await this.#buildRequest()
@@ -398,9 +488,26 @@ export class DurableStream {
     }
 
     // For JSON mode, wrap body in array to match protocol (server flattens one level)
+    // Input is pre-serialized JSON string
     const isJson = normalizeContentType(contentType) === `application/json`
-    const bodyToEncode = isJson ? [body] : body
-    const encodedBody = encodeBody(bodyToEncode)
+    let encodedBody: BodyInit
+    if (isJson) {
+      // JSON mode: decode as UTF-8 string and wrap in array
+      const bodyStr =
+        typeof body === `string` ? body : new TextDecoder().decode(body)
+      encodedBody = `[${bodyStr}]`
+    } else {
+      // Binary mode: preserve raw bytes
+      // Use ArrayBuffer for cross-platform BodyInit compatibility
+      if (typeof body === `string`) {
+        encodedBody = body
+      } else {
+        encodedBody = body.buffer.slice(
+          body.byteOffset,
+          body.byteOffset + body.byteLength
+        ) as ArrayBuffer
+      }
+    }
 
     const response = await this.#fetchClient(fetchUrl.toString(), {
       method: `POST`,
@@ -418,7 +525,7 @@ export class DurableStream {
    * Append with batching - buffers messages and sends them in batches.
    */
   async #appendWithBatching(
-    body: unknown,
+    body: Uint8Array | string,
     opts?: AppendOptions
   ): Promise<void> {
     return new Promise<void>((resolve, reject) => {
@@ -511,29 +618,49 @@ export class DurableStream {
       // For JSON mode: always send as array (server flattens one level)
       // Single append: [value] → server stores value
       // Multiple appends: [val1, val2] → server stores val1, val2
-      const values = batch.map((m) => m.data)
-      batchedBody = JSON.stringify(values)
+      // Input is pre-serialized JSON strings, join them into an array
+      const jsonStrings = batch.map((m) =>
+        typeof m.data === `string` ? m.data : new TextDecoder().decode(m.data)
+      )
+      batchedBody = `[${jsonStrings.join(`,`)}]`
     } else {
-      // For byte mode: concatenate all chunks
-      const totalSize = batch.reduce((sum, m) => {
-        const size =
-          typeof m.data === `string`
-            ? new TextEncoder().encode(m.data).length
-            : (m.data as Uint8Array).length
-        return sum + size
-      }, 0)
-
-      const concatenated = new Uint8Array(totalSize)
-      let offset = 0
-      for (const msg of batch) {
-        const bytes =
-          typeof msg.data === `string`
-            ? new TextEncoder().encode(msg.data)
-            : (msg.data as Uint8Array)
-        concatenated.set(bytes, offset)
-        offset += bytes.length
+      // For byte mode: preserve original data types
+      // - Strings are concatenated as strings (for text/* content types)
+      // - Uint8Arrays are concatenated as binary (for application/octet-stream)
+      // - Mixed types: convert all to binary to avoid data corruption
+      const hasUint8Array = batch.some((m) => m.data instanceof Uint8Array)
+      const hasString = batch.some((m) => typeof m.data === `string`)
+
+      if (hasUint8Array && !hasString) {
+        // All binary: concatenate Uint8Arrays
+        const chunks = batch.map((m) => m.data as Uint8Array)
+        const totalLength = chunks.reduce((sum, c) => sum + c.length, 0)
+        const combined = new Uint8Array(totalLength)
+        let offset = 0
+        for (const chunk of chunks) {
+          combined.set(chunk, offset)
+          offset += chunk.length
+        }
+        batchedBody = combined
+      } else if (hasString && !hasUint8Array) {
+        // All strings: concatenate as string
+        batchedBody = batch.map((m) => m.data as string).join(``)
+      } else {
+        // Mixed types: convert strings to binary and concatenate
+        // This preserves binary data integrity
+        const encoder = new TextEncoder()
+        const chunks = batch.map((m) =>
+          typeof m.data === `string` ? encoder.encode(m.data) : m.data
+        )
+        const totalLength = chunks.reduce((sum, c) => sum + c.length, 0)
+        const combined = new Uint8Array(totalLength)
+        let offset = 0
+        for (const chunk of chunks) {
+          combined.set(chunk, offset)
+          offset += chunk.length
+        }
+        batchedBody = combined
       }
-      batchedBody = concatenated
     }
 
     // Combine signals: stream-level signal + any per-message signals
@@ -634,6 +761,11 @@ export class DurableStream {
    * Returns a WritableStream that can be used with `pipeTo()` or
    * `pipeThrough()` from any ReadableStream source.
    *
+   * Uses IdempotentProducer internally for:
+   * - Automatic batching (controlled by lingerMs, maxBatchBytes)
+   * - Exactly-once delivery semantics
+   * - Streaming writes (doesn't buffer entire content in memory)
+   *
    * @example
    * ```typescript
    * // Pipe from fetch response
@@ -643,32 +775,56 @@ export class DurableStream {
    * // Pipe through a transform
    * const readable = someStream.pipeThrough(new TextEncoderStream());
    * await readable.pipeTo(stream.writable());
+   *
+   * // With custom producer options
+   * await source.pipeTo(stream.writable({
+   *   producerId: "my-producer",
+   *   lingerMs: 10,
+   *   maxBatchBytes: 64 * 1024,
+   * }));
    * ```
    */
-  writable(opts?: AppendOptions): WritableStream<Uint8Array | string> {
-    const chunks: Array<Uint8Array | string> = []
-    const stream = this
+  writable(
+    opts?: Pick<
+      IdempotentProducerOptions,
+      `lingerMs` | `maxBatchBytes` | `onError`
+    > & {
+      producerId?: string
+      signal?: AbortSignal
+    }
+  ): WritableStream<Uint8Array | string> {
+    // Generate a random producer ID if not provided
+    const producerId =
+      opts?.producerId ?? `writable-${crypto.randomUUID().slice(0, 8)}`
+
+    // Track async errors to surface in close() so pipeTo() rejects on failure
+    let writeError: Error | null = null
+
+    const producer = new IdempotentProducer(this, producerId, {
+      autoClaim: true, // Ephemeral producer, auto-claim epoch
+      lingerMs: opts?.lingerMs,
+      maxBatchBytes: opts?.maxBatchBytes,
+      onError: (error) => {
+        if (!writeError) writeError = error // Capture first error
+        opts?.onError?.(error) // Still call user's handler
+      },
+      signal: opts?.signal ?? this.#options.signal,
+    })
 
     return new WritableStream<Uint8Array | string>({
       write(chunk) {
-        chunks.push(chunk)
+        producer.append(chunk)
       },
       async close() {
-        if (chunks.length > 0) {
-          // Create a ReadableStream from collected chunks
-          const readable = new ReadableStream<Uint8Array | string>({
-            start(controller) {
-              for (const chunk of chunks) {
-                controller.enqueue(chunk)
-              }
-              controller.close()
-            },
-          })
-          await stream.appendStream(readable, opts)
-        }
+        // close() flushes pending and closes the stream (EOF)
+        await producer.close()
+        if (writeError) throw writeError // Causes pipeTo() to reject
       },
-      abort(reason) {
-        console.error(`WritableStream aborted:`, reason)
+      abort(_reason) {
+        // detach() stops the producer without closing the stream
+        producer.detach().catch((err) => {
+          opts?.onError?.(err) // Report instead of swallowing
+        })
       },
     })
   }
@@ -715,20 +871,6 @@ export class DurableStream {
   async stream<TJson = unknown>(
     options?: Omit<StreamOptions, `url`>
   ): Promise<StreamResponse<TJson>> {
-    // Check SSE compatibility if SSE mode is requested
-    if (options?.live === `sse` && this.contentType) {
-      const isSSECompatible = SSE_COMPATIBLE_CONTENT_TYPES.some((prefix) =>
-        this.contentType!.startsWith(prefix)
-      )
-      if (!isSSECompatible) {
-        throw new DurableStreamError(
-          `SSE is not supported for content-type: ${this.contentType}`,
-          `SSE_NOT_SUPPORTED`,
-          400
-        )
-      }
-    }
-
     // Merge handle-level and call-specific headers
     const mergedHeaders: HeadersRecord = {
       ...this.#options.headers,