openai 7.4.0 → 7.6.0

package/src/core/streaming.ts

@@ -1,36 +1,90 @@
-import { OpenAIError } from './error';
-import { type ReadableStream } from '../internal/shim-types';
-import { makeReadableStream } from '../internal/shims';
+import { OpenAIError, APIError } from './error';
+import type { ReadableStream } from '../internal/shim-types';
+import { makeReadableStream, ReadableStreamToAsyncIterable } from '../internal/shims';
 import { findDoubleNewlineIndex, LineDecoder } from '../internal/decoders/line';
-import { ReadableStreamToAsyncIterable } from '../internal/shims';
 import { isAbortError } from '../internal/errors';
 import { encodeUTF8 } from '../internal/utils/bytes';
 import { loggerFor } from '../internal/utils/log';
 import type { OpenAI } from '../client';
 
-import { APIError } from './error';
-
 type Bytes = string | ArrayBuffer | Uint8Array | null | undefined;
 
+type StreamTeeQueue<Item> = {
+  readonly length: number;
+  enqueue: (value: Promise<IteratorResult<Item>>) => void;
+  dequeue: () => Promise<IteratorResult<Item>> | undefined;
+};
+
+function createStreamTeeQueue<Item>(): StreamTeeQueue<Item> {
+  let entries: (Promise<IteratorResult<Item>> | undefined)[] = [];
+  let head = 0;
+
+  return {
+    get length() {
+      return entries.length - head;
+    },
+    enqueue(value) {
+      entries.push(value);
+    },
+    dequeue() {
+      if (head === entries.length) {
+        return undefined;
+      }
+
+      const value = entries[head];
+      entries[head] = undefined;
+      head += 1;
+
+      if (head === entries.length) {
+        entries = [];
+        head = 0;
+      } else if (head >= 1024 && head * 2 >= entries.length) {
+        entries = entries.slice(head);
+        head = 0;
+      }
+
+      return value;
+    },
+  };
+}
+
+/** A decoded server-sent event before its JSON payload has been parsed. */
 export type ServerSentEvent = {
+  /** Explicit SSE event name, or `null` when the event has no `event:` field. */
   event: string | null;
+  /** Joined contents of the event's `data:` fields. */
   data: string;
+  /** Original event lines retained for diagnostics when parsing fails. */
   raw: string[];
 };
 
+/**
+ * A single-consumption asynchronous API response stream.
+ *
+ * Use {@link Stream.tee} when two consumers need the same events. Breaking out of
+ * a response-backed stream early aborts its request; branches created by `tee()`
+ * instead share {@link Stream.controller} for explicit cancellation.
+ */
 export class Stream<Item> implements AsyncIterable<Item> {
+  /** Abort controller for the underlying request and all branches created with `tee()`. */
   controller: AbortController;
   #client: OpenAI | undefined;
+  private iterator: () => AsyncIterator<Item>;
 
-  constructor(
-    private iterator: () => AsyncIterator<Item>,
-    controller: AbortController,
-    client?: OpenAI,
-  ) {
+  /** Wraps an asynchronous event iterator and the controller that owns its request. */
+  constructor(iterator: () => AsyncIterator<Item>, controller: AbortController, client?: OpenAI) {
+    this.iterator = iterator;
     this.controller = controller;
     this.#client = client;
   }
 
+  /**
+   * Decodes an SSE response into parsed JSON events.
+   *
+   * The resulting stream can be consumed only once, ignores events after `[DONE]`, and
+   * surfaces API error payloads as `APIError` instances. When
+   * `synthesizeEventData` is enabled, each item also includes its SSE event name.
+   */
   static fromSSEResponse<Item>(
     response: Response,
     controller: AbortController,
@@ -46,13 +100,12 @@ export class Stream<Item> implements AsyncIterable<Item> {
       }
       consumed = true;
       let done = false;
+      let receivedCompletionSentinel = false;
       try {
         for await (const sse of _iterSSEMessages(response, controller)) {
-          if (done) continue;
-
-          if (sse.data.startsWith('[DONE]')) {
-            done = true;
-            continue;
+          if (sse.data === '[DONE]') {
+            receivedCompletionSentinel = true;
+            break;
           }
 
           if (sse.event === null || !sse.event.startsWith('thread.')) {
@@ -60,10 +113,14 @@ export class Stream<Item> implements AsyncIterable<Item> {
 
             try {
               data = JSON.parse(sse.data) as any;
-            } catch (e) {
-              logger.error(`Could not parse message into JSON:`, sse.data);
-              logger.error(`From chunk:`, sse.raw);
-              throw e;
+            } catch {
+              logger.error(`Could not parse message into JSON:`);
+              logger.error(`From chunk:`);
+              throw new SyntaxError('Error reading response: malformed server-sent event JSON.');
+            }
+
+            if (sse.event === 'error') {
+              throw new APIError(undefined, data?.error ?? data, undefined, response.headers);
             }
 
             if (data && data.error) {
@@ -75,26 +132,30 @@ export class Stream<Item> implements AsyncIterable<Item> {
             let data;
             try {
               data = JSON.parse(sse.data);
-            } catch (e) {
-              console.error(`Could not parse message into JSON:`, sse.data);
-              console.error(`From chunk:`, sse.raw);
-              throw e;
+            } catch {
+              logger.error(`Could not parse message into JSON:`);
+              logger.error(`From chunk:`);
+              throw new SyntaxError('Error reading response: malformed server-sent event JSON.');
             }
-            // TODO: Is this where the error should be thrown?
-            if (sse.event == 'error') {
-              throw new APIError(undefined, data.error, data.message, undefined);
-            }
-            yield { event: sse.event, data: data } as any;
+            yield { event: sse.event, data } as any;
           }
         }
         done = true;
       } catch (e) {
-        // If the user calls `stream.controller.abort()`, we should exit without throwing.
-        if (isAbortError(e)) return;
+        // Abort errors and cleanup failures after the completion sentinel are non-fatal.
+        if (
+          receivedCompletionSentinel ||
+          isAbortError(e) ||
+          (controller.signal.aborted && e === controller.signal.reason)
+        ) {
+          return;
+        }
         throw e;
       } finally {
         // If the user `break`s, abort the ongoing request.
-        if (!done) controller.abort();
+        if (!done) {
+          controller.abort();
+        }
       }
     }
 
@@ -119,7 +180,7 @@ export class Stream<Item> implements AsyncIterable<Item> {
       let cancelPromise: Promise<void> | undefined;
       const cancel = () => {
         cancelPromise ??= reader.cancel();
-        cancelPromise.catch(() => {});
+        cancelPromise.catch(() => undefined);
       };
 
       controller.signal.addEventListener('abort', cancel, { once: true });
@@ -135,22 +196,32 @@ export class Stream<Item> implements AsyncIterable<Item> {
             closed = true;
             break;
           }
-          if (controller.signal.aborted) return;
+          if (controller.signal.aborted) {
+            return;
+          }
 
           for (const line of lineDecoder.decode(chunk)) {
-            if (controller.signal.aborted) return;
+            if (controller.signal.aborted) {
+              return;
+            }
             yield line;
           }
         }
 
-        if (controller.signal.aborted) return;
+        if (controller.signal.aborted) {
+          return;
+        }
         for (const line of lineDecoder.flush()) {
-          if (controller.signal.aborted) return;
+          if (controller.signal.aborted) {
+            return;
+          }
           yield line;
         }
       } finally {
         controller.signal.removeEventListener('abort', cancel);
-        if (!closed) cancel();
+        if (!closed) {
+          cancel();
+        }
         reader.releaseLock();
       }
     }
@@ -163,23 +234,43 @@ export class Stream<Item> implements AsyncIterable<Item> {
       let done = false;
       try {
         for await (const line of iterLines()) {
-          if (done) continue;
-          if (line) yield JSON.parse(line) as Item;
+          if (done) {
+            continue;
+          }
+          if (line) {
+            let data: Item;
+            try {
+              data = JSON.parse(line) as Item;
+            } catch (error) {
+              if (error instanceof SyntaxError) {
+                throw new SyntaxError('Error reading response: malformed newline-delimited JSON.');
+              }
+
+              throw error;
+            }
+
+            yield data;
+          }
         }
         done = true;
       } catch (e) {
         // If the user calls `stream.controller.abort()`, we should exit without throwing.
-        if (controller.signal.aborted || isAbortError(e)) return;
+        if (controller.signal.aborted || isAbortError(e)) {
+          return;
+        }
         throw e;
       } finally {
         // If the user `break`s, abort the ongoing request.
-        if (!done) controller.abort();
+        if (!done) {
+          controller.abort();
+        }
       }
     }
 
     return new Stream(iterator, controller, client);
   }
 
+  /** Starts consuming this stream; attempting to consume it again throws. */
   [Symbol.asyncIterator](): AsyncIterator<Item> {
     return this.iterator();
   }
@@ -189,22 +280,20 @@ export class Stream<Item> implements AsyncIterable<Item> {
    * independently read from at different speeds.
    */
   tee(): [Stream<Item>, Stream<Item>] {
-    const left: Array<Promise<IteratorResult<Item>>> = [];
-    const right: Array<Promise<IteratorResult<Item>>> = [];
+    const left = createStreamTeeQueue<Item>();
+    const right = createStreamTeeQueue<Item>();
     const iterator = this.iterator();
 
-    const teeIterator = (queue: Array<Promise<IteratorResult<Item>>>): AsyncIterator<Item> => {
-      return {
-        next: () => {
-          if (queue.length === 0) {
-            const result = iterator.next();
-            left.push(result);
-            right.push(result);
-          }
-          return queue.shift()!;
-        },
-      };
-    };
+    const teeIterator = (queue: StreamTeeQueue<Item>): AsyncIterator<Item> => ({
+      next: () => {
+        if (queue.length === 0) {
+          const result = iterator.next();
+          left.enqueue(result);
+          right.enqueue(result);
+        }
+        return queue.dequeue()!;
+      },
+    });
 
     return [
       new Stream(() => teeIterator(left), this.controller, this.#client),
@@ -218,17 +307,18 @@ export class Stream<Item> implements AsyncIterable<Item> {
    * which can be turned back into a Stream with `Stream.fromReadableStream()`.
    */
   toReadableStream(): ReadableStream {
-    const self = this;
     let iter: AsyncIterator<Item>;
 
     return makeReadableStream({
-      async start() {
-        iter = self[Symbol.asyncIterator]();
+      start: async () => {
+        iter = this[Symbol.asyncIterator]();
       },
       async pull(ctrl: any) {
         try {
           const { value, done } = await iter.next();
-          if (done) return ctrl.close();
+          if (done) {
+            return ctrl.close();
+          }
 
           const bytes = encodeUTF8(JSON.stringify(value) + '\n');
 
@@ -244,6 +334,12 @@ export class Stream<Item> implements AsyncIterable<Item> {
   }
 }
 
+/**
+ * Decodes complete SSE records from a response and aborts when its body is absent.
+ * Complete events are decoded on demand without imposing a line or event size limit.
+ *
+ * @yields {ServerSentEvent} Each decoded server-sent event in wire order.
+ */
 export async function* _iterSSEMessages(
   response: Response,
   controller: AbortController,
@@ -251,7 +347,7 @@ export async function* _iterSSEMessages(
   if (!response.body) {
     controller.abort();
     if (
-      typeof (globalThis as any).navigator !== 'undefined' &&
+      (globalThis as any).navigator !== undefined &&
       (globalThis as any).navigator.product === 'ReactNative'
     ) {
       throw new OpenAIError(
@@ -268,69 +364,105 @@ export async function* _iterSSEMessages(
   for await (const sseChunk of iterSSEChunks(iter)) {
     for (const line of lineDecoder.decode(sseChunk)) {
       const sse = sseDecoder.decode(line);
-      if (sse) yield sse;
+      if (sse) {
+        yield sse;
+      }
     }
   }
 
   for (const line of lineDecoder.flush()) {
     const sse = sseDecoder.decode(line);
-    if (sse) yield sse;
+    if (sse) {
+      yield sse;
+    }
   }
 }
 
+// A `\r\n\r\n` separator may retain up to three bytes from the previous chunk.
+const DOUBLE_NEWLINE_DELIMITER_MAX_OVERLAP_BYTES = 3;
+
 /**
  * Given an async iterable iterator, iterates over it and yields full
  * SSE chunks, i.e. yields when a double new-line is encountered.
+ *
+ * @yields {Uint8Array} A complete SSE chunk.
  */
 async function* iterSSEChunks(iterator: AsyncIterableIterator<Bytes>): AsyncGenerator<Uint8Array> {
   let data = new Uint8Array();
+  let dataStart = 0;
+  let dataEnd = 0;
+  let searchStartIndex = 0;
 
   for await (const chunk of iterator) {
     if (chunk == null) {
       continue;
     }
 
-    const binaryChunk =
-      chunk instanceof ArrayBuffer ? new Uint8Array(chunk)
-      : typeof chunk === 'string' ? encodeUTF8(chunk)
-      : chunk;
+    let binaryChunk: Uint8Array;
+    if (chunk instanceof ArrayBuffer) {
+      binaryChunk = new Uint8Array(chunk);
+    } else if (typeof chunk === 'string') {
+      binaryChunk = encodeUTF8(chunk);
+    } else {
+      binaryChunk = chunk;
+    }
+
+    if (dataEnd + binaryChunk.length > data.length) {
+      const bufferedLength = dataEnd - dataStart;
+
+      // Compact only when it reclaims substantial space without moving a large live tail repeatedly.
+      if (dataStart >= data.length / 2 && bufferedLength + binaryChunk.length <= data.length) {
+        data.copyWithin(0, dataStart, dataEnd);
+      } else {
+        const newData = new Uint8Array(Math.max(data.length * 2, bufferedLength + binaryChunk.length));
+        newData.set(data.subarray(dataStart, dataEnd));
+        data = newData;
+      }
 
-    let newData = new Uint8Array(data.length + binaryChunk.length);
-    newData.set(data);
-    newData.set(binaryChunk, data.length);
-    data = newData;
+      searchStartIndex -= dataStart;
+      dataStart = 0;
+      dataEnd = bufferedLength;
+    }
+
+    data.set(binaryChunk, dataEnd);
+    dataEnd += binaryChunk.length;
 
     let patternIndex;
-    while ((patternIndex = findDoubleNewlineIndex(data)) !== -1) {
-      yield data.slice(0, patternIndex);
-      data = data.slice(patternIndex);
+    while ((patternIndex = findDoubleNewlineIndex(data.subarray(searchStartIndex, dataEnd))) !== -1) {
+      patternIndex += searchStartIndex;
+      yield data.slice(dataStart, patternIndex);
+      dataStart = patternIndex;
+      searchStartIndex = dataStart;
     }
+
+    searchStartIndex = Math.max(dataStart, dataEnd - DOUBLE_NEWLINE_DELIMITER_MAX_OVERLAP_BYTES);
   }
 
-  if (data.length > 0) {
-    yield data;
+  if (dataEnd > dataStart) {
+    yield data.slice(dataStart, dataEnd);
   }
 }
 
 class SSEDecoder {
   private data: string[];
-  private event: string | null;
+  private event: string | null = null;
   private chunks: string[];
 
   constructor() {
-    this.event = null;
     this.data = [];
     this.chunks = [];
   }
 
   decode(line: string) {
     if (line.endsWith('\r')) {
-      line = line.substring(0, line.length - 1);
+      line = line.slice(0, -1);
     }
 
     if (!line) {
       // empty line and we didn't previously encounter any messages
-      if (!this.event && !this.data.length) return null;
+      if (!this.event && !this.data.length) {
+        return null;
+      }
 
       const sse: ServerSentEvent = {
         event: this.event,
@@ -351,10 +483,11 @@ class SSEDecoder {
       return null;
     }
 
-    let [fieldname, _, value] = partition(line, ':');
+    const [fieldname, , initialValue] = partition(line, ':');
+    let value = initialValue;
 
     if (value.startsWith(' ')) {
-      value = value.substring(1);
+      value = value.slice(1);
     }
 
     if (fieldname === 'event') {
@@ -370,7 +503,7 @@ class SSEDecoder {
 function partition(str: string, delimiter: string): [string, string, string] {
   const index = str.indexOf(delimiter);
   if (index !== -1) {
-    return [str.substring(0, index), delimiter, str.substring(index + delimiter.length)];
+    return [str.slice(0, index), delimiter, str.slice(index + delimiter.length)];
   }
 
   return [str, '', ''];