openai 4.53.2 → 4.55.0

package/src/lib/ChatCompletionStream.ts

@@ -1,10 +1,16 @@
 import * as Core from "../core";
-import { OpenAIError, APIUserAbortError } from "../error";
 import {
-  Completions,
+  OpenAIError,
+  APIUserAbortError,
+  LengthFinishReasonError,
+  ContentFilterFinishReasonError,
+} from "../error";
+import {
+  ChatCompletionTokenLogprob,
   type ChatCompletion,
   type ChatCompletionChunk,
   type ChatCompletionCreateParams,
+  type ChatCompletionCreateParamsStreaming,
   type ChatCompletionCreateParamsBase,
 } from "../resources/chat/completions";
 import {
@@ -13,22 +19,125 @@ import {
 } from './AbstractChatCompletionRunner';
 import { type ReadableStream } from "../_shims/index";
 import { Stream } from "../streaming";
+import OpenAI from "../index";
+import { ParsedChatCompletion } from "../resources/beta/chat/completions";
+import {
+  AutoParseableResponseFormat,
+  hasAutoParseableInput,
+  isAutoParsableResponseFormat,
+  isAutoParsableTool,
+  maybeParseChatCompletion,
+  shouldParseToolCall,
+} from "./parser";
+import { partialParse } from '../_vendor/partial-json-parser/parser';
+
+export interface ContentDeltaEvent {
+  delta: string;
+  snapshot: string;
+  parsed: unknown | null;
+}
+
+export interface ContentDoneEvent<ParsedT = null> {
+  content: string;
+  parsed: ParsedT | null;
+}
+
+export interface RefusalDeltaEvent {
+  delta: string;
+  snapshot: string;
+}
+
+export interface RefusalDoneEvent {
+  refusal: string;
+}
+
+export interface FunctionToolCallArgumentsDeltaEvent {
+  name: string;
+
+  index: number;
+
+  arguments: string;
+
+  parsed_arguments: unknown;
+
+  arguments_delta: string;
+}
+
+export interface FunctionToolCallArgumentsDoneEvent {
+  name: string;
+
+  index: number;
+
+  arguments: string;
 
-export interface ChatCompletionStreamEvents extends AbstractChatCompletionRunnerEvents {
+  parsed_arguments: unknown;
+}
+
+export interface LogProbsContentDeltaEvent {
+  content: Array<ChatCompletionTokenLogprob>;
+  snapshot: Array<ChatCompletionTokenLogprob>;
+}
+
+export interface LogProbsContentDoneEvent {
+  content: Array<ChatCompletionTokenLogprob>;
+}
+
+export interface LogProbsRefusalDeltaEvent {
+  refusal: Array<ChatCompletionTokenLogprob>;
+  snapshot: Array<ChatCompletionTokenLogprob>;
+}
+
+export interface LogProbsRefusalDoneEvent {
+  refusal: Array<ChatCompletionTokenLogprob>;
+}
+
+export interface ChatCompletionStreamEvents<ParsedT = null> extends AbstractChatCompletionRunnerEvents {
   content: (contentDelta: string, contentSnapshot: string) => void;
   chunk: (chunk: ChatCompletionChunk, snapshot: ChatCompletionSnapshot) => void;
+
+  'content.delta': (props: ContentDeltaEvent) => void;
+  'content.done': (props: ContentDoneEvent<ParsedT>) => void;
+
+  'refusal.delta': (props: RefusalDeltaEvent) => void;
+  'refusal.done': (props: RefusalDoneEvent) => void;
+
+  'tool_calls.function.arguments.delta': (props: FunctionToolCallArgumentsDeltaEvent) => void;
+  'tool_calls.function.arguments.done': (props: FunctionToolCallArgumentsDoneEvent) => void;
+
+  'logprobs.content.delta': (props: LogProbsContentDeltaEvent) => void;
+  'logprobs.content.done': (props: LogProbsContentDoneEvent) => void;
+
+  'logprobs.refusal.delta': (props: LogProbsRefusalDeltaEvent) => void;
+  'logprobs.refusal.done': (props: LogProbsRefusalDoneEvent) => void;
 }
 
 export type ChatCompletionStreamParams = Omit<ChatCompletionCreateParamsBase, 'stream'> & {
   stream?: true;
 };
 
-export class ChatCompletionStream
-  extends AbstractChatCompletionRunner<ChatCompletionStreamEvents>
+interface ChoiceEventState {
+  content_done: boolean;
+  refusal_done: boolean;
+  logprobs_content_done: boolean;
+  logprobs_refusal_done: boolean;
+  current_tool_call_index: number | null;
+  done_tool_calls: Set<number>;
+}
+
+export class ChatCompletionStream<ParsedT = null>
+  extends AbstractChatCompletionRunner<ChatCompletionStreamEvents<ParsedT>, ParsedT>
   implements AsyncIterable<ChatCompletionChunk>
 {
+  #params: ChatCompletionCreateParams | null;
+  #choiceEventStates: ChoiceEventState[];
   #currentChatCompletionSnapshot: ChatCompletionSnapshot | undefined;
 
+  constructor(params: ChatCompletionCreateParams | null) {
+    super();
+    this.#params = params;
+    this.#choiceEventStates = [];
+  }
+
   get currentChatCompletionSnapshot(): ChatCompletionSnapshot | undefined {
     return this.#currentChatCompletionSnapshot;
   }
@@ -40,21 +149,21 @@ export class ChatCompletionStream
    * Note that messages sent to the model do not appear in `.on('message')`
    * in this context.
    */
-  static fromReadableStream(stream: ReadableStream): ChatCompletionStream {
-    const runner = new ChatCompletionStream();
+  static fromReadableStream(stream: ReadableStream): ChatCompletionStream<null> {
+    const runner = new ChatCompletionStream(null);
     runner._run(() => runner._fromReadableStream(stream));
     return runner;
   }
 
-  static createChatCompletion(
-    completions: Completions,
+  static createChatCompletion<ParsedT>(
+    client: OpenAI,
     params: ChatCompletionStreamParams,
     options?: Core.RequestOptions,
-  ): ChatCompletionStream {
-    const runner = new ChatCompletionStream();
+  ): ChatCompletionStream<ParsedT> {
+    const runner = new ChatCompletionStream<ParsedT>(params as ChatCompletionCreateParamsStreaming);
     runner._run(() =>
       runner._runChatCompletion(
-        completions,
+        client,
         { ...params, stream: true },
         { ...options, headers: { ...options?.headers, 'X-Stainless-Helper-Method': 'stream' } },
       ),
@@ -66,17 +175,184 @@ export class ChatCompletionStream
     if (this.ended) return;
     this.#currentChatCompletionSnapshot = undefined;
   }
-  #addChunk(chunk: ChatCompletionChunk) {
+
+  #getChoiceEventState(choice: ChatCompletionSnapshot.Choice): ChoiceEventState {
+    let state = this.#choiceEventStates[choice.index];
+    if (state) {
+      return state;
+    }
+
+    state = {
+      content_done: false,
+      refusal_done: false,
+      logprobs_content_done: false,
+      logprobs_refusal_done: false,
+      done_tool_calls: new Set(),
+      current_tool_call_index: null,
+    };
+    this.#choiceEventStates[choice.index] = state;
+    return state;
+  }
+
+  #addChunk(this: ChatCompletionStream<ParsedT>, chunk: ChatCompletionChunk) {
     if (this.ended) return;
+
     const completion = this.#accumulateChatCompletion(chunk);
     this._emit('chunk', chunk, completion);
-    const delta = chunk.choices[0]?.delta?.content;
-    const snapshot = completion.choices[0]?.message;
-    if (delta != null && snapshot?.role === 'assistant' && snapshot?.content) {
-      this._emit('content', delta, snapshot.content);
+
+    for (const choice of chunk.choices) {
+      const choiceSnapshot = completion.choices[choice.index]!;
+
+      if (
+        choice.delta.content != null &&
+        choiceSnapshot.message?.role === 'assistant' &&
+        choiceSnapshot.message?.content
+      ) {
+        this._emit('content', choice.delta.content, choiceSnapshot.message.content);
+        this._emit('content.delta', {
+          delta: choice.delta.content,
+          snapshot: choiceSnapshot.message.content,
+          parsed: choiceSnapshot.message.parsed,
+        });
+      }
+
+      if (
+        choice.delta.refusal != null &&
+        choiceSnapshot.message?.role === 'assistant' &&
+        choiceSnapshot.message?.refusal
+      ) {
+        this._emit('refusal.delta', {
+          delta: choice.delta.refusal,
+          snapshot: choiceSnapshot.message.refusal,
+        });
+      }
+
+      if (choice.logprobs?.content != null && choiceSnapshot.message?.role === 'assistant') {
+        this._emit('logprobs.content.delta', {
+          content: choice.logprobs?.content,
+          snapshot: choiceSnapshot.logprobs?.content ?? [],
+        });
+      }
+
+      if (choice.logprobs?.refusal != null && choiceSnapshot.message?.role === 'assistant') {
+        this._emit('logprobs.refusal.delta', {
+          refusal: choice.logprobs?.refusal,
+          snapshot: choiceSnapshot.logprobs?.refusal ?? [],
+        });
+      }
+
+      const state = this.#getChoiceEventState(choiceSnapshot);
+
+      if (choiceSnapshot.finish_reason) {
+        this.#emitContentDoneEvents(choiceSnapshot);
+
+        if (state.current_tool_call_index != null) {
+          this.#emitToolCallDoneEvent(choiceSnapshot, state.current_tool_call_index);
+        }
+      }
+
+      for (const toolCall of choice.delta.tool_calls ?? []) {
+        if (state.current_tool_call_index !== toolCall.index) {
+          this.#emitContentDoneEvents(choiceSnapshot);
+
+          // new tool call started, the previous one is done
+          if (state.current_tool_call_index != null) {
+            this.#emitToolCallDoneEvent(choiceSnapshot, state.current_tool_call_index);
+          }
+        }
+
+        state.current_tool_call_index = toolCall.index;
+      }
+
+      for (const toolCallDelta of choice.delta.tool_calls ?? []) {
+        const toolCallSnapshot = choiceSnapshot.message.tool_calls?.[toolCallDelta.index];
+        if (!toolCallSnapshot?.type) {
+          continue;
+        }
+
+        if (toolCallSnapshot?.type === 'function') {
+          this._emit('tool_calls.function.arguments.delta', {
+            name: toolCallSnapshot.function?.name,
+            index: toolCallDelta.index,
+            arguments: toolCallSnapshot.function.arguments,
+            parsed_arguments: toolCallSnapshot.function.parsed_arguments,
+            arguments_delta: toolCallDelta.function?.arguments ?? '',
+          });
+        } else {
+          assertNever(toolCallSnapshot?.type);
+        }
+      }
     }
   }
-  #endRequest(): ChatCompletion {
+
+  #emitToolCallDoneEvent(choiceSnapshot: ChatCompletionSnapshot.Choice, toolCallIndex: number) {
+    const state = this.#getChoiceEventState(choiceSnapshot);
+    if (state.done_tool_calls.has(toolCallIndex)) {
+      // we've already fired the done event
+      return;
+    }
+
+    const toolCallSnapshot = choiceSnapshot.message.tool_calls?.[toolCallIndex];
+    if (!toolCallSnapshot) {
+      throw new Error('no tool call snapshot');
+    }
+    if (!toolCallSnapshot.type) {
+      throw new Error('tool call snapshot missing `type`');
+    }
+
+    if (toolCallSnapshot.type === 'function') {
+      const inputTool = this.#params?.tools?.find(
+        (tool) => tool.type === 'function' && tool.function.name === toolCallSnapshot.function.name,
+      );
+
+      this._emit('tool_calls.function.arguments.done', {
+        name: toolCallSnapshot.function.name,
+        index: toolCallIndex,
+        arguments: toolCallSnapshot.function.arguments,
+        parsed_arguments:
+          isAutoParsableTool(inputTool) ? inputTool.$parseRaw(toolCallSnapshot.function.arguments)
+          : inputTool?.function.strict ? JSON.parse(toolCallSnapshot.function.arguments)
+          : null,
+      });
+    } else {
+      assertNever(toolCallSnapshot.type);
+    }
+  }
+
+  #emitContentDoneEvents(choiceSnapshot: ChatCompletionSnapshot.Choice) {
+    const state = this.#getChoiceEventState(choiceSnapshot);
+
+    if (choiceSnapshot.message.content && !state.content_done) {
+      state.content_done = true;
+
+      const responseFormat = this.#getAutoParseableResponseFormat();
+
+      this._emit('content.done', {
+        content: choiceSnapshot.message.content,
+        parsed: responseFormat ? responseFormat.$parseRaw(choiceSnapshot.message.content) : (null as any),
+      });
+    }
+
+    if (choiceSnapshot.message.refusal && !state.refusal_done) {
+      state.refusal_done = true;
+
+      this._emit('refusal.done', { refusal: choiceSnapshot.message.refusal });
+    }
+
+    if (choiceSnapshot.logprobs?.content && !state.logprobs_content_done) {
+      state.logprobs_content_done = true;
+
+      this._emit('logprobs.content.done', { content: choiceSnapshot.logprobs.content });
+    }
+
+    if (choiceSnapshot.logprobs?.refusal && !state.logprobs_refusal_done) {
+      state.logprobs_refusal_done = true;
+
+      this._emit('logprobs.refusal.done', { refusal: choiceSnapshot.logprobs.refusal });
+    }
+  }
+
+  #endRequest(): ParsedChatCompletion<ParsedT> {
     if (this.ended) {
       throw new OpenAIError(`stream has ended, this shouldn't happen`);
     }
@@ -85,21 +361,24 @@ export class ChatCompletionStream
       throw new OpenAIError(`request ended without sending any chunks`);
     }
     this.#currentChatCompletionSnapshot = undefined;
-    return finalizeChatCompletion(snapshot);
+    this.#choiceEventStates = [];
+    return finalizeChatCompletion(snapshot, this.#params);
   }
 
   protected override async _createChatCompletion(
-    completions: Completions,
+    client: OpenAI,
     params: ChatCompletionCreateParams,
     options?: Core.RequestOptions,
-  ): Promise<ChatCompletion> {
+  ): Promise<ParsedChatCompletion<ParsedT>> {
+    super._createChatCompletion;
     const signal = options?.signal;
     if (signal) {
       if (signal.aborted) this.controller.abort();
       signal.addEventListener('abort', () => this.controller.abort());
     }
     this.#beginRequest();
-    const stream = await completions.create(
+
+    const stream = await client.chat.completions.create(
       { ...params, stream: true },
       { ...options, signal: this.controller.signal },
     );
@@ -141,6 +420,15 @@ export class ChatCompletionStream
     return this._addChatCompletion(this.#endRequest());
   }
 
+  #getAutoParseableResponseFormat(): AutoParseableResponseFormat<ParsedT> | null {
+    const responseFormat = this.#params?.response_format;
+    if (isAutoParsableResponseFormat<ParsedT>(responseFormat)) {
+      return responseFormat;
+    }
+
+    return null;
+  }
+
   #accumulateChatCompletion(chunk: ChatCompletionChunk): ChatCompletionSnapshot {
     let snapshot = this.#currentChatCompletionSnapshot;
     const { choices, ...rest } = chunk;
@@ -163,23 +451,48 @@ export class ChatCompletionStream
         if (!choice.logprobs) {
           choice.logprobs = Object.assign({}, logprobs);
         } else {
-          const { content, ...rest } = logprobs;
+          const { content, refusal, ...rest } = logprobs;
+          assertIsEmpty(rest);
           Object.assign(choice.logprobs, rest);
+
           if (content) {
             choice.logprobs.content ??= [];
             choice.logprobs.content.push(...content);
           }
+
+          if (refusal) {
+            choice.logprobs.refusal ??= [];
+            choice.logprobs.refusal.push(...refusal);
+          }
+        }
+      }
+
+      if (finish_reason) {
+        choice.finish_reason = finish_reason;
+
+        if (this.#params && hasAutoParseableInput(this.#params)) {
+          if (finish_reason === 'length') {
+            throw new LengthFinishReasonError();
+          }
+
+          if (finish_reason === 'content_filter') {
+            throw new ContentFilterFinishReasonError();
+          }
         }
       }
 
-      if (finish_reason) choice.finish_reason = finish_reason;
       Object.assign(choice, other);
 
       if (!delta) continue; // Shouldn't happen; just in case.
-      const { content, function_call, role, tool_calls, ...rest } = delta;
+
+      const { content, refusal, function_call, role, tool_calls, ...rest } = delta;
+      assertIsEmpty(rest);
       Object.assign(choice.message, rest);
 
-      if (content) choice.message.content = (choice.message.content || '') + content;
+      if (refusal) {
+        choice.message.refusal = (choice.message.refusal || '') + refusal;
+      }
+
       if (role) choice.message.role = role;
       if (function_call) {
         if (!choice.message.function_call) {
@@ -192,23 +505,39 @@ export class ChatCompletionStream
           }
         }
       }
+      if (content) {
+        choice.message.content = (choice.message.content || '') + content;
+
+        if (!choice.message.refusal && this.#getAutoParseableResponseFormat()) {
+          choice.message.parsed = partialParse(choice.message.content);
+        }
+      }
+
       if (tool_calls) {
         if (!choice.message.tool_calls) choice.message.tool_calls = [];
+
         for (const { index, id, type, function: fn, ...rest } of tool_calls) {
-          const tool_call = (choice.message.tool_calls[index] ??= {});
+          const tool_call = (choice.message.tool_calls[index] ??=
+            {} as ChatCompletionSnapshot.Choice.Message.ToolCall);
           Object.assign(tool_call, rest);
           if (id) tool_call.id = id;
           if (type) tool_call.type = type;
-          if (fn) tool_call.function ??= { arguments: '' };
+          if (fn) tool_call.function ??= { name: fn.name ?? '', arguments: '' };
           if (fn?.name) tool_call.function!.name = fn.name;
-          if (fn?.arguments) tool_call.function!.arguments += fn.arguments;
+          if (fn?.arguments) {
+            tool_call.function!.arguments += fn.arguments;
+
+            if (shouldParseToolCall(this.#params, tool_call)) {
+              tool_call.function!.parsed_arguments = partialParse(tool_call.function!.arguments);
+            }
+          }
         }
       }
     }
     return snapshot;
   }
 
-  [Symbol.asyncIterator](): AsyncIterator<ChatCompletionChunk> {
+  [Symbol.asyncIterator](this: ChatCompletionStream<ParsedT>): AsyncIterator<ChatCompletionChunk> {
     const pushQueue: ChatCompletionChunk[] = [];
     const readQueue: {
       resolve: (chunk: ChatCompletionChunk | undefined) => void;
@@ -275,29 +604,50 @@ export class ChatCompletionStream
   }
 }
 
-function finalizeChatCompletion(snapshot: ChatCompletionSnapshot): ChatCompletion {
+function finalizeChatCompletion<ParsedT>(
+  snapshot: ChatCompletionSnapshot,
+  params: ChatCompletionCreateParams | null,
+): ParsedChatCompletion<ParsedT> {
   const { id, choices, created, model, system_fingerprint, ...rest } = snapshot;
-  return {
+  const completion: ChatCompletion = {
     ...rest,
     id,
     choices: choices.map(
       ({ message, finish_reason, index, logprobs, ...choiceRest }): ChatCompletion.Choice => {
-        if (!finish_reason) throw new OpenAIError(`missing finish_reason for choice ${index}`);
+        if (!finish_reason) {
+          throw new OpenAIError(`missing finish_reason for choice ${index}`);
+        }
+
         const { content = null, function_call, tool_calls, ...messageRest } = message;
         const role = message.role as 'assistant'; // this is what we expect; in theory it could be different which would make our types a slight lie but would be fine.
-        if (!role) throw new OpenAIError(`missing role for choice ${index}`);
+        if (!role) {
+          throw new OpenAIError(`missing role for choice ${index}`);
+        }
+
         if (function_call) {
           const { arguments: args, name } = function_call;
-          if (args == null) throw new OpenAIError(`missing function_call.arguments for choice ${index}`);
-          if (!name) throw new OpenAIError(`missing function_call.name for choice ${index}`);
+          if (args == null) {
+            throw new OpenAIError(`missing function_call.arguments for choice ${index}`);
+          }
+
+          if (!name) {
+            throw new OpenAIError(`missing function_call.name for choice ${index}`);
+          }
+
           return {
             ...choiceRest,
-            message: { content, function_call: { arguments: args, name }, role },
+            message: {
+              content,
+              function_call: { arguments: args, name },
+              role,
+              refusal: message.refusal ?? null,
+            },
             finish_reason,
             index,
             logprobs,
           };
         }
+
         if (tool_calls) {
           return {
             ...choiceRest,
@@ -308,21 +658,26 @@ function finalizeChatCompletion(snapshot: ChatCompletionSnapshot): ChatCompletio
               ...messageRest,
               role,
               content,
+              refusal: message.refusal ?? null,
               tool_calls: tool_calls.map((tool_call, i) => {
                 const { function: fn, type, id, ...toolRest } = tool_call;
                 const { arguments: args, name, ...fnRest } = fn || {};
-                if (id == null)
+                if (id == null) {
                   throw new OpenAIError(`missing choices[${index}].tool_calls[${i}].id\n${str(snapshot)}`);
-                if (type == null)
+                }
+                if (type == null) {
                   throw new OpenAIError(`missing choices[${index}].tool_calls[${i}].type\n${str(snapshot)}`);
-                if (name == null)
+                }
+                if (name == null) {
                   throw new OpenAIError(
                     `missing choices[${index}].tool_calls[${i}].function.name\n${str(snapshot)}`,
                   );
-                if (args == null)
+                }
+                if (args == null) {
                   throw new OpenAIError(
                     `missing choices[${index}].tool_calls[${i}].function.arguments\n${str(snapshot)}`,
                   );
+                }
 
                 return { ...toolRest, id, type, function: { ...fnRest, name, arguments: args } };
               }),
@@ -331,7 +686,7 @@ function finalizeChatCompletion(snapshot: ChatCompletionSnapshot): ChatCompletio
         }
         return {
           ...choiceRest,
-          message: { ...messageRest, content, role },
+          message: { ...messageRest, content, role, refusal: message.refusal ?? null },
           finish_reason,
           index,
           logprobs,
@@ -343,6 +698,8 @@ function finalizeChatCompletion(snapshot: ChatCompletionSnapshot): ChatCompletio
     object: 'chat.completion',
     ...(system_fingerprint ? { system_fingerprint } : {}),
   };
+
+  return maybeParseChatCompletion(completion, params);
 }
 
 function str(x: unknown) {
@@ -425,6 +782,10 @@ export namespace ChatCompletionSnapshot {
        */
       content?: string | null;
 
+      refusal?: string | null;
+
+      parsed?: unknown | null;
+
       /**
        * The name and arguments of a function that should be called, as generated by the
        * model.
@@ -444,14 +805,14 @@ export namespace ChatCompletionSnapshot {
         /**
          * The ID of the tool call.
          */
-        id?: string;
+        id: string;
 
-        function?: ToolCall.Function;
+        function: ToolCall.Function;
 
         /**
          * The type of the tool.
          */
-        type?: 'function';
+        type: 'function';
       }
 
       export namespace ToolCall {
@@ -462,12 +823,14 @@ export namespace ChatCompletionSnapshot {
            * hallucinate parameters not defined by your function schema. Validate the
            * arguments in your code before calling your function.
            */
-          arguments?: string;
+          arguments: string;
+
+          parsed_arguments?: unknown;
 
           /**
            * The name of the function to call.
            */
-          name?: string;
+          name: string;
         }
       }
 
@@ -492,3 +855,16 @@ export namespace ChatCompletionSnapshot {
     }
   }
 }
+
+type AssertIsEmpty<T extends {}> = keyof T extends never ? T : never;
+
+/**
+ * Ensures the given argument is an empty object, useful for
+ * asserting that all known properties on an object have been
+ * destructured.
+ */
+function assertIsEmpty<T extends {}>(obj: AssertIsEmpty<T>): asserts obj is AssertIsEmpty<T> {
+  return;
+}
+
+function assertNever(_x: never) {}