openai 4.87.4 → 4.89.0

package/src/helpers/audio.ts

@@ -0,0 +1,145 @@
+import { File } from 'formdata-node';
+import { spawn } from 'node:child_process';
+import { Readable } from 'node:stream';
+import { platform, versions } from 'node:process';
+import { Response } from "../_shims";
+
+const DEFAULT_SAMPLE_RATE = 24000;
+const DEFAULT_CHANNELS = 1;
+
+const isNode = Boolean(versions?.node);
+
+const recordingProviders: Record<NodeJS.Platform, string> = {
+  win32: 'dshow',
+  darwin: 'avfoundation',
+  linux: 'alsa',
+  aix: 'alsa',
+  android: 'alsa',
+  freebsd: 'alsa',
+  haiku: 'alsa',
+  sunos: 'alsa',
+  netbsd: 'alsa',
+  openbsd: 'alsa',
+  cygwin: 'dshow',
+};
+
+function isResponse(stream: NodeJS.ReadableStream | Response | File): stream is Response {
+  return typeof (stream as any).body !== 'undefined';
+}
+
+function isFile(stream: NodeJS.ReadableStream | Response | File): stream is File {
+  return stream instanceof File;
+}
+
+async function nodejsPlayAudio(stream: NodeJS.ReadableStream | Response | File): Promise<void> {
+  return new Promise((resolve, reject) => {
+    try {
+      const ffplay = spawn('ffplay', ['-autoexit', '-nodisp', '-i', 'pipe:0']);
+
+      if (isResponse(stream)) {
+        stream.body.pipe(ffplay.stdin);
+      } else if (isFile(stream)) {
+        Readable.from(stream.stream()).pipe(ffplay.stdin);
+      } else {
+        stream.pipe(ffplay.stdin);
+      }
+
+      ffplay.on('close', (code: number) => {
+        if (code !== 0) {
+          reject(new Error(`ffplay process exited with code ${code}`));
+        }
+        resolve();
+      });
+    } catch (error) {
+      reject(error);
+    }
+  });
+}
+
+export async function playAudio(input: NodeJS.ReadableStream | Response | File): Promise<void> {
+  if (isNode) {
+    return nodejsPlayAudio(input);
+  }
+
+  throw new Error(
+    'Play audio is not supported in the browser yet. Check out https://npm.im/wavtools as an alternative.',
+  );
+}
+
+type RecordAudioOptions = {
+  signal?: AbortSignal;
+  device?: number;
+  timeout?: number;
+};
+
+function nodejsRecordAudio({ signal, device, timeout }: RecordAudioOptions = {}): Promise<File> {
+  return new Promise((resolve, reject) => {
+    const data: any[] = [];
+    const provider = recordingProviders[platform];
+    try {
+      const ffmpeg = spawn(
+        'ffmpeg',
+        [
+          '-f',
+          provider,
+          '-i',
+          `:${device ?? 0}`, // default audio input device; adjust as needed
+          '-ar',
+          DEFAULT_SAMPLE_RATE.toString(),
+          '-ac',
+          DEFAULT_CHANNELS.toString(),
+          '-f',
+          'wav',
+          'pipe:1',
+        ],
+        {
+          stdio: ['ignore', 'pipe', 'pipe'],
+        },
+      );
+
+      ffmpeg.stdout.on('data', (chunk) => {
+        data.push(chunk);
+      });
+
+      ffmpeg.on('error', (error) => {
+        console.error(error);
+        reject(error);
+      });
+
+      ffmpeg.on('close', (code) => {
+        returnData();
+      });
+
+      function returnData() {
+        const audioBuffer = Buffer.concat(data);
+        const audioFile = new File([audioBuffer], 'audio.wav', { type: 'audio/wav' });
+        resolve(audioFile);
+      }
+
+      if (typeof timeout === 'number' && timeout > 0) {
+        const internalSignal = AbortSignal.timeout(timeout);
+        internalSignal.addEventListener('abort', () => {
+          ffmpeg.kill('SIGTERM');
+        });
+      }
+
+      if (signal) {
+        signal.addEventListener('abort', () => {
+          ffmpeg.kill('SIGTERM');
+        });
+      }
+    } catch (error) {
+      reject(error);
+    }
+  });
+}
+
+export async function recordAudio(options: RecordAudioOptions = {}) {
+  if (isNode) {
+    return nodejsRecordAudio(options);
+  }
+
+  throw new Error(
+    'Record audio is not supported in the browser. Check out https://npm.im/wavtools as an alternative.',
+  );
+}

package/src/index.ts

@@ -508,6 +508,7 @@ export declare namespace OpenAI {
 
   export { Responses as Responses };
 
+  export type AllModels = API.AllModels;
   export type ChatModel = API.ChatModel;
   export type ComparisonFilter = API.ComparisonFilter;
   export type CompoundFilter = API.CompoundFilter;
@@ -520,6 +521,7 @@ export declare namespace OpenAI {
   export type ResponseFormatJSONObject = API.ResponseFormatJSONObject;
   export type ResponseFormatJSONSchema = API.ResponseFormatJSONSchema;
   export type ResponseFormatText = API.ResponseFormatText;
+  export type ResponsesModel = API.ResponsesModel;
 }
 
 // ---------------------- Azure ----------------------

package/src/resources/audio/audio.ts

@@ -7,8 +7,14 @@ import * as TranscriptionsAPI from './transcriptions';
 import {
   Transcription,
   TranscriptionCreateParams,
+  TranscriptionCreateParamsNonStreaming,
+  TranscriptionCreateParamsStreaming,
   TranscriptionCreateResponse,
+  TranscriptionInclude,
   TranscriptionSegment,
+  TranscriptionStreamEvent,
+  TranscriptionTextDeltaEvent,
+  TranscriptionTextDoneEvent,
   TranscriptionVerbose,
   TranscriptionWord,
   Transcriptions,
@@ -28,11 +34,12 @@ export class Audio extends APIResource {
   speech: SpeechAPI.Speech = new SpeechAPI.Speech(this._client);
 }
 
-export type AudioModel = 'whisper-1';
+export type AudioModel = 'whisper-1' | 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe';
 
 /**
  * The format of the output, in one of these options: `json`, `text`, `srt`,
- * `verbose_json`, or `vtt`.
+ * `verbose_json`, or `vtt`. For `gpt-4o-transcribe` and `gpt-4o-mini-transcribe`,
+ * the only supported format is `json`.
  */
 export type AudioResponseFormat = 'json' | 'text' | 'srt' | 'verbose_json' | 'vtt';
 
@@ -46,11 +53,17 @@ export declare namespace Audio {
   export {
     Transcriptions as Transcriptions,
     type Transcription as Transcription,
+    type TranscriptionInclude as TranscriptionInclude,
     type TranscriptionSegment as TranscriptionSegment,
+    type TranscriptionStreamEvent as TranscriptionStreamEvent,
+    type TranscriptionTextDeltaEvent as TranscriptionTextDeltaEvent,
+    type TranscriptionTextDoneEvent as TranscriptionTextDoneEvent,
     type TranscriptionVerbose as TranscriptionVerbose,
     type TranscriptionWord as TranscriptionWord,
     type TranscriptionCreateResponse as TranscriptionCreateResponse,
     type TranscriptionCreateParams as TranscriptionCreateParams,
+    type TranscriptionCreateParamsNonStreaming as TranscriptionCreateParamsNonStreaming,
+    type TranscriptionCreateParamsStreaming as TranscriptionCreateParamsStreaming,
   };
 
   export {

package/src/resources/audio/index.ts

@@ -5,11 +5,17 @@ export { Speech, type SpeechModel, type SpeechCreateParams } from './speech';
 export {
   Transcriptions,
   type Transcription,
+  type TranscriptionInclude,
   type TranscriptionSegment,
+  type TranscriptionStreamEvent,
+  type TranscriptionTextDeltaEvent,
+  type TranscriptionTextDoneEvent,
   type TranscriptionVerbose,
   type TranscriptionWord,
   type TranscriptionCreateResponse,
   type TranscriptionCreateParams,
+  type TranscriptionCreateParamsNonStreaming,
+  type TranscriptionCreateParamsStreaming,
 } from './transcriptions';
 export {
   Translations,

package/src/resources/audio/speech.ts

@@ -18,7 +18,7 @@ export class Speech extends APIResource {
   }
 }
 
-export type SpeechModel = 'tts-1' | 'tts-1-hd';
+export type SpeechModel = 'tts-1' | 'tts-1-hd' | 'gpt-4o-mini-tts';
 
 export interface SpeechCreateParams {
   /**
@@ -28,7 +28,7 @@ export interface SpeechCreateParams {
 
   /**
    * One of the available [TTS models](https://platform.openai.com/docs/models#tts):
-   * `tts-1` or `tts-1-hd`
+   * `tts-1`, `tts-1-hd` or `gpt-4o-mini-tts`.
    */
   model: (string & {}) | SpeechModel;
 
@@ -40,6 +40,12 @@ export interface SpeechCreateParams {
    */
   voice: 'alloy' | 'ash' | 'coral' | 'echo' | 'fable' | 'onyx' | 'nova' | 'sage' | 'shimmer';
 
+  /**
+   * Control the voice of your generated audio with additional instructions. Does not
+   * work with `tts-1` or `tts-1-hd`.
+   */
+  instructions?: string;
+
   /**
    * The format to audio in. Supported formats are `mp3`, `opus`, `aac`, `flac`,
    * `wav`, and `pcm`.

package/src/resources/audio/transcriptions.ts

@@ -2,29 +2,42 @@
 
 import { APIResource } from '../../resource';
 import * as Core from '../../core';
+import * as TranscriptionsAPI from './transcriptions';
 import * as AudioAPI from './audio';
+import { Stream } from '../../streaming';
 
 export class Transcriptions extends APIResource {
   /**
    * Transcribes audio into the input language.
    */
   create(
-    body: TranscriptionCreateParams<'json' | undefined>,
+    body: TranscriptionCreateParamsNonStreaming<'json' | undefined>,
     options?: Core.RequestOptions,
   ): Core.APIPromise<Transcription>;
   create(
-    body: TranscriptionCreateParams<'verbose_json'>,
+    body: TranscriptionCreateParamsNonStreaming<'verbose_json'>,
     options?: Core.RequestOptions,
   ): Core.APIPromise<TranscriptionVerbose>;
   create(
-    body: TranscriptionCreateParams<'srt' | 'vtt' | 'text'>,
+    body: TranscriptionCreateParamsNonStreaming<'srt' | 'vtt' | 'text'>,
     options?: Core.RequestOptions,
   ): Core.APIPromise<string>;
-  create(body: TranscriptionCreateParams, options?: Core.RequestOptions): Core.APIPromise<Transcription>;
+  create(
+    body: TranscriptionCreateParamsNonStreaming,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<Transcription>;
+  create(
+    body: TranscriptionCreateParamsStreaming,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<Stream<TranscriptionStreamEvent>>;
+  create(
+    body: TranscriptionCreateParamsStreaming,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<TranscriptionCreateResponse | string | Stream<TranscriptionStreamEvent>>;
   create(
     body: TranscriptionCreateParams,
     options?: Core.RequestOptions,
-  ): Core.APIPromise<TranscriptionCreateResponse | string> {
+  ): Core.APIPromise<TranscriptionCreateResponse | string | Stream<TranscriptionStreamEvent>> {
     return this._client.post(
       '/audio/transcriptions',
       Core.multipartFormRequestOptions({ body, ...options, __metadata: { model: body.model } }),
@@ -41,8 +54,36 @@ export interface Transcription {
    * The transcribed text.
    */
   text: string;
+
+  /**
+   * The log probabilities of the tokens in the transcription. Only returned with the
+   * models `gpt-4o-transcribe` and `gpt-4o-mini-transcribe` if `logprobs` is added
+   * to the `include` array.
+   */
+  logprobs?: Array<Transcription.Logprob>;
 }
 
+export namespace Transcription {
+  export interface Logprob {
+    /**
+     * The token in the transcription.
+     */
+    token?: string;
+
+    /**
+     * The bytes of the token.
+     */
+    bytes?: Array<number>;
+
+    /**
+     * The log probability of the token.
+     */
+    logprob?: number;
+  }
+}
+
+export type TranscriptionInclude = 'logprobs';
+
 export interface TranscriptionSegment {
   /**
    * Unique identifier of the segment.
@@ -98,6 +139,103 @@ export interface TranscriptionSegment {
   tokens: Array<number>;
 }
 
+/**
+ * Emitted when there is an additional text delta. This is also the first event
+ * emitted when the transcription starts. Only emitted when you
+ * [create a transcription](https://platform.openai.com/docs/api-reference/audio/create-transcription)
+ * with the `Stream` parameter set to `true`.
+ */
+export type TranscriptionStreamEvent = TranscriptionTextDeltaEvent | TranscriptionTextDoneEvent;
+
+/**
+ * Emitted when there is an additional text delta. This is also the first event
+ * emitted when the transcription starts. Only emitted when you
+ * [create a transcription](https://platform.openai.com/docs/api-reference/audio/create-transcription)
+ * with the `Stream` parameter set to `true`.
+ */
+export interface TranscriptionTextDeltaEvent {
+  /**
+   * The text delta that was additionally transcribed.
+   */
+  delta: string;
+
+  /**
+   * The type of the event. Always `transcript.text.delta`.
+   */
+  type: 'transcript.text.delta';
+
+  /**
+   * The log probabilities of the delta. Only included if you
+   * [create a transcription](https://platform.openai.com/docs/api-reference/audio/create-transcription)
+   * with the `include[]` parameter set to `logprobs`.
+   */
+  logprobs?: Array<TranscriptionTextDeltaEvent.Logprob>;
+}
+
+export namespace TranscriptionTextDeltaEvent {
+  export interface Logprob {
+    /**
+     * The token that was used to generate the log probability.
+     */
+    token?: string;
+
+    /**
+     * The bytes that were used to generate the log probability.
+     */
+    bytes?: Array<unknown>;
+
+    /**
+     * The log probability of the token.
+     */
+    logprob?: number;
+  }
+}
+
+/**
+ * Emitted when the transcription is complete. Contains the complete transcription
+ * text. Only emitted when you
+ * [create a transcription](https://platform.openai.com/docs/api-reference/audio/create-transcription)
+ * with the `Stream` parameter set to `true`.
+ */
+export interface TranscriptionTextDoneEvent {
+  /**
+   * The text that was transcribed.
+   */
+  text: string;
+
+  /**
+   * The type of the event. Always `transcript.text.done`.
+   */
+  type: 'transcript.text.done';
+
+  /**
+   * The log probabilities of the individual tokens in the transcription. Only
+   * included if you
+   * [create a transcription](https://platform.openai.com/docs/api-reference/audio/create-transcription)
+   * with the `include[]` parameter set to `logprobs`.
+   */
+  logprobs?: Array<TranscriptionTextDoneEvent.Logprob>;
+}
+
+export namespace TranscriptionTextDoneEvent {
+  export interface Logprob {
+    /**
+     * The token that was used to generate the log probability.
+     */
+    token?: string;
+
+    /**
+     * The bytes that were used to generate the log probability.
+     */
+    bytes?: Array<unknown>;
+
+    /**
+     * The log probability of the token.
+     */
+    logprob?: number;
+  }
+}
+
 /**
  * Represents a verbose json transcription response returned by model, based on the
  * provided input.
@@ -152,7 +290,11 @@ export interface TranscriptionWord {
  */
 export type TranscriptionCreateResponse = Transcription | TranscriptionVerbose;
 
-export interface TranscriptionCreateParams<
+export type TranscriptionCreateParams<
+  ResponseFormat extends AudioAPI.AudioResponseFormat | undefined = AudioAPI.AudioResponseFormat | undefined,
+> = TranscriptionCreateParamsNonStreaming<ResponseFormat> | TranscriptionCreateParamsStreaming;
+
+export interface TranscriptionCreateParamsBase<
   ResponseFormat extends AudioAPI.AudioResponseFormat | undefined = AudioAPI.AudioResponseFormat | undefined,
 > {
   /**
@@ -162,11 +304,21 @@ export interface TranscriptionCreateParams<
   file: Core.Uploadable;
 
   /**
-   * ID of the model to use. Only `whisper-1` (which is powered by our open source
-   * Whisper V2 model) is currently available.
+   * ID of the model to use. The options are `gpt-4o-transcribe`,
+   * `gpt-4o-mini-transcribe`, and `whisper-1` (which is powered by our open source
+   * Whisper V2 model).
    */
   model: (string & {}) | AudioAPI.AudioModel;
 
+  /**
+   * Additional information to include in the transcription response. `logprobs` will
+   * return the log probabilities of the tokens in the response to understand the
+   * model's confidence in the transcription. `logprobs` only works with
+   * response_format set to `json` and only with the models `gpt-4o-transcribe` and
+   * `gpt-4o-mini-transcribe`.
+   */
+  include?: Array<TranscriptionInclude>;
+
   /**
    * The language of the input audio. Supplying the input language in
    * [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (e.g. `en`)
@@ -184,10 +336,23 @@ export interface TranscriptionCreateParams<
 
   /**
    * The format of the output, in one of these options: `json`, `text`, `srt`,
-   * `verbose_json`, or `vtt`.
+   * `verbose_json`, or `vtt`. For `gpt-4o-transcribe` and `gpt-4o-mini-transcribe`,
+   * the only supported format is `json`.
    */
   response_format?: ResponseFormat;
 
+  /**
+   * If set to true, the model response data will be streamed to the client as it is
+   * generated using
+   * [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format).
+   * See the
+   * [Streaming section of the Speech-to-Text guide](https://platform.openai.com/docs/guides/speech-to-text?lang=curl#streaming-transcriptions)
+   * for more information.
+   *
+   * Note: Streaming is not supported for the `whisper-1` model and will be ignored.
+   */
+  stream?: boolean | null;
+
   /**
    * The sampling temperature, between 0 and 1. Higher values like 0.8 will make the
    * output more random, while lower values like 0.2 will make it more focused and
@@ -207,13 +372,54 @@ export interface TranscriptionCreateParams<
   timestamp_granularities?: Array<'word' | 'segment'>;
 }
 
+export namespace TranscriptionCreateParams {
+  export type TranscriptionCreateParamsNonStreaming = TranscriptionsAPI.TranscriptionCreateParamsNonStreaming;
+  export type TranscriptionCreateParamsStreaming = TranscriptionsAPI.TranscriptionCreateParamsStreaming;
+}
+
+export interface TranscriptionCreateParamsNonStreaming<
+  ResponseFormat extends AudioAPI.AudioResponseFormat | undefined = AudioAPI.AudioResponseFormat | undefined,
+> extends TranscriptionCreateParamsBase<ResponseFormat> {
+  /**
+   * If set to true, the model response data will be streamed to the client as it is
+   * generated using
+   * [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format).
+   * See the
+   * [Streaming section of the Speech-to-Text guide](https://platform.openai.com/docs/guides/speech-to-text?lang=curl#streaming-transcriptions)
+   * for more information.
+   *
+   * Note: Streaming is not supported for the `whisper-1` model and will be ignored.
+   */
+  stream?: false | null;
+}
+
+export interface TranscriptionCreateParamsStreaming extends TranscriptionCreateParamsBase {
+  /**
+   * If set to true, the model response data will be streamed to the client as it is
+   * generated using
+   * [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format).
+   * See the
+   * [Streaming section of the Speech-to-Text guide](https://platform.openai.com/docs/guides/speech-to-text?lang=curl#streaming-transcriptions)
+   * for more information.
+   *
+   * Note: Streaming is not supported for the `whisper-1` model and will be ignored.
+   */
+  stream: true;
+}
+
 export declare namespace Transcriptions {
   export {
     type Transcription as Transcription,
+    type TranscriptionInclude as TranscriptionInclude,
     type TranscriptionSegment as TranscriptionSegment,
+    type TranscriptionStreamEvent as TranscriptionStreamEvent,
+    type TranscriptionTextDeltaEvent as TranscriptionTextDeltaEvent,
+    type TranscriptionTextDoneEvent as TranscriptionTextDoneEvent,
     type TranscriptionVerbose as TranscriptionVerbose,
     type TranscriptionWord as TranscriptionWord,
     type TranscriptionCreateResponse as TranscriptionCreateResponse,
     type TranscriptionCreateParams as TranscriptionCreateParams,
+    type TranscriptionCreateParamsNonStreaming as TranscriptionCreateParamsNonStreaming,
+    type TranscriptionCreateParamsStreaming as TranscriptionCreateParamsStreaming,
   };
 }

package/src/resources/audio/translations.ts

@@ -88,7 +88,7 @@ export interface TranslationCreateParams<
    * The format of the output, in one of these options: `json`, `text`, `srt`,
    * `verbose_json`, or `vtt`.
    */
-  response_format?: ResponseFormat;
+  response_format?: 'json' | 'text' | 'srt' | 'verbose_json' | 'vtt';
 
   /**
    * The sampling temperature, between 0 and 1. Higher values like 0.8 will make the

package/src/resources/beta/realtime/index.ts

@@ -2,3 +2,8 @@
 
 export { Realtime } from './realtime';
 export { Sessions, type Session, type SessionCreateResponse, type SessionCreateParams } from './sessions';
+export {
+  TranscriptionSessions,
+  type TranscriptionSession,
+  type TranscriptionSessionCreateParams,
+} from './transcription-sessions';