openai 4.87.4 → 4.89.0

package/src/resources/beta/realtime/realtime.ts

@@ -10,9 +10,17 @@ import {
   SessionCreateResponse,
   Sessions,
 } from './sessions';
+import * as TranscriptionSessionsAPI from './transcription-sessions';
+import {
+  TranscriptionSession,
+  TranscriptionSessionCreateParams,
+  TranscriptionSessions,
+} from './transcription-sessions';
 
 export class Realtime extends APIResource {
   sessions: SessionsAPI.Sessions = new SessionsAPI.Sessions(this._client);
+  transcriptionSessions: TranscriptionSessionsAPI.TranscriptionSessions =
+    new TranscriptionSessionsAPI.TranscriptionSessions(this._client);
 }
 
 /**
@@ -300,6 +308,91 @@ export interface ConversationItemInputAudioTranscriptionCompletedEvent {
    * The event type, must be `conversation.item.input_audio_transcription.completed`.
    */
   type: 'conversation.item.input_audio_transcription.completed';
+
+  /**
+   * The log probabilities of the transcription.
+   */
+  logprobs?: Array<ConversationItemInputAudioTranscriptionCompletedEvent.Logprob> | null;
+}
+
+export namespace ConversationItemInputAudioTranscriptionCompletedEvent {
+  /**
+   * A log probability object.
+   */
+  export interface Logprob {
+    /**
+     * The token that was used to generate the log probability.
+     */
+    token: string;
+
+    /**
+     * The bytes that were used to generate the log probability.
+     */
+    bytes: Array<number>;
+
+    /**
+     * The log probability of the token.
+     */
+    logprob: number;
+  }
+}
+
+/**
+ * Returned when the text value of an input audio transcription content part is
+ * updated.
+ */
+export interface ConversationItemInputAudioTranscriptionDeltaEvent {
+  /**
+   * The unique ID of the server event.
+   */
+  event_id: string;
+
+  /**
+   * The ID of the item.
+   */
+  item_id: string;
+
+  /**
+   * The event type, must be `conversation.item.input_audio_transcription.delta`.
+   */
+  type: 'conversation.item.input_audio_transcription.delta';
+
+  /**
+   * The index of the content part in the item's content array.
+   */
+  content_index?: number;
+
+  /**
+   * The text delta.
+   */
+  delta?: string;
+
+  /**
+   * The log probabilities of the transcription.
+   */
+  logprobs?: Array<ConversationItemInputAudioTranscriptionDeltaEvent.Logprob> | null;
+}
+
+export namespace ConversationItemInputAudioTranscriptionDeltaEvent {
+  /**
+   * A log probability object.
+   */
+  export interface Logprob {
+    /**
+     * The token that was used to generate the log probability.
+     */
+    token: string;
+
+    /**
+     * The bytes that were used to generate the log probability.
+     */
+    bytes: Array<number>;
+
+    /**
+     * The log probability of the token.
+     */
+    logprob: number;
+  }
 }
 
 /**
@@ -361,6 +454,30 @@ export namespace ConversationItemInputAudioTranscriptionFailedEvent {
   }
 }
 
+/**
+ * Send this event when you want to retrieve the server's representation of a
+ * specific item in the conversation history. This is useful, for example, to
+ * inspect user audio after noise cancellation and VAD. The server will respond
+ * with a `conversation.item.retrieved` event, unless the item does not exist in
+ * the conversation history, in which case the server will respond with an error.
+ */
+export interface ConversationItemRetrieveEvent {
+  /**
+   * The ID of the item to retrieve.
+   */
+  item_id: string;
+
+  /**
+   * The event type, must be `conversation.item.retrieve`.
+   */
+  type: 'conversation.item.retrieve';
+
+  /**
+   * Optional client-generated ID used to identify this event.
+   */
+  event_id?: string;
+}
+
 /**
  * Send this event to truncate a previous assistant message’s audio. The server
  * will produce audio faster than realtime, so this event is useful when the user
@@ -789,18 +906,20 @@ export namespace RateLimitsUpdatedEvent {
 }
 
 /**
- * All events that the client can send to the Realtime API
+ * A realtime client event.
  */
 export type RealtimeClientEvent =
-  | SessionUpdateEvent
-  | InputAudioBufferAppendEvent
-  | InputAudioBufferCommitEvent
-  | InputAudioBufferClearEvent
   | ConversationItemCreateEvent
-  | ConversationItemTruncateEvent
   | ConversationItemDeleteEvent
+  | ConversationItemRetrieveEvent
+  | ConversationItemTruncateEvent
+  | InputAudioBufferAppendEvent
+  | InputAudioBufferClearEvent
+  | InputAudioBufferCommitEvent
+  | ResponseCancelEvent
   | ResponseCreateEvent
-  | ResponseCancelEvent;
+  | SessionUpdateEvent
+  | TranscriptionSessionUpdate;
 
 /**
  * The response resource.
@@ -1009,37 +1128,63 @@ export namespace RealtimeResponseUsage {
 }
 
 /**
- * All events that the Realtime API can send back
+ * A realtime server event.
  */
 export type RealtimeServerEvent =
-  | ErrorEvent
-  | SessionCreatedEvent
-  | SessionUpdatedEvent
   | ConversationCreatedEvent
-  | InputAudioBufferCommittedEvent
-  | InputAudioBufferClearedEvent
-  | InputAudioBufferSpeechStartedEvent
-  | InputAudioBufferSpeechStoppedEvent
   | ConversationItemCreatedEvent
+  | ConversationItemDeletedEvent
   | ConversationItemInputAudioTranscriptionCompletedEvent
+  | ConversationItemInputAudioTranscriptionDeltaEvent
   | ConversationItemInputAudioTranscriptionFailedEvent
+  | RealtimeServerEvent.ConversationItemRetrieved
   | ConversationItemTruncatedEvent
-  | ConversationItemDeletedEvent
+  | ErrorEvent
+  | InputAudioBufferClearedEvent
+  | InputAudioBufferCommittedEvent
+  | InputAudioBufferSpeechStartedEvent
+  | InputAudioBufferSpeechStoppedEvent
+  | RateLimitsUpdatedEvent
+  | ResponseAudioDeltaEvent
+  | ResponseAudioDoneEvent
+  | ResponseAudioTranscriptDeltaEvent
+  | ResponseAudioTranscriptDoneEvent
+  | ResponseContentPartAddedEvent
+  | ResponseContentPartDoneEvent
   | ResponseCreatedEvent
   | ResponseDoneEvent
+  | ResponseFunctionCallArgumentsDeltaEvent
+  | ResponseFunctionCallArgumentsDoneEvent
   | ResponseOutputItemAddedEvent
   | ResponseOutputItemDoneEvent
-  | ResponseContentPartAddedEvent
-  | ResponseContentPartDoneEvent
   | ResponseTextDeltaEvent
   | ResponseTextDoneEvent
-  | ResponseAudioTranscriptDeltaEvent
-  | ResponseAudioTranscriptDoneEvent
-  | ResponseAudioDeltaEvent
-  | ResponseAudioDoneEvent
-  | ResponseFunctionCallArgumentsDeltaEvent
-  | ResponseFunctionCallArgumentsDoneEvent
-  | RateLimitsUpdatedEvent;
+  | SessionCreatedEvent
+  | SessionUpdatedEvent
+  | TranscriptionSessionUpdatedEvent;
+
+export namespace RealtimeServerEvent {
+  /**
+   * Returned when a conversation item is retrieved with
+   * `conversation.item.retrieve`.
+   */
+  export interface ConversationItemRetrieved {
+    /**
+     * The unique ID of the server event.
+     */
+    event_id: string;
+
+    /**
+     * The item to add to the conversation.
+     */
+    item: RealtimeAPI.ConversationItem;
+
+    /**
+     * The event type, must be `conversation.item.retrieved`.
+     */
+    type: 'conversation.item.retrieved';
+  }
+}
 
 /**
  * Returned when the model-generated audio is updated.
@@ -1834,15 +1979,24 @@ export namespace SessionUpdateEvent {
      */
     input_audio_format?: 'pcm16' | 'g711_ulaw' | 'g711_alaw';
 
+    /**
+     * Configuration for input audio noise reduction. This can be set to `null` to turn
+     * off. Noise reduction filters audio added to the input audio buffer before it is
+     * sent to VAD and the model. Filtering the audio can improve VAD and turn
+     * detection accuracy (reducing false positives) and model performance by improving
+     * perception of the input audio.
+     */
+    input_audio_noise_reduction?: Session.InputAudioNoiseReduction;
+
     /**
      * Configuration for input audio transcription, defaults to off and can be set to
      * `null` to turn off once on. Input audio transcription is not native to the
      * model, since the model consumes audio directly. Transcription runs
      * asynchronously through
-     * [OpenAI Whisper transcription](https://platform.openai.com/docs/api-reference/audio/createTranscription)
-     * and should be treated as rough guidance rather than the representation
-     * understood by the model. The client can optionally set the language and prompt
-     * for transcription, these fields will be passed to the Whisper API.
+     * [the /audio/transcriptions endpoint](https://platform.openai.com/docs/api-reference/audio/createTranscription)
+     * and should be treated as guidance of input audio content rather than precisely
+     * what the model heard. The client can optionally set the language and prompt for
+     * transcription, these offer additional guidance to the transcription service.
      */
     input_audio_transcription?: Session.InputAudioTranscription;
 
@@ -1891,7 +2045,8 @@ export namespace SessionUpdateEvent {
     output_audio_format?: 'pcm16' | 'g711_ulaw' | 'g711_alaw';
 
     /**
-     * Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8.
+     * Sampling temperature for the model, limited to [0.6, 1.2]. For audio models a
+     * temperature of 0.8 is highly recommended for best performance.
      */
     temperature?: number;
 
@@ -1907,9 +2062,16 @@ export namespace SessionUpdateEvent {
     tools?: Array<Session.Tool>;
 
     /**
-     * Configuration for turn detection. Can be set to `null` to turn off. Server VAD
-     * means that the model will detect the start and end of speech based on audio
-     * volume and respond at the end of user speech.
+     * Configuration for turn detection, ether Server VAD or Semantic VAD. This can be
+     * set to `null` to turn off, in which case the client must manually trigger model
+     * response. Server VAD means that the model will detect the start and end of
+     * speech based on audio volume and respond at the end of user speech. Semantic VAD
+     * is more advanced and uses a turn detection model (in conjuction with VAD) to
+     * semantically estimate whether the user has finished speaking, then dynamically
+     * sets a timeout based on this probability. For example, if user audio trails off
+     * with "uhhm", the model will score a low probability of turn end and wait longer
+     * for the user to continue speaking. This can be useful for more natural
+     * conversations, but may have a higher latency.
      */
     turn_detection?: Session.TurnDetection;
 
@@ -1922,15 +2084,31 @@ export namespace SessionUpdateEvent {
   }
 
   export namespace Session {
+    /**
+     * Configuration for input audio noise reduction. This can be set to `null` to turn
+     * off. Noise reduction filters audio added to the input audio buffer before it is
+     * sent to VAD and the model. Filtering the audio can improve VAD and turn
+     * detection accuracy (reducing false positives) and model performance by improving
+     * perception of the input audio.
+     */
+    export interface InputAudioNoiseReduction {
+      /**
+       * Type of noise reduction. `near_field` is for close-talking microphones such as
+       * headphones, `far_field` is for far-field microphones such as laptop or
+       * conference room microphones.
+       */
+      type?: 'near_field' | 'far_field';
+    }
+
     /**
      * Configuration for input audio transcription, defaults to off and can be set to
      * `null` to turn off once on. Input audio transcription is not native to the
      * model, since the model consumes audio directly. Transcription runs
      * asynchronously through
-     * [OpenAI Whisper transcription](https://platform.openai.com/docs/api-reference/audio/createTranscription)
-     * and should be treated as rough guidance rather than the representation
-     * understood by the model. The client can optionally set the language and prompt
-     * for transcription, these fields will be passed to the Whisper API.
+     * [the /audio/transcriptions endpoint](https://platform.openai.com/docs/api-reference/audio/createTranscription)
+     * and should be treated as guidance of input audio content rather than precisely
+     * what the model heard. The client can optionally set the language and prompt for
+     * transcription, these offer additional guidance to the transcription service.
      */
     export interface InputAudioTranscription {
       /**
@@ -1941,16 +2119,17 @@ export namespace SessionUpdateEvent {
       language?: string;
 
       /**
-       * The model to use for transcription, `whisper-1` is the only currently supported
-       * model.
+       * The model to use for transcription, current options are `gpt-4o-transcribe`,
+       * `gpt-4o-mini-transcribe`, and `whisper-1`.
        */
       model?: string;
 
       /**
        * An optional text to guide the model's style or continue a previous audio
-       * segment. The
-       * [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
-       * should match the audio language.
+       * segment. For `whisper-1`, the
+       * [prompt is a list of keywords](https://platform.openai.com/docs/guides/speech-to-text#prompting).
+       * For `gpt-4o-transcribe` models, the prompt is a free text string, for example
+       * "expect words related to technology".
        */
       prompt?: string;
     }
@@ -1979,48 +2158,62 @@ export namespace SessionUpdateEvent {
     }
 
     /**
-     * Configuration for turn detection. Can be set to `null` to turn off. Server VAD
-     * means that the model will detect the start and end of speech based on audio
-     * volume and respond at the end of user speech.
+     * Configuration for turn detection, ether Server VAD or Semantic VAD. This can be
+     * set to `null` to turn off, in which case the client must manually trigger model
+     * response. Server VAD means that the model will detect the start and end of
+     * speech based on audio volume and respond at the end of user speech. Semantic VAD
+     * is more advanced and uses a turn detection model (in conjuction with VAD) to
+     * semantically estimate whether the user has finished speaking, then dynamically
+     * sets a timeout based on this probability. For example, if user audio trails off
+     * with "uhhm", the model will score a low probability of turn end and wait longer
+     * for the user to continue speaking. This can be useful for more natural
+     * conversations, but may have a higher latency.
      */
     export interface TurnDetection {
       /**
        * Whether or not to automatically generate a response when a VAD stop event
-       * occurs. `true` by default.
+       * occurs.
        */
       create_response?: boolean;
 
+      /**
+       * Used only for `semantic_vad` mode. The eagerness of the model to respond. `low`
+       * will wait longer for the user to continue speaking, `high` will respond more
+       * quickly. `auto` is the default and is equivalent to `medium`.
+       */
+      eagerness?: 'low' | 'medium' | 'high' | 'auto';
+
       /**
        * Whether or not to automatically interrupt any ongoing response with output to
        * the default conversation (i.e. `conversation` of `auto`) when a VAD start event
-       * occurs. `true` by default.
+       * occurs.
        */
       interrupt_response?: boolean;
 
       /**
-       * Amount of audio to include before the VAD detected speech (in milliseconds).
-       * Defaults to 300ms.
+       * Used only for `server_vad` mode. Amount of audio to include before the VAD
+       * detected speech (in milliseconds). Defaults to 300ms.
        */
       prefix_padding_ms?: number;
 
       /**
-       * Duration of silence to detect speech stop (in milliseconds). Defaults to 500ms.
-       * With shorter values the model will respond more quickly, but may jump in on
-       * short pauses from the user.
+       * Used only for `server_vad` mode. Duration of silence to detect speech stop (in
+       * milliseconds). Defaults to 500ms. With shorter values the model will respond
+       * more quickly, but may jump in on short pauses from the user.
        */
       silence_duration_ms?: number;
 
       /**
-       * Activation threshold for VAD (0.0 to 1.0), this defaults to 0.5. A higher
-       * threshold will require louder audio to activate the model, and thus might
-       * perform better in noisy environments.
+       * Used only for `server_vad` mode. Activation threshold for VAD (0.0 to 1.0), this
+       * defaults to 0.5. A higher threshold will require louder audio to activate the
+       * model, and thus might perform better in noisy environments.
        */
       threshold?: number;
 
       /**
-       * Type of turn detection, only `server_vad` is currently supported.
+       * Type of turn detection.
        */
-      type?: string;
+      type?: 'server_vad' | 'semantic_vad';
     }
   }
 }
@@ -2046,7 +2239,216 @@ export interface SessionUpdatedEvent {
   type: 'session.updated';
 }
 
+/**
+ * Send this event to update a transcription session.
+ */
+export interface TranscriptionSessionUpdate {
+  /**
+   * Realtime transcription session object configuration.
+   */
+  session: TranscriptionSessionUpdate.Session;
+
+  /**
+   * The event type, must be `transcription_session.update`.
+   */
+  type: 'transcription_session.update';
+
+  /**
+   * Optional client-generated ID used to identify this event.
+   */
+  event_id?: string;
+}
+
+export namespace TranscriptionSessionUpdate {
+  /**
+   * Realtime transcription session object configuration.
+   */
+  export interface Session {
+    /**
+     * The set of items to include in the transcription. Current available items are:
+     *
+     * - `item.input_audio_transcription.logprobs`
+     */
+    include?: Array<string>;
+
+    /**
+     * The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`. For
+     * `pcm16`, input audio must be 16-bit PCM at a 24kHz sample rate, single channel
+     * (mono), and little-endian byte order.
+     */
+    input_audio_format?: 'pcm16' | 'g711_ulaw' | 'g711_alaw';
+
+    /**
+     * Configuration for input audio noise reduction. This can be set to `null` to turn
+     * off. Noise reduction filters audio added to the input audio buffer before it is
+     * sent to VAD and the model. Filtering the audio can improve VAD and turn
+     * detection accuracy (reducing false positives) and model performance by improving
+     * perception of the input audio.
+     */
+    input_audio_noise_reduction?: Session.InputAudioNoiseReduction;
+
+    /**
+     * Configuration for input audio transcription. The client can optionally set the
+     * language and prompt for transcription, these offer additional guidance to the
+     * transcription service.
+     */
+    input_audio_transcription?: Session.InputAudioTranscription;
+
+    /**
+     * The set of modalities the model can respond with. To disable audio, set this to
+     * ["text"].
+     */
+    modalities?: Array<'text' | 'audio'>;
+
+    /**
+     * Configuration for turn detection, ether Server VAD or Semantic VAD. This can be
+     * set to `null` to turn off, in which case the client must manually trigger model
+     * response. Server VAD means that the model will detect the start and end of
+     * speech based on audio volume and respond at the end of user speech. Semantic VAD
+     * is more advanced and uses a turn detection model (in conjuction with VAD) to
+     * semantically estimate whether the user has finished speaking, then dynamically
+     * sets a timeout based on this probability. For example, if user audio trails off
+     * with "uhhm", the model will score a low probability of turn end and wait longer
+     * for the user to continue speaking. This can be useful for more natural
+     * conversations, but may have a higher latency.
+     */
+    turn_detection?: Session.TurnDetection;
+  }
+
+  export namespace Session {
+    /**
+     * Configuration for input audio noise reduction. This can be set to `null` to turn
+     * off. Noise reduction filters audio added to the input audio buffer before it is
+     * sent to VAD and the model. Filtering the audio can improve VAD and turn
+     * detection accuracy (reducing false positives) and model performance by improving
+     * perception of the input audio.
+     */
+    export interface InputAudioNoiseReduction {
+      /**
+       * Type of noise reduction. `near_field` is for close-talking microphones such as
+       * headphones, `far_field` is for far-field microphones such as laptop or
+       * conference room microphones.
+       */
+      type?: 'near_field' | 'far_field';
+    }
+
+    /**
+     * Configuration for input audio transcription. The client can optionally set the
+     * language and prompt for transcription, these offer additional guidance to the
+     * transcription service.
+     */
+    export interface InputAudioTranscription {
+      /**
+       * The language of the input audio. Supplying the input language in
+       * [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (e.g. `en`)
+       * format will improve accuracy and latency.
+       */
+      language?: string;
+
+      /**
+       * The model to use for transcription, current options are `gpt-4o-transcribe`,
+       * `gpt-4o-mini-transcribe`, and `whisper-1`.
+       */
+      model?: 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe' | 'whisper-1';
+
+      /**
+       * An optional text to guide the model's style or continue a previous audio
+       * segment. For `whisper-1`, the
+       * [prompt is a list of keywords](https://platform.openai.com/docs/guides/speech-to-text#prompting).
+       * For `gpt-4o-transcribe` models, the prompt is a free text string, for example
+       * "expect words related to technology".
+       */
+      prompt?: string;
+    }
+
+    /**
+     * Configuration for turn detection, ether Server VAD or Semantic VAD. This can be
+     * set to `null` to turn off, in which case the client must manually trigger model
+     * response. Server VAD means that the model will detect the start and end of
+     * speech based on audio volume and respond at the end of user speech. Semantic VAD
+     * is more advanced and uses a turn detection model (in conjuction with VAD) to
+     * semantically estimate whether the user has finished speaking, then dynamically
+     * sets a timeout based on this probability. For example, if user audio trails off
+     * with "uhhm", the model will score a low probability of turn end and wait longer
+     * for the user to continue speaking. This can be useful for more natural
+     * conversations, but may have a higher latency.
+     */
+    export interface TurnDetection {
+      /**
+       * Whether or not to automatically generate a response when a VAD stop event
+       * occurs.
+       */
+      create_response?: boolean;
+
+      /**
+       * Used only for `semantic_vad` mode. The eagerness of the model to respond. `low`
+       * will wait longer for the user to continue speaking, `high` will respond more
+       * quickly. `auto` is the default and is equivalent to `medium`.
+       */
+      eagerness?: 'low' | 'medium' | 'high' | 'auto';
+
+      /**
+       * Whether or not to automatically interrupt any ongoing response with output to
+       * the default conversation (i.e. `conversation` of `auto`) when a VAD start event
+       * occurs.
+       */
+      interrupt_response?: boolean;
+
+      /**
+       * Used only for `server_vad` mode. Amount of audio to include before the VAD
+       * detected speech (in milliseconds). Defaults to 300ms.
+       */
+      prefix_padding_ms?: number;
+
+      /**
+       * Used only for `server_vad` mode. Duration of silence to detect speech stop (in
+       * milliseconds). Defaults to 500ms. With shorter values the model will respond
+       * more quickly, but may jump in on short pauses from the user.
+       */
+      silence_duration_ms?: number;
+
+      /**
+       * Used only for `server_vad` mode. Activation threshold for VAD (0.0 to 1.0), this
+       * defaults to 0.5. A higher threshold will require louder audio to activate the
+       * model, and thus might perform better in noisy environments.
+       */
+      threshold?: number;
+
+      /**
+       * Type of turn detection.
+       */
+      type?: 'server_vad' | 'semantic_vad';
+    }
+  }
+}
+
+/**
+ * Returned when a transcription session is updated with a
+ * `transcription_session.update` event, unless there is an error.
+ */
+export interface TranscriptionSessionUpdatedEvent {
+  /**
+   * The unique ID of the server event.
+   */
+  event_id: string;
+
+  /**
+   * A new Realtime transcription session configuration.
+   *
+   * When a session is created on the server via REST API, the session object also
+   * contains an ephemeral key. Default TTL for keys is one minute. This property is
+   * not present when a session is updated via the WebSocket API.
+   */
+  session: TranscriptionSessionsAPI.TranscriptionSession;
+
+  /**
+   * The event type, must be `transcription_session.updated`.
+   */
+  type: 'transcription_session.updated';
+}
+
 Realtime.Sessions = Sessions;
+Realtime.TranscriptionSessions = TranscriptionSessions;
 
 export declare namespace Realtime {
   export {
@@ -2055,4 +2457,10 @@ export declare namespace Realtime {
     type SessionCreateResponse as SessionCreateResponse,
     type SessionCreateParams as SessionCreateParams,
   };
+
+  export {
+    TranscriptionSessions as TranscriptionSessions,
+    type TranscriptionSession as TranscriptionSession,
+    type TranscriptionSessionCreateParams as TranscriptionSessionCreateParams,
+  };
 }