openai 4.87.4 → 4.89.0

package/src/resources/beta/realtime/sessions.ts

@@ -27,7 +27,7 @@ export class Sessions extends APIResource {
  */
 export interface Session {
   /**
-   * Unique identifier for the session object.
+   * Unique identifier for the session that looks like `sess_1234567890abcdef`.
    */
   id?: string;
 
@@ -38,12 +38,24 @@ export interface Session {
    */
   input_audio_format?: 'pcm16' | 'g711_ulaw' | 'g711_alaw';
 
+  /**
+   * Configuration for input audio noise reduction. This can be set to `null` to turn
+   * off. Noise reduction filters audio added to the input audio buffer before it is
+   * sent to VAD and the model. Filtering the audio can improve VAD and turn
+   * detection accuracy (reducing false positives) and model performance by improving
+   * perception of the input audio.
+   */
+  input_audio_noise_reduction?: Session.InputAudioNoiseReduction;
+
   /**
    * Configuration for input audio transcription, defaults to off and can be set to
    * `null` to turn off once on. Input audio transcription is not native to the
    * model, since the model consumes audio directly. Transcription runs
-   * asynchronously through Whisper and should be treated as rough guidance rather
-   * than the representation understood by the model.
+   * asynchronously through
+   * [the /audio/transcriptions endpoint](https://platform.openai.com/docs/api-reference/audio/createTranscription)
+   * and should be treated as guidance of input audio content rather than precisely
+   * what the model heard. The client can optionally set the language and prompt for
+   * transcription, these offer additional guidance to the transcription service.
    */
   input_audio_transcription?: Session.InputAudioTranscription;
 
@@ -79,7 +91,6 @@ export interface Session {
    * The Realtime model used for this session.
    */
   model?:
-    | (string & {})
     | 'gpt-4o-realtime-preview'
     | 'gpt-4o-realtime-preview-2024-10-01'
     | 'gpt-4o-realtime-preview-2024-12-17'
@@ -93,7 +104,8 @@ export interface Session {
   output_audio_format?: 'pcm16' | 'g711_ulaw' | 'g711_alaw';
 
   /**
-   * Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8.
+   * Sampling temperature for the model, limited to [0.6, 1.2]. For audio models a
+   * temperature of 0.8 is highly recommended for best performance.
    */
   temperature?: number;
 
@@ -109,11 +121,18 @@ export interface Session {
   tools?: Array<Session.Tool>;
 
   /**
-   * Configuration for turn detection. Can be set to `null` to turn off. Server VAD
-   * means that the model will detect the start and end of speech based on audio
-   * volume and respond at the end of user speech.
+   * Configuration for turn detection, ether Server VAD or Semantic VAD. This can be
+   * set to `null` to turn off, in which case the client must manually trigger model
+   * response. Server VAD means that the model will detect the start and end of
+   * speech based on audio volume and respond at the end of user speech. Semantic VAD
+   * is more advanced and uses a turn detection model (in conjuction with VAD) to
+   * semantically estimate whether the user has finished speaking, then dynamically
+   * sets a timeout based on this probability. For example, if user audio trails off
+   * with "uhhm", the model will score a low probability of turn end and wait longer
+   * for the user to continue speaking. This can be useful for more natural
+   * conversations, but may have a higher latency.
    */
-  turn_detection?: Session.TurnDetection | null;
+  turn_detection?: Session.TurnDetection;
 
   /**
    * The voice the model uses to respond. Voice cannot be changed during the session
@@ -124,19 +143,54 @@ export interface Session {
 }
 
 export namespace Session {
+  /**
+   * Configuration for input audio noise reduction. This can be set to `null` to turn
+   * off. Noise reduction filters audio added to the input audio buffer before it is
+   * sent to VAD and the model. Filtering the audio can improve VAD and turn
+   * detection accuracy (reducing false positives) and model performance by improving
+   * perception of the input audio.
+   */
+  export interface InputAudioNoiseReduction {
+    /**
+     * Type of noise reduction. `near_field` is for close-talking microphones such as
+     * headphones, `far_field` is for far-field microphones such as laptop or
+     * conference room microphones.
+     */
+    type?: 'near_field' | 'far_field';
+  }
+
   /**
    * Configuration for input audio transcription, defaults to off and can be set to
    * `null` to turn off once on. Input audio transcription is not native to the
    * model, since the model consumes audio directly. Transcription runs
-   * asynchronously through Whisper and should be treated as rough guidance rather
-   * than the representation understood by the model.
+   * asynchronously through
+   * [the /audio/transcriptions endpoint](https://platform.openai.com/docs/api-reference/audio/createTranscription)
+   * and should be treated as guidance of input audio content rather than precisely
+   * what the model heard. The client can optionally set the language and prompt for
+   * transcription, these offer additional guidance to the transcription service.
    */
   export interface InputAudioTranscription {
     /**
-     * The model to use for transcription, `whisper-1` is the only currently supported
-     * model.
+     * The language of the input audio. Supplying the input language in
+     * [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (e.g. `en`)
+     * format will improve accuracy and latency.
+     */
+    language?: string;
+
+    /**
+     * The model to use for transcription, current options are `gpt-4o-transcribe`,
+     * `gpt-4o-mini-transcribe`, and `whisper-1`.
      */
     model?: string;
+
+    /**
+     * An optional text to guide the model's style or continue a previous audio
+     * segment. For `whisper-1`, the
+     * [prompt is a list of keywords](https://platform.openai.com/docs/guides/speech-to-text#prompting).
+     * For `gpt-4o-transcribe` models, the prompt is a free text string, for example
+     * "expect words related to technology".
+     */
+    prompt?: string;
   }
 
   export interface Tool {
@@ -163,48 +217,62 @@ export namespace Session {
   }
 
   /**
-   * Configuration for turn detection. Can be set to `null` to turn off. Server VAD
-   * means that the model will detect the start and end of speech based on audio
-   * volume and respond at the end of user speech.
+   * Configuration for turn detection, ether Server VAD or Semantic VAD. This can be
+   * set to `null` to turn off, in which case the client must manually trigger model
+   * response. Server VAD means that the model will detect the start and end of
+   * speech based on audio volume and respond at the end of user speech. Semantic VAD
+   * is more advanced and uses a turn detection model (in conjuction with VAD) to
+   * semantically estimate whether the user has finished speaking, then dynamically
+   * sets a timeout based on this probability. For example, if user audio trails off
+   * with "uhhm", the model will score a low probability of turn end and wait longer
+   * for the user to continue speaking. This can be useful for more natural
+   * conversations, but may have a higher latency.
    */
   export interface TurnDetection {
     /**
      * Whether or not to automatically generate a response when a VAD stop event
-     * occurs. `true` by default.
+     * occurs.
      */
     create_response?: boolean;
 
+    /**
+     * Used only for `semantic_vad` mode. The eagerness of the model to respond. `low`
+     * will wait longer for the user to continue speaking, `high` will respond more
+     * quickly. `auto` is the default and is equivalent to `medium`.
+     */
+    eagerness?: 'low' | 'medium' | 'high' | 'auto';
+
     /**
      * Whether or not to automatically interrupt any ongoing response with output to
      * the default conversation (i.e. `conversation` of `auto`) when a VAD start event
-     * occurs. `true` by default.
+     * occurs.
      */
     interrupt_response?: boolean;
 
     /**
-     * Amount of audio to include before the VAD detected speech (in milliseconds).
-     * Defaults to 300ms.
+     * Used only for `server_vad` mode. Amount of audio to include before the VAD
+     * detected speech (in milliseconds). Defaults to 300ms.
      */
     prefix_padding_ms?: number;
 
     /**
-     * Duration of silence to detect speech stop (in milliseconds). Defaults to 500ms.
-     * With shorter values the model will respond more quickly, but may jump in on
-     * short pauses from the user.
+     * Used only for `server_vad` mode. Duration of silence to detect speech stop (in
+     * milliseconds). Defaults to 500ms. With shorter values the model will respond
+     * more quickly, but may jump in on short pauses from the user.
      */
     silence_duration_ms?: number;
 
     /**
-     * Activation threshold for VAD (0.0 to 1.0), this defaults to 0.5. A higher
-     * threshold will require louder audio to activate the model, and thus might
-     * perform better in noisy environments.
+     * Used only for `server_vad` mode. Activation threshold for VAD (0.0 to 1.0), this
+     * defaults to 0.5. A higher threshold will require louder audio to activate the
+     * model, and thus might perform better in noisy environments.
      */
     threshold?: number;
 
     /**
-     * Type of turn detection, only `server_vad` is currently supported.
+     * Type of turn detection.
      */
-    type?: 'server_vad';
+    type?: 'server_vad' | 'semantic_vad';
   }
 }
 
@@ -394,15 +462,24 @@ export interface SessionCreateParams {
    */
   input_audio_format?: 'pcm16' | 'g711_ulaw' | 'g711_alaw';
 
+  /**
+   * Configuration for input audio noise reduction. This can be set to `null` to turn
+   * off. Noise reduction filters audio added to the input audio buffer before it is
+   * sent to VAD and the model. Filtering the audio can improve VAD and turn
+   * detection accuracy (reducing false positives) and model performance by improving
+   * perception of the input audio.
+   */
+  input_audio_noise_reduction?: SessionCreateParams.InputAudioNoiseReduction;
+
   /**
    * Configuration for input audio transcription, defaults to off and can be set to
    * `null` to turn off once on. Input audio transcription is not native to the
    * model, since the model consumes audio directly. Transcription runs
    * asynchronously through
-   * [OpenAI Whisper transcription](https://platform.openai.com/docs/api-reference/audio/createTranscription)
-   * and should be treated as rough guidance rather than the representation
-   * understood by the model. The client can optionally set the language and prompt
-   * for transcription, these fields will be passed to the Whisper API.
+   * [the /audio/transcriptions endpoint](https://platform.openai.com/docs/api-reference/audio/createTranscription)
+   * and should be treated as guidance of input audio content rather than precisely
+   * what the model heard. The client can optionally set the language and prompt for
+   * transcription, these offer additional guidance to the transcription service.
    */
   input_audio_transcription?: SessionCreateParams.InputAudioTranscription;
 
@@ -451,7 +528,8 @@ export interface SessionCreateParams {
   output_audio_format?: 'pcm16' | 'g711_ulaw' | 'g711_alaw';
 
   /**
-   * Sampling temperature for the model, limited to [0.6, 1.2]. Defaults to 0.8.
+   * Sampling temperature for the model, limited to [0.6, 1.2]. For audio models a
+   * temperature of 0.8 is highly recommended for best performance.
    */
   temperature?: number;
 
@@ -467,9 +545,16 @@ export interface SessionCreateParams {
   tools?: Array<SessionCreateParams.Tool>;
 
   /**
-   * Configuration for turn detection. Can be set to `null` to turn off. Server VAD
-   * means that the model will detect the start and end of speech based on audio
-   * volume and respond at the end of user speech.
+   * Configuration for turn detection, ether Server VAD or Semantic VAD. This can be
+   * set to `null` to turn off, in which case the client must manually trigger model
+   * response. Server VAD means that the model will detect the start and end of
+   * speech based on audio volume and respond at the end of user speech. Semantic VAD
+   * is more advanced and uses a turn detection model (in conjuction with VAD) to
+   * semantically estimate whether the user has finished speaking, then dynamically
+   * sets a timeout based on this probability. For example, if user audio trails off
+   * with "uhhm", the model will score a low probability of turn end and wait longer
+   * for the user to continue speaking. This can be useful for more natural
+   * conversations, but may have a higher latency.
    */
   turn_detection?: SessionCreateParams.TurnDetection;
 
@@ -482,15 +567,31 @@ export interface SessionCreateParams {
 }
 
 export namespace SessionCreateParams {
+  /**
+   * Configuration for input audio noise reduction. This can be set to `null` to turn
+   * off. Noise reduction filters audio added to the input audio buffer before it is
+   * sent to VAD and the model. Filtering the audio can improve VAD and turn
+   * detection accuracy (reducing false positives) and model performance by improving
+   * perception of the input audio.
+   */
+  export interface InputAudioNoiseReduction {
+    /**
+     * Type of noise reduction. `near_field` is for close-talking microphones such as
+     * headphones, `far_field` is for far-field microphones such as laptop or
+     * conference room microphones.
+     */
+    type?: 'near_field' | 'far_field';
+  }
+
   /**
    * Configuration for input audio transcription, defaults to off and can be set to
    * `null` to turn off once on. Input audio transcription is not native to the
    * model, since the model consumes audio directly. Transcription runs
    * asynchronously through
-   * [OpenAI Whisper transcription](https://platform.openai.com/docs/api-reference/audio/createTranscription)
-   * and should be treated as rough guidance rather than the representation
-   * understood by the model. The client can optionally set the language and prompt
-   * for transcription, these fields will be passed to the Whisper API.
+   * [the /audio/transcriptions endpoint](https://platform.openai.com/docs/api-reference/audio/createTranscription)
+   * and should be treated as guidance of input audio content rather than precisely
+   * what the model heard. The client can optionally set the language and prompt for
+   * transcription, these offer additional guidance to the transcription service.
    */
   export interface InputAudioTranscription {
     /**
@@ -501,16 +602,17 @@ export namespace SessionCreateParams {
     language?: string;
 
     /**
-     * The model to use for transcription, `whisper-1` is the only currently supported
-     * model.
+     * The model to use for transcription, current options are `gpt-4o-transcribe`,
+     * `gpt-4o-mini-transcribe`, and `whisper-1`.
      */
     model?: string;
 
     /**
      * An optional text to guide the model's style or continue a previous audio
-     * segment. The
-     * [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
-     * should match the audio language.
+     * segment. For `whisper-1`, the
+     * [prompt is a list of keywords](https://platform.openai.com/docs/guides/speech-to-text#prompting).
+     * For `gpt-4o-transcribe` models, the prompt is a free text string, for example
+     * "expect words related to technology".
      */
     prompt?: string;
   }
@@ -539,48 +641,62 @@ export namespace SessionCreateParams {
   }
 
   /**
-   * Configuration for turn detection. Can be set to `null` to turn off. Server VAD
-   * means that the model will detect the start and end of speech based on audio
-   * volume and respond at the end of user speech.
+   * Configuration for turn detection, ether Server VAD or Semantic VAD. This can be
+   * set to `null` to turn off, in which case the client must manually trigger model
+   * response. Server VAD means that the model will detect the start and end of
+   * speech based on audio volume and respond at the end of user speech. Semantic VAD
+   * is more advanced and uses a turn detection model (in conjuction with VAD) to
+   * semantically estimate whether the user has finished speaking, then dynamically
+   * sets a timeout based on this probability. For example, if user audio trails off
+   * with "uhhm", the model will score a low probability of turn end and wait longer
+   * for the user to continue speaking. This can be useful for more natural
+   * conversations, but may have a higher latency.
    */
   export interface TurnDetection {
     /**
      * Whether or not to automatically generate a response when a VAD stop event
-     * occurs. `true` by default.
+     * occurs.
      */
     create_response?: boolean;
 
+    /**
+     * Used only for `semantic_vad` mode. The eagerness of the model to respond. `low`
+     * will wait longer for the user to continue speaking, `high` will respond more
+     * quickly. `auto` is the default and is equivalent to `medium`.
+     */
+    eagerness?: 'low' | 'medium' | 'high' | 'auto';
+
     /**
      * Whether or not to automatically interrupt any ongoing response with output to
      * the default conversation (i.e. `conversation` of `auto`) when a VAD start event
-     * occurs. `true` by default.
+     * occurs.
      */
     interrupt_response?: boolean;
 
     /**
-     * Amount of audio to include before the VAD detected speech (in milliseconds).
-     * Defaults to 300ms.
+     * Used only for `server_vad` mode. Amount of audio to include before the VAD
+     * detected speech (in milliseconds). Defaults to 300ms.
      */
     prefix_padding_ms?: number;
 
     /**
-     * Duration of silence to detect speech stop (in milliseconds). Defaults to 500ms.
-     * With shorter values the model will respond more quickly, but may jump in on
-     * short pauses from the user.
+     * Used only for `server_vad` mode. Duration of silence to detect speech stop (in
+     * milliseconds). Defaults to 500ms. With shorter values the model will respond
+     * more quickly, but may jump in on short pauses from the user.
      */
     silence_duration_ms?: number;
 
     /**
-     * Activation threshold for VAD (0.0 to 1.0), this defaults to 0.5. A higher
-     * threshold will require louder audio to activate the model, and thus might
-     * perform better in noisy environments.
+     * Used only for `server_vad` mode. Activation threshold for VAD (0.0 to 1.0), this
+     * defaults to 0.5. A higher threshold will require louder audio to activate the
+     * model, and thus might perform better in noisy environments.
      */
     threshold?: number;
 
     /**
-     * Type of turn detection, only `server_vad` is currently supported.
+     * Type of turn detection.
      */
-    type?: string;
+    type?: 'server_vad' | 'semantic_vad';
   }
 }