@pipedog/openai 0.0.0 → 0.4.0

package/dist/stt.d.ts

@@ -0,0 +1,409 @@
+/**
+ * OpenAI Speech-to-Text service implementations.
+ *
+ * Provides two STT services:
+ *
+ * - `OpenAISTTService`: REST-based transcription using the Audio API
+ *   (Whisper / GPT transcription models).
+ * - `OpenAIRealtimeSTTService`: WebSocket-based streaming transcription
+ *   using the Realtime API in transcription-only mode.
+ *
+ * Porting notes:
+ * - `OpenAISTTService` uses the `openai` JS SDK client built by
+ *   `BaseWhisperSTTService` (`@pipedog/whisper`).
+ * - `OpenAIRealtimeSTTService` uses the global `WebSocket` through
+ *   `WebsocketService._websocketConnect()`. The `Authorization` header is sent
+ *   on the opening handshake, which the Bun, Node and Deno `WebSocket`
+ *   support; on Cloudflare Workers override `_connectWebsocket()` or the
+ *   connect options to open the socket through `fetch`.
+ * - Deprecated APIs are not ported: the `model`, `language`, `prompt`,
+ *   `temperature` (`OpenAISTTService`) and `model`, `language`, `prompt`,
+ *   `noiseReduction` (`OpenAIRealtimeSTTService`) constructor options.
+ */
+import { type Task } from "@pipedog/asyncio";
+import type { BaseAudioResampler } from "@pipedog/core/audio/resamplers/base_audio_resampler";
+import { type CancelFrame, type EndFrame, type Frame, type STTMetadataFrame } from "@pipedog/core/frames/frames";
+import type { FrameDirection, FrameProcessorSetup } from "@pipedog/core/processors/frame_processor";
+import { STTSettings, type STTSettingsOptions } from "@pipedog/core/services/settings";
+import { WebsocketSTTService, type WebsocketSTTServiceOptions } from "@pipedog/core/services/stt_service";
+import { Language } from "@pipedog/core/transcriptions/language";
+import { type NotGiven } from "@pipedog/core/utils/types";
+import { BaseWhisperSTTService, type BaseWhisperSTTServiceOptions, BaseWhisperSTTSettings, type BaseWhisperSTTSettingsOptions, type Transcription } from "@pipedog/whisper/base_stt";
+type Dict = Record<string, any>;
+export interface OpenAISTTSettingsOptions extends BaseWhisperSTTSettingsOptions {
+    /**
+     * Words or phrases that guide transcription of the input audio, such as
+     * product names or acronyms. Supported by "gpt-transcribe".
+     */
+    keywords?: string[] | null | NotGiven;
+}
+/**
+ * Settings for the OpenAI STT service.
+ *
+ * - `keywords`: words or phrases that guide transcription of the input
+ *   audio, such as product names or acronyms. Supported by
+ *   "gpt-transcribe".
+ */
+export declare class OpenAISTTSettings extends BaseWhisperSTTSettings {
+    keywords: string[] | null | NotGiven;
+    constructor(options?: OpenAISTTSettingsOptions);
+}
+export interface OpenAISTTServiceOptions extends BaseWhisperSTTServiceOptions {
+    /** Runtime-updatable settings. */
+    settings?: OpenAISTTSettings | null;
+}
+/**
+ * OpenAI Speech-to-Text service that generates text from audio.
+ *
+ * Uses OpenAI's transcription API to convert audio to text. Requires an
+ * OpenAI API key set via the apiKey option or OPENAI_API_KEY environment
+ * variable.
+ *
+ * With `includeProbMetrics: true`, GPT transcription models report per-token
+ * logprobs and Whisper models report per-segment logprobs. Diarization models
+ * support neither, so they report no probabilities.
+ */
+export declare class OpenAISTTService extends BaseWhisperSTTService {
+    static Settings: typeof OpenAISTTSettings;
+    protected _settings: OpenAISTTSettings;
+    /**
+     * Model already reported as unable to supply probabilities, so that warning
+     * is logged once per model rather than once per utterance.
+     */
+    protected _probMetricsWarnedModel: string | null;
+    /**
+     * Initialize OpenAI STT service.
+     *
+     * @param options.apiKey OpenAI API key. Defaults to null.
+     * @param options.baseUrl API base URL. Defaults to null.
+     * @param options.settings Runtime-updatable settings. Its given fields
+     *   override the defaults (model "gpt-transcribe", language English).
+     * @param options.ttfsP99Latency P99 latency from speech end to final
+     *   transcript in seconds. Override for your deployment. See
+     *   https://github.com/pipecat-ai/stt-benchmark
+     *   Other options are passed to BaseWhisperSTTService.
+     */
+    constructor(options?: OpenAISTTServiceOptions);
+    protected _transcribe(audio: Uint8Array): Promise<Transcription>;
+}
+export type OpenAIRealtimeNoiseReduction = "near_field" | "far_field";
+export type OpenAIRealtimeDelay = "minimal" | "low" | "medium" | "high" | "xhigh";
+export interface OpenAIRealtimeSTTSettingsOptions extends STTSettingsOptions {
+    /**
+     * Optional prompt text to guide transcription style. Not supported by
+     * "gpt-realtime-whisper".
+     */
+    prompt?: string | null | NotGiven;
+    /**
+     * Noise reduction mode. "near_field" for close microphones, "far_field"
+     * for distant microphones, or null to disable.
+     */
+    noiseReduction?: OpenAIRealtimeNoiseReduction | null | NotGiven;
+    /**
+     * How long the model waits before emitting transcription text, trading
+     * latency for accuracy. Only supported by "gpt-realtime-whisper".
+     */
+    delay?: OpenAIRealtimeDelay | null | NotGiven;
+}
+/**
+ * Settings for OpenAIRealtimeSTTService.
+ *
+ * - `prompt`: optional prompt text to guide transcription style. Not
+ *   supported by "gpt-realtime-whisper".
+ * - `noiseReduction`: noise reduction mode. "near_field" for close
+ *   microphones, "far_field" for distant microphones, or null to disable.
+ * - `delay`: how long the model waits before emitting transcription text,
+ *   trading latency for accuracy. Only supported by "gpt-realtime-whisper".
+ */
+export declare class OpenAIRealtimeSTTSettings extends STTSettings {
+    prompt: string | null | NotGiven;
+    noiseReduction: OpenAIRealtimeNoiseReduction | null | NotGiven;
+    delay: OpenAIRealtimeDelay | null | NotGiven;
+    constructor(options?: OpenAIRealtimeSTTSettingsOptions);
+}
+export interface OpenAIRealtimeSTTServiceOptions extends WebsocketSTTServiceOptions {
+    /** OpenAI API key for authentication. */
+    apiKey: string;
+    /**
+     * WebSocket base URL for the Realtime API. Defaults to
+     * "wss://api.openai.com/v1/realtime".
+     */
+    baseUrl?: string;
+    /**
+     * Server-side VAD configuration. Defaults to `false` (disabled), which
+     * relies on a local VAD processor in the pipeline. Pass an object to enable
+     * it — at minimum `{ type: "server_vad" }`, optionally with custom settings
+     * (e.g. `{ type: "server_vad", threshold: 0.5 }`). Requires a model that
+     * supports turn detection. `null` omits the field entirely, leaving the
+     * session's own default in place.
+     */
+    turnDetection?: Dict | false | null;
+    /**
+     * Whether to interrupt bot output when speech is detected by server-side
+     * VAD. Only applies when turn detection is enabled. Passed along to the
+     * user turn strategies this service recommends, which own the
+     * interruption; a user-supplied `userTurnStrategies` overrides the
+     * recommendation and this setting with it. Defaults to true.
+     */
+    shouldInterrupt?: boolean;
+    /** Runtime-updatable settings. */
+    settings?: OpenAIRealtimeSTTSettings | null;
+}
+/**
+ * OpenAI Realtime Speech-to-Text service using WebSocket transcription sessions.
+ *
+ * Uses OpenAI's Realtime API in transcription-only mode for real-time
+ * streaming speech recognition with optional server-side VAD and noise
+ * reduction. The model does not generate conversational responses — only
+ * transcription output.
+ *
+ * This service supports two VAD modes:
+ *
+ * **Local VAD** (default): Disable server-side VAD and use a local VAD
+ * processor in the pipeline instead. When a `VADUserStoppedSpeakingFrame` is
+ * received, the service commits the audio buffer so that the server begins
+ * transcription for the completed speech segment.
+ *
+ * **Server-side VAD** (`turnDetection: { type: "server_vad" }`): The OpenAI
+ * server performs voice-activity detection. The service proposes turn
+ * boundaries when the server detects them, and recommends the external user
+ * turn strategies that resolve those proposals into turn frames. Do **not**
+ * use a separate VAD processor in the pipeline in this mode. Requires a model
+ * that supports turn detection — `gpt-transcribe` does, the default
+ * `gpt-realtime-whisper` does not.
+ *
+ * Audio is sent as 24 kHz 16-bit mono PCM as required by the OpenAI Realtime
+ * API. If the pipeline runs at a different sample rate (e.g. 16 kHz for
+ * Silero VAD compatibility), audio is automatically upsampled before sending.
+ *
+ * @example
+ * const stt = new OpenAIRealtimeSTTService({
+ *   apiKey: "sk-...",
+ *   settings: new OpenAIRealtimeSTTService.Settings({
+ *     model: "gpt-realtime-whisper",
+ *     noiseReduction: "near_field",
+ *   }),
+ * });
+ */
+export declare class OpenAIRealtimeSTTService extends WebsocketSTTService {
+    static Settings: typeof OpenAIRealtimeSTTSettings;
+    protected _settings: OpenAIRealtimeSTTSettings;
+    protected _apiKey: string;
+    protected _baseUrl: string;
+    protected _turnDetection: Dict | false | null;
+    protected _shouldInterrupt: boolean;
+    protected _receiveTask: Task<unknown> | null;
+    protected _sessionReady: boolean;
+    protected _resampler: BaseAudioResampler;
+    protected _serverVadEnabled: boolean;
+    /**
+     * Initialize the OpenAI Realtime STT service.
+     *
+     * @param options.apiKey OpenAI API key for authentication.
+     * @param options.baseUrl WebSocket base URL for the Realtime API. Defaults
+     *   to "wss://api.openai.com/v1/realtime".
+     * @param options.turnDetection Server-side VAD configuration. Defaults to
+     *   `false` (disabled).
+     * @param options.shouldInterrupt Whether to interrupt bot output when speech
+     *   is detected by server-side VAD. Defaults to true.
+     * @param options.settings Runtime-updatable settings. Its given fields
+     *   override the defaults (model "gpt-realtime-whisper", language English).
+     * @param options.ttfsP99Latency P99 latency from speech end to final
+     *   transcript in seconds. Override for your deployment. See
+     *   https://github.com/pipecat-ai/stt-benchmark
+     *   Other options are passed to parent WebsocketSTTService.
+     */
+    constructor(options: OpenAIRealtimeSTTServiceOptions);
+    /** Drop prompt settings that are not accepted by the selected model. */
+    static _omitUnsupportedPrompt(settings: OpenAIRealtimeSTTSettings): Record<string, unknown>;
+    /**
+     * Convert a Language value to an ISO-639-1 code.
+     *
+     * @param language The Language value.
+     * @returns Two-letter ISO-639-1 language code.
+     */
+    static _languageToCode(language: Language | string): string;
+    /**
+     * Check if the service can generate processing metrics.
+     *
+     * @returns True, as this service supports metrics generation.
+     */
+    canGenerateMetrics(): boolean;
+    /**
+     * Request external turn strategies when server-side VAD drives turns.
+     *
+     * With server-side VAD enabled the OpenAI server detects speech boundaries
+     * and this service proposes turns from them, so the user aggregator
+     * resolves those rather than running local VAD/smart-turn. With
+     * `turnDetection: false` (the default) the server emits no VAD events and
+     * the defaults are left in place. Applied unless the user passed their own
+     * `userTurnStrategies`.
+     */
+    serviceMetadataFrame(): STTMetadataFrame;
+    /**
+     * Apply a settings delta and send session update if needed.
+     *
+     * Sends a `session.update` to the server when the session is active.
+     *
+     * @param delta An STTSettings (or `OpenAIRealtimeSTTService.Settings`) delta.
+     * @returns Record mapping changed field names to their previous values.
+     */
+    protected _updateSettings(delta: STTSettings): Promise<Record<string, unknown>>;
+    /**
+     * Set up the service and connect.
+     *
+     * @param setup Configuration object containing setup parameters.
+     */
+    setup(setup: FrameProcessorSetup): Promise<void>;
+    /**
+     * Stop the service and close WebSocket connection.
+     *
+     * @param frame The end frame triggering service shutdown.
+     */
+    stop(frame: EndFrame): Promise<void>;
+    /**
+     * Cancel the service and close WebSocket connection.
+     *
+     * @param frame The cancel frame triggering service cancellation.
+     */
+    cancel(frame: CancelFrame): Promise<void>;
+    /**
+     * Send audio data to the transcription session.
+     *
+     * Audio is streamed over the WebSocket. Transcription results arrive
+     * asynchronously via the receive task and are pushed as
+     * `InterimTranscriptionFrame` or `TranscriptionFrame`.
+     *
+     * @param audio Raw audio bytes (16-bit mono PCM at the pipeline sample
+     *   rate). Automatically resampled to 24 kHz.
+     * @returns null — results are delivered via the WebSocket receive task.
+     */
+    runStt(audio: Uint8Array): AsyncGenerator<Frame | null, void, unknown>;
+    /**
+     * Process frames from the pipeline.
+     *
+     * Extends the base STT service to handle local VAD events when server-side
+     * VAD is disabled. On `VADUserStoppedSpeakingFrame`, commits the audio
+     * buffer so the server begins transcription for the completed speech
+     * segment.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow in the pipeline.
+     */
+    processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
+    /** Connect to the transcription endpoint and start receiving. */
+    _connect(): Promise<void>;
+    /** Disconnect and clean up background tasks. */
+    _disconnect(): Promise<void>;
+    /** Establish the WebSocket connection to the transcription endpoint. */
+    _connectWebsocket(): Promise<void>;
+    /** Close the WebSocket connection. */
+    _disconnectWebsocket(): Promise<void>;
+    /**
+     * Send a JSON message over the WebSocket.
+     *
+     * @param message The message object to serialize and send.
+     */
+    protected _wsSend(message: Dict): Promise<void>;
+    /** Send `session.update` to configure the transcription session. */
+    protected _sendSessionUpdate(): Promise<void>;
+    /**
+     * Send audio data via `input_audio_buffer.append`.
+     *
+     * Resamples from the pipeline sample rate to 24 kHz if needed.
+     *
+     * @param audio Raw audio bytes at the pipeline sample rate.
+     */
+    protected _sendAudio(audio: Uint8Array): Promise<void>;
+    /** Commit the current audio buffer for transcription. */
+    protected _commitAudioBuffer(): Promise<void>;
+    /** Clear the current audio buffer. */
+    protected _clearAudioBuffer(): Promise<void>;
+    /**
+     * Receive and dispatch server events from the transcription session.
+     *
+     * Called by `WebsocketService._receiveTaskHandler` which wraps this method
+     * with automatic reconnection on connection errors.
+     */
+    _receiveMessages(): Promise<void>;
+    /**
+     * Handle `session.created`.
+     *
+     * Sent immediately after connecting. We respond by configuring the session
+     * with our desired settings.
+     *
+     * @param _evt The session created event from the server.
+     */
+    protected _handleSessionCreated(_evt: Dict): Promise<void>;
+    /**
+     * Handle `session.updated`.
+     *
+     * The session is now fully configured and ready to transcribe.
+     *
+     * @param _evt The session updated event from the server.
+     */
+    protected _handleSessionUpdated(_evt: Dict): Promise<void>;
+    /**
+     * Handle incremental transcription text.
+     *
+     * For "gpt-realtime-whisper", "gpt-live-transcribe", and "gpt-transcribe",
+     * deltas contain low-latency streaming partial text.
+     *
+     * @param evt The delta event from the server.
+     */
+    protected _handleTranscriptionDelta(evt: Dict): Promise<void>;
+    /**
+     * Handle a completed transcription for a speech segment.
+     *
+     * Pushes a `TranscriptionFrame` and records the result for tracing.
+     *
+     * @param evt The completed event containing the full transcript.
+     */
+    protected _handleTranscriptionCompleted(evt: Dict): Promise<void>;
+    /**
+     * Record transcription result for tracing.
+     *
+     * @param _transcript The transcribed text.
+     * @param _isFinal Whether this is a final transcription result.
+     * @param _language Optional language of the transcription.
+     */
+    protected _handleTranscriptionTrace(_transcript: string, _isFinal: boolean, _language?: Language | null): Promise<void>;
+    /**
+     * Handle server-side VAD speech start.
+     *
+     * Proposes a turn start, which the user turn strategies resolve into a
+     * `UserStartedSpeakingFrame` and an interruption.
+     *
+     * @param _evt The `input_audio_buffer.speech_started` event.
+     */
+    protected _handleSpeechStarted(_evt: Dict): Promise<void>;
+    /**
+     * Handle server-side VAD speech stop.
+     *
+     * Proposes a turn stop. The audio buffer is automatically committed by the
+     * server when VAD is enabled.
+     *
+     * @param _evt The `input_audio_buffer.speech_stopped` event.
+     */
+    protected _handleSpeechStopped(_evt: Dict): Promise<void>;
+    /**
+     * Handle a transcription failure for a speech segment.
+     *
+     * Logs the error but does not treat it as fatal — the session remains
+     * active for subsequent turns.
+     *
+     * @param evt The failed event containing error details.
+     */
+    protected _handleTranscriptionFailed(evt: Dict): Promise<void>;
+    /**
+     * Handle a fatal error from the transcription session.
+     *
+     * Throws so that `WebsocketService` can decide whether to attempt
+     * reconnection.
+     *
+     * @param evt The error event.
+     */
+    protected _handleError(evt: Dict): Promise<void>;
+}
+export {};
+//# sourceMappingURL=stt.d.ts.map