@pipedog/openai 0.0.0 → 0.3.0

package/dist/realtime/index.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/realtime/index.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/services/openai/realtime/__init__.py
+// Upstream's package __init__ is empty: import the submodules directly.
+export {};
+//# sourceMappingURL=index.js.map

package/dist/realtime/llm.d.ts

@@ -0,0 +1,457 @@
+/**
+ * OpenAI Realtime LLM service implementation with WebSocket support.
+ *
+ * Porting notes:
+ * - The deprecated `model` and `sessionProperties` constructor options are
+ *   not ported; use `settings: new OpenAIRealtimeLLMService.Settings({ model,
+ *   sessionProperties })` instead.
+ * - The websocket is `websocketConnect()` from core (the global `WebSocket`).
+ * - Video frames are encoded with core's `encodeImage()` (upstream: Pillow).
+ * - The tracing decorators are applied in a static block (decorator syntax is
+ *   not erasable).
+ */
+import { Future, type Task } from "@pipedog/asyncio";
+import type { BaseLLMAdapter } from "@pipedog/core/adapters/base_llm_adapter";
+import type { ToolsSchema } from "@pipedog/core/adapters/schemas/tools_schema";
+import { type CancelFrame, type EndFrame, type Frame, FunctionCallCancelFrame, FunctionCallResultFrame, InputAudioRawFrame, InputImageRawFrame, LLMMessagesAppendFrame, LLMServiceMetadataFrame, SpeechControlParamsFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame } from "@pipedog/core/frames/frames";
+import { type LLMContext } from "@pipedog/core/processors/aggregators/llm_context";
+import { FrameDirection, type FrameProcessorSetup } from "@pipedog/core/processors/frame_processor";
+import { type LLMAdapterClass, LLMService, type LLMServiceOptions } from "@pipedog/core/services/llm_service";
+import { LLMSettings, type LLMSettingsOptions, type ServiceSettings } from "@pipedog/core/services/settings";
+import { type WebsocketClientConnection } from "@pipedog/core/services/websocket_service";
+import { Language } from "@pipedog/core/transcriptions/language";
+import { type NotGiven } from "@pipedog/core/utils/types";
+import { OpenAIRealtimeLLMAdapter } from "../open_ai_realtime_adapter.ts";
+import * as events from "./events.ts";
+export declare const AUTOSIZED_USER_AUDIO_PREROLL_MARGIN_SECS = 0.3;
+export declare const DEFAULT_USER_AUDIO_PREROLL_SECS = 0.5;
+type RealtimeAdapter = OpenAIRealtimeLLMAdapter & BaseLLMAdapter;
+export interface CurrentAudioResponseOptions {
+    /** Unique identifier for the audio response item. */
+    itemId: string;
+    /** Index of the audio content within the item. */
+    contentIndex: number;
+    /** Timestamp when the audio response started in milliseconds. */
+    startTimeMs: number;
+    /** Total size of audio data received in bytes. Defaults to 0. */
+    totalSize?: number;
+    /** ID of the server response the item belongs to. Defaults to "". */
+    responseId?: string;
+}
+/**
+ * Tracks the current audio response from the assistant.
+ *
+ * - `itemId`: unique identifier for the audio response item.
+ * - `contentIndex`: index of the audio content within the item.
+ * - `startTimeMs`: timestamp when the audio response started in milliseconds.
+ * - `totalSize`: total size of audio data received in bytes. Defaults to 0.
+ * - `responseId`: ID of the server response the item belongs to. Defaults to "".
+ */
+export declare class CurrentAudioResponse {
+    itemId: string;
+    contentIndex: number;
+    startTimeMs: number;
+    totalSize: number;
+    responseId: string;
+    constructor(options: CurrentAudioResponseOptions);
+}
+export interface OpenAIRealtimeLLMSettingsOptions extends LLMSettingsOptions {
+    /**
+     * OpenAI Realtime session properties (modalities, audio config, tools,
+     * etc.). `model` and `instructions` are synced bidirectionally with the
+     * top-level `model` and `systemInstruction` fields.
+     */
+    sessionProperties?: events.SessionProperties | NotGiven;
+}
+/**
+ * Settings for OpenAIRealtimeLLMService.
+ *
+ * - `sessionProperties`: OpenAI Realtime session properties (modalities,
+ *   audio config, tools, etc.). `model` and `instructions` are synced
+ *   bidirectionally with the top-level `model` and `systemInstruction` fields.
+ */
+export declare class OpenAIRealtimeLLMSettings extends LLMSettings {
+    sessionProperties: events.SessionProperties | NotGiven;
+    constructor(options?: OpenAIRealtimeLLMSettingsOptions);
+    /** Push top-level `model`/`systemInstruction` into `sessionProperties`. */
+    static _syncTopLevelToSp(settings: OpenAIRealtimeLLMSettings): void;
+    /**
+     * Merge a delta, keeping `model`/`systemInstruction` in sync with SP.
+     *
+     * When the delta contains `sessionProperties`, it **replaces** the stored
+     * SP wholesale (matching legacy behaviour). Top-level field values always
+     * take precedence over conflicting SP values.
+     */
+    applyUpdate(delta: this): Record<string, unknown>;
+    /**
+     * Build a delta from a plain dict, routing SP keys into `sessionProperties`.
+     *
+     * Keys that correspond to `SessionProperties` fields (except `model`) are
+     * collected into a nested `sessionProperties` value. `model` is always
+     * routed to the top-level field. Unknown keys go to `extra`. Keys may use
+     * upstream's snake_case spelling or the camelCase field names.
+     */
+    static fromMapping<T extends ServiceSettings>(this: new (options?: object) => T, settings: Record<string, unknown>): T;
+}
+export interface OpenAIRealtimeLLMServiceOptions extends LLMServiceOptions {
+    /** OpenAI API key for authentication. */
+    apiKey: string;
+    /**
+     * WebSocket base URL for the realtime API. Defaults to
+     * "wss://api.openai.com/v1/realtime".
+     */
+    baseUrl?: string;
+    /** Runtime-updatable settings for this service. */
+    settings?: OpenAIRealtimeLLMSettings | null;
+    /** Whether to start with audio input paused. Defaults to false. */
+    startAudioPaused?: boolean;
+    /** Whether to start with video input paused. Defaults to false. */
+    startVideoPaused?: boolean;
+    /**
+     * Detail level for video processing. Can be "auto", "low", or "high". This
+     * sets the image_detail parameter in the OpenAI Realtime API. "auto" lets
+     * the model decide, "low" is faster and uses fewer tokens, "high" provides
+     * more detail. Defaults to "auto".
+     */
+    videoFrameDetail?: events.ImageDetail;
+    /**
+     * In manual turn-detection mode (`turn_detection=False`, locally-driven
+     * turns), how much recent audio to replay after an interruption clears the
+     * input audio buffer, so the speech onset isn't lost. Defaults to null:
+     * auto-sized to the upstream VAD's `startSecs` plus a small margin, falling
+     * back to `DEFAULT_USER_AUDIO_PREROLL_SECS` when no VAD is present.
+     * Auto-sizing assumes VAD drives turn starts (the default
+     * `VADUserTurnStartStrategy`); set this explicitly if you use a non-VAD
+     * turn-start strategy. No effect when server-side turn detection is enabled.
+     */
+    userAudioPrerollSecs?: number | null;
+}
+/**
+ * OpenAI Realtime LLM service providing real-time audio and text communication.
+ *
+ * Implements the OpenAI Realtime API with WebSocket communication for
+ * low-latency bidirectional audio and text interactions. Supports function
+ * calling, conversation management, and real-time transcription.
+ *
+ * Proposes turn boundaries from OpenAI's server-side VAD events, which the
+ * recommended external user turn strategies resolve into
+ * `UserStartedSpeakingFrame` / `UserStoppedSpeakingFrame`, so pipeline
+ * processors that depend on those frames (RTVI client speech events,
+ * `TurnTrackingObserver`, `AudioBufferProcessor` turn recording,
+ * `UserIdleController`, user mute strategies, voicemail detector) work out of
+ * the box. `LLMContextAggregatorPair` auto-detects this realtime service and
+ * decouples context writes from those frames; see the
+ * `examples/realtime/realtime-openai.py` example.
+ *
+ * If you wire local VAD (`LLMUserAggregatorParams.vadAnalyzer`) on top of this
+ * service, disable OpenAI's server-side turn detection first
+ * (`turn_detection=False`); otherwise both sources broadcast duplicate
+ * user-turn frames. See
+ * `examples/realtime/realtime-openai-locally-driven-turns.py`.
+ */
+export declare class OpenAIRealtimeLLMService extends LLMService<RealtimeAdapter> {
+    static Settings: typeof OpenAIRealtimeLLMSettings;
+    protected _settings: OpenAIRealtimeLLMSettings;
+    static adapterClass: LLMAdapterClass | null;
+    static _REASONING_CAPABLE_MODEL_SUBSTRINGS: readonly string[];
+    apiKey: string;
+    baseUrl: string;
+    protected _audioInputPaused: boolean;
+    protected _videoInputPaused: boolean;
+    protected _videoFrameDetail: events.ImageDetail;
+    protected _lastSentTime: number;
+    protected _websocket: WebsocketClientConnection | null;
+    protected _receiveTask: Task<unknown> | null;
+    protected _context: LLMContext | null;
+    protected _llmNeedsConversationSetup: boolean;
+    protected _disconnecting: boolean;
+    protected _apiSessionReady: boolean;
+    protected _runLlmWhenApiSessionReady: boolean;
+    protected _currentAssistantResponse: events.ConversationItem | null;
+    protected _currentAudioResponse: CurrentAudioResponse | null;
+    protected _resampler: import("@pipedog/core/audio/resamplers/base_audio_resampler").BaseAudioResampler;
+    protected _userAudioPrerollBuffer: Uint8Array;
+    protected _userAudioPrerollSecsOverride: number | null;
+    protected _userAudioPrerollSecs: number;
+    protected _messagesAddedManually: Map<string, boolean>;
+    protected _pendingFunctionCalls: Map<string, events.ConversationItem>;
+    protected _completedToolCalls: Set<string>;
+    protected _openFunctionCalls: Set<string>;
+    protected _resultsAwaitingResponse: boolean;
+    protected _responseCreateOwed: boolean;
+    protected _responseActive: boolean;
+    protected _reasoningStripWarned: boolean;
+    protected _retrieveConversationItemFutures: Map<string, Future<events.ConversationItem>[]>;
+    /**
+     * Initialize the OpenAI Realtime LLM service.
+     *
+     * @param options.apiKey OpenAI API key for authentication.
+     * @param options.baseUrl WebSocket base URL for the realtime API.
+     * @param options.settings Runtime-updatable settings for this service.
+     * @param options.startAudioPaused Whether to start with audio input paused.
+     * @param options.startVideoPaused Whether to start with video input paused.
+     * @param options.videoFrameDetail Detail level for video processing.
+     * @param options.userAudioPrerollSecs Pre-roll replayed after an
+     *   interruption clears the input audio buffer, in manual turn-detection
+     *   mode. See `OpenAIRealtimeLLMServiceOptions.userAudioPrerollSecs`.
+     *   Other options are passed to the parent LLMService.
+     */
+    constructor(options: OpenAIRealtimeLLMServiceOptions);
+    /** Drop input transcription prompt settings unsupported by the selected model. */
+    static _omitUnsupportedInputAudioTranscriptionPrompt(sessionProperties: events.SessionProperties): boolean;
+    /**
+     * Check if the service can generate usage metrics.
+     *
+     * @returns True if metrics generation is supported.
+     */
+    canGenerateMetrics(): boolean;
+    /**
+     * Set whether audio input is paused.
+     *
+     * @param paused True to pause audio input, false to resume.
+     */
+    setAudioInputPaused(paused: boolean): void;
+    /**
+     * Set whether video input is paused.
+     *
+     * @param paused True to pause video input, false to resume.
+     */
+    setVideoInputPaused(paused: boolean): void;
+    /**
+     * Set the detail level for video processing.
+     *
+     * @param detail Detail level - "auto", "low", or "high".
+     */
+    setVideoFrameDetail(detail: string): void;
+    /** Check if a specific modality is enabled, "text" or "audio". */
+    protected _isModalityEnabled(modality: string): boolean;
+    /** Get the list of enabled modalities. */
+    protected _getEnabledModalities(): events.Modality[];
+    /**
+     * Retrieve a conversation item by ID from the server.
+     *
+     * @param itemId The ID of the conversation item to retrieve.
+     * @returns The retrieved conversation item.
+     */
+    retrieveConversationItem(itemId: string): Promise<events.ConversationItem>;
+    /**
+     * Set up the service and connect.
+     *
+     * @param setup Configuration object containing setup parameters.
+     */
+    setup(setup: FrameProcessorSetup): Promise<void>;
+    /** Release realtime LLM resources at teardown. */
+    cleanup(): Promise<void>;
+    /**
+     * Stop the service and close WebSocket connection.
+     *
+     * @param frame The end frame triggering service shutdown.
+     */
+    stop(frame: EndFrame): Promise<void>;
+    /**
+     * Cancel the service and close WebSocket connection.
+     *
+     * @param frame The cancel frame triggering service cancellation.
+     */
+    cancel(frame: CancelFrame): Promise<void>;
+    /**
+     * Whether server-side turn detection is explicitly disabled.
+     *
+     * `null` and `false` are different. `null` means we're using OpenAI's
+     * built-in turn detection defaults (so server VAD is active); `false` is
+     * the opt-out that puts the service in manual mode.
+     */
+    protected _isTurnDetectionDisabled(): boolean;
+    /**
+     * Intermediate results reach the model as conversation items.
+     *
+     * The API takes one output per function call, so an intermediate result
+     * goes in beside the call instead of through it (see `pushFrame`).
+     */
+    get acceptsIntermediateFunctionCallResults(): boolean;
+    /** Realtime service; recommends external turn strategies when server-side VAD is active. */
+    serviceMetadataFrame(): LLMServiceMetadataFrame;
+    protected _handleInterruption(): Promise<void>;
+    protected _handleUserStartedSpeaking(_frame: UserStartedSpeakingFrame): Promise<void>;
+    protected _handleUserStoppedSpeaking(_frame: UserStoppedSpeakingFrame): Promise<void>;
+    protected _handleBotStoppedSpeaking(): Promise<void>;
+    /** Calculate audio duration in milliseconds based on PCM audio parameters. */
+    protected _calculateAudioDurationMs(totalBytes: number, sampleRate?: number, bytesPerSample?: number): number;
+    /**
+     * Truncates the current audio response at the appropriate duration.
+     *
+     * Calculates the actual duration of the audio content and truncates at the
+     * shorter of either the wall clock time or the actual audio duration to
+     * prevent invalid truncation requests.
+     */
+    protected _truncateCurrentAudioResponse(): Promise<void>;
+    /** Return the tools configured on `sessionProperties`, if any. */
+    protected _serviceTools(): ToolsSchema | unknown[] | null;
+    /**
+     * Process incoming frames from the pipeline.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow in the pipeline.
+     */
+    processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
+    /**
+     * Push a frame, delivering function call outcomes to the API on the way.
+     *
+     * Function call results and cancellations are broadcast by the base
+     * service; the downstream copy is observed here and sent to the API as it
+     * is produced, intermediate results included. The frames travel on to the
+     * assistant aggregator, which records them in the context and decides when
+     * inference should run.
+     *
+     * @param frame The frame to push.
+     * @param direction The direction of frame pushing.
+     */
+    pushFrame(frame: Frame, direction?: FrameDirection): Promise<void>;
+    protected _handleContext(context: LLMContext, direction: FrameDirection): Promise<void>;
+    /**
+     * Put appended messages into the conversation as items, for the model to
+     * take in.
+     *
+     * Whether the model answers them is the aggregator's call: an append that
+     * asks to run comes back to this service as the context frame the assistant
+     * aggregator pushes upstream, which creates the response (see
+     * `_handleContext`). Before the session's conversation is set up, nothing
+     * is sent: the setup sends the context, which the aggregator records the
+     * messages in.
+     */
+    protected _handleMessagesAppend(frame: LLMMessagesAppendFrame): Promise<void>;
+    /**
+     * Send a client event to the OpenAI Realtime API.
+     *
+     * @param event The client event to send.
+     */
+    sendClientEvent(event: events.ClientEvent): Promise<void>;
+    protected _connect(): Promise<void>;
+    protected _disconnect(): Promise<void>;
+    protected _wsSend(realtimeMessage: Record<string, unknown>): Promise<void>;
+    /** Apply a settings delta, sending a session update when needed. */
+    protected _updateSettings(delta: LLMSettings): Promise<Record<string, unknown>>;
+    /**
+     * Drop `reasoning` from an outgoing session.update if the model can't use it.
+     *
+     * The server otherwise rejects the whole update and kills the session.
+     * Returns a copy when stripping; the user's stored config is preserved.
+     */
+    protected _stripUnsupportedReasoning(settings: events.SessionProperties): events.SessionProperties;
+    protected _sendSessionUpdate(): Promise<void>;
+    protected _receiveTaskHandler(): Promise<void>;
+    protected _handleEvtSessionCreated(_evt: events.SessionCreatedEvent): Promise<void>;
+    protected _handleEvtSessionUpdated(_evt: events.SessionUpdatedEvent): Promise<void>;
+    protected _handleEvtAudioDelta(evt: events.ResponseAudioDelta): Promise<void>;
+    /** Handle conversation.item.added event - item is added but may still be processing. */
+    protected _handleEvtConversationItemAdded(evt: events.ConversationItemAdded): Promise<void>;
+    /** Handle conversation.item.done event - item is fully completed. */
+    protected _handleEvtConversationItemDone(evt: events.ConversationItemDone): Promise<void>;
+    protected _handleEvtInputAudioTranscriptionDelta(evt: events.ConversationItemInputAudioTranscriptionDelta): Promise<void>;
+    /** Handle a transcription result with tracing. */
+    protected _handleUserTranscription(_transcript: string, _isFinal: boolean, _language?: Language | null): Promise<void>;
+    /**
+     * Handle completion of input audio transcription.
+     *
+     * @param evt The transcription completed event.
+     */
+    handleEvtInputAudioTranscriptionCompleted(evt: events.ConversationItemInputAudioTranscriptionCompleted): Promise<void>;
+    protected _handleConversationItemRetrieved(evt: events.ConversationItemRetrieved): Promise<void>;
+    /** Note the response, which takes in everything delivered before it. */
+    protected _handleEvtResponseCreated(_evt: events.ResponseCreated): Promise<void>;
+    protected _handleEvtResponseDone(evt: events.ResponseDone): Promise<void>;
+    protected _handleEvtTextDelta(evt: events.ResponseTextDelta): Promise<void>;
+    protected _handleEvtAudioTranscriptDelta(evt: events.ResponseAudioTranscriptDelta): Promise<void>;
+    protected _pushOutputTranscriptTextFrames(text: string): Promise<void>;
+    /**
+     * Handle completion of function call arguments.
+     *
+     * @param evt The response.function_call_arguments.done event.
+     */
+    protected _handleEvtFunctionCallArgumentsDone(evt: events.ResponseFunctionCallArgumentsDone): Promise<void>;
+    protected _handleEvtSpeechStarted(_evt: events.InputAudioBufferSpeechStarted): Promise<void>;
+    protected _handleEvtSpeechStopped(_evt: events.InputAudioBufferSpeechStopped): Promise<void>;
+    /**
+     * Maybe handle an error event related to retrieving a conversation item.
+     *
+     * If the given error event is an error retrieving a conversation item:
+     *
+     * - set an exception on the future that `retrieveConversationItem()` is
+     *   waiting on
+     * - return true
+     *
+     * Otherwise:
+     *
+     * - return false
+     */
+    protected _maybeHandleEvtRetrieveConversationItemError(evt: events.ErrorEvent): Promise<boolean>;
+    protected _handleEvtError(evt: events.ErrorEvent): Promise<void>;
+    /**
+     * Reset the conversation by disconnecting and reconnecting.
+     *
+     * This is the safest way to start a new conversation. Note that this will
+     * fail if called from the receive task.
+     */
+    resetConversation(): Promise<void>;
+    protected _createResponse(): Promise<void>;
+    /**
+     * Send any tool result the context holds that hasn't reached the API yet.
+     *
+     * Results this service produced are already sent, as they are produced (see
+     * `pushFrame`), so what this finds is a result written into the context
+     * some other way, plus, with `sendNewResults` false, the results a context
+     * brings with it at the start of a session.
+     */
+    protected _processCompletedFunctionCalls(sendNewResults: boolean): Promise<void>;
+    /**
+     * Auto-size the pre-roll from the upstream VAD's startSecs.
+     *
+     * `VADController` broadcasts the current `VADParams` at pipeline start (and
+     * on any runtime change), so the pre-roll tracks `startSecs` — the leading
+     * speech local turn detection consumes before confirming the turn — plus a
+     * margin.
+     *
+     * This assumes VAD drives turn starts (the default
+     * `VADUserTurnStartStrategy`). With a non-VAD turn-start strategy the
+     * onset-to-turn-start gap isn't governed by `startSecs`; in that case the
+     * developer should pin a pre-roll duration via `userAudioPrerollSecs`
+     * (which short-circuits this).
+     *
+     * This mechanism is harmless when server-side turn detection is enabled
+     * (the buffer is simply unused).
+     */
+    protected _handleSpeechControlParams(frame: SpeechControlParamsFrame): void;
+    protected _sendUserAudio(frame: InputAudioRawFrame): Promise<void>;
+    /**
+     * Re-append the buffered pre-roll audio to the input buffer.
+     *
+     * Called right after `InputAudioBufferClearEvent` so the speech onset
+     * (appended before local turn detection fired the interruption) survives
+     * the clear. The buffer is left intact — it's a rolling window, so a later
+     * interruption can restore the onset again.
+     */
+    protected _replayUserAudioPreroll(): Promise<void>;
+    /**
+     * Send user video frame to OpenAI Realtime API.
+     *
+     * @param frame The InputImageRawFrame to send.
+     */
+    protected _sendUserVideo(frame: InputImageRawFrame): Promise<void>;
+    protected _sendToolResult(toolCallId: string, result: string | null): Promise<void>;
+    /** Put text into the conversation for the model to take in, without answering it. */
+    protected _sendMessageItem(text: string, role?: string): Promise<void>;
+    /**
+     * Send one function call result to the API as it is produced.
+     *
+     * An intermediate result can't go through the tool-result channel — the
+     * API takes one output per call — so it goes in as a message carrying the
+     * same envelope a text LLM would read from the context. A final result
+     * answers the call, unless a reconnect has left the API without the id to
+     * answer, in which case it goes in as a message too.
+     */
+    protected _handleFunctionCallResult(frame: FunctionCallResultFrame): Promise<void>;
+    /** Settle a cancelled call, so the API isn't left waiting on it. */
+    protected _handleFunctionCallCancel(frame: FunctionCallCancelFrame): Promise<void>;
+}
+export {};
+//# sourceMappingURL=llm.d.ts.map