@pipedog/core 0.0.0 → 0.2.0

package/dist/transports/base_output.d.ts

@@ -0,0 +1,432 @@
+/**
+ * Base output transport implementation for Pipecat.
+ *
+ * This module provides the BaseOutputTransport class which handles audio and
+ * video output processing, including frame buffering, mixing, timing, and
+ * media streaming.
+ */
+import { Queue, type Task } from "@pipedog/asyncio";
+import type { BaseAudioFilter } from "../audio/filters/base_audio_filter.ts";
+import type { BaseAudioMixer } from "../audio/mixers/base_audio_mixer.ts";
+import type { BaseAudioResampler } from "../audio/resamplers/base_audio_resampler.ts";
+import { CancelFrame, EndFrame, type Frame, InterruptionFrame, MixerControlFrame, OutputAudioRawFrame, OutputDTMFFrame, OutputDTMFUrgentFrame, OutputImageRawFrame, OutputTransportMessageFrame, OutputTransportMessageUrgentFrame, SpeechOutputAudioRawFrame, SpriteFrame, StartFrame, TTSStoppedFrame } from "../frames/frames.ts";
+import { BotCapabilities } from "../pipeline/capabilities.ts";
+import { FrameDirection, FrameProcessor, type FrameProcessorOptions, type FrameProcessorSetup } from "../processors/frame_processor.ts";
+import { FramePriorityQueue, FrameQueue } from "../utils/frame_queue.ts";
+import type { TransportParams } from "./base_transport.ts";
+export declare const BOT_VAD_STOP_SECS = 0.35;
+/** Only used as a fallback. */
+export declare const BOT_VAD_STOP_FALLBACK_SECS = 3;
+/** An OutputAudioRawFrame subclass, so chunks keep the type of the audio they came from. */
+type OutputAudioFrameClass = new (options: {
+    audio: Uint8Array;
+    sampleRate: number;
+    numChannels: number;
+}) => OutputAudioRawFrame;
+/** A clock queue item: (presentation timestamp, insertion counter, frame). */
+type ClockItem = [number, number, Frame];
+/** A run of buffered audio from frames with the same `interruptible` flag. */
+type AudioRun = [Uint8Array, boolean];
+export interface MediaSenderOptions {
+    /** The destination identifier for this sender. */
+    destination: string | null;
+    /** The audio sample rate in Hz. */
+    sampleRate: number;
+    /** The size of audio chunks in bytes. */
+    audioChunkSize: number;
+    /** Transport configuration parameters. */
+    params: TransportParams;
+}
+/**
+ * Handles media streaming for a specific destination.
+ *
+ * Manages audio and video output processing including buffering, timing,
+ * mixing, and frame delivery for a single output destination.
+ *
+ * Upstream this is the nested class `BaseOutputTransport.MediaSender`, which
+ * is also reachable here as `BaseOutputTransport.MediaSender`.
+ */
+export declare class MediaSender {
+    protected _transport: BaseOutputTransport;
+    protected _destination: string | null;
+    protected _sampleRate: number;
+    protected _audioChunkSize: number;
+    protected _params: TransportParams;
+    protected _audioRuns: AudioRun[];
+    protected _audioBufferCls: OutputAudioFrameClass;
+    protected _resampler: BaseAudioResampler;
+    protected _mixer: BaseAudioMixer | null;
+    protected _filter: BaseAudioFilter | null;
+    protected _volume: number;
+    protected _volumeGain: number;
+    protected _videoImages: Iterator<OutputImageRawFrame> | null;
+    protected _botSpeaking: boolean;
+    protected _ttsAudioReceived: boolean;
+    protected _botSpeakingFrameTime: number;
+    protected _botSpeakingFramePeriod: number;
+    protected _botSpeechLastTime: number;
+    protected _audioTask: Task<unknown> | null;
+    protected _videoTask: Task<unknown> | null;
+    protected _clockTask: Task<unknown> | null;
+    protected _audioQueue: FrameQueue<Frame>;
+    protected _videoQueue: Queue<OutputImageRawFrame>;
+    protected _clockQueue: FramePriorityQueue<ClockItem>;
+    protected _clockFrame: Frame | null;
+    protected _clockQueueCounter: number;
+    protected _videoStartTime: number | null;
+    protected _videoFrameIndex: number;
+    protected _videoFrameDuration: number;
+    protected _videoFrameReset: number;
+    /**
+     * Initialize the media sender.
+     *
+     * @param transport The parent transport instance.
+     * @param options.destination The destination identifier for this sender.
+     * @param options.sampleRate The audio sample rate in Hz.
+     * @param options.audioChunkSize The size of audio chunks in bytes.
+     * @param options.params Transport configuration parameters.
+     */
+    constructor(transport: BaseOutputTransport, options: MediaSenderOptions);
+    /** The audio sample rate in Hz. */
+    get sampleRate(): number;
+    /** The size of audio chunks in bytes. */
+    get audioChunkSize(): number;
+    /**
+     * Start the media sender and initialize components.
+     *
+     * @param frame The start frame containing initialization parameters.
+     */
+    start(_frame: StartFrame): Promise<void>;
+    /**
+     * Stop the media sender and cleanup resources.
+     *
+     * @param frame The end frame signaling sender shutdown.
+     */
+    stop(frame: EndFrame): Promise<void>;
+    /**
+     * Cancel the media sender and stop all processing.
+     *
+     * @param frame The cancel frame signaling immediate cancellation.
+     */
+    cancel(_frame: CancelFrame): Promise<void>;
+    /** Release media sender resources at teardown. */
+    cleanup(): Promise<void>;
+    /**
+     * Handle interruption events by restarting tasks and clearing buffers.
+     *
+     * @param _ The start interruption frame (unused).
+     */
+    handleInterruptions(_: InterruptionFrame): Promise<void>;
+    /**
+     * Handle incoming audio frames by buffering and chunking.
+     *
+     * @param frame The output audio frame to handle.
+     */
+    handleAudioFrame(frame: OutputAudioRawFrame): Promise<void>;
+    /**
+     * Handle incoming image frames for video output.
+     *
+     * @param frame The output image or sprite frame to handle.
+     */
+    handleImageFrame(frame: OutputImageRawFrame | SpriteFrame): Promise<void>;
+    /**
+     * Handle frames with presentation timestamps.
+     *
+     * @param frame The frame with timing information to handle.
+     */
+    handleTimedFrame(frame: Frame): Promise<void>;
+    /**
+     * Handle frames that need synchronized processing.
+     *
+     * @param frame The frame to handle synchronously.
+     */
+    handleSyncFrame(frame: Frame): Promise<void>;
+    /**
+     * Queue a TTSStoppedFrame, flushing any trailing buffered audio first.
+     *
+     * `handleAudioFrame` only queues complete `audioChunkSize` chunks, so up to
+     * one chunk's worth of trailing audio can still be sitting in `_audioRuns`.
+     * Queue it now (padded to a full chunk with silence) so it plays before the
+     * stop frame is handled, instead of being discarded when the buffer is
+     * cleared in `_botStoppedSpeaking`.
+     *
+     * @param frame The TTS stopped frame to queue.
+     */
+    handleTtsStopped(frame: TTSStoppedFrame): Promise<void>;
+    /**
+     * Handle audio mixer control frames.
+     *
+     * @param frame The mixer control frame to handle.
+     */
+    handleMixerControlFrame(frame: MixerControlFrame): Promise<void>;
+    /** Create the audio processing task. */
+    protected _createAudioTask(): void;
+    /** Cancel and cleanup the audio processing task. */
+    protected _cancelAudioTask(): Promise<void>;
+    /** Handle bot started speaking event. */
+    protected _botStartedSpeaking(): Promise<void>;
+    /**
+     * Queue every bit of audio still held back at the end of a speech run.
+     *
+     * That means both what the resampler is holding in its filter and what is
+     * left in `_audioRuns`, padded with silence to a full chunk. The audio goes
+     * out as frames of the same type (e.g. TTSAudioRawFrame) as the audio it
+     * was buffered from, on `_audioQueue`, so it takes the normal playback path
+     * (write, error handling, bot speaking tracking) like any other chunk, in
+     * order relative to whatever is queued after it (e.g. a TTSStoppedFrame).
+     */
+    protected _enqueueFlushedAudioBuffer(): Promise<void>;
+    protected get _bufferedAudioBytes(): number;
+    /** Add output-rate audio to the buffer, noting whether it came from uninterruptible frames. */
+    protected _bufferAudio(audio: Uint8Array, options: {
+        uninterruptible: boolean;
+    }): void;
+    /** Cut up to a chunk from the front of the buffer, and whether any of it is uninterruptible. */
+    protected _takeAudioChunk(): [Uint8Array, boolean];
+    protected _clearAudioBuffer(): void;
+    /** Queue one full chunk of output-rate audio for playback. */
+    protected _enqueueAudioChunk(audio: Uint8Array, uninterruptible: boolean): Promise<void>;
+    /** Handle bot stopped speaking event. */
+    protected _botStoppedSpeaking(): Promise<void>;
+    /** Handle bot speaking event. */
+    protected _botCurrentlySpeaking(): Promise<void>;
+    protected _maybeBotCurrentlySpeaking(frame: SpeechOutputAudioRawFrame): Promise<void>;
+    protected _handleBotSpeech(frame: Frame): Promise<void>;
+    /**
+     * Handle various frame types with appropriate processing.
+     *
+     * @param frame The frame to handle.
+     */
+    protected _handleFrame(frame: Frame): Promise<void>;
+    /**
+     * Run the output filter over a frame taken from the audio queue.
+     *
+     * @param frame The queued frame. Only audio frames are filtered.
+     */
+    protected _filterAudio(frame: Frame): Promise<void>;
+    /** Scale an audio frame taken from the audio queue by the volume and its gain. */
+    protected _applyVolume(frame: Frame): void;
+    /**
+     * Generate the next frame for audio processing.
+     *
+     * @returns An async generator yielding frames for processing.
+     */
+    protected _nextFrame(): AsyncGenerator<Frame, void, undefined>;
+    protected _sendSilence(secs: number): Promise<void>;
+    /** Main audio processing task handler. */
+    protected _audioTaskHandler(): Promise<void>;
+    /**
+     * Write a frame to the transport, giving up if the write never returns.
+     *
+     * A client that stops reading is the common way to get there: the
+     * connection stays up and nothing fails, so the transport has neither an
+     * error to report nor state to check. That leaves how long the write takes
+     * as the only signal available.
+     *
+     * @returns Whether the transport took the frame.
+     */
+    protected _internalWriteAudioFrame(frame: OutputAudioRawFrame): Promise<boolean>;
+    /** Create the video processing task if video output is enabled. */
+    protected _createVideoTask(): void;
+    /** Cancel and cleanup the video processing task. */
+    protected _cancelVideoTask(): Promise<void>;
+    /**
+     * Set a single video image for cycling output.
+     *
+     * @param image The image frame to cycle for video output.
+     */
+    protected _setVideoImage(image: OutputImageRawFrame): Promise<void>;
+    /**
+     * Set multiple video images for cycling output.
+     *
+     * @param images The list of image frames to cycle for video output.
+     */
+    protected _setVideoImages(images: OutputImageRawFrame[]): Promise<void>;
+    /** Main video processing task handler. */
+    protected _videoTaskHandler(): Promise<void>;
+    /** Handle live video streaming with frame timing. */
+    protected _videoIsLiveHandler(): Promise<void>;
+    /**
+     * Draw/render an image frame with resizing if needed.
+     *
+     * @param frame The image frame to draw.
+     */
+    protected _drawImage(frame: OutputImageRawFrame): Promise<void>;
+    /**
+     * Resize an image frame to the transport's output size.
+     *
+     * Upstream decodes the raw bytes with PIL (`Image.frombytes`), resizes them
+     * and re-encodes them in the image's mode. There is no edge-compatible PIL
+     * equivalent yet; override this (or provide a resizer) to support images
+     * whose size differs from `videoOutWidth` x `videoOutHeight`.
+     *
+     * @param frame The image frame to resize (its `format` is set).
+     * @param desiredSize The output size as [width, height].
+     * @returns A new OutputImageRawFrame with the resized image.
+     */
+    protected _resizeImage(_frame: OutputImageRawFrame, _desiredSize: [number, number]): OutputImageRawFrame;
+    /** Create the clock/timing processing task. */
+    protected _createClockTask(): void;
+    /** Cancel and cleanup the clock processing task. */
+    protected _cancelClockTask(): Promise<void>;
+    /** Main clock/timing task handler for timed frame delivery. */
+    protected _clockTaskHandler(): Promise<void>;
+    toString(): string;
+}
+/**
+ * Base class for output transport implementations.
+ *
+ * Handles audio and video output processing including frame buffering, audio
+ * mixing, timing coordination, and media streaming. Supports multiple output
+ * destinations and provides interruption handling for real-time
+ * communication.
+ *
+ * The deprecated `TransportParams.video_out_bitrate` (and its deprecation
+ * warning) is not ported.
+ */
+export declare class BaseOutputTransport extends FrameProcessor {
+    /** Upstream's nested class `BaseOutputTransport.MediaSender`. */
+    static readonly MediaSender: typeof MediaSender;
+    protected _params: TransportParams;
+    protected _sampleRate: number;
+    protected _audioChunkSize: number;
+    protected _mediaSenders: Map<string | null, MediaSender>;
+    /**
+     * Initialize the base output transport.
+     *
+     * @param params Transport configuration parameters.
+     * @param options Additional options passed to the parent class.
+     */
+    constructor(params: TransportParams, options?: FrameProcessorOptions);
+    /** The current audio sample rate in Hz. */
+    get sampleRate(): number;
+    /** The size of audio chunks for output processing, in bytes. */
+    get audioChunkSize(): number;
+    /** The media this transport sends to the user: the `audio_out` and `video_out` capabilities. */
+    get capabilities(): BotCapabilities;
+    /**
+     * Set up the transport.
+     *
+     * @param setup Configuration object containing setup parameters.
+     */
+    setup(setup: FrameProcessorSetup): Promise<void>;
+    /** Release output transport resources at teardown. */
+    cleanup(): Promise<void>;
+    /**
+     * Start the output transport.
+     *
+     * Base hook for subclasses, which reach it through `super`. The transport's
+     * own audio configuration is resolved in `setup`.
+     *
+     * @param frame The start frame containing initialization parameters.
+     */
+    start(_frame: StartFrame): Promise<void>;
+    /**
+     * Stop the output transport and cleanup resources.
+     *
+     * @param frame The end frame signaling transport shutdown.
+     */
+    stop(frame: EndFrame): Promise<void>;
+    /**
+     * Cancel the output transport and stop all processing.
+     *
+     * @param frame The cancel frame signaling immediate cancellation.
+     */
+    cancel(frame: CancelFrame): Promise<void>;
+    /**
+     * Called when the transport is ready to stream.
+     *
+     * @param frame The start frame containing initialization parameters.
+     */
+    setTransportReady(frame: StartFrame): Promise<void>;
+    /**
+     * Send a transport message.
+     *
+     * @param frame The transport message frame to send.
+     */
+    sendMessage(_frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame): Promise<void>;
+    /**
+     * Register a video output destination.
+     *
+     * @param destination The destination identifier to register.
+     */
+    registerVideoDestination(_destination: string): Promise<void>;
+    /**
+     * Register an audio output destination.
+     *
+     * @param destination The destination identifier to register.
+     */
+    registerAudioDestination(_destination: string): Promise<void>;
+    /**
+     * Write a video frame to the transport.
+     *
+     * @param frame The output video frame to write.
+     * @returns True if the video frame was written successfully, false otherwise.
+     */
+    writeVideoFrame(_frame: OutputImageRawFrame): Promise<boolean>;
+    /**
+     * Write an audio frame to the transport.
+     *
+     * @param frame The output audio frame to write.
+     * @returns Whether the transport took the frame. A serializer that emits no
+     *   payload has still taken it; see `FrameSerializer.serialize`.
+     */
+    writeAudioFrame(_frame: OutputAudioRawFrame): Promise<boolean>;
+    /**
+     * Write a DTMF tone using the transport's preferred method.
+     *
+     * @param frame The DTMF frame to write.
+     */
+    writeDtmf(frame: OutputDTMFFrame | OutputDTMFUrgentFrame): Promise<void>;
+    /**
+     * Handle a queued frame after preceding audio has been sent.
+     *
+     * Override in transport subclasses to handle custom frame types that flow
+     * through the audio queue. Called by the media sender after the frame has
+     * waited for any preceding audio to finish.
+     *
+     * @param frame The frame to handle.
+     */
+    writeTransportFrame(_frame: Frame): Promise<void>;
+    /**
+     * Override in transport implementations that support native DTMF.
+     *
+     * @returns True if the transport supports native DTMF, false otherwise.
+     */
+    protected _supportsNativeDtmf(): boolean;
+    /**
+     * Override in transport implementations for native DTMF.
+     *
+     * @param frame The DTMF frame to write.
+     */
+    protected _writeDtmfNative(_frame: OutputDTMFFrame | OutputDTMFUrgentFrame): Promise<void>;
+    /**
+     * Generate and send audio tones for DTMF.
+     *
+     * @param frame The DTMF frame to write.
+     */
+    protected _writeDtmfAudio(frame: OutputDTMFFrame | OutputDTMFUrgentFrame): Promise<void>;
+    /**
+     * Send an audio frame downstream.
+     *
+     * @param frame The audio frame to send.
+     */
+    sendAudio(frame: OutputAudioRawFrame): Promise<void>;
+    /**
+     * Send an image frame downstream.
+     *
+     * @param frame The image frame to send.
+     */
+    sendImage(frame: OutputImageRawFrame | SpriteFrame): Promise<void>;
+    /**
+     * Process incoming frames and handle transport-specific logic.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow in the pipeline.
+     */
+    processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
+    /** Handle frames by routing them to appropriate media senders. */
+    protected _handleFrame(frame: Frame): Promise<void>;
+}
+export {};
+//# sourceMappingURL=base_output.d.ts.map