@pipedog/core 0.0.0 → 0.2.0

package/dist/processors/frameworks/rtvi/observer.d.ts

@@ -0,0 +1,340 @@
+import { AudioVolumeTracker } from "../../../audio/volume.ts";
+import { AggregatedTextFrame, AggregatedTextProgressFrame, type Frame, InterimTranscriptionFrame, LLMContextFrame, LLMTextFrame, MetricsFrame, TextType, TranscriptionFrame } from "../../../frames/frames.ts";
+import { BaseObserver, type BaseObserverOptions, type FramePushed } from "../../../observers/base_observer.ts";
+import { type FrameProcessor } from "../../frame_processor.ts";
+import { RTVIConfigureObserverFrame, RTVIServerResponseFrame } from "./frames.ts";
+import * as RTVI from "./models.ts";
+import { BotOutputTransformResult } from "./models.ts";
+import type { RTVIModelLike, RTVIProcessor } from "./processor.ts";
+/**
+ * Level of detail to include in function call RTVI events.
+ *
+ * Controls what information is exposed in function call events for security.
+ *
+ * - DISABLED: No events emitted for this function call.
+ * - NONE: Events only with tool_call_id, no function name or metadata (most secure).
+ * - NAME: Events with function name, no arguments or results.
+ * - ARGUMENTS: Events with function name and arguments, no results.
+ * - FULL: Events with function name, arguments, and results.
+ */
+export declare const RTVIFunctionCallReportLevel: {
+    readonly DISABLED: "disabled";
+    readonly NONE: "none";
+    readonly NAME: "name";
+    readonly ARGUMENTS: "arguments";
+    readonly FULL: "full";
+};
+export type RTVIFunctionCallReportLevel = (typeof RTVIFunctionCallReportLevel)[keyof typeof RTVIFunctionCallReportLevel];
+/**
+ * A bot output transform.
+ *
+ * When `accumulatedText` and `remainingText` are null the transform is being
+ * called for the full segment text (from `bot-output`). When they are
+ * provided the transform is being called for a progress event and must return
+ * a `BotOutputTransformResult` with `accumulatedText` and `remainingText` set.
+ */
+export type BotOutputTransform = (text: string, textType: TextType | string, accumulatedText: string | null, remainingText: string | null) => Promise<BotOutputTransformResult | string>;
+/** Options for `RTVIObserverParams`. Every field is optional. */
+export interface RTVIObserverParamsOptions {
+    botOutputEnabled?: boolean;
+    botLlmEnabled?: boolean;
+    botLlmMarkerEnabled?: boolean;
+    botTtsEnabled?: boolean;
+    botSpeakingEnabled?: boolean;
+    botAudioLevelEnabled?: boolean;
+    userLlmEnabled?: boolean;
+    userSpeakingEnabled?: boolean;
+    vadUserSpeakingEnabled?: boolean;
+    userMuteEnabled?: boolean;
+    userInputEnabled?: boolean;
+    userAudioLevelEnabled?: boolean;
+    metricsEnabled?: boolean;
+    systemLogsEnabled?: boolean;
+    sttRawTextEnabled?: boolean;
+    llmRawTextEnabled?: boolean;
+    ttsRawTextEnabled?: boolean;
+    ignoredSources?: FrameProcessor[];
+    skipTextTypes?: (TextType | string)[] | null;
+    botOutputTransforms?: [TextType | string, BotOutputTransform][] | null;
+    audioLevelPeriodSecs?: number;
+    functionCallReportLevel?: Record<string, RTVIFunctionCallReportLevel>;
+}
+/**
+ * Parameters for configuring RTVI Observer behavior.
+ *
+ * - `botOutputEnabled`: Indicates if bot output messages should be sent.
+ * - `botLlmEnabled`: Indicates if the bot's LLM messages should be sent. Of
+ *   these, the `bot-llm-text` message is deprecated in favor of `llm-raw-text`
+ *   (`llmRawTextEnabled`) and will be removed in 2.0.0. This flag stays, for
+ *   `bot-llm-started` and `bot-llm-stopped`.
+ * - `botLlmMarkerEnabled`: Indicates if the bot's LLM marker reports
+ *   (`LLMMarkerResponseFrame`, e.g. from `filterIncompleteUserTurns`) should
+ *   be sent. A report carries the marker the LLM emitted and the raw text of
+ *   the whole response, which is meant for evaluation, not for clients, so
+ *   this is off by default. Defaults to false.
+ * - `botTtsEnabled`: Indicates if the bot's TTS messages should be sent. Of
+ *   these, the `bot-tts-text` message is deprecated in favor of
+ *   `tts-raw-text` (`ttsRawTextEnabled`) and will be removed in 2.0.0. This
+ *   flag stays, for `bot-tts-started` and `bot-tts-stopped`.
+ * - `botSpeakingEnabled`: Indicates if the bot's started/stopped speaking and
+ *   interrupted messages should be sent.
+ * - `botAudioLevelEnabled`: Indicates if bot's audio level messages should be sent.
+ * - `userLlmEnabled`: Indicates if the user's LLM input messages should be sent.
+ * - `userSpeakingEnabled`: Indicates if the user's started/stopped speaking
+ *   messages should be sent.
+ * - `vadUserSpeakingEnabled`: Indicates if raw VAD user started/stopped
+ *   speaking messages should be sent. These reflect the VAD signal directly,
+ *   independent of turn finalization (unlike `userSpeakingEnabled`, which a
+ *   turn strategy may gate or defer). Off by default. Defaults to false.
+ * - `userMuteEnabled`: Indicates if user mute started/stopped messages should be sent.
+ * - `userInputEnabled`: Indicates if the user's input messages should be sent.
+ *   They report what the user said or typed: transcriptions, including
+ *   interim ones, backchannels, and the text the client sent.
+ * - `userAudioLevelEnabled`: Indicates if user's audio level messages should be sent.
+ * - `metricsEnabled`: Indicates if metrics messages should be sent.
+ * - `systemLogsEnabled`: Indicates if system logs should be sent.
+ * - `sttRawTextEnabled`: Indicates if the STT's raw text messages should be
+ *   sent. They report each transcription, interim ones included, as the STT
+ *   produced it. Only sent to clients of protocol 2.2.0 or later. Defaults to false.
+ * - `llmRawTextEnabled`: Indicates if the LLM's raw text messages should be
+ *   sent. They report each chunk of text the LLM streams. Only sent to
+ *   clients of protocol 2.2.0 or later. Defaults to false.
+ * - `ttsRawTextEnabled`: Indicates if the TTS's raw text messages should be
+ *   sent. They report the text the TTS speaks, as it is spoken, and leave out
+ *   the same text types as bot output messages. Only sent to clients of
+ *   protocol 2.2.0 or later. Defaults to false.
+ * - `ignoredSources`: List of frame processors whose frames should be
+ *   silently ignored by this observer. Useful for suppressing RTVI messages
+ *   from secondary pipeline branches (e.g. a silent evaluation LLM) that
+ *   should not be visible to clients. Sources can also be added and removed
+ *   dynamically via `addIgnoredSource()` and `removeIgnoredSource()`.
+ * - `skipTextTypes`: List of text types to skip sending as tts/output
+ *   messages. Text of type `backchannel` is only sent to clients of protocol
+ *   2.2.0 or later; add `"backchannel"` here to keep it from every client.
+ *   Note: if using this to avoid sending secure information, be sure to also
+ *   disable botLlmEnabled, and leave llmRawTextEnabled off, to avoid leaking
+ *   through LLM messages.
+ * - `botOutputTransforms`: A list of `[textType | "*", transform]` tuples to
+ *   transform text before sending it to the client. See `BotOutputTransform`.
+ * - `audioLevelPeriodSecs`: How often audio levels should be sent if enabled.
+ * - `functionCallReportLevel`: Controls what information is exposed in
+ *   function call events for security. A record mapping function names to
+ *   levels, where `"*"` sets the default level for unlisted functions.
+ *   Defaults to `{ "*": RTVIFunctionCallReportLevel.NONE }`.
+ *
+ * @example
+ * new RTVIObserverParams({
+ *   functionCallReportLevel: {
+ *     "*": RTVIFunctionCallReportLevel.NONE, // Default: events with no metadata
+ *     get_weather: RTVIFunctionCallReportLevel.FULL, // Expose everything
+ *   },
+ * });
+ */
+export declare class RTVIObserverParams {
+    botOutputEnabled: boolean;
+    botLlmEnabled: boolean;
+    botLlmMarkerEnabled: boolean;
+    botTtsEnabled: boolean;
+    botSpeakingEnabled: boolean;
+    botAudioLevelEnabled: boolean;
+    userLlmEnabled: boolean;
+    userSpeakingEnabled: boolean;
+    vadUserSpeakingEnabled: boolean;
+    userMuteEnabled: boolean;
+    userInputEnabled: boolean;
+    userAudioLevelEnabled: boolean;
+    metricsEnabled: boolean;
+    systemLogsEnabled: boolean;
+    sttRawTextEnabled: boolean;
+    llmRawTextEnabled: boolean;
+    ttsRawTextEnabled: boolean;
+    ignoredSources: FrameProcessor[];
+    skipTextTypes: (TextType | string)[] | null;
+    botOutputTransforms: [TextType | string, BotOutputTransform][] | null;
+    audioLevelPeriodSecs: number;
+    functionCallReportLevel: Record<string, RTVIFunctionCallReportLevel>;
+    constructor(options?: RTVIObserverParamsOptions);
+}
+/** Options for `RTVIObserver`. */
+export interface RTVIObserverOptions extends BaseObserverOptions {
+    /** Settings to enable/disable specific messages. */
+    params?: RTVIObserverParams | null;
+}
+type FunctionCallPhase = "started" | "in_progress" | "stopped";
+/**
+ * Pipeline frame observer for RTVI server message handling.
+ *
+ * This observer monitors pipeline frames and converts them into appropriate
+ * RTVI messages for client communication. It handles various frame types
+ * including speech events, transcriptions, LLM responses, and TTS events.
+ *
+ * Note: this observer only handles outgoing messages. Incoming RTVI client
+ * messages are handled by the RTVIProcessor.
+ */
+export declare class RTVIObserver extends BaseObserver {
+    #private;
+    protected _rtvi: RTVIProcessor | null;
+    protected _params: RTVIObserverParams;
+    protected _ignoredSources: Set<FrameProcessor>;
+    protected _botTranscription: string;
+    protected _lastUserAudioLevel: number;
+    protected _lastBotAudioLevel: number;
+    protected _userVolumeTracker: AudioVolumeTracker;
+    protected _botVolumeTracker: AudioVolumeTracker;
+    protected _botIsSpeaking: boolean;
+    protected _queuedAggregatedTextFrames: AggregatedTextFrame[];
+    protected _skippedSegmentIds: Set<number>;
+    protected _skippedSegmentsInterrupted: boolean;
+    protected _systemLoggerRemove: (() => void) | null;
+    protected _aggregationTransforms: [TextType | string, BotOutputTransform][];
+    /**
+     * Initialize the RTVI observer.
+     *
+     * @param rtvi The RTVI processor to push frames to.
+     * @param options.params Settings to enable/disable specific messages.
+     *   Other options are passed to the parent class.
+     */
+    constructor(rtvi?: RTVIProcessor | null, options?: RTVIObserverOptions);
+    /** Whether the connected client uses a deprecated 1.x protocol. */
+    protected get _isLegacyClient(): boolean;
+    /** Whether the connected client's protocol is `version` or later. */
+    protected _clientIsAtLeast(version: number[]): boolean;
+    /** Whether the connected client supports the backchannel text type (2.2.0+). */
+    protected get _clientSupportsBackchannels(): boolean;
+    /** Whether the connected client supports the raw text messages (2.2.0+). */
+    protected get _clientSupportsRawText(): boolean;
+    /**
+     * Register a text transformer for a specific text type.
+     *
+     * When `accumulatedText` and `remainingText` are null the transform is
+     * being called for the full segment text (`bot-output` message). When they
+     * are provided it is being called for a progress event; return a
+     * `BotOutputTransformResult` with `accumulatedText` and `remainingText`
+     * populated so the client can display word-level progress on the
+     * transformed text.
+     *
+     * @param transformFunction The transform callable.
+     * @param textType Text type to match, or `"*"` for all types.
+     */
+    addBotOutputTransformer(transformFunction: BotOutputTransform, textType?: TextType | string): void;
+    /**
+     * Remove a text transformer for a specific text type.
+     *
+     * @param transformFunction The function to remove.
+     * @param textType The type of text to remove the transformer for.
+     */
+    removeBotOutputTransformer(transformFunction: BotOutputTransform, textType?: TextType | string): void;
+    /**
+     * Ignore all frames pushed by the given processor.
+     *
+     * Any frame whose source matches `source` will be silently skipped,
+     * preventing RTVI messages from being emitted for activity in that
+     * processor. Useful for suppressing events from secondary pipeline
+     * branches (e.g. a silent evaluation LLM) that should not be visible
+     * to clients.
+     *
+     * @param source The frame processor to ignore.
+     */
+    addIgnoredSource(source: FrameProcessor): void;
+    /**
+     * Stop ignoring frames pushed by the given processor.
+     *
+     * Reverses a previous call to `addIgnoredSource()`. If `source` was not
+     * previously ignored this is a no-op.
+     *
+     * @param source The frame processor to stop ignoring.
+     */
+    removeIgnoredSource(source: FrameProcessor): void;
+    /**
+     * Get the report level for a specific function call.
+     *
+     * @param functionName The name of the function to get the report level for.
+     * @returns The report level for the function. Looks up the function name
+     *   first, then falls back to "*" key, then NONE.
+     */
+    protected _getFunctionCallReportLevel(functionName: string): RTVIFunctionCallReportLevel;
+    /**
+     * Apply a dynamic observer-config update (only the fields that are set).
+     *
+     * @param frame The config frame; null fields leave the current value unchanged.
+     */
+    protected _applyConfig(frame: RTVIConfigureObserverFrame): void;
+    /** Logger sink so we can send system logs to RTVI clients. */
+    protected _loggerSink(message: string): Promise<void>;
+    /** Cleanup RTVI observer resources. */
+    cleanup(): Promise<void>;
+    /**
+     * Send an RTVI message.
+     *
+     * By default, we push a transport frame. But this function can be
+     * overridden by subclass to send RTVI messages in different ways.
+     *
+     * @param model The message to send.
+     * @param excludeNone Whether to exclude null values from the model dump.
+     */
+    sendRtviMessage(model: RTVIModelLike, excludeNone?: boolean): Promise<void>;
+    /** Report a function-call frame, the pipeline's own or an external one. */
+    protected _reportFunctionCallFrame(frame: Frame): Promise<void>;
+    /**
+     * Send the function-call message for a phase, as the report level allows.
+     *
+     * The pipeline's own calls and calls reported from elsewhere (the
+     * `ExternalFunctionCall*Frame` family) go through here alike, so both obey
+     * the per-function report level.
+     */
+    protected _reportFunctionCall(phase: FunctionCallPhase, functionName: string, toolCallId: string, options?: {
+        arguments?: unknown;
+        result?: unknown;
+        cancelled?: boolean;
+    }): Promise<void>;
+    /**
+     * Process a frame being pushed through the pipeline.
+     *
+     * @param data Frame push event data containing source, frame, direction,
+     *   and timestamp.
+     */
+    onPushFrame(data: FramePushed): Promise<void>;
+    /** Handle user speaking interruption frames. */
+    protected _handleInterruptions(frame: Frame): Promise<void>;
+    /** Emit raw VAD user started/stopped speaking messages. */
+    protected _handleVadSpeaking(frame: Frame): Promise<void>;
+    /** Handle user mute/unmute frames. */
+    protected _handleUserMute(frame: Frame): Promise<void>;
+    /** Track whether the bot is speaking, and tell clients if they want to know. */
+    protected _handleBotSpeaking(frame: Frame): Promise<void>;
+    /** Tell clients the bot's output was cut off, and drop what it won't say. */
+    protected _handleBotInterrupted(): Promise<void>;
+    /** Handle aggregated LLM text output frames. */
+    protected _handleAggregatedLlmText(frame: AggregatedTextFrame): Promise<void>;
+    /** Handle progress frames. */
+    protected _handleAggregatedProgress(frame: AggregatedTextProgressFrame): Promise<void>;
+    /** Forget the skipped segments an interruption cut off. */
+    protected _forgetInterruptedSegments(): void;
+    /** Whether text of this type is kept out of bot output and TTS text messages. */
+    protected _isSkippedType(textType: TextType | string): boolean;
+    /** Whether the frame's text type, or its segment's, keeps it from bot output. */
+    protected _isSkipped(frame: AggregatedTextFrame): boolean;
+    /** Send aggregated LLM text messages. */
+    protected _sendAggregatedLlmText(frame: AggregatedTextFrame): Promise<void>;
+    /** Handle LLM text output frames. */
+    protected _handleLlmTextFrame(frame: LLMTextFrame): Promise<void>;
+    /** Handle user transcription frames. */
+    protected _handleUserTranscriptions(frame: TranscriptionFrame | InterimTranscriptionFrame): Promise<void>;
+    /** Send a user input message. */
+    protected _sendUserInput(text: string, inputType: RTVI.UserInputType, options: {
+        timestamp: string;
+        final: boolean;
+        userId?: string | null;
+        msgId?: string | null;
+    }): Promise<void>;
+    /** Process LLM context frames to extract user messages for the RTVI client. */
+    protected _handleContext(frame: LLMContextFrame): Promise<void>;
+    /** Handle metrics frames and convert to RTVI metrics messages. */
+    protected _handleMetrics(frame: MetricsFrame): Promise<void>;
+    /** Send a response to the client for a specific request. */
+    protected _sendServerResponse(frame: RTVIServerResponseFrame): Promise<void>;
+    /** Send a response to the client for a specific request. */
+    protected _sendErrorResponse(frame: RTVIServerResponseFrame): Promise<void>;
+}
+export {};
+//# sourceMappingURL=observer.d.ts.map