@pipedog/core 0.0.0 → 0.1.0

package/dist/processors/aggregators/llm_response_universal.d.ts

@@ -0,0 +1,966 @@
+/**
+ * LLM response aggregators for handling conversation context and message aggregation.
+ *
+ * This module provides aggregators that process and accumulate LLM responses,
+ * user inputs, and conversation context. These aggregators handle the flow
+ * between speech-to-text, LLM processing, and text-to-speech components in
+ * conversational AI pipelines.
+ *
+ * Porting notes: the deprecated `LLMUserAggregatorParams.filterIncompleteUserTurns`
+ * / `userTurnCompletionConfig` and `LLMAssistantAggregatorParams.enableContextSummarization`
+ * / `contextSummarizationConfig` fields are not ported.
+ */
+import { type Task } from "@pipedog/asyncio";
+import type { ToolsSchema } from "../../adapters/schemas/tools_schema.ts";
+import type { VADAnalyzer } from "../../audio/vad/vad_analyzer.ts";
+import { VADController } from "../../audio/vad/vad_controller.ts";
+import { AssistantImageRawFrame, CancelFrame, EndFrame, type Frame, FunctionCallCancelFrame, FunctionCallInProgressFrame, FunctionCallResultFrame, FunctionCallsStartedFrame, InterruptionFrame, LLMContextFrame, type LLMContextSummaryRequestFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMarkerFrame, LLMMessagesAppendFrame, LLMMessagesTransformFrame, LLMMessagesUpdateFrame, LLMRunFrame, LLMServiceMetadataFrame, LLMThoughtEndFrame, LLMThoughtStartFrame, LLMThoughtTextFrame, ServiceMetadataFrame, STTMetadataFrame, StartFrame, TextFrame, TranscriptionFrame, TTSStartedFrame, UserFileRawFrame, UserImageRawFrame } from "../../frames/frames.ts";
+import { EmptyUserTurnConfig } from "../../turns/empty_user_turn.ts";
+import type { UserTurnSpeculation } from "../../turns/types.ts";
+import { UserIdleController } from "../../turns/user_idle_controller.ts";
+import type { BaseUserMuteStrategy } from "../../turns/user_mute/base_user_mute_strategy.ts";
+import type { BaseUserTurnStartStrategy, UserTurnStartedParams } from "../../turns/user_start/base_user_turn_start_strategy.ts";
+import type { BaseUserTurnStopStrategy, UserTurnStoppedParams } from "../../turns/user_stop/base_user_turn_stop_strategy.ts";
+import { UserTurnController } from "../../turns/user_turn_controller.ts";
+import { UserTurnStrategies } from "../../turns/user_turn_strategies.ts";
+import type { LLMAutoContextSummarizationConfig } from "../../utils/context/llm_context_summarization.ts";
+import { TextPartForConcatenation } from "../../utils/string.ts";
+import { FrameDirection, FrameProcessor, type FrameProcessorOptions, type FrameProcessorSetup } from "../frame_processor.ts";
+import { LLMContext, type LLMContextMessage, type NotGiven } from "./llm_context.ts";
+import { LLMContextSummarizer, type SummaryAppliedEvent } from "./llm_context_summarizer.ts";
+export interface LLMUserAggregatorParamsOptions {
+    /**
+     * When true, on each `LLMSetToolsFrame` the aggregator computes the diff
+     * against the currently advertised tools and appends a developer-role
+     * message to the context describing additions/removals. Helps the LLM stay
+     * coherent across mid-conversation tool changes, mitigating several
+     * flavors of tool-call-related hallucination: calling tools that have been
+     * removed, avoiding tools that have been re-added, and hallucinating output
+     * (made-up answers or tool-call-shaped non-tool-calls) when tools are
+     * unavailable. Only standard tools are diffed; custom (LLM-specific) tools
+     * are ignored. When using `LLMContextAggregatorPair`, prefer setting this
+     * via its `addToolChangeMessages` argument instead. Defaults to false.
+     */
+    addToolChangeMessages?: boolean;
+    /**
+     * Timeout in seconds to force speech stop when no audio frames are
+     * received while in SPEAKING state (e.g. user mutes mic mid-speech). Set to
+     * 0 to disable. Defaults to 1.0.
+     */
+    audioIdleTimeout?: number;
+    /** User turn start and stop strategies. */
+    userTurnStrategies?: UserTurnStrategies | null;
+    /** List of user mute strategies. */
+    userMuteStrategies?: BaseUserMuteStrategy[];
+    /** Time in seconds to wait before considering the user's turn finished. */
+    userTurnStopTimeout?: number;
+    /**
+     * Timeout in seconds for detecting user idle state. The aggregator will
+     * emit an `on_user_turn_idle` event when the user has been idle (not
+     * speaking) for this duration. Set to 0 to disable idle detection.
+     */
+    userIdleTimeout?: number;
+    /** Voice Activity Detection analyzer instance. */
+    vadAnalyzer?: VADAnalyzer | null;
+    /**
+     * How to respond to a user turn that ends with no transcript. By default,
+     * the bot answers such a turn when it interrupted the bot, and leaves it
+     * unanswered otherwise. `null` leaves every such turn unanswered. Ignored
+     * with a realtime LLM service, which hears the user's audio directly.
+     */
+    emptyUserTurn?: EmptyUserTurnConfig | null;
+}
+/**
+ * Parameters for configuring LLM user aggregation behavior.
+ *
+ * See `LLMUserAggregatorParamsOptions` for the fields.
+ */
+export declare class LLMUserAggregatorParams {
+    addToolChangeMessages: boolean;
+    audioIdleTimeout: number;
+    userTurnStrategies: UserTurnStrategies | null;
+    userMuteStrategies: BaseUserMuteStrategy[];
+    userTurnStopTimeout: number;
+    userIdleTimeout: number;
+    vadAnalyzer: VADAnalyzer | null;
+    emptyUserTurn: EmptyUserTurnConfig | null;
+    constructor(options?: LLMUserAggregatorParamsOptions);
+}
+export interface LLMAssistantAggregatorParamsOptions {
+    /**
+     * Enable automatic context summarization when token or message-count
+     * limits are reached (disabled by default). When enabled, older
+     * conversation messages are automatically compressed into summaries to
+     * manage context size.
+     */
+    enableAutoContextSummarization?: boolean;
+    /**
+     * Configuration for automatic context summarization. Controls trigger
+     * thresholds, message preservation, and summarization prompts. If null,
+     * uses default `LLMAutoContextSummarizationConfig` values.
+     */
+    autoContextSummarizationConfig?: LLMAutoContextSummarizationConfig | null;
+    /**
+     * When true, on each `LLMSetToolsFrame` the aggregator computes the diff
+     * against the currently advertised tools and appends a developer-role
+     * message to the context describing additions/removals. See
+     * `LLMUserAggregatorParamsOptions.addToolChangeMessages`. Defaults to false.
+     */
+    addToolChangeMessages?: boolean;
+}
+/**
+ * Parameters for configuring LLM assistant aggregation behavior.
+ *
+ * See `LLMAssistantAggregatorParamsOptions` for the fields.
+ */
+export declare class LLMAssistantAggregatorParams {
+    enableAutoContextSummarization: boolean;
+    autoContextSummarizationConfig: LLMAutoContextSummarizationConfig | null;
+    addToolChangeMessages: boolean;
+    constructor(options?: LLMAssistantAggregatorParamsOptions);
+}
+export interface UserTurnStoppedMessageOptions {
+    content: string | null;
+    timestamp: string;
+    userId?: string | null;
+}
+/**
+ * A user turn stopped message containing a user transcript update.
+ *
+ * A message in a conversation transcript containing the user content. This
+ * is the aggregated transcript that is then used in the context.
+ *
+ * - `content`: The message content/text. `null` in realtime mode
+ *   (`realtimeServiceMode: true`) when fired from a user-turn-stop frame: in
+ *   realtime mode the user message isn't finalized until the assistant
+ *   response start acts as the effective end-of-turn signal. Subscribers that
+ *   need the finalized text should listen to `on_user_turn_message_added`
+ *   instead.
+ * - `timestamp`: When the user turn started.
+ * - `userId`: Optional identifier for the user.
+ */
+export declare class UserTurnStoppedMessage {
+    content: string | null;
+    timestamp: string;
+    userId: string | null;
+    constructor(options: UserTurnStoppedMessageOptions);
+}
+export interface UserTurnMessageAddedMessageOptions {
+    content: string;
+    timestamp: string;
+    userId?: string | null;
+}
+/**
+ * A message accompanying `on_user_turn_message_added`.
+ *
+ * Fired when a user message is written to the LLM context. In cascade mode
+ * (`realtimeServiceMode: false`) this coincides with `on_user_turn_stopped`.
+ * In realtime mode (`realtimeServiceMode: true`) the write is triggered by
+ * the assistant response start (or session end), so this event fires
+ * decoupled from user-turn-end frames. `content` is always populated.
+ *
+ * - `content`: The aggregated user transcript.
+ * - `timestamp`: When the user turn started.
+ * - `userId`: Optional identifier for the user.
+ */
+export declare class UserTurnMessageAddedMessage {
+    content: string;
+    timestamp: string;
+    userId: string | null;
+    constructor(options: UserTurnMessageAddedMessageOptions);
+}
+export interface UserBackchannelMessageOptions {
+    content: string;
+    timestamp: string;
+    userId?: string | null;
+}
+/**
+ * A message accompanying `on_user_backchannel`.
+ *
+ * A short acknowledgment the user said while the bot talks, such as "mhm",
+ * which stays out of the LLM context.
+ *
+ * - `content`: What the user said.
+ * - `timestamp`: When the user said it.
+ * - `userId`: Optional identifier for the user.
+ */
+export declare class UserBackchannelMessage {
+    content: string;
+    timestamp: string;
+    userId: string | null;
+    constructor(options: UserBackchannelMessageOptions);
+}
+export interface AssistantTurnStoppedMessageOptions {
+    content: string;
+    interrupted: boolean;
+    timestamp: string;
+}
+/**
+ * An assistant turn stopped message containing an assistant transcript update.
+ *
+ * A message in a conversation transcript containing the assistant content.
+ * This is the aggregated transcript that is then used in the context.
+ *
+ * - `content`: The message content/text. May be empty if the LLM returned
+ *   zero tokens (e.g. turn was interrupted before any tokens were received or
+ *   pushed).
+ * - `interrupted`: Whether the assistant turn was interrupted.
+ * - `timestamp`: When the assistant turn started.
+ */
+export declare class AssistantTurnStoppedMessage {
+    content: string;
+    interrupted: boolean;
+    timestamp: string;
+    constructor(options: AssistantTurnStoppedMessageOptions);
+}
+export interface AssistantBackchannelMessageOptions {
+    content: string;
+    timestamp: string;
+}
+/**
+ * A message accompanying `on_assistant_backchannel`.
+ *
+ * A short acknowledgment the assistant said while the user talks, such as
+ * "Mm-hmm.".
+ *
+ * - `content`: What the assistant said.
+ * - `timestamp`: When the assistant started saying it.
+ */
+export declare class AssistantBackchannelMessage {
+    content: string;
+    timestamp: string;
+    constructor(options: AssistantBackchannelMessageOptions);
+}
+export interface AssistantThoughtMessageOptions {
+    content: string;
+    timestamp: string;
+}
+/**
+ * An assistant thought message containing an assistant thought update.
+ *
+ * A message in a conversation transcript containing the assistant thought
+ * content.
+ *
+ * - `content`: The message content/text.
+ * - `timestamp`: When the thought started.
+ */
+export declare class AssistantThoughtMessage {
+    content: string;
+    timestamp: string;
+    constructor(options: AssistantThoughtMessageOptions);
+}
+export interface LLMContextAggregatorOptions extends FrameProcessorOptions {
+    /** The LLM context to use for conversation storage. */
+    context: LLMContext;
+    /** The role this aggregator represents (e.g. "user", "assistant"). */
+    role: string;
+    /**
+     * See the field of the same name on the aggregator-specific params
+     * classes. Subclasses propagate this from their `params`.
+     */
+    addToolChangeMessages?: boolean;
+}
+/**
+ * Base LLM aggregator that uses an LLMContext for conversation storage.
+ *
+ * This aggregator maintains conversation state using an LLMContext and pushes
+ * LLMContextFrame objects as aggregation frames. It provides common
+ * functionality for context-based conversation management.
+ */
+export declare abstract class LLMContextAggregator extends FrameProcessor {
+    static readonly TOOL_ACTIVATION_MESSAGE_TEMPLATE: string;
+    static readonly TOOL_DEACTIVATION_MESSAGE_TEMPLATE: string;
+    protected _context: LLMContext;
+    protected _role: string;
+    protected _addToolChangeMessages: boolean;
+    protected _aggregation: TextPartForConcatenation[];
+    /** Initialize the context response aggregator. */
+    constructor(options: LLMContextAggregatorOptions);
+    /**
+     * Append a developer message describing tool add/remove deltas.
+     *
+     * No-op unless `addToolChangeMessages` was enabled on the aggregator, and
+     * no-op when the diff against the currently advertised tools is empty.
+     * Custom (LLM-specific) tools are ignored — only standard tools are diffed.
+     *
+     * Both aggregators call this on every `LLMSetToolsFrame` they handle.
+     * Whichever aggregator handles the frame first computes a real diff against
+     * the shared context and adds the announcement; by the time the other
+     * aggregator sees it (if at all), the context already reflects the new
+     * tools, so its diff is empty and no duplicate message is added. This is
+     * order-independent: it works whether the frame flows downstream (user
+     * aggregator first) or upstream (assistant aggregator first, and consumed
+     * without being forwarded).
+     */
+    protected _maybeAddToolChangeMessages(newTools: ToolsSchema | NotGiven): void;
+    /** Messages from the LLM context. */
+    get messages(): LLMContextMessage[];
+    /** The role for this aggregator. */
+    get role(): string;
+    /** The LLMContext instance used by this aggregator. */
+    get context(): LLMContext;
+    /**
+     * Create a context frame with the current context.
+     *
+     * @returns LLMContextFrame containing the current context.
+     */
+    protected _getContextFrame(): LLMContextFrame;
+    /**
+     * Push a context frame in the specified direction.
+     *
+     * @param direction The direction to push the frame (upstream or downstream).
+     */
+    pushContextFrame(direction?: FrameDirection): Promise<void>;
+    /**
+     * Add messages to the context.
+     *
+     * @param messages Messages to add to the conversation context.
+     */
+    addMessages(messages: LLMContextMessage[]): void;
+    /**
+     * Set the context messages.
+     *
+     * @param messages Messages to replace the current context messages.
+     */
+    setMessages(messages: LLMContextMessage[]): void;
+    /**
+     * Transform the context messages using a provided function.
+     *
+     * @param transform A function that takes the current list of messages and
+     *   returns a modified list of messages to set in the context.
+     */
+    transformMessages(transform: (messages: LLMContextMessage[]) => LLMContextMessage[]): void;
+    /**
+     * Set tools in the context.
+     *
+     * @param tools List of tool definitions to set in the context.
+     */
+    setTools(tools: ToolsSchema | NotGiven): void;
+    /**
+     * Set tool choice in the context.
+     *
+     * @param toolChoice Tool choice configuration for the context.
+     */
+    setToolChoice(toolChoice: "none" | "auto" | "required" | Record<string, unknown>): void;
+    /** Reset the aggregation state. */
+    reset(): Promise<void>;
+    /**
+     * Push the current aggregation downstream.
+     *
+     * @returns The pushed aggregation.
+     */
+    abstract pushAggregation(): Promise<string>;
+    /**
+     * Get the current aggregation as a string.
+     *
+     * @returns The concatenated aggregation string.
+     */
+    aggregationString(): string;
+}
+export interface LLMUserAggregatorOptions extends FrameProcessorOptions {
+    /** Configuration parameters for aggregation behavior. */
+    params?: LLMUserAggregatorParams | null;
+    /**
+     * Pair-internal. Realtime-mode flag propagated from
+     * `LLMContextAggregatorPair` (`null` = auto-configure from service
+     * metadata). Not intended for direct use — construct the aggregators via
+     * the pair.
+     */
+    _realtimeServiceMode?: boolean | null;
+}
+/**
+ * User LLM aggregator that aggregates user input during active user turns.
+ *
+ * This aggregator uses a turn controller and operates within turn boundaries
+ * defined by the controller's configured user turn strategies. User turn
+ * start strategies indicate when a user turn begins, while user turn stop
+ * strategies signal when the user turn has ended.
+ *
+ * The aggregator collects and aggregates speech-to-text transcriptions that
+ * occur while a user turn is active and pushes the final aggregation when the
+ * user turn is finished.
+ *
+ * Event handlers available:
+ *
+ * - on_user_turn_started: Called when the user turn starts
+ * - on_user_turn_stopped: Called when the user turn ends
+ * - on_user_turn_stop_timeout: Called when no user turn stop strategy triggers
+ * - on_user_turn_idle: Called when the user has been idle for the configured timeout
+ * - on_user_turn_message_added: Called when a user message is written to
+ *   context. In realtime mode (`realtimeServiceMode: true`) the write is
+ *   triggered by the assistant response start rather than the user-turn-end
+ *   frame, so this fires decoupled from `on_user_turn_stopped` — subscribe
+ *   here for the finalized user text.
+ * - on_user_mute_started: Called when the user becomes muted
+ * - on_user_mute_stopped: Called when the user becomes unmuted
+ * - on_user_backchannel: Called when the user acknowledges the bot without
+ *   interrupting it, from a `UserBackchannelFrame`
+ *
+ * @example
+ * aggregator.eventHandler("on_user_turn_started", async (aggregator, strategy: BaseUserTurnStartStrategy) => {});
+ * aggregator.eventHandler(
+ *   "on_user_turn_stopped",
+ *   async (aggregator, strategy: BaseUserTurnStopStrategy, message: UserTurnStoppedMessage) => {},
+ * );
+ * aggregator.eventHandler("on_user_turn_stop_timeout", async (aggregator) => {});
+ * aggregator.eventHandler("on_user_turn_idle", async (aggregator) => {});
+ * // In realtime mode the user message is written when the assistant response
+ * // starts, not at user-turn-end — subscribe here for the finalized text.
+ * aggregator.eventHandler(
+ *   "on_user_turn_message_added",
+ *   async (aggregator, message: UserTurnMessageAddedMessage) => {},
+ * );
+ * aggregator.eventHandler("on_user_mute_started", async (aggregator) => {});
+ * aggregator.eventHandler("on_user_mute_stopped", async (aggregator) => {});
+ * aggregator.eventHandler("on_user_backchannel", async (aggregator, message: UserBackchannelMessage) => {});
+ */
+export declare class LLMUserAggregator extends LLMContextAggregator {
+    protected _params: LLMUserAggregatorParams;
+    protected _realtimeServiceMode: boolean | null;
+    protected _userIsMuted: boolean;
+    protected _userTurnStartTimestamp: string;
+    protected _realtimeMetadataHandled: boolean;
+    protected _ttfsP99Latency: number | null;
+    protected _realtimeHandoffFlushTask: Task<unknown> | null;
+    protected _fullUserTurnAggregation: string | null;
+    protected _userTurnInterruptedBot: boolean;
+    protected _consecutiveEmptyUserTurnRecoveries: number;
+    _userTurnController: UserTurnController;
+    protected _userIdleController: UserIdleController;
+    protected _vadController: VADController | null;
+    /**
+     * Initialize the user context aggregator.
+     *
+     * @param context The LLM context for conversation storage.
+     */
+    constructor(context: LLMContext, options?: LLMUserAggregatorOptions);
+    /**
+     * Set up the aggregator.
+     *
+     * @param setup Configuration object containing setup parameters.
+     */
+    setup(setup: FrameProcessorSetup): Promise<void>;
+    /** Release this aggregator's resources at teardown. */
+    cleanup(): Promise<void>;
+    /**
+     * Process frames for user speech aggregation and context management.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow in the pipeline.
+     */
+    processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
+    /** Push the current aggregation. */
+    pushAggregation(): Promise<string>;
+    /**
+     * Write the aggregated user turn to the context.
+     *
+     * @param options.runLlm Whether to emit the LLMContextFrame that runs
+     *   inference. False when the response is already generated — an eager end
+     *   of turn that held — and only the context write is still owed.
+     * @returns The text written, or "" when there was nothing to write.
+     */
+    protected _pushAggregation(options?: {
+        runLlm?: boolean;
+    }): Promise<string>;
+    protected _start(_frame: StartFrame): Promise<void>;
+    protected _stop(_frame: EndFrame): Promise<void>;
+    protected _cancel(_frame: CancelFrame): Promise<void>;
+    /**
+     * Mutate turn strategies for realtime mode.
+     *
+     * Drops `TranscriptionUserTurnStartStrategy` from the start strategies
+     * (transcripts shouldn't start a turn when the realtime service drives the
+     * conversation) and flips `waitForTranscript = false` on stop strategies
+     * that expose the flag, so end-of-turn fires as soon as VAD / the turn
+     * analyzer reports end-of-speech.
+     */
+    protected _applyRealtimeModeStrategyMutations(userTurnStrategies: UserTurnStrategies, areUserProvidedCustomStrategies: boolean): void;
+    /**
+     * Turn off empty user turn recovery for realtime mode.
+     *
+     * A realtime LLM service hears the user's audio directly, so an empty
+     * transcript doesn't mean the model missed the speech.
+     */
+    protected _disableEmptyUserTurnRecovery(): void;
+    /**
+     * Dispatch a service metadata frame.
+     *
+     * Any service may recommend user turn strategies via the base
+     * `userTurnStrategies` field, so apply that first — regardless of service
+     * type or realtime mode — then run type-specific handling.
+     */
+    protected _handleServiceMetadata(frame: ServiceMetadataFrame): Promise<void>;
+    /**
+     * Handle an `STTMetadataFrame` broadcast by an STT service at start.
+     *
+     * Captures the STT TTFS P99 so the realtime-mode deferred handoff flush can
+     * size itself to the real transcript-arrival latency. Any recommended user
+     * turn strategies were already applied by `_handleServiceMetadata`.
+     */
+    protected _handleSttMetadata(frame: STTMetadataFrame): Promise<void>;
+    /**
+     * Apply user turn strategies a service recommends via its metadata frame.
+     *
+     * Honored only when the user did not pass their own `userTurnStrategies`,
+     * which always wins. `updateStrategies` is idempotent, so this is safe to
+     * re-run on a later broadcast (e.g. a ServiceSwitcher switch).
+     */
+    protected _handleServiceUserTurnStrategies(serviceName: string, userTurnStrategies: UserTurnStrategies | null): Promise<void>;
+    /**
+     * Warn when (maybe unintentionally) overruling a service silently turns interruptions back on.
+     *
+     * A service recommends turn strategies by putting them on its
+     * `ServiceMetadataFrame.userTurnStrategies`, and one configured with
+     * `shouldInterrupt: false` carries that setting there. User-provided
+     * strategies discard the recommendation whole — so the pipeline would
+     * start interrupting with no indication why.
+     *
+     * Passing `ExternalUserTurnStrategies` explicitly was a safe way to opt out
+     * of the recommendation until 1.8.0, because the external start strategy
+     * never interrupted no matter how it was constructed. It broadcasts the
+     * interruption now, so explicit strategies can contradict the service, and
+     * this warning names the two ways to reconcile them.
+     *
+     * Only that container is checked. A `UserTurnStrategies` assembled by hand
+     * around a bare `ExternalUserTurnStartStrategy` contradicts the service the
+     * same way and goes unwarned — not expected to come up often enough to
+     * justify inspecting the individual strategies.
+     */
+    protected _warnOnDiscardedInterruptionSetting(serviceName: string, recommended: UserTurnStrategies): void;
+    /**
+     * Handle an `LLMServiceMetadataFrame` broadcast by an LLM service.
+     *
+     * Non-realtime services need no configuration. For a realtime service,
+     * auto-configure `realtimeServiceMode` (once) when the user left it unset
+     * (`null`); an explicit `true`/`false` is honored as-is (`false` opts out
+     * silently). When the mode is active, strip transcript dependence from the
+     * active strategies — including any external strategies
+     * `_handleServiceMetadata` just adopted.
+     *
+     * The strategy mutation re-runs on every broadcast (so a re-adopted
+     * recommendation is re-mutated, e.g. after a ServiceSwitcher switch); only
+     * the auto-configuration + log is one-shot. `updateStrategies` is
+     * idempotent, so re-applying is safe.
+     */
+    protected _handleLlmServiceMetadata(frame: LLMServiceMetadataFrame): Promise<void>;
+    /**
+     * Commit the user message in realtime mode, allowing for late transcripts.
+     *
+     * Called by the paired assistant half when the assistant response starts —
+     * realtime mode treats that as the user turn's end signal for
+     * context-writing purposes (in place of `UserStoppedSpeakingFrame`).
+     * Realtime services may deliver the finalized user `TranscriptionFrame` in
+     * pieces, and some deliver it only after the assistant starts responding,
+     * so we always defer the flush by `ttfsP99Latency` to let any in-flight
+     * transcript land. The `_realtimeHandoffFlushImmediate` failsafe on
+     * `LLMFullResponseEndFrame` catches any stragglers.
+     *
+     * Pair-internal: called by the paired LLMAssistantAggregator.
+     */
+    _realtimeHandoffFlush(): Promise<void>;
+    /** Wait one `ttfsP99Latency` window, then flush whatever has arrived. */
+    protected _realtimeDeferredHandoffFlush(): Promise<void>;
+    /**
+     * Failsafe flush invoked when the assistant response ends.
+     *
+     * By the time the assistant response has fully completed, even slow
+     * transcripts should have landed. Cancel any pending deferred flush and
+     * write the user message now so it precedes the assistant message in
+     * context.
+     *
+     * Pair-internal: called by the paired LLMAssistantAggregator.
+     */
+    _realtimeHandoffFlushImmediate(): Promise<void>;
+    protected _cancelRealtimeHandoffFlushTask(): Promise<void>;
+    protected _startControllers(): Promise<void>;
+    protected _stopControllers(): Promise<void>;
+    protected _maybeMuteFrame(frame: Frame): Promise<boolean>;
+    protected _handleLlmRun(_frame: LLMRunFrame): Promise<void>;
+    protected _handleLlmMessagesAppend(frame: LLMMessagesAppendFrame): Promise<void>;
+    protected _handleLlmMessagesUpdate(frame: LLMMessagesUpdateFrame): Promise<void>;
+    protected _handleLlmMessagesTransform(frame: LLMMessagesTransformFrame): Promise<void>;
+    protected _handleTranscription(frame: TranscriptionFrame): Promise<void>;
+    /**
+     * Broadcasts a frame upstream and queues it for internal processing.
+     *
+     * Queues the frame so it flows through `processFrame` and is handled
+     * internally (e.g. by the `UserTurnController`). The upstream frame is
+     * pushed directly.
+     *
+     * @param frameCls The class of the frame to be broadcasted.
+     * @param options Options to be passed to the frame's constructor.
+     */
+    protected _queuedBroadcastFrame<O>(frameCls: new (options: O) => Frame, options?: O): Promise<void>;
+    protected _onPushFrame(_controller: unknown, frame: Frame, direction?: FrameDirection): Promise<void>;
+    protected _onBroadcastFrame<O>(_controller: unknown, frameCls: new (options: O) => Frame, options?: O): Promise<void>;
+    protected _onUserTurnSpeculationCancelled(_controller: unknown): Promise<void>;
+    protected _onVadSpeechStarted(controller: VADController): Promise<void>;
+    protected _onVadSpeechStopped(controller: VADController): Promise<void>;
+    protected _onVadSpeechActivity(_controller: VADController): Promise<void>;
+    protected _onUserTurnStarted(_controller: UserTurnController, strategy: BaseUserTurnStartStrategy, params: UserTurnStartedParams): Promise<void>;
+    protected _onUserTurnInferenceTriggered(_controller: UserTurnController, strategy: BaseUserTurnStopStrategy, speculation?: UserTurnSpeculation | null): Promise<void>;
+    /**
+     * Run an inference for a turn that hasn't ended yet.
+     *
+     * The turn is still open, so the context must not record it. The inference
+     * runs against a provisional copy carrying the speculated turn text:
+     * nothing here mutates the real context, and `_aggregation` keeps
+     * accumulating for whenever the turn does end. The response is held
+     * downstream until the turn is confirmed, and discarded if it isn't.
+     *
+     * @param speculation The speculation to run, from the stop strategy that
+     *   started it.
+     */
+    protected _runSpeculativeInference(speculation: UserTurnSpeculation): Promise<void>;
+    protected _onUserTurnStopped(_controller: UserTurnController, strategy: BaseUserTurnStopStrategy, params: UserTurnStoppedParams): Promise<void>;
+    protected _onResetAggregation(_controller: UserTurnController, strategy: BaseUserTurnStartStrategy): Promise<void>;
+    protected _onUserTurnStopTimeout(_controller: unknown): Promise<void>;
+    protected _onUserTurnIdle(_controller: unknown): Promise<void>;
+    /**
+     * Maybe emit user turn stopped event.
+     *
+     * Earlier inference triggers in the same turn have already pushed their
+     * segments to the context and accumulated them into
+     * `this._fullUserTurnAggregation`. Any aggregation that arrived after the
+     * last inference trigger is flushed here so end-of-turn content is never
+     * lost from the public event.
+     *
+     * @param options.strategy The strategy that triggered the turn stop.
+     * @param options.onSessionEnd If true, only emit if there's unemitted
+     *   content (avoids duplicate events when session ends).
+     * @param options.runLlm Whether the context write should run inference.
+     */
+    protected _maybeEmitUserTurnStopped(options?: {
+        strategy?: BaseUserTurnStopStrategy | null;
+        onSessionEnd?: boolean;
+        runLlm?: boolean;
+    }): Promise<void>;
+    /**
+     * Run the LLM for a user turn that ended with no transcript.
+     *
+     * @param interruptedBot Whether the turn interrupted a response in progress.
+     * @returns Whether the LLM was asked to respond.
+     */
+    protected _maybeRecoverEmptyUserTurn(interruptedBot: boolean): Promise<boolean>;
+}
+export interface LLMAssistantAggregatorOptions extends FrameProcessorOptions {
+    /** Configuration parameters for aggregation behavior. */
+    params?: LLMAssistantAggregatorParams | null;
+    /**
+     * Pair-internal. Realtime-mode flag propagated from
+     * `LLMContextAggregatorPair` (`null` = auto-configure from service
+     * metadata). Not intended for direct use — construct the aggregators via
+     * the pair.
+     */
+    _realtimeServiceMode?: boolean | null;
+    /**
+     * Pair-internal. Back-reference to the paired `LLMUserAggregator`. The
+     * assistant flushes it on `LLMFullResponseStartFrame` so the user message
+     * lands in context before the assistant turn starts.
+     */
+    _pairedUserAggregator?: LLMUserAggregator | null;
+}
+/**
+ * Assistant LLM aggregator that processes bot responses and function calls.
+ *
+ * This aggregator handles the complex logic of processing assistant
+ * responses including:
+ *
+ * - Text frame aggregation between response start/end markers
+ * - Function call lifecycle management
+ * - Context updates with timestamps
+ * - Tool execution and result handling
+ * - Interruption handling during responses
+ *
+ * The aggregator manages function calls in progress and coordinates between
+ * text generation and tool execution phases of LLM responses.
+ *
+ * Event handlers available:
+ *
+ * - on_assistant_turn_started: Called when the assistant turn starts
+ * - on_assistant_turn_stopped: Called when the assistant turn ends
+ * - on_assistant_thought: Called when an assistant thought is available
+ * - on_assistant_backchannel: Called when the assistant starts saying text of
+ *   type `backchannel`, such as a `BotBackchannelFrame`
+ * - on_summary_applied: Called when a context summarization is applied
+ *
+ * @example
+ * aggregator.eventHandler("on_assistant_turn_started", async (aggregator) => {});
+ * aggregator.eventHandler(
+ *   "on_assistant_turn_stopped",
+ *   async (aggregator, message: AssistantTurnStoppedMessage) => {},
+ * );
+ * aggregator.eventHandler("on_assistant_thought", async (aggregator, message: AssistantThoughtMessage) => {});
+ * aggregator.eventHandler(
+ *   "on_assistant_backchannel",
+ *   async (aggregator, message: AssistantBackchannelMessage) => {},
+ * );
+ * aggregator.eventHandler(
+ *   "on_summary_applied",
+ *   async (aggregator, summarizer: LLMContextSummarizer, event: SummaryAppliedEvent) => {},
+ * );
+ */
+export declare class LLMAssistantAggregator extends LLMContextAggregator {
+    protected _params: LLMAssistantAggregatorParams;
+    protected _realtimeServiceMode: boolean | null;
+    protected _pairedUserAggregator: LLMUserAggregator | null;
+    protected _functionCallsInProgress: Map<string, FunctionCallInProgressFrame | null>;
+    protected _functionCallsImageResults: Map<string, UserImageRawFrame>;
+    protected _seenTurnMarkers: Set<string>;
+    protected _contextUpdatedTasks: Set<Task<unknown>>;
+    protected _userSpeaking: boolean;
+    protected _botSpeaking: boolean;
+    protected _pushContextOnBotStoppedSpeaking: boolean;
+    protected _contextPushOwed: boolean;
+    protected _assistantTurnStartTimestamp: string;
+    protected _thoughtAppendToContext: boolean;
+    protected _thoughtLlm: string;
+    protected _thoughtAggregation: TextPartForConcatenation[];
+    protected _thoughtStartTime: string;
+    protected _summarizer: LLMContextSummarizer | null;
+    /**
+     * Initialize the assistant context aggregator.
+     *
+     * @param context The LLM context for conversation storage.
+     */
+    constructor(context: LLMContext, options?: LLMAssistantAggregatorOptions);
+    /** Whether there are any function calls currently in progress. */
+    get hasFunctionCallsInProgress(): boolean;
+    /**
+     * Whether a synchronous call is in flight, one whose result the model waits for.
+     *
+     * A synchronous call leaves the context without its result until it
+     * returns, so the model cannot run in the meantime; an asynchronous one
+     * (`cancelOnInterruption: false`) has a placeholder result in the context
+     * and the model runs on. A call announced but not yet started counts as
+     * synchronous.
+     */
+    get hasSynchronousFunctionCallsInProgress(): boolean;
+    /**
+     * Set up the aggregator.
+     *
+     * @param setup Configuration object containing setup parameters.
+     */
+    setup(setup: FrameProcessorSetup): Promise<void>;
+    /** Release this aggregator's resources at teardown. */
+    cleanup(): Promise<void>;
+    /** Reset the aggregation state. */
+    reset(): Promise<void>;
+    /** Reset the thought aggregation state. */
+    protected _resetThoughtAggregation(): Promise<void>;
+    /**
+     * Process frames for assistant response aggregation and function call management.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow in the pipeline.
+     */
+    processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
+    protected _start(_frame: StartFrame): Promise<void>;
+    /**
+     * Throw if realtime mode is active without a paired user aggregator.
+     *
+     * Realtime mode needs the assistant half to hold a back-reference to the
+     * user half so it can flush the user message on `LLMFullResponseStartFrame`.
+     * The pair wires this unconditionally (so the halves are always configured
+     * together); this guards unsupported direct construction, surfacing the
+     * missing flush loudly instead of silently dropping user messages. Checked
+     * at `StartFrame` for an explicit mode and again when the mode auto-enables.
+     *
+     * @throws Error (upstream: RuntimeError) when no user aggregator is paired.
+     */
+    protected _requirePairedUserAggregator(): void;
+    /** Push the current assistant aggregation with timestamp. */
+    pushAggregation(): Promise<string>;
+    /**
+     * Commit the held assistant aggregation as a context message, pushing no frames.
+     *
+     * For flushes in the middle of an open assistant turn (a user file or image
+     * arriving mid-reply), where the downstream context and timestamp frames —
+     * and pushContextFrame()'s side effects — belong to the turn's real end,
+     * not to this commit.
+     *
+     * @returns The committed aggregation, or an empty string if none was held.
+     */
+    protected _addAggregationToContext(): Promise<string>;
+    /**
+     * Push a context frame in the specified direction.
+     *
+     * @param direction The direction to push the frame (upstream or downstream).
+     */
+    pushContextFrame(direction?: FrameDirection): Promise<void>;
+    protected _handleLlmRun(_frame: LLMRunFrame): Promise<void>;
+    protected _handleLlmMessagesAppend(frame: LLMMessagesAppendFrame): Promise<void>;
+    protected _handleLlmMessagesUpdate(frame: LLMMessagesUpdateFrame): Promise<void>;
+    protected _handleLlmMessagesTransform(frame: LLMMessagesTransformFrame): Promise<void>;
+    protected _handleInterruptions(_frame: InterruptionFrame): Promise<void>;
+    protected _handleEndOrCancel(frame: Frame): Promise<void>;
+    protected _cleanup(): Promise<void>;
+    protected _handleFunctionCallsStarted(frame: FunctionCallsStartedFrame): Promise<void>;
+    protected _handleFunctionCallInProgress(frame: FunctionCallInProgressFrame): Promise<void>;
+    protected _handleFunctionCallResult(frame: FunctionCallResultFrame): Promise<void>;
+    /**
+     * Decide whether to push a context frame after a function call settles or a message is appended.
+     *
+     * Push an `LLMContextFrame` upstream (with care to avoid duplicate pushes
+     * while results or appends are queued or the bot is still speaking).
+     * Cascade LLMs use the context frame to re-run inference with the new tool
+     * result in scope. Realtime LLMs read the new tool result out of the
+     * context the same way — they don't get function results from
+     * `FunctionCallResultFrame` directly — so the same push is load-bearing for
+     * both modes.
+     */
+    protected _maybePushContext(): Promise<void>;
+    /**
+     * Handle an intermediate result for an async function call.
+     *
+     * Injects an intermediate developer message into the context without
+     * removing the call from the in-progress map.
+     */
+    protected _handleFunctionCallIntermediateResult(frame: FunctionCallResultFrame, _inProgressFrame: FunctionCallInProgressFrame): Promise<void>;
+    /**
+     * Handle the final result of a function call.
+     *
+     * Removes the call from the in-progress map, updates the context, and
+     * triggers LLM inference when appropriate.
+     */
+    protected _handleFunctionCallFinished(frame: FunctionCallResultFrame, inProgressFrame: FunctionCallInProgressFrame): Promise<void>;
+    protected _handleFunctionCallCancel(frame: FunctionCallCancelFrame): Promise<void>;
+    protected _handleUserImageFrame(frame: UserImageRawFrame): Promise<void>;
+    protected _handleUserFileFrame(frame: UserFileRawFrame): Promise<void>;
+    protected _handleAssistantImageFrame(frame: AssistantImageRawFrame): Promise<void>;
+    protected _handleLlmStart(_frame: LLMFullResponseStartFrame): Promise<void>;
+    /**
+     * Realtime mode: commit the paired user message, then start the assistant turn.
+     *
+     * Realtime mode treats `LLMFullResponseStartFrame` as the user turn's end
+     * signal for context-writing purposes (in place of
+     * `UserStoppedSpeakingFrame`). The paired user aggregator may defer the
+     * actual write for one `ttfsP99Latency` window to allow a late-arriving
+     * transcript to land.
+     */
+    protected _realtimeHandleLlmStart(): Promise<void>;
+    protected _handleLlmEnd(_frame: LLMFullResponseEndFrame): Promise<void>;
+    protected _handleTtsStarted(frame: TTSStartedFrame): Promise<void>;
+    protected _handlePushAggregation(): Promise<void>;
+    protected _handleText(frame: TextFrame): Promise<void>;
+    protected _handleMarkerFrame(frame: LLMMarkerFrame): Promise<void>;
+    protected _handleThoughtStart(frame: LLMThoughtStartFrame): Promise<void>;
+    protected _handleThoughtText(frame: LLMThoughtTextFrame): Promise<void>;
+    protected _handleThoughtEnd(frame: LLMThoughtEndFrame): Promise<void>;
+    protected _maybeAppendImageToContext(frame: UserImageRawFrame): Promise<boolean>;
+    /**
+     * Whether a call's final result must be delivered as a deferred message.
+     *
+     * An async call (`cancelOnInterruption: false`) whose result arrives before
+     * the conversation moved on is indistinguishable from a synchronous one,
+     * and settles in place: its "started" placeholder becomes the tool result,
+     * and the LLM sees an ordinary call. The context alone decides. The result
+     * is deferred when a user or developer message follows the placeholder:
+     * something the user said, new task instructions, or an intermediate update
+     * from this call. Assistant messages are ignored, so filler spoken with a
+     * `TTSSpeakFrame` while the call runs does not defer it, whether the tool
+     * handler or an event handler spoke it. The same holds for text the model
+     * wrote after seeing the placeholder with no user turn in between, as when
+     * an ungrouped sibling's result runs inference; the result still settles in
+     * place. Other calls' placeholders, results, and deferred messages are
+     * ignored too. A placeholder that is no longer in the context, because the
+     * context was rebuilt while the call ran, also defers.
+     *
+     * @param inProgressFrame The call's in-progress frame.
+     * @returns True when the result goes into a developer message.
+     */
+    protected _isDeferred(inProgressFrame: FunctionCallInProgressFrame): boolean;
+    protected _updateFunctionCallResult(_functionName: string, toolCallId: string, result: unknown): void;
+    protected _contextUpdatedTaskFinished(task: Task<unknown>): void;
+    protected _triggerAssistantTurnStarted(): Promise<void>;
+    protected _triggerAssistantTurnStopped(options?: {
+        interrupted?: boolean;
+    }): Promise<void>;
+    /**
+     * Strip turn completion markers from assistant transcript.
+     *
+     * Turn completion markers are used internally for turn completion detection
+     * and shouldn't appear in the final transcript. Both the default markers
+     * and any seen on this turn's marker frames are removed, so a custom marker
+     * set is stripped too.
+     */
+    protected _maybeStripTurnCompletionMarkers(text: string): string;
+    /**
+     * Handle summarization request from the summarizer.
+     *
+     * Push the request frame UPSTREAM to the LLM service for processing.
+     *
+     * @param summarizer The summarizer that generated the request.
+     * @param frame The summarization request frame to broadcast.
+     */
+    protected _onRequestSummarization(_summarizer: LLMContextSummarizer, frame: LLMContextSummaryRequestFrame): Promise<void>;
+    /**
+     * Handle summary applied event from the summarizer.
+     *
+     * Forwards the event to any registered `on_summary_applied` handlers.
+     *
+     * @param summarizer The summarizer that applied the summary.
+     * @param event The summary applied event.
+     */
+    protected _onSummaryApplied(summarizer: LLMContextSummarizer, event: SummaryAppliedEvent): Promise<void>;
+}
+export interface LLMContextAggregatorPairOptions {
+    /** Parameters for the user context aggregator. */
+    userParams?: LLMUserAggregatorParams | null;
+    /** Parameters for the assistant context aggregator. */
+    assistantParams?: LLMAssistantAggregatorParams | null;
+    /**
+     * When provided, sets the field of the same name on both `userParams` and
+     * `assistantParams`, overriding any value already set on either. This is
+     * the preferred way to enable tool-change announcements: it ensures both
+     * aggregators participate, which makes the feature robust regardless of
+     * which aggregator handles a given `LLMSetToolsFrame`. The shared context
+     * guarantees the announcement is added exactly once (the second
+     * aggregator's diff is empty by the time it sees the frame). Leave as
+     * `null` to respect per-params settings.
+     */
+    addToolChangeMessages?: boolean | null;
+    /**
+     * Configures the pair for use with a realtime (speech-to-speech) LLM
+     * service. When enabled, context writes become trailing — driven by the
+     * content stream itself (transcripts, `LLMFullResponseStartFrame`) rather
+     * than turn frames — and turn-end strategies stop waiting for transcripts.
+     * `null` (the default) auto-configures: the mode turns on when a realtime
+     * service announces itself via service metadata, and stays off otherwise.
+     * `true` / `false` force it on or off (`false` keeps legacy pre-realtime
+     * context-write behavior). Both halves share this setting.
+     */
+    realtimeServiceMode?: boolean | null;
+}
+/** Pair of LLM context aggregators for updating context with user and assistant messages. */
+export declare class LLMContextAggregatorPair {
+    protected _user: LLMUserAggregator;
+    protected _assistant: LLMAssistantAggregator;
+    /**
+     * Initialize the LLM context aggregator pair.
+     *
+     * @param context The context to be managed by the aggregators.
+     */
+    constructor(context: LLMContext, options?: LLMContextAggregatorPairOptions);
+    /**
+     * Get the user context aggregator.
+     *
+     * @returns The user context aggregator instance.
+     */
+    user(): LLMUserAggregator;
+    /**
+     * Get the assistant context aggregator.
+     *
+     * @returns The assistant context aggregator instance.
+     */
+    assistant(): LLMAssistantAggregator;
+    /**
+     * Allow destructuring of the aggregator pair.
+     *
+     * @example
+     * const pair = new LLMContextAggregatorPair(context); // The instance
+     * const [user, assistant] = new LLMContextAggregatorPair(context); // Destructured
+     */
+    [Symbol.iterator](): IterableIterator<LLMUserAggregator | LLMAssistantAggregator>;
+}
+//# sourceMappingURL=llm_response_universal.d.ts.map