@pipedog/core 0.0.0 → 0.2.0

package/dist/observers/turn_tracking_observer.js

@@ -0,0 +1,189 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/observers/turn_tracking_observer.py
+/**
+ * Turn tracking observer for conversation flow monitoring.
+ *
+ * Provides an observer that monitors conversation turns in a pipeline,
+ * tracking when turns start and end based on user and bot speech patterns.
+ */
+import { createTask, sleep } from "@pipedog/asyncio";
+import { BotStartedSpeakingFrame, BotStoppedSpeakingFrame, CancelFrame, EndFrame, StartFrame, UserStartedSpeakingFrame, } from "../frames/frames.js";
+import { logger } from "../utils/logger.js";
+import { BaseObserver } from "./base_observer.js";
+/**
+ * Observer that tracks conversation turns in a pipeline.
+ *
+ * This observer monitors the flow of conversation by tracking when turns
+ * start and end based on user and bot speaking patterns. It handles
+ * interruptions, timeouts, and maintains turn state throughout the pipeline.
+ *
+ * Turn tracking logic:
+ *
+ * - The first turn starts immediately when the pipeline starts (StartFrame)
+ * - Subsequent turns start when the user starts speaking
+ * - A turn ends when the bot stops speaking and either:
+ *   - The user starts speaking again
+ *   - A timeout period elapses with no more bot speech
+ *
+ * Events:
+ *
+ * - on_turn_started: Triggered when a turn starts, providing the turn number
+ * - on_turn_ended: Triggered when a turn ends, providing the turn number, the
+ *   turn's duration in seconds, and whether it was interrupted
+ */
+export class TurnTrackingObserver extends BaseObserver {
+    _turnCount = 0;
+    _isTurnActive = false;
+    _isBotSpeaking = false;
+    _hasBotSpoken = false;
+    _turnStartTime = 0;
+    _turnEndTimeoutSecs;
+    // Stands in for upstream's `loop.call_later` handle: a task that sleeps for
+    // the timeout and then ends the turn.
+    _endTurnTimer = null;
+    constructor(options = {}) {
+        super({ ...options, observeEveryPush: false });
+        this._turnEndTimeoutSecs = options.turnEndTimeoutSecs ?? 2.5;
+        this._registerEventHandler("on_turn_started");
+        this._registerEventHandler("on_turn_ended");
+    }
+    /**
+     * Process frame events for turn tracking.
+     *
+     * @param data Frame push event data containing the frame and metadata.
+     */
+    async onPushFrame(data) {
+        if (data.frame instanceof StartFrame) {
+            // Start the first turn immediately when the pipeline starts
+            if (this._turnCount === 0) {
+                await this._startTurn(data);
+            }
+        }
+        else if (data.frame instanceof UserStartedSpeakingFrame) {
+            await this._handleUserStartedSpeaking(data);
+        }
+        else if (data.frame instanceof BotStartedSpeakingFrame) {
+            await this._handleBotStartedSpeaking(data);
+        }
+        // A BotStoppedSpeakingFrame can arrive after a UserStartedSpeakingFrame
+        // following an interruption. We only want to end the turn if the bot was
+        // previously speaking.
+        else if (data.frame instanceof BotStoppedSpeakingFrame && this._isBotSpeaking) {
+            await this._handleBotStoppedSpeaking(data);
+        }
+        else if (data.frame instanceof EndFrame || data.frame instanceof CancelFrame) {
+            await this._handlePipelineEnd(data);
+        }
+    }
+    /**
+     * Cancel the pending turn end timer.
+     *
+     * The pipeline can be torn down before this observer receives its EndFrame
+     * or CancelFrame, and the timer's callback holds the pipeline until it fires.
+     */
+    async cleanup() {
+        this._cancelTurnEndTimer();
+        await super.cleanup();
+    }
+    /** Schedule turn end with a timeout. */
+    _scheduleTurnEnd(data) {
+        // Cancel any existing timer
+        this._cancelTurnEndTimer();
+        // Create a new timer
+        const timer = createTask(async () => {
+            await sleep(this._turnEndTimeoutSecs);
+            // The timer has fired: like a `call_later` handle, cancelling it from
+            // now on no longer stops the callback.
+            if (this._endTurnTimer === timer) {
+                this._endTurnTimer = null;
+            }
+            await this._endTurnAfterTimeout(data);
+        }, "TurnTrackingObserver::_end_turn_after_timeout");
+        this._endTurnTimer = timer;
+    }
+    /** Cancel the turn end timer if it exists. */
+    _cancelTurnEndTimer() {
+        if (this._endTurnTimer) {
+            this._endTurnTimer.cancel();
+            this._endTurnTimer = null;
+        }
+    }
+    /** End turn after timeout has expired. */
+    async _endTurnAfterTimeout(data) {
+        if (this._isTurnActive && !this._isBotSpeaking) {
+            logger.trace(`Turn ${this._turnCount} ending due to timeout`);
+            await this._endTurn(data, false);
+            this._endTurnTimer = null;
+        }
+    }
+    /** Handle user speaking events, including interruptions. */
+    async _handleUserStartedSpeaking(data) {
+        if (this._isBotSpeaking) {
+            // Handle interruption - end current turn and start a new one
+            this._cancelTurnEndTimer(); // Cancel any pending end turn timer
+            await this._endTurn(data, true);
+            this._isBotSpeaking = false; // Bot is considered interrupted
+            await this._startTurn(data);
+        }
+        else if (this._isTurnActive && this._hasBotSpoken) {
+            // User started speaking during the turnEndTimeoutSecs period after bot speech
+            this._cancelTurnEndTimer(); // Cancel any pending end turn timer
+            await this._endTurn(data, false);
+            await this._startTurn(data);
+        }
+        else if (!this._isTurnActive) {
+            // Start a new turn after previous one ended
+            await this._startTurn(data);
+        }
+        else {
+            // User is speaking within the same turn (before bot has responded)
+            logger.trace(`User is already speaking in Turn ${this._turnCount}`);
+        }
+    }
+    /** Handle bot speaking events. */
+    async _handleBotStartedSpeaking(_data) {
+        this._isBotSpeaking = true;
+        this._hasBotSpoken = true;
+        // Cancel any pending turn end timer when bot starts speaking again
+        this._cancelTurnEndTimer();
+    }
+    /** Handle bot stopped speaking events. */
+    async _handleBotStoppedSpeaking(data) {
+        this._isBotSpeaking = false;
+        // Schedule turn end with timeout.
+        // This is needed to handle cases where the bot's speech ends and then
+        // resumes. This can happen with HTTP TTS services or function calls.
+        this._scheduleTurnEnd(data);
+    }
+    /** Handle pipeline end or cancellation by flushing any active turn. */
+    async _handlePipelineEnd(data) {
+        this._cancelTurnEndTimer();
+        if (this._isTurnActive) {
+            await this._endTurn(data, true);
+        }
+    }
+    /** Start a new turn. */
+    async _startTurn(data) {
+        this._isTurnActive = true;
+        this._hasBotSpoken = false;
+        this._turnCount += 1;
+        this._turnStartTime = data.timestamp;
+        logger.trace(`Turn ${this._turnCount} started`);
+        await this._callEventHandler("on_turn_started", this._turnCount);
+    }
+    /** End the current turn. */
+    async _endTurn(data, wasInterrupted) {
+        if (!this._isTurnActive) {
+            return;
+        }
+        const duration = (data.timestamp - this._turnStartTime) / 1_000_000_000; // Convert to seconds
+        this._isTurnActive = false;
+        const status = wasInterrupted ? "interrupted" : "completed";
+        logger.trace(`Turn ${this._turnCount} ${status} after ${duration.toFixed(2)}s`);
+        await this._callEventHandler("on_turn_ended", this._turnCount, duration, wasInterrupted);
+    }
+}
+//# sourceMappingURL=turn_tracking_observer.js.map

package/dist/observers/user_bot_latency_observer.d.ts

@@ -0,0 +1,411 @@
+import { MetricsFrame } from "../frames/frames.ts";
+import { BaseObserver, type BaseObserverOptions, type FramePushed } from "./base_observer.ts";
+export interface TTFBBreakdownMetricsOptions {
+    processor: string;
+    model?: string | null;
+    startTime: number;
+    durationSecs: number;
+}
+/**
+ * TTFB measurement with timestamp for timeline placement.
+ *
+ * - `processor`: name of the processor that reported the TTFB.
+ * - `model`: optional model name associated with the metric.
+ * - `startTime`: Unix timestamp (seconds) when the TTFB measurement started.
+ * - `durationSecs`: TTFB duration in seconds.
+ */
+export declare class TTFBBreakdownMetrics {
+    processor: string;
+    model: string | null;
+    startTime: number;
+    durationSecs: number;
+    constructor(options: TTFBBreakdownMetricsOptions);
+}
+export interface TextAggregationBreakdownMetricsOptions {
+    processor: string;
+    startTime: number;
+    durationSecs: number;
+}
+/**
+ * Text aggregation measurement with timestamp for timeline placement.
+ *
+ * - `processor`: name of the processor that reported the metric.
+ * - `startTime`: Unix timestamp (seconds) when text aggregation started.
+ * - `durationSecs`: aggregation duration in seconds.
+ */
+export declare class TextAggregationBreakdownMetrics {
+    processor: string;
+    startTime: number;
+    durationSecs: number;
+    constructor(options: TextAggregationBreakdownMetricsOptions);
+}
+export interface FunctionCallMetricsOptions {
+    functionName: string;
+    startTime: number;
+    durationSecs: number;
+}
+/**
+ * Latency for a single function call execution.
+ *
+ * - `functionName`: name of the function that was called.
+ * - `startTime`: Unix timestamp (seconds) when execution started.
+ * - `durationSecs`: time in seconds from execution start to result.
+ */
+export declare class FunctionCallMetrics {
+    functionName: string;
+    startTime: number;
+    durationSecs: number;
+    constructor(options: FunctionCallMetricsOptions);
+}
+/**
+ * Anything shorter than this is a frame hop rather than work worth naming, so
+ * it is rolled into the single pipeline contribution.
+ */
+export declare const MIN_CONTRIBUTION_SECS = 0.005;
+/**
+ * Half of the last digit `turnContributionLines()` prints, below which a span
+ * would show as zero and so is not part of the timeline at all.
+ */
+export declare const PRINTS_AS_ZERO_SECS = 0.0005;
+/**
+ * A point in a user-to-bot cycle that a contribution can start or end at.
+ *
+ * Most are observed as frames. `LLM_CHUNK` and `SENTENCE` are worked back from
+ * a service's own metric, because no frame marks them.
+ */
+declare const _MomentKind: {
+    readonly SILENCE: "silence";
+    readonly READY: "ready";
+    readonly VAD_STOP: "vad stop";
+    readonly TRANSCRIPT: "transcript";
+    readonly LLM_REQUEST: "llm request";
+    readonly LLM_CHUNK: "llm chunk";
+    readonly MARKER_COMPLETE: "marker complete";
+    readonly MARKER_INCOMPLETE: "marker incomplete";
+    readonly HANDLERS_START: "handlers start";
+    readonly HANDLERS_END: "handlers end";
+    readonly FIRST_TEXT: "first text";
+    readonly SENTENCE: "sentence";
+    readonly FIRST_AUDIO: "first audio";
+    readonly BOT_SPEAKING: "bot speaking";
+};
+type _MomentKind = (typeof _MomentKind)[keyof typeof _MomentKind];
+/**
+ * One moment in a cycle.
+ *
+ * - `at`: Unix timestamp (seconds) of the moment.
+ * - `kind`: what happened.
+ * - `source`: name of whatever produced it, used when a span takes its owner
+ *   from one end.
+ * - `derived`: whether it was worked back from a metric rather than observed
+ *   as a frame. A derived moment sits at or before the frame that reported
+ *   it, which decides the order when the two share a time.
+ */
+declare class _Moment {
+    readonly at: number;
+    readonly kind: _MomentKind;
+    readonly source: string;
+    readonly derived: boolean;
+    constructor(options: {
+        at: number;
+        kind: _MomentKind;
+        source?: string;
+        derived?: boolean;
+    });
+}
+/**
+ * Where a measured interval was anchored.
+ *
+ * A greeting is timed from the client connecting, a turn from the user
+ * falling silent, so a partial interval is never mistaken for a whole one.
+ */
+export declare const MeasuredFrom: {
+    readonly USER_SILENCE: "user_silence";
+    readonly CLIENT_CONNECTED: "client_connected";
+};
+export type MeasuredFrom = (typeof MeasuredFrom)[keyof typeof MeasuredFrom];
+/**
+ * What kind of thing a contribution's owner is.
+ *
+ * Grouping on this answers where a turn's time went without matching on the
+ * owner's name: a service that could be swapped, a setting the bot chose, the
+ * bot's own code, or the pipeline between them.
+ */
+export declare const LatencyOwnerKind: {
+    readonly SERVICE: "service";
+    readonly SETTING: "setting";
+    readonly BOT: "bot";
+    readonly PIPELINE: "pipeline";
+};
+export type LatencyOwnerKind = (typeof LatencyOwnerKind)[keyof typeof LatencyOwnerKind];
+export interface LatencyContributionOptions {
+    key: string;
+    label: string;
+    owner: string;
+    ownerKind: LatencyOwnerKind;
+    startTime: number;
+    durationSecs: number;
+}
+/**
+ * One named part of the user-to-bot interval.
+ *
+ * Contributions account for the whole interval, so their durations sum to the
+ * measured latency. Unlike a TTFB, which reports what a service spent once it
+ * was asked, a contribution can also name time no service is measuring: the
+ * silence a VAD waits out, the tokens a turn-completion marker occupies before
+ * any speakable text, or a hold while the pipeline waits for the user to
+ * finish a sentence.
+ *
+ * - `key`: stable identifier for this part of a turn. Unlike `label`, it is
+ *   safe to group on: it survives the label being reworded.
+ * - `label`: what the time was spent on.
+ * - `owner`: what spent it — a processor name, or a `config:` tag naming the
+ *   setting that governs it.
+ * - `ownerKind`: what kind of thing the owner is.
+ * - `startTime`: Unix timestamp (seconds) when it started.
+ * - `durationSecs`: how long it took, in seconds.
+ */
+export declare class LatencyContribution {
+    key: string;
+    label: string;
+    owner: string;
+    ownerKind: LatencyOwnerKind;
+    startTime: number;
+    durationSecs: number;
+    constructor(options: LatencyContributionOptions);
+}
+export interface LatencyBreakdownOptions {
+    ttfb?: TTFBBreakdownMetrics[];
+    contributions?: LatencyContribution[];
+    measuredFrom?: MeasuredFrom | null;
+    totalSecs?: number;
+    textAggregation?: TextAggregationBreakdownMetrics | null;
+    userTurnStartTime?: number | null;
+    userTurnSecs?: number | null;
+    functionCalls?: FunctionCallMetrics[];
+}
+/**
+ * Per-service latency breakdown for a single user-to-bot cycle.
+ *
+ * Collected between `VADUserStoppedSpeakingFrame` and
+ * `BotStartedSpeakingFrame` when `enableMetrics` is true in `PipelineParams`.
+ *
+ * - `ttfb`: time-to-first-byte metrics from each service in the pipeline.
+ * - `textAggregation`: first text aggregation measurement, representing the
+ *   latency cost of sentence aggregation in the TTS pipeline.
+ * - `userTurnStartTime`: Unix timestamp when the user turn started (actual
+ *   user silence, adjusted for VAD stop_secs). null if no
+ *   `VADUserStoppedSpeakingFrame` was observed.
+ * - `userTurnSecs`: duration in seconds of the user's turn, measured from
+ *   when the user actually stopped speaking to when the turn was released
+ *   (`UserStoppedSpeakingFrame`). This includes VAD silence detection, STT
+ *   finalization, and any turn analyzer wait. null if no
+ *   `UserStoppedSpeakingFrame` was observed (e.g. no turn analyzer
+ *   configured).
+ * - `functionCalls`: latency for each function call executed during this
+ *   cycle. Empty if no function calls occurred.
+ * - `contributions`: named parts of the interval, in chronological order,
+ *   summing to the measured latency. A part is listed only if it happened, so
+ *   a bot without turn completion has no marker or wait entries.
+ * - `measuredFrom`: where the interval was anchored, so a greeting is not
+ *   compared with a turn.
+ * - `totalSecs`: the measured interval, which the contributions sum to.
+ */
+export declare class LatencyBreakdown {
+    ttfb: TTFBBreakdownMetrics[];
+    contributions: LatencyContribution[];
+    measuredFrom: MeasuredFrom | null;
+    totalSecs: number;
+    textAggregation: TextAggregationBreakdownMetrics | null;
+    userTurnStartTime: number | null;
+    userTurnSecs: number | null;
+    functionCalls: FunctionCallMetrics[];
+    constructor(options?: LatencyBreakdownOptions);
+    /**
+     * Format the contributions for logging, one per line plus a total.
+     *
+     * One breakdown covers one user-to-bot cycle, so these lines describe a
+     * single turn — or, for the first thing the bot says, the wait before it
+     * from the pipeline being ready with a client on it, which the total line
+     * names.
+     *
+     * @param options.byCost Order by duration, largest first, rather than in
+     *   the order things happened. Useful when the question is what to optimize
+     *   rather than what the turn did.
+     * @returns One formatted line per contribution, then a total line. Empty if
+     *   no contributions were collected.
+     */
+    turnContributionLines(options?: {
+        byCost?: boolean;
+    }): string[];
+}
+export interface UserBotLatencyObserverOptions extends BaseObserverOptions {
+    /**
+     * Contributions shorter than this are rolled into the single pipeline entry
+     * rather than listed. Pass 0 to list every one, including individual frame
+     * hops.
+     */
+    minContributionSecs?: number;
+    /**
+     * Reads the current time in seconds. Supplying one lets a test drive a
+     * cycle without waiting out the intervals it describes.
+     */
+    timeSource?: () => number;
+}
+/**
+ * Observer that tracks user-to-bot response latency.
+ *
+ * Measures the time between when a user stops speaking
+ * (VADUserStoppedSpeakingFrame) and when the bot starts speaking
+ * (BotStartedSpeakingFrame). Emits events when latency is measured, allowing
+ * consumers to log, trace, or otherwise process the latency data.
+ *
+ * When `enableMetrics` is true in pipeline params, also collects a latency
+ * breakdown and emits an `on_latency_breakdown` event alongside the existing
+ * latency measurement. The breakdown carries per-service metrics (TTFB, text
+ * aggregation) and, in `contributions`, the whole interval named part by part:
+ *
+ * @example
+ * for (const line of breakdown.turnContributionLines()) {
+ *   logger.info(line);
+ * }
+ * // 0.200s  endpointing wait     [config: VAD stop_secs]
+ * // 0.125s  transcription        [DeepgramSTTService#0]
+ * // 0.336s  LLM inference        [OpenAILLMService#0]
+ * // 0.359s  speech synthesis     [CartesiaTTSService#0]
+ * // 1.020s  TOTAL
+ *
+ * This observer follows the composition pattern used by
+ * TurnTrackingObserver, acting as a reusable component for latency
+ * measurement.
+ *
+ * Events:
+ *
+ * - on_latency_measured(observer, latencySeconds): emitted when
+ *   time-to-first-bot-speech is calculated. Measures the time from when the
+ *   user stopped speaking to when the bot starts speaking.
+ * - on_latency_breakdown(observer, breakdown): emitted at each
+ *   `BotStartedSpeakingFrame` with a `LatencyBreakdown` containing
+ *   per-service metrics collected during the user→bot cycle.
+ * - on_first_bot_speech_latency(observer, latencySeconds): emitted once, the
+ *   first time `BotStartedSpeakingFrame` arrives after `ClientConnectedFrame`.
+ *   Measures the time from client connection to the first bot speech.
+ */
+export declare class UserBotLatencyObserver extends BaseObserver {
+    protected _minContributionSecs: number;
+    protected _now: () => number;
+    protected _userStoppedTime: number | null;
+    protected _userTurnStartTime: number | null;
+    protected _userTurn: number | null;
+    protected _clientConnectedTime: number | null;
+    protected _pipelineStartedTime: number | null;
+    protected _firstBotSpeechMeasured: boolean;
+    protected _moments: _Moment[];
+    protected _llmRequest: _Moment | null;
+    protected _markersSeen: boolean;
+    protected _ttfb: TTFBBreakdownMetrics[];
+    protected _textAggregation: TextAggregationBreakdownMetrics | null;
+    protected _functionCallStarts: Map<string, [string, number]>;
+    protected _functionCallMetrics: FunctionCallMetrics[];
+    /**
+     * Initialize the user-bot latency observer.
+     *
+     * @param options.minContributionSecs Contributions shorter than this are
+     *   rolled into the single pipeline entry rather than listed.
+     * @param options.timeSource Reads the current time in seconds.
+     */
+    constructor(options?: UserBotLatencyObserverOptions);
+    /**
+     * Record that the pipeline finished starting.
+     *
+     * The `StartFrame` has reached every processor by now, which is where
+     * `StartupTimingObserver` stops measuring. A greeting is timed from here so
+     * the two reports meet rather than overlap.
+     */
+    onPipelineStarted(): Promise<void>;
+    /**
+     * Process frames to track speech timing and calculate latency.
+     *
+     * Tracks VAD events and bot speaking events to measure the time between
+     * user stopping speech and bot starting speech. Also accumulates metrics
+     * from MetricsFrame for the latency breakdown.
+     *
+     * @param data Frame push event containing the frame and direction information.
+     */
+    onPushFrame(data: FramePushed): Promise<void>;
+    /**
+     * Record a moment, if a cycle is being measured.
+     *
+     * @param kind What happened.
+     * @param options.at When, defaulting to now. Derived moments pass their own time.
+     * @param options.source Name of whatever produced it.
+     * @param options.once Keep only the first of this kind, for frames that
+     *   repeat within a cycle such as text and audio.
+     * @param options.derived Whether it comes from a metric rather than from a frame.
+     * @returns The recorded moment, or null if it was not recorded.
+     */
+    protected _mark(kind: _MomentKind, options?: {
+        at?: number | null;
+        source?: string;
+        once?: boolean;
+        derived?: boolean;
+    }): _Moment | null;
+    /** Whether a moment of this kind has been recorded this cycle. */
+    protected _seen(kind: _MomentKind): boolean;
+    /** Whether a cycle is open: a user turn, or the wait for the greeting. */
+    protected get _measuring(): boolean;
+    /**
+     * Moments that come from metrics rather than from frames.
+     *
+     * Handlers dispatched together run concurrently, so they become one pair of
+     * moments named for the one the wait actually depends on: listing each
+     * would count the same stretch of wall clock more than once.
+     *
+     * @returns The sentence-aggregation and function-handler moments, if any.
+     */
+    protected _derivedMoments(): _Moment[];
+    /**
+     * Group function handlers whose executions overlap in time.
+     *
+     * @returns Groups of overlapping calls, in the order they started.
+     */
+    protected _concurrentHandlers(): FunctionCallMetrics[][];
+    /**
+     * Name each stretch between one moment and the next.
+     *
+     * Spans run from one moment to the following one, so they tile the cycle
+     * and cannot overlap. A stretch whose pair of moments has no entry in the
+     * table is left unnamed and reported as pipeline time, which is what makes
+     * a gap in the accounting visible rather than silently absorbed.
+     *
+     * @returns Contributions in chronological order, or empty if the cycle was
+     *   never measured.
+     */
+    protected _buildContributions(): LatencyContribution[];
+    /**
+     * Join neighbouring spans that say the same thing.
+     *
+     * A marker sits in the middle of what a reader thinks of as one stretch, so
+     * the table names both halves the same way and they are joined here. Only
+     * spans that meet are joined: two of the same name either side of an
+     * unnamed gap stay apart, so the gap is reported rather than absorbed.
+     *
+     * @param spans Contributions in chronological order.
+     * @returns The same coverage, with runs of one label and owner joined.
+     */
+    protected static _mergeAdjacent(spans: LatencyContribution[]): LatencyContribution[];
+    /** Handle BotStartedSpeakingFrame to emit latency and breakdown. */
+    protected _handleBotStartedSpeaking(): Promise<void>;
+    /**
+     * Extract latency metrics from a MetricsFrame.
+     *
+     * Accumulates metrics when a measurement is in progress: either a user→bot
+     * cycle (after `VADUserStoppedSpeakingFrame`) or the first-bot-speech
+     * window (after `ClientConnectedFrame`).
+     */
+    protected _handleMetricsFrame(frame: MetricsFrame): void;
+    /** Clear per-cycle metric accumulators. */
+    protected _resetAccumulators(): void;
+}
+export {};
+//# sourceMappingURL=user_bot_latency_observer.d.ts.map