@pipedog/core 0.0.0 → 0.1.0

package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.js

@@ -0,0 +1,332 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/turns/user_stop/turn_analyzer_user_turn_stop_strategy.py
+/** User turn stop strategy based on turn detection analyzers. */
+import { CancelledError, sleep, time } from "@pipedog/asyncio";
+import { EndOfTurnState } from "../../audio/turn/base_turn_analyzer.js";
+import { VAD_STOP_SECS } from "../../audio/vad/vad_analyzer.js";
+import { InputAudioRawFrame, InterimTranscriptionFrame, MetricsFrame, SpeechControlParamsFrame, STTMetadataFrame, StartFrame, TranscriptionFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame, } from "../../frames/frames.js";
+import { logger } from "../../utils/logger.js";
+import { ProcessFrameResult } from "../types.js";
+import { BaseUserTurnStopStrategy, } from "./base_user_turn_stop_strategy.js";
+/**
+ * User turn stop strategy that uses a turn detection model to determine if the
+ * user is done speaking.
+ *
+ * This strategy feeds audio, VAD, and transcription frames to a turn
+ * detection model (`BaseTurnAnalyzer`) that predicts when the user has
+ * finished their turn. Once the model indicates the turn is complete, the
+ * strategy waits for a final transcription before triggering the end of
+ * the user's turn.
+ *
+ * For services that support finalization (`TranscriptionFrame.finalized`),
+ * the turn can be triggered immediately once the finalized transcript is
+ * received. Otherwise, an STT timeout is used as a fallback: the turn is
+ * released once `ttfsP99Latency` has elapsed since the end of the user's
+ * speech (as reported by VAD, discounting its `stopSecs` silence window).
+ */
+export class TurnAnalyzerUserTurnStopStrategy extends BaseUserTurnStopStrategy {
+    _turnAnalyzer;
+    _waitForTranscript;
+    _sttTimeout = 0.0; // STT P99 latency from STTMetadataFrame
+    _stopSecs = 0.0; // VAD stop_secs from VADUserStoppedSpeakingFrame
+    _stopSecsWarned = false;
+    _text = "";
+    _turnComplete = false;
+    _vadUserSpeaking = false;
+    _vadStopped = false; // Whether VAD reported a stop this turn
+    _transcriptFinalized = false;
+    _timeoutTask = null;
+    _timeoutExpired = false;
+    constructor(options) {
+        super(options);
+        this._turnAnalyzer = options.turnAnalyzer;
+        this._waitForTranscript = options.waitForTranscript ?? true;
+    }
+    /** Whether transcripts gate end-of-turn signalling. */
+    get waitForTranscript() {
+        return this._waitForTranscript;
+    }
+    set waitForTranscript(value) {
+        this._waitForTranscript = value;
+    }
+    /** Ready the strategy to detect the end of the turn now starting. */
+    async handleUserTurnStarted() {
+        await this._reset();
+    }
+    /**
+     * Clear the analyzer's buffered speech when the turn ends.
+     *
+     * Beyond the usual per-turn reset, drop the analyzer's buffered speech
+     * state so a stale silence timer can't fire a phantom end-of-turn after
+     * an externally-ended turn (for example a forced stop while a mute
+     * strategy held audio back). The analyzer already clears itself on its
+     * own COMPLETE, so the clear is a no-op on the normal path; the
+     * pre-speech buffer refills continuously before the next turn starts.
+     */
+    async handleUserTurnStopped() {
+        await this._reset();
+        this._turnAnalyzer.clear();
+    }
+    /**
+     * Clear turn-scoped state. Runs at both turn boundaries.
+     *
+     * `_vadUserSpeaking` is deliberately left alone: whether the user is
+     * speaking belongs to the user rather than to the turn, and VAD reports it
+     * only on transitions. Clearing it at a turn start VAD didn't drive — a
+     * turn begun from a transcript, mid-utterance — would leave it wrong until
+     * the user next stops.
+     */
+    async _reset() {
+        this._text = "";
+        await this._discardPendingEndOfTurn();
+    }
+    /**
+     * Drop any end-of-turn conclusion reached so far.
+     *
+     * Runs at a turn boundary, and whenever VAD reports the user speaking
+     * again — which makes an earlier conclusion stale mid-turn.
+     */
+    async _discardPendingEndOfTurn() {
+        this._turnComplete = false;
+        this._transcriptFinalized = false;
+        this._timeoutExpired = false;
+        this._vadStopped = false;
+        if (this._timeoutTask) {
+            await this.taskManager.cancelTask(this._timeoutTask);
+            this._timeoutTask = null;
+        }
+    }
+    /**
+     * Set up the strategy.
+     *
+     * @param setup Configuration object containing setup parameters.
+     */
+    async setup(setup) {
+        await super.setup(setup);
+        this._turnAnalyzer.setSampleRate(setup.audioInSampleRate);
+    }
+    /** Cleanup the strategy. */
+    async cleanup() {
+        await super.cleanup();
+        await this._turnAnalyzer.cleanup();
+        if (this._timeoutTask) {
+            await this.taskManager.cancelTask(this._timeoutTask);
+            this._timeoutTask = null;
+        }
+    }
+    /**
+     * Process an incoming frame to update the turn analyzer and strategy state.
+     *
+     * @param frame The frame to be analyzed.
+     * @returns Always returns CONTINUE so subsequent stop strategies are evaluated.
+     */
+    async processFrame(frame) {
+        await super.processFrame(frame);
+        if (frame instanceof StartFrame) {
+            await this._start(frame);
+        }
+        else if (frame instanceof STTMetadataFrame) {
+            this._sttTimeout = frame.ttfsP99Latency;
+            this._stopSecsWarned = false;
+        }
+        else if (frame instanceof VADUserStartedSpeakingFrame) {
+            await this._handleVadUserStartedSpeaking(frame);
+        }
+        else if (frame instanceof VADUserStoppedSpeakingFrame) {
+            await this._handleVadUserStoppedSpeaking(frame);
+        }
+        else if (frame instanceof InputAudioRawFrame) {
+            await this._handleInputAudio(frame);
+        }
+        else if (frame instanceof TranscriptionFrame) {
+            await this._handleTranscription(frame);
+        }
+        else if (frame instanceof InterimTranscriptionFrame) {
+            // An interim means more transcription is still on the way, so an
+            // earlier finalized transcript no longer covers all of the user's
+            // speech.
+            // Without this, a transcript finalized during a pause too short for
+            // VAD to report a stop (and thus a new start, which is what normally
+            // clears the flag) would leave the flag stale and trigger the turn
+            // immediately at the next VAD stop while the tail of the utterance
+            // is still in flight. This can happen when the STT endpointer
+            // finalizes on silences shorter than the VAD stop_secs — e.g. an
+            // aggressive STT endpoint or a manually raised stop_secs.
+            this._transcriptFinalized = false;
+        }
+        return ProcessFrameResult.CONTINUE;
+    }
+    /** Process the start frame to configure the turn analyzer. */
+    async _start(_frame) {
+        await this.broadcastFrame(SpeechControlParamsFrame, {
+            turnParams: this._turnAnalyzer.params,
+        });
+    }
+    /** Handle input audio to check if the turn is completed. */
+    async _handleInputAudio(frame) {
+        const state = this._turnAnalyzer.appendAudio(frame.audio, this._vadUserSpeaking);
+        // Streaming analyzers (e.g. KrispVivaTurn) detect turn completion
+        // frame-by-frame inside appendAudio, so COMPLETE is returned here
+        // rather than in analyzeEndOfTurn. Batch analyzers (BaseSmartTurn)
+        // return COMPLETE here only on a silence timeout. In either case we
+        // consume and push metrics immediately while they're fresh.
+        if (state === EndOfTurnState.COMPLETE) {
+            const [, prediction] = await this._turnAnalyzer.analyzeEndOfTurn();
+            await this._handlePredictionResult(prediction);
+            this._turnComplete = true;
+            await this._maybeTriggerUserTurnStopped();
+        }
+    }
+    /** Handle when the VAD indicates the user is speaking. */
+    async _handleVadUserStartedSpeaking(frame) {
+        // Sync Smart Turn pre-speech buffering with VAD start delay
+        this._turnAnalyzer.updateVadStartSecs(frame.startSecs);
+        this._vadUserSpeaking = true;
+        await this._discardPendingEndOfTurn();
+    }
+    /** Handle when the VAD indicates the user has stopped speaking. */
+    async _handleVadUserStoppedSpeaking(frame) {
+        this._vadUserSpeaking = false;
+        this._stopSecs = frame.stopSecs;
+        this._vadStopped = true;
+        // The STT p99 budget is measured from when the user actually stopped
+        // speaking, which VAD only reports stop_secs later. Anchoring the
+        // safety net to that absolute deadline keeps the wait fixed no matter
+        // how long end-of-turn analysis (e.g. ML inference) takes before the
+        // timer is armed.
+        const sttDeadline = frame.timestamp - frame.stopSecs + this._sttTimeout;
+        const [state, prediction] = await this._turnAnalyzer.analyzeEndOfTurn();
+        await this._handlePredictionResult(prediction);
+        // The user stopped speaking and the turn is complete, we now need to
+        // wait for transcriptions.
+        this._turnComplete = state === EndOfTurnState.COMPLETE;
+        const timeout = Math.max(0, sttDeadline - time());
+        if (!this._stopSecsWarned) {
+            if (this._stopSecs !== VAD_STOP_SECS) {
+                this._stopSecsWarned = true;
+                logger.warning(`${this}: VAD stop_secs (${this._stopSecs}s) differs from the ` +
+                    `recommended default (${VAD_STOP_SECS}s). Built-in p99 latency ` +
+                    `values assume stop_secs=${VAD_STOP_SECS}. Re-run ` +
+                    "https://github.com/pipecat-ai/stt-benchmark with your settings " +
+                    "and pass the TTFS P99 latency result as ttfs_p99_latency to " +
+                    "your STT service.");
+            }
+            if (this._sttTimeout > 0 && this._stopSecs >= this._sttTimeout) {
+                this._stopSecsWarned = true;
+                logger.warning(`${this}: VAD stop_secs (${this._stopSecs}s) >= STT p99 latency ` +
+                    `(${this._sttTimeout}s). STT wait timeout collapsed to 0s, which ` +
+                    "may cause delayed turn detection specified by the " +
+                    "user_turn_stop_timeout parameter in the LLMUserAggregatorParams.");
+            }
+        }
+        this._timeoutTask = this.taskManager.createTask(() => this._timeoutHandler(timeout), `${this}::_timeout_handler`);
+        // A finalized transcript (or COMPLETE itself, when waitForTranscript
+        // is false) may already have satisfied the trigger conditions while the
+        // analyzer was running. The analyzer's verdict is only known now, so
+        // re-check here rather than wait out the safety-net timer.
+        await this._maybeTriggerUserTurnStopped();
+    }
+    /** Handle user transcription. */
+    async _handleTranscription(frame) {
+        // We don't really care about the content.
+        this._text = frame.text;
+        if (frame.finalized) {
+            this._transcriptFinalized = true;
+            // For finalized transcripts, trigger immediately if turn is complete
+            await this._maybeTriggerUserTurnStopped();
+        }
+        else if (this._timeoutExpired && this._turnComplete) {
+            // The p99 timeout already elapsed without a transcript. Now that
+            // we have one, trigger the turn stop immediately. This handles the
+            // case where the transcript is slower to arrive than the p99 timeout,
+            // trigger the user turn to stop immediately.
+            await this.triggerUserTurnStopped();
+            return;
+        }
+        // Fallback: handle transcripts when no VAD stop was received.
+        // This handles edge cases where transcripts arrive without VAD firing.
+        // In fallback mode, reset timeout on each transcript to wait for inactivity.
+        if (!this._vadUserSpeaking && !this._vadStopped) {
+            // Cancel existing fallback timeout if any
+            if (this._timeoutTask) {
+                await this.taskManager.cancelTask(this._timeoutTask);
+            }
+            // Without VAD/turn analyzer data, assume turn is complete
+            this._turnComplete = true;
+            const timeout = Math.max(0, this._sttTimeout - this._stopSecs);
+            this._timeoutTask = this.taskManager.createTask(() => this._timeoutHandler(timeout), `${this}::_timeout_handler`);
+        }
+    }
+    /** Handle a prediction result event from the turn analyzer. */
+    async _handlePredictionResult(result) {
+        if (result) {
+            await this.pushFrame(new MetricsFrame({ data: [result] }));
+        }
+    }
+    /**
+     * Wait for the timeout then trigger user turn stopped if conditions met.
+     *
+     * @param timeout The timeout in seconds to wait.
+     */
+    async _timeoutHandler(timeout) {
+        try {
+            await sleep(timeout);
+        }
+        catch (e) {
+            if (e instanceof CancelledError)
+                return;
+            throw e;
+        }
+        finally {
+            this._timeoutTask = null;
+        }
+        this._timeoutExpired = true;
+        await this._maybeTriggerUserTurnStopped();
+    }
+    /**
+     * Trigger user turn stopped if conditions are met.
+     *
+     * Conditions:
+     * - Turn analyzer indicates turn is complete
+     * - When `waitForTranscript` is true (default): we have transcription text
+     *   *and* either the safety-net timeout has elapsed or a finalized
+     *   transcript arrived.
+     * - When `waitForTranscript` is false: fire as soon as the turn analyzer
+     *   reports COMPLETE — independent of transcripts.
+     */
+    async _maybeTriggerUserTurnStopped() {
+        if (!this._turnComplete) {
+            return;
+        }
+        if (!this._waitForTranscript) {
+            // Turn-end is driven by the analyzer; transcripts are bookkeeping.
+            if (this._timeoutTask) {
+                await this.taskManager.cancelTask(this._timeoutTask);
+                this._timeoutTask = null;
+            }
+            await this.triggerUserTurnStopped();
+            return;
+        }
+        if (!this._text) {
+            return;
+        }
+        // For finalized transcripts, trigger immediately
+        if (this._transcriptFinalized) {
+            // Cancel any remaining timeout since we're triggering now
+            if (this._timeoutTask) {
+                await this.taskManager.cancelTask(this._timeoutTask);
+                this._timeoutTask = null;
+            }
+            await this.triggerUserTurnStopped();
+            return;
+        }
+        // For non-finalized, only trigger if timeout task has completed
+        if (this._timeoutTask === null) {
+            await this.triggerUserTurnStopped();
+        }
+    }
+}
+//# sourceMappingURL=turn_analyzer_user_turn_stop_strategy.js.map

package/dist/turns/user_turn_completion_mixin.d.ts

@@ -0,0 +1,147 @@
+/**
+ * Mixin for adding turn completion detection to LLM services.
+ *
+ * This mixin enables LLM services to detect and process turn completion
+ * markers (COMPLETE/INCOMPLETE) in LLM responses, allowing for smarter
+ * conversation flow where the LLM can indicate whether the user's input was
+ * complete or if they were interrupted mid-thought.
+ */
+import { type Task } from "@pipedog/asyncio";
+import { type Ctor } from "../utils/mixin.ts";
+export declare const USER_TURN_COMPLETE_MARKER = "\u25CF";
+/** Short wait - user likely continues soon. */
+export declare const USER_TURN_INCOMPLETE_SHORT_MARKER = "\u25D0";
+/** Long wait - user needs more time. */
+export declare const USER_TURN_INCOMPLETE_LONG_MARKER = "\u25CB";
+/**
+ * Completion verdict detected in the current LLM response.
+ *
+ * - `COMPLETE`:   ● detected, response flows through as speech.
+ * - `INCOMPLETE`: ◐/○ detected, response suppressed, timeout armed.
+ */
+export declare const TurnMarker: {
+    readonly COMPLETE: "complete";
+    readonly INCOMPLETE: "incomplete";
+};
+export type TurnMarker = (typeof TurnMarker)[keyof typeof TurnMarker];
+/**
+ * How long to wait before re-prompting after an incomplete turn.
+ *
+ * - `SHORT`: ◐ detected, the user was cut off and likely continues soon.
+ * - `LONG`:  ○ detected, the user needs more time to think.
+ */
+export declare const IncompleteType: {
+    readonly SHORT: "short";
+    readonly LONG: "long";
+};
+export type IncompleteType = (typeof IncompleteType)[keyof typeof IncompleteType];
+export declare const DEFAULT_INCOMPLETE_SHORT_PROMPT: string;
+export declare const DEFAULT_INCOMPLETE_LONG_PROMPT: string;
+export declare const USER_TURN_COMPLETION_INSTRUCTIONS: string;
+export interface UserTurnCompletionConfigOptions {
+    /**
+     * Custom instructions for turn completion. If not provided, the default
+     * instructions are rendered from the configured markers.
+     */
+    instructions?: string | null;
+    /**
+     * Marker the LLM emits when the user turn is complete. It is generated
+     * before any speakable text, so prefer a character that is a single token
+     * in the model's tokenizer.
+     */
+    completeMarker?: string;
+    /** Marker for a turn cut off mid-thought. */
+    incompleteShortMarker?: string;
+    /** Marker for a user who needs more time. */
+    incompleteLongMarker?: string;
+    /** Seconds to wait after a short incomplete before prompting. */
+    incompleteShortTimeout?: number;
+    /** Seconds to wait after a long incomplete before prompting. */
+    incompleteLongTimeout?: number;
+    /** Custom prompt when short timeout expires. */
+    incompleteShortPrompt?: string | null;
+    /** Custom prompt when long timeout expires. */
+    incompleteLongPrompt?: string | null;
+}
+/** Configuration for turn completion behavior. See UserTurnCompletionConfigOptions. */
+export declare class UserTurnCompletionConfig {
+    instructions: string | null;
+    completeMarker: string;
+    incompleteShortMarker: string;
+    incompleteLongMarker: string;
+    incompleteShortTimeout: number;
+    incompleteLongTimeout: number;
+    incompleteShortPrompt: string | null;
+    incompleteLongPrompt: string | null;
+    constructor(options?: UserTurnCompletionConfigOptions);
+    /** The complete, short and long markers, in that order. */
+    get markers(): [string, string, string];
+    /** Turn completion instructions, using default if not set. */
+    get completionInstructions(): string;
+    /** Short incomplete prompt, using default if not set. */
+    get shortPrompt(): string;
+    /** Long incomplete prompt, using default if not set. */
+    get longPrompt(): string;
+}
+/**
+ * Members the turn completion mixin adds to an LLM service.
+ *
+ * The underscore-prefixed members are protected upstream; they are public
+ * here because a mixin's members have to appear on its interface.
+ */
+export interface UserTurnCompletionLLMServiceMixin {
+    _turnTextBuffer: string;
+    _responseRaw: string;
+    _responseMarker: [string, string] | null;
+    _turnMarker: TurnMarker | null;
+    _turnCompletionBroadcasted: boolean;
+    _userTurnCompletionVoiced: boolean;
+    _userSpeaking: boolean;
+    _userTurnCompletionConfig: UserTurnCompletionConfig;
+    _incompleteTimeoutTask: Task<unknown> | null;
+    setUserTurnCompletionConfig(config: UserTurnCompletionConfig | null): void;
+    _broadcastTurnCompletion(): Promise<void>;
+    _startIncompleteTimeout(incompleteType: IncompleteType): Promise<void>;
+    _cancelIncompleteTimeout(): Promise<void>;
+    _incompleteTimeoutHandler(incompleteType: IncompleteType, timeout: number): Promise<void>;
+    _turnReset(): Promise<void>;
+    _pushMarker(marker: string, kind: string, appendToContextImmediately?: boolean): Promise<void>;
+    _reportResponse(): Promise<void>;
+    _pushTurnText(text: string): Promise<void>;
+}
+/**
+ * Mixin that adds turn completion detection to LLM services.
+ *
+ * This mixin provides methods to push LLM text with turn completion
+ * detection. It processes turn completion markers to enable smarter
+ * conversation flow:
+ *
+ * - ● (COMPLETE): Push response normally
+ * - ◐ (INCOMPLETE SHORT): Suppress response, wait 5s, then prompt
+ * - ○ (INCOMPLETE LONG): Suppress response, wait 10s, then prompt
+ *
+ * A ● that arrives while VAD still hears the user is stale (the user resumed
+ * speaking after the inference was triggered) and is handled as ◐.
+ *
+ * When incomplete timeouts expire, the mixin automatically prompts the LLM
+ * with a contextual follow-up message to re-engage the user.
+ *
+ * The LLM service controls when to use turn completion by calling
+ * `_pushTurnText` instead of `pushFrame`.
+ *
+ * The mixin requires that the base class is a FrameProcessor (it relies on
+ * `pushFrame` with FrameProcessor's signature).
+ *
+ * Upstream: `class LLMService(UserTurnCompletionLLMServiceMixin, AIService)`
+ * → `class LLMService extends UserTurnCompletionLLMServiceMixin(AIService)`.
+ *
+ * @example
+ * // With turn completion:
+ * if (this._filterIncompleteUserTurns) {
+ *   await this._pushTurnText(chunk.text);
+ * } else {
+ *   await this.pushFrame(new LLMTextFrame({ text: chunk.text }));
+ * }
+ */
+export declare const UserTurnCompletionLLMServiceMixin: import("../utils/mixin.ts").Mixin<<B extends Ctor>(base: B) => B & Ctor<UserTurnCompletionLLMServiceMixin>, UserTurnCompletionLLMServiceMixin>;
+//# sourceMappingURL=user_turn_completion_mixin.d.ts.map