@pipedog/core 0.0.0 → 0.2.0

package/dist/audio/turn/smart_turn/base_smart_turn.js

@@ -0,0 +1,239 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/audio/turn/smart_turn/base_smart_turn.py
+/**
+ * Smart turn analyzer base class using ML models for end-of-turn detection.
+ *
+ * Provides the base implementation for smart turn analyzers that use machine
+ * learning models to determine when a user has finished speaking, going
+ * beyond simple silence-based detection.
+ */
+import { abortable, monotonic } from "@pipedog/asyncio";
+import { TurnMetricsData } from "../../../metrics/metrics.js";
+import { logger } from "../../../utils/logger.js";
+import { BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState, } from "../base_turn_analyzer.js";
+// Default timing parameters
+export const STOP_SECS = 3;
+export const PRE_SPEECH_MS = 500;
+export const MAX_DURATION_SECONDS = 8; // Max allowed segment duration
+/** Configuration parameters for smart turn analysis. */
+export class SmartTurnParams extends BaseTurnParams {
+    stopSecs;
+    preSpeechMs;
+    maxDurationSecs;
+    constructor(options = {}) {
+        super(options);
+        this.stopSecs = options.stopSecs ?? STOP_SECS;
+        this.preSpeechMs = options.preSpeechMs ?? PRE_SPEECH_MS;
+        this.maxDurationSecs = options.maxDurationSecs ?? MAX_DURATION_SECONDS;
+    }
+    toString() {
+        return (`stop_secs=${this.stopSecs} pre_speech_ms=${this.preSpeechMs} ` +
+            `max_duration_secs=${this.maxDurationSecs}`);
+    }
+}
+/** Exception raised when smart turn analysis times out. */
+export class SmartTurnTimeoutException extends Error {
+    name = "SmartTurnTimeoutException";
+}
+function endOfTurnStateName(state) {
+    return state === EndOfTurnState.COMPLETE
+        ? "EndOfTurnState.COMPLETE"
+        : "EndOfTurnState.INCOMPLETE";
+}
+/**
+ * Base class for smart turn analyzers using ML models.
+ *
+ * Provides common functionality for smart turn detection including audio
+ * buffering, speech tracking, and ML model integration. Subclasses must
+ * implement the specific model prediction logic.
+ *
+ * Upstream runs the model on a dedicated thread; there are no threads on every
+ * runtime we target, so prediction is an async method run inline.
+ */
+export class BaseSmartTurn extends BaseTurnAnalyzer {
+    _params;
+    // Configuration
+    _stopMs;
+    // Inference state
+    _audioBuffer = [];
+    _speechTriggered = false;
+    _silenceMs = 0;
+    _speechStartTime = 0;
+    _vadStartSecs = 0.0;
+    constructor(options = {}) {
+        super(options);
+        this._params = options.params ?? new SmartTurnParams();
+        this._stopMs = this._params.stopSecs * 1000; // silence threshold in ms
+    }
+    /** Whether speech has been detected and turn analysis is active. */
+    get speechTriggered() {
+        return this._speechTriggered;
+    }
+    /** The current smart turn configuration parameters. */
+    get params() {
+        return this._params;
+    }
+    /**
+     * Append audio data for turn analysis.
+     *
+     * @param buffer Raw audio data bytes to append for analysis.
+     * @param isSpeech Whether the audio buffer contains detected speech.
+     * @returns Current end-of-turn state after processing the audio.
+     */
+    appendAudio(buffer, isSpeech) {
+        // Append the raw int16 PCM view to the buffer. Conversion to float32 is
+        // deferred until _processSpeechSegment, where it runs once per turn on
+        // the concatenated segment instead of once per ~20 ms audio frame. The
+        // samples are copied so later mutation of the caller's buffer can't leak
+        // in (Python relies on `bytes` being immutable).
+        const audioInt16 = new Int16Array(buffer.slice(0, buffer.length - (buffer.length % 2)).buffer);
+        this._audioBuffer.push([monotonic(), audioInt16]);
+        let state = EndOfTurnState.INCOMPLETE;
+        if (isSpeech) {
+            // Reset silence tracking on speech
+            this._silenceMs = 0;
+            this._speechTriggered = true;
+            if (this._speechStartTime === 0) {
+                this._speechStartTime = monotonic();
+            }
+        }
+        else {
+            if (this._speechTriggered) {
+                const chunkDurationMs = audioInt16.length / (this._sampleRate / 1000);
+                this._silenceMs += chunkDurationMs;
+                // If silence exceeds threshold, mark end of turn
+                if (this._silenceMs >= this._stopMs) {
+                    logger.debug(`End of Turn complete due to stop_secs. Silence in ms: ${this._silenceMs}`);
+                    state = EndOfTurnState.COMPLETE;
+                    this._clear(state);
+                }
+            }
+            else {
+                // Trim buffer to prevent unbounded growth before speech
+                const maxBufferTime = this._params.preSpeechMs / 1000 + this._params.stopSecs + this._params.maxDurationSecs;
+                while (this._audioBuffer.length > 0 &&
+                    this._audioBuffer[0][0] < monotonic() - maxBufferTime) {
+                    this._audioBuffer.shift();
+                }
+            }
+        }
+        return state;
+    }
+    /**
+     * Analyze the current audio state to determine if turn has ended.
+     *
+     * @returns The end-of-turn state and optional metrics data from the ML
+     *   model analysis.
+     */
+    async analyzeEndOfTurn() {
+        const [state, result] = await this._processSpeechSegment(this._audioBuffer);
+        if (state === EndOfTurnState.COMPLETE) {
+            this._clear(state);
+        }
+        logger.debug(`End of Turn result: ${endOfTurnStateName(state)}`);
+        return [state, result];
+    }
+    /** Store the new vadStartSecs value. */
+    updateVadStartSecs(vadStartSecs) {
+        this._vadStartSecs = vadStartSecs;
+    }
+    /** Reset the turn analyzer to its initial state. */
+    clear() {
+        this._clear(EndOfTurnState.COMPLETE);
+    }
+    /**
+     * Release the analyzer's resources.
+     *
+     * The analyzer stays usable: cleaning up is not only teardown here, since a
+     * turn strategy is cleaned up and re-applied whenever the strategies are
+     * updated. (Upstream stops its model thread here; there is none in the
+     * port.)
+     */
+    async cleanup() {
+        await super.cleanup();
+    }
+    /** Clear internal state based on turn completion status. */
+    _clear(turnState) {
+        // If the state is still incomplete, keep the _speechTriggered as true
+        this._speechTriggered = turnState === EndOfTurnState.INCOMPLETE;
+        this._audioBuffer = [];
+        this._speechStartTime = 0;
+        this._silenceMs = 0;
+    }
+    /** Process accumulated audio segment using ML model. */
+    async _processSpeechSegment(audioBuffer) {
+        let state = EndOfTurnState.INCOMPLETE;
+        if (audioBuffer.length === 0) {
+            return [state, null];
+        }
+        // Extract recent audio segment for prediction
+        const effectivePreSpeechMs = this._params.preSpeechMs + this._vadStartSecs * 1000;
+        const startTime = this._speechStartTime - effectivePreSpeechMs / 1000;
+        let startIndex = 0;
+        for (let i = 0; i < audioBuffer.length; i++) {
+            const [t] = audioBuffer[i];
+            if (t >= startTime) {
+                startIndex = i;
+                break;
+            }
+        }
+        const endIndex = audioBuffer.length - 1;
+        // Extract the audio segment. The buffer holds int16 PCM; defer the
+        // float32 conversion until after concatenation so it runs once per turn
+        // instead of once per appended frame.
+        const segmentChunks = audioBuffer.slice(startIndex, endIndex + 1).map(([, chunk]) => chunk);
+        const totalLength = segmentChunks.reduce((n, chunk) => n + chunk.length, 0);
+        let segmentAudio = new Float32Array(totalLength);
+        let offset = 0;
+        for (const chunk of segmentChunks) {
+            for (let i = 0; i < chunk.length; i++) {
+                segmentAudio[offset + i] = chunk[i] / 32768.0;
+            }
+            offset += chunk.length;
+        }
+        // Limit maximum duration
+        const maxSamples = Math.trunc(this._params.maxDurationSecs * this.sampleRate);
+        if (segmentAudio.length > maxSamples) {
+            // Keep the last maxSamples samples, discarding the earlier part.
+            segmentAudio = segmentAudio.slice(segmentAudio.length - maxSamples);
+        }
+        let resultData = null;
+        if (segmentAudio.length > 0) {
+            const start = performance.now();
+            try {
+                // Upstream awaits the model on an executor, which is a cancellation
+                // point; `abortable` keeps it one here.
+                const result = await abortable(this._predictEndpoint(segmentAudio));
+                state = result.prediction === 1 ? EndOfTurnState.COMPLETE : EndOfTurnState.INCOMPLETE;
+                const end = performance.now();
+                // Calculate processing time
+                const e2eProcessingTimeMs = end - start;
+                // Prepare the result data
+                resultData = new TurnMetricsData({
+                    processor: "BaseSmartTurn",
+                    isComplete: result.prediction === 1,
+                    probability: result.probability,
+                    e2eProcessingTimeMs,
+                });
+                logger.trace(`Prediction: ${resultData.isComplete ? "Complete" : "Incomplete"}`);
+                logger.trace(`Probability of complete: ${resultData.probability.toFixed(4)}`);
+                logger.trace(`E2E processing time: ${resultData.e2eProcessingTimeMs.toFixed(2)}ms`);
+            }
+            catch (e) {
+                if (!(e instanceof SmartTurnTimeoutException))
+                    throw e;
+                logger.debug(`End of Turn complete due to stop_secs. Silence in ms: ${this._silenceMs}`);
+                state = EndOfTurnState.COMPLETE;
+            }
+        }
+        else {
+            logger.trace(`params: ${this._params}, stop_ms: ${this._stopMs}`);
+            logger.trace("Captured empty audio segment, skipping prediction.");
+        }
+        return [state, resultData];
+    }
+}
+//# sourceMappingURL=base_smart_turn.js.map

package/dist/audio/turn/smart_turn/local_smart_turn_v3.d.ts

@@ -0,0 +1,76 @@
+import { BaseSmartTurn, type BaseSmartTurnOptions, type SmartTurnPrediction } from "./base_smart_turn.ts";
+/**
+ * The Whisper-based ONNX model expects 16 kHz audio input.
+ *
+ * @internal
+ */
+export declare const MODEL_SAMPLE_RATE = 16000;
+/** Session options matching upstream's `onnxruntime.SessionOptions`. */
+export interface SmartTurnSessionOptions {
+    executionMode: "sequential";
+    interOpNumThreads: number;
+    intraOpNumThreads: number;
+    graphOptimizationLevel: "all";
+}
+/** A tensor fed to the model: row-major data plus its dimensions. */
+export interface SmartTurnTensor {
+    data: Float32Array;
+    dims: number[];
+}
+/** The part of an ONNX inference session the analyzer uses. */
+export interface SmartTurnInferenceSession {
+    /**
+     * Run the model and return its first output as a flat array (the sigmoid
+     * probability of turn completion is element 0).
+     */
+    run(feeds: {
+        input_features: SmartTurnTensor;
+    }): Promise<ArrayLike<number>>;
+}
+/** Creates an inference session for a model file/URL. */
+export type SmartTurnSessionFactory = (modelPath: string, options: SmartTurnSessionOptions) => Promise<SmartTurnInferenceSession>;
+export interface LocalSmartTurnAnalyzerV3Options extends BaseSmartTurnOptions {
+    /**
+     * Path/URL to the ONNX model file. If not set, the upstream bundled model
+     * name (smart-turn-v3.2-cpu.onnx) is used.
+     */
+    smartTurnModelPath?: string | null;
+    /** The number of CPUs to use for inference. Defaults to 1. */
+    cpuCount?: number;
+    /** Creates the ONNX inference session (injected runtime). */
+    createSession?: SmartTurnSessionFactory;
+}
+/**
+ * Local turn analyzer using the smart-turn-v3 ONNX model.
+ *
+ * Provides end-of-turn detection using a locally-stored ONNX model, enabling
+ * offline operation without network dependencies.
+ */
+export declare class LocalSmartTurnAnalyzerV3 extends BaseSmartTurn {
+    protected _logData: boolean;
+    protected _smartTurnModelPath: string;
+    protected _sessionOptions: SmartTurnSessionOptions;
+    protected _createSession: SmartTurnSessionFactory;
+    protected _session: Promise<SmartTurnInferenceSession>;
+    constructor(options?: LocalSmartTurnAnalyzerV3Options);
+    /**
+     * Write audio data to a WAV file.
+     *
+     * STUB: core has no filesystem access; logs a warning instead.
+     *
+     * @param audioArray The audio data (float32, normalized to [-1, 1]).
+     * @param sampleRate The sample rate of the audio data.
+     * @param suffix Optional suffix to append to the filename.
+     */
+    protected _writeAudioToWav(_audioArray: Float32Array, _sampleRate?: number, suffix?: string): void;
+    /**
+     * Resample audio to the model's expected sample rate (16 kHz).
+     *
+     * @param audioArray Audio data as a float32 array.
+     * @returns Resampled audio array at 16 kHz.
+     */
+    protected _resampleToModelRate(audioArray: Float32Array): Float32Array;
+    /** Predict end-of-turn using local ONNX model. */
+    protected _predictEndpoint(audioArray: Float32Array): Promise<SmartTurnPrediction>;
+}
+//# sourceMappingURL=local_smart_turn_v3.d.ts.map

package/dist/audio/turn/smart_turn/local_smart_turn_v3.js

@@ -0,0 +1,151 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py
+/**
+ * Local turn analyzer for on-device ML inference using the smart-turn-v3 model.
+ *
+ * Provides a smart turn analyzer that uses an ONNX model for local
+ * end-of-turn detection without requiring network connectivity.
+ *
+ * PORTING NOTES:
+ * - There is no ONNX runtime in core. The analyzer takes an injected
+ *   `createSession`; `createLocalSmartTurnAnalyzerV3()` in
+ *   `@pipedog/audio-smart-turn` wires it to `@pipedog/onnx` (onnxruntime-node
+ *   or onnxruntime-web) and the model. Without one, the default factory throws.
+ * - The model is not in core; `smartTurnModelPath` defaults to the upstream
+ *   file name and is handed to `createSession` as is (`@pipedog/audio-smart-turn`
+ *   ships the file).
+ * - Resampling to 16 kHz uses the resampler engine (`audio/resamplers/engine`)
+ *   instead of soxr. `_predictEndpoint` makes sure it is loaded before the
+ *   (synchronous) `_resampleToModelRate` runs.
+ * - `PIPECAT_SMART_TURN_LOG_DATA` WAV dumps need a filesystem, which core
+ *   doesn't use; the dump is a stub that logs a warning.
+ */
+import { envTruthy } from "../../../utils/env.js";
+import { logger } from "../../../utils/logger.js";
+import { getResamplerEngine } from "../../resamplers/engine.js";
+import { soxrResample } from "../../resamplers/soxr_resampler.js";
+import { computeWhisperLogMelFeatures, WHISPER_NUM_FRAMES, WHISPER_NUM_MELS, } from "./_whisper_features.js";
+import { BaseSmartTurn, } from "./base_smart_turn.js";
+/**
+ * The Whisper-based ONNX model expects 16 kHz audio input.
+ *
+ * @internal
+ */
+export const MODEL_SAMPLE_RATE = 16000;
+const notPortedSessionFactory = async () => {
+    throw new Error("LocalSmartTurnAnalyzerV3: core has no ONNX runtime; use createLocalSmartTurnAnalyzerV3() " +
+        "from @pipedog/audio-smart-turn, or pass `createSession`");
+};
+function truncateAudioToLastNSeconds(audioArray, nSeconds = 8, sampleRate = MODEL_SAMPLE_RATE) {
+    // Truncate audio to last n seconds or pad with zeros to meet n seconds.
+    const maxSamples = nSeconds * sampleRate;
+    if (audioArray.length > maxSamples) {
+        return audioArray.slice(audioArray.length - maxSamples);
+    }
+    if (audioArray.length < maxSamples) {
+        // Pad with zeros at the beginning
+        const padded = new Float32Array(maxSamples);
+        padded.set(audioArray, maxSamples - audioArray.length);
+        return padded;
+    }
+    return audioArray;
+}
+/**
+ * Local turn analyzer using the smart-turn-v3 ONNX model.
+ *
+ * Provides end-of-turn detection using a locally-stored ONNX model, enabling
+ * offline operation without network dependencies.
+ */
+export class LocalSmartTurnAnalyzerV3 extends BaseSmartTurn {
+    _logData;
+    _smartTurnModelPath;
+    _sessionOptions;
+    _createSession;
+    _session;
+    constructor(options = {}) {
+        super(options);
+        this._logData = envTruthy("PIPECAT_SMART_TURN_LOG_DATA", false);
+        // Upstream loads the model bundled with the package by default.
+        this._smartTurnModelPath = options.smartTurnModelPath || "smart-turn-v3.2-cpu.onnx";
+        logger.debug(`Loading Local Smart Turn v3.x model from ${this._smartTurnModelPath}...`);
+        this._sessionOptions = {
+            executionMode: "sequential",
+            interOpNumThreads: 1,
+            intraOpNumThreads: options.cpuCount ?? 1,
+            graphOptimizationLevel: "all",
+        };
+        this._createSession = options.createSession ?? notPortedSessionFactory;
+        // Session creation is async in JS; it starts now and is awaited on first
+        // prediction.
+        this._session = this._createSession(this._smartTurnModelPath, this._sessionOptions).then((session) => {
+            logger.debug("Loaded Local Smart Turn v3.x");
+            return session;
+        });
+        // Errors surface on first use; don't report an unhandled rejection now.
+        this._session.catch(() => { });
+    }
+    /**
+     * Write audio data to a WAV file.
+     *
+     * STUB: core has no filesystem access; logs a warning instead.
+     *
+     * @param audioArray The audio data (float32, normalized to [-1, 1]).
+     * @param sampleRate The sample rate of the audio data.
+     * @param suffix Optional suffix to append to the filename.
+     */
+    _writeAudioToWav(_audioArray, _sampleRate = MODEL_SAMPLE_RATE, suffix = "") {
+        logger.warning(`Smart turn audio logging (${suffix}) is not ported yet: no filesystem access in core`);
+    }
+    /**
+     * Resample audio to the model's expected sample rate (16 kHz).
+     *
+     * @param audioArray Audio data as a float32 array.
+     * @returns Resampled audio array at 16 kHz.
+     */
+    _resampleToModelRate(audioArray) {
+        const actualRate = this._sampleRate || MODEL_SAMPLE_RATE;
+        if (actualRate === MODEL_SAMPLE_RATE) {
+            return audioArray;
+        }
+        return soxrResample(audioArray, actualRate, MODEL_SAMPLE_RATE, "HQ");
+    }
+    /** Predict end-of-turn using local ONNX model. */
+    async _predictEndpoint(audioArray) {
+        const audioForLogging = audioArray;
+        const actualRate = this._sampleRate || MODEL_SAMPLE_RATE;
+        // Resample to 16 kHz if the pipeline uses a different sample rate
+        if (actualRate !== MODEL_SAMPLE_RATE) {
+            // soxr is synchronous; our engine may still need loading.
+            await getResamplerEngine();
+        }
+        audioArray = this._resampleToModelRate(audioArray);
+        // Truncate to 8 seconds (keeping the end) or pad to 8 seconds
+        audioArray = truncateAudioToLastNSeconds(audioArray, 8);
+        // Compute Whisper-style log-mel features (vendored implementation).
+        const logMel = computeWhisperLogMelFeatures(audioArray, { doNormalize: true });
+        // Add batch dimension
+        const inputFeatures = {
+            data: logMel,
+            dims: [1, WHISPER_NUM_MELS, WHISPER_NUM_FRAMES],
+        };
+        // Run ONNX inference
+        const session = await this._session;
+        const output = await session.run({ input_features: inputFeatures });
+        // Extract probability (ONNX model returns sigmoid probabilities)
+        const probability = Number(output[0]);
+        // Make prediction (1 for Complete, 0 for Incomplete)
+        const prediction = probability > 0.5 ? 1 : 0;
+        if (this._logData) {
+            const suffix = prediction === 1 ? "_complete" : "_incomplete";
+            this._writeAudioToWav(audioForLogging, actualRate, suffix);
+        }
+        return {
+            prediction,
+            probability,
+        };
+    }
+}
+//# sourceMappingURL=local_smart_turn_v3.js.map