@pipedog/core 0.0.0 → 0.1.0

package/dist/audio/vad/vad_analyzer.d.ts

@@ -0,0 +1,126 @@
+import { AudioVolumeTracker } from "../volume.ts";
+export declare const VAD_CONFIDENCE = 0.7;
+export declare const VAD_START_SECS = 0.2;
+export declare const VAD_STOP_SECS = 0.2;
+export declare const VAD_MIN_VOLUME = 0.6;
+/**
+ * Voice Activity Detection states.
+ *
+ * - QUIET: no voice activity detected.
+ * - STARTING: voice activity beginning, transitioning from quiet.
+ * - SPEAKING: active voice detected and confirmed.
+ * - STOPPING: voice activity ending, transitioning to quiet.
+ */
+export declare const VADState: {
+    readonly QUIET: 1;
+    readonly STARTING: 2;
+    readonly SPEAKING: 3;
+    readonly STOPPING: 4;
+};
+export type VADState = (typeof VADState)[keyof typeof VADState];
+export interface VADParamsOptions {
+    /** Minimum confidence threshold for voice detection. */
+    confidence?: number;
+    /** Duration to wait before confirming voice start. */
+    startSecs?: number;
+    /** Duration to wait before confirming voice stop. */
+    stopSecs?: number;
+    /** Minimum audio volume threshold for voice detection. */
+    minVolume?: number;
+}
+/** Configuration parameters for Voice Activity Detection. */
+export declare class VADParams {
+    confidence: number;
+    startSecs: number;
+    stopSecs: number;
+    minVolume: number;
+    constructor(options?: VADParamsOptions);
+    toString(): string;
+}
+export interface VADAnalyzerOptions {
+    /** Audio sample rate in Hz. If null, will be set later. */
+    sampleRate?: number | null;
+    /** VAD parameters for detection configuration. */
+    params?: VADParams | null;
+}
+/**
+ * Abstract base class for Voice Activity Detection analyzers.
+ *
+ * Provides the framework for implementing VAD analysis with configurable
+ * parameters, state management, and audio processing capabilities.
+ * Subclasses must implement the core voice confidence calculation.
+ *
+ * Upstream runs the model on a single-thread executor; there are no threads
+ * on every runtime we target, so the analysis runs inline.
+ */
+export declare abstract class VADAnalyzer {
+    protected _initSampleRate: number | null;
+    protected _sampleRate: number;
+    protected _params: VADParams;
+    protected _numChannels: number;
+    protected _vadBuffer: Uint8Array;
+    protected _volumeTracker: AudioVolumeTracker;
+    protected _smoothingFactor: number;
+    protected _prevVolume: number;
+    protected _vadFrames: number;
+    protected _vadFramesNumBytes: number;
+    protected _vadStartFrames: number;
+    protected _vadStopFrames: number;
+    protected _vadStartingCount: number;
+    protected _vadStoppingCount: number;
+    protected _vadState: VADState;
+    constructor(options?: VADAnalyzerOptions);
+    /** Current audio sample rate in Hz. */
+    get sampleRate(): number;
+    /** Number of audio channels (always 1 for mono). */
+    get numChannels(): number;
+    /** Current VAD configuration parameters. */
+    get params(): VADParams;
+    /**
+     * Get the number of audio frames required for analysis.
+     *
+     * @returns Number of frames needed for VAD processing.
+     */
+    abstract numFramesRequired(): number;
+    /**
+     * Calculate voice activity confidence for the given audio buffer.
+     *
+     * @param buffer Audio buffer to analyze.
+     * @returns Voice confidence score between 0.0 and 1.0.
+     */
+    abstract voiceConfidence(buffer: Uint8Array): number;
+    /**
+     * Set the sample rate for audio processing.
+     *
+     * @param sampleRate Audio sample rate in Hz.
+     */
+    setSampleRate(sampleRate: number): void;
+    /**
+     * Set VAD parameters and recalculate internal values.
+     *
+     * @param params VAD parameters for detection configuration.
+     */
+    setParams(params: VADParams): void;
+    /** Calculate smoothed audio volume using exponential smoothing. */
+    protected _getSmoothedVolume(audio: Uint8Array): number;
+    /**
+     * Analyze audio buffer and return current VAD state.
+     *
+     * Processes incoming audio data, maintains internal state, and determines
+     * voice activity status based on confidence and volume thresholds.
+     *
+     * @param buffer Audio buffer to analyze.
+     * @returns Current VAD state after processing the buffer.
+     */
+    analyzeAudio(buffer: Uint8Array): Promise<VADState>;
+    /** Analyze audio buffer and return current VAD state. */
+    protected _runAnalyzer(buffer: Uint8Array): VADState;
+    /**
+     * Release the analyzer's resources.
+     *
+     * This method should be called when the object is no longer needed.
+     * (Upstream shuts down its model thread here; there is none in the port.)
+     */
+    cleanup(): Promise<void>;
+}
+//# sourceMappingURL=vad_analyzer.d.ts.map

package/dist/audio/vad/vad_analyzer.js

@@ -0,0 +1,211 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/audio/vad/vad_analyzer.py
+/**
+ * Voice Activity Detection (VAD) analyzer base classes and utilities.
+ *
+ * Provides the abstract base class for VAD analyzers and associated data
+ * structures for voice activity detection in audio streams. Includes state
+ * management, parameter configuration, and audio analysis framework.
+ */
+import { logger } from "../../utils/logger.js";
+import { expSmoothing } from "../utils.js";
+import { AudioVolumeTracker } from "../volume.js";
+export const VAD_CONFIDENCE = 0.7;
+export const VAD_START_SECS = 0.2;
+export const VAD_STOP_SECS = 0.2;
+export const VAD_MIN_VOLUME = 0.6;
+/**
+ * Voice Activity Detection states.
+ *
+ * - QUIET: no voice activity detected.
+ * - STARTING: voice activity beginning, transitioning from quiet.
+ * - SPEAKING: active voice detected and confirmed.
+ * - STOPPING: voice activity ending, transitioning to quiet.
+ */
+export const VADState = {
+    QUIET: 1,
+    STARTING: 2,
+    SPEAKING: 3,
+    STOPPING: 4,
+};
+/** Configuration parameters for Voice Activity Detection. */
+export class VADParams {
+    confidence;
+    startSecs;
+    stopSecs;
+    minVolume;
+    constructor(options = {}) {
+        this.confidence = options.confidence ?? VAD_CONFIDENCE;
+        this.startSecs = options.startSecs ?? VAD_START_SECS;
+        this.stopSecs = options.stopSecs ?? VAD_STOP_SECS;
+        this.minVolume = options.minVolume ?? VAD_MIN_VOLUME;
+    }
+    toString() {
+        return (`confidence=${this.confidence} start_secs=${this.startSecs} ` +
+            `stop_secs=${this.stopSecs} min_volume=${this.minVolume}`);
+    }
+}
+/**
+ * Abstract base class for Voice Activity Detection analyzers.
+ *
+ * Provides the framework for implementing VAD analysis with configurable
+ * parameters, state management, and audio processing capabilities.
+ * Subclasses must implement the core voice confidence calculation.
+ *
+ * Upstream runs the model on a single-thread executor; there are no threads
+ * on every runtime we target, so the analysis runs inline.
+ */
+export class VADAnalyzer {
+    _initSampleRate;
+    _sampleRate = 0;
+    _params;
+    _numChannels = 1;
+    _vadBuffer = new Uint8Array(0);
+    _volumeTracker = new AudioVolumeTracker();
+    // Volume exponential smoothing
+    _smoothingFactor = 0.2;
+    _prevVolume = 0.0;
+    _vadFrames = 0;
+    _vadFramesNumBytes = 0;
+    _vadStartFrames = 0;
+    _vadStopFrames = 0;
+    _vadStartingCount = 0;
+    _vadStoppingCount = 0;
+    _vadState = VADState.QUIET;
+    constructor(options = {}) {
+        this._initSampleRate = options.sampleRate ?? null;
+        this._params = options.params ?? new VADParams();
+    }
+    /** Current audio sample rate in Hz. */
+    get sampleRate() {
+        return this._sampleRate;
+    }
+    /** Number of audio channels (always 1 for mono). */
+    get numChannels() {
+        return this._numChannels;
+    }
+    /** Current VAD configuration parameters. */
+    get params() {
+        return this._params;
+    }
+    /**
+     * Set the sample rate for audio processing.
+     *
+     * @param sampleRate Audio sample rate in Hz.
+     */
+    setSampleRate(sampleRate) {
+        this._sampleRate = this._initSampleRate || sampleRate;
+        this.setParams(this._params);
+    }
+    /**
+     * Set VAD parameters and recalculate internal values.
+     *
+     * @param params VAD parameters for detection configuration.
+     */
+    setParams(params) {
+        logger.debug(`Setting VAD params to: ${params}`);
+        this._params = params;
+        this._vadFrames = this.numFramesRequired();
+        this._vadFramesNumBytes = this._vadFrames * this._numChannels * 2;
+        const vadFramesPerSec = this._vadFrames / this.sampleRate;
+        this._vadStartFrames = pyRound(this._params.startSecs / vadFramesPerSec);
+        this._vadStopFrames = pyRound(this._params.stopSecs / vadFramesPerSec);
+        // VAD state resets, but volume state doesn't: the rolling window and its
+        // smoothing follow the audio stream, which is continuous across parameter
+        // changes.
+        this._vadStartingCount = 0;
+        this._vadStoppingCount = 0;
+        this._vadState = VADState.QUIET;
+    }
+    /** Calculate smoothed audio volume using exponential smoothing. */
+    _getSmoothedVolume(audio) {
+        this._volumeTracker.update(audio, this.sampleRate);
+        return expSmoothing(this._volumeTracker.volume, this._prevVolume, this._smoothingFactor);
+    }
+    /**
+     * Analyze audio buffer and return current VAD state.
+     *
+     * Processes incoming audio data, maintains internal state, and determines
+     * voice activity status based on confidence and volume thresholds.
+     *
+     * @param buffer Audio buffer to analyze.
+     * @returns Current VAD state after processing the buffer.
+     */
+    async analyzeAudio(buffer) {
+        return this._runAnalyzer(buffer);
+    }
+    /** Analyze audio buffer and return current VAD state. */
+    _runAnalyzer(buffer) {
+        const combined = new Uint8Array(this._vadBuffer.length + buffer.length);
+        combined.set(this._vadBuffer, 0);
+        combined.set(buffer, this._vadBuffer.length);
+        this._vadBuffer = combined;
+        const numRequiredBytes = this._vadFramesNumBytes;
+        if (this._vadBuffer.length < numRequiredBytes) {
+            return this._vadState;
+        }
+        while (this._vadBuffer.length >= numRequiredBytes) {
+            const audioFrames = this._vadBuffer.slice(0, numRequiredBytes);
+            this._vadBuffer = this._vadBuffer.subarray(numRequiredBytes);
+            const confidence = this.voiceConfidence(audioFrames);
+            const volume = this._getSmoothedVolume(audioFrames);
+            this._prevVolume = volume;
+            const speaking = confidence >= this._params.confidence && volume >= this._params.minVolume;
+            if (speaking) {
+                switch (this._vadState) {
+                    case VADState.QUIET:
+                        this._vadState = VADState.STARTING;
+                        this._vadStartingCount = 1;
+                        break;
+                    case VADState.STARTING:
+                        this._vadStartingCount += 1;
+                        break;
+                    case VADState.STOPPING:
+                        this._vadState = VADState.SPEAKING;
+                        this._vadStoppingCount = 0;
+                        break;
+                }
+            }
+            else {
+                switch (this._vadState) {
+                    case VADState.STARTING:
+                        this._vadState = VADState.QUIET;
+                        this._vadStartingCount = 0;
+                        break;
+                    case VADState.SPEAKING:
+                        this._vadState = VADState.STOPPING;
+                        this._vadStoppingCount = 1;
+                        break;
+                    case VADState.STOPPING:
+                        this._vadStoppingCount += 1;
+                        break;
+                }
+            }
+        }
+        if (this._vadState === VADState.STARTING && this._vadStartingCount >= this._vadStartFrames) {
+            this._vadState = VADState.SPEAKING;
+            this._vadStartingCount = 0;
+        }
+        if (this._vadState === VADState.STOPPING && this._vadStoppingCount >= this._vadStopFrames) {
+            this._vadState = VADState.QUIET;
+            this._vadStoppingCount = 0;
+        }
+        return this._vadState;
+    }
+    /**
+     * Release the analyzer's resources.
+     *
+     * This method should be called when the object is no longer needed.
+     * (Upstream shuts down its model thread here; there is none in the port.)
+     */
+    async cleanup() { }
+}
+/** Python's round(): half to even. */
+function pyRound(x) {
+    const r = Math.round(x);
+    return Math.abs(x % 1) === 0.5 && r % 2 !== 0 ? r - 1 : r;
+}
+//# sourceMappingURL=vad_analyzer.js.map

package/dist/audio/vad/vad_controller.d.ts

@@ -0,0 +1,143 @@
+/**
+ * Voice Activity Detection controller for managing speech state transitions.
+ *
+ * Provides a controller that wraps a VADAnalyzer to track speech state and
+ * emit events when speech starts, stops, or is actively detected.
+ */
+import { type Task } from "@pipedog/asyncio";
+import { type Frame, InputAudioRawFrame } from "../../frames/frames.ts";
+import { FrameDirection, type FrameProcessorSetup } from "../../processors/frame_processor.ts";
+import { BaseObject } from "../../utils/base_object.ts";
+import { type VADAnalyzer, VADState } from "./vad_analyzer.ts";
+export interface VADControllerOptions {
+    /**
+     * Minimum interval in seconds between `on_speech_activity` events.
+     * Defaults to 0.2.
+     */
+    speechActivityPeriod?: number;
+    /**
+     * Timeout in seconds to force speech stop when no audio frames are
+     * received while in SPEAKING state. This handles cases like mic mute
+     * mid-speech. Set to 0 to disable. Defaults to 1.0.
+     */
+    audioIdleTimeout?: number;
+}
+/**
+ * Manages voice activity detection state and emits speech events.
+ *
+ * Wraps a `VADAnalyzer` to process audio and trigger events based on speech
+ * state transitions. Tracks whether the user is speaking, quiet, or
+ * transitioning between states.
+ *
+ * Event handlers available:
+ *
+ * - on_speech_started: Called when speech begins.
+ * - on_speech_stopped: Called when speech ends, including forced stop when
+ *   the audio stream goes idle (no frames received while speaking).
+ * - on_speech_activity: Called periodically while speech is detected.
+ * - on_push_frame: Called when the controller wants to push a frame.
+ * - on_broadcast_frame: Called when the controller wants to broadcast a frame.
+ *
+ * @example
+ * vadController.eventHandler("on_speech_started", async (controller) => { ... });
+ * vadController.eventHandler("on_speech_stopped", async (controller) => { ... });
+ * vadController.eventHandler("on_speech_activity", async (controller) => { ... });
+ * vadController.eventHandler("on_push_frame", async (controller, frame, direction) => { ... });
+ * vadController.eventHandler("on_broadcast_frame", async (controller, frameCls, options) => { ... });
+ */
+export declare class VADController extends BaseObject {
+    protected _vadAnalyzer: VADAnalyzer;
+    protected _vadState: VADState;
+    protected _speechActivityTime: number;
+    protected _speechActivityPeriod: number;
+    protected _lastAudioTime: number;
+    protected _audioIdleTimeout: number;
+    protected _audioIdleTask: Task<unknown> | null;
+    /**
+     * Initialize the VAD controller.
+     *
+     * @param vadAnalyzer The `VADAnalyzer` instance for processing audio.
+     * @param options Speech activity period and audio idle timeout.
+     */
+    constructor(vadAnalyzer: VADAnalyzer, options?: VADControllerOptions);
+    /**
+     * Set up the controller.
+     *
+     * @param setup Configuration object containing setup parameters.
+     */
+    setup(setup: FrameProcessorSetup): Promise<void>;
+    /**
+     * Process a frame and handle VAD-related events.
+     *
+     * Handles `InputAudioRawFrame` to analyze audio for voice activity, and
+     * `VADParamsUpdateFrame` to reconfigure the analyzer.
+     *
+     * @param frame The frame to process.
+     */
+    processFrame(frame: Frame): Promise<void>;
+    /**
+     * Announce the analyzer's params and start the idle timer.
+     *
+     * Paired with `stop()`.
+     */
+    start(): Promise<void>;
+    /**
+     * Stop the idle timer, leaving the analyzer alone.
+     *
+     * Called at session end so the timer can't report silence that only means
+     * the session is over. The analyzer may be shared, so releasing it waits
+     * for `cleanup()`.
+     */
+    stop(): Promise<void>;
+    /**
+     * Clean up resources.
+     *
+     * This method should be called when the object is no longer needed. It
+     * waits for all currently executing event handler tasks to finish before
+     * returning.
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Process an audio chunk and emit speech events as needed.
+     *
+     * Analyzes the audio for voice activity and triggers `on_speech_started`,
+     * `on_speech_stopped`, or `on_speech_activity` events based on state
+     * changes.
+     *
+     * @param frame Audio frame to process.
+     */
+    protected _handleAudio(frame: InputAudioRawFrame): Promise<void>;
+    /** Handle Voice Activity Detection results and trigger appropriate events. */
+    protected _handleVad(audio: Uint8Array, vadState: VADState): Promise<VADState>;
+    /**
+     * Monitor for an idle audio stream while in SPEAKING state.
+     *
+     * When no audio frames arrive for `audioIdleTimeout` seconds (e.g. user
+     * mutes mic mid-speech), forces a transition to QUIET and emits
+     * `on_speech_stopped`.
+     */
+    protected _audioIdleHandler(): Promise<void>;
+    /** Handle user speaking frame. */
+    protected _maybeSpeechActivity(): Promise<void>;
+    /**
+     * Request a frame to be pushed through the pipeline.
+     *
+     * This emits an on_push_frame event that must be handled by a processor to
+     * actually push the frame into the pipeline.
+     *
+     * @param frame The frame to push.
+     * @param direction The direction to push the frame.
+     */
+    pushFrame(frame: Frame, direction?: FrameDirection): Promise<void>;
+    /**
+     * Request a frame to be broadcast upstream and downstream.
+     *
+     * This emits an on_broadcast_frame event that must be handled by a
+     * processor to actually broadcast the frame in the pipeline.
+     *
+     * @param frameCls The class of the frame to broadcast.
+     * @param options Arguments to pass to the frame constructor.
+     */
+    broadcastFrame<O>(frameCls: new (options: O) => Frame, options?: O): Promise<void>;
+}
+//# sourceMappingURL=vad_controller.d.ts.map