@livekit/agents-plugin-silero 0.4.0

package/dist/vad.js

@@ -0,0 +1,240 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { ExpFilter, VADEventType, VADStream as baseStream, VAD as baseVAD, log, mergeFrames, } from '@livekit/agents';
+import { AudioFrame, AudioResampler, AudioResamplerQuality } from '@livekit/rtc-node';
+import { OnnxModel, newInferenceSession } from './onnx_model.js';
+const SLOW_INFERENCE_THRESHOLD = 200; // late by 200ms
+const defaultVADOptions = {
+    minSpeechDuration: 50,
+    minSilenceDuration: 250,
+    prefixPaddingDuration: 100,
+    maxBufferedSpeech: 60000,
+    activationThreshold: 0.5,
+    sampleRate: 16000,
+    forceCPU: true,
+};
+export class VAD extends baseVAD {
+    #session;
+    #opts;
+    constructor(session, opts) {
+        super({ updateInterval: 32 });
+        this.#session = session;
+        this.#opts = opts;
+    }
+    /**
+     * Load and initialize the Silero VAD model.
+     *
+     * This method loads the ONNX model and prepares it for inference. When options are not provided,
+     * sane defaults are used.
+     *
+     * @remarks
+     * This method may take time to load the model into memory.
+     * It is recommended to call this method inside your prewarm mechanism.
+     *
+     * @example
+     * ```ts
+     * export default defineAgent({
+     *   prewarm: async (proc: JobProcess) => {
+     *     proc.userData.vad = await VAD.load();
+     *   },
+     *   entry: async (ctx: JobContext) => {
+     *     const vad = ctx.proc.userData.vad! as VAD;
+     *     // the rest of your agent logic
+     *   },
+     * });
+     * ```
+     *
+     * @param options -
+     * @returns Promise\<{@link VAD}\>: An instance of the VAD class ready for streaming.
+     */
+    static async load(opts = defaultVADOptions) {
+        const session = await newInferenceSession(opts.forceCPU);
+        return new VAD(session, opts);
+    }
+    stream() {
+        return new VADStream(this.#opts, new OnnxModel(this.#session, this.#opts.sampleRate));
+    }
+}
+export class VADStream extends baseStream {
+    #opts;
+    #model;
+    #task;
+    #expFilter = new ExpFilter(0.35);
+    #extraInferenceTime = 0;
+    #logger = log();
+    constructor(opts, model) {
+        super();
+        this.#opts = opts;
+        this.#model = model;
+        this.#task = new Promise(async () => {
+            let inferenceData = new Float32Array(this.#model.windowSizeSamples);
+            // a copy is exposed to the user in END_OF_SPEECH
+            let speechBuffer = null;
+            let speechBufferMaxReached = false;
+            let speechBufferIndex = 0;
+            // "pub" means public, these values are exposed to the users through events
+            let pubSpeaking = false;
+            let pubSpeechDuration = 0;
+            let pubSilenceDuration = 0;
+            let pubCurrentSample = 0;
+            let pubTimestamp = 0;
+            let pubSampleRate = 0;
+            let pubPrefixPaddingSamples = 0; // size in samples of padding data
+            let speechThresholdDuration = 0;
+            let silenceThresholdDuration = 0;
+            let inputFrames = [];
+            let inferenceFrames = [];
+            let resampler = null;
+            // used to avoid drift when the sampleRate ratio is not an integer
+            let inputCopyRemainingFrac = 0.0;
+            for await (const frame of this.input) {
+                if (typeof frame === 'symbol') {
+                    continue; // ignore flush sentinel for now
+                }
+                if (!pubSampleRate || !speechBuffer) {
+                    pubSampleRate = frame.sampleRate;
+                    pubPrefixPaddingSamples = Math.ceil(this.#opts.prefixPaddingDuration * pubSampleRate);
+                    speechBuffer = new Int16Array((this.#opts.maxBufferedSpeech + this.#opts.prefixPaddingDuration) * pubSampleRate);
+                    if (this.#opts.sampleRate !== pubSampleRate) {
+                        // resampling needed: the input sample rate isn't the same as the model's
+                        // sample rate used for inference
+                        resampler = new AudioResampler(pubSampleRate, this.#opts.sampleRate, 1, AudioResamplerQuality.QUICK);
+                    }
+                }
+                else if (frame.sampleRate !== pubSampleRate) {
+                    this.#logger.error('a frame with a different sample rate was already published');
+                    continue;
+                }
+                inputFrames.push(frame);
+                if (resampler) {
+                    inferenceFrames.push(...resampler.push(frame));
+                }
+                else {
+                    inferenceFrames.push(frame);
+                }
+                while (true) {
+                    const startTime = process.hrtime.bigint();
+                    const availableInferenceSamples = inferenceFrames
+                        .map((x) => x.samplesPerChannel)
+                        .reduce((acc, x) => acc + x, 0);
+                    if (availableInferenceSamples < this.#model.windowSizeSamples) {
+                        break; // not enough samples to run inference
+                    }
+                    const inputFrame = mergeFrames(inputFrames);
+                    const inferenceFrame = mergeFrames(inferenceFrames);
+                    // convert data to f32
+                    inferenceData = Float32Array.from(inferenceFrame.data.subarray(0, this.#model.windowSizeSamples), (x) => x / 32767);
+                    const p = await this.#model
+                        .run(inferenceData)
+                        .then((data) => this.#expFilter.apply(1, data));
+                    const windowDuration = (this.#model.windowSizeSamples / this.#opts.sampleRate) * 1000;
+                    pubCurrentSample += this.#model.windowSizeSamples;
+                    pubTimestamp += windowDuration;
+                    const resamplingRatio = pubSampleRate / this.#model.sampleRate;
+                    const toCopy = this.#model.windowSizeSamples * resamplingRatio + inputCopyRemainingFrac;
+                    const toCopyInt = Math.trunc(toCopy);
+                    inputCopyRemainingFrac = toCopy - toCopyInt;
+                    // copy the inference window to the speech buffer
+                    const availableSpace = speechBuffer.length - speechBufferIndex;
+                    const toCopyBuffer = Math.min(this.#model.windowSizeSamples, availableSpace);
+                    if (toCopyBuffer > 0) {
+                        speechBuffer.set(inputFrame.data.subarray(0, toCopyBuffer), speechBufferIndex);
+                    }
+                    else if (!speechBufferMaxReached) {
+                        speechBufferMaxReached = true;
+                        this.#logger.warn('maxBufferedSpeech reached, ignoring further data for the current speech input');
+                    }
+                    const inferenceDuration = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));
+                    this.#extraInferenceTime = Math.max(0, this.#extraInferenceTime + inferenceDuration - windowDuration);
+                    if (this.#extraInferenceTime > SLOW_INFERENCE_THRESHOLD) {
+                        this.#logger
+                            .child({ delay: this.#extraInferenceTime })
+                            .warn('inference is slower than realtime');
+                    }
+                    if (pubSpeaking) {
+                        pubSpeechDuration += inferenceDuration;
+                    }
+                    else {
+                        pubSilenceDuration += inferenceDuration;
+                    }
+                    this.queue.put({
+                        type: VADEventType.INFERENCE_DONE,
+                        samplesIndex: pubCurrentSample,
+                        timestamp: pubTimestamp,
+                        silenceDuration: pubSilenceDuration,
+                        speechDuration: pubSpeechDuration,
+                        probability: p,
+                        inferenceDuration,
+                        frames: [
+                            new AudioFrame(new Int16Array(inputFrame.data.subarray(0, toCopyInt)), pubSampleRate, 1, toCopyInt),
+                        ],
+                        speaking: pubSpeaking,
+                    });
+                    const copySpeechBuffer = () => {
+                        if (!speechBuffer)
+                            throw new Error('speechBuffer is empty');
+                        return new AudioFrame(new Int16Array(speechBuffer.subarray(0, speechBufferIndex)), pubSampleRate, 1, speechBufferIndex);
+                    };
+                    if (p > this.#opts.activationThreshold) {
+                        speechThresholdDuration += windowDuration;
+                        silenceThresholdDuration = 0;
+                        if (!pubSpeaking && speechThresholdDuration >= this.#opts.minSpeechDuration) {
+                            pubSpeaking = true;
+                            pubSilenceDuration = 0;
+                            pubSpeechDuration = speechThresholdDuration;
+                            this.queue.put({
+                                type: VADEventType.START_OF_SPEECH,
+                                samplesIndex: pubCurrentSample,
+                                timestamp: pubTimestamp,
+                                silenceDuration: pubSilenceDuration,
+                                speechDuration: pubSpeechDuration,
+                                probability: p,
+                                inferenceDuration,
+                                frames: [copySpeechBuffer()],
+                                speaking: pubSpeaking,
+                            });
+                        }
+                    }
+                    else {
+                        silenceThresholdDuration += windowDuration;
+                        speechThresholdDuration = 0;
+                        if (!pubSpeaking && speechBufferIndex <= pubPrefixPaddingSamples) {
+                            const paddingData = speechBuffer.subarray(speechBufferIndex - pubPrefixPaddingSamples, speechBufferIndex);
+                            speechBuffer.set(paddingData, 0);
+                            speechBufferIndex = pubPrefixPaddingSamples;
+                            speechBufferMaxReached = false;
+                        }
+                        if (pubSpeaking && silenceThresholdDuration > this.#opts.minSilenceDuration) {
+                            pubSpeaking = false;
+                            pubSpeechDuration = 0;
+                            pubSilenceDuration = silenceThresholdDuration;
+                            this.queue.put({
+                                type: VADEventType.END_OF_SPEECH,
+                                samplesIndex: pubCurrentSample,
+                                timestamp: pubTimestamp,
+                                silenceDuration: pubSilenceDuration,
+                                speechDuration: pubSpeechDuration,
+                                probability: p,
+                                inferenceDuration,
+                                frames: [copySpeechBuffer()],
+                                speaking: pubSpeaking,
+                            });
+                        }
+                    }
+                    inputFrames = [];
+                    inferenceFrames = [];
+                    if (inputFrame.data.length > toCopyInt) {
+                        const data = new Int16Array(inputFrame.data.subarray(toCopyInt));
+                        inputFrames.push(new AudioFrame(data, pubSampleRate, 1, Math.trunc(data.length / 2)));
+                    }
+                    if (inferenceFrame.data.length > this.#model.windowSizeSamples) {
+                        const data = new Int16Array(inferenceFrame.data.subarray(this.#model.windowSizeSamples));
+                        inferenceFrames.push(new AudioFrame(data, this.#opts.sampleRate, 1, Math.trunc(data.length / 2)));
+                    }
+                }
+            }
+        });
+    }
+}
+//# sourceMappingURL=vad.js.map

package/dist/vad.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vad.js","sourceRoot":"","sources":["../src/vad.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EACL,SAAS,EACT,YAAY,EACZ,SAAS,IAAI,UAAU,EACvB,GAAG,IAAI,OAAO,EACd,GAAG,EACH,WAAW,GACZ,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAGtF,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,wBAAwB,GAAG,GAAG,CAAC,CAAC,gBAAgB;AAmBtD,MAAM,iBAAiB,GAAe;IACpC,iBAAiB,EAAE,EAAE;IACrB,kBAAkB,EAAE,GAAG;IACvB,qBAAqB,EAAE,GAAG;IAC1B,iBAAiB,EAAE,KAAK;IACxB,mBAAmB,EAAE,GAAG;IACxB,UAAU,EAAE,KAAK;IACjB,QAAQ,EAAE,IAAI;CACf,CAAC;AAEF,MAAM,OAAO,GAAI,SAAQ,OAAO;IAC9B,QAAQ,CAAmB;IAC3B,KAAK,CAAa;IAElB,YAAY,OAAyB,EAAE,IAAgB;QACrD,KAAK,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,iBAAiB;QACxC,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzD,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IACxF,CAAC;CACF;AAED,MAAM,OAAO,SAAU,SAAQ,UAAU;IACvC,KAAK,CAAa;IAClB,MAAM,CAAY;IAClB,KAAK,CAAgB;IACrB,UAAU,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC;IACjC,mBAAmB,GAAG,CAAC,CAAC;IACxB,OAAO,GAAG,GAAG,EAAE,CAAC;IAEhB,YAAY,IAAgB,EAAE,KAAgB;QAC5C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QAEpB,IAAI,CAAC,KAAK,GAAG,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE;YAClC,IAAI,aAAa,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YAEpE,iDAAiD;YACjD,IAAI,YAAY,GAAsB,IAAI,CAAC;YAC3C,IAAI,sBAAsB,GAAG,KAAK,CAAC;YACnC,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAE1B,2EAA2E;YAC3E,IAAI,WAAW,GAAG,KAAK,CAAC;YACxB,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAC3B,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,IAAI,aAAa,GAAG,CAAC,CAAC;YACtB,IAAI,uBAAuB,GAAG,CAAC,CAAC,CAAC,kCAAkC;YAEnE,IAAI,uBAAuB,GAAG,CAAC,CAAC;YAChC,IAAI,wBAAwB,GAAG,CAAC,CAAC;YAEjC,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,IAAI,eAAe,GAAiB,EAAE,CAAC;YACvC,IAAI,SAAS,GAA0B,IAAI,CAAC;YAE5C,kEAAkE;YAClE,IAAI,sBAAsB,GAAG,GAAG,CAAC;YAEjC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAC9B,SAAS,CAAC,gCAAgC;gBAC5C,CAAC;gBAED,IAAI,CAAC,aAAa,IAAI,CAAC,YAAY,EAAE,CAAC;oBACpC,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC;oBACjC,uBAAuB,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,qBAAqB,GAAG,aAAa,CAAC,CAAC;oBAEtF,YAAY,GAAG,IAAI,UAAU,CAC3B,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,GAAG,aAAa,CAClF,CAAC;oBAEF,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,KAAK,aAAa,EAAE,CAAC;wBAC5C,yEAAyE;wBACzE,iCAAiC;wBACjC,SAAS,GAAG,IAAI,cAAc,CAC5B,aAAa,EACb,IAAI,CAAC,KAAK,CAAC,UAAU,EACrB,CAAC,EACD,qBAAqB,CAAC,KAAK,CAC5B,CAAC;oBACJ,CAAC;gBACH,CAAC;qBAAM,IAAI,KAAK,CAAC,UAAU,KAAK,aAAa,EAAE,CAAC;oBAC9C,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;oBACjF,SAAS;gBACX,CAAC;gBAED,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACxB,IAAI,SAAS,EAAE,CAAC;oBACd,eAAe,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjD,CAAC;qBAAM,CAAC;oBACN,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC9B,CAAC;gBAED,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC1C,MAAM,yBAAyB,GAAG,eAAe;yBAC9C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC;yBAC/B,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;oBAElC,IAAI,yBAAyB,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;wBAC9D,MAAM,CAAC,sCAAsC;oBAC/C,CAAC;oBAED,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;oBAC5C,MAAM,cAAc,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;oBAEpD,sBAAsB;oBACtB,aAAa,GAAG,YAAY,CAAC,IAAI,CAC/B,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,EAC9D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CACjB,CAAC;oBAEF,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM;yBACxB,GAAG,CAAC,aAAa,CAAC;yBAClB,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;oBAElD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;oBACtF,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC;oBAClD,YAAY,IAAI,cAAc,CAAC;oBAC/B,MAAM,eAAe,GAAG,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;oBAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,GAAG,eAAe,GAAG,sBAAsB,CAAC;oBACxF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBACrC,sBAAsB,GAAG,MAAM,GAAG,SAAS,CAAC;oBAE5C,iDAAiD;oBACjD,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,GAAG,iBAAiB,CAAC;oBAC/D,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;oBAC7E,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;wBACrB,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,iBAAiB,CAAC,CAAC;oBACjF,CAAC;yBAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;wBACnC,sBAAsB,GAAG,IAAI,CAAC;wBAC9B,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,+EAA+E,CAChF,CAAC;oBACJ,CAAC;oBAED,MAAM,iBAAiB,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC1F,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CACjC,CAAC,EACD,IAAI,CAAC,mBAAmB,GAAG,iBAAiB,GAAG,cAAc,CAC9D,CAAC;oBACF,IAAI,IAAI,CAAC,mBAAmB,GAAG,wBAAwB,EAAE,CAAC;wBACxD,IAAI,CAAC,OAAO;6BACT,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC;6BAC1C,IAAI,CAAC,mCAAmC,CAAC,CAAC;oBAC/C,CAAC;oBAED,IAAI,WAAW,EAAE,CAAC;wBAChB,iBAAiB,IAAI,iBAAiB,CAAC;oBACzC,CAAC;yBAAM,CAAC;wBACN,kBAAkB,IAAI,iBAAiB,CAAC;oBAC1C,CAAC;oBAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;wBACb,IAAI,EAAE,YAAY,CAAC,cAAc;wBACjC,YAAY,EAAE,gBAAgB;wBAC9B,SAAS,EAAE,YAAY;wBACvB,eAAe,EAAE,kBAAkB;wBACnC,cAAc,EAAE,iBAAiB;wBACjC,WAAW,EAAE,CAAC;wBACd,iBAAiB;wBACjB,MAAM,EAAE;4BACN,IAAI,UAAU,CACZ,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EACtD,aAAa,EACb,CAAC,EACD,SAAS,CACV;yBACF;wBACD,QAAQ,EAAE,WAAW;qBACtB,CAAC,CAAC;oBAEH,MAAM,gBAAgB,GAAG,GAAe,EAAE;wBACxC,IAAI,CAAC,YAAY;4BAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;wBAC5D,OAAO,IAAI,UAAU,CACnB,IAAI,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,EAC3D,aAAa,EACb,CAAC,EACD,iBAAiB,CAClB,CAAC;oBACJ,CAAC,CAAC;oBAEF,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC;wBACvC,uBAAuB,IAAI,cAAc,CAAC;wBAC1C,wBAAwB,GAAG,CAAC,CAAC;wBAC7B,IAAI,CAAC,WAAW,IAAI,uBAAuB,IAAI,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC;4BAC5E,WAAW,GAAG,IAAI,CAAC;4BACnB,kBAAkB,GAAG,CAAC,CAAC;4BACvB,iBAAiB,GAAG,uBAAuB,CAAC;4BAE5C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;gCACb,IAAI,EAAE,YAAY,CAAC,eAAe;gCAClC,YAAY,EAAE,gBAAgB;gCAC9B,SAAS,EAAE,YAAY;gCACvB,eAAe,EAAE,kBAAkB;gCACnC,cAAc,EAAE,iBAAiB;gCACjC,WAAW,EAAE,CAAC;gCACd,iBAAiB;gCACjB,MAAM,EAAE,CAAC,gBAAgB,EAAE,CAAC;gCAC5B,QAAQ,EAAE,WAAW;6BACtB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,wBAAwB,IAAI,cAAc,CAAC;wBAC3C,uBAAuB,GAAG,CAAC,CAAC;wBAE5B,IAAI,CAAC,WAAW,IAAI,iBAAiB,IAAI,uBAAuB,EAAE,CAAC;4BACjE,MAAM,WAAW,GAAG,YAAY,CAAC,QAAQ,CACvC,iBAAiB,GAAG,uBAAuB,EAC3C,iBAAiB,CAClB,CAAC;4BACF,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;4BACjC,iBAAiB,GAAG,uBAAuB,CAAC;4BAC5C,sBAAsB,GAAG,KAAK,CAAC;wBACjC,CAAC;wBAED,IAAI,WAAW,IAAI,wBAAwB,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC;4BAC5E,WAAW,GAAG,KAAK,CAAC;4BACpB,iBAAiB,GAAG,CAAC,CAAC;4BACtB,kBAAkB,GAAG,wBAAwB,CAAC;4BAE9C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;gCACb,IAAI,EAAE,YAAY,CAAC,aAAa;gCAChC,YAAY,EAAE,gBAAgB;gCAC9B,SAAS,EAAE,YAAY;gCACvB,eAAe,EAAE,kBAAkB;gCACnC,cAAc,EAAE,iBAAiB;gCACjC,WAAW,EAAE,CAAC;gCACd,iBAAiB;gCACjB,MAAM,EAAE,CAAC,gBAAgB,EAAE,CAAC;gCAC5B,QAAQ,EAAE,WAAW;6BACtB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,WAAW,GAAG,EAAE,CAAC;oBACjB,eAAe,GAAG,EAAE,CAAC;oBAErB,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;wBACvC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;wBACjE,WAAW,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBACxF,CAAC;oBACD,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;wBAC/D,MAAM,IAAI,GAAG,IAAI,UAAU,CACzB,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAC5D,CAAC;wBACF,eAAe,CAAC,IAAI,CAClB,IAAI,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAC5E,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@livekit/agents-plugin-silero",
+  "version": "0.4.0",
+  "description": "Silero voice activity detection LiveKit Node Agents",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "author": "LiveKit",
+  "type": "module",
+  "devDependencies": {
+    "@microsoft/api-extractor": "^7.35.0",
+    "@types/ws": "^8.5.10",
+    "onnxruntime-common": "^1.19.2",
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "@livekit/rtc-node": "^0.11.1",
+    "onnxruntime-node": "^1.19.2",
+    "ws": "^8.16.0",
+    "@livekit/agents": "0.4.0"
+  },
+  "scripts": {
+    "build": "tsc && cp src/*.onnx dist/",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  }
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export { VAD, VADStream } from './vad.js';

package/src/onnx_model.ts

@@ -0,0 +1,80 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { InferenceSession, Tensor } from 'onnxruntime-node';
+
+export type SampleRate = 8000 | 16000;
+
+export const newInferenceSession = (forceCPU: boolean) => {
+  return InferenceSession.create(new URL('silero_vad.onnx', import.meta.url).pathname, {
+    interOpNumThreads: 1,
+    intraOpNumThreads: 1,
+    executionMode: 'sequential',
+    executionProviders: forceCPU ? [{ name: 'cpu' }] : undefined,
+  });
+};
+
+export class OnnxModel {
+  #session: InferenceSession;
+  #sampleRate: number;
+  #windowSizeSamples: number;
+  #contextSize: number;
+  #sampleRateNd: BigInt64Array;
+  #context: Float32Array;
+  // #state: Float32Array;
+  #rnnState: Float32Array;
+  #inputBuffer: Float32Array;
+
+  constructor(session: InferenceSession, sampleRate: SampleRate) {
+    this.#session = session;
+    this.#sampleRate = sampleRate;
+
+    switch (sampleRate) {
+      case 8000:
+        this.#windowSizeSamples = 256;
+        this.#contextSize = 32;
+        break;
+      case 16000:
+        this.#windowSizeSamples = 512;
+        this.#contextSize = 64;
+        break;
+    }
+
+    this.#sampleRateNd = BigInt64Array.from([BigInt(sampleRate)]);
+    this.#context = new Float32Array(this.#contextSize);
+    this.#rnnState = new Float32Array(2 * 1 * 128);
+    this.#inputBuffer = new Float32Array(this.#contextSize + this.#windowSizeSamples);
+  }
+
+  get sampleRate(): number {
+    return this.#sampleRate;
+  }
+
+  get windowSizeSamples(): number {
+    return this.#windowSizeSamples;
+  }
+
+  get contextSize(): number {
+    return this.#contextSize;
+  }
+
+  async run(x: Float32Array): Promise<number> {
+    this.#inputBuffer.set(this.#context, 0);
+    this.#inputBuffer.set(x, this.#contextSize);
+
+    return await this.#session
+      .run({
+        input: new Tensor('float32', this.#inputBuffer, [
+          1,
+          this.#contextSize + this.#windowSizeSamples,
+        ]),
+        state: new Tensor('float32', this.#rnnState, [2, 1, 128]),
+        sr: new Tensor('int64', this.#sampleRateNd),
+      })
+      .then((result) => {
+        // this.#state = result.output.data as Float32Array,
+        this.#context = this.#inputBuffer.subarray(0, this.#contextSize);
+        return (result.output.data as Float32Array).at(0)!;
+      });
+  }
+}

package/src/onnxruntime.d.ts

@@ -0,0 +1,8 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// https://github.com/microsoft/onnxruntime/issues/17979
+declare module 'onnxruntime-node' {
+  export * from 'onnxruntime-common';
+}

package/src/silero_vad.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b99cbfd39246b6706f98ec13c7c50c6b299181f2474fa05cbc8046acc274396
+size 2313101