@livekit/agents-plugin-silero 0.4.5 → 0.5.0

package/dist/vad.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n  ExpFilter,\n  VADEventType,\n  VADStream as baseStream,\n  VAD as baseVAD,\n  log,\n  mergeFrames,\n} from '@livekit/agents';\nimport { AudioFrame, AudioResampler, AudioResamplerQuality } from '@livekit/rtc-node';\nimport type { InferenceSession } from 'onnxruntime-node';\nimport type { SampleRate } from './onnx_model.js';\nimport { OnnxModel, newInferenceSession } from './onnx_model.js';\n\nconst SLOW_INFERENCE_THRESHOLD = 200; // late by 200ms\n\nexport interface VADOptions {\n  /** Minimum duration of speech to start a new speech chunk */\n  minSpeechDuration: number;\n  /** At the end of each speech, wait this duration before ending the speech */\n  minSilenceDuration: number;\n  /** Duration of padding to add to the beginning of each speech chunk */\n  prefixPaddingDuration: number;\n  /** Maximum duration of speech to keep in the buffer */\n  maxBufferedSpeech: number;\n  /** Maximum duration of speech to keep in the buffer*/\n  activationThreshold: number;\n  /** Sample rate for the inference (only 8KHz and 16KHz are supported) */\n  sampleRate: SampleRate;\n  /** Force the use of CPU for inference */\n  forceCPU: boolean;\n}\n\nconst defaultVADOptions: VADOptions = {\n  minSpeechDuration: 50,\n  minSilenceDuration: 250,\n  prefixPaddingDuration: 500,\n  maxBufferedSpeech: 60000,\n  activationThreshold: 0.5,\n  sampleRate: 16000,\n  forceCPU: true,\n};\n\nexport class VAD extends baseVAD {\n  #session: InferenceSession;\n  #opts: VADOptions;\n\n  constructor(session: InferenceSession, opts: VADOptions) {\n    super({ updateInterval: 32 });\n    this.#session = session;\n    this.#opts = opts;\n  }\n\n  /**\n   * Load and initialize the Silero VAD model.\n   *\n   * This method loads the ONNX model and prepares it for inference. When options are not provided,\n   * sane defaults are used.\n   *\n   * @remarks\n   * This method may take time to load the model into memory.\n   * It is recommended to call this method inside your prewarm mechanism.\n   *\n   * @example\n   * ```ts\n   * export default defineAgent({\n   *   prewarm: async (proc: JobProcess) => {\n   *     proc.userData.vad = await VAD.load();\n   *   },\n   *   entry: async (ctx: JobContext) => {\n   *     const vad = ctx.proc.userData.vad! as VAD;\n   *     // the rest of your agent logic\n   *   },\n   * });\n   * ```\n   *\n   * @param options -\n   * @returns Promise\\<{@link VAD}\\>: An instance of the VAD class ready for streaming.\n   */\n  static async load(opts: Partial<VADOptions> = {}): Promise<VAD> {\n    const mergedOpts: VADOptions = { ...defaultVADOptions, ...opts };\n    const session = await newInferenceSession(mergedOpts.forceCPU);\n    return new VAD(session, mergedOpts);\n  }\n\n  stream(): VADStream {\n    return new VADStream(this.#opts, new OnnxModel(this.#session, this.#opts.sampleRate));\n  }\n}\n\nexport class VADStream extends baseStream {\n  #opts: VADOptions;\n  #model: OnnxModel;\n  #task: Promise<void>;\n  #expFilter = new ExpFilter(0.35);\n  #extraInferenceTime = 0;\n  #logger = log();\n\n  constructor(opts: VADOptions, model: OnnxModel) {\n    super();\n    this.#opts = opts;\n    this.#model = model;\n\n    this.#task = new Promise(async () => {\n      let inferenceData = new Float32Array(this.#model.windowSizeSamples);\n\n      // a copy is exposed to the user in END_OF_SPEECH\n      let speechBuffer: Int16Array | null = null;\n      let speechBufferMaxReached = false;\n      let speechBufferIndex = 0;\n\n      // \"pub\" means public, these values are exposed to the users through events\n      let pubSpeaking = false;\n      let pubSpeechDuration = 0;\n      let pubSilenceDuration = 0;\n      let pubCurrentSample = 0;\n      let pubTimestamp = 0;\n      let pubSampleRate = 0;\n      let pubPrefixPaddingSamples = 0; // size in samples of padding data\n\n      let speechThresholdDuration = 0;\n      let silenceThresholdDuration = 0;\n\n      let inputFrames = [];\n      let inferenceFrames: AudioFrame[] = [];\n      let resampler: AudioResampler | null = null;\n\n      // used to avoid drift when the sampleRate ratio is not an integer\n      let inputCopyRemainingFrac = 0.0;\n\n      for await (const frame of this.input) {\n        if (typeof frame === 'symbol') {\n          continue; // ignore flush sentinel for now\n        }\n\n        if (!pubSampleRate || !speechBuffer) {\n          pubSampleRate = frame.sampleRate;\n          pubPrefixPaddingSamples = Math.trunc(\n            (this.#opts.prefixPaddingDuration * pubSampleRate) / 1000,\n          );\n\n          speechBuffer = new Int16Array(\n            this.#opts.maxBufferedSpeech * pubSampleRate + pubPrefixPaddingSamples,\n          );\n\n          if (this.#opts.sampleRate !== pubSampleRate) {\n            // resampling needed: the input sample rate isn't the same as the model's\n            // sample rate used for inference\n            resampler = new AudioResampler(\n              pubSampleRate,\n              this.#opts.sampleRate,\n              1,\n              AudioResamplerQuality.QUICK, // VAD doesn't need high quality\n            );\n          }\n        } else if (frame.sampleRate !== pubSampleRate) {\n          this.#logger.error('a frame with a different sample rate was already published');\n          continue;\n        }\n\n        inputFrames.push(frame);\n        if (resampler) {\n          inferenceFrames.push(...resampler.push(frame));\n        } else {\n          inferenceFrames.push(frame);\n        }\n\n        while (true) {\n          const startTime = process.hrtime.bigint();\n          const availableInferenceSamples = inferenceFrames\n            .map((x) => x.samplesPerChannel)\n            .reduce((acc, x) => acc + x, 0);\n\n          if (availableInferenceSamples < this.#model.windowSizeSamples) {\n            break; // not enough samples to run inference\n          }\n\n          const inputFrame = mergeFrames(inputFrames);\n          const inferenceFrame = mergeFrames(inferenceFrames);\n\n          // convert data to f32\n          inferenceData = Float32Array.from(\n            inferenceFrame.data.subarray(0, this.#model.windowSizeSamples),\n            (x) => x / 32767,\n          );\n\n          const p = await this.#model\n            .run(inferenceData)\n            .then((data) => this.#expFilter.apply(1, data));\n\n          const windowDuration = (this.#model.windowSizeSamples / this.#opts.sampleRate) * 1000;\n          pubCurrentSample += this.#model.windowSizeSamples;\n          pubTimestamp += windowDuration;\n          const resamplingRatio = pubSampleRate / this.#model.sampleRate;\n          const toCopy = this.#model.windowSizeSamples * resamplingRatio + inputCopyRemainingFrac;\n          const toCopyInt = Math.trunc(toCopy);\n          inputCopyRemainingFrac = toCopy - toCopyInt;\n\n          // copy the inference window to the speech buffer\n          const availableSpace = speechBuffer.length - speechBufferIndex;\n          const toCopyBuffer = Math.min(this.#model.windowSizeSamples, availableSpace);\n          if (toCopyBuffer > 0) {\n            speechBuffer.set(inputFrame.data.subarray(0, toCopyBuffer), speechBufferIndex);\n            speechBufferIndex += toCopyBuffer;\n          } else if (!speechBufferMaxReached) {\n            speechBufferMaxReached = true;\n            this.#logger.warn(\n              'maxBufferedSpeech reached, ignoring further data for the current speech input',\n            );\n          }\n\n          const inferenceDuration = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));\n          this.#extraInferenceTime = Math.max(\n            0,\n            this.#extraInferenceTime + inferenceDuration - windowDuration,\n          );\n          if (this.#extraInferenceTime > SLOW_INFERENCE_THRESHOLD) {\n            this.#logger\n              .child({ delay: this.#extraInferenceTime })\n              .warn('inference is slower than realtime');\n          }\n\n          if (pubSpeaking) {\n            pubSpeechDuration += inferenceDuration;\n          } else {\n            pubSilenceDuration += inferenceDuration;\n          }\n\n          this.queue.put({\n            type: VADEventType.INFERENCE_DONE,\n            samplesIndex: pubCurrentSample,\n            timestamp: pubTimestamp,\n            silenceDuration: pubSilenceDuration,\n            speechDuration: pubSpeechDuration,\n            probability: p,\n            inferenceDuration,\n            frames: [\n              new AudioFrame(inputFrame.data.subarray(0, toCopyInt), pubSampleRate, 1, toCopyInt),\n            ],\n            speaking: pubSpeaking,\n          });\n\n          const resetWriteCursor = () => {\n            if (!speechBuffer) throw new Error('speechBuffer is empty');\n            if (speechBufferIndex <= pubPrefixPaddingSamples) {\n              return;\n            }\n\n            const paddingData = speechBuffer.subarray(\n              speechBufferIndex - pubPrefixPaddingSamples,\n              speechBufferIndex,\n            );\n            speechBuffer.set(paddingData, 0);\n            speechBufferIndex = pubPrefixPaddingSamples;\n            speechBufferMaxReached = false;\n          };\n\n          const copySpeechBuffer = (): AudioFrame => {\n            if (!speechBuffer) throw new Error('speechBuffer is empty');\n            return new AudioFrame(\n              speechBuffer.subarray(0, speechBufferIndex),\n              pubSampleRate,\n              1,\n              speechBufferIndex,\n            );\n          };\n\n          if (p > this.#opts.activationThreshold) {\n            speechThresholdDuration += windowDuration;\n            silenceThresholdDuration = 0;\n            if (!pubSpeaking && speechThresholdDuration >= this.#opts.minSpeechDuration) {\n              pubSpeaking = true;\n              pubSilenceDuration = 0;\n              pubSpeechDuration = speechThresholdDuration;\n\n              this.queue.put({\n                type: VADEventType.START_OF_SPEECH,\n                samplesIndex: pubCurrentSample,\n                timestamp: pubTimestamp,\n                silenceDuration: pubSilenceDuration,\n                speechDuration: pubSpeechDuration,\n                probability: p,\n                inferenceDuration,\n                frames: [copySpeechBuffer()],\n                speaking: pubSpeaking,\n              });\n            }\n          } else {\n            silenceThresholdDuration += windowDuration;\n            speechThresholdDuration = 0;\n\n            if (!pubSpeaking) {\n              resetWriteCursor();\n            }\n\n            if (pubSpeaking && silenceThresholdDuration > this.#opts.minSilenceDuration) {\n              pubSpeaking = false;\n              pubSpeechDuration = 0;\n              pubSilenceDuration = silenceThresholdDuration;\n\n              this.queue.put({\n                type: VADEventType.END_OF_SPEECH,\n                samplesIndex: pubCurrentSample,\n                timestamp: pubTimestamp,\n                silenceDuration: pubSilenceDuration,\n                speechDuration: pubSpeechDuration,\n                probability: p,\n                inferenceDuration,\n                frames: [copySpeechBuffer()],\n                speaking: pubSpeaking,\n              });\n\n              resetWriteCursor();\n            }\n          }\n\n          inputFrames = [];\n          inferenceFrames = [];\n\n          if (inputFrame.data.length > toCopyInt) {\n            const data = inputFrame.data.subarray(toCopyInt);\n            inputFrames.push(new AudioFrame(data, pubSampleRate, 1, Math.trunc(data.length / 2)));\n          }\n          if (inferenceFrame.data.length > this.#model.windowSizeSamples) {\n            const data = inferenceFrame.data.subarray(this.#model.windowSizeSamples);\n            inferenceFrames.push(\n              new AudioFrame(data, this.#opts.sampleRate, 1, Math.trunc(data.length / 2)),\n            );\n          }\n        }\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAOO;AACP,sBAAkE;AAGlE,wBAA+C;AAE/C,MAAM,2BAA2B;AAmBjC,MAAM,oBAAgC;AAAA,EACpC,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EACpB,uBAAuB;AAAA,EACvB,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,YAAY;AAAA,EACZ,UAAU;AACZ;AAEO,MAAM,YAAY,cAAAA,IAAQ;AAAA,EAC/B;AAAA,EACA;AAAA,EAEA,YAAY,SAA2B,MAAkB;AACvD,UAAM,EAAE,gBAAgB,GAAG,CAAC;AAC5B,SAAK,WAAW;AAChB,SAAK,QAAQ;AAAA,EACf;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA4BA,aAAa,KAAK,OAA4B,CAAC,GAAiB;AAC9D,UAAM,aAAyB,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC/D,UAAM,UAAU,UAAM,uCAAoB,WAAW,QAAQ;AAC7D,WAAO,IAAI,IAAI,SAAS,UAAU;AAAA,EACpC;AAAA,EAEA,SAAoB;AAClB,WAAO,IAAI,UAAU,KAAK,OAAO,IAAI,4BAAU,KAAK,UAAU,KAAK,MAAM,UAAU,CAAC;AAAA,EACtF;AACF;AAEO,MAAM,kBAAkB,cAAAC,UAAW;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAa,IAAI,wBAAU,IAAI;AAAA,EAC/B,sBAAsB;AAAA,EACtB,cAAU,mBAAI;AAAA,EAEd,YAAY,MAAkB,OAAkB;AAC9C,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,SAAS;AAEd,SAAK,QAAQ,IAAI,QAAQ,YAAY;AACnC,UAAI,gBAAgB,IAAI,aAAa,KAAK,OAAO,iBAAiB;AAGlE,UAAI,eAAkC;AACtC,UAAI,yBAAyB;AAC7B,UAAI,oBAAoB;AAGxB,UAAI,cAAc;AAClB,UAAI,oBAAoB;AACxB,UAAI,qBAAqB;AACzB,UAAI,mBAAmB;AACvB,UAAI,eAAe;AACnB,UAAI,gBAAgB;AACpB,UAAI,0BAA0B;AAE9B,UAAI,0BAA0B;AAC9B,UAAI,2BAA2B;AAE/B,UAAI,cAAc,CAAC;AACnB,UAAI,kBAAgC,CAAC;AACrC,UAAI,YAAmC;AAGvC,UAAI,yBAAyB;AAE7B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,OAAO,UAAU,UAAU;AAC7B;AAAA,QACF;AAEA,YAAI,CAAC,iBAAiB,CAAC,cAAc;AACnC,0BAAgB,MAAM;AACtB,oCAA0B,KAAK;AAAA,YAC5B,KAAK,MAAM,wBAAwB,gBAAiB;AAAA,UACvD;AAEA,yBAAe,IAAI;AAAA,YACjB,KAAK,MAAM,oBAAoB,gBAAgB;AAAA,UACjD;AAEA,cAAI,KAAK,MAAM,eAAe,eAAe;AAG3C,wBAAY,IAAI;AAAA,cACd;AAAA,cACA,KAAK,MAAM;AAAA,cACX;AAAA,cACA,sCAAsB;AAAA;AAAA,YACxB;AAAA,UACF;AAAA,QACF,WAAW,MAAM,eAAe,eAAe;AAC7C,eAAK,QAAQ,MAAM,4DAA4D;AAC/E;AAAA,QACF;AAEA,oBAAY,KAAK,KAAK;AACtB,YAAI,WAAW;AACb,0BAAgB,KAAK,GAAG,UAAU,KAAK,KAAK,CAAC;AAAA,QAC/C,OAAO;AACL,0BAAgB,KAAK,KAAK;AAAA,QAC5B;AAEA,eAAO,MAAM;AACX,gBAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,gBAAM,4BAA4B,gBAC/B,IAAI,CAAC,MAAM,EAAE,iBAAiB,EAC9B,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAC;AAEhC,cAAI,4BAA4B,KAAK,OAAO,mBAAmB;AAC7D;AAAA,UACF;AAEA,gBAAM,iBAAa,2BAAY,WAAW;AAC1C,gBAAM,qBAAiB,2BAAY,eAAe;AAGlD,0BAAgB,aAAa;AAAA,YAC3B,eAAe,KAAK,SAAS,GAAG,KAAK,OAAO,iBAAiB;AAAA,YAC7D,CAAC,MAAM,IAAI;AAAA,UACb;AAEA,gBAAM,IAAI,MAAM,KAAK,OAClB,IAAI,aAAa,EACjB,KAAK,CAAC,SAAS,KAAK,WAAW,MAAM,GAAG,IAAI,CAAC;AAEhD,gBAAM,iBAAkB,KAAK,OAAO,oBAAoB,KAAK,MAAM,aAAc;AACjF,8BAAoB,KAAK,OAAO;AAChC,0BAAgB;AAChB,gBAAM,kBAAkB,gBAAgB,KAAK,OAAO;AACpD,gBAAM,SAAS,KAAK,OAAO,oBAAoB,kBAAkB;AACjE,gBAAM,YAAY,KAAK,MAAM,MAAM;AACnC,mCAAyB,SAAS;AAGlC,gBAAM,iBAAiB,aAAa,SAAS;AAC7C,gBAAM,eAAe,KAAK,IAAI,KAAK,OAAO,mBAAmB,cAAc;AAC3E,cAAI,eAAe,GAAG;AACpB,yBAAa,IAAI,WAAW,KAAK,SAAS,GAAG,YAAY,GAAG,iBAAiB;AAC7E,iCAAqB;AAAA,UACvB,WAAW,CAAC,wBAAwB;AAClC,qCAAyB;AACzB,iBAAK,QAAQ;AAAA,cACX;AAAA,YACF;AAAA,UACF;AAEA,gBAAM,oBAAoB,QAAQ,QAAQ,OAAO,OAAO,IAAI,aAAa,OAAO,GAAO,CAAC;AACxF,eAAK,sBAAsB,KAAK;AAAA,YAC9B;AAAA,YACA,KAAK,sBAAsB,oBAAoB;AAAA,UACjD;AACA,cAAI,KAAK,sBAAsB,0BAA0B;AACvD,iBAAK,QACF,MAAM,EAAE,OAAO,KAAK,oBAAoB,CAAC,EACzC,KAAK,mCAAmC;AAAA,UAC7C;AAEA,cAAI,aAAa;AACf,iCAAqB;AAAA,UACvB,OAAO;AACL,kCAAsB;AAAA,UACxB;AAEA,eAAK,MAAM,IAAI;AAAA,YACb,MAAM,2BAAa;AAAA,YACnB,cAAc;AAAA,YACd,WAAW;AAAA,YACX,iBAAiB;AAAA,YACjB,gBAAgB;AAAA,YAChB,aAAa;AAAA,YACb;AAAA,YACA,QAAQ;AAAA,cACN,IAAI,2BAAW,WAAW,KAAK,SAAS,GAAG,SAAS,GAAG,eAAe,GAAG,SAAS;AAAA,YACpF;AAAA,YACA,UAAU;AAAA,UACZ,CAAC;AAED,gBAAM,mBAAmB,MAAM;AAC7B,gBAAI,CAAC,aAAc,OAAM,IAAI,MAAM,uBAAuB;AAC1D,gBAAI,qBAAqB,yBAAyB;AAChD;AAAA,YACF;AAEA,kBAAM,cAAc,aAAa;AAAA,cAC/B,oBAAoB;AAAA,cACpB;AAAA,YACF;AACA,yBAAa,IAAI,aAAa,CAAC;AAC/B,gCAAoB;AACpB,qCAAyB;AAAA,UAC3B;AAEA,gBAAM,mBAAmB,MAAkB;AACzC,gBAAI,CAAC,aAAc,OAAM,IAAI,MAAM,uBAAuB;AAC1D,mBAAO,IAAI;AAAA,cACT,aAAa,SAAS,GAAG,iBAAiB;AAAA,cAC1C;AAAA,cACA;AAAA,cACA;AAAA,YACF;AAAA,UACF;AAEA,cAAI,IAAI,KAAK,MAAM,qBAAqB;AACtC,uCAA2B;AAC3B,uCAA2B;AAC3B,gBAAI,CAAC,eAAe,2BAA2B,KAAK,MAAM,mBAAmB;AAC3E,4BAAc;AACd,mCAAqB;AACrB,kCAAoB;AAEpB,mBAAK,MAAM,IAAI;AAAA,gBACb,MAAM,2BAAa;AAAA,gBACnB,cAAc;AAAA,gBACd,WAAW;AAAA,gBACX,iBAAiB;AAAA,gBACjB,gBAAgB;AAAA,gBAChB,aAAa;AAAA,gBACb;AAAA,gBACA,QAAQ,CAAC,iBAAiB,CAAC;AAAA,gBAC3B,UAAU;AAAA,cACZ,CAAC;AAAA,YACH;AAAA,UACF,OAAO;AACL,wCAA4B;AAC5B,sCAA0B;AAE1B,gBAAI,CAAC,aAAa;AAChB,+BAAiB;AAAA,YACnB;AAEA,gBAAI,eAAe,2BAA2B,KAAK,MAAM,oBAAoB;AAC3E,4BAAc;AACd,kCAAoB;AACpB,mCAAqB;AAErB,mBAAK,MAAM,IAAI;AAAA,gBACb,MAAM,2BAAa;AAAA,gBACnB,cAAc;AAAA,gBACd,WAAW;AAAA,gBACX,iBAAiB;AAAA,gBACjB,gBAAgB;AAAA,gBAChB,aAAa;AAAA,gBACb;AAAA,gBACA,QAAQ,CAAC,iBAAiB,CAAC;AAAA,gBAC3B,UAAU;AAAA,cACZ,CAAC;AAED,+BAAiB;AAAA,YACnB;AAAA,UACF;AAEA,wBAAc,CAAC;AACf,4BAAkB,CAAC;AAEnB,cAAI,WAAW,KAAK,SAAS,WAAW;AACtC,kBAAM,OAAO,WAAW,KAAK,SAAS,SAAS;AAC/C,wBAAY,KAAK,IAAI,2BAAW,MAAM,eAAe,GAAG,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC;AAAA,UACtF;AACA,cAAI,eAAe,KAAK,SAAS,KAAK,OAAO,mBAAmB;AAC9D,kBAAM,OAAO,eAAe,KAAK,SAAS,KAAK,OAAO,iBAAiB;AACvE,4BAAgB;AAAA,cACd,IAAI,2BAAW,MAAM,KAAK,MAAM,YAAY,GAAG,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,YAC5E;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AACF;","names":["baseVAD","baseStream"]}

package/dist/vad.d.ts

@@ -48,7 +48,7 @@ export declare class VAD extends baseVAD {
      * @param options -
      * @returns Promise\<{@link VAD}\>: An instance of the VAD class ready for streaming.
      */
-    static load(opts?: VADOptions): Promise<VAD>;
+    static load(opts?: Partial<VADOptions>): Promise<VAD>;
     stream(): VADStream;
 }
 export declare class VADStream extends baseStream {

package/dist/vad.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vad.d.ts","sourceRoot":"","sources":["../src/vad.ts"],"names":[],"mappings":";AAGA,OAAO,EAGL,SAAS,IAAI,UAAU,EACvB,GAAG,IAAI,OAAO,EAGf,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAuB,MAAM,iBAAiB,CAAC;AAIjE,MAAM,WAAW,UAAU;IACzB,6DAA6D;IAC7D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,6EAA6E;IAC7E,kBAAkB,EAAE,MAAM,CAAC;IAC3B,uEAAuE;IACvE,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uDAAuD;IACvD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sDAAsD;IACtD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,wEAAwE;IACxE,UAAU,EAAE,UAAU,CAAC;IACvB,yCAAyC;IACzC,QAAQ,EAAE,OAAO,CAAC;CACnB;AAYD,qBAAa,GAAI,SAAQ,OAAO;;gBAIlB,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU;IAMvD;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;WACU,IAAI,CAAC,IAAI,aAAoB,GAAG,OAAO,CAAC,GAAG,CAAC;IAKzD,MAAM,IAAI,SAAS;CAGpB;AAED,qBAAa,SAAU,SAAQ,UAAU;;gBAQ3B,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS;CA2O/C"}
+{"version":3,"file":"vad.d.ts","sourceRoot":"","sources":["../src/vad.ts"],"names":[],"mappings":";AAGA,OAAO,EAGL,SAAS,IAAI,UAAU,EACvB,GAAG,IAAI,OAAO,EAGf,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAuB,MAAM,iBAAiB,CAAC;AAIjE,MAAM,WAAW,UAAU;IACzB,6DAA6D;IAC7D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,6EAA6E;IAC7E,kBAAkB,EAAE,MAAM,CAAC;IAC3B,uEAAuE;IACvE,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uDAAuD;IACvD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sDAAsD;IACtD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,wEAAwE;IACxE,UAAU,EAAE,UAAU,CAAC;IACvB,yCAAyC;IACzC,QAAQ,EAAE,OAAO,CAAC;CACnB;AAYD,qBAAa,GAAI,SAAQ,OAAO;;gBAIlB,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU;IAMvD;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;WACU,IAAI,CAAC,IAAI,GAAE,OAAO,CAAC,UAAU,CAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAM/D,MAAM,IAAI,SAAS;CAGpB;AAED,qBAAa,SAAU,SAAQ,UAAU;;gBAQ3B,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS;CA2O/C"}

package/dist/vad.js

@@ -1,250 +1,267 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-import { ExpFilter, VADEventType, VADStream as baseStream, VAD as baseVAD, log, mergeFrames, } from '@livekit/agents';
-import { AudioFrame, AudioResampler, AudioResamplerQuality } from '@livekit/rtc-node';
-import { OnnxModel, newInferenceSession } from './onnx_model.js';
-const SLOW_INFERENCE_THRESHOLD = 200; // late by 200ms
+import {
+  ExpFilter,
+  VADEventType,
+  VADStream as baseStream,
+  VAD as baseVAD,
+  log,
+  mergeFrames
+} from "@livekit/agents";
+import { AudioFrame, AudioResampler, AudioResamplerQuality } from "@livekit/rtc-node";
+import { OnnxModel, newInferenceSession } from "./onnx_model.js";
+const SLOW_INFERENCE_THRESHOLD = 200;
 const defaultVADOptions = {
-    minSpeechDuration: 50,
-    minSilenceDuration: 250,
-    prefixPaddingDuration: 500,
-    maxBufferedSpeech: 60000,
-    activationThreshold: 0.5,
-    sampleRate: 16000,
-    forceCPU: true,
+  minSpeechDuration: 50,
+  minSilenceDuration: 250,
+  prefixPaddingDuration: 500,
+  maxBufferedSpeech: 6e4,
+  activationThreshold: 0.5,
+  sampleRate: 16e3,
+  forceCPU: true
 };
-export class VAD extends baseVAD {
-    #session;
-    #opts;
-    constructor(session, opts) {
-        super({ updateInterval: 32 });
-        this.#session = session;
-        this.#opts = opts;
-    }
-    /**
-     * Load and initialize the Silero VAD model.
-     *
-     * This method loads the ONNX model and prepares it for inference. When options are not provided,
-     * sane defaults are used.
-     *
-     * @remarks
-     * This method may take time to load the model into memory.
-     * It is recommended to call this method inside your prewarm mechanism.
-     *
-     * @example
-     * ```ts
-     * export default defineAgent({
-     *   prewarm: async (proc: JobProcess) => {
-     *     proc.userData.vad = await VAD.load();
-     *   },
-     *   entry: async (ctx: JobContext) => {
-     *     const vad = ctx.proc.userData.vad! as VAD;
-     *     // the rest of your agent logic
-     *   },
-     * });
-     * ```
-     *
-     * @param options -
-     * @returns Promise\<{@link VAD}\>: An instance of the VAD class ready for streaming.
-     */
-    static async load(opts = defaultVADOptions) {
-        const session = await newInferenceSession(opts.forceCPU);
-        return new VAD(session, opts);
-    }
-    stream() {
-        return new VADStream(this.#opts, new OnnxModel(this.#session, this.#opts.sampleRate));
-    }
+class VAD extends baseVAD {
+  #session;
+  #opts;
+  constructor(session, opts) {
+    super({ updateInterval: 32 });
+    this.#session = session;
+    this.#opts = opts;
+  }
+  /**
+   * Load and initialize the Silero VAD model.
+   *
+   * This method loads the ONNX model and prepares it for inference. When options are not provided,
+   * sane defaults are used.
+   *
+   * @remarks
+   * This method may take time to load the model into memory.
+   * It is recommended to call this method inside your prewarm mechanism.
+   *
+   * @example
+   * ```ts
+   * export default defineAgent({
+   *   prewarm: async (proc: JobProcess) => {
+   *     proc.userData.vad = await VAD.load();
+   *   },
+   *   entry: async (ctx: JobContext) => {
+   *     const vad = ctx.proc.userData.vad! as VAD;
+   *     // the rest of your agent logic
+   *   },
+   * });
+   * ```
+   *
+   * @param options -
+   * @returns Promise\<{@link VAD}\>: An instance of the VAD class ready for streaming.
+   */
+  static async load(opts = {}) {
+    const mergedOpts = { ...defaultVADOptions, ...opts };
+    const session = await newInferenceSession(mergedOpts.forceCPU);
+    return new VAD(session, mergedOpts);
+  }
+  stream() {
+    return new VADStream(this.#opts, new OnnxModel(this.#session, this.#opts.sampleRate));
+  }
 }
-export class VADStream extends baseStream {
-    #opts;
-    #model;
-    #task;
-    #expFilter = new ExpFilter(0.35);
-    #extraInferenceTime = 0;
-    #logger = log();
-    constructor(opts, model) {
-        super();
-        this.#opts = opts;
-        this.#model = model;
-        this.#task = new Promise(async () => {
-            let inferenceData = new Float32Array(this.#model.windowSizeSamples);
-            // a copy is exposed to the user in END_OF_SPEECH
-            let speechBuffer = null;
-            let speechBufferMaxReached = false;
-            let speechBufferIndex = 0;
-            // "pub" means public, these values are exposed to the users through events
-            let pubSpeaking = false;
-            let pubSpeechDuration = 0;
-            let pubSilenceDuration = 0;
-            let pubCurrentSample = 0;
-            let pubTimestamp = 0;
-            let pubSampleRate = 0;
-            let pubPrefixPaddingSamples = 0; // size in samples of padding data
-            let speechThresholdDuration = 0;
-            let silenceThresholdDuration = 0;
-            let inputFrames = [];
-            let inferenceFrames = [];
-            let resampler = null;
-            // used to avoid drift when the sampleRate ratio is not an integer
-            let inputCopyRemainingFrac = 0.0;
-            for await (const frame of this.input) {
-                if (typeof frame === 'symbol') {
-                    continue; // ignore flush sentinel for now
-                }
-                if (!pubSampleRate || !speechBuffer) {
-                    pubSampleRate = frame.sampleRate;
-                    pubPrefixPaddingSamples = Math.trunc((this.#opts.prefixPaddingDuration * pubSampleRate) / 1000);
-                    speechBuffer = new Int16Array(this.#opts.maxBufferedSpeech * pubSampleRate + pubPrefixPaddingSamples);
-                    if (this.#opts.sampleRate !== pubSampleRate) {
-                        // resampling needed: the input sample rate isn't the same as the model's
-                        // sample rate used for inference
-                        resampler = new AudioResampler(pubSampleRate, this.#opts.sampleRate, 1, AudioResamplerQuality.QUICK);
-                    }
-                }
-                else if (frame.sampleRate !== pubSampleRate) {
-                    this.#logger.error('a frame with a different sample rate was already published');
-                    continue;
-                }
-                inputFrames.push(frame);
-                if (resampler) {
-                    inferenceFrames.push(...resampler.push(frame));
-                }
-                else {
-                    inferenceFrames.push(frame);
-                }
-                while (true) {
-                    const startTime = process.hrtime.bigint();
-                    const availableInferenceSamples = inferenceFrames
-                        .map((x) => x.samplesPerChannel)
-                        .reduce((acc, x) => acc + x, 0);
-                    if (availableInferenceSamples < this.#model.windowSizeSamples) {
-                        break; // not enough samples to run inference
-                    }
-                    const inputFrame = mergeFrames(inputFrames);
-                    const inferenceFrame = mergeFrames(inferenceFrames);
-                    // convert data to f32
-                    inferenceData = Float32Array.from(inferenceFrame.data.subarray(0, this.#model.windowSizeSamples), (x) => x / 32767);
-                    const p = await this.#model
-                        .run(inferenceData)
-                        .then((data) => this.#expFilter.apply(1, data));
-                    const windowDuration = (this.#model.windowSizeSamples / this.#opts.sampleRate) * 1000;
-                    pubCurrentSample += this.#model.windowSizeSamples;
-                    pubTimestamp += windowDuration;
-                    const resamplingRatio = pubSampleRate / this.#model.sampleRate;
-                    const toCopy = this.#model.windowSizeSamples * resamplingRatio + inputCopyRemainingFrac;
-                    const toCopyInt = Math.trunc(toCopy);
-                    inputCopyRemainingFrac = toCopy - toCopyInt;
-                    // copy the inference window to the speech buffer
-                    const availableSpace = speechBuffer.length - speechBufferIndex;
-                    const toCopyBuffer = Math.min(this.#model.windowSizeSamples, availableSpace);
-                    if (toCopyBuffer > 0) {
-                        speechBuffer.set(inputFrame.data.subarray(0, toCopyBuffer), speechBufferIndex);
-                        speechBufferIndex += toCopyBuffer;
-                    }
-                    else if (!speechBufferMaxReached) {
-                        speechBufferMaxReached = true;
-                        this.#logger.warn('maxBufferedSpeech reached, ignoring further data for the current speech input');
-                    }
-                    const inferenceDuration = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));
-                    this.#extraInferenceTime = Math.max(0, this.#extraInferenceTime + inferenceDuration - windowDuration);
-                    if (this.#extraInferenceTime > SLOW_INFERENCE_THRESHOLD) {
-                        this.#logger
-                            .child({ delay: this.#extraInferenceTime })
-                            .warn('inference is slower than realtime');
-                    }
-                    if (pubSpeaking) {
-                        pubSpeechDuration += inferenceDuration;
-                    }
-                    else {
-                        pubSilenceDuration += inferenceDuration;
-                    }
-                    this.queue.put({
-                        type: VADEventType.INFERENCE_DONE,
-                        samplesIndex: pubCurrentSample,
-                        timestamp: pubTimestamp,
-                        silenceDuration: pubSilenceDuration,
-                        speechDuration: pubSpeechDuration,
-                        probability: p,
-                        inferenceDuration,
-                        frames: [
-                            new AudioFrame(inputFrame.data.subarray(0, toCopyInt), pubSampleRate, 1, toCopyInt),
-                        ],
-                        speaking: pubSpeaking,
-                    });
-                    const resetWriteCursor = () => {
-                        if (!speechBuffer)
-                            throw new Error('speechBuffer is empty');
-                        if (speechBufferIndex <= pubPrefixPaddingSamples) {
-                            return;
-                        }
-                        const paddingData = speechBuffer.subarray(speechBufferIndex - pubPrefixPaddingSamples, speechBufferIndex);
-                        speechBuffer.set(paddingData, 0);
-                        speechBufferIndex = pubPrefixPaddingSamples;
-                        speechBufferMaxReached = false;
-                    };
-                    const copySpeechBuffer = () => {
-                        if (!speechBuffer)
-                            throw new Error('speechBuffer is empty');
-                        return new AudioFrame(speechBuffer.subarray(0, speechBufferIndex), pubSampleRate, 1, speechBufferIndex);
-                    };
-                    if (p > this.#opts.activationThreshold) {
-                        speechThresholdDuration += windowDuration;
-                        silenceThresholdDuration = 0;
-                        if (!pubSpeaking && speechThresholdDuration >= this.#opts.minSpeechDuration) {
-                            pubSpeaking = true;
-                            pubSilenceDuration = 0;
-                            pubSpeechDuration = speechThresholdDuration;
-                            this.queue.put({
-                                type: VADEventType.START_OF_SPEECH,
-                                samplesIndex: pubCurrentSample,
-                                timestamp: pubTimestamp,
-                                silenceDuration: pubSilenceDuration,
-                                speechDuration: pubSpeechDuration,
-                                probability: p,
-                                inferenceDuration,
-                                frames: [copySpeechBuffer()],
-                                speaking: pubSpeaking,
-                            });
-                        }
-                    }
-                    else {
-                        silenceThresholdDuration += windowDuration;
-                        speechThresholdDuration = 0;
-                        if (!pubSpeaking) {
-                            resetWriteCursor();
-                        }
-                        if (pubSpeaking && silenceThresholdDuration > this.#opts.minSilenceDuration) {
-                            pubSpeaking = false;
-                            pubSpeechDuration = 0;
-                            pubSilenceDuration = silenceThresholdDuration;
-                            this.queue.put({
-                                type: VADEventType.END_OF_SPEECH,
-                                samplesIndex: pubCurrentSample,
-                                timestamp: pubTimestamp,
-                                silenceDuration: pubSilenceDuration,
-                                speechDuration: pubSpeechDuration,
-                                probability: p,
-                                inferenceDuration,
-                                frames: [copySpeechBuffer()],
-                                speaking: pubSpeaking,
-                            });
-                            resetWriteCursor();
-                        }
-                    }
-                    inputFrames = [];
-                    inferenceFrames = [];
-                    if (inputFrame.data.length > toCopyInt) {
-                        const data = inputFrame.data.subarray(toCopyInt);
-                        inputFrames.push(new AudioFrame(data, pubSampleRate, 1, Math.trunc(data.length / 2)));
-                    }
-                    if (inferenceFrame.data.length > this.#model.windowSizeSamples) {
-                        const data = inferenceFrame.data.subarray(this.#model.windowSizeSamples);
-                        inferenceFrames.push(new AudioFrame(data, this.#opts.sampleRate, 1, Math.trunc(data.length / 2)));
-                    }
-                }
+class VADStream extends baseStream {
+  #opts;
+  #model;
+  #task;
+  #expFilter = new ExpFilter(0.35);
+  #extraInferenceTime = 0;
+  #logger = log();
+  constructor(opts, model) {
+    super();
+    this.#opts = opts;
+    this.#model = model;
+    this.#task = new Promise(async () => {
+      let inferenceData = new Float32Array(this.#model.windowSizeSamples);
+      let speechBuffer = null;
+      let speechBufferMaxReached = false;
+      let speechBufferIndex = 0;
+      let pubSpeaking = false;
+      let pubSpeechDuration = 0;
+      let pubSilenceDuration = 0;
+      let pubCurrentSample = 0;
+      let pubTimestamp = 0;
+      let pubSampleRate = 0;
+      let pubPrefixPaddingSamples = 0;
+      let speechThresholdDuration = 0;
+      let silenceThresholdDuration = 0;
+      let inputFrames = [];
+      let inferenceFrames = [];
+      let resampler = null;
+      let inputCopyRemainingFrac = 0;
+      for await (const frame of this.input) {
+        if (typeof frame === "symbol") {
+          continue;
+        }
+        if (!pubSampleRate || !speechBuffer) {
+          pubSampleRate = frame.sampleRate;
+          pubPrefixPaddingSamples = Math.trunc(
+            this.#opts.prefixPaddingDuration * pubSampleRate / 1e3
+          );
+          speechBuffer = new Int16Array(
+            this.#opts.maxBufferedSpeech * pubSampleRate + pubPrefixPaddingSamples
+          );
+          if (this.#opts.sampleRate !== pubSampleRate) {
+            resampler = new AudioResampler(
+              pubSampleRate,
+              this.#opts.sampleRate,
+              1,
+              AudioResamplerQuality.QUICK
+              // VAD doesn't need high quality
+            );
+          }
+        } else if (frame.sampleRate !== pubSampleRate) {
+          this.#logger.error("a frame with a different sample rate was already published");
+          continue;
+        }
+        inputFrames.push(frame);
+        if (resampler) {
+          inferenceFrames.push(...resampler.push(frame));
+        } else {
+          inferenceFrames.push(frame);
+        }
+        while (true) {
+          const startTime = process.hrtime.bigint();
+          const availableInferenceSamples = inferenceFrames.map((x) => x.samplesPerChannel).reduce((acc, x) => acc + x, 0);
+          if (availableInferenceSamples < this.#model.windowSizeSamples) {
+            break;
+          }
+          const inputFrame = mergeFrames(inputFrames);
+          const inferenceFrame = mergeFrames(inferenceFrames);
+          inferenceData = Float32Array.from(
+            inferenceFrame.data.subarray(0, this.#model.windowSizeSamples),
+            (x) => x / 32767
+          );
+          const p = await this.#model.run(inferenceData).then((data) => this.#expFilter.apply(1, data));
+          const windowDuration = this.#model.windowSizeSamples / this.#opts.sampleRate * 1e3;
+          pubCurrentSample += this.#model.windowSizeSamples;
+          pubTimestamp += windowDuration;
+          const resamplingRatio = pubSampleRate / this.#model.sampleRate;
+          const toCopy = this.#model.windowSizeSamples * resamplingRatio + inputCopyRemainingFrac;
+          const toCopyInt = Math.trunc(toCopy);
+          inputCopyRemainingFrac = toCopy - toCopyInt;
+          const availableSpace = speechBuffer.length - speechBufferIndex;
+          const toCopyBuffer = Math.min(this.#model.windowSizeSamples, availableSpace);
+          if (toCopyBuffer > 0) {
+            speechBuffer.set(inputFrame.data.subarray(0, toCopyBuffer), speechBufferIndex);
+            speechBufferIndex += toCopyBuffer;
+          } else if (!speechBufferMaxReached) {
+            speechBufferMaxReached = true;
+            this.#logger.warn(
+              "maxBufferedSpeech reached, ignoring further data for the current speech input"
+            );
+          }
+          const inferenceDuration = Number((process.hrtime.bigint() - startTime) / BigInt(1e6));
+          this.#extraInferenceTime = Math.max(
+            0,
+            this.#extraInferenceTime + inferenceDuration - windowDuration
+          );
+          if (this.#extraInferenceTime > SLOW_INFERENCE_THRESHOLD) {
+            this.#logger.child({ delay: this.#extraInferenceTime }).warn("inference is slower than realtime");
+          }
+          if (pubSpeaking) {
+            pubSpeechDuration += inferenceDuration;
+          } else {
+            pubSilenceDuration += inferenceDuration;
+          }
+          this.queue.put({
+            type: VADEventType.INFERENCE_DONE,
+            samplesIndex: pubCurrentSample,
+            timestamp: pubTimestamp,
+            silenceDuration: pubSilenceDuration,
+            speechDuration: pubSpeechDuration,
+            probability: p,
+            inferenceDuration,
+            frames: [
+              new AudioFrame(inputFrame.data.subarray(0, toCopyInt), pubSampleRate, 1, toCopyInt)
+            ],
+            speaking: pubSpeaking
+          });
+          const resetWriteCursor = () => {
+            if (!speechBuffer) throw new Error("speechBuffer is empty");
+            if (speechBufferIndex <= pubPrefixPaddingSamples) {
+              return;
             }
-        });
-    }
+            const paddingData = speechBuffer.subarray(
+              speechBufferIndex - pubPrefixPaddingSamples,
+              speechBufferIndex
+            );
+            speechBuffer.set(paddingData, 0);
+            speechBufferIndex = pubPrefixPaddingSamples;
+            speechBufferMaxReached = false;
+          };
+          const copySpeechBuffer = () => {
+            if (!speechBuffer) throw new Error("speechBuffer is empty");
+            return new AudioFrame(
+              speechBuffer.subarray(0, speechBufferIndex),
+              pubSampleRate,
+              1,
+              speechBufferIndex
+            );
+          };
+          if (p > this.#opts.activationThreshold) {
+            speechThresholdDuration += windowDuration;
+            silenceThresholdDuration = 0;
+            if (!pubSpeaking && speechThresholdDuration >= this.#opts.minSpeechDuration) {
+              pubSpeaking = true;
+              pubSilenceDuration = 0;
+              pubSpeechDuration = speechThresholdDuration;
+              this.queue.put({
+                type: VADEventType.START_OF_SPEECH,
+                samplesIndex: pubCurrentSample,
+                timestamp: pubTimestamp,
+                silenceDuration: pubSilenceDuration,
+                speechDuration: pubSpeechDuration,
+                probability: p,
+                inferenceDuration,
+                frames: [copySpeechBuffer()],
+                speaking: pubSpeaking
+              });
+            }
+          } else {
+            silenceThresholdDuration += windowDuration;
+            speechThresholdDuration = 0;
+            if (!pubSpeaking) {
+              resetWriteCursor();
+            }
+            if (pubSpeaking && silenceThresholdDuration > this.#opts.minSilenceDuration) {
+              pubSpeaking = false;
+              pubSpeechDuration = 0;
+              pubSilenceDuration = silenceThresholdDuration;
+              this.queue.put({
+                type: VADEventType.END_OF_SPEECH,
+                samplesIndex: pubCurrentSample,
+                timestamp: pubTimestamp,
+                silenceDuration: pubSilenceDuration,
+                speechDuration: pubSpeechDuration,
+                probability: p,
+                inferenceDuration,
+                frames: [copySpeechBuffer()],
+                speaking: pubSpeaking
+              });
+              resetWriteCursor();
+            }
+          }
+          inputFrames = [];
+          inferenceFrames = [];
+          if (inputFrame.data.length > toCopyInt) {
+            const data = inputFrame.data.subarray(toCopyInt);
+            inputFrames.push(new AudioFrame(data, pubSampleRate, 1, Math.trunc(data.length / 2)));
+          }
+          if (inferenceFrame.data.length > this.#model.windowSizeSamples) {
+            const data = inferenceFrame.data.subarray(this.#model.windowSizeSamples);
+            inferenceFrames.push(
+              new AudioFrame(data, this.#opts.sampleRate, 1, Math.trunc(data.length / 2))
+            );
+          }
+        }
+      }
+    });
+  }
 }
+export {
+  VAD,
+  VADStream
+};
 //# sourceMappingURL=vad.js.map

package/dist/vad.js.map

@@ -1 +1 @@
-{"version":3,"file":"vad.js","sourceRoot":"","sources":["../src/vad.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EACL,SAAS,EACT,YAAY,EACZ,SAAS,IAAI,UAAU,EACvB,GAAG,IAAI,OAAO,EACd,GAAG,EACH,WAAW,GACZ,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAGtF,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,wBAAwB,GAAG,GAAG,CAAC,CAAC,gBAAgB;AAmBtD,MAAM,iBAAiB,GAAe;IACpC,iBAAiB,EAAE,EAAE;IACrB,kBAAkB,EAAE,GAAG;IACvB,qBAAqB,EAAE,GAAG;IAC1B,iBAAiB,EAAE,KAAK;IACxB,mBAAmB,EAAE,GAAG;IACxB,UAAU,EAAE,KAAK;IACjB,QAAQ,EAAE,IAAI;CACf,CAAC;AAEF,MAAM,OAAO,GAAI,SAAQ,OAAO;IAC9B,QAAQ,CAAmB;IAC3B,KAAK,CAAa;IAElB,YAAY,OAAyB,EAAE,IAAgB;QACrD,KAAK,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,iBAAiB;QACxC,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzD,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IACxF,CAAC;CACF;AAED,MAAM,OAAO,SAAU,SAAQ,UAAU;IACvC,KAAK,CAAa;IAClB,MAAM,CAAY;IAClB,KAAK,CAAgB;IACrB,UAAU,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC;IACjC,mBAAmB,GAAG,CAAC,CAAC;IACxB,OAAO,GAAG,GAAG,EAAE,CAAC;IAEhB,YAAY,IAAgB,EAAE,KAAgB;QAC5C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QAEpB,IAAI,CAAC,KAAK,GAAG,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE;YAClC,IAAI,aAAa,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YAEpE,iDAAiD;YACjD,IAAI,YAAY,GAAsB,IAAI,CAAC;YAC3C,IAAI,sBAAsB,GAAG,KAAK,CAAC;YACnC,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAE1B,2EAA2E;YAC3E,IAAI,WAAW,GAAG,KAAK,CAAC;YACxB,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAC3B,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,IAAI,aAAa,GAAG,CAAC,CAAC;YACtB,IAAI,uBAAuB,GAAG,CAAC,CAAC,CAAC,kCAAkC;YAEnE,IAAI,uBAAuB,GAAG,CAAC,CAAC;YAChC,IAAI,wBAAwB,GAAG,CAAC,CAAC;YAEjC,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,IAAI,eAAe,GAAiB,EAAE,CAAC;YACvC,IAAI,SAAS,GAA0B,IAAI,CAAC;YAE5C,kEAAkE;YAClE,IAAI,sBAAsB,GAAG,GAAG,CAAC;YAEjC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAC9B,SAAS,CAAC,gCAAgC;gBAC5C,CAAC;gBAED,IAAI,CAAC,aAAa,IAAI,CAAC,YAAY,EAAE,CAAC;oBACpC,aAAa,GAAG,KAAK,CAAC,UAAU,CAAC;oBACjC,uBAAuB,GAAG,IAAI,CAAC,KAAK,CAClC,CAAC,IAAI,CAAC,KAAK,CAAC,qBAAqB,GAAG,aAAa,CAAC,GAAG,IAAI,CAC1D,CAAC;oBAEF,YAAY,GAAG,IAAI,UAAU,CAC3B,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,aAAa,GAAG,uBAAuB,CACvE,CAAC;oBAEF,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,KAAK,aAAa,EAAE,CAAC;wBAC5C,yEAAyE;wBACzE,iCAAiC;wBACjC,SAAS,GAAG,IAAI,cAAc,CAC5B,aAAa,EACb,IAAI,CAAC,KAAK,CAAC,UAAU,EACrB,CAAC,EACD,qBAAqB,CAAC,KAAK,CAC5B,CAAC;oBACJ,CAAC;gBACH,CAAC;qBAAM,IAAI,KAAK,CAAC,UAAU,KAAK,aAAa,EAAE,CAAC;oBAC9C,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;oBACjF,SAAS;gBACX,CAAC;gBAED,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACxB,IAAI,SAAS,EAAE,CAAC;oBACd,eAAe,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjD,CAAC;qBAAM,CAAC;oBACN,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC9B,CAAC;gBAED,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC1C,MAAM,yBAAyB,GAAG,eAAe;yBAC9C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC;yBAC/B,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;oBAElC,IAAI,yBAAyB,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;wBAC9D,MAAM,CAAC,sCAAsC;oBAC/C,CAAC;oBAED,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;oBAC5C,MAAM,cAAc,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;oBAEpD,sBAAsB;oBACtB,aAAa,GAAG,YAAY,CAAC,IAAI,CAC/B,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,EAC9D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CACjB,CAAC;oBAEF,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM;yBACxB,GAAG,CAAC,aAAa,CAAC;yBAClB,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;oBAElD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;oBACtF,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC;oBAClD,YAAY,IAAI,cAAc,CAAC;oBAC/B,MAAM,eAAe,GAAG,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;oBAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,GAAG,eAAe,GAAG,sBAAsB,CAAC;oBACxF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBACrC,sBAAsB,GAAG,MAAM,GAAG,SAAS,CAAC;oBAE5C,iDAAiD;oBACjD,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,GAAG,iBAAiB,CAAC;oBAC/D,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;oBAC7E,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;wBACrB,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,iBAAiB,CAAC,CAAC;wBAC/E,iBAAiB,IAAI,YAAY,CAAC;oBACpC,CAAC;yBAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;wBACnC,sBAAsB,GAAG,IAAI,CAAC;wBAC9B,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,+EAA+E,CAChF,CAAC;oBACJ,CAAC;oBAED,MAAM,iBAAiB,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC1F,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CACjC,CAAC,EACD,IAAI,CAAC,mBAAmB,GAAG,iBAAiB,GAAG,cAAc,CAC9D,CAAC;oBACF,IAAI,IAAI,CAAC,mBAAmB,GAAG,wBAAwB,EAAE,CAAC;wBACxD,IAAI,CAAC,OAAO;6BACT,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC;6BAC1C,IAAI,CAAC,mCAAmC,CAAC,CAAC;oBAC/C,CAAC;oBAED,IAAI,WAAW,EAAE,CAAC;wBAChB,iBAAiB,IAAI,iBAAiB,CAAC;oBACzC,CAAC;yBAAM,CAAC;wBACN,kBAAkB,IAAI,iBAAiB,CAAC;oBAC1C,CAAC;oBAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;wBACb,IAAI,EAAE,YAAY,CAAC,cAAc;wBACjC,YAAY,EAAE,gBAAgB;wBAC9B,SAAS,EAAE,YAAY;wBACvB,eAAe,EAAE,kBAAkB;wBACnC,cAAc,EAAE,iBAAiB;wBACjC,WAAW,EAAE,CAAC;wBACd,iBAAiB;wBACjB,MAAM,EAAE;4BACN,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,SAAS,CAAC;yBACpF;wBACD,QAAQ,EAAE,WAAW;qBACtB,CAAC,CAAC;oBAEH,MAAM,gBAAgB,GAAG,GAAG,EAAE;wBAC5B,IAAI,CAAC,YAAY;4BAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;wBAC5D,IAAI,iBAAiB,IAAI,uBAAuB,EAAE,CAAC;4BACjD,OAAO;wBACT,CAAC;wBAED,MAAM,WAAW,GAAG,YAAY,CAAC,QAAQ,CACvC,iBAAiB,GAAG,uBAAuB,EAC3C,iBAAiB,CAClB,CAAC;wBACF,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;wBACjC,iBAAiB,GAAG,uBAAuB,CAAC;wBAC5C,sBAAsB,GAAG,KAAK,CAAC;oBACjC,CAAC,CAAC;oBAEF,MAAM,gBAAgB,GAAG,GAAe,EAAE;wBACxC,IAAI,CAAC,YAAY;4BAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;wBAC5D,OAAO,IAAI,UAAU,CACnB,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,iBAAiB,CAAC,EAC3C,aAAa,EACb,CAAC,EACD,iBAAiB,CAClB,CAAC;oBACJ,CAAC,CAAC;oBAEF,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC;wBACvC,uBAAuB,IAAI,cAAc,CAAC;wBAC1C,wBAAwB,GAAG,CAAC,CAAC;wBAC7B,IAAI,CAAC,WAAW,IAAI,uBAAuB,IAAI,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC;4BAC5E,WAAW,GAAG,IAAI,CAAC;4BACnB,kBAAkB,GAAG,CAAC,CAAC;4BACvB,iBAAiB,GAAG,uBAAuB,CAAC;4BAE5C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;gCACb,IAAI,EAAE,YAAY,CAAC,eAAe;gCAClC,YAAY,EAAE,gBAAgB;gCAC9B,SAAS,EAAE,YAAY;gCACvB,eAAe,EAAE,kBAAkB;gCACnC,cAAc,EAAE,iBAAiB;gCACjC,WAAW,EAAE,CAAC;gCACd,iBAAiB;gCACjB,MAAM,EAAE,CAAC,gBAAgB,EAAE,CAAC;gCAC5B,QAAQ,EAAE,WAAW;6BACtB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,wBAAwB,IAAI,cAAc,CAAC;wBAC3C,uBAAuB,GAAG,CAAC,CAAC;wBAE5B,IAAI,CAAC,WAAW,EAAE,CAAC;4BACjB,gBAAgB,EAAE,CAAC;wBACrB,CAAC;wBAED,IAAI,WAAW,IAAI,wBAAwB,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC;4BAC5E,WAAW,GAAG,KAAK,CAAC;4BACpB,iBAAiB,GAAG,CAAC,CAAC;4BACtB,kBAAkB,GAAG,wBAAwB,CAAC;4BAE9C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;gCACb,IAAI,EAAE,YAAY,CAAC,aAAa;gCAChC,YAAY,EAAE,gBAAgB;gCAC9B,SAAS,EAAE,YAAY;gCACvB,eAAe,EAAE,kBAAkB;gCACnC,cAAc,EAAE,iBAAiB;gCACjC,WAAW,EAAE,CAAC;gCACd,iBAAiB;gCACjB,MAAM,EAAE,CAAC,gBAAgB,EAAE,CAAC;gCAC5B,QAAQ,EAAE,WAAW;6BACtB,CAAC,CAAC;4BAEH,gBAAgB,EAAE,CAAC;wBACrB,CAAC;oBACH,CAAC;oBAED,WAAW,GAAG,EAAE,CAAC;oBACjB,eAAe,GAAG,EAAE,CAAC;oBAErB,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;wBACvC,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;wBACjD,WAAW,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBACxF,CAAC;oBACD,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;wBAC/D,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;wBACzE,eAAe,CAAC,IAAI,CAClB,IAAI,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAC5E,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
+{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n  ExpFilter,\n  VADEventType,\n  VADStream as baseStream,\n  VAD as baseVAD,\n  log,\n  mergeFrames,\n} from '@livekit/agents';\nimport { AudioFrame, AudioResampler, AudioResamplerQuality } from '@livekit/rtc-node';\nimport type { InferenceSession } from 'onnxruntime-node';\nimport type { SampleRate } from './onnx_model.js';\nimport { OnnxModel, newInferenceSession } from './onnx_model.js';\n\nconst SLOW_INFERENCE_THRESHOLD = 200; // late by 200ms\n\nexport interface VADOptions {\n  /** Minimum duration of speech to start a new speech chunk */\n  minSpeechDuration: number;\n  /** At the end of each speech, wait this duration before ending the speech */\n  minSilenceDuration: number;\n  /** Duration of padding to add to the beginning of each speech chunk */\n  prefixPaddingDuration: number;\n  /** Maximum duration of speech to keep in the buffer */\n  maxBufferedSpeech: number;\n  /** Maximum duration of speech to keep in the buffer*/\n  activationThreshold: number;\n  /** Sample rate for the inference (only 8KHz and 16KHz are supported) */\n  sampleRate: SampleRate;\n  /** Force the use of CPU for inference */\n  forceCPU: boolean;\n}\n\nconst defaultVADOptions: VADOptions = {\n  minSpeechDuration: 50,\n  minSilenceDuration: 250,\n  prefixPaddingDuration: 500,\n  maxBufferedSpeech: 60000,\n  activationThreshold: 0.5,\n  sampleRate: 16000,\n  forceCPU: true,\n};\n\nexport class VAD extends baseVAD {\n  #session: InferenceSession;\n  #opts: VADOptions;\n\n  constructor(session: InferenceSession, opts: VADOptions) {\n    super({ updateInterval: 32 });\n    this.#session = session;\n    this.#opts = opts;\n  }\n\n  /**\n   * Load and initialize the Silero VAD model.\n   *\n   * This method loads the ONNX model and prepares it for inference. When options are not provided,\n   * sane defaults are used.\n   *\n   * @remarks\n   * This method may take time to load the model into memory.\n   * It is recommended to call this method inside your prewarm mechanism.\n   *\n   * @example\n   * ```ts\n   * export default defineAgent({\n   *   prewarm: async (proc: JobProcess) => {\n   *     proc.userData.vad = await VAD.load();\n   *   },\n   *   entry: async (ctx: JobContext) => {\n   *     const vad = ctx.proc.userData.vad! as VAD;\n   *     // the rest of your agent logic\n   *   },\n   * });\n   * ```\n   *\n   * @param options -\n   * @returns Promise\\<{@link VAD}\\>: An instance of the VAD class ready for streaming.\n   */\n  static async load(opts: Partial<VADOptions> = {}): Promise<VAD> {\n    const mergedOpts: VADOptions = { ...defaultVADOptions, ...opts };\n    const session = await newInferenceSession(mergedOpts.forceCPU);\n    return new VAD(session, mergedOpts);\n  }\n\n  stream(): VADStream {\n    return new VADStream(this.#opts, new OnnxModel(this.#session, this.#opts.sampleRate));\n  }\n}\n\nexport class VADStream extends baseStream {\n  #opts: VADOptions;\n  #model: OnnxModel;\n  #task: Promise<void>;\n  #expFilter = new ExpFilter(0.35);\n  #extraInferenceTime = 0;\n  #logger = log();\n\n  constructor(opts: VADOptions, model: OnnxModel) {\n    super();\n    this.#opts = opts;\n    this.#model = model;\n\n    this.#task = new Promise(async () => {\n      let inferenceData = new Float32Array(this.#model.windowSizeSamples);\n\n      // a copy is exposed to the user in END_OF_SPEECH\n      let speechBuffer: Int16Array | null = null;\n      let speechBufferMaxReached = false;\n      let speechBufferIndex = 0;\n\n      // \"pub\" means public, these values are exposed to the users through events\n      let pubSpeaking = false;\n      let pubSpeechDuration = 0;\n      let pubSilenceDuration = 0;\n      let pubCurrentSample = 0;\n      let pubTimestamp = 0;\n      let pubSampleRate = 0;\n      let pubPrefixPaddingSamples = 0; // size in samples of padding data\n\n      let speechThresholdDuration = 0;\n      let silenceThresholdDuration = 0;\n\n      let inputFrames = [];\n      let inferenceFrames: AudioFrame[] = [];\n      let resampler: AudioResampler | null = null;\n\n      // used to avoid drift when the sampleRate ratio is not an integer\n      let inputCopyRemainingFrac = 0.0;\n\n      for await (const frame of this.input) {\n        if (typeof frame === 'symbol') {\n          continue; // ignore flush sentinel for now\n        }\n\n        if (!pubSampleRate || !speechBuffer) {\n          pubSampleRate = frame.sampleRate;\n          pubPrefixPaddingSamples = Math.trunc(\n            (this.#opts.prefixPaddingDuration * pubSampleRate) / 1000,\n          );\n\n          speechBuffer = new Int16Array(\n            this.#opts.maxBufferedSpeech * pubSampleRate + pubPrefixPaddingSamples,\n          );\n\n          if (this.#opts.sampleRate !== pubSampleRate) {\n            // resampling needed: the input sample rate isn't the same as the model's\n            // sample rate used for inference\n            resampler = new AudioResampler(\n              pubSampleRate,\n              this.#opts.sampleRate,\n              1,\n              AudioResamplerQuality.QUICK, // VAD doesn't need high quality\n            );\n          }\n        } else if (frame.sampleRate !== pubSampleRate) {\n          this.#logger.error('a frame with a different sample rate was already published');\n          continue;\n        }\n\n        inputFrames.push(frame);\n        if (resampler) {\n          inferenceFrames.push(...resampler.push(frame));\n        } else {\n          inferenceFrames.push(frame);\n        }\n\n        while (true) {\n          const startTime = process.hrtime.bigint();\n          const availableInferenceSamples = inferenceFrames\n            .map((x) => x.samplesPerChannel)\n            .reduce((acc, x) => acc + x, 0);\n\n          if (availableInferenceSamples < this.#model.windowSizeSamples) {\n            break; // not enough samples to run inference\n          }\n\n          const inputFrame = mergeFrames(inputFrames);\n          const inferenceFrame = mergeFrames(inferenceFrames);\n\n          // convert data to f32\n          inferenceData = Float32Array.from(\n            inferenceFrame.data.subarray(0, this.#model.windowSizeSamples),\n            (x) => x / 32767,\n          );\n\n          const p = await this.#model\n            .run(inferenceData)\n            .then((data) => this.#expFilter.apply(1, data));\n\n          const windowDuration = (this.#model.windowSizeSamples / this.#opts.sampleRate) * 1000;\n          pubCurrentSample += this.#model.windowSizeSamples;\n          pubTimestamp += windowDuration;\n          const resamplingRatio = pubSampleRate / this.#model.sampleRate;\n          const toCopy = this.#model.windowSizeSamples * resamplingRatio + inputCopyRemainingFrac;\n          const toCopyInt = Math.trunc(toCopy);\n          inputCopyRemainingFrac = toCopy - toCopyInt;\n\n          // copy the inference window to the speech buffer\n          const availableSpace = speechBuffer.length - speechBufferIndex;\n          const toCopyBuffer = Math.min(this.#model.windowSizeSamples, availableSpace);\n          if (toCopyBuffer > 0) {\n            speechBuffer.set(inputFrame.data.subarray(0, toCopyBuffer), speechBufferIndex);\n            speechBufferIndex += toCopyBuffer;\n          } else if (!speechBufferMaxReached) {\n            speechBufferMaxReached = true;\n            this.#logger.warn(\n              'maxBufferedSpeech reached, ignoring further data for the current speech input',\n            );\n          }\n\n          const inferenceDuration = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));\n          this.#extraInferenceTime = Math.max(\n            0,\n            this.#extraInferenceTime + inferenceDuration - windowDuration,\n          );\n          if (this.#extraInferenceTime > SLOW_INFERENCE_THRESHOLD) {\n            this.#logger\n              .child({ delay: this.#extraInferenceTime })\n              .warn('inference is slower than realtime');\n          }\n\n          if (pubSpeaking) {\n            pubSpeechDuration += inferenceDuration;\n          } else {\n            pubSilenceDuration += inferenceDuration;\n          }\n\n          this.queue.put({\n            type: VADEventType.INFERENCE_DONE,\n            samplesIndex: pubCurrentSample,\n            timestamp: pubTimestamp,\n            silenceDuration: pubSilenceDuration,\n            speechDuration: pubSpeechDuration,\n            probability: p,\n            inferenceDuration,\n            frames: [\n              new AudioFrame(inputFrame.data.subarray(0, toCopyInt), pubSampleRate, 1, toCopyInt),\n            ],\n            speaking: pubSpeaking,\n          });\n\n          const resetWriteCursor = () => {\n            if (!speechBuffer) throw new Error('speechBuffer is empty');\n            if (speechBufferIndex <= pubPrefixPaddingSamples) {\n              return;\n            }\n\n            const paddingData = speechBuffer.subarray(\n              speechBufferIndex - pubPrefixPaddingSamples,\n              speechBufferIndex,\n            );\n            speechBuffer.set(paddingData, 0);\n            speechBufferIndex = pubPrefixPaddingSamples;\n            speechBufferMaxReached = false;\n          };\n\n          const copySpeechBuffer = (): AudioFrame => {\n            if (!speechBuffer) throw new Error('speechBuffer is empty');\n            return new AudioFrame(\n              speechBuffer.subarray(0, speechBufferIndex),\n              pubSampleRate,\n              1,\n              speechBufferIndex,\n            );\n          };\n\n          if (p > this.#opts.activationThreshold) {\n            speechThresholdDuration += windowDuration;\n            silenceThresholdDuration = 0;\n            if (!pubSpeaking && speechThresholdDuration >= this.#opts.minSpeechDuration) {\n              pubSpeaking = true;\n              pubSilenceDuration = 0;\n              pubSpeechDuration = speechThresholdDuration;\n\n              this.queue.put({\n                type: VADEventType.START_OF_SPEECH,\n                samplesIndex: pubCurrentSample,\n                timestamp: pubTimestamp,\n                silenceDuration: pubSilenceDuration,\n                speechDuration: pubSpeechDuration,\n                probability: p,\n                inferenceDuration,\n                frames: [copySpeechBuffer()],\n                speaking: pubSpeaking,\n              });\n            }\n          } else {\n            silenceThresholdDuration += windowDuration;\n            speechThresholdDuration = 0;\n\n            if (!pubSpeaking) {\n              resetWriteCursor();\n            }\n\n            if (pubSpeaking && silenceThresholdDuration > this.#opts.minSilenceDuration) {\n              pubSpeaking = false;\n              pubSpeechDuration = 0;\n              pubSilenceDuration = silenceThresholdDuration;\n\n              this.queue.put({\n                type: VADEventType.END_OF_SPEECH,\n                samplesIndex: pubCurrentSample,\n                timestamp: pubTimestamp,\n                silenceDuration: pubSilenceDuration,\n                speechDuration: pubSpeechDuration,\n                probability: p,\n                inferenceDuration,\n                frames: [copySpeechBuffer()],\n                speaking: pubSpeaking,\n              });\n\n              resetWriteCursor();\n            }\n          }\n\n          inputFrames = [];\n          inferenceFrames = [];\n\n          if (inputFrame.data.length > toCopyInt) {\n            const data = inputFrame.data.subarray(toCopyInt);\n            inputFrames.push(new AudioFrame(data, pubSampleRate, 1, Math.trunc(data.length / 2)));\n          }\n          if (inferenceFrame.data.length > this.#model.windowSizeSamples) {\n            const data = inferenceFrame.data.subarray(this.#model.windowSizeSamples);\n            inferenceFrames.push(\n              new AudioFrame(data, this.#opts.sampleRate, 1, Math.trunc(data.length / 2)),\n            );\n          }\n        }\n      }\n    });\n  }\n}\n"],"mappings":"AAGA;AAAA,EACE;AAAA,EACA;AAAA,EACA,aAAa;AAAA,EACb,OAAO;AAAA,EACP;AAAA,EACA;AAAA,OACK;AACP,SAAS,YAAY,gBAAgB,6BAA6B;AAGlE,SAAS,WAAW,2BAA2B;AAE/C,MAAM,2BAA2B;AAmBjC,MAAM,oBAAgC;AAAA,EACpC,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EACpB,uBAAuB;AAAA,EACvB,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,YAAY;AAAA,EACZ,UAAU;AACZ;AAEO,MAAM,YAAY,QAAQ;AAAA,EAC/B;AAAA,EACA;AAAA,EAEA,YAAY,SAA2B,MAAkB;AACvD,UAAM,EAAE,gBAAgB,GAAG,CAAC;AAC5B,SAAK,WAAW;AAChB,SAAK,QAAQ;AAAA,EACf;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA4BA,aAAa,KAAK,OAA4B,CAAC,GAAiB;AAC9D,UAAM,aAAyB,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC/D,UAAM,UAAU,MAAM,oBAAoB,WAAW,QAAQ;AAC7D,WAAO,IAAI,IAAI,SAAS,UAAU;AAAA,EACpC;AAAA,EAEA,SAAoB;AAClB,WAAO,IAAI,UAAU,KAAK,OAAO,IAAI,UAAU,KAAK,UAAU,KAAK,MAAM,UAAU,CAAC;AAAA,EACtF;AACF;AAEO,MAAM,kBAAkB,WAAW;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAa,IAAI,UAAU,IAAI;AAAA,EAC/B,sBAAsB;AAAA,EACtB,UAAU,IAAI;AAAA,EAEd,YAAY,MAAkB,OAAkB;AAC9C,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,SAAS;AAEd,SAAK,QAAQ,IAAI,QAAQ,YAAY;AACnC,UAAI,gBAAgB,IAAI,aAAa,KAAK,OAAO,iBAAiB;AAGlE,UAAI,eAAkC;AACtC,UAAI,yBAAyB;AAC7B,UAAI,oBAAoB;AAGxB,UAAI,cAAc;AAClB,UAAI,oBAAoB;AACxB,UAAI,qBAAqB;AACzB,UAAI,mBAAmB;AACvB,UAAI,eAAe;AACnB,UAAI,gBAAgB;AACpB,UAAI,0BAA0B;AAE9B,UAAI,0BAA0B;AAC9B,UAAI,2BAA2B;AAE/B,UAAI,cAAc,CAAC;AACnB,UAAI,kBAAgC,CAAC;AACrC,UAAI,YAAmC;AAGvC,UAAI,yBAAyB;AAE7B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,OAAO,UAAU,UAAU;AAC7B;AAAA,QACF;AAEA,YAAI,CAAC,iBAAiB,CAAC,cAAc;AACnC,0BAAgB,MAAM;AACtB,oCAA0B,KAAK;AAAA,YAC5B,KAAK,MAAM,wBAAwB,gBAAiB;AAAA,UACvD;AAEA,yBAAe,IAAI;AAAA,YACjB,KAAK,MAAM,oBAAoB,gBAAgB;AAAA,UACjD;AAEA,cAAI,KAAK,MAAM,eAAe,eAAe;AAG3C,wBAAY,IAAI;AAAA,cACd;AAAA,cACA,KAAK,MAAM;AAAA,cACX;AAAA,cACA,sBAAsB;AAAA;AAAA,YACxB;AAAA,UACF;AAAA,QACF,WAAW,MAAM,eAAe,eAAe;AAC7C,eAAK,QAAQ,MAAM,4DAA4D;AAC/E;AAAA,QACF;AAEA,oBAAY,KAAK,KAAK;AACtB,YAAI,WAAW;AACb,0BAAgB,KAAK,GAAG,UAAU,KAAK,KAAK,CAAC;AAAA,QAC/C,OAAO;AACL,0BAAgB,KAAK,KAAK;AAAA,QAC5B;AAEA,eAAO,MAAM;AACX,gBAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,gBAAM,4BAA4B,gBAC/B,IAAI,CAAC,MAAM,EAAE,iBAAiB,EAC9B,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAC;AAEhC,cAAI,4BAA4B,KAAK,OAAO,mBAAmB;AAC7D;AAAA,UACF;AAEA,gBAAM,aAAa,YAAY,WAAW;AAC1C,gBAAM,iBAAiB,YAAY,eAAe;AAGlD,0BAAgB,aAAa;AAAA,YAC3B,eAAe,KAAK,SAAS,GAAG,KAAK,OAAO,iBAAiB;AAAA,YAC7D,CAAC,MAAM,IAAI;AAAA,UACb;AAEA,gBAAM,IAAI,MAAM,KAAK,OAClB,IAAI,aAAa,EACjB,KAAK,CAAC,SAAS,KAAK,WAAW,MAAM,GAAG,IAAI,CAAC;AAEhD,gBAAM,iBAAkB,KAAK,OAAO,oBAAoB,KAAK,MAAM,aAAc;AACjF,8BAAoB,KAAK,OAAO;AAChC,0BAAgB;AAChB,gBAAM,kBAAkB,gBAAgB,KAAK,OAAO;AACpD,gBAAM,SAAS,KAAK,OAAO,oBAAoB,kBAAkB;AACjE,gBAAM,YAAY,KAAK,MAAM,MAAM;AACnC,mCAAyB,SAAS;AAGlC,gBAAM,iBAAiB,aAAa,SAAS;AAC7C,gBAAM,eAAe,KAAK,IAAI,KAAK,OAAO,mBAAmB,cAAc;AAC3E,cAAI,eAAe,GAAG;AACpB,yBAAa,IAAI,WAAW,KAAK,SAAS,GAAG,YAAY,GAAG,iBAAiB;AAC7E,iCAAqB;AAAA,UACvB,WAAW,CAAC,wBAAwB;AAClC,qCAAyB;AACzB,iBAAK,QAAQ;AAAA,cACX;AAAA,YACF;AAAA,UACF;AAEA,gBAAM,oBAAoB,QAAQ,QAAQ,OAAO,OAAO,IAAI,aAAa,OAAO,GAAO,CAAC;AACxF,eAAK,sBAAsB,KAAK;AAAA,YAC9B;AAAA,YACA,KAAK,sBAAsB,oBAAoB;AAAA,UACjD;AACA,cAAI,KAAK,sBAAsB,0BAA0B;AACvD,iBAAK,QACF,MAAM,EAAE,OAAO,KAAK,oBAAoB,CAAC,EACzC,KAAK,mCAAmC;AAAA,UAC7C;AAEA,cAAI,aAAa;AACf,iCAAqB;AAAA,UACvB,OAAO;AACL,kCAAsB;AAAA,UACxB;AAEA,eAAK,MAAM,IAAI;AAAA,YACb,MAAM,aAAa;AAAA,YACnB,cAAc;AAAA,YACd,WAAW;AAAA,YACX,iBAAiB;AAAA,YACjB,gBAAgB;AAAA,YAChB,aAAa;AAAA,YACb;AAAA,YACA,QAAQ;AAAA,cACN,IAAI,WAAW,WAAW,KAAK,SAAS,GAAG,SAAS,GAAG,eAAe,GAAG,SAAS;AAAA,YACpF;AAAA,YACA,UAAU;AAAA,UACZ,CAAC;AAED,gBAAM,mBAAmB,MAAM;AAC7B,gBAAI,CAAC,aAAc,OAAM,IAAI,MAAM,uBAAuB;AAC1D,gBAAI,qBAAqB,yBAAyB;AAChD;AAAA,YACF;AAEA,kBAAM,cAAc,aAAa;AAAA,cAC/B,oBAAoB;AAAA,cACpB;AAAA,YACF;AACA,yBAAa,IAAI,aAAa,CAAC;AAC/B,gCAAoB;AACpB,qCAAyB;AAAA,UAC3B;AAEA,gBAAM,mBAAmB,MAAkB;AACzC,gBAAI,CAAC,aAAc,OAAM,IAAI,MAAM,uBAAuB;AAC1D,mBAAO,IAAI;AAAA,cACT,aAAa,SAAS,GAAG,iBAAiB;AAAA,cAC1C;AAAA,cACA;AAAA,cACA;AAAA,YACF;AAAA,UACF;AAEA,cAAI,IAAI,KAAK,MAAM,qBAAqB;AACtC,uCAA2B;AAC3B,uCAA2B;AAC3B,gBAAI,CAAC,eAAe,2BAA2B,KAAK,MAAM,mBAAmB;AAC3E,4BAAc;AACd,mCAAqB;AACrB,kCAAoB;AAEpB,mBAAK,MAAM,IAAI;AAAA,gBACb,MAAM,aAAa;AAAA,gBACnB,cAAc;AAAA,gBACd,WAAW;AAAA,gBACX,iBAAiB;AAAA,gBACjB,gBAAgB;AAAA,gBAChB,aAAa;AAAA,gBACb;AAAA,gBACA,QAAQ,CAAC,iBAAiB,CAAC;AAAA,gBAC3B,UAAU;AAAA,cACZ,CAAC;AAAA,YACH;AAAA,UACF,OAAO;AACL,wCAA4B;AAC5B,sCAA0B;AAE1B,gBAAI,CAAC,aAAa;AAChB,+BAAiB;AAAA,YACnB;AAEA,gBAAI,eAAe,2BAA2B,KAAK,MAAM,oBAAoB;AAC3E,4BAAc;AACd,kCAAoB;AACpB,mCAAqB;AAErB,mBAAK,MAAM,IAAI;AAAA,gBACb,MAAM,aAAa;AAAA,gBACnB,cAAc;AAAA,gBACd,WAAW;AAAA,gBACX,iBAAiB;AAAA,gBACjB,gBAAgB;AAAA,gBAChB,aAAa;AAAA,gBACb;AAAA,gBACA,QAAQ,CAAC,iBAAiB,CAAC;AAAA,gBAC3B,UAAU;AAAA,cACZ,CAAC;AAED,+BAAiB;AAAA,YACnB;AAAA,UACF;AAEA,wBAAc,CAAC;AACf,4BAAkB,CAAC;AAEnB,cAAI,WAAW,KAAK,SAAS,WAAW;AACtC,kBAAM,OAAO,WAAW,KAAK,SAAS,SAAS;AAC/C,wBAAY,KAAK,IAAI,WAAW,MAAM,eAAe,GAAG,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC;AAAA,UACtF;AACA,cAAI,eAAe,KAAK,SAAS,KAAK,OAAO,mBAAmB;AAC9D,kBAAM,OAAO,eAAe,KAAK,SAAS,KAAK,OAAO,iBAAiB;AACvE,4BAAgB;AAAA,cACd,IAAI,WAAW,MAAM,KAAK,MAAM,YAAY,GAAG,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,YAC5E;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AACF;","names":[]}