@fonoster/autopilot 0.7.18 → 0.7.19

package/dist/assistants/AssistantSchema.d.ts

@@ -32,11 +32,33 @@ declare const conversationSettingsSchema: z.ZodObject<{
         timeout: number;
         maxTimeoutCount: number;
     }>>>;
+    vad: z.ZodObject<{
+        pathToModel: z.ZodOptional<z.ZodString>;
+        activationThreshold: z.ZodNumber;
+        deactivationThreshold: z.ZodNumber;
+        debounceFrames: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        activationThreshold: number;
+        deactivationThreshold: number;
+        debounceFrames: number;
+        pathToModel?: string | undefined;
+    }, {
+        activationThreshold: number;
+        deactivationThreshold: number;
+        debounceFrames: number;
+        pathToModel?: string | undefined;
+    }>;
 }, "strip", z.ZodTypeAny, {
     firstMessage: string;
     systemTemplate: string;
     goodbyeMessage: string;
     systemErrorMessage: string;
+    vad: {
+        activationThreshold: number;
+        deactivationThreshold: number;
+        debounceFrames: number;
+        pathToModel?: string | undefined;
+    };
     initialDtmf?: string | null | undefined;
     transferOptions?: {
         message: string;
@@ -53,6 +75,12 @@ declare const conversationSettingsSchema: z.ZodObject<{
     systemTemplate: string;
     goodbyeMessage: string;
     systemErrorMessage: string;
+    vad: {
+        activationThreshold: number;
+        deactivationThreshold: number;
+        debounceFrames: number;
+        pathToModel?: string | undefined;
+    };
     initialDtmf?: string | null | undefined;
     transferOptions?: {
         message: string;
@@ -293,11 +321,33 @@ declare const assistantSchema: z.ZodObject<{
             timeout: number;
             maxTimeoutCount: number;
         }>>>;
+        vad: z.ZodObject<{
+            pathToModel: z.ZodOptional<z.ZodString>;
+            activationThreshold: z.ZodNumber;
+            deactivationThreshold: z.ZodNumber;
+            debounceFrames: z.ZodNumber;
+        }, "strip", z.ZodTypeAny, {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        }, {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        }>;
     }, "strip", z.ZodTypeAny, {
         firstMessage: string;
         systemTemplate: string;
         goodbyeMessage: string;
         systemErrorMessage: string;
+        vad: {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        };
         initialDtmf?: string | null | undefined;
         transferOptions?: {
             message: string;
@@ -314,6 +364,12 @@ declare const assistantSchema: z.ZodObject<{
         systemTemplate: string;
         goodbyeMessage: string;
         systemErrorMessage: string;
+        vad: {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        };
         initialDtmf?: string | null | undefined;
         transferOptions?: {
             message: string;
@@ -527,6 +583,12 @@ declare const assistantSchema: z.ZodObject<{
         systemTemplate: string;
         goodbyeMessage: string;
         systemErrorMessage: string;
+        vad: {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        };
         initialDtmf?: string | null | undefined;
         transferOptions?: {
             message: string;
@@ -578,6 +640,12 @@ declare const assistantSchema: z.ZodObject<{
         systemTemplate: string;
         goodbyeMessage: string;
         systemErrorMessage: string;
+        vad: {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        };
         initialDtmf?: string | null | undefined;
         transferOptions?: {
             message: string;

package/dist/assistants/AssistantSchema.js

@@ -43,7 +43,13 @@ const conversationSettingsSchema = zod_1.z.object({
         maxTimeoutCount: zod_1.z.number()
     })
         .optional()
-        .nullable()
+        .nullable(),
+    vad: zod_1.z.object({
+        pathToModel: zod_1.z.string().optional(),
+        activationThreshold: zod_1.z.number(),
+        deactivationThreshold: zod_1.z.number(),
+        debounceFrames: zod_1.z.number()
+    })
 });
 exports.conversationSettingsSchema = conversationSettingsSchema;
 const languageModelConfigSchema = zod_1.z.object({

package/dist/handleVoiceRequest.js

@@ -60,7 +60,7 @@ async function handleVoiceRequest(req, res) {
     const assistantConfig = (0, loadAssistantConfig_1.loadAssistantConfig)();
     const knowledgeBase = await (0, loadKnowledgeBase_1.loadKnowledgeBase)();
     const voice = new _1.VoiceImpl(sessionRef, res);
-    const vad = new _1.SileroVad();
+    const vad = new _1.SileroVad(assistantConfig.conversationSettings.vad);
     const languageModel = (0, createLanguageModel_1.createLanguageModel)({
         voice,
         assistantConfig,

package/dist/vad/SileroVad.d.ts

@@ -1,7 +1,17 @@
 import { Vad } from "./types";
 declare class SileroVad implements Vad {
     private vad;
-    constructor();
+    private params;
+    constructor(params: {
+        pathToModel?: string;
+        activationThreshold: number;
+        deactivationThreshold: number;
+        debounceFrames: number;
+    });
+    pathToModel?: string;
+    activationThreshold: number;
+    deactivationThreshold: number;
+    debounceFrames: number;
     private init;
     processChunk(data: Uint8Array, callback: (event: "SPEECH_START" | "SPEECH_END") => void): void;
 }

package/dist/vad/SileroVad.js

@@ -20,14 +20,18 @@ exports.SileroVad = void 0;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+const logger_1 = require("@fonoster/logger");
 const makeVad_1 = require("./makeVad");
+const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
 class SileroVad {
-    constructor() {
+    constructor(params) {
+        logger.verbose("starting instance of silero vad", { ...params });
+        this.params = params;
         this.init();
     }
     async init() {
         // FIXME: It feels strange to do this in the constructor
-        this.vad = await (0, makeVad_1.makeVad)();
+        this.vad = await (0, makeVad_1.makeVad)(this.params);
     }
     processChunk(data, callback) {
         if (!this.vad) {

package/dist/vad/makeVad.d.ts

@@ -1,2 +1,7 @@
-declare function makeVad(pathToModel?: string): Promise<(chunk: Uint8Array, callback: (event: "SPEECH_START" | "SPEECH_END") => void) => Promise<void>>;
+declare function makeVad(params: {
+    pathToModel?: string;
+    activationThreshold: number;
+    deactivationThreshold: number;
+    debounceFrames: number;
+}): Promise<(chunk: Uint8Array, callback: (event: "SPEECH_START" | "SPEECH_END") => void) => Promise<void>>;
 export { makeVad };

package/dist/vad/makeVad.js

@@ -50,11 +50,14 @@ const chunkToFloat32Array_1 = require("./chunkToFloat32Array");
 const SileroVadModel_1 = require("./SileroVadModel");
 const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
 const BUFFER_SIZE = 16000;
-async function makeVad(pathToModel) {
+async function makeVad(params) {
+    const { pathToModel, activationThreshold, deactivationThreshold, debounceFrames } = params;
     const effectivePath = pathToModel || (0, path_1.join)(__dirname, "..", "..", "silero_vad.onnx");
     const silero = await SileroVadModel_1.SileroVadModel.new(ort, effectivePath);
     let audioBuffer = [];
     let isSpeechActive = false;
+    let consecutiveSpeechFrames = 0;
+    let consecutiveNonSpeechFrames = 0;
     return async function process(chunk, callback) {
         const float32Array = (0, chunkToFloat32Array_1.chunkToFloat32Array)(chunk);
         audioBuffer.push(...float32Array);
@@ -65,20 +68,24 @@ async function makeVad(pathToModel) {
             const remainingBuffer = buffer.slice(BUFFER_SIZE);
             const result = await silero.process(new Float32Array(audioFrame));
             logger.silly("last vad result", { ...result });
-            if (result.isSpeech > 0.5) {
-                if (!isSpeechActive) {
+            if (result.isSpeech > activationThreshold) {
+                consecutiveNonSpeechFrames = 0; // Reset non-speech counter
+                consecutiveSpeechFrames++;
+                if (consecutiveSpeechFrames >= debounceFrames && !isSpeechActive) {
                     isSpeechActive = true;
                     callback("SPEECH_START");
-                    return processBuffer(remainingBuffer);
                 }
             }
-            else if (isSpeechActive) {
-                isSpeechActive = false;
-                callback("SPEECH_END");
-                // WARNING: I'm unsure if this has any effect on the model
-                // but it seems to work fine to ensure the model works optimally
-                silero.resetState();
-                return processBuffer(remainingBuffer);
+            else {
+                consecutiveSpeechFrames = 0; // Reset speech counter
+                consecutiveNonSpeechFrames++;
+                if (consecutiveNonSpeechFrames >= debounceFrames &&
+                    isSpeechActive &&
+                    result.isSpeech < deactivationThreshold) {
+                    isSpeechActive = false;
+                    callback("SPEECH_END");
+                    silero.resetState(); // Reset VAD state after speech ends
+                }
             }
             return processBuffer(remainingBuffer);
         };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fonoster/autopilot",
-  "version": "0.7.18",
+  "version": "0.7.19",
   "description": "Voice AI for the Fonoster platform",
   "author": "Pedro Sanders <psanders@fonoster.com>",
   "homepage": "https://github.com/fonoster/fonoster#readme",
@@ -56,5 +56,5 @@
   "devDependencies": {
     "typescript": "^5.5.4"
   },
-  "gitHead": "4150dcb8086de182d0650df0c6d990ee76658058"
+  "gitHead": "5250aa76f6c4b72a3b26beabe71ea7a7c227d7c1"
 }