@fonoster/autopilot 0.7.17 → 0.7.19

package/README.md

@@ -88,8 +88,9 @@ The Autopilot supports multiple language model providers. The following is a lis
 
 | Provider   | Description                                                | Supported models
 |------------|------------------------------------------------------------|------------------------------------------------------------------------------|
-| OpenAI     | OpenAI provides various GPT models for conversational AI   | `gpt-4o`, `gpt-40-mini` |
+| OpenAI     | OpenAI provides various GPT models for conversational AI   | `gpt-4o`, `gpt-4o-mini`                                                      |
 | Groq       | Groq offers high-performance AI models optimized for speed | `gemm-7b-it`, `llama3-groq-70b-8192-tool-use-preview`, `llama3-1-8b-instant` |
+| Ollama     | Self-hosted Ollama models                                  | `lama3.1`                                                                    |
 
 ## Adding Knowledge Base
 
@@ -141,7 +142,7 @@ You can configure a new tool by adding a new entry in the `tools` array in the c
 
 In addition to the `get` operation type, you can use the `post` operation type. The `post` operation type is used when sending data to the tool. When sending a post, you can optionally set `waitForResponse` to false, which will "fire and forget" the request. The default behavior is to wait for the response.
 
-If your tool needs the number of the caller or the number that received the call, you can use the reserved variables `ingressNumber` and `callerNumber`. Similarly, you can use the reserved variable `callReceivedAt` to get the date and time when the call was received in `ISO 8601` format.
+If your tool needs the number of the caller or the number that received the call, you can use the reserved variables `ingressNumber` and `callerNumber`. Similarly, you can use the reserved variable `callReceivedAt` to get the date and time when the call was received in `ISO 8601` format and the `callDirection` variable to check if the call was originated from the PSTN.
 
 The expected format for the response is a JSON object with the following structure:
 

package/dist/assistants/AssistantSchema.d.ts

@@ -32,11 +32,33 @@ declare const conversationSettingsSchema: z.ZodObject<{
         timeout: number;
         maxTimeoutCount: number;
     }>>>;
+    vad: z.ZodObject<{
+        pathToModel: z.ZodOptional<z.ZodString>;
+        activationThreshold: z.ZodNumber;
+        deactivationThreshold: z.ZodNumber;
+        debounceFrames: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        activationThreshold: number;
+        deactivationThreshold: number;
+        debounceFrames: number;
+        pathToModel?: string | undefined;
+    }, {
+        activationThreshold: number;
+        deactivationThreshold: number;
+        debounceFrames: number;
+        pathToModel?: string | undefined;
+    }>;
 }, "strip", z.ZodTypeAny, {
-    systemTemplate: string;
     firstMessage: string;
+    systemTemplate: string;
     goodbyeMessage: string;
     systemErrorMessage: string;
+    vad: {
+        activationThreshold: number;
+        deactivationThreshold: number;
+        debounceFrames: number;
+        pathToModel?: string | undefined;
+    };
     initialDtmf?: string | null | undefined;
     transferOptions?: {
         message: string;
@@ -49,10 +71,16 @@ declare const conversationSettingsSchema: z.ZodObject<{
         maxTimeoutCount: number;
     } | null | undefined;
 }, {
-    systemTemplate: string;
     firstMessage: string;
+    systemTemplate: string;
     goodbyeMessage: string;
     systemErrorMessage: string;
+    vad: {
+        activationThreshold: number;
+        deactivationThreshold: number;
+        debounceFrames: number;
+        pathToModel?: string | undefined;
+    };
     initialDtmf?: string | null | undefined;
     transferOptions?: {
         message: string;
@@ -129,6 +157,7 @@ declare const languageModelConfigSchema: z.ZodObject<{
             }>;
             required?: string[] | undefined;
         }>;
+        requestStartMessage: z.ZodOptional<z.ZodString>;
         operation: z.ZodEffects<z.ZodObject<{
             type: z.ZodNativeEnum<typeof import("../tools/ToolSchema").AllowedOperations>;
             url: z.ZodOptional<z.ZodString>;
@@ -173,6 +202,7 @@ declare const languageModelConfigSchema: z.ZodObject<{
             waitForResponse?: boolean | undefined;
             headers?: Record<string, string> | undefined;
         };
+        requestStartMessage?: string | undefined;
     }, {
         name: string;
         description: string;
@@ -191,6 +221,7 @@ declare const languageModelConfigSchema: z.ZodObject<{
             waitForResponse?: boolean | undefined;
             headers?: Record<string, string> | undefined;
         };
+        requestStartMessage?: string | undefined;
     }>, "many">;
 }, "strip", z.ZodTypeAny, {
     model: string;
@@ -217,6 +248,7 @@ declare const languageModelConfigSchema: z.ZodObject<{
             waitForResponse?: boolean | undefined;
             headers?: Record<string, string> | undefined;
         };
+        requestStartMessage?: string | undefined;
     }[];
     temperature: number;
     maxTokens: number;
@@ -248,6 +280,7 @@ declare const languageModelConfigSchema: z.ZodObject<{
             waitForResponse?: boolean | undefined;
             headers?: Record<string, string> | undefined;
         };
+        requestStartMessage?: string | undefined;
     }[];
     temperature: number;
     maxTokens: number;
@@ -288,11 +321,33 @@ declare const assistantSchema: z.ZodObject<{
             timeout: number;
             maxTimeoutCount: number;
         }>>>;
+        vad: z.ZodObject<{
+            pathToModel: z.ZodOptional<z.ZodString>;
+            activationThreshold: z.ZodNumber;
+            deactivationThreshold: z.ZodNumber;
+            debounceFrames: z.ZodNumber;
+        }, "strip", z.ZodTypeAny, {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        }, {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        }>;
     }, "strip", z.ZodTypeAny, {
-        systemTemplate: string;
         firstMessage: string;
+        systemTemplate: string;
         goodbyeMessage: string;
         systemErrorMessage: string;
+        vad: {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        };
         initialDtmf?: string | null | undefined;
         transferOptions?: {
             message: string;
@@ -305,10 +360,16 @@ declare const assistantSchema: z.ZodObject<{
             maxTimeoutCount: number;
         } | null | undefined;
     }, {
-        systemTemplate: string;
         firstMessage: string;
+        systemTemplate: string;
         goodbyeMessage: string;
         systemErrorMessage: string;
+        vad: {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        };
         initialDtmf?: string | null | undefined;
         transferOptions?: {
             message: string;
@@ -385,6 +446,7 @@ declare const assistantSchema: z.ZodObject<{
                 }>;
                 required?: string[] | undefined;
             }>;
+            requestStartMessage: z.ZodOptional<z.ZodString>;
             operation: z.ZodEffects<z.ZodObject<{
                 type: z.ZodNativeEnum<typeof import("../tools/ToolSchema").AllowedOperations>;
                 url: z.ZodOptional<z.ZodString>;
@@ -429,6 +491,7 @@ declare const assistantSchema: z.ZodObject<{
                 waitForResponse?: boolean | undefined;
                 headers?: Record<string, string> | undefined;
             };
+            requestStartMessage?: string | undefined;
         }, {
             name: string;
             description: string;
@@ -447,6 +510,7 @@ declare const assistantSchema: z.ZodObject<{
                 waitForResponse?: boolean | undefined;
                 headers?: Record<string, string> | undefined;
             };
+            requestStartMessage?: string | undefined;
         }>, "many">;
     }, "strip", z.ZodTypeAny, {
         model: string;
@@ -473,6 +537,7 @@ declare const assistantSchema: z.ZodObject<{
                 waitForResponse?: boolean | undefined;
                 headers?: Record<string, string> | undefined;
             };
+            requestStartMessage?: string | undefined;
         }[];
         temperature: number;
         maxTokens: number;
@@ -504,6 +569,7 @@ declare const assistantSchema: z.ZodObject<{
                 waitForResponse?: boolean | undefined;
                 headers?: Record<string, string> | undefined;
             };
+            requestStartMessage?: string | undefined;
         }[];
         temperature: number;
         maxTokens: number;
@@ -513,10 +579,16 @@ declare const assistantSchema: z.ZodObject<{
     }>;
 }, "strip", z.ZodTypeAny, {
     conversationSettings: {
-        systemTemplate: string;
         firstMessage: string;
+        systemTemplate: string;
         goodbyeMessage: string;
         systemErrorMessage: string;
+        vad: {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        };
         initialDtmf?: string | null | undefined;
         transferOptions?: {
             message: string;
@@ -554,6 +626,7 @@ declare const assistantSchema: z.ZodObject<{
                 waitForResponse?: boolean | undefined;
                 headers?: Record<string, string> | undefined;
             };
+            requestStartMessage?: string | undefined;
         }[];
         temperature: number;
         maxTokens: number;
@@ -563,10 +636,16 @@ declare const assistantSchema: z.ZodObject<{
     };
 }, {
     conversationSettings: {
-        systemTemplate: string;
         firstMessage: string;
+        systemTemplate: string;
         goodbyeMessage: string;
         systemErrorMessage: string;
+        vad: {
+            activationThreshold: number;
+            deactivationThreshold: number;
+            debounceFrames: number;
+            pathToModel?: string | undefined;
+        };
         initialDtmf?: string | null | undefined;
         transferOptions?: {
             message: string;
@@ -604,6 +683,7 @@ declare const assistantSchema: z.ZodObject<{
                 waitForResponse?: boolean | undefined;
                 headers?: Record<string, string> | undefined;
             };
+            requestStartMessage?: string | undefined;
         }[];
         temperature: number;
         maxTokens: number;

package/dist/assistants/AssistantSchema.js

@@ -43,7 +43,13 @@ const conversationSettingsSchema = zod_1.z.object({
         maxTimeoutCount: zod_1.z.number()
     })
         .optional()
-        .nullable()
+        .nullable(),
+    vad: zod_1.z.object({
+        pathToModel: zod_1.z.string().optional(),
+        activationThreshold: zod_1.z.number(),
+        deactivationThreshold: zod_1.z.number(),
+        debounceFrames: zod_1.z.number()
+    })
 });
 exports.conversationSettingsSchema = conversationSettingsSchema;
 const languageModelConfigSchema = zod_1.z.object({

package/dist/createLanguageModel.d.ts

@@ -1,3 +1,8 @@
-import { AssistantConfig, FilesKnowledgeBase, TelephonyContext } from ".";
-declare function createLanguageModel(assistantConfig: AssistantConfig, knowledgeBase: FilesKnowledgeBase, telephonyContext: TelephonyContext): import("./models/AbstractLanguageModel").AbstractLanguageModel;
+import { AssistantConfig, FilesKnowledgeBase, TelephonyContext, Voice } from ".";
+declare function createLanguageModel(params: {
+    voice: Voice;
+    assistantConfig: AssistantConfig;
+    knowledgeBase: FilesKnowledgeBase;
+    telephonyContext: TelephonyContext;
+}): import("./models/AbstractLanguageModel").AbstractLanguageModel;
 export { createLanguageModel };

package/dist/createLanguageModel.js

@@ -20,14 +20,16 @@ exports.createLanguageModel = createLanguageModel;
  * limitations under the License.
  */
 const _1 = require(".");
-function createLanguageModel(assistantConfig, knowledgeBase, telephonyContext) {
+function createLanguageModel(params) {
+    const { voice, assistantConfig, knowledgeBase, telephonyContext } = params;
     const { languageModel: languageModelSettings, conversationSettings } = assistantConfig;
     return _1.LanguageModelFactory.getLanguageModel(languageModelSettings.provider, {
-        apiKey: languageModelSettings.apiKey,
         // @ts-expect-error don't know the model type here
         model: languageModelSettings.model,
+        apiKey: languageModelSettings.apiKey,
         maxTokens: languageModelSettings.maxTokens,
         temperature: languageModelSettings.temperature,
+        firstMessage: conversationSettings.firstMessage,
         systemTemplate: conversationSettings.systemTemplate,
         baseUrl: languageModelSettings.baseUrl,
         knowledgeBase,
@@ -36,5 +38,5 @@ function createLanguageModel(assistantConfig, knowledgeBase, telephonyContext) {
             _1.hangupToolDefinition,
             _1.transferToolDefinition
         ]
-    }, telephonyContext);
+    }, voice, telephonyContext);
 }

package/dist/handleVoiceRequest.js

@@ -50,15 +50,26 @@ const loadKnowledgeBase_1 = require("./loadKnowledgeBase");
 const _1 = __importStar(require("."));
 const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
 async function handleVoiceRequest(req, res) {
-    const { ingressNumber, sessionRef, appRef } = req;
-    logger.verbose("voice request", { ingressNumber, sessionRef, appRef });
+    const { ingressNumber, sessionRef, appRef, callDirection } = req;
+    logger.verbose("voice request", {
+        ingressNumber,
+        sessionRef,
+        appRef,
+        metadata: req.metadata
+    });
     const assistantConfig = (0, loadAssistantConfig_1.loadAssistantConfig)();
     const knowledgeBase = await (0, loadKnowledgeBase_1.loadKnowledgeBase)();
     const voice = new _1.VoiceImpl(sessionRef, res);
-    const vad = new _1.SileroVad();
-    const languageModel = (0, createLanguageModel_1.createLanguageModel)(assistantConfig, knowledgeBase, {
-        ingressNumber: req.ingressNumber,
-        callerNumber: req.callerNumber
+    const vad = new _1.SileroVad(assistantConfig.conversationSettings.vad);
+    const languageModel = (0, createLanguageModel_1.createLanguageModel)({
+        voice,
+        assistantConfig,
+        knowledgeBase,
+        telephonyContext: {
+            callDirection,
+            ingressNumber: req.ingressNumber,
+            callerNumber: req.callerNumber
+        }
     });
     const autopilot = new _1.default({
         conversationSettings: assistantConfig.conversationSettings,

package/dist/machine/machine.d.ts

@@ -1,25 +1,8 @@
+import { AutopilotContext } from "./types";
 import { ConversationSettings } from "../assistants";
 import { LanguageModel } from "../models";
 import { Voice } from "../voice";
-declare const machine: import("xstate").StateMachine<{
-    sessionRef: string;
-    languageModel: LanguageModel;
-    voice: Voice;
-    firstMessage: string;
-    goodbyeMessage: string;
-    transferMessage?: string;
-    transferPhoneNumber?: string;
-    systemErrorMessage: string;
-    idleMessage: string;
-    idleTimeout: number;
-    idleTimeoutCount: number;
-    maxIdleTimeoutCount: number;
-    speechBuffer: string;
-    speechResponseStartTime: number;
-    speechResponseTime: number;
-    isSpeaking: boolean;
-    knowledgeBaseSourceUrl?: string;
-}, {
+declare const machine: import("xstate").StateMachine<AutopilotContext, {
     type: "SPEECH_START";
 } | {
     type: "SPEECH_END";
@@ -28,7 +11,17 @@ declare const machine: import("xstate").StateMachine<{
     speech: string;
 } | {
     type: "USER_REQUEST_PROCESSED";
-}, {}, never, import("xstate").Values<{
+}, {
+    [x: string]: import("xstate").ActorRefFromLogic<import("xstate").PromiseActorLogic<void, {
+        context: AutopilotContext;
+    }, import("xstate").EventObject>> | undefined;
+}, {
+    src: "doProcessUserRequest";
+    logic: import("xstate").PromiseActorLogic<void, {
+        context: AutopilotContext;
+    }, import("xstate").EventObject>;
+    id: string | undefined;
+}, import("xstate").Values<{
     greetUser: {
         type: "greetUser";
         params: unknown;
@@ -45,10 +38,6 @@ declare const machine: import("xstate").StateMachine<{
         type: "interruptPlayback";
         params: unknown;
     };
-    processUserRequest: {
-        type: "processUserRequest";
-        params: unknown;
-    };
     announceIdleTimeout: {
         type: "announceIdleTimeout";
         params: unknown;
@@ -90,14 +79,39 @@ declare const machine: import("xstate").StateMachine<{
         type: "isNotSpeaking";
         params: unknown;
     };
-}>, "IDLE_TIMEOUT", "hangup" | "greeting" | "idle" | "waitingForUserRequest" | "hackingTimeout" | "updatingSpeech" | "processingUserRequest", string, {
+}>, "IDLE_TIMEOUT", "hangup" | "greeting" | "idle" | "waitingForUserRequest" | "transitioningToIdle" | "updatingSpeech" | "processingUserRequest", string, {
     conversationSettings: ConversationSettings;
     languageModel: LanguageModel;
     voice: Voice;
 }, import("xstate").NonReducibleUnknown, import("xstate").EventObject, import("xstate").MetaObject, {
     readonly context: ({ input }: {
         spawn: {
-            <TSrc extends never>(logic: TSrc, ...[options]: never): import("xstate").ActorRefFromLogic<never>;
+            <TSrc extends "doProcessUserRequest">(logic: TSrc, ...[options]: {
+                src: "doProcessUserRequest";
+                logic: import("xstate").PromiseActorLogic<void, {
+                    context: AutopilotContext;
+                }, import("xstate").EventObject>;
+                id: string | undefined;
+            } extends infer T ? T extends {
+                src: "doProcessUserRequest";
+                logic: import("xstate").PromiseActorLogic<void, {
+                    context: AutopilotContext;
+                }, import("xstate").EventObject>;
+                id: string | undefined;
+            } ? T extends {
+                src: TSrc;
+            } ? import("xstate").ConditionalRequired<[options?: ({
+                id?: T["id"] | undefined;
+                systemId?: string;
+                input?: import("xstate").InputFrom<T["logic"]> | undefined;
+                syncSnapshot?: boolean;
+            } & { [K in import("xstate").RequiredActorOptions<T>]: unknown; }) | undefined], import("xstate").IsNotNever<import("xstate").RequiredActorOptions<T>>> : never : never : never): import("xstate").ActorRefFromLogic<import("xstate").GetConcreteByKey<{
+                src: "doProcessUserRequest";
+                logic: import("xstate").PromiseActorLogic<void, {
+                    context: AutopilotContext;
+                }, import("xstate").EventObject>;
+                id: string | undefined;
+            }, "src", TSrc>["logic"]>;
             <TLogic extends import("xstate").AnyActorLogic>(src: TLogic, options?: {
                 id?: never;
                 systemId?: string;
@@ -110,25 +124,7 @@ declare const machine: import("xstate").StateMachine<{
             languageModel: LanguageModel;
             voice: Voice;
         };
-        self: import("xstate").ActorRef<import("xstate").MachineSnapshot<{
-            sessionRef: string;
-            languageModel: LanguageModel;
-            voice: Voice;
-            firstMessage: string;
-            goodbyeMessage: string;
-            transferMessage?: string;
-            transferPhoneNumber?: string;
-            systemErrorMessage: string;
-            idleMessage: string;
-            idleTimeout: number;
-            idleTimeoutCount: number;
-            maxIdleTimeoutCount: number;
-            speechBuffer: string;
-            speechResponseStartTime: number;
-            speechResponseTime: number;
-            isSpeaking: boolean;
-            knowledgeBaseSourceUrl?: string;
-        }, {
+        self: import("xstate").ActorRef<import("xstate").MachineSnapshot<AutopilotContext, {
             type: "SPEECH_START";
         } | {
             type: "SPEECH_END";
@@ -193,7 +189,7 @@ declare const machine: import("xstate").StateMachine<{
                         readonly type: "idleTimeoutCountExceedsMax";
                     };
                 }, {
-                    readonly target: "hackingTimeout";
+                    readonly target: "transitioningToIdle";
                     readonly actions: readonly [{
                         readonly type: "increaseIdleTimeoutCount";
                     }, {
@@ -219,7 +215,7 @@ declare const machine: import("xstate").StateMachine<{
         readonly hangup: {
             readonly type: "final";
         };
-        readonly hackingTimeout: {
+        readonly transitioningToIdle: {
             readonly always: {
                 readonly target: "idle";
             };
@@ -264,14 +260,47 @@ declare const machine: import("xstate").StateMachine<{
                     readonly target: "waitingForUserRequest";
                     readonly description: "Event from VAD or similar system.";
                 };
-                readonly USER_REQUEST_PROCESSED: {
+            };
+            readonly invoke: {
+                readonly src: "doProcessUserRequest";
+                readonly input: ({ context }: {
+                    context: AutopilotContext;
+                    event: {
+                        type: "SPEECH_START";
+                    } | {
+                        type: "SPEECH_END";
+                    } | {
+                        type: "SPEECH_RESULT";
+                        speech: string;
+                    } | {
+                        type: "USER_REQUEST_PROCESSED";
+                    };
+                    self: import("xstate").ActorRef<import("xstate").MachineSnapshot<AutopilotContext, {
+                        type: "SPEECH_START";
+                    } | {
+                        type: "SPEECH_END";
+                    } | {
+                        type: "SPEECH_RESULT";
+                        speech: string;
+                    } | {
+                        type: "USER_REQUEST_PROCESSED";
+                    }, Record<string, import("xstate").AnyActorRef>, import("xstate").StateValue, string, unknown, any, any>, {
+                        type: "SPEECH_START";
+                    } | {
+                        type: "SPEECH_END";
+                    } | {
+                        type: "SPEECH_RESULT";
+                        speech: string;
+                    } | {
+                        type: "USER_REQUEST_PROCESSED";
+                    }, import("xstate").AnyEventObject>;
+                }) => {
+                    context: AutopilotContext;
+                };
+                readonly onDone: {
                     readonly target: "idle";
-                    readonly description: "Go back home.";
                 };
             };
-            readonly entry: {
-                readonly type: "processUserRequest";
-            };
         };
     };
 }>;

package/dist/machine/machine.js

@@ -56,55 +56,6 @@ const machine = (0, xstate_1.setup)({
             });
             await context.voice.stopSpeech();
         },
-        processUserRequest: async ({ context }) => {
-            logger.verbose("called processUserRequest action", {
-                speechBuffer: context.speechBuffer
-            });
-            // Stop any speech that might be playing
-            await context.voice.stopSpeech();
-            const speech = context.speechBuffer.trim();
-            const languageModel = context.languageModel;
-            const response = await languageModel.invoke(speech);
-            const speechResponseTime = Date.now() - context.speechResponseStartTime;
-            context.speechResponseTime = speechResponseTime;
-            context.speechResponseStartTime = 0;
-            logger.verbose("response from language model", {
-                speechResponseTime
-            });
-            try {
-                if (response.type === "say" && !response.content) {
-                    logger.verbose("call might already be hung up");
-                    (0, xstate_1.raise)({ type: "USER_REQUEST_PROCESSED" });
-                    return;
-                }
-                else if (response.type === "hangup") {
-                    const message = context.goodbyeMessage;
-                    await context.voice.say(message);
-                    await context.voice.hangup();
-                    return;
-                }
-                else if (response.type === "transfer") {
-                    logger.verbose("transferring call to a number in the pstn", {
-                        phoneNumber: context.transferPhoneNumber
-                    });
-                    const message = context.transferMessage;
-                    await context.voice.say(message);
-                    await context.voice.transfer(context.transferPhoneNumber, {
-                        record: true,
-                        timeout: 30
-                    });
-                    return;
-                }
-                await context.voice.say(response.content);
-            }
-            catch (error) {
-                logger.error("error processing user request", {
-                    error
-                });
-                await context.voice.say(context.systemErrorMessage);
-            }
-            (0, xstate_1.raise)({ type: "USER_REQUEST_PROCESSED" });
-        },
         announceIdleTimeout: async ({ context }) => {
             logger.verbose("called announceIdleTimeout action", {
                 idleMessage: context.idleMessage
@@ -174,6 +125,56 @@ const machine = (0, xstate_1.setup)({
         IDLE_TIMEOUT: ({ context }) => {
             return context.idleTimeout;
         }
+    },
+    actors: {
+        doProcessUserRequest: (0, xstate_1.fromPromise)(async ({ input }) => {
+            const { context } = input;
+            logger.verbose("called processUserRequest action", {
+                speechBuffer: context.speechBuffer
+            });
+            // Stop any speech that might be playing
+            await context.voice.stopSpeech();
+            const speech = context.speechBuffer.trim();
+            const languageModel = context.languageModel;
+            const response = await languageModel.invoke(speech);
+            const speechResponseTime = Date.now() - context.speechResponseStartTime;
+            context.speechResponseTime = speechResponseTime;
+            context.speechResponseStartTime = 0;
+            logger.verbose("response from language model", {
+                speechResponseTime
+            });
+            try {
+                if (response.type === "say" && !response.content) {
+                    logger.verbose("call might already be hung up");
+                    return;
+                }
+                else if (response.type === "hangup") {
+                    const message = context.goodbyeMessage;
+                    await context.voice.say(message);
+                    await context.voice.hangup();
+                    return;
+                }
+                else if (response.type === "transfer") {
+                    logger.verbose("transferring call to a number in the pstn", {
+                        phoneNumber: context.transferPhoneNumber
+                    });
+                    const message = context.transferMessage;
+                    await context.voice.say(message);
+                    await context.voice.transfer(context.transferPhoneNumber, {
+                        record: true,
+                        timeout: 30
+                    });
+                    return;
+                }
+                await context.voice.say(response.content);
+            }
+            catch (error) {
+                logger.error("error processing user request", {
+                    error
+                });
+                await context.voice.say(context.systemErrorMessage);
+            }
+        })
     }
 }).createMachine({
     context: ({ input }) => ({
@@ -224,7 +225,7 @@ const machine = (0, xstate_1.setup)({
                         }
                     },
                     {
-                        target: "hackingTimeout",
+                        target: "transitioningToIdle",
                         actions: [
                             {
                                 type: "increaseIdleTimeoutCount"
@@ -259,7 +260,7 @@ const machine = (0, xstate_1.setup)({
         hangup: {
             type: "final"
         },
-        hackingTimeout: {
+        transitioningToIdle: {
             always: {
                 target: "idle"
             }
@@ -309,14 +310,14 @@ const machine = (0, xstate_1.setup)({
                 SPEECH_START: {
                     target: "waitingForUserRequest",
                     description: "Event from VAD or similar system."
-                },
-                USER_REQUEST_PROCESSED: {
-                    target: "idle",
-                    description: "Go back home."
                 }
             },
-            entry: {
-                type: "processUserRequest"
+            invoke: {
+                src: "doProcessUserRequest",
+                input: ({ context }) => ({ context }),
+                onDone: {
+                    target: "idle"
+                }
             }
         }
     }

package/dist/machine/types.d.ts

@@ -0,0 +1,22 @@
+import { LanguageModel } from "../models";
+import { Voice } from "../voice";
+type AutopilotContext = {
+    sessionRef: string;
+    languageModel: LanguageModel;
+    voice: Voice;
+    firstMessage: string;
+    goodbyeMessage: string;
+    transferMessage?: string;
+    transferPhoneNumber?: string;
+    systemErrorMessage: string;
+    idleMessage: string;
+    idleTimeout: number;
+    idleTimeoutCount: number;
+    maxIdleTimeoutCount: number;
+    speechBuffer: string;
+    speechResponseStartTime: number;
+    speechResponseTime: number;
+    isSpeaking: boolean;
+    knowledgeBaseSourceUrl?: string;
+};
+export { AutopilotContext };

package/dist/machine/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/models/AbstractLanguageModel.d.ts

@@ -1,9 +1,11 @@
 import { InvocationResult, LanguageModel, LanguageModelParams, TelephonyContext } from "./types";
+import { Voice } from "../voice";
 declare abstract class AbstractLanguageModel implements LanguageModel {
     private chain;
     private chatHistory;
     private toolsCatalog;
-    constructor(params: LanguageModelParams, telephonyContext: TelephonyContext);
+    private voice;
+    constructor(params: LanguageModelParams, voice: Voice, telephonyContext: TelephonyContext);
     invoke(text: string): Promise<InvocationResult>;
 }
 export { AbstractLanguageModel };

package/dist/models/AbstractLanguageModel.js

@@ -23,24 +23,33 @@ const logger_1 = require("@fonoster/logger");
 const chatHistory_1 = require("./chatHistory");
 const createChain_1 = require("./createChain");
 const createPromptTemplate_1 = require("./createPromptTemplate");
+const toolInvocation_1 = require("./toolInvocation");
 const tools_1 = require("../tools");
 const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
 class AbstractLanguageModel {
-    constructor(params, telephonyContext) {
-        const { model, systemTemplate, knowledgeBase, tools } = params;
-        const promptTemplate = (0, createPromptTemplate_1.createPromptTemplate)(systemTemplate, telephonyContext);
+    constructor(params, voice, telephonyContext) {
+        const { model, firstMessage, systemTemplate, knowledgeBase, tools } = params;
         this.chatHistory = (0, chatHistory_1.createChatHistory)();
         this.toolsCatalog = new tools_1.ToolsCatalog(tools);
+        this.voice = voice;
+        const promptTemplate = (0, createPromptTemplate_1.createPromptTemplate)({
+            firstMessage,
+            systemTemplate,
+            telephonyContext
+        });
         this.chain = (0, createChain_1.createChain)(model, knowledgeBase, promptTemplate, this.chatHistory);
     }
     async invoke(text) {
         const { chain, chatHistory, toolsCatalog } = this;
         const response = (await chain.invoke({ text }));
+        let firstInvocation = true;
         if (response.additional_kwargs?.tool_calls) {
             // eslint-disable-next-line no-loops/no-loops
             for (const toolCall of response.additional_kwargs.tool_calls) {
                 const { arguments: args, name } = toolCall.function;
-                logger.verbose(`invoking tool: ${name} with args: ${args}`);
+                logger.verbose(`invoking tool: ${name} with args: ${args}`, {
+                    firstInvocation
+                });
                 switch (name) {
                     case "hangup":
                         await chatHistory.addAIMessage("tool result: call hangup initiated");
@@ -49,15 +58,15 @@ class AbstractLanguageModel {
                         await chatHistory.addAIMessage("tool result: call transfer initiated");
                         return { type: "transfer" };
                     default:
-                        try {
-                            const toolResult = await toolsCatalog.invokeTool(name, JSON.parse(args));
-                            logger.verbose("tool result: ", toolResult);
-                            await chatHistory.addAIMessage(`tool result: ${toolResult.result}`);
-                        }
-                        catch (error) {
-                            logger.error(`tool error: ${error.message}`);
-                            await chatHistory.addAIMessage(`tool error: ${error.message}`);
-                        }
+                        await (0, toolInvocation_1.toolInvocation)({
+                            args,
+                            chatHistory,
+                            firstInvocation,
+                            toolName: name,
+                            toolsCatalog,
+                            voice: this.voice
+                        });
+                        firstInvocation = false;
                 }
             }
             const finalResponse = (await chain.invoke({

package/dist/models/LanguageModelFactory.d.ts

@@ -4,7 +4,8 @@ import { OllamaParams } from "./ollama";
 import { OpenAIParams } from "./openai";
 import { BaseModelParams, TelephonyContext } from "./types";
 import { LANGUAGE_MODEL_PROVIDER } from "../types";
-type LanguageModelConstructor<T extends BaseModelParams = BaseModelParams> = new (options: T, telephonyContext: TelephonyContext) => AbstractLanguageModel;
+import { Voice } from "../voice";
+type LanguageModelConstructor<T extends BaseModelParams = BaseModelParams> = new (options: T, voice: Voice, telephonyContext: TelephonyContext) => AbstractLanguageModel;
 type LanguageModelConfigMap = {
     [LANGUAGE_MODEL_PROVIDER.OPENAI]: OpenAIParams;
     [LANGUAGE_MODEL_PROVIDER.GROQ]: GroqParams;
@@ -13,6 +14,6 @@ type LanguageModelConfigMap = {
 declare class LanguageModelFactory {
     private static languageModels;
     static registerLanguageModel<T extends BaseModelParams>(name: string, ctor: LanguageModelConstructor<T>): void;
-    static getLanguageModel<T extends keyof LanguageModelConfigMap>(languageModel: T, config: LanguageModelConfigMap[T], telephonyContext: TelephonyContext): AbstractLanguageModel;
+    static getLanguageModel<T extends keyof LanguageModelConfigMap>(languageModel: T, config: LanguageModelConfigMap[T], voice: Voice, telephonyContext: TelephonyContext): AbstractLanguageModel;
 }
 export { LanguageModelFactory };

package/dist/models/LanguageModelFactory.js

@@ -30,12 +30,12 @@ class LanguageModelFactory {
         logger.verbose("registering llm provider", { name });
         this.languageModels.set(name, ctor);
     }
-    static getLanguageModel(languageModel, config, telephonyContext) {
+    static getLanguageModel(languageModel, config, voice, telephonyContext) {
         const LanguageModelConstructor = this.languageModels.get(`llm.${languageModel}`);
         if (!LanguageModelConstructor) {
             throw new Error(`Language model ${languageModel} not found`);
         }
-        return new LanguageModelConstructor(config, telephonyContext);
+        return new LanguageModelConstructor(config, voice, telephonyContext);
     }
 }
 exports.LanguageModelFactory = LanguageModelFactory;

package/dist/models/createPromptTemplate.d.ts

@@ -1,3 +1,7 @@
 import { ChatPromptTemplate } from "@langchain/core/prompts";
 import { TelephonyContext } from "./types";
-export declare function createPromptTemplate(systemTemplate: string, telephonyContext: TelephonyContext): ChatPromptTemplate<any, any>;
+export declare function createPromptTemplate(params: {
+    firstMessage: string;
+    systemTemplate: string;
+    telephonyContext: TelephonyContext;
+}): ChatPromptTemplate<any, any>;

package/dist/models/createPromptTemplate.js

@@ -20,14 +20,17 @@ exports.createPromptTemplate = createPromptTemplate;
  * limitations under the License.
  */
 const prompts_1 = require("@langchain/core/prompts");
-function createPromptTemplate(systemTemplate, telephonyContext) {
+function createPromptTemplate(params) {
+    const { firstMessage, systemTemplate, telephonyContext } = params;
     return prompts_1.ChatPromptTemplate.fromMessages([
         new prompts_1.MessagesPlaceholder("history"),
+        prompts_1.SystemMessagePromptTemplate.fromTemplate(`firstMessage: ${firstMessage}`),
         prompts_1.SystemMessagePromptTemplate.fromTemplate(systemTemplate),
         prompts_1.SystemMessagePromptTemplate.fromTemplate("{context}"),
         prompts_1.SystemMessagePromptTemplate.fromTemplate(`callReceivedAt:${new Date().toISOString()}
        ingressNumber:${telephonyContext.ingressNumber}
-       callerNumber:${telephonyContext.callerNumber}`),
+       callerNumber:${telephonyContext.callerNumber}
+       callDirection:${telephonyContext.callDirection}`),
         prompts_1.HumanMessagePromptTemplate.fromTemplate("{input}")
     ]);
 }

package/dist/models/groq/Groq.d.ts

@@ -1,8 +1,9 @@
 import { GroqParams } from "./types";
+import { Voice } from "../../voice";
 import { AbstractLanguageModel } from "../AbstractLanguageModel";
 import { TelephonyContext } from "../types";
 declare const LANGUAGE_MODEL_NAME = "llm.groq";
 declare class Groq extends AbstractLanguageModel {
-    constructor(params: GroqParams, telephonyContext: TelephonyContext);
+    constructor(params: GroqParams, voice: Voice, telephonyContext: TelephonyContext);
 }
 export { Groq, LANGUAGE_MODEL_NAME };

package/dist/models/groq/Groq.js

@@ -7,7 +7,7 @@ const AbstractLanguageModel_1 = require("../AbstractLanguageModel");
 const LANGUAGE_MODEL_NAME = "llm.groq";
 exports.LANGUAGE_MODEL_NAME = LANGUAGE_MODEL_NAME;
 class Groq extends AbstractLanguageModel_1.AbstractLanguageModel {
-    constructor(params, telephonyContext) {
+    constructor(params, voice, telephonyContext) {
         const model = new groq_1.ChatGroq({
             ...params
         }).bind({
@@ -16,7 +16,7 @@ class Groq extends AbstractLanguageModel_1.AbstractLanguageModel {
         super({
             ...params,
             model
-        }, telephonyContext);
+        }, voice, telephonyContext);
     }
 }
 exports.Groq = Groq;

package/dist/models/ollama/Ollama.d.ts

@@ -1,8 +1,9 @@
 import { OllamaParams } from "./types";
+import { Voice } from "../../voice";
 import { AbstractLanguageModel } from "../AbstractLanguageModel";
 import { TelephonyContext } from "../types";
 declare const LANGUAGE_MODEL_NAME = "llm.ollama";
 declare class Ollama extends AbstractLanguageModel {
-    constructor(params: OllamaParams, telephonyContext: TelephonyContext);
+    constructor(params: OllamaParams, voice: Voice, telephonyContext: TelephonyContext);
 }
 export { LANGUAGE_MODEL_NAME, Ollama };

package/dist/models/ollama/Ollama.js

@@ -7,7 +7,7 @@ const AbstractLanguageModel_1 = require("../AbstractLanguageModel");
 const LANGUAGE_MODEL_NAME = "llm.ollama";
 exports.LANGUAGE_MODEL_NAME = LANGUAGE_MODEL_NAME;
 class Ollama extends AbstractLanguageModel_1.AbstractLanguageModel {
-    constructor(params, telephonyContext) {
+    constructor(params, voice, telephonyContext) {
         const model = new ollama_1.ChatOllama({
             ...params
         }).bind({
@@ -16,7 +16,7 @@ class Ollama extends AbstractLanguageModel_1.AbstractLanguageModel {
         super({
             ...params,
             model
-        }, telephonyContext);
+        }, voice, telephonyContext);
     }
 }
 exports.Ollama = Ollama;

package/dist/models/openai/OpenAI.d.ts

@@ -1,8 +1,9 @@
 import { OpenAIParams } from "./types";
+import { Voice } from "../../voice";
 import { AbstractLanguageModel } from "../AbstractLanguageModel";
 import { TelephonyContext } from "../types";
 declare const LANGUAGE_MODEL_NAME = "llm.openai";
 declare class OpenAI extends AbstractLanguageModel {
-    constructor(params: OpenAIParams, telephonyContext: TelephonyContext);
+    constructor(params: OpenAIParams, voice: Voice, telephonyContext: TelephonyContext);
 }
 export { LANGUAGE_MODEL_NAME, OpenAI };

package/dist/models/openai/OpenAI.js

@@ -7,7 +7,7 @@ const AbstractLanguageModel_1 = require("../AbstractLanguageModel");
 const LANGUAGE_MODEL_NAME = "llm.openai";
 exports.LANGUAGE_MODEL_NAME = LANGUAGE_MODEL_NAME;
 class OpenAI extends AbstractLanguageModel_1.AbstractLanguageModel {
-    constructor(params, telephonyContext) {
+    constructor(params, voice, telephonyContext) {
         const model = new openai_1.ChatOpenAI({
             ...params
         }).bind({
@@ -16,7 +16,7 @@ class OpenAI extends AbstractLanguageModel_1.AbstractLanguageModel {
         super({
             ...params,
             model
-        }, telephonyContext);
+        }, voice, telephonyContext);
     }
 }
 exports.OpenAI = OpenAI;

package/dist/models/toolInvocation.d.ts

@@ -0,0 +1,12 @@
+import { createChatHistory } from "./chatHistory";
+import { ToolsCatalog } from "../tools";
+import { Voice } from "../voice";
+declare function toolInvocation(params: {
+    toolName: string;
+    chatHistory: ReturnType<typeof createChatHistory>;
+    toolsCatalog: ToolsCatalog;
+    firstInvocation: boolean;
+    args: string;
+    voice: Voice;
+}): Promise<void>;
+export { toolInvocation };

package/dist/models/toolInvocation.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toolInvocation = toolInvocation;
+/*
+ * Copyright (C) 2024 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const logger_1 = require("@fonoster/logger");
+const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
+async function toolInvocation(params) {
+    const { firstInvocation, voice, args, toolName, chatHistory, toolsCatalog } = params;
+    try {
+        if (firstInvocation) {
+            const tool = toolsCatalog.getTool(toolName);
+            const message = tool?.requestStartMessage ?? "";
+            if (message) {
+                await voice.say(message);
+            }
+        }
+        const toolResult = await toolsCatalog.invokeTool(toolName, JSON.parse(args));
+        logger.verbose("tool result: ", toolResult);
+        await chatHistory.addAIMessage(`tool result: ${toolResult.result}`);
+    }
+    catch (error) {
+        logger.error(`tool error: ${error.message}`);
+        await chatHistory.addAIMessage(`tool error: ${error.message}`);
+    }
+}

package/dist/models/types.d.ts

@@ -1,3 +1,4 @@
+import { CallDirection } from "@fonoster/types";
 import { BaseChatModel } from "@langchain/core/language_models/chat_models";
 import { KnowledgeBase } from "../knowledge";
 import { Tool } from "../tools/type";
@@ -5,6 +6,7 @@ type LanguageModel = {
     invoke: (text: string) => Promise<InvocationResult>;
 };
 type BaseModelParams = {
+    firstMessage: string;
     systemTemplate: string;
     knowledgeBase: KnowledgeBase;
     tools: Tool[];
@@ -18,6 +20,7 @@ type InvocationResult = {
     content?: string;
 };
 type TelephonyContext = {
+    callDirection: CallDirection;
     ingressNumber: string;
     callerNumber: string;
 };

package/dist/tools/ToolSchema.d.ts

@@ -48,6 +48,7 @@ declare const toolSchema: z.ZodObject<{
         }>;
         required?: string[] | undefined;
     }>;
+    requestStartMessage: z.ZodOptional<z.ZodString>;
     operation: z.ZodEffects<z.ZodObject<{
         type: z.ZodNativeEnum<typeof AllowedOperations>;
         url: z.ZodOptional<z.ZodString>;
@@ -92,6 +93,7 @@ declare const toolSchema: z.ZodObject<{
         waitForResponse?: boolean | undefined;
         headers?: Record<string, string> | undefined;
     };
+    requestStartMessage?: string | undefined;
 }, {
     name: string;
     description: string;
@@ -110,5 +112,6 @@ declare const toolSchema: z.ZodObject<{
         waitForResponse?: boolean | undefined;
         headers?: Record<string, string> | undefined;
     };
+    requestStartMessage?: string | undefined;
 }>;
 export { AllowedOperations, toolSchema };

package/dist/tools/ToolSchema.js

@@ -45,6 +45,7 @@ const toolSchema = zod_1.z.object({
         properties: zod_1.z.record(propertySchema),
         required: zod_1.z.array(zod_1.z.string()).optional()
     }),
+    requestStartMessage: zod_1.z.string().optional(),
     operation: zod_1.z
         .object({
         type: zod_1.z.nativeEnum(AllowedOperations),

package/dist/tools/ToolsCatalog.d.ts

@@ -5,6 +5,7 @@ declare class ToolsCatalog {
     invokeTool(toolName: string, args: Record<string, unknown>): Promise<{
         result: string;
     }>;
+    getTool(toolName: string): Tool | undefined;
     addTool(toolDef: Tool): void;
     listTools(): Tool[];
 }

package/dist/tools/ToolsCatalog.js

@@ -43,6 +43,9 @@ class ToolsCatalog {
             body: args
         });
     }
+    getTool(toolName) {
+        return this.tools.get(toolName);
+    }
     addTool(toolDef) {
         this.tools.set(toolDef.name, toolDef);
     }

package/dist/tools/sendRequest.js

@@ -19,18 +19,24 @@ exports.sendRequest = sendRequest;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+const logger_1 = require("@fonoster/logger");
 const zod_1 = require("zod");
 const ToolSchema_1 = require("./ToolSchema");
 const responseSchema = zod_1.z.object({
     result: zod_1.z.string()
 });
+const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
 async function sendRequest(input) {
     const { url, method, body, headers, waitForResponse } = input;
     const options = {
         method,
-        headers,
+        headers: {
+            "Content-Type": "application/json",
+            ...headers
+        },
         body: method === ToolSchema_1.AllowedOperations.POST ? JSON.stringify(body) : undefined
     };
+    logger.verbose(`sending request to ${url}`, { body, method });
     if (waitForResponse && method === ToolSchema_1.AllowedOperations.POST) {
         setTimeout(() => fetch(url, options), 0);
         return { result: "request sent" };

package/dist/vad/SileroVad.d.ts

@@ -1,7 +1,17 @@
 import { Vad } from "./types";
 declare class SileroVad implements Vad {
     private vad;
-    constructor();
+    private params;
+    constructor(params: {
+        pathToModel?: string;
+        activationThreshold: number;
+        deactivationThreshold: number;
+        debounceFrames: number;
+    });
+    pathToModel?: string;
+    activationThreshold: number;
+    deactivationThreshold: number;
+    debounceFrames: number;
     private init;
     processChunk(data: Uint8Array, callback: (event: "SPEECH_START" | "SPEECH_END") => void): void;
 }

package/dist/vad/SileroVad.js

@@ -20,14 +20,18 @@ exports.SileroVad = void 0;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+const logger_1 = require("@fonoster/logger");
 const makeVad_1 = require("./makeVad");
+const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
 class SileroVad {
-    constructor() {
+    constructor(params) {
+        logger.verbose("starting instance of silero vad", { ...params });
+        this.params = params;
         this.init();
     }
     async init() {
         // FIXME: It feels strange to do this in the constructor
-        this.vad = await (0, makeVad_1.makeVad)();
+        this.vad = await (0, makeVad_1.makeVad)(this.params);
     }
     processChunk(data, callback) {
         if (!this.vad) {

package/dist/vad/makeVad.d.ts

@@ -1,2 +1,7 @@
-declare function makeVad(pathToModel?: string): Promise<(chunk: Uint8Array, callback: (event: "SPEECH_START" | "SPEECH_END") => void) => Promise<void>>;
+declare function makeVad(params: {
+    pathToModel?: string;
+    activationThreshold: number;
+    deactivationThreshold: number;
+    debounceFrames: number;
+}): Promise<(chunk: Uint8Array, callback: (event: "SPEECH_START" | "SPEECH_END") => void) => Promise<void>>;
 export { makeVad };

package/dist/vad/makeVad.js

@@ -50,11 +50,14 @@ const chunkToFloat32Array_1 = require("./chunkToFloat32Array");
 const SileroVadModel_1 = require("./SileroVadModel");
 const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
 const BUFFER_SIZE = 16000;
-async function makeVad(pathToModel) {
+async function makeVad(params) {
+    const { pathToModel, activationThreshold, deactivationThreshold, debounceFrames } = params;
     const effectivePath = pathToModel || (0, path_1.join)(__dirname, "..", "..", "silero_vad.onnx");
     const silero = await SileroVadModel_1.SileroVadModel.new(ort, effectivePath);
     let audioBuffer = [];
     let isSpeechActive = false;
+    let consecutiveSpeechFrames = 0;
+    let consecutiveNonSpeechFrames = 0;
     return async function process(chunk, callback) {
         const float32Array = (0, chunkToFloat32Array_1.chunkToFloat32Array)(chunk);
         audioBuffer.push(...float32Array);
@@ -65,20 +68,24 @@ async function makeVad(pathToModel) {
             const remainingBuffer = buffer.slice(BUFFER_SIZE);
             const result = await silero.process(new Float32Array(audioFrame));
             logger.silly("last vad result", { ...result });
-            if (result.isSpeech > 0.5) {
-                if (!isSpeechActive) {
+            if (result.isSpeech > activationThreshold) {
+                consecutiveNonSpeechFrames = 0; // Reset non-speech counter
+                consecutiveSpeechFrames++;
+                if (consecutiveSpeechFrames >= debounceFrames && !isSpeechActive) {
                     isSpeechActive = true;
                     callback("SPEECH_START");
-                    return processBuffer(remainingBuffer);
                 }
             }
-            else if (isSpeechActive) {
-                isSpeechActive = false;
-                callback("SPEECH_END");
-                // WARNING: I'm unsure if this has any effect on the model
-                // but it seems to work fine to ensure the model works optimally
-                silero.resetState();
-                return processBuffer(remainingBuffer);
+            else {
+                consecutiveSpeechFrames = 0; // Reset speech counter
+                consecutiveNonSpeechFrames++;
+                if (consecutiveNonSpeechFrames >= debounceFrames &&
+                    isSpeechActive &&
+                    result.isSpeech < deactivationThreshold) {
+                    isSpeechActive = false;
+                    callback("SPEECH_END");
+                    silero.resetState(); // Reset VAD state after speech ends
+                }
             }
             return processBuffer(remainingBuffer);
         };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fonoster/autopilot",
-  "version": "0.7.17",
+  "version": "0.7.19",
   "description": "Voice AI for the Fonoster platform",
   "author": "Pedro Sanders <psanders@fonoster.com>",
   "homepage": "https://github.com/fonoster/fonoster#readme",
@@ -35,10 +35,10 @@
     "url": "https://github.com/fonoster/fonoster/issues"
   },
   "dependencies": {
-    "@fonoster/common": "^0.7.16",
-    "@fonoster/logger": "^0.7.16",
-    "@fonoster/types": "^0.7.16",
-    "@fonoster/voice": "^0.7.16",
+    "@fonoster/common": "^0.7.18",
+    "@fonoster/logger": "^0.7.18",
+    "@fonoster/types": "^0.7.18",
+    "@fonoster/voice": "^0.7.18",
     "@langchain/community": "^0.2.31",
     "@langchain/core": "^0.2.32",
     "@langchain/groq": "^0.0.17",
@@ -56,5 +56,5 @@
   "devDependencies": {
     "typescript": "^5.5.4"
   },
-  "gitHead": "4408dbb6ca8aafe086ac86ebd6b5daa255e8c99f"
+  "gitHead": "5250aa76f6c4b72a3b26beabe71ea7a7c227d7c1"
 }