@fonoster/autopilot 0.8.52 → 0.8.59

package/dist/Autopilot.js

@@ -39,7 +39,7 @@ class Autopilot {
         this.actor = (0, xstate_1.createActor)(machine_1.machine, {
             input: {
                 conversationSettings,
-                languageModel: languageModel,
+                languageModel,
                 voice
             }
         });

package/dist/handleVoiceRequest.js

@@ -58,7 +58,7 @@ exports.handleVoiceRequest = handleVoiceRequest;
  */
 const common_1 = require("@fonoster/common");
 const logger_1 = require("@fonoster/logger");
-const createLanguageModel_1 = require("./createLanguageModel");
+const createLanguageModel_1 = require("./models/createLanguageModel");
 const envs_1 = require("./envs");
 const loadAssistantConfigFromFile_1 = require("./loadAssistantConfigFromFile");
 const _1 = __importStar(require("."));
@@ -66,7 +66,7 @@ const loadAssistantFromAPI_1 = require("./loadAssistantFromAPI");
 const fs_1 = __importDefault(require("fs"));
 const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
 async function handleVoiceRequest(req, res) {
-    const { accessKeyId, ingressNumber, sessionRef, appRef, callDirection } = req;
+    const { accessKeyId, callerNumber, ingressNumber, sessionRef, appRef, callDirection } = req;
     logger.verbose("voice request", {
         accessKeyId,
         ingressNumber,
@@ -105,8 +105,8 @@ async function handleVoiceRequest(req, res) {
         knowledgeBase,
         telephonyContext: {
             callDirection,
-            ingressNumber: req.ingressNumber,
-            callerNumber: req.callerNumber
+            ingressNumber,
+            callerNumber
         }
     });
     const { conversationSettings } = assistantConfig;

package/dist/models/AbstractLanguageModel.js

@@ -42,44 +42,63 @@ class AbstractLanguageModel {
     async invoke(text) {
         const { chain, chatHistory, toolsCatalog } = this;
         const response = (await chain.invoke({ text }));
-        let firstInvocation = true;
+        let isFirstTool = true;
+        logger.verbose("invoke", { text });
+        logger.verbose("response", { content: response.content });
+        logger.verbose("tools?", {
+            hasTools: response.tool_calls?.length > 0,
+            tools: response.tool_calls?.map((tool) => tool.name)
+        });
         if (response.tool_calls && response.tool_calls.length > 0) {
             // eslint-disable-next-line no-loops/no-loops
             for (const toolCall of response.tool_calls) {
-                const { args, name } = toolCall;
-                logger.verbose(`invoking tool: ${name} with args: ${JSON.stringify(args)}`, {
-                    firstInvocation
+                const { args, name: toolName } = toolCall;
+                logger.verbose(`invoking tool: ${toolName} with args: ${JSON.stringify(args)}`, {
+                    isFirstTool
                 });
-                switch (name) {
+                switch (toolName) {
                     case "hangup":
                         await chatHistory.addAIMessage("tool result: call hangup initiated");
-                        return { type: "hangup" };
+                        return {
+                            type: "hangup",
+                            content: "tool result: call hangup initiated",
+                            toolCalls: response.tool_calls
+                        };
                     case "transfer":
                         await chatHistory.addAIMessage("tool result: call transfer initiated");
-                        return { type: "transfer" };
+                        return {
+                            type: "transfer",
+                            content: "tool result: call transfer initiated",
+                            toolCalls: response.tool_calls
+                        };
                     default:
+                        if (isFirstTool) {
+                            const tool = toolsCatalog.getTool(toolName);
+                            await this.voice.say(tool?.requestStartMessage ?? "");
+                        }
                         await (0, toolInvocation_1.toolInvocation)({
                             args,
                             chatHistory,
-                            firstInvocation,
-                            toolName: name,
-                            toolsCatalog,
-                            voice: this.voice
+                            isFirstTool,
+                            toolName,
+                            toolsCatalog
                         });
-                        firstInvocation = false;
+                        isFirstTool = false;
                 }
             }
             const finalResponse = (await chain.invoke({
-                text: "Please provide a final response based on the tool's results."
+                text: "Write a quick message based on the tools results"
             }));
-            response.content = finalResponse.content ?? "";
+            logger.verbose("finalResponse by AI", { content: finalResponse.content });
+            response.content = finalResponse.content?.toString() ?? "";
         }
         await chatHistory.addUserMessage(text);
         await chatHistory.addAIMessage(response.content?.toString() ?? "");
         logger.verbose("system will say", { content: response.content });
         return {
             type: "say",
-            content: response.content.toString()
+            content: response.content.toString(),
+            toolCalls: response.tool_calls
         };
     }
 }

package/dist/{createLanguageModel.d.ts → models/createLanguageModel.d.ts}

@@ -1,8 +1,8 @@
-import { AssistantConfig, KnowledgeBase, TelephonyContext, Voice } from ".";
+import { AssistantConfig, KnowledgeBase, TelephonyContext, Voice } from "..";
 declare function createLanguageModel(params: {
     voice: Voice;
     assistantConfig: AssistantConfig;
     knowledgeBase: KnowledgeBase;
     telephonyContext: TelephonyContext;
-}): import("./models/AbstractLanguageModel").AbstractLanguageModel;
+}): import("./AbstractLanguageModel").AbstractLanguageModel;
 export { createLanguageModel };

package/dist/{createLanguageModel.js → models/createLanguageModel.js}

@@ -20,22 +20,17 @@ exports.createLanguageModel = createLanguageModel;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-const _1 = require(".");
+const __1 = require("..");
 function createLanguageModel(params) {
     const { voice, assistantConfig, knowledgeBase, telephonyContext } = params;
     const { languageModel: languageModelSettings, conversationSettings } = assistantConfig;
-    // Ensure that the transfer tool is only added if the transfer options exist
+    // The transfer tool is only added if the transfer options exist
     const tools = languageModelSettings.tools.concat(assistantConfig.conversationSettings.transferOptions
-        ? [_1.hangupToolDefinition, _1.transferToolDefinition]
-        : [_1.hangupToolDefinition]);
-    return _1.LanguageModelFactory.getLanguageModel(languageModelSettings.provider, {
-        model: languageModelSettings.model,
-        apiKey: languageModelSettings.apiKey,
-        maxTokens: languageModelSettings.maxTokens,
-        temperature: languageModelSettings.temperature,
-        firstMessage: conversationSettings.firstMessage,
-        systemPrompt: conversationSettings.systemPrompt,
-        baseUrl: languageModelSettings.baseUrl,
+        ? [__1.hangupToolDefinition, __1.transferToolDefinition]
+        : [__1.hangupToolDefinition]);
+    return __1.LanguageModelFactory.getLanguageModel(languageModelSettings.provider, {
+        ...languageModelSettings,
+        ...conversationSettings,
         knowledgeBase,
         tools
     }, voice, telephonyContext);

package/dist/models/evaluations/createTestTextSimilarity.d.ts

@@ -0,0 +1,6 @@
+export declare function createTestTextSimilarity(evalsLanguageModel: {
+    provider: any;
+    model: string;
+    baseUrl?: string;
+    apiKey?: string;
+}, systemPrompt: string): (text1: string, text2: string) => Promise<boolean>;

package/dist/models/evaluations/createTestTextSimilarity.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTestTextSimilarity = createTestTextSimilarity;
+/*
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const messages_1 = require("@langchain/core/messages");
+const openai_1 = require("@langchain/openai");
+function createTestTextSimilarity(evalsLanguageModel, systemPrompt) {
+    if (!evalsLanguageModel.apiKey) {
+        throw new Error("API key is required for text similarity evaluation.");
+    }
+    return async function testTextSimilarity(text1, text2) {
+        const llm = new openai_1.ChatOpenAI({
+            modelName: evalsLanguageModel.model,
+            temperature: 0,
+            openAIApiKey: evalsLanguageModel.apiKey,
+            maxTokens: 10
+        });
+        const messages = [
+            new messages_1.SystemMessage(systemPrompt),
+            new messages_1.HumanMessage(`Text 1: ${text1}\nText 2: ${text2}`)
+        ];
+        const response = await llm.invoke(messages);
+        const reply = response.content?.toString().trim().toLowerCase();
+        return new Boolean(reply).valueOf();
+    };
+}

package/dist/models/evaluations/evals.d.ts

@@ -0,0 +1,3 @@
+import { AssistantConfig } from "../../assistants/types";
+import { ScenarioEvaluationReport } from "./types";
+export declare function evalTestCases(assistantConfig: AssistantConfig): Promise<ScenarioEvaluationReport[]>;

package/dist/models/evaluations/evals.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evalTestCases = evalTestCases;
+const createLanguageModel_1 = require("../createLanguageModel");
+const createTestTextSimilarity_1 = require("./createTestTextSimilarity");
+const evaluateScenario_1 = require("./evaluateScenario");
+const textSimilaryPrompt_1 = require("./textSimilaryPrompt");
+async function evalTestCases(assistantConfig) {
+    const { testCases } = assistantConfig;
+    const voice = {
+        say: async (_) => { }
+    };
+    const evaluationReports = [];
+    for (const scenario of testCases?.scenarios) {
+        const languageModel = (0, createLanguageModel_1.createLanguageModel)({
+            voice,
+            assistantConfig,
+            knowledgeBase: {
+                load: async () => { },
+                queryKnowledgeBase: async (query, k) => query
+            },
+            telephonyContext: scenario.telephonyContext
+        });
+        const testTextSimilarity = (0, createTestTextSimilarity_1.createTestTextSimilarity)({
+            provider: assistantConfig.testCases?.evalsLanguageModel?.provider,
+            model: assistantConfig.testCases?.evalsLanguageModel?.model,
+            apiKey: assistantConfig.testCases?.evalsLanguageModel?.apiKey
+        }, assistantConfig.testCases?.evalsSystemPrompt || textSimilaryPrompt_1.textSimilaryPrompt);
+        const evaluationReport = await (0, evaluateScenario_1.evaluateScenario)({
+            assistantConfig,
+            scenario,
+            languageModel,
+            testTextSimilarity
+        });
+        evaluationReports.push(evaluationReport);
+    }
+    return evaluationReports;
+}

package/dist/models/evaluations/evaluateScenario.d.ts

@@ -0,0 +1,2 @@
+import { ScenarioEvaluationConfig, ScenarioEvaluationReport } from "./types";
+export declare function evaluateScenario(config: ScenarioEvaluationConfig): Promise<ScenarioEvaluationReport>;

package/dist/models/evaluations/evaluateScenario.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluateScenario = evaluateScenario;
+const evaluateStep_1 = require("./evaluateStep");
+async function evaluateScenario(config) {
+    const { scenario, languageModel, testTextSimilarity, assistantConfig } = config;
+    const results = [];
+    for (const step of scenario.conversation) {
+        const stepResult = await (0, evaluateStep_1.evaluateStep)({
+            step,
+            languageModel,
+            testTextSimilarity,
+            assistantConfig
+        });
+        results.push(stepResult);
+    }
+    const overallPassed = results.every((step) => step.passed);
+    return {
+        scenarioRef: scenario.ref,
+        overallPassed,
+        steps: results
+    };
+}

package/dist/models/evaluations/evaluateStep.d.ts

@@ -0,0 +1,2 @@
+import { EvaluateStepParams, StepEvaluationReport } from "./types";
+export declare function evaluateStep({ step, languageModel, testTextSimilarity, assistantConfig }: EvaluateStepParams): Promise<StepEvaluationReport>;

package/dist/models/evaluations/evaluateStep.js

@@ -0,0 +1,55 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluateStep = evaluateStep;
+const evaluateTextResponse_1 = require("./evaluateTextResponse");
+const evaluateToolCalls_1 = require("./evaluateToolCalls");
+async function evaluateStep({ step, languageModel, testTextSimilarity, assistantConfig }) {
+    const stepResult = {
+        humanInput: step.userInput,
+        expectedResponse: step.expected.text.response,
+        aiResponse: "", // will be filled if invoke is successful
+        evaluationType: step.expected.text.type,
+        passed: true
+    };
+    try {
+        const response = await languageModel.invoke(step.userInput);
+        // Hangup and transfer are special cases
+        if (response.toolCalls && response.toolCalls.length > 0) {
+            const topTool = response.toolCalls[0];
+            if (topTool.name === "hangup") {
+                stepResult.aiResponse =
+                    assistantConfig.conversationSettings?.goodbyeMessage;
+            }
+            else if (topTool.name === "transfer") {
+                stepResult.aiResponse =
+                    assistantConfig.conversationSettings?.transferOptions?.message ?? "";
+            }
+            else {
+                stepResult.aiResponse = response.content;
+            }
+        }
+        else {
+            stepResult.aiResponse = response.content;
+        }
+        const textEvaluation = await (0, evaluateTextResponse_1.evaluateTextResponse)(step.expected.text, stepResult.aiResponse, testTextSimilarity);
+        if (!textEvaluation.passed) {
+            stepResult.passed = false;
+            stepResult.errorMessage = textEvaluation.errorMessage;
+        }
+        if (step.expected.tools && step.expected.tools.length > 0) {
+            const toolsEvaluation = (0, evaluateToolCalls_1.evaluateToolCalls)(step.expected.tools, response.toolCalls);
+            stepResult.toolEvaluations = toolsEvaluation.evaluations;
+            if (!toolsEvaluation.passed) {
+                stepResult.passed = false;
+                stepResult.errorMessage = stepResult.errorMessage
+                    ? `${stepResult.errorMessage} ${toolsEvaluation.errorMessage}`
+                    : toolsEvaluation.errorMessage;
+            }
+        }
+    }
+    catch (error) {
+        stepResult.passed = false;
+        stepResult.errorMessage = `Language model error for input "${step.userInput}": ${error}`;
+    }
+    return stepResult;
+}

package/dist/models/evaluations/evaluateTextResponse.d.ts

@@ -0,0 +1,8 @@
+import { ExpectedTextType } from "./types";
+export declare function evaluateTextResponse(expected: {
+    type: ExpectedTextType;
+    response: string;
+}, aiResponse: string, testTextSimilarity: (text1: string, text2: string) => Promise<boolean>): Promise<{
+    passed: boolean;
+    errorMessage?: string;
+}>;

package/dist/models/evaluations/evaluateTextResponse.js

@@ -0,0 +1,28 @@
+"use strict";
+/*
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * Licensed under the MIT License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluateTextResponse = evaluateTextResponse;
+const types_1 = require("./types");
+async function evaluateTextResponse(expected, aiResponse, testTextSimilarity) {
+    if (expected.type === types_1.ExpectedTextType.EXACT) {
+        if (aiResponse !== expected.response) {
+            return {
+                passed: false,
+                errorMessage: `Expected exact response "${expected.response}", but got "${aiResponse}".`
+            };
+        }
+    }
+    else if (expected.type === types_1.ExpectedTextType.SIMILAR) {
+        const isSimilar = await testTextSimilarity(expected.response, aiResponse);
+        if (!isSimilar) {
+            return {
+                passed: false,
+                errorMessage: `Expected similar response to "${expected.response}", but got "${aiResponse}".`
+            };
+        }
+    }
+    return { passed: true };
+}

package/dist/models/evaluations/evaluateToolCalls.d.ts

@@ -0,0 +1,6 @@
+import { ToolEvaluationReport } from "./types";
+export declare function evaluateToolCalls(expectedTools: any[], toolCalls: any[] | undefined): {
+    evaluations: ToolEvaluationReport[];
+    passed: boolean;
+    errorMessage?: string;
+};

package/dist/models/evaluations/evaluateToolCalls.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluateToolCalls = evaluateToolCalls;
+function evaluateToolCalls(expectedTools, toolCalls) {
+    const evaluations = [];
+    let overallPassed = true;
+    if (!toolCalls || toolCalls.length !== expectedTools.length) {
+        overallPassed = false;
+        evaluations.push({
+            expectedTool: "",
+            actualTool: "",
+            passed: false,
+            expectedParameters: undefined,
+            actualParameters: undefined,
+            errorMessage: `Expected ${expectedTools.length} tool invocation(s), but got ${toolCalls ? toolCalls.length : 0}.`
+        });
+        return {
+            evaluations,
+            passed: overallPassed,
+            errorMessage: `Tool invocation count mismatch.`
+        };
+    }
+    for (let i = 0; i < expectedTools.length; i++) {
+        const expectedTool = expectedTools[i];
+        const actualCall = toolCalls[i];
+        let toolPassed = true;
+        let errorMessage = "";
+        if (actualCall.name !== expectedTool.tool) {
+            toolPassed = false;
+            errorMessage = `Expected tool "${expectedTool.tool}" but got "${actualCall.name}".`;
+        }
+        // Validate expected parameters against the actual ones
+        const expectedParams = expectedTool.parameters || {};
+        const actualParams = actualCall.args || {};
+        for (const key of Object.keys(expectedParams)) {
+            if (actualParams[key] !== expectedParams[key]) {
+                toolPassed = false;
+                const paramMsg = `Expected parameter "${key}" to have value ${JSON.stringify(expectedParams[key])}, but got ${JSON.stringify(actualParams[key])}.`;
+                errorMessage = errorMessage ? errorMessage + " " + paramMsg : paramMsg;
+            }
+        }
+        if (!toolPassed) {
+            overallPassed = false;
+        }
+        evaluations.push({
+            expectedTool: expectedTool.tool,
+            actualTool: actualCall.name,
+            passed: toolPassed,
+            expectedParameters: expectedTool.parameters,
+            actualParameters: actualCall.args,
+            errorMessage: errorMessage || undefined
+        });
+    }
+    return {
+        evaluations,
+        passed: overallPassed,
+        errorMessage: overallPassed
+            ? undefined
+            : "One or more tool evaluations failed."
+    };
+}

package/dist/models/evaluations/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./evals";
+export * from "./printEval";

package/dist/models/evaluations/index.js

@@ -0,0 +1,36 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+/*
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+__exportStar(require("./evals"), exports);
+__exportStar(require("./printEval"), exports);

package/dist/models/evaluations/printEval.d.ts

@@ -0,0 +1,2 @@
+import { ScenarioEvaluationReport } from "./types";
+export declare function printEval(results: ScenarioEvaluationReport[]): void;

package/dist/models/evaluations/printEval.js

@@ -0,0 +1,79 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.printEval = printEval;
+/*
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const cli_table3_1 = __importDefault(require("cli-table3"));
+const chalk_1 = __importDefault(require("chalk"));
+function printEval(results) {
+    results.forEach((result) => {
+        console.log(chalk_1.default.bold.blue(`\nScenario: ${result.scenarioRef}`));
+        console.log(chalk_1.default.bold(`Overall Passed: ${result.overallPassed ? chalk_1.default.green("✔") : chalk_1.default.red("✘")}`));
+        const table = new cli_table3_1.default({
+            head: [
+                "Step",
+                "Human Input",
+                "Expected",
+                "AI Response",
+                "Tool Calls",
+                "Passed"
+            ],
+            colWidths: [8, 25, 25, 25, 25, 8],
+            wordWrap: true
+        });
+        result.steps.forEach((step, index) => {
+            // Format tool evaluations if they exist
+            let toolEvalText = "";
+            if (step.toolEvaluations && step.toolEvaluations.length > 0) {
+                toolEvalText = step.toolEvaluations
+                    .map((toolEval) => {
+                    const params = JSON.stringify(toolEval.actualParameters || {});
+                    return `${toolEval.actualTool}(${params})`;
+                })
+                    .join("\n");
+            }
+            table.push([
+                index + 1,
+                step.humanInput,
+                step.expectedResponse,
+                step.aiResponse,
+                toolEvalText,
+                step.passed ? chalk_1.default.green("✔") : chalk_1.default.red("✘")
+            ]);
+            // Print error message if step failed
+            if (!step.passed && step.errorMessage) {
+                console.log(chalk_1.default.red(`\nError in step ${index + 1}:`));
+                console.log(chalk_1.default.red(step.errorMessage));
+            }
+            // Print tool evaluation errors if any
+            if (step.toolEvaluations) {
+                step.toolEvaluations.forEach((toolEval) => {
+                    if (!toolEval.passed && toolEval.errorMessage) {
+                        console.log(chalk_1.default.red(`\nTool Error in step ${index + 1}:`));
+                        console.log(chalk_1.default.red(toolEval.errorMessage));
+                    }
+                });
+            }
+        });
+        console.log(table.toString());
+    });
+}

package/dist/models/evaluations/textSimilaryPrompt.d.ts

@@ -0,0 +1 @@
+export declare const textSimilaryPrompt = "\nYou are a text similarity evaluator for a Voice Assistant application. \n\nGive Text1 and Text2, you use the following process to evaluate the similarity between the two texts:\n\n- Take the first text and determmine the intent of the text.\n- Take the second text and determine the intent of the text.\n- Compare the intents of the two texts ignoring the actual text content and the entities, and length of the text.\n\n## Example 1\n\nText1: \"You're welcome. Have a great day!\"\nText2: \"You're welcome [name]. Your appointment is confirmed. Goodbye!\"\n\nAnswer: true\n\n=== \n\nAre the intents of the two texts the same? Respond with true.\n";

package/dist/models/evaluations/textSimilaryPrompt.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.textSimilaryPrompt = void 0;
+/*
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+exports.textSimilaryPrompt = `
+You are a text similarity evaluator for a Voice Assistant application. 
+
+Give Text1 and Text2, you use the following process to evaluate the similarity between the two texts:
+
+- Take the first text and determmine the intent of the text.
+- Take the second text and determine the intent of the text.
+- Compare the intents of the two texts ignoring the actual text content and the entities, and length of the text.
+
+## Example 1
+
+Text1: "You're welcome. Have a great day!"
+Text2: "You're welcome [name]. Your appointment is confirmed. Goodbye!"
+
+Answer: true
+
+=== 
+
+Are the intents of the two texts the same? Respond with true.
+`;

package/dist/models/evaluations/types.d.ts

@@ -0,0 +1,41 @@
+import { AssistantConfig } from "../../assistants";
+import { LanguageModel } from "../types";
+declare enum ExpectedTextType {
+    EXACT = "exact",
+    SIMILAR = "similar"
+}
+type ScenarioEvaluationReport = {
+    scenarioRef: string;
+    overallPassed: boolean;
+    steps: StepEvaluationReport[];
+};
+type StepEvaluationReport = {
+    humanInput: string;
+    expectedResponse: string;
+    aiResponse: string;
+    evaluationType: ExpectedTextType;
+    passed: boolean;
+    errorMessage?: string;
+    toolEvaluations?: ToolEvaluationReport[];
+};
+type EvaluateStepParams = {
+    step: any;
+    languageModel: LanguageModel;
+    testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
+    assistantConfig: AssistantConfig;
+};
+type ToolEvaluationReport = {
+    expectedTool: string;
+    actualTool: string;
+    passed: boolean;
+    expectedParameters?: Record<string, unknown>;
+    actualParameters?: Record<string, unknown>;
+    errorMessage?: string;
+};
+type ScenarioEvaluationConfig = {
+    assistantConfig: AssistantConfig;
+    scenario: any;
+    languageModel: LanguageModel;
+    testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
+};
+export { ExpectedTextType, ScenarioEvaluationReport, StepEvaluationReport, ToolEvaluationReport, ScenarioEvaluationConfig, EvaluateStepParams };

package/dist/models/evaluations/types.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExpectedTextType = void 0;
+var ExpectedTextType;
+(function (ExpectedTextType) {
+    ExpectedTextType["EXACT"] = "exact";
+    ExpectedTextType["SIMILAR"] = "similar";
+})(ExpectedTextType || (exports.ExpectedTextType = ExpectedTextType = {}));

package/dist/models/index.d.ts

@@ -1,2 +1,3 @@
 export * from "./LanguageModelFactory";
 export * from "./types";
+export * from "./evaluations";

package/dist/models/index.js

@@ -34,3 +34,4 @@ __exportStar(require("./LanguageModelFactory"), exports);
  * limitations under the License.
  */
 __exportStar(require("./types"), exports);
+__exportStar(require("./evaluations"), exports);

package/dist/models/toolInvocation.d.ts

@@ -1,12 +1,10 @@
 import { createChatHistory } from "./chatHistory";
 import { ToolsCatalog } from "../tools";
-import { Voice } from "../voice";
 declare function toolInvocation(params: {
     toolName: string;
     chatHistory: ReturnType<typeof createChatHistory>;
     toolsCatalog: ToolsCatalog;
-    firstInvocation: boolean;
+    isFirstTool: boolean;
     args: Record<string, unknown>;
-    voice: Voice;
-}): Promise<void>;
+}): Promise<string>;
 export { toolInvocation };

package/dist/models/toolInvocation.js

@@ -22,21 +22,23 @@ exports.toolInvocation = toolInvocation;
 const logger_1 = require("@fonoster/logger");
 const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
 async function toolInvocation(params) {
-    const { firstInvocation, voice, args, toolName, chatHistory, toolsCatalog } = params;
+    const { isFirstTool, args, toolName, chatHistory, toolsCatalog } = params;
     try {
-        if (firstInvocation) {
+        if (isFirstTool) {
             const tool = toolsCatalog.getTool(toolName);
             const message = tool?.requestStartMessage ?? "";
             if (message) {
-                await voice.say(message);
+                await chatHistory.addAIMessage(message);
             }
         }
         const toolResult = await toolsCatalog.invokeTool(toolName, args);
-        logger.verbose("tool result: ", toolResult);
-        await chatHistory.addAIMessage(`tool result: ${toolResult.result}`);
+        logger.verbose(`tool result (${toolName}):`, { result: toolResult.result });
+        await chatHistory.addAIMessage(`tool result (${toolName}): ${toolResult.result}`);
+        return toolResult.result;
     }
     catch (error) {
         logger.error(`tool error: ${error.message}`);
         await chatHistory.addAIMessage(`tool error: ${error.message}`);
+        return "";
     }
 }

package/dist/models/types.d.ts

@@ -2,6 +2,7 @@ import { CallDirection } from "@fonoster/types";
 import { BaseChatModel } from "@langchain/core/language_models/chat_models";
 import { KnowledgeBase } from "../knowledge";
 import { Tool } from "../tools/type";
+import { ToolCall } from "@langchain/core/messages/tool";
 type LanguageModel = {
     invoke: (text: string) => Promise<InvocationResult>;
 };
@@ -18,6 +19,7 @@ type LanguageModelParams = BaseModelParams & {
 type InvocationResult = {
     type: "say" | "hangup" | "transfer";
     content?: string;
+    toolCalls?: ToolCall[];
 };
 type TelephonyContext = {
     callDirection: CallDirection;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fonoster/autopilot",
-  "version": "0.8.52",
+  "version": "0.8.59",
   "description": "Voice AI for the Fonoster platform",
   "author": "Pedro Sanders <psanders@fonoster.com>",
   "homepage": "https://github.com/fonoster/fonoster#readme",
@@ -33,18 +33,20 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.712.0",
-    "@fonoster/common": "^0.8.52",
-    "@fonoster/logger": "^0.8.51",
-    "@fonoster/sdk": "^0.8.52",
-    "@fonoster/types": "^0.8.51",
-    "@fonoster/voice": "^0.8.52",
-    "@langchain/community": "^0.3.19",
-    "@langchain/core": "^0.3.23",
-    "@langchain/groq": "^0.1.2",
-    "@langchain/ollama": "^0.1.2",
-    "@langchain/openai": "^0.3.14",
+    "@fonoster/common": "^0.8.59",
+    "@fonoster/logger": "^0.8.59",
+    "@fonoster/sdk": "^0.8.59",
+    "@fonoster/types": "^0.8.59",
+    "@fonoster/voice": "^0.8.59",
+    "@langchain/community": "^0.3.29",
+    "@langchain/core": "^0.3.39",
+    "@langchain/groq": "^0.1.3",
+    "@langchain/ollama": "^0.1.5",
+    "@langchain/openai": "^0.4.3",
     "cheerio": "^1.0.0",
+    "cli-table3": "^0.6.5",
     "dotenv": "^16.4.5",
+    "js-yaml": "^4.1.0",
     "langchain": "^0.3.6",
     "onnxruntime-node": "^1.19.0",
     "pdf-parse": "^1.1.1",
@@ -55,5 +57,5 @@
   "devDependencies": {
     "typescript": "^5.5.4"
   },
-  "gitHead": "772398ea115078012f8e270943ec36b61343ea36"
+  "gitHead": "6a5255febb9e4bcd9184d93abe72281f945c8c4d"
 }