@fonoster/autopilot 0.13.3 → 0.13.5

package/dist/models/evaluations/createEvalEffectiveConfig.d.ts

@@ -0,0 +1,120 @@
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { AssistantConfig } from "../../assistants";
+export declare function createEvalEffectiveConfig(config: AssistantConfig, credentials: {
+    apiKey: string;
+}, evaluationApiKey: {
+    apiKey: string;
+}): {
+    languageModel: {
+        apiKey: string;
+        provider?: import("@fonoster/common").LanguageModelProvider;
+        model?: string;
+        temperature?: number;
+        maxTokens?: number;
+        baseUrl?: string;
+        knowledgeBase?: {
+            type?: "s3";
+            title?: string;
+            document?: string;
+        }[];
+        tools?: {
+            name?: string;
+            description?: string;
+            parameters?: {
+                type?: "object" | "array";
+                properties?: Record<string, {
+                    type?: string;
+                    format?: "enum" | "date-time";
+                    pattern?: string;
+                }>;
+                required?: string[];
+            };
+            requestStartMessage?: string;
+            operation?: {
+                url?: string;
+                method?: import("@fonoster/common").AllowedHttpMethod;
+                headers?: Record<string, string>;
+                waitForResponse?: boolean;
+            };
+        }[];
+    };
+    testCases: {
+        evalsLanguageModel: {
+            apiKey: string;
+            provider?: "openai";
+            model?: string;
+        };
+        evalsSystemPrompt?: string;
+        scenarios?: {
+            description?: string;
+            ref?: string;
+            telephonyContext?: {
+                callDirection?: import("@fonoster/types").CallDirection;
+                ingressNumber?: string;
+                callerNumber?: string;
+                metadata?: Record<string, string>;
+            };
+            conversation?: {
+                expected?: {
+                    tools?: {
+                        parameters?: Record<string, any>;
+                        tool?: string;
+                    }[];
+                    text?: {
+                        type?: "EXACT" | "SIMILAR";
+                        response?: string;
+                    };
+                };
+                userInput?: string;
+            }[];
+        }[];
+    };
+    conversationSettings?: {
+        firstMessage?: string;
+        systemPrompt?: string;
+        goodbyeMessage?: string;
+        systemErrorMessage?: string;
+        initialDtmf?: string;
+        maxSessionDuration?: number;
+        maxSpeechWaitTimeout?: number;
+        transferOptions?: {
+            message?: string;
+            phoneNumber?: string;
+            timeout?: number;
+        };
+        idleOptions?: {
+            message?: string;
+            timeout?: number;
+            maxTimeoutCount?: number;
+        };
+        allowUserBargeIn?: boolean;
+        vad?: {
+            pathToModel?: string;
+            activationThreshold?: number;
+            deactivationThreshold?: number;
+            debounceFrames?: number;
+        };
+    };
+    eventsHook?: {
+        url?: string;
+        headers?: Record<string, string>;
+        events?: import("@fonoster/common").EventsHookAllowedEvents[];
+    };
+};

package/dist/models/evaluations/createEvalEffectiveConfig.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createEvalEffectiveConfig = createEvalEffectiveConfig;
+function createEvalEffectiveConfig(config, credentials, evaluationApiKey) {
+    return {
+        ...config,
+        languageModel: {
+            ...config.languageModel,
+            apiKey: credentials.apiKey
+        },
+        testCases: {
+            ...config.testCases,
+            evalsLanguageModel: {
+                ...config.testCases.evalsLanguageModel,
+                apiKey: evaluationApiKey.apiKey
+            }
+        }
+    };
+}

package/dist/models/evaluations/createEvaluateIntelligence.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { GrpcErrorMessage, IntegrationConfig } from "@fonoster/common";
+declare function createEvaluateIntelligence(integrations: IntegrationConfig[]): (call: {
+    request: unknown;
+}, callback: (error?: GrpcErrorMessage, response?: unknown) => void) => Promise<void>;
+export { createEvaluateIntelligence };

package/dist/models/evaluations/createEvaluateIntelligence.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createEvaluateIntelligence = createEvaluateIntelligence;
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const common_1 = require("@fonoster/common");
+const logger_1 = require("@fonoster/logger");
+const pb_util_1 = require("pb-util");
+const zod_1 = require("zod");
+const createEvalEffectiveConfig_1 = require("./createEvalEffectiveConfig");
+const evalTestCases_1 = require("./evalTestCases");
+const logger = (0, logger_1.getLogger)({ service: "apiserver", filePath: __filename });
+function createEvaluateIntelligence(integrations) {
+    const evaluateIntelligence = async (call, callback) => {
+        const { request } = call;
+        const { intelligence } = request;
+        const accessKeyId = (0, common_1.getAccessKeyIdFromCall)(call);
+        logger.verbose("call to evaluateIntelligence", {
+            accessKeyId,
+            llmProductRef: intelligence.productRef,
+            evalLlmProductRef: "llm.openai"
+        });
+        const config = pb_util_1.struct.decode(intelligence.config);
+        const parsedIntelligence = zod_1.z
+            .object({
+            productRef: zod_1.z.string(),
+            config: common_1.assistantSchema
+        })
+            .parse({
+            productRef: intelligence.productRef,
+            config: config
+        });
+        const credentials = (0, common_1.findIntegrationsCredentials)(integrations, intelligence.productRef);
+        const evaluationApiKey = (0, common_1.findIntegrationsCredentials)(integrations, "llm.openai");
+        const effectiveConfig = (0, createEvalEffectiveConfig_1.createEvalEffectiveConfig)(parsedIntelligence.config, credentials, evaluationApiKey);
+        const results = await (0, evalTestCases_1.evalTestCases)({
+            intelligence: {
+                config: effectiveConfig
+            }
+        });
+        callback(null, { results });
+    };
+    return (0, common_1.withErrorHandling)(evaluateIntelligence);
+}

package/dist/models/evaluations/evalTestCases.d.ts

@@ -16,8 +16,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { ScenarioEvaluationReport } from "@fonoster/types";
 import { AssistantConfig } from "../../assistants";
-import { ScenarioEvaluationReport } from "./types";
 export declare function evalTestCases(autopilotApplication: {
     intelligence: {
         config: AssistantConfig;

package/dist/models/evaluations/evaluateScenario.d.ts

@@ -1,2 +1,21 @@
-import { ScenarioEvaluationConfig, ScenarioEvaluationReport } from "./types";
-export declare function evaluateScenario(config: ScenarioEvaluationConfig): Promise<ScenarioEvaluationReport>;
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ScenarioEvaluationReport } from "@fonoster/types";
+import { ScenarioEvaluationRequest } from "./types";
+export declare function evaluateScenario(config: ScenarioEvaluationRequest): Promise<ScenarioEvaluationReport>;

package/dist/models/evaluations/evaluateScenario.js

@@ -1,24 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.evaluateScenario = evaluateScenario;
-/**
- * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
- * http://github.com/fonoster/fonoster
- *
- * This file is part of Fonoster
- *
- * Licensed under the MIT License (the "License");
- * you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    https://opensource.org/licenses/MIT
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 const evaluateStep_1 = require("./evaluateStep");
 async function evaluateScenario(config) {
     const { scenario, languageModel, testTextSimilarity, assistantConfig } = config;

package/dist/models/evaluations/evaluateStep.d.ts

@@ -1,2 +1,21 @@
-import { EvaluateStepParams, StepEvaluationReport } from "./types";
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { StepEvaluationReport } from "@fonoster/types";
+import { EvaluateStepParams } from "./types";
 export declare function evaluateStep({ step, languageModel, testTextSimilarity, assistantConfig }: EvaluateStepParams): Promise<StepEvaluationReport>;

package/dist/models/evaluations/evaluateStep.js

@@ -1,24 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.evaluateStep = evaluateStep;
-/**
- * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
- * http://github.com/fonoster/fonoster
- *
- * This file is part of Fonoster
- *
- * Licensed under the MIT License (the "License");
- * you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    https://opensource.org/licenses/MIT
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 const evaluateTextResponse_1 = require("./evaluateTextResponse");
 const evaluateToolCalls_1 = require("./evaluateToolCalls");
 async function evaluateStep({ step, languageModel, testTextSimilarity, assistantConfig }) {

package/dist/models/evaluations/evaluateTextResponse.d.ts

@@ -16,7 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { ExpectedTextType } from "./types";
+import { ExpectedTextType } from "@fonoster/types";
 export declare function evaluateTextResponse(expected: {
     type: ExpectedTextType;
     response: string;

package/dist/models/evaluations/evaluateTextResponse.js

@@ -19,7 +19,7 @@ exports.evaluateTextResponse = evaluateTextResponse;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-const types_1 = require("./types");
+const types_1 = require("@fonoster/types");
 async function evaluateTextResponse(expected, aiResponse, testTextSimilarity) {
     if (expected.type === types_1.ExpectedTextType.EXACT) {
         if (aiResponse !== expected.response) {

package/dist/models/evaluations/evaluateToolCalls.d.ts

@@ -1,4 +1,22 @@
-import { ToolEvaluationReport } from "./types";
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ToolEvaluationReport } from "@fonoster/types";
 export declare function evaluateToolCalls(expectedTools: any[], toolCalls: any[] | undefined): {
     evaluations: ToolEvaluationReport[];
     passed: boolean;

package/dist/models/evaluations/evaluateToolCalls.js

@@ -4,24 +4,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.evaluateToolCalls = evaluateToolCalls;
-/**
- * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
- * http://github.com/fonoster/fonoster
- *
- * This file is part of Fonoster
- *
- * Licensed under the MIT License (the "License");
- * you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    https://opensource.org/licenses/MIT
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 const moment_1 = __importDefault(require("moment"));
 function evaluateToolCalls(expectedTools, toolCalls) {
     const evaluations = [];

package/dist/models/evaluations/index.d.ts

@@ -17,5 +17,5 @@
  * limitations under the License.
  */
 export * from "./evalTestCases";
-export * from "./printEval";
 export * from "./types";
+export * from "./createEvaluateIntelligence";

package/dist/models/evaluations/index.js

@@ -33,5 +33,5 @@ Object.defineProperty(exports, "__esModule", { value: true });
  * limitations under the License.
  */
 __exportStar(require("./evalTestCases"), exports);
-__exportStar(require("./printEval"), exports);
 __exportStar(require("./types"), exports);
+__exportStar(require("./createEvaluateIntelligence"), exports);

package/dist/models/evaluations/types.d.ts

@@ -18,23 +18,11 @@
  */
 import { AssistantConfig } from "../../assistants";
 import { LanguageModel } from "../types";
-declare enum ExpectedTextType {
-    EXACT = "exact",
-    SIMILAR = "similar"
-}
-type ScenarioEvaluationReport = {
-    scenarioRef: string;
-    overallPassed: boolean;
-    steps: StepEvaluationReport[];
-};
-type StepEvaluationReport = {
-    humanInput: string;
-    expectedResponse: string;
-    aiResponse: string;
-    evaluationType: ExpectedTextType;
-    passed: boolean;
-    errorMessage?: string;
-    toolEvaluations?: ToolEvaluationReport[];
+type EvaluateIntelligenceRequest = {
+    intelligence: {
+        productRef: string;
+        config: AssistantConfig;
+    };
 };
 type EvaluateStepParams = {
     step: any;
@@ -42,18 +30,10 @@ type EvaluateStepParams = {
     testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
     assistantConfig: AssistantConfig;
 };
-type ToolEvaluationReport = {
-    expectedTool: string;
-    actualTool: string;
-    passed: boolean;
-    expectedParameters?: Record<string, unknown>;
-    actualParameters?: Record<string, unknown>;
-    errorMessage?: string;
-};
-type ScenarioEvaluationConfig = {
+type ScenarioEvaluationRequest = {
     assistantConfig: AssistantConfig;
     scenario: any;
     languageModel: LanguageModel;
     testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
 };
-export { ExpectedTextType, ScenarioEvaluationReport, StepEvaluationReport, ToolEvaluationReport, ScenarioEvaluationConfig, EvaluateStepParams };
+export { EvaluateIntelligenceRequest, ScenarioEvaluationRequest, EvaluateStepParams };

package/dist/models/evaluations/types.js

@@ -1,8 +1,2 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ExpectedTextType = void 0;
-var ExpectedTextType;
-(function (ExpectedTextType) {
-    ExpectedTextType["EXACT"] = "exact";
-    ExpectedTextType["SIMILAR"] = "similar";
-})(ExpectedTextType || (exports.ExpectedTextType = ExpectedTextType = {}));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fonoster/autopilot",
-  "version": "0.13.3",
+  "version": "0.13.5",
   "description": "Voice AI for the Fonoster platform",
   "author": "Pedro Sanders <psanders@fonoster.com>",
   "homepage": "https://github.com/fonoster/fonoster#readme",
@@ -34,11 +34,11 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.712.0",
     "@dmitryrechkin/json-schema-to-zod": "^1.0.1",
-    "@fonoster/common": "^0.13.2",
-    "@fonoster/logger": "^0.13.2",
-    "@fonoster/sdk": "^0.13.3",
-    "@fonoster/types": "^0.13.2",
-    "@fonoster/voice": "^0.13.2",
+    "@fonoster/common": "^0.13.5",
+    "@fonoster/logger": "^0.13.5",
+    "@fonoster/sdk": "^0.13.5",
+    "@fonoster/types": "^0.13.5",
+    "@fonoster/voice": "^0.13.5",
     "@langchain/anthropic": "^0.3.15",
     "@langchain/community": "^0.3.32",
     "@langchain/core": "^0.3.40",
@@ -46,9 +46,7 @@
     "@langchain/groq": "^0.1.3",
     "@langchain/ollama": "^0.1.6",
     "@langchain/openai": "^0.4.4",
-    "ansis": "^3.17.0",
     "cheerio": "^1.0.0",
-    "cli-table3": "^0.6.5",
     "dotenv": "^16.4.5",
     "js-yaml": "^4.1.0",
     "langchain": "^0.3.6",
@@ -59,5 +57,5 @@
     "xstate": "^5.17.3",
     "zod": "^3.23.8"
   },
-  "gitHead": "d83554cf5f574a46a4cac71c9083c591b4d0ba98"
+  "gitHead": "a502dc44c7b180a4424eea51952e41194e4ffaf4"
 }

package/dist/models/evaluations/printEval.d.ts

@@ -1,2 +0,0 @@
-import { ScenarioEvaluationReport } from "./types";
-export declare function printEval(results: ScenarioEvaluationReport[]): void;

package/dist/models/evaluations/printEval.js

@@ -1,98 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.printEval = printEval;
-/**
- * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
- * http://github.com/fonoster/fonoster
- *
- * This file is part of Fonoster
- *
- * Licensed under the MIT License (the "License");
- * you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    https://opensource.org/licenses/MIT
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-const ansis_1 = __importDefault(require("ansis"));
-const cli_table3_1 = __importDefault(require("cli-table3"));
-function printEval(results) {
-    results.forEach((result) => {
-        console.log(ansis_1.default.bold.blue(`\nScenario: ${result.scenarioRef}`));
-        console.log(ansis_1.default.bold(`Overall Passed: ${result.overallPassed ? ansis_1.default.green("✔") : ansis_1.default.red("✘")}`));
-        const table = new cli_table3_1.default({
-            head: [
-                "Step",
-                "Human Input",
-                "Expected",
-                "AI Response",
-                "Tool Calls",
-                "Passed"
-            ],
-            colWidths: [
-                6, // Step
-                28, // Human Input
-                28, // Expected
-                28, // AI Response
-                null, // Tool Calls - dynamic width
-                8 // Passed
-            ],
-            wordWrap: true
-        });
-        result.steps.forEach((step, index) => {
-            // Format tool evaluations if they exist
-            let toolEvalText = "";
-            if (step.toolEvaluations && step.toolEvaluations.length > 0) {
-                toolEvalText = step.toolEvaluations
-                    .map((toolEval) => {
-                    if (!Object.keys(toolEval.actualParameters || {}).length) {
-                        return `${toolEval.actualTool}()`;
-                    }
-                    const params = JSON.stringify(toolEval.actualParameters || {}, null, 1)
-                        .split("\n")
-                        .map((line, index, arr) => {
-                        if (index === 0)
-                            return "";
-                        if (index === arr.length - 1)
-                            return "";
-                        return " " + line.trim();
-                    })
-                        .join("\n");
-                    return `${toolEval.actualTool}({${params}})`;
-                })
-                    .join("\n\n"); // Add extra line between multiple tool calls
-            }
-            table.push([
-                index + 1,
-                step.humanInput,
-                step.expectedResponse,
-                step.aiResponse,
-                toolEvalText,
-                step.passed ? ansis_1.default.green("✔") : ansis_1.default.red("✘")
-            ]);
-            // Print error message if step failed
-            if (!step.passed && step.errorMessage) {
-                console.log(ansis_1.default.red(`\nError in step ${index + 1}:`));
-                console.log(ansis_1.default.red(step.errorMessage));
-            }
-            // Print tool evaluation errors if any
-            if (step.toolEvaluations) {
-                step.toolEvaluations.forEach((toolEval) => {
-                    if (!toolEval.passed && toolEval.errorMessage) {
-                        console.log(ansis_1.default.red(`\nTool Error in step ${index + 1}:`));
-                        console.log(ansis_1.default.red(toolEval.errorMessage));
-                    }
-                });
-            }
-        });
-        console.log(table.toString());
-    });
-}