@fonoster/autopilot 0.27.0 → 0.31.0

package/dist/models/evaluations/session/evaluateSessionScenario.js

@@ -0,0 +1,182 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evalToolPort = void 0;
+exports.evaluateSessionScenario = evaluateSessionScenario;
+const createAutopilot_1 = require("../../../session/createAutopilot");
+const evaluateTextResponse_1 = require("../evaluateTextResponse");
+const evaluateToolCalls_1 = require("../evaluateToolCalls");
+const EvalSession_1 = require("./EvalSession");
+/** The default eval tool port: records nothing, calls nothing, every tool succeeds. */
+exports.evalToolPort = {
+    run: async () => ({
+        ok: true,
+        result: { status: "ok" }
+    })
+};
+const SETTLE_POLL_MS = 10;
+/** Resolves once the session has had no new output and no say in flight for two polls. */
+async function settle(session) {
+    let seen = -1;
+    let quiet = 0;
+    while (quiet < 2) {
+        await new Promise((r) => setTimeout(r, SETTLE_POLL_MS));
+        if (!session.busy && session.outputs.length === seen)
+            quiet++;
+        else
+            quiet = 0;
+        seen = session.outputs.length;
+    }
+}
+function parseArgs(call) {
+    try {
+        return call.arguments ? JSON.parse(call.arguments) : {};
+    }
+    catch {
+        return {};
+    }
+}
+/**
+ * Runs one test-case scenario through the session engine (rework-autopilot 1.4): the real state
+ * machine and Agent on an in-memory session, replies streamed sentence by sentence, tool calls
+ * read back from the structured tool messages in the history.
+ */
+async function evaluateSessionScenario(options, scenario, onStepResult) {
+    const { agent: agentConfig, testTextSimilarity } = options;
+    const stepTimeoutMs = options.stepTimeoutMs ?? 60000;
+    const session = new EvalSession_1.EvalSession(scenario.telephonyContext);
+    let agent;
+    let signal;
+    let thinkError;
+    const traces = [];
+    // Think errors are swallowed by the engine (it apologises); keep them for the report.
+    const think = {
+        stream: (messages, tools, abort) => (async function* () {
+            try {
+                yield* options.think.stream(messages, tools, abort);
+            }
+            catch (e) {
+                if (!abort.aborted) {
+                    thinkError = e;
+                    signal?.();
+                }
+                throw e;
+            }
+        })()
+    };
+    const handler = (0, createAutopilot_1.createAutopilotHandler)({
+        agent: { ...agentConfig, testCases: undefined },
+        think,
+        tools: options.tools ?? exports.evalToolPort,
+        recorder: {
+            turn: (t) => {
+                traces.push(t);
+                signal?.();
+            },
+            note: () => undefined
+        },
+        onAgent: (a) => (agent = a)
+    });
+    session.onOutput((o) => {
+        if (o.kind === "hangup" || o.kind === "transfer")
+            signal?.();
+    });
+    const running = handler(session);
+    const results = [];
+    try {
+        await settle(session); // greeting
+        for (const step of scenario.conversation) {
+            const type = step.expected.text.type;
+            const report = {
+                humanInput: step.userInput,
+                expectedResponse: step.expected.text.response,
+                aiResponse: "",
+                evaluationType: type,
+                passed: true,
+                streaming: { pieces: [] },
+                toolMessages: []
+            };
+            results.push(report);
+            if (session.ended) {
+                report.passed = false;
+                report.errorMessage = `The call had already ended before "${step.userInput}".`;
+                if (onStepResult)
+                    await onStepResult(scenario.ref, report);
+                continue;
+            }
+            const outputsFrom = session.outputs.length;
+            const historyFrom = agent?.history.length ?? 0;
+            const tracesFrom = traces.length;
+            thinkError = undefined;
+            let timer;
+            const done = new Promise((resolve) => {
+                signal = () => resolve(true);
+                timer = setTimeout(() => resolve(false), stepTimeoutMs);
+            });
+            session.turn(step.userInput);
+            const finished = await done;
+            clearTimeout(timer);
+            signal = undefined;
+            await settle(session);
+            const outputs = session.outputs.slice(outputsFrom);
+            const says = outputs.filter((o) => o.kind === "say");
+            const reply = says.filter((s) => s.streamed);
+            const history = agent?.history.slice(historyFrom) ?? [];
+            const toolCalls = history.flatMap((m) => m.role === "assistant" ? (m.tool_calls ?? []) : []);
+            const builtIn = toolCalls.find((c) => c.name === "hangup" || c.name === "transfer");
+            const trace = traces[tracesFrom];
+            report.streaming = {
+                pieces: reply.flatMap((s) => s.pieces),
+                firstTokenMs: trace?.firstTokenMs,
+                firstSpeakMs: trace?.firstSpeakMs
+            };
+            report.toolMessages = history.filter((m) => m.role === "tool" || (m.role === "assistant" && m.tool_calls));
+            // What the caller hears: the streamed reply; for hangup or transfer, the message the
+            // engine says for it (as the legacy evals compared).
+            report.aiResponse = builtIn
+                ? builtIn.name === "hangup"
+                    ? agentConfig.conversationSettings.goodbyeMessage
+                    : (agentConfig.conversationSettings.transferOptions?.message ?? "")
+                : reply
+                    .map((s) => s.text)
+                    .join("")
+                    .trim();
+            if (thinkError) {
+                report.passed = false;
+                report.aiResponse = says.map((s) => s.text).join(" ");
+                report.errorMessage = `Language model error for input "${step.userInput}": ${thinkError.message}`;
+            }
+            else if (!finished) {
+                report.passed = false;
+                report.errorMessage = `No reply within ${stepTimeoutMs} ms for "${step.userInput}".`;
+            }
+            else {
+                const text = await (0, evaluateTextResponse_1.evaluateTextResponse)({ type, response: step.expected.text.response }, report.aiResponse, testTextSimilarity);
+                if (!text.passed) {
+                    report.passed = false;
+                    report.errorMessage = text.errorMessage;
+                }
+                if (step.expected.tools && step.expected.tools.length > 0) {
+                    const tools = (0, evaluateToolCalls_1.evaluateToolCalls)(step.expected.tools, toolCalls.map((c) => ({ name: c.name, args: parseArgs(c) })));
+                    report.toolEvaluations = tools.evaluations;
+                    if (!tools.passed) {
+                        report.passed = false;
+                        report.errorMessage = report.errorMessage
+                            ? `${report.errorMessage} ${tools.errorMessage}`
+                            : tools.errorMessage;
+                    }
+                }
+            }
+            if (onStepResult)
+                await onStepResult(scenario.ref, report);
+        }
+    }
+    finally {
+        session.end();
+        await running.catch(() => undefined);
+    }
+    return {
+        scenarioRef: scenario.ref,
+        overallPassed: results.every((s) => s.passed),
+        steps: results
+    };
+}

package/dist/{vad → models/evaluations/session}/index.d.ts

@@ -16,5 +16,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export * from "./SileroVad";
-export * from "./types";
+export * from "./EvalSession";
+export * from "./evaluateSessionScenario";
+export * from "./runSessionEval";

package/dist/{vad → models/evaluations/session}/index.js

@@ -32,5 +32,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-__exportStar(require("./SileroVad"), exports);
-__exportStar(require("./types"), exports);
+__exportStar(require("./EvalSession"), exports);
+__exportStar(require("./evaluateSessionScenario"), exports);
+__exportStar(require("./runSessionEval"), exports);

package/dist/models/evaluations/session/runSessionEval.d.ts

@@ -0,0 +1,20 @@
+import { SessionAgent } from "../../../session/config";
+import { Think } from "../../../session/think";
+import { createTestTextSimilarity } from "../createTestTextSimilarity";
+import { RunEvalCallbacks } from "../types";
+import { SessionEvalOptions, SessionScenarioReport } from "./evaluateSessionScenario";
+export type RunSessionEvalOptions = Omit<SessionEvalOptions, "testTextSimilarity"> & {
+    /** Defaults to the OpenAI similarity judge from the agent's testCases. */
+    testTextSimilarity?: SessionEvalOptions["testTextSimilarity"];
+    /** The judge's key (the testCases model is always OpenAI). */
+    evaluationApiKey?: string;
+    /** Meters the judge's tokens (add-usage-metering). */
+    judgeUsage?: Parameters<typeof createTestTextSimilarity>[2];
+};
+/**
+ * The session-engine Think for an agent whose catalog provider speaks OpenAI-compatible Chat
+ * Completions with a known endpoint (openai, groq, ollama); undefined otherwise.
+ */
+export declare function sessionThinkFor(agent: SessionAgent, apiKey: string | undefined): Think | undefined;
+/** Runs every test-case scenario through the session engine, streaming results. */
+export declare function runSessionEval(options: RunSessionEvalOptions, callbacks?: Partial<RunEvalCallbacks>): Promise<SessionScenarioReport[]>;

package/dist/models/evaluations/session/runSessionEval.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sessionThinkFor = sessionThinkFor;
+exports.runSessionEval = runSessionEval;
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const common_1 = require("@fonoster/common");
+const openai_1 = require("@fonoster/community/openai");
+const createTestTextSimilarity_1 = require("../createTestTextSimilarity");
+const textSimilaryPrompt_1 = require("../textSimilaryPrompt");
+const evaluateSessionScenario_1 = require("./evaluateSessionScenario");
+/**
+ * The session-engine Think for an agent whose catalog provider speaks OpenAI-compatible Chat
+ * Completions with a known endpoint (openai, groq, ollama); undefined otherwise.
+ */
+function sessionThinkFor(agent, apiKey) {
+    const id = agent.languageModel.model;
+    const slash = id.indexOf("/");
+    const baseUrl = openai_1.OPENAI_COMPATIBLE[id.slice(0, slash)];
+    if (!baseUrl)
+        return undefined;
+    return new openai_1.OpenAiCompatibleThink({ baseUrl, apiKey }, id.slice(slash + 1), {
+        temperature: agent.languageModel.temperature,
+        maxTokens: agent.languageModel.maxTokens
+    });
+}
+/** Runs every test-case scenario through the session engine, streaming results. */
+async function runSessionEval(options, callbacks) {
+    const testCases = common_1.testCasesSchema.parse(options.agent.testCases);
+    const testTextSimilarity = options.testTextSimilarity ??
+        (0, createTestTextSimilarity_1.createTestTextSimilarity)({
+            provider: testCases.evalsLanguageModel.provider,
+            model: testCases.evalsLanguageModel.model,
+            apiKey: options.evaluationApiKey ?? testCases.evalsLanguageModel.apiKey
+        }, testCases.evalsSystemPrompt || textSimilaryPrompt_1.textSimilaryPrompt, options.judgeUsage);
+    const reports = [];
+    for (const scenario of testCases.scenarios) {
+        const report = await (0, evaluateSessionScenario_1.evaluateSessionScenario)({ ...options, testTextSimilarity }, scenario, callbacks?.onStepResult);
+        reports.push(report);
+        await callbacks?.onScenarioComplete?.(report.scenarioRef, report.overallPassed);
+    }
+    return reports;
+}

package/dist/models/evaluations/textSimilaryPrompt.d.ts

@@ -16,4 +16,4 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export declare const textSimilaryPrompt = "\nYou are a text similarity evaluator for a Voice Assistant application. \n\nGive Text1 and Text2, you use the following process to evaluate the similarity between the two texts:\n\n- Take the first text and determmine the intent of the text.\n- Take the second text and determine the intent of the text.\n- Compare the intents of the two texts ignoring the actual text content, the entities, and length of the text.\n\n## Example 1\n\nText1: \"You're welcome. Have a great day!\"\nText2: \"You're welcome [name]. Your appointment is confirmed. Goodbye!\"\n\nAnswer: true\n\n=== \n\nAre the intents of the two texts the same? Respond with true.\n";
+export declare const textSimilaryPrompt = "\nYou are a text similarity evaluator for a Voice Agent application. \n\nGive Text1 and Text2, you use the following process to evaluate the similarity between the two texts:\n\n- Take the first text and determmine the intent of the text.\n- Take the second text and determine the intent of the text.\n- Compare the intents of the two texts ignoring the actual text content, the entities, and length of the text.\n\n## Example 1\n\nText1: \"You're welcome. Have a great day!\"\nText2: \"You're welcome [name]. Your appointment is confirmed. Goodbye!\"\n\nAnswer: true\n\n=== \n\nAre the intents of the two texts the same? Respond with true.\n";

package/dist/models/evaluations/textSimilaryPrompt.js

@@ -20,7 +20,7 @@ exports.textSimilaryPrompt = void 0;
  * limitations under the License.
  */
 exports.textSimilaryPrompt = `
-You are a text similarity evaluator for a Voice Assistant application. 
+You are a text similarity evaluator for a Voice Agent application. 
 
 Give Text1 and Text2, you use the following process to evaluate the similarity between the two texts:
 

package/dist/models/evaluations/types.d.ts

@@ -17,8 +17,7 @@
  * limitations under the License.
  */
 import { StepEvaluationReport } from "@fonoster/types";
-import { AssistantConfig } from "../../assistants";
-import { LanguageModel } from "../types";
+import { EvalAgent, LanguageModel } from "../types";
 type EvalExpectedTool = {
     tool: string;
     parameters?: Record<string, unknown>;
@@ -46,18 +45,20 @@ type EvalScenario = {
 };
 type EvaluateIntelligenceRequest = {
     intelligence: {
-        productRef: string;
-        config: AssistantConfig;
+        /** Ignored: the agent's catalog id names the provider. */
+        productRef?: string;
+        /** A session agent (with testCases), as a protobuf Struct. */
+        config: unknown;
     };
 };
 type EvaluateStepParams = {
     step: EvalConversationStep;
     languageModel: LanguageModel;
     testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
-    assistantConfig: AssistantConfig;
+    agentConfig: EvalAgent;
 };
 type ScenarioEvaluationRequest = {
-    assistantConfig: AssistantConfig;
+    agentConfig: EvalAgent;
     scenario: EvalScenario;
     languageModel: LanguageModel;
     testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;

package/dist/models/types.d.ts

@@ -16,11 +16,29 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { SessionConversationSettings, SessionLanguageModel, testCasesSchema } from "@fonoster/common";
 import { CallDirection } from "@fonoster/types";
 import { BaseChatModel } from "@langchain/core/language_models/chat_models";
 import { ToolCall } from "@langchain/core/messages/tool";
+import { z } from "zod";
 import { KnowledgeBase } from "../knowledge";
 import { Tool } from "../tools/types";
+/** The language model vendors the LangChain models (used by the evaluations only) implement. */
+type LlmProvider = "openai" | "groq" | "ollama" | "google" | "anthropic";
+/**
+ * A session agent as the evaluations run it (rework-autopilot 8.2): the catalog id split into
+ * provider and model, and the keys the control plane resolved for this run. Never stored.
+ */
+type EvalAgent = {
+    conversationSettings: SessionConversationSettings;
+    languageModel: Omit<SessionLanguageModel, "model" | "provider" | "apiKey" | "baseUrl"> & {
+        provider: LlmProvider;
+        model: string;
+        apiKey?: string;
+        baseUrl?: string;
+    };
+    testCases?: z.infer<typeof testCasesSchema>;
+};
 type LanguageModel = {
     invoke: (text: string, isReentry?: boolean) => Promise<InvocationResult>;
 };
@@ -49,4 +67,4 @@ type TelephonyContext = {
     callerNumber: string;
     metadata?: Record<string, string>;
 };
-export { BaseModelParams, InvocationResult, LanguageModel, LanguageModelParams, TelephonyContext };
+export { BaseModelParams, EvalAgent, LlmProvider, InvocationResult, LanguageModel, LanguageModelParams, TelephonyContext };

package/dist/session/Agent.d.ts

@@ -0,0 +1,173 @@
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Usage } from "@fonoster/common";
+import { SessionAgent } from "./config";
+import { KnowledgePort } from "./knowledge";
+import { ChatMessage, Think } from "./think";
+import { ToolPort } from "./tools";
+import type { Session } from "@fonoster/voice";
+/** What Autopilot says when a transfer fails and the agent sets no `failedMessage`. */
+export declare const DEFAULT_TRANSFER_FAILED_MESSAGE = "Sorry, I couldn't connect you. How else can I help?";
+export type SpokenSource = "greeting" | "reply" | "idle" | "error";
+export type AgentEvent = {
+    type: "TURN_USER_STARTED";
+} | {
+    type: "TURN_LIKELY_ENDED";
+    text: string;
+} | {
+    type: "TURN_RESUMED";
+} | {
+    type: "TURN_ENDED";
+    text: string;
+} | {
+    type: "TURN_BACKCHANNEL";
+} | {
+    type: "REPLY_SPEAKING";
+}
+/** An output finished; `source` says which, so stale events from an earlier phase are ignored. */
+ | {
+    type: "SPOKEN";
+    source: SpokenSource;
+} | {
+    type: "IDLE";
+} | {
+    type: "MAX_DURATION";
+} | {
+    type: "HANGUP_REQUESTED";
+} | {
+    type: "TRANSFER_REQUESTED";
+} | {
+    type: "SESSION_END";
+};
+/** One turn's trace for the call record (rework-autopilot A10). */
+export type TurnTrace = {
+    userText: string;
+    replyText: string;
+    heardText: string;
+    interrupted: boolean;
+    toolCalls: {
+        name: string;
+        ok: boolean;
+    }[];
+    /** ms from turn.ended to the first token, and to the first speak sent. */
+    firstTokenMs?: number;
+    firstSpeakMs?: number;
+};
+export interface Recorder {
+    turn(trace: TurnTrace): void;
+    note(event: Record<string, unknown>): void;
+    /** Model token usage for the call (usage.report); absent where there is no call. */
+    usage?(lines: Usage.UsageLine[]): void;
+}
+/** Writes turn traces and notes to the call record through the session (record.note). */
+export declare function sessionRecorder(session: Session): Recorder;
+export declare const logRecorder: Recorder;
+/** The side effects of Autopilot: everything the state machine asks it to do on the session. */
+export declare class Agent {
+    private readonly session;
+    private readonly agent;
+    private readonly think;
+    private readonly tools;
+    private readonly recorder;
+    private readonly knowledge?;
+    readonly history: ChatMessage[];
+    private reply?;
+    private inflight?;
+    private prepared?;
+    private idleTimer?;
+    /** The model's `transfer` tool call, answered in the history if the transfer fails. */
+    private transferCallId?;
+    private maxTimer?;
+    private idleCount;
+    private resumes;
+    /** Whether a reply is prepared on turn.likely_ended (off at high patience or once adapted). */
+    private prepareEarly;
+    private stopped;
+    send: (event: AgentEvent) => void;
+    constructor(session: Session, agent: SessionAgent, think: Think, tools: ToolPort, recorder?: Recorder, knowledge?: KnowledgePort);
+    get settings(): {
+        firstMessage?: string;
+        systemPrompt?: string;
+        goodbyeMessage?: string;
+        systemErrorMessage?: string;
+        initialDtmf?: string;
+        maxSessionDuration?: number;
+        transferOptions?: {
+            message?: string;
+            phoneNumber?: string;
+            failedMessage?: string;
+        };
+        idleOptions?: {
+            message?: string;
+            timeout?: number;
+            maxTimeoutCount?: number;
+        };
+        allowUserBargeIn?: boolean;
+        patience?: "low" | "normal" | "high";
+        hints?: string[];
+        filler?: {
+            url?: string;
+        };
+        voice?: string;
+        language?: string;
+        vad?: any;
+        maxSpeechWaitTimeout?: any;
+    };
+    /** Greeting: subscribe to the estimate, send initial DTMF, speak the first message. */
+    greet(): Promise<void>;
+    stop(): void;
+    armIdle(): void;
+    disarmIdle(): void;
+    /** The next idle timeout ends the call (the goodbye counts as the last announcement). */
+    idleExhausted(): boolean;
+    sayIdle(): Promise<void>;
+    /** turn.likely_ended: start generating silently (when preparing early). */
+    prepare(text: string): void;
+    /** turn.resumed: the caller kept talking; the prepared reply is dropped. */
+    resumed(): void;
+    clear(): Promise<void>;
+    abortReply(): void;
+    /** turn.ended: answer the caller, streaming sentences into one speak id. */
+    answer(text: string): Promise<void>;
+    /** One turn (STT final, LLM, TTS) as an autopilot.turn span. */
+    private respond;
+    private respondTurn;
+    private speakRound;
+    private runTools;
+    goodbye(): Promise<void>;
+    transfer(): Promise<void>;
+    /**
+     * The transfer failed: say so and keep helping (autopilot "Transfer fails"). The model learns
+     * the outcome as its transfer call's result, and the machine returns to listening.
+     */
+    private transferFailed;
+    private messages;
+    /** Knowledge for this turn, as a system message for the request only (never in history). */
+    private contextFor;
+    private toolSpecs;
+    private startStream;
+    /** think.stream inside an autopilot.llm.request span, with request and latency metrics. */
+    private traced;
+    private takePrepared;
+    private discardPrepared;
+    private pump;
+    private speechOptions;
+    /** Runs session work; a session that has ended stops everything quietly. */
+    private safely;
+}