@fonoster/autopilot 0.27.0 → 0.30.0

package/dist/models/evaluations/session/EvalSession.d.ts

@@ -0,0 +1,73 @@
+/** What the engine did on the session, in order. */
+export type EvalOutput = {
+    kind: "say";
+    text: string;
+    /** True when the engine streamed the reply (say with an async iterable). */
+    streamed: boolean;
+    /** The pieces as they arrived (sentences, for a streamed reply). */
+    pieces: string[];
+} | {
+    kind: "play";
+    url: string;
+} | {
+    kind: "dtmf";
+    digits: string;
+} | {
+    kind: "hangup";
+} | {
+    kind: "transfer";
+    to: string;
+};
+export type EvalTelephony = {
+    callDirection?: string;
+    ingressNumber?: string;
+    callerNumber?: string;
+    metadata?: Record<string, string>;
+};
+/**
+ * An in-memory session for conversation evals (rework-autopilot 1.4): the session engine runs on
+ * it exactly as on a call, but there is no gateway, socket or audio. Turns go in as `turn.ended`;
+ * every reply is "heard" in full (no barge-in in an eval).
+ */
+export declare class EvalSession {
+    readonly outputs: EvalOutput[];
+    readonly sessionRef = "eval-session";
+    readonly callRef = "eval-call";
+    readonly appRef = "eval-app";
+    readonly direction: string;
+    readonly from: string;
+    readonly to: string;
+    readonly metadata: Record<string, string>;
+    private readonly events;
+    private pending;
+    private endedReason?;
+    constructor(telephony?: EvalTelephony);
+    get ended(): boolean;
+    /** Says in flight (a streamed reply is in flight until its last piece). */
+    get busy(): boolean;
+    on(event: string, listener: (payload: never) => void): this;
+    /** Observe outputs as they are recorded (the evaluator waits on these). */
+    onOutput(listener: (output: EvalOutput) => void): void;
+    /** The caller finished a turn. */
+    turn(text: string): void;
+    /** The call is over (end of scenario, or the engine hung up or transferred). */
+    end(reason?: string): void;
+    say(input: string | AsyncIterable<string>, _options?: unknown): Promise<{
+        interrupted: boolean;
+        heardText: string;
+        heardMs: number;
+    }>;
+    stopSay(): Promise<void>;
+    play(url: string, _options?: unknown): Promise<{
+        interrupted: boolean;
+        heardMs: number;
+    }>;
+    playDtmf(digits: string): Promise<void>;
+    subscribe(_events: string[]): Promise<void>;
+    listen(_options: unknown): Promise<void>;
+    note(_data: Record<string, unknown>): Promise<void>;
+    hangup(): Promise<void>;
+    transfer(to: string): Promise<void>;
+    private record;
+    private assertOpen;
+}

package/dist/models/evaluations/session/EvalSession.js

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EvalSession = void 0;
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const events_1 = require("events");
+const voice_1 = require("@fonoster/voice");
+/**
+ * An in-memory session for conversation evals (rework-autopilot 1.4): the session engine runs on
+ * it exactly as on a call, but there is no gateway, socket or audio. Turns go in as `turn.ended`;
+ * every reply is "heard" in full (no barge-in in an eval).
+ */
+class EvalSession {
+    constructor(telephony = {}) {
+        this.outputs = [];
+        this.sessionRef = "eval-session";
+        this.callRef = "eval-call";
+        this.appRef = "eval-app";
+        this.events = new events_1.EventEmitter();
+        this.pending = 0;
+        this.direction = (telephony.callDirection ?? "FROM_PSTN").startsWith("TO")
+            ? "outbound"
+            : "inbound";
+        this.from = telephony.callerNumber ?? "";
+        this.to = telephony.ingressNumber ?? "";
+        this.metadata = telephony.metadata ?? {};
+        this.events.setMaxListeners(50);
+    }
+    get ended() {
+        return this.endedReason !== undefined;
+    }
+    /** Says in flight (a streamed reply is in flight until its last piece). */
+    get busy() {
+        return this.pending > 0;
+    }
+    on(event, listener) {
+        this.events.on(event, listener);
+        return this;
+    }
+    /** Observe outputs as they are recorded (the evaluator waits on these). */
+    onOutput(listener) {
+        this.events.on("__output", listener);
+    }
+    /** The caller finished a turn. */
+    turn(text) {
+        this.events.emit("turn.ended", {
+            type: "turn.ended",
+            text,
+            language: "en",
+            confidence: 1,
+            durationMs: 0
+        });
+    }
+    /** The call is over (end of scenario, or the engine hung up or transferred). */
+    end(reason = "eval_done") {
+        if (this.ended)
+            return;
+        this.endedReason = reason;
+        this.events.emit("end", { type: "session.end", reason });
+    }
+    // ---- the Session surface the engine uses ---------------------------------------------
+    async say(input, _options = {}) {
+        this.assertOpen();
+        this.pending++;
+        try {
+            const pieces = [];
+            if (typeof input === "string")
+                pieces.push(input);
+            else
+                for await (const piece of input)
+                    pieces.push(piece);
+            const text = pieces.join("");
+            this.record({
+                kind: "say",
+                text,
+                streamed: typeof input !== "string",
+                pieces
+            });
+            return { interrupted: false, heardText: text, heardMs: 0 };
+        }
+        finally {
+            this.pending--;
+        }
+    }
+    async stopSay() { }
+    async play(url, _options = {}) {
+        this.assertOpen();
+        this.record({ kind: "play", url });
+        return { interrupted: false, heardMs: 0 };
+    }
+    async playDtmf(digits) {
+        this.assertOpen();
+        this.record({ kind: "dtmf", digits });
+    }
+    async subscribe(_events) { }
+    async listen(_options) { }
+    async note(_data) { }
+    async hangup() {
+        if (this.ended)
+            return;
+        this.record({ kind: "hangup" });
+        this.end("hangup");
+    }
+    async transfer(to) {
+        this.assertOpen();
+        this.record({ kind: "transfer", to });
+        this.end("transferred");
+    }
+    record(output) {
+        this.outputs.push(output);
+        this.events.emit("__output", output);
+    }
+    assertOpen() {
+        if (this.endedReason)
+            throw new voice_1.SessionEndedError(this.endedReason);
+    }
+}
+exports.EvalSession = EvalSession;

package/dist/models/evaluations/session/evaluateSessionScenario.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ScenarioEvaluationReport, StepEvaluationReport } from "@fonoster/types";
+import { SessionAssistant } from "../../../session/config";
+import { ChatMessage, Think } from "../../../session/think";
+import { ToolPort } from "../../../session/tools";
+import { EvalScenario } from "../types";
+/** A step report with what the session engine streamed (extra fields, ignored on the wire). */
+export type SessionStepReport = StepEvaluationReport & {
+    streaming: {
+        /** The pieces of the reply as the engine sent them to `say` (sentences). */
+        pieces: string[];
+        firstTokenMs?: number;
+        firstSpeakMs?: number;
+    };
+    /** Tool calls and results as stored in the history (structured tool messages). */
+    toolMessages: ChatMessage[];
+};
+export type SessionScenarioReport = ScenarioEvaluationReport & {
+    steps: SessionStepReport[];
+};
+export type SessionEvalOptions = {
+    assistant: SessionAssistant;
+    /** The language model the engine streams from (a registry adapter, or the fake LLM). */
+    think: Think;
+    testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
+    /**
+     * Runs the assistant's own tools. Defaults to `evalToolPort`: evals never call tool endpoints
+     * (an eval must not book an appointment), every call succeeds with `{ status: "ok" }`.
+     */
+    tools?: ToolPort;
+    /** Upper bound for one step (default 60 s). */
+    stepTimeoutMs?: number;
+};
+/** The default eval tool port: records nothing, calls nothing, every tool succeeds. */
+export declare const evalToolPort: ToolPort;
+/**
+ * Runs one test-case scenario through the session engine (rework-autopilot 1.4): the real state
+ * machine and Agent on an in-memory session, replies streamed sentence by sentence, tool calls
+ * read back from the structured tool messages in the history.
+ */
+export declare function evaluateSessionScenario(options: SessionEvalOptions, scenario: EvalScenario, onStepResult?: (scenarioRef: string, report: StepEvaluationReport) => void | Promise<void>): Promise<SessionScenarioReport>;

package/dist/models/evaluations/session/evaluateSessionScenario.js

@@ -0,0 +1,182 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evalToolPort = void 0;
+exports.evaluateSessionScenario = evaluateSessionScenario;
+const createAutopilot_1 = require("../../../session/createAutopilot");
+const evaluateTextResponse_1 = require("../evaluateTextResponse");
+const evaluateToolCalls_1 = require("../evaluateToolCalls");
+const EvalSession_1 = require("./EvalSession");
+/** The default eval tool port: records nothing, calls nothing, every tool succeeds. */
+exports.evalToolPort = {
+    run: async () => ({
+        ok: true,
+        result: { status: "ok" }
+    })
+};
+const SETTLE_POLL_MS = 10;
+/** Resolves once the session has had no new output and no say in flight for two polls. */
+async function settle(session) {
+    let seen = -1;
+    let quiet = 0;
+    while (quiet < 2) {
+        await new Promise((r) => setTimeout(r, SETTLE_POLL_MS));
+        if (!session.busy && session.outputs.length === seen)
+            quiet++;
+        else
+            quiet = 0;
+        seen = session.outputs.length;
+    }
+}
+function parseArgs(call) {
+    try {
+        return call.arguments ? JSON.parse(call.arguments) : {};
+    }
+    catch {
+        return {};
+    }
+}
+/**
+ * Runs one test-case scenario through the session engine (rework-autopilot 1.4): the real state
+ * machine and Agent on an in-memory session, replies streamed sentence by sentence, tool calls
+ * read back from the structured tool messages in the history.
+ */
+async function evaluateSessionScenario(options, scenario, onStepResult) {
+    const { assistant, testTextSimilarity } = options;
+    const stepTimeoutMs = options.stepTimeoutMs ?? 60000;
+    const session = new EvalSession_1.EvalSession(scenario.telephonyContext);
+    let agent;
+    let signal;
+    let thinkError;
+    const traces = [];
+    // Think errors are swallowed by the engine (it apologises); keep them for the report.
+    const think = {
+        stream: (messages, tools, abort) => (async function* () {
+            try {
+                yield* options.think.stream(messages, tools, abort);
+            }
+            catch (e) {
+                if (!abort.aborted) {
+                    thinkError = e;
+                    signal?.();
+                }
+                throw e;
+            }
+        })()
+    };
+    const handler = (0, createAutopilot_1.createAutopilotHandler)({
+        assistant: { ...assistant, testCases: undefined },
+        think,
+        tools: options.tools ?? exports.evalToolPort,
+        recorder: {
+            turn: (t) => {
+                traces.push(t);
+                signal?.();
+            },
+            note: () => undefined
+        },
+        onAgent: (a) => (agent = a)
+    });
+    session.onOutput((o) => {
+        if (o.kind === "hangup" || o.kind === "transfer")
+            signal?.();
+    });
+    const running = handler(session);
+    const results = [];
+    try {
+        await settle(session); // greeting
+        for (const step of scenario.conversation) {
+            const type = step.expected.text.type;
+            const report = {
+                humanInput: step.userInput,
+                expectedResponse: step.expected.text.response,
+                aiResponse: "",
+                evaluationType: type,
+                passed: true,
+                streaming: { pieces: [] },
+                toolMessages: []
+            };
+            results.push(report);
+            if (session.ended) {
+                report.passed = false;
+                report.errorMessage = `The call had already ended before "${step.userInput}".`;
+                if (onStepResult)
+                    await onStepResult(scenario.ref, report);
+                continue;
+            }
+            const outputsFrom = session.outputs.length;
+            const historyFrom = agent?.history.length ?? 0;
+            const tracesFrom = traces.length;
+            thinkError = undefined;
+            let timer;
+            const done = new Promise((resolve) => {
+                signal = () => resolve(true);
+                timer = setTimeout(() => resolve(false), stepTimeoutMs);
+            });
+            session.turn(step.userInput);
+            const finished = await done;
+            clearTimeout(timer);
+            signal = undefined;
+            await settle(session);
+            const outputs = session.outputs.slice(outputsFrom);
+            const says = outputs.filter((o) => o.kind === "say");
+            const reply = says.filter((s) => s.streamed);
+            const history = agent?.history.slice(historyFrom) ?? [];
+            const toolCalls = history.flatMap((m) => m.role === "assistant" ? (m.tool_calls ?? []) : []);
+            const builtIn = toolCalls.find((c) => c.name === "hangup" || c.name === "transfer");
+            const trace = traces[tracesFrom];
+            report.streaming = {
+                pieces: reply.flatMap((s) => s.pieces),
+                firstTokenMs: trace?.firstTokenMs,
+                firstSpeakMs: trace?.firstSpeakMs
+            };
+            report.toolMessages = history.filter((m) => m.role === "tool" || (m.role === "assistant" && m.tool_calls));
+            // What the caller hears: the streamed reply; for hangup or transfer, the message the
+            // engine says for it (as the legacy evals compared).
+            report.aiResponse = builtIn
+                ? builtIn.name === "hangup"
+                    ? assistant.conversationSettings.goodbyeMessage
+                    : (assistant.conversationSettings.transferOptions?.message ?? "")
+                : reply
+                    .map((s) => s.text)
+                    .join("")
+                    .trim();
+            if (thinkError) {
+                report.passed = false;
+                report.aiResponse = says.map((s) => s.text).join(" ");
+                report.errorMessage = `Language model error for input "${step.userInput}": ${thinkError.message}`;
+            }
+            else if (!finished) {
+                report.passed = false;
+                report.errorMessage = `No reply within ${stepTimeoutMs} ms for "${step.userInput}".`;
+            }
+            else {
+                const text = await (0, evaluateTextResponse_1.evaluateTextResponse)({ type, response: step.expected.text.response }, report.aiResponse, testTextSimilarity);
+                if (!text.passed) {
+                    report.passed = false;
+                    report.errorMessage = text.errorMessage;
+                }
+                if (step.expected.tools && step.expected.tools.length > 0) {
+                    const tools = (0, evaluateToolCalls_1.evaluateToolCalls)(step.expected.tools, toolCalls.map((c) => ({ name: c.name, args: parseArgs(c) })));
+                    report.toolEvaluations = tools.evaluations;
+                    if (!tools.passed) {
+                        report.passed = false;
+                        report.errorMessage = report.errorMessage
+                            ? `${report.errorMessage} ${tools.errorMessage}`
+                            : tools.errorMessage;
+                    }
+                }
+            }
+            if (onStepResult)
+                await onStepResult(scenario.ref, report);
+        }
+    }
+    finally {
+        session.end();
+        await running.catch(() => undefined);
+    }
+    return {
+        scenarioRef: scenario.ref,
+        overallPassed: results.every((s) => s.passed),
+        steps: results
+    };
+}

package/dist/{vad → models/evaluations/session}/index.d.ts

@@ -16,5 +16,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export * from "./SileroVad";
-export * from "./types";
+export * from "./EvalSession";
+export * from "./evaluateSessionScenario";
+export * from "./runSessionEval";

package/dist/{vad → models/evaluations/session}/index.js

@@ -32,5 +32,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-__exportStar(require("./SileroVad"), exports);
-__exportStar(require("./types"), exports);
+__exportStar(require("./EvalSession"), exports);
+__exportStar(require("./evaluateSessionScenario"), exports);
+__exportStar(require("./runSessionEval"), exports);

package/dist/models/evaluations/session/runSessionEval.d.ts

@@ -0,0 +1,17 @@
+import { SessionAssistant } from "../../../session/config";
+import { Think } from "../../../session/think";
+import { RunEvalCallbacks } from "../types";
+import { SessionEvalOptions, SessionScenarioReport } from "./evaluateSessionScenario";
+export type RunSessionEvalOptions = Omit<SessionEvalOptions, "testTextSimilarity"> & {
+    /** Defaults to the OpenAI similarity judge from the assistant's testCases. */
+    testTextSimilarity?: SessionEvalOptions["testTextSimilarity"];
+    /** The judge's key (the testCases model is always OpenAI). */
+    evaluationApiKey?: string;
+};
+/**
+ * The session-engine Think for an assistant whose catalog provider speaks OpenAI-compatible Chat
+ * Completions with a known endpoint (openai, groq, ollama); undefined otherwise.
+ */
+export declare function sessionThinkFor(assistant: SessionAssistant, apiKey: string | undefined): Think | undefined;
+/** Runs every test-case scenario through the session engine, streaming results. */
+export declare function runSessionEval(options: RunSessionEvalOptions, callbacks?: Partial<RunEvalCallbacks>): Promise<SessionScenarioReport[]>;

package/dist/models/evaluations/session/runSessionEval.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sessionThinkFor = sessionThinkFor;
+exports.runSessionEval = runSessionEval;
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const common_1 = require("@fonoster/common");
+const openai_1 = require("@fonoster/community/openai");
+const createTestTextSimilarity_1 = require("../createTestTextSimilarity");
+const textSimilaryPrompt_1 = require("../textSimilaryPrompt");
+const evaluateSessionScenario_1 = require("./evaluateSessionScenario");
+/**
+ * The session-engine Think for an assistant whose catalog provider speaks OpenAI-compatible Chat
+ * Completions with a known endpoint (openai, groq, ollama); undefined otherwise.
+ */
+function sessionThinkFor(assistant, apiKey) {
+    const id = assistant.languageModel.model;
+    const slash = id.indexOf("/");
+    const baseUrl = openai_1.OPENAI_COMPATIBLE[id.slice(0, slash)];
+    if (!baseUrl)
+        return undefined;
+    return new openai_1.OpenAiCompatibleThink({ baseUrl, apiKey }, id.slice(slash + 1), {
+        temperature: assistant.languageModel.temperature,
+        maxTokens: assistant.languageModel.maxTokens
+    });
+}
+/** Runs every test-case scenario through the session engine, streaming results. */
+async function runSessionEval(options, callbacks) {
+    const testCases = common_1.testCasesSchema.parse(options.assistant.testCases);
+    const testTextSimilarity = options.testTextSimilarity ??
+        (0, createTestTextSimilarity_1.createTestTextSimilarity)({
+            provider: testCases.evalsLanguageModel.provider,
+            model: testCases.evalsLanguageModel.model,
+            apiKey: options.evaluationApiKey ?? testCases.evalsLanguageModel.apiKey
+        }, testCases.evalsSystemPrompt || textSimilaryPrompt_1.textSimilaryPrompt);
+    const reports = [];
+    for (const scenario of testCases.scenarios) {
+        const report = await (0, evaluateSessionScenario_1.evaluateSessionScenario)({ ...options, testTextSimilarity }, scenario, callbacks?.onStepResult);
+        reports.push(report);
+        await callbacks?.onScenarioComplete?.(report.scenarioRef, report.overallPassed);
+    }
+    return reports;
+}

package/dist/models/evaluations/types.d.ts

@@ -17,8 +17,7 @@
  * limitations under the License.
  */
 import { StepEvaluationReport } from "@fonoster/types";
-import { AssistantConfig } from "../../assistants";
-import { LanguageModel } from "../types";
+import { EvalAssistant, LanguageModel } from "../types";
 type EvalExpectedTool = {
     tool: string;
     parameters?: Record<string, unknown>;
@@ -46,18 +45,20 @@ type EvalScenario = {
 };
 type EvaluateIntelligenceRequest = {
     intelligence: {
-        productRef: string;
-        config: AssistantConfig;
+        /** Ignored: the assistant's catalog id names the provider. */
+        productRef?: string;
+        /** A session assistant (with testCases), as a protobuf Struct. */
+        config: unknown;
     };
 };
 type EvaluateStepParams = {
     step: EvalConversationStep;
     languageModel: LanguageModel;
     testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
-    assistantConfig: AssistantConfig;
+    assistantConfig: EvalAssistant;
 };
 type ScenarioEvaluationRequest = {
-    assistantConfig: AssistantConfig;
+    assistantConfig: EvalAssistant;
     scenario: EvalScenario;
     languageModel: LanguageModel;
     testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;

package/dist/models/types.d.ts

@@ -16,11 +16,29 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { SessionConversationSettings, SessionLanguageModel, testCasesSchema } from "@fonoster/common";
 import { CallDirection } from "@fonoster/types";
 import { BaseChatModel } from "@langchain/core/language_models/chat_models";
 import { ToolCall } from "@langchain/core/messages/tool";
+import { z } from "zod";
 import { KnowledgeBase } from "../knowledge";
 import { Tool } from "../tools/types";
+/** The language model vendors the LangChain models (used by the evaluations only) implement. */
+type LlmProvider = "openai" | "groq" | "ollama" | "google" | "anthropic";
+/**
+ * A session assistant as the evaluations run it (rework-autopilot 8.2): the catalog id split into
+ * provider and model, and the keys the control plane resolved for this run. Never stored.
+ */
+type EvalAssistant = {
+    conversationSettings: SessionConversationSettings;
+    languageModel: Omit<SessionLanguageModel, "model" | "provider" | "apiKey" | "baseUrl"> & {
+        provider: LlmProvider;
+        model: string;
+        apiKey?: string;
+        baseUrl?: string;
+    };
+    testCases?: z.infer<typeof testCasesSchema>;
+};
 type LanguageModel = {
     invoke: (text: string, isReentry?: boolean) => Promise<InvocationResult>;
 };
@@ -49,4 +67,4 @@ type TelephonyContext = {
     callerNumber: string;
     metadata?: Record<string, string>;
 };
-export { BaseModelParams, InvocationResult, LanguageModel, LanguageModelParams, TelephonyContext };
+export { BaseModelParams, EvalAssistant, LlmProvider, InvocationResult, LanguageModel, LanguageModelParams, TelephonyContext };