@fonoster/autopilot 0.27.0 → 0.31.0

package/dist/models/evaluations/createEvaluateIntelligence.js

@@ -19,48 +19,78 @@ exports.createEvaluateIntelligence = createEvaluateIntelligence;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+const crypto_1 = require("crypto");
 const common_1 = require("@fonoster/common");
 const logger_1 = require("@fonoster/logger");
 const pb_util_1 = require("pb-util");
-const zod_1 = require("zod");
+const meteredThink_1 = require("../../session/meteredThink");
 const createEvalEffectiveConfig_1 = require("./createEvalEffectiveConfig");
 const runEval_1 = require("./runEval");
+const session_1 = require("./session");
 const stepReportToEventPayload_1 = require("./stepReportToEventPayload");
 const logger = (0, logger_1.getLogger)({ service: "apiserver", filePath: __filename });
-function createEvaluateIntelligence(integrations) {
+const keyOf = (k) => typeof k === "string" ? { key: k, source: "platform" } : k;
+/**
+ * Runs an agent's test cases (rework-autopilot 8.2: the agent is a session agent; its
+ * catalog id picks the provider, and keys come from `keys`, never from the agent).
+ */
+function createEvaluateIntelligence(keys, reportUsage) {
     const evaluateIntelligence = async (call) => {
         const { request } = call;
         const { intelligence } = request;
         const accessKeyId = (0, common_1.getAccessKeyIdFromCall)(call);
-        logger.verbose("call to evaluateIntelligence", {
-            accessKeyId,
-            llmProductRef: intelligence.productRef,
-            evalLlmProductRef: "llm.openai"
-        });
         try {
             const config = pb_util_1.struct.decode(intelligence.config);
-            const parsedIntelligence = zod_1.z
-                .object({
-                productRef: zod_1.z.string(),
-                config: common_1.assistantSchema
-            })
-                .parse({
-                productRef: intelligence.productRef,
-                config
+            const agent = common_1.sessionAgentSchema.parse(config);
+            const provider = agent.languageModel.model.split("/")[0];
+            logger.verbose("call to evaluateIntelligence", {
+                accessKeyId,
+                model: agent.languageModel.model
             });
-            const credentials = (0, common_1.findIntegrationsCredentials)(integrations, intelligence.productRef);
-            const evaluationApiKey = (0, common_1.findIntegrationsCredentials)(integrations, "llm.openai");
-            const effectiveConfig = (0, createEvalEffectiveConfig_1.createEvalEffectiveConfig)(parsedIntelligence.config, credentials, evaluationApiKey);
-            await (0, runEval_1.runEval)({ intelligence: { config: effectiveConfig } }, {
+            const agentKey = keyOf(await keys(provider, accessKeyId));
+            const judgeKey = keyOf(await keys("openai", accessKeyId));
+            const apiKey = agentKey?.key;
+            const evaluationApiKey = judgeKey?.key;
+            // Eval tokens are metered with origin eval, one scope per request of this run.
+            const runId = (0, crypto_1.randomUUID)();
+            let request = 0;
+            const sink = {
+                usage: (lines) => reportUsage?.(accessKeyId, lines, `eval|${runId}|${request++}`),
+                missing: (catalogId) => logger.warn(`eval: ${catalogId} reported no usage`, { accessKeyId })
+            };
+            if (!apiKey && provider !== "ollama")
+                throw new Error(`no ${provider} key for this workspace or platform`);
+            if (!evaluationApiKey)
+                throw new Error("no openai key for the evaluation model (workspace or platform)");
+            const callbacks = {
                 onStepResult: (scenarioRef, stepReport) => {
-                    const payload = (0, stepReportToEventPayload_1.stepReportToEventPayload)(scenarioRef, stepReport);
-                    call.write(payload);
+                    call.write((0, stepReportToEventPayload_1.stepReportToEventPayload)(scenarioRef, stepReport));
                 },
                 onScenarioComplete: (scenarioRef, overallPassed) => {
-                    const payload = (0, stepReportToEventPayload_1.scenarioSummaryToEventPayload)(scenarioRef, overallPassed);
-                    call.write(payload);
+                    call.write((0, stepReportToEventPayload_1.scenarioSummaryToEventPayload)(scenarioRef, overallPassed));
                 }
+            };
+            // The session engine (streaming replies, structured tool messages) for OpenAI-compatible
+            // providers; the LangChain path stays for the rest until their adapters land (EV-B1).
+            const sessionThink = (0, session_1.sessionThinkFor)(agent, apiKey);
+            if (sessionThink) {
+                const think = (0, meteredThink_1.meteredThink)(sessionThink, {
+                    catalogId: agent.languageModel.model,
+                    keySource: agentKey?.source ?? "platform"
+                }, sink);
+                await (0, session_1.runSessionEval)({
+                    agent,
+                    think,
+                    evaluationApiKey,
+                    judgeUsage: { sink, keySource: judgeKey?.source ?? "platform" }
+                }, callbacks);
+                return;
+            }
+            const effectiveConfig = (0, createEvalEffectiveConfig_1.createEvalEffectiveConfig)(agent, {
+                apiKey,
+                evaluationApiKey
             });
+            await (0, runEval_1.runEval)({ intelligence: { config: effectiveConfig } }, callbacks);
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);

package/dist/models/evaluations/createTestTextSimilarity.d.ts

@@ -1,6 +1,12 @@
+import { MeteredModel, UsageSink } from "../../session/meteredThink";
 export declare function createTestTextSimilarity(evalsLanguageModel: {
     provider: any;
     model: string;
     baseUrl?: string;
     apiKey?: string;
-}, systemPrompt: string): (text1: string, text2: string) => Promise<boolean>;
+}, systemPrompt: string, 
+/** Meters the judge's tokens (add-usage-metering); the judge is always OpenAI. */
+usage?: {
+    sink: UsageSink;
+    keySource: MeteredModel["keySource"];
+}): (text1: string, text2: string) => Promise<boolean>;

package/dist/models/evaluations/createTestTextSimilarity.js

@@ -21,7 +21,10 @@ exports.createTestTextSimilarity = createTestTextSimilarity;
  */
 const messages_1 = require("@langchain/core/messages");
 const openai_1 = require("@langchain/openai");
-function createTestTextSimilarity(evalsLanguageModel, systemPrompt) {
+const meteredThink_1 = require("../../session/meteredThink");
+function createTestTextSimilarity(evalsLanguageModel, systemPrompt, 
+/** Meters the judge's tokens (add-usage-metering); the judge is always OpenAI. */
+usage) {
     if (!evalsLanguageModel.apiKey) {
         throw new Error("API key is required for text similarity evaluation.");
     }
@@ -37,6 +40,21 @@ function createTestTextSimilarity(evalsLanguageModel, systemPrompt) {
             new messages_1.HumanMessage(`Text 1: ${text1}\nText 2: ${text2}`)
         ];
         const response = await llm.invoke(messages);
+        if (usage) {
+            const model = {
+                catalogId: `openai/${evalsLanguageModel.model}`,
+                keySource: usage.keySource
+            };
+            const m = response.usage_metadata;
+            if (m)
+                usage.sink.usage((0, meteredThink_1.usageLines)(model, {
+                    inputTokens: m.input_tokens,
+                    outputTokens: m.output_tokens,
+                    cachedTokens: m.input_token_details?.cache_read ?? 0
+                }));
+            else
+                usage.sink.missing(model.catalogId);
+        }
         const reply = response.content?.toString().trim().toLowerCase();
         return reply === "true" || reply === "yes";
     };

package/dist/models/evaluations/evalTestCases.d.ts

@@ -17,9 +17,9 @@
  * limitations under the License.
  */
 import { ScenarioEvaluationReport } from "@fonoster/types";
-import { AssistantConfig } from "../../assistants";
+import { EvalAgent } from "../types";
 export declare function evalTestCases(autopilotApplication: {
     intelligence: {
-        config: AssistantConfig;
+        config: EvalAgent;
     };
 }): Promise<ScenarioEvaluationReport[]>;

package/dist/models/evaluations/evalTestCases.js

@@ -7,7 +7,7 @@ const evaluateScenario_1 = require("./evaluateScenario");
 const textSimilaryPrompt_1 = require("./textSimilaryPrompt");
 async function evalTestCases(autopilotApplication) {
     const { testCases } = autopilotApplication.intelligence.config;
-    const { config: assistantConfig } = autopilotApplication.intelligence;
+    const { config: agentConfig } = autopilotApplication.intelligence;
     const voice = {
         say: async (_) => { }
     };
@@ -17,7 +17,7 @@ async function evalTestCases(autopilotApplication) {
         const scenario = scenarios[i];
         const languageModel = (0, createLanguageModel_1.createLanguageModel)({
             voice,
-            assistantConfig: autopilotApplication.intelligence.config,
+            agentConfig: autopilotApplication.intelligence.config,
             knowledgeBase: {
                 load: async () => { },
                 queryKnowledgeBase: async (query) => query
@@ -25,12 +25,12 @@ async function evalTestCases(autopilotApplication) {
             telephonyContext: scenario.telephonyContext
         });
         const testTextSimilarity = (0, createTestTextSimilarity_1.createTestTextSimilarity)({
-            provider: assistantConfig.testCases?.evalsLanguageModel?.provider,
-            model: assistantConfig.testCases?.evalsLanguageModel?.model ?? "",
-            apiKey: assistantConfig.testCases?.evalsLanguageModel?.apiKey
-        }, assistantConfig.testCases?.evalsSystemPrompt || textSimilaryPrompt_1.textSimilaryPrompt);
+            provider: agentConfig.testCases?.evalsLanguageModel?.provider,
+            model: agentConfig.testCases?.evalsLanguageModel?.model ?? "",
+            apiKey: agentConfig.testCases?.evalsLanguageModel?.apiKey
+        }, agentConfig.testCases?.evalsSystemPrompt || textSimilaryPrompt_1.textSimilaryPrompt);
         const evaluationReport = await (0, evaluateScenario_1.evaluateScenario)({
-            assistantConfig,
+            agentConfig,
             scenario,
             languageModel,
             testTextSimilarity

package/dist/models/evaluations/evaluateScenario.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.evaluateScenario = evaluateScenario;
 const evaluateStep_1 = require("./evaluateStep");
 async function evaluateScenario(config, onStepResult) {
-    const { scenario, languageModel, testTextSimilarity, assistantConfig } = config;
+    const { scenario, languageModel, testTextSimilarity, agentConfig } = config;
     const results = [];
     const steps = scenario.conversation;
     for (let i = 0; i < steps.length; i++) {
@@ -12,7 +12,7 @@ async function evaluateScenario(config, onStepResult) {
             step,
             languageModel,
             testTextSimilarity,
-            assistantConfig
+            agentConfig
         });
         results.push(stepResult);
         if (onStepResult)

package/dist/models/evaluations/evaluateStep.d.ts

@@ -18,4 +18,4 @@
  */
 import { StepEvaluationReport } from "@fonoster/types";
 import { EvaluateStepParams } from "./types";
-export declare function evaluateStep({ step, languageModel, testTextSimilarity, assistantConfig }: EvaluateStepParams): Promise<StepEvaluationReport>;
+export declare function evaluateStep({ step, languageModel, testTextSimilarity, agentConfig }: EvaluateStepParams): Promise<StepEvaluationReport>;

package/dist/models/evaluations/evaluateStep.js

@@ -4,7 +4,7 @@ exports.evaluateStep = evaluateStep;
 const evaluateTextResponse_1 = require("./evaluateTextResponse");
 const evaluateToolCalls_1 = require("./evaluateToolCalls");
 const extractAiResponse_1 = require("./extractAiResponse");
-async function evaluateStep({ step, languageModel, testTextSimilarity, assistantConfig }) {
+async function evaluateStep({ step, languageModel, testTextSimilarity, agentConfig }) {
     const stepResult = {
         humanInput: step.userInput,
         expectedResponse: step.expected.text.response,
@@ -14,7 +14,7 @@ async function evaluateStep({ step, languageModel, testTextSimilarity, assistant
     };
     try {
         const response = await languageModel.invoke(step.userInput);
-        stepResult.aiResponse = (0, extractAiResponse_1.extractAiResponse)(response, assistantConfig);
+        stepResult.aiResponse = (0, extractAiResponse_1.extractAiResponse)(response, agentConfig);
         const textEvaluation = await (0, evaluateTextResponse_1.evaluateTextResponse)({
             type: step.expected.text.type,
             response: step.expected.text.response

package/dist/models/evaluations/extractAiResponse.d.ts

@@ -16,10 +16,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { AssistantConfig } from "../../assistants";
-import { InvocationResult } from "../types";
+import { EvalAgent, InvocationResult } from "../types";
 /**
  * Maps language model invocation result to the text that should be compared
  * in eval (hangup/transfer use config messages).
  */
-export declare function extractAiResponse(response: InvocationResult, assistantConfig: AssistantConfig): string;
+export declare function extractAiResponse(response: InvocationResult, agentConfig: EvalAgent): string;

package/dist/models/evaluations/extractAiResponse.js

@@ -5,14 +5,14 @@ exports.extractAiResponse = extractAiResponse;
  * Maps language model invocation result to the text that should be compared
  * in eval (hangup/transfer use config messages).
  */
-function extractAiResponse(response, assistantConfig) {
+function extractAiResponse(response, agentConfig) {
     if (response.toolCalls && response.toolCalls.length > 0) {
         const topTool = response.toolCalls[0];
         if (topTool.name === "hangup") {
-            return assistantConfig.conversationSettings?.goodbyeMessage ?? "";
+            return agentConfig.conversationSettings?.goodbyeMessage ?? "";
         }
         if (topTool.name === "transfer") {
-            return (assistantConfig.conversationSettings?.transferOptions?.message ?? "");
+            return agentConfig.conversationSettings?.transferOptions?.message ?? "";
         }
     }
     return response.content ?? "";

package/dist/models/evaluations/index.d.ts

@@ -28,3 +28,4 @@ export * from "./extractAiResponse";
 export * from "./isValidIso8601Date";
 export * from "./runEval";
 export * from "./types";
+export * from "./session";

package/dist/models/evaluations/index.js

@@ -44,3 +44,4 @@ __exportStar(require("./extractAiResponse"), exports);
 __exportStar(require("./isValidIso8601Date"), exports);
 __exportStar(require("./runEval"), exports);
 __exportStar(require("./types"), exports);
+__exportStar(require("./session"), exports);

package/dist/models/evaluations/runEval.d.ts

@@ -1,26 +1,8 @@
-/**
- * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
- * http://github.com/fonoster/fonoster
- *
- * This file is part of Fonoster
- *
- * Licensed under the MIT License (the "License");
- * you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    https://opensource.org/licenses/MIT
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import { AssistantConfig } from "../../assistants";
+import { EvalAgent } from "../types";
 import { RunEvalCallbacks } from "./types";
 export type RunEvalConfig = {
     intelligence: {
-        config: AssistantConfig;
+        config: EvalAgent;
     };
 };
 export declare function runEval(autopilotApplication: RunEvalConfig, callbacks: RunEvalCallbacks): Promise<void>;

package/dist/models/evaluations/runEval.js

@@ -7,7 +7,7 @@ const evaluateScenario_1 = require("./evaluateScenario");
 const textSimilaryPrompt_1 = require("./textSimilaryPrompt");
 async function runEval(autopilotApplication, callbacks) {
     const { testCases } = autopilotApplication.intelligence.config;
-    const assistantConfig = autopilotApplication.intelligence.config;
+    const agentConfig = autopilotApplication.intelligence.config;
     const voice = {
         say: async (_) => { }
     };
@@ -15,7 +15,7 @@ async function runEval(autopilotApplication, callbacks) {
     for (const scenario of scenarios) {
         const languageModel = (0, createLanguageModel_1.createLanguageModel)({
             voice,
-            assistantConfig,
+            agentConfig,
             knowledgeBase: {
                 load: async () => { },
                 queryKnowledgeBase: async (query) => query
@@ -23,12 +23,12 @@ async function runEval(autopilotApplication, callbacks) {
             telephonyContext: scenario.telephonyContext
         });
         const testTextSimilarity = (0, createTestTextSimilarity_1.createTestTextSimilarity)({
-            provider: assistantConfig.testCases?.evalsLanguageModel?.provider,
-            model: assistantConfig.testCases?.evalsLanguageModel?.model ?? "",
-            apiKey: assistantConfig.testCases?.evalsLanguageModel?.apiKey
-        }, assistantConfig.testCases?.evalsSystemPrompt ?? textSimilaryPrompt_1.textSimilaryPrompt);
+            provider: agentConfig.testCases?.evalsLanguageModel?.provider,
+            model: agentConfig.testCases?.evalsLanguageModel?.model ?? "",
+            apiKey: agentConfig.testCases?.evalsLanguageModel?.apiKey
+        }, agentConfig.testCases?.evalsSystemPrompt ?? textSimilaryPrompt_1.textSimilaryPrompt);
         const report = await (0, evaluateScenario_1.evaluateScenario)({
-            assistantConfig,
+            agentConfig,
             scenario,
             languageModel,
             testTextSimilarity

package/dist/models/evaluations/session/EvalSession.d.ts

@@ -0,0 +1,73 @@
+/** What the engine did on the session, in order. */
+export type EvalOutput = {
+    kind: "say";
+    text: string;
+    /** True when the engine streamed the reply (say with an async iterable). */
+    streamed: boolean;
+    /** The pieces as they arrived (sentences, for a streamed reply). */
+    pieces: string[];
+} | {
+    kind: "play";
+    url: string;
+} | {
+    kind: "dtmf";
+    digits: string;
+} | {
+    kind: "hangup";
+} | {
+    kind: "transfer";
+    to: string;
+};
+export type EvalTelephony = {
+    callDirection?: string;
+    ingressNumber?: string;
+    callerNumber?: string;
+    metadata?: Record<string, string>;
+};
+/**
+ * An in-memory session for conversation evals (rework-autopilot 1.4): the session engine runs on
+ * it exactly as on a call, but there is no AppBridge, socket or audio. Turns go in as `turn.ended`;
+ * every reply is "heard" in full (no barge-in in an eval).
+ */
+export declare class EvalSession {
+    readonly outputs: EvalOutput[];
+    readonly sessionRef = "eval-session";
+    readonly callRef = "eval-call";
+    readonly appRef = "eval-app";
+    readonly direction: string;
+    readonly from: string;
+    readonly to: string;
+    readonly metadata: Record<string, string>;
+    private readonly events;
+    private pending;
+    private endedReason?;
+    constructor(telephony?: EvalTelephony);
+    get ended(): boolean;
+    /** Says in flight (a streamed reply is in flight until its last piece). */
+    get busy(): boolean;
+    on(event: string, listener: (payload: never) => void): this;
+    /** Observe outputs as they are recorded (the evaluator waits on these). */
+    onOutput(listener: (output: EvalOutput) => void): void;
+    /** The caller finished a turn. */
+    turn(text: string): void;
+    /** The call is over (end of scenario, or the engine hung up or transferred). */
+    end(reason?: string): void;
+    say(input: string | AsyncIterable<string>, _options?: unknown): Promise<{
+        interrupted: boolean;
+        heardText: string;
+        heardMs: number;
+    }>;
+    stopSay(): Promise<void>;
+    play(url: string, _options?: unknown): Promise<{
+        interrupted: boolean;
+        heardMs: number;
+    }>;
+    playDtmf(digits: string): Promise<void>;
+    subscribe(_events: string[]): Promise<void>;
+    listen(_options: unknown): Promise<void>;
+    note(_data: Record<string, unknown>): Promise<void>;
+    hangup(): Promise<void>;
+    transfer(to: string): Promise<void>;
+    private record;
+    private assertOpen;
+}

package/dist/models/evaluations/session/EvalSession.js

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EvalSession = void 0;
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const events_1 = require("events");
+const voice_1 = require("@fonoster/voice");
+/**
+ * An in-memory session for conversation evals (rework-autopilot 1.4): the session engine runs on
+ * it exactly as on a call, but there is no AppBridge, socket or audio. Turns go in as `turn.ended`;
+ * every reply is "heard" in full (no barge-in in an eval).
+ */
+class EvalSession {
+    constructor(telephony = {}) {
+        this.outputs = [];
+        this.sessionRef = "eval-session";
+        this.callRef = "eval-call";
+        this.appRef = "eval-app";
+        this.events = new events_1.EventEmitter();
+        this.pending = 0;
+        this.direction = (telephony.callDirection ?? "FROM_PSTN").startsWith("TO")
+            ? "outbound"
+            : "inbound";
+        this.from = telephony.callerNumber ?? "";
+        this.to = telephony.ingressNumber ?? "";
+        this.metadata = telephony.metadata ?? {};
+        this.events.setMaxListeners(50);
+    }
+    get ended() {
+        return this.endedReason !== undefined;
+    }
+    /** Says in flight (a streamed reply is in flight until its last piece). */
+    get busy() {
+        return this.pending > 0;
+    }
+    on(event, listener) {
+        this.events.on(event, listener);
+        return this;
+    }
+    /** Observe outputs as they are recorded (the evaluator waits on these). */
+    onOutput(listener) {
+        this.events.on("__output", listener);
+    }
+    /** The caller finished a turn. */
+    turn(text) {
+        this.events.emit("turn.ended", {
+            type: "turn.ended",
+            text,
+            language: "en",
+            confidence: 1,
+            durationMs: 0
+        });
+    }
+    /** The call is over (end of scenario, or the engine hung up or transferred). */
+    end(reason = "eval_done") {
+        if (this.ended)
+            return;
+        this.endedReason = reason;
+        this.events.emit("end", { type: "session.end", reason });
+    }
+    // ---- the Session surface the engine uses ---------------------------------------------
+    async say(input, _options = {}) {
+        this.assertOpen();
+        this.pending++;
+        try {
+            const pieces = [];
+            if (typeof input === "string")
+                pieces.push(input);
+            else
+                for await (const piece of input)
+                    pieces.push(piece);
+            const text = pieces.join("");
+            this.record({
+                kind: "say",
+                text,
+                streamed: typeof input !== "string",
+                pieces
+            });
+            return { interrupted: false, heardText: text, heardMs: 0 };
+        }
+        finally {
+            this.pending--;
+        }
+    }
+    async stopSay() { }
+    async play(url, _options = {}) {
+        this.assertOpen();
+        this.record({ kind: "play", url });
+        return { interrupted: false, heardMs: 0 };
+    }
+    async playDtmf(digits) {
+        this.assertOpen();
+        this.record({ kind: "dtmf", digits });
+    }
+    async subscribe(_events) { }
+    async listen(_options) { }
+    async note(_data) { }
+    async hangup() {
+        if (this.ended)
+            return;
+        this.record({ kind: "hangup" });
+        this.end("hangup");
+    }
+    async transfer(to) {
+        this.assertOpen();
+        this.record({ kind: "transfer", to });
+        this.end("transferred");
+    }
+    record(output) {
+        this.outputs.push(output);
+        this.events.emit("__output", output);
+    }
+    assertOpen() {
+        if (this.endedReason)
+            throw new voice_1.SessionEndedError(this.endedReason);
+    }
+}
+exports.EvalSession = EvalSession;

package/dist/models/evaluations/session/evaluateSessionScenario.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ScenarioEvaluationReport, StepEvaluationReport } from "@fonoster/types";
+import { SessionAgent } from "../../../session/config";
+import { ChatMessage, Think } from "../../../session/think";
+import { ToolPort } from "../../../session/tools";
+import { EvalScenario } from "../types";
+/** A step report with what the session engine streamed (extra fields, ignored on the wire). */
+export type SessionStepReport = StepEvaluationReport & {
+    streaming: {
+        /** The pieces of the reply as the engine sent them to `say` (sentences). */
+        pieces: string[];
+        firstTokenMs?: number;
+        firstSpeakMs?: number;
+    };
+    /** Tool calls and results as stored in the history (structured tool messages). */
+    toolMessages: ChatMessage[];
+};
+export type SessionScenarioReport = ScenarioEvaluationReport & {
+    steps: SessionStepReport[];
+};
+export type SessionEvalOptions = {
+    agent: SessionAgent;
+    /** The language model the engine streams from (a registry adapter, or the fake LLM). */
+    think: Think;
+    testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
+    /**
+     * Runs the agent's own tools. Defaults to `evalToolPort`: evals never call tool endpoints
+     * (an eval must not book an appointment), every call succeeds with `{ status: "ok" }`.
+     */
+    tools?: ToolPort;
+    /** Upper bound for one step (default 60 s). */
+    stepTimeoutMs?: number;
+};
+/** The default eval tool port: records nothing, calls nothing, every tool succeeds. */
+export declare const evalToolPort: ToolPort;
+/**
+ * Runs one test-case scenario through the session engine (rework-autopilot 1.4): the real state
+ * machine and Agent on an in-memory session, replies streamed sentence by sentence, tool calls
+ * read back from the structured tool messages in the history.
+ */
+export declare function evaluateSessionScenario(options: SessionEvalOptions, scenario: EvalScenario, onStepResult?: (scenarioRef: string, report: StepEvaluationReport) => void | Promise<void>): Promise<SessionScenarioReport>;