@fonoster/autopilot 0.50.0 → 0.51.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { Usage } from "@fonoster/common";
1
+ import { Registry, Usage } from "@fonoster/common";
2
2
  import { EvaluateIntelligenceRequest } from "./types";
3
3
  type ServerStreamCall = {
4
4
  request: EvaluateIntelligenceRequest;
@@ -20,9 +20,11 @@ export type EvalKey = {
20
20
  * is unique per request within the run, for the lines' ids.
21
21
  */
22
22
  export type EvalUsageReporter = (workspaceRef: string, lines: Usage.UsageLine[], scope: string) => void;
23
+ /** The provider catalog, read fresh for each eval, so a model runs the way a live call runs it. */
24
+ export type EvalCatalog = () => Promise<Registry.Registry>;
23
25
  /**
24
26
  * Runs an agent's test cases (rework-autopilot 8.2: the agent is a session agent; its
25
27
  * catalog id picks the provider, and keys come from `keys`, never from the agent).
26
28
  */
27
- declare function createEvaluateIntelligence(keys: EvalKeySource, reportUsage?: EvalUsageReporter): (call: ServerStreamCall) => Promise<void>;
29
+ declare function createEvaluateIntelligence(keys: EvalKeySource, reportUsage?: EvalUsageReporter, catalog?: EvalCatalog): (call: ServerStreamCall) => Promise<void>;
28
30
  export { createEvaluateIntelligence };
@@ -23,6 +23,7 @@ const crypto_1 = require("crypto");
23
23
  const common_1 = require("@fonoster/common");
24
24
  const logger_1 = require("@fonoster/logger");
25
25
  const pb_util_1 = require("pb-util");
26
+ const llm_1 = require("../../session/llm");
26
27
  const meteredThink_1 = require("../../session/meteredThink");
27
28
  const createEvalEffectiveConfig_1 = require("./createEvalEffectiveConfig");
28
29
  const runEval_1 = require("./runEval");
@@ -34,7 +35,7 @@ const keyOf = (k) => typeof k === "string" ? { key: k, source: "platform" } : k;
34
35
  * Runs an agent's test cases (rework-autopilot 8.2: the agent is a session agent; its
35
36
  * catalog id picks the provider, and keys come from `keys`, never from the agent).
36
37
  */
37
- function createEvaluateIntelligence(keys, reportUsage) {
38
+ function createEvaluateIntelligence(keys, reportUsage, catalog) {
38
39
  const evaluateIntelligence = async (call) => {
39
40
  const { request } = call;
40
41
  const { intelligence } = request;
@@ -58,7 +59,8 @@ function createEvaluateIntelligence(keys, reportUsage) {
58
59
  usage: (lines) => reportUsage?.(accessKeyId, lines, `eval|${runId}|${request++}`),
59
60
  missing: (catalogId) => logger.warn(`eval: ${catalogId} reported no usage`, { accessKeyId })
60
61
  };
61
- if (!apiKey && provider !== "ollama")
62
+ // With the catalog, the resolver knows which providers are keyless.
63
+ if (!catalog && !apiKey && provider !== "ollama")
62
64
  throw new Error(`no ${provider} key for this workspace or platform`);
63
65
  if (!evaluationApiKey)
64
66
  throw new Error("no openai key for the evaluation model (workspace or platform)");
@@ -70,12 +72,22 @@ function createEvaluateIntelligence(keys, reportUsage) {
70
72
  call.write((0, stepReportToEventPayload_1.scenarioSummaryToEventPayload)(scenarioRef, overallPassed));
71
73
  }
72
74
  };
73
- // The session engine (streaming replies, structured tool messages) for OpenAI-compatible
74
- // providers; the LangChain path stays for the rest until their adapters land (EV-B1).
75
- const sessionThink = (0, session_1.sessionThinkFor)(agent, apiKey);
75
+ // With the catalog, the model runs as in a live call: same provider, adapter and features.
76
+ // Without it, the session engine for OpenAI-compatible providers and the LangChain path for
77
+ // the rest.
78
+ const resolved = catalog
79
+ ? await new llm_1.RegistryLlmResolver({
80
+ registry: await catalog(),
81
+ credentials: { get: () => apiKey }
82
+ }).resolve(agent.languageModel.model, {
83
+ temperature: agent.languageModel.temperature,
84
+ maxTokens: agent.languageModel.maxTokens
85
+ }, {})
86
+ : undefined;
87
+ const sessionThink = resolved?.think ?? (0, session_1.sessionThinkFor)(agent, apiKey);
76
88
  if (sessionThink) {
77
89
  const think = (0, meteredThink_1.meteredThink)(sessionThink, {
78
- catalogId: agent.languageModel.model,
90
+ catalogId: resolved?.id ?? agent.languageModel.model,
79
91
  keySource: agentKey?.source ?? "platform"
80
92
  }, sink);
81
93
  await (0, session_1.runSessionEval)({
@@ -11,7 +11,7 @@ function evaluateToolCalls(expectedTools, toolCalls) {
11
11
  passed: false,
12
12
  expectedParameters: undefined,
13
13
  actualParameters: undefined,
14
- errorMessage: `Expected ${expectedTools.length} tool invocation(s), but got ${toolCalls ? toolCalls.length : 0}.`
14
+ errorMessage: `Expected ${expectedTools.length} tool invocation(s), but got ${toolCalls?.length ?? 0}${toolCalls?.length ? ` (${toolCalls.map((c) => c.name).join(", ")})` : ""}.`
15
15
  });
16
16
  return {
17
17
  evaluations,
@@ -37,6 +37,8 @@ export type SessionScenarioReport = ScenarioEvaluationReport & {
37
37
  };
38
38
  export type SessionEvalOptions = {
39
39
  agent: SessionAgent;
40
+ /** The test cases' canned tool results, used on any step that gives none of its own. */
41
+ toolResults?: Record<string, unknown>;
40
42
  /** The language model the engine streams from (a registry adapter, or the fake LLM). */
41
43
  think: Think;
42
44
  testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
@@ -44,6 +44,13 @@ async function evaluateSessionScenario(options, scenario, onStepResult) {
44
44
  const { agent: agentConfig, testTextSimilarity } = options;
45
45
  const stepTimeoutMs = options.stepTimeoutMs ?? 60000;
46
46
  const session = new EvalSession_1.EvalSession(scenario.telephonyContext);
47
+ // The canned results of the step being evaluated.
48
+ let cannedResults = new Map();
49
+ const tools = options.tools ?? {
50
+ run: async (name, args, signal) => cannedResults.has(name)
51
+ ? { ok: true, result: cannedResults.get(name) }
52
+ : exports.evalToolPort.run(name, args, signal)
53
+ };
47
54
  let agent;
48
55
  let signal;
49
56
  let thinkError;
@@ -66,7 +73,7 @@ async function evaluateSessionScenario(options, scenario, onStepResult) {
66
73
  const handler = (0, createAutopilot_1.createAutopilotHandler)({
67
74
  agent: { ...agentConfig, testCases: undefined },
68
75
  think,
69
- tools: options.tools ?? exports.evalToolPort,
76
+ tools,
70
77
  recorder: {
71
78
  turn: (t) => {
72
79
  traces.push(t);
@@ -86,6 +93,12 @@ async function evaluateSessionScenario(options, scenario, onStepResult) {
86
93
  await settle(session); // greeting
87
94
  for (const step of scenario.conversation) {
88
95
  const type = step.expected.text.type;
96
+ cannedResults = new Map([
97
+ ...Object.entries(options.toolResults ?? {}),
98
+ ...(step.expected.tools ?? [])
99
+ .filter((t) => t.result !== undefined)
100
+ .map((t) => [t.tool, t.result])
101
+ ]);
89
102
  const report = {
90
103
  humanInput: step.userInput,
91
104
  expectedResponse: step.expected.text.response,
@@ -124,6 +137,7 @@ async function evaluateSessionScenario(options, scenario, onStepResult) {
124
137
  const toolCalls = history.flatMap((m) => m.role === "assistant" ? (m.tool_calls ?? []) : []);
125
138
  const builtIn = toolCalls.find((c) => c.name === "hangup" || c.name === "transfer");
126
139
  const trace = traces[tracesFrom];
140
+ report.firstTokenMs = trace?.firstTokenMs;
127
141
  report.streaming = {
128
142
  pieces: reply.flatMap((s) => s.pieces),
129
143
  firstTokenMs: trace?.firstTokenMs,
@@ -155,7 +169,8 @@ async function evaluateSessionScenario(options, scenario, onStepResult) {
155
169
  report.passed = false;
156
170
  report.errorMessage = text.errorMessage;
157
171
  }
158
- if (step.expected.tools && step.expected.tools.length > 0) {
172
+ // An empty list is a check too: the model must not call a tool on this step.
173
+ if (step.expected.tools) {
159
174
  const tools = (0, evaluateToolCalls_1.evaluateToolCalls)(step.expected.tools, toolCalls.map((c) => ({ name: c.name, args: parseArgs(c) })));
160
175
  report.toolEvaluations = tools.evaluations;
161
176
  if (!tools.passed) {
@@ -51,7 +51,7 @@ async function runSessionEval(options, callbacks) {
51
51
  }, testCases.evalsSystemPrompt || textSimilaryPrompt_1.textSimilaryPrompt, options.judgeUsage);
52
52
  const reports = [];
53
53
  for (const scenario of testCases.scenarios) {
54
- const report = await (0, evaluateSessionScenario_1.evaluateSessionScenario)({ ...options, testTextSimilarity }, scenario, callbacks?.onStepResult);
54
+ const report = await (0, evaluateSessionScenario_1.evaluateSessionScenario)({ ...options, testTextSimilarity, toolResults: testCases.toolResults }, scenario, callbacks?.onStepResult);
55
55
  reports.push(report);
56
56
  await callbacks?.onScenarioComplete?.(report.scenarioRef, report.overallPassed);
57
57
  }
@@ -51,7 +51,8 @@ function stepReportToEventPayload(scenarioRef, report) {
51
51
  evaluationType: EVALUATION_TYPE_TO_NUM[report.evaluationType] ?? 0,
52
52
  passed: report.passed,
53
53
  errorMessage: report.errorMessage ?? "",
54
- toolEvaluations: (report.toolEvaluations ?? []).map(toolReportToPayload)
54
+ toolEvaluations: (report.toolEvaluations ?? []).map(toolReportToPayload),
55
+ firstTokenMs: report.firstTokenMs ?? 0
55
56
  }
56
57
  }
57
58
  };
@@ -21,6 +21,7 @@ import { EvalAgent, LanguageModel } from "../types";
21
21
  type EvalExpectedTool = {
22
22
  tool: string;
23
23
  parameters?: Record<string, unknown>;
24
+ result?: unknown;
24
25
  };
25
26
  type EvalConversationStep = {
26
27
  userInput: string;
@@ -113,9 +113,16 @@ class RegistryLlmResolver {
113
113
  keySource: credential?.source ?? "platform",
114
114
  think: llm.create(
115
115
  // The catalog provider (e.g. together) names a rejected key, not the adapter (openai).
116
- { baseUrl, apiKey, name: provider.id }, id.slice(providerId.length + 1), model.features.includes("reasoning")
117
- ? { ...settings, reasoning: true }
118
- : settings),
116
+ { baseUrl, apiKey, name: provider.id }, id.slice(providerId.length + 1), {
117
+ maxTokens: settings.maxTokens,
118
+ ...(model.features.includes("fixed-sampling")
119
+ ? {}
120
+ : { temperature: settings.temperature }),
121
+ ...(model.features.includes("reasoning") ? { reasoning: true } : {}),
122
+ ...(model.features.includes("reasoning-low")
123
+ ? { reasoningLow: true }
124
+ : {})
125
+ }),
119
126
  ...(resolution.event
120
127
  ? {
121
128
  event: {
@@ -33,7 +33,12 @@ function usageLines(model, usage) {
33
33
  keySource: model.keySource
34
34
  };
35
35
  const lines = [
36
- { meter: "llm.tokens.in", quantity: usage.inputTokens, ...key },
36
+ // Cached tokens have their own meter and price, so each token is billed once.
37
+ {
38
+ meter: "llm.tokens.in",
39
+ quantity: Math.max(0, usage.inputTokens - usage.cachedTokens),
40
+ ...key
41
+ },
37
42
  { meter: "llm.tokens.out", quantity: usage.outputTokens, ...key }
38
43
  ];
39
44
  if (usage.cachedTokens > 0)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fonoster/autopilot",
3
- "version": "0.50.0",
3
+ "version": "0.51.0",
4
4
  "description": "Voice AI for the Fonoster platform",
5
5
  "author": "Pedro Sanders <psanders@fonoster.com>",
6
6
  "homepage": "https://github.com/fonoster/fonoster#readme",
@@ -32,11 +32,11 @@
32
32
  },
33
33
  "dependencies": {
34
34
  "@aws-sdk/client-s3": "^3.958.0",
35
- "@fonoster/common": "^0.50.0",
36
- "@fonoster/community": "^0.50.0",
37
- "@fonoster/logger": "^0.50.0",
38
- "@fonoster/types": "^0.50.0",
39
- "@fonoster/voice": "^0.50.0",
35
+ "@fonoster/common": "^0.51.0",
36
+ "@fonoster/community": "^0.51.0",
37
+ "@fonoster/logger": "^0.51.0",
38
+ "@fonoster/types": "^0.51.0",
39
+ "@fonoster/voice": "^0.51.0",
40
40
  "@grpc/grpc-js": "~1.14.6",
41
41
  "@langchain/anthropic": "^1.3.3",
42
42
  "@langchain/community": "^1.1.1",
@@ -54,7 +54,7 @@
54
54
  "xstate": "^5.17.3",
55
55
  "zod": "^4.6.7"
56
56
  },
57
- "gitHead": "75d5debf2e3a113cc2275312a17cadaf7986a73a",
57
+ "gitHead": "51c3bd44b843ca1462d170fc279b2ce391aa8366",
58
58
  "devDependencies": {
59
59
  "yaml": "^2.9.0"
60
60
  }