@fonoster/autopilot 0.50.0 → 0.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/models/evaluations/createEvaluateIntelligence.d.ts +4 -2
- package/dist/models/evaluations/createEvaluateIntelligence.js +18 -6
- package/dist/models/evaluations/evaluateToolCalls.js +1 -1
- package/dist/models/evaluations/session/evaluateSessionScenario.d.ts +2 -0
- package/dist/models/evaluations/session/evaluateSessionScenario.js +17 -2
- package/dist/models/evaluations/session/runSessionEval.js +1 -1
- package/dist/models/evaluations/stepReportToEventPayload.js +2 -1
- package/dist/models/evaluations/types.d.ts +1 -0
- package/dist/session/llm.js +10 -3
- package/dist/session/meteredThink.js +6 -1
- package/package.json +7 -7
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Usage } from "@fonoster/common";
|
|
1
|
+
import { Registry, Usage } from "@fonoster/common";
|
|
2
2
|
import { EvaluateIntelligenceRequest } from "./types";
|
|
3
3
|
type ServerStreamCall = {
|
|
4
4
|
request: EvaluateIntelligenceRequest;
|
|
@@ -20,9 +20,11 @@ export type EvalKey = {
|
|
|
20
20
|
* is unique per request within the run, for the lines' ids.
|
|
21
21
|
*/
|
|
22
22
|
export type EvalUsageReporter = (workspaceRef: string, lines: Usage.UsageLine[], scope: string) => void;
|
|
23
|
+
/** The provider catalog, read fresh for each eval, so a model runs the way a live call runs it. */
|
|
24
|
+
export type EvalCatalog = () => Promise<Registry.Registry>;
|
|
23
25
|
/**
|
|
24
26
|
* Runs an agent's test cases (rework-autopilot 8.2: the agent is a session agent; its
|
|
25
27
|
* catalog id picks the provider, and keys come from `keys`, never from the agent).
|
|
26
28
|
*/
|
|
27
|
-
declare function createEvaluateIntelligence(keys: EvalKeySource, reportUsage?: EvalUsageReporter): (call: ServerStreamCall) => Promise<void>;
|
|
29
|
+
declare function createEvaluateIntelligence(keys: EvalKeySource, reportUsage?: EvalUsageReporter, catalog?: EvalCatalog): (call: ServerStreamCall) => Promise<void>;
|
|
28
30
|
export { createEvaluateIntelligence };
|
|
@@ -23,6 +23,7 @@ const crypto_1 = require("crypto");
|
|
|
23
23
|
const common_1 = require("@fonoster/common");
|
|
24
24
|
const logger_1 = require("@fonoster/logger");
|
|
25
25
|
const pb_util_1 = require("pb-util");
|
|
26
|
+
const llm_1 = require("../../session/llm");
|
|
26
27
|
const meteredThink_1 = require("../../session/meteredThink");
|
|
27
28
|
const createEvalEffectiveConfig_1 = require("./createEvalEffectiveConfig");
|
|
28
29
|
const runEval_1 = require("./runEval");
|
|
@@ -34,7 +35,7 @@ const keyOf = (k) => typeof k === "string" ? { key: k, source: "platform" } : k;
|
|
|
34
35
|
* Runs an agent's test cases (rework-autopilot 8.2: the agent is a session agent; its
|
|
35
36
|
* catalog id picks the provider, and keys come from `keys`, never from the agent).
|
|
36
37
|
*/
|
|
37
|
-
function createEvaluateIntelligence(keys, reportUsage) {
|
|
38
|
+
function createEvaluateIntelligence(keys, reportUsage, catalog) {
|
|
38
39
|
const evaluateIntelligence = async (call) => {
|
|
39
40
|
const { request } = call;
|
|
40
41
|
const { intelligence } = request;
|
|
@@ -58,7 +59,8 @@ function createEvaluateIntelligence(keys, reportUsage) {
|
|
|
58
59
|
usage: (lines) => reportUsage?.(accessKeyId, lines, `eval|${runId}|${request++}`),
|
|
59
60
|
missing: (catalogId) => logger.warn(`eval: ${catalogId} reported no usage`, { accessKeyId })
|
|
60
61
|
};
|
|
61
|
-
|
|
62
|
+
// With the catalog, the resolver knows which providers are keyless.
|
|
63
|
+
if (!catalog && !apiKey && provider !== "ollama")
|
|
62
64
|
throw new Error(`no ${provider} key for this workspace or platform`);
|
|
63
65
|
if (!evaluationApiKey)
|
|
64
66
|
throw new Error("no openai key for the evaluation model (workspace or platform)");
|
|
@@ -70,12 +72,22 @@ function createEvaluateIntelligence(keys, reportUsage) {
|
|
|
70
72
|
call.write((0, stepReportToEventPayload_1.scenarioSummaryToEventPayload)(scenarioRef, overallPassed));
|
|
71
73
|
}
|
|
72
74
|
};
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
|
|
75
|
+
// With the catalog, the model runs as in a live call: same provider, adapter and features.
|
|
76
|
+
// Without it, the session engine for OpenAI-compatible providers and the LangChain path for
|
|
77
|
+
// the rest.
|
|
78
|
+
const resolved = catalog
|
|
79
|
+
? await new llm_1.RegistryLlmResolver({
|
|
80
|
+
registry: await catalog(),
|
|
81
|
+
credentials: { get: () => apiKey }
|
|
82
|
+
}).resolve(agent.languageModel.model, {
|
|
83
|
+
temperature: agent.languageModel.temperature,
|
|
84
|
+
maxTokens: agent.languageModel.maxTokens
|
|
85
|
+
}, {})
|
|
86
|
+
: undefined;
|
|
87
|
+
const sessionThink = resolved?.think ?? (0, session_1.sessionThinkFor)(agent, apiKey);
|
|
76
88
|
if (sessionThink) {
|
|
77
89
|
const think = (0, meteredThink_1.meteredThink)(sessionThink, {
|
|
78
|
-
catalogId: agent.languageModel.model,
|
|
90
|
+
catalogId: resolved?.id ?? agent.languageModel.model,
|
|
79
91
|
keySource: agentKey?.source ?? "platform"
|
|
80
92
|
}, sink);
|
|
81
93
|
await (0, session_1.runSessionEval)({
|
|
@@ -11,7 +11,7 @@ function evaluateToolCalls(expectedTools, toolCalls) {
|
|
|
11
11
|
passed: false,
|
|
12
12
|
expectedParameters: undefined,
|
|
13
13
|
actualParameters: undefined,
|
|
14
|
-
errorMessage: `Expected ${expectedTools.length} tool invocation(s), but got ${toolCalls ? toolCalls.
|
|
14
|
+
errorMessage: `Expected ${expectedTools.length} tool invocation(s), but got ${toolCalls?.length ?? 0}${toolCalls?.length ? ` (${toolCalls.map((c) => c.name).join(", ")})` : ""}.`
|
|
15
15
|
});
|
|
16
16
|
return {
|
|
17
17
|
evaluations,
|
|
@@ -37,6 +37,8 @@ export type SessionScenarioReport = ScenarioEvaluationReport & {
|
|
|
37
37
|
};
|
|
38
38
|
export type SessionEvalOptions = {
|
|
39
39
|
agent: SessionAgent;
|
|
40
|
+
/** The test cases' canned tool results, used on any step that gives none of its own. */
|
|
41
|
+
toolResults?: Record<string, unknown>;
|
|
40
42
|
/** The language model the engine streams from (a registry adapter, or the fake LLM). */
|
|
41
43
|
think: Think;
|
|
42
44
|
testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
|
|
@@ -44,6 +44,13 @@ async function evaluateSessionScenario(options, scenario, onStepResult) {
|
|
|
44
44
|
const { agent: agentConfig, testTextSimilarity } = options;
|
|
45
45
|
const stepTimeoutMs = options.stepTimeoutMs ?? 60000;
|
|
46
46
|
const session = new EvalSession_1.EvalSession(scenario.telephonyContext);
|
|
47
|
+
// The canned results of the step being evaluated.
|
|
48
|
+
let cannedResults = new Map();
|
|
49
|
+
const tools = options.tools ?? {
|
|
50
|
+
run: async (name, args, signal) => cannedResults.has(name)
|
|
51
|
+
? { ok: true, result: cannedResults.get(name) }
|
|
52
|
+
: exports.evalToolPort.run(name, args, signal)
|
|
53
|
+
};
|
|
47
54
|
let agent;
|
|
48
55
|
let signal;
|
|
49
56
|
let thinkError;
|
|
@@ -66,7 +73,7 @@ async function evaluateSessionScenario(options, scenario, onStepResult) {
|
|
|
66
73
|
const handler = (0, createAutopilot_1.createAutopilotHandler)({
|
|
67
74
|
agent: { ...agentConfig, testCases: undefined },
|
|
68
75
|
think,
|
|
69
|
-
tools
|
|
76
|
+
tools,
|
|
70
77
|
recorder: {
|
|
71
78
|
turn: (t) => {
|
|
72
79
|
traces.push(t);
|
|
@@ -86,6 +93,12 @@ async function evaluateSessionScenario(options, scenario, onStepResult) {
|
|
|
86
93
|
await settle(session); // greeting
|
|
87
94
|
for (const step of scenario.conversation) {
|
|
88
95
|
const type = step.expected.text.type;
|
|
96
|
+
cannedResults = new Map([
|
|
97
|
+
...Object.entries(options.toolResults ?? {}),
|
|
98
|
+
...(step.expected.tools ?? [])
|
|
99
|
+
.filter((t) => t.result !== undefined)
|
|
100
|
+
.map((t) => [t.tool, t.result])
|
|
101
|
+
]);
|
|
89
102
|
const report = {
|
|
90
103
|
humanInput: step.userInput,
|
|
91
104
|
expectedResponse: step.expected.text.response,
|
|
@@ -124,6 +137,7 @@ async function evaluateSessionScenario(options, scenario, onStepResult) {
|
|
|
124
137
|
const toolCalls = history.flatMap((m) => m.role === "assistant" ? (m.tool_calls ?? []) : []);
|
|
125
138
|
const builtIn = toolCalls.find((c) => c.name === "hangup" || c.name === "transfer");
|
|
126
139
|
const trace = traces[tracesFrom];
|
|
140
|
+
report.firstTokenMs = trace?.firstTokenMs;
|
|
127
141
|
report.streaming = {
|
|
128
142
|
pieces: reply.flatMap((s) => s.pieces),
|
|
129
143
|
firstTokenMs: trace?.firstTokenMs,
|
|
@@ -155,7 +169,8 @@ async function evaluateSessionScenario(options, scenario, onStepResult) {
|
|
|
155
169
|
report.passed = false;
|
|
156
170
|
report.errorMessage = text.errorMessage;
|
|
157
171
|
}
|
|
158
|
-
|
|
172
|
+
// An empty list is a check too: the model must not call a tool on this step.
|
|
173
|
+
if (step.expected.tools) {
|
|
159
174
|
const tools = (0, evaluateToolCalls_1.evaluateToolCalls)(step.expected.tools, toolCalls.map((c) => ({ name: c.name, args: parseArgs(c) })));
|
|
160
175
|
report.toolEvaluations = tools.evaluations;
|
|
161
176
|
if (!tools.passed) {
|
|
@@ -51,7 +51,7 @@ async function runSessionEval(options, callbacks) {
|
|
|
51
51
|
}, testCases.evalsSystemPrompt || textSimilaryPrompt_1.textSimilaryPrompt, options.judgeUsage);
|
|
52
52
|
const reports = [];
|
|
53
53
|
for (const scenario of testCases.scenarios) {
|
|
54
|
-
const report = await (0, evaluateSessionScenario_1.evaluateSessionScenario)({ ...options, testTextSimilarity }, scenario, callbacks?.onStepResult);
|
|
54
|
+
const report = await (0, evaluateSessionScenario_1.evaluateSessionScenario)({ ...options, testTextSimilarity, toolResults: testCases.toolResults }, scenario, callbacks?.onStepResult);
|
|
55
55
|
reports.push(report);
|
|
56
56
|
await callbacks?.onScenarioComplete?.(report.scenarioRef, report.overallPassed);
|
|
57
57
|
}
|
|
@@ -51,7 +51,8 @@ function stepReportToEventPayload(scenarioRef, report) {
|
|
|
51
51
|
evaluationType: EVALUATION_TYPE_TO_NUM[report.evaluationType] ?? 0,
|
|
52
52
|
passed: report.passed,
|
|
53
53
|
errorMessage: report.errorMessage ?? "",
|
|
54
|
-
toolEvaluations: (report.toolEvaluations ?? []).map(toolReportToPayload)
|
|
54
|
+
toolEvaluations: (report.toolEvaluations ?? []).map(toolReportToPayload),
|
|
55
|
+
firstTokenMs: report.firstTokenMs ?? 0
|
|
55
56
|
}
|
|
56
57
|
}
|
|
57
58
|
};
|
package/dist/session/llm.js
CHANGED
|
@@ -113,9 +113,16 @@ class RegistryLlmResolver {
|
|
|
113
113
|
keySource: credential?.source ?? "platform",
|
|
114
114
|
think: llm.create(
|
|
115
115
|
// The catalog provider (e.g. together) names a rejected key, not the adapter (openai).
|
|
116
|
-
{ baseUrl, apiKey, name: provider.id }, id.slice(providerId.length + 1),
|
|
117
|
-
|
|
118
|
-
|
|
116
|
+
{ baseUrl, apiKey, name: provider.id }, id.slice(providerId.length + 1), {
|
|
117
|
+
maxTokens: settings.maxTokens,
|
|
118
|
+
...(model.features.includes("fixed-sampling")
|
|
119
|
+
? {}
|
|
120
|
+
: { temperature: settings.temperature }),
|
|
121
|
+
...(model.features.includes("reasoning") ? { reasoning: true } : {}),
|
|
122
|
+
...(model.features.includes("reasoning-low")
|
|
123
|
+
? { reasoningLow: true }
|
|
124
|
+
: {})
|
|
125
|
+
}),
|
|
119
126
|
...(resolution.event
|
|
120
127
|
? {
|
|
121
128
|
event: {
|
|
@@ -33,7 +33,12 @@ function usageLines(model, usage) {
|
|
|
33
33
|
keySource: model.keySource
|
|
34
34
|
};
|
|
35
35
|
const lines = [
|
|
36
|
-
|
|
36
|
+
// Cached tokens have their own meter and price, so each token is billed once.
|
|
37
|
+
{
|
|
38
|
+
meter: "llm.tokens.in",
|
|
39
|
+
quantity: Math.max(0, usage.inputTokens - usage.cachedTokens),
|
|
40
|
+
...key
|
|
41
|
+
},
|
|
37
42
|
{ meter: "llm.tokens.out", quantity: usage.outputTokens, ...key }
|
|
38
43
|
];
|
|
39
44
|
if (usage.cachedTokens > 0)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fonoster/autopilot",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.51.0",
|
|
4
4
|
"description": "Voice AI for the Fonoster platform",
|
|
5
5
|
"author": "Pedro Sanders <psanders@fonoster.com>",
|
|
6
6
|
"homepage": "https://github.com/fonoster/fonoster#readme",
|
|
@@ -32,11 +32,11 @@
|
|
|
32
32
|
},
|
|
33
33
|
"dependencies": {
|
|
34
34
|
"@aws-sdk/client-s3": "^3.958.0",
|
|
35
|
-
"@fonoster/common": "^0.
|
|
36
|
-
"@fonoster/community": "^0.
|
|
37
|
-
"@fonoster/logger": "^0.
|
|
38
|
-
"@fonoster/types": "^0.
|
|
39
|
-
"@fonoster/voice": "^0.
|
|
35
|
+
"@fonoster/common": "^0.51.0",
|
|
36
|
+
"@fonoster/community": "^0.51.0",
|
|
37
|
+
"@fonoster/logger": "^0.51.0",
|
|
38
|
+
"@fonoster/types": "^0.51.0",
|
|
39
|
+
"@fonoster/voice": "^0.51.0",
|
|
40
40
|
"@grpc/grpc-js": "~1.14.6",
|
|
41
41
|
"@langchain/anthropic": "^1.3.3",
|
|
42
42
|
"@langchain/community": "^1.1.1",
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"xstate": "^5.17.3",
|
|
55
55
|
"zod": "^4.6.7"
|
|
56
56
|
},
|
|
57
|
-
"gitHead": "
|
|
57
|
+
"gitHead": "51c3bd44b843ca1462d170fc279b2ce391aa8366",
|
|
58
58
|
"devDependencies": {
|
|
59
59
|
"yaml": "^2.9.0"
|
|
60
60
|
}
|