@fonoster/autopilot 0.30.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/knowledge/MeteredEmbeddings.d.ts +47 -0
- package/dist/knowledge/MeteredEmbeddings.js +58 -0
- package/dist/models/createLanguageModel.d.ts +2 -2
- package/dist/models/createLanguageModel.js +2 -2
- package/dist/models/evaluations/createEvalEffectiveConfig.d.ts +6 -6
- package/dist/models/evaluations/createEvalEffectiveConfig.js +7 -7
- package/dist/models/evaluations/createEvaluateIntelligence.d.ts +15 -4
- package/dist/models/evaluations/createEvaluateIntelligence.js +33 -12
- package/dist/models/evaluations/createTestTextSimilarity.d.ts +7 -1
- package/dist/models/evaluations/createTestTextSimilarity.js +19 -1
- package/dist/models/evaluations/evalTestCases.d.ts +2 -2
- package/dist/models/evaluations/evalTestCases.js +7 -7
- package/dist/models/evaluations/evaluateScenario.js +2 -2
- package/dist/models/evaluations/evaluateStep.d.ts +1 -1
- package/dist/models/evaluations/evaluateStep.js +2 -2
- package/dist/models/evaluations/extractAiResponse.d.ts +2 -2
- package/dist/models/evaluations/extractAiResponse.js +3 -3
- package/dist/models/evaluations/runEval.d.ts +2 -2
- package/dist/models/evaluations/runEval.js +7 -7
- package/dist/models/evaluations/session/EvalSession.d.ts +1 -1
- package/dist/models/evaluations/session/EvalSession.js +1 -1
- package/dist/models/evaluations/session/evaluateSessionScenario.d.ts +3 -3
- package/dist/models/evaluations/session/evaluateSessionScenario.js +4 -4
- package/dist/models/evaluations/session/runSessionEval.d.ts +7 -4
- package/dist/models/evaluations/session/runSessionEval.js +7 -7
- package/dist/models/evaluations/textSimilaryPrompt.d.ts +1 -1
- package/dist/models/evaluations/textSimilaryPrompt.js +1 -1
- package/dist/models/evaluations/types.d.ts +5 -5
- package/dist/models/types.d.ts +3 -3
- package/dist/session/Agent.d.ts +38 -3
- package/dist/session/Agent.js +90 -16
- package/dist/session/config.d.ts +2 -2
- package/dist/session/config.js +4 -4
- package/dist/session/createAutopilot.d.ts +12 -11
- package/dist/session/createAutopilot.js +48 -16
- package/dist/session/hosted.d.ts +23 -25
- package/dist/session/hosted.js +43 -48
- package/dist/session/index.d.ts +1 -1
- package/dist/session/index.js +1 -1
- package/dist/session/knowledge.d.ts +1 -1
- package/dist/session/llm.d.ts +28 -1
- package/dist/session/llm.js +50 -6
- package/dist/session/main.js +2 -1
- package/dist/session/meteredThink.d.ts +45 -0
- package/dist/session/meteredThink.js +125 -0
- package/dist/session/telemetry.d.ts +34 -0
- package/dist/session/telemetry.js +55 -0
- package/dist/session/tools.d.ts +1 -1
- package/dist/session/tools.js +1 -1
- package/package.json +9 -9
- package/dist/session/convertLegacyAssistant.d.ts +0 -77
- package/dist/session/convertLegacyAssistant.js +0 -118
|
@@ -1,17 +1,20 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { SessionAgent } from "../../../session/config";
|
|
2
2
|
import { Think } from "../../../session/think";
|
|
3
|
+
import { createTestTextSimilarity } from "../createTestTextSimilarity";
|
|
3
4
|
import { RunEvalCallbacks } from "../types";
|
|
4
5
|
import { SessionEvalOptions, SessionScenarioReport } from "./evaluateSessionScenario";
|
|
5
6
|
export type RunSessionEvalOptions = Omit<SessionEvalOptions, "testTextSimilarity"> & {
|
|
6
|
-
/** Defaults to the OpenAI similarity judge from the
|
|
7
|
+
/** Defaults to the OpenAI similarity judge from the agent's testCases. */
|
|
7
8
|
testTextSimilarity?: SessionEvalOptions["testTextSimilarity"];
|
|
8
9
|
/** The judge's key (the testCases model is always OpenAI). */
|
|
9
10
|
evaluationApiKey?: string;
|
|
11
|
+
/** Meters the judge's tokens (add-usage-metering). */
|
|
12
|
+
judgeUsage?: Parameters<typeof createTestTextSimilarity>[2];
|
|
10
13
|
};
|
|
11
14
|
/**
|
|
12
|
-
* The session-engine Think for an
|
|
15
|
+
* The session-engine Think for an agent whose catalog provider speaks OpenAI-compatible Chat
|
|
13
16
|
* Completions with a known endpoint (openai, groq, ollama); undefined otherwise.
|
|
14
17
|
*/
|
|
15
|
-
export declare function sessionThinkFor(
|
|
18
|
+
export declare function sessionThinkFor(agent: SessionAgent, apiKey: string | undefined): Think | undefined;
|
|
16
19
|
/** Runs every test-case scenario through the session engine, streaming results. */
|
|
17
20
|
export declare function runSessionEval(options: RunSessionEvalOptions, callbacks?: Partial<RunEvalCallbacks>): Promise<SessionScenarioReport[]>;
|
|
@@ -26,29 +26,29 @@ const createTestTextSimilarity_1 = require("../createTestTextSimilarity");
|
|
|
26
26
|
const textSimilaryPrompt_1 = require("../textSimilaryPrompt");
|
|
27
27
|
const evaluateSessionScenario_1 = require("./evaluateSessionScenario");
|
|
28
28
|
/**
|
|
29
|
-
* The session-engine Think for an
|
|
29
|
+
* The session-engine Think for an agent whose catalog provider speaks OpenAI-compatible Chat
|
|
30
30
|
* Completions with a known endpoint (openai, groq, ollama); undefined otherwise.
|
|
31
31
|
*/
|
|
32
|
-
function sessionThinkFor(
|
|
33
|
-
const id =
|
|
32
|
+
function sessionThinkFor(agent, apiKey) {
|
|
33
|
+
const id = agent.languageModel.model;
|
|
34
34
|
const slash = id.indexOf("/");
|
|
35
35
|
const baseUrl = openai_1.OPENAI_COMPATIBLE[id.slice(0, slash)];
|
|
36
36
|
if (!baseUrl)
|
|
37
37
|
return undefined;
|
|
38
38
|
return new openai_1.OpenAiCompatibleThink({ baseUrl, apiKey }, id.slice(slash + 1), {
|
|
39
|
-
temperature:
|
|
40
|
-
maxTokens:
|
|
39
|
+
temperature: agent.languageModel.temperature,
|
|
40
|
+
maxTokens: agent.languageModel.maxTokens
|
|
41
41
|
});
|
|
42
42
|
}
|
|
43
43
|
/** Runs every test-case scenario through the session engine, streaming results. */
|
|
44
44
|
async function runSessionEval(options, callbacks) {
|
|
45
|
-
const testCases = common_1.testCasesSchema.parse(options.
|
|
45
|
+
const testCases = common_1.testCasesSchema.parse(options.agent.testCases);
|
|
46
46
|
const testTextSimilarity = options.testTextSimilarity ??
|
|
47
47
|
(0, createTestTextSimilarity_1.createTestTextSimilarity)({
|
|
48
48
|
provider: testCases.evalsLanguageModel.provider,
|
|
49
49
|
model: testCases.evalsLanguageModel.model,
|
|
50
50
|
apiKey: options.evaluationApiKey ?? testCases.evalsLanguageModel.apiKey
|
|
51
|
-
}, testCases.evalsSystemPrompt || textSimilaryPrompt_1.textSimilaryPrompt);
|
|
51
|
+
}, testCases.evalsSystemPrompt || textSimilaryPrompt_1.textSimilaryPrompt, options.judgeUsage);
|
|
52
52
|
const reports = [];
|
|
53
53
|
for (const scenario of testCases.scenarios) {
|
|
54
54
|
const report = await (0, evaluateSessionScenario_1.evaluateSessionScenario)({ ...options, testTextSimilarity }, scenario, callbacks?.onStepResult);
|
|
@@ -16,4 +16,4 @@
|
|
|
16
16
|
* See the License for the specific language governing permissions and
|
|
17
17
|
* limitations under the License.
|
|
18
18
|
*/
|
|
19
|
-
export declare const textSimilaryPrompt = "\nYou are a text similarity evaluator for a Voice
|
|
19
|
+
export declare const textSimilaryPrompt = "\nYou are a text similarity evaluator for a Voice Agent application. \n\nGive Text1 and Text2, you use the following process to evaluate the similarity between the two texts:\n\n- Take the first text and determmine the intent of the text.\n- Take the second text and determine the intent of the text.\n- Compare the intents of the two texts ignoring the actual text content, the entities, and length of the text.\n\n## Example 1\n\nText1: \"You're welcome. Have a great day!\"\nText2: \"You're welcome [name]. Your appointment is confirmed. Goodbye!\"\n\nAnswer: true\n\n=== \n\nAre the intents of the two texts the same? Respond with true.\n";
|
|
@@ -20,7 +20,7 @@ exports.textSimilaryPrompt = void 0;
|
|
|
20
20
|
* limitations under the License.
|
|
21
21
|
*/
|
|
22
22
|
exports.textSimilaryPrompt = `
|
|
23
|
-
You are a text similarity evaluator for a Voice
|
|
23
|
+
You are a text similarity evaluator for a Voice Agent application.
|
|
24
24
|
|
|
25
25
|
Give Text1 and Text2, you use the following process to evaluate the similarity between the two texts:
|
|
26
26
|
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* limitations under the License.
|
|
18
18
|
*/
|
|
19
19
|
import { StepEvaluationReport } from "@fonoster/types";
|
|
20
|
-
import {
|
|
20
|
+
import { EvalAgent, LanguageModel } from "../types";
|
|
21
21
|
type EvalExpectedTool = {
|
|
22
22
|
tool: string;
|
|
23
23
|
parameters?: Record<string, unknown>;
|
|
@@ -45,9 +45,9 @@ type EvalScenario = {
|
|
|
45
45
|
};
|
|
46
46
|
type EvaluateIntelligenceRequest = {
|
|
47
47
|
intelligence: {
|
|
48
|
-
/** Ignored: the
|
|
48
|
+
/** Ignored: the agent's catalog id names the provider. */
|
|
49
49
|
productRef?: string;
|
|
50
|
-
/** A session
|
|
50
|
+
/** A session agent (with testCases), as a protobuf Struct. */
|
|
51
51
|
config: unknown;
|
|
52
52
|
};
|
|
53
53
|
};
|
|
@@ -55,10 +55,10 @@ type EvaluateStepParams = {
|
|
|
55
55
|
step: EvalConversationStep;
|
|
56
56
|
languageModel: LanguageModel;
|
|
57
57
|
testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
|
|
58
|
-
|
|
58
|
+
agentConfig: EvalAgent;
|
|
59
59
|
};
|
|
60
60
|
type ScenarioEvaluationRequest = {
|
|
61
|
-
|
|
61
|
+
agentConfig: EvalAgent;
|
|
62
62
|
scenario: EvalScenario;
|
|
63
63
|
languageModel: LanguageModel;
|
|
64
64
|
testTextSimilarity: (text1: string, text2: string) => Promise<boolean>;
|
package/dist/models/types.d.ts
CHANGED
|
@@ -26,10 +26,10 @@ import { Tool } from "../tools/types";
|
|
|
26
26
|
/** The language model vendors the LangChain models (used by the evaluations only) implement. */
|
|
27
27
|
type LlmProvider = "openai" | "groq" | "ollama" | "google" | "anthropic";
|
|
28
28
|
/**
|
|
29
|
-
* A session
|
|
29
|
+
* A session agent as the evaluations run it (rework-autopilot 8.2): the catalog id split into
|
|
30
30
|
* provider and model, and the keys the control plane resolved for this run. Never stored.
|
|
31
31
|
*/
|
|
32
|
-
type
|
|
32
|
+
type EvalAgent = {
|
|
33
33
|
conversationSettings: SessionConversationSettings;
|
|
34
34
|
languageModel: Omit<SessionLanguageModel, "model" | "provider" | "apiKey" | "baseUrl"> & {
|
|
35
35
|
provider: LlmProvider;
|
|
@@ -67,4 +67,4 @@ type TelephonyContext = {
|
|
|
67
67
|
callerNumber: string;
|
|
68
68
|
metadata?: Record<string, string>;
|
|
69
69
|
};
|
|
70
|
-
export { BaseModelParams,
|
|
70
|
+
export { BaseModelParams, EvalAgent, LlmProvider, InvocationResult, LanguageModel, LanguageModelParams, TelephonyContext };
|
package/dist/session/Agent.d.ts
CHANGED
|
@@ -1,8 +1,29 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
3
|
+
* http://github.com/fonoster/fonoster
|
|
4
|
+
*
|
|
5
|
+
* This file is part of Fonoster
|
|
6
|
+
*
|
|
7
|
+
* Licensed under the MIT License (the "License");
|
|
8
|
+
* you may not use this file except in compliance with
|
|
9
|
+
* the License. You may obtain a copy of the License at
|
|
10
|
+
*
|
|
11
|
+
* https://opensource.org/licenses/MIT
|
|
12
|
+
*
|
|
13
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
* See the License for the specific language governing permissions and
|
|
17
|
+
* limitations under the License.
|
|
18
|
+
*/
|
|
19
|
+
import { Usage } from "@fonoster/common";
|
|
20
|
+
import { SessionAgent } from "./config";
|
|
2
21
|
import { KnowledgePort } from "./knowledge";
|
|
3
22
|
import { ChatMessage, Think } from "./think";
|
|
4
23
|
import { ToolPort } from "./tools";
|
|
5
24
|
import type { Session } from "@fonoster/voice";
|
|
25
|
+
/** What Autopilot says when a transfer fails and the agent sets no `failedMessage`. */
|
|
26
|
+
export declare const DEFAULT_TRANSFER_FAILED_MESSAGE = "Sorry, I couldn't connect you. How else can I help?";
|
|
6
27
|
export type SpokenSource = "greeting" | "reply" | "idle" | "error";
|
|
7
28
|
export type AgentEvent = {
|
|
8
29
|
type: "TURN_USER_STARTED";
|
|
@@ -51,6 +72,8 @@ export type TurnTrace = {
|
|
|
51
72
|
export interface Recorder {
|
|
52
73
|
turn(trace: TurnTrace): void;
|
|
53
74
|
note(event: Record<string, unknown>): void;
|
|
75
|
+
/** Model token usage for the call (usage.report); absent where there is no call. */
|
|
76
|
+
usage?(lines: Usage.UsageLine[]): void;
|
|
54
77
|
}
|
|
55
78
|
/** Writes turn traces and notes to the call record through the session (record.note). */
|
|
56
79
|
export declare function sessionRecorder(session: Session): Recorder;
|
|
@@ -58,7 +81,7 @@ export declare const logRecorder: Recorder;
|
|
|
58
81
|
/** The side effects of Autopilot: everything the state machine asks it to do on the session. */
|
|
59
82
|
export declare class Agent {
|
|
60
83
|
private readonly session;
|
|
61
|
-
private readonly
|
|
84
|
+
private readonly agent;
|
|
62
85
|
private readonly think;
|
|
63
86
|
private readonly tools;
|
|
64
87
|
private readonly recorder;
|
|
@@ -68,6 +91,8 @@ export declare class Agent {
|
|
|
68
91
|
private inflight?;
|
|
69
92
|
private prepared?;
|
|
70
93
|
private idleTimer?;
|
|
94
|
+
/** The model's `transfer` tool call, answered in the history if the transfer fails. */
|
|
95
|
+
private transferCallId?;
|
|
71
96
|
private maxTimer?;
|
|
72
97
|
private idleCount;
|
|
73
98
|
private resumes;
|
|
@@ -75,7 +100,7 @@ export declare class Agent {
|
|
|
75
100
|
private prepareEarly;
|
|
76
101
|
private stopped;
|
|
77
102
|
send: (event: AgentEvent) => void;
|
|
78
|
-
constructor(session: Session,
|
|
103
|
+
constructor(session: Session, agent: SessionAgent, think: Think, tools: ToolPort, recorder?: Recorder, knowledge?: KnowledgePort);
|
|
79
104
|
get settings(): {
|
|
80
105
|
firstMessage?: string;
|
|
81
106
|
systemPrompt?: string;
|
|
@@ -86,6 +111,7 @@ export declare class Agent {
|
|
|
86
111
|
transferOptions?: {
|
|
87
112
|
message?: string;
|
|
88
113
|
phoneNumber?: string;
|
|
114
|
+
failedMessage?: string;
|
|
89
115
|
};
|
|
90
116
|
idleOptions?: {
|
|
91
117
|
message?: string;
|
|
@@ -119,16 +145,25 @@ export declare class Agent {
|
|
|
119
145
|
abortReply(): void;
|
|
120
146
|
/** turn.ended: answer the caller, streaming sentences into one speak id. */
|
|
121
147
|
answer(text: string): Promise<void>;
|
|
148
|
+
/** One turn (STT final, LLM, TTS) as an autopilot.turn span. */
|
|
122
149
|
private respond;
|
|
150
|
+
private respondTurn;
|
|
123
151
|
private speakRound;
|
|
124
152
|
private runTools;
|
|
125
153
|
goodbye(): Promise<void>;
|
|
126
154
|
transfer(): Promise<void>;
|
|
155
|
+
/**
|
|
156
|
+
* The transfer failed: say so and keep helping (autopilot "Transfer fails"). The model learns
|
|
157
|
+
* the outcome as its transfer call's result, and the machine returns to listening.
|
|
158
|
+
*/
|
|
159
|
+
private transferFailed;
|
|
127
160
|
private messages;
|
|
128
161
|
/** Knowledge for this turn, as a system message for the request only (never in history). */
|
|
129
162
|
private contextFor;
|
|
130
163
|
private toolSpecs;
|
|
131
164
|
private startStream;
|
|
165
|
+
/** think.stream inside an autopilot.llm.request span, with request and latency metrics. */
|
|
166
|
+
private traced;
|
|
132
167
|
private takePrepared;
|
|
133
168
|
private discardPrepared;
|
|
134
169
|
private pump;
|
package/dist/session/Agent.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Agent = exports.logRecorder = void 0;
|
|
3
|
+
exports.Agent = exports.logRecorder = exports.DEFAULT_TRANSFER_FAILED_MESSAGE = void 0;
|
|
4
4
|
exports.sessionRecorder = sessionRecorder;
|
|
5
5
|
/**
|
|
6
6
|
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
@@ -20,11 +20,15 @@ exports.sessionRecorder = sessionRecorder;
|
|
|
20
20
|
* See the License for the specific language governing permissions and
|
|
21
21
|
* limitations under the License.
|
|
22
22
|
*/
|
|
23
|
+
const common_1 = require("@fonoster/common");
|
|
23
24
|
const logger_1 = require("@fonoster/logger");
|
|
24
25
|
const voice_1 = require("@fonoster/voice");
|
|
25
26
|
const AsyncQueue_1 = require("./AsyncQueue");
|
|
26
27
|
const knowledge_1 = require("./knowledge");
|
|
27
28
|
const sentences_1 = require("./sentences");
|
|
29
|
+
const telemetry_1 = require("./telemetry");
|
|
30
|
+
/** What Autopilot says when a transfer fails and the agent sets no `failedMessage`. */
|
|
31
|
+
exports.DEFAULT_TRANSFER_FAILED_MESSAGE = "Sorry, I couldn't connect you. How else can I help?";
|
|
28
32
|
const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
|
|
29
33
|
const MAX_TOOL_ROUNDS = 4;
|
|
30
34
|
const BUILT_IN_TOOLS = [
|
|
@@ -44,7 +48,8 @@ function sessionRecorder(session) {
|
|
|
44
48
|
const write = (data) => void session.note(data).catch(() => undefined);
|
|
45
49
|
return {
|
|
46
50
|
turn: (trace) => write({ kind: "autopilot.turn", ...trace }),
|
|
47
|
-
note: (event) => write({ kind: "autopilot.note", ...event })
|
|
51
|
+
note: (event) => write({ kind: "autopilot.note", ...event }),
|
|
52
|
+
usage: (lines) => void session.reportUsage(lines).catch(() => undefined)
|
|
48
53
|
};
|
|
49
54
|
}
|
|
50
55
|
exports.logRecorder = {
|
|
@@ -53,9 +58,9 @@ exports.logRecorder = {
|
|
|
53
58
|
};
|
|
54
59
|
/** The side effects of Autopilot: everything the state machine asks it to do on the session. */
|
|
55
60
|
class Agent {
|
|
56
|
-
constructor(session,
|
|
61
|
+
constructor(session, agent, think, tools, recorder = exports.logRecorder, knowledge) {
|
|
57
62
|
this.session = session;
|
|
58
|
-
this.
|
|
63
|
+
this.agent = agent;
|
|
59
64
|
this.think = think;
|
|
60
65
|
this.tools = tools;
|
|
61
66
|
this.recorder = recorder;
|
|
@@ -65,10 +70,10 @@ class Agent {
|
|
|
65
70
|
this.resumes = 0;
|
|
66
71
|
this.stopped = false;
|
|
67
72
|
this.send = () => undefined;
|
|
68
|
-
this.prepareEarly =
|
|
73
|
+
this.prepareEarly = agent.conversationSettings.patience !== "high";
|
|
69
74
|
}
|
|
70
75
|
get settings() {
|
|
71
|
-
return this.
|
|
76
|
+
return this.agent.conversationSettings;
|
|
72
77
|
}
|
|
73
78
|
// ---- lifecycle ----------------------------------------------------------------------
|
|
74
79
|
/** Greeting: subscribe to the estimate, send initial DTMF, speak the first message. */
|
|
@@ -76,8 +81,8 @@ class Agent {
|
|
|
76
81
|
this.maxTimer = setTimeout(() => this.send({ type: "MAX_DURATION" }), this.settings.maxSessionDuration);
|
|
77
82
|
await this.safely(async () => {
|
|
78
83
|
await this.session.subscribe(["turn.estimate"]);
|
|
79
|
-
//
|
|
80
|
-
// Recognition hints (names, brands) add to the app's defaults at
|
|
84
|
+
// AppBridge's Pacing policy owns turn timing; Autopilot only tells it how patient to be.
|
|
85
|
+
// Recognition hints (names, brands) add to the app's defaults at AppBridge (VO-B3).
|
|
81
86
|
const hints = this.settings.hints?.length
|
|
82
87
|
? this.settings.hints
|
|
83
88
|
: undefined;
|
|
@@ -144,7 +149,7 @@ class Agent {
|
|
|
144
149
|
...this.messages(prepared.context),
|
|
145
150
|
{ role: "user", content: text }
|
|
146
151
|
];
|
|
147
|
-
await this.pump(this.
|
|
152
|
+
await this.pump(this.traced(messages, abort.signal), chunks);
|
|
148
153
|
})();
|
|
149
154
|
}
|
|
150
155
|
/** turn.resumed: the caller kept talking; the prepared reply is dropped. */
|
|
@@ -152,7 +157,7 @@ class Agent {
|
|
|
152
157
|
this.discardPrepared();
|
|
153
158
|
this.resumes++;
|
|
154
159
|
if (this.resumes >= 3 && this.prepareEarly) {
|
|
155
|
-
// A caller who pauses often: stop preparing early for the rest of this call.
|
|
160
|
+
// A caller who pauses often: stop preparing early for the rest of this call. AppBridge
|
|
156
161
|
// adapts its own turn timing within the call (TT-B6), so Autopilot does not also change
|
|
157
162
|
// the patience it asked for (AP-B2).
|
|
158
163
|
this.prepareEarly = false;
|
|
@@ -181,7 +186,23 @@ class Agent {
|
|
|
181
186
|
this.inflight = run;
|
|
182
187
|
return run;
|
|
183
188
|
}
|
|
189
|
+
/** One turn (STT final, LLM, TTS) as an autopilot.turn span. */
|
|
184
190
|
async respond(text, abort, previous) {
|
|
191
|
+
const callRef = this.session.callRef;
|
|
192
|
+
const span = telemetry_1.telemetry.startSpan("autopilot.turn", { [common_1.CALL_REF_ATTR]: callRef }, telemetry_1.telemetry.traceparent(callRef));
|
|
193
|
+
try {
|
|
194
|
+
await this.respondTurn(text, abort, previous);
|
|
195
|
+
span.end({ "turn.cancelled": abort.signal.aborted });
|
|
196
|
+
}
|
|
197
|
+
catch (e) {
|
|
198
|
+
span.end({}, e.message);
|
|
199
|
+
throw e;
|
|
200
|
+
}
|
|
201
|
+
finally {
|
|
202
|
+
telemetry_1.telemetry.count("fonoster.autopilot.turns", 1);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
async respondTurn(text, abort, previous) {
|
|
185
206
|
// Let a cancelled reply settle first, so what the caller heard of it is recorded before
|
|
186
207
|
// this turn (honest history, A6).
|
|
187
208
|
await previous?.catch(() => undefined);
|
|
@@ -232,6 +253,8 @@ class Agent {
|
|
|
232
253
|
break;
|
|
233
254
|
const builtIn = outcome.toolCalls.find((c) => c.name === "hangup" || c.name === "transfer");
|
|
234
255
|
if (builtIn) {
|
|
256
|
+
if (builtIn.name === "transfer")
|
|
257
|
+
this.transferCallId = builtIn.id;
|
|
235
258
|
this.recorder.turn({
|
|
236
259
|
...trace,
|
|
237
260
|
toolCalls: [...trace.toolCalls, { name: builtIn.name, ok: true }]
|
|
@@ -279,7 +302,7 @@ class Agent {
|
|
|
279
302
|
full += chunk.text;
|
|
280
303
|
text.push(chunk.text);
|
|
281
304
|
}
|
|
282
|
-
else
|
|
305
|
+
else if (chunk.type === "tool_calls")
|
|
283
306
|
toolCalls = chunk.calls;
|
|
284
307
|
}
|
|
285
308
|
text.end();
|
|
@@ -323,8 +346,8 @@ class Agent {
|
|
|
323
346
|
}
|
|
324
347
|
async runTools(calls, abort, trace) {
|
|
325
348
|
for (const call of calls) {
|
|
326
|
-
const tool = this.
|
|
327
|
-
// The start message is spoken while the tool runs;
|
|
349
|
+
const tool = this.agent.languageModel.tools.find((t) => t.name === call.name);
|
|
350
|
+
// The start message is spoken while the tool runs; AppBridge queues the reply after it.
|
|
328
351
|
if (tool?.requestStartMessage) {
|
|
329
352
|
void this.session
|
|
330
353
|
.say(tool.requestStartMessage, this.speechOptions())
|
|
@@ -373,9 +396,33 @@ class Agent {
|
|
|
373
396
|
...this.speechOptions(),
|
|
374
397
|
interruptible: false
|
|
375
398
|
});
|
|
376
|
-
|
|
399
|
+
try {
|
|
400
|
+
await this.session.transfer(options.phoneNumber);
|
|
401
|
+
}
|
|
402
|
+
catch (e) {
|
|
403
|
+
if (!(e instanceof voice_1.SessionError) || e.code !== "transfer_failed")
|
|
404
|
+
throw e;
|
|
405
|
+
await this.transferFailed(options.failedMessage);
|
|
406
|
+
}
|
|
377
407
|
});
|
|
378
408
|
}
|
|
409
|
+
/**
|
|
410
|
+
* The transfer failed: say so and keep helping (autopilot "Transfer fails"). The model learns
|
|
411
|
+
* the outcome as its transfer call's result, and the machine returns to listening.
|
|
412
|
+
*/
|
|
413
|
+
async transferFailed(message) {
|
|
414
|
+
const callId = this.transferCallId;
|
|
415
|
+
this.transferCallId = undefined;
|
|
416
|
+
if (callId)
|
|
417
|
+
this.history.push({
|
|
418
|
+
role: "tool",
|
|
419
|
+
tool_call_id: callId,
|
|
420
|
+
content: JSON.stringify({ error: "the transfer failed" })
|
|
421
|
+
});
|
|
422
|
+
const r = await this.session.say(message || exports.DEFAULT_TRANSFER_FAILED_MESSAGE, this.speechOptions());
|
|
423
|
+
this.history.push({ role: "assistant", content: r.heardText });
|
|
424
|
+
this.send({ type: "SPOKEN", source: "error" });
|
|
425
|
+
}
|
|
379
426
|
// ---- internals --------------------------------------------------------------------
|
|
380
427
|
messages(context = []) {
|
|
381
428
|
return [
|
|
@@ -408,7 +455,7 @@ class Agent {
|
|
|
408
455
|
}
|
|
409
456
|
}
|
|
410
457
|
toolSpecs() {
|
|
411
|
-
const own = this.
|
|
458
|
+
const own = this.agent.languageModel.tools.map((t) => ({
|
|
412
459
|
name: t.name,
|
|
413
460
|
description: t.description,
|
|
414
461
|
parameters: t.parameters
|
|
@@ -418,9 +465,36 @@ class Agent {
|
|
|
418
465
|
}
|
|
419
466
|
startStream(signal, context = []) {
|
|
420
467
|
const chunks = new AsyncQueue_1.AsyncQueue();
|
|
421
|
-
void this.pump(this.
|
|
468
|
+
void this.pump(this.traced(this.messages(context), signal), chunks);
|
|
422
469
|
return chunks;
|
|
423
470
|
}
|
|
471
|
+
/** think.stream inside an autopilot.llm.request span, with request and latency metrics. */
|
|
472
|
+
async *traced(messages, signal) {
|
|
473
|
+
const callRef = this.session.callRef;
|
|
474
|
+
const span = telemetry_1.telemetry.startSpan("autopilot.llm.request", { [common_1.CALL_REF_ATTR]: callRef, "llm.messages": messages.length }, telemetry_1.telemetry.traceparent(callRef));
|
|
475
|
+
const started = Date.now();
|
|
476
|
+
let first;
|
|
477
|
+
let failure;
|
|
478
|
+
try {
|
|
479
|
+
for await (const chunk of this.think.stream(messages, this.toolSpecs(), signal)) {
|
|
480
|
+
first ?? (first = Date.now() - started);
|
|
481
|
+
yield chunk;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
catch (e) {
|
|
485
|
+
failure = e.message;
|
|
486
|
+
throw e;
|
|
487
|
+
}
|
|
488
|
+
finally {
|
|
489
|
+
telemetry_1.telemetry.count("fonoster.autopilot.llm.requests", 1);
|
|
490
|
+
if (first !== undefined)
|
|
491
|
+
telemetry_1.telemetry.count("fonoster.autopilot.llm.latency_ms", first);
|
|
492
|
+
span.end({
|
|
493
|
+
"llm.first_chunk_ms": first ?? -1,
|
|
494
|
+
"llm.cancelled": signal.aborted
|
|
495
|
+
}, failure);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
424
498
|
takePrepared(text) {
|
|
425
499
|
const p = this.prepared;
|
|
426
500
|
this.prepared = undefined;
|
package/dist/session/config.d.ts
CHANGED
|
@@ -16,5 +16,5 @@
|
|
|
16
16
|
* See the License for the specific language governing permissions and
|
|
17
17
|
* limitations under the License.
|
|
18
18
|
*/
|
|
19
|
-
export {
|
|
20
|
-
export type {
|
|
19
|
+
export { sessionAgentSchema, sessionConversationSettingsSchema, sessionLanguageModelSchema } from "@fonoster/common";
|
|
20
|
+
export type { SessionAgent, SessionConversationSettings, SessionLanguageModel, ToolDefinition } from "@fonoster/common";
|
package/dist/session/config.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.sessionLanguageModelSchema = exports.sessionConversationSettingsSchema = exports.
|
|
3
|
+
exports.sessionLanguageModelSchema = exports.sessionConversationSettingsSchema = exports.sessionAgentSchema = void 0;
|
|
4
4
|
/**
|
|
5
5
|
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
6
6
|
* http://github.com/fonoster/fonoster
|
|
@@ -19,9 +19,9 @@ exports.sessionLanguageModelSchema = exports.sessionConversationSettingsSchema =
|
|
|
19
19
|
* See the License for the specific language governing permissions and
|
|
20
20
|
* limitations under the License.
|
|
21
21
|
*/
|
|
22
|
-
// The session
|
|
23
|
-
// plane validates
|
|
22
|
+
// The session agent schema lives in @fonoster/common (rework-autopilot 5.1), so the control
|
|
23
|
+
// plane validates agents with the same rules Autopilot runs with.
|
|
24
24
|
var common_1 = require("@fonoster/common");
|
|
25
|
-
Object.defineProperty(exports, "
|
|
25
|
+
Object.defineProperty(exports, "sessionAgentSchema", { enumerable: true, get: function () { return common_1.sessionAgentSchema; } });
|
|
26
26
|
Object.defineProperty(exports, "sessionConversationSettingsSchema", { enumerable: true, get: function () { return common_1.sessionConversationSettingsSchema; } });
|
|
27
27
|
Object.defineProperty(exports, "sessionLanguageModelSchema", { enumerable: true, get: function () { return common_1.sessionLanguageModelSchema; } });
|
|
@@ -1,24 +1,25 @@
|
|
|
1
1
|
import { Agent, Recorder } from "./Agent";
|
|
2
|
-
import {
|
|
2
|
+
import { SessionAgent } from "./config";
|
|
3
3
|
import { KnowledgePort } from "./knowledge";
|
|
4
4
|
import { LlmResolver } from "./llm";
|
|
5
|
+
import { UsageSink } from "./meteredThink";
|
|
5
6
|
import { Think, ThinkProvider } from "./think";
|
|
6
7
|
import { ToolPort } from "./tools";
|
|
7
8
|
import type { Session } from "@fonoster/voice";
|
|
8
9
|
/** What the hosted Autopilot knows about the app a session belongs to. */
|
|
9
|
-
export type
|
|
10
|
-
|
|
10
|
+
export type LoadedAgent = {
|
|
11
|
+
agent: unknown;
|
|
11
12
|
/** The workspace that owns the app: scope for its own keys (BYOK) and knowledge. */
|
|
12
13
|
workspaceRef?: string;
|
|
13
14
|
};
|
|
14
15
|
export type AutopilotOptions = {
|
|
15
|
-
/** The
|
|
16
|
-
|
|
16
|
+
/** The agent for a session (validated with the session agent schema). */
|
|
17
|
+
agent?: unknown | ((session: Session) => Promise<unknown> | unknown);
|
|
17
18
|
/**
|
|
18
|
-
* The hosted path: loads the app's
|
|
19
|
-
* app has no
|
|
19
|
+
* The hosted path: loads the app's agent and workspace for a session; undefined means the
|
|
20
|
+
* app has no agent (the call is ended).
|
|
20
21
|
*/
|
|
21
|
-
load?: (session: Session) => Promise<
|
|
22
|
+
load?: (session: Session) => Promise<LoadedAgent | undefined>;
|
|
22
23
|
/** Language models through the provider registry (task 3.1). */
|
|
23
24
|
llm?: LlmResolver;
|
|
24
25
|
/** Language model endpoints by provider name (the stand-in used before the registry). */
|
|
@@ -32,10 +33,10 @@ export type AutopilotOptions = {
|
|
|
32
33
|
allowedDestinations?: string[];
|
|
33
34
|
/** Finds what is relevant to the caller's question (the knowledge port). */
|
|
34
35
|
knowledge?: KnowledgePort;
|
|
35
|
-
/** Builds the knowledge port for an
|
|
36
|
-
knowledgeFor?: (
|
|
36
|
+
/** Builds the knowledge port for an agent (operator wiring, task 4.1). */
|
|
37
|
+
knowledgeFor?: (agent: SessionAgent, scope: {
|
|
37
38
|
workspaceRef?: string;
|
|
38
|
-
}) => KnowledgePort | undefined;
|
|
39
|
+
}, usage?: UsageSink) => KnowledgePort | undefined;
|
|
39
40
|
recorder?: Recorder;
|
|
40
41
|
/** Called with each session's Agent, for tests and observability. */
|
|
41
42
|
onAgent?: (agent: Agent) => void;
|