@fonoster/autopilot 0.32.0 → 0.34.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/knowledge/AbstractKnowledgeBase.d.ts +3 -0
- package/dist/knowledge/AbstractKnowledgeBase.js +17 -4
- package/dist/knowledge/MeteredEmbeddings.d.ts +12 -0
- package/dist/knowledge/MeteredEmbeddings.js +20 -1
- package/dist/knowledge/index.d.ts +1 -0
- package/dist/knowledge/index.js +1 -0
- package/dist/knowledge/passages.d.ts +32 -0
- package/dist/knowledge/passages.js +18 -0
- package/dist/knowledge/types.d.ts +2 -0
- package/dist/session/Agent.d.ts +24 -4
- package/dist/session/Agent.js +90 -24
- package/dist/session/createAutopilot.js +7 -0
- package/dist/session/hosted.d.ts +2 -1
- package/dist/session/hosted.js +4 -0
- package/dist/session/index.d.ts +1 -0
- package/dist/session/index.js +1 -0
- package/dist/session/knowledge.d.ts +8 -0
- package/dist/session/knowledge.js +35 -2
- package/dist/session/llm.js +3 -1
- package/dist/session/meteredThink.d.ts +6 -0
- package/dist/session/tools.d.ts +3 -0
- package/dist/session/tools.js +18 -14
- package/dist/session/traceDetails.d.ts +94 -0
- package/dist/session/traceDetails.js +113 -0
- package/package.json +7 -7
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Embeddings } from "@langchain/core/embeddings";
|
|
2
2
|
import { VectorStore } from "@langchain/core/vectorstores";
|
|
3
|
+
import { KnowledgePassage } from "./passages";
|
|
3
4
|
import { KnowledgeBase } from "./types";
|
|
4
5
|
declare abstract class AbstractKnowledgeBase implements KnowledgeBase {
|
|
5
6
|
protected embeddings: Embeddings;
|
|
@@ -11,5 +12,7 @@ declare abstract class AbstractKnowledgeBase implements KnowledgeBase {
|
|
|
11
12
|
abstract getLoaders(): Promise<unknown>;
|
|
12
13
|
load(): Promise<void>;
|
|
13
14
|
queryKnowledgeBase(query: string, k?: number): Promise<string>;
|
|
15
|
+
/** The best `k` passages for the query, with their document and similarity score. */
|
|
16
|
+
searchKnowledgeBase(query: string, k?: number): Promise<KnowledgePassage[]>;
|
|
14
17
|
}
|
|
15
18
|
export { AbstractKnowledgeBase };
|
|
@@ -19,9 +19,11 @@ exports.AbstractKnowledgeBase = void 0;
|
|
|
19
19
|
* See the License for the specific language governing permissions and
|
|
20
20
|
* limitations under the License.
|
|
21
21
|
*/
|
|
22
|
+
const node_path_1 = require("node:path");
|
|
22
23
|
const logger_1 = require("@fonoster/logger");
|
|
23
24
|
const memory_1 = require("@langchain/classic/vectorstores/memory");
|
|
24
25
|
const textsplitters_1 = require("@langchain/textsplitters");
|
|
26
|
+
const passages_1 = require("./passages");
|
|
25
27
|
const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
|
|
26
28
|
class AbstractKnowledgeBase {
|
|
27
29
|
/** The embeddings come from the caller (the hosted Autopilot resolves their key by credentialRef). */
|
|
@@ -44,13 +46,24 @@ class AbstractKnowledgeBase {
|
|
|
44
46
|
this.vectorStore = await memory_1.MemoryVectorStore.fromDocuments(splitDocs.flat(), this.embeddings);
|
|
45
47
|
}
|
|
46
48
|
async queryKnowledgeBase(query, k = 2) {
|
|
49
|
+
return (0, passages_1.passagesText)(await this.searchKnowledgeBase(query, k));
|
|
50
|
+
}
|
|
51
|
+
/** The best `k` passages for the query, with their document and similarity score. */
|
|
52
|
+
async searchKnowledgeBase(query, k = 2) {
|
|
47
53
|
const { vectorStore } = this;
|
|
48
54
|
if (!vectorStore) {
|
|
49
|
-
logger.verbose("vector store is not initialized, returning
|
|
50
|
-
return
|
|
55
|
+
logger.verbose("vector store is not initialized, returning nothing");
|
|
56
|
+
return [];
|
|
51
57
|
}
|
|
52
|
-
const results = await vectorStore.
|
|
53
|
-
return results.
|
|
58
|
+
const results = await vectorStore.similaritySearchWithScore(query, k);
|
|
59
|
+
return results.map(([doc, score]) => {
|
|
60
|
+
const source = doc.metadata?.filename ?? doc.metadata?.source;
|
|
61
|
+
return {
|
|
62
|
+
text: doc.pageContent,
|
|
63
|
+
...(source ? { source: (0, node_path_1.basename)(String(source)) } : {}),
|
|
64
|
+
score
|
|
65
|
+
};
|
|
66
|
+
});
|
|
54
67
|
}
|
|
55
68
|
}
|
|
56
69
|
exports.AbstractKnowledgeBase = AbstractKnowledgeBase;
|
|
@@ -21,6 +21,12 @@ import { Usage } from "@fonoster/common";
|
|
|
21
21
|
export type EmbeddingUsageSink = {
|
|
22
22
|
usage(lines: Usage.UsageLine[]): void;
|
|
23
23
|
missing(catalogId: string): void;
|
|
24
|
+
/** The vendor rejected the embeddings key (HTTP 401/403): reported once per instance. */
|
|
25
|
+
keyRejected?(event: {
|
|
26
|
+
provider: string;
|
|
27
|
+
catalogId: string;
|
|
28
|
+
status: number;
|
|
29
|
+
}): void;
|
|
24
30
|
};
|
|
25
31
|
/**
|
|
26
32
|
* OpenAI embeddings that report their token usage (add-usage-metering): the knowledge base's
|
|
@@ -32,6 +38,11 @@ export declare class MeteredEmbeddings {
|
|
|
32
38
|
private readonly options;
|
|
33
39
|
private readonly model;
|
|
34
40
|
private readonly baseUrl;
|
|
41
|
+
/**
|
|
42
|
+
* The status a rejected key was answered with. Once set, no more requests go out: the key
|
|
43
|
+
* won't start working mid-call, and knowledge is optional, so the call goes on without it.
|
|
44
|
+
*/
|
|
45
|
+
private rejected?;
|
|
35
46
|
constructor(options: {
|
|
36
47
|
apiKey: string;
|
|
37
48
|
keySource: Usage.KeySource;
|
|
@@ -44,4 +55,5 @@ export declare class MeteredEmbeddings {
|
|
|
44
55
|
embedDocuments(texts: string[]): Promise<number[][]>;
|
|
45
56
|
embedQuery(text: string): Promise<number[]>;
|
|
46
57
|
private embed;
|
|
58
|
+
private rejectedError;
|
|
47
59
|
}
|
|
@@ -26,6 +26,8 @@ class MeteredEmbeddings {
|
|
|
26
26
|
return (await this.embed([text]))[0];
|
|
27
27
|
}
|
|
28
28
|
async embed(input) {
|
|
29
|
+
if (this.rejected)
|
|
30
|
+
throw this.rejectedError(this.rejected);
|
|
29
31
|
const res = await fetch(`${this.baseUrl}/v1/embeddings`, {
|
|
30
32
|
method: "POST",
|
|
31
33
|
headers: {
|
|
@@ -34,8 +36,22 @@ class MeteredEmbeddings {
|
|
|
34
36
|
},
|
|
35
37
|
body: JSON.stringify({ model: this.model, input })
|
|
36
38
|
});
|
|
37
|
-
if (
|
|
39
|
+
if (res.status === 401 || res.status === 403) {
|
|
40
|
+
await res.body?.cancel().catch(() => undefined);
|
|
41
|
+
if (!this.rejected) {
|
|
42
|
+
this.rejected = res.status;
|
|
43
|
+
this.options.sink?.keyRejected?.({
|
|
44
|
+
provider: "openai",
|
|
45
|
+
catalogId: this.catalogId,
|
|
46
|
+
status: res.status
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
throw this.rejectedError(res.status);
|
|
50
|
+
}
|
|
51
|
+
if (!res.ok) {
|
|
52
|
+
await res.body?.cancel().catch(() => undefined);
|
|
38
53
|
throw new Error(`embeddings answered HTTP ${res.status}`);
|
|
54
|
+
}
|
|
39
55
|
const body = (await res.json());
|
|
40
56
|
const tokens = body.usage?.prompt_tokens;
|
|
41
57
|
if (tokens === undefined)
|
|
@@ -54,5 +70,8 @@ class MeteredEmbeddings {
|
|
|
54
70
|
.sort((a, b) => a.index - b.index)
|
|
55
71
|
.map((d) => d.embedding);
|
|
56
72
|
}
|
|
73
|
+
rejectedError(status) {
|
|
74
|
+
return new Error(`embeddings rejected the credentials (HTTP ${status})`);
|
|
75
|
+
}
|
|
57
76
|
}
|
|
58
77
|
exports.MeteredEmbeddings = MeteredEmbeddings;
|
package/dist/knowledge/index.js
CHANGED
|
@@ -34,5 +34,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
34
34
|
*/
|
|
35
35
|
__exportStar(require("./AbstractKnowledgeBase"), exports);
|
|
36
36
|
__exportStar(require("./FilesKnowledgeBase"), exports);
|
|
37
|
+
__exportStar(require("./passages"), exports);
|
|
37
38
|
__exportStar(require("./S3KnowledgeBase"), exports);
|
|
38
39
|
__exportStar(require("./types"), exports);
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
3
|
+
* http://github.com/fonoster/fonoster
|
|
4
|
+
*
|
|
5
|
+
* This file is part of Fonoster
|
|
6
|
+
*
|
|
7
|
+
* Licensed under the MIT License (the "License");
|
|
8
|
+
* you may not use this file except in compliance with
|
|
9
|
+
* the License. You may obtain a copy of the License at
|
|
10
|
+
*
|
|
11
|
+
* https://opensource.org/licenses/MIT
|
|
12
|
+
*
|
|
13
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
* See the License for the specific language governing permissions and
|
|
17
|
+
* limitations under the License.
|
|
18
|
+
*/
|
|
19
|
+
/** One passage a knowledge search found: its text, the document it came from and its score. */
|
|
20
|
+
export type KnowledgePassage = {
|
|
21
|
+
text: string;
|
|
22
|
+
source?: string;
|
|
23
|
+
score?: number;
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* Each passage is cut at this many characters before it goes into the model's prompt: the
|
|
27
|
+
* splitter's chunk size, so a normal passage goes in whole and a loader that skips splitting
|
|
28
|
+
* can't flood the prompt.
|
|
29
|
+
*/
|
|
30
|
+
export declare const MAX_PROMPT_PASSAGE_TEXT = 1000;
|
|
31
|
+
/** The passages as the model sees them: their text, each capped, one per line. */
|
|
32
|
+
export declare function passagesText(passages: KnowledgePassage[]): string;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MAX_PROMPT_PASSAGE_TEXT = void 0;
|
|
4
|
+
exports.passagesText = passagesText;
|
|
5
|
+
/**
|
|
6
|
+
* Each passage is cut at this many characters before it goes into the model's prompt: the
|
|
7
|
+
* splitter's chunk size, so a normal passage goes in whole and a loader that skips splitting
|
|
8
|
+
* can't flood the prompt.
|
|
9
|
+
*/
|
|
10
|
+
exports.MAX_PROMPT_PASSAGE_TEXT = 1000;
|
|
11
|
+
/** The passages as the model sees them: their text, each capped, one per line. */
|
|
12
|
+
function passagesText(passages) {
|
|
13
|
+
return passages
|
|
14
|
+
.map((p) => p.text.length > exports.MAX_PROMPT_PASSAGE_TEXT
|
|
15
|
+
? `${p.text.slice(0, exports.MAX_PROMPT_PASSAGE_TEXT)}…`
|
|
16
|
+
: p.text)
|
|
17
|
+
.join("\n");
|
|
18
|
+
}
|
|
@@ -17,9 +17,11 @@
|
|
|
17
17
|
* limitations under the License.
|
|
18
18
|
*/
|
|
19
19
|
import { Embeddings } from "@langchain/core/embeddings";
|
|
20
|
+
import { KnowledgePassage } from "./passages";
|
|
20
21
|
type KnowledgeBase = {
|
|
21
22
|
load: () => Promise<void>;
|
|
22
23
|
queryKnowledgeBase: (query: string, k?: number) => Promise<string>;
|
|
24
|
+
searchKnowledgeBase?: (query: string, k?: number) => Promise<KnowledgePassage[]>;
|
|
23
25
|
};
|
|
24
26
|
type S3KnowledgeBaseParams = {
|
|
25
27
|
embeddings: Embeddings;
|
package/dist/session/Agent.d.ts
CHANGED
|
@@ -21,6 +21,7 @@ import { SessionAgent } from "./config";
|
|
|
21
21
|
import { KnowledgePort } from "./knowledge";
|
|
22
22
|
import { ChatMessage, Think } from "./think";
|
|
23
23
|
import { ToolPort } from "./tools";
|
|
24
|
+
import { KnowledgeSearchTrace, ToolCallTrace } from "./traceDetails";
|
|
24
25
|
import type { Session } from "@fonoster/voice";
|
|
25
26
|
/** What Autopilot says when a transfer fails and the agent sets no `failedMessage`. */
|
|
26
27
|
export declare const DEFAULT_TRANSFER_FAILED_MESSAGE = "Sorry, I couldn't connect you. How else can I help?";
|
|
@@ -61,13 +62,23 @@ export type TurnTrace = {
|
|
|
61
62
|
replyText: string;
|
|
62
63
|
heardText: string;
|
|
63
64
|
interrupted: boolean;
|
|
64
|
-
toolCalls:
|
|
65
|
-
name: string;
|
|
66
|
-
ok: boolean;
|
|
67
|
-
}[];
|
|
65
|
+
toolCalls: ToolCallTrace[];
|
|
68
66
|
/** ms from turn.ended to the first token, and to the first speak sent. */
|
|
69
67
|
firstTokenMs?: number;
|
|
70
68
|
firstSpeakMs?: number;
|
|
69
|
+
/**
|
|
70
|
+
* Where the caller's wait went, in ms, when the turn's first sentence was also its first audio
|
|
71
|
+
* and the reply played to the end: `llmMs` from turn.ended to that sentence handed to speech
|
|
72
|
+
* (knowledge lookups and tool calls before the first words count here), `ttsMs` from that
|
|
73
|
+
* sentence to its own first audio (output.started). Absent when the turn was interrupted, or
|
|
74
|
+
* when other speech (a tool's start message) went out before the reply.
|
|
75
|
+
*/
|
|
76
|
+
llmMs?: number;
|
|
77
|
+
ttsMs?: number;
|
|
78
|
+
/** Set when speech other than the reply (a tool's start message) went out during the turn. */
|
|
79
|
+
spokeAhead?: boolean;
|
|
80
|
+
/** What the knowledge port found for the caller's words (agents with knowledge only). */
|
|
81
|
+
knowledgeSearches?: KnowledgeSearchTrace[];
|
|
71
82
|
};
|
|
72
83
|
export interface Recorder {
|
|
73
84
|
turn(trace: TurnTrace): void;
|
|
@@ -99,6 +110,8 @@ export declare class Agent {
|
|
|
99
110
|
/** Whether a reply is prepared on turn.likely_ended (off at high patience or once adapted). */
|
|
100
111
|
private prepareEarly;
|
|
101
112
|
private stopped;
|
|
113
|
+
/** A vendor rejected the model's key and the call is failing (#1146). */
|
|
114
|
+
private keyFailed;
|
|
102
115
|
send: (event: AgentEvent) => void;
|
|
103
116
|
constructor(session: Session, agent: SessionAgent, think: Think, tools: ToolPort, recorder?: Recorder, knowledge?: KnowledgePort);
|
|
104
117
|
get settings(): {
|
|
@@ -132,6 +145,8 @@ export declare class Agent {
|
|
|
132
145
|
/** Greeting: subscribe to the estimate, send initial DTMF, speak the first message. */
|
|
133
146
|
greet(): Promise<void>;
|
|
134
147
|
stop(): void;
|
|
148
|
+
/** Nothing more is said or prepared: the reply, the prepared reply and the timers stop. */
|
|
149
|
+
private endTurns;
|
|
135
150
|
armIdle(): void;
|
|
136
151
|
disarmIdle(): void;
|
|
137
152
|
/** The next idle timeout ends the call (the goodbye counts as the last announcement). */
|
|
@@ -150,6 +165,11 @@ export declare class Agent {
|
|
|
150
165
|
private respondTurn;
|
|
151
166
|
private speakRound;
|
|
152
167
|
private runTools;
|
|
168
|
+
/**
|
|
169
|
+
* The model's vendor rejected the key: retrying can't help, so the call fails and hangs up at
|
|
170
|
+
* once, without an error message (#1146).
|
|
171
|
+
*/
|
|
172
|
+
private keyRejected;
|
|
153
173
|
goodbye(): Promise<void>;
|
|
154
174
|
transfer(): Promise<void>;
|
|
155
175
|
/**
|
package/dist/session/Agent.js
CHANGED
|
@@ -21,12 +21,14 @@ exports.sessionRecorder = sessionRecorder;
|
|
|
21
21
|
* limitations under the License.
|
|
22
22
|
*/
|
|
23
23
|
const common_1 = require("@fonoster/common");
|
|
24
|
+
const community_1 = require("@fonoster/community");
|
|
24
25
|
const logger_1 = require("@fonoster/logger");
|
|
25
26
|
const voice_1 = require("@fonoster/voice");
|
|
26
27
|
const AsyncQueue_1 = require("./AsyncQueue");
|
|
27
28
|
const knowledge_1 = require("./knowledge");
|
|
28
29
|
const sentences_1 = require("./sentences");
|
|
29
30
|
const telemetry_1 = require("./telemetry");
|
|
31
|
+
const traceDetails_1 = require("./traceDetails");
|
|
30
32
|
/** What Autopilot says when a transfer fails and the agent sets no `failedMessage`. */
|
|
31
33
|
exports.DEFAULT_TRANSFER_FAILED_MESSAGE = "Sorry, I couldn't connect you. How else can I help?";
|
|
32
34
|
const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
|
|
@@ -43,6 +45,16 @@ const BUILT_IN_TOOLS = [
|
|
|
43
45
|
parameters: { type: "object", properties: {} }
|
|
44
46
|
}
|
|
45
47
|
];
|
|
48
|
+
/** The note for a finished turn: an interrupted reply carries no stages. */
|
|
49
|
+
function turnNote(trace) {
|
|
50
|
+
const note = { ...trace };
|
|
51
|
+
delete note.spokeAhead;
|
|
52
|
+
if (note.interrupted) {
|
|
53
|
+
delete note.llmMs;
|
|
54
|
+
delete note.ttsMs;
|
|
55
|
+
}
|
|
56
|
+
return note;
|
|
57
|
+
}
|
|
46
58
|
/** Writes turn traces and notes to the call record through the session (record.note). */
|
|
47
59
|
function sessionRecorder(session) {
|
|
48
60
|
const write = (data) => void session.note(data).catch(() => undefined);
|
|
@@ -53,7 +65,8 @@ function sessionRecorder(session) {
|
|
|
53
65
|
};
|
|
54
66
|
}
|
|
55
67
|
exports.logRecorder = {
|
|
56
|
-
|
|
68
|
+
// Without a call record, tool payloads and knowledge queries stay out of the logs.
|
|
69
|
+
turn: (trace) => logger.verbose("autopilot turn", (0, traceDetails_1.loggedTurn)(trace)),
|
|
57
70
|
note: (event) => logger.verbose("autopilot note", event)
|
|
58
71
|
};
|
|
59
72
|
/** The side effects of Autopilot: everything the state machine asks it to do on the session. */
|
|
@@ -69,6 +82,8 @@ class Agent {
|
|
|
69
82
|
this.idleCount = 0;
|
|
70
83
|
this.resumes = 0;
|
|
71
84
|
this.stopped = false;
|
|
85
|
+
/** A vendor rejected the model's key and the call is failing (#1146). */
|
|
86
|
+
this.keyFailed = false;
|
|
72
87
|
this.send = () => undefined;
|
|
73
88
|
this.prepareEarly = agent.conversationSettings.patience !== "high";
|
|
74
89
|
}
|
|
@@ -104,6 +119,10 @@ class Agent {
|
|
|
104
119
|
}
|
|
105
120
|
stop() {
|
|
106
121
|
this.stopped = true;
|
|
122
|
+
this.endTurns();
|
|
123
|
+
}
|
|
124
|
+
/** Nothing more is said or prepared: the reply, the prepared reply and the timers stop. */
|
|
125
|
+
endTurns() {
|
|
107
126
|
this.abortReply();
|
|
108
127
|
this.discardPrepared();
|
|
109
128
|
clearTimeout(this.idleTimer);
|
|
@@ -139,10 +158,16 @@ class Agent {
|
|
|
139
158
|
this.discardPrepared();
|
|
140
159
|
const abort = new AbortController();
|
|
141
160
|
const chunks = new AsyncQueue_1.AsyncQueue();
|
|
142
|
-
const prepared = {
|
|
161
|
+
const prepared = {
|
|
162
|
+
text,
|
|
163
|
+
abort,
|
|
164
|
+
chunks,
|
|
165
|
+
context: [],
|
|
166
|
+
searches: []
|
|
167
|
+
};
|
|
143
168
|
this.prepared = prepared;
|
|
144
169
|
void (async () => {
|
|
145
|
-
prepared.context = await this.contextFor(text, abort.signal);
|
|
170
|
+
prepared.context = await this.contextFor(text, abort.signal, prepared.searches);
|
|
146
171
|
if (abort.signal.aborted)
|
|
147
172
|
return chunks.end();
|
|
148
173
|
const messages = [
|
|
@@ -227,9 +252,12 @@ class Agent {
|
|
|
227
252
|
}
|
|
228
253
|
try {
|
|
229
254
|
const prepared = this.takePrepared(text);
|
|
255
|
+
const searches = prepared?.searches ?? [];
|
|
256
|
+
if (this.knowledge)
|
|
257
|
+
trace.knowledgeSearches = searches;
|
|
230
258
|
const context = prepared
|
|
231
259
|
? prepared.context
|
|
232
|
-
: await this.contextFor(text, abort.signal);
|
|
260
|
+
: await this.contextFor(text, abort.signal, searches);
|
|
233
261
|
let source = prepared?.chunks ?? this.startStream(abort.signal, context);
|
|
234
262
|
for (let round = 0; round < MAX_TOOL_ROUNDS; round++) {
|
|
235
263
|
const outcome = await this.speakRound(source, abort, preempt, turnEnded, trace);
|
|
@@ -255,10 +283,13 @@ class Agent {
|
|
|
255
283
|
if (builtIn) {
|
|
256
284
|
if (builtIn.name === "transfer")
|
|
257
285
|
this.transferCallId = builtIn.id;
|
|
258
|
-
this.recorder.turn({
|
|
286
|
+
this.recorder.turn(turnNote({
|
|
259
287
|
...trace,
|
|
260
|
-
toolCalls: [
|
|
261
|
-
|
|
288
|
+
toolCalls: [
|
|
289
|
+
...trace.toolCalls,
|
|
290
|
+
{ name: builtIn.name, ok: true, startedAtMs: Date.now() }
|
|
291
|
+
]
|
|
292
|
+
}));
|
|
262
293
|
this.send({
|
|
263
294
|
type: builtIn.name === "hangup"
|
|
264
295
|
? "HANGUP_REQUESTED"
|
|
@@ -271,7 +302,7 @@ class Agent {
|
|
|
271
302
|
return;
|
|
272
303
|
source = this.startStream(abort.signal, context);
|
|
273
304
|
}
|
|
274
|
-
this.recorder.turn(trace);
|
|
305
|
+
this.recorder.turn(turnNote(trace));
|
|
275
306
|
if (this.reply === abort)
|
|
276
307
|
this.send({ type: "SPOKEN", source: "reply" });
|
|
277
308
|
}
|
|
@@ -280,6 +311,8 @@ class Agent {
|
|
|
280
311
|
e instanceof voice_1.SessionEndedError ||
|
|
281
312
|
this.stopped)
|
|
282
313
|
return;
|
|
314
|
+
if ((0, community_1.isProviderAuthError)(e))
|
|
315
|
+
return this.keyRejected(e);
|
|
283
316
|
logger.error(`autopilot reply failed: ${e.message}`);
|
|
284
317
|
await this.safely(async () => {
|
|
285
318
|
await this.session.say(this.settings.systemErrorMessage, this.speechOptions());
|
|
@@ -318,7 +351,11 @@ class Agent {
|
|
|
318
351
|
let completed = false;
|
|
319
352
|
if (!first.done && !abort.signal.aborted) {
|
|
320
353
|
this.send({ type: "REPLY_SPEAKING" });
|
|
321
|
-
|
|
354
|
+
const spokeAt = Date.now();
|
|
355
|
+
// Stages are measured on the turn's first sentence only, and only when nothing else was
|
|
356
|
+
// queued ahead of it, so its first audio is the first audio the caller heard.
|
|
357
|
+
const measure = trace.firstSpeakMs === undefined && !trace.spokeAhead;
|
|
358
|
+
trace.firstSpeakMs ?? (trace.firstSpeakMs = spokeAt - turnEnded);
|
|
322
359
|
const pieces = (async function* () {
|
|
323
360
|
yield first.value;
|
|
324
361
|
for (;;) {
|
|
@@ -332,6 +369,11 @@ class Agent {
|
|
|
332
369
|
...this.speechOptions(),
|
|
333
370
|
preempt: preempt || undefined
|
|
334
371
|
});
|
|
372
|
+
// startedAt is this say's own output.started, on this clock.
|
|
373
|
+
if (measure && !result.interrupted && result.startedAt !== undefined) {
|
|
374
|
+
trace.llmMs = spokeAt - turnEnded;
|
|
375
|
+
trace.ttsMs = Math.max(0, result.startedAt - spokeAt);
|
|
376
|
+
}
|
|
335
377
|
heard = result.heardText;
|
|
336
378
|
interrupted = result.interrupted;
|
|
337
379
|
completed = !interrupted;
|
|
@@ -349,6 +391,7 @@ class Agent {
|
|
|
349
391
|
const tool = this.agent.languageModel.tools.find((t) => t.name === call.name);
|
|
350
392
|
// The start message is spoken while the tool runs; AppBridge queues the reply after it.
|
|
351
393
|
if (tool?.requestStartMessage) {
|
|
394
|
+
trace.spokeAhead = true;
|
|
352
395
|
void this.session
|
|
353
396
|
.say(tool.requestStartMessage, this.speechOptions())
|
|
354
397
|
.catch(() => undefined);
|
|
@@ -360,8 +403,9 @@ class Agent {
|
|
|
360
403
|
catch {
|
|
361
404
|
args = {};
|
|
362
405
|
}
|
|
406
|
+
const started = Date.now();
|
|
363
407
|
const outcome = await this.tools.run(call.name, args, abort.signal);
|
|
364
|
-
trace.toolCalls.push(
|
|
408
|
+
trace.toolCalls.push((0, traceDetails_1.toolCallTrace)(call.name, args, outcome, started, Date.now() - started));
|
|
365
409
|
this.history.push({
|
|
366
410
|
role: "tool",
|
|
367
411
|
tool_call_id: call.id,
|
|
@@ -370,6 +414,20 @@ class Agent {
|
|
|
370
414
|
}
|
|
371
415
|
}
|
|
372
416
|
// ---- endings ----------------------------------------------------------------------
|
|
417
|
+
/**
|
|
418
|
+
* The model's vendor rejected the key: retrying can't help, so the call fails and hangs up at
|
|
419
|
+
* once, without an error message (#1146).
|
|
420
|
+
*/
|
|
421
|
+
async keyRejected(e) {
|
|
422
|
+
if (this.keyFailed || this.stopped)
|
|
423
|
+
return;
|
|
424
|
+
this.keyFailed = true;
|
|
425
|
+
logger.error(`autopilot cannot use its language model: ${e.message}`);
|
|
426
|
+
this.endTurns();
|
|
427
|
+
await this.safely(() => this.session.hangup({
|
|
428
|
+
failure: { reason: e.reason, provider: e.provider, kind: e.kind }
|
|
429
|
+
}));
|
|
430
|
+
}
|
|
373
431
|
async goodbye() {
|
|
374
432
|
this.abortReply();
|
|
375
433
|
this.discardPrepared();
|
|
@@ -432,27 +490,28 @@ class Agent {
|
|
|
432
490
|
];
|
|
433
491
|
}
|
|
434
492
|
/** Knowledge for this turn, as a system message for the request only (never in history). */
|
|
435
|
-
async contextFor(question, signal) {
|
|
436
|
-
|
|
493
|
+
async contextFor(question, signal, searches) {
|
|
494
|
+
const knowledge = this.knowledge;
|
|
495
|
+
if (!knowledge)
|
|
437
496
|
return [];
|
|
438
|
-
const timeout = new Promise((resolve) => setTimeout(() => resolve(
|
|
497
|
+
const timeout = new Promise((resolve) => setTimeout(() => resolve(undefined), knowledge_1.KNOWLEDGE_TIMEOUT_MS));
|
|
498
|
+
const started = Date.now();
|
|
499
|
+
let passages;
|
|
439
500
|
try {
|
|
440
|
-
|
|
441
|
-
|
|
501
|
+
passages = await Promise.race([
|
|
502
|
+
(0, knowledge_1.searchKnowledge)(knowledge, question, signal),
|
|
442
503
|
timeout
|
|
443
504
|
]);
|
|
444
|
-
return found?.trim()
|
|
445
|
-
? [
|
|
446
|
-
{
|
|
447
|
-
role: "system",
|
|
448
|
-
content: `Relevant information:\n${found.trim()}`
|
|
449
|
-
}
|
|
450
|
-
]
|
|
451
|
-
: [];
|
|
452
505
|
}
|
|
453
|
-
catch {
|
|
506
|
+
catch (error) {
|
|
507
|
+
searches?.push((0, traceDetails_1.knowledgeSearchTrace)(question, { error }, started, Date.now() - started));
|
|
454
508
|
return [];
|
|
455
509
|
}
|
|
510
|
+
searches?.push((0, traceDetails_1.knowledgeSearchTrace)(question, passages ? { passages } : { timedOut: true }, started, Date.now() - started));
|
|
511
|
+
const found = (0, knowledge_1.passagesText)(passages ?? []).trim();
|
|
512
|
+
return found
|
|
513
|
+
? [{ role: "system", content: `Relevant information:\n${found}` }]
|
|
514
|
+
: [];
|
|
456
515
|
}
|
|
457
516
|
toolSpecs() {
|
|
458
517
|
const own = this.agent.languageModel.tools.map((t) => ({
|
|
@@ -515,6 +574,9 @@ class Agent {
|
|
|
515
574
|
this.prepared = undefined;
|
|
516
575
|
}
|
|
517
576
|
async pump(stream, into) {
|
|
577
|
+
// The key was rejected and the call is failing: the model is not asked again.
|
|
578
|
+
if (this.keyFailed)
|
|
579
|
+
return into.end();
|
|
518
580
|
try {
|
|
519
581
|
for await (const chunk of stream)
|
|
520
582
|
into.push(chunk);
|
|
@@ -522,6 +584,10 @@ class Agent {
|
|
|
522
584
|
}
|
|
523
585
|
catch (e) {
|
|
524
586
|
into.fail(e);
|
|
587
|
+
// Fatal at once, even for a reply prepared early that is never spoken: a reply discarded
|
|
588
|
+
// after its key was rejected must not lead to a second request (#1146).
|
|
589
|
+
if ((0, community_1.isProviderAuthError)(e))
|
|
590
|
+
void this.keyRejected(e);
|
|
525
591
|
}
|
|
526
592
|
}
|
|
527
593
|
speechOptions() {
|
|
@@ -103,6 +103,13 @@ function createSessionRunner(options) {
|
|
|
103
103
|
missing: (catalogId) => {
|
|
104
104
|
telemetry_1.telemetry.count("fonoster.usage.missing", 1, { catalog_id: catalogId });
|
|
105
105
|
recorder.note({ type: "usage_missing", catalogId });
|
|
106
|
+
},
|
|
107
|
+
// Knowledge is optional: the call goes on without it, and the record says why (#1146).
|
|
108
|
+
keyRejected: (event) => {
|
|
109
|
+
telemetry_1.telemetry.count("fonoster.autopilot.knowledge.key_rejected", 1, {
|
|
110
|
+
provider: event.provider
|
|
111
|
+
});
|
|
112
|
+
recorder.note({ type: "knowledge.key_rejected", ...event });
|
|
106
113
|
}
|
|
107
114
|
};
|
|
108
115
|
let think = options.think;
|
package/dist/session/hosted.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { Session, VoiceServer } from "@fonoster/voice";
|
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { SessionAgent } from "./config";
|
|
5
5
|
import { AutopilotOptions, LoadedAgent } from "./createAutopilot";
|
|
6
|
-
import { KnowledgePort } from "./knowledge";
|
|
6
|
+
import { KnowledgePassage, KnowledgePort } from "./knowledge";
|
|
7
7
|
import { CredentialSource } from "./llm";
|
|
8
8
|
import { UsageSink } from "./meteredThink";
|
|
9
9
|
export declare const hostedConfigSchema: z.ZodObject<{
|
|
@@ -420,6 +420,7 @@ export declare class FileWorkspaceSecrets implements CredentialSource {
|
|
|
420
420
|
type KnowledgeBaseLike = {
|
|
421
421
|
load(): Promise<void>;
|
|
422
422
|
queryKnowledgeBase(query: string, k?: number): Promise<string>;
|
|
423
|
+
searchKnowledgeBase?(query: string, k?: number): Promise<KnowledgePassage[]>;
|
|
423
424
|
};
|
|
424
425
|
/**
|
|
425
426
|
* Builds the knowledge port for an agent from the operator's S3 + Unstructured settings,
|
package/dist/session/hosted.js
CHANGED
|
@@ -462,6 +462,10 @@ function eagerKnowledge(build) {
|
|
|
462
462
|
find: async (question, signal) => {
|
|
463
463
|
const p = await port;
|
|
464
464
|
return p ? p.find(question, signal) : "";
|
|
465
|
+
},
|
|
466
|
+
search: async (question, signal) => {
|
|
467
|
+
const p = await port;
|
|
468
|
+
return p ? (0, knowledge_1.searchKnowledge)(p, question, signal) : [];
|
|
465
469
|
}
|
|
466
470
|
};
|
|
467
471
|
};
|
package/dist/session/index.d.ts
CHANGED
package/dist/session/index.js
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
* See the License for the specific language governing permissions and
|
|
17
17
|
* limitations under the License.
|
|
18
18
|
*/
|
|
19
|
+
import { KnowledgePassage, passagesText } from "../knowledge/passages";
|
|
19
20
|
/**
|
|
20
21
|
* The knowledge port (rework-autopilot A8): one operation, find what is relevant to a question.
|
|
21
22
|
* Implementations (S3 + Unstructured today) are wired by the operator; their keys never live in
|
|
@@ -23,10 +24,17 @@
|
|
|
23
24
|
*/
|
|
24
25
|
export interface KnowledgePort {
|
|
25
26
|
find(question: string, signal: AbortSignal): Promise<string>;
|
|
27
|
+
/** The same lookup as passages, with where each came from; used instead of `find` when present. */
|
|
28
|
+
search?(question: string, signal: AbortSignal): Promise<KnowledgePassage[]>;
|
|
26
29
|
}
|
|
30
|
+
export type { KnowledgePassage };
|
|
31
|
+
export { passagesText };
|
|
32
|
+
/** A port's lookup as passages: its own `search`, or what `find` returned as one passage. */
|
|
33
|
+
export declare function searchKnowledge(port: KnowledgePort, question: string, signal: AbortSignal): Promise<KnowledgePassage[]>;
|
|
27
34
|
/** Knowledge lookups never hold a reply back longer than this. */
|
|
28
35
|
export declare const KNOWLEDGE_TIMEOUT_MS = 800;
|
|
29
36
|
/** Adapts the current knowledge bases (mods/autopilot/src/knowledge) to the port. */
|
|
30
37
|
export declare function fromKnowledgeBase(kb: {
|
|
31
38
|
queryKnowledgeBase(query: string, k?: number): Promise<string>;
|
|
39
|
+
searchKnowledgeBase?(query: string, k?: number): Promise<KnowledgePassage[]>;
|
|
32
40
|
}): KnowledgePort;
|
|
@@ -1,10 +1,43 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.KNOWLEDGE_TIMEOUT_MS = void 0;
|
|
3
|
+
exports.KNOWLEDGE_TIMEOUT_MS = exports.passagesText = void 0;
|
|
4
|
+
exports.searchKnowledge = searchKnowledge;
|
|
4
5
|
exports.fromKnowledgeBase = fromKnowledgeBase;
|
|
6
|
+
/**
|
|
7
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
8
|
+
* http://github.com/fonoster/fonoster
|
|
9
|
+
*
|
|
10
|
+
* This file is part of Fonoster
|
|
11
|
+
*
|
|
12
|
+
* Licensed under the MIT License (the "License");
|
|
13
|
+
* you may not use this file except in compliance with
|
|
14
|
+
* the License. You may obtain a copy of the License at
|
|
15
|
+
*
|
|
16
|
+
* https://opensource.org/licenses/MIT
|
|
17
|
+
*
|
|
18
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
19
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
20
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
21
|
+
* See the License for the specific language governing permissions and
|
|
22
|
+
* limitations under the License.
|
|
23
|
+
*/
|
|
24
|
+
const passages_1 = require("../knowledge/passages");
|
|
25
|
+
Object.defineProperty(exports, "passagesText", { enumerable: true, get: function () { return passages_1.passagesText; } });
|
|
26
|
+
/** A port's lookup as passages: its own `search`, or what `find` returned as one passage. */
|
|
27
|
+
async function searchKnowledge(port, question, signal) {
|
|
28
|
+
if (port.search)
|
|
29
|
+
return port.search(question, signal);
|
|
30
|
+
const text = (await port.find(question, signal))?.trim();
|
|
31
|
+
return text ? [{ text }] : [];
|
|
32
|
+
}
|
|
5
33
|
/** Knowledge lookups never hold a reply back longer than this. */
|
|
6
34
|
exports.KNOWLEDGE_TIMEOUT_MS = 800;
|
|
7
35
|
/** Adapts the current knowledge bases (mods/autopilot/src/knowledge) to the port. */
|
|
8
36
|
function fromKnowledgeBase(kb) {
|
|
9
|
-
return {
|
|
37
|
+
return {
|
|
38
|
+
find: (question) => kb.queryKnowledgeBase(question),
|
|
39
|
+
...(kb.searchKnowledgeBase
|
|
40
|
+
? { search: (question) => kb.searchKnowledgeBase(question) }
|
|
41
|
+
: {})
|
|
42
|
+
};
|
|
10
43
|
}
|
package/dist/session/llm.js
CHANGED
|
@@ -111,7 +111,9 @@ class RegistryLlmResolver {
|
|
|
111
111
|
return {
|
|
112
112
|
id,
|
|
113
113
|
keySource: credential?.source ?? "platform",
|
|
114
|
-
think: llm.create(
|
|
114
|
+
think: llm.create(
|
|
115
|
+
// The catalog provider (e.g. together) names a rejected key, not the adapter (openai).
|
|
116
|
+
{ baseUrl, apiKey, name: provider.id }, id.slice(providerId.length + 1), settings),
|
|
115
117
|
...(resolution.event
|
|
116
118
|
? {
|
|
117
119
|
event: {
|
|
@@ -23,6 +23,12 @@ export type UsageSink = {
|
|
|
23
23
|
usage(lines: Usage.UsageLine[]): void;
|
|
24
24
|
/** A completed request whose vendor reported no usage (add-usage-metering). */
|
|
25
25
|
missing(catalogId: string): void;
|
|
26
|
+
/** The knowledge base's embeddings key was rejected: knowledge is off for this call. */
|
|
27
|
+
keyRejected?(event: {
|
|
28
|
+
provider: string;
|
|
29
|
+
catalogId: string;
|
|
30
|
+
status: number;
|
|
31
|
+
}): void;
|
|
26
32
|
};
|
|
27
33
|
export type MeteredModel = {
|
|
28
34
|
/** The catalog id (`<provider>/<model>`). */
|
package/dist/session/tools.d.ts
CHANGED
|
@@ -4,6 +4,9 @@ export type ToolOutcome = {
|
|
|
4
4
|
ok: boolean;
|
|
5
5
|
result?: unknown;
|
|
6
6
|
error?: string;
|
|
7
|
+
/** HTTP tools: the method used and the response status (for the call record). */
|
|
8
|
+
method?: string;
|
|
9
|
+
status?: number;
|
|
7
10
|
};
|
|
8
11
|
/** The tool port (rework-autopilot A7): one operation, run a tool with inputs. */
|
|
9
12
|
export interface ToolPort {
|
package/dist/session/tools.js
CHANGED
|
@@ -36,16 +36,17 @@ class HttpToolPort {
|
|
|
36
36
|
if (!tool)
|
|
37
37
|
return { ok: false, error: `unknown tool ${name}` };
|
|
38
38
|
const method = String(tool.operation.method ?? "get").toUpperCase();
|
|
39
|
-
const url = new url_1.URL(tool.operation.url);
|
|
40
|
-
let body;
|
|
41
|
-
if (method === "GET") {
|
|
42
|
-
for (const [k, v] of Object.entries(args))
|
|
43
|
-
url.searchParams.set(k, String(v));
|
|
44
|
-
}
|
|
45
|
-
else {
|
|
46
|
-
body = JSON.stringify(args);
|
|
47
|
-
}
|
|
48
39
|
try {
|
|
40
|
+
// Inside the try: an invalid tool URL is a failed call, recorded like any other.
|
|
41
|
+
const url = new url_1.URL(tool.operation.url);
|
|
42
|
+
let body;
|
|
43
|
+
if (method === "GET") {
|
|
44
|
+
for (const [k, v] of Object.entries(args))
|
|
45
|
+
url.searchParams.set(k, String(v));
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
body = JSON.stringify(args);
|
|
49
|
+
}
|
|
49
50
|
const request = (0, common_1.safeFetch)(url, {
|
|
50
51
|
method,
|
|
51
52
|
headers: {
|
|
@@ -59,24 +60,27 @@ class HttpToolPort {
|
|
|
59
60
|
});
|
|
60
61
|
if (tool.operation.waitForResponse === false) {
|
|
61
62
|
request.catch(() => undefined);
|
|
62
|
-
return { ok: true, result: { status: "sent" } };
|
|
63
|
+
return { ok: true, result: { status: "sent" }, method };
|
|
63
64
|
}
|
|
64
65
|
const res = await request;
|
|
65
66
|
const text = await res.text();
|
|
67
|
+
const status = res.status;
|
|
66
68
|
if (!res.ok)
|
|
67
69
|
return {
|
|
68
70
|
ok: false,
|
|
69
|
-
error: `HTTP ${
|
|
71
|
+
error: `HTTP ${status}: ${text.slice(0, 200)}`,
|
|
72
|
+
method,
|
|
73
|
+
status
|
|
70
74
|
};
|
|
71
75
|
try {
|
|
72
|
-
return { ok: true, result: JSON.parse(text) };
|
|
76
|
+
return { ok: true, result: JSON.parse(text), method, status };
|
|
73
77
|
}
|
|
74
78
|
catch {
|
|
75
|
-
return { ok: true, result: text };
|
|
79
|
+
return { ok: true, result: text, method, status };
|
|
76
80
|
}
|
|
77
81
|
}
|
|
78
82
|
catch (e) {
|
|
79
|
-
return { ok: false, error: e.message };
|
|
83
|
+
return { ok: false, error: e.message, method };
|
|
80
84
|
}
|
|
81
85
|
}
|
|
82
86
|
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
3
|
+
* http://github.com/fonoster/fonoster
|
|
4
|
+
*
|
|
5
|
+
* This file is part of Fonoster
|
|
6
|
+
*
|
|
7
|
+
* Licensed under the MIT License (the "License");
|
|
8
|
+
* you may not use this file except in compliance with
|
|
9
|
+
* the License. You may obtain a copy of the License at
|
|
10
|
+
*
|
|
11
|
+
* https://opensource.org/licenses/MIT
|
|
12
|
+
*
|
|
13
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
* See the License for the specific language governing permissions and
|
|
17
|
+
* limitations under the License.
|
|
18
|
+
*/
|
|
19
|
+
import { KnowledgePassage } from "./knowledge";
|
|
20
|
+
import { ToolOutcome } from "./tools";
|
|
21
|
+
/** Tool parameters, responses and errors are cut at this many characters. */
|
|
22
|
+
export declare const MAX_TOOL_TEXT = 2048;
|
|
23
|
+
/** Knowledge queries and passages are cut at this many characters. */
|
|
24
|
+
export declare const MAX_KNOWLEDGE_TEXT = 500;
|
|
25
|
+
/** At most this many passages are kept per knowledge search. */
|
|
26
|
+
export declare const MAX_PASSAGES = 3;
|
|
27
|
+
/** One tool call of a turn. */
|
|
28
|
+
export type ToolCallTrace = {
|
|
29
|
+
name: string;
|
|
30
|
+
ok: boolean;
|
|
31
|
+
/** When the call started, in ms since the epoch (orders it within the turn). */
|
|
32
|
+
startedAtMs?: number;
|
|
33
|
+
/** HTTP tools: the method and the response status. */
|
|
34
|
+
method?: string;
|
|
35
|
+
status?: number;
|
|
36
|
+
durationMs?: number;
|
|
37
|
+
/** JSON, capped. */
|
|
38
|
+
parameters?: string;
|
|
39
|
+
response?: string;
|
|
40
|
+
error?: string;
|
|
41
|
+
};
|
|
42
|
+
/** One knowledge search of a turn: the passages found for the caller's words. */
|
|
43
|
+
export type KnowledgeSearchTrace = {
|
|
44
|
+
query: string;
|
|
45
|
+
/** At most MAX_PASSAGES, best first; the first one's `source` is the top match's document. */
|
|
46
|
+
passages: KnowledgePassage[];
|
|
47
|
+
/** When the search started, in ms since the epoch (orders it within the turn). */
|
|
48
|
+
startedAtMs?: number;
|
|
49
|
+
durationMs: number;
|
|
50
|
+
/** The search took longer than a reply waits for; the reply went without it. */
|
|
51
|
+
timedOut?: boolean;
|
|
52
|
+
/** The search failed; the reply went without it. `error` is the message, short. */
|
|
53
|
+
failed?: boolean;
|
|
54
|
+
error?: string;
|
|
55
|
+
};
|
|
56
|
+
/** A failed knowledge search's error is cut at this many characters. */
|
|
57
|
+
export declare const MAX_KNOWLEDGE_ERROR = 200;
|
|
58
|
+
/**
|
|
59
|
+
* Caps text at `max` characters, ending with an ellipsis. A run of digits or digit words cut at
|
|
60
|
+
* the end is dropped when, with what follows the cut, it is long enough to be part of a card
|
|
61
|
+
* number (the redaction only recognizes whole ones); amounts and ordinary words stay.
|
|
62
|
+
*/
|
|
63
|
+
export declare function capText(text: string, max: number): string;
|
|
64
|
+
/** The trace entry for a tool the model called. */
|
|
65
|
+
export declare function toolCallTrace(name: string, args: Record<string, unknown>, outcome: ToolOutcome, startedAtMs: number, durationMs: number): ToolCallTrace;
|
|
66
|
+
/** The trace entry for a knowledge search: what it found, or that it timed out or failed. */
|
|
67
|
+
export declare function knowledgeSearchTrace(query: string, found: {
|
|
68
|
+
passages: KnowledgePassage[];
|
|
69
|
+
} | {
|
|
70
|
+
timedOut: true;
|
|
71
|
+
} | {
|
|
72
|
+
error: unknown;
|
|
73
|
+
}, startedAtMs: number, durationMs: number): KnowledgeSearchTrace;
|
|
74
|
+
/**
|
|
75
|
+
* The turn as the log sees it when there is no call record: tool names, outcomes, counts and
|
|
76
|
+
* durations only. Parameters, responses, errors, queries and passages stay out of the logs.
|
|
77
|
+
*/
|
|
78
|
+
export declare function loggedTurn<T extends {
|
|
79
|
+
toolCalls: ToolCallTrace[];
|
|
80
|
+
knowledgeSearches?: KnowledgeSearchTrace[];
|
|
81
|
+
}>(trace: T): T & {
|
|
82
|
+
knowledgeSearches?: {
|
|
83
|
+
failed?: boolean;
|
|
84
|
+
timedOut?: boolean;
|
|
85
|
+
passages: number;
|
|
86
|
+
durationMs: number;
|
|
87
|
+
}[];
|
|
88
|
+
toolCalls: {
|
|
89
|
+
durationMs?: number;
|
|
90
|
+
status?: number;
|
|
91
|
+
name: string;
|
|
92
|
+
ok: boolean;
|
|
93
|
+
}[];
|
|
94
|
+
};
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MAX_KNOWLEDGE_ERROR = exports.MAX_PASSAGES = exports.MAX_KNOWLEDGE_TEXT = exports.MAX_TOOL_TEXT = void 0;
|
|
4
|
+
exports.capText = capText;
|
|
5
|
+
exports.toolCallTrace = toolCallTrace;
|
|
6
|
+
exports.knowledgeSearchTrace = knowledgeSearchTrace;
|
|
7
|
+
exports.loggedTurn = loggedTurn;
|
|
8
|
+
/** Tool parameters, responses and errors are cut at this many characters. */
|
|
9
|
+
exports.MAX_TOOL_TEXT = 2048;
|
|
10
|
+
/** Knowledge queries and passages are cut at this many characters. */
|
|
11
|
+
exports.MAX_KNOWLEDGE_TEXT = 500;
|
|
12
|
+
/** At most this many passages are kept per knowledge search. */
|
|
13
|
+
exports.MAX_PASSAGES = 3;
|
|
14
|
+
/** A failed knowledge search's error is cut at this many characters. */
|
|
15
|
+
exports.MAX_KNOWLEDGE_ERROR = 200;
|
|
16
|
+
/** A trailing run of at least this many digits could be part of a card number. */
|
|
17
|
+
const CARD_DIGITS = 13;
|
|
18
|
+
const DIGIT_WORDS = "zero|oh|one|two|three|four|five|six|seven|eight|nine|cero|uno|dos|tres|cuatro|cinco|seis|siete|ocho|nueve";
|
|
19
|
+
const DIGIT = `(?:\\d|\\b(?:${DIGIT_WORDS})\\b)`;
|
|
20
|
+
// A run of digits or digit words at the end of the text, and at the start, with their separators.
|
|
21
|
+
const TRAILING_NUMBER = new RegExp(`(?:[\\s,.-]*${DIGIT})+[\\s,.-]*$`, "i");
|
|
22
|
+
const LEADING_NUMBER = new RegExp(`^(?:[\\s,.-]*${DIGIT})+`, "i");
|
|
23
|
+
const DIGITS = new RegExp(DIGIT, "gi");
|
|
24
|
+
const digitCount = (text) => text.match(DIGITS)?.length ?? 0;
|
|
25
|
+
/**
|
|
26
|
+
* Caps text at `max` characters, ending with an ellipsis. A run of digits or digit words cut at
|
|
27
|
+
* the end is dropped when, with what follows the cut, it is long enough to be part of a card
|
|
28
|
+
* number (the redaction only recognizes whole ones); amounts and ordinary words stay.
|
|
29
|
+
*/
|
|
30
|
+
function capText(text, max) {
|
|
31
|
+
if (text.length <= max)
|
|
32
|
+
return text;
|
|
33
|
+
const head = text.slice(0, max);
|
|
34
|
+
const trailing = head.match(TRAILING_NUMBER)?.[0] ?? "";
|
|
35
|
+
const continued = text.slice(max).match(LEADING_NUMBER)?.[0] ?? "";
|
|
36
|
+
const run = digitCount(trailing) + digitCount(continued);
|
|
37
|
+
const kept = trailing && run >= CARD_DIGITS ? head.slice(0, -trailing.length) : head;
|
|
38
|
+
return `${kept}…`;
|
|
39
|
+
}
|
|
40
|
+
function json(value) {
|
|
41
|
+
if (typeof value === "string")
|
|
42
|
+
return value;
|
|
43
|
+
try {
|
|
44
|
+
return JSON.stringify(value) ?? "";
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return String(value);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/** The trace entry for a tool the model called. */
|
|
51
|
+
function toolCallTrace(name, args, outcome, startedAtMs, durationMs) {
|
|
52
|
+
return {
|
|
53
|
+
name,
|
|
54
|
+
ok: outcome.ok,
|
|
55
|
+
startedAtMs,
|
|
56
|
+
...(outcome.method ? { method: outcome.method } : {}),
|
|
57
|
+
...(outcome.status !== undefined ? { status: outcome.status } : {}),
|
|
58
|
+
durationMs,
|
|
59
|
+
...(Object.keys(args).length
|
|
60
|
+
? { parameters: capText(json(args), exports.MAX_TOOL_TEXT) }
|
|
61
|
+
: {}),
|
|
62
|
+
...(outcome.ok && outcome.result !== undefined
|
|
63
|
+
? { response: capText(json(outcome.result), exports.MAX_TOOL_TEXT) }
|
|
64
|
+
: {}),
|
|
65
|
+
...(!outcome.ok && outcome.error
|
|
66
|
+
? { error: capText(outcome.error, exports.MAX_TOOL_TEXT) }
|
|
67
|
+
: {})
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
/** The trace entry for a knowledge search: what it found, or that it timed out or failed. */
|
|
71
|
+
function knowledgeSearchTrace(query, found, startedAtMs, durationMs) {
|
|
72
|
+
const passages = "passages" in found ? found.passages : [];
|
|
73
|
+
const error = "error" in found
|
|
74
|
+
? capText(found.error?.message ?? String(found.error), exports.MAX_KNOWLEDGE_ERROR)
|
|
75
|
+
: undefined;
|
|
76
|
+
return {
|
|
77
|
+
query: capText(query, exports.MAX_KNOWLEDGE_TEXT),
|
|
78
|
+
passages: passages.slice(0, exports.MAX_PASSAGES).map((p) => ({
|
|
79
|
+
text: capText(p.text, exports.MAX_KNOWLEDGE_TEXT),
|
|
80
|
+
...(p.source ? { source: p.source } : {}),
|
|
81
|
+
...(p.score !== undefined ? { score: p.score } : {})
|
|
82
|
+
})),
|
|
83
|
+
startedAtMs,
|
|
84
|
+
durationMs,
|
|
85
|
+
...("timedOut" in found ? { timedOut: true } : {}),
|
|
86
|
+
...("error" in found ? { failed: true, ...(error ? { error } : {}) } : {})
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* The turn as the log sees it when there is no call record: tool names, outcomes, counts and
|
|
91
|
+
* durations only. Parameters, responses, errors, queries and passages stay out of the logs.
|
|
92
|
+
*/
|
|
93
|
+
function loggedTurn(trace) {
|
|
94
|
+
return {
|
|
95
|
+
...trace,
|
|
96
|
+
toolCalls: trace.toolCalls.map((c) => ({
|
|
97
|
+
name: c.name,
|
|
98
|
+
ok: c.ok,
|
|
99
|
+
...(c.status !== undefined ? { status: c.status } : {}),
|
|
100
|
+
...(c.durationMs !== undefined ? { durationMs: c.durationMs } : {})
|
|
101
|
+
})),
|
|
102
|
+
...(trace.knowledgeSearches
|
|
103
|
+
? {
|
|
104
|
+
knowledgeSearches: trace.knowledgeSearches.map((k) => ({
|
|
105
|
+
passages: k.passages.length,
|
|
106
|
+
durationMs: k.durationMs,
|
|
107
|
+
...(k.timedOut ? { timedOut: true } : {}),
|
|
108
|
+
...(k.failed ? { failed: true } : {})
|
|
109
|
+
}))
|
|
110
|
+
}
|
|
111
|
+
: {})
|
|
112
|
+
};
|
|
113
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fonoster/autopilot",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.34.1",
|
|
4
4
|
"description": "Voice AI for the Fonoster platform",
|
|
5
5
|
"author": "Pedro Sanders <psanders@fonoster.com>",
|
|
6
6
|
"homepage": "https://github.com/fonoster/fonoster#readme",
|
|
@@ -33,11 +33,11 @@
|
|
|
33
33
|
"dependencies": {
|
|
34
34
|
"@aws-sdk/client-s3": "^3.958.0",
|
|
35
35
|
"@dmitryrechkin/json-schema-to-zod": "^1.0.1",
|
|
36
|
-
"@fonoster/common": "^0.
|
|
37
|
-
"@fonoster/community": "^0.
|
|
38
|
-
"@fonoster/logger": "^0.
|
|
39
|
-
"@fonoster/types": "^0.
|
|
40
|
-
"@fonoster/voice": "^0.
|
|
36
|
+
"@fonoster/common": "^0.34.1",
|
|
37
|
+
"@fonoster/community": "^0.34.1",
|
|
38
|
+
"@fonoster/logger": "^0.34.1",
|
|
39
|
+
"@fonoster/types": "^0.34.1",
|
|
40
|
+
"@fonoster/voice": "^0.34.1",
|
|
41
41
|
"@grpc/grpc-js": "~1.14.6",
|
|
42
42
|
"@langchain/anthropic": "^1.3.3",
|
|
43
43
|
"@langchain/community": "^1.1.1",
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
"xstate": "^5.17.3",
|
|
56
56
|
"zod": "^3.25.76"
|
|
57
57
|
},
|
|
58
|
-
"gitHead": "
|
|
58
|
+
"gitHead": "f521d836e94a24a5b17a25fba3e03e680637016d",
|
|
59
59
|
"devDependencies": {
|
|
60
60
|
"yaml": "^2.9.0"
|
|
61
61
|
}
|