@fonoster/autopilot 0.32.0 → 0.34.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  import { Embeddings } from "@langchain/core/embeddings";
2
2
  import { VectorStore } from "@langchain/core/vectorstores";
3
+ import { KnowledgePassage } from "./passages";
3
4
  import { KnowledgeBase } from "./types";
4
5
  declare abstract class AbstractKnowledgeBase implements KnowledgeBase {
5
6
  protected embeddings: Embeddings;
@@ -11,5 +12,7 @@ declare abstract class AbstractKnowledgeBase implements KnowledgeBase {
11
12
  abstract getLoaders(): Promise<unknown>;
12
13
  load(): Promise<void>;
13
14
  queryKnowledgeBase(query: string, k?: number): Promise<string>;
15
+ /** The best `k` passages for the query, with their document and similarity score. */
16
+ searchKnowledgeBase(query: string, k?: number): Promise<KnowledgePassage[]>;
14
17
  }
15
18
  export { AbstractKnowledgeBase };
@@ -19,9 +19,11 @@ exports.AbstractKnowledgeBase = void 0;
19
19
  * See the License for the specific language governing permissions and
20
20
  * limitations under the License.
21
21
  */
22
+ const node_path_1 = require("node:path");
22
23
  const logger_1 = require("@fonoster/logger");
23
24
  const memory_1 = require("@langchain/classic/vectorstores/memory");
24
25
  const textsplitters_1 = require("@langchain/textsplitters");
26
+ const passages_1 = require("./passages");
25
27
  const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
26
28
  class AbstractKnowledgeBase {
27
29
  /** The embeddings come from the caller (the hosted Autopilot resolves their key by credentialRef). */
@@ -44,13 +46,24 @@ class AbstractKnowledgeBase {
44
46
  this.vectorStore = await memory_1.MemoryVectorStore.fromDocuments(splitDocs.flat(), this.embeddings);
45
47
  }
46
48
  async queryKnowledgeBase(query, k = 2) {
49
+ return (0, passages_1.passagesText)(await this.searchKnowledgeBase(query, k));
50
+ }
51
+ /** The best `k` passages for the query, with their document and similarity score. */
52
+ async searchKnowledgeBase(query, k = 2) {
47
53
  const { vectorStore } = this;
48
54
  if (!vectorStore) {
49
- logger.verbose("vector store is not initialized, returning empty string");
50
- return "";
55
+ logger.verbose("vector store is not initialized, returning nothing");
56
+ return [];
51
57
  }
52
- const results = await vectorStore.similaritySearch(query, k);
53
- return results.join("\n");
58
+ const results = await vectorStore.similaritySearchWithScore(query, k);
59
+ return results.map(([doc, score]) => {
60
+ const source = doc.metadata?.filename ?? doc.metadata?.source;
61
+ return {
62
+ text: doc.pageContent,
63
+ ...(source ? { source: (0, node_path_1.basename)(String(source)) } : {}),
64
+ score
65
+ };
66
+ });
54
67
  }
55
68
  }
56
69
  exports.AbstractKnowledgeBase = AbstractKnowledgeBase;
@@ -21,6 +21,12 @@ import { Usage } from "@fonoster/common";
21
21
  export type EmbeddingUsageSink = {
22
22
  usage(lines: Usage.UsageLine[]): void;
23
23
  missing(catalogId: string): void;
24
+ /** The vendor rejected the embeddings key (HTTP 401/403): reported once per instance. */
25
+ keyRejected?(event: {
26
+ provider: string;
27
+ catalogId: string;
28
+ status: number;
29
+ }): void;
24
30
  };
25
31
  /**
26
32
  * OpenAI embeddings that report their token usage (add-usage-metering): the knowledge base's
@@ -32,6 +38,11 @@ export declare class MeteredEmbeddings {
32
38
  private readonly options;
33
39
  private readonly model;
34
40
  private readonly baseUrl;
41
+ /**
42
+ * The status a rejected key was answered with. Once set, no more requests go out: the key
43
+ * won't start working mid-call, and knowledge is optional, so the call goes on without it.
44
+ */
45
+ private rejected?;
35
46
  constructor(options: {
36
47
  apiKey: string;
37
48
  keySource: Usage.KeySource;
@@ -44,4 +55,5 @@ export declare class MeteredEmbeddings {
44
55
  embedDocuments(texts: string[]): Promise<number[][]>;
45
56
  embedQuery(text: string): Promise<number[]>;
46
57
  private embed;
58
+ private rejectedError;
47
59
  }
@@ -26,6 +26,8 @@ class MeteredEmbeddings {
26
26
  return (await this.embed([text]))[0];
27
27
  }
28
28
  async embed(input) {
29
+ if (this.rejected)
30
+ throw this.rejectedError(this.rejected);
29
31
  const res = await fetch(`${this.baseUrl}/v1/embeddings`, {
30
32
  method: "POST",
31
33
  headers: {
@@ -34,8 +36,22 @@ class MeteredEmbeddings {
34
36
  },
35
37
  body: JSON.stringify({ model: this.model, input })
36
38
  });
37
- if (!res.ok)
39
+ if (res.status === 401 || res.status === 403) {
40
+ await res.body?.cancel().catch(() => undefined);
41
+ if (!this.rejected) {
42
+ this.rejected = res.status;
43
+ this.options.sink?.keyRejected?.({
44
+ provider: "openai",
45
+ catalogId: this.catalogId,
46
+ status: res.status
47
+ });
48
+ }
49
+ throw this.rejectedError(res.status);
50
+ }
51
+ if (!res.ok) {
52
+ await res.body?.cancel().catch(() => undefined);
38
53
  throw new Error(`embeddings answered HTTP ${res.status}`);
54
+ }
39
55
  const body = (await res.json());
40
56
  const tokens = body.usage?.prompt_tokens;
41
57
  if (tokens === undefined)
@@ -54,5 +70,8 @@ class MeteredEmbeddings {
54
70
  .sort((a, b) => a.index - b.index)
55
71
  .map((d) => d.embedding);
56
72
  }
73
+ rejectedError(status) {
74
+ return new Error(`embeddings rejected the credentials (HTTP ${status})`);
75
+ }
57
76
  }
58
77
  exports.MeteredEmbeddings = MeteredEmbeddings;
@@ -18,5 +18,6 @@
18
18
  */
19
19
  export * from "./AbstractKnowledgeBase";
20
20
  export * from "./FilesKnowledgeBase";
21
+ export * from "./passages";
21
22
  export * from "./S3KnowledgeBase";
22
23
  export * from "./types";
@@ -34,5 +34,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
34
34
  */
35
35
  __exportStar(require("./AbstractKnowledgeBase"), exports);
36
36
  __exportStar(require("./FilesKnowledgeBase"), exports);
37
+ __exportStar(require("./passages"), exports);
37
38
  __exportStar(require("./S3KnowledgeBase"), exports);
38
39
  __exportStar(require("./types"), exports);
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
3
+ * http://github.com/fonoster/fonoster
4
+ *
5
+ * This file is part of Fonoster
6
+ *
7
+ * Licensed under the MIT License (the "License");
8
+ * you may not use this file except in compliance with
9
+ * the License. You may obtain a copy of the License at
10
+ *
11
+ * https://opensource.org/licenses/MIT
12
+ *
13
+ * Unless required by applicable law or agreed to in writing, software
14
+ * distributed under the License is distributed on an "AS IS" BASIS,
15
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ * See the License for the specific language governing permissions and
17
+ * limitations under the License.
18
+ */
19
+ /** One passage a knowledge search found: its text, the document it came from and its score. */
20
+ export type KnowledgePassage = {
21
+ text: string;
22
+ source?: string;
23
+ score?: number;
24
+ };
25
+ /**
26
+ * Each passage is cut at this many characters before it goes into the model's prompt: the
27
+ * splitter's chunk size, so a normal passage goes in whole and a loader that skips splitting
28
+ * can't flood the prompt.
29
+ */
30
+ export declare const MAX_PROMPT_PASSAGE_TEXT = 1000;
31
+ /** The passages as the model sees them: their text, each capped, one per line. */
32
+ export declare function passagesText(passages: KnowledgePassage[]): string;
@@ -0,0 +1,18 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.MAX_PROMPT_PASSAGE_TEXT = void 0;
4
+ exports.passagesText = passagesText;
5
+ /**
6
+ * Each passage is cut at this many characters before it goes into the model's prompt: the
7
+ * splitter's chunk size, so a normal passage goes in whole and a loader that skips splitting
8
+ * can't flood the prompt.
9
+ */
10
+ exports.MAX_PROMPT_PASSAGE_TEXT = 1000;
11
+ /** The passages as the model sees them: their text, each capped, one per line. */
12
+ function passagesText(passages) {
13
+ return passages
14
+ .map((p) => p.text.length > exports.MAX_PROMPT_PASSAGE_TEXT
15
+ ? `${p.text.slice(0, exports.MAX_PROMPT_PASSAGE_TEXT)}…`
16
+ : p.text)
17
+ .join("\n");
18
+ }
@@ -17,9 +17,11 @@
17
17
  * limitations under the License.
18
18
  */
19
19
  import { Embeddings } from "@langchain/core/embeddings";
20
+ import { KnowledgePassage } from "./passages";
20
21
  type KnowledgeBase = {
21
22
  load: () => Promise<void>;
22
23
  queryKnowledgeBase: (query: string, k?: number) => Promise<string>;
24
+ searchKnowledgeBase?: (query: string, k?: number) => Promise<KnowledgePassage[]>;
23
25
  };
24
26
  type S3KnowledgeBaseParams = {
25
27
  embeddings: Embeddings;
@@ -21,6 +21,7 @@ import { SessionAgent } from "./config";
21
21
  import { KnowledgePort } from "./knowledge";
22
22
  import { ChatMessage, Think } from "./think";
23
23
  import { ToolPort } from "./tools";
24
+ import { KnowledgeSearchTrace, ToolCallTrace } from "./traceDetails";
24
25
  import type { Session } from "@fonoster/voice";
25
26
  /** What Autopilot says when a transfer fails and the agent sets no `failedMessage`. */
26
27
  export declare const DEFAULT_TRANSFER_FAILED_MESSAGE = "Sorry, I couldn't connect you. How else can I help?";
@@ -61,13 +62,23 @@ export type TurnTrace = {
61
62
  replyText: string;
62
63
  heardText: string;
63
64
  interrupted: boolean;
64
- toolCalls: {
65
- name: string;
66
- ok: boolean;
67
- }[];
65
+ toolCalls: ToolCallTrace[];
68
66
  /** ms from turn.ended to the first token, and to the first speak sent. */
69
67
  firstTokenMs?: number;
70
68
  firstSpeakMs?: number;
69
+ /**
70
+ * Where the caller's wait went, in ms, when the turn's first sentence was also its first audio
71
+ * and the reply played to the end: `llmMs` from turn.ended to that sentence handed to speech
72
+ * (knowledge lookups and tool calls before the first words count here), `ttsMs` from that
73
+ * sentence to its own first audio (output.started). Absent when the turn was interrupted, or
74
+ * when other speech (a tool's start message) went out before the reply.
75
+ */
76
+ llmMs?: number;
77
+ ttsMs?: number;
78
+ /** Set when speech other than the reply (a tool's start message) went out during the turn. */
79
+ spokeAhead?: boolean;
80
+ /** What the knowledge port found for the caller's words (agents with knowledge only). */
81
+ knowledgeSearches?: KnowledgeSearchTrace[];
71
82
  };
72
83
  export interface Recorder {
73
84
  turn(trace: TurnTrace): void;
@@ -99,6 +110,8 @@ export declare class Agent {
99
110
  /** Whether a reply is prepared on turn.likely_ended (off at high patience or once adapted). */
100
111
  private prepareEarly;
101
112
  private stopped;
113
+ /** A vendor rejected the model's key and the call is failing (#1146). */
114
+ private keyFailed;
102
115
  send: (event: AgentEvent) => void;
103
116
  constructor(session: Session, agent: SessionAgent, think: Think, tools: ToolPort, recorder?: Recorder, knowledge?: KnowledgePort);
104
117
  get settings(): {
@@ -132,6 +145,8 @@ export declare class Agent {
132
145
  /** Greeting: subscribe to the estimate, send initial DTMF, speak the first message. */
133
146
  greet(): Promise<void>;
134
147
  stop(): void;
148
+ /** Nothing more is said or prepared: the reply, the prepared reply and the timers stop. */
149
+ private endTurns;
135
150
  armIdle(): void;
136
151
  disarmIdle(): void;
137
152
  /** The next idle timeout ends the call (the goodbye counts as the last announcement). */
@@ -150,6 +165,11 @@ export declare class Agent {
150
165
  private respondTurn;
151
166
  private speakRound;
152
167
  private runTools;
168
+ /**
169
+ * The model's vendor rejected the key: retrying can't help, so the call fails and hangs up at
170
+ * once, without an error message (#1146).
171
+ */
172
+ private keyRejected;
153
173
  goodbye(): Promise<void>;
154
174
  transfer(): Promise<void>;
155
175
  /**
@@ -21,12 +21,14 @@ exports.sessionRecorder = sessionRecorder;
21
21
  * limitations under the License.
22
22
  */
23
23
  const common_1 = require("@fonoster/common");
24
+ const community_1 = require("@fonoster/community");
24
25
  const logger_1 = require("@fonoster/logger");
25
26
  const voice_1 = require("@fonoster/voice");
26
27
  const AsyncQueue_1 = require("./AsyncQueue");
27
28
  const knowledge_1 = require("./knowledge");
28
29
  const sentences_1 = require("./sentences");
29
30
  const telemetry_1 = require("./telemetry");
31
+ const traceDetails_1 = require("./traceDetails");
30
32
  /** What Autopilot says when a transfer fails and the agent sets no `failedMessage`. */
31
33
  exports.DEFAULT_TRANSFER_FAILED_MESSAGE = "Sorry, I couldn't connect you. How else can I help?";
32
34
  const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
@@ -43,6 +45,16 @@ const BUILT_IN_TOOLS = [
43
45
  parameters: { type: "object", properties: {} }
44
46
  }
45
47
  ];
48
+ /** The note for a finished turn: an interrupted reply carries no stages. */
49
+ function turnNote(trace) {
50
+ const note = { ...trace };
51
+ delete note.spokeAhead;
52
+ if (note.interrupted) {
53
+ delete note.llmMs;
54
+ delete note.ttsMs;
55
+ }
56
+ return note;
57
+ }
46
58
  /** Writes turn traces and notes to the call record through the session (record.note). */
47
59
  function sessionRecorder(session) {
48
60
  const write = (data) => void session.note(data).catch(() => undefined);
@@ -53,7 +65,8 @@ function sessionRecorder(session) {
53
65
  };
54
66
  }
55
67
  exports.logRecorder = {
56
- turn: (trace) => logger.verbose("autopilot turn", trace),
68
+ // Without a call record, tool payloads and knowledge queries stay out of the logs.
69
+ turn: (trace) => logger.verbose("autopilot turn", (0, traceDetails_1.loggedTurn)(trace)),
57
70
  note: (event) => logger.verbose("autopilot note", event)
58
71
  };
59
72
  /** The side effects of Autopilot: everything the state machine asks it to do on the session. */
@@ -69,6 +82,8 @@ class Agent {
69
82
  this.idleCount = 0;
70
83
  this.resumes = 0;
71
84
  this.stopped = false;
85
+ /** A vendor rejected the model's key and the call is failing (#1146). */
86
+ this.keyFailed = false;
72
87
  this.send = () => undefined;
73
88
  this.prepareEarly = agent.conversationSettings.patience !== "high";
74
89
  }
@@ -104,6 +119,10 @@ class Agent {
104
119
  }
105
120
  stop() {
106
121
  this.stopped = true;
122
+ this.endTurns();
123
+ }
124
+ /** Nothing more is said or prepared: the reply, the prepared reply and the timers stop. */
125
+ endTurns() {
107
126
  this.abortReply();
108
127
  this.discardPrepared();
109
128
  clearTimeout(this.idleTimer);
@@ -139,10 +158,16 @@ class Agent {
139
158
  this.discardPrepared();
140
159
  const abort = new AbortController();
141
160
  const chunks = new AsyncQueue_1.AsyncQueue();
142
- const prepared = { text, abort, chunks, context: [] };
161
+ const prepared = {
162
+ text,
163
+ abort,
164
+ chunks,
165
+ context: [],
166
+ searches: []
167
+ };
143
168
  this.prepared = prepared;
144
169
  void (async () => {
145
- prepared.context = await this.contextFor(text, abort.signal);
170
+ prepared.context = await this.contextFor(text, abort.signal, prepared.searches);
146
171
  if (abort.signal.aborted)
147
172
  return chunks.end();
148
173
  const messages = [
@@ -227,9 +252,12 @@ class Agent {
227
252
  }
228
253
  try {
229
254
  const prepared = this.takePrepared(text);
255
+ const searches = prepared?.searches ?? [];
256
+ if (this.knowledge)
257
+ trace.knowledgeSearches = searches;
230
258
  const context = prepared
231
259
  ? prepared.context
232
- : await this.contextFor(text, abort.signal);
260
+ : await this.contextFor(text, abort.signal, searches);
233
261
  let source = prepared?.chunks ?? this.startStream(abort.signal, context);
234
262
  for (let round = 0; round < MAX_TOOL_ROUNDS; round++) {
235
263
  const outcome = await this.speakRound(source, abort, preempt, turnEnded, trace);
@@ -255,10 +283,13 @@ class Agent {
255
283
  if (builtIn) {
256
284
  if (builtIn.name === "transfer")
257
285
  this.transferCallId = builtIn.id;
258
- this.recorder.turn({
286
+ this.recorder.turn(turnNote({
259
287
  ...trace,
260
- toolCalls: [...trace.toolCalls, { name: builtIn.name, ok: true }]
261
- });
288
+ toolCalls: [
289
+ ...trace.toolCalls,
290
+ { name: builtIn.name, ok: true, startedAtMs: Date.now() }
291
+ ]
292
+ }));
262
293
  this.send({
263
294
  type: builtIn.name === "hangup"
264
295
  ? "HANGUP_REQUESTED"
@@ -271,7 +302,7 @@ class Agent {
271
302
  return;
272
303
  source = this.startStream(abort.signal, context);
273
304
  }
274
- this.recorder.turn(trace);
305
+ this.recorder.turn(turnNote(trace));
275
306
  if (this.reply === abort)
276
307
  this.send({ type: "SPOKEN", source: "reply" });
277
308
  }
@@ -280,6 +311,8 @@ class Agent {
280
311
  e instanceof voice_1.SessionEndedError ||
281
312
  this.stopped)
282
313
  return;
314
+ if ((0, community_1.isProviderAuthError)(e))
315
+ return this.keyRejected(e);
283
316
  logger.error(`autopilot reply failed: ${e.message}`);
284
317
  await this.safely(async () => {
285
318
  await this.session.say(this.settings.systemErrorMessage, this.speechOptions());
@@ -318,7 +351,11 @@ class Agent {
318
351
  let completed = false;
319
352
  if (!first.done && !abort.signal.aborted) {
320
353
  this.send({ type: "REPLY_SPEAKING" });
321
- trace.firstSpeakMs = Date.now() - turnEnded;
354
+ const spokeAt = Date.now();
355
+ // Stages are measured on the turn's first sentence only, and only when nothing else was
356
+ // queued ahead of it, so its first audio is the first audio the caller heard.
357
+ const measure = trace.firstSpeakMs === undefined && !trace.spokeAhead;
358
+ trace.firstSpeakMs ?? (trace.firstSpeakMs = spokeAt - turnEnded);
322
359
  const pieces = (async function* () {
323
360
  yield first.value;
324
361
  for (;;) {
@@ -332,6 +369,11 @@ class Agent {
332
369
  ...this.speechOptions(),
333
370
  preempt: preempt || undefined
334
371
  });
372
+ // startedAt is this say's own output.started, on this clock.
373
+ if (measure && !result.interrupted && result.startedAt !== undefined) {
374
+ trace.llmMs = spokeAt - turnEnded;
375
+ trace.ttsMs = Math.max(0, result.startedAt - spokeAt);
376
+ }
335
377
  heard = result.heardText;
336
378
  interrupted = result.interrupted;
337
379
  completed = !interrupted;
@@ -349,6 +391,7 @@ class Agent {
349
391
  const tool = this.agent.languageModel.tools.find((t) => t.name === call.name);
350
392
  // The start message is spoken while the tool runs; AppBridge queues the reply after it.
351
393
  if (tool?.requestStartMessage) {
394
+ trace.spokeAhead = true;
352
395
  void this.session
353
396
  .say(tool.requestStartMessage, this.speechOptions())
354
397
  .catch(() => undefined);
@@ -360,8 +403,9 @@ class Agent {
360
403
  catch {
361
404
  args = {};
362
405
  }
406
+ const started = Date.now();
363
407
  const outcome = await this.tools.run(call.name, args, abort.signal);
364
- trace.toolCalls.push({ name: call.name, ok: outcome.ok });
408
+ trace.toolCalls.push((0, traceDetails_1.toolCallTrace)(call.name, args, outcome, started, Date.now() - started));
365
409
  this.history.push({
366
410
  role: "tool",
367
411
  tool_call_id: call.id,
@@ -370,6 +414,20 @@ class Agent {
370
414
  }
371
415
  }
372
416
  // ---- endings ----------------------------------------------------------------------
417
+ /**
418
+ * The model's vendor rejected the key: retrying can't help, so the call fails and hangs up at
419
+ * once, without an error message (#1146).
420
+ */
421
+ async keyRejected(e) {
422
+ if (this.keyFailed || this.stopped)
423
+ return;
424
+ this.keyFailed = true;
425
+ logger.error(`autopilot cannot use its language model: ${e.message}`);
426
+ this.endTurns();
427
+ await this.safely(() => this.session.hangup({
428
+ failure: { reason: e.reason, provider: e.provider, kind: e.kind }
429
+ }));
430
+ }
373
431
  async goodbye() {
374
432
  this.abortReply();
375
433
  this.discardPrepared();
@@ -432,27 +490,28 @@ class Agent {
432
490
  ];
433
491
  }
434
492
  /** Knowledge for this turn, as a system message for the request only (never in history). */
435
- async contextFor(question, signal) {
436
- if (!this.knowledge)
493
+ async contextFor(question, signal, searches) {
494
+ const knowledge = this.knowledge;
495
+ if (!knowledge)
437
496
  return [];
438
- const timeout = new Promise((resolve) => setTimeout(() => resolve(""), knowledge_1.KNOWLEDGE_TIMEOUT_MS));
497
+ const timeout = new Promise((resolve) => setTimeout(() => resolve(undefined), knowledge_1.KNOWLEDGE_TIMEOUT_MS));
498
+ const started = Date.now();
499
+ let passages;
439
500
  try {
440
- const found = await Promise.race([
441
- this.knowledge.find(question, signal),
501
+ passages = await Promise.race([
502
+ (0, knowledge_1.searchKnowledge)(knowledge, question, signal),
442
503
  timeout
443
504
  ]);
444
- return found?.trim()
445
- ? [
446
- {
447
- role: "system",
448
- content: `Relevant information:\n${found.trim()}`
449
- }
450
- ]
451
- : [];
452
505
  }
453
- catch {
506
+ catch (error) {
507
+ searches?.push((0, traceDetails_1.knowledgeSearchTrace)(question, { error }, started, Date.now() - started));
454
508
  return [];
455
509
  }
510
+ searches?.push((0, traceDetails_1.knowledgeSearchTrace)(question, passages ? { passages } : { timedOut: true }, started, Date.now() - started));
511
+ const found = (0, knowledge_1.passagesText)(passages ?? []).trim();
512
+ return found
513
+ ? [{ role: "system", content: `Relevant information:\n${found}` }]
514
+ : [];
456
515
  }
457
516
  toolSpecs() {
458
517
  const own = this.agent.languageModel.tools.map((t) => ({
@@ -515,6 +574,9 @@ class Agent {
515
574
  this.prepared = undefined;
516
575
  }
517
576
  async pump(stream, into) {
577
+ // The key was rejected and the call is failing: the model is not asked again.
578
+ if (this.keyFailed)
579
+ return into.end();
518
580
  try {
519
581
  for await (const chunk of stream)
520
582
  into.push(chunk);
@@ -522,6 +584,10 @@ class Agent {
522
584
  }
523
585
  catch (e) {
524
586
  into.fail(e);
587
+ // Fatal at once, even for a reply prepared early that is never spoken: a reply discarded
588
+ // after its key was rejected must not lead to a second request (#1146).
589
+ if ((0, community_1.isProviderAuthError)(e))
590
+ void this.keyRejected(e);
525
591
  }
526
592
  }
527
593
  speechOptions() {
@@ -103,6 +103,13 @@ function createSessionRunner(options) {
103
103
  missing: (catalogId) => {
104
104
  telemetry_1.telemetry.count("fonoster.usage.missing", 1, { catalog_id: catalogId });
105
105
  recorder.note({ type: "usage_missing", catalogId });
106
+ },
107
+ // Knowledge is optional: the call goes on without it, and the record says why (#1146).
108
+ keyRejected: (event) => {
109
+ telemetry_1.telemetry.count("fonoster.autopilot.knowledge.key_rejected", 1, {
110
+ provider: event.provider
111
+ });
112
+ recorder.note({ type: "knowledge.key_rejected", ...event });
106
113
  }
107
114
  };
108
115
  let think = options.think;
@@ -3,7 +3,7 @@ import { Session, VoiceServer } from "@fonoster/voice";
3
3
  import { z } from "zod";
4
4
  import { SessionAgent } from "./config";
5
5
  import { AutopilotOptions, LoadedAgent } from "./createAutopilot";
6
- import { KnowledgePort } from "./knowledge";
6
+ import { KnowledgePassage, KnowledgePort } from "./knowledge";
7
7
  import { CredentialSource } from "./llm";
8
8
  import { UsageSink } from "./meteredThink";
9
9
  export declare const hostedConfigSchema: z.ZodObject<{
@@ -420,6 +420,7 @@ export declare class FileWorkspaceSecrets implements CredentialSource {
420
420
  type KnowledgeBaseLike = {
421
421
  load(): Promise<void>;
422
422
  queryKnowledgeBase(query: string, k?: number): Promise<string>;
423
+ searchKnowledgeBase?(query: string, k?: number): Promise<KnowledgePassage[]>;
423
424
  };
424
425
  /**
425
426
  * Builds the knowledge port for an agent from the operator's S3 + Unstructured settings,
@@ -462,6 +462,10 @@ function eagerKnowledge(build) {
462
462
  find: async (question, signal) => {
463
463
  const p = await port;
464
464
  return p ? p.find(question, signal) : "";
465
+ },
466
+ search: async (question, signal) => {
467
+ const p = await port;
468
+ return p ? (0, knowledge_1.searchKnowledge)(p, question, signal) : [];
465
469
  }
466
470
  };
467
471
  };
@@ -27,3 +27,4 @@ export * from "./sentences";
27
27
  export * from "./telemetry";
28
28
  export * from "./think";
29
29
  export * from "./tools";
30
+ export * from "./traceDetails";
@@ -43,3 +43,4 @@ __exportStar(require("./sentences"), exports);
43
43
  __exportStar(require("./telemetry"), exports);
44
44
  __exportStar(require("./think"), exports);
45
45
  __exportStar(require("./tools"), exports);
46
+ __exportStar(require("./traceDetails"), exports);
@@ -16,6 +16,7 @@
16
16
  * See the License for the specific language governing permissions and
17
17
  * limitations under the License.
18
18
  */
19
+ import { KnowledgePassage, passagesText } from "../knowledge/passages";
19
20
  /**
20
21
  * The knowledge port (rework-autopilot A8): one operation, find what is relevant to a question.
21
22
  * Implementations (S3 + Unstructured today) are wired by the operator; their keys never live in
@@ -23,10 +24,17 @@
23
24
  */
24
25
  export interface KnowledgePort {
25
26
  find(question: string, signal: AbortSignal): Promise<string>;
27
+ /** The same lookup as passages, with where each came from; used instead of `find` when present. */
28
+ search?(question: string, signal: AbortSignal): Promise<KnowledgePassage[]>;
26
29
  }
30
+ export type { KnowledgePassage };
31
+ export { passagesText };
32
+ /** A port's lookup as passages: its own `search`, or what `find` returned as one passage. */
33
+ export declare function searchKnowledge(port: KnowledgePort, question: string, signal: AbortSignal): Promise<KnowledgePassage[]>;
27
34
  /** Knowledge lookups never hold a reply back longer than this. */
28
35
  export declare const KNOWLEDGE_TIMEOUT_MS = 800;
29
36
  /** Adapts the current knowledge bases (mods/autopilot/src/knowledge) to the port. */
30
37
  export declare function fromKnowledgeBase(kb: {
31
38
  queryKnowledgeBase(query: string, k?: number): Promise<string>;
39
+ searchKnowledgeBase?(query: string, k?: number): Promise<KnowledgePassage[]>;
32
40
  }): KnowledgePort;
@@ -1,10 +1,43 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.KNOWLEDGE_TIMEOUT_MS = void 0;
3
+ exports.KNOWLEDGE_TIMEOUT_MS = exports.passagesText = void 0;
4
+ exports.searchKnowledge = searchKnowledge;
4
5
  exports.fromKnowledgeBase = fromKnowledgeBase;
6
+ /**
7
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
8
+ * http://github.com/fonoster/fonoster
9
+ *
10
+ * This file is part of Fonoster
11
+ *
12
+ * Licensed under the MIT License (the "License");
13
+ * you may not use this file except in compliance with
14
+ * the License. You may obtain a copy of the License at
15
+ *
16
+ * https://opensource.org/licenses/MIT
17
+ *
18
+ * Unless required by applicable law or agreed to in writing, software
19
+ * distributed under the License is distributed on an "AS IS" BASIS,
20
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+ * See the License for the specific language governing permissions and
22
+ * limitations under the License.
23
+ */
24
+ const passages_1 = require("../knowledge/passages");
25
+ Object.defineProperty(exports, "passagesText", { enumerable: true, get: function () { return passages_1.passagesText; } });
26
+ /** A port's lookup as passages: its own `search`, or what `find` returned as one passage. */
27
+ async function searchKnowledge(port, question, signal) {
28
+ if (port.search)
29
+ return port.search(question, signal);
30
+ const text = (await port.find(question, signal))?.trim();
31
+ return text ? [{ text }] : [];
32
+ }
5
33
  /** Knowledge lookups never hold a reply back longer than this. */
6
34
  exports.KNOWLEDGE_TIMEOUT_MS = 800;
7
35
  /** Adapts the current knowledge bases (mods/autopilot/src/knowledge) to the port. */
8
36
  function fromKnowledgeBase(kb) {
9
- return { find: (question) => kb.queryKnowledgeBase(question) };
37
+ return {
38
+ find: (question) => kb.queryKnowledgeBase(question),
39
+ ...(kb.searchKnowledgeBase
40
+ ? { search: (question) => kb.searchKnowledgeBase(question) }
41
+ : {})
42
+ };
10
43
  }
@@ -111,7 +111,9 @@ class RegistryLlmResolver {
111
111
  return {
112
112
  id,
113
113
  keySource: credential?.source ?? "platform",
114
- think: llm.create({ baseUrl, apiKey }, id.slice(providerId.length + 1), settings),
114
+ think: llm.create(
115
+ // The catalog provider (e.g. together) names a rejected key, not the adapter (openai).
116
+ { baseUrl, apiKey, name: provider.id }, id.slice(providerId.length + 1), settings),
115
117
  ...(resolution.event
116
118
  ? {
117
119
  event: {
@@ -23,6 +23,12 @@ export type UsageSink = {
23
23
  usage(lines: Usage.UsageLine[]): void;
24
24
  /** A completed request whose vendor reported no usage (add-usage-metering). */
25
25
  missing(catalogId: string): void;
26
+ /** The knowledge base's embeddings key was rejected: knowledge is off for this call. */
27
+ keyRejected?(event: {
28
+ provider: string;
29
+ catalogId: string;
30
+ status: number;
31
+ }): void;
26
32
  };
27
33
  export type MeteredModel = {
28
34
  /** The catalog id (`<provider>/<model>`). */
@@ -4,6 +4,9 @@ export type ToolOutcome = {
4
4
  ok: boolean;
5
5
  result?: unknown;
6
6
  error?: string;
7
+ /** HTTP tools: the method used and the response status (for the call record). */
8
+ method?: string;
9
+ status?: number;
7
10
  };
8
11
  /** The tool port (rework-autopilot A7): one operation, run a tool with inputs. */
9
12
  export interface ToolPort {
@@ -36,16 +36,17 @@ class HttpToolPort {
36
36
  if (!tool)
37
37
  return { ok: false, error: `unknown tool ${name}` };
38
38
  const method = String(tool.operation.method ?? "get").toUpperCase();
39
- const url = new url_1.URL(tool.operation.url);
40
- let body;
41
- if (method === "GET") {
42
- for (const [k, v] of Object.entries(args))
43
- url.searchParams.set(k, String(v));
44
- }
45
- else {
46
- body = JSON.stringify(args);
47
- }
48
39
  try {
40
+ // Inside the try: an invalid tool URL is a failed call, recorded like any other.
41
+ const url = new url_1.URL(tool.operation.url);
42
+ let body;
43
+ if (method === "GET") {
44
+ for (const [k, v] of Object.entries(args))
45
+ url.searchParams.set(k, String(v));
46
+ }
47
+ else {
48
+ body = JSON.stringify(args);
49
+ }
49
50
  const request = (0, common_1.safeFetch)(url, {
50
51
  method,
51
52
  headers: {
@@ -59,24 +60,27 @@ class HttpToolPort {
59
60
  });
60
61
  if (tool.operation.waitForResponse === false) {
61
62
  request.catch(() => undefined);
62
- return { ok: true, result: { status: "sent" } };
63
+ return { ok: true, result: { status: "sent" }, method };
63
64
  }
64
65
  const res = await request;
65
66
  const text = await res.text();
67
+ const status = res.status;
66
68
  if (!res.ok)
67
69
  return {
68
70
  ok: false,
69
- error: `HTTP ${res.status}: ${text.slice(0, 200)}`
71
+ error: `HTTP ${status}: ${text.slice(0, 200)}`,
72
+ method,
73
+ status
70
74
  };
71
75
  try {
72
- return { ok: true, result: JSON.parse(text) };
76
+ return { ok: true, result: JSON.parse(text), method, status };
73
77
  }
74
78
  catch {
75
- return { ok: true, result: text };
79
+ return { ok: true, result: text, method, status };
76
80
  }
77
81
  }
78
82
  catch (e) {
79
- return { ok: false, error: e.message };
83
+ return { ok: false, error: e.message, method };
80
84
  }
81
85
  }
82
86
  }
@@ -0,0 +1,94 @@
1
+ /**
2
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
3
+ * http://github.com/fonoster/fonoster
4
+ *
5
+ * This file is part of Fonoster
6
+ *
7
+ * Licensed under the MIT License (the "License");
8
+ * you may not use this file except in compliance with
9
+ * the License. You may obtain a copy of the License at
10
+ *
11
+ * https://opensource.org/licenses/MIT
12
+ *
13
+ * Unless required by applicable law or agreed to in writing, software
14
+ * distributed under the License is distributed on an "AS IS" BASIS,
15
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ * See the License for the specific language governing permissions and
17
+ * limitations under the License.
18
+ */
19
+ import { KnowledgePassage } from "./knowledge";
20
+ import { ToolOutcome } from "./tools";
21
+ /** Tool parameters, responses and errors are cut at this many characters. */
22
+ export declare const MAX_TOOL_TEXT = 2048;
23
+ /** Knowledge queries and passages are cut at this many characters. */
24
+ export declare const MAX_KNOWLEDGE_TEXT = 500;
25
+ /** At most this many passages are kept per knowledge search. */
26
+ export declare const MAX_PASSAGES = 3;
27
+ /** One tool call of a turn. */
28
+ export type ToolCallTrace = {
29
+ name: string;
30
+ ok: boolean;
31
+ /** When the call started, in ms since the epoch (orders it within the turn). */
32
+ startedAtMs?: number;
33
+ /** HTTP tools: the method and the response status. */
34
+ method?: string;
35
+ status?: number;
36
+ durationMs?: number;
37
+ /** JSON, capped. */
38
+ parameters?: string;
39
+ response?: string;
40
+ error?: string;
41
+ };
42
+ /** One knowledge search of a turn: the passages found for the caller's words. */
43
+ export type KnowledgeSearchTrace = {
44
+ query: string;
45
+ /** At most MAX_PASSAGES, best first; the first one's `source` is the top match's document. */
46
+ passages: KnowledgePassage[];
47
+ /** When the search started, in ms since the epoch (orders it within the turn). */
48
+ startedAtMs?: number;
49
+ durationMs: number;
50
+ /** The search took longer than a reply waits for; the reply went without it. */
51
+ timedOut?: boolean;
52
+ /** The search failed; the reply went without it. `error` is the message, short. */
53
+ failed?: boolean;
54
+ error?: string;
55
+ };
56
+ /** A failed knowledge search's error is cut at this many characters. */
57
+ export declare const MAX_KNOWLEDGE_ERROR = 200;
58
+ /**
59
+ * Caps text at `max` characters, ending with an ellipsis. A run of digits or digit words cut at
60
+ * the end is dropped when, with what follows the cut, it is long enough to be part of a card
61
+ * number (the redaction only recognizes whole ones); amounts and ordinary words stay.
62
+ */
63
+ export declare function capText(text: string, max: number): string;
64
+ /** The trace entry for a tool the model called. */
65
+ export declare function toolCallTrace(name: string, args: Record<string, unknown>, outcome: ToolOutcome, startedAtMs: number, durationMs: number): ToolCallTrace;
66
+ /** The trace entry for a knowledge search: what it found, or that it timed out or failed. */
67
+ export declare function knowledgeSearchTrace(query: string, found: {
68
+ passages: KnowledgePassage[];
69
+ } | {
70
+ timedOut: true;
71
+ } | {
72
+ error: unknown;
73
+ }, startedAtMs: number, durationMs: number): KnowledgeSearchTrace;
74
+ /**
75
+ * The turn as the log sees it when there is no call record: tool names, outcomes, counts and
76
+ * durations only. Parameters, responses, errors, queries and passages stay out of the logs.
77
+ */
78
+ export declare function loggedTurn<T extends {
79
+ toolCalls: ToolCallTrace[];
80
+ knowledgeSearches?: KnowledgeSearchTrace[];
81
+ }>(trace: T): T & {
82
+ knowledgeSearches?: {
83
+ failed?: boolean;
84
+ timedOut?: boolean;
85
+ passages: number;
86
+ durationMs: number;
87
+ }[];
88
+ toolCalls: {
89
+ durationMs?: number;
90
+ status?: number;
91
+ name: string;
92
+ ok: boolean;
93
+ }[];
94
+ };
@@ -0,0 +1,113 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.MAX_KNOWLEDGE_ERROR = exports.MAX_PASSAGES = exports.MAX_KNOWLEDGE_TEXT = exports.MAX_TOOL_TEXT = void 0;
4
+ exports.capText = capText;
5
+ exports.toolCallTrace = toolCallTrace;
6
+ exports.knowledgeSearchTrace = knowledgeSearchTrace;
7
+ exports.loggedTurn = loggedTurn;
8
+ /** Tool parameters, responses and errors are cut at this many characters. */
9
+ exports.MAX_TOOL_TEXT = 2048;
10
+ /** Knowledge queries and passages are cut at this many characters. */
11
+ exports.MAX_KNOWLEDGE_TEXT = 500;
12
+ /** At most this many passages are kept per knowledge search. */
13
+ exports.MAX_PASSAGES = 3;
14
+ /** A failed knowledge search's error is cut at this many characters. */
15
+ exports.MAX_KNOWLEDGE_ERROR = 200;
16
+ /** A trailing run of at least this many digits could be part of a card number. */
17
+ const CARD_DIGITS = 13;
18
+ const DIGIT_WORDS = "zero|oh|one|two|three|four|five|six|seven|eight|nine|cero|uno|dos|tres|cuatro|cinco|seis|siete|ocho|nueve";
19
+ const DIGIT = `(?:\\d|\\b(?:${DIGIT_WORDS})\\b)`;
20
+ // A run of digits or digit words at the end of the text, and at the start, with their separators.
21
+ const TRAILING_NUMBER = new RegExp(`(?:[\\s,.-]*${DIGIT})+[\\s,.-]*$`, "i");
22
+ const LEADING_NUMBER = new RegExp(`^(?:[\\s,.-]*${DIGIT})+`, "i");
23
+ const DIGITS = new RegExp(DIGIT, "gi");
24
+ const digitCount = (text) => text.match(DIGITS)?.length ?? 0;
25
+ /**
26
+ * Caps text at `max` characters, ending with an ellipsis. A run of digits or digit words cut at
27
+ * the end is dropped when, with what follows the cut, it is long enough to be part of a card
28
+ * number (the redaction only recognizes whole ones); amounts and ordinary words stay.
29
+ */
30
+ function capText(text, max) {
31
+ if (text.length <= max)
32
+ return text;
33
+ const head = text.slice(0, max);
34
+ const trailing = head.match(TRAILING_NUMBER)?.[0] ?? "";
35
+ const continued = text.slice(max).match(LEADING_NUMBER)?.[0] ?? "";
36
+ const run = digitCount(trailing) + digitCount(continued);
37
+ const kept = trailing && run >= CARD_DIGITS ? head.slice(0, -trailing.length) : head;
38
+ return `${kept}…`;
39
+ }
40
+ function json(value) {
41
+ if (typeof value === "string")
42
+ return value;
43
+ try {
44
+ return JSON.stringify(value) ?? "";
45
+ }
46
+ catch {
47
+ return String(value);
48
+ }
49
+ }
50
+ /** The trace entry for a tool the model called. */
51
+ function toolCallTrace(name, args, outcome, startedAtMs, durationMs) {
52
+ return {
53
+ name,
54
+ ok: outcome.ok,
55
+ startedAtMs,
56
+ ...(outcome.method ? { method: outcome.method } : {}),
57
+ ...(outcome.status !== undefined ? { status: outcome.status } : {}),
58
+ durationMs,
59
+ ...(Object.keys(args).length
60
+ ? { parameters: capText(json(args), exports.MAX_TOOL_TEXT) }
61
+ : {}),
62
+ ...(outcome.ok && outcome.result !== undefined
63
+ ? { response: capText(json(outcome.result), exports.MAX_TOOL_TEXT) }
64
+ : {}),
65
+ ...(!outcome.ok && outcome.error
66
+ ? { error: capText(outcome.error, exports.MAX_TOOL_TEXT) }
67
+ : {})
68
+ };
69
+ }
70
+ /** The trace entry for a knowledge search: what it found, or that it timed out or failed. */
71
+ function knowledgeSearchTrace(query, found, startedAtMs, durationMs) {
72
+ const passages = "passages" in found ? found.passages : [];
73
+ const error = "error" in found
74
+ ? capText(found.error?.message ?? String(found.error), exports.MAX_KNOWLEDGE_ERROR)
75
+ : undefined;
76
+ return {
77
+ query: capText(query, exports.MAX_KNOWLEDGE_TEXT),
78
+ passages: passages.slice(0, exports.MAX_PASSAGES).map((p) => ({
79
+ text: capText(p.text, exports.MAX_KNOWLEDGE_TEXT),
80
+ ...(p.source ? { source: p.source } : {}),
81
+ ...(p.score !== undefined ? { score: p.score } : {})
82
+ })),
83
+ startedAtMs,
84
+ durationMs,
85
+ ...("timedOut" in found ? { timedOut: true } : {}),
86
+ ...("error" in found ? { failed: true, ...(error ? { error } : {}) } : {})
87
+ };
88
+ }
89
+ /**
90
+ * The turn as the log sees it when there is no call record: tool names, outcomes, counts and
91
+ * durations only. Parameters, responses, errors, queries and passages stay out of the logs.
92
+ */
93
+ function loggedTurn(trace) {
94
+ return {
95
+ ...trace,
96
+ toolCalls: trace.toolCalls.map((c) => ({
97
+ name: c.name,
98
+ ok: c.ok,
99
+ ...(c.status !== undefined ? { status: c.status } : {}),
100
+ ...(c.durationMs !== undefined ? { durationMs: c.durationMs } : {})
101
+ })),
102
+ ...(trace.knowledgeSearches
103
+ ? {
104
+ knowledgeSearches: trace.knowledgeSearches.map((k) => ({
105
+ passages: k.passages.length,
106
+ durationMs: k.durationMs,
107
+ ...(k.timedOut ? { timedOut: true } : {}),
108
+ ...(k.failed ? { failed: true } : {})
109
+ }))
110
+ }
111
+ : {})
112
+ };
113
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fonoster/autopilot",
3
- "version": "0.32.0",
3
+ "version": "0.34.1",
4
4
  "description": "Voice AI for the Fonoster platform",
5
5
  "author": "Pedro Sanders <psanders@fonoster.com>",
6
6
  "homepage": "https://github.com/fonoster/fonoster#readme",
@@ -33,11 +33,11 @@
33
33
  "dependencies": {
34
34
  "@aws-sdk/client-s3": "^3.958.0",
35
35
  "@dmitryrechkin/json-schema-to-zod": "^1.0.1",
36
- "@fonoster/common": "^0.32.0",
37
- "@fonoster/community": "^0.32.0",
38
- "@fonoster/logger": "^0.32.0",
39
- "@fonoster/types": "^0.32.0",
40
- "@fonoster/voice": "^0.32.0",
36
+ "@fonoster/common": "^0.34.1",
37
+ "@fonoster/community": "^0.34.1",
38
+ "@fonoster/logger": "^0.34.1",
39
+ "@fonoster/types": "^0.34.1",
40
+ "@fonoster/voice": "^0.34.1",
41
41
  "@grpc/grpc-js": "~1.14.6",
42
42
  "@langchain/anthropic": "^1.3.3",
43
43
  "@langchain/community": "^1.1.1",
@@ -55,7 +55,7 @@
55
55
  "xstate": "^5.17.3",
56
56
  "zod": "^3.25.76"
57
57
  },
58
- "gitHead": "26503065668ba982e416f9fec44e3b090fb9ae88",
58
+ "gitHead": "f521d836e94a24a5b17a25fba3e03e680637016d",
59
59
  "devDependencies": {
60
60
  "yaml": "^2.9.0"
61
61
  }