@fonoster/autopilot 0.27.0 → 0.31.0

package/README.md

@@ -12,15 +12,19 @@ The autopilot module provides a way to create Voice AI applications in Fonoster.
 * [Example](#example)
 * [Adding Knowledge Base](#configuring-the-knowledge-base)
 * [Adding Tools](#configuring-the-tools)
-* [What's Next](#whats-next)
 
 ## Installation
 
+Autopilot is an ordinary session app that AppBridge dials. Run the
+`fonoster/autopilot` image (built from `mods/autopilot/Dockerfile`), or from the repository:
+
 ```sh-session
-$ npm install -g @fonoster/autopilot
+$ cp config/autopilot-session.example.yaml config/autopilot-session.yaml
+$ node mods/autopilot/dist/session/main.js --config config/autopilot-session.yaml
 ```
 
-The previous command will install the module globally.
+See [docs/autopilot.md](../../docs/autopilot.md) for the engine, pacing and the agent
+schema.
 
 ## Example
 
@@ -60,7 +64,7 @@ The configuration file has two major sections: `conversationSettings` and `langu
 To run the Autopilot, you can use the following command:
 
 ```sh-session
-$ ASSISTANTS=$(pwd)/assistants.json OPEN_API_KEY=your-key autopilot
+$ AGENT=$(pwd)/agents.json OPEN_API_KEY=your-key autopilot
 ```
 
 You will see a message like this:
@@ -179,53 +183,12 @@ For example:
 { "result": "The available appointment times are 9:00 AM, 10:00 AM, and 11:00 AM." }
 ```
 
-## Environment Variables
-
-Please use the following environment variables to configure the Autopilot:
-
-- `AUTOPILOT_AWS_S3_ACCESS_KEY_ID`: The AWS S3 access key.
-- `AUTOPILOT_AWS_S3_ENDPOINT`: The AWS S3 endpoint.
-- `AUTOPILOT_AWS_S3_REGION`: The AWS S3 region.
-- `AUTOPILOT_AWS_S3_SECRET_ACCESS_KEY`: The AWS S3 secret access key.
-- `AUTOPILOT_CONVERSATION_PROVIDER`: The conversation provider.
-- `AUTOPILOT_KNOWLEDGE_BASE_ENABLED`: Enable knowledge base powered by AWS S3 and Unstructured API.
-- `AUTOPILOT_LOGS_FORMAT`: The logs format.
-- `AUTOPILOT_LOGS_LEVEL`: The logs level.
-- `AUTOPILOT_LOGS_TRANSPORT`: The logs transport.
-- `AUTOPILOT_OPENAI_API_KEY`: The OpenAI API key needed for embeddings support.
-- `AUTOPILOT_UNSTRUCTURED_API_KEY`: The Unstructured API key.
-- `AUTOPILOT_UNSTRUCTURED_API_URL`: The Unstructured API URL. Default is `https://api.unstructured.com`.
-- `AUTOPILOT_RECORDING_BASE_URL`: The recording base URL. Default is `http://localhost:9876/api/recordings`.
-
-## What's Next
-
-The Autopilot is still under heavy development. The next steps are to add support for Retrieval-Augmented Generation(RAG), improve the language model integration, and improve the Finite State Machine (FSM) that powers the conversations. If you have any suggestions or feedback, please let us know.
-
-## Audio filters
-
-Background voices reach speech recognition and the voice activity detector, and the assistant
-answers them. An assistant can ask the Media Server to filter the caller's audio first, by adding
-`audioFilters` to its configuration — the same block whether the assistant comes from a file or
-from an Application:
-
-```yaml
-audioFilters:
-  - name: aiCoustics
-    options:
-      enhancementLevel: 0.8 # 0 to 1; higher also suppresses competing voices
-
-conversationSettings:
-  firstMessage: "Hola, ¿en qué puedo ayudarte?"
-  # ...
-```
-
-For an Application, the same block goes inside `intelligence.config`, and is applied with:
+## Configuration
 
-```sh
-fonoster applications:create -f app.yaml
-```
+Autopilot reads `autopilot-session.yaml`, found via `--config <path>`, the
+`AUTOPILOT_SESSION_CONFIG` variable or `./config/autopilot-session.yaml`. It names the signing
+secret AppBridge uses for Autopilot apps, where apps and agents come from, the registry
+catalog, platform keys by credential ref and the knowledge settings. See
+`config/autopilot-session.example.yaml` for every key.
 
-Assistant files may be YAML or JSON. Filtering needs the optional `@ai-coustics/aic-sdk` package
-and an `AIC_SDK_LICENSE` key on the apiserver; the key never appears in assistant configuration,
-so these files stay safe to commit. If the filter cannot be applied, the call continues with
-unfiltered audio and the reason is logged.
+Audio filters are part of the app's configuration (AppBridge runs them), not the agent's.

package/dist/index.d.ts

@@ -16,12 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Autopilot } from "./Autopilot";
-export default Autopilot;
-export * from "./assistants";
 export * from "./knowledge";
 export * from "./models";
 export * from "./tools";
-export * from "./types";
-export * from "./vad";
 export * from "./voice";

package/dist/index.js

@@ -32,12 +32,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-const Autopilot_1 = require("./Autopilot");
-exports.default = Autopilot_1.Autopilot;
-__exportStar(require("./assistants"), exports);
 __exportStar(require("./knowledge"), exports);
 __exportStar(require("./models"), exports);
 __exportStar(require("./tools"), exports);
-__exportStar(require("./types"), exports);
-__exportStar(require("./vad"), exports);
 __exportStar(require("./voice"), exports);

package/dist/knowledge/AbstractKnowledgeBase.d.ts

@@ -4,8 +4,9 @@ import { KnowledgeBase } from "./types";
 declare abstract class AbstractKnowledgeBase implements KnowledgeBase {
     protected embeddings: Embeddings;
     protected vectorStore: VectorStore;
-    constructor(params?: {
-        embeddings?: Embeddings;
+    /** The embeddings come from the caller (the hosted Autopilot resolves their key by credentialRef). */
+    constructor(params: {
+        embeddings: Embeddings;
     });
     abstract getLoaders(): Promise<unknown>;
     load(): Promise<void>;

package/dist/knowledge/AbstractKnowledgeBase.js

@@ -21,17 +21,12 @@ exports.AbstractKnowledgeBase = void 0;
  */
 const logger_1 = require("@fonoster/logger");
 const memory_1 = require("@langchain/classic/vectorstores/memory");
-const openai_1 = require("@langchain/openai");
 const textsplitters_1 = require("@langchain/textsplitters");
-const envs_1 = require("../envs");
 const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
 class AbstractKnowledgeBase {
+    /** The embeddings come from the caller (the hosted Autopilot resolves their key by credentialRef). */
     constructor(params) {
-        this.embeddings =
-            params?.embeddings ||
-                new openai_1.OpenAIEmbeddings({
-                    apiKey: envs_1.OPENAI_API_KEY
-                });
+        this.embeddings = params.embeddings;
     }
     async load() {
         const loaders = (await this.getLoaders());

package/dist/knowledge/FilesKnowledgeBase.d.ts

@@ -22,7 +22,7 @@ import { AbstractKnowledgeBase } from "./AbstractKnowledgeBase";
 declare class FilesKnowledgeBase extends AbstractKnowledgeBase {
     private params;
     constructor(params: {
-        embeddings?: Embeddings;
+        embeddings: Embeddings;
         files: string[];
     });
     getLoaders(): Promise<PDFLoader[]>;

package/dist/knowledge/MeteredEmbeddings.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Usage } from "@fonoster/common";
+/** Where embedding usage goes (the session's usage sink). */
+export type EmbeddingUsageSink = {
+    usage(lines: Usage.UsageLine[]): void;
+    missing(catalogId: string): void;
+};
+/**
+ * OpenAI embeddings that report their token usage (add-usage-metering): the knowledge base's
+ * embeddings, called directly so the vendor's `usage.prompt_tokens` is not lost. Same defaults
+ * as LangChain's OpenAIEmbeddings (model text-embedding-ada-002, 512 texts per request); it
+ * implements the embedDocuments/embedQuery pair LangChain's vector stores use.
+ */
+export declare class MeteredEmbeddings {
+    private readonly options;
+    private readonly model;
+    private readonly baseUrl;
+    constructor(options: {
+        apiKey: string;
+        keySource: Usage.KeySource;
+        sink?: EmbeddingUsageSink;
+        model?: string;
+        baseUrl?: string;
+        batchSize?: number;
+    });
+    get catalogId(): string;
+    embedDocuments(texts: string[]): Promise<number[][]>;
+    embedQuery(text: string): Promise<number[]>;
+    private embed;
+}

package/dist/knowledge/MeteredEmbeddings.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MeteredEmbeddings = void 0;
+/**
+ * OpenAI embeddings that report their token usage (add-usage-metering): the knowledge base's
+ * embeddings, called directly so the vendor's `usage.prompt_tokens` is not lost. Same defaults
+ * as LangChain's OpenAIEmbeddings (model text-embedding-ada-002, 512 texts per request); it
+ * implements the embedDocuments/embedQuery pair LangChain's vector stores use.
+ */
+class MeteredEmbeddings {
+    constructor(options) {
+        this.options = options;
+        this.model = options.model ?? "text-embedding-ada-002";
+        this.baseUrl = (options.baseUrl ?? "https://api.openai.com").replace(/\/$/, "");
+    }
+    get catalogId() {
+        return `openai/${this.model}`;
+    }
+    async embedDocuments(texts) {
+        const size = this.options.batchSize ?? 512;
+        const batches = Array.from({ length: Math.ceil(texts.length / size) }, (_, i) => texts.slice(i * size, (i + 1) * size));
+        const vectors = await Promise.all(batches.map((b) => this.embed(b)));
+        return vectors.flat();
+    }
+    async embedQuery(text) {
+        return (await this.embed([text]))[0];
+    }
+    async embed(input) {
+        const res = await fetch(`${this.baseUrl}/v1/embeddings`, {
+            method: "POST",
+            headers: {
+                "content-type": "application/json",
+                authorization: `Bearer ${this.options.apiKey}`
+            },
+            body: JSON.stringify({ model: this.model, input })
+        });
+        if (!res.ok)
+            throw new Error(`embeddings answered HTTP ${res.status}`);
+        const body = (await res.json());
+        const tokens = body.usage?.prompt_tokens;
+        if (tokens === undefined)
+            this.options.sink?.missing(this.catalogId);
+        else
+            this.options.sink?.usage([
+                {
+                    meter: "embedding.tokens",
+                    quantity: tokens,
+                    provider: "openai",
+                    model: this.catalogId,
+                    keySource: this.options.keySource
+                }
+            ]);
+        return [...body.data]
+            .sort((a, b) => a.index - b.index)
+            .map((d) => d.embedding);
+    }
+}
+exports.MeteredEmbeddings = MeteredEmbeddings;

package/dist/knowledge/types.d.ts

@@ -22,7 +22,7 @@ type KnowledgeBase = {
     queryKnowledgeBase: (query: string, k?: number) => Promise<string>;
 };
 type S3KnowledgeBaseParams = {
-    embeddings?: Embeddings;
+    embeddings: Embeddings;
     bucket: string;
     documents: string[];
     s3Config: {

package/dist/models/LanguageModelFactory.d.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
- * http://github.com/fonoster/fonoster
- *
- * This file is part of Fonoster
- *
- * Licensed under the MIT License (the "License");
- * you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    https://opensource.org/licenses/MIT
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import { LanguageModelProvider } from "@fonoster/common";
 import { Voice } from "../voice";
 import { AbstractLanguageModel } from "./AbstractLanguageModel";
 import { AnthropicParams } from "./anthropic";
@@ -27,11 +8,11 @@ import { OpenAIParams } from "./openai";
 import { BaseModelParams, TelephonyContext } from "./types";
 type LanguageModelConstructor<T extends BaseModelParams = BaseModelParams> = new (options: T, voice: Voice, telephonyContext: TelephonyContext) => AbstractLanguageModel;
 type LanguageModelConfigMap = {
-    [LanguageModelProvider.OPENAI]: OpenAIParams;
-    [LanguageModelProvider.GROQ]: GroqParams;
-    [LanguageModelProvider.OLLAMA]: OllamaParams;
-    [LanguageModelProvider.GOOGLE]: GoogleParams;
-    [LanguageModelProvider.ANTHROPIC]: AnthropicParams;
+    openai: OpenAIParams;
+    groq: GroqParams;
+    ollama: OllamaParams;
+    google: GoogleParams;
+    anthropic: AnthropicParams;
 };
 declare class LanguageModelFactory {
     private static readonly languageModels;

package/dist/models/LanguageModelFactory.js

@@ -19,7 +19,6 @@ exports.LanguageModelFactory = void 0;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-const common_1 = require("@fonoster/common");
 const logger_1 = require("@fonoster/logger");
 const anthropic_1 = require("./anthropic");
 const google_1 = require("./google");

package/dist/models/createLanguageModel.d.ts

@@ -16,10 +16,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { AssistantConfig, KnowledgeBase, TelephonyContext, Voice } from "..";
+import { EvalAgent, KnowledgeBase, TelephonyContext, Voice } from "..";
 declare function createLanguageModel(params: {
     voice: Voice;
-    assistantConfig: AssistantConfig;
+    agentConfig: EvalAgent;
     knowledgeBase: KnowledgeBase;
     telephonyContext: TelephonyContext;
 }): import("./AbstractLanguageModel").AbstractLanguageModel;

package/dist/models/createLanguageModel.js

@@ -21,8 +21,8 @@ exports.createLanguageModel = createLanguageModel;
  */
 const __1 = require("..");
 function createLanguageModel(params) {
-    const { voice, assistantConfig, knowledgeBase, telephonyContext } = params;
-    const { languageModel, conversationSettings } = assistantConfig;
+    const { voice, agentConfig, knowledgeBase, telephonyContext } = params;
+    const { languageModel, conversationSettings } = agentConfig;
     // The transfer tool is only added if the transfer options exist
     const tools = languageModel.tools.concat(conversationSettings.transferOptions
         ? [__1.hangupToolDefinition, __1.transferToolDefinition]

package/dist/models/evaluations/createEvalEffectiveConfig.d.ts

@@ -16,109 +16,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { AssistantConfig } from "../../assistants";
-export declare function createEvalEffectiveConfig(config: AssistantConfig, credentials: {
-    apiKey: string;
-}, evaluationApiKey: {
-    apiKey: string;
-}): {
-    languageModel: {
-        apiKey: string;
-        provider?: import("@fonoster/common").LanguageModelProvider;
-        model?: string;
-        temperature?: number;
-        maxTokens?: number;
-        baseUrl?: string;
-        knowledgeBase?: {
-            type?: "s3";
-            title?: string;
-            document?: string;
-        }[];
-        tools?: {
-            name?: string;
-            description?: string;
-            parameters?: {
-                type?: "object" | "array";
-                properties?: Record<string, {
-                    type?: string;
-                    format?: "enum" | "date-time";
-                    pattern?: string;
-                }>;
-                required?: string[];
-            };
-            requestStartMessage?: string;
-            operation?: {
-                url?: string;
-                method?: import("@fonoster/common").AllowedHttpMethod;
-                headers?: Record<string, string>;
-                waitForResponse?: boolean;
-            };
-        }[];
-    };
-    testCases: {
-        evalsLanguageModel: {
-            apiKey: string;
-            provider?: "openai";
-            model?: string;
-        };
-        evalsSystemPrompt?: string;
-        scenarios?: {
-            description?: string;
-            ref?: string;
-            telephonyContext?: {
-                callDirection?: import("@fonoster/types").CallDirection;
-                ingressNumber?: string;
-                callerNumber?: string;
-                metadata?: Record<string, string>;
-            };
-            conversation?: {
-                expected?: {
-                    tools?: {
-                        parameters?: Record<string, any>;
-                        tool?: string;
-                    }[];
-                    text?: {
-                        type?: "EXACT" | "SIMILAR";
-                        response?: string;
-                    };
-                };
-                userInput?: string;
-            }[];
-        }[];
-    };
-    audioFilters?: {
-        options?: Record<string, unknown>;
-        name?: string;
-    }[];
-    conversationSettings?: {
-        firstMessage?: string;
-        systemPrompt?: string;
-        goodbyeMessage?: string;
-        systemErrorMessage?: string;
-        initialDtmf?: string;
-        maxSessionDuration?: number;
-        maxSpeechWaitTimeout?: number;
-        transferOptions?: {
-            message?: string;
-            phoneNumber?: string;
-            timeout?: number;
-        };
-        idleOptions?: {
-            message?: string;
-            timeout?: number;
-            maxTimeoutCount?: number;
-        };
-        allowUserBargeIn?: boolean;
-        vad?: {
-            pathToModel?: string;
-            activationThreshold?: number;
-            deactivationThreshold?: number;
-            debounceFrames?: number;
-        };
-    };
-    eventsHook?: {
-        url?: string;
-        headers?: Record<string, string>;
-        events?: import("@fonoster/common").EventsHookAllowedEvents[];
-    };
-};
+import { SessionAgent } from "@fonoster/common";
+import { EvalAgent } from "../types";
+/**
+ * The session agent as the evaluations run it: the catalog id split into provider and model,
+ * with the keys the caller resolved (never stored in the agent).
+ */
+export declare function createEvalEffectiveConfig(agent: SessionAgent, keys: {
+    apiKey?: string;
+    evaluationApiKey?: string;
+}): EvalAgent;

package/dist/models/evaluations/createEvalEffectiveConfig.js

@@ -1,18 +1,58 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createEvalEffectiveConfig = createEvalEffectiveConfig;
-function createEvalEffectiveConfig(config, credentials, evaluationApiKey) {
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const common_1 = require("@fonoster/common");
+const LLM_PROVIDERS = [
+    "openai",
+    "groq",
+    "ollama",
+    "google",
+    "anthropic"
+];
+/**
+ * The session agent as the evaluations run it: the catalog id split into provider and model,
+ * with the keys the caller resolved (never stored in the agent).
+ */
+function createEvalEffectiveConfig(agent, keys) {
+    const [provider, ...rest] = agent.languageModel.model.split("/");
+    if (!LLM_PROVIDERS.includes(provider))
+        throw new Error(`evaluations do not support ${provider} models`);
+    const testCases = common_1.testCasesSchema.parse(agent.testCases);
+    const { provider: _p, apiKey: _k, baseUrl: _b, ...languageModel } = agent.languageModel;
+    void _p;
+    void _k;
+    void _b;
     return {
-        ...config,
+        conversationSettings: agent.conversationSettings,
         languageModel: {
-            ...config.languageModel,
-            apiKey: credentials.apiKey
+            ...languageModel,
+            provider: provider,
+            model: rest.join("/"),
+            apiKey: keys.apiKey
         },
         testCases: {
-            ...config.testCases,
+            ...testCases,
             evalsLanguageModel: {
-                ...config.testCases.evalsLanguageModel,
-                apiKey: evaluationApiKey.apiKey
+                ...testCases.evalsLanguageModel,
+                apiKey: keys.evaluationApiKey
             }
         }
     };

package/dist/models/evaluations/createEvaluateIntelligence.d.ts

@@ -1,27 +1,28 @@
-/**
- * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
- * http://github.com/fonoster/fonoster
- *
- * This file is part of Fonoster
- *
- * Licensed under the MIT License (the "License");
- * you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    https://opensource.org/licenses/MIT
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import { IntegrationConfig } from "@fonoster/common";
+import { Usage } from "@fonoster/common";
 import { EvaluateIntelligenceRequest } from "./types";
 type ServerStreamCall = {
     request: EvaluateIntelligenceRequest;
     write: (chunk: Record<string, unknown>) => void;
     end: () => void;
 };
-declare function createEvaluateIntelligence(integrations: IntegrationConfig[]): (call: ServerStreamCall) => Promise<void>;
+/**
+ * A model provider's key for a workspace: the workspace's own vendor key, else the platform key
+ * by the provider's credentialRef (the control plane decides; integrations.json is gone).
+ */
+export type EvalKeySource = (providerId: string, workspaceRef: string) => Promise<EvalKey | string | undefined>;
+/** A key and whose it is; a bare string is the platform's. */
+export type EvalKey = {
+    key: string;
+    source: Usage.KeySource;
+};
+/**
+ * Where an eval's usage goes (add-usage-metering D5): the workspace, the lines, and a scope that
+ * is unique per request within the run, for the lines' ids.
+ */
+export type EvalUsageReporter = (workspaceRef: string, lines: Usage.UsageLine[], scope: string) => void;
+/**
+ * Runs an agent's test cases (rework-autopilot 8.2: the agent is a session agent; its
+ * catalog id picks the provider, and keys come from `keys`, never from the agent).
+ */
+declare function createEvaluateIntelligence(keys: EvalKeySource, reportUsage?: EvalUsageReporter): (call: ServerStreamCall) => Promise<void>;
 export { createEvaluateIntelligence };