@fonoster/autopilot 0.27.0 → 0.31.0

package/dist/session/Agent.js

@@ -0,0 +1,548 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Agent = exports.logRecorder = exports.DEFAULT_TRANSFER_FAILED_MESSAGE = void 0;
+exports.sessionRecorder = sessionRecorder;
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const common_1 = require("@fonoster/common");
+const logger_1 = require("@fonoster/logger");
+const voice_1 = require("@fonoster/voice");
+const AsyncQueue_1 = require("./AsyncQueue");
+const knowledge_1 = require("./knowledge");
+const sentences_1 = require("./sentences");
+const telemetry_1 = require("./telemetry");
+/** What Autopilot says when a transfer fails and the agent sets no `failedMessage`. */
+exports.DEFAULT_TRANSFER_FAILED_MESSAGE = "Sorry, I couldn't connect you. How else can I help?";
+const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
+const MAX_TOOL_ROUNDS = 4;
+const BUILT_IN_TOOLS = [
+    {
+        name: "hangup",
+        description: "End the call when the conversation is over.",
+        parameters: { type: "object", properties: {} }
+    },
+    {
+        name: "transfer",
+        description: "Transfer the caller to an operator (a person).",
+        parameters: { type: "object", properties: {} }
+    }
+];
+/** Writes turn traces and notes to the call record through the session (record.note). */
+function sessionRecorder(session) {
+    const write = (data) => void session.note(data).catch(() => undefined);
+    return {
+        turn: (trace) => write({ kind: "autopilot.turn", ...trace }),
+        note: (event) => write({ kind: "autopilot.note", ...event }),
+        usage: (lines) => void session.reportUsage(lines).catch(() => undefined)
+    };
+}
+exports.logRecorder = {
+    turn: (trace) => logger.verbose("autopilot turn", trace),
+    note: (event) => logger.verbose("autopilot note", event)
+};
+/** The side effects of Autopilot: everything the state machine asks it to do on the session. */
+class Agent {
+    constructor(session, agent, think, tools, recorder = exports.logRecorder, knowledge) {
+        this.session = session;
+        this.agent = agent;
+        this.think = think;
+        this.tools = tools;
+        this.recorder = recorder;
+        this.knowledge = knowledge;
+        this.history = [];
+        this.idleCount = 0;
+        this.resumes = 0;
+        this.stopped = false;
+        this.send = () => undefined;
+        this.prepareEarly = agent.conversationSettings.patience !== "high";
+    }
+    get settings() {
+        return this.agent.conversationSettings;
+    }
+    // ---- lifecycle ----------------------------------------------------------------------
+    /** Greeting: subscribe to the estimate, send initial DTMF, speak the first message. */
+    async greet() {
+        this.maxTimer = setTimeout(() => this.send({ type: "MAX_DURATION" }), this.settings.maxSessionDuration);
+        await this.safely(async () => {
+            await this.session.subscribe(["turn.estimate"]);
+            // AppBridge's Pacing policy owns turn timing; Autopilot only tells it how patient to be.
+            // Recognition hints (names, brands) add to the app's defaults at AppBridge (VO-B3).
+            const hints = this.settings.hints?.length
+                ? this.settings.hints
+                : undefined;
+            if (this.settings.patience !== "normal" || hints)
+                await this.session.listen({
+                    ...(this.settings.patience !== "normal"
+                        ? { patience: this.settings.patience }
+                        : {}),
+                    ...(hints ? { hints } : {})
+                });
+            if (this.settings.initialDtmf)
+                await this.session.playDtmf(this.settings.initialDtmf);
+            if (this.settings.firstMessage) {
+                const r = await this.session.say(this.settings.firstMessage, this.speechOptions());
+                this.history.push({ role: "assistant", content: r.heardText });
+            }
+            this.send({ type: "SPOKEN", source: "greeting" });
+        });
+    }
+    stop() {
+        this.stopped = true;
+        this.abortReply();
+        this.discardPrepared();
+        clearTimeout(this.idleTimer);
+        clearTimeout(this.maxTimer);
+    }
+    // ---- idle -------------------------------------------------------------------------
+    armIdle() {
+        clearTimeout(this.idleTimer);
+        const idle = this.settings.idleOptions;
+        if (idle)
+            this.idleTimer = setTimeout(() => this.send({ type: "IDLE" }), idle.timeout);
+    }
+    disarmIdle() {
+        clearTimeout(this.idleTimer);
+    }
+    /** The next idle timeout ends the call (the goodbye counts as the last announcement). */
+    idleExhausted() {
+        const max = this.settings.idleOptions?.maxTimeoutCount ?? 2;
+        return this.idleCount + 1 >= max;
+    }
+    async sayIdle() {
+        this.idleCount++;
+        await this.safely(async () => {
+            await this.session.say(this.settings.idleOptions.message, this.speechOptions());
+            this.send({ type: "SPOKEN", source: "idle" });
+        });
+    }
+    // ---- pacing -----------------------------------------------------------------------
+    /** turn.likely_ended: start generating silently (when preparing early). */
+    prepare(text) {
+        if (!this.prepareEarly || !text.trim())
+            return;
+        this.discardPrepared();
+        const abort = new AbortController();
+        const chunks = new AsyncQueue_1.AsyncQueue();
+        const prepared = { text, abort, chunks, context: [] };
+        this.prepared = prepared;
+        void (async () => {
+            prepared.context = await this.contextFor(text, abort.signal);
+            if (abort.signal.aborted)
+                return chunks.end();
+            const messages = [
+                ...this.messages(prepared.context),
+                { role: "user", content: text }
+            ];
+            await this.pump(this.traced(messages, abort.signal), chunks);
+        })();
+    }
+    /** turn.resumed: the caller kept talking; the prepared reply is dropped. */
+    resumed() {
+        this.discardPrepared();
+        this.resumes++;
+        if (this.resumes >= 3 && this.prepareEarly) {
+            // A caller who pauses often: stop preparing early for the rest of this call. AppBridge
+            // adapts its own turn timing within the call (TT-B6), so Autopilot does not also change
+            // the patience it asked for (AP-B2).
+            this.prepareEarly = false;
+            this.recorder.note({
+                type: "pacing.adapted",
+                prepare: "on turn.ended",
+                resumes: this.resumes
+            });
+        }
+    }
+    // ---- replies ----------------------------------------------------------------------
+    async clear() {
+        await this.safely(() => this.session.stopSay());
+    }
+    abortReply() {
+        this.reply?.abort();
+        this.reply = undefined;
+    }
+    /** turn.ended: answer the caller, streaming sentences into one speak id. */
+    answer(text) {
+        const previous = this.inflight;
+        this.abortReply();
+        const abort = new AbortController();
+        this.reply = abort;
+        const run = this.respond(text, abort, previous);
+        this.inflight = run;
+        return run;
+    }
+    /** One turn (STT final, LLM, TTS) as an autopilot.turn span. */
+    async respond(text, abort, previous) {
+        const callRef = this.session.callRef;
+        const span = telemetry_1.telemetry.startSpan("autopilot.turn", { [common_1.CALL_REF_ATTR]: callRef }, telemetry_1.telemetry.traceparent(callRef));
+        try {
+            await this.respondTurn(text, abort, previous);
+            span.end({ "turn.cancelled": abort.signal.aborted });
+        }
+        catch (e) {
+            span.end({}, e.message);
+            throw e;
+        }
+        finally {
+            telemetry_1.telemetry.count("fonoster.autopilot.turns", 1);
+        }
+    }
+    async respondTurn(text, abort, previous) {
+        // Let a cancelled reply settle first, so what the caller heard of it is recorded before
+        // this turn (honest history, A6).
+        await previous?.catch(() => undefined);
+        this.idleCount = 0;
+        const turnEnded = Date.now();
+        this.history.push({ role: "user", content: text });
+        if (abort.signal.aborted)
+            return;
+        const trace = {
+            userText: text,
+            replyText: "",
+            heardText: "",
+            interrupted: false,
+            toolCalls: []
+        };
+        let preempt = false;
+        if (this.settings.filler) {
+            preempt = true;
+            void this.session
+                .play(this.settings.filler.url, { interruptible: true })
+                .catch(() => undefined);
+        }
+        try {
+            const prepared = this.takePrepared(text);
+            const context = prepared
+                ? prepared.context
+                : await this.contextFor(text, abort.signal);
+            let source = prepared?.chunks ?? this.startStream(abort.signal, context);
+            for (let round = 0; round < MAX_TOOL_ROUNDS; round++) {
+                const outcome = await this.speakRound(source, abort, preempt, turnEnded, trace);
+                preempt = false;
+                if (abort.signal.aborted &&
+                    !outcome.interrupted &&
+                    !outcome.completed) {
+                    // Cancelled by a new turn: only what the caller already heard is kept.
+                    if (outcome.heard)
+                        this.history.push({ role: "assistant", content: outcome.heard });
+                    return;
+                }
+                this.history.push({
+                    role: "assistant",
+                    content: outcome.heard,
+                    ...(outcome.toolCalls.length ? { tool_calls: outcome.toolCalls } : {})
+                });
+                if (outcome.interrupted ||
+                    !outcome.toolCalls.length ||
+                    abort.signal.aborted)
+                    break;
+                const builtIn = outcome.toolCalls.find((c) => c.name === "hangup" || c.name === "transfer");
+                if (builtIn) {
+                    if (builtIn.name === "transfer")
+                        this.transferCallId = builtIn.id;
+                    this.recorder.turn({
+                        ...trace,
+                        toolCalls: [...trace.toolCalls, { name: builtIn.name, ok: true }]
+                    });
+                    this.send({
+                        type: builtIn.name === "hangup"
+                            ? "HANGUP_REQUESTED"
+                            : "TRANSFER_REQUESTED"
+                    });
+                    return;
+                }
+                await this.runTools(outcome.toolCalls, abort, trace);
+                if (abort.signal.aborted)
+                    return;
+                source = this.startStream(abort.signal, context);
+            }
+            this.recorder.turn(trace);
+            if (this.reply === abort)
+                this.send({ type: "SPOKEN", source: "reply" });
+        }
+        catch (e) {
+            if (abort.signal.aborted ||
+                e instanceof voice_1.SessionEndedError ||
+                this.stopped)
+                return;
+            logger.error(`autopilot reply failed: ${e.message}`);
+            await this.safely(async () => {
+                await this.session.say(this.settings.systemErrorMessage, this.speechOptions());
+                this.send({ type: "SPOKEN", source: "error" });
+            });
+        }
+    }
+    async speakRound(source, abort, preempt, turnEnded, trace) {
+        const text = new AsyncQueue_1.AsyncQueue();
+        let toolCalls = [];
+        let full = "";
+        const pump = (async () => {
+            try {
+                for await (const chunk of source) {
+                    if (abort.signal.aborted)
+                        break;
+                    if (chunk.type === "text") {
+                        if (trace.firstTokenMs === undefined)
+                            trace.firstTokenMs = Date.now() - turnEnded;
+                        full += chunk.text;
+                        text.push(chunk.text);
+                    }
+                    else if (chunk.type === "tool_calls")
+                        toolCalls = chunk.calls;
+                }
+                text.end();
+            }
+            catch (e) {
+                text.fail(e);
+            }
+        })();
+        const iterator = (0, sentences_1.sentences)(text)[Symbol.asyncIterator]();
+        const first = await iterator.next();
+        let heard = "";
+        let interrupted = false;
+        let completed = false;
+        if (!first.done && !abort.signal.aborted) {
+            this.send({ type: "REPLY_SPEAKING" });
+            trace.firstSpeakMs = Date.now() - turnEnded;
+            const pieces = (async function* () {
+                yield first.value;
+                for (;;) {
+                    const next = await iterator.next();
+                    if (next.done)
+                        return;
+                    yield next.value;
+                }
+            })();
+            const result = await this.session.say(pieces, {
+                ...this.speechOptions(),
+                preempt: preempt || undefined
+            });
+            heard = result.heardText;
+            interrupted = result.interrupted;
+            completed = !interrupted;
+            if (interrupted)
+                abort.abort();
+        }
+        await pump;
+        trace.replyText += full;
+        trace.heardText += heard;
+        trace.interrupted || (trace.interrupted = interrupted);
+        return { heard, interrupted, completed, toolCalls };
+    }
+    async runTools(calls, abort, trace) {
+        for (const call of calls) {
+            const tool = this.agent.languageModel.tools.find((t) => t.name === call.name);
+            // The start message is spoken while the tool runs; AppBridge queues the reply after it.
+            if (tool?.requestStartMessage) {
+                void this.session
+                    .say(tool.requestStartMessage, this.speechOptions())
+                    .catch(() => undefined);
+            }
+            let args = {};
+            try {
+                args = call.arguments ? JSON.parse(call.arguments) : {};
+            }
+            catch {
+                args = {};
+            }
+            const outcome = await this.tools.run(call.name, args, abort.signal);
+            trace.toolCalls.push({ name: call.name, ok: outcome.ok });
+            this.history.push({
+                role: "tool",
+                tool_call_id: call.id,
+                content: JSON.stringify(outcome.ok ? outcome.result : { error: outcome.error })
+            });
+        }
+    }
+    // ---- endings ----------------------------------------------------------------------
+    async goodbye() {
+        this.abortReply();
+        this.discardPrepared();
+        clearTimeout(this.idleTimer);
+        await this.safely(async () => {
+            await this.session.say(this.settings.goodbyeMessage, {
+                ...this.speechOptions(),
+                interruptible: false
+            });
+            await this.session.hangup();
+        });
+    }
+    async transfer() {
+        this.abortReply();
+        clearTimeout(this.idleTimer);
+        const options = this.settings.transferOptions;
+        await this.safely(async () => {
+            if (!options) {
+                await this.session.say(this.settings.systemErrorMessage, this.speechOptions());
+                this.send({ type: "SPOKEN", source: "error" });
+                return;
+            }
+            await this.session.say(options.message, {
+                ...this.speechOptions(),
+                interruptible: false
+            });
+            try {
+                await this.session.transfer(options.phoneNumber);
+            }
+            catch (e) {
+                if (!(e instanceof voice_1.SessionError) || e.code !== "transfer_failed")
+                    throw e;
+                await this.transferFailed(options.failedMessage);
+            }
+        });
+    }
+    /**
+     * The transfer failed: say so and keep helping (autopilot "Transfer fails"). The model learns
+     * the outcome as its transfer call's result, and the machine returns to listening.
+     */
+    async transferFailed(message) {
+        const callId = this.transferCallId;
+        this.transferCallId = undefined;
+        if (callId)
+            this.history.push({
+                role: "tool",
+                tool_call_id: callId,
+                content: JSON.stringify({ error: "the transfer failed" })
+            });
+        const r = await this.session.say(message || exports.DEFAULT_TRANSFER_FAILED_MESSAGE, this.speechOptions());
+        this.history.push({ role: "assistant", content: r.heardText });
+        this.send({ type: "SPOKEN", source: "error" });
+    }
+    // ---- internals --------------------------------------------------------------------
+    messages(context = []) {
+        return [
+            { role: "system", content: this.settings.systemPrompt },
+            ...context,
+            ...this.history
+        ];
+    }
+    /** Knowledge for this turn, as a system message for the request only (never in history). */
+    async contextFor(question, signal) {
+        if (!this.knowledge)
+            return [];
+        const timeout = new Promise((resolve) => setTimeout(() => resolve(""), knowledge_1.KNOWLEDGE_TIMEOUT_MS));
+        try {
+            const found = await Promise.race([
+                this.knowledge.find(question, signal),
+                timeout
+            ]);
+            return found?.trim()
+                ? [
+                    {
+                        role: "system",
+                        content: `Relevant information:\n${found.trim()}`
+                    }
+                ]
+                : [];
+        }
+        catch {
+            return [];
+        }
+    }
+    toolSpecs() {
+        const own = this.agent.languageModel.tools.map((t) => ({
+            name: t.name,
+            description: t.description,
+            parameters: t.parameters
+        }));
+        const builtIn = BUILT_IN_TOOLS.filter((b) => b.name !== "transfer" || this.settings.transferOptions);
+        return [...own, ...builtIn];
+    }
+    startStream(signal, context = []) {
+        const chunks = new AsyncQueue_1.AsyncQueue();
+        void this.pump(this.traced(this.messages(context), signal), chunks);
+        return chunks;
+    }
+    /** think.stream inside an autopilot.llm.request span, with request and latency metrics. */
+    async *traced(messages, signal) {
+        const callRef = this.session.callRef;
+        const span = telemetry_1.telemetry.startSpan("autopilot.llm.request", { [common_1.CALL_REF_ATTR]: callRef, "llm.messages": messages.length }, telemetry_1.telemetry.traceparent(callRef));
+        const started = Date.now();
+        let first;
+        let failure;
+        try {
+            for await (const chunk of this.think.stream(messages, this.toolSpecs(), signal)) {
+                first ?? (first = Date.now() - started);
+                yield chunk;
+            }
+        }
+        catch (e) {
+            failure = e.message;
+            throw e;
+        }
+        finally {
+            telemetry_1.telemetry.count("fonoster.autopilot.llm.requests", 1);
+            if (first !== undefined)
+                telemetry_1.telemetry.count("fonoster.autopilot.llm.latency_ms", first);
+            span.end({
+                "llm.first_chunk_ms": first ?? -1,
+                "llm.cancelled": signal.aborted
+            }, failure);
+        }
+    }
+    takePrepared(text) {
+        const p = this.prepared;
+        this.prepared = undefined;
+        if (!p)
+            return undefined;
+        if (p.text === text && !p.abort.signal.aborted) {
+            // The reply prepared on turn.likely_ended is still valid; adopt its abort controller.
+            this.reply?.signal.addEventListener("abort", () => p.abort.abort(), {
+                once: true
+            });
+            return p;
+        }
+        p.abort.abort();
+        return undefined;
+    }
+    discardPrepared() {
+        this.prepared?.abort.abort();
+        this.prepared = undefined;
+    }
+    async pump(stream, into) {
+        try {
+            for await (const chunk of stream)
+                into.push(chunk);
+            into.end();
+        }
+        catch (e) {
+            into.fail(e);
+        }
+    }
+    speechOptions() {
+        return {
+            voice: this.settings.voice,
+            language: this.settings.language,
+            interruptible: this.settings.allowUserBargeIn
+        };
+    }
+    /** Runs session work; a session that has ended stops everything quietly. */
+    async safely(fn) {
+        if (this.stopped)
+            return;
+        try {
+            await fn();
+        }
+        catch (e) {
+            if (e instanceof voice_1.SessionEndedError || this.stopped)
+                return;
+            logger.error(`autopilot session error: ${e.message}`);
+        }
+    }
+}
+exports.Agent = Agent;

package/dist/{loadAssistantFromAPI.d.ts → session/AsyncQueue.d.ts}

@@ -16,10 +16,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { VoiceRequest } from "@fonoster/common";
-import { AssistantConfig } from "./assistants";
-declare function loadAssistantFromAPI(req: VoiceRequest, integrations: {
-    productRef: string;
-    credentials: Record<string, unknown>;
-}[]): Promise<AssistantConfig>;
-export { loadAssistantFromAPI };
+/** A push-based async iterable: producers push, one consumer iterates. */
+export declare class AsyncQueue<T> implements AsyncIterable<T> {
+    private items;
+    private waiting?;
+    private failWaiting?;
+    private ended;
+    private error?;
+    push(item: T): void;
+    end(): void;
+    fail(error: Error): void;
+    [Symbol.asyncIterator](): AsyncIterator<T>;
+}

package/dist/session/AsyncQueue.js

@@ -0,0 +1,67 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AsyncQueue = void 0;
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** A push-based async iterable: producers push, one consumer iterates. */
+class AsyncQueue {
+    constructor() {
+        this.items = [];
+        this.ended = false;
+    }
+    push(item) {
+        if (this.ended)
+            return;
+        if (this.waiting) {
+            const resolve = this.waiting;
+            this.waiting = undefined;
+            resolve({ value: item, done: false });
+        }
+        else
+            this.items.push(item);
+    }
+    end() {
+        this.ended = true;
+        this.waiting?.({ value: undefined, done: true });
+        this.waiting = undefined;
+    }
+    fail(error) {
+        this.error = error;
+        this.ended = true;
+        this.failWaiting?.(error);
+        this.waiting = undefined;
+    }
+    [Symbol.asyncIterator]() {
+        return {
+            next: () => {
+                if (this.items.length)
+                    return Promise.resolve({ value: this.items.shift(), done: false });
+                if (this.error)
+                    return Promise.reject(this.error);
+                if (this.ended)
+                    return Promise.resolve({ value: undefined, done: true });
+                return new Promise((resolve, reject) => {
+                    this.waiting = resolve;
+                    this.failWaiting = reject;
+                });
+            }
+        };
+    }
+}
+exports.AsyncQueue = AsyncQueue;

package/dist/{machine/guards/index.d.ts → session/config.d.ts}

@@ -16,5 +16,5 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export { idleTimeoutCountExceedsMax } from "./idleTimeoutCountExceedsMax";
-export { hasSpeechResult } from "./hasSpeechResult";
+export { sessionAgentSchema, sessionConversationSettingsSchema, sessionLanguageModelSchema } from "@fonoster/common";
+export type { SessionAgent, SessionConversationSettings, SessionLanguageModel, ToolDefinition } from "@fonoster/common";

package/dist/session/config.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sessionLanguageModelSchema = exports.sessionConversationSettingsSchema = exports.sessionAgentSchema = void 0;
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// The session agent schema lives in @fonoster/common (rework-autopilot 5.1), so the control
+// plane validates agents with the same rules Autopilot runs with.
+var common_1 = require("@fonoster/common");
+Object.defineProperty(exports, "sessionAgentSchema", { enumerable: true, get: function () { return common_1.sessionAgentSchema; } });
+Object.defineProperty(exports, "sessionConversationSettingsSchema", { enumerable: true, get: function () { return common_1.sessionConversationSettingsSchema; } });
+Object.defineProperty(exports, "sessionLanguageModelSchema", { enumerable: true, get: function () { return common_1.sessionLanguageModelSchema; } });