@fonoster/autopilot 0.27.0 → 0.30.0

package/dist/session/Agent.d.ts

@@ -0,0 +1,138 @@
+import { SessionAssistant } from "./config";
+import { KnowledgePort } from "./knowledge";
+import { ChatMessage, Think } from "./think";
+import { ToolPort } from "./tools";
+import type { Session } from "@fonoster/voice";
+export type SpokenSource = "greeting" | "reply" | "idle" | "error";
+export type AgentEvent = {
+    type: "TURN_USER_STARTED";
+} | {
+    type: "TURN_LIKELY_ENDED";
+    text: string;
+} | {
+    type: "TURN_RESUMED";
+} | {
+    type: "TURN_ENDED";
+    text: string;
+} | {
+    type: "TURN_BACKCHANNEL";
+} | {
+    type: "REPLY_SPEAKING";
+}
+/** An output finished; `source` says which, so stale events from an earlier phase are ignored. */
+ | {
+    type: "SPOKEN";
+    source: SpokenSource;
+} | {
+    type: "IDLE";
+} | {
+    type: "MAX_DURATION";
+} | {
+    type: "HANGUP_REQUESTED";
+} | {
+    type: "TRANSFER_REQUESTED";
+} | {
+    type: "SESSION_END";
+};
+/** One turn's trace for the call record (rework-autopilot A10). */
+export type TurnTrace = {
+    userText: string;
+    replyText: string;
+    heardText: string;
+    interrupted: boolean;
+    toolCalls: {
+        name: string;
+        ok: boolean;
+    }[];
+    /** ms from turn.ended to the first token, and to the first speak sent. */
+    firstTokenMs?: number;
+    firstSpeakMs?: number;
+};
+export interface Recorder {
+    turn(trace: TurnTrace): void;
+    note(event: Record<string, unknown>): void;
+}
+/** Writes turn traces and notes to the call record through the session (record.note). */
+export declare function sessionRecorder(session: Session): Recorder;
+export declare const logRecorder: Recorder;
+/** The side effects of Autopilot: everything the state machine asks it to do on the session. */
+export declare class Agent {
+    private readonly session;
+    private readonly assistant;
+    private readonly think;
+    private readonly tools;
+    private readonly recorder;
+    private readonly knowledge?;
+    readonly history: ChatMessage[];
+    private reply?;
+    private inflight?;
+    private prepared?;
+    private idleTimer?;
+    private maxTimer?;
+    private idleCount;
+    private resumes;
+    /** Whether a reply is prepared on turn.likely_ended (off at high patience or once adapted). */
+    private prepareEarly;
+    private stopped;
+    send: (event: AgentEvent) => void;
+    constructor(session: Session, assistant: SessionAssistant, think: Think, tools: ToolPort, recorder?: Recorder, knowledge?: KnowledgePort);
+    get settings(): {
+        firstMessage?: string;
+        systemPrompt?: string;
+        goodbyeMessage?: string;
+        systemErrorMessage?: string;
+        initialDtmf?: string;
+        maxSessionDuration?: number;
+        transferOptions?: {
+            message?: string;
+            phoneNumber?: string;
+        };
+        idleOptions?: {
+            message?: string;
+            timeout?: number;
+            maxTimeoutCount?: number;
+        };
+        allowUserBargeIn?: boolean;
+        patience?: "low" | "normal" | "high";
+        hints?: string[];
+        filler?: {
+            url?: string;
+        };
+        voice?: string;
+        language?: string;
+        vad?: any;
+        maxSpeechWaitTimeout?: any;
+    };
+    /** Greeting: subscribe to the estimate, send initial DTMF, speak the first message. */
+    greet(): Promise<void>;
+    stop(): void;
+    armIdle(): void;
+    disarmIdle(): void;
+    /** The next idle timeout ends the call (the goodbye counts as the last announcement). */
+    idleExhausted(): boolean;
+    sayIdle(): Promise<void>;
+    /** turn.likely_ended: start generating silently (when preparing early). */
+    prepare(text: string): void;
+    /** turn.resumed: the caller kept talking; the prepared reply is dropped. */
+    resumed(): void;
+    clear(): Promise<void>;
+    abortReply(): void;
+    /** turn.ended: answer the caller, streaming sentences into one speak id. */
+    answer(text: string): Promise<void>;
+    private respond;
+    private speakRound;
+    private runTools;
+    goodbye(): Promise<void>;
+    transfer(): Promise<void>;
+    private messages;
+    /** Knowledge for this turn, as a system message for the request only (never in history). */
+    private contextFor;
+    private toolSpecs;
+    private startStream;
+    private takePrepared;
+    private discardPrepared;
+    private pump;
+    private speechOptions;
+    /** Runs session work; a session that has ended stops everything quietly. */
+    private safely;
+}

package/dist/session/Agent.js

@@ -0,0 +1,474 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Agent = exports.logRecorder = void 0;
+exports.sessionRecorder = sessionRecorder;
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const logger_1 = require("@fonoster/logger");
+const voice_1 = require("@fonoster/voice");
+const AsyncQueue_1 = require("./AsyncQueue");
+const knowledge_1 = require("./knowledge");
+const sentences_1 = require("./sentences");
+const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
+const MAX_TOOL_ROUNDS = 4;
+const BUILT_IN_TOOLS = [
+    {
+        name: "hangup",
+        description: "End the call when the conversation is over.",
+        parameters: { type: "object", properties: {} }
+    },
+    {
+        name: "transfer",
+        description: "Transfer the caller to an operator (a person).",
+        parameters: { type: "object", properties: {} }
+    }
+];
+/** Writes turn traces and notes to the call record through the session (record.note). */
+function sessionRecorder(session) {
+    const write = (data) => void session.note(data).catch(() => undefined);
+    return {
+        turn: (trace) => write({ kind: "autopilot.turn", ...trace }),
+        note: (event) => write({ kind: "autopilot.note", ...event })
+    };
+}
+exports.logRecorder = {
+    turn: (trace) => logger.verbose("autopilot turn", trace),
+    note: (event) => logger.verbose("autopilot note", event)
+};
+/** The side effects of Autopilot: everything the state machine asks it to do on the session. */
+class Agent {
+    constructor(session, assistant, think, tools, recorder = exports.logRecorder, knowledge) {
+        this.session = session;
+        this.assistant = assistant;
+        this.think = think;
+        this.tools = tools;
+        this.recorder = recorder;
+        this.knowledge = knowledge;
+        this.history = [];
+        this.idleCount = 0;
+        this.resumes = 0;
+        this.stopped = false;
+        this.send = () => undefined;
+        this.prepareEarly = assistant.conversationSettings.patience !== "high";
+    }
+    get settings() {
+        return this.assistant.conversationSettings;
+    }
+    // ---- lifecycle ----------------------------------------------------------------------
+    /** Greeting: subscribe to the estimate, send initial DTMF, speak the first message. */
+    async greet() {
+        this.maxTimer = setTimeout(() => this.send({ type: "MAX_DURATION" }), this.settings.maxSessionDuration);
+        await this.safely(async () => {
+            await this.session.subscribe(["turn.estimate"]);
+            // The gateway's Pacing policy owns turn timing; Autopilot only tells it how patient to be.
+            // Recognition hints (names, brands) add to the app's defaults at the gateway (VO-B3).
+            const hints = this.settings.hints?.length
+                ? this.settings.hints
+                : undefined;
+            if (this.settings.patience !== "normal" || hints)
+                await this.session.listen({
+                    ...(this.settings.patience !== "normal"
+                        ? { patience: this.settings.patience }
+                        : {}),
+                    ...(hints ? { hints } : {})
+                });
+            if (this.settings.initialDtmf)
+                await this.session.playDtmf(this.settings.initialDtmf);
+            if (this.settings.firstMessage) {
+                const r = await this.session.say(this.settings.firstMessage, this.speechOptions());
+                this.history.push({ role: "assistant", content: r.heardText });
+            }
+            this.send({ type: "SPOKEN", source: "greeting" });
+        });
+    }
+    stop() {
+        this.stopped = true;
+        this.abortReply();
+        this.discardPrepared();
+        clearTimeout(this.idleTimer);
+        clearTimeout(this.maxTimer);
+    }
+    // ---- idle -------------------------------------------------------------------------
+    armIdle() {
+        clearTimeout(this.idleTimer);
+        const idle = this.settings.idleOptions;
+        if (idle)
+            this.idleTimer = setTimeout(() => this.send({ type: "IDLE" }), idle.timeout);
+    }
+    disarmIdle() {
+        clearTimeout(this.idleTimer);
+    }
+    /** The next idle timeout ends the call (the goodbye counts as the last announcement). */
+    idleExhausted() {
+        const max = this.settings.idleOptions?.maxTimeoutCount ?? 2;
+        return this.idleCount + 1 >= max;
+    }
+    async sayIdle() {
+        this.idleCount++;
+        await this.safely(async () => {
+            await this.session.say(this.settings.idleOptions.message, this.speechOptions());
+            this.send({ type: "SPOKEN", source: "idle" });
+        });
+    }
+    // ---- pacing -----------------------------------------------------------------------
+    /** turn.likely_ended: start generating silently (when preparing early). */
+    prepare(text) {
+        if (!this.prepareEarly || !text.trim())
+            return;
+        this.discardPrepared();
+        const abort = new AbortController();
+        const chunks = new AsyncQueue_1.AsyncQueue();
+        const prepared = { text, abort, chunks, context: [] };
+        this.prepared = prepared;
+        void (async () => {
+            prepared.context = await this.contextFor(text, abort.signal);
+            if (abort.signal.aborted)
+                return chunks.end();
+            const messages = [
+                ...this.messages(prepared.context),
+                { role: "user", content: text }
+            ];
+            await this.pump(this.think.stream(messages, this.toolSpecs(), abort.signal), chunks);
+        })();
+    }
+    /** turn.resumed: the caller kept talking; the prepared reply is dropped. */
+    resumed() {
+        this.discardPrepared();
+        this.resumes++;
+        if (this.resumes >= 3 && this.prepareEarly) {
+            // A caller who pauses often: stop preparing early for the rest of this call. The gateway
+            // adapts its own turn timing within the call (TT-B6), so Autopilot does not also change
+            // the patience it asked for (AP-B2).
+            this.prepareEarly = false;
+            this.recorder.note({
+                type: "pacing.adapted",
+                prepare: "on turn.ended",
+                resumes: this.resumes
+            });
+        }
+    }
+    // ---- replies ----------------------------------------------------------------------
+    async clear() {
+        await this.safely(() => this.session.stopSay());
+    }
+    abortReply() {
+        this.reply?.abort();
+        this.reply = undefined;
+    }
+    /** turn.ended: answer the caller, streaming sentences into one speak id. */
+    answer(text) {
+        const previous = this.inflight;
+        this.abortReply();
+        const abort = new AbortController();
+        this.reply = abort;
+        const run = this.respond(text, abort, previous);
+        this.inflight = run;
+        return run;
+    }
+    async respond(text, abort, previous) {
+        // Let a cancelled reply settle first, so what the caller heard of it is recorded before
+        // this turn (honest history, A6).
+        await previous?.catch(() => undefined);
+        this.idleCount = 0;
+        const turnEnded = Date.now();
+        this.history.push({ role: "user", content: text });
+        if (abort.signal.aborted)
+            return;
+        const trace = {
+            userText: text,
+            replyText: "",
+            heardText: "",
+            interrupted: false,
+            toolCalls: []
+        };
+        let preempt = false;
+        if (this.settings.filler) {
+            preempt = true;
+            void this.session
+                .play(this.settings.filler.url, { interruptible: true })
+                .catch(() => undefined);
+        }
+        try {
+            const prepared = this.takePrepared(text);
+            const context = prepared
+                ? prepared.context
+                : await this.contextFor(text, abort.signal);
+            let source = prepared?.chunks ?? this.startStream(abort.signal, context);
+            for (let round = 0; round < MAX_TOOL_ROUNDS; round++) {
+                const outcome = await this.speakRound(source, abort, preempt, turnEnded, trace);
+                preempt = false;
+                if (abort.signal.aborted &&
+                    !outcome.interrupted &&
+                    !outcome.completed) {
+                    // Cancelled by a new turn: only what the caller already heard is kept.
+                    if (outcome.heard)
+                        this.history.push({ role: "assistant", content: outcome.heard });
+                    return;
+                }
+                this.history.push({
+                    role: "assistant",
+                    content: outcome.heard,
+                    ...(outcome.toolCalls.length ? { tool_calls: outcome.toolCalls } : {})
+                });
+                if (outcome.interrupted ||
+                    !outcome.toolCalls.length ||
+                    abort.signal.aborted)
+                    break;
+                const builtIn = outcome.toolCalls.find((c) => c.name === "hangup" || c.name === "transfer");
+                if (builtIn) {
+                    this.recorder.turn({
+                        ...trace,
+                        toolCalls: [...trace.toolCalls, { name: builtIn.name, ok: true }]
+                    });
+                    this.send({
+                        type: builtIn.name === "hangup"
+                            ? "HANGUP_REQUESTED"
+                            : "TRANSFER_REQUESTED"
+                    });
+                    return;
+                }
+                await this.runTools(outcome.toolCalls, abort, trace);
+                if (abort.signal.aborted)
+                    return;
+                source = this.startStream(abort.signal, context);
+            }
+            this.recorder.turn(trace);
+            if (this.reply === abort)
+                this.send({ type: "SPOKEN", source: "reply" });
+        }
+        catch (e) {
+            if (abort.signal.aborted ||
+                e instanceof voice_1.SessionEndedError ||
+                this.stopped)
+                return;
+            logger.error(`autopilot reply failed: ${e.message}`);
+            await this.safely(async () => {
+                await this.session.say(this.settings.systemErrorMessage, this.speechOptions());
+                this.send({ type: "SPOKEN", source: "error" });
+            });
+        }
+    }
+    async speakRound(source, abort, preempt, turnEnded, trace) {
+        const text = new AsyncQueue_1.AsyncQueue();
+        let toolCalls = [];
+        let full = "";
+        const pump = (async () => {
+            try {
+                for await (const chunk of source) {
+                    if (abort.signal.aborted)
+                        break;
+                    if (chunk.type === "text") {
+                        if (trace.firstTokenMs === undefined)
+                            trace.firstTokenMs = Date.now() - turnEnded;
+                        full += chunk.text;
+                        text.push(chunk.text);
+                    }
+                    else
+                        toolCalls = chunk.calls;
+                }
+                text.end();
+            }
+            catch (e) {
+                text.fail(e);
+            }
+        })();
+        const iterator = (0, sentences_1.sentences)(text)[Symbol.asyncIterator]();
+        const first = await iterator.next();
+        let heard = "";
+        let interrupted = false;
+        let completed = false;
+        if (!first.done && !abort.signal.aborted) {
+            this.send({ type: "REPLY_SPEAKING" });
+            trace.firstSpeakMs = Date.now() - turnEnded;
+            const pieces = (async function* () {
+                yield first.value;
+                for (;;) {
+                    const next = await iterator.next();
+                    if (next.done)
+                        return;
+                    yield next.value;
+                }
+            })();
+            const result = await this.session.say(pieces, {
+                ...this.speechOptions(),
+                preempt: preempt || undefined
+            });
+            heard = result.heardText;
+            interrupted = result.interrupted;
+            completed = !interrupted;
+            if (interrupted)
+                abort.abort();
+        }
+        await pump;
+        trace.replyText += full;
+        trace.heardText += heard;
+        trace.interrupted || (trace.interrupted = interrupted);
+        return { heard, interrupted, completed, toolCalls };
+    }
+    async runTools(calls, abort, trace) {
+        for (const call of calls) {
+            const tool = this.assistant.languageModel.tools.find((t) => t.name === call.name);
+            // The start message is spoken while the tool runs; the gateway queues the reply after it.
+            if (tool?.requestStartMessage) {
+                void this.session
+                    .say(tool.requestStartMessage, this.speechOptions())
+                    .catch(() => undefined);
+            }
+            let args = {};
+            try {
+                args = call.arguments ? JSON.parse(call.arguments) : {};
+            }
+            catch {
+                args = {};
+            }
+            const outcome = await this.tools.run(call.name, args, abort.signal);
+            trace.toolCalls.push({ name: call.name, ok: outcome.ok });
+            this.history.push({
+                role: "tool",
+                tool_call_id: call.id,
+                content: JSON.stringify(outcome.ok ? outcome.result : { error: outcome.error })
+            });
+        }
+    }
+    // ---- endings ----------------------------------------------------------------------
+    async goodbye() {
+        this.abortReply();
+        this.discardPrepared();
+        clearTimeout(this.idleTimer);
+        await this.safely(async () => {
+            await this.session.say(this.settings.goodbyeMessage, {
+                ...this.speechOptions(),
+                interruptible: false
+            });
+            await this.session.hangup();
+        });
+    }
+    async transfer() {
+        this.abortReply();
+        clearTimeout(this.idleTimer);
+        const options = this.settings.transferOptions;
+        await this.safely(async () => {
+            if (!options) {
+                await this.session.say(this.settings.systemErrorMessage, this.speechOptions());
+                this.send({ type: "SPOKEN", source: "error" });
+                return;
+            }
+            await this.session.say(options.message, {
+                ...this.speechOptions(),
+                interruptible: false
+            });
+            await this.session.transfer(options.phoneNumber);
+        });
+    }
+    // ---- internals --------------------------------------------------------------------
+    messages(context = []) {
+        return [
+            { role: "system", content: this.settings.systemPrompt },
+            ...context,
+            ...this.history
+        ];
+    }
+    /** Knowledge for this turn, as a system message for the request only (never in history). */
+    async contextFor(question, signal) {
+        if (!this.knowledge)
+            return [];
+        const timeout = new Promise((resolve) => setTimeout(() => resolve(""), knowledge_1.KNOWLEDGE_TIMEOUT_MS));
+        try {
+            const found = await Promise.race([
+                this.knowledge.find(question, signal),
+                timeout
+            ]);
+            return found?.trim()
+                ? [
+                    {
+                        role: "system",
+                        content: `Relevant information:\n${found.trim()}`
+                    }
+                ]
+                : [];
+        }
+        catch {
+            return [];
+        }
+    }
+    toolSpecs() {
+        const own = this.assistant.languageModel.tools.map((t) => ({
+            name: t.name,
+            description: t.description,
+            parameters: t.parameters
+        }));
+        const builtIn = BUILT_IN_TOOLS.filter((b) => b.name !== "transfer" || this.settings.transferOptions);
+        return [...own, ...builtIn];
+    }
+    startStream(signal, context = []) {
+        const chunks = new AsyncQueue_1.AsyncQueue();
+        void this.pump(this.think.stream(this.messages(context), this.toolSpecs(), signal), chunks);
+        return chunks;
+    }
+    takePrepared(text) {
+        const p = this.prepared;
+        this.prepared = undefined;
+        if (!p)
+            return undefined;
+        if (p.text === text && !p.abort.signal.aborted) {
+            // The reply prepared on turn.likely_ended is still valid; adopt its abort controller.
+            this.reply?.signal.addEventListener("abort", () => p.abort.abort(), {
+                once: true
+            });
+            return p;
+        }
+        p.abort.abort();
+        return undefined;
+    }
+    discardPrepared() {
+        this.prepared?.abort.abort();
+        this.prepared = undefined;
+    }
+    async pump(stream, into) {
+        try {
+            for await (const chunk of stream)
+                into.push(chunk);
+            into.end();
+        }
+        catch (e) {
+            into.fail(e);
+        }
+    }
+    speechOptions() {
+        return {
+            voice: this.settings.voice,
+            language: this.settings.language,
+            interruptible: this.settings.allowUserBargeIn
+        };
+    }
+    /** Runs session work; a session that has ended stops everything quietly. */
+    async safely(fn) {
+        if (this.stopped)
+            return;
+        try {
+            await fn();
+        }
+        catch (e) {
+            if (e instanceof voice_1.SessionEndedError || this.stopped)
+                return;
+            logger.error(`autopilot session error: ${e.message}`);
+        }
+    }
+}
+exports.Agent = Agent;

package/dist/{loadAssistantFromAPI.d.ts → session/AsyncQueue.d.ts}

@@ -16,10 +16,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { VoiceRequest } from "@fonoster/common";
-import { AssistantConfig } from "./assistants";
-declare function loadAssistantFromAPI(req: VoiceRequest, integrations: {
-    productRef: string;
-    credentials: Record<string, unknown>;
-}[]): Promise<AssistantConfig>;
-export { loadAssistantFromAPI };
+/** A push-based async iterable: producers push, one consumer iterates. */
+export declare class AsyncQueue<T> implements AsyncIterable<T> {
+    private items;
+    private waiting?;
+    private failWaiting?;
+    private ended;
+    private error?;
+    push(item: T): void;
+    end(): void;
+    fail(error: Error): void;
+    [Symbol.asyncIterator](): AsyncIterator<T>;
+}

package/dist/session/AsyncQueue.js

@@ -0,0 +1,67 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AsyncQueue = void 0;
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** A push-based async iterable: producers push, one consumer iterates. */
+class AsyncQueue {
+    constructor() {
+        this.items = [];
+        this.ended = false;
+    }
+    push(item) {
+        if (this.ended)
+            return;
+        if (this.waiting) {
+            const resolve = this.waiting;
+            this.waiting = undefined;
+            resolve({ value: item, done: false });
+        }
+        else
+            this.items.push(item);
+    }
+    end() {
+        this.ended = true;
+        this.waiting?.({ value: undefined, done: true });
+        this.waiting = undefined;
+    }
+    fail(error) {
+        this.error = error;
+        this.ended = true;
+        this.failWaiting?.(error);
+        this.waiting = undefined;
+    }
+    [Symbol.asyncIterator]() {
+        return {
+            next: () => {
+                if (this.items.length)
+                    return Promise.resolve({ value: this.items.shift(), done: false });
+                if (this.error)
+                    return Promise.reject(this.error);
+                if (this.ended)
+                    return Promise.resolve({ value: undefined, done: true });
+                return new Promise((resolve, reject) => {
+                    this.waiting = resolve;
+                    this.failWaiting = reject;
+                });
+            }
+        };
+    }
+}
+exports.AsyncQueue = AsyncQueue;