@komaa/standin-sdk 0.1.0

package/dist/lane.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Caller audio in, a spoken answer out, for agents that are not speech to speech.
+ *
+ * A speech-to-speech model hears the caller and talks back, and a plugin for one
+ * is mostly a socket. Most agents are not that. They read text, they write text,
+ * and getting them onto a phone call means four things in a row: work out where
+ * the caller stopped talking, turn that into words, ask the agent, and say the
+ * answer back at the rate a call consumes audio.
+ *
+ * `voice.ts` has each of those pieces. This is the thing that runs them in
+ * order, holds the turn together, and gets the awkward parts right:
+ *
+ * **One turn at a time.** An agent asked two questions at once answers neither
+ * well. A new utterance supersedes the one in flight rather than racing it.
+ *
+ * **Barge-in actually stops the answer.** Somebody who interrupts has stopped
+ * listening, and a worker that keeps streaming a paragraph at them is talking to
+ * nobody. The buffered audio is dropped at the same moment the new utterance
+ * opens, not when the old one finishes.
+ *
+ * **A silence is not a turn.** A cough, a door, a second of traffic: the
+ * segmenter opens on any loud frame, and waking the agent for every one of them
+ * is a bill and a caller being answered at random.
+ *
+ * **Nothing here throws into the call.** A provider that fails says so, in a
+ * sentence, out loud. Silence is the one thing a caller cannot interpret.
+ *
+ * Identical in shape to the Python SDK's `standin.lane`.
+ */
+import type { CallSession } from "./handler.js";
+import { UtteranceSegmenter } from "./voice.js";
+/**
+ * What the caller hears when a step of the lane fails.
+ *
+ * Spoken, not logged and swallowed. Somebody on a phone call cannot tell a
+ * broken transcriber from an agent that is thinking, and will keep waiting.
+ */
+export declare const TROUBLE_HEARING = "Sorry, I did not catch that.";
+export declare const TROUBLE_ANSWERING = "Sorry, I am having trouble answering just now.";
+export declare const TROUBLE_SPEAKING = "Sorry, I am having trouble speaking just now.";
+/** Caller audio (PCM16 mono, 16 kHz) to words. Empty means nothing was said. */
+export type Transcribe = (pcm: Buffer) => Promise<string>;
+/** Words to an answer. Either the whole thing, or sentences as they are written. */
+export type Answer = (text: string) => Promise<string> | AsyncIterable<string>;
+/** An answer to speech (PCM16 mono, 16 kHz). Either one buffer, or chunks. */
+export type Synthesize = (text: string) => Promise<Buffer> | AsyncIterable<Buffer>;
+/** One exchange, after it is over. */
+export interface VoiceTurn {
+    readonly heard: string;
+    readonly said: string;
+    /**
+     * Whether the caller cut the answer short. Not a failure: it is the most
+     * common way a real conversation goes.
+     */
+    readonly interrupted: boolean;
+    readonly error?: string;
+}
+/** Options for {@link VoiceLane}. */
+export interface VoiceLaneOptions {
+    segmenter?: UtteranceSegmenter;
+    onTurn?: (turn: VoiceTurn) => void;
+}
+/**
+ * Runs one call's worth of listen, transcribe, answer, speak.
+ *
+ * Built by a plugin, which supplies the three steps. Everything about pacing,
+ * interruption and turn-taking is here, because getting those wrong is what
+ * makes a working provider sound broken.
+ */
+export declare class VoiceLane {
+    #private;
+    constructor(session: CallSession, transcribe: Transcribe, answer: Answer, synthesize: Synthesize, options?: VoiceLaneOptions);
+    /** Whether the agent is talking right now. */
+    get speaking(): boolean;
+    /** Whether a turn is in flight, including the model's own thinking. */
+    get busy(): boolean;
+    /** The turn in flight, if any. Await it to let one finish. */
+    get turn(): Promise<void> | undefined;
+    /** Take one frame of caller audio. Never throws, never blocks. */
+    feed(pcm: Buffer): Promise<void>;
+    /** Stop talking, immediately. The caller interrupted. */
+    bargeIn(): Promise<void>;
+    /**
+     * Speak a line the agent did not have to be asked for.
+     *
+     * A greeting, a handover, something that arrived from outside the call.
+     */
+    say(text: string): Promise<VoiceTurn>;
+    /** Stop everything. Called once, on teardown. */
+    aclose(): Promise<void>;
+}
+//# sourceMappingURL=lane.d.ts.map

package/dist/lane.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lane.d.ts","sourceRoot":"","sources":["../src/lane.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,OAAO,EAAiB,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE/D;;;;;GAKG;AACH,eAAO,MAAM,eAAe,iCAAiC,CAAC;AAC9D,eAAO,MAAM,iBAAiB,mDACoB,CAAC;AACnD,eAAO,MAAM,gBAAgB,kDAAkD,CAAC;AAEhF,gFAAgF;AAChF,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAE1D,oFAAoF;AACpF,MAAM,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;AAE/E,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,CACvB,IAAI,EAAE,MAAM,KACT,OAAO,CAAC,MAAM,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;AAE7C,sCAAsC;AACtC,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,qCAAqC;AACrC,MAAM,WAAW,gBAAgB;IAC/B,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,IAAI,CAAC;CACpC;AAED;;;;;;GAMG;AACH,qBAAa,SAAS;;gBAalB,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACtB,OAAO,GAAE,gBAAqB;IAWhC,8CAA8C;IAC9C,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,uEAAuE;IACvE,IAAI,IAAI,IAAI,OAAO,CAElB;IAED,8DAA8D;IAC9D,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAEpC;IAED,kEAAkE;IAC5D,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IActC,yDAAyD;IACnD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAS9B;;;;OAIG;IACG,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAa3C,iDAAiD;IAC3C,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAyK9B"}

package/dist/lane.js

@@ -0,0 +1,269 @@
+// Copyright (c) 2026 Komaa DigiTech
+// SPDX-License-Identifier: MIT
+import { logger } from "./log.js";
+import { PacedPlayback, UtteranceSegmenter } from "./voice.js";
+/**
+ * What the caller hears when a step of the lane fails.
+ *
+ * Spoken, not logged and swallowed. Somebody on a phone call cannot tell a
+ * broken transcriber from an agent that is thinking, and will keep waiting.
+ */
+export const TROUBLE_HEARING = "Sorry, I did not catch that.";
+export const TROUBLE_ANSWERING = "Sorry, I am having trouble answering just now.";
+export const TROUBLE_SPEAKING = "Sorry, I am having trouble speaking just now.";
+/**
+ * Runs one call's worth of listen, transcribe, answer, speak.
+ *
+ * Built by a plugin, which supplies the three steps. Everything about pacing,
+ * interruption and turn-taking is here, because getting those wrong is what
+ * makes a working provider sound broken.
+ */
+export class VoiceLane {
+    #session;
+    #transcribe;
+    #answer;
+    #synthesize;
+    #segmenter;
+    #onTurn;
+    #playback;
+    #turn;
+    #generation = 0;
+    #closed = false;
+    constructor(session, transcribe, answer, synthesize, options = {}) {
+        this.#session = session;
+        this.#transcribe = transcribe;
+        this.#answer = answer;
+        this.#synthesize = synthesize;
+        this.#segmenter = options.segmenter ?? new UtteranceSegmenter();
+        this.#onTurn = options.onTurn;
+        this.#playback = new PacedPlayback((pcm) => session.sendAudio(pcm));
+    }
+    /** Whether the agent is talking right now. */
+    get speaking() {
+        return this.#playback.playing;
+    }
+    /** Whether a turn is in flight, including the model's own thinking. */
+    get busy() {
+        return this.#turn !== undefined;
+    }
+    /** The turn in flight, if any. Await it to let one finish. */
+    get turn() {
+        return this.#turn;
+    }
+    /** Take one frame of caller audio. Never throws, never blocks. */
+    async feed(pcm) {
+        if (this.#closed)
+            return;
+        const wasSpeaking = this.#segmenter.speaking;
+        const utterance = this.#segmenter.feed(pcm);
+        if (!wasSpeaking && this.#segmenter.speaking && this.#playback.playing) {
+            // The caller started over the top of the answer. Drop what is buffered
+            // NOW rather than when this utterance finishes: the extra second of
+            // talking at somebody who has stopped listening is the whole difference
+            // between a call that feels alive and one that does not.
+            await this.bargeIn();
+        }
+        if (utterance !== undefined)
+            this.#begin(utterance);
+    }
+    /** Stop talking, immediately. The caller interrupted. */
+    async bargeIn() {
+        this.#playback.cancel();
+        try {
+            await this.#session.cancelPlayback();
+        }
+        catch {
+            // Best effort. A failed cancel is not worth ending a call over.
+        }
+    }
+    /**
+     * Speak a line the agent did not have to be asked for.
+     *
+     * A greeting, a handover, something that arrived from outside the call.
+     */
+    async say(text) {
+        // Same guard as feed(). A line handed in after teardown would otherwise
+        // synthesize and send on a call that has already gone.
+        if (this.#closed)
+            return {
+                heard: "",
+                said: "",
+                interrupted: false,
+                error: "the call has ended",
+            };
+        return await this.#speak(text, "");
+    }
+    /** Stop everything. Called once, on teardown. */
+    async aclose() {
+        this.#closed = true;
+        this.#generation += 1;
+        this.#playback.cancel();
+        this.#segmenter.reset();
+        const turn = this.#turn;
+        this.#turn = undefined;
+        if (turn !== undefined)
+            await turn.catch(() => undefined);
+    }
+    // ---- one turn ---------------------------------------------------------
+    /**
+     * Start a turn, superseding whatever was in flight.
+     *
+     * Detached on purpose: this is reached from the receive path of a live call,
+     * and awaiting a model there stops frames arriving.
+     */
+    #begin(utterance) {
+        // One turn at a time. An agent asked two questions at once answers neither
+        // well, and both answers would be spoken over each other. JavaScript cannot
+        // cancel a promise, so the older turn is retired by generation: it runs to
+        // completion but nothing it produces is ever spoken.
+        const generation = ++this.#generation;
+        this.#playback.cancel();
+        const running = this.#run(utterance, generation).finally(() => {
+            if (this.#turn === running)
+                this.#turn = undefined;
+        });
+        this.#turn = running;
+    }
+    async #run(utterance, generation) {
+        try {
+            const heard = await this.#hear(utterance, generation);
+            if (heard === undefined || generation !== this.#generation)
+                return;
+            await this.#respond(heard, generation);
+        }
+        catch (err) {
+            logger.warn(`standin: the voice turn failed: ${String(err)}`);
+        }
+    }
+    async #hear(utterance, generation) {
+        let heard;
+        try {
+            heard = ((await this.#transcribe(utterance)) ?? "").trim();
+        }
+        catch (err) {
+            logger.warn(`standin: could not transcribe the caller: ${String(err)}`);
+            if (generation === this.#generation) {
+                await this.#speak(TROUBLE_HEARING, "", String(err));
+            }
+            return undefined;
+        }
+        if (heard === "") {
+            // A cough, a door, a second of traffic. The segmenter opens on any loud
+            // frame, and waking the agent for every one of them is a bill and a
+            // caller being answered at random.
+            logger.debug("standin: an utterance transcribed to nothing; no turn");
+            return undefined;
+        }
+        return heard;
+    }
+    async #respond(heard, generation) {
+        let reply;
+        try {
+            reply = this.#answer(heard);
+        }
+        catch (err) {
+            logger.warn(`standin: the agent did not answer: ${String(err)}`);
+            // Gated, like every other thing this turn might say. A turn superseded
+            // while its answer was pending would otherwise apologise over the top of
+            // the turn that replaced it.
+            if (generation === this.#generation) {
+                await this.#speak(TROUBLE_ANSWERING, heard, String(err));
+            }
+            return;
+        }
+        if (isAsyncIterable(reply)) {
+            // Sentence by sentence, so the caller hears the beginning of a long
+            // answer while the rest is still being written.
+            const spoken = [];
+            let interrupted = false;
+            try {
+                for await (const piece of reply) {
+                    if (generation !== this.#generation)
+                        return;
+                    if (piece.trim() === "")
+                        continue;
+                    const turn = await this.#speak(piece, heard);
+                    spoken.push(turn.said);
+                    if (turn.interrupted) {
+                        interrupted = true;
+                        break;
+                    }
+                }
+            }
+            catch (err) {
+                logger.warn(`standin: the agent did not answer: ${String(err)}`);
+                if (generation === this.#generation) {
+                    await this.#speak(TROUBLE_ANSWERING, heard, String(err));
+                }
+                return;
+            }
+            if (spoken.length > 0)
+                this.#finished({ heard, said: spoken.join(" "), interrupted });
+            return;
+        }
+        let said;
+        try {
+            said = ((await reply) ?? "").trim();
+        }
+        catch (err) {
+            logger.warn(`standin: the agent did not answer: ${String(err)}`);
+            if (generation === this.#generation) {
+                await this.#speak(TROUBLE_ANSWERING, heard, String(err));
+            }
+            return;
+        }
+        if (said === "") {
+            logger.debug("standin: the agent answered with nothing; staying quiet");
+            return;
+        }
+        if (generation !== this.#generation)
+            return;
+        this.#finished(await this.#speak(said, heard));
+    }
+    /** Say one piece of an answer, and report what the caller heard. */
+    async #speak(text, heard, error) {
+        const line = (text ?? "").trim();
+        if (line === "")
+            return { heard, said: "", interrupted: false, error };
+        try {
+            const audio = this.#synthesize(line);
+            if (isAsyncIterable(audio)) {
+                let interrupted = false;
+                for await (const chunk of audio) {
+                    const played = await this.#playback.say(chunk);
+                    if (played.interrupted) {
+                        interrupted = true;
+                        break;
+                    }
+                }
+                return { heard, said: line, interrupted, error };
+            }
+            const played = await this.#playback.say(await audio);
+            return { heard, said: line, interrupted: played.interrupted, error };
+        }
+        catch (err) {
+            logger.warn(`standin: could not speak: ${String(err)}`);
+            if (line !== TROUBLE_SPEAKING) {
+                // One retry, with the sentence that says what happened. Without it a
+                // synthesis failure is indistinguishable from a dropped call, and the
+                // caller waits for an answer that is not coming.
+                return await this.#speak(TROUBLE_SPEAKING, heard, String(err));
+            }
+            return { heard, said: "", interrupted: false, error: String(err) };
+        }
+    }
+    #finished(turn) {
+        if (this.#onTurn === undefined)
+            return;
+        try {
+            this.#onTurn(turn);
+        }
+        catch {
+            // A plugin's own bookkeeping must not end a call.
+        }
+    }
+}
+function isAsyncIterable(value) {
+    return (typeof value === "object" && value !== null && Symbol.asyncIterator in value);
+}
+//# sourceMappingURL=lane.js.map

package/dist/lane.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lane.js","sourceRoot":"","sources":["../src/lane.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,+BAA+B;AAiC/B,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE/D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,8BAA8B,CAAC;AAC9D,MAAM,CAAC,MAAM,iBAAiB,GAC5B,gDAAgD,CAAC;AACnD,MAAM,CAAC,MAAM,gBAAgB,GAAG,+CAA+C,CAAC;AA+BhF;;;;;;GAMG;AACH,MAAM,OAAO,SAAS;IACX,QAAQ,CAAc;IACtB,WAAW,CAAa;IACxB,OAAO,CAAS;IAChB,WAAW,CAAa;IACxB,UAAU,CAAqB;IAC/B,OAAO,CAA0C;IACjD,SAAS,CAAgB;IAClC,KAAK,CAA4B;IACjC,WAAW,GAAG,CAAC,CAAC;IAChB,OAAO,GAAG,KAAK,CAAC;IAEhB,YACE,OAAoB,EACpB,UAAsB,EACtB,MAAc,EACd,UAAsB,EACtB,UAA4B,EAAE;QAE9B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,kBAAkB,EAAE,CAAC;QAChE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,aAAa,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,8CAA8C;IAC9C,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;IAChC,CAAC;IAED,uEAAuE;IACvE,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC;IAClC,CAAC;IAED,8DAA8D;IAC9D,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,IAAI,CAAC,GAAW;QACpB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YACvE,uEAAuE;YACvE,oEAAoE;YACpE,wEAAwE;YACxE,yDAAyD;YACzD,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;QACD,IAAI,SAAS,KAAK,SAAS;YAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACtD,CAAC;IAED,yDAAyD;IACzD,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,gEAAgE;QAClE,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,GAAG,CAAC,IAAY;QACpB,wEAAwE;QACxE,uDAAuD;QACvD,IAAI,IAAI,CAAC,OAAO;YACd,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,IAAI,EAAE,EAAE;gBACR,WAAW,EAAE,KAAK;gBAClB,KAAK,EAAE,oBAAoB;aAC5B,CAAC;QACJ,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,iDAAiD;IACjD,KAAK,CAAC,MAAM;QACV,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QACxB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED,0EAA0E;IAE1E;;;;;OAKG;IACH,MAAM,CAAC,SAAiB;QACtB,2EAA2E;QAC3E,4EAA4E;QAC5E,2EAA2E;QAC3E,qDAAqD;QACrD,MAAM,UAAU,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;YAC5D,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO;gBAAE,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACrD,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAiB,EAAE,UAAkB;QAC9C,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACtD,IAAI,KAAK,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI,CAAC,WAAW;gBAAE,OAAO;YACnE,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,mCAAmC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CACT,SAAiB,EACjB,UAAkB;QAElB,IAAI,KAAa,CAAC;QAClB,IAAI,CAAC;YACH,KAAK,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,6CAA6C,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxE,IAAI,UAAU,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;gBACpC,MAAM,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;YACjB,wEAAwE;YACxE,oEAAoE;YACpE,mCAAmC;YACnC,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;YACtE,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,UAAkB;QAC9C,IAAI,KAA8C,CAAC;QACnD,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,sCAAsC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjE,uEAAuE;YACvE,yEAAyE;YACzE,6BAA6B;YAC7B,IAAI,UAAU,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;gBACpC,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,oEAAoE;YACpE,gDAAgD;YAChD,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,IAAI,WAAW,GAAG,KAAK,CAAC;YACxB,IAAI,CAAC;gBACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;oBAChC,IAAI,UAAU,KAAK,IAAI,CAAC,WAAW;wBAAE,OAAO;oBAC5C,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;wBAAE,SAAS;oBAClC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;oBAC7C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACvB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;wBACrB,WAAW,GAAG,IAAI,CAAC;wBACnB,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAC,sCAAsC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACjE,IAAI,UAAU,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;oBACpC,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBACD,OAAO;YACT,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBACnB,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;YACjE,OAAO;QACT,CAAC;QAED,IAAI,IAAY,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,sCAAsC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjE,IAAI,UAAU,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;gBACpC,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YAChB,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;YACxE,OAAO;QACT,CAAC;QACD,IAAI,UAAU,KAAK,IAAI,CAAC,WAAW;YAAE,OAAO;QAC5C,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,oEAAoE;IACpE,KAAK,CAAC,MAAM,CACV,IAAY,EACZ,KAAa,EACb,KAAc;QAEd,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,IAAI,KAAK,EAAE;YAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACvE,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3B,IAAI,WAAW,GAAG,KAAK,CAAC;gBACxB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;oBAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAC/C,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;wBACvB,WAAW,GAAG,IAAI,CAAC;wBACnB,MAAM;oBACR,CAAC;gBACH,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;YACnD,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC;YACrD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC;QACvE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,6BAA6B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxD,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;gBAC9B,qEAAqE;gBACrE,sEAAsE;gBACtE,iDAAiD;gBACjD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACrE,CAAC;IACH,CAAC;IAED,SAAS,CAAC,IAAe;QACvB,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO;QACvC,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QAAC,MAAM,CAAC;YACP,kDAAkD;QACpD,CAAC;IACH,CAAC;CACF;AAED,SAAS,eAAe,CAAI,KAAc;IACxC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,CAAC,aAAa,IAAI,KAAK,CAC7E,CAAC;AACJ,CAAC"}

package/dist/lipsync.d.ts

@@ -0,0 +1,136 @@
+import type { SpeechMark } from "./avatar.js";
+/**
+ * A closed mouth, and what the space between two words becomes. Every other
+ * unmapped character is skipped instead: punching silence into "3.5%" would
+ * close the mouth in the middle of a spoken number.
+ */
+export declare const SILENCE_VISEME = 0;
+/**
+ * Character to mouth shape, read-only. Sukun, shadda, tanween, the tatweel and
+ * every presentation form are deliberately absent: a stretch mark and a doubling
+ * mark carry no mouth shape of their own, and mapping them would insert phantom
+ * mouth changes into an otherwise correct timeline.
+ */
+export declare const CHAR_VISEMES: Readonly<Record<string, number>>;
+/**
+ * The mouth shape a character wears, or undefined when it has none.
+ *
+ * The lookup is on the raw character after lowercasing, with no Unicode
+ * normalization at all: adding NFKC here would change which characters map and
+ * the two SDKs would disagree on the same string. Digits, punctuation, the
+ * tatweel and the non-vowel diacritics come back undefined, and every caller
+ * skips them rather than holding the mouth closed over them.
+ */
+export declare function visemeForChar(ch: string): number | undefined;
+/**
+ * Spread `text` over `durationMs` as a viseme timeline.
+ *
+ * Pass the duration of the audio you actually sent for the turn, which
+ * {@link TurnLipSync} counts for you. Anything else is a guess, and a guessed
+ * timeline is worse than no timeline at all.
+ *
+ * `text` is lowercased, its whitespace runs collapsed, and trimmed. Anything
+ * that is not a positive finite duration returns no marks rather than being
+ * divided by, because an infinite or not-a-number timestamp desynchronises the
+ * mouth for the rest of the utterance and reaches the wire as a null.
+ *
+ * Returns `{tMs, visemeId}` marks, strictly increasing in time, ready for
+ * {@link speechMarks}. Empty when there is nothing to say: no text, no duration,
+ * or nothing in the text that has a mouth shape, which is the right answer for
+ * "3.5%" or an emoji on its own.
+ *
+ * ```ts
+ * const marks = estimateVisemes(finalTranscript, lipsync.durationMs);
+ * if (marks.length > 0) await session.sendSpeechMarks(marks);
+ * ```
+ */
+export declare function estimateVisemes(text: string | null | undefined, durationMs: number): SpeechMark[];
+/**
+ * Build the timeline from per-character timings the speech provider gave you.
+ *
+ * Real times are strictly better than an estimate and cost nothing when the
+ * provider already returns them, so prefer this whenever a synthesis call can
+ * hand back an alignment. Core takes the two plain arrays: `characters` as the
+ * provider spoke them, and `startTimesSeconds` counting from the start of the
+ * utterance. Normalising a vendor's field names is the speech plugin's job.
+ *
+ * Returns marks, or an empty array when the alignment holds no mouth shape at
+ * all (all spaces, all punctuation). Fall back to {@link estimateVisemes} on an
+ * EMPTY result rather than on a missing alignment: a provider that returns
+ * timings for punctuation only has an alignment and still needs the estimate.
+ *
+ * Ragged arrays are tolerated: the walk stops at the shorter of the two.
+ * Providers do return mismatched lengths, and throwing there would lose the turn
+ * over a cosmetic hint.
+ */
+export declare function visemesFromAlignment(characters: readonly string[], startTimesSeconds: readonly number[]): SpeechMark[];
+/** How {@link TurnLipSync} reads the buffers it is handed. */
+export interface TurnLipSyncOptions {
+    /**
+     * The rate of the PCM16 mono buffers passed to {@link TurnLipSync.audioSent}.
+     * Defaults to the wire's own rate, which is what a plugin sending frames to
+     * the call is holding.
+     */
+    sampleRateHz?: number;
+}
+/**
+ * Counts the audio one turn actually sent, then times the mouth to it.
+ *
+ * Feed it every buffer you hand to the call, and ask it for the timeline when
+ * that turn's text is final. It resets itself, so the next turn starts from
+ * zero:
+ *
+ * ```ts
+ * const lipsync = new TurnLipSync();
+ *
+ * // the audio sink
+ * await session.sendAudio(chunk);
+ * lipsync.audioSent(chunk);
+ *
+ * // the final transcript only
+ * if (isFinal) {
+ *   const marks = lipsync.finish(text);
+ *   if (marks.length > 0) await session.sendSpeechMarks(marks);
+ * }
+ *
+ * // playback cancelled
+ * lipsync.cancel();
+ * ```
+ *
+ * Emit once per turn, on the final transcript. A partial would send an
+ * ever-lengthening timeline several times over and the avatar would restart the
+ * mouth mid-sentence.
+ *
+ * {@link cancel} is not optional. On a barge-in the service drops audio the
+ * caller never heard, and a counter that keeps those milliseconds spreads the
+ * next turn's text over its own audio plus the discarded audio: the mouth runs
+ * long for the whole of that turn and every turn after it.
+ */
+export declare class TurnLipSync {
+    #private;
+    constructor(options?: TurnLipSyncOptions);
+    /** Milliseconds of audio sent for the turn in progress. Starts at 0. */
+    get durationMs(): number;
+    /** Add one PCM16 mono buffer that has gone out to the call. */
+    audioSent(pcm: Uint8Array): void;
+    /**
+     * Add a duration directly, for a sink that hands over encoded audio.
+     *
+     * Rounded per chunk rather than kept as a running float, so both SDKs
+     * accumulate the same integer for the same stream of chunks. A chunk that
+     * measures as nothing, or as no number at all, is ignored rather than taking
+     * the turn's count with it.
+     */
+    audioSentMs(ms: number): void;
+    /** Drop the count on a barge-in or a playback cancel, emitting nothing. */
+    cancel(): void;
+    /**
+     * Return the turn's timeline and reset the counter.
+     *
+     * Empty when no audio was sent or the text carries no mouth shape, and a
+     * caller sends nothing in that case. The reset happens either way: the next
+     * turn must not inherit these milliseconds.
+     */
+    finish(text: string | null | undefined): SpeechMark[];
+}
+//# sourceMappingURL=lipsync.d.ts.map

package/dist/lipsync.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lipsync.d.ts","sourceRoot":"","sources":["../src/lipsync.ts"],"names":[],"mappings":"AA4CA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAG9C;;;;GAIG;AACH,eAAO,MAAM,cAAc,IAAI,CAAC;AA4DhC;;;;;GAKG;AACH,eAAO,MAAM,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAC7B,CAAC;AAE9B;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAE5D;AAoCD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAC/B,UAAU,EAAE,MAAM,GACjB,UAAU,EAAE,CAkBd;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,oBAAoB,CAClC,UAAU,EAAE,SAAS,MAAM,EAAE,EAC7B,iBAAiB,EAAE,SAAS,MAAM,EAAE,GACnC,UAAU,EAAE,CAmBd;AAED,8DAA8D;AAC9D,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,qBAAa,WAAW;;gBAIV,OAAO,GAAE,kBAAuB;IAI5C,wEAAwE;IACxE,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,+DAA+D;IAC/D,SAAS,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI;IAMhC;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAK7B,2EAA2E;IAC3E,MAAM,IAAI,IAAI;IAId;;;;;;OAMG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,UAAU,EAAE;CAKtD"}