@komaa/standin-sdk 0.1.0

package/dist/audio.d.ts

@@ -0,0 +1,80 @@
+/** 16-bit mono. */
+export declare const BYTES_PER_SAMPLE = 2;
+/** The nominal frame StandIn sends: 20 ms of PCM16 at 16 kHz = 320 samples. */
+export declare const FRAME_MS = 20;
+export declare const FRAME_BYTES: number;
+/**
+ * What the realtime speech-to-speech models speak. Named here rather than in a
+ * plugin because the RATIO is what forces the residual buffer below, and that is
+ * an audio concern rather than a provider one.
+ */
+export declare const REALTIME_SAMPLE_RATE_HZ = 24000;
+/**
+ * Duration of a PCM16 mono buffer in milliseconds.
+ *
+ * Use this for a playout clock rather than counting frames: outbound chunk
+ * lengths are NOT fixed, so a frame count drifts against real time.
+ */
+export declare function frameDurationMs(pcm: Buffer): number;
+/**
+ * Linear-interpolation resample of PCM16 mono.
+ *
+ * An odd trailing byte is dropped rather than throwing: a truncated frame is a
+ * glitch, but an exception in the audio path is a dropped call.
+ */
+export declare function resamplePcm16(pcm: Buffer, srcHz: number, dstHz: number): Buffer;
+/**
+ * Chops arbitrary-length PCM buffers into whole wire frames, carrying the
+ * remainder.
+ *
+ * Resampled 24 kHz deltas do not divide evenly into the wire's 640-byte frame,
+ * so without a residual the leftover bytes are dropped and every turn loses a
+ * few milliseconds at the seams. Over a call that is audible as clipped word
+ * endings.
+ *
+ * ```ts
+ * const aligner = new FrameAligner();
+ * for (const chunk of providerAudio) {          // arbitrary lengths
+ *   for (const frame of aligner.push(chunk)) {  // whole 640-byte frames
+ *     await session.sendAudio(frame);
+ *   }
+ * }
+ * const tail = aligner.flush();                 // end of turn
+ * if (tail) await session.sendAudio(tail);
+ * ```
+ */
+export declare class FrameAligner {
+    #private;
+    constructor(frameBytes?: number);
+    /** Bytes held back, waiting for a whole frame. */
+    get pending(): number;
+    /** Add a buffer; return whatever whole frames are now available. */
+    push(pcm: Buffer): Buffer[];
+    /**
+     * Zero-pad and return the residual at end of turn, or undefined when empty.
+     *
+     * Padding rather than dropping: the tail of the last word matters more than a
+     * few milliseconds of silence.
+     */
+    flush(): Buffer | undefined;
+    /**
+     * Drop the residual without emitting it - use on a barge-in, where the
+     * held-back bytes belong to a turn the caller just interrupted.
+     */
+    reset(): void;
+}
+/**
+ * Root-mean-square amplitude of PCM16 mono little-endian, normalised to 0.0 - 1.0.
+ *
+ * How loud a frame is, which is what an echo guard, a barge-in check and a voice
+ * segmenter each need. It lives here because all three want it and every plugin
+ * that wanted it had been writing it again.
+ *
+ * Dependency-free for the same reason the resampler above is: this runs on every
+ * inbound frame of every call.
+ *
+ * An odd trailing byte is dropped rather than throwing. A truncated frame is a
+ * glitch; an exception in the audio path is a dropped call.
+ */
+export declare function pcm16Rms(pcm: Buffer): number;
+//# sourceMappingURL=audio.d.ts.map

package/dist/audio.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio.d.ts","sourceRoot":"","sources":["../src/audio.ts"],"names":[],"mappings":"AA2BA,mBAAmB;AACnB,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,+EAA+E;AAC/E,eAAO,MAAM,QAAQ,KAAK,CAAC;AAC3B,eAAO,MAAM,WAAW,QAC+B,CAAC;AAExD;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,QAAS,CAAC;AAE9C;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,MAAM,CA8BR;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,YAAY;;gBAIX,UAAU,GAAE,MAAoB;IAI5C,kDAAkD;IAClD,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,oEAAoE;IACpE,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE;IAa3B;;;;;OAKG;IACH,KAAK,IAAI,MAAM,GAAG,SAAS;IAO3B;;;OAGG;IACH,KAAK,IAAI,IAAI;CAGd;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAS5C"}

package/dist/audio.js

@@ -0,0 +1,170 @@
+// Copyright (c) 2026 Komaa DigiTech
+// SPDX-License-Identifier: MIT
+/**
+ * The audio the wire carries, and the two helpers every plugin needs.
+ *
+ * The StandIn wire is PCM 16 kHz, 16-bit, mono, little-endian in both
+ * directions. Almost nothing else is. The realtime speech-to-speech models speak
+ * 24 kHz, most TTS vendors emit 22.05 or 24 kHz, and none of them chunk on the
+ * wire's frame boundary. So every plugin that is not a pure passthrough
+ * ends up writing the same two things:
+ *
+ * - a resampler, because the rates differ
+ * - a frame aligner, because a resampled buffer does not divide evenly into the
+ *   wire's 640-byte frame, and dropping the remainder clips the end of every turn
+ *
+ * They live here rather than in each plugin because they are properties of the
+ * WIRE, not of any framework - the same reason the sequence number and the
+ * outbound timeline live in {@link CallServer}. A plugin that has to reimplement
+ * them is a plugin the SDK failed.
+ *
+ * Byte-for-byte equivalent to the Python SDK's `standin/sdk/audio.py`; the
+ * shared conformance vectors assert both produce identical output.
+ */
+import { SAMPLE_RATE_HZ } from "./protocol.js";
+/** 16-bit mono. */
+export const BYTES_PER_SAMPLE = 2;
+/** The nominal frame StandIn sends: 20 ms of PCM16 at 16 kHz = 320 samples. */
+export const FRAME_MS = 20;
+export const FRAME_BYTES = (SAMPLE_RATE_HZ * BYTES_PER_SAMPLE * FRAME_MS) / 1000; // 640
+/**
+ * What the realtime speech-to-speech models speak. Named here rather than in a
+ * plugin because the RATIO is what forces the residual buffer below, and that is
+ * an audio concern rather than a provider one.
+ */
+export const REALTIME_SAMPLE_RATE_HZ = 24_000;
+/**
+ * Duration of a PCM16 mono buffer in milliseconds.
+ *
+ * Use this for a playout clock rather than counting frames: outbound chunk
+ * lengths are NOT fixed, so a frame count drifts against real time.
+ */
+export function frameDurationMs(pcm) {
+    return (pcm.length / (SAMPLE_RATE_HZ * BYTES_PER_SAMPLE)) * 1000.0;
+}
+/**
+ * Linear-interpolation resample of PCM16 mono.
+ *
+ * An odd trailing byte is dropped rather than throwing: a truncated frame is a
+ * glitch, but an exception in the audio path is a dropped call.
+ */
+export function resamplePcm16(pcm, srcHz, dstHz) {
+    if (srcHz === dstHz || pcm.length === 0)
+        return pcm;
+    const usable = pcm.length % 2 === 0 ? pcm : pcm.subarray(0, pcm.length - 1);
+    const nIn = usable.length / 2;
+    if (nIn === 0)
+        return Buffer.alloc(0);
+    // Math.round matches Python's round() for the .5 cases these rates produce
+    // (2:3 and 3:2 never land exactly on .5), so both SDKs agree on n_out.
+    const nOut = Math.max(1, Math.round((nIn * dstHz) / srcHz));
+    const step = nIn / nOut;
+    const out = Buffer.alloc(nOut * 2);
+    for (let i = 0; i < nOut; i++) {
+        const pos = i * step;
+        const j = Math.trunc(pos);
+        let value;
+        if (j >= nIn - 1) {
+            value = usable.readInt16LE((nIn - 1) * 2);
+        }
+        else {
+            const a = usable.readInt16LE(j * 2);
+            const b = usable.readInt16LE((j + 1) * 2);
+            // Math.trunc, not Math.floor: Python's int() truncates toward zero, and
+            // interpolating between a negative and a less-negative sample makes the
+            // difference visible.
+            value = Math.trunc(a + (b - a) * (pos - j));
+        }
+        out.writeInt16LE(value, i * 2);
+    }
+    return out;
+}
+/**
+ * Chops arbitrary-length PCM buffers into whole wire frames, carrying the
+ * remainder.
+ *
+ * Resampled 24 kHz deltas do not divide evenly into the wire's 640-byte frame,
+ * so without a residual the leftover bytes are dropped and every turn loses a
+ * few milliseconds at the seams. Over a call that is audible as clipped word
+ * endings.
+ *
+ * ```ts
+ * const aligner = new FrameAligner();
+ * for (const chunk of providerAudio) {          // arbitrary lengths
+ *   for (const frame of aligner.push(chunk)) {  // whole 640-byte frames
+ *     await session.sendAudio(frame);
+ *   }
+ * }
+ * const tail = aligner.flush();                 // end of turn
+ * if (tail) await session.sendAudio(tail);
+ * ```
+ */
+export class FrameAligner {
+    #frameBytes;
+    #buf = Buffer.alloc(0);
+    constructor(frameBytes = FRAME_BYTES) {
+        this.#frameBytes = frameBytes;
+    }
+    /** Bytes held back, waiting for a whole frame. */
+    get pending() {
+        return this.#buf.length;
+    }
+    /** Add a buffer; return whatever whole frames are now available. */
+    push(pcm) {
+        this.#buf =
+            this.#buf.length === 0
+                ? Buffer.from(pcm)
+                : Buffer.concat([this.#buf, pcm]);
+        const out = [];
+        while (this.#buf.length >= this.#frameBytes) {
+            out.push(this.#buf.subarray(0, this.#frameBytes));
+            this.#buf = this.#buf.subarray(this.#frameBytes);
+        }
+        return out;
+    }
+    /**
+     * Zero-pad and return the residual at end of turn, or undefined when empty.
+     *
+     * Padding rather than dropping: the tail of the last word matters more than a
+     * few milliseconds of silence.
+     */
+    flush() {
+        if (this.#buf.length === 0)
+            return undefined;
+        const tail = Buffer.concat([this.#buf], this.#frameBytes);
+        this.#buf = Buffer.alloc(0);
+        return tail;
+    }
+    /**
+     * Drop the residual without emitting it - use on a barge-in, where the
+     * held-back bytes belong to a turn the caller just interrupted.
+     */
+    reset() {
+        this.#buf = Buffer.alloc(0);
+    }
+}
+/**
+ * Root-mean-square amplitude of PCM16 mono little-endian, normalised to 0.0 - 1.0.
+ *
+ * How loud a frame is, which is what an echo guard, a barge-in check and a voice
+ * segmenter each need. It lives here because all three want it and every plugin
+ * that wanted it had been writing it again.
+ *
+ * Dependency-free for the same reason the resampler above is: this runs on every
+ * inbound frame of every call.
+ *
+ * An odd trailing byte is dropped rather than throwing. A truncated frame is a
+ * glitch; an exception in the audio path is a dropped call.
+ */
+export function pcm16Rms(pcm) {
+    const samples = Math.floor(pcm.length / 2);
+    if (samples === 0)
+        return 0;
+    let sum = 0;
+    for (let i = 0; i < samples; i += 1) {
+        const sample = pcm.readInt16LE(i * 2) / 32768;
+        sum += sample * sample;
+    }
+    return Math.sqrt(sum / samples);
+}
+//# sourceMappingURL=audio.js.map

package/dist/audio.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio.js","sourceRoot":"","sources":["../src/audio.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,+BAA+B;AAE/B;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C,mBAAmB;AACnB,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,+EAA+E;AAC/E,MAAM,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAC;AAC3B,MAAM,CAAC,MAAM,WAAW,GACtB,CAAC,cAAc,GAAG,gBAAgB,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAE9C;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,cAAc,GAAG,gBAAgB,CAAC,CAAC,GAAG,MAAM,CAAC;AACrE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,GAAW,EACX,KAAa,EACb,KAAa;IAEb,IAAI,KAAK,KAAK,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAEpD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5E,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAC9B,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEtC,2EAA2E;IAC3E,uEAAuE;IACvE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC;IACxB,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC;QACrB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,KAAa,CAAC;QAClB,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YACjB,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACpC,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,wEAAwE;YACxE,wEAAwE;YACxE,sBAAsB;YACtB,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC;QACD,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,OAAO,YAAY;IACd,WAAW,CAAS;IAC7B,IAAI,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE/B,YAAY,aAAqB,WAAW;QAC1C,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAChC,CAAC;IAED,kDAAkD;IAClD,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1B,CAAC;IAED,oEAAoE;IACpE,IAAI,CAAC,GAAW;QACd,IAAI,CAAC,IAAI;YACP,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBACpB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;gBAClB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACtC,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5C,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;YAClD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACnD,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;OAKG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,SAAS,CAAC;QAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC1D,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9B,CAAC;CACF;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3C,IAAI,OAAO,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC5B,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;QAC9C,GAAG,IAAI,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC;AAClC,CAAC"}

package/dist/avatar.d.ts

@@ -0,0 +1,111 @@
+/**
+ * The emotions StandIn knows by name. An open set: sending something else is
+ * allowed and renders as neutral, so a newer sender and an older service still
+ * interoperate.
+ */
+export declare const EMOTIONS: readonly ["neutral", "happy", "sad", "surprised", "thinking"];
+/** A well-known emotion, or any other string the service may learn later. */
+export type Emotion = (typeof EMOTIONS)[number] | (string & {});
+/**
+ * Visemes use the Azure Speech numbering, 0 to 21, which is what the avatar
+ * expects. A mark outside that range is dropped rather than sent.
+ */
+export declare const MAX_VISEME_ID = 21;
+/**
+ * An emotion reaches the avatar tile, and the string usually came from a model
+ * that whoever is on the call is steering. Bounded here, where the message is
+ * built, so a plugin cannot forget to bound it.
+ */
+export declare const MAX_EMOTION_CHARS = 40;
+/** One viseme mark: milliseconds from the start of the utterance, and which mouth shape to hold. */
+export interface SpeechMark {
+    /** Time offset in ms, relative to the utterance's audio start. */
+    readonly tMs: number;
+    /** Viseme id in the Azure Speech numbering, 0-21. */
+    readonly visemeId: number;
+}
+/**
+ * Build an `expression`: the emotion the avatar should wear.
+ *
+ * Affects the video tile only, never the audio. Throws on an empty value or one
+ * longer than {@link MAX_EMOTION_CHARS}; everything else is the service's to
+ * interpret, and an unknown emotion renders as neutral.
+ */
+export declare function expression(emotion: Emotion): string;
+/** The emotions {@link inferEmotion} can read out of a reply. */
+export type InferredEmotion = "surprised" | "sad" | "happy" | "neutral";
+/**
+ * Read an emotion out of a reply, for {@link expression} to carry.
+ *
+ * A lexicon rather than a model: it runs on every turn of a live call, so it has
+ * to cost nothing and add no latency. First match wins in priority order,
+ * surprised then sad then happy, which is deliberate - a "wow!" must not be
+ * masked by a polite "thanks", and an apology must not be masked by an
+ * incidental "nice". A reply that is genuinely mixed resolves to the
+ * higher-priority one, and on a streaming path the next chunk re-reads it.
+ *
+ * **The lexicon is English only.** An Arabic or other non-English reply always
+ * infers "neutral". That is by design and not a bug to work around: a language
+ * guess that got it wrong would put the wrong face on, and neutral is always a
+ * safe face.
+ */
+export declare function inferEmotion(text: string | null | undefined): InferredEmotion;
+/**
+ * Decides WHEN an emotion is worth sending, so the face changes and nothing
+ * else does.
+ *
+ * One instance per call. Both methods return the emotion to hand to
+ * {@link expression}, or null when nothing should be sent:
+ *
+ * ```ts
+ * const cue = new ExpressionCue();
+ * // on every assistant transcript, partial and final alike
+ * const emotion = cue.cue(chunk);
+ * if (emotion !== null) await call.express(emotion);
+ *
+ * // around a tool that keeps the caller waiting
+ * const waiting = cue.thinking(true);
+ * try {
+ *   ...
+ * } finally {
+ *   const done = cue.thinking(false);
+ * }
+ * ```
+ *
+ * Re-cue on partials: waiting for the final transcript leaves the face stale
+ * for the whole time an apologetic reply is already being spoken. The de-dupe
+ * on the last value sent is what keeps a word-by-word stream from sending
+ * dozens of identical cues.
+ */
+export declare class ExpressionCue {
+    #private;
+    /** The emotion last handed out, or null while none has been. */
+    get lastSent(): Emotion | null;
+    /**
+     * Read `text` and return the emotion to send, or null when it is unchanged.
+     *
+     * Answers null for everything while {@link thinking} is on, so a transcript
+     * chunk arriving mid-tool cannot make the avatar look done while it is still
+     * working.
+     */
+    cue(text: string | null | undefined): Emotion | null;
+    /**
+     * Enter or leave the waiting face around a tool call.
+     *
+     * Only transitions produce anything; setting the same state twice is a no-op.
+     * Leaving it returns "neutral" when the thinking face is still the last thing
+     * sent, because the model may stay silent after a tool result and there would
+     * be no transcript left to re-cue from. Call it from a `finally` so a tool
+     * that threw still puts the face back.
+     */
+    thinking(on: boolean): Emotion | null;
+}
+/**
+ * Build a `speech.marks`: the viseme timeline for one utterance.
+ *
+ * Marks are sorted ascending and marks outside the viseme range are dropped,
+ * because the avatar reads the timeline in order and one bad entry would
+ * desynchronise the mouth for the rest of the utterance.
+ */
+export declare function speechMarks(marks: Iterable<SpeechMark>): string;
+//# sourceMappingURL=avatar.d.ts.map

package/dist/avatar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"avatar.d.ts","sourceRoot":"","sources":["../src/avatar.ts"],"names":[],"mappings":"AAqCA;;;;GAIG;AACH,eAAO,MAAM,QAAQ,+DAMX,CAAC;AAEX,6EAA6E;AAC7E,MAAM,MAAM,OAAO,GAAG,CAAC,OAAO,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;AAEhE;;;GAGG;AACH,eAAO,MAAM,aAAa,KAAK,CAAC;AAEhC;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAEpC,oGAAoG;AACpG,MAAM,WAAW,UAAU;IACzB,kEAAkE;IAClE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,qDAAqD;IACrD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CASnD;AAED,iEAAiE;AACjE,MAAM,MAAM,eAAe,GAAG,WAAW,GAAG,KAAK,GAAG,OAAO,GAAG,SAAS,CAAC;AAcxE;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,eAAe,CAW7E;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,aAAa;;IAIxB,gEAAgE;IAChE,IAAI,QAAQ,IAAI,OAAO,GAAG,IAAI,CAE7B;IAED;;;;;;OAMG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,GAAG,IAAI;IAQpD;;;;;;;;OAQG;IACH,QAAQ,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,GAAG,IAAI;CAWtC;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,MAAM,CAO/D"}

package/dist/avatar.js

@@ -0,0 +1,201 @@
+// Copyright (c) 2026 Komaa DigiTech
+// SPDX-License-Identifier: MIT
+/**
+ * The avatar lane: the face the caller sees while your agent talks.
+ *
+ * StandIn renders the avatar tile. Your worker does not draw it, stream it, or
+ * know how it is made - it sends two hints and the service does the rest:
+ *
+ * - {@link expression} names the emotion the face should wear.
+ * - {@link speechMarks} carries the viseme timeline for one utterance, which is
+ *   what makes the mouth match the words.
+ *
+ * Both are **additive and best-effort**. A service that does not implement one
+ * ignores it, an unknown emotion falls back to neutral, and neither ever affects
+ * the audio the caller hears.
+ *
+ * Reach for {@link speechMarks} whenever you can put a timeline behind it. Real
+ * viseme timings from the provider are best, and an ESTIMATE spread over the
+ * audio the turn actually sent is the right default on a realtime path, where no
+ * provider offers timings at all: a mouth on a measured clock beats a still one.
+ * What is still worse than none is a timeline whose DURATION was guessed, from
+ * text length or a words-per-minute rate, because that one drifts away from the
+ * voice as the sentence goes on. `lipsync.ts` is where both live.
+ *
+ * {@link inferEmotion} and {@link ExpressionCue} sit here rather than in their
+ * own module because what they produce is {@link expression}'s argument: the
+ * first reads an emotion out of the reply text, the second decides when that is
+ * worth sending.
+ *
+ * Identical in shape to the Python SDK's `standin.avatar`, translated to TS
+ * naming.
+ */
+import { encode } from "./protocolRuntime.js";
+import { TYPE_EXPRESSION, TYPE_SPEECH_MARKS } from "./protocol.js";
+/**
+ * The emotions StandIn knows by name. An open set: sending something else is
+ * allowed and renders as neutral, so a newer sender and an older service still
+ * interoperate.
+ */
+export const EMOTIONS = [
+    "neutral",
+    "happy",
+    "sad",
+    "surprised",
+    "thinking",
+];
+/**
+ * Visemes use the Azure Speech numbering, 0 to 21, which is what the avatar
+ * expects. A mark outside that range is dropped rather than sent.
+ */
+export const MAX_VISEME_ID = 21;
+/**
+ * An emotion reaches the avatar tile, and the string usually came from a model
+ * that whoever is on the call is steering. Bounded here, where the message is
+ * built, so a plugin cannot forget to bound it.
+ */
+export const MAX_EMOTION_CHARS = 40;
+/**
+ * Build an `expression`: the emotion the avatar should wear.
+ *
+ * Affects the video tile only, never the audio. Throws on an empty value or one
+ * longer than {@link MAX_EMOTION_CHARS}; everything else is the service's to
+ * interpret, and an unknown emotion renders as neutral.
+ */
+export function expression(emotion) {
+    const text = emotion.trim();
+    if (text === "")
+        throw new Error("expression needs an emotion");
+    if (text.length > MAX_EMOTION_CHARS) {
+        throw new Error(`an emotion must be at most ${MAX_EMOTION_CHARS} characters`);
+    }
+    return encode({ type: TYPE_EXPRESSION, emotion: text });
+}
+// Two or more of them together. One "?" is a question, "?!" is a reaction.
+const SURPRISED_MARKS = /[?!]{2,}/;
+// Word tests are \b-bounded throughout, or "nicety" reads as happy and
+// "greatly" masks an apology.
+const SURPRISED_WORDS = /\b(?:wow|whoa|woah|oh no|oh my|no way|unbelievable|incredible|astonish\w*|surpris\w*)\b/;
+const SAD_WORDS = /\b(?:sorry|apolog\w*|unfortunately|regret\w*|afraid|sadly|bad news|failed|unable to|i can't|i cannot|i'm unable)\b/;
+const HAPPY_WORDS = /\b(?:glad|great|awesome|wonderful|fantastic|excellent|congrat\w*|happy|love|perfect|good news|success\w*|thank\w*|welcome|nice|well done)\b/;
+/**
+ * Read an emotion out of a reply, for {@link expression} to carry.
+ *
+ * A lexicon rather than a model: it runs on every turn of a live call, so it has
+ * to cost nothing and add no latency. First match wins in priority order,
+ * surprised then sad then happy, which is deliberate - a "wow!" must not be
+ * masked by a polite "thanks", and an apology must not be masked by an
+ * incidental "nice". A reply that is genuinely mixed resolves to the
+ * higher-priority one, and on a streaming path the next chunk re-reads it.
+ *
+ * **The lexicon is English only.** An Arabic or other non-English reply always
+ * infers "neutral". That is by design and not a bug to work around: a language
+ * guess that got it wrong would put the wrong face on, and neutral is always a
+ * safe face.
+ */
+export function inferEmotion(text) {
+    const raw = (text ?? "").trim();
+    if (raw === "")
+        return "neutral";
+    if (SURPRISED_MARKS.test(raw))
+        return "surprised";
+    // Models emit the typographic apostrophe routinely, and "I can’t" is the most
+    // common apologetic phrasing there is.
+    const words = raw.toLowerCase().replace(/’/g, "'");
+    if (SURPRISED_WORDS.test(words))
+        return "surprised";
+    if (SAD_WORDS.test(words))
+        return "sad";
+    if (HAPPY_WORDS.test(words))
+        return "happy";
+    return "neutral";
+}
+/**
+ * Decides WHEN an emotion is worth sending, so the face changes and nothing
+ * else does.
+ *
+ * One instance per call. Both methods return the emotion to hand to
+ * {@link expression}, or null when nothing should be sent:
+ *
+ * ```ts
+ * const cue = new ExpressionCue();
+ * // on every assistant transcript, partial and final alike
+ * const emotion = cue.cue(chunk);
+ * if (emotion !== null) await call.express(emotion);
+ *
+ * // around a tool that keeps the caller waiting
+ * const waiting = cue.thinking(true);
+ * try {
+ *   ...
+ * } finally {
+ *   const done = cue.thinking(false);
+ * }
+ * ```
+ *
+ * Re-cue on partials: waiting for the final transcript leaves the face stale
+ * for the whole time an apologetic reply is already being spoken. The de-dupe
+ * on the last value sent is what keeps a word-by-word stream from sending
+ * dozens of identical cues.
+ */
+export class ExpressionCue {
+    #lastSent = null;
+    #thinking = false;
+    /** The emotion last handed out, or null while none has been. */
+    get lastSent() {
+        return this.#lastSent;
+    }
+    /**
+     * Read `text` and return the emotion to send, or null when it is unchanged.
+     *
+     * Answers null for everything while {@link thinking} is on, so a transcript
+     * chunk arriving mid-tool cannot make the avatar look done while it is still
+     * working.
+     */
+    cue(text) {
+        if (this.#thinking)
+            return null;
+        const emotion = inferEmotion(text);
+        if (emotion === this.#lastSent)
+            return null;
+        this.#lastSent = emotion;
+        return emotion;
+    }
+    /**
+     * Enter or leave the waiting face around a tool call.
+     *
+     * Only transitions produce anything; setting the same state twice is a no-op.
+     * Leaving it returns "neutral" when the thinking face is still the last thing
+     * sent, because the model may stay silent after a tool result and there would
+     * be no transcript left to re-cue from. Call it from a `finally` so a tool
+     * that threw still puts the face back.
+     */
+    thinking(on) {
+        if (on === this.#thinking)
+            return null;
+        this.#thinking = on;
+        if (on) {
+            this.#lastSent = "thinking";
+            return "thinking";
+        }
+        if (this.#lastSent !== "thinking")
+            return null;
+        this.#lastSent = "neutral";
+        return "neutral";
+    }
+}
+/**
+ * Build a `speech.marks`: the viseme timeline for one utterance.
+ *
+ * Marks are sorted ascending and marks outside the viseme range are dropped,
+ * because the avatar reads the timeline in order and one bad entry would
+ * desynchronise the mouth for the rest of the utterance.
+ */
+export function speechMarks(marks) {
+    const cleaned = [...marks]
+        .filter((m) => m.tMs >= 0 && m.visemeId >= 0 && m.visemeId <= MAX_VISEME_ID)
+        .map((m) => ({ tMs: Math.trunc(m.tMs), visemeId: Math.trunc(m.visemeId) }))
+        .sort((a, b) => a.tMs - b.tMs || a.visemeId - b.visemeId);
+    // A reserved timeline anchor the lane does not use yet. Both SDKs send 0.
+    return encode({ type: TYPE_SPEECH_MARKS, ts: 0, marks: cleaned });
+}
+//# sourceMappingURL=avatar.js.map

package/dist/avatar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"avatar.js","sourceRoot":"","sources":["../src/avatar.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,+BAA+B;AAE/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEnE;;;;GAIG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,SAAS;IACT,OAAO;IACP,KAAK;IACL,WAAW;IACX,UAAU;CACF,CAAC;AAKX;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC;AAEhC;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAUpC;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,OAAgB;IACzC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,IAAI,KAAK,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAChE,IAAI,IAAI,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CACb,8BAA8B,iBAAiB,aAAa,CAC7D,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;AAC1D,CAAC;AAKD,2EAA2E;AAC3E,MAAM,eAAe,GAAG,UAAU,CAAC;AAEnC,uEAAuE;AACvE,8BAA8B;AAC9B,MAAM,eAAe,GACnB,yFAAyF,CAAC;AAC5F,MAAM,SAAS,GACb,oHAAoH,CAAC;AACvH,MAAM,WAAW,GACf,6IAA6I,CAAC;AAEhJ;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,YAAY,CAAC,IAA+B;IAC1D,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,GAAG,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IACjC,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,WAAW,CAAC;IAClD,8EAA8E;IAC9E,uCAAuC;IACvC,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACnD,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,WAAW,CAAC;IACpD,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAC5C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,OAAO,aAAa;IACxB,SAAS,GAAmB,IAAI,CAAC;IACjC,SAAS,GAAG,KAAK,CAAC;IAElB,gEAAgE;IAChE,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;;;;OAMG;IACH,GAAG,CAAC,IAA+B;QACjC,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAChC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,OAAO,KAAK,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAC5C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC;QACzB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;;;OAQG;IACH,QAAQ,CAAC,EAAW;QAClB,IAAI,EAAE,KAAK,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,EAAE,EAAE,CAAC;YACP,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC;YAC5B,OAAO,UAAU,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,KAAK,UAAU;YAAE,OAAO,IAAI,CAAC;QAC/C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,OAAO,SAAS,CAAC;IACnB,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,KAA2B;IACrD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC;SACvB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,aAAa,CAAC;SAC3E,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;SAC1E,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC5D,0EAA0E;IAC1E,OAAO,MAAM,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;AACpE,CAAC"}