@voqalize/avatar 0.0.1

package/client/dist/types.d.ts

@@ -0,0 +1,101 @@
+/**
+ * types.ts — the avatar wire vocabulary, client side.
+ *
+ * The binding definition is `docs/contract-protocol.md`; this file is its
+ * TypeScript restatement and must not drift from it. The Python half of the
+ * same vocabulary is `py/src/voqalize_avatar/messages.py` — the three are
+ * maintained together, and a command added to one without the others is
+ * incomplete.
+ *
+ * A server pushes these as RTVI `server-message`s under the envelope
+ * `{ type: "avatar", v: 1, ...cmd-specific fields }`. {@link AvatarCommand}
+ * describes the *payload*, not the envelope, because the payload is what
+ * arrives however the host chose to carry it — `AvatarClient.dispatch()`
+ * accepts anything with a string `cmd`, so an application that tunnels these
+ * through its own message type can hand them straight over.
+ */
+/** A viseme cue: `t` is a ms offset into the utterance's clock, `v` is a Rhubarb A–H (or X) letter. */
+export interface AvatarCue {
+    t: number;
+    v: string;
+    i?: number;
+}
+/** `perform()` timeline action — see docs/contract-protocol.md § Composing behavior. */
+export interface AvatarPerformAction {
+    t: number;
+    do: "state" | "emotion" | "gaze" | "interject";
+    name?: string;
+    id?: string;
+    i?: number;
+    keepGaze?: boolean;
+}
+export interface AvatarStateCmd {
+    cmd: "state";
+    name: string;
+    emotion?: string;
+    gaze?: string;
+}
+export interface AvatarInterjectCmd {
+    cmd: "interject";
+    id: string;
+}
+export interface AvatarPerformCmd {
+    cmd: "perform";
+    actions: AvatarPerformAction[];
+    ctx?: string;
+}
+export interface AvatarCuesCmd {
+    cmd: "cues";
+    ctx: string;
+    /** Discard queued cues at or after this offset (ms), then append `cues`. */
+    from_ms: number;
+    cues: AvatarCue[];
+    /**
+     * True on the one chunk that completes this turn's track: the TTS context is
+     * closed, so no further chunk will splice into `ctx`. What a client may
+     * assume, exactly — nothing about playout. The audio it describes is still
+     * ahead, and `speech stop` remains the end of the turn. It is safe to release
+     * per-turn cue state (the splice buffer for `ctx`) once the last cue has
+     * played, and safe to stop expecting more.
+     *
+     * Absent on an interrupted turn, deliberately: a turn that was cut never
+     * claims to have completed. Absent chunks are the normal case — the widget's
+     * own track already completes on the trailing `X`, so ignoring `final`
+     * entirely is a correct implementation.
+     */
+    final?: boolean;
+}
+export interface AvatarSpeechCmd {
+    cmd: "speech";
+    event: "start" | "stop";
+    ctx: string;
+}
+export interface AvatarUserCmd {
+    cmd: "user";
+    speaking: boolean;
+}
+export interface AvatarHintCmd {
+    cmd: "hint";
+    kind: "eager_eot" | (string & {});
+}
+/** A cmd this build doesn't recognize — dispatched to nothing, ignored for forward compat. */
+export interface AvatarUnknownCmd {
+    cmd: string;
+    [key: string]: unknown;
+}
+export type AvatarCommand = AvatarStateCmd | AvatarInterjectCmd | AvatarPerformCmd | AvatarCuesCmd | AvatarSpeechCmd | AvatarUserCmd | AvatarHintCmd | AvatarUnknownCmd;
+/** The full server-message payload: the avatar envelope plus its `cmd`. */
+export type AvatarServerMessage = AvatarCommand & {
+    type?: "avatar";
+    v?: number;
+};
+/** Narrows an unknown server-message payload to an avatar command. */
+export declare function isAvatarMessage(msg: unknown): msg is AvatarServerMessage;
+/** The envelope `type` the protocol reserves for avatar traffic. */
+export declare const AVATAR_MESSAGE_TYPE = "avatar";
+/** The protocol version this client speaks — matches `AVATAR_PROTOCOL_VERSION`
+ * in the Python package. Sent as `v` and, today, never checked: an unknown
+ * `cmd` is ignored rather than version-gated, which is the forward-compat rule
+ * the contract states. */
+export declare const AVATAR_PROTOCOL_VERSION = 1;
+//# sourceMappingURL=types.d.ts.map

package/client/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,uGAAuG;AACvG,MAAM,WAAW,SAAS;IACxB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,CAAC,EAAE,MAAM,CAAC;CACZ;AAED,wFAAwF;AACxF,MAAM,WAAW,mBAAmB;IAClC,CAAC,EAAE,MAAM,CAAC;IACV,EAAE,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,GAAG,WAAW,CAAC;IAC/C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,OAAO,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,WAAW,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,SAAS,CAAC;IACf,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,4EAA4E;IAC5E,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,QAAQ,CAAC;IACd,KAAK,EAAE,OAAO,GAAG,MAAM,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,WAAW,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;CACnC;AAED,8FAA8F;AAC9F,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,MAAM,aAAa,GACrB,cAAc,GACd,kBAAkB,GAClB,gBAAgB,GAChB,aAAa,GACb,eAAe,GACf,aAAa,GACb,aAAa,GACb,gBAAgB,CAAC;AAErB,2EAA2E;AAC3E,MAAM,MAAM,mBAAmB,GAAG,aAAa,GAAG;IAChD,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,CAAC,CAAC,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,sEAAsE;AACtE,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,GAAG,IAAI,mBAAmB,CAIxE;AAED,oEAAoE;AACpE,eAAO,MAAM,mBAAmB,WAAW,CAAC;AAE5C;;;0BAG0B;AAC1B,eAAO,MAAM,uBAAuB,IAAI,CAAC"}

package/client/dist/types.js

@@ -0,0 +1,31 @@
+/**
+ * types.ts — the avatar wire vocabulary, client side.
+ *
+ * The binding definition is `docs/contract-protocol.md`; this file is its
+ * TypeScript restatement and must not drift from it. The Python half of the
+ * same vocabulary is `py/src/voqalize_avatar/messages.py` — the three are
+ * maintained together, and a command added to one without the others is
+ * incomplete.
+ *
+ * A server pushes these as RTVI `server-message`s under the envelope
+ * `{ type: "avatar", v: 1, ...cmd-specific fields }`. {@link AvatarCommand}
+ * describes the *payload*, not the envelope, because the payload is what
+ * arrives however the host chose to carry it — `AvatarClient.dispatch()`
+ * accepts anything with a string `cmd`, so an application that tunnels these
+ * through its own message type can hand them straight over.
+ */
+/** Narrows an unknown server-message payload to an avatar command. */
+export function isAvatarMessage(msg) {
+    if (typeof msg !== "object" || msg === null)
+        return false;
+    const m = msg;
+    return typeof m.cmd === "string";
+}
+/** The envelope `type` the protocol reserves for avatar traffic. */
+export const AVATAR_MESSAGE_TYPE = "avatar";
+/** The protocol version this client speaks — matches `AVATAR_PROTOCOL_VERSION`
+ * in the Python package. Sent as `v` and, today, never checked: an unknown
+ * `cmd` is ignored rather than version-gated, which is the forward-compat rule
+ * the contract states. */
+export const AVATAR_PROTOCOL_VERSION = 1;
+//# sourceMappingURL=types.js.map

package/client/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAiGH,sEAAsE;AACtE,MAAM,UAAU,eAAe,CAAC,GAAY;IAC1C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,MAAM,CAAC,GAAG,GAA8B,CAAC;IACzC,OAAO,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC;AACnC,CAAC;AAED,oEAAoE;AACpE,MAAM,CAAC,MAAM,mBAAmB,GAAG,QAAQ,CAAC;AAE5C;;;0BAG0B;AAC1B,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC"}

package/client/dist/useAvatar.d.ts

@@ -0,0 +1,53 @@
+/**
+ * useAvatar — mount the widget, wire it to a live session, dispatch its
+ * server-messages, and clean up.
+ *
+ * Options are read through a ref so the effect doesn't re-subscribe on every
+ * render. Split in two effects: mounting the widget happens once (an avatar or
+ * theme swap remounts by design — see the note in the effect); attaching to
+ * the pipecat client re-runs whenever the client identity changes (a session
+ * reconnect mints a new one) or once the widget instance becomes available.
+ */
+import type { PipecatClient } from "@pipecat-ai/client-js";
+import { type AvatarApi, type CreateAvatarOptions } from "../../src/avatar.js";
+import { AvatarClient, type AvatarClientOptions } from "./AvatarClient.js";
+export interface UseAvatarOptions extends AvatarClientOptions {
+    /** Name from `AVATAR_NAMES`. Omit for the widget's own `DEFAULT_AVATAR`. */
+    avatar?: string;
+    theme?: CreateAvatarOptions["theme"];
+    /** Articulation gains — see docs/contract-protocol.md § Events, gains, introspection. */
+    mouthGain?: number;
+    gestureGain?: number;
+    /** The live `PipecatClient` to dispatch server-messages from, or `null`
+     * before connect. `useAvatar` (dis)connects the subscription as this
+     * changes; it does not create or own the client. */
+    client?: PipecatClient | null;
+}
+/**
+ * The mount ref's type, written out rather than named as React's `RefObject`.
+ *
+ * React 18 and 19 declare that alias with different type arguments — 18's
+ * `useRef<T>(null)` yields `RefObject<T>`, 19's yields `RefObject<T | null>` —
+ * and because both are the *same alias*, TypeScript compares them by variance
+ * and rejects whichever one we didn't pick. An anonymous shape forces a
+ * structural comparison instead, which both versions satisfy, and which the
+ * `ref` prop accepts on both. This is the only place the 18-vs-19 split shows
+ * up in the binding; keep it that way.
+ */
+export type AvatarMountRef = {
+    current: HTMLDivElement | null;
+};
+export interface UseAvatarHandle {
+    /** Attach to the mount element: `<div ref={containerRef} />`. */
+    containerRef: AvatarMountRef;
+    /** The live widget instance once mounted, else `null`. */
+    avatar: AvatarApi | null;
+    /** The dispatcher wrapping `avatar` — `null` until mounted. Exposed for
+     * tests and telemetry (`turnCtx`, `turnCues`) and for manual dispatch. */
+    client: AvatarClient | null;
+    /** Dispatch one avatar command by hand — e.g. from a dev-tools console, or
+     * from a transport that isn't a `PipecatClient`. No-ops before mount. */
+    dispatch: (msg: unknown) => void;
+}
+export declare function useAvatar(options?: UseAvatarOptions): UseAvatarHandle;
+//# sourceMappingURL=useAvatar.d.ts.map

package/client/dist/useAvatar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"useAvatar.d.ts","sourceRoot":"","sources":["../src/useAvatar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAgB,KAAK,SAAS,EAAE,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC7F,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAE3E,MAAM,WAAW,gBAAiB,SAAQ,mBAAmB;IAC3D,4EAA4E;IAC5E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,mBAAmB,CAAC,OAAO,CAAC,CAAC;IACrC,yFAAyF;IACzF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;wDAEoD;IACpD,MAAM,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;CAC/B;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,cAAc,GAAG;IAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAAA;CAAE,CAAC;AAEhE,MAAM,WAAW,eAAe;IAC9B,iEAAiE;IACjE,YAAY,EAAE,cAAc,CAAC;IAC7B,0DAA0D;IAC1D,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC;IACzB;8EAC0E;IAC1E,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5B;6EACyE;IACzE,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,CAAC;CAClC;AAED,wBAAgB,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,eAAe,CAyDzE"}

package/client/dist/useAvatar.js

@@ -0,0 +1,68 @@
+/**
+ * useAvatar — mount the widget, wire it to a live session, dispatch its
+ * server-messages, and clean up.
+ *
+ * Options are read through a ref so the effect doesn't re-subscribe on every
+ * render. Split in two effects: mounting the widget happens once (an avatar or
+ * theme swap remounts by design — see the note in the effect); attaching to
+ * the pipecat client re-runs whenever the client identity changes (a session
+ * reconnect mints a new one) or once the widget instance becomes available.
+ */
+import { useCallback, useEffect, useRef, useState } from "react";
+import { createAvatar } from "../../src/avatar.js";
+import { AvatarClient } from "./AvatarClient.js";
+export function useAvatar(options = {}) {
+    const containerRef = useRef(null);
+    const [avatar, setAvatar] = useState(null);
+    const avatarClientRef = useRef(null);
+    // Latest-options ref, so the mount effect (which runs once) still reads live
+    // callback props without re-subscribing.
+    const optionsRef = useRef(options);
+    optionsRef.current = options;
+    useEffect(() => {
+        const mount = containerRef.current;
+        if (!mount)
+            return;
+        const instance = createAvatar({
+            mount,
+            avatar: optionsRef.current.avatar,
+            theme: optionsRef.current.theme,
+            mouthGain: optionsRef.current.mouthGain,
+            gestureGain: optionsRef.current.gestureGain,
+        });
+        const wrapper = new AvatarClient(instance, {
+            onHint: (kind, msg) => optionsRef.current.onHint?.(kind, msg),
+            onUnknownCmd: (msg) => optionsRef.current.onUnknownCmd?.(msg),
+            onError: (err, msg) => optionsRef.current.onError?.(err, msg),
+            onSpeakingDrift: (info) => optionsRef.current.onSpeakingDrift?.(info),
+            accept: optionsRef.current.accept,
+            now: optionsRef.current.now,
+        });
+        avatarClientRef.current = wrapper;
+        setAvatar(instance);
+        return () => {
+            instance.destroy();
+            avatarClientRef.current = null;
+            setAvatar(null);
+        };
+        // Mount once. `avatar`/`theme`/the gains are read at mount time only — the
+        // widget has no hot-swap-avatar API (`createFace` runs once per mount), so
+        // changing them re-renders nothing here by design; a caller that needs a
+        // different avatar remounts with a `key` prop (see the component's doc).
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, []);
+    useEffect(() => {
+        const wrapper = avatarClientRef.current;
+        const pipecatClient = options.client;
+        if (!wrapper || !pipecatClient)
+            return;
+        return wrapper.attach(pipecatClient);
+        // Re-subscribe when the widget mounts or the session's client changes.
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [avatar, options.client]);
+    const dispatch = useCallback((msg) => {
+        avatarClientRef.current?.dispatch(msg);
+    }, []);
+    return { containerRef, avatar, client: avatarClientRef.current, dispatch };
+}
+//# sourceMappingURL=useAvatar.js.map

package/client/dist/useAvatar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"useAvatar.js","sourceRoot":"","sources":["../src/useAvatar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAEjE,OAAO,EAAE,YAAY,EAA4C,MAAM,qBAAqB,CAAC;AAC7F,OAAO,EAAE,YAAY,EAA4B,MAAM,mBAAmB,CAAC;AAyC3E,MAAM,UAAU,SAAS,CAAC,UAA4B,EAAE;IACtD,MAAM,YAAY,GAAG,MAAM,CAAiB,IAAI,CAAC,CAAC;IAClD,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,QAAQ,CAAmB,IAAI,CAAC,CAAC;IAC7D,MAAM,eAAe,GAAG,MAAM,CAAsB,IAAI,CAAC,CAAC;IAE1D,6EAA6E;IAC7E,yCAAyC;IACzC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IACnC,UAAU,CAAC,OAAO,GAAG,OAAO,CAAC;IAE7B,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC;QACnC,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,MAAM,QAAQ,GAAG,YAAY,CAAC;YAC5B,KAAK;YACL,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,MAAM;YACjC,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,KAAK;YAC/B,SAAS,EAAE,UAAU,CAAC,OAAO,CAAC,SAAS;YACvC,WAAW,EAAE,UAAU,CAAC,OAAO,CAAC,WAAW;SAC5C,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,QAAQ,EAAE;YACzC,MAAM,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;YAC7D,YAAY,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC;YAC7D,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;YAC7D,eAAe,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC;YACrE,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,MAAM;YACjC,GAAG,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG;SAC5B,CAAC,CAAC;QACH,eAAe,CAAC,OAAO,GAAG,OAAO,CAAC;QAClC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAEpB,OAAO,GAAG,EAAE;YACV,QAAQ,CAAC,OAAO,EAAE,CAAC;YACnB,eAAe,CAAC,OAAO,GAAG,IAAI,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC,CAAC;QACF,2EAA2E;QAC3E,2EAA2E;QAC3E,yEAAyE;QACzE,yEAAyE;QACzE,uDAAuD;IACzD,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;QACrC,IAAI,CAAC,OAAO,IAAI,CAAC,aAAa;YAAE,OAAO;QACvC,OAAO,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACrC,uEAAuE;QACvE,uDAAuD;IACzD,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAE7B,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,GAAY,EAAE,EAAE;QAC5C,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC;AAC7E,CAAC"}

package/client/src/Avatar.tsx

@@ -0,0 +1,38 @@
+/**
+ * Avatar — a call-tile-ready wrapper around the widget.
+ *
+ * Thin by design: `useAvatar` does the work; this is the div it mounts into
+ * plus prop plumbing.
+ *
+ *     <Avatar client={session.client} className="avatar-tile" />
+ *
+ * The widget has no hot-swap-avatar API — `createFace` runs once per mount —
+ * so `avatar`/`theme`/`mouthGain`/`gestureGain` are read once, at mount. To
+ * switch avatars at runtime, remount with a `key` prop:
+ *
+ *     <Avatar key={name} avatar={name} client={session.client} />
+ */
+
+import type { CSSProperties } from "react";
+import type { UseAvatarOptions } from "./useAvatar.js";
+import { useAvatar } from "./useAvatar.js";
+
+export interface AvatarProps extends UseAvatarOptions {
+  className?: string;
+  style?: CSSProperties;
+  /** Forwarded to the mount `<div>`. */
+  "aria-label"?: string;
+}
+
+export function Avatar({ className, style, "aria-label": ariaLabel, ...options }: AvatarProps) {
+  const { containerRef } = useAvatar(options);
+  return (
+    <div
+      ref={containerRef}
+      className={className}
+      style={style}
+      aria-label={ariaLabel ?? "avatar"}
+      role="img"
+    />
+  );
+}

package/client/src/AvatarClient.ts

@@ -0,0 +1,343 @@
+/**
+ * AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
+ * splice, framework-free (no React; the hook and component wrap this).
+ *
+ * ## Turn clock anchoring
+ *
+ * A turn's `t0` is anchored to `performance.now()` **at the moment this client
+ * receives the `{cmd:"speech", event:"start"}` message** — cues are
+ * client-anchored, not per-cue server-released. That message rides the RTVI
+ * data channel, ahead of the jitter-buffered audio path, so the residual error
+ * lands on the video-leads side — the side `docs/contract-protocol.md` says
+ * perceptual tolerance favours (+125 ms vs -45 ms).
+ *
+ * We investigated anchoring on pipecat client-js's own `RTVIEvent
+ * .BotStartedSpeaking`/`BotStoppedSpeaking` instead (or as a refinement) and
+ * chose not to, for two reasons:
+ *
+ *   1. **No turn correlation.** Those events carry no payload — no `ctx` — so
+ *      there is no way to tell which turn a firing belongs to. Our own
+ *      `speech` command carries `ctx`, which the splice logic below needs
+ *      regardless, so anchoring off it costs nothing extra.
+ *   2. **Same source, same path, no accuracy gain.** The `AvatarProcessor`
+ *      sits between the TTS service and the output transport and observes the
+ *      transport's own `BotStarted/StoppedSpeakingFrame` broadcasts — the exact
+ *      frame pipecat's built-in speaking detection is *also* driven from. Both
+ *      notifications travel the same data-channel path to the browser. There is
+ *      no local "truly audible now" signal cheaply available: the audio arrives
+ *      on a `MediaStreamTrack` whose only lifecycle events (`unmute`/`mute`)
+ *      fire once per call, not per utterance. Tapping the decoded remote audio
+ *      with a WebAudio `AnalyserNode` RMS gate *would* give one, but it adds
+ *      its own onset latency and a real audio pipeline to build and tune, and
+ *      it would eat into the intentional video-first safety margin rather than
+ *      improve it. Left as a documented option, not built.
+ *
+ * `attach()` still subscribes to both pipecat events, but only to report a
+ * **diagnostic** drift (`onSpeakingDrift`) between our anchor and pipecat's —
+ * useful for noticing in logs if the two ever separate by more than jitter,
+ * never used to move `t0` itself.
+ *
+ * ## Cue splice
+ *
+ * The widget has two cue-track primitives: `speak({cues, clock})` (a full
+ * replace) and `pushCues(cues)` (a pure union that can only grow the track,
+ * never shrink it). Neither is "discard queued cues at or after `from_ms`,
+ * then append" on its own — `pushCues` has no way to drop a stale tail. So
+ * this client keeps the turn's canonical cue array itself (kept portion +
+ * every appended chunk, spliced on each `cues` message) and picks the cheapest
+ * widget call that stays correct:
+ *
+ *   - if the splice's `from_ms` doesn't reach back into anything already
+ *     queued — the common case past a turn's first sentence, since only the
+ *     first sentence genuinely plays fast-leg cues — nothing needs discarding:
+ *     `pushCues(newCues)` is the cheap, correct append.
+ *   - if it does reach back (a real fast→accurate splice), `pushCues` cannot
+ *     express the discard; we call `speak()` again with the full spliced
+ *     canonical array on the turn's original clock. `speak()` is otherwise
+ *     documented as also killing an in-flight spoken interjection and
+ *     re-entering `SPEAKING` — both harmless mid-splice (an interjection
+ *     should not be running while a server track owns the mouth; re-entering
+ *     an unchanged state is a no-op past the profile/gaze reset the widget
+ *     already does for a same-name `setState`).
+ *
+ * Cues commonly arrive **before** `speech start` — the fast leg starts the
+ * moment a sentence is handed to TTS, well before `BotStartedSpeakingFrame`.
+ * Chunks that arrive before the clock is anchored are spliced into the
+ * canonical array but not yet handed to the widget; `speech start` hands over
+ * whatever has accumulated as the turn's first `speak()` call. So "the first
+ * chunk of a turn starts speak()" means the first *widget* call, not
+ * necessarily the first *message*.
+ */
+
+import type { PipecatClient, RTVIEvent } from "@pipecat-ai/client-js";
+import type { AvatarApi } from "../../src/avatar.js";
+import {
+  AVATAR_MESSAGE_TYPE,
+  isAvatarMessage,
+  type AvatarCommand,
+  type AvatarCue,
+  type AvatarCuesCmd,
+  type AvatarHintCmd,
+  type AvatarPerformCmd,
+  type AvatarSpeechCmd,
+  type AvatarStateCmd,
+  type AvatarUnknownCmd,
+} from "./types.js";
+
+interface Turn {
+  ctx: string;
+  /** The canonical, already-spliced cue track for this turn. */
+  cues: AvatarCue[];
+  /** Whether `speech start` has anchored a clock and issued the first `speak()`. */
+  started: boolean;
+  clock: (() => number) | null;
+  t0: number | null;
+}
+
+export interface AvatarClientOptions {
+  /** `{cmd:"hint"}` is a no-op hook today — the widget's listening engine
+   * already handles acks; a host may still want to know a hint arrived. */
+  onHint?: (kind: string, msg: AvatarHintCmd) => void;
+  /** An unrecognized `cmd` (forward compat) — the protocol says ignore
+   * silently, so this is purely an observability hook, not required. */
+  onUnknownCmd?: (msg: AvatarUnknownCmd) => void;
+  /** A dispatch threw (e.g. an unknown state or interjection id, which the
+   * widget throws on). Defaults to `console.warn`. */
+  onError?: (err: unknown, msg: AvatarCommand) => void;
+  /** Diagnostic only (see the class doc's "Turn clock anchoring" section) —
+   * never moves the anchor, just reports how far pipecat's own
+   * botStartedSpeaking/botStoppedSpeaking landed from it. */
+  onSpeakingDrift?: (info: { event: "start" | "stop"; ctx: string | null; driftMs: number }) => void;
+  /**
+   * Which server-messages `attach()` should look inside. Defaults to the
+   * protocol's own envelope, `type === "avatar"`.
+   *
+   * The escape hatch exists because an application may tunnel avatar commands
+   * inside a message type of its own — one deployment routes them through a
+   * generic `ui_command` envelope so an LLM tool call can drive the face — and
+   * teaching this library that envelope would be teaching it one consumer's
+   * private vocabulary. Widen it here instead:
+   *
+   *     accept: (m) => m.type === "avatar" ||
+   *                    (m.type === "ui_command" && m.action === "avatar")
+   *
+   * The predicate only decides *whether to look*; the payload still has to
+   * carry a string `cmd` to dispatch at all.
+   */
+  accept?: (message: Record<string, unknown>) => boolean;
+  /** Override for tests. Defaults to `performance.now`. */
+  now?: () => number;
+}
+
+/**
+ * The three `RTVIEvent` members `attach()` subscribes to, spelled as their
+ * values.
+ *
+ * Written out rather than imported because that enum was this module's *only*
+ * runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
+ * the whole `/pipecat` subpath fail to load without the peer installed — even
+ * for a host that drives `dispatch()` from its own transport and never calls
+ * `attach()`. The peer is declared optional; this is what makes that true
+ * rather than aspirational.
+ *
+ * String enums are nominal in TypeScript, so the compiler cannot check these
+ * against the real ones from a type-only import. `client/test/AvatarClient.test.ts`
+ * does it instead, against the actual enum — the devDependency is present
+ * exactly where the check belongs and absent from what we ship.
+ */
+export const RTVI_EVENTS = {
+  serverMessage: "serverMessage",
+  botStartedSpeaking: "botStartedSpeaking",
+  botStoppedSpeaking: "botStoppedSpeaking",
+} as const satisfies Record<string, string>;
+
+/** Defensive unwrap for the `RTVIEvent.ServerMessage` `{ data }` quirk: some
+ * transports deliver the payload directly and some wrap it once more. */
+function unwrapServerMessage(raw: unknown): Record<string, unknown> {
+  const obj = (raw ?? {}) as Record<string, unknown>;
+  const inner = obj["data"] as Record<string, unknown> | undefined;
+  return inner && "type" in inner ? inner : obj;
+}
+
+export class AvatarClient {
+  private readonly avatar: AvatarApi;
+  private readonly opts: AvatarClientOptions;
+  private readonly now: () => number;
+  private readonly accept: (message: Record<string, unknown>) => boolean;
+  private turn: Turn | null = null;
+
+  constructor(avatar: AvatarApi, opts: AvatarClientOptions = {}) {
+    this.avatar = avatar;
+    this.opts = opts;
+    this.now = opts.now ?? (() => performance.now());
+    this.accept = opts.accept ?? ((m) => m.type === AVATAR_MESSAGE_TYPE);
+  }
+
+  /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
+  get turnCtx(): string | null {
+    return this.turn?.ctx ?? null;
+  }
+
+  /** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
+  get turnCues(): AvatarCue[] {
+    return this.turn ? [...this.turn.cues] : [];
+  }
+
+  /** Dispatch one avatar command. Accepts anything with a string `cmd` — an
+   * already-unwrapped `{type:"avatar", cmd, ...}` server message, or a bare
+   * `{cmd, ...}` payload from whatever else the host is carrying them in.
+   * Unknown `cmd`s are ignored, per the wire protocol's forward-compat rule. */
+  dispatch(raw: unknown): void {
+    if (!isAvatarMessage(raw)) return;
+    const msg = raw;
+    try {
+      switch (msg.cmd) {
+        case "state":
+          this.handleState(msg as AvatarStateCmd);
+          break;
+        case "interject":
+          this.avatar.interject((msg as { id: string }).id);
+          break;
+        case "perform":
+          this.handlePerform(msg as AvatarPerformCmd);
+          break;
+        case "cues":
+          this.handleCues(msg as AvatarCuesCmd);
+          break;
+        case "speech":
+          this.handleSpeech(msg as AvatarSpeechCmd);
+          break;
+        case "user":
+          this.avatar.setUserSpeaking((msg as { speaking: boolean }).speaking);
+          break;
+        case "hint": {
+          const hint = msg as AvatarHintCmd;
+          this.opts.onHint?.(hint.kind, hint);
+          break;
+        }
+        default:
+          this.opts.onUnknownCmd?.(msg as AvatarUnknownCmd);
+          break;
+      }
+    } catch (err) {
+      if (this.opts.onError) this.opts.onError(err, msg);
+      else console.warn("[avatar] dispatch failed", msg, err);
+    }
+  }
+
+  private handleState(msg: AvatarStateCmd) {
+    // Deliberately no client-side dedup: pass every `state` command straight
+    // through. The widget's own setState already no-ops the parts that matter
+    // for an unchanged name (`changed` gates the blink and the 'state' event in
+    // avatar.js), and a server resending the same state name as a
+    // keepalive/resync must still land so an `emotion`/`gaze` override on this
+    // particular message takes effect.
+    this.avatar.setState(msg.name, { emotion: msg.emotion, gaze: msg.gaze });
+  }
+
+  private handlePerform(msg: AvatarPerformCmd) {
+    this.avatar.perform(msg.actions, { clock: this.resolveClock(msg.ctx) });
+  }
+
+  /** Ride the named turn's clock if it's the one we're currently anchored to;
+   * otherwise (no active turn, or `perform` names a ctx we never saw a
+   * `speech start` for) fall back to a fresh clock anchored at this call — the
+   * same "elapsed ms since this call" default `avatar.perform()` itself uses
+   * when given no clock and no audio. */
+  private resolveClock(ctx: string | undefined): () => number {
+    if (ctx && this.turn && this.turn.ctx === ctx && this.turn.clock) {
+      return this.turn.clock;
+    }
+    const start = this.now();
+    return () => this.now() - start;
+  }
+
+  private ensureTurn(ctx: string): Turn {
+    if (!this.turn || this.turn.ctx !== ctx) {
+      // A different ctx supersedes whatever turn we had — a stale trailing
+      // message for the old ctx will find `this.turn.ctx !== ctx` in
+      // handleSpeech's stop-guard and be ignored, rather than cutting off the
+      // new turn.
+      this.turn = { ctx, cues: [], started: false, clock: null, t0: null };
+    }
+    return this.turn;
+  }
+
+  private handleCues(msg: AvatarCuesCmd) {
+    const turn = this.ensureTurn(msg.ctx);
+    const kept = turn.cues.filter((c) => c.t < msg.from_ms);
+    const discarded = turn.cues.length - kept.length;
+    turn.cues = [...kept, ...msg.cues].sort((a, b) => a.t - b.t);
+
+    if (!turn.started) {
+      // No clock yet — buffer. `speech start` will hand this over as the turn's
+      // first speak() call.
+      return;
+    }
+    if (discarded === 0) {
+      this.avatar.pushCues(msg.cues);
+    } else {
+      this.avatar.speak({ cues: turn.cues, clock: turn.clock! });
+    }
+  }
+
+  private handleSpeech(msg: AvatarSpeechCmd) {
+    if (msg.event === "start") {
+      const turn = this.ensureTurn(msg.ctx);
+      const t0 = this.now();
+      const clock = () => this.now() - t0;
+      turn.t0 = t0;
+      turn.clock = clock;
+      turn.started = true;
+      this.avatar.speak({ cues: turn.cues, clock });
+      return;
+    }
+    // "stop": only act if it names the turn we're actually riding. A stale stop
+    // for an already-superseded ctx must not cut off a newer turn.
+    if (this.turn && this.turn.ctx === msg.ctx) {
+      this.avatar.stopSpeaking();
+      this.turn = null;
+    }
+  }
+
+  private reportDrift(event: "start" | "stop") {
+    if (!this.opts.onSpeakingDrift) return;
+    const t0 = this.turn?.t0;
+    if (t0 == null) return;
+    this.opts.onSpeakingDrift({ event, ctx: this.turn?.ctx ?? null, driftMs: this.now() - t0 });
+  }
+
+  /**
+   * Subscribe to a live `PipecatClient`'s server messages and dispatch the
+   * avatar commands among them. Which messages count is the `accept` option;
+   * by default, the protocol's own `{type:"avatar"}` envelope.
+   *
+   * Also wires the diagnostic drift cross-check described in the class doc.
+   * Never throws on a malformed or irrelevant message.
+   *
+   * @returns an unsubscribe function; call it on unmount or disconnect.
+   */
+  attach(client: PipecatClient): () => void {
+    const onServerMessage = (raw: unknown) => {
+      const message = unwrapServerMessage(raw);
+      if (!this.accept(message)) return;
+      this.dispatch(message);
+    };
+    const onBotStartedSpeaking = () => this.reportDrift("start");
+    const onBotStoppedSpeaking = () => this.reportDrift("stop");
+
+    const serverMessage = RTVI_EVENTS.serverMessage as RTVIEvent;
+    const started = RTVI_EVENTS.botStartedSpeaking as RTVIEvent;
+    const stopped = RTVI_EVENTS.botStoppedSpeaking as RTVIEvent;
+
+    client.on(serverMessage, onServerMessage);
+    client.on(started, onBotStartedSpeaking);
+    client.on(stopped, onBotStoppedSpeaking);
+
+    return () => {
+      client.off(serverMessage, onServerMessage);
+      client.off(started, onBotStartedSpeaking);
+      client.off(stopped, onBotStoppedSpeaking);
+    };
+  }
+}