@voqalize/avatar 0.2.2 → 0.4.0

package/src/rig.js

@@ -0,0 +1,32 @@
+/**
+ * Renderer-agnostic frame types expressed as runtime helpers.
+ *
+ * `createSvgRig()` is a migration adapter for the existing SVG faces. It lets
+ * the mixer submit one AvatarFrame per tick today while new renderers can
+ * implement the same small `{ apply(frame), destroy() }` contract directly.
+ */
+
+export const HAND_GESTURE_NAMES = Object.freeze(['greet', 'farewell', 'approve', 'wait']);
+
+/** @param {Record<string, number>} pose @param {object|undefined} hand */
+export function avatarFrame(pose, hand) {
+  return hand ? { pose, hand } : { pose };
+}
+
+/**
+ * Adapt an existing SVG face plus its first-class hand renderer to AvatarRig.
+ * `face` and `hand` retain their own SVG-private geometry; callers only submit
+ * semantic frames.
+ */
+export function createSvgRig(face, hand = null) {
+  return {
+    apply(frame) {
+      face.apply(frame.pose);
+      if (hand && typeof hand.applyFrame === 'function') hand.applyFrame(frame.hand);
+    },
+    destroy() {
+      if (hand) hand.destroy();
+      face.destroy();
+    },
+  };
+}

package/src/speech-timing.js

@@ -0,0 +1,23 @@
+/**
+ * Timing policy for the mouth track.
+ *
+ * Keep perceptual cleanup, cue presentation, and mouth response constants
+ * named by their job. Network delivery is deliberately absent: it cannot be
+ * corrected by shifting every cue on the utterance clock.
+ */
+
+/** Cues shorter than this do not survive visual normalization. */
+export const MIN_VISIBLE_CUE_MS = 30;
+
+/** Cue timestamps render verbatim; the clock's epoch is supplied by the caller. */
+export const CUE_TRACK_LEAD_MS = 0;
+
+/** Extra time after a final silence cue before a general speech track ends. */
+export const SPEECH_TRACK_TAIL_MS = 120;
+
+/** Shorter completion tail for self-contained spoken interjection clips. */
+export const INTERJECTION_TRACK_TAIL_MS = 60;
+
+/** First-order response constants: lips settle before the jaw does. */
+export const MOUTH_RESPONSE_TAU_S = 0.042;
+export const JAW_RESPONSE_TAU_S = 0.07;

package/src/visemes.js

@@ -17,12 +17,28 @@
  *   H  tongue up            L
  *   X  idle / silence
  *
- * A cue is `{ t, v, i? }` — millisecond offset into the utterance, the letter,
- * and an optional 0..1 intensity (loudness). Intensity is cheap for the server
- * to derive from TTS energy and is the single biggest realism win available:
- * the same viseme shouted and murmured should not look identical.
+ * A cue is `{ t, v, i?, p? }` — millisecond offset into the utterance, the
+ * letter, an optional 0..1 intensity (loudness): the same viseme shouted and
+ * murmured should not look identical, and an optional phone label. The backend
+ * measures `i` from the RMS under each cue, but only on the leg that has audio
+ * to measure — a predicted cue describes speech nobody has generated yet, so it
+ * arrives without one and is read here as 1.
+ *
+ * `p` is the phone being articulated under the letter, and the nine letters are
+ * a lossy projection of it: `B` above lists four phone families and absorbs
+ * fourteen more. Nothing in this file reads it — `VISEME_SHAPES` is the mouth
+ * every face owes the server and it is keyed by letter. It rides through
+ * normalization and out of `sample()` so that a rig with a finer mouth than
+ * nine shapes can use it without the server sending a second track, and so that
+ * a rig without one is unaffected.
  */
 
+import {
+  CUE_TRACK_LEAD_MS,
+  MIN_VISIBLE_CUE_MS,
+  SPEECH_TRACK_TAIL_MS,
+} from './speech-timing.js';
+
 export const VISEME_SHAPES = {
   X: { mouthOpen: 0.02, mouthWidth: 0.42, mouthRound: 0.10, mouthPress: 0.15, mouthTuck: 0, teethUpper: 0.00, tongue: 0.0 },
   A: { mouthOpen: 0.00, mouthWidth: 0.40, mouthRound: 0.18, mouthPress: 0.55, mouthTuck: 0, teethUpper: 0.00, tongue: 0.0 },
@@ -63,42 +79,106 @@ export function shapeFor(letter, intensity = 1) {
 // ---------------------------------------------------------------------------
 // Cue track hygiene
 // ---------------------------------------------------------------------------
-const MIN_CUE_MS = 30; // shorter than this and the mouth just flutters
+
+/**
+ * Group a time-ordered cue list into runs that share a mouth shape.
+ *
+ * A backend emits one cue per intersection of its shape and phone timelines, so
+ * a held shape arrives as several cues differing only in `p`. Grouping them is
+ * what lets the visibility rules stay rules about shapes.
+ */
+function shapeRuns(cues) {
+  const runs = [];
+  for (const c of cues) {
+    const v = VISEME_SHAPES[c.v] ? c.v : SILENT;
+    const clean = { t: c.t, v, i: c.i == null ? 1 : c.i, p: c.p ?? null };
+    if (runs.length && runs[runs.length - 1][0].v === v) runs[runs.length - 1].push(clean);
+    else runs.push([clean]);
+  }
+  return runs;
+}
 
 /**
  * Sort, merge consecutive duplicates, and drop sub-perceptual cues. Servers
  * emit noisy tracks; this makes them watchable.
+ *
+ * **Every decision here is a decision about shapes**, taken over the run heads
+ * and nothing else. MIN_VISIBLE_CUE_MS and the closure swap are rules about what
+ * the face can be *seen* to do, and a phone transition inside a held shape is
+ * not something the face does at all — letting one participate would mean a
+ * phone changing 15 ms before a real shape change could swallow that change,
+ * which is a lipsync regression bought with a field this file does not read.
+ * Phone detail is re-attached afterwards, bounded by the mouth positions either
+ * side of it. Mirrors `normalize_cues` server-side.
  */
 export function normalizeCues(cues) {
+  const runs = shapeRuns([...cues].sort((a, b) => a.t - b.t));
+
+  // Pass one: the shape track, by exactly the rule that predates phone detail.
+  // `kept` maps each surviving cue back to the run it speaks for, which the
+  // closure swap can change without moving the cue's timestamp.
   const out = [];
-  const sorted = [...cues].sort((a, b) => a.t - b.t);
-  for (const c of sorted) {
-    const v = VISEME_SHAPES[c.v] ? c.v : SILENT;
+  const kept = [];
+  for (let index = 0; index < runs.length; index++) {
+    const head = runs[index][0];
+    const v = head.v;
     const prev = out[out.length - 1];
     if (prev && prev.v === v) continue; // merge repeats
-    if (prev && c.t - prev.t < MIN_CUE_MS) {
+    if (prev && head.t - prev.t < MIN_VISIBLE_CUE_MS) {
       // Too short to read. Keep whichever is more visually salient: a closure
       // (A/G) carries more lip-reading information than a mid-open vowel.
-      if (v === 'A' || v === 'G') out[out.length - 1] = { ...c, v };
+      if (v === 'A' || v === 'G') {
+        // The short cue can sit between two identical closures (G → F → G).
+        // Its replacement would otherwise create a duplicate visible shape;
+        // preserving the first G is both the stable wire form and the face the
+        // viewer actually saw.
+        if (out.length > 1 && out[out.length - 2].v === v) { out.pop(); kept.pop(); }
+        // A winning closure replaces the preceding shape for the entire
+        // sub-perceptual interval. Preserve that cue's timestamp while taking
+        // the closure's intensity, matching the server-side wire normalizer.
+        else {
+          out[out.length - 1] = { ...head, t: prev.t, v };
+          kept[kept.length - 1] = index;
+        }
+      }
       continue;
     }
-    out.push({ t: c.t, v, i: c.i == null ? 1 : c.i });
+    out.push({ t: head.t, v, i: head.i, p: head.p });
+    kept.push(index);
   }
-  return out;
+
+  // Pass two: the phone detail inside each surviving mouth position. Silence is
+  // never split — nothing is being articulated under a rest.
+  const result = [];
+  for (let slot = 0; slot < out.length; slot++) {
+    const cue = out[slot];
+    result.push(cue);
+    if (cue.v === SILENT) continue;
+    const limit = slot + 1 < out.length ? out[slot + 1].t : Infinity;
+    for (const sub of runs[kept[slot]].slice(1)) {
+      // Strictly after whatever was last emitted, not merely after the head: a
+      // splice joins two legs' tracks and both can name the same millisecond.
+      if (sub.t <= result[result.length - 1].t || sub.t >= limit) continue;
+      if (sub.p === result[result.length - 1].p) continue;
+      result.push({ t: sub.t, v: cue.v, i: sub.i, p: sub.p });
+    }
+  }
+  return result;
 }
 
 /**
- * Schedules a cue track against an audio clock.
+ * Schedules a cue track against an utterance clock.
  *
- * The clock must come from the audio itself (`audioEl.currentTime * 1000` or
- * `AudioContext.currentTime`), never from wall time — wall time drifts against
- * playback and you will spend the rest of your life chasing it.
+ * An audio-owned clock (`audioEl.currentTime * 1000` or
+ * `AudioContext.currentTime`) is the strongest source when the caller owns
+ * playback. The Pipecat adapter cannot see browser device playout; it supplies
+ * elapsed time from `botStartedSpeaking`, Pipecat's output-lifecycle epoch.
  *
- * LEAD_MS biases the mouth slightly ahead of the sound. Perceptually the
- * tolerance is asymmetric: roughly -45ms (audio first) to +125ms (video first),
- * so leading is the safe side to err on.
+ * There is deliberately no renderer-wide lead. Network/media skew cannot be
+ * corrected by moving every cue, and the predicted backend leg owns its own
+ * explicit prediction cushion.
  */
-export const LEAD_MS = 40;
+export const LEAD_MS = CUE_TRACK_LEAD_MS;
 
 export class VisemeTrack {
   constructor() {
@@ -107,9 +187,14 @@ export class VisemeTrack {
     this.playing = false;
     this._idx = 0;
     this.onEnd = null;
-    this.tailMs = 120; // how long past the last cue before we call it done
+    this.tailMs = SPEECH_TRACK_TAIL_MS;
+    // Where the last sample() landed, for the layers that read the track's
+    // future (prosody.js looks ahead to the end of a pause).
+    this.now = 0;
   }
 
+  get index() { return this._idx; }
+
   /** @param {() => number} clock returns elapsed ms of the audio being played */
   start(cues, clock) {
     this.cues = normalizeCues(cues);
@@ -133,10 +218,11 @@ export class VisemeTrack {
     this._idx = 0;
   }
 
-  /** @returns {{letter: string, intensity: number} | null} */
+  /** @returns {{letter: string, intensity: number, phone: string | null} | null} */
   sample() {
     if (!this.playing || !this.cues.length || !this.clock) return null;
     const now = this.clock() + LEAD_MS;
+    this.now = now;
 
     // Cues are time-ordered and `now` is mostly monotonic, so this walk is O(1)
     // amortized. Reset on seek-backward.
@@ -151,8 +237,8 @@ export class VisemeTrack {
     }
 
     const cue = this.cues[this._idx];
-    if (cue.t > now) return { letter: SILENT, intensity: 1 };
-    return { letter: cue.v, intensity: cue.i == null ? 1 : cue.i };
+    if (cue.t > now) return { letter: SILENT, intensity: 1, phone: null };
+    return { letter: cue.v, intensity: cue.i == null ? 1 : cue.i, phone: cue.p ?? null };
   }
 }
 

package/client/dist/Avatar.d.ts

@@ -1,27 +0,0 @@
-/**
- * Avatar — a call-tile-ready wrapper around the widget, and the package's
- * whole public surface.
- *
- *     <Avatar client={session.client} className="avatar-tile" />
- *
- * Two props of its own; everything else is forwarded to the mount `<div>`, so
- * it sizes and styles like the tile it lives in. There is nothing to configure
- * because there is nothing the server does not already say: the
- * `AvatarProcessor` in the pipeline drives the state, the gaze and the mouth.
- *
- * The widget has no hot-swap-avatar API — `createFace` runs once per mount —
- * so `avatar` is read once, at mount. To switch faces at runtime, remount with
- * a `key` prop:
- *
- *     <Avatar key={name} avatar={name} client={session.client} />
- */
-import type { HTMLAttributes } from "react";
-import type { PipecatClient } from "@pipecat-ai/client-js";
-export interface AvatarProps extends HTMLAttributes<HTMLDivElement> {
-    /** The live `PipecatClient`, or `null` before connect. */
-    client?: PipecatClient | null;
-    /** Which face. Omit for the default. Read at mount only — see above. */
-    avatar?: string;
-}
-export declare function Avatar({ client, avatar, ...rest }: AvatarProps): import("react").JSX.Element;
-//# sourceMappingURL=Avatar.d.ts.map

package/client/dist/Avatar.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"Avatar.d.ts","sourceRoot":"","sources":["../src/Avatar.tsx"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,OAAO,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAG3D,MAAM,WAAW,WAAY,SAAQ,cAAc,CAAC,cAAc,CAAC;IACjE,0DAA0D;IAC1D,MAAM,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,EAAE,WAAW,+BAG9D"}

package/client/dist/Avatar.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"Avatar.js","sourceRoot":"","sources":["../src/Avatar.tsx"],"names":[],"mappings":";AAoBA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAS3C,MAAM,UAAU,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAe;IAC7D,MAAM,EAAE,YAAY,EAAE,GAAG,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACvD,OAAO,cAAK,IAAI,EAAC,KAAK,gBAAY,QAAQ,KAAK,IAAI,EAAE,GAAG,EAAE,YAAY,GAAI,CAAC;AAC7E,CAAC"}

package/client/dist/AvatarClient.d.ts

@@ -1,139 +0,0 @@
-/**
- * AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
- * splice, framework-free (no React; the hook and component wrap this).
- *
- * ## Turn clock anchoring
- *
- * A turn's `t0` is anchored to `performance.now()` **at the moment this client
- * receives the `{cmd:"speech", event:"start"}` message** — cues are
- * client-anchored, not per-cue server-released. That message rides the RTVI
- * data channel, ahead of the jitter-buffered audio path, so the residual error
- * lands on the video-leads side — the side `docs/contract-protocol.md` says
- * perceptual tolerance favours (+125 ms vs -45 ms).
- *
- * We investigated anchoring on pipecat client-js's own `RTVIEvent
- * .BotStartedSpeaking`/`BotStoppedSpeaking` instead (or as a refinement) and
- * chose not to, for two reasons:
- *
- *   1. **No turn correlation.** Those events carry no payload — no `ctx` — so
- *      there is no way to tell which turn a firing belongs to. Our own
- *      `speech` command carries `ctx`, which the splice logic below needs
- *      regardless, so anchoring off it costs nothing extra.
- *   2. **Same source, same path, no accuracy gain.** The `AvatarProcessor`
- *      sits between the TTS service and the output transport and observes the
- *      transport's own `BotStarted/StoppedSpeakingFrame` broadcasts — the exact
- *      frame pipecat's built-in speaking detection is *also* driven from. Both
- *      notifications travel the same data-channel path to the browser. There is
- *      no local "truly audible now" signal cheaply available: the audio arrives
- *      on a `MediaStreamTrack` whose only lifecycle events (`unmute`/`mute`)
- *      fire once per call, not per utterance. Tapping the decoded remote audio
- *      with a WebAudio `AnalyserNode` RMS gate *would* give one, but it adds
- *      its own onset latency and a real audio pipeline to build and tune, and
- *      it would eat into the intentional video-first safety margin rather than
- *      improve it. Left as a documented option, not built.
- *
- * `attach()` therefore subscribes to exactly one pipecat event,
- * `serverMessage`. It used to also subscribe to both speaking events to report
- * a diagnostic drift between our anchor and pipecat's; that hook is gone with
- * the rest of the observability surface (`docs/removed.md` § Client callbacks).
- *
- * ## Cue splice
- *
- * The widget has two cue-track primitives: `speak({cues, clock})` (a full
- * replace) and `pushCues(cues)` (a pure union that can only grow the track,
- * never shrink it). Neither is "discard queued cues at or after `from_ms`,
- * then append" on its own — `pushCues` has no way to drop a stale tail. So
- * this client keeps the turn's canonical cue array itself (kept portion +
- * every appended chunk, spliced on each `cues` message) and picks the cheapest
- * widget call that stays correct:
- *
- *   - if the splice's `from_ms` doesn't reach back into anything already
- *     queued — the common case past a turn's first sentence, since only the
- *     first sentence genuinely plays fast-leg cues — nothing needs discarding:
- *     `pushCues(newCues)` is the cheap, correct append.
- *   - if it does reach back (a real fast→accurate splice), `pushCues` cannot
- *     express the discard; we call `speak()` again with the full spliced
- *     canonical array on the turn's original clock. `speak()` is otherwise
- *     documented as also killing an in-flight spoken interjection and
- *     re-entering `SPEAKING` — both harmless mid-splice (an interjection
- *     should not be running while a server track owns the mouth; re-entering
- *     an unchanged state is a no-op past the profile/gaze reset the widget
- *     already does for a same-name `setState`).
- *
- * Cues commonly arrive **before** `speech start` — the fast leg starts the
- * moment a sentence is handed to TTS, well before `BotStartedSpeakingFrame`.
- * Chunks that arrive before the clock is anchored are spliced into the
- * canonical array but not yet handed to the widget; `speech start` hands over
- * whatever has accumulated as the turn's first `speak()` call. So "the first
- * chunk of a turn starts speak()" means the first *widget* call, not
- * necessarily the first *message*.
- */
-import type { PipecatClient } from "@pipecat-ai/client-js";
-import type { AvatarApi } from "../../src/avatar.js";
-import { type AvatarCommand, type AvatarCue } from "./types.js";
-/**
- * Internal. Not exported from the package — see `index.ts` for the public
- * surface, which is `<Avatar>` and nothing else.
- *
- * There is deliberately no `accept` predicate here any more. Avatar commands
- * travel in one envelope, `{type:"avatar"}`, in both directions and from every
- * source: a `AvatarProcessor` in the pipeline and a brain driving the face
- * out of band emit the same shape. A per-deployment predicate meant the
- * library could not state what an avatar message *is*, which is the one thing
- * a wire format has to be able to say. See docs/removed.md § The accept
- * predicate.
- */
-export interface AvatarClientOptions {
-    /** A dispatch threw (e.g. an unknown state or interjection id, which the
-     * widget throws on). Defaults to `console.warn`. */
-    onError?: (err: unknown, msg: AvatarCommand) => void;
-    /** Override for tests. Defaults to `performance.now`. */
-    now?: () => number;
-}
-/**
- * The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
- *
- * Written out rather than imported because that enum was this module's *only*
- * runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
- * the whole `/pipecat` subpath fail to load without the peer installed — even
- * for a host that drives `dispatch()` from its own transport and never calls
- * `attach()`. The peer is declared optional; this is what makes that true
- * rather than aspirational.
- *
- * String enums are nominal in TypeScript, so the compiler cannot check these
- * against the real ones from a type-only import. `client/test/AvatarClient.test.ts`
- * does it instead, against the actual enum — the devDependency is present
- * exactly where the check belongs and absent from what we ship.
- */
-export declare const RTVI_EVENTS: {
-    readonly serverMessage: "serverMessage";
-};
-export declare class AvatarClient {
-    private readonly avatar;
-    private readonly opts;
-    private readonly now;
-    private turn;
-    constructor(avatar: AvatarApi, opts?: AvatarClientOptions);
-    /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
-    get turnCtx(): string | null;
-    /** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
-    get turnCues(): AvatarCue[];
-    /** Dispatch one server message. Anything that isn't in the avatar envelope
-     * is not ours and is ignored; so is an envelope carrying a `cmd` this build
-     * has never heard of, per the wire protocol's forward-compat rule. */
-    dispatch(raw: unknown): void;
-    private handleState;
-    private ensureTurn;
-    private handleCues;
-    private handleSpeech;
-    /**
-     * Subscribe to a live `PipecatClient`'s server messages and dispatch the
-     * avatar commands among them — the ones in the protocol's own
-     * `{type:"avatar"}` envelope, which `isAvatarMessage` is the definition of.
-     * Never throws on a malformed or irrelevant message.
-     *
-     * @returns an unsubscribe function; call it on unmount or disconnect.
-     */
-    attach(client: PipecatClient): () => void;
-}
-//# sourceMappingURL=AvatarClient.d.ts.map

package/client/dist/AvatarClient.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"AvatarClient.d.ts","sourceRoot":"","sources":["../src/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAa,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAEL,KAAK,aAAa,EAClB,KAAK,SAAS,EAIf,MAAM,YAAY,CAAC;AAYpB;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,mBAAmB;IAClC;wDACoD;IACpD,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC;IACrD,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,WAAW;;CAEmB,CAAC;AAU5C,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,IAAI,CAAqB;gBAErB,MAAM,EAAE,SAAS,EAAE,IAAI,GAAE,mBAAwB;IAM7D,+EAA+E;IAC/E,IAAI,OAAO,IAAI,MAAM,GAAG,IAAI,CAE3B;IAED,wFAAwF;IACxF,IAAI,QAAQ,IAAI,SAAS,EAAE,CAE1B;IAED;;0EAEsE;IACtE,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IAkC5B,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,UAAU;IAWlB,OAAO,CAAC,UAAU;IAkBlB,OAAO,CAAC,YAAY;IAkBpB;;;;;;;OAOG;IACH,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,IAAI;CAM1C"}

package/client/dist/AvatarClient.js

@@ -1,223 +0,0 @@
-/**
- * AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
- * splice, framework-free (no React; the hook and component wrap this).
- *
- * ## Turn clock anchoring
- *
- * A turn's `t0` is anchored to `performance.now()` **at the moment this client
- * receives the `{cmd:"speech", event:"start"}` message** — cues are
- * client-anchored, not per-cue server-released. That message rides the RTVI
- * data channel, ahead of the jitter-buffered audio path, so the residual error
- * lands on the video-leads side — the side `docs/contract-protocol.md` says
- * perceptual tolerance favours (+125 ms vs -45 ms).
- *
- * We investigated anchoring on pipecat client-js's own `RTVIEvent
- * .BotStartedSpeaking`/`BotStoppedSpeaking` instead (or as a refinement) and
- * chose not to, for two reasons:
- *
- *   1. **No turn correlation.** Those events carry no payload — no `ctx` — so
- *      there is no way to tell which turn a firing belongs to. Our own
- *      `speech` command carries `ctx`, which the splice logic below needs
- *      regardless, so anchoring off it costs nothing extra.
- *   2. **Same source, same path, no accuracy gain.** The `AvatarProcessor`
- *      sits between the TTS service and the output transport and observes the
- *      transport's own `BotStarted/StoppedSpeakingFrame` broadcasts — the exact
- *      frame pipecat's built-in speaking detection is *also* driven from. Both
- *      notifications travel the same data-channel path to the browser. There is
- *      no local "truly audible now" signal cheaply available: the audio arrives
- *      on a `MediaStreamTrack` whose only lifecycle events (`unmute`/`mute`)
- *      fire once per call, not per utterance. Tapping the decoded remote audio
- *      with a WebAudio `AnalyserNode` RMS gate *would* give one, but it adds
- *      its own onset latency and a real audio pipeline to build and tune, and
- *      it would eat into the intentional video-first safety margin rather than
- *      improve it. Left as a documented option, not built.
- *
- * `attach()` therefore subscribes to exactly one pipecat event,
- * `serverMessage`. It used to also subscribe to both speaking events to report
- * a diagnostic drift between our anchor and pipecat's; that hook is gone with
- * the rest of the observability surface (`docs/removed.md` § Client callbacks).
- *
- * ## Cue splice
- *
- * The widget has two cue-track primitives: `speak({cues, clock})` (a full
- * replace) and `pushCues(cues)` (a pure union that can only grow the track,
- * never shrink it). Neither is "discard queued cues at or after `from_ms`,
- * then append" on its own — `pushCues` has no way to drop a stale tail. So
- * this client keeps the turn's canonical cue array itself (kept portion +
- * every appended chunk, spliced on each `cues` message) and picks the cheapest
- * widget call that stays correct:
- *
- *   - if the splice's `from_ms` doesn't reach back into anything already
- *     queued — the common case past a turn's first sentence, since only the
- *     first sentence genuinely plays fast-leg cues — nothing needs discarding:
- *     `pushCues(newCues)` is the cheap, correct append.
- *   - if it does reach back (a real fast→accurate splice), `pushCues` cannot
- *     express the discard; we call `speak()` again with the full spliced
- *     canonical array on the turn's original clock. `speak()` is otherwise
- *     documented as also killing an in-flight spoken interjection and
- *     re-entering `SPEAKING` — both harmless mid-splice (an interjection
- *     should not be running while a server track owns the mouth; re-entering
- *     an unchanged state is a no-op past the profile/gaze reset the widget
- *     already does for a same-name `setState`).
- *
- * Cues commonly arrive **before** `speech start` — the fast leg starts the
- * moment a sentence is handed to TTS, well before `BotStartedSpeakingFrame`.
- * Chunks that arrive before the clock is anchored are spliced into the
- * canonical array but not yet handed to the widget; `speech start` hands over
- * whatever has accumulated as the turn's first `speak()` call. So "the first
- * chunk of a turn starts speak()" means the first *widget* call, not
- * necessarily the first *message*.
- */
-import { isAvatarMessage, } from "./types.js";
-/**
- * The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
- *
- * Written out rather than imported because that enum was this module's *only*
- * runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
- * the whole `/pipecat` subpath fail to load without the peer installed — even
- * for a host that drives `dispatch()` from its own transport and never calls
- * `attach()`. The peer is declared optional; this is what makes that true
- * rather than aspirational.
- *
- * String enums are nominal in TypeScript, so the compiler cannot check these
- * against the real ones from a type-only import. `client/test/AvatarClient.test.ts`
- * does it instead, against the actual enum — the devDependency is present
- * exactly where the check belongs and absent from what we ship.
- */
-export const RTVI_EVENTS = {
-    serverMessage: "serverMessage",
-};
-/** Defensive unwrap for the `RTVIEvent.ServerMessage` `{ data }` quirk: some
- * transports deliver the payload directly and some wrap it once more. */
-function unwrapServerMessage(raw) {
-    const obj = (raw ?? {});
-    const inner = obj["data"];
-    return inner && "type" in inner ? inner : obj;
-}
-export class AvatarClient {
-    avatar;
-    opts;
-    now;
-    turn = null;
-    constructor(avatar, opts = {}) {
-        this.avatar = avatar;
-        this.opts = opts;
-        this.now = opts.now ?? (() => performance.now());
-    }
-    /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
-    get turnCtx() {
-        return this.turn?.ctx ?? null;
-    }
-    /** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
-    get turnCues() {
-        return this.turn ? [...this.turn.cues] : [];
-    }
-    /** Dispatch one server message. Anything that isn't in the avatar envelope
-     * is not ours and is ignored; so is an envelope carrying a `cmd` this build
-     * has never heard of, per the wire protocol's forward-compat rule. */
-    dispatch(raw) {
-        if (!isAvatarMessage(raw))
-            return;
-        const msg = raw;
-        try {
-            switch (msg.cmd) {
-                case "state":
-                    this.handleState(msg);
-                    break;
-                case "interject":
-                    this.avatar.interject(msg.id);
-                    break;
-                case "gesture":
-                    this.avatar.gesture(msg.id);
-                    break;
-                case "cues":
-                    this.handleCues(msg);
-                    break;
-                case "speech":
-                    this.handleSpeech(msg);
-                    break;
-                case "user":
-                    this.avatar.setUserSpeaking(msg.speaking);
-                    break;
-                // No default: an unknown `cmd` is a newer server talking to an older
-                // widget, and the protocol's forward-compat rule says ignore it. There
-                // is no callback for it — a hook nobody could act on is observability,
-                // not an interface (`docs/removed.md` § Client callbacks).
-            }
-        }
-        catch (err) {
-            if (this.opts.onError)
-                this.opts.onError(err, msg);
-            else
-                console.warn("[avatar] dispatch failed", msg, err);
-        }
-    }
-    handleState(msg) {
-        // Deliberately no client-side dedup: pass every `state` command straight
-        // through. The widget's own setState already no-ops the parts that matter
-        // for an unchanged name (`changed` gates the blink and the 'state' event in
-        // avatar.js), and a server resending the same state name as a
-        // keepalive/resync must still land so an `emotion`/`gaze` override on this
-        // particular message takes effect.
-        this.avatar.setState(msg.name, { emotion: msg.emotion, gaze: msg.gaze });
-    }
-    ensureTurn(ctx) {
-        if (!this.turn || this.turn.ctx !== ctx) {
-            // A different ctx supersedes whatever turn we had — a stale trailing
-            // message for the old ctx will find `this.turn.ctx !== ctx` in
-            // handleSpeech's stop-guard and be ignored, rather than cutting off the
-            // new turn.
-            this.turn = { ctx, cues: [], started: false, clock: null };
-        }
-        return this.turn;
-    }
-    handleCues(msg) {
-        const turn = this.ensureTurn(msg.ctx);
-        const kept = turn.cues.filter((c) => c.t < msg.from_ms);
-        const discarded = turn.cues.length - kept.length;
-        turn.cues = [...kept, ...msg.cues].sort((a, b) => a.t - b.t);
-        if (!turn.started) {
-            // No clock yet — buffer. `speech start` will hand this over as the turn's
-            // first speak() call.
-            return;
-        }
-        if (discarded === 0) {
-            this.avatar.pushCues(msg.cues);
-        }
-        else {
-            this.avatar.speak({ cues: turn.cues, clock: turn.clock });
-        }
-    }
-    handleSpeech(msg) {
-        if (msg.event === "start") {
-            const turn = this.ensureTurn(msg.ctx);
-            const t0 = this.now();
-            const clock = () => this.now() - t0;
-            turn.clock = clock;
-            turn.started = true;
-            this.avatar.speak({ cues: turn.cues, clock });
-            return;
-        }
-        // "stop": only act if it names the turn we're actually riding. A stale stop
-        // for an already-superseded ctx must not cut off a newer turn.
-        if (this.turn && this.turn.ctx === msg.ctx) {
-            this.avatar.stopSpeaking();
-            this.turn = null;
-        }
-    }
-    /**
-     * Subscribe to a live `PipecatClient`'s server messages and dispatch the
-     * avatar commands among them — the ones in the protocol's own
-     * `{type:"avatar"}` envelope, which `isAvatarMessage` is the definition of.
-     * Never throws on a malformed or irrelevant message.
-     *
-     * @returns an unsubscribe function; call it on unmount or disconnect.
-     */
-    attach(client) {
-        const onServerMessage = (raw) => this.dispatch(unwrapServerMessage(raw));
-        const serverMessage = RTVI_EVENTS.serverMessage;
-        client.on(serverMessage, onServerMessage);
-        return () => client.off(serverMessage, onServerMessage);
-    }
-}
-//# sourceMappingURL=AvatarClient.js.map