@voqalize/avatar 0.3.0 → 0.4.0

package/client/AvatarClient.ts

@@ -7,17 +7,36 @@
  * Base Pipecat TTS gives each serialized TTS context an opaque `context_id`.
  * The server uses it only to group and splice cue chunks. `botStartedSpeaking`
  * has no context payload, so the browser FIFO-claims the next buffered context
- * at that Pipecat output-lifecycle event and anchors its clock there.
- * `botStoppedSpeaking` closes the active context. Neither event observes the
- * browser's audio device, and no avatar-specific speech marker exists.
+ * at that Pipecat output-lifecycle event. `botStoppedSpeaking` closes it. No
+ * avatar-specific speech marker exists.
+ *
+ * The event says *which* audio is starting, not *when* it is heard, and the
+ * turn clock is anchored to the sound itself (`playout.ts` has the
+ * measurements that forced it). `attach()` listens to the bot's audio track;
+ * at the event, the turn's zero is backdated to a sound that has already begun,
+ * or the mouth is held shut until one does. When the track cannot be heard —
+ * no `attach()`, no track, a suspended audio graph, or a sound that never
+ * arrives within `ONSET_WAIT_MS` — the event is the anchor, as it always was.
+ * Which one won is logged once per turn, at `info`, because a desync report is
+ * unanswerable without it.
+ *
+ * The clock then runs `VISUAL_LEAD_MS` ahead of the sound, since the mixer's
+ * smoothing makes every mouth shape late by about that much. It is pulled back
+ * by any output-device latency beyond what the display already matches (a
+ * Bluetooth headset). Within a turn, each resumption after a pause in the
+ * track is a second chance to hear where the sound really is — but a mouth
+ * opens before its sound, so a resumption heard within the track's usual
+ * anticipation of it agrees with the clock. Only a disagreement beyond that is
+ * slewed out, at no more than 10 % of clock rate, never jumped, because a
+ * mouth that skips is seen and one that runs briefly fast is not.
  *
  * `attach()` subscribes to the avatar server-message channel *and* Pipecat's
  * standard lifecycle events. Server messages carry only what Pipecat cannot:
  * correlated speech/cue timing and deliberate application instructions. The
  * lifecycle events project the factual presence states locally. The server
- * supplies only lower-priority `THINKING` / `WORKING` claims and deliberate,
+ * supplies only the three lower-priority states and deliberate,
  * self-completing actions. This keeps the face tied to Pipecat's output truth
- * even if a server claim is delayed or stale.
+ * even if a server state is delayed or stale.
  *
  * ## Cue splice
  *
@@ -52,8 +71,8 @@
  */
 
 import type { PipecatClient, RTVIEvent } from "@pipecat-ai/client-js";
-import type { AvatarActionId, AvatarApi } from "../src/avatar.js";
 import { BehaviorController } from "../src/behavior.js";
+import { createPlayoutProbe, type PlayoutProbe } from "./playout.js";
 import {
   isAvatarMessage,
   parseAvatarCommand,
@@ -70,6 +89,84 @@ interface Turn {
   /** Whether Pipecat output has anchored a clock and issued `speak()`. */
   started: boolean;
   clock: (() => number) | null;
+  /** Timeline zero on the `now()` clock; `null` while waiting to hear the
+   * audio begin, during which the clock reads before the first cue. */
+  t0: number | null;
+  /** Where a re-anchor has placed zero; `t0` slews toward it. */
+  target: number | null;
+  /** `now()` at the clock's last read, which bounds how far a slew may move. */
+  read: number;
+  /** How far the clock runs ahead of the sound, fixed for the turn. */
+  lead: number;
+  /** The last resumption (track ms) already listened for. */
+  checked: number;
+}
+
+/**
+ * Longest the mouth is held for audio `botStartedSpeaking` has announced but
+ * the track has not yet carried. The event has been measured leading the first
+ * sample by ~160 ms on a local stack; twice that is a stall, and a mouth that
+ * starts without its sound is better than one that waits indefinitely.
+ */
+const ONSET_WAIT_MS = 350;
+const ONSET_POLL_MS = 10;
+
+/**
+ * How far the cue clock runs ahead of the sound. The mixer eases every mouth
+ * channel toward its target with a 42 ms time constant
+ * (`MOUTH_RESPONSE_TAU_S`), which delays a shape by about that much, and a
+ * frame is drawn on average half a frame after its time. Picture ahead of sound
+ * is the side people forgive (ITU-R BT.1359: sound leading is noticed at about
+ * 45 ms, picture leading at about 125 ms), so a small surplus is the safe error.
+ */
+export const VISUAL_LEAD_MS = 50;
+/** Output latency the display already matches: a compositor and a screen are
+ * late by about as much as a wired speaker, so only the excess counts. */
+const DISPLAY_LATENCY_MS = 40;
+/** A reported latency beyond this is a broken report, not a device. */
+const MAX_OUTPUT_LATENCY_MS = 250;
+/** A pause in the track at least this long is one the audio can be heard to
+ * resume after — shorter ones are usually not digital silence on the wire. */
+const REANCHOR_GAP_MS = 250;
+/** When to look, after the resumption is due, and for how long. The probe
+ * sees 170 ms back, so one look catches an early sound as well as a late one. */
+const REANCHOR_LOOK_MS = 100;
+const REANCHOR_WINDOW_MS = 150;
+/**
+ * Where a resumption is heard, after the cue that marks it. The track opens
+ * the mouth before the sound leaves digital silence: by 30–41 ms at most of the
+ * demo corpus's silent resumptions (median 32, both voices) and by up to 105 at
+ * a few, and the browser hears the sound a few ms after it leaves the wire. A
+ * sound heard anywhere in that band is the clock being right. Taking
+ * it as the resumption itself set every later phrase of a turn that much late —
+ * the direction people notice — and a mouth left early by an underrun smaller
+ * than the band is the direction they forgive (see `VISUAL_LEAD_MS`).
+ */
+const RESUME_HEARD_MS = 30;
+const RESUME_HEARD_RANGE_MS = [0, 130] as const;
+/** The most a re-anchor may speed or slow the clock, as a fraction of real time. */
+const MAX_SLEW = 0.1;
+
+/**
+ * The private renderer-side vocabulary the lifecycle driver actually uses.
+ *
+ * This stays narrower than `AvatarApi` on purpose. The latter is the SVG
+ * mixer's driving API; requiring a second renderer to impersonate every one
+ * of its inspection and authoring methods would turn that implementation
+ * detail into a renderer contract. The public seam remains `createAvatar`.
+ */
+export interface AvatarDriver {
+  setState(name: string): unknown;
+  /**
+   * One motion by name — a core intent or one of this avatar's own. **A name
+   * it does not have is a no-op, not a throw**: the wire's action vocabulary is
+   * open, so a face that cannot do the thing is as expected as a newer server.
+   */
+  action(id: string): unknown;
+  speak(options: { cues: AvatarCue[]; clock: () => number }): unknown;
+  pushCues(cues: AvatarCue[]): unknown;
+  stopSpeaking(): unknown;
+  setUserSpeaking(speaking: boolean | null): unknown;
 }
 
 /**
@@ -89,6 +186,9 @@ export interface AvatarClientOptions {
   onError?: (err: unknown, msg: AvatarCommand) => void;
   /** Override for tests. Defaults to `performance.now`. */
   now?: () => number;
+  /** Override for tests. Defaults to listening to the bot's audio track, once
+   * `attach()` has found one. */
+  playoutProbe?: PlayoutProbe;
   /** Quiet time in listening before the client-owned idle loop begins. */
   idleDelayMs?: number;
   /** Timer seams keep lifecycle behavior deterministic in tests. */
@@ -130,9 +230,12 @@ export const RTVI_EVENTS = {
   userStoppedSpeaking: "userStoppedSpeaking",
   botStartedSpeaking: "botStartedSpeaking",
   botStoppedSpeaking: "botStoppedSpeaking",
-  // Mute is a Pipecat fact, not a claim: the server's mute strategy emits
-  // `UserMuteStarted/StoppedFrame`, the RTVI observer forwards them, and the
-  // browser client raises these. So "has muted you" costs no wire verb —
+  // Only to find the bot's audio track once the transport has it; which track
+  // is re-read from `tracks()`, the one authority for whose it is.
+  trackStarted: "trackStarted",
+  // Mute is a Pipecat fact, and there is no server state for it: the server's
+  // mute strategy emits `UserMuteStarted/StoppedFrame`, the RTVI observer
+  // forwards them, and the browser client raises these. So "has muted you" costs no wire verb —
   // reading the events the peer already sends is exactly the authority model.
   userMuteStarted: "userMuteStarted",
   userMuteStopped: "userMuteStopped",
@@ -140,10 +243,10 @@ export const RTVI_EVENTS = {
 
 /** The resolved, factual presence state a host may render around the avatar. */
 export type AvatarPresenceState =
-  | "IDLE" | "LISTENING" | "STRAINING" | "THINKING" | "WORKING"
+  | "IDLE" | "LISTENING" | "CANT_HEAR" | "THINKING" | "WORKING"
   | "MUTED" | "SPEAKING" | "DEGRADED" | "OFFLINE";
 type LifecycleState = AvatarPresenceState;
-type ServerClaim = "STRAINING" | "THINKING" | "WORKING" | null;
+type ServerState = "CANT_HEAR" | "THINKING" | "WORKING" | null;
 
 /** Defensive unwrap for the `RTVIEvent.ServerMessage` `{ data }` quirk: some
  * transports deliver the payload directly and some wrap it once more. */
@@ -154,7 +257,7 @@ function unwrapServerMessage(raw: unknown): Record<string, unknown> {
 }
 
 export class AvatarClient {
-  private readonly avatar: AvatarApi;
+  private readonly avatar: AvatarDriver;
   /** Maps factual/wire intent into the broader client behavior catalog. */
   private readonly behavior: BehaviorController;
   private readonly opts: AvatarClientOptions;
@@ -164,7 +267,7 @@ export class AvatarClient {
   private readonly pendingCtxs: string[] = [];
   private readonly closedCtxs = new Set<string>();
   private projected: LifecycleState | null = null;
-  private serverClaim: ServerClaim = null;
+  private serverState: ServerState = null;
   private userSpeaking = false;
   private botSpeaking = false;
   private muted = false;
@@ -179,8 +282,15 @@ export class AvatarClient {
   private readonly idleDelayMs: number;
   private readonly setTimer: typeof setTimeout;
   private readonly clearTimer: typeof clearTimeout;
-
-  constructor(avatar: AvatarApi, opts: AvatarClientOptions = {}) {
+  private probe: PlayoutProbe | null;
+  private probeTrack: MediaStreamTrack | null = null;
+  /** Whether the probe has ever heard an onset. One that has not, and times
+   * out, is deaf to this track rather than early to it. */
+  private probeHeard = false;
+  private onsetTimer: ReturnType<typeof setTimeout> | null = null;
+  private reanchorTimer: ReturnType<typeof setTimeout> | null = null;
+
+  constructor(avatar: AvatarDriver, opts: AvatarClientOptions = {}) {
     this.avatar = avatar;
     this.behavior = new BehaviorController(avatar);
     this.opts = opts;
@@ -188,6 +298,7 @@ export class AvatarClient {
     this.idleDelayMs = opts.idleDelayMs ?? 12_000;
     this.setTimer = opts.setTimeout ?? globalThis.setTimeout.bind(globalThis);
     this.clearTimer = opts.clearTimeout ?? globalThis.clearTimeout.bind(globalThis);
+    this.probe = opts.playoutProbe ?? null;
   }
 
   /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
@@ -223,8 +334,8 @@ export class AvatarClient {
     if (!msg) return;
     try {
       switch (msg.cmd) {
-        case "claim":
-          this.handleClaim(msg.state);
+        case "state":
+          this.handleState(msg.state);
           break;
         case "action":
           this.handleAction(msg.id);
@@ -243,14 +354,14 @@ export class AvatarClient {
     }
   }
 
-  private handleClaim(state: ServerClaim): void {
-    this.serverClaim = state;
+  private handleState(state: ServerState): void {
+    this.serverState = state;
     if (state) this.idle = false;
     this.applyProjection();
     this.armIdleIfEligible();
   }
 
-  private handleAction(id: AvatarActionId): void {
+  private handleAction(id: string): void {
     // An interruption is a server-confirmed explanation of a transition, not
     // authority to steal the mouth while bot audio is still playing. Hold it
     // until Pipecat output has released the speaking state.
@@ -264,14 +375,17 @@ export class AvatarClient {
     this.playAction(id);
   }
 
-  private playAction(id: AvatarActionId): void {
+  private playAction(id: string): void {
     this.behavior.wireAction(id);
   }
 
   private ensureTurn(ctx: string): Turn {
     const existing = this.turns.get(ctx);
     if (existing) return existing;
-    const turn = { ctx, cues: [], started: false, clock: null };
+    const turn: Turn = {
+      ctx, cues: [], started: false, clock: null, t0: null,
+      target: null, read: 0, lead: 0, checked: -Infinity,
+    };
     this.turns.set(ctx, turn);
     this.pendingCtxs.push(ctx);
     return turn;
@@ -294,6 +408,8 @@ export class AvatarClient {
     } else {
       this.avatar.speak({ cues: turn.cues, clock: turn.clock! });
     }
+    // A resumption this chunk added may be the next one worth listening for.
+    this.scheduleReanchor(turn);
   }
 
   private activateNextTurn(): void {
@@ -303,11 +419,163 @@ export class AvatarClient {
       turn = this.turns.get(this.pendingCtxs.shift()!);
     }
     if (!turn) return;
-    const t0 = this.now();
-    turn.clock = () => this.now() - t0;
-    turn.started = true;
-    this.turn = turn;
-    this.avatar.speak({ cues: turn.cues, clock: turn.clock });
+    const active = turn;
+    active.lead = this.leadMs();
+    active.clock = () => {
+      if (active.t0 === null) return -1;
+      const now = this.now();
+      if (active.target !== null) {
+        const room = MAX_SLEW * Math.max(0, now - active.read);
+        const off = active.target - active.t0;
+        if (Math.abs(off) <= room) {
+          active.t0 = active.target;
+          active.target = null;
+        } else {
+          active.t0 += Math.sign(off) * room;
+        }
+      }
+      active.read = now;
+      return now - active.t0 + active.lead;
+    };
+    active.started = true;
+    this.turn = active;
+    this.anchor(active);
+    this.avatar.speak({ cues: active.cues, clock: active.clock });
+  }
+
+  /** Put the turn's zero where its sound began — see the header. */
+  private anchor(turn: Turn): void {
+    const event = this.now();
+    // A context made before any gesture can sit suspended; every turn asks again.
+    this.probe?.resume?.();
+    const heard = this.probe?.onset();
+    if (typeof heard === "number") {
+      this.probeHeard = true;
+      this.anchored(turn, heard, "the sound", event);
+      return;
+    }
+    if (heard !== null) {
+      this.anchored(turn, event, this.probe ? "the event (probe cannot say)" : "the event (no probe)", event);
+      return;
+    }
+    const poll = () => {
+      this.onsetTimer = null;
+      if (this.turn !== turn) return;
+      const onset = this.probe?.onset();
+      if (typeof onset === "number") {
+        this.probeHeard = true;
+        this.anchored(turn, onset, "the sound", event);
+      } else if (onset === null && this.now() - event < ONSET_WAIT_MS) {
+        this.onsetTimer = this.setTimer(poll, ONSET_POLL_MS);
+      } else {
+        if (!this.probeHeard) this.dropProbe();
+        this.anchored(turn, event, onset === null ? "the event (no sound in time)" : "the event (probe cannot say)", event);
+      }
+    };
+    this.onsetTimer = this.setTimer(poll, ONSET_POLL_MS);
+  }
+
+  private anchored(turn: Turn, t0: number, how: string, event: number): void {
+    turn.t0 = t0;
+    turn.read = this.now();
+    console.info(
+      `[avatar] ${turn.ctx} anchored on ${how}: sound ${Math.round(event - t0)} ms before the event, `
+        + `mouth ${Math.round(turn.lead)} ms ahead`,
+    );
+    this.scheduleReanchor(turn);
+  }
+
+  /** The lead for a turn starting now — see `VISUAL_LEAD_MS`. */
+  private leadMs(): number {
+    const out = this.probe?.outputLatencyMs?.() ?? 0;
+    const excess = Math.min(MAX_OUTPUT_LATENCY_MS, Math.max(0, out - DISPLAY_LATENCY_MS));
+    return VISUAL_LEAD_MS - excess;
+  }
+
+  /** The first resumption after `after` (track ms) that follows a pause long
+   * enough to be heard as one. */
+  private nextResumption(turn: Turn, after: number): number | null {
+    let silentFrom: number | null = null;
+    for (const c of turn.cues) {
+      if (c.v === "X") {
+        silentFrom ??= c.t;
+        continue;
+      }
+      if (silentFrom !== null && c.t - silentFrom >= REANCHOR_GAP_MS && c.t > after) return c.t;
+      silentFrom = null;
+    }
+    return null;
+  }
+
+  /** Listen for the sound at the turn's next resumption, once. */
+  private scheduleReanchor(turn: Turn): void {
+    if (this.reanchorTimer !== null || !this.probe || turn.t0 === null || this.turn !== turn) return;
+    const zero = turn.target ?? turn.t0;
+    const at = this.nextResumption(turn, Math.max(turn.checked, this.now() - zero - REANCHOR_WINDOW_MS));
+    if (at === null) return;
+    const poll = () => {
+      this.reanchorTimer = null;
+      if (this.turn !== turn || !this.probe || turn.t0 === null) return;
+      const due = (turn.target ?? turn.t0) + at;
+      const onset = this.probe.onset();
+      if (onset === null && this.now() < due + RESUME_HEARD_MS + REANCHOR_WINDOW_MS) {
+        this.reanchorTimer = this.setTimer(poll, ONSET_POLL_MS);
+        return;
+      }
+      turn.checked = at;
+      // `undefined` is sound all the way back: the pause was not silence on
+      // the wire, or it resumed long before the track says. Either way there
+      // is nothing to measure.
+      if (typeof onset === "number") {
+        const heard = onset - due;
+        const [earliest, latest] = RESUME_HEARD_RANGE_MS;
+        const agrees = heard >= earliest && heard <= latest;
+        if (!agrees && Math.abs(heard - RESUME_HEARD_MS) <= REANCHOR_WINDOW_MS) {
+          turn.target = onset - at - RESUME_HEARD_MS;
+        }
+      }
+      this.scheduleReanchor(turn);
+    };
+    const zeroNow = turn.target ?? turn.t0;
+    this.reanchorTimer = this.setTimer(poll, Math.max(0, zeroNow + at + REANCHOR_LOOK_MS - this.now()));
+  }
+
+  private clearReanchorTimer(): void {
+    if (this.reanchorTimer !== null) {
+      this.clearTimer(this.reanchorTimer);
+      this.reanchorTimer = null;
+    }
+  }
+
+  private clearOnsetTimer(): void {
+    if (this.onsetTimer !== null) {
+      this.clearTimer(this.onsetTimer);
+      this.onsetTimer = null;
+    }
+  }
+
+  private dropProbe(): void {
+    this.probe?.dispose();
+    this.probe = null;
+    this.probeTrack = null;
+    this.probeHeard = false;
+  }
+
+  /** Listen to the bot's audio track, if the transport has one yet. */
+  private listenTo(client: PipecatClient, heard?: MediaStreamTrack): void {
+    if (this.opts.playoutProbe) return;
+    let track: MediaStreamTrack | undefined = heard;
+    if (!track) {
+      try {
+        track = client.tracks().bot?.audio;
+      } catch {
+        return;
+      }
+    }
+    if (!track || track === this.probeTrack) return;
+    this.dropProbe();
+    this.probe = createPlayoutProbe(track, this.now);
+    this.probeTrack = this.probe ? track : null;
   }
 
   private discardQueuedTurns(): void {
@@ -321,7 +589,7 @@ export class AvatarClient {
     // The one copy of the ladder is docs/pipecat-lifecycle-protocol.md; this is
     // its implementation, in the same order.
     //
-    // Audio truth is the P0 invariant: no lower claim or microphone event may
+    // Audio truth is the P0 invariant: no lower state or microphone event may
     // put the face in a non-speaking pose while bot speech is audible.
     if (this.botSpeaking) return "SPEAKING";
     if (this.userSpeaking) return "LISTENING";
@@ -329,13 +597,13 @@ export class AvatarClient {
     // not the thing worth saying about a dead call.
     if (this.failure) return this.failure;
     if (this.muted) return "MUTED";
-    // Every claim resolves to the presence state of the same name, so there is
-    // nothing to rank here — a claim is a single value and only one can be in
-    // flight. STRAINING > THINKING > WORKING is decided where more than one
-    // condition can hold at once, which is the server: `AvatarStateMachine.
-    // _resolve()`. Ranking them again on this side would be a second, silent
-    // copy of that ladder, and the two would drift.
-    if (this.serverClaim) return this.serverClaim;
+    // A server state *is* the presence state of the same name, so there is
+    // nothing to rank here — only one can be in flight, and the command sets
+    // rather than adds. CANT_HEAR > THINKING > WORKING is decided where more
+    // than one condition can hold at once, which is the server:
+    // `AvatarStateMachine._resolve()`. Ranking them again on this side would be
+    // a second, silent copy of that ladder, and the two would drift.
+    if (this.serverState) return this.serverState;
     if (this.idle) return "IDLE";
     return "LISTENING";
   }
@@ -373,7 +641,7 @@ export class AvatarClient {
 
   private eligibleForIdle(): boolean {
     return this.listening && !this.userSpeaking && !this.botSpeaking
-      && !this.muted && !this.serverClaim && !this.failure;
+      && !this.muted && !this.serverState && !this.failure;
   }
 
   private enterListening(): void {
@@ -383,10 +651,10 @@ export class AvatarClient {
     this.armIdleIfEligible();
   }
 
-  private clearClaimForTurnBoundary(): void {
-    // Claims are lower-priority hints. A fresh user turn or bot output means
-    // any prior thinking/work claim is no longer allowed to reappear later.
-    this.serverClaim = null;
+  private clearServerStateForTurnBoundary(): void {
+    // A server state is a lower-priority hint. A fresh user turn or bot output
+    // means any prior thinking/working one may not reappear later.
+    this.serverState = null;
   }
 
   private maybePlayInterrupted(): void {
@@ -406,7 +674,7 @@ export class AvatarClient {
   private onUserStartedSpeaking = (): void => {
     this.clearRecoverableFailure();
     this.userSpeaking = true;
-    this.clearClaimForTurnBoundary();
+    this.clearServerStateForTurnBoundary();
     this.listening = true;
     this.idle = false;
     this.clearIdleTimer();
@@ -423,7 +691,7 @@ export class AvatarClient {
   private onBotStartedSpeaking = (): void => {
     this.clearRecoverableFailure();
     this.botSpeaking = true;
-    this.clearClaimForTurnBoundary();
+    this.clearServerStateForTurnBoundary();
     this.idle = false;
     this.clearIdleTimer();
     this.applyProjection();
@@ -432,6 +700,8 @@ export class AvatarClient {
 
   private onBotStoppedSpeaking = (): void => {
     this.botSpeaking = false;
+    this.clearOnsetTimer();
+    this.clearReanchorTimer();
     // Playout truth releases the only active mouth track. A late cue chunk for
     // this context is ignored rather than reviving a silent mouth.
     if (this.turn) {
@@ -495,6 +765,14 @@ export class AvatarClient {
    */
   attach(client: PipecatClient): () => void {
     const onServerMessage = (raw: unknown) => this.dispatch(unwrapServerMessage(raw));
+    // The bot's track has to come from the event. SmallWebRTC never lists a
+    // remote track in `tracks()`; it only hands it to this callback, with no
+    // participant. Asking `tracks()` alone meant no probe on that transport,
+    // so every turn fell back to the event.
+    const onTrackStarted = (track?: MediaStreamTrack, participant?: { local?: boolean }) => {
+      if (participant?.local || (track && track.kind !== "audio")) return;
+      this.listenTo(client, track);
+    };
     const subscriptions: Array<[string, (...args: any[]) => void]> = [
       [RTVI_EVENTS.serverMessage, onServerMessage],
       [RTVI_EVENTS.connected, this.onConnectedOrReady],
@@ -507,10 +785,24 @@ export class AvatarClient {
       [RTVI_EVENTS.botStoppedSpeaking, this.onBotStoppedSpeaking],
       [RTVI_EVENTS.userMuteStarted, this.onUserMuteStarted],
       [RTVI_EVENTS.userMuteStopped, this.onUserMuteStopped],
+      [RTVI_EVENTS.trackStarted, onTrackStarted],
     ];
     for (const [event, listener] of subscriptions) client.on(event as RTVIEvent, listener as never);
+    this.listenTo(client);
+    // Some browsers keep an audio context suspended until it is resumed inside
+    // a gesture, and the connect click usually lands before the bot's track
+    // exists. Any later click or key is the next chance.
+    const wake = () => this.probe?.resume?.();
+    const doc = typeof document === "undefined" ? null : document;
+    doc?.addEventListener("pointerdown", wake, true);
+    doc?.addEventListener("keydown", wake, true);
     return () => {
+      doc?.removeEventListener("pointerdown", wake, true);
+      doc?.removeEventListener("keydown", wake, true);
       this.clearIdleTimer();
+      this.clearOnsetTimer();
+      this.clearReanchorTimer();
+      if (!this.opts.playoutProbe) this.dropProbe();
       for (const [event, listener] of subscriptions) client.off(event as RTVIEvent, listener as never);
     };
   }
@@ -518,6 +810,9 @@ export class AvatarClient {
   /** Dispose controller-owned timers when its mounted avatar is destroyed. */
   destroy(): void {
     this.clearIdleTimer();
+    this.clearOnsetTimer();
+    this.clearReanchorTimer();
+    this.dropProbe();
     this.behavior.destroy();
   }
 }

package/client/arjun.ts

@@ -13,6 +13,8 @@ const IMAGES = {
 const FACE = () => import('../src/canvas/avatars/round/face.mjs');
 
 export type { CanvasAvatarOptions };
+// Same mixer, same clip library, so the same declaration (`supports.ts`).
+export { supports } from './supports.js';
 
 export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
   return createCanvasAvatar(options, {

package/client/createAvatar.ts

@@ -41,6 +41,7 @@
 import type { PipecatClient } from "@pipecat-ai/client-js";
 import { createAvatar as createSvgWidget } from "../src/avatar.js";
 import type { Face, FaceTheme, Gain, HandSide } from "../src/avatar.js";
+import type { BehaviorStateId } from "../src/behavior.js";
 import { peep } from "../src/face-peep.js";
 import { AvatarClient } from "./AvatarClient.js";
 
@@ -75,6 +76,43 @@ export interface AvatarInstance {
 export type AvatarFactory<O extends AvatarOptions = AvatarOptions> =
   (options: O) => AvatarInstance;
 
+/**
+ * What a mounted avatar answers to — the one *optional* export beside
+ * `createAvatar`.
+ *
+ * A page that drives an avatar has a problem an ordinary consumer does not.
+ * The wire's action id is open and an unknown one is ignored in silence
+ * (docs/contract-wire.md § Action), which is right for a protocol and useless
+ * for a control surface: a button per name cannot tell "this face has no such
+ * motion" from "nothing happened". This is the avatar answering that question
+ * for a driving UI, and nothing else reads it — `createAvatar` does not take
+ * it, the library never consults it, and an avatar that omits it is fully
+ * conforming. It is a *declaration*: nobody verifies it, so a list that has
+ * drifted from the drawing is a bug in the avatar and not something the
+ * library can catch.
+ */
+export interface AvatarSupport {
+  /**
+   * Every action id this avatar answers to, the two required ones included.
+   * An id absent from here is a documented no-op, never an error.
+   */
+  readonly actions: readonly string[];
+  /**
+   * Which of the nine states this avatar draws *distinguishably*.
+   *
+   * Every avatar must accept all nine; a renderer that draws `CANT_HEAR` as
+   * ordinary listening is conforming (docs/contract-behavior.md). So this is
+   * the narrower claim — which of them a reviewer can expect to tell apart on
+   * the face — and omitting it claims all nine, which is what every avatar in
+   * this repo does.
+   */
+  readonly states?: readonly BehaviorStateId[];
+}
+
+/** The bundled avatars' own `supports` value is in `supports.ts`, not here: it
+ *  is a value, so this module's `peep` import would follow it into the bundle of
+ *  anything that read it — including the canvas avatars, which draw no SVG face. */
+
 /**
  * Options for the bundled SVG avatars — ours alone; nothing outside this
  * package reads them.

package/client/index.ts

@@ -23,14 +23,19 @@
  *   because they are an interface.
  *
  * To ship your own avatar, publish a module exporting `createAvatar` and import
- * that instead — see `createAvatar.ts` and docs/design-avatar-interface.md.
+ * that instead — see `createAvatar.ts` and docs/design-avatar-interface.md. A
+ * module may also export `supports`, which is how a page that *drives* an
+ * avatar learns which action ids this one answers to; every consumer that only
+ * mounts a face can ignore it.
  */
 
 export { createAvatar } from "./createAvatar.js";
+export { supports } from "./supports.js";
 export type {
   AvatarOptions,
   AvatarInstance,
   AvatarFactory,
+  AvatarSupport,
   SvgAvatarOptions,
   Face,
   FaceTheme,