@voqalize/avatar 0.2.2 → 0.4.0

package/dist/AvatarClient.js

@@ -0,0 +1,712 @@
+/**
+ * AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
+ * splice, framework-free (no React; the hook and component wrap this).
+ *
+ * ## Turn clock anchoring
+ *
+ * Base Pipecat TTS gives each serialized TTS context an opaque `context_id`.
+ * The server uses it only to group and splice cue chunks. `botStartedSpeaking`
+ * has no context payload, so the browser FIFO-claims the next buffered context
+ * at that Pipecat output-lifecycle event. `botStoppedSpeaking` closes it. No
+ * avatar-specific speech marker exists.
+ *
+ * The event says *which* audio is starting, not *when* it is heard, and the
+ * turn clock is anchored to the sound itself (`playout.ts` has the
+ * measurements that forced it). `attach()` listens to the bot's audio track;
+ * at the event, the turn's zero is backdated to a sound that has already begun,
+ * or the mouth is held shut until one does. When the track cannot be heard —
+ * no `attach()`, no track, a suspended audio graph, or a sound that never
+ * arrives within `ONSET_WAIT_MS` — the event is the anchor, as it always was.
+ * Which one won is logged once per turn, at `info`, because a desync report is
+ * unanswerable without it.
+ *
+ * The clock then runs `VISUAL_LEAD_MS` ahead of the sound, since the mixer's
+ * smoothing makes every mouth shape late by about that much. It is pulled back
+ * by any output-device latency beyond what the display already matches (a
+ * Bluetooth headset). Within a turn, each resumption after a pause in the
+ * track is a second chance to hear where the sound really is — but a mouth
+ * opens before its sound, so a resumption heard within the track's usual
+ * anticipation of it agrees with the clock. Only a disagreement beyond that is
+ * slewed out, at no more than 10 % of clock rate, never jumped, because a
+ * mouth that skips is seen and one that runs briefly fast is not.
+ *
+ * `attach()` subscribes to the avatar server-message channel *and* Pipecat's
+ * standard lifecycle events. Server messages carry only what Pipecat cannot:
+ * correlated speech/cue timing and deliberate application instructions. The
+ * lifecycle events project the factual presence states locally. The server
+ * supplies only the three lower-priority states and deliberate,
+ * self-completing actions. This keeps the face tied to Pipecat's output truth
+ * even if a server state is delayed or stale.
+ *
+ * ## Cue splice
+ *
+ * The widget has two cue-track primitives: `speak({cues, clock})` (a full
+ * replace) and `pushCues(cues)` (a pure union that can only grow the track,
+ * never shrink it). Neither is "discard queued cues at or after `from_ms`,
+ * then append" on its own — `pushCues` has no way to drop a stale tail. So
+ * this client keeps the turn's canonical cue array itself (kept portion +
+ * every appended chunk, spliced on each `cues` message) and picks the cheapest
+ * widget call that stays correct:
+ *
+ *   - if the splice's `from_ms` doesn't reach back into anything already
+ *     queued — the common case past a turn's first sentence, since only the
+ *     first sentence genuinely plays fast-leg cues — nothing needs discarding:
+ *     `pushCues(newCues)` is the cheap, correct append.
+ *   - if it does reach back (a real fast→accurate splice), `pushCues` cannot
+ *     express the discard; we call `speak()` again with the full spliced
+ *     canonical array on the turn's original clock. `speak()` is otherwise
+ *     documented as also killing an in-flight spoken interjection and
+ *     re-entering `SPEAKING` — both harmless mid-splice (an interjection
+ *     should not be running while a server track owns the mouth; re-entering
+ *     an unchanged state is a no-op past the profile/gaze reset the widget
+ *     already does for a same-name `setState`).
+ *
+ * Cues commonly arrive **before** `botStartedSpeaking` — the fast leg starts
+ * the moment a sentence is handed to TTS, well before bot output begins.
+ * Chunks that arrive before the clock is anchored are spliced into the
+ * canonical array but not yet handed to the widget; `botStartedSpeaking` hands over
+ * whatever has accumulated as the turn's first `speak()` call. So "the first
+ * chunk of a turn starts speak()" means the first *widget* call, not
+ * necessarily the first *message*.
+ */
+import { BehaviorController } from "../src/behavior.js";
+import { createPlayoutProbe } from "./playout.js";
+import { isAvatarMessage, parseAvatarCommand, } from "./types.js";
+/**
+ * Longest the mouth is held for audio `botStartedSpeaking` has announced but
+ * the track has not yet carried. The event has been measured leading the first
+ * sample by ~160 ms on a local stack; twice that is a stall, and a mouth that
+ * starts without its sound is better than one that waits indefinitely.
+ */
+const ONSET_WAIT_MS = 350;
+const ONSET_POLL_MS = 10;
+/**
+ * How far the cue clock runs ahead of the sound. The mixer eases every mouth
+ * channel toward its target with a 42 ms time constant
+ * (`MOUTH_RESPONSE_TAU_S`), which delays a shape by about that much, and a
+ * frame is drawn on average half a frame after its time. Picture ahead of sound
+ * is the side people forgive (ITU-R BT.1359: sound leading is noticed at about
+ * 45 ms, picture leading at about 125 ms), so a small surplus is the safe error.
+ */
+export const VISUAL_LEAD_MS = 50;
+/** Output latency the display already matches: a compositor and a screen are
+ * late by about as much as a wired speaker, so only the excess counts. */
+const DISPLAY_LATENCY_MS = 40;
+/** A reported latency beyond this is a broken report, not a device. */
+const MAX_OUTPUT_LATENCY_MS = 250;
+/** A pause in the track at least this long is one the audio can be heard to
+ * resume after — shorter ones are usually not digital silence on the wire. */
+const REANCHOR_GAP_MS = 250;
+/** When to look, after the resumption is due, and for how long. The probe
+ * sees 170 ms back, so one look catches an early sound as well as a late one. */
+const REANCHOR_LOOK_MS = 100;
+const REANCHOR_WINDOW_MS = 150;
+/**
+ * Where a resumption is heard, after the cue that marks it. The track opens
+ * the mouth before the sound leaves digital silence: by 30–41 ms at most of the
+ * demo corpus's silent resumptions (median 32, both voices) and by up to 105 at
+ * a few, and the browser hears the sound a few ms after it leaves the wire. A
+ * sound heard anywhere in that band is the clock being right. Taking
+ * it as the resumption itself set every later phrase of a turn that much late —
+ * the direction people notice — and a mouth left early by an underrun smaller
+ * than the band is the direction they forgive (see `VISUAL_LEAD_MS`).
+ */
+const RESUME_HEARD_MS = 30;
+const RESUME_HEARD_RANGE_MS = [0, 130];
+/** The most a re-anchor may speed or slow the clock, as a fraction of real time. */
+const MAX_SLEW = 0.1;
+/**
+ * The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
+ *
+ * Written out rather than imported because that enum was this module's *only*
+ * runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
+ * the whole `/pipecat` subpath fail to load without the peer installed — even
+ * for a host that drives `dispatch()` from its own transport and never calls
+ * `attach()`. The peer is declared optional; this is what makes that true
+ * rather than aspirational.
+ *
+ * String enums are nominal in TypeScript, so the compiler cannot check these
+ * against the real ones from a type-only import. `packages/avatar/test/AvatarClient.test.ts`
+ * does it instead, against the actual enum — the devDependency is present
+ * exactly where the check belongs and absent from what we ship.
+ */
+export const RTVI_EVENTS = {
+    serverMessage: "serverMessage",
+    connected: "connected",
+    disconnected: "disconnected",
+    botReady: "botReady",
+    error: "error",
+    userStartedSpeaking: "userStartedSpeaking",
+    userStoppedSpeaking: "userStoppedSpeaking",
+    botStartedSpeaking: "botStartedSpeaking",
+    botStoppedSpeaking: "botStoppedSpeaking",
+    // Only to find the bot's audio track once the transport has it; which track
+    // is re-read from `tracks()`, the one authority for whose it is.
+    trackStarted: "trackStarted",
+    // Mute is a Pipecat fact, and there is no server state for it: the server's
+    // mute strategy emits `UserMuteStarted/StoppedFrame`, the RTVI observer
+    // forwards them, and the browser client raises these. So "has muted you" costs no wire verb —
+    // reading the events the peer already sends is exactly the authority model.
+    userMuteStarted: "userMuteStarted",
+    userMuteStopped: "userMuteStopped",
+};
+/** Defensive unwrap for the `RTVIEvent.ServerMessage` `{ data }` quirk: some
+ * transports deliver the payload directly and some wrap it once more. */
+function unwrapServerMessage(raw) {
+    const obj = (raw ?? {});
+    const inner = obj["data"];
+    return inner && "type" in inner ? inner : obj;
+}
+export class AvatarClient {
+    avatar;
+    /** Maps factual/wire intent into the broader client behavior catalog. */
+    behavior;
+    opts;
+    now;
+    turn = null;
+    turns = new Map();
+    pendingCtxs = [];
+    closedCtxs = new Set();
+    projected = null;
+    serverState = null;
+    userSpeaking = false;
+    botSpeaking = false;
+    muted = false;
+    listening = false;
+    // Idle is earned after a connected, quiet listening interval. A newly
+    // mounted avatar is available, not already "stepped aside".
+    idle = false;
+    failure = null;
+    idleTimer = null;
+    pendingInterruptedAction = false;
+    discardQueuedContextsOnBotStop = false;
+    idleDelayMs;
+    setTimer;
+    clearTimer;
+    probe;
+    probeTrack = null;
+    /** Whether the probe has ever heard an onset. One that has not, and times
+     * out, is deaf to this track rather than early to it. */
+    probeHeard = false;
+    onsetTimer = null;
+    reanchorTimer = null;
+    constructor(avatar, opts = {}) {
+        this.avatar = avatar;
+        this.behavior = new BehaviorController(avatar);
+        this.opts = opts;
+        this.now = opts.now ?? (() => performance.now());
+        this.idleDelayMs = opts.idleDelayMs ?? 12_000;
+        this.setTimer = opts.setTimeout ?? globalThis.setTimeout.bind(globalThis);
+        this.clearTimer = opts.clearTimeout ?? globalThis.clearTimeout.bind(globalThis);
+        this.probe = opts.playoutProbe ?? null;
+    }
+    /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
+    get turnCtx() {
+        return this.turn?.ctx ?? null;
+    }
+    /** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
+    get turnCues() {
+        return this.turn ? [...this.turn.cues] : [];
+    }
+    /** Current resolved projection. Internal — for tests and Studio's inspector.
+     * Before the first lifecycle fact the renderer is simply at its ready rest
+     * pose, so `LISTENING` is the safe value; it is not a session status. */
+    get presenceState() {
+        return this.projected ?? "LISTENING";
+    }
+    /** Re-apply the currently resolved factual projection after an embedding
+     * tool has temporarily used raw renderer controls. This does not invent a
+     * lifecycle event or cancel a finite action. */
+    restoreProjection() {
+        this.applyProjection(true);
+    }
+    /** Dispatch one server message. Anything that isn't in the avatar envelope
+     * is not ours and is ignored; so is an envelope carrying a `cmd` this build
+     * has never heard of, per the wire protocol's forward-compat rule. */
+    dispatch(raw) {
+        if (!isAvatarMessage(raw))
+            return;
+        const msg = parseAvatarCommand(raw);
+        if (!msg)
+            return;
+        try {
+            switch (msg.cmd) {
+                case "state":
+                    this.handleState(msg.state);
+                    break;
+                case "action":
+                    this.handleAction(msg.id);
+                    break;
+                case "cues":
+                    this.handleCues(msg);
+                    break;
+                // No default: an unknown `cmd` is a newer server talking to an older
+                // widget, and the protocol's forward-compat rule says ignore it. There
+                // is no callback for it — a hook nobody could act on is observability,
+                // not an interface.
+            }
+        }
+        catch (err) {
+            if (this.opts.onError)
+                this.opts.onError(err, msg);
+            else
+                console.warn("[avatar] dispatch failed", msg, err);
+        }
+    }
+    handleState(state) {
+        this.serverState = state;
+        if (state)
+            this.idle = false;
+        this.applyProjection();
+        this.armIdleIfEligible();
+    }
+    handleAction(id) {
+        // An interruption is a server-confirmed explanation of a transition, not
+        // authority to steal the mouth while bot audio is still playing. Hold it
+        // until Pipecat output has released the speaking state.
+        if (id === "RESPONSE_INTERRUPTED" && this.botSpeaking) {
+            this.pendingInterruptedAction = true;
+            // Any prefetched TTS contexts behind interrupted output belong to
+            // audio Pipecat will now discard. Never let one animate a later reply.
+            this.discardQueuedContextsOnBotStop = true;
+            return;
+        }
+        this.playAction(id);
+    }
+    playAction(id) {
+        this.behavior.wireAction(id);
+    }
+    ensureTurn(ctx) {
+        const existing = this.turns.get(ctx);
+        if (existing)
+            return existing;
+        const turn = {
+            ctx, cues: [], started: false, clock: null, t0: null,
+            target: null, read: 0, lead: 0, checked: -Infinity,
+        };
+        this.turns.set(ctx, turn);
+        this.pendingCtxs.push(ctx);
+        return turn;
+    }
+    handleCues(msg) {
+        if (this.closedCtxs.has(msg.ctx))
+            return;
+        const turn = this.ensureTurn(msg.ctx);
+        const kept = turn.cues.filter((c) => c.t < msg.from_ms);
+        const discarded = turn.cues.length - kept.length;
+        turn.cues = [...kept, ...msg.cues].sort((a, b) => a.t - b.t);
+        if (!turn.started) {
+            // No clock yet — buffer. Pipecat output will claim this FIFO context.
+            if (this.botSpeaking && this.turn === null)
+                this.activateNextTurn();
+            return;
+        }
+        if (discarded === 0) {
+            this.avatar.pushCues(msg.cues);
+        }
+        else {
+            this.avatar.speak({ cues: turn.cues, clock: turn.clock });
+        }
+        // A resumption this chunk added may be the next one worth listening for.
+        this.scheduleReanchor(turn);
+    }
+    activateNextTurn() {
+        if (this.turn)
+            return;
+        let turn;
+        while (this.pendingCtxs.length && !turn) {
+            turn = this.turns.get(this.pendingCtxs.shift());
+        }
+        if (!turn)
+            return;
+        const active = turn;
+        active.lead = this.leadMs();
+        active.clock = () => {
+            if (active.t0 === null)
+                return -1;
+            const now = this.now();
+            if (active.target !== null) {
+                const room = MAX_SLEW * Math.max(0, now - active.read);
+                const off = active.target - active.t0;
+                if (Math.abs(off) <= room) {
+                    active.t0 = active.target;
+                    active.target = null;
+                }
+                else {
+                    active.t0 += Math.sign(off) * room;
+                }
+            }
+            active.read = now;
+            return now - active.t0 + active.lead;
+        };
+        active.started = true;
+        this.turn = active;
+        this.anchor(active);
+        this.avatar.speak({ cues: active.cues, clock: active.clock });
+    }
+    /** Put the turn's zero where its sound began — see the header. */
+    anchor(turn) {
+        const event = this.now();
+        // A context made before any gesture can sit suspended; every turn asks again.
+        this.probe?.resume?.();
+        const heard = this.probe?.onset();
+        if (typeof heard === "number") {
+            this.probeHeard = true;
+            this.anchored(turn, heard, "the sound", event);
+            return;
+        }
+        if (heard !== null) {
+            this.anchored(turn, event, this.probe ? "the event (probe cannot say)" : "the event (no probe)", event);
+            return;
+        }
+        const poll = () => {
+            this.onsetTimer = null;
+            if (this.turn !== turn)
+                return;
+            const onset = this.probe?.onset();
+            if (typeof onset === "number") {
+                this.probeHeard = true;
+                this.anchored(turn, onset, "the sound", event);
+            }
+            else if (onset === null && this.now() - event < ONSET_WAIT_MS) {
+                this.onsetTimer = this.setTimer(poll, ONSET_POLL_MS);
+            }
+            else {
+                if (!this.probeHeard)
+                    this.dropProbe();
+                this.anchored(turn, event, onset === null ? "the event (no sound in time)" : "the event (probe cannot say)", event);
+            }
+        };
+        this.onsetTimer = this.setTimer(poll, ONSET_POLL_MS);
+    }
+    anchored(turn, t0, how, event) {
+        turn.t0 = t0;
+        turn.read = this.now();
+        console.info(`[avatar] ${turn.ctx} anchored on ${how}: sound ${Math.round(event - t0)} ms before the event, `
+            + `mouth ${Math.round(turn.lead)} ms ahead`);
+        this.scheduleReanchor(turn);
+    }
+    /** The lead for a turn starting now — see `VISUAL_LEAD_MS`. */
+    leadMs() {
+        const out = this.probe?.outputLatencyMs?.() ?? 0;
+        const excess = Math.min(MAX_OUTPUT_LATENCY_MS, Math.max(0, out - DISPLAY_LATENCY_MS));
+        return VISUAL_LEAD_MS - excess;
+    }
+    /** The first resumption after `after` (track ms) that follows a pause long
+     * enough to be heard as one. */
+    nextResumption(turn, after) {
+        let silentFrom = null;
+        for (const c of turn.cues) {
+            if (c.v === "X") {
+                silentFrom ??= c.t;
+                continue;
+            }
+            if (silentFrom !== null && c.t - silentFrom >= REANCHOR_GAP_MS && c.t > after)
+                return c.t;
+            silentFrom = null;
+        }
+        return null;
+    }
+    /** Listen for the sound at the turn's next resumption, once. */
+    scheduleReanchor(turn) {
+        if (this.reanchorTimer !== null || !this.probe || turn.t0 === null || this.turn !== turn)
+            return;
+        const zero = turn.target ?? turn.t0;
+        const at = this.nextResumption(turn, Math.max(turn.checked, this.now() - zero - REANCHOR_WINDOW_MS));
+        if (at === null)
+            return;
+        const poll = () => {
+            this.reanchorTimer = null;
+            if (this.turn !== turn || !this.probe || turn.t0 === null)
+                return;
+            const due = (turn.target ?? turn.t0) + at;
+            const onset = this.probe.onset();
+            if (onset === null && this.now() < due + RESUME_HEARD_MS + REANCHOR_WINDOW_MS) {
+                this.reanchorTimer = this.setTimer(poll, ONSET_POLL_MS);
+                return;
+            }
+            turn.checked = at;
+            // `undefined` is sound all the way back: the pause was not silence on
+            // the wire, or it resumed long before the track says. Either way there
+            // is nothing to measure.
+            if (typeof onset === "number") {
+                const heard = onset - due;
+                const [earliest, latest] = RESUME_HEARD_RANGE_MS;
+                const agrees = heard >= earliest && heard <= latest;
+                if (!agrees && Math.abs(heard - RESUME_HEARD_MS) <= REANCHOR_WINDOW_MS) {
+                    turn.target = onset - at - RESUME_HEARD_MS;
+                }
+            }
+            this.scheduleReanchor(turn);
+        };
+        const zeroNow = turn.target ?? turn.t0;
+        this.reanchorTimer = this.setTimer(poll, Math.max(0, zeroNow + at + REANCHOR_LOOK_MS - this.now()));
+    }
+    clearReanchorTimer() {
+        if (this.reanchorTimer !== null) {
+            this.clearTimer(this.reanchorTimer);
+            this.reanchorTimer = null;
+        }
+    }
+    clearOnsetTimer() {
+        if (this.onsetTimer !== null) {
+            this.clearTimer(this.onsetTimer);
+            this.onsetTimer = null;
+        }
+    }
+    dropProbe() {
+        this.probe?.dispose();
+        this.probe = null;
+        this.probeTrack = null;
+        this.probeHeard = false;
+    }
+    /** Listen to the bot's audio track, if the transport has one yet. */
+    listenTo(client, heard) {
+        if (this.opts.playoutProbe)
+            return;
+        let track = heard;
+        if (!track) {
+            try {
+                track = client.tracks().bot?.audio;
+            }
+            catch {
+                return;
+            }
+        }
+        if (!track || track === this.probeTrack)
+            return;
+        this.dropProbe();
+        this.probe = createPlayoutProbe(track, this.now);
+        this.probeTrack = this.probe ? track : null;
+    }
+    discardQueuedTurns() {
+        for (const ctx of this.pendingCtxs.splice(0)) {
+            this.closedCtxs.add(ctx);
+            this.turns.delete(ctx);
+        }
+    }
+    lifecycleState() {
+        // The one copy of the ladder is docs/pipecat-lifecycle-protocol.md; this is
+        // its implementation, in the same order.
+        //
+        // Audio truth is the P0 invariant: no lower state or microphone event may
+        // put the face in a non-speaking pose while bot speech is audible.
+        if (this.botSpeaking)
+            return "SPEAKING";
+        if (this.userSpeaking)
+            return "LISTENING";
+        // A broken session outranks mute, because a mute nobody can hear about is
+        // not the thing worth saying about a dead call.
+        if (this.failure)
+            return this.failure;
+        if (this.muted)
+            return "MUTED";
+        // A server state *is* the presence state of the same name, so there is
+        // nothing to rank here — only one can be in flight, and the command sets
+        // rather than adds. CANT_HEAR > THINKING > WORKING is decided where more
+        // than one condition can hold at once, which is the server:
+        // `AvatarStateMachine._resolve()`. Ranking them again on this side would be
+        // a second, silent copy of that ladder, and the two would drift.
+        if (this.serverState)
+            return this.serverState;
+        if (this.idle)
+            return "IDLE";
+        return "LISTENING";
+    }
+    applyProjection(force = false) {
+        const state = this.lifecycleState();
+        if (force || this.projected !== state) {
+            this.behavior.setState(state);
+            this.projected = state;
+            this.opts.onPresenceChange?.(state);
+        }
+    }
+    clearIdleTimer() {
+        if (this.idleTimer !== null) {
+            this.clearTimer(this.idleTimer);
+            this.idleTimer = null;
+        }
+    }
+    armIdleIfEligible() {
+        this.clearIdleTimer();
+        // Quiet under a mute is not the quiet that earns IDLE — the silence was
+        // imposed, and letting the timer run behind it would reveal a stepped-aside
+        // face the moment the microphone came back.
+        if (!this.eligibleForIdle())
+            return;
+        this.idleTimer = this.setTimer(() => {
+            this.idleTimer = null;
+            if (this.eligibleForIdle()) {
+                this.idle = true;
+                this.applyProjection();
+            }
+        }, this.idleDelayMs);
+    }
+    eligibleForIdle() {
+        return this.listening && !this.userSpeaking && !this.botSpeaking
+            && !this.muted && !this.serverState && !this.failure;
+    }
+    enterListening() {
+        this.listening = true;
+        this.idle = false;
+        this.applyProjection();
+        this.armIdleIfEligible();
+    }
+    clearServerStateForTurnBoundary() {
+        // A server state is a lower-priority hint. A fresh user turn or bot output
+        // means any prior thinking/working one may not reappear later.
+        this.serverState = null;
+    }
+    maybePlayInterrupted() {
+        if (this.pendingInterruptedAction && !this.botSpeaking) {
+            this.pendingInterruptedAction = false;
+            this.playAction("RESPONSE_INTERRUPTED");
+        }
+    }
+    clearRecoverableFailure() {
+        if (this.failure === "DEGRADED") {
+            this.failure = null;
+            this.applyProjection();
+        }
+    }
+    onUserStartedSpeaking = () => {
+        this.clearRecoverableFailure();
+        this.userSpeaking = true;
+        this.clearServerStateForTurnBoundary();
+        this.listening = true;
+        this.idle = false;
+        this.clearIdleTimer();
+        this.avatar.setUserSpeaking(true);
+        this.applyProjection();
+    };
+    onUserStoppedSpeaking = () => {
+        this.userSpeaking = false;
+        this.avatar.setUserSpeaking(false);
+        this.enterListening();
+    };
+    onBotStartedSpeaking = () => {
+        this.clearRecoverableFailure();
+        this.botSpeaking = true;
+        this.clearServerStateForTurnBoundary();
+        this.idle = false;
+        this.clearIdleTimer();
+        this.applyProjection();
+        this.activateNextTurn();
+    };
+    onBotStoppedSpeaking = () => {
+        this.botSpeaking = false;
+        this.clearOnsetTimer();
+        this.clearReanchorTimer();
+        // Playout truth releases the only active mouth track. A late cue chunk for
+        // this context is ignored rather than reviving a silent mouth.
+        if (this.turn) {
+            this.avatar.stopSpeaking();
+            this.closedCtxs.add(this.turn.ctx);
+            this.turns.delete(this.turn.ctx);
+            this.turn = null;
+        }
+        if (this.discardQueuedContextsOnBotStop) {
+            this.discardQueuedContextsOnBotStop = false;
+            this.discardQueuedTurns();
+        }
+        this.enterListening();
+        this.maybePlayInterrupted();
+    };
+    onUserMuteStarted = () => {
+        this.muted = true;
+        this.idle = false;
+        this.clearIdleTimer();
+        this.applyProjection();
+    };
+    onUserMuteStopped = () => {
+        this.muted = false;
+        // Whatever the avatar was waiting on before the mute, it is waiting on the
+        // user again now — the same place a turn ends.
+        this.enterListening();
+    };
+    onError = (raw) => {
+        const data = raw?.data;
+        this.failure = data?.fatal === true ? "OFFLINE" : "DEGRADED";
+        this.applyProjection();
+    };
+    onDisconnected = () => {
+        this.failure = "OFFLINE";
+        this.clearIdleTimer();
+        this.applyProjection();
+    };
+    onConnectedOrReady = () => {
+        if (this.failure === "OFFLINE")
+            this.failure = null;
+        // Pipecat has established a session. Until a factual speech event says
+        // otherwise, the avatar is available to listen; it earns IDLE only after
+        // the regular quiet timer expires.
+        this.listening = true;
+        this.idle = false;
+        this.applyProjection();
+        this.armIdleIfEligible();
+    };
+    /**
+     * Subscribe to a live `PipecatClient`. Standard client events own the normal
+     * lifecycle projection; avatar server-messages carry correlated visemes and
+     * explicit application intent. Never throws on malformed or irrelevant
+     * server messages.
+     *
+     * @returns an unsubscribe function; call it on unmount or disconnect.
+     */
+    attach(client) {
+        const onServerMessage = (raw) => this.dispatch(unwrapServerMessage(raw));
+        // The bot's track has to come from the event. SmallWebRTC never lists a
+        // remote track in `tracks()`; it only hands it to this callback, with no
+        // participant. Asking `tracks()` alone meant no probe on that transport,
+        // so every turn fell back to the event.
+        const onTrackStarted = (track, participant) => {
+            if (participant?.local || (track && track.kind !== "audio"))
+                return;
+            this.listenTo(client, track);
+        };
+        const subscriptions = [
+            [RTVI_EVENTS.serverMessage, onServerMessage],
+            [RTVI_EVENTS.connected, this.onConnectedOrReady],
+            [RTVI_EVENTS.botReady, this.onConnectedOrReady],
+            [RTVI_EVENTS.disconnected, this.onDisconnected],
+            [RTVI_EVENTS.error, this.onError],
+            [RTVI_EVENTS.userStartedSpeaking, this.onUserStartedSpeaking],
+            [RTVI_EVENTS.userStoppedSpeaking, this.onUserStoppedSpeaking],
+            [RTVI_EVENTS.botStartedSpeaking, this.onBotStartedSpeaking],
+            [RTVI_EVENTS.botStoppedSpeaking, this.onBotStoppedSpeaking],
+            [RTVI_EVENTS.userMuteStarted, this.onUserMuteStarted],
+            [RTVI_EVENTS.userMuteStopped, this.onUserMuteStopped],
+            [RTVI_EVENTS.trackStarted, onTrackStarted],
+        ];
+        for (const [event, listener] of subscriptions)
+            client.on(event, listener);
+        this.listenTo(client);
+        // Some browsers keep an audio context suspended until it is resumed inside
+        // a gesture, and the connect click usually lands before the bot's track
+        // exists. Any later click or key is the next chance.
+        const wake = () => this.probe?.resume?.();
+        const doc = typeof document === "undefined" ? null : document;
+        doc?.addEventListener("pointerdown", wake, true);
+        doc?.addEventListener("keydown", wake, true);
+        return () => {
+            doc?.removeEventListener("pointerdown", wake, true);
+            doc?.removeEventListener("keydown", wake, true);
+            this.clearIdleTimer();
+            this.clearOnsetTimer();
+            this.clearReanchorTimer();
+            if (!this.opts.playoutProbe)
+                this.dropProbe();
+            for (const [event, listener] of subscriptions)
+                client.off(event, listener);
+        };
+    }
+    /** Dispose controller-owned timers when its mounted avatar is destroyed. */
+    destroy() {
+        this.clearIdleTimer();
+        this.clearOnsetTimer();
+        this.clearReanchorTimer();
+        this.dropProbe();
+        this.behavior.destroy();
+    }
+}
+//# sourceMappingURL=AvatarClient.js.map