@voqalize/avatar 0.2.2 → 0.4.0

package/dist/AvatarClient.d.ts

@@ -0,0 +1,274 @@
+/**
+ * AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
+ * splice, framework-free (no React; the hook and component wrap this).
+ *
+ * ## Turn clock anchoring
+ *
+ * Base Pipecat TTS gives each serialized TTS context an opaque `context_id`.
+ * The server uses it only to group and splice cue chunks. `botStartedSpeaking`
+ * has no context payload, so the browser FIFO-claims the next buffered context
+ * at that Pipecat output-lifecycle event. `botStoppedSpeaking` closes it. No
+ * avatar-specific speech marker exists.
+ *
+ * The event says *which* audio is starting, not *when* it is heard, and the
+ * turn clock is anchored to the sound itself (`playout.ts` has the
+ * measurements that forced it). `attach()` listens to the bot's audio track;
+ * at the event, the turn's zero is backdated to a sound that has already begun,
+ * or the mouth is held shut until one does. When the track cannot be heard —
+ * no `attach()`, no track, a suspended audio graph, or a sound that never
+ * arrives within `ONSET_WAIT_MS` — the event is the anchor, as it always was.
+ * Which one won is logged once per turn, at `info`, because a desync report is
+ * unanswerable without it.
+ *
+ * The clock then runs `VISUAL_LEAD_MS` ahead of the sound, since the mixer's
+ * smoothing makes every mouth shape late by about that much. It is pulled back
+ * by any output-device latency beyond what the display already matches (a
+ * Bluetooth headset). Within a turn, each resumption after a pause in the
+ * track is a second chance to hear where the sound really is — but a mouth
+ * opens before its sound, so a resumption heard within the track's usual
+ * anticipation of it agrees with the clock. Only a disagreement beyond that is
+ * slewed out, at no more than 10 % of clock rate, never jumped, because a
+ * mouth that skips is seen and one that runs briefly fast is not.
+ *
+ * `attach()` subscribes to the avatar server-message channel *and* Pipecat's
+ * standard lifecycle events. Server messages carry only what Pipecat cannot:
+ * correlated speech/cue timing and deliberate application instructions. The
+ * lifecycle events project the factual presence states locally. The server
+ * supplies only the three lower-priority states and deliberate,
+ * self-completing actions. This keeps the face tied to Pipecat's output truth
+ * even if a server state is delayed or stale.
+ *
+ * ## Cue splice
+ *
+ * The widget has two cue-track primitives: `speak({cues, clock})` (a full
+ * replace) and `pushCues(cues)` (a pure union that can only grow the track,
+ * never shrink it). Neither is "discard queued cues at or after `from_ms`,
+ * then append" on its own — `pushCues` has no way to drop a stale tail. So
+ * this client keeps the turn's canonical cue array itself (kept portion +
+ * every appended chunk, spliced on each `cues` message) and picks the cheapest
+ * widget call that stays correct:
+ *
+ *   - if the splice's `from_ms` doesn't reach back into anything already
+ *     queued — the common case past a turn's first sentence, since only the
+ *     first sentence genuinely plays fast-leg cues — nothing needs discarding:
+ *     `pushCues(newCues)` is the cheap, correct append.
+ *   - if it does reach back (a real fast→accurate splice), `pushCues` cannot
+ *     express the discard; we call `speak()` again with the full spliced
+ *     canonical array on the turn's original clock. `speak()` is otherwise
+ *     documented as also killing an in-flight spoken interjection and
+ *     re-entering `SPEAKING` — both harmless mid-splice (an interjection
+ *     should not be running while a server track owns the mouth; re-entering
+ *     an unchanged state is a no-op past the profile/gaze reset the widget
+ *     already does for a same-name `setState`).
+ *
+ * Cues commonly arrive **before** `botStartedSpeaking` — the fast leg starts
+ * the moment a sentence is handed to TTS, well before bot output begins.
+ * Chunks that arrive before the clock is anchored are spliced into the
+ * canonical array but not yet handed to the widget; `botStartedSpeaking` hands over
+ * whatever has accumulated as the turn's first `speak()` call. So "the first
+ * chunk of a turn starts speak()" means the first *widget* call, not
+ * necessarily the first *message*.
+ */
+import type { PipecatClient } from "@pipecat-ai/client-js";
+import { type PlayoutProbe } from "./playout.js";
+import { type AvatarCommand, type AvatarCue } from "./types.js";
+/**
+ * How far the cue clock runs ahead of the sound. The mixer eases every mouth
+ * channel toward its target with a 42 ms time constant
+ * (`MOUTH_RESPONSE_TAU_S`), which delays a shape by about that much, and a
+ * frame is drawn on average half a frame after its time. Picture ahead of sound
+ * is the side people forgive (ITU-R BT.1359: sound leading is noticed at about
+ * 45 ms, picture leading at about 125 ms), so a small surplus is the safe error.
+ */
+export declare const VISUAL_LEAD_MS = 50;
+/**
+ * The private renderer-side vocabulary the lifecycle driver actually uses.
+ *
+ * This stays narrower than `AvatarApi` on purpose. The latter is the SVG
+ * mixer's driving API; requiring a second renderer to impersonate every one
+ * of its inspection and authoring methods would turn that implementation
+ * detail into a renderer contract. The public seam remains `createAvatar`.
+ */
+export interface AvatarDriver {
+    setState(name: string): unknown;
+    /**
+     * One motion by name — a core intent or one of this avatar's own. **A name
+     * it does not have is a no-op, not a throw**: the wire's action vocabulary is
+     * open, so a face that cannot do the thing is as expected as a newer server.
+     */
+    action(id: string): unknown;
+    speak(options: {
+        cues: AvatarCue[];
+        clock: () => number;
+    }): unknown;
+    pushCues(cues: AvatarCue[]): unknown;
+    stopSpeaking(): unknown;
+    setUserSpeaking(speaking: boolean | null): unknown;
+}
+/**
+ * Internal. Not exported from the package — the public surface is
+ * `createAvatar({ mount, client })` and nothing else.
+ *
+ * There is deliberately no `accept` predicate here any more. Avatar commands
+ * travel in one envelope, `{type:"avatar"}`, in both directions and from every
+ * source: a `AvatarProcessor` in the pipeline and a brain driving the face
+ * out of band emit the same shape. A per-deployment predicate meant the
+ * library could not state what an avatar message *is*, which is the one thing
+ * a wire format has to be able to say.
+ */
+export interface AvatarClientOptions {
+    /** A dispatch threw (e.g. an unknown state or interjection id, which the
+     * widget throws on). Defaults to `console.warn`. */
+    onError?: (err: unknown, msg: AvatarCommand) => void;
+    /** Override for tests. Defaults to `performance.now`. */
+    now?: () => number;
+    /** Override for tests. Defaults to listening to the bot's audio track, once
+     * `attach()` has found one. */
+    playoutProbe?: PlayoutProbe;
+    /** Quiet time in listening before the client-owned idle loop begins. */
+    idleDelayMs?: number;
+    /** Timer seams keep lifecycle behavior deterministic in tests. */
+    setTimeout?: typeof setTimeout;
+    clearTimeout?: typeof clearTimeout;
+    /**
+     * Internal, for Studio's inspector — the resolved projection, so a developer
+     * tool can show what the lifecycle decided. It is *not* on the public
+     * surface and must not become one: a presence callback is a contract, and
+     * publishing it would oblige every avatar implementation to emit these seven
+     * states with this precedence, which is exactly the second public contract
+     * the design exists to avoid.
+     */
+    onPresenceChange?: (state: AvatarPresenceState) => void;
+}
+/**
+ * The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
+ *
+ * Written out rather than imported because that enum was this module's *only*
+ * runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
+ * the whole `/pipecat` subpath fail to load without the peer installed — even
+ * for a host that drives `dispatch()` from its own transport and never calls
+ * `attach()`. The peer is declared optional; this is what makes that true
+ * rather than aspirational.
+ *
+ * String enums are nominal in TypeScript, so the compiler cannot check these
+ * against the real ones from a type-only import. `packages/avatar/test/AvatarClient.test.ts`
+ * does it instead, against the actual enum — the devDependency is present
+ * exactly where the check belongs and absent from what we ship.
+ */
+export declare const RTVI_EVENTS: {
+    readonly serverMessage: "serverMessage";
+    readonly connected: "connected";
+    readonly disconnected: "disconnected";
+    readonly botReady: "botReady";
+    readonly error: "error";
+    readonly userStartedSpeaking: "userStartedSpeaking";
+    readonly userStoppedSpeaking: "userStoppedSpeaking";
+    readonly botStartedSpeaking: "botStartedSpeaking";
+    readonly botStoppedSpeaking: "botStoppedSpeaking";
+    readonly trackStarted: "trackStarted";
+    readonly userMuteStarted: "userMuteStarted";
+    readonly userMuteStopped: "userMuteStopped";
+};
+/** The resolved, factual presence state a host may render around the avatar. */
+export type AvatarPresenceState = "IDLE" | "LISTENING" | "CANT_HEAR" | "THINKING" | "WORKING" | "MUTED" | "SPEAKING" | "DEGRADED" | "OFFLINE";
+export declare class AvatarClient {
+    private readonly avatar;
+    /** Maps factual/wire intent into the broader client behavior catalog. */
+    private readonly behavior;
+    private readonly opts;
+    private readonly now;
+    private turn;
+    private readonly turns;
+    private readonly pendingCtxs;
+    private readonly closedCtxs;
+    private projected;
+    private serverState;
+    private userSpeaking;
+    private botSpeaking;
+    private muted;
+    private listening;
+    private idle;
+    private failure;
+    private idleTimer;
+    private pendingInterruptedAction;
+    private discardQueuedContextsOnBotStop;
+    private readonly idleDelayMs;
+    private readonly setTimer;
+    private readonly clearTimer;
+    private probe;
+    private probeTrack;
+    /** Whether the probe has ever heard an onset. One that has not, and times
+     * out, is deaf to this track rather than early to it. */
+    private probeHeard;
+    private onsetTimer;
+    private reanchorTimer;
+    constructor(avatar: AvatarDriver, opts?: AvatarClientOptions);
+    /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
+    get turnCtx(): string | null;
+    /** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
+    get turnCues(): AvatarCue[];
+    /** Current resolved projection. Internal — for tests and Studio's inspector.
+     * Before the first lifecycle fact the renderer is simply at its ready rest
+     * pose, so `LISTENING` is the safe value; it is not a session status. */
+    get presenceState(): AvatarPresenceState;
+    /** Re-apply the currently resolved factual projection after an embedding
+     * tool has temporarily used raw renderer controls. This does not invent a
+     * lifecycle event or cancel a finite action. */
+    restoreProjection(): void;
+    /** Dispatch one server message. Anything that isn't in the avatar envelope
+     * is not ours and is ignored; so is an envelope carrying a `cmd` this build
+     * has never heard of, per the wire protocol's forward-compat rule. */
+    dispatch(raw: unknown): void;
+    private handleState;
+    private handleAction;
+    private playAction;
+    private ensureTurn;
+    private handleCues;
+    private activateNextTurn;
+    /** Put the turn's zero where its sound began — see the header. */
+    private anchor;
+    private anchored;
+    /** The lead for a turn starting now — see `VISUAL_LEAD_MS`. */
+    private leadMs;
+    /** The first resumption after `after` (track ms) that follows a pause long
+     * enough to be heard as one. */
+    private nextResumption;
+    /** Listen for the sound at the turn's next resumption, once. */
+    private scheduleReanchor;
+    private clearReanchorTimer;
+    private clearOnsetTimer;
+    private dropProbe;
+    /** Listen to the bot's audio track, if the transport has one yet. */
+    private listenTo;
+    private discardQueuedTurns;
+    private lifecycleState;
+    private applyProjection;
+    private clearIdleTimer;
+    private armIdleIfEligible;
+    private eligibleForIdle;
+    private enterListening;
+    private clearServerStateForTurnBoundary;
+    private maybePlayInterrupted;
+    private clearRecoverableFailure;
+    private onUserStartedSpeaking;
+    private onUserStoppedSpeaking;
+    private onBotStartedSpeaking;
+    private onBotStoppedSpeaking;
+    private onUserMuteStarted;
+    private onUserMuteStopped;
+    private onError;
+    private onDisconnected;
+    private onConnectedOrReady;
+    /**
+     * Subscribe to a live `PipecatClient`. Standard client events own the normal
+     * lifecycle projection; avatar server-messages carry correlated visemes and
+     * explicit application intent. Never throws on malformed or irrelevant
+     * server messages.
+     *
+     * @returns an unsubscribe function; call it on unmount or disconnect.
+     */
+    attach(client: PipecatClient): () => void;
+    /** Dispose controller-owned timers when its mounted avatar is destroyed. */
+    destroy(): void;
+}
+//# sourceMappingURL=AvatarClient.d.ts.map

package/dist/AvatarClient.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AvatarClient.d.ts","sourceRoot":"","sources":["../client/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAa,MAAM,uBAAuB,CAAC;AAEtE,OAAO,EAAsB,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,EAGL,KAAK,aAAa,EAClB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAgCpB;;;;;;;GAOG;AACH,eAAO,MAAM,cAAc,KAAK,CAAC;AA4BjC;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IAChC;;;;OAIG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE;QAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;IACpE,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IACrC,YAAY,IAAI,OAAO,CAAC;IACxB,eAAe,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC;CACpD;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC;wDACoD;IACpD,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC;IACrD,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB;mCAC+B;IAC/B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,UAAU,CAAC,EAAE,OAAO,UAAU,CAAC;IAC/B,YAAY,CAAC,EAAE,OAAO,YAAY,CAAC;IACnC;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;CACzD;AAED;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,WAAW;;;;;;;;;;;;;CAmBmB,CAAC;AAE5C,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAC3B,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,GAC3D,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAYlD,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2B;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,SAAS,CAAS;IAG1B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,SAAS,CAA8C;IAC/D,OAAO,CAAC,wBAAwB,CAAS;IACzC,OAAO,CAAC,8BAA8B,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoB;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;IACjD,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,UAAU,CAAiC;IACnD;6DACyD;IACzD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,UAAU,CAA8C;IAChE,OAAO,CAAC,aAAa,CAA8C;gBAEvD,MAAM,EAAE,YAAY,EAAE,IAAI,GAAE,mBAAwB;IAWhE,+EAA+E;IAC/E,IAAI,OAAO,IAAI,MAAM,GAAG,IAAI,CAE3B;IAED,wFAAwF;IACxF,IAAI,QAAQ,IAAI,SAAS,EAAE,CAE1B;IAED;;6EAEyE;IACzE,IAAI,aAAa,IAAI,mBAAmB,CAEvC;IAED;;oDAEgD;IAChD,iBAAiB,IAAI,IAAI;IAIzB;;0EAEsE;IACtE,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IA0B5B,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,YAAY;IAcpB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,UAAU;IAYlB,OAAO,CAAC,UAAU;IAqBlB,OAAO,CAAC,gBAAgB;IA+BxB,kEAAkE;IAClE,OAAO,CAAC,MAAM;IA+Bd,OAAO,CAAC,QAAQ;IAUhB,+DAA+D;IAC/D,OAAO,CAAC,MAAM;IAMd;oCACgC;IAChC,OAAO,CAAC,cAAc;IAatB,gEAAgE;IAChE,OAAO,CAAC,gBAAgB;IAgCxB,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,SAAS;IAOjB,qEAAqE;IACrE,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,eAAe;IAKvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,+BAA+B;IAMvC,OAAO,CAAC,oBAAoB;IAO5B,OAAO,CAAC,uBAAuB;IAO/B,OAAO,CAAC,qBAAqB,CAS3B;IAEF,OAAO,CAAC,qBAAqB,CAI3B;IAEF,OAAO,CAAC,oBAAoB,CAQ1B;IAEF,OAAO,CAAC,oBAAoB,CAkB1B;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,OAAO,CAIb;IAEF,OAAO,CAAC,cAAc,CAIpB;IAEF,OAAO,CAAC,kBAAkB,CASxB;IAEF;;;;;;;OAOG;IACH,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,IAAI;IA4CzC,4EAA4E;IAC5E,OAAO,IAAI,IAAI;CAOhB"}