@voqalize/avatar 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +692 -0
- package/client/dist/Avatar.d.ts +24 -0
- package/client/dist/Avatar.d.ts.map +1 -0
- package/client/dist/Avatar.js +7 -0
- package/client/dist/Avatar.js.map +1 -0
- package/client/dist/AvatarClient.d.ts +173 -0
- package/client/dist/AvatarClient.d.ts.map +1 -0
- package/client/dist/AvatarClient.js +274 -0
- package/client/dist/AvatarClient.js.map +1 -0
- package/client/dist/pipecat.d.ts +21 -0
- package/client/dist/pipecat.d.ts.map +1 -0
- package/client/dist/pipecat.js +21 -0
- package/client/dist/pipecat.js.map +1 -0
- package/client/dist/react.d.ts +16 -0
- package/client/dist/react.d.ts.map +1 -0
- package/client/dist/react.js +17 -0
- package/client/dist/react.js.map +1 -0
- package/client/dist/types.d.ts +101 -0
- package/client/dist/types.d.ts.map +1 -0
- package/client/dist/types.js +31 -0
- package/client/dist/types.js.map +1 -0
- package/client/dist/useAvatar.d.ts +53 -0
- package/client/dist/useAvatar.d.ts.map +1 -0
- package/client/dist/useAvatar.js +68 -0
- package/client/dist/useAvatar.js.map +1 -0
- package/client/src/Avatar.tsx +38 -0
- package/client/src/AvatarClient.ts +343 -0
- package/client/src/pipecat.ts +38 -0
- package/client/src/react.ts +34 -0
- package/client/src/types.ts +127 -0
- package/client/src/useAvatar.ts +113 -0
- package/docs/contract-avatar.md +337 -0
- package/docs/contract-protocol.md +401 -0
- package/package.json +89 -0
- package/src/audio-fallback.js +100 -0
- package/src/avatar.d.ts +241 -0
- package/src/avatar.js +722 -0
- package/src/clips.js +144 -0
- package/src/emotions.js +55 -0
- package/src/face-core.js +154 -0
- package/src/face-myna.js +725 -0
- package/src/face-peep.js +767 -0
- package/src/face-wren.js +470 -0
- package/src/gaze.js +155 -0
- package/src/idle.js +535 -0
- package/src/interjections.js +578 -0
- package/src/line-art.js +111 -0
- package/src/params.js +176 -0
- package/src/perform.js +105 -0
- package/src/visemes.js +230 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Avatar — a call-tile-ready wrapper around the widget.
|
|
3
|
+
*
|
|
4
|
+
* Thin by design: `useAvatar` does the work; this is the div it mounts into
|
|
5
|
+
* plus prop plumbing.
|
|
6
|
+
*
|
|
7
|
+
* <Avatar client={session.client} className="avatar-tile" />
|
|
8
|
+
*
|
|
9
|
+
* The widget has no hot-swap-avatar API — `createFace` runs once per mount —
|
|
10
|
+
* so `avatar`/`theme`/`mouthGain`/`gestureGain` are read once, at mount. To
|
|
11
|
+
* switch avatars at runtime, remount with a `key` prop:
|
|
12
|
+
*
|
|
13
|
+
* <Avatar key={name} avatar={name} client={session.client} />
|
|
14
|
+
*/
|
|
15
|
+
import type { CSSProperties } from "react";
|
|
16
|
+
import type { UseAvatarOptions } from "./useAvatar.js";
|
|
17
|
+
export interface AvatarProps extends UseAvatarOptions {
|
|
18
|
+
className?: string;
|
|
19
|
+
style?: CSSProperties;
|
|
20
|
+
/** Forwarded to the mount `<div>`. */
|
|
21
|
+
"aria-label"?: string;
|
|
22
|
+
}
|
|
23
|
+
export declare function Avatar({ className, style, "aria-label": ariaLabel, ...options }: AvatarProps): import("react").JSX.Element;
|
|
24
|
+
//# sourceMappingURL=Avatar.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Avatar.d.ts","sourceRoot":"","sources":["../src/Avatar.tsx"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAC3C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGvD,MAAM,WAAW,WAAY,SAAQ,gBAAgB;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,sCAAsC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,MAAM,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,EAAE,WAAW,+BAW5F"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { jsx as _jsx } from "react/jsx-runtime";
|
|
2
|
+
import { useAvatar } from "./useAvatar.js";
|
|
3
|
+
export function Avatar({ className, style, "aria-label": ariaLabel, ...options }) {
|
|
4
|
+
const { containerRef } = useAvatar(options);
|
|
5
|
+
return (_jsx("div", { ref: containerRef, className: className, style: style, "aria-label": ariaLabel ?? "avatar", role: "img" }));
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=Avatar.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Avatar.js","sourceRoot":"","sources":["../src/Avatar.tsx"],"names":[],"mappings":";AAiBA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAS3C,MAAM,UAAU,MAAM,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,OAAO,EAAe;IAC3F,MAAM,EAAE,YAAY,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,CACL,cACE,GAAG,EAAE,YAAY,EACjB,SAAS,EAAE,SAAS,EACpB,KAAK,EAAE,KAAK,gBACA,SAAS,IAAI,QAAQ,EACjC,IAAI,EAAC,KAAK,GACV,CACH,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
|
|
3
|
+
* splice, framework-free (no React; the hook and component wrap this).
|
|
4
|
+
*
|
|
5
|
+
* ## Turn clock anchoring
|
|
6
|
+
*
|
|
7
|
+
* A turn's `t0` is anchored to `performance.now()` **at the moment this client
|
|
8
|
+
* receives the `{cmd:"speech", event:"start"}` message** — cues are
|
|
9
|
+
* client-anchored, not per-cue server-released. That message rides the RTVI
|
|
10
|
+
* data channel, ahead of the jitter-buffered audio path, so the residual error
|
|
11
|
+
* lands on the video-leads side — the side `docs/contract-protocol.md` says
|
|
12
|
+
* perceptual tolerance favours (+125 ms vs -45 ms).
|
|
13
|
+
*
|
|
14
|
+
* We investigated anchoring on pipecat client-js's own `RTVIEvent
|
|
15
|
+
* .BotStartedSpeaking`/`BotStoppedSpeaking` instead (or as a refinement) and
|
|
16
|
+
* chose not to, for two reasons:
|
|
17
|
+
*
|
|
18
|
+
* 1. **No turn correlation.** Those events carry no payload — no `ctx` — so
|
|
19
|
+
* there is no way to tell which turn a firing belongs to. Our own
|
|
20
|
+
* `speech` command carries `ctx`, which the splice logic below needs
|
|
21
|
+
* regardless, so anchoring off it costs nothing extra.
|
|
22
|
+
* 2. **Same source, same path, no accuracy gain.** The `AvatarProcessor`
|
|
23
|
+
* sits between the TTS service and the output transport and observes the
|
|
24
|
+
* transport's own `BotStarted/StoppedSpeakingFrame` broadcasts — the exact
|
|
25
|
+
* frame pipecat's built-in speaking detection is *also* driven from. Both
|
|
26
|
+
* notifications travel the same data-channel path to the browser. There is
|
|
27
|
+
* no local "truly audible now" signal cheaply available: the audio arrives
|
|
28
|
+
* on a `MediaStreamTrack` whose only lifecycle events (`unmute`/`mute`)
|
|
29
|
+
* fire once per call, not per utterance. Tapping the decoded remote audio
|
|
30
|
+
* with a WebAudio `AnalyserNode` RMS gate *would* give one, but it adds
|
|
31
|
+
* its own onset latency and a real audio pipeline to build and tune, and
|
|
32
|
+
* it would eat into the intentional video-first safety margin rather than
|
|
33
|
+
* improve it. Left as a documented option, not built.
|
|
34
|
+
*
|
|
35
|
+
* `attach()` still subscribes to both pipecat events, but only to report a
|
|
36
|
+
* **diagnostic** drift (`onSpeakingDrift`) between our anchor and pipecat's —
|
|
37
|
+
* useful for noticing in logs if the two ever separate by more than jitter,
|
|
38
|
+
* never used to move `t0` itself.
|
|
39
|
+
*
|
|
40
|
+
* ## Cue splice
|
|
41
|
+
*
|
|
42
|
+
* The widget has two cue-track primitives: `speak({cues, clock})` (a full
|
|
43
|
+
* replace) and `pushCues(cues)` (a pure union that can only grow the track,
|
|
44
|
+
* never shrink it). Neither is "discard queued cues at or after `from_ms`,
|
|
45
|
+
* then append" on its own — `pushCues` has no way to drop a stale tail. So
|
|
46
|
+
* this client keeps the turn's canonical cue array itself (kept portion +
|
|
47
|
+
* every appended chunk, spliced on each `cues` message) and picks the cheapest
|
|
48
|
+
* widget call that stays correct:
|
|
49
|
+
*
|
|
50
|
+
* - if the splice's `from_ms` doesn't reach back into anything already
|
|
51
|
+
* queued — the common case past a turn's first sentence, since only the
|
|
52
|
+
* first sentence genuinely plays fast-leg cues — nothing needs discarding:
|
|
53
|
+
* `pushCues(newCues)` is the cheap, correct append.
|
|
54
|
+
* - if it does reach back (a real fast→accurate splice), `pushCues` cannot
|
|
55
|
+
* express the discard; we call `speak()` again with the full spliced
|
|
56
|
+
* canonical array on the turn's original clock. `speak()` is otherwise
|
|
57
|
+
* documented as also killing an in-flight spoken interjection and
|
|
58
|
+
* re-entering `SPEAKING` — both harmless mid-splice (an interjection
|
|
59
|
+
* should not be running while a server track owns the mouth; re-entering
|
|
60
|
+
* an unchanged state is a no-op past the profile/gaze reset the widget
|
|
61
|
+
* already does for a same-name `setState`).
|
|
62
|
+
*
|
|
63
|
+
* Cues commonly arrive **before** `speech start` — the fast leg starts the
|
|
64
|
+
* moment a sentence is handed to TTS, well before `BotStartedSpeakingFrame`.
|
|
65
|
+
* Chunks that arrive before the clock is anchored are spliced into the
|
|
66
|
+
* canonical array but not yet handed to the widget; `speech start` hands over
|
|
67
|
+
* whatever has accumulated as the turn's first `speak()` call. So "the first
|
|
68
|
+
* chunk of a turn starts speak()" means the first *widget* call, not
|
|
69
|
+
* necessarily the first *message*.
|
|
70
|
+
*/
|
|
71
|
+
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
72
|
+
import type { AvatarApi } from "../../src/avatar.js";
|
|
73
|
+
import { type AvatarCommand, type AvatarCue, type AvatarHintCmd, type AvatarUnknownCmd } from "./types.js";
|
|
74
|
+
export interface AvatarClientOptions {
|
|
75
|
+
/** `{cmd:"hint"}` is a no-op hook today — the widget's listening engine
|
|
76
|
+
* already handles acks; a host may still want to know a hint arrived. */
|
|
77
|
+
onHint?: (kind: string, msg: AvatarHintCmd) => void;
|
|
78
|
+
/** An unrecognized `cmd` (forward compat) — the protocol says ignore
|
|
79
|
+
* silently, so this is purely an observability hook, not required. */
|
|
80
|
+
onUnknownCmd?: (msg: AvatarUnknownCmd) => void;
|
|
81
|
+
/** A dispatch threw (e.g. an unknown state or interjection id, which the
|
|
82
|
+
* widget throws on). Defaults to `console.warn`. */
|
|
83
|
+
onError?: (err: unknown, msg: AvatarCommand) => void;
|
|
84
|
+
/** Diagnostic only (see the class doc's "Turn clock anchoring" section) —
|
|
85
|
+
* never moves the anchor, just reports how far pipecat's own
|
|
86
|
+
* botStartedSpeaking/botStoppedSpeaking landed from it. */
|
|
87
|
+
onSpeakingDrift?: (info: {
|
|
88
|
+
event: "start" | "stop";
|
|
89
|
+
ctx: string | null;
|
|
90
|
+
driftMs: number;
|
|
91
|
+
}) => void;
|
|
92
|
+
/**
|
|
93
|
+
* Which server-messages `attach()` should look inside. Defaults to the
|
|
94
|
+
* protocol's own envelope, `type === "avatar"`.
|
|
95
|
+
*
|
|
96
|
+
* The escape hatch exists because an application may tunnel avatar commands
|
|
97
|
+
* inside a message type of its own — one deployment routes them through a
|
|
98
|
+
* generic `ui_command` envelope so an LLM tool call can drive the face — and
|
|
99
|
+
* teaching this library that envelope would be teaching it one consumer's
|
|
100
|
+
* private vocabulary. Widen it here instead:
|
|
101
|
+
*
|
|
102
|
+
* accept: (m) => m.type === "avatar" ||
|
|
103
|
+
* (m.type === "ui_command" && m.action === "avatar")
|
|
104
|
+
*
|
|
105
|
+
* The predicate only decides *whether to look*; the payload still has to
|
|
106
|
+
* carry a string `cmd` to dispatch at all.
|
|
107
|
+
*/
|
|
108
|
+
accept?: (message: Record<string, unknown>) => boolean;
|
|
109
|
+
/** Override for tests. Defaults to `performance.now`. */
|
|
110
|
+
now?: () => number;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* The three `RTVIEvent` members `attach()` subscribes to, spelled as their
|
|
114
|
+
* values.
|
|
115
|
+
*
|
|
116
|
+
* Written out rather than imported because that enum was this module's *only*
|
|
117
|
+
* runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
|
|
118
|
+
* the whole `/pipecat` subpath fail to load without the peer installed — even
|
|
119
|
+
* for a host that drives `dispatch()` from its own transport and never calls
|
|
120
|
+
* `attach()`. The peer is declared optional; this is what makes that true
|
|
121
|
+
* rather than aspirational.
|
|
122
|
+
*
|
|
123
|
+
* String enums are nominal in TypeScript, so the compiler cannot check these
|
|
124
|
+
* against the real ones from a type-only import. `client/test/AvatarClient.test.ts`
|
|
125
|
+
* does it instead, against the actual enum — the devDependency is present
|
|
126
|
+
* exactly where the check belongs and absent from what we ship.
|
|
127
|
+
*/
|
|
128
|
+
export declare const RTVI_EVENTS: {
|
|
129
|
+
readonly serverMessage: "serverMessage";
|
|
130
|
+
readonly botStartedSpeaking: "botStartedSpeaking";
|
|
131
|
+
readonly botStoppedSpeaking: "botStoppedSpeaking";
|
|
132
|
+
};
|
|
133
|
+
export declare class AvatarClient {
|
|
134
|
+
private readonly avatar;
|
|
135
|
+
private readonly opts;
|
|
136
|
+
private readonly now;
|
|
137
|
+
private readonly accept;
|
|
138
|
+
private turn;
|
|
139
|
+
constructor(avatar: AvatarApi, opts?: AvatarClientOptions);
|
|
140
|
+
/** The active turn's ctx, or `null` between turns. For tests and telemetry. */
|
|
141
|
+
get turnCtx(): string | null;
|
|
142
|
+
/** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
|
|
143
|
+
get turnCues(): AvatarCue[];
|
|
144
|
+
/** Dispatch one avatar command. Accepts anything with a string `cmd` — an
|
|
145
|
+
* already-unwrapped `{type:"avatar", cmd, ...}` server message, or a bare
|
|
146
|
+
* `{cmd, ...}` payload from whatever else the host is carrying them in.
|
|
147
|
+
* Unknown `cmd`s are ignored, per the wire protocol's forward-compat rule. */
|
|
148
|
+
dispatch(raw: unknown): void;
|
|
149
|
+
private handleState;
|
|
150
|
+
private handlePerform;
|
|
151
|
+
/** Ride the named turn's clock if it's the one we're currently anchored to;
|
|
152
|
+
* otherwise (no active turn, or `perform` names a ctx we never saw a
|
|
153
|
+
* `speech start` for) fall back to a fresh clock anchored at this call — the
|
|
154
|
+
* same "elapsed ms since this call" default `avatar.perform()` itself uses
|
|
155
|
+
* when given no clock and no audio. */
|
|
156
|
+
private resolveClock;
|
|
157
|
+
private ensureTurn;
|
|
158
|
+
private handleCues;
|
|
159
|
+
private handleSpeech;
|
|
160
|
+
private reportDrift;
|
|
161
|
+
/**
|
|
162
|
+
* Subscribe to a live `PipecatClient`'s server messages and dispatch the
|
|
163
|
+
* avatar commands among them. Which messages count is the `accept` option;
|
|
164
|
+
* by default, the protocol's own `{type:"avatar"}` envelope.
|
|
165
|
+
*
|
|
166
|
+
* Also wires the diagnostic drift cross-check described in the class doc.
|
|
167
|
+
* Never throws on a malformed or irrelevant message.
|
|
168
|
+
*
|
|
169
|
+
* @returns an unsubscribe function; call it on unmount or disconnect.
|
|
170
|
+
*/
|
|
171
|
+
attach(client: PipecatClient): () => void;
|
|
172
|
+
}
|
|
173
|
+
//# sourceMappingURL=AvatarClient.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AvatarClient.d.ts","sourceRoot":"","sources":["../src/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAa,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAGL,KAAK,aAAa,EAClB,KAAK,SAAS,EAEd,KAAK,aAAa,EAIlB,KAAK,gBAAgB,EACtB,MAAM,YAAY,CAAC;AAYpB,MAAM,WAAW,mBAAmB;IAClC;6EACyE;IACzE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC;IACpD;0EACsE;IACtE,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC/C;wDACoD;IACpD,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC;IACrD;;+DAE2D;IAC3D,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,OAAO,GAAG,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACnG;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC;IACvD,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,WAAW;;;;CAImB,CAAC;AAU5C,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgD;IACvE,OAAO,CAAC,IAAI,CAAqB;gBAErB,MAAM,EAAE,SAAS,EAAE,IAAI,GAAE,mBAAwB;IAO7D,+EAA+E;IAC/E,IAAI,OAAO,IAAI,MAAM,GAAG,IAAI,CAE3B;IAED,wFAAwF;IACxF,IAAI,QAAQ,IAAI,SAAS,EAAE,CAE1B;IAED;;;kFAG8E;IAC9E,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IAsC5B,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,aAAa;IAIrB;;;;2CAIuC;IACvC,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,UAAU;IAWlB,OAAO,CAAC,UAAU;IAkBlB,OAAO,CAAC,YAAY;IAmBpB,OAAO,CAAC,WAAW;IAOnB;;;;;;;;;OASG;IACH,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,IAAI;CAuB1C"}
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
|
|
3
|
+
* splice, framework-free (no React; the hook and component wrap this).
|
|
4
|
+
*
|
|
5
|
+
* ## Turn clock anchoring
|
|
6
|
+
*
|
|
7
|
+
* A turn's `t0` is anchored to `performance.now()` **at the moment this client
|
|
8
|
+
* receives the `{cmd:"speech", event:"start"}` message** — cues are
|
|
9
|
+
* client-anchored, not per-cue server-released. That message rides the RTVI
|
|
10
|
+
* data channel, ahead of the jitter-buffered audio path, so the residual error
|
|
11
|
+
* lands on the video-leads side — the side `docs/contract-protocol.md` says
|
|
12
|
+
* perceptual tolerance favours (+125 ms vs -45 ms).
|
|
13
|
+
*
|
|
14
|
+
* We investigated anchoring on pipecat client-js's own `RTVIEvent
|
|
15
|
+
* .BotStartedSpeaking`/`BotStoppedSpeaking` instead (or as a refinement) and
|
|
16
|
+
* chose not to, for two reasons:
|
|
17
|
+
*
|
|
18
|
+
* 1. **No turn correlation.** Those events carry no payload — no `ctx` — so
|
|
19
|
+
* there is no way to tell which turn a firing belongs to. Our own
|
|
20
|
+
* `speech` command carries `ctx`, which the splice logic below needs
|
|
21
|
+
* regardless, so anchoring off it costs nothing extra.
|
|
22
|
+
* 2. **Same source, same path, no accuracy gain.** The `AvatarProcessor`
|
|
23
|
+
* sits between the TTS service and the output transport and observes the
|
|
24
|
+
* transport's own `BotStarted/StoppedSpeakingFrame` broadcasts — the exact
|
|
25
|
+
* frame pipecat's built-in speaking detection is *also* driven from. Both
|
|
26
|
+
* notifications travel the same data-channel path to the browser. There is
|
|
27
|
+
* no local "truly audible now" signal cheaply available: the audio arrives
|
|
28
|
+
* on a `MediaStreamTrack` whose only lifecycle events (`unmute`/`mute`)
|
|
29
|
+
* fire once per call, not per utterance. Tapping the decoded remote audio
|
|
30
|
+
* with a WebAudio `AnalyserNode` RMS gate *would* give one, but it adds
|
|
31
|
+
* its own onset latency and a real audio pipeline to build and tune, and
|
|
32
|
+
* it would eat into the intentional video-first safety margin rather than
|
|
33
|
+
* improve it. Left as a documented option, not built.
|
|
34
|
+
*
|
|
35
|
+
* `attach()` still subscribes to both pipecat events, but only to report a
|
|
36
|
+
* **diagnostic** drift (`onSpeakingDrift`) between our anchor and pipecat's —
|
|
37
|
+
* useful for noticing in logs if the two ever separate by more than jitter,
|
|
38
|
+
* never used to move `t0` itself.
|
|
39
|
+
*
|
|
40
|
+
* ## Cue splice
|
|
41
|
+
*
|
|
42
|
+
* The widget has two cue-track primitives: `speak({cues, clock})` (a full
|
|
43
|
+
* replace) and `pushCues(cues)` (a pure union that can only grow the track,
|
|
44
|
+
* never shrink it). Neither is "discard queued cues at or after `from_ms`,
|
|
45
|
+
* then append" on its own — `pushCues` has no way to drop a stale tail. So
|
|
46
|
+
* this client keeps the turn's canonical cue array itself (kept portion +
|
|
47
|
+
* every appended chunk, spliced on each `cues` message) and picks the cheapest
|
|
48
|
+
* widget call that stays correct:
|
|
49
|
+
*
|
|
50
|
+
* - if the splice's `from_ms` doesn't reach back into anything already
|
|
51
|
+
* queued — the common case past a turn's first sentence, since only the
|
|
52
|
+
* first sentence genuinely plays fast-leg cues — nothing needs discarding:
|
|
53
|
+
* `pushCues(newCues)` is the cheap, correct append.
|
|
54
|
+
* - if it does reach back (a real fast→accurate splice), `pushCues` cannot
|
|
55
|
+
* express the discard; we call `speak()` again with the full spliced
|
|
56
|
+
* canonical array on the turn's original clock. `speak()` is otherwise
|
|
57
|
+
* documented as also killing an in-flight spoken interjection and
|
|
58
|
+
* re-entering `SPEAKING` — both harmless mid-splice (an interjection
|
|
59
|
+
* should not be running while a server track owns the mouth; re-entering
|
|
60
|
+
* an unchanged state is a no-op past the profile/gaze reset the widget
|
|
61
|
+
* already does for a same-name `setState`).
|
|
62
|
+
*
|
|
63
|
+
* Cues commonly arrive **before** `speech start` — the fast leg starts the
|
|
64
|
+
* moment a sentence is handed to TTS, well before `BotStartedSpeakingFrame`.
|
|
65
|
+
* Chunks that arrive before the clock is anchored are spliced into the
|
|
66
|
+
* canonical array but not yet handed to the widget; `speech start` hands over
|
|
67
|
+
* whatever has accumulated as the turn's first `speak()` call. So "the first
|
|
68
|
+
* chunk of a turn starts speak()" means the first *widget* call, not
|
|
69
|
+
* necessarily the first *message*.
|
|
70
|
+
*/
|
|
71
|
+
import { AVATAR_MESSAGE_TYPE, isAvatarMessage, } from "./types.js";
|
|
72
|
+
/**
|
|
73
|
+
* The three `RTVIEvent` members `attach()` subscribes to, spelled as their
|
|
74
|
+
* values.
|
|
75
|
+
*
|
|
76
|
+
* Written out rather than imported because that enum was this module's *only*
|
|
77
|
+
* runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
|
|
78
|
+
* the whole `/pipecat` subpath fail to load without the peer installed — even
|
|
79
|
+
* for a host that drives `dispatch()` from its own transport and never calls
|
|
80
|
+
* `attach()`. The peer is declared optional; this is what makes that true
|
|
81
|
+
* rather than aspirational.
|
|
82
|
+
*
|
|
83
|
+
* String enums are nominal in TypeScript, so the compiler cannot check these
|
|
84
|
+
* against the real ones from a type-only import. `client/test/AvatarClient.test.ts`
|
|
85
|
+
* does it instead, against the actual enum — the devDependency is present
|
|
86
|
+
* exactly where the check belongs and absent from what we ship.
|
|
87
|
+
*/
|
|
88
|
+
export const RTVI_EVENTS = {
|
|
89
|
+
serverMessage: "serverMessage",
|
|
90
|
+
botStartedSpeaking: "botStartedSpeaking",
|
|
91
|
+
botStoppedSpeaking: "botStoppedSpeaking",
|
|
92
|
+
};
|
|
93
|
+
/** Defensive unwrap for the `RTVIEvent.ServerMessage` `{ data }` quirk: some
|
|
94
|
+
* transports deliver the payload directly and some wrap it once more. */
|
|
95
|
+
function unwrapServerMessage(raw) {
|
|
96
|
+
const obj = (raw ?? {});
|
|
97
|
+
const inner = obj["data"];
|
|
98
|
+
return inner && "type" in inner ? inner : obj;
|
|
99
|
+
}
|
|
100
|
+
export class AvatarClient {
|
|
101
|
+
avatar;
|
|
102
|
+
opts;
|
|
103
|
+
now;
|
|
104
|
+
accept;
|
|
105
|
+
turn = null;
|
|
106
|
+
constructor(avatar, opts = {}) {
|
|
107
|
+
this.avatar = avatar;
|
|
108
|
+
this.opts = opts;
|
|
109
|
+
this.now = opts.now ?? (() => performance.now());
|
|
110
|
+
this.accept = opts.accept ?? ((m) => m.type === AVATAR_MESSAGE_TYPE);
|
|
111
|
+
}
|
|
112
|
+
/** The active turn's ctx, or `null` between turns. For tests and telemetry. */
|
|
113
|
+
get turnCtx() {
|
|
114
|
+
return this.turn?.ctx ?? null;
|
|
115
|
+
}
|
|
116
|
+
/** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
|
|
117
|
+
get turnCues() {
|
|
118
|
+
return this.turn ? [...this.turn.cues] : [];
|
|
119
|
+
}
|
|
120
|
+
/** Dispatch one avatar command. Accepts anything with a string `cmd` — an
|
|
121
|
+
* already-unwrapped `{type:"avatar", cmd, ...}` server message, or a bare
|
|
122
|
+
* `{cmd, ...}` payload from whatever else the host is carrying them in.
|
|
123
|
+
* Unknown `cmd`s are ignored, per the wire protocol's forward-compat rule. */
|
|
124
|
+
dispatch(raw) {
|
|
125
|
+
if (!isAvatarMessage(raw))
|
|
126
|
+
return;
|
|
127
|
+
const msg = raw;
|
|
128
|
+
try {
|
|
129
|
+
switch (msg.cmd) {
|
|
130
|
+
case "state":
|
|
131
|
+
this.handleState(msg);
|
|
132
|
+
break;
|
|
133
|
+
case "interject":
|
|
134
|
+
this.avatar.interject(msg.id);
|
|
135
|
+
break;
|
|
136
|
+
case "perform":
|
|
137
|
+
this.handlePerform(msg);
|
|
138
|
+
break;
|
|
139
|
+
case "cues":
|
|
140
|
+
this.handleCues(msg);
|
|
141
|
+
break;
|
|
142
|
+
case "speech":
|
|
143
|
+
this.handleSpeech(msg);
|
|
144
|
+
break;
|
|
145
|
+
case "user":
|
|
146
|
+
this.avatar.setUserSpeaking(msg.speaking);
|
|
147
|
+
break;
|
|
148
|
+
case "hint": {
|
|
149
|
+
const hint = msg;
|
|
150
|
+
this.opts.onHint?.(hint.kind, hint);
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
default:
|
|
154
|
+
this.opts.onUnknownCmd?.(msg);
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
catch (err) {
|
|
159
|
+
if (this.opts.onError)
|
|
160
|
+
this.opts.onError(err, msg);
|
|
161
|
+
else
|
|
162
|
+
console.warn("[avatar] dispatch failed", msg, err);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
handleState(msg) {
|
|
166
|
+
// Deliberately no client-side dedup: pass every `state` command straight
|
|
167
|
+
// through. The widget's own setState already no-ops the parts that matter
|
|
168
|
+
// for an unchanged name (`changed` gates the blink and the 'state' event in
|
|
169
|
+
// avatar.js), and a server resending the same state name as a
|
|
170
|
+
// keepalive/resync must still land so an `emotion`/`gaze` override on this
|
|
171
|
+
// particular message takes effect.
|
|
172
|
+
this.avatar.setState(msg.name, { emotion: msg.emotion, gaze: msg.gaze });
|
|
173
|
+
}
|
|
174
|
+
handlePerform(msg) {
|
|
175
|
+
this.avatar.perform(msg.actions, { clock: this.resolveClock(msg.ctx) });
|
|
176
|
+
}
|
|
177
|
+
/** Ride the named turn's clock if it's the one we're currently anchored to;
|
|
178
|
+
* otherwise (no active turn, or `perform` names a ctx we never saw a
|
|
179
|
+
* `speech start` for) fall back to a fresh clock anchored at this call — the
|
|
180
|
+
* same "elapsed ms since this call" default `avatar.perform()` itself uses
|
|
181
|
+
* when given no clock and no audio. */
|
|
182
|
+
resolveClock(ctx) {
|
|
183
|
+
if (ctx && this.turn && this.turn.ctx === ctx && this.turn.clock) {
|
|
184
|
+
return this.turn.clock;
|
|
185
|
+
}
|
|
186
|
+
const start = this.now();
|
|
187
|
+
return () => this.now() - start;
|
|
188
|
+
}
|
|
189
|
+
ensureTurn(ctx) {
|
|
190
|
+
if (!this.turn || this.turn.ctx !== ctx) {
|
|
191
|
+
// A different ctx supersedes whatever turn we had — a stale trailing
|
|
192
|
+
// message for the old ctx will find `this.turn.ctx !== ctx` in
|
|
193
|
+
// handleSpeech's stop-guard and be ignored, rather than cutting off the
|
|
194
|
+
// new turn.
|
|
195
|
+
this.turn = { ctx, cues: [], started: false, clock: null, t0: null };
|
|
196
|
+
}
|
|
197
|
+
return this.turn;
|
|
198
|
+
}
|
|
199
|
+
handleCues(msg) {
|
|
200
|
+
const turn = this.ensureTurn(msg.ctx);
|
|
201
|
+
const kept = turn.cues.filter((c) => c.t < msg.from_ms);
|
|
202
|
+
const discarded = turn.cues.length - kept.length;
|
|
203
|
+
turn.cues = [...kept, ...msg.cues].sort((a, b) => a.t - b.t);
|
|
204
|
+
if (!turn.started) {
|
|
205
|
+
// No clock yet — buffer. `speech start` will hand this over as the turn's
|
|
206
|
+
// first speak() call.
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
if (discarded === 0) {
|
|
210
|
+
this.avatar.pushCues(msg.cues);
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
this.avatar.speak({ cues: turn.cues, clock: turn.clock });
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
handleSpeech(msg) {
|
|
217
|
+
if (msg.event === "start") {
|
|
218
|
+
const turn = this.ensureTurn(msg.ctx);
|
|
219
|
+
const t0 = this.now();
|
|
220
|
+
const clock = () => this.now() - t0;
|
|
221
|
+
turn.t0 = t0;
|
|
222
|
+
turn.clock = clock;
|
|
223
|
+
turn.started = true;
|
|
224
|
+
this.avatar.speak({ cues: turn.cues, clock });
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
// "stop": only act if it names the turn we're actually riding. A stale stop
|
|
228
|
+
// for an already-superseded ctx must not cut off a newer turn.
|
|
229
|
+
if (this.turn && this.turn.ctx === msg.ctx) {
|
|
230
|
+
this.avatar.stopSpeaking();
|
|
231
|
+
this.turn = null;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
reportDrift(event) {
|
|
235
|
+
if (!this.opts.onSpeakingDrift)
|
|
236
|
+
return;
|
|
237
|
+
const t0 = this.turn?.t0;
|
|
238
|
+
if (t0 == null)
|
|
239
|
+
return;
|
|
240
|
+
this.opts.onSpeakingDrift({ event, ctx: this.turn?.ctx ?? null, driftMs: this.now() - t0 });
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Subscribe to a live `PipecatClient`'s server messages and dispatch the
|
|
244
|
+
* avatar commands among them. Which messages count is the `accept` option;
|
|
245
|
+
* by default, the protocol's own `{type:"avatar"}` envelope.
|
|
246
|
+
*
|
|
247
|
+
* Also wires the diagnostic drift cross-check described in the class doc.
|
|
248
|
+
* Never throws on a malformed or irrelevant message.
|
|
249
|
+
*
|
|
250
|
+
* @returns an unsubscribe function; call it on unmount or disconnect.
|
|
251
|
+
*/
|
|
252
|
+
attach(client) {
|
|
253
|
+
const onServerMessage = (raw) => {
|
|
254
|
+
const message = unwrapServerMessage(raw);
|
|
255
|
+
if (!this.accept(message))
|
|
256
|
+
return;
|
|
257
|
+
this.dispatch(message);
|
|
258
|
+
};
|
|
259
|
+
const onBotStartedSpeaking = () => this.reportDrift("start");
|
|
260
|
+
const onBotStoppedSpeaking = () => this.reportDrift("stop");
|
|
261
|
+
const serverMessage = RTVI_EVENTS.serverMessage;
|
|
262
|
+
const started = RTVI_EVENTS.botStartedSpeaking;
|
|
263
|
+
const stopped = RTVI_EVENTS.botStoppedSpeaking;
|
|
264
|
+
client.on(serverMessage, onServerMessage);
|
|
265
|
+
client.on(started, onBotStartedSpeaking);
|
|
266
|
+
client.on(stopped, onBotStoppedSpeaking);
|
|
267
|
+
return () => {
|
|
268
|
+
client.off(serverMessage, onServerMessage);
|
|
269
|
+
client.off(started, onBotStartedSpeaking);
|
|
270
|
+
client.off(stopped, onBotStoppedSpeaking);
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
//# sourceMappingURL=AvatarClient.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AvatarClient.js","sourceRoot":"","sources":["../src/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqEG;AAIH,OAAO,EACL,mBAAmB,EACnB,eAAe,GAShB,MAAM,YAAY,CAAC;AA+CpB;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,aAAa,EAAE,eAAe;IAC9B,kBAAkB,EAAE,oBAAoB;IACxC,kBAAkB,EAAE,oBAAoB;CACC,CAAC;AAE5C;yEACyE;AACzE,SAAS,mBAAmB,CAAC,GAAY;IACvC,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,EAAE,CAA4B,CAAC;IACnD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAwC,CAAC;IACjE,OAAO,KAAK,IAAI,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;AAChD,CAAC;AAED,MAAM,OAAO,YAAY;IACN,MAAM,CAAY;IAClB,IAAI,CAAsB;IAC1B,GAAG,CAAe;IAClB,MAAM,CAAgD;IAC/D,IAAI,GAAgB,IAAI,CAAC;IAEjC,YAAY,MAAiB,EAAE,OAA4B,EAAE;QAC3D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,mBAAmB,CAAC,CAAC;IACvE,CAAC;IAED,+EAA+E;IAC/E,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,CAAC;IAChC,CAAC;IAED,wFAAwF;IACxF,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9C,CAAC;IAED;;;kFAG8E;IAC9E,QAAQ,CAAC,GAAY;QACnB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC;YAAE,OAAO;QAClC,MAAM,GAAG,GAAG,GAAG,CAAC;QAChB,IAAI,CAAC;YACH,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC;gBAChB,KAAK,OAAO;oBACV,IAAI,CAAC,WAAW,CAAC,GAAqB,CAAC,CAAC;oBACxC,MAAM;gBACR,KAAK,WAAW;oBACd,IAAI,CAAC,MAAM,CAAC,SAAS,CAAE,GAAsB,CAAC,EAAE,CAAC,CAAC;oBAClD,MAAM;gBACR,KAAK,SAAS;oBACZ,IAAI,CAAC,aAAa,CAAC,GAAuB,CAAC,CAAC;oBAC5C,MAAM;gBACR,KAAK,MAAM;oBACT,IAAI,CAAC,UAAU,CAAC,GAAoB,CAAC,CAAC;oBACtC,MAAM;gBACR,KAAK,QAAQ;oBACX,IAAI,CAAC,YAAY,CAAC,GAAsB,CAAC,CAAC;oBAC1C,MAAM;gBACR,KAAK,MAAM;oBACT,IAAI,CAAC,MAAM,CAAC,eAAe,CAAE,GAA6B,CAAC,QAAQ,CAAC,CAAC;oBACrE,MAAM;gBACR,KAAK,MAAM,CAAC,CAAC,CAAC;oBACZ,MAAM,IAAI,GAAG,GAAoB,CAAC;oBAClC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBACpC,MAAM;gBACR,CAAC;gBACD;oBACE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,GAAuB,CAAC,CAAC;oBAClD,MAAM;YACV,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;;gBAC9C,OAAO,CAAC,IAAI,CAAC,0BAA0B,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,GAAmB;QACrC,yEAAyE;QACzE,0EAA0E;QAC1E,4EAA4E;QAC5E,8DAA8D;QAC9D,2EAA2E;QAC3E,mCAAmC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC;IAEO,aAAa,CAAC,GAAqB;QACzC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED;;;;2CAIuC;IAC/B,YAAY,CAAC,GAAuB;QAC1C,IAAI,GAAG,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjE,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IAClC,CAAC;IAEO,UAAU,CAAC,GAAW;QAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC;YACxC,qEAAqE;YACrE,+DAA+D;YAC/D,wEAAwE;YACxE,YAAY;YACZ,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;QACvE,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAEO,UAAU,CAAC,GAAkB;QACnC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACjD,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAE7D,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,0EAA0E;YAC1E,sBAAsB;YACtB,OAAO;QACT,CAAC;QACD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,GAAoB;QACvC,IAAI,GAAG,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACpC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAC9C,OAAO;QACT,CAAC;QACD,4EAA4E;QAC5E,+DAA+D;QAC/D,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,EAAE,CAAC;YAC3C,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,KAAuB;QACzC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACzB,IAAI,EAAE,IAAI,IAAI;YAAE,OAAO;QACvB,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IAC9F,CAAC;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,MAAqB;QAC1B,MAAM,eAAe,GAAG,CAAC,GAAY,EAAE,EAAE;YACvC,MAAM,OAAO,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAAE,OAAO;YAClC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC,CAAC;QACF,MAAM,oBAAoB,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC7D,MAAM,oBAAoB,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAE5D,MAAM,aAAa,GAAG,WAAW,CAAC,aAA0B,CAAC;QAC7D,MAAM,OAAO,GAAG,WAAW,CAAC,kBAA+B,CAAC;QAC5D,MAAM,OAAO,GAAG,WAAW,CAAC,kBAA+B,CAAC;QAE5D,MAAM,CAAC,EAAE,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;QAC1C,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;QACzC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;QAEzC,OAAO,GAAG,EAAE;YACV,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;YAC3C,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;YAC1C,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;QAC5C,CAAC,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@voqalize/avatar/pipecat` — drive the widget from a pipecat session.
|
|
3
|
+
*
|
|
4
|
+
* Framework-free: everything here is plain TypeScript over the `AvatarApi` the
|
|
5
|
+
* root export returns. `AvatarClient` is the whole surface — construct it
|
|
6
|
+
* around a mounted widget and either `attach()` it to a live `PipecatClient`
|
|
7
|
+
* or feed it messages yourself with `dispatch()`.
|
|
8
|
+
*
|
|
9
|
+
* import { createAvatar } from "@voqalize/avatar";
|
|
10
|
+
* import { AvatarClient } from "@voqalize/avatar/pipecat";
|
|
11
|
+
*
|
|
12
|
+
* const avatar = createAvatar({ mount: "#tile" });
|
|
13
|
+
* const detach = new AvatarClient(avatar).attach(pipecatClient);
|
|
14
|
+
*
|
|
15
|
+
* `@pipecat-ai/client-js` is a peer dependency of this subpath only — the root
|
|
16
|
+
* export has no dependencies at all, and a host that carries avatar commands
|
|
17
|
+
* over its own transport can import this module and never call `attach()`.
|
|
18
|
+
*/
|
|
19
|
+
export { AvatarClient, type AvatarClientOptions } from "./AvatarClient.js";
|
|
20
|
+
export { isAvatarMessage, AVATAR_MESSAGE_TYPE, AVATAR_PROTOCOL_VERSION, type AvatarCommand, type AvatarCue, type AvatarCuesCmd, type AvatarHintCmd, type AvatarInterjectCmd, type AvatarPerformAction, type AvatarPerformCmd, type AvatarServerMessage, type AvatarSpeechCmd, type AvatarStateCmd, type AvatarUnknownCmd, type AvatarUserCmd, } from "./types.js";
|
|
21
|
+
//# sourceMappingURL=pipecat.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipecat.d.ts","sourceRoot":"","sources":["../src/pipecat.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAE3E,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,uBAAuB,EACvB,KAAK,aAAa,EAClB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,aAAa,GACnB,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@voqalize/avatar/pipecat` — drive the widget from a pipecat session.
|
|
3
|
+
*
|
|
4
|
+
* Framework-free: everything here is plain TypeScript over the `AvatarApi` the
|
|
5
|
+
* root export returns. `AvatarClient` is the whole surface — construct it
|
|
6
|
+
* around a mounted widget and either `attach()` it to a live `PipecatClient`
|
|
7
|
+
* or feed it messages yourself with `dispatch()`.
|
|
8
|
+
*
|
|
9
|
+
* import { createAvatar } from "@voqalize/avatar";
|
|
10
|
+
* import { AvatarClient } from "@voqalize/avatar/pipecat";
|
|
11
|
+
*
|
|
12
|
+
* const avatar = createAvatar({ mount: "#tile" });
|
|
13
|
+
* const detach = new AvatarClient(avatar).attach(pipecatClient);
|
|
14
|
+
*
|
|
15
|
+
* `@pipecat-ai/client-js` is a peer dependency of this subpath only — the root
|
|
16
|
+
* export has no dependencies at all, and a host that carries avatar commands
|
|
17
|
+
* over its own transport can import this module and never call `attach()`.
|
|
18
|
+
*/
|
|
19
|
+
export { AvatarClient } from "./AvatarClient.js";
|
|
20
|
+
export { isAvatarMessage, AVATAR_MESSAGE_TYPE, AVATAR_PROTOCOL_VERSION, } from "./types.js";
|
|
21
|
+
//# sourceMappingURL=pipecat.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipecat.js","sourceRoot":"","sources":["../src/pipecat.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,YAAY,EAA4B,MAAM,mBAAmB,CAAC;AAE3E,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,uBAAuB,GAaxB,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@voqalize/avatar/react` — the React binding.
|
|
3
|
+
*
|
|
4
|
+
* import { Avatar } from "@voqalize/avatar/react";
|
|
5
|
+
*
|
|
6
|
+
* <Avatar client={pipecatClient} className="tile" />
|
|
7
|
+
*
|
|
8
|
+
* Peers: `react >= 18` and `@pipecat-ai/client-js`. Everything a non-React
|
|
9
|
+
* host needs is in `@voqalize/avatar/pipecat`; this module adds a mount
|
|
10
|
+
* lifecycle and nothing else.
|
|
11
|
+
*/
|
|
12
|
+
export { useAvatar, type UseAvatarHandle, type UseAvatarOptions } from "./useAvatar.js";
|
|
13
|
+
export { Avatar, type AvatarProps } from "./Avatar.js";
|
|
14
|
+
export { AvatarClient, type AvatarClientOptions } from "./AvatarClient.js";
|
|
15
|
+
export { isAvatarMessage, AVATAR_MESSAGE_TYPE, AVATAR_PROTOCOL_VERSION, type AvatarCommand, type AvatarCue, type AvatarCuesCmd, type AvatarHintCmd, type AvatarInterjectCmd, type AvatarPerformAction, type AvatarPerformCmd, type AvatarServerMessage, type AvatarSpeechCmd, type AvatarStateCmd, type AvatarUnknownCmd, type AvatarUserCmd, } from "./types.js";
|
|
16
|
+
//# sourceMappingURL=react.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"react.d.ts","sourceRoot":"","sources":["../src/react.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,eAAe,EAAE,KAAK,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACxF,OAAO,EAAE,MAAM,EAAE,KAAK,WAAW,EAAE,MAAM,aAAa,CAAC;AAGvD,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,uBAAuB,EACvB,KAAK,aAAa,EAClB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,aAAa,GACnB,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@voqalize/avatar/react` — the React binding.
|
|
3
|
+
*
|
|
4
|
+
* import { Avatar } from "@voqalize/avatar/react";
|
|
5
|
+
*
|
|
6
|
+
* <Avatar client={pipecatClient} className="tile" />
|
|
7
|
+
*
|
|
8
|
+
* Peers: `react >= 18` and `@pipecat-ai/client-js`. Everything a non-React
|
|
9
|
+
* host needs is in `@voqalize/avatar/pipecat`; this module adds a mount
|
|
10
|
+
* lifecycle and nothing else.
|
|
11
|
+
*/
|
|
12
|
+
export { useAvatar } from "./useAvatar.js";
|
|
13
|
+
export { Avatar } from "./Avatar.js";
|
|
14
|
+
// Re-exported so a React consumer needs one import for the common case.
|
|
15
|
+
export { AvatarClient } from "./AvatarClient.js";
|
|
16
|
+
export { isAvatarMessage, AVATAR_MESSAGE_TYPE, AVATAR_PROTOCOL_VERSION, } from "./types.js";
|
|
17
|
+
//# sourceMappingURL=react.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"react.js","sourceRoot":"","sources":["../src/react.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,SAAS,EAA+C,MAAM,gBAAgB,CAAC;AACxF,OAAO,EAAE,MAAM,EAAoB,MAAM,aAAa,CAAC;AAEvD,wEAAwE;AACxE,OAAO,EAAE,YAAY,EAA4B,MAAM,mBAAmB,CAAC;AAC3E,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,uBAAuB,GAaxB,MAAM,YAAY,CAAC"}
|