@voqalize/avatar 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/LICENSE +21 -661
  2. package/README.md +144 -663
  3. package/client/Avatar.tsx +57 -0
  4. package/client/AvatarClient.ts +523 -0
  5. package/client/arjun.ts +24 -0
  6. package/client/createAvatar.ts +139 -0
  7. package/client/createCanvasAvatar.ts +72 -0
  8. package/client/index.ts +39 -0
  9. package/client/internal.ts +80 -0
  10. package/client/interviewer-female.ts +4 -0
  11. package/client/interviewer-male.ts +4 -0
  12. package/client/ishita.ts +24 -0
  13. package/client/kabir.ts +24 -0
  14. package/client/meera.ts +24 -0
  15. package/client/naina.ts +24 -0
  16. package/client/professional-female-a.ts +4 -0
  17. package/client/professional-female-b.ts +4 -0
  18. package/client/professional-male-a.ts +4 -0
  19. package/client/professional-male-b.ts +4 -0
  20. package/client/react.ts +13 -0
  21. package/client/types.ts +146 -0
  22. package/client/useAvatar.ts +74 -0
  23. package/client/vikram.ts +24 -0
  24. package/dist/Avatar.d.ts +44 -0
  25. package/dist/Avatar.d.ts.map +1 -0
  26. package/{client/dist → dist}/Avatar.js +2 -2
  27. package/dist/Avatar.js.map +1 -0
  28. package/dist/AvatarClient.d.ts +196 -0
  29. package/dist/AvatarClient.d.ts.map +1 -0
  30. package/dist/AvatarClient.js +450 -0
  31. package/dist/AvatarClient.js.map +1 -0
  32. package/dist/arjun.d.ts +6 -0
  33. package/dist/arjun.d.ts.map +1 -0
  34. package/dist/arjun.js +18 -0
  35. package/dist/arjun.js.map +1 -0
  36. package/dist/createAvatar.d.ts +99 -0
  37. package/dist/createAvatar.d.ts.map +1 -0
  38. package/dist/createAvatar.js +75 -0
  39. package/dist/createAvatar.js.map +1 -0
  40. package/dist/createCanvasAvatar.d.ts +22 -0
  41. package/dist/createCanvasAvatar.d.ts.map +1 -0
  42. package/dist/createCanvasAvatar.js +47 -0
  43. package/dist/createCanvasAvatar.js.map +1 -0
  44. package/dist/index.d.ts +30 -0
  45. package/dist/index.d.ts.map +1 -0
  46. package/dist/index.js +29 -0
  47. package/dist/index.js.map +1 -0
  48. package/dist/internal.d.ts +30 -0
  49. package/dist/internal.d.ts.map +1 -0
  50. package/dist/internal.js +33 -0
  51. package/dist/internal.js.map +1 -0
  52. package/dist/interviewer-female.d.ts +4 -0
  53. package/dist/interviewer-female.d.ts.map +1 -0
  54. package/dist/interviewer-female.js +3 -0
  55. package/dist/interviewer-female.js.map +1 -0
  56. package/dist/interviewer-male.d.ts +4 -0
  57. package/dist/interviewer-male.d.ts.map +1 -0
  58. package/dist/interviewer-male.js +3 -0
  59. package/dist/interviewer-male.js.map +1 -0
  60. package/dist/ishita.d.ts +6 -0
  61. package/dist/ishita.d.ts.map +1 -0
  62. package/dist/ishita.js +18 -0
  63. package/dist/ishita.js.map +1 -0
  64. package/dist/kabir.d.ts +6 -0
  65. package/dist/kabir.d.ts.map +1 -0
  66. package/dist/kabir.js +18 -0
  67. package/dist/kabir.js.map +1 -0
  68. package/dist/meera.d.ts +6 -0
  69. package/dist/meera.d.ts.map +1 -0
  70. package/dist/meera.js +18 -0
  71. package/dist/meera.js.map +1 -0
  72. package/dist/naina.d.ts +6 -0
  73. package/dist/naina.d.ts.map +1 -0
  74. package/dist/naina.js +18 -0
  75. package/dist/naina.js.map +1 -0
  76. package/dist/professional-female-a.d.ts +4 -0
  77. package/dist/professional-female-a.d.ts.map +1 -0
  78. package/dist/professional-female-a.js +3 -0
  79. package/dist/professional-female-a.js.map +1 -0
  80. package/dist/professional-female-b.d.ts +4 -0
  81. package/dist/professional-female-b.d.ts.map +1 -0
  82. package/dist/professional-female-b.js +3 -0
  83. package/dist/professional-female-b.js.map +1 -0
  84. package/dist/professional-male-a.d.ts +4 -0
  85. package/dist/professional-male-a.d.ts.map +1 -0
  86. package/dist/professional-male-a.js +3 -0
  87. package/dist/professional-male-a.js.map +1 -0
  88. package/dist/professional-male-b.d.ts +4 -0
  89. package/dist/professional-male-b.d.ts.map +1 -0
  90. package/dist/professional-male-b.js +3 -0
  91. package/dist/professional-male-b.js.map +1 -0
  92. package/dist/react.d.ts +13 -0
  93. package/dist/react.d.ts.map +1 -0
  94. package/dist/react.js +13 -0
  95. package/dist/react.js.map +1 -0
  96. package/dist/types.d.ts +94 -0
  97. package/dist/types.d.ts.map +1 -0
  98. package/dist/types.js +93 -0
  99. package/dist/types.js.map +1 -0
  100. package/dist/useAvatar.d.ts +39 -0
  101. package/dist/useAvatar.d.ts.map +1 -0
  102. package/dist/useAvatar.js +35 -0
  103. package/dist/useAvatar.js.map +1 -0
  104. package/dist/vikram.d.ts +6 -0
  105. package/dist/vikram.d.ts.map +1 -0
  106. package/dist/vikram.js +18 -0
  107. package/dist/vikram.js.map +1 -0
  108. package/package.json +81 -24
  109. package/src/avatar.d.ts +161 -120
  110. package/src/avatar.js +234 -146
  111. package/src/behavior.d.ts +26 -0
  112. package/src/behavior.js +102 -0
  113. package/src/camera.js +29 -0
  114. package/src/canvas/author/parts/eye.mjs +722 -0
  115. package/src/canvas/author/parts/hand.mjs +1156 -0
  116. package/src/canvas/author/parts/mouth.mjs +741 -0
  117. package/src/canvas/author/parts/nose.mjs +100 -0
  118. package/src/canvas/author/parts/skin-detail.mjs +67 -0
  119. package/src/canvas/author/path.mjs +283 -0
  120. package/src/canvas/author/rig.mjs +405 -0
  121. package/src/canvas/avatars/round/face.d.mts +3 -0
  122. package/src/canvas/avatars/round/face.mjs +1307 -0
  123. package/src/canvas/create-rig.d.ts +15 -0
  124. package/src/canvas/create-rig.js +100 -0
  125. package/src/canvas/data/img/professional-female-a-hair-back.webp +0 -0
  126. package/src/canvas/data/img/professional-female-a-hair-front.webp +0 -0
  127. package/src/canvas/data/img/professional-female-a-top-body.webp +0 -0
  128. package/src/canvas/data/img/professional-female-b-hair-back.webp +0 -0
  129. package/src/canvas/data/img/professional-female-b-hair-front.webp +0 -0
  130. package/src/canvas/data/img/professional-female-b-top-body.webp +0 -0
  131. package/src/canvas/data/img/professional-male-a-hair-back.webp +0 -0
  132. package/src/canvas/data/img/professional-male-a-hair-front.webp +0 -0
  133. package/src/canvas/data/img/professional-male-a-top-body.webp +0 -0
  134. package/src/canvas/data/img/professional-male-b-hair-back.webp +0 -0
  135. package/src/canvas/data/img/professional-male-b-hair-front.webp +0 -0
  136. package/src/canvas/data/img/professional-male-b-top-body.webp +0 -0
  137. package/src/canvas/data/img/round-m3-hair-back.webp +0 -0
  138. package/src/canvas/data/img/round-m3-hair-front.webp +0 -0
  139. package/src/canvas/data/img/round-m3-top-body.webp +0 -0
  140. package/src/canvas/data/img/round-w1-hair-back.webp +0 -0
  141. package/src/canvas/data/img/round-w1-hair-front.webp +0 -0
  142. package/src/canvas/data/img/round-w1-top-body.webp +0 -0
  143. package/src/canvas/data/interviewer-female.rig.json +1 -0
  144. package/src/canvas/data/interviewer-male.rig.json +1 -0
  145. package/src/canvas/data/professional-female-a.rig.json +1 -0
  146. package/src/canvas/data/professional-female-b.rig.json +1 -0
  147. package/src/canvas/data/professional-male-a.rig.json +1 -0
  148. package/src/canvas/data/professional-male-b.rig.json +1 -0
  149. package/src/canvas/src/live.js +508 -0
  150. package/src/canvas/src/render2d.js +218 -0
  151. package/src/canvas/src/rig.js +297 -0
  152. package/src/canvas/src/vocab.js +96 -0
  153. package/src/clips.js +18 -7
  154. package/src/conformance.js +119 -0
  155. package/src/emotions.js +4 -3
  156. package/src/face-core.js +27 -1
  157. package/src/face-myna.d.ts +7 -0
  158. package/src/face-myna.js +192 -131
  159. package/src/face-peep-control-plane.js +167 -0
  160. package/src/face-peep.d.ts +7 -0
  161. package/src/face-peep.js +274 -116
  162. package/src/face-wren.d.ts +7 -0
  163. package/src/face-wren.js +17 -19
  164. package/src/faces.d.ts +12 -0
  165. package/src/faces.js +53 -0
  166. package/src/gaze.js +119 -4
  167. package/src/hand.js +120 -91
  168. package/src/idle.js +18 -101
  169. package/src/interjections.js +156 -31
  170. package/src/params.js +8 -4
  171. package/src/perform.js +7 -9
  172. package/src/rig.d.ts +24 -0
  173. package/src/rig.js +32 -0
  174. package/src/speech-timing.js +23 -0
  175. package/src/visemes.js +28 -12
  176. package/client/dist/Avatar.d.ts +0 -27
  177. package/client/dist/Avatar.d.ts.map +0 -1
  178. package/client/dist/Avatar.js.map +0 -1
  179. package/client/dist/AvatarClient.d.ts +0 -139
  180. package/client/dist/AvatarClient.d.ts.map +0 -1
  181. package/client/dist/AvatarClient.js +0 -223
  182. package/client/dist/AvatarClient.js.map +0 -1
  183. package/client/dist/index.d.ts +0 -22
  184. package/client/dist/index.d.ts.map +0 -1
  185. package/client/dist/index.js +0 -22
  186. package/client/dist/index.js.map +0 -1
  187. package/client/dist/types.d.ts +0 -86
  188. package/client/dist/types.d.ts.map +0 -1
  189. package/client/dist/types.js +0 -31
  190. package/client/dist/types.js.map +0 -1
  191. package/client/dist/useAvatar.d.ts +0 -42
  192. package/client/dist/useAvatar.d.ts.map +0 -1
  193. package/client/dist/useAvatar.js +0 -51
  194. package/client/dist/useAvatar.js.map +0 -1
  195. package/client/src/Avatar.tsx +0 -33
  196. package/client/src/AvatarClient.ts +0 -270
  197. package/client/src/index.ts +0 -22
  198. package/client/src/types.ts +0 -104
  199. package/client/src/useAvatar.ts +0 -85
  200. package/docs/contract-avatar.md +0 -371
  201. package/docs/contract-protocol.md +0 -461
@@ -0,0 +1,450 @@
1
+ /**
2
+ * AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
3
+ * splice, framework-free (no React; the hook and component wrap this).
4
+ *
5
+ * ## Turn clock anchoring
6
+ *
7
+ * Base Pipecat TTS gives each serialized TTS context an opaque `context_id`.
8
+ * The server uses it only to group and splice cue chunks. `botStartedSpeaking`
9
+ * has no context payload, so the browser FIFO-claims the next buffered context
10
+ * at that Pipecat output-lifecycle event and anchors its clock there.
11
+ * `botStoppedSpeaking` closes the active context. Neither event observes the
12
+ * browser's audio device, and no avatar-specific speech marker exists.
13
+ *
14
+ * `attach()` subscribes to the avatar server-message channel *and* Pipecat's
15
+ * standard lifecycle events. Server messages carry only what Pipecat cannot:
16
+ * correlated speech/cue timing and deliberate application instructions. The
17
+ * lifecycle events project the factual presence states locally. The server
18
+ * supplies only lower-priority `THINKING` / `WORKING` claims and deliberate,
19
+ * self-completing actions. This keeps the face tied to Pipecat's output truth
20
+ * even if a server claim is delayed or stale.
21
+ *
22
+ * ## Cue splice
23
+ *
24
+ * The widget has two cue-track primitives: `speak({cues, clock})` (a full
25
+ * replace) and `pushCues(cues)` (a pure union that can only grow the track,
26
+ * never shrink it). Neither is "discard queued cues at or after `from_ms`,
27
+ * then append" on its own — `pushCues` has no way to drop a stale tail. So
28
+ * this client keeps the turn's canonical cue array itself (kept portion +
29
+ * every appended chunk, spliced on each `cues` message) and picks the cheapest
30
+ * widget call that stays correct:
31
+ *
32
+ * - if the splice's `from_ms` doesn't reach back into anything already
33
+ * queued — the common case past a turn's first sentence, since only the
34
+ * first sentence genuinely plays fast-leg cues — nothing needs discarding:
35
+ * `pushCues(newCues)` is the cheap, correct append.
36
+ * - if it does reach back (a real fast→accurate splice), `pushCues` cannot
37
+ * express the discard; we call `speak()` again with the full spliced
38
+ * canonical array on the turn's original clock. `speak()` is otherwise
39
+ * documented as also killing an in-flight spoken interjection and
40
+ * re-entering `SPEAKING` — both harmless mid-splice (an interjection
41
+ * should not be running while a server track owns the mouth; re-entering
42
+ * an unchanged state is a no-op past the profile/gaze reset the widget
43
+ * already does for a same-name `setState`).
44
+ *
45
+ * Cues commonly arrive **before** `botStartedSpeaking` — the fast leg starts
46
+ * the moment a sentence is handed to TTS, well before bot output begins.
47
+ * Chunks that arrive before the clock is anchored are spliced into the
48
+ * canonical array but not yet handed to the widget; `botStartedSpeaking` hands over
49
+ * whatever has accumulated as the turn's first `speak()` call. So "the first
50
+ * chunk of a turn starts speak()" means the first *widget* call, not
51
+ * necessarily the first *message*.
52
+ */
53
+ import { BehaviorController } from "../src/behavior.js";
54
+ import { isAvatarMessage, parseAvatarCommand, } from "./types.js";
55
+ /**
56
+ * The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
57
+ *
58
+ * Written out rather than imported because that enum was this module's *only*
59
+ * runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
60
+ * the whole `/pipecat` subpath fail to load without the peer installed — even
61
+ * for a host that drives `dispatch()` from its own transport and never calls
62
+ * `attach()`. The peer is declared optional; this is what makes that true
63
+ * rather than aspirational.
64
+ *
65
+ * String enums are nominal in TypeScript, so the compiler cannot check these
66
+ * against the real ones from a type-only import. `packages/avatar/test/AvatarClient.test.ts`
67
+ * does it instead, against the actual enum — the devDependency is present
68
+ * exactly where the check belongs and absent from what we ship.
69
+ */
70
+ export const RTVI_EVENTS = {
71
+ serverMessage: "serverMessage",
72
+ connected: "connected",
73
+ disconnected: "disconnected",
74
+ botReady: "botReady",
75
+ error: "error",
76
+ userStartedSpeaking: "userStartedSpeaking",
77
+ userStoppedSpeaking: "userStoppedSpeaking",
78
+ botStartedSpeaking: "botStartedSpeaking",
79
+ botStoppedSpeaking: "botStoppedSpeaking",
80
+ // Mute is a Pipecat fact, not a claim: the server's mute strategy emits
81
+ // `UserMuteStarted/StoppedFrame`, the RTVI observer forwards them, and the
82
+ // browser client raises these. So "has muted you" costs no wire verb —
83
+ // reading the events the peer already sends is exactly the authority model.
84
+ userMuteStarted: "userMuteStarted",
85
+ userMuteStopped: "userMuteStopped",
86
+ };
87
+ /** Defensive unwrap for the `RTVIEvent.ServerMessage` `{ data }` quirk: some
88
+ * transports deliver the payload directly and some wrap it once more. */
89
+ function unwrapServerMessage(raw) {
90
+ const obj = (raw ?? {});
91
+ const inner = obj["data"];
92
+ return inner && "type" in inner ? inner : obj;
93
+ }
94
+ export class AvatarClient {
95
+ avatar;
96
+ /** Maps factual/wire intent into the broader client behavior catalog. */
97
+ behavior;
98
+ opts;
99
+ now;
100
+ turn = null;
101
+ turns = new Map();
102
+ pendingCtxs = [];
103
+ closedCtxs = new Set();
104
+ projected = null;
105
+ serverClaim = null;
106
+ userSpeaking = false;
107
+ botSpeaking = false;
108
+ muted = false;
109
+ listening = false;
110
+ // Idle is earned after a connected, quiet listening interval. A newly
111
+ // mounted avatar is available, not already "stepped aside".
112
+ idle = false;
113
+ failure = null;
114
+ idleTimer = null;
115
+ pendingInterruptedAction = false;
116
+ discardQueuedContextsOnBotStop = false;
117
+ idleDelayMs;
118
+ setTimer;
119
+ clearTimer;
120
+ constructor(avatar, opts = {}) {
121
+ this.avatar = avatar;
122
+ this.behavior = new BehaviorController(avatar);
123
+ this.opts = opts;
124
+ this.now = opts.now ?? (() => performance.now());
125
+ this.idleDelayMs = opts.idleDelayMs ?? 12_000;
126
+ this.setTimer = opts.setTimeout ?? globalThis.setTimeout.bind(globalThis);
127
+ this.clearTimer = opts.clearTimeout ?? globalThis.clearTimeout.bind(globalThis);
128
+ }
129
+ /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
130
+ get turnCtx() {
131
+ return this.turn?.ctx ?? null;
132
+ }
133
+ /** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
134
+ get turnCues() {
135
+ return this.turn ? [...this.turn.cues] : [];
136
+ }
137
+ /** Current resolved projection. Internal — for tests and Studio's inspector.
138
+ * Before the first lifecycle fact the renderer is simply at its ready rest
139
+ * pose, so `LISTENING` is the safe value; it is not a session status. */
140
+ get presenceState() {
141
+ return this.projected ?? "LISTENING";
142
+ }
143
+ /** Re-apply the currently resolved factual projection after an embedding
144
+ * tool has temporarily used raw renderer controls. This does not invent a
145
+ * lifecycle event or cancel a finite action. */
146
+ restoreProjection() {
147
+ this.applyProjection(true);
148
+ }
149
+ /** Dispatch one server message. Anything that isn't in the avatar envelope
150
+ * is not ours and is ignored; so is an envelope carrying a `cmd` this build
151
+ * has never heard of, per the wire protocol's forward-compat rule. */
152
+ dispatch(raw) {
153
+ if (!isAvatarMessage(raw))
154
+ return;
155
+ const msg = parseAvatarCommand(raw);
156
+ if (!msg)
157
+ return;
158
+ try {
159
+ switch (msg.cmd) {
160
+ case "claim":
161
+ this.handleClaim(msg.state);
162
+ break;
163
+ case "action":
164
+ this.handleAction(msg.id);
165
+ break;
166
+ case "cues":
167
+ this.handleCues(msg);
168
+ break;
169
+ // No default: an unknown `cmd` is a newer server talking to an older
170
+ // widget, and the protocol's forward-compat rule says ignore it. There
171
+ // is no callback for it — a hook nobody could act on is observability,
172
+ // not an interface.
173
+ }
174
+ }
175
+ catch (err) {
176
+ if (this.opts.onError)
177
+ this.opts.onError(err, msg);
178
+ else
179
+ console.warn("[avatar] dispatch failed", msg, err);
180
+ }
181
+ }
182
+ handleClaim(state) {
183
+ this.serverClaim = state;
184
+ if (state)
185
+ this.idle = false;
186
+ this.applyProjection();
187
+ this.armIdleIfEligible();
188
+ }
189
+ handleAction(id) {
190
+ // An interruption is a server-confirmed explanation of a transition, not
191
+ // authority to steal the mouth while bot audio is still playing. Hold it
192
+ // until Pipecat output has released the speaking state.
193
+ if (id === "RESPONSE_INTERRUPTED" && this.botSpeaking) {
194
+ this.pendingInterruptedAction = true;
195
+ // Any prefetched TTS contexts behind interrupted output belong to
196
+ // audio Pipecat will now discard. Never let one animate a later reply.
197
+ this.discardQueuedContextsOnBotStop = true;
198
+ return;
199
+ }
200
+ this.playAction(id);
201
+ }
202
+ playAction(id) {
203
+ this.behavior.wireAction(id);
204
+ }
205
+ ensureTurn(ctx) {
206
+ const existing = this.turns.get(ctx);
207
+ if (existing)
208
+ return existing;
209
+ const turn = { ctx, cues: [], started: false, clock: null };
210
+ this.turns.set(ctx, turn);
211
+ this.pendingCtxs.push(ctx);
212
+ return turn;
213
+ }
214
+ handleCues(msg) {
215
+ if (this.closedCtxs.has(msg.ctx))
216
+ return;
217
+ const turn = this.ensureTurn(msg.ctx);
218
+ const kept = turn.cues.filter((c) => c.t < msg.from_ms);
219
+ const discarded = turn.cues.length - kept.length;
220
+ turn.cues = [...kept, ...msg.cues].sort((a, b) => a.t - b.t);
221
+ if (!turn.started) {
222
+ // No clock yet — buffer. Pipecat output will claim this FIFO context.
223
+ if (this.botSpeaking && this.turn === null)
224
+ this.activateNextTurn();
225
+ return;
226
+ }
227
+ if (discarded === 0) {
228
+ this.avatar.pushCues(msg.cues);
229
+ }
230
+ else {
231
+ this.avatar.speak({ cues: turn.cues, clock: turn.clock });
232
+ }
233
+ }
234
+ activateNextTurn() {
235
+ if (this.turn)
236
+ return;
237
+ let turn;
238
+ while (this.pendingCtxs.length && !turn) {
239
+ turn = this.turns.get(this.pendingCtxs.shift());
240
+ }
241
+ if (!turn)
242
+ return;
243
+ const t0 = this.now();
244
+ turn.clock = () => this.now() - t0;
245
+ turn.started = true;
246
+ this.turn = turn;
247
+ this.avatar.speak({ cues: turn.cues, clock: turn.clock });
248
+ }
249
+ discardQueuedTurns() {
250
+ for (const ctx of this.pendingCtxs.splice(0)) {
251
+ this.closedCtxs.add(ctx);
252
+ this.turns.delete(ctx);
253
+ }
254
+ }
255
+ lifecycleState() {
256
+ // The one copy of the ladder is docs/pipecat-lifecycle-protocol.md; this is
257
+ // its implementation, in the same order.
258
+ //
259
+ // Audio truth is the P0 invariant: no lower claim or microphone event may
260
+ // put the face in a non-speaking pose while bot speech is audible.
261
+ if (this.botSpeaking)
262
+ return "SPEAKING";
263
+ if (this.userSpeaking)
264
+ return "LISTENING";
265
+ // A broken session outranks mute, because a mute nobody can hear about is
266
+ // not the thing worth saying about a dead call.
267
+ if (this.failure)
268
+ return this.failure;
269
+ if (this.muted)
270
+ return "MUTED";
271
+ // Every claim resolves to the presence state of the same name, so there is
272
+ // nothing to rank here — a claim is a single value and only one can be in
273
+ // flight. STRAINING > THINKING > WORKING is decided where more than one
274
+ // condition can hold at once, which is the server: `AvatarStateMachine.
275
+ // _resolve()`. Ranking them again on this side would be a second, silent
276
+ // copy of that ladder, and the two would drift.
277
+ if (this.serverClaim)
278
+ return this.serverClaim;
279
+ if (this.idle)
280
+ return "IDLE";
281
+ return "LISTENING";
282
+ }
283
+ applyProjection(force = false) {
284
+ const state = this.lifecycleState();
285
+ if (force || this.projected !== state) {
286
+ this.behavior.setState(state);
287
+ this.projected = state;
288
+ this.opts.onPresenceChange?.(state);
289
+ }
290
+ }
291
+ clearIdleTimer() {
292
+ if (this.idleTimer !== null) {
293
+ this.clearTimer(this.idleTimer);
294
+ this.idleTimer = null;
295
+ }
296
+ }
297
+ armIdleIfEligible() {
298
+ this.clearIdleTimer();
299
+ // Quiet under a mute is not the quiet that earns IDLE — the silence was
300
+ // imposed, and letting the timer run behind it would reveal a stepped-aside
301
+ // face the moment the microphone came back.
302
+ if (!this.eligibleForIdle())
303
+ return;
304
+ this.idleTimer = this.setTimer(() => {
305
+ this.idleTimer = null;
306
+ if (this.eligibleForIdle()) {
307
+ this.idle = true;
308
+ this.applyProjection();
309
+ }
310
+ }, this.idleDelayMs);
311
+ }
312
+ eligibleForIdle() {
313
+ return this.listening && !this.userSpeaking && !this.botSpeaking
314
+ && !this.muted && !this.serverClaim && !this.failure;
315
+ }
316
+ enterListening() {
317
+ this.listening = true;
318
+ this.idle = false;
319
+ this.applyProjection();
320
+ this.armIdleIfEligible();
321
+ }
322
+ clearClaimForTurnBoundary() {
323
+ // Claims are lower-priority hints. A fresh user turn or bot output means
324
+ // any prior thinking/work claim is no longer allowed to reappear later.
325
+ this.serverClaim = null;
326
+ }
327
+ maybePlayInterrupted() {
328
+ if (this.pendingInterruptedAction && !this.botSpeaking) {
329
+ this.pendingInterruptedAction = false;
330
+ this.playAction("RESPONSE_INTERRUPTED");
331
+ }
332
+ }
333
+ clearRecoverableFailure() {
334
+ if (this.failure === "DEGRADED") {
335
+ this.failure = null;
336
+ this.applyProjection();
337
+ }
338
+ }
339
+ onUserStartedSpeaking = () => {
340
+ this.clearRecoverableFailure();
341
+ this.userSpeaking = true;
342
+ this.clearClaimForTurnBoundary();
343
+ this.listening = true;
344
+ this.idle = false;
345
+ this.clearIdleTimer();
346
+ this.avatar.setUserSpeaking(true);
347
+ this.applyProjection();
348
+ };
349
+ onUserStoppedSpeaking = () => {
350
+ this.userSpeaking = false;
351
+ this.avatar.setUserSpeaking(false);
352
+ this.enterListening();
353
+ };
354
+ onBotStartedSpeaking = () => {
355
+ this.clearRecoverableFailure();
356
+ this.botSpeaking = true;
357
+ this.clearClaimForTurnBoundary();
358
+ this.idle = false;
359
+ this.clearIdleTimer();
360
+ this.applyProjection();
361
+ this.activateNextTurn();
362
+ };
363
+ onBotStoppedSpeaking = () => {
364
+ this.botSpeaking = false;
365
+ // Playout truth releases the only active mouth track. A late cue chunk for
366
+ // this context is ignored rather than reviving a silent mouth.
367
+ if (this.turn) {
368
+ this.avatar.stopSpeaking();
369
+ this.closedCtxs.add(this.turn.ctx);
370
+ this.turns.delete(this.turn.ctx);
371
+ this.turn = null;
372
+ }
373
+ if (this.discardQueuedContextsOnBotStop) {
374
+ this.discardQueuedContextsOnBotStop = false;
375
+ this.discardQueuedTurns();
376
+ }
377
+ this.enterListening();
378
+ this.maybePlayInterrupted();
379
+ };
380
+ onUserMuteStarted = () => {
381
+ this.muted = true;
382
+ this.idle = false;
383
+ this.clearIdleTimer();
384
+ this.applyProjection();
385
+ };
386
+ onUserMuteStopped = () => {
387
+ this.muted = false;
388
+ // Whatever the avatar was waiting on before the mute, it is waiting on the
389
+ // user again now — the same place a turn ends.
390
+ this.enterListening();
391
+ };
392
+ onError = (raw) => {
393
+ const data = raw?.data;
394
+ this.failure = data?.fatal === true ? "OFFLINE" : "DEGRADED";
395
+ this.applyProjection();
396
+ };
397
+ onDisconnected = () => {
398
+ this.failure = "OFFLINE";
399
+ this.clearIdleTimer();
400
+ this.applyProjection();
401
+ };
402
+ onConnectedOrReady = () => {
403
+ if (this.failure === "OFFLINE")
404
+ this.failure = null;
405
+ // Pipecat has established a session. Until a factual speech event says
406
+ // otherwise, the avatar is available to listen; it earns IDLE only after
407
+ // the regular quiet timer expires.
408
+ this.listening = true;
409
+ this.idle = false;
410
+ this.applyProjection();
411
+ this.armIdleIfEligible();
412
+ };
413
+ /**
414
+ * Subscribe to a live `PipecatClient`. Standard client events own the normal
415
+ * lifecycle projection; avatar server-messages carry correlated visemes and
416
+ * explicit application intent. Never throws on malformed or irrelevant
417
+ * server messages.
418
+ *
419
+ * @returns an unsubscribe function; call it on unmount or disconnect.
420
+ */
421
+ attach(client) {
422
+ const onServerMessage = (raw) => this.dispatch(unwrapServerMessage(raw));
423
+ const subscriptions = [
424
+ [RTVI_EVENTS.serverMessage, onServerMessage],
425
+ [RTVI_EVENTS.connected, this.onConnectedOrReady],
426
+ [RTVI_EVENTS.botReady, this.onConnectedOrReady],
427
+ [RTVI_EVENTS.disconnected, this.onDisconnected],
428
+ [RTVI_EVENTS.error, this.onError],
429
+ [RTVI_EVENTS.userStartedSpeaking, this.onUserStartedSpeaking],
430
+ [RTVI_EVENTS.userStoppedSpeaking, this.onUserStoppedSpeaking],
431
+ [RTVI_EVENTS.botStartedSpeaking, this.onBotStartedSpeaking],
432
+ [RTVI_EVENTS.botStoppedSpeaking, this.onBotStoppedSpeaking],
433
+ [RTVI_EVENTS.userMuteStarted, this.onUserMuteStarted],
434
+ [RTVI_EVENTS.userMuteStopped, this.onUserMuteStopped],
435
+ ];
436
+ for (const [event, listener] of subscriptions)
437
+ client.on(event, listener);
438
+ return () => {
439
+ this.clearIdleTimer();
440
+ for (const [event, listener] of subscriptions)
441
+ client.off(event, listener);
442
+ };
443
+ }
444
+ /** Dispose controller-owned timers when its mounted avatar is destroyed. */
445
+ destroy() {
446
+ this.clearIdleTimer();
447
+ this.behavior.destroy();
448
+ }
449
+ }
450
+ //# sourceMappingURL=AvatarClient.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AvatarClient.js","sourceRoot":"","sources":["../client/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAIH,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,eAAe,EACf,kBAAkB,GAInB,MAAM,YAAY,CAAC;AA6CpB;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,aAAa,EAAE,eAAe;IAC9B,SAAS,EAAE,WAAW;IACtB,YAAY,EAAE,cAAc;IAC5B,QAAQ,EAAE,UAAU;IACpB,KAAK,EAAE,OAAO;IACd,mBAAmB,EAAE,qBAAqB;IAC1C,mBAAmB,EAAE,qBAAqB;IAC1C,kBAAkB,EAAE,oBAAoB;IACxC,kBAAkB,EAAE,oBAAoB;IACxC,wEAAwE;IACxE,2EAA2E;IAC3E,uEAAuE;IACvE,4EAA4E;IAC5E,eAAe,EAAE,iBAAiB;IAClC,eAAe,EAAE,iBAAiB;CACO,CAAC;AAS5C;yEACyE;AACzE,SAAS,mBAAmB,CAAC,GAAY;IACvC,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,EAAE,CAA4B,CAAC;IACnD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAwC,CAAC;IACjE,OAAO,KAAK,IAAI,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;AAChD,CAAC;AAED,MAAM,OAAO,YAAY;IACN,MAAM,CAAY;IACnC,yEAAyE;IACxD,QAAQ,CAAqB;IAC7B,IAAI,CAAsB;IAC1B,GAAG,CAAe;IAC3B,IAAI,GAAgB,IAAI,CAAC;IAChB,KAAK,GAAG,IAAI,GAAG,EAAgB,CAAC;IAChC,WAAW,GAAa,EAAE,CAAC;IAC3B,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACxC,SAAS,GAA0B,IAAI,CAAC;IACxC,WAAW,GAAgB,IAAI,CAAC;IAChC,YAAY,GAAG,KAAK,CAAC;IACrB,WAAW,GAAG,KAAK,CAAC;IACpB,KAAK,GAAG,KAAK,CAAC;IACd,SAAS,GAAG,KAAK,CAAC;IAC1B,sEAAsE;IACtE,4DAA4D;IACpD,IAAI,GAAG,KAAK,CAAC;IACb,OAAO,GAAkC,IAAI,CAAC;IAC9C,SAAS,GAAyC,IAAI,CAAC;IACvD,wBAAwB,GAAG,KAAK,CAAC;IACjC,8BAA8B,GAAG,KAAK,CAAC;IAC9B,WAAW,CAAS;IACpB,QAAQ,CAAoB;IAC5B,UAAU,CAAsB;IAEjD,YAAY,MAAiB,EAAE,OAA4B,EAAE;QAC3D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC;QAC9C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1E,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,YAAY,IAAI,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAClF,CAAC;IAED,+EAA+E;IAC/E,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,CAAC;IAChC,CAAC;IAED,wFAAwF;IACxF,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9C,CAAC;IAED;;6EAEyE;IACzE,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,SAAS,IAAI,WAAW,CAAC;IACvC,CAAC;IAED;;oDAEgD;IAChD,iBAAiB;QACf,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED;;0EAEsE;IACtE,QAAQ,CAAC,GAAY;QACnB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC;YAAE,OAAO;QAClC,MAAM,GAAG,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC;YACH,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC;gBAChB,KAAK,OAAO;oBACV,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAC5B,MAAM;gBACR,KAAK,QAAQ;oBACX,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC1B,MAAM;gBACR,KAAK,MAAM;oBACT,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;oBACrB,MAAM;gBACR,qEAAqE;gBACrE,uEAAuE;gBACvE,uEAAuE;gBACvE,oBAAoB;YACtB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;;gBAC9C,OAAO,CAAC,IAAI,CAAC,0BAA0B,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,KAAkB;QACpC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QACzB,IAAI,KAAK;YAAE,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEO,YAAY,CAAC,EAAkB;QACrC,yEAAyE;QACzE,yEAAyE;QACzE,wDAAwD;QACxD,IAAI,EAAE,KAAK,sBAAsB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtD,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;YACrC,kEAAkE;YAClE,uEAAuE;YACvE,IAAI,CAAC,8BAA8B,GAAG,IAAI,CAAC;YAC3C,OAAO;QACT,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACtB,CAAC;IAEO,UAAU,CAAC,EAAkB;QACnC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,GAAW;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAC9B,MAAM,IAAI,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,UAAU,CAAC,GAAkB;QACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACjD,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAE7D,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,sEAAsE;YACtE,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI;gBAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpE,OAAO;QACT,CAAC;QACD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAEO,gBAAgB;QACtB,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO;QACtB,IAAI,IAAsB,CAAC;QAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;YACxC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAG,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IAC5D,CAAC;IAEO,kBAAkB;QACxB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,cAAc;QACpB,4EAA4E;QAC5E,yCAAyC;QACzC,EAAE;QACF,0EAA0E;QAC1E,mEAAmE;QACnE,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,UAAU,CAAC;QACxC,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO,WAAW,CAAC;QAC1C,0EAA0E;QAC1E,gDAAgD;QAChD,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,OAAO,CAAC;QACtC,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,OAAO,CAAC;QAC/B,2EAA2E;QAC3E,0EAA0E;QAC1E,wEAAwE;QACxE,wEAAwE;QACxE,yEAAyE;QACzE,gDAAgD;QAChD,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,WAAW,CAAC;QAC9C,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO,MAAM,CAAC;QAC7B,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,eAAe,CAAC,KAAK,GAAG,KAAK;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACpC,IAAI,KAAK,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YACtC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,cAAc;QACpB,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,iBAAiB;QACvB,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,wEAAwE;QACxE,4EAA4E;QAC5E,4CAA4C;QAC5C,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YAAE,OAAO;QACpC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YAClC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC;gBAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;gBACjB,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IACvB,CAAC;IAEO,eAAe;QACrB,OAAO,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,WAAW;eAC3D,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;IACzD,CAAC;IAEO,cAAc;QACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEO,yBAAyB;QAC/B,yEAAyE;QACzE,wEAAwE;QACxE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAEO,oBAAoB;QAC1B,IAAI,IAAI,CAAC,wBAAwB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACvD,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;YACtC,IAAI,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAEO,uBAAuB;QAC7B,IAAI,IAAI,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,eAAe,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,qBAAqB,GAAG,GAAS,EAAE;QACzC,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC,CAAC;IAEM,qBAAqB,GAAG,GAAS,EAAE;QACzC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC,CAAC;IAEM,oBAAoB,GAAG,GAAS,EAAE;QACxC,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC1B,CAAC,CAAC;IAEM,oBAAoB,GAAG,GAAS,EAAE;QACxC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QACzB,2EAA2E;QAC3E,+DAA+D;QAC/D,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,8BAA8B,EAAE,CAAC;YACxC,IAAI,CAAC,8BAA8B,GAAG,KAAK,CAAC;YAC5C,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC9B,CAAC,CAAC;IAEM,iBAAiB,GAAG,GAAS,EAAE;QACrC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC,CAAC;IAEM,iBAAiB,GAAG,GAAS,EAAE;QACrC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,2EAA2E;QAC3E,+CAA+C;QAC/C,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC,CAAC;IAEM,OAAO,GAAG,CAAC,GAAY,EAAQ,EAAE;QACvC,MAAM,IAAI,GAAI,GAA0B,EAAE,IAAuC,CAAC;QAClF,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC;QAC7D,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC,CAAC;IAEM,cAAc,GAAG,GAAS,EAAE;QAClC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QACzB,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC,CAAC;IAEM,kBAAkB,GAAG,GAAS,EAAE;QACtC,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;YAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpD,uEAAuE;QACvE,yEAAyE;QACzE,mCAAmC;QACnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC,CAAC;IAEF;;;;;;;OAOG;IACH,MAAM,CAAC,MAAqB;QAC1B,MAAM,eAAe,GAAG,CAAC,GAAY,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC;QAClF,MAAM,aAAa,GAA8C;YAC/D,CAAC,WAAW,CAAC,aAAa,EAAE,eAAe,CAAC;YAC5C,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC;YAChD,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,kBAAkB,CAAC;YAC/C,CAAC,WAAW,CAAC,YAAY,EAAE,IAAI,CAAC,cAAc,CAAC;YAC/C,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC;YACjC,CAAC,WAAW,CAAC,mBAAmB,EAAE,IAAI,CAAC,qBAAqB,CAAC;YAC7D,CAAC,WAAW,CAAC,mBAAmB,EAAE,IAAI,CAAC,qBAAqB,CAAC;YAC7D,CAAC,WAAW,CAAC,kBAAkB,EAAE,IAAI,CAAC,oBAAoB,CAAC;YAC3D,CAAC,WAAW,CAAC,kBAAkB,EAAE,IAAI,CAAC,oBAAoB,CAAC;YAC3D,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,iBAAiB,CAAC;YACrD,CAAC,WAAW,CAAC,eAAe,EAAE,IAAI,CAAC,iBAAiB,CAAC;SACtD,CAAC;QACF,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,aAAa;YAAE,MAAM,CAAC,EAAE,CAAC,KAAkB,EAAE,QAAiB,CAAC,CAAC;QAChG,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,aAAa;gBAAE,MAAM,CAAC,GAAG,CAAC,KAAkB,EAAE,QAAiB,CAAC,CAAC;QACnG,CAAC,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,OAAO;QACL,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;CACF"}
@@ -0,0 +1,6 @@
1
+ /** Professional Indian male interviewer, authored to read at call-tile size. */
2
+ import type { CanvasAvatarOptions } from './createCanvasAvatar.js';
3
+ import type { AvatarInstance } from './createAvatar.js';
4
+ export type { CanvasAvatarOptions };
5
+ export declare function createAvatar(options: CanvasAvatarOptions): AvatarInstance;
6
+ //# sourceMappingURL=arjun.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"arjun.d.ts","sourceRoot":"","sources":["../client/arjun.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAGhF,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAUxD,YAAY,EAAE,mBAAmB,EAAE,CAAC;AAEpC,wBAAgB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,cAAc,CAOzE"}
package/dist/arjun.js ADDED
@@ -0,0 +1,18 @@
1
+ /** Professional Indian male interviewer, authored to read at call-tile size. */
2
+ import { createCanvasAvatar } from './createCanvasAvatar.js';
3
+ const RIG_URL = new URL('../src/canvas/data/interviewer-male.rig.json', import.meta.url);
4
+ const IMAGES = {
5
+ 'round-m3-hair-back.webp': new URL('../src/canvas/data/img/round-m3-hair-back.webp', import.meta.url),
6
+ 'round-m3-top-body.webp': new URL('../src/canvas/data/img/round-m3-top-body.webp', import.meta.url),
7
+ 'round-m3-hair-front.webp': new URL('../src/canvas/data/img/round-m3-hair-front.webp', import.meta.url),
8
+ };
9
+ const FACE = () => import('../src/canvas/avatars/round/face.mjs');
10
+ export function createAvatar(options) {
11
+ return createCanvasAvatar(options, {
12
+ rigUrl: RIG_URL,
13
+ images: IMAGES,
14
+ face: FACE,
15
+ label: 'Arjun — professional male interviewer avatar',
16
+ });
17
+ }
18
+ //# sourceMappingURL=arjun.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"arjun.js","sourceRoot":"","sources":["../client/arjun.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAEhF,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAI7D,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,8CAA8C,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzF,MAAM,MAAM,GAAG;IACb,yBAAyB,EAAE,IAAI,GAAG,CAAC,gDAAgD,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;IACrG,wBAAwB,EAAE,IAAI,GAAG,CAAC,+CAA+C,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;IACnG,0BAA0B,EAAE,IAAI,GAAG,CAAC,iDAAiD,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;CAC/F,CAAC;AACX,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,sCAAsC,CAAC,CAAC;AAIlE,MAAM,UAAU,YAAY,CAAC,OAA4B;IACvD,OAAO,kBAAkB,CAAC,OAAO,EAAE;QACjC,MAAM,EAAE,OAAO;QACf,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI;QACV,KAAK,EAAE,8CAA8C;KACtD,CAAC,CAAC;AACL,CAAC"}
@@ -0,0 +1,99 @@
1
+ /**
2
+ * `createAvatar` — the entire public surface of an avatar.
3
+ *
4
+ * const avatar = createAvatar({ mount: el, client: pipecatClient });
5
+ * // …
6
+ * avatar.destroy();
7
+ *
8
+ * An avatar is an embodiment of a `PipecatClient`. You hand it a mount and a
9
+ * live client; it subscribes and reacts. There is nothing else: no methods to
10
+ * drive it, no callbacks to observe it, no state to read back. Everything it
11
+ * needs to know is already on the wire — Pipecat's factual lifecycle events
12
+ * plus the avatar server-message envelope (docs/contract-wire.md).
13
+ *
14
+ * ## This is also the extension point
15
+ *
16
+ * There is no registry, no loader and no plug-in system. **You add an avatar by
17
+ * publishing a module that exports `createAvatar` with this signature**, and
18
+ * importing yours instead of ours:
19
+ *
20
+ * import { createAvatar } from "@acme/our-mascot";
21
+ *
22
+ * That is the whole mechanism, and it is deliberate. A registry would make us
23
+ * own resolution, versioning and asset paths for code we have never seen; a
24
+ * renderer interface would make us commit to a second public contract before we
25
+ * know what the second renderer actually needs. An ES module export we get for
26
+ * free from the platform.
27
+ *
28
+ * Everything past `mount` and `client` belongs to the implementation. Ours takes
29
+ * `SvgAvatarOptions`; yours takes whatever it likes, and `AvatarFactory<O>` is
30
+ * generic so the caller still gets your options checked.
31
+ *
32
+ * ## What an implementation owes the caller
33
+ *
34
+ * Only that `destroy()` leaves the mount as it found it and unsubscribes from
35
+ * the client. Not returning to rest, not honouring every action, not even
36
+ * having a mouth — a handless, mouthless avatar that only changes colour is a
37
+ * conforming avatar. The obligations that *matter* are perceptual, not typed,
38
+ * and they live in docs/contract-behavior.md.
39
+ */
40
+ import type { PipecatClient } from "@pipecat-ai/client-js";
41
+ import type { Face, FaceTheme, Gain, HandSide } from "../src/avatar.js";
42
+ export type { Face, FaceTheme, Gain, HandSide };
43
+ /**
44
+ * What every `createAvatar` takes. Implementations extend it with their own
45
+ * options; the two named here are the only ones the caller can count on.
46
+ */
47
+ export interface AvatarOptions {
48
+ /** The element to render into. The implementation owns its contents. */
49
+ readonly mount: HTMLElement;
50
+ /**
51
+ * A live `PipecatClient`. Required, and not nullable: an avatar with nothing
52
+ * to embody has no reason to exist yet. React callers get the forgiving
53
+ * version — `<Avatar>` waits for a non-null client before mounting.
54
+ */
55
+ readonly client: PipecatClient;
56
+ }
57
+ /** What every `createAvatar` returns. */
58
+ export interface AvatarInstance {
59
+ /** Unsubscribe from the client and remove everything from the mount. */
60
+ destroy(): void;
61
+ }
62
+ /**
63
+ * The shape a third-party avatar module exports, parameterised by its own
64
+ * options so a caller passing them gets them checked. `AvatarFactory` bare is
65
+ * the common denominator — the two options everyone shares.
66
+ */
67
+ export type AvatarFactory<O extends AvatarOptions = AvatarOptions> = (options: O) => AvatarInstance;
68
+ /**
69
+ * Options for the bundled SVG avatars — ours alone; nothing outside this
70
+ * package reads them.
71
+ *
72
+ * `face` is a value, not a name: a name needs a table, and a table needs every
73
+ * face imported to answer any lookup. Import the one you want and the other two
74
+ * never enter your bundle.
75
+ *
76
+ * import { createAvatar } from "@voqalize/avatar";
77
+ * import { wren } from "@voqalize/avatar/faces/wren";
78
+ *
79
+ * createAvatar({ mount, client, face: wren });
80
+ */
81
+ export interface SvgAvatarOptions extends AvatarOptions {
82
+ /** Defaults to `peep`, the face this entry point already carries. */
83
+ readonly face?: Face;
84
+ /** Palette for that face. Its keys are the face's own; see `THEME` in its
85
+ * module, and CLAUDE.md on why `peep` has exactly one. */
86
+ readonly theme?: FaceTheme;
87
+ /** Viseme amplitude, 0..2. `1` is as authored. */
88
+ readonly mouthGain?: Gain;
89
+ /** Gesture-clip amplitude, 0..2. */
90
+ readonly gestureGain?: Gain;
91
+ /** Idle/liveness amplitude, 0..2. Low by design — see CLAUDE.md. */
92
+ readonly motionGain?: Gain;
93
+ /** Render the frame-edge hand at all. Default true. */
94
+ readonly hand?: boolean;
95
+ /** `1` puts it on the viewer's right, `-1` the other side. */
96
+ readonly handSide?: HandSide;
97
+ }
98
+ export declare function createAvatar(options: SvgAvatarOptions): AvatarInstance;
99
+ //# sourceMappingURL=createAvatar.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createAvatar.d.ts","sourceRoot":"","sources":["../client/createAvatar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAIxE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AAEhD;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,wEAAwE;IACxE,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;CAChC;AAED,yCAAyC;AACzC,MAAM,WAAW,cAAc;IAC7B,wEAAwE;IACxE,OAAO,IAAI,IAAI,CAAC;CACjB;AAED;;;;GAIG;AACH,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,aAAa,GAAG,aAAa,IAC/D,CAAC,OAAO,EAAE,CAAC,KAAK,cAAc,CAAC;AAEjC;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACrD,qEAAqE;IACrE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC;IACrB;8DAC0D;IAC1D,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC;IAC3B,kDAAkD;IAClD,QAAQ,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC;IAC1B,oCAAoC;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,IAAI,CAAC;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC;IAC3B,uDAAuD;IACvD,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IACxB,8DAA8D;IAC9D,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;CAC9B;AAID,wBAAgB,YAAY,CAAC,OAAO,EAAE,gBAAgB,GAAG,cAAc,CA4BtE"}