@voqalize/avatar 0.2.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (265) hide show
  1. package/LICENSE +21 -661
  2. package/LICENSE-CC-BY-4.0 +396 -0
  3. package/README.md +191 -662
  4. package/assets/README.md +48 -0
  5. package/assets/tanya.glb +0 -0
  6. package/assets/tara.glb +0 -0
  7. package/assets/tushar.glb +0 -0
  8. package/client/Avatar.tsx +57 -0
  9. package/client/AvatarClient.ts +818 -0
  10. package/client/arjun.ts +26 -0
  11. package/client/createAvatar.ts +177 -0
  12. package/client/createCanvasAvatar.ts +72 -0
  13. package/client/index.ts +44 -0
  14. package/client/internal.ts +108 -0
  15. package/client/interviewer-female.ts +4 -0
  16. package/client/interviewer-male.ts +4 -0
  17. package/client/ishita.ts +26 -0
  18. package/client/kabir.ts +26 -0
  19. package/client/meera.ts +26 -0
  20. package/client/naina.ts +26 -0
  21. package/client/playout.ts +95 -0
  22. package/client/professional-female-a.ts +4 -0
  23. package/client/professional-female-b.ts +4 -0
  24. package/client/professional-male-a.ts +4 -0
  25. package/client/professional-male-b.ts +4 -0
  26. package/client/react.ts +13 -0
  27. package/client/supports.ts +28 -0
  28. package/client/three/assets.ts +24 -0
  29. package/client/three/budgets.ts +27 -0
  30. package/client/three/holds.ts +64 -0
  31. package/client/three/internal.ts +54 -0
  32. package/client/three/motion-limits.json +116 -0
  33. package/client/three/sequences.ts +281 -0
  34. package/client/three/tanya.ts +69 -0
  35. package/client/three/tara-rig.ts +1562 -0
  36. package/client/three/tara.ts +76 -0
  37. package/client/three/tushar.ts +65 -0
  38. package/client/types.ts +235 -0
  39. package/client/useAvatar.ts +74 -0
  40. package/client/vikram.ts +26 -0
  41. package/dist/Avatar.d.ts +44 -0
  42. package/dist/Avatar.d.ts.map +1 -0
  43. package/{client/dist → dist}/Avatar.js +2 -2
  44. package/dist/Avatar.js.map +1 -0
  45. package/dist/AvatarClient.d.ts +274 -0
  46. package/dist/AvatarClient.d.ts.map +1 -0
  47. package/dist/AvatarClient.js +712 -0
  48. package/dist/AvatarClient.js.map +1 -0
  49. package/dist/arjun.d.ts +7 -0
  50. package/dist/arjun.d.ts.map +1 -0
  51. package/dist/arjun.js +20 -0
  52. package/dist/arjun.js.map +1 -0
  53. package/dist/createAvatar.d.ts +135 -0
  54. package/dist/createAvatar.d.ts.map +1 -0
  55. package/dist/createAvatar.js +75 -0
  56. package/dist/createAvatar.js.map +1 -0
  57. package/dist/createCanvasAvatar.d.ts +22 -0
  58. package/dist/createCanvasAvatar.d.ts.map +1 -0
  59. package/dist/createCanvasAvatar.js +47 -0
  60. package/dist/createCanvasAvatar.js.map +1 -0
  61. package/dist/index.d.ts +34 -0
  62. package/dist/index.d.ts.map +1 -0
  63. package/dist/index.js +33 -0
  64. package/dist/index.js.map +1 -0
  65. package/dist/internal.d.ts +34 -0
  66. package/dist/internal.d.ts.map +1 -0
  67. package/dist/internal.js +41 -0
  68. package/dist/internal.js.map +1 -0
  69. package/dist/interviewer-female.d.ts +4 -0
  70. package/dist/interviewer-female.d.ts.map +1 -0
  71. package/dist/interviewer-female.js +3 -0
  72. package/dist/interviewer-female.js.map +1 -0
  73. package/dist/interviewer-male.d.ts +4 -0
  74. package/dist/interviewer-male.d.ts.map +1 -0
  75. package/dist/interviewer-male.js +3 -0
  76. package/dist/interviewer-male.js.map +1 -0
  77. package/dist/ishita.d.ts +7 -0
  78. package/dist/ishita.d.ts.map +1 -0
  79. package/dist/ishita.js +20 -0
  80. package/dist/ishita.js.map +1 -0
  81. package/dist/kabir.d.ts +7 -0
  82. package/dist/kabir.d.ts.map +1 -0
  83. package/dist/kabir.js +20 -0
  84. package/dist/kabir.js.map +1 -0
  85. package/dist/meera.d.ts +7 -0
  86. package/dist/meera.d.ts.map +1 -0
  87. package/dist/meera.js +20 -0
  88. package/dist/meera.js.map +1 -0
  89. package/dist/naina.d.ts +7 -0
  90. package/dist/naina.d.ts.map +1 -0
  91. package/dist/naina.js +20 -0
  92. package/dist/naina.js.map +1 -0
  93. package/dist/playout.d.ts +42 -0
  94. package/dist/playout.d.ts.map +1 -0
  95. package/dist/playout.js +85 -0
  96. package/dist/playout.js.map +1 -0
  97. package/dist/professional-female-a.d.ts +4 -0
  98. package/dist/professional-female-a.d.ts.map +1 -0
  99. package/dist/professional-female-a.js +3 -0
  100. package/dist/professional-female-a.js.map +1 -0
  101. package/dist/professional-female-b.d.ts +4 -0
  102. package/dist/professional-female-b.d.ts.map +1 -0
  103. package/dist/professional-female-b.js +3 -0
  104. package/dist/professional-female-b.js.map +1 -0
  105. package/dist/professional-male-a.d.ts +4 -0
  106. package/dist/professional-male-a.d.ts.map +1 -0
  107. package/dist/professional-male-a.js +3 -0
  108. package/dist/professional-male-a.js.map +1 -0
  109. package/dist/professional-male-b.d.ts +4 -0
  110. package/dist/professional-male-b.d.ts.map +1 -0
  111. package/dist/professional-male-b.js +3 -0
  112. package/dist/professional-male-b.js.map +1 -0
  113. package/dist/react.d.ts +13 -0
  114. package/dist/react.d.ts.map +1 -0
  115. package/dist/react.js +13 -0
  116. package/dist/react.js.map +1 -0
  117. package/dist/supports.d.ts +20 -0
  118. package/dist/supports.d.ts.map +1 -0
  119. package/dist/supports.js +26 -0
  120. package/dist/supports.js.map +1 -0
  121. package/dist/three/assets.d.ts +24 -0
  122. package/dist/three/assets.d.ts.map +1 -0
  123. package/dist/three/assets.js +22 -0
  124. package/dist/three/assets.js.map +1 -0
  125. package/dist/three/budgets.d.ts +17 -0
  126. package/dist/three/budgets.d.ts.map +1 -0
  127. package/dist/three/budgets.js +21 -0
  128. package/dist/three/budgets.js.map +1 -0
  129. package/dist/three/holds.d.ts +33 -0
  130. package/dist/three/holds.d.ts.map +1 -0
  131. package/dist/three/holds.js +56 -0
  132. package/dist/three/holds.js.map +1 -0
  133. package/dist/three/internal.d.ts +52 -0
  134. package/dist/three/internal.d.ts.map +1 -0
  135. package/dist/three/internal.js +49 -0
  136. package/dist/three/internal.js.map +1 -0
  137. package/dist/three/motion-limits.json +116 -0
  138. package/dist/three/sequences.d.ts +71 -0
  139. package/dist/three/sequences.d.ts.map +1 -0
  140. package/dist/three/sequences.js +262 -0
  141. package/dist/three/sequences.js.map +1 -0
  142. package/dist/three/tanya.d.ts +35 -0
  143. package/dist/three/tanya.d.ts.map +1 -0
  144. package/dist/three/tanya.js +51 -0
  145. package/dist/three/tanya.js.map +1 -0
  146. package/dist/three/tara-rig.d.ts +403 -0
  147. package/dist/three/tara-rig.d.ts.map +1 -0
  148. package/dist/three/tara-rig.js +1500 -0
  149. package/dist/three/tara-rig.js.map +1 -0
  150. package/dist/three/tara.d.ts +35 -0
  151. package/dist/three/tara.d.ts.map +1 -0
  152. package/dist/three/tara.js +58 -0
  153. package/dist/three/tara.js.map +1 -0
  154. package/dist/three/tushar.d.ts +31 -0
  155. package/dist/three/tushar.d.ts.map +1 -0
  156. package/dist/three/tushar.js +47 -0
  157. package/dist/three/tushar.js.map +1 -0
  158. package/dist/types.d.ts +149 -0
  159. package/dist/types.d.ts.map +1 -0
  160. package/dist/types.js +140 -0
  161. package/dist/types.js.map +1 -0
  162. package/dist/useAvatar.d.ts +39 -0
  163. package/dist/useAvatar.d.ts.map +1 -0
  164. package/dist/useAvatar.js +35 -0
  165. package/dist/useAvatar.js.map +1 -0
  166. package/dist/vikram.d.ts +7 -0
  167. package/dist/vikram.d.ts.map +1 -0
  168. package/dist/vikram.js +20 -0
  169. package/dist/vikram.js.map +1 -0
  170. package/package.json +110 -25
  171. package/src/avatar.d.ts +218 -120
  172. package/src/avatar.js +950 -213
  173. package/src/behavior.d.ts +42 -0
  174. package/src/behavior.js +114 -0
  175. package/src/camera.js +29 -0
  176. package/src/canvas/author/parts/eye.mjs +722 -0
  177. package/src/canvas/author/parts/hand.mjs +1156 -0
  178. package/src/canvas/author/parts/mouth.mjs +741 -0
  179. package/src/canvas/author/parts/nose.mjs +100 -0
  180. package/src/canvas/author/parts/skin-detail.mjs +67 -0
  181. package/src/canvas/author/path.mjs +283 -0
  182. package/src/canvas/author/rig.mjs +405 -0
  183. package/src/canvas/avatars/round/face.d.mts +3 -0
  184. package/src/canvas/avatars/round/face.mjs +1307 -0
  185. package/src/canvas/create-rig.d.ts +15 -0
  186. package/src/canvas/create-rig.js +100 -0
  187. package/src/canvas/data/img/professional-female-a-hair-back.webp +0 -0
  188. package/src/canvas/data/img/professional-female-a-hair-front.webp +0 -0
  189. package/src/canvas/data/img/professional-female-a-top-body.webp +0 -0
  190. package/src/canvas/data/img/professional-female-b-hair-back.webp +0 -0
  191. package/src/canvas/data/img/professional-female-b-hair-front.webp +0 -0
  192. package/src/canvas/data/img/professional-female-b-top-body.webp +0 -0
  193. package/src/canvas/data/img/professional-male-a-hair-back.webp +0 -0
  194. package/src/canvas/data/img/professional-male-a-hair-front.webp +0 -0
  195. package/src/canvas/data/img/professional-male-a-top-body.webp +0 -0
  196. package/src/canvas/data/img/professional-male-b-hair-back.webp +0 -0
  197. package/src/canvas/data/img/professional-male-b-hair-front.webp +0 -0
  198. package/src/canvas/data/img/professional-male-b-top-body.webp +0 -0
  199. package/src/canvas/data/img/round-m3-hair-back.webp +0 -0
  200. package/src/canvas/data/img/round-m3-hair-front.webp +0 -0
  201. package/src/canvas/data/img/round-m3-top-body.webp +0 -0
  202. package/src/canvas/data/img/round-w1-hair-back.webp +0 -0
  203. package/src/canvas/data/img/round-w1-hair-front.webp +0 -0
  204. package/src/canvas/data/img/round-w1-top-body.webp +0 -0
  205. package/src/canvas/data/interviewer-female.rig.json +1 -0
  206. package/src/canvas/data/interviewer-male.rig.json +1 -0
  207. package/src/canvas/data/professional-female-a.rig.json +1 -0
  208. package/src/canvas/data/professional-female-b.rig.json +1 -0
  209. package/src/canvas/data/professional-male-a.rig.json +1 -0
  210. package/src/canvas/data/professional-male-b.rig.json +1 -0
  211. package/src/canvas/src/live.js +508 -0
  212. package/src/canvas/src/render2d.js +218 -0
  213. package/src/canvas/src/rig.js +297 -0
  214. package/src/canvas/src/vocab.js +96 -0
  215. package/src/clips.js +18 -7
  216. package/src/conformance.js +119 -0
  217. package/src/emotions.js +8 -5
  218. package/src/face-core.js +27 -1
  219. package/src/face-myna.d.ts +7 -0
  220. package/src/face-myna.js +192 -131
  221. package/src/face-peep-control-plane.js +167 -0
  222. package/src/face-peep.d.ts +7 -0
  223. package/src/face-peep.js +274 -116
  224. package/src/face-wren.d.ts +7 -0
  225. package/src/face-wren.js +17 -19
  226. package/src/faces.d.ts +12 -0
  227. package/src/faces.js +53 -0
  228. package/src/gaze.js +434 -58
  229. package/src/hand.js +120 -91
  230. package/src/head.js +134 -0
  231. package/src/idle.js +227 -119
  232. package/src/interjections.js +200 -31
  233. package/src/params.js +8 -4
  234. package/src/perform.js +7 -9
  235. package/src/prosody.js +647 -0
  236. package/src/rig.d.ts +24 -0
  237. package/src/rig.js +32 -0
  238. package/src/speech-timing.js +23 -0
  239. package/src/visemes.js +110 -24
  240. package/client/dist/Avatar.d.ts +0 -27
  241. package/client/dist/Avatar.d.ts.map +0 -1
  242. package/client/dist/Avatar.js.map +0 -1
  243. package/client/dist/AvatarClient.d.ts +0 -139
  244. package/client/dist/AvatarClient.d.ts.map +0 -1
  245. package/client/dist/AvatarClient.js +0 -223
  246. package/client/dist/AvatarClient.js.map +0 -1
  247. package/client/dist/index.d.ts +0 -22
  248. package/client/dist/index.d.ts.map +0 -1
  249. package/client/dist/index.js +0 -22
  250. package/client/dist/index.js.map +0 -1
  251. package/client/dist/types.d.ts +0 -86
  252. package/client/dist/types.d.ts.map +0 -1
  253. package/client/dist/types.js +0 -31
  254. package/client/dist/types.js.map +0 -1
  255. package/client/dist/useAvatar.d.ts +0 -42
  256. package/client/dist/useAvatar.d.ts.map +0 -1
  257. package/client/dist/useAvatar.js +0 -51
  258. package/client/dist/useAvatar.js.map +0 -1
  259. package/client/src/Avatar.tsx +0 -33
  260. package/client/src/AvatarClient.ts +0 -270
  261. package/client/src/index.ts +0 -22
  262. package/client/src/types.ts +0 -104
  263. package/client/src/useAvatar.ts +0 -85
  264. package/docs/contract-avatar.md +0 -371
  265. package/docs/contract-protocol.md +0 -461
@@ -0,0 +1,76 @@
1
+ /**
2
+ * tara — the first 3-D character, as a complete `createAvatar` module.
3
+ *
4
+ * Structurally this is the canvas avatars' shape, one layer down: the mixer
5
+ * (`@voqalize/avatar/internal`) does states, layers, gaze, idle, blinks, clips
6
+ * and per-channel smoothing exactly as it does for every SVG face, and the only
7
+ * thing tara replaces is the renderer at the end of it. That is the whole
8
+ * argument for authoring her morph targets under the library's pose-channel
9
+ * names: nothing above this line knows there is a GPU involved, and every clip,
10
+ * viseme and co-articulation rule the SVG faces have works here unmodified.
11
+ *
12
+ * The Three.js import lives behind this module rather than in the barrel so an
13
+ * SVG or Canvas consumer never downloads it.
14
+ */
15
+
16
+ import type { PipecatClient } from "@pipecat-ai/client-js";
17
+ import { AvatarClient, createSvgAvatar } from "../internal.js";
18
+ import { BLENDER_ACTIONS, BLENDER_SEQUENCES, BLENDER_SUPPORTS } from "./sequences.js";
19
+ import { headHold } from "./holds.js";
20
+ import { createTaraRig, TARA_TUNING } from "./tara-rig.js";
21
+ import type { TaraRigOptions } from "./tara-rig.js";
22
+
23
+ export interface AvatarOptions {
24
+ readonly mount: HTMLElement;
25
+ readonly client: PipecatClient;
26
+ /** Viseme amplitude, 0..2. `1` is as authored. */
27
+ readonly mouthGain?: number;
28
+ /** Gesture-clip amplitude, 0..2. */
29
+ readonly gestureGain?: number;
30
+ /** Idle/liveness amplitude, 0..2. */
31
+ readonly motionGain?: number;
32
+ /** Fires when the GLB is in the scene — for a capture tool, not a consumer. */
33
+ readonly onReady?: () => void;
34
+ }
35
+
36
+ export interface AvatarInstance { destroy(): void; }
37
+
38
+ /** The optional driving-UI declaration; one list for all three characters
39
+ * (`sequences.ts`), because they are one rig. */
40
+ export const supports = BLENDER_SUPPORTS;
41
+
42
+ export function createAvatar(options: AvatarOptions): AvatarInstance {
43
+ const { mount, client, onReady, ...gains } = options;
44
+ if (!mount) throw new TypeError("createAvatar: `mount` is required");
45
+ if (!client) throw new TypeError("createAvatar: `client` is required");
46
+
47
+ const rigOptions: TaraRigOptions = { onReady };
48
+ // `hand: false` disables the bundled SVG hand renderer only; the semantic
49
+ // hand frame still reaches the rig, which ignores it — tara has no arms, and
50
+ // that is the library's oldest standing constraint rather than an omission.
51
+ // `sequences` is what a server can address on *this* avatar beyond the two
52
+ // ids every avatar owes it — the three nod types the listening research
53
+ // separates, and a head shake sized for a rig whose pose unit is a degree.
54
+ // The wire's action id is open, so nothing here needed promoting; a server
55
+ // that does not know tara is mounted sends `ACKNOWLEDGE` and is never wrong.
56
+ // `actions` is the other half: her own shape for an id the mixer already has,
57
+ // whose shared keys land outside what it means on a head that turns in degrees.
58
+ const widget = createSvgAvatar({
59
+ mount, rig: createTaraRig, rigOptions, hand: false,
60
+ sequences: BLENDER_SEQUENCES, actions: BLENDER_ACTIONS, ...TARA_TUNING,
61
+ headHold: headHold("tara"), ...gains,
62
+ });
63
+ const driver = new AvatarClient(widget);
64
+ const detach = driver.attach(client);
65
+
66
+ let destroyed = false;
67
+ return {
68
+ destroy() {
69
+ if (destroyed) return;
70
+ destroyed = true;
71
+ detach();
72
+ driver.destroy();
73
+ widget.destroy();
74
+ },
75
+ };
76
+ }
@@ -0,0 +1,65 @@
1
+ /**
2
+ * tushar — the second 3-D character, and deliberately a copy of `tara.ts`.
3
+ *
4
+ * His GLB is tara's scripts run with his landmarks and atlas layout in place of
5
+ * hers (`characters/tushar/`, whose README has the provenance and the commands),
6
+ * so the rig, the mixer, the sequences and her tuning are all hers, unmodified;
7
+ * the one seam is `TaraRigOptions.url`. What turns out to be hers rather than
8
+ * general is marked TARA-SPECIFIC where it lives, and generalised only once a
9
+ * third character agrees.
10
+ */
11
+
12
+ import type { PipecatClient } from "@pipecat-ai/client-js";
13
+ import { AvatarClient, createSvgAvatar } from "../internal.js";
14
+ import { ASSETS } from "./assets.js";
15
+ import { BLENDER_ACTIONS, BLENDER_SEQUENCES, BLENDER_SUPPORTS } from "./sequences.js";
16
+ import { headHold } from "./holds.js";
17
+ import { createTaraRig, TARA_TUNING } from "./tara-rig.js";
18
+ import type { TaraRigOptions } from "./tara-rig.js";
19
+
20
+ export interface AvatarOptions {
21
+ readonly mount: HTMLElement;
22
+ readonly client: PipecatClient;
23
+ /** Viseme amplitude, 0..2. `1` is as authored. */
24
+ readonly mouthGain?: number;
25
+ /** Gesture-clip amplitude, 0..2. */
26
+ readonly gestureGain?: number;
27
+ /** Idle/liveness amplitude, 0..2. */
28
+ readonly motionGain?: number;
29
+ /** Fires when the GLB is in the scene — for a capture tool, not a consumer. */
30
+ readonly onReady?: () => void;
31
+ }
32
+
33
+ export interface AvatarInstance { destroy(): void; }
34
+
35
+ /** The optional driving-UI declaration; one list for all three characters
36
+ * (`sequences.ts`), because they are one rig. */
37
+ export const supports = BLENDER_SUPPORTS;
38
+
39
+ export function createAvatar(options: AvatarOptions): AvatarInstance {
40
+ const { mount, client, onReady, ...gains } = options;
41
+ if (!mount) throw new TypeError("createAvatar: `mount` is required");
42
+ if (!client) throw new TypeError("createAvatar: `client` is required");
43
+
44
+ const rigOptions: TaraRigOptions = { onReady, url: ASSETS.tushar };
45
+ // TARA-SPECIFIC, kept on purpose: `TARA_TUNING` (her mouth and motion gains)
46
+ // is the first thing to question if he reads wrong in a call.
47
+ const widget = createSvgAvatar({
48
+ mount, rig: createTaraRig, rigOptions, hand: false,
49
+ sequences: BLENDER_SEQUENCES, actions: BLENDER_ACTIONS, ...TARA_TUNING,
50
+ headHold: headHold("tushar"), ...gains,
51
+ });
52
+ const driver = new AvatarClient(widget);
53
+ const detach = driver.attach(client);
54
+
55
+ let destroyed = false;
56
+ return {
57
+ destroy() {
58
+ if (destroyed) return;
59
+ destroyed = true;
60
+ detach();
61
+ driver.destroy();
62
+ widget.destroy();
63
+ },
64
+ };
65
+ }
@@ -0,0 +1,235 @@
1
+ /**
2
+ * types.ts — the avatar wire vocabulary, client side.
3
+ *
4
+ * The binding definition is `docs/contract-wire.md`; this file is its
5
+ * TypeScript restatement and must not drift from it. The Python half of the
6
+ * same vocabulary is `packages/avatar-py/src/voqalize_avatar/messages.py` — the three are
7
+ * maintained together, and a command added to one without the others is
8
+ * incomplete.
9
+ *
10
+ * A server pushes these as RTVI `server-message`s under the envelope
11
+ * `{ type: "avatar", ...cmd-specific fields }`. {@link AvatarCommand} describes
12
+ * what rides inside that envelope; {@link isAvatarMessage} is the envelope
13
+ * itself, and is the only definition of "this message is for the avatar" the
14
+ * client has. There is no protocol version field — an unknown `cmd` is simply
15
+ * ignored, which is the whole forward-compatibility rule.
16
+ */
17
+
18
+ import { VISEME_LETTERS } from "../src/avatar.js";
19
+ import type { VisemeLetter } from "../src/avatar.js";
20
+
21
+ /** A viseme cue: `t` is a ms offset into the utterance's clock, `v` is a Rhubarb A–H (or X) letter. */
22
+ export interface AvatarCue {
23
+ t: number;
24
+ v: VisemeLetter;
25
+ /** 0..1 loudness. Absent means full. */
26
+ i?: number;
27
+ /**
28
+ * The phone being articulated under `v`, absent during silence.
29
+ *
30
+ * `v` is a nine-way projection of the recogniser's ~40 phones and the loss is
31
+ * concentrated — `B` alone absorbs IY, IH, T, D, CH, JH, TH, DH, S, Z, SH,
32
+ * ZH, N and Y — so a renderer with a mouth for "tongue between the teeth"
33
+ * cannot ask for it from `v` and can from here. **Not a closed set**: it is
34
+ * whatever the engine labels its own segments with, which today is Arpabet
35
+ * plus `Schwa` and four non-speech labels, and a later engine may say
36
+ * something else. Validated as an identifier and passed through unread —
37
+ * the letter stays the thing every face must handle, and a face that wants
38
+ * the finer signal opts in.
39
+ */
40
+ p?: string;
41
+ }
42
+
43
+ /**
44
+ * A durable, lower-priority server state. `null` explicitly clears it.
45
+ *
46
+ * Three values, and the server can send no others: the remaining six of the
47
+ * nine are Pipecat facts the browser already holds, and a server spelling of a
48
+ * fact would be a second, lower-authority copy of it. It was `cmd: "claim"`
49
+ * until the wire redesign, when the concept and the command were given the one
50
+ * name the rest of the system already used for it.
51
+ */
52
+ export interface AvatarStateCmd {
53
+ cmd: "state";
54
+ state: "CANT_HEAR" | "THINKING" | "WORKING" | null;
55
+ }
56
+
57
+ /**
58
+ * One self-completing motion, by name: face, body, and optionally a hand.
59
+ *
60
+ * **The id is open.** Two of them — {@link CORE_ACTION_IDS} — are intents every
61
+ * renderer owes a server and are the only ones a server may send without
62
+ * knowing what is mounted. Everything else is a name from the mounted avatar's
63
+ * own catalogue, and **an unknown one is ignored, not an error**: that is the
64
+ * same forward-compatibility rule an unknown `cmd` gets, arrived at from the
65
+ * other direction. There, a newer server meets an older widget; here, any
66
+ * server meets a face that cannot do the thing. Neither is worth breaking a
67
+ * call over, and there is deliberately no fallback — an action sits on top of a
68
+ * state, so nothing is missing when one is dropped.
69
+ *
70
+ * This replaced a closed seven-id vocabulary plus a second command,
71
+ * `sequence`, for a renderer's own motions. The split cost a promotion ritual
72
+ * for every new portable intent and still could not say what the closed set was
73
+ * for: the listening research separates a continuer nod from an assessment nod
74
+ * from a realisation, and all three were the one id `ACK_NOD` spelled. Now the
75
+ * server says *when* to acknowledge and the avatar owns the variety.
76
+ */
77
+ export interface AvatarActionCmd {
78
+ cmd: "action";
79
+ id: string;
80
+ }
81
+
82
+ /**
83
+ * The two actions every renderer must answer to.
84
+ *
85
+ * `ACKNOWLEDGE` is the whole backchannel family in one word — receipt, nod,
86
+ * realisation, empathy — because which of those a face does is a rendering
87
+ * decision and the server is not the one holding the drawing.
88
+ * `RESPONSE_INTERRUPTED` is the one transition a server can explain and the
89
+ * browser cannot infer.
90
+ *
91
+ * A renderer may draw either in its own shape; it may never redefine one, and
92
+ * a name in its own catalogue never shadows one of these.
93
+ */
94
+ export const CORE_ACTION_IDS = Object.freeze(["ACKNOWLEDGE", "RESPONSE_INTERRUPTED"] as const);
95
+
96
+ export type CoreActionId = (typeof CORE_ACTION_IDS)[number];
97
+
98
+ export interface AvatarCuesCmd {
99
+ cmd: "cues";
100
+ ctx: string;
101
+ /** Discard queued cues at or after this offset (ms), then append `cues`. */
102
+ from_ms: number;
103
+ cues: AvatarCue[];
104
+ /**
105
+ * True on the one chunk that completes this turn's track: the TTS context is
106
+ * closed, so no further chunk will splice into `ctx`. What a client may
107
+ * assume, exactly — nothing about playout. The audio it describes is still
108
+ * ahead, and Pipecat's `botStoppedSpeaking` remains the end of the turn. It
109
+ * is safe to stop expecting more cue chunks after `final`.
110
+ *
111
+ * Absent on an interrupted turn, deliberately: a turn that was cut never
112
+ * claims to have completed. Absent chunks are the normal case — the widget's
113
+ * own track already completes on the trailing `X`, so ignoring `final`
114
+ * entirely is a correct implementation.
115
+ */
116
+ final?: boolean;
117
+ }
118
+
119
+ export type AvatarCommand =
120
+ | AvatarStateCmd
121
+ | AvatarActionCmd
122
+ | AvatarCuesCmd;
123
+
124
+ /** The full server-message payload: the envelope plus its command. */
125
+ export type AvatarServerMessage = AvatarCommand & { type: "avatar" };
126
+
127
+ /** An envelope addressed to the avatar, before its payload has been read. */
128
+ export type AvatarEnvelope = { type: "avatar"; cmd: string } & Record<string, unknown>;
129
+
130
+ /**
131
+ * Is this server-message payload the avatar's? The envelope is the whole
132
+ * answer: `{type:"avatar"}` with a string `cmd`. It used to be a per-deployment
133
+ * `accept` predicate on the client, which meant the library could not state
134
+ * what an avatar message *is*.
135
+ *
136
+ * Addressed to us is not the same as understood by us: {@link parseAvatarCommand}
137
+ * is the second half.
138
+ */
139
+ export function isAvatarMessage(msg: unknown): msg is AvatarEnvelope {
140
+ if (typeof msg !== "object" || msg === null) return false;
141
+ const m = msg as Record<string, unknown>;
142
+ return m.type === AVATAR_MESSAGE_TYPE && typeof m.cmd === "string";
143
+ }
144
+
145
+ /**
146
+ * What an action id may look like. Not which ones exist — that is the mounted
147
+ * avatar's business and this function has never met it. `CATEGORY_INTENT` in
148
+ * upper case, bounded so a malformed or hostile message cannot smuggle
149
+ * anything through as a name.
150
+ */
151
+ const ACTION_ID = /^[A-Z][A-Z0-9_]{1,63}$/;
152
+ const LETTERS = new Set<string>(VISEME_LETTERS);
153
+ /**
154
+ * What a phone label may look like. Letters only, because every label any
155
+ * engine we have seen is one word of them — `AO`, `NG`, `Schwa`, `Breath` —
156
+ * and the bound is what stops a name being a payload.
157
+ */
158
+ const PHONE_LABEL = /^[A-Za-z]{1,12}$/;
159
+
160
+ /**
161
+ * Read an envelope's payload into the wire vocabulary, or `null` if this build
162
+ * cannot act on it — an unknown `cmd`, a state outside the three, an action id
163
+ * that is not a name, a malformed `cues` chunk.
164
+ *
165
+ * `null` is the forward-compat rule with a type attached: a newer server
166
+ * talking to an older widget is *expected*, and the older widget ignores what
167
+ * it does not know rather than guessing. Doing the check here rather than at
168
+ * each use site is what lets the wire types be closed unions instead of
169
+ * `string` — the boundary is one function, so it can be the only place that
170
+ * has to be honest about untrusted input.
171
+ *
172
+ * Cues survive individually: one unrecognised letter in a chunk drops that cue,
173
+ * not the utterance around it. Losing a frame of articulation is a far smaller
174
+ * regression than losing a sentence of it.
175
+ */
176
+ export function parseAvatarCommand(msg: AvatarEnvelope): AvatarCommand | null {
177
+ switch (msg.cmd) {
178
+ // `claim` was this command's name, and `STRAINING` one of its values, until
179
+ // the wire redesign. Both are accepted and translated here, in the one
180
+ // place that reads untrusted input, so no layer above has ever heard of
181
+ // either spelling. Servers that predate the rename keep working; the old
182
+ // names appear nowhere else and come out when the last one has shipped.
183
+ case "claim":
184
+ case "state": {
185
+ const state = msg.state === "STRAINING" ? "CANT_HEAR" : msg.state;
186
+ if (state === null || state === "CANT_HEAR" || state === "THINKING" || state === "WORKING") {
187
+ return { cmd: "state", state };
188
+ }
189
+ return null;
190
+ }
191
+ // `sequence` was a second command for the same thing: a name resolved
192
+ // against the mounted avatar rather than against a closed list. Now that
193
+ // `action` is that, the two are one command and the old one is an alias.
194
+ case "sequence":
195
+ case "action":
196
+ // Shape only, both for a core id and for one of the avatar's own. There
197
+ // is nothing to check a name against here — the catalogue belongs to
198
+ // whatever is mounted, and this function has never met it.
199
+ return typeof msg.id === "string" && ACTION_ID.test(msg.id)
200
+ ? { cmd: "action", id: msg.id }
201
+ : null;
202
+ case "cues": {
203
+ if (typeof msg.ctx !== "string" || !Number.isFinite(msg.from_ms) || !Array.isArray(msg.cues)) {
204
+ return null;
205
+ }
206
+ const cues: AvatarCue[] = [];
207
+ for (const c of msg.cues as unknown[]) {
208
+ if (typeof c !== "object" || c === null) continue;
209
+ const { t, v, i, p } = c as Record<string, unknown>;
210
+ if (typeof t !== "number" || !Number.isFinite(t)) continue;
211
+ if (typeof v !== "string" || !LETTERS.has(v)) continue;
212
+ const cue: AvatarCue = { t, v: v as VisemeLetter };
213
+ if (typeof i === "number") cue.i = i;
214
+ // Bounded as a name, not checked against a list. The set belongs to
215
+ // whatever produced the cue, so enumerating it here would make this
216
+ // library the thing that has to be released before a better recogniser
217
+ // can say a new word.
218
+ if (typeof p === "string" && PHONE_LABEL.test(p)) cue.p = p;
219
+ cues.push(cue);
220
+ }
221
+ return {
222
+ cmd: "cues",
223
+ ctx: msg.ctx,
224
+ from_ms: msg.from_ms as number,
225
+ cues,
226
+ ...(msg.final === true ? { final: true as const } : {}),
227
+ };
228
+ }
229
+ default:
230
+ return null;
231
+ }
232
+ }
233
+
234
+ /** The envelope `type` the protocol reserves for avatar traffic. */
235
+ export const AVATAR_MESSAGE_TYPE = "avatar";
@@ -0,0 +1,74 @@
1
+ /**
2
+ * useAvatar — mount an avatar into a ref'd element for as long as there is a
3
+ * client to embody, and tear it down after.
4
+ *
5
+ * Internal; `<Avatar>` is the only thing the React entry exports. The whole
6
+ * hook is one effect, because the factory takes the client at construction:
7
+ * there is no separate attach step to keep in its own lifecycle any more.
8
+ */
9
+
10
+ import { useEffect, useRef } from "react";
11
+ import type { PipecatClient } from "@pipecat-ai/client-js";
12
+ import {
13
+ createAvatar as createSvgAvatar,
14
+ type AvatarFactory,
15
+ type AvatarOptions,
16
+ type SvgAvatarOptions,
17
+ } from "./createAvatar.js";
18
+
19
+ export interface UseAvatarOptions<O extends AvatarOptions = SvgAvatarOptions> {
20
+ /** The live `PipecatClient`, or `null` before connect. Nothing mounts until
21
+ * this is non-null — an avatar with nothing to embody has nothing to do. */
22
+ client?: PipecatClient | null;
23
+ /** The avatar implementation. Defaults to the bundled SVG faces. */
24
+ create?: AvatarFactory<O>;
25
+ /** Implementation options, forwarded verbatim. Read at mount only. */
26
+ options?: Omit<O, keyof AvatarOptions>;
27
+ }
28
+
29
+ /**
30
+ * The mount ref's type, written out rather than named as React's `RefObject`.
31
+ *
32
+ * React 18 and 19 declare that alias with different type arguments — 18's
33
+ * `useRef<T>(null)` yields `RefObject<T>`, 19's yields `RefObject<T | null>` —
34
+ * and because both are the *same alias*, TypeScript compares them by variance
35
+ * and rejects whichever one we didn't pick. An anonymous shape forces a
36
+ * structural comparison instead, which both versions satisfy, and which the
37
+ * `ref` prop accepts on both. This is the only place the 18-vs-19 split shows
38
+ * up in the binding; keep it that way.
39
+ */
40
+ export type AvatarMountRef = { current: HTMLDivElement | null };
41
+
42
+ export interface UseAvatarHandle {
43
+ /** Attach to the mount element: `<div ref={containerRef} />`. */
44
+ containerRef: AvatarMountRef;
45
+ }
46
+
47
+ export function useAvatar<O extends AvatarOptions = SvgAvatarOptions>(
48
+ { client, create, options }: UseAvatarOptions<O> = {},
49
+ ): UseAvatarHandle {
50
+ const containerRef = useRef<HTMLDivElement>(null);
51
+
52
+ // Latest-options ref, so the mount effect reads the live values without
53
+ // remounting the face every time a caller passes a fresh object literal.
54
+ const optionsRef = useRef(options);
55
+ optionsRef.current = options;
56
+
57
+ useEffect(() => {
58
+ const mount = containerRef.current;
59
+ if (!mount || !client) return;
60
+ const factory = (create ?? createSvgAvatar) as AvatarFactory<O>;
61
+ // The cast is the seam between "O minus the two we supply" and O. It is
62
+ // sound by construction and TypeScript cannot see through the spread of a
63
+ // generic; the two halves are typed at the boundary the caller touches.
64
+ const instance = factory({ mount, client, ...optionsRef.current } as unknown as O);
65
+ return () => instance.destroy();
66
+ // A new client identity is a new thing to embody, so the avatar is rebuilt
67
+ // rather than re-pointed. Hosts keep one `PipecatClient` across
68
+ // connect/disconnect cycles, so this does not fire on an ordinary
69
+ // reconnect; if yours constructs a fresh client per session, expect the
70
+ // face to remount with it.
71
+ }, [client, create]);
72
+
73
+ return { containerRef };
74
+ }
@@ -0,0 +1,26 @@
1
+ /** Polished professional male avatar, authored to read at call-tile size. */
2
+
3
+ import { createCanvasAvatar } from './createCanvasAvatar.js';
4
+ import type { CanvasAvatarOptions } from './createCanvasAvatar.js';
5
+ import type { AvatarInstance } from './createAvatar.js';
6
+
7
+ const RIG_URL = new URL('../src/canvas/data/professional-male-a.rig.json', import.meta.url);
8
+ const IMAGES = {
9
+ 'professional-male-a-top-body.webp': new URL('../src/canvas/data/img/professional-male-a-top-body.webp', import.meta.url),
10
+ 'professional-male-a-hair-back.webp': new URL('../src/canvas/data/img/professional-male-a-hair-back.webp', import.meta.url),
11
+ 'professional-male-a-hair-front.webp': new URL('../src/canvas/data/img/professional-male-a-hair-front.webp', import.meta.url),
12
+ } as const;
13
+ const FACE = () => import('../src/canvas/avatars/round/face.mjs');
14
+
15
+ export type { CanvasAvatarOptions };
16
+ // Same mixer, same clip library, so the same declaration (`supports.ts`).
17
+ export { supports } from './supports.js';
18
+
19
+ export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
20
+ return createCanvasAvatar(options, {
21
+ rigUrl: RIG_URL,
22
+ images: IMAGES,
23
+ face: FACE,
24
+ label: 'Vikram — polished professional male avatar',
25
+ });
26
+ }
@@ -0,0 +1,44 @@
1
+ /**
2
+ * Avatar — `createAvatar` as a call tile, and the whole of `@voqalize/avatar/react`.
3
+ *
4
+ * <Avatar client={session.client} className="avatar-tile" />
5
+ *
6
+ * Three props of its own; everything else is forwarded to the mount `<div>`, so
7
+ * it sizes and styles like the tile it lives in. There is nothing to configure
8
+ * because there is nothing the server does not already say: the
9
+ * `AvatarProcessor` in the pipeline drives the state, the gaze and the mouth.
10
+ *
11
+ * `create` is how you use a different avatar — any module exporting
12
+ * `createAvatar` (docs/design-avatar-interface.md), including a Rive or WebGL
13
+ * one. The component is generic over that implementation's options, so
14
+ * `options` is checked against the factory you passed rather than being a bag:
15
+ *
16
+ * <Avatar client={c} create={createMascot} options={{ mood: "sly" }} />
17
+ *
18
+ * With no `create` it is the bundled SVG avatar and `options` is
19
+ * `SvgAvatarOptions` minus `mount`/`client`, which the component supplies.
20
+ *
21
+ * The component renders an empty div and a static `role="img"`. It does not
22
+ * label the avatar with its current state: the implementation owns the DOM
23
+ * inside the mount and is the only thing that knows what it is portraying —
24
+ * and a live label would be this package reading back an avatar's internal
25
+ * state, which is the one thing the interface refuses to promise. Pass your
26
+ * own `aria-label` to override.
27
+ */
28
+ import type { HTMLAttributes } from "react";
29
+ import type { PipecatClient } from "@pipecat-ai/client-js";
30
+ import type { AvatarFactory, AvatarOptions, SvgAvatarOptions } from "./createAvatar.js";
31
+ /** An implementation's own options: everything it takes past the two the
32
+ * component supplies itself. */
33
+ export type ImplementationOptions<O extends AvatarOptions> = Omit<O, keyof AvatarOptions>;
34
+ export type AvatarProps<O extends AvatarOptions = SvgAvatarOptions> = Omit<HTMLAttributes<HTMLDivElement>, "children"> & {
35
+ /** The live `PipecatClient`, or `null` before connect. Nothing renders
36
+ * until it is non-null. */
37
+ client?: PipecatClient | null;
38
+ /** An avatar implementation. Omit for the bundled SVG faces. */
39
+ create?: AvatarFactory<O>;
40
+ /** Options for that implementation. Read at mount only. */
41
+ options?: ImplementationOptions<O>;
42
+ };
43
+ export declare function Avatar<O extends AvatarOptions = SvgAvatarOptions>({ client, create, options, ...rest }: AvatarProps<O>): import("react").JSX.Element;
44
+ //# sourceMappingURL=Avatar.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Avatar.d.ts","sourceRoot":"","sources":["../client/Avatar.tsx"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,OAAO,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAExF;gCACgC;AAChC,MAAM,MAAM,qBAAqB,CAAC,CAAC,SAAS,aAAa,IAAI,IAAI,CAAC,CAAC,EAAE,MAAM,aAAa,CAAC,CAAC;AAE1F,MAAM,MAAM,WAAW,CAAC,CAAC,SAAS,aAAa,GAAG,gBAAgB,IAChE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,EAAE,UAAU,CAAC,GAAG;IACjD;+BAC2B;IAC3B,MAAM,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,gEAAgE;IAChE,MAAM,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;IAC1B,2DAA2D;IAC3D,OAAO,CAAC,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC;CACpC,CAAC;AAEJ,wBAAgB,MAAM,CAAC,CAAC,SAAS,aAAa,GAAG,gBAAgB,EAAE,EACjE,MAAM,EACN,MAAM,EACN,OAAO,EACP,GAAG,IAAI,EACR,EAAE,WAAW,CAAC,CAAC,CAAC,+BAGhB"}
@@ -1,7 +1,7 @@
1
1
  import { jsx as _jsx } from "react/jsx-runtime";
2
2
  import { useAvatar } from "./useAvatar.js";
3
- export function Avatar({ client, avatar, ...rest }) {
4
- const { containerRef } = useAvatar({ client, avatar });
3
+ export function Avatar({ client, create, options, ...rest }) {
4
+ const { containerRef } = useAvatar({ client, create, options });
5
5
  return _jsx("div", { role: "img", "aria-label": "avatar", ...rest, ref: containerRef });
6
6
  }
7
7
  //# sourceMappingURL=Avatar.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Avatar.js","sourceRoot":"","sources":["../client/Avatar.tsx"],"names":[],"mappings":";AA8BA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAkB3C,MAAM,UAAU,MAAM,CAA6C,EACjE,MAAM,EACN,MAAM,EACN,OAAO,EACP,GAAG,IAAI,EACQ;IACf,MAAM,EAAE,YAAY,EAAE,GAAG,SAAS,CAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACnE,OAAO,cAAK,IAAI,EAAC,KAAK,gBAAY,QAAQ,KAAK,IAAI,EAAE,GAAG,EAAE,YAAY,GAAI,CAAC;AAC7E,CAAC"}