@voqalize/avatar 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/LICENSE +21 -661
  2. package/README.md +144 -663
  3. package/client/Avatar.tsx +57 -0
  4. package/client/AvatarClient.ts +523 -0
  5. package/client/arjun.ts +24 -0
  6. package/client/createAvatar.ts +139 -0
  7. package/client/createCanvasAvatar.ts +72 -0
  8. package/client/index.ts +39 -0
  9. package/client/internal.ts +80 -0
  10. package/client/interviewer-female.ts +4 -0
  11. package/client/interviewer-male.ts +4 -0
  12. package/client/ishita.ts +24 -0
  13. package/client/kabir.ts +24 -0
  14. package/client/meera.ts +24 -0
  15. package/client/naina.ts +24 -0
  16. package/client/professional-female-a.ts +4 -0
  17. package/client/professional-female-b.ts +4 -0
  18. package/client/professional-male-a.ts +4 -0
  19. package/client/professional-male-b.ts +4 -0
  20. package/client/react.ts +13 -0
  21. package/client/types.ts +146 -0
  22. package/client/useAvatar.ts +74 -0
  23. package/client/vikram.ts +24 -0
  24. package/dist/Avatar.d.ts +44 -0
  25. package/dist/Avatar.d.ts.map +1 -0
  26. package/{client/dist → dist}/Avatar.js +2 -2
  27. package/dist/Avatar.js.map +1 -0
  28. package/dist/AvatarClient.d.ts +196 -0
  29. package/dist/AvatarClient.d.ts.map +1 -0
  30. package/dist/AvatarClient.js +450 -0
  31. package/dist/AvatarClient.js.map +1 -0
  32. package/dist/arjun.d.ts +6 -0
  33. package/dist/arjun.d.ts.map +1 -0
  34. package/dist/arjun.js +18 -0
  35. package/dist/arjun.js.map +1 -0
  36. package/dist/createAvatar.d.ts +99 -0
  37. package/dist/createAvatar.d.ts.map +1 -0
  38. package/dist/createAvatar.js +75 -0
  39. package/dist/createAvatar.js.map +1 -0
  40. package/dist/createCanvasAvatar.d.ts +22 -0
  41. package/dist/createCanvasAvatar.d.ts.map +1 -0
  42. package/dist/createCanvasAvatar.js +47 -0
  43. package/dist/createCanvasAvatar.js.map +1 -0
  44. package/dist/index.d.ts +30 -0
  45. package/dist/index.d.ts.map +1 -0
  46. package/dist/index.js +29 -0
  47. package/dist/index.js.map +1 -0
  48. package/dist/internal.d.ts +30 -0
  49. package/dist/internal.d.ts.map +1 -0
  50. package/dist/internal.js +33 -0
  51. package/dist/internal.js.map +1 -0
  52. package/dist/interviewer-female.d.ts +4 -0
  53. package/dist/interviewer-female.d.ts.map +1 -0
  54. package/dist/interviewer-female.js +3 -0
  55. package/dist/interviewer-female.js.map +1 -0
  56. package/dist/interviewer-male.d.ts +4 -0
  57. package/dist/interviewer-male.d.ts.map +1 -0
  58. package/dist/interviewer-male.js +3 -0
  59. package/dist/interviewer-male.js.map +1 -0
  60. package/dist/ishita.d.ts +6 -0
  61. package/dist/ishita.d.ts.map +1 -0
  62. package/dist/ishita.js +18 -0
  63. package/dist/ishita.js.map +1 -0
  64. package/dist/kabir.d.ts +6 -0
  65. package/dist/kabir.d.ts.map +1 -0
  66. package/dist/kabir.js +18 -0
  67. package/dist/kabir.js.map +1 -0
  68. package/dist/meera.d.ts +6 -0
  69. package/dist/meera.d.ts.map +1 -0
  70. package/dist/meera.js +18 -0
  71. package/dist/meera.js.map +1 -0
  72. package/dist/naina.d.ts +6 -0
  73. package/dist/naina.d.ts.map +1 -0
  74. package/dist/naina.js +18 -0
  75. package/dist/naina.js.map +1 -0
  76. package/dist/professional-female-a.d.ts +4 -0
  77. package/dist/professional-female-a.d.ts.map +1 -0
  78. package/dist/professional-female-a.js +3 -0
  79. package/dist/professional-female-a.js.map +1 -0
  80. package/dist/professional-female-b.d.ts +4 -0
  81. package/dist/professional-female-b.d.ts.map +1 -0
  82. package/dist/professional-female-b.js +3 -0
  83. package/dist/professional-female-b.js.map +1 -0
  84. package/dist/professional-male-a.d.ts +4 -0
  85. package/dist/professional-male-a.d.ts.map +1 -0
  86. package/dist/professional-male-a.js +3 -0
  87. package/dist/professional-male-a.js.map +1 -0
  88. package/dist/professional-male-b.d.ts +4 -0
  89. package/dist/professional-male-b.d.ts.map +1 -0
  90. package/dist/professional-male-b.js +3 -0
  91. package/dist/professional-male-b.js.map +1 -0
  92. package/dist/react.d.ts +13 -0
  93. package/dist/react.d.ts.map +1 -0
  94. package/dist/react.js +13 -0
  95. package/dist/react.js.map +1 -0
  96. package/dist/types.d.ts +94 -0
  97. package/dist/types.d.ts.map +1 -0
  98. package/dist/types.js +93 -0
  99. package/dist/types.js.map +1 -0
  100. package/dist/useAvatar.d.ts +39 -0
  101. package/dist/useAvatar.d.ts.map +1 -0
  102. package/dist/useAvatar.js +35 -0
  103. package/dist/useAvatar.js.map +1 -0
  104. package/dist/vikram.d.ts +6 -0
  105. package/dist/vikram.d.ts.map +1 -0
  106. package/dist/vikram.js +18 -0
  107. package/dist/vikram.js.map +1 -0
  108. package/package.json +81 -24
  109. package/src/avatar.d.ts +161 -120
  110. package/src/avatar.js +234 -146
  111. package/src/behavior.d.ts +26 -0
  112. package/src/behavior.js +102 -0
  113. package/src/camera.js +29 -0
  114. package/src/canvas/author/parts/eye.mjs +722 -0
  115. package/src/canvas/author/parts/hand.mjs +1156 -0
  116. package/src/canvas/author/parts/mouth.mjs +741 -0
  117. package/src/canvas/author/parts/nose.mjs +100 -0
  118. package/src/canvas/author/parts/skin-detail.mjs +67 -0
  119. package/src/canvas/author/path.mjs +283 -0
  120. package/src/canvas/author/rig.mjs +405 -0
  121. package/src/canvas/avatars/round/face.d.mts +3 -0
  122. package/src/canvas/avatars/round/face.mjs +1307 -0
  123. package/src/canvas/create-rig.d.ts +15 -0
  124. package/src/canvas/create-rig.js +100 -0
  125. package/src/canvas/data/img/professional-female-a-hair-back.webp +0 -0
  126. package/src/canvas/data/img/professional-female-a-hair-front.webp +0 -0
  127. package/src/canvas/data/img/professional-female-a-top-body.webp +0 -0
  128. package/src/canvas/data/img/professional-female-b-hair-back.webp +0 -0
  129. package/src/canvas/data/img/professional-female-b-hair-front.webp +0 -0
  130. package/src/canvas/data/img/professional-female-b-top-body.webp +0 -0
  131. package/src/canvas/data/img/professional-male-a-hair-back.webp +0 -0
  132. package/src/canvas/data/img/professional-male-a-hair-front.webp +0 -0
  133. package/src/canvas/data/img/professional-male-a-top-body.webp +0 -0
  134. package/src/canvas/data/img/professional-male-b-hair-back.webp +0 -0
  135. package/src/canvas/data/img/professional-male-b-hair-front.webp +0 -0
  136. package/src/canvas/data/img/professional-male-b-top-body.webp +0 -0
  137. package/src/canvas/data/img/round-m3-hair-back.webp +0 -0
  138. package/src/canvas/data/img/round-m3-hair-front.webp +0 -0
  139. package/src/canvas/data/img/round-m3-top-body.webp +0 -0
  140. package/src/canvas/data/img/round-w1-hair-back.webp +0 -0
  141. package/src/canvas/data/img/round-w1-hair-front.webp +0 -0
  142. package/src/canvas/data/img/round-w1-top-body.webp +0 -0
  143. package/src/canvas/data/interviewer-female.rig.json +1 -0
  144. package/src/canvas/data/interviewer-male.rig.json +1 -0
  145. package/src/canvas/data/professional-female-a.rig.json +1 -0
  146. package/src/canvas/data/professional-female-b.rig.json +1 -0
  147. package/src/canvas/data/professional-male-a.rig.json +1 -0
  148. package/src/canvas/data/professional-male-b.rig.json +1 -0
  149. package/src/canvas/src/live.js +508 -0
  150. package/src/canvas/src/render2d.js +218 -0
  151. package/src/canvas/src/rig.js +297 -0
  152. package/src/canvas/src/vocab.js +96 -0
  153. package/src/clips.js +18 -7
  154. package/src/conformance.js +119 -0
  155. package/src/emotions.js +4 -3
  156. package/src/face-core.js +27 -1
  157. package/src/face-myna.d.ts +7 -0
  158. package/src/face-myna.js +192 -131
  159. package/src/face-peep-control-plane.js +167 -0
  160. package/src/face-peep.d.ts +7 -0
  161. package/src/face-peep.js +274 -116
  162. package/src/face-wren.d.ts +7 -0
  163. package/src/face-wren.js +17 -19
  164. package/src/faces.d.ts +12 -0
  165. package/src/faces.js +53 -0
  166. package/src/gaze.js +119 -4
  167. package/src/hand.js +120 -91
  168. package/src/idle.js +18 -101
  169. package/src/interjections.js +156 -31
  170. package/src/params.js +8 -4
  171. package/src/perform.js +7 -9
  172. package/src/rig.d.ts +24 -0
  173. package/src/rig.js +32 -0
  174. package/src/speech-timing.js +23 -0
  175. package/src/visemes.js +28 -12
  176. package/client/dist/Avatar.d.ts +0 -27
  177. package/client/dist/Avatar.d.ts.map +0 -1
  178. package/client/dist/Avatar.js.map +0 -1
  179. package/client/dist/AvatarClient.d.ts +0 -139
  180. package/client/dist/AvatarClient.d.ts.map +0 -1
  181. package/client/dist/AvatarClient.js +0 -223
  182. package/client/dist/AvatarClient.js.map +0 -1
  183. package/client/dist/index.d.ts +0 -22
  184. package/client/dist/index.d.ts.map +0 -1
  185. package/client/dist/index.js +0 -22
  186. package/client/dist/index.js.map +0 -1
  187. package/client/dist/types.d.ts +0 -86
  188. package/client/dist/types.d.ts.map +0 -1
  189. package/client/dist/types.js +0 -31
  190. package/client/dist/types.js.map +0 -1
  191. package/client/dist/useAvatar.d.ts +0 -42
  192. package/client/dist/useAvatar.d.ts.map +0 -1
  193. package/client/dist/useAvatar.js +0 -51
  194. package/client/dist/useAvatar.js.map +0 -1
  195. package/client/src/Avatar.tsx +0 -33
  196. package/client/src/AvatarClient.ts +0 -270
  197. package/client/src/index.ts +0 -22
  198. package/client/src/types.ts +0 -104
  199. package/client/src/useAvatar.ts +0 -85
  200. package/docs/contract-avatar.md +0 -371
  201. package/docs/contract-protocol.md +0 -461
@@ -1 +0,0 @@
1
- {"version":3,"file":"useAvatar.js","sourceRoot":"","sources":["../src/useAvatar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAEpD,OAAO,EAAE,YAAY,EAAkB,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AA+BjD,MAAM,UAAU,SAAS,CAAC,UAA4B,EAAE;IACtD,MAAM,YAAY,GAAG,MAAM,CAAiB,IAAI,CAAC,CAAC;IAClD,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,QAAQ,CAAmB,IAAI,CAAC,CAAC;IAC7D,MAAM,eAAe,GAAG,MAAM,CAAsB,IAAI,CAAC,CAAC;IAE1D,4EAA4E;IAC5E,qCAAqC;IACrC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IACnC,UAAU,CAAC,OAAO,GAAG,OAAO,CAAC;IAE7B,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC;QACnC,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5E,eAAe,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,QAAQ,CAAC,CAAC;QACrD,SAAS,CAAC,QAAQ,CAAC,CAAC;QAEpB,OAAO,GAAG,EAAE;YACV,QAAQ,CAAC,OAAO,EAAE,CAAC;YACnB,eAAe,CAAC,OAAO,GAAG,IAAI,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC,CAAC;QACF,sEAAsE;QACtE,yEAAyE;QACzE,0EAA0E;QAC1E,wDAAwD;QACxD,uDAAuD;IACzD,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;QACrC,IAAI,CAAC,OAAO,IAAI,CAAC,aAAa;YAAE,OAAO;QACvC,OAAO,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACrC,uEAAuE;QACvE,uDAAuD;IACzD,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAE7B,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;AAClC,CAAC"}
@@ -1,33 +0,0 @@
1
- /**
2
- * Avatar — a call-tile-ready wrapper around the widget, and the package's
3
- * whole public surface.
4
- *
5
- * <Avatar client={session.client} className="avatar-tile" />
6
- *
7
- * Two props of its own; everything else is forwarded to the mount `<div>`, so
8
- * it sizes and styles like the tile it lives in. There is nothing to configure
9
- * because there is nothing the server does not already say: the
10
- * `AvatarProcessor` in the pipeline drives the state, the gaze and the mouth.
11
- *
12
- * The widget has no hot-swap-avatar API — `createFace` runs once per mount —
13
- * so `avatar` is read once, at mount. To switch faces at runtime, remount with
14
- * a `key` prop:
15
- *
16
- * <Avatar key={name} avatar={name} client={session.client} />
17
- */
18
-
19
- import type { HTMLAttributes } from "react";
20
- import type { PipecatClient } from "@pipecat-ai/client-js";
21
- import { useAvatar } from "./useAvatar.js";
22
-
23
- export interface AvatarProps extends HTMLAttributes<HTMLDivElement> {
24
- /** The live `PipecatClient`, or `null` before connect. */
25
- client?: PipecatClient | null;
26
- /** Which face. Omit for the default. Read at mount only — see above. */
27
- avatar?: string;
28
- }
29
-
30
- export function Avatar({ client, avatar, ...rest }: AvatarProps) {
31
- const { containerRef } = useAvatar({ client, avatar });
32
- return <div role="img" aria-label="avatar" {...rest} ref={containerRef} />;
33
- }
@@ -1,270 +0,0 @@
1
- /**
2
- * AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
3
- * splice, framework-free (no React; the hook and component wrap this).
4
- *
5
- * ## Turn clock anchoring
6
- *
7
- * A turn's `t0` is anchored to `performance.now()` **at the moment this client
8
- * receives the `{cmd:"speech", event:"start"}` message** — cues are
9
- * client-anchored, not per-cue server-released. That message rides the RTVI
10
- * data channel, ahead of the jitter-buffered audio path, so the residual error
11
- * lands on the video-leads side — the side `docs/contract-protocol.md` says
12
- * perceptual tolerance favours (+125 ms vs -45 ms).
13
- *
14
- * We investigated anchoring on pipecat client-js's own `RTVIEvent
15
- * .BotStartedSpeaking`/`BotStoppedSpeaking` instead (or as a refinement) and
16
- * chose not to, for two reasons:
17
- *
18
- * 1. **No turn correlation.** Those events carry no payload — no `ctx` — so
19
- * there is no way to tell which turn a firing belongs to. Our own
20
- * `speech` command carries `ctx`, which the splice logic below needs
21
- * regardless, so anchoring off it costs nothing extra.
22
- * 2. **Same source, same path, no accuracy gain.** The `AvatarProcessor`
23
- * sits between the TTS service and the output transport and observes the
24
- * transport's own `BotStarted/StoppedSpeakingFrame` broadcasts — the exact
25
- * frame pipecat's built-in speaking detection is *also* driven from. Both
26
- * notifications travel the same data-channel path to the browser. There is
27
- * no local "truly audible now" signal cheaply available: the audio arrives
28
- * on a `MediaStreamTrack` whose only lifecycle events (`unmute`/`mute`)
29
- * fire once per call, not per utterance. Tapping the decoded remote audio
30
- * with a WebAudio `AnalyserNode` RMS gate *would* give one, but it adds
31
- * its own onset latency and a real audio pipeline to build and tune, and
32
- * it would eat into the intentional video-first safety margin rather than
33
- * improve it. Left as a documented option, not built.
34
- *
35
- * `attach()` therefore subscribes to exactly one pipecat event,
36
- * `serverMessage`. It used to also subscribe to both speaking events to report
37
- * a diagnostic drift between our anchor and pipecat's; that hook is gone with
38
- * the rest of the observability surface (`docs/removed.md` § Client callbacks).
39
- *
40
- * ## Cue splice
41
- *
42
- * The widget has two cue-track primitives: `speak({cues, clock})` (a full
43
- * replace) and `pushCues(cues)` (a pure union that can only grow the track,
44
- * never shrink it). Neither is "discard queued cues at or after `from_ms`,
45
- * then append" on its own — `pushCues` has no way to drop a stale tail. So
46
- * this client keeps the turn's canonical cue array itself (kept portion +
47
- * every appended chunk, spliced on each `cues` message) and picks the cheapest
48
- * widget call that stays correct:
49
- *
50
- * - if the splice's `from_ms` doesn't reach back into anything already
51
- * queued — the common case past a turn's first sentence, since only the
52
- * first sentence genuinely plays fast-leg cues — nothing needs discarding:
53
- * `pushCues(newCues)` is the cheap, correct append.
54
- * - if it does reach back (a real fast→accurate splice), `pushCues` cannot
55
- * express the discard; we call `speak()` again with the full spliced
56
- * canonical array on the turn's original clock. `speak()` is otherwise
57
- * documented as also killing an in-flight spoken interjection and
58
- * re-entering `SPEAKING` — both harmless mid-splice (an interjection
59
- * should not be running while a server track owns the mouth; re-entering
60
- * an unchanged state is a no-op past the profile/gaze reset the widget
61
- * already does for a same-name `setState`).
62
- *
63
- * Cues commonly arrive **before** `speech start` — the fast leg starts the
64
- * moment a sentence is handed to TTS, well before `BotStartedSpeakingFrame`.
65
- * Chunks that arrive before the clock is anchored are spliced into the
66
- * canonical array but not yet handed to the widget; `speech start` hands over
67
- * whatever has accumulated as the turn's first `speak()` call. So "the first
68
- * chunk of a turn starts speak()" means the first *widget* call, not
69
- * necessarily the first *message*.
70
- */
71
-
72
- import type { PipecatClient, RTVIEvent } from "@pipecat-ai/client-js";
73
- import type { AvatarApi } from "../../src/avatar.js";
74
- import {
75
- isAvatarMessage,
76
- type AvatarCommand,
77
- type AvatarCue,
78
- type AvatarCuesCmd,
79
- type AvatarSpeechCmd,
80
- type AvatarStateCmd,
81
- } from "./types.js";
82
-
83
-
84
- interface Turn {
85
- ctx: string;
86
- /** The canonical, already-spliced cue track for this turn. */
87
- cues: AvatarCue[];
88
- /** Whether `speech start` has anchored a clock and issued the first `speak()`. */
89
- started: boolean;
90
- clock: (() => number) | null;
91
- }
92
-
93
- /**
94
- * Internal. Not exported from the package — see `index.ts` for the public
95
- * surface, which is `<Avatar>` and nothing else.
96
- *
97
- * There is deliberately no `accept` predicate here any more. Avatar commands
98
- * travel in one envelope, `{type:"avatar"}`, in both directions and from every
99
- * source: a `AvatarProcessor` in the pipeline and a brain driving the face
100
- * out of band emit the same shape. A per-deployment predicate meant the
101
- * library could not state what an avatar message *is*, which is the one thing
102
- * a wire format has to be able to say. See docs/removed.md § The accept
103
- * predicate.
104
- */
105
- export interface AvatarClientOptions {
106
- /** A dispatch threw (e.g. an unknown state or interjection id, which the
107
- * widget throws on). Defaults to `console.warn`. */
108
- onError?: (err: unknown, msg: AvatarCommand) => void;
109
- /** Override for tests. Defaults to `performance.now`. */
110
- now?: () => number;
111
- }
112
-
113
- /**
114
- * The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
115
- *
116
- * Written out rather than imported because that enum was this module's *only*
117
- * runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
118
- * the whole `/pipecat` subpath fail to load without the peer installed — even
119
- * for a host that drives `dispatch()` from its own transport and never calls
120
- * `attach()`. The peer is declared optional; this is what makes that true
121
- * rather than aspirational.
122
- *
123
- * String enums are nominal in TypeScript, so the compiler cannot check these
124
- * against the real ones from a type-only import. `client/test/AvatarClient.test.ts`
125
- * does it instead, against the actual enum — the devDependency is present
126
- * exactly where the check belongs and absent from what we ship.
127
- */
128
- export const RTVI_EVENTS = {
129
- serverMessage: "serverMessage",
130
- } as const satisfies Record<string, string>;
131
-
132
- /** Defensive unwrap for the `RTVIEvent.ServerMessage` `{ data }` quirk: some
133
- * transports deliver the payload directly and some wrap it once more. */
134
- function unwrapServerMessage(raw: unknown): Record<string, unknown> {
135
- const obj = (raw ?? {}) as Record<string, unknown>;
136
- const inner = obj["data"] as Record<string, unknown> | undefined;
137
- return inner && "type" in inner ? inner : obj;
138
- }
139
-
140
- export class AvatarClient {
141
- private readonly avatar: AvatarApi;
142
- private readonly opts: AvatarClientOptions;
143
- private readonly now: () => number;
144
- private turn: Turn | null = null;
145
-
146
- constructor(avatar: AvatarApi, opts: AvatarClientOptions = {}) {
147
- this.avatar = avatar;
148
- this.opts = opts;
149
- this.now = opts.now ?? (() => performance.now());
150
- }
151
-
152
- /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
153
- get turnCtx(): string | null {
154
- return this.turn?.ctx ?? null;
155
- }
156
-
157
- /** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
158
- get turnCues(): AvatarCue[] {
159
- return this.turn ? [...this.turn.cues] : [];
160
- }
161
-
162
- /** Dispatch one server message. Anything that isn't in the avatar envelope
163
- * is not ours and is ignored; so is an envelope carrying a `cmd` this build
164
- * has never heard of, per the wire protocol's forward-compat rule. */
165
- dispatch(raw: unknown): void {
166
- if (!isAvatarMessage(raw)) return;
167
- const msg: AvatarCommand = raw;
168
- try {
169
- switch (msg.cmd) {
170
- case "state":
171
- this.handleState(msg);
172
- break;
173
- case "interject":
174
- this.avatar.interject(msg.id);
175
- break;
176
- case "gesture":
177
- this.avatar.gesture(msg.id);
178
- break;
179
- case "cues":
180
- this.handleCues(msg);
181
- break;
182
- case "speech":
183
- this.handleSpeech(msg);
184
- break;
185
- case "user":
186
- this.avatar.setUserSpeaking(msg.speaking);
187
- break;
188
- // No default: an unknown `cmd` is a newer server talking to an older
189
- // widget, and the protocol's forward-compat rule says ignore it. There
190
- // is no callback for it — a hook nobody could act on is observability,
191
- // not an interface (`docs/removed.md` § Client callbacks).
192
- }
193
- } catch (err) {
194
- if (this.opts.onError) this.opts.onError(err, msg);
195
- else console.warn("[avatar] dispatch failed", msg, err);
196
- }
197
- }
198
-
199
- private handleState(msg: AvatarStateCmd) {
200
- // Deliberately no client-side dedup: pass every `state` command straight
201
- // through. The widget's own setState already no-ops the parts that matter
202
- // for an unchanged name (`changed` gates the blink and the 'state' event in
203
- // avatar.js), and a server resending the same state name as a
204
- // keepalive/resync must still land so an `emotion`/`gaze` override on this
205
- // particular message takes effect.
206
- this.avatar.setState(msg.name, { emotion: msg.emotion, gaze: msg.gaze });
207
- }
208
-
209
- private ensureTurn(ctx: string): Turn {
210
- if (!this.turn || this.turn.ctx !== ctx) {
211
- // A different ctx supersedes whatever turn we had — a stale trailing
212
- // message for the old ctx will find `this.turn.ctx !== ctx` in
213
- // handleSpeech's stop-guard and be ignored, rather than cutting off the
214
- // new turn.
215
- this.turn = { ctx, cues: [], started: false, clock: null };
216
- }
217
- return this.turn;
218
- }
219
-
220
- private handleCues(msg: AvatarCuesCmd) {
221
- const turn = this.ensureTurn(msg.ctx);
222
- const kept = turn.cues.filter((c) => c.t < msg.from_ms);
223
- const discarded = turn.cues.length - kept.length;
224
- turn.cues = [...kept, ...msg.cues].sort((a, b) => a.t - b.t);
225
-
226
- if (!turn.started) {
227
- // No clock yet — buffer. `speech start` will hand this over as the turn's
228
- // first speak() call.
229
- return;
230
- }
231
- if (discarded === 0) {
232
- this.avatar.pushCues(msg.cues);
233
- } else {
234
- this.avatar.speak({ cues: turn.cues, clock: turn.clock! });
235
- }
236
- }
237
-
238
- private handleSpeech(msg: AvatarSpeechCmd) {
239
- if (msg.event === "start") {
240
- const turn = this.ensureTurn(msg.ctx);
241
- const t0 = this.now();
242
- const clock = () => this.now() - t0;
243
- turn.clock = clock;
244
- turn.started = true;
245
- this.avatar.speak({ cues: turn.cues, clock });
246
- return;
247
- }
248
- // "stop": only act if it names the turn we're actually riding. A stale stop
249
- // for an already-superseded ctx must not cut off a newer turn.
250
- if (this.turn && this.turn.ctx === msg.ctx) {
251
- this.avatar.stopSpeaking();
252
- this.turn = null;
253
- }
254
- }
255
-
256
- /**
257
- * Subscribe to a live `PipecatClient`'s server messages and dispatch the
258
- * avatar commands among them — the ones in the protocol's own
259
- * `{type:"avatar"}` envelope, which `isAvatarMessage` is the definition of.
260
- * Never throws on a malformed or irrelevant message.
261
- *
262
- * @returns an unsubscribe function; call it on unmount or disconnect.
263
- */
264
- attach(client: PipecatClient): () => void {
265
- const onServerMessage = (raw: unknown) => this.dispatch(unwrapServerMessage(raw));
266
- const serverMessage = RTVI_EVENTS.serverMessage as RTVIEvent;
267
- client.on(serverMessage, onServerMessage);
268
- return () => client.off(serverMessage, onServerMessage);
269
- }
270
- }
@@ -1,22 +0,0 @@
1
- /**
2
- * `@voqalize/avatar` — the avatar as one React component.
3
- *
4
- * import { Avatar } from "@voqalize/avatar";
5
- *
6
- * <Avatar client={pipecatClient} className="avatar-tile" />
7
- *
8
- * That is the whole public surface. Put an `AvatarProcessor` in the pipecat
9
- * pipeline (`pip install voqalize-avatar`), drop this component into the bot's
10
- * tile, and the face listens, thinks, claims the floor and lipsyncs what the
11
- * TTS says.
12
- *
13
- * The widget underneath is framework-free, and the dispatcher between it and
14
- * the RTVI data channel is plain TypeScript — but neither is exported. Two
15
- * consumers wanted a call tile, both are React, and a public API is a promise
16
- * we have to keep across versions. `docs/removed.md` lists what used to be
17
- * here and how to get it back if a real third case argues for it.
18
- *
19
- * Peers: `react >= 18` and `@pipecat-ai/client-js`.
20
- */
21
-
22
- export { Avatar, type AvatarProps } from "./Avatar.js";
@@ -1,104 +0,0 @@
1
- /**
2
- * types.ts — the avatar wire vocabulary, client side.
3
- *
4
- * The binding definition is `docs/contract-protocol.md`; this file is its
5
- * TypeScript restatement and must not drift from it. The Python half of the
6
- * same vocabulary is `py/src/voqalize_avatar/messages.py` — the three are
7
- * maintained together, and a command added to one without the others is
8
- * incomplete.
9
- *
10
- * A server pushes these as RTVI `server-message`s under the envelope
11
- * `{ type: "avatar", ...cmd-specific fields }`. {@link AvatarCommand} describes
12
- * what rides inside that envelope; {@link isAvatarMessage} is the envelope
13
- * itself, and is the only definition of "this message is for the avatar" the
14
- * client has. There is no protocol version field — see `docs/removed.md`
15
- * § The `v` field.
16
- */
17
-
18
- /** A viseme cue: `t` is a ms offset into the utterance's clock, `v` is a Rhubarb A–H (or X) letter. */
19
- export interface AvatarCue {
20
- t: number;
21
- v: string;
22
- i?: number;
23
- }
24
-
25
- export interface AvatarStateCmd {
26
- cmd: "state";
27
- name: string;
28
- emotion?: string;
29
- gaze?: string;
30
- }
31
-
32
- export interface AvatarInterjectCmd {
33
- cmd: "interject";
34
- id: string;
35
- }
36
-
37
- /**
38
- * A hand gesture — the hand at the frame edge plus its face half. Separate from
39
- * `interject` on purpose: `interject("WAVE")` is the face alone and always was,
40
- * so a server that upgrades gets no hand until it asks for one.
41
- */
42
- export interface AvatarGestureCmd {
43
- cmd: "gesture";
44
- id: string;
45
- }
46
-
47
- export interface AvatarCuesCmd {
48
- cmd: "cues";
49
- ctx: string;
50
- /** Discard queued cues at or after this offset (ms), then append `cues`. */
51
- from_ms: number;
52
- cues: AvatarCue[];
53
- /**
54
- * True on the one chunk that completes this turn's track: the TTS context is
55
- * closed, so no further chunk will splice into `ctx`. What a client may
56
- * assume, exactly — nothing about playout. The audio it describes is still
57
- * ahead, and `speech stop` remains the end of the turn. It is safe to release
58
- * per-turn cue state (the splice buffer for `ctx`) once the last cue has
59
- * played, and safe to stop expecting more.
60
- *
61
- * Absent on an interrupted turn, deliberately: a turn that was cut never
62
- * claims to have completed. Absent chunks are the normal case — the widget's
63
- * own track already completes on the trailing `X`, so ignoring `final`
64
- * entirely is a correct implementation.
65
- */
66
- final?: boolean;
67
- }
68
-
69
- export interface AvatarSpeechCmd {
70
- cmd: "speech";
71
- event: "start" | "stop";
72
- ctx: string;
73
- }
74
-
75
- export interface AvatarUserCmd {
76
- cmd: "user";
77
- speaking: boolean;
78
- }
79
-
80
- export type AvatarCommand =
81
- | AvatarStateCmd
82
- | AvatarInterjectCmd
83
- | AvatarGestureCmd
84
- | AvatarCuesCmd
85
- | AvatarSpeechCmd
86
- | AvatarUserCmd;
87
-
88
- /** The full server-message payload: the envelope plus its command. */
89
- export type AvatarServerMessage = AvatarCommand & { type: "avatar" };
90
-
91
- /**
92
- * Is this server-message payload the avatar's? The envelope is the whole
93
- * answer: `{type:"avatar"}` with a string `cmd`. It used to be a per-deployment
94
- * `accept` predicate on the client, which meant the library could not state
95
- * what an avatar message *is* — see `docs/removed.md` § The accept predicate.
96
- */
97
- export function isAvatarMessage(msg: unknown): msg is AvatarServerMessage {
98
- if (typeof msg !== "object" || msg === null) return false;
99
- const m = msg as Record<string, unknown>;
100
- return m.type === AVATAR_MESSAGE_TYPE && typeof m.cmd === "string";
101
- }
102
-
103
- /** The envelope `type` the protocol reserves for avatar traffic. */
104
- export const AVATAR_MESSAGE_TYPE = "avatar";
@@ -1,85 +0,0 @@
1
- /**
2
- * useAvatar — mount the widget, wire it to a live session, dispatch its
3
- * server-messages, and clean up.
4
- *
5
- * Internal: `<Avatar>` is the only thing the package exports. It is a separate
6
- * module anyway because the two lifecycles genuinely differ — mounting the
7
- * widget happens once (an avatar swap remounts by design; see the note in the
8
- * effect), while attaching to the pipecat client re-runs whenever the client
9
- * identity changes, which a session reconnect makes it do.
10
- */
11
-
12
- import { useEffect, useRef, useState } from "react";
13
- import type { PipecatClient } from "@pipecat-ai/client-js";
14
- import { createAvatar, type AvatarApi } from "../../src/avatar.js";
15
- import { AvatarClient } from "./AvatarClient.js";
16
-
17
- export interface UseAvatarOptions {
18
- /** Which face. Omit for the widget's own default. */
19
- avatar?: string;
20
- /** The live `PipecatClient` to dispatch server-messages from, or `null`
21
- * before connect. `useAvatar` (dis)connects the subscription as this
22
- * changes; it does not create or own the client. */
23
- client?: PipecatClient | null;
24
- }
25
-
26
- /**
27
- * The mount ref's type, written out rather than named as React's `RefObject`.
28
- *
29
- * React 18 and 19 declare that alias with different type arguments — 18's
30
- * `useRef<T>(null)` yields `RefObject<T>`, 19's yields `RefObject<T | null>` —
31
- * and because both are the *same alias*, TypeScript compares them by variance
32
- * and rejects whichever one we didn't pick. An anonymous shape forces a
33
- * structural comparison instead, which both versions satisfy, and which the
34
- * `ref` prop accepts on both. This is the only place the 18-vs-19 split shows
35
- * up in the binding; keep it that way.
36
- */
37
- export type AvatarMountRef = { current: HTMLDivElement | null };
38
-
39
- export interface UseAvatarHandle {
40
- /** Attach to the mount element: `<div ref={containerRef} />`. */
41
- containerRef: AvatarMountRef;
42
- /** The live widget instance once mounted, else `null`. */
43
- avatar: AvatarApi | null;
44
- }
45
-
46
- export function useAvatar(options: UseAvatarOptions = {}): UseAvatarHandle {
47
- const containerRef = useRef<HTMLDivElement>(null);
48
- const [avatar, setAvatar] = useState<AvatarApi | null>(null);
49
- const avatarClientRef = useRef<AvatarClient | null>(null);
50
-
51
- // Latest-options ref, so the mount effect (which runs once) still reads the
52
- // live props without re-subscribing.
53
- const optionsRef = useRef(options);
54
- optionsRef.current = options;
55
-
56
- useEffect(() => {
57
- const mount = containerRef.current;
58
- if (!mount) return;
59
- const instance = createAvatar({ mount, avatar: optionsRef.current.avatar });
60
- avatarClientRef.current = new AvatarClient(instance);
61
- setAvatar(instance);
62
-
63
- return () => {
64
- instance.destroy();
65
- avatarClientRef.current = null;
66
- setAvatar(null);
67
- };
68
- // Mount once. `avatar` is read at mount time only — the widget has no
69
- // hot-swap-avatar API (`createFace` runs once per mount), so changing it
70
- // re-renders nothing here by design; a caller that needs a different face
71
- // remounts with a `key` prop (see the component's doc).
72
- // eslint-disable-next-line react-hooks/exhaustive-deps
73
- }, []);
74
-
75
- useEffect(() => {
76
- const wrapper = avatarClientRef.current;
77
- const pipecatClient = options.client;
78
- if (!wrapper || !pipecatClient) return;
79
- return wrapper.attach(pipecatClient);
80
- // Re-subscribe when the widget mounts or the session's client changes.
81
- // eslint-disable-next-line react-hooks/exhaustive-deps
82
- }, [avatar, options.client]);
83
-
84
- return { containerRef, avatar };
85
- }