@voqalize/avatar 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/LICENSE +21 -661
  2. package/README.md +144 -663
  3. package/client/Avatar.tsx +57 -0
  4. package/client/AvatarClient.ts +523 -0
  5. package/client/arjun.ts +24 -0
  6. package/client/createAvatar.ts +139 -0
  7. package/client/createCanvasAvatar.ts +72 -0
  8. package/client/index.ts +39 -0
  9. package/client/internal.ts +80 -0
  10. package/client/interviewer-female.ts +4 -0
  11. package/client/interviewer-male.ts +4 -0
  12. package/client/ishita.ts +24 -0
  13. package/client/kabir.ts +24 -0
  14. package/client/meera.ts +24 -0
  15. package/client/naina.ts +24 -0
  16. package/client/professional-female-a.ts +4 -0
  17. package/client/professional-female-b.ts +4 -0
  18. package/client/professional-male-a.ts +4 -0
  19. package/client/professional-male-b.ts +4 -0
  20. package/client/react.ts +13 -0
  21. package/client/types.ts +146 -0
  22. package/client/useAvatar.ts +74 -0
  23. package/client/vikram.ts +24 -0
  24. package/dist/Avatar.d.ts +44 -0
  25. package/dist/Avatar.d.ts.map +1 -0
  26. package/{client/dist → dist}/Avatar.js +2 -2
  27. package/dist/Avatar.js.map +1 -0
  28. package/dist/AvatarClient.d.ts +196 -0
  29. package/dist/AvatarClient.d.ts.map +1 -0
  30. package/dist/AvatarClient.js +450 -0
  31. package/dist/AvatarClient.js.map +1 -0
  32. package/dist/arjun.d.ts +6 -0
  33. package/dist/arjun.d.ts.map +1 -0
  34. package/dist/arjun.js +18 -0
  35. package/dist/arjun.js.map +1 -0
  36. package/dist/createAvatar.d.ts +99 -0
  37. package/dist/createAvatar.d.ts.map +1 -0
  38. package/dist/createAvatar.js +75 -0
  39. package/dist/createAvatar.js.map +1 -0
  40. package/dist/createCanvasAvatar.d.ts +22 -0
  41. package/dist/createCanvasAvatar.d.ts.map +1 -0
  42. package/dist/createCanvasAvatar.js +47 -0
  43. package/dist/createCanvasAvatar.js.map +1 -0
  44. package/dist/index.d.ts +30 -0
  45. package/dist/index.d.ts.map +1 -0
  46. package/dist/index.js +29 -0
  47. package/dist/index.js.map +1 -0
  48. package/dist/internal.d.ts +30 -0
  49. package/dist/internal.d.ts.map +1 -0
  50. package/dist/internal.js +33 -0
  51. package/dist/internal.js.map +1 -0
  52. package/dist/interviewer-female.d.ts +4 -0
  53. package/dist/interviewer-female.d.ts.map +1 -0
  54. package/dist/interviewer-female.js +3 -0
  55. package/dist/interviewer-female.js.map +1 -0
  56. package/dist/interviewer-male.d.ts +4 -0
  57. package/dist/interviewer-male.d.ts.map +1 -0
  58. package/dist/interviewer-male.js +3 -0
  59. package/dist/interviewer-male.js.map +1 -0
  60. package/dist/ishita.d.ts +6 -0
  61. package/dist/ishita.d.ts.map +1 -0
  62. package/dist/ishita.js +18 -0
  63. package/dist/ishita.js.map +1 -0
  64. package/dist/kabir.d.ts +6 -0
  65. package/dist/kabir.d.ts.map +1 -0
  66. package/dist/kabir.js +18 -0
  67. package/dist/kabir.js.map +1 -0
  68. package/dist/meera.d.ts +6 -0
  69. package/dist/meera.d.ts.map +1 -0
  70. package/dist/meera.js +18 -0
  71. package/dist/meera.js.map +1 -0
  72. package/dist/naina.d.ts +6 -0
  73. package/dist/naina.d.ts.map +1 -0
  74. package/dist/naina.js +18 -0
  75. package/dist/naina.js.map +1 -0
  76. package/dist/professional-female-a.d.ts +4 -0
  77. package/dist/professional-female-a.d.ts.map +1 -0
  78. package/dist/professional-female-a.js +3 -0
  79. package/dist/professional-female-a.js.map +1 -0
  80. package/dist/professional-female-b.d.ts +4 -0
  81. package/dist/professional-female-b.d.ts.map +1 -0
  82. package/dist/professional-female-b.js +3 -0
  83. package/dist/professional-female-b.js.map +1 -0
  84. package/dist/professional-male-a.d.ts +4 -0
  85. package/dist/professional-male-a.d.ts.map +1 -0
  86. package/dist/professional-male-a.js +3 -0
  87. package/dist/professional-male-a.js.map +1 -0
  88. package/dist/professional-male-b.d.ts +4 -0
  89. package/dist/professional-male-b.d.ts.map +1 -0
  90. package/dist/professional-male-b.js +3 -0
  91. package/dist/professional-male-b.js.map +1 -0
  92. package/dist/react.d.ts +13 -0
  93. package/dist/react.d.ts.map +1 -0
  94. package/dist/react.js +13 -0
  95. package/dist/react.js.map +1 -0
  96. package/dist/types.d.ts +94 -0
  97. package/dist/types.d.ts.map +1 -0
  98. package/dist/types.js +93 -0
  99. package/dist/types.js.map +1 -0
  100. package/dist/useAvatar.d.ts +39 -0
  101. package/dist/useAvatar.d.ts.map +1 -0
  102. package/dist/useAvatar.js +35 -0
  103. package/dist/useAvatar.js.map +1 -0
  104. package/dist/vikram.d.ts +6 -0
  105. package/dist/vikram.d.ts.map +1 -0
  106. package/dist/vikram.js +18 -0
  107. package/dist/vikram.js.map +1 -0
  108. package/package.json +81 -24
  109. package/src/avatar.d.ts +161 -120
  110. package/src/avatar.js +234 -146
  111. package/src/behavior.d.ts +26 -0
  112. package/src/behavior.js +102 -0
  113. package/src/camera.js +29 -0
  114. package/src/canvas/author/parts/eye.mjs +722 -0
  115. package/src/canvas/author/parts/hand.mjs +1156 -0
  116. package/src/canvas/author/parts/mouth.mjs +741 -0
  117. package/src/canvas/author/parts/nose.mjs +100 -0
  118. package/src/canvas/author/parts/skin-detail.mjs +67 -0
  119. package/src/canvas/author/path.mjs +283 -0
  120. package/src/canvas/author/rig.mjs +405 -0
  121. package/src/canvas/avatars/round/face.d.mts +3 -0
  122. package/src/canvas/avatars/round/face.mjs +1307 -0
  123. package/src/canvas/create-rig.d.ts +15 -0
  124. package/src/canvas/create-rig.js +100 -0
  125. package/src/canvas/data/img/professional-female-a-hair-back.webp +0 -0
  126. package/src/canvas/data/img/professional-female-a-hair-front.webp +0 -0
  127. package/src/canvas/data/img/professional-female-a-top-body.webp +0 -0
  128. package/src/canvas/data/img/professional-female-b-hair-back.webp +0 -0
  129. package/src/canvas/data/img/professional-female-b-hair-front.webp +0 -0
  130. package/src/canvas/data/img/professional-female-b-top-body.webp +0 -0
  131. package/src/canvas/data/img/professional-male-a-hair-back.webp +0 -0
  132. package/src/canvas/data/img/professional-male-a-hair-front.webp +0 -0
  133. package/src/canvas/data/img/professional-male-a-top-body.webp +0 -0
  134. package/src/canvas/data/img/professional-male-b-hair-back.webp +0 -0
  135. package/src/canvas/data/img/professional-male-b-hair-front.webp +0 -0
  136. package/src/canvas/data/img/professional-male-b-top-body.webp +0 -0
  137. package/src/canvas/data/img/round-m3-hair-back.webp +0 -0
  138. package/src/canvas/data/img/round-m3-hair-front.webp +0 -0
  139. package/src/canvas/data/img/round-m3-top-body.webp +0 -0
  140. package/src/canvas/data/img/round-w1-hair-back.webp +0 -0
  141. package/src/canvas/data/img/round-w1-hair-front.webp +0 -0
  142. package/src/canvas/data/img/round-w1-top-body.webp +0 -0
  143. package/src/canvas/data/interviewer-female.rig.json +1 -0
  144. package/src/canvas/data/interviewer-male.rig.json +1 -0
  145. package/src/canvas/data/professional-female-a.rig.json +1 -0
  146. package/src/canvas/data/professional-female-b.rig.json +1 -0
  147. package/src/canvas/data/professional-male-a.rig.json +1 -0
  148. package/src/canvas/data/professional-male-b.rig.json +1 -0
  149. package/src/canvas/src/live.js +508 -0
  150. package/src/canvas/src/render2d.js +218 -0
  151. package/src/canvas/src/rig.js +297 -0
  152. package/src/canvas/src/vocab.js +96 -0
  153. package/src/clips.js +18 -7
  154. package/src/conformance.js +119 -0
  155. package/src/emotions.js +4 -3
  156. package/src/face-core.js +27 -1
  157. package/src/face-myna.d.ts +7 -0
  158. package/src/face-myna.js +192 -131
  159. package/src/face-peep-control-plane.js +167 -0
  160. package/src/face-peep.d.ts +7 -0
  161. package/src/face-peep.js +274 -116
  162. package/src/face-wren.d.ts +7 -0
  163. package/src/face-wren.js +17 -19
  164. package/src/faces.d.ts +12 -0
  165. package/src/faces.js +53 -0
  166. package/src/gaze.js +119 -4
  167. package/src/hand.js +120 -91
  168. package/src/idle.js +18 -101
  169. package/src/interjections.js +156 -31
  170. package/src/params.js +8 -4
  171. package/src/perform.js +7 -9
  172. package/src/rig.d.ts +24 -0
  173. package/src/rig.js +32 -0
  174. package/src/speech-timing.js +23 -0
  175. package/src/visemes.js +28 -12
  176. package/client/dist/Avatar.d.ts +0 -27
  177. package/client/dist/Avatar.d.ts.map +0 -1
  178. package/client/dist/Avatar.js.map +0 -1
  179. package/client/dist/AvatarClient.d.ts +0 -139
  180. package/client/dist/AvatarClient.d.ts.map +0 -1
  181. package/client/dist/AvatarClient.js +0 -223
  182. package/client/dist/AvatarClient.js.map +0 -1
  183. package/client/dist/index.d.ts +0 -22
  184. package/client/dist/index.d.ts.map +0 -1
  185. package/client/dist/index.js +0 -22
  186. package/client/dist/index.js.map +0 -1
  187. package/client/dist/types.d.ts +0 -86
  188. package/client/dist/types.d.ts.map +0 -1
  189. package/client/dist/types.js +0 -31
  190. package/client/dist/types.js.map +0 -1
  191. package/client/dist/useAvatar.d.ts +0 -42
  192. package/client/dist/useAvatar.d.ts.map +0 -1
  193. package/client/dist/useAvatar.js +0 -51
  194. package/client/dist/useAvatar.js.map +0 -1
  195. package/client/src/Avatar.tsx +0 -33
  196. package/client/src/AvatarClient.ts +0 -270
  197. package/client/src/index.ts +0 -22
  198. package/client/src/types.ts +0 -104
  199. package/client/src/useAvatar.ts +0 -85
  200. package/docs/contract-avatar.md +0 -371
  201. package/docs/contract-protocol.md +0 -461
@@ -0,0 +1,74 @@
1
+ /**
2
+ * useAvatar — mount an avatar into a ref'd element for as long as there is a
3
+ * client to embody, and tear it down after.
4
+ *
5
+ * Internal; `<Avatar>` is the only thing the React entry exports. The whole
6
+ * hook is one effect, because the factory takes the client at construction:
7
+ * there is no separate attach step to keep in its own lifecycle any more.
8
+ */
9
+
10
+ import { useEffect, useRef } from "react";
11
+ import type { PipecatClient } from "@pipecat-ai/client-js";
12
+ import {
13
+ createAvatar as createSvgAvatar,
14
+ type AvatarFactory,
15
+ type AvatarOptions,
16
+ type SvgAvatarOptions,
17
+ } from "./createAvatar.js";
18
+
19
+ export interface UseAvatarOptions<O extends AvatarOptions = SvgAvatarOptions> {
20
+ /** The live `PipecatClient`, or `null` before connect. Nothing mounts until
21
+ * this is non-null — an avatar with nothing to embody has nothing to do. */
22
+ client?: PipecatClient | null;
23
+ /** The avatar implementation. Defaults to the bundled SVG faces. */
24
+ create?: AvatarFactory<O>;
25
+ /** Implementation options, forwarded verbatim. Read at mount only. */
26
+ options?: Omit<O, keyof AvatarOptions>;
27
+ }
28
+
29
+ /**
30
+ * The mount ref's type, written out rather than named as React's `RefObject`.
31
+ *
32
+ * React 18 and 19 declare that alias with different type arguments — 18's
33
+ * `useRef<T>(null)` yields `RefObject<T>`, 19's yields `RefObject<T | null>` —
34
+ * and because both are the *same alias*, TypeScript compares them by variance
35
+ * and rejects whichever one we didn't pick. An anonymous shape forces a
36
+ * structural comparison instead, which both versions satisfy, and which the
37
+ * `ref` prop accepts on both. This is the only place the 18-vs-19 split shows
38
+ * up in the binding; keep it that way.
39
+ */
40
+ export type AvatarMountRef = { current: HTMLDivElement | null };
41
+
42
+ export interface UseAvatarHandle {
43
+ /** Attach to the mount element: `<div ref={containerRef} />`. */
44
+ containerRef: AvatarMountRef;
45
+ }
46
+
47
+ export function useAvatar<O extends AvatarOptions = SvgAvatarOptions>(
48
+ { client, create, options }: UseAvatarOptions<O> = {},
49
+ ): UseAvatarHandle {
50
+ const containerRef = useRef<HTMLDivElement>(null);
51
+
52
+ // Latest-options ref, so the mount effect reads the live values without
53
+ // remounting the face every time a caller passes a fresh object literal.
54
+ const optionsRef = useRef(options);
55
+ optionsRef.current = options;
56
+
57
+ useEffect(() => {
58
+ const mount = containerRef.current;
59
+ if (!mount || !client) return;
60
+ const factory = (create ?? createSvgAvatar) as AvatarFactory<O>;
61
+ // The cast is the seam between "O minus the two we supply" and O. It is
62
+ // sound by construction and TypeScript cannot see through the spread of a
63
+ // generic; the two halves are typed at the boundary the caller touches.
64
+ const instance = factory({ mount, client, ...optionsRef.current } as unknown as O);
65
+ return () => instance.destroy();
66
+ // A new client identity is a new thing to embody, so the avatar is rebuilt
67
+ // rather than re-pointed. Hosts keep one `PipecatClient` across
68
+ // connect/disconnect cycles, so this does not fire on an ordinary
69
+ // reconnect; if yours constructs a fresh client per session, expect the
70
+ // face to remount with it.
71
+ }, [client, create]);
72
+
73
+ return { containerRef };
74
+ }
@@ -0,0 +1,24 @@
1
+ /** Polished professional male avatar, authored to read at call-tile size. */
2
+
3
+ import { createCanvasAvatar } from './createCanvasAvatar.js';
4
+ import type { CanvasAvatarOptions } from './createCanvasAvatar.js';
5
+ import type { AvatarInstance } from './createAvatar.js';
6
+
7
+ const RIG_URL = new URL('../src/canvas/data/professional-male-a.rig.json', import.meta.url);
8
+ const IMAGES = {
9
+ 'professional-male-a-top-body.webp': new URL('../src/canvas/data/img/professional-male-a-top-body.webp', import.meta.url),
10
+ 'professional-male-a-hair-back.webp': new URL('../src/canvas/data/img/professional-male-a-hair-back.webp', import.meta.url),
11
+ 'professional-male-a-hair-front.webp': new URL('../src/canvas/data/img/professional-male-a-hair-front.webp', import.meta.url),
12
+ } as const;
13
+ const FACE = () => import('../src/canvas/avatars/round/face.mjs');
14
+
15
+ export type { CanvasAvatarOptions };
16
+
17
+ export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
18
+ return createCanvasAvatar(options, {
19
+ rigUrl: RIG_URL,
20
+ images: IMAGES,
21
+ face: FACE,
22
+ label: 'Vikram — polished professional male avatar',
23
+ });
24
+ }
@@ -0,0 +1,44 @@
1
+ /**
2
+ * Avatar — `createAvatar` as a call tile, and the whole of `@voqalize/avatar/react`.
3
+ *
4
+ * <Avatar client={session.client} className="avatar-tile" />
5
+ *
6
+ * Three props of its own; everything else is forwarded to the mount `<div>`, so
7
+ * it sizes and styles like the tile it lives in. There is nothing to configure
8
+ * because there is nothing the server does not already say: the
9
+ * `AvatarProcessor` in the pipeline drives the state, the gaze and the mouth.
10
+ *
11
+ * `create` is how you use a different avatar — any module exporting
12
+ * `createAvatar` (docs/design-avatar-interface.md), including a Rive or WebGL
13
+ * one. The component is generic over that implementation's options, so
14
+ * `options` is checked against the factory you passed rather than being a bag:
15
+ *
16
+ * <Avatar client={c} create={createMascot} options={{ mood: "sly" }} />
17
+ *
18
+ * With no `create` it is the bundled SVG avatar and `options` is
19
+ * `SvgAvatarOptions` minus `mount`/`client`, which the component supplies.
20
+ *
21
+ * The component renders an empty div and a static `role="img"`. It does not
22
+ * label the avatar with its current state: the implementation owns the DOM
23
+ * inside the mount and is the only thing that knows what it is portraying —
24
+ * and a live label would be this package reading back an avatar's internal
25
+ * state, which is the one thing the interface refuses to promise. Pass your
26
+ * own `aria-label` to override.
27
+ */
28
+ import type { HTMLAttributes } from "react";
29
+ import type { PipecatClient } from "@pipecat-ai/client-js";
30
+ import type { AvatarFactory, AvatarOptions, SvgAvatarOptions } from "./createAvatar.js";
31
+ /** An implementation's own options: everything it takes past the two the
32
+ * component supplies itself. */
33
+ export type ImplementationOptions<O extends AvatarOptions> = Omit<O, keyof AvatarOptions>;
34
+ export type AvatarProps<O extends AvatarOptions = SvgAvatarOptions> = Omit<HTMLAttributes<HTMLDivElement>, "children"> & {
35
+ /** The live `PipecatClient`, or `null` before connect. Nothing renders
36
+ * until it is non-null. */
37
+ client?: PipecatClient | null;
38
+ /** An avatar implementation. Omit for the bundled SVG faces. */
39
+ create?: AvatarFactory<O>;
40
+ /** Options for that implementation. Read at mount only. */
41
+ options?: ImplementationOptions<O>;
42
+ };
43
+ export declare function Avatar<O extends AvatarOptions = SvgAvatarOptions>({ client, create, options, ...rest }: AvatarProps<O>): import("react").JSX.Element;
44
+ //# sourceMappingURL=Avatar.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Avatar.d.ts","sourceRoot":"","sources":["../client/Avatar.tsx"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,OAAO,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAExF;gCACgC;AAChC,MAAM,MAAM,qBAAqB,CAAC,CAAC,SAAS,aAAa,IAAI,IAAI,CAAC,CAAC,EAAE,MAAM,aAAa,CAAC,CAAC;AAE1F,MAAM,MAAM,WAAW,CAAC,CAAC,SAAS,aAAa,GAAG,gBAAgB,IAChE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,EAAE,UAAU,CAAC,GAAG;IACjD;+BAC2B;IAC3B,MAAM,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,gEAAgE;IAChE,MAAM,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;IAC1B,2DAA2D;IAC3D,OAAO,CAAC,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC;CACpC,CAAC;AAEJ,wBAAgB,MAAM,CAAC,CAAC,SAAS,aAAa,GAAG,gBAAgB,EAAE,EACjE,MAAM,EACN,MAAM,EACN,OAAO,EACP,GAAG,IAAI,EACR,EAAE,WAAW,CAAC,CAAC,CAAC,+BAGhB"}
@@ -1,7 +1,7 @@
1
1
  import { jsx as _jsx } from "react/jsx-runtime";
2
2
  import { useAvatar } from "./useAvatar.js";
3
- export function Avatar({ client, avatar, ...rest }) {
4
- const { containerRef } = useAvatar({ client, avatar });
3
+ export function Avatar({ client, create, options, ...rest }) {
4
+ const { containerRef } = useAvatar({ client, create, options });
5
5
  return _jsx("div", { role: "img", "aria-label": "avatar", ...rest, ref: containerRef });
6
6
  }
7
7
  //# sourceMappingURL=Avatar.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Avatar.js","sourceRoot":"","sources":["../client/Avatar.tsx"],"names":[],"mappings":";AA8BA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAkB3C,MAAM,UAAU,MAAM,CAA6C,EACjE,MAAM,EACN,MAAM,EACN,OAAO,EACP,GAAG,IAAI,EACQ;IACf,MAAM,EAAE,YAAY,EAAE,GAAG,SAAS,CAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACnE,OAAO,cAAK,IAAI,EAAC,KAAK,gBAAY,QAAQ,KAAK,IAAI,EAAE,GAAG,EAAE,YAAY,GAAI,CAAC;AAC7E,CAAC"}
@@ -0,0 +1,196 @@
1
+ /**
2
+ * AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
3
+ * splice, framework-free (no React; the hook and component wrap this).
4
+ *
5
+ * ## Turn clock anchoring
6
+ *
7
+ * Base Pipecat TTS gives each serialized TTS context an opaque `context_id`.
8
+ * The server uses it only to group and splice cue chunks. `botStartedSpeaking`
9
+ * has no context payload, so the browser FIFO-claims the next buffered context
10
+ * at that Pipecat output-lifecycle event and anchors its clock there.
11
+ * `botStoppedSpeaking` closes the active context. Neither event observes the
12
+ * browser's audio device, and no avatar-specific speech marker exists.
13
+ *
14
+ * `attach()` subscribes to the avatar server-message channel *and* Pipecat's
15
+ * standard lifecycle events. Server messages carry only what Pipecat cannot:
16
+ * correlated speech/cue timing and deliberate application instructions. The
17
+ * lifecycle events project the factual presence states locally. The server
18
+ * supplies only lower-priority `THINKING` / `WORKING` claims and deliberate,
19
+ * self-completing actions. This keeps the face tied to Pipecat's output truth
20
+ * even if a server claim is delayed or stale.
21
+ *
22
+ * ## Cue splice
23
+ *
24
+ * The widget has two cue-track primitives: `speak({cues, clock})` (a full
25
+ * replace) and `pushCues(cues)` (a pure union that can only grow the track,
26
+ * never shrink it). Neither is "discard queued cues at or after `from_ms`,
27
+ * then append" on its own — `pushCues` has no way to drop a stale tail. So
28
+ * this client keeps the turn's canonical cue array itself (kept portion +
29
+ * every appended chunk, spliced on each `cues` message) and picks the cheapest
30
+ * widget call that stays correct:
31
+ *
32
+ * - if the splice's `from_ms` doesn't reach back into anything already
33
+ * queued — the common case past a turn's first sentence, since only the
34
+ * first sentence genuinely plays fast-leg cues — nothing needs discarding:
35
+ * `pushCues(newCues)` is the cheap, correct append.
36
+ * - if it does reach back (a real fast→accurate splice), `pushCues` cannot
37
+ * express the discard; we call `speak()` again with the full spliced
38
+ * canonical array on the turn's original clock. `speak()` is otherwise
39
+ * documented as also killing an in-flight spoken interjection and
40
+ * re-entering `SPEAKING` — both harmless mid-splice (an interjection
41
+ * should not be running while a server track owns the mouth; re-entering
42
+ * an unchanged state is a no-op past the profile/gaze reset the widget
43
+ * already does for a same-name `setState`).
44
+ *
45
+ * Cues commonly arrive **before** `botStartedSpeaking` — the fast leg starts
46
+ * the moment a sentence is handed to TTS, well before bot output begins.
47
+ * Chunks that arrive before the clock is anchored are spliced into the
48
+ * canonical array but not yet handed to the widget; `botStartedSpeaking` hands over
49
+ * whatever has accumulated as the turn's first `speak()` call. So "the first
50
+ * chunk of a turn starts speak()" means the first *widget* call, not
51
+ * necessarily the first *message*.
52
+ */
53
+ import type { PipecatClient } from "@pipecat-ai/client-js";
54
+ import type { AvatarApi } from "../src/avatar.js";
55
+ import { type AvatarCommand, type AvatarCue } from "./types.js";
56
+ /**
57
+ * Internal. Not exported from the package — the public surface is
58
+ * `createAvatar({ mount, client })` and nothing else.
59
+ *
60
+ * There is deliberately no `accept` predicate here any more. Avatar commands
61
+ * travel in one envelope, `{type:"avatar"}`, in both directions and from every
62
+ * source: a `AvatarProcessor` in the pipeline and a brain driving the face
63
+ * out of band emit the same shape. A per-deployment predicate meant the
64
+ * library could not state what an avatar message *is*, which is the one thing
65
+ * a wire format has to be able to say.
66
+ */
67
+ export interface AvatarClientOptions {
68
+ /** A dispatch threw (e.g. an unknown state or interjection id, which the
69
+ * widget throws on). Defaults to `console.warn`. */
70
+ onError?: (err: unknown, msg: AvatarCommand) => void;
71
+ /** Override for tests. Defaults to `performance.now`. */
72
+ now?: () => number;
73
+ /** Quiet time in listening before the client-owned idle loop begins. */
74
+ idleDelayMs?: number;
75
+ /** Timer seams keep lifecycle behavior deterministic in tests. */
76
+ setTimeout?: typeof setTimeout;
77
+ clearTimeout?: typeof clearTimeout;
78
+ /**
79
+ * Internal, for Studio's inspector — the resolved projection, so a developer
80
+ * tool can show what the lifecycle decided. It is *not* on the public
81
+ * surface and must not become one: a presence callback is a contract, and
82
+ * publishing it would oblige every avatar implementation to emit these seven
83
+ * states with this precedence, which is exactly the second public contract
84
+ * the design exists to avoid.
85
+ */
86
+ onPresenceChange?: (state: AvatarPresenceState) => void;
87
+ }
88
+ /**
89
+ * The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
90
+ *
91
+ * Written out rather than imported because that enum was this module's *only*
92
+ * runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
93
+ * the whole `/pipecat` subpath fail to load without the peer installed — even
94
+ * for a host that drives `dispatch()` from its own transport and never calls
95
+ * `attach()`. The peer is declared optional; this is what makes that true
96
+ * rather than aspirational.
97
+ *
98
+ * String enums are nominal in TypeScript, so the compiler cannot check these
99
+ * against the real ones from a type-only import. `packages/avatar/test/AvatarClient.test.ts`
100
+ * does it instead, against the actual enum — the devDependency is present
101
+ * exactly where the check belongs and absent from what we ship.
102
+ */
103
+ export declare const RTVI_EVENTS: {
104
+ readonly serverMessage: "serverMessage";
105
+ readonly connected: "connected";
106
+ readonly disconnected: "disconnected";
107
+ readonly botReady: "botReady";
108
+ readonly error: "error";
109
+ readonly userStartedSpeaking: "userStartedSpeaking";
110
+ readonly userStoppedSpeaking: "userStoppedSpeaking";
111
+ readonly botStartedSpeaking: "botStartedSpeaking";
112
+ readonly botStoppedSpeaking: "botStoppedSpeaking";
113
+ readonly userMuteStarted: "userMuteStarted";
114
+ readonly userMuteStopped: "userMuteStopped";
115
+ };
116
+ /** The resolved, factual presence state a host may render around the avatar. */
117
+ export type AvatarPresenceState = "IDLE" | "LISTENING" | "STRAINING" | "THINKING" | "WORKING" | "MUTED" | "SPEAKING" | "DEGRADED" | "OFFLINE";
118
+ export declare class AvatarClient {
119
+ private readonly avatar;
120
+ /** Maps factual/wire intent into the broader client behavior catalog. */
121
+ private readonly behavior;
122
+ private readonly opts;
123
+ private readonly now;
124
+ private turn;
125
+ private readonly turns;
126
+ private readonly pendingCtxs;
127
+ private readonly closedCtxs;
128
+ private projected;
129
+ private serverClaim;
130
+ private userSpeaking;
131
+ private botSpeaking;
132
+ private muted;
133
+ private listening;
134
+ private idle;
135
+ private failure;
136
+ private idleTimer;
137
+ private pendingInterruptedAction;
138
+ private discardQueuedContextsOnBotStop;
139
+ private readonly idleDelayMs;
140
+ private readonly setTimer;
141
+ private readonly clearTimer;
142
+ constructor(avatar: AvatarApi, opts?: AvatarClientOptions);
143
+ /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
144
+ get turnCtx(): string | null;
145
+ /** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
146
+ get turnCues(): AvatarCue[];
147
+ /** Current resolved projection. Internal — for tests and Studio's inspector.
148
+ * Before the first lifecycle fact the renderer is simply at its ready rest
149
+ * pose, so `LISTENING` is the safe value; it is not a session status. */
150
+ get presenceState(): AvatarPresenceState;
151
+ /** Re-apply the currently resolved factual projection after an embedding
152
+ * tool has temporarily used raw renderer controls. This does not invent a
153
+ * lifecycle event or cancel a finite action. */
154
+ restoreProjection(): void;
155
+ /** Dispatch one server message. Anything that isn't in the avatar envelope
156
+ * is not ours and is ignored; so is an envelope carrying a `cmd` this build
157
+ * has never heard of, per the wire protocol's forward-compat rule. */
158
+ dispatch(raw: unknown): void;
159
+ private handleClaim;
160
+ private handleAction;
161
+ private playAction;
162
+ private ensureTurn;
163
+ private handleCues;
164
+ private activateNextTurn;
165
+ private discardQueuedTurns;
166
+ private lifecycleState;
167
+ private applyProjection;
168
+ private clearIdleTimer;
169
+ private armIdleIfEligible;
170
+ private eligibleForIdle;
171
+ private enterListening;
172
+ private clearClaimForTurnBoundary;
173
+ private maybePlayInterrupted;
174
+ private clearRecoverableFailure;
175
+ private onUserStartedSpeaking;
176
+ private onUserStoppedSpeaking;
177
+ private onBotStartedSpeaking;
178
+ private onBotStoppedSpeaking;
179
+ private onUserMuteStarted;
180
+ private onUserMuteStopped;
181
+ private onError;
182
+ private onDisconnected;
183
+ private onConnectedOrReady;
184
+ /**
185
+ * Subscribe to a live `PipecatClient`. Standard client events own the normal
186
+ * lifecycle projection; avatar server-messages carry correlated visemes and
187
+ * explicit application intent. Never throws on malformed or irrelevant
188
+ * server messages.
189
+ *
190
+ * @returns an unsubscribe function; call it on unmount or disconnect.
191
+ */
192
+ attach(client: PipecatClient): () => void;
193
+ /** Dispose controller-owned timers when its mounted avatar is destroyed. */
194
+ destroy(): void;
195
+ }
196
+ //# sourceMappingURL=AvatarClient.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AvatarClient.d.ts","sourceRoot":"","sources":["../client/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAa,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAkB,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,EAGL,KAAK,aAAa,EAClB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAYpB;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC;wDACoD;IACpD,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC;IACrD,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,UAAU,CAAC,EAAE,OAAO,UAAU,CAAC;IAC/B,YAAY,CAAC,EAAE,OAAO,YAAY,CAAC;IACnC;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;CACzD;AAED;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,WAAW;;;;;;;;;;;;CAgBmB,CAAC;AAE5C,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAC3B,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,GAC3D,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAYlD,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2B;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,SAAS,CAAS;IAG1B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,SAAS,CAA8C;IAC/D,OAAO,CAAC,wBAAwB,CAAS;IACzC,OAAO,CAAC,8BAA8B,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoB;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;gBAErC,MAAM,EAAE,SAAS,EAAE,IAAI,GAAE,mBAAwB;IAU7D,+EAA+E;IAC/E,IAAI,OAAO,IAAI,MAAM,GAAG,IAAI,CAE3B;IAED,wFAAwF;IACxF,IAAI,QAAQ,IAAI,SAAS,EAAE,CAE1B;IAED;;6EAEyE;IACzE,IAAI,aAAa,IAAI,mBAAmB,CAEvC;IAED;;oDAEgD;IAChD,iBAAiB,IAAI,IAAI;IAIzB;;0EAEsE;IACtE,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IA0B5B,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,YAAY;IAcpB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,UAAU;IAmBlB,OAAO,CAAC,gBAAgB;IAcxB,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,eAAe;IAKvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,yBAAyB;IAMjC,OAAO,CAAC,oBAAoB;IAO5B,OAAO,CAAC,uBAAuB;IAO/B,OAAO,CAAC,qBAAqB,CAS3B;IAEF,OAAO,CAAC,qBAAqB,CAI3B;IAEF,OAAO,CAAC,oBAAoB,CAQ1B;IAEF,OAAO,CAAC,oBAAoB,CAgB1B;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,OAAO,CAIb;IAEF,OAAO,CAAC,cAAc,CAIpB;IAEF,OAAO,CAAC,kBAAkB,CASxB;IAEF;;;;;;;OAOG;IACH,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,IAAI;IAsBzC,4EAA4E;IAC5E,OAAO,IAAI,IAAI;CAIhB"}