@voqalize/avatar 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/LICENSE +21 -661
  2. package/README.md +144 -663
  3. package/client/Avatar.tsx +57 -0
  4. package/client/AvatarClient.ts +523 -0
  5. package/client/arjun.ts +24 -0
  6. package/client/createAvatar.ts +139 -0
  7. package/client/createCanvasAvatar.ts +72 -0
  8. package/client/index.ts +39 -0
  9. package/client/internal.ts +80 -0
  10. package/client/interviewer-female.ts +4 -0
  11. package/client/interviewer-male.ts +4 -0
  12. package/client/ishita.ts +24 -0
  13. package/client/kabir.ts +24 -0
  14. package/client/meera.ts +24 -0
  15. package/client/naina.ts +24 -0
  16. package/client/professional-female-a.ts +4 -0
  17. package/client/professional-female-b.ts +4 -0
  18. package/client/professional-male-a.ts +4 -0
  19. package/client/professional-male-b.ts +4 -0
  20. package/client/react.ts +13 -0
  21. package/client/types.ts +146 -0
  22. package/client/useAvatar.ts +74 -0
  23. package/client/vikram.ts +24 -0
  24. package/dist/Avatar.d.ts +44 -0
  25. package/dist/Avatar.d.ts.map +1 -0
  26. package/{client/dist → dist}/Avatar.js +2 -2
  27. package/dist/Avatar.js.map +1 -0
  28. package/dist/AvatarClient.d.ts +196 -0
  29. package/dist/AvatarClient.d.ts.map +1 -0
  30. package/dist/AvatarClient.js +450 -0
  31. package/dist/AvatarClient.js.map +1 -0
  32. package/dist/arjun.d.ts +6 -0
  33. package/dist/arjun.d.ts.map +1 -0
  34. package/dist/arjun.js +18 -0
  35. package/dist/arjun.js.map +1 -0
  36. package/dist/createAvatar.d.ts +99 -0
  37. package/dist/createAvatar.d.ts.map +1 -0
  38. package/dist/createAvatar.js +75 -0
  39. package/dist/createAvatar.js.map +1 -0
  40. package/dist/createCanvasAvatar.d.ts +22 -0
  41. package/dist/createCanvasAvatar.d.ts.map +1 -0
  42. package/dist/createCanvasAvatar.js +47 -0
  43. package/dist/createCanvasAvatar.js.map +1 -0
  44. package/dist/index.d.ts +30 -0
  45. package/dist/index.d.ts.map +1 -0
  46. package/dist/index.js +29 -0
  47. package/dist/index.js.map +1 -0
  48. package/dist/internal.d.ts +30 -0
  49. package/dist/internal.d.ts.map +1 -0
  50. package/dist/internal.js +33 -0
  51. package/dist/internal.js.map +1 -0
  52. package/dist/interviewer-female.d.ts +4 -0
  53. package/dist/interviewer-female.d.ts.map +1 -0
  54. package/dist/interviewer-female.js +3 -0
  55. package/dist/interviewer-female.js.map +1 -0
  56. package/dist/interviewer-male.d.ts +4 -0
  57. package/dist/interviewer-male.d.ts.map +1 -0
  58. package/dist/interviewer-male.js +3 -0
  59. package/dist/interviewer-male.js.map +1 -0
  60. package/dist/ishita.d.ts +6 -0
  61. package/dist/ishita.d.ts.map +1 -0
  62. package/dist/ishita.js +18 -0
  63. package/dist/ishita.js.map +1 -0
  64. package/dist/kabir.d.ts +6 -0
  65. package/dist/kabir.d.ts.map +1 -0
  66. package/dist/kabir.js +18 -0
  67. package/dist/kabir.js.map +1 -0
  68. package/dist/meera.d.ts +6 -0
  69. package/dist/meera.d.ts.map +1 -0
  70. package/dist/meera.js +18 -0
  71. package/dist/meera.js.map +1 -0
  72. package/dist/naina.d.ts +6 -0
  73. package/dist/naina.d.ts.map +1 -0
  74. package/dist/naina.js +18 -0
  75. package/dist/naina.js.map +1 -0
  76. package/dist/professional-female-a.d.ts +4 -0
  77. package/dist/professional-female-a.d.ts.map +1 -0
  78. package/dist/professional-female-a.js +3 -0
  79. package/dist/professional-female-a.js.map +1 -0
  80. package/dist/professional-female-b.d.ts +4 -0
  81. package/dist/professional-female-b.d.ts.map +1 -0
  82. package/dist/professional-female-b.js +3 -0
  83. package/dist/professional-female-b.js.map +1 -0
  84. package/dist/professional-male-a.d.ts +4 -0
  85. package/dist/professional-male-a.d.ts.map +1 -0
  86. package/dist/professional-male-a.js +3 -0
  87. package/dist/professional-male-a.js.map +1 -0
  88. package/dist/professional-male-b.d.ts +4 -0
  89. package/dist/professional-male-b.d.ts.map +1 -0
  90. package/dist/professional-male-b.js +3 -0
  91. package/dist/professional-male-b.js.map +1 -0
  92. package/dist/react.d.ts +13 -0
  93. package/dist/react.d.ts.map +1 -0
  94. package/dist/react.js +13 -0
  95. package/dist/react.js.map +1 -0
  96. package/dist/types.d.ts +94 -0
  97. package/dist/types.d.ts.map +1 -0
  98. package/dist/types.js +93 -0
  99. package/dist/types.js.map +1 -0
  100. package/dist/useAvatar.d.ts +39 -0
  101. package/dist/useAvatar.d.ts.map +1 -0
  102. package/dist/useAvatar.js +35 -0
  103. package/dist/useAvatar.js.map +1 -0
  104. package/dist/vikram.d.ts +6 -0
  105. package/dist/vikram.d.ts.map +1 -0
  106. package/dist/vikram.js +18 -0
  107. package/dist/vikram.js.map +1 -0
  108. package/package.json +81 -24
  109. package/src/avatar.d.ts +161 -120
  110. package/src/avatar.js +234 -146
  111. package/src/behavior.d.ts +26 -0
  112. package/src/behavior.js +102 -0
  113. package/src/camera.js +29 -0
  114. package/src/canvas/author/parts/eye.mjs +722 -0
  115. package/src/canvas/author/parts/hand.mjs +1156 -0
  116. package/src/canvas/author/parts/mouth.mjs +741 -0
  117. package/src/canvas/author/parts/nose.mjs +100 -0
  118. package/src/canvas/author/parts/skin-detail.mjs +67 -0
  119. package/src/canvas/author/path.mjs +283 -0
  120. package/src/canvas/author/rig.mjs +405 -0
  121. package/src/canvas/avatars/round/face.d.mts +3 -0
  122. package/src/canvas/avatars/round/face.mjs +1307 -0
  123. package/src/canvas/create-rig.d.ts +15 -0
  124. package/src/canvas/create-rig.js +100 -0
  125. package/src/canvas/data/img/professional-female-a-hair-back.webp +0 -0
  126. package/src/canvas/data/img/professional-female-a-hair-front.webp +0 -0
  127. package/src/canvas/data/img/professional-female-a-top-body.webp +0 -0
  128. package/src/canvas/data/img/professional-female-b-hair-back.webp +0 -0
  129. package/src/canvas/data/img/professional-female-b-hair-front.webp +0 -0
  130. package/src/canvas/data/img/professional-female-b-top-body.webp +0 -0
  131. package/src/canvas/data/img/professional-male-a-hair-back.webp +0 -0
  132. package/src/canvas/data/img/professional-male-a-hair-front.webp +0 -0
  133. package/src/canvas/data/img/professional-male-a-top-body.webp +0 -0
  134. package/src/canvas/data/img/professional-male-b-hair-back.webp +0 -0
  135. package/src/canvas/data/img/professional-male-b-hair-front.webp +0 -0
  136. package/src/canvas/data/img/professional-male-b-top-body.webp +0 -0
  137. package/src/canvas/data/img/round-m3-hair-back.webp +0 -0
  138. package/src/canvas/data/img/round-m3-hair-front.webp +0 -0
  139. package/src/canvas/data/img/round-m3-top-body.webp +0 -0
  140. package/src/canvas/data/img/round-w1-hair-back.webp +0 -0
  141. package/src/canvas/data/img/round-w1-hair-front.webp +0 -0
  142. package/src/canvas/data/img/round-w1-top-body.webp +0 -0
  143. package/src/canvas/data/interviewer-female.rig.json +1 -0
  144. package/src/canvas/data/interviewer-male.rig.json +1 -0
  145. package/src/canvas/data/professional-female-a.rig.json +1 -0
  146. package/src/canvas/data/professional-female-b.rig.json +1 -0
  147. package/src/canvas/data/professional-male-a.rig.json +1 -0
  148. package/src/canvas/data/professional-male-b.rig.json +1 -0
  149. package/src/canvas/src/live.js +508 -0
  150. package/src/canvas/src/render2d.js +218 -0
  151. package/src/canvas/src/rig.js +297 -0
  152. package/src/canvas/src/vocab.js +96 -0
  153. package/src/clips.js +18 -7
  154. package/src/conformance.js +119 -0
  155. package/src/emotions.js +4 -3
  156. package/src/face-core.js +27 -1
  157. package/src/face-myna.d.ts +7 -0
  158. package/src/face-myna.js +192 -131
  159. package/src/face-peep-control-plane.js +167 -0
  160. package/src/face-peep.d.ts +7 -0
  161. package/src/face-peep.js +274 -116
  162. package/src/face-wren.d.ts +7 -0
  163. package/src/face-wren.js +17 -19
  164. package/src/faces.d.ts +12 -0
  165. package/src/faces.js +53 -0
  166. package/src/gaze.js +119 -4
  167. package/src/hand.js +120 -91
  168. package/src/idle.js +18 -101
  169. package/src/interjections.js +156 -31
  170. package/src/params.js +8 -4
  171. package/src/perform.js +7 -9
  172. package/src/rig.d.ts +24 -0
  173. package/src/rig.js +32 -0
  174. package/src/speech-timing.js +23 -0
  175. package/src/visemes.js +28 -12
  176. package/client/dist/Avatar.d.ts +0 -27
  177. package/client/dist/Avatar.d.ts.map +0 -1
  178. package/client/dist/Avatar.js.map +0 -1
  179. package/client/dist/AvatarClient.d.ts +0 -139
  180. package/client/dist/AvatarClient.d.ts.map +0 -1
  181. package/client/dist/AvatarClient.js +0 -223
  182. package/client/dist/AvatarClient.js.map +0 -1
  183. package/client/dist/index.d.ts +0 -22
  184. package/client/dist/index.d.ts.map +0 -1
  185. package/client/dist/index.js +0 -22
  186. package/client/dist/index.js.map +0 -1
  187. package/client/dist/types.d.ts +0 -86
  188. package/client/dist/types.d.ts.map +0 -1
  189. package/client/dist/types.js +0 -31
  190. package/client/dist/types.js.map +0 -1
  191. package/client/dist/useAvatar.d.ts +0 -42
  192. package/client/dist/useAvatar.d.ts.map +0 -1
  193. package/client/dist/useAvatar.js +0 -51
  194. package/client/dist/useAvatar.js.map +0 -1
  195. package/client/src/Avatar.tsx +0 -33
  196. package/client/src/AvatarClient.ts +0 -270
  197. package/client/src/index.ts +0 -22
  198. package/client/src/types.ts +0 -104
  199. package/client/src/useAvatar.ts +0 -85
  200. package/docs/contract-avatar.md +0 -371
  201. package/docs/contract-protocol.md +0 -461
@@ -0,0 +1,57 @@
1
+ /**
2
+ * Avatar — `createAvatar` as a call tile, and the whole of `@voqalize/avatar/react`.
3
+ *
4
+ * <Avatar client={session.client} className="avatar-tile" />
5
+ *
6
+ * Three props of its own; everything else is forwarded to the mount `<div>`, so
7
+ * it sizes and styles like the tile it lives in. There is nothing to configure
8
+ * because there is nothing the server does not already say: the
9
+ * `AvatarProcessor` in the pipeline drives the state, the gaze and the mouth.
10
+ *
11
+ * `create` is how you use a different avatar — any module exporting
12
+ * `createAvatar` (docs/design-avatar-interface.md), including a Rive or WebGL
13
+ * one. The component is generic over that implementation's options, so
14
+ * `options` is checked against the factory you passed rather than being a bag:
15
+ *
16
+ * <Avatar client={c} create={createMascot} options={{ mood: "sly" }} />
17
+ *
18
+ * With no `create` it is the bundled SVG avatar and `options` is
19
+ * `SvgAvatarOptions` minus `mount`/`client`, which the component supplies.
20
+ *
21
+ * The component renders an empty div and a static `role="img"`. It does not
22
+ * label the avatar with its current state: the implementation owns the DOM
23
+ * inside the mount and is the only thing that knows what it is portraying —
24
+ * and a live label would be this package reading back an avatar's internal
25
+ * state, which is the one thing the interface refuses to promise. Pass your
26
+ * own `aria-label` to override.
27
+ */
28
+
29
+ import type { HTMLAttributes } from "react";
30
+ import type { PipecatClient } from "@pipecat-ai/client-js";
31
+ import { useAvatar } from "./useAvatar.js";
32
+ import type { AvatarFactory, AvatarOptions, SvgAvatarOptions } from "./createAvatar.js";
33
+
34
+ /** An implementation's own options: everything it takes past the two the
35
+ * component supplies itself. */
36
+ export type ImplementationOptions<O extends AvatarOptions> = Omit<O, keyof AvatarOptions>;
37
+
38
+ export type AvatarProps<O extends AvatarOptions = SvgAvatarOptions> =
39
+ Omit<HTMLAttributes<HTMLDivElement>, "children"> & {
40
+ /** The live `PipecatClient`, or `null` before connect. Nothing renders
41
+ * until it is non-null. */
42
+ client?: PipecatClient | null;
43
+ /** An avatar implementation. Omit for the bundled SVG faces. */
44
+ create?: AvatarFactory<O>;
45
+ /** Options for that implementation. Read at mount only. */
46
+ options?: ImplementationOptions<O>;
47
+ };
48
+
49
+ export function Avatar<O extends AvatarOptions = SvgAvatarOptions>({
50
+ client,
51
+ create,
52
+ options,
53
+ ...rest
54
+ }: AvatarProps<O>) {
55
+ const { containerRef } = useAvatar<O>({ client, create, options });
56
+ return <div role="img" aria-label="avatar" {...rest} ref={containerRef} />;
57
+ }
@@ -0,0 +1,523 @@
1
+ /**
2
+ * AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
3
+ * splice, framework-free (no React; the hook and component wrap this).
4
+ *
5
+ * ## Turn clock anchoring
6
+ *
7
+ * Base Pipecat TTS gives each serialized TTS context an opaque `context_id`.
8
+ * The server uses it only to group and splice cue chunks. `botStartedSpeaking`
9
+ * has no context payload, so the browser FIFO-claims the next buffered context
10
+ * at that Pipecat output-lifecycle event and anchors its clock there.
11
+ * `botStoppedSpeaking` closes the active context. Neither event observes the
12
+ * browser's audio device, and no avatar-specific speech marker exists.
13
+ *
14
+ * `attach()` subscribes to the avatar server-message channel *and* Pipecat's
15
+ * standard lifecycle events. Server messages carry only what Pipecat cannot:
16
+ * correlated speech/cue timing and deliberate application instructions. The
17
+ * lifecycle events project the factual presence states locally. The server
18
+ * supplies only lower-priority `THINKING` / `WORKING` claims and deliberate,
19
+ * self-completing actions. This keeps the face tied to Pipecat's output truth
20
+ * even if a server claim is delayed or stale.
21
+ *
22
+ * ## Cue splice
23
+ *
24
+ * The widget has two cue-track primitives: `speak({cues, clock})` (a full
25
+ * replace) and `pushCues(cues)` (a pure union that can only grow the track,
26
+ * never shrink it). Neither is "discard queued cues at or after `from_ms`,
27
+ * then append" on its own — `pushCues` has no way to drop a stale tail. So
28
+ * this client keeps the turn's canonical cue array itself (kept portion +
29
+ * every appended chunk, spliced on each `cues` message) and picks the cheapest
30
+ * widget call that stays correct:
31
+ *
32
+ * - if the splice's `from_ms` doesn't reach back into anything already
33
+ * queued — the common case past a turn's first sentence, since only the
34
+ * first sentence genuinely plays fast-leg cues — nothing needs discarding:
35
+ * `pushCues(newCues)` is the cheap, correct append.
36
+ * - if it does reach back (a real fast→accurate splice), `pushCues` cannot
37
+ * express the discard; we call `speak()` again with the full spliced
38
+ * canonical array on the turn's original clock. `speak()` is otherwise
39
+ * documented as also killing an in-flight spoken interjection and
40
+ * re-entering `SPEAKING` — both harmless mid-splice (an interjection
41
+ * should not be running while a server track owns the mouth; re-entering
42
+ * an unchanged state is a no-op past the profile/gaze reset the widget
43
+ * already does for a same-name `setState`).
44
+ *
45
+ * Cues commonly arrive **before** `botStartedSpeaking` — the fast leg starts
46
+ * the moment a sentence is handed to TTS, well before bot output begins.
47
+ * Chunks that arrive before the clock is anchored are spliced into the
48
+ * canonical array but not yet handed to the widget; `botStartedSpeaking` hands over
49
+ * whatever has accumulated as the turn's first `speak()` call. So "the first
50
+ * chunk of a turn starts speak()" means the first *widget* call, not
51
+ * necessarily the first *message*.
52
+ */
53
+
54
+ import type { PipecatClient, RTVIEvent } from "@pipecat-ai/client-js";
55
+ import type { AvatarActionId, AvatarApi } from "../src/avatar.js";
56
+ import { BehaviorController } from "../src/behavior.js";
57
+ import {
58
+ isAvatarMessage,
59
+ parseAvatarCommand,
60
+ type AvatarCommand,
61
+ type AvatarCue,
62
+ type AvatarCuesCmd,
63
+ } from "./types.js";
64
+
65
+
66
+ interface Turn {
67
+ ctx: string;
68
+ /** The canonical, already-spliced cue track for this turn. */
69
+ cues: AvatarCue[];
70
+ /** Whether Pipecat output has anchored a clock and issued `speak()`. */
71
+ started: boolean;
72
+ clock: (() => number) | null;
73
+ }
74
+
75
+ /**
76
+ * Internal. Not exported from the package — the public surface is
77
+ * `createAvatar({ mount, client })` and nothing else.
78
+ *
79
+ * There is deliberately no `accept` predicate here any more. Avatar commands
80
+ * travel in one envelope, `{type:"avatar"}`, in both directions and from every
81
+ * source: a `AvatarProcessor` in the pipeline and a brain driving the face
82
+ * out of band emit the same shape. A per-deployment predicate meant the
83
+ * library could not state what an avatar message *is*, which is the one thing
84
+ * a wire format has to be able to say.
85
+ */
86
+ export interface AvatarClientOptions {
87
+ /** A dispatch threw (e.g. an unknown state or interjection id, which the
88
+ * widget throws on). Defaults to `console.warn`. */
89
+ onError?: (err: unknown, msg: AvatarCommand) => void;
90
+ /** Override for tests. Defaults to `performance.now`. */
91
+ now?: () => number;
92
+ /** Quiet time in listening before the client-owned idle loop begins. */
93
+ idleDelayMs?: number;
94
+ /** Timer seams keep lifecycle behavior deterministic in tests. */
95
+ setTimeout?: typeof setTimeout;
96
+ clearTimeout?: typeof clearTimeout;
97
+ /**
98
+ * Internal, for Studio's inspector — the resolved projection, so a developer
99
+ * tool can show what the lifecycle decided. It is *not* on the public
100
+ * surface and must not become one: a presence callback is a contract, and
101
+ * publishing it would oblige every avatar implementation to emit these seven
102
+ * states with this precedence, which is exactly the second public contract
103
+ * the design exists to avoid.
104
+ */
105
+ onPresenceChange?: (state: AvatarPresenceState) => void;
106
+ }
107
+
108
+ /**
109
+ * The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
110
+ *
111
+ * Written out rather than imported because that enum was this module's *only*
112
+ * runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
113
+ * the whole `/pipecat` subpath fail to load without the peer installed — even
114
+ * for a host that drives `dispatch()` from its own transport and never calls
115
+ * `attach()`. The peer is declared optional; this is what makes that true
116
+ * rather than aspirational.
117
+ *
118
+ * String enums are nominal in TypeScript, so the compiler cannot check these
119
+ * against the real ones from a type-only import. `packages/avatar/test/AvatarClient.test.ts`
120
+ * does it instead, against the actual enum — the devDependency is present
121
+ * exactly where the check belongs and absent from what we ship.
122
+ */
123
+ export const RTVI_EVENTS = {
124
+ serverMessage: "serverMessage",
125
+ connected: "connected",
126
+ disconnected: "disconnected",
127
+ botReady: "botReady",
128
+ error: "error",
129
+ userStartedSpeaking: "userStartedSpeaking",
130
+ userStoppedSpeaking: "userStoppedSpeaking",
131
+ botStartedSpeaking: "botStartedSpeaking",
132
+ botStoppedSpeaking: "botStoppedSpeaking",
133
+ // Mute is a Pipecat fact, not a claim: the server's mute strategy emits
134
+ // `UserMuteStarted/StoppedFrame`, the RTVI observer forwards them, and the
135
+ // browser client raises these. So "has muted you" costs no wire verb —
136
+ // reading the events the peer already sends is exactly the authority model.
137
+ userMuteStarted: "userMuteStarted",
138
+ userMuteStopped: "userMuteStopped",
139
+ } as const satisfies Record<string, string>;
140
+
141
+ /** The resolved, factual presence state a host may render around the avatar. */
142
+ export type AvatarPresenceState =
143
+ | "IDLE" | "LISTENING" | "STRAINING" | "THINKING" | "WORKING"
144
+ | "MUTED" | "SPEAKING" | "DEGRADED" | "OFFLINE";
145
+ type LifecycleState = AvatarPresenceState;
146
+ type ServerClaim = "STRAINING" | "THINKING" | "WORKING" | null;
147
+
148
+ /** Defensive unwrap for the `RTVIEvent.ServerMessage` `{ data }` quirk: some
149
+ * transports deliver the payload directly and some wrap it once more. */
150
+ function unwrapServerMessage(raw: unknown): Record<string, unknown> {
151
+ const obj = (raw ?? {}) as Record<string, unknown>;
152
+ const inner = obj["data"] as Record<string, unknown> | undefined;
153
+ return inner && "type" in inner ? inner : obj;
154
+ }
155
+
156
+ export class AvatarClient {
157
+ private readonly avatar: AvatarApi;
158
+ /** Maps factual/wire intent into the broader client behavior catalog. */
159
+ private readonly behavior: BehaviorController;
160
+ private readonly opts: AvatarClientOptions;
161
+ private readonly now: () => number;
162
+ private turn: Turn | null = null;
163
+ private readonly turns = new Map<string, Turn>();
164
+ private readonly pendingCtxs: string[] = [];
165
+ private readonly closedCtxs = new Set<string>();
166
+ private projected: LifecycleState | null = null;
167
+ private serverClaim: ServerClaim = null;
168
+ private userSpeaking = false;
169
+ private botSpeaking = false;
170
+ private muted = false;
171
+ private listening = false;
172
+ // Idle is earned after a connected, quiet listening interval. A newly
173
+ // mounted avatar is available, not already "stepped aside".
174
+ private idle = false;
175
+ private failure: "DEGRADED" | "OFFLINE" | null = null;
176
+ private idleTimer: ReturnType<typeof setTimeout> | null = null;
177
+ private pendingInterruptedAction = false;
178
+ private discardQueuedContextsOnBotStop = false;
179
+ private readonly idleDelayMs: number;
180
+ private readonly setTimer: typeof setTimeout;
181
+ private readonly clearTimer: typeof clearTimeout;
182
+
183
+ constructor(avatar: AvatarApi, opts: AvatarClientOptions = {}) {
184
+ this.avatar = avatar;
185
+ this.behavior = new BehaviorController(avatar);
186
+ this.opts = opts;
187
+ this.now = opts.now ?? (() => performance.now());
188
+ this.idleDelayMs = opts.idleDelayMs ?? 12_000;
189
+ this.setTimer = opts.setTimeout ?? globalThis.setTimeout.bind(globalThis);
190
+ this.clearTimer = opts.clearTimeout ?? globalThis.clearTimeout.bind(globalThis);
191
+ }
192
+
193
+ /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
194
+ get turnCtx(): string | null {
195
+ return this.turn?.ctx ?? null;
196
+ }
197
+
198
+ /** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
199
+ get turnCues(): AvatarCue[] {
200
+ return this.turn ? [...this.turn.cues] : [];
201
+ }
202
+
203
+ /** Current resolved projection. Internal — for tests and Studio's inspector.
204
+ * Before the first lifecycle fact the renderer is simply at its ready rest
205
+ * pose, so `LISTENING` is the safe value; it is not a session status. */
206
+ get presenceState(): AvatarPresenceState {
207
+ return this.projected ?? "LISTENING";
208
+ }
209
+
210
+ /** Re-apply the currently resolved factual projection after an embedding
211
+ * tool has temporarily used raw renderer controls. This does not invent a
212
+ * lifecycle event or cancel a finite action. */
213
+ restoreProjection(): void {
214
+ this.applyProjection(true);
215
+ }
216
+
217
+ /** Dispatch one server message. Anything that isn't in the avatar envelope
218
+ * is not ours and is ignored; so is an envelope carrying a `cmd` this build
219
+ * has never heard of, per the wire protocol's forward-compat rule. */
220
+ dispatch(raw: unknown): void {
221
+ if (!isAvatarMessage(raw)) return;
222
+ const msg = parseAvatarCommand(raw);
223
+ if (!msg) return;
224
+ try {
225
+ switch (msg.cmd) {
226
+ case "claim":
227
+ this.handleClaim(msg.state);
228
+ break;
229
+ case "action":
230
+ this.handleAction(msg.id);
231
+ break;
232
+ case "cues":
233
+ this.handleCues(msg);
234
+ break;
235
+ // No default: an unknown `cmd` is a newer server talking to an older
236
+ // widget, and the protocol's forward-compat rule says ignore it. There
237
+ // is no callback for it — a hook nobody could act on is observability,
238
+ // not an interface.
239
+ }
240
+ } catch (err) {
241
+ if (this.opts.onError) this.opts.onError(err, msg);
242
+ else console.warn("[avatar] dispatch failed", msg, err);
243
+ }
244
+ }
245
+
246
+ private handleClaim(state: ServerClaim): void {
247
+ this.serverClaim = state;
248
+ if (state) this.idle = false;
249
+ this.applyProjection();
250
+ this.armIdleIfEligible();
251
+ }
252
+
253
+ private handleAction(id: AvatarActionId): void {
254
+ // An interruption is a server-confirmed explanation of a transition, not
255
+ // authority to steal the mouth while bot audio is still playing. Hold it
256
+ // until Pipecat output has released the speaking state.
257
+ if (id === "RESPONSE_INTERRUPTED" && this.botSpeaking) {
258
+ this.pendingInterruptedAction = true;
259
+ // Any prefetched TTS contexts behind interrupted output belong to
260
+ // audio Pipecat will now discard. Never let one animate a later reply.
261
+ this.discardQueuedContextsOnBotStop = true;
262
+ return;
263
+ }
264
+ this.playAction(id);
265
+ }
266
+
267
+ private playAction(id: AvatarActionId): void {
268
+ this.behavior.wireAction(id);
269
+ }
270
+
271
+ private ensureTurn(ctx: string): Turn {
272
+ const existing = this.turns.get(ctx);
273
+ if (existing) return existing;
274
+ const turn = { ctx, cues: [], started: false, clock: null };
275
+ this.turns.set(ctx, turn);
276
+ this.pendingCtxs.push(ctx);
277
+ return turn;
278
+ }
279
+
280
+ private handleCues(msg: AvatarCuesCmd) {
281
+ if (this.closedCtxs.has(msg.ctx)) return;
282
+ const turn = this.ensureTurn(msg.ctx);
283
+ const kept = turn.cues.filter((c) => c.t < msg.from_ms);
284
+ const discarded = turn.cues.length - kept.length;
285
+ turn.cues = [...kept, ...msg.cues].sort((a, b) => a.t - b.t);
286
+
287
+ if (!turn.started) {
288
+ // No clock yet — buffer. Pipecat output will claim this FIFO context.
289
+ if (this.botSpeaking && this.turn === null) this.activateNextTurn();
290
+ return;
291
+ }
292
+ if (discarded === 0) {
293
+ this.avatar.pushCues(msg.cues);
294
+ } else {
295
+ this.avatar.speak({ cues: turn.cues, clock: turn.clock! });
296
+ }
297
+ }
298
+
299
+ private activateNextTurn(): void {
300
+ if (this.turn) return;
301
+ let turn: Turn | undefined;
302
+ while (this.pendingCtxs.length && !turn) {
303
+ turn = this.turns.get(this.pendingCtxs.shift()!);
304
+ }
305
+ if (!turn) return;
306
+ const t0 = this.now();
307
+ turn.clock = () => this.now() - t0;
308
+ turn.started = true;
309
+ this.turn = turn;
310
+ this.avatar.speak({ cues: turn.cues, clock: turn.clock });
311
+ }
312
+
313
+ private discardQueuedTurns(): void {
314
+ for (const ctx of this.pendingCtxs.splice(0)) {
315
+ this.closedCtxs.add(ctx);
316
+ this.turns.delete(ctx);
317
+ }
318
+ }
319
+
320
+ private lifecycleState(): LifecycleState {
321
+ // The one copy of the ladder is docs/pipecat-lifecycle-protocol.md; this is
322
+ // its implementation, in the same order.
323
+ //
324
+ // Audio truth is the P0 invariant: no lower claim or microphone event may
325
+ // put the face in a non-speaking pose while bot speech is audible.
326
+ if (this.botSpeaking) return "SPEAKING";
327
+ if (this.userSpeaking) return "LISTENING";
328
+ // A broken session outranks mute, because a mute nobody can hear about is
329
+ // not the thing worth saying about a dead call.
330
+ if (this.failure) return this.failure;
331
+ if (this.muted) return "MUTED";
332
+ // Every claim resolves to the presence state of the same name, so there is
333
+ // nothing to rank here — a claim is a single value and only one can be in
334
+ // flight. STRAINING > THINKING > WORKING is decided where more than one
335
+ // condition can hold at once, which is the server: `AvatarStateMachine.
336
+ // _resolve()`. Ranking them again on this side would be a second, silent
337
+ // copy of that ladder, and the two would drift.
338
+ if (this.serverClaim) return this.serverClaim;
339
+ if (this.idle) return "IDLE";
340
+ return "LISTENING";
341
+ }
342
+
343
+ private applyProjection(force = false): void {
344
+ const state = this.lifecycleState();
345
+ if (force || this.projected !== state) {
346
+ this.behavior.setState(state);
347
+ this.projected = state;
348
+ this.opts.onPresenceChange?.(state);
349
+ }
350
+ }
351
+
352
+ private clearIdleTimer(): void {
353
+ if (this.idleTimer !== null) {
354
+ this.clearTimer(this.idleTimer);
355
+ this.idleTimer = null;
356
+ }
357
+ }
358
+
359
+ private armIdleIfEligible(): void {
360
+ this.clearIdleTimer();
361
+ // Quiet under a mute is not the quiet that earns IDLE — the silence was
362
+ // imposed, and letting the timer run behind it would reveal a stepped-aside
363
+ // face the moment the microphone came back.
364
+ if (!this.eligibleForIdle()) return;
365
+ this.idleTimer = this.setTimer(() => {
366
+ this.idleTimer = null;
367
+ if (this.eligibleForIdle()) {
368
+ this.idle = true;
369
+ this.applyProjection();
370
+ }
371
+ }, this.idleDelayMs);
372
+ }
373
+
374
+ private eligibleForIdle(): boolean {
375
+ return this.listening && !this.userSpeaking && !this.botSpeaking
376
+ && !this.muted && !this.serverClaim && !this.failure;
377
+ }
378
+
379
+ private enterListening(): void {
380
+ this.listening = true;
381
+ this.idle = false;
382
+ this.applyProjection();
383
+ this.armIdleIfEligible();
384
+ }
385
+
386
+ private clearClaimForTurnBoundary(): void {
387
+ // Claims are lower-priority hints. A fresh user turn or bot output means
388
+ // any prior thinking/work claim is no longer allowed to reappear later.
389
+ this.serverClaim = null;
390
+ }
391
+
392
+ private maybePlayInterrupted(): void {
393
+ if (this.pendingInterruptedAction && !this.botSpeaking) {
394
+ this.pendingInterruptedAction = false;
395
+ this.playAction("RESPONSE_INTERRUPTED");
396
+ }
397
+ }
398
+
399
+ private clearRecoverableFailure(): void {
400
+ if (this.failure === "DEGRADED") {
401
+ this.failure = null;
402
+ this.applyProjection();
403
+ }
404
+ }
405
+
406
+ private onUserStartedSpeaking = (): void => {
407
+ this.clearRecoverableFailure();
408
+ this.userSpeaking = true;
409
+ this.clearClaimForTurnBoundary();
410
+ this.listening = true;
411
+ this.idle = false;
412
+ this.clearIdleTimer();
413
+ this.avatar.setUserSpeaking(true);
414
+ this.applyProjection();
415
+ };
416
+
417
+ private onUserStoppedSpeaking = (): void => {
418
+ this.userSpeaking = false;
419
+ this.avatar.setUserSpeaking(false);
420
+ this.enterListening();
421
+ };
422
+
423
+ private onBotStartedSpeaking = (): void => {
424
+ this.clearRecoverableFailure();
425
+ this.botSpeaking = true;
426
+ this.clearClaimForTurnBoundary();
427
+ this.idle = false;
428
+ this.clearIdleTimer();
429
+ this.applyProjection();
430
+ this.activateNextTurn();
431
+ };
432
+
433
+ private onBotStoppedSpeaking = (): void => {
434
+ this.botSpeaking = false;
435
+ // Playout truth releases the only active mouth track. A late cue chunk for
436
+ // this context is ignored rather than reviving a silent mouth.
437
+ if (this.turn) {
438
+ this.avatar.stopSpeaking();
439
+ this.closedCtxs.add(this.turn.ctx);
440
+ this.turns.delete(this.turn.ctx);
441
+ this.turn = null;
442
+ }
443
+ if (this.discardQueuedContextsOnBotStop) {
444
+ this.discardQueuedContextsOnBotStop = false;
445
+ this.discardQueuedTurns();
446
+ }
447
+ this.enterListening();
448
+ this.maybePlayInterrupted();
449
+ };
450
+
451
+ private onUserMuteStarted = (): void => {
452
+ this.muted = true;
453
+ this.idle = false;
454
+ this.clearIdleTimer();
455
+ this.applyProjection();
456
+ };
457
+
458
+ private onUserMuteStopped = (): void => {
459
+ this.muted = false;
460
+ // Whatever the avatar was waiting on before the mute, it is waiting on the
461
+ // user again now — the same place a turn ends.
462
+ this.enterListening();
463
+ };
464
+
465
+ private onError = (raw: unknown): void => {
466
+ const data = (raw as { data?: unknown })?.data as { fatal?: unknown } | undefined;
467
+ this.failure = data?.fatal === true ? "OFFLINE" : "DEGRADED";
468
+ this.applyProjection();
469
+ };
470
+
471
+ private onDisconnected = (): void => {
472
+ this.failure = "OFFLINE";
473
+ this.clearIdleTimer();
474
+ this.applyProjection();
475
+ };
476
+
477
+ private onConnectedOrReady = (): void => {
478
+ if (this.failure === "OFFLINE") this.failure = null;
479
+ // Pipecat has established a session. Until a factual speech event says
480
+ // otherwise, the avatar is available to listen; it earns IDLE only after
481
+ // the regular quiet timer expires.
482
+ this.listening = true;
483
+ this.idle = false;
484
+ this.applyProjection();
485
+ this.armIdleIfEligible();
486
+ };
487
+
488
+ /**
489
+ * Subscribe to a live `PipecatClient`. Standard client events own the normal
490
+ * lifecycle projection; avatar server-messages carry correlated visemes and
491
+ * explicit application intent. Never throws on malformed or irrelevant
492
+ * server messages.
493
+ *
494
+ * @returns an unsubscribe function; call it on unmount or disconnect.
495
+ */
496
+ attach(client: PipecatClient): () => void {
497
+ const onServerMessage = (raw: unknown) => this.dispatch(unwrapServerMessage(raw));
498
+ const subscriptions: Array<[string, (...args: any[]) => void]> = [
499
+ [RTVI_EVENTS.serverMessage, onServerMessage],
500
+ [RTVI_EVENTS.connected, this.onConnectedOrReady],
501
+ [RTVI_EVENTS.botReady, this.onConnectedOrReady],
502
+ [RTVI_EVENTS.disconnected, this.onDisconnected],
503
+ [RTVI_EVENTS.error, this.onError],
504
+ [RTVI_EVENTS.userStartedSpeaking, this.onUserStartedSpeaking],
505
+ [RTVI_EVENTS.userStoppedSpeaking, this.onUserStoppedSpeaking],
506
+ [RTVI_EVENTS.botStartedSpeaking, this.onBotStartedSpeaking],
507
+ [RTVI_EVENTS.botStoppedSpeaking, this.onBotStoppedSpeaking],
508
+ [RTVI_EVENTS.userMuteStarted, this.onUserMuteStarted],
509
+ [RTVI_EVENTS.userMuteStopped, this.onUserMuteStopped],
510
+ ];
511
+ for (const [event, listener] of subscriptions) client.on(event as RTVIEvent, listener as never);
512
+ return () => {
513
+ this.clearIdleTimer();
514
+ for (const [event, listener] of subscriptions) client.off(event as RTVIEvent, listener as never);
515
+ };
516
+ }
517
+
518
+ /** Dispose controller-owned timers when its mounted avatar is destroyed. */
519
+ destroy(): void {
520
+ this.clearIdleTimer();
521
+ this.behavior.destroy();
522
+ }
523
+ }
@@ -0,0 +1,24 @@
1
+ /** Professional Indian male interviewer, authored to read at call-tile size. */
2
+
3
+ import { createCanvasAvatar } from './createCanvasAvatar.js';
4
+ import type { CanvasAvatarOptions } from './createCanvasAvatar.js';
5
+ import type { AvatarInstance } from './createAvatar.js';
6
+
7
+ const RIG_URL = new URL('../src/canvas/data/interviewer-male.rig.json', import.meta.url);
8
+ const IMAGES = {
9
+ 'round-m3-hair-back.webp': new URL('../src/canvas/data/img/round-m3-hair-back.webp', import.meta.url),
10
+ 'round-m3-top-body.webp': new URL('../src/canvas/data/img/round-m3-top-body.webp', import.meta.url),
11
+ 'round-m3-hair-front.webp': new URL('../src/canvas/data/img/round-m3-hair-front.webp', import.meta.url),
12
+ } as const;
13
+ const FACE = () => import('../src/canvas/avatars/round/face.mjs');
14
+
15
+ export type { CanvasAvatarOptions };
16
+
17
+ export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
18
+ return createCanvasAvatar(options, {
19
+ rigUrl: RIG_URL,
20
+ images: IMAGES,
21
+ face: FACE,
22
+ label: 'Arjun — professional male interviewer avatar',
23
+ });
24
+ }