@voqalize/avatar 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE-CC-BY-4.0 +396 -0
  2. package/README.md +65 -17
  3. package/assets/README.md +48 -0
  4. package/assets/tanya.glb +0 -0
  5. package/assets/tara.glb +0 -0
  6. package/assets/tushar.glb +0 -0
  7. package/client/AvatarClient.ts +337 -42
  8. package/client/arjun.ts +2 -0
  9. package/client/createAvatar.ts +38 -0
  10. package/client/index.ts +6 -1
  11. package/client/internal.ts +31 -3
  12. package/client/ishita.ts +2 -0
  13. package/client/kabir.ts +2 -0
  14. package/client/meera.ts +2 -0
  15. package/client/naina.ts +2 -0
  16. package/client/playout.ts +95 -0
  17. package/client/supports.ts +28 -0
  18. package/client/three/assets.ts +24 -0
  19. package/client/three/budgets.ts +27 -0
  20. package/client/three/holds.ts +64 -0
  21. package/client/three/internal.ts +54 -0
  22. package/client/three/motion-limits.json +116 -0
  23. package/client/three/sequences.ts +281 -0
  24. package/client/three/tanya.ts +69 -0
  25. package/client/three/tara-rig.ts +1562 -0
  26. package/client/three/tara.ts +76 -0
  27. package/client/three/tushar.ts +65 -0
  28. package/client/types.ts +109 -20
  29. package/client/vikram.ts +2 -0
  30. package/dist/AvatarClient.d.ts +89 -11
  31. package/dist/AvatarClient.d.ts.map +1 -1
  32. package/dist/AvatarClient.js +297 -35
  33. package/dist/AvatarClient.js.map +1 -1
  34. package/dist/arjun.d.ts +1 -0
  35. package/dist/arjun.d.ts.map +1 -1
  36. package/dist/arjun.js +2 -0
  37. package/dist/arjun.js.map +1 -1
  38. package/dist/createAvatar.d.ts +36 -0
  39. package/dist/createAvatar.d.ts.map +1 -1
  40. package/dist/createAvatar.js.map +1 -1
  41. package/dist/index.d.ts +6 -2
  42. package/dist/index.d.ts.map +1 -1
  43. package/dist/index.js +5 -1
  44. package/dist/index.js.map +1 -1
  45. package/dist/internal.d.ts +9 -5
  46. package/dist/internal.d.ts.map +1 -1
  47. package/dist/internal.js +13 -5
  48. package/dist/internal.js.map +1 -1
  49. package/dist/ishita.d.ts +1 -0
  50. package/dist/ishita.d.ts.map +1 -1
  51. package/dist/ishita.js +2 -0
  52. package/dist/ishita.js.map +1 -1
  53. package/dist/kabir.d.ts +1 -0
  54. package/dist/kabir.d.ts.map +1 -1
  55. package/dist/kabir.js +2 -0
  56. package/dist/kabir.js.map +1 -1
  57. package/dist/meera.d.ts +1 -0
  58. package/dist/meera.d.ts.map +1 -1
  59. package/dist/meera.js +2 -0
  60. package/dist/meera.js.map +1 -1
  61. package/dist/naina.d.ts +1 -0
  62. package/dist/naina.d.ts.map +1 -1
  63. package/dist/naina.js +2 -0
  64. package/dist/naina.js.map +1 -1
  65. package/dist/playout.d.ts +42 -0
  66. package/dist/playout.d.ts.map +1 -0
  67. package/dist/playout.js +85 -0
  68. package/dist/playout.js.map +1 -0
  69. package/dist/supports.d.ts +20 -0
  70. package/dist/supports.d.ts.map +1 -0
  71. package/dist/supports.js +26 -0
  72. package/dist/supports.js.map +1 -0
  73. package/dist/three/assets.d.ts +24 -0
  74. package/dist/three/assets.d.ts.map +1 -0
  75. package/dist/three/assets.js +22 -0
  76. package/dist/three/assets.js.map +1 -0
  77. package/dist/three/budgets.d.ts +17 -0
  78. package/dist/three/budgets.d.ts.map +1 -0
  79. package/dist/three/budgets.js +21 -0
  80. package/dist/three/budgets.js.map +1 -0
  81. package/dist/three/holds.d.ts +33 -0
  82. package/dist/three/holds.d.ts.map +1 -0
  83. package/dist/three/holds.js +56 -0
  84. package/dist/three/holds.js.map +1 -0
  85. package/dist/three/internal.d.ts +52 -0
  86. package/dist/three/internal.d.ts.map +1 -0
  87. package/dist/three/internal.js +49 -0
  88. package/dist/three/internal.js.map +1 -0
  89. package/dist/three/motion-limits.json +116 -0
  90. package/dist/three/sequences.d.ts +71 -0
  91. package/dist/three/sequences.d.ts.map +1 -0
  92. package/dist/three/sequences.js +262 -0
  93. package/dist/three/sequences.js.map +1 -0
  94. package/dist/three/tanya.d.ts +35 -0
  95. package/dist/three/tanya.d.ts.map +1 -0
  96. package/dist/three/tanya.js +51 -0
  97. package/dist/three/tanya.js.map +1 -0
  98. package/dist/three/tara-rig.d.ts +403 -0
  99. package/dist/three/tara-rig.d.ts.map +1 -0
  100. package/dist/three/tara-rig.js +1500 -0
  101. package/dist/three/tara-rig.js.map +1 -0
  102. package/dist/three/tara.d.ts +35 -0
  103. package/dist/three/tara.d.ts.map +1 -0
  104. package/dist/three/tara.js +58 -0
  105. package/dist/three/tara.js.map +1 -0
  106. package/dist/three/tushar.d.ts +31 -0
  107. package/dist/three/tushar.d.ts.map +1 -0
  108. package/dist/three/tushar.js +47 -0
  109. package/dist/three/tushar.js.map +1 -0
  110. package/dist/types.d.ts +65 -10
  111. package/dist/types.d.ts.map +1 -1
  112. package/dist/types.js +58 -11
  113. package/dist/types.js.map +1 -1
  114. package/dist/vikram.d.ts +1 -0
  115. package/dist/vikram.d.ts.map +1 -1
  116. package/dist/vikram.js +2 -0
  117. package/dist/vikram.js.map +1 -1
  118. package/package.json +34 -6
  119. package/src/avatar.d.ts +59 -2
  120. package/src/avatar.js +743 -94
  121. package/src/behavior.d.ts +28 -12
  122. package/src/behavior.js +54 -42
  123. package/src/canvas/create-rig.js +1 -1
  124. package/src/canvas/src/live.js +2 -2
  125. package/src/conformance.js +1 -1
  126. package/src/emotions.js +4 -2
  127. package/src/gaze.js +334 -73
  128. package/src/head.js +134 -0
  129. package/src/idle.js +210 -19
  130. package/src/interjections.js +70 -26
  131. package/src/prosody.js +647 -0
  132. package/src/visemes.js +85 -15
@@ -0,0 +1,76 @@
1
+ /**
2
+ * tara — the first 3-D character, as a complete `createAvatar` module.
3
+ *
4
+ * Structurally this is the canvas avatars' shape, one layer down: the mixer
5
+ * (`@voqalize/avatar/internal`) does states, layers, gaze, idle, blinks, clips
6
+ * and per-channel smoothing exactly as it does for every SVG face, and the only
7
+ * thing tara replaces is the renderer at the end of it. That is the whole
8
+ * argument for authoring her morph targets under the library's pose-channel
9
+ * names: nothing above this line knows there is a GPU involved, and every clip,
10
+ * viseme and co-articulation rule the SVG faces have works here unmodified.
11
+ *
12
+ * The Three.js import lives behind this module rather than in the barrel so an
13
+ * SVG or Canvas consumer never downloads it.
14
+ */
15
+
16
+ import type { PipecatClient } from "@pipecat-ai/client-js";
17
+ import { AvatarClient, createSvgAvatar } from "../internal.js";
18
+ import { BLENDER_ACTIONS, BLENDER_SEQUENCES, BLENDER_SUPPORTS } from "./sequences.js";
19
+ import { headHold } from "./holds.js";
20
+ import { createTaraRig, TARA_TUNING } from "./tara-rig.js";
21
+ import type { TaraRigOptions } from "./tara-rig.js";
22
+
23
+ export interface AvatarOptions {
24
+ readonly mount: HTMLElement;
25
+ readonly client: PipecatClient;
26
+ /** Viseme amplitude, 0..2. `1` is as authored. */
27
+ readonly mouthGain?: number;
28
+ /** Gesture-clip amplitude, 0..2. */
29
+ readonly gestureGain?: number;
30
+ /** Idle/liveness amplitude, 0..2. */
31
+ readonly motionGain?: number;
32
+ /** Fires when the GLB is in the scene — for a capture tool, not a consumer. */
33
+ readonly onReady?: () => void;
34
+ }
35
+
36
+ export interface AvatarInstance { destroy(): void; }
37
+
38
+ /** The optional driving-UI declaration; one list for all three characters
39
+ * (`sequences.ts`), because they are one rig. */
40
+ export const supports = BLENDER_SUPPORTS;
41
+
42
+ export function createAvatar(options: AvatarOptions): AvatarInstance {
43
+ const { mount, client, onReady, ...gains } = options;
44
+ if (!mount) throw new TypeError("createAvatar: `mount` is required");
45
+ if (!client) throw new TypeError("createAvatar: `client` is required");
46
+
47
+ const rigOptions: TaraRigOptions = { onReady };
48
+ // `hand: false` disables the bundled SVG hand renderer only; the semantic
49
+ // hand frame still reaches the rig, which ignores it — tara has no arms, and
50
+ // that is the library's oldest standing constraint rather than an omission.
51
+ // `sequences` is what a server can address on *this* avatar beyond the two
52
+ // ids every avatar owes it — the three nod types the listening research
53
+ // separates, and a head shake sized for a rig whose pose unit is a degree.
54
+ // The wire's action id is open, so nothing here needed promoting; a server
55
+ // that does not know tara is mounted sends `ACKNOWLEDGE` and is never wrong.
56
+ // `actions` is the other half: her own shape for an id the mixer already has,
57
+ // whose shared keys land outside what it means on a head that turns in degrees.
58
+ const widget = createSvgAvatar({
59
+ mount, rig: createTaraRig, rigOptions, hand: false,
60
+ sequences: BLENDER_SEQUENCES, actions: BLENDER_ACTIONS, ...TARA_TUNING,
61
+ headHold: headHold("tara"), ...gains,
62
+ });
63
+ const driver = new AvatarClient(widget);
64
+ const detach = driver.attach(client);
65
+
66
+ let destroyed = false;
67
+ return {
68
+ destroy() {
69
+ if (destroyed) return;
70
+ destroyed = true;
71
+ detach();
72
+ driver.destroy();
73
+ widget.destroy();
74
+ },
75
+ };
76
+ }
@@ -0,0 +1,65 @@
1
+ /**
2
+ * tushar — the second 3-D character, and deliberately a copy of `tara.ts`.
3
+ *
4
+ * His GLB is tara's scripts run with his landmarks and atlas layout in place of
5
+ * hers (`characters/tushar/`, whose README has the provenance and the commands),
6
+ * so the rig, the mixer, the sequences and her tuning are all hers, unmodified;
7
+ * the one seam is `TaraRigOptions.url`. What turns out to be hers rather than
8
+ * general is marked TARA-SPECIFIC where it lives, and generalised only once a
9
+ * third character agrees.
10
+ */
11
+
12
+ import type { PipecatClient } from "@pipecat-ai/client-js";
13
+ import { AvatarClient, createSvgAvatar } from "../internal.js";
14
+ import { ASSETS } from "./assets.js";
15
+ import { BLENDER_ACTIONS, BLENDER_SEQUENCES, BLENDER_SUPPORTS } from "./sequences.js";
16
+ import { headHold } from "./holds.js";
17
+ import { createTaraRig, TARA_TUNING } from "./tara-rig.js";
18
+ import type { TaraRigOptions } from "./tara-rig.js";
19
+
20
+ export interface AvatarOptions {
21
+ readonly mount: HTMLElement;
22
+ readonly client: PipecatClient;
23
+ /** Viseme amplitude, 0..2. `1` is as authored. */
24
+ readonly mouthGain?: number;
25
+ /** Gesture-clip amplitude, 0..2. */
26
+ readonly gestureGain?: number;
27
+ /** Idle/liveness amplitude, 0..2. */
28
+ readonly motionGain?: number;
29
+ /** Fires when the GLB is in the scene — for a capture tool, not a consumer. */
30
+ readonly onReady?: () => void;
31
+ }
32
+
33
+ export interface AvatarInstance { destroy(): void; }
34
+
35
+ /** The optional driving-UI declaration; one list for all three characters
36
+ * (`sequences.ts`), because they are one rig. */
37
+ export const supports = BLENDER_SUPPORTS;
38
+
39
+ export function createAvatar(options: AvatarOptions): AvatarInstance {
40
+ const { mount, client, onReady, ...gains } = options;
41
+ if (!mount) throw new TypeError("createAvatar: `mount` is required");
42
+ if (!client) throw new TypeError("createAvatar: `client` is required");
43
+
44
+ const rigOptions: TaraRigOptions = { onReady, url: ASSETS.tushar };
45
+ // TARA-SPECIFIC, kept on purpose: `TARA_TUNING` (her mouth and motion gains)
46
+ // is the first thing to question if he reads wrong in a call.
47
+ const widget = createSvgAvatar({
48
+ mount, rig: createTaraRig, rigOptions, hand: false,
49
+ sequences: BLENDER_SEQUENCES, actions: BLENDER_ACTIONS, ...TARA_TUNING,
50
+ headHold: headHold("tushar"), ...gains,
51
+ });
52
+ const driver = new AvatarClient(widget);
53
+ const detach = driver.attach(client);
54
+
55
+ let destroyed = false;
56
+ return {
57
+ destroy() {
58
+ if (destroyed) return;
59
+ destroyed = true;
60
+ detach();
61
+ driver.destroy();
62
+ widget.destroy();
63
+ },
64
+ };
65
+ }
package/client/types.ts CHANGED
@@ -15,8 +15,8 @@
15
15
  * ignored, which is the whole forward-compatibility rule.
16
16
  */
17
17
 
18
- import { ACTION_IDS, VISEME_LETTERS } from "../src/avatar.js";
19
- import type { AvatarActionId, VisemeLetter } from "../src/avatar.js";
18
+ import { VISEME_LETTERS } from "../src/avatar.js";
19
+ import type { VisemeLetter } from "../src/avatar.js";
20
20
 
21
21
  /** A viseme cue: `t` is a ms offset into the utterance's clock, `v` is a Rhubarb A–H (or X) letter. */
22
22
  export interface AvatarCue {
@@ -24,20 +24,77 @@ export interface AvatarCue {
24
24
  v: VisemeLetter;
25
25
  /** 0..1 loudness. Absent means full. */
26
26
  i?: number;
27
+ /**
28
+ * The phone being articulated under `v`, absent during silence.
29
+ *
30
+ * `v` is a nine-way projection of the recogniser's ~40 phones and the loss is
31
+ * concentrated — `B` alone absorbs IY, IH, T, D, CH, JH, TH, DH, S, Z, SH,
32
+ * ZH, N and Y — so a renderer with a mouth for "tongue between the teeth"
33
+ * cannot ask for it from `v` and can from here. **Not a closed set**: it is
34
+ * whatever the engine labels its own segments with, which today is Arpabet
35
+ * plus `Schwa` and four non-speech labels, and a later engine may say
36
+ * something else. Validated as an identifier and passed through unread —
37
+ * the letter stays the thing every face must handle, and a face that wants
38
+ * the finer signal opts in.
39
+ */
40
+ p?: string;
27
41
  }
28
42
 
29
- /** A durable, lower-priority server claim. `null` explicitly clears it. */
30
- export interface AvatarClaimCmd {
31
- cmd: "claim";
32
- state: "STRAINING" | "THINKING" | "WORKING" | null;
43
+ /**
44
+ * A durable, lower-priority server state. `null` explicitly clears it.
45
+ *
46
+ * Three values, and the server can send no others: the remaining six of the
47
+ * nine are Pipecat facts the browser already holds, and a server spelling of a
48
+ * fact would be a second, lower-authority copy of it. It was `cmd: "claim"`
49
+ * until the wire redesign, when the concept and the command were given the one
50
+ * name the rest of the system already used for it.
51
+ */
52
+ export interface AvatarStateCmd {
53
+ cmd: "state";
54
+ state: "CANT_HEAR" | "THINKING" | "WORKING" | null;
33
55
  }
34
56
 
35
- /** A self-completing authored sequence: face, body, and optionally a hand. */
57
+ /**
58
+ * One self-completing motion, by name: face, body, and optionally a hand.
59
+ *
60
+ * **The id is open.** Two of them — {@link CORE_ACTION_IDS} — are intents every
61
+ * renderer owes a server and are the only ones a server may send without
62
+ * knowing what is mounted. Everything else is a name from the mounted avatar's
63
+ * own catalogue, and **an unknown one is ignored, not an error**: that is the
64
+ * same forward-compatibility rule an unknown `cmd` gets, arrived at from the
65
+ * other direction. There, a newer server meets an older widget; here, any
66
+ * server meets a face that cannot do the thing. Neither is worth breaking a
67
+ * call over, and there is deliberately no fallback — an action sits on top of a
68
+ * state, so nothing is missing when one is dropped.
69
+ *
70
+ * This replaced a closed seven-id vocabulary plus a second command,
71
+ * `sequence`, for a renderer's own motions. The split cost a promotion ritual
72
+ * for every new portable intent and still could not say what the closed set was
73
+ * for: the listening research separates a continuer nod from an assessment nod
74
+ * from a realisation, and all three were the one id `ACK_NOD` spelled. Now the
75
+ * server says *when* to acknowledge and the avatar owns the variety.
76
+ */
36
77
  export interface AvatarActionCmd {
37
78
  cmd: "action";
38
- id: AvatarActionId;
79
+ id: string;
39
80
  }
40
81
 
82
+ /**
83
+ * The two actions every renderer must answer to.
84
+ *
85
+ * `ACKNOWLEDGE` is the whole backchannel family in one word — receipt, nod,
86
+ * realisation, empathy — because which of those a face does is a rendering
87
+ * decision and the server is not the one holding the drawing.
88
+ * `RESPONSE_INTERRUPTED` is the one transition a server can explain and the
89
+ * browser cannot infer.
90
+ *
91
+ * A renderer may draw either in its own shape; it may never redefine one, and
92
+ * a name in its own catalogue never shadows one of these.
93
+ */
94
+ export const CORE_ACTION_IDS = Object.freeze(["ACKNOWLEDGE", "RESPONSE_INTERRUPTED"] as const);
95
+
96
+ export type CoreActionId = (typeof CORE_ACTION_IDS)[number];
97
+
41
98
  export interface AvatarCuesCmd {
42
99
  cmd: "cues";
43
100
  ctx: string;
@@ -60,7 +117,7 @@ export interface AvatarCuesCmd {
60
117
  }
61
118
 
62
119
  export type AvatarCommand =
63
- | AvatarClaimCmd
120
+ | AvatarStateCmd
64
121
  | AvatarActionCmd
65
122
  | AvatarCuesCmd;
66
123
 
@@ -85,13 +142,25 @@ export function isAvatarMessage(msg: unknown): msg is AvatarEnvelope {
85
142
  return m.type === AVATAR_MESSAGE_TYPE && typeof m.cmd === "string";
86
143
  }
87
144
 
88
- const ACTIONS = new Set<string>(ACTION_IDS);
145
+ /**
146
+ * What an action id may look like. Not which ones exist — that is the mounted
147
+ * avatar's business and this function has never met it. `CATEGORY_INTENT` in
148
+ * upper case, bounded so a malformed or hostile message cannot smuggle
149
+ * anything through as a name.
150
+ */
151
+ const ACTION_ID = /^[A-Z][A-Z0-9_]{1,63}$/;
89
152
  const LETTERS = new Set<string>(VISEME_LETTERS);
153
+ /**
154
+ * What a phone label may look like. Letters only, because every label any
155
+ * engine we have seen is one word of them — `AO`, `NG`, `Schwa`, `Breath` —
156
+ * and the bound is what stops a name being a payload.
157
+ */
158
+ const PHONE_LABEL = /^[A-Za-z]{1,12}$/;
90
159
 
91
160
  /**
92
161
  * Read an envelope's payload into the wire vocabulary, or `null` if this build
93
- * cannot act on it — an unknown `cmd`, an id or claim outside the enum, a
94
- * malformed `cues` chunk.
162
+ * cannot act on it — an unknown `cmd`, a state outside the three, an action id
163
+ * that is not a name, a malformed `cues` chunk.
95
164
  *
96
165
  * `null` is the forward-compat rule with a type attached: a newer server
97
166
  * talking to an older widget is *expected*, and the older widget ignores what
@@ -106,16 +175,29 @@ const LETTERS = new Set<string>(VISEME_LETTERS);
106
175
  */
107
176
  export function parseAvatarCommand(msg: AvatarEnvelope): AvatarCommand | null {
108
177
  switch (msg.cmd) {
109
- case "claim": {
110
- const state = msg.state;
111
- if (state === null || state === "STRAINING" || state === "THINKING" || state === "WORKING") {
112
- return { cmd: "claim", state };
178
+ // `claim` was this command's name, and `STRAINING` one of its values, until
179
+ // the wire redesign. Both are accepted and translated here, in the one
180
+ // place that reads untrusted input, so no layer above has ever heard of
181
+ // either spelling. Servers that predate the rename keep working; the old
182
+ // names appear nowhere else and come out when the last one has shipped.
183
+ case "claim":
184
+ case "state": {
185
+ const state = msg.state === "STRAINING" ? "CANT_HEAR" : msg.state;
186
+ if (state === null || state === "CANT_HEAR" || state === "THINKING" || state === "WORKING") {
187
+ return { cmd: "state", state };
113
188
  }
114
189
  return null;
115
190
  }
191
+ // `sequence` was a second command for the same thing: a name resolved
192
+ // against the mounted avatar rather than against a closed list. Now that
193
+ // `action` is that, the two are one command and the old one is an alias.
194
+ case "sequence":
116
195
  case "action":
117
- return typeof msg.id === "string" && ACTIONS.has(msg.id)
118
- ? { cmd: "action", id: msg.id as AvatarActionId }
196
+ // Shape only, both for a core id and for one of the avatar's own. There
197
+ // is nothing to check a name against here — the catalogue belongs to
198
+ // whatever is mounted, and this function has never met it.
199
+ return typeof msg.id === "string" && ACTION_ID.test(msg.id)
200
+ ? { cmd: "action", id: msg.id }
119
201
  : null;
120
202
  case "cues": {
121
203
  if (typeof msg.ctx !== "string" || !Number.isFinite(msg.from_ms) || !Array.isArray(msg.cues)) {
@@ -124,10 +206,17 @@ export function parseAvatarCommand(msg: AvatarEnvelope): AvatarCommand | null {
124
206
  const cues: AvatarCue[] = [];
125
207
  for (const c of msg.cues as unknown[]) {
126
208
  if (typeof c !== "object" || c === null) continue;
127
- const { t, v, i } = c as Record<string, unknown>;
209
+ const { t, v, i, p } = c as Record<string, unknown>;
128
210
  if (typeof t !== "number" || !Number.isFinite(t)) continue;
129
211
  if (typeof v !== "string" || !LETTERS.has(v)) continue;
130
- cues.push(typeof i === "number" ? { t, v: v as VisemeLetter, i } : { t, v: v as VisemeLetter });
212
+ const cue: AvatarCue = { t, v: v as VisemeLetter };
213
+ if (typeof i === "number") cue.i = i;
214
+ // Bounded as a name, not checked against a list. The set belongs to
215
+ // whatever produced the cue, so enumerating it here would make this
216
+ // library the thing that has to be released before a better recogniser
217
+ // can say a new word.
218
+ if (typeof p === "string" && PHONE_LABEL.test(p)) cue.p = p;
219
+ cues.push(cue);
131
220
  }
132
221
  return {
133
222
  cmd: "cues",
package/client/vikram.ts CHANGED
@@ -13,6 +13,8 @@ const IMAGES = {
13
13
  const FACE = () => import('../src/canvas/avatars/round/face.mjs');
14
14
 
15
15
  export type { CanvasAvatarOptions };
16
+ // Same mixer, same clip library, so the same declaration (`supports.ts`).
17
+ export { supports } from './supports.js';
16
18
 
17
19
  export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
18
20
  return createCanvasAvatar(options, {
@@ -7,17 +7,36 @@
7
7
  * Base Pipecat TTS gives each serialized TTS context an opaque `context_id`.
8
8
  * The server uses it only to group and splice cue chunks. `botStartedSpeaking`
9
9
  * has no context payload, so the browser FIFO-claims the next buffered context
10
- * at that Pipecat output-lifecycle event and anchors its clock there.
11
- * `botStoppedSpeaking` closes the active context. Neither event observes the
12
- * browser's audio device, and no avatar-specific speech marker exists.
10
+ * at that Pipecat output-lifecycle event. `botStoppedSpeaking` closes it. No
11
+ * avatar-specific speech marker exists.
12
+ *
13
+ * The event says *which* audio is starting, not *when* it is heard, and the
14
+ * turn clock is anchored to the sound itself (`playout.ts` has the
15
+ * measurements that forced it). `attach()` listens to the bot's audio track;
16
+ * at the event, the turn's zero is backdated to a sound that has already begun,
17
+ * or the mouth is held shut until one does. When the track cannot be heard —
18
+ * no `attach()`, no track, a suspended audio graph, or a sound that never
19
+ * arrives within `ONSET_WAIT_MS` — the event is the anchor, as it always was.
20
+ * Which one won is logged once per turn, at `info`, because a desync report is
21
+ * unanswerable without it.
22
+ *
23
+ * The clock then runs `VISUAL_LEAD_MS` ahead of the sound, since the mixer's
24
+ * smoothing makes every mouth shape late by about that much. It is pulled back
25
+ * by any output-device latency beyond what the display already matches (a
26
+ * Bluetooth headset). Within a turn, each resumption after a pause in the
27
+ * track is a second chance to hear where the sound really is — but a mouth
28
+ * opens before its sound, so a resumption heard within the track's usual
29
+ * anticipation of it agrees with the clock. Only a disagreement beyond that is
30
+ * slewed out, at no more than 10 % of clock rate, never jumped, because a
31
+ * mouth that skips is seen and one that runs briefly fast is not.
13
32
  *
14
33
  * `attach()` subscribes to the avatar server-message channel *and* Pipecat's
15
34
  * standard lifecycle events. Server messages carry only what Pipecat cannot:
16
35
  * correlated speech/cue timing and deliberate application instructions. The
17
36
  * lifecycle events project the factual presence states locally. The server
18
- * supplies only lower-priority `THINKING` / `WORKING` claims and deliberate,
37
+ * supplies only the three lower-priority states and deliberate,
19
38
  * self-completing actions. This keeps the face tied to Pipecat's output truth
20
- * even if a server claim is delayed or stale.
39
+ * even if a server state is delayed or stale.
21
40
  *
22
41
  * ## Cue splice
23
42
  *
@@ -51,8 +70,41 @@
51
70
  * necessarily the first *message*.
52
71
  */
53
72
  import type { PipecatClient } from "@pipecat-ai/client-js";
54
- import type { AvatarApi } from "../src/avatar.js";
73
+ import { type PlayoutProbe } from "./playout.js";
55
74
  import { type AvatarCommand, type AvatarCue } from "./types.js";
75
+ /**
76
+ * How far the cue clock runs ahead of the sound. The mixer eases every mouth
77
+ * channel toward its target with a 42 ms time constant
78
+ * (`MOUTH_RESPONSE_TAU_S`), which delays a shape by about that much, and a
79
+ * frame is drawn on average half a frame after its time. Picture ahead of sound
80
+ * is the side people forgive (ITU-R BT.1359: sound leading is noticed at about
81
+ * 45 ms, picture leading at about 125 ms), so a small surplus is the safe error.
82
+ */
83
+ export declare const VISUAL_LEAD_MS = 50;
84
+ /**
85
+ * The private renderer-side vocabulary the lifecycle driver actually uses.
86
+ *
87
+ * This stays narrower than `AvatarApi` on purpose. The latter is the SVG
88
+ * mixer's driving API; requiring a second renderer to impersonate every one
89
+ * of its inspection and authoring methods would turn that implementation
90
+ * detail into a renderer contract. The public seam remains `createAvatar`.
91
+ */
92
+ export interface AvatarDriver {
93
+ setState(name: string): unknown;
94
+ /**
95
+ * One motion by name — a core intent or one of this avatar's own. **A name
96
+ * it does not have is a no-op, not a throw**: the wire's action vocabulary is
97
+ * open, so a face that cannot do the thing is as expected as a newer server.
98
+ */
99
+ action(id: string): unknown;
100
+ speak(options: {
101
+ cues: AvatarCue[];
102
+ clock: () => number;
103
+ }): unknown;
104
+ pushCues(cues: AvatarCue[]): unknown;
105
+ stopSpeaking(): unknown;
106
+ setUserSpeaking(speaking: boolean | null): unknown;
107
+ }
56
108
  /**
57
109
  * Internal. Not exported from the package — the public surface is
58
110
  * `createAvatar({ mount, client })` and nothing else.
@@ -70,6 +122,9 @@ export interface AvatarClientOptions {
70
122
  onError?: (err: unknown, msg: AvatarCommand) => void;
71
123
  /** Override for tests. Defaults to `performance.now`. */
72
124
  now?: () => number;
125
+ /** Override for tests. Defaults to listening to the bot's audio track, once
126
+ * `attach()` has found one. */
127
+ playoutProbe?: PlayoutProbe;
73
128
  /** Quiet time in listening before the client-owned idle loop begins. */
74
129
  idleDelayMs?: number;
75
130
  /** Timer seams keep lifecycle behavior deterministic in tests. */
@@ -110,11 +165,12 @@ export declare const RTVI_EVENTS: {
110
165
  readonly userStoppedSpeaking: "userStoppedSpeaking";
111
166
  readonly botStartedSpeaking: "botStartedSpeaking";
112
167
  readonly botStoppedSpeaking: "botStoppedSpeaking";
168
+ readonly trackStarted: "trackStarted";
113
169
  readonly userMuteStarted: "userMuteStarted";
114
170
  readonly userMuteStopped: "userMuteStopped";
115
171
  };
116
172
  /** The resolved, factual presence state a host may render around the avatar. */
117
- export type AvatarPresenceState = "IDLE" | "LISTENING" | "STRAINING" | "THINKING" | "WORKING" | "MUTED" | "SPEAKING" | "DEGRADED" | "OFFLINE";
173
+ export type AvatarPresenceState = "IDLE" | "LISTENING" | "CANT_HEAR" | "THINKING" | "WORKING" | "MUTED" | "SPEAKING" | "DEGRADED" | "OFFLINE";
118
174
  export declare class AvatarClient {
119
175
  private readonly avatar;
120
176
  /** Maps factual/wire intent into the broader client behavior catalog. */
@@ -126,7 +182,7 @@ export declare class AvatarClient {
126
182
  private readonly pendingCtxs;
127
183
  private readonly closedCtxs;
128
184
  private projected;
129
- private serverClaim;
185
+ private serverState;
130
186
  private userSpeaking;
131
187
  private botSpeaking;
132
188
  private muted;
@@ -139,7 +195,14 @@ export declare class AvatarClient {
139
195
  private readonly idleDelayMs;
140
196
  private readonly setTimer;
141
197
  private readonly clearTimer;
142
- constructor(avatar: AvatarApi, opts?: AvatarClientOptions);
198
+ private probe;
199
+ private probeTrack;
200
+ /** Whether the probe has ever heard an onset. One that has not, and times
201
+ * out, is deaf to this track rather than early to it. */
202
+ private probeHeard;
203
+ private onsetTimer;
204
+ private reanchorTimer;
205
+ constructor(avatar: AvatarDriver, opts?: AvatarClientOptions);
143
206
  /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
144
207
  get turnCtx(): string | null;
145
208
  /** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
@@ -156,12 +219,27 @@ export declare class AvatarClient {
156
219
  * is not ours and is ignored; so is an envelope carrying a `cmd` this build
157
220
  * has never heard of, per the wire protocol's forward-compat rule. */
158
221
  dispatch(raw: unknown): void;
159
- private handleClaim;
222
+ private handleState;
160
223
  private handleAction;
161
224
  private playAction;
162
225
  private ensureTurn;
163
226
  private handleCues;
164
227
  private activateNextTurn;
228
+ /** Put the turn's zero where its sound began — see the header. */
229
+ private anchor;
230
+ private anchored;
231
+ /** The lead for a turn starting now — see `VISUAL_LEAD_MS`. */
232
+ private leadMs;
233
+ /** The first resumption after `after` (track ms) that follows a pause long
234
+ * enough to be heard as one. */
235
+ private nextResumption;
236
+ /** Listen for the sound at the turn's next resumption, once. */
237
+ private scheduleReanchor;
238
+ private clearReanchorTimer;
239
+ private clearOnsetTimer;
240
+ private dropProbe;
241
+ /** Listen to the bot's audio track, if the transport has one yet. */
242
+ private listenTo;
165
243
  private discardQueuedTurns;
166
244
  private lifecycleState;
167
245
  private applyProjection;
@@ -169,7 +247,7 @@ export declare class AvatarClient {
169
247
  private armIdleIfEligible;
170
248
  private eligibleForIdle;
171
249
  private enterListening;
172
- private clearClaimForTurnBoundary;
250
+ private clearServerStateForTurnBoundary;
173
251
  private maybePlayInterrupted;
174
252
  private clearRecoverableFailure;
175
253
  private onUserStartedSpeaking;
@@ -1 +1 @@
1
- {"version":3,"file":"AvatarClient.d.ts","sourceRoot":"","sources":["../client/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAa,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAkB,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,EAGL,KAAK,aAAa,EAClB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAYpB;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC;wDACoD;IACpD,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC;IACrD,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,UAAU,CAAC,EAAE,OAAO,UAAU,CAAC;IAC/B,YAAY,CAAC,EAAE,OAAO,YAAY,CAAC;IACnC;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;CACzD;AAED;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,WAAW;;;;;;;;;;;;CAgBmB,CAAC;AAE5C,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAC3B,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,GAC3D,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAYlD,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2B;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,SAAS,CAAS;IAG1B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,SAAS,CAA8C;IAC/D,OAAO,CAAC,wBAAwB,CAAS;IACzC,OAAO,CAAC,8BAA8B,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoB;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;gBAErC,MAAM,EAAE,SAAS,EAAE,IAAI,GAAE,mBAAwB;IAU7D,+EAA+E;IAC/E,IAAI,OAAO,IAAI,MAAM,GAAG,IAAI,CAE3B;IAED,wFAAwF;IACxF,IAAI,QAAQ,IAAI,SAAS,EAAE,CAE1B;IAED;;6EAEyE;IACzE,IAAI,aAAa,IAAI,mBAAmB,CAEvC;IAED;;oDAEgD;IAChD,iBAAiB,IAAI,IAAI;IAIzB;;0EAEsE;IACtE,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IA0B5B,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,YAAY;IAcpB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,UAAU;IAmBlB,OAAO,CAAC,gBAAgB;IAcxB,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,eAAe;IAKvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,yBAAyB;IAMjC,OAAO,CAAC,oBAAoB;IAO5B,OAAO,CAAC,uBAAuB;IAO/B,OAAO,CAAC,qBAAqB,CAS3B;IAEF,OAAO,CAAC,qBAAqB,CAI3B;IAEF,OAAO,CAAC,oBAAoB,CAQ1B;IAEF,OAAO,CAAC,oBAAoB,CAgB1B;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,OAAO,CAIb;IAEF,OAAO,CAAC,cAAc,CAIpB;IAEF,OAAO,CAAC,kBAAkB,CASxB;IAEF;;;;;;;OAOG;IACH,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,IAAI;IAsBzC,4EAA4E;IAC5E,OAAO,IAAI,IAAI;CAIhB"}
1
+ {"version":3,"file":"AvatarClient.d.ts","sourceRoot":"","sources":["../client/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAa,MAAM,uBAAuB,CAAC;AAEtE,OAAO,EAAsB,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,EAGL,KAAK,aAAa,EAClB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAgCpB;;;;;;;GAOG;AACH,eAAO,MAAM,cAAc,KAAK,CAAC;AA4BjC;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IAChC;;;;OAIG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE;QAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;IACpE,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IACrC,YAAY,IAAI,OAAO,CAAC;IACxB,eAAe,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC;CACpD;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC;wDACoD;IACpD,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC;IACrD,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB;mCAC+B;IAC/B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,UAAU,CAAC,EAAE,OAAO,UAAU,CAAC;IAC/B,YAAY,CAAC,EAAE,OAAO,YAAY,CAAC;IACnC;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;CACzD;AAED;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,WAAW;;;;;;;;;;;;;CAmBmB,CAAC;AAE5C,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAC3B,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,GAC3D,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAYlD,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2B;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,SAAS,CAAS;IAG1B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,SAAS,CAA8C;IAC/D,OAAO,CAAC,wBAAwB,CAAS;IACzC,OAAO,CAAC,8BAA8B,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoB;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;IACjD,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,UAAU,CAAiC;IACnD;6DACyD;IACzD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,UAAU,CAA8C;IAChE,OAAO,CAAC,aAAa,CAA8C;gBAEvD,MAAM,EAAE,YAAY,EAAE,IAAI,GAAE,mBAAwB;IAWhE,+EAA+E;IAC/E,IAAI,OAAO,IAAI,MAAM,GAAG,IAAI,CAE3B;IAED,wFAAwF;IACxF,IAAI,QAAQ,IAAI,SAAS,EAAE,CAE1B;IAED;;6EAEyE;IACzE,IAAI,aAAa,IAAI,mBAAmB,CAEvC;IAED;;oDAEgD;IAChD,iBAAiB,IAAI,IAAI;IAIzB;;0EAEsE;IACtE,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IA0B5B,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,YAAY;IAcpB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,UAAU;IAYlB,OAAO,CAAC,UAAU;IAqBlB,OAAO,CAAC,gBAAgB;IA+BxB,kEAAkE;IAClE,OAAO,CAAC,MAAM;IA+Bd,OAAO,CAAC,QAAQ;IAUhB,+DAA+D;IAC/D,OAAO,CAAC,MAAM;IAMd;oCACgC;IAChC,OAAO,CAAC,cAAc;IAatB,gEAAgE;IAChE,OAAO,CAAC,gBAAgB;IAgCxB,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,SAAS;IAOjB,qEAAqE;IACrE,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,eAAe;IAKvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,+BAA+B;IAMvC,OAAO,CAAC,oBAAoB;IAO5B,OAAO,CAAC,uBAAuB;IAO/B,OAAO,CAAC,qBAAqB,CAS3B;IAEF,OAAO,CAAC,qBAAqB,CAI3B;IAEF,OAAO,CAAC,oBAAoB,CAQ1B;IAEF,OAAO,CAAC,oBAAoB,CAkB1B;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,OAAO,CAIb;IAEF,OAAO,CAAC,cAAc,CAIpB;IAEF,OAAO,CAAC,kBAAkB,CASxB;IAEF;;;;;;;OAOG;IACH,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,IAAI;IA4CzC,4EAA4E;IAC5E,OAAO,IAAI,IAAI;CAOhB"}