@voqalize/avatar 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +86 -51
  2. package/client/dist/Avatar.d.ts +17 -14
  3. package/client/dist/Avatar.d.ts.map +1 -1
  4. package/client/dist/Avatar.js +3 -3
  5. package/client/dist/Avatar.js.map +1 -1
  6. package/client/dist/AvatarClient.d.ts +23 -57
  7. package/client/dist/AvatarClient.d.ts.map +1 -1
  8. package/client/dist/AvatarClient.js +20 -71
  9. package/client/dist/AvatarClient.js.map +1 -1
  10. package/client/dist/index.d.ts +22 -0
  11. package/client/dist/index.d.ts.map +1 -0
  12. package/client/dist/index.js +22 -0
  13. package/client/dist/index.js.map +1 -0
  14. package/client/dist/types.d.ts +22 -37
  15. package/client/dist/types.d.ts.map +1 -1
  16. package/client/dist/types.js +12 -12
  17. package/client/dist/types.js.map +1 -1
  18. package/client/dist/useAvatar.d.ts +8 -19
  19. package/client/dist/useAvatar.d.ts.map +1 -1
  20. package/client/dist/useAvatar.js +15 -32
  21. package/client/dist/useAvatar.js.map +1 -1
  22. package/client/src/Avatar.tsx +19 -24
  23. package/client/src/AvatarClient.ts +38 -111
  24. package/client/src/index.ts +22 -0
  25. package/client/src/types.ts +24 -47
  26. package/client/src/useAvatar.ts +19 -47
  27. package/docs/contract-avatar.md +37 -3
  28. package/docs/contract-protocol.md +90 -30
  29. package/package.json +3 -11
  30. package/src/avatar.d.ts +38 -6
  31. package/src/avatar.js +72 -37
  32. package/src/hand.js +680 -0
  33. package/src/idle.js +8 -25
  34. package/src/line-art.js +26 -0
  35. package/src/perform.js +8 -4
  36. package/client/dist/pipecat.d.ts +0 -21
  37. package/client/dist/pipecat.d.ts.map +0 -1
  38. package/client/dist/pipecat.js +0 -21
  39. package/client/dist/pipecat.js.map +0 -1
  40. package/client/dist/react.d.ts +0 -16
  41. package/client/dist/react.d.ts.map +0 -1
  42. package/client/dist/react.js +0 -17
  43. package/client/dist/react.js.map +0 -1
  44. package/client/src/pipecat.ts +0 -38
  45. package/client/src/react.ts +0 -34
  46. package/src/audio-fallback.js +0 -100
@@ -32,10 +32,10 @@
32
32
  * it would eat into the intentional video-first safety margin rather than
33
33
  * improve it. Left as a documented option, not built.
34
34
  *
35
- * `attach()` still subscribes to both pipecat events, but only to report a
36
- * **diagnostic** drift (`onSpeakingDrift`) between our anchor and pipecat's —
37
- * useful for noticing in logs if the two ever separate by more than jitter,
38
- * never used to move `t0` itself.
35
+ * `attach()` therefore subscribes to exactly one pipecat event,
36
+ * `serverMessage`. It used to also subscribe to both speaking events to report
37
+ * a diagnostic drift between our anchor and pipecat's; that hook is gone with
38
+ * the rest of the observability surface (`docs/removed.md` § Client callbacks).
39
39
  *
40
40
  * ## Cue splice
41
41
  *
@@ -72,18 +72,15 @@
72
72
  import type { PipecatClient, RTVIEvent } from "@pipecat-ai/client-js";
73
73
  import type { AvatarApi } from "../../src/avatar.js";
74
74
  import {
75
- AVATAR_MESSAGE_TYPE,
76
75
  isAvatarMessage,
77
76
  type AvatarCommand,
78
77
  type AvatarCue,
79
78
  type AvatarCuesCmd,
80
- type AvatarHintCmd,
81
- type AvatarPerformCmd,
82
79
  type AvatarSpeechCmd,
83
80
  type AvatarStateCmd,
84
- type AvatarUnknownCmd,
85
81
  } from "./types.js";
86
82
 
83
+
87
84
  interface Turn {
88
85
  ctx: string;
89
86
  /** The canonical, already-spliced cue track for this turn. */
@@ -91,47 +88,30 @@ interface Turn {
91
88
  /** Whether `speech start` has anchored a clock and issued the first `speak()`. */
92
89
  started: boolean;
93
90
  clock: (() => number) | null;
94
- t0: number | null;
95
91
  }
96
92
 
93
+ /**
94
+ * Internal. Not exported from the package — see `index.ts` for the public
95
+ * surface, which is `<Avatar>` and nothing else.
96
+ *
97
+ * There is deliberately no `accept` predicate here any more. Avatar commands
98
+ * travel in one envelope, `{type:"avatar"}`, in both directions and from every
99
+ * source: a `AvatarProcessor` in the pipeline and a brain driving the face
100
+ * out of band emit the same shape. A per-deployment predicate meant the
101
+ * library could not state what an avatar message *is*, which is the one thing
102
+ * a wire format has to be able to say. See docs/removed.md § The accept
103
+ * predicate.
104
+ */
97
105
  export interface AvatarClientOptions {
98
- /** `{cmd:"hint"}` is a no-op hook today — the widget's listening engine
99
- * already handles acks; a host may still want to know a hint arrived. */
100
- onHint?: (kind: string, msg: AvatarHintCmd) => void;
101
- /** An unrecognized `cmd` (forward compat) — the protocol says ignore
102
- * silently, so this is purely an observability hook, not required. */
103
- onUnknownCmd?: (msg: AvatarUnknownCmd) => void;
104
106
  /** A dispatch threw (e.g. an unknown state or interjection id, which the
105
107
  * widget throws on). Defaults to `console.warn`. */
106
108
  onError?: (err: unknown, msg: AvatarCommand) => void;
107
- /** Diagnostic only (see the class doc's "Turn clock anchoring" section) —
108
- * never moves the anchor, just reports how far pipecat's own
109
- * botStartedSpeaking/botStoppedSpeaking landed from it. */
110
- onSpeakingDrift?: (info: { event: "start" | "stop"; ctx: string | null; driftMs: number }) => void;
111
- /**
112
- * Which server-messages `attach()` should look inside. Defaults to the
113
- * protocol's own envelope, `type === "avatar"`.
114
- *
115
- * The escape hatch exists because an application may tunnel avatar commands
116
- * inside a message type of its own — one deployment routes them through a
117
- * generic `ui_command` envelope so an LLM tool call can drive the face — and
118
- * teaching this library that envelope would be teaching it one consumer's
119
- * private vocabulary. Widen it here instead:
120
- *
121
- * accept: (m) => m.type === "avatar" ||
122
- * (m.type === "ui_command" && m.action === "avatar")
123
- *
124
- * The predicate only decides *whether to look*; the payload still has to
125
- * carry a string `cmd` to dispatch at all.
126
- */
127
- accept?: (message: Record<string, unknown>) => boolean;
128
109
  /** Override for tests. Defaults to `performance.now`. */
129
110
  now?: () => number;
130
111
  }
131
112
 
132
113
  /**
133
- * The three `RTVIEvent` members `attach()` subscribes to, spelled as their
134
- * values.
114
+ * The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
135
115
  *
136
116
  * Written out rather than imported because that enum was this module's *only*
137
117
  * runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
@@ -147,8 +127,6 @@ export interface AvatarClientOptions {
147
127
  */
148
128
  export const RTVI_EVENTS = {
149
129
  serverMessage: "serverMessage",
150
- botStartedSpeaking: "botStartedSpeaking",
151
- botStoppedSpeaking: "botStoppedSpeaking",
152
130
  } as const satisfies Record<string, string>;
153
131
 
154
132
  /** Defensive unwrap for the `RTVIEvent.ServerMessage` `{ data }` quirk: some
@@ -163,14 +141,12 @@ export class AvatarClient {
163
141
  private readonly avatar: AvatarApi;
164
142
  private readonly opts: AvatarClientOptions;
165
143
  private readonly now: () => number;
166
- private readonly accept: (message: Record<string, unknown>) => boolean;
167
144
  private turn: Turn | null = null;
168
145
 
169
146
  constructor(avatar: AvatarApi, opts: AvatarClientOptions = {}) {
170
147
  this.avatar = avatar;
171
148
  this.opts = opts;
172
149
  this.now = opts.now ?? (() => performance.now());
173
- this.accept = opts.accept ?? ((m) => m.type === AVATAR_MESSAGE_TYPE);
174
150
  }
175
151
 
176
152
  /** The active turn's ctx, or `null` between turns. For tests and telemetry. */
@@ -183,41 +159,36 @@ export class AvatarClient {
183
159
  return this.turn ? [...this.turn.cues] : [];
184
160
  }
185
161
 
186
- /** Dispatch one avatar command. Accepts anything with a string `cmd` — an
187
- * already-unwrapped `{type:"avatar", cmd, ...}` server message, or a bare
188
- * `{cmd, ...}` payload from whatever else the host is carrying them in.
189
- * Unknown `cmd`s are ignored, per the wire protocol's forward-compat rule. */
162
+ /** Dispatch one server message. Anything that isn't in the avatar envelope
163
+ * is not ours and is ignored; so is an envelope carrying a `cmd` this build
164
+ * has never heard of, per the wire protocol's forward-compat rule. */
190
165
  dispatch(raw: unknown): void {
191
166
  if (!isAvatarMessage(raw)) return;
192
- const msg = raw;
167
+ const msg: AvatarCommand = raw;
193
168
  try {
194
169
  switch (msg.cmd) {
195
170
  case "state":
196
- this.handleState(msg as AvatarStateCmd);
171
+ this.handleState(msg);
197
172
  break;
198
173
  case "interject":
199
- this.avatar.interject((msg as { id: string }).id);
174
+ this.avatar.interject(msg.id);
200
175
  break;
201
- case "perform":
202
- this.handlePerform(msg as AvatarPerformCmd);
176
+ case "gesture":
177
+ this.avatar.gesture(msg.id);
203
178
  break;
204
179
  case "cues":
205
- this.handleCues(msg as AvatarCuesCmd);
180
+ this.handleCues(msg);
206
181
  break;
207
182
  case "speech":
208
- this.handleSpeech(msg as AvatarSpeechCmd);
183
+ this.handleSpeech(msg);
209
184
  break;
210
185
  case "user":
211
- this.avatar.setUserSpeaking((msg as { speaking: boolean }).speaking);
212
- break;
213
- case "hint": {
214
- const hint = msg as AvatarHintCmd;
215
- this.opts.onHint?.(hint.kind, hint);
216
- break;
217
- }
218
- default:
219
- this.opts.onUnknownCmd?.(msg as AvatarUnknownCmd);
186
+ this.avatar.setUserSpeaking(msg.speaking);
220
187
  break;
188
+ // No default: an unknown `cmd` is a newer server talking to an older
189
+ // widget, and the protocol's forward-compat rule says ignore it. There
190
+ // is no callback for it — a hook nobody could act on is observability,
191
+ // not an interface (`docs/removed.md` § Client callbacks).
221
192
  }
222
193
  } catch (err) {
223
194
  if (this.opts.onError) this.opts.onError(err, msg);
@@ -235,30 +206,13 @@ export class AvatarClient {
235
206
  this.avatar.setState(msg.name, { emotion: msg.emotion, gaze: msg.gaze });
236
207
  }
237
208
 
238
- private handlePerform(msg: AvatarPerformCmd) {
239
- this.avatar.perform(msg.actions, { clock: this.resolveClock(msg.ctx) });
240
- }
241
-
242
- /** Ride the named turn's clock if it's the one we're currently anchored to;
243
- * otherwise (no active turn, or `perform` names a ctx we never saw a
244
- * `speech start` for) fall back to a fresh clock anchored at this call — the
245
- * same "elapsed ms since this call" default `avatar.perform()` itself uses
246
- * when given no clock and no audio. */
247
- private resolveClock(ctx: string | undefined): () => number {
248
- if (ctx && this.turn && this.turn.ctx === ctx && this.turn.clock) {
249
- return this.turn.clock;
250
- }
251
- const start = this.now();
252
- return () => this.now() - start;
253
- }
254
-
255
209
  private ensureTurn(ctx: string): Turn {
256
210
  if (!this.turn || this.turn.ctx !== ctx) {
257
211
  // A different ctx supersedes whatever turn we had — a stale trailing
258
212
  // message for the old ctx will find `this.turn.ctx !== ctx` in
259
213
  // handleSpeech's stop-guard and be ignored, rather than cutting off the
260
214
  // new turn.
261
- this.turn = { ctx, cues: [], started: false, clock: null, t0: null };
215
+ this.turn = { ctx, cues: [], started: false, clock: null };
262
216
  }
263
217
  return this.turn;
264
218
  }
@@ -286,7 +240,6 @@ export class AvatarClient {
286
240
  const turn = this.ensureTurn(msg.ctx);
287
241
  const t0 = this.now();
288
242
  const clock = () => this.now() - t0;
289
- turn.t0 = t0;
290
243
  turn.clock = clock;
291
244
  turn.started = true;
292
245
  this.avatar.speak({ cues: turn.cues, clock });
@@ -300,44 +253,18 @@ export class AvatarClient {
300
253
  }
301
254
  }
302
255
 
303
- private reportDrift(event: "start" | "stop") {
304
- if (!this.opts.onSpeakingDrift) return;
305
- const t0 = this.turn?.t0;
306
- if (t0 == null) return;
307
- this.opts.onSpeakingDrift({ event, ctx: this.turn?.ctx ?? null, driftMs: this.now() - t0 });
308
- }
309
-
310
256
  /**
311
257
  * Subscribe to a live `PipecatClient`'s server messages and dispatch the
312
- * avatar commands among them. Which messages count is the `accept` option;
313
- * by default, the protocol's own `{type:"avatar"}` envelope.
314
- *
315
- * Also wires the diagnostic drift cross-check described in the class doc.
258
+ * avatar commands among them — the ones in the protocol's own
259
+ * `{type:"avatar"}` envelope, which `isAvatarMessage` is the definition of.
316
260
  * Never throws on a malformed or irrelevant message.
317
261
  *
318
262
  * @returns an unsubscribe function; call it on unmount or disconnect.
319
263
  */
320
264
  attach(client: PipecatClient): () => void {
321
- const onServerMessage = (raw: unknown) => {
322
- const message = unwrapServerMessage(raw);
323
- if (!this.accept(message)) return;
324
- this.dispatch(message);
325
- };
326
- const onBotStartedSpeaking = () => this.reportDrift("start");
327
- const onBotStoppedSpeaking = () => this.reportDrift("stop");
328
-
265
+ const onServerMessage = (raw: unknown) => this.dispatch(unwrapServerMessage(raw));
329
266
  const serverMessage = RTVI_EVENTS.serverMessage as RTVIEvent;
330
- const started = RTVI_EVENTS.botStartedSpeaking as RTVIEvent;
331
- const stopped = RTVI_EVENTS.botStoppedSpeaking as RTVIEvent;
332
-
333
267
  client.on(serverMessage, onServerMessage);
334
- client.on(started, onBotStartedSpeaking);
335
- client.on(stopped, onBotStoppedSpeaking);
336
-
337
- return () => {
338
- client.off(serverMessage, onServerMessage);
339
- client.off(started, onBotStartedSpeaking);
340
- client.off(stopped, onBotStoppedSpeaking);
341
- };
268
+ return () => client.off(serverMessage, onServerMessage);
342
269
  }
343
270
  }
@@ -0,0 +1,22 @@
1
+ /**
2
+ * `@voqalize/avatar` — the avatar as one React component.
3
+ *
4
+ * import { Avatar } from "@voqalize/avatar";
5
+ *
6
+ * <Avatar client={pipecatClient} className="avatar-tile" />
7
+ *
8
+ * That is the whole public surface. Put an `AvatarProcessor` in the pipecat
9
+ * pipeline (`pip install voqalize-avatar`), drop this component into the bot's
10
+ * tile, and the face listens, thinks, claims the floor and lipsyncs what the
11
+ * TTS says.
12
+ *
13
+ * The widget underneath is framework-free, and the dispatcher between it and
14
+ * the RTVI data channel is plain TypeScript — but neither is exported. Two
15
+ * consumers wanted a call tile, both are React, and a public API is a promise
16
+ * we have to keep across versions. `docs/removed.md` lists what used to be
17
+ * here and how to get it back if a real third case argues for it.
18
+ *
19
+ * Peers: `react >= 18` and `@pipecat-ai/client-js`.
20
+ */
21
+
22
+ export { Avatar, type AvatarProps } from "./Avatar.js";
@@ -8,11 +8,11 @@
8
8
  * incomplete.
9
9
  *
10
10
  * A server pushes these as RTVI `server-message`s under the envelope
11
- * `{ type: "avatar", v: 1, ...cmd-specific fields }`. {@link AvatarCommand}
12
- * describes the *payload*, not the envelope, because the payload is what
13
- * arrives however the host chose to carry it — `AvatarClient.dispatch()`
14
- * accepts anything with a string `cmd`, so an application that tunnels these
15
- * through its own message type can hand them straight over.
11
+ * `{ type: "avatar", ...cmd-specific fields }`. {@link AvatarCommand} describes
12
+ * what rides inside that envelope; {@link isAvatarMessage} is the envelope
13
+ * itself, and is the only definition of "this message is for the avatar" the
14
+ * client has. There is no protocol version field — see `docs/removed.md`
15
+ * § The `v` field.
16
16
  */
17
17
 
18
18
  /** A viseme cue: `t` is a ms offset into the utterance's clock, `v` is a Rhubarb A–H (or X) letter. */
@@ -22,16 +22,6 @@ export interface AvatarCue {
22
22
  i?: number;
23
23
  }
24
24
 
25
- /** `perform()` timeline action — see docs/contract-protocol.md § Composing behavior. */
26
- export interface AvatarPerformAction {
27
- t: number;
28
- do: "state" | "emotion" | "gaze" | "interject";
29
- name?: string;
30
- id?: string;
31
- i?: number;
32
- keepGaze?: boolean;
33
- }
34
-
35
25
  export interface AvatarStateCmd {
36
26
  cmd: "state";
37
27
  name: string;
@@ -44,10 +34,14 @@ export interface AvatarInterjectCmd {
44
34
  id: string;
45
35
  }
46
36
 
47
- export interface AvatarPerformCmd {
48
- cmd: "perform";
49
- actions: AvatarPerformAction[];
50
- ctx?: string;
37
+ /**
38
+ * A hand gesture — the hand at the frame edge plus its face half. Separate from
39
+ * `interject` on purpose: `interject("WAVE")` is the face alone and always was,
40
+ * so a server that upgrades gets no hand until it asks for one.
41
+ */
42
+ export interface AvatarGestureCmd {
43
+ cmd: "gesture";
44
+ id: string;
51
45
  }
52
46
 
53
47
  export interface AvatarCuesCmd {
@@ -83,45 +77,28 @@ export interface AvatarUserCmd {
83
77
  speaking: boolean;
84
78
  }
85
79
 
86
- export interface AvatarHintCmd {
87
- cmd: "hint";
88
- kind: "eager_eot" | (string & {});
89
- }
90
-
91
- /** A cmd this build doesn't recognize — dispatched to nothing, ignored for forward compat. */
92
- export interface AvatarUnknownCmd {
93
- cmd: string;
94
- [key: string]: unknown;
95
- }
96
-
97
80
  export type AvatarCommand =
98
81
  | AvatarStateCmd
99
82
  | AvatarInterjectCmd
100
- | AvatarPerformCmd
83
+ | AvatarGestureCmd
101
84
  | AvatarCuesCmd
102
85
  | AvatarSpeechCmd
103
- | AvatarUserCmd
104
- | AvatarHintCmd
105
- | AvatarUnknownCmd;
86
+ | AvatarUserCmd;
106
87
 
107
- /** The full server-message payload: the avatar envelope plus its `cmd`. */
108
- export type AvatarServerMessage = AvatarCommand & {
109
- type?: "avatar";
110
- v?: number;
111
- };
88
+ /** The full server-message payload: the envelope plus its command. */
89
+ export type AvatarServerMessage = AvatarCommand & { type: "avatar" };
112
90
 
113
- /** Narrows an unknown server-message payload to an avatar command. */
91
+ /**
92
+ * Is this server-message payload the avatar's? The envelope is the whole
93
+ * answer: `{type:"avatar"}` with a string `cmd`. It used to be a per-deployment
94
+ * `accept` predicate on the client, which meant the library could not state
95
+ * what an avatar message *is* — see `docs/removed.md` § The accept predicate.
96
+ */
114
97
  export function isAvatarMessage(msg: unknown): msg is AvatarServerMessage {
115
98
  if (typeof msg !== "object" || msg === null) return false;
116
99
  const m = msg as Record<string, unknown>;
117
- return typeof m.cmd === "string";
100
+ return m.type === AVATAR_MESSAGE_TYPE && typeof m.cmd === "string";
118
101
  }
119
102
 
120
103
  /** The envelope `type` the protocol reserves for avatar traffic. */
121
104
  export const AVATAR_MESSAGE_TYPE = "avatar";
122
-
123
- /** The protocol version this client speaks — matches `AVATAR_PROTOCOL_VERSION`
124
- * in the Python package. Sent as `v` and, today, never checked: an unknown
125
- * `cmd` is ignored rather than version-gated, which is the forward-compat rule
126
- * the contract states. */
127
- export const AVATAR_PROTOCOL_VERSION = 1;
@@ -2,25 +2,21 @@
2
2
  * useAvatar — mount the widget, wire it to a live session, dispatch its
3
3
  * server-messages, and clean up.
4
4
  *
5
- * Options are read through a ref so the effect doesn't re-subscribe on every
6
- * render. Split in two effects: mounting the widget happens once (an avatar or
7
- * theme swap remounts by design — see the note in the effect); attaching to
8
- * the pipecat client re-runs whenever the client identity changes (a session
9
- * reconnect mints a new one) or once the widget instance becomes available.
5
+ * Internal: `<Avatar>` is the only thing the package exports. It is a separate
6
+ * module anyway because the two lifecycles genuinely differ — mounting the
7
+ * widget happens once (an avatar swap remounts by design; see the note in the
8
+ * effect), while attaching to the pipecat client re-runs whenever the client
9
+ * identity changes, which a session reconnect makes it do.
10
10
  */
11
11
 
12
- import { useCallback, useEffect, useRef, useState } from "react";
12
+ import { useEffect, useRef, useState } from "react";
13
13
  import type { PipecatClient } from "@pipecat-ai/client-js";
14
- import { createAvatar, type AvatarApi, type CreateAvatarOptions } from "../../src/avatar.js";
15
- import { AvatarClient, type AvatarClientOptions } from "./AvatarClient.js";
14
+ import { createAvatar, type AvatarApi } from "../../src/avatar.js";
15
+ import { AvatarClient } from "./AvatarClient.js";
16
16
 
17
- export interface UseAvatarOptions extends AvatarClientOptions {
18
- /** Name from `AVATAR_NAMES`. Omit for the widget's own `DEFAULT_AVATAR`. */
17
+ export interface UseAvatarOptions {
18
+ /** Which face. Omit for the widget's own default. */
19
19
  avatar?: string;
20
- theme?: CreateAvatarOptions["theme"];
21
- /** Articulation gains — see docs/contract-protocol.md § Events, gains, introspection. */
22
- mouthGain?: number;
23
- gestureGain?: number;
24
20
  /** The live `PipecatClient` to dispatch server-messages from, or `null`
25
21
  * before connect. `useAvatar` (dis)connects the subscription as this
26
22
  * changes; it does not create or own the client. */
@@ -45,12 +41,6 @@ export interface UseAvatarHandle {
45
41
  containerRef: AvatarMountRef;
46
42
  /** The live widget instance once mounted, else `null`. */
47
43
  avatar: AvatarApi | null;
48
- /** The dispatcher wrapping `avatar` — `null` until mounted. Exposed for
49
- * tests and telemetry (`turnCtx`, `turnCues`) and for manual dispatch. */
50
- client: AvatarClient | null;
51
- /** Dispatch one avatar command by hand — e.g. from a dev-tools console, or
52
- * from a transport that isn't a `PipecatClient`. No-ops before mount. */
53
- dispatch: (msg: unknown) => void;
54
44
  }
55
45
 
56
46
  export function useAvatar(options: UseAvatarOptions = {}): UseAvatarHandle {
@@ -58,30 +48,16 @@ export function useAvatar(options: UseAvatarOptions = {}): UseAvatarHandle {
58
48
  const [avatar, setAvatar] = useState<AvatarApi | null>(null);
59
49
  const avatarClientRef = useRef<AvatarClient | null>(null);
60
50
 
61
- // Latest-options ref, so the mount effect (which runs once) still reads live
62
- // callback props without re-subscribing.
51
+ // Latest-options ref, so the mount effect (which runs once) still reads the
52
+ // live props without re-subscribing.
63
53
  const optionsRef = useRef(options);
64
54
  optionsRef.current = options;
65
55
 
66
56
  useEffect(() => {
67
57
  const mount = containerRef.current;
68
58
  if (!mount) return;
69
- const instance = createAvatar({
70
- mount,
71
- avatar: optionsRef.current.avatar,
72
- theme: optionsRef.current.theme,
73
- mouthGain: optionsRef.current.mouthGain,
74
- gestureGain: optionsRef.current.gestureGain,
75
- });
76
- const wrapper = new AvatarClient(instance, {
77
- onHint: (kind, msg) => optionsRef.current.onHint?.(kind, msg),
78
- onUnknownCmd: (msg) => optionsRef.current.onUnknownCmd?.(msg),
79
- onError: (err, msg) => optionsRef.current.onError?.(err, msg),
80
- onSpeakingDrift: (info) => optionsRef.current.onSpeakingDrift?.(info),
81
- accept: optionsRef.current.accept,
82
- now: optionsRef.current.now,
83
- });
84
- avatarClientRef.current = wrapper;
59
+ const instance = createAvatar({ mount, avatar: optionsRef.current.avatar });
60
+ avatarClientRef.current = new AvatarClient(instance);
85
61
  setAvatar(instance);
86
62
 
87
63
  return () => {
@@ -89,10 +65,10 @@ export function useAvatar(options: UseAvatarOptions = {}): UseAvatarHandle {
89
65
  avatarClientRef.current = null;
90
66
  setAvatar(null);
91
67
  };
92
- // Mount once. `avatar`/`theme`/the gains are read at mount time only — the
93
- // widget has no hot-swap-avatar API (`createFace` runs once per mount), so
94
- // changing them re-renders nothing here by design; a caller that needs a
95
- // different avatar remounts with a `key` prop (see the component's doc).
68
+ // Mount once. `avatar` is read at mount time only — the widget has no
69
+ // hot-swap-avatar API (`createFace` runs once per mount), so changing it
70
+ // re-renders nothing here by design; a caller that needs a different face
71
+ // remounts with a `key` prop (see the component's doc).
96
72
  // eslint-disable-next-line react-hooks/exhaustive-deps
97
73
  }, []);
98
74
 
@@ -105,9 +81,5 @@ export function useAvatar(options: UseAvatarOptions = {}): UseAvatarHandle {
105
81
  // eslint-disable-next-line react-hooks/exhaustive-deps
106
82
  }, [avatar, options.client]);
107
83
 
108
- const dispatch = useCallback((msg: unknown) => {
109
- avatarClientRef.current?.dispatch(msg);
110
- }, []);
111
-
112
- return { containerRef, avatar, client: avatarClientRef.current, dispatch };
84
+ return { containerRef, avatar };
113
85
  }
@@ -201,6 +201,36 @@ no dark palette **by decision** (inverting two-value line art recolours the
201
201
  hair and ages the character; that is geometry wearing a palette's clothes) —
202
202
  its theme keys stay overridable, but do not add a `dark` selector.
203
203
 
204
+ ## The hand — a layer no face draws
205
+
206
+ `src/hand.js` puts a hand into the bottom of the frame for `gesture(id)`
207
+ (protocol side: [contract-protocol.md](contract-protocol.md) § Hand gestures).
208
+ It is deliberately **not** part of this contract's parameter space: it writes a
209
+ transform on its own `<g>` appended over the face's svg, it has no channel in
210
+ `params.js`, and a face that never plays a gesture renders byte-for-byte what
211
+ it rendered before. That is the whole reason it could be added at all — a hand
212
+ channel only one avatar could draw is precisely the mistake CLAUDE.md
213
+ constraint 9 names.
214
+
215
+ **What a face owes it: a `META.viewBox`, and `theme.ink` / `theme.paper`.**
216
+ Nothing else, and no new META field. Placement derives four numbers from the
217
+ window itself — centre `x + w/2`, floor `y + h`, a reach scaled off `w`, and an
218
+ outboard limit of `w/2 − 8` — and every gesture timeline is authored in wrist
219
+ depth *below the floor* rather than absolute `y`, so the same drawing lands
220
+ correctly on windows of different heights. peep's bottom is 876 and wren's and
221
+ myna's is 850; all three place identically.
222
+
223
+ Two framing rules are asserted, not assumed. `checkHandFraming(meta)` throws if
224
+ any keyframe would let the wrist rise into the window (the hand must always be
225
+ *cut* by the bottom edge, never end in a floating stump) or let the hand's
226
+ rotated width cross the window's side (a portrait window pillarboxed in a 16:9
227
+ tile slices anything outboard with a hard vertical line that reads as a
228
+ rendering fault). `sweep()` runs it for every registered avatar, so a new face
229
+ with an unusual window fails the gate rather than the eye.
230
+
231
+ If a character's idiom cannot carry it, mount with `hand: false`; `gesture()`
232
+ then plays the face half alone.
233
+
204
234
  ## Checklist for a new avatar
205
235
 
206
236
  1. Serve with `python3 serve.py 8777` (never `python3 -m http.server` — its
@@ -218,9 +248,12 @@ its theme keys stay overridable, but do not add a `dark` selector.
218
248
  through the mixer's own smoothing, as a filmstrip.
219
249
  5. `demo/rig/rig-check.html` → `await sweep()` — conformance: params finite,
220
250
  `|v| ≤ 2`, svg connected, across every state/emotion/gaze/interjection and
221
- a viseme track. Sweep also cannot see *looks*; it reaches shoulders/torso
251
+ a viseme track, plus `checkHandFraming` against your window and a pass of
252
+ every hand gesture. Sweep also cannot see *looks*; it reaches shoulders/torso
222
253
  only through clips, so drive those with a `setOverrides` loop over
223
- `[-1, 0, 1]` per channel.
254
+ `[-1, 0, 1]` per channel — and look at one hand gesture at peak extension
255
+ (`demo/rig/body-lab.html?face=NAME&gesture=HI&at=0.4`), because figure/ground
256
+ between hand and shirt is a judgement the framing check cannot make.
224
257
  6. Auto-traced art has known failure modes to budget for: zero-margin abutting
225
258
  contours open seams under parallax; the trace stops at the source crop;
226
259
  hard horizontal edges invisible in the source appear under motion.
@@ -324,7 +357,8 @@ A new face module supplies:
324
357
  5. **A registry entry** — `{ create, meta }` in `src/avatar.js`.
325
358
 
326
359
  What you get for free: the mixer, visemes, emotions, gaze, idle, clips,
327
- interjections, the pose mechanics, the memoizer, and every host page and rig
360
+ interjections, the frame-edge hand (§ The hand — it needs only your viewBox and
361
+ two theme keys), the pose mechanics, the memoizer, and every host page and rig
328
362
  tool — the demos' avatar pickers, contact sheet, torso check, clip strip and
329
363
  `sweep()` all enumerate the registry. The wren run measured the split: the
330
364
  plumbing steps (2, 4, 5) are mechanical; the art (step 1) and the read of