@voqalize/avatar 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -51
- package/client/dist/Avatar.d.ts +17 -14
- package/client/dist/Avatar.d.ts.map +1 -1
- package/client/dist/Avatar.js +3 -3
- package/client/dist/Avatar.js.map +1 -1
- package/client/dist/AvatarClient.d.ts +23 -57
- package/client/dist/AvatarClient.d.ts.map +1 -1
- package/client/dist/AvatarClient.js +20 -71
- package/client/dist/AvatarClient.js.map +1 -1
- package/client/dist/index.d.ts +22 -0
- package/client/dist/index.d.ts.map +1 -0
- package/client/dist/index.js +22 -0
- package/client/dist/index.js.map +1 -0
- package/client/dist/types.d.ts +22 -37
- package/client/dist/types.d.ts.map +1 -1
- package/client/dist/types.js +12 -12
- package/client/dist/types.js.map +1 -1
- package/client/dist/useAvatar.d.ts +8 -19
- package/client/dist/useAvatar.d.ts.map +1 -1
- package/client/dist/useAvatar.js +15 -32
- package/client/dist/useAvatar.js.map +1 -1
- package/client/src/Avatar.tsx +19 -24
- package/client/src/AvatarClient.ts +38 -111
- package/client/src/index.ts +22 -0
- package/client/src/types.ts +24 -47
- package/client/src/useAvatar.ts +19 -47
- package/docs/contract-avatar.md +37 -3
- package/docs/contract-protocol.md +90 -30
- package/package.json +3 -11
- package/src/avatar.d.ts +38 -6
- package/src/avatar.js +72 -37
- package/src/hand.js +680 -0
- package/src/idle.js +8 -25
- package/src/line-art.js +26 -0
- package/src/perform.js +8 -4
- package/client/dist/pipecat.d.ts +0 -21
- package/client/dist/pipecat.d.ts.map +0 -1
- package/client/dist/pipecat.js +0 -21
- package/client/dist/pipecat.js.map +0 -1
- package/client/dist/react.d.ts +0 -16
- package/client/dist/react.d.ts.map +0 -1
- package/client/dist/react.js +0 -17
- package/client/dist/react.js.map +0 -1
- package/client/src/pipecat.ts +0 -38
- package/client/src/react.ts +0 -34
- package/src/audio-fallback.js +0 -100
|
@@ -32,10 +32,10 @@
|
|
|
32
32
|
* it would eat into the intentional video-first safety margin rather than
|
|
33
33
|
* improve it. Left as a documented option, not built.
|
|
34
34
|
*
|
|
35
|
-
* `attach()`
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
35
|
+
* `attach()` therefore subscribes to exactly one pipecat event,
|
|
36
|
+
* `serverMessage`. It used to also subscribe to both speaking events to report
|
|
37
|
+
* a diagnostic drift between our anchor and pipecat's; that hook is gone with
|
|
38
|
+
* the rest of the observability surface (`docs/removed.md` § Client callbacks).
|
|
39
39
|
*
|
|
40
40
|
* ## Cue splice
|
|
41
41
|
*
|
|
@@ -72,18 +72,15 @@
|
|
|
72
72
|
import type { PipecatClient, RTVIEvent } from "@pipecat-ai/client-js";
|
|
73
73
|
import type { AvatarApi } from "../../src/avatar.js";
|
|
74
74
|
import {
|
|
75
|
-
AVATAR_MESSAGE_TYPE,
|
|
76
75
|
isAvatarMessage,
|
|
77
76
|
type AvatarCommand,
|
|
78
77
|
type AvatarCue,
|
|
79
78
|
type AvatarCuesCmd,
|
|
80
|
-
type AvatarHintCmd,
|
|
81
|
-
type AvatarPerformCmd,
|
|
82
79
|
type AvatarSpeechCmd,
|
|
83
80
|
type AvatarStateCmd,
|
|
84
|
-
type AvatarUnknownCmd,
|
|
85
81
|
} from "./types.js";
|
|
86
82
|
|
|
83
|
+
|
|
87
84
|
interface Turn {
|
|
88
85
|
ctx: string;
|
|
89
86
|
/** The canonical, already-spliced cue track for this turn. */
|
|
@@ -91,47 +88,30 @@ interface Turn {
|
|
|
91
88
|
/** Whether `speech start` has anchored a clock and issued the first `speak()`. */
|
|
92
89
|
started: boolean;
|
|
93
90
|
clock: (() => number) | null;
|
|
94
|
-
t0: number | null;
|
|
95
91
|
}
|
|
96
92
|
|
|
93
|
+
/**
|
|
94
|
+
* Internal. Not exported from the package — see `index.ts` for the public
|
|
95
|
+
* surface, which is `<Avatar>` and nothing else.
|
|
96
|
+
*
|
|
97
|
+
* There is deliberately no `accept` predicate here any more. Avatar commands
|
|
98
|
+
* travel in one envelope, `{type:"avatar"}`, in both directions and from every
|
|
99
|
+
* source: a `AvatarProcessor` in the pipeline and a brain driving the face
|
|
100
|
+
* out of band emit the same shape. A per-deployment predicate meant the
|
|
101
|
+
* library could not state what an avatar message *is*, which is the one thing
|
|
102
|
+
* a wire format has to be able to say. See docs/removed.md § The accept
|
|
103
|
+
* predicate.
|
|
104
|
+
*/
|
|
97
105
|
export interface AvatarClientOptions {
|
|
98
|
-
/** `{cmd:"hint"}` is a no-op hook today — the widget's listening engine
|
|
99
|
-
* already handles acks; a host may still want to know a hint arrived. */
|
|
100
|
-
onHint?: (kind: string, msg: AvatarHintCmd) => void;
|
|
101
|
-
/** An unrecognized `cmd` (forward compat) — the protocol says ignore
|
|
102
|
-
* silently, so this is purely an observability hook, not required. */
|
|
103
|
-
onUnknownCmd?: (msg: AvatarUnknownCmd) => void;
|
|
104
106
|
/** A dispatch threw (e.g. an unknown state or interjection id, which the
|
|
105
107
|
* widget throws on). Defaults to `console.warn`. */
|
|
106
108
|
onError?: (err: unknown, msg: AvatarCommand) => void;
|
|
107
|
-
/** Diagnostic only (see the class doc's "Turn clock anchoring" section) —
|
|
108
|
-
* never moves the anchor, just reports how far pipecat's own
|
|
109
|
-
* botStartedSpeaking/botStoppedSpeaking landed from it. */
|
|
110
|
-
onSpeakingDrift?: (info: { event: "start" | "stop"; ctx: string | null; driftMs: number }) => void;
|
|
111
|
-
/**
|
|
112
|
-
* Which server-messages `attach()` should look inside. Defaults to the
|
|
113
|
-
* protocol's own envelope, `type === "avatar"`.
|
|
114
|
-
*
|
|
115
|
-
* The escape hatch exists because an application may tunnel avatar commands
|
|
116
|
-
* inside a message type of its own — one deployment routes them through a
|
|
117
|
-
* generic `ui_command` envelope so an LLM tool call can drive the face — and
|
|
118
|
-
* teaching this library that envelope would be teaching it one consumer's
|
|
119
|
-
* private vocabulary. Widen it here instead:
|
|
120
|
-
*
|
|
121
|
-
* accept: (m) => m.type === "avatar" ||
|
|
122
|
-
* (m.type === "ui_command" && m.action === "avatar")
|
|
123
|
-
*
|
|
124
|
-
* The predicate only decides *whether to look*; the payload still has to
|
|
125
|
-
* carry a string `cmd` to dispatch at all.
|
|
126
|
-
*/
|
|
127
|
-
accept?: (message: Record<string, unknown>) => boolean;
|
|
128
109
|
/** Override for tests. Defaults to `performance.now`. */
|
|
129
110
|
now?: () => number;
|
|
130
111
|
}
|
|
131
112
|
|
|
132
113
|
/**
|
|
133
|
-
* The
|
|
134
|
-
* values.
|
|
114
|
+
* The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
|
|
135
115
|
*
|
|
136
116
|
* Written out rather than imported because that enum was this module's *only*
|
|
137
117
|
* runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
|
|
@@ -147,8 +127,6 @@ export interface AvatarClientOptions {
|
|
|
147
127
|
*/
|
|
148
128
|
export const RTVI_EVENTS = {
|
|
149
129
|
serverMessage: "serverMessage",
|
|
150
|
-
botStartedSpeaking: "botStartedSpeaking",
|
|
151
|
-
botStoppedSpeaking: "botStoppedSpeaking",
|
|
152
130
|
} as const satisfies Record<string, string>;
|
|
153
131
|
|
|
154
132
|
/** Defensive unwrap for the `RTVIEvent.ServerMessage` `{ data }` quirk: some
|
|
@@ -163,14 +141,12 @@ export class AvatarClient {
|
|
|
163
141
|
private readonly avatar: AvatarApi;
|
|
164
142
|
private readonly opts: AvatarClientOptions;
|
|
165
143
|
private readonly now: () => number;
|
|
166
|
-
private readonly accept: (message: Record<string, unknown>) => boolean;
|
|
167
144
|
private turn: Turn | null = null;
|
|
168
145
|
|
|
169
146
|
constructor(avatar: AvatarApi, opts: AvatarClientOptions = {}) {
|
|
170
147
|
this.avatar = avatar;
|
|
171
148
|
this.opts = opts;
|
|
172
149
|
this.now = opts.now ?? (() => performance.now());
|
|
173
|
-
this.accept = opts.accept ?? ((m) => m.type === AVATAR_MESSAGE_TYPE);
|
|
174
150
|
}
|
|
175
151
|
|
|
176
152
|
/** The active turn's ctx, or `null` between turns. For tests and telemetry. */
|
|
@@ -183,41 +159,36 @@ export class AvatarClient {
|
|
|
183
159
|
return this.turn ? [...this.turn.cues] : [];
|
|
184
160
|
}
|
|
185
161
|
|
|
186
|
-
/** Dispatch one
|
|
187
|
-
*
|
|
188
|
-
*
|
|
189
|
-
* Unknown `cmd`s are ignored, per the wire protocol's forward-compat rule. */
|
|
162
|
+
/** Dispatch one server message. Anything that isn't in the avatar envelope
|
|
163
|
+
* is not ours and is ignored; so is an envelope carrying a `cmd` this build
|
|
164
|
+
* has never heard of, per the wire protocol's forward-compat rule. */
|
|
190
165
|
dispatch(raw: unknown): void {
|
|
191
166
|
if (!isAvatarMessage(raw)) return;
|
|
192
|
-
const msg = raw;
|
|
167
|
+
const msg: AvatarCommand = raw;
|
|
193
168
|
try {
|
|
194
169
|
switch (msg.cmd) {
|
|
195
170
|
case "state":
|
|
196
|
-
this.handleState(msg
|
|
171
|
+
this.handleState(msg);
|
|
197
172
|
break;
|
|
198
173
|
case "interject":
|
|
199
|
-
this.avatar.interject(
|
|
174
|
+
this.avatar.interject(msg.id);
|
|
200
175
|
break;
|
|
201
|
-
case "
|
|
202
|
-
this.
|
|
176
|
+
case "gesture":
|
|
177
|
+
this.avatar.gesture(msg.id);
|
|
203
178
|
break;
|
|
204
179
|
case "cues":
|
|
205
|
-
this.handleCues(msg
|
|
180
|
+
this.handleCues(msg);
|
|
206
181
|
break;
|
|
207
182
|
case "speech":
|
|
208
|
-
this.handleSpeech(msg
|
|
183
|
+
this.handleSpeech(msg);
|
|
209
184
|
break;
|
|
210
185
|
case "user":
|
|
211
|
-
this.avatar.setUserSpeaking(
|
|
212
|
-
break;
|
|
213
|
-
case "hint": {
|
|
214
|
-
const hint = msg as AvatarHintCmd;
|
|
215
|
-
this.opts.onHint?.(hint.kind, hint);
|
|
216
|
-
break;
|
|
217
|
-
}
|
|
218
|
-
default:
|
|
219
|
-
this.opts.onUnknownCmd?.(msg as AvatarUnknownCmd);
|
|
186
|
+
this.avatar.setUserSpeaking(msg.speaking);
|
|
220
187
|
break;
|
|
188
|
+
// No default: an unknown `cmd` is a newer server talking to an older
|
|
189
|
+
// widget, and the protocol's forward-compat rule says ignore it. There
|
|
190
|
+
// is no callback for it — a hook nobody could act on is observability,
|
|
191
|
+
// not an interface (`docs/removed.md` § Client callbacks).
|
|
221
192
|
}
|
|
222
193
|
} catch (err) {
|
|
223
194
|
if (this.opts.onError) this.opts.onError(err, msg);
|
|
@@ -235,30 +206,13 @@ export class AvatarClient {
|
|
|
235
206
|
this.avatar.setState(msg.name, { emotion: msg.emotion, gaze: msg.gaze });
|
|
236
207
|
}
|
|
237
208
|
|
|
238
|
-
private handlePerform(msg: AvatarPerformCmd) {
|
|
239
|
-
this.avatar.perform(msg.actions, { clock: this.resolveClock(msg.ctx) });
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
/** Ride the named turn's clock if it's the one we're currently anchored to;
|
|
243
|
-
* otherwise (no active turn, or `perform` names a ctx we never saw a
|
|
244
|
-
* `speech start` for) fall back to a fresh clock anchored at this call — the
|
|
245
|
-
* same "elapsed ms since this call" default `avatar.perform()` itself uses
|
|
246
|
-
* when given no clock and no audio. */
|
|
247
|
-
private resolveClock(ctx: string | undefined): () => number {
|
|
248
|
-
if (ctx && this.turn && this.turn.ctx === ctx && this.turn.clock) {
|
|
249
|
-
return this.turn.clock;
|
|
250
|
-
}
|
|
251
|
-
const start = this.now();
|
|
252
|
-
return () => this.now() - start;
|
|
253
|
-
}
|
|
254
|
-
|
|
255
209
|
private ensureTurn(ctx: string): Turn {
|
|
256
210
|
if (!this.turn || this.turn.ctx !== ctx) {
|
|
257
211
|
// A different ctx supersedes whatever turn we had — a stale trailing
|
|
258
212
|
// message for the old ctx will find `this.turn.ctx !== ctx` in
|
|
259
213
|
// handleSpeech's stop-guard and be ignored, rather than cutting off the
|
|
260
214
|
// new turn.
|
|
261
|
-
this.turn = { ctx, cues: [], started: false, clock: null
|
|
215
|
+
this.turn = { ctx, cues: [], started: false, clock: null };
|
|
262
216
|
}
|
|
263
217
|
return this.turn;
|
|
264
218
|
}
|
|
@@ -286,7 +240,6 @@ export class AvatarClient {
|
|
|
286
240
|
const turn = this.ensureTurn(msg.ctx);
|
|
287
241
|
const t0 = this.now();
|
|
288
242
|
const clock = () => this.now() - t0;
|
|
289
|
-
turn.t0 = t0;
|
|
290
243
|
turn.clock = clock;
|
|
291
244
|
turn.started = true;
|
|
292
245
|
this.avatar.speak({ cues: turn.cues, clock });
|
|
@@ -300,44 +253,18 @@ export class AvatarClient {
|
|
|
300
253
|
}
|
|
301
254
|
}
|
|
302
255
|
|
|
303
|
-
private reportDrift(event: "start" | "stop") {
|
|
304
|
-
if (!this.opts.onSpeakingDrift) return;
|
|
305
|
-
const t0 = this.turn?.t0;
|
|
306
|
-
if (t0 == null) return;
|
|
307
|
-
this.opts.onSpeakingDrift({ event, ctx: this.turn?.ctx ?? null, driftMs: this.now() - t0 });
|
|
308
|
-
}
|
|
309
|
-
|
|
310
256
|
/**
|
|
311
257
|
* Subscribe to a live `PipecatClient`'s server messages and dispatch the
|
|
312
|
-
* avatar commands among them
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
* Also wires the diagnostic drift cross-check described in the class doc.
|
|
258
|
+
* avatar commands among them — the ones in the protocol's own
|
|
259
|
+
* `{type:"avatar"}` envelope, which `isAvatarMessage` is the definition of.
|
|
316
260
|
* Never throws on a malformed or irrelevant message.
|
|
317
261
|
*
|
|
318
262
|
* @returns an unsubscribe function; call it on unmount or disconnect.
|
|
319
263
|
*/
|
|
320
264
|
attach(client: PipecatClient): () => void {
|
|
321
|
-
const onServerMessage = (raw: unknown) =>
|
|
322
|
-
const message = unwrapServerMessage(raw);
|
|
323
|
-
if (!this.accept(message)) return;
|
|
324
|
-
this.dispatch(message);
|
|
325
|
-
};
|
|
326
|
-
const onBotStartedSpeaking = () => this.reportDrift("start");
|
|
327
|
-
const onBotStoppedSpeaking = () => this.reportDrift("stop");
|
|
328
|
-
|
|
265
|
+
const onServerMessage = (raw: unknown) => this.dispatch(unwrapServerMessage(raw));
|
|
329
266
|
const serverMessage = RTVI_EVENTS.serverMessage as RTVIEvent;
|
|
330
|
-
const started = RTVI_EVENTS.botStartedSpeaking as RTVIEvent;
|
|
331
|
-
const stopped = RTVI_EVENTS.botStoppedSpeaking as RTVIEvent;
|
|
332
|
-
|
|
333
267
|
client.on(serverMessage, onServerMessage);
|
|
334
|
-
client.
|
|
335
|
-
client.on(stopped, onBotStoppedSpeaking);
|
|
336
|
-
|
|
337
|
-
return () => {
|
|
338
|
-
client.off(serverMessage, onServerMessage);
|
|
339
|
-
client.off(started, onBotStartedSpeaking);
|
|
340
|
-
client.off(stopped, onBotStoppedSpeaking);
|
|
341
|
-
};
|
|
268
|
+
return () => client.off(serverMessage, onServerMessage);
|
|
342
269
|
}
|
|
343
270
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@voqalize/avatar` — the avatar as one React component.
|
|
3
|
+
*
|
|
4
|
+
* import { Avatar } from "@voqalize/avatar";
|
|
5
|
+
*
|
|
6
|
+
* <Avatar client={pipecatClient} className="avatar-tile" />
|
|
7
|
+
*
|
|
8
|
+
* That is the whole public surface. Put an `AvatarProcessor` in the pipecat
|
|
9
|
+
* pipeline (`pip install voqalize-avatar`), drop this component into the bot's
|
|
10
|
+
* tile, and the face listens, thinks, claims the floor and lipsyncs what the
|
|
11
|
+
* TTS says.
|
|
12
|
+
*
|
|
13
|
+
* The widget underneath is framework-free, and the dispatcher between it and
|
|
14
|
+
* the RTVI data channel is plain TypeScript — but neither is exported. Two
|
|
15
|
+
* consumers wanted a call tile, both are React, and a public API is a promise
|
|
16
|
+
* we have to keep across versions. `docs/removed.md` lists what used to be
|
|
17
|
+
* here and how to get it back if a real third case argues for it.
|
|
18
|
+
*
|
|
19
|
+
* Peers: `react >= 18` and `@pipecat-ai/client-js`.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
export { Avatar, type AvatarProps } from "./Avatar.js";
|
package/client/src/types.ts
CHANGED
|
@@ -8,11 +8,11 @@
|
|
|
8
8
|
* incomplete.
|
|
9
9
|
*
|
|
10
10
|
* A server pushes these as RTVI `server-message`s under the envelope
|
|
11
|
-
* `{ type: "avatar",
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* `{ type: "avatar", ...cmd-specific fields }`. {@link AvatarCommand} describes
|
|
12
|
+
* what rides inside that envelope; {@link isAvatarMessage} is the envelope
|
|
13
|
+
* itself, and is the only definition of "this message is for the avatar" the
|
|
14
|
+
* client has. There is no protocol version field — see `docs/removed.md`
|
|
15
|
+
* § The `v` field.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
18
|
/** A viseme cue: `t` is a ms offset into the utterance's clock, `v` is a Rhubarb A–H (or X) letter. */
|
|
@@ -22,16 +22,6 @@ export interface AvatarCue {
|
|
|
22
22
|
i?: number;
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
/** `perform()` timeline action — see docs/contract-protocol.md § Composing behavior. */
|
|
26
|
-
export interface AvatarPerformAction {
|
|
27
|
-
t: number;
|
|
28
|
-
do: "state" | "emotion" | "gaze" | "interject";
|
|
29
|
-
name?: string;
|
|
30
|
-
id?: string;
|
|
31
|
-
i?: number;
|
|
32
|
-
keepGaze?: boolean;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
25
|
export interface AvatarStateCmd {
|
|
36
26
|
cmd: "state";
|
|
37
27
|
name: string;
|
|
@@ -44,10 +34,14 @@ export interface AvatarInterjectCmd {
|
|
|
44
34
|
id: string;
|
|
45
35
|
}
|
|
46
36
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
37
|
+
/**
|
|
38
|
+
* A hand gesture — the hand at the frame edge plus its face half. Separate from
|
|
39
|
+
* `interject` on purpose: `interject("WAVE")` is the face alone and always was,
|
|
40
|
+
* so a server that upgrades gets no hand until it asks for one.
|
|
41
|
+
*/
|
|
42
|
+
export interface AvatarGestureCmd {
|
|
43
|
+
cmd: "gesture";
|
|
44
|
+
id: string;
|
|
51
45
|
}
|
|
52
46
|
|
|
53
47
|
export interface AvatarCuesCmd {
|
|
@@ -83,45 +77,28 @@ export interface AvatarUserCmd {
|
|
|
83
77
|
speaking: boolean;
|
|
84
78
|
}
|
|
85
79
|
|
|
86
|
-
export interface AvatarHintCmd {
|
|
87
|
-
cmd: "hint";
|
|
88
|
-
kind: "eager_eot" | (string & {});
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
/** A cmd this build doesn't recognize — dispatched to nothing, ignored for forward compat. */
|
|
92
|
-
export interface AvatarUnknownCmd {
|
|
93
|
-
cmd: string;
|
|
94
|
-
[key: string]: unknown;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
80
|
export type AvatarCommand =
|
|
98
81
|
| AvatarStateCmd
|
|
99
82
|
| AvatarInterjectCmd
|
|
100
|
-
|
|
|
83
|
+
| AvatarGestureCmd
|
|
101
84
|
| AvatarCuesCmd
|
|
102
85
|
| AvatarSpeechCmd
|
|
103
|
-
| AvatarUserCmd
|
|
104
|
-
| AvatarHintCmd
|
|
105
|
-
| AvatarUnknownCmd;
|
|
86
|
+
| AvatarUserCmd;
|
|
106
87
|
|
|
107
|
-
/** The full server-message payload: the
|
|
108
|
-
export type AvatarServerMessage = AvatarCommand & {
|
|
109
|
-
type?: "avatar";
|
|
110
|
-
v?: number;
|
|
111
|
-
};
|
|
88
|
+
/** The full server-message payload: the envelope plus its command. */
|
|
89
|
+
export type AvatarServerMessage = AvatarCommand & { type: "avatar" };
|
|
112
90
|
|
|
113
|
-
/**
|
|
91
|
+
/**
|
|
92
|
+
* Is this server-message payload the avatar's? The envelope is the whole
|
|
93
|
+
* answer: `{type:"avatar"}` with a string `cmd`. It used to be a per-deployment
|
|
94
|
+
* `accept` predicate on the client, which meant the library could not state
|
|
95
|
+
* what an avatar message *is* — see `docs/removed.md` § The accept predicate.
|
|
96
|
+
*/
|
|
114
97
|
export function isAvatarMessage(msg: unknown): msg is AvatarServerMessage {
|
|
115
98
|
if (typeof msg !== "object" || msg === null) return false;
|
|
116
99
|
const m = msg as Record<string, unknown>;
|
|
117
|
-
return typeof m.cmd === "string";
|
|
100
|
+
return m.type === AVATAR_MESSAGE_TYPE && typeof m.cmd === "string";
|
|
118
101
|
}
|
|
119
102
|
|
|
120
103
|
/** The envelope `type` the protocol reserves for avatar traffic. */
|
|
121
104
|
export const AVATAR_MESSAGE_TYPE = "avatar";
|
|
122
|
-
|
|
123
|
-
/** The protocol version this client speaks — matches `AVATAR_PROTOCOL_VERSION`
|
|
124
|
-
* in the Python package. Sent as `v` and, today, never checked: an unknown
|
|
125
|
-
* `cmd` is ignored rather than version-gated, which is the forward-compat rule
|
|
126
|
-
* the contract states. */
|
|
127
|
-
export const AVATAR_PROTOCOL_VERSION = 1;
|
package/client/src/useAvatar.ts
CHANGED
|
@@ -2,25 +2,21 @@
|
|
|
2
2
|
* useAvatar — mount the widget, wire it to a live session, dispatch its
|
|
3
3
|
* server-messages, and clean up.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* the pipecat client re-runs whenever the client
|
|
9
|
-
*
|
|
5
|
+
* Internal: `<Avatar>` is the only thing the package exports. It is a separate
|
|
6
|
+
* module anyway because the two lifecycles genuinely differ — mounting the
|
|
7
|
+
* widget happens once (an avatar swap remounts by design; see the note in the
|
|
8
|
+
* effect), while attaching to the pipecat client re-runs whenever the client
|
|
9
|
+
* identity changes, which a session reconnect makes it do.
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
-
import {
|
|
12
|
+
import { useEffect, useRef, useState } from "react";
|
|
13
13
|
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
14
|
-
import { createAvatar, type AvatarApi
|
|
15
|
-
import { AvatarClient
|
|
14
|
+
import { createAvatar, type AvatarApi } from "../../src/avatar.js";
|
|
15
|
+
import { AvatarClient } from "./AvatarClient.js";
|
|
16
16
|
|
|
17
|
-
export interface UseAvatarOptions
|
|
18
|
-
/**
|
|
17
|
+
export interface UseAvatarOptions {
|
|
18
|
+
/** Which face. Omit for the widget's own default. */
|
|
19
19
|
avatar?: string;
|
|
20
|
-
theme?: CreateAvatarOptions["theme"];
|
|
21
|
-
/** Articulation gains — see docs/contract-protocol.md § Events, gains, introspection. */
|
|
22
|
-
mouthGain?: number;
|
|
23
|
-
gestureGain?: number;
|
|
24
20
|
/** The live `PipecatClient` to dispatch server-messages from, or `null`
|
|
25
21
|
* before connect. `useAvatar` (dis)connects the subscription as this
|
|
26
22
|
* changes; it does not create or own the client. */
|
|
@@ -45,12 +41,6 @@ export interface UseAvatarHandle {
|
|
|
45
41
|
containerRef: AvatarMountRef;
|
|
46
42
|
/** The live widget instance once mounted, else `null`. */
|
|
47
43
|
avatar: AvatarApi | null;
|
|
48
|
-
/** The dispatcher wrapping `avatar` — `null` until mounted. Exposed for
|
|
49
|
-
* tests and telemetry (`turnCtx`, `turnCues`) and for manual dispatch. */
|
|
50
|
-
client: AvatarClient | null;
|
|
51
|
-
/** Dispatch one avatar command by hand — e.g. from a dev-tools console, or
|
|
52
|
-
* from a transport that isn't a `PipecatClient`. No-ops before mount. */
|
|
53
|
-
dispatch: (msg: unknown) => void;
|
|
54
44
|
}
|
|
55
45
|
|
|
56
46
|
export function useAvatar(options: UseAvatarOptions = {}): UseAvatarHandle {
|
|
@@ -58,30 +48,16 @@ export function useAvatar(options: UseAvatarOptions = {}): UseAvatarHandle {
|
|
|
58
48
|
const [avatar, setAvatar] = useState<AvatarApi | null>(null);
|
|
59
49
|
const avatarClientRef = useRef<AvatarClient | null>(null);
|
|
60
50
|
|
|
61
|
-
// Latest-options ref, so the mount effect (which runs once) still reads
|
|
62
|
-
//
|
|
51
|
+
// Latest-options ref, so the mount effect (which runs once) still reads the
|
|
52
|
+
// live props without re-subscribing.
|
|
63
53
|
const optionsRef = useRef(options);
|
|
64
54
|
optionsRef.current = options;
|
|
65
55
|
|
|
66
56
|
useEffect(() => {
|
|
67
57
|
const mount = containerRef.current;
|
|
68
58
|
if (!mount) return;
|
|
69
|
-
const instance = createAvatar({
|
|
70
|
-
|
|
71
|
-
avatar: optionsRef.current.avatar,
|
|
72
|
-
theme: optionsRef.current.theme,
|
|
73
|
-
mouthGain: optionsRef.current.mouthGain,
|
|
74
|
-
gestureGain: optionsRef.current.gestureGain,
|
|
75
|
-
});
|
|
76
|
-
const wrapper = new AvatarClient(instance, {
|
|
77
|
-
onHint: (kind, msg) => optionsRef.current.onHint?.(kind, msg),
|
|
78
|
-
onUnknownCmd: (msg) => optionsRef.current.onUnknownCmd?.(msg),
|
|
79
|
-
onError: (err, msg) => optionsRef.current.onError?.(err, msg),
|
|
80
|
-
onSpeakingDrift: (info) => optionsRef.current.onSpeakingDrift?.(info),
|
|
81
|
-
accept: optionsRef.current.accept,
|
|
82
|
-
now: optionsRef.current.now,
|
|
83
|
-
});
|
|
84
|
-
avatarClientRef.current = wrapper;
|
|
59
|
+
const instance = createAvatar({ mount, avatar: optionsRef.current.avatar });
|
|
60
|
+
avatarClientRef.current = new AvatarClient(instance);
|
|
85
61
|
setAvatar(instance);
|
|
86
62
|
|
|
87
63
|
return () => {
|
|
@@ -89,10 +65,10 @@ export function useAvatar(options: UseAvatarOptions = {}): UseAvatarHandle {
|
|
|
89
65
|
avatarClientRef.current = null;
|
|
90
66
|
setAvatar(null);
|
|
91
67
|
};
|
|
92
|
-
// Mount once. `avatar
|
|
93
|
-
//
|
|
94
|
-
//
|
|
95
|
-
//
|
|
68
|
+
// Mount once. `avatar` is read at mount time only — the widget has no
|
|
69
|
+
// hot-swap-avatar API (`createFace` runs once per mount), so changing it
|
|
70
|
+
// re-renders nothing here by design; a caller that needs a different face
|
|
71
|
+
// remounts with a `key` prop (see the component's doc).
|
|
96
72
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
97
73
|
}, []);
|
|
98
74
|
|
|
@@ -105,9 +81,5 @@ export function useAvatar(options: UseAvatarOptions = {}): UseAvatarHandle {
|
|
|
105
81
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
106
82
|
}, [avatar, options.client]);
|
|
107
83
|
|
|
108
|
-
|
|
109
|
-
avatarClientRef.current?.dispatch(msg);
|
|
110
|
-
}, []);
|
|
111
|
-
|
|
112
|
-
return { containerRef, avatar, client: avatarClientRef.current, dispatch };
|
|
84
|
+
return { containerRef, avatar };
|
|
113
85
|
}
|
package/docs/contract-avatar.md
CHANGED
|
@@ -201,6 +201,36 @@ no dark palette **by decision** (inverting two-value line art recolours the
|
|
|
201
201
|
hair and ages the character; that is geometry wearing a palette's clothes) —
|
|
202
202
|
its theme keys stay overridable, but do not add a `dark` selector.
|
|
203
203
|
|
|
204
|
+
## The hand — a layer no face draws
|
|
205
|
+
|
|
206
|
+
`src/hand.js` puts a hand into the bottom of the frame for `gesture(id)`
|
|
207
|
+
(protocol side: [contract-protocol.md](contract-protocol.md) § Hand gestures).
|
|
208
|
+
It is deliberately **not** part of this contract's parameter space: it writes a
|
|
209
|
+
transform on its own `<g>` appended over the face's svg, it has no channel in
|
|
210
|
+
`params.js`, and a face that never plays a gesture renders byte-for-byte what
|
|
211
|
+
it rendered before. That is the whole reason it could be added at all — a hand
|
|
212
|
+
channel only one avatar could draw is precisely the mistake CLAUDE.md
|
|
213
|
+
constraint 9 names.
|
|
214
|
+
|
|
215
|
+
**What a face owes it: a `META.viewBox`, and `theme.ink` / `theme.paper`.**
|
|
216
|
+
Nothing else, and no new META field. Placement derives four numbers from the
|
|
217
|
+
window itself — centre `x + w/2`, floor `y + h`, a reach scaled off `w`, and an
|
|
218
|
+
outboard limit of `w/2 − 8` — and every gesture timeline is authored in wrist
|
|
219
|
+
depth *below the floor* rather than absolute `y`, so the same drawing lands
|
|
220
|
+
correctly on windows of different heights. peep's bottom is 876 and wren's and
|
|
221
|
+
myna's is 850; all three place identically.
|
|
222
|
+
|
|
223
|
+
Two framing rules are asserted, not assumed. `checkHandFraming(meta)` throws if
|
|
224
|
+
any keyframe would let the wrist rise into the window (the hand must always be
|
|
225
|
+
*cut* by the bottom edge, never end in a floating stump) or let the hand's
|
|
226
|
+
rotated width cross the window's side (a portrait window pillarboxed in a 16:9
|
|
227
|
+
tile slices anything outboard with a hard vertical line that reads as a
|
|
228
|
+
rendering fault). `sweep()` runs it for every registered avatar, so a new face
|
|
229
|
+
with an unusual window fails the gate rather than the eye.
|
|
230
|
+
|
|
231
|
+
If a character's idiom cannot carry it, mount with `hand: false`; `gesture()`
|
|
232
|
+
then plays the face half alone.
|
|
233
|
+
|
|
204
234
|
## Checklist for a new avatar
|
|
205
235
|
|
|
206
236
|
1. Serve with `python3 serve.py 8777` (never `python3 -m http.server` — its
|
|
@@ -218,9 +248,12 @@ its theme keys stay overridable, but do not add a `dark` selector.
|
|
|
218
248
|
through the mixer's own smoothing, as a filmstrip.
|
|
219
249
|
5. `demo/rig/rig-check.html` → `await sweep()` — conformance: params finite,
|
|
220
250
|
`|v| ≤ 2`, svg connected, across every state/emotion/gaze/interjection and
|
|
221
|
-
a viseme track
|
|
251
|
+
a viseme track, plus `checkHandFraming` against your window and a pass of
|
|
252
|
+
every hand gesture. Sweep also cannot see *looks*; it reaches shoulders/torso
|
|
222
253
|
only through clips, so drive those with a `setOverrides` loop over
|
|
223
|
-
`[-1, 0, 1]` per channel
|
|
254
|
+
`[-1, 0, 1]` per channel — and look at one hand gesture at peak extension
|
|
255
|
+
(`demo/rig/body-lab.html?face=NAME&gesture=HI&at=0.4`), because figure/ground
|
|
256
|
+
between hand and shirt is a judgement the framing check cannot make.
|
|
224
257
|
6. Auto-traced art has known failure modes to budget for: zero-margin abutting
|
|
225
258
|
contours open seams under parallax; the trace stops at the source crop;
|
|
226
259
|
hard horizontal edges invisible in the source appear under motion.
|
|
@@ -324,7 +357,8 @@ A new face module supplies:
|
|
|
324
357
|
5. **A registry entry** — `{ create, meta }` in `src/avatar.js`.
|
|
325
358
|
|
|
326
359
|
What you get for free: the mixer, visemes, emotions, gaze, idle, clips,
|
|
327
|
-
interjections, the
|
|
360
|
+
interjections, the frame-edge hand (§ The hand — it needs only your viewBox and
|
|
361
|
+
two theme keys), the pose mechanics, the memoizer, and every host page and rig
|
|
328
362
|
tool — the demos' avatar pickers, contact sheet, torso check, clip strip and
|
|
329
363
|
`sweep()` all enumerate the registry. The wren run measured the split: the
|
|
330
364
|
plumbing steps (2, 4, 5) are mechanical; the art (step 1) and the read of
|