@voqalize/avatar 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -51
- package/client/dist/Avatar.d.ts +17 -14
- package/client/dist/Avatar.d.ts.map +1 -1
- package/client/dist/Avatar.js +3 -3
- package/client/dist/Avatar.js.map +1 -1
- package/client/dist/AvatarClient.d.ts +23 -57
- package/client/dist/AvatarClient.d.ts.map +1 -1
- package/client/dist/AvatarClient.js +20 -71
- package/client/dist/AvatarClient.js.map +1 -1
- package/client/dist/index.d.ts +22 -0
- package/client/dist/index.d.ts.map +1 -0
- package/client/dist/index.js +22 -0
- package/client/dist/index.js.map +1 -0
- package/client/dist/types.d.ts +22 -37
- package/client/dist/types.d.ts.map +1 -1
- package/client/dist/types.js +12 -12
- package/client/dist/types.js.map +1 -1
- package/client/dist/useAvatar.d.ts +8 -19
- package/client/dist/useAvatar.d.ts.map +1 -1
- package/client/dist/useAvatar.js +15 -32
- package/client/dist/useAvatar.js.map +1 -1
- package/client/src/Avatar.tsx +19 -24
- package/client/src/AvatarClient.ts +38 -111
- package/client/src/index.ts +22 -0
- package/client/src/types.ts +24 -47
- package/client/src/useAvatar.ts +19 -47
- package/docs/contract-avatar.md +37 -3
- package/docs/contract-protocol.md +90 -30
- package/package.json +3 -11
- package/src/avatar.d.ts +38 -6
- package/src/avatar.js +72 -37
- package/src/hand.js +680 -0
- package/src/idle.js +8 -25
- package/src/line-art.js +26 -0
- package/src/perform.js +8 -4
- package/client/dist/pipecat.d.ts +0 -21
- package/client/dist/pipecat.d.ts.map +0 -1
- package/client/dist/pipecat.js +0 -21
- package/client/dist/pipecat.js.map +0 -1
- package/client/dist/react.d.ts +0 -16
- package/client/dist/react.d.ts.map +0 -1
- package/client/dist/react.js +0 -17
- package/client/dist/react.js.map +0 -1
- package/client/src/pipecat.ts +0 -38
- package/client/src/react.ts +0 -34
- package/src/audio-fallback.js +0 -100
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AvatarClient.js","sourceRoot":"","sources":["../src/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqEG;AAIH,OAAO,EACL,
|
|
1
|
+
{"version":3,"file":"AvatarClient.js","sourceRoot":"","sources":["../src/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqEG;AAIH,OAAO,EACL,eAAe,GAMhB,MAAM,YAAY,CAAC;AAgCpB;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,aAAa,EAAE,eAAe;CACW,CAAC;AAE5C;yEACyE;AACzE,SAAS,mBAAmB,CAAC,GAAY;IACvC,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,EAAE,CAA4B,CAAC;IACnD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAwC,CAAC;IACjE,OAAO,KAAK,IAAI,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;AAChD,CAAC;AAED,MAAM,OAAO,YAAY;IACN,MAAM,CAAY;IAClB,IAAI,CAAsB;IAC1B,GAAG,CAAe;IAC3B,IAAI,GAAgB,IAAI,CAAC;IAEjC,YAAY,MAAiB,EAAE,OAA4B,EAAE;QAC3D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,+EAA+E;IAC/E,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,CAAC;IAChC,CAAC;IAED,wFAAwF;IACxF,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9C,CAAC;IAED;;0EAEsE;IACtE,QAAQ,CAAC,GAAY;QACnB,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC;YAAE,OAAO;QAClC,MAAM,GAAG,GAAkB,GAAG,CAAC;QAC/B,IAAI,CAAC;YACH,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC;gBAChB,KAAK,OAAO;oBACV,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;oBACtB,MAAM;gBACR,KAAK,WAAW;oBACd,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC9B,MAAM;gBACR,KAAK,SAAS;oBACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC5B,MAAM;gBACR,KAAK,MAAM;oBACT,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;oBACrB,MAAM;gBACR,KAAK,QAAQ;oBACX,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;oBACvB,MAAM;gBACR,KAAK,MAAM;oBACT,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;oBAC1C,MAAM;gBACR,qEAAqE;gBACrE,uEAAuE;gBACvE,uEAAuE;gBACvE,2DAA2D;YAC7D,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;;gBAC9C,OAAO,CAAC,IAAI,CAAC,0BAA0B,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,GAAmB;QACrC,yEAAyE;QACzE,0EAA0E;QAC1E,4EAA4E;QAC5E,8DAA8D;QAC9D,2EAA2E;QAC3E,mCAAmC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC;IAEO,UAAU,CAAC,GAAW;QAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC;YACxC,qEAAqE;YACrE,+DAA+D;YAC/D,wEAAwE;YACxE,YAAY;YACZ,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QAC7D,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAEO,UAAU,CAAC,GAAkB;QACnC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACjD,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAE7D,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,0EAA0E;YAC1E,sBAAsB;YACtB,OAAO;QACT,CAAC;QACD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,GAAoB;QACvC,IAAI,GAAG,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACpC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAC9C,OAAO;QACT,CAAC;QACD,4EAA4E;QAC5E,+DAA+D;QAC/D,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,EAAE,CAAC;YAC3C,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,MAAqB;QAC1B,MAAM,eAAe,GAAG,CAAC,GAAY,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC;QAClF,MAAM,aAAa,GAAG,WAAW,CAAC,aAA0B,CAAC;QAC7D,MAAM,CAAC,EAAE,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;QAC1C,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IAC1D,CAAC;CACF"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@voqalize/avatar` — the avatar as one React component.
|
|
3
|
+
*
|
|
4
|
+
* import { Avatar } from "@voqalize/avatar";
|
|
5
|
+
*
|
|
6
|
+
* <Avatar client={pipecatClient} className="avatar-tile" />
|
|
7
|
+
*
|
|
8
|
+
* That is the whole public surface. Put an `AvatarProcessor` in the pipecat
|
|
9
|
+
* pipeline (`pip install voqalize-avatar`), drop this component into the bot's
|
|
10
|
+
* tile, and the face listens, thinks, claims the floor and lipsyncs what the
|
|
11
|
+
* TTS says.
|
|
12
|
+
*
|
|
13
|
+
* The widget underneath is framework-free, and the dispatcher between it and
|
|
14
|
+
* the RTVI data channel is plain TypeScript — but neither is exported. Two
|
|
15
|
+
* consumers wanted a call tile, both are React, and a public API is a promise
|
|
16
|
+
* we have to keep across versions. `docs/removed.md` lists what used to be
|
|
17
|
+
* here and how to get it back if a real third case argues for it.
|
|
18
|
+
*
|
|
19
|
+
* Peers: `react >= 18` and `@pipecat-ai/client-js`.
|
|
20
|
+
*/
|
|
21
|
+
export { Avatar, type AvatarProps } from "./Avatar.js";
|
|
22
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,MAAM,EAAE,KAAK,WAAW,EAAE,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@voqalize/avatar` — the avatar as one React component.
|
|
3
|
+
*
|
|
4
|
+
* import { Avatar } from "@voqalize/avatar";
|
|
5
|
+
*
|
|
6
|
+
* <Avatar client={pipecatClient} className="avatar-tile" />
|
|
7
|
+
*
|
|
8
|
+
* That is the whole public surface. Put an `AvatarProcessor` in the pipecat
|
|
9
|
+
* pipeline (`pip install voqalize-avatar`), drop this component into the bot's
|
|
10
|
+
* tile, and the face listens, thinks, claims the floor and lipsyncs what the
|
|
11
|
+
* TTS says.
|
|
12
|
+
*
|
|
13
|
+
* The widget underneath is framework-free, and the dispatcher between it and
|
|
14
|
+
* the RTVI data channel is plain TypeScript — but neither is exported. Two
|
|
15
|
+
* consumers wanted a call tile, both are React, and a public API is a promise
|
|
16
|
+
* we have to keep across versions. `docs/removed.md` lists what used to be
|
|
17
|
+
* here and how to get it back if a real third case argues for it.
|
|
18
|
+
*
|
|
19
|
+
* Peers: `react >= 18` and `@pipecat-ai/client-js`.
|
|
20
|
+
*/
|
|
21
|
+
export { Avatar } from "./Avatar.js";
|
|
22
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,MAAM,EAAoB,MAAM,aAAa,CAAC"}
|
package/client/dist/types.d.ts
CHANGED
|
@@ -8,11 +8,11 @@
|
|
|
8
8
|
* incomplete.
|
|
9
9
|
*
|
|
10
10
|
* A server pushes these as RTVI `server-message`s under the envelope
|
|
11
|
-
* `{ type: "avatar",
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* `{ type: "avatar", ...cmd-specific fields }`. {@link AvatarCommand} describes
|
|
12
|
+
* what rides inside that envelope; {@link isAvatarMessage} is the envelope
|
|
13
|
+
* itself, and is the only definition of "this message is for the avatar" the
|
|
14
|
+
* client has. There is no protocol version field — see `docs/removed.md`
|
|
15
|
+
* § The `v` field.
|
|
16
16
|
*/
|
|
17
17
|
/** A viseme cue: `t` is a ms offset into the utterance's clock, `v` is a Rhubarb A–H (or X) letter. */
|
|
18
18
|
export interface AvatarCue {
|
|
@@ -20,15 +20,6 @@ export interface AvatarCue {
|
|
|
20
20
|
v: string;
|
|
21
21
|
i?: number;
|
|
22
22
|
}
|
|
23
|
-
/** `perform()` timeline action — see docs/contract-protocol.md § Composing behavior. */
|
|
24
|
-
export interface AvatarPerformAction {
|
|
25
|
-
t: number;
|
|
26
|
-
do: "state" | "emotion" | "gaze" | "interject";
|
|
27
|
-
name?: string;
|
|
28
|
-
id?: string;
|
|
29
|
-
i?: number;
|
|
30
|
-
keepGaze?: boolean;
|
|
31
|
-
}
|
|
32
23
|
export interface AvatarStateCmd {
|
|
33
24
|
cmd: "state";
|
|
34
25
|
name: string;
|
|
@@ -39,10 +30,14 @@ export interface AvatarInterjectCmd {
|
|
|
39
30
|
cmd: "interject";
|
|
40
31
|
id: string;
|
|
41
32
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
33
|
+
/**
|
|
34
|
+
* A hand gesture — the hand at the frame edge plus its face half. Separate from
|
|
35
|
+
* `interject` on purpose: `interject("WAVE")` is the face alone and always was,
|
|
36
|
+
* so a server that upgrades gets no hand until it asks for one.
|
|
37
|
+
*/
|
|
38
|
+
export interface AvatarGestureCmd {
|
|
39
|
+
cmd: "gesture";
|
|
40
|
+
id: string;
|
|
46
41
|
}
|
|
47
42
|
export interface AvatarCuesCmd {
|
|
48
43
|
cmd: "cues";
|
|
@@ -74,28 +69,18 @@ export interface AvatarUserCmd {
|
|
|
74
69
|
cmd: "user";
|
|
75
70
|
speaking: boolean;
|
|
76
71
|
}
|
|
77
|
-
export
|
|
78
|
-
|
|
79
|
-
kind: "eager_eot" | (string & {});
|
|
80
|
-
}
|
|
81
|
-
/** A cmd this build doesn't recognize — dispatched to nothing, ignored for forward compat. */
|
|
82
|
-
export interface AvatarUnknownCmd {
|
|
83
|
-
cmd: string;
|
|
84
|
-
[key: string]: unknown;
|
|
85
|
-
}
|
|
86
|
-
export type AvatarCommand = AvatarStateCmd | AvatarInterjectCmd | AvatarPerformCmd | AvatarCuesCmd | AvatarSpeechCmd | AvatarUserCmd | AvatarHintCmd | AvatarUnknownCmd;
|
|
87
|
-
/** The full server-message payload: the avatar envelope plus its `cmd`. */
|
|
72
|
+
export type AvatarCommand = AvatarStateCmd | AvatarInterjectCmd | AvatarGestureCmd | AvatarCuesCmd | AvatarSpeechCmd | AvatarUserCmd;
|
|
73
|
+
/** The full server-message payload: the envelope plus its command. */
|
|
88
74
|
export type AvatarServerMessage = AvatarCommand & {
|
|
89
|
-
type
|
|
90
|
-
v?: number;
|
|
75
|
+
type: "avatar";
|
|
91
76
|
};
|
|
92
|
-
/**
|
|
77
|
+
/**
|
|
78
|
+
* Is this server-message payload the avatar's? The envelope is the whole
|
|
79
|
+
* answer: `{type:"avatar"}` with a string `cmd`. It used to be a per-deployment
|
|
80
|
+
* `accept` predicate on the client, which meant the library could not state
|
|
81
|
+
* what an avatar message *is* — see `docs/removed.md` § The accept predicate.
|
|
82
|
+
*/
|
|
93
83
|
export declare function isAvatarMessage(msg: unknown): msg is AvatarServerMessage;
|
|
94
84
|
/** The envelope `type` the protocol reserves for avatar traffic. */
|
|
95
85
|
export declare const AVATAR_MESSAGE_TYPE = "avatar";
|
|
96
|
-
/** The protocol version this client speaks — matches `AVATAR_PROTOCOL_VERSION`
|
|
97
|
-
* in the Python package. Sent as `v` and, today, never checked: an unknown
|
|
98
|
-
* `cmd` is ignored rather than version-gated, which is the forward-compat rule
|
|
99
|
-
* the contract states. */
|
|
100
|
-
export declare const AVATAR_PROTOCOL_VERSION = 1;
|
|
101
86
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,uGAAuG;AACvG,MAAM,WAAW,SAAS;IACxB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,CAAC,EAAE,MAAM,CAAC;CACZ;AAED,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,uGAAuG;AACvG,MAAM,WAAW,SAAS;IACxB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,CAAC,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,OAAO,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,WAAW,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,SAAS,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,4EAA4E;IAC5E,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,QAAQ,CAAC;IACd,KAAK,EAAE,OAAO,GAAG,MAAM,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,MAAM,aAAa,GACrB,cAAc,GACd,kBAAkB,GAClB,gBAAgB,GAChB,aAAa,GACb,eAAe,GACf,aAAa,CAAC;AAElB,sEAAsE;AACtE,MAAM,MAAM,mBAAmB,GAAG,aAAa,GAAG;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CAAC;AAErE;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,GAAG,IAAI,mBAAmB,CAIxE;AAED,oEAAoE;AACpE,eAAO,MAAM,mBAAmB,WAAW,CAAC"}
|
package/client/dist/types.js
CHANGED
|
@@ -8,24 +8,24 @@
|
|
|
8
8
|
* incomplete.
|
|
9
9
|
*
|
|
10
10
|
* A server pushes these as RTVI `server-message`s under the envelope
|
|
11
|
-
* `{ type: "avatar",
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* `{ type: "avatar", ...cmd-specific fields }`. {@link AvatarCommand} describes
|
|
12
|
+
* what rides inside that envelope; {@link isAvatarMessage} is the envelope
|
|
13
|
+
* itself, and is the only definition of "this message is for the avatar" the
|
|
14
|
+
* client has. There is no protocol version field — see `docs/removed.md`
|
|
15
|
+
* § The `v` field.
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* Is this server-message payload the avatar's? The envelope is the whole
|
|
19
|
+
* answer: `{type:"avatar"}` with a string `cmd`. It used to be a per-deployment
|
|
20
|
+
* `accept` predicate on the client, which meant the library could not state
|
|
21
|
+
* what an avatar message *is* — see `docs/removed.md` § The accept predicate.
|
|
16
22
|
*/
|
|
17
|
-
/** Narrows an unknown server-message payload to an avatar command. */
|
|
18
23
|
export function isAvatarMessage(msg) {
|
|
19
24
|
if (typeof msg !== "object" || msg === null)
|
|
20
25
|
return false;
|
|
21
26
|
const m = msg;
|
|
22
|
-
return typeof m.cmd === "string";
|
|
27
|
+
return m.type === AVATAR_MESSAGE_TYPE && typeof m.cmd === "string";
|
|
23
28
|
}
|
|
24
29
|
/** The envelope `type` the protocol reserves for avatar traffic. */
|
|
25
30
|
export const AVATAR_MESSAGE_TYPE = "avatar";
|
|
26
|
-
/** The protocol version this client speaks — matches `AVATAR_PROTOCOL_VERSION`
|
|
27
|
-
* in the Python package. Sent as `v` and, today, never checked: an unknown
|
|
28
|
-
* `cmd` is ignored rather than version-gated, which is the forward-compat rule
|
|
29
|
-
* the contract states. */
|
|
30
|
-
export const AVATAR_PROTOCOL_VERSION = 1;
|
|
31
31
|
//# sourceMappingURL=types.js.map
|
package/client/dist/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AA2EH;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,GAAY;IAC1C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,MAAM,CAAC,GAAG,GAA8B,CAAC;IACzC,OAAO,CAAC,CAAC,IAAI,KAAK,mBAAmB,IAAI,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC;AACrE,CAAC;AAED,oEAAoE;AACpE,MAAM,CAAC,MAAM,mBAAmB,GAAG,QAAQ,CAAC"}
|
|
@@ -2,22 +2,17 @@
|
|
|
2
2
|
* useAvatar — mount the widget, wire it to a live session, dispatch its
|
|
3
3
|
* server-messages, and clean up.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* the pipecat client re-runs whenever the client
|
|
9
|
-
*
|
|
5
|
+
* Internal: `<Avatar>` is the only thing the package exports. It is a separate
|
|
6
|
+
* module anyway because the two lifecycles genuinely differ — mounting the
|
|
7
|
+
* widget happens once (an avatar swap remounts by design; see the note in the
|
|
8
|
+
* effect), while attaching to the pipecat client re-runs whenever the client
|
|
9
|
+
* identity changes, which a session reconnect makes it do.
|
|
10
10
|
*/
|
|
11
11
|
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
12
|
-
import { type AvatarApi
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
/** Name from `AVATAR_NAMES`. Omit for the widget's own `DEFAULT_AVATAR`. */
|
|
12
|
+
import { type AvatarApi } from "../../src/avatar.js";
|
|
13
|
+
export interface UseAvatarOptions {
|
|
14
|
+
/** Which face. Omit for the widget's own default. */
|
|
16
15
|
avatar?: string;
|
|
17
|
-
theme?: CreateAvatarOptions["theme"];
|
|
18
|
-
/** Articulation gains — see docs/contract-protocol.md § Events, gains, introspection. */
|
|
19
|
-
mouthGain?: number;
|
|
20
|
-
gestureGain?: number;
|
|
21
16
|
/** The live `PipecatClient` to dispatch server-messages from, or `null`
|
|
22
17
|
* before connect. `useAvatar` (dis)connects the subscription as this
|
|
23
18
|
* changes; it does not create or own the client. */
|
|
@@ -42,12 +37,6 @@ export interface UseAvatarHandle {
|
|
|
42
37
|
containerRef: AvatarMountRef;
|
|
43
38
|
/** The live widget instance once mounted, else `null`. */
|
|
44
39
|
avatar: AvatarApi | null;
|
|
45
|
-
/** The dispatcher wrapping `avatar` — `null` until mounted. Exposed for
|
|
46
|
-
* tests and telemetry (`turnCtx`, `turnCues`) and for manual dispatch. */
|
|
47
|
-
client: AvatarClient | null;
|
|
48
|
-
/** Dispatch one avatar command by hand — e.g. from a dev-tools console, or
|
|
49
|
-
* from a transport that isn't a `PipecatClient`. No-ops before mount. */
|
|
50
|
-
dispatch: (msg: unknown) => void;
|
|
51
40
|
}
|
|
52
41
|
export declare function useAvatar(options?: UseAvatarOptions): UseAvatarHandle;
|
|
53
42
|
//# sourceMappingURL=useAvatar.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"useAvatar.d.ts","sourceRoot":"","sources":["../src/useAvatar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAgB,KAAK,SAAS,EAAE,
|
|
1
|
+
{"version":3,"file":"useAvatar.d.ts","sourceRoot":"","sources":["../src/useAvatar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAgB,KAAK,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGnE,MAAM,WAAW,gBAAgB;IAC/B,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;wDAEoD;IACpD,MAAM,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;CAC/B;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,cAAc,GAAG;IAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAAA;CAAE,CAAC;AAEhE,MAAM,WAAW,eAAe;IAC9B,iEAAiE;IACjE,YAAY,EAAE,cAAc,CAAC;IAC7B,0DAA0D;IAC1D,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC;CAC1B;AAED,wBAAgB,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,eAAe,CAuCzE"}
|
package/client/dist/useAvatar.js
CHANGED
|
@@ -2,53 +2,39 @@
|
|
|
2
2
|
* useAvatar — mount the widget, wire it to a live session, dispatch its
|
|
3
3
|
* server-messages, and clean up.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* the pipecat client re-runs whenever the client
|
|
9
|
-
*
|
|
5
|
+
* Internal: `<Avatar>` is the only thing the package exports. It is a separate
|
|
6
|
+
* module anyway because the two lifecycles genuinely differ — mounting the
|
|
7
|
+
* widget happens once (an avatar swap remounts by design; see the note in the
|
|
8
|
+
* effect), while attaching to the pipecat client re-runs whenever the client
|
|
9
|
+
* identity changes, which a session reconnect makes it do.
|
|
10
10
|
*/
|
|
11
|
-
import {
|
|
11
|
+
import { useEffect, useRef, useState } from "react";
|
|
12
12
|
import { createAvatar } from "../../src/avatar.js";
|
|
13
13
|
import { AvatarClient } from "./AvatarClient.js";
|
|
14
14
|
export function useAvatar(options = {}) {
|
|
15
15
|
const containerRef = useRef(null);
|
|
16
16
|
const [avatar, setAvatar] = useState(null);
|
|
17
17
|
const avatarClientRef = useRef(null);
|
|
18
|
-
// Latest-options ref, so the mount effect (which runs once) still reads
|
|
19
|
-
//
|
|
18
|
+
// Latest-options ref, so the mount effect (which runs once) still reads the
|
|
19
|
+
// live props without re-subscribing.
|
|
20
20
|
const optionsRef = useRef(options);
|
|
21
21
|
optionsRef.current = options;
|
|
22
22
|
useEffect(() => {
|
|
23
23
|
const mount = containerRef.current;
|
|
24
24
|
if (!mount)
|
|
25
25
|
return;
|
|
26
|
-
const instance = createAvatar({
|
|
27
|
-
|
|
28
|
-
avatar: optionsRef.current.avatar,
|
|
29
|
-
theme: optionsRef.current.theme,
|
|
30
|
-
mouthGain: optionsRef.current.mouthGain,
|
|
31
|
-
gestureGain: optionsRef.current.gestureGain,
|
|
32
|
-
});
|
|
33
|
-
const wrapper = new AvatarClient(instance, {
|
|
34
|
-
onHint: (kind, msg) => optionsRef.current.onHint?.(kind, msg),
|
|
35
|
-
onUnknownCmd: (msg) => optionsRef.current.onUnknownCmd?.(msg),
|
|
36
|
-
onError: (err, msg) => optionsRef.current.onError?.(err, msg),
|
|
37
|
-
onSpeakingDrift: (info) => optionsRef.current.onSpeakingDrift?.(info),
|
|
38
|
-
accept: optionsRef.current.accept,
|
|
39
|
-
now: optionsRef.current.now,
|
|
40
|
-
});
|
|
41
|
-
avatarClientRef.current = wrapper;
|
|
26
|
+
const instance = createAvatar({ mount, avatar: optionsRef.current.avatar });
|
|
27
|
+
avatarClientRef.current = new AvatarClient(instance);
|
|
42
28
|
setAvatar(instance);
|
|
43
29
|
return () => {
|
|
44
30
|
instance.destroy();
|
|
45
31
|
avatarClientRef.current = null;
|
|
46
32
|
setAvatar(null);
|
|
47
33
|
};
|
|
48
|
-
// Mount once. `avatar
|
|
49
|
-
//
|
|
50
|
-
//
|
|
51
|
-
//
|
|
34
|
+
// Mount once. `avatar` is read at mount time only — the widget has no
|
|
35
|
+
// hot-swap-avatar API (`createFace` runs once per mount), so changing it
|
|
36
|
+
// re-renders nothing here by design; a caller that needs a different face
|
|
37
|
+
// remounts with a `key` prop (see the component's doc).
|
|
52
38
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
53
39
|
}, []);
|
|
54
40
|
useEffect(() => {
|
|
@@ -60,9 +46,6 @@ export function useAvatar(options = {}) {
|
|
|
60
46
|
// Re-subscribe when the widget mounts or the session's client changes.
|
|
61
47
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
62
48
|
}, [avatar, options.client]);
|
|
63
|
-
|
|
64
|
-
avatarClientRef.current?.dispatch(msg);
|
|
65
|
-
}, []);
|
|
66
|
-
return { containerRef, avatar, client: avatarClientRef.current, dispatch };
|
|
49
|
+
return { containerRef, avatar };
|
|
67
50
|
}
|
|
68
51
|
//# sourceMappingURL=useAvatar.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"useAvatar.js","sourceRoot":"","sources":["../src/useAvatar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"useAvatar.js","sourceRoot":"","sources":["../src/useAvatar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAEpD,OAAO,EAAE,YAAY,EAAkB,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AA+BjD,MAAM,UAAU,SAAS,CAAC,UAA4B,EAAE;IACtD,MAAM,YAAY,GAAG,MAAM,CAAiB,IAAI,CAAC,CAAC;IAClD,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,QAAQ,CAAmB,IAAI,CAAC,CAAC;IAC7D,MAAM,eAAe,GAAG,MAAM,CAAsB,IAAI,CAAC,CAAC;IAE1D,4EAA4E;IAC5E,qCAAqC;IACrC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IACnC,UAAU,CAAC,OAAO,GAAG,OAAO,CAAC;IAE7B,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC;QACnC,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5E,eAAe,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,QAAQ,CAAC,CAAC;QACrD,SAAS,CAAC,QAAQ,CAAC,CAAC;QAEpB,OAAO,GAAG,EAAE;YACV,QAAQ,CAAC,OAAO,EAAE,CAAC;YACnB,eAAe,CAAC,OAAO,GAAG,IAAI,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC,CAAC;QACF,sEAAsE;QACtE,yEAAyE;QACzE,0EAA0E;QAC1E,wDAAwD;QACxD,uDAAuD;IACzD,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;QACrC,IAAI,CAAC,OAAO,IAAI,CAAC,aAAa;YAAE,OAAO;QACvC,OAAO,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACrC,uEAAuE;QACvE,uDAAuD;IACzD,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAE7B,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;AAClC,CAAC"}
|
package/client/src/Avatar.tsx
CHANGED
|
@@ -1,38 +1,33 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Avatar — a call-tile-ready wrapper around the widget
|
|
3
|
-
*
|
|
4
|
-
* Thin by design: `useAvatar` does the work; this is the div it mounts into
|
|
5
|
-
* plus prop plumbing.
|
|
2
|
+
* Avatar — a call-tile-ready wrapper around the widget, and the package's
|
|
3
|
+
* whole public surface.
|
|
6
4
|
*
|
|
7
5
|
* <Avatar client={session.client} className="avatar-tile" />
|
|
8
6
|
*
|
|
7
|
+
* Two props of its own; everything else is forwarded to the mount `<div>`, so
|
|
8
|
+
* it sizes and styles like the tile it lives in. There is nothing to configure
|
|
9
|
+
* because there is nothing the server does not already say: the
|
|
10
|
+
* `AvatarProcessor` in the pipeline drives the state, the gaze and the mouth.
|
|
11
|
+
*
|
|
9
12
|
* The widget has no hot-swap-avatar API — `createFace` runs once per mount —
|
|
10
|
-
* so `avatar
|
|
11
|
-
*
|
|
13
|
+
* so `avatar` is read once, at mount. To switch faces at runtime, remount with
|
|
14
|
+
* a `key` prop:
|
|
12
15
|
*
|
|
13
16
|
* <Avatar key={name} avatar={name} client={session.client} />
|
|
14
17
|
*/
|
|
15
18
|
|
|
16
|
-
import type {
|
|
17
|
-
import type {
|
|
19
|
+
import type { HTMLAttributes } from "react";
|
|
20
|
+
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
18
21
|
import { useAvatar } from "./useAvatar.js";
|
|
19
22
|
|
|
20
|
-
export interface AvatarProps extends
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
/**
|
|
24
|
-
|
|
23
|
+
export interface AvatarProps extends HTMLAttributes<HTMLDivElement> {
|
|
24
|
+
/** The live `PipecatClient`, or `null` before connect. */
|
|
25
|
+
client?: PipecatClient | null;
|
|
26
|
+
/** Which face. Omit for the default. Read at mount only — see above. */
|
|
27
|
+
avatar?: string;
|
|
25
28
|
}
|
|
26
29
|
|
|
27
|
-
export function Avatar({
|
|
28
|
-
const { containerRef } = useAvatar(
|
|
29
|
-
return
|
|
30
|
-
<div
|
|
31
|
-
ref={containerRef}
|
|
32
|
-
className={className}
|
|
33
|
-
style={style}
|
|
34
|
-
aria-label={ariaLabel ?? "avatar"}
|
|
35
|
-
role="img"
|
|
36
|
-
/>
|
|
37
|
-
);
|
|
30
|
+
export function Avatar({ client, avatar, ...rest }: AvatarProps) {
|
|
31
|
+
const { containerRef } = useAvatar({ client, avatar });
|
|
32
|
+
return <div role="img" aria-label="avatar" {...rest} ref={containerRef} />;
|
|
38
33
|
}
|