@voqalize/avatar 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-CC-BY-4.0 +396 -0
- package/README.md +65 -17
- package/assets/README.md +48 -0
- package/assets/tanya.glb +0 -0
- package/assets/tara.glb +0 -0
- package/assets/tushar.glb +0 -0
- package/client/AvatarClient.ts +337 -42
- package/client/arjun.ts +2 -0
- package/client/createAvatar.ts +38 -0
- package/client/index.ts +6 -1
- package/client/internal.ts +31 -3
- package/client/ishita.ts +2 -0
- package/client/kabir.ts +2 -0
- package/client/meera.ts +2 -0
- package/client/naina.ts +2 -0
- package/client/playout.ts +95 -0
- package/client/supports.ts +28 -0
- package/client/three/assets.ts +22 -0
- package/client/three/budgets.ts +27 -0
- package/client/three/holds.ts +64 -0
- package/client/three/internal.ts +54 -0
- package/client/three/motion-limits.json +116 -0
- package/client/three/sequences.ts +281 -0
- package/client/three/tanya-asset.ts +6 -0
- package/client/three/tanya.ts +69 -0
- package/client/three/tara-asset.ts +25 -0
- package/client/three/tara-rig.ts +1571 -0
- package/client/three/tara.ts +77 -0
- package/client/three/tushar-asset.ts +6 -0
- package/client/three/tushar.ts +65 -0
- package/client/types.ts +109 -20
- package/client/vikram.ts +2 -0
- package/dist/AvatarClient.d.ts +89 -11
- package/dist/AvatarClient.d.ts.map +1 -1
- package/dist/AvatarClient.js +297 -35
- package/dist/AvatarClient.js.map +1 -1
- package/dist/arjun.d.ts +1 -0
- package/dist/arjun.d.ts.map +1 -1
- package/dist/arjun.js +2 -0
- package/dist/arjun.js.map +1 -1
- package/dist/createAvatar.d.ts +36 -0
- package/dist/createAvatar.d.ts.map +1 -1
- package/dist/createAvatar.js.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/internal.d.ts +9 -5
- package/dist/internal.d.ts.map +1 -1
- package/dist/internal.js +13 -5
- package/dist/internal.js.map +1 -1
- package/dist/ishita.d.ts +1 -0
- package/dist/ishita.d.ts.map +1 -1
- package/dist/ishita.js +2 -0
- package/dist/ishita.js.map +1 -1
- package/dist/kabir.d.ts +1 -0
- package/dist/kabir.d.ts.map +1 -1
- package/dist/kabir.js +2 -0
- package/dist/kabir.js.map +1 -1
- package/dist/meera.d.ts +1 -0
- package/dist/meera.d.ts.map +1 -1
- package/dist/meera.js +2 -0
- package/dist/meera.js.map +1 -1
- package/dist/naina.d.ts +1 -0
- package/dist/naina.d.ts.map +1 -1
- package/dist/naina.js +2 -0
- package/dist/naina.js.map +1 -1
- package/dist/playout.d.ts +42 -0
- package/dist/playout.d.ts.map +1 -0
- package/dist/playout.js +85 -0
- package/dist/playout.js.map +1 -0
- package/dist/supports.d.ts +20 -0
- package/dist/supports.d.ts.map +1 -0
- package/dist/supports.js +26 -0
- package/dist/supports.js.map +1 -0
- package/dist/three/assets.d.ts +8 -0
- package/dist/three/assets.d.ts.map +1 -0
- package/dist/three/assets.js +19 -0
- package/dist/three/assets.js.map +1 -0
- package/dist/three/budgets.d.ts +17 -0
- package/dist/three/budgets.d.ts.map +1 -0
- package/dist/three/budgets.js +21 -0
- package/dist/three/budgets.js.map +1 -0
- package/dist/three/holds.d.ts +33 -0
- package/dist/three/holds.d.ts.map +1 -0
- package/dist/three/holds.js +56 -0
- package/dist/three/holds.js.map +1 -0
- package/dist/three/internal.d.ts +52 -0
- package/dist/three/internal.d.ts.map +1 -0
- package/dist/three/internal.js +49 -0
- package/dist/three/internal.js.map +1 -0
- package/dist/three/motion-limits.json +116 -0
- package/dist/three/sequences.d.ts +71 -0
- package/dist/three/sequences.d.ts.map +1 -0
- package/dist/three/sequences.js +262 -0
- package/dist/three/sequences.js.map +1 -0
- package/dist/three/tanya-asset.d.ts +7 -0
- package/dist/three/tanya-asset.d.ts.map +1 -0
- package/dist/three/tanya-asset.js +7 -0
- package/dist/three/tanya-asset.js.map +1 -0
- package/dist/three/tanya.d.ts +35 -0
- package/dist/three/tanya.d.ts.map +1 -0
- package/dist/three/tanya.js +51 -0
- package/dist/three/tanya.js.map +1 -0
- package/dist/three/tara-asset.d.ts +26 -0
- package/dist/three/tara-asset.d.ts.map +1 -0
- package/dist/three/tara-asset.js +26 -0
- package/dist/three/tara-asset.js.map +1 -0
- package/dist/three/tara-rig.d.ts +409 -0
- package/dist/three/tara-rig.d.ts.map +1 -0
- package/dist/three/tara-rig.js +1504 -0
- package/dist/three/tara-rig.js.map +1 -0
- package/dist/three/tara.d.ts +35 -0
- package/dist/three/tara.d.ts.map +1 -0
- package/dist/three/tara.js +59 -0
- package/dist/three/tara.js.map +1 -0
- package/dist/three/tushar-asset.d.ts +7 -0
- package/dist/three/tushar-asset.d.ts.map +1 -0
- package/dist/three/tushar-asset.js +7 -0
- package/dist/three/tushar-asset.js.map +1 -0
- package/dist/three/tushar.d.ts +31 -0
- package/dist/three/tushar.d.ts.map +1 -0
- package/dist/three/tushar.js +47 -0
- package/dist/three/tushar.js.map +1 -0
- package/dist/types.d.ts +65 -10
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +58 -11
- package/dist/types.js.map +1 -1
- package/dist/vikram.d.ts +1 -0
- package/dist/vikram.d.ts.map +1 -1
- package/dist/vikram.js +2 -0
- package/dist/vikram.js.map +1 -1
- package/package.json +34 -6
- package/src/avatar.d.ts +59 -2
- package/src/avatar.js +743 -94
- package/src/behavior.d.ts +28 -12
- package/src/behavior.js +54 -42
- package/src/canvas/create-rig.js +1 -1
- package/src/canvas/src/live.js +2 -2
- package/src/conformance.js +1 -1
- package/src/emotions.js +4 -2
- package/src/gaze.js +334 -73
- package/src/head.js +134 -0
- package/src/idle.js +210 -19
- package/src/interjections.js +70 -26
- package/src/prosody.js +647 -0
- package/src/visemes.js +85 -15
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tara — the first 3-D character, as a complete `createAvatar` module.
|
|
3
|
+
*
|
|
4
|
+
* Structurally this is the canvas avatars' shape, one layer down: the mixer
|
|
5
|
+
* (`@voqalize/avatar/internal`) does states, layers, gaze, idle, blinks, clips
|
|
6
|
+
* and per-channel smoothing exactly as it does for every SVG face, and the only
|
|
7
|
+
* thing tara replaces is the renderer at the end of it. That is the whole
|
|
8
|
+
* argument for authoring her morph targets under the library's pose-channel
|
|
9
|
+
* names: nothing above this line knows there is a GPU involved, and every clip,
|
|
10
|
+
* viseme and co-articulation rule the SVG faces have works here unmodified.
|
|
11
|
+
*
|
|
12
|
+
* The Three.js import lives behind this module rather than in the barrel so an
|
|
13
|
+
* SVG or Canvas consumer never downloads it.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
17
|
+
import { AvatarClient, createSvgAvatar } from "../internal.js";
|
|
18
|
+
import { BLENDER_ACTIONS, BLENDER_SEQUENCES, BLENDER_SUPPORTS } from "./sequences.js";
|
|
19
|
+
import { TARA_GLB } from "./tara-asset.js";
|
|
20
|
+
import { headHold } from "./holds.js";
|
|
21
|
+
import { createTaraRig, TARA_TUNING } from "./tara-rig.js";
|
|
22
|
+
import type { TaraRigOptions } from "./tara-rig.js";
|
|
23
|
+
|
|
24
|
+
export interface AvatarOptions {
|
|
25
|
+
readonly mount: HTMLElement;
|
|
26
|
+
readonly client: PipecatClient;
|
|
27
|
+
/** Viseme amplitude, 0..2. `1` is as authored. */
|
|
28
|
+
readonly mouthGain?: number;
|
|
29
|
+
/** Gesture-clip amplitude, 0..2. */
|
|
30
|
+
readonly gestureGain?: number;
|
|
31
|
+
/** Idle/liveness amplitude, 0..2. */
|
|
32
|
+
readonly motionGain?: number;
|
|
33
|
+
/** Fires when the GLB is in the scene — for a capture tool, not a consumer. */
|
|
34
|
+
readonly onReady?: () => void;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface AvatarInstance { destroy(): void; }
|
|
38
|
+
|
|
39
|
+
/** The optional driving-UI declaration; one list for all three characters
|
|
40
|
+
* (`sequences.ts`), because they are one rig. */
|
|
41
|
+
export const supports = BLENDER_SUPPORTS;
|
|
42
|
+
|
|
43
|
+
export function createAvatar(options: AvatarOptions): AvatarInstance {
|
|
44
|
+
const { mount, client, onReady, ...gains } = options;
|
|
45
|
+
if (!mount) throw new TypeError("createAvatar: `mount` is required");
|
|
46
|
+
if (!client) throw new TypeError("createAvatar: `client` is required");
|
|
47
|
+
|
|
48
|
+
const rigOptions: TaraRigOptions = { onReady, url: TARA_GLB };
|
|
49
|
+
// `hand: false` disables the bundled SVG hand renderer only; the semantic
|
|
50
|
+
// hand frame still reaches the rig, which ignores it — tara has no arms, and
|
|
51
|
+
// that is the library's oldest standing constraint rather than an omission.
|
|
52
|
+
// `sequences` is what a server can address on *this* avatar beyond the two
|
|
53
|
+
// ids every avatar owes it — the three nod types the listening research
|
|
54
|
+
// separates, and a head shake sized for a rig whose pose unit is a degree.
|
|
55
|
+
// The wire's action id is open, so nothing here needed promoting; a server
|
|
56
|
+
// that does not know tara is mounted sends `ACKNOWLEDGE` and is never wrong.
|
|
57
|
+
// `actions` is the other half: her own shape for an id the mixer already has,
|
|
58
|
+
// whose shared keys land outside what it means on a head that turns in degrees.
|
|
59
|
+
const widget = createSvgAvatar({
|
|
60
|
+
mount, rig: createTaraRig, rigOptions, hand: false,
|
|
61
|
+
sequences: BLENDER_SEQUENCES, actions: BLENDER_ACTIONS, ...TARA_TUNING,
|
|
62
|
+
headHold: headHold("tara"), ...gains,
|
|
63
|
+
});
|
|
64
|
+
const driver = new AvatarClient(widget);
|
|
65
|
+
const detach = driver.attach(client);
|
|
66
|
+
|
|
67
|
+
let destroyed = false;
|
|
68
|
+
return {
|
|
69
|
+
destroy() {
|
|
70
|
+
if (destroyed) return;
|
|
71
|
+
destroyed = true;
|
|
72
|
+
detach();
|
|
73
|
+
driver.destroy();
|
|
74
|
+
widget.destroy();
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where tushar's compiled GLB is. One character, one module — see
|
|
3
|
+
* [tara-asset.ts](./tara-asset.ts) for why that separation is load-bearing and
|
|
4
|
+
* why the `new URL` literal is spelled exactly this way.
|
|
5
|
+
*/
|
|
6
|
+
export const TUSHAR_GLB = new URL("../../assets/tushar.glb", import.meta.url).href;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tushar — the second 3-D character, and deliberately a copy of `tara.ts`.
|
|
3
|
+
*
|
|
4
|
+
* His GLB is tara's scripts run with his landmarks and atlas layout in place of
|
|
5
|
+
* hers (`characters/tushar/`, whose README has the provenance and the commands),
|
|
6
|
+
* so the rig, the mixer, the sequences and her tuning are all hers, unmodified;
|
|
7
|
+
* the one seam is `TaraRigOptions.url`. What turns out to be hers rather than
|
|
8
|
+
* general is marked TARA-SPECIFIC where it lives, and generalised only once a
|
|
9
|
+
* third character agrees.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
13
|
+
import { AvatarClient, createSvgAvatar } from "../internal.js";
|
|
14
|
+
import { TUSHAR_GLB } from "./tushar-asset.js";
|
|
15
|
+
import { BLENDER_ACTIONS, BLENDER_SEQUENCES, BLENDER_SUPPORTS } from "./sequences.js";
|
|
16
|
+
import { headHold } from "./holds.js";
|
|
17
|
+
import { createTaraRig, TARA_TUNING } from "./tara-rig.js";
|
|
18
|
+
import type { TaraRigOptions } from "./tara-rig.js";
|
|
19
|
+
|
|
20
|
+
export interface AvatarOptions {
|
|
21
|
+
readonly mount: HTMLElement;
|
|
22
|
+
readonly client: PipecatClient;
|
|
23
|
+
/** Viseme amplitude, 0..2. `1` is as authored. */
|
|
24
|
+
readonly mouthGain?: number;
|
|
25
|
+
/** Gesture-clip amplitude, 0..2. */
|
|
26
|
+
readonly gestureGain?: number;
|
|
27
|
+
/** Idle/liveness amplitude, 0..2. */
|
|
28
|
+
readonly motionGain?: number;
|
|
29
|
+
/** Fires when the GLB is in the scene — for a capture tool, not a consumer. */
|
|
30
|
+
readonly onReady?: () => void;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface AvatarInstance { destroy(): void; }
|
|
34
|
+
|
|
35
|
+
/** The optional driving-UI declaration; one list for all three characters
|
|
36
|
+
* (`sequences.ts`), because they are one rig. */
|
|
37
|
+
export const supports = BLENDER_SUPPORTS;
|
|
38
|
+
|
|
39
|
+
export function createAvatar(options: AvatarOptions): AvatarInstance {
|
|
40
|
+
const { mount, client, onReady, ...gains } = options;
|
|
41
|
+
if (!mount) throw new TypeError("createAvatar: `mount` is required");
|
|
42
|
+
if (!client) throw new TypeError("createAvatar: `client` is required");
|
|
43
|
+
|
|
44
|
+
const rigOptions: TaraRigOptions = { onReady, url: TUSHAR_GLB };
|
|
45
|
+
// TARA-SPECIFIC, kept on purpose: `TARA_TUNING` (her mouth and motion gains)
|
|
46
|
+
// is the first thing to question if he reads wrong in a call.
|
|
47
|
+
const widget = createSvgAvatar({
|
|
48
|
+
mount, rig: createTaraRig, rigOptions, hand: false,
|
|
49
|
+
sequences: BLENDER_SEQUENCES, actions: BLENDER_ACTIONS, ...TARA_TUNING,
|
|
50
|
+
headHold: headHold("tushar"), ...gains,
|
|
51
|
+
});
|
|
52
|
+
const driver = new AvatarClient(widget);
|
|
53
|
+
const detach = driver.attach(client);
|
|
54
|
+
|
|
55
|
+
let destroyed = false;
|
|
56
|
+
return {
|
|
57
|
+
destroy() {
|
|
58
|
+
if (destroyed) return;
|
|
59
|
+
destroyed = true;
|
|
60
|
+
detach();
|
|
61
|
+
driver.destroy();
|
|
62
|
+
widget.destroy();
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
package/client/types.ts
CHANGED
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
* ignored, which is the whole forward-compatibility rule.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import {
|
|
19
|
-
import type {
|
|
18
|
+
import { VISEME_LETTERS } from "../src/avatar.js";
|
|
19
|
+
import type { VisemeLetter } from "../src/avatar.js";
|
|
20
20
|
|
|
21
21
|
/** A viseme cue: `t` is a ms offset into the utterance's clock, `v` is a Rhubarb A–H (or X) letter. */
|
|
22
22
|
export interface AvatarCue {
|
|
@@ -24,20 +24,77 @@ export interface AvatarCue {
|
|
|
24
24
|
v: VisemeLetter;
|
|
25
25
|
/** 0..1 loudness. Absent means full. */
|
|
26
26
|
i?: number;
|
|
27
|
+
/**
|
|
28
|
+
* The phone being articulated under `v`, absent during silence.
|
|
29
|
+
*
|
|
30
|
+
* `v` is a nine-way projection of the recogniser's ~40 phones and the loss is
|
|
31
|
+
* concentrated — `B` alone absorbs IY, IH, T, D, CH, JH, TH, DH, S, Z, SH,
|
|
32
|
+
* ZH, N and Y — so a renderer with a mouth for "tongue between the teeth"
|
|
33
|
+
* cannot ask for it from `v` and can from here. **Not a closed set**: it is
|
|
34
|
+
* whatever the engine labels its own segments with, which today is Arpabet
|
|
35
|
+
* plus `Schwa` and four non-speech labels, and a later engine may say
|
|
36
|
+
* something else. Validated as an identifier and passed through unread —
|
|
37
|
+
* the letter stays the thing every face must handle, and a face that wants
|
|
38
|
+
* the finer signal opts in.
|
|
39
|
+
*/
|
|
40
|
+
p?: string;
|
|
27
41
|
}
|
|
28
42
|
|
|
29
|
-
/**
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
43
|
+
/**
|
|
44
|
+
* A durable, lower-priority server state. `null` explicitly clears it.
|
|
45
|
+
*
|
|
46
|
+
* Three values, and the server can send no others: the remaining six of the
|
|
47
|
+
* nine are Pipecat facts the browser already holds, and a server spelling of a
|
|
48
|
+
* fact would be a second, lower-authority copy of it. It was `cmd: "claim"`
|
|
49
|
+
* until the wire redesign, when the concept and the command were given the one
|
|
50
|
+
* name the rest of the system already used for it.
|
|
51
|
+
*/
|
|
52
|
+
export interface AvatarStateCmd {
|
|
53
|
+
cmd: "state";
|
|
54
|
+
state: "CANT_HEAR" | "THINKING" | "WORKING" | null;
|
|
33
55
|
}
|
|
34
56
|
|
|
35
|
-
/**
|
|
57
|
+
/**
|
|
58
|
+
* One self-completing motion, by name: face, body, and optionally a hand.
|
|
59
|
+
*
|
|
60
|
+
* **The id is open.** Two of them — {@link CORE_ACTION_IDS} — are intents every
|
|
61
|
+
* renderer owes a server and are the only ones a server may send without
|
|
62
|
+
* knowing what is mounted. Everything else is a name from the mounted avatar's
|
|
63
|
+
* own catalogue, and **an unknown one is ignored, not an error**: that is the
|
|
64
|
+
* same forward-compatibility rule an unknown `cmd` gets, arrived at from the
|
|
65
|
+
* other direction. There, a newer server meets an older widget; here, any
|
|
66
|
+
* server meets a face that cannot do the thing. Neither is worth breaking a
|
|
67
|
+
* call over, and there is deliberately no fallback — an action sits on top of a
|
|
68
|
+
* state, so nothing is missing when one is dropped.
|
|
69
|
+
*
|
|
70
|
+
* This replaced a closed seven-id vocabulary plus a second command,
|
|
71
|
+
* `sequence`, for a renderer's own motions. The split cost a promotion ritual
|
|
72
|
+
* for every new portable intent and still could not say what the closed set was
|
|
73
|
+
* for: the listening research separates a continuer nod from an assessment nod
|
|
74
|
+
* from a realisation, and all three were the one id `ACK_NOD` spelled. Now the
|
|
75
|
+
* server says *when* to acknowledge and the avatar owns the variety.
|
|
76
|
+
*/
|
|
36
77
|
export interface AvatarActionCmd {
|
|
37
78
|
cmd: "action";
|
|
38
|
-
id:
|
|
79
|
+
id: string;
|
|
39
80
|
}
|
|
40
81
|
|
|
82
|
+
/**
|
|
83
|
+
* The two actions every renderer must answer to.
|
|
84
|
+
*
|
|
85
|
+
* `ACKNOWLEDGE` is the whole backchannel family in one word — receipt, nod,
|
|
86
|
+
* realisation, empathy — because which of those a face does is a rendering
|
|
87
|
+
* decision and the server is not the one holding the drawing.
|
|
88
|
+
* `RESPONSE_INTERRUPTED` is the one transition a server can explain and the
|
|
89
|
+
* browser cannot infer.
|
|
90
|
+
*
|
|
91
|
+
* A renderer may draw either in its own shape; it may never redefine one, and
|
|
92
|
+
* a name in its own catalogue never shadows one of these.
|
|
93
|
+
*/
|
|
94
|
+
export const CORE_ACTION_IDS = Object.freeze(["ACKNOWLEDGE", "RESPONSE_INTERRUPTED"] as const);
|
|
95
|
+
|
|
96
|
+
export type CoreActionId = (typeof CORE_ACTION_IDS)[number];
|
|
97
|
+
|
|
41
98
|
export interface AvatarCuesCmd {
|
|
42
99
|
cmd: "cues";
|
|
43
100
|
ctx: string;
|
|
@@ -60,7 +117,7 @@ export interface AvatarCuesCmd {
|
|
|
60
117
|
}
|
|
61
118
|
|
|
62
119
|
export type AvatarCommand =
|
|
63
|
-
|
|
|
120
|
+
| AvatarStateCmd
|
|
64
121
|
| AvatarActionCmd
|
|
65
122
|
| AvatarCuesCmd;
|
|
66
123
|
|
|
@@ -85,13 +142,25 @@ export function isAvatarMessage(msg: unknown): msg is AvatarEnvelope {
|
|
|
85
142
|
return m.type === AVATAR_MESSAGE_TYPE && typeof m.cmd === "string";
|
|
86
143
|
}
|
|
87
144
|
|
|
88
|
-
|
|
145
|
+
/**
|
|
146
|
+
* What an action id may look like. Not which ones exist — that is the mounted
|
|
147
|
+
* avatar's business and this function has never met it. `CATEGORY_INTENT` in
|
|
148
|
+
* upper case, bounded so a malformed or hostile message cannot smuggle
|
|
149
|
+
* anything through as a name.
|
|
150
|
+
*/
|
|
151
|
+
const ACTION_ID = /^[A-Z][A-Z0-9_]{1,63}$/;
|
|
89
152
|
const LETTERS = new Set<string>(VISEME_LETTERS);
|
|
153
|
+
/**
|
|
154
|
+
* What a phone label may look like. Letters only, because every label any
|
|
155
|
+
* engine we have seen is one word of them — `AO`, `NG`, `Schwa`, `Breath` —
|
|
156
|
+
* and the bound is what stops a name being a payload.
|
|
157
|
+
*/
|
|
158
|
+
const PHONE_LABEL = /^[A-Za-z]{1,12}$/;
|
|
90
159
|
|
|
91
160
|
/**
|
|
92
161
|
* Read an envelope's payload into the wire vocabulary, or `null` if this build
|
|
93
|
-
* cannot act on it — an unknown `cmd`,
|
|
94
|
-
* malformed `cues` chunk.
|
|
162
|
+
* cannot act on it — an unknown `cmd`, a state outside the three, an action id
|
|
163
|
+
* that is not a name, a malformed `cues` chunk.
|
|
95
164
|
*
|
|
96
165
|
* `null` is the forward-compat rule with a type attached: a newer server
|
|
97
166
|
* talking to an older widget is *expected*, and the older widget ignores what
|
|
@@ -106,16 +175,29 @@ const LETTERS = new Set<string>(VISEME_LETTERS);
|
|
|
106
175
|
*/
|
|
107
176
|
export function parseAvatarCommand(msg: AvatarEnvelope): AvatarCommand | null {
|
|
108
177
|
switch (msg.cmd) {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
178
|
+
// `claim` was this command's name, and `STRAINING` one of its values, until
|
|
179
|
+
// the wire redesign. Both are accepted and translated here, in the one
|
|
180
|
+
// place that reads untrusted input, so no layer above has ever heard of
|
|
181
|
+
// either spelling. Servers that predate the rename keep working; the old
|
|
182
|
+
// names appear nowhere else and come out when the last one has shipped.
|
|
183
|
+
case "claim":
|
|
184
|
+
case "state": {
|
|
185
|
+
const state = msg.state === "STRAINING" ? "CANT_HEAR" : msg.state;
|
|
186
|
+
if (state === null || state === "CANT_HEAR" || state === "THINKING" || state === "WORKING") {
|
|
187
|
+
return { cmd: "state", state };
|
|
113
188
|
}
|
|
114
189
|
return null;
|
|
115
190
|
}
|
|
191
|
+
// `sequence` was a second command for the same thing: a name resolved
|
|
192
|
+
// against the mounted avatar rather than against a closed list. Now that
|
|
193
|
+
// `action` is that, the two are one command and the old one is an alias.
|
|
194
|
+
case "sequence":
|
|
116
195
|
case "action":
|
|
117
|
-
|
|
118
|
-
|
|
196
|
+
// Shape only, both for a core id and for one of the avatar's own. There
|
|
197
|
+
// is nothing to check a name against here — the catalogue belongs to
|
|
198
|
+
// whatever is mounted, and this function has never met it.
|
|
199
|
+
return typeof msg.id === "string" && ACTION_ID.test(msg.id)
|
|
200
|
+
? { cmd: "action", id: msg.id }
|
|
119
201
|
: null;
|
|
120
202
|
case "cues": {
|
|
121
203
|
if (typeof msg.ctx !== "string" || !Number.isFinite(msg.from_ms) || !Array.isArray(msg.cues)) {
|
|
@@ -124,10 +206,17 @@ export function parseAvatarCommand(msg: AvatarEnvelope): AvatarCommand | null {
|
|
|
124
206
|
const cues: AvatarCue[] = [];
|
|
125
207
|
for (const c of msg.cues as unknown[]) {
|
|
126
208
|
if (typeof c !== "object" || c === null) continue;
|
|
127
|
-
const { t, v, i } = c as Record<string, unknown>;
|
|
209
|
+
const { t, v, i, p } = c as Record<string, unknown>;
|
|
128
210
|
if (typeof t !== "number" || !Number.isFinite(t)) continue;
|
|
129
211
|
if (typeof v !== "string" || !LETTERS.has(v)) continue;
|
|
130
|
-
|
|
212
|
+
const cue: AvatarCue = { t, v: v as VisemeLetter };
|
|
213
|
+
if (typeof i === "number") cue.i = i;
|
|
214
|
+
// Bounded as a name, not checked against a list. The set belongs to
|
|
215
|
+
// whatever produced the cue, so enumerating it here would make this
|
|
216
|
+
// library the thing that has to be released before a better recogniser
|
|
217
|
+
// can say a new word.
|
|
218
|
+
if (typeof p === "string" && PHONE_LABEL.test(p)) cue.p = p;
|
|
219
|
+
cues.push(cue);
|
|
131
220
|
}
|
|
132
221
|
return {
|
|
133
222
|
cmd: "cues",
|
package/client/vikram.ts
CHANGED
|
@@ -13,6 +13,8 @@ const IMAGES = {
|
|
|
13
13
|
const FACE = () => import('../src/canvas/avatars/round/face.mjs');
|
|
14
14
|
|
|
15
15
|
export type { CanvasAvatarOptions };
|
|
16
|
+
// Same mixer, same clip library, so the same declaration (`supports.ts`).
|
|
17
|
+
export { supports } from './supports.js';
|
|
16
18
|
|
|
17
19
|
export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
|
|
18
20
|
return createCanvasAvatar(options, {
|
package/dist/AvatarClient.d.ts
CHANGED
|
@@ -7,17 +7,36 @@
|
|
|
7
7
|
* Base Pipecat TTS gives each serialized TTS context an opaque `context_id`.
|
|
8
8
|
* The server uses it only to group and splice cue chunks. `botStartedSpeaking`
|
|
9
9
|
* has no context payload, so the browser FIFO-claims the next buffered context
|
|
10
|
-
* at that Pipecat output-lifecycle event
|
|
11
|
-
*
|
|
12
|
-
*
|
|
10
|
+
* at that Pipecat output-lifecycle event. `botStoppedSpeaking` closes it. No
|
|
11
|
+
* avatar-specific speech marker exists.
|
|
12
|
+
*
|
|
13
|
+
* The event says *which* audio is starting, not *when* it is heard, and the
|
|
14
|
+
* turn clock is anchored to the sound itself (`playout.ts` has the
|
|
15
|
+
* measurements that forced it). `attach()` listens to the bot's audio track;
|
|
16
|
+
* at the event, the turn's zero is backdated to a sound that has already begun,
|
|
17
|
+
* or the mouth is held shut until one does. When the track cannot be heard —
|
|
18
|
+
* no `attach()`, no track, a suspended audio graph, or a sound that never
|
|
19
|
+
* arrives within `ONSET_WAIT_MS` — the event is the anchor, as it always was.
|
|
20
|
+
* Which one won is logged once per turn, at `info`, because a desync report is
|
|
21
|
+
* unanswerable without it.
|
|
22
|
+
*
|
|
23
|
+
* The clock then runs `VISUAL_LEAD_MS` ahead of the sound, since the mixer's
|
|
24
|
+
* smoothing makes every mouth shape late by about that much. It is pulled back
|
|
25
|
+
* by any output-device latency beyond what the display already matches (a
|
|
26
|
+
* Bluetooth headset). Within a turn, each resumption after a pause in the
|
|
27
|
+
* track is a second chance to hear where the sound really is — but a mouth
|
|
28
|
+
* opens before its sound, so a resumption heard within the track's usual
|
|
29
|
+
* anticipation of it agrees with the clock. Only a disagreement beyond that is
|
|
30
|
+
* slewed out, at no more than 10 % of clock rate, never jumped, because a
|
|
31
|
+
* mouth that skips is seen and one that runs briefly fast is not.
|
|
13
32
|
*
|
|
14
33
|
* `attach()` subscribes to the avatar server-message channel *and* Pipecat's
|
|
15
34
|
* standard lifecycle events. Server messages carry only what Pipecat cannot:
|
|
16
35
|
* correlated speech/cue timing and deliberate application instructions. The
|
|
17
36
|
* lifecycle events project the factual presence states locally. The server
|
|
18
|
-
* supplies only lower-priority
|
|
37
|
+
* supplies only the three lower-priority states and deliberate,
|
|
19
38
|
* self-completing actions. This keeps the face tied to Pipecat's output truth
|
|
20
|
-
* even if a server
|
|
39
|
+
* even if a server state is delayed or stale.
|
|
21
40
|
*
|
|
22
41
|
* ## Cue splice
|
|
23
42
|
*
|
|
@@ -51,8 +70,41 @@
|
|
|
51
70
|
* necessarily the first *message*.
|
|
52
71
|
*/
|
|
53
72
|
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
54
|
-
import type
|
|
73
|
+
import { type PlayoutProbe } from "./playout.js";
|
|
55
74
|
import { type AvatarCommand, type AvatarCue } from "./types.js";
|
|
75
|
+
/**
|
|
76
|
+
* How far the cue clock runs ahead of the sound. The mixer eases every mouth
|
|
77
|
+
* channel toward its target with a 42 ms time constant
|
|
78
|
+
* (`MOUTH_RESPONSE_TAU_S`), which delays a shape by about that much, and a
|
|
79
|
+
* frame is drawn on average half a frame after its time. Picture ahead of sound
|
|
80
|
+
* is the side people forgive (ITU-R BT.1359: sound leading is noticed at about
|
|
81
|
+
* 45 ms, picture leading at about 125 ms), so a small surplus is the safe error.
|
|
82
|
+
*/
|
|
83
|
+
export declare const VISUAL_LEAD_MS = 50;
|
|
84
|
+
/**
|
|
85
|
+
* The private renderer-side vocabulary the lifecycle driver actually uses.
|
|
86
|
+
*
|
|
87
|
+
* This stays narrower than `AvatarApi` on purpose. The latter is the SVG
|
|
88
|
+
* mixer's driving API; requiring a second renderer to impersonate every one
|
|
89
|
+
* of its inspection and authoring methods would turn that implementation
|
|
90
|
+
* detail into a renderer contract. The public seam remains `createAvatar`.
|
|
91
|
+
*/
|
|
92
|
+
export interface AvatarDriver {
|
|
93
|
+
setState(name: string): unknown;
|
|
94
|
+
/**
|
|
95
|
+
* One motion by name — a core intent or one of this avatar's own. **A name
|
|
96
|
+
* it does not have is a no-op, not a throw**: the wire's action vocabulary is
|
|
97
|
+
* open, so a face that cannot do the thing is as expected as a newer server.
|
|
98
|
+
*/
|
|
99
|
+
action(id: string): unknown;
|
|
100
|
+
speak(options: {
|
|
101
|
+
cues: AvatarCue[];
|
|
102
|
+
clock: () => number;
|
|
103
|
+
}): unknown;
|
|
104
|
+
pushCues(cues: AvatarCue[]): unknown;
|
|
105
|
+
stopSpeaking(): unknown;
|
|
106
|
+
setUserSpeaking(speaking: boolean | null): unknown;
|
|
107
|
+
}
|
|
56
108
|
/**
|
|
57
109
|
* Internal. Not exported from the package — the public surface is
|
|
58
110
|
* `createAvatar({ mount, client })` and nothing else.
|
|
@@ -70,6 +122,9 @@ export interface AvatarClientOptions {
|
|
|
70
122
|
onError?: (err: unknown, msg: AvatarCommand) => void;
|
|
71
123
|
/** Override for tests. Defaults to `performance.now`. */
|
|
72
124
|
now?: () => number;
|
|
125
|
+
/** Override for tests. Defaults to listening to the bot's audio track, once
|
|
126
|
+
* `attach()` has found one. */
|
|
127
|
+
playoutProbe?: PlayoutProbe;
|
|
73
128
|
/** Quiet time in listening before the client-owned idle loop begins. */
|
|
74
129
|
idleDelayMs?: number;
|
|
75
130
|
/** Timer seams keep lifecycle behavior deterministic in tests. */
|
|
@@ -110,11 +165,12 @@ export declare const RTVI_EVENTS: {
|
|
|
110
165
|
readonly userStoppedSpeaking: "userStoppedSpeaking";
|
|
111
166
|
readonly botStartedSpeaking: "botStartedSpeaking";
|
|
112
167
|
readonly botStoppedSpeaking: "botStoppedSpeaking";
|
|
168
|
+
readonly trackStarted: "trackStarted";
|
|
113
169
|
readonly userMuteStarted: "userMuteStarted";
|
|
114
170
|
readonly userMuteStopped: "userMuteStopped";
|
|
115
171
|
};
|
|
116
172
|
/** The resolved, factual presence state a host may render around the avatar. */
|
|
117
|
-
export type AvatarPresenceState = "IDLE" | "LISTENING" | "
|
|
173
|
+
export type AvatarPresenceState = "IDLE" | "LISTENING" | "CANT_HEAR" | "THINKING" | "WORKING" | "MUTED" | "SPEAKING" | "DEGRADED" | "OFFLINE";
|
|
118
174
|
export declare class AvatarClient {
|
|
119
175
|
private readonly avatar;
|
|
120
176
|
/** Maps factual/wire intent into the broader client behavior catalog. */
|
|
@@ -126,7 +182,7 @@ export declare class AvatarClient {
|
|
|
126
182
|
private readonly pendingCtxs;
|
|
127
183
|
private readonly closedCtxs;
|
|
128
184
|
private projected;
|
|
129
|
-
private
|
|
185
|
+
private serverState;
|
|
130
186
|
private userSpeaking;
|
|
131
187
|
private botSpeaking;
|
|
132
188
|
private muted;
|
|
@@ -139,7 +195,14 @@ export declare class AvatarClient {
|
|
|
139
195
|
private readonly idleDelayMs;
|
|
140
196
|
private readonly setTimer;
|
|
141
197
|
private readonly clearTimer;
|
|
142
|
-
|
|
198
|
+
private probe;
|
|
199
|
+
private probeTrack;
|
|
200
|
+
/** Whether the probe has ever heard an onset. One that has not, and times
|
|
201
|
+
* out, is deaf to this track rather than early to it. */
|
|
202
|
+
private probeHeard;
|
|
203
|
+
private onsetTimer;
|
|
204
|
+
private reanchorTimer;
|
|
205
|
+
constructor(avatar: AvatarDriver, opts?: AvatarClientOptions);
|
|
143
206
|
/** The active turn's ctx, or `null` between turns. For tests and telemetry. */
|
|
144
207
|
get turnCtx(): string | null;
|
|
145
208
|
/** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
|
|
@@ -156,12 +219,27 @@ export declare class AvatarClient {
|
|
|
156
219
|
* is not ours and is ignored; so is an envelope carrying a `cmd` this build
|
|
157
220
|
* has never heard of, per the wire protocol's forward-compat rule. */
|
|
158
221
|
dispatch(raw: unknown): void;
|
|
159
|
-
private
|
|
222
|
+
private handleState;
|
|
160
223
|
private handleAction;
|
|
161
224
|
private playAction;
|
|
162
225
|
private ensureTurn;
|
|
163
226
|
private handleCues;
|
|
164
227
|
private activateNextTurn;
|
|
228
|
+
/** Put the turn's zero where its sound began — see the header. */
|
|
229
|
+
private anchor;
|
|
230
|
+
private anchored;
|
|
231
|
+
/** The lead for a turn starting now — see `VISUAL_LEAD_MS`. */
|
|
232
|
+
private leadMs;
|
|
233
|
+
/** The first resumption after `after` (track ms) that follows a pause long
|
|
234
|
+
* enough to be heard as one. */
|
|
235
|
+
private nextResumption;
|
|
236
|
+
/** Listen for the sound at the turn's next resumption, once. */
|
|
237
|
+
private scheduleReanchor;
|
|
238
|
+
private clearReanchorTimer;
|
|
239
|
+
private clearOnsetTimer;
|
|
240
|
+
private dropProbe;
|
|
241
|
+
/** Listen to the bot's audio track, if the transport has one yet. */
|
|
242
|
+
private listenTo;
|
|
165
243
|
private discardQueuedTurns;
|
|
166
244
|
private lifecycleState;
|
|
167
245
|
private applyProjection;
|
|
@@ -169,7 +247,7 @@ export declare class AvatarClient {
|
|
|
169
247
|
private armIdleIfEligible;
|
|
170
248
|
private eligibleForIdle;
|
|
171
249
|
private enterListening;
|
|
172
|
-
private
|
|
250
|
+
private clearServerStateForTurnBoundary;
|
|
173
251
|
private maybePlayInterrupted;
|
|
174
252
|
private clearRecoverableFailure;
|
|
175
253
|
private onUserStartedSpeaking;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AvatarClient.d.ts","sourceRoot":"","sources":["../client/AvatarClient.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"AvatarClient.d.ts","sourceRoot":"","sources":["../client/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAa,MAAM,uBAAuB,CAAC;AAEtE,OAAO,EAAsB,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,EAGL,KAAK,aAAa,EAClB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAgCpB;;;;;;;GAOG;AACH,eAAO,MAAM,cAAc,KAAK,CAAC;AA4BjC;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IAChC;;;;OAIG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE;QAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;IACpE,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IACrC,YAAY,IAAI,OAAO,CAAC;IACxB,eAAe,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC;CACpD;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC;wDACoD;IACpD,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC;IACrD,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB;mCAC+B;IAC/B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,UAAU,CAAC,EAAE,OAAO,UAAU,CAAC;IAC/B,YAAY,CAAC,EAAE,OAAO,YAAY,CAAC;IACnC;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;CACzD;AAED;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,WAAW;;;;;;;;;;;;;CAmBmB,CAAC;AAE5C,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAC3B,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,GAC3D,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAYlD,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2B;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,SAAS,CAAS;IAG1B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,SAAS,CAA8C;IAC/D,OAAO,CAAC,wBAAwB,CAAS;IACzC,OAAO,CAAC,8BAA8B,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoB;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;IACjD,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,UAAU,CAAiC;IACnD;6DACyD;IACzD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,UAAU,CAA8C;IAChE,OAAO,CAAC,aAAa,CAA8C;gBAEvD,MAAM,EAAE,YAAY,EAAE,IAAI,GAAE,mBAAwB;IAWhE,+EAA+E;IAC/E,IAAI,OAAO,IAAI,MAAM,GAAG,IAAI,CAE3B;IAED,wFAAwF;IACxF,IAAI,QAAQ,IAAI,SAAS,EAAE,CAE1B;IAED;;6EAEyE;IACzE,IAAI,aAAa,IAAI,mBAAmB,CAEvC;IAED;;oDAEgD;IAChD,iBAAiB,IAAI,IAAI;IAIzB;;0EAEsE;IACtE,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IA0B5B,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,YAAY;IAcpB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,UAAU;IAYlB,OAAO,CAAC,UAAU;IAqBlB,OAAO,CAAC,gBAAgB;IA+BxB,kEAAkE;IAClE,OAAO,CAAC,MAAM;IA+Bd,OAAO,CAAC,QAAQ;IAUhB,+DAA+D;IAC/D,OAAO,CAAC,MAAM;IAMd;oCACgC;IAChC,OAAO,CAAC,cAAc;IAatB,gEAAgE;IAChE,OAAO,CAAC,gBAAgB;IAgCxB,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,SAAS;IAOjB,qEAAqE;IACrE,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,eAAe;IAKvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,+BAA+B;IAMvC,OAAO,CAAC,oBAAoB;IAO5B,OAAO,CAAC,uBAAuB;IAO/B,OAAO,CAAC,qBAAqB,CAS3B;IAEF,OAAO,CAAC,qBAAqB,CAI3B;IAEF,OAAO,CAAC,oBAAoB,CAQ1B;IAEF,OAAO,CAAC,oBAAoB,CAkB1B;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,OAAO,CAIb;IAEF,OAAO,CAAC,cAAc,CAIpB;IAEF,OAAO,CAAC,kBAAkB,CASxB;IAEF;;;;;;;OAOG;IACH,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,IAAI;IA4CzC,4EAA4E;IAC5E,OAAO,IAAI,IAAI;CAOhB"}
|