@voqalize/avatar 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -661
- package/README.md +144 -663
- package/client/Avatar.tsx +57 -0
- package/client/AvatarClient.ts +523 -0
- package/client/arjun.ts +24 -0
- package/client/createAvatar.ts +139 -0
- package/client/createCanvasAvatar.ts +72 -0
- package/client/index.ts +39 -0
- package/client/internal.ts +80 -0
- package/client/interviewer-female.ts +4 -0
- package/client/interviewer-male.ts +4 -0
- package/client/ishita.ts +24 -0
- package/client/kabir.ts +24 -0
- package/client/meera.ts +24 -0
- package/client/naina.ts +24 -0
- package/client/professional-female-a.ts +4 -0
- package/client/professional-female-b.ts +4 -0
- package/client/professional-male-a.ts +4 -0
- package/client/professional-male-b.ts +4 -0
- package/client/react.ts +13 -0
- package/client/types.ts +146 -0
- package/client/useAvatar.ts +74 -0
- package/client/vikram.ts +24 -0
- package/dist/Avatar.d.ts +44 -0
- package/dist/Avatar.d.ts.map +1 -0
- package/{client/dist → dist}/Avatar.js +2 -2
- package/dist/Avatar.js.map +1 -0
- package/dist/AvatarClient.d.ts +196 -0
- package/dist/AvatarClient.d.ts.map +1 -0
- package/dist/AvatarClient.js +450 -0
- package/dist/AvatarClient.js.map +1 -0
- package/dist/arjun.d.ts +6 -0
- package/dist/arjun.d.ts.map +1 -0
- package/dist/arjun.js +18 -0
- package/dist/arjun.js.map +1 -0
- package/dist/createAvatar.d.ts +99 -0
- package/dist/createAvatar.d.ts.map +1 -0
- package/dist/createAvatar.js +75 -0
- package/dist/createAvatar.js.map +1 -0
- package/dist/createCanvasAvatar.d.ts +22 -0
- package/dist/createCanvasAvatar.d.ts.map +1 -0
- package/dist/createCanvasAvatar.js +47 -0
- package/dist/createCanvasAvatar.js.map +1 -0
- package/dist/index.d.ts +30 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +29 -0
- package/dist/index.js.map +1 -0
- package/dist/internal.d.ts +30 -0
- package/dist/internal.d.ts.map +1 -0
- package/dist/internal.js +33 -0
- package/dist/internal.js.map +1 -0
- package/dist/interviewer-female.d.ts +4 -0
- package/dist/interviewer-female.d.ts.map +1 -0
- package/dist/interviewer-female.js +3 -0
- package/dist/interviewer-female.js.map +1 -0
- package/dist/interviewer-male.d.ts +4 -0
- package/dist/interviewer-male.d.ts.map +1 -0
- package/dist/interviewer-male.js +3 -0
- package/dist/interviewer-male.js.map +1 -0
- package/dist/ishita.d.ts +6 -0
- package/dist/ishita.d.ts.map +1 -0
- package/dist/ishita.js +18 -0
- package/dist/ishita.js.map +1 -0
- package/dist/kabir.d.ts +6 -0
- package/dist/kabir.d.ts.map +1 -0
- package/dist/kabir.js +18 -0
- package/dist/kabir.js.map +1 -0
- package/dist/meera.d.ts +6 -0
- package/dist/meera.d.ts.map +1 -0
- package/dist/meera.js +18 -0
- package/dist/meera.js.map +1 -0
- package/dist/naina.d.ts +6 -0
- package/dist/naina.d.ts.map +1 -0
- package/dist/naina.js +18 -0
- package/dist/naina.js.map +1 -0
- package/dist/professional-female-a.d.ts +4 -0
- package/dist/professional-female-a.d.ts.map +1 -0
- package/dist/professional-female-a.js +3 -0
- package/dist/professional-female-a.js.map +1 -0
- package/dist/professional-female-b.d.ts +4 -0
- package/dist/professional-female-b.d.ts.map +1 -0
- package/dist/professional-female-b.js +3 -0
- package/dist/professional-female-b.js.map +1 -0
- package/dist/professional-male-a.d.ts +4 -0
- package/dist/professional-male-a.d.ts.map +1 -0
- package/dist/professional-male-a.js +3 -0
- package/dist/professional-male-a.js.map +1 -0
- package/dist/professional-male-b.d.ts +4 -0
- package/dist/professional-male-b.d.ts.map +1 -0
- package/dist/professional-male-b.js +3 -0
- package/dist/professional-male-b.js.map +1 -0
- package/dist/react.d.ts +13 -0
- package/dist/react.d.ts.map +1 -0
- package/dist/react.js +13 -0
- package/dist/react.js.map +1 -0
- package/dist/types.d.ts +94 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +93 -0
- package/dist/types.js.map +1 -0
- package/dist/useAvatar.d.ts +39 -0
- package/dist/useAvatar.d.ts.map +1 -0
- package/dist/useAvatar.js +35 -0
- package/dist/useAvatar.js.map +1 -0
- package/dist/vikram.d.ts +6 -0
- package/dist/vikram.d.ts.map +1 -0
- package/dist/vikram.js +18 -0
- package/dist/vikram.js.map +1 -0
- package/package.json +81 -24
- package/src/avatar.d.ts +161 -120
- package/src/avatar.js +234 -146
- package/src/behavior.d.ts +26 -0
- package/src/behavior.js +102 -0
- package/src/camera.js +29 -0
- package/src/canvas/author/parts/eye.mjs +722 -0
- package/src/canvas/author/parts/hand.mjs +1156 -0
- package/src/canvas/author/parts/mouth.mjs +741 -0
- package/src/canvas/author/parts/nose.mjs +100 -0
- package/src/canvas/author/parts/skin-detail.mjs +67 -0
- package/src/canvas/author/path.mjs +283 -0
- package/src/canvas/author/rig.mjs +405 -0
- package/src/canvas/avatars/round/face.d.mts +3 -0
- package/src/canvas/avatars/round/face.mjs +1307 -0
- package/src/canvas/create-rig.d.ts +15 -0
- package/src/canvas/create-rig.js +100 -0
- package/src/canvas/data/img/professional-female-a-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-female-a-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-female-a-top-body.webp +0 -0
- package/src/canvas/data/img/professional-female-b-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-female-b-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-female-b-top-body.webp +0 -0
- package/src/canvas/data/img/professional-male-a-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-male-a-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-male-a-top-body.webp +0 -0
- package/src/canvas/data/img/professional-male-b-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-male-b-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-male-b-top-body.webp +0 -0
- package/src/canvas/data/img/round-m3-hair-back.webp +0 -0
- package/src/canvas/data/img/round-m3-hair-front.webp +0 -0
- package/src/canvas/data/img/round-m3-top-body.webp +0 -0
- package/src/canvas/data/img/round-w1-hair-back.webp +0 -0
- package/src/canvas/data/img/round-w1-hair-front.webp +0 -0
- package/src/canvas/data/img/round-w1-top-body.webp +0 -0
- package/src/canvas/data/interviewer-female.rig.json +1 -0
- package/src/canvas/data/interviewer-male.rig.json +1 -0
- package/src/canvas/data/professional-female-a.rig.json +1 -0
- package/src/canvas/data/professional-female-b.rig.json +1 -0
- package/src/canvas/data/professional-male-a.rig.json +1 -0
- package/src/canvas/data/professional-male-b.rig.json +1 -0
- package/src/canvas/src/live.js +508 -0
- package/src/canvas/src/render2d.js +218 -0
- package/src/canvas/src/rig.js +297 -0
- package/src/canvas/src/vocab.js +96 -0
- package/src/clips.js +18 -7
- package/src/conformance.js +119 -0
- package/src/emotions.js +4 -3
- package/src/face-core.js +27 -1
- package/src/face-myna.d.ts +7 -0
- package/src/face-myna.js +192 -131
- package/src/face-peep-control-plane.js +167 -0
- package/src/face-peep.d.ts +7 -0
- package/src/face-peep.js +274 -116
- package/src/face-wren.d.ts +7 -0
- package/src/face-wren.js +17 -19
- package/src/faces.d.ts +12 -0
- package/src/faces.js +53 -0
- package/src/gaze.js +119 -4
- package/src/hand.js +120 -91
- package/src/idle.js +18 -101
- package/src/interjections.js +156 -31
- package/src/params.js +8 -4
- package/src/perform.js +7 -9
- package/src/rig.d.ts +24 -0
- package/src/rig.js +32 -0
- package/src/speech-timing.js +23 -0
- package/src/visemes.js +28 -12
- package/client/dist/Avatar.d.ts +0 -27
- package/client/dist/Avatar.d.ts.map +0 -1
- package/client/dist/Avatar.js.map +0 -1
- package/client/dist/AvatarClient.d.ts +0 -139
- package/client/dist/AvatarClient.d.ts.map +0 -1
- package/client/dist/AvatarClient.js +0 -223
- package/client/dist/AvatarClient.js.map +0 -1
- package/client/dist/index.d.ts +0 -22
- package/client/dist/index.d.ts.map +0 -1
- package/client/dist/index.js +0 -22
- package/client/dist/index.js.map +0 -1
- package/client/dist/types.d.ts +0 -86
- package/client/dist/types.d.ts.map +0 -1
- package/client/dist/types.js +0 -31
- package/client/dist/types.js.map +0 -1
- package/client/dist/useAvatar.d.ts +0 -42
- package/client/dist/useAvatar.d.ts.map +0 -1
- package/client/dist/useAvatar.js +0 -51
- package/client/dist/useAvatar.js.map +0 -1
- package/client/src/Avatar.tsx +0 -33
- package/client/src/AvatarClient.ts +0 -270
- package/client/src/index.ts +0 -22
- package/client/src/types.ts +0 -104
- package/client/src/useAvatar.ts +0 -85
- package/docs/contract-avatar.md +0 -371
- package/docs/contract-protocol.md +0 -461
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Avatar — `createAvatar` as a call tile, and the whole of `@voqalize/avatar/react`.
|
|
3
|
+
*
|
|
4
|
+
* <Avatar client={session.client} className="avatar-tile" />
|
|
5
|
+
*
|
|
6
|
+
* Three props of its own; everything else is forwarded to the mount `<div>`, so
|
|
7
|
+
* it sizes and styles like the tile it lives in. There is nothing to configure
|
|
8
|
+
* because there is nothing the server does not already say: the
|
|
9
|
+
* `AvatarProcessor` in the pipeline drives the state, the gaze and the mouth.
|
|
10
|
+
*
|
|
11
|
+
* `create` is how you use a different avatar — any module exporting
|
|
12
|
+
* `createAvatar` (docs/design-avatar-interface.md), including a Rive or WebGL
|
|
13
|
+
* one. The component is generic over that implementation's options, so
|
|
14
|
+
* `options` is checked against the factory you passed rather than being a bag:
|
|
15
|
+
*
|
|
16
|
+
* <Avatar client={c} create={createMascot} options={{ mood: "sly" }} />
|
|
17
|
+
*
|
|
18
|
+
* With no `create` it is the bundled SVG avatar and `options` is
|
|
19
|
+
* `SvgAvatarOptions` minus `mount`/`client`, which the component supplies.
|
|
20
|
+
*
|
|
21
|
+
* The component renders an empty div and a static `role="img"`. It does not
|
|
22
|
+
* label the avatar with its current state: the implementation owns the DOM
|
|
23
|
+
* inside the mount and is the only thing that knows what it is portraying —
|
|
24
|
+
* and a live label would be this package reading back an avatar's internal
|
|
25
|
+
* state, which is the one thing the interface refuses to promise. Pass your
|
|
26
|
+
* own `aria-label` to override.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import type { HTMLAttributes } from "react";
|
|
30
|
+
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
31
|
+
import { useAvatar } from "./useAvatar.js";
|
|
32
|
+
import type { AvatarFactory, AvatarOptions, SvgAvatarOptions } from "./createAvatar.js";
|
|
33
|
+
|
|
34
|
+
/** An implementation's own options: everything it takes past the two the
|
|
35
|
+
* component supplies itself. */
|
|
36
|
+
export type ImplementationOptions<O extends AvatarOptions> = Omit<O, keyof AvatarOptions>;
|
|
37
|
+
|
|
38
|
+
export type AvatarProps<O extends AvatarOptions = SvgAvatarOptions> =
|
|
39
|
+
Omit<HTMLAttributes<HTMLDivElement>, "children"> & {
|
|
40
|
+
/** The live `PipecatClient`, or `null` before connect. Nothing renders
|
|
41
|
+
* until it is non-null. */
|
|
42
|
+
client?: PipecatClient | null;
|
|
43
|
+
/** An avatar implementation. Omit for the bundled SVG faces. */
|
|
44
|
+
create?: AvatarFactory<O>;
|
|
45
|
+
/** Options for that implementation. Read at mount only. */
|
|
46
|
+
options?: ImplementationOptions<O>;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
export function Avatar<O extends AvatarOptions = SvgAvatarOptions>({
|
|
50
|
+
client,
|
|
51
|
+
create,
|
|
52
|
+
options,
|
|
53
|
+
...rest
|
|
54
|
+
}: AvatarProps<O>) {
|
|
55
|
+
const { containerRef } = useAvatar<O>({ client, create, options });
|
|
56
|
+
return <div role="img" aria-label="avatar" {...rest} ref={containerRef} />;
|
|
57
|
+
}
|
|
@@ -0,0 +1,523 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
|
|
3
|
+
* splice, framework-free (no React; the hook and component wrap this).
|
|
4
|
+
*
|
|
5
|
+
* ## Turn clock anchoring
|
|
6
|
+
*
|
|
7
|
+
* Base Pipecat TTS gives each serialized TTS context an opaque `context_id`.
|
|
8
|
+
* The server uses it only to group and splice cue chunks. `botStartedSpeaking`
|
|
9
|
+
* has no context payload, so the browser FIFO-claims the next buffered context
|
|
10
|
+
* at that Pipecat output-lifecycle event and anchors its clock there.
|
|
11
|
+
* `botStoppedSpeaking` closes the active context. Neither event observes the
|
|
12
|
+
* browser's audio device, and no avatar-specific speech marker exists.
|
|
13
|
+
*
|
|
14
|
+
* `attach()` subscribes to the avatar server-message channel *and* Pipecat's
|
|
15
|
+
* standard lifecycle events. Server messages carry only what Pipecat cannot:
|
|
16
|
+
* correlated speech/cue timing and deliberate application instructions. The
|
|
17
|
+
* lifecycle events project the factual presence states locally. The server
|
|
18
|
+
* supplies only lower-priority `THINKING` / `WORKING` claims and deliberate,
|
|
19
|
+
* self-completing actions. This keeps the face tied to Pipecat's output truth
|
|
20
|
+
* even if a server claim is delayed or stale.
|
|
21
|
+
*
|
|
22
|
+
* ## Cue splice
|
|
23
|
+
*
|
|
24
|
+
* The widget has two cue-track primitives: `speak({cues, clock})` (a full
|
|
25
|
+
* replace) and `pushCues(cues)` (a pure union that can only grow the track,
|
|
26
|
+
* never shrink it). Neither is "discard queued cues at or after `from_ms`,
|
|
27
|
+
* then append" on its own — `pushCues` has no way to drop a stale tail. So
|
|
28
|
+
* this client keeps the turn's canonical cue array itself (kept portion +
|
|
29
|
+
* every appended chunk, spliced on each `cues` message) and picks the cheapest
|
|
30
|
+
* widget call that stays correct:
|
|
31
|
+
*
|
|
32
|
+
* - if the splice's `from_ms` doesn't reach back into anything already
|
|
33
|
+
* queued — the common case past a turn's first sentence, since only the
|
|
34
|
+
* first sentence genuinely plays fast-leg cues — nothing needs discarding:
|
|
35
|
+
* `pushCues(newCues)` is the cheap, correct append.
|
|
36
|
+
* - if it does reach back (a real fast→accurate splice), `pushCues` cannot
|
|
37
|
+
* express the discard; we call `speak()` again with the full spliced
|
|
38
|
+
* canonical array on the turn's original clock. `speak()` is otherwise
|
|
39
|
+
* documented as also killing an in-flight spoken interjection and
|
|
40
|
+
* re-entering `SPEAKING` — both harmless mid-splice (an interjection
|
|
41
|
+
* should not be running while a server track owns the mouth; re-entering
|
|
42
|
+
* an unchanged state is a no-op past the profile/gaze reset the widget
|
|
43
|
+
* already does for a same-name `setState`).
|
|
44
|
+
*
|
|
45
|
+
* Cues commonly arrive **before** `botStartedSpeaking` — the fast leg starts
|
|
46
|
+
* the moment a sentence is handed to TTS, well before bot output begins.
|
|
47
|
+
* Chunks that arrive before the clock is anchored are spliced into the
|
|
48
|
+
* canonical array but not yet handed to the widget; `botStartedSpeaking` hands over
|
|
49
|
+
* whatever has accumulated as the turn's first `speak()` call. So "the first
|
|
50
|
+
* chunk of a turn starts speak()" means the first *widget* call, not
|
|
51
|
+
* necessarily the first *message*.
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
import type { PipecatClient, RTVIEvent } from "@pipecat-ai/client-js";
|
|
55
|
+
import type { AvatarActionId, AvatarApi } from "../src/avatar.js";
|
|
56
|
+
import { BehaviorController } from "../src/behavior.js";
|
|
57
|
+
import {
|
|
58
|
+
isAvatarMessage,
|
|
59
|
+
parseAvatarCommand,
|
|
60
|
+
type AvatarCommand,
|
|
61
|
+
type AvatarCue,
|
|
62
|
+
type AvatarCuesCmd,
|
|
63
|
+
} from "./types.js";
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
interface Turn {
|
|
67
|
+
ctx: string;
|
|
68
|
+
/** The canonical, already-spliced cue track for this turn. */
|
|
69
|
+
cues: AvatarCue[];
|
|
70
|
+
/** Whether Pipecat output has anchored a clock and issued `speak()`. */
|
|
71
|
+
started: boolean;
|
|
72
|
+
clock: (() => number) | null;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Internal. Not exported from the package — the public surface is
|
|
77
|
+
* `createAvatar({ mount, client })` and nothing else.
|
|
78
|
+
*
|
|
79
|
+
* There is deliberately no `accept` predicate here any more. Avatar commands
|
|
80
|
+
* travel in one envelope, `{type:"avatar"}`, in both directions and from every
|
|
81
|
+
* source: a `AvatarProcessor` in the pipeline and a brain driving the face
|
|
82
|
+
* out of band emit the same shape. A per-deployment predicate meant the
|
|
83
|
+
* library could not state what an avatar message *is*, which is the one thing
|
|
84
|
+
* a wire format has to be able to say.
|
|
85
|
+
*/
|
|
86
|
+
export interface AvatarClientOptions {
|
|
87
|
+
/** A dispatch threw (e.g. an unknown state or interjection id, which the
|
|
88
|
+
* widget throws on). Defaults to `console.warn`. */
|
|
89
|
+
onError?: (err: unknown, msg: AvatarCommand) => void;
|
|
90
|
+
/** Override for tests. Defaults to `performance.now`. */
|
|
91
|
+
now?: () => number;
|
|
92
|
+
/** Quiet time in listening before the client-owned idle loop begins. */
|
|
93
|
+
idleDelayMs?: number;
|
|
94
|
+
/** Timer seams keep lifecycle behavior deterministic in tests. */
|
|
95
|
+
setTimeout?: typeof setTimeout;
|
|
96
|
+
clearTimeout?: typeof clearTimeout;
|
|
97
|
+
/**
|
|
98
|
+
* Internal, for Studio's inspector — the resolved projection, so a developer
|
|
99
|
+
* tool can show what the lifecycle decided. It is *not* on the public
|
|
100
|
+
* surface and must not become one: a presence callback is a contract, and
|
|
101
|
+
* publishing it would oblige every avatar implementation to emit these seven
|
|
102
|
+
* states with this precedence, which is exactly the second public contract
|
|
103
|
+
* the design exists to avoid.
|
|
104
|
+
*/
|
|
105
|
+
onPresenceChange?: (state: AvatarPresenceState) => void;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
|
|
110
|
+
*
|
|
111
|
+
* Written out rather than imported because that enum was this module's *only*
|
|
112
|
+
* runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
|
|
113
|
+
* the whole `/pipecat` subpath fail to load without the peer installed — even
|
|
114
|
+
* for a host that drives `dispatch()` from its own transport and never calls
|
|
115
|
+
* `attach()`. The peer is declared optional; this is what makes that true
|
|
116
|
+
* rather than aspirational.
|
|
117
|
+
*
|
|
118
|
+
* String enums are nominal in TypeScript, so the compiler cannot check these
|
|
119
|
+
* against the real ones from a type-only import. `packages/avatar/test/AvatarClient.test.ts`
|
|
120
|
+
* does it instead, against the actual enum — the devDependency is present
|
|
121
|
+
* exactly where the check belongs and absent from what we ship.
|
|
122
|
+
*/
|
|
123
|
+
export const RTVI_EVENTS = {
|
|
124
|
+
serverMessage: "serverMessage",
|
|
125
|
+
connected: "connected",
|
|
126
|
+
disconnected: "disconnected",
|
|
127
|
+
botReady: "botReady",
|
|
128
|
+
error: "error",
|
|
129
|
+
userStartedSpeaking: "userStartedSpeaking",
|
|
130
|
+
userStoppedSpeaking: "userStoppedSpeaking",
|
|
131
|
+
botStartedSpeaking: "botStartedSpeaking",
|
|
132
|
+
botStoppedSpeaking: "botStoppedSpeaking",
|
|
133
|
+
// Mute is a Pipecat fact, not a claim: the server's mute strategy emits
|
|
134
|
+
// `UserMuteStarted/StoppedFrame`, the RTVI observer forwards them, and the
|
|
135
|
+
// browser client raises these. So "has muted you" costs no wire verb —
|
|
136
|
+
// reading the events the peer already sends is exactly the authority model.
|
|
137
|
+
userMuteStarted: "userMuteStarted",
|
|
138
|
+
userMuteStopped: "userMuteStopped",
|
|
139
|
+
} as const satisfies Record<string, string>;
|
|
140
|
+
|
|
141
|
+
/** The resolved, factual presence state a host may render around the avatar. */
|
|
142
|
+
export type AvatarPresenceState =
|
|
143
|
+
| "IDLE" | "LISTENING" | "STRAINING" | "THINKING" | "WORKING"
|
|
144
|
+
| "MUTED" | "SPEAKING" | "DEGRADED" | "OFFLINE";
|
|
145
|
+
type LifecycleState = AvatarPresenceState;
|
|
146
|
+
type ServerClaim = "STRAINING" | "THINKING" | "WORKING" | null;
|
|
147
|
+
|
|
148
|
+
/** Defensive unwrap for the `RTVIEvent.ServerMessage` `{ data }` quirk: some
|
|
149
|
+
* transports deliver the payload directly and some wrap it once more. */
|
|
150
|
+
function unwrapServerMessage(raw: unknown): Record<string, unknown> {
|
|
151
|
+
const obj = (raw ?? {}) as Record<string, unknown>;
|
|
152
|
+
const inner = obj["data"] as Record<string, unknown> | undefined;
|
|
153
|
+
return inner && "type" in inner ? inner : obj;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export class AvatarClient {
|
|
157
|
+
private readonly avatar: AvatarApi;
|
|
158
|
+
/** Maps factual/wire intent into the broader client behavior catalog. */
|
|
159
|
+
private readonly behavior: BehaviorController;
|
|
160
|
+
private readonly opts: AvatarClientOptions;
|
|
161
|
+
private readonly now: () => number;
|
|
162
|
+
private turn: Turn | null = null;
|
|
163
|
+
private readonly turns = new Map<string, Turn>();
|
|
164
|
+
private readonly pendingCtxs: string[] = [];
|
|
165
|
+
private readonly closedCtxs = new Set<string>();
|
|
166
|
+
private projected: LifecycleState | null = null;
|
|
167
|
+
private serverClaim: ServerClaim = null;
|
|
168
|
+
private userSpeaking = false;
|
|
169
|
+
private botSpeaking = false;
|
|
170
|
+
private muted = false;
|
|
171
|
+
private listening = false;
|
|
172
|
+
// Idle is earned after a connected, quiet listening interval. A newly
|
|
173
|
+
// mounted avatar is available, not already "stepped aside".
|
|
174
|
+
private idle = false;
|
|
175
|
+
private failure: "DEGRADED" | "OFFLINE" | null = null;
|
|
176
|
+
private idleTimer: ReturnType<typeof setTimeout> | null = null;
|
|
177
|
+
private pendingInterruptedAction = false;
|
|
178
|
+
private discardQueuedContextsOnBotStop = false;
|
|
179
|
+
private readonly idleDelayMs: number;
|
|
180
|
+
private readonly setTimer: typeof setTimeout;
|
|
181
|
+
private readonly clearTimer: typeof clearTimeout;
|
|
182
|
+
|
|
183
|
+
constructor(avatar: AvatarApi, opts: AvatarClientOptions = {}) {
|
|
184
|
+
this.avatar = avatar;
|
|
185
|
+
this.behavior = new BehaviorController(avatar);
|
|
186
|
+
this.opts = opts;
|
|
187
|
+
this.now = opts.now ?? (() => performance.now());
|
|
188
|
+
this.idleDelayMs = opts.idleDelayMs ?? 12_000;
|
|
189
|
+
this.setTimer = opts.setTimeout ?? globalThis.setTimeout.bind(globalThis);
|
|
190
|
+
this.clearTimer = opts.clearTimeout ?? globalThis.clearTimeout.bind(globalThis);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** The active turn's ctx, or `null` between turns. For tests and telemetry. */
|
|
194
|
+
get turnCtx(): string | null {
|
|
195
|
+
return this.turn?.ctx ?? null;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
|
|
199
|
+
get turnCues(): AvatarCue[] {
|
|
200
|
+
return this.turn ? [...this.turn.cues] : [];
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Current resolved projection. Internal — for tests and Studio's inspector.
|
|
204
|
+
* Before the first lifecycle fact the renderer is simply at its ready rest
|
|
205
|
+
* pose, so `LISTENING` is the safe value; it is not a session status. */
|
|
206
|
+
get presenceState(): AvatarPresenceState {
|
|
207
|
+
return this.projected ?? "LISTENING";
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Re-apply the currently resolved factual projection after an embedding
|
|
211
|
+
* tool has temporarily used raw renderer controls. This does not invent a
|
|
212
|
+
* lifecycle event or cancel a finite action. */
|
|
213
|
+
restoreProjection(): void {
|
|
214
|
+
this.applyProjection(true);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Dispatch one server message. Anything that isn't in the avatar envelope
|
|
218
|
+
* is not ours and is ignored; so is an envelope carrying a `cmd` this build
|
|
219
|
+
* has never heard of, per the wire protocol's forward-compat rule. */
|
|
220
|
+
dispatch(raw: unknown): void {
|
|
221
|
+
if (!isAvatarMessage(raw)) return;
|
|
222
|
+
const msg = parseAvatarCommand(raw);
|
|
223
|
+
if (!msg) return;
|
|
224
|
+
try {
|
|
225
|
+
switch (msg.cmd) {
|
|
226
|
+
case "claim":
|
|
227
|
+
this.handleClaim(msg.state);
|
|
228
|
+
break;
|
|
229
|
+
case "action":
|
|
230
|
+
this.handleAction(msg.id);
|
|
231
|
+
break;
|
|
232
|
+
case "cues":
|
|
233
|
+
this.handleCues(msg);
|
|
234
|
+
break;
|
|
235
|
+
// No default: an unknown `cmd` is a newer server talking to an older
|
|
236
|
+
// widget, and the protocol's forward-compat rule says ignore it. There
|
|
237
|
+
// is no callback for it — a hook nobody could act on is observability,
|
|
238
|
+
// not an interface.
|
|
239
|
+
}
|
|
240
|
+
} catch (err) {
|
|
241
|
+
if (this.opts.onError) this.opts.onError(err, msg);
|
|
242
|
+
else console.warn("[avatar] dispatch failed", msg, err);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
private handleClaim(state: ServerClaim): void {
|
|
247
|
+
this.serverClaim = state;
|
|
248
|
+
if (state) this.idle = false;
|
|
249
|
+
this.applyProjection();
|
|
250
|
+
this.armIdleIfEligible();
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
private handleAction(id: AvatarActionId): void {
|
|
254
|
+
// An interruption is a server-confirmed explanation of a transition, not
|
|
255
|
+
// authority to steal the mouth while bot audio is still playing. Hold it
|
|
256
|
+
// until Pipecat output has released the speaking state.
|
|
257
|
+
if (id === "RESPONSE_INTERRUPTED" && this.botSpeaking) {
|
|
258
|
+
this.pendingInterruptedAction = true;
|
|
259
|
+
// Any prefetched TTS contexts behind interrupted output belong to
|
|
260
|
+
// audio Pipecat will now discard. Never let one animate a later reply.
|
|
261
|
+
this.discardQueuedContextsOnBotStop = true;
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
this.playAction(id);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
private playAction(id: AvatarActionId): void {
|
|
268
|
+
this.behavior.wireAction(id);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
private ensureTurn(ctx: string): Turn {
|
|
272
|
+
const existing = this.turns.get(ctx);
|
|
273
|
+
if (existing) return existing;
|
|
274
|
+
const turn = { ctx, cues: [], started: false, clock: null };
|
|
275
|
+
this.turns.set(ctx, turn);
|
|
276
|
+
this.pendingCtxs.push(ctx);
|
|
277
|
+
return turn;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
private handleCues(msg: AvatarCuesCmd) {
|
|
281
|
+
if (this.closedCtxs.has(msg.ctx)) return;
|
|
282
|
+
const turn = this.ensureTurn(msg.ctx);
|
|
283
|
+
const kept = turn.cues.filter((c) => c.t < msg.from_ms);
|
|
284
|
+
const discarded = turn.cues.length - kept.length;
|
|
285
|
+
turn.cues = [...kept, ...msg.cues].sort((a, b) => a.t - b.t);
|
|
286
|
+
|
|
287
|
+
if (!turn.started) {
|
|
288
|
+
// No clock yet — buffer. Pipecat output will claim this FIFO context.
|
|
289
|
+
if (this.botSpeaking && this.turn === null) this.activateNextTurn();
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
if (discarded === 0) {
|
|
293
|
+
this.avatar.pushCues(msg.cues);
|
|
294
|
+
} else {
|
|
295
|
+
this.avatar.speak({ cues: turn.cues, clock: turn.clock! });
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
private activateNextTurn(): void {
|
|
300
|
+
if (this.turn) return;
|
|
301
|
+
let turn: Turn | undefined;
|
|
302
|
+
while (this.pendingCtxs.length && !turn) {
|
|
303
|
+
turn = this.turns.get(this.pendingCtxs.shift()!);
|
|
304
|
+
}
|
|
305
|
+
if (!turn) return;
|
|
306
|
+
const t0 = this.now();
|
|
307
|
+
turn.clock = () => this.now() - t0;
|
|
308
|
+
turn.started = true;
|
|
309
|
+
this.turn = turn;
|
|
310
|
+
this.avatar.speak({ cues: turn.cues, clock: turn.clock });
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
private discardQueuedTurns(): void {
|
|
314
|
+
for (const ctx of this.pendingCtxs.splice(0)) {
|
|
315
|
+
this.closedCtxs.add(ctx);
|
|
316
|
+
this.turns.delete(ctx);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
private lifecycleState(): LifecycleState {
|
|
321
|
+
// The one copy of the ladder is docs/pipecat-lifecycle-protocol.md; this is
|
|
322
|
+
// its implementation, in the same order.
|
|
323
|
+
//
|
|
324
|
+
// Audio truth is the P0 invariant: no lower claim or microphone event may
|
|
325
|
+
// put the face in a non-speaking pose while bot speech is audible.
|
|
326
|
+
if (this.botSpeaking) return "SPEAKING";
|
|
327
|
+
if (this.userSpeaking) return "LISTENING";
|
|
328
|
+
// A broken session outranks mute, because a mute nobody can hear about is
|
|
329
|
+
// not the thing worth saying about a dead call.
|
|
330
|
+
if (this.failure) return this.failure;
|
|
331
|
+
if (this.muted) return "MUTED";
|
|
332
|
+
// Every claim resolves to the presence state of the same name, so there is
|
|
333
|
+
// nothing to rank here — a claim is a single value and only one can be in
|
|
334
|
+
// flight. STRAINING > THINKING > WORKING is decided where more than one
|
|
335
|
+
// condition can hold at once, which is the server: `AvatarStateMachine.
|
|
336
|
+
// _resolve()`. Ranking them again on this side would be a second, silent
|
|
337
|
+
// copy of that ladder, and the two would drift.
|
|
338
|
+
if (this.serverClaim) return this.serverClaim;
|
|
339
|
+
if (this.idle) return "IDLE";
|
|
340
|
+
return "LISTENING";
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
private applyProjection(force = false): void {
|
|
344
|
+
const state = this.lifecycleState();
|
|
345
|
+
if (force || this.projected !== state) {
|
|
346
|
+
this.behavior.setState(state);
|
|
347
|
+
this.projected = state;
|
|
348
|
+
this.opts.onPresenceChange?.(state);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
private clearIdleTimer(): void {
|
|
353
|
+
if (this.idleTimer !== null) {
|
|
354
|
+
this.clearTimer(this.idleTimer);
|
|
355
|
+
this.idleTimer = null;
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
private armIdleIfEligible(): void {
|
|
360
|
+
this.clearIdleTimer();
|
|
361
|
+
// Quiet under a mute is not the quiet that earns IDLE — the silence was
|
|
362
|
+
// imposed, and letting the timer run behind it would reveal a stepped-aside
|
|
363
|
+
// face the moment the microphone came back.
|
|
364
|
+
if (!this.eligibleForIdle()) return;
|
|
365
|
+
this.idleTimer = this.setTimer(() => {
|
|
366
|
+
this.idleTimer = null;
|
|
367
|
+
if (this.eligibleForIdle()) {
|
|
368
|
+
this.idle = true;
|
|
369
|
+
this.applyProjection();
|
|
370
|
+
}
|
|
371
|
+
}, this.idleDelayMs);
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
private eligibleForIdle(): boolean {
|
|
375
|
+
return this.listening && !this.userSpeaking && !this.botSpeaking
|
|
376
|
+
&& !this.muted && !this.serverClaim && !this.failure;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
private enterListening(): void {
|
|
380
|
+
this.listening = true;
|
|
381
|
+
this.idle = false;
|
|
382
|
+
this.applyProjection();
|
|
383
|
+
this.armIdleIfEligible();
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
private clearClaimForTurnBoundary(): void {
|
|
387
|
+
// Claims are lower-priority hints. A fresh user turn or bot output means
|
|
388
|
+
// any prior thinking/work claim is no longer allowed to reappear later.
|
|
389
|
+
this.serverClaim = null;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
private maybePlayInterrupted(): void {
|
|
393
|
+
if (this.pendingInterruptedAction && !this.botSpeaking) {
|
|
394
|
+
this.pendingInterruptedAction = false;
|
|
395
|
+
this.playAction("RESPONSE_INTERRUPTED");
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
private clearRecoverableFailure(): void {
|
|
400
|
+
if (this.failure === "DEGRADED") {
|
|
401
|
+
this.failure = null;
|
|
402
|
+
this.applyProjection();
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
private onUserStartedSpeaking = (): void => {
|
|
407
|
+
this.clearRecoverableFailure();
|
|
408
|
+
this.userSpeaking = true;
|
|
409
|
+
this.clearClaimForTurnBoundary();
|
|
410
|
+
this.listening = true;
|
|
411
|
+
this.idle = false;
|
|
412
|
+
this.clearIdleTimer();
|
|
413
|
+
this.avatar.setUserSpeaking(true);
|
|
414
|
+
this.applyProjection();
|
|
415
|
+
};
|
|
416
|
+
|
|
417
|
+
private onUserStoppedSpeaking = (): void => {
|
|
418
|
+
this.userSpeaking = false;
|
|
419
|
+
this.avatar.setUserSpeaking(false);
|
|
420
|
+
this.enterListening();
|
|
421
|
+
};
|
|
422
|
+
|
|
423
|
+
private onBotStartedSpeaking = (): void => {
|
|
424
|
+
this.clearRecoverableFailure();
|
|
425
|
+
this.botSpeaking = true;
|
|
426
|
+
this.clearClaimForTurnBoundary();
|
|
427
|
+
this.idle = false;
|
|
428
|
+
this.clearIdleTimer();
|
|
429
|
+
this.applyProjection();
|
|
430
|
+
this.activateNextTurn();
|
|
431
|
+
};
|
|
432
|
+
|
|
433
|
+
private onBotStoppedSpeaking = (): void => {
|
|
434
|
+
this.botSpeaking = false;
|
|
435
|
+
// Playout truth releases the only active mouth track. A late cue chunk for
|
|
436
|
+
// this context is ignored rather than reviving a silent mouth.
|
|
437
|
+
if (this.turn) {
|
|
438
|
+
this.avatar.stopSpeaking();
|
|
439
|
+
this.closedCtxs.add(this.turn.ctx);
|
|
440
|
+
this.turns.delete(this.turn.ctx);
|
|
441
|
+
this.turn = null;
|
|
442
|
+
}
|
|
443
|
+
if (this.discardQueuedContextsOnBotStop) {
|
|
444
|
+
this.discardQueuedContextsOnBotStop = false;
|
|
445
|
+
this.discardQueuedTurns();
|
|
446
|
+
}
|
|
447
|
+
this.enterListening();
|
|
448
|
+
this.maybePlayInterrupted();
|
|
449
|
+
};
|
|
450
|
+
|
|
451
|
+
private onUserMuteStarted = (): void => {
|
|
452
|
+
this.muted = true;
|
|
453
|
+
this.idle = false;
|
|
454
|
+
this.clearIdleTimer();
|
|
455
|
+
this.applyProjection();
|
|
456
|
+
};
|
|
457
|
+
|
|
458
|
+
private onUserMuteStopped = (): void => {
|
|
459
|
+
this.muted = false;
|
|
460
|
+
// Whatever the avatar was waiting on before the mute, it is waiting on the
|
|
461
|
+
// user again now — the same place a turn ends.
|
|
462
|
+
this.enterListening();
|
|
463
|
+
};
|
|
464
|
+
|
|
465
|
+
private onError = (raw: unknown): void => {
|
|
466
|
+
const data = (raw as { data?: unknown })?.data as { fatal?: unknown } | undefined;
|
|
467
|
+
this.failure = data?.fatal === true ? "OFFLINE" : "DEGRADED";
|
|
468
|
+
this.applyProjection();
|
|
469
|
+
};
|
|
470
|
+
|
|
471
|
+
private onDisconnected = (): void => {
|
|
472
|
+
this.failure = "OFFLINE";
|
|
473
|
+
this.clearIdleTimer();
|
|
474
|
+
this.applyProjection();
|
|
475
|
+
};
|
|
476
|
+
|
|
477
|
+
private onConnectedOrReady = (): void => {
|
|
478
|
+
if (this.failure === "OFFLINE") this.failure = null;
|
|
479
|
+
// Pipecat has established a session. Until a factual speech event says
|
|
480
|
+
// otherwise, the avatar is available to listen; it earns IDLE only after
|
|
481
|
+
// the regular quiet timer expires.
|
|
482
|
+
this.listening = true;
|
|
483
|
+
this.idle = false;
|
|
484
|
+
this.applyProjection();
|
|
485
|
+
this.armIdleIfEligible();
|
|
486
|
+
};
|
|
487
|
+
|
|
488
|
+
/**
|
|
489
|
+
* Subscribe to a live `PipecatClient`. Standard client events own the normal
|
|
490
|
+
* lifecycle projection; avatar server-messages carry correlated visemes and
|
|
491
|
+
* explicit application intent. Never throws on malformed or irrelevant
|
|
492
|
+
* server messages.
|
|
493
|
+
*
|
|
494
|
+
* @returns an unsubscribe function; call it on unmount or disconnect.
|
|
495
|
+
*/
|
|
496
|
+
attach(client: PipecatClient): () => void {
|
|
497
|
+
const onServerMessage = (raw: unknown) => this.dispatch(unwrapServerMessage(raw));
|
|
498
|
+
const subscriptions: Array<[string, (...args: any[]) => void]> = [
|
|
499
|
+
[RTVI_EVENTS.serverMessage, onServerMessage],
|
|
500
|
+
[RTVI_EVENTS.connected, this.onConnectedOrReady],
|
|
501
|
+
[RTVI_EVENTS.botReady, this.onConnectedOrReady],
|
|
502
|
+
[RTVI_EVENTS.disconnected, this.onDisconnected],
|
|
503
|
+
[RTVI_EVENTS.error, this.onError],
|
|
504
|
+
[RTVI_EVENTS.userStartedSpeaking, this.onUserStartedSpeaking],
|
|
505
|
+
[RTVI_EVENTS.userStoppedSpeaking, this.onUserStoppedSpeaking],
|
|
506
|
+
[RTVI_EVENTS.botStartedSpeaking, this.onBotStartedSpeaking],
|
|
507
|
+
[RTVI_EVENTS.botStoppedSpeaking, this.onBotStoppedSpeaking],
|
|
508
|
+
[RTVI_EVENTS.userMuteStarted, this.onUserMuteStarted],
|
|
509
|
+
[RTVI_EVENTS.userMuteStopped, this.onUserMuteStopped],
|
|
510
|
+
];
|
|
511
|
+
for (const [event, listener] of subscriptions) client.on(event as RTVIEvent, listener as never);
|
|
512
|
+
return () => {
|
|
513
|
+
this.clearIdleTimer();
|
|
514
|
+
for (const [event, listener] of subscriptions) client.off(event as RTVIEvent, listener as never);
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/** Dispose controller-owned timers when its mounted avatar is destroyed. */
|
|
519
|
+
destroy(): void {
|
|
520
|
+
this.clearIdleTimer();
|
|
521
|
+
this.behavior.destroy();
|
|
522
|
+
}
|
|
523
|
+
}
|
package/client/arjun.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Professional Indian male interviewer, authored to read at call-tile size. */
|
|
2
|
+
|
|
3
|
+
import { createCanvasAvatar } from './createCanvasAvatar.js';
|
|
4
|
+
import type { CanvasAvatarOptions } from './createCanvasAvatar.js';
|
|
5
|
+
import type { AvatarInstance } from './createAvatar.js';
|
|
6
|
+
|
|
7
|
+
const RIG_URL = new URL('../src/canvas/data/interviewer-male.rig.json', import.meta.url);
|
|
8
|
+
const IMAGES = {
|
|
9
|
+
'round-m3-hair-back.webp': new URL('../src/canvas/data/img/round-m3-hair-back.webp', import.meta.url),
|
|
10
|
+
'round-m3-top-body.webp': new URL('../src/canvas/data/img/round-m3-top-body.webp', import.meta.url),
|
|
11
|
+
'round-m3-hair-front.webp': new URL('../src/canvas/data/img/round-m3-hair-front.webp', import.meta.url),
|
|
12
|
+
} as const;
|
|
13
|
+
const FACE = () => import('../src/canvas/avatars/round/face.mjs');
|
|
14
|
+
|
|
15
|
+
export type { CanvasAvatarOptions };
|
|
16
|
+
|
|
17
|
+
export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
|
|
18
|
+
return createCanvasAvatar(options, {
|
|
19
|
+
rigUrl: RIG_URL,
|
|
20
|
+
images: IMAGES,
|
|
21
|
+
face: FACE,
|
|
22
|
+
label: 'Arjun — professional male interviewer avatar',
|
|
23
|
+
});
|
|
24
|
+
}
|