@voqalize/avatar 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -661
- package/README.md +144 -663
- package/client/Avatar.tsx +57 -0
- package/client/AvatarClient.ts +523 -0
- package/client/arjun.ts +24 -0
- package/client/createAvatar.ts +139 -0
- package/client/createCanvasAvatar.ts +72 -0
- package/client/index.ts +39 -0
- package/client/internal.ts +80 -0
- package/client/interviewer-female.ts +4 -0
- package/client/interviewer-male.ts +4 -0
- package/client/ishita.ts +24 -0
- package/client/kabir.ts +24 -0
- package/client/meera.ts +24 -0
- package/client/naina.ts +24 -0
- package/client/professional-female-a.ts +4 -0
- package/client/professional-female-b.ts +4 -0
- package/client/professional-male-a.ts +4 -0
- package/client/professional-male-b.ts +4 -0
- package/client/react.ts +13 -0
- package/client/types.ts +146 -0
- package/client/useAvatar.ts +74 -0
- package/client/vikram.ts +24 -0
- package/dist/Avatar.d.ts +44 -0
- package/dist/Avatar.d.ts.map +1 -0
- package/{client/dist → dist}/Avatar.js +2 -2
- package/dist/Avatar.js.map +1 -0
- package/dist/AvatarClient.d.ts +196 -0
- package/dist/AvatarClient.d.ts.map +1 -0
- package/dist/AvatarClient.js +450 -0
- package/dist/AvatarClient.js.map +1 -0
- package/dist/arjun.d.ts +6 -0
- package/dist/arjun.d.ts.map +1 -0
- package/dist/arjun.js +18 -0
- package/dist/arjun.js.map +1 -0
- package/dist/createAvatar.d.ts +99 -0
- package/dist/createAvatar.d.ts.map +1 -0
- package/dist/createAvatar.js +75 -0
- package/dist/createAvatar.js.map +1 -0
- package/dist/createCanvasAvatar.d.ts +22 -0
- package/dist/createCanvasAvatar.d.ts.map +1 -0
- package/dist/createCanvasAvatar.js +47 -0
- package/dist/createCanvasAvatar.js.map +1 -0
- package/dist/index.d.ts +30 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +29 -0
- package/dist/index.js.map +1 -0
- package/dist/internal.d.ts +30 -0
- package/dist/internal.d.ts.map +1 -0
- package/dist/internal.js +33 -0
- package/dist/internal.js.map +1 -0
- package/dist/interviewer-female.d.ts +4 -0
- package/dist/interviewer-female.d.ts.map +1 -0
- package/dist/interviewer-female.js +3 -0
- package/dist/interviewer-female.js.map +1 -0
- package/dist/interviewer-male.d.ts +4 -0
- package/dist/interviewer-male.d.ts.map +1 -0
- package/dist/interviewer-male.js +3 -0
- package/dist/interviewer-male.js.map +1 -0
- package/dist/ishita.d.ts +6 -0
- package/dist/ishita.d.ts.map +1 -0
- package/dist/ishita.js +18 -0
- package/dist/ishita.js.map +1 -0
- package/dist/kabir.d.ts +6 -0
- package/dist/kabir.d.ts.map +1 -0
- package/dist/kabir.js +18 -0
- package/dist/kabir.js.map +1 -0
- package/dist/meera.d.ts +6 -0
- package/dist/meera.d.ts.map +1 -0
- package/dist/meera.js +18 -0
- package/dist/meera.js.map +1 -0
- package/dist/naina.d.ts +6 -0
- package/dist/naina.d.ts.map +1 -0
- package/dist/naina.js +18 -0
- package/dist/naina.js.map +1 -0
- package/dist/professional-female-a.d.ts +4 -0
- package/dist/professional-female-a.d.ts.map +1 -0
- package/dist/professional-female-a.js +3 -0
- package/dist/professional-female-a.js.map +1 -0
- package/dist/professional-female-b.d.ts +4 -0
- package/dist/professional-female-b.d.ts.map +1 -0
- package/dist/professional-female-b.js +3 -0
- package/dist/professional-female-b.js.map +1 -0
- package/dist/professional-male-a.d.ts +4 -0
- package/dist/professional-male-a.d.ts.map +1 -0
- package/dist/professional-male-a.js +3 -0
- package/dist/professional-male-a.js.map +1 -0
- package/dist/professional-male-b.d.ts +4 -0
- package/dist/professional-male-b.d.ts.map +1 -0
- package/dist/professional-male-b.js +3 -0
- package/dist/professional-male-b.js.map +1 -0
- package/dist/react.d.ts +13 -0
- package/dist/react.d.ts.map +1 -0
- package/dist/react.js +13 -0
- package/dist/react.js.map +1 -0
- package/dist/types.d.ts +94 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +93 -0
- package/dist/types.js.map +1 -0
- package/dist/useAvatar.d.ts +39 -0
- package/dist/useAvatar.d.ts.map +1 -0
- package/dist/useAvatar.js +35 -0
- package/dist/useAvatar.js.map +1 -0
- package/dist/vikram.d.ts +6 -0
- package/dist/vikram.d.ts.map +1 -0
- package/dist/vikram.js +18 -0
- package/dist/vikram.js.map +1 -0
- package/package.json +81 -24
- package/src/avatar.d.ts +161 -120
- package/src/avatar.js +234 -146
- package/src/behavior.d.ts +26 -0
- package/src/behavior.js +102 -0
- package/src/camera.js +29 -0
- package/src/canvas/author/parts/eye.mjs +722 -0
- package/src/canvas/author/parts/hand.mjs +1156 -0
- package/src/canvas/author/parts/mouth.mjs +741 -0
- package/src/canvas/author/parts/nose.mjs +100 -0
- package/src/canvas/author/parts/skin-detail.mjs +67 -0
- package/src/canvas/author/path.mjs +283 -0
- package/src/canvas/author/rig.mjs +405 -0
- package/src/canvas/avatars/round/face.d.mts +3 -0
- package/src/canvas/avatars/round/face.mjs +1307 -0
- package/src/canvas/create-rig.d.ts +15 -0
- package/src/canvas/create-rig.js +100 -0
- package/src/canvas/data/img/professional-female-a-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-female-a-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-female-a-top-body.webp +0 -0
- package/src/canvas/data/img/professional-female-b-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-female-b-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-female-b-top-body.webp +0 -0
- package/src/canvas/data/img/professional-male-a-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-male-a-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-male-a-top-body.webp +0 -0
- package/src/canvas/data/img/professional-male-b-hair-back.webp +0 -0
- package/src/canvas/data/img/professional-male-b-hair-front.webp +0 -0
- package/src/canvas/data/img/professional-male-b-top-body.webp +0 -0
- package/src/canvas/data/img/round-m3-hair-back.webp +0 -0
- package/src/canvas/data/img/round-m3-hair-front.webp +0 -0
- package/src/canvas/data/img/round-m3-top-body.webp +0 -0
- package/src/canvas/data/img/round-w1-hair-back.webp +0 -0
- package/src/canvas/data/img/round-w1-hair-front.webp +0 -0
- package/src/canvas/data/img/round-w1-top-body.webp +0 -0
- package/src/canvas/data/interviewer-female.rig.json +1 -0
- package/src/canvas/data/interviewer-male.rig.json +1 -0
- package/src/canvas/data/professional-female-a.rig.json +1 -0
- package/src/canvas/data/professional-female-b.rig.json +1 -0
- package/src/canvas/data/professional-male-a.rig.json +1 -0
- package/src/canvas/data/professional-male-b.rig.json +1 -0
- package/src/canvas/src/live.js +508 -0
- package/src/canvas/src/render2d.js +218 -0
- package/src/canvas/src/rig.js +297 -0
- package/src/canvas/src/vocab.js +96 -0
- package/src/clips.js +18 -7
- package/src/conformance.js +119 -0
- package/src/emotions.js +4 -3
- package/src/face-core.js +27 -1
- package/src/face-myna.d.ts +7 -0
- package/src/face-myna.js +192 -131
- package/src/face-peep-control-plane.js +167 -0
- package/src/face-peep.d.ts +7 -0
- package/src/face-peep.js +274 -116
- package/src/face-wren.d.ts +7 -0
- package/src/face-wren.js +17 -19
- package/src/faces.d.ts +12 -0
- package/src/faces.js +53 -0
- package/src/gaze.js +119 -4
- package/src/hand.js +120 -91
- package/src/idle.js +18 -101
- package/src/interjections.js +156 -31
- package/src/params.js +8 -4
- package/src/perform.js +7 -9
- package/src/rig.d.ts +24 -0
- package/src/rig.js +32 -0
- package/src/speech-timing.js +23 -0
- package/src/visemes.js +28 -12
- package/client/dist/Avatar.d.ts +0 -27
- package/client/dist/Avatar.d.ts.map +0 -1
- package/client/dist/Avatar.js.map +0 -1
- package/client/dist/AvatarClient.d.ts +0 -139
- package/client/dist/AvatarClient.d.ts.map +0 -1
- package/client/dist/AvatarClient.js +0 -223
- package/client/dist/AvatarClient.js.map +0 -1
- package/client/dist/index.d.ts +0 -22
- package/client/dist/index.d.ts.map +0 -1
- package/client/dist/index.js +0 -22
- package/client/dist/index.js.map +0 -1
- package/client/dist/types.d.ts +0 -86
- package/client/dist/types.d.ts.map +0 -1
- package/client/dist/types.js +0 -31
- package/client/dist/types.js.map +0 -1
- package/client/dist/useAvatar.d.ts +0 -42
- package/client/dist/useAvatar.d.ts.map +0 -1
- package/client/dist/useAvatar.js +0 -51
- package/client/dist/useAvatar.js.map +0 -1
- package/client/src/Avatar.tsx +0 -33
- package/client/src/AvatarClient.ts +0 -270
- package/client/src/index.ts +0 -22
- package/client/src/types.ts +0 -104
- package/client/src/useAvatar.ts +0 -85
- package/docs/contract-avatar.md +0 -371
- package/docs/contract-protocol.md +0 -461
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* useAvatar — mount an avatar into a ref'd element for as long as there is a
|
|
3
|
+
* client to embody, and tear it down after.
|
|
4
|
+
*
|
|
5
|
+
* Internal; `<Avatar>` is the only thing the React entry exports. The whole
|
|
6
|
+
* hook is one effect, because the factory takes the client at construction:
|
|
7
|
+
* there is no separate attach step to keep in its own lifecycle any more.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { useEffect, useRef } from "react";
|
|
11
|
+
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
12
|
+
import {
|
|
13
|
+
createAvatar as createSvgAvatar,
|
|
14
|
+
type AvatarFactory,
|
|
15
|
+
type AvatarOptions,
|
|
16
|
+
type SvgAvatarOptions,
|
|
17
|
+
} from "./createAvatar.js";
|
|
18
|
+
|
|
19
|
+
export interface UseAvatarOptions<O extends AvatarOptions = SvgAvatarOptions> {
|
|
20
|
+
/** The live `PipecatClient`, or `null` before connect. Nothing mounts until
|
|
21
|
+
* this is non-null — an avatar with nothing to embody has nothing to do. */
|
|
22
|
+
client?: PipecatClient | null;
|
|
23
|
+
/** The avatar implementation. Defaults to the bundled SVG faces. */
|
|
24
|
+
create?: AvatarFactory<O>;
|
|
25
|
+
/** Implementation options, forwarded verbatim. Read at mount only. */
|
|
26
|
+
options?: Omit<O, keyof AvatarOptions>;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* The mount ref's type, written out rather than named as React's `RefObject`.
|
|
31
|
+
*
|
|
32
|
+
* React 18 and 19 declare that alias with different type arguments — 18's
|
|
33
|
+
* `useRef<T>(null)` yields `RefObject<T>`, 19's yields `RefObject<T | null>` —
|
|
34
|
+
* and because both are the *same alias*, TypeScript compares them by variance
|
|
35
|
+
* and rejects whichever one we didn't pick. An anonymous shape forces a
|
|
36
|
+
* structural comparison instead, which both versions satisfy, and which the
|
|
37
|
+
* `ref` prop accepts on both. This is the only place the 18-vs-19 split shows
|
|
38
|
+
* up in the binding; keep it that way.
|
|
39
|
+
*/
|
|
40
|
+
export type AvatarMountRef = { current: HTMLDivElement | null };
|
|
41
|
+
|
|
42
|
+
export interface UseAvatarHandle {
|
|
43
|
+
/** Attach to the mount element: `<div ref={containerRef} />`. */
|
|
44
|
+
containerRef: AvatarMountRef;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function useAvatar<O extends AvatarOptions = SvgAvatarOptions>(
|
|
48
|
+
{ client, create, options }: UseAvatarOptions<O> = {},
|
|
49
|
+
): UseAvatarHandle {
|
|
50
|
+
const containerRef = useRef<HTMLDivElement>(null);
|
|
51
|
+
|
|
52
|
+
// Latest-options ref, so the mount effect reads the live values without
|
|
53
|
+
// remounting the face every time a caller passes a fresh object literal.
|
|
54
|
+
const optionsRef = useRef(options);
|
|
55
|
+
optionsRef.current = options;
|
|
56
|
+
|
|
57
|
+
useEffect(() => {
|
|
58
|
+
const mount = containerRef.current;
|
|
59
|
+
if (!mount || !client) return;
|
|
60
|
+
const factory = (create ?? createSvgAvatar) as AvatarFactory<O>;
|
|
61
|
+
// The cast is the seam between "O minus the two we supply" and O. It is
|
|
62
|
+
// sound by construction and TypeScript cannot see through the spread of a
|
|
63
|
+
// generic; the two halves are typed at the boundary the caller touches.
|
|
64
|
+
const instance = factory({ mount, client, ...optionsRef.current } as unknown as O);
|
|
65
|
+
return () => instance.destroy();
|
|
66
|
+
// A new client identity is a new thing to embody, so the avatar is rebuilt
|
|
67
|
+
// rather than re-pointed. Hosts keep one `PipecatClient` across
|
|
68
|
+
// connect/disconnect cycles, so this does not fire on an ordinary
|
|
69
|
+
// reconnect; if yours constructs a fresh client per session, expect the
|
|
70
|
+
// face to remount with it.
|
|
71
|
+
}, [client, create]);
|
|
72
|
+
|
|
73
|
+
return { containerRef };
|
|
74
|
+
}
|
package/client/vikram.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Polished professional male avatar, authored to read at call-tile size. */
|
|
2
|
+
|
|
3
|
+
import { createCanvasAvatar } from './createCanvasAvatar.js';
|
|
4
|
+
import type { CanvasAvatarOptions } from './createCanvasAvatar.js';
|
|
5
|
+
import type { AvatarInstance } from './createAvatar.js';
|
|
6
|
+
|
|
7
|
+
const RIG_URL = new URL('../src/canvas/data/professional-male-a.rig.json', import.meta.url);
|
|
8
|
+
const IMAGES = {
|
|
9
|
+
'professional-male-a-top-body.webp': new URL('../src/canvas/data/img/professional-male-a-top-body.webp', import.meta.url),
|
|
10
|
+
'professional-male-a-hair-back.webp': new URL('../src/canvas/data/img/professional-male-a-hair-back.webp', import.meta.url),
|
|
11
|
+
'professional-male-a-hair-front.webp': new URL('../src/canvas/data/img/professional-male-a-hair-front.webp', import.meta.url),
|
|
12
|
+
} as const;
|
|
13
|
+
const FACE = () => import('../src/canvas/avatars/round/face.mjs');
|
|
14
|
+
|
|
15
|
+
export type { CanvasAvatarOptions };
|
|
16
|
+
|
|
17
|
+
export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
|
|
18
|
+
return createCanvasAvatar(options, {
|
|
19
|
+
rigUrl: RIG_URL,
|
|
20
|
+
images: IMAGES,
|
|
21
|
+
face: FACE,
|
|
22
|
+
label: 'Vikram — polished professional male avatar',
|
|
23
|
+
});
|
|
24
|
+
}
|
package/dist/Avatar.d.ts
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Avatar — `createAvatar` as a call tile, and the whole of `@voqalize/avatar/react`.
|
|
3
|
+
*
|
|
4
|
+
* <Avatar client={session.client} className="avatar-tile" />
|
|
5
|
+
*
|
|
6
|
+
* Three props of its own; everything else is forwarded to the mount `<div>`, so
|
|
7
|
+
* it sizes and styles like the tile it lives in. There is nothing to configure
|
|
8
|
+
* because there is nothing the server does not already say: the
|
|
9
|
+
* `AvatarProcessor` in the pipeline drives the state, the gaze and the mouth.
|
|
10
|
+
*
|
|
11
|
+
* `create` is how you use a different avatar — any module exporting
|
|
12
|
+
* `createAvatar` (docs/design-avatar-interface.md), including a Rive or WebGL
|
|
13
|
+
* one. The component is generic over that implementation's options, so
|
|
14
|
+
* `options` is checked against the factory you passed rather than being a bag:
|
|
15
|
+
*
|
|
16
|
+
* <Avatar client={c} create={createMascot} options={{ mood: "sly" }} />
|
|
17
|
+
*
|
|
18
|
+
* With no `create` it is the bundled SVG avatar and `options` is
|
|
19
|
+
* `SvgAvatarOptions` minus `mount`/`client`, which the component supplies.
|
|
20
|
+
*
|
|
21
|
+
* The component renders an empty div and a static `role="img"`. It does not
|
|
22
|
+
* label the avatar with its current state: the implementation owns the DOM
|
|
23
|
+
* inside the mount and is the only thing that knows what it is portraying —
|
|
24
|
+
* and a live label would be this package reading back an avatar's internal
|
|
25
|
+
* state, which is the one thing the interface refuses to promise. Pass your
|
|
26
|
+
* own `aria-label` to override.
|
|
27
|
+
*/
|
|
28
|
+
import type { HTMLAttributes } from "react";
|
|
29
|
+
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
30
|
+
import type { AvatarFactory, AvatarOptions, SvgAvatarOptions } from "./createAvatar.js";
|
|
31
|
+
/** An implementation's own options: everything it takes past the two the
|
|
32
|
+
* component supplies itself. */
|
|
33
|
+
export type ImplementationOptions<O extends AvatarOptions> = Omit<O, keyof AvatarOptions>;
|
|
34
|
+
export type AvatarProps<O extends AvatarOptions = SvgAvatarOptions> = Omit<HTMLAttributes<HTMLDivElement>, "children"> & {
|
|
35
|
+
/** The live `PipecatClient`, or `null` before connect. Nothing renders
|
|
36
|
+
* until it is non-null. */
|
|
37
|
+
client?: PipecatClient | null;
|
|
38
|
+
/** An avatar implementation. Omit for the bundled SVG faces. */
|
|
39
|
+
create?: AvatarFactory<O>;
|
|
40
|
+
/** Options for that implementation. Read at mount only. */
|
|
41
|
+
options?: ImplementationOptions<O>;
|
|
42
|
+
};
|
|
43
|
+
export declare function Avatar<O extends AvatarOptions = SvgAvatarOptions>({ client, create, options, ...rest }: AvatarProps<O>): import("react").JSX.Element;
|
|
44
|
+
//# sourceMappingURL=Avatar.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Avatar.d.ts","sourceRoot":"","sources":["../client/Avatar.tsx"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,OAAO,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAExF;gCACgC;AAChC,MAAM,MAAM,qBAAqB,CAAC,CAAC,SAAS,aAAa,IAAI,IAAI,CAAC,CAAC,EAAE,MAAM,aAAa,CAAC,CAAC;AAE1F,MAAM,MAAM,WAAW,CAAC,CAAC,SAAS,aAAa,GAAG,gBAAgB,IAChE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,EAAE,UAAU,CAAC,GAAG;IACjD;+BAC2B;IAC3B,MAAM,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,gEAAgE;IAChE,MAAM,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;IAC1B,2DAA2D;IAC3D,OAAO,CAAC,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC;CACpC,CAAC;AAEJ,wBAAgB,MAAM,CAAC,CAAC,SAAS,aAAa,GAAG,gBAAgB,EAAE,EACjE,MAAM,EACN,MAAM,EACN,OAAO,EACP,GAAG,IAAI,EACR,EAAE,WAAW,CAAC,CAAC,CAAC,+BAGhB"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { jsx as _jsx } from "react/jsx-runtime";
|
|
2
2
|
import { useAvatar } from "./useAvatar.js";
|
|
3
|
-
export function Avatar({ client,
|
|
4
|
-
const { containerRef } = useAvatar({ client,
|
|
3
|
+
export function Avatar({ client, create, options, ...rest }) {
|
|
4
|
+
const { containerRef } = useAvatar({ client, create, options });
|
|
5
5
|
return _jsx("div", { role: "img", "aria-label": "avatar", ...rest, ref: containerRef });
|
|
6
6
|
}
|
|
7
7
|
//# sourceMappingURL=Avatar.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Avatar.js","sourceRoot":"","sources":["../client/Avatar.tsx"],"names":[],"mappings":";AA8BA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAkB3C,MAAM,UAAU,MAAM,CAA6C,EACjE,MAAM,EACN,MAAM,EACN,OAAO,EACP,GAAG,IAAI,EACQ;IACf,MAAM,EAAE,YAAY,EAAE,GAAG,SAAS,CAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACnE,OAAO,cAAK,IAAI,EAAC,KAAK,gBAAY,QAAQ,KAAK,IAAI,EAAE,GAAG,EAAE,YAAY,GAAI,CAAC;AAC7E,CAAC"}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AvatarClient — the avatar's server-message dispatcher, turn clock, and cue
|
|
3
|
+
* splice, framework-free (no React; the hook and component wrap this).
|
|
4
|
+
*
|
|
5
|
+
* ## Turn clock anchoring
|
|
6
|
+
*
|
|
7
|
+
* Base Pipecat TTS gives each serialized TTS context an opaque `context_id`.
|
|
8
|
+
* The server uses it only to group and splice cue chunks. `botStartedSpeaking`
|
|
9
|
+
* has no context payload, so the browser FIFO-claims the next buffered context
|
|
10
|
+
* at that Pipecat output-lifecycle event and anchors its clock there.
|
|
11
|
+
* `botStoppedSpeaking` closes the active context. Neither event observes the
|
|
12
|
+
* browser's audio device, and no avatar-specific speech marker exists.
|
|
13
|
+
*
|
|
14
|
+
* `attach()` subscribes to the avatar server-message channel *and* Pipecat's
|
|
15
|
+
* standard lifecycle events. Server messages carry only what Pipecat cannot:
|
|
16
|
+
* correlated speech/cue timing and deliberate application instructions. The
|
|
17
|
+
* lifecycle events project the factual presence states locally. The server
|
|
18
|
+
* supplies only lower-priority `THINKING` / `WORKING` claims and deliberate,
|
|
19
|
+
* self-completing actions. This keeps the face tied to Pipecat's output truth
|
|
20
|
+
* even if a server claim is delayed or stale.
|
|
21
|
+
*
|
|
22
|
+
* ## Cue splice
|
|
23
|
+
*
|
|
24
|
+
* The widget has two cue-track primitives: `speak({cues, clock})` (a full
|
|
25
|
+
* replace) and `pushCues(cues)` (a pure union that can only grow the track,
|
|
26
|
+
* never shrink it). Neither is "discard queued cues at or after `from_ms`,
|
|
27
|
+
* then append" on its own — `pushCues` has no way to drop a stale tail. So
|
|
28
|
+
* this client keeps the turn's canonical cue array itself (kept portion +
|
|
29
|
+
* every appended chunk, spliced on each `cues` message) and picks the cheapest
|
|
30
|
+
* widget call that stays correct:
|
|
31
|
+
*
|
|
32
|
+
* - if the splice's `from_ms` doesn't reach back into anything already
|
|
33
|
+
* queued — the common case past a turn's first sentence, since only the
|
|
34
|
+
* first sentence genuinely plays fast-leg cues — nothing needs discarding:
|
|
35
|
+
* `pushCues(newCues)` is the cheap, correct append.
|
|
36
|
+
* - if it does reach back (a real fast→accurate splice), `pushCues` cannot
|
|
37
|
+
* express the discard; we call `speak()` again with the full spliced
|
|
38
|
+
* canonical array on the turn's original clock. `speak()` is otherwise
|
|
39
|
+
* documented as also killing an in-flight spoken interjection and
|
|
40
|
+
* re-entering `SPEAKING` — both harmless mid-splice (an interjection
|
|
41
|
+
* should not be running while a server track owns the mouth; re-entering
|
|
42
|
+
* an unchanged state is a no-op past the profile/gaze reset the widget
|
|
43
|
+
* already does for a same-name `setState`).
|
|
44
|
+
*
|
|
45
|
+
* Cues commonly arrive **before** `botStartedSpeaking` — the fast leg starts
|
|
46
|
+
* the moment a sentence is handed to TTS, well before bot output begins.
|
|
47
|
+
* Chunks that arrive before the clock is anchored are spliced into the
|
|
48
|
+
* canonical array but not yet handed to the widget; `botStartedSpeaking` hands over
|
|
49
|
+
* whatever has accumulated as the turn's first `speak()` call. So "the first
|
|
50
|
+
* chunk of a turn starts speak()" means the first *widget* call, not
|
|
51
|
+
* necessarily the first *message*.
|
|
52
|
+
*/
|
|
53
|
+
import type { PipecatClient } from "@pipecat-ai/client-js";
|
|
54
|
+
import type { AvatarApi } from "../src/avatar.js";
|
|
55
|
+
import { type AvatarCommand, type AvatarCue } from "./types.js";
|
|
56
|
+
/**
|
|
57
|
+
* Internal. Not exported from the package — the public surface is
|
|
58
|
+
* `createAvatar({ mount, client })` and nothing else.
|
|
59
|
+
*
|
|
60
|
+
* There is deliberately no `accept` predicate here any more. Avatar commands
|
|
61
|
+
* travel in one envelope, `{type:"avatar"}`, in both directions and from every
|
|
62
|
+
* source: a `AvatarProcessor` in the pipeline and a brain driving the face
|
|
63
|
+
* out of band emit the same shape. A per-deployment predicate meant the
|
|
64
|
+
* library could not state what an avatar message *is*, which is the one thing
|
|
65
|
+
* a wire format has to be able to say.
|
|
66
|
+
*/
|
|
67
|
+
export interface AvatarClientOptions {
|
|
68
|
+
/** A dispatch threw (e.g. an unknown state or interjection id, which the
|
|
69
|
+
* widget throws on). Defaults to `console.warn`. */
|
|
70
|
+
onError?: (err: unknown, msg: AvatarCommand) => void;
|
|
71
|
+
/** Override for tests. Defaults to `performance.now`. */
|
|
72
|
+
now?: () => number;
|
|
73
|
+
/** Quiet time in listening before the client-owned idle loop begins. */
|
|
74
|
+
idleDelayMs?: number;
|
|
75
|
+
/** Timer seams keep lifecycle behavior deterministic in tests. */
|
|
76
|
+
setTimeout?: typeof setTimeout;
|
|
77
|
+
clearTimeout?: typeof clearTimeout;
|
|
78
|
+
/**
|
|
79
|
+
* Internal, for Studio's inspector — the resolved projection, so a developer
|
|
80
|
+
* tool can show what the lifecycle decided. It is *not* on the public
|
|
81
|
+
* surface and must not become one: a presence callback is a contract, and
|
|
82
|
+
* publishing it would oblige every avatar implementation to emit these seven
|
|
83
|
+
* states with this precedence, which is exactly the second public contract
|
|
84
|
+
* the design exists to avoid.
|
|
85
|
+
*/
|
|
86
|
+
onPresenceChange?: (state: AvatarPresenceState) => void;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* The one `RTVIEvent` member `attach()` subscribes to, spelled as its value.
|
|
90
|
+
*
|
|
91
|
+
* Written out rather than imported because that enum was this module's *only*
|
|
92
|
+
* runtime reference to `@pipecat-ai/client-js`, and one runtime reference makes
|
|
93
|
+
* the whole `/pipecat` subpath fail to load without the peer installed — even
|
|
94
|
+
* for a host that drives `dispatch()` from its own transport and never calls
|
|
95
|
+
* `attach()`. The peer is declared optional; this is what makes that true
|
|
96
|
+
* rather than aspirational.
|
|
97
|
+
*
|
|
98
|
+
* String enums are nominal in TypeScript, so the compiler cannot check these
|
|
99
|
+
* against the real ones from a type-only import. `packages/avatar/test/AvatarClient.test.ts`
|
|
100
|
+
* does it instead, against the actual enum — the devDependency is present
|
|
101
|
+
* exactly where the check belongs and absent from what we ship.
|
|
102
|
+
*/
|
|
103
|
+
export declare const RTVI_EVENTS: {
|
|
104
|
+
readonly serverMessage: "serverMessage";
|
|
105
|
+
readonly connected: "connected";
|
|
106
|
+
readonly disconnected: "disconnected";
|
|
107
|
+
readonly botReady: "botReady";
|
|
108
|
+
readonly error: "error";
|
|
109
|
+
readonly userStartedSpeaking: "userStartedSpeaking";
|
|
110
|
+
readonly userStoppedSpeaking: "userStoppedSpeaking";
|
|
111
|
+
readonly botStartedSpeaking: "botStartedSpeaking";
|
|
112
|
+
readonly botStoppedSpeaking: "botStoppedSpeaking";
|
|
113
|
+
readonly userMuteStarted: "userMuteStarted";
|
|
114
|
+
readonly userMuteStopped: "userMuteStopped";
|
|
115
|
+
};
|
|
116
|
+
/** The resolved, factual presence state a host may render around the avatar. */
|
|
117
|
+
export type AvatarPresenceState = "IDLE" | "LISTENING" | "STRAINING" | "THINKING" | "WORKING" | "MUTED" | "SPEAKING" | "DEGRADED" | "OFFLINE";
|
|
118
|
+
export declare class AvatarClient {
|
|
119
|
+
private readonly avatar;
|
|
120
|
+
/** Maps factual/wire intent into the broader client behavior catalog. */
|
|
121
|
+
private readonly behavior;
|
|
122
|
+
private readonly opts;
|
|
123
|
+
private readonly now;
|
|
124
|
+
private turn;
|
|
125
|
+
private readonly turns;
|
|
126
|
+
private readonly pendingCtxs;
|
|
127
|
+
private readonly closedCtxs;
|
|
128
|
+
private projected;
|
|
129
|
+
private serverClaim;
|
|
130
|
+
private userSpeaking;
|
|
131
|
+
private botSpeaking;
|
|
132
|
+
private muted;
|
|
133
|
+
private listening;
|
|
134
|
+
private idle;
|
|
135
|
+
private failure;
|
|
136
|
+
private idleTimer;
|
|
137
|
+
private pendingInterruptedAction;
|
|
138
|
+
private discardQueuedContextsOnBotStop;
|
|
139
|
+
private readonly idleDelayMs;
|
|
140
|
+
private readonly setTimer;
|
|
141
|
+
private readonly clearTimer;
|
|
142
|
+
constructor(avatar: AvatarApi, opts?: AvatarClientOptions);
|
|
143
|
+
/** The active turn's ctx, or `null` between turns. For tests and telemetry. */
|
|
144
|
+
get turnCtx(): string | null;
|
|
145
|
+
/** The active turn's canonical (already-spliced) cue track. For tests and telemetry. */
|
|
146
|
+
get turnCues(): AvatarCue[];
|
|
147
|
+
/** Current resolved projection. Internal — for tests and Studio's inspector.
|
|
148
|
+
* Before the first lifecycle fact the renderer is simply at its ready rest
|
|
149
|
+
* pose, so `LISTENING` is the safe value; it is not a session status. */
|
|
150
|
+
get presenceState(): AvatarPresenceState;
|
|
151
|
+
/** Re-apply the currently resolved factual projection after an embedding
|
|
152
|
+
* tool has temporarily used raw renderer controls. This does not invent a
|
|
153
|
+
* lifecycle event or cancel a finite action. */
|
|
154
|
+
restoreProjection(): void;
|
|
155
|
+
/** Dispatch one server message. Anything that isn't in the avatar envelope
|
|
156
|
+
* is not ours and is ignored; so is an envelope carrying a `cmd` this build
|
|
157
|
+
* has never heard of, per the wire protocol's forward-compat rule. */
|
|
158
|
+
dispatch(raw: unknown): void;
|
|
159
|
+
private handleClaim;
|
|
160
|
+
private handleAction;
|
|
161
|
+
private playAction;
|
|
162
|
+
private ensureTurn;
|
|
163
|
+
private handleCues;
|
|
164
|
+
private activateNextTurn;
|
|
165
|
+
private discardQueuedTurns;
|
|
166
|
+
private lifecycleState;
|
|
167
|
+
private applyProjection;
|
|
168
|
+
private clearIdleTimer;
|
|
169
|
+
private armIdleIfEligible;
|
|
170
|
+
private eligibleForIdle;
|
|
171
|
+
private enterListening;
|
|
172
|
+
private clearClaimForTurnBoundary;
|
|
173
|
+
private maybePlayInterrupted;
|
|
174
|
+
private clearRecoverableFailure;
|
|
175
|
+
private onUserStartedSpeaking;
|
|
176
|
+
private onUserStoppedSpeaking;
|
|
177
|
+
private onBotStartedSpeaking;
|
|
178
|
+
private onBotStoppedSpeaking;
|
|
179
|
+
private onUserMuteStarted;
|
|
180
|
+
private onUserMuteStopped;
|
|
181
|
+
private onError;
|
|
182
|
+
private onDisconnected;
|
|
183
|
+
private onConnectedOrReady;
|
|
184
|
+
/**
|
|
185
|
+
* Subscribe to a live `PipecatClient`. Standard client events own the normal
|
|
186
|
+
* lifecycle projection; avatar server-messages carry correlated visemes and
|
|
187
|
+
* explicit application intent. Never throws on malformed or irrelevant
|
|
188
|
+
* server messages.
|
|
189
|
+
*
|
|
190
|
+
* @returns an unsubscribe function; call it on unmount or disconnect.
|
|
191
|
+
*/
|
|
192
|
+
attach(client: PipecatClient): () => void;
|
|
193
|
+
/** Dispose controller-owned timers when its mounted avatar is destroyed. */
|
|
194
|
+
destroy(): void;
|
|
195
|
+
}
|
|
196
|
+
//# sourceMappingURL=AvatarClient.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AvatarClient.d.ts","sourceRoot":"","sources":["../client/AvatarClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAa,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAkB,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,EAGL,KAAK,aAAa,EAClB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAYpB;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC;wDACoD;IACpD,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC;IACrD,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,UAAU,CAAC,EAAE,OAAO,UAAU,CAAC;IAC/B,YAAY,CAAC,EAAE,OAAO,YAAY,CAAC;IACnC;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,CAAC;CACzD;AAED;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,WAAW;;;;;;;;;;;;CAgBmB,CAAC;AAE5C,gFAAgF;AAChF,MAAM,MAAM,mBAAmB,GAC3B,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,GAC3D,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;AAYlD,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2B;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,SAAS,CAAS;IAG1B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,SAAS,CAA8C;IAC/D,OAAO,CAAC,wBAAwB,CAAS;IACzC,OAAO,CAAC,8BAA8B,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoB;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;gBAErC,MAAM,EAAE,SAAS,EAAE,IAAI,GAAE,mBAAwB;IAU7D,+EAA+E;IAC/E,IAAI,OAAO,IAAI,MAAM,GAAG,IAAI,CAE3B;IAED,wFAAwF;IACxF,IAAI,QAAQ,IAAI,SAAS,EAAE,CAE1B;IAED;;6EAEyE;IACzE,IAAI,aAAa,IAAI,mBAAmB,CAEvC;IAED;;oDAEgD;IAChD,iBAAiB,IAAI,IAAI;IAIzB;;0EAEsE;IACtE,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IA0B5B,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,YAAY;IAcpB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,UAAU;IAmBlB,OAAO,CAAC,gBAAgB;IAcxB,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,eAAe;IAKvB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,yBAAyB;IAMjC,OAAO,CAAC,oBAAoB;IAO5B,OAAO,CAAC,uBAAuB;IAO/B,OAAO,CAAC,qBAAqB,CAS3B;IAEF,OAAO,CAAC,qBAAqB,CAI3B;IAEF,OAAO,CAAC,oBAAoB,CAQ1B;IAEF,OAAO,CAAC,oBAAoB,CAgB1B;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF,OAAO,CAAC,OAAO,CAIb;IAEF,OAAO,CAAC,cAAc,CAIpB;IAEF,OAAO,CAAC,kBAAkB,CASxB;IAEF;;;;;;;OAOG;IACH,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,IAAI;IAsBzC,4EAA4E;IAC5E,OAAO,IAAI,IAAI;CAIhB"}
|