@convai/embody 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +93 -0
- package/README.md +114 -30
- package/dist/character/createCharacter.d.ts +108 -3
- package/dist/character/createCharacter.js +118 -7
- package/dist/connect/session.d.ts +51 -0
- package/dist/connect/session.js +62 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.js +23 -0
- package/dist/motion/breathe.d.ts +1 -1
- package/dist/motion/breathe.js +1 -1
- package/dist/react/CharacterModel.d.ts +8 -0
- package/dist/react/CharacterModel.js +39 -0
- package/dist/react/ConvaiCharacter.d.ts +20 -6
- package/dist/react/ConvaiCharacter.js +37 -22
- package/dist/react/index.d.ts +35 -1
- package/dist/react/index.js +33 -1
- package/dist/react/useCharacter.d.ts +13 -2
- package/dist/react/useCharacter.js +17 -8
- package/dist/react/useWidgetClient.d.ts +18 -0
- package/dist/react/useWidgetClient.js +105 -0
- package/dist/resolve/resolver.d.ts +14 -1
- package/dist/resolve/resolver.js +33 -3
- package/package.json +8 -4
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,98 @@
|
|
|
1
1
|
# @convai/embody
|
|
2
2
|
|
|
3
|
+
## 0.2.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Add a `default` condition to the package exports map.
|
|
8
|
+
|
|
9
|
+
The map previously declared only `types` and `import`, so any resolver using
|
|
10
|
+
CommonJS semantics could not resolve the package at all — Jest's default
|
|
11
|
+
resolver failed to resolve `@convai/embody/react` even to mock it, and
|
|
12
|
+
Next.js's `next/dynamic` build transform (which resolves its target via
|
|
13
|
+
`require.resolveWeak`) failed with "Package path './react' is not exported".
|
|
14
|
+
The first real consumer needed three separate workarounds for this one
|
|
15
|
+
omission. `default` points at the same ESM files; resolvers that only need
|
|
16
|
+
resolution now succeed, and runtimes with `require(esm)` can load them.
|
|
17
|
+
|
|
18
|
+
## 0.2.0
|
|
19
|
+
|
|
20
|
+
### Minor Changes
|
|
21
|
+
|
|
22
|
+
- 32ea2de: **Breaking.** The front door is now a credential and a character id. Everything
|
|
23
|
+
else has a working default.
|
|
24
|
+
|
|
25
|
+
```tsx
|
|
26
|
+
import ConvaiCharacter from "@convai/embody/react";
|
|
27
|
+
|
|
28
|
+
<ConvaiCharacter apiKey={apiKey} characterId={characterId} />;
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
That renders the character: no `<Canvas>`, no lights, no camera, no
|
|
32
|
+
`ConvaiClient`, no resolver, no frame loop, no lipsync queue. embody resolves
|
|
33
|
+
the body, opens the `@convai/web-sdk` connection, renders its audio and drives
|
|
34
|
+
lipsync, gaze, breathing and blink.
|
|
35
|
+
|
|
36
|
+
Previously an app had to build a resolver, hand it in, construct its _own_
|
|
37
|
+
`ConvaiClient` with the same credentials, connect it, attach an
|
|
38
|
+
`AudioRenderer`, and thread `client.blendshapeQueue` through every frame —
|
|
39
|
+
two connections and two copies of the credentials for one character.
|
|
40
|
+
|
|
41
|
+
- `createCharacter` takes `apiKey` / `token` / `authToken` and builds the
|
|
42
|
+
resolver itself. `character.client` is the SDK client, and `update(delta)`
|
|
43
|
+
no longer needs a queue argument.
|
|
44
|
+
- `ConvaiCharacter` (default export of `/react`) brings its own canvas.
|
|
45
|
+
`CharacterModel` is the in-scene form for apps composing their own scene.
|
|
46
|
+
- The SDK's conversational surface is re-exported, so `@convai/embody` is the
|
|
47
|
+
only import an app needs. `useWidgetClient` adapts embody's client to
|
|
48
|
+
`<ConvaiWidget/>` — the SDK's `useConvaiClient` would construct a second
|
|
49
|
+
client, and so a second connection.
|
|
50
|
+
- Escape hatches for everything the old API required: `resolver`, `client`,
|
|
51
|
+
`connect: false`, `session`, and an explicit `queue` on `update()`.
|
|
52
|
+
|
|
53
|
+
Fixes, all found by running it rather than by typechecking it:
|
|
54
|
+
|
|
55
|
+
- **Draco assets could not load.** A GLB declaring
|
|
56
|
+
`KHR_draco_mesh_compression` failed outright with "No DRACOLoader instance
|
|
57
|
+
provided", which took out the entire default male body. A decoder is now
|
|
58
|
+
wired; `dracoDecoderPath` overrides where its wasm comes from.
|
|
59
|
+
- **Characters loaded in their bind pose**, arms out, until the app called
|
|
60
|
+
`play()`. An idle clip now starts automatically; `idleClip` overrides it.
|
|
61
|
+
- **Auth tokens were being dropped.** The resolver sent
|
|
62
|
+
`Authorization: Bearer`, which the backend ignores, silently downgrading the
|
|
63
|
+
request. It now sends `API-AUTH-TOKEN`, matching the SDK.
|
|
64
|
+
- **A missing credential is refused** before any request. There is no
|
|
65
|
+
anonymous access to Convai, and the SDK does not use the backend's guest
|
|
66
|
+
path either.
|
|
67
|
+
- `CharacterBundle.fallback` is typed. The backend has always returned it;
|
|
68
|
+
without it no consumer could tell a placeholder body from the character's
|
|
69
|
+
own.
|
|
70
|
+
- Resolver failures now carry the backend's message. "Character has no 3D
|
|
71
|
+
embodiment configured" points at the avatar config; a bare `HTTP 404` sends
|
|
72
|
+
people to check the character id.
|
|
73
|
+
|
|
74
|
+
- 266904e: `httpResolver`'s `baseUrl` is now optional, defaulting to `DEFAULT_BASE_URL` —
|
|
75
|
+
a source constant pinned per release branch, mirroring how `@convai/web-sdk`
|
|
76
|
+
pins its connect host.
|
|
77
|
+
|
|
78
|
+
`stable-release` and `beta-release` target production; `stg-release` targets
|
|
79
|
+
staging. `scripts/check-hosts.mjs` enforces it, and `build:stg` / `build:beta` /
|
|
80
|
+
`build:prod` each assert the policy for the release they claim to be, so a
|
|
81
|
+
staging artifact cannot be cut from prod-host source.
|
|
82
|
+
|
|
83
|
+
- f972846: Initial motion slice: rig abstraction, staged frame pipeline, and the behaviours
|
|
84
|
+
that make a character read as alive.
|
|
85
|
+
|
|
86
|
+
- `RigProfile` / `resolveRig` — bone and morph roles as data, `convai-mha-1` registered
|
|
87
|
+
- `Pipeline` — explicit six-stage frame order (normalize, mixer, gaze, additive, ik, attach, morph);
|
|
88
|
+
intra-stage order is the contract that puts blink after lipsync
|
|
89
|
+
- `GazeSystem` — head/neck solve, idle drift, look-away shifts, turn banking,
|
|
90
|
+
camera lock and clip deviation pull-back
|
|
91
|
+
- `BreathingSystem`, `BlinkSystem`, `EyeGazeSystem`
|
|
92
|
+
- `prefers-reduced-motion` support across every additive layer
|
|
93
|
+
- Thin React binding: `<ConvaiCharacter/>` and `useCharacter()`, with the
|
|
94
|
+
create/cancel/dispose lifecycle extracted framework-free
|
|
95
|
+
|
|
3
96
|
All notable changes are recorded here. Versions are managed with
|
|
4
97
|
[changesets](https://github.com/changesets/changesets); add one with
|
|
5
98
|
`pnpm changeset` and the release flow writes this file.
|
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
**Give a Convai character a body.**
|
|
4
4
|
|
|
5
|
-
[](https://www.npmjs.com/package/@convai/embody)
|
|
6
6
|
[](https://www.typescriptlang.org/)
|
|
7
7
|
[](./LICENSE)
|
|
8
8
|
|
|
@@ -15,8 +15,9 @@ lipsync, gaze, breathing and blink. Layered on
|
|
|
15
15
|
> **Branch: `stg`** — this branch defaults to the staging backend, which has
|
|
16
16
|
> its own character database. Use `main` or `prod` for production.
|
|
17
17
|
>
|
|
18
|
-
> **Status: early release
|
|
19
|
-
>
|
|
18
|
+
> **Status: early release.** The front door — a credential plus a
|
|
19
|
+
> `characterId` — is settled; what is still moving is listed in
|
|
20
|
+
> [Roadmap](#roadmap).
|
|
20
21
|
|
|
21
22
|
## Install
|
|
22
23
|
|
|
@@ -29,40 +30,105 @@ pnpm add @react-three/fiber react
|
|
|
29
30
|
`three` and `@convai/web-sdk` are peer dependencies. `react` and
|
|
30
31
|
`@react-three/fiber` are optional peers — the `/core` path needs neither.
|
|
31
32
|
|
|
32
|
-
## Quick start
|
|
33
|
+
## Quick start
|
|
33
34
|
|
|
34
35
|
```tsx
|
|
35
|
-
import
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
<ConvaiCharacter
|
|
39
|
-
|
|
40
|
-
characterId={characterId}
|
|
41
|
-
queue={client.blendshapeQueue} // only while this character owns the turn
|
|
42
|
-
/>
|
|
36
|
+
import ConvaiCharacter from '@convai/embody/react'
|
|
37
|
+
|
|
38
|
+
export default function App() {
|
|
39
|
+
return <ConvaiCharacter apiKey={apiKey} characterId={characterId} />
|
|
40
|
+
}
|
|
43
41
|
```
|
|
44
42
|
|
|
45
|
-
|
|
43
|
+
That renders the character. Nothing else is needed — no `<Canvas>`, no lights,
|
|
44
|
+
no camera, no `ConvaiClient`, no resolver, no frame loop. embody resolves the
|
|
45
|
+
body, opens the `@convai/web-sdk` connection, renders its audio, and drives
|
|
46
|
+
lipsync, gaze, breathing and blink.
|
|
47
|
+
|
|
48
|
+
### Credentials
|
|
49
|
+
|
|
50
|
+
`apiKey` is the fastest way to see something on screen, and it is what the
|
|
51
|
+
demo uses. It is a **workspace-wide** credential, so treat a build that ships
|
|
52
|
+
it as trusted-only.
|
|
53
|
+
|
|
54
|
+
For anything user-facing, mint a short-lived auth token server-side and pass
|
|
55
|
+
`token` instead — same component, one prop different:
|
|
56
|
+
|
|
57
|
+
```tsx
|
|
58
|
+
<ConvaiCharacter token={token} characterId={characterId} />
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### In a scene you already have
|
|
62
|
+
|
|
63
|
+
`ConvaiCharacter` brings its own canvas. When the character should join a
|
|
64
|
+
scene you are composing yourself, use `CharacterModel` inside your `<Canvas>`:
|
|
65
|
+
|
|
66
|
+
```tsx
|
|
67
|
+
import { CharacterModel } from '@convai/embody/react'
|
|
68
|
+
|
|
69
|
+
<Canvas>
|
|
70
|
+
<MyLights />
|
|
71
|
+
<CharacterModel apiKey={apiKey} characterId={characterId} />
|
|
72
|
+
<MyProps />
|
|
73
|
+
</Canvas>
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### Talking to the character
|
|
77
|
+
|
|
78
|
+
`character.client` is the `@convai/web-sdk` client — the same connection, not
|
|
79
|
+
a second one. Lift it out with `onReady`; chat UI is DOM, so it lives outside
|
|
80
|
+
the canvas:
|
|
81
|
+
|
|
82
|
+
```tsx
|
|
83
|
+
import {
|
|
84
|
+
ConvaiCharacter, ConvaiWidget, useWidgetClient, type ConvaiClient,
|
|
85
|
+
} from '@convai/embody/react'
|
|
86
|
+
|
|
87
|
+
function App() {
|
|
88
|
+
const [client, setClient] = useState<ConvaiClient | null>(null)
|
|
89
|
+
const widgetClient = useWidgetClient(client)
|
|
90
|
+
return (
|
|
91
|
+
<>
|
|
92
|
+
<ConvaiCharacter
|
|
93
|
+
apiKey={apiKey}
|
|
94
|
+
characterId={characterId}
|
|
95
|
+
onReady={(c) => setClient(c.client)}
|
|
96
|
+
/>
|
|
97
|
+
{widgetClient && <ConvaiWidget convaiClient={widgetClient} />}
|
|
98
|
+
</>
|
|
99
|
+
)
|
|
100
|
+
}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`ConvaiWidget` and `ConvaiClient` are re-exported from `@convai/web-sdk`, so
|
|
104
|
+
`@convai/embody` is the only import your app needs. `useWidgetClient` adapts
|
|
105
|
+
embody's client to the widget's props — the SDK's own `useConvaiClient` would
|
|
106
|
+
*construct a second client*, and so open a second connection, which is exactly
|
|
107
|
+
what embody exists to avoid.
|
|
108
|
+
|
|
109
|
+
## Without React
|
|
110
|
+
|
|
111
|
+
The root entry is framework-free — it pulls in neither React nor
|
|
112
|
+
`@react-three/fiber`, so a vanilla three.js app can use it directly:
|
|
46
113
|
|
|
47
114
|
```ts
|
|
48
|
-
import { createCharacter
|
|
115
|
+
import { createCharacter } from '@convai/embody'
|
|
49
116
|
|
|
50
|
-
const character = await createCharacter({
|
|
51
|
-
resolver: httpResolver({ apiKey }), // host defaults per release channel
|
|
52
|
-
characterId,
|
|
53
|
-
})
|
|
117
|
+
const character = await createCharacter({ apiKey, characterId })
|
|
54
118
|
|
|
55
119
|
scene.add(character.root)
|
|
56
120
|
character.camera = camera
|
|
121
|
+
character.client?.sendUserTextMessage('hello')
|
|
57
122
|
|
|
58
123
|
renderer.setAnimationLoop(() => {
|
|
59
|
-
character.update(clock.getDelta()
|
|
124
|
+
character.update(clock.getDelta())
|
|
60
125
|
renderer.render(scene, camera)
|
|
61
126
|
})
|
|
62
127
|
```
|
|
63
128
|
|
|
64
129
|
`character.dispose()` releases the model, its materials and textures, the
|
|
65
|
-
mixer, and the reduced-motion listener
|
|
130
|
+
mixer, and the reduced-motion listener, and closes the connection embody
|
|
131
|
+
opened. A client you passed in yourself is left alone.
|
|
66
132
|
|
|
67
133
|
## What it does
|
|
68
134
|
|
|
@@ -95,12 +161,18 @@ load-bearing, not incidental:
|
|
|
95
161
|
ships but is deliberately not registered. Bone and morph names are data on the
|
|
96
162
|
profile, so a new rig is a value rather than a code change.
|
|
97
163
|
|
|
98
|
-
##
|
|
164
|
+
## Escape hatches
|
|
99
165
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
166
|
+
The defaults cover the normal case; each of these exists for a specific one.
|
|
167
|
+
|
|
168
|
+
| Option | For |
|
|
169
|
+
|---|---|
|
|
170
|
+
| `resolver` | A private asset backend, a fixture in tests, or a pre-fetched manifest. Anything returning a `CharacterBundle` works — the demo ships an offline fixture resolver for backend-free development. `httpResolver` (the default) talks to `GET /character/{id}/embodiment`. |
|
|
171
|
+
| `client` | An SDK client your app already created and connected. embody drives lipsync from it and never disconnects it. |
|
|
172
|
+
| `connect: false` | A body with no mind — thumbnails, previews, asset review tools. `character.client` is then `null`. |
|
|
173
|
+
| `session` | Extra SDK options (voice, memory, transport, `endUserId`). Credentials and `characterId` are not repeated here. |
|
|
174
|
+
| `baseUrl` | A different backend environment. Defaults per release channel. |
|
|
175
|
+
| `queue` on `update()` | One client driving several bodies, when you are deciding which owns the current turn. |
|
|
104
176
|
|
|
105
177
|
## Docs
|
|
106
178
|
|
|
@@ -112,6 +184,7 @@ Design, plan and decision history live in [`docs/`](./docs):
|
|
|
112
184
|
| [motion-implementation-plan.md](./docs/motion-implementation-plan.md) | the 12-task TDD plan |
|
|
113
185
|
| [motion-decisions.md](./docs/motion-decisions.md) | 73 rulings, each with its cost if wrong |
|
|
114
186
|
| [motion-execution-ledger.md](./docs/motion-execution-ledger.md) | the full execution record |
|
|
187
|
+
| [testing-spec.md](./docs/testing-spec.md) | standing policy: the module → test manifest CI enforces |
|
|
115
188
|
|
|
116
189
|
## Release channels
|
|
117
190
|
|
|
@@ -144,19 +217,30 @@ pnpm check:hosts # auto-detects the current branch
|
|
|
144
217
|
|
|
145
218
|
```bash
|
|
146
219
|
pnpm install
|
|
147
|
-
pnpm test #
|
|
220
|
+
pnpm test # 183 tests
|
|
148
221
|
pnpm typecheck
|
|
222
|
+
pnpm check:tests # every src/ module has a named test — see docs/testing-spec.md
|
|
149
223
|
pnpm build # emits dist/ with .d.ts
|
|
150
|
-
pnpm verify # hosts + typecheck + test + build
|
|
224
|
+
pnpm verify # hosts + test manifest + typecheck + test + build
|
|
151
225
|
pnpm dev # the demo (see demo/verify.md)
|
|
152
226
|
```
|
|
153
227
|
|
|
228
|
+
Every module in `src/` has a row in [`docs/testing-spec.md`](./docs/testing-spec.md)
|
|
229
|
+
naming its test and what that test must prove, and `pnpm check:tests` enforces
|
|
230
|
+
it in CI. A new file under `src/` with no row fails the build, so adding a
|
|
231
|
+
feature and adding its test are the same commit.
|
|
232
|
+
|
|
154
233
|
## Roadmap
|
|
155
234
|
|
|
156
|
-
- **
|
|
157
|
-
|
|
158
|
-
|
|
235
|
+
- **Zero requests of our own.** Return the body manifest from the connection
|
|
236
|
+
the SDK already opens, so embody makes no API call at all. Designed in
|
|
237
|
+
[`docs/manifest-in-connect.md`](./docs/manifest-in-connect.md); the work is
|
|
238
|
+
mostly in `core-service`.
|
|
159
239
|
- **Retry on expired asset URLs** — re-resolve and retry once on a 403.
|
|
240
|
+
- **More rigs.** `convai-mha-1` is the only registered profile, so a character
|
|
241
|
+
on any other rig has no body.
|
|
242
|
+
- **Compressed assets.** The default bodies are ~30 MB; the manifest already
|
|
243
|
+
has `lod` and `encodings` fields that nothing fills in.
|
|
160
244
|
- Foot lock, locomotion, hair physics, idle gestures.
|
|
161
245
|
- Eye saccades and the speech lock.
|
|
162
246
|
|
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import * as THREE from 'three';
|
|
6
6
|
import { type CharacterResolver, type CharacterBundle, type Representation } from '../resolve/resolver.js';
|
|
7
|
+
import { type SessionOptions } from '../connect/session.js';
|
|
8
|
+
import type { ConvaiClient } from '@convai/web-sdk/core';
|
|
7
9
|
import { type UnitNormalizer } from '../rig/units.js';
|
|
8
10
|
import { AnimationController } from '../clips/animations.js';
|
|
9
11
|
import { LipsyncSystem, type MouthTuning } from '../speech/lipsync.js';
|
|
@@ -16,9 +18,69 @@ import { type BreathingConfig } from '../motion/breathe.js';
|
|
|
16
18
|
import { type EyeGazeConfig } from '../motion/eyeGaze.js';
|
|
17
19
|
import { type BlinkConfig } from '../motion/blink.js';
|
|
18
20
|
import { type ReducedMotion } from '../motion/reducedMotion.js';
|
|
21
|
+
/**
|
|
22
|
+
* Where the Draco decoder is fetched from. Google's versioned CDN, which is
|
|
23
|
+
* what three's own examples use and is served with permissive CORS.
|
|
24
|
+
*
|
|
25
|
+
* Override via `dracoDecoderPath` to self-host — required for air-gapped
|
|
26
|
+
* deployments, or any app whose CSP does not allow gstatic.
|
|
27
|
+
*/
|
|
28
|
+
export declare const DEFAULT_DRACO_DECODER_PATH = "https://www.gstatic.com/draco/versioned/decoders/1.5.7/";
|
|
19
29
|
export interface CreateCharacterOptions {
|
|
20
|
-
|
|
30
|
+
/**
|
|
31
|
+
* The character to embody. This plus a credential is the whole required
|
|
32
|
+
* API — everything below has a working default.
|
|
33
|
+
*/
|
|
21
34
|
characterId: string;
|
|
35
|
+
/**
|
|
36
|
+
* Convai auth token — the credential a browser app should ship, since it
|
|
37
|
+
* is scoped and short-lived. Alias of `authToken`; supply either.
|
|
38
|
+
*/
|
|
39
|
+
token?: string;
|
|
40
|
+
/** Convai API key. Server-side or trusted contexts. */
|
|
41
|
+
apiKey?: string;
|
|
42
|
+
/** Same as `token`, under the name @convai/web-sdk uses. */
|
|
43
|
+
authToken?: string;
|
|
44
|
+
/**
|
|
45
|
+
* Backend host. Defaults to the release channel's host (see
|
|
46
|
+
* `DEFAULT_BASE_URL`); override only to point at a different environment.
|
|
47
|
+
*/
|
|
48
|
+
baseUrl?: string;
|
|
49
|
+
/**
|
|
50
|
+
* Escape hatch: resolve the body manifest yourself. Almost nobody needs
|
|
51
|
+
* this — with credentials above, embody builds an `httpResolver` for you.
|
|
52
|
+
* Supply it for a private asset backend, a fixture in tests, or a
|
|
53
|
+
* pre-fetched manifest.
|
|
54
|
+
*/
|
|
55
|
+
resolver?: CharacterResolver;
|
|
56
|
+
/**
|
|
57
|
+
* Extra @convai/web-sdk connection options (voice, memory, transport,
|
|
58
|
+
* `endUserId`, …). `characterId`, `apiKey` and `authToken` come from
|
|
59
|
+
* above and are not repeated here.
|
|
60
|
+
*/
|
|
61
|
+
session?: Omit<SessionOptions, 'characterId' | 'apiKey' | 'authToken'>;
|
|
62
|
+
/**
|
|
63
|
+
* Reuse a client the app already created and connected, instead of having
|
|
64
|
+
* embody open one. embody drives lipsync from it but never disconnects it.
|
|
65
|
+
*
|
|
66
|
+
* Read once, when the character is created. In React that means it must be
|
|
67
|
+
* present on the first render that mounts the character — a client supplied
|
|
68
|
+
* later is not picked up, because re-reading it would mean rebuilding the
|
|
69
|
+
* character and re-downloading its model. Gate the mount on your client
|
|
70
|
+
* being ready.
|
|
71
|
+
*/
|
|
72
|
+
client?: ConvaiClient;
|
|
73
|
+
/**
|
|
74
|
+
* Set `false` for a body with no mind — no SDK connection, no audio, no
|
|
75
|
+
* lipsync source. Useful for thumbnails, previews and asset review tools
|
|
76
|
+
* that only need to render the model. Default `true`.
|
|
77
|
+
*/
|
|
78
|
+
connect?: boolean;
|
|
79
|
+
/**
|
|
80
|
+
* Base URL for the Draco decoder files. Defaults to
|
|
81
|
+
* `DEFAULT_DRACO_DECODER_PATH`; set this to self-host them.
|
|
82
|
+
*/
|
|
83
|
+
dracoDecoderPath?: string;
|
|
22
84
|
/** Rig profiles this build understands, in preference order. */
|
|
23
85
|
supportedRigs?: string[];
|
|
24
86
|
targetHeight?: number;
|
|
@@ -28,6 +90,13 @@ export interface CreateCharacterOptions {
|
|
|
28
90
|
decorators?: CharacterDecorator[];
|
|
29
91
|
/** Extra local clips (name → url), merged OVER manifest clips by name. */
|
|
30
92
|
animations?: Record<string, string>;
|
|
93
|
+
/**
|
|
94
|
+
* Which clip to play on load. Defaults to a clip named "idle"
|
|
95
|
+
* (case-insensitive), falling back to the first clip the character ships,
|
|
96
|
+
* so a character is never left standing in its bind pose. Pass `false` to
|
|
97
|
+
* start paused and drive playback yourself.
|
|
98
|
+
*/
|
|
99
|
+
idleClip?: string | false;
|
|
31
100
|
/** Stage 2 head/neck gaze tracking config. */
|
|
32
101
|
gaze?: GazeConfig;
|
|
33
102
|
/** Stage 6 eyelid/blink config. */
|
|
@@ -51,11 +120,31 @@ export interface CharacterHandle {
|
|
|
51
120
|
rig: ResolvedRig;
|
|
52
121
|
pipeline: Pipeline;
|
|
53
122
|
gaze: GazeSystem;
|
|
123
|
+
/**
|
|
124
|
+
* The live @convai/web-sdk client — the character's mind. `null` only when
|
|
125
|
+
* `connect: false`. Use it for everything conversational:
|
|
126
|
+
*
|
|
127
|
+
* ```ts
|
|
128
|
+
* character.client?.sendUserTextMessage('hello')
|
|
129
|
+
* character.client?.on('transcript', (t) => console.log(t))
|
|
130
|
+
* ```
|
|
131
|
+
*/
|
|
132
|
+
client: ConvaiClient | null;
|
|
54
133
|
/** Set by the app once its camera exists; gaze tracking no-ops without one. */
|
|
55
134
|
camera?: THREE.Camera;
|
|
56
|
-
/**
|
|
135
|
+
/**
|
|
136
|
+
* Tick every frame. The lipsync queue comes from this character's own
|
|
137
|
+
* session, so a single-character app passes nothing.
|
|
138
|
+
*
|
|
139
|
+
* Pass `queue` explicitly only when one client drives several bodies and
|
|
140
|
+
* you are deciding which of them owns the current turn; it overrides the
|
|
141
|
+
* session queue for that frame.
|
|
142
|
+
*/
|
|
57
143
|
update(delta: number, queue?: BlendshapeQueue): void;
|
|
58
|
-
/**
|
|
144
|
+
/**
|
|
145
|
+
* Release GPU resources and, when embody opened the connection, close it.
|
|
146
|
+
* A client passed in as `client` is left alone — the app owns its lifetime.
|
|
147
|
+
*/
|
|
59
148
|
dispose(): void;
|
|
60
149
|
}
|
|
61
150
|
/**
|
|
@@ -76,6 +165,22 @@ export declare function buildPipeline(deps: {
|
|
|
76
165
|
pipeline: Pipeline;
|
|
77
166
|
gaze: GazeSystem;
|
|
78
167
|
};
|
|
168
|
+
/**
|
|
169
|
+
* Which clip a freshly loaded character should play, if any.
|
|
170
|
+
*
|
|
171
|
+
* Extracted as its own seam (same rationale as `buildPipeline`) so the
|
|
172
|
+
* choice can be tested without a GLB.
|
|
173
|
+
*
|
|
174
|
+
* `requested` is `CreateCharacterOptions.idleClip`: a name to insist on,
|
|
175
|
+
* `false` to start paused, or undefined for the default. Clip names come from
|
|
176
|
+
* the manifest and are not guaranteed, so the default prefers a clip actually
|
|
177
|
+
* called idle and otherwise falls back to whatever the character shipped
|
|
178
|
+
* first — anything is better than the bind pose.
|
|
179
|
+
*
|
|
180
|
+
* A requested name that the character does not have returns null rather than
|
|
181
|
+
* silently substituting: the caller asked for something specific.
|
|
182
|
+
*/
|
|
183
|
+
export declare function pickIdleClip(names: string[], requested?: string | false): string | null;
|
|
79
184
|
/**
|
|
80
185
|
* The teardown block, factored out as its own seam (same rationale as
|
|
81
186
|
* `buildPipeline` above) so it can be tested without a GLB — real,
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js';
|
|
2
|
-
import {
|
|
2
|
+
import { DRACOLoader } from 'three/examples/jsm/loaders/DRACOLoader.js';
|
|
3
|
+
import { pickRepresentation, httpResolver, MISSING_CREDENTIAL } from '../resolve/resolver.js';
|
|
4
|
+
import { createSession, adoptSession } from '../connect/session.js';
|
|
3
5
|
import { createUnitNormalizer } from '../rig/units.js';
|
|
4
6
|
import { AnimationController } from '../clips/animations.js';
|
|
5
7
|
import { LipsyncSystem } from '../speech/lipsync.js';
|
|
@@ -12,6 +14,14 @@ import { BreathingSystem } from '../motion/breathe.js';
|
|
|
12
14
|
import { EyeGazeSystem } from '../motion/eyeGaze.js';
|
|
13
15
|
import { BlinkSystem } from '../motion/blink.js';
|
|
14
16
|
import { createReducedMotion } from '../motion/reducedMotion.js';
|
|
17
|
+
/**
|
|
18
|
+
* Where the Draco decoder is fetched from. Google's versioned CDN, which is
|
|
19
|
+
* what three's own examples use and is served with permissive CORS.
|
|
20
|
+
*
|
|
21
|
+
* Override via `dracoDecoderPath` to self-host — required for air-gapped
|
|
22
|
+
* deployments, or any app whose CSP does not allow gstatic.
|
|
23
|
+
*/
|
|
24
|
+
export const DEFAULT_DRACO_DECODER_PATH = 'https://www.gstatic.com/draco/versioned/decoders/1.5.7/';
|
|
15
25
|
/**
|
|
16
26
|
* The registration block, factored out as its own seam so it can be tested
|
|
17
27
|
* without a GLB or a network resolver — it only needs objects of the right
|
|
@@ -41,6 +51,28 @@ export function buildPipeline(deps, opts) {
|
|
|
41
51
|
}
|
|
42
52
|
return { pipeline, gaze };
|
|
43
53
|
}
|
|
54
|
+
/**
|
|
55
|
+
* Which clip a freshly loaded character should play, if any.
|
|
56
|
+
*
|
|
57
|
+
* Extracted as its own seam (same rationale as `buildPipeline`) so the
|
|
58
|
+
* choice can be tested without a GLB.
|
|
59
|
+
*
|
|
60
|
+
* `requested` is `CreateCharacterOptions.idleClip`: a name to insist on,
|
|
61
|
+
* `false` to start paused, or undefined for the default. Clip names come from
|
|
62
|
+
* the manifest and are not guaranteed, so the default prefers a clip actually
|
|
63
|
+
* called idle and otherwise falls back to whatever the character shipped
|
|
64
|
+
* first — anything is better than the bind pose.
|
|
65
|
+
*
|
|
66
|
+
* A requested name that the character does not have returns null rather than
|
|
67
|
+
* silently substituting: the caller asked for something specific.
|
|
68
|
+
*/
|
|
69
|
+
export function pickIdleClip(names, requested) {
|
|
70
|
+
if (requested === false)
|
|
71
|
+
return null;
|
|
72
|
+
if (requested)
|
|
73
|
+
return names.includes(requested) ? requested : null;
|
|
74
|
+
return names.find((n) => n.toLowerCase() === 'idle') ?? names[0] ?? null;
|
|
75
|
+
}
|
|
44
76
|
/** Every material property that can hold a disposable `THREE.Texture`. */
|
|
45
77
|
const TEXTURE_PROPS = [
|
|
46
78
|
'map', 'normalMap', 'roughnessMap', 'metalnessMap', 'emissiveMap',
|
|
@@ -95,9 +127,52 @@ export function disposeCharacter(deps) {
|
|
|
95
127
|
reducedMotion.dispose();
|
|
96
128
|
}
|
|
97
129
|
export async function createCharacter(opts) {
|
|
98
|
-
|
|
130
|
+
// The resolver is an escape hatch, not a requirement: the common path is
|
|
131
|
+
// credentials in, character out. Same credentials for body and mind, so
|
|
132
|
+
// the caller never supplies them twice.
|
|
133
|
+
// `token` is the name the API is documented under; `authToken` is what the
|
|
134
|
+
// SDK config calls the same thing. Accept both, resolve once, here.
|
|
135
|
+
const authToken = opts.token ?? opts.authToken;
|
|
136
|
+
// Fail before the connection and a multi-megabyte download rather than
|
|
137
|
+
// after. Only the DEFAULT resolver needs this — a caller supplying their
|
|
138
|
+
// own resolver (fixtures, a private backend) authenticates it themselves.
|
|
139
|
+
if (!opts.resolver && !authToken && !opts.apiKey)
|
|
140
|
+
throw new Error(MISSING_CREDENTIAL);
|
|
141
|
+
const resolver = opts.resolver ??
|
|
142
|
+
httpResolver({ baseUrl: opts.baseUrl, apiKey: opts.apiKey, authToken });
|
|
143
|
+
// Body and mind are started together — the connection is the slower of the
|
|
144
|
+
// two and the GLB is tens of megabytes, so serialising them would add the
|
|
145
|
+
// whole handshake to time-to-first-frame for no reason.
|
|
146
|
+
const wantsSession = opts.connect !== false;
|
|
147
|
+
const sessionPromise = !wantsSession
|
|
148
|
+
? Promise.resolve(null)
|
|
149
|
+
: opts.client
|
|
150
|
+
? Promise.resolve(adoptSession(opts.client))
|
|
151
|
+
: createSession({
|
|
152
|
+
...opts.session,
|
|
153
|
+
characterId: opts.characterId,
|
|
154
|
+
apiKey: opts.apiKey,
|
|
155
|
+
authToken,
|
|
156
|
+
});
|
|
157
|
+
// Nothing awaits this yet, so an early connection failure would be an
|
|
158
|
+
// unhandled rejection while the GLB is still downloading. Park the outcome
|
|
159
|
+
// and re-throw it below, after the body has settled.
|
|
160
|
+
const settledSession = sessionPromise.then((s) => ({ session: s, error: null }), (e) => ({ session: null, error: e }));
|
|
161
|
+
const bundle = await resolver.resolve(opts.characterId);
|
|
99
162
|
const rep = pickRepresentation(bundle, opts.supportedRigs ?? ['convai-mha-1']);
|
|
163
|
+
// Draco is not optional: japanese-m-animations.glb — the default male body's
|
|
164
|
+
// clips — declares KHR_draco_mesh_compression in extensionsRequired, and a
|
|
165
|
+
// GLTFLoader without a decoder fails it outright ("No DRACOLoader instance
|
|
166
|
+
// provided"). Any compressed asset the art pipeline produces will be the
|
|
167
|
+
// same, so the decoder is wired unconditionally rather than sniffed for.
|
|
168
|
+
//
|
|
169
|
+
// The decoder itself is fetched at load time from `dracoDecoderPath`; three
|
|
170
|
+
// does not bundle the wasm, and shipping it inside this package would put a
|
|
171
|
+
// binary in every consumer's bundle whether or not their character uses it.
|
|
100
172
|
const loader = new GLTFLoader();
|
|
173
|
+
const draco = new DRACOLoader();
|
|
174
|
+
draco.setDecoderPath(opts.dracoDecoderPath ?? DEFAULT_DRACO_DECODER_PATH);
|
|
175
|
+
loader.setDRACOLoader(draco);
|
|
101
176
|
const modelUrl = rep.model[0].url;
|
|
102
177
|
// clip sources: manifest clips, with local overrides winning by name
|
|
103
178
|
const clipUrls = new Map();
|
|
@@ -106,10 +181,20 @@ export async function createCharacter(opts) {
|
|
|
106
181
|
for (const [name, url] of Object.entries(opts.animations ?? {}))
|
|
107
182
|
clipUrls.set(name, url);
|
|
108
183
|
const uniqueUrls = [...new Set(clipUrls.values())];
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
184
|
+
let modelGltf, clipGltfs;
|
|
185
|
+
try {
|
|
186
|
+
;
|
|
187
|
+
[modelGltf, ...clipGltfs] = await Promise.all([
|
|
188
|
+
loader.loadAsync(modelUrl),
|
|
189
|
+
...uniqueUrls.map((u) => loader.loadAsync(u)),
|
|
190
|
+
]);
|
|
191
|
+
}
|
|
192
|
+
finally {
|
|
193
|
+
// Frees the decoder's worker pool. Safe once every loadAsync above has
|
|
194
|
+
// settled, and it must run on the failure path too or a character that
|
|
195
|
+
// fails to load leaks workers.
|
|
196
|
+
draco.dispose();
|
|
197
|
+
}
|
|
113
198
|
const root = modelGltf.scene;
|
|
114
199
|
// gather every clip whose name the manifest/overrides declared
|
|
115
200
|
const wanted = new Set(clipUrls.keys());
|
|
@@ -120,6 +205,17 @@ export async function createCharacter(opts) {
|
|
|
120
205
|
clips.push(a);
|
|
121
206
|
const animations = new AnimationController(root);
|
|
122
207
|
animations.register(clips);
|
|
208
|
+
// Start idling. Without this a zero-config character stands in its bind
|
|
209
|
+
// pose — arms out, visibly wrong — until the app happens to call play().
|
|
210
|
+
// Making the caller do that is exactly the kind of required extra step
|
|
211
|
+
// this API is meant not to have.
|
|
212
|
+
//
|
|
213
|
+
// Name resolution is by convention with a fallback, because clip names come
|
|
214
|
+
// from the manifest and are not guaranteed: prefer a clip actually called
|
|
215
|
+
// idle, otherwise just play the first one the character shipped.
|
|
216
|
+
const idle = pickIdleClip(animations.names, opts.idleClip);
|
|
217
|
+
if (idle)
|
|
218
|
+
animations.play(idle);
|
|
123
219
|
const lipsync = new LipsyncSystem(opts.mouth);
|
|
124
220
|
lipsync.bind(root);
|
|
125
221
|
for (const d of opts.decorators ?? []) {
|
|
@@ -142,12 +238,23 @@ export async function createCharacter(opts) {
|
|
|
142
238
|
energy: 0,
|
|
143
239
|
reducedMotion: false,
|
|
144
240
|
};
|
|
241
|
+
// The body is ready; now surface any connection failure. Ordering matters:
|
|
242
|
+
// reporting it here means a mind that fails to connect never leaks a
|
|
243
|
+
// half-built body — we dispose what we loaded before throwing.
|
|
244
|
+
const { session, error: sessionError } = await settledSession;
|
|
245
|
+
if (sessionError) {
|
|
246
|
+
disposeCharacter({ root, mixer: animations.mixer, reducedMotion: reduced });
|
|
247
|
+
throw sessionError;
|
|
248
|
+
}
|
|
145
249
|
const handle = {
|
|
146
250
|
root, bundle, representation: rep, animations, lipsync, rig, pipeline, gaze,
|
|
251
|
+
client: session?.client ?? null,
|
|
147
252
|
camera: undefined,
|
|
148
253
|
update(delta, queue) {
|
|
149
254
|
frameCtx.camera = handle.camera;
|
|
150
|
-
|
|
255
|
+
// An explicit queue wins — that is the multi-body turn-ownership case.
|
|
256
|
+
// Otherwise this character listens to its own session.
|
|
257
|
+
frameCtx.queue = queue ?? session?.queue;
|
|
151
258
|
// Produced in stage 6 last frame — deliberately one frame stale (see
|
|
152
259
|
// FrameContext's doc comment on `speaking`/`energy`).
|
|
153
260
|
frameCtx.speaking = lipsync.speaking;
|
|
@@ -157,6 +264,10 @@ export async function createCharacter(opts) {
|
|
|
157
264
|
},
|
|
158
265
|
dispose() {
|
|
159
266
|
disposeCharacter({ root, mixer: animations.mixer, reducedMotion: reduced });
|
|
267
|
+
// Fire-and-forget: `dispose()` is called from React cleanup and
|
|
268
|
+
// `requestAnimationFrame` teardown, neither of which can await. The
|
|
269
|
+
// session swallows its own double-dispose.
|
|
270
|
+
void session?.dispose();
|
|
160
271
|
},
|
|
161
272
|
};
|
|
162
273
|
return handle;
|