@convai/web-sdk 1.5.0 → 1.6.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -36
- package/dist/core/BlendshapeQueue.d.ts +75 -4
- package/dist/core/BlendshapeQueue.d.ts.map +1 -1
- package/dist/core/BlendshapeQueue.js +400 -24
- package/dist/core/BlendshapeQueue.js.map +1 -1
- package/dist/core/ConvaiClient.d.ts +10 -2
- package/dist/core/ConvaiClient.d.ts.map +1 -1
- package/dist/core/ConvaiClient.js +208 -57
- package/dist/core/ConvaiClient.js.map +1 -1
- package/dist/core/MessageHandler.d.ts +8 -0
- package/dist/core/MessageHandler.d.ts.map +1 -1
- package/dist/core/MessageHandler.js +63 -12
- package/dist/core/MessageHandler.js.map +1 -1
- package/dist/core/VideoManager.d.ts +40 -5
- package/dist/core/VideoManager.d.ts.map +1 -1
- package/dist/core/VideoManager.js +579 -30
- package/dist/core/VideoManager.js.map +1 -1
- package/dist/core/connectRequest.d.ts +23 -0
- package/dist/core/connectRequest.d.ts.map +1 -0
- package/dist/core/connectRequest.js +31 -0
- package/dist/core/connectRequest.js.map +1 -0
- package/dist/core/contextUpdateRequest.d.ts +4 -0
- package/dist/core/contextUpdateRequest.d.ts.map +1 -0
- package/dist/core/contextUpdateRequest.js +10 -0
- package/dist/core/contextUpdateRequest.js.map +1 -0
- package/dist/core/index.d.ts +1 -1
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/types.d.ts +172 -3
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/videoSourceState.d.ts +4 -0
- package/dist/core/videoSourceState.d.ts.map +1 -0
- package/dist/core/videoSourceState.js +9 -0
- package/dist/core/videoSourceState.js.map +1 -0
- package/dist/core/visionRequest.d.ts +4 -0
- package/dist/core/visionRequest.d.ts.map +1 -0
- package/dist/core/visionRequest.js +42 -0
- package/dist/core/visionRequest.js.map +1 -0
- package/dist/react/hooks/useConvaiClient.d.ts.map +1 -1
- package/dist/react/hooks/useConvaiClient.js +3 -0
- package/dist/react/hooks/useConvaiClient.js.map +1 -1
- package/dist/vanilla/types.d.ts +6 -0
- package/dist/vanilla/types.d.ts.map +1 -1
- package/package.json +5 -2
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
**Real-time conversational AI characters for the web.**
|
|
4
4
|
|
|
5
|
-
[](https://www.npmjs.com/package/@convai/web-sdk)
|
|
6
6
|
[](https://www.typescriptlang.org/)
|
|
7
7
|
[](./LICENSE)
|
|
8
8
|
|
|
@@ -16,7 +16,7 @@ TypeScript-first SDK for embedding Convai AI characters into React and vanilla J
|
|
|
16
16
|
- **Real-time audio/video** — full-duplex WebRTC with echo cancellation, camera, and screen share
|
|
17
17
|
- **Lipsync** — ARKit and MetaHuman blendshape streams for facial animation
|
|
18
18
|
- **Emotions** — per-turn emotion detection with intensity scale
|
|
19
|
-
- **Dynamic context** — inject
|
|
19
|
+
- **Dynamic context & vision** — inject text state, scene metadata, and LiveKit video frames mid-session
|
|
20
20
|
- **Long-term memory** — persistent cross-session memories scoped to each end user
|
|
21
21
|
- **File upload** — send images to the character during a live session
|
|
22
22
|
- **WebSocket transport** — opt-in alternative to WebRTC for constrained networks
|
|
@@ -45,7 +45,7 @@ yarn add @convai/web-sdk
|
|
|
45
45
|
### React
|
|
46
46
|
|
|
47
47
|
```tsx
|
|
48
|
-
import { useConvaiClient, ConvaiWidget } from
|
|
48
|
+
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk";
|
|
49
49
|
|
|
50
50
|
export function App() {
|
|
51
51
|
const client = useConvaiClient({
|
|
@@ -60,7 +60,7 @@ export function App() {
|
|
|
60
60
|
### Vanilla TypeScript
|
|
61
61
|
|
|
62
62
|
```ts
|
|
63
|
-
import { ConvaiClient, createConvaiWidget } from
|
|
63
|
+
import { ConvaiClient, createConvaiWidget } from "@convai/web-sdk/vanilla";
|
|
64
64
|
|
|
65
65
|
const client = new ConvaiClient({
|
|
66
66
|
apiKey: import.meta.env.VITE_CONVAI_API_KEY,
|
|
@@ -69,7 +69,7 @@ const client = new ConvaiClient({
|
|
|
69
69
|
|
|
70
70
|
const widget = createConvaiWidget(document.body, { convaiClient: client });
|
|
71
71
|
|
|
72
|
-
window.addEventListener(
|
|
72
|
+
window.addEventListener("beforeunload", () => {
|
|
73
73
|
widget.destroy();
|
|
74
74
|
void client.disconnect();
|
|
75
75
|
});
|
|
@@ -81,34 +81,34 @@ window.addEventListener('beforeunload', () => {
|
|
|
81
81
|
|
|
82
82
|
Full documentation is at **<a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">docs.convai.com</a>**.
|
|
83
83
|
|
|
84
|
-
| Guide
|
|
85
|
-
|
|
86
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">Quick Start</a>
|
|
87
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">Configuration</a>
|
|
88
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">React Integration</a>
|
|
89
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">Vanilla JS</a>
|
|
90
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/event-reference" target="_blank" rel="noopener noreferrer">Events</a>
|
|
91
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/dynamic-context" target="_blank" rel="noopener noreferrer">Context Management</a>
|
|
92
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/emotions" target="_blank" rel="noopener noreferrer">Emotions</a>
|
|
93
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">Lipsync</a>
|
|
94
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/actions" target="_blank" rel="noopener noreferrer">Actions</a>
|
|
95
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/long-term-memory" target="_blank" rel="noopener noreferrer">Memory</a>
|
|
96
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">Audio & Video</a>
|
|
97
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/error-handling" target="_blank" rel="noopener noreferrer">Error Handling</a>
|
|
98
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/auth-tokens" target="_blank" rel="noopener noreferrer">Auth Tokens</a>
|
|
99
|
-
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/websocket-transport-layer" target="_blank" rel="noopener noreferrer">WebSocket Transport</a> | Alternative transport for WebRTC-constrained environments
|
|
84
|
+
| Guide | Description |
|
|
85
|
+
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------- |
|
|
86
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">Quick Start</a> | First working integration in under 5 minutes |
|
|
87
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">Configuration</a> | All `ConvaiConfig` options |
|
|
88
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">React Integration</a> | `useConvaiClient`, `ConvaiWidget`, and React-specific patterns |
|
|
89
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">Vanilla JS</a> | `ConvaiClient`, `createConvaiWidget`, and audio setup |
|
|
90
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/event-reference" target="_blank" rel="noopener noreferrer">Events</a> | Full event reference — `botReady`, `stateChange`, `messagesChange`, `interactionCreated`, and more |
|
|
91
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/dynamic-context" target="_blank" rel="noopener noreferrer">Context Management</a> | Dynamic context, `updateContext`, file upload, session management |
|
|
92
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/emotions" target="_blank" rel="noopener noreferrer">Emotions</a> | Per-turn emotion detection with provider options |
|
|
93
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">Lipsync</a> | ARKit / MetaHuman blendshape streams and `BlendshapeQueue` API |
|
|
94
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/actions" target="_blank" rel="noopener noreferrer">Actions</a> | Trigger character behaviors and scene actions |
|
|
95
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/long-term-memory" target="_blank" rel="noopener noreferrer">Memory</a> | Long-term memory scoped to end users |
|
|
96
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk" target="_blank" rel="noopener noreferrer">Audio & Video</a> | Microphone, camera, screen share controls |
|
|
97
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/error-handling" target="_blank" rel="noopener noreferrer">Error Handling</a> | `error`, `disconnect`, `serverResponse`, retry patterns |
|
|
98
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/auth-tokens" target="_blank" rel="noopener noreferrer">Auth Tokens</a> | Server-side token exchange for production |
|
|
99
|
+
| <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-plugins/convai-web-sdk/websocket-transport-layer" target="_blank" rel="noopener noreferrer">WebSocket Transport</a> | Alternative transport for WebRTC-constrained environments |
|
|
100
100
|
|
|
101
101
|
---
|
|
102
102
|
|
|
103
103
|
## Package entry points
|
|
104
104
|
|
|
105
|
-
| Import path
|
|
106
|
-
|
|
107
|
-
| `@convai/web-sdk`
|
|
108
|
-
| `@convai/web-sdk/react`
|
|
109
|
-
| `@convai/web-sdk/vanilla`
|
|
110
|
-
| `@convai/web-sdk/core`
|
|
111
|
-
| `@convai/web-sdk/lipsync-helpers`
|
|
105
|
+
| Import path | Contents |
|
|
106
|
+
| ----------------------------------- | --------------------------------------------------------------------------------------------------------------- |
|
|
107
|
+
| `@convai/web-sdk` | React hooks, components, and re-exported core types |
|
|
108
|
+
| `@convai/web-sdk/react` | Same as default (React-explicit alias) |
|
|
109
|
+
| `@convai/web-sdk/vanilla` | `ConvaiClient`, `createConvaiWidget`, `AudioRenderer` |
|
|
110
|
+
| `@convai/web-sdk/core` | Framework-agnostic `ConvaiClient`, managers, and all types |
|
|
111
|
+
| `@convai/web-sdk/lipsync-helpers` | Blendshape format utilities and queue helpers |
|
|
112
112
|
| `@convai/web-sdk/vanilla/websocket` | **Opt-in.** Registers the WebSocket transport. Import alongside `/vanilla` when using `transport: "websocket"`. |
|
|
113
113
|
|
|
114
114
|
---
|
|
@@ -119,13 +119,13 @@ The default transport is WebRTC (LiveKit). A WebSocket-based transport is availa
|
|
|
119
119
|
|
|
120
120
|
```ts
|
|
121
121
|
// Add this import once (e.g. in your app entry file)
|
|
122
|
-
import
|
|
123
|
-
import { ConvaiClient } from
|
|
122
|
+
import "@convai/web-sdk/vanilla/websocket";
|
|
123
|
+
import { ConvaiClient } from "@convai/web-sdk/vanilla";
|
|
124
124
|
|
|
125
125
|
const client = new ConvaiClient({
|
|
126
|
-
apiKey:
|
|
127
|
-
characterId:
|
|
128
|
-
transport:
|
|
126
|
+
apiKey: "...",
|
|
127
|
+
characterId: "...",
|
|
128
|
+
transport: "websocket",
|
|
129
129
|
});
|
|
130
130
|
```
|
|
131
131
|
|
|
@@ -135,13 +135,66 @@ See the <a href="https://docs.convai.com/api-docs/plugins-and-integrations/web-p
|
|
|
135
135
|
|
|
136
136
|
---
|
|
137
137
|
|
|
138
|
-
##
|
|
138
|
+
## Vision dynamic context beta
|
|
139
139
|
|
|
140
|
-
|
|
140
|
+
Vision dynamic context is the default WebRTC/LiveKit vision path when `enableVideo: true`. Camera, screen, canvas, and custom video tracks can feed unified vision context; set `visionInputConfig.enabled: false` only when you need to keep the video channel while opting out.
|
|
141
|
+
|
|
142
|
+
```tsx
|
|
143
|
+
import { useConvaiClient } from "@convai/web-sdk";
|
|
144
|
+
|
|
145
|
+
const client = useConvaiClient({
|
|
146
|
+
apiKey: import.meta.env.VITE_CONVAI_API_KEY,
|
|
147
|
+
characterId: import.meta.env.VITE_CONVAI_CHARACTER_ID,
|
|
148
|
+
enableVideo: true,
|
|
149
|
+
visionInputConfig: {
|
|
150
|
+
framesPerTurn: 12,
|
|
151
|
+
bufferFrames: 30,
|
|
152
|
+
samplingWindows: [
|
|
153
|
+
{ count: 6, intervalMs: 300 },
|
|
154
|
+
{ count: 6, intervalMs: 1000 },
|
|
155
|
+
],
|
|
156
|
+
stalenessSeconds: 10,
|
|
157
|
+
replacePreviousVisionContext: true,
|
|
158
|
+
},
|
|
159
|
+
respondModes: {
|
|
160
|
+
vision: "silent",
|
|
161
|
+
contextUpdate: "auto",
|
|
162
|
+
sceneMetadata: "silent",
|
|
163
|
+
trigger: "must_respond",
|
|
164
|
+
},
|
|
165
|
+
});
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Publish visual sources with semantic labels so acknowledgments can distinguish webcam, canvas, screen, and custom feeds:
|
|
169
|
+
|
|
170
|
+
```ts
|
|
171
|
+
await client.videoControls.enableVideo(); // webcam source
|
|
172
|
+
|
|
173
|
+
const handle = await client.videoControls.publishCanvas(canvas, {
|
|
174
|
+
source: "canvas",
|
|
175
|
+
name: "canvas-pov",
|
|
176
|
+
fps: 1,
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
const statusId = client.visionStatus();
|
|
180
|
+
const triggerId = client.visionTrigger({
|
|
181
|
+
respondMode: "silent",
|
|
182
|
+
frameIndices: [-1, -1],
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
await client.videoControls.unpublishVisionSource(handle);
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
`visionStatus()` and `visionTrigger()` return an update id. Match it against `serverResponse` events for outcomes such as `frames_available`, `buffer_empty`, `attached`, or `no_active_video`.
|
|
189
|
+
|
|
190
|
+
After the beta package is published, install it with:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
npm install @convai/web-sdk@beta
|
|
194
|
+
```
|
|
141
195
|
|
|
142
196
|
---
|
|
143
197
|
|
|
144
198
|
## License
|
|
145
199
|
|
|
146
200
|
Licensed under the [Apache License 2.0](./LICENSE). Copyright 2025 Convai.
|
|
147
|
-
|
|
@@ -27,8 +27,19 @@ export interface TurnStats {
|
|
|
27
27
|
total_blendshapes: number;
|
|
28
28
|
total_turn_duration_ms: number;
|
|
29
29
|
}
|
|
30
|
+
export interface BlendshapeChunkMetadata {
|
|
31
|
+
fps?: number;
|
|
32
|
+
startFrameIndex?: number;
|
|
33
|
+
responseId?: string;
|
|
34
|
+
neurosyncTurnId?: number;
|
|
35
|
+
epoch?: number;
|
|
36
|
+
}
|
|
30
37
|
export declare class BlendshapeQueue {
|
|
31
38
|
private queue;
|
|
39
|
+
private pendingTimelineFrames;
|
|
40
|
+
private activeTimelineAliases;
|
|
41
|
+
private retiredTimelineAliases;
|
|
42
|
+
private retiredOwnerlessTimelineAliases;
|
|
32
43
|
private conversationActive;
|
|
33
44
|
private botSpeaking;
|
|
34
45
|
private conversationEnded;
|
|
@@ -39,17 +50,61 @@ export declare class BlendshapeQueue {
|
|
|
39
50
|
private interruptedAtFrame;
|
|
40
51
|
private needsNormalization;
|
|
41
52
|
private _awaitingBotSpeak;
|
|
53
|
+
private playbackFps;
|
|
54
|
+
private ownerReplacementStoppedSpeaking;
|
|
55
|
+
private missingExpectedFramesSinceMs;
|
|
56
|
+
private readonly now;
|
|
42
57
|
private static readonly TARGET_FPS;
|
|
58
|
+
private static readonly MAX_RETIRED_TIMELINE_ALIASES;
|
|
59
|
+
private static readonly MISSING_INDEXED_FRAME_GRACE_MS;
|
|
43
60
|
/**
|
|
44
61
|
* Create a new BlendshapeQueue
|
|
45
62
|
* @param mapper - Optional function to map/transform blendshape data as it's added
|
|
46
63
|
*/
|
|
47
|
-
constructor(mapper?: (input: number[] | Float32Array) => Float32Array);
|
|
64
|
+
constructor(mapper?: (input: number[] | Float32Array) => Float32Array, now?: () => number);
|
|
48
65
|
/**
|
|
49
66
|
* Add a chunk of blendshapes to the queue
|
|
50
67
|
* @param blendshapes - Array of blendshape frames (each frame is an array of numbers)
|
|
68
|
+
* @param metadata - Optional timeline metadata for ahead-delivered chunks
|
|
69
|
+
*/
|
|
70
|
+
addChunk(blendshapes: number[][], metadata?: BlendshapeChunkMetadata): void;
|
|
71
|
+
private mapFrame;
|
|
72
|
+
private addTimelineChunk;
|
|
73
|
+
private flushContiguousTimelineFrames;
|
|
74
|
+
private timelineAliases;
|
|
75
|
+
private hasActiveTimelineOwner;
|
|
76
|
+
private responseAlias;
|
|
77
|
+
private turnEpochAlias;
|
|
78
|
+
private turnAlias;
|
|
79
|
+
private activeHasResponseAlias;
|
|
80
|
+
private activeHasTurnEpochAlias;
|
|
81
|
+
private matchesActiveTimelineOwner;
|
|
82
|
+
private matchesActiveTimelineChunkOwner;
|
|
83
|
+
private mergeActiveTimelineAliases;
|
|
84
|
+
private isRetiredTimelineOwner;
|
|
85
|
+
private rememberRetiredTimelineAliases;
|
|
86
|
+
private rememberRetiredOwnerlessTimelineAliases;
|
|
87
|
+
private rememberBoundedAliases;
|
|
88
|
+
private adoptTimelineOwner;
|
|
89
|
+
private clearTimelineOwner;
|
|
90
|
+
/**
|
|
91
|
+
* True when a lifecycle event belongs to an owner that has already been
|
|
92
|
+
* interrupted/cancelled or does not match the active indexed timeline.
|
|
93
|
+
* Ownerless events are treated as current for backward compatibility.
|
|
94
|
+
*/
|
|
95
|
+
isStaleOwner(metadata?: BlendshapeChunkMetadata): boolean;
|
|
96
|
+
/**
|
|
97
|
+
* True when an owner was explicitly interrupted/cancelled/completed.
|
|
98
|
+
* Chunk messages use this narrower predicate so a newer owner can still
|
|
99
|
+
* reach addChunk() and atomically replace the active owner there.
|
|
100
|
+
*/
|
|
101
|
+
isRetiredOwner(metadata?: BlendshapeChunkMetadata): boolean;
|
|
102
|
+
/**
|
|
103
|
+
* Drop buffered ahead-delivered frames for a server-cancelled owner.
|
|
104
|
+
* Returns true when the active owner was flushed.
|
|
51
105
|
*/
|
|
52
|
-
|
|
106
|
+
cancelOwner(metadata?: BlendshapeChunkMetadata): boolean;
|
|
107
|
+
hasBufferedFrames(): boolean;
|
|
53
108
|
/**
|
|
54
109
|
* Set or update the mapper function
|
|
55
110
|
* @param mapper - Function to map/transform blendshape data
|
|
@@ -112,7 +167,11 @@ export declare class BlendshapeQueue {
|
|
|
112
167
|
*/
|
|
113
168
|
isConversationEnded(): boolean;
|
|
114
169
|
/**
|
|
115
|
-
* Check if server
|
|
170
|
+
* Check if the server end signal is safe to expose to playback consumers.
|
|
171
|
+
*
|
|
172
|
+
* Ahead-delivery can deliver blendshape-turn-stats before bot-started-speaking.
|
|
173
|
+
* Keep those early stats internally for expected-frame accounting, but do not
|
|
174
|
+
* expose a public drain signal until audio playback has actually started.
|
|
116
175
|
* This does NOT check if all frames have been consumed - use isConversationEnded() for that.
|
|
117
176
|
*/
|
|
118
177
|
hasReceivedEndSignal(): boolean;
|
|
@@ -135,12 +194,17 @@ export declare class BlendshapeQueue {
|
|
|
135
194
|
* Lipsync player should only play frames when this is true
|
|
136
195
|
*/
|
|
137
196
|
isBotSpeaking(): boolean;
|
|
197
|
+
/**
|
|
198
|
+
* Consume the signal that a newer ahead-delivered owner replaced an active
|
|
199
|
+
* speaking owner before the old bot-stopped-speaking event arrived.
|
|
200
|
+
*/
|
|
201
|
+
consumeOwnerReplacementStoppedSpeaking(): boolean;
|
|
138
202
|
/**
|
|
139
203
|
* Mark conversation end with turn stats
|
|
140
204
|
* Called when blendshape-turn-stats received
|
|
141
205
|
* Stops accepting new frames and marks conversation as ended
|
|
142
206
|
*/
|
|
143
|
-
endConversation(stats?: TurnStats): void;
|
|
207
|
+
endConversation(stats?: TurnStats, metadata?: BlendshapeChunkMetadata): void;
|
|
144
208
|
/**
|
|
145
209
|
* Handle interruption - mark for fade out
|
|
146
210
|
* Keeps last 10 frames in queue and fades them out
|
|
@@ -161,6 +225,10 @@ export declare class BlendshapeQueue {
|
|
|
161
225
|
* Get number of frames consumed
|
|
162
226
|
*/
|
|
163
227
|
getFramesConsumed(): number;
|
|
228
|
+
/**
|
|
229
|
+
* Get playback FPS from chunk metadata, falling back to 60fps.
|
|
230
|
+
*/
|
|
231
|
+
getPlaybackFps(): number;
|
|
164
232
|
/**
|
|
165
233
|
* Calculate time left in milliseconds based on turn stats
|
|
166
234
|
* Returns null if stats not available
|
|
@@ -170,6 +238,7 @@ export declare class BlendshapeQueue {
|
|
|
170
238
|
* Check if all frames have been consumed based on turn stats
|
|
171
239
|
*/
|
|
172
240
|
isAllFramesConsumed(): boolean;
|
|
241
|
+
private hasUnconsumedExpectedFrames;
|
|
173
242
|
/**
|
|
174
243
|
* Reset queue for new conversation
|
|
175
244
|
* Should be called after all frames have been consumed
|
|
@@ -197,6 +266,8 @@ export declare class BlendshapeQueue {
|
|
|
197
266
|
framesConsumed: number;
|
|
198
267
|
turnStats: TurnStats | null;
|
|
199
268
|
timeLeftMs: number | null;
|
|
269
|
+
pendingTimelineFrames: number;
|
|
270
|
+
playbackFps: number;
|
|
200
271
|
};
|
|
201
272
|
}
|
|
202
273
|
//# sourceMappingURL=BlendshapeQueue.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BlendshapeQueue.d.ts","sourceRoot":"","sources":["../../src/core/BlendshapeQueue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB,EAAE,MAAM,CAAC;IAC1B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,kBAAkB,CAAkB;IAC5C,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,MAAM,CAAC,CAAmD;IAClE,OAAO,CAAC,SAAS,CAA0B;IAC3C,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,kBAAkB,CAAa;IACvC,OAAO,CAAC,kBAAkB,CAAkB;IAC5C,OAAO,CAAC,iBAAiB,CAAkB;
|
|
1
|
+
{"version":3,"file":"BlendshapeQueue.d.ts","sourceRoot":"","sources":["../../src/core/BlendshapeQueue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB,EAAE,MAAM,CAAC;IAC1B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,uBAAuB;IACtC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,qBAAqB,CAAmC;IAChE,OAAO,CAAC,qBAAqB,CAAqB;IAClD,OAAO,CAAC,sBAAsB,CAAqB;IACnD,OAAO,CAAC,+BAA+B,CAAqB;IAC5D,OAAO,CAAC,kBAAkB,CAAkB;IAC5C,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,MAAM,CAAC,CAAmD;IAClE,OAAO,CAAC,SAAS,CAA0B;IAC3C,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,kBAAkB,CAAa;IACvC,OAAO,CAAC,kBAAkB,CAAkB;IAC5C,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,+BAA+B,CAAkB;IACzD,OAAO,CAAC,4BAA4B,CAAuB;IAC3D,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IAGnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAM;IACxC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAAM;IAC1D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CACX;IAE3C;;;OAGG;gBAED,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,YAAY,KAAK,YAAY,EACzD,GAAG,GAAE,MAAM,MAAyB;IAMtC;;;;OAIG;IACH,QAAQ,CACN,WAAW,EAAE,MAAM,EAAE,EAAE,EACvB,QAAQ,GAAE,uBAA4B,GACrC,IAAI;IA2EP,OAAO,CAAC,QAAQ;IAIhB,OAAO,CAAC,gBAAgB;IAyBxB,OAAO,CAAC,6BAA6B;IAarC,OAAO,CAAC,eAAe;IAevB,OAAO,CAAC,sBAAsB;IAI9B,OAAO,CAAC,aAAa;IAMrB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,SAAS;IAMjB,OAAO,CAAC,sBAAsB;IAM9B,OAAO,CAAC,uBAAuB;IAM/B,OAAO,CAAC,0BAA0B;IAqBlC,OAAO,CAAC,+BAA+B;IA0BvC,OAAO,CAAC,0BAA0B;IAIlC,OAAO,CAAC,sBAAsB;IA0C9B,OAAO,CAAC,8BAA8B;IAWtC,OAAO,CAAC,uCAAuC;IAW/C,OAAO,CAAC,sBAAsB;IAyB9B,OAAO,CAAC,kBAAkB;IAsB1B,OAAO,CAAC,kBAAkB;IAI1B;;;;OAIG;IACH,YAAY,CAAC,QAAQ,GAAE,uBAA4B,GAAG,OAAO;IAc7D;;;;OAIG;IACH,cAAc,CAAC,QAAQ,GAAE,uBAA4B,GAAG,OAAO;IAI/D;;;OAGG;IACH,WAAW,CAAC,QAAQ,GAAE,uBAA4B,GAAG,OAAO;IA0C5D,iBAAiB,IAAI,OAAO;IAI5B;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,YAAY,KAAK,YAAY,GAAG,IAAI;IAIzE;;OAEG;IACH,WAAW,IAAI,IAAI;IAInB;;OAEG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,SAAS,IAAI,YAAY,EAAE;IAI3B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI;IAO5C;;;;;;OAMG;IACH,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI;IA+CrD;;;OAGG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAQlC;;OAEG;IACH,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;OAEG;IACH,SAAS,IAAI,OAAO;IAIpB;;OAEG;IACH,oBAAoB,IAAI,OAAO;IAI/B;;;;;;;;;;OAUG;IACH,mBAAmB,IAAI,OAAO;IA4C9B;;;;;;;OAOG;IACH,oBAAoB,IAAI,OAAO;IAI/B;;;OAGG;IACH,iBAAiB,IAAI,IAAI;IA+BzB;;;OAGG;IACH,gBAAgB,IAAI,OAAO;IAiB3B;;OAEG;IACH,eAAe,IAAI,IAAI;IAIvB;;;OAGG;IACH,aAAa,IAAI,OAAO;IAIxB;;;OAGG;IACH,sCAAsC,IAAI,OAAO;IAMjD;;;;OAIG;IACH,eAAe,CACb,KAAK,CAAC,EAAE,SAAS,EACjB,QAAQ,GAAE,uBAA4B,GACrC,IAAI;IAoCP;;;OAGG;IACH,SAAS,IAAI,IAAI;IAkBjB;;;;;OAKG;IACH,0BAA0B,IAAI,OAAO;IAMrC;;OAEG;IACH,YAAY,IAAI,SAAS,GAAG,IAAI;IAIhC;;OAEG;IACH,iBAAiB,IAAI,MAAM;IAI3B;;OAEG;IACH,cAAc,IAAI,MAAM;IAIxB;;;OAGG;IACH,aAAa,IAAI,MAAM,GAAG,IAAI;IAiB9B;;OAEG;IACH,mBAAmB,IAAI,OAAO;IAK9B,OAAO,CAAC,2BAA2B;IAOnC;;;OAGG;IACH,KAAK,IAAI,IAAI;IAiBb;;;;OAIG;IACH,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG;QACnC,KAAK,EAAE,YAAY,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;KACpB,GAAG,IAAI;IAiBR;;OAEG;IACH,YAAY,IAAI;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,kBAAkB,EAAE,OAAO,CAAC;QAC5B,WAAW,EAAE,OAAO,CAAC;QACrB,iBAAiB,EAAE,OAAO,CAAC;QAC3B,WAAW,EAAE,OAAO,CAAC;QACrB,gBAAgB,EAAE,OAAO,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;QAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,qBAAqB,EAAE,MAAM,CAAC;QAC9B,WAAW,EAAE,MAAM,CAAC;KACrB;CAeF"}
|