@keyframelabs/sdk 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -8
- package/dist/PersonaSession.d.ts +2 -9
- package/dist/PersonaSession.d.ts.map +1 -1
- package/dist/PersonaSession.js +19 -42
- package/dist/PersonaSession.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +4 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -49,6 +49,7 @@ import { createClient } from '@keyframelabs/sdk';
|
|
|
49
49
|
const persona = createClient({
|
|
50
50
|
serverUrl: "wss://...",
|
|
51
51
|
participantToken: "A6gB...",
|
|
52
|
+
agentIdentity: "agent",
|
|
52
53
|
onVideoTrack: (track) => {
|
|
53
54
|
// Some HTML video element
|
|
54
55
|
videoElement.srcObject = new MediaStream([track]);
|
|
@@ -57,6 +58,9 @@ const persona = createClient({
|
|
|
57
58
|
// Some HTML audio element
|
|
58
59
|
audioElement.srcObject = new MediaStream([track]);
|
|
59
60
|
},
|
|
61
|
+
onAgentStateChange: (state) => {
|
|
62
|
+
console.log('Agent state:', state); // 'listening' | 'speaking'
|
|
63
|
+
},
|
|
60
64
|
});
|
|
61
65
|
|
|
62
66
|
// Connect to the session
|
|
@@ -113,14 +117,17 @@ The SDK is intentionally minimal—it only handles the avatar connection. You br
|
|
|
113
117
|
|
|
114
118
|
Initializes the WebSocket connection and media managers.
|
|
115
119
|
|
|
116
|
-
| Option
|
|
117
|
-
|
|
|
118
|
-
| `serverUrl`
|
|
119
|
-
| `participantToken`
|
|
120
|
-
| `
|
|
121
|
-
| `
|
|
122
|
-
| `
|
|
123
|
-
| `
|
|
120
|
+
| Option | Type | Default | Description |
|
|
121
|
+
| -------------------- | ----------------- | -------- | --------------------------------------------------------- |
|
|
122
|
+
| `serverUrl` | `string` | Required | The WSS URL returned by the `/session` API endpoint. |
|
|
123
|
+
| `participantToken` | `string` | Required | The access token returned by the `/session` API endpoint. |
|
|
124
|
+
| `agentIdentity` | `string` | Required | Identity of the agent participant in the room. |
|
|
125
|
+
| `onVideoTrack` | `(track) => void` | Required | Fired when the WebRTC video track is ready. |
|
|
126
|
+
| `onAudioTrack` | `(track) => void` | Required | Fired when the WebRTC audio track is ready. |
|
|
127
|
+
| `onStateChange` | `(state) => void` | — | Fired when session state changes. |
|
|
128
|
+
| `onAgentStateChange` | `(state) => void` | — | Fired when agent playback state changes (`'listening'` or `'speaking'`). |
|
|
129
|
+
| `onClose` | `(reason) => void` | — | Fired when the session is closed (by server or client). |
|
|
130
|
+
| `onError` | `(err) => void` | — | Fired on errors. |
|
|
124
131
|
|
|
125
132
|
### `PersonaSession`
|
|
126
133
|
|
package/dist/PersonaSession.d.ts
CHANGED
|
@@ -35,7 +35,6 @@ export declare class PersonaSession {
|
|
|
35
35
|
private _state;
|
|
36
36
|
private byteStreamWriter;
|
|
37
37
|
private writeQueue;
|
|
38
|
-
private autoFlushTimer;
|
|
39
38
|
constructor(options: PersonaSessionOptions);
|
|
40
39
|
/** Current session state */
|
|
41
40
|
get state(): SessionState;
|
|
@@ -53,11 +52,6 @@ export declare class PersonaSession {
|
|
|
53
52
|
* @param pcmData - 16-bit PCM audio at 24kHz
|
|
54
53
|
*/
|
|
55
54
|
sendAudio(pcmData: Uint8Array): void;
|
|
56
|
-
/**
|
|
57
|
-
* Schedule auto-flush after a timeout.
|
|
58
|
-
* Called on each sendAudio() to reset the timer.
|
|
59
|
-
*/
|
|
60
|
-
private scheduleAutoFlush;
|
|
61
55
|
/**
|
|
62
56
|
* Flush all pending audio writes.
|
|
63
57
|
* Call before close to ensure all audio is sent.
|
|
@@ -66,9 +60,8 @@ export declare class PersonaSession {
|
|
|
66
60
|
/**
|
|
67
61
|
* End the current audio turn and flush any buffered data.
|
|
68
62
|
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
* if you need immediate flushing.
|
|
63
|
+
* Call this when the voice agent signals that all audio for a turn
|
|
64
|
+
* has been sent (e.g., on the agent's turnEnd event).
|
|
72
65
|
*/
|
|
73
66
|
endAudioTurn(): Promise<void>;
|
|
74
67
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersonaSession.d.ts","sourceRoot":"","sources":["../src/PersonaSession.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAaH,OAAO,KAAK,EAEV,OAAO,EACP,qBAAqB,EACrB,YAAY,EACb,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"PersonaSession.d.ts","sourceRoot":"","sources":["../src/PersonaSession.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAaH,OAAO,KAAK,EAEV,OAAO,EACP,qBAAqB,EACrB,YAAY,EACb,MAAM,YAAY,CAAC;AA4BpB;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,OAAO,CACS;IACxB,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,MAAM,CAAgC;IAC9C,OAAO,CAAC,gBAAgB,CAAiC;IAGzD,OAAO,CAAC,UAAU,CAAoC;gBAE1C,OAAO,EAAE,qBAAqB;IAI1C,4BAA4B;IAC5B,IAAI,KAAK,IAAI,YAAY,CAExB;IAED,OAAO,CAAC,QAAQ;IAOhB;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IA8B9B,OAAO,CAAC,kBAAkB;IAoD1B;;;;OAIG;IACH,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI;IAyBpC;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;;;OAKG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBnC;;;;;;;;OAQG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBhC;;;;;;;OAOG;IACG,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBjD;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAsB7B"}
|
package/dist/PersonaSession.js
CHANGED
|
@@ -28,8 +28,9 @@ function mapDisconnectReason(reason) {
|
|
|
28
28
|
const AUDIO_STREAM_TOPIC = 'lk.audio_stream';
|
|
29
29
|
/** DataStream topic for control messages (interrupt, etc.) */
|
|
30
30
|
const CONTROL_STREAM_TOPIC = 'lk.control';
|
|
31
|
-
/**
|
|
32
|
-
const
|
|
31
|
+
/** RPC methods sent by the GPU worker */
|
|
32
|
+
const RPC_PLAYBACK_STARTED = 'lk.playback_started';
|
|
33
|
+
const RPC_PLAYBACK_FINISHED = 'lk.playback_finished';
|
|
33
34
|
/**
|
|
34
35
|
* PersonaSession manages the connection between your app and a Persona avatar.
|
|
35
36
|
*
|
|
@@ -57,8 +58,6 @@ export class PersonaSession {
|
|
|
57
58
|
byteStreamWriter = null;
|
|
58
59
|
// Write queue for proper ordering and completion tracking
|
|
59
60
|
writeQueue = Promise.resolve();
|
|
60
|
-
// Auto-flush timer for turn end detection
|
|
61
|
-
autoFlushTimer = null;
|
|
62
61
|
constructor(options) {
|
|
63
62
|
this.options = { ...options };
|
|
64
63
|
}
|
|
@@ -131,6 +130,17 @@ export class PersonaSession {
|
|
|
131
130
|
console.error('[PersonaSession] Media devices error:', error);
|
|
132
131
|
this.options.onError?.(error);
|
|
133
132
|
});
|
|
133
|
+
// Register RPC handlers for avatar playback state signals
|
|
134
|
+
this.room.registerRpcMethod(RPC_PLAYBACK_STARTED, async () => {
|
|
135
|
+
console.log('[PersonaSession] Playback started (speaking)');
|
|
136
|
+
this.options.onAgentStateChange?.('speaking');
|
|
137
|
+
return '';
|
|
138
|
+
});
|
|
139
|
+
this.room.registerRpcMethod(RPC_PLAYBACK_FINISHED, async () => {
|
|
140
|
+
console.log('[PersonaSession] Playback finished (listening)');
|
|
141
|
+
this.options.onAgentStateChange?.('listening');
|
|
142
|
+
return '';
|
|
143
|
+
});
|
|
134
144
|
}
|
|
135
145
|
/**
|
|
136
146
|
* Send audio data to the persona for video synthesis.
|
|
@@ -144,7 +154,7 @@ export class PersonaSession {
|
|
|
144
154
|
}
|
|
145
155
|
// Chain writes to ensure ordering and track completion
|
|
146
156
|
// Lazily create ByteStreamWriter if needed (e.g., after endAudioTurn closed it)
|
|
147
|
-
console.
|
|
157
|
+
console.debug(`[PersonaSession] Writing ${pcmData.length} bytes to ByteStream`);
|
|
148
158
|
this.writeQueue = this.writeQueue
|
|
149
159
|
.then(async () => {
|
|
150
160
|
if (!this.byteStreamWriter && this.room) {
|
|
@@ -158,23 +168,6 @@ export class PersonaSession {
|
|
|
158
168
|
.catch((err) => {
|
|
159
169
|
console.warn('[PersonaSession] Failed to write audio:', err);
|
|
160
170
|
});
|
|
161
|
-
// Schedule auto-flush after timeout (reset on each audio chunk)
|
|
162
|
-
this.scheduleAutoFlush();
|
|
163
|
-
}
|
|
164
|
-
/**
|
|
165
|
-
* Schedule auto-flush after a timeout.
|
|
166
|
-
* Called on each sendAudio() to reset the timer.
|
|
167
|
-
*/
|
|
168
|
-
scheduleAutoFlush() {
|
|
169
|
-
if (this.autoFlushTimer) {
|
|
170
|
-
clearTimeout(this.autoFlushTimer);
|
|
171
|
-
}
|
|
172
|
-
this.autoFlushTimer = setTimeout(() => {
|
|
173
|
-
if (this.byteStreamWriter) {
|
|
174
|
-
console.log('[PersonaSession] Auto-flush: no audio for', AUTO_FLUSH_TIMEOUT_MS, 'ms');
|
|
175
|
-
this.endAudioTurn();
|
|
176
|
-
}
|
|
177
|
-
}, AUTO_FLUSH_TIMEOUT_MS);
|
|
178
171
|
}
|
|
179
172
|
/**
|
|
180
173
|
* Flush all pending audio writes.
|
|
@@ -186,16 +179,10 @@ export class PersonaSession {
|
|
|
186
179
|
/**
|
|
187
180
|
* End the current audio turn and flush any buffered data.
|
|
188
181
|
*
|
|
189
|
-
*
|
|
190
|
-
*
|
|
191
|
-
* if you need immediate flushing.
|
|
182
|
+
* Call this when the voice agent signals that all audio for a turn
|
|
183
|
+
* has been sent (e.g., on the agent's turnEnd event).
|
|
192
184
|
*/
|
|
193
185
|
async endAudioTurn() {
|
|
194
|
-
// Clear auto-flush timer to prevent double-flush
|
|
195
|
-
if (this.autoFlushTimer) {
|
|
196
|
-
clearTimeout(this.autoFlushTimer);
|
|
197
|
-
this.autoFlushTimer = null;
|
|
198
|
-
}
|
|
199
186
|
console.debug('[PersonaSession] endAudioTurn() called, awaiting writeQueue...');
|
|
200
187
|
await this.writeQueue;
|
|
201
188
|
console.debug('[PersonaSession] writeQueue complete, closing stream...');
|
|
@@ -223,11 +210,6 @@ export class PersonaSession {
|
|
|
223
210
|
* while the agent is still talking).
|
|
224
211
|
*/
|
|
225
212
|
async interrupt() {
|
|
226
|
-
// Clear auto-flush timer - we're interrupting, don't need to flush
|
|
227
|
-
if (this.autoFlushTimer) {
|
|
228
|
-
clearTimeout(this.autoFlushTimer);
|
|
229
|
-
this.autoFlushTimer = null;
|
|
230
|
-
}
|
|
231
213
|
if (!this.room || this._state !== 'connected') {
|
|
232
214
|
console.warn('[PersonaSession] interrupt() called but not connected');
|
|
233
215
|
return;
|
|
@@ -255,9 +237,9 @@ export class PersonaSession {
|
|
|
255
237
|
return;
|
|
256
238
|
}
|
|
257
239
|
try {
|
|
258
|
-
console.
|
|
240
|
+
console.debug('[PersonaSession] Setting emotion:', emotion);
|
|
259
241
|
await this.room.localParticipant.publishData(new TextEncoder().encode(`emotion:${emotion}`), { reliable: false, topic: CONTROL_STREAM_TOPIC, destinationIdentities: [this.options.agentIdentity] });
|
|
260
|
-
console.
|
|
242
|
+
console.debug('[PersonaSession] Emotion sent:', emotion);
|
|
261
243
|
}
|
|
262
244
|
catch (error) {
|
|
263
245
|
console.warn('[PersonaSession] Failed to set emotion:', error);
|
|
@@ -267,11 +249,6 @@ export class PersonaSession {
|
|
|
267
249
|
* Close the session and clean up resources.
|
|
268
250
|
*/
|
|
269
251
|
async close() {
|
|
270
|
-
// Clear auto-flush timer
|
|
271
|
-
if (this.autoFlushTimer) {
|
|
272
|
-
clearTimeout(this.autoFlushTimer);
|
|
273
|
-
this.autoFlushTimer = null;
|
|
274
|
-
}
|
|
275
252
|
// Wait for pending writes to complete before closing
|
|
276
253
|
await this.flush();
|
|
277
254
|
// Close byte stream
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersonaSession.js","sourceRoot":"","sources":["../src/PersonaSession.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,IAAI,EACJ,SAAS,EACT,KAAK,EAGL,gBAAgB,GACjB,MAAM,gBAAgB,CAAC;AAWxB,sDAAsD;AACtD,SAAS,mBAAmB,CAAC,MAAyB;IACpD,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,gBAAgB,CAAC,gBAAgB;YACpC,OAAO,kBAAkB,CAAC;QAC5B,KAAK,gBAAgB,CAAC,YAAY;YAChC,OAAO,cAAc,CAAC;QACxB,KAAK,gBAAgB,CAAC,eAAe;YACnC,OAAO,iBAAiB,CAAC;QAC3B,KAAK,gBAAgB,CAAC,mBAAmB;YACvC,OAAO,qBAAqB,CAAC;QAC/B;YACE,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,kBAAkB,GAAG,iBAAiB,CAAC;AAE7C,8DAA8D;AAC9D,MAAM,oBAAoB,GAAG,YAAY,CAAC;AAE1C,
|
|
1
|
+
{"version":3,"file":"PersonaSession.js","sourceRoot":"","sources":["../src/PersonaSession.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,IAAI,EACJ,SAAS,EACT,KAAK,EAGL,gBAAgB,GACjB,MAAM,gBAAgB,CAAC;AAWxB,sDAAsD;AACtD,SAAS,mBAAmB,CAAC,MAAyB;IACpD,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,gBAAgB,CAAC,gBAAgB;YACpC,OAAO,kBAAkB,CAAC;QAC5B,KAAK,gBAAgB,CAAC,YAAY;YAChC,OAAO,cAAc,CAAC;QACxB,KAAK,gBAAgB,CAAC,eAAe;YACnC,OAAO,iBAAiB,CAAC;QAC3B,KAAK,gBAAgB,CAAC,mBAAmB;YACvC,OAAO,qBAAqB,CAAC;QAC/B;YACE,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,kBAAkB,GAAG,iBAAiB,CAAC;AAE7C,8DAA8D;AAC9D,MAAM,oBAAoB,GAAG,YAAY,CAAC;AAE1C,yCAAyC;AACzC,MAAM,oBAAoB,GAAG,qBAAqB,CAAC;AACnD,MAAM,qBAAqB,GAAG,sBAAsB,CAAC;AAErD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,OAAO,cAAc;IACjB,OAAO,CACS;IAChB,IAAI,GAAgB,IAAI,CAAC;IACzB,MAAM,GAAiB,cAAc,CAAC;IACtC,gBAAgB,GAA4B,IAAI,CAAC;IAEzD,0DAA0D;IAClD,UAAU,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEtD,YAAY,OAA8B;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAED,4BAA4B;IAC5B,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAEO,QAAQ,CAAC,KAAmB;QAClC,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE5B,IAAI,CAAC;YACH,6BAA6B;YAC7B,IAAI,CAAC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAE1B,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;YAE/E,4BAA4B;YAC5B,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC;gBACnE,KAAK,EAAE,kBAAkB;gBACzB,qBAAqB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;aACpD,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;YAEpD,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAClF,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,kBAAkB;QACxB,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QAEvB,6BAA6B;QAC7B,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC,KAAkB,EAAE,IAAI,EAAE,WAA8B,EAAE,EAAE;YACnG,IAAI,WAAW,CAAC,QAAQ,KAAK,IAAI,CAAC,OAAO,CAAC,aAAa;gBAAE,OAAO;YAEhE,MAAM,UAAU,GAAG,KAAK,CAAC,gBAAgB,CAAC;YAE1C,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;gBAC7D,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,UAAU,CAAC,CAAC;YAC1C,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3C,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;gBAC7D,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,UAAU,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,uBAAuB;QACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,MAAyB,EAAE,EAAE;YACjE,MAAM,WAAW,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAChD,OAAO,CAAC,GAAG,CAAC,0CAA0C,EAAE,WAAW,CAAC,CAAC;YACrE,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,gBAAgB;QAChB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,iBAAiB,EAAE,CAAC,KAAY,EAAE,EAAE;YACzD,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,KAAK,CAAC,CAAC;YAC9D,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,0DAA0D;QAC1D,IAAI,CAAC,IAAI,CAAC,iBAAiB,CACzB,oBAAoB,EACpB,KAAK,IAAI,EAAE;YACT,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;YAC5D,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,UAAU,CAAC,CAAC;YAC9C,OAAO,EAAE,CAAC;QACZ,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,iBAAiB,CACzB,qBAAqB,EACrB,KAAK,IAAI,EAAE;YACT,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC9D,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,WAAW,CAAC,CAAC;YAC/C,OAAO,EAAE,CAAC;QACZ,CAAC,CACF,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,OAAmB;QAC3B,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;YACnE,OAAO;QACT,CAAC;QAED,uDAAuD;QACvD,gFAAgF;QAChF,OAAO,CAAC,KAAK,CAAC,4BAA4B,OAAO,CAAC,MAAM,sBAAsB,CAAC,CAAC;QAChF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU;aAC9B,IAAI,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACxC,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC;oBACnE,KAAK,EAAE,kBAAkB;oBACzB,qBAAqB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;iBACpD,CAAC,CAAC;YACL,CAAC;YACD,MAAM,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9C,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,OAAO,CAAC,IAAI,CAAC,yCAAyC,EAAE,GAAG,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IAEP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,UAAU,CAAC;IACxB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY;QAChB,OAAO,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAChF,MAAM,IAAI,CAAC,UAAU,CAAC;QACtB,OAAO,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QACzE,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;gBACpC,OAAO,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;YAC/D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAE,GAAG,CAAC,CAAC;YAC5D,CAAC;YACD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;YACtE,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAC1C,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EACrC,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CACtG,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,4CAA4C,EAAE,KAAK,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU,CAAC,OAAgB;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACvE,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,OAAO,CAAC,CAAC;YAC5D,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAC1C,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,WAAW,OAAO,EAAE,CAAC,EAC9C,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CACtG,CAAC;YACF,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,OAAO,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,qDAAqD;QACrD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAEnB,oBAAoB;QACpB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;YACD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC/B,CAAC;QAED,uBAAuB;QACvB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACnB,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;IAChC,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -30,6 +30,6 @@ import type { PersonaSessionOptions } from './types.js';
|
|
|
30
30
|
*/
|
|
31
31
|
declare const createClient: (options: PersonaSessionOptions) => PersonaSession;
|
|
32
32
|
export { createClient, PersonaSession };
|
|
33
|
-
export type { CloseReason, Emotion, PersonaSessionOptions, SessionState, } from './types.js';
|
|
33
|
+
export type { AgentState, CloseReason, Emotion, PersonaSessionOptions, SessionState, } from './types.js';
|
|
34
34
|
export { SAMPLE_RATE, base64ToBytes, bytesToBase64, createEventEmitter, resamplePcm, } from './audio-utils.js';
|
|
35
35
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAExD;;;;;GAKG;AACH,QAAA,MAAM,YAAY,GAAI,SAAS,qBAAqB,KAAG,cAEtD,CAAC;AAGF,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC;AAGxC,YAAY,EACV,WAAW,EACX,OAAO,EACP,qBAAqB,EACrB,YAAY,GACb,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,WAAW,EACX,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,WAAW,GACZ,MAAM,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAExD;;;;;GAKG;AACH,QAAA,MAAM,YAAY,GAAI,SAAS,qBAAqB,KAAG,cAEtD,CAAC;AAGF,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC;AAGxC,YAAY,EACV,UAAU,EACV,WAAW,EACX,OAAO,EACP,qBAAqB,EACrB,YAAY,GACb,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,WAAW,EACX,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,WAAW,GACZ,MAAM,kBAAkB,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAGrD;;;;;GAKG;AACH,MAAM,YAAY,GAAG,CAAC,OAA8B,EAAkB,EAAE;IACtE,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;AACrC,CAAC,CAAC;AAEF,OAAO;AACP,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAGrD;;;;;GAKG;AACH,MAAM,YAAY,GAAG,CAAC,OAA8B,EAAkB,EAAE;IACtE,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;AACrC,CAAC,CAAC;AAEF,OAAO;AACP,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC;AAWxC,YAAY;AACZ,OAAO,EACL,WAAW,EACX,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,WAAW,GACZ,MAAM,kBAAkB,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -7,6 +7,8 @@ export type Emotion = 'neutral' | 'angry' | 'sad' | 'happy';
|
|
|
7
7
|
export type SessionState = 'disconnected' | 'connecting' | 'connected' | 'error';
|
|
8
8
|
/** Reason for session close */
|
|
9
9
|
export type CloseReason = 'client_initiated' | 'room_deleted' | 'server_shutdown' | 'participant_removed' | 'unknown';
|
|
10
|
+
/** Avatar playback state, signaled by the GPU worker via RPC */
|
|
11
|
+
export type AgentState = 'listening' | 'speaking';
|
|
10
12
|
/** Options for creating a PersonaSession */
|
|
11
13
|
export interface PersonaSessionOptions {
|
|
12
14
|
/** WebRTC URL (returned from KeyframeLab API) */
|
|
@@ -21,6 +23,8 @@ export interface PersonaSessionOptions {
|
|
|
21
23
|
onAudioTrack?: (track: MediaStreamTrack) => void;
|
|
22
24
|
/** Called when session state changes */
|
|
23
25
|
onStateChange?: (state: SessionState) => void;
|
|
26
|
+
/** Called when avatar playback state changes (listening/speaking) */
|
|
27
|
+
onAgentStateChange?: (state: AgentState) => void;
|
|
24
28
|
/** Called on error */
|
|
25
29
|
onError?: (error: Error) => void;
|
|
26
30
|
/** Called when session is closed (by server or client) */
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,2CAA2C;AAC3C,MAAM,MAAM,OAAO,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC;AAE5D,4BAA4B;AAC5B,MAAM,MAAM,YAAY,GACpB,cAAc,GACd,YAAY,GACZ,WAAW,GACX,OAAO,CAAC;AAEZ,+BAA+B;AAC/B,MAAM,MAAM,WAAW,GACnB,kBAAkB,GAClB,cAAc,GACd,iBAAiB,GACjB,qBAAqB,GACrB,SAAS,CAAC;AAEd,4CAA4C;AAC5C,MAAM,WAAW,qBAAqB;IACpC,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,gBAAgB,EAAE,MAAM,CAAC;IACzB,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IAEtB,2CAA2C;IAC3C,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAEjD,2CAA2C;IAC3C,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAEjD,wCAAwC;IACxC,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IAE9C,sBAAsB;IACtB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IAEjC,0DAA0D;IAC1D,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,CAAC;CACzC"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,2CAA2C;AAC3C,MAAM,MAAM,OAAO,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC;AAE5D,4BAA4B;AAC5B,MAAM,MAAM,YAAY,GACpB,cAAc,GACd,YAAY,GACZ,WAAW,GACX,OAAO,CAAC;AAEZ,+BAA+B;AAC/B,MAAM,MAAM,WAAW,GACnB,kBAAkB,GAClB,cAAc,GACd,iBAAiB,GACjB,qBAAqB,GACrB,SAAS,CAAC;AAEd,gEAAgE;AAChE,MAAM,MAAM,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAElD,4CAA4C;AAC5C,MAAM,WAAW,qBAAqB;IACpC,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,gBAAgB,EAAE,MAAM,CAAC;IACzB,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IAEtB,2CAA2C;IAC3C,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAEjD,2CAA2C;IAC3C,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAEjD,wCAAwC;IACxC,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IAE9C,qEAAqE;IACrE,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;IAEjD,sBAAsB;IACtB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IAEjC,0DAA0D;IAC1D,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,CAAC;CACzC"}
|