@keyframelabs/sdk 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -8
- package/dist/PersonaSession.d.ts.map +1 -1
- package/dist/PersonaSession.js +44 -31
- package/dist/PersonaSession.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +4 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -49,6 +49,7 @@ import { createClient } from '@keyframelabs/sdk';
|
|
|
49
49
|
const persona = createClient({
|
|
50
50
|
serverUrl: "wss://...",
|
|
51
51
|
participantToken: "A6gB...",
|
|
52
|
+
agentIdentity: "agent",
|
|
52
53
|
onVideoTrack: (track) => {
|
|
53
54
|
// Some HTML video element
|
|
54
55
|
videoElement.srcObject = new MediaStream([track]);
|
|
@@ -57,6 +58,9 @@ const persona = createClient({
|
|
|
57
58
|
// Some HTML audio element
|
|
58
59
|
audioElement.srcObject = new MediaStream([track]);
|
|
59
60
|
},
|
|
61
|
+
onAgentStateChange: (state) => {
|
|
62
|
+
console.log('Agent state:', state); // 'listening' | 'speaking'
|
|
63
|
+
},
|
|
60
64
|
});
|
|
61
65
|
|
|
62
66
|
// Connect to the session
|
|
@@ -113,14 +117,17 @@ The SDK is intentionally minimal—it only handles the avatar connection. You br
|
|
|
113
117
|
|
|
114
118
|
Initializes the WebSocket connection and media managers.
|
|
115
119
|
|
|
116
|
-
| Option
|
|
117
|
-
|
|
|
118
|
-
| `serverUrl`
|
|
119
|
-
| `participantToken`
|
|
120
|
-
| `
|
|
121
|
-
| `
|
|
122
|
-
| `
|
|
123
|
-
| `
|
|
120
|
+
| Option | Type | Default | Description |
|
|
121
|
+
| -------------------- | ----------------- | -------- | --------------------------------------------------------- |
|
|
122
|
+
| `serverUrl` | `string` | Required | The WSS URL returned by the `/session` API endpoint. |
|
|
123
|
+
| `participantToken` | `string` | Required | The access token returned by the `/session` API endpoint. |
|
|
124
|
+
| `agentIdentity` | `string` | Required | Identity of the agent participant in the room. |
|
|
125
|
+
| `onVideoTrack` | `(track) => void` | Required | Fired when the WebRTC video track is ready. |
|
|
126
|
+
| `onAudioTrack` | `(track) => void` | Required | Fired when the WebRTC audio track is ready. |
|
|
127
|
+
| `onStateChange` | `(state) => void` | — | Fired when session state changes. |
|
|
128
|
+
| `onAgentStateChange` | `(state) => void` | — | Fired when agent playback state changes (`'listening'` or `'speaking'`). |
|
|
129
|
+
| `onClose` | `(reason) => void` | — | Fired when the session is closed (by server or client). |
|
|
130
|
+
| `onError` | `(err) => void` | — | Fired on errors. |
|
|
124
131
|
|
|
125
132
|
### `PersonaSession`
|
|
126
133
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersonaSession.d.ts","sourceRoot":"","sources":["../src/PersonaSession.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAaH,OAAO,KAAK,EAEV,OAAO,EACP,qBAAqB,EACrB,YAAY,EACb,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"PersonaSession.d.ts","sourceRoot":"","sources":["../src/PersonaSession.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAaH,OAAO,KAAK,EAEV,OAAO,EACP,qBAAqB,EACrB,YAAY,EACb,MAAM,YAAY,CAAC;AA4BpB;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,OAAO,CACS;IACxB,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,MAAM,CAAgC;IAC9C,OAAO,CAAC,gBAAgB,CAAiC;IAGzD,OAAO,CAAC,UAAU,CAAoC;gBAE1C,OAAO,EAAE,qBAAqB;IAI1C,4BAA4B;IAC5B,IAAI,KAAK,IAAI,YAAY,CAExB;IAED,OAAO,CAAC,QAAQ;IAOhB;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IA8B9B,OAAO,CAAC,kBAAkB;IAoD1B;;;;OAIG;IACH,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI;IAuBpC;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;;;OAKG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBnC;;;;;;;;OAQG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBhC;;;;;;;OAOG;IACG,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBjD;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAoB7B"}
|
package/dist/PersonaSession.js
CHANGED
|
@@ -28,6 +28,9 @@ function mapDisconnectReason(reason) {
|
|
|
28
28
|
const AUDIO_STREAM_TOPIC = 'lk.audio_stream';
|
|
29
29
|
/** DataStream topic for control messages (interrupt, etc.) */
|
|
30
30
|
const CONTROL_STREAM_TOPIC = 'lk.control';
|
|
31
|
+
/** RPC methods sent by the GPU worker */
|
|
32
|
+
const RPC_PLAYBACK_STARTED = 'lk.playback_started';
|
|
33
|
+
const RPC_PLAYBACK_FINISHED = 'lk.playback_finished';
|
|
31
34
|
/**
|
|
32
35
|
* PersonaSession manages the connection between your app and a Persona avatar.
|
|
33
36
|
*
|
|
@@ -127,6 +130,17 @@ export class PersonaSession {
|
|
|
127
130
|
console.error('[PersonaSession] Media devices error:', error);
|
|
128
131
|
this.options.onError?.(error);
|
|
129
132
|
});
|
|
133
|
+
// Register RPC handlers for avatar playback state signals
|
|
134
|
+
this.room.registerRpcMethod(RPC_PLAYBACK_STARTED, async () => {
|
|
135
|
+
console.log('[PersonaSession] Playback started (speaking)');
|
|
136
|
+
this.options.onAgentStateChange?.('speaking');
|
|
137
|
+
return '';
|
|
138
|
+
});
|
|
139
|
+
this.room.registerRpcMethod(RPC_PLAYBACK_FINISHED, async () => {
|
|
140
|
+
console.log('[PersonaSession] Playback finished (listening)');
|
|
141
|
+
this.options.onAgentStateChange?.('listening');
|
|
142
|
+
return '';
|
|
143
|
+
});
|
|
130
144
|
}
|
|
131
145
|
/**
|
|
132
146
|
* Send audio data to the persona for video synthesis.
|
|
@@ -138,18 +152,18 @@ export class PersonaSession {
|
|
|
138
152
|
console.warn('[PersonaSession] sendAudio dropped - not connected');
|
|
139
153
|
return;
|
|
140
154
|
}
|
|
141
|
-
// Chain writes to ensure ordering and track completion
|
|
142
|
-
// Lazily create ByteStreamWriter if needed (e.g., after endAudioTurn closed it)
|
|
143
155
|
console.debug(`[PersonaSession] Writing ${pcmData.length} bytes to ByteStream`);
|
|
144
156
|
this.writeQueue = this.writeQueue
|
|
145
157
|
.then(async () => {
|
|
146
|
-
if (!this.
|
|
158
|
+
if (!this.room || this._state !== 'connected')
|
|
159
|
+
return;
|
|
160
|
+
if (!this.byteStreamWriter) {
|
|
147
161
|
this.byteStreamWriter = await this.room.localParticipant.streamBytes({
|
|
148
162
|
topic: AUDIO_STREAM_TOPIC,
|
|
149
163
|
destinationIdentities: [this.options.agentIdentity],
|
|
150
164
|
});
|
|
151
165
|
}
|
|
152
|
-
await this.byteStreamWriter
|
|
166
|
+
await this.byteStreamWriter.write(pcmData);
|
|
153
167
|
})
|
|
154
168
|
.catch((err) => {
|
|
155
169
|
console.warn('[PersonaSession] Failed to write audio:', err);
|
|
@@ -169,22 +183,22 @@ export class PersonaSession {
|
|
|
169
183
|
* has been sent (e.g., on the agent's turnEnd event).
|
|
170
184
|
*/
|
|
171
185
|
async endAudioTurn() {
|
|
172
|
-
console.debug('[PersonaSession] endAudioTurn() called
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
186
|
+
console.debug('[PersonaSession] endAudioTurn() called');
|
|
187
|
+
const closeOp = this.writeQueue.then(async () => {
|
|
188
|
+
if (this.byteStreamWriter) {
|
|
189
|
+
const writer = this.byteStreamWriter;
|
|
190
|
+
this.byteStreamWriter = null;
|
|
191
|
+
try {
|
|
192
|
+
await writer.close();
|
|
193
|
+
console.debug('[PersonaSession] stream closed successfully');
|
|
194
|
+
}
|
|
195
|
+
catch (err) {
|
|
196
|
+
console.warn('[PersonaSession] stream close error:', err);
|
|
197
|
+
}
|
|
182
198
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
console.debug('[PersonaSession] no stream to close');
|
|
187
|
-
}
|
|
199
|
+
});
|
|
200
|
+
this.writeQueue = closeOp;
|
|
201
|
+
await closeOp;
|
|
188
202
|
}
|
|
189
203
|
/**
|
|
190
204
|
* Signal an interruption to the avatar.
|
|
@@ -235,19 +249,18 @@ export class PersonaSession {
|
|
|
235
249
|
* Close the session and clean up resources.
|
|
236
250
|
*/
|
|
237
251
|
async close() {
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
252
|
+
this.writeQueue = this.writeQueue.then(async () => {
|
|
253
|
+
if (this.byteStreamWriter) {
|
|
254
|
+
try {
|
|
255
|
+
await this.byteStreamWriter.close();
|
|
256
|
+
}
|
|
257
|
+
catch {
|
|
258
|
+
// Ignore close errors during teardown
|
|
259
|
+
}
|
|
260
|
+
this.byteStreamWriter = null;
|
|
247
261
|
}
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
// Disconnect from room
|
|
262
|
+
});
|
|
263
|
+
await this.writeQueue;
|
|
251
264
|
if (this.room) {
|
|
252
265
|
this.room.disconnect();
|
|
253
266
|
this.room = null;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersonaSession.js","sourceRoot":"","sources":["../src/PersonaSession.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,IAAI,EACJ,SAAS,EACT,KAAK,EAGL,gBAAgB,GACjB,MAAM,gBAAgB,CAAC;AAWxB,sDAAsD;AACtD,SAAS,mBAAmB,CAAC,MAAyB;IACpD,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,gBAAgB,CAAC,gBAAgB;YACpC,OAAO,kBAAkB,CAAC;QAC5B,KAAK,gBAAgB,CAAC,YAAY;YAChC,OAAO,cAAc,CAAC;QACxB,KAAK,gBAAgB,CAAC,eAAe;YACnC,OAAO,iBAAiB,CAAC;QAC3B,KAAK,gBAAgB,CAAC,mBAAmB;YACvC,OAAO,qBAAqB,CAAC;QAC/B;YACE,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,kBAAkB,GAAG,iBAAiB,CAAC;AAE7C,8DAA8D;AAC9D,MAAM,oBAAoB,GAAG,YAAY,CAAC;AAE1C;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,OAAO,cAAc;IACjB,OAAO,CACS;IAChB,IAAI,GAAgB,IAAI,CAAC;IACzB,MAAM,GAAiB,cAAc,CAAC;IACtC,gBAAgB,GAA4B,IAAI,CAAC;IAEzD,0DAA0D;IAClD,UAAU,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEtD,YAAY,OAA8B;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAED,4BAA4B;IAC5B,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAEO,QAAQ,CAAC,KAAmB;QAClC,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE5B,IAAI,CAAC;YACH,6BAA6B;YAC7B,IAAI,CAAC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAE1B,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;YAE/E,4BAA4B;YAC5B,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC;gBACnE,KAAK,EAAE,kBAAkB;gBACzB,qBAAqB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;aACpD,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;YAEpD,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAClF,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,kBAAkB;QACxB,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QAEvB,6BAA6B;QAC7B,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC,KAAkB,EAAE,IAAI,EAAE,WAA8B,EAAE,EAAE;YACnG,IAAI,WAAW,CAAC,QAAQ,KAAK,IAAI,CAAC,OAAO,CAAC,aAAa;gBAAE,OAAO;YAEhE,MAAM,UAAU,GAAG,KAAK,CAAC,gBAAgB,CAAC;YAE1C,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;gBAC7D,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,UAAU,CAAC,CAAC;YAC1C,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3C,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;gBAC7D,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,UAAU,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,uBAAuB;QACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,MAAyB,EAAE,EAAE;YACjE,MAAM,WAAW,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAChD,OAAO,CAAC,GAAG,CAAC,0CAA0C,EAAE,WAAW,CAAC,CAAC;YACrE,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,gBAAgB;QAChB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,iBAAiB,EAAE,CAAC,KAAY,EAAE,EAAE;YACzD,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,KAAK,CAAC,CAAC;YAC9D,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"PersonaSession.js","sourceRoot":"","sources":["../src/PersonaSession.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,IAAI,EACJ,SAAS,EACT,KAAK,EAGL,gBAAgB,GACjB,MAAM,gBAAgB,CAAC;AAWxB,sDAAsD;AACtD,SAAS,mBAAmB,CAAC,MAAyB;IACpD,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,gBAAgB,CAAC,gBAAgB;YACpC,OAAO,kBAAkB,CAAC;QAC5B,KAAK,gBAAgB,CAAC,YAAY;YAChC,OAAO,cAAc,CAAC;QACxB,KAAK,gBAAgB,CAAC,eAAe;YACnC,OAAO,iBAAiB,CAAC;QAC3B,KAAK,gBAAgB,CAAC,mBAAmB;YACvC,OAAO,qBAAqB,CAAC;QAC/B;YACE,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,kBAAkB,GAAG,iBAAiB,CAAC;AAE7C,8DAA8D;AAC9D,MAAM,oBAAoB,GAAG,YAAY,CAAC;AAE1C,yCAAyC;AACzC,MAAM,oBAAoB,GAAG,qBAAqB,CAAC;AACnD,MAAM,qBAAqB,GAAG,sBAAsB,CAAC;AAErD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,OAAO,cAAc;IACjB,OAAO,CACS;IAChB,IAAI,GAAgB,IAAI,CAAC;IACzB,MAAM,GAAiB,cAAc,CAAC;IACtC,gBAAgB,GAA4B,IAAI,CAAC;IAEzD,0DAA0D;IAClD,UAAU,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEtD,YAAY,OAA8B;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAED,4BAA4B;IAC5B,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAEO,QAAQ,CAAC,KAAmB;QAClC,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE5B,IAAI,CAAC;YACH,6BAA6B;YAC7B,IAAI,CAAC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAE1B,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;YAE/E,4BAA4B;YAC5B,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC;gBACnE,KAAK,EAAE,kBAAkB;gBACzB,qBAAqB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;aACpD,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;YAEpD,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAClF,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,kBAAkB;QACxB,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QAEvB,6BAA6B;QAC7B,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC,KAAkB,EAAE,IAAI,EAAE,WAA8B,EAAE,EAAE;YACnG,IAAI,WAAW,CAAC,QAAQ,KAAK,IAAI,CAAC,OAAO,CAAC,aAAa;gBAAE,OAAO;YAEhE,MAAM,UAAU,GAAG,KAAK,CAAC,gBAAgB,CAAC;YAE1C,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;gBAC7D,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,UAAU,CAAC,CAAC;YAC1C,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3C,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;gBAC7D,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,UAAU,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,uBAAuB;QACvB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,MAAyB,EAAE,EAAE;YACjE,MAAM,WAAW,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAChD,OAAO,CAAC,GAAG,CAAC,0CAA0C,EAAE,WAAW,CAAC,CAAC;YACrE,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,gBAAgB;QAChB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,iBAAiB,EAAE,CAAC,KAAY,EAAE,EAAE;YACzD,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,KAAK,CAAC,CAAC;YAC9D,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,0DAA0D;QAC1D,IAAI,CAAC,IAAI,CAAC,iBAAiB,CACzB,oBAAoB,EACpB,KAAK,IAAI,EAAE;YACT,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;YAC5D,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,UAAU,CAAC,CAAC;YAC9C,OAAO,EAAE,CAAC;QACZ,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,iBAAiB,CACzB,qBAAqB,EACrB,KAAK,IAAI,EAAE;YACT,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC9D,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,WAAW,CAAC,CAAC;YAC/C,OAAO,EAAE,CAAC;QACZ,CAAC,CACF,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,OAAmB;QAC3B,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;YACnE,OAAO;QACT,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,4BAA4B,OAAO,CAAC,MAAM,sBAAsB,CAAC,CAAC;QAChF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU;aAC9B,IAAI,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW;gBAAE,OAAO;YACtD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC3B,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC;oBACnE,KAAK,EAAE,kBAAkB;oBACzB,qBAAqB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;iBACpD,CAAC,CAAC;YACL,CAAC;YACD,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7C,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,OAAO,CAAC,IAAI,CAAC,yCAAyC,EAAE,GAAG,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,UAAU,CAAC;IACxB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY;QAChB,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC9C,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC;gBACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;oBACrB,OAAO,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;gBAC/D,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAE,GAAG,CAAC,CAAC;gBAC5D,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC;QAC1B,MAAM,OAAO,CAAC;IAChB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;YACtE,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAC1C,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EACrC,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CACtG,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,4CAA4C,EAAE,KAAK,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU,CAAC,OAAgB;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACvE,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,OAAO,CAAC,CAAC;YAC5D,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAC1C,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,WAAW,OAAO,EAAE,CAAC,EAC9C,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CACtG,CAAC;YACF,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,OAAO,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAChD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC1B,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;gBACtC,CAAC;gBAAC,MAAM,CAAC;oBACP,sCAAsC;gBACxC,CAAC;gBACD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,UAAU,CAAC;QAEtB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACnB,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;IAChC,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -30,6 +30,6 @@ import type { PersonaSessionOptions } from './types.js';
|
|
|
30
30
|
*/
|
|
31
31
|
declare const createClient: (options: PersonaSessionOptions) => PersonaSession;
|
|
32
32
|
export { createClient, PersonaSession };
|
|
33
|
-
export type { CloseReason, Emotion, PersonaSessionOptions, SessionState, } from './types.js';
|
|
33
|
+
export type { AgentState, CloseReason, Emotion, PersonaSessionOptions, SessionState, } from './types.js';
|
|
34
34
|
export { SAMPLE_RATE, base64ToBytes, bytesToBase64, createEventEmitter, resamplePcm, } from './audio-utils.js';
|
|
35
35
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAExD;;;;;GAKG;AACH,QAAA,MAAM,YAAY,GAAI,SAAS,qBAAqB,KAAG,cAEtD,CAAC;AAGF,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC;AAGxC,YAAY,EACV,WAAW,EACX,OAAO,EACP,qBAAqB,EACrB,YAAY,GACb,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,WAAW,EACX,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,WAAW,GACZ,MAAM,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAExD;;;;;GAKG;AACH,QAAA,MAAM,YAAY,GAAI,SAAS,qBAAqB,KAAG,cAEtD,CAAC;AAGF,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC;AAGxC,YAAY,EACV,UAAU,EACV,WAAW,EACX,OAAO,EACP,qBAAqB,EACrB,YAAY,GACb,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,WAAW,EACX,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,WAAW,GACZ,MAAM,kBAAkB,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAGrD;;;;;GAKG;AACH,MAAM,YAAY,GAAG,CAAC,OAA8B,EAAkB,EAAE;IACtE,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;AACrC,CAAC,CAAC;AAEF,OAAO;AACP,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAGrD;;;;;GAKG;AACH,MAAM,YAAY,GAAG,CAAC,OAA8B,EAAkB,EAAE;IACtE,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;AACrC,CAAC,CAAC;AAEF,OAAO;AACP,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC;AAWxC,YAAY;AACZ,OAAO,EACL,WAAW,EACX,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,WAAW,GACZ,MAAM,kBAAkB,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -7,6 +7,8 @@ export type Emotion = 'neutral' | 'angry' | 'sad' | 'happy';
|
|
|
7
7
|
export type SessionState = 'disconnected' | 'connecting' | 'connected' | 'error';
|
|
8
8
|
/** Reason for session close */
|
|
9
9
|
export type CloseReason = 'client_initiated' | 'room_deleted' | 'server_shutdown' | 'participant_removed' | 'unknown';
|
|
10
|
+
/** Avatar playback state, signaled by the GPU worker via RPC */
|
|
11
|
+
export type AgentState = 'listening' | 'speaking';
|
|
10
12
|
/** Options for creating a PersonaSession */
|
|
11
13
|
export interface PersonaSessionOptions {
|
|
12
14
|
/** WebRTC URL (returned from KeyframeLab API) */
|
|
@@ -21,6 +23,8 @@ export interface PersonaSessionOptions {
|
|
|
21
23
|
onAudioTrack?: (track: MediaStreamTrack) => void;
|
|
22
24
|
/** Called when session state changes */
|
|
23
25
|
onStateChange?: (state: SessionState) => void;
|
|
26
|
+
/** Called when avatar playback state changes (listening/speaking) */
|
|
27
|
+
onAgentStateChange?: (state: AgentState) => void;
|
|
24
28
|
/** Called on error */
|
|
25
29
|
onError?: (error: Error) => void;
|
|
26
30
|
/** Called when session is closed (by server or client) */
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,2CAA2C;AAC3C,MAAM,MAAM,OAAO,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC;AAE5D,4BAA4B;AAC5B,MAAM,MAAM,YAAY,GACpB,cAAc,GACd,YAAY,GACZ,WAAW,GACX,OAAO,CAAC;AAEZ,+BAA+B;AAC/B,MAAM,MAAM,WAAW,GACnB,kBAAkB,GAClB,cAAc,GACd,iBAAiB,GACjB,qBAAqB,GACrB,SAAS,CAAC;AAEd,4CAA4C;AAC5C,MAAM,WAAW,qBAAqB;IACpC,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,gBAAgB,EAAE,MAAM,CAAC;IACzB,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IAEtB,2CAA2C;IAC3C,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAEjD,2CAA2C;IAC3C,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAEjD,wCAAwC;IACxC,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IAE9C,sBAAsB;IACtB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IAEjC,0DAA0D;IAC1D,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,CAAC;CACzC"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,2CAA2C;AAC3C,MAAM,MAAM,OAAO,GAAG,SAAS,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC;AAE5D,4BAA4B;AAC5B,MAAM,MAAM,YAAY,GACpB,cAAc,GACd,YAAY,GACZ,WAAW,GACX,OAAO,CAAC;AAEZ,+BAA+B;AAC/B,MAAM,MAAM,WAAW,GACnB,kBAAkB,GAClB,cAAc,GACd,iBAAiB,GACjB,qBAAqB,GACrB,SAAS,CAAC;AAEd,gEAAgE;AAChE,MAAM,MAAM,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAElD,4CAA4C;AAC5C,MAAM,WAAW,qBAAqB;IACpC,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,gBAAgB,EAAE,MAAM,CAAC;IACzB,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IAEtB,2CAA2C;IAC3C,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAEjD,2CAA2C;IAC3C,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAEjD,wCAAwC;IACxC,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IAE9C,qEAAqE;IACrE,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;IAEjD,sBAAsB;IACtB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IAEjC,0DAA0D;IAC1D,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,CAAC;CACzC"}
|