@estuary-ai/sdk 0.1.22 → 0.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +24 -2
- package/dist/{chunk-6M5LSBMK.mjs → chunk-W5QYPYX3.mjs} +2 -2
- package/dist/chunk-W5QYPYX3.mjs.map +1 -0
- package/dist/index.d.mts +33 -1
- package/dist/index.d.ts +33 -1
- package/dist/index.js +173 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +88 -8
- package/dist/index.mjs.map +1 -1
- package/dist/{livekit-voice-RWXL7IXC.mjs → livekit-voice-PV3TGH2Q.mjs} +95 -3
- package/dist/livekit-voice-PV3TGH2Q.mjs.map +1 -0
- package/dist/{websocket-voice-IFM6J5ES.mjs → websocket-voice-6DMYBGHP.mjs} +3 -3
- package/dist/websocket-voice-6DMYBGHP.mjs.map +1 -0
- package/package.json +71 -71
- package/dist/chunk-6M5LSBMK.mjs.map +0 -1
- package/dist/livekit-voice-RWXL7IXC.mjs.map +0 -1
- package/dist/websocket-voice-IFM6J5ES.mjs.map +0 -1
package/LICENSE
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2026 estuary.ai
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 estuary.ai
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://www.npmjs.com/package/@estuary-ai/sdk)
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Web SDK for the [Estuary](https://www.estuary-ai.com) real-time AI conversation platform. Build applications with persistent AI characters that remember, hear, and see.
|
|
6
6
|
|
|
7
7
|
## Installation
|
|
8
8
|
|
|
@@ -120,6 +120,15 @@ import { parseActions } from '@estuary-ai/sdk';
|
|
|
120
120
|
const { actions, cleanText } = parseActions(rawBotText);
|
|
121
121
|
```
|
|
122
122
|
|
|
123
|
+
### Character Info
|
|
124
|
+
|
|
125
|
+
Fetch character details (name, avatar, 3D model URLs):
|
|
126
|
+
|
|
127
|
+
```typescript
|
|
128
|
+
const character = await client.getCharacter();
|
|
129
|
+
console.log(character.name, character.avatar);
|
|
130
|
+
```
|
|
131
|
+
|
|
123
132
|
### Memory & Knowledge Graph
|
|
124
133
|
|
|
125
134
|
```typescript
|
|
@@ -237,12 +246,24 @@ interface EstuaryConfig {
|
|
|
237
246
|
voiceTransport?: 'websocket' | 'livekit' | 'auto'; // Default: 'auto'
|
|
238
247
|
realtimeMemory?: boolean; // Enable real-time memory extraction events. Default: false
|
|
239
248
|
suppressMicDuringPlayback?: boolean; // Mute mic while bot audio plays (software AEC). Default: false
|
|
249
|
+
autoInterruptOnSpeech?: boolean; // Interrupt bot audio when user speaks. Default: true
|
|
240
250
|
}
|
|
241
251
|
```
|
|
242
252
|
|
|
253
|
+
## Runtime Properties
|
|
254
|
+
|
|
255
|
+
```typescript
|
|
256
|
+
client.connectionState // ConnectionState enum (Disconnected, Connecting, Connected, ...)
|
|
257
|
+
client.isConnected // boolean shorthand
|
|
258
|
+
client.isVoiceActive // true while voice session is running
|
|
259
|
+
client.isMuted // current mute state
|
|
260
|
+
client.suppressMicDuringPlayback // get/set at runtime without reconnecting
|
|
261
|
+
client.session // SessionInfo | null after connect
|
|
262
|
+
```
|
|
263
|
+
|
|
243
264
|
## Exports
|
|
244
265
|
|
|
245
|
-
Key exports
|
|
266
|
+
Key exports:
|
|
246
267
|
|
|
247
268
|
```typescript
|
|
248
269
|
// Client
|
|
@@ -261,6 +282,7 @@ import { parseActions } from '@estuary-ai/sdk';
|
|
|
261
282
|
import type {
|
|
262
283
|
EstuaryConfig,
|
|
263
284
|
SessionInfo,
|
|
285
|
+
CharacterInfo,
|
|
264
286
|
BotResponse,
|
|
265
287
|
BotVoice,
|
|
266
288
|
SttResponse,
|
|
@@ -26,5 +26,5 @@ var EstuaryError = class extends Error {
|
|
|
26
26
|
};
|
|
27
27
|
|
|
28
28
|
export { ErrorCode, EstuaryError };
|
|
29
|
-
//# sourceMappingURL=chunk-
|
|
30
|
-
//# sourceMappingURL=chunk-
|
|
29
|
+
//# sourceMappingURL=chunk-W5QYPYX3.mjs.map
|
|
30
|
+
//# sourceMappingURL=chunk-W5QYPYX3.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/errors.ts"],"names":["ErrorCode"],"mappings":";AAAO,IAAK,SAAA,qBAAAA,UAAAA,KAAL;AACL,EAAAA,WAAA,mBAAA,CAAA,GAAoB,mBAAA;AACpB,EAAAA,WAAA,aAAA,CAAA,GAAc,aAAA;AACd,EAAAA,WAAA,oBAAA,CAAA,GAAqB,oBAAA;AACrB,EAAAA,WAAA,gBAAA,CAAA,GAAiB,gBAAA;AACjB,EAAAA,WAAA,qBAAA,CAAA,GAAsB,qBAAA;AACtB,EAAAA,WAAA,sBAAA,CAAA,GAAuB,sBAAA;AACvB,EAAAA,WAAA,kBAAA,CAAA,GAAmB,kBAAA;AACnB,EAAAA,WAAA,qBAAA,CAAA,GAAsB,qBAAA;AACtB,EAAAA,WAAA,mBAAA,CAAA,GAAoB,mBAAA;AACpB,EAAAA,WAAA,eAAA,CAAA,GAAgB,eAAA;AAChB,EAAAA,WAAA,YAAA,CAAA,GAAa,YAAA;AACb,EAAAA,WAAA,SAAA,CAAA,GAAU,SAAA;AAZA,EAAA,OAAAA,UAAAA;AAAA,CAAA,EAAA,SAAA,IAAA,EAAA;AAeL,IAAM,YAAA,GAAN,cAA2B,KAAA,CAAM;AAAA,EAC7B,IAAA;AAAA,EACA,OAAA;AAAA,EAET,WAAA,CAAY,IAAA,EAAiB,OAAA,EAAiB,OAAA,EAAmB;AAC/D,IAAA,KAAA,CAAM,OAAO,CAAA;AACb,IAAA,IAAA,CAAK,IAAA,GAAO,cAAA;AACZ,IAAA,IAAA,CAAK,IAAA,GAAO,IAAA;AACZ,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AAAA,EACjB;AACF","file":"chunk-W5QYPYX3.mjs","sourcesContent":["export enum ErrorCode {\n CONNECTION_FAILED = 'CONNECTION_FAILED',\n AUTH_FAILED = 'AUTH_FAILED',\n CONNECTION_TIMEOUT = 'CONNECTION_TIMEOUT',\n QUOTA_EXCEEDED = 'QUOTA_EXCEEDED',\n VOICE_NOT_SUPPORTED = 'VOICE_NOT_SUPPORTED',\n VOICE_ALREADY_ACTIVE = 'VOICE_ALREADY_ACTIVE',\n VOICE_NOT_ACTIVE = 'VOICE_NOT_ACTIVE',\n LIVEKIT_UNAVAILABLE = 'LIVEKIT_UNAVAILABLE',\n MICROPHONE_DENIED = 'MICROPHONE_DENIED',\n NOT_CONNECTED = 'NOT_CONNECTED',\n REST_ERROR = 'REST_ERROR',\n UNKNOWN = 'UNKNOWN',\n}\n\nexport class EstuaryError extends Error {\n readonly code: ErrorCode;\n readonly details?: unknown;\n\n constructor(code: ErrorCode, message: string, details?: unknown) {\n super(message);\n this.name = 'EstuaryError';\n this.code = code;\n this.details = details;\n }\n}\n"]}
|
package/dist/index.d.mts
CHANGED
|
@@ -109,6 +109,16 @@ interface MemoryUpdatedEvent {
|
|
|
109
109
|
newMemories: MemoryData[];
|
|
110
110
|
timestamp: string;
|
|
111
111
|
}
|
|
112
|
+
interface CharacterInfo {
|
|
113
|
+
id: string;
|
|
114
|
+
name: string;
|
|
115
|
+
tagline: string | null;
|
|
116
|
+
avatar: string | null;
|
|
117
|
+
modelUrl: string | null;
|
|
118
|
+
modelPreviewUrl: string | null;
|
|
119
|
+
modelStatus: string | null;
|
|
120
|
+
sourceImageUrl: string | null;
|
|
121
|
+
}
|
|
112
122
|
interface CharacterAction {
|
|
113
123
|
/** Action name (e.g., "follow_user", "sit", "look_at") */
|
|
114
124
|
name: string;
|
|
@@ -137,6 +147,8 @@ type EstuaryEventMap = {
|
|
|
137
147
|
livekitDisconnected: () => void;
|
|
138
148
|
audioPlaybackStarted: (messageId: string) => void;
|
|
139
149
|
audioPlaybackComplete: (messageId: string) => void;
|
|
150
|
+
/** Bot audio level 0.0–1.0, emitted during playback for both transports. */
|
|
151
|
+
botAudioLevel: (level: number) => void;
|
|
140
152
|
memoryUpdated: (event: MemoryUpdatedEvent) => void;
|
|
141
153
|
};
|
|
142
154
|
interface VoiceManager {
|
|
@@ -145,6 +157,10 @@ interface VoiceManager {
|
|
|
145
157
|
toggleMute(): void;
|
|
146
158
|
/** Suppress audio sending (software AEC). No-op if not supported. */
|
|
147
159
|
setSuppressed?(suppressed: boolean): void;
|
|
160
|
+
/** Set callback for speaking state from participant attributes (LiveKit only). */
|
|
161
|
+
setSpeakingStateCallback?(cb: (speaking: boolean) => void): void;
|
|
162
|
+
/** Set callback for audio level updates (0-1) during bot speech. */
|
|
163
|
+
setAudioLevelCallback?(cb: (level: number) => void): void;
|
|
148
164
|
readonly isMuted: boolean;
|
|
149
165
|
readonly isActive: boolean;
|
|
150
166
|
dispose(): void;
|
|
@@ -309,6 +325,7 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
|
|
|
309
325
|
private voiceManager;
|
|
310
326
|
private audioPlayer;
|
|
311
327
|
private _memory;
|
|
328
|
+
private _character;
|
|
312
329
|
private _sessionInfo;
|
|
313
330
|
private actionParsers;
|
|
314
331
|
private _hasAutoInterrupted;
|
|
@@ -316,6 +333,8 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
|
|
|
316
333
|
constructor(config: EstuaryConfig);
|
|
317
334
|
/** Memory API client for querying memories, graphs, and facts */
|
|
318
335
|
get memory(): MemoryClient;
|
|
336
|
+
/** Fetch character details including 3D model and avatar URLs. */
|
|
337
|
+
getCharacter(characterId?: string): Promise<CharacterInfo>;
|
|
319
338
|
/** Current session info (null if not connected) */
|
|
320
339
|
get session(): SessionInfo | null;
|
|
321
340
|
/** Current connection state */
|
|
@@ -346,12 +365,17 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
|
|
|
346
365
|
toggleMute(): void;
|
|
347
366
|
/** Whether the microphone is muted */
|
|
348
367
|
get isMuted(): boolean;
|
|
368
|
+
/** Get/set suppressMicDuringPlayback at runtime (no reconnect needed) */
|
|
369
|
+
get suppressMicDuringPlayback(): boolean;
|
|
370
|
+
set suppressMicDuringPlayback(enabled: boolean);
|
|
349
371
|
/** Whether voice is currently active */
|
|
350
372
|
get isVoiceActive(): boolean;
|
|
351
373
|
private ensureConnected;
|
|
352
374
|
private forwardSocketEvents;
|
|
353
375
|
private handleBotResponse;
|
|
354
376
|
private handleBotVoice;
|
|
377
|
+
/** Compute RMS audio level (0-1) from base64-encoded Int16 PCM. */
|
|
378
|
+
private computeAudioLevel;
|
|
355
379
|
private maybeAutoInterrupt;
|
|
356
380
|
}
|
|
357
381
|
|
|
@@ -394,4 +418,12 @@ declare function parseActions(text: string): {
|
|
|
394
418
|
cleanText: string;
|
|
395
419
|
};
|
|
396
420
|
|
|
397
|
-
|
|
421
|
+
declare class CharacterClient {
|
|
422
|
+
private rest;
|
|
423
|
+
constructor(rest: RestClient);
|
|
424
|
+
/** Fetch character details including 3D model and avatar URLs. */
|
|
425
|
+
getCharacter(characterId: string): Promise<CharacterInfo>;
|
|
426
|
+
dispose(): void;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
export { type BotResponse, type BotVoice, type CameraCaptureRequest, type CharacterAction, CharacterClient, type CharacterInfo, ConnectionState, type CoreFact, type CoreFactsResponse, ErrorCode, EstuaryClient, type EstuaryConfig, EstuaryError, type EstuaryEventMap, type InterruptData, type LiveKitTokenResponse, MemoryClient, type MemoryData, type MemoryGraphEdge, type MemoryGraphNode, type MemoryGraphOptions, type MemoryGraphResponse, type MemoryListOptions, type MemoryListResponse, type MemorySearchOptions, type MemorySearchResponse, type MemoryStatsResponse, type MemoryTimelineOptions, type MemoryTimelineResponse, type MemoryUpdatedEvent, type ParsedAction, type QuotaExceededData, type SessionInfo, type SttResponse, type VoiceManager, type VoiceTransport, parseActions };
|
package/dist/index.d.ts
CHANGED
|
@@ -109,6 +109,16 @@ interface MemoryUpdatedEvent {
|
|
|
109
109
|
newMemories: MemoryData[];
|
|
110
110
|
timestamp: string;
|
|
111
111
|
}
|
|
112
|
+
interface CharacterInfo {
|
|
113
|
+
id: string;
|
|
114
|
+
name: string;
|
|
115
|
+
tagline: string | null;
|
|
116
|
+
avatar: string | null;
|
|
117
|
+
modelUrl: string | null;
|
|
118
|
+
modelPreviewUrl: string | null;
|
|
119
|
+
modelStatus: string | null;
|
|
120
|
+
sourceImageUrl: string | null;
|
|
121
|
+
}
|
|
112
122
|
interface CharacterAction {
|
|
113
123
|
/** Action name (e.g., "follow_user", "sit", "look_at") */
|
|
114
124
|
name: string;
|
|
@@ -137,6 +147,8 @@ type EstuaryEventMap = {
|
|
|
137
147
|
livekitDisconnected: () => void;
|
|
138
148
|
audioPlaybackStarted: (messageId: string) => void;
|
|
139
149
|
audioPlaybackComplete: (messageId: string) => void;
|
|
150
|
+
/** Bot audio level 0.0–1.0, emitted during playback for both transports. */
|
|
151
|
+
botAudioLevel: (level: number) => void;
|
|
140
152
|
memoryUpdated: (event: MemoryUpdatedEvent) => void;
|
|
141
153
|
};
|
|
142
154
|
interface VoiceManager {
|
|
@@ -145,6 +157,10 @@ interface VoiceManager {
|
|
|
145
157
|
toggleMute(): void;
|
|
146
158
|
/** Suppress audio sending (software AEC). No-op if not supported. */
|
|
147
159
|
setSuppressed?(suppressed: boolean): void;
|
|
160
|
+
/** Set callback for speaking state from participant attributes (LiveKit only). */
|
|
161
|
+
setSpeakingStateCallback?(cb: (speaking: boolean) => void): void;
|
|
162
|
+
/** Set callback for audio level updates (0-1) during bot speech. */
|
|
163
|
+
setAudioLevelCallback?(cb: (level: number) => void): void;
|
|
148
164
|
readonly isMuted: boolean;
|
|
149
165
|
readonly isActive: boolean;
|
|
150
166
|
dispose(): void;
|
|
@@ -309,6 +325,7 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
|
|
|
309
325
|
private voiceManager;
|
|
310
326
|
private audioPlayer;
|
|
311
327
|
private _memory;
|
|
328
|
+
private _character;
|
|
312
329
|
private _sessionInfo;
|
|
313
330
|
private actionParsers;
|
|
314
331
|
private _hasAutoInterrupted;
|
|
@@ -316,6 +333,8 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
|
|
|
316
333
|
constructor(config: EstuaryConfig);
|
|
317
334
|
/** Memory API client for querying memories, graphs, and facts */
|
|
318
335
|
get memory(): MemoryClient;
|
|
336
|
+
/** Fetch character details including 3D model and avatar URLs. */
|
|
337
|
+
getCharacter(characterId?: string): Promise<CharacterInfo>;
|
|
319
338
|
/** Current session info (null if not connected) */
|
|
320
339
|
get session(): SessionInfo | null;
|
|
321
340
|
/** Current connection state */
|
|
@@ -346,12 +365,17 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
|
|
|
346
365
|
toggleMute(): void;
|
|
347
366
|
/** Whether the microphone is muted */
|
|
348
367
|
get isMuted(): boolean;
|
|
368
|
+
/** Get/set suppressMicDuringPlayback at runtime (no reconnect needed) */
|
|
369
|
+
get suppressMicDuringPlayback(): boolean;
|
|
370
|
+
set suppressMicDuringPlayback(enabled: boolean);
|
|
349
371
|
/** Whether voice is currently active */
|
|
350
372
|
get isVoiceActive(): boolean;
|
|
351
373
|
private ensureConnected;
|
|
352
374
|
private forwardSocketEvents;
|
|
353
375
|
private handleBotResponse;
|
|
354
376
|
private handleBotVoice;
|
|
377
|
+
/** Compute RMS audio level (0-1) from base64-encoded Int16 PCM. */
|
|
378
|
+
private computeAudioLevel;
|
|
355
379
|
private maybeAutoInterrupt;
|
|
356
380
|
}
|
|
357
381
|
|
|
@@ -394,4 +418,12 @@ declare function parseActions(text: string): {
|
|
|
394
418
|
cleanText: string;
|
|
395
419
|
};
|
|
396
420
|
|
|
397
|
-
|
|
421
|
+
declare class CharacterClient {
|
|
422
|
+
private rest;
|
|
423
|
+
constructor(rest: RestClient);
|
|
424
|
+
/** Fetch character details including 3D model and avatar URLs. */
|
|
425
|
+
getCharacter(characterId: string): Promise<CharacterInfo>;
|
|
426
|
+
dispose(): void;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
export { type BotResponse, type BotVoice, type CameraCaptureRequest, type CharacterAction, CharacterClient, type CharacterInfo, ConnectionState, type CoreFact, type CoreFactsResponse, ErrorCode, EstuaryClient, type EstuaryConfig, EstuaryError, type EstuaryEventMap, type InterruptData, type LiveKitTokenResponse, MemoryClient, type MemoryData, type MemoryGraphEdge, type MemoryGraphNode, type MemoryGraphOptions, type MemoryGraphResponse, type MemoryListOptions, type MemoryListResponse, type MemorySearchOptions, type MemorySearchResponse, type MemoryStatsResponse, type MemoryTimelineOptions, type MemoryTimelineResponse, type MemoryUpdatedEvent, type ParsedAction, type QuotaExceededData, type SessionInfo, type SttResponse, type VoiceManager, type VoiceTransport, parseActions };
|
package/dist/index.js
CHANGED
|
@@ -247,6 +247,13 @@ var init_livekit_voice = __esm({
|
|
|
247
247
|
// livekit-client Room (dynamically imported)
|
|
248
248
|
_isMuted = false;
|
|
249
249
|
_isActive = false;
|
|
250
|
+
speakingStateCallback = null;
|
|
251
|
+
audioLevelCallback = null;
|
|
252
|
+
// Audio analyser (via livekit-client's createAudioAnalyser)
|
|
253
|
+
calculateVolume = null;
|
|
254
|
+
analyserCleanup = null;
|
|
255
|
+
audioLevelPollTimer = null;
|
|
256
|
+
_isBotSpeaking = false;
|
|
250
257
|
constructor(socketManager, logger) {
|
|
251
258
|
this.socketManager = socketManager;
|
|
252
259
|
this.logger = logger;
|
|
@@ -257,6 +264,12 @@ var init_livekit_voice = __esm({
|
|
|
257
264
|
get isActive() {
|
|
258
265
|
return this._isActive;
|
|
259
266
|
}
|
|
267
|
+
setSpeakingStateCallback(cb) {
|
|
268
|
+
this.speakingStateCallback = cb;
|
|
269
|
+
}
|
|
270
|
+
setAudioLevelCallback(cb) {
|
|
271
|
+
this.audioLevelCallback = cb;
|
|
272
|
+
}
|
|
260
273
|
async start() {
|
|
261
274
|
if (this._isActive) {
|
|
262
275
|
throw new exports.EstuaryError("VOICE_ALREADY_ACTIVE" /* VOICE_ALREADY_ACTIVE */, "Voice is already active");
|
|
@@ -296,16 +309,24 @@ var init_livekit_voice = __esm({
|
|
|
296
309
|
}
|
|
297
310
|
audioElement.play().catch(() => {
|
|
298
311
|
});
|
|
312
|
+
this.setupAnalyser(track);
|
|
313
|
+
if (this._isBotSpeaking) {
|
|
314
|
+
setTimeout(() => this.startAudioLevelPolling(), 50);
|
|
315
|
+
}
|
|
299
316
|
}
|
|
300
317
|
});
|
|
301
318
|
this.room.on(RoomEvent.TrackUnsubscribed, (track) => {
|
|
302
319
|
if (track.kind === Track.Kind.Audio) {
|
|
320
|
+
this.teardownAnalyser();
|
|
303
321
|
track.detach().forEach((el) => el.remove());
|
|
304
322
|
}
|
|
305
323
|
});
|
|
306
324
|
this.room.on(RoomEvent.Disconnected, () => {
|
|
307
325
|
this.logger.debug("LiveKit room disconnected");
|
|
308
326
|
this._isActive = false;
|
|
327
|
+
this._isBotSpeaking = false;
|
|
328
|
+
this.teardownAnalyser();
|
|
329
|
+
this.speakingStateCallback?.(false);
|
|
309
330
|
});
|
|
310
331
|
try {
|
|
311
332
|
await this.room.connect(tokenData.url, tokenData.token);
|
|
@@ -319,6 +340,23 @@ var init_livekit_voice = __esm({
|
|
|
319
340
|
err
|
|
320
341
|
);
|
|
321
342
|
}
|
|
343
|
+
this.room.on(
|
|
344
|
+
RoomEvent.ParticipantAttributesChanged,
|
|
345
|
+
(changedAttributes, participant) => {
|
|
346
|
+
if (participant === this.room?.localParticipant) return;
|
|
347
|
+
const state = changedAttributes["estuary.state"];
|
|
348
|
+
if (state === "speaking") {
|
|
349
|
+
this._isBotSpeaking = true;
|
|
350
|
+
this.speakingStateCallback?.(true);
|
|
351
|
+
this.startAudioLevelPolling();
|
|
352
|
+
} else if (state === "idle") {
|
|
353
|
+
this._isBotSpeaking = false;
|
|
354
|
+
this.stopAudioLevelPolling();
|
|
355
|
+
this.speakingStateCallback?.(false);
|
|
356
|
+
this.audioLevelCallback?.(0);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
);
|
|
322
360
|
try {
|
|
323
361
|
await this.room.localParticipant.setMicrophoneEnabled(true);
|
|
324
362
|
this.logger.debug("Microphone enabled");
|
|
@@ -342,6 +380,9 @@ var init_livekit_voice = __esm({
|
|
|
342
380
|
this.socketManager.emitEvent("livekit_leave");
|
|
343
381
|
} catch {
|
|
344
382
|
}
|
|
383
|
+
this._isBotSpeaking = false;
|
|
384
|
+
this.teardownAnalyser();
|
|
385
|
+
this.speakingStateCallback?.(false);
|
|
345
386
|
if (this.room) {
|
|
346
387
|
for (const [, publication] of this.room.localParticipant.trackPublications) {
|
|
347
388
|
if (publication.track) {
|
|
@@ -362,6 +403,10 @@ var init_livekit_voice = __esm({
|
|
|
362
403
|
this.logger.debug("Mute toggled:", this._isMuted);
|
|
363
404
|
}
|
|
364
405
|
dispose() {
|
|
406
|
+
this.speakingStateCallback = null;
|
|
407
|
+
this.audioLevelCallback = null;
|
|
408
|
+
this._isBotSpeaking = false;
|
|
409
|
+
this.teardownAnalyser();
|
|
365
410
|
if (this.room) {
|
|
366
411
|
this.room.disconnect();
|
|
367
412
|
this.room = null;
|
|
@@ -369,6 +414,53 @@ var init_livekit_voice = __esm({
|
|
|
369
414
|
this._isActive = false;
|
|
370
415
|
this._isMuted = false;
|
|
371
416
|
}
|
|
417
|
+
// ─── Audio Analyser (livekit-client built-in) ───────────────────
|
|
418
|
+
async setupAnalyser(track) {
|
|
419
|
+
this.teardownAnalyser();
|
|
420
|
+
try {
|
|
421
|
+
const { createAudioAnalyser } = await import('livekit-client');
|
|
422
|
+
const { analyser, calculateVolume, cleanup } = createAudioAnalyser(track, {
|
|
423
|
+
fftSize: 256,
|
|
424
|
+
smoothingTimeConstant: 0.3
|
|
425
|
+
});
|
|
426
|
+
if (analyser.context.state === "suspended") {
|
|
427
|
+
await analyser.context.resume();
|
|
428
|
+
}
|
|
429
|
+
this.calculateVolume = calculateVolume;
|
|
430
|
+
this.analyserCleanup = cleanup;
|
|
431
|
+
this.logger.debug("Audio analyser created for bot track");
|
|
432
|
+
} catch (err) {
|
|
433
|
+
this.logger.debug("Failed to create audio analyser:", err);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
teardownAnalyser() {
|
|
437
|
+
this.stopAudioLevelPolling();
|
|
438
|
+
if (this.analyserCleanup) {
|
|
439
|
+
this.analyserCleanup().catch(() => {
|
|
440
|
+
});
|
|
441
|
+
this.analyserCleanup = null;
|
|
442
|
+
}
|
|
443
|
+
this.calculateVolume = null;
|
|
444
|
+
}
|
|
445
|
+
startAudioLevelPolling() {
|
|
446
|
+
if (this.audioLevelPollTimer !== null) return;
|
|
447
|
+
if (!this.calculateVolume) return;
|
|
448
|
+
this.audioLevelPollTimer = setInterval(() => {
|
|
449
|
+
if (!this.calculateVolume) {
|
|
450
|
+
this.stopAudioLevelPolling();
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
const vol = this.calculateVolume();
|
|
454
|
+
this.audioLevelCallback?.(vol);
|
|
455
|
+
}, 33);
|
|
456
|
+
}
|
|
457
|
+
stopAudioLevelPolling() {
|
|
458
|
+
if (this.audioLevelPollTimer !== null) {
|
|
459
|
+
clearInterval(this.audioLevelPollTimer);
|
|
460
|
+
this.audioLevelPollTimer = null;
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
// ─── Private ────────────────────────────────────────────────────
|
|
372
464
|
requestToken() {
|
|
373
465
|
return new Promise((resolve, reject) => {
|
|
374
466
|
const timeout = setTimeout(() => {
|
|
@@ -880,6 +972,30 @@ var MemoryClient = class {
|
|
|
880
972
|
}
|
|
881
973
|
};
|
|
882
974
|
|
|
975
|
+
// src/rest/character-client.ts
|
|
976
|
+
var CharacterClient = class {
|
|
977
|
+
rest;
|
|
978
|
+
constructor(rest) {
|
|
979
|
+
this.rest = rest;
|
|
980
|
+
}
|
|
981
|
+
/** Fetch character details including 3D model and avatar URLs. */
|
|
982
|
+
async getCharacter(characterId) {
|
|
983
|
+
const raw = await this.rest.get(`/api/agents/${characterId}`);
|
|
984
|
+
return {
|
|
985
|
+
id: raw.id,
|
|
986
|
+
name: raw.name,
|
|
987
|
+
tagline: raw.tagline ?? null,
|
|
988
|
+
avatar: raw.avatar ?? null,
|
|
989
|
+
modelUrl: raw.modelUrl ?? null,
|
|
990
|
+
modelPreviewUrl: raw.modelPreviewUrl ?? null,
|
|
991
|
+
modelStatus: raw.modelStatus ?? null,
|
|
992
|
+
sourceImageUrl: raw.sourceImageUrl ?? null
|
|
993
|
+
};
|
|
994
|
+
}
|
|
995
|
+
dispose() {
|
|
996
|
+
}
|
|
997
|
+
};
|
|
998
|
+
|
|
883
999
|
// src/audio/audio-player.ts
|
|
884
1000
|
var AudioPlayer = class {
|
|
885
1001
|
sampleRate;
|
|
@@ -1130,6 +1246,7 @@ var EstuaryClient = class extends TypedEventEmitter {
|
|
|
1130
1246
|
voiceManager = null;
|
|
1131
1247
|
audioPlayer = null;
|
|
1132
1248
|
_memory;
|
|
1249
|
+
_character;
|
|
1133
1250
|
_sessionInfo = null;
|
|
1134
1251
|
actionParsers = /* @__PURE__ */ new Map();
|
|
1135
1252
|
_hasAutoInterrupted = false;
|
|
@@ -1142,11 +1259,16 @@ var EstuaryClient = class extends TypedEventEmitter {
|
|
|
1142
1259
|
this.forwardSocketEvents();
|
|
1143
1260
|
const restClient = new RestClient(config.serverUrl, config.apiKey);
|
|
1144
1261
|
this._memory = new MemoryClient(restClient, config.characterId, config.playerId);
|
|
1262
|
+
this._character = new CharacterClient(restClient);
|
|
1145
1263
|
}
|
|
1146
1264
|
/** Memory API client for querying memories, graphs, and facts */
|
|
1147
1265
|
get memory() {
|
|
1148
1266
|
return this._memory;
|
|
1149
1267
|
}
|
|
1268
|
+
/** Fetch character details including 3D model and avatar URLs. */
|
|
1269
|
+
async getCharacter(characterId) {
|
|
1270
|
+
return this._character.getCharacter(characterId ?? this.config.characterId);
|
|
1271
|
+
}
|
|
1150
1272
|
/** Current session info (null if not connected) */
|
|
1151
1273
|
get session() {
|
|
1152
1274
|
return this._sessionInfo;
|
|
@@ -1247,6 +1369,7 @@ var EstuaryClient = class extends TypedEventEmitter {
|
|
|
1247
1369
|
}
|
|
1248
1370
|
} else if (event.type === "complete") {
|
|
1249
1371
|
this.emit("audioPlaybackComplete", event.messageId);
|
|
1372
|
+
this.emit("botAudioLevel", 0);
|
|
1250
1373
|
this.notifyAudioPlaybackComplete(event.messageId);
|
|
1251
1374
|
if (this.config.suppressMicDuringPlayback) {
|
|
1252
1375
|
this.voiceManager?.setSuppressed?.(false);
|
|
@@ -1255,6 +1378,24 @@ var EstuaryClient = class extends TypedEventEmitter {
|
|
|
1255
1378
|
});
|
|
1256
1379
|
}
|
|
1257
1380
|
await this.voiceManager.start();
|
|
1381
|
+
this.voiceManager.setSpeakingStateCallback?.((speaking) => {
|
|
1382
|
+
if (speaking) {
|
|
1383
|
+
this.emit("audioPlaybackStarted", "livekit-audio");
|
|
1384
|
+
if (this.config.suppressMicDuringPlayback) {
|
|
1385
|
+
this.voiceManager?.setSuppressed?.(true);
|
|
1386
|
+
}
|
|
1387
|
+
} else {
|
|
1388
|
+
this.emit("audioPlaybackComplete", "livekit-audio");
|
|
1389
|
+
this.emit("botAudioLevel", 0);
|
|
1390
|
+
this.notifyAudioPlaybackComplete("livekit-audio");
|
|
1391
|
+
if (this.config.suppressMicDuringPlayback) {
|
|
1392
|
+
this.voiceManager?.setSuppressed?.(false);
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
});
|
|
1396
|
+
this.voiceManager.setAudioLevelCallback?.((level) => {
|
|
1397
|
+
this.emit("botAudioLevel", level);
|
|
1398
|
+
});
|
|
1258
1399
|
this.emit("voiceStarted");
|
|
1259
1400
|
}
|
|
1260
1401
|
/** Stop voice input */
|
|
@@ -1277,6 +1418,13 @@ var EstuaryClient = class extends TypedEventEmitter {
|
|
|
1277
1418
|
get isMuted() {
|
|
1278
1419
|
return this.voiceManager?.isMuted ?? false;
|
|
1279
1420
|
}
|
|
1421
|
+
/** Get/set suppressMicDuringPlayback at runtime (no reconnect needed) */
|
|
1422
|
+
get suppressMicDuringPlayback() {
|
|
1423
|
+
return this.config.suppressMicDuringPlayback ?? false;
|
|
1424
|
+
}
|
|
1425
|
+
set suppressMicDuringPlayback(enabled) {
|
|
1426
|
+
this.config.suppressMicDuringPlayback = enabled;
|
|
1427
|
+
}
|
|
1280
1428
|
/** Whether voice is currently active */
|
|
1281
1429
|
get isVoiceActive() {
|
|
1282
1430
|
return this.voiceManager?.isActive ?? false;
|
|
@@ -1346,7 +1494,31 @@ var EstuaryClient = class extends TypedEventEmitter {
|
|
|
1346
1494
|
}
|
|
1347
1495
|
handleBotVoice(voice) {
|
|
1348
1496
|
this.emit("botVoice", voice);
|
|
1349
|
-
|
|
1497
|
+
if (voice.audio) {
|
|
1498
|
+
this.audioPlayer?.enqueue(voice);
|
|
1499
|
+
this.emit("botAudioLevel", this.computeAudioLevel(voice.audio));
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
/** Compute RMS audio level (0-1) from base64-encoded Int16 PCM. */
|
|
1503
|
+
computeAudioLevel(base64Audio) {
|
|
1504
|
+
try {
|
|
1505
|
+
const binaryStr = atob(base64Audio);
|
|
1506
|
+
const len = binaryStr.length;
|
|
1507
|
+
const step = 16;
|
|
1508
|
+
let sum = 0;
|
|
1509
|
+
let count = 0;
|
|
1510
|
+
for (let i = 0; i + 1 < len; i += step) {
|
|
1511
|
+
const sample = binaryStr.charCodeAt(i) | binaryStr.charCodeAt(i + 1) << 8;
|
|
1512
|
+
const signed = sample > 32767 ? sample - 65536 : sample;
|
|
1513
|
+
const normalized = signed / 32768;
|
|
1514
|
+
sum += normalized * normalized;
|
|
1515
|
+
count++;
|
|
1516
|
+
}
|
|
1517
|
+
if (count === 0) return 0;
|
|
1518
|
+
return Math.min(1, Math.sqrt(sum / count) * 5);
|
|
1519
|
+
} catch {
|
|
1520
|
+
return 0;
|
|
1521
|
+
}
|
|
1350
1522
|
}
|
|
1351
1523
|
maybeAutoInterrupt(stt) {
|
|
1352
1524
|
if ((this.config.autoInterruptOnSpeech ?? true) === false) return;
|