@convai/web-sdk 1.3.2-beta.0 → 1.4.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/AudioManager.d.ts +4 -0
- package/dist/core/AudioManager.d.ts.map +1 -1
- package/dist/core/AudioManager.js +26 -0
- package/dist/core/AudioManager.js.map +1 -1
- package/dist/core/ConvaiClient.d.ts +20 -0
- package/dist/core/ConvaiClient.d.ts.map +1 -1
- package/dist/core/ConvaiClient.js +215 -224
- package/dist/core/ConvaiClient.js.map +1 -1
- package/dist/core/MessageHandler.d.ts +5 -0
- package/dist/core/MessageHandler.d.ts.map +1 -1
- package/dist/core/MessageHandler.js +44 -20
- package/dist/core/MessageHandler.js.map +1 -1
- package/dist/core/WebSocketSession.d.ts +45 -0
- package/dist/core/WebSocketSession.d.ts.map +1 -0
- package/dist/core/WebSocketSession.js +133 -0
- package/dist/core/WebSocketSession.js.map +1 -0
- package/dist/core/types.d.ts +12 -49
- package/dist/core/types.d.ts.map +1 -1
- package/dist/react/components/ConvaiWidget.d.ts.map +1 -1
- package/dist/react/components/ConvaiWidget.js +25 -14
- package/dist/react/components/ConvaiWidget.js.map +1 -1
- package/dist/react/hooks/useConvaiClient.d.ts.map +1 -1
- package/dist/react/hooks/useConvaiClient.js +16 -0
- package/dist/react/hooks/useConvaiClient.js.map +1 -1
- package/dist/vanilla/AudioRenderer.d.ts +6 -0
- package/dist/vanilla/AudioRenderer.d.ts.map +1 -1
- package/dist/vanilla/AudioRenderer.js +21 -0
- package/dist/vanilla/AudioRenderer.js.map +1 -1
- package/dist/vanilla/types.d.ts +1 -1
- package/dist/vanilla/types.d.ts.map +1 -1
- package/package.json +3 -1
|
@@ -6,6 +6,7 @@ import { MessageHandler } from "./MessageHandler";
|
|
|
6
6
|
import { EventEmitter } from "./EventEmitter";
|
|
7
7
|
import { MemoryManager } from "./MemoryManager";
|
|
8
8
|
import { ConnectionStateHandler } from "./ConnectionStateHandler";
|
|
9
|
+
import { WebSocketSession } from "./WebSocketSession";
|
|
9
10
|
/**
|
|
10
11
|
* Custom reconnect policy that disables automatic reconnection
|
|
11
12
|
*/
|
|
@@ -64,6 +65,8 @@ export class ConvaiClient extends EventEmitter {
|
|
|
64
65
|
this._storedConfig = null;
|
|
65
66
|
this._endUserId = null;
|
|
66
67
|
this._endUserMetadata = null;
|
|
68
|
+
this._wsSession = null;
|
|
69
|
+
this._activeTransport = null;
|
|
67
70
|
// Conversation session tracking
|
|
68
71
|
this._conversationSessionId = 0;
|
|
69
72
|
this._conversationStartTime = 0;
|
|
@@ -280,6 +283,9 @@ export class ConvaiClient extends EventEmitter {
|
|
|
280
283
|
this._messageHandler.on("userMuteStopped", () => {
|
|
281
284
|
this.emit("userMuteStopped");
|
|
282
285
|
});
|
|
286
|
+
this._messageHandler.on("actionResponse", (data) => {
|
|
287
|
+
this.emit("actionResponse", data);
|
|
288
|
+
});
|
|
283
289
|
}
|
|
284
290
|
/**
|
|
285
291
|
* Setup MessageHandler to respond to conversation start events
|
|
@@ -318,6 +324,30 @@ export class ConvaiClient extends EventEmitter {
|
|
|
318
324
|
this.emit("stateChange", this._state);
|
|
319
325
|
}
|
|
320
326
|
}
|
|
327
|
+
/**
|
|
328
|
+
* Send a data message via the active transport.
|
|
329
|
+
* For LiveKit: publishes via the data channel.
|
|
330
|
+
* For WebSocket: sends via PipecatClient.sendClientMessage (wrapped in {t,d} envelope).
|
|
331
|
+
*/
|
|
332
|
+
publishMessage(type, data) {
|
|
333
|
+
if (this._activeTransport === "websocket" && this._wsSession?.isConnected) {
|
|
334
|
+
this._wsSession.sendMessage(type, data);
|
|
335
|
+
}
|
|
336
|
+
else if (this._room?.localParticipant &&
|
|
337
|
+
this._room.state !== "disconnected") {
|
|
338
|
+
const encodedData = new TextEncoder().encode(JSON.stringify({ type, data }));
|
|
339
|
+
this._room.localParticipant.publishData(encodedData, { reliable: true });
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Returns true when a message can be sent on the active transport.
|
|
344
|
+
*/
|
|
345
|
+
isTransportReady() {
|
|
346
|
+
if (this._activeTransport === "websocket") {
|
|
347
|
+
return this._wsSession?.isConnected ?? false;
|
|
348
|
+
}
|
|
349
|
+
return (!!this._room?.localParticipant && this._room.state !== "disconnected");
|
|
350
|
+
}
|
|
321
351
|
/**
|
|
322
352
|
* Connect to a Convai character
|
|
323
353
|
*/
|
|
@@ -336,7 +366,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
336
366
|
}
|
|
337
367
|
// Add default URL if not provided
|
|
338
368
|
const configWithDefaults = {
|
|
339
|
-
url: "https://realtime-api
|
|
369
|
+
url: "https://realtime-api.convai.com",
|
|
340
370
|
...finalConfig,
|
|
341
371
|
};
|
|
342
372
|
const hasApiKey = Boolean(configWithDefaults.apiKey);
|
|
@@ -354,6 +384,8 @@ export class ConvaiClient extends EventEmitter {
|
|
|
354
384
|
// Determine connection type based on enableVideo
|
|
355
385
|
const connType = configWithDefaults.enableVideo ? "video" : "audio";
|
|
356
386
|
this._connectionType = connType;
|
|
387
|
+
const transportType = configWithDefaults.transport ?? "livekit";
|
|
388
|
+
this._activeTransport = transportType;
|
|
357
389
|
// Prepare request body with required parameters
|
|
358
390
|
const characterSessionIdToSend = configWithDefaults.characterSessionId ?? this._characterSessionId;
|
|
359
391
|
const requestBody = {
|
|
@@ -364,7 +396,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
364
396
|
...(configWithDefaults.endUserMetadata != null && {
|
|
365
397
|
end_user_metadata: configWithDefaults.endUserMetadata,
|
|
366
398
|
}),
|
|
367
|
-
transport:
|
|
399
|
+
transport: transportType,
|
|
368
400
|
connection_type: connType,
|
|
369
401
|
blendshape_provider: configWithDefaults.enableLipsync
|
|
370
402
|
? "neurosync"
|
|
@@ -377,7 +409,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
377
409
|
output_fps: 90,
|
|
378
410
|
},
|
|
379
411
|
llm_provider: "dynamic",
|
|
380
|
-
...(configWithDefaults.enableEmotion
|
|
412
|
+
...(configWithDefaults.enableEmotion === true && {
|
|
381
413
|
emotion_config: {
|
|
382
414
|
provider: configWithDefaults.emotionConfig?.provider ?? "llm",
|
|
383
415
|
min_word_threshold: configWithDefaults.emotionConfig?.min_word_threshold ?? 3,
|
|
@@ -395,19 +427,14 @@ export class ConvaiClient extends EventEmitter {
|
|
|
395
427
|
character_session_id: characterSessionIdToSend,
|
|
396
428
|
}),
|
|
397
429
|
...(configWithDefaults.actionConfig && {
|
|
398
|
-
action_config:
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
...(configWithDefaults.currentAttentionObject && {
|
|
407
|
-
current_attention_object: configWithDefaults.currentAttentionObject,
|
|
408
|
-
}),
|
|
409
|
-
...(configWithDefaults.sceneDescription && {
|
|
410
|
-
scene_description: configWithDefaults.sceneDescription,
|
|
430
|
+
action_config: {
|
|
431
|
+
actions: configWithDefaults.actionConfig.actions,
|
|
432
|
+
objects: configWithDefaults.actionConfig.objects,
|
|
433
|
+
characters: configWithDefaults.actionConfig.characters,
|
|
434
|
+
...(configWithDefaults.actionConfig.current_attention_object && {
|
|
435
|
+
current_attention_object: configWithDefaults.actionConfig.current_attention_object,
|
|
436
|
+
}),
|
|
437
|
+
},
|
|
411
438
|
}),
|
|
412
439
|
...(configWithDefaults.dynamicInfo && {
|
|
413
440
|
dynamic_info: {
|
|
@@ -429,18 +456,19 @@ export class ConvaiClient extends EventEmitter {
|
|
|
429
456
|
}),
|
|
430
457
|
},
|
|
431
458
|
};
|
|
432
|
-
const
|
|
459
|
+
const connectHeaders = {
|
|
433
460
|
"Content-Type": "application/json",
|
|
434
461
|
};
|
|
435
462
|
if (configWithDefaults.authToken) {
|
|
436
|
-
|
|
463
|
+
connectHeaders["API-AUTH-TOKEN"] = configWithDefaults.authToken;
|
|
437
464
|
}
|
|
438
465
|
else if (configWithDefaults.apiKey) {
|
|
439
|
-
|
|
466
|
+
connectHeaders["X-API-Key"] = configWithDefaults.apiKey;
|
|
440
467
|
}
|
|
468
|
+
// HTTP POST /connect — same request for both transports
|
|
441
469
|
const response = await fetch(`${configWithDefaults.url}/connect`, {
|
|
442
470
|
method: "POST",
|
|
443
|
-
headers,
|
|
471
|
+
headers: connectHeaders,
|
|
444
472
|
body: JSON.stringify(requestBody),
|
|
445
473
|
});
|
|
446
474
|
if (!response.ok) {
|
|
@@ -466,7 +494,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
466
494
|
throw new Error(errorMessage);
|
|
467
495
|
}
|
|
468
496
|
const connectionData = await response.json();
|
|
469
|
-
// Capture
|
|
497
|
+
// Capture session identifiers from /connect response
|
|
470
498
|
if (connectionData.character_session_id) {
|
|
471
499
|
this._characterSessionId = connectionData.character_session_id;
|
|
472
500
|
this._storedConfig = {
|
|
@@ -474,39 +502,87 @@ export class ConvaiClient extends EventEmitter {
|
|
|
474
502
|
characterSessionId: connectionData.character_session_id,
|
|
475
503
|
};
|
|
476
504
|
}
|
|
477
|
-
// Capture end_user_id and end_user_metadata from connection response
|
|
478
505
|
if (connectionData.end_user_id) {
|
|
479
506
|
this._endUserId = connectionData.end_user_id;
|
|
480
507
|
}
|
|
481
508
|
if (connectionData.end_user_metadata) {
|
|
482
509
|
this._endUserMetadata = connectionData.end_user_metadata;
|
|
483
510
|
}
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
511
|
+
if (transportType === "websocket") {
|
|
512
|
+
// ── WebSocket (Pipecat) transport path ────────────────────────────
|
|
513
|
+
// Server returns the WebSocket URL in room_url (same field as LiveKit).
|
|
514
|
+
const wsUrl = connectionData.room_url;
|
|
515
|
+
if (!wsUrl) {
|
|
516
|
+
throw new Error("No WebSocket URL returned from /connect endpoint (expected room_url)");
|
|
517
|
+
}
|
|
518
|
+
// Always enable mic at construction so WavMediaManager initializes the
|
|
519
|
+
// audio stream during connect(). If startWithAudioOn is false, we mute
|
|
520
|
+
// after the connection is established (enableMic(false) works once the
|
|
521
|
+
// stream exists, but not before it is initialized).
|
|
522
|
+
this._wsSession = new WebSocketSession((payload) => this._messageHandler.handleDataReceivedPublic(payload), true);
|
|
523
|
+
this._wsSession.on("botAudioTrack", (track) => {
|
|
524
|
+
this.emit("botAudioTrack", track);
|
|
525
|
+
});
|
|
526
|
+
this._wsSession.on("disconnected", () => {
|
|
527
|
+
this._connectionStateHandler.handleDisconnected();
|
|
528
|
+
});
|
|
529
|
+
this._audioManager.setWebSocketSession(this._wsSession);
|
|
530
|
+
// Wait for WebSocket transport to connect (fires before bot-ready)
|
|
531
|
+
await new Promise((resolve, reject) => {
|
|
532
|
+
const onConnected = () => {
|
|
533
|
+
this._wsSession.off("connected", onConnected);
|
|
534
|
+
this._wsSession.off("error", onError);
|
|
535
|
+
resolve();
|
|
536
|
+
};
|
|
537
|
+
const onError = (err) => {
|
|
538
|
+
this._wsSession.off("connected", onConnected);
|
|
539
|
+
this._wsSession.off("error", onError);
|
|
540
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
541
|
+
};
|
|
542
|
+
this._wsSession.on("connected", onConnected);
|
|
543
|
+
this._wsSession.on("error", onError);
|
|
544
|
+
// connectWithUrl calls initDevices() then connect({ wsUrl }) — matches sandbox flow
|
|
545
|
+
this._wsSession.connectWithUrl(wsUrl).catch((err) => {
|
|
546
|
+
this._wsSession.off("connected", onConnected);
|
|
547
|
+
this._wsSession.off("error", onError);
|
|
548
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
549
|
+
});
|
|
499
550
|
});
|
|
551
|
+
// For WebSocket, mic is already streaming after connect.
|
|
552
|
+
// Only mute if the caller explicitly sets startWithAudioOn: false.
|
|
553
|
+
// Defaulting to on matches the sandbox behavior and ensures the bot
|
|
554
|
+
// sees audio when it initializes (avoids server "tap mic to talk" prompt).
|
|
555
|
+
if (configWithDefaults.startWithAudioOn === false) {
|
|
556
|
+
this._wsSession.enableMic(false);
|
|
557
|
+
this._audioManager.syncWsAudioState(false);
|
|
558
|
+
}
|
|
559
|
+
else {
|
|
560
|
+
this._audioManager.syncWsAudioState(true);
|
|
561
|
+
}
|
|
500
562
|
}
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
563
|
+
else {
|
|
564
|
+
// ── LiveKit transport path (default) ─────────────────────────────
|
|
565
|
+
await this._room.connect(connectionData.room_url, connectionData.token, {
|
|
566
|
+
rtcConfig: {
|
|
567
|
+
iceTransportPolicy: "relay",
|
|
568
|
+
},
|
|
569
|
+
});
|
|
570
|
+
if (configWithDefaults.startWithAudioOn) {
|
|
571
|
+
await this._room.localParticipant.setMicrophoneEnabled(true, {
|
|
572
|
+
echoCancellation: this._audioSettings.echoCancellation,
|
|
573
|
+
noiseSuppression: this._audioSettings.noiseSuppression,
|
|
574
|
+
autoGainControl: this._audioSettings.autoGainControl,
|
|
575
|
+
sampleRate: this._audioSettings.sampleRate,
|
|
576
|
+
channelCount: this._audioSettings.channelCount,
|
|
577
|
+
});
|
|
578
|
+
}
|
|
579
|
+
if (configWithDefaults.enableVideo &&
|
|
580
|
+
configWithDefaults.startWithVideoOn) {
|
|
581
|
+
await this._room.localParticipant.setCameraEnabled(true);
|
|
582
|
+
}
|
|
583
|
+
this._audioManager.syncStateFromRoom({ emit: true });
|
|
584
|
+
this._participantSid = this._room.localParticipant.sid;
|
|
505
585
|
}
|
|
506
|
-
// Ensure audio manager mirrors the actual microphone permission state
|
|
507
|
-
this._audioManager.syncStateFromRoom({ emit: true });
|
|
508
|
-
// Capture participant SID
|
|
509
|
-
this._participantSid = this._room.localParticipant.sid;
|
|
510
586
|
// Apply custom mapper to blendshape queue if provided
|
|
511
587
|
if (configWithDefaults.blendshapeConfig?.customMapper) {
|
|
512
588
|
this.blendshapeQueue.setMapper(configWithDefaults.blendshapeConfig.customMapper);
|
|
@@ -545,7 +621,18 @@ export class ConvaiClient extends EventEmitter {
|
|
|
545
621
|
* Disconnect from the current character session
|
|
546
622
|
*/
|
|
547
623
|
async disconnect() {
|
|
548
|
-
if (this.
|
|
624
|
+
if (this._activeTransport === "websocket" && this._wsSession) {
|
|
625
|
+
try {
|
|
626
|
+
await this._wsSession.disconnect();
|
|
627
|
+
}
|
|
628
|
+
catch {
|
|
629
|
+
// ignore disconnect errors
|
|
630
|
+
}
|
|
631
|
+
this._wsSession = null;
|
|
632
|
+
this._connectionStateHandler.resetConnectionState();
|
|
633
|
+
this.clearAllManagers();
|
|
634
|
+
}
|
|
635
|
+
else if (this._room && this._room.state !== "disconnected") {
|
|
549
636
|
try {
|
|
550
637
|
await this._room.disconnect();
|
|
551
638
|
this._connectionStateHandler.resetConnectionState();
|
|
@@ -563,6 +650,8 @@ export class ConvaiClient extends EventEmitter {
|
|
|
563
650
|
*/
|
|
564
651
|
clearAllManagers() {
|
|
565
652
|
this._connectionType = null;
|
|
653
|
+
this._activeTransport = null;
|
|
654
|
+
this._wsSession = null;
|
|
566
655
|
this._apiKey = null;
|
|
567
656
|
this._authToken = null;
|
|
568
657
|
this._characterId = null;
|
|
@@ -570,6 +659,7 @@ export class ConvaiClient extends EventEmitter {
|
|
|
570
659
|
this._endUserId = null;
|
|
571
660
|
this._endUserMetadata = null;
|
|
572
661
|
this._memoryManager = null; // Clear memory manager on disconnect
|
|
662
|
+
this._audioManager.setWebSocketSession(null);
|
|
573
663
|
this._audioManager.reset();
|
|
574
664
|
this._videoManager.reset();
|
|
575
665
|
this._screenShareManager.reset();
|
|
@@ -596,14 +686,10 @@ export class ConvaiClient extends EventEmitter {
|
|
|
596
686
|
* Send a text message to the character
|
|
597
687
|
*/
|
|
598
688
|
sendUserTextMessage(text) {
|
|
599
|
-
if (!this.
|
|
600
|
-
this._room.state === "disconnected" ||
|
|
601
|
-
!this._room.localParticipant) {
|
|
689
|
+
if (!this.isTransportReady())
|
|
602
690
|
return;
|
|
603
|
-
|
|
604
|
-
if (!text || !text.trim()) {
|
|
691
|
+
if (!text || !text.trim())
|
|
605
692
|
return;
|
|
606
|
-
}
|
|
607
693
|
try {
|
|
608
694
|
// Reset blendshape queue if mid-conversation, but don't interrupt the server
|
|
609
695
|
if (this.blendshapeQueue.isConversationActive() ||
|
|
@@ -619,16 +705,9 @@ export class ConvaiClient extends EventEmitter {
|
|
|
619
705
|
userMessage: text.trim(),
|
|
620
706
|
timestamp: this._conversationStartTime,
|
|
621
707
|
});
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
text: text.trim(),
|
|
626
|
-
participant_sid: this._participantSid || this._room.localParticipant.sid,
|
|
627
|
-
},
|
|
628
|
-
};
|
|
629
|
-
const encodedData = new TextEncoder().encode(JSON.stringify(message));
|
|
630
|
-
this._room.localParticipant.publishData(encodedData, {
|
|
631
|
-
reliable: true,
|
|
708
|
+
this.publishMessage("user_text_message", {
|
|
709
|
+
text: text.trim(),
|
|
710
|
+
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
632
711
|
});
|
|
633
712
|
}
|
|
634
713
|
catch (error) {
|
|
@@ -639,32 +718,24 @@ export class ConvaiClient extends EventEmitter {
|
|
|
639
718
|
* Send a trigger message to invoke specific character actions
|
|
640
719
|
*/
|
|
641
720
|
sendTriggerMessage(triggerName, triggerMessage) {
|
|
642
|
-
if (this.
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
participant_sid: this._participantSid || this._room.localParticipant.sid,
|
|
661
|
-
},
|
|
662
|
-
};
|
|
663
|
-
const encodedData = new TextEncoder().encode(JSON.stringify(message));
|
|
664
|
-
this._room.localParticipant.publishData(encodedData, {
|
|
665
|
-
reliable: true,
|
|
666
|
-
});
|
|
667
|
-
}
|
|
721
|
+
if (!this.isTransportReady())
|
|
722
|
+
return;
|
|
723
|
+
// Reset blendshape queue if mid-conversation, but don't interrupt the server
|
|
724
|
+
if (this.blendshapeQueue.isConversationActive() || this._state.isSpeaking) {
|
|
725
|
+
this.blendshapeQueue.interrupt();
|
|
726
|
+
}
|
|
727
|
+
this._conversationSessionId++;
|
|
728
|
+
this._conversationStartTime = Date.now();
|
|
729
|
+
this.emit("conversationStart", {
|
|
730
|
+
sessionId: this._conversationSessionId,
|
|
731
|
+
userMessage: `[trigger:${triggerName ?? ""}]`,
|
|
732
|
+
timestamp: this._conversationStartTime,
|
|
733
|
+
});
|
|
734
|
+
this.publishMessage("trigger-message", {
|
|
735
|
+
...(triggerName && { trigger_name: triggerName }),
|
|
736
|
+
...(triggerMessage && { trigger_message: triggerMessage }),
|
|
737
|
+
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
738
|
+
});
|
|
668
739
|
}
|
|
669
740
|
/**
|
|
670
741
|
* If the bot is currently speaking or a conversation is active, send an
|
|
@@ -676,17 +747,10 @@ export class ConvaiClient extends EventEmitter {
|
|
|
676
747
|
const botSpeaking = this._state.isSpeaking;
|
|
677
748
|
if (!queueActive && !botSpeaking)
|
|
678
749
|
return;
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
const message = {
|
|
684
|
-
type: "interrupt-bot",
|
|
685
|
-
data: {
|
|
686
|
-
participant_sid: this._participantSid || this._room.localParticipant.sid,
|
|
687
|
-
},
|
|
688
|
-
};
|
|
689
|
-
this._room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(message)), { reliable: true });
|
|
750
|
+
if (this.isTransportReady()) {
|
|
751
|
+
this.publishMessage("interrupt-bot", {
|
|
752
|
+
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
753
|
+
});
|
|
690
754
|
}
|
|
691
755
|
// End the blendshape queue so the lipsync player finishes cleanly
|
|
692
756
|
this.blendshapeQueue.interrupt();
|
|
@@ -695,26 +759,20 @@ export class ConvaiClient extends EventEmitter {
|
|
|
695
759
|
* Send an interrupt message to stop the bot's current response
|
|
696
760
|
*/
|
|
697
761
|
sendInterruptMessage() {
|
|
698
|
-
if (!this.
|
|
699
|
-
this._room.state === "disconnected" ||
|
|
700
|
-
!this._room.localParticipant) {
|
|
762
|
+
if (!this.isTransportReady())
|
|
701
763
|
return;
|
|
702
|
-
}
|
|
703
764
|
try {
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
data: {
|
|
707
|
-
participant_sid: this._participantSid || this._room.localParticipant.sid,
|
|
708
|
-
},
|
|
709
|
-
};
|
|
710
|
-
const encodedData = new TextEncoder().encode(JSON.stringify(message));
|
|
711
|
-
this._room.localParticipant.publishData(encodedData, {
|
|
712
|
-
reliable: true,
|
|
765
|
+
this.publishMessage("interrupt-bot", {
|
|
766
|
+
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
713
767
|
});
|
|
714
|
-
//
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
768
|
+
// Blendshape fade-out only for LiveKit (WebSocket transport has no lipsync;
|
|
769
|
+
// calling interrupt() would set conversationEnded=true and block the next
|
|
770
|
+
// bot-started-speaking from ever firing isSpeaking).
|
|
771
|
+
if (this._activeTransport !== "websocket") {
|
|
772
|
+
if (this.blendshapeQueue.isConversationActive() ||
|
|
773
|
+
this._state.isSpeaking) {
|
|
774
|
+
this.blendshapeQueue.interrupt();
|
|
775
|
+
}
|
|
718
776
|
}
|
|
719
777
|
}
|
|
720
778
|
catch (error) {
|
|
@@ -725,39 +783,23 @@ export class ConvaiClient extends EventEmitter {
|
|
|
725
783
|
* Update template keys in the character's context
|
|
726
784
|
*/
|
|
727
785
|
updateTemplateKeys(templateKeys) {
|
|
728
|
-
if (this.
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
template_keys: templateKeys,
|
|
735
|
-
participant_sid: this._participantSid || this._room.localParticipant.sid,
|
|
736
|
-
},
|
|
737
|
-
};
|
|
738
|
-
const encodedData = new TextEncoder().encode(JSON.stringify(message));
|
|
739
|
-
this._room.localParticipant.publishData(encodedData, {
|
|
740
|
-
reliable: true,
|
|
741
|
-
});
|
|
742
|
-
}
|
|
786
|
+
if (!this.isTransportReady() || Object.keys(templateKeys).length === 0)
|
|
787
|
+
return;
|
|
788
|
+
this.publishMessage("update-template-keys", {
|
|
789
|
+
template_keys: templateKeys,
|
|
790
|
+
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
791
|
+
});
|
|
743
792
|
}
|
|
744
793
|
/**
|
|
745
794
|
* Update dynamic information about the current context
|
|
746
795
|
*/
|
|
747
796
|
updateDynamicInfo(dynamicInfo) {
|
|
748
|
-
if (this.
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
},
|
|
755
|
-
};
|
|
756
|
-
const encodedData = new TextEncoder().encode(JSON.stringify(message));
|
|
757
|
-
this._room.localParticipant.publishData(encodedData, {
|
|
758
|
-
reliable: true,
|
|
759
|
-
});
|
|
760
|
-
}
|
|
797
|
+
if (!this.isTransportReady() || !dynamicInfo?.trim())
|
|
798
|
+
return;
|
|
799
|
+
this.publishMessage("update-dynamic-info", {
|
|
800
|
+
dynamic_info: { text: dynamicInfo },
|
|
801
|
+
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
802
|
+
});
|
|
761
803
|
}
|
|
762
804
|
/**
|
|
763
805
|
* Update the bot's temporary runtime context in a unified way.
|
|
@@ -786,19 +828,15 @@ export class ConvaiClient extends EventEmitter {
|
|
|
786
828
|
* ```
|
|
787
829
|
*/
|
|
788
830
|
updateContext(options) {
|
|
789
|
-
if (!this.
|
|
831
|
+
if (!this.isTransportReady())
|
|
790
832
|
return;
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
833
|
+
const hasText = !!options.text?.trim();
|
|
834
|
+
const hasAttention = options.current_attention_object !== undefined;
|
|
835
|
+
// text is required unless mode is "reset" or only updating attention
|
|
836
|
+
if (options.mode !== "reset" && !hasText && !hasAttention) {
|
|
837
|
+
console.warn("[ConvaiClient] updateContext: text or current_attention_object is required unless mode is 'reset'");
|
|
795
838
|
return;
|
|
796
839
|
}
|
|
797
|
-
// When run_llm is "false" the server won't respond, so no conversation
|
|
798
|
-
// lifecycle is needed. For "true", "auto", or unset the server may respond,
|
|
799
|
-
// so we treat it the same as sendUserTextMessage: interrupt any active turn
|
|
800
|
-
// and start a new conversation so the blendshape queue is ready to accept
|
|
801
|
-
// the incoming frames.
|
|
802
840
|
const willTriggerLlm = options.run_llm !== "false";
|
|
803
841
|
if (willTriggerLlm) {
|
|
804
842
|
if (this.blendshapeQueue.isConversationActive() ||
|
|
@@ -813,70 +851,38 @@ export class ConvaiClient extends EventEmitter {
|
|
|
813
851
|
timestamp: this._conversationStartTime,
|
|
814
852
|
});
|
|
815
853
|
}
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
participant_sid: this._participantSid || this._room.localParticipant.sid,
|
|
823
|
-
},
|
|
824
|
-
};
|
|
825
|
-
const encodedData = new TextEncoder().encode(JSON.stringify(message));
|
|
826
|
-
this._room.localParticipant.publishData(encodedData, {
|
|
827
|
-
reliable: true,
|
|
854
|
+
this.publishMessage("context-update", {
|
|
855
|
+
...(hasText && { text: options.text.trim() }),
|
|
856
|
+
...(options.mode && { mode: options.mode }),
|
|
857
|
+
...(options.run_llm && { run_llm: options.run_llm }),
|
|
858
|
+
...(hasAttention && { current_attention_object: options.current_attention_object }),
|
|
859
|
+
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
828
860
|
});
|
|
829
861
|
}
|
|
862
|
+
/**
|
|
863
|
+
* Send descriptive scene updates. Use when environment description changes
|
|
864
|
+
* and the bot should know what is visible or nearby. Does not modify action_config.
|
|
865
|
+
*/
|
|
866
|
+
updateSceneMetadata(items) {
|
|
867
|
+
if (!this.isTransportReady())
|
|
868
|
+
return;
|
|
869
|
+
this.publishMessage("update-scene-metadata", { scene_metadata: items });
|
|
870
|
+
}
|
|
830
871
|
/**
|
|
831
872
|
* Toggle text-to-speech on or off
|
|
832
873
|
*/
|
|
833
874
|
toggleTts(enabled) {
|
|
834
|
-
if (!this.
|
|
835
|
-
this._room.state === "disconnected" ||
|
|
836
|
-
!this._room.localParticipant) {
|
|
875
|
+
if (!this.isTransportReady())
|
|
837
876
|
return;
|
|
838
|
-
}
|
|
839
|
-
try {
|
|
840
|
-
const message = {
|
|
841
|
-
type: "tts-toggle",
|
|
842
|
-
data: {
|
|
843
|
-
enabled: enabled,
|
|
844
|
-
},
|
|
845
|
-
};
|
|
846
|
-
const encodedData = new TextEncoder().encode(JSON.stringify(message));
|
|
847
|
-
this._room.localParticipant.publishData(encodedData, {
|
|
848
|
-
reliable: true,
|
|
849
|
-
});
|
|
850
|
-
}
|
|
851
|
-
catch (error) {
|
|
852
|
-
throw error;
|
|
853
|
-
}
|
|
877
|
+
this.publishMessage("tts-toggle", { enabled });
|
|
854
878
|
}
|
|
855
879
|
/**
|
|
856
880
|
* Toggle speech-to-text on or off
|
|
857
881
|
*/
|
|
858
882
|
toggleStt(enabled) {
|
|
859
|
-
if (!this.
|
|
860
|
-
this._room.state === "disconnected" ||
|
|
861
|
-
!this._room.localParticipant) {
|
|
883
|
+
if (!this.isTransportReady())
|
|
862
884
|
return;
|
|
863
|
-
}
|
|
864
|
-
try {
|
|
865
|
-
const message = {
|
|
866
|
-
lable: "rtvi-ai",
|
|
867
|
-
type: "stt-toggle",
|
|
868
|
-
data: {
|
|
869
|
-
muted: !enabled,
|
|
870
|
-
},
|
|
871
|
-
};
|
|
872
|
-
const encodedData = new TextEncoder().encode(JSON.stringify(message));
|
|
873
|
-
this._room.localParticipant.publishData(encodedData, {
|
|
874
|
-
reliable: true,
|
|
875
|
-
});
|
|
876
|
-
}
|
|
877
|
-
catch (error) {
|
|
878
|
-
throw error;
|
|
879
|
-
}
|
|
885
|
+
this.publishMessage("stt-toggle", { muted: !enabled });
|
|
880
886
|
}
|
|
881
887
|
/**
|
|
882
888
|
* Reset the server-side idle timer.
|
|
@@ -884,26 +890,11 @@ export class ConvaiClient extends EventEmitter {
|
|
|
884
890
|
* the session from being disconnected due to inactivity.
|
|
885
891
|
*/
|
|
886
892
|
resetIdleTimer() {
|
|
887
|
-
if (!this.
|
|
888
|
-
this._room.state === "disconnected" ||
|
|
889
|
-
!this._room.localParticipant) {
|
|
893
|
+
if (!this.isTransportReady())
|
|
890
894
|
return;
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
type: "reset-idle-timer",
|
|
895
|
-
data: {
|
|
896
|
-
participant_sid: this._participantSid || this._room.localParticipant.sid,
|
|
897
|
-
},
|
|
898
|
-
};
|
|
899
|
-
const encodedData = new TextEncoder().encode(JSON.stringify(message));
|
|
900
|
-
this._room.localParticipant.publishData(encodedData, {
|
|
901
|
-
reliable: true,
|
|
902
|
-
});
|
|
903
|
-
}
|
|
904
|
-
catch (error) {
|
|
905
|
-
throw error;
|
|
906
|
-
}
|
|
895
|
+
this.publishMessage("reset-idle-timer", {
|
|
896
|
+
participant_sid: this._participantSid || this._room.localParticipant?.sid,
|
|
897
|
+
});
|
|
907
898
|
}
|
|
908
899
|
}
|
|
909
900
|
//# sourceMappingURL=ConvaiClient.js.map
|