@convai/web-sdk 1.3.2-beta.0 → 1.4.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,6 +6,7 @@ import { MessageHandler } from "./MessageHandler";
6
6
  import { EventEmitter } from "./EventEmitter";
7
7
  import { MemoryManager } from "./MemoryManager";
8
8
  import { ConnectionStateHandler } from "./ConnectionStateHandler";
9
+ import { WebSocketSession } from "./WebSocketSession";
9
10
  /**
10
11
  * Custom reconnect policy that disables automatic reconnection
11
12
  */
@@ -64,6 +65,8 @@ export class ConvaiClient extends EventEmitter {
64
65
  this._storedConfig = null;
65
66
  this._endUserId = null;
66
67
  this._endUserMetadata = null;
68
+ this._wsSession = null;
69
+ this._activeTransport = null;
67
70
  // Conversation session tracking
68
71
  this._conversationSessionId = 0;
69
72
  this._conversationStartTime = 0;
@@ -280,6 +283,9 @@ export class ConvaiClient extends EventEmitter {
280
283
  this._messageHandler.on("userMuteStopped", () => {
281
284
  this.emit("userMuteStopped");
282
285
  });
286
+ this._messageHandler.on("actionResponse", (data) => {
287
+ this.emit("actionResponse", data);
288
+ });
283
289
  }
284
290
  /**
285
291
  * Setup MessageHandler to respond to conversation start events
@@ -318,6 +324,30 @@ export class ConvaiClient extends EventEmitter {
318
324
  this.emit("stateChange", this._state);
319
325
  }
320
326
  }
327
+ /**
328
+ * Send a data message via the active transport.
329
+ * For LiveKit: publishes via the data channel.
330
+ * For WebSocket: sends via PipecatClient.sendClientMessage (wrapped in {t,d} envelope).
331
+ */
332
+ publishMessage(type, data) {
333
+ if (this._activeTransport === "websocket" && this._wsSession?.isConnected) {
334
+ this._wsSession.sendMessage(type, data);
335
+ }
336
+ else if (this._room?.localParticipant &&
337
+ this._room.state !== "disconnected") {
338
+ const encodedData = new TextEncoder().encode(JSON.stringify({ type, data }));
339
+ this._room.localParticipant.publishData(encodedData, { reliable: true });
340
+ }
341
+ }
342
+ /**
343
+ * Returns true when a message can be sent on the active transport.
344
+ */
345
+ isTransportReady() {
346
+ if (this._activeTransport === "websocket") {
347
+ return this._wsSession?.isConnected ?? false;
348
+ }
349
+ return (!!this._room?.localParticipant && this._room.state !== "disconnected");
350
+ }
321
351
  /**
322
352
  * Connect to a Convai character
323
353
  */
@@ -336,7 +366,7 @@ export class ConvaiClient extends EventEmitter {
336
366
  }
337
367
  // Add default URL if not provided
338
368
  const configWithDefaults = {
339
- url: "https://realtime-api-preview.convai.com",
369
+ url: "https://realtime-api.convai.com",
340
370
  ...finalConfig,
341
371
  };
342
372
  const hasApiKey = Boolean(configWithDefaults.apiKey);
@@ -354,6 +384,8 @@ export class ConvaiClient extends EventEmitter {
354
384
  // Determine connection type based on enableVideo
355
385
  const connType = configWithDefaults.enableVideo ? "video" : "audio";
356
386
  this._connectionType = connType;
387
+ const transportType = configWithDefaults.transport ?? "livekit";
388
+ this._activeTransport = transportType;
357
389
  // Prepare request body with required parameters
358
390
  const characterSessionIdToSend = configWithDefaults.characterSessionId ?? this._characterSessionId;
359
391
  const requestBody = {
@@ -364,7 +396,7 @@ export class ConvaiClient extends EventEmitter {
364
396
  ...(configWithDefaults.endUserMetadata != null && {
365
397
  end_user_metadata: configWithDefaults.endUserMetadata,
366
398
  }),
367
- transport: "livekit",
399
+ transport: transportType,
368
400
  connection_type: connType,
369
401
  blendshape_provider: configWithDefaults.enableLipsync
370
402
  ? "neurosync"
@@ -377,7 +409,7 @@ export class ConvaiClient extends EventEmitter {
377
409
  output_fps: 90,
378
410
  },
379
411
  llm_provider: "dynamic",
380
- ...(configWithDefaults.enableEmotion !== false && {
412
+ ...(configWithDefaults.enableEmotion === true && {
381
413
  emotion_config: {
382
414
  provider: configWithDefaults.emotionConfig?.provider ?? "llm",
383
415
  min_word_threshold: configWithDefaults.emotionConfig?.min_word_threshold ?? 3,
@@ -395,19 +427,14 @@ export class ConvaiClient extends EventEmitter {
395
427
  character_session_id: characterSessionIdToSend,
396
428
  }),
397
429
  ...(configWithDefaults.actionConfig && {
398
- action_config: configWithDefaults.actionConfig,
399
- }),
400
- ...(configWithDefaults.characters && {
401
- characters: configWithDefaults.characters,
402
- }),
403
- ...(configWithDefaults.objects && {
404
- objects: configWithDefaults.objects,
405
- }),
406
- ...(configWithDefaults.currentAttentionObject && {
407
- current_attention_object: configWithDefaults.currentAttentionObject,
408
- }),
409
- ...(configWithDefaults.sceneDescription && {
410
- scene_description: configWithDefaults.sceneDescription,
430
+ action_config: {
431
+ actions: configWithDefaults.actionConfig.actions,
432
+ objects: configWithDefaults.actionConfig.objects,
433
+ characters: configWithDefaults.actionConfig.characters,
434
+ ...(configWithDefaults.actionConfig.current_attention_object && {
435
+ current_attention_object: configWithDefaults.actionConfig.current_attention_object,
436
+ }),
437
+ },
411
438
  }),
412
439
  ...(configWithDefaults.dynamicInfo && {
413
440
  dynamic_info: {
@@ -429,18 +456,19 @@ export class ConvaiClient extends EventEmitter {
429
456
  }),
430
457
  },
431
458
  };
432
- const headers = {
459
+ const connectHeaders = {
433
460
  "Content-Type": "application/json",
434
461
  };
435
462
  if (configWithDefaults.authToken) {
436
- headers["API-AUTH-TOKEN"] = configWithDefaults.authToken;
463
+ connectHeaders["API-AUTH-TOKEN"] = configWithDefaults.authToken;
437
464
  }
438
465
  else if (configWithDefaults.apiKey) {
439
- headers["X-API-Key"] = configWithDefaults.apiKey;
466
+ connectHeaders["X-API-Key"] = configWithDefaults.apiKey;
440
467
  }
468
+ // HTTP POST /connect — same request for both transports
441
469
  const response = await fetch(`${configWithDefaults.url}/connect`, {
442
470
  method: "POST",
443
- headers,
471
+ headers: connectHeaders,
444
472
  body: JSON.stringify(requestBody),
445
473
  });
446
474
  if (!response.ok) {
@@ -466,7 +494,7 @@ export class ConvaiClient extends EventEmitter {
466
494
  throw new Error(errorMessage);
467
495
  }
468
496
  const connectionData = await response.json();
469
- // Capture character_session_id from connection response and persist in stored config for reconnection
497
+ // Capture session identifiers from /connect response
470
498
  if (connectionData.character_session_id) {
471
499
  this._characterSessionId = connectionData.character_session_id;
472
500
  this._storedConfig = {
@@ -474,39 +502,87 @@ export class ConvaiClient extends EventEmitter {
474
502
  characterSessionId: connectionData.character_session_id,
475
503
  };
476
504
  }
477
- // Capture end_user_id and end_user_metadata from connection response
478
505
  if (connectionData.end_user_id) {
479
506
  this._endUserId = connectionData.end_user_id;
480
507
  }
481
508
  if (connectionData.end_user_metadata) {
482
509
  this._endUserMetadata = connectionData.end_user_metadata;
483
510
  }
484
- // Connect to LiveKit room
485
- await this._room.connect(connectionData.room_url, connectionData.token, {
486
- rtcConfig: {
487
- iceTransportPolicy: "relay",
488
- },
489
- });
490
- // Enable microphone only if startWithAudioOn is true (default: false)
491
- // If false, microphone stays off until user enables it via audioControls
492
- if (configWithDefaults.startWithAudioOn) {
493
- await this._room.localParticipant.setMicrophoneEnabled(true, {
494
- echoCancellation: this._audioSettings.echoCancellation,
495
- noiseSuppression: this._audioSettings.noiseSuppression,
496
- autoGainControl: this._audioSettings.autoGainControl,
497
- sampleRate: this._audioSettings.sampleRate,
498
- channelCount: this._audioSettings.channelCount,
511
+ if (transportType === "websocket") {
512
+ // ── WebSocket (Pipecat) transport path ────────────────────────────
513
+ // Server returns the WebSocket URL in room_url (same field as LiveKit).
514
+ const wsUrl = connectionData.room_url;
515
+ if (!wsUrl) {
516
+ throw new Error("No WebSocket URL returned from /connect endpoint (expected room_url)");
517
+ }
518
+ // Always enable mic at construction so WavMediaManager initializes the
519
+ // audio stream during connect(). If startWithAudioOn is false, we mute
520
+ // after the connection is established (enableMic(false) works once the
521
+ // stream exists, but not before it is initialized).
522
+ this._wsSession = new WebSocketSession((payload) => this._messageHandler.handleDataReceivedPublic(payload), true);
523
+ this._wsSession.on("botAudioTrack", (track) => {
524
+ this.emit("botAudioTrack", track);
525
+ });
526
+ this._wsSession.on("disconnected", () => {
527
+ this._connectionStateHandler.handleDisconnected();
528
+ });
529
+ this._audioManager.setWebSocketSession(this._wsSession);
530
+ // Wait for WebSocket transport to connect (fires before bot-ready)
531
+ await new Promise((resolve, reject) => {
532
+ const onConnected = () => {
533
+ this._wsSession.off("connected", onConnected);
534
+ this._wsSession.off("error", onError);
535
+ resolve();
536
+ };
537
+ const onError = (err) => {
538
+ this._wsSession.off("connected", onConnected);
539
+ this._wsSession.off("error", onError);
540
+ reject(err instanceof Error ? err : new Error(String(err)));
541
+ };
542
+ this._wsSession.on("connected", onConnected);
543
+ this._wsSession.on("error", onError);
544
+ // connectWithUrl calls initDevices() then connect({ wsUrl }) — matches sandbox flow
545
+ this._wsSession.connectWithUrl(wsUrl).catch((err) => {
546
+ this._wsSession.off("connected", onConnected);
547
+ this._wsSession.off("error", onError);
548
+ reject(err instanceof Error ? err : new Error(String(err)));
549
+ });
499
550
  });
551
+ // For WebSocket, mic is already streaming after connect.
552
+ // Only mute if the caller explicitly sets startWithAudioOn: false.
553
+ // Defaulting to on matches the sandbox behavior and ensures the bot
554
+ // sees audio when it initializes (avoids server "tap mic to talk" prompt).
555
+ if (configWithDefaults.startWithAudioOn === false) {
556
+ this._wsSession.enableMic(false);
557
+ this._audioManager.syncWsAudioState(false);
558
+ }
559
+ else {
560
+ this._audioManager.syncWsAudioState(true);
561
+ }
500
562
  }
501
- // Enable camera only if enableVideo is true AND startWithVideoOn is true
502
- if (configWithDefaults.enableVideo &&
503
- configWithDefaults.startWithVideoOn) {
504
- await this._room.localParticipant.setCameraEnabled(true);
563
+ else {
564
+ // ── LiveKit transport path (default) ─────────────────────────────
565
+ await this._room.connect(connectionData.room_url, connectionData.token, {
566
+ rtcConfig: {
567
+ iceTransportPolicy: "relay",
568
+ },
569
+ });
570
+ if (configWithDefaults.startWithAudioOn) {
571
+ await this._room.localParticipant.setMicrophoneEnabled(true, {
572
+ echoCancellation: this._audioSettings.echoCancellation,
573
+ noiseSuppression: this._audioSettings.noiseSuppression,
574
+ autoGainControl: this._audioSettings.autoGainControl,
575
+ sampleRate: this._audioSettings.sampleRate,
576
+ channelCount: this._audioSettings.channelCount,
577
+ });
578
+ }
579
+ if (configWithDefaults.enableVideo &&
580
+ configWithDefaults.startWithVideoOn) {
581
+ await this._room.localParticipant.setCameraEnabled(true);
582
+ }
583
+ this._audioManager.syncStateFromRoom({ emit: true });
584
+ this._participantSid = this._room.localParticipant.sid;
505
585
  }
506
- // Ensure audio manager mirrors the actual microphone permission state
507
- this._audioManager.syncStateFromRoom({ emit: true });
508
- // Capture participant SID
509
- this._participantSid = this._room.localParticipant.sid;
510
586
  // Apply custom mapper to blendshape queue if provided
511
587
  if (configWithDefaults.blendshapeConfig?.customMapper) {
512
588
  this.blendshapeQueue.setMapper(configWithDefaults.blendshapeConfig.customMapper);
@@ -545,7 +621,18 @@ export class ConvaiClient extends EventEmitter {
545
621
  * Disconnect from the current character session
546
622
  */
547
623
  async disconnect() {
548
- if (this._room && this._room.state !== "disconnected") {
624
+ if (this._activeTransport === "websocket" && this._wsSession) {
625
+ try {
626
+ await this._wsSession.disconnect();
627
+ }
628
+ catch {
629
+ // ignore disconnect errors
630
+ }
631
+ this._wsSession = null;
632
+ this._connectionStateHandler.resetConnectionState();
633
+ this.clearAllManagers();
634
+ }
635
+ else if (this._room && this._room.state !== "disconnected") {
549
636
  try {
550
637
  await this._room.disconnect();
551
638
  this._connectionStateHandler.resetConnectionState();
@@ -563,6 +650,8 @@ export class ConvaiClient extends EventEmitter {
563
650
  */
564
651
  clearAllManagers() {
565
652
  this._connectionType = null;
653
+ this._activeTransport = null;
654
+ this._wsSession = null;
566
655
  this._apiKey = null;
567
656
  this._authToken = null;
568
657
  this._characterId = null;
@@ -570,6 +659,7 @@ export class ConvaiClient extends EventEmitter {
570
659
  this._endUserId = null;
571
660
  this._endUserMetadata = null;
572
661
  this._memoryManager = null; // Clear memory manager on disconnect
662
+ this._audioManager.setWebSocketSession(null);
573
663
  this._audioManager.reset();
574
664
  this._videoManager.reset();
575
665
  this._screenShareManager.reset();
@@ -596,14 +686,10 @@ export class ConvaiClient extends EventEmitter {
596
686
  * Send a text message to the character
597
687
  */
598
688
  sendUserTextMessage(text) {
599
- if (!this._room ||
600
- this._room.state === "disconnected" ||
601
- !this._room.localParticipant) {
689
+ if (!this.isTransportReady())
602
690
  return;
603
- }
604
- if (!text || !text.trim()) {
691
+ if (!text || !text.trim())
605
692
  return;
606
- }
607
693
  try {
608
694
  // Reset blendshape queue if mid-conversation, but don't interrupt the server
609
695
  if (this.blendshapeQueue.isConversationActive() ||
@@ -619,16 +705,9 @@ export class ConvaiClient extends EventEmitter {
619
705
  userMessage: text.trim(),
620
706
  timestamp: this._conversationStartTime,
621
707
  });
622
- const message = {
623
- type: "user_text_message",
624
- data: {
625
- text: text.trim(),
626
- participant_sid: this._participantSid || this._room.localParticipant.sid,
627
- },
628
- };
629
- const encodedData = new TextEncoder().encode(JSON.stringify(message));
630
- this._room.localParticipant.publishData(encodedData, {
631
- reliable: true,
708
+ this.publishMessage("user_text_message", {
709
+ text: text.trim(),
710
+ participant_sid: this._participantSid || this._room.localParticipant?.sid,
632
711
  });
633
712
  }
634
713
  catch (error) {
@@ -639,32 +718,24 @@ export class ConvaiClient extends EventEmitter {
639
718
  * Send a trigger message to invoke specific character actions
640
719
  */
641
720
  sendTriggerMessage(triggerName, triggerMessage) {
642
- if (this._room && this._room.localParticipant) {
643
- // Reset blendshape queue if mid-conversation, but don't interrupt the server
644
- if (this.blendshapeQueue.isConversationActive() ||
645
- this._state.isSpeaking) {
646
- this.blendshapeQueue.interrupt();
647
- }
648
- this._conversationSessionId++;
649
- this._conversationStartTime = Date.now();
650
- this.emit("conversationStart", {
651
- sessionId: this._conversationSessionId,
652
- userMessage: `[trigger:${triggerName ?? ""}]`,
653
- timestamp: this._conversationStartTime,
654
- });
655
- const message = {
656
- type: "trigger-message",
657
- data: {
658
- ...(triggerName && { trigger_name: triggerName }),
659
- ...(triggerMessage && { trigger_message: triggerMessage }),
660
- participant_sid: this._participantSid || this._room.localParticipant.sid,
661
- },
662
- };
663
- const encodedData = new TextEncoder().encode(JSON.stringify(message));
664
- this._room.localParticipant.publishData(encodedData, {
665
- reliable: true,
666
- });
667
- }
721
+ if (!this.isTransportReady())
722
+ return;
723
+ // Reset blendshape queue if mid-conversation, but don't interrupt the server
724
+ if (this.blendshapeQueue.isConversationActive() || this._state.isSpeaking) {
725
+ this.blendshapeQueue.interrupt();
726
+ }
727
+ this._conversationSessionId++;
728
+ this._conversationStartTime = Date.now();
729
+ this.emit("conversationStart", {
730
+ sessionId: this._conversationSessionId,
731
+ userMessage: `[trigger:${triggerName ?? ""}]`,
732
+ timestamp: this._conversationStartTime,
733
+ });
734
+ this.publishMessage("trigger-message", {
735
+ ...(triggerName && { trigger_name: triggerName }),
736
+ ...(triggerMessage && { trigger_message: triggerMessage }),
737
+ participant_sid: this._participantSid || this._room.localParticipant?.sid,
738
+ });
668
739
  }
669
740
  /**
670
741
  * If the bot is currently speaking or a conversation is active, send an
@@ -676,17 +747,10 @@ export class ConvaiClient extends EventEmitter {
676
747
  const botSpeaking = this._state.isSpeaking;
677
748
  if (!queueActive && !botSpeaking)
678
749
  return;
679
- // Tell the server to stop the current turn
680
- if (this._room &&
681
- this._room.state !== "disconnected" &&
682
- this._room.localParticipant) {
683
- const message = {
684
- type: "interrupt-bot",
685
- data: {
686
- participant_sid: this._participantSid || this._room.localParticipant.sid,
687
- },
688
- };
689
- this._room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(message)), { reliable: true });
750
+ if (this.isTransportReady()) {
751
+ this.publishMessage("interrupt-bot", {
752
+ participant_sid: this._participantSid || this._room.localParticipant?.sid,
753
+ });
690
754
  }
691
755
  // End the blendshape queue so the lipsync player finishes cleanly
692
756
  this.blendshapeQueue.interrupt();
@@ -695,26 +759,20 @@ export class ConvaiClient extends EventEmitter {
695
759
  * Send an interrupt message to stop the bot's current response
696
760
  */
697
761
  sendInterruptMessage() {
698
- if (!this._room ||
699
- this._room.state === "disconnected" ||
700
- !this._room.localParticipant) {
762
+ if (!this.isTransportReady())
701
763
  return;
702
- }
703
764
  try {
704
- const message = {
705
- type: "interrupt-bot",
706
- data: {
707
- participant_sid: this._participantSid || this._room.localParticipant.sid,
708
- },
709
- };
710
- const encodedData = new TextEncoder().encode(JSON.stringify(message));
711
- this._room.localParticipant.publishData(encodedData, {
712
- reliable: true,
765
+ this.publishMessage("interrupt-bot", {
766
+ participant_sid: this._participantSid || this._room.localParticipant?.sid,
713
767
  });
714
- // Trigger fade out for interruption - keeps last 10 frames
715
- if (this.blendshapeQueue.isConversationActive() ||
716
- this._state.isSpeaking) {
717
- this.blendshapeQueue.interrupt();
768
+ // Blendshape fade-out only for LiveKit (WebSocket transport has no lipsync;
769
+ // calling interrupt() would set conversationEnded=true and block the next
770
+ // bot-started-speaking from ever firing isSpeaking).
771
+ if (this._activeTransport !== "websocket") {
772
+ if (this.blendshapeQueue.isConversationActive() ||
773
+ this._state.isSpeaking) {
774
+ this.blendshapeQueue.interrupt();
775
+ }
718
776
  }
719
777
  }
720
778
  catch (error) {
@@ -725,39 +783,23 @@ export class ConvaiClient extends EventEmitter {
725
783
  * Update template keys in the character's context
726
784
  */
727
785
  updateTemplateKeys(templateKeys) {
728
- if (this._room &&
729
- this._room.localParticipant &&
730
- Object.keys(templateKeys).length > 0) {
731
- const message = {
732
- type: "update-template-keys",
733
- data: {
734
- template_keys: templateKeys,
735
- participant_sid: this._participantSid || this._room.localParticipant.sid,
736
- },
737
- };
738
- const encodedData = new TextEncoder().encode(JSON.stringify(message));
739
- this._room.localParticipant.publishData(encodedData, {
740
- reliable: true,
741
- });
742
- }
786
+ if (!this.isTransportReady() || Object.keys(templateKeys).length === 0)
787
+ return;
788
+ this.publishMessage("update-template-keys", {
789
+ template_keys: templateKeys,
790
+ participant_sid: this._participantSid || this._room.localParticipant?.sid,
791
+ });
743
792
  }
744
793
  /**
745
794
  * Update dynamic information about the current context
746
795
  */
747
796
  updateDynamicInfo(dynamicInfo) {
748
- if (this._room && this._room.localParticipant && dynamicInfo?.trim()) {
749
- const message = {
750
- type: "update-dynamic-info",
751
- data: {
752
- dynamic_info: { text: dynamicInfo },
753
- participant_sid: this._participantSid || this._room.localParticipant.sid,
754
- },
755
- };
756
- const encodedData = new TextEncoder().encode(JSON.stringify(message));
757
- this._room.localParticipant.publishData(encodedData, {
758
- reliable: true,
759
- });
760
- }
797
+ if (!this.isTransportReady() || !dynamicInfo?.trim())
798
+ return;
799
+ this.publishMessage("update-dynamic-info", {
800
+ dynamic_info: { text: dynamicInfo },
801
+ participant_sid: this._participantSid || this._room.localParticipant?.sid,
802
+ });
761
803
  }
762
804
  /**
763
805
  * Update the bot's temporary runtime context in a unified way.
@@ -786,19 +828,15 @@ export class ConvaiClient extends EventEmitter {
786
828
  * ```
787
829
  */
788
830
  updateContext(options) {
789
- if (!this._room || !this._room.localParticipant) {
831
+ if (!this.isTransportReady())
790
832
  return;
791
- }
792
- // Validate: text is required unless mode is "reset"
793
- if (options.mode !== "reset" && !options.text?.trim()) {
794
- console.warn("[ConvaiClient] updateContext: text is required unless mode is 'reset'");
833
+ const hasText = !!options.text?.trim();
834
+ const hasAttention = options.current_attention_object !== undefined;
835
+ // text is required unless mode is "reset" or only updating attention
836
+ if (options.mode !== "reset" && !hasText && !hasAttention) {
837
+ console.warn("[ConvaiClient] updateContext: text or current_attention_object is required unless mode is 'reset'");
795
838
  return;
796
839
  }
797
- // When run_llm is "false" the server won't respond, so no conversation
798
- // lifecycle is needed. For "true", "auto", or unset the server may respond,
799
- // so we treat it the same as sendUserTextMessage: interrupt any active turn
800
- // and start a new conversation so the blendshape queue is ready to accept
801
- // the incoming frames.
802
840
  const willTriggerLlm = options.run_llm !== "false";
803
841
  if (willTriggerLlm) {
804
842
  if (this.blendshapeQueue.isConversationActive() ||
@@ -813,70 +851,38 @@ export class ConvaiClient extends EventEmitter {
813
851
  timestamp: this._conversationStartTime,
814
852
  });
815
853
  }
816
- const message = {
817
- type: "context-update",
818
- data: {
819
- ...(options.text && { text: options.text.trim() }),
820
- ...(options.mode && { mode: options.mode }),
821
- ...(options.run_llm && { run_llm: options.run_llm }),
822
- participant_sid: this._participantSid || this._room.localParticipant.sid,
823
- },
824
- };
825
- const encodedData = new TextEncoder().encode(JSON.stringify(message));
826
- this._room.localParticipant.publishData(encodedData, {
827
- reliable: true,
854
+ this.publishMessage("context-update", {
855
+ ...(hasText && { text: options.text.trim() }),
856
+ ...(options.mode && { mode: options.mode }),
857
+ ...(options.run_llm && { run_llm: options.run_llm }),
858
+ ...(hasAttention && { current_attention_object: options.current_attention_object }),
859
+ participant_sid: this._participantSid || this._room.localParticipant?.sid,
828
860
  });
829
861
  }
862
+ /**
863
+ * Send descriptive scene updates. Use when environment description changes
864
+ * and the bot should know what is visible or nearby. Does not modify action_config.
865
+ */
866
+ updateSceneMetadata(items) {
867
+ if (!this.isTransportReady())
868
+ return;
869
+ this.publishMessage("update-scene-metadata", { scene_metadata: items });
870
+ }
830
871
  /**
831
872
  * Toggle text-to-speech on or off
832
873
  */
833
874
  toggleTts(enabled) {
834
- if (!this._room ||
835
- this._room.state === "disconnected" ||
836
- !this._room.localParticipant) {
875
+ if (!this.isTransportReady())
837
876
  return;
838
- }
839
- try {
840
- const message = {
841
- type: "tts-toggle",
842
- data: {
843
- enabled: enabled,
844
- },
845
- };
846
- const encodedData = new TextEncoder().encode(JSON.stringify(message));
847
- this._room.localParticipant.publishData(encodedData, {
848
- reliable: true,
849
- });
850
- }
851
- catch (error) {
852
- throw error;
853
- }
877
+ this.publishMessage("tts-toggle", { enabled });
854
878
  }
855
879
  /**
856
880
  * Toggle speech-to-text on or off
857
881
  */
858
882
  toggleStt(enabled) {
859
- if (!this._room ||
860
- this._room.state === "disconnected" ||
861
- !this._room.localParticipant) {
883
+ if (!this.isTransportReady())
862
884
  return;
863
- }
864
- try {
865
- const message = {
866
- lable: "rtvi-ai",
867
- type: "stt-toggle",
868
- data: {
869
- muted: !enabled,
870
- },
871
- };
872
- const encodedData = new TextEncoder().encode(JSON.stringify(message));
873
- this._room.localParticipant.publishData(encodedData, {
874
- reliable: true,
875
- });
876
- }
877
- catch (error) {
878
- throw error;
879
- }
885
+ this.publishMessage("stt-toggle", { muted: !enabled });
880
886
  }
881
887
  /**
882
888
  * Reset the server-side idle timer.
@@ -884,26 +890,11 @@ export class ConvaiClient extends EventEmitter {
884
890
  * the session from being disconnected due to inactivity.
885
891
  */
886
892
  resetIdleTimer() {
887
- if (!this._room ||
888
- this._room.state === "disconnected" ||
889
- !this._room.localParticipant) {
893
+ if (!this.isTransportReady())
890
894
  return;
891
- }
892
- try {
893
- const message = {
894
- type: "reset-idle-timer",
895
- data: {
896
- participant_sid: this._participantSid || this._room.localParticipant.sid,
897
- },
898
- };
899
- const encodedData = new TextEncoder().encode(JSON.stringify(message));
900
- this._room.localParticipant.publishData(encodedData, {
901
- reliable: true,
902
- });
903
- }
904
- catch (error) {
905
- throw error;
906
- }
895
+ this.publishMessage("reset-idle-timer", {
896
+ participant_sid: this._participantSid || this._room.localParticipant?.sid,
897
+ });
907
898
  }
908
899
  }
909
900
  //# sourceMappingURL=ConvaiClient.js.map