@convai/web-sdk 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +239 -0
- package/dist/components/RTCWidget.d.ts +56 -0
- package/dist/components/RTCWidget.d.ts.map +1 -0
- package/dist/components/RTCWidget.js +419 -0
- package/dist/components/RTCWidget.js.map +1 -0
- package/dist/components/index.d.ts +3 -0
- package/dist/components/index.d.ts.map +1 -0
- package/dist/components/index.js +5 -0
- package/dist/components/index.js.map +1 -0
- package/dist/components/rtc-widget/components/AudioSettingsPanel.d.ts +10 -0
- package/dist/components/rtc-widget/components/AudioSettingsPanel.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/AudioSettingsPanel.js +316 -0
- package/dist/components/rtc-widget/components/AudioSettingsPanel.js.map +1 -0
- package/dist/components/rtc-widget/components/ConviMessage.d.ts +10 -0
- package/dist/components/rtc-widget/components/ConviMessage.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/ConviMessage.js +14 -0
- package/dist/components/rtc-widget/components/ConviMessage.js.map +1 -0
- package/dist/components/rtc-widget/components/FloatingVideo.d.ts +9 -0
- package/dist/components/rtc-widget/components/FloatingVideo.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/FloatingVideo.js +122 -0
- package/dist/components/rtc-widget/components/FloatingVideo.js.map +1 -0
- package/dist/components/rtc-widget/components/MarkdownRenderer.d.ts +7 -0
- package/dist/components/rtc-widget/components/MarkdownRenderer.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/MarkdownRenderer.js +68 -0
- package/dist/components/rtc-widget/components/MarkdownRenderer.js.map +1 -0
- package/dist/components/rtc-widget/components/MessageBubble.d.ts +10 -0
- package/dist/components/rtc-widget/components/MessageBubble.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/MessageBubble.js +23 -0
- package/dist/components/rtc-widget/components/MessageBubble.js.map +1 -0
- package/dist/components/rtc-widget/components/MessageList.d.ts +11 -0
- package/dist/components/rtc-widget/components/MessageList.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/MessageList.js +89 -0
- package/dist/components/rtc-widget/components/MessageList.js.map +1 -0
- package/dist/components/rtc-widget/components/UserMessage.d.ts +9 -0
- package/dist/components/rtc-widget/components/UserMessage.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/UserMessage.js +15 -0
- package/dist/components/rtc-widget/components/UserMessage.js.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviButton.d.ts +6 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviButton.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviButton.js +15 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviButton.js.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviFooter.d.ts +25 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviFooter.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviFooter.js +172 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviFooter.js.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviHeader.d.ts +17 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviHeader.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviHeader.js +66 -0
- package/dist/components/rtc-widget/components/conviComponents/ConviHeader.js.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/SettingsTray.d.ts +12 -0
- package/dist/components/rtc-widget/components/conviComponents/SettingsTray.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/SettingsTray.js +68 -0
- package/dist/components/rtc-widget/components/conviComponents/SettingsTray.js.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/VoiceModeOverlay.d.ts +8 -0
- package/dist/components/rtc-widget/components/conviComponents/VoiceModeOverlay.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/VoiceModeOverlay.js +199 -0
- package/dist/components/rtc-widget/components/conviComponents/VoiceModeOverlay.js.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/index.d.ts +6 -0
- package/dist/components/rtc-widget/components/conviComponents/index.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/conviComponents/index.js +6 -0
- package/dist/components/rtc-widget/components/conviComponents/index.js.map +1 -0
- package/dist/components/rtc-widget/components/index.d.ts +9 -0
- package/dist/components/rtc-widget/components/index.d.ts.map +1 -0
- package/dist/components/rtc-widget/components/index.js +13 -0
- package/dist/components/rtc-widget/components/index.js.map +1 -0
- package/dist/components/rtc-widget/index.d.ts +6 -0
- package/dist/components/rtc-widget/index.d.ts.map +1 -0
- package/dist/components/rtc-widget/index.js +7 -0
- package/dist/components/rtc-widget/index.js.map +1 -0
- package/dist/components/rtc-widget/styles/framerConfig.d.ts +116 -0
- package/dist/components/rtc-widget/styles/framerConfig.d.ts.map +1 -0
- package/dist/components/rtc-widget/styles/framerConfig.js +73 -0
- package/dist/components/rtc-widget/styles/framerConfig.js.map +1 -0
- package/dist/components/rtc-widget/styles/icons.d.ts +28 -0
- package/dist/components/rtc-widget/styles/icons.d.ts.map +1 -0
- package/dist/components/rtc-widget/styles/icons.js +257 -0
- package/dist/components/rtc-widget/styles/icons.js.map +1 -0
- package/dist/components/rtc-widget/styles/index.d.ts +6 -0
- package/dist/components/rtc-widget/styles/index.d.ts.map +1 -0
- package/dist/components/rtc-widget/styles/index.js +9 -0
- package/dist/components/rtc-widget/styles/index.js.map +1 -0
- package/dist/components/rtc-widget/styles/styledComponents.d.ts +90 -0
- package/dist/components/rtc-widget/styles/styledComponents.d.ts.map +1 -0
- package/dist/components/rtc-widget/styles/styledComponents.js +640 -0
- package/dist/components/rtc-widget/styles/styledComponents.js.map +1 -0
- package/dist/components/rtc-widget/styles/theme.d.ts +188 -0
- package/dist/components/rtc-widget/styles/theme.d.ts.map +1 -0
- package/dist/components/rtc-widget/styles/theme.js +290 -0
- package/dist/components/rtc-widget/styles/theme.js.map +1 -0
- package/dist/components/rtc-widget/types/index.d.ts +60 -0
- package/dist/components/rtc-widget/types/index.d.ts.map +1 -0
- package/dist/components/rtc-widget/types/index.js +2 -0
- package/dist/components/rtc-widget/types/index.js.map +1 -0
- package/dist/hooks/index.d.ts +13 -0
- package/dist/hooks/index.d.ts.map +1 -0
- package/dist/hooks/index.js +14 -0
- package/dist/hooks/index.js.map +1 -0
- package/dist/hooks/useAudioControls.d.ts +41 -0
- package/dist/hooks/useAudioControls.d.ts.map +1 -0
- package/dist/hooks/useAudioControls.js +219 -0
- package/dist/hooks/useAudioControls.js.map +1 -0
- package/dist/hooks/useCharacterInfo.d.ts +17 -0
- package/dist/hooks/useCharacterInfo.d.ts.map +1 -0
- package/dist/hooks/useCharacterInfo.js +61 -0
- package/dist/hooks/useCharacterInfo.js.map +1 -0
- package/dist/hooks/useConvaiClient.d.ts +37 -0
- package/dist/hooks/useConvaiClient.d.ts.map +1 -0
- package/dist/hooks/useConvaiClient.js +348 -0
- package/dist/hooks/useConvaiClient.js.map +1 -0
- package/dist/hooks/useDynamicInfoUpdater.d.ts +33 -0
- package/dist/hooks/useDynamicInfoUpdater.d.ts.map +1 -0
- package/dist/hooks/useDynamicInfoUpdater.js +51 -0
- package/dist/hooks/useDynamicInfoUpdater.js.map +1 -0
- package/dist/hooks/useLocalCameraTrack.d.ts +22 -0
- package/dist/hooks/useLocalCameraTrack.d.ts.map +1 -0
- package/dist/hooks/useLocalCameraTrack.js +34 -0
- package/dist/hooks/useLocalCameraTrack.js.map +1 -0
- package/dist/hooks/useMessageHandler.d.ts +28 -0
- package/dist/hooks/useMessageHandler.d.ts.map +1 -0
- package/dist/hooks/useMessageHandler.js +317 -0
- package/dist/hooks/useMessageHandler.js.map +1 -0
- package/dist/hooks/useScreenShare.d.ts +45 -0
- package/dist/hooks/useScreenShare.d.ts.map +1 -0
- package/dist/hooks/useScreenShare.js +198 -0
- package/dist/hooks/useScreenShare.js.map +1 -0
- package/dist/hooks/useTemplateKeysUpdater.d.ts +35 -0
- package/dist/hooks/useTemplateKeysUpdater.d.ts.map +1 -0
- package/dist/hooks/useTemplateKeysUpdater.js +53 -0
- package/dist/hooks/useTemplateKeysUpdater.js.map +1 -0
- package/dist/hooks/useTriggerMessageSender.d.ts +28 -0
- package/dist/hooks/useTriggerMessageSender.d.ts.map +1 -0
- package/dist/hooks/useTriggerMessageSender.js +52 -0
- package/dist/hooks/useTriggerMessageSender.js.map +1 -0
- package/dist/hooks/useTtsToggle.d.ts +37 -0
- package/dist/hooks/useTtsToggle.d.ts.map +1 -0
- package/dist/hooks/useTtsToggle.js +65 -0
- package/dist/hooks/useTtsToggle.js.map +1 -0
- package/dist/hooks/useUserTextMessageSender.d.ts +28 -0
- package/dist/hooks/useUserTextMessageSender.d.ts.map +1 -0
- package/dist/hooks/useUserTextMessageSender.js +61 -0
- package/dist/hooks/useUserTextMessageSender.js.map +1 -0
- package/dist/hooks/useVideoControls.d.ts +39 -0
- package/dist/hooks/useVideoControls.d.ts.map +1 -0
- package/dist/hooks/useVideoControls.js +211 -0
- package/dist/hooks/useVideoControls.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +9 -0
- package/dist/index.js.map +1 -0
- package/dist/types/index.d.ts +225 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +2 -0
- package/dist/types/index.js.map +1 -0
- package/dist/utils/logger.d.ts +27 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +96 -0
- package/dist/utils/logger.js.map +1 -0
- package/package.json +94 -0
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
import { useState, useCallback, useMemo, useEffect } from "react";
|
|
2
|
+
import { Room, RoomEvent } from "livekit-client";
|
|
3
|
+
import { useUserTextMessageSender } from "./useUserTextMessageSender";
|
|
4
|
+
import { useTriggerMessageSender } from "./useTriggerMessageSender";
|
|
5
|
+
import { useTemplateKeysUpdater } from "./useTemplateKeysUpdater";
|
|
6
|
+
import { useDynamicInfoUpdater } from "./useDynamicInfoUpdater";
|
|
7
|
+
import { useMessageHandler } from "./useMessageHandler";
|
|
8
|
+
import { useAudioControls } from "./useAudioControls";
|
|
9
|
+
import { useVideoControls } from "./useVideoControls";
|
|
10
|
+
import { useScreenShare } from "./useScreenShare";
|
|
11
|
+
import { logger } from "../utils/logger";
|
|
12
|
+
import { useTtsToggle } from "./useTtsToggle";
|
|
13
|
+
/**
|
|
14
|
+
* Main hook for managing Convai client connection and functionality.
|
|
15
|
+
*
|
|
16
|
+
* Provides a complete interface for connecting to Convai's AI-powered voice assistants,
|
|
17
|
+
* managing real-time audio/video conversations, and handling various message types.
|
|
18
|
+
*
|
|
19
|
+
* @returns {ConvaiClient & { activity: string; chatMessages: ChatMessage[] }} Complete client interface with connection state, methods, and message history
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```tsx
|
|
23
|
+
* function App() {
|
|
24
|
+
* const convaiClient = useConvaiClient();
|
|
25
|
+
*
|
|
26
|
+
* const connect = async () => {
|
|
27
|
+
* await convaiClient.connect({
|
|
28
|
+
* apiKey: 'your-api-key',
|
|
29
|
+
* characterId: 'your-character-id',
|
|
30
|
+
* enableVideo: true,
|
|
31
|
+
* enableAudio: true
|
|
32
|
+
* });
|
|
33
|
+
* };
|
|
34
|
+
*
|
|
35
|
+
* return (
|
|
36
|
+
* <div>
|
|
37
|
+
* <button onClick={connect}>Connect</button>
|
|
38
|
+
* <p>Status: {convaiClient.state.isConnected ? 'Connected' : 'Disconnected'}</p>
|
|
39
|
+
* </div>
|
|
40
|
+
* );
|
|
41
|
+
* }
|
|
42
|
+
* ```
|
|
43
|
+
*/
|
|
44
|
+
export const useConvaiClient = () => {
|
|
45
|
+
const [room] = useState(new Room());
|
|
46
|
+
const [isConnected, setIsConnected] = useState(false);
|
|
47
|
+
const [isConnecting, setIsConnecting] = useState(false);
|
|
48
|
+
const [isBotReady, setIsBotReady] = useState(false); // Bot is ready to receive messages
|
|
49
|
+
const [activity, setActivity] = useState("Idle");
|
|
50
|
+
const [chatMessages, setChatMessages] = useState([]);
|
|
51
|
+
const [userTranscription, setUserTranscription] = useState("");
|
|
52
|
+
const [isBotResponding, setIsBotResponding] = useState(false);
|
|
53
|
+
const [participantSid, setParticipantSid] = useState("");
|
|
54
|
+
const [isSpeaking, setIsSpeaking] = useState(false);
|
|
55
|
+
const [characterSessionId, setCharacterSessionId] = useState("-1");
|
|
56
|
+
// Default to Noisy Mode for optimal interruption handling
|
|
57
|
+
const [audioSettings, setAudioSettings] = useState({
|
|
58
|
+
echoCancellation: true,
|
|
59
|
+
noiseSuppression: true,
|
|
60
|
+
autoGainControl: true,
|
|
61
|
+
sampleRate: 48000,
|
|
62
|
+
channelCount: 1,
|
|
63
|
+
});
|
|
64
|
+
// Initialize message sender hooks
|
|
65
|
+
const { sendUserTextMessage } = useUserTextMessageSender(room, participantSid);
|
|
66
|
+
const { sendTriggerMessage } = useTriggerMessageSender(room, participantSid);
|
|
67
|
+
const { updateTemplateKeys } = useTemplateKeysUpdater(room, participantSid);
|
|
68
|
+
const { updateDynamicInfo } = useDynamicInfoUpdater(room, participantSid);
|
|
69
|
+
const { toggleTts } = useTtsToggle(room);
|
|
70
|
+
// Initialize message handler
|
|
71
|
+
const { setupMessageListener } = useMessageHandler(room, setChatMessages, setUserTranscription, setIsBotResponding, isBotResponding, setIsSpeaking, setIsConnected, // Pass callback to update connection state when bot-ready is received
|
|
72
|
+
setActivity, // Pass callback to update activity state when bot-ready is received
|
|
73
|
+
setIsBotReady);
|
|
74
|
+
// Initialize media controls
|
|
75
|
+
const audioControls = useAudioControls(room);
|
|
76
|
+
const videoControls = useVideoControls(room);
|
|
77
|
+
const screenShareControls = useScreenShare(room);
|
|
78
|
+
// Data message listener effect
|
|
79
|
+
useEffect(() => {
|
|
80
|
+
const cleanup = setupMessageListener();
|
|
81
|
+
return cleanup;
|
|
82
|
+
}, [setupMessageListener]);
|
|
83
|
+
// Listen for room disconnection events (PC manager closed, network issues, etc.)
|
|
84
|
+
useEffect(() => {
|
|
85
|
+
if (!room)
|
|
86
|
+
return;
|
|
87
|
+
const handleDisconnected = () => {
|
|
88
|
+
logger.log("Room disconnected (PC manager closed or connection lost)");
|
|
89
|
+
setIsConnected(false);
|
|
90
|
+
setIsBotReady(false); // Reset bot-ready state on disconnect
|
|
91
|
+
setActivity("Disconnected");
|
|
92
|
+
setChatMessages([]);
|
|
93
|
+
setIsBotResponding(false);
|
|
94
|
+
};
|
|
95
|
+
const handleConnectionStateChanged = (state) => {
|
|
96
|
+
logger.log("Room connection state changed:", state);
|
|
97
|
+
if (state === "disconnected") {
|
|
98
|
+
setIsConnected(false);
|
|
99
|
+
setActivity("Disconnected");
|
|
100
|
+
setChatMessages([]);
|
|
101
|
+
setIsBotResponding(false);
|
|
102
|
+
}
|
|
103
|
+
else if (state === "connected") {
|
|
104
|
+
setIsConnected(true);
|
|
105
|
+
setIsBotReady(false); // Room connected but bot not ready yet
|
|
106
|
+
setActivity("Connected");
|
|
107
|
+
}
|
|
108
|
+
else if (state === "connecting") {
|
|
109
|
+
setIsConnecting(true);
|
|
110
|
+
setIsBotReady(false); // Reset bot-ready when reconnecting
|
|
111
|
+
setActivity("Connecting");
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
// Listen for disconnection events
|
|
115
|
+
room.on(RoomEvent.Disconnected, handleDisconnected);
|
|
116
|
+
room.on(RoomEvent.ConnectionStateChanged, handleConnectionStateChanged);
|
|
117
|
+
return () => {
|
|
118
|
+
room.off(RoomEvent.Disconnected, handleDisconnected);
|
|
119
|
+
room.off(RoomEvent.ConnectionStateChanged, handleConnectionStateChanged);
|
|
120
|
+
};
|
|
121
|
+
}, [room]);
|
|
122
|
+
// Create client state
|
|
123
|
+
const state = useMemo(() => ({
|
|
124
|
+
isConnected,
|
|
125
|
+
isConnecting,
|
|
126
|
+
isListening: false,
|
|
127
|
+
isThinking: false,
|
|
128
|
+
isSpeaking,
|
|
129
|
+
agentState: "disconnected",
|
|
130
|
+
}), [isConnected, isConnecting, isSpeaking]);
|
|
131
|
+
// Connect function
|
|
132
|
+
const connect = useCallback(async (config) => {
|
|
133
|
+
if (!config.apiKey ||
|
|
134
|
+
!config.characterId ||
|
|
135
|
+
!config.url) {
|
|
136
|
+
throw new Error("apiKey, characterId, and url are required");
|
|
137
|
+
}
|
|
138
|
+
setIsConnecting(true);
|
|
139
|
+
setActivity("Connecting...");
|
|
140
|
+
try {
|
|
141
|
+
// Prepare request body with required parameters
|
|
142
|
+
const requestBody = {
|
|
143
|
+
character_id: config.characterId,
|
|
144
|
+
transport: "livekit",
|
|
145
|
+
connection_type: "video",
|
|
146
|
+
llm_provider: "dynamic",
|
|
147
|
+
default_tts_enabled: true, // Set TTS enabled by default
|
|
148
|
+
...(characterSessionId &&
|
|
149
|
+
characterSessionId !== "-1" && {
|
|
150
|
+
character_session_id: characterSessionId,
|
|
151
|
+
}), // Resume with existing session if available (skip if -1)
|
|
152
|
+
...(config.actionConfig && { action_config: config.actionConfig }),
|
|
153
|
+
};
|
|
154
|
+
logger.log("Connecting to Convai with config:", {
|
|
155
|
+
url: config.url,
|
|
156
|
+
characterId: config.characterId,
|
|
157
|
+
llmProvider: "dynamic",
|
|
158
|
+
hasActionConfig: !!config.actionConfig,
|
|
159
|
+
characterSessionId: characterSessionId === "-1" ? "new session" : characterSessionId,
|
|
160
|
+
});
|
|
161
|
+
// Call Core Service API
|
|
162
|
+
const response = await fetch(`${config.url}/connect`, {
|
|
163
|
+
method: "POST",
|
|
164
|
+
headers: {
|
|
165
|
+
"x-api-key": config.apiKey,
|
|
166
|
+
"Content-Type": "application/json",
|
|
167
|
+
},
|
|
168
|
+
body: JSON.stringify(requestBody),
|
|
169
|
+
});
|
|
170
|
+
if (!response.ok) {
|
|
171
|
+
const errorText = await response.text();
|
|
172
|
+
logger.error("API Error Response:", {
|
|
173
|
+
status: response.status,
|
|
174
|
+
statusText: response.statusText,
|
|
175
|
+
body: errorText,
|
|
176
|
+
});
|
|
177
|
+
let errorMessage = `HTTP error! status: ${response.status}`;
|
|
178
|
+
try {
|
|
179
|
+
const errorData = JSON.parse(errorText);
|
|
180
|
+
if (errorData.message) {
|
|
181
|
+
errorMessage = errorData.message;
|
|
182
|
+
}
|
|
183
|
+
else if (errorData.error) {
|
|
184
|
+
errorMessage = errorData.error;
|
|
185
|
+
}
|
|
186
|
+
else if (errorData.detail) {
|
|
187
|
+
errorMessage = errorData.detail;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
catch (e) {
|
|
191
|
+
// If we can't parse the error, use the raw text
|
|
192
|
+
if (errorText) {
|
|
193
|
+
errorMessage = `${errorMessage}: ${errorText}`;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
throw new Error(errorMessage);
|
|
197
|
+
}
|
|
198
|
+
const connectionData = await response.json();
|
|
199
|
+
logger.log("Connection data received:", connectionData);
|
|
200
|
+
// Capture character_session_id from connection response
|
|
201
|
+
if (connectionData.character_session_id) {
|
|
202
|
+
setCharacterSessionId(connectionData.character_session_id);
|
|
203
|
+
logger.log("Character Session ID:", connectionData.character_session_id);
|
|
204
|
+
}
|
|
205
|
+
// Update audio settings if provided
|
|
206
|
+
if (config.audioSettings) {
|
|
207
|
+
setAudioSettings(config.audioSettings);
|
|
208
|
+
}
|
|
209
|
+
// Connect to LiveKit room
|
|
210
|
+
await room.connect(connectionData.room_url, connectionData.token, {
|
|
211
|
+
rtcConfig: {
|
|
212
|
+
iceTransportPolicy: "relay",
|
|
213
|
+
},
|
|
214
|
+
});
|
|
215
|
+
// Enable mic/camera based on config
|
|
216
|
+
if (config.enableAudio !== false) {
|
|
217
|
+
const currentSettings = config.audioSettings || audioSettings;
|
|
218
|
+
await room.localParticipant.setMicrophoneEnabled(true, {
|
|
219
|
+
echoCancellation: currentSettings.echoCancellation,
|
|
220
|
+
noiseSuppression: currentSettings.noiseSuppression,
|
|
221
|
+
autoGainControl: currentSettings.autoGainControl,
|
|
222
|
+
sampleRate: currentSettings.sampleRate,
|
|
223
|
+
channelCount: currentSettings.channelCount,
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
if (config.enableVideo !== false) {
|
|
227
|
+
await room.localParticipant.setCameraEnabled(true);
|
|
228
|
+
}
|
|
229
|
+
// Capture participant SID
|
|
230
|
+
setParticipantSid(room.localParticipant.sid);
|
|
231
|
+
setIsConnected(true);
|
|
232
|
+
setActivity("Connected");
|
|
233
|
+
logger.log("Connected to room:", connectionData.room_name);
|
|
234
|
+
logger.log("Session ID:", connectionData.session_id);
|
|
235
|
+
logger.log("Participant SID:", room.localParticipant.sid);
|
|
236
|
+
}
|
|
237
|
+
catch (error) {
|
|
238
|
+
logger.error("Connection failed:", error);
|
|
239
|
+
setIsConnected(false);
|
|
240
|
+
setActivity("Connection failed");
|
|
241
|
+
throw error;
|
|
242
|
+
}
|
|
243
|
+
finally {
|
|
244
|
+
setIsConnecting(false);
|
|
245
|
+
}
|
|
246
|
+
}, [room, characterSessionId]);
|
|
247
|
+
// Disconnect function
|
|
248
|
+
const disconnect = useCallback(async () => {
|
|
249
|
+
if (room && room.state !== "disconnected") {
|
|
250
|
+
try {
|
|
251
|
+
await room.disconnect();
|
|
252
|
+
setIsConnected(false);
|
|
253
|
+
setIsBotReady(false); // Reset bot-ready state on disconnect
|
|
254
|
+
setActivity("Disconnected");
|
|
255
|
+
setChatMessages([]);
|
|
256
|
+
setIsBotResponding(false);
|
|
257
|
+
// Keep characterSessionId for reconnection - only clear on explicit reset
|
|
258
|
+
logger.log("Disconnected from room");
|
|
259
|
+
}
|
|
260
|
+
catch (error) {
|
|
261
|
+
// Handle disconnect errors gracefully
|
|
262
|
+
// ConnectionError "Client initiated disconnect" is expected in some cases
|
|
263
|
+
if (error instanceof Error && error.message.includes("disconnect")) {
|
|
264
|
+
logger.debug("Disconnect completed (expected):", error.message);
|
|
265
|
+
}
|
|
266
|
+
else {
|
|
267
|
+
logger.error("Disconnect error:", error);
|
|
268
|
+
}
|
|
269
|
+
// Still update state even if disconnect throws
|
|
270
|
+
setIsConnected(false);
|
|
271
|
+
setIsBotReady(false); // Reset bot-ready state on disconnect
|
|
272
|
+
setActivity("Disconnected");
|
|
273
|
+
setChatMessages([]);
|
|
274
|
+
setIsBotResponding(false);
|
|
275
|
+
// Keep characterSessionId for reconnection - only clear on explicit reset
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}, [room]);
|
|
279
|
+
// Reset session function - clears character_session_id for a fresh session
|
|
280
|
+
const resetSession = useCallback(() => {
|
|
281
|
+
setCharacterSessionId("-1");
|
|
282
|
+
logger.log("Character session reset - next connection will create a new session");
|
|
283
|
+
}, []);
|
|
284
|
+
// Send RTVI trigger
|
|
285
|
+
const sendRTVI = useCallback((triggerName, message) => {
|
|
286
|
+
if (room && room.localParticipant) {
|
|
287
|
+
const data = {
|
|
288
|
+
type: "rtvi-trigger",
|
|
289
|
+
trigger: triggerName,
|
|
290
|
+
message: message || "",
|
|
291
|
+
};
|
|
292
|
+
room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(data)), {
|
|
293
|
+
reliable: true,
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
}, [room]);
|
|
297
|
+
// Update audio settings dynamically
|
|
298
|
+
const updateAudioSettings = useCallback(async (newSettings) => {
|
|
299
|
+
const updatedSettings = { ...audioSettings, ...newSettings };
|
|
300
|
+
setAudioSettings(updatedSettings);
|
|
301
|
+
// If connected and audio is enabled, restart the microphone with new settings
|
|
302
|
+
if (room && room.state === "connected" && audioControls.isAudioEnabled) {
|
|
303
|
+
try {
|
|
304
|
+
logger.log("Updating audio settings:", updatedSettings);
|
|
305
|
+
// Disable and re-enable microphone with new settings
|
|
306
|
+
await room.localParticipant.setMicrophoneEnabled(false);
|
|
307
|
+
await room.localParticipant.setMicrophoneEnabled(true, {
|
|
308
|
+
echoCancellation: updatedSettings.echoCancellation,
|
|
309
|
+
noiseSuppression: updatedSettings.noiseSuppression,
|
|
310
|
+
autoGainControl: updatedSettings.autoGainControl,
|
|
311
|
+
sampleRate: updatedSettings.sampleRate,
|
|
312
|
+
channelCount: updatedSettings.channelCount,
|
|
313
|
+
});
|
|
314
|
+
logger.log("Audio settings updated successfully");
|
|
315
|
+
}
|
|
316
|
+
catch (error) {
|
|
317
|
+
logger.error("Failed to update audio settings:", error);
|
|
318
|
+
throw error;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}, [room, audioSettings, audioControls.isAudioEnabled]);
|
|
322
|
+
// Return client object with activity and chat messages
|
|
323
|
+
return {
|
|
324
|
+
state,
|
|
325
|
+
connect,
|
|
326
|
+
disconnect,
|
|
327
|
+
resetSession,
|
|
328
|
+
room,
|
|
329
|
+
videoTrack: null,
|
|
330
|
+
audioTrack: null,
|
|
331
|
+
sendUserTextMessage,
|
|
332
|
+
sendTriggerMessage,
|
|
333
|
+
updateTemplateKeys,
|
|
334
|
+
updateDynamicInfo,
|
|
335
|
+
activity,
|
|
336
|
+
chatMessages,
|
|
337
|
+
userTranscription,
|
|
338
|
+
characterSessionId,
|
|
339
|
+
isBotReady,
|
|
340
|
+
audioControls,
|
|
341
|
+
videoControls,
|
|
342
|
+
screenShareControls,
|
|
343
|
+
toggleTts,
|
|
344
|
+
audioSettings,
|
|
345
|
+
updateAudioSettings,
|
|
346
|
+
};
|
|
347
|
+
};
|
|
348
|
+
//# sourceMappingURL=useConvaiClient.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useConvaiClient.js","sourceRoot":"","sources":["../../src/hooks/useConvaiClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAQjD,OAAO,EAAE,wBAAwB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,GAG7B,EAAE;IACF,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACpC,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACtD,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACxD,MAAM,CAAC,UAAU,EAAE,aAAa,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,mCAAmC;IACxF,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAS,MAAM,CAAC,CAAC;IACzD,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,GAAG,QAAQ,CAAgB,EAAE,CAAC,CAAC;IACpE,MAAM,CAAC,iBAAiB,EAAE,oBAAoB,CAAC,GAAG,QAAQ,CAAS,EAAE,CAAC,CAAC;IACvE,MAAM,CAAC,eAAe,EAAE,kBAAkB,CAAC,GAAG,QAAQ,CAAU,KAAK,CAAC,CAAC;IACvE,MAAM,CAAC,cAAc,EAAE,iBAAiB,CAAC,GAAG,QAAQ,CAAS,EAAE,CAAC,CAAC;IACjE,MAAM,CAAC,UAAU,EAAE,aAAa,CAAC,GAAG,QAAQ,CAAU,KAAK,CAAC,CAAC;IAC7D,MAAM,CAAC,kBAAkB,EAAE,qBAAqB,CAAC,GAAG,QAAQ,CAC1D,IAAI,CACL,CAAC;IACF,0DAA0D;IAC1D,MAAM,CAAC,aAAa,EAAE,gBAAgB,CAAC,GAAG,QAAQ,CAAgB;QAChE,gBAAgB,EAAE,IAAI;QACtB,gBAAgB,EAAE,IAAI;QACtB,eAAe,EAAE,IAAI;QACrB,UAAU,EAAE,KAAK;QACjB,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IAEH,kCAAkC;IAClC,MAAM,EAAE,mBAAmB,EAAE,GAAG,wBAAwB,CACtD,IAAI,EACJ,cAAc,CACf,CAAC;IACF,MAAM,EAAE,kBAAkB,EAAE,GAAG,uBAAuB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAC7E,MAAM,EAAE,kBAAkB,EAAE,GAAG,sBAAsB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAC5E,MAAM,EAAE,iBAAiB,EAAE,GAAG,qBAAqB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAC1E,MAAM,EAAE,SAAS,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAEzC,6BAA6B;IAC7B,MAAM,EAAE,oBAAoB,EAAE,GAAG,iBAAiB,CAChD,IAAI,EACJ,eAAe,EACf,oBAAoB,EACpB,kBAAkB,EAClB,eAAe,EACf,aAAa,EACb,cAAc,EAAE,sEAAsE;IACtF,WAAW,EAAE,oEAAoE;IACjF,aAAa,CACd,CAAC;IAEF,4BAA4B;IAC5B,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,mBAAmB,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAEjD,+BAA+B;IAC/B,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,OAAO,GAAG,oBAAoB,EAAE,CAAC;QACvC,OAAO,OAAO,CAAC;IACjB,CAAC,EAAE,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAE3B,iFAAiF;IACjF,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,CAAC,IAAI;YAAE,OAAO;QAElB,MAAM,kBAAkB,GAAG,GAAG,EAAE;YAC9B,MAAM,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;YACvE,cAAc,CAAC,KAAK,CAAC,CAAC;YACtB,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,sCAAsC;YAC5D,WAAW,CAAC,cAAc,CAAC,CAAC;YAC5B,eAAe,CAAC,EAAE,CAAC,CAAC;YACpB,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC,CAAC;QAEF,MAAM,4BAA4B,GAAG,CAAC,KAAU,EAAE,EAAE;YAClD,MAAM,CAAC,GAAG,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YACpD,IAAI,KAAK,KAAK,cAAc,EAAE,CAAC;gBAC7B,cAAc,CAAC,KAAK,CAAC,CAAC;gBACtB,WAAW,CAAC,cAAc,CAAC,CAAC;gBAC5B,eAAe,CAAC,EAAE,CAAC,CAAC;gBACpB,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;iBAAM,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;gBACjC,cAAc,CAAC,IAAI,CAAC,CAAC;gBACrB,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,uCAAuC;gBAC7D,WAAW,CAAC,WAAW,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;gBAClC,eAAe,CAAC,IAAI,CAAC,CAAC;gBACtB,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,oCAAoC;gBAC1D,WAAW,CAAC,YAAY,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC;QAEF,kCAAkC;QAClC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;QACpD,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,sBAAsB,EAAE,4BAA4B,CAAC,CAAC;QAExE,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;YACrD,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,sBAAsB,EAAE,4BAA4B,CAAC,CAAC;QAC3E,CAAC,CAAC;IACJ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAEX,sBAAsB;IACtB,MAAM,KAAK,GAAsB,OAAO,CACtC,GAAG,EAAE,CAAC,CAAC;QACL,WAAW;QACX,YAAY;QACZ,WAAW,EAAE,KAAK;QAClB,UAAU,EAAE,KAAK;QACjB,UAAU;QACV,UAAU,EAAE,cAAuB;KACpC,CAAC,EACF,CAAC,WAAW,EAAE,YAAY,EAAE,UAAU,CAAC,CACxC,CAAC;IAEF,mBAAmB;IACnB,MAAM,OAAO,GAAG,WAAW,CACzB,KAAK,EAAE,MAAoB,EAAE,EAAE;QAC7B,IACE,CAAC,MAAM,CAAC,MAAM;YACd,CAAC,MAAM,CAAC,WAAW;YACnB,CAAC,MAAM,CAAC,GAAG,EACX,CAAC;YACD,MAAM,IAAI,KAAK,CACb,2CAA2C,CAC5C,CAAC;QACJ,CAAC;QAED,eAAe,CAAC,IAAI,CAAC,CAAC;QACtB,WAAW,CAAC,eAAe,CAAC,CAAC;QAE7B,IAAI,CAAC;YACH,gDAAgD;YAChD,MAAM,WAAW,GAAG;gBAClB,YAAY,EAAE,MAAM,CAAC,WAAW;gBAChC,SAAS,EAAE,SAAS;gBACpB,eAAe,EAAE,OAAO;gBACxB,YAAY,EAAE,SAAS;gBACvB,mBAAmB,EAAE,IAAI,EAAE,6BAA6B;gBACxD,GAAG,CAAC,kBAAkB;oBACpB,kBAAkB,KAAK,IAAI,IAAI;oBAC7B,oBAAoB,EAAE,kBAAkB;iBACzC,CAAC,EAAE,yDAAyD;gBAC/D,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;aACnE,CAAC;YAEF,MAAM,CAAC,GAAG,CAAC,mCAAmC,EAAE;gBAC9C,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,WAAW,EAAE,SAAS;gBACtB,eAAe,EAAE,CAAC,CAAC,MAAM,CAAC,YAAY;gBACtC,kBAAkB,EAChB,kBAAkB,KAAK,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,kBAAkB;aACnE,CAAC,CAAC;YAEH,wBAAwB;YACxB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,UAAU,EAAE;gBACpD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM,CAAC,MAAM;oBAC1B,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;aAClC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACxC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;oBAClC,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,IAAI,EAAE,SAAS;iBAChB,CAAC,CAAC;gBAEH,IAAI,YAAY,GAAG,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAC5D,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;oBACxC,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;wBACtB,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC;oBACnC,CAAC;yBAAM,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;wBAC3B,YAAY,GAAG,SAAS,CAAC,KAAK,CAAC;oBACjC,CAAC;yBAAM,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;wBAC5B,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC;oBAClC,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,gDAAgD;oBAChD,IAAI,SAAS,EAAE,CAAC;wBACd,YAAY,GAAG,GAAG,YAAY,KAAK,SAAS,EAAE,CAAC;oBACjD,CAAC;gBACH,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;YAChC,CAAC;YAED,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC7C,MAAM,CAAC,GAAG,CAAC,2BAA2B,EAAE,cAAc,CAAC,CAAC;YAExD,wDAAwD;YACxD,IAAI,cAAc,CAAC,oBAAoB,EAAE,CAAC;gBACxC,qBAAqB,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;gBAC3D,MAAM,CAAC,GAAG,CACR,uBAAuB,EACvB,cAAc,CAAC,oBAAoB,CACpC,CAAC;YACJ,CAAC;YAED,oCAAoC;YACpC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;gBACzB,gBAAgB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACzC,CAAC;YAED,0BAA0B;YAC1B,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,cAAc,CAAC,KAAK,EAAE;gBAChE,SAAS,EAAE;oBACT,kBAAkB,EAAE,OAAO;iBAC5B;aACF,CAAC,CAAC;YAEH,oCAAoC;YACpC,IAAI,MAAM,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;gBACjC,MAAM,eAAe,GAAG,MAAM,CAAC,aAAa,IAAI,aAAa,CAAC;gBAC9D,MAAM,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,IAAI,EAAE;oBACrD,gBAAgB,EAAE,eAAe,CAAC,gBAAgB;oBAClD,gBAAgB,EAAE,eAAe,CAAC,gBAAgB;oBAClD,eAAe,EAAE,eAAe,CAAC,eAAe;oBAChD,UAAU,EAAE,eAAe,CAAC,UAAU;oBACtC,YAAY,EAAE,eAAe,CAAC,YAAY;iBAC3C,CAAC,CAAC;YACL,CAAC;YACD,IAAI,MAAM,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;gBACjC,MAAM,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACrD,CAAC;YAED,0BAA0B;YAC1B,iBAAiB,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC7C,cAAc,CAAC,IAAI,CAAC,CAAC;YACrB,WAAW,CAAC,WAAW,CAAC,CAAC;YAEzB,MAAM,CAAC,GAAG,CAAC,oBAAoB,EAAE,cAAc,CAAC,SAAS,CAAC,CAAC;YAC3D,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;YACrD,MAAM,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;YAC1C,cAAc,CAAC,KAAK,CAAC,CAAC;YACtB,WAAW,CAAC,mBAAmB,CAAC,CAAC;YACjC,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,eAAe,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;IACH,CAAC,EACD,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAC3B,CAAC;IAEF,sBAAsB;IACtB,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;QACxC,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,cAAc,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxB,cAAc,CAAC,KAAK,CAAC,CAAC;gBACtB,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,sCAAsC;gBAC5D,WAAW,CAAC,cAAc,CAAC,CAAC;gBAC5B,eAAe,CAAC,EAAE,CAAC,CAAC;gBACpB,kBAAkB,CAAC,KAAK,CAAC,CAAC;gBAC1B,0EAA0E;gBAC1E,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,sCAAsC;gBACtC,0EAA0E;gBAC1E,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACnE,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;gBAClE,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,KAAK,CAAC,CAAC;gBAC3C,CAAC;gBACD,+CAA+C;gBAC/C,cAAc,CAAC,KAAK,CAAC,CAAC;gBACtB,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,sCAAsC;gBAC5D,WAAW,CAAC,cAAc,CAAC,CAAC;gBAC5B,eAAe,CAAC,EAAE,CAAC,CAAC;gBACpB,kBAAkB,CAAC,KAAK,CAAC,CAAC;gBAC1B,0EAA0E;YAC5E,CAAC;QACH,CAAC;IACH,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAEX,2EAA2E;IAC3E,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE;QACpC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CACR,qEAAqE,CACtE,CAAC;IACJ,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,oBAAoB;IACpB,MAAM,QAAQ,GAAG,WAAW,CAC1B,CAAC,WAAmB,EAAE,OAAgB,EAAE,EAAE;QACxC,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG;gBACX,IAAI,EAAE,cAAc;gBACpB,OAAO,EAAE,WAAW;gBACpB,OAAO,EAAE,OAAO,IAAI,EAAE;aACvB,CAAC;YACF,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAC/B,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAC9C;gBACE,QAAQ,EAAE,IAAI;aACf,CACF,CAAC;QACJ,CAAC;IACH,CAAC,EACD,CAAC,IAAI,CAAC,CACP,CAAC;IAEF,oCAAoC;IACpC,MAAM,mBAAmB,GAAG,WAAW,CACrC,KAAK,EAAE,WAAmC,EAAE,EAAE;QAC5C,MAAM,eAAe,GAAG,EAAE,GAAG,aAAa,EAAE,GAAG,WAAW,EAAE,CAAC;QAC7D,gBAAgB,CAAC,eAAe,CAAC,CAAC;QAElC,8EAA8E;QAC9E,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,IAAI,aAAa,CAAC,cAAc,EAAE,CAAC;YACvE,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;gBAExD,qDAAqD;gBACrD,MAAM,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;gBACxD,MAAM,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,IAAI,EAAE;oBACrD,gBAAgB,EAAE,eAAe,CAAC,gBAAgB;oBAClD,gBAAgB,EAAE,eAAe,CAAC,gBAAgB;oBAClD,eAAe,EAAE,eAAe,CAAC,eAAe;oBAChD,UAAU,EAAE,eAAe,CAAC,UAAU;oBACtC,YAAY,EAAE,eAAe,CAAC,YAAY;iBAC3C,CAAC,CAAC;gBAEH,MAAM,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;gBACxD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC,EACD,CAAC,IAAI,EAAE,aAAa,EAAE,aAAa,CAAC,cAAc,CAAC,CACpD,CAAC;IAEF,uDAAuD;IACvD,OAAO;QACL,KAAK;QACL,OAAO;QACP,UAAU;QACV,YAAY;QACZ,IAAI;QACJ,UAAU,EAAE,IAAI;QAChB,UAAU,EAAE,IAAI;QAChB,mBAAmB;QACnB,kBAAkB;QAClB,kBAAkB;QAClB,iBAAiB;QACjB,QAAQ;QACR,YAAY;QACZ,iBAAiB;QACjB,kBAAkB;QAClB,UAAU;QACV,aAAa;QACb,aAAa;QACb,mBAAmB;QACnB,SAAS;QACT,aAAa;QACb,mBAAmB;KACpB,CAAC;AACJ,CAAC,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { Room } from "livekit-client";
|
|
2
|
+
export interface DynamicInfo {
|
|
3
|
+
text: string;
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Hook for updating dynamic information in Convai.
|
|
7
|
+
*
|
|
8
|
+
* Updates contextual information that can be used by the AI assistant
|
|
9
|
+
* to provide more relevant and up-to-date responses.
|
|
10
|
+
*
|
|
11
|
+
* @param {Room} room - LiveKit room instance
|
|
12
|
+
* @param {string} participantSid - Participant session ID
|
|
13
|
+
* @returns {Object} Object containing updateDynamicInfo function
|
|
14
|
+
*
|
|
15
|
+
* @example
|
|
16
|
+
* ```tsx
|
|
17
|
+
* function DynamicInfoComponent() {
|
|
18
|
+
* const { updateDynamicInfo } = useDynamicInfoUpdater(room, participantSid);
|
|
19
|
+
*
|
|
20
|
+
* const handleUpdate = () => {
|
|
21
|
+
* updateDynamicInfo({
|
|
22
|
+
* text: "The user is now exploring a virtual museum"
|
|
23
|
+
* });
|
|
24
|
+
* };
|
|
25
|
+
*
|
|
26
|
+
* return <button onClick={handleUpdate}>Update Info</button>;
|
|
27
|
+
* }
|
|
28
|
+
* ```
|
|
29
|
+
*/
|
|
30
|
+
export declare const useDynamicInfoUpdater: (room: Room, participantSid: string) => {
|
|
31
|
+
updateDynamicInfo: (dynamicInfo: DynamicInfo) => void;
|
|
32
|
+
};
|
|
33
|
+
//# sourceMappingURL=useDynamicInfoUpdater.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useDynamicInfoUpdater.d.ts","sourceRoot":"","sources":["../../src/hooks/useDynamicInfoUpdater.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAGtC,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,eAAO,MAAM,qBAAqB,GAChC,MAAM,IAAI,EACV,gBAAgB,MAAM;qCAGN,WAAW;CA0B5B,CAAC"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { useCallback } from "react";
|
|
2
|
+
import { logger } from "../utils/logger";
|
|
3
|
+
/**
|
|
4
|
+
* Hook for updating dynamic information in Convai.
|
|
5
|
+
*
|
|
6
|
+
* Updates contextual information that can be used by the AI assistant
|
|
7
|
+
* to provide more relevant and up-to-date responses.
|
|
8
|
+
*
|
|
9
|
+
* @param {Room} room - LiveKit room instance
|
|
10
|
+
* @param {string} participantSid - Participant session ID
|
|
11
|
+
* @returns {Object} Object containing updateDynamicInfo function
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```tsx
|
|
15
|
+
* function DynamicInfoComponent() {
|
|
16
|
+
* const { updateDynamicInfo } = useDynamicInfoUpdater(room, participantSid);
|
|
17
|
+
*
|
|
18
|
+
* const handleUpdate = () => {
|
|
19
|
+
* updateDynamicInfo({
|
|
20
|
+
* text: "The user is now exploring a virtual museum"
|
|
21
|
+
* });
|
|
22
|
+
* };
|
|
23
|
+
*
|
|
24
|
+
* return <button onClick={handleUpdate}>Update Info</button>;
|
|
25
|
+
* }
|
|
26
|
+
* ```
|
|
27
|
+
*/
|
|
28
|
+
export const useDynamicInfoUpdater = (room, participantSid) => {
|
|
29
|
+
const updateDynamicInfo = useCallback((dynamicInfo) => {
|
|
30
|
+
if (room && room.localParticipant && dynamicInfo.text?.trim()) {
|
|
31
|
+
const message = {
|
|
32
|
+
type: "update-dynamic-info",
|
|
33
|
+
data: {
|
|
34
|
+
dynamic_info: {
|
|
35
|
+
text: dynamicInfo.text.trim(),
|
|
36
|
+
},
|
|
37
|
+
participant_sid: participantSid || room.localParticipant.sid,
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
const encodedData = new TextEncoder().encode(JSON.stringify(message));
|
|
41
|
+
room.localParticipant.publishData(encodedData, {
|
|
42
|
+
reliable: true,
|
|
43
|
+
});
|
|
44
|
+
logger.log("🔄 Dynamic info updated:", dynamicInfo.text, "SID:", participantSid || room.localParticipant.sid);
|
|
45
|
+
}
|
|
46
|
+
}, [room, participantSid]);
|
|
47
|
+
return {
|
|
48
|
+
updateDynamicInfo,
|
|
49
|
+
};
|
|
50
|
+
};
|
|
51
|
+
//# sourceMappingURL=useDynamicInfoUpdater.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useDynamicInfoUpdater.js","sourceRoot":"","sources":["../../src/hooks/useDynamicInfoUpdater.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AAEpC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAMzC;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,IAAU,EACV,cAAsB,EACtB,EAAE;IACF,MAAM,iBAAiB,GAAG,WAAW,CACnC,CAAC,WAAwB,EAAE,EAAE;QAC3B,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,IAAI,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;YAC9D,MAAM,OAAO,GAAG;gBACd,IAAI,EAAE,qBAAqB;gBAC3B,IAAI,EAAE;oBACJ,YAAY,EAAE;wBACZ,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE;qBAC9B;oBACD,eAAe,EAAE,cAAc,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG;iBAC7D;aACF,CAAC;YAEF,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;YACtE,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,WAAW,EAAE;gBAC7C,QAAQ,EAAE,IAAI;aACf,CAAC,CAAC;YAEH,MAAM,CAAC,GAAG,CAAC,0BAA0B,EAAE,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChH,CAAC;IACH,CAAC,EACD,CAAC,IAAI,EAAE,cAAc,CAAC,CACvB,CAAC;IAEF,OAAO;QACL,iBAAiB;KAClB,CAAC;AACJ,CAAC,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { TrackReferenceOrPlaceholder } from "@livekit/components-react";
|
|
2
|
+
/**
|
|
3
|
+
* Hook to get the local camera track reference from LiveKit.
|
|
4
|
+
*
|
|
5
|
+
* Returns a track reference that can be used with LiveKit components
|
|
6
|
+
* to display the local participant's camera feed.
|
|
7
|
+
*
|
|
8
|
+
* @returns {TrackReferenceOrPlaceholder} Track reference for the local camera
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```tsx
|
|
12
|
+
* function VideoComponent() {
|
|
13
|
+
* const cameraTrackRef = useLocalCameraTrack();
|
|
14
|
+
*
|
|
15
|
+
* return (
|
|
16
|
+
* <VideoTrack trackRef={cameraTrackRef} />
|
|
17
|
+
* );
|
|
18
|
+
* }
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
export declare const useLocalCameraTrack: () => TrackReferenceOrPlaceholder;
|
|
22
|
+
//# sourceMappingURL=useLocalCameraTrack.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useLocalCameraTrack.d.ts","sourceRoot":"","sources":["../../src/hooks/useLocalCameraTrack.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,2BAA2B,EAE5B,MAAM,2BAA2B,CAAC;AAInC;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,mBAAmB,mCAY/B,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { useLocalParticipant, } from "@livekit/components-react";
|
|
2
|
+
import { Track } from "livekit-client";
|
|
3
|
+
import { useMemo } from "react";
|
|
4
|
+
/**
|
|
5
|
+
* Hook to get the local camera track reference from LiveKit.
|
|
6
|
+
*
|
|
7
|
+
* Returns a track reference that can be used with LiveKit components
|
|
8
|
+
* to display the local participant's camera feed.
|
|
9
|
+
*
|
|
10
|
+
* @returns {TrackReferenceOrPlaceholder} Track reference for the local camera
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```tsx
|
|
14
|
+
* function VideoComponent() {
|
|
15
|
+
* const cameraTrackRef = useLocalCameraTrack();
|
|
16
|
+
*
|
|
17
|
+
* return (
|
|
18
|
+
* <VideoTrack trackRef={cameraTrackRef} />
|
|
19
|
+
* );
|
|
20
|
+
* }
|
|
21
|
+
* ```
|
|
22
|
+
*/
|
|
23
|
+
export const useLocalCameraTrack = () => {
|
|
24
|
+
const { cameraTrack, localParticipant } = useLocalParticipant();
|
|
25
|
+
const cameraTrackRef = useMemo(() => {
|
|
26
|
+
return {
|
|
27
|
+
participant: localParticipant,
|
|
28
|
+
source: Track.Source.Camera,
|
|
29
|
+
publication: cameraTrack,
|
|
30
|
+
};
|
|
31
|
+
}, [localParticipant, cameraTrack]);
|
|
32
|
+
return cameraTrackRef;
|
|
33
|
+
};
|
|
34
|
+
//# sourceMappingURL=useLocalCameraTrack.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useLocalCameraTrack.js","sourceRoot":"","sources":["../../src/hooks/useLocalCameraTrack.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,mBAAmB,GACpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AAEhC;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,EAAE;IACtC,MAAM,EAAE,WAAW,EAAE,gBAAgB,EAAE,GAAG,mBAAmB,EAAE,CAAC;IAEhE,MAAM,cAAc,GAAgC,OAAO,CAAC,GAAG,EAAE;QAC/D,OAAO;YACL,WAAW,EAAE,gBAAgB;YAC7B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;YAC3B,WAAW,EAAE,WAAW;SACzB,CAAC;IACJ,CAAC,EAAE,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC,CAAC;IAEpC,OAAO,cAAc,CAAC;AACxB,CAAC,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { Room } from "livekit-client";
|
|
2
|
+
import { ChatMessage } from "../types";
|
|
3
|
+
/**
|
|
4
|
+
* Hook for handling incoming messages from Convai.
|
|
5
|
+
*
|
|
6
|
+
* Sets up data message listeners to process various message types
|
|
7
|
+
* from the AI assistant and updates the chat message history.
|
|
8
|
+
*
|
|
9
|
+
* @param {Room} room - LiveKit room instance
|
|
10
|
+
* @param {Function} setChatMessages - Function to update chat messages state
|
|
11
|
+
* @returns {Function} Cleanup function to remove message listeners
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```tsx
|
|
15
|
+
* function ChatComponent() {
|
|
16
|
+
* const [chatMessages, setChatMessages] = useState([]);
|
|
17
|
+
* const cleanup = useMessageHandler(room, setChatMessages);
|
|
18
|
+
*
|
|
19
|
+
* useEffect(() => {
|
|
20
|
+
* return cleanup;
|
|
21
|
+
* }, []);
|
|
22
|
+
* }
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export declare const useMessageHandler: (room: Room, setChatMessages: React.Dispatch<React.SetStateAction<ChatMessage[]>>, onUserTranscription?: (text: string) => void, setIsBotResponding?: (responding: boolean) => void, isBotResponding?: boolean, setIsSpeaking?: (speaking: boolean) => void, onConnectionStateChange?: (isConnected: boolean) => void, setActivity?: (activity: string) => void, setIsBotReady?: (isReady: boolean) => void) => {
|
|
26
|
+
setupMessageListener: () => (() => void) | undefined;
|
|
27
|
+
};
|
|
28
|
+
//# sourceMappingURL=useMessageHandler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useMessageHandler.d.ts","sourceRoot":"","sources":["../../src/hooks/useMessageHandler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAA8B,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAGvC;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,iBAAiB,GAC5B,MAAM,IAAI,EACV,iBAAiB,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,EACpE,sBAAsB,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,EAC5C,qBAAqB,CAAC,UAAU,EAAE,OAAO,KAAK,IAAI,EAClD,kBAAkB,OAAO,EACzB,gBAAgB,CAAC,QAAQ,EAAE,OAAO,KAAK,IAAI,EAC3C,0BAA0B,CAAC,WAAW,EAAE,OAAO,KAAK,IAAI,EACxD,cAAc,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACxC,gBAAgB,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI;;CAya3C,CAAC"}
|