@bytexbyte/nxtlinq-ai-agent-ui-react-development 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/NxtlinqAgentChat.d.ts +26 -0
- package/dist/NxtlinqAgentChat.d.ts.map +1 -0
- package/dist/NxtlinqAgentChat.js +28 -0
- package/dist/components/AgentAssistantShell.d.ts +5 -0
- package/dist/components/AgentAssistantShell.d.ts.map +1 -0
- package/dist/components/AgentAssistantShell.js +52 -0
- package/dist/components/AgentComposer.d.ts +3 -0
- package/dist/components/AgentComposer.d.ts.map +1 -0
- package/dist/components/AgentComposer.js +60 -0
- package/dist/components/AgentMessageList.d.ts +3 -0
- package/dist/components/AgentMessageList.d.ts.map +1 -0
- package/dist/components/AgentMessageList.js +37 -0
- package/dist/components/AgentRemoteAudio.d.ts +4 -0
- package/dist/components/AgentRemoteAudio.d.ts.map +1 -0
- package/dist/components/AgentRemoteAudio.js +34 -0
- package/dist/components/AgentVoiceBar.d.ts +3 -0
- package/dist/components/AgentVoiceBar.d.ts.map +1 -0
- package/dist/components/AgentVoiceBar.js +91 -0
- package/dist/components/PresetMessageChips.d.ts +3 -0
- package/dist/components/PresetMessageChips.d.ts.map +1 -0
- package/dist/components/PresetMessageChips.js +23 -0
- package/dist/context/AgentAssistantContext.d.ts +32 -0
- package/dist/context/AgentAssistantContext.d.ts.map +1 -0
- package/dist/context/AgentAssistantContext.js +159 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/legacy/assets/images/adiSideItalicDataUri.d.ts +2 -0
- package/dist/legacy/assets/images/adiSideItalicDataUri.d.ts.map +1 -0
- package/dist/legacy/assets/images/adiSideItalicDataUri.js +1 -0
- package/dist/legacy/chatbot/ChatBot.d.ts +5 -0
- package/dist/legacy/chatbot/ChatBot.d.ts.map +1 -0
- package/dist/legacy/chatbot/ChatBot.js +35 -0
- package/dist/legacy/chatbot/context/ChatBotContext.d.ts +5 -0
- package/dist/legacy/chatbot/context/ChatBotContext.d.ts.map +1 -0
- package/dist/legacy/chatbot/context/ChatBotContext.js +2908 -0
- package/dist/legacy/chatbot/types/ChatBotTypes.d.ts +166 -0
- package/dist/legacy/chatbot/types/ChatBotTypes.d.ts.map +1 -0
- package/dist/legacy/chatbot/types/ChatBotTypes.js +1 -0
- package/dist/legacy/chatbot/ui/BerifyMeModal.d.ts +17 -0
- package/dist/legacy/chatbot/ui/BerifyMeModal.d.ts.map +1 -0
- package/dist/legacy/chatbot/ui/BerifyMeModal.js +110 -0
- package/dist/legacy/chatbot/ui/ChatBotUI.d.ts +3 -0
- package/dist/legacy/chatbot/ui/ChatBotUI.d.ts.map +1 -0
- package/dist/legacy/chatbot/ui/ChatBotUI.js +625 -0
- package/dist/legacy/chatbot/ui/MessageInput.d.ts +3 -0
- package/dist/legacy/chatbot/ui/MessageInput.d.ts.map +1 -0
- package/dist/legacy/chatbot/ui/MessageInput.js +321 -0
- package/dist/legacy/chatbot/ui/MessageList.d.ts +4 -0
- package/dist/legacy/chatbot/ui/MessageList.d.ts.map +1 -0
- package/dist/legacy/chatbot/ui/MessageList.js +455 -0
- package/dist/legacy/chatbot/ui/ModelSelector.d.ts +4 -0
- package/dist/legacy/chatbot/ui/ModelSelector.d.ts.map +1 -0
- package/dist/legacy/chatbot/ui/ModelSelector.js +122 -0
- package/dist/legacy/chatbot/ui/NotificationModal.d.ts +15 -0
- package/dist/legacy/chatbot/ui/NotificationModal.d.ts.map +1 -0
- package/dist/legacy/chatbot/ui/NotificationModal.js +53 -0
- package/dist/legacy/chatbot/ui/PermissionForm.d.ts +8 -0
- package/dist/legacy/chatbot/ui/PermissionForm.d.ts.map +1 -0
- package/dist/legacy/chatbot/ui/PermissionForm.js +465 -0
- package/dist/legacy/chatbot/ui/PresetMessages.d.ts +4 -0
- package/dist/legacy/chatbot/ui/PresetMessages.d.ts.map +1 -0
- package/dist/legacy/chatbot/ui/PresetMessages.js +33 -0
- package/dist/legacy/chatbot/ui/VoiceModePanel.d.ts +3 -0
- package/dist/legacy/chatbot/ui/VoiceModePanel.d.ts.map +1 -0
- package/dist/legacy/chatbot/ui/VoiceModePanel.js +95 -0
- package/dist/legacy/chatbot/ui/styles/isolatedStyles.d.ts +73 -0
- package/dist/legacy/chatbot/ui/styles/isolatedStyles.d.ts.map +1 -0
- package/dist/legacy/chatbot/ui/styles/isolatedStyles.js +985 -0
- package/dist/legacy/index.d.ts +14 -0
- package/dist/legacy/index.d.ts.map +1 -0
- package/dist/legacy/index.js +12 -0
- package/dist/theme/defaultTheme.d.ts +3 -0
- package/dist/theme/defaultTheme.d.ts.map +1 -0
- package/dist/theme/defaultTheme.js +20 -0
- package/dist/types.d.ts +62 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +1 -0
- package/dist/voice/useVoiceConnectOrchestration.d.ts +21 -0
- package/dist/voice/useVoiceConnectOrchestration.d.ts.map +1 -0
- package/dist/voice/useVoiceConnectOrchestration.js +86 -0
- package/dist/voice/useVoiceMicState.d.ts +15 -0
- package/dist/voice/useVoiceMicState.d.ts.map +1 -0
- package/dist/voice/useVoiceMicState.js +94 -0
- package/dist/voice/useVoiceSilenceCommit.d.ts +10 -0
- package/dist/voice/useVoiceSilenceCommit.d.ts.map +1 -0
- package/dist/voice/useVoiceSilenceCommit.js +67 -0
- package/dist/voice/useVoiceTranscriptMessages.d.ts +16 -0
- package/dist/voice/useVoiceTranscriptMessages.d.ts.map +1 -0
- package/dist/voice/useVoiceTranscriptMessages.js +129 -0
- package/dist/voice/useWsRealtimeAudio.d.ts +18 -0
- package/dist/voice/useWsRealtimeAudio.d.ts.map +1 -0
- package/dist/voice/useWsRealtimeAudio.js +102 -0
- package/dist/voice/voiceMicConstants.d.ts +4 -0
- package/dist/voice/voiceMicConstants.d.ts.map +1 -0
- package/dist/voice/voiceMicConstants.js +10 -0
- package/dist/voice/ws/BrowserWsPcmPlayer.d.ts +23 -0
- package/dist/voice/ws/BrowserWsPcmPlayer.d.ts.map +1 -0
- package/dist/voice/ws/BrowserWsPcmPlayer.js +137 -0
- package/dist/voice/ws/BrowserWsPcmRecorder.d.ts +17 -0
- package/dist/voice/ws/BrowserWsPcmRecorder.d.ts.map +1 -0
- package/dist/voice/ws/BrowserWsPcmRecorder.js +71 -0
- package/dist/voice/ws/float32ToPcm16.d.ts +2 -0
- package/dist/voice/ws/float32ToPcm16.d.ts.map +1 -0
- package/dist/voice/ws/float32ToPcm16.js +8 -0
- package/dist/voice/ws/voiceSilenceConstants.d.ts +5 -0
- package/dist/voice/ws/voiceSilenceConstants.d.ts.map +1 -0
- package/dist/voice/ws/voiceSilenceConstants.js +4 -0
- package/dist/voice/ws/wsRealtimeConstants.d.ts +2 -0
- package/dist/voice/ws/wsRealtimeConstants.d.ts.map +1 -0
- package/dist/voice/ws/wsRealtimeConstants.js +1 -0
- package/package.json +60 -0
- package/src/NxtlinqAgentChat.tsx +79 -0
- package/src/components/AgentAssistantShell.tsx +104 -0
- package/src/components/AgentComposer.tsx +134 -0
- package/src/components/AgentMessageList.tsx +78 -0
- package/src/components/AgentRemoteAudio.tsx +34 -0
- package/src/components/AgentVoiceBar.tsx +173 -0
- package/src/components/PresetMessageChips.tsx +41 -0
- package/src/context/AgentAssistantContext.tsx +276 -0
- package/src/index.ts +78 -0
- package/src/legacy/assets/images/adiSideItalicDataUri.ts +1 -0
- package/src/legacy/chatbot/ChatBot.tsx +61 -0
- package/src/legacy/chatbot/context/ChatBotContext.tsx +3227 -0
- package/src/legacy/chatbot/types/ChatBotTypes.ts +195 -0
- package/src/legacy/chatbot/ui/BerifyMeModal.tsx +145 -0
- package/src/legacy/chatbot/ui/ChatBotUI.tsx +949 -0
- package/src/legacy/chatbot/ui/MessageInput.tsx +517 -0
- package/src/legacy/chatbot/ui/MessageList.tsx +764 -0
- package/src/legacy/chatbot/ui/ModelSelector.tsx +190 -0
- package/src/legacy/chatbot/ui/NotificationModal.tsx +110 -0
- package/src/legacy/chatbot/ui/PermissionForm.tsx +632 -0
- package/src/legacy/chatbot/ui/PresetMessages.tsx +50 -0
- package/src/legacy/chatbot/ui/VoiceModePanel.tsx +168 -0
- package/src/legacy/chatbot/ui/styles/isolatedStyles.ts +1058 -0
- package/src/legacy/index.ts +26 -0
- package/src/theme/defaultTheme.ts +22 -0
- package/src/types.ts +65 -0
- package/src/voice/useVoiceConnectOrchestration.ts +117 -0
- package/src/voice/useVoiceMicState.ts +117 -0
- package/src/voice/useVoiceTranscriptMessages.ts +173 -0
- package/src/voice/voiceMicConstants.ts +13 -0
|
@@ -0,0 +1,3227 @@
|
|
|
1
|
+
import { ethers } from 'ethers';
|
|
2
|
+
import stringify from 'fast-json-stable-stringify';
|
|
3
|
+
import * as React from 'react';
|
|
4
|
+
import { flushSync } from 'react-dom';
|
|
5
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
6
|
+
import {
|
|
7
|
+
createNxtlinqApi,
|
|
8
|
+
setApiHosts,
|
|
9
|
+
synthesizeSpeechToBuffer,
|
|
10
|
+
useLocalStorage,
|
|
11
|
+
useSessionStorage,
|
|
12
|
+
useSpeechToTextFromMic,
|
|
13
|
+
useVoiceMode,
|
|
14
|
+
metakeepClient,
|
|
15
|
+
getEthers,
|
|
16
|
+
sleep,
|
|
17
|
+
walletTextUtils,
|
|
18
|
+
} from '@bytexbyte/nxtlinq-ai-agent-web-development';
|
|
19
|
+
import type {
|
|
20
|
+
AgentResponse,
|
|
21
|
+
AIT,
|
|
22
|
+
Attachment,
|
|
23
|
+
ClientTtsVoiceSettings,
|
|
24
|
+
Message,
|
|
25
|
+
ServicePermission,
|
|
26
|
+
} from '@bytexbyte/nxtlinq-ai-agent-core-development';
|
|
27
|
+
import {
|
|
28
|
+
AIModel,
|
|
29
|
+
ChatBotContextType,
|
|
30
|
+
ChatBotProps,
|
|
31
|
+
PresetMessage
|
|
32
|
+
} from '../types/ChatBotTypes';
|
|
33
|
+
|
|
34
|
+
const MIC_ENABLED_SESSION_KEY = 'chatbot-mic-enabled';
|
|
35
|
+
|
|
36
|
+
const ChatBotContext = React.createContext<ChatBotContextType | undefined>(undefined);
|
|
37
|
+
|
|
38
|
+
export const useChatBot = () => {
|
|
39
|
+
const context = React.useContext(ChatBotContext);
|
|
40
|
+
if (!context) {
|
|
41
|
+
throw new Error('useChatBot must be used within a ChatBotProvider');
|
|
42
|
+
}
|
|
43
|
+
return context;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
export const ChatBotProvider: React.FC<ChatBotProps> = ({
|
|
47
|
+
onMessage,
|
|
48
|
+
onError,
|
|
49
|
+
onToolUse,
|
|
50
|
+
presetMessages = [],
|
|
51
|
+
placeholder = 'Type a message...',
|
|
52
|
+
className = '',
|
|
53
|
+
maxRetries = 3,
|
|
54
|
+
retryDelay = 2000,
|
|
55
|
+
serviceId,
|
|
56
|
+
apiKey,
|
|
57
|
+
apiSecret,
|
|
58
|
+
environment = 'production',
|
|
59
|
+
onVerifyWallet,
|
|
60
|
+
permissionGroup,
|
|
61
|
+
children,
|
|
62
|
+
// AI Model related attributes
|
|
63
|
+
onModelChange,
|
|
64
|
+
// Storage mode configuration
|
|
65
|
+
storageMode = "local-storage" as "session-storage" | "local-storage",
|
|
66
|
+
// Wallet verification configuration
|
|
67
|
+
requireWalletIDVVerification = true,
|
|
68
|
+
isSemiAutomaticMode = false,
|
|
69
|
+
// Custom user identity information
|
|
70
|
+
customUserInfo,
|
|
71
|
+
// Custom username for wallet verification
|
|
72
|
+
customUsername,
|
|
73
|
+
// IDV suggestion banner configuration
|
|
74
|
+
idvBannerDismissSeconds = 86400,
|
|
75
|
+
// 24 hours in seconds
|
|
76
|
+
isStopRecordingOnSend = false,
|
|
77
|
+
// Custom error message to display in chat
|
|
78
|
+
customError,
|
|
79
|
+
// PII display mode
|
|
80
|
+
piiDisplayMode = 'redacted',
|
|
81
|
+
}) => {
|
|
82
|
+
// Set API hosts immediately based on environment (before any API calls)
|
|
83
|
+
setApiHosts(environment);
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
const nxtlinqApi = React.useMemo(() => createNxtlinqApi(apiKey, apiSecret), [apiKey, apiSecret]);
|
|
87
|
+
|
|
88
|
+
const apiKeyRef = React.useRef(apiKey);
|
|
89
|
+
const apiSecretRef = React.useRef(apiSecret);
|
|
90
|
+
const clientTtsVoiceRef = React.useRef<ClientTtsVoiceSettings | undefined>(undefined);
|
|
91
|
+
React.useEffect(() => {
|
|
92
|
+
apiKeyRef.current = apiKey;
|
|
93
|
+
apiSecretRef.current = apiSecret;
|
|
94
|
+
}, [apiKey, apiSecret]);
|
|
95
|
+
|
|
96
|
+
// Custom hook
|
|
97
|
+
const { isMicEnabled, transcript, partialTranscript, start: startRecording, stop: stopRecording, clear: clearRecording } = useSpeechToTextFromMic({
|
|
98
|
+
apiKey,
|
|
99
|
+
apiSecret
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
// Messages: Use storage based on storageMode to support cross-tab sync (local-storage) or session isolation (session-storage)
|
|
103
|
+
const [messages, setMessages] = storageMode === "local-storage"
|
|
104
|
+
? useLocalStorage<Message[]>('chatbot-messages', [])
|
|
105
|
+
: useSessionStorage<Message[]>('chatbot-messages', []);
|
|
106
|
+
const [isOpen, setIsOpen] = storageMode === "session-storage"
|
|
107
|
+
? useSessionStorage<boolean>('chatbot-is-open', false)
|
|
108
|
+
: React.useState<boolean>(false);
|
|
109
|
+
|
|
110
|
+
const [inputValue, setInputValue] = React.useState('');
|
|
111
|
+
const [isLoading, setIsLoading] = React.useState(false);
|
|
112
|
+
const [hitAddress, setHitAddress] = React.useState<string | null>(null);
|
|
113
|
+
const [ait, setAit] = React.useState<AIT | null>(null);
|
|
114
|
+
const [permissions, setPermissions] = React.useState<string[]>([]);
|
|
115
|
+
const [availablePermissions, setAvailablePermissions] = React.useState<ServicePermission[]>([]);
|
|
116
|
+
const [showPermissionForm, setShowPermissionForm] = React.useState(false);
|
|
117
|
+
const [isPermissionFormOpen, setIsPermissionFormOpen] = React.useState(false);
|
|
118
|
+
const [isAITLoading, setIsAITLoading] = React.useState(false);
|
|
119
|
+
const [isDisabled, setIsDisabled] = React.useState(true);
|
|
120
|
+
const [signer, setSigner] = React.useState<ethers.JsonRpcSigner | null>(null);
|
|
121
|
+
const [walletInfo, setWalletInfo] = React.useState<any>(null);
|
|
122
|
+
const [isWalletLoading, setIsWalletLoading] = React.useState(false);
|
|
123
|
+
const [isAutoConnecting, setIsAutoConnecting] = React.useState(false);
|
|
124
|
+
|
|
125
|
+
// Persistent data (always use localStorage)
|
|
126
|
+
const [nxtlinqAITServiceAccessToken, setNxtlinqAITServiceAccessToken] = useLocalStorage<string>('nxtlinqAITServiceAccessToken', '');
|
|
127
|
+
const [pseudoId, setPseudoId] = useLocalStorage<string>('pseudoId', uuidv4());
|
|
128
|
+
|
|
129
|
+
// Suggestions: Use storage based on storageMode to support cross-tab sync (local-storage) or session isolation (session-storage)
|
|
130
|
+
const [suggestions, setSuggestions] = storageMode === "local-storage"
|
|
131
|
+
? useLocalStorage<PresetMessage[]>('chatbot-suggestions', presetMessages)
|
|
132
|
+
: useSessionStorage<PresetMessage[]>('chatbot-suggestions', presetMessages);
|
|
133
|
+
const [isAITEnabling, setIsAITEnabling] = React.useState(false);
|
|
134
|
+
const [isAwaitingMicGesture, setIsAwaitingMicGesture] = React.useState(false);
|
|
135
|
+
// autoSendEnabled: Always use localStorage (do not sync across tabs to avoid conflicts with speech-to-text)
|
|
136
|
+
const [autoSendEnabled, setAutoSendEnabled] = useLocalStorage<boolean>('chatbot-auto-send-enabled', true);
|
|
137
|
+
|
|
138
|
+
// Speech related state
|
|
139
|
+
const [textToSpeechEnabled, setTextToSpeechEnabled] = useLocalStorage<boolean>('chatbot-text-to-speech-enabled', false);
|
|
140
|
+
const [preferredChatMode, setPreferredChatMode] = useLocalStorage<'text' | 'voice'>('chatbot-active-mode', 'text');
|
|
141
|
+
const [speechingIndex, setSpeechingIndex] = React.useState<number | undefined>(undefined);
|
|
142
|
+
|
|
143
|
+
// Speech related refs
|
|
144
|
+
const audioCtxRef = React.useRef<AudioContext | null>(null);
|
|
145
|
+
const audioSourceRef = React.useRef<AudioBufferSourceNode | null>(null);
|
|
146
|
+
const audioElementRef = React.useRef<HTMLAudioElement | null>(null);
|
|
147
|
+
const speechingRef = React.useRef(false);
|
|
148
|
+
const [isTtsProcessing, setIsTtsProcessing] = React.useState(false);
|
|
149
|
+
const [requiresGesture, setRequiresGesture] = React.useState(false);
|
|
150
|
+
const pendingTtsRef = React.useRef<string | null>(null);
|
|
151
|
+
/** Azure Speech STT:每次麥克風成功開啟時記錄,供寫入後端 usage log(開麥→送出) */
|
|
152
|
+
const messagesRef = React.useRef(messages);
|
|
153
|
+
messagesRef.current = messages;
|
|
154
|
+
const micSessionStartForLogRef = React.useRef<number | null>(null);
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
// Clean up stale PII scanning state on mount (e.g. page refresh during active SSE)
|
|
158
|
+
React.useEffect(() => {
|
|
159
|
+
setMessages(prev => {
|
|
160
|
+
const hasStale = prev.some(m => m.piiStatus === 'scanning');
|
|
161
|
+
if (!hasStale) return prev;
|
|
162
|
+
return prev.map(m =>
|
|
163
|
+
m.piiStatus === 'scanning'
|
|
164
|
+
? { ...m, piiStatus: undefined, piiStep: undefined }
|
|
165
|
+
: m
|
|
166
|
+
);
|
|
167
|
+
});
|
|
168
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
169
|
+
}, []);
|
|
170
|
+
|
|
171
|
+
// Use refs to get latest state values in hasPermission function
|
|
172
|
+
const hitAddressRef = React.useRef(hitAddress);
|
|
173
|
+
const aitRef = React.useRef(ait);
|
|
174
|
+
const permissionsRef = React.useRef(permissions);
|
|
175
|
+
const nxtlinqAITServiceAccessTokenRef = React.useRef(nxtlinqAITServiceAccessToken);
|
|
176
|
+
const signerRef = React.useRef(signer);
|
|
177
|
+
|
|
178
|
+
// Helper function to combine customUsername with Adilas customUserInfo
|
|
179
|
+
const getFinalCustomUsername = React.useCallback((username: string | undefined): string | undefined => {
|
|
180
|
+
if (!username) return username;
|
|
181
|
+
|
|
182
|
+
// For Adilas: combine customUsername, corpId and serverDomain
|
|
183
|
+
if (walletTextUtils.isAdilasService(serviceId) && customUserInfo) {
|
|
184
|
+
const { corpId, serverDomain } = customUserInfo;
|
|
185
|
+
if (corpId && serverDomain) {
|
|
186
|
+
// Format: customUsername|(corpId)|[serverDomain]
|
|
187
|
+
const parts = [username];
|
|
188
|
+
if (corpId) parts.push(`(${corpId})`);
|
|
189
|
+
if (serverDomain) parts.push(`[${serverDomain}]`);
|
|
190
|
+
return parts.join('|');
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return username;
|
|
195
|
+
}, [serviceId, customUserInfo]);
|
|
196
|
+
|
|
197
|
+
// Refs for input value and recording state
|
|
198
|
+
const isMicEnabledRef = React.useRef(false);
|
|
199
|
+
const isRestoringRecordingRef = React.useRef(false);
|
|
200
|
+
const hasSyncedMicStateRef = React.useRef(false);
|
|
201
|
+
const isReacquiringMicRef = React.useRef(false);
|
|
202
|
+
const pendingMicAutoStartRef = React.useRef(false);
|
|
203
|
+
const autoStartGestureCleanupRef = React.useRef<(() => void) | null>(null);
|
|
204
|
+
const textInputRef = React.useRef<HTMLInputElement | null>(null);
|
|
205
|
+
const lastPartialRangeRef = React.useRef<{ start: number; end: number } | null>(null);
|
|
206
|
+
const lastAutoSentTranscriptRef = React.useRef<string>('');
|
|
207
|
+
const autoSendTimerRef = React.useRef<NodeJS.Timeout | null>(null);
|
|
208
|
+
const lastCustomErrorRef = React.useRef<string | undefined>(undefined);
|
|
209
|
+
|
|
210
|
+
function insertPartial(input: string, partial: string, caret: number) {
|
|
211
|
+
let start = caret;
|
|
212
|
+
let end = caret;
|
|
213
|
+
|
|
214
|
+
if (lastPartialRangeRef.current) {
|
|
215
|
+
start = lastPartialRangeRef.current.start;
|
|
216
|
+
end = lastPartialRangeRef.current.end;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const before = input.slice(0, start);
|
|
220
|
+
const after = input.slice(end);
|
|
221
|
+
|
|
222
|
+
const next = before + partial + after;
|
|
223
|
+
const newCaret = start + partial.length;
|
|
224
|
+
|
|
225
|
+
lastPartialRangeRef.current = { start, end: newCaret };
|
|
226
|
+
return { next, caret: newCaret };
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function finalizePartial(input: string, finalText: string) {
|
|
230
|
+
if (!lastPartialRangeRef.current) {
|
|
231
|
+
return { next: input, caret: input.length };
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const { start, end } = lastPartialRangeRef.current;
|
|
235
|
+
const before = input.slice(0, start);
|
|
236
|
+
const after = input.slice(end);
|
|
237
|
+
|
|
238
|
+
const next = before + finalText + after;
|
|
239
|
+
const caret = start + finalText.length;
|
|
240
|
+
|
|
241
|
+
lastPartialRangeRef.current = null;
|
|
242
|
+
return { next, caret };
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function normalizeTranscript(text: string): string {
|
|
246
|
+
return text
|
|
247
|
+
.toLowerCase()
|
|
248
|
+
// Remove period only when NOT between digits (preserve decimals like 130.00)
|
|
249
|
+
.replace(/(?<!\d)\.(?!\d)/g, '')
|
|
250
|
+
.replace(/[,!?;:]/g, '')
|
|
251
|
+
.replace(/\s+/g, ' ')
|
|
252
|
+
.trim();
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// Simple token cleanup function
|
|
256
|
+
const clearExpiredToken = React.useCallback(() => {
|
|
257
|
+
try {
|
|
258
|
+
localStorage.removeItem('nxtlinqAITServiceAccessToken');
|
|
259
|
+
setNxtlinqAITServiceAccessToken('');
|
|
260
|
+
} catch (error) {
|
|
261
|
+
console.error('Error clearing expired token:', error);
|
|
262
|
+
}
|
|
263
|
+
}, []);
|
|
264
|
+
|
|
265
|
+
// Internal: Play text-to-speech with retry mechanism
|
|
266
|
+
const playTextToSpeechWithRetry = React.useCallback(async (text: string, messageIndex?: number, attempt = 0): Promise<void> => {
|
|
267
|
+
if (attempt > 2) {
|
|
268
|
+
console.error('TTS playback failed after retries');
|
|
269
|
+
setIsTtsProcessing(false);
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
const synth = await synthesizeSpeechToBuffer({
|
|
273
|
+
text,
|
|
274
|
+
apiKey: apiKeyRef.current,
|
|
275
|
+
apiSecret: apiSecretRef.current,
|
|
276
|
+
settings: clientTtsVoiceRef.current,
|
|
277
|
+
});
|
|
278
|
+
if (!synth) throw new Error('Failed to get audio buffer');
|
|
279
|
+
|
|
280
|
+
const bufferData = synth.buffer;
|
|
281
|
+
const audioMimeType = synth.mimeType;
|
|
282
|
+
|
|
283
|
+
const isMobileDevice = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent);
|
|
284
|
+
if (isMobileDevice) {
|
|
285
|
+
const blob = new Blob([bufferData], { type: audioMimeType });
|
|
286
|
+
const audioUrl = URL.createObjectURL(blob);
|
|
287
|
+
const audio = new Audio(audioUrl);
|
|
288
|
+
if ('setSinkId' in HTMLMediaElement.prototype) {
|
|
289
|
+
try {
|
|
290
|
+
const devices = await (navigator.mediaDevices as any)?.enumerateDevices?.();
|
|
291
|
+
const speakerDevice = devices?.find((d: any) => d.kind === 'audiooutput' && d.label.toLowerCase().includes('speaker'));
|
|
292
|
+
if (speakerDevice) await (audio as any).setSinkId(speakerDevice.deviceId);
|
|
293
|
+
else await (audio as any).setSinkId('default');
|
|
294
|
+
} catch (err) {
|
|
295
|
+
console.debug('Could not set audio sink:', err);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
audio.volume = 1.0;
|
|
299
|
+
audioElementRef.current = audio;
|
|
300
|
+
speechingRef.current = true;
|
|
301
|
+
if (messageIndex !== undefined) setSpeechingIndex(messageIndex);
|
|
302
|
+
|
|
303
|
+
// Track if onplay has been called to avoid clearing state multiple times
|
|
304
|
+
let playStarted = false;
|
|
305
|
+
const clearProcessingState = () => {
|
|
306
|
+
if (!playStarted) {
|
|
307
|
+
playStarted = true;
|
|
308
|
+
setIsTtsProcessing(false);
|
|
309
|
+
}
|
|
310
|
+
};
|
|
311
|
+
|
|
312
|
+
await new Promise<void>(async (resolve, reject) => {
|
|
313
|
+
audio.onended = () => {
|
|
314
|
+
speechingRef.current = false;
|
|
315
|
+
setSpeechingIndex(undefined);
|
|
316
|
+
URL.revokeObjectURL(audioUrl);
|
|
317
|
+
audioElementRef.current = null;
|
|
318
|
+
resolve();
|
|
319
|
+
};
|
|
320
|
+
audio.onerror = (error) => {
|
|
321
|
+
speechingRef.current = false;
|
|
322
|
+
setSpeechingIndex(undefined);
|
|
323
|
+
URL.revokeObjectURL(audioUrl);
|
|
324
|
+
audioElementRef.current = null;
|
|
325
|
+
clearProcessingState();
|
|
326
|
+
reject(error as any);
|
|
327
|
+
};
|
|
328
|
+
audio.onplay = () => {
|
|
329
|
+
// Clear processing state when audio actually starts playing
|
|
330
|
+
setRequiresGesture(false);
|
|
331
|
+
clearProcessingState();
|
|
332
|
+
};
|
|
333
|
+
|
|
334
|
+
// Handle play() promise - some mobile browsers require user interaction
|
|
335
|
+
audio.play().then(() => {
|
|
336
|
+
// Play promise resolved, but wait for onplay event to confirm actual playback
|
|
337
|
+
// Don't clear requiresGesture here - let onplay handle it
|
|
338
|
+
setTimeout(clearProcessingState, 100);
|
|
339
|
+
}).catch((playError) => {
|
|
340
|
+
// Mark requires gesture and store pending text
|
|
341
|
+
setRequiresGesture(true);
|
|
342
|
+
pendingTtsRef.current = text;
|
|
343
|
+
// Fallback: try WebAudio (AudioContext) playback on mobile
|
|
344
|
+
try {
|
|
345
|
+
if (audioCtxRef.current === null) {
|
|
346
|
+
audioCtxRef.current = new AudioContext();
|
|
347
|
+
}
|
|
348
|
+
audioSourceRef.current = audioCtxRef.current.createBufferSource();
|
|
349
|
+
if (!audioSourceRef.current) {
|
|
350
|
+
throw playError;
|
|
351
|
+
}
|
|
352
|
+
audioCtxRef.current.decodeAudioData(
|
|
353
|
+
bufferData.slice(0),
|
|
354
|
+
(decodedBuffer) => {
|
|
355
|
+
if (!audioSourceRef.current) {
|
|
356
|
+
reject(playError);
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
audioSourceRef.current.buffer = decodedBuffer;
|
|
360
|
+
audioSourceRef.current.connect(audioCtxRef.current!.destination);
|
|
361
|
+
speechingRef.current = true;
|
|
362
|
+
clearProcessingState();
|
|
363
|
+
audioSourceRef.current.onended = () => {
|
|
364
|
+
speechingRef.current = false;
|
|
365
|
+
setSpeechingIndex(undefined);
|
|
366
|
+
audioSourceRef.current = null;
|
|
367
|
+
// cleanup audio element blob URL
|
|
368
|
+
URL.revokeObjectURL(audioUrl);
|
|
369
|
+
audioElementRef.current = null;
|
|
370
|
+
resolve();
|
|
371
|
+
};
|
|
372
|
+
try {
|
|
373
|
+
// Ensure AudioContext is running before starting
|
|
374
|
+
if (audioCtxRef.current?.state === 'suspended') {
|
|
375
|
+
audioCtxRef.current.resume().then(() => {
|
|
376
|
+
audioSourceRef.current?.start();
|
|
377
|
+
// Only clear requiresGesture after successfully starting
|
|
378
|
+
setRequiresGesture(false);
|
|
379
|
+
}).catch((resumeErr) => {
|
|
380
|
+
// AudioContext resume failed, keep requiresGesture true
|
|
381
|
+
reject(resumeErr);
|
|
382
|
+
});
|
|
383
|
+
} else {
|
|
384
|
+
audioSourceRef.current.start();
|
|
385
|
+
// Only clear requiresGesture after successfully starting
|
|
386
|
+
setRequiresGesture(false);
|
|
387
|
+
}
|
|
388
|
+
} catch (e) {
|
|
389
|
+
// start() failed, keep requiresGesture true
|
|
390
|
+
reject(e as any);
|
|
391
|
+
}
|
|
392
|
+
},
|
|
393
|
+
(decodeErr) => {
|
|
394
|
+
// If decode also fails, give up
|
|
395
|
+
speechingRef.current = false;
|
|
396
|
+
setSpeechingIndex(undefined);
|
|
397
|
+
URL.revokeObjectURL(audioUrl);
|
|
398
|
+
audioElementRef.current = null;
|
|
399
|
+
clearProcessingState();
|
|
400
|
+
reject(decodeErr as any);
|
|
401
|
+
}
|
|
402
|
+
);
|
|
403
|
+
} catch (fallbackErr) {
|
|
404
|
+
// Play failed (e.g., user interaction required)
|
|
405
|
+
speechingRef.current = false;
|
|
406
|
+
setSpeechingIndex(undefined);
|
|
407
|
+
URL.revokeObjectURL(audioUrl);
|
|
408
|
+
audioElementRef.current = null;
|
|
409
|
+
clearProcessingState();
|
|
410
|
+
reject(fallbackErr as any);
|
|
411
|
+
}
|
|
412
|
+
});
|
|
413
|
+
});
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
if (audioCtxRef.current === null) {
|
|
418
|
+
audioCtxRef.current = new AudioContext();
|
|
419
|
+
}
|
|
420
|
+
audioSourceRef.current = audioCtxRef.current.createBufferSource();
|
|
421
|
+
if (!audioSourceRef.current) return;
|
|
422
|
+
const buffer = await new Promise<AudioBuffer>((resolve, reject) => {
|
|
423
|
+
if (audioCtxRef.current !== null && bufferData) {
|
|
424
|
+
audioCtxRef.current.decodeAudioData(
|
|
425
|
+
bufferData,
|
|
426
|
+
(decodedBuffer) => resolve(decodedBuffer),
|
|
427
|
+
(error) => reject(error)
|
|
428
|
+
);
|
|
429
|
+
} else {
|
|
430
|
+
reject(new Error('Audio context or buffer data is missing'));
|
|
431
|
+
}
|
|
432
|
+
});
|
|
433
|
+
if (!audioSourceRef.current.buffer) {
|
|
434
|
+
audioSourceRef.current.buffer = buffer;
|
|
435
|
+
audioSourceRef.current.connect(audioCtxRef.current.destination);
|
|
436
|
+
speechingRef.current = true;
|
|
437
|
+
if (messageIndex !== undefined) setSpeechingIndex(messageIndex);
|
|
438
|
+
// Clear processing state when audio starts playing
|
|
439
|
+
setRequiresGesture(false);
|
|
440
|
+
setIsTtsProcessing(false);
|
|
441
|
+
await new Promise<void>((resolve) => {
|
|
442
|
+
audioSourceRef.current!.onended = () => {
|
|
443
|
+
speechingRef.current = false;
|
|
444
|
+
setSpeechingIndex(undefined);
|
|
445
|
+
audioSourceRef.current = null;
|
|
446
|
+
resolve();
|
|
447
|
+
};
|
|
448
|
+
audioSourceRef.current!.start();
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
}, []);
|
|
452
|
+
|
|
453
|
+
// Update refs when state changes
|
|
454
|
+
React.useEffect(() => {
|
|
455
|
+
hitAddressRef.current = hitAddress;
|
|
456
|
+
}, [hitAddress]);
|
|
457
|
+
|
|
458
|
+
React.useEffect(() => {
|
|
459
|
+
aitRef.current = ait;
|
|
460
|
+
}, [ait]);
|
|
461
|
+
|
|
462
|
+
React.useEffect(() => {
|
|
463
|
+
permissionsRef.current = permissions;
|
|
464
|
+
}, [permissions]);
|
|
465
|
+
|
|
466
|
+
React.useEffect(() => {
|
|
467
|
+
nxtlinqAITServiceAccessTokenRef.current = nxtlinqAITServiceAccessToken;
|
|
468
|
+
}, [nxtlinqAITServiceAccessToken]);
|
|
469
|
+
|
|
470
|
+
React.useEffect(() => {
|
|
471
|
+
signerRef.current = signer;
|
|
472
|
+
}, [signer]);
|
|
473
|
+
|
|
474
|
+
React.useEffect(() => {
|
|
475
|
+
if (!textInputRef.current) return;
|
|
476
|
+
|
|
477
|
+
const el = textInputRef.current;
|
|
478
|
+
const selStart = el.selectionStart ?? inputValue.length;
|
|
479
|
+
|
|
480
|
+
if (partialTranscript) {
|
|
481
|
+
const { next, caret } = insertPartial(inputValue, partialTranscript, selStart);
|
|
482
|
+
setInputValue(next);
|
|
483
|
+
|
|
484
|
+
setTimeout(() => {
|
|
485
|
+
if (textInputRef.current) {
|
|
486
|
+
textInputRef.current.selectionStart = caret;
|
|
487
|
+
textInputRef.current.selectionEnd = caret;
|
|
488
|
+
}
|
|
489
|
+
}, 0);
|
|
490
|
+
}
|
|
491
|
+
}, [partialTranscript]);
|
|
492
|
+
|
|
493
|
+
React.useEffect(() => {
|
|
494
|
+
if (!transcript) return;
|
|
495
|
+
if (!textInputRef.current) return;
|
|
496
|
+
|
|
497
|
+
// 每段新辨識結果重設 STT 計時起點(長期開麥、多輪自動送出時,每則請求各自一段,避免只記到第一則)
|
|
498
|
+
micSessionStartForLogRef.current = performance.now();
|
|
499
|
+
|
|
500
|
+
const { next, caret } = finalizePartial(inputValue, transcript);
|
|
501
|
+
const normalizedText = normalizeTranscript(next);
|
|
502
|
+
setInputValue(normalizedText);
|
|
503
|
+
|
|
504
|
+
setTimeout(() => {
|
|
505
|
+
if (textInputRef.current) {
|
|
506
|
+
textInputRef.current.selectionStart = caret;
|
|
507
|
+
textInputRef.current.selectionEnd = caret;
|
|
508
|
+
}
|
|
509
|
+
}, 0);
|
|
510
|
+
|
|
511
|
+
// Auto send on speech complete (if user enabled the toggle)
|
|
512
|
+
if (autoSendEnabled && isMicEnabled && normalizedText.trim()) {
|
|
513
|
+
// Avoid duplicate sends
|
|
514
|
+
if (lastAutoSentTranscriptRef.current !== normalizedText.trim()) {
|
|
515
|
+
// Clear any existing timer
|
|
516
|
+
if (autoSendTimerRef.current) {
|
|
517
|
+
clearTimeout(autoSendTimerRef.current);
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
const currentTranscript = normalizedText.trim();
|
|
521
|
+
|
|
522
|
+
// Wait 3 seconds before auto-sending
|
|
523
|
+
// Timer will be cleared if user continues speaking (transcript/partialTranscript changes)
|
|
524
|
+
autoSendTimerRef.current = setTimeout(() => {
|
|
525
|
+
if (lastAutoSentTranscriptRef.current !== currentTranscript && !isLoading) {
|
|
526
|
+
lastAutoSentTranscriptRef.current = currentTranscript;
|
|
527
|
+
sendMessage(currentTranscript).catch(error => {
|
|
528
|
+
console.error('Failed to auto-send message from speech:', error);
|
|
529
|
+
lastAutoSentTranscriptRef.current = '';
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
autoSendTimerRef.current = null;
|
|
533
|
+
}, 3000); // 3 seconds delay
|
|
534
|
+
}
|
|
535
|
+
} else if (autoSendTimerRef.current) {
|
|
536
|
+
// Clear timer if auto-send disabled or mic disabled
|
|
537
|
+
clearTimeout(autoSendTimerRef.current);
|
|
538
|
+
autoSendTimerRef.current = null;
|
|
539
|
+
}
|
|
540
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
541
|
+
}, [transcript, autoSendEnabled, isMicEnabled, isLoading]);
|
|
542
|
+
|
|
543
|
+
// Clear timer when user continues speaking (partialTranscript updates)
|
|
544
|
+
React.useEffect(() => {
|
|
545
|
+
if (partialTranscript && partialTranscript.trim() && autoSendEnabled && isMicEnabled && autoSendTimerRef.current) {
|
|
546
|
+
clearTimeout(autoSendTimerRef.current as NodeJS.Timeout);
|
|
547
|
+
autoSendTimerRef.current = null;
|
|
548
|
+
}
|
|
549
|
+
}, [partialTranscript, autoSendEnabled, isMicEnabled]);
|
|
550
|
+
|
|
551
|
+
// Cleanup timer when mic is disabled or component unmounts
|
|
552
|
+
React.useEffect(() => {
|
|
553
|
+
if (!isMicEnabled) {
|
|
554
|
+
if (autoSendTimerRef.current) {
|
|
555
|
+
clearTimeout(autoSendTimerRef.current as NodeJS.Timeout);
|
|
556
|
+
autoSendTimerRef.current = null;
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
return () => {
|
|
560
|
+
if (autoSendTimerRef.current) {
|
|
561
|
+
clearTimeout(autoSendTimerRef.current as NodeJS.Timeout);
|
|
562
|
+
autoSendTimerRef.current = null;
|
|
563
|
+
}
|
|
564
|
+
};
|
|
565
|
+
}, [isMicEnabled]);
|
|
566
|
+
|
|
567
|
+
React.useEffect(() => {
|
|
568
|
+
isMicEnabledRef.current = isMicEnabled;
|
|
569
|
+
// Reset auto-sent transcript ref when starting new recording
|
|
570
|
+
if (isMicEnabled) {
|
|
571
|
+
lastAutoSentTranscriptRef.current = '';
|
|
572
|
+
// Clear any pending auto-send timer
|
|
573
|
+
if (autoSendTimerRef.current) {
|
|
574
|
+
clearTimeout(autoSendTimerRef.current as NodeJS.Timeout);
|
|
575
|
+
autoSendTimerRef.current = null;
|
|
576
|
+
}
|
|
577
|
+
// Hide gesture prompt once mic is actually enabled
|
|
578
|
+
setIsAwaitingMicGesture(false);
|
|
579
|
+
// Remove any pending gesture listeners
|
|
580
|
+
if (autoStartGestureCleanupRef.current) {
|
|
581
|
+
autoStartGestureCleanupRef.current();
|
|
582
|
+
autoStartGestureCleanupRef.current = null;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
}, [isMicEnabled]);
|
|
586
|
+
|
|
587
|
+
React.useEffect(() => {
|
|
588
|
+
if (typeof window === 'undefined') {
|
|
589
|
+
return;
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
if (storageMode !== "session-storage") {
|
|
593
|
+
sessionStorage.removeItem(MIC_ENABLED_SESSION_KEY);
|
|
594
|
+
hasSyncedMicStateRef.current = false;
|
|
595
|
+
return;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
if (!hasSyncedMicStateRef.current) {
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
try {
|
|
603
|
+
sessionStorage.setItem(MIC_ENABLED_SESSION_KEY, JSON.stringify(isMicEnabled));
|
|
604
|
+
} catch (error) {
|
|
605
|
+
console.warn('Failed to persist recording state to sessionStorage:', error);
|
|
606
|
+
}
|
|
607
|
+
}, [isMicEnabled, storageMode]);
|
|
608
|
+
|
|
609
|
+
React.useEffect(() => {
|
|
610
|
+
if (typeof window === 'undefined') {
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
const cleanupAutoStartListeners = () => {
|
|
615
|
+
if (autoStartGestureCleanupRef.current) {
|
|
616
|
+
autoStartGestureCleanupRef.current();
|
|
617
|
+
autoStartGestureCleanupRef.current = null;
|
|
618
|
+
}
|
|
619
|
+
};
|
|
620
|
+
|
|
621
|
+
if (storageMode !== "session-storage") {
|
|
622
|
+
cleanupAutoStartListeners();
|
|
623
|
+
pendingMicAutoStartRef.current = false;
|
|
624
|
+
setIsAwaitingMicGesture(false);
|
|
625
|
+
return;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
if (hasSyncedMicStateRef.current || isReacquiringMicRef.current) {
|
|
629
|
+
return;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
const rawStoredValue = sessionStorage.getItem(MIC_ENABLED_SESSION_KEY);
|
|
633
|
+
const shouldRestore = rawStoredValue === 'true';
|
|
634
|
+
|
|
635
|
+
if (!shouldRestore || isMicEnabledRef.current || isRestoringRecordingRef.current) {
|
|
636
|
+
hasSyncedMicStateRef.current = true;
|
|
637
|
+
setIsAwaitingMicGesture(false);
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
pendingMicAutoStartRef.current = true;
|
|
642
|
+
|
|
643
|
+
function scheduleRetryOnUserGesture() {
|
|
644
|
+
if (autoStartGestureCleanupRef.current) return;
|
|
645
|
+
|
|
646
|
+
setIsAwaitingMicGesture(true);
|
|
647
|
+
const events: Array<keyof WindowEventMap> = ['pointerdown', 'keydown', 'touchstart'];
|
|
648
|
+
const handler = async () => {
|
|
649
|
+
cleanupAutoStartListeners();
|
|
650
|
+
await attemptAutoStart('user-gesture');
|
|
651
|
+
};
|
|
652
|
+
|
|
653
|
+
events.forEach(event => window.addEventListener(event, handler, { once: true }));
|
|
654
|
+
autoStartGestureCleanupRef.current = () => {
|
|
655
|
+
events.forEach(event => window.removeEventListener(event, handler));
|
|
656
|
+
autoStartGestureCleanupRef.current = null;
|
|
657
|
+
};
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
async function attemptAutoStart(reason: 'initial' | 'user-gesture') {
|
|
661
|
+
isReacquiringMicRef.current = true;
|
|
662
|
+
|
|
663
|
+
// For initial restore, show gesture prompt immediately while checking
|
|
664
|
+
// This gives user early feedback and allows them to click anytime
|
|
665
|
+
if (reason === 'initial') {
|
|
666
|
+
scheduleRetryOnUserGesture();
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
if (typeof navigator !== 'undefined' && navigator.mediaDevices?.getUserMedia) {
|
|
670
|
+
try {
|
|
671
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
672
|
+
stream.getTracks().forEach(track => track.stop());
|
|
673
|
+
} catch (error: any) {
|
|
674
|
+
console.error('Failed to reacquire microphone stream:', error);
|
|
675
|
+
const errorName = error?.name || '';
|
|
676
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
677
|
+
|
|
678
|
+
if (reason === 'initial') {
|
|
679
|
+
const isPermissionPermanentlyDenied = errorName === 'NotAllowedError' &&
|
|
680
|
+
/permission.*denied|blocked|not.*granted/i.test(errorMessage);
|
|
681
|
+
|
|
682
|
+
if (isPermissionPermanentlyDenied) {
|
|
683
|
+
// Permission permanently denied, hide the prompt
|
|
684
|
+
pendingMicAutoStartRef.current = false;
|
|
685
|
+
setIsAwaitingMicGesture(false);
|
|
686
|
+
try {
|
|
687
|
+
sessionStorage.setItem(MIC_ENABLED_SESSION_KEY, JSON.stringify(false));
|
|
688
|
+
} catch (storageError) {
|
|
689
|
+
console.warn('Failed to reset recording state in sessionStorage:', storageError);
|
|
690
|
+
}
|
|
691
|
+
cleanupAutoStartListeners();
|
|
692
|
+
}
|
|
693
|
+
// Otherwise, keep the prompt showing (already shown above)
|
|
694
|
+
isReacquiringMicRef.current = false;
|
|
695
|
+
return;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
// For user-gesture retry, treat as error
|
|
699
|
+
pendingMicAutoStartRef.current = false;
|
|
700
|
+
setIsAwaitingMicGesture(false);
|
|
701
|
+
try {
|
|
702
|
+
sessionStorage.setItem(MIC_ENABLED_SESSION_KEY, JSON.stringify(false));
|
|
703
|
+
} catch (storageError) {
|
|
704
|
+
console.warn('Failed to reset recording state in sessionStorage:', storageError);
|
|
705
|
+
}
|
|
706
|
+
isReacquiringMicRef.current = false;
|
|
707
|
+
hasSyncedMicStateRef.current = true;
|
|
708
|
+
cleanupAutoStartListeners();
|
|
709
|
+
return;
|
|
710
|
+
}
|
|
711
|
+
} else if (reason === 'initial') {
|
|
712
|
+
// getUserMedia not available, prompt already shown above
|
|
713
|
+
isReacquiringMicRef.current = false;
|
|
714
|
+
return;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
isRestoringRecordingRef.current = true;
|
|
718
|
+
|
|
719
|
+
// Intercept console.warn to detect AudioContext warnings
|
|
720
|
+
let audioContextWarningDetected = false;
|
|
721
|
+
const originalWarn = console.warn;
|
|
722
|
+
console.warn = (...args: any[]) => {
|
|
723
|
+
const message = args.join(' ');
|
|
724
|
+
if (message.includes('AudioContext was not allowed to start')) {
|
|
725
|
+
audioContextWarningDetected = true;
|
|
726
|
+
}
|
|
727
|
+
originalWarn.apply(console, args);
|
|
728
|
+
};
|
|
729
|
+
|
|
730
|
+
try {
|
|
731
|
+
await startRecording();
|
|
732
|
+
console.warn = originalWarn;
|
|
733
|
+
|
|
734
|
+
// Check mic status after a short delay
|
|
735
|
+
await new Promise(resolve => setTimeout(resolve, 300));
|
|
736
|
+
|
|
737
|
+
// If mic started successfully, hide the prompt
|
|
738
|
+
if (isMicEnabledRef.current) {
|
|
739
|
+
pendingMicAutoStartRef.current = false;
|
|
740
|
+
setIsAwaitingMicGesture(false);
|
|
741
|
+
cleanupAutoStartListeners();
|
|
742
|
+
} else if (reason === 'initial') {
|
|
743
|
+
// Mic didn't start, keep the prompt showing (already shown earlier)
|
|
744
|
+
isRestoringRecordingRef.current = false;
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
747
|
+
} catch (error) {
|
|
748
|
+
console.error('startRecording threw an error:', error);
|
|
749
|
+
try {
|
|
750
|
+
sessionStorage.setItem(MIC_ENABLED_SESSION_KEY, JSON.stringify(false));
|
|
751
|
+
} catch (storageError) {
|
|
752
|
+
console.warn('Failed to reset recording state in sessionStorage:', storageError);
|
|
753
|
+
}
|
|
754
|
+
} finally {
|
|
755
|
+
console.warn = originalWarn;
|
|
756
|
+
isRestoringRecordingRef.current = false;
|
|
757
|
+
isReacquiringMicRef.current = false;
|
|
758
|
+
hasSyncedMicStateRef.current = true;
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
attemptAutoStart('initial');
|
|
763
|
+
|
|
764
|
+
return () => {
|
|
765
|
+
cleanupAutoStartListeners();
|
|
766
|
+
pendingMicAutoStartRef.current = false;
|
|
767
|
+
setIsAwaitingMicGesture(false);
|
|
768
|
+
};
|
|
769
|
+
}, [storageMode, startRecording]);
|
|
770
|
+
|
|
771
|
+
const [notification, setNotification] = React.useState<{
|
|
772
|
+
show: boolean;
|
|
773
|
+
type: 'success' | 'error' | 'warning' | 'info';
|
|
774
|
+
message: string;
|
|
775
|
+
autoHide?: boolean;
|
|
776
|
+
duration?: number;
|
|
777
|
+
}>({
|
|
778
|
+
show: false,
|
|
779
|
+
type: 'info',
|
|
780
|
+
message: '',
|
|
781
|
+
autoHide: true,
|
|
782
|
+
duration: 5000
|
|
783
|
+
});
|
|
784
|
+
|
|
785
|
+
// AI Model related state
|
|
786
|
+
const [modelsFromApi, setModelsFromApi] = React.useState<AIModel[]>([]);
|
|
787
|
+
const [selectedModelIndex, setSelectedModelIndex] = useLocalStorage<number>('selectedAIModelIndex', 0);
|
|
788
|
+
|
|
789
|
+
// Use models from API only (no default fallback)
|
|
790
|
+
// This ensures SDK always uses Dashboard-configured models
|
|
791
|
+
const effectiveAvailableModels = React.useMemo(() => {
|
|
792
|
+
return modelsFromApi;
|
|
793
|
+
}, [modelsFromApi]);
|
|
794
|
+
|
|
795
|
+
// Auto determine if selector should be shown based on number of models
|
|
796
|
+
// Show selector only when there are 2 or more models
|
|
797
|
+
const showModelSelector = React.useMemo(() => {
|
|
798
|
+
return effectiveAvailableModels.length >= 2;
|
|
799
|
+
}, [effectiveAvailableModels]);
|
|
800
|
+
|
|
801
|
+
// Fetch available models from API on mount
|
|
802
|
+
React.useEffect(() => {
|
|
803
|
+
const fetchModels = async () => {
|
|
804
|
+
try {
|
|
805
|
+
const result = await nxtlinqApi.agent.getServiceModels({ apiKey, apiSecret });
|
|
806
|
+
|
|
807
|
+
if ('availableModels' in result && Array.isArray(result.availableModels)) {
|
|
808
|
+
setModelsFromApi(result.availableModels);
|
|
809
|
+
}
|
|
810
|
+
} catch (error) {
|
|
811
|
+
console.error('Failed to fetch available models from API:', error);
|
|
812
|
+
// Silently fail and use default models
|
|
813
|
+
}
|
|
814
|
+
};
|
|
815
|
+
|
|
816
|
+
fetchModels();
|
|
817
|
+
}, [apiKey, apiSecret, nxtlinqApi]);
|
|
818
|
+
|
|
819
|
+
// Validate selectedModelIndex against effectiveAvailableModels
|
|
820
|
+
React.useEffect(() => {
|
|
821
|
+
// Check if selectedModelIndex is out of bounds
|
|
822
|
+
if (selectedModelIndex >= effectiveAvailableModels.length) {
|
|
823
|
+
setSelectedModelIndex(0);
|
|
824
|
+
return;
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
// Check if the selected model still exists in effectiveAvailableModels
|
|
828
|
+
const selectedModel = effectiveAvailableModels[selectedModelIndex];
|
|
829
|
+
if (!selectedModel) {
|
|
830
|
+
setSelectedModelIndex(0);
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
833
|
+
}, [effectiveAvailableModels, selectedModelIndex, setSelectedModelIndex]);
|
|
834
|
+
|
|
835
|
+
// Notification functions
|
|
836
|
+
const showNotification = (type: 'success' | 'error' | 'warning' | 'info', message: string, duration = 5000) => {
|
|
837
|
+
setNotification({
|
|
838
|
+
show: true,
|
|
839
|
+
type,
|
|
840
|
+
message,
|
|
841
|
+
autoHide: true,
|
|
842
|
+
duration
|
|
843
|
+
});
|
|
844
|
+
|
|
845
|
+
if (duration > 0) {
|
|
846
|
+
setTimeout(() => {
|
|
847
|
+
setNotification(prev => ({ ...prev, show: false }));
|
|
848
|
+
}, duration);
|
|
849
|
+
}
|
|
850
|
+
};
|
|
851
|
+
|
|
852
|
+
const showSuccess = (message: string) => showNotification('success', message, 3000);
|
|
853
|
+
const showError = (message: string) => showNotification('error', message, 5000);
|
|
854
|
+
const showWarning = (message: string) => showNotification('warning', message, 4000);
|
|
855
|
+
const showInfo = (message: string) => showNotification('info', message, 3000);
|
|
856
|
+
|
|
857
|
+
// Show "Please log in again." instead of raw "Invalid or expired token" to users (English)
|
|
858
|
+
const normalizeErrorForUser = (message: string): string =>
|
|
859
|
+
/invalid\s+or\s+expired\s+token/i.test(message) ? 'Please log in again.' : message;
|
|
860
|
+
|
|
861
|
+
// Fetch available permissions. Returns the service's permission list when successful, otherwise undefined.
|
|
862
|
+
const fetchAvailablePermissions = async (): Promise<ServicePermission[] | undefined> => {
|
|
863
|
+
if (!serviceId) return undefined;
|
|
864
|
+
|
|
865
|
+
try {
|
|
866
|
+
const result = await nxtlinqApi.permissions.getServicePermissions({
|
|
867
|
+
serviceId,
|
|
868
|
+
...(permissionGroup && { groupName: permissionGroup })
|
|
869
|
+
});
|
|
870
|
+
if ('error' in result) {
|
|
871
|
+
console.error('Failed to fetch permissions:', result.error);
|
|
872
|
+
return undefined;
|
|
873
|
+
}
|
|
874
|
+
setAvailablePermissions(result.permissions);
|
|
875
|
+
return result.permissions;
|
|
876
|
+
} catch (error) {
|
|
877
|
+
console.error('Error fetching permissions:', error);
|
|
878
|
+
return undefined;
|
|
879
|
+
}
|
|
880
|
+
};
|
|
881
|
+
|
|
882
|
+
// Refresh AIT
|
|
883
|
+
const refreshAIT = async (forceUpdatePermissions = false) => {
|
|
884
|
+
if (!hitAddress) {
|
|
885
|
+
setAit(null);
|
|
886
|
+
setPermissions([]);
|
|
887
|
+
setWalletInfo(null);
|
|
888
|
+
setIsAITLoading(false);
|
|
889
|
+
return;
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
setIsAITLoading(true);
|
|
893
|
+
try {
|
|
894
|
+
// Get wallet info first - always try to get wallet info if we have a token
|
|
895
|
+
if (nxtlinqAITServiceAccessToken) {
|
|
896
|
+
try {
|
|
897
|
+
const walletResponse = await nxtlinqApi.wallet.getWallet({ address: hitAddress }, nxtlinqAITServiceAccessToken);
|
|
898
|
+
if (!('error' in walletResponse)) {
|
|
899
|
+
setWalletInfo(walletResponse);
|
|
900
|
+
} else {
|
|
901
|
+
// Check if the error is due to invalid/expired token
|
|
902
|
+
if (walletResponse.error.includes('Invalid or expired token')) {
|
|
903
|
+
console.log('Token appears to be invalid during wallet info fetch, clearing it');
|
|
904
|
+
clearExpiredToken();
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
} catch (error) {
|
|
908
|
+
console.error('Failed to fetch wallet info:', error);
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
// Only try to fetch AIT if we have a token
|
|
913
|
+
if (nxtlinqAITServiceAccessToken) {
|
|
914
|
+
const response = await nxtlinqApi.ait.getAITByServiceIdAndController({
|
|
915
|
+
serviceId,
|
|
916
|
+
controller: hitAddress,
|
|
917
|
+
customUsername: (!requireWalletIDVVerification && customUsername) ? getFinalCustomUsername(customUsername) : undefined
|
|
918
|
+
}, nxtlinqAITServiceAccessToken);
|
|
919
|
+
|
|
920
|
+
if ('error' in response) {
|
|
921
|
+
console.error('Failed to fetch AIT:', response.error);
|
|
922
|
+
setAit(null);
|
|
923
|
+
setPermissions([]);
|
|
924
|
+
return;
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
setAit(response);
|
|
928
|
+
if (!isPermissionFormOpen || forceUpdatePermissions) {
|
|
929
|
+
const newPermissions = response.metadata?.permissions || [];
|
|
930
|
+
setPermissions(newPermissions);
|
|
931
|
+
}
|
|
932
|
+
} else {
|
|
933
|
+
// No token available, clear AIT data
|
|
934
|
+
setAit(null);
|
|
935
|
+
setPermissions([]);
|
|
936
|
+
}
|
|
937
|
+
} catch (error) {
|
|
938
|
+
console.error('Failed to fetch AIT:', error);
|
|
939
|
+
setAit(null);
|
|
940
|
+
setPermissions([]);
|
|
941
|
+
} finally {
|
|
942
|
+
setIsAITLoading(false);
|
|
943
|
+
}
|
|
944
|
+
};
|
|
945
|
+
|
|
946
|
+
// Check if user needs to sign in
|
|
947
|
+
const isNeedSignInWithWallet = React.useMemo(() => {
|
|
948
|
+
if (!hitAddress) return false;
|
|
949
|
+
if (!nxtlinqAITServiceAccessToken) return true;
|
|
950
|
+
|
|
951
|
+
try {
|
|
952
|
+
const payload = JSON.parse(atob(nxtlinqAITServiceAccessToken.split('.')[1]));
|
|
953
|
+
const address = payload.address;
|
|
954
|
+
if (address !== hitAddress) return true;
|
|
955
|
+
|
|
956
|
+
return false;
|
|
957
|
+
} catch (error) {
|
|
958
|
+
console.error('Error parsing token:', error);
|
|
959
|
+
return true;
|
|
960
|
+
}
|
|
961
|
+
}, [hitAddress, nxtlinqAITServiceAccessToken]);
|
|
962
|
+
|
|
963
|
+
// Connect wallet
|
|
964
|
+
const connectWallet = React.useCallback(async (autoShowSignInMessage = true) => {
|
|
965
|
+
if (typeof window === 'undefined') {
|
|
966
|
+
console.error('Web3 is not available in server-side rendering');
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
try {
|
|
971
|
+
getEthers();
|
|
972
|
+
|
|
973
|
+
const web3Provider = await metakeepClient.ethereum;
|
|
974
|
+
if (!web3Provider) {
|
|
975
|
+
throw new Error('Web3 provider not available');
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
await web3Provider.enable();
|
|
979
|
+
|
|
980
|
+
// ethers v6: Use BrowserProvider instead of Web3Provider
|
|
981
|
+
const ethersProvider = new ethers.BrowserProvider(web3Provider);
|
|
982
|
+
|
|
983
|
+
// ethers v6: getSigner() and getAddress() are now async
|
|
984
|
+
const userSigner = await ethersProvider.getSigner();
|
|
985
|
+
const userAddress = await userSigner.getAddress();
|
|
986
|
+
|
|
987
|
+
localStorage.setItem('walletAddress', userAddress);
|
|
988
|
+
|
|
989
|
+
setHitAddress(userAddress);
|
|
990
|
+
setSigner(userSigner);
|
|
991
|
+
|
|
992
|
+
// Immediately update refs to ensure they're available for subsequent operations
|
|
993
|
+
hitAddressRef.current = userAddress;
|
|
994
|
+
signerRef.current = userSigner;
|
|
995
|
+
|
|
996
|
+
// Check if we have a valid token before trying to load AIT
|
|
997
|
+
if (nxtlinqAITServiceAccessToken) {
|
|
998
|
+
try {
|
|
999
|
+
const payload = JSON.parse(atob(nxtlinqAITServiceAccessToken.split('.')[1]));
|
|
1000
|
+
const address = payload.address;
|
|
1001
|
+
|
|
1002
|
+
// Check if address matches
|
|
1003
|
+
if (address !== userAddress) {
|
|
1004
|
+
console.log('Token address mismatch, clearing token');
|
|
1005
|
+
clearExpiredToken();
|
|
1006
|
+
// Don't call refreshAIT with invalid token
|
|
1007
|
+
} else {
|
|
1008
|
+
// Token address matches, try to load AIT
|
|
1009
|
+
await refreshAIT();
|
|
1010
|
+
await sleep(1000);
|
|
1011
|
+
}
|
|
1012
|
+
} catch (error) {
|
|
1013
|
+
console.error('Error parsing token during wallet connection:', error);
|
|
1014
|
+
clearExpiredToken();
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
// If requireIDV is false and we have customUsername, check if wallet exists in AIT Service
|
|
1019
|
+
if (!requireWalletIDVVerification && customUsername) {
|
|
1020
|
+
try {
|
|
1021
|
+
// Check if wallet exists in AIT Service
|
|
1022
|
+
const walletResponse = await nxtlinqApi.wallet.getWallet({ address: userAddress }, '');
|
|
1023
|
+
if ('error' in walletResponse) {
|
|
1024
|
+
// Wallet doesn't exist, create it with custom method
|
|
1025
|
+
console.log('Wallet not found in AIT Service, creating with custom method...');
|
|
1026
|
+
|
|
1027
|
+
const verifyPayload = {
|
|
1028
|
+
address: userAddress,
|
|
1029
|
+
method: 'custom' as const,
|
|
1030
|
+
timestamp: Date.now(),
|
|
1031
|
+
customUsername: getFinalCustomUsername(customUsername)
|
|
1032
|
+
};
|
|
1033
|
+
const verifyResponse = await nxtlinqApi.wallet.verifyWallet(verifyPayload, '');
|
|
1034
|
+
if ('error' in verifyResponse) {
|
|
1035
|
+
console.error('Failed to create wallet with custom method:', verifyResponse.error);
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
} catch (error) {
|
|
1039
|
+
console.error('Error checking/creating wallet in AIT Service:', error);
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
if (autoShowSignInMessage) {
|
|
1044
|
+
if (isNeedSignInWithWallet) {
|
|
1045
|
+
setMessages(prev => [...prev, {
|
|
1046
|
+
id: Date.now().toString(),
|
|
1047
|
+
content: walletTextUtils.getWalletText('Please sign in with your wallet to continue.', serviceId),
|
|
1048
|
+
role: 'assistant',
|
|
1049
|
+
timestamp: new Date().toISOString(),
|
|
1050
|
+
button: 'signIn'
|
|
1051
|
+
}]);
|
|
1052
|
+
} else {
|
|
1053
|
+
showSuccess(walletTextUtils.getWalletText('Successfully connected your HIT wallet. You are already signed in and can use the AI agent.', serviceId));
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
return userAddress;
|
|
1058
|
+
} catch (error) {
|
|
1059
|
+
console.error('Failed to connect wallet:', error);
|
|
1060
|
+
localStorage.removeItem('walletAddress');
|
|
1061
|
+
setHitAddress(null);
|
|
1062
|
+
setSigner(null);
|
|
1063
|
+
if (autoShowSignInMessage) {
|
|
1064
|
+
throw error;
|
|
1065
|
+
}
|
|
1066
|
+
return false;
|
|
1067
|
+
}
|
|
1068
|
+
}, [nxtlinqAITServiceAccessToken, refreshAIT, isNeedSignInWithWallet, requireWalletIDVVerification, customUsername, nxtlinqApi]);
|
|
1069
|
+
|
|
1070
|
+
const signInWallet = async (autoShowSuccessMessage = true) => {
|
|
1071
|
+
// Use refs to get latest state values for consistency with hasPermission
|
|
1072
|
+
const currentHitAddress = hitAddressRef.current;
|
|
1073
|
+
const currentSigner = signerRef.current;
|
|
1074
|
+
|
|
1075
|
+
// If refs don't have the values, try using current state values
|
|
1076
|
+
let addressToUse = currentHitAddress || hitAddress;
|
|
1077
|
+
let signerToUse = currentSigner || signer;
|
|
1078
|
+
|
|
1079
|
+
// If we have an address but no signer, try to auto-connect wallet
|
|
1080
|
+
if (addressToUse && !signerToUse) {
|
|
1081
|
+
const storedWalletAddress = localStorage.getItem('walletAddress');
|
|
1082
|
+
if (storedWalletAddress === addressToUse) {
|
|
1083
|
+
try {
|
|
1084
|
+
const autoConnectResult = await connectWallet(false); // Don't show sign-in message yet
|
|
1085
|
+
if (autoConnectResult) {
|
|
1086
|
+
// Wait for state to update
|
|
1087
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
1088
|
+
// Get the updated signer
|
|
1089
|
+
signerToUse = signerRef.current || signer;
|
|
1090
|
+
addressToUse = hitAddressRef.current || hitAddress;
|
|
1091
|
+
}
|
|
1092
|
+
} catch (error) {
|
|
1093
|
+
console.error('Auto-connect failed during sign in:', error);
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
if (!addressToUse) {
|
|
1099
|
+
console.log('No address available, returning early');
|
|
1100
|
+
if (autoShowSuccessMessage) {
|
|
1101
|
+
showError(walletTextUtils.getWalletText('Please connect your wallet first.', serviceId));
|
|
1102
|
+
}
|
|
1103
|
+
return;
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
if (!signerToUse) {
|
|
1107
|
+
console.log('No signer available, returning early');
|
|
1108
|
+
if (autoShowSuccessMessage) {
|
|
1109
|
+
showError(walletTextUtils.getWalletText('Please connect your wallet first.', serviceId));
|
|
1110
|
+
}
|
|
1111
|
+
return;
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
try {
|
|
1115
|
+
const nonceResponse = await nxtlinqApi.auth.getNonce({ address: addressToUse! });
|
|
1116
|
+
if ('error' in nonceResponse) {
|
|
1117
|
+
if (autoShowSuccessMessage) {
|
|
1118
|
+
showError(normalizeErrorForUser(nonceResponse.error));
|
|
1119
|
+
}
|
|
1120
|
+
return;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
const payload = {
|
|
1124
|
+
address: addressToUse!,
|
|
1125
|
+
code: nonceResponse.code,
|
|
1126
|
+
timestamp: nonceResponse.timestamp
|
|
1127
|
+
};
|
|
1128
|
+
|
|
1129
|
+
const stringToSign = stringify(payload);
|
|
1130
|
+
const signature = await signerToUse.signMessage(stringToSign || '');
|
|
1131
|
+
|
|
1132
|
+
const response = await nxtlinqApi.auth.signIn({
|
|
1133
|
+
...payload,
|
|
1134
|
+
signature
|
|
1135
|
+
});
|
|
1136
|
+
|
|
1137
|
+
if ('error' in response) {
|
|
1138
|
+
if (autoShowSuccessMessage) {
|
|
1139
|
+
showError(normalizeErrorForUser(response.error));
|
|
1140
|
+
}
|
|
1141
|
+
return;
|
|
1142
|
+
}
|
|
1143
|
+
const { accessToken } = response;
|
|
1144
|
+
setNxtlinqAITServiceAccessToken(accessToken);
|
|
1145
|
+
|
|
1146
|
+
// Wait for state to update before refreshing AIT
|
|
1147
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
1148
|
+
await refreshAIT();
|
|
1149
|
+
|
|
1150
|
+
if (autoShowSuccessMessage) {
|
|
1151
|
+
showSuccess(walletTextUtils.getWalletText('Successfully signed in with your HIT wallet. You can now use the AI agent.', serviceId));
|
|
1152
|
+
}
|
|
1153
|
+
} catch (error) {
|
|
1154
|
+
console.error('Failed to sign in:', error);
|
|
1155
|
+
if (autoShowSuccessMessage) {
|
|
1156
|
+
showError('Failed to sign in. Please try again.');
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
};
|
|
1160
|
+
|
|
1161
|
+
// Check permissions
|
|
1162
|
+
const hasPermission = async (requiredPermission: string, autoRetry = true, onAutoConnect?: () => void, onAutoSignIn?: () => void): Promise<boolean> => {
|
|
1163
|
+
// Use refs to get latest state values
|
|
1164
|
+
const currentHitAddress = hitAddressRef.current;
|
|
1165
|
+
const currentAit = aitRef.current;
|
|
1166
|
+
const currentPermissions = permissionsRef.current;
|
|
1167
|
+
const currentToken = nxtlinqAITServiceAccessTokenRef.current;
|
|
1168
|
+
|
|
1169
|
+
if (!currentHitAddress) {
|
|
1170
|
+
if (autoRetry) {
|
|
1171
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1172
|
+
setMessages(prev => [...prev, {
|
|
1173
|
+
id: Date.now().toString(),
|
|
1174
|
+
content: walletTextUtils.getWalletText('Please connect your HIT wallet to continue.', serviceId),
|
|
1175
|
+
role: 'assistant',
|
|
1176
|
+
timestamp: new Date().toISOString(),
|
|
1177
|
+
button: 'connectWallet'
|
|
1178
|
+
}]);
|
|
1179
|
+
|
|
1180
|
+
try {
|
|
1181
|
+
setIsAutoConnecting(true); // Mark as auto-connecting
|
|
1182
|
+
await connectWallet(false); // Don't show sign-in message yet
|
|
1183
|
+
setIsAutoConnecting(false); // Clear auto-connecting state
|
|
1184
|
+
|
|
1185
|
+
// Show brief success message for auto-connect
|
|
1186
|
+
showSuccess(walletTextUtils.getWalletText('Auto wallet connection successful', serviceId));
|
|
1187
|
+
|
|
1188
|
+
|
|
1189
|
+
onAutoConnect?.(); // Call callback if provided
|
|
1190
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1191
|
+
|
|
1192
|
+
// After auto connect, if not signed in, then auto sign-in
|
|
1193
|
+
const tokenAfterConnect = nxtlinqAITServiceAccessTokenRef.current;
|
|
1194
|
+
if (!tokenAfterConnect) {
|
|
1195
|
+
setIsAutoConnecting(true);
|
|
1196
|
+
await signInWallet(false);
|
|
1197
|
+
onAutoSignIn?.(); // Call callback if provided
|
|
1198
|
+
setIsAutoConnecting(false);
|
|
1199
|
+
showSuccess(walletTextUtils.getWalletText('Auto sign-in successful after wallet connect', serviceId));
|
|
1200
|
+
await refreshAIT();
|
|
1201
|
+
// Wait for AIT to be fully loaded with polling
|
|
1202
|
+
let attempts = 0;
|
|
1203
|
+
const maxAttempts = 5;
|
|
1204
|
+
while (!aitRef.current && attempts < maxAttempts) {
|
|
1205
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1206
|
+
attempts++;
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
// If connection (and sign-in if needed) successful, continue with permission check
|
|
1211
|
+
const result = await hasPermission(requiredPermission, false);
|
|
1212
|
+
return result;
|
|
1213
|
+
} catch (error) {
|
|
1214
|
+
console.error('Failed to auto-connect wallet:', error);
|
|
1215
|
+
setIsAutoConnecting(false); // Clear auto-connecting state on error
|
|
1216
|
+
return false;
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
// If autoRetry is false, don't show message again, just return false
|
|
1220
|
+
return false;
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
if (!currentToken) {
|
|
1224
|
+
if (autoRetry) {
|
|
1225
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1226
|
+
setMessages(prev => [...prev, {
|
|
1227
|
+
id: Date.now().toString(),
|
|
1228
|
+
content: walletTextUtils.getWalletText('Please sign in with your HIT wallet to continue.', serviceId),
|
|
1229
|
+
role: 'assistant',
|
|
1230
|
+
timestamp: new Date().toISOString(),
|
|
1231
|
+
button: 'signIn'
|
|
1232
|
+
}]);
|
|
1233
|
+
|
|
1234
|
+
try {
|
|
1235
|
+
setIsAutoConnecting(true); // Mark as auto-signing
|
|
1236
|
+
await signInWallet(false); // Don't show success message yet
|
|
1237
|
+
onAutoSignIn?.(); // Call callback if provided
|
|
1238
|
+
setIsAutoConnecting(false); // Clear auto-signing state
|
|
1239
|
+
|
|
1240
|
+
// Show brief success message for auto-sign-in
|
|
1241
|
+
showSuccess('Auto sign-in successful');
|
|
1242
|
+
|
|
1243
|
+
// Ensure AIT is refreshed after sign-in
|
|
1244
|
+
await refreshAIT();
|
|
1245
|
+
|
|
1246
|
+
// Wait for AIT to be fully loaded with polling
|
|
1247
|
+
let attempts = 0;
|
|
1248
|
+
const maxAttempts = 5; // Wait up to 10 seconds (5 * 2000ms)
|
|
1249
|
+
while (!aitRef.current && attempts < maxAttempts) {
|
|
1250
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1251
|
+
attempts++;
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
// Only continue if AIT is actually loaded
|
|
1255
|
+
if (aitRef.current) {
|
|
1256
|
+
// Wait a bit more to ensure permissions are also loaded
|
|
1257
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1258
|
+
// If sign-in successful, continue with permission check
|
|
1259
|
+
const result = await hasPermission(requiredPermission, false);
|
|
1260
|
+
return result;
|
|
1261
|
+
} else {
|
|
1262
|
+
return false;
|
|
1263
|
+
}
|
|
1264
|
+
} catch (error) {
|
|
1265
|
+
console.error('Failed to auto-sign-in wallet:', error);
|
|
1266
|
+
setIsAutoConnecting(false); // Clear auto-signing state on error
|
|
1267
|
+
return false;
|
|
1268
|
+
}
|
|
1269
|
+
}
|
|
1270
|
+
// If autoRetry is false, don't show message again, just return false
|
|
1271
|
+
return false;
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
try {
|
|
1275
|
+
const payload = JSON.parse(atob(currentToken.split('.')[1]));
|
|
1276
|
+
const address = payload.address;
|
|
1277
|
+
if (address !== currentHitAddress) {
|
|
1278
|
+
setNxtlinqAITServiceAccessToken('');
|
|
1279
|
+
if (autoRetry) {
|
|
1280
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1281
|
+
setMessages(prev => [...prev, {
|
|
1282
|
+
id: Date.now().toString(),
|
|
1283
|
+
content: walletTextUtils.getWalletText('Wallet address mismatch. Please sign in with the correct wallet.', serviceId),
|
|
1284
|
+
role: 'assistant',
|
|
1285
|
+
timestamp: new Date().toISOString(),
|
|
1286
|
+
button: 'signIn'
|
|
1287
|
+
}]);
|
|
1288
|
+
|
|
1289
|
+
try {
|
|
1290
|
+
setIsAutoConnecting(true); // Mark as auto-signing
|
|
1291
|
+
await signInWallet(false); // Don't show success message yet
|
|
1292
|
+
onAutoSignIn?.(); // Call callback if provided
|
|
1293
|
+
setIsAutoConnecting(false); // Clear auto-signing state
|
|
1294
|
+
|
|
1295
|
+
// Show brief success message for auto-sign-in after address mismatch
|
|
1296
|
+
showSuccess('Auto sign-in successful after address mismatch');
|
|
1297
|
+
|
|
1298
|
+
// Ensure AIT is refreshed after sign-in
|
|
1299
|
+
await refreshAIT();
|
|
1300
|
+
|
|
1301
|
+
// Wait for AIT to be fully loaded with polling
|
|
1302
|
+
let attempts = 0;
|
|
1303
|
+
const maxAttempts = 5; // Wait up to 10 seconds (5 * 2000ms)
|
|
1304
|
+
while (!aitRef.current && attempts < maxAttempts) {
|
|
1305
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1306
|
+
attempts++;
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
// Only continue if AIT is actually loaded
|
|
1310
|
+
if (aitRef.current) {
|
|
1311
|
+
// Wait a bit more to ensure permissions are also loaded
|
|
1312
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1313
|
+
// If sign-in successful, continue with permission check
|
|
1314
|
+
const result = await hasPermission(requiredPermission, false);
|
|
1315
|
+
return result;
|
|
1316
|
+
} else {
|
|
1317
|
+
return false;
|
|
1318
|
+
}
|
|
1319
|
+
} catch (error) {
|
|
1320
|
+
console.error('Failed to auto-sign-in after address mismatch:', error);
|
|
1321
|
+
setIsAutoConnecting(false); // Clear auto-signing state on error
|
|
1322
|
+
return false;
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
// If autoRetry is false, don't show message again, just return false
|
|
1326
|
+
return false;
|
|
1327
|
+
}
|
|
1328
|
+
} catch (error) {
|
|
1329
|
+
console.error('Error parsing token:', error);
|
|
1330
|
+
setNxtlinqAITServiceAccessToken('');
|
|
1331
|
+
if (autoRetry) {
|
|
1332
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1333
|
+
setMessages(prev => [...prev, {
|
|
1334
|
+
id: Date.now().toString(),
|
|
1335
|
+
content: walletTextUtils.getWalletText('Invalid wallet session. Please sign in again.', serviceId),
|
|
1336
|
+
role: 'assistant',
|
|
1337
|
+
timestamp: new Date().toISOString(),
|
|
1338
|
+
button: 'signIn'
|
|
1339
|
+
}]);
|
|
1340
|
+
|
|
1341
|
+
try {
|
|
1342
|
+
setIsAutoConnecting(true); // Mark as auto-signing
|
|
1343
|
+
await signInWallet(false); // Don't show success message yet
|
|
1344
|
+
onAutoSignIn?.(); // Call callback if provided
|
|
1345
|
+
setIsAutoConnecting(false); // Clear auto-signing state
|
|
1346
|
+
|
|
1347
|
+
// Show brief success message for auto-sign-in after token parse error
|
|
1348
|
+
showSuccess('Auto sign-in successful after token error');
|
|
1349
|
+
|
|
1350
|
+
// Ensure AIT is refreshed after sign-in
|
|
1351
|
+
await refreshAIT();
|
|
1352
|
+
|
|
1353
|
+
// Wait for AIT to be fully loaded with polling
|
|
1354
|
+
let attempts = 0;
|
|
1355
|
+
const maxAttempts = 5; // Wait up to 10 seconds (5 * 2000ms)
|
|
1356
|
+
while (!aitRef.current && attempts < maxAttempts) {
|
|
1357
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1358
|
+
attempts++;
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
// Only continue if AIT is actually loaded
|
|
1362
|
+
if (aitRef.current) {
|
|
1363
|
+
// Wait a bit more to ensure permissions are also loaded
|
|
1364
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1365
|
+
// If sign-in successful, continue with permission check
|
|
1366
|
+
const result = await hasPermission(requiredPermission, false);
|
|
1367
|
+
return result;
|
|
1368
|
+
} else {
|
|
1369
|
+
return false;
|
|
1370
|
+
}
|
|
1371
|
+
} catch (signInError) {
|
|
1372
|
+
console.error('Failed to auto-sign-in after token parse error:', signInError);
|
|
1373
|
+
setIsAutoConnecting(false); // Clear auto-signing state on error
|
|
1374
|
+
return false;
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
// If autoRetry is false, don't show message again, just return false
|
|
1378
|
+
return false;
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
if (!currentAit) {
|
|
1382
|
+
// Show loading message if AIT is still loading
|
|
1383
|
+
if (isAITLoading) {
|
|
1384
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1385
|
+
setMessages(prev => [...prev, {
|
|
1386
|
+
id: Date.now().toString(),
|
|
1387
|
+
content: walletTextUtils.getWalletText('Loading your wallet configuration... Please wait a moment.', serviceId),
|
|
1388
|
+
role: 'assistant',
|
|
1389
|
+
timestamp: new Date().toISOString()
|
|
1390
|
+
}]);
|
|
1391
|
+
return false;
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
// If AIT is not loaded but we have a token, try to refresh it once
|
|
1395
|
+
if (currentToken && !isAITLoading) {
|
|
1396
|
+
try {
|
|
1397
|
+
await refreshAIT();
|
|
1398
|
+
// Wait for AIT to be loaded with polling
|
|
1399
|
+
let attempts = 0;
|
|
1400
|
+
const maxAttempts = 5; // Wait up to 10 seconds (5 * 2000ms)
|
|
1401
|
+
while (!aitRef.current && attempts < maxAttempts) {
|
|
1402
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1403
|
+
attempts++;
|
|
1404
|
+
}
|
|
1405
|
+
// Check again after refresh
|
|
1406
|
+
if (!aitRef.current) {
|
|
1407
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1408
|
+
setMessages(prev => [...prev, {
|
|
1409
|
+
id: Date.now().toString(),
|
|
1410
|
+
content: walletTextUtils.getWalletText('No AIT found for your wallet. Please click the settings button (⚙️) to configure your AIT permissions.', serviceId),
|
|
1411
|
+
role: 'assistant',
|
|
1412
|
+
timestamp: new Date().toISOString()
|
|
1413
|
+
}]);
|
|
1414
|
+
return false;
|
|
1415
|
+
}
|
|
1416
|
+
} catch (error) {
|
|
1417
|
+
console.error('Failed to refresh AIT during permission check:', error);
|
|
1418
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1419
|
+
setMessages(prev => [...prev, {
|
|
1420
|
+
id: Date.now().toString(),
|
|
1421
|
+
content: 'No AIT found for your wallet. Please click the settings button (⚙️) to configure your AIT permissions.',
|
|
1422
|
+
role: 'assistant',
|
|
1423
|
+
timestamp: new Date().toISOString()
|
|
1424
|
+
}]);
|
|
1425
|
+
return false;
|
|
1426
|
+
}
|
|
1427
|
+
} else {
|
|
1428
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1429
|
+
setMessages(prev => [...prev, {
|
|
1430
|
+
id: Date.now().toString(),
|
|
1431
|
+
content: 'No AIT found for your wallet. Please click the settings button (⚙️) to configure your AIT permissions.',
|
|
1432
|
+
role: 'assistant',
|
|
1433
|
+
timestamp: new Date().toISOString()
|
|
1434
|
+
}]);
|
|
1435
|
+
return false;
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
if (availablePermissions.length === 0) {
|
|
1440
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1441
|
+
setMessages(prev => [...prev, {
|
|
1442
|
+
id: Date.now().toString(),
|
|
1443
|
+
content: `No permissions available for your current identity provider. Please check your service configuration or contact support. Service ID: ${serviceId}, Permission Group: ${permissionGroup || 'None'}`,
|
|
1444
|
+
role: 'assistant',
|
|
1445
|
+
timestamp: new Date().toISOString()
|
|
1446
|
+
}]);
|
|
1447
|
+
return false;
|
|
1448
|
+
}
|
|
1449
|
+
|
|
1450
|
+
const checkToolPermissionResult = await nxtlinqApi.agent.checkToolPermission({
|
|
1451
|
+
apiKey,
|
|
1452
|
+
apiSecret,
|
|
1453
|
+
aitToken: currentToken!,
|
|
1454
|
+
controller: currentHitAddress!,
|
|
1455
|
+
toolName: requiredPermission
|
|
1456
|
+
})
|
|
1457
|
+
|
|
1458
|
+
if ('error' in checkToolPermissionResult) {
|
|
1459
|
+
// Check if AIT is still loading or if we just auto-connected
|
|
1460
|
+
if (isAITLoading || isAutoConnecting) {
|
|
1461
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1462
|
+
setMessages(prev => [...prev, {
|
|
1463
|
+
id: Date.now().toString(),
|
|
1464
|
+
content: walletTextUtils.getWalletText('Loading your wallet configuration... Please wait a moment.', serviceId),
|
|
1465
|
+
role: 'assistant',
|
|
1466
|
+
timestamp: new Date().toISOString()
|
|
1467
|
+
}]);
|
|
1468
|
+
return false;
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
// Refresh service permissions before judging; use latest result to decide message
|
|
1472
|
+
const freshAvailablePermissions = await fetchAvailablePermissions();
|
|
1473
|
+
const latestAvailablePermissions = freshAvailablePermissions ?? availablePermissions;
|
|
1474
|
+
|
|
1475
|
+
const requiredPermission = checkToolPermissionResult.requiredPermission;
|
|
1476
|
+
const isInServiceList = latestAvailablePermissions.some((p) => p.label === requiredPermission);
|
|
1477
|
+
|
|
1478
|
+
// Permission not in service's available list → "not available for your current identity provider"
|
|
1479
|
+
if (!isInServiceList) {
|
|
1480
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1481
|
+
setMessages(prev => [...prev, {
|
|
1482
|
+
id: Date.now().toString(),
|
|
1483
|
+
content: `This permission (${requiredPermission}) is not available for your current identity provider.`,
|
|
1484
|
+
role: 'assistant',
|
|
1485
|
+
timestamp: new Date().toISOString()
|
|
1486
|
+
}]);
|
|
1487
|
+
return false;
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
// AIT loaded but permissions empty (no AIT found)
|
|
1491
|
+
if (currentAit && currentPermissions.length === 0) {
|
|
1492
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1493
|
+
setMessages(prev => [...prev, {
|
|
1494
|
+
id: Date.now().toString(),
|
|
1495
|
+
content: 'No AIT found for your wallet. Please click the settings button (⚙️) to configure your AIT permissions.',
|
|
1496
|
+
role: 'assistant',
|
|
1497
|
+
timestamp: new Date().toISOString()
|
|
1498
|
+
}]);
|
|
1499
|
+
return false;
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1502
|
+
// User has AIT but hasn't enabled this permission → prompt to enable
|
|
1503
|
+
setIsLoading(false); // Stop thinking before showing message
|
|
1504
|
+
const permissionMsg = `You don't have the required AIT permission: ${requiredPermission}. Would you like to enable AIT permission?`;
|
|
1505
|
+
setMessages(prev => [
|
|
1506
|
+
...prev,
|
|
1507
|
+
{
|
|
1508
|
+
id: Date.now().toString(),
|
|
1509
|
+
content: permissionMsg,
|
|
1510
|
+
role: 'assistant',
|
|
1511
|
+
timestamp: new Date().toISOString(),
|
|
1512
|
+
button: 'enableAIT',
|
|
1513
|
+
metadata: { requiredPermission }
|
|
1514
|
+
}
|
|
1515
|
+
]);
|
|
1516
|
+
return false;
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1519
|
+
return true;
|
|
1520
|
+
};
|
|
1521
|
+
|
|
1522
|
+
// AI Model related functions
|
|
1523
|
+
const handleModelChange = React.useCallback((modelIndex: number) => {
|
|
1524
|
+
setSelectedModelIndex(modelIndex);
|
|
1525
|
+
const selectedModel = effectiveAvailableModels[modelIndex];
|
|
1526
|
+
onModelChange?.(selectedModel);
|
|
1527
|
+
}, [effectiveAvailableModels, onModelChange, setSelectedModelIndex]);
|
|
1528
|
+
|
|
1529
|
+
const getCurrentModel = React.useCallback(() => {
|
|
1530
|
+
// Safety check: ensure selectedModelIndex is within bounds
|
|
1531
|
+
let safeIndex = selectedModelIndex;
|
|
1532
|
+
|
|
1533
|
+
if (selectedModelIndex >= effectiveAvailableModels.length) {
|
|
1534
|
+
safeIndex = 0;
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
// If no models available, return a default model
|
|
1538
|
+
if (effectiveAvailableModels.length === 0) {
|
|
1539
|
+
return { label: 'Unknown', value: 'unknown' };
|
|
1540
|
+
}
|
|
1541
|
+
|
|
1542
|
+
return effectiveAvailableModels[safeIndex];
|
|
1543
|
+
}, [effectiveAvailableModels, selectedModelIndex]);
|
|
1544
|
+
|
|
1545
|
+
const updateSuggestions = React.useCallback(async (pseudoId: string, externalId?: string) => {
|
|
1546
|
+
const result = await nxtlinqApi.agent.generateSuggestions({
|
|
1547
|
+
apiKey,
|
|
1548
|
+
apiSecret,
|
|
1549
|
+
pseudoId,
|
|
1550
|
+
externalId
|
|
1551
|
+
});
|
|
1552
|
+
|
|
1553
|
+
if ('error' in result) {
|
|
1554
|
+
console.error('Failed to fetch suggestions:', result.error);
|
|
1555
|
+
setSuggestions([]);
|
|
1556
|
+
return;
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
setSuggestions(result.suggestions.map((sug: string) => ({
|
|
1560
|
+
text: sug,
|
|
1561
|
+
autoSend: true
|
|
1562
|
+
})));
|
|
1563
|
+
}, [])
|
|
1564
|
+
|
|
1565
|
+
// Updated sendMessage function to support different AI models and attachments
|
|
1566
|
+
const sendMessage = async (
|
|
1567
|
+
content: string,
|
|
1568
|
+
retryCount = 0,
|
|
1569
|
+
isPresetMessage = false,
|
|
1570
|
+
attachments?: Attachment[],
|
|
1571
|
+
clientPipelineOverride?: Array<{ name: string; durationMs: number }>
|
|
1572
|
+
): Promise<void> => {
|
|
1573
|
+
const hasContent = content.trim() || (attachments && attachments.length > 0);
|
|
1574
|
+
if (!hasContent || isLoading) return;
|
|
1575
|
+
|
|
1576
|
+
const currentModel = getCurrentModel();
|
|
1577
|
+
// Initialize with current model, will be updated with actual model from backend response
|
|
1578
|
+
let actualModelUsed = currentModel.value;
|
|
1579
|
+
|
|
1580
|
+
// Only add user message on first attempt, not on retries
|
|
1581
|
+
// Also check if this is not a preset message (preset messages are added separately)
|
|
1582
|
+
if (retryCount === 0 && !isPresetMessage) {
|
|
1583
|
+
const userMessage: Message = {
|
|
1584
|
+
id: Date.now().toString(),
|
|
1585
|
+
content: content || (attachments && attachments.length > 0 ? `Uploaded ${attachments.length} file(s)` : ''),
|
|
1586
|
+
role: 'user',
|
|
1587
|
+
timestamp: new Date().toISOString(),
|
|
1588
|
+
attachments: attachments,
|
|
1589
|
+
metadata: {
|
|
1590
|
+
model: currentModel.value,
|
|
1591
|
+
permissions: permissions,
|
|
1592
|
+
issuedBy: hitAddress || ''
|
|
1593
|
+
},
|
|
1594
|
+
piiStatus: piiDisplayMode === 'redacted' ? 'scanning' : undefined,
|
|
1595
|
+
};
|
|
1596
|
+
setMessages(prev => [...prev, userMessage]);
|
|
1597
|
+
setInputValue('');
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
setIsLoading(true);
|
|
1601
|
+
|
|
1602
|
+
let streamAssistantId: string | null = null;
|
|
1603
|
+
|
|
1604
|
+
try {
|
|
1605
|
+
// Prepare attachments for API
|
|
1606
|
+
const apiAttachments = attachments?.map(att => ({
|
|
1607
|
+
type: att.type,
|
|
1608
|
+
url: att.url,
|
|
1609
|
+
name: att.name,
|
|
1610
|
+
mimeType: att.mimeType
|
|
1611
|
+
}));
|
|
1612
|
+
|
|
1613
|
+
let clientPipelineTimings: Array<{ name: string; durationMs: number }> | undefined;
|
|
1614
|
+
if (retryCount === 0) {
|
|
1615
|
+
if (clientPipelineOverride?.length) {
|
|
1616
|
+
clientPipelineTimings = clientPipelineOverride;
|
|
1617
|
+
} else if (micSessionStartForLogRef.current != null) {
|
|
1618
|
+
clientPipelineTimings = [
|
|
1619
|
+
{
|
|
1620
|
+
name: 'speech_to_text',
|
|
1621
|
+
durationMs: Math.round(performance.now() - micSessionStartForLogRef.current),
|
|
1622
|
+
},
|
|
1623
|
+
];
|
|
1624
|
+
micSessionStartForLogRef.current = null;
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
const isAgentStreamModel = currentModel.value.endsWith('-stream');
|
|
1629
|
+
let streamedReplyBuffer = '';
|
|
1630
|
+
if (isAgentStreamModel && retryCount === 0) {
|
|
1631
|
+
const sid = `stream-assistant-${Date.now()}`;
|
|
1632
|
+
streamAssistantId = sid;
|
|
1633
|
+
setMessages(prev => [...prev, {
|
|
1634
|
+
id: sid,
|
|
1635
|
+
role: 'assistant',
|
|
1636
|
+
content: '',
|
|
1637
|
+
partialContent: '',
|
|
1638
|
+
isStreaming: true,
|
|
1639
|
+
timestamp: new Date().toISOString(),
|
|
1640
|
+
metadata: {
|
|
1641
|
+
model: currentModel.value,
|
|
1642
|
+
agentLlmStream: true,
|
|
1643
|
+
permissions: permissions,
|
|
1644
|
+
issuedBy: hitAddress || ''
|
|
1645
|
+
}
|
|
1646
|
+
}]);
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
const response: AgentResponse = await nxtlinqApi.agent.sendMessage({
|
|
1650
|
+
model: currentModel.value,
|
|
1651
|
+
apiKey,
|
|
1652
|
+
apiSecret,
|
|
1653
|
+
pseudoId: pseudoId,
|
|
1654
|
+
externalId: localStorage.getItem('walletAddress') || undefined,
|
|
1655
|
+
customUserInfo,
|
|
1656
|
+
customUsername,
|
|
1657
|
+
message: content || (attachments && attachments.length > 0 ? `Uploaded ${attachments.length} file(s)` : ''),
|
|
1658
|
+
attachments: apiAttachments,
|
|
1659
|
+
isRetry: retryCount > 0,
|
|
1660
|
+
...(clientPipelineTimings?.length ? { clientPipelineTimings } : {}),
|
|
1661
|
+
onStreamDelta: streamAssistantId
|
|
1662
|
+
? (text: string) => {
|
|
1663
|
+
streamedReplyBuffer += text;
|
|
1664
|
+
flushSync(() => {
|
|
1665
|
+
setMessages(prev =>
|
|
1666
|
+
prev.map(m =>
|
|
1667
|
+
m.id === streamAssistantId
|
|
1668
|
+
? { ...m, partialContent: (m.partialContent || '') + text }
|
|
1669
|
+
: m
|
|
1670
|
+
)
|
|
1671
|
+
);
|
|
1672
|
+
});
|
|
1673
|
+
if (text) {
|
|
1674
|
+
setIsLoading(false);
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1677
|
+
: undefined,
|
|
1678
|
+
onPiiProgress: piiDisplayMode === 'redacted' ? (step: 'scan_start' | 'scan_complete' | 'send_start' | 'done', data?: any) => {
|
|
1679
|
+
const stepMap: Record<string, 0 | 1 | 2> = {
|
|
1680
|
+
scan_start: 0, // Scan active
|
|
1681
|
+
scan_complete: 1, // {N} protected — data available, show redacted
|
|
1682
|
+
send_start: 2, // Sending to AI
|
|
1683
|
+
};
|
|
1684
|
+
const piiStep = stepMap[step];
|
|
1685
|
+
if (piiStep === undefined) return;
|
|
1686
|
+
setMessages(prev => {
|
|
1687
|
+
const updated = [...prev];
|
|
1688
|
+
for (let i = updated.length - 1; i >= 0; i--) {
|
|
1689
|
+
if (updated[i].role === 'user' && updated[i].piiStatus === 'scanning') {
|
|
1690
|
+
const patch: Partial<typeof updated[0]> = { piiStep };
|
|
1691
|
+
// scan_complete carries anonymized data — set piiProtection early for immediate redact
|
|
1692
|
+
if (step === 'scan_complete' && data?.anonymizedUserMessage) {
|
|
1693
|
+
patch.piiProtection = {
|
|
1694
|
+
anonymizedContent: data.anonymizedUserMessage,
|
|
1695
|
+
mapping: data.mapping || {},
|
|
1696
|
+
};
|
|
1697
|
+
}
|
|
1698
|
+
updated[i] = { ...updated[i], ...patch };
|
|
1699
|
+
break;
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
return updated;
|
|
1703
|
+
});
|
|
1704
|
+
} : undefined,
|
|
1705
|
+
});
|
|
1706
|
+
|
|
1707
|
+
if (!('error' in response && (response as { error?: string }).error)) {
|
|
1708
|
+
const tr = response as AgentResponse;
|
|
1709
|
+
if (
|
|
1710
|
+
tr.ttsVoice &&
|
|
1711
|
+
typeof tr.ttsVoice === 'object' &&
|
|
1712
|
+
(tr.ttsVoice.provider === 'azure' || tr.ttsVoice.provider === 'openai')
|
|
1713
|
+
) {
|
|
1714
|
+
clientTtsVoiceRef.current = tr.ttsVoice;
|
|
1715
|
+
} else {
|
|
1716
|
+
clientTtsVoiceRef.current = undefined;
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
// Get the actual model used from response (may differ due to fallback)
|
|
1721
|
+
actualModelUsed = (response as any).model || currentModel.value;
|
|
1722
|
+
|
|
1723
|
+
if ('error' in response || (response as any).result === 'error') {
|
|
1724
|
+
// Check if it's an AIT-related error
|
|
1725
|
+
const errorResponse = response as any;
|
|
1726
|
+
if (errorResponse.requiresWalletConnection) {
|
|
1727
|
+
// Case where wallet connection is required - trigger auto-connect logic
|
|
1728
|
+
// Try to auto-connect wallet first
|
|
1729
|
+
const autoConnectResult = await connectWallet(false); // Don't show sign-in message yet
|
|
1730
|
+
|
|
1731
|
+
if (autoConnectResult) {
|
|
1732
|
+
// If auto-connect successful, try to sign in and then retry
|
|
1733
|
+
try {
|
|
1734
|
+
// Try to sign in wallet
|
|
1735
|
+
await signInWallet(false); // Don't show success message yet
|
|
1736
|
+
|
|
1737
|
+
if (isSemiAutomaticMode) {
|
|
1738
|
+
setIsLoading(false);
|
|
1739
|
+
setMessages(prev => [...prev, {
|
|
1740
|
+
id: Date.now().toString(),
|
|
1741
|
+
content: 'Click button to continue using tool',
|
|
1742
|
+
role: 'assistant',
|
|
1743
|
+
timestamp: new Date().toISOString(),
|
|
1744
|
+
button: 'continue'
|
|
1745
|
+
}]);
|
|
1746
|
+
return
|
|
1747
|
+
} else {
|
|
1748
|
+
// Wait a bit for everything to settle
|
|
1749
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1750
|
+
|
|
1751
|
+
// Retry the message
|
|
1752
|
+
sendMessage(content, retryCount + 1, isPresetMessage);
|
|
1753
|
+
}
|
|
1754
|
+
} catch (signInError) {
|
|
1755
|
+
setIsLoading(false);
|
|
1756
|
+
setMessages(prev => [...prev, {
|
|
1757
|
+
id: Date.now().toString(),
|
|
1758
|
+
content: 'Wallet connected but sign-in failed. Please try signing in manually.',
|
|
1759
|
+
role: 'assistant',
|
|
1760
|
+
timestamp: new Date().toISOString(),
|
|
1761
|
+
button: 'signIn',
|
|
1762
|
+
metadata: {
|
|
1763
|
+
requiresWalletConnection: true,
|
|
1764
|
+
toolName: errorResponse.toolName
|
|
1765
|
+
}
|
|
1766
|
+
}]);
|
|
1767
|
+
}
|
|
1768
|
+
return;
|
|
1769
|
+
} else {
|
|
1770
|
+
// If auto-connect failed, show the error message with connect button
|
|
1771
|
+
setIsLoading(false);
|
|
1772
|
+
setMessages(prev => [...prev, {
|
|
1773
|
+
id: Date.now().toString(),
|
|
1774
|
+
content: normalizeErrorForUser(errorResponse.message),
|
|
1775
|
+
role: 'assistant',
|
|
1776
|
+
timestamp: new Date().toISOString(),
|
|
1777
|
+
button: 'connectWallet',
|
|
1778
|
+
metadata: {
|
|
1779
|
+
requiresWalletConnection: true,
|
|
1780
|
+
toolName: errorResponse.toolName
|
|
1781
|
+
}
|
|
1782
|
+
}]);
|
|
1783
|
+
return;
|
|
1784
|
+
}
|
|
1785
|
+
} else if (errorResponse.requiresPermissionUpdate) {
|
|
1786
|
+
// Case where permission update is required - show manual enable option
|
|
1787
|
+
setIsLoading(false);
|
|
1788
|
+
setMessages(prev => [...prev, {
|
|
1789
|
+
id: Date.now().toString(),
|
|
1790
|
+
content: normalizeErrorForUser(errorResponse.message),
|
|
1791
|
+
role: 'assistant',
|
|
1792
|
+
timestamp: new Date().toISOString(),
|
|
1793
|
+
button: 'enableAIT',
|
|
1794
|
+
metadata: {
|
|
1795
|
+
requiresPermissionUpdate: true,
|
|
1796
|
+
toolName: errorResponse.toolName,
|
|
1797
|
+
requiredPermission: errorResponse.requiredPermission
|
|
1798
|
+
}
|
|
1799
|
+
}]);
|
|
1800
|
+
return;
|
|
1801
|
+
} else {
|
|
1802
|
+
// Other errors (show user-friendly message for invalid/expired token)
|
|
1803
|
+
throw new Error(normalizeErrorForUser(errorResponse.error || errorResponse.message));
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
|
|
1807
|
+
let botResponse: Message | undefined;
|
|
1808
|
+
|
|
1809
|
+
let handledOpenAiStreamAssistant = false;
|
|
1810
|
+
|
|
1811
|
+
if (streamAssistantId && response.toolCall?.toolUse) {
|
|
1812
|
+
setMessages(prev => prev.filter(m => m.id !== streamAssistantId));
|
|
1813
|
+
streamAssistantId = null;
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
if (streamAssistantId && !response.toolCall?.toolUse) {
|
|
1817
|
+
handledOpenAiStreamAssistant = true;
|
|
1818
|
+
let replyText = '';
|
|
1819
|
+
if (response.reply) {
|
|
1820
|
+
replyText = Array.isArray(response.reply)
|
|
1821
|
+
? response.reply
|
|
1822
|
+
.map((item: any) => item.text)
|
|
1823
|
+
.join(' ') || ''
|
|
1824
|
+
: response.reply || '';
|
|
1825
|
+
} else {
|
|
1826
|
+
replyText = response.fullResponse?.output?.message?.content
|
|
1827
|
+
?.find((item: any) => item.text)?.text || '';
|
|
1828
|
+
}
|
|
1829
|
+
if (!replyText.trim() && streamedReplyBuffer.trim()) {
|
|
1830
|
+
replyText = streamedReplyBuffer.trim();
|
|
1831
|
+
}
|
|
1832
|
+
if (!replyText.trim()) {
|
|
1833
|
+
replyText = 'Sorry, I cannot understand your question';
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
updateSuggestions(pseudoId, localStorage.getItem('walletAddress') || undefined);
|
|
1837
|
+
|
|
1838
|
+
setMessages(prev => prev.map(m =>
|
|
1839
|
+
m.id === streamAssistantId
|
|
1840
|
+
? {
|
|
1841
|
+
...m,
|
|
1842
|
+
content: replyText,
|
|
1843
|
+
isStreaming: false,
|
|
1844
|
+
partialContent: undefined,
|
|
1845
|
+
metadata: {
|
|
1846
|
+
model: actualModelUsed,
|
|
1847
|
+
agentLlmStream: true,
|
|
1848
|
+
permissions: permissions,
|
|
1849
|
+
issuedBy: hitAddress || ''
|
|
1850
|
+
},
|
|
1851
|
+
piiProtection: response.piiProtection?.anonymizedReply
|
|
1852
|
+
? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
|
|
1853
|
+
: undefined,
|
|
1854
|
+
}
|
|
1855
|
+
: m
|
|
1856
|
+
));
|
|
1857
|
+
|
|
1858
|
+
botResponse = {
|
|
1859
|
+
id: streamAssistantId,
|
|
1860
|
+
content: replyText,
|
|
1861
|
+
role: 'assistant',
|
|
1862
|
+
timestamp: new Date().toISOString(),
|
|
1863
|
+
metadata: {
|
|
1864
|
+
model: actualModelUsed,
|
|
1865
|
+
agentLlmStream: true,
|
|
1866
|
+
permissions: permissions,
|
|
1867
|
+
issuedBy: hitAddress || ''
|
|
1868
|
+
},
|
|
1869
|
+
piiProtection: response.piiProtection?.anonymizedReply
|
|
1870
|
+
? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
|
|
1871
|
+
: undefined,
|
|
1872
|
+
};
|
|
1873
|
+
}
|
|
1874
|
+
|
|
1875
|
+
// Store redirectUrl for later use (don't redirect yet)
|
|
1876
|
+
const redirectUrl = response?.redirectUrl;
|
|
1877
|
+
|
|
1878
|
+
if (!handledOpenAiStreamAssistant && response.toolCall?.toolUse) {
|
|
1879
|
+
// Handle tool calls
|
|
1880
|
+
const toolUse = response.toolCall.toolUse;
|
|
1881
|
+
|
|
1882
|
+
let toolMsg = '';
|
|
1883
|
+
if (onToolUse) {
|
|
1884
|
+
let wasAutoConnected = false;
|
|
1885
|
+
let wasAutoSignedIn = false;
|
|
1886
|
+
// Added: Mark if permission denied due to missing AIT permission
|
|
1887
|
+
let permissionDenied = false;
|
|
1888
|
+
// Use requiredPermission from response if available, otherwise fall back to toolUse.name
|
|
1889
|
+
const permissionToCheck = (response as any)?.requiredPermission || toolUse.name;
|
|
1890
|
+
|
|
1891
|
+
const isToolAllowed = await hasPermission(
|
|
1892
|
+
permissionToCheck,
|
|
1893
|
+
true,
|
|
1894
|
+
() => { wasAutoConnected = true },
|
|
1895
|
+
() => { wasAutoSignedIn = true }
|
|
1896
|
+
);
|
|
1897
|
+
|
|
1898
|
+
// If currentPermissions does not include permissionToCheck and availablePermissionLabels includes permissionToCheck, it means AIT permission is missing
|
|
1899
|
+
if (!isToolAllowed && !permissions.includes(permissionToCheck) && availablePermissions.map(p => p.label).includes(permissionToCheck)) {
|
|
1900
|
+
permissionDenied = true;
|
|
1901
|
+
}
|
|
1902
|
+
if (!isToolAllowed) {
|
|
1903
|
+
// If permission denied due to missing AIT permission return
|
|
1904
|
+
if (permissionDenied) {
|
|
1905
|
+
setIsLoading(false);
|
|
1906
|
+
return;
|
|
1907
|
+
}
|
|
1908
|
+
|
|
1909
|
+
if (isSemiAutomaticMode) {
|
|
1910
|
+
setIsLoading(false);
|
|
1911
|
+
setMessages(prev => [...prev, {
|
|
1912
|
+
id: Date.now().toString(),
|
|
1913
|
+
content: 'Click button to continue using tool',
|
|
1914
|
+
role: 'assistant',
|
|
1915
|
+
timestamp: new Date().toISOString(),
|
|
1916
|
+
button: 'continue'
|
|
1917
|
+
}]);
|
|
1918
|
+
return;
|
|
1919
|
+
} else {
|
|
1920
|
+
// Only retry for auto-connect/auto-sign-in scenarios
|
|
1921
|
+
if (wasAutoConnected && retryCount < 1) {
|
|
1922
|
+
// Clear loading state and retry immediately
|
|
1923
|
+
setIsLoading(false);
|
|
1924
|
+
|
|
1925
|
+
// Check if wallet is already signed in
|
|
1926
|
+
const currentToken = nxtlinqAITServiceAccessTokenRef.current;
|
|
1927
|
+
|
|
1928
|
+
if (!currentToken) {
|
|
1929
|
+
// If not signed in, directly retry the message without waiting for AIT
|
|
1930
|
+
setTimeout(() => {
|
|
1931
|
+
sendMessage(content, retryCount + 1, isPresetMessage);
|
|
1932
|
+
}, 2000);
|
|
1933
|
+
} else {
|
|
1934
|
+
// If already signed in, wait for AIT to be fully loaded before retrying
|
|
1935
|
+
setTimeout(async () => {
|
|
1936
|
+
// Wait for AIT to be loaded if needed
|
|
1937
|
+
if (!aitRef.current && nxtlinqAITServiceAccessTokenRef.current) {
|
|
1938
|
+
await refreshAIT();
|
|
1939
|
+
}
|
|
1940
|
+
|
|
1941
|
+
// Wait for AIT to be fully loaded with polling
|
|
1942
|
+
let attempts = 0;
|
|
1943
|
+
const maxAttempts = 5; // Wait up to 10 seconds (5 * 2000ms)
|
|
1944
|
+
while (!aitRef.current && attempts < maxAttempts) {
|
|
1945
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1946
|
+
attempts++;
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
// Only retry if AIT is actually loaded
|
|
1950
|
+
if (aitRef.current) {
|
|
1951
|
+
// Wait a bit more to ensure permissions are also loaded
|
|
1952
|
+
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
1953
|
+
sendMessage(content, retryCount + 1, isPresetMessage);
|
|
1954
|
+
}
|
|
1955
|
+
}, 2000);
|
|
1956
|
+
}
|
|
1957
|
+
}
|
|
1958
|
+
return;
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
if (isSemiAutomaticMode && wasAutoSignedIn) {
|
|
1963
|
+
setIsLoading(false);
|
|
1964
|
+
setMessages(prev => [...prev, {
|
|
1965
|
+
id: Date.now().toString(),
|
|
1966
|
+
content: 'Click button to continue using tool',
|
|
1967
|
+
role: 'assistant',
|
|
1968
|
+
timestamp: new Date().toISOString(),
|
|
1969
|
+
button: 'continue'
|
|
1970
|
+
}]);
|
|
1971
|
+
return;
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
// Create streaming message for tool execution
|
|
1975
|
+
const streamingMessageId = `streaming-${Date.now()}`;
|
|
1976
|
+
const streamingMessage: Message = {
|
|
1977
|
+
id: streamingMessageId,
|
|
1978
|
+
content: '',
|
|
1979
|
+
role: 'assistant',
|
|
1980
|
+
timestamp: new Date().toISOString(),
|
|
1981
|
+
isStreaming: true,
|
|
1982
|
+
streamingToolName: toolUse.name,
|
|
1983
|
+
streamingStatus: `Starting ${toolUse.name}...`,
|
|
1984
|
+
streamingProgress: 0,
|
|
1985
|
+
streamingSteps: [],
|
|
1986
|
+
metadata: {
|
|
1987
|
+
model: actualModelUsed,
|
|
1988
|
+
permissions: permissions,
|
|
1989
|
+
issuedBy: hitAddress || '',
|
|
1990
|
+
toolUse: toolUse
|
|
1991
|
+
}
|
|
1992
|
+
};
|
|
1993
|
+
|
|
1994
|
+
// Add streaming message to chat
|
|
1995
|
+
setMessages(prev => [...prev, streamingMessage]);
|
|
1996
|
+
|
|
1997
|
+
// Execute tool with streaming updates
|
|
1998
|
+
const toolResult = await onToolUse(toolUse, (update) => {
|
|
1999
|
+
// Update streaming message with progress
|
|
2000
|
+
setMessages(prev => prev.map(msg =>
|
|
2001
|
+
msg.id === streamingMessageId
|
|
2002
|
+
? {
|
|
2003
|
+
...msg,
|
|
2004
|
+
content: update.partialResult || msg.content,
|
|
2005
|
+
partialContent: update.partialContent, // Token-level streaming
|
|
2006
|
+
streamingProgress: update.progress,
|
|
2007
|
+
streamingStatus: update.status,
|
|
2008
|
+
streamingSteps: update.steps || msg.streamingSteps
|
|
2009
|
+
}
|
|
2010
|
+
: msg
|
|
2011
|
+
));
|
|
2012
|
+
});
|
|
2013
|
+
|
|
2014
|
+
toolMsg = toolResult?.content || `Tool ${toolUse.name} executed successfully`;
|
|
2015
|
+
|
|
2016
|
+
// Merge AI original reply
|
|
2017
|
+
let replyText = '';
|
|
2018
|
+
if (response.reply) {
|
|
2019
|
+
replyText = Array.isArray(response.reply)
|
|
2020
|
+
? response.reply
|
|
2021
|
+
.map((item: any) => item.text)
|
|
2022
|
+
.join(' ') || ''
|
|
2023
|
+
: response.reply || '';
|
|
2024
|
+
} else if (response.fullResponse?.output?.message?.content) {
|
|
2025
|
+
replyText = response.fullResponse.output.message.content
|
|
2026
|
+
?.find((item: any) => item.text)?.text || '';
|
|
2027
|
+
}
|
|
2028
|
+
|
|
2029
|
+
let finalContent = toolMsg;
|
|
2030
|
+
if (replyText && replyText.trim()) {
|
|
2031
|
+
finalContent += `\n\n${replyText}`;
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
// Mark streaming as complete with final content
|
|
2035
|
+
setMessages(prev => prev.map(msg =>
|
|
2036
|
+
msg.id === streamingMessageId
|
|
2037
|
+
? {
|
|
2038
|
+
...msg,
|
|
2039
|
+
isStreaming: false,
|
|
2040
|
+
streamingProgress: 100,
|
|
2041
|
+
content: finalContent,
|
|
2042
|
+
metadata: {
|
|
2043
|
+
...msg.metadata,
|
|
2044
|
+
model: actualModelUsed,
|
|
2045
|
+
toolUse: toolUse
|
|
2046
|
+
},
|
|
2047
|
+
piiProtection: response.piiProtection?.anonymizedReply
|
|
2048
|
+
? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
|
|
2049
|
+
: undefined,
|
|
2050
|
+
}
|
|
2051
|
+
: msg
|
|
2052
|
+
));
|
|
2053
|
+
|
|
2054
|
+
// Update message content in database if assistantMessageId is available
|
|
2055
|
+
if (response.assistantMessageId && apiKey && apiSecret) {
|
|
2056
|
+
try {
|
|
2057
|
+
await nxtlinqApi.agent.updateMessageContent({
|
|
2058
|
+
apiKey,
|
|
2059
|
+
apiSecret,
|
|
2060
|
+
messageId: response.assistantMessageId,
|
|
2061
|
+
content: finalContent,
|
|
2062
|
+
});
|
|
2063
|
+
} catch (error) {
|
|
2064
|
+
console.error('Failed to update message content in database:', error);
|
|
2065
|
+
// Don't block the UI if update fails
|
|
2066
|
+
}
|
|
2067
|
+
}
|
|
2068
|
+
|
|
2069
|
+
updateSuggestions(pseudoId, localStorage.getItem('walletAddress') || undefined);
|
|
2070
|
+
|
|
2071
|
+
// Skip creating a new botResponse since we already updated the streaming message
|
|
2072
|
+
} else {
|
|
2073
|
+
toolMsg = `Tool ${toolUse.name} executed successfully`;
|
|
2074
|
+
|
|
2075
|
+
// No onToolUse callback - create message without streaming
|
|
2076
|
+
let replyText = '';
|
|
2077
|
+
if (response.reply) {
|
|
2078
|
+
replyText = Array.isArray(response.reply)
|
|
2079
|
+
? response.reply
|
|
2080
|
+
.map((item: any) => item.text)
|
|
2081
|
+
.join(' ') || ''
|
|
2082
|
+
: response.reply || '';
|
|
2083
|
+
} else if (response.fullResponse?.output?.message?.content) {
|
|
2084
|
+
replyText = response.fullResponse.output.message.content
|
|
2085
|
+
?.find((item: any) => item.text)?.text || '';
|
|
2086
|
+
}
|
|
2087
|
+
let mergedContent = '';
|
|
2088
|
+
if (toolMsg) {
|
|
2089
|
+
mergedContent += toolMsg;
|
|
2090
|
+
}
|
|
2091
|
+
if (replyText && replyText.trim()) {
|
|
2092
|
+
mergedContent += `\n\n${replyText}`;
|
|
2093
|
+
}
|
|
2094
|
+
|
|
2095
|
+
// Update message content in database if assistantMessageId is available
|
|
2096
|
+
if (response.assistantMessageId && apiKey && apiSecret) {
|
|
2097
|
+
try {
|
|
2098
|
+
await nxtlinqApi.agent.updateMessageContent({
|
|
2099
|
+
apiKey,
|
|
2100
|
+
apiSecret,
|
|
2101
|
+
messageId: response.assistantMessageId,
|
|
2102
|
+
content: mergedContent,
|
|
2103
|
+
});
|
|
2104
|
+
} catch (error) {
|
|
2105
|
+
console.error('Failed to update message content in database:', error);
|
|
2106
|
+
// Don't block the UI if update fails
|
|
2107
|
+
}
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2110
|
+
updateSuggestions(pseudoId, localStorage.getItem('walletAddress') || undefined);
|
|
2111
|
+
const newBotResponse: Message = {
|
|
2112
|
+
id: (Date.now() + 1).toString(),
|
|
2113
|
+
content: mergedContent,
|
|
2114
|
+
role: 'assistant',
|
|
2115
|
+
timestamp: new Date().toISOString(),
|
|
2116
|
+
metadata: {
|
|
2117
|
+
model: actualModelUsed,
|
|
2118
|
+
permissions: permissions,
|
|
2119
|
+
issuedBy: hitAddress || '',
|
|
2120
|
+
toolUse: toolUse
|
|
2121
|
+
},
|
|
2122
|
+
piiProtection: response.piiProtection?.anonymizedReply
|
|
2123
|
+
? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
|
|
2124
|
+
: undefined,
|
|
2125
|
+
};
|
|
2126
|
+
setMessages(prev => [...prev, newBotResponse]);
|
|
2127
|
+
botResponse = newBotResponse;
|
|
2128
|
+
}
|
|
2129
|
+
} else if (!handledOpenAiStreamAssistant && response.reply) {
|
|
2130
|
+
const replyText = Array.isArray(response.reply)
|
|
2131
|
+
? response.reply
|
|
2132
|
+
.map((item: any) => item.text)
|
|
2133
|
+
.join(' ') || 'Sorry, I cannot understand your question'
|
|
2134
|
+
: response.reply || 'Sorry, I cannot understand your question';
|
|
2135
|
+
|
|
2136
|
+
updateSuggestions(pseudoId, localStorage.getItem('walletAddress') || undefined);
|
|
2137
|
+
|
|
2138
|
+
const newBotResponse: Message = {
|
|
2139
|
+
id: (Date.now() + 1).toString(),
|
|
2140
|
+
content: replyText,
|
|
2141
|
+
role: 'assistant',
|
|
2142
|
+
timestamp: new Date().toISOString(),
|
|
2143
|
+
metadata: {
|
|
2144
|
+
model: actualModelUsed,
|
|
2145
|
+
permissions: permissions,
|
|
2146
|
+
issuedBy: hitAddress || ''
|
|
2147
|
+
},
|
|
2148
|
+
piiProtection: response.piiProtection?.anonymizedReply
|
|
2149
|
+
? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
|
|
2150
|
+
: undefined,
|
|
2151
|
+
};
|
|
2152
|
+
setMessages(prev => [...prev, newBotResponse]);
|
|
2153
|
+
botResponse = newBotResponse;
|
|
2154
|
+
} else if (!handledOpenAiStreamAssistant) {
|
|
2155
|
+
const replyText = response.fullResponse?.output?.message?.content
|
|
2156
|
+
?.find((item: any) => item.text)?.text || 'Sorry, I cannot understand your question';
|
|
2157
|
+
|
|
2158
|
+
const newBotResponse: Message = {
|
|
2159
|
+
id: (Date.now() + 1).toString(),
|
|
2160
|
+
content: replyText,
|
|
2161
|
+
role: 'assistant',
|
|
2162
|
+
timestamp: new Date().toISOString(),
|
|
2163
|
+
metadata: {
|
|
2164
|
+
model: actualModelUsed,
|
|
2165
|
+
permissions: permissions,
|
|
2166
|
+
issuedBy: hitAddress || ''
|
|
2167
|
+
},
|
|
2168
|
+
piiProtection: response.piiProtection?.anonymizedReply
|
|
2169
|
+
? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
|
|
2170
|
+
: undefined,
|
|
2171
|
+
};
|
|
2172
|
+
setMessages(prev => [...prev, newBotResponse]);
|
|
2173
|
+
botResponse = newBotResponse;
|
|
2174
|
+
}
|
|
2175
|
+
|
|
2176
|
+
// ===== PII Protection: Update user message with anonymized version =====
|
|
2177
|
+
const anonymizedUserMsg = response.piiProtection?.anonymizedUserMessage;
|
|
2178
|
+
const piiMappingData = response.piiProtection?.mapping ?? undefined;
|
|
2179
|
+
if (anonymizedUserMsg) {
|
|
2180
|
+
setMessages(prev => {
|
|
2181
|
+
const updated = [...prev];
|
|
2182
|
+
for (let i = updated.length - 1; i >= 0; i--) {
|
|
2183
|
+
if (updated[i].role === 'user') {
|
|
2184
|
+
updated[i] = {
|
|
2185
|
+
...updated[i],
|
|
2186
|
+
piiProtection: { anonymizedContent: anonymizedUserMsg, mapping: piiMappingData },
|
|
2187
|
+
piiStatus: 'complete',
|
|
2188
|
+
};
|
|
2189
|
+
break;
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
2192
|
+
return updated;
|
|
2193
|
+
});
|
|
2194
|
+
} else if (piiDisplayMode === 'redacted') {
|
|
2195
|
+
// No PII detected — set piiStatus to 'none' to trigger "No sensitive data" indicator
|
|
2196
|
+
setMessages(prev => {
|
|
2197
|
+
const updated = [...prev];
|
|
2198
|
+
for (let i = updated.length - 1; i >= 0; i--) {
|
|
2199
|
+
if (updated[i].role === 'user' && updated[i].piiStatus === 'scanning') {
|
|
2200
|
+
updated[i] = { ...updated[i], piiStatus: 'none' };
|
|
2201
|
+
break;
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
return updated;
|
|
2205
|
+
});
|
|
2206
|
+
}
|
|
2207
|
+
|
|
2208
|
+
// Execute redirect after all message processing is complete
|
|
2209
|
+
if (redirectUrl) {
|
|
2210
|
+
// Use setTimeout to ensure the message is displayed before redirect
|
|
2211
|
+
setTimeout(() => {
|
|
2212
|
+
window.location.href = redirectUrl;
|
|
2213
|
+
}, 100);
|
|
2214
|
+
}
|
|
2215
|
+
|
|
2216
|
+
// Play text-to-speech for AI response if enabled
|
|
2217
|
+
if (botResponse && botResponse.role === 'assistant' && textToSpeechEnabled) {
|
|
2218
|
+
const ttsContent = botResponse.content;
|
|
2219
|
+
setTimeout(() => {
|
|
2220
|
+
playTextToSpeech(ttsContent).catch(error => {
|
|
2221
|
+
console.error('Failed to play text-to-speech:', error);
|
|
2222
|
+
});
|
|
2223
|
+
}, 100);
|
|
2224
|
+
}
|
|
2225
|
+
} catch (error) {
|
|
2226
|
+
console.error('Failed to send message:', error);
|
|
2227
|
+
|
|
2228
|
+
if (streamAssistantId) {
|
|
2229
|
+
setMessages(prev => prev.filter(m => m.id !== streamAssistantId));
|
|
2230
|
+
streamAssistantId = null;
|
|
2231
|
+
}
|
|
2232
|
+
|
|
2233
|
+
if (retryCount < maxRetries) {
|
|
2234
|
+
// Show retry indicator to user
|
|
2235
|
+
const retryMessage: Message = {
|
|
2236
|
+
id: `retry-${Date.now()}-${retryCount}`,
|
|
2237
|
+
content: `Retrying... (Attempt ${retryCount + 1}/${maxRetries})`,
|
|
2238
|
+
role: 'assistant',
|
|
2239
|
+
timestamp: new Date().toISOString(),
|
|
2240
|
+
metadata: {
|
|
2241
|
+
model: actualModelUsed,
|
|
2242
|
+
permissions: permissions,
|
|
2243
|
+
issuedBy: hitAddress || '',
|
|
2244
|
+
isRetry: true
|
|
2245
|
+
}
|
|
2246
|
+
};
|
|
2247
|
+
setMessages(prev => [...prev, retryMessage]);
|
|
2248
|
+
|
|
2249
|
+
setTimeout(() => {
|
|
2250
|
+
// Remove retry message before next attempt
|
|
2251
|
+
setMessages(prev => prev.filter(msg => msg.id !== retryMessage.id));
|
|
2252
|
+
sendMessage(content, retryCount + 1, isPresetMessage);
|
|
2253
|
+
}, retryDelay);
|
|
2254
|
+
return;
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
// Clear piiStatus on error so stepper doesn't get stuck
|
|
2258
|
+
if (piiDisplayMode === 'redacted') {
|
|
2259
|
+
setMessages(prev => {
|
|
2260
|
+
const updated = [...prev];
|
|
2261
|
+
for (let i = updated.length - 1; i >= 0; i--) {
|
|
2262
|
+
if (updated[i].role === 'user' && updated[i].piiStatus === 'scanning') {
|
|
2263
|
+
updated[i] = { ...updated[i], piiStatus: undefined };
|
|
2264
|
+
break;
|
|
2265
|
+
}
|
|
2266
|
+
}
|
|
2267
|
+
return updated;
|
|
2268
|
+
});
|
|
2269
|
+
}
|
|
2270
|
+
|
|
2271
|
+
const rawMessage = error instanceof Error ? error.message : 'An error occurred while sending the message';
|
|
2272
|
+
const errorMessage: Message = {
|
|
2273
|
+
id: (Date.now() + 1).toString(),
|
|
2274
|
+
content: normalizeErrorForUser(rawMessage),
|
|
2275
|
+
role: 'assistant',
|
|
2276
|
+
timestamp: new Date().toISOString(),
|
|
2277
|
+
metadata: {
|
|
2278
|
+
model: currentModel.value,
|
|
2279
|
+
permissions: permissions,
|
|
2280
|
+
issuedBy: hitAddress || ''
|
|
2281
|
+
}
|
|
2282
|
+
};
|
|
2283
|
+
setMessages(prev => [...prev, errorMessage]);
|
|
2284
|
+
onError?.(error instanceof Error ? error : new Error('Unknown error'));
|
|
2285
|
+
} finally {
|
|
2286
|
+
setIsLoading(false);
|
|
2287
|
+
}
|
|
2288
|
+
};
|
|
2289
|
+
|
|
2290
|
+
// Handle submit
|
|
2291
|
+
const handleSubmit = async (e: React.FormEvent, attachments?: Attachment[]) => {
|
|
2292
|
+
e.preventDefault();
|
|
2293
|
+
|
|
2294
|
+
if (isStopRecordingOnSend) {
|
|
2295
|
+
while (isMicEnabledRef.current) {
|
|
2296
|
+
stopRecording();
|
|
2297
|
+
await sleep(1000);
|
|
2298
|
+
}
|
|
2299
|
+
} else {
|
|
2300
|
+
if (isMicEnabledRef.current) {
|
|
2301
|
+
await sleep(1000);
|
|
2302
|
+
clearRecording();
|
|
2303
|
+
}
|
|
2304
|
+
}
|
|
2305
|
+
|
|
2306
|
+
if (!textInputRef.current) return;
|
|
2307
|
+
|
|
2308
|
+
const hasContent = textInputRef.current.value.trim() || (attachments && attachments.length > 0);
|
|
2309
|
+
if (!hasContent || isLoading) return;
|
|
2310
|
+
|
|
2311
|
+
// 如果此時有語音自動發送的計時器,優先取消,避免與手動發送同內容時重複送出
|
|
2312
|
+
if (autoSendTimerRef.current) {
|
|
2313
|
+
clearTimeout(autoSendTimerRef.current as NodeJS.Timeout);
|
|
2314
|
+
autoSendTimerRef.current = null;
|
|
2315
|
+
}
|
|
2316
|
+
|
|
2317
|
+
// 標記這次手動送出的內容,讓語音自動發送邏輯視為已送出,避免同一段文字再自動發送一次
|
|
2318
|
+
const manualContent = textInputRef.current.value.trim();
|
|
2319
|
+
if (manualContent) {
|
|
2320
|
+
lastAutoSentTranscriptRef.current = manualContent;
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2323
|
+
// Stop current speech playback when user sends new message
|
|
2324
|
+
stopTextToSpeechAndReset();
|
|
2325
|
+
|
|
2326
|
+
// Show loading message if AIT is still loading
|
|
2327
|
+
if (isAITLoading) {
|
|
2328
|
+
const loadingMessage: Message = {
|
|
2329
|
+
id: Date.now().toString(),
|
|
2330
|
+
content: 'Loading your wallet configuration... Please wait a moment.',
|
|
2331
|
+
role: 'assistant',
|
|
2332
|
+
timestamp: new Date().toISOString(),
|
|
2333
|
+
metadata: {
|
|
2334
|
+
model: getCurrentModel().value,
|
|
2335
|
+
permissions: permissions,
|
|
2336
|
+
issuedBy: hitAddress || ''
|
|
2337
|
+
}
|
|
2338
|
+
};
|
|
2339
|
+
setMessages(prev => [...prev, loadingMessage]);
|
|
2340
|
+
return;
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
try {
|
|
2344
|
+
const clientPipelineOverride =
|
|
2345
|
+
micSessionStartForLogRef.current != null
|
|
2346
|
+
? [
|
|
2347
|
+
{
|
|
2348
|
+
name: 'speech_to_text',
|
|
2349
|
+
durationMs: Math.round(performance.now() - micSessionStartForLogRef.current),
|
|
2350
|
+
},
|
|
2351
|
+
]
|
|
2352
|
+
: undefined;
|
|
2353
|
+
if (micSessionStartForLogRef.current != null) {
|
|
2354
|
+
micSessionStartForLogRef.current = null;
|
|
2355
|
+
}
|
|
2356
|
+
await sendMessage(textInputRef.current.value, 0, false, attachments, clientPipelineOverride);
|
|
2357
|
+
} catch (error) {
|
|
2358
|
+
console.error('Failed to send message:', error);
|
|
2359
|
+
onError?.(error instanceof Error ? error : new Error('Failed to send message'));
|
|
2360
|
+
const errorMessage: Message = {
|
|
2361
|
+
id: Date.now().toString(),
|
|
2362
|
+
content: 'Sorry, there was an error processing your message. Please try again.',
|
|
2363
|
+
role: 'assistant',
|
|
2364
|
+
timestamp: new Date().toISOString(),
|
|
2365
|
+
metadata: {
|
|
2366
|
+
model: getCurrentModel().value,
|
|
2367
|
+
permissions: permissions,
|
|
2368
|
+
issuedBy: hitAddress || ''
|
|
2369
|
+
}
|
|
2370
|
+
};
|
|
2371
|
+
setMessages(prev => [...prev, errorMessage]);
|
|
2372
|
+
}
|
|
2373
|
+
};
|
|
2374
|
+
|
|
2375
|
+
// Stop text-to-speech (does not clear queue)
|
|
2376
|
+
const stopTextToSpeech = React.useCallback(() => {
|
|
2377
|
+
if (speechingRef.current) {
|
|
2378
|
+
// Stop AudioContext source if exists
|
|
2379
|
+
if (audioSourceRef.current) {
|
|
2380
|
+
audioSourceRef.current.stop();
|
|
2381
|
+
audioSourceRef.current.disconnect();
|
|
2382
|
+
audioSourceRef.current = null;
|
|
2383
|
+
}
|
|
2384
|
+
// Stop HTML Audio element if exists
|
|
2385
|
+
if (audioElementRef.current) {
|
|
2386
|
+
audioElementRef.current.pause();
|
|
2387
|
+
audioElementRef.current.currentTime = 0;
|
|
2388
|
+
if (audioElementRef.current.src.startsWith('blob:')) {
|
|
2389
|
+
URL.revokeObjectURL(audioElementRef.current.src);
|
|
2390
|
+
}
|
|
2391
|
+
audioElementRef.current = null;
|
|
2392
|
+
}
|
|
2393
|
+
speechingRef.current = false;
|
|
2394
|
+
}
|
|
2395
|
+
setSpeechingIndex(undefined);
|
|
2396
|
+
}, []);
|
|
2397
|
+
|
|
2398
|
+
// Stop text-to-speech (used when user sends new message)
|
|
2399
|
+
const stopTextToSpeechAndReset = React.useCallback(() => {
|
|
2400
|
+
stopTextToSpeech();
|
|
2401
|
+
setRequiresGesture(false);
|
|
2402
|
+
setIsTtsProcessing(false);
|
|
2403
|
+
pendingTtsRef.current = null;
|
|
2404
|
+
}, [stopTextToSpeech]);
|
|
2405
|
+
|
|
2406
|
+
// When TTS is disabled during playback, stop immediately
|
|
2407
|
+
React.useEffect(() => {
|
|
2408
|
+
if (!textToSpeechEnabled) {
|
|
2409
|
+
stopTextToSpeechAndReset();
|
|
2410
|
+
}
|
|
2411
|
+
}, [textToSpeechEnabled, stopTextToSpeechAndReset]);
|
|
2412
|
+
|
|
2413
|
+
const {
|
|
2414
|
+
isVoiceMode,
|
|
2415
|
+
voiceStatus,
|
|
2416
|
+
isVoiceConnecting,
|
|
2417
|
+
isMicMuted,
|
|
2418
|
+
remoteAudioRef,
|
|
2419
|
+
enterVoiceMode,
|
|
2420
|
+
exitVoiceMode,
|
|
2421
|
+
toggleMicMute: toggleVoiceMicMute,
|
|
2422
|
+
interruptVoice,
|
|
2423
|
+
} = useVoiceMode({
|
|
2424
|
+
apiKey,
|
|
2425
|
+
apiSecret,
|
|
2426
|
+
pseudoId,
|
|
2427
|
+
externalId: hitAddress || undefined,
|
|
2428
|
+
aitId: ait?.aitId,
|
|
2429
|
+
walletAddress: hitAddress || undefined,
|
|
2430
|
+
aitToken: nxtlinqAITServiceAccessToken || undefined,
|
|
2431
|
+
metadata: customUserInfo,
|
|
2432
|
+
nxtlinqApi,
|
|
2433
|
+
getMessages: () => messagesRef.current,
|
|
2434
|
+
setMessages,
|
|
2435
|
+
onError,
|
|
2436
|
+
stopRecording,
|
|
2437
|
+
stopTextToSpeech: stopTextToSpeechAndReset,
|
|
2438
|
+
voiceTransport: 'ws-realtime',
|
|
2439
|
+
});
|
|
2440
|
+
|
|
2441
|
+
React.useEffect(() => {
|
|
2442
|
+
if (!isOpen && isVoiceMode) {
|
|
2443
|
+
void exitVoiceMode();
|
|
2444
|
+
}
|
|
2445
|
+
}, [isOpen, isVoiceMode, exitVoiceMode]);
|
|
2446
|
+
|
|
2447
|
+
React.useEffect(() => {
|
|
2448
|
+
if (isOpen && preferredChatMode === 'voice' && !isVoiceMode && !isVoiceConnecting) {
|
|
2449
|
+
void enterVoiceMode();
|
|
2450
|
+
}
|
|
2451
|
+
}, [isOpen, preferredChatMode, isVoiceMode, isVoiceConnecting, enterVoiceMode]);
|
|
2452
|
+
|
|
2453
|
+
React.useEffect(() => {
|
|
2454
|
+
if (isVoiceMode || isVoiceConnecting) {
|
|
2455
|
+
stopRecording();
|
|
2456
|
+
}
|
|
2457
|
+
}, [isVoiceMode, isVoiceConnecting, stopRecording]);
|
|
2458
|
+
|
|
2459
|
+
const handleEnterVoiceMode = React.useCallback(async () => {
|
|
2460
|
+
stopRecording();
|
|
2461
|
+
setPreferredChatMode('voice');
|
|
2462
|
+
await enterVoiceMode();
|
|
2463
|
+
}, [enterVoiceMode, setPreferredChatMode, stopRecording]);
|
|
2464
|
+
|
|
2465
|
+
const handleExitVoiceMode = React.useCallback(async () => {
|
|
2466
|
+
setPreferredChatMode('text');
|
|
2467
|
+
await exitVoiceMode();
|
|
2468
|
+
}, [exitVoiceMode, setPreferredChatMode]);
|
|
2469
|
+
|
|
2470
|
+
// Play text-to-speech (simplified, no queue)
|
|
2471
|
+
const playTextToSpeech = React.useCallback(async (text: string, messageIndex?: number): Promise<void> => {
|
|
2472
|
+
if (!textToSpeechEnabled || !text.trim()) return;
|
|
2473
|
+
|
|
2474
|
+
// Stop any ongoing speech
|
|
2475
|
+
stopTextToSpeech();
|
|
2476
|
+
|
|
2477
|
+
// Show processing indicator
|
|
2478
|
+
setIsTtsProcessing(true);
|
|
2479
|
+
|
|
2480
|
+
// Retry up to 2 times (3 attempts total)
|
|
2481
|
+
let lastError: any;
|
|
2482
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
2483
|
+
try {
|
|
2484
|
+
await playTextToSpeechWithRetry(text, messageIndex, attempt);
|
|
2485
|
+
return; // Success, exit
|
|
2486
|
+
} catch (err) {
|
|
2487
|
+
lastError = err;
|
|
2488
|
+
if (attempt < 2) {
|
|
2489
|
+
// Continue to next attempt
|
|
2490
|
+
continue;
|
|
2491
|
+
}
|
|
2492
|
+
}
|
|
2493
|
+
}
|
|
2494
|
+
|
|
2495
|
+
// All attempts failed
|
|
2496
|
+
console.error('TTS playback failed after retries:', lastError);
|
|
2497
|
+
setIsTtsProcessing(false);
|
|
2498
|
+
}, [textToSpeechEnabled, stopTextToSpeech, playTextToSpeechWithRetry]);
|
|
2499
|
+
|
|
2500
|
+
const retryTtsWithGesture = React.useCallback(async () => {
|
|
2501
|
+
try {
|
|
2502
|
+
setRequiresGesture(false);
|
|
2503
|
+
// Try to resume audio context
|
|
2504
|
+
if (audioCtxRef.current) {
|
|
2505
|
+
try { await audioCtxRef.current.resume(); } catch { }
|
|
2506
|
+
}
|
|
2507
|
+
const pending = pendingTtsRef.current;
|
|
2508
|
+
if (pending && textToSpeechEnabled) {
|
|
2509
|
+
setIsTtsProcessing(true);
|
|
2510
|
+
await playTextToSpeech(pending);
|
|
2511
|
+
pendingTtsRef.current = null;
|
|
2512
|
+
}
|
|
2513
|
+
} catch (err) {
|
|
2514
|
+
console.error('Retry TTS with gesture failed:', err);
|
|
2515
|
+
setIsTtsProcessing(false);
|
|
2516
|
+
}
|
|
2517
|
+
}, [textToSpeechEnabled, playTextToSpeech]);
|
|
2518
|
+
|
|
2519
|
+
// When browser blocks autoplay (requires gesture), retry on next user interaction
|
|
2520
|
+
React.useEffect(() => {
|
|
2521
|
+
if (!requiresGesture || !textToSpeechEnabled || !pendingTtsRef.current) return;
|
|
2522
|
+
const handleUserGesture = () => {
|
|
2523
|
+
retryTtsWithGesture();
|
|
2524
|
+
};
|
|
2525
|
+
window.addEventListener('pointerdown', handleUserGesture, { once: true });
|
|
2526
|
+
return () => window.removeEventListener('pointerdown', handleUserGesture);
|
|
2527
|
+
}, [requiresGesture, textToSpeechEnabled, retryTtsWithGesture]);
|
|
2528
|
+
|
|
2529
|
+
const uploadAttachment = React.useCallback(async (file: File): Promise<{ url: string } | { error: string }> => {
|
|
2530
|
+
const result = await nxtlinqApi.agent.uploadAttachment({
|
|
2531
|
+
apiKey,
|
|
2532
|
+
apiSecret,
|
|
2533
|
+
pseudoId,
|
|
2534
|
+
file,
|
|
2535
|
+
});
|
|
2536
|
+
if ('error' in result) return result;
|
|
2537
|
+
return { url: result.url };
|
|
2538
|
+
}, [nxtlinqApi, apiKey, apiSecret, pseudoId]);
|
|
2539
|
+
|
|
2540
|
+
// Handle preset message
|
|
2541
|
+
const handlePresetMessage = (message: PresetMessage) => {
|
|
2542
|
+
// If preset is configured as auto-send, avoid duplicate sends when user clicks repeatedly
|
|
2543
|
+
if (message.autoSend) {
|
|
2544
|
+
const trimmedText = (message.text || '').trim();
|
|
2545
|
+
if (!trimmedText) return;
|
|
2546
|
+
|
|
2547
|
+
// Prevent sending messages while AI Agent is processing
|
|
2548
|
+
if (isLoading) {
|
|
2549
|
+
return;
|
|
2550
|
+
}
|
|
2551
|
+
|
|
2552
|
+
// If this exact preset text was just sent (by auto-send / manual / preset), skip to prevent duplicates
|
|
2553
|
+
if (lastAutoSentTranscriptRef.current === trimmedText) {
|
|
2554
|
+
return;
|
|
2555
|
+
}
|
|
2556
|
+
|
|
2557
|
+
// For preset messages, we need to add the user message first since sendMessage won't add it on retries
|
|
2558
|
+
const userMessage: Message = {
|
|
2559
|
+
id: Date.now().toString(),
|
|
2560
|
+
content: trimmedText,
|
|
2561
|
+
role: 'user',
|
|
2562
|
+
timestamp: new Date().toISOString(),
|
|
2563
|
+
metadata: {
|
|
2564
|
+
model: getCurrentModel().value,
|
|
2565
|
+
permissions: permissions,
|
|
2566
|
+
issuedBy: hitAddress || ''
|
|
2567
|
+
}
|
|
2568
|
+
};
|
|
2569
|
+
setMessages(prev => [...prev, userMessage]);
|
|
2570
|
+
|
|
2571
|
+
// Mark as last sent to guard against rapid re-clicks and other duplicate flows
|
|
2572
|
+
lastAutoSentTranscriptRef.current = trimmedText;
|
|
2573
|
+
|
|
2574
|
+
// Pass a flag to indicate this is a preset message so sendMessage won't add user message again
|
|
2575
|
+
sendMessage(trimmedText, 0, true);
|
|
2576
|
+
} else {
|
|
2577
|
+
setInputValue(message.text);
|
|
2578
|
+
}
|
|
2579
|
+
};
|
|
2580
|
+
|
|
2581
|
+
// Generate and register AIT
|
|
2582
|
+
const generateAndRegisterAIT = async (newPermissions?: string[], isFromAIAgent = false) => {
|
|
2583
|
+
let currentSigner = signer;
|
|
2584
|
+
let currentAddress = hitAddress;
|
|
2585
|
+
|
|
2586
|
+
// If we have an address but no signer, try to auto-connect wallet
|
|
2587
|
+
if (currentAddress && !currentSigner) {
|
|
2588
|
+
const storedWalletAddress = localStorage.getItem('walletAddress');
|
|
2589
|
+
if (storedWalletAddress === currentAddress) {
|
|
2590
|
+
try {
|
|
2591
|
+
const autoConnectResult = await connectWallet(false); // Don't show sign-in message yet
|
|
2592
|
+
if (autoConnectResult) {
|
|
2593
|
+
// Wait for state to update
|
|
2594
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
2595
|
+
// Get the updated signer and address
|
|
2596
|
+
currentSigner = signerRef.current || signer;
|
|
2597
|
+
currentAddress = hitAddressRef.current || hitAddress;
|
|
2598
|
+
}
|
|
2599
|
+
} catch (error) {
|
|
2600
|
+
console.error('Auto-connect failed during AIT generation:', error);
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2603
|
+
}
|
|
2604
|
+
|
|
2605
|
+
if (!currentSigner || !currentAddress) {
|
|
2606
|
+
throw new Error('Missing signer or wallet address');
|
|
2607
|
+
}
|
|
2608
|
+
|
|
2609
|
+
return generateAndRegisterAITWithSigner(newPermissions, isFromAIAgent, currentSigner, currentAddress);
|
|
2610
|
+
};
|
|
2611
|
+
|
|
2612
|
+
// Generate and register AIT with explicit signer and address
|
|
2613
|
+
const generateAndRegisterAITWithSigner = async (
|
|
2614
|
+
newPermissions?: string[],
|
|
2615
|
+
isFromAIAgent = false,
|
|
2616
|
+
explicitSigner?: ethers.JsonRpcSigner,
|
|
2617
|
+
explicitAddress?: string
|
|
2618
|
+
) => {
|
|
2619
|
+
const currentSigner = explicitSigner || signer;
|
|
2620
|
+
const currentAddress = explicitAddress || hitAddress;
|
|
2621
|
+
|
|
2622
|
+
if (!currentSigner || !currentAddress) {
|
|
2623
|
+
throw new Error('Missing signer or wallet address');
|
|
2624
|
+
}
|
|
2625
|
+
|
|
2626
|
+
// If AI Agent is creating, must check if user has existing AIT
|
|
2627
|
+
if (isFromAIAgent && !aitRef.current) {
|
|
2628
|
+
throw new Error('You must have an existing AIT before AI Agent can create new AITs');
|
|
2629
|
+
}
|
|
2630
|
+
|
|
2631
|
+
const timestamp = Math.floor(Date.now() / 1000);
|
|
2632
|
+
const aitId = `did:polygon:nxtlinq:${currentAddress}:${timestamp}`;
|
|
2633
|
+
|
|
2634
|
+
const metadata = {
|
|
2635
|
+
permissions: newPermissions || permissions,
|
|
2636
|
+
issuedBy: currentAddress,
|
|
2637
|
+
};
|
|
2638
|
+
|
|
2639
|
+
const metadataStr = stringify(metadata);
|
|
2640
|
+
// ethers v6: utils.keccak256 -> keccak256, utils.toUtf8Bytes -> toUtf8Bytes
|
|
2641
|
+
const metadataHash = ethers.keccak256(ethers.toUtf8Bytes(metadataStr));
|
|
2642
|
+
|
|
2643
|
+
const uploadResponse = await nxtlinqApi.metadata.createMetadata(metadata, nxtlinqAITServiceAccessToken || '');
|
|
2644
|
+
if ('error' in uploadResponse) {
|
|
2645
|
+
throw new Error(`Failed to upload metadata: ${uploadResponse.error}`);
|
|
2646
|
+
}
|
|
2647
|
+
|
|
2648
|
+
const { metadataCid } = uploadResponse;
|
|
2649
|
+
|
|
2650
|
+
const createAITParams = {
|
|
2651
|
+
aitId,
|
|
2652
|
+
controller: currentAddress,
|
|
2653
|
+
serviceId,
|
|
2654
|
+
metadataHash,
|
|
2655
|
+
metadataCid,
|
|
2656
|
+
isFromAIAgent: isFromAIAgent,
|
|
2657
|
+
parentAITId: isFromAIAgent ? aitRef.current?.aitId : undefined,
|
|
2658
|
+
};
|
|
2659
|
+
|
|
2660
|
+
const createAITResponse = await nxtlinqApi.ait.createAIT(createAITParams, nxtlinqAITServiceAccessToken || '');
|
|
2661
|
+
|
|
2662
|
+
if ('error' in createAITResponse) {
|
|
2663
|
+
throw new Error(`Failed to create AIT: ${createAITResponse.error}`);
|
|
2664
|
+
}
|
|
2665
|
+
|
|
2666
|
+
const aitInfo = {
|
|
2667
|
+
aitId,
|
|
2668
|
+
controller: currentAddress,
|
|
2669
|
+
metadata,
|
|
2670
|
+
metadataHash,
|
|
2671
|
+
metadataCid,
|
|
2672
|
+
};
|
|
2673
|
+
|
|
2674
|
+
setAit(aitInfo);
|
|
2675
|
+
setPermissions(newPermissions || permissions);
|
|
2676
|
+
};
|
|
2677
|
+
|
|
2678
|
+
|
|
2679
|
+
|
|
2680
|
+
// Auto enable AIT permission
|
|
2681
|
+
const enableAIT = async (toolName: string) => {
|
|
2682
|
+
if (isAITEnabling) return false; // Prevent duplicate
|
|
2683
|
+
setIsAITEnabling(true);
|
|
2684
|
+
|
|
2685
|
+
let currentSigner = signer;
|
|
2686
|
+
let currentHitAddress = hitAddress;
|
|
2687
|
+
|
|
2688
|
+
// If no wallet connected, try to auto-connect first
|
|
2689
|
+
if (!currentSigner || !currentHitAddress) {
|
|
2690
|
+
const autoConnectResult = await connectWallet(false); // Don't show sign-in message yet
|
|
2691
|
+
|
|
2692
|
+
if (autoConnectResult) {
|
|
2693
|
+
// Wait for state to update - increase wait time
|
|
2694
|
+
await new Promise(resolve => setTimeout(resolve, 1500));
|
|
2695
|
+
|
|
2696
|
+
// Get the latest state values after connection
|
|
2697
|
+
currentSigner = signerRef.current || signer;
|
|
2698
|
+
currentHitAddress = hitAddressRef.current || hitAddress;
|
|
2699
|
+
|
|
2700
|
+
// Double check that we have the required state
|
|
2701
|
+
if (!currentSigner || !currentHitAddress) {
|
|
2702
|
+
await new Promise(resolve => setTimeout(resolve, 1500));
|
|
2703
|
+
currentSigner = signerRef.current || signer;
|
|
2704
|
+
currentHitAddress = hitAddressRef.current || hitAddress;
|
|
2705
|
+
}
|
|
2706
|
+
|
|
2707
|
+
try {
|
|
2708
|
+
await signInWallet(false); // Don't show success message yet
|
|
2709
|
+
} catch (signInError) {
|
|
2710
|
+
showError(walletTextUtils.getWalletText('Wallet connected but sign-in failed. Please try signing in manually.', serviceId));
|
|
2711
|
+
setIsAITEnabling(false);
|
|
2712
|
+
return false;
|
|
2713
|
+
}
|
|
2714
|
+
} else {
|
|
2715
|
+
showError(walletTextUtils.getWalletText('Please connect your wallet first', serviceId));
|
|
2716
|
+
setIsAITEnabling(false);
|
|
2717
|
+
return false;
|
|
2718
|
+
}
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2721
|
+
// Final check before proceeding
|
|
2722
|
+
if (!currentSigner || !currentHitAddress) {
|
|
2723
|
+
console.error('Missing signer or wallet address after connection process');
|
|
2724
|
+
console.error('Current signer:', !!currentSigner);
|
|
2725
|
+
console.error('Current hitAddress:', currentHitAddress);
|
|
2726
|
+
console.error('Signer ref:', !!signerRef.current);
|
|
2727
|
+
console.error('HitAddress ref:', hitAddressRef.current);
|
|
2728
|
+
showError(walletTextUtils.getWalletText('Wallet connection failed. Please try connecting your wallet manually.', serviceId));
|
|
2729
|
+
setIsAITEnabling(false);
|
|
2730
|
+
return false;
|
|
2731
|
+
}
|
|
2732
|
+
|
|
2733
|
+
// If we don't have AIT loaded but we have a token, try to refresh it first
|
|
2734
|
+
if (!aitRef.current && nxtlinqAITServiceAccessToken) {
|
|
2735
|
+
try {
|
|
2736
|
+
await refreshAIT();
|
|
2737
|
+
// Wait a bit for AIT to be loaded
|
|
2738
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
2739
|
+
} catch (error) {
|
|
2740
|
+
// Continue anyway, we'll create a new AIT if needed
|
|
2741
|
+
console.error('Failed to refresh AIT:', error);
|
|
2742
|
+
}
|
|
2743
|
+
}
|
|
2744
|
+
|
|
2745
|
+
try {
|
|
2746
|
+
// Get available permissions to find the tool
|
|
2747
|
+
const availablePermissionLabels = availablePermissions.map(p => p.label);
|
|
2748
|
+
if (!availablePermissionLabels.includes(toolName)) {
|
|
2749
|
+
showError(`Tool ${toolName} is not available for your current identity provider`);
|
|
2750
|
+
setIsAITEnabling(false);
|
|
2751
|
+
return false;
|
|
2752
|
+
}
|
|
2753
|
+
|
|
2754
|
+
// Get current permissions from AIT metadata instead of React state
|
|
2755
|
+
// This ensures we don't lose existing permissions when enabling new ones
|
|
2756
|
+
const currentAITPermissions = aitRef.current?.metadata?.permissions || permissions;
|
|
2757
|
+
const newPermissions = [...currentAITPermissions];
|
|
2758
|
+
if (!newPermissions.includes(toolName)) {
|
|
2759
|
+
newPermissions.push(toolName);
|
|
2760
|
+
}
|
|
2761
|
+
|
|
2762
|
+
// Filter out deleted permissions before saving
|
|
2763
|
+
const validPermissions = newPermissions.filter(permission =>
|
|
2764
|
+
availablePermissionLabels.includes(permission)
|
|
2765
|
+
);
|
|
2766
|
+
|
|
2767
|
+
// Generate and register AIT with new permissions
|
|
2768
|
+
// For auto-enable, we should create a regular AIT (not AI Agent AIT) if user doesn't have existing AIT
|
|
2769
|
+
const shouldCreateAsAIAgent = !!aitRef.current; // Only create as AI Agent if user already has an AIT
|
|
2770
|
+
|
|
2771
|
+
await generateAndRegisterAITWithSigner(validPermissions, shouldCreateAsAIAgent, currentSigner, currentHitAddress);
|
|
2772
|
+
showSuccess('AIT permission enabled successfully! You can now use the AI agent.');
|
|
2773
|
+
await refreshAIT(true);
|
|
2774
|
+
setIsAITEnabling(false);
|
|
2775
|
+
return true;
|
|
2776
|
+
} catch (error) {
|
|
2777
|
+
console.error('Failed to auto-enable AIT:', error);
|
|
2778
|
+
setIsAITEnabling(false);
|
|
2779
|
+
if (error instanceof Error) {
|
|
2780
|
+
showError(error.message);
|
|
2781
|
+
} else {
|
|
2782
|
+
showError('Failed to enable AIT permission. Please try again.');
|
|
2783
|
+
}
|
|
2784
|
+
return false;
|
|
2785
|
+
}
|
|
2786
|
+
};
|
|
2787
|
+
|
|
2788
|
+
// Save permissions
|
|
2789
|
+
const savePermissions = async (newPermissions?: string[]) => {
|
|
2790
|
+
setIsDisabled(true);
|
|
2791
|
+
try {
|
|
2792
|
+
// Filter out deleted permissions before saving
|
|
2793
|
+
const permissionsToSave = newPermissions || permissions;
|
|
2794
|
+
const availablePermissionLabels = availablePermissions.map(p => p.label);
|
|
2795
|
+
const validPermissions = permissionsToSave.filter(permission =>
|
|
2796
|
+
availablePermissionLabels.includes(permission)
|
|
2797
|
+
);
|
|
2798
|
+
|
|
2799
|
+
await generateAndRegisterAIT(validPermissions, false);
|
|
2800
|
+
showSuccess('AIT permissions saved successfully! You can now use the AI agent with your configured permissions.');
|
|
2801
|
+
setShowPermissionForm(false);
|
|
2802
|
+
setIsPermissionFormOpen(false);
|
|
2803
|
+
await refreshAIT(true);
|
|
2804
|
+
} catch (error) {
|
|
2805
|
+
console.error('Failed to generate AIT:', error);
|
|
2806
|
+
setIsDisabled(false);
|
|
2807
|
+
if (error instanceof Error) {
|
|
2808
|
+
showError(error.message);
|
|
2809
|
+
} else {
|
|
2810
|
+
showError('Failed to save permissions. Please try again.');
|
|
2811
|
+
}
|
|
2812
|
+
}
|
|
2813
|
+
};
|
|
2814
|
+
|
|
2815
|
+
// Handle verify wallet click
|
|
2816
|
+
const handleVerifyWalletClick = async (method: 'berifyme' | 'custom') => {
|
|
2817
|
+
if (!hitAddress) {
|
|
2818
|
+
showError('Please connect your wallet first.');
|
|
2819
|
+
return;
|
|
2820
|
+
}
|
|
2821
|
+
|
|
2822
|
+
try {
|
|
2823
|
+
setIsLoading(true);
|
|
2824
|
+
|
|
2825
|
+
if (method === 'berifyme') {
|
|
2826
|
+
if (!onVerifyWallet) {
|
|
2827
|
+
showError('Berify.me verification is not available. Please provide onVerifyWallet callback.');
|
|
2828
|
+
setIsLoading(false);
|
|
2829
|
+
return;
|
|
2830
|
+
}
|
|
2831
|
+
|
|
2832
|
+
const result = await onVerifyWallet();
|
|
2833
|
+
if (!result) {
|
|
2834
|
+
setIsLoading(false);
|
|
2835
|
+
return;
|
|
2836
|
+
}
|
|
2837
|
+
const { token } = result;
|
|
2838
|
+
const address = hitAddress;
|
|
2839
|
+
if (token && address) {
|
|
2840
|
+
const payload = {
|
|
2841
|
+
address: hitAddress,
|
|
2842
|
+
token,
|
|
2843
|
+
timestamp: Date.now(),
|
|
2844
|
+
method: 'berifyme'
|
|
2845
|
+
};
|
|
2846
|
+
try {
|
|
2847
|
+
const verifyWalletResponse = await nxtlinqApi.wallet.verifyWallet({ ...payload }, token);
|
|
2848
|
+
if ('error' in verifyWalletResponse) {
|
|
2849
|
+
if (verifyWalletResponse.error === 'Wallet already exists') {
|
|
2850
|
+
// Wallet already exists with the same method, just refresh AIT
|
|
2851
|
+
setIsWalletLoading(true);
|
|
2852
|
+
try {
|
|
2853
|
+
const walletResponse = await nxtlinqApi.wallet.getWallet({ address }, token);
|
|
2854
|
+
if (!('error' in walletResponse)) {
|
|
2855
|
+
setWalletInfo(walletResponse);
|
|
2856
|
+
const aitResponse = await nxtlinqApi.ait.getAITByServiceIdAndController({
|
|
2857
|
+
serviceId,
|
|
2858
|
+
controller: address,
|
|
2859
|
+
customUsername: (!requireWalletIDVVerification && customUsername) ? getFinalCustomUsername(customUsername) : undefined
|
|
2860
|
+
}, token);
|
|
2861
|
+
if (!('error' in aitResponse)) {
|
|
2862
|
+
setAit(aitResponse);
|
|
2863
|
+
}
|
|
2864
|
+
}
|
|
2865
|
+
} finally {
|
|
2866
|
+
setIsWalletLoading(false);
|
|
2867
|
+
}
|
|
2868
|
+
if (typeof window !== 'undefined') {
|
|
2869
|
+
try {
|
|
2870
|
+
const url = new URL(window.location.href);
|
|
2871
|
+
url.searchParams.delete('token');
|
|
2872
|
+
url.searchParams.delete('isAutoConnect');
|
|
2873
|
+
url.searchParams.delete('method');
|
|
2874
|
+
url.searchParams.delete('returnUrl');
|
|
2875
|
+
window.history.replaceState({}, '', url.toString());
|
|
2876
|
+
} catch (e) {
|
|
2877
|
+
console.error('Failed to clean URL:', e);
|
|
2878
|
+
}
|
|
2879
|
+
}
|
|
2880
|
+
setIsLoading(false);
|
|
2881
|
+
showSuccess(walletTextUtils.getWalletText('Wallet verification completed successfully! Your wallet is now verified and ready to use.', serviceId));
|
|
2882
|
+
refreshAIT();
|
|
2883
|
+
return;
|
|
2884
|
+
}
|
|
2885
|
+
showError(verifyWalletResponse.error);
|
|
2886
|
+
setIsLoading(false);
|
|
2887
|
+
return;
|
|
2888
|
+
}
|
|
2889
|
+
setIsWalletLoading(true);
|
|
2890
|
+
try {
|
|
2891
|
+
const walletResponse = await nxtlinqApi.wallet.getWallet({ address }, token);
|
|
2892
|
+
if (!('error' in walletResponse)) {
|
|
2893
|
+
setWalletInfo(walletResponse);
|
|
2894
|
+
const aitResponse = await nxtlinqApi.ait.getAITByServiceIdAndController({
|
|
2895
|
+
serviceId,
|
|
2896
|
+
controller: address,
|
|
2897
|
+
customUsername: (!requireWalletIDVVerification && customUsername) ? getFinalCustomUsername(customUsername) : undefined
|
|
2898
|
+
}, token);
|
|
2899
|
+
if (!('error' in aitResponse)) {
|
|
2900
|
+
setAit(aitResponse);
|
|
2901
|
+
}
|
|
2902
|
+
}
|
|
2903
|
+
} finally {
|
|
2904
|
+
setIsWalletLoading(false);
|
|
2905
|
+
}
|
|
2906
|
+
if (typeof window !== 'undefined') {
|
|
2907
|
+
try {
|
|
2908
|
+
const url = new URL(window.location.href);
|
|
2909
|
+
url.searchParams.delete('token');
|
|
2910
|
+
url.searchParams.delete('isAutoConnect');
|
|
2911
|
+
url.searchParams.delete('method');
|
|
2912
|
+
url.searchParams.delete('returnUrl');
|
|
2913
|
+
window.history.replaceState({}, '', url.toString());
|
|
2914
|
+
} catch (e) {
|
|
2915
|
+
console.error('Failed to clean URL:', e);
|
|
2916
|
+
}
|
|
2917
|
+
}
|
|
2918
|
+
setIsLoading(false);
|
|
2919
|
+
showSuccess('Wallet verification completed successfully! Your wallet is now verified and ready to use.');
|
|
2920
|
+
refreshAIT();
|
|
2921
|
+
return;
|
|
2922
|
+
} catch (error) {
|
|
2923
|
+
let msg = 'Verification failed';
|
|
2924
|
+
if (typeof error === 'object' && error !== null && 'response' in error) {
|
|
2925
|
+
msg = (error as any).response?.data?.error || (error as unknown as Error).message || msg;
|
|
2926
|
+
} else if (error instanceof Error) {
|
|
2927
|
+
msg = error.message;
|
|
2928
|
+
}
|
|
2929
|
+
console.error('Wallet verification failed:', error);
|
|
2930
|
+
showError(msg);
|
|
2931
|
+
setIsLoading(false);
|
|
2932
|
+
throw error;
|
|
2933
|
+
}
|
|
2934
|
+
}
|
|
2935
|
+
setIsLoading(false);
|
|
2936
|
+
return;
|
|
2937
|
+
}
|
|
2938
|
+
|
|
2939
|
+
if (method === 'custom') {
|
|
2940
|
+
if (!customUsername) {
|
|
2941
|
+
showError('Custom username is required for custom verification method.');
|
|
2942
|
+
setIsLoading(false);
|
|
2943
|
+
return;
|
|
2944
|
+
}
|
|
2945
|
+
|
|
2946
|
+
const payload = {
|
|
2947
|
+
address: hitAddress,
|
|
2948
|
+
method: 'custom' as const,
|
|
2949
|
+
timestamp: Date.now(),
|
|
2950
|
+
customUsername: getFinalCustomUsername(customUsername)
|
|
2951
|
+
};
|
|
2952
|
+
try {
|
|
2953
|
+
const verifyWalletResponse = await nxtlinqApi.wallet.verifyWallet(payload, '');
|
|
2954
|
+
if ('error' in verifyWalletResponse) {
|
|
2955
|
+
if (verifyWalletResponse.error === 'Wallet already exists') {
|
|
2956
|
+
// Wallet already exists with the same method, just refresh AIT
|
|
2957
|
+
await refreshAIT();
|
|
2958
|
+
setIsLoading(false);
|
|
2959
|
+
showSuccess(walletTextUtils.getWalletText('Wallet verification completed successfully! Your wallet is now verified and ready to use.', serviceId));
|
|
2960
|
+
return;
|
|
2961
|
+
}
|
|
2962
|
+
// Handle specific error messages
|
|
2963
|
+
if (verifyWalletResponse.error === 'Cannot downgrade from Berify.me verification to custom verification') {
|
|
2964
|
+
showError(walletTextUtils.getWalletText('This wallet is already verified with Berify.me. Custom verification cannot override Berify.me verification.', serviceId));
|
|
2965
|
+
} else {
|
|
2966
|
+
showError(verifyWalletResponse.error);
|
|
2967
|
+
}
|
|
2968
|
+
setIsLoading(false);
|
|
2969
|
+
return;
|
|
2970
|
+
}
|
|
2971
|
+
// Successfully created wallet with custom method
|
|
2972
|
+
await refreshAIT();
|
|
2973
|
+
setIsLoading(false);
|
|
2974
|
+
showSuccess(walletTextUtils.getWalletText('Wallet verification completed successfully! Your wallet is now verified and ready to use.', serviceId));
|
|
2975
|
+
return;
|
|
2976
|
+
} catch (error) {
|
|
2977
|
+
console.error('Custom wallet verification failed:', error);
|
|
2978
|
+
showError(walletTextUtils.getWalletText('Failed to verify wallet with custom method. Please try again.', serviceId));
|
|
2979
|
+
setIsLoading(false);
|
|
2980
|
+
return;
|
|
2981
|
+
}
|
|
2982
|
+
}
|
|
2983
|
+
|
|
2984
|
+
// If we reach here, the method is not supported
|
|
2985
|
+
showError(`Unsupported verification method: ${method}`);
|
|
2986
|
+
setIsLoading(false);
|
|
2987
|
+
} catch (error) {
|
|
2988
|
+
console.error('Failed to verify wallet:', error);
|
|
2989
|
+
setIsLoading(false);
|
|
2990
|
+
showError(walletTextUtils.getWalletText('Failed to verify wallet. Please try again.', serviceId));
|
|
2991
|
+
throw error;
|
|
2992
|
+
}
|
|
2993
|
+
};
|
|
2994
|
+
|
|
2995
|
+
// Initialize wallet address from localStorage on component mount
|
|
2996
|
+
React.useEffect(() => {
|
|
2997
|
+
const storedWalletAddress = localStorage.getItem('walletAddress');
|
|
2998
|
+
if (storedWalletAddress && !hitAddress) {
|
|
2999
|
+
setHitAddress(storedWalletAddress);
|
|
3000
|
+
hitAddressRef.current = storedWalletAddress;
|
|
3001
|
+
}
|
|
3002
|
+
}, []);
|
|
3003
|
+
|
|
3004
|
+
React.useEffect(() => {
|
|
3005
|
+
if (hitAddress && nxtlinqAITServiceAccessToken) {
|
|
3006
|
+
refreshAIT();
|
|
3007
|
+
}
|
|
3008
|
+
}, [hitAddress, nxtlinqAITServiceAccessToken]);
|
|
3009
|
+
|
|
3010
|
+
// Set loading state when permission form opens
|
|
3011
|
+
React.useEffect(() => {
|
|
3012
|
+
if (isPermissionFormOpen && hitAddress) {
|
|
3013
|
+
setIsAITLoading(true);
|
|
3014
|
+
}
|
|
3015
|
+
}, [isPermissionFormOpen, hitAddress]);
|
|
3016
|
+
|
|
3017
|
+
React.useEffect(() => {
|
|
3018
|
+
fetchAvailablePermissions();
|
|
3019
|
+
}, [serviceId, permissionGroup]);
|
|
3020
|
+
|
|
3021
|
+
React.useEffect(() => {
|
|
3022
|
+
if (typeof window !== 'undefined') {
|
|
3023
|
+
try {
|
|
3024
|
+
const urlParams = new URLSearchParams(window.location.search);
|
|
3025
|
+
const token = urlParams.get('token');
|
|
3026
|
+
if (token && hitAddress) {
|
|
3027
|
+
handleVerifyWalletClick('berifyme');
|
|
3028
|
+
}
|
|
3029
|
+
} catch (e) {
|
|
3030
|
+
console.error('Failed to get URL params:', e);
|
|
3031
|
+
}
|
|
3032
|
+
}
|
|
3033
|
+
}, [hitAddress]);
|
|
3034
|
+
|
|
3035
|
+
// Handle custom error message display
|
|
3036
|
+
React.useEffect(() => {
|
|
3037
|
+
const trimmedError = customError?.trim();
|
|
3038
|
+
// Only display error if it's a new value (different from last displayed error)
|
|
3039
|
+
if (trimmedError && trimmedError !== lastCustomErrorRef.current) {
|
|
3040
|
+
lastCustomErrorRef.current = trimmedError;
|
|
3041
|
+
const errorMessage: Message = {
|
|
3042
|
+
id: `custom-error-${Date.now()}`,
|
|
3043
|
+
content: trimmedError,
|
|
3044
|
+
role: 'assistant',
|
|
3045
|
+
timestamp: new Date().toISOString(),
|
|
3046
|
+
metadata: {
|
|
3047
|
+
model: getCurrentModel().value,
|
|
3048
|
+
permissions: permissions,
|
|
3049
|
+
issuedBy: hitAddress || ''
|
|
3050
|
+
}
|
|
3051
|
+
};
|
|
3052
|
+
setMessages(prev => [...prev, errorMessage]);
|
|
3053
|
+
onMessage?.(errorMessage);
|
|
3054
|
+
} else if (!trimmedError) {
|
|
3055
|
+
// Reset ref when customError is cleared
|
|
3056
|
+
lastCustomErrorRef.current = undefined;
|
|
3057
|
+
}
|
|
3058
|
+
}, [customError, getCurrentModel, permissions, hitAddress, onMessage]);
|
|
3059
|
+
|
|
3060
|
+
React.useEffect(() => {
|
|
3061
|
+
if (messages.length > 0) {
|
|
3062
|
+
const lastMessage = messages[messages.length - 1];
|
|
3063
|
+
onMessage?.(lastMessage);
|
|
3064
|
+
}
|
|
3065
|
+
}, [messages, onMessage]);
|
|
3066
|
+
|
|
3067
|
+
React.useEffect(() => {
|
|
3068
|
+
if (!hitAddress) {
|
|
3069
|
+
return
|
|
3070
|
+
}
|
|
3071
|
+
|
|
3072
|
+
const updateExternalId = async () => {
|
|
3073
|
+
const externalId = hitAddress;
|
|
3074
|
+
await nxtlinqApi.agent.updateMessagesExternalIdByPseudoId({
|
|
3075
|
+
apiKey,
|
|
3076
|
+
apiSecret,
|
|
3077
|
+
pseudoId,
|
|
3078
|
+
externalId: externalId,
|
|
3079
|
+
})
|
|
3080
|
+
|
|
3081
|
+
await nxtlinqApi.agent.cloneUserProfileByPseudoId({
|
|
3082
|
+
apiKey,
|
|
3083
|
+
apiSecret,
|
|
3084
|
+
pseudoId,
|
|
3085
|
+
externalId: externalId,
|
|
3086
|
+
});
|
|
3087
|
+
|
|
3088
|
+
await nxtlinqApi.agent.cloneUserTopicByPseudoId({
|
|
3089
|
+
apiKey,
|
|
3090
|
+
apiSecret,
|
|
3091
|
+
pseudoId,
|
|
3092
|
+
externalId: externalId,
|
|
3093
|
+
})
|
|
3094
|
+
}
|
|
3095
|
+
|
|
3096
|
+
updateExternalId()
|
|
3097
|
+
}, [hitAddress, nxtlinqApi, apiKey, apiSecret, pseudoId]);
|
|
3098
|
+
|
|
3099
|
+
const contextValue: ChatBotContextType = {
|
|
3100
|
+
// State
|
|
3101
|
+
messages,
|
|
3102
|
+
inputValue,
|
|
3103
|
+
isLoading,
|
|
3104
|
+
isOpen,
|
|
3105
|
+
hitAddress,
|
|
3106
|
+
ait,
|
|
3107
|
+
permissions,
|
|
3108
|
+
availablePermissions,
|
|
3109
|
+
showPermissionForm,
|
|
3110
|
+
isPermissionFormOpen,
|
|
3111
|
+
isAITLoading,
|
|
3112
|
+
isDisabled,
|
|
3113
|
+
walletInfo,
|
|
3114
|
+
isWalletLoading,
|
|
3115
|
+
isAutoConnecting,
|
|
3116
|
+
notification,
|
|
3117
|
+
isMicEnabled,
|
|
3118
|
+
isAwaitingMicGesture,
|
|
3119
|
+
transcript,
|
|
3120
|
+
textInputRef,
|
|
3121
|
+
autoSendEnabled,
|
|
3122
|
+
// Speech related state
|
|
3123
|
+
textToSpeechEnabled,
|
|
3124
|
+
speechingIndex,
|
|
3125
|
+
isTtsProcessing,
|
|
3126
|
+
requiresGesture,
|
|
3127
|
+
// AI Model related state
|
|
3128
|
+
availableModels: effectiveAvailableModels,
|
|
3129
|
+
selectedModelIndex,
|
|
3130
|
+
showModelSelector,
|
|
3131
|
+
suggestions,
|
|
3132
|
+
isAITEnabling,
|
|
3133
|
+
piiDisplayMode,
|
|
3134
|
+
isVoiceMode,
|
|
3135
|
+
voiceStatus,
|
|
3136
|
+
isVoiceConnecting,
|
|
3137
|
+
isMicMuted,
|
|
3138
|
+
remoteAudioRef,
|
|
3139
|
+
|
|
3140
|
+
// Actions
|
|
3141
|
+
setInputValue,
|
|
3142
|
+
setIsOpen,
|
|
3143
|
+
setShowPermissionForm,
|
|
3144
|
+
setIsPermissionFormOpen,
|
|
3145
|
+
setPermissions,
|
|
3146
|
+
setIsDisabled,
|
|
3147
|
+
setIsWalletLoading,
|
|
3148
|
+
setNotification,
|
|
3149
|
+
// AI Model related actions
|
|
3150
|
+
setSelectedModelIndex,
|
|
3151
|
+
setSuggestions,
|
|
3152
|
+
setAutoSendEnabled,
|
|
3153
|
+
setTextToSpeechEnabled,
|
|
3154
|
+
|
|
3155
|
+
// Functions
|
|
3156
|
+
connectWallet,
|
|
3157
|
+
signInWallet,
|
|
3158
|
+
sendMessage,
|
|
3159
|
+
handleSubmit,
|
|
3160
|
+
handlePresetMessage,
|
|
3161
|
+
savePermissions,
|
|
3162
|
+
enableAIT,
|
|
3163
|
+
handleVerifyWalletClick,
|
|
3164
|
+
showSuccess,
|
|
3165
|
+
showError,
|
|
3166
|
+
showWarning,
|
|
3167
|
+
showInfo,
|
|
3168
|
+
refreshAIT,
|
|
3169
|
+
startRecording,
|
|
3170
|
+
stopRecording,
|
|
3171
|
+
clearRecording,
|
|
3172
|
+
// AI Model related functions
|
|
3173
|
+
handleModelChange,
|
|
3174
|
+
getCurrentModel,
|
|
3175
|
+
// Speech related functions
|
|
3176
|
+
playTextToSpeech,
|
|
3177
|
+
stopTextToSpeech,
|
|
3178
|
+
retryTtsWithGesture,
|
|
3179
|
+
uploadAttachment,
|
|
3180
|
+
enterVoiceMode: handleEnterVoiceMode,
|
|
3181
|
+
exitVoiceMode: handleExitVoiceMode,
|
|
3182
|
+
toggleVoiceMicMute,
|
|
3183
|
+
interruptVoice,
|
|
3184
|
+
|
|
3185
|
+
// Additional properties for PermissionForm
|
|
3186
|
+
onSave: savePermissions,
|
|
3187
|
+
onConnectWallet: () => connectWallet(false),
|
|
3188
|
+
onSignIn: () => signInWallet(false),
|
|
3189
|
+
isNeedSignInWithWallet,
|
|
3190
|
+
onVerifyWallet: (method: 'berifyme' | 'custom') => handleVerifyWalletClick(method),
|
|
3191
|
+
serviceId,
|
|
3192
|
+
permissionGroup,
|
|
3193
|
+
|
|
3194
|
+
// Props
|
|
3195
|
+
props: {
|
|
3196
|
+
onMessage,
|
|
3197
|
+
onError,
|
|
3198
|
+
onToolUse,
|
|
3199
|
+
presetMessages,
|
|
3200
|
+
placeholder,
|
|
3201
|
+
className,
|
|
3202
|
+
maxRetries,
|
|
3203
|
+
retryDelay,
|
|
3204
|
+
serviceId,
|
|
3205
|
+
apiKey,
|
|
3206
|
+
apiSecret,
|
|
3207
|
+
onVerifyWallet,
|
|
3208
|
+
permissionGroup,
|
|
3209
|
+
onModelChange,
|
|
3210
|
+
storageMode,
|
|
3211
|
+
requireWalletIDVVerification,
|
|
3212
|
+
isSemiAutomaticMode,
|
|
3213
|
+
customUserInfo,
|
|
3214
|
+
customUsername,
|
|
3215
|
+
idvBannerDismissSeconds,
|
|
3216
|
+
isStopRecordingOnSend,
|
|
3217
|
+
customError,
|
|
3218
|
+
},
|
|
3219
|
+
nxtlinqApi
|
|
3220
|
+
};
|
|
3221
|
+
|
|
3222
|
+
return (
|
|
3223
|
+
<ChatBotContext.Provider value={contextValue}>
|
|
3224
|
+
{children}
|
|
3225
|
+
</ChatBotContext.Provider>
|
|
3226
|
+
);
|
|
3227
|
+
};
|