@bytexbyte/nxtlinq-ai-agent-ui-react-development 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/dist/NxtlinqAgentChat.d.ts +26 -0
  2. package/dist/NxtlinqAgentChat.d.ts.map +1 -0
  3. package/dist/NxtlinqAgentChat.js +28 -0
  4. package/dist/components/AgentAssistantShell.d.ts +5 -0
  5. package/dist/components/AgentAssistantShell.d.ts.map +1 -0
  6. package/dist/components/AgentAssistantShell.js +52 -0
  7. package/dist/components/AgentComposer.d.ts +3 -0
  8. package/dist/components/AgentComposer.d.ts.map +1 -0
  9. package/dist/components/AgentComposer.js +60 -0
  10. package/dist/components/AgentMessageList.d.ts +3 -0
  11. package/dist/components/AgentMessageList.d.ts.map +1 -0
  12. package/dist/components/AgentMessageList.js +37 -0
  13. package/dist/components/AgentRemoteAudio.d.ts +4 -0
  14. package/dist/components/AgentRemoteAudio.d.ts.map +1 -0
  15. package/dist/components/AgentRemoteAudio.js +34 -0
  16. package/dist/components/AgentVoiceBar.d.ts +3 -0
  17. package/dist/components/AgentVoiceBar.d.ts.map +1 -0
  18. package/dist/components/AgentVoiceBar.js +91 -0
  19. package/dist/components/PresetMessageChips.d.ts +3 -0
  20. package/dist/components/PresetMessageChips.d.ts.map +1 -0
  21. package/dist/components/PresetMessageChips.js +23 -0
  22. package/dist/context/AgentAssistantContext.d.ts +32 -0
  23. package/dist/context/AgentAssistantContext.d.ts.map +1 -0
  24. package/dist/context/AgentAssistantContext.js +159 -0
  25. package/dist/index.d.ts +16 -0
  26. package/dist/index.d.ts.map +1 -0
  27. package/dist/index.js +12 -0
  28. package/dist/legacy/assets/images/adiSideItalicDataUri.d.ts +2 -0
  29. package/dist/legacy/assets/images/adiSideItalicDataUri.d.ts.map +1 -0
  30. package/dist/legacy/assets/images/adiSideItalicDataUri.js +1 -0
  31. package/dist/legacy/chatbot/ChatBot.d.ts +5 -0
  32. package/dist/legacy/chatbot/ChatBot.d.ts.map +1 -0
  33. package/dist/legacy/chatbot/ChatBot.js +35 -0
  34. package/dist/legacy/chatbot/context/ChatBotContext.d.ts +5 -0
  35. package/dist/legacy/chatbot/context/ChatBotContext.d.ts.map +1 -0
  36. package/dist/legacy/chatbot/context/ChatBotContext.js +2908 -0
  37. package/dist/legacy/chatbot/types/ChatBotTypes.d.ts +166 -0
  38. package/dist/legacy/chatbot/types/ChatBotTypes.d.ts.map +1 -0
  39. package/dist/legacy/chatbot/types/ChatBotTypes.js +1 -0
  40. package/dist/legacy/chatbot/ui/BerifyMeModal.d.ts +17 -0
  41. package/dist/legacy/chatbot/ui/BerifyMeModal.d.ts.map +1 -0
  42. package/dist/legacy/chatbot/ui/BerifyMeModal.js +110 -0
  43. package/dist/legacy/chatbot/ui/ChatBotUI.d.ts +3 -0
  44. package/dist/legacy/chatbot/ui/ChatBotUI.d.ts.map +1 -0
  45. package/dist/legacy/chatbot/ui/ChatBotUI.js +625 -0
  46. package/dist/legacy/chatbot/ui/MessageInput.d.ts +3 -0
  47. package/dist/legacy/chatbot/ui/MessageInput.d.ts.map +1 -0
  48. package/dist/legacy/chatbot/ui/MessageInput.js +321 -0
  49. package/dist/legacy/chatbot/ui/MessageList.d.ts +4 -0
  50. package/dist/legacy/chatbot/ui/MessageList.d.ts.map +1 -0
  51. package/dist/legacy/chatbot/ui/MessageList.js +455 -0
  52. package/dist/legacy/chatbot/ui/ModelSelector.d.ts +4 -0
  53. package/dist/legacy/chatbot/ui/ModelSelector.d.ts.map +1 -0
  54. package/dist/legacy/chatbot/ui/ModelSelector.js +122 -0
  55. package/dist/legacy/chatbot/ui/NotificationModal.d.ts +15 -0
  56. package/dist/legacy/chatbot/ui/NotificationModal.d.ts.map +1 -0
  57. package/dist/legacy/chatbot/ui/NotificationModal.js +53 -0
  58. package/dist/legacy/chatbot/ui/PermissionForm.d.ts +8 -0
  59. package/dist/legacy/chatbot/ui/PermissionForm.d.ts.map +1 -0
  60. package/dist/legacy/chatbot/ui/PermissionForm.js +465 -0
  61. package/dist/legacy/chatbot/ui/PresetMessages.d.ts +4 -0
  62. package/dist/legacy/chatbot/ui/PresetMessages.d.ts.map +1 -0
  63. package/dist/legacy/chatbot/ui/PresetMessages.js +33 -0
  64. package/dist/legacy/chatbot/ui/VoiceModePanel.d.ts +3 -0
  65. package/dist/legacy/chatbot/ui/VoiceModePanel.d.ts.map +1 -0
  66. package/dist/legacy/chatbot/ui/VoiceModePanel.js +95 -0
  67. package/dist/legacy/chatbot/ui/styles/isolatedStyles.d.ts +73 -0
  68. package/dist/legacy/chatbot/ui/styles/isolatedStyles.d.ts.map +1 -0
  69. package/dist/legacy/chatbot/ui/styles/isolatedStyles.js +985 -0
  70. package/dist/legacy/index.d.ts +14 -0
  71. package/dist/legacy/index.d.ts.map +1 -0
  72. package/dist/legacy/index.js +12 -0
  73. package/dist/theme/defaultTheme.d.ts +3 -0
  74. package/dist/theme/defaultTheme.d.ts.map +1 -0
  75. package/dist/theme/defaultTheme.js +20 -0
  76. package/dist/types.d.ts +62 -0
  77. package/dist/types.d.ts.map +1 -0
  78. package/dist/types.js +1 -0
  79. package/dist/voice/useVoiceConnectOrchestration.d.ts +21 -0
  80. package/dist/voice/useVoiceConnectOrchestration.d.ts.map +1 -0
  81. package/dist/voice/useVoiceConnectOrchestration.js +86 -0
  82. package/dist/voice/useVoiceMicState.d.ts +15 -0
  83. package/dist/voice/useVoiceMicState.d.ts.map +1 -0
  84. package/dist/voice/useVoiceMicState.js +94 -0
  85. package/dist/voice/useVoiceSilenceCommit.d.ts +10 -0
  86. package/dist/voice/useVoiceSilenceCommit.d.ts.map +1 -0
  87. package/dist/voice/useVoiceSilenceCommit.js +67 -0
  88. package/dist/voice/useVoiceTranscriptMessages.d.ts +16 -0
  89. package/dist/voice/useVoiceTranscriptMessages.d.ts.map +1 -0
  90. package/dist/voice/useVoiceTranscriptMessages.js +129 -0
  91. package/dist/voice/useWsRealtimeAudio.d.ts +18 -0
  92. package/dist/voice/useWsRealtimeAudio.d.ts.map +1 -0
  93. package/dist/voice/useWsRealtimeAudio.js +102 -0
  94. package/dist/voice/voiceMicConstants.d.ts +4 -0
  95. package/dist/voice/voiceMicConstants.d.ts.map +1 -0
  96. package/dist/voice/voiceMicConstants.js +10 -0
  97. package/dist/voice/ws/BrowserWsPcmPlayer.d.ts +23 -0
  98. package/dist/voice/ws/BrowserWsPcmPlayer.d.ts.map +1 -0
  99. package/dist/voice/ws/BrowserWsPcmPlayer.js +137 -0
  100. package/dist/voice/ws/BrowserWsPcmRecorder.d.ts +17 -0
  101. package/dist/voice/ws/BrowserWsPcmRecorder.d.ts.map +1 -0
  102. package/dist/voice/ws/BrowserWsPcmRecorder.js +71 -0
  103. package/dist/voice/ws/float32ToPcm16.d.ts +2 -0
  104. package/dist/voice/ws/float32ToPcm16.d.ts.map +1 -0
  105. package/dist/voice/ws/float32ToPcm16.js +8 -0
  106. package/dist/voice/ws/voiceSilenceConstants.d.ts +5 -0
  107. package/dist/voice/ws/voiceSilenceConstants.d.ts.map +1 -0
  108. package/dist/voice/ws/voiceSilenceConstants.js +4 -0
  109. package/dist/voice/ws/wsRealtimeConstants.d.ts +2 -0
  110. package/dist/voice/ws/wsRealtimeConstants.d.ts.map +1 -0
  111. package/dist/voice/ws/wsRealtimeConstants.js +1 -0
  112. package/package.json +60 -0
  113. package/src/NxtlinqAgentChat.tsx +79 -0
  114. package/src/components/AgentAssistantShell.tsx +104 -0
  115. package/src/components/AgentComposer.tsx +134 -0
  116. package/src/components/AgentMessageList.tsx +78 -0
  117. package/src/components/AgentRemoteAudio.tsx +34 -0
  118. package/src/components/AgentVoiceBar.tsx +173 -0
  119. package/src/components/PresetMessageChips.tsx +41 -0
  120. package/src/context/AgentAssistantContext.tsx +276 -0
  121. package/src/index.ts +78 -0
  122. package/src/legacy/assets/images/adiSideItalicDataUri.ts +1 -0
  123. package/src/legacy/chatbot/ChatBot.tsx +61 -0
  124. package/src/legacy/chatbot/context/ChatBotContext.tsx +3227 -0
  125. package/src/legacy/chatbot/types/ChatBotTypes.ts +195 -0
  126. package/src/legacy/chatbot/ui/BerifyMeModal.tsx +145 -0
  127. package/src/legacy/chatbot/ui/ChatBotUI.tsx +949 -0
  128. package/src/legacy/chatbot/ui/MessageInput.tsx +517 -0
  129. package/src/legacy/chatbot/ui/MessageList.tsx +764 -0
  130. package/src/legacy/chatbot/ui/ModelSelector.tsx +190 -0
  131. package/src/legacy/chatbot/ui/NotificationModal.tsx +110 -0
  132. package/src/legacy/chatbot/ui/PermissionForm.tsx +632 -0
  133. package/src/legacy/chatbot/ui/PresetMessages.tsx +50 -0
  134. package/src/legacy/chatbot/ui/VoiceModePanel.tsx +168 -0
  135. package/src/legacy/chatbot/ui/styles/isolatedStyles.ts +1058 -0
  136. package/src/legacy/index.ts +26 -0
  137. package/src/theme/defaultTheme.ts +22 -0
  138. package/src/types.ts +65 -0
  139. package/src/voice/useVoiceConnectOrchestration.ts +117 -0
  140. package/src/voice/useVoiceMicState.ts +117 -0
  141. package/src/voice/useVoiceTranscriptMessages.ts +173 -0
  142. package/src/voice/voiceMicConstants.ts +13 -0
@@ -0,0 +1,3227 @@
1
+ import { ethers } from 'ethers';
2
+ import stringify from 'fast-json-stable-stringify';
3
+ import * as React from 'react';
4
+ import { flushSync } from 'react-dom';
5
+ import { v4 as uuidv4 } from 'uuid';
6
+ import {
7
+ createNxtlinqApi,
8
+ setApiHosts,
9
+ synthesizeSpeechToBuffer,
10
+ useLocalStorage,
11
+ useSessionStorage,
12
+ useSpeechToTextFromMic,
13
+ useVoiceMode,
14
+ metakeepClient,
15
+ getEthers,
16
+ sleep,
17
+ walletTextUtils,
18
+ } from '@bytexbyte/nxtlinq-ai-agent-web-development';
19
+ import type {
20
+ AgentResponse,
21
+ AIT,
22
+ Attachment,
23
+ ClientTtsVoiceSettings,
24
+ Message,
25
+ ServicePermission,
26
+ } from '@bytexbyte/nxtlinq-ai-agent-core-development';
27
+ import {
28
+ AIModel,
29
+ ChatBotContextType,
30
+ ChatBotProps,
31
+ PresetMessage
32
+ } from '../types/ChatBotTypes';
33
+
34
+ const MIC_ENABLED_SESSION_KEY = 'chatbot-mic-enabled';
35
+
36
+ const ChatBotContext = React.createContext<ChatBotContextType | undefined>(undefined);
37
+
38
+ export const useChatBot = () => {
39
+ const context = React.useContext(ChatBotContext);
40
+ if (!context) {
41
+ throw new Error('useChatBot must be used within a ChatBotProvider');
42
+ }
43
+ return context;
44
+ };
45
+
46
+ export const ChatBotProvider: React.FC<ChatBotProps> = ({
47
+ onMessage,
48
+ onError,
49
+ onToolUse,
50
+ presetMessages = [],
51
+ placeholder = 'Type a message...',
52
+ className = '',
53
+ maxRetries = 3,
54
+ retryDelay = 2000,
55
+ serviceId,
56
+ apiKey,
57
+ apiSecret,
58
+ environment = 'production',
59
+ onVerifyWallet,
60
+ permissionGroup,
61
+ children,
62
+ // AI Model related attributes
63
+ onModelChange,
64
+ // Storage mode configuration
65
+ storageMode = "local-storage" as "session-storage" | "local-storage",
66
+ // Wallet verification configuration
67
+ requireWalletIDVVerification = true,
68
+ isSemiAutomaticMode = false,
69
+ // Custom user identity information
70
+ customUserInfo,
71
+ // Custom username for wallet verification
72
+ customUsername,
73
+ // IDV suggestion banner configuration
74
+ idvBannerDismissSeconds = 86400,
75
+ // 24 hours in seconds
76
+ isStopRecordingOnSend = false,
77
+ // Custom error message to display in chat
78
+ customError,
79
+ // PII display mode
80
+ piiDisplayMode = 'redacted',
81
+ }) => {
82
+ // Set API hosts immediately based on environment (before any API calls)
83
+ setApiHosts(environment);
84
+
85
+
86
+ const nxtlinqApi = React.useMemo(() => createNxtlinqApi(apiKey, apiSecret), [apiKey, apiSecret]);
87
+
88
+ const apiKeyRef = React.useRef(apiKey);
89
+ const apiSecretRef = React.useRef(apiSecret);
90
+ const clientTtsVoiceRef = React.useRef<ClientTtsVoiceSettings | undefined>(undefined);
91
+ React.useEffect(() => {
92
+ apiKeyRef.current = apiKey;
93
+ apiSecretRef.current = apiSecret;
94
+ }, [apiKey, apiSecret]);
95
+
96
+ // Custom hook
97
+ const { isMicEnabled, transcript, partialTranscript, start: startRecording, stop: stopRecording, clear: clearRecording } = useSpeechToTextFromMic({
98
+ apiKey,
99
+ apiSecret
100
+ });
101
+
102
+ // Messages: Use storage based on storageMode to support cross-tab sync (local-storage) or session isolation (session-storage)
103
+ const [messages, setMessages] = storageMode === "local-storage"
104
+ ? useLocalStorage<Message[]>('chatbot-messages', [])
105
+ : useSessionStorage<Message[]>('chatbot-messages', []);
106
+ const [isOpen, setIsOpen] = storageMode === "session-storage"
107
+ ? useSessionStorage<boolean>('chatbot-is-open', false)
108
+ : React.useState<boolean>(false);
109
+
110
+ const [inputValue, setInputValue] = React.useState('');
111
+ const [isLoading, setIsLoading] = React.useState(false);
112
+ const [hitAddress, setHitAddress] = React.useState<string | null>(null);
113
+ const [ait, setAit] = React.useState<AIT | null>(null);
114
+ const [permissions, setPermissions] = React.useState<string[]>([]);
115
+ const [availablePermissions, setAvailablePermissions] = React.useState<ServicePermission[]>([]);
116
+ const [showPermissionForm, setShowPermissionForm] = React.useState(false);
117
+ const [isPermissionFormOpen, setIsPermissionFormOpen] = React.useState(false);
118
+ const [isAITLoading, setIsAITLoading] = React.useState(false);
119
+ const [isDisabled, setIsDisabled] = React.useState(true);
120
+ const [signer, setSigner] = React.useState<ethers.JsonRpcSigner | null>(null);
121
+ const [walletInfo, setWalletInfo] = React.useState<any>(null);
122
+ const [isWalletLoading, setIsWalletLoading] = React.useState(false);
123
+ const [isAutoConnecting, setIsAutoConnecting] = React.useState(false);
124
+
125
+ // Persistent data (always use localStorage)
126
+ const [nxtlinqAITServiceAccessToken, setNxtlinqAITServiceAccessToken] = useLocalStorage<string>('nxtlinqAITServiceAccessToken', '');
127
+ const [pseudoId, setPseudoId] = useLocalStorage<string>('pseudoId', uuidv4());
128
+
129
+ // Suggestions: Use storage based on storageMode to support cross-tab sync (local-storage) or session isolation (session-storage)
130
+ const [suggestions, setSuggestions] = storageMode === "local-storage"
131
+ ? useLocalStorage<PresetMessage[]>('chatbot-suggestions', presetMessages)
132
+ : useSessionStorage<PresetMessage[]>('chatbot-suggestions', presetMessages);
133
+ const [isAITEnabling, setIsAITEnabling] = React.useState(false);
134
+ const [isAwaitingMicGesture, setIsAwaitingMicGesture] = React.useState(false);
135
+ // autoSendEnabled: Always use localStorage (do not sync across tabs to avoid conflicts with speech-to-text)
136
+ const [autoSendEnabled, setAutoSendEnabled] = useLocalStorage<boolean>('chatbot-auto-send-enabled', true);
137
+
138
+ // Speech related state
139
+ const [textToSpeechEnabled, setTextToSpeechEnabled] = useLocalStorage<boolean>('chatbot-text-to-speech-enabled', false);
140
+ const [preferredChatMode, setPreferredChatMode] = useLocalStorage<'text' | 'voice'>('chatbot-active-mode', 'text');
141
+ const [speechingIndex, setSpeechingIndex] = React.useState<number | undefined>(undefined);
142
+
143
+ // Speech related refs
144
+ const audioCtxRef = React.useRef<AudioContext | null>(null);
145
+ const audioSourceRef = React.useRef<AudioBufferSourceNode | null>(null);
146
+ const audioElementRef = React.useRef<HTMLAudioElement | null>(null);
147
+ const speechingRef = React.useRef(false);
148
+ const [isTtsProcessing, setIsTtsProcessing] = React.useState(false);
149
+ const [requiresGesture, setRequiresGesture] = React.useState(false);
150
+ const pendingTtsRef = React.useRef<string | null>(null);
151
+ /** Azure Speech STT:每次麥克風成功開啟時記錄,供寫入後端 usage log(開麥→送出) */
152
+ const messagesRef = React.useRef(messages);
153
+ messagesRef.current = messages;
154
+ const micSessionStartForLogRef = React.useRef<number | null>(null);
155
+
156
+
157
+ // Clean up stale PII scanning state on mount (e.g. page refresh during active SSE)
158
+ React.useEffect(() => {
159
+ setMessages(prev => {
160
+ const hasStale = prev.some(m => m.piiStatus === 'scanning');
161
+ if (!hasStale) return prev;
162
+ return prev.map(m =>
163
+ m.piiStatus === 'scanning'
164
+ ? { ...m, piiStatus: undefined, piiStep: undefined }
165
+ : m
166
+ );
167
+ });
168
+ // eslint-disable-next-line react-hooks/exhaustive-deps
169
+ }, []);
170
+
171
+ // Use refs to get latest state values in hasPermission function
172
+ const hitAddressRef = React.useRef(hitAddress);
173
+ const aitRef = React.useRef(ait);
174
+ const permissionsRef = React.useRef(permissions);
175
+ const nxtlinqAITServiceAccessTokenRef = React.useRef(nxtlinqAITServiceAccessToken);
176
+ const signerRef = React.useRef(signer);
177
+
178
+ // Helper function to combine customUsername with Adilas customUserInfo
179
+ const getFinalCustomUsername = React.useCallback((username: string | undefined): string | undefined => {
180
+ if (!username) return username;
181
+
182
+ // For Adilas: combine customUsername, corpId and serverDomain
183
+ if (walletTextUtils.isAdilasService(serviceId) && customUserInfo) {
184
+ const { corpId, serverDomain } = customUserInfo;
185
+ if (corpId && serverDomain) {
186
+ // Format: customUsername|(corpId)|[serverDomain]
187
+ const parts = [username];
188
+ if (corpId) parts.push(`(${corpId})`);
189
+ if (serverDomain) parts.push(`[${serverDomain}]`);
190
+ return parts.join('|');
191
+ }
192
+ }
193
+
194
+ return username;
195
+ }, [serviceId, customUserInfo]);
196
+
197
+ // Refs for input value and recording state
198
+ const isMicEnabledRef = React.useRef(false);
199
+ const isRestoringRecordingRef = React.useRef(false);
200
+ const hasSyncedMicStateRef = React.useRef(false);
201
+ const isReacquiringMicRef = React.useRef(false);
202
+ const pendingMicAutoStartRef = React.useRef(false);
203
+ const autoStartGestureCleanupRef = React.useRef<(() => void) | null>(null);
204
+ const textInputRef = React.useRef<HTMLInputElement | null>(null);
205
+ const lastPartialRangeRef = React.useRef<{ start: number; end: number } | null>(null);
206
+ const lastAutoSentTranscriptRef = React.useRef<string>('');
207
+ const autoSendTimerRef = React.useRef<NodeJS.Timeout | null>(null);
208
+ const lastCustomErrorRef = React.useRef<string | undefined>(undefined);
209
+
210
+ function insertPartial(input: string, partial: string, caret: number) {
211
+ let start = caret;
212
+ let end = caret;
213
+
214
+ if (lastPartialRangeRef.current) {
215
+ start = lastPartialRangeRef.current.start;
216
+ end = lastPartialRangeRef.current.end;
217
+ }
218
+
219
+ const before = input.slice(0, start);
220
+ const after = input.slice(end);
221
+
222
+ const next = before + partial + after;
223
+ const newCaret = start + partial.length;
224
+
225
+ lastPartialRangeRef.current = { start, end: newCaret };
226
+ return { next, caret: newCaret };
227
+ }
228
+
229
+ function finalizePartial(input: string, finalText: string) {
230
+ if (!lastPartialRangeRef.current) {
231
+ return { next: input, caret: input.length };
232
+ }
233
+
234
+ const { start, end } = lastPartialRangeRef.current;
235
+ const before = input.slice(0, start);
236
+ const after = input.slice(end);
237
+
238
+ const next = before + finalText + after;
239
+ const caret = start + finalText.length;
240
+
241
+ lastPartialRangeRef.current = null;
242
+ return { next, caret };
243
+ }
244
+
245
+ function normalizeTranscript(text: string): string {
246
+ return text
247
+ .toLowerCase()
248
+ // Remove period only when NOT between digits (preserve decimals like 130.00)
249
+ .replace(/(?<!\d)\.(?!\d)/g, '')
250
+ .replace(/[,!?;:]/g, '')
251
+ .replace(/\s+/g, ' ')
252
+ .trim();
253
+ }
254
+
255
+ // Simple token cleanup function
256
+ const clearExpiredToken = React.useCallback(() => {
257
+ try {
258
+ localStorage.removeItem('nxtlinqAITServiceAccessToken');
259
+ setNxtlinqAITServiceAccessToken('');
260
+ } catch (error) {
261
+ console.error('Error clearing expired token:', error);
262
+ }
263
+ }, []);
264
+
265
+ // Internal: Play text-to-speech with retry mechanism
266
+ const playTextToSpeechWithRetry = React.useCallback(async (text: string, messageIndex?: number, attempt = 0): Promise<void> => {
267
+ if (attempt > 2) {
268
+ console.error('TTS playback failed after retries');
269
+ setIsTtsProcessing(false);
270
+ return;
271
+ }
272
+ const synth = await synthesizeSpeechToBuffer({
273
+ text,
274
+ apiKey: apiKeyRef.current,
275
+ apiSecret: apiSecretRef.current,
276
+ settings: clientTtsVoiceRef.current,
277
+ });
278
+ if (!synth) throw new Error('Failed to get audio buffer');
279
+
280
+ const bufferData = synth.buffer;
281
+ const audioMimeType = synth.mimeType;
282
+
283
+ const isMobileDevice = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent);
284
+ if (isMobileDevice) {
285
+ const blob = new Blob([bufferData], { type: audioMimeType });
286
+ const audioUrl = URL.createObjectURL(blob);
287
+ const audio = new Audio(audioUrl);
288
+ if ('setSinkId' in HTMLMediaElement.prototype) {
289
+ try {
290
+ const devices = await (navigator.mediaDevices as any)?.enumerateDevices?.();
291
+ const speakerDevice = devices?.find((d: any) => d.kind === 'audiooutput' && d.label.toLowerCase().includes('speaker'));
292
+ if (speakerDevice) await (audio as any).setSinkId(speakerDevice.deviceId);
293
+ else await (audio as any).setSinkId('default');
294
+ } catch (err) {
295
+ console.debug('Could not set audio sink:', err);
296
+ }
297
+ }
298
+ audio.volume = 1.0;
299
+ audioElementRef.current = audio;
300
+ speechingRef.current = true;
301
+ if (messageIndex !== undefined) setSpeechingIndex(messageIndex);
302
+
303
+ // Track if onplay has been called to avoid clearing state multiple times
304
+ let playStarted = false;
305
+ const clearProcessingState = () => {
306
+ if (!playStarted) {
307
+ playStarted = true;
308
+ setIsTtsProcessing(false);
309
+ }
310
+ };
311
+
312
+ await new Promise<void>(async (resolve, reject) => {
313
+ audio.onended = () => {
314
+ speechingRef.current = false;
315
+ setSpeechingIndex(undefined);
316
+ URL.revokeObjectURL(audioUrl);
317
+ audioElementRef.current = null;
318
+ resolve();
319
+ };
320
+ audio.onerror = (error) => {
321
+ speechingRef.current = false;
322
+ setSpeechingIndex(undefined);
323
+ URL.revokeObjectURL(audioUrl);
324
+ audioElementRef.current = null;
325
+ clearProcessingState();
326
+ reject(error as any);
327
+ };
328
+ audio.onplay = () => {
329
+ // Clear processing state when audio actually starts playing
330
+ setRequiresGesture(false);
331
+ clearProcessingState();
332
+ };
333
+
334
+ // Handle play() promise - some mobile browsers require user interaction
335
+ audio.play().then(() => {
336
+ // Play promise resolved, but wait for onplay event to confirm actual playback
337
+ // Don't clear requiresGesture here - let onplay handle it
338
+ setTimeout(clearProcessingState, 100);
339
+ }).catch((playError) => {
340
+ // Mark requires gesture and store pending text
341
+ setRequiresGesture(true);
342
+ pendingTtsRef.current = text;
343
+ // Fallback: try WebAudio (AudioContext) playback on mobile
344
+ try {
345
+ if (audioCtxRef.current === null) {
346
+ audioCtxRef.current = new AudioContext();
347
+ }
348
+ audioSourceRef.current = audioCtxRef.current.createBufferSource();
349
+ if (!audioSourceRef.current) {
350
+ throw playError;
351
+ }
352
+ audioCtxRef.current.decodeAudioData(
353
+ bufferData.slice(0),
354
+ (decodedBuffer) => {
355
+ if (!audioSourceRef.current) {
356
+ reject(playError);
357
+ return;
358
+ }
359
+ audioSourceRef.current.buffer = decodedBuffer;
360
+ audioSourceRef.current.connect(audioCtxRef.current!.destination);
361
+ speechingRef.current = true;
362
+ clearProcessingState();
363
+ audioSourceRef.current.onended = () => {
364
+ speechingRef.current = false;
365
+ setSpeechingIndex(undefined);
366
+ audioSourceRef.current = null;
367
+ // cleanup audio element blob URL
368
+ URL.revokeObjectURL(audioUrl);
369
+ audioElementRef.current = null;
370
+ resolve();
371
+ };
372
+ try {
373
+ // Ensure AudioContext is running before starting
374
+ if (audioCtxRef.current?.state === 'suspended') {
375
+ audioCtxRef.current.resume().then(() => {
376
+ audioSourceRef.current?.start();
377
+ // Only clear requiresGesture after successfully starting
378
+ setRequiresGesture(false);
379
+ }).catch((resumeErr) => {
380
+ // AudioContext resume failed, keep requiresGesture true
381
+ reject(resumeErr);
382
+ });
383
+ } else {
384
+ audioSourceRef.current.start();
385
+ // Only clear requiresGesture after successfully starting
386
+ setRequiresGesture(false);
387
+ }
388
+ } catch (e) {
389
+ // start() failed, keep requiresGesture true
390
+ reject(e as any);
391
+ }
392
+ },
393
+ (decodeErr) => {
394
+ // If decode also fails, give up
395
+ speechingRef.current = false;
396
+ setSpeechingIndex(undefined);
397
+ URL.revokeObjectURL(audioUrl);
398
+ audioElementRef.current = null;
399
+ clearProcessingState();
400
+ reject(decodeErr as any);
401
+ }
402
+ );
403
+ } catch (fallbackErr) {
404
+ // Play failed (e.g., user interaction required)
405
+ speechingRef.current = false;
406
+ setSpeechingIndex(undefined);
407
+ URL.revokeObjectURL(audioUrl);
408
+ audioElementRef.current = null;
409
+ clearProcessingState();
410
+ reject(fallbackErr as any);
411
+ }
412
+ });
413
+ });
414
+ return;
415
+ }
416
+
417
+ if (audioCtxRef.current === null) {
418
+ audioCtxRef.current = new AudioContext();
419
+ }
420
+ audioSourceRef.current = audioCtxRef.current.createBufferSource();
421
+ if (!audioSourceRef.current) return;
422
+ const buffer = await new Promise<AudioBuffer>((resolve, reject) => {
423
+ if (audioCtxRef.current !== null && bufferData) {
424
+ audioCtxRef.current.decodeAudioData(
425
+ bufferData,
426
+ (decodedBuffer) => resolve(decodedBuffer),
427
+ (error) => reject(error)
428
+ );
429
+ } else {
430
+ reject(new Error('Audio context or buffer data is missing'));
431
+ }
432
+ });
433
+ if (!audioSourceRef.current.buffer) {
434
+ audioSourceRef.current.buffer = buffer;
435
+ audioSourceRef.current.connect(audioCtxRef.current.destination);
436
+ speechingRef.current = true;
437
+ if (messageIndex !== undefined) setSpeechingIndex(messageIndex);
438
+ // Clear processing state when audio starts playing
439
+ setRequiresGesture(false);
440
+ setIsTtsProcessing(false);
441
+ await new Promise<void>((resolve) => {
442
+ audioSourceRef.current!.onended = () => {
443
+ speechingRef.current = false;
444
+ setSpeechingIndex(undefined);
445
+ audioSourceRef.current = null;
446
+ resolve();
447
+ };
448
+ audioSourceRef.current!.start();
449
+ });
450
+ }
451
+ }, []);
452
+
453
+ // Update refs when state changes
454
+ React.useEffect(() => {
455
+ hitAddressRef.current = hitAddress;
456
+ }, [hitAddress]);
457
+
458
+ React.useEffect(() => {
459
+ aitRef.current = ait;
460
+ }, [ait]);
461
+
462
+ React.useEffect(() => {
463
+ permissionsRef.current = permissions;
464
+ }, [permissions]);
465
+
466
+ React.useEffect(() => {
467
+ nxtlinqAITServiceAccessTokenRef.current = nxtlinqAITServiceAccessToken;
468
+ }, [nxtlinqAITServiceAccessToken]);
469
+
470
+ React.useEffect(() => {
471
+ signerRef.current = signer;
472
+ }, [signer]);
473
+
474
+ React.useEffect(() => {
475
+ if (!textInputRef.current) return;
476
+
477
+ const el = textInputRef.current;
478
+ const selStart = el.selectionStart ?? inputValue.length;
479
+
480
+ if (partialTranscript) {
481
+ const { next, caret } = insertPartial(inputValue, partialTranscript, selStart);
482
+ setInputValue(next);
483
+
484
+ setTimeout(() => {
485
+ if (textInputRef.current) {
486
+ textInputRef.current.selectionStart = caret;
487
+ textInputRef.current.selectionEnd = caret;
488
+ }
489
+ }, 0);
490
+ }
491
+ }, [partialTranscript]);
492
+
493
+ React.useEffect(() => {
494
+ if (!transcript) return;
495
+ if (!textInputRef.current) return;
496
+
497
+ // 每段新辨識結果重設 STT 計時起點(長期開麥、多輪自動送出時,每則請求各自一段,避免只記到第一則)
498
+ micSessionStartForLogRef.current = performance.now();
499
+
500
+ const { next, caret } = finalizePartial(inputValue, transcript);
501
+ const normalizedText = normalizeTranscript(next);
502
+ setInputValue(normalizedText);
503
+
504
+ setTimeout(() => {
505
+ if (textInputRef.current) {
506
+ textInputRef.current.selectionStart = caret;
507
+ textInputRef.current.selectionEnd = caret;
508
+ }
509
+ }, 0);
510
+
511
+ // Auto send on speech complete (if user enabled the toggle)
512
+ if (autoSendEnabled && isMicEnabled && normalizedText.trim()) {
513
+ // Avoid duplicate sends
514
+ if (lastAutoSentTranscriptRef.current !== normalizedText.trim()) {
515
+ // Clear any existing timer
516
+ if (autoSendTimerRef.current) {
517
+ clearTimeout(autoSendTimerRef.current);
518
+ }
519
+
520
+ const currentTranscript = normalizedText.trim();
521
+
522
+ // Wait 3 seconds before auto-sending
523
+ // Timer will be cleared if user continues speaking (transcript/partialTranscript changes)
524
+ autoSendTimerRef.current = setTimeout(() => {
525
+ if (lastAutoSentTranscriptRef.current !== currentTranscript && !isLoading) {
526
+ lastAutoSentTranscriptRef.current = currentTranscript;
527
+ sendMessage(currentTranscript).catch(error => {
528
+ console.error('Failed to auto-send message from speech:', error);
529
+ lastAutoSentTranscriptRef.current = '';
530
+ });
531
+ }
532
+ autoSendTimerRef.current = null;
533
+ }, 3000); // 3 seconds delay
534
+ }
535
+ } else if (autoSendTimerRef.current) {
536
+ // Clear timer if auto-send disabled or mic disabled
537
+ clearTimeout(autoSendTimerRef.current);
538
+ autoSendTimerRef.current = null;
539
+ }
540
+ // eslint-disable-next-line react-hooks/exhaustive-deps
541
+ }, [transcript, autoSendEnabled, isMicEnabled, isLoading]);
542
+
543
+ // Clear timer when user continues speaking (partialTranscript updates)
544
+ React.useEffect(() => {
545
+ if (partialTranscript && partialTranscript.trim() && autoSendEnabled && isMicEnabled && autoSendTimerRef.current) {
546
+ clearTimeout(autoSendTimerRef.current as NodeJS.Timeout);
547
+ autoSendTimerRef.current = null;
548
+ }
549
+ }, [partialTranscript, autoSendEnabled, isMicEnabled]);
550
+
551
+ // Cleanup timer when mic is disabled or component unmounts
552
+ React.useEffect(() => {
553
+ if (!isMicEnabled) {
554
+ if (autoSendTimerRef.current) {
555
+ clearTimeout(autoSendTimerRef.current as NodeJS.Timeout);
556
+ autoSendTimerRef.current = null;
557
+ }
558
+ }
559
+ return () => {
560
+ if (autoSendTimerRef.current) {
561
+ clearTimeout(autoSendTimerRef.current as NodeJS.Timeout);
562
+ autoSendTimerRef.current = null;
563
+ }
564
+ };
565
+ }, [isMicEnabled]);
566
+
567
+ React.useEffect(() => {
568
+ isMicEnabledRef.current = isMicEnabled;
569
+ // Reset auto-sent transcript ref when starting new recording
570
+ if (isMicEnabled) {
571
+ lastAutoSentTranscriptRef.current = '';
572
+ // Clear any pending auto-send timer
573
+ if (autoSendTimerRef.current) {
574
+ clearTimeout(autoSendTimerRef.current as NodeJS.Timeout);
575
+ autoSendTimerRef.current = null;
576
+ }
577
+ // Hide gesture prompt once mic is actually enabled
578
+ setIsAwaitingMicGesture(false);
579
+ // Remove any pending gesture listeners
580
+ if (autoStartGestureCleanupRef.current) {
581
+ autoStartGestureCleanupRef.current();
582
+ autoStartGestureCleanupRef.current = null;
583
+ }
584
+ }
585
+ }, [isMicEnabled]);
586
+
587
+ React.useEffect(() => {
588
+ if (typeof window === 'undefined') {
589
+ return;
590
+ }
591
+
592
+ if (storageMode !== "session-storage") {
593
+ sessionStorage.removeItem(MIC_ENABLED_SESSION_KEY);
594
+ hasSyncedMicStateRef.current = false;
595
+ return;
596
+ }
597
+
598
+ if (!hasSyncedMicStateRef.current) {
599
+ return;
600
+ }
601
+
602
+ try {
603
+ sessionStorage.setItem(MIC_ENABLED_SESSION_KEY, JSON.stringify(isMicEnabled));
604
+ } catch (error) {
605
+ console.warn('Failed to persist recording state to sessionStorage:', error);
606
+ }
607
+ }, [isMicEnabled, storageMode]);
608
+
609
+ React.useEffect(() => {
610
+ if (typeof window === 'undefined') {
611
+ return;
612
+ }
613
+
614
+ const cleanupAutoStartListeners = () => {
615
+ if (autoStartGestureCleanupRef.current) {
616
+ autoStartGestureCleanupRef.current();
617
+ autoStartGestureCleanupRef.current = null;
618
+ }
619
+ };
620
+
621
+ if (storageMode !== "session-storage") {
622
+ cleanupAutoStartListeners();
623
+ pendingMicAutoStartRef.current = false;
624
+ setIsAwaitingMicGesture(false);
625
+ return;
626
+ }
627
+
628
+ if (hasSyncedMicStateRef.current || isReacquiringMicRef.current) {
629
+ return;
630
+ }
631
+
632
+ const rawStoredValue = sessionStorage.getItem(MIC_ENABLED_SESSION_KEY);
633
+ const shouldRestore = rawStoredValue === 'true';
634
+
635
+ if (!shouldRestore || isMicEnabledRef.current || isRestoringRecordingRef.current) {
636
+ hasSyncedMicStateRef.current = true;
637
+ setIsAwaitingMicGesture(false);
638
+ return;
639
+ }
640
+
641
+ pendingMicAutoStartRef.current = true;
642
+
643
+ function scheduleRetryOnUserGesture() {
644
+ if (autoStartGestureCleanupRef.current) return;
645
+
646
+ setIsAwaitingMicGesture(true);
647
+ const events: Array<keyof WindowEventMap> = ['pointerdown', 'keydown', 'touchstart'];
648
+ const handler = async () => {
649
+ cleanupAutoStartListeners();
650
+ await attemptAutoStart('user-gesture');
651
+ };
652
+
653
+ events.forEach(event => window.addEventListener(event, handler, { once: true }));
654
+ autoStartGestureCleanupRef.current = () => {
655
+ events.forEach(event => window.removeEventListener(event, handler));
656
+ autoStartGestureCleanupRef.current = null;
657
+ };
658
+ }
659
+
660
+ async function attemptAutoStart(reason: 'initial' | 'user-gesture') {
661
+ isReacquiringMicRef.current = true;
662
+
663
+ // For initial restore, show gesture prompt immediately while checking
664
+ // This gives user early feedback and allows them to click anytime
665
+ if (reason === 'initial') {
666
+ scheduleRetryOnUserGesture();
667
+ }
668
+
669
+ if (typeof navigator !== 'undefined' && navigator.mediaDevices?.getUserMedia) {
670
+ try {
671
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
672
+ stream.getTracks().forEach(track => track.stop());
673
+ } catch (error: any) {
674
+ console.error('Failed to reacquire microphone stream:', error);
675
+ const errorName = error?.name || '';
676
+ const errorMessage = error instanceof Error ? error.message : String(error);
677
+
678
+ if (reason === 'initial') {
679
+ const isPermissionPermanentlyDenied = errorName === 'NotAllowedError' &&
680
+ /permission.*denied|blocked|not.*granted/i.test(errorMessage);
681
+
682
+ if (isPermissionPermanentlyDenied) {
683
+ // Permission permanently denied, hide the prompt
684
+ pendingMicAutoStartRef.current = false;
685
+ setIsAwaitingMicGesture(false);
686
+ try {
687
+ sessionStorage.setItem(MIC_ENABLED_SESSION_KEY, JSON.stringify(false));
688
+ } catch (storageError) {
689
+ console.warn('Failed to reset recording state in sessionStorage:', storageError);
690
+ }
691
+ cleanupAutoStartListeners();
692
+ }
693
+ // Otherwise, keep the prompt showing (already shown above)
694
+ isReacquiringMicRef.current = false;
695
+ return;
696
+ }
697
+
698
+ // For user-gesture retry, treat as error
699
+ pendingMicAutoStartRef.current = false;
700
+ setIsAwaitingMicGesture(false);
701
+ try {
702
+ sessionStorage.setItem(MIC_ENABLED_SESSION_KEY, JSON.stringify(false));
703
+ } catch (storageError) {
704
+ console.warn('Failed to reset recording state in sessionStorage:', storageError);
705
+ }
706
+ isReacquiringMicRef.current = false;
707
+ hasSyncedMicStateRef.current = true;
708
+ cleanupAutoStartListeners();
709
+ return;
710
+ }
711
+ } else if (reason === 'initial') {
712
+ // getUserMedia not available, prompt already shown above
713
+ isReacquiringMicRef.current = false;
714
+ return;
715
+ }
716
+
717
+ isRestoringRecordingRef.current = true;
718
+
719
+ // Intercept console.warn to detect AudioContext warnings
720
+ let audioContextWarningDetected = false;
721
+ const originalWarn = console.warn;
722
+ console.warn = (...args: any[]) => {
723
+ const message = args.join(' ');
724
+ if (message.includes('AudioContext was not allowed to start')) {
725
+ audioContextWarningDetected = true;
726
+ }
727
+ originalWarn.apply(console, args);
728
+ };
729
+
730
+ try {
731
+ await startRecording();
732
+ console.warn = originalWarn;
733
+
734
+ // Check mic status after a short delay
735
+ await new Promise(resolve => setTimeout(resolve, 300));
736
+
737
+ // If mic started successfully, hide the prompt
738
+ if (isMicEnabledRef.current) {
739
+ pendingMicAutoStartRef.current = false;
740
+ setIsAwaitingMicGesture(false);
741
+ cleanupAutoStartListeners();
742
+ } else if (reason === 'initial') {
743
+ // Mic didn't start, keep the prompt showing (already shown earlier)
744
+ isRestoringRecordingRef.current = false;
745
+ return;
746
+ }
747
+ } catch (error) {
748
+ console.error('startRecording threw an error:', error);
749
+ try {
750
+ sessionStorage.setItem(MIC_ENABLED_SESSION_KEY, JSON.stringify(false));
751
+ } catch (storageError) {
752
+ console.warn('Failed to reset recording state in sessionStorage:', storageError);
753
+ }
754
+ } finally {
755
+ console.warn = originalWarn;
756
+ isRestoringRecordingRef.current = false;
757
+ isReacquiringMicRef.current = false;
758
+ hasSyncedMicStateRef.current = true;
759
+ }
760
+ }
761
+
762
+ attemptAutoStart('initial');
763
+
764
+ return () => {
765
+ cleanupAutoStartListeners();
766
+ pendingMicAutoStartRef.current = false;
767
+ setIsAwaitingMicGesture(false);
768
+ };
769
+ }, [storageMode, startRecording]);
770
+
771
+ const [notification, setNotification] = React.useState<{
772
+ show: boolean;
773
+ type: 'success' | 'error' | 'warning' | 'info';
774
+ message: string;
775
+ autoHide?: boolean;
776
+ duration?: number;
777
+ }>({
778
+ show: false,
779
+ type: 'info',
780
+ message: '',
781
+ autoHide: true,
782
+ duration: 5000
783
+ });
784
+
785
+ // AI Model related state
786
+ const [modelsFromApi, setModelsFromApi] = React.useState<AIModel[]>([]);
787
+ const [selectedModelIndex, setSelectedModelIndex] = useLocalStorage<number>('selectedAIModelIndex', 0);
788
+
789
+ // Use models from API only (no default fallback)
790
+ // This ensures SDK always uses Dashboard-configured models
791
+ const effectiveAvailableModels = React.useMemo(() => {
792
+ return modelsFromApi;
793
+ }, [modelsFromApi]);
794
+
795
+ // Auto determine if selector should be shown based on number of models
796
+ // Show selector only when there are 2 or more models
797
+ const showModelSelector = React.useMemo(() => {
798
+ return effectiveAvailableModels.length >= 2;
799
+ }, [effectiveAvailableModels]);
800
+
801
+ // Fetch available models from API on mount
802
+ React.useEffect(() => {
803
+ const fetchModels = async () => {
804
+ try {
805
+ const result = await nxtlinqApi.agent.getServiceModels({ apiKey, apiSecret });
806
+
807
+ if ('availableModels' in result && Array.isArray(result.availableModels)) {
808
+ setModelsFromApi(result.availableModels);
809
+ }
810
+ } catch (error) {
811
+ console.error('Failed to fetch available models from API:', error);
812
+ // Silently fail and use default models
813
+ }
814
+ };
815
+
816
+ fetchModels();
817
+ }, [apiKey, apiSecret, nxtlinqApi]);
818
+
819
+ // Validate selectedModelIndex against effectiveAvailableModels
820
+ React.useEffect(() => {
821
+ // Check if selectedModelIndex is out of bounds
822
+ if (selectedModelIndex >= effectiveAvailableModels.length) {
823
+ setSelectedModelIndex(0);
824
+ return;
825
+ }
826
+
827
+ // Check if the selected model still exists in effectiveAvailableModels
828
+ const selectedModel = effectiveAvailableModels[selectedModelIndex];
829
+ if (!selectedModel) {
830
+ setSelectedModelIndex(0);
831
+ return;
832
+ }
833
+ }, [effectiveAvailableModels, selectedModelIndex, setSelectedModelIndex]);
834
+
835
+ // Notification functions
836
+ const showNotification = (type: 'success' | 'error' | 'warning' | 'info', message: string, duration = 5000) => {
837
+ setNotification({
838
+ show: true,
839
+ type,
840
+ message,
841
+ autoHide: true,
842
+ duration
843
+ });
844
+
845
+ if (duration > 0) {
846
+ setTimeout(() => {
847
+ setNotification(prev => ({ ...prev, show: false }));
848
+ }, duration);
849
+ }
850
+ };
851
+
852
+ const showSuccess = (message: string) => showNotification('success', message, 3000);
853
+ const showError = (message: string) => showNotification('error', message, 5000);
854
+ const showWarning = (message: string) => showNotification('warning', message, 4000);
855
+ const showInfo = (message: string) => showNotification('info', message, 3000);
856
+
857
+ // Show "Please log in again." instead of raw "Invalid or expired token" to users (English)
858
+ const normalizeErrorForUser = (message: string): string =>
859
+ /invalid\s+or\s+expired\s+token/i.test(message) ? 'Please log in again.' : message;
860
+
861
+ // Fetch available permissions. Returns the service's permission list when successful, otherwise undefined.
862
+ const fetchAvailablePermissions = async (): Promise<ServicePermission[] | undefined> => {
863
+ if (!serviceId) return undefined;
864
+
865
+ try {
866
+ const result = await nxtlinqApi.permissions.getServicePermissions({
867
+ serviceId,
868
+ ...(permissionGroup && { groupName: permissionGroup })
869
+ });
870
+ if ('error' in result) {
871
+ console.error('Failed to fetch permissions:', result.error);
872
+ return undefined;
873
+ }
874
+ setAvailablePermissions(result.permissions);
875
+ return result.permissions;
876
+ } catch (error) {
877
+ console.error('Error fetching permissions:', error);
878
+ return undefined;
879
+ }
880
+ };
881
+
882
+ // Refresh AIT
883
+ const refreshAIT = async (forceUpdatePermissions = false) => {
884
+ if (!hitAddress) {
885
+ setAit(null);
886
+ setPermissions([]);
887
+ setWalletInfo(null);
888
+ setIsAITLoading(false);
889
+ return;
890
+ }
891
+
892
+ setIsAITLoading(true);
893
+ try {
894
+ // Get wallet info first - always try to get wallet info if we have a token
895
+ if (nxtlinqAITServiceAccessToken) {
896
+ try {
897
+ const walletResponse = await nxtlinqApi.wallet.getWallet({ address: hitAddress }, nxtlinqAITServiceAccessToken);
898
+ if (!('error' in walletResponse)) {
899
+ setWalletInfo(walletResponse);
900
+ } else {
901
+ // Check if the error is due to invalid/expired token
902
+ if (walletResponse.error.includes('Invalid or expired token')) {
903
+ console.log('Token appears to be invalid during wallet info fetch, clearing it');
904
+ clearExpiredToken();
905
+ }
906
+ }
907
+ } catch (error) {
908
+ console.error('Failed to fetch wallet info:', error);
909
+ }
910
+ }
911
+
912
+ // Only try to fetch AIT if we have a token
913
+ if (nxtlinqAITServiceAccessToken) {
914
+ const response = await nxtlinqApi.ait.getAITByServiceIdAndController({
915
+ serviceId,
916
+ controller: hitAddress,
917
+ customUsername: (!requireWalletIDVVerification && customUsername) ? getFinalCustomUsername(customUsername) : undefined
918
+ }, nxtlinqAITServiceAccessToken);
919
+
920
+ if ('error' in response) {
921
+ console.error('Failed to fetch AIT:', response.error);
922
+ setAit(null);
923
+ setPermissions([]);
924
+ return;
925
+ }
926
+
927
+ setAit(response);
928
+ if (!isPermissionFormOpen || forceUpdatePermissions) {
929
+ const newPermissions = response.metadata?.permissions || [];
930
+ setPermissions(newPermissions);
931
+ }
932
+ } else {
933
+ // No token available, clear AIT data
934
+ setAit(null);
935
+ setPermissions([]);
936
+ }
937
+ } catch (error) {
938
+ console.error('Failed to fetch AIT:', error);
939
+ setAit(null);
940
+ setPermissions([]);
941
+ } finally {
942
+ setIsAITLoading(false);
943
+ }
944
+ };
945
+
946
+ // Check if user needs to sign in
947
+ const isNeedSignInWithWallet = React.useMemo(() => {
948
+ if (!hitAddress) return false;
949
+ if (!nxtlinqAITServiceAccessToken) return true;
950
+
951
+ try {
952
+ const payload = JSON.parse(atob(nxtlinqAITServiceAccessToken.split('.')[1]));
953
+ const address = payload.address;
954
+ if (address !== hitAddress) return true;
955
+
956
+ return false;
957
+ } catch (error) {
958
+ console.error('Error parsing token:', error);
959
+ return true;
960
+ }
961
+ }, [hitAddress, nxtlinqAITServiceAccessToken]);
962
+
963
+ // Connect wallet
964
+ const connectWallet = React.useCallback(async (autoShowSignInMessage = true) => {
965
+ if (typeof window === 'undefined') {
966
+ console.error('Web3 is not available in server-side rendering');
967
+ return;
968
+ }
969
+
970
+ try {
971
+ getEthers();
972
+
973
+ const web3Provider = await metakeepClient.ethereum;
974
+ if (!web3Provider) {
975
+ throw new Error('Web3 provider not available');
976
+ }
977
+
978
+ await web3Provider.enable();
979
+
980
+ // ethers v6: Use BrowserProvider instead of Web3Provider
981
+ const ethersProvider = new ethers.BrowserProvider(web3Provider);
982
+
983
+ // ethers v6: getSigner() and getAddress() are now async
984
+ const userSigner = await ethersProvider.getSigner();
985
+ const userAddress = await userSigner.getAddress();
986
+
987
+ localStorage.setItem('walletAddress', userAddress);
988
+
989
+ setHitAddress(userAddress);
990
+ setSigner(userSigner);
991
+
992
+ // Immediately update refs to ensure they're available for subsequent operations
993
+ hitAddressRef.current = userAddress;
994
+ signerRef.current = userSigner;
995
+
996
+ // Check if we have a valid token before trying to load AIT
997
+ if (nxtlinqAITServiceAccessToken) {
998
+ try {
999
+ const payload = JSON.parse(atob(nxtlinqAITServiceAccessToken.split('.')[1]));
1000
+ const address = payload.address;
1001
+
1002
+ // Check if address matches
1003
+ if (address !== userAddress) {
1004
+ console.log('Token address mismatch, clearing token');
1005
+ clearExpiredToken();
1006
+ // Don't call refreshAIT with invalid token
1007
+ } else {
1008
+ // Token address matches, try to load AIT
1009
+ await refreshAIT();
1010
+ await sleep(1000);
1011
+ }
1012
+ } catch (error) {
1013
+ console.error('Error parsing token during wallet connection:', error);
1014
+ clearExpiredToken();
1015
+ }
1016
+ }
1017
+
1018
+ // If requireIDV is false and we have customUsername, check if wallet exists in AIT Service
1019
+ if (!requireWalletIDVVerification && customUsername) {
1020
+ try {
1021
+ // Check if wallet exists in AIT Service
1022
+ const walletResponse = await nxtlinqApi.wallet.getWallet({ address: userAddress }, '');
1023
+ if ('error' in walletResponse) {
1024
+ // Wallet doesn't exist, create it with custom method
1025
+ console.log('Wallet not found in AIT Service, creating with custom method...');
1026
+
1027
+ const verifyPayload = {
1028
+ address: userAddress,
1029
+ method: 'custom' as const,
1030
+ timestamp: Date.now(),
1031
+ customUsername: getFinalCustomUsername(customUsername)
1032
+ };
1033
+ const verifyResponse = await nxtlinqApi.wallet.verifyWallet(verifyPayload, '');
1034
+ if ('error' in verifyResponse) {
1035
+ console.error('Failed to create wallet with custom method:', verifyResponse.error);
1036
+ }
1037
+ }
1038
+ } catch (error) {
1039
+ console.error('Error checking/creating wallet in AIT Service:', error);
1040
+ }
1041
+ }
1042
+
1043
+ if (autoShowSignInMessage) {
1044
+ if (isNeedSignInWithWallet) {
1045
+ setMessages(prev => [...prev, {
1046
+ id: Date.now().toString(),
1047
+ content: walletTextUtils.getWalletText('Please sign in with your wallet to continue.', serviceId),
1048
+ role: 'assistant',
1049
+ timestamp: new Date().toISOString(),
1050
+ button: 'signIn'
1051
+ }]);
1052
+ } else {
1053
+ showSuccess(walletTextUtils.getWalletText('Successfully connected your HIT wallet. You are already signed in and can use the AI agent.', serviceId));
1054
+ }
1055
+ }
1056
+
1057
+ return userAddress;
1058
+ } catch (error) {
1059
+ console.error('Failed to connect wallet:', error);
1060
+ localStorage.removeItem('walletAddress');
1061
+ setHitAddress(null);
1062
+ setSigner(null);
1063
+ if (autoShowSignInMessage) {
1064
+ throw error;
1065
+ }
1066
+ return false;
1067
+ }
1068
+ }, [nxtlinqAITServiceAccessToken, refreshAIT, isNeedSignInWithWallet, requireWalletIDVVerification, customUsername, nxtlinqApi]);
1069
+
1070
+ const signInWallet = async (autoShowSuccessMessage = true) => {
1071
+ // Use refs to get latest state values for consistency with hasPermission
1072
+ const currentHitAddress = hitAddressRef.current;
1073
+ const currentSigner = signerRef.current;
1074
+
1075
+ // If refs don't have the values, try using current state values
1076
+ let addressToUse = currentHitAddress || hitAddress;
1077
+ let signerToUse = currentSigner || signer;
1078
+
1079
+ // If we have an address but no signer, try to auto-connect wallet
1080
+ if (addressToUse && !signerToUse) {
1081
+ const storedWalletAddress = localStorage.getItem('walletAddress');
1082
+ if (storedWalletAddress === addressToUse) {
1083
+ try {
1084
+ const autoConnectResult = await connectWallet(false); // Don't show sign-in message yet
1085
+ if (autoConnectResult) {
1086
+ // Wait for state to update
1087
+ await new Promise(resolve => setTimeout(resolve, 1000));
1088
+ // Get the updated signer
1089
+ signerToUse = signerRef.current || signer;
1090
+ addressToUse = hitAddressRef.current || hitAddress;
1091
+ }
1092
+ } catch (error) {
1093
+ console.error('Auto-connect failed during sign in:', error);
1094
+ }
1095
+ }
1096
+ }
1097
+
1098
+ if (!addressToUse) {
1099
+ console.log('No address available, returning early');
1100
+ if (autoShowSuccessMessage) {
1101
+ showError(walletTextUtils.getWalletText('Please connect your wallet first.', serviceId));
1102
+ }
1103
+ return;
1104
+ }
1105
+
1106
+ if (!signerToUse) {
1107
+ console.log('No signer available, returning early');
1108
+ if (autoShowSuccessMessage) {
1109
+ showError(walletTextUtils.getWalletText('Please connect your wallet first.', serviceId));
1110
+ }
1111
+ return;
1112
+ }
1113
+
1114
+ try {
1115
+ const nonceResponse = await nxtlinqApi.auth.getNonce({ address: addressToUse! });
1116
+ if ('error' in nonceResponse) {
1117
+ if (autoShowSuccessMessage) {
1118
+ showError(normalizeErrorForUser(nonceResponse.error));
1119
+ }
1120
+ return;
1121
+ }
1122
+
1123
+ const payload = {
1124
+ address: addressToUse!,
1125
+ code: nonceResponse.code,
1126
+ timestamp: nonceResponse.timestamp
1127
+ };
1128
+
1129
+ const stringToSign = stringify(payload);
1130
+ const signature = await signerToUse.signMessage(stringToSign || '');
1131
+
1132
+ const response = await nxtlinqApi.auth.signIn({
1133
+ ...payload,
1134
+ signature
1135
+ });
1136
+
1137
+ if ('error' in response) {
1138
+ if (autoShowSuccessMessage) {
1139
+ showError(normalizeErrorForUser(response.error));
1140
+ }
1141
+ return;
1142
+ }
1143
+ const { accessToken } = response;
1144
+ setNxtlinqAITServiceAccessToken(accessToken);
1145
+
1146
+ // Wait for state to update before refreshing AIT
1147
+ await new Promise(resolve => setTimeout(resolve, 100));
1148
+ await refreshAIT();
1149
+
1150
+ if (autoShowSuccessMessage) {
1151
+ showSuccess(walletTextUtils.getWalletText('Successfully signed in with your HIT wallet. You can now use the AI agent.', serviceId));
1152
+ }
1153
+ } catch (error) {
1154
+ console.error('Failed to sign in:', error);
1155
+ if (autoShowSuccessMessage) {
1156
+ showError('Failed to sign in. Please try again.');
1157
+ }
1158
+ }
1159
+ };
1160
+
1161
+ // Check permissions
1162
+ const hasPermission = async (requiredPermission: string, autoRetry = true, onAutoConnect?: () => void, onAutoSignIn?: () => void): Promise<boolean> => {
1163
+ // Use refs to get latest state values
1164
+ const currentHitAddress = hitAddressRef.current;
1165
+ const currentAit = aitRef.current;
1166
+ const currentPermissions = permissionsRef.current;
1167
+ const currentToken = nxtlinqAITServiceAccessTokenRef.current;
1168
+
1169
+ if (!currentHitAddress) {
1170
+ if (autoRetry) {
1171
+ setIsLoading(false); // Stop thinking before showing message
1172
+ setMessages(prev => [...prev, {
1173
+ id: Date.now().toString(),
1174
+ content: walletTextUtils.getWalletText('Please connect your HIT wallet to continue.', serviceId),
1175
+ role: 'assistant',
1176
+ timestamp: new Date().toISOString(),
1177
+ button: 'connectWallet'
1178
+ }]);
1179
+
1180
+ try {
1181
+ setIsAutoConnecting(true); // Mark as auto-connecting
1182
+ await connectWallet(false); // Don't show sign-in message yet
1183
+ setIsAutoConnecting(false); // Clear auto-connecting state
1184
+
1185
+ // Show brief success message for auto-connect
1186
+ showSuccess(walletTextUtils.getWalletText('Auto wallet connection successful', serviceId));
1187
+
1188
+
1189
+ onAutoConnect?.(); // Call callback if provided
1190
+ await new Promise(resolve => setTimeout(resolve, 2000));
1191
+
1192
+ // After auto connect, if not signed in, then auto sign-in
1193
+ const tokenAfterConnect = nxtlinqAITServiceAccessTokenRef.current;
1194
+ if (!tokenAfterConnect) {
1195
+ setIsAutoConnecting(true);
1196
+ await signInWallet(false);
1197
+ onAutoSignIn?.(); // Call callback if provided
1198
+ setIsAutoConnecting(false);
1199
+ showSuccess(walletTextUtils.getWalletText('Auto sign-in successful after wallet connect', serviceId));
1200
+ await refreshAIT();
1201
+ // Wait for AIT to be fully loaded with polling
1202
+ let attempts = 0;
1203
+ const maxAttempts = 5;
1204
+ while (!aitRef.current && attempts < maxAttempts) {
1205
+ await new Promise(resolve => setTimeout(resolve, 2000));
1206
+ attempts++;
1207
+ }
1208
+ }
1209
+
1210
+ // If connection (and sign-in if needed) successful, continue with permission check
1211
+ const result = await hasPermission(requiredPermission, false);
1212
+ return result;
1213
+ } catch (error) {
1214
+ console.error('Failed to auto-connect wallet:', error);
1215
+ setIsAutoConnecting(false); // Clear auto-connecting state on error
1216
+ return false;
1217
+ }
1218
+ }
1219
+ // If autoRetry is false, don't show message again, just return false
1220
+ return false;
1221
+ }
1222
+
1223
+ if (!currentToken) {
1224
+ if (autoRetry) {
1225
+ setIsLoading(false); // Stop thinking before showing message
1226
+ setMessages(prev => [...prev, {
1227
+ id: Date.now().toString(),
1228
+ content: walletTextUtils.getWalletText('Please sign in with your HIT wallet to continue.', serviceId),
1229
+ role: 'assistant',
1230
+ timestamp: new Date().toISOString(),
1231
+ button: 'signIn'
1232
+ }]);
1233
+
1234
+ try {
1235
+ setIsAutoConnecting(true); // Mark as auto-signing
1236
+ await signInWallet(false); // Don't show success message yet
1237
+ onAutoSignIn?.(); // Call callback if provided
1238
+ setIsAutoConnecting(false); // Clear auto-signing state
1239
+
1240
+ // Show brief success message for auto-sign-in
1241
+ showSuccess('Auto sign-in successful');
1242
+
1243
+ // Ensure AIT is refreshed after sign-in
1244
+ await refreshAIT();
1245
+
1246
+ // Wait for AIT to be fully loaded with polling
1247
+ let attempts = 0;
1248
+ const maxAttempts = 5; // Wait up to 10 seconds (5 * 2000ms)
1249
+ while (!aitRef.current && attempts < maxAttempts) {
1250
+ await new Promise(resolve => setTimeout(resolve, 2000));
1251
+ attempts++;
1252
+ }
1253
+
1254
+ // Only continue if AIT is actually loaded
1255
+ if (aitRef.current) {
1256
+ // Wait a bit more to ensure permissions are also loaded
1257
+ await new Promise(resolve => setTimeout(resolve, 2000));
1258
+ // If sign-in successful, continue with permission check
1259
+ const result = await hasPermission(requiredPermission, false);
1260
+ return result;
1261
+ } else {
1262
+ return false;
1263
+ }
1264
+ } catch (error) {
1265
+ console.error('Failed to auto-sign-in wallet:', error);
1266
+ setIsAutoConnecting(false); // Clear auto-signing state on error
1267
+ return false;
1268
+ }
1269
+ }
1270
+ // If autoRetry is false, don't show message again, just return false
1271
+ return false;
1272
+ }
1273
+
1274
+ try {
1275
+ const payload = JSON.parse(atob(currentToken.split('.')[1]));
1276
+ const address = payload.address;
1277
+ if (address !== currentHitAddress) {
1278
+ setNxtlinqAITServiceAccessToken('');
1279
+ if (autoRetry) {
1280
+ setIsLoading(false); // Stop thinking before showing message
1281
+ setMessages(prev => [...prev, {
1282
+ id: Date.now().toString(),
1283
+ content: walletTextUtils.getWalletText('Wallet address mismatch. Please sign in with the correct wallet.', serviceId),
1284
+ role: 'assistant',
1285
+ timestamp: new Date().toISOString(),
1286
+ button: 'signIn'
1287
+ }]);
1288
+
1289
+ try {
1290
+ setIsAutoConnecting(true); // Mark as auto-signing
1291
+ await signInWallet(false); // Don't show success message yet
1292
+ onAutoSignIn?.(); // Call callback if provided
1293
+ setIsAutoConnecting(false); // Clear auto-signing state
1294
+
1295
+ // Show brief success message for auto-sign-in after address mismatch
1296
+ showSuccess('Auto sign-in successful after address mismatch');
1297
+
1298
+ // Ensure AIT is refreshed after sign-in
1299
+ await refreshAIT();
1300
+
1301
+ // Wait for AIT to be fully loaded with polling
1302
+ let attempts = 0;
1303
+ const maxAttempts = 5; // Wait up to 10 seconds (5 * 2000ms)
1304
+ while (!aitRef.current && attempts < maxAttempts) {
1305
+ await new Promise(resolve => setTimeout(resolve, 2000));
1306
+ attempts++;
1307
+ }
1308
+
1309
+ // Only continue if AIT is actually loaded
1310
+ if (aitRef.current) {
1311
+ // Wait a bit more to ensure permissions are also loaded
1312
+ await new Promise(resolve => setTimeout(resolve, 2000));
1313
+ // If sign-in successful, continue with permission check
1314
+ const result = await hasPermission(requiredPermission, false);
1315
+ return result;
1316
+ } else {
1317
+ return false;
1318
+ }
1319
+ } catch (error) {
1320
+ console.error('Failed to auto-sign-in after address mismatch:', error);
1321
+ setIsAutoConnecting(false); // Clear auto-signing state on error
1322
+ return false;
1323
+ }
1324
+ }
1325
+ // If autoRetry is false, don't show message again, just return false
1326
+ return false;
1327
+ }
1328
+ } catch (error) {
1329
+ console.error('Error parsing token:', error);
1330
+ setNxtlinqAITServiceAccessToken('');
1331
+ if (autoRetry) {
1332
+ setIsLoading(false); // Stop thinking before showing message
1333
+ setMessages(prev => [...prev, {
1334
+ id: Date.now().toString(),
1335
+ content: walletTextUtils.getWalletText('Invalid wallet session. Please sign in again.', serviceId),
1336
+ role: 'assistant',
1337
+ timestamp: new Date().toISOString(),
1338
+ button: 'signIn'
1339
+ }]);
1340
+
1341
+ try {
1342
+ setIsAutoConnecting(true); // Mark as auto-signing
1343
+ await signInWallet(false); // Don't show success message yet
1344
+ onAutoSignIn?.(); // Call callback if provided
1345
+ setIsAutoConnecting(false); // Clear auto-signing state
1346
+
1347
+ // Show brief success message for auto-sign-in after token parse error
1348
+ showSuccess('Auto sign-in successful after token error');
1349
+
1350
+ // Ensure AIT is refreshed after sign-in
1351
+ await refreshAIT();
1352
+
1353
+ // Wait for AIT to be fully loaded with polling
1354
+ let attempts = 0;
1355
+ const maxAttempts = 5; // Wait up to 10 seconds (5 * 2000ms)
1356
+ while (!aitRef.current && attempts < maxAttempts) {
1357
+ await new Promise(resolve => setTimeout(resolve, 2000));
1358
+ attempts++;
1359
+ }
1360
+
1361
+ // Only continue if AIT is actually loaded
1362
+ if (aitRef.current) {
1363
+ // Wait a bit more to ensure permissions are also loaded
1364
+ await new Promise(resolve => setTimeout(resolve, 2000));
1365
+ // If sign-in successful, continue with permission check
1366
+ const result = await hasPermission(requiredPermission, false);
1367
+ return result;
1368
+ } else {
1369
+ return false;
1370
+ }
1371
+ } catch (signInError) {
1372
+ console.error('Failed to auto-sign-in after token parse error:', signInError);
1373
+ setIsAutoConnecting(false); // Clear auto-signing state on error
1374
+ return false;
1375
+ }
1376
+ }
1377
+ // If autoRetry is false, don't show message again, just return false
1378
+ return false;
1379
+ }
1380
+
1381
+ if (!currentAit) {
1382
+ // Show loading message if AIT is still loading
1383
+ if (isAITLoading) {
1384
+ setIsLoading(false); // Stop thinking before showing message
1385
+ setMessages(prev => [...prev, {
1386
+ id: Date.now().toString(),
1387
+ content: walletTextUtils.getWalletText('Loading your wallet configuration... Please wait a moment.', serviceId),
1388
+ role: 'assistant',
1389
+ timestamp: new Date().toISOString()
1390
+ }]);
1391
+ return false;
1392
+ }
1393
+
1394
+ // If AIT is not loaded but we have a token, try to refresh it once
1395
+ if (currentToken && !isAITLoading) {
1396
+ try {
1397
+ await refreshAIT();
1398
+ // Wait for AIT to be loaded with polling
1399
+ let attempts = 0;
1400
+ const maxAttempts = 5; // Wait up to 10 seconds (5 * 2000ms)
1401
+ while (!aitRef.current && attempts < maxAttempts) {
1402
+ await new Promise(resolve => setTimeout(resolve, 2000));
1403
+ attempts++;
1404
+ }
1405
+ // Check again after refresh
1406
+ if (!aitRef.current) {
1407
+ setIsLoading(false); // Stop thinking before showing message
1408
+ setMessages(prev => [...prev, {
1409
+ id: Date.now().toString(),
1410
+ content: walletTextUtils.getWalletText('No AIT found for your wallet. Please click the settings button (⚙️) to configure your AIT permissions.', serviceId),
1411
+ role: 'assistant',
1412
+ timestamp: new Date().toISOString()
1413
+ }]);
1414
+ return false;
1415
+ }
1416
+ } catch (error) {
1417
+ console.error('Failed to refresh AIT during permission check:', error);
1418
+ setIsLoading(false); // Stop thinking before showing message
1419
+ setMessages(prev => [...prev, {
1420
+ id: Date.now().toString(),
1421
+ content: 'No AIT found for your wallet. Please click the settings button (⚙️) to configure your AIT permissions.',
1422
+ role: 'assistant',
1423
+ timestamp: new Date().toISOString()
1424
+ }]);
1425
+ return false;
1426
+ }
1427
+ } else {
1428
+ setIsLoading(false); // Stop thinking before showing message
1429
+ setMessages(prev => [...prev, {
1430
+ id: Date.now().toString(),
1431
+ content: 'No AIT found for your wallet. Please click the settings button (⚙️) to configure your AIT permissions.',
1432
+ role: 'assistant',
1433
+ timestamp: new Date().toISOString()
1434
+ }]);
1435
+ return false;
1436
+ }
1437
+ }
1438
+
1439
+ if (availablePermissions.length === 0) {
1440
+ setIsLoading(false); // Stop thinking before showing message
1441
+ setMessages(prev => [...prev, {
1442
+ id: Date.now().toString(),
1443
+ content: `No permissions available for your current identity provider. Please check your service configuration or contact support. Service ID: ${serviceId}, Permission Group: ${permissionGroup || 'None'}`,
1444
+ role: 'assistant',
1445
+ timestamp: new Date().toISOString()
1446
+ }]);
1447
+ return false;
1448
+ }
1449
+
1450
+ const checkToolPermissionResult = await nxtlinqApi.agent.checkToolPermission({
1451
+ apiKey,
1452
+ apiSecret,
1453
+ aitToken: currentToken!,
1454
+ controller: currentHitAddress!,
1455
+ toolName: requiredPermission
1456
+ })
1457
+
1458
+ if ('error' in checkToolPermissionResult) {
1459
+ // Check if AIT is still loading or if we just auto-connected
1460
+ if (isAITLoading || isAutoConnecting) {
1461
+ setIsLoading(false); // Stop thinking before showing message
1462
+ setMessages(prev => [...prev, {
1463
+ id: Date.now().toString(),
1464
+ content: walletTextUtils.getWalletText('Loading your wallet configuration... Please wait a moment.', serviceId),
1465
+ role: 'assistant',
1466
+ timestamp: new Date().toISOString()
1467
+ }]);
1468
+ return false;
1469
+ }
1470
+
1471
+ // Refresh service permissions before judging; use latest result to decide message
1472
+ const freshAvailablePermissions = await fetchAvailablePermissions();
1473
+ const latestAvailablePermissions = freshAvailablePermissions ?? availablePermissions;
1474
+
1475
+ const requiredPermission = checkToolPermissionResult.requiredPermission;
1476
+ const isInServiceList = latestAvailablePermissions.some((p) => p.label === requiredPermission);
1477
+
1478
+ // Permission not in service's available list → "not available for your current identity provider"
1479
+ if (!isInServiceList) {
1480
+ setIsLoading(false); // Stop thinking before showing message
1481
+ setMessages(prev => [...prev, {
1482
+ id: Date.now().toString(),
1483
+ content: `This permission (${requiredPermission}) is not available for your current identity provider.`,
1484
+ role: 'assistant',
1485
+ timestamp: new Date().toISOString()
1486
+ }]);
1487
+ return false;
1488
+ }
1489
+
1490
+ // AIT loaded but permissions empty (no AIT found)
1491
+ if (currentAit && currentPermissions.length === 0) {
1492
+ setIsLoading(false); // Stop thinking before showing message
1493
+ setMessages(prev => [...prev, {
1494
+ id: Date.now().toString(),
1495
+ content: 'No AIT found for your wallet. Please click the settings button (⚙️) to configure your AIT permissions.',
1496
+ role: 'assistant',
1497
+ timestamp: new Date().toISOString()
1498
+ }]);
1499
+ return false;
1500
+ }
1501
+
1502
+ // User has AIT but hasn't enabled this permission → prompt to enable
1503
+ setIsLoading(false); // Stop thinking before showing message
1504
+ const permissionMsg = `You don't have the required AIT permission: ${requiredPermission}. Would you like to enable AIT permission?`;
1505
+ setMessages(prev => [
1506
+ ...prev,
1507
+ {
1508
+ id: Date.now().toString(),
1509
+ content: permissionMsg,
1510
+ role: 'assistant',
1511
+ timestamp: new Date().toISOString(),
1512
+ button: 'enableAIT',
1513
+ metadata: { requiredPermission }
1514
+ }
1515
+ ]);
1516
+ return false;
1517
+ }
1518
+
1519
+ return true;
1520
+ };
1521
+
1522
+ // AI Model related functions
1523
+ const handleModelChange = React.useCallback((modelIndex: number) => {
1524
+ setSelectedModelIndex(modelIndex);
1525
+ const selectedModel = effectiveAvailableModels[modelIndex];
1526
+ onModelChange?.(selectedModel);
1527
+ }, [effectiveAvailableModels, onModelChange, setSelectedModelIndex]);
1528
+
1529
+ const getCurrentModel = React.useCallback(() => {
1530
+ // Safety check: ensure selectedModelIndex is within bounds
1531
+ let safeIndex = selectedModelIndex;
1532
+
1533
+ if (selectedModelIndex >= effectiveAvailableModels.length) {
1534
+ safeIndex = 0;
1535
+ }
1536
+
1537
+ // If no models available, return a default model
1538
+ if (effectiveAvailableModels.length === 0) {
1539
+ return { label: 'Unknown', value: 'unknown' };
1540
+ }
1541
+
1542
+ return effectiveAvailableModels[safeIndex];
1543
+ }, [effectiveAvailableModels, selectedModelIndex]);
1544
+
1545
+ const updateSuggestions = React.useCallback(async (pseudoId: string, externalId?: string) => {
1546
+ const result = await nxtlinqApi.agent.generateSuggestions({
1547
+ apiKey,
1548
+ apiSecret,
1549
+ pseudoId,
1550
+ externalId
1551
+ });
1552
+
1553
+ if ('error' in result) {
1554
+ console.error('Failed to fetch suggestions:', result.error);
1555
+ setSuggestions([]);
1556
+ return;
1557
+ }
1558
+
1559
+ setSuggestions(result.suggestions.map((sug: string) => ({
1560
+ text: sug,
1561
+ autoSend: true
1562
+ })));
1563
+ }, [])
1564
+
1565
+ // Updated sendMessage function to support different AI models and attachments
1566
+ const sendMessage = async (
1567
+ content: string,
1568
+ retryCount = 0,
1569
+ isPresetMessage = false,
1570
+ attachments?: Attachment[],
1571
+ clientPipelineOverride?: Array<{ name: string; durationMs: number }>
1572
+ ): Promise<void> => {
1573
+ const hasContent = content.trim() || (attachments && attachments.length > 0);
1574
+ if (!hasContent || isLoading) return;
1575
+
1576
+ const currentModel = getCurrentModel();
1577
+ // Initialize with current model, will be updated with actual model from backend response
1578
+ let actualModelUsed = currentModel.value;
1579
+
1580
+ // Only add user message on first attempt, not on retries
1581
+ // Also check if this is not a preset message (preset messages are added separately)
1582
+ if (retryCount === 0 && !isPresetMessage) {
1583
+ const userMessage: Message = {
1584
+ id: Date.now().toString(),
1585
+ content: content || (attachments && attachments.length > 0 ? `Uploaded ${attachments.length} file(s)` : ''),
1586
+ role: 'user',
1587
+ timestamp: new Date().toISOString(),
1588
+ attachments: attachments,
1589
+ metadata: {
1590
+ model: currentModel.value,
1591
+ permissions: permissions,
1592
+ issuedBy: hitAddress || ''
1593
+ },
1594
+ piiStatus: piiDisplayMode === 'redacted' ? 'scanning' : undefined,
1595
+ };
1596
+ setMessages(prev => [...prev, userMessage]);
1597
+ setInputValue('');
1598
+ }
1599
+
1600
+ setIsLoading(true);
1601
+
1602
+ let streamAssistantId: string | null = null;
1603
+
1604
+ try {
1605
+ // Prepare attachments for API
1606
+ const apiAttachments = attachments?.map(att => ({
1607
+ type: att.type,
1608
+ url: att.url,
1609
+ name: att.name,
1610
+ mimeType: att.mimeType
1611
+ }));
1612
+
1613
+ let clientPipelineTimings: Array<{ name: string; durationMs: number }> | undefined;
1614
+ if (retryCount === 0) {
1615
+ if (clientPipelineOverride?.length) {
1616
+ clientPipelineTimings = clientPipelineOverride;
1617
+ } else if (micSessionStartForLogRef.current != null) {
1618
+ clientPipelineTimings = [
1619
+ {
1620
+ name: 'speech_to_text',
1621
+ durationMs: Math.round(performance.now() - micSessionStartForLogRef.current),
1622
+ },
1623
+ ];
1624
+ micSessionStartForLogRef.current = null;
1625
+ }
1626
+ }
1627
+
1628
+ const isAgentStreamModel = currentModel.value.endsWith('-stream');
1629
+ let streamedReplyBuffer = '';
1630
+ if (isAgentStreamModel && retryCount === 0) {
1631
+ const sid = `stream-assistant-${Date.now()}`;
1632
+ streamAssistantId = sid;
1633
+ setMessages(prev => [...prev, {
1634
+ id: sid,
1635
+ role: 'assistant',
1636
+ content: '',
1637
+ partialContent: '',
1638
+ isStreaming: true,
1639
+ timestamp: new Date().toISOString(),
1640
+ metadata: {
1641
+ model: currentModel.value,
1642
+ agentLlmStream: true,
1643
+ permissions: permissions,
1644
+ issuedBy: hitAddress || ''
1645
+ }
1646
+ }]);
1647
+ }
1648
+
1649
+ const response: AgentResponse = await nxtlinqApi.agent.sendMessage({
1650
+ model: currentModel.value,
1651
+ apiKey,
1652
+ apiSecret,
1653
+ pseudoId: pseudoId,
1654
+ externalId: localStorage.getItem('walletAddress') || undefined,
1655
+ customUserInfo,
1656
+ customUsername,
1657
+ message: content || (attachments && attachments.length > 0 ? `Uploaded ${attachments.length} file(s)` : ''),
1658
+ attachments: apiAttachments,
1659
+ isRetry: retryCount > 0,
1660
+ ...(clientPipelineTimings?.length ? { clientPipelineTimings } : {}),
1661
+ onStreamDelta: streamAssistantId
1662
+ ? (text: string) => {
1663
+ streamedReplyBuffer += text;
1664
+ flushSync(() => {
1665
+ setMessages(prev =>
1666
+ prev.map(m =>
1667
+ m.id === streamAssistantId
1668
+ ? { ...m, partialContent: (m.partialContent || '') + text }
1669
+ : m
1670
+ )
1671
+ );
1672
+ });
1673
+ if (text) {
1674
+ setIsLoading(false);
1675
+ }
1676
+ }
1677
+ : undefined,
1678
+ onPiiProgress: piiDisplayMode === 'redacted' ? (step: 'scan_start' | 'scan_complete' | 'send_start' | 'done', data?: any) => {
1679
+ const stepMap: Record<string, 0 | 1 | 2> = {
1680
+ scan_start: 0, // Scan active
1681
+ scan_complete: 1, // {N} protected — data available, show redacted
1682
+ send_start: 2, // Sending to AI
1683
+ };
1684
+ const piiStep = stepMap[step];
1685
+ if (piiStep === undefined) return;
1686
+ setMessages(prev => {
1687
+ const updated = [...prev];
1688
+ for (let i = updated.length - 1; i >= 0; i--) {
1689
+ if (updated[i].role === 'user' && updated[i].piiStatus === 'scanning') {
1690
+ const patch: Partial<typeof updated[0]> = { piiStep };
1691
+ // scan_complete carries anonymized data — set piiProtection early for immediate redact
1692
+ if (step === 'scan_complete' && data?.anonymizedUserMessage) {
1693
+ patch.piiProtection = {
1694
+ anonymizedContent: data.anonymizedUserMessage,
1695
+ mapping: data.mapping || {},
1696
+ };
1697
+ }
1698
+ updated[i] = { ...updated[i], ...patch };
1699
+ break;
1700
+ }
1701
+ }
1702
+ return updated;
1703
+ });
1704
+ } : undefined,
1705
+ });
1706
+
1707
+ if (!('error' in response && (response as { error?: string }).error)) {
1708
+ const tr = response as AgentResponse;
1709
+ if (
1710
+ tr.ttsVoice &&
1711
+ typeof tr.ttsVoice === 'object' &&
1712
+ (tr.ttsVoice.provider === 'azure' || tr.ttsVoice.provider === 'openai')
1713
+ ) {
1714
+ clientTtsVoiceRef.current = tr.ttsVoice;
1715
+ } else {
1716
+ clientTtsVoiceRef.current = undefined;
1717
+ }
1718
+ }
1719
+
1720
+ // Get the actual model used from response (may differ due to fallback)
1721
+ actualModelUsed = (response as any).model || currentModel.value;
1722
+
1723
+ if ('error' in response || (response as any).result === 'error') {
1724
+ // Check if it's an AIT-related error
1725
+ const errorResponse = response as any;
1726
+ if (errorResponse.requiresWalletConnection) {
1727
+ // Case where wallet connection is required - trigger auto-connect logic
1728
+ // Try to auto-connect wallet first
1729
+ const autoConnectResult = await connectWallet(false); // Don't show sign-in message yet
1730
+
1731
+ if (autoConnectResult) {
1732
+ // If auto-connect successful, try to sign in and then retry
1733
+ try {
1734
+ // Try to sign in wallet
1735
+ await signInWallet(false); // Don't show success message yet
1736
+
1737
+ if (isSemiAutomaticMode) {
1738
+ setIsLoading(false);
1739
+ setMessages(prev => [...prev, {
1740
+ id: Date.now().toString(),
1741
+ content: 'Click button to continue using tool',
1742
+ role: 'assistant',
1743
+ timestamp: new Date().toISOString(),
1744
+ button: 'continue'
1745
+ }]);
1746
+ return
1747
+ } else {
1748
+ // Wait a bit for everything to settle
1749
+ await new Promise(resolve => setTimeout(resolve, 2000));
1750
+
1751
+ // Retry the message
1752
+ sendMessage(content, retryCount + 1, isPresetMessage);
1753
+ }
1754
+ } catch (signInError) {
1755
+ setIsLoading(false);
1756
+ setMessages(prev => [...prev, {
1757
+ id: Date.now().toString(),
1758
+ content: 'Wallet connected but sign-in failed. Please try signing in manually.',
1759
+ role: 'assistant',
1760
+ timestamp: new Date().toISOString(),
1761
+ button: 'signIn',
1762
+ metadata: {
1763
+ requiresWalletConnection: true,
1764
+ toolName: errorResponse.toolName
1765
+ }
1766
+ }]);
1767
+ }
1768
+ return;
1769
+ } else {
1770
+ // If auto-connect failed, show the error message with connect button
1771
+ setIsLoading(false);
1772
+ setMessages(prev => [...prev, {
1773
+ id: Date.now().toString(),
1774
+ content: normalizeErrorForUser(errorResponse.message),
1775
+ role: 'assistant',
1776
+ timestamp: new Date().toISOString(),
1777
+ button: 'connectWallet',
1778
+ metadata: {
1779
+ requiresWalletConnection: true,
1780
+ toolName: errorResponse.toolName
1781
+ }
1782
+ }]);
1783
+ return;
1784
+ }
1785
+ } else if (errorResponse.requiresPermissionUpdate) {
1786
+ // Case where permission update is required - show manual enable option
1787
+ setIsLoading(false);
1788
+ setMessages(prev => [...prev, {
1789
+ id: Date.now().toString(),
1790
+ content: normalizeErrorForUser(errorResponse.message),
1791
+ role: 'assistant',
1792
+ timestamp: new Date().toISOString(),
1793
+ button: 'enableAIT',
1794
+ metadata: {
1795
+ requiresPermissionUpdate: true,
1796
+ toolName: errorResponse.toolName,
1797
+ requiredPermission: errorResponse.requiredPermission
1798
+ }
1799
+ }]);
1800
+ return;
1801
+ } else {
1802
+ // Other errors (show user-friendly message for invalid/expired token)
1803
+ throw new Error(normalizeErrorForUser(errorResponse.error || errorResponse.message));
1804
+ }
1805
+ }
1806
+
1807
+ let botResponse: Message | undefined;
1808
+
1809
+ let handledOpenAiStreamAssistant = false;
1810
+
1811
+ if (streamAssistantId && response.toolCall?.toolUse) {
1812
+ setMessages(prev => prev.filter(m => m.id !== streamAssistantId));
1813
+ streamAssistantId = null;
1814
+ }
1815
+
1816
+ if (streamAssistantId && !response.toolCall?.toolUse) {
1817
+ handledOpenAiStreamAssistant = true;
1818
+ let replyText = '';
1819
+ if (response.reply) {
1820
+ replyText = Array.isArray(response.reply)
1821
+ ? response.reply
1822
+ .map((item: any) => item.text)
1823
+ .join(' ') || ''
1824
+ : response.reply || '';
1825
+ } else {
1826
+ replyText = response.fullResponse?.output?.message?.content
1827
+ ?.find((item: any) => item.text)?.text || '';
1828
+ }
1829
+ if (!replyText.trim() && streamedReplyBuffer.trim()) {
1830
+ replyText = streamedReplyBuffer.trim();
1831
+ }
1832
+ if (!replyText.trim()) {
1833
+ replyText = 'Sorry, I cannot understand your question';
1834
+ }
1835
+
1836
+ updateSuggestions(pseudoId, localStorage.getItem('walletAddress') || undefined);
1837
+
1838
+ setMessages(prev => prev.map(m =>
1839
+ m.id === streamAssistantId
1840
+ ? {
1841
+ ...m,
1842
+ content: replyText,
1843
+ isStreaming: false,
1844
+ partialContent: undefined,
1845
+ metadata: {
1846
+ model: actualModelUsed,
1847
+ agentLlmStream: true,
1848
+ permissions: permissions,
1849
+ issuedBy: hitAddress || ''
1850
+ },
1851
+ piiProtection: response.piiProtection?.anonymizedReply
1852
+ ? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
1853
+ : undefined,
1854
+ }
1855
+ : m
1856
+ ));
1857
+
1858
+ botResponse = {
1859
+ id: streamAssistantId,
1860
+ content: replyText,
1861
+ role: 'assistant',
1862
+ timestamp: new Date().toISOString(),
1863
+ metadata: {
1864
+ model: actualModelUsed,
1865
+ agentLlmStream: true,
1866
+ permissions: permissions,
1867
+ issuedBy: hitAddress || ''
1868
+ },
1869
+ piiProtection: response.piiProtection?.anonymizedReply
1870
+ ? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
1871
+ : undefined,
1872
+ };
1873
+ }
1874
+
1875
+ // Store redirectUrl for later use (don't redirect yet)
1876
+ const redirectUrl = response?.redirectUrl;
1877
+
1878
+ if (!handledOpenAiStreamAssistant && response.toolCall?.toolUse) {
1879
+ // Handle tool calls
1880
+ const toolUse = response.toolCall.toolUse;
1881
+
1882
+ let toolMsg = '';
1883
+ if (onToolUse) {
1884
+ let wasAutoConnected = false;
1885
+ let wasAutoSignedIn = false;
1886
+ // Added: Mark if permission denied due to missing AIT permission
1887
+ let permissionDenied = false;
1888
+ // Use requiredPermission from response if available, otherwise fall back to toolUse.name
1889
+ const permissionToCheck = (response as any)?.requiredPermission || toolUse.name;
1890
+
1891
+ const isToolAllowed = await hasPermission(
1892
+ permissionToCheck,
1893
+ true,
1894
+ () => { wasAutoConnected = true },
1895
+ () => { wasAutoSignedIn = true }
1896
+ );
1897
+
1898
+ // If currentPermissions does not include permissionToCheck and availablePermissionLabels includes permissionToCheck, it means AIT permission is missing
1899
+ if (!isToolAllowed && !permissions.includes(permissionToCheck) && availablePermissions.map(p => p.label).includes(permissionToCheck)) {
1900
+ permissionDenied = true;
1901
+ }
1902
+ if (!isToolAllowed) {
1903
+ // If permission denied due to missing AIT permission return
1904
+ if (permissionDenied) {
1905
+ setIsLoading(false);
1906
+ return;
1907
+ }
1908
+
1909
+ if (isSemiAutomaticMode) {
1910
+ setIsLoading(false);
1911
+ setMessages(prev => [...prev, {
1912
+ id: Date.now().toString(),
1913
+ content: 'Click button to continue using tool',
1914
+ role: 'assistant',
1915
+ timestamp: new Date().toISOString(),
1916
+ button: 'continue'
1917
+ }]);
1918
+ return;
1919
+ } else {
1920
+ // Only retry for auto-connect/auto-sign-in scenarios
1921
+ if (wasAutoConnected && retryCount < 1) {
1922
+ // Clear loading state and retry immediately
1923
+ setIsLoading(false);
1924
+
1925
+ // Check if wallet is already signed in
1926
+ const currentToken = nxtlinqAITServiceAccessTokenRef.current;
1927
+
1928
+ if (!currentToken) {
1929
+ // If not signed in, directly retry the message without waiting for AIT
1930
+ setTimeout(() => {
1931
+ sendMessage(content, retryCount + 1, isPresetMessage);
1932
+ }, 2000);
1933
+ } else {
1934
+ // If already signed in, wait for AIT to be fully loaded before retrying
1935
+ setTimeout(async () => {
1936
+ // Wait for AIT to be loaded if needed
1937
+ if (!aitRef.current && nxtlinqAITServiceAccessTokenRef.current) {
1938
+ await refreshAIT();
1939
+ }
1940
+
1941
+ // Wait for AIT to be fully loaded with polling
1942
+ let attempts = 0;
1943
+ const maxAttempts = 5; // Wait up to 10 seconds (5 * 2000ms)
1944
+ while (!aitRef.current && attempts < maxAttempts) {
1945
+ await new Promise(resolve => setTimeout(resolve, 2000));
1946
+ attempts++;
1947
+ }
1948
+
1949
+ // Only retry if AIT is actually loaded
1950
+ if (aitRef.current) {
1951
+ // Wait a bit more to ensure permissions are also loaded
1952
+ await new Promise(resolve => setTimeout(resolve, 3000));
1953
+ sendMessage(content, retryCount + 1, isPresetMessage);
1954
+ }
1955
+ }, 2000);
1956
+ }
1957
+ }
1958
+ return;
1959
+ }
1960
+ }
1961
+
1962
+ if (isSemiAutomaticMode && wasAutoSignedIn) {
1963
+ setIsLoading(false);
1964
+ setMessages(prev => [...prev, {
1965
+ id: Date.now().toString(),
1966
+ content: 'Click button to continue using tool',
1967
+ role: 'assistant',
1968
+ timestamp: new Date().toISOString(),
1969
+ button: 'continue'
1970
+ }]);
1971
+ return;
1972
+ }
1973
+
1974
+ // Create streaming message for tool execution
1975
+ const streamingMessageId = `streaming-${Date.now()}`;
1976
+ const streamingMessage: Message = {
1977
+ id: streamingMessageId,
1978
+ content: '',
1979
+ role: 'assistant',
1980
+ timestamp: new Date().toISOString(),
1981
+ isStreaming: true,
1982
+ streamingToolName: toolUse.name,
1983
+ streamingStatus: `Starting ${toolUse.name}...`,
1984
+ streamingProgress: 0,
1985
+ streamingSteps: [],
1986
+ metadata: {
1987
+ model: actualModelUsed,
1988
+ permissions: permissions,
1989
+ issuedBy: hitAddress || '',
1990
+ toolUse: toolUse
1991
+ }
1992
+ };
1993
+
1994
+ // Add streaming message to chat
1995
+ setMessages(prev => [...prev, streamingMessage]);
1996
+
1997
+ // Execute tool with streaming updates
1998
+ const toolResult = await onToolUse(toolUse, (update) => {
1999
+ // Update streaming message with progress
2000
+ setMessages(prev => prev.map(msg =>
2001
+ msg.id === streamingMessageId
2002
+ ? {
2003
+ ...msg,
2004
+ content: update.partialResult || msg.content,
2005
+ partialContent: update.partialContent, // Token-level streaming
2006
+ streamingProgress: update.progress,
2007
+ streamingStatus: update.status,
2008
+ streamingSteps: update.steps || msg.streamingSteps
2009
+ }
2010
+ : msg
2011
+ ));
2012
+ });
2013
+
2014
+ toolMsg = toolResult?.content || `Tool ${toolUse.name} executed successfully`;
2015
+
2016
+ // Merge AI original reply
2017
+ let replyText = '';
2018
+ if (response.reply) {
2019
+ replyText = Array.isArray(response.reply)
2020
+ ? response.reply
2021
+ .map((item: any) => item.text)
2022
+ .join(' ') || ''
2023
+ : response.reply || '';
2024
+ } else if (response.fullResponse?.output?.message?.content) {
2025
+ replyText = response.fullResponse.output.message.content
2026
+ ?.find((item: any) => item.text)?.text || '';
2027
+ }
2028
+
2029
+ let finalContent = toolMsg;
2030
+ if (replyText && replyText.trim()) {
2031
+ finalContent += `\n\n${replyText}`;
2032
+ }
2033
+
2034
+ // Mark streaming as complete with final content
2035
+ setMessages(prev => prev.map(msg =>
2036
+ msg.id === streamingMessageId
2037
+ ? {
2038
+ ...msg,
2039
+ isStreaming: false,
2040
+ streamingProgress: 100,
2041
+ content: finalContent,
2042
+ metadata: {
2043
+ ...msg.metadata,
2044
+ model: actualModelUsed,
2045
+ toolUse: toolUse
2046
+ },
2047
+ piiProtection: response.piiProtection?.anonymizedReply
2048
+ ? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
2049
+ : undefined,
2050
+ }
2051
+ : msg
2052
+ ));
2053
+
2054
+ // Update message content in database if assistantMessageId is available
2055
+ if (response.assistantMessageId && apiKey && apiSecret) {
2056
+ try {
2057
+ await nxtlinqApi.agent.updateMessageContent({
2058
+ apiKey,
2059
+ apiSecret,
2060
+ messageId: response.assistantMessageId,
2061
+ content: finalContent,
2062
+ });
2063
+ } catch (error) {
2064
+ console.error('Failed to update message content in database:', error);
2065
+ // Don't block the UI if update fails
2066
+ }
2067
+ }
2068
+
2069
+ updateSuggestions(pseudoId, localStorage.getItem('walletAddress') || undefined);
2070
+
2071
+ // Skip creating a new botResponse since we already updated the streaming message
2072
+ } else {
2073
+ toolMsg = `Tool ${toolUse.name} executed successfully`;
2074
+
2075
+ // No onToolUse callback - create message without streaming
2076
+ let replyText = '';
2077
+ if (response.reply) {
2078
+ replyText = Array.isArray(response.reply)
2079
+ ? response.reply
2080
+ .map((item: any) => item.text)
2081
+ .join(' ') || ''
2082
+ : response.reply || '';
2083
+ } else if (response.fullResponse?.output?.message?.content) {
2084
+ replyText = response.fullResponse.output.message.content
2085
+ ?.find((item: any) => item.text)?.text || '';
2086
+ }
2087
+ let mergedContent = '';
2088
+ if (toolMsg) {
2089
+ mergedContent += toolMsg;
2090
+ }
2091
+ if (replyText && replyText.trim()) {
2092
+ mergedContent += `\n\n${replyText}`;
2093
+ }
2094
+
2095
+ // Update message content in database if assistantMessageId is available
2096
+ if (response.assistantMessageId && apiKey && apiSecret) {
2097
+ try {
2098
+ await nxtlinqApi.agent.updateMessageContent({
2099
+ apiKey,
2100
+ apiSecret,
2101
+ messageId: response.assistantMessageId,
2102
+ content: mergedContent,
2103
+ });
2104
+ } catch (error) {
2105
+ console.error('Failed to update message content in database:', error);
2106
+ // Don't block the UI if update fails
2107
+ }
2108
+ }
2109
+
2110
+ updateSuggestions(pseudoId, localStorage.getItem('walletAddress') || undefined);
2111
+ const newBotResponse: Message = {
2112
+ id: (Date.now() + 1).toString(),
2113
+ content: mergedContent,
2114
+ role: 'assistant',
2115
+ timestamp: new Date().toISOString(),
2116
+ metadata: {
2117
+ model: actualModelUsed,
2118
+ permissions: permissions,
2119
+ issuedBy: hitAddress || '',
2120
+ toolUse: toolUse
2121
+ },
2122
+ piiProtection: response.piiProtection?.anonymizedReply
2123
+ ? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
2124
+ : undefined,
2125
+ };
2126
+ setMessages(prev => [...prev, newBotResponse]);
2127
+ botResponse = newBotResponse;
2128
+ }
2129
+ } else if (!handledOpenAiStreamAssistant && response.reply) {
2130
+ const replyText = Array.isArray(response.reply)
2131
+ ? response.reply
2132
+ .map((item: any) => item.text)
2133
+ .join(' ') || 'Sorry, I cannot understand your question'
2134
+ : response.reply || 'Sorry, I cannot understand your question';
2135
+
2136
+ updateSuggestions(pseudoId, localStorage.getItem('walletAddress') || undefined);
2137
+
2138
+ const newBotResponse: Message = {
2139
+ id: (Date.now() + 1).toString(),
2140
+ content: replyText,
2141
+ role: 'assistant',
2142
+ timestamp: new Date().toISOString(),
2143
+ metadata: {
2144
+ model: actualModelUsed,
2145
+ permissions: permissions,
2146
+ issuedBy: hitAddress || ''
2147
+ },
2148
+ piiProtection: response.piiProtection?.anonymizedReply
2149
+ ? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
2150
+ : undefined,
2151
+ };
2152
+ setMessages(prev => [...prev, newBotResponse]);
2153
+ botResponse = newBotResponse;
2154
+ } else if (!handledOpenAiStreamAssistant) {
2155
+ const replyText = response.fullResponse?.output?.message?.content
2156
+ ?.find((item: any) => item.text)?.text || 'Sorry, I cannot understand your question';
2157
+
2158
+ const newBotResponse: Message = {
2159
+ id: (Date.now() + 1).toString(),
2160
+ content: replyText,
2161
+ role: 'assistant',
2162
+ timestamp: new Date().toISOString(),
2163
+ metadata: {
2164
+ model: actualModelUsed,
2165
+ permissions: permissions,
2166
+ issuedBy: hitAddress || ''
2167
+ },
2168
+ piiProtection: response.piiProtection?.anonymizedReply
2169
+ ? { anonymizedContent: response.piiProtection.anonymizedReply ?? undefined, mapping: response.piiProtection.mapping ?? undefined }
2170
+ : undefined,
2171
+ };
2172
+ setMessages(prev => [...prev, newBotResponse]);
2173
+ botResponse = newBotResponse;
2174
+ }
2175
+
2176
+ // ===== PII Protection: Update user message with anonymized version =====
2177
+ const anonymizedUserMsg = response.piiProtection?.anonymizedUserMessage;
2178
+ const piiMappingData = response.piiProtection?.mapping ?? undefined;
2179
+ if (anonymizedUserMsg) {
2180
+ setMessages(prev => {
2181
+ const updated = [...prev];
2182
+ for (let i = updated.length - 1; i >= 0; i--) {
2183
+ if (updated[i].role === 'user') {
2184
+ updated[i] = {
2185
+ ...updated[i],
2186
+ piiProtection: { anonymizedContent: anonymizedUserMsg, mapping: piiMappingData },
2187
+ piiStatus: 'complete',
2188
+ };
2189
+ break;
2190
+ }
2191
+ }
2192
+ return updated;
2193
+ });
2194
+ } else if (piiDisplayMode === 'redacted') {
2195
+ // No PII detected — set piiStatus to 'none' to trigger "No sensitive data" indicator
2196
+ setMessages(prev => {
2197
+ const updated = [...prev];
2198
+ for (let i = updated.length - 1; i >= 0; i--) {
2199
+ if (updated[i].role === 'user' && updated[i].piiStatus === 'scanning') {
2200
+ updated[i] = { ...updated[i], piiStatus: 'none' };
2201
+ break;
2202
+ }
2203
+ }
2204
+ return updated;
2205
+ });
2206
+ }
2207
+
2208
+ // Execute redirect after all message processing is complete
2209
+ if (redirectUrl) {
2210
+ // Use setTimeout to ensure the message is displayed before redirect
2211
+ setTimeout(() => {
2212
+ window.location.href = redirectUrl;
2213
+ }, 100);
2214
+ }
2215
+
2216
+ // Play text-to-speech for AI response if enabled
2217
+ if (botResponse && botResponse.role === 'assistant' && textToSpeechEnabled) {
2218
+ const ttsContent = botResponse.content;
2219
+ setTimeout(() => {
2220
+ playTextToSpeech(ttsContent).catch(error => {
2221
+ console.error('Failed to play text-to-speech:', error);
2222
+ });
2223
+ }, 100);
2224
+ }
2225
+ } catch (error) {
2226
+ console.error('Failed to send message:', error);
2227
+
2228
+ if (streamAssistantId) {
2229
+ setMessages(prev => prev.filter(m => m.id !== streamAssistantId));
2230
+ streamAssistantId = null;
2231
+ }
2232
+
2233
+ if (retryCount < maxRetries) {
2234
+ // Show retry indicator to user
2235
+ const retryMessage: Message = {
2236
+ id: `retry-${Date.now()}-${retryCount}`,
2237
+ content: `Retrying... (Attempt ${retryCount + 1}/${maxRetries})`,
2238
+ role: 'assistant',
2239
+ timestamp: new Date().toISOString(),
2240
+ metadata: {
2241
+ model: actualModelUsed,
2242
+ permissions: permissions,
2243
+ issuedBy: hitAddress || '',
2244
+ isRetry: true
2245
+ }
2246
+ };
2247
+ setMessages(prev => [...prev, retryMessage]);
2248
+
2249
+ setTimeout(() => {
2250
+ // Remove retry message before next attempt
2251
+ setMessages(prev => prev.filter(msg => msg.id !== retryMessage.id));
2252
+ sendMessage(content, retryCount + 1, isPresetMessage);
2253
+ }, retryDelay);
2254
+ return;
2255
+ }
2256
+
2257
+ // Clear piiStatus on error so stepper doesn't get stuck
2258
+ if (piiDisplayMode === 'redacted') {
2259
+ setMessages(prev => {
2260
+ const updated = [...prev];
2261
+ for (let i = updated.length - 1; i >= 0; i--) {
2262
+ if (updated[i].role === 'user' && updated[i].piiStatus === 'scanning') {
2263
+ updated[i] = { ...updated[i], piiStatus: undefined };
2264
+ break;
2265
+ }
2266
+ }
2267
+ return updated;
2268
+ });
2269
+ }
2270
+
2271
+ const rawMessage = error instanceof Error ? error.message : 'An error occurred while sending the message';
2272
+ const errorMessage: Message = {
2273
+ id: (Date.now() + 1).toString(),
2274
+ content: normalizeErrorForUser(rawMessage),
2275
+ role: 'assistant',
2276
+ timestamp: new Date().toISOString(),
2277
+ metadata: {
2278
+ model: currentModel.value,
2279
+ permissions: permissions,
2280
+ issuedBy: hitAddress || ''
2281
+ }
2282
+ };
2283
+ setMessages(prev => [...prev, errorMessage]);
2284
+ onError?.(error instanceof Error ? error : new Error('Unknown error'));
2285
+ } finally {
2286
+ setIsLoading(false);
2287
+ }
2288
+ };
2289
+
2290
+ // Handle submit
2291
+ const handleSubmit = async (e: React.FormEvent, attachments?: Attachment[]) => {
2292
+ e.preventDefault();
2293
+
2294
+ if (isStopRecordingOnSend) {
2295
+ while (isMicEnabledRef.current) {
2296
+ stopRecording();
2297
+ await sleep(1000);
2298
+ }
2299
+ } else {
2300
+ if (isMicEnabledRef.current) {
2301
+ await sleep(1000);
2302
+ clearRecording();
2303
+ }
2304
+ }
2305
+
2306
+ if (!textInputRef.current) return;
2307
+
2308
+ const hasContent = textInputRef.current.value.trim() || (attachments && attachments.length > 0);
2309
+ if (!hasContent || isLoading) return;
2310
+
2311
+ // 如果此時有語音自動發送的計時器,優先取消,避免與手動發送同內容時重複送出
2312
+ if (autoSendTimerRef.current) {
2313
+ clearTimeout(autoSendTimerRef.current as NodeJS.Timeout);
2314
+ autoSendTimerRef.current = null;
2315
+ }
2316
+
2317
+ // 標記這次手動送出的內容,讓語音自動發送邏輯視為已送出,避免同一段文字再自動發送一次
2318
+ const manualContent = textInputRef.current.value.trim();
2319
+ if (manualContent) {
2320
+ lastAutoSentTranscriptRef.current = manualContent;
2321
+ }
2322
+
2323
+ // Stop current speech playback when user sends new message
2324
+ stopTextToSpeechAndReset();
2325
+
2326
+ // Show loading message if AIT is still loading
2327
+ if (isAITLoading) {
2328
+ const loadingMessage: Message = {
2329
+ id: Date.now().toString(),
2330
+ content: 'Loading your wallet configuration... Please wait a moment.',
2331
+ role: 'assistant',
2332
+ timestamp: new Date().toISOString(),
2333
+ metadata: {
2334
+ model: getCurrentModel().value,
2335
+ permissions: permissions,
2336
+ issuedBy: hitAddress || ''
2337
+ }
2338
+ };
2339
+ setMessages(prev => [...prev, loadingMessage]);
2340
+ return;
2341
+ }
2342
+
2343
+ try {
2344
+ const clientPipelineOverride =
2345
+ micSessionStartForLogRef.current != null
2346
+ ? [
2347
+ {
2348
+ name: 'speech_to_text',
2349
+ durationMs: Math.round(performance.now() - micSessionStartForLogRef.current),
2350
+ },
2351
+ ]
2352
+ : undefined;
2353
+ if (micSessionStartForLogRef.current != null) {
2354
+ micSessionStartForLogRef.current = null;
2355
+ }
2356
+ await sendMessage(textInputRef.current.value, 0, false, attachments, clientPipelineOverride);
2357
+ } catch (error) {
2358
+ console.error('Failed to send message:', error);
2359
+ onError?.(error instanceof Error ? error : new Error('Failed to send message'));
2360
+ const errorMessage: Message = {
2361
+ id: Date.now().toString(),
2362
+ content: 'Sorry, there was an error processing your message. Please try again.',
2363
+ role: 'assistant',
2364
+ timestamp: new Date().toISOString(),
2365
+ metadata: {
2366
+ model: getCurrentModel().value,
2367
+ permissions: permissions,
2368
+ issuedBy: hitAddress || ''
2369
+ }
2370
+ };
2371
+ setMessages(prev => [...prev, errorMessage]);
2372
+ }
2373
+ };
2374
+
2375
+ // Stop text-to-speech (does not clear queue)
2376
+ const stopTextToSpeech = React.useCallback(() => {
2377
+ if (speechingRef.current) {
2378
+ // Stop AudioContext source if exists
2379
+ if (audioSourceRef.current) {
2380
+ audioSourceRef.current.stop();
2381
+ audioSourceRef.current.disconnect();
2382
+ audioSourceRef.current = null;
2383
+ }
2384
+ // Stop HTML Audio element if exists
2385
+ if (audioElementRef.current) {
2386
+ audioElementRef.current.pause();
2387
+ audioElementRef.current.currentTime = 0;
2388
+ if (audioElementRef.current.src.startsWith('blob:')) {
2389
+ URL.revokeObjectURL(audioElementRef.current.src);
2390
+ }
2391
+ audioElementRef.current = null;
2392
+ }
2393
+ speechingRef.current = false;
2394
+ }
2395
+ setSpeechingIndex(undefined);
2396
+ }, []);
2397
+
2398
+ // Stop text-to-speech (used when user sends new message)
2399
+ const stopTextToSpeechAndReset = React.useCallback(() => {
2400
+ stopTextToSpeech();
2401
+ setRequiresGesture(false);
2402
+ setIsTtsProcessing(false);
2403
+ pendingTtsRef.current = null;
2404
+ }, [stopTextToSpeech]);
2405
+
2406
+ // When TTS is disabled during playback, stop immediately
2407
+ React.useEffect(() => {
2408
+ if (!textToSpeechEnabled) {
2409
+ stopTextToSpeechAndReset();
2410
+ }
2411
+ }, [textToSpeechEnabled, stopTextToSpeechAndReset]);
2412
+
2413
+ const {
2414
+ isVoiceMode,
2415
+ voiceStatus,
2416
+ isVoiceConnecting,
2417
+ isMicMuted,
2418
+ remoteAudioRef,
2419
+ enterVoiceMode,
2420
+ exitVoiceMode,
2421
+ toggleMicMute: toggleVoiceMicMute,
2422
+ interruptVoice,
2423
+ } = useVoiceMode({
2424
+ apiKey,
2425
+ apiSecret,
2426
+ pseudoId,
2427
+ externalId: hitAddress || undefined,
2428
+ aitId: ait?.aitId,
2429
+ walletAddress: hitAddress || undefined,
2430
+ aitToken: nxtlinqAITServiceAccessToken || undefined,
2431
+ metadata: customUserInfo,
2432
+ nxtlinqApi,
2433
+ getMessages: () => messagesRef.current,
2434
+ setMessages,
2435
+ onError,
2436
+ stopRecording,
2437
+ stopTextToSpeech: stopTextToSpeechAndReset,
2438
+ voiceTransport: 'ws-realtime',
2439
+ });
2440
+
2441
+ React.useEffect(() => {
2442
+ if (!isOpen && isVoiceMode) {
2443
+ void exitVoiceMode();
2444
+ }
2445
+ }, [isOpen, isVoiceMode, exitVoiceMode]);
2446
+
2447
+ React.useEffect(() => {
2448
+ if (isOpen && preferredChatMode === 'voice' && !isVoiceMode && !isVoiceConnecting) {
2449
+ void enterVoiceMode();
2450
+ }
2451
+ }, [isOpen, preferredChatMode, isVoiceMode, isVoiceConnecting, enterVoiceMode]);
2452
+
2453
+ React.useEffect(() => {
2454
+ if (isVoiceMode || isVoiceConnecting) {
2455
+ stopRecording();
2456
+ }
2457
+ }, [isVoiceMode, isVoiceConnecting, stopRecording]);
2458
+
2459
+ const handleEnterVoiceMode = React.useCallback(async () => {
2460
+ stopRecording();
2461
+ setPreferredChatMode('voice');
2462
+ await enterVoiceMode();
2463
+ }, [enterVoiceMode, setPreferredChatMode, stopRecording]);
2464
+
2465
+ const handleExitVoiceMode = React.useCallback(async () => {
2466
+ setPreferredChatMode('text');
2467
+ await exitVoiceMode();
2468
+ }, [exitVoiceMode, setPreferredChatMode]);
2469
+
2470
+ // Play text-to-speech (simplified, no queue)
2471
+ const playTextToSpeech = React.useCallback(async (text: string, messageIndex?: number): Promise<void> => {
2472
+ if (!textToSpeechEnabled || !text.trim()) return;
2473
+
2474
+ // Stop any ongoing speech
2475
+ stopTextToSpeech();
2476
+
2477
+ // Show processing indicator
2478
+ setIsTtsProcessing(true);
2479
+
2480
+ // Retry up to 2 times (3 attempts total)
2481
+ let lastError: any;
2482
+ for (let attempt = 0; attempt < 3; attempt++) {
2483
+ try {
2484
+ await playTextToSpeechWithRetry(text, messageIndex, attempt);
2485
+ return; // Success, exit
2486
+ } catch (err) {
2487
+ lastError = err;
2488
+ if (attempt < 2) {
2489
+ // Continue to next attempt
2490
+ continue;
2491
+ }
2492
+ }
2493
+ }
2494
+
2495
+ // All attempts failed
2496
+ console.error('TTS playback failed after retries:', lastError);
2497
+ setIsTtsProcessing(false);
2498
+ }, [textToSpeechEnabled, stopTextToSpeech, playTextToSpeechWithRetry]);
2499
+
2500
+ const retryTtsWithGesture = React.useCallback(async () => {
2501
+ try {
2502
+ setRequiresGesture(false);
2503
+ // Try to resume audio context
2504
+ if (audioCtxRef.current) {
2505
+ try { await audioCtxRef.current.resume(); } catch { }
2506
+ }
2507
+ const pending = pendingTtsRef.current;
2508
+ if (pending && textToSpeechEnabled) {
2509
+ setIsTtsProcessing(true);
2510
+ await playTextToSpeech(pending);
2511
+ pendingTtsRef.current = null;
2512
+ }
2513
+ } catch (err) {
2514
+ console.error('Retry TTS with gesture failed:', err);
2515
+ setIsTtsProcessing(false);
2516
+ }
2517
+ }, [textToSpeechEnabled, playTextToSpeech]);
2518
+
2519
+ // When browser blocks autoplay (requires gesture), retry on next user interaction
2520
+ React.useEffect(() => {
2521
+ if (!requiresGesture || !textToSpeechEnabled || !pendingTtsRef.current) return;
2522
+ const handleUserGesture = () => {
2523
+ retryTtsWithGesture();
2524
+ };
2525
+ window.addEventListener('pointerdown', handleUserGesture, { once: true });
2526
+ return () => window.removeEventListener('pointerdown', handleUserGesture);
2527
+ }, [requiresGesture, textToSpeechEnabled, retryTtsWithGesture]);
2528
+
2529
+ const uploadAttachment = React.useCallback(async (file: File): Promise<{ url: string } | { error: string }> => {
2530
+ const result = await nxtlinqApi.agent.uploadAttachment({
2531
+ apiKey,
2532
+ apiSecret,
2533
+ pseudoId,
2534
+ file,
2535
+ });
2536
+ if ('error' in result) return result;
2537
+ return { url: result.url };
2538
+ }, [nxtlinqApi, apiKey, apiSecret, pseudoId]);
2539
+
2540
+ // Handle preset message
2541
+ const handlePresetMessage = (message: PresetMessage) => {
2542
+ // If preset is configured as auto-send, avoid duplicate sends when user clicks repeatedly
2543
+ if (message.autoSend) {
2544
+ const trimmedText = (message.text || '').trim();
2545
+ if (!trimmedText) return;
2546
+
2547
+ // Prevent sending messages while AI Agent is processing
2548
+ if (isLoading) {
2549
+ return;
2550
+ }
2551
+
2552
+ // If this exact preset text was just sent (by auto-send / manual / preset), skip to prevent duplicates
2553
+ if (lastAutoSentTranscriptRef.current === trimmedText) {
2554
+ return;
2555
+ }
2556
+
2557
+ // For preset messages, we need to add the user message first since sendMessage won't add it on retries
2558
+ const userMessage: Message = {
2559
+ id: Date.now().toString(),
2560
+ content: trimmedText,
2561
+ role: 'user',
2562
+ timestamp: new Date().toISOString(),
2563
+ metadata: {
2564
+ model: getCurrentModel().value,
2565
+ permissions: permissions,
2566
+ issuedBy: hitAddress || ''
2567
+ }
2568
+ };
2569
+ setMessages(prev => [...prev, userMessage]);
2570
+
2571
+ // Mark as last sent to guard against rapid re-clicks and other duplicate flows
2572
+ lastAutoSentTranscriptRef.current = trimmedText;
2573
+
2574
+ // Pass a flag to indicate this is a preset message so sendMessage won't add user message again
2575
+ sendMessage(trimmedText, 0, true);
2576
+ } else {
2577
+ setInputValue(message.text);
2578
+ }
2579
+ };
2580
+
2581
+ // Generate and register AIT
2582
+ const generateAndRegisterAIT = async (newPermissions?: string[], isFromAIAgent = false) => {
2583
+ let currentSigner = signer;
2584
+ let currentAddress = hitAddress;
2585
+
2586
+ // If we have an address but no signer, try to auto-connect wallet
2587
+ if (currentAddress && !currentSigner) {
2588
+ const storedWalletAddress = localStorage.getItem('walletAddress');
2589
+ if (storedWalletAddress === currentAddress) {
2590
+ try {
2591
+ const autoConnectResult = await connectWallet(false); // Don't show sign-in message yet
2592
+ if (autoConnectResult) {
2593
+ // Wait for state to update
2594
+ await new Promise(resolve => setTimeout(resolve, 1000));
2595
+ // Get the updated signer and address
2596
+ currentSigner = signerRef.current || signer;
2597
+ currentAddress = hitAddressRef.current || hitAddress;
2598
+ }
2599
+ } catch (error) {
2600
+ console.error('Auto-connect failed during AIT generation:', error);
2601
+ }
2602
+ }
2603
+ }
2604
+
2605
+ if (!currentSigner || !currentAddress) {
2606
+ throw new Error('Missing signer or wallet address');
2607
+ }
2608
+
2609
+ return generateAndRegisterAITWithSigner(newPermissions, isFromAIAgent, currentSigner, currentAddress);
2610
+ };
2611
+
2612
+ // Generate and register AIT with explicit signer and address
2613
+ const generateAndRegisterAITWithSigner = async (
2614
+ newPermissions?: string[],
2615
+ isFromAIAgent = false,
2616
+ explicitSigner?: ethers.JsonRpcSigner,
2617
+ explicitAddress?: string
2618
+ ) => {
2619
+ const currentSigner = explicitSigner || signer;
2620
+ const currentAddress = explicitAddress || hitAddress;
2621
+
2622
+ if (!currentSigner || !currentAddress) {
2623
+ throw new Error('Missing signer or wallet address');
2624
+ }
2625
+
2626
+ // If AI Agent is creating, must check if user has existing AIT
2627
+ if (isFromAIAgent && !aitRef.current) {
2628
+ throw new Error('You must have an existing AIT before AI Agent can create new AITs');
2629
+ }
2630
+
2631
+ const timestamp = Math.floor(Date.now() / 1000);
2632
+ const aitId = `did:polygon:nxtlinq:${currentAddress}:${timestamp}`;
2633
+
2634
+ const metadata = {
2635
+ permissions: newPermissions || permissions,
2636
+ issuedBy: currentAddress,
2637
+ };
2638
+
2639
+ const metadataStr = stringify(metadata);
2640
+ // ethers v6: utils.keccak256 -> keccak256, utils.toUtf8Bytes -> toUtf8Bytes
2641
+ const metadataHash = ethers.keccak256(ethers.toUtf8Bytes(metadataStr));
2642
+
2643
+ const uploadResponse = await nxtlinqApi.metadata.createMetadata(metadata, nxtlinqAITServiceAccessToken || '');
2644
+ if ('error' in uploadResponse) {
2645
+ throw new Error(`Failed to upload metadata: ${uploadResponse.error}`);
2646
+ }
2647
+
2648
+ const { metadataCid } = uploadResponse;
2649
+
2650
+ const createAITParams = {
2651
+ aitId,
2652
+ controller: currentAddress,
2653
+ serviceId,
2654
+ metadataHash,
2655
+ metadataCid,
2656
+ isFromAIAgent: isFromAIAgent,
2657
+ parentAITId: isFromAIAgent ? aitRef.current?.aitId : undefined,
2658
+ };
2659
+
2660
+ const createAITResponse = await nxtlinqApi.ait.createAIT(createAITParams, nxtlinqAITServiceAccessToken || '');
2661
+
2662
+ if ('error' in createAITResponse) {
2663
+ throw new Error(`Failed to create AIT: ${createAITResponse.error}`);
2664
+ }
2665
+
2666
+ const aitInfo = {
2667
+ aitId,
2668
+ controller: currentAddress,
2669
+ metadata,
2670
+ metadataHash,
2671
+ metadataCid,
2672
+ };
2673
+
2674
+ setAit(aitInfo);
2675
+ setPermissions(newPermissions || permissions);
2676
+ };
2677
+
2678
+
2679
+
2680
+ // Auto enable AIT permission
2681
+ const enableAIT = async (toolName: string) => {
2682
+ if (isAITEnabling) return false; // Prevent duplicate
2683
+ setIsAITEnabling(true);
2684
+
2685
+ let currentSigner = signer;
2686
+ let currentHitAddress = hitAddress;
2687
+
2688
+ // If no wallet connected, try to auto-connect first
2689
+ if (!currentSigner || !currentHitAddress) {
2690
+ const autoConnectResult = await connectWallet(false); // Don't show sign-in message yet
2691
+
2692
+ if (autoConnectResult) {
2693
+ // Wait for state to update - increase wait time
2694
+ await new Promise(resolve => setTimeout(resolve, 1500));
2695
+
2696
+ // Get the latest state values after connection
2697
+ currentSigner = signerRef.current || signer;
2698
+ currentHitAddress = hitAddressRef.current || hitAddress;
2699
+
2700
+ // Double check that we have the required state
2701
+ if (!currentSigner || !currentHitAddress) {
2702
+ await new Promise(resolve => setTimeout(resolve, 1500));
2703
+ currentSigner = signerRef.current || signer;
2704
+ currentHitAddress = hitAddressRef.current || hitAddress;
2705
+ }
2706
+
2707
+ try {
2708
+ await signInWallet(false); // Don't show success message yet
2709
+ } catch (signInError) {
2710
+ showError(walletTextUtils.getWalletText('Wallet connected but sign-in failed. Please try signing in manually.', serviceId));
2711
+ setIsAITEnabling(false);
2712
+ return false;
2713
+ }
2714
+ } else {
2715
+ showError(walletTextUtils.getWalletText('Please connect your wallet first', serviceId));
2716
+ setIsAITEnabling(false);
2717
+ return false;
2718
+ }
2719
+ }
2720
+
2721
+ // Final check before proceeding
2722
+ if (!currentSigner || !currentHitAddress) {
2723
+ console.error('Missing signer or wallet address after connection process');
2724
+ console.error('Current signer:', !!currentSigner);
2725
+ console.error('Current hitAddress:', currentHitAddress);
2726
+ console.error('Signer ref:', !!signerRef.current);
2727
+ console.error('HitAddress ref:', hitAddressRef.current);
2728
+ showError(walletTextUtils.getWalletText('Wallet connection failed. Please try connecting your wallet manually.', serviceId));
2729
+ setIsAITEnabling(false);
2730
+ return false;
2731
+ }
2732
+
2733
+ // If we don't have AIT loaded but we have a token, try to refresh it first
2734
+ if (!aitRef.current && nxtlinqAITServiceAccessToken) {
2735
+ try {
2736
+ await refreshAIT();
2737
+ // Wait a bit for AIT to be loaded
2738
+ await new Promise(resolve => setTimeout(resolve, 1000));
2739
+ } catch (error) {
2740
+ // Continue anyway, we'll create a new AIT if needed
2741
+ console.error('Failed to refresh AIT:', error);
2742
+ }
2743
+ }
2744
+
2745
+ try {
2746
+ // Get available permissions to find the tool
2747
+ const availablePermissionLabels = availablePermissions.map(p => p.label);
2748
+ if (!availablePermissionLabels.includes(toolName)) {
2749
+ showError(`Tool ${toolName} is not available for your current identity provider`);
2750
+ setIsAITEnabling(false);
2751
+ return false;
2752
+ }
2753
+
2754
+ // Get current permissions from AIT metadata instead of React state
2755
+ // This ensures we don't lose existing permissions when enabling new ones
2756
+ const currentAITPermissions = aitRef.current?.metadata?.permissions || permissions;
2757
+ const newPermissions = [...currentAITPermissions];
2758
+ if (!newPermissions.includes(toolName)) {
2759
+ newPermissions.push(toolName);
2760
+ }
2761
+
2762
+ // Filter out deleted permissions before saving
2763
+ const validPermissions = newPermissions.filter(permission =>
2764
+ availablePermissionLabels.includes(permission)
2765
+ );
2766
+
2767
+ // Generate and register AIT with new permissions
2768
+ // For auto-enable, we should create a regular AIT (not AI Agent AIT) if user doesn't have existing AIT
2769
+ const shouldCreateAsAIAgent = !!aitRef.current; // Only create as AI Agent if user already has an AIT
2770
+
2771
+ await generateAndRegisterAITWithSigner(validPermissions, shouldCreateAsAIAgent, currentSigner, currentHitAddress);
2772
+ showSuccess('AIT permission enabled successfully! You can now use the AI agent.');
2773
+ await refreshAIT(true);
2774
+ setIsAITEnabling(false);
2775
+ return true;
2776
+ } catch (error) {
2777
+ console.error('Failed to auto-enable AIT:', error);
2778
+ setIsAITEnabling(false);
2779
+ if (error instanceof Error) {
2780
+ showError(error.message);
2781
+ } else {
2782
+ showError('Failed to enable AIT permission. Please try again.');
2783
+ }
2784
+ return false;
2785
+ }
2786
+ };
2787
+
2788
+ // Save permissions
2789
+ const savePermissions = async (newPermissions?: string[]) => {
2790
+ setIsDisabled(true);
2791
+ try {
2792
+ // Filter out deleted permissions before saving
2793
+ const permissionsToSave = newPermissions || permissions;
2794
+ const availablePermissionLabels = availablePermissions.map(p => p.label);
2795
+ const validPermissions = permissionsToSave.filter(permission =>
2796
+ availablePermissionLabels.includes(permission)
2797
+ );
2798
+
2799
+ await generateAndRegisterAIT(validPermissions, false);
2800
+ showSuccess('AIT permissions saved successfully! You can now use the AI agent with your configured permissions.');
2801
+ setShowPermissionForm(false);
2802
+ setIsPermissionFormOpen(false);
2803
+ await refreshAIT(true);
2804
+ } catch (error) {
2805
+ console.error('Failed to generate AIT:', error);
2806
+ setIsDisabled(false);
2807
+ if (error instanceof Error) {
2808
+ showError(error.message);
2809
+ } else {
2810
+ showError('Failed to save permissions. Please try again.');
2811
+ }
2812
+ }
2813
+ };
2814
+
2815
+ // Handle verify wallet click
2816
+ const handleVerifyWalletClick = async (method: 'berifyme' | 'custom') => {
2817
+ if (!hitAddress) {
2818
+ showError('Please connect your wallet first.');
2819
+ return;
2820
+ }
2821
+
2822
+ try {
2823
+ setIsLoading(true);
2824
+
2825
+ if (method === 'berifyme') {
2826
+ if (!onVerifyWallet) {
2827
+ showError('Berify.me verification is not available. Please provide onVerifyWallet callback.');
2828
+ setIsLoading(false);
2829
+ return;
2830
+ }
2831
+
2832
+ const result = await onVerifyWallet();
2833
+ if (!result) {
2834
+ setIsLoading(false);
2835
+ return;
2836
+ }
2837
+ const { token } = result;
2838
+ const address = hitAddress;
2839
+ if (token && address) {
2840
+ const payload = {
2841
+ address: hitAddress,
2842
+ token,
2843
+ timestamp: Date.now(),
2844
+ method: 'berifyme'
2845
+ };
2846
+ try {
2847
+ const verifyWalletResponse = await nxtlinqApi.wallet.verifyWallet({ ...payload }, token);
2848
+ if ('error' in verifyWalletResponse) {
2849
+ if (verifyWalletResponse.error === 'Wallet already exists') {
2850
+ // Wallet already exists with the same method, just refresh AIT
2851
+ setIsWalletLoading(true);
2852
+ try {
2853
+ const walletResponse = await nxtlinqApi.wallet.getWallet({ address }, token);
2854
+ if (!('error' in walletResponse)) {
2855
+ setWalletInfo(walletResponse);
2856
+ const aitResponse = await nxtlinqApi.ait.getAITByServiceIdAndController({
2857
+ serviceId,
2858
+ controller: address,
2859
+ customUsername: (!requireWalletIDVVerification && customUsername) ? getFinalCustomUsername(customUsername) : undefined
2860
+ }, token);
2861
+ if (!('error' in aitResponse)) {
2862
+ setAit(aitResponse);
2863
+ }
2864
+ }
2865
+ } finally {
2866
+ setIsWalletLoading(false);
2867
+ }
2868
+ if (typeof window !== 'undefined') {
2869
+ try {
2870
+ const url = new URL(window.location.href);
2871
+ url.searchParams.delete('token');
2872
+ url.searchParams.delete('isAutoConnect');
2873
+ url.searchParams.delete('method');
2874
+ url.searchParams.delete('returnUrl');
2875
+ window.history.replaceState({}, '', url.toString());
2876
+ } catch (e) {
2877
+ console.error('Failed to clean URL:', e);
2878
+ }
2879
+ }
2880
+ setIsLoading(false);
2881
+ showSuccess(walletTextUtils.getWalletText('Wallet verification completed successfully! Your wallet is now verified and ready to use.', serviceId));
2882
+ refreshAIT();
2883
+ return;
2884
+ }
2885
+ showError(verifyWalletResponse.error);
2886
+ setIsLoading(false);
2887
+ return;
2888
+ }
2889
+ setIsWalletLoading(true);
2890
+ try {
2891
+ const walletResponse = await nxtlinqApi.wallet.getWallet({ address }, token);
2892
+ if (!('error' in walletResponse)) {
2893
+ setWalletInfo(walletResponse);
2894
+ const aitResponse = await nxtlinqApi.ait.getAITByServiceIdAndController({
2895
+ serviceId,
2896
+ controller: address,
2897
+ customUsername: (!requireWalletIDVVerification && customUsername) ? getFinalCustomUsername(customUsername) : undefined
2898
+ }, token);
2899
+ if (!('error' in aitResponse)) {
2900
+ setAit(aitResponse);
2901
+ }
2902
+ }
2903
+ } finally {
2904
+ setIsWalletLoading(false);
2905
+ }
2906
+ if (typeof window !== 'undefined') {
2907
+ try {
2908
+ const url = new URL(window.location.href);
2909
+ url.searchParams.delete('token');
2910
+ url.searchParams.delete('isAutoConnect');
2911
+ url.searchParams.delete('method');
2912
+ url.searchParams.delete('returnUrl');
2913
+ window.history.replaceState({}, '', url.toString());
2914
+ } catch (e) {
2915
+ console.error('Failed to clean URL:', e);
2916
+ }
2917
+ }
2918
+ setIsLoading(false);
2919
+ showSuccess('Wallet verification completed successfully! Your wallet is now verified and ready to use.');
2920
+ refreshAIT();
2921
+ return;
2922
+ } catch (error) {
2923
+ let msg = 'Verification failed';
2924
+ if (typeof error === 'object' && error !== null && 'response' in error) {
2925
+ msg = (error as any).response?.data?.error || (error as unknown as Error).message || msg;
2926
+ } else if (error instanceof Error) {
2927
+ msg = error.message;
2928
+ }
2929
+ console.error('Wallet verification failed:', error);
2930
+ showError(msg);
2931
+ setIsLoading(false);
2932
+ throw error;
2933
+ }
2934
+ }
2935
+ setIsLoading(false);
2936
+ return;
2937
+ }
2938
+
2939
+ if (method === 'custom') {
2940
+ if (!customUsername) {
2941
+ showError('Custom username is required for custom verification method.');
2942
+ setIsLoading(false);
2943
+ return;
2944
+ }
2945
+
2946
+ const payload = {
2947
+ address: hitAddress,
2948
+ method: 'custom' as const,
2949
+ timestamp: Date.now(),
2950
+ customUsername: getFinalCustomUsername(customUsername)
2951
+ };
2952
+ try {
2953
+ const verifyWalletResponse = await nxtlinqApi.wallet.verifyWallet(payload, '');
2954
+ if ('error' in verifyWalletResponse) {
2955
+ if (verifyWalletResponse.error === 'Wallet already exists') {
2956
+ // Wallet already exists with the same method, just refresh AIT
2957
+ await refreshAIT();
2958
+ setIsLoading(false);
2959
+ showSuccess(walletTextUtils.getWalletText('Wallet verification completed successfully! Your wallet is now verified and ready to use.', serviceId));
2960
+ return;
2961
+ }
2962
+ // Handle specific error messages
2963
+ if (verifyWalletResponse.error === 'Cannot downgrade from Berify.me verification to custom verification') {
2964
+ showError(walletTextUtils.getWalletText('This wallet is already verified with Berify.me. Custom verification cannot override Berify.me verification.', serviceId));
2965
+ } else {
2966
+ showError(verifyWalletResponse.error);
2967
+ }
2968
+ setIsLoading(false);
2969
+ return;
2970
+ }
2971
+ // Successfully created wallet with custom method
2972
+ await refreshAIT();
2973
+ setIsLoading(false);
2974
+ showSuccess(walletTextUtils.getWalletText('Wallet verification completed successfully! Your wallet is now verified and ready to use.', serviceId));
2975
+ return;
2976
+ } catch (error) {
2977
+ console.error('Custom wallet verification failed:', error);
2978
+ showError(walletTextUtils.getWalletText('Failed to verify wallet with custom method. Please try again.', serviceId));
2979
+ setIsLoading(false);
2980
+ return;
2981
+ }
2982
+ }
2983
+
2984
+ // If we reach here, the method is not supported
2985
+ showError(`Unsupported verification method: ${method}`);
2986
+ setIsLoading(false);
2987
+ } catch (error) {
2988
+ console.error('Failed to verify wallet:', error);
2989
+ setIsLoading(false);
2990
+ showError(walletTextUtils.getWalletText('Failed to verify wallet. Please try again.', serviceId));
2991
+ throw error;
2992
+ }
2993
+ };
2994
+
2995
+ // Initialize wallet address from localStorage on component mount
2996
+ React.useEffect(() => {
2997
+ const storedWalletAddress = localStorage.getItem('walletAddress');
2998
+ if (storedWalletAddress && !hitAddress) {
2999
+ setHitAddress(storedWalletAddress);
3000
+ hitAddressRef.current = storedWalletAddress;
3001
+ }
3002
+ }, []);
3003
+
3004
+ React.useEffect(() => {
3005
+ if (hitAddress && nxtlinqAITServiceAccessToken) {
3006
+ refreshAIT();
3007
+ }
3008
+ }, [hitAddress, nxtlinqAITServiceAccessToken]);
3009
+
3010
+ // Set loading state when permission form opens
3011
+ React.useEffect(() => {
3012
+ if (isPermissionFormOpen && hitAddress) {
3013
+ setIsAITLoading(true);
3014
+ }
3015
+ }, [isPermissionFormOpen, hitAddress]);
3016
+
3017
+ React.useEffect(() => {
3018
+ fetchAvailablePermissions();
3019
+ }, [serviceId, permissionGroup]);
3020
+
3021
+ React.useEffect(() => {
3022
+ if (typeof window !== 'undefined') {
3023
+ try {
3024
+ const urlParams = new URLSearchParams(window.location.search);
3025
+ const token = urlParams.get('token');
3026
+ if (token && hitAddress) {
3027
+ handleVerifyWalletClick('berifyme');
3028
+ }
3029
+ } catch (e) {
3030
+ console.error('Failed to get URL params:', e);
3031
+ }
3032
+ }
3033
+ }, [hitAddress]);
3034
+
3035
+ // Handle custom error message display
3036
+ React.useEffect(() => {
3037
+ const trimmedError = customError?.trim();
3038
+ // Only display error if it's a new value (different from last displayed error)
3039
+ if (trimmedError && trimmedError !== lastCustomErrorRef.current) {
3040
+ lastCustomErrorRef.current = trimmedError;
3041
+ const errorMessage: Message = {
3042
+ id: `custom-error-${Date.now()}`,
3043
+ content: trimmedError,
3044
+ role: 'assistant',
3045
+ timestamp: new Date().toISOString(),
3046
+ metadata: {
3047
+ model: getCurrentModel().value,
3048
+ permissions: permissions,
3049
+ issuedBy: hitAddress || ''
3050
+ }
3051
+ };
3052
+ setMessages(prev => [...prev, errorMessage]);
3053
+ onMessage?.(errorMessage);
3054
+ } else if (!trimmedError) {
3055
+ // Reset ref when customError is cleared
3056
+ lastCustomErrorRef.current = undefined;
3057
+ }
3058
+ }, [customError, getCurrentModel, permissions, hitAddress, onMessage]);
3059
+
3060
+ React.useEffect(() => {
3061
+ if (messages.length > 0) {
3062
+ const lastMessage = messages[messages.length - 1];
3063
+ onMessage?.(lastMessage);
3064
+ }
3065
+ }, [messages, onMessage]);
3066
+
3067
+ React.useEffect(() => {
3068
+ if (!hitAddress) {
3069
+ return
3070
+ }
3071
+
3072
+ const updateExternalId = async () => {
3073
+ const externalId = hitAddress;
3074
+ await nxtlinqApi.agent.updateMessagesExternalIdByPseudoId({
3075
+ apiKey,
3076
+ apiSecret,
3077
+ pseudoId,
3078
+ externalId: externalId,
3079
+ })
3080
+
3081
+ await nxtlinqApi.agent.cloneUserProfileByPseudoId({
3082
+ apiKey,
3083
+ apiSecret,
3084
+ pseudoId,
3085
+ externalId: externalId,
3086
+ });
3087
+
3088
+ await nxtlinqApi.agent.cloneUserTopicByPseudoId({
3089
+ apiKey,
3090
+ apiSecret,
3091
+ pseudoId,
3092
+ externalId: externalId,
3093
+ })
3094
+ }
3095
+
3096
+ updateExternalId()
3097
+ }, [hitAddress, nxtlinqApi, apiKey, apiSecret, pseudoId]);
3098
+
3099
+ const contextValue: ChatBotContextType = {
3100
+ // State
3101
+ messages,
3102
+ inputValue,
3103
+ isLoading,
3104
+ isOpen,
3105
+ hitAddress,
3106
+ ait,
3107
+ permissions,
3108
+ availablePermissions,
3109
+ showPermissionForm,
3110
+ isPermissionFormOpen,
3111
+ isAITLoading,
3112
+ isDisabled,
3113
+ walletInfo,
3114
+ isWalletLoading,
3115
+ isAutoConnecting,
3116
+ notification,
3117
+ isMicEnabled,
3118
+ isAwaitingMicGesture,
3119
+ transcript,
3120
+ textInputRef,
3121
+ autoSendEnabled,
3122
+ // Speech related state
3123
+ textToSpeechEnabled,
3124
+ speechingIndex,
3125
+ isTtsProcessing,
3126
+ requiresGesture,
3127
+ // AI Model related state
3128
+ availableModels: effectiveAvailableModels,
3129
+ selectedModelIndex,
3130
+ showModelSelector,
3131
+ suggestions,
3132
+ isAITEnabling,
3133
+ piiDisplayMode,
3134
+ isVoiceMode,
3135
+ voiceStatus,
3136
+ isVoiceConnecting,
3137
+ isMicMuted,
3138
+ remoteAudioRef,
3139
+
3140
+ // Actions
3141
+ setInputValue,
3142
+ setIsOpen,
3143
+ setShowPermissionForm,
3144
+ setIsPermissionFormOpen,
3145
+ setPermissions,
3146
+ setIsDisabled,
3147
+ setIsWalletLoading,
3148
+ setNotification,
3149
+ // AI Model related actions
3150
+ setSelectedModelIndex,
3151
+ setSuggestions,
3152
+ setAutoSendEnabled,
3153
+ setTextToSpeechEnabled,
3154
+
3155
+ // Functions
3156
+ connectWallet,
3157
+ signInWallet,
3158
+ sendMessage,
3159
+ handleSubmit,
3160
+ handlePresetMessage,
3161
+ savePermissions,
3162
+ enableAIT,
3163
+ handleVerifyWalletClick,
3164
+ showSuccess,
3165
+ showError,
3166
+ showWarning,
3167
+ showInfo,
3168
+ refreshAIT,
3169
+ startRecording,
3170
+ stopRecording,
3171
+ clearRecording,
3172
+ // AI Model related functions
3173
+ handleModelChange,
3174
+ getCurrentModel,
3175
+ // Speech related functions
3176
+ playTextToSpeech,
3177
+ stopTextToSpeech,
3178
+ retryTtsWithGesture,
3179
+ uploadAttachment,
3180
+ enterVoiceMode: handleEnterVoiceMode,
3181
+ exitVoiceMode: handleExitVoiceMode,
3182
+ toggleVoiceMicMute,
3183
+ interruptVoice,
3184
+
3185
+ // Additional properties for PermissionForm
3186
+ onSave: savePermissions,
3187
+ onConnectWallet: () => connectWallet(false),
3188
+ onSignIn: () => signInWallet(false),
3189
+ isNeedSignInWithWallet,
3190
+ onVerifyWallet: (method: 'berifyme' | 'custom') => handleVerifyWalletClick(method),
3191
+ serviceId,
3192
+ permissionGroup,
3193
+
3194
+ // Props
3195
+ props: {
3196
+ onMessage,
3197
+ onError,
3198
+ onToolUse,
3199
+ presetMessages,
3200
+ placeholder,
3201
+ className,
3202
+ maxRetries,
3203
+ retryDelay,
3204
+ serviceId,
3205
+ apiKey,
3206
+ apiSecret,
3207
+ onVerifyWallet,
3208
+ permissionGroup,
3209
+ onModelChange,
3210
+ storageMode,
3211
+ requireWalletIDVVerification,
3212
+ isSemiAutomaticMode,
3213
+ customUserInfo,
3214
+ customUsername,
3215
+ idvBannerDismissSeconds,
3216
+ isStopRecordingOnSend,
3217
+ customError,
3218
+ },
3219
+ nxtlinqApi
3220
+ };
3221
+
3222
+ return (
3223
+ <ChatBotContext.Provider value={contextValue}>
3224
+ {children}
3225
+ </ChatBotContext.Provider>
3226
+ );
3227
+ };