@mobileai/react-native 0.9.27 → 0.9.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +24 -11
  2. package/android/build.gradle +17 -0
  3. package/android/src/main/java/com/mobileai/overlay/FloatingOverlayDialogRootViewGroup.kt +243 -0
  4. package/android/src/main/java/com/mobileai/overlay/FloatingOverlayView.kt +281 -87
  5. package/android/src/newarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +52 -17
  6. package/android/src/oldarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +49 -2
  7. package/bin/generate-map.cjs +45 -6
  8. package/ios/Podfile +63 -0
  9. package/ios/Podfile.lock +2290 -0
  10. package/ios/Podfile.properties.json +4 -0
  11. package/ios/mobileaireactnative/AppDelegate.swift +69 -0
  12. package/ios/mobileaireactnative/Images.xcassets/AppIcon.appiconset/Contents.json +13 -0
  13. package/ios/mobileaireactnative/Images.xcassets/Contents.json +6 -0
  14. package/ios/mobileaireactnative/Images.xcassets/SplashScreenLegacy.imageset/Contents.json +21 -0
  15. package/ios/mobileaireactnative/Images.xcassets/SplashScreenLegacy.imageset/SplashScreenLegacy.png +0 -0
  16. package/ios/mobileaireactnative/Info.plist +55 -0
  17. package/ios/mobileaireactnative/PrivacyInfo.xcprivacy +48 -0
  18. package/ios/mobileaireactnative/SplashScreen.storyboard +47 -0
  19. package/ios/mobileaireactnative/Supporting/Expo.plist +6 -0
  20. package/ios/mobileaireactnative/mobileaireactnative-Bridging-Header.h +3 -0
  21. package/ios/mobileaireactnative.xcodeproj/project.pbxproj +547 -0
  22. package/ios/mobileaireactnative.xcodeproj/xcshareddata/xcschemes/mobileaireactnative.xcscheme +88 -0
  23. package/ios/mobileaireactnative.xcworkspace/contents.xcworkspacedata +10 -0
  24. package/lib/module/components/AIAgent.js +405 -168
  25. package/lib/module/components/AgentChatBar.js +250 -59
  26. package/lib/module/components/FloatingOverlayWrapper.js +68 -32
  27. package/lib/module/config/endpoints.js +22 -1
  28. package/lib/module/core/AgentRuntime.js +103 -1
  29. package/lib/module/core/FiberTreeWalker.js +98 -0
  30. package/lib/module/core/OutcomeVerifier.js +149 -0
  31. package/lib/module/core/systemPrompt.js +96 -25
  32. package/lib/module/providers/GeminiProvider.js +9 -3
  33. package/lib/module/services/telemetry/TelemetryService.js +21 -2
  34. package/lib/module/services/telemetry/TouchAutoCapture.js +45 -35
  35. package/lib/module/specs/FloatingOverlayNativeComponent.ts +7 -1
  36. package/lib/module/support/supportPrompt.js +22 -7
  37. package/lib/module/support/supportStyle.js +55 -0
  38. package/lib/module/support/types.js +2 -0
  39. package/lib/module/tools/typeTool.js +20 -0
  40. package/lib/module/utils/humanizeScreenName.js +49 -0
  41. package/lib/typescript/src/components/AIAgent.d.ts +6 -2
  42. package/lib/typescript/src/components/AgentChatBar.d.ts +15 -1
  43. package/lib/typescript/src/components/FloatingOverlayWrapper.d.ts +22 -10
  44. package/lib/typescript/src/config/endpoints.d.ts +4 -0
  45. package/lib/typescript/src/core/AgentRuntime.d.ts +9 -0
  46. package/lib/typescript/src/core/FiberTreeWalker.d.ts +12 -1
  47. package/lib/typescript/src/core/OutcomeVerifier.d.ts +46 -0
  48. package/lib/typescript/src/core/systemPrompt.d.ts +3 -10
  49. package/lib/typescript/src/core/types.d.ts +35 -0
  50. package/lib/typescript/src/index.d.ts +1 -0
  51. package/lib/typescript/src/services/telemetry/TelemetryService.d.ts +7 -1
  52. package/lib/typescript/src/services/telemetry/types.d.ts +1 -1
  53. package/lib/typescript/src/specs/FloatingOverlayNativeComponent.d.ts +5 -0
  54. package/lib/typescript/src/support/index.d.ts +1 -0
  55. package/lib/typescript/src/support/supportStyle.d.ts +9 -0
  56. package/lib/typescript/src/support/types.d.ts +3 -0
  57. package/lib/typescript/src/utils/humanizeScreenName.d.ts +6 -0
  58. package/package.json +5 -2
  59. package/src/specs/FloatingOverlayNativeComponent.ts +7 -1
  60. package/ios/MobileAIFloatingOverlayComponentView.mm +0 -73
  61. package/ios/MobileAIPilotIntents.swift +0 -51
@@ -11,8 +11,10 @@
11
11
  */
12
12
 
13
13
  import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
14
- import { View, StyleSheet } from 'react-native';
14
+ import { View, StyleSheet, InteractionManager, Platform } from 'react-native';
15
15
  import { AgentRuntime } from "../core/AgentRuntime.js";
16
+ import { captureWireframe } from "../core/FiberTreeWalker.js";
17
+ import { humanizeScreenName } from "../utils/humanizeScreenName.js";
16
18
  import { createProvider } from "../providers/ProviderFactory.js";
17
19
  import { AgentContext } from "../hooks/useAction.js";
18
20
  import { AgentChatBar } from "./AgentChatBar.js";
@@ -41,13 +43,12 @@ import { SupportChatModal } from "../support/SupportChatModal.js";
41
43
  import { ENDPOINTS } from "../config/endpoints.js";
42
44
  import * as ConversationService from "../services/ConversationService.js";
43
45
  import { createMobileAIKnowledgeRetriever } from "../services/MobileAIKnowledgeRetriever.js";
44
-
46
+ import { jsx as _jsx, Fragment as _Fragment, jsxs as _jsxs } from "react/jsx-runtime";
45
47
  // ─── Context ───────────────────────────────────────────────────
46
48
 
47
49
  // ─── AsyncStorage Helper (same pattern as TicketStore) ─────────
48
50
 
49
51
  /** Try to load AsyncStorage for tooltip persistence. Optional peer dep. */
50
- import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
51
52
  function getTooltipStorage() {
52
53
  try {
53
54
  const origError = console.error;
@@ -81,6 +82,8 @@ export function AIAgent({
81
82
  voiceProxyHeaders,
82
83
  provider: providerName = 'gemini',
83
84
  model,
85
+ supportStyle = 'warm-concise',
86
+ verifier,
84
87
  navRef,
85
88
  maxSteps = 25,
86
89
  showChatBar = true,
@@ -205,6 +208,9 @@ export function AIAgent({
205
208
  // ── Onboarding Journey State ────────────────────────────────
206
209
  const [isOnboardingActive, setIsOnboardingActive] = useState(false);
207
210
  const [currentOnboardingIndex, setCurrentOnboardingIndex] = useState(0);
211
+ const [androidWindowMetrics, setAndroidWindowMetrics] = useState(null);
212
+ const androidWindowMetricsRef = useRef(null);
213
+ const floatingOverlayRef = useRef(null);
208
214
  useEffect(() => {
209
215
  if (!onboarding?.enabled) return;
210
216
  if (onboarding.firstLaunchOnly !== false) {
@@ -269,6 +275,49 @@ export function AIAgent({
269
275
  } catch {/* graceful */}
270
276
  })();
271
277
  }, []);
278
+ useEffect(() => {
279
+ if (!showChatBar) {
280
+ androidWindowMetricsRef.current = null;
281
+ setAndroidWindowMetrics(null);
282
+ }
283
+ }, [showChatBar]);
284
+ const handleAndroidWindowMetricsChange = useCallback(metrics => {
285
+ if (Platform.OS !== 'android') {
286
+ return;
287
+ }
288
+ if (!showChatBar) {
289
+ androidWindowMetricsRef.current = null;
290
+ setAndroidWindowMetrics(null);
291
+ return;
292
+ }
293
+ const previousMetrics = androidWindowMetricsRef.current;
294
+ if (previousMetrics && previousMetrics.x === metrics.x && previousMetrics.y === metrics.y && previousMetrics.width === metrics.width && previousMetrics.height === metrics.height) {
295
+ return;
296
+ }
297
+ androidWindowMetricsRef.current = metrics;
298
+ if (floatingOverlayRef.current && previousMetrics) {
299
+ floatingOverlayRef.current.setAndroidWindowMetrics(metrics);
300
+ return;
301
+ }
302
+ setAndroidWindowMetrics(metrics);
303
+ }, [showChatBar]);
304
+ const handleAndroidWindowDragEnd = useCallback(metrics => {
305
+ if (Platform.OS !== 'android') {
306
+ return;
307
+ }
308
+ if (!showChatBar) {
309
+ androidWindowMetricsRef.current = null;
310
+ setAndroidWindowMetrics(null);
311
+ return;
312
+ }
313
+ androidWindowMetricsRef.current = metrics;
314
+ setAndroidWindowMetrics(prev => {
315
+ if (prev && prev.x === metrics.x && prev.y === metrics.y && prev.width === metrics.width && prev.height === metrics.height) {
316
+ return prev;
317
+ }
318
+ return metrics;
319
+ });
320
+ }, [showChatBar]);
272
321
 
273
322
  // CRITICAL: clearSupport uses REFS and functional setters — never closure values.
274
323
  // This function is captured by long-lived callbacks (escalation sockets, restored
@@ -505,6 +554,7 @@ export function AIAgent({
505
554
  if (!humanFrtFiredRef.current[ticketId]) {
506
555
  humanFrtFiredRef.current[ticketId] = true;
507
556
  telemetryRef.current?.track('human_first_response', {
557
+ canonical_type: 'human_first_response_sent',
508
558
  ticketId
509
559
  });
510
560
  }
@@ -703,7 +753,7 @@ export function AIAgent({
703
753
  },
704
754
  onTypingChange: setIsLiveAgentTyping,
705
755
  onTicketClosed: () => clearSupport(ticket.id),
706
- onError: err => logger.error('AIAgent', '★ Restored socket error:', err)
756
+ onError: err => logger.warn('AIAgent', '★ Restored socket error:', err)
707
757
  });
708
758
  if (ticket.wsUrl) {
709
759
  socket.connect(ticket.wsUrl);
@@ -715,7 +765,7 @@ export function AIAgent({
715
765
  logger.info('AIAgent', '★ Single ticket restored and socket cached:', ticket.id);
716
766
  }
717
767
  } catch (err) {
718
- logger.error('AIAgent', '★ Failed to restore tickets:', err);
768
+ logger.warn('AIAgent', '★ Failed to restore tickets:', err);
719
769
  }
720
770
  })();
721
771
  // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -949,7 +999,7 @@ export function AIAgent({
949
999
  }
950
1000
  clearSupport(ticketId);
951
1001
  },
952
- onError: err => logger.error('AIAgent', '★ Socket error on select:', err)
1002
+ onError: err => logger.warn('AIAgent', '★ Socket error on select:', err)
953
1003
  });
954
1004
  if (freshWsUrl) {
955
1005
  socket.connect(freshWsUrl);
@@ -1023,16 +1073,50 @@ export function AIAgent({
1023
1073
  const [pendingApprovalQuestion, setPendingApprovalQuestion] = useState(null);
1024
1074
  const overlayVisible = isThinking || !!pendingApprovalQuestion;
1025
1075
  const overlayStatusText = pendingApprovalQuestion ? 'Waiting for your approval...' : statusText;
1076
+ const effectiveProxyHeaders = useMemo(() => {
1077
+ if (!analyticsKey) return proxyHeaders;
1078
+ const isAuthMissing = !proxyHeaders || !Object.keys(proxyHeaders).some(k => k.toLowerCase() === 'authorization');
1079
+ if (isAuthMissing) {
1080
+ return {
1081
+ ...proxyHeaders,
1082
+ Authorization: `Bearer ${analyticsKey}`
1083
+ };
1084
+ }
1085
+ return proxyHeaders;
1086
+ }, [proxyHeaders, analyticsKey]);
1087
+ const effectiveVoiceProxyHeaders = useMemo(() => {
1088
+ if (!analyticsKey) return voiceProxyHeaders;
1089
+ const isAuthMissing = !voiceProxyHeaders || !Object.keys(voiceProxyHeaders).some(k => k.toLowerCase() === 'authorization');
1090
+ if (isAuthMissing) {
1091
+ return {
1092
+ ...voiceProxyHeaders,
1093
+ Authorization: `Bearer ${analyticsKey}`
1094
+ };
1095
+ }
1096
+ return voiceProxyHeaders;
1097
+ }, [voiceProxyHeaders, analyticsKey]);
1098
+ const resolvedProxyUrl = useMemo(() => {
1099
+ if (proxyUrl) return proxyUrl;
1100
+ if (analyticsKey) return ENDPOINTS.hostedTextProxy;
1101
+ return undefined;
1102
+ }, [proxyUrl, analyticsKey]);
1103
+ const resolvedVoiceProxyUrl = useMemo(() => {
1104
+ if (voiceProxyUrl) return voiceProxyUrl;
1105
+ if (analyticsKey) return ENDPOINTS.hostedVoiceProxy;
1106
+ return resolvedProxyUrl;
1107
+ }, [voiceProxyUrl, analyticsKey, resolvedProxyUrl]);
1026
1108
 
1027
1109
  // ─── Create Runtime ──────────────────────────────────────────
1028
1110
 
1029
1111
  const config = useMemo(() => ({
1030
1112
  apiKey,
1031
- proxyUrl,
1113
+ proxyUrl: resolvedProxyUrl,
1032
1114
  proxyHeaders,
1033
- voiceProxyUrl,
1115
+ voiceProxyUrl: resolvedVoiceProxyUrl,
1034
1116
  voiceProxyHeaders,
1035
1117
  model,
1118
+ supportStyle,
1119
+ verifier,
1036
1120
  language: 'en',
1037
1121
  maxSteps,
1038
1122
  interactiveBlacklist,
@@ -1139,33 +1223,11 @@ export function AIAgent({
1139
1223
  ...(event.data ?? {})
1140
1224
  });
1141
1225
  }
1142
- }), [mode, apiKey, proxyUrl, proxyHeaders, voiceProxyUrl, voiceProxyHeaders, model, maxSteps, interactiveBlacklist, interactiveWhitelist, onBeforeStep, onAfterStep, onBeforeTask, onAfterTask, transformScreenContent, customTools, instructions, stepDelay, mcpServerUrl, router, pathname, onTokenUsage, resolvedKnowledgeBase, knowledgeMaxTokens, enableUIControl, screenMap, useScreenMap, maxTokenBudget, maxCostUSD, interactionMode]);
1226
+ }), [mode, apiKey, resolvedProxyUrl, proxyHeaders, resolvedVoiceProxyUrl, voiceProxyHeaders, model, maxSteps, interactiveBlacklist, interactiveWhitelist, onBeforeStep, onAfterStep, onBeforeTask, onAfterTask, transformScreenContent, customTools, instructions, stepDelay, mcpServerUrl, router, pathname, onTokenUsage, resolvedKnowledgeBase, knowledgeMaxTokens, enableUIControl, screenMap, useScreenMap, maxTokenBudget, maxCostUSD, interactionMode, verifier, supportStyle]);
1143
1227
  useEffect(() => {
1144
1228
  logger.info('AIAgent', `⚙️ Runtime config recomputed: mode=${mode} interactionMode=${interactionMode || 'copilot(default)'} onAskUser=${mode !== 'voice'} mergedTools=${Object.keys(mergedCustomTools).join(', ') || '(none)'}`);
1145
1229
  }, [mode, interactionMode, mergedCustomTools]);
1146
- const effectiveProxyHeaders = useMemo(() => {
1147
- if (!analyticsKey) return proxyHeaders;
1148
- const isAuthMissing = !proxyHeaders || !Object.keys(proxyHeaders).some(k => k.toLowerCase() === 'authorization');
1149
- if (isAuthMissing) {
1150
- return {
1151
- ...proxyHeaders,
1152
- Authorization: `Bearer ${analyticsKey}`
1153
- };
1154
- }
1155
- return proxyHeaders;
1156
- }, [proxyHeaders, analyticsKey]);
1157
- const effectiveVoiceProxyHeaders = useMemo(() => {
1158
- if (!analyticsKey) return voiceProxyHeaders;
1159
- const isAuthMissing = !voiceProxyHeaders || !Object.keys(voiceProxyHeaders).some(k => k.toLowerCase() === 'authorization');
1160
- if (isAuthMissing) {
1161
- return {
1162
- ...voiceProxyHeaders,
1163
- Authorization: `Bearer ${analyticsKey}`
1164
- };
1165
- }
1166
- return voiceProxyHeaders;
1167
- }, [voiceProxyHeaders, analyticsKey]);
1168
- const provider = useMemo(() => createProvider(providerName, apiKey, model, proxyUrl, effectiveProxyHeaders), [providerName, apiKey, model, proxyUrl, effectiveProxyHeaders]);
1230
+ const provider = useMemo(() => createProvider(providerName, apiKey, model, resolvedProxyUrl, effectiveProxyHeaders), [providerName, apiKey, model, resolvedProxyUrl, effectiveProxyHeaders]);
1169
1231
  const runtime = useMemo(() => new AgentRuntime(provider, config, rootViewRef.current, navRef),
1170
1232
  // eslint-disable-next-line react-hooks/exhaustive-deps
1171
1233
  [provider, config]);
@@ -1193,12 +1255,12 @@ export function AIAgent({
1193
1255
  debug,
1194
1256
  onEvent: event => {
1195
1257
  // Proactive behavior triggers
1196
- if (event.type === 'rage_tap' || event.type === 'error_screen' || event.type === 'repeated_navigation') {
1258
+ if (event.type === 'rage_tap' || event.type === 'rage_click' || event.type === 'rage_click_detected' || event.type === 'error_screen' || event.type === 'repeated_navigation') {
1197
1259
  idleDetectorRef.current?.triggerBehavior(event.type, event.screen);
1198
1260
  }
1199
1261
 
1200
1262
  // Customer Success features
1201
- if (customerSuccess?.enabled && event.type === 'user_interaction' && event.data) {
1263
+ if (customerSuccess?.enabled && (event.type === 'user_interaction' || event.type === 'user_action') && event.data) {
1202
1264
  const action = String(event.data.label || event.data.action || '');
1203
1265
 
1204
1266
  // Check milestones
@@ -1235,7 +1297,10 @@ export function AIAgent({
1235
1297
  attempts++;
1236
1298
  const route = navRef?.getCurrentRoute?.();
1237
1299
  if (route?.name) {
1238
- telemetry.setScreen(route.name);
1300
+ const cleanName = humanizeScreenName(route.name);
1301
+ if (cleanName) {
1302
+ telemetry.setScreen(cleanName);
1303
+ }
1239
1304
  clearInterval(timer);
1240
1305
  } else if (attempts >= maxAttempts) {
1241
1306
  clearInterval(timer);
@@ -1250,16 +1315,36 @@ export function AIAgent({
1250
1315
 
1251
1316
  useEffect(() => {
1252
1317
  // @ts-ignore
1253
- if (typeof __DEV__ !== 'undefined' && !__DEV__ && apiKey && !proxyUrl) {
1318
+ if (typeof __DEV__ !== 'undefined' && !__DEV__ && apiKey && !resolvedProxyUrl) {
1254
1319
  logger.warn('[MobileAI] ⚠️ SECURITY WARNING: You are using `apiKey` directly in a production build. ' + 'This exposes your LLM provider key in the app binary. ' + 'Use `apiProxyUrl` to route requests through your backend instead. ' + 'See docs for details.');
1255
1320
  }
1256
- }, [apiKey, proxyUrl]);
1321
+ }, [apiKey, resolvedProxyUrl]);
1257
1322
 
1258
1323
  // Track screen changes via navRef
1259
1324
  useEffect(() => {
1260
1325
  if (!navRef?.addListener || !telemetryRef.current) return;
1261
1326
  const checkScreenMilestone = screenName => {
1262
1327
  telemetryRef.current?.setScreen(screenName);
1328
+
1329
+ // Auto-capture wireframe snapshot for privacy-safe heatmaps.
1330
+ // Deferred: wait for all animations/interactions to finish, then
1331
+ // wait one more frame for layout to settle. Zero perf impact.
1332
+ if (rootViewRef.current) {
1333
+ const handle = InteractionManager.runAfterInteractions(() => {
1334
+ requestAnimationFrame(() => {
1335
+ captureWireframe(rootViewRef, {
1336
+ screenName
1337
+ }).then(wireframe => {
1338
+ if (wireframe && telemetryRef.current) {
1339
+ telemetryRef.current.trackWireframe(wireframe);
1340
+ }
1341
+ }).catch(err => {
1342
+ if (debug) logger.debug('AIAgent', 'Wireframe capture failed:', err);
1343
+ });
1344
+ });
1345
+ });
1346
+ void handle;
1347
+ }
1263
1348
  if (customerSuccess?.enabled) {
1264
1349
  customerSuccess.successMilestones?.forEach(m => {
1265
1350
  if (m.screen && m.screen === screenName) {
@@ -1295,7 +1380,10 @@ export function AIAgent({
1295
1380
  const unsubscribe = navRef.addListener('state', () => {
1296
1381
  const currentRoute = navRef.getCurrentRoute?.();
1297
1382
  if (currentRoute?.name) {
1298
- checkScreenMilestone(currentRoute.name);
1383
+ const cleanName = humanizeScreenName(currentRoute.name);
1384
+ if (cleanName) {
1385
+ checkScreenMilestone(cleanName);
1386
+ }
1299
1387
  }
1300
1388
  });
1301
1389
  return () => unsubscribe?.();
@@ -1365,11 +1453,11 @@ export function AIAgent({
1365
1453
  logger.info('AIAgent', `Registering ${runtimeTools.length} tools with VoiceService: ${runtimeTools.map(t => t.name).join(', ')}`);
1366
1454
  // Use voice-adapted system prompt — same core rules as text mode
1367
1455
  // but without agent-loop directives that trigger autonomous actions
1368
- const voicePrompt = buildVoiceSystemPrompt('en', instructions?.system, !!knowledgeBase);
1456
+ const voicePrompt = buildVoiceSystemPrompt('en', instructions?.system, !!knowledgeBase, supportStyle);
1369
1457
  logger.info('AIAgent', `📝 Voice system prompt (${voicePrompt.length} chars):\n${voicePrompt}`);
1370
1458
  voiceServiceRef.current = new VoiceService({
1371
1459
  apiKey,
1372
- proxyUrl: voiceProxyUrl || proxyUrl,
1460
+ proxyUrl: resolvedVoiceProxyUrl,
1373
1461
  proxyHeaders: effectiveVoiceProxyHeaders || effectiveProxyHeaders,
1374
1462
  systemPrompt: voicePrompt,
1375
1463
  tools: runtimeTools,
@@ -1609,7 +1697,7 @@ export function AIAgent({
1609
1697
  setIsVoiceConnected(false);
1610
1698
  };
1611
1699
  // eslint-disable-next-line react-hooks/exhaustive-deps
1612
- }, [mode, apiKey, proxyUrl, proxyHeaders, voiceProxyUrl, voiceProxyHeaders, runtime, instructions]);
1700
+ }, [mode, apiKey, resolvedVoiceProxyUrl, effectiveVoiceProxyHeaders, effectiveProxyHeaders, runtime, instructions, supportStyle, knowledgeBase]);
1613
1701
 
1614
1702
  // ─── Stop Voice Session (full cleanup) ─────────────────────
1615
1703
 
@@ -1754,12 +1842,15 @@ export function AIAgent({
1754
1842
  setIsThinking(true);
1755
1843
  setStatusText('Thinking...');
1756
1844
  setLastResult(null);
1845
+ const requestStartedAt = Date.now();
1757
1846
  logger.info('AIAgent', `📨 New user request received in ${mode} mode | interactionMode=${interactionMode || 'copilot(default)'} | text="${message.trim()}"`);
1758
1847
 
1759
1848
  // Telemetry: track agent request
1760
1849
  telemetryRef.current?.track('agent_request', {
1850
+ canonical_type: 'ai_question_asked',
1761
1851
  query: message.trim(),
1762
1852
  transcript: message.trim(),
1853
+ request_topic: message.trim().slice(0, 120),
1763
1854
  mode
1764
1855
  });
1765
1856
  telemetryRef.current?.track('agent_trace', {
@@ -1778,8 +1869,11 @@ export function AIAgent({
1778
1869
  if (HIGH_RISK_ESCALATION_REGEX.test(message)) {
1779
1870
  logger.warn('AIAgent', 'High-risk support signal detected — auto-escalating to human');
1780
1871
  telemetryRef.current?.track('business_escalation', {
1872
+ canonical_type: 'support_escalated',
1781
1873
  message,
1782
- trigger: 'high_risk'
1874
+ trigger: 'high_risk',
1875
+ escalation_reason: 'high_risk',
1876
+ topic: message.trim().slice(0, 120)
1783
1877
  });
1784
1878
  const escalationResult = await escalateTool.execute({
1785
1879
  reason: `Customer needs human support: ${message.trim()}`
@@ -1880,7 +1974,9 @@ export function AIAgent({
1880
1974
  if (telemetryRef.current) {
1881
1975
  if (!agentFrtFiredRef.current) {
1882
1976
  agentFrtFiredRef.current = true;
1883
- telemetryRef.current.track('agent_first_response');
1977
+ telemetryRef.current.track('agent_first_response', {
1978
+ canonical_type: 'ai_first_response_sent'
1979
+ });
1884
1980
  }
1885
1981
  for (const step of normalizedResult.steps ?? []) {
1886
1982
  telemetryRef.current.track('agent_step', {
@@ -1894,7 +1990,12 @@ export function AIAgent({
1894
1990
  });
1895
1991
  }
1896
1992
  telemetryRef.current.track('agent_complete', {
1993
+ canonical_type: normalizedResult.success ? 'ai_answer_completed' : 'ai_answer_failed',
1897
1994
  success: normalizedResult.success,
1995
+ resolved: normalizedResult.success,
1996
+ resolution_type: normalizedResult.success ? 'ai_resolved' : 'needs_follow_up',
1997
+ latency_ms: Date.now() - requestStartedAt,
1998
+ request_topic: message.trim().slice(0, 120),
1898
1999
  steps: normalizedResult.steps?.length ?? 0,
1899
2000
  tokens: normalizedResult.tokenUsage?.totalTokens ?? 0,
1900
2001
  cost: normalizedResult.tokenUsage?.estimatedCostUSD ?? 0,
@@ -1989,7 +2090,12 @@ export function AIAgent({
1989
2090
 
1990
2091
  // Telemetry: track agent failure
1991
2092
  telemetryRef.current?.track('agent_complete', {
2093
+ canonical_type: 'ai_answer_failed',
1992
2094
  success: false,
2095
+ resolved: false,
2096
+ resolution_type: 'execution_error',
2097
+ latency_ms: Date.now() - requestStartedAt,
2098
+ request_topic: message.trim().slice(0, 120),
1993
2099
  error: error.message,
1994
2100
  response: `Error: ${error.message}`,
1995
2101
  conversation: {
@@ -2026,8 +2132,7 @@ export function AIAgent({
2026
2132
 
2027
2133
  const handleCancel = useCallback(() => {
2028
2134
  runtime.cancel();
2029
- setIsThinking(false);
2030
- setStatusText('');
2135
+ setStatusText('Stopping...');
2031
2136
  }, [runtime]);
2032
2137
 
2033
2138
  // ─── Conversation History Handlers ─────────────────────────────
@@ -2084,9 +2189,12 @@ export function AIAgent({
2084
2189
  if (telemetryRef.current && !telemetryRef.current.isAgentActing) {
2085
2190
  const label = extractTouchLabel(event);
2086
2191
  if (label && label !== 'Unknown Element' && label !== '[pressable]') {
2087
- telemetryRef.current.track('user_interaction', {
2192
+ telemetryRef.current.track('user_action', {
2193
+ canonical_type: 'button_tapped',
2088
2194
  type: 'tap',
2195
+ action: label,
2089
2196
  label,
2197
+ element_label: label,
2090
2198
  actor: 'user',
2091
2199
  x: Math.round(event.nativeEvent.pageX),
2092
2200
  y: Math.round(event.nativeEvent.pageY)
@@ -2097,9 +2205,11 @@ export function AIAgent({
2097
2205
  } else {
2098
2206
  // Tapped an unlabelled/empty area
2099
2207
  telemetryRef.current.track('dead_click', {
2208
+ canonical_type: 'dead_click_detected',
2100
2209
  x: Math.round(event.nativeEvent.pageX),
2101
2210
  y: Math.round(event.nativeEvent.pageY),
2102
- screen: telemetryRef.current.screen
2211
+ screen: telemetryRef.current.screen,
2212
+ screen_area: 'unknown'
2103
2213
  });
2104
2214
  }
2105
2215
  }
@@ -2115,128 +2225,255 @@ export function AIAgent({
2115
2225
  },
2116
2226
  children: children
2117
2227
  })
2118
- }), /*#__PURE__*/_jsx(FloatingOverlayWrapper, {
2228
+ }), /*#__PURE__*/_jsx(HighlightOverlay, {}), showChatBar && /*#__PURE__*/_jsx(FloatingOverlayWrapper, {
2229
+ ref: Platform.OS === 'android' ? floatingOverlayRef : undefined,
2230
+ androidWindowMetrics: Platform.OS === 'android' ? androidWindowMetricsRef.current ?? androidWindowMetrics : null,
2231
+ onAndroidWindowDragEnd: Platform.OS === 'android' ? handleAndroidWindowDragEnd : undefined,
2119
2232
  fallbackStyle: styles.floatingLayer,
2120
- children: /*#__PURE__*/_jsxs(View, {
2121
- style: StyleSheet.absoluteFill,
2122
- pointerEvents: "box-none",
2123
- children: [/*#__PURE__*/_jsx(HighlightOverlay, {}), showChatBar && /*#__PURE__*/_jsx(ProactiveHint, {
2124
- stage: proactiveStage,
2125
- badgeText: proactiveBadgeText,
2126
- onDismiss: () => idleDetectorRef.current?.dismiss(),
2127
- children: /*#__PURE__*/_jsx(AgentChatBar, {
2128
- onSend: handleSend,
2129
- isThinking: isThinking,
2130
- statusText: overlayStatusText,
2131
- lastResult: lastResult,
2132
- lastUserMessage: lastUserMessage,
2133
- chatMessages: messages,
2134
- pendingApprovalQuestion: pendingApprovalQuestion,
2135
- onPendingApprovalAction: action => {
2136
- const resolver = askUserResolverRef.current;
2137
- logger.info('AIAgent', `🔘 Approval button tapped: action=${action} | resolver=${resolver ? 'EXISTS' : 'NULL'} | pendingApprovalQuestion="${pendingApprovalQuestion}" | pendingAppApprovalRef=${pendingAppApprovalRef.current}`);
2138
- if (!resolver) {
2139
- logger.error('AIAgent', '🚫 ABORT: resolver is null when button was tapped — this means ask_user Promise was already resolved without clearing the buttons. This is a state sync bug.');
2140
- return;
2141
- }
2142
- askUserResolverRef.current = null;
2143
- pendingAskUserKindRef.current = null;
2144
- // Button was actually tapped — clear the approval gate and any queued message
2145
- pendingAppApprovalRef.current = false;
2146
- queuedApprovalAnswerRef.current = null;
2147
- setPendingApprovalQuestion(null);
2148
- const response = action === 'approve' ? '__APPROVAL_GRANTED__' : '__APPROVAL_REJECTED__';
2149
- // Restore the thinking overlay so the user can see the agent working
2150
- // after approval. onAskUser set isThinking=false when buttons appeared,
2151
- // but the agent is still running — restore the visual indicator.
2152
- if (action === 'approve') {
2153
- setIsThinking(true);
2154
- setStatusText('Working...');
2155
- }
2156
- telemetryRef.current?.track('agent_trace', {
2157
- stage: 'approval_button_pressed',
2158
- action
2159
- });
2160
- resolver(response);
2161
- },
2162
- language: 'en',
2163
- onDismiss: () => {
2164
- setLastResult(null);
2165
- setLastUserMessage(null);
2166
- },
2167
- theme: accentColor || theme ? {
2168
- ...(accentColor ? {
2169
- primaryColor: accentColor
2170
- } : {}),
2171
- ...theme
2172
- } : undefined,
2173
- availableModes: availableModes,
2174
- mode: mode,
2175
- onModeChange: newMode => {
2176
- logger.info('AIAgent', '★ onModeChange:', mode, '→', newMode, '| tickets:', tickets.length, 'selectedTicketId:', selectedTicketId);
2177
- setMode(newMode);
2178
- },
2179
- isMicActive: isMicActive,
2180
- isSpeakerMuted: isSpeakerMuted,
2181
- isAISpeaking: isAISpeaking,
2182
- isAgentTyping: isLiveAgentTyping,
2183
- onStopSession: stopVoiceSession,
2184
- isVoiceConnected: isVoiceConnected,
2185
- onMicToggle: active => {
2186
- if (active && !isVoiceConnected) {
2187
- logger.warn('AIAgent', 'Cannot toggle mic — VoiceService not connected yet');
2188
- return;
2189
- }
2190
- logger.info('AIAgent', `Mic toggle: ${active ? 'ON' : 'OFF'}`);
2191
- setIsMicActive(active);
2192
- if (active) {
2193
- logger.info('AIAgent', 'Starting AudioInput...');
2194
- audioInputRef.current?.start().then(ok => {
2195
- logger.info('AIAgent', `AudioInput start result: ${ok}`);
2196
- });
2197
- } else {
2198
- logger.info('AIAgent', 'Stopping AudioInput...');
2199
- audioInputRef.current?.stop();
2200
- }
2201
- },
2202
- onSpeakerToggle: muted => {
2203
- logger.info('AIAgent', `Speaker toggle: ${muted ? 'MUTED' : 'UNMUTED'}`);
2204
- setIsSpeakerMuted(muted);
2205
- if (muted) {
2206
- audioOutputRef.current?.mute();
2207
- } else {
2208
- audioOutputRef.current?.unmute();
2209
- }
2210
- },
2211
- tickets: tickets,
2212
- selectedTicketId: selectedTicketId,
2213
- onTicketSelect: handleTicketSelect,
2214
- onBackToTickets: handleBackToTickets,
2215
- autoExpandTrigger: autoExpandTrigger,
2216
- unreadCounts: unreadCounts,
2217
- totalUnread: totalUnread,
2218
- showDiscoveryTooltip: tooltipVisible,
2219
- discoveryTooltipMessage: discoveryTooltipMessage,
2220
- onTooltipDismiss: handleTooltipDismiss,
2221
- conversations: conversations,
2222
- isLoadingHistory: isLoadingHistory,
2223
- onConversationSelect: handleConversationSelect,
2224
- onNewConversation: handleNewConversation
2225
- })
2226
- }), /*#__PURE__*/_jsx(AgentOverlay, {
2227
- visible: overlayVisible,
2228
- statusText: overlayStatusText,
2229
- onCancel: handleCancel
2230
- }), /*#__PURE__*/_jsx(SupportChatModal, {
2231
- visible: mode === 'human' && !!selectedTicketId,
2232
- messages: supportMessages,
2233
+ children: Platform.OS === 'android' ? /*#__PURE__*/_jsx(AgentChatBar, {
2234
+ onSend: handleSend,
2235
+ onCancel: handleCancel,
2236
+ isThinking: isThinking,
2237
+ statusText: overlayStatusText,
2238
+ lastResult: lastResult,
2239
+ lastUserMessage: lastUserMessage,
2240
+ chatMessages: messages,
2241
+ pendingApprovalQuestion: pendingApprovalQuestion,
2242
+ onPendingApprovalAction: action => {
2243
+ const resolver = askUserResolverRef.current;
2244
+ logger.info('AIAgent', `🔘 Approval button tapped: action=${action} | resolver=${resolver ? 'EXISTS' : 'NULL'} | pendingApprovalQuestion="${pendingApprovalQuestion}" | pendingAppApprovalRef=${pendingAppApprovalRef.current}`);
2245
+ if (!resolver) {
2246
+ logger.error('AIAgent', '🚫 ABORT: resolver is null when button was tapped — this means ask_user Promise was already resolved without clearing the buttons. This is a state sync bug.');
2247
+ return;
2248
+ }
2249
+ askUserResolverRef.current = null;
2250
+ pendingAskUserKindRef.current = null;
2251
+ pendingAppApprovalRef.current = false;
2252
+ queuedApprovalAnswerRef.current = null;
2253
+ setPendingApprovalQuestion(null);
2254
+ const response = action === 'approve' ? '__APPROVAL_GRANTED__' : '__APPROVAL_REJECTED__';
2255
+ if (action === 'approve') {
2256
+ setIsThinking(true);
2257
+ setStatusText('Working...');
2258
+ }
2259
+ telemetryRef.current?.track('agent_trace', {
2260
+ stage: 'approval_button_pressed',
2261
+ action
2262
+ });
2263
+ resolver(response);
2264
+ },
2265
+ language: 'en',
2266
+ onDismiss: () => {
2267
+ setLastResult(null);
2268
+ setLastUserMessage(null);
2269
+ },
2270
+ theme: accentColor || theme ? {
2271
+ ...(accentColor ? {
2272
+ primaryColor: accentColor
2273
+ } : {}),
2274
+ ...theme
2275
+ } : undefined,
2276
+ availableModes: availableModes,
2277
+ mode: mode,
2278
+ onModeChange: newMode => {
2279
+ logger.info('AIAgent', '★ onModeChange:', mode, '→', newMode, '| tickets:', tickets.length, 'selectedTicketId:', selectedTicketId);
2280
+ setMode(newMode);
2281
+ },
2282
+ isMicActive: isMicActive,
2283
+ isSpeakerMuted: isSpeakerMuted,
2284
+ isAISpeaking: isAISpeaking,
2285
+ isAgentTyping: isLiveAgentTyping,
2286
+ onStopSession: stopVoiceSession,
2287
+ isVoiceConnected: isVoiceConnected,
2288
+ onMicToggle: active => {
2289
+ if (active && !isVoiceConnected) {
2290
+ logger.warn('AIAgent', 'Cannot toggle mic — VoiceService not connected yet');
2291
+ return;
2292
+ }
2293
+ logger.info('AIAgent', `Mic toggle: ${active ? 'ON' : 'OFF'}`);
2294
+ setIsMicActive(active);
2295
+ if (active) {
2296
+ logger.info('AIAgent', 'Starting AudioInput...');
2297
+ audioInputRef.current?.start().then(ok => {
2298
+ logger.info('AIAgent', `AudioInput start result: ${ok}`);
2299
+ });
2300
+ } else {
2301
+ logger.info('AIAgent', 'Stopping AudioInput...');
2302
+ audioInputRef.current?.stop();
2303
+ }
2304
+ },
2305
+ onSpeakerToggle: muted => {
2306
+ logger.info('AIAgent', `Speaker toggle: ${muted ? 'MUTED' : 'UNMUTED'}`);
2307
+ setIsSpeakerMuted(muted);
2308
+ if (muted) {
2309
+ audioOutputRef.current?.mute();
2310
+ } else {
2311
+ audioOutputRef.current?.unmute();
2312
+ }
2313
+ },
2314
+ tickets: tickets,
2315
+ selectedTicketId: selectedTicketId,
2316
+ onTicketSelect: handleTicketSelect,
2317
+ onBackToTickets: handleBackToTickets,
2318
+ autoExpandTrigger: autoExpandTrigger,
2319
+ unreadCounts: unreadCounts,
2320
+ totalUnread: totalUnread,
2321
+ showDiscoveryTooltip: tooltipVisible,
2322
+ discoveryTooltipMessage: discoveryTooltipMessage,
2323
+ onTooltipDismiss: handleTooltipDismiss,
2324
+ conversations: conversations,
2325
+ isLoadingHistory: isLoadingHistory,
2326
+ onConversationSelect: handleConversationSelect,
2327
+ onNewConversation: handleNewConversation,
2328
+ renderMode: "android-native-window",
2329
+ windowMetrics: androidWindowMetricsRef.current ?? androidWindowMetrics,
2330
+ onWindowMetricsChange: handleAndroidWindowMetricsChange
2331
+ }) : /*#__PURE__*/_jsx(ProactiveHint, {
2332
+ stage: proactiveStage,
2333
+ badgeText: proactiveBadgeText,
2334
+ onDismiss: () => idleDetectorRef.current?.dismiss(),
2335
+ children: /*#__PURE__*/_jsx(AgentChatBar, {
2233
2336
  onSend: handleSend,
2234
- onClose: handleBackToTickets,
2235
- isAgentTyping: isLiveAgentTyping,
2337
+ onCancel: handleCancel,
2236
2338
  isThinking: isThinking,
2237
- scrollToEndTrigger: chatScrollTrigger,
2238
- ticketStatus: tickets.find(t => t.id === selectedTicketId)?.status
2239
- }), /*#__PURE__*/_jsx(AIConsentDialog, {
2339
+ statusText: overlayStatusText,
2340
+ lastResult: lastResult,
2341
+ lastUserMessage: lastUserMessage,
2342
+ chatMessages: messages,
2343
+ pendingApprovalQuestion: pendingApprovalQuestion,
2344
+ onPendingApprovalAction: action => {
2345
+ const resolver = askUserResolverRef.current;
2346
+ logger.info('AIAgent', `🔘 Approval button tapped: action=${action} | resolver=${resolver ? 'EXISTS' : 'NULL'} | pendingApprovalQuestion="${pendingApprovalQuestion}" | pendingAppApprovalRef=${pendingAppApprovalRef.current}`);
2347
+ if (!resolver) {
2348
+ logger.error('AIAgent', '🚫 ABORT: resolver is null when button was tapped — this means ask_user Promise was already resolved without clearing the buttons. This is a state sync bug.');
2349
+ return;
2350
+ }
2351
+ askUserResolverRef.current = null;
2352
+ pendingAskUserKindRef.current = null;
2353
+ pendingAppApprovalRef.current = false;
2354
+ queuedApprovalAnswerRef.current = null;
2355
+ setPendingApprovalQuestion(null);
2356
+ const response = action === 'approve' ? '__APPROVAL_GRANTED__' : '__APPROVAL_REJECTED__';
2357
+ if (action === 'approve') {
2358
+ setIsThinking(true);
2359
+ setStatusText('Working...');
2360
+ }
2361
+ telemetryRef.current?.track('agent_trace', {
2362
+ stage: 'approval_button_pressed',
2363
+ action
2364
+ });
2365
+ resolver(response);
2366
+ },
2367
+ language: 'en',
2368
+ onDismiss: () => {
2369
+ setLastResult(null);
2370
+ setLastUserMessage(null);
2371
+ },
2372
+ theme: accentColor || theme ? {
2373
+ ...(accentColor ? {
2374
+ primaryColor: accentColor
2375
+ } : {}),
2376
+ ...theme
2377
+ } : undefined,
2378
+ availableModes: availableModes,
2379
+ mode: mode,
2380
+ onModeChange: newMode => {
2381
+ logger.info('AIAgent', '★ onModeChange:', mode, '→', newMode, '| tickets:', tickets.length, 'selectedTicketId:', selectedTicketId);
2382
+ setMode(newMode);
2383
+ },
2384
+ isMicActive: isMicActive,
2385
+ isSpeakerMuted: isSpeakerMuted,
2386
+ isAISpeaking: isAISpeaking,
2387
+ isAgentTyping: isLiveAgentTyping,
2388
+ onStopSession: stopVoiceSession,
2389
+ isVoiceConnected: isVoiceConnected,
2390
+ onMicToggle: active => {
2391
+ if (active && !isVoiceConnected) {
2392
+ logger.warn('AIAgent', 'Cannot toggle mic — VoiceService not connected yet');
2393
+ return;
2394
+ }
2395
+ logger.info('AIAgent', `Mic toggle: ${active ? 'ON' : 'OFF'}`);
2396
+ setIsMicActive(active);
2397
+ if (active) {
2398
+ logger.info('AIAgent', 'Starting AudioInput...');
2399
+ audioInputRef.current?.start().then(ok => {
2400
+ logger.info('AIAgent', `AudioInput start result: ${ok}`);
2401
+ });
2402
+ } else {
2403
+ logger.info('AIAgent', 'Stopping AudioInput...');
2404
+ audioInputRef.current?.stop();
2405
+ }
2406
+ },
2407
+ onSpeakerToggle: muted => {
2408
+ logger.info('AIAgent', `Speaker toggle: ${muted ? 'MUTED' : 'UNMUTED'}`);
2409
+ setIsSpeakerMuted(muted);
2410
+ if (muted) {
2411
+ audioOutputRef.current?.mute();
2412
+ } else {
2413
+ audioOutputRef.current?.unmute();
2414
+ }
2415
+ },
2416
+ tickets: tickets,
2417
+ selectedTicketId: selectedTicketId,
2418
+ onTicketSelect: handleTicketSelect,
2419
+ onBackToTickets: handleBackToTickets,
2420
+ autoExpandTrigger: autoExpandTrigger,
2421
+ unreadCounts: unreadCounts,
2422
+ totalUnread: totalUnread,
2423
+ showDiscoveryTooltip: tooltipVisible,
2424
+ discoveryTooltipMessage: discoveryTooltipMessage,
2425
+ onTooltipDismiss: handleTooltipDismiss,
2426
+ conversations: conversations,
2427
+ isLoadingHistory: isLoadingHistory,
2428
+ onConversationSelect: handleConversationSelect,
2429
+ onNewConversation: handleNewConversation,
2430
+ renderMode: "default"
2431
+ })
2432
+ })
2433
+ }), /*#__PURE__*/_jsx(AgentOverlay, {
2434
+ visible: overlayVisible,
2435
+ statusText: overlayStatusText,
2436
+ onCancel: handleCancel
2437
+ }), Platform.OS !== 'android' && /*#__PURE__*/_jsxs(_Fragment, {
2438
+ children: [/*#__PURE__*/_jsx(SupportChatModal, {
2439
+ visible: mode === 'human' && !!selectedTicketId,
2440
+ messages: supportMessages,
2441
+ onSend: handleSend,
2442
+ onClose: handleBackToTickets,
2443
+ isAgentTyping: isLiveAgentTyping,
2444
+ isThinking: isThinking,
2445
+ scrollToEndTrigger: chatScrollTrigger,
2446
+ ticketStatus: tickets.find(t => t.id === selectedTicketId)?.status
2447
+ }), /*#__PURE__*/_jsx(AIConsentDialog, {
2448
+ visible: showConsentDialog,
2449
+ provider: providerName,
2450
+ config: consentConfig,
2451
+ language: 'en',
2452
+ onConsent: async () => {
2453
+ await grantConsent();
2454
+ setShowConsentDialog(false);
2455
+ consentConfig.onConsent?.();
2456
+ logger.info('AIAgent', '✅ AI consent granted by user');
2457
+ },
2458
+ onDecline: () => {
2459
+ setShowConsentDialog(false);
2460
+ consentConfig.onDecline?.();
2461
+ logger.info('AIAgent', '❌ AI consent declined by user');
2462
+ }
2463
+ })]
2464
+ }), Platform.OS === 'android' && /*#__PURE__*/_jsxs(_Fragment, {
2465
+ children: [/*#__PURE__*/_jsx(SupportChatModal, {
2466
+ visible: mode === 'human' && !!selectedTicketId,
2467
+ messages: supportMessages,
2468
+ onSend: handleSend,
2469
+ onClose: handleBackToTickets,
2470
+ isAgentTyping: isLiveAgentTyping,
2471
+ isThinking: isThinking,
2472
+ scrollToEndTrigger: chatScrollTrigger,
2473
+ ticketStatus: tickets.find(t => t.id === selectedTicketId)?.status
2474
+ }), /*#__PURE__*/_jsx(FloatingOverlayWrapper, {
2475
+ fallbackStyle: styles.floatingLayer,
2476
+ children: /*#__PURE__*/_jsx(AIConsentDialog, {
2240
2477
  visible: showConsentDialog,
2241
2478
  provider: providerName,
2242
2479
  config: consentConfig,
@@ -2252,8 +2489,8 @@ export function AIAgent({
2252
2489
  consentConfig.onDecline?.();
2253
2490
  logger.info('AIAgent', '❌ AI consent declined by user');
2254
2491
  }
2255
- })]
2256
- })
2492
+ })
2493
+ })]
2257
2494
  })]
2258
2495
  })
2259
2496
  });