@mobileai/react-native 0.9.26 → 0.9.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +28 -15
  2. package/android/build.gradle +17 -0
  3. package/android/src/main/java/com/mobileai/overlay/FloatingOverlayDialogRootViewGroup.kt +243 -0
  4. package/android/src/main/java/com/mobileai/overlay/FloatingOverlayView.kt +281 -87
  5. package/android/src/newarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +52 -17
  6. package/android/src/oldarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +49 -2
  7. package/bin/generate-map.cjs +556 -126
  8. package/ios/Podfile +63 -0
  9. package/ios/Podfile.lock +2290 -0
  10. package/ios/Podfile.properties.json +4 -0
  11. package/ios/mobileaireactnative/AppDelegate.swift +69 -0
  12. package/ios/mobileaireactnative/Images.xcassets/AppIcon.appiconset/Contents.json +13 -0
  13. package/ios/mobileaireactnative/Images.xcassets/Contents.json +6 -0
  14. package/ios/mobileaireactnative/Images.xcassets/SplashScreenLegacy.imageset/Contents.json +21 -0
  15. package/ios/mobileaireactnative/Images.xcassets/SplashScreenLegacy.imageset/SplashScreenLegacy.png +0 -0
  16. package/ios/mobileaireactnative/Info.plist +55 -0
  17. package/ios/mobileaireactnative/PrivacyInfo.xcprivacy +48 -0
  18. package/ios/mobileaireactnative/SplashScreen.storyboard +47 -0
  19. package/ios/mobileaireactnative/Supporting/Expo.plist +6 -0
  20. package/ios/mobileaireactnative/mobileaireactnative-Bridging-Header.h +3 -0
  21. package/ios/mobileaireactnative.xcodeproj/project.pbxproj +547 -0
  22. package/ios/mobileaireactnative.xcodeproj/xcshareddata/xcschemes/mobileaireactnative.xcscheme +88 -0
  23. package/ios/mobileaireactnative.xcworkspace/contents.xcworkspacedata +10 -0
  24. package/lib/module/components/AIAgent.js +407 -148
  25. package/lib/module/components/AgentChatBar.js +253 -62
  26. package/lib/module/components/FloatingOverlayWrapper.js +68 -32
  27. package/lib/module/config/endpoints.js +22 -1
  28. package/lib/module/core/AgentRuntime.js +192 -24
  29. package/lib/module/core/FiberTreeWalker.js +410 -34
  30. package/lib/module/core/OutcomeVerifier.js +149 -0
  31. package/lib/module/core/systemPrompt.js +126 -44
  32. package/lib/module/providers/GeminiProvider.js +9 -3
  33. package/lib/module/services/MobileAIKnowledgeRetriever.js +1 -1
  34. package/lib/module/services/telemetry/MobileAI.js +1 -1
  35. package/lib/module/services/telemetry/TelemetryService.js +21 -2
  36. package/lib/module/services/telemetry/TouchAutoCapture.js +45 -35
  37. package/lib/module/specs/FloatingOverlayNativeComponent.ts +7 -1
  38. package/lib/module/support/supportPrompt.js +22 -7
  39. package/lib/module/support/supportStyle.js +55 -0
  40. package/lib/module/support/types.js +2 -0
  41. package/lib/module/tools/tapTool.js +77 -6
  42. package/lib/module/tools/typeTool.js +20 -0
  43. package/lib/module/utils/humanizeScreenName.js +49 -0
  44. package/lib/typescript/src/components/AIAgent.d.ts +6 -2
  45. package/lib/typescript/src/components/AgentChatBar.d.ts +15 -1
  46. package/lib/typescript/src/components/FloatingOverlayWrapper.d.ts +22 -10
  47. package/lib/typescript/src/config/endpoints.d.ts +4 -0
  48. package/lib/typescript/src/core/AgentRuntime.d.ts +17 -1
  49. package/lib/typescript/src/core/FiberTreeWalker.d.ts +12 -1
  50. package/lib/typescript/src/core/OutcomeVerifier.d.ts +46 -0
  51. package/lib/typescript/src/core/systemPrompt.d.ts +3 -10
  52. package/lib/typescript/src/core/types.d.ts +37 -1
  53. package/lib/typescript/src/index.d.ts +1 -0
  54. package/lib/typescript/src/services/MobileAIKnowledgeRetriever.d.ts +1 -1
  55. package/lib/typescript/src/services/telemetry/TelemetryService.d.ts +7 -1
  56. package/lib/typescript/src/services/telemetry/types.d.ts +1 -1
  57. package/lib/typescript/src/specs/FloatingOverlayNativeComponent.d.ts +5 -0
  58. package/lib/typescript/src/support/index.d.ts +1 -0
  59. package/lib/typescript/src/support/supportStyle.d.ts +9 -0
  60. package/lib/typescript/src/support/types.d.ts +3 -0
  61. package/lib/typescript/src/tools/tapTool.d.ts +3 -2
  62. package/lib/typescript/src/utils/humanizeScreenName.d.ts +6 -0
  63. package/lib/typescript/test-tree.d.ts +2 -0
  64. package/package.json +5 -2
  65. package/src/specs/FloatingOverlayNativeComponent.ts +7 -1
  66. package/ios/MobileAIFloatingOverlayComponentView.mm +0 -73
  67. package/ios/MobileAIPilotIntents.swift +0 -51
@@ -11,8 +11,10 @@
11
11
  */
12
12
 
13
13
  import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
14
- import { View, StyleSheet } from 'react-native';
14
+ import { View, StyleSheet, InteractionManager, Platform } from 'react-native';
15
15
  import { AgentRuntime } from "../core/AgentRuntime.js";
16
+ import { captureWireframe } from "../core/FiberTreeWalker.js";
17
+ import { humanizeScreenName } from "../utils/humanizeScreenName.js";
16
18
  import { createProvider } from "../providers/ProviderFactory.js";
17
19
  import { AgentContext } from "../hooks/useAction.js";
18
20
  import { AgentChatBar } from "./AgentChatBar.js";
@@ -41,13 +43,12 @@ import { SupportChatModal } from "../support/SupportChatModal.js";
41
43
  import { ENDPOINTS } from "../config/endpoints.js";
42
44
  import * as ConversationService from "../services/ConversationService.js";
43
45
  import { createMobileAIKnowledgeRetriever } from "../services/MobileAIKnowledgeRetriever.js";
44
-
46
+ import { jsx as _jsx, Fragment as _Fragment, jsxs as _jsxs } from "react/jsx-runtime";
45
47
  // ─── Context ───────────────────────────────────────────────────
46
48
 
47
49
  // ─── AsyncStorage Helper (same pattern as TicketStore) ─────────
48
50
 
49
51
  /** Try to load AsyncStorage for tooltip persistence. Optional peer dep. */
50
- import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
51
52
  function getTooltipStorage() {
52
53
  try {
53
54
  const origError = console.error;
@@ -81,6 +82,8 @@ export function AIAgent({
81
82
  voiceProxyHeaders,
82
83
  provider: providerName = 'gemini',
83
84
  model,
85
+ supportStyle = 'warm-concise',
86
+ verifier,
84
87
  navRef,
85
88
  maxSteps = 25,
86
89
  showChatBar = true,
@@ -205,6 +208,9 @@ export function AIAgent({
205
208
  // ── Onboarding Journey State ────────────────────────────────
206
209
  const [isOnboardingActive, setIsOnboardingActive] = useState(false);
207
210
  const [currentOnboardingIndex, setCurrentOnboardingIndex] = useState(0);
211
+ const [androidWindowMetrics, setAndroidWindowMetrics] = useState(null);
212
+ const androidWindowMetricsRef = useRef(null);
213
+ const floatingOverlayRef = useRef(null);
208
214
  useEffect(() => {
209
215
  if (!onboarding?.enabled) return;
210
216
  if (onboarding.firstLaunchOnly !== false) {
@@ -269,6 +275,49 @@ export function AIAgent({
269
275
  } catch {/* graceful */}
270
276
  })();
271
277
  }, []);
278
+ useEffect(() => {
279
+ if (!showChatBar) {
280
+ androidWindowMetricsRef.current = null;
281
+ setAndroidWindowMetrics(null);
282
+ }
283
+ }, [showChatBar]);
284
+ const handleAndroidWindowMetricsChange = useCallback(metrics => {
285
+ if (Platform.OS !== 'android') {
286
+ return;
287
+ }
288
+ if (!showChatBar) {
289
+ androidWindowMetricsRef.current = null;
290
+ setAndroidWindowMetrics(null);
291
+ return;
292
+ }
293
+ const previousMetrics = androidWindowMetricsRef.current;
294
+ if (previousMetrics && previousMetrics.x === metrics.x && previousMetrics.y === metrics.y && previousMetrics.width === metrics.width && previousMetrics.height === metrics.height) {
295
+ return;
296
+ }
297
+ androidWindowMetricsRef.current = metrics;
298
+ if (floatingOverlayRef.current && previousMetrics) {
299
+ floatingOverlayRef.current.setAndroidWindowMetrics(metrics);
300
+ return;
301
+ }
302
+ setAndroidWindowMetrics(metrics);
303
+ }, [showChatBar]);
304
+ const handleAndroidWindowDragEnd = useCallback(metrics => {
305
+ if (Platform.OS !== 'android') {
306
+ return;
307
+ }
308
+ if (!showChatBar) {
309
+ androidWindowMetricsRef.current = null;
310
+ setAndroidWindowMetrics(null);
311
+ return;
312
+ }
313
+ androidWindowMetricsRef.current = metrics;
314
+ setAndroidWindowMetrics(prev => {
315
+ if (prev && prev.x === metrics.x && prev.y === metrics.y && prev.width === metrics.width && prev.height === metrics.height) {
316
+ return prev;
317
+ }
318
+ return metrics;
319
+ });
320
+ }, [showChatBar]);
272
321
 
273
322
  // CRITICAL: clearSupport uses REFS and functional setters — never closure values.
274
323
  // This function is captured by long-lived callbacks (escalation sockets, restored
@@ -400,7 +449,7 @@ export function AIAgent({
400
449
  if (knowledgeBase) return knowledgeBase;
401
450
  if (!analyticsKey) return undefined;
402
451
  return createMobileAIKnowledgeRetriever({
403
- publishableKey: analyticsKey,
452
+ analyticsKey: analyticsKey,
404
453
  baseUrl: analyticsProxyUrl ?? ENDPOINTS.escalation,
405
454
  headers: analyticsProxyHeaders
406
455
  });
@@ -505,6 +554,7 @@ export function AIAgent({
505
554
  if (!humanFrtFiredRef.current[ticketId]) {
506
555
  humanFrtFiredRef.current[ticketId] = true;
507
556
  telemetryRef.current?.track('human_first_response', {
557
+ canonical_type: 'human_first_response_sent',
508
558
  ticketId
509
559
  });
510
560
  }
@@ -703,7 +753,7 @@ export function AIAgent({
703
753
  },
704
754
  onTypingChange: setIsLiveAgentTyping,
705
755
  onTicketClosed: () => clearSupport(ticket.id),
706
- onError: err => logger.error('AIAgent', '★ Restored socket error:', err)
756
+ onError: err => logger.warn('AIAgent', '★ Restored socket error:', err)
707
757
  });
708
758
  if (ticket.wsUrl) {
709
759
  socket.connect(ticket.wsUrl);
@@ -715,7 +765,7 @@ export function AIAgent({
715
765
  logger.info('AIAgent', '★ Single ticket restored and socket cached:', ticket.id);
716
766
  }
717
767
  } catch (err) {
718
- logger.error('AIAgent', '★ Failed to restore tickets:', err);
768
+ logger.warn('AIAgent', '★ Failed to restore tickets:', err);
719
769
  }
720
770
  })();
721
771
  // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -949,7 +999,7 @@ export function AIAgent({
949
999
  }
950
1000
  clearSupport(ticketId);
951
1001
  },
952
- onError: err => logger.error('AIAgent', '★ Socket error on select:', err)
1002
+ onError: err => logger.warn('AIAgent', '★ Socket error on select:', err)
953
1003
  });
954
1004
  if (freshWsUrl) {
955
1005
  socket.connect(freshWsUrl);
@@ -1023,16 +1073,50 @@ export function AIAgent({
1023
1073
  const [pendingApprovalQuestion, setPendingApprovalQuestion] = useState(null);
1024
1074
  const overlayVisible = isThinking || !!pendingApprovalQuestion;
1025
1075
  const overlayStatusText = pendingApprovalQuestion ? 'Waiting for your approval...' : statusText;
1076
+ const effectiveProxyHeaders = useMemo(() => {
1077
+ if (!analyticsKey) return proxyHeaders;
1078
+ const isAuthMissing = !proxyHeaders || !Object.keys(proxyHeaders).some(k => k.toLowerCase() === 'authorization');
1079
+ if (isAuthMissing) {
1080
+ return {
1081
+ ...proxyHeaders,
1082
+ Authorization: `Bearer ${analyticsKey}`
1083
+ };
1084
+ }
1085
+ return proxyHeaders;
1086
+ }, [proxyHeaders, analyticsKey]);
1087
+ const effectiveVoiceProxyHeaders = useMemo(() => {
1088
+ if (!analyticsKey) return voiceProxyHeaders;
1089
+ const isAuthMissing = !voiceProxyHeaders || !Object.keys(voiceProxyHeaders).some(k => k.toLowerCase() === 'authorization');
1090
+ if (isAuthMissing) {
1091
+ return {
1092
+ ...voiceProxyHeaders,
1093
+ Authorization: `Bearer ${analyticsKey}`
1094
+ };
1095
+ }
1096
+ return voiceProxyHeaders;
1097
+ }, [voiceProxyHeaders, analyticsKey]);
1098
+ const resolvedProxyUrl = useMemo(() => {
1099
+ if (proxyUrl) return proxyUrl;
1100
+ if (analyticsKey) return ENDPOINTS.hostedTextProxy;
1101
+ return undefined;
1102
+ }, [proxyUrl, analyticsKey]);
1103
+ const resolvedVoiceProxyUrl = useMemo(() => {
1104
+ if (voiceProxyUrl) return voiceProxyUrl;
1105
+ if (analyticsKey) return ENDPOINTS.hostedVoiceProxy;
1106
+ return resolvedProxyUrl;
1107
+ }, [voiceProxyUrl, analyticsKey, resolvedProxyUrl]);
1026
1108
 
1027
1109
  // ─── Create Runtime ──────────────────────────────────────────
1028
1110
 
1029
1111
  const config = useMemo(() => ({
1030
1112
  apiKey,
1031
- proxyUrl,
1113
+ proxyUrl: resolvedProxyUrl,
1032
1114
  proxyHeaders,
1033
- voiceProxyUrl,
1115
+ voiceProxyUrl: resolvedVoiceProxyUrl,
1034
1116
  voiceProxyHeaders,
1035
1117
  model,
1118
+ supportStyle,
1119
+ verifier,
1036
1120
  language: 'en',
1037
1121
  maxSteps,
1038
1122
  interactiveBlacklist,
@@ -1139,11 +1223,11 @@ export function AIAgent({
1139
1223
  ...(event.data ?? {})
1140
1224
  });
1141
1225
  }
1142
- }), [mode, apiKey, proxyUrl, proxyHeaders, voiceProxyUrl, voiceProxyHeaders, model, maxSteps, interactiveBlacklist, interactiveWhitelist, onBeforeStep, onAfterStep, onBeforeTask, onAfterTask, transformScreenContent, customTools, instructions, stepDelay, mcpServerUrl, router, pathname, onTokenUsage, resolvedKnowledgeBase, knowledgeMaxTokens, enableUIControl, screenMap, useScreenMap, maxTokenBudget, maxCostUSD, interactionMode]);
1226
+ }), [mode, apiKey, resolvedProxyUrl, proxyHeaders, resolvedVoiceProxyUrl, voiceProxyHeaders, model, maxSteps, interactiveBlacklist, interactiveWhitelist, onBeforeStep, onAfterStep, onBeforeTask, onAfterTask, transformScreenContent, customTools, instructions, stepDelay, mcpServerUrl, router, pathname, onTokenUsage, resolvedKnowledgeBase, knowledgeMaxTokens, enableUIControl, screenMap, useScreenMap, maxTokenBudget, maxCostUSD, interactionMode, verifier, supportStyle]);
1143
1227
  useEffect(() => {
1144
1228
  logger.info('AIAgent', `⚙️ Runtime config recomputed: mode=${mode} interactionMode=${interactionMode || 'copilot(default)'} onAskUser=${mode !== 'voice'} mergedTools=${Object.keys(mergedCustomTools).join(', ') || '(none)'}`);
1145
1229
  }, [mode, interactionMode, mergedCustomTools]);
1146
- const provider = useMemo(() => createProvider(providerName, apiKey, model, proxyUrl, proxyHeaders), [providerName, apiKey, model, proxyUrl, proxyHeaders]);
1230
+ const provider = useMemo(() => createProvider(providerName, apiKey, model, resolvedProxyUrl, effectiveProxyHeaders), [providerName, apiKey, model, resolvedProxyUrl, effectiveProxyHeaders]);
1147
1231
  const runtime = useMemo(() => new AgentRuntime(provider, config, rootViewRef.current, navRef),
1148
1232
  // eslint-disable-next-line react-hooks/exhaustive-deps
1149
1233
  [provider, config]);
@@ -1171,12 +1255,12 @@ export function AIAgent({
1171
1255
  debug,
1172
1256
  onEvent: event => {
1173
1257
  // Proactive behavior triggers
1174
- if (event.type === 'rage_tap' || event.type === 'error_screen' || event.type === 'repeated_navigation') {
1258
+ if (event.type === 'rage_tap' || event.type === 'rage_click' || event.type === 'rage_click_detected' || event.type === 'error_screen' || event.type === 'repeated_navigation') {
1175
1259
  idleDetectorRef.current?.triggerBehavior(event.type, event.screen);
1176
1260
  }
1177
1261
 
1178
1262
  // Customer Success features
1179
- if (customerSuccess?.enabled && event.type === 'user_interaction' && event.data) {
1263
+ if (customerSuccess?.enabled && (event.type === 'user_interaction' || event.type === 'user_action') && event.data) {
1180
1264
  const action = String(event.data.label || event.data.action || '');
1181
1265
 
1182
1266
  // Check milestones
@@ -1213,7 +1297,10 @@ export function AIAgent({
1213
1297
  attempts++;
1214
1298
  const route = navRef?.getCurrentRoute?.();
1215
1299
  if (route?.name) {
1216
- telemetry.setScreen(route.name);
1300
+ const cleanName = humanizeScreenName(route.name);
1301
+ if (cleanName) {
1302
+ telemetry.setScreen(cleanName);
1303
+ }
1217
1304
  clearInterval(timer);
1218
1305
  } else if (attempts >= maxAttempts) {
1219
1306
  clearInterval(timer);
@@ -1228,16 +1315,36 @@ export function AIAgent({
1228
1315
 
1229
1316
  useEffect(() => {
1230
1317
  // @ts-ignore
1231
- if (typeof __DEV__ !== 'undefined' && !__DEV__ && apiKey && !proxyUrl) {
1318
+ if (typeof __DEV__ !== 'undefined' && !__DEV__ && apiKey && !resolvedProxyUrl) {
1232
1319
  logger.warn('[MobileAI] ⚠️ SECURITY WARNING: You are using `apiKey` directly in a production build. ' + 'This exposes your LLM provider key in the app binary. ' + 'Use `apiProxyUrl` to route requests through your backend instead. ' + 'See docs for details.');
1233
1320
  }
1234
- }, [apiKey, proxyUrl]);
1321
+ }, [apiKey, resolvedProxyUrl]);
1235
1322
 
1236
1323
  // Track screen changes via navRef
1237
1324
  useEffect(() => {
1238
1325
  if (!navRef?.addListener || !telemetryRef.current) return;
1239
1326
  const checkScreenMilestone = screenName => {
1240
1327
  telemetryRef.current?.setScreen(screenName);
1328
+
1329
+ // Auto-capture wireframe snapshot for privacy-safe heatmaps.
1330
+ // Deferred: wait for all animations/interactions to finish, then
1331
+ // wait one more frame for layout to settle. Zero perf impact.
1332
+ if (rootViewRef.current) {
1333
+ const handle = InteractionManager.runAfterInteractions(() => {
1334
+ requestAnimationFrame(() => {
1335
+ captureWireframe(rootViewRef, {
1336
+ screenName
1337
+ }).then(wireframe => {
1338
+ if (wireframe && telemetryRef.current) {
1339
+ telemetryRef.current.trackWireframe(wireframe);
1340
+ }
1341
+ }).catch(err => {
1342
+ if (debug) logger.debug('AIAgent', 'Wireframe capture failed:', err);
1343
+ });
1344
+ });
1345
+ });
1346
+ void handle;
1347
+ }
1241
1348
  if (customerSuccess?.enabled) {
1242
1349
  customerSuccess.successMilestones?.forEach(m => {
1243
1350
  if (m.screen && m.screen === screenName) {
@@ -1273,7 +1380,10 @@ export function AIAgent({
1273
1380
  const unsubscribe = navRef.addListener('state', () => {
1274
1381
  const currentRoute = navRef.getCurrentRoute?.();
1275
1382
  if (currentRoute?.name) {
1276
- checkScreenMilestone(currentRoute.name);
1383
+ const cleanName = humanizeScreenName(currentRoute.name);
1384
+ if (cleanName) {
1385
+ checkScreenMilestone(cleanName);
1386
+ }
1277
1387
  }
1278
1388
  });
1279
1389
  return () => unsubscribe?.();
@@ -1343,12 +1453,12 @@ export function AIAgent({
1343
1453
  logger.info('AIAgent', `Registering ${runtimeTools.length} tools with VoiceService: ${runtimeTools.map(t => t.name).join(', ')}`);
1344
1454
  // Use voice-adapted system prompt — same core rules as text mode
1345
1455
  // but without agent-loop directives that trigger autonomous actions
1346
- const voicePrompt = buildVoiceSystemPrompt('en', instructions?.system, !!knowledgeBase);
1456
+ const voicePrompt = buildVoiceSystemPrompt('en', instructions?.system, !!knowledgeBase, supportStyle);
1347
1457
  logger.info('AIAgent', `📝 Voice system prompt (${voicePrompt.length} chars):\n${voicePrompt}`);
1348
1458
  voiceServiceRef.current = new VoiceService({
1349
1459
  apiKey,
1350
- proxyUrl: voiceProxyUrl || proxyUrl,
1351
- proxyHeaders: voiceProxyHeaders || proxyHeaders,
1460
+ proxyUrl: resolvedVoiceProxyUrl,
1461
+ proxyHeaders: effectiveVoiceProxyHeaders || effectiveProxyHeaders,
1352
1462
  systemPrompt: voicePrompt,
1353
1463
  tools: runtimeTools,
1354
1464
  language: 'en'
@@ -1587,7 +1697,7 @@ export function AIAgent({
1587
1697
  setIsVoiceConnected(false);
1588
1698
  };
1589
1699
  // eslint-disable-next-line react-hooks/exhaustive-deps
1590
- }, [mode, apiKey, proxyUrl, proxyHeaders, voiceProxyUrl, voiceProxyHeaders, runtime, instructions]);
1700
+ }, [mode, apiKey, resolvedVoiceProxyUrl, effectiveVoiceProxyHeaders, effectiveProxyHeaders, runtime, instructions, supportStyle, knowledgeBase]);
1591
1701
 
1592
1702
  // ─── Stop Voice Session (full cleanup) ─────────────────────
1593
1703
 
@@ -1732,12 +1842,15 @@ export function AIAgent({
1732
1842
  setIsThinking(true);
1733
1843
  setStatusText('Thinking...');
1734
1844
  setLastResult(null);
1845
+ const requestStartedAt = Date.now();
1735
1846
  logger.info('AIAgent', `📨 New user request received in ${mode} mode | interactionMode=${interactionMode || 'copilot(default)'} | text="${message.trim()}"`);
1736
1847
 
1737
1848
  // Telemetry: track agent request
1738
1849
  telemetryRef.current?.track('agent_request', {
1850
+ canonical_type: 'ai_question_asked',
1739
1851
  query: message.trim(),
1740
1852
  transcript: message.trim(),
1853
+ request_topic: message.trim().slice(0, 120),
1741
1854
  mode
1742
1855
  });
1743
1856
  telemetryRef.current?.track('agent_trace', {
@@ -1756,8 +1869,11 @@ export function AIAgent({
1756
1869
  if (HIGH_RISK_ESCALATION_REGEX.test(message)) {
1757
1870
  logger.warn('AIAgent', 'High-risk support signal detected — auto-escalating to human');
1758
1871
  telemetryRef.current?.track('business_escalation', {
1872
+ canonical_type: 'support_escalated',
1759
1873
  message,
1760
- trigger: 'high_risk'
1874
+ trigger: 'high_risk',
1875
+ escalation_reason: 'high_risk',
1876
+ topic: message.trim().slice(0, 120)
1761
1877
  });
1762
1878
  const escalationResult = await escalateTool.execute({
1763
1879
  reason: `Customer needs human support: ${message.trim()}`
@@ -1858,7 +1974,9 @@ export function AIAgent({
1858
1974
  if (telemetryRef.current) {
1859
1975
  if (!agentFrtFiredRef.current) {
1860
1976
  agentFrtFiredRef.current = true;
1861
- telemetryRef.current.track('agent_first_response');
1977
+ telemetryRef.current.track('agent_first_response', {
1978
+ canonical_type: 'ai_first_response_sent'
1979
+ });
1862
1980
  }
1863
1981
  for (const step of normalizedResult.steps ?? []) {
1864
1982
  telemetryRef.current.track('agent_step', {
@@ -1872,7 +1990,12 @@ export function AIAgent({
1872
1990
  });
1873
1991
  }
1874
1992
  telemetryRef.current.track('agent_complete', {
1993
+ canonical_type: normalizedResult.success ? 'ai_answer_completed' : 'ai_answer_failed',
1875
1994
  success: normalizedResult.success,
1995
+ resolved: normalizedResult.success,
1996
+ resolution_type: normalizedResult.success ? 'ai_resolved' : 'needs_follow_up',
1997
+ latency_ms: Date.now() - requestStartedAt,
1998
+ request_topic: message.trim().slice(0, 120),
1876
1999
  steps: normalizedResult.steps?.length ?? 0,
1877
2000
  tokens: normalizedResult.tokenUsage?.totalTokens ?? 0,
1878
2001
  cost: normalizedResult.tokenUsage?.estimatedCostUSD ?? 0,
@@ -1967,7 +2090,12 @@ export function AIAgent({
1967
2090
 
1968
2091
  // Telemetry: track agent failure
1969
2092
  telemetryRef.current?.track('agent_complete', {
2093
+ canonical_type: 'ai_answer_failed',
1970
2094
  success: false,
2095
+ resolved: false,
2096
+ resolution_type: 'execution_error',
2097
+ latency_ms: Date.now() - requestStartedAt,
2098
+ request_topic: message.trim().slice(0, 120),
1971
2099
  error: error.message,
1972
2100
  response: `Error: ${error.message}`,
1973
2101
  conversation: {
@@ -2004,8 +2132,7 @@ export function AIAgent({
2004
2132
 
2005
2133
  const handleCancel = useCallback(() => {
2006
2134
  runtime.cancel();
2007
- setIsThinking(false);
2008
- setStatusText('');
2135
+ setStatusText('Stopping...');
2009
2136
  }, [runtime]);
2010
2137
 
2011
2138
  // ─── Conversation History Handlers ─────────────────────────────
@@ -2062,9 +2189,12 @@ export function AIAgent({
2062
2189
  if (telemetryRef.current && !telemetryRef.current.isAgentActing) {
2063
2190
  const label = extractTouchLabel(event);
2064
2191
  if (label && label !== 'Unknown Element' && label !== '[pressable]') {
2065
- telemetryRef.current.track('user_interaction', {
2192
+ telemetryRef.current.track('user_action', {
2193
+ canonical_type: 'button_tapped',
2066
2194
  type: 'tap',
2195
+ action: label,
2067
2196
  label,
2197
+ element_label: label,
2068
2198
  actor: 'user',
2069
2199
  x: Math.round(event.nativeEvent.pageX),
2070
2200
  y: Math.round(event.nativeEvent.pageY)
@@ -2075,9 +2205,11 @@ export function AIAgent({
2075
2205
  } else {
2076
2206
  // Tapped an unlabelled/empty area
2077
2207
  telemetryRef.current.track('dead_click', {
2208
+ canonical_type: 'dead_click_detected',
2078
2209
  x: Math.round(event.nativeEvent.pageX),
2079
2210
  y: Math.round(event.nativeEvent.pageY),
2080
- screen: telemetryRef.current.screen
2211
+ screen: telemetryRef.current.screen,
2212
+ screen_area: 'unknown'
2081
2213
  });
2082
2214
  }
2083
2215
  }
@@ -2093,128 +2225,255 @@ export function AIAgent({
2093
2225
  },
2094
2226
  children: children
2095
2227
  })
2096
- }), /*#__PURE__*/_jsx(FloatingOverlayWrapper, {
2228
+ }), /*#__PURE__*/_jsx(HighlightOverlay, {}), showChatBar && /*#__PURE__*/_jsx(FloatingOverlayWrapper, {
2229
+ ref: Platform.OS === 'android' ? floatingOverlayRef : undefined,
2230
+ androidWindowMetrics: Platform.OS === 'android' ? androidWindowMetricsRef.current ?? androidWindowMetrics : null,
2231
+ onAndroidWindowDragEnd: Platform.OS === 'android' ? handleAndroidWindowDragEnd : undefined,
2097
2232
  fallbackStyle: styles.floatingLayer,
2098
- children: /*#__PURE__*/_jsxs(View, {
2099
- style: StyleSheet.absoluteFill,
2100
- pointerEvents: "box-none",
2101
- children: [/*#__PURE__*/_jsx(HighlightOverlay, {}), showChatBar && /*#__PURE__*/_jsx(ProactiveHint, {
2102
- stage: proactiveStage,
2103
- badgeText: proactiveBadgeText,
2104
- onDismiss: () => idleDetectorRef.current?.dismiss(),
2105
- children: /*#__PURE__*/_jsx(AgentChatBar, {
2106
- onSend: handleSend,
2107
- isThinking: isThinking,
2108
- statusText: overlayStatusText,
2109
- lastResult: lastResult,
2110
- lastUserMessage: lastUserMessage,
2111
- chatMessages: messages,
2112
- pendingApprovalQuestion: pendingApprovalQuestion,
2113
- onPendingApprovalAction: action => {
2114
- const resolver = askUserResolverRef.current;
2115
- logger.info('AIAgent', `🔘 Approval button tapped: action=${action} | resolver=${resolver ? 'EXISTS' : 'NULL'} | pendingApprovalQuestion="${pendingApprovalQuestion}" | pendingAppApprovalRef=${pendingAppApprovalRef.current}`);
2116
- if (!resolver) {
2117
- logger.error('AIAgent', '🚫 ABORT: resolver is null when button was tapped — this means ask_user Promise was already resolved without clearing the buttons. This is a state sync bug.');
2118
- return;
2119
- }
2120
- askUserResolverRef.current = null;
2121
- pendingAskUserKindRef.current = null;
2122
- // Button was actually tapped — clear the approval gate and any queued message
2123
- pendingAppApprovalRef.current = false;
2124
- queuedApprovalAnswerRef.current = null;
2125
- setPendingApprovalQuestion(null);
2126
- const response = action === 'approve' ? '__APPROVAL_GRANTED__' : '__APPROVAL_REJECTED__';
2127
- // Restore the thinking overlay so the user can see the agent working
2128
- // after approval. onAskUser set isThinking=false when buttons appeared,
2129
- // but the agent is still running — restore the visual indicator.
2130
- if (action === 'approve') {
2131
- setIsThinking(true);
2132
- setStatusText('Working...');
2133
- }
2134
- telemetryRef.current?.track('agent_trace', {
2135
- stage: 'approval_button_pressed',
2136
- action
2137
- });
2138
- resolver(response);
2139
- },
2140
- language: 'en',
2141
- onDismiss: () => {
2142
- setLastResult(null);
2143
- setLastUserMessage(null);
2144
- },
2145
- theme: accentColor || theme ? {
2146
- ...(accentColor ? {
2147
- primaryColor: accentColor
2148
- } : {}),
2149
- ...theme
2150
- } : undefined,
2151
- availableModes: availableModes,
2152
- mode: mode,
2153
- onModeChange: newMode => {
2154
- logger.info('AIAgent', '★ onModeChange:', mode, '→', newMode, '| tickets:', tickets.length, 'selectedTicketId:', selectedTicketId);
2155
- setMode(newMode);
2156
- },
2157
- isMicActive: isMicActive,
2158
- isSpeakerMuted: isSpeakerMuted,
2159
- isAISpeaking: isAISpeaking,
2160
- isAgentTyping: isLiveAgentTyping,
2161
- onStopSession: stopVoiceSession,
2162
- isVoiceConnected: isVoiceConnected,
2163
- onMicToggle: active => {
2164
- if (active && !isVoiceConnected) {
2165
- logger.warn('AIAgent', 'Cannot toggle mic — VoiceService not connected yet');
2166
- return;
2167
- }
2168
- logger.info('AIAgent', `Mic toggle: ${active ? 'ON' : 'OFF'}`);
2169
- setIsMicActive(active);
2170
- if (active) {
2171
- logger.info('AIAgent', 'Starting AudioInput...');
2172
- audioInputRef.current?.start().then(ok => {
2173
- logger.info('AIAgent', `AudioInput start result: ${ok}`);
2174
- });
2175
- } else {
2176
- logger.info('AIAgent', 'Stopping AudioInput...');
2177
- audioInputRef.current?.stop();
2178
- }
2179
- },
2180
- onSpeakerToggle: muted => {
2181
- logger.info('AIAgent', `Speaker toggle: ${muted ? 'MUTED' : 'UNMUTED'}`);
2182
- setIsSpeakerMuted(muted);
2183
- if (muted) {
2184
- audioOutputRef.current?.mute();
2185
- } else {
2186
- audioOutputRef.current?.unmute();
2187
- }
2188
- },
2189
- tickets: tickets,
2190
- selectedTicketId: selectedTicketId,
2191
- onTicketSelect: handleTicketSelect,
2192
- onBackToTickets: handleBackToTickets,
2193
- autoExpandTrigger: autoExpandTrigger,
2194
- unreadCounts: unreadCounts,
2195
- totalUnread: totalUnread,
2196
- showDiscoveryTooltip: tooltipVisible,
2197
- discoveryTooltipMessage: discoveryTooltipMessage,
2198
- onTooltipDismiss: handleTooltipDismiss,
2199
- conversations: conversations,
2200
- isLoadingHistory: isLoadingHistory,
2201
- onConversationSelect: handleConversationSelect,
2202
- onNewConversation: handleNewConversation
2203
- })
2204
- }), /*#__PURE__*/_jsx(AgentOverlay, {
2205
- visible: overlayVisible,
2206
- statusText: overlayStatusText,
2207
- onCancel: handleCancel
2208
- }), /*#__PURE__*/_jsx(SupportChatModal, {
2209
- visible: mode === 'human' && !!selectedTicketId,
2210
- messages: supportMessages,
2233
+ children: Platform.OS === 'android' ? /*#__PURE__*/_jsx(AgentChatBar, {
2234
+ onSend: handleSend,
2235
+ onCancel: handleCancel,
2236
+ isThinking: isThinking,
2237
+ statusText: overlayStatusText,
2238
+ lastResult: lastResult,
2239
+ lastUserMessage: lastUserMessage,
2240
+ chatMessages: messages,
2241
+ pendingApprovalQuestion: pendingApprovalQuestion,
2242
+ onPendingApprovalAction: action => {
2243
+ const resolver = askUserResolverRef.current;
2244
+ logger.info('AIAgent', `🔘 Approval button tapped: action=${action} | resolver=${resolver ? 'EXISTS' : 'NULL'} | pendingApprovalQuestion="${pendingApprovalQuestion}" | pendingAppApprovalRef=${pendingAppApprovalRef.current}`);
2245
+ if (!resolver) {
2246
+ logger.error('AIAgent', '🚫 ABORT: resolver is null when button was tapped — this means ask_user Promise was already resolved without clearing the buttons. This is a state sync bug.');
2247
+ return;
2248
+ }
2249
+ askUserResolverRef.current = null;
2250
+ pendingAskUserKindRef.current = null;
2251
+ pendingAppApprovalRef.current = false;
2252
+ queuedApprovalAnswerRef.current = null;
2253
+ setPendingApprovalQuestion(null);
2254
+ const response = action === 'approve' ? '__APPROVAL_GRANTED__' : '__APPROVAL_REJECTED__';
2255
+ if (action === 'approve') {
2256
+ setIsThinking(true);
2257
+ setStatusText('Working...');
2258
+ }
2259
+ telemetryRef.current?.track('agent_trace', {
2260
+ stage: 'approval_button_pressed',
2261
+ action
2262
+ });
2263
+ resolver(response);
2264
+ },
2265
+ language: 'en',
2266
+ onDismiss: () => {
2267
+ setLastResult(null);
2268
+ setLastUserMessage(null);
2269
+ },
2270
+ theme: accentColor || theme ? {
2271
+ ...(accentColor ? {
2272
+ primaryColor: accentColor
2273
+ } : {}),
2274
+ ...theme
2275
+ } : undefined,
2276
+ availableModes: availableModes,
2277
+ mode: mode,
2278
+ onModeChange: newMode => {
2279
+ logger.info('AIAgent', '★ onModeChange:', mode, '→', newMode, '| tickets:', tickets.length, 'selectedTicketId:', selectedTicketId);
2280
+ setMode(newMode);
2281
+ },
2282
+ isMicActive: isMicActive,
2283
+ isSpeakerMuted: isSpeakerMuted,
2284
+ isAISpeaking: isAISpeaking,
2285
+ isAgentTyping: isLiveAgentTyping,
2286
+ onStopSession: stopVoiceSession,
2287
+ isVoiceConnected: isVoiceConnected,
2288
+ onMicToggle: active => {
2289
+ if (active && !isVoiceConnected) {
2290
+ logger.warn('AIAgent', 'Cannot toggle mic — VoiceService not connected yet');
2291
+ return;
2292
+ }
2293
+ logger.info('AIAgent', `Mic toggle: ${active ? 'ON' : 'OFF'}`);
2294
+ setIsMicActive(active);
2295
+ if (active) {
2296
+ logger.info('AIAgent', 'Starting AudioInput...');
2297
+ audioInputRef.current?.start().then(ok => {
2298
+ logger.info('AIAgent', `AudioInput start result: ${ok}`);
2299
+ });
2300
+ } else {
2301
+ logger.info('AIAgent', 'Stopping AudioInput...');
2302
+ audioInputRef.current?.stop();
2303
+ }
2304
+ },
2305
+ onSpeakerToggle: muted => {
2306
+ logger.info('AIAgent', `Speaker toggle: ${muted ? 'MUTED' : 'UNMUTED'}`);
2307
+ setIsSpeakerMuted(muted);
2308
+ if (muted) {
2309
+ audioOutputRef.current?.mute();
2310
+ } else {
2311
+ audioOutputRef.current?.unmute();
2312
+ }
2313
+ },
2314
+ tickets: tickets,
2315
+ selectedTicketId: selectedTicketId,
2316
+ onTicketSelect: handleTicketSelect,
2317
+ onBackToTickets: handleBackToTickets,
2318
+ autoExpandTrigger: autoExpandTrigger,
2319
+ unreadCounts: unreadCounts,
2320
+ totalUnread: totalUnread,
2321
+ showDiscoveryTooltip: tooltipVisible,
2322
+ discoveryTooltipMessage: discoveryTooltipMessage,
2323
+ onTooltipDismiss: handleTooltipDismiss,
2324
+ conversations: conversations,
2325
+ isLoadingHistory: isLoadingHistory,
2326
+ onConversationSelect: handleConversationSelect,
2327
+ onNewConversation: handleNewConversation,
2328
+ renderMode: "android-native-window",
2329
+ windowMetrics: androidWindowMetricsRef.current ?? androidWindowMetrics,
2330
+ onWindowMetricsChange: handleAndroidWindowMetricsChange
2331
+ }) : /*#__PURE__*/_jsx(ProactiveHint, {
2332
+ stage: proactiveStage,
2333
+ badgeText: proactiveBadgeText,
2334
+ onDismiss: () => idleDetectorRef.current?.dismiss(),
2335
+ children: /*#__PURE__*/_jsx(AgentChatBar, {
2211
2336
  onSend: handleSend,
2212
- onClose: handleBackToTickets,
2213
- isAgentTyping: isLiveAgentTyping,
2337
+ onCancel: handleCancel,
2214
2338
  isThinking: isThinking,
2215
- scrollToEndTrigger: chatScrollTrigger,
2216
- ticketStatus: tickets.find(t => t.id === selectedTicketId)?.status
2217
- }), /*#__PURE__*/_jsx(AIConsentDialog, {
2339
+ statusText: overlayStatusText,
2340
+ lastResult: lastResult,
2341
+ lastUserMessage: lastUserMessage,
2342
+ chatMessages: messages,
2343
+ pendingApprovalQuestion: pendingApprovalQuestion,
2344
+ onPendingApprovalAction: action => {
2345
+ const resolver = askUserResolverRef.current;
2346
+ logger.info('AIAgent', `🔘 Approval button tapped: action=${action} | resolver=${resolver ? 'EXISTS' : 'NULL'} | pendingApprovalQuestion="${pendingApprovalQuestion}" | pendingAppApprovalRef=${pendingAppApprovalRef.current}`);
2347
+ if (!resolver) {
2348
+ logger.error('AIAgent', '🚫 ABORT: resolver is null when button was tapped — this means ask_user Promise was already resolved without clearing the buttons. This is a state sync bug.');
2349
+ return;
2350
+ }
2351
+ askUserResolverRef.current = null;
2352
+ pendingAskUserKindRef.current = null;
2353
+ pendingAppApprovalRef.current = false;
2354
+ queuedApprovalAnswerRef.current = null;
2355
+ setPendingApprovalQuestion(null);
2356
+ const response = action === 'approve' ? '__APPROVAL_GRANTED__' : '__APPROVAL_REJECTED__';
2357
+ if (action === 'approve') {
2358
+ setIsThinking(true);
2359
+ setStatusText('Working...');
2360
+ }
2361
+ telemetryRef.current?.track('agent_trace', {
2362
+ stage: 'approval_button_pressed',
2363
+ action
2364
+ });
2365
+ resolver(response);
2366
+ },
2367
+ language: 'en',
2368
+ onDismiss: () => {
2369
+ setLastResult(null);
2370
+ setLastUserMessage(null);
2371
+ },
2372
+ theme: accentColor || theme ? {
2373
+ ...(accentColor ? {
2374
+ primaryColor: accentColor
2375
+ } : {}),
2376
+ ...theme
2377
+ } : undefined,
2378
+ availableModes: availableModes,
2379
+ mode: mode,
2380
+ onModeChange: newMode => {
2381
+ logger.info('AIAgent', '★ onModeChange:', mode, '→', newMode, '| tickets:', tickets.length, 'selectedTicketId:', selectedTicketId);
2382
+ setMode(newMode);
2383
+ },
2384
+ isMicActive: isMicActive,
2385
+ isSpeakerMuted: isSpeakerMuted,
2386
+ isAISpeaking: isAISpeaking,
2387
+ isAgentTyping: isLiveAgentTyping,
2388
+ onStopSession: stopVoiceSession,
2389
+ isVoiceConnected: isVoiceConnected,
2390
+ onMicToggle: active => {
2391
+ if (active && !isVoiceConnected) {
2392
+ logger.warn('AIAgent', 'Cannot toggle mic — VoiceService not connected yet');
2393
+ return;
2394
+ }
2395
+ logger.info('AIAgent', `Mic toggle: ${active ? 'ON' : 'OFF'}`);
2396
+ setIsMicActive(active);
2397
+ if (active) {
2398
+ logger.info('AIAgent', 'Starting AudioInput...');
2399
+ audioInputRef.current?.start().then(ok => {
2400
+ logger.info('AIAgent', `AudioInput start result: ${ok}`);
2401
+ });
2402
+ } else {
2403
+ logger.info('AIAgent', 'Stopping AudioInput...');
2404
+ audioInputRef.current?.stop();
2405
+ }
2406
+ },
2407
+ onSpeakerToggle: muted => {
2408
+ logger.info('AIAgent', `Speaker toggle: ${muted ? 'MUTED' : 'UNMUTED'}`);
2409
+ setIsSpeakerMuted(muted);
2410
+ if (muted) {
2411
+ audioOutputRef.current?.mute();
2412
+ } else {
2413
+ audioOutputRef.current?.unmute();
2414
+ }
2415
+ },
2416
+ tickets: tickets,
2417
+ selectedTicketId: selectedTicketId,
2418
+ onTicketSelect: handleTicketSelect,
2419
+ onBackToTickets: handleBackToTickets,
2420
+ autoExpandTrigger: autoExpandTrigger,
2421
+ unreadCounts: unreadCounts,
2422
+ totalUnread: totalUnread,
2423
+ showDiscoveryTooltip: tooltipVisible,
2424
+ discoveryTooltipMessage: discoveryTooltipMessage,
2425
+ onTooltipDismiss: handleTooltipDismiss,
2426
+ conversations: conversations,
2427
+ isLoadingHistory: isLoadingHistory,
2428
+ onConversationSelect: handleConversationSelect,
2429
+ onNewConversation: handleNewConversation,
2430
+ renderMode: "default"
2431
+ })
2432
+ })
2433
+ }), /*#__PURE__*/_jsx(AgentOverlay, {
2434
+ visible: overlayVisible,
2435
+ statusText: overlayStatusText,
2436
+ onCancel: handleCancel
2437
+ }), Platform.OS !== 'android' && /*#__PURE__*/_jsxs(_Fragment, {
2438
+ children: [/*#__PURE__*/_jsx(SupportChatModal, {
2439
+ visible: mode === 'human' && !!selectedTicketId,
2440
+ messages: supportMessages,
2441
+ onSend: handleSend,
2442
+ onClose: handleBackToTickets,
2443
+ isAgentTyping: isLiveAgentTyping,
2444
+ isThinking: isThinking,
2445
+ scrollToEndTrigger: chatScrollTrigger,
2446
+ ticketStatus: tickets.find(t => t.id === selectedTicketId)?.status
2447
+ }), /*#__PURE__*/_jsx(AIConsentDialog, {
2448
+ visible: showConsentDialog,
2449
+ provider: providerName,
2450
+ config: consentConfig,
2451
+ language: 'en',
2452
+ onConsent: async () => {
2453
+ await grantConsent();
2454
+ setShowConsentDialog(false);
2455
+ consentConfig.onConsent?.();
2456
+ logger.info('AIAgent', '✅ AI consent granted by user');
2457
+ },
2458
+ onDecline: () => {
2459
+ setShowConsentDialog(false);
2460
+ consentConfig.onDecline?.();
2461
+ logger.info('AIAgent', '❌ AI consent declined by user');
2462
+ }
2463
+ })]
2464
+ }), Platform.OS === 'android' && /*#__PURE__*/_jsxs(_Fragment, {
2465
+ children: [/*#__PURE__*/_jsx(SupportChatModal, {
2466
+ visible: mode === 'human' && !!selectedTicketId,
2467
+ messages: supportMessages,
2468
+ onSend: handleSend,
2469
+ onClose: handleBackToTickets,
2470
+ isAgentTyping: isLiveAgentTyping,
2471
+ isThinking: isThinking,
2472
+ scrollToEndTrigger: chatScrollTrigger,
2473
+ ticketStatus: tickets.find(t => t.id === selectedTicketId)?.status
2474
+ }), /*#__PURE__*/_jsx(FloatingOverlayWrapper, {
2475
+ fallbackStyle: styles.floatingLayer,
2476
+ children: /*#__PURE__*/_jsx(AIConsentDialog, {
2218
2477
  visible: showConsentDialog,
2219
2478
  provider: providerName,
2220
2479
  config: consentConfig,
@@ -2230,8 +2489,8 @@ export function AIAgent({
2230
2489
  consentConfig.onDecline?.();
2231
2490
  logger.info('AIAgent', '❌ AI consent declined by user');
2232
2491
  }
2233
- })]
2234
- })
2492
+ })
2493
+ })]
2235
2494
  })]
2236
2495
  })
2237
2496
  });