@mobileai/react-native 0.9.17 → 0.9.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (292) hide show
  1. package/LICENSE +28 -20
  2. package/MobileAIFloatingOverlay.podspec +25 -0
  3. package/android/build.gradle +61 -0
  4. package/android/src/main/AndroidManifest.xml +3 -0
  5. package/android/src/main/java/com/mobileai/overlay/FloatingOverlayView.kt +151 -0
  6. package/android/src/main/java/com/mobileai/overlay/MobileAIOverlayPackage.kt +23 -0
  7. package/android/src/newarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +45 -0
  8. package/android/src/oldarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +29 -0
  9. package/ios/MobileAIFloatingOverlayComponentView.mm +73 -0
  10. package/lib/module/components/AIAgent.js +902 -136
  11. package/lib/module/components/AIConsentDialog.js +439 -0
  12. package/lib/module/components/AgentChatBar.js +828 -134
  13. package/lib/module/components/AgentOverlay.js +2 -1
  14. package/lib/module/components/DiscoveryTooltip.js +21 -9
  15. package/lib/module/components/FloatingOverlayWrapper.js +108 -0
  16. package/lib/module/components/Icons.js +123 -0
  17. package/lib/module/config/endpoints.js +12 -2
  18. package/lib/module/core/AgentRuntime.js +373 -27
  19. package/lib/module/core/FiberAdapter.js +56 -0
  20. package/lib/module/core/FiberTreeWalker.js +186 -80
  21. package/lib/module/core/IdleDetector.js +19 -0
  22. package/lib/module/core/NativeAlertInterceptor.js +191 -0
  23. package/lib/module/core/systemPrompt.js +203 -45
  24. package/lib/module/index.js +3 -0
  25. package/lib/module/providers/GeminiProvider.js +72 -56
  26. package/lib/module/providers/ProviderFactory.js +6 -2
  27. package/lib/module/services/AudioInputService.js +3 -12
  28. package/lib/module/services/AudioOutputService.js +1 -13
  29. package/lib/module/services/ConversationService.js +166 -0
  30. package/lib/module/services/MobileAIKnowledgeRetriever.js +41 -0
  31. package/lib/module/services/VoiceService.js +29 -8
  32. package/lib/module/services/telemetry/MobileAI.js +44 -0
  33. package/lib/module/services/telemetry/TelemetryService.js +13 -1
  34. package/lib/module/services/telemetry/TouchAutoCapture.js +44 -18
  35. package/lib/module/specs/FloatingOverlayNativeComponent.ts +19 -0
  36. package/lib/module/support/CSATSurvey.js +95 -12
  37. package/lib/module/support/EscalationSocket.js +70 -1
  38. package/lib/module/support/ReportedIssueEventSource.js +148 -0
  39. package/lib/module/support/escalateTool.js +4 -2
  40. package/lib/module/support/index.js +1 -0
  41. package/lib/module/support/reportIssueTool.js +127 -0
  42. package/lib/module/support/supportPrompt.js +77 -9
  43. package/lib/module/tools/guideTool.js +2 -1
  44. package/lib/module/tools/longPressTool.js +4 -3
  45. package/lib/module/tools/pickerTool.js +6 -4
  46. package/lib/module/tools/tapTool.js +12 -3
  47. package/lib/module/tools/typeTool.js +19 -10
  48. package/lib/module/utils/logger.js +175 -6
  49. package/lib/typescript/react-native.config.d.ts +11 -0
  50. package/lib/typescript/src/components/AIAgent.d.ts +28 -2
  51. package/lib/typescript/src/components/AIConsentDialog.d.ts +153 -0
  52. package/lib/typescript/src/components/AgentChatBar.d.ts +15 -2
  53. package/lib/typescript/src/components/DiscoveryTooltip.d.ts +3 -1
  54. package/lib/typescript/src/components/FloatingOverlayWrapper.d.ts +51 -0
  55. package/lib/typescript/src/components/Icons.d.ts +8 -0
  56. package/lib/typescript/src/config/endpoints.d.ts +5 -3
  57. package/lib/typescript/src/core/AgentRuntime.d.ts +4 -0
  58. package/lib/typescript/src/core/FiberAdapter.d.ts +25 -0
  59. package/lib/typescript/src/core/FiberTreeWalker.d.ts +2 -0
  60. package/lib/typescript/src/core/IdleDetector.d.ts +11 -0
  61. package/lib/typescript/src/core/NativeAlertInterceptor.d.ts +55 -0
  62. package/lib/typescript/src/core/types.d.ts +106 -1
  63. package/lib/typescript/src/index.d.ts +9 -4
  64. package/lib/typescript/src/providers/GeminiProvider.d.ts +6 -5
  65. package/lib/typescript/src/services/ConversationService.d.ts +55 -0
  66. package/lib/typescript/src/services/MobileAIKnowledgeRetriever.d.ts +9 -0
  67. package/lib/typescript/src/services/telemetry/MobileAI.d.ts +7 -0
  68. package/lib/typescript/src/services/telemetry/TelemetryService.d.ts +1 -1
  69. package/lib/typescript/src/services/telemetry/TouchAutoCapture.d.ts +9 -6
  70. package/lib/typescript/src/services/telemetry/types.d.ts +3 -1
  71. package/lib/typescript/src/specs/FloatingOverlayNativeComponent.d.ts +17 -0
  72. package/lib/typescript/src/support/EscalationSocket.d.ts +17 -0
  73. package/lib/typescript/src/support/ReportedIssueEventSource.d.ts +24 -0
  74. package/lib/typescript/src/support/escalateTool.d.ts +5 -0
  75. package/lib/typescript/src/support/index.d.ts +2 -1
  76. package/lib/typescript/src/support/reportIssueTool.d.ts +20 -0
  77. package/lib/typescript/src/support/types.d.ts +56 -1
  78. package/lib/typescript/src/utils/logger.d.ts +15 -0
  79. package/package.json +20 -9
  80. package/react-native.config.js +12 -0
  81. package/lib/module/__cli_tmp__.js.map +0 -1
  82. package/lib/module/components/AIAgent.js.map +0 -1
  83. package/lib/module/components/AIZone.js.map +0 -1
  84. package/lib/module/components/AgentChatBar.js.map +0 -1
  85. package/lib/module/components/AgentErrorBoundary.js.map +0 -1
  86. package/lib/module/components/AgentOverlay.js.map +0 -1
  87. package/lib/module/components/DiscoveryTooltip.js.map +0 -1
  88. package/lib/module/components/HighlightOverlay.js.map +0 -1
  89. package/lib/module/components/Icons.js.map +0 -1
  90. package/lib/module/components/ProactiveHint.js.map +0 -1
  91. package/lib/module/components/cards/InfoCard.js.map +0 -1
  92. package/lib/module/components/cards/ReviewSummary.js.map +0 -1
  93. package/lib/module/config/endpoints.js.map +0 -1
  94. package/lib/module/core/ActionRegistry.js.map +0 -1
  95. package/lib/module/core/AgentRuntime.js.map +0 -1
  96. package/lib/module/core/FiberTreeWalker.js.map +0 -1
  97. package/lib/module/core/IdleDetector.js.map +0 -1
  98. package/lib/module/core/MCPBridge.js.map +0 -1
  99. package/lib/module/core/ScreenDehydrator.js.map +0 -1
  100. package/lib/module/core/ZoneRegistry.js.map +0 -1
  101. package/lib/module/core/systemPrompt.js.map +0 -1
  102. package/lib/module/core/types.js.map +0 -1
  103. package/lib/module/hooks/useAction.js.map +0 -1
  104. package/lib/module/index.js.map +0 -1
  105. package/lib/module/plugin/withAppIntents.js.map +0 -1
  106. package/lib/module/providers/GeminiProvider.js.map +0 -1
  107. package/lib/module/providers/OpenAIProvider.js.map +0 -1
  108. package/lib/module/providers/ProviderFactory.js.map +0 -1
  109. package/lib/module/services/AudioInputService.js.map +0 -1
  110. package/lib/module/services/AudioOutputService.js.map +0 -1
  111. package/lib/module/services/KnowledgeBaseService.js.map +0 -1
  112. package/lib/module/services/VoiceService.js.map +0 -1
  113. package/lib/module/services/flags/FlagService.js.map +0 -1
  114. package/lib/module/services/telemetry/MobileAI.js.map +0 -1
  115. package/lib/module/services/telemetry/PiiScrubber.js.map +0 -1
  116. package/lib/module/services/telemetry/TelemetryService.js.map +0 -1
  117. package/lib/module/services/telemetry/TouchAutoCapture.js.map +0 -1
  118. package/lib/module/services/telemetry/device.js.map +0 -1
  119. package/lib/module/services/telemetry/deviceMetadata.js.map +0 -1
  120. package/lib/module/services/telemetry/index.js.map +0 -1
  121. package/lib/module/services/telemetry/types.js.map +0 -1
  122. package/lib/module/support/CSATSurvey.js.map +0 -1
  123. package/lib/module/support/EscalationEventSource.js.map +0 -1
  124. package/lib/module/support/EscalationSocket.js.map +0 -1
  125. package/lib/module/support/SupportChatModal.js.map +0 -1
  126. package/lib/module/support/SupportGreeting.js.map +0 -1
  127. package/lib/module/support/TicketStore.js.map +0 -1
  128. package/lib/module/support/escalateTool.js.map +0 -1
  129. package/lib/module/support/index.js.map +0 -1
  130. package/lib/module/support/supportPrompt.js.map +0 -1
  131. package/lib/module/support/types.js.map +0 -1
  132. package/lib/module/tools/datePickerTool.js.map +0 -1
  133. package/lib/module/tools/guideTool.js.map +0 -1
  134. package/lib/module/tools/index.js.map +0 -1
  135. package/lib/module/tools/keyboardTool.js.map +0 -1
  136. package/lib/module/tools/longPressTool.js.map +0 -1
  137. package/lib/module/tools/pickerTool.js.map +0 -1
  138. package/lib/module/tools/restoreTool.js.map +0 -1
  139. package/lib/module/tools/scrollTool.js.map +0 -1
  140. package/lib/module/tools/simplifyTool.js.map +0 -1
  141. package/lib/module/tools/sliderTool.js.map +0 -1
  142. package/lib/module/tools/tapTool.js.map +0 -1
  143. package/lib/module/tools/typeTool.js.map +0 -1
  144. package/lib/module/tools/types.js.map +0 -1
  145. package/lib/module/types/jsx.d.js.map +0 -1
  146. package/lib/module/utils/audioUtils.js.map +0 -1
  147. package/lib/module/utils/logger.js.map +0 -1
  148. package/lib/typescript/babel.config.d.ts.map +0 -1
  149. package/lib/typescript/bin/generate-map.d.cts.map +0 -1
  150. package/lib/typescript/eslint.config.d.mts.map +0 -1
  151. package/lib/typescript/generate-map.d.ts.map +0 -1
  152. package/lib/typescript/src/__cli_tmp__.d.ts.map +0 -1
  153. package/lib/typescript/src/components/AIAgent.d.ts.map +0 -1
  154. package/lib/typescript/src/components/AIZone.d.ts.map +0 -1
  155. package/lib/typescript/src/components/AgentChatBar.d.ts.map +0 -1
  156. package/lib/typescript/src/components/AgentErrorBoundary.d.ts.map +0 -1
  157. package/lib/typescript/src/components/AgentOverlay.d.ts.map +0 -1
  158. package/lib/typescript/src/components/DiscoveryTooltip.d.ts.map +0 -1
  159. package/lib/typescript/src/components/HighlightOverlay.d.ts.map +0 -1
  160. package/lib/typescript/src/components/Icons.d.ts.map +0 -1
  161. package/lib/typescript/src/components/ProactiveHint.d.ts.map +0 -1
  162. package/lib/typescript/src/components/cards/InfoCard.d.ts.map +0 -1
  163. package/lib/typescript/src/components/cards/ReviewSummary.d.ts.map +0 -1
  164. package/lib/typescript/src/config/endpoints.d.ts.map +0 -1
  165. package/lib/typescript/src/core/ActionRegistry.d.ts.map +0 -1
  166. package/lib/typescript/src/core/AgentRuntime.d.ts.map +0 -1
  167. package/lib/typescript/src/core/FiberTreeWalker.d.ts.map +0 -1
  168. package/lib/typescript/src/core/IdleDetector.d.ts.map +0 -1
  169. package/lib/typescript/src/core/MCPBridge.d.ts.map +0 -1
  170. package/lib/typescript/src/core/ScreenDehydrator.d.ts.map +0 -1
  171. package/lib/typescript/src/core/ZoneRegistry.d.ts.map +0 -1
  172. package/lib/typescript/src/core/systemPrompt.d.ts.map +0 -1
  173. package/lib/typescript/src/core/types.d.ts.map +0 -1
  174. package/lib/typescript/src/hooks/useAction.d.ts.map +0 -1
  175. package/lib/typescript/src/index.d.ts.map +0 -1
  176. package/lib/typescript/src/plugin/withAppIntents.d.ts.map +0 -1
  177. package/lib/typescript/src/providers/GeminiProvider.d.ts.map +0 -1
  178. package/lib/typescript/src/providers/OpenAIProvider.d.ts.map +0 -1
  179. package/lib/typescript/src/providers/ProviderFactory.d.ts.map +0 -1
  180. package/lib/typescript/src/services/AudioInputService.d.ts.map +0 -1
  181. package/lib/typescript/src/services/AudioOutputService.d.ts.map +0 -1
  182. package/lib/typescript/src/services/KnowledgeBaseService.d.ts.map +0 -1
  183. package/lib/typescript/src/services/VoiceService.d.ts.map +0 -1
  184. package/lib/typescript/src/services/flags/FlagService.d.ts.map +0 -1
  185. package/lib/typescript/src/services/telemetry/MobileAI.d.ts.map +0 -1
  186. package/lib/typescript/src/services/telemetry/PiiScrubber.d.ts.map +0 -1
  187. package/lib/typescript/src/services/telemetry/TelemetryService.d.ts.map +0 -1
  188. package/lib/typescript/src/services/telemetry/TouchAutoCapture.d.ts.map +0 -1
  189. package/lib/typescript/src/services/telemetry/device.d.ts.map +0 -1
  190. package/lib/typescript/src/services/telemetry/deviceMetadata.d.ts.map +0 -1
  191. package/lib/typescript/src/services/telemetry/index.d.ts.map +0 -1
  192. package/lib/typescript/src/services/telemetry/types.d.ts.map +0 -1
  193. package/lib/typescript/src/support/CSATSurvey.d.ts.map +0 -1
  194. package/lib/typescript/src/support/EscalationEventSource.d.ts.map +0 -1
  195. package/lib/typescript/src/support/EscalationSocket.d.ts.map +0 -1
  196. package/lib/typescript/src/support/SupportChatModal.d.ts.map +0 -1
  197. package/lib/typescript/src/support/SupportGreeting.d.ts.map +0 -1
  198. package/lib/typescript/src/support/TicketStore.d.ts.map +0 -1
  199. package/lib/typescript/src/support/escalateTool.d.ts.map +0 -1
  200. package/lib/typescript/src/support/index.d.ts.map +0 -1
  201. package/lib/typescript/src/support/supportPrompt.d.ts.map +0 -1
  202. package/lib/typescript/src/support/types.d.ts.map +0 -1
  203. package/lib/typescript/src/tools/datePickerTool.d.ts.map +0 -1
  204. package/lib/typescript/src/tools/guideTool.d.ts.map +0 -1
  205. package/lib/typescript/src/tools/index.d.ts.map +0 -1
  206. package/lib/typescript/src/tools/keyboardTool.d.ts.map +0 -1
  207. package/lib/typescript/src/tools/longPressTool.d.ts.map +0 -1
  208. package/lib/typescript/src/tools/pickerTool.d.ts.map +0 -1
  209. package/lib/typescript/src/tools/restoreTool.d.ts.map +0 -1
  210. package/lib/typescript/src/tools/scrollTool.d.ts.map +0 -1
  211. package/lib/typescript/src/tools/simplifyTool.d.ts.map +0 -1
  212. package/lib/typescript/src/tools/sliderTool.d.ts.map +0 -1
  213. package/lib/typescript/src/tools/tapTool.d.ts.map +0 -1
  214. package/lib/typescript/src/tools/typeTool.d.ts.map +0 -1
  215. package/lib/typescript/src/tools/types.d.ts.map +0 -1
  216. package/lib/typescript/src/utils/audioUtils.d.ts.map +0 -1
  217. package/lib/typescript/src/utils/logger.d.ts.map +0 -1
  218. package/src/__cli_tmp__.tsx +0 -9
  219. package/src/cli/analyzers/chain-analyzer.ts +0 -183
  220. package/src/cli/extractors/ai-extractor.ts +0 -6
  221. package/src/cli/extractors/ast-extractor.ts +0 -551
  222. package/src/cli/generate-intents.ts +0 -140
  223. package/src/cli/generate-map.ts +0 -121
  224. package/src/cli/generate-swift.ts +0 -116
  225. package/src/cli/scanners/expo-scanner.ts +0 -203
  226. package/src/cli/scanners/rn-scanner.ts +0 -445
  227. package/src/components/AIAgent.tsx +0 -1716
  228. package/src/components/AIZone.tsx +0 -147
  229. package/src/components/AgentChatBar.tsx +0 -1143
  230. package/src/components/AgentErrorBoundary.tsx +0 -78
  231. package/src/components/AgentOverlay.tsx +0 -73
  232. package/src/components/DiscoveryTooltip.tsx +0 -148
  233. package/src/components/HighlightOverlay.tsx +0 -136
  234. package/src/components/Icons.tsx +0 -253
  235. package/src/components/ProactiveHint.tsx +0 -145
  236. package/src/components/cards/InfoCard.tsx +0 -58
  237. package/src/components/cards/ReviewSummary.tsx +0 -76
  238. package/src/config/endpoints.ts +0 -22
  239. package/src/core/ActionRegistry.ts +0 -105
  240. package/src/core/AgentRuntime.ts +0 -1471
  241. package/src/core/FiberTreeWalker.ts +0 -930
  242. package/src/core/IdleDetector.ts +0 -72
  243. package/src/core/MCPBridge.ts +0 -163
  244. package/src/core/ScreenDehydrator.ts +0 -53
  245. package/src/core/ZoneRegistry.ts +0 -44
  246. package/src/core/systemPrompt.ts +0 -431
  247. package/src/core/types.ts +0 -521
  248. package/src/hooks/useAction.ts +0 -182
  249. package/src/index.ts +0 -83
  250. package/src/plugin/withAppIntents.ts +0 -98
  251. package/src/providers/GeminiProvider.ts +0 -357
  252. package/src/providers/OpenAIProvider.ts +0 -379
  253. package/src/providers/ProviderFactory.ts +0 -36
  254. package/src/services/AudioInputService.ts +0 -226
  255. package/src/services/AudioOutputService.ts +0 -236
  256. package/src/services/KnowledgeBaseService.ts +0 -156
  257. package/src/services/VoiceService.ts +0 -451
  258. package/src/services/flags/FlagService.ts +0 -137
  259. package/src/services/telemetry/MobileAI.ts +0 -66
  260. package/src/services/telemetry/PiiScrubber.ts +0 -17
  261. package/src/services/telemetry/TelemetryService.ts +0 -323
  262. package/src/services/telemetry/TouchAutoCapture.ts +0 -165
  263. package/src/services/telemetry/device.ts +0 -93
  264. package/src/services/telemetry/deviceMetadata.ts +0 -13
  265. package/src/services/telemetry/index.ts +0 -13
  266. package/src/services/telemetry/types.ts +0 -75
  267. package/src/support/CSATSurvey.tsx +0 -304
  268. package/src/support/EscalationEventSource.ts +0 -190
  269. package/src/support/EscalationSocket.ts +0 -152
  270. package/src/support/SupportChatModal.tsx +0 -563
  271. package/src/support/SupportGreeting.tsx +0 -161
  272. package/src/support/TicketStore.ts +0 -100
  273. package/src/support/escalateTool.ts +0 -174
  274. package/src/support/index.ts +0 -29
  275. package/src/support/supportPrompt.ts +0 -55
  276. package/src/support/types.ts +0 -155
  277. package/src/tools/datePickerTool.ts +0 -60
  278. package/src/tools/guideTool.ts +0 -76
  279. package/src/tools/index.ts +0 -20
  280. package/src/tools/keyboardTool.ts +0 -30
  281. package/src/tools/longPressTool.ts +0 -61
  282. package/src/tools/pickerTool.ts +0 -115
  283. package/src/tools/restoreTool.ts +0 -33
  284. package/src/tools/scrollTool.ts +0 -156
  285. package/src/tools/simplifyTool.ts +0 -33
  286. package/src/tools/sliderTool.ts +0 -65
  287. package/src/tools/tapTool.ts +0 -93
  288. package/src/tools/typeTool.ts +0 -113
  289. package/src/tools/types.ts +0 -58
  290. package/src/types/jsx.d.ts +0 -20
  291. package/src/utils/audioUtils.ts +0 -54
  292. package/src/utils/logger.ts +0 -38
@@ -1,1471 +0,0 @@
1
- /**
2
- * AgentRuntime — The main agent loop.
3
- *
4
- * Flow:
5
- * 1. Walk Fiber tree → detect interactive elements
6
- * 2. Dehydrate screen → text for LLM
7
- * 3. Send to AI provider with tools
8
- * 4. Parse tool call → execute (tap, type, navigate, done)
9
- * 5. If not done, repeat from step 1 (re-dehydrate after UI change)
10
- */
11
-
12
- import { logger } from '../utils/logger';
13
- import { walkFiberTree } from './FiberTreeWalker';
14
- import type { WalkConfig } from './FiberTreeWalker';
15
- import { dehydrateScreen } from './ScreenDehydrator';
16
- import { buildSystemPrompt, buildKnowledgeOnlyPrompt } from './systemPrompt';
17
- import { KnowledgeBaseService } from '../services/KnowledgeBaseService';
18
- import {
19
- createTapTool,
20
- createLongPressTool,
21
- createTypeTool,
22
- createScrollTool,
23
- createSliderTool,
24
- createPickerTool,
25
- createDatePickerTool,
26
- createKeyboardTool,
27
- createGuideTool,
28
- createSimplifyTool,
29
- createRestoreTool,
30
- } from '../tools';
31
- import type { ToolContext } from '../tools';
32
- import type {
33
- AIProvider,
34
- AgentConfig,
35
- AgentStep,
36
- ExecutionResult,
37
- ToolDefinition,
38
- TokenUsage,
39
- } from './types';
40
- import { actionRegistry } from './ActionRegistry';
41
-
42
- const DEFAULT_MAX_STEPS = 25;
43
-
44
- // ─── Agent Runtime ─────────────────────────────────────────────
45
-
46
- export class AgentRuntime {
47
- private provider: AIProvider;
48
- private config: AgentConfig;
49
- private rootRef: any;
50
- private navRef: any;
51
- private tools: Map<string, ToolDefinition> = new Map();
52
- private history: AgentStep[] = [];
53
- private isRunning = false;
54
- private isCancelRequested = false;
55
- private lastAskUserQuestion: string | null = null;
56
- private knowledgeService: KnowledgeBaseService | null = null;
57
- private uiControlOverride?: boolean;
58
- private lastDehydratedRoot: any = null;
59
-
60
- // ─── Task-scoped error suppression ──────────────────────────
61
- // Installed once at execute() start, removed after grace period.
62
- // Catches ALL async errors (useEffect, native callbacks, PagerView)
63
- // that would otherwise crash the host app during agent execution.
64
- private originalErrorHandler: ((error: Error, isFatal?: boolean) => void) | null = null;
65
- private lastSuppressedError: Error | null = null;
66
- private graceTimer: ReturnType<typeof setTimeout> | null = null;
67
- private originalReportErrorsAsExceptions: boolean | undefined = undefined;
68
-
69
- public getConfig(): AgentConfig {
70
- return this.config;
71
- }
72
-
73
- constructor(
74
- provider: AIProvider,
75
- config: AgentConfig,
76
- rootRef: any,
77
- navRef: any,
78
- ) {
79
- this.provider = provider;
80
- this.config = config;
81
- this.rootRef = rootRef;
82
- this.navRef = navRef;
83
- logger.debug('AgentRuntime', 'constructor: config.screenMap exists:', !!config.screenMap);
84
-
85
- // Initialize knowledge base service if configured
86
- if (config.knowledgeBase) {
87
- this.knowledgeService = new KnowledgeBaseService(
88
- config.knowledgeBase,
89
- config.knowledgeMaxTokens
90
- );
91
- }
92
-
93
- // Register tools based on mode
94
- if (config.enableUIControl === false) {
95
- this.registerKnowledgeOnlyTools();
96
- } else {
97
- this.registerBuiltInTools();
98
- }
99
-
100
- // Apply customTools
101
- if (config.customTools) {
102
- for (const [name, tool] of Object.entries(config.customTools)) {
103
- if (tool === null) {
104
- this.tools.delete(name);
105
- logger.info('AgentRuntime', `Removed tool: ${name}`);
106
- } else {
107
- this.tools.set(name, tool);
108
- logger.info('AgentRuntime', `Overrode tool: ${name}`);
109
- }
110
- }
111
- }
112
- }
113
-
114
- // ─── Tool Registration ─────────────────────────────────────
115
-
116
- private registerBuiltInTools(): void {
117
- // ── Tool Context — shared dependencies for modular tools ──
118
- const toolContext: ToolContext = {
119
- getRootRef: () => this.rootRef,
120
- getWalkConfig: () => this.getWalkConfig(),
121
- getCurrentScreenName: () => this.getCurrentScreenName(),
122
- getNavRef: () => this.navRef,
123
- routerRef: this.config.router,
124
- getRouteNames: () => this.getRouteNames(),
125
- findScreenPath: (name: string) => this.findScreenPath(name),
126
- buildNestedParams: (path: string[], params?: any) => this.buildNestedParams(path, params),
127
- captureScreenshot: async () => (await this.captureScreenshot()) ?? null,
128
- getLastDehydratedRoot: () => this.lastDehydratedRoot,
129
- };
130
-
131
- // ── Register modular tools (extracted to src/tools/) ──
132
- const modularTools = [
133
- createTapTool(toolContext),
134
- createLongPressTool(toolContext),
135
- createTypeTool(toolContext),
136
- createScrollTool(toolContext),
137
- createSliderTool(toolContext),
138
- createPickerTool(toolContext),
139
- createDatePickerTool(toolContext),
140
- createKeyboardTool(),
141
- createGuideTool(toolContext),
142
- createSimplifyTool(),
143
- createRestoreTool(),
144
- ];
145
-
146
- for (const tool of modularTools) {
147
- this.tools.set(tool.name, tool);
148
- }
149
-
150
-
151
- // navigate — navigate to a screen (supports React Navigation + Expo Router)
152
- this.tools.set('navigate', {
153
- name: 'navigate',
154
- description: 'Navigate to a top-level screen by name. ONLY use this for top-level screens that do NOT require route params (e.g. Login, Settings, Cart, TabBar). NEVER use this for parameterized screens that require an ID or selection (e.g. DishDetail, SelectCategory, ProfileDetail, OrderDetails) — those screens will crash without required params. For parameterized screens, always navigate by TAPPING the relevant item in the parent screen instead.',
155
- parameters: {
156
- screen: { type: 'string', description: 'Top-level screen name to navigate to (must not require route params)', required: true },
157
- params: { type: 'string', description: 'Optional JSON params object for screens that accept them', required: false },
158
- },
159
- execute: async (args) => {
160
- // Expo Router path: use router.push()
161
- if (this.config.router) {
162
- try {
163
- const path = args.screen.startsWith('/') ? args.screen : `/${args.screen}`;
164
- this.config.router.push(path);
165
- await new Promise(resolve => setTimeout(resolve, 500));
166
- return `✅ Navigated to "${path}"`;
167
- } catch (error: any) {
168
- return `❌ Navigation error: ${error.message}`;
169
- }
170
- }
171
-
172
- // React Navigation path: use navRef
173
- if (!this.navRef) {
174
- return '❌ Navigation ref not available.';
175
- }
176
- if (!this.navRef.isReady()) {
177
- await new Promise(resolve => setTimeout(resolve, 1000));
178
- if (!this.navRef.isReady()) {
179
- return '❌ Navigation is not ready yet.';
180
- }
181
- }
182
- try {
183
- const params = args.params ? (typeof args.params === 'string' ? JSON.parse(args.params) : args.params) : undefined;
184
- // Case-insensitive screen name matching
185
- const availableRoutes = this.getRouteNames();
186
- logger.info('AgentRuntime', `🧭 Navigate requested: "${args.screen}" | Available: [${availableRoutes.join(', ')}] | Params: ${JSON.stringify(params)}`);
187
- const matchedScreen = availableRoutes.find(
188
- r => r.toLowerCase() === args.screen.toLowerCase()
189
- );
190
-
191
- // Guard: screen must exist in the navigation tree
192
- if (!matchedScreen) {
193
- const errMsg = `❌ "${args.screen}" is not a screen — it may be content within a screen. Available screens: ${availableRoutes.join(', ')}. Look at the current screen context for "${args.screen}" as a section, category, or element, and scroll/tap to find it. If it's on a different screen, navigate to the correct screen first.`;
194
- logger.warn('AgentRuntime', `🧭 Navigate REJECTED: ${errMsg}`);
195
- return errMsg;
196
- }
197
- logger.info('AgentRuntime', `🧭 Navigate matched: "${args.screen}" → "${matchedScreen}"`);
198
-
199
- // Find the path to the screen (handles nested navigators)
200
- const screenPath = this.findScreenPath(matchedScreen);
201
- if (screenPath.length > 1) {
202
- // Nested screen: navigate using parent → { screen: child } pattern
203
- // e.g. navigate('HomeTab', { screen: 'Home', params })
204
- logger.info('AgentRuntime', `Nested navigation: ${screenPath.join(' → ')}`);
205
- const nestedParams = this.buildNestedParams(screenPath, params);
206
- this.navRef.navigate(screenPath[0], nestedParams);
207
- } else {
208
- // Top-level screen: direct navigate
209
- this.navRef.navigate(matchedScreen, params);
210
- }
211
- await new Promise(resolve => setTimeout(resolve, 500));
212
- return `✅ Navigated to "${matchedScreen}"${params ? ` with params: ${JSON.stringify(params)}` : ''}`;
213
- } catch (error: any) {
214
- return `❌ Navigation error: ${error.message}. Available screens: ${this.getRouteNames().join(', ')}`;
215
- }
216
- },
217
- });
218
-
219
- // done — complete the task
220
- this.tools.set('done', {
221
- name: 'done',
222
- description: 'Complete the task with a message to the user.',
223
- parameters: {
224
- text: { type: 'string', description: 'Response message to the user', required: true },
225
- success: { type: 'boolean', description: 'Whether the task was completed successfully', required: true },
226
- },
227
- execute: async (args) => {
228
- return args.text;
229
- },
230
- });
231
-
232
- // wait — explicitly wait for loading states
233
- this.tools.set('wait', {
234
- name: 'wait',
235
- description: 'Wait for a specified number of seconds before taking the next action. Use this when the screen explicitly shows "Loading...", "Please wait", or loading skeletons, to give the app time to fetch data.',
236
- parameters: {
237
- seconds: { type: 'number', description: 'Number of seconds to wait (max 5)', required: true },
238
- },
239
- execute: async (args) => {
240
- const seconds = Math.min(Number(args.seconds) || 2, 5);
241
- await new Promise(resolve => setTimeout(resolve, seconds * 1000));
242
- return `âŗ Waited ${seconds} seconds for the screen to update.`;
243
- },
244
- });
245
-
246
- // ask_user — ask for clarification
247
- this.tools.set('ask_user', {
248
- name: 'ask_user',
249
- description: 'Ask the user a question and wait for their answer. Use this if you need more information or clarification.',
250
- parameters: {
251
- question: { type: 'string', description: 'Question to ask the user', required: true },
252
- },
253
- execute: async (args) => {
254
- if (this.config.onAskUser) {
255
- // Block until user responds, then continue the loop
256
- this.config.onStatusUpdate?.('Waiting for your answer...');
257
- const answer = await this.config.onAskUser(args.question);
258
- return `User answered: ${answer}`;
259
- }
260
- // Legacy fallback: break the loop (context will be lost)
261
- return `❓ ${args.question}`;
262
- },
263
- });
264
-
265
- // capture_screenshot — on-demand visual capture (for image/video content questions)
266
- this.tools.set('capture_screenshot', {
267
- name: 'capture_screenshot',
268
- description: 'Capture a screenshot of the current screen. Use when the user asks about visual content (images, videos, colors, layout appearance) that cannot be determined from the element tree alone.',
269
- parameters: {},
270
- execute: async () => {
271
- const screenshot = await this.captureScreenshot();
272
- if (screenshot) {
273
- return `✅ Screenshot captured (${Math.round(screenshot.length / 1024)}KB). Visual content is now available for analysis.`;
274
- }
275
- return '❌ Screenshot capture failed. react-native-view-shot may not be installed.';
276
- },
277
- });
278
-
279
-
280
-
281
-
282
- // query_knowledge — retrieve domain-specific knowledge (only if knowledgeBase is configured)
283
- if (this.knowledgeService) {
284
- this.tools.set('query_knowledge', {
285
- name: 'query_knowledge',
286
- description:
287
- 'Search the app knowledge base for domain-specific information '
288
- + '(policies, FAQs, product details, delivery areas, allergens, etc). '
289
- + 'Use when the user asks about the business or app and the answer is NOT visible on screen.',
290
- parameters: {
291
- question: {
292
- type: 'string',
293
- description: 'The question or topic to search for',
294
- required: true,
295
- },
296
- },
297
- execute: async (args) => {
298
- const screenName = this.getCurrentScreenName();
299
- return this.knowledgeService!.retrieve(args.question, screenName);
300
- },
301
- });
302
- }
303
- }
304
-
305
- /**
306
- * Register only knowledge-assistant tools (no UI control).
307
- * Used when enableUIControl = false — the AI can only answer questions.
308
- */
309
- private registerKnowledgeOnlyTools(): void {
310
- // done — complete the task
311
- this.tools.set('done', {
312
- name: 'done',
313
- description: 'Complete the task with a message to the user.',
314
- parameters: {
315
- text: { type: 'string', description: 'Response message to the user', required: true },
316
- success: { type: 'boolean', description: 'Whether the task was completed successfully', required: true },
317
- },
318
- execute: async (args) => {
319
- return args.text;
320
- },
321
- });
322
-
323
- // query_knowledge — retrieve domain-specific knowledge (only if knowledgeBase is configured)
324
- if (this.knowledgeService) {
325
- this.tools.set('query_knowledge', {
326
- name: 'query_knowledge',
327
- description:
328
- 'Search the app knowledge base for domain-specific information '
329
- + '(policies, FAQs, product details, delivery areas, allergens, etc). '
330
- + 'Use when the user asks about the business or app and the answer is NOT visible on screen.',
331
- parameters: {
332
- question: {
333
- type: 'string',
334
- description: 'The question or topic to search for',
335
- required: true,
336
- },
337
- },
338
- execute: async (args) => {
339
- const screenName = this.getCurrentScreenName();
340
- return this.knowledgeService!.retrieve(args.question, screenName);
341
- },
342
- });
343
- }
344
- }
345
-
346
- // ─── Navigation Helpers ────────────────────────────────────
347
-
348
- /**
349
- * Recursively collect ALL screen names from the navigation state tree.
350
- * This handles tabs, drawers, and nested stacks.
351
- */
352
- private getRouteNames(): string[] {
353
- try {
354
- if (!this.navRef?.isReady?.()) return [];
355
- const state = this.navRef?.getRootState?.() || this.navRef?.getState?.();
356
- if (!state) return [];
357
- const names = this.collectRouteNames(state);
358
- logger.debug('AgentRuntime', 'Available routes:', names.join(', '));
359
- return names;
360
- } catch {
361
- return [];
362
- }
363
- }
364
-
365
- private collectRouteNames(state: any): string[] {
366
- const names: string[] = [];
367
- // routeNames contains ALL defined screens (including unvisited)
368
- if (state?.routeNames) {
369
- names.push(...state.routeNames);
370
- }
371
- if (state?.routes) {
372
- for (const route of state.routes) {
373
- names.push(route.name);
374
- // Recurse into nested navigator states
375
- if (route.state) {
376
- names.push(...this.collectRouteNames(route.state));
377
- }
378
- }
379
- }
380
- return [...new Set(names)];
381
- }
382
-
383
- /**
384
- * Find the path from root navigator to a target screen.
385
- * Returns [parentTab, screen] for nested screens, or [screen] for top-level.
386
- * Example: findScreenPath('Home') → ['HomeTab', 'Home']
387
- */
388
- private findScreenPath(targetScreen: string): string[] {
389
- try {
390
- const state = this.navRef?.getRootState?.() || this.navRef?.getState?.();
391
- if (!state?.routes) return [targetScreen];
392
-
393
- // Check if target is a direct top-level route
394
- if (state.routes.some((r: any) => r.name === targetScreen)) {
395
- return [targetScreen];
396
- }
397
-
398
- // Search nested navigators
399
- for (const route of state.routes) {
400
- const nestedNames = route.state ? this.collectRouteNames(route.state) : [];
401
- if (nestedNames.includes(targetScreen)) {
402
- return [route.name, targetScreen];
403
- }
404
- }
405
-
406
- return [targetScreen]; // Fallback: try direct
407
- } catch {
408
- return [targetScreen];
409
- }
410
- }
411
-
412
-
413
- /**
414
- * Build nested params for React Navigation nested screen navigation.
415
- * ['HomeTab', 'Home'] → { screen: 'Home', params }
416
- * ['Tab', 'Stack', 'Screen'] → { screen: 'Stack', params: { screen: 'Screen', params } }
417
- */
418
- private buildNestedParams(path: string[], leafParams?: any): any {
419
- // Build from the end: innermost screen gets the leafParams
420
- let result = leafParams;
421
- for (let i = path.length - 1; i >= 1; i--) {
422
- result = { screen: path[i], ...(result !== undefined ? { params: result } : {}) };
423
- }
424
- return result;
425
- }
426
-
427
- /**
428
- * Recursively find the deepest active screen name.
429
- * For tabs: follows active tab → active screen inside that tab.
430
- */
431
- private getCurrentScreenName(): string {
432
- // Expo Router: use pathname
433
- if (this.config.pathname) {
434
- const segments = this.config.pathname.split('/').filter(Boolean);
435
- return segments[segments.length - 1] || 'Unknown';
436
- }
437
-
438
- try {
439
- if (!this.navRef?.isReady?.()) return 'Unknown';
440
- const state = this.navRef?.getRootState?.() || this.navRef?.getState?.();
441
- if (!state) return 'Unknown';
442
- return this.getDeepestScreenName(state);
443
- } catch {
444
- return 'Unknown';
445
- }
446
- }
447
-
448
- private getDeepestScreenName(state: any): string {
449
- if (!state?.routes || state.index == null) return 'Unknown';
450
- const route = state.routes[state.index];
451
- if (!route) return 'Unknown';
452
- // If this route has a nested state, recurse deeper
453
- if (route.state) {
454
- return this.getDeepestScreenName(route.state);
455
- }
456
- return route.name || 'Unknown';
457
- }
458
-
459
- // ─── Dynamic Config Overrides ────────────────────────────────
460
-
461
- public setUIControlOverride(enabled: boolean | undefined) {
462
- this.uiControlOverride = enabled;
463
- }
464
-
465
- private isUIEnabled(): boolean {
466
- if (this.uiControlOverride !== undefined) return this.uiControlOverride;
467
- return this.config.enableUIControl !== false; // defaults to true
468
- }
469
-
470
- /** Maps a tool call to a user-friendly status label for the loading overlay. */
471
- private getToolStatusLabel(toolName: string, args: Record<string, any>): string {
472
- switch (toolName) {
473
- case 'tap':
474
- return 'Tapping a button...';
475
- case 'type':
476
- return 'Typing into a field...';
477
- case 'navigate':
478
- return `Navigating to ${args.screen || 'another screen'}...`;
479
- case 'done':
480
- return 'Wrapping up...';
481
- case 'ask_user':
482
- return 'Asking you a question...';
483
- case 'query_knowledge':
484
- return 'Searching knowledge base...';
485
- case 'scroll':
486
- return `Scrolling ${args.direction || 'down'}...`;
487
- case 'wait':
488
- return 'Waiting for the screen to load...';
489
- case 'long_press':
490
- return 'Long-pressing an element...';
491
- case 'adjust_slider':
492
- return `Adjusting slider to ${Math.round((args.value ?? 0) * 100)}%...`;
493
- case 'select_picker':
494
- return `Selecting "${args.value || ''}" from a dropdown...`;
495
- case 'set_date':
496
- return `Setting date to ${args.date || ''}...`;
497
- case 'dismiss_keyboard':
498
- return 'Dismissing keyboard...';
499
- default:
500
- return `Running ${toolName}...`;
501
- }
502
- }
503
-
504
- // ─── Screenshot Capture (optional react-native-view-shot) ─────
505
-
506
- /**
507
- * Captures the current screen as a base64 JPEG for Gemini vision.
508
- * Uses react-native-view-shot as an optional peer dependency.
509
- * Returns null if the library is not installed (graceful fallback).
510
- */
511
- private async captureScreenshot(): Promise<string | undefined> {
512
- try {
513
- // Static require — Metro needs a literal string; the try/catch handles MODULE_NOT_FOUND.
514
- const viewShot = require('react-native-view-shot');
515
- const captureRef = viewShot.captureRef || viewShot.default?.captureRef;
516
- if (!captureRef || !this.rootRef) return undefined;
517
-
518
- const uri = await captureRef(this.rootRef, {
519
- format: 'jpg',
520
- quality: 0.4,
521
- width: 720,
522
- result: 'base64',
523
- });
524
-
525
- logger.info('AgentRuntime', `Screenshot captured (${Math.round((uri?.length || 0) / 1024)}KB base64)`);
526
- return uri || undefined;
527
- } catch (error: any) {
528
- if (error.message?.includes('Cannot find module') || error.code === 'MODULE_NOT_FOUND' || error.message?.includes('unknown module')) {
529
- logger.warn('AgentRuntime', 'Screenshot requires react-native-view-shot. Install with: npx expo install react-native-view-shot');
530
- } else {
531
- logger.debug('AgentRuntime', `Screenshot skipped: ${error.message}`);
532
- }
533
- return undefined;
534
- }
535
- }
536
-
537
- // ─── Screen Context for Voice Mode ──────────────────────
538
-
539
- /**
540
- * Get current screen context as formatted text.
541
- * Used by voice mode: sent once at connect + after each tool call.
542
- * Tree goes in user prompt, not system instructions.
543
- */
544
- public getScreenContext(): string {
545
- try {
546
- logger.debug('AgentRuntime', 'getScreenContext called');
547
- logger.debug('AgentRuntime', 'config.screenMap exists:', !!this.config.screenMap);
548
- if (this.config.screenMap) {
549
- logger.debug('AgentRuntime', 'screenMap.screens count:', Object.keys(this.config.screenMap.screens).length);
550
- logger.debug('AgentRuntime', 'screenMap.chains count:', this.config.screenMap.chains?.length);
551
- }
552
-
553
- const walkResult = walkFiberTree(this.rootRef, this.getWalkConfig());
554
- const screenName = this.getCurrentScreenName();
555
- logger.debug('AgentRuntime', 'current screen:', screenName);
556
-
557
- const screen = dehydrateScreen(
558
- screenName,
559
- this.getRouteNames(),
560
- walkResult.elementsText,
561
- walkResult.interactives,
562
- );
563
-
564
- const routeNames = this.getRouteNames();
565
- logger.debug('AgentRuntime', 'routeNames:', routeNames);
566
- let availableScreensText: string;
567
- let appMapText = '';
568
-
569
- if (this.config.screenMap) {
570
- const map = this.config.screenMap;
571
- logger.debug('AgentRuntime', 'USING SCREEN MAP - enriching context');
572
-
573
- const screenLines = routeNames.map(name => {
574
- const entry = map.screens[name];
575
- if (entry) {
576
- const title = entry.title ? ` (${entry.title})` : '';
577
- const line = `- ${name}${title}: ${entry.description}`;
578
- logger.debug('AgentRuntime', 'matched:', line);
579
- return line;
580
- }
581
- logger.debug('AgentRuntime', 'NO MATCH for route:', name);
582
- return `- ${name}`;
583
- });
584
- availableScreensText = `Available Screens:\n${screenLines.join('\n')}`;
585
-
586
- if (map.chains && map.chains.length > 0) {
587
- const chainLines = map.chains.map(chain => ` ${chain.join(' → ')}`);
588
- appMapText = `\nNavigation Chains:\n${chainLines.join('\n')}`;
589
- logger.debug('AgentRuntime', 'chains:', chainLines.length);
590
- }
591
-
592
- this.detectStaleMap(routeNames, map);
593
- } else {
594
- logger.debug('AgentRuntime', 'NO SCREEN MAP - using flat list');
595
- availableScreensText = `Available Screens: ${routeNames.join(', ')}`;
596
- }
597
-
598
- const context = `<screen_update>
599
- Current Screen: ${screenName}
600
- ${availableScreensText}${appMapText}
601
-
602
- ${screen.elementsText}
603
- </screen_update>`;
604
- logger.debug('AgentRuntime', 'FULL CONTEXT:', context.substring(0, 500));
605
- return context;
606
- } catch (error: any) {
607
- logger.debug('AgentRuntime', 'getScreenContext ERROR:', error.message);
608
- logger.error('AgentRuntime', `getScreenContext failed: ${error.message}`);
609
- return '<screen_update>Error reading screen</screen_update>';
610
- }
611
- }
612
- // ─── Stale Map Detection ─────────────────────────────────────
613
-
614
- private staleMapWarned = false;
615
-
616
- private detectStaleMap(routeNames: string[], map: { screens: Record<string, any> }) {
617
- if (this.staleMapWarned) return; // Only warn once
618
-
619
- const mapScreens = new Set(Object.keys(map.screens));
620
- const missing = routeNames.filter(r => !mapScreens.has(r));
621
-
622
- if (missing.length > 0) {
623
- this.staleMapWarned = true;
624
- console.warn(
625
- `âš ī¸ [AIAgent] Screens not in map: "${missing.join('", "')}". ` +
626
- `Run 'npx react-native-ai-agent generate-map' to update.`
627
- );
628
- }
629
- }
630
-
631
- // ─── Build Tools Array for Provider ────────────────────────
632
-
633
- private buildToolsForProvider(): ToolDefinition[] {
634
- const allTools = [...this.tools.values()];
635
-
636
- // Add registered actions as tools
637
- for (const action of actionRegistry.getAll()) {
638
- const toolParams: Record<string, any> = {};
639
- for (const [key, val] of Object.entries(action.parameters)) {
640
- if (typeof val === 'string') {
641
- toolParams[key] = { type: 'string', description: val, required: true };
642
- } else {
643
- toolParams[key] = {
644
- type: val.type,
645
- description: val.description,
646
- required: val.required !== false,
647
- enum: val.enum
648
- };
649
- }
650
- }
651
-
652
- allTools.push({
653
- name: action.name,
654
- description: action.description,
655
- parameters: toolParams,
656
- execute: async (args) => {
657
- try {
658
- const result = await action.handler(args);
659
- logger.info('AgentRuntime', `Action "${action.name}" result:`, JSON.stringify(result));
660
- return typeof result === 'string' ? result : JSON.stringify(result);
661
- } catch (error: any) {
662
- return `❌ Action "${action.name}" failed: ${error.message}`;
663
- }
664
- },
665
- });
666
- }
667
-
668
- return allTools;
669
- }
670
-
671
- /** Public accessor for voice mode — returns all registered tool definitions. */
672
- public getTools(): ToolDefinition[] {
673
- return this.buildToolsForProvider();
674
- }
675
-
676
- /** Execute a tool by name (for voice mode tool calls from WebSocket). */
677
- public async executeTool(name: string, args: Record<string, any>): Promise<string> {
678
- const tool = this.tools.get(name) ||
679
- this.buildToolsForProvider().find(t => t.name === name);
680
- if (!tool) {
681
- return `❌ Unknown tool: ${name}`;
682
- }
683
- return this.executeToolSafely(tool, args, name);
684
- }
685
- /**
686
- * Start 3-layer error suppression for the agent task lifecycle.
687
- *
688
- * Layer 1 — ErrorUtils: Catches non-React async errors (setTimeout, fetch, native callbacks).
689
- * Layer 2 — console.reportErrorsAsExceptions: React Native dev-mode flag. When false,
690
- * console.error calls don't trigger ExceptionsManager.handleException(),
691
- * preventing the red "Render Error" screen for errors that React surfaces
692
- * via console.error (useEffect, lifecycle, invariant violations).
693
- * Layer 3 — Grace period (in _stopErrorSuppression): Keeps suppression active
694
- * for N ms after task completion, covering delayed useEffect effects.
695
- *
696
- * Same compound approach used by Sentry React Native SDK (ErrorUtils + ExceptionsManager override).
697
- */
698
- private _startErrorSuppression(): void {
699
- // Cancel any pending grace timer from a previous task
700
- if (this.graceTimer) {
701
- clearTimeout(this.graceTimer);
702
- this.graceTimer = null;
703
- }
704
-
705
- // Layer 1: ErrorUtils global handler
706
- const ErrorUtils = (global as any).ErrorUtils;
707
- if (ErrorUtils?.setGlobalHandler) {
708
- this.originalErrorHandler = ErrorUtils.getGlobalHandler?.() ?? null;
709
- this.lastSuppressedError = null;
710
- ErrorUtils.setGlobalHandler((error: Error, isFatal?: boolean) => {
711
- this.lastSuppressedError = error;
712
- logger.warn(
713
- 'AgentRuntime',
714
- `đŸ›Ąī¸ Suppressed ${isFatal ? 'FATAL' : 'non-fatal'} error during agent task: ${error.message}`
715
- );
716
- // Don't re-throw — suppress the crash entirely.
717
- });
718
- }
719
-
720
- // Layer 2: Suppress dev-mode red screen
721
- // In RN dev mode, useEffect errors trigger console.error → ExceptionsManager → red screen.
722
- // This flag is the official RN mechanism to disable that pipeline.
723
- const consoleAny = console as any;
724
- if (consoleAny.reportErrorsAsExceptions !== undefined) {
725
- this.originalReportErrorsAsExceptions = consoleAny.reportErrorsAsExceptions;
726
- consoleAny.reportErrorsAsExceptions = false;
727
- }
728
- }
729
-
730
- /**
731
- * Stop error suppression after a grace period.
732
- * The grace period covers delayed React side-effects (useEffect, PagerView onPageSelected,
733
- * scrollToIndex) that can fire AFTER execute() returns.
734
- */
735
- private _stopErrorSuppression(gracePeriodMs: number = 0): void {
736
- const restore = () => {
737
- // Restore Layer 1: ErrorUtils
738
- const ErrorUtils = (global as any).ErrorUtils;
739
- if (ErrorUtils?.setGlobalHandler && this.originalErrorHandler) {
740
- ErrorUtils.setGlobalHandler(this.originalErrorHandler);
741
- this.originalErrorHandler = null;
742
- }
743
- this.lastSuppressedError = null;
744
-
745
- // Restore Layer 2: console.reportErrorsAsExceptions
746
- const consoleAny = console as any;
747
- if (this.originalReportErrorsAsExceptions !== undefined) {
748
- consoleAny.reportErrorsAsExceptions = this.originalReportErrorsAsExceptions;
749
- this.originalReportErrorsAsExceptions = undefined;
750
- }
751
-
752
- this.graceTimer = null;
753
- };
754
-
755
- if (gracePeriodMs > 0) {
756
- this.graceTimer = setTimeout(restore, gracePeriodMs);
757
- } else {
758
- restore();
759
- }
760
- }
761
-
762
- /**
763
- * Execute a tool with safety checks.
764
- * Validates args before execution (Detox/Appium pattern).
765
- * Checks for async errors that were suppressed during the settle window.
766
- * The global ErrorUtils handler is task-scoped (installed in execute()),
767
- * so this method only needs to CHECK for errors, not install/remove.
768
- */
769
- private async executeToolSafely(
770
- tool: { execute: (args: any) => Promise<string> },
771
- args: any,
772
- toolName: string
773
- ): Promise<string> {
774
- // Clear any previous suppressed error before this tool
775
- this.lastSuppressedError = null;
776
-
777
- // Signal analytics that the AGENT is acting (not the user).
778
- // This prevents AI-driven taps from being tracked as user_interaction events.
779
- this.config.onToolExecute?.(true);
780
-
781
- try {
782
- // ── Argument Validation (Pattern from Detox/Appium: typeof checks before native dispatch) ──
783
- const validationError = this.validateToolArgs(args, toolName);
784
- if (validationError) {
785
- logger.warn('AgentRuntime', `đŸ›Ąī¸ Arg validation rejected "${toolName}": ${validationError}`);
786
- return validationError;
787
- }
788
-
789
- // ── Copilot aiConfirm gate ──────────────────────────────────
790
- // In copilot mode, elements marked with aiConfirm={true} require
791
- // user confirmation before execution. This is the code-level safety net
792
- // complementing the prompt-level copilot instructions.
793
- if (this.config.interactionMode !== 'autopilot') {
794
- const confirmResult = await this.checkCopilotConfirmation(toolName, args);
795
- if (confirmResult) return confirmResult;
796
- }
797
-
798
- const result = await tool.execute(args);
799
-
800
- // Settle window for async side-effects (useEffect, native callbacks)
801
- // The global ErrorUtils handler catches any errors during this window
802
- await new Promise(resolve => setTimeout(resolve, 2000));
803
-
804
- const suppressedError = this.lastSuppressedError as Error | null;
805
- if (suppressedError) {
806
- logger.warn('AgentRuntime', `đŸ›Ąī¸ Tool "${toolName}" caused async error (suppressed): ${suppressedError.message}`);
807
- this.lastSuppressedError = null;
808
- return `${result} (âš ī¸ a background error was safely caught: ${suppressedError.message})`;
809
- }
810
- return result;
811
- } catch (error: any) {
812
- logger.error('AgentRuntime', `Tool "${toolName}" threw: ${error.message}`);
813
- return `❌ Tool "${toolName}" failed: ${error.message}`;
814
- } finally {
815
- // Always restore the flag — even on error or validation rejection
816
- this.config.onToolExecute?.(false);
817
- }
818
- }
819
-
820
- /**
821
- * Validate tool arguments before execution.
822
- * Pattern from Detox: `typeof index !== 'number' → throw Error`
823
- * Pattern from Appium: `_.isFinite(x) && _.isFinite(y)` for coordinates
824
- * Returns error string if validation fails, null if valid.
825
- */
826
- private validateToolArgs(args: any, toolName: string): string | null {
827
- if (!args || typeof args !== 'object') return null;
828
-
829
- // Reject any null/undefined values that could crash native components
830
- for (const [key, value] of Object.entries(args)) {
831
- if (value === undefined) {
832
- return `❌ Argument "${key}" is undefined for tool "${toolName}". Provide a valid value.`;
833
- }
834
- }
835
-
836
- // Tool-specific number validation (like Detox's typeof checks)
837
- const numericArgs = ['containerIndex', 'index', 'x', 'y', 'offset'];
838
- for (const key of numericArgs) {
839
- if (key in args && args[key] !== null && args[key] !== undefined) {
840
- const val = args[key];
841
- if (typeof val !== 'number' || !Number.isFinite(val)) {
842
- return `❌ Argument "${key}" must be a finite number for tool "${toolName}", got ${typeof val}: ${val}`;
843
- }
844
- }
845
- }
846
-
847
- return null;
848
- }
849
-
850
- // ─── Copilot Confirmation ─────────────────────────────────────
851
-
852
- /** Write tools that can mutate state — only these are checked for aiConfirm */
853
- private static readonly WRITE_TOOLS = new Set([
854
- 'tap', 'type', 'long_press', 'adjust_slider', 'select_picker', 'set_date',
855
- ]);
856
-
857
- /**
858
- * Check if a tool call targets an aiConfirm element and request user confirmation.
859
- * Returns null if the action should proceed, or an error string if rejected.
860
- */
861
- private async checkCopilotConfirmation(
862
- toolName: string,
863
- args: Record<string, any>,
864
- ): Promise<string | null> {
865
- // Only gate write tools
866
- if (!AgentRuntime.WRITE_TOOLS.has(toolName)) return null;
867
-
868
- // Look up the target element by index
869
- const index = args.index;
870
- if (typeof index !== 'number') return null;
871
-
872
- const screen = this.lastDehydratedRoot as import('./types').DehydratedScreen | null;
873
- if (!screen?.elements) return null;
874
-
875
- const element = screen.elements.find(e => e.index === index);
876
- if (!element?.requiresConfirmation) return null;
877
-
878
- // Element has aiConfirm — request user confirmation
879
- const label = element.label || `[${element.type}]`;
880
- const description = this.getToolStatusLabel(toolName, args);
881
- const question = `I'm about to ${description} on "${label}". Should I proceed?`;
882
-
883
- logger.info('AgentRuntime', `đŸ›Ąī¸ Copilot: aiConfirm gate triggered for "${toolName}" on "${label}"`);
884
-
885
- // Use onAskUser if available (integrated into chat UI), otherwise Alert.alert
886
- if (this.config.onAskUser) {
887
- const response = await this.config.onAskUser(question);
888
- const approved = /^(yes|ok|sure|go|proceed|confirm|y)/i.test(response.trim());
889
- if (!approved) {
890
- logger.info('AgentRuntime', `🛑 User rejected "${toolName}" on "${label}"`);
891
- return `❌ User rejected "${toolName}" action on "${label}". Ask what they would like to do instead.`;
892
- }
893
- return null;
894
- }
895
-
896
- // Fallback: React Native Alert
897
- const { Alert } = require('react-native');
898
- const approved = await new Promise<boolean>(resolve => {
899
- Alert.alert(
900
- 'Confirm Action',
901
- question,
902
- [
903
- { text: 'Cancel', style: 'cancel', onPress: () => resolve(false) },
904
- { text: 'Continue', onPress: () => resolve(true) },
905
- ],
906
- { cancelable: false },
907
- );
908
- });
909
-
910
- if (!approved) {
911
- logger.info('AgentRuntime', `🛑 User rejected "${toolName}" on "${label}"`);
912
- return `❌ User rejected "${toolName}" action on "${label}". Ask what they would like to do instead.`;
913
- }
914
-
915
- return null;
916
- }
917
-
918
- // ─── Walk Config (passes security settings to FiberTreeWalker) ─
919
-
920
- private getWalkConfig(): WalkConfig {
921
- return {
922
- interactiveBlacklist: this.config.interactiveBlacklist,
923
- interactiveWhitelist: this.config.interactiveWhitelist,
924
- screenName: this.getCurrentScreenName(),
925
- };
926
- }
927
-
928
- // ─── Instructions ───────
929
-
930
- private getInstructions(screenName: string): string {
931
- const { instructions } = this.config;
932
- if (!instructions) return '';
933
-
934
- let result = '';
935
- if (instructions.system?.trim()) {
936
- result += `<system_instructions>\n${instructions.system.trim()}\n</system_instructions>\n`;
937
- }
938
-
939
- if (instructions.getScreenInstructions) {
940
- try {
941
- const screenInstructions = instructions.getScreenInstructions(screenName)?.trim();
942
- if (screenInstructions) {
943
- result += `<screen_instructions>\n${screenInstructions}\n</screen_instructions>\n`;
944
- }
945
- } catch (error) {
946
- logger.error('AgentRuntime', 'Failed to get screen instructions:', error);
947
- }
948
- }
949
-
950
- return result ? `<instructions>\n${result}</instructions>\n\n` : '';
951
- }
952
-
953
- // ─── Observation System ──
954
-
955
- private observations: string[] = [];
956
- private lastScreenName: string = '';
957
-
958
- private handleObservations(step: number, maxSteps: number, screenName: string): void {
959
- // Screen change detection
960
- if (this.lastScreenName && screenName !== this.lastScreenName) {
961
- this.observations.push(`Screen navigated to → ${screenName}`);
962
- }
963
- this.lastScreenName = screenName;
964
-
965
- // Remaining steps warning
966
- const remaining = maxSteps - step;
967
- if (remaining === 5) {
968
- this.observations.push(
969
- `âš ī¸ Only ${remaining} steps remaining. Consider wrapping up or calling done with partial results.`
970
- );
971
- } else if (remaining === 2) {
972
- this.observations.push(
973
- `âš ī¸ Critical: Only ${remaining} steps left! You must finish the task or call done immediately.`
974
- );
975
- }
976
- }
977
-
978
- // ─── User Prompt Assembly ──
979
-
980
- private assembleUserPrompt(
981
- step: number,
982
- maxSteps: number,
983
- contextualMessage: string,
984
- screenName: string,
985
- screenContent: string,
986
- chatHistory?: { role: string; content: string }[]
987
- ): string {
988
- let prompt = '';
989
-
990
- // 1. <instructions> (optional system/screen instructions)
991
- prompt += this.getInstructions(screenName);
992
-
993
- // 2. <agent_state> — user request + step info
994
- prompt += '<agent_state>\n';
995
- prompt += '<user_request>\n';
996
- prompt += `${contextualMessage}\n`;
997
- prompt += '</user_request>\n';
998
-
999
- if (chatHistory && chatHistory.length > 0) {
1000
- prompt += '<chat_history>\n';
1001
- // Only include the last 10 messages to manage context length
1002
- const recentHistory = chatHistory.slice(-10);
1003
- for (const msg of recentHistory) {
1004
- prompt += `[${msg.role}]: ${msg.content}\n`;
1005
- }
1006
- prompt += '</chat_history>\n';
1007
- }
1008
-
1009
- prompt += '<step_info>\n';
1010
- prompt += `Step ${step + 1} of ${maxSteps} max possible steps\n`;
1011
- prompt += '</step_info>\n';
1012
- prompt += '</agent_state>\n\n';
1013
-
1014
- // 3. <agent_history> — structured per-step history
1015
- prompt += '<agent_history>\n';
1016
-
1017
- // History summarization: when steps > 8, compress middle steps
1018
- // to bound prompt growth for long tasks (approaching 25-step limit).
1019
- // Keep first 2 (initial context) + last 4 (recent context) as full detail.
1020
- const SUMMARIZE_THRESHOLD = 8;
1021
- const KEEP_HEAD = 2;
1022
- const KEEP_TAIL = 4;
1023
- const shouldSummarize = this.history.length > SUMMARIZE_THRESHOLD;
1024
-
1025
- let stepIndex = 0;
1026
- for (let i = 0; i < this.history.length; i++) {
1027
- const event = this.history[i]!;
1028
- stepIndex++;
1029
-
1030
- if (shouldSummarize && i >= KEEP_HEAD && i < this.history.length - KEEP_TAIL) {
1031
- // First compressed step emits the summary block
1032
- if (i === KEEP_HEAD) {
1033
- prompt += '<steps_summary>\n';
1034
- for (let j = KEEP_HEAD; j < this.history.length - KEEP_TAIL; j++) {
1035
- const h = this.history[j]!;
1036
- const actionName = h.action.name || 'unknown';
1037
- const succeeded = h.action.output?.startsWith('✅') ? 'success' : 'fail';
1038
- prompt += `Step ${j + 1}: ${actionName} → ${succeeded}\n`;
1039
- }
1040
- prompt += '</steps_summary>\n';
1041
- }
1042
- continue; // Skip full detail for middle steps
1043
- }
1044
-
1045
- prompt += `<step_${stepIndex}>\n`;
1046
- prompt += `Previous Goal Eval: ${event.reflection.previousGoalEval}\n`;
1047
- prompt += `Memory: ${event.reflection.memory}\n`;
1048
- prompt += `Plan: ${event.reflection.plan}\n`;
1049
- prompt += `Action Result: ${event.action.output}\n`;
1050
- prompt += `</step_${stepIndex}>\n`;
1051
- }
1052
-
1053
- // Inject system observations
1054
- for (const obs of this.observations) {
1055
- prompt += `<sys>${obs}</sys>\n`;
1056
- }
1057
- this.observations = [];
1058
-
1059
- prompt += '</agent_history>\n\n';
1060
-
1061
- // 4. <screen_state> — dehydrated screen content + screen map enrichment
1062
- logger.debug('AgentRuntime', 'assembleUserPrompt: screenMap exists:', !!this.config.screenMap);
1063
- prompt += '<screen_state>\n';
1064
- prompt += `Current Screen: ${screenName}\n`;
1065
-
1066
- // Inject screen map descriptions & navigation chains if available
1067
- if (this.config.screenMap) {
1068
- const map = this.config.screenMap;
1069
- const routeNames = this.getRouteNames();
1070
- logger.debug('AgentRuntime', 'ENRICHING prompt with screenMap for screen:', screenName);
1071
-
1072
- // Build enriched screen list with descriptions
1073
- const screenLines = routeNames.map(name => {
1074
- const entry = map.screens[name];
1075
- if (entry) {
1076
- const title = entry.title ? ` (${entry.title})` : '';
1077
- return `- ${name}${title}: ${entry.description}`;
1078
- }
1079
- return `- ${name}`;
1080
- });
1081
- prompt += `\nAvailable Screens:\n${screenLines.join('\n')}\n`;
1082
-
1083
- // Add navigation chains
1084
- if (map.chains && map.chains.length > 0) {
1085
- const chainLines = map.chains.map(chain => ` ${chain.join(' → ')}`);
1086
- prompt += `\nNavigation Chains:\n${chainLines.join('\n')}\n`;
1087
- }
1088
- } else {
1089
- // Flat list fallback
1090
- const routeNames = this.getRouteNames();
1091
- prompt += `Available Screens: ${routeNames.join(', ')}\n`;
1092
- }
1093
-
1094
- prompt += '\n' + screenContent + '\n';
1095
- prompt += '</screen_state>\n';
1096
-
1097
- return prompt;
1098
- }
1099
-
1100
- // ─── Main Execution Loop ──────────────────────────────────────
1101
-
1102
- async execute(userMessage: string, chatHistory?: { role: string; content: string }[]): Promise<ExecutionResult> {
1103
- if (this.isRunning) {
1104
- return { success: false, message: 'Agent is already running.', steps: [] };
1105
- }
1106
-
1107
- this.isRunning = true;
1108
- this.isCancelRequested = false;
1109
- this.history = [];
1110
- this.observations = [];
1111
- this.lastScreenName = '';
1112
- const maxSteps = this.config.maxSteps || DEFAULT_MAX_STEPS;
1113
- const stepDelay = this.config.stepDelay ?? 300;
1114
-
1115
- // Token usage accumulator for the entire task
1116
- const sessionUsage: TokenUsage = {
1117
- promptTokens: 0,
1118
- completionTokens: 0,
1119
- totalTokens: 0,
1120
- estimatedCostUSD: 0,
1121
- };
1122
-
1123
- // Inject conversational context if we are answering the AI's question
1124
- let contextualMessage = userMessage;
1125
- if (this.lastAskUserQuestion) {
1126
- contextualMessage = `(Note: You just asked the user: "${this.lastAskUserQuestion}")\n\nUser replied: ${userMessage}`;
1127
- this.lastAskUserQuestion = null; // Consume the question
1128
- }
1129
-
1130
- logger.info('AgentRuntime', `Starting execution: "${contextualMessage}"`);
1131
-
1132
- // Lifecycle: onBeforeTask
1133
- await this.config.onBeforeTask?.();
1134
-
1135
- try {
1136
- // ── Start error suppression (3 layers) ──────────────────
1137
- this._startErrorSuppression();
1138
-
1139
- // ─── Knowledge-only fast path ─────────────────────────────────
1140
- // Skip fiber walk, dehydration, screenshots, and multi-step loop.
1141
- // Only sends the user question → single LLM call → done.
1142
- if (!this.isUIEnabled()) {
1143
- this.config.onStatusUpdate?.('Thinking...');
1144
- const hasKnowledge = !!this.knowledgeService;
1145
- const systemPrompt = buildKnowledgeOnlyPrompt(
1146
- 'en', hasKnowledge, this.config.instructions?.system,
1147
- );
1148
- const tools = this.buildToolsForProvider();
1149
- const screenName = this.getCurrentScreenName();
1150
-
1151
- // Minimal user prompt — just the question + screen name for context
1152
- const userPrompt = `Current screen: ${screenName}\n\nUser: ${contextualMessage}`;
1153
-
1154
- const response = await this.provider.generateContent(
1155
- systemPrompt, userPrompt, tools, [], undefined,
1156
- );
1157
-
1158
- // Track token usage
1159
- if (response.tokenUsage) {
1160
- sessionUsage.promptTokens += response.tokenUsage.promptTokens;
1161
- sessionUsage.completionTokens += response.tokenUsage.completionTokens;
1162
- sessionUsage.totalTokens += response.tokenUsage.totalTokens;
1163
- sessionUsage.estimatedCostUSD += response.tokenUsage.estimatedCostUSD;
1164
- this.config.onTokenUsage?.(response.tokenUsage);
1165
- }
1166
-
1167
- // Execute tool calls (done / query_knowledge)
1168
- let message = response.text || '';
1169
- if (response.toolCalls) {
1170
- for (const tc of response.toolCalls) {
1171
- const tool = this.tools.get(tc.name);
1172
- if (tool) {
1173
- const result = await this.executeToolSafely(tool, tc.args, tc.name);
1174
- if (tc.name === 'done') {
1175
- message = result;
1176
- } else if (tc.name === 'query_knowledge') {
1177
- // Knowledge retrieved — need a second call with the results
1178
- const followUp = `Knowledge result:\n${result}\n\nUser question: ${contextualMessage}\n\nAnswer the user based on this knowledge. Call done() with your answer.`;
1179
- const followUpResponse = await this.provider.generateContent(
1180
- systemPrompt, followUp, tools, [], undefined,
1181
- );
1182
- if (followUpResponse.tokenUsage) {
1183
- sessionUsage.promptTokens += followUpResponse.tokenUsage.promptTokens;
1184
- sessionUsage.completionTokens += followUpResponse.tokenUsage.completionTokens;
1185
- sessionUsage.totalTokens += followUpResponse.tokenUsage.totalTokens;
1186
- sessionUsage.estimatedCostUSD += followUpResponse.tokenUsage.estimatedCostUSD;
1187
- this.config.onTokenUsage?.(followUpResponse.tokenUsage);
1188
- }
1189
- if (followUpResponse.toolCalls) {
1190
- for (const ftc of followUpResponse.toolCalls) {
1191
- if (ftc.name === 'done') {
1192
- const doneResult = await this.tools.get('done')!.execute(ftc.args);
1193
- message = doneResult;
1194
- }
1195
- }
1196
- }
1197
- if (!message && followUpResponse.text) {
1198
- message = followUpResponse.text;
1199
- }
1200
- }
1201
- }
1202
- }
1203
- }
1204
-
1205
- const result: ExecutionResult = {
1206
- success: true,
1207
- message: message || 'I could not find an answer.',
1208
- steps: [],
1209
- tokenUsage: sessionUsage,
1210
- };
1211
- await this.config.onAfterTask?.(result);
1212
- return result;
1213
- }
1214
-
1215
- // ─── Full agent loop (UI control enabled) ─────────────────────
1216
- for (let step = 0; step < maxSteps; step++) {
1217
- // ── Cancel check ──
1218
- if (this.isCancelRequested) {
1219
- logger.info('AgentRuntime', `Task cancelled by user at step ${step + 1}`);
1220
- const cancelResult: ExecutionResult = {
1221
- success: false,
1222
- message: 'Task was cancelled.',
1223
- steps: this.history,
1224
- tokenUsage: sessionUsage,
1225
- };
1226
- await this.config.onAfterTask?.(cancelResult);
1227
- return cancelResult;
1228
- }
1229
- logger.info('AgentRuntime', `===== Step ${step + 1}/${maxSteps} =====`);
1230
-
1231
- // Lifecycle: onBeforeStep
1232
- await this.config.onBeforeStep?.(step);
1233
-
1234
- // 1. Walk Fiber tree with security config and dehydrate screen
1235
- const walkResult = walkFiberTree(this.rootRef, this.getWalkConfig());
1236
- const screenName = this.getCurrentScreenName();
1237
- const screen = dehydrateScreen(
1238
- screenName,
1239
- this.getRouteNames(),
1240
- walkResult.elementsText,
1241
- walkResult.interactives,
1242
- );
1243
-
1244
- // Store root for tooling access (e.g., GuideTool measuring)
1245
- this.lastDehydratedRoot = screen;
1246
-
1247
- logger.info('AgentRuntime', `Screen: ${screen.screenName}`);
1248
- logger.debug('AgentRuntime', `Dehydrated:\n${screen.elementsText}`);
1249
-
1250
- // 2. Apply transformScreenContent
1251
- let screenContent = screen.elementsText;
1252
- if (this.config.transformScreenContent) {
1253
- screenContent = await this.config.transformScreenContent(screenContent);
1254
- }
1255
-
1256
- // 3. Handle observations
1257
- this.handleObservations(step, maxSteps, screenName);
1258
-
1259
- // 4. Assemble structured user prompt
1260
- const contextMessage = this.assembleUserPrompt(
1261
- step, maxSteps, contextualMessage, screenName, screenContent, chatHistory
1262
- );
1263
-
1264
- // 4.5. Capture screenshot for Gemini vision (optional)
1265
- const screenshot = await this.captureScreenshot();
1266
-
1267
- // 5. Send to AI provider
1268
- this.config.onStatusUpdate?.('Analyzing screen...');
1269
- const hasKnowledge = !!this.knowledgeService;
1270
- const isCopilot = this.config.interactionMode !== 'autopilot';
1271
- const systemPrompt = buildSystemPrompt('en', hasKnowledge, isCopilot);
1272
- const tools = this.buildToolsForProvider();
1273
-
1274
- logger.info('AgentRuntime', `Sending to AI with ${tools.length} tools...`);
1275
- logger.debug('AgentRuntime', 'System prompt length:', systemPrompt.length);
1276
- logger.debug('AgentRuntime', 'User context message:', contextMessage.substring(0, 300));
1277
-
1278
- const response = await this.provider.generateContent(
1279
- systemPrompt,
1280
- contextMessage,
1281
- tools,
1282
- this.history,
1283
- screenshot,
1284
- );
1285
-
1286
- // Accumulate token usage
1287
- if (response.tokenUsage) {
1288
- sessionUsage.promptTokens += response.tokenUsage.promptTokens;
1289
- sessionUsage.completionTokens += response.tokenUsage.completionTokens;
1290
- sessionUsage.totalTokens += response.tokenUsage.totalTokens;
1291
- sessionUsage.estimatedCostUSD += response.tokenUsage.estimatedCostUSD;
1292
- this.config.onTokenUsage?.(response.tokenUsage);
1293
- }
1294
-
1295
- // ── Budget Guards ──────────────────────────────────────
1296
- if (this.config.maxTokenBudget && sessionUsage.totalTokens >= this.config.maxTokenBudget) {
1297
- logger.warn('AgentRuntime', `Token budget exceeded: ${sessionUsage.totalTokens} >= ${this.config.maxTokenBudget}`);
1298
- const budgetResult: ExecutionResult = {
1299
- success: false,
1300
- message: `Task stopped: token budget exceeded (used ${sessionUsage.totalTokens.toLocaleString()} of ${this.config.maxTokenBudget.toLocaleString()} tokens)`,
1301
- steps: this.history,
1302
- tokenUsage: sessionUsage,
1303
- };
1304
- await this.config.onAfterTask?.(budgetResult);
1305
- return budgetResult;
1306
- }
1307
- if (this.config.maxCostUSD && sessionUsage.estimatedCostUSD >= this.config.maxCostUSD) {
1308
- logger.warn('AgentRuntime', `Cost budget exceeded: $${sessionUsage.estimatedCostUSD.toFixed(4)} >= $${this.config.maxCostUSD}`);
1309
- const budgetResult: ExecutionResult = {
1310
- success: false,
1311
- message: `Task stopped: cost budget exceeded ($${sessionUsage.estimatedCostUSD.toFixed(4)} of $${this.config.maxCostUSD.toFixed(2)} max)`,
1312
- steps: this.history,
1313
- tokenUsage: sessionUsage,
1314
- };
1315
- await this.config.onAfterTask?.(budgetResult);
1316
- return budgetResult;
1317
- }
1318
-
1319
- // 6. Process tool calls
1320
- if (!response.toolCalls || response.toolCalls.length === 0) {
1321
- logger.warn('AgentRuntime', 'No tool calls in response. Text:', response.text);
1322
- const result: ExecutionResult = {
1323
- success: true,
1324
- message: response.text || 'Task completed.',
1325
- steps: this.history,
1326
- tokenUsage: sessionUsage,
1327
- };
1328
- await this.config.onAfterTask?.(result);
1329
- return result;
1330
- }
1331
-
1332
- // 7. Structured reasoning from provider (no regex parsing needed)
1333
- const { reasoning } = response;
1334
- logger.info('AgentRuntime', `🧠 Plan: ${reasoning.plan}`);
1335
- if (reasoning.memory) {
1336
- logger.debug('AgentRuntime', `💾 Memory: ${reasoning.memory}`);
1337
- }
1338
-
1339
- // Only process the FIRST tool call per step (one action per step).
1340
- // After one action, the loop re-reads the screen with fresh indexes.
1341
- const toolCall = response.toolCalls[0]!;
1342
- if (response.toolCalls.length > 1) {
1343
- logger.warn('AgentRuntime', `AI returned ${response.toolCalls.length} tool calls, executing only the first one.`);
1344
- }
1345
-
1346
- logger.info('AgentRuntime', `Tool: ${toolCall.name}(${JSON.stringify(toolCall.args)})`);
1347
-
1348
- // Dynamic status update based on tool being executed + Reasoning
1349
- const statusLabel = this.getToolStatusLabel(toolCall.name, toolCall.args);
1350
- // Prefer the human-readable plan over the raw tool status if available to avoid double statuses
1351
- const statusDisplay = reasoning.plan || statusLabel;
1352
- this.config.onStatusUpdate?.(statusDisplay);
1353
-
1354
- // Find and execute the tool
1355
- const tool = this.tools.get(toolCall.name) ||
1356
- this.buildToolsForProvider().find(t => t.name === toolCall.name);
1357
-
1358
- let output: string;
1359
- if (tool) {
1360
- output = await this.executeToolSafely(tool, toolCall.args, toolCall.name);
1361
- } else {
1362
- output = `❌ Unknown tool: ${toolCall.name}`;
1363
- }
1364
-
1365
- logger.info('AgentRuntime', `Result: ${output}`);
1366
-
1367
- // Record step with structured reasoning
1368
- const agentStep: AgentStep = {
1369
- stepIndex: step,
1370
- reflection: reasoning,
1371
- action: {
1372
- name: toolCall.name,
1373
- input: toolCall.args,
1374
- output,
1375
- },
1376
- };
1377
- this.history.push(agentStep);
1378
-
1379
- // Lifecycle: onAfterStep
1380
- await this.config.onAfterStep?.(this.history);
1381
-
1382
- // Check if done
1383
- if (toolCall.name === 'done') {
1384
- const result: ExecutionResult = {
1385
- success: toolCall.args.success !== false,
1386
- message: toolCall.args.text || output,
1387
- steps: this.history,
1388
- tokenUsage: sessionUsage,
1389
- };
1390
- logger.info('AgentRuntime', `Task completed: ${result.message}`);
1391
- await this.config.onAfterTask?.(result);
1392
- return result;
1393
- }
1394
-
1395
- // Check if asking user (legacy path — only breaks loop when onAskUser is NOT set)
1396
- if (toolCall.name === 'ask_user' && !this.config.onAskUser) {
1397
- this.lastAskUserQuestion = toolCall.args.question || output;
1398
-
1399
- const result: ExecutionResult = {
1400
- success: true,
1401
- message: output,
1402
- steps: this.history,
1403
- tokenUsage: sessionUsage,
1404
- };
1405
- await this.config.onAfterTask?.(result);
1406
- return result;
1407
- }
1408
-
1409
- // Step delay
1410
- await new Promise(resolve => setTimeout(resolve, stepDelay));
1411
- }
1412
-
1413
- // Max steps reached
1414
- const result: ExecutionResult = {
1415
- success: false,
1416
- message: `Reached maximum steps (${maxSteps}) without completing the task.`,
1417
- steps: this.history,
1418
- tokenUsage: sessionUsage,
1419
- };
1420
-
1421
- // Dev warning: remind developers to add aiConfirm for extra safety
1422
- if (__DEV__ && this.config.interactionMode !== 'autopilot') {
1423
- logger.info('AgentRuntime',
1424
- 'â„šī¸ Copilot mode active. Tip: Add aiConfirm={true} to critical buttons (e.g. "Place Order", "Delete") for extra safety.'
1425
- );
1426
- }
1427
-
1428
- await this.config.onAfterTask?.(result);
1429
- return result;
1430
- } catch (error: any) {
1431
- logger.error('AgentRuntime', 'Execution error:', error);
1432
- const result: ExecutionResult = {
1433
- success: false,
1434
- message: `Error: ${error.message}`,
1435
- steps: this.history,
1436
- tokenUsage: sessionUsage,
1437
- };
1438
- await this.config.onAfterTask?.(result);
1439
- return result;
1440
- } finally {
1441
- this.isRunning = false;
1442
- // ── Grace period: keep error suppression for delayed side-effects ──
1443
- // useEffect callbacks, PagerView onPageSelected, scrollToIndex, etc.
1444
- // can fire AFTER execute() returns. Keep suppression active for 10s.
1445
- this._stopErrorSuppression(10000);
1446
- }
1447
- }
1448
-
1449
- /** Update refs (called when component re-renders) */
1450
- updateRefs(rootRef: any, navRef: any): void {
1451
- this.rootRef = rootRef;
1452
- this.navRef = navRef;
1453
- }
1454
-
1455
- /** Check if agent is currently executing */
1456
- getIsRunning(): boolean {
1457
- return this.isRunning;
1458
- }
1459
-
1460
- /**
1461
- * Cancel the currently running task.
1462
- * The agent loop checks this flag at the start of each step,
1463
- * so the current step will complete before the task stops.
1464
- */
1465
- cancel(): void {
1466
- if (this.isRunning) {
1467
- this.isCancelRequested = true;
1468
- logger.info('AgentRuntime', 'Cancel requested — will stop after current step completes');
1469
- }
1470
- }
1471
- }