@mobileai/react-native 0.9.17 → 0.9.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (292) hide show
  1. package/LICENSE +28 -20
  2. package/MobileAIFloatingOverlay.podspec +25 -0
  3. package/android/build.gradle +61 -0
  4. package/android/src/main/AndroidManifest.xml +3 -0
  5. package/android/src/main/java/com/mobileai/overlay/FloatingOverlayView.kt +151 -0
  6. package/android/src/main/java/com/mobileai/overlay/MobileAIOverlayPackage.kt +23 -0
  7. package/android/src/newarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +45 -0
  8. package/android/src/oldarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +29 -0
  9. package/ios/MobileAIFloatingOverlayComponentView.mm +73 -0
  10. package/lib/module/components/AIAgent.js +902 -136
  11. package/lib/module/components/AIConsentDialog.js +439 -0
  12. package/lib/module/components/AgentChatBar.js +828 -134
  13. package/lib/module/components/AgentOverlay.js +2 -1
  14. package/lib/module/components/DiscoveryTooltip.js +21 -9
  15. package/lib/module/components/FloatingOverlayWrapper.js +108 -0
  16. package/lib/module/components/Icons.js +123 -0
  17. package/lib/module/config/endpoints.js +12 -2
  18. package/lib/module/core/AgentRuntime.js +373 -27
  19. package/lib/module/core/FiberAdapter.js +56 -0
  20. package/lib/module/core/FiberTreeWalker.js +186 -80
  21. package/lib/module/core/IdleDetector.js +19 -0
  22. package/lib/module/core/NativeAlertInterceptor.js +191 -0
  23. package/lib/module/core/systemPrompt.js +203 -45
  24. package/lib/module/index.js +3 -0
  25. package/lib/module/providers/GeminiProvider.js +72 -56
  26. package/lib/module/providers/ProviderFactory.js +6 -2
  27. package/lib/module/services/AudioInputService.js +3 -12
  28. package/lib/module/services/AudioOutputService.js +1 -13
  29. package/lib/module/services/ConversationService.js +166 -0
  30. package/lib/module/services/MobileAIKnowledgeRetriever.js +41 -0
  31. package/lib/module/services/VoiceService.js +29 -8
  32. package/lib/module/services/telemetry/MobileAI.js +44 -0
  33. package/lib/module/services/telemetry/TelemetryService.js +13 -1
  34. package/lib/module/services/telemetry/TouchAutoCapture.js +44 -18
  35. package/lib/module/specs/FloatingOverlayNativeComponent.ts +19 -0
  36. package/lib/module/support/CSATSurvey.js +95 -12
  37. package/lib/module/support/EscalationSocket.js +70 -1
  38. package/lib/module/support/ReportedIssueEventSource.js +148 -0
  39. package/lib/module/support/escalateTool.js +4 -2
  40. package/lib/module/support/index.js +1 -0
  41. package/lib/module/support/reportIssueTool.js +127 -0
  42. package/lib/module/support/supportPrompt.js +77 -9
  43. package/lib/module/tools/guideTool.js +2 -1
  44. package/lib/module/tools/longPressTool.js +4 -3
  45. package/lib/module/tools/pickerTool.js +6 -4
  46. package/lib/module/tools/tapTool.js +12 -3
  47. package/lib/module/tools/typeTool.js +19 -10
  48. package/lib/module/utils/logger.js +175 -6
  49. package/lib/typescript/react-native.config.d.ts +11 -0
  50. package/lib/typescript/src/components/AIAgent.d.ts +28 -2
  51. package/lib/typescript/src/components/AIConsentDialog.d.ts +153 -0
  52. package/lib/typescript/src/components/AgentChatBar.d.ts +15 -2
  53. package/lib/typescript/src/components/DiscoveryTooltip.d.ts +3 -1
  54. package/lib/typescript/src/components/FloatingOverlayWrapper.d.ts +51 -0
  55. package/lib/typescript/src/components/Icons.d.ts +8 -0
  56. package/lib/typescript/src/config/endpoints.d.ts +5 -3
  57. package/lib/typescript/src/core/AgentRuntime.d.ts +4 -0
  58. package/lib/typescript/src/core/FiberAdapter.d.ts +25 -0
  59. package/lib/typescript/src/core/FiberTreeWalker.d.ts +2 -0
  60. package/lib/typescript/src/core/IdleDetector.d.ts +11 -0
  61. package/lib/typescript/src/core/NativeAlertInterceptor.d.ts +55 -0
  62. package/lib/typescript/src/core/types.d.ts +106 -1
  63. package/lib/typescript/src/index.d.ts +9 -4
  64. package/lib/typescript/src/providers/GeminiProvider.d.ts +6 -5
  65. package/lib/typescript/src/services/ConversationService.d.ts +55 -0
  66. package/lib/typescript/src/services/MobileAIKnowledgeRetriever.d.ts +9 -0
  67. package/lib/typescript/src/services/telemetry/MobileAI.d.ts +7 -0
  68. package/lib/typescript/src/services/telemetry/TelemetryService.d.ts +1 -1
  69. package/lib/typescript/src/services/telemetry/TouchAutoCapture.d.ts +9 -6
  70. package/lib/typescript/src/services/telemetry/types.d.ts +3 -1
  71. package/lib/typescript/src/specs/FloatingOverlayNativeComponent.d.ts +17 -0
  72. package/lib/typescript/src/support/EscalationSocket.d.ts +17 -0
  73. package/lib/typescript/src/support/ReportedIssueEventSource.d.ts +24 -0
  74. package/lib/typescript/src/support/escalateTool.d.ts +5 -0
  75. package/lib/typescript/src/support/index.d.ts +2 -1
  76. package/lib/typescript/src/support/reportIssueTool.d.ts +20 -0
  77. package/lib/typescript/src/support/types.d.ts +56 -1
  78. package/lib/typescript/src/utils/logger.d.ts +15 -0
  79. package/package.json +20 -9
  80. package/react-native.config.js +12 -0
  81. package/lib/module/__cli_tmp__.js.map +0 -1
  82. package/lib/module/components/AIAgent.js.map +0 -1
  83. package/lib/module/components/AIZone.js.map +0 -1
  84. package/lib/module/components/AgentChatBar.js.map +0 -1
  85. package/lib/module/components/AgentErrorBoundary.js.map +0 -1
  86. package/lib/module/components/AgentOverlay.js.map +0 -1
  87. package/lib/module/components/DiscoveryTooltip.js.map +0 -1
  88. package/lib/module/components/HighlightOverlay.js.map +0 -1
  89. package/lib/module/components/Icons.js.map +0 -1
  90. package/lib/module/components/ProactiveHint.js.map +0 -1
  91. package/lib/module/components/cards/InfoCard.js.map +0 -1
  92. package/lib/module/components/cards/ReviewSummary.js.map +0 -1
  93. package/lib/module/config/endpoints.js.map +0 -1
  94. package/lib/module/core/ActionRegistry.js.map +0 -1
  95. package/lib/module/core/AgentRuntime.js.map +0 -1
  96. package/lib/module/core/FiberTreeWalker.js.map +0 -1
  97. package/lib/module/core/IdleDetector.js.map +0 -1
  98. package/lib/module/core/MCPBridge.js.map +0 -1
  99. package/lib/module/core/ScreenDehydrator.js.map +0 -1
  100. package/lib/module/core/ZoneRegistry.js.map +0 -1
  101. package/lib/module/core/systemPrompt.js.map +0 -1
  102. package/lib/module/core/types.js.map +0 -1
  103. package/lib/module/hooks/useAction.js.map +0 -1
  104. package/lib/module/index.js.map +0 -1
  105. package/lib/module/plugin/withAppIntents.js.map +0 -1
  106. package/lib/module/providers/GeminiProvider.js.map +0 -1
  107. package/lib/module/providers/OpenAIProvider.js.map +0 -1
  108. package/lib/module/providers/ProviderFactory.js.map +0 -1
  109. package/lib/module/services/AudioInputService.js.map +0 -1
  110. package/lib/module/services/AudioOutputService.js.map +0 -1
  111. package/lib/module/services/KnowledgeBaseService.js.map +0 -1
  112. package/lib/module/services/VoiceService.js.map +0 -1
  113. package/lib/module/services/flags/FlagService.js.map +0 -1
  114. package/lib/module/services/telemetry/MobileAI.js.map +0 -1
  115. package/lib/module/services/telemetry/PiiScrubber.js.map +0 -1
  116. package/lib/module/services/telemetry/TelemetryService.js.map +0 -1
  117. package/lib/module/services/telemetry/TouchAutoCapture.js.map +0 -1
  118. package/lib/module/services/telemetry/device.js.map +0 -1
  119. package/lib/module/services/telemetry/deviceMetadata.js.map +0 -1
  120. package/lib/module/services/telemetry/index.js.map +0 -1
  121. package/lib/module/services/telemetry/types.js.map +0 -1
  122. package/lib/module/support/CSATSurvey.js.map +0 -1
  123. package/lib/module/support/EscalationEventSource.js.map +0 -1
  124. package/lib/module/support/EscalationSocket.js.map +0 -1
  125. package/lib/module/support/SupportChatModal.js.map +0 -1
  126. package/lib/module/support/SupportGreeting.js.map +0 -1
  127. package/lib/module/support/TicketStore.js.map +0 -1
  128. package/lib/module/support/escalateTool.js.map +0 -1
  129. package/lib/module/support/index.js.map +0 -1
  130. package/lib/module/support/supportPrompt.js.map +0 -1
  131. package/lib/module/support/types.js.map +0 -1
  132. package/lib/module/tools/datePickerTool.js.map +0 -1
  133. package/lib/module/tools/guideTool.js.map +0 -1
  134. package/lib/module/tools/index.js.map +0 -1
  135. package/lib/module/tools/keyboardTool.js.map +0 -1
  136. package/lib/module/tools/longPressTool.js.map +0 -1
  137. package/lib/module/tools/pickerTool.js.map +0 -1
  138. package/lib/module/tools/restoreTool.js.map +0 -1
  139. package/lib/module/tools/scrollTool.js.map +0 -1
  140. package/lib/module/tools/simplifyTool.js.map +0 -1
  141. package/lib/module/tools/sliderTool.js.map +0 -1
  142. package/lib/module/tools/tapTool.js.map +0 -1
  143. package/lib/module/tools/typeTool.js.map +0 -1
  144. package/lib/module/tools/types.js.map +0 -1
  145. package/lib/module/types/jsx.d.js.map +0 -1
  146. package/lib/module/utils/audioUtils.js.map +0 -1
  147. package/lib/module/utils/logger.js.map +0 -1
  148. package/lib/typescript/babel.config.d.ts.map +0 -1
  149. package/lib/typescript/bin/generate-map.d.cts.map +0 -1
  150. package/lib/typescript/eslint.config.d.mts.map +0 -1
  151. package/lib/typescript/generate-map.d.ts.map +0 -1
  152. package/lib/typescript/src/__cli_tmp__.d.ts.map +0 -1
  153. package/lib/typescript/src/components/AIAgent.d.ts.map +0 -1
  154. package/lib/typescript/src/components/AIZone.d.ts.map +0 -1
  155. package/lib/typescript/src/components/AgentChatBar.d.ts.map +0 -1
  156. package/lib/typescript/src/components/AgentErrorBoundary.d.ts.map +0 -1
  157. package/lib/typescript/src/components/AgentOverlay.d.ts.map +0 -1
  158. package/lib/typescript/src/components/DiscoveryTooltip.d.ts.map +0 -1
  159. package/lib/typescript/src/components/HighlightOverlay.d.ts.map +0 -1
  160. package/lib/typescript/src/components/Icons.d.ts.map +0 -1
  161. package/lib/typescript/src/components/ProactiveHint.d.ts.map +0 -1
  162. package/lib/typescript/src/components/cards/InfoCard.d.ts.map +0 -1
  163. package/lib/typescript/src/components/cards/ReviewSummary.d.ts.map +0 -1
  164. package/lib/typescript/src/config/endpoints.d.ts.map +0 -1
  165. package/lib/typescript/src/core/ActionRegistry.d.ts.map +0 -1
  166. package/lib/typescript/src/core/AgentRuntime.d.ts.map +0 -1
  167. package/lib/typescript/src/core/FiberTreeWalker.d.ts.map +0 -1
  168. package/lib/typescript/src/core/IdleDetector.d.ts.map +0 -1
  169. package/lib/typescript/src/core/MCPBridge.d.ts.map +0 -1
  170. package/lib/typescript/src/core/ScreenDehydrator.d.ts.map +0 -1
  171. package/lib/typescript/src/core/ZoneRegistry.d.ts.map +0 -1
  172. package/lib/typescript/src/core/systemPrompt.d.ts.map +0 -1
  173. package/lib/typescript/src/core/types.d.ts.map +0 -1
  174. package/lib/typescript/src/hooks/useAction.d.ts.map +0 -1
  175. package/lib/typescript/src/index.d.ts.map +0 -1
  176. package/lib/typescript/src/plugin/withAppIntents.d.ts.map +0 -1
  177. package/lib/typescript/src/providers/GeminiProvider.d.ts.map +0 -1
  178. package/lib/typescript/src/providers/OpenAIProvider.d.ts.map +0 -1
  179. package/lib/typescript/src/providers/ProviderFactory.d.ts.map +0 -1
  180. package/lib/typescript/src/services/AudioInputService.d.ts.map +0 -1
  181. package/lib/typescript/src/services/AudioOutputService.d.ts.map +0 -1
  182. package/lib/typescript/src/services/KnowledgeBaseService.d.ts.map +0 -1
  183. package/lib/typescript/src/services/VoiceService.d.ts.map +0 -1
  184. package/lib/typescript/src/services/flags/FlagService.d.ts.map +0 -1
  185. package/lib/typescript/src/services/telemetry/MobileAI.d.ts.map +0 -1
  186. package/lib/typescript/src/services/telemetry/PiiScrubber.d.ts.map +0 -1
  187. package/lib/typescript/src/services/telemetry/TelemetryService.d.ts.map +0 -1
  188. package/lib/typescript/src/services/telemetry/TouchAutoCapture.d.ts.map +0 -1
  189. package/lib/typescript/src/services/telemetry/device.d.ts.map +0 -1
  190. package/lib/typescript/src/services/telemetry/deviceMetadata.d.ts.map +0 -1
  191. package/lib/typescript/src/services/telemetry/index.d.ts.map +0 -1
  192. package/lib/typescript/src/services/telemetry/types.d.ts.map +0 -1
  193. package/lib/typescript/src/support/CSATSurvey.d.ts.map +0 -1
  194. package/lib/typescript/src/support/EscalationEventSource.d.ts.map +0 -1
  195. package/lib/typescript/src/support/EscalationSocket.d.ts.map +0 -1
  196. package/lib/typescript/src/support/SupportChatModal.d.ts.map +0 -1
  197. package/lib/typescript/src/support/SupportGreeting.d.ts.map +0 -1
  198. package/lib/typescript/src/support/TicketStore.d.ts.map +0 -1
  199. package/lib/typescript/src/support/escalateTool.d.ts.map +0 -1
  200. package/lib/typescript/src/support/index.d.ts.map +0 -1
  201. package/lib/typescript/src/support/supportPrompt.d.ts.map +0 -1
  202. package/lib/typescript/src/support/types.d.ts.map +0 -1
  203. package/lib/typescript/src/tools/datePickerTool.d.ts.map +0 -1
  204. package/lib/typescript/src/tools/guideTool.d.ts.map +0 -1
  205. package/lib/typescript/src/tools/index.d.ts.map +0 -1
  206. package/lib/typescript/src/tools/keyboardTool.d.ts.map +0 -1
  207. package/lib/typescript/src/tools/longPressTool.d.ts.map +0 -1
  208. package/lib/typescript/src/tools/pickerTool.d.ts.map +0 -1
  209. package/lib/typescript/src/tools/restoreTool.d.ts.map +0 -1
  210. package/lib/typescript/src/tools/scrollTool.d.ts.map +0 -1
  211. package/lib/typescript/src/tools/simplifyTool.d.ts.map +0 -1
  212. package/lib/typescript/src/tools/sliderTool.d.ts.map +0 -1
  213. package/lib/typescript/src/tools/tapTool.d.ts.map +0 -1
  214. package/lib/typescript/src/tools/typeTool.d.ts.map +0 -1
  215. package/lib/typescript/src/tools/types.d.ts.map +0 -1
  216. package/lib/typescript/src/utils/audioUtils.d.ts.map +0 -1
  217. package/lib/typescript/src/utils/logger.d.ts.map +0 -1
  218. package/src/__cli_tmp__.tsx +0 -9
  219. package/src/cli/analyzers/chain-analyzer.ts +0 -183
  220. package/src/cli/extractors/ai-extractor.ts +0 -6
  221. package/src/cli/extractors/ast-extractor.ts +0 -551
  222. package/src/cli/generate-intents.ts +0 -140
  223. package/src/cli/generate-map.ts +0 -121
  224. package/src/cli/generate-swift.ts +0 -116
  225. package/src/cli/scanners/expo-scanner.ts +0 -203
  226. package/src/cli/scanners/rn-scanner.ts +0 -445
  227. package/src/components/AIAgent.tsx +0 -1716
  228. package/src/components/AIZone.tsx +0 -147
  229. package/src/components/AgentChatBar.tsx +0 -1143
  230. package/src/components/AgentErrorBoundary.tsx +0 -78
  231. package/src/components/AgentOverlay.tsx +0 -73
  232. package/src/components/DiscoveryTooltip.tsx +0 -148
  233. package/src/components/HighlightOverlay.tsx +0 -136
  234. package/src/components/Icons.tsx +0 -253
  235. package/src/components/ProactiveHint.tsx +0 -145
  236. package/src/components/cards/InfoCard.tsx +0 -58
  237. package/src/components/cards/ReviewSummary.tsx +0 -76
  238. package/src/config/endpoints.ts +0 -22
  239. package/src/core/ActionRegistry.ts +0 -105
  240. package/src/core/AgentRuntime.ts +0 -1471
  241. package/src/core/FiberTreeWalker.ts +0 -930
  242. package/src/core/IdleDetector.ts +0 -72
  243. package/src/core/MCPBridge.ts +0 -163
  244. package/src/core/ScreenDehydrator.ts +0 -53
  245. package/src/core/ZoneRegistry.ts +0 -44
  246. package/src/core/systemPrompt.ts +0 -431
  247. package/src/core/types.ts +0 -521
  248. package/src/hooks/useAction.ts +0 -182
  249. package/src/index.ts +0 -83
  250. package/src/plugin/withAppIntents.ts +0 -98
  251. package/src/providers/GeminiProvider.ts +0 -357
  252. package/src/providers/OpenAIProvider.ts +0 -379
  253. package/src/providers/ProviderFactory.ts +0 -36
  254. package/src/services/AudioInputService.ts +0 -226
  255. package/src/services/AudioOutputService.ts +0 -236
  256. package/src/services/KnowledgeBaseService.ts +0 -156
  257. package/src/services/VoiceService.ts +0 -451
  258. package/src/services/flags/FlagService.ts +0 -137
  259. package/src/services/telemetry/MobileAI.ts +0 -66
  260. package/src/services/telemetry/PiiScrubber.ts +0 -17
  261. package/src/services/telemetry/TelemetryService.ts +0 -323
  262. package/src/services/telemetry/TouchAutoCapture.ts +0 -165
  263. package/src/services/telemetry/device.ts +0 -93
  264. package/src/services/telemetry/deviceMetadata.ts +0 -13
  265. package/src/services/telemetry/index.ts +0 -13
  266. package/src/services/telemetry/types.ts +0 -75
  267. package/src/support/CSATSurvey.tsx +0 -304
  268. package/src/support/EscalationEventSource.ts +0 -190
  269. package/src/support/EscalationSocket.ts +0 -152
  270. package/src/support/SupportChatModal.tsx +0 -563
  271. package/src/support/SupportGreeting.tsx +0 -161
  272. package/src/support/TicketStore.ts +0 -100
  273. package/src/support/escalateTool.ts +0 -174
  274. package/src/support/index.ts +0 -29
  275. package/src/support/supportPrompt.ts +0 -55
  276. package/src/support/types.ts +0 -155
  277. package/src/tools/datePickerTool.ts +0 -60
  278. package/src/tools/guideTool.ts +0 -76
  279. package/src/tools/index.ts +0 -20
  280. package/src/tools/keyboardTool.ts +0 -30
  281. package/src/tools/longPressTool.ts +0 -61
  282. package/src/tools/pickerTool.ts +0 -115
  283. package/src/tools/restoreTool.ts +0 -33
  284. package/src/tools/scrollTool.ts +0 -156
  285. package/src/tools/simplifyTool.ts +0 -33
  286. package/src/tools/sliderTool.ts +0 -65
  287. package/src/tools/tapTool.ts +0 -93
  288. package/src/tools/typeTool.ts +0 -113
  289. package/src/tools/types.ts +0 -58
  290. package/src/types/jsx.d.ts +0 -20
  291. package/src/utils/audioUtils.ts +0 -54
  292. package/src/utils/logger.ts +0 -38
@@ -1,451 +0,0 @@
1
- /**
2
- * VoiceService — @google/genai SDK Live API connection.
3
- *
4
- * Uses the official `ai.live.connect()` method instead of raw WebSocket.
5
- * This fixes function calling reliability: the SDK handles protocol details
6
- * (binary framing, message transforms, model name prefixes) that our
7
- * previous raw WebSocket implementation missed.
8
- *
9
- * Handles bidirectional audio streaming between the app and Gemini:
10
- * - Sends PCM 16kHz 16-bit audio chunks (mic input)
11
- * - Receives PCM 24kHz 16-bit audio chunks (AI responses)
12
- * - Receives function calls (tap, navigate, etc.) for agentic actions
13
- * - Sends screen context (DOM text) for live mode
14
- */
15
-
16
- // Platform-specific import: Metro can't resolve '@google/genai/web' sub-path
17
- // export, so we use the full path to the web bundle. This is what the SDK
18
- // recommends ('use a platform specific import') — RN's WebSocket API is
19
- // browser-compatible so the web bundle works correctly.
20
- // @ts-ignore — TS can't find declarations for the deep path
21
- import { GoogleGenAI, Modality } from '@google/genai/dist/web/index.mjs';
22
- // @ts-ignore
23
- import type { Session } from '@google/genai/dist/web/index.mjs';
24
- import { logger } from '../utils/logger';
25
- import type { ToolDefinition } from '../core/types';
26
-
27
- // ─── Types ─────────────────────────────────────────────────────
28
-
29
- export interface VoiceServiceConfig {
30
- apiKey?: string;
31
- proxyUrl?: string;
32
- proxyHeaders?: Record<string, string>;
33
- model?: string;
34
- systemPrompt?: string;
35
- tools?: ToolDefinition[];
36
- /** Audio sample rate for mic input (default: 16000) */
37
- inputSampleRate?: number;
38
- /** Language for Gemini speech generation (e.g., 'en', 'ar') */
39
- language?: string;
40
- }
41
-
42
- export interface VoiceServiceCallbacks {
43
- onAudioResponse?: (base64Audio: string) => void;
44
- onToolCall?: (toolCall: { name: string; args: Record<string, any>; id: string }) => void;
45
- onTranscript?: (text: string, isFinal: boolean, role: 'user' | 'model') => void;
46
- onStatusChange?: (status: VoiceStatus) => void;
47
- onError?: (error: string) => void;
48
- /** Called when AI turn is complete (all audio sent) */
49
- onTurnComplete?: () => void;
50
- /** Called when SDK setup is complete — safe to send screen context */
51
- onSetupComplete?: () => void;
52
- }
53
-
54
- export type VoiceStatus = 'disconnected' | 'connecting' | 'connected' | 'error';
55
-
56
- // ─── Constants ─────────────────────────────────────────────────
57
-
58
- const DEFAULT_MODEL = 'gemini-2.5-flash-native-audio-preview-12-2025';
59
- const DEFAULT_INPUT_SAMPLE_RATE = 16000;
60
-
61
- // ─── Service ───────────────────────────────────────────────────
62
-
63
- export class VoiceService {
64
- private session: Session | null = null;
65
- private config: VoiceServiceConfig;
66
- private callbacks: VoiceServiceCallbacks = {};
67
- public lastCallbacks: VoiceServiceCallbacks | null = null;
68
- private _status: VoiceStatus = 'disconnected';
69
- public intentionalDisconnect = false;
70
-
71
- constructor(config: VoiceServiceConfig) {
72
- this.config = config;
73
- }
74
-
75
- // ─── Connection ────────────────────────────────────────────
76
-
77
- /**
78
- * Connect to Gemini Live API via the official SDK.
79
- * Now async because `ai.live.connect()` returns a Promise.
80
- */
81
- async connect(callbacks: VoiceServiceCallbacks): Promise<void> {
82
- if (this.session) {
83
- logger.info('VoiceService', 'Already connected');
84
- return;
85
- }
86
-
87
- this.callbacks = callbacks;
88
- this.lastCallbacks = callbacks;
89
- this.setStatus('connecting');
90
- this.intentionalDisconnect = false;
91
-
92
- const model = this.config.model || DEFAULT_MODEL;
93
- logger.info('VoiceService', `Connecting via SDK (model: ${model})`);
94
-
95
- try {
96
- const genAiConfig: any = {};
97
-
98
- if (this.config.proxyUrl) {
99
- genAiConfig.apiKey = 'proxy-key';
100
- genAiConfig.httpOptions = {
101
- baseUrl: this.config.proxyUrl,
102
- headers: this.config.proxyHeaders || {},
103
- };
104
- } else if (this.config.apiKey) {
105
- genAiConfig.apiKey = this.config.apiKey;
106
- } else {
107
- throw new Error('[mobileai] Must provide apiKey or proxyUrl');
108
- }
109
-
110
- const ai = new GoogleGenAI(genAiConfig);
111
-
112
- const toolDeclarations = this.buildToolDeclarations();
113
-
114
- // Build SDK config matching the official docs pattern
115
- const sdkConfig: Record<string, any> = {
116
- responseModalities: [Modality.AUDIO],
117
- };
118
-
119
- // Enable transcription for debugging and UX
120
- sdkConfig.inputAudioTranscription = {};
121
- sdkConfig.outputAudioTranscription = {};
122
- logger.info('VoiceService', 'Transcription enabled');
123
-
124
- if (this.config.systemPrompt) {
125
- sdkConfig.systemInstruction = {
126
- parts: [{ text: this.config.systemPrompt }],
127
- };
128
- }
129
-
130
- if (toolDeclarations.length > 0) {
131
- sdkConfig.tools = [{ functionDeclarations: toolDeclarations }];
132
- }
133
-
134
- // FULL CONFIG DUMP — see exactly what we send to SDK
135
- const configDump = JSON.stringify({
136
- ...sdkConfig,
137
- systemInstruction: sdkConfig.systemInstruction ? '(present)' : '(none)',
138
- tools: sdkConfig.tools ? `${toolDeclarations.length} declarations` : '(none)',
139
- });
140
- logger.info('VoiceService', `📋 SDK config: ${configDump}`);
141
- logger.info('VoiceService', `📋 Tool names: ${toolDeclarations.map((t: any) => t.name).join(', ')}`);
142
-
143
- const session = await ai.live.connect({
144
- model: model,
145
- config: sdkConfig,
146
- callbacks: {
147
- onopen: () => {
148
- logger.info('VoiceService', '✅ SDK session connected');
149
- this.setStatus('connected');
150
- },
151
- onmessage: (message: any) => {
152
- this.handleSDKMessage(message);
153
- },
154
- onerror: (error: any) => {
155
- const errDetail = error
156
- ? JSON.stringify(error, Object.getOwnPropertyNames(error)).substring(0, 500)
157
- : 'null';
158
- logger.error('VoiceService', `SDK error: ${errDetail}`);
159
- this.setStatus('error');
160
- this.callbacks.onError?.(error?.message || 'SDK connection error');
161
- },
162
- onclose: (event: any) => {
163
- const closeDetail = event
164
- ? JSON.stringify(event, Object.getOwnPropertyNames(event)).substring(0, 500)
165
- : 'null';
166
- if (this.intentionalDisconnect) {
167
- logger.info('VoiceService', `SDK session closed (intentional)`);
168
- } else {
169
- logger.error('VoiceService', `SDK session closed UNEXPECTEDLY — code: ${event?.code}, reason: ${event?.reason}, detail: ${closeDetail}`);
170
- this.callbacks.onError?.(`Connection lost (code: ${event?.code || 'unknown'})`);
171
- }
172
- this.session = null;
173
- this.setStatus('disconnected');
174
- },
175
- },
176
- });
177
-
178
- this.session = session;
179
- logger.info('VoiceService', 'SDK session established');
180
-
181
- } catch (error: any) {
182
- logger.error('VoiceService', `Connection failed: ${error.message}`);
183
- this.setStatus('error');
184
- this.callbacks.onError?.(error.message || 'Failed to connect');
185
- }
186
- }
187
-
188
- disconnect(): void {
189
- if (this.session) {
190
- logger.info('VoiceService', 'Disconnecting (intentional)...');
191
- this.intentionalDisconnect = true;
192
- this.session.close();
193
- this.session = null;
194
- this.setStatus('disconnected');
195
- }
196
- }
197
-
198
- get isConnected(): boolean {
199
- return this.session !== null && this._status === 'connected';
200
- }
201
-
202
- get currentStatus(): VoiceStatus {
203
- return this._status;
204
- }
205
-
206
- // ─── Send Audio ────────────────────────────────────────────
207
-
208
- /** Send PCM audio chunk (base64 encoded) via SDK's sendRealtimeInput */
209
- private sendCount = 0;
210
- sendAudio(base64Audio: string): void {
211
- this.sendCount++;
212
- if (!this.isConnected || !this.session) {
213
- if (this.sendCount % 20 === 0) {
214
- logger.warn('VoiceService', `sendAudio #${this.sendCount} DROPPED — not connected`);
215
- }
216
- return;
217
- }
218
-
219
- const mimeType = `audio/pcm;rate=${this.config.inputSampleRate || DEFAULT_INPUT_SAMPLE_RATE}`;
220
-
221
- // DEBUG: log every send call
222
- if (this.sendCount <= 5 || this.sendCount % 10 === 0) {
223
- logger.info('VoiceService', `📡 sendAudio #${this.sendCount}: len=${base64Audio.length}, mime=${mimeType}, preview=${base64Audio.substring(0, 30)}...`);
224
- }
225
-
226
- try {
227
- this.session.sendRealtimeInput({
228
- audio: { data: base64Audio, mimeType },
229
- });
230
- // Log every 50th successful send to confirm data is reaching WebSocket
231
- if (this.sendCount % 50 === 0) {
232
- logger.info('VoiceService', `✅ sendAudio #${this.sendCount} OK — session.isOpen=${!!this.session}`);
233
- }
234
- } catch (error: any) {
235
- logger.error('VoiceService', `❌ sendAudio EXCEPTION: ${error.message}\n${error.stack?.substring(0, 300)}`);
236
- this.session = null;
237
- this.setStatus('disconnected');
238
- }
239
- }
240
-
241
- // ─── Send Text ─────────────────────────────────────────────
242
-
243
- /** Send text message via SDK's sendClientContent */
244
- sendText(text: string): void {
245
- if (!this.isConnected || !this.session) return;
246
-
247
- logger.info('VoiceService', `🗣️ USER (text): "${text}"`);
248
- try {
249
- this.session.sendClientContent({
250
- turns: [{ role: 'user', parts: [{ text }] }],
251
- turnComplete: true,
252
- });
253
- } catch (error: any) {
254
- logger.error('VoiceService', `sendText failed: ${error.message}`);
255
- }
256
- }
257
-
258
- /**
259
- * Send DOM tree as passive context during live conversation.
260
- * Uses turnComplete: false — the model receives context without responding.
261
- */
262
- sendScreenContext(domText: string): void {
263
- if (!this.isConnected || !this.session) return;
264
-
265
- try {
266
- this.session.sendClientContent({
267
- turns: [{ role: 'user', parts: [{ text: domText }] }],
268
- turnComplete: true,
269
- });
270
- logger.info('VoiceService', `📤 Screen context sent (${domText.length} chars)`);
271
- } catch (error: any) {
272
- logger.error('VoiceService', `sendScreenContext failed: ${error.message}`);
273
- }
274
- }
275
-
276
- // ─── Send Function Response ────────────────────────────────
277
-
278
- /** Send function call result back via SDK's sendToolResponse */
279
- sendFunctionResponse(name: string, id: string, result: any): void {
280
- if (!this.isConnected || !this.session) return;
281
-
282
- logger.info('VoiceService', `📤 Sending tool response for ${name} (id=${id})`);
283
-
284
- try {
285
- this.session.sendToolResponse({
286
- functionResponses: [{ name, id, response: result }],
287
- });
288
- } catch (error: any) {
289
- logger.error('VoiceService', `sendFunctionResponse failed: ${error.message}`);
290
- }
291
- }
292
-
293
- // ─── Internal: Tool Declarations ───────────────────────────
294
-
295
- /**
296
- * Builds function declarations from configured tools.
297
- * Converts BOOLEAN params to STRING (native audio model limitation).
298
- */
299
- private buildToolDeclarations(): any[] {
300
- if (!this.config.tools?.length) return [];
301
-
302
- const validTools = this.config.tools.filter(t => t.name !== 'capture_screenshot');
303
- if (validTools.length === 0) return [];
304
-
305
- return validTools.map(tool => {
306
- const hasParams = Object.keys(tool.parameters || {}).length > 0;
307
- const functionDecl: any = {
308
- name: tool.name,
309
- description: tool.description,
310
- };
311
-
312
- if (hasParams) {
313
- functionDecl.parameters = {
314
- type: 'OBJECT',
315
- properties: Object.fromEntries(
316
- Object.entries(tool.parameters).map(([key, param]) => {
317
- let paramType = param.type.toUpperCase();
318
- let desc = param.description;
319
- if (paramType === 'BOOLEAN') {
320
- paramType = 'STRING';
321
- desc = `${desc} (use "true" or "false")`;
322
- }
323
- return [key, { type: paramType, description: desc }];
324
- })
325
- ),
326
- required: Object.entries(tool.parameters)
327
- .filter(([, param]) => param.required)
328
- .map(([key]) => key),
329
- };
330
- }
331
- return functionDecl;
332
- });
333
- }
334
-
335
- // ─── Internal: Message Handling ────────────────────────────
336
-
337
- /**
338
- * Handle messages from the SDK's onmessage callback.
339
- * The SDK parses binary/JSON automatically — we get clean objects.
340
- *
341
- * Per official docs, tool calls come at the top level as
342
- * `response.toolCall.functionCalls`.
343
- */
344
- private handleSDKMessage(message: any): void {
345
- try {
346
- // RAW MESSAGE DUMP — full session visibility
347
- const msgKeys = Object.keys(message || {}).join(', ');
348
- logger.info('VoiceService', `📨 SDK message keys: [${msgKeys}]`);
349
-
350
- // Full raw dump for non-audio messages (audio is too large)
351
- if (!message.serverContent?.modelTurn?.parts?.some((p: any) => p.inlineData)) {
352
- const rawDump = JSON.stringify(message).substring(0, 1000);
353
- logger.info('VoiceService', `📨 RAW: ${rawDump}`);
354
- }
355
-
356
- // Tool calls — top-level (per official docs)
357
- if (message.toolCall?.functionCalls) {
358
- this.handleToolCalls(message.toolCall.functionCalls);
359
- return;
360
- }
361
-
362
- // Server content (audio, text, transcripts, turn events)
363
- if (message.serverContent) {
364
- this.handleServerContent(message.serverContent);
365
- }
366
-
367
- // Setup complete acknowledgment
368
- if (message.setupComplete !== undefined) {
369
- logger.info('VoiceService', '✅ Setup complete — ready for audio');
370
- this.callbacks.onSetupComplete?.();
371
- }
372
-
373
- // Error messages
374
- if (message.error) {
375
- logger.error('VoiceService', `Server error: ${JSON.stringify(message.error)}`);
376
- this.callbacks.onError?.(message.error.message || 'Server error');
377
- }
378
- } catch (error: any) {
379
- logger.error('VoiceService', `Error handling SDK message: ${error.message}`);
380
- }
381
- }
382
-
383
- /** Process tool calls from the model */
384
- private handleToolCalls(functionCalls: any[]): void {
385
- for (const fn of functionCalls) {
386
- logger.info('VoiceService', `🎯 Tool call: ${fn.name}(${JSON.stringify(fn.args)}) [id=${fn.id}]`);
387
- this.callbacks.onToolCall?.({
388
- name: fn.name,
389
- args: fn.args || {},
390
- id: fn.id,
391
- });
392
- }
393
- }
394
-
395
- private audioResponseCount = 0;
396
-
397
- /** Process server content (audio responses, transcripts, turn events) */
398
- private handleServerContent(content: any): void {
399
- // Log all keys for full visibility
400
- const contentKeys = Object.keys(content || {}).join(', ');
401
- logger.debug('VoiceService', `📦 serverContent keys: [${contentKeys}]`);
402
-
403
- // Turn complete
404
- if (content.turnComplete) {
405
- logger.info('VoiceService', `🏁 Turn complete (audioChunks sent: ${this.audioResponseCount})`);
406
- this.audioResponseCount = 0;
407
- this.callbacks.onTurnComplete?.();
408
- }
409
-
410
- // Model output parts (audio + optional thinking text)
411
- if (content.modelTurn?.parts) {
412
- for (const part of content.modelTurn.parts) {
413
- if (part.inlineData?.data) {
414
- this.audioResponseCount++;
415
- if (this.audioResponseCount <= 3 || this.audioResponseCount % 20 === 0) {
416
- logger.info('VoiceService', `🔊 Audio chunk #${this.audioResponseCount}: ${part.inlineData.data.length} b64 chars, mime=${part.inlineData.mimeType || 'unknown'}`);
417
- }
418
- this.callbacks.onAudioResponse?.(part.inlineData.data);
419
- }
420
- if (part.text) {
421
- logger.info('VoiceService', `🤖 MODEL: "${part.text}"`);
422
- this.callbacks.onTranscript?.(part.text, true, 'model');
423
- }
424
- }
425
- }
426
-
427
- // Input transcription (user's speech-to-text)
428
- if (content.inputTranscription?.text) {
429
- logger.info('VoiceService', `🗣️ USER (voice): "${content.inputTranscription.text}"`);
430
- this.callbacks.onTranscript?.(content.inputTranscription.text, true, 'user');
431
- }
432
-
433
- // Output transcription (model's speech-to-text)
434
- if (content.outputTranscription?.text) {
435
- logger.info('VoiceService', `🤖 MODEL (voice): "${content.outputTranscription.text}"`);
436
- this.callbacks.onTranscript?.(content.outputTranscription.text, true, 'model');
437
- }
438
-
439
- // Tool calls inside serverContent (some SDK versions deliver here)
440
- if (content.toolCall?.functionCalls) {
441
- this.handleToolCalls(content.toolCall.functionCalls);
442
- }
443
- }
444
-
445
- // ─── Helpers ───────────────────────────────────────────────
446
-
447
- private setStatus(newStatus: VoiceStatus): void {
448
- this._status = newStatus;
449
- this.callbacks.onStatusChange?.(newStatus);
450
- }
451
- }
@@ -1,137 +0,0 @@
1
- import { logger } from '../../utils/logger';
2
- import { getDeviceId } from '../telemetry/device';
3
-
4
- const LOG_TAG = 'FlagService';
5
-
6
- /**
7
- * MurmurHash3 (32-bit) implementation
8
- */
9
- function murmurhash3_32_gc(key: string, seed: number = 0): number {
10
- let remainder, bytes, h1, h1b, c1, c2, k1, i;
11
-
12
- remainder = key.length & 3; // key.length % 4
13
- bytes = key.length - remainder;
14
- h1 = seed;
15
- c1 = 0xcc9e2d51;
16
- c2 = 0x1b873593;
17
- i = 0;
18
-
19
- while (i < bytes) {
20
- k1 =
21
- ((key.charCodeAt(i) & 0xff)) |
22
- ((key.charCodeAt(++i) & 0xff) << 8) |
23
- ((key.charCodeAt(++i) & 0xff) << 16) |
24
- ((key.charCodeAt(++i) & 0xff) << 24);
25
- ++i;
26
-
27
- k1 = ((((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16))) & 0xffffffff;
28
- k1 = (k1 << 15) | (k1 >>> 17);
29
- k1 = ((((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16))) & 0xffffffff;
30
-
31
- h1 ^= k1;
32
- h1 = (h1 << 13) | (h1 >>> 19);
33
- h1b = ((((h1 & 0xffff) * 5) + ((((h1 >>> 16) * 5) & 0xffff) << 16))) & 0xffffffff;
34
- h1 = (((h1b & 0xffff) + 0x6b64) + ((((h1b >>> 16) + 0xe654) & 0xffff) << 16));
35
- }
36
-
37
- if (remainder >= 1) {
38
- k1 = 0;
39
- if (remainder >= 3) k1 ^= (key.charCodeAt(i + 2) & 0xff) << 16;
40
- if (remainder >= 2) k1 ^= (key.charCodeAt(i + 1) & 0xff) << 8;
41
- k1 ^= (key.charCodeAt(i) & 0xff);
42
- k1 = (((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16)) & 0xffffffff;
43
- k1 = (k1 << 15) | (k1 >>> 17);
44
- k1 = (((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16)) & 0xffffffff;
45
- h1 ^= k1;
46
- }
47
-
48
- h1 ^= key.length;
49
-
50
- h1 ^= h1 >>> 16;
51
- h1 = (((h1 & 0xffff) * 0x85ebca6b) + ((((h1 >>> 16) * 0x85ebca6b) & 0xffff) << 16)) & 0xffffffff;
52
- h1 ^= h1 >>> 13;
53
- h1 = ((((h1 & 0xffff) * 0xc2b2ae35) + ((((h1 >>> 16) * 0xc2b2ae35) & 0xffff) << 16))) & 0xffffffff;
54
- h1 ^= h1 >>> 16;
55
-
56
- return h1 >>> 0;
57
- }
58
-
59
- export interface FeatureFlagPayload {
60
- key: string;
61
- variants: string[];
62
- rollout: number[];
63
- }
64
-
65
- export class FlagService {
66
- private assignments: Record<string, string> = {};
67
- private fetched: boolean = false;
68
-
69
- constructor(private hostUrl: string) {}
70
-
71
- /**
72
- * Fetch feature flags from the dashboard backend
73
- */
74
- async fetch(analyticsKey: string, userId?: string): Promise<void> {
75
- try {
76
- // Avoid fetching if already loaded, unless explicitly forced?
77
- // For now, allow refetching just in case.
78
- const res = await fetch(`${this.hostUrl}/api/v1/flags/sync?key=${analyticsKey}`);
79
- if (!res.ok) {
80
- throw new Error(`Failed to fetch flags: ${res.status}`);
81
- }
82
-
83
- const data = await res.json();
84
- const flags: FeatureFlagPayload[] = data.flags || [];
85
-
86
- this.assignAll(flags, userId);
87
- this.fetched = true;
88
- logger.info(LOG_TAG, `Fetched ${flags.length} flags`);
89
- } catch (err: any) {
90
- logger.warn(LOG_TAG, `Could not sync feature flags: ${err.message}`);
91
- }
92
- }
93
-
94
- /**
95
- * Deterministically assign a variant using murmurhash.
96
- */
97
- private assignVariant(userIdentifier: string, flagKey: string, variants: string[], rollout: number[]): string {
98
- const hash = murmurhash3_32_gc(`${userIdentifier}_${flagKey}`) % 100;
99
- let cumulative = 0;
100
-
101
- for (let i = 0; i < rollout.length; i++) {
102
- cumulative += rollout[i]!;
103
- if (hash < cumulative) {
104
- return variants[i]!;
105
- }
106
- }
107
-
108
- // Fallback if rollout doesn't equal exactly 100 or edge case
109
- return variants[0]!
110
- }
111
-
112
- private assignAll(flags: FeatureFlagPayload[], userId?: string) {
113
- const identifier = userId || getDeviceId() || 'unknown';
114
-
115
- const newAssignments: Record<string, string> = {};
116
- for (const flag of flags) {
117
- if (flag.variants.length > 0 && flag.rollout.length === flag.variants.length) {
118
- newAssignments[flag.key] = this.assignVariant(identifier, flag.key, flag.variants, flag.rollout);
119
- }
120
- }
121
-
122
- this.assignments = newAssignments;
123
- }
124
-
125
- /** Get a specific flag value */
126
- getFlag(key: string, defaultValue?: string): string {
127
- if (!this.fetched) {
128
- logger.debug(LOG_TAG, `getFlag("${key}") called before flags were fetched. Returning default.`);
129
- }
130
- return this.assignments[key] ?? defaultValue ?? '';
131
- }
132
-
133
- /** Get all active assignments for telemetry */
134
- getAllFlags(): Record<string, string> {
135
- return { ...this.assignments };
136
- }
137
- }
@@ -1,66 +0,0 @@
1
- /**
2
- * MobileAI — Public static API for consumer event tracking.
3
- *
4
- * Usage:
5
- * import { MobileAI } from '@mobileai/react-native';
6
- * MobileAI.track('purchase_complete', { total: 29.99 });
7
- *
8
- * The TelemetryService instance is injected by the <AIAgent> component.
9
- * If no analyticsKey is configured, all calls are no-ops.
10
- */
11
-
12
- import type { TelemetryService } from './TelemetryService';
13
- import { logger } from '../../utils/logger';
14
-
15
- const LOG_TAG = 'MobileAI';
16
-
17
- let service: TelemetryService | null = null;
18
-
19
- export const MobileAI = {
20
- /**
21
- * Track a custom business event.
22
- * @param eventName - Name of the event (e.g., 'purchase_complete')
23
- * @param data - Event-specific key-value data
24
- */
25
- track(eventName: string, data: Record<string, unknown> = {}): void {
26
- if (!service) {
27
- logger.debug(LOG_TAG, `track('${eventName}') ignored — no analyticsKey configured`);
28
- return;
29
- }
30
- service.track(eventName, data);
31
- },
32
-
33
- /**
34
- * Identify the current user (optional, for user-level analytics).
35
- * @param userId - Unique user identifier (hashed by consumer)
36
- * @param traits - Optional user traits (plan, role, etc.)
37
- */
38
- identify(userId: string, traits: Record<string, unknown> = {}): void {
39
- if (!service) {
40
- logger.debug(LOG_TAG, 'identify() ignored — no analyticsKey configured');
41
- return;
42
- }
43
- service.track('identify', { user_id: userId, ...traits });
44
- },
45
-
46
- /**
47
- * Get an assigned feature flag variation for the current device.
48
- * Deterministic via murmurhash. Call after MobileAI has initialized.
49
- * @param key Flag key
50
- * @param defaultValue Fallback if not assigned
51
- */
52
- getFlag(key: string, defaultValue?: string): string {
53
- if (!service) {
54
- return defaultValue ?? '';
55
- }
56
- return service.flags.getFlag(key, defaultValue);
57
- },
58
- };
59
-
60
- /**
61
- * Internal: Bind the TelemetryService instance (called by AIAgent on mount).
62
- * Not exported to consumers.
63
- */
64
- export function bindTelemetryService(instance: TelemetryService | null): void {
65
- service = instance;
66
- }
@@ -1,17 +0,0 @@
1
- const EMAIL_RE = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
2
- const PHONE_RE = /(\+?\d[\d\s\-().]{7,}\d)/g;
3
- const CARD_RE = /\b\d{4}[\s\-]?\d{4}[\s\-]?\d{4}[\s\-]?\d{4}\b/g;
4
-
5
- /**
6
- * Scrubs common PII patterns from strings.
7
- * Used by TelemetryService to sanitize auto-captured touch labels.
8
- */
9
- export function scrubPII(value: string): string {
10
- if (typeof value !== 'string') return value;
11
-
12
- return value
13
- .replace(EMAIL_RE, '[email]')
14
- .replace(CARD_RE, '[card]')
15
- // Phone numbers are tricky, we rely on the 7+ digit regex
16
- .replace(PHONE_RE, '[phone]');
17
- }