@mobileai/react-native 0.9.16 → 0.9.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/README.md +2 -2
  2. package/package.json +5 -8
  3. package/lib/module/__cli_tmp__.js.map +0 -1
  4. package/lib/module/components/AIAgent.js.map +0 -1
  5. package/lib/module/components/AIZone.js.map +0 -1
  6. package/lib/module/components/AgentChatBar.js.map +0 -1
  7. package/lib/module/components/AgentErrorBoundary.js.map +0 -1
  8. package/lib/module/components/AgentOverlay.js.map +0 -1
  9. package/lib/module/components/DiscoveryTooltip.js.map +0 -1
  10. package/lib/module/components/HighlightOverlay.js.map +0 -1
  11. package/lib/module/components/Icons.js.map +0 -1
  12. package/lib/module/components/ProactiveHint.js.map +0 -1
  13. package/lib/module/components/cards/InfoCard.js.map +0 -1
  14. package/lib/module/components/cards/ReviewSummary.js.map +0 -1
  15. package/lib/module/config/endpoints.js.map +0 -1
  16. package/lib/module/core/ActionRegistry.js.map +0 -1
  17. package/lib/module/core/AgentRuntime.js.map +0 -1
  18. package/lib/module/core/FiberTreeWalker.js.map +0 -1
  19. package/lib/module/core/IdleDetector.js.map +0 -1
  20. package/lib/module/core/MCPBridge.js.map +0 -1
  21. package/lib/module/core/ScreenDehydrator.js.map +0 -1
  22. package/lib/module/core/ZoneRegistry.js.map +0 -1
  23. package/lib/module/core/systemPrompt.js.map +0 -1
  24. package/lib/module/core/types.js.map +0 -1
  25. package/lib/module/hooks/useAction.js.map +0 -1
  26. package/lib/module/index.js.map +0 -1
  27. package/lib/module/plugin/withAppIntents.js.map +0 -1
  28. package/lib/module/providers/GeminiProvider.js.map +0 -1
  29. package/lib/module/providers/OpenAIProvider.js.map +0 -1
  30. package/lib/module/providers/ProviderFactory.js.map +0 -1
  31. package/lib/module/services/AudioInputService.js.map +0 -1
  32. package/lib/module/services/AudioOutputService.js.map +0 -1
  33. package/lib/module/services/KnowledgeBaseService.js.map +0 -1
  34. package/lib/module/services/VoiceService.js.map +0 -1
  35. package/lib/module/services/flags/FlagService.js.map +0 -1
  36. package/lib/module/services/telemetry/MobileAI.js.map +0 -1
  37. package/lib/module/services/telemetry/PiiScrubber.js.map +0 -1
  38. package/lib/module/services/telemetry/TelemetryService.js.map +0 -1
  39. package/lib/module/services/telemetry/TouchAutoCapture.js.map +0 -1
  40. package/lib/module/services/telemetry/device.js.map +0 -1
  41. package/lib/module/services/telemetry/deviceMetadata.js.map +0 -1
  42. package/lib/module/services/telemetry/index.js.map +0 -1
  43. package/lib/module/services/telemetry/types.js.map +0 -1
  44. package/lib/module/support/CSATSurvey.js.map +0 -1
  45. package/lib/module/support/EscalationEventSource.js.map +0 -1
  46. package/lib/module/support/EscalationSocket.js.map +0 -1
  47. package/lib/module/support/SupportChatModal.js.map +0 -1
  48. package/lib/module/support/SupportGreeting.js.map +0 -1
  49. package/lib/module/support/TicketStore.js.map +0 -1
  50. package/lib/module/support/escalateTool.js.map +0 -1
  51. package/lib/module/support/index.js.map +0 -1
  52. package/lib/module/support/supportPrompt.js.map +0 -1
  53. package/lib/module/support/types.js.map +0 -1
  54. package/lib/module/tools/datePickerTool.js.map +0 -1
  55. package/lib/module/tools/guideTool.js.map +0 -1
  56. package/lib/module/tools/index.js.map +0 -1
  57. package/lib/module/tools/keyboardTool.js.map +0 -1
  58. package/lib/module/tools/longPressTool.js.map +0 -1
  59. package/lib/module/tools/pickerTool.js.map +0 -1
  60. package/lib/module/tools/restoreTool.js.map +0 -1
  61. package/lib/module/tools/scrollTool.js.map +0 -1
  62. package/lib/module/tools/simplifyTool.js.map +0 -1
  63. package/lib/module/tools/sliderTool.js.map +0 -1
  64. package/lib/module/tools/tapTool.js.map +0 -1
  65. package/lib/module/tools/typeTool.js.map +0 -1
  66. package/lib/module/tools/types.js.map +0 -1
  67. package/lib/module/types/jsx.d.js.map +0 -1
  68. package/lib/module/utils/audioUtils.js.map +0 -1
  69. package/lib/module/utils/logger.js.map +0 -1
  70. package/lib/typescript/babel.config.d.ts.map +0 -1
  71. package/lib/typescript/bin/generate-map.d.cts.map +0 -1
  72. package/lib/typescript/eslint.config.d.mts.map +0 -1
  73. package/lib/typescript/generate-map.d.ts.map +0 -1
  74. package/lib/typescript/src/__cli_tmp__.d.ts.map +0 -1
  75. package/lib/typescript/src/components/AIAgent.d.ts.map +0 -1
  76. package/lib/typescript/src/components/AIZone.d.ts.map +0 -1
  77. package/lib/typescript/src/components/AgentChatBar.d.ts.map +0 -1
  78. package/lib/typescript/src/components/AgentErrorBoundary.d.ts.map +0 -1
  79. package/lib/typescript/src/components/AgentOverlay.d.ts.map +0 -1
  80. package/lib/typescript/src/components/DiscoveryTooltip.d.ts.map +0 -1
  81. package/lib/typescript/src/components/HighlightOverlay.d.ts.map +0 -1
  82. package/lib/typescript/src/components/Icons.d.ts.map +0 -1
  83. package/lib/typescript/src/components/ProactiveHint.d.ts.map +0 -1
  84. package/lib/typescript/src/components/cards/InfoCard.d.ts.map +0 -1
  85. package/lib/typescript/src/components/cards/ReviewSummary.d.ts.map +0 -1
  86. package/lib/typescript/src/config/endpoints.d.ts.map +0 -1
  87. package/lib/typescript/src/core/ActionRegistry.d.ts.map +0 -1
  88. package/lib/typescript/src/core/AgentRuntime.d.ts.map +0 -1
  89. package/lib/typescript/src/core/FiberTreeWalker.d.ts.map +0 -1
  90. package/lib/typescript/src/core/IdleDetector.d.ts.map +0 -1
  91. package/lib/typescript/src/core/MCPBridge.d.ts.map +0 -1
  92. package/lib/typescript/src/core/ScreenDehydrator.d.ts.map +0 -1
  93. package/lib/typescript/src/core/ZoneRegistry.d.ts.map +0 -1
  94. package/lib/typescript/src/core/systemPrompt.d.ts.map +0 -1
  95. package/lib/typescript/src/core/types.d.ts.map +0 -1
  96. package/lib/typescript/src/hooks/useAction.d.ts.map +0 -1
  97. package/lib/typescript/src/index.d.ts.map +0 -1
  98. package/lib/typescript/src/plugin/withAppIntents.d.ts.map +0 -1
  99. package/lib/typescript/src/providers/GeminiProvider.d.ts.map +0 -1
  100. package/lib/typescript/src/providers/OpenAIProvider.d.ts.map +0 -1
  101. package/lib/typescript/src/providers/ProviderFactory.d.ts.map +0 -1
  102. package/lib/typescript/src/services/AudioInputService.d.ts.map +0 -1
  103. package/lib/typescript/src/services/AudioOutputService.d.ts.map +0 -1
  104. package/lib/typescript/src/services/KnowledgeBaseService.d.ts.map +0 -1
  105. package/lib/typescript/src/services/VoiceService.d.ts.map +0 -1
  106. package/lib/typescript/src/services/flags/FlagService.d.ts.map +0 -1
  107. package/lib/typescript/src/services/telemetry/MobileAI.d.ts.map +0 -1
  108. package/lib/typescript/src/services/telemetry/PiiScrubber.d.ts.map +0 -1
  109. package/lib/typescript/src/services/telemetry/TelemetryService.d.ts.map +0 -1
  110. package/lib/typescript/src/services/telemetry/TouchAutoCapture.d.ts.map +0 -1
  111. package/lib/typescript/src/services/telemetry/device.d.ts.map +0 -1
  112. package/lib/typescript/src/services/telemetry/deviceMetadata.d.ts.map +0 -1
  113. package/lib/typescript/src/services/telemetry/index.d.ts.map +0 -1
  114. package/lib/typescript/src/services/telemetry/types.d.ts.map +0 -1
  115. package/lib/typescript/src/support/CSATSurvey.d.ts.map +0 -1
  116. package/lib/typescript/src/support/EscalationEventSource.d.ts.map +0 -1
  117. package/lib/typescript/src/support/EscalationSocket.d.ts.map +0 -1
  118. package/lib/typescript/src/support/SupportChatModal.d.ts.map +0 -1
  119. package/lib/typescript/src/support/SupportGreeting.d.ts.map +0 -1
  120. package/lib/typescript/src/support/TicketStore.d.ts.map +0 -1
  121. package/lib/typescript/src/support/escalateTool.d.ts.map +0 -1
  122. package/lib/typescript/src/support/index.d.ts.map +0 -1
  123. package/lib/typescript/src/support/supportPrompt.d.ts.map +0 -1
  124. package/lib/typescript/src/support/types.d.ts.map +0 -1
  125. package/lib/typescript/src/tools/datePickerTool.d.ts.map +0 -1
  126. package/lib/typescript/src/tools/guideTool.d.ts.map +0 -1
  127. package/lib/typescript/src/tools/index.d.ts.map +0 -1
  128. package/lib/typescript/src/tools/keyboardTool.d.ts.map +0 -1
  129. package/lib/typescript/src/tools/longPressTool.d.ts.map +0 -1
  130. package/lib/typescript/src/tools/pickerTool.d.ts.map +0 -1
  131. package/lib/typescript/src/tools/restoreTool.d.ts.map +0 -1
  132. package/lib/typescript/src/tools/scrollTool.d.ts.map +0 -1
  133. package/lib/typescript/src/tools/simplifyTool.d.ts.map +0 -1
  134. package/lib/typescript/src/tools/sliderTool.d.ts.map +0 -1
  135. package/lib/typescript/src/tools/tapTool.d.ts.map +0 -1
  136. package/lib/typescript/src/tools/typeTool.d.ts.map +0 -1
  137. package/lib/typescript/src/tools/types.d.ts.map +0 -1
  138. package/lib/typescript/src/utils/audioUtils.d.ts.map +0 -1
  139. package/lib/typescript/src/utils/logger.d.ts.map +0 -1
  140. package/src/__cli_tmp__.tsx +0 -9
  141. package/src/cli/analyzers/chain-analyzer.ts +0 -183
  142. package/src/cli/extractors/ai-extractor.ts +0 -6
  143. package/src/cli/extractors/ast-extractor.ts +0 -551
  144. package/src/cli/generate-intents.ts +0 -140
  145. package/src/cli/generate-map.ts +0 -121
  146. package/src/cli/generate-swift.ts +0 -116
  147. package/src/cli/scanners/expo-scanner.ts +0 -203
  148. package/src/cli/scanners/rn-scanner.ts +0 -445
  149. package/src/components/AIAgent.tsx +0 -1716
  150. package/src/components/AIZone.tsx +0 -147
  151. package/src/components/AgentChatBar.tsx +0 -1143
  152. package/src/components/AgentErrorBoundary.tsx +0 -78
  153. package/src/components/AgentOverlay.tsx +0 -73
  154. package/src/components/DiscoveryTooltip.tsx +0 -148
  155. package/src/components/HighlightOverlay.tsx +0 -136
  156. package/src/components/Icons.tsx +0 -253
  157. package/src/components/ProactiveHint.tsx +0 -145
  158. package/src/components/cards/InfoCard.tsx +0 -58
  159. package/src/components/cards/ReviewSummary.tsx +0 -76
  160. package/src/config/endpoints.ts +0 -22
  161. package/src/core/ActionRegistry.ts +0 -105
  162. package/src/core/AgentRuntime.ts +0 -1471
  163. package/src/core/FiberTreeWalker.ts +0 -930
  164. package/src/core/IdleDetector.ts +0 -72
  165. package/src/core/MCPBridge.ts +0 -163
  166. package/src/core/ScreenDehydrator.ts +0 -53
  167. package/src/core/ZoneRegistry.ts +0 -44
  168. package/src/core/systemPrompt.ts +0 -431
  169. package/src/core/types.ts +0 -521
  170. package/src/hooks/useAction.ts +0 -182
  171. package/src/index.ts +0 -83
  172. package/src/plugin/withAppIntents.ts +0 -98
  173. package/src/providers/GeminiProvider.ts +0 -357
  174. package/src/providers/OpenAIProvider.ts +0 -379
  175. package/src/providers/ProviderFactory.ts +0 -36
  176. package/src/services/AudioInputService.ts +0 -226
  177. package/src/services/AudioOutputService.ts +0 -236
  178. package/src/services/KnowledgeBaseService.ts +0 -156
  179. package/src/services/VoiceService.ts +0 -451
  180. package/src/services/flags/FlagService.ts +0 -137
  181. package/src/services/telemetry/MobileAI.ts +0 -66
  182. package/src/services/telemetry/PiiScrubber.ts +0 -17
  183. package/src/services/telemetry/TelemetryService.ts +0 -323
  184. package/src/services/telemetry/TouchAutoCapture.ts +0 -165
  185. package/src/services/telemetry/device.ts +0 -93
  186. package/src/services/telemetry/deviceMetadata.ts +0 -13
  187. package/src/services/telemetry/index.ts +0 -13
  188. package/src/services/telemetry/types.ts +0 -75
  189. package/src/support/CSATSurvey.tsx +0 -304
  190. package/src/support/EscalationEventSource.ts +0 -190
  191. package/src/support/EscalationSocket.ts +0 -152
  192. package/src/support/SupportChatModal.tsx +0 -563
  193. package/src/support/SupportGreeting.tsx +0 -161
  194. package/src/support/TicketStore.ts +0 -100
  195. package/src/support/escalateTool.ts +0 -174
  196. package/src/support/index.ts +0 -29
  197. package/src/support/supportPrompt.ts +0 -55
  198. package/src/support/types.ts +0 -155
  199. package/src/tools/datePickerTool.ts +0 -60
  200. package/src/tools/guideTool.ts +0 -76
  201. package/src/tools/index.ts +0 -20
  202. package/src/tools/keyboardTool.ts +0 -30
  203. package/src/tools/longPressTool.ts +0 -61
  204. package/src/tools/pickerTool.ts +0 -115
  205. package/src/tools/restoreTool.ts +0 -33
  206. package/src/tools/scrollTool.ts +0 -156
  207. package/src/tools/simplifyTool.ts +0 -33
  208. package/src/tools/sliderTool.ts +0 -65
  209. package/src/tools/tapTool.ts +0 -93
  210. package/src/tools/typeTool.ts +0 -113
  211. package/src/tools/types.ts +0 -58
  212. package/src/types/jsx.d.ts +0 -20
  213. package/src/utils/audioUtils.ts +0 -54
  214. package/src/utils/logger.ts +0 -38
@@ -1,451 +0,0 @@
1
- /**
2
- * VoiceService — @google/genai SDK Live API connection.
3
- *
4
- * Uses the official `ai.live.connect()` method instead of raw WebSocket.
5
- * This fixes function calling reliability: the SDK handles protocol details
6
- * (binary framing, message transforms, model name prefixes) that our
7
- * previous raw WebSocket implementation missed.
8
- *
9
- * Handles bidirectional audio streaming between the app and Gemini:
10
- * - Sends PCM 16kHz 16-bit audio chunks (mic input)
11
- * - Receives PCM 24kHz 16-bit audio chunks (AI responses)
12
- * - Receives function calls (tap, navigate, etc.) for agentic actions
13
- * - Sends screen context (DOM text) for live mode
14
- */
15
-
16
- // Platform-specific import: Metro can't resolve '@google/genai/web' sub-path
17
- // export, so we use the full path to the web bundle. This is what the SDK
18
- // recommends ('use a platform specific import') — RN's WebSocket API is
19
- // browser-compatible so the web bundle works correctly.
20
- // @ts-ignore — TS can't find declarations for the deep path
21
- import { GoogleGenAI, Modality } from '@google/genai/dist/web/index.mjs';
22
- // @ts-ignore
23
- import type { Session } from '@google/genai/dist/web/index.mjs';
24
- import { logger } from '../utils/logger';
25
- import type { ToolDefinition } from '../core/types';
26
-
27
- // ─── Types ─────────────────────────────────────────────────────
28
-
29
- export interface VoiceServiceConfig {
30
- apiKey?: string;
31
- proxyUrl?: string;
32
- proxyHeaders?: Record<string, string>;
33
- model?: string;
34
- systemPrompt?: string;
35
- tools?: ToolDefinition[];
36
- /** Audio sample rate for mic input (default: 16000) */
37
- inputSampleRate?: number;
38
- /** Language for Gemini speech generation (e.g., 'en', 'ar') */
39
- language?: string;
40
- }
41
-
42
- export interface VoiceServiceCallbacks {
43
- onAudioResponse?: (base64Audio: string) => void;
44
- onToolCall?: (toolCall: { name: string; args: Record<string, any>; id: string }) => void;
45
- onTranscript?: (text: string, isFinal: boolean, role: 'user' | 'model') => void;
46
- onStatusChange?: (status: VoiceStatus) => void;
47
- onError?: (error: string) => void;
48
- /** Called when AI turn is complete (all audio sent) */
49
- onTurnComplete?: () => void;
50
- /** Called when SDK setup is complete — safe to send screen context */
51
- onSetupComplete?: () => void;
52
- }
53
-
54
- export type VoiceStatus = 'disconnected' | 'connecting' | 'connected' | 'error';
55
-
56
- // ─── Constants ─────────────────────────────────────────────────
57
-
58
- const DEFAULT_MODEL = 'gemini-2.5-flash-native-audio-preview-12-2025';
59
- const DEFAULT_INPUT_SAMPLE_RATE = 16000;
60
-
61
- // ─── Service ───────────────────────────────────────────────────
62
-
63
- export class VoiceService {
64
- private session: Session | null = null;
65
- private config: VoiceServiceConfig;
66
- private callbacks: VoiceServiceCallbacks = {};
67
- public lastCallbacks: VoiceServiceCallbacks | null = null;
68
- private _status: VoiceStatus = 'disconnected';
69
- public intentionalDisconnect = false;
70
-
71
- constructor(config: VoiceServiceConfig) {
72
- this.config = config;
73
- }
74
-
75
- // ─── Connection ────────────────────────────────────────────
76
-
77
- /**
78
- * Connect to Gemini Live API via the official SDK.
79
- * Now async because `ai.live.connect()` returns a Promise.
80
- */
81
- async connect(callbacks: VoiceServiceCallbacks): Promise<void> {
82
- if (this.session) {
83
- logger.info('VoiceService', 'Already connected');
84
- return;
85
- }
86
-
87
- this.callbacks = callbacks;
88
- this.lastCallbacks = callbacks;
89
- this.setStatus('connecting');
90
- this.intentionalDisconnect = false;
91
-
92
- const model = this.config.model || DEFAULT_MODEL;
93
- logger.info('VoiceService', `Connecting via SDK (model: ${model})`);
94
-
95
- try {
96
- const genAiConfig: any = {};
97
-
98
- if (this.config.proxyUrl) {
99
- genAiConfig.apiKey = 'proxy-key';
100
- genAiConfig.httpOptions = {
101
- baseUrl: this.config.proxyUrl,
102
- headers: this.config.proxyHeaders || {},
103
- };
104
- } else if (this.config.apiKey) {
105
- genAiConfig.apiKey = this.config.apiKey;
106
- } else {
107
- throw new Error('[mobileai] Must provide apiKey or proxyUrl');
108
- }
109
-
110
- const ai = new GoogleGenAI(genAiConfig);
111
-
112
- const toolDeclarations = this.buildToolDeclarations();
113
-
114
- // Build SDK config matching the official docs pattern
115
- const sdkConfig: Record<string, any> = {
116
- responseModalities: [Modality.AUDIO],
117
- };
118
-
119
- // Enable transcription for debugging and UX
120
- sdkConfig.inputAudioTranscription = {};
121
- sdkConfig.outputAudioTranscription = {};
122
- logger.info('VoiceService', 'Transcription enabled');
123
-
124
- if (this.config.systemPrompt) {
125
- sdkConfig.systemInstruction = {
126
- parts: [{ text: this.config.systemPrompt }],
127
- };
128
- }
129
-
130
- if (toolDeclarations.length > 0) {
131
- sdkConfig.tools = [{ functionDeclarations: toolDeclarations }];
132
- }
133
-
134
- // FULL CONFIG DUMP — see exactly what we send to SDK
135
- const configDump = JSON.stringify({
136
- ...sdkConfig,
137
- systemInstruction: sdkConfig.systemInstruction ? '(present)' : '(none)',
138
- tools: sdkConfig.tools ? `${toolDeclarations.length} declarations` : '(none)',
139
- });
140
- logger.info('VoiceService', `📋 SDK config: ${configDump}`);
141
- logger.info('VoiceService', `📋 Tool names: ${toolDeclarations.map((t: any) => t.name).join(', ')}`);
142
-
143
- const session = await ai.live.connect({
144
- model: model,
145
- config: sdkConfig,
146
- callbacks: {
147
- onopen: () => {
148
- logger.info('VoiceService', '✅ SDK session connected');
149
- this.setStatus('connected');
150
- },
151
- onmessage: (message: any) => {
152
- this.handleSDKMessage(message);
153
- },
154
- onerror: (error: any) => {
155
- const errDetail = error
156
- ? JSON.stringify(error, Object.getOwnPropertyNames(error)).substring(0, 500)
157
- : 'null';
158
- logger.error('VoiceService', `SDK error: ${errDetail}`);
159
- this.setStatus('error');
160
- this.callbacks.onError?.(error?.message || 'SDK connection error');
161
- },
162
- onclose: (event: any) => {
163
- const closeDetail = event
164
- ? JSON.stringify(event, Object.getOwnPropertyNames(event)).substring(0, 500)
165
- : 'null';
166
- if (this.intentionalDisconnect) {
167
- logger.info('VoiceService', `SDK session closed (intentional)`);
168
- } else {
169
- logger.error('VoiceService', `SDK session closed UNEXPECTEDLY — code: ${event?.code}, reason: ${event?.reason}, detail: ${closeDetail}`);
170
- this.callbacks.onError?.(`Connection lost (code: ${event?.code || 'unknown'})`);
171
- }
172
- this.session = null;
173
- this.setStatus('disconnected');
174
- },
175
- },
176
- });
177
-
178
- this.session = session;
179
- logger.info('VoiceService', 'SDK session established');
180
-
181
- } catch (error: any) {
182
- logger.error('VoiceService', `Connection failed: ${error.message}`);
183
- this.setStatus('error');
184
- this.callbacks.onError?.(error.message || 'Failed to connect');
185
- }
186
- }
187
-
188
- disconnect(): void {
189
- if (this.session) {
190
- logger.info('VoiceService', 'Disconnecting (intentional)...');
191
- this.intentionalDisconnect = true;
192
- this.session.close();
193
- this.session = null;
194
- this.setStatus('disconnected');
195
- }
196
- }
197
-
198
- get isConnected(): boolean {
199
- return this.session !== null && this._status === 'connected';
200
- }
201
-
202
- get currentStatus(): VoiceStatus {
203
- return this._status;
204
- }
205
-
206
- // ─── Send Audio ────────────────────────────────────────────
207
-
208
- /** Send PCM audio chunk (base64 encoded) via SDK's sendRealtimeInput */
209
- private sendCount = 0;
210
- sendAudio(base64Audio: string): void {
211
- this.sendCount++;
212
- if (!this.isConnected || !this.session) {
213
- if (this.sendCount % 20 === 0) {
214
- logger.warn('VoiceService', `sendAudio #${this.sendCount} DROPPED — not connected`);
215
- }
216
- return;
217
- }
218
-
219
- const mimeType = `audio/pcm;rate=${this.config.inputSampleRate || DEFAULT_INPUT_SAMPLE_RATE}`;
220
-
221
- // DEBUG: log every send call
222
- if (this.sendCount <= 5 || this.sendCount % 10 === 0) {
223
- logger.info('VoiceService', `📡 sendAudio #${this.sendCount}: len=${base64Audio.length}, mime=${mimeType}, preview=${base64Audio.substring(0, 30)}...`);
224
- }
225
-
226
- try {
227
- this.session.sendRealtimeInput({
228
- audio: { data: base64Audio, mimeType },
229
- });
230
- // Log every 50th successful send to confirm data is reaching WebSocket
231
- if (this.sendCount % 50 === 0) {
232
- logger.info('VoiceService', `✅ sendAudio #${this.sendCount} OK — session.isOpen=${!!this.session}`);
233
- }
234
- } catch (error: any) {
235
- logger.error('VoiceService', `❌ sendAudio EXCEPTION: ${error.message}\n${error.stack?.substring(0, 300)}`);
236
- this.session = null;
237
- this.setStatus('disconnected');
238
- }
239
- }
240
-
241
- // ─── Send Text ─────────────────────────────────────────────
242
-
243
- /** Send text message via SDK's sendClientContent */
244
- sendText(text: string): void {
245
- if (!this.isConnected || !this.session) return;
246
-
247
- logger.info('VoiceService', `🗣️ USER (text): "${text}"`);
248
- try {
249
- this.session.sendClientContent({
250
- turns: [{ role: 'user', parts: [{ text }] }],
251
- turnComplete: true,
252
- });
253
- } catch (error: any) {
254
- logger.error('VoiceService', `sendText failed: ${error.message}`);
255
- }
256
- }
257
-
258
- /**
259
- * Send DOM tree as passive context during live conversation.
260
- * Uses turnComplete: false — the model receives context without responding.
261
- */
262
- sendScreenContext(domText: string): void {
263
- if (!this.isConnected || !this.session) return;
264
-
265
- try {
266
- this.session.sendClientContent({
267
- turns: [{ role: 'user', parts: [{ text: domText }] }],
268
- turnComplete: true,
269
- });
270
- logger.info('VoiceService', `📤 Screen context sent (${domText.length} chars)`);
271
- } catch (error: any) {
272
- logger.error('VoiceService', `sendScreenContext failed: ${error.message}`);
273
- }
274
- }
275
-
276
- // ─── Send Function Response ────────────────────────────────
277
-
278
- /** Send function call result back via SDK's sendToolResponse */
279
- sendFunctionResponse(name: string, id: string, result: any): void {
280
- if (!this.isConnected || !this.session) return;
281
-
282
- logger.info('VoiceService', `📤 Sending tool response for ${name} (id=${id})`);
283
-
284
- try {
285
- this.session.sendToolResponse({
286
- functionResponses: [{ name, id, response: result }],
287
- });
288
- } catch (error: any) {
289
- logger.error('VoiceService', `sendFunctionResponse failed: ${error.message}`);
290
- }
291
- }
292
-
293
- // ─── Internal: Tool Declarations ───────────────────────────
294
-
295
- /**
296
- * Builds function declarations from configured tools.
297
- * Converts BOOLEAN params to STRING (native audio model limitation).
298
- */
299
- private buildToolDeclarations(): any[] {
300
- if (!this.config.tools?.length) return [];
301
-
302
- const validTools = this.config.tools.filter(t => t.name !== 'capture_screenshot');
303
- if (validTools.length === 0) return [];
304
-
305
- return validTools.map(tool => {
306
- const hasParams = Object.keys(tool.parameters || {}).length > 0;
307
- const functionDecl: any = {
308
- name: tool.name,
309
- description: tool.description,
310
- };
311
-
312
- if (hasParams) {
313
- functionDecl.parameters = {
314
- type: 'OBJECT',
315
- properties: Object.fromEntries(
316
- Object.entries(tool.parameters).map(([key, param]) => {
317
- let paramType = param.type.toUpperCase();
318
- let desc = param.description;
319
- if (paramType === 'BOOLEAN') {
320
- paramType = 'STRING';
321
- desc = `${desc} (use "true" or "false")`;
322
- }
323
- return [key, { type: paramType, description: desc }];
324
- })
325
- ),
326
- required: Object.entries(tool.parameters)
327
- .filter(([, param]) => param.required)
328
- .map(([key]) => key),
329
- };
330
- }
331
- return functionDecl;
332
- });
333
- }
334
-
335
- // ─── Internal: Message Handling ────────────────────────────
336
-
337
- /**
338
- * Handle messages from the SDK's onmessage callback.
339
- * The SDK parses binary/JSON automatically — we get clean objects.
340
- *
341
- * Per official docs, tool calls come at the top level as
342
- * `response.toolCall.functionCalls`.
343
- */
344
- private handleSDKMessage(message: any): void {
345
- try {
346
- // RAW MESSAGE DUMP — full session visibility
347
- const msgKeys = Object.keys(message || {}).join(', ');
348
- logger.info('VoiceService', `📨 SDK message keys: [${msgKeys}]`);
349
-
350
- // Full raw dump for non-audio messages (audio is too large)
351
- if (!message.serverContent?.modelTurn?.parts?.some((p: any) => p.inlineData)) {
352
- const rawDump = JSON.stringify(message).substring(0, 1000);
353
- logger.info('VoiceService', `📨 RAW: ${rawDump}`);
354
- }
355
-
356
- // Tool calls — top-level (per official docs)
357
- if (message.toolCall?.functionCalls) {
358
- this.handleToolCalls(message.toolCall.functionCalls);
359
- return;
360
- }
361
-
362
- // Server content (audio, text, transcripts, turn events)
363
- if (message.serverContent) {
364
- this.handleServerContent(message.serverContent);
365
- }
366
-
367
- // Setup complete acknowledgment
368
- if (message.setupComplete !== undefined) {
369
- logger.info('VoiceService', '✅ Setup complete — ready for audio');
370
- this.callbacks.onSetupComplete?.();
371
- }
372
-
373
- // Error messages
374
- if (message.error) {
375
- logger.error('VoiceService', `Server error: ${JSON.stringify(message.error)}`);
376
- this.callbacks.onError?.(message.error.message || 'Server error');
377
- }
378
- } catch (error: any) {
379
- logger.error('VoiceService', `Error handling SDK message: ${error.message}`);
380
- }
381
- }
382
-
383
- /** Process tool calls from the model */
384
- private handleToolCalls(functionCalls: any[]): void {
385
- for (const fn of functionCalls) {
386
- logger.info('VoiceService', `🎯 Tool call: ${fn.name}(${JSON.stringify(fn.args)}) [id=${fn.id}]`);
387
- this.callbacks.onToolCall?.({
388
- name: fn.name,
389
- args: fn.args || {},
390
- id: fn.id,
391
- });
392
- }
393
- }
394
-
395
- private audioResponseCount = 0;
396
-
397
- /** Process server content (audio responses, transcripts, turn events) */
398
- private handleServerContent(content: any): void {
399
- // Log all keys for full visibility
400
- const contentKeys = Object.keys(content || {}).join(', ');
401
- logger.debug('VoiceService', `📦 serverContent keys: [${contentKeys}]`);
402
-
403
- // Turn complete
404
- if (content.turnComplete) {
405
- logger.info('VoiceService', `🏁 Turn complete (audioChunks sent: ${this.audioResponseCount})`);
406
- this.audioResponseCount = 0;
407
- this.callbacks.onTurnComplete?.();
408
- }
409
-
410
- // Model output parts (audio + optional thinking text)
411
- if (content.modelTurn?.parts) {
412
- for (const part of content.modelTurn.parts) {
413
- if (part.inlineData?.data) {
414
- this.audioResponseCount++;
415
- if (this.audioResponseCount <= 3 || this.audioResponseCount % 20 === 0) {
416
- logger.info('VoiceService', `🔊 Audio chunk #${this.audioResponseCount}: ${part.inlineData.data.length} b64 chars, mime=${part.inlineData.mimeType || 'unknown'}`);
417
- }
418
- this.callbacks.onAudioResponse?.(part.inlineData.data);
419
- }
420
- if (part.text) {
421
- logger.info('VoiceService', `🤖 MODEL: "${part.text}"`);
422
- this.callbacks.onTranscript?.(part.text, true, 'model');
423
- }
424
- }
425
- }
426
-
427
- // Input transcription (user's speech-to-text)
428
- if (content.inputTranscription?.text) {
429
- logger.info('VoiceService', `🗣️ USER (voice): "${content.inputTranscription.text}"`);
430
- this.callbacks.onTranscript?.(content.inputTranscription.text, true, 'user');
431
- }
432
-
433
- // Output transcription (model's speech-to-text)
434
- if (content.outputTranscription?.text) {
435
- logger.info('VoiceService', `🤖 MODEL (voice): "${content.outputTranscription.text}"`);
436
- this.callbacks.onTranscript?.(content.outputTranscription.text, true, 'model');
437
- }
438
-
439
- // Tool calls inside serverContent (some SDK versions deliver here)
440
- if (content.toolCall?.functionCalls) {
441
- this.handleToolCalls(content.toolCall.functionCalls);
442
- }
443
- }
444
-
445
- // ─── Helpers ───────────────────────────────────────────────
446
-
447
- private setStatus(newStatus: VoiceStatus): void {
448
- this._status = newStatus;
449
- this.callbacks.onStatusChange?.(newStatus);
450
- }
451
- }
@@ -1,137 +0,0 @@
1
- import { logger } from '../../utils/logger';
2
- import { getDeviceId } from '../telemetry/device';
3
-
4
- const LOG_TAG = 'FlagService';
5
-
6
- /**
7
- * MurmurHash3 (32-bit) implementation
8
- */
9
- function murmurhash3_32_gc(key: string, seed: number = 0): number {
10
- let remainder, bytes, h1, h1b, c1, c2, k1, i;
11
-
12
- remainder = key.length & 3; // key.length % 4
13
- bytes = key.length - remainder;
14
- h1 = seed;
15
- c1 = 0xcc9e2d51;
16
- c2 = 0x1b873593;
17
- i = 0;
18
-
19
- while (i < bytes) {
20
- k1 =
21
- ((key.charCodeAt(i) & 0xff)) |
22
- ((key.charCodeAt(++i) & 0xff) << 8) |
23
- ((key.charCodeAt(++i) & 0xff) << 16) |
24
- ((key.charCodeAt(++i) & 0xff) << 24);
25
- ++i;
26
-
27
- k1 = ((((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16))) & 0xffffffff;
28
- k1 = (k1 << 15) | (k1 >>> 17);
29
- k1 = ((((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16))) & 0xffffffff;
30
-
31
- h1 ^= k1;
32
- h1 = (h1 << 13) | (h1 >>> 19);
33
- h1b = ((((h1 & 0xffff) * 5) + ((((h1 >>> 16) * 5) & 0xffff) << 16))) & 0xffffffff;
34
- h1 = (((h1b & 0xffff) + 0x6b64) + ((((h1b >>> 16) + 0xe654) & 0xffff) << 16));
35
- }
36
-
37
- if (remainder >= 1) {
38
- k1 = 0;
39
- if (remainder >= 3) k1 ^= (key.charCodeAt(i + 2) & 0xff) << 16;
40
- if (remainder >= 2) k1 ^= (key.charCodeAt(i + 1) & 0xff) << 8;
41
- k1 ^= (key.charCodeAt(i) & 0xff);
42
- k1 = (((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16)) & 0xffffffff;
43
- k1 = (k1 << 15) | (k1 >>> 17);
44
- k1 = (((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16)) & 0xffffffff;
45
- h1 ^= k1;
46
- }
47
-
48
- h1 ^= key.length;
49
-
50
- h1 ^= h1 >>> 16;
51
- h1 = (((h1 & 0xffff) * 0x85ebca6b) + ((((h1 >>> 16) * 0x85ebca6b) & 0xffff) << 16)) & 0xffffffff;
52
- h1 ^= h1 >>> 13;
53
- h1 = ((((h1 & 0xffff) * 0xc2b2ae35) + ((((h1 >>> 16) * 0xc2b2ae35) & 0xffff) << 16))) & 0xffffffff;
54
- h1 ^= h1 >>> 16;
55
-
56
- return h1 >>> 0;
57
- }
58
-
59
- export interface FeatureFlagPayload {
60
- key: string;
61
- variants: string[];
62
- rollout: number[];
63
- }
64
-
65
- export class FlagService {
66
- private assignments: Record<string, string> = {};
67
- private fetched: boolean = false;
68
-
69
- constructor(private hostUrl: string) {}
70
-
71
- /**
72
- * Fetch feature flags from the dashboard backend
73
- */
74
- async fetch(analyticsKey: string, userId?: string): Promise<void> {
75
- try {
76
- // Avoid fetching if already loaded, unless explicitly forced?
77
- // For now, allow refetching just in case.
78
- const res = await fetch(`${this.hostUrl}/api/v1/flags/sync?key=${analyticsKey}`);
79
- if (!res.ok) {
80
- throw new Error(`Failed to fetch flags: ${res.status}`);
81
- }
82
-
83
- const data = await res.json();
84
- const flags: FeatureFlagPayload[] = data.flags || [];
85
-
86
- this.assignAll(flags, userId);
87
- this.fetched = true;
88
- logger.info(LOG_TAG, `Fetched ${flags.length} flags`);
89
- } catch (err: any) {
90
- logger.warn(LOG_TAG, `Could not sync feature flags: ${err.message}`);
91
- }
92
- }
93
-
94
- /**
95
- * Deterministically assign a variant using murmurhash.
96
- */
97
- private assignVariant(userIdentifier: string, flagKey: string, variants: string[], rollout: number[]): string {
98
- const hash = murmurhash3_32_gc(`${userIdentifier}_${flagKey}`) % 100;
99
- let cumulative = 0;
100
-
101
- for (let i = 0; i < rollout.length; i++) {
102
- cumulative += rollout[i]!;
103
- if (hash < cumulative) {
104
- return variants[i]!;
105
- }
106
- }
107
-
108
- // Fallback if rollout doesn't equal exactly 100 or edge case
109
- return variants[0]!
110
- }
111
-
112
- private assignAll(flags: FeatureFlagPayload[], userId?: string) {
113
- const identifier = userId || getDeviceId() || 'unknown';
114
-
115
- const newAssignments: Record<string, string> = {};
116
- for (const flag of flags) {
117
- if (flag.variants.length > 0 && flag.rollout.length === flag.variants.length) {
118
- newAssignments[flag.key] = this.assignVariant(identifier, flag.key, flag.variants, flag.rollout);
119
- }
120
- }
121
-
122
- this.assignments = newAssignments;
123
- }
124
-
125
- /** Get a specific flag value */
126
- getFlag(key: string, defaultValue?: string): string {
127
- if (!this.fetched) {
128
- logger.debug(LOG_TAG, `getFlag("${key}") called before flags were fetched. Returning default.`);
129
- }
130
- return this.assignments[key] ?? defaultValue ?? '';
131
- }
132
-
133
- /** Get all active assignments for telemetry */
134
- getAllFlags(): Record<string, string> {
135
- return { ...this.assignments };
136
- }
137
- }
@@ -1,66 +0,0 @@
1
- /**
2
- * MobileAI — Public static API for consumer event tracking.
3
- *
4
- * Usage:
5
- * import { MobileAI } from '@mobileai/react-native';
6
- * MobileAI.track('purchase_complete', { total: 29.99 });
7
- *
8
- * The TelemetryService instance is injected by the <AIAgent> component.
9
- * If no analyticsKey is configured, all calls are no-ops.
10
- */
11
-
12
- import type { TelemetryService } from './TelemetryService';
13
- import { logger } from '../../utils/logger';
14
-
15
- const LOG_TAG = 'MobileAI';
16
-
17
- let service: TelemetryService | null = null;
18
-
19
- export const MobileAI = {
20
- /**
21
- * Track a custom business event.
22
- * @param eventName - Name of the event (e.g., 'purchase_complete')
23
- * @param data - Event-specific key-value data
24
- */
25
- track(eventName: string, data: Record<string, unknown> = {}): void {
26
- if (!service) {
27
- logger.debug(LOG_TAG, `track('${eventName}') ignored — no analyticsKey configured`);
28
- return;
29
- }
30
- service.track(eventName, data);
31
- },
32
-
33
- /**
34
- * Identify the current user (optional, for user-level analytics).
35
- * @param userId - Unique user identifier (hashed by consumer)
36
- * @param traits - Optional user traits (plan, role, etc.)
37
- */
38
- identify(userId: string, traits: Record<string, unknown> = {}): void {
39
- if (!service) {
40
- logger.debug(LOG_TAG, 'identify() ignored — no analyticsKey configured');
41
- return;
42
- }
43
- service.track('identify', { user_id: userId, ...traits });
44
- },
45
-
46
- /**
47
- * Get an assigned feature flag variation for the current device.
48
- * Deterministic via murmurhash. Call after MobileAI has initialized.
49
- * @param key Flag key
50
- * @param defaultValue Fallback if not assigned
51
- */
52
- getFlag(key: string, defaultValue?: string): string {
53
- if (!service) {
54
- return defaultValue ?? '';
55
- }
56
- return service.flags.getFlag(key, defaultValue);
57
- },
58
- };
59
-
60
- /**
61
- * Internal: Bind the TelemetryService instance (called by AIAgent on mount).
62
- * Not exported to consumers.
63
- */
64
- export function bindTelemetryService(instance: TelemetryService | null): void {
65
- service = instance;
66
- }
@@ -1,17 +0,0 @@
1
- const EMAIL_RE = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
2
- const PHONE_RE = /(\+?\d[\d\s\-().]{7,}\d)/g;
3
- const CARD_RE = /\b\d{4}[\s\-]?\d{4}[\s\-]?\d{4}[\s\-]?\d{4}\b/g;
4
-
5
- /**
6
- * Scrubs common PII patterns from strings.
7
- * Used by TelemetryService to sanitize auto-captured touch labels.
8
- */
9
- export function scrubPII(value: string): string {
10
- if (typeof value !== 'string') return value;
11
-
12
- return value
13
- .replace(EMAIL_RE, '[email]')
14
- .replace(CARD_RE, '[card]')
15
- // Phone numbers are tricky, we rely on the 7+ digit regex
16
- .replace(PHONE_RE, '[phone]');
17
- }