@mobileai/react-native 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +121 -0
- package/lib/module/components/AIAgent.js +248 -53
- package/lib/module/components/AIAgent.js.map +1 -1
- package/lib/module/components/AIZone.js +140 -0
- package/lib/module/components/AIZone.js.map +1 -0
- package/lib/module/components/AgentErrorBoundary.js +9 -0
- package/lib/module/components/AgentErrorBoundary.js.map +1 -1
- package/lib/module/components/HighlightOverlay.js +138 -0
- package/lib/module/components/HighlightOverlay.js.map +1 -0
- package/lib/module/components/ProactiveHint.js +138 -0
- package/lib/module/components/ProactiveHint.js.map +1 -0
- package/lib/module/components/cards/InfoCard.js +65 -0
- package/lib/module/components/cards/InfoCard.js.map +1 -0
- package/lib/module/components/cards/ReviewSummary.js +74 -0
- package/lib/module/components/cards/ReviewSummary.js.map +1 -0
- package/lib/module/core/AgentRuntime.js +16 -3
- package/lib/module/core/AgentRuntime.js.map +1 -1
- package/lib/module/core/FiberTreeWalker.js +62 -85
- package/lib/module/core/FiberTreeWalker.js.map +1 -1
- package/lib/module/core/IdleDetector.js +51 -0
- package/lib/module/core/IdleDetector.js.map +1 -0
- package/lib/module/core/ZoneRegistry.js +47 -0
- package/lib/module/core/ZoneRegistry.js.map +1 -0
- package/lib/module/core/systemPrompt.js +2 -0
- package/lib/module/core/systemPrompt.js.map +1 -1
- package/lib/module/index.js +21 -0
- package/lib/module/index.js.map +1 -1
- package/lib/module/services/AudioOutputService.js +10 -0
- package/lib/module/services/AudioOutputService.js.map +1 -1
- package/lib/module/services/flags/FlagService.js +117 -0
- package/lib/module/services/flags/FlagService.js.map +1 -0
- package/lib/module/services/telemetry/MobileAI.js +66 -0
- package/lib/module/services/telemetry/MobileAI.js.map +1 -0
- package/lib/module/services/telemetry/PiiScrubber.js +17 -0
- package/lib/module/services/telemetry/PiiScrubber.js.map +1 -0
- package/lib/module/services/telemetry/TelemetryService.js +260 -0
- package/lib/module/services/telemetry/TelemetryService.js.map +1 -0
- package/lib/module/services/telemetry/TouchAutoCapture.js +159 -0
- package/lib/module/services/telemetry/TouchAutoCapture.js.map +1 -0
- package/lib/module/services/telemetry/device.js +19 -0
- package/lib/module/services/telemetry/device.js.map +1 -0
- package/lib/module/services/telemetry/index.js +9 -0
- package/lib/module/services/telemetry/index.js.map +1 -0
- package/lib/module/services/telemetry/types.js +2 -0
- package/lib/module/services/telemetry/types.js.map +1 -0
- package/lib/module/support/CSATSurvey.js +273 -0
- package/lib/module/support/CSATSurvey.js.map +1 -0
- package/lib/module/support/EscalationSocket.js +92 -0
- package/lib/module/support/EscalationSocket.js.map +1 -0
- package/lib/module/support/SupportGreeting.js +142 -0
- package/lib/module/support/SupportGreeting.js.map +1 -0
- package/lib/module/support/escalateTool.js +120 -0
- package/lib/module/support/escalateTool.js.map +1 -0
- package/lib/module/support/index.js +18 -0
- package/lib/module/support/index.js.map +1 -0
- package/lib/module/support/supportPrompt.js +47 -0
- package/lib/module/support/supportPrompt.js.map +1 -0
- package/lib/module/support/types.js +2 -0
- package/lib/module/support/types.js.map +1 -0
- package/lib/module/tools/guideTool.js +61 -0
- package/lib/module/tools/guideTool.js.map +1 -0
- package/lib/module/tools/index.js +3 -0
- package/lib/module/tools/index.js.map +1 -1
- package/lib/module/tools/restoreTool.js +31 -0
- package/lib/module/tools/restoreTool.js.map +1 -0
- package/lib/module/tools/simplifyTool.js +31 -0
- package/lib/module/tools/simplifyTool.js.map +1 -0
- package/lib/module/types/jsx.d.js +4 -0
- package/lib/module/types/jsx.d.js.map +1 -0
- package/lib/typescript/src/components/AIAgent.d.ts +21 -2
- package/lib/typescript/src/components/AIAgent.d.ts.map +1 -1
- package/lib/typescript/src/components/AIZone.d.ts +16 -0
- package/lib/typescript/src/components/AIZone.d.ts.map +1 -0
- package/lib/typescript/src/components/AgentErrorBoundary.d.ts +1 -0
- package/lib/typescript/src/components/AgentErrorBoundary.d.ts.map +1 -1
- package/lib/typescript/src/components/HighlightOverlay.d.ts +10 -0
- package/lib/typescript/src/components/HighlightOverlay.d.ts.map +1 -0
- package/lib/typescript/src/components/ProactiveHint.d.ts +10 -0
- package/lib/typescript/src/components/ProactiveHint.d.ts.map +1 -0
- package/lib/typescript/src/components/cards/InfoCard.d.ts +19 -0
- package/lib/typescript/src/components/cards/InfoCard.d.ts.map +1 -0
- package/lib/typescript/src/components/cards/ReviewSummary.d.ts +19 -0
- package/lib/typescript/src/components/cards/ReviewSummary.d.ts.map +1 -0
- package/lib/typescript/src/core/AgentRuntime.d.ts +1 -0
- package/lib/typescript/src/core/AgentRuntime.d.ts.map +1 -1
- package/lib/typescript/src/core/FiberTreeWalker.d.ts.map +1 -1
- package/lib/typescript/src/core/IdleDetector.d.ts +27 -0
- package/lib/typescript/src/core/IdleDetector.d.ts.map +1 -0
- package/lib/typescript/src/core/ZoneRegistry.d.ts +13 -0
- package/lib/typescript/src/core/ZoneRegistry.d.ts.map +1 -0
- package/lib/typescript/src/core/systemPrompt.d.ts.map +1 -1
- package/lib/typescript/src/core/types.d.ts +54 -0
- package/lib/typescript/src/core/types.d.ts.map +1 -1
- package/lib/typescript/src/index.d.ts +5 -0
- package/lib/typescript/src/index.d.ts.map +1 -1
- package/lib/typescript/src/services/AudioOutputService.d.ts.map +1 -1
- package/lib/typescript/src/services/flags/FlagService.d.ts +25 -0
- package/lib/typescript/src/services/flags/FlagService.d.ts.map +1 -0
- package/lib/typescript/src/services/telemetry/MobileAI.d.ts +38 -0
- package/lib/typescript/src/services/telemetry/MobileAI.d.ts.map +1 -0
- package/lib/typescript/src/services/telemetry/PiiScrubber.d.ts +6 -0
- package/lib/typescript/src/services/telemetry/PiiScrubber.d.ts.map +1 -0
- package/lib/typescript/src/services/telemetry/TelemetryService.d.ts +49 -0
- package/lib/typescript/src/services/telemetry/TelemetryService.d.ts.map +1 -0
- package/lib/typescript/src/services/telemetry/TouchAutoCapture.d.ts +28 -0
- package/lib/typescript/src/services/telemetry/TouchAutoCapture.d.ts.map +1 -0
- package/lib/typescript/src/services/telemetry/device.d.ts +7 -0
- package/lib/typescript/src/services/telemetry/device.d.ts.map +1 -0
- package/lib/typescript/src/services/telemetry/index.d.ts +7 -0
- package/lib/typescript/src/services/telemetry/index.d.ts.map +1 -0
- package/lib/typescript/src/services/telemetry/types.d.ts +50 -0
- package/lib/typescript/src/services/telemetry/types.d.ts.map +1 -0
- package/lib/typescript/src/support/CSATSurvey.d.ts +20 -0
- package/lib/typescript/src/support/CSATSurvey.d.ts.map +1 -0
- package/lib/typescript/src/support/EscalationSocket.d.ts +38 -0
- package/lib/typescript/src/support/EscalationSocket.d.ts.map +1 -0
- package/lib/typescript/src/support/SupportGreeting.d.ts +19 -0
- package/lib/typescript/src/support/SupportGreeting.d.ts.map +1 -0
- package/lib/typescript/src/support/escalateTool.d.ts +25 -0
- package/lib/typescript/src/support/escalateTool.d.ts.map +1 -0
- package/lib/typescript/src/support/index.d.ts +11 -0
- package/lib/typescript/src/support/index.d.ts.map +1 -0
- package/lib/typescript/src/support/supportPrompt.d.ts +12 -0
- package/lib/typescript/src/support/supportPrompt.d.ts.map +1 -0
- package/lib/typescript/src/support/types.d.ts +114 -0
- package/lib/typescript/src/support/types.d.ts.map +1 -0
- package/lib/typescript/src/tools/guideTool.d.ts +4 -0
- package/lib/typescript/src/tools/guideTool.d.ts.map +1 -0
- package/lib/typescript/src/tools/index.d.ts +3 -0
- package/lib/typescript/src/tools/index.d.ts.map +1 -1
- package/lib/typescript/src/tools/restoreTool.d.ts +3 -0
- package/lib/typescript/src/tools/restoreTool.d.ts.map +1 -0
- package/lib/typescript/src/tools/simplifyTool.d.ts +3 -0
- package/lib/typescript/src/tools/simplifyTool.d.ts.map +1 -0
- package/lib/typescript/src/tools/types.d.ts +2 -0
- package/lib/typescript/src/tools/types.d.ts.map +1 -1
- package/package.json +5 -1
- package/src/components/AIAgent.tsx +253 -15
- package/src/components/AIZone.tsx +147 -0
- package/src/components/AgentErrorBoundary.tsx +10 -0
- package/src/components/HighlightOverlay.tsx +136 -0
- package/src/components/ProactiveHint.tsx +145 -0
- package/src/components/cards/InfoCard.tsx +58 -0
- package/src/components/cards/ReviewSummary.tsx +76 -0
- package/src/core/AgentRuntime.ts +18 -0
- package/src/core/FiberTreeWalker.ts +71 -93
- package/src/core/IdleDetector.ts +72 -0
- package/src/core/ZoneRegistry.ts +44 -0
- package/src/core/systemPrompt.ts +2 -0
- package/src/core/types.ts +60 -0
- package/src/index.ts +31 -0
- package/src/services/AudioOutputService.ts +13 -0
- package/src/services/flags/FlagService.ts +137 -0
- package/src/services/telemetry/MobileAI.ts +66 -0
- package/src/services/telemetry/PiiScrubber.ts +17 -0
- package/src/services/telemetry/TelemetryService.ts +291 -0
- package/src/services/telemetry/TouchAutoCapture.ts +165 -0
- package/src/services/telemetry/device.ts +16 -0
- package/src/services/telemetry/index.ts +13 -0
- package/src/services/telemetry/types.ts +75 -0
- package/src/support/CSATSurvey.tsx +304 -0
- package/src/support/EscalationSocket.ts +113 -0
- package/src/support/SupportGreeting.tsx +161 -0
- package/src/support/escalateTool.ts +134 -0
- package/src/support/index.ts +27 -0
- package/src/support/supportPrompt.ts +55 -0
- package/src/support/types.ts +141 -0
- package/src/tools/guideTool.ts +67 -0
- package/src/tools/index.ts +3 -0
- package/src/tools/restoreTool.ts +33 -0
- package/src/tools/simplifyTool.ts +33 -0
- package/src/tools/types.ts +2 -0
- package/src/types/jsx.d.ts +20 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/tools/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAI1D,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAID,MAAM,WAAW,SAAS;IACxB,kDAAkD;IAClD,IAAI,EAAE,MAAM,CAAC;IACb,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,2CAA2C;IAC3C,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAID,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,UAAU,EAAE,MAAM,GAAG,CAAC;IACtB,8DAA8D;IAC9D,aAAa,EAAE,MAAM,UAAU,CAAC;IAChC,yCAAyC;IACzC,oBAAoB,EAAE,MAAM,MAAM,CAAC;IACnC,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,GAAG,CAAC;IACtB,sBAAsB;IACtB,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,gCAAgC;IAChC,aAAa,CAAC,EAAE,MAAM,MAAM,EAAE,CAAC;IAC/B,8BAA8B;IAC9B,cAAc,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;IAClD,qCAAqC;IACrC,iBAAiB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,GAAG,KAAK,GAAG,CAAC;IAC1D,oCAAoC;IACpC,iBAAiB,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/tools/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAI1D,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAID,MAAM,WAAW,SAAS;IACxB,kDAAkD;IAClD,IAAI,EAAE,MAAM,CAAC;IACb,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC1C,2CAA2C;IAC3C,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAID,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,UAAU,EAAE,MAAM,GAAG,CAAC;IACtB,8DAA8D;IAC9D,aAAa,EAAE,MAAM,UAAU,CAAC;IAChC,yCAAyC;IACzC,oBAAoB,EAAE,MAAM,MAAM,CAAC;IACnC,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,GAAG,CAAC;IACtB,sBAAsB;IACtB,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,gCAAgC;IAChC,aAAa,CAAC,EAAE,MAAM,MAAM,EAAE,CAAC;IAC/B,8BAA8B;IAC9B,cAAc,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;IAClD,qCAAqC;IACrC,iBAAiB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,GAAG,KAAK,GAAG,CAAC;IAC1D,oCAAoC;IACpC,iBAAiB,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACjD,mEAAmE;IACnE,qBAAqB,CAAC,EAAE,MAAM,GAAG,CAAC;CACnC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mobileai/react-native",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.2",
|
|
4
4
|
"description": "Build autonomous AI agents for React Native and Expo apps. Provides AI-native UI traversal, tool calling, and structured reasoning.",
|
|
5
5
|
"main": "./lib/module/index.js",
|
|
6
6
|
"source": "./src/index.ts",
|
|
@@ -99,6 +99,7 @@
|
|
|
99
99
|
"@eslint/js": "^9.35.0",
|
|
100
100
|
"@react-native/babel-preset": "0.83.0",
|
|
101
101
|
"@react-native/eslint-config": "0.83.0",
|
|
102
|
+
"@testing-library/react-native": "^13.3.3",
|
|
102
103
|
"@types/jest": "^29.5.14",
|
|
103
104
|
"@types/react": "^19.1.12",
|
|
104
105
|
"@types/react-native": "^0.72.8",
|
|
@@ -168,6 +169,9 @@
|
|
|
168
169
|
},
|
|
169
170
|
"jest": {
|
|
170
171
|
"preset": "react-native",
|
|
172
|
+
"testMatch": [
|
|
173
|
+
"**/?(*.)+(spec|test).[jt]s?(x)"
|
|
174
|
+
],
|
|
171
175
|
"modulePathIgnorePatterns": [
|
|
172
176
|
"<rootDir>/example-react-navigation/node_modules",
|
|
173
177
|
"<rootDir>/lib/"
|
|
@@ -27,8 +27,14 @@ import { MCPBridge } from '../core/MCPBridge';
|
|
|
27
27
|
import { VoiceService } from '../services/VoiceService';
|
|
28
28
|
import { AudioInputService } from '../services/AudioInputService';
|
|
29
29
|
import { AudioOutputService } from '../services/AudioOutputService';
|
|
30
|
-
import
|
|
30
|
+
import { TelemetryService, bindTelemetryService } from '../services/telemetry';
|
|
31
|
+
import { extractTouchLabel, checkRageClick } from '../services/telemetry/TouchAutoCapture';
|
|
32
|
+
import type { AgentConfig, AgentMode, ExecutionResult, ToolDefinition, AgentStep, TokenUsage, KnowledgeBaseConfig, ChatBarTheme, AIMessage, AIProviderName, ScreenMap, ProactiveHelpConfig } from '../core/types';
|
|
31
33
|
import { AgentErrorBoundary } from './AgentErrorBoundary';
|
|
34
|
+
import { HighlightOverlay } from './HighlightOverlay';
|
|
35
|
+
import { IdleDetector } from '../core/IdleDetector';
|
|
36
|
+
import { ProactiveHint } from './ProactiveHint';
|
|
37
|
+
import { createEscalateTool } from '../support/escalateTool';
|
|
32
38
|
|
|
33
39
|
// ─── Context ───────────────────────────────────────────────────
|
|
34
40
|
|
|
@@ -164,8 +170,32 @@ interface AIAgentProps {
|
|
|
164
170
|
* @default true
|
|
165
171
|
*/
|
|
166
172
|
useScreenMap?: boolean;
|
|
173
|
+
|
|
174
|
+
// ── Analytics (opt-in) ── @internal — requires api.mobileai.dev ──
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* @internal Requires api.mobileai.dev — not yet available.
|
|
178
|
+
* Publishable analytics key (mobileai_pub_xxx).
|
|
179
|
+
*/
|
|
180
|
+
analyticsKey?: string;
|
|
181
|
+
/**
|
|
182
|
+
* @internal Requires api.mobileai.dev — not yet available.
|
|
183
|
+
* Proxy URL for enterprise customers — routes events through your backend.
|
|
184
|
+
*/
|
|
185
|
+
analyticsProxyUrl?: string;
|
|
186
|
+
/**
|
|
187
|
+
* @internal Requires api.mobileai.dev — not yet available.
|
|
188
|
+
* Custom headers for analyticsProxyUrl (e.g., auth tokens).
|
|
189
|
+
*/
|
|
190
|
+
analyticsProxyHeaders?: Record<string, string>;
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Proactive agent configuration (detects user hesitation)
|
|
194
|
+
*/
|
|
195
|
+
proactiveHelp?: ProactiveHelpConfig;
|
|
167
196
|
}
|
|
168
197
|
|
|
198
|
+
|
|
169
199
|
// ─── Component ─────────────────────────────────────────────────
|
|
170
200
|
|
|
171
201
|
export function AIAgent({
|
|
@@ -208,6 +238,10 @@ export function AIAgent({
|
|
|
208
238
|
useScreenMap = true,
|
|
209
239
|
maxTokenBudget,
|
|
210
240
|
maxCostUSD,
|
|
241
|
+
analyticsKey,
|
|
242
|
+
analyticsProxyUrl,
|
|
243
|
+
analyticsProxyHeaders,
|
|
244
|
+
proactiveHelp,
|
|
211
245
|
}: AIAgentProps) {
|
|
212
246
|
// Configure logger based on debug prop
|
|
213
247
|
React.useEffect(() => {
|
|
@@ -228,6 +262,43 @@ export function AIAgent({
|
|
|
228
262
|
setLastResult(null);
|
|
229
263
|
}, []);
|
|
230
264
|
|
|
265
|
+
// ─── Auto-create MobileAI escalation tool ─────────────────────
|
|
266
|
+
// When analyticsKey is present and consumer hasn't provided their own
|
|
267
|
+
// escalate_to_human tool, auto-wire the MobileAI platform provider.
|
|
268
|
+
// Human replies from the dashboard inbox are injected into chat messages.
|
|
269
|
+
const autoEscalateTool = useMemo(() => {
|
|
270
|
+
if (!analyticsKey) return null;
|
|
271
|
+
if (customTools?.['escalate_to_human']) return null; // consumer overrides
|
|
272
|
+
return createEscalateTool({
|
|
273
|
+
config: { provider: 'mobileai' },
|
|
274
|
+
analyticsKey,
|
|
275
|
+
getContext: () => ({
|
|
276
|
+
currentScreen: (navRef as any)?.getCurrentRoute?.()?.name ?? 'unknown',
|
|
277
|
+
originalQuery: '',
|
|
278
|
+
stepsBeforeEscalation: 0,
|
|
279
|
+
}),
|
|
280
|
+
getHistory: () =>
|
|
281
|
+
messages.map((m) => ({ role: m.role, content: m.content })),
|
|
282
|
+
onHumanReply: (reply: string) => {
|
|
283
|
+
setMessages((prev) => [
|
|
284
|
+
...prev,
|
|
285
|
+
{
|
|
286
|
+
id: `human-${Date.now()}`,
|
|
287
|
+
role: 'assistant' as const,
|
|
288
|
+
content: `👤 Human Agent: ${reply}`,
|
|
289
|
+
timestamp: Date.now(),
|
|
290
|
+
},
|
|
291
|
+
]);
|
|
292
|
+
},
|
|
293
|
+
});
|
|
294
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
295
|
+
}, [analyticsKey, navRef, customTools]);
|
|
296
|
+
|
|
297
|
+
const mergedCustomTools = useMemo(() => {
|
|
298
|
+
if (!autoEscalateTool) return customTools;
|
|
299
|
+
return { escalate_to_human: autoEscalateTool, ...customTools };
|
|
300
|
+
}, [autoEscalateTool, customTools]);
|
|
301
|
+
|
|
231
302
|
// ─── Voice/Live Mode State ──────────────────────────────────
|
|
232
303
|
const [mode, setMode] = useState<AgentMode>('text');
|
|
233
304
|
const [isMicActive, setIsMicActive] = useState(false);
|
|
@@ -272,7 +343,7 @@ export function AIAgent({
|
|
|
272
343
|
onAfterStep,
|
|
273
344
|
onBeforeTask,
|
|
274
345
|
onAfterTask,
|
|
275
|
-
customTools: mode === 'voice' ? { ...
|
|
346
|
+
customTools: mode === 'voice' ? { ...mergedCustomTools, ask_user: null } : mergedCustomTools,
|
|
276
347
|
instructions,
|
|
277
348
|
stepDelay,
|
|
278
349
|
mcpServerUrl,
|
|
@@ -296,6 +367,11 @@ export function AIAgent({
|
|
|
296
367
|
setStatusText('');
|
|
297
368
|
});
|
|
298
369
|
}),
|
|
370
|
+
// Toggle isAgentActing flag on TelemetryService before/after every tool
|
|
371
|
+
// so that AI-driven taps are never tracked as user_interaction events.
|
|
372
|
+
onToolExecute: (active: boolean) => {
|
|
373
|
+
telemetryRef.current?.setAgentActing(active);
|
|
374
|
+
},
|
|
299
375
|
}), [
|
|
300
376
|
mode, apiKey, proxyUrl, proxyHeaders, voiceProxyUrl, voiceProxyHeaders, model, maxSteps,
|
|
301
377
|
interactiveBlacklist, interactiveWhitelist,
|
|
@@ -322,6 +398,61 @@ export function AIAgent({
|
|
|
322
398
|
runtime.updateRefs(rootViewRef.current, navRef);
|
|
323
399
|
}, [runtime, navRef]);
|
|
324
400
|
|
|
401
|
+
// ─── Telemetry ─────────────────────────────────────────────
|
|
402
|
+
|
|
403
|
+
const telemetryRef = useRef<TelemetryService | null>(null);
|
|
404
|
+
|
|
405
|
+
useEffect(() => {
|
|
406
|
+
if (!analyticsKey && !analyticsProxyUrl) {
|
|
407
|
+
bindTelemetryService(null);
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
const telemetry = new TelemetryService({
|
|
412
|
+
analyticsKey,
|
|
413
|
+
analyticsProxyUrl,
|
|
414
|
+
analyticsProxyHeaders,
|
|
415
|
+
debug,
|
|
416
|
+
});
|
|
417
|
+
telemetryRef.current = telemetry;
|
|
418
|
+
bindTelemetryService(telemetry);
|
|
419
|
+
telemetry.start();
|
|
420
|
+
|
|
421
|
+
return () => {
|
|
422
|
+
telemetry.stop();
|
|
423
|
+
telemetryRef.current = null;
|
|
424
|
+
bindTelemetryService(null);
|
|
425
|
+
};
|
|
426
|
+
}, [analyticsKey, analyticsProxyUrl, analyticsProxyHeaders, debug]);
|
|
427
|
+
|
|
428
|
+
// ─── Security warnings ──────────────────────────────────────
|
|
429
|
+
|
|
430
|
+
useEffect(() => {
|
|
431
|
+
// @ts-ignore
|
|
432
|
+
if (typeof __DEV__ !== 'undefined' && !__DEV__ && apiKey && !proxyUrl) {
|
|
433
|
+
console.warn(
|
|
434
|
+
'[MobileAI] ⚠️ SECURITY WARNING: You are using `apiKey` directly in a production build. ' +
|
|
435
|
+
'This exposes your LLM provider key in the app binary. ' +
|
|
436
|
+
'Use `apiProxyUrl` to route requests through your backend instead. ' +
|
|
437
|
+
'See docs for details.'
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
}, [apiKey, proxyUrl]);
|
|
441
|
+
|
|
442
|
+
// Track screen changes via navRef
|
|
443
|
+
useEffect(() => {
|
|
444
|
+
if (!navRef?.addListener || !telemetryRef.current) return;
|
|
445
|
+
|
|
446
|
+
const unsubscribe = navRef.addListener('state', () => {
|
|
447
|
+
const currentRoute = navRef.getCurrentRoute?.();
|
|
448
|
+
if (currentRoute?.name) {
|
|
449
|
+
telemetryRef.current?.setScreen(currentRoute.name);
|
|
450
|
+
}
|
|
451
|
+
});
|
|
452
|
+
|
|
453
|
+
return () => unsubscribe?.();
|
|
454
|
+
}, [navRef]);
|
|
455
|
+
|
|
325
456
|
// ─── MCP Bridge ──────────────────────────────────────────────
|
|
326
457
|
|
|
327
458
|
useEffect(() => {
|
|
@@ -335,6 +466,42 @@ export function AIAgent({
|
|
|
335
466
|
};
|
|
336
467
|
}, [mcpServerUrl, runtime]);
|
|
337
468
|
|
|
469
|
+
// ─── Proactive Idle Agent ────────────────────────────────────
|
|
470
|
+
|
|
471
|
+
const idleDetectorRef = useRef<IdleDetector | null>(null);
|
|
472
|
+
const [proactiveStage, setProactiveStage] = useState<'hidden' | 'pulse' | 'badge'>('hidden');
|
|
473
|
+
const [proactiveBadgeText, setProactiveBadgeText] = useState('');
|
|
474
|
+
|
|
475
|
+
useEffect(() => {
|
|
476
|
+
if (proactiveHelp?.enabled === false) {
|
|
477
|
+
idleDetectorRef.current?.destroy();
|
|
478
|
+
idleDetectorRef.current = null;
|
|
479
|
+
setProactiveStage('hidden');
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
if (!idleDetectorRef.current) {
|
|
484
|
+
idleDetectorRef.current = new IdleDetector();
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
idleDetectorRef.current.start({
|
|
488
|
+
pulseAfterMs: (proactiveHelp?.pulseAfterMinutes || 2) * 60000,
|
|
489
|
+
badgeAfterMs: (proactiveHelp?.badgeAfterMinutes || 4) * 60000,
|
|
490
|
+
onPulse: () => setProactiveStage('pulse'),
|
|
491
|
+
onBadge: (suggestion: string) => {
|
|
492
|
+
setProactiveBadgeText(suggestion);
|
|
493
|
+
setProactiveStage('badge');
|
|
494
|
+
},
|
|
495
|
+
onReset: () => setProactiveStage('hidden'),
|
|
496
|
+
generateSuggestion: () => proactiveHelp?.generateSuggestion?.(telemetryRef.current?.screen || 'Home') || proactiveHelp?.badgeText || "Need help with this screen?",
|
|
497
|
+
});
|
|
498
|
+
|
|
499
|
+
return () => {
|
|
500
|
+
idleDetectorRef.current?.destroy();
|
|
501
|
+
idleDetectorRef.current = null;
|
|
502
|
+
};
|
|
503
|
+
}, [proactiveHelp, telemetryRef]);
|
|
504
|
+
|
|
338
505
|
// ─── Voice/Live Service Initialization ──────────────────────
|
|
339
506
|
|
|
340
507
|
// Initialize voice services when mode changes to voice or live
|
|
@@ -665,12 +832,36 @@ export function AIAgent({
|
|
|
665
832
|
setStatusText('Thinking...');
|
|
666
833
|
setLastResult(null);
|
|
667
834
|
|
|
835
|
+
// Telemetry: track agent request
|
|
836
|
+
telemetryRef.current?.track('agent_request', {
|
|
837
|
+
query: message.trim(),
|
|
838
|
+
});
|
|
839
|
+
|
|
668
840
|
try {
|
|
669
841
|
// Ensure we have the latest Fiber tree ref
|
|
670
842
|
runtime.updateRefs(rootViewRef.current, navRef);
|
|
671
843
|
|
|
672
844
|
const result = await runtime.execute(message, messages);
|
|
673
845
|
|
|
846
|
+
// Telemetry: track agent completion and per-step details
|
|
847
|
+
if (telemetryRef.current) {
|
|
848
|
+
for (const step of result.steps ?? []) {
|
|
849
|
+
telemetryRef.current.track('agent_step', {
|
|
850
|
+
tool: step.action.name,
|
|
851
|
+
args: step.action.input,
|
|
852
|
+
result: typeof step.action.output === 'string'
|
|
853
|
+
? step.action.output.substring(0, 200)
|
|
854
|
+
: String(step.action.output),
|
|
855
|
+
});
|
|
856
|
+
}
|
|
857
|
+
telemetryRef.current.track('agent_complete', {
|
|
858
|
+
success: result.success,
|
|
859
|
+
steps: result.steps?.length ?? 0,
|
|
860
|
+
tokens: result.tokenUsage?.totalTokens ?? 0,
|
|
861
|
+
cost: result.tokenUsage?.estimatedCostUSD ?? 0,
|
|
862
|
+
});
|
|
863
|
+
}
|
|
864
|
+
|
|
674
865
|
setLastResult(result);
|
|
675
866
|
|
|
676
867
|
// Append assistant message
|
|
@@ -694,6 +885,13 @@ export function AIAgent({
|
|
|
694
885
|
logger.info('AIAgent', `Result: ${result.success ? '✅' : '❌'} ${result.message}`);
|
|
695
886
|
} catch (error: any) {
|
|
696
887
|
logger.error('AIAgent', 'Execution failed:', error);
|
|
888
|
+
|
|
889
|
+
// Telemetry: track agent failure
|
|
890
|
+
telemetryRef.current?.track('agent_complete', {
|
|
891
|
+
success: false,
|
|
892
|
+
error: error.message,
|
|
893
|
+
});
|
|
894
|
+
|
|
697
895
|
setLastResult({
|
|
698
896
|
success: false,
|
|
699
897
|
message: `Error: ${error.message}`,
|
|
@@ -730,8 +928,42 @@ export function AIAgent({
|
|
|
730
928
|
<AgentContext.Provider value={contextValue}>
|
|
731
929
|
<View style={styles.root}>
|
|
732
930
|
{/* App content — rootViewRef captures Fiber tree for element detection */}
|
|
733
|
-
<View
|
|
931
|
+
<View
|
|
932
|
+
ref={rootViewRef}
|
|
933
|
+
style={styles.root}
|
|
934
|
+
collapsable={false}
|
|
935
|
+
onStartShouldSetResponderCapture={(event) => {
|
|
936
|
+
// Auto-capture every tap for analytics (zero-config)
|
|
937
|
+
// Skip if the AI agent is currently executing a tool — those are
|
|
938
|
+
// already tracked as `agent_step` events with full context.
|
|
939
|
+
if (telemetryRef.current && !telemetryRef.current.isAgentActing) {
|
|
940
|
+
const label = extractTouchLabel(event.nativeEvent);
|
|
941
|
+
if (label && label !== 'Unknown Element' && label !== '[pressable]') {
|
|
942
|
+
telemetryRef.current.track('user_interaction', {
|
|
943
|
+
type: 'tap',
|
|
944
|
+
label,
|
|
945
|
+
actor: 'user',
|
|
946
|
+
x: Math.round(event.nativeEvent.pageX),
|
|
947
|
+
y: Math.round(event.nativeEvent.pageY),
|
|
948
|
+
});
|
|
949
|
+
|
|
950
|
+
// Track if user is rage-tapping this specific element
|
|
951
|
+
checkRageClick(label, telemetryRef.current);
|
|
952
|
+
} else {
|
|
953
|
+
// Tapped an unlabelled/empty area
|
|
954
|
+
telemetryRef.current.track('dead_click', {
|
|
955
|
+
x: Math.round(event.nativeEvent.pageX),
|
|
956
|
+
y: Math.round(event.nativeEvent.pageY),
|
|
957
|
+
screen: telemetryRef.current.screen,
|
|
958
|
+
});
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
// IMPORTANT: return false so we don't steal the touch from the actual button
|
|
962
|
+
return false;
|
|
963
|
+
}}
|
|
964
|
+
>
|
|
734
965
|
<AgentErrorBoundary
|
|
966
|
+
telemetryRef={telemetryRef}
|
|
735
967
|
onError={(error, componentStack) => {
|
|
736
968
|
const errorMsg = `⚠️ A rendering error occurred: ${error.message}`;
|
|
737
969
|
lastAgentErrorRef.current = errorMsg;
|
|
@@ -742,14 +974,21 @@ export function AIAgent({
|
|
|
742
974
|
</AgentErrorBoundary>
|
|
743
975
|
</View>
|
|
744
976
|
|
|
745
|
-
{/* Floating UI — absolute-positioned View that passes touches through */}
|
|
746
|
-
{
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
977
|
+
{/* Floating UI — absolute-positioned View that passes touches pass-through unless interacting */}
|
|
978
|
+
<View style={styles.floatingLayer} pointerEvents="box-none">
|
|
979
|
+
{/* Highlight Overlay (always active, listens to events) */}
|
|
980
|
+
<HighlightOverlay />
|
|
981
|
+
|
|
982
|
+
{/* Overlay (shown while thinking) */}
|
|
983
|
+
<AgentOverlay visible={isThinking} statusText={statusText} onCancel={handleCancel} />
|
|
984
|
+
|
|
985
|
+
{/* Chat bar wrapped in Proactive Hint */}
|
|
986
|
+
{showChatBar && (
|
|
987
|
+
<ProactiveHint
|
|
988
|
+
stage={proactiveStage}
|
|
989
|
+
badgeText={proactiveBadgeText}
|
|
990
|
+
onDismiss={() => idleDetectorRef.current?.dismiss()}
|
|
991
|
+
>
|
|
753
992
|
<AgentChatBar
|
|
754
993
|
onSend={handleSend}
|
|
755
994
|
isThinking={isThinking}
|
|
@@ -797,11 +1036,10 @@ export function AIAgent({
|
|
|
797
1036
|
audioOutputRef.current?.unmute();
|
|
798
1037
|
}
|
|
799
1038
|
}}
|
|
800
|
-
|
|
801
1039
|
/>
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
1040
|
+
</ProactiveHint>
|
|
1041
|
+
)}
|
|
1042
|
+
</View>
|
|
805
1043
|
</View>
|
|
806
1044
|
</AgentContext.Provider>
|
|
807
1045
|
);
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import React, { useRef, useEffect, useContext, useState } from 'react';
|
|
2
|
+
import type { ReactElement } from 'react';
|
|
3
|
+
import { View, StyleSheet, Pressable, Text } from 'react-native';
|
|
4
|
+
import { ZoneRegistryContext } from '../core/ZoneRegistry';
|
|
5
|
+
import type { AIZoneConfig } from '../core/types';
|
|
6
|
+
|
|
7
|
+
interface AIZoneProps extends AIZoneConfig {
|
|
8
|
+
children: React.ReactNode;
|
|
9
|
+
style?: any;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// React context to broadcast simplified state down strictly to children
|
|
13
|
+
export const AIZoneStateContext = React.createContext<{ simplified: boolean }>({ simplified: false });
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Declarative boundary that grants the AI permission to modify its subtree.
|
|
17
|
+
* Has zero visual impact by default.
|
|
18
|
+
*/
|
|
19
|
+
export function AIZone({
|
|
20
|
+
id,
|
|
21
|
+
allowHighlight,
|
|
22
|
+
allowInjectHint,
|
|
23
|
+
allowSimplify,
|
|
24
|
+
allowInjectCard,
|
|
25
|
+
templates,
|
|
26
|
+
children,
|
|
27
|
+
style,
|
|
28
|
+
}: AIZoneProps) {
|
|
29
|
+
const zoneRef = useRef<any>(null);
|
|
30
|
+
const registry = useContext(ZoneRegistryContext);
|
|
31
|
+
|
|
32
|
+
// State managed by AI tools
|
|
33
|
+
const [simplified, setSimplified] = useState(false);
|
|
34
|
+
const [injectedCard, setInjectedCard] = useState<ReactElement | null>(null);
|
|
35
|
+
|
|
36
|
+
useEffect(() => {
|
|
37
|
+
// Register zone permissions on mount
|
|
38
|
+
registry.register({
|
|
39
|
+
id,
|
|
40
|
+
allowHighlight,
|
|
41
|
+
allowInjectHint,
|
|
42
|
+
allowSimplify,
|
|
43
|
+
allowInjectCard,
|
|
44
|
+
templates,
|
|
45
|
+
}, zoneRef);
|
|
46
|
+
|
|
47
|
+
// Unregister on unmount
|
|
48
|
+
return () => registry.unregister(id);
|
|
49
|
+
}, [id, allowHighlight, allowInjectHint, allowSimplify, allowInjectCard, templates, registry]);
|
|
50
|
+
|
|
51
|
+
// If the zone exposes an API to manipulate itself locally (outside of AI),
|
|
52
|
+
// we would attach it to the ref or a secondary context. For now, the tools
|
|
53
|
+
// will dispatch events or we can expose a global setter.
|
|
54
|
+
// Actually, the easiest way for the AI tools to mutate this state is
|
|
55
|
+
// an EventEmitter or assigning a controller object to the registry.
|
|
56
|
+
|
|
57
|
+
useEffect(() => {
|
|
58
|
+
// Attach controller to the registry so tools can act on this specific mount instance
|
|
59
|
+
const zone = registry.get(id);
|
|
60
|
+
if (zone) {
|
|
61
|
+
(zone as any)._controller = {
|
|
62
|
+
simplify: () => setSimplified(true),
|
|
63
|
+
restore: () => {
|
|
64
|
+
setSimplified(false);
|
|
65
|
+
setInjectedCard(null);
|
|
66
|
+
},
|
|
67
|
+
injectCard: (card: ReactElement) => setInjectedCard(card),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
}, [id, registry]);
|
|
71
|
+
|
|
72
|
+
return (
|
|
73
|
+
<View ref={zoneRef} style={style} collapsable={false}>
|
|
74
|
+
<AIZoneStateContext.Provider value={{ simplified }}>
|
|
75
|
+
{children}
|
|
76
|
+
|
|
77
|
+
{/* Render AI-Injected Card slot at the bottom of the zone */}
|
|
78
|
+
{injectedCard && (
|
|
79
|
+
<View style={styles.cardWrapper}>
|
|
80
|
+
{injectedCard}
|
|
81
|
+
<Pressable
|
|
82
|
+
style={styles.closeCardBtn}
|
|
83
|
+
onPress={() => setInjectedCard(null)}
|
|
84
|
+
accessibilityLabel="Dismiss AI Card"
|
|
85
|
+
>
|
|
86
|
+
<Text style={styles.closeCardText}>×</Text>
|
|
87
|
+
</Pressable>
|
|
88
|
+
</View>
|
|
89
|
+
)}
|
|
90
|
+
|
|
91
|
+
{/* User cancellation button for simplification */}
|
|
92
|
+
{simplified && (
|
|
93
|
+
<Pressable
|
|
94
|
+
style={styles.showAllBtn}
|
|
95
|
+
onPress={() => setSimplified(false)}
|
|
96
|
+
accessibilityLabel="Show all options"
|
|
97
|
+
>
|
|
98
|
+
<Text style={styles.showAllText}>Show all options</Text>
|
|
99
|
+
</Pressable>
|
|
100
|
+
)}
|
|
101
|
+
</AIZoneStateContext.Provider>
|
|
102
|
+
</View>
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const styles = StyleSheet.create({
|
|
107
|
+
cardWrapper: {
|
|
108
|
+
marginVertical: 12,
|
|
109
|
+
position: 'relative',
|
|
110
|
+
backgroundColor: '#fff',
|
|
111
|
+
borderRadius: 8,
|
|
112
|
+
shadowColor: '#000',
|
|
113
|
+
shadowOpacity: 0.1,
|
|
114
|
+
shadowRadius: 10,
|
|
115
|
+
elevation: 3,
|
|
116
|
+
},
|
|
117
|
+
closeCardBtn: {
|
|
118
|
+
position: 'absolute',
|
|
119
|
+
top: -8,
|
|
120
|
+
right: -8,
|
|
121
|
+
backgroundColor: '#444',
|
|
122
|
+
width: 24,
|
|
123
|
+
height: 24,
|
|
124
|
+
borderRadius: 12,
|
|
125
|
+
alignItems: 'center',
|
|
126
|
+
justifyContent: 'center',
|
|
127
|
+
zIndex: 10,
|
|
128
|
+
},
|
|
129
|
+
closeCardText: {
|
|
130
|
+
color: '#fff',
|
|
131
|
+
fontSize: 14,
|
|
132
|
+
fontWeight: 'bold',
|
|
133
|
+
},
|
|
134
|
+
showAllBtn: {
|
|
135
|
+
marginTop: 8,
|
|
136
|
+
paddingVertical: 10,
|
|
137
|
+
alignItems: 'center',
|
|
138
|
+
justifyContent: 'center',
|
|
139
|
+
backgroundColor: '#f0f0f0',
|
|
140
|
+
borderRadius: 8,
|
|
141
|
+
},
|
|
142
|
+
showAllText: {
|
|
143
|
+
color: '#0066cc',
|
|
144
|
+
fontSize: 14,
|
|
145
|
+
fontWeight: '500',
|
|
146
|
+
}
|
|
147
|
+
});
|
|
@@ -19,6 +19,7 @@ interface Props {
|
|
|
19
19
|
children: React.ReactNode;
|
|
20
20
|
/** Called when an error is caught — reports back to agent runtime */
|
|
21
21
|
onError?: (error: Error, componentStack?: string) => void;
|
|
22
|
+
telemetryRef?: React.RefObject<any>; // Using any to avoid circular import, we duck-type track()
|
|
22
23
|
}
|
|
23
24
|
|
|
24
25
|
interface State {
|
|
@@ -40,6 +41,15 @@ export class AgentErrorBoundary extends React.Component<Props, State> {
|
|
|
40
41
|
`🛡️ Caught rendering error: ${error.message}\n${componentStack}`
|
|
41
42
|
);
|
|
42
43
|
this.props.onError?.(error, componentStack);
|
|
44
|
+
|
|
45
|
+
// Track the render error silently in analytics
|
|
46
|
+
if (this.props.telemetryRef?.current?.track) {
|
|
47
|
+
this.props.telemetryRef.current.track('render_error', {
|
|
48
|
+
message: error.message,
|
|
49
|
+
component: componentStack?.split('\n')[1]?.trim() ?? 'unknown',
|
|
50
|
+
screen: this.props.telemetryRef.current.screen,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
43
53
|
}
|
|
44
54
|
|
|
45
55
|
componentDidUpdate(_prevProps: Props, prevState: State): void {
|