@mobileai/react-native 0.9.18 → 0.9.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -20
- package/MobileAIFloatingOverlay.podspec +25 -0
- package/android/build.gradle +61 -0
- package/android/src/main/AndroidManifest.xml +3 -0
- package/android/src/main/java/com/mobileai/overlay/FloatingOverlayView.kt +151 -0
- package/android/src/main/java/com/mobileai/overlay/MobileAIOverlayPackage.kt +23 -0
- package/android/src/newarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +45 -0
- package/android/src/oldarch/com/mobileai/overlay/FloatingOverlayViewManager.kt +29 -0
- package/ios/MobileAIFloatingOverlayComponentView.mm +73 -0
- package/lib/module/components/AIAgent.js +902 -136
- package/lib/module/components/AIConsentDialog.js +439 -0
- package/lib/module/components/AgentChatBar.js +828 -134
- package/lib/module/components/AgentOverlay.js +2 -1
- package/lib/module/components/DiscoveryTooltip.js +21 -9
- package/lib/module/components/FloatingOverlayWrapper.js +108 -0
- package/lib/module/components/Icons.js +123 -0
- package/lib/module/config/endpoints.js +12 -2
- package/lib/module/core/AgentRuntime.js +373 -27
- package/lib/module/core/FiberAdapter.js +56 -0
- package/lib/module/core/FiberTreeWalker.js +186 -80
- package/lib/module/core/IdleDetector.js +19 -0
- package/lib/module/core/NativeAlertInterceptor.js +191 -0
- package/lib/module/core/systemPrompt.js +203 -45
- package/lib/module/index.js +3 -0
- package/lib/module/providers/GeminiProvider.js +72 -56
- package/lib/module/providers/ProviderFactory.js +6 -2
- package/lib/module/services/AudioInputService.js +3 -12
- package/lib/module/services/AudioOutputService.js +1 -13
- package/lib/module/services/ConversationService.js +166 -0
- package/lib/module/services/MobileAIKnowledgeRetriever.js +41 -0
- package/lib/module/services/VoiceService.js +29 -8
- package/lib/module/services/telemetry/MobileAI.js +44 -0
- package/lib/module/services/telemetry/TelemetryService.js +13 -1
- package/lib/module/services/telemetry/TouchAutoCapture.js +44 -18
- package/lib/module/specs/FloatingOverlayNativeComponent.ts +19 -0
- package/lib/module/support/CSATSurvey.js +95 -12
- package/lib/module/support/EscalationSocket.js +70 -1
- package/lib/module/support/ReportedIssueEventSource.js +148 -0
- package/lib/module/support/escalateTool.js +4 -2
- package/lib/module/support/index.js +1 -0
- package/lib/module/support/reportIssueTool.js +127 -0
- package/lib/module/support/supportPrompt.js +77 -9
- package/lib/module/tools/guideTool.js +2 -1
- package/lib/module/tools/longPressTool.js +4 -3
- package/lib/module/tools/pickerTool.js +6 -4
- package/lib/module/tools/tapTool.js +12 -3
- package/lib/module/tools/typeTool.js +19 -10
- package/lib/module/utils/logger.js +175 -6
- package/lib/typescript/react-native.config.d.ts +11 -0
- package/lib/typescript/src/components/AIAgent.d.ts +28 -2
- package/lib/typescript/src/components/AIConsentDialog.d.ts +153 -0
- package/lib/typescript/src/components/AgentChatBar.d.ts +15 -2
- package/lib/typescript/src/components/DiscoveryTooltip.d.ts +3 -1
- package/lib/typescript/src/components/FloatingOverlayWrapper.d.ts +51 -0
- package/lib/typescript/src/components/Icons.d.ts +8 -0
- package/lib/typescript/src/config/endpoints.d.ts +5 -3
- package/lib/typescript/src/core/AgentRuntime.d.ts +4 -0
- package/lib/typescript/src/core/FiberAdapter.d.ts +25 -0
- package/lib/typescript/src/core/FiberTreeWalker.d.ts +2 -0
- package/lib/typescript/src/core/IdleDetector.d.ts +11 -0
- package/lib/typescript/src/core/NativeAlertInterceptor.d.ts +55 -0
- package/lib/typescript/src/core/types.d.ts +106 -1
- package/lib/typescript/src/index.d.ts +9 -4
- package/lib/typescript/src/providers/GeminiProvider.d.ts +6 -5
- package/lib/typescript/src/services/ConversationService.d.ts +55 -0
- package/lib/typescript/src/services/MobileAIKnowledgeRetriever.d.ts +9 -0
- package/lib/typescript/src/services/telemetry/MobileAI.d.ts +7 -0
- package/lib/typescript/src/services/telemetry/TelemetryService.d.ts +1 -1
- package/lib/typescript/src/services/telemetry/TouchAutoCapture.d.ts +9 -6
- package/lib/typescript/src/services/telemetry/types.d.ts +3 -1
- package/lib/typescript/src/specs/FloatingOverlayNativeComponent.d.ts +17 -0
- package/lib/typescript/src/support/EscalationSocket.d.ts +17 -0
- package/lib/typescript/src/support/ReportedIssueEventSource.d.ts +24 -0
- package/lib/typescript/src/support/escalateTool.d.ts +5 -0
- package/lib/typescript/src/support/index.d.ts +2 -1
- package/lib/typescript/src/support/reportIssueTool.d.ts +20 -0
- package/lib/typescript/src/support/types.d.ts +56 -1
- package/lib/typescript/src/utils/logger.d.ts +15 -0
- package/package.json +20 -5
- package/react-native.config.js +12 -0
- package/src/specs/FloatingOverlayNativeComponent.ts +19 -0
|
@@ -12,18 +12,41 @@
|
|
|
12
12
|
* with OpenAIProvider, AnthropicProvider, etc.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
import { GoogleGenAI, FunctionCallingConfigMode, Type } from '@google/genai';
|
|
16
15
|
import { logger } from "../utils/logger.js";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Lazy-loads @google/genai on first call.
|
|
19
|
+
* Using require() instead of a static import allows:
|
|
20
|
+
* 1. Older Metro bundlers (RN 0.72-0.73) to bundle the library without
|
|
21
|
+
* choking on the SDK's ESM sub-path exports.
|
|
22
|
+
* 2. Users who pick OpenAI to never pay the SDK startup cost.
|
|
23
|
+
*
|
|
24
|
+
* NOTE: We do NOT cache the result in module-scope variables because that
|
|
25
|
+
* would break Jest's mock isolation — jest.mock() replaces the module in the
|
|
26
|
+
* require registry, so every call to require() in a test sees the mock.
|
|
27
|
+
* Node's own require() cache handles de-duplication in production.
|
|
28
|
+
*/
|
|
29
|
+
function loadGenAI() {
|
|
30
|
+
try {
|
|
31
|
+
const mod = require('@google/genai');
|
|
32
|
+
return {
|
|
33
|
+
GoogleGenAI: mod.GoogleGenAI,
|
|
34
|
+
FunctionCallingConfigMode: mod.FunctionCallingConfigMode,
|
|
35
|
+
Type: mod.Type
|
|
36
|
+
};
|
|
37
|
+
} catch (e) {
|
|
38
|
+
throw new Error('[mobileai] @google/genai is required for the Gemini provider. ' + 'Install it: npm install @google/genai');
|
|
39
|
+
}
|
|
40
|
+
}
|
|
17
41
|
// ─── Constants ─────────────────────────────────────────────────
|
|
18
42
|
|
|
19
43
|
const AGENT_STEP_FN = 'agent_step';
|
|
20
44
|
|
|
21
|
-
// Reasoning fields always present in the agent_step schema
|
|
22
|
-
const REASONING_FIELDS = ['previous_goal_eval', 'memory', 'plan'];
|
|
23
|
-
|
|
24
45
|
// ─── Provider ──────────────────────────────────────────────────
|
|
25
46
|
|
|
26
47
|
export class GeminiProvider {
|
|
48
|
+
// GoogleGenAI instance, loaded lazily
|
|
49
|
+
|
|
27
50
|
constructor(apiKey, model = 'gemini-2.5-flash', proxyUrl, proxyHeaders) {
|
|
28
51
|
const config = {};
|
|
29
52
|
if (proxyUrl) {
|
|
@@ -37,6 +60,9 @@ export class GeminiProvider {
|
|
|
37
60
|
} else {
|
|
38
61
|
throw new Error('[mobileai] You must provide either "apiKey" or "proxyUrl" to AIAgent.');
|
|
39
62
|
}
|
|
63
|
+
const {
|
|
64
|
+
GoogleGenAI
|
|
65
|
+
} = loadGenAI();
|
|
40
66
|
this.ai = new GoogleGenAI(config);
|
|
41
67
|
this.model = model;
|
|
42
68
|
}
|
|
@@ -60,7 +86,7 @@ export class GeminiProvider {
|
|
|
60
86
|
}],
|
|
61
87
|
toolConfig: {
|
|
62
88
|
functionCallingConfig: {
|
|
63
|
-
mode: FunctionCallingConfigMode.ANY,
|
|
89
|
+
mode: loadGenAI().FunctionCallingConfigMode.ANY,
|
|
64
90
|
allowedFunctionNames: [AGENT_STEP_FN]
|
|
65
91
|
}
|
|
66
92
|
},
|
|
@@ -91,36 +117,27 @@ export class GeminiProvider {
|
|
|
91
117
|
// ─── Build agent_step Declaration ──────────────────────────
|
|
92
118
|
|
|
93
119
|
/**
|
|
94
|
-
* Builds a single `agent_step` function declaration that
|
|
95
|
-
*
|
|
120
|
+
* Builds a single `agent_step` function declaration that keeps Gemini's
|
|
121
|
+
* served schema intentionally narrow:
|
|
122
|
+
* - Structured reasoning fields
|
|
96
123
|
* - action_name (enum of all available tool names)
|
|
97
|
-
* -
|
|
124
|
+
* - action_input as a JSON object string
|
|
98
125
|
*
|
|
99
|
-
*
|
|
126
|
+
* Flattening every tool parameter into top-level properties can trigger
|
|
127
|
+
* Gemini's "too much branching for serving" error once the toolset grows.
|
|
100
128
|
*/
|
|
101
129
|
buildAgentStepDeclaration(tools) {
|
|
102
130
|
const toolNames = tools.map(t => t.name);
|
|
103
131
|
|
|
104
|
-
// Collect all unique parameter fields across all tools
|
|
105
|
-
const actionProperties = {};
|
|
106
|
-
for (const tool of tools) {
|
|
107
|
-
for (const [paramName, param] of Object.entries(tool.parameters)) {
|
|
108
|
-
if (actionProperties[paramName]) continue;
|
|
109
|
-
actionProperties[paramName] = {
|
|
110
|
-
type: this.mapParamType(param.type),
|
|
111
|
-
description: param.description,
|
|
112
|
-
...(param.enum ? {
|
|
113
|
-
enum: param.enum
|
|
114
|
-
} : {})
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
|
|
119
132
|
// Build tool descriptions for the action_name enum
|
|
120
133
|
const toolDescriptions = tools.map(t => {
|
|
121
|
-
const params = Object.keys(t.parameters)
|
|
122
|
-
|
|
134
|
+
const params = Object.keys(t.parameters);
|
|
135
|
+
const inputGuide = params.length === 0 ? 'Use {} for action_input.' : `Provide action_input as a JSON object string with keys: ${params.join(', ')}.`;
|
|
136
|
+
return `- ${t.name}: ${t.description} ${inputGuide}`;
|
|
123
137
|
}).join('\n');
|
|
138
|
+
const {
|
|
139
|
+
Type
|
|
140
|
+
} = loadGenAI();
|
|
124
141
|
return {
|
|
125
142
|
name: AGENT_STEP_FN,
|
|
126
143
|
description: `Execute one agent step. Choose an action and provide reasoning.\n\nAvailable actions:\n${toolDescriptions}`,
|
|
@@ -144,25 +161,15 @@ export class GeminiProvider {
|
|
|
144
161
|
description: 'Which action to execute.',
|
|
145
162
|
enum: toolNames
|
|
146
163
|
},
|
|
147
|
-
|
|
164
|
+
action_input: {
|
|
165
|
+
type: Type.STRING,
|
|
166
|
+
description: 'JSON object string containing only the arguments for action_name. Use "{}" when the action takes no parameters.'
|
|
167
|
+
}
|
|
148
168
|
},
|
|
149
169
|
required: ['plan', 'action_name']
|
|
150
170
|
}
|
|
151
171
|
};
|
|
152
172
|
}
|
|
153
|
-
mapParamType(type) {
|
|
154
|
-
switch (type) {
|
|
155
|
-
case 'number':
|
|
156
|
-
return Type.NUMBER;
|
|
157
|
-
case 'integer':
|
|
158
|
-
return Type.INTEGER;
|
|
159
|
-
case 'boolean':
|
|
160
|
-
return Type.BOOLEAN;
|
|
161
|
-
case 'string':
|
|
162
|
-
default:
|
|
163
|
-
return Type.STRING;
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
173
|
|
|
167
174
|
// ─── Build Contents ────────────────────────────────────────
|
|
168
175
|
|
|
@@ -265,23 +272,23 @@ export class GeminiProvider {
|
|
|
265
272
|
text: textPart?.text
|
|
266
273
|
};
|
|
267
274
|
}
|
|
268
|
-
|
|
269
|
-
// Build action args: extract only the params that belong to the matched tool
|
|
270
|
-
const actionArgs = {};
|
|
271
|
-
const reservedKeys = new Set([...REASONING_FIELDS, 'action_name']);
|
|
272
275
|
const matchedTool = tools.find(t => t.name === actionName);
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
276
|
+
let actionArgs = {};
|
|
277
|
+
const rawActionInput = args.action_input;
|
|
278
|
+
if (typeof rawActionInput === 'string' && rawActionInput.trim().length > 0) {
|
|
279
|
+
try {
|
|
280
|
+
const parsed = JSON.parse(rawActionInput);
|
|
281
|
+
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
282
|
+
actionArgs = parsed;
|
|
277
283
|
}
|
|
284
|
+
} catch (error) {
|
|
285
|
+
logger.warn('GeminiProvider', `Invalid action_input JSON for ${actionName}: ${error.message}`);
|
|
278
286
|
}
|
|
287
|
+
}
|
|
288
|
+
if (matchedTool) {
|
|
289
|
+
actionArgs = Object.fromEntries(Object.entries(actionArgs).filter(([key]) => key in matchedTool.parameters));
|
|
279
290
|
} else {
|
|
280
|
-
|
|
281
|
-
if (!reservedKeys.has(key)) {
|
|
282
|
-
actionArgs[key] = value;
|
|
283
|
-
}
|
|
284
|
-
}
|
|
291
|
+
actionArgs = {};
|
|
285
292
|
}
|
|
286
293
|
logger.info('GeminiProvider', `Parsed: action=${actionName}, plan="${reasoning.plan}"`);
|
|
287
294
|
return {
|
|
@@ -311,8 +318,8 @@ export class GeminiProvider {
|
|
|
311
318
|
const totalTokens = meta.totalTokenCount ?? promptTokens + completionTokens;
|
|
312
319
|
|
|
313
320
|
// Cost estimation based on Gemini 2.5 Flash pricing
|
|
314
|
-
const INPUT_COST_PER_M = 0.
|
|
315
|
-
const OUTPUT_COST_PER_M = 2.
|
|
321
|
+
const INPUT_COST_PER_M = 0.3;
|
|
322
|
+
const OUTPUT_COST_PER_M = 2.5;
|
|
316
323
|
const estimatedCostUSD = promptTokens / 1_000_000 * INPUT_COST_PER_M + completionTokens / 1_000_000 * OUTPUT_COST_PER_M;
|
|
317
324
|
return {
|
|
318
325
|
promptTokens,
|
|
@@ -331,9 +338,11 @@ export class GeminiProvider {
|
|
|
331
338
|
formatProviderError(status, rawMessage) {
|
|
332
339
|
// Try to extract the human-readable message from JSON body
|
|
333
340
|
let humanMessage = '';
|
|
341
|
+
let errorCode = '';
|
|
334
342
|
try {
|
|
335
343
|
const parsed = JSON.parse(rawMessage);
|
|
336
344
|
humanMessage = parsed?.error?.message || parsed?.message || '';
|
|
345
|
+
errorCode = parsed?.error?.code || parsed?.code || '';
|
|
337
346
|
} catch {
|
|
338
347
|
// rawMessage may contain JSON embedded in a string like "503: {json}"
|
|
339
348
|
const jsonMatch = rawMessage.match(/\{[\s\S]*\}/);
|
|
@@ -341,9 +350,16 @@ export class GeminiProvider {
|
|
|
341
350
|
try {
|
|
342
351
|
const parsed = JSON.parse(jsonMatch[0]);
|
|
343
352
|
humanMessage = parsed?.error?.message || parsed?.message || '';
|
|
344
|
-
|
|
353
|
+
errorCode = parsed?.error?.code || parsed?.code || '';
|
|
354
|
+
} catch {
|
|
355
|
+
/* ignore */
|
|
356
|
+
}
|
|
345
357
|
}
|
|
346
358
|
}
|
|
359
|
+
if (errorCode === 'proxy_blocked') {
|
|
360
|
+
logger.error('GeminiProvider', 'Proxy blocked: Credit limit reached or budget exhausted.');
|
|
361
|
+
return 'The AI assistant is temporarily unavailable. Please try again later.';
|
|
362
|
+
}
|
|
347
363
|
|
|
348
364
|
// Map status codes to friendly descriptions
|
|
349
365
|
switch (status) {
|
|
@@ -7,7 +7,6 @@
|
|
|
7
7
|
* know about individual provider implementations.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import { GeminiProvider } from "./GeminiProvider.js";
|
|
11
10
|
import { OpenAIProvider } from "./OpenAIProvider.js";
|
|
12
11
|
export function createProvider(provider = 'gemini', apiKey, model, proxyUrl, proxyHeaders) {
|
|
13
12
|
switch (provider) {
|
|
@@ -15,7 +14,12 @@ export function createProvider(provider = 'gemini', apiKey, model, proxyUrl, pro
|
|
|
15
14
|
return new OpenAIProvider(apiKey, model || 'gpt-4.1-mini', proxyUrl, proxyHeaders);
|
|
16
15
|
case 'gemini':
|
|
17
16
|
default:
|
|
18
|
-
|
|
17
|
+
{
|
|
18
|
+
const {
|
|
19
|
+
GeminiProvider
|
|
20
|
+
} = require('./GeminiProvider');
|
|
21
|
+
return new GeminiProvider(apiKey, model || 'gemini-2.5-flash', proxyUrl, proxyHeaders);
|
|
22
|
+
}
|
|
19
23
|
}
|
|
20
24
|
}
|
|
21
25
|
//# sourceMappingURL=ProviderFactory.js.map
|
|
@@ -42,20 +42,11 @@ export class AudioInputService {
|
|
|
42
42
|
// Lazy-load react-native-audio-api (optional peer dependency)
|
|
43
43
|
let audioApi;
|
|
44
44
|
try {
|
|
45
|
-
const {
|
|
46
|
-
NativeModules
|
|
47
|
-
} = require('react-native');
|
|
48
|
-
if (!NativeModules.AudioApiModule) {
|
|
49
|
-
const msg = '[mobileai] react-native-audio-api native module not found. ' + 'Voice mode requires a development build (not Expo Go).';
|
|
50
|
-
logger.warn('AudioInput', msg);
|
|
51
|
-
this.config.onError?.(msg);
|
|
52
|
-
return false;
|
|
53
|
-
}
|
|
54
45
|
// Static require — Metro needs a literal string for bundling.
|
|
55
46
|
audioApi = require('react-native-audio-api');
|
|
56
47
|
} catch {
|
|
57
48
|
const msg = 'Voice mode requires react-native-audio-api. Install with: npm install react-native-audio-api';
|
|
58
|
-
logger.
|
|
49
|
+
logger.warn('AudioInput', msg);
|
|
59
50
|
this.config.onError?.(msg);
|
|
60
51
|
return false;
|
|
61
52
|
}
|
|
@@ -63,8 +54,8 @@ export class AudioInputService {
|
|
|
63
54
|
// Request mic permission (Android)
|
|
64
55
|
try {
|
|
65
56
|
const {
|
|
66
|
-
|
|
67
|
-
|
|
57
|
+
Platform,
|
|
58
|
+
PermissionsAndroid
|
|
68
59
|
} = require('react-native');
|
|
69
60
|
if (Platform.OS === 'android') {
|
|
70
61
|
const result = await PermissionsAndroid.request(PermissionsAndroid.PERMISSIONS.RECORD_AUDIO);
|
|
@@ -12,8 +12,6 @@
|
|
|
12
12
|
|
|
13
13
|
import { logger } from "../utils/logger.js";
|
|
14
14
|
import { base64ToFloat32 } from "../utils/audioUtils.js";
|
|
15
|
-
import { NativeModules } from 'react-native';
|
|
16
|
-
|
|
17
15
|
// ─── Types ─────────────────────────────────────────────────────
|
|
18
16
|
|
|
19
17
|
/** Gemini Live API outputs 24kHz 16-bit mono PCM */
|
|
@@ -37,21 +35,11 @@ export class AudioOutputService {
|
|
|
37
35
|
try {
|
|
38
36
|
let audioApi;
|
|
39
37
|
try {
|
|
40
|
-
// Guard: NativeModules.AudioApiModule is only present in dev/prod builds.
|
|
41
|
-
// In Expo Go it is undefined, and require() throws a native bridge error
|
|
42
|
-
// that cannot be caught by a standard try/catch.
|
|
43
|
-
if (!NativeModules.AudioApiModule) {
|
|
44
|
-
const msg = '[mobileai] react-native-audio-api native module not found. ' + 'Voice audio output requires a development build (not Expo Go). ' + 'Run: npx expo run:ios';
|
|
45
|
-
logger.warn('AudioOutput', msg);
|
|
46
|
-
this.config.onError?.(msg);
|
|
47
|
-
return false;
|
|
48
|
-
}
|
|
49
38
|
// Static require — Metro needs a literal string.
|
|
50
|
-
// The NativeModules guard above already prevents this from running in Expo Go.
|
|
51
39
|
audioApi = require('react-native-audio-api');
|
|
52
40
|
} catch {
|
|
53
41
|
const msg = 'react-native-audio-api is required for audio output. Install with: npm install react-native-audio-api';
|
|
54
|
-
logger.
|
|
42
|
+
logger.warn('AudioOutput', msg);
|
|
55
43
|
this.config.onError?.(msg);
|
|
56
44
|
return false;
|
|
57
45
|
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* ConversationService — backend-persisted AI conversation history.
|
|
5
|
+
*
|
|
6
|
+
* Saves and retrieves AI chat sessions from the MobileAI backend so users
|
|
7
|
+
* can browse and continue previous conversations across app launches.
|
|
8
|
+
*
|
|
9
|
+
* All methods are no-ops when analyticsKey is absent (graceful degradation).
|
|
10
|
+
* All network errors are silently swallowed — history is best-effort and must
|
|
11
|
+
* never break the core agent flow.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { ENDPOINTS } from "../config/endpoints.js";
|
|
15
|
+
import { logger } from "../utils/logger.js";
|
|
16
|
+
|
|
17
|
+
// ─── Types ───────────────────────────────────────────────────────
|
|
18
|
+
|
|
19
|
+
// ─── Serialization ───────────────────────────────────────────────
|
|
20
|
+
|
|
21
|
+
function toPayload(msgs) {
|
|
22
|
+
return msgs.filter(m => m.role === 'user' || m.role === 'assistant').filter(m => typeof m.content === 'string' && m.content.trim().length > 0).map(m => ({
|
|
23
|
+
role: m.role,
|
|
24
|
+
content: typeof m.content === 'string' ? m.content : String(m.content),
|
|
25
|
+
timestamp: m.timestamp
|
|
26
|
+
}));
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// ─── Service ─────────────────────────────────────────────────────
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Start a new conversation on the backend.
|
|
33
|
+
* Call this when the first AI response arrives in a new session.
|
|
34
|
+
* Returns the backend conversationId, or null on failure.
|
|
35
|
+
*/
|
|
36
|
+
export async function startConversation({
|
|
37
|
+
analyticsKey,
|
|
38
|
+
userId,
|
|
39
|
+
deviceId,
|
|
40
|
+
messages
|
|
41
|
+
}) {
|
|
42
|
+
if (!analyticsKey) return null;
|
|
43
|
+
const payload = toPayload(messages);
|
|
44
|
+
if (!payload.length) return null;
|
|
45
|
+
try {
|
|
46
|
+
const res = await fetch(ENDPOINTS.conversations, {
|
|
47
|
+
method: 'POST',
|
|
48
|
+
headers: {
|
|
49
|
+
'Content-Type': 'application/json'
|
|
50
|
+
},
|
|
51
|
+
body: JSON.stringify({
|
|
52
|
+
analyticsKey,
|
|
53
|
+
userId: userId || undefined,
|
|
54
|
+
deviceId: deviceId || undefined,
|
|
55
|
+
messages: payload
|
|
56
|
+
})
|
|
57
|
+
});
|
|
58
|
+
if (!res.ok) {
|
|
59
|
+
logger.warn('ConversationService', `startConversation failed: ${res.status}`);
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
const data = await res.json();
|
|
63
|
+
logger.info('ConversationService', `Started conversation: ${data.conversationId}`);
|
|
64
|
+
return data.conversationId;
|
|
65
|
+
} catch (err) {
|
|
66
|
+
logger.warn('ConversationService', `startConversation error: ${err}`);
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Append new messages to an existing conversation.
|
|
73
|
+
* Fire-and-forget — call after each exchange (debounce in caller).
|
|
74
|
+
*/
|
|
75
|
+
export async function appendMessages({
|
|
76
|
+
conversationId,
|
|
77
|
+
analyticsKey,
|
|
78
|
+
messages
|
|
79
|
+
}) {
|
|
80
|
+
if (!analyticsKey || !conversationId) return;
|
|
81
|
+
const payload = toPayload(messages);
|
|
82
|
+
if (!payload.length) return;
|
|
83
|
+
try {
|
|
84
|
+
const res = await fetch(ENDPOINTS.conversations, {
|
|
85
|
+
method: 'POST',
|
|
86
|
+
headers: {
|
|
87
|
+
'Content-Type': 'application/json'
|
|
88
|
+
},
|
|
89
|
+
body: JSON.stringify({
|
|
90
|
+
analyticsKey,
|
|
91
|
+
conversationId,
|
|
92
|
+
messages: payload
|
|
93
|
+
})
|
|
94
|
+
});
|
|
95
|
+
if (!res.ok) {
|
|
96
|
+
logger.warn('ConversationService', `appendMessages failed: ${res.status}`);
|
|
97
|
+
}
|
|
98
|
+
} catch (err) {
|
|
99
|
+
logger.warn('ConversationService', `appendMessages error: ${err}`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Fetch the user's conversation list (most-recent-first).
|
|
105
|
+
* Returns empty array on failure — never throws.
|
|
106
|
+
*/
|
|
107
|
+
export async function fetchConversations({
|
|
108
|
+
analyticsKey,
|
|
109
|
+
userId,
|
|
110
|
+
deviceId,
|
|
111
|
+
limit = 20
|
|
112
|
+
}) {
|
|
113
|
+
if (!analyticsKey) return [];
|
|
114
|
+
if (!userId && !deviceId) return [];
|
|
115
|
+
try {
|
|
116
|
+
const params = new URLSearchParams({
|
|
117
|
+
analyticsKey,
|
|
118
|
+
limit: String(limit)
|
|
119
|
+
});
|
|
120
|
+
if (userId) params.set('userId', userId);
|
|
121
|
+
if (deviceId) params.set('deviceId', deviceId);
|
|
122
|
+
const res = await fetch(`${ENDPOINTS.conversations}?${params.toString()}`);
|
|
123
|
+
if (!res.ok) {
|
|
124
|
+
logger.warn('ConversationService', `fetchConversations failed: ${res.status}`);
|
|
125
|
+
return [];
|
|
126
|
+
}
|
|
127
|
+
const data = await res.json();
|
|
128
|
+
return data.conversations ?? [];
|
|
129
|
+
} catch (err) {
|
|
130
|
+
logger.warn('ConversationService', `fetchConversations error: ${err}`);
|
|
131
|
+
return [];
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Fetch the full message history of a single conversation.
|
|
137
|
+
* Returns null on failure.
|
|
138
|
+
*/
|
|
139
|
+
export async function fetchConversation({
|
|
140
|
+
conversationId,
|
|
141
|
+
analyticsKey
|
|
142
|
+
}) {
|
|
143
|
+
if (!analyticsKey || !conversationId) return null;
|
|
144
|
+
try {
|
|
145
|
+
const params = new URLSearchParams({
|
|
146
|
+
analyticsKey
|
|
147
|
+
});
|
|
148
|
+
const res = await fetch(`${ENDPOINTS.conversations}/${conversationId}?${params.toString()}`);
|
|
149
|
+
if (!res.ok) {
|
|
150
|
+
logger.warn('ConversationService', `fetchConversation failed: ${res.status}`);
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
const data = await res.json();
|
|
154
|
+
const msgs = (data.messages ?? []).map(m => ({
|
|
155
|
+
id: m.id,
|
|
156
|
+
role: m.role,
|
|
157
|
+
content: m.content,
|
|
158
|
+
timestamp: m.timestamp
|
|
159
|
+
}));
|
|
160
|
+
return msgs;
|
|
161
|
+
} catch (err) {
|
|
162
|
+
logger.warn('ConversationService', `fetchConversation error: ${err}`);
|
|
163
|
+
return null;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
//# sourceMappingURL=ConversationService.js.map
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
import { logger } from "../utils/logger.js";
|
|
4
|
+
function normalizeBaseUrl(baseUrl) {
|
|
5
|
+
if (!baseUrl) return 'http://localhost:3001';
|
|
6
|
+
const trimmed = baseUrl.replace(/\/$/, '');
|
|
7
|
+
return trimmed.endsWith('/api/v1/analytics') ? trimmed.replace(/\/api\/v1\/analytics$/, '') : trimmed;
|
|
8
|
+
}
|
|
9
|
+
export function createMobileAIKnowledgeRetriever(options) {
|
|
10
|
+
const baseUrl = normalizeBaseUrl(options.baseUrl);
|
|
11
|
+
const url = `${baseUrl}/api/v1/knowledge/query`;
|
|
12
|
+
return {
|
|
13
|
+
async retrieve(query, screenName) {
|
|
14
|
+
try {
|
|
15
|
+
const response = await fetch(url, {
|
|
16
|
+
method: 'POST',
|
|
17
|
+
headers: {
|
|
18
|
+
'Content-Type': 'application/json',
|
|
19
|
+
Authorization: `Bearer ${options.publishableKey}`,
|
|
20
|
+
...(options.headers ?? {})
|
|
21
|
+
},
|
|
22
|
+
body: JSON.stringify({
|
|
23
|
+
query,
|
|
24
|
+
screenName,
|
|
25
|
+
limit: options.limit
|
|
26
|
+
})
|
|
27
|
+
});
|
|
28
|
+
if (!response.ok) {
|
|
29
|
+
logger.warn('MobileAIKnowledge', `Knowledge query failed: HTTP ${response.status}`);
|
|
30
|
+
return [];
|
|
31
|
+
}
|
|
32
|
+
const payload = await response.json();
|
|
33
|
+
return Array.isArray(payload?.entries) ? payload.entries : [];
|
|
34
|
+
} catch (error) {
|
|
35
|
+
logger.error('MobileAIKnowledge', `Knowledge query failed: ${error?.message ?? 'unknown error'}`);
|
|
36
|
+
return [];
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=MobileAIKnowledgeRetriever.js.map
|
|
@@ -15,14 +15,21 @@
|
|
|
15
15
|
* - Sends screen context (DOM text) for live mode
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
+
// @ts-ignore — TS can't find declarations for the deep path at build time
|
|
18
19
|
// Platform-specific import: Metro can't resolve '@google/genai/web' sub-path
|
|
19
|
-
//
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
20
|
+
// so we use the full path to the web bundle (works because RN's WebSocket = browser API).
|
|
21
|
+
|
|
22
|
+
function loadVoiceGenAI() {
|
|
23
|
+
try {
|
|
24
|
+
const mod = require('@google/genai/dist/web/index.mjs');
|
|
25
|
+
return {
|
|
26
|
+
GoogleGenAI: mod.GoogleGenAI,
|
|
27
|
+
Modality: mod.Modality
|
|
28
|
+
};
|
|
29
|
+
} catch (e) {
|
|
30
|
+
throw new Error('[mobileai] @google/genai is required for Voice Mode. ' + 'Install it: npm install @google/genai');
|
|
31
|
+
}
|
|
32
|
+
}
|
|
26
33
|
import { logger } from "../utils/logger.js";
|
|
27
34
|
|
|
28
35
|
// ─── Types ─────────────────────────────────────────────────────
|
|
@@ -64,7 +71,17 @@ export class VoiceService {
|
|
|
64
71
|
try {
|
|
65
72
|
const genAiConfig = {};
|
|
66
73
|
if (this.config.proxyUrl) {
|
|
67
|
-
|
|
74
|
+
// The @google/genai SDK sends apiKey as ?key=<value> in the WebSocket URL.
|
|
75
|
+
// For HTTP text proxy, the Authorization header carries the secret key.
|
|
76
|
+
// For WebSocket voice proxy, browser WS APIs don't support custom headers —
|
|
77
|
+
// the only way to pass auth is via the URL query string.
|
|
78
|
+
// So we extract the real secret key from proxyHeaders and use it as apiKey,
|
|
79
|
+
// which the SDK will append as ?key=<secret> in the WS URL.
|
|
80
|
+
// Our proxy reads it there, validates it, then replaces it with the real
|
|
81
|
+
// Gemini API key before forwarding upstream.
|
|
82
|
+
const authHeader = this.config.proxyHeaders?.['Authorization'] ?? this.config.proxyHeaders?.['authorization'];
|
|
83
|
+
const secretKey = authHeader?.startsWith('Bearer ') ? authHeader.replace('Bearer ', '').trim() : authHeader ?? 'proxy-key';
|
|
84
|
+
genAiConfig.apiKey = secretKey;
|
|
68
85
|
genAiConfig.httpOptions = {
|
|
69
86
|
baseUrl: this.config.proxyUrl,
|
|
70
87
|
headers: this.config.proxyHeaders || {}
|
|
@@ -74,6 +91,10 @@ export class VoiceService {
|
|
|
74
91
|
} else {
|
|
75
92
|
throw new Error('[mobileai] Must provide apiKey or proxyUrl');
|
|
76
93
|
}
|
|
94
|
+
const {
|
|
95
|
+
GoogleGenAI,
|
|
96
|
+
Modality
|
|
97
|
+
} = loadVoiceGenAI();
|
|
77
98
|
const ai = new GoogleGenAI(genAiConfig);
|
|
78
99
|
const toolDeclarations = this.buildToolDeclarations();
|
|
79
100
|
|
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
14
|
import { logger } from "../../utils/logger.js";
|
|
15
|
+
import { getDeviceId } from "./device.js";
|
|
16
|
+
import { ENDPOINTS } from "../../config/endpoints.js";
|
|
15
17
|
const LOG_TAG = 'MobileAI';
|
|
16
18
|
let service = null;
|
|
17
19
|
export const MobileAI = {
|
|
@@ -53,6 +55,48 @@ export const MobileAI = {
|
|
|
53
55
|
return defaultValue ?? '';
|
|
54
56
|
}
|
|
55
57
|
return service.flags.getFlag(key, defaultValue);
|
|
58
|
+
},
|
|
59
|
+
/**
|
|
60
|
+
* Helper function to securely consume a global WOW action limit (like a discount)
|
|
61
|
+
* natively on the MobileAI Server to prevent prompt injection bypasses.
|
|
62
|
+
* @param actionName - The exact registered name of the WOW action
|
|
63
|
+
* @returns true if allowed, false if rejected or error
|
|
64
|
+
*/
|
|
65
|
+
async consumeWowAction(actionName) {
|
|
66
|
+
if (!service || !service.config.analyticsKey) {
|
|
67
|
+
logger.warn(LOG_TAG, 'consumeWowAction failed: SDK not initialized with analyticsKey or publishableKey in AIAgent');
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
try {
|
|
71
|
+
const baseUrl = (service.config.analyticsProxyUrl ?? ENDPOINTS.escalation).replace(/\/$/, '').replace(/\/api\/v1\/analytics$/, '');
|
|
72
|
+
const url = `${baseUrl}/api/v1/wow-actions/consume`;
|
|
73
|
+
const deviceId = getDeviceId() ?? 'unknown';
|
|
74
|
+
const res = await fetch(url, {
|
|
75
|
+
method: 'POST',
|
|
76
|
+
headers: {
|
|
77
|
+
'Content-Type': 'application/json',
|
|
78
|
+
'Authorization': `Bearer ${service.config.analyticsKey}`,
|
|
79
|
+
...(service.config.analyticsProxyHeaders ?? {})
|
|
80
|
+
},
|
|
81
|
+
body: JSON.stringify({
|
|
82
|
+
actionName,
|
|
83
|
+
deviceId
|
|
84
|
+
})
|
|
85
|
+
});
|
|
86
|
+
if (!res.ok) {
|
|
87
|
+
if (res.status === 429) {
|
|
88
|
+
logger.info(LOG_TAG, `consumeWowAction denied: Global limit reached for '${actionName}'`);
|
|
89
|
+
} else {
|
|
90
|
+
logger.warn(LOG_TAG, `consumeWowAction failed: HTTP ${res.status}`);
|
|
91
|
+
}
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
const data = await res.json();
|
|
95
|
+
return data.allowed === true;
|
|
96
|
+
} catch (e) {
|
|
97
|
+
logger.error(LOG_TAG, `consumeWowAction network error: ${e.message}`);
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
56
100
|
}
|
|
57
101
|
};
|
|
58
102
|
|
|
@@ -184,6 +184,9 @@ export class TelemetryService {
|
|
|
184
184
|
sessionId: this.sessionId
|
|
185
185
|
};
|
|
186
186
|
this.queue.push(event);
|
|
187
|
+
if (this.config.onEvent) {
|
|
188
|
+
this.config.onEvent(event);
|
|
189
|
+
}
|
|
187
190
|
if (this.config.debug) {
|
|
188
191
|
logger.debug(LOG_TAG, `→ ${type}`, data);
|
|
189
192
|
}
|
|
@@ -212,7 +215,16 @@ export class TelemetryService {
|
|
|
212
215
|
|
|
213
216
|
/** Send queued events to the cloud API */
|
|
214
217
|
async flush() {
|
|
215
|
-
if (!this.isEnabled() || this.
|
|
218
|
+
if (!this.isEnabled() || this.isFlushing) return;
|
|
219
|
+
|
|
220
|
+
// Extract any pending SDK debug logs to sink them natively to the backend
|
|
221
|
+
const unflushedLogs = logger.extractUnflushedLines();
|
|
222
|
+
if (unflushedLogs.length > 0) {
|
|
223
|
+
this.track('sdk_trace_dump', {
|
|
224
|
+
logs: unflushedLogs.join('\n')
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
if (this.queue.length === 0) return;
|
|
216
228
|
this.isFlushing = true;
|
|
217
229
|
const eventsToSend = [...this.queue];
|
|
218
230
|
this.queue = [];
|