@mobileai/react-native 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -0
- package/README.md +190 -0
- package/lib/module/components/AIAgent.js +149 -0
- package/lib/module/components/AIAgent.js.map +1 -0
- package/lib/module/components/AgentChatBar.js +120 -0
- package/lib/module/components/AgentChatBar.js.map +1 -0
- package/lib/module/components/AgentOverlay.js +53 -0
- package/lib/module/components/AgentOverlay.js.map +1 -0
- package/lib/module/core/AgentRuntime.js +498 -0
- package/lib/module/core/AgentRuntime.js.map +1 -0
- package/lib/module/core/FiberTreeWalker.js +308 -0
- package/lib/module/core/FiberTreeWalker.js.map +1 -0
- package/lib/module/core/MCPBridge.js +98 -0
- package/lib/module/core/MCPBridge.js.map +1 -0
- package/lib/module/core/ScreenDehydrator.js +46 -0
- package/lib/module/core/ScreenDehydrator.js.map +1 -0
- package/lib/module/core/types.js +2 -0
- package/lib/module/core/types.js.map +1 -0
- package/lib/module/hooks/useAction.js +32 -0
- package/lib/module/hooks/useAction.js.map +1 -0
- package/lib/module/index.js +17 -0
- package/lib/module/index.js.map +1 -0
- package/lib/module/package.json +1 -0
- package/lib/module/providers/GeminiProvider.js +178 -0
- package/lib/module/providers/GeminiProvider.js.map +1 -0
- package/lib/module/utils/logger.js +17 -0
- package/lib/module/utils/logger.js.map +1 -0
- package/lib/typescript/package.json +1 -0
- package/lib/typescript/src/components/AIAgent.d.ts +57 -0
- package/lib/typescript/src/components/AIAgent.d.ts.map +1 -0
- package/lib/typescript/src/components/AgentChatBar.d.ts +14 -0
- package/lib/typescript/src/components/AgentChatBar.d.ts.map +1 -0
- package/lib/typescript/src/components/AgentOverlay.d.ts +10 -0
- package/lib/typescript/src/components/AgentOverlay.d.ts.map +1 -0
- package/lib/typescript/src/core/AgentRuntime.d.ts +37 -0
- package/lib/typescript/src/core/AgentRuntime.d.ts.map +1 -0
- package/lib/typescript/src/core/FiberTreeWalker.d.ts +26 -0
- package/lib/typescript/src/core/FiberTreeWalker.d.ts.map +1 -0
- package/lib/typescript/src/core/MCPBridge.d.ts +23 -0
- package/lib/typescript/src/core/MCPBridge.d.ts.map +1 -0
- package/lib/typescript/src/core/ScreenDehydrator.d.ts +20 -0
- package/lib/typescript/src/core/ScreenDehydrator.d.ts.map +1 -0
- package/lib/typescript/src/core/types.d.ts +138 -0
- package/lib/typescript/src/core/types.d.ts.map +1 -0
- package/lib/typescript/src/hooks/useAction.d.ts +13 -0
- package/lib/typescript/src/hooks/useAction.d.ts.map +1 -0
- package/lib/typescript/src/index.d.ts +10 -0
- package/lib/typescript/src/index.d.ts.map +1 -0
- package/lib/typescript/src/providers/GeminiProvider.d.ts +23 -0
- package/lib/typescript/src/providers/GeminiProvider.d.ts.map +1 -0
- package/lib/typescript/src/utils/logger.d.ts +7 -0
- package/lib/typescript/src/utils/logger.d.ts.map +1 -0
- package/package.json +143 -0
- package/src/components/AIAgent.tsx +222 -0
- package/src/components/AgentChatBar.tsx +136 -0
- package/src/components/AgentOverlay.tsx +48 -0
- package/src/core/AgentRuntime.ts +505 -0
- package/src/core/FiberTreeWalker.ts +349 -0
- package/src/core/MCPBridge.ts +110 -0
- package/src/core/ScreenDehydrator.ts +53 -0
- package/src/core/types.ts +185 -0
- package/src/hooks/useAction.ts +40 -0
- package/src/index.ts +22 -0
- package/src/providers/GeminiProvider.ts +210 -0
- package/src/utils/logger.ts +21 -0
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* GeminiProvider — Simplified Gemini API integration.
|
|
5
|
+
* Sends dehydrated screen state + tools to Gemini and parses tool call responses.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { logger } from "../utils/logger.js";
|
|
9
|
+
|
|
10
|
+
// ─── Gemini API Types ──────────────────────────────────────────
|
|
11
|
+
|
|
12
|
+
// ─── Provider ──────────────────────────────────────────────────
|
|
13
|
+
|
|
14
|
+
export class GeminiProvider {
|
|
15
|
+
constructor(apiKey, model = 'gemini-2.5-flash') {
|
|
16
|
+
this.apiKey = apiKey;
|
|
17
|
+
this.model = model;
|
|
18
|
+
}
|
|
19
|
+
async generateContent(systemPrompt, userMessage, tools, history) {
|
|
20
|
+
logger.info('GeminiProvider', `Sending request. Model: ${this.model}, Tools: ${tools.length}`);
|
|
21
|
+
|
|
22
|
+
// Build Gemini tools
|
|
23
|
+
const geminiTools = this.buildGeminiTools(tools);
|
|
24
|
+
|
|
25
|
+
// Build conversation history
|
|
26
|
+
const contents = this.buildContents(userMessage, history);
|
|
27
|
+
|
|
28
|
+
// Make API request
|
|
29
|
+
const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`;
|
|
30
|
+
const body = {
|
|
31
|
+
contents,
|
|
32
|
+
tools: geminiTools.length > 0 ? geminiTools : undefined,
|
|
33
|
+
systemInstruction: {
|
|
34
|
+
parts: [{
|
|
35
|
+
text: systemPrompt
|
|
36
|
+
}]
|
|
37
|
+
},
|
|
38
|
+
generationConfig: {
|
|
39
|
+
temperature: 0.2,
|
|
40
|
+
maxOutputTokens: 2048
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
const startTime = Date.now();
|
|
44
|
+
try {
|
|
45
|
+
const response = await fetch(url, {
|
|
46
|
+
method: 'POST',
|
|
47
|
+
headers: {
|
|
48
|
+
'Content-Type': 'application/json'
|
|
49
|
+
},
|
|
50
|
+
body: JSON.stringify(body)
|
|
51
|
+
});
|
|
52
|
+
const elapsed = Date.now() - startTime;
|
|
53
|
+
logger.info('GeminiProvider', `Response received in ${elapsed}ms`);
|
|
54
|
+
if (!response.ok) {
|
|
55
|
+
const errorText = await response.text();
|
|
56
|
+
logger.error('GeminiProvider', `API error ${response.status}: ${errorText}`);
|
|
57
|
+
throw new Error(`Gemini API error ${response.status}: ${errorText}`);
|
|
58
|
+
}
|
|
59
|
+
const data = await response.json();
|
|
60
|
+
|
|
61
|
+
// Parse response
|
|
62
|
+
return this.parseResponse(data);
|
|
63
|
+
} catch (error) {
|
|
64
|
+
logger.error('GeminiProvider', 'Request failed:', error.message);
|
|
65
|
+
throw error;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// ─── Build Gemini Tools ────────────────────────────────────
|
|
70
|
+
|
|
71
|
+
buildGeminiTools(tools) {
|
|
72
|
+
const declarations = tools.map(tool => ({
|
|
73
|
+
name: tool.name,
|
|
74
|
+
description: tool.description,
|
|
75
|
+
parameters: {
|
|
76
|
+
type: 'OBJECT',
|
|
77
|
+
properties: Object.fromEntries(Object.entries(tool.parameters).map(([key, param]) => [key, {
|
|
78
|
+
type: this.mapParamType(param.type),
|
|
79
|
+
description: param.description,
|
|
80
|
+
...(param.enum ? {
|
|
81
|
+
enum: param.enum
|
|
82
|
+
} : {})
|
|
83
|
+
}])),
|
|
84
|
+
required: Object.entries(tool.parameters).filter(([, param]) => param.required !== false).map(([key]) => key)
|
|
85
|
+
}
|
|
86
|
+
}));
|
|
87
|
+
return [{
|
|
88
|
+
functionDeclarations: declarations
|
|
89
|
+
}];
|
|
90
|
+
}
|
|
91
|
+
mapParamType(type) {
|
|
92
|
+
switch (type) {
|
|
93
|
+
case 'number':
|
|
94
|
+
return 'NUMBER';
|
|
95
|
+
case 'boolean':
|
|
96
|
+
return 'BOOLEAN';
|
|
97
|
+
case 'string':
|
|
98
|
+
default:
|
|
99
|
+
return 'STRING';
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// ─── Build Contents ────────────────────────────────────────
|
|
104
|
+
|
|
105
|
+
buildContents(userMessage, history) {
|
|
106
|
+
const contents = [];
|
|
107
|
+
|
|
108
|
+
// Add history as conversation turns
|
|
109
|
+
for (const step of history) {
|
|
110
|
+
// User turn (screen state was sent)
|
|
111
|
+
contents.push({
|
|
112
|
+
role: 'user',
|
|
113
|
+
parts: [{
|
|
114
|
+
text: `Step ${step.stepIndex + 1} result: ${step.action.output}`
|
|
115
|
+
}]
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Current user message
|
|
120
|
+
contents.push({
|
|
121
|
+
role: 'user',
|
|
122
|
+
parts: [{
|
|
123
|
+
text: userMessage
|
|
124
|
+
}]
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
// Ensure alternating roles (Gemini requirement)
|
|
128
|
+
return this.ensureAlternatingRoles(contents);
|
|
129
|
+
}
|
|
130
|
+
ensureAlternatingRoles(contents) {
|
|
131
|
+
if (contents.length <= 1) return contents;
|
|
132
|
+
const merged = [contents[0]];
|
|
133
|
+
for (let i = 1; i < contents.length; i++) {
|
|
134
|
+
const prev = merged[merged.length - 1];
|
|
135
|
+
const curr = contents[i];
|
|
136
|
+
if (prev.role === curr.role) {
|
|
137
|
+
// Merge same-role messages
|
|
138
|
+
prev.parts.push(...curr.parts);
|
|
139
|
+
} else {
|
|
140
|
+
merged.push(curr);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return merged;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// ─── Parse Response ────────────────────────────────────────
|
|
147
|
+
|
|
148
|
+
parseResponse(data) {
|
|
149
|
+
const toolCalls = [];
|
|
150
|
+
let text;
|
|
151
|
+
if (!data.candidates || data.candidates.length === 0) {
|
|
152
|
+
logger.warn('GeminiProvider', 'No candidates in response');
|
|
153
|
+
return {
|
|
154
|
+
toolCalls,
|
|
155
|
+
text: 'No response generated.'
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
const candidate = data.candidates[0];
|
|
159
|
+
const parts = candidate.content?.parts || [];
|
|
160
|
+
for (const part of parts) {
|
|
161
|
+
if (part.functionCall) {
|
|
162
|
+
toolCalls.push({
|
|
163
|
+
name: part.functionCall.name,
|
|
164
|
+
args: part.functionCall.args || {}
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
if (part.text) {
|
|
168
|
+
text = (text || '') + part.text;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
logger.info('GeminiProvider', `Parsed: ${toolCalls.length} tool calls, text: ${text ? 'yes' : 'no'}`);
|
|
172
|
+
return {
|
|
173
|
+
toolCalls,
|
|
174
|
+
text
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
//# sourceMappingURL=GeminiProvider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["logger","GeminiProvider","constructor","apiKey","model","generateContent","systemPrompt","userMessage","tools","history","info","length","geminiTools","buildGeminiTools","contents","buildContents","url","body","undefined","systemInstruction","parts","text","generationConfig","temperature","maxOutputTokens","startTime","Date","now","response","fetch","method","headers","JSON","stringify","elapsed","ok","errorText","error","status","Error","data","json","parseResponse","message","declarations","map","tool","name","description","parameters","type","properties","Object","fromEntries","entries","key","param","mapParamType","enum","required","filter","functionDeclarations","step","push","role","stepIndex","action","output","ensureAlternatingRoles","merged","i","prev","curr","toolCalls","candidates","warn","candidate","content","part","functionCall","args"],"sourceRoot":"../../../src","sources":["providers/GeminiProvider.ts"],"mappings":";;AAAA;AACA;AACA;AACA;;AAEA,SAASA,MAAM,QAAQ,oBAAiB;;AAGxC;;AAqBA;;AAEA,OAAO,MAAMC,cAAc,CAAuB;EAKhDC,WAAWA,CAACC,MAAc,EAAEC,KAAa,GAAG,kBAAkB,EAAE;IAC9D,IAAI,CAACD,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACC,KAAK,GAAGA,KAAK;EACpB;EAEA,MAAMC,eAAeA,CACnBC,YAAoB,EACpBC,WAAmB,EACnBC,KAAuB,EACvBC,OAAoB,EACuE;IAE3FT,MAAM,CAACU,IAAI,CAAC,gBAAgB,EAAE,2BAA2B,IAAI,CAACN,KAAK,YAAYI,KAAK,CAACG,MAAM,EAAE,CAAC;;IAE9F;IACA,MAAMC,WAAW,GAAG,IAAI,CAACC,gBAAgB,CAACL,KAAK,CAAC;;IAEhD;IACA,MAAMM,QAAQ,GAAG,IAAI,CAACC,aAAa,CAACR,WAAW,EAAEE,OAAO,CAAC;;IAEzD;IACA,MAAMO,GAAG,GAAG,2DAA2D,IAAI,CAACZ,KAAK,wBAAwB,IAAI,CAACD,MAAM,EAAE;IAEtH,MAAMc,IAAS,GAAG;MAChBH,QAAQ;MACRN,KAAK,EAAEI,WAAW,CAACD,MAAM,GAAG,CAAC,GAAGC,WAAW,GAAGM,SAAS;MACvDC,iBAAiB,EAAE;QAAEC,KAAK,EAAE,CAAC;UAAEC,IAAI,EAAEf;QAAa,CAAC;MAAE,CAAC;MACtDgB,gBAAgB,EAAE;QAChBC,WAAW,EAAE,GAAG;QAChBC,eAAe,EAAE;MACnB;IACF,CAAC;IAED,MAAMC,SAAS,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC;IAE5B,IAAI;MACF,MAAMC,QAAQ,GAAG,MAAMC,KAAK,CAACb,GAAG,EAAE;QAChCc,MAAM,EAAE,MAAM;QACdC,OAAO,EAAE;UAAE,cAAc,EAAE;QAAmB,CAAC;QAC/Cd,IAAI,EAAEe,IAAI,CAACC,SAAS,CAAChB,IAAI;MAC3B,CAAC,CAAC;MAEF,MAAMiB,OAAO,GAAGR,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGF,SAAS;MACtCzB,MAAM,CAACU,IAAI,CAAC,gBAAgB,EAAE,wBAAwBwB,OAAO,IAAI,CAAC;MAElE,IAAI,CAACN,QAAQ,CAACO,EAAE,EAAE;QAChB,MAAMC,SAAS,GAAG,MAAMR,QAAQ,CAACP,IAAI,CAAC,CAAC;QACvCrB,MAAM,CAACqC,KAAK,CAAC,gBAAgB,EAAE,aAAaT,QAAQ,CAACU,MAAM,KAAKF,SAAS,EAAE,CAAC;QAC5E,MAAM,IAAIG,KAAK,CAAC,oBAAoBX,QAAQ,CAACU,MAAM,KAAKF,SAAS,EAAE,CAAC;MACtE;MAEA,MAAMI,IAAI,GAAG,MAAMZ,QAAQ,CAACa,IAAI,CAAC,CAAC;;MAElC;MACA,OAAO,IAAI,CAACC,aAAa,CAACF,IAAI,CAAC;IACjC,CAAC,CAAC,OAAOH,KAAU,EAAE;MACnBrC,MAAM,CAACqC,KAAK,CAAC,gBAAgB,EAAE,iBAAiB,EAAEA,KAAK,CAACM,OAAO,CAAC;MAChE,MAAMN,KAAK;IACb;EACF;;EAEA;;EAEQxB,gBAAgBA,CAACL,KAAuB,EAAgB;IAC9D,MAAMoC,YAAyC,GAAGpC,KAAK,CAACqC,GAAG,CAACC,IAAI,KAAK;MACnEC,IAAI,EAAED,IAAI,CAACC,IAAI;MACfC,WAAW,EAAEF,IAAI,CAACE,WAAW;MAC7BC,UAAU,EAAE;QACVC,IAAI,EAAE,QAAQ;QACdC,UAAU,EAAEC,MAAM,CAACC,WAAW,CAC5BD,MAAM,CAACE,OAAO,CAACR,IAAI,CAACG,UAAU,CAAC,CAACJ,GAAG,CAAC,CAAC,CAACU,GAAG,EAAEC,KAAK,CAAC,KAAK,CACpDD,GAAG,EACH;UACEL,IAAI,EAAE,IAAI,CAACO,YAAY,CAACD,KAAK,CAACN,IAAI,CAAC;UACnCF,WAAW,EAAEQ,KAAK,CAACR,WAAW;UAC9B,IAAIQ,KAAK,CAACE,IAAI,GAAG;YAAEA,IAAI,EAAEF,KAAK,CAACE;UAAK,CAAC,GAAG,CAAC,CAAC;QAC5C,CAAC,CACF,CACH,CAAC;QACDC,QAAQ,EAAEP,MAAM,CAACE,OAAO,CAACR,IAAI,CAACG,UAAU,CAAC,CACtCW,MAAM,CAAC,CAAC,GAAGJ,KAAK,CAAC,KAAKA,KAAK,CAACG,QAAQ,KAAK,KAAK,CAAC,CAC/Cd,GAAG,CAAC,CAAC,CAACU,GAAG,CAAC,KAAKA,GAAG;MACvB;IACF,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC;MAAEM,oBAAoB,EAAEjB;IAAa,CAAC,CAAC;EACjD;EAEQa,YAAYA,CAACP,IAAY,EAAU;IACzC,QAAQA,IAAI;MACV,KAAK,QAAQ;QAAE,OAAO,QAAQ;MAC9B,KAAK,SAAS;QAAE,OAAO,SAAS;MAChC,KAAK,QAAQ;MACb;QAAS,OAAO,QAAQ;IAC1B;EACF;;EAEA;;EAEQnC,aAAaA,CAACR,WAAmB,EAAEE,OAAoB,EAAmB;IAChF,MAAMK,QAAyB,GAAG,EAAE;;IAEpC;IACA,KAAK,MAAMgD,IAAI,IAAIrD,OAAO,EAAE;MAC1B;MACAK,QAAQ,CAACiD,IAAI,CAAC;QACZC,IAAI,EAAE,MAAM;QACZ5C,KAAK,EAAE,CAAC;UAAEC,IAAI,EAAE,QAAQyC,IAAI,CAACG,SAAS,GAAG,CAAC,YAAYH,IAAI,CAACI,MAAM,CAACC,MAAM;QAAG,CAAC;MAC9E,CAAC,CAAC;IACJ;;IAEA;IACArD,QAAQ,CAACiD,IAAI,CAAC;MACZC,IAAI,EAAE,MAAM;MACZ5C,KAAK,EAAE,CAAC;QAAEC,IAAI,EAAEd;MAAY,CAAC;IAC/B,CAAC,CAAC;;IAEF;IACA,OAAO,IAAI,CAAC6D,sBAAsB,CAACtD,QAAQ,CAAC;EAC9C;EAEQsD,sBAAsBA,CAACtD,QAAyB,EAAmB;IACzE,IAAIA,QAAQ,CAACH,MAAM,IAAI,CAAC,EAAE,OAAOG,QAAQ;IAEzC,MAAMuD,MAAuB,GAAG,CAACvD,QAAQ,CAAC,CAAC,CAAC,CAAE;IAE9C,KAAK,IAAIwD,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGxD,QAAQ,CAACH,MAAM,EAAE2D,CAAC,EAAE,EAAE;MACxC,MAAMC,IAAI,GAAGF,MAAM,CAACA,MAAM,CAAC1D,MAAM,GAAG,CAAC,CAAE;MACvC,MAAM6D,IAAI,GAAG1D,QAAQ,CAACwD,CAAC,CAAE;MAEzB,IAAIC,IAAI,CAACP,IAAI,KAAKQ,IAAI,CAACR,IAAI,EAAE;QAC3B;QACAO,IAAI,CAACnD,KAAK,CAAC2C,IAAI,CAAC,GAAGS,IAAI,CAACpD,KAAK,CAAC;MAChC,CAAC,MAAM;QACLiD,MAAM,CAACN,IAAI,CAACS,IAAI,CAAC;MACnB;IACF;IAEA,OAAOH,MAAM;EACf;;EAEA;;EAEQ3B,aAAaA,CAACF,IAAS,EAAoF;IACjH,MAAMiC,SAA6D,GAAG,EAAE;IACxE,IAAIpD,IAAwB;IAE5B,IAAI,CAACmB,IAAI,CAACkC,UAAU,IAAIlC,IAAI,CAACkC,UAAU,CAAC/D,MAAM,KAAK,CAAC,EAAE;MACpDX,MAAM,CAAC2E,IAAI,CAAC,gBAAgB,EAAE,2BAA2B,CAAC;MAC1D,OAAO;QAAEF,SAAS;QAAEpD,IAAI,EAAE;MAAyB,CAAC;IACtD;IAEA,MAAMuD,SAAS,GAAGpC,IAAI,CAACkC,UAAU,CAAC,CAAC,CAAC;IACpC,MAAMtD,KAAK,GAAGwD,SAAS,CAACC,OAAO,EAAEzD,KAAK,IAAI,EAAE;IAE5C,KAAK,MAAM0D,IAAI,IAAI1D,KAAK,EAAE;MACxB,IAAI0D,IAAI,CAACC,YAAY,EAAE;QACrBN,SAAS,CAACV,IAAI,CAAC;UACbhB,IAAI,EAAE+B,IAAI,CAACC,YAAY,CAAChC,IAAI;UAC5BiC,IAAI,EAAEF,IAAI,CAACC,YAAY,CAACC,IAAI,IAAI,CAAC;QACnC,CAAC,CAAC;MACJ;MACA,IAAIF,IAAI,CAACzD,IAAI,EAAE;QACbA,IAAI,GAAG,CAACA,IAAI,IAAI,EAAE,IAAIyD,IAAI,CAACzD,IAAI;MACjC;IACF;IAEArB,MAAM,CAACU,IAAI,CAAC,gBAAgB,EAAE,WAAW+D,SAAS,CAAC9D,MAAM,sBAAsBU,IAAI,GAAG,KAAK,GAAG,IAAI,EAAE,CAAC;IAErG,OAAO;MAAEoD,SAAS;MAAEpD;IAAK,CAAC;EAC5B;AAGF","ignoreList":[]}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Logger utility — prefixed console output for easy filtering.
|
|
5
|
+
*/
|
|
6
|
+
const TAG = '[AIAgent]';
|
|
7
|
+
export const logger = {
|
|
8
|
+
info: (context, ...args) => console.log(`${TAG} [${context}]`, ...args),
|
|
9
|
+
warn: (context, ...args) => console.warn(`${TAG} [${context}]`, ...args),
|
|
10
|
+
error: (context, ...args) => console.error(`${TAG} [${context}]`, ...args),
|
|
11
|
+
debug: (context, ...args) => {
|
|
12
|
+
if (__DEV__) {
|
|
13
|
+
console.log(`${TAG} [${context}] 🐛`, ...args);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
};
|
|
17
|
+
//# sourceMappingURL=logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["TAG","logger","info","context","args","console","log","warn","error","debug","__DEV__"],"sourceRoot":"../../../src","sources":["utils/logger.ts"],"mappings":";;AAAA;AACA;AACA;AACA,MAAMA,GAAG,GAAG,WAAW;AAEvB,OAAO,MAAMC,MAAM,GAAG;EACpBC,IAAI,EAAEA,CAACC,OAAe,EAAE,GAAGC,IAAW,KACpCC,OAAO,CAACC,GAAG,CAAC,GAAGN,GAAG,KAAKG,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAE7CG,IAAI,EAAEA,CAACJ,OAAe,EAAE,GAAGC,IAAW,KACpCC,OAAO,CAACE,IAAI,CAAC,GAAGP,GAAG,KAAKG,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAE9CI,KAAK,EAAEA,CAACL,OAAe,EAAE,GAAGC,IAAW,KACrCC,OAAO,CAACG,KAAK,CAAC,GAAGR,GAAG,KAAKG,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAE/CK,KAAK,EAAEA,CAACN,OAAe,EAAE,GAAGC,IAAW,KAAK;IAC1C,IAAIM,OAAO,EAAE;MACXL,OAAO,CAACC,GAAG,CAAC,GAAGN,GAAG,KAAKG,OAAO,MAAM,EAAE,GAAGC,IAAI,CAAC;IAChD;EACF;AACF,CAAC","ignoreList":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"type":"module"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AIAgent — Root provider component for the AI agent.
|
|
3
|
+
*
|
|
4
|
+
* Wraps the app and provides:
|
|
5
|
+
* - Fiber tree root ref for element auto-detection
|
|
6
|
+
* - Navigation ref for auto-navigation
|
|
7
|
+
* - Floating chat bar for user input
|
|
8
|
+
* - Agent runtime context for useAction hooks
|
|
9
|
+
*/
|
|
10
|
+
import React from 'react';
|
|
11
|
+
import type { ExecutionResult, ToolDefinition, AgentStep } from '../core/types';
|
|
12
|
+
interface AIAgentProps {
|
|
13
|
+
/** Gemini API key */
|
|
14
|
+
apiKey: string;
|
|
15
|
+
/** Gemini model name */
|
|
16
|
+
model?: string;
|
|
17
|
+
/** Navigation container ref (from useNavigationContainerRef) */
|
|
18
|
+
navRef?: any;
|
|
19
|
+
/** UI language */
|
|
20
|
+
language?: 'en' | 'ar';
|
|
21
|
+
/** Max agent steps per request */
|
|
22
|
+
maxSteps?: number;
|
|
23
|
+
/** Show/hide the chat bar */
|
|
24
|
+
showChatBar?: boolean;
|
|
25
|
+
/** Children — the actual app */
|
|
26
|
+
children: React.ReactNode;
|
|
27
|
+
/** Callback when agent completes */
|
|
28
|
+
onResult?: (result: ExecutionResult) => void;
|
|
29
|
+
/** Refs of elements the AI must NOT interact with */
|
|
30
|
+
interactiveBlacklist?: React.RefObject<any>[];
|
|
31
|
+
/** If set, AI can ONLY interact with these elements */
|
|
32
|
+
interactiveWhitelist?: React.RefObject<any>[];
|
|
33
|
+
/** Called before each step */
|
|
34
|
+
onBeforeStep?: (stepCount: number) => Promise<void> | void;
|
|
35
|
+
/** Called after each step */
|
|
36
|
+
onAfterStep?: (history: AgentStep[]) => Promise<void> | void;
|
|
37
|
+
/** Called before task starts */
|
|
38
|
+
onBeforeTask?: () => Promise<void> | void;
|
|
39
|
+
/** Called after task completes */
|
|
40
|
+
onAfterTask?: (result: ExecutionResult) => Promise<void> | void;
|
|
41
|
+
/** Transform screen content before LLM sees it (for data masking) */
|
|
42
|
+
transformScreenContent?: (content: string) => Promise<string> | string;
|
|
43
|
+
/** Override or remove built-in tools (null = remove) */
|
|
44
|
+
customTools?: Record<string, ToolDefinition | null>;
|
|
45
|
+
/** Instructions to guide agent behavior */
|
|
46
|
+
instructions?: {
|
|
47
|
+
system?: string;
|
|
48
|
+
getScreenInstructions?: (screenName: string) => string | undefined | null;
|
|
49
|
+
};
|
|
50
|
+
/** Delay between steps in ms */
|
|
51
|
+
stepDelay?: number;
|
|
52
|
+
/** WebSocket URL to companion MCP server bridge (e.g., ws://localhost:3101) */
|
|
53
|
+
mcpServerUrl?: string;
|
|
54
|
+
}
|
|
55
|
+
export declare function AIAgent({ apiKey, model, navRef, language, maxSteps, showChatBar, children, onResult, interactiveBlacklist, interactiveWhitelist, onBeforeStep, onAfterStep, onBeforeTask, onAfterTask, transformScreenContent, customTools, instructions, stepDelay, mcpServerUrl, }: AIAgentProps): import("react/jsx-runtime").JSX.Element;
|
|
56
|
+
export {};
|
|
57
|
+
//# sourceMappingURL=AIAgent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AIAgent.d.ts","sourceRoot":"","sources":["../../../../src/components/AIAgent.tsx"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAMN,MAAM,OAAO,CAAC;AASf,OAAO,KAAK,EAAe,eAAe,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAO7F,UAAU,YAAY;IACpB,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,kBAAkB;IAClB,QAAQ,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,gCAAgC;IAChC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC;IAC1B,oCAAoC;IACpC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IAI7C,qDAAqD;IACrD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,uDAAuD;IACvD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,8BAA8B;IAC9B,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3D,6BAA6B;IAC7B,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7D,gCAAgC;IAChC,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1C,kCAAkC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAChE,qEAAqE;IACrE,sBAAsB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IACvE,wDAAwD;IACxD,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC,CAAC;IACpD,2CAA2C;IAC3C,YAAY,CAAC,EAAE;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,qBAAqB,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,GAAG,IAAI,CAAC;KAC3E,CAAC;IACF,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAID,wBAAgB,OAAO,CAAC,EACtB,MAAM,EACN,KAA0B,EAC1B,MAAM,EACN,QAAe,EACf,QAAa,EACb,WAAkB,EAClB,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACpB,oBAAoB,EACpB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,YAAY,GACb,EAAE,YAAY,2CAiHd"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AgentChatBar — Floating chat input at the bottom of the screen.
|
|
3
|
+
* User sends messages here, and results are shown inline.
|
|
4
|
+
*/
|
|
5
|
+
import type { ExecutionResult } from '../core/types';
|
|
6
|
+
interface AgentChatBarProps {
|
|
7
|
+
onSend: (message: string) => void;
|
|
8
|
+
isThinking: boolean;
|
|
9
|
+
lastResult: ExecutionResult | null;
|
|
10
|
+
language: 'en' | 'ar';
|
|
11
|
+
}
|
|
12
|
+
export declare function AgentChatBar({ onSend, isThinking, lastResult, language }: AgentChatBarProps): import("react/jsx-runtime").JSX.Element;
|
|
13
|
+
export {};
|
|
14
|
+
//# sourceMappingURL=AgentChatBar.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AgentChatBar.d.ts","sourceRoot":"","sources":["../../../../src/components/AgentChatBar.tsx"],"names":[],"mappings":"AAAA;;;GAGG;AAYH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAErD,UAAU,iBAAiB;IACzB,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,eAAe,GAAG,IAAI,CAAC;IACnC,QAAQ,EAAE,IAAI,GAAG,IAAI,CAAC;CACvB;AAED,wBAAgB,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,iBAAiB,2CAgD3F"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AgentOverlay — Subtle thinking indicator shown while the AI agent is processing.
|
|
3
|
+
*/
|
|
4
|
+
interface AgentOverlayProps {
|
|
5
|
+
visible: boolean;
|
|
6
|
+
statusText: string;
|
|
7
|
+
}
|
|
8
|
+
export declare function AgentOverlay({ visible, statusText }: AgentOverlayProps): import("react/jsx-runtime").JSX.Element | null;
|
|
9
|
+
export {};
|
|
10
|
+
//# sourceMappingURL=AgentOverlay.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AgentOverlay.d.ts","sourceRoot":"","sources":["../../../../src/components/AgentOverlay.tsx"],"names":[],"mappings":"AAAA;;GAEG;AAIH,UAAU,iBAAiB;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,wBAAgB,YAAY,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,iBAAiB,kDAWtE"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AgentRuntime — The main agent loop, inspired by page-agent.js.
|
|
3
|
+
*
|
|
4
|
+
* Flow:
|
|
5
|
+
* 1. Walk Fiber tree → detect interactive elements
|
|
6
|
+
* 2. Dehydrate screen → text for LLM
|
|
7
|
+
* 3. Send to AI provider with tools
|
|
8
|
+
* 4. Parse tool call → execute (tap, type, navigate, done)
|
|
9
|
+
* 5. If not done, repeat from step 1 (re-dehydrate after UI change)
|
|
10
|
+
*/
|
|
11
|
+
import type { AIProvider, AgentConfig, ExecutionResult, ActionDefinition } from './types';
|
|
12
|
+
export declare class AgentRuntime {
|
|
13
|
+
private provider;
|
|
14
|
+
private config;
|
|
15
|
+
private rootRef;
|
|
16
|
+
private navRef;
|
|
17
|
+
private tools;
|
|
18
|
+
private actions;
|
|
19
|
+
private history;
|
|
20
|
+
private isRunning;
|
|
21
|
+
private lastAskUserQuestion;
|
|
22
|
+
constructor(provider: AIProvider, config: AgentConfig, rootRef: any, navRef: any);
|
|
23
|
+
private registerBuiltInTools;
|
|
24
|
+
registerAction(action: ActionDefinition): void;
|
|
25
|
+
unregisterAction(name: string): void;
|
|
26
|
+
private getRouteNames;
|
|
27
|
+
private getCurrentScreenName;
|
|
28
|
+
private buildToolsForProvider;
|
|
29
|
+
private getWalkConfig;
|
|
30
|
+
private getInstructions;
|
|
31
|
+
execute(userMessage: string): Promise<ExecutionResult>;
|
|
32
|
+
/** Update refs (called when component re-renders) */
|
|
33
|
+
updateRefs(rootRef: any, navRef: any): void;
|
|
34
|
+
/** Check if agent is currently executing */
|
|
35
|
+
getIsRunning(): boolean;
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=AgentRuntime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AgentRuntime.d.ts","sourceRoot":"","sources":["../../../../src/core/AgentRuntime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,OAAO,KAAK,EACV,UAAU,EACV,WAAW,EAEX,eAAe,EAEf,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAmDjB,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,OAAO,CAAM;IACrB,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,KAAK,CAA0C;IACvD,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,mBAAmB,CAAuB;gBAGhD,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,GAAG,EACZ,MAAM,EAAE,GAAG;IAyBb,OAAO,CAAC,oBAAoB;IAkH5B,cAAc,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAK9C,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAMpC,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,oBAAoB;IAc5B,OAAO,CAAC,qBAAqB;IA8B7B,OAAO,CAAC,aAAa;IASrB,OAAO,CAAC,eAAe;IAyBjB,OAAO,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAsK5D,qDAAqD;IACrD,UAAU,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,GAAG,IAAI;IAK3C,4CAA4C;IAC5C,YAAY,IAAI,OAAO;CAGxB"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FiberTreeWalker — Traverses React's Fiber tree to discover interactive elements.
|
|
3
|
+
*
|
|
4
|
+
* This is the React Native equivalent of page-agent.js reading the DOM.
|
|
5
|
+
* Instead of traversing HTML nodes, we traverse React Fiber nodes and detect
|
|
6
|
+
* interactive elements by their type and props (onPress, onChangeText, etc.).
|
|
7
|
+
*
|
|
8
|
+
* Architecture inspired by: https://github.com/alibaba/page-agent
|
|
9
|
+
*/
|
|
10
|
+
import type { InteractiveElement } from './types';
|
|
11
|
+
export interface WalkConfig {
|
|
12
|
+
/** React refs of elements to exclude — mirrors page-agent interactiveBlacklist */
|
|
13
|
+
interactiveBlacklist?: React.RefObject<any>[];
|
|
14
|
+
/** If set, only these elements are interactive — mirrors page-agent interactiveWhitelist */
|
|
15
|
+
interactiveWhitelist?: React.RefObject<any>[];
|
|
16
|
+
}
|
|
17
|
+
export interface WalkResult {
|
|
18
|
+
elementsText: string;
|
|
19
|
+
interactives: InteractiveElement[];
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Walk the React Fiber tree from a root and collect all interactive elements
|
|
23
|
+
* as well as a hierarchical layout representation for the LLM.
|
|
24
|
+
*/
|
|
25
|
+
export declare function walkFiberTree(rootRef: any, config?: WalkConfig): WalkResult;
|
|
26
|
+
//# sourceMappingURL=FiberTreeWalker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FiberTreeWalker.d.ts","sourceRoot":"","sources":["../../../../src/core/FiberTreeWalker.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,kBAAkB,EAAe,MAAM,SAAS,CAAC;AAI/D,MAAM,WAAW,UAAU;IACzB,kFAAkF;IAClF,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,4FAA4F;IAC5F,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;CAC/C;AAmOD,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,kBAAkB,EAAE,CAAC;CACpC;AAID;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU,CA0F3E"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MCPBridge — Connects the React Native app to the local MCP Server bridge.
|
|
3
|
+
*
|
|
4
|
+
* Flow:
|
|
5
|
+
* - Connects via WebSocket to the Node.js MCP server
|
|
6
|
+
* - Listens for 'request' messages containing an MCP command
|
|
7
|
+
* - Forwards the command to AgentRuntime.execute()
|
|
8
|
+
* - Sends the ExecutionResult back via WebSocket as a 'response'
|
|
9
|
+
*/
|
|
10
|
+
import type { AgentRuntime } from './AgentRuntime';
|
|
11
|
+
export declare class MCPBridge {
|
|
12
|
+
private url;
|
|
13
|
+
private ws;
|
|
14
|
+
private runtime;
|
|
15
|
+
private reconnectTimer;
|
|
16
|
+
private isDestroyed;
|
|
17
|
+
constructor(url: string, runtime: AgentRuntime);
|
|
18
|
+
private connect;
|
|
19
|
+
private sendResponse;
|
|
20
|
+
private scheduleReconnect;
|
|
21
|
+
destroy(): void;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=MCPBridge.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MCPBridge.d.ts","sourceRoot":"","sources":["../../../../src/core/MCPBridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,qBAAa,SAAS;IACpB,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,WAAW,CAAS;gBAEhB,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY;IAM9C,OAAO,CAAC,OAAO;IAsDf,OAAO,CAAC,YAAY;IAUpB,OAAO,CAAC,iBAAiB;IAQlB,OAAO;CAWf"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ScreenDehydrator — Converts discovered interactive elements into
|
|
3
|
+
* a text representation for the LLM, matching page-agent.js format.
|
|
4
|
+
*
|
|
5
|
+
* Output example:
|
|
6
|
+
* ```
|
|
7
|
+
* Screen: Home | Available screens: Home, Menu, Cart
|
|
8
|
+
* Interactive elements:
|
|
9
|
+
* [0]<pressable>🍕 Pizzas</>
|
|
10
|
+
* [1]<pressable>🍔 Burgers</>
|
|
11
|
+
* [2]<pressable>🥤 Drinks</>
|
|
12
|
+
* [3]<pressable>🛒 View Cart</>
|
|
13
|
+
* ```
|
|
14
|
+
*/
|
|
15
|
+
import type { InteractiveElement, DehydratedScreen } from './types';
|
|
16
|
+
/**
|
|
17
|
+
* Dehydrate the current screen state into a text format for the LLM.
|
|
18
|
+
*/
|
|
19
|
+
export declare function dehydrateScreen(screenName: string, availableScreens: string[], elementsText: string, elements: InteractiveElement[]): DehydratedScreen;
|
|
20
|
+
//# sourceMappingURL=ScreenDehydrator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ScreenDehydrator.d.ts","sourceRoot":"","sources":["../../../../src/core/ScreenDehydrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAEpE;;GAEG;AACH,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,gBAAgB,EAAE,MAAM,EAAE,EAC1B,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,kBAAkB,EAAE,GAC7B,gBAAgB,CA2BlB"}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Core types for the page-agent-style React Native AI SDK.
|
|
3
|
+
*/
|
|
4
|
+
export type ElementType = 'pressable' | 'text-input' | 'switch' | 'scrollable';
|
|
5
|
+
export interface InteractiveElement {
|
|
6
|
+
/** Unique index assigned during tree walk */
|
|
7
|
+
index: number;
|
|
8
|
+
/** Element type */
|
|
9
|
+
type: ElementType;
|
|
10
|
+
/** Human-readable label (extracted from Text children or accessibilityLabel) */
|
|
11
|
+
label: string;
|
|
12
|
+
/** Reference to the Fiber node for execution */
|
|
13
|
+
fiberNode: any;
|
|
14
|
+
/** Key props snapshot */
|
|
15
|
+
props: {
|
|
16
|
+
onPress?: (...args: any[]) => void;
|
|
17
|
+
onChangeText?: (text: string) => void;
|
|
18
|
+
value?: string;
|
|
19
|
+
placeholder?: string;
|
|
20
|
+
checked?: boolean;
|
|
21
|
+
disabled?: boolean;
|
|
22
|
+
accessibilityLabel?: string;
|
|
23
|
+
accessibilityRole?: string;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export interface DehydratedScreen {
|
|
27
|
+
/** Current screen name (from navigation state) */
|
|
28
|
+
screenName: string;
|
|
29
|
+
/** All available screen names (from routeNames) */
|
|
30
|
+
availableScreens: string[];
|
|
31
|
+
/** Indexed interactive elements as text */
|
|
32
|
+
elementsText: string;
|
|
33
|
+
/** Raw elements array */
|
|
34
|
+
elements: InteractiveElement[];
|
|
35
|
+
}
|
|
36
|
+
export interface AgentStep {
|
|
37
|
+
stepIndex: number;
|
|
38
|
+
reflection: {
|
|
39
|
+
evaluationPreviousGoal: string;
|
|
40
|
+
memory: string;
|
|
41
|
+
nextGoal: string;
|
|
42
|
+
};
|
|
43
|
+
action: {
|
|
44
|
+
name: string;
|
|
45
|
+
input: Record<string, any>;
|
|
46
|
+
output: string;
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
export interface AgentConfig {
|
|
50
|
+
apiKey: string;
|
|
51
|
+
model?: string;
|
|
52
|
+
language?: 'en' | 'ar';
|
|
53
|
+
/** Maximum steps per task (page-agent default: 40) */
|
|
54
|
+
maxSteps?: number;
|
|
55
|
+
/**
|
|
56
|
+
* React refs of elements the AI must NOT interact with.
|
|
57
|
+
* Mirrors page-agent.js `interactiveBlacklist: Element[]`.
|
|
58
|
+
* The Fiber walker skips any node whose ref matches one in this list.
|
|
59
|
+
*/
|
|
60
|
+
interactiveBlacklist?: React.RefObject<any>[];
|
|
61
|
+
/**
|
|
62
|
+
* If set, the AI can ONLY interact with these elements.
|
|
63
|
+
* Mirrors page-agent.js `interactiveWhitelist: Element[]`.
|
|
64
|
+
*/
|
|
65
|
+
interactiveWhitelist?: React.RefObject<any>[];
|
|
66
|
+
/** Called before each agent step. */
|
|
67
|
+
onBeforeStep?: (stepCount: number) => Promise<void> | void;
|
|
68
|
+
/** Called after each agent step with full history. */
|
|
69
|
+
onAfterStep?: (history: AgentStep[]) => Promise<void> | void;
|
|
70
|
+
/** Called before task execution starts. */
|
|
71
|
+
onBeforeTask?: () => Promise<void> | void;
|
|
72
|
+
/** Called after task completes (success or failure). */
|
|
73
|
+
onAfterTask?: (result: ExecutionResult) => Promise<void> | void;
|
|
74
|
+
/**
|
|
75
|
+
* Transform dehydrated screen text before sending to LLM.
|
|
76
|
+
* Use to mask sensitive data (credit cards, PII, etc).
|
|
77
|
+
* Mirrors page-agent.js `transformPageContent`.
|
|
78
|
+
*/
|
|
79
|
+
transformScreenContent?: (content: string) => Promise<string> | string;
|
|
80
|
+
/**
|
|
81
|
+
* Override or remove built-in tools.
|
|
82
|
+
* Set tool to `null` to remove it (e.g., `{ navigate: null }`).
|
|
83
|
+
* Mirrors page-agent.js `customTools: Record<string, Tool | null>`.
|
|
84
|
+
*/
|
|
85
|
+
customTools?: Record<string, ToolDefinition | null>;
|
|
86
|
+
/** Instructions to guide the agent's behavior. */
|
|
87
|
+
instructions?: {
|
|
88
|
+
/** Global system-level instructions, applied to all tasks. */
|
|
89
|
+
system?: string;
|
|
90
|
+
/**
|
|
91
|
+
* Dynamic per-screen instructions callback.
|
|
92
|
+
* Called before each step to get instructions for the current screen.
|
|
93
|
+
* Mirrors page-agent.js `getPageInstructions(url)`.
|
|
94
|
+
*/
|
|
95
|
+
getScreenInstructions?: (screenName: string) => string | undefined | null;
|
|
96
|
+
};
|
|
97
|
+
/** Delay between steps in ms (page-agent default: 400ms). */
|
|
98
|
+
stepDelay?: number;
|
|
99
|
+
/**
|
|
100
|
+
* Optional URL of the companion Node.js MCP server bridge (e.g. ws://localhost:3101).
|
|
101
|
+
* If set, the SDK will connect to this server and listen for execution requests
|
|
102
|
+
* from external AI agents (like OpenClaw, Claude Desktop, etc).
|
|
103
|
+
*/
|
|
104
|
+
mcpServerUrl?: string;
|
|
105
|
+
}
|
|
106
|
+
export interface ExecutionResult {
|
|
107
|
+
success: boolean;
|
|
108
|
+
message: string;
|
|
109
|
+
steps: AgentStep[];
|
|
110
|
+
}
|
|
111
|
+
export interface ToolDefinition {
|
|
112
|
+
name: string;
|
|
113
|
+
description: string;
|
|
114
|
+
parameters: Record<string, ToolParam>;
|
|
115
|
+
execute: (args: Record<string, any>) => Promise<string>;
|
|
116
|
+
}
|
|
117
|
+
export interface ToolParam {
|
|
118
|
+
type: 'string' | 'number' | 'boolean';
|
|
119
|
+
description: string;
|
|
120
|
+
required?: boolean;
|
|
121
|
+
enum?: string[];
|
|
122
|
+
}
|
|
123
|
+
export interface ActionDefinition {
|
|
124
|
+
name: string;
|
|
125
|
+
description: string;
|
|
126
|
+
parameters: Record<string, string>;
|
|
127
|
+
handler: (args: Record<string, any>) => any;
|
|
128
|
+
}
|
|
129
|
+
export interface AIProvider {
|
|
130
|
+
generateContent(systemPrompt: string, userMessage: string, tools: ToolDefinition[], history: AgentStep[]): Promise<{
|
|
131
|
+
toolCalls: Array<{
|
|
132
|
+
name: string;
|
|
133
|
+
args: Record<string, any>;
|
|
134
|
+
}>;
|
|
135
|
+
text?: string;
|
|
136
|
+
}>;
|
|
137
|
+
}
|
|
138
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/core/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,YAAY,GAAG,QAAQ,GAAG,YAAY,CAAC;AAE/E,MAAM,WAAW,kBAAkB;IACjC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,mBAAmB;IACnB,IAAI,EAAE,WAAW,CAAC;IAClB,gFAAgF;IAChF,KAAK,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,SAAS,EAAE,GAAG,CAAC;IACf,yBAAyB;IACzB,KAAK,EAAE;QACL,OAAO,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,CAAC;QACnC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;QACtC,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B,CAAC;CACH;AAID,MAAM,WAAW,gBAAgB;IAC/B,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,mDAAmD;IACnD,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,2CAA2C;IAC3C,YAAY,EAAE,MAAM,CAAC;IACrB,yBAAyB;IACzB,QAAQ,EAAE,kBAAkB,EAAE,CAAC;CAChC;AAID,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE;QACV,sBAAsB,EAAE,MAAM,CAAC;QAC/B,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC3B,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IAEvB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAIlB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAE9C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAI9C,qCAAqC;IACrC,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAE3D,sDAAsD;IACtD,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAE7D,2CAA2C;IAC3C,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAE1C,wDAAwD;IACxD,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAIhE;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IAIvE;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC,CAAC;IAIpD,kDAAkD;IAClD,YAAY,CAAC,EAAE;QACb,8DAA8D;QAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;;WAIG;QACH,qBAAqB,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,GAAG,IAAI,CAAC;KAC3E,CAAC;IAEF,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IAInB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,SAAS,EAAE,CAAC;CACpB;AAID,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACtC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACzD;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAID,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC;CAC7C;AAID,MAAM,WAAW,UAAU;IACzB,eAAe,CACb,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,cAAc,EAAE,EACvB,OAAO,EAAE,SAAS,EAAE,GACnB,OAAO,CAAC;QACT,SAAS,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAAE,CAAC,CAAC;QAC9D,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;CACJ"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* useAction — Optional hook to register non-UI actions for the AI agent.
|
|
3
|
+
*
|
|
4
|
+
* Use this for business logic that doesn't correspond to a visible UI element,
|
|
5
|
+
* e.g., API calls, cart operations, calculations.
|
|
6
|
+
*
|
|
7
|
+
* The Fiber tree walker handles visible UI elements automatically.
|
|
8
|
+
* useAction is for invisible operations the AI should be able to trigger.
|
|
9
|
+
*/
|
|
10
|
+
import type { AgentRuntime } from '../core/AgentRuntime';
|
|
11
|
+
export declare const AgentContext: import("react").Context<AgentRuntime | null>;
|
|
12
|
+
export declare function useAction(name: string, description: string, parameters: Record<string, string>, handler: (args: Record<string, any>) => any): void;
|
|
13
|
+
//# sourceMappingURL=useAction.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useAction.d.ts","sourceRoot":"","sources":["../../../../src/hooks/useAction.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,eAAO,MAAM,YAAY,8CAA2C,CAAC;AAErE,wBAAgB,SAAS,CACvB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,GAC1C,IAAI,CAkBN"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @mobileai/react-native — Page-Agent Architecture
|
|
3
|
+
*
|
|
4
|
+
* Zero-wrapper AI agent for React Native.
|
|
5
|
+
* Auto-detects interactive elements via React Fiber tree traversal.
|
|
6
|
+
*/
|
|
7
|
+
export { AIAgent } from './components/AIAgent';
|
|
8
|
+
export { useAction } from './hooks/useAction';
|
|
9
|
+
export type { AgentConfig, ExecutionResult, InteractiveElement, DehydratedScreen, ToolDefinition, ActionDefinition, } from './core/types';
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAG/C,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAG9C,YAAY,EACV,WAAW,EACX,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GeminiProvider — Simplified Gemini API integration.
|
|
3
|
+
* Sends dehydrated screen state + tools to Gemini and parses tool call responses.
|
|
4
|
+
*/
|
|
5
|
+
import type { AIProvider, ToolDefinition, AgentStep } from '../core/types';
|
|
6
|
+
export declare class GeminiProvider implements AIProvider {
|
|
7
|
+
private apiKey;
|
|
8
|
+
private model;
|
|
9
|
+
constructor(apiKey: string, model?: string);
|
|
10
|
+
generateContent(systemPrompt: string, userMessage: string, tools: ToolDefinition[], history: AgentStep[]): Promise<{
|
|
11
|
+
toolCalls: Array<{
|
|
12
|
+
name: string;
|
|
13
|
+
args: Record<string, any>;
|
|
14
|
+
}>;
|
|
15
|
+
text?: string;
|
|
16
|
+
}>;
|
|
17
|
+
private buildGeminiTools;
|
|
18
|
+
private mapParamType;
|
|
19
|
+
private buildContents;
|
|
20
|
+
private ensureAlternatingRoles;
|
|
21
|
+
private parseResponse;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=GeminiProvider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"GeminiProvider.d.ts","sourceRoot":"","sources":["../../../../src/providers/GeminiProvider.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAyB3E,qBAAa,cAAe,YAAW,UAAU;IAC/C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAGV,MAAM,EAAE,MAAM,EAAE,KAAK,GAAE,MAA2B;IAKxD,eAAe,CACnB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,cAAc,EAAE,EACvB,OAAO,EAAE,SAAS,EAAE,GACnB,OAAO,CAAC;QAAE,SAAS,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAAE,CAAC,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAqD5F,OAAO,CAAC,gBAAgB;IAyBxB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,aAAa;IAsBrB,OAAO,CAAC,sBAAsB;IAsB9B,OAAO,CAAC,aAAa;CA8BtB"}
|