@mobileai/react-native 0.5.5 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -12
- package/lib/module/components/AIAgent.js +55 -56
- package/lib/module/components/AIAgent.js.map +1 -1
- package/lib/module/core/AgentRuntime.js +18 -18
- package/lib/module/core/AgentRuntime.js.map +1 -1
- package/lib/module/core/FiberTreeWalker.js +6 -7
- package/lib/module/core/FiberTreeWalker.js.map +1 -1
- package/lib/module/core/ScreenDehydrator.js +1 -1
- package/lib/module/core/systemPrompt.js +2 -2
- package/lib/module/index.js +1 -1
- package/lib/typescript/src/components/AIAgent.d.ts +1 -3
- package/lib/typescript/src/components/AIAgent.d.ts.map +1 -1
- package/lib/typescript/src/core/AgentRuntime.d.ts +2 -2
- package/lib/typescript/src/core/FiberTreeWalker.d.ts +3 -4
- package/lib/typescript/src/core/FiberTreeWalker.d.ts.map +1 -1
- package/lib/typescript/src/core/ScreenDehydrator.d.ts +1 -1
- package/lib/typescript/src/core/systemPrompt.d.ts +2 -2
- package/lib/typescript/src/core/types.d.ts +9 -10
- package/lib/typescript/src/core/types.d.ts.map +1 -1
- package/lib/typescript/src/index.d.ts +1 -1
- package/package.json +4 -5
- package/src/components/AIAgent.tsx +60 -61
- package/src/core/AgentRuntime.ts +18 -18
- package/src/core/FiberTreeWalker.ts +8 -9
- package/src/core/ScreenDehydrator.ts +1 -1
- package/src/core/systemPrompt.ts +2 -2
- package/src/core/types.ts +15 -15
- package/src/index.ts +1 -1
- package/lib/typescript/fetch-models.d.mts +0 -2
- package/lib/typescript/fetch-models.d.mts.map +0 -1
- package/lib/typescript/list-all-models.d.mts +0 -2
- package/lib/typescript/list-all-models.d.mts.map +0 -1
- package/lib/typescript/list-models.d.mts +0 -2
- package/lib/typescript/list-models.d.mts.map +0 -1
package/src/core/AgentRuntime.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* AgentRuntime — The main agent loop
|
|
2
|
+
* AgentRuntime — The main agent loop.
|
|
3
3
|
*
|
|
4
4
|
* Flow:
|
|
5
5
|
* 1. Walk Fiber tree → detect interactive elements
|
|
@@ -52,7 +52,7 @@ export class AgentRuntime {
|
|
|
52
52
|
|
|
53
53
|
this.registerBuiltInTools();
|
|
54
54
|
|
|
55
|
-
// Apply customTools
|
|
55
|
+
// Apply customTools
|
|
56
56
|
if (config.customTools) {
|
|
57
57
|
for (const [name, tool] of Object.entries(config.customTools)) {
|
|
58
58
|
if (tool === null) {
|
|
@@ -237,7 +237,7 @@ export class AgentRuntime {
|
|
|
237
237
|
},
|
|
238
238
|
});
|
|
239
239
|
|
|
240
|
-
// ask_user — ask for clarification
|
|
240
|
+
// ask_user — ask for clarification
|
|
241
241
|
this.tools.set('ask_user', {
|
|
242
242
|
name: 'ask_user',
|
|
243
243
|
description: 'Ask the user a question and wait for their answer. Use this if you need more information or clarification.',
|
|
@@ -246,7 +246,7 @@ export class AgentRuntime {
|
|
|
246
246
|
},
|
|
247
247
|
execute: async (args) => {
|
|
248
248
|
if (this.config.onAskUser) {
|
|
249
|
-
//
|
|
249
|
+
// Block until user responds, then continue the loop
|
|
250
250
|
this.config.onStatusUpdate?.('Waiting for your answer...');
|
|
251
251
|
const answer = await this.config.onAskUser(args.question);
|
|
252
252
|
return `User answered: ${answer}`;
|
|
@@ -444,7 +444,7 @@ export class AgentRuntime {
|
|
|
444
444
|
/**
|
|
445
445
|
* Get current screen context as formatted text.
|
|
446
446
|
* Used by voice mode: sent once at connect + after each tool call.
|
|
447
|
-
*
|
|
447
|
+
* Tree goes in user prompt, not system instructions.
|
|
448
448
|
*/
|
|
449
449
|
public getScreenContext(): string {
|
|
450
450
|
try {
|
|
@@ -531,7 +531,7 @@ ${screen.elementsText}
|
|
|
531
531
|
};
|
|
532
532
|
}
|
|
533
533
|
|
|
534
|
-
// ─── Instructions
|
|
534
|
+
// ─── Instructions ───────
|
|
535
535
|
|
|
536
536
|
private getInstructions(screenName: string): string {
|
|
537
537
|
const { instructions } = this.config;
|
|
@@ -556,7 +556,7 @@ ${screen.elementsText}
|
|
|
556
556
|
return result ? `<instructions>\n${result}</instructions>\n\n` : '';
|
|
557
557
|
}
|
|
558
558
|
|
|
559
|
-
// ─── Observation System
|
|
559
|
+
// ─── Observation System ──
|
|
560
560
|
|
|
561
561
|
private observations: string[] = [];
|
|
562
562
|
private lastScreenName: string = '';
|
|
@@ -581,7 +581,7 @@ ${screen.elementsText}
|
|
|
581
581
|
}
|
|
582
582
|
}
|
|
583
583
|
|
|
584
|
-
// ─── User Prompt Assembly
|
|
584
|
+
// ─── User Prompt Assembly ──
|
|
585
585
|
|
|
586
586
|
private assembleUserPrompt(
|
|
587
587
|
step: number,
|
|
@@ -595,7 +595,7 @@ ${screen.elementsText}
|
|
|
595
595
|
// 1. <instructions> (optional system/screen instructions)
|
|
596
596
|
prompt += this.getInstructions(screenName);
|
|
597
597
|
|
|
598
|
-
// 2. <agent_state> — user request + step info
|
|
598
|
+
// 2. <agent_state> — user request + step info
|
|
599
599
|
prompt += '<agent_state>\n';
|
|
600
600
|
prompt += '<user_request>\n';
|
|
601
601
|
prompt += `${contextualMessage}\n`;
|
|
@@ -605,7 +605,7 @@ ${screen.elementsText}
|
|
|
605
605
|
prompt += '</step_info>\n';
|
|
606
606
|
prompt += '</agent_state>\n\n';
|
|
607
607
|
|
|
608
|
-
// 3. <agent_history> — structured per-step history
|
|
608
|
+
// 3. <agent_history> — structured per-step history
|
|
609
609
|
prompt += '<agent_history>\n';
|
|
610
610
|
|
|
611
611
|
let stepIndex = 0;
|
|
@@ -667,14 +667,14 @@ ${screen.elementsText}
|
|
|
667
667
|
|
|
668
668
|
logger.info('AgentRuntime', `Starting execution: "${contextualMessage}"`);
|
|
669
669
|
|
|
670
|
-
// Lifecycle: onBeforeTask
|
|
670
|
+
// Lifecycle: onBeforeTask
|
|
671
671
|
await this.config.onBeforeTask?.();
|
|
672
672
|
|
|
673
673
|
try {
|
|
674
674
|
for (let step = 0; step < maxSteps; step++) {
|
|
675
675
|
logger.info('AgentRuntime', `===== Step ${step + 1}/${maxSteps} =====`);
|
|
676
676
|
|
|
677
|
-
// Lifecycle: onBeforeStep
|
|
677
|
+
// Lifecycle: onBeforeStep
|
|
678
678
|
await this.config.onBeforeStep?.(step);
|
|
679
679
|
|
|
680
680
|
// 1. Walk Fiber tree with security config and dehydrate screen
|
|
@@ -690,16 +690,16 @@ ${screen.elementsText}
|
|
|
690
690
|
logger.info('AgentRuntime', `Screen: ${screen.screenName}`);
|
|
691
691
|
logger.debug('AgentRuntime', `Dehydrated:\n${screen.elementsText}`);
|
|
692
692
|
|
|
693
|
-
// 2. Apply transformScreenContent
|
|
693
|
+
// 2. Apply transformScreenContent
|
|
694
694
|
let screenContent = screen.elementsText;
|
|
695
695
|
if (this.config.transformScreenContent) {
|
|
696
696
|
screenContent = await this.config.transformScreenContent(screenContent);
|
|
697
697
|
}
|
|
698
698
|
|
|
699
|
-
// 3. Handle observations
|
|
699
|
+
// 3. Handle observations
|
|
700
700
|
this.handleObservations(step, maxSteps, screenName);
|
|
701
701
|
|
|
702
|
-
// 4. Assemble structured user prompt
|
|
702
|
+
// 4. Assemble structured user prompt
|
|
703
703
|
const contextMessage = this.assembleUserPrompt(
|
|
704
704
|
step, maxSteps, contextualMessage, screenName, screenContent,
|
|
705
705
|
);
|
|
@@ -709,7 +709,7 @@ ${screen.elementsText}
|
|
|
709
709
|
|
|
710
710
|
// 5. Send to AI provider
|
|
711
711
|
this.config.onStatusUpdate?.('Analyzing screen...');
|
|
712
|
-
const systemPrompt = buildSystemPrompt(
|
|
712
|
+
const systemPrompt = buildSystemPrompt('en');
|
|
713
713
|
const tools = this.buildToolsForProvider();
|
|
714
714
|
|
|
715
715
|
logger.info('AgentRuntime', `Sending to AI with ${tools.length} tools...`);
|
|
@@ -789,7 +789,7 @@ ${screen.elementsText}
|
|
|
789
789
|
};
|
|
790
790
|
this.history.push(agentStep);
|
|
791
791
|
|
|
792
|
-
// Lifecycle: onAfterStep
|
|
792
|
+
// Lifecycle: onAfterStep
|
|
793
793
|
await this.config.onAfterStep?.(this.history);
|
|
794
794
|
|
|
795
795
|
// Check if done
|
|
@@ -817,7 +817,7 @@ ${screen.elementsText}
|
|
|
817
817
|
return result;
|
|
818
818
|
}
|
|
819
819
|
|
|
820
|
-
// Step delay
|
|
820
|
+
// Step delay
|
|
821
821
|
await new Promise(resolve => setTimeout(resolve, stepDelay));
|
|
822
822
|
}
|
|
823
823
|
|
|
@@ -1,22 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* FiberTreeWalker — Traverses React's Fiber tree to discover interactive elements.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* Walks the React Native fiber tree to extract a text representation of the UI.
|
|
5
5
|
* Instead of traversing HTML nodes, we traverse React Fiber nodes and detect
|
|
6
6
|
* interactive elements by their type and props (onPress, onChangeText, etc.).
|
|
7
7
|
*
|
|
8
|
-
* Architecture inspired by: https://github.com/alibaba/page-agent
|
|
9
8
|
*/
|
|
10
9
|
|
|
11
10
|
import { logger } from '../utils/logger';
|
|
12
11
|
import type { InteractiveElement, ElementType } from './types';
|
|
13
12
|
|
|
14
|
-
// ─── Walk Configuration
|
|
13
|
+
// ─── Walk Configuration ─────────
|
|
15
14
|
|
|
16
15
|
export interface WalkConfig {
|
|
17
|
-
/** React refs of elements to exclude
|
|
16
|
+
/** React refs of elements to exclude */
|
|
18
17
|
interactiveBlacklist?: React.RefObject<any>[];
|
|
19
|
-
/** If set, only these elements are interactive
|
|
18
|
+
/** If set, only these elements are interactive */
|
|
20
19
|
interactiveWhitelist?: React.RefObject<any>[];
|
|
21
20
|
}
|
|
22
21
|
|
|
@@ -53,14 +52,14 @@ const RN_INTERNAL_NAMES = new Set([
|
|
|
53
52
|
'AnimatedComponentWrapper', 'Animated',
|
|
54
53
|
]);
|
|
55
54
|
|
|
56
|
-
// ─── State Extraction
|
|
55
|
+
// ─── State Extraction ──
|
|
57
56
|
|
|
58
57
|
/** Props to extract as state attributes — covers lazy devs who skip accessibility */
|
|
59
58
|
const STATE_PROPS = ['value', 'checked', 'selected', 'active', 'on', 'isOn', 'toggled', 'enabled'];
|
|
60
59
|
|
|
61
60
|
/**
|
|
62
61
|
* Extract state attributes from a fiber node's props.
|
|
63
|
-
*
|
|
62
|
+
* Extracts meaningful state from a fiber node.
|
|
64
63
|
* Priority: accessibilityState > accessibilityRole > direct scalar props.
|
|
65
64
|
*/
|
|
66
65
|
function extractStateAttributes(props: any): string {
|
|
@@ -319,7 +318,7 @@ function getFiberFromRef(ref: any): any | null {
|
|
|
319
318
|
|
|
320
319
|
/**
|
|
321
320
|
* Check if a Fiber node matches any ref in the given list.
|
|
322
|
-
*
|
|
321
|
+
* Checks if an element is excluded from AI interaction
|
|
323
322
|
* We compare the Fiber's stateNode (native instance) against ref.current.
|
|
324
323
|
*/
|
|
325
324
|
function matchesRefList(node: any, refs?: React.RefObject<any>[]): boolean {
|
|
@@ -402,7 +401,7 @@ export function walkFiberTree(rootRef: any, config?: WalkConfig): WalkResult {
|
|
|
402
401
|
props: { ...props },
|
|
403
402
|
});
|
|
404
403
|
|
|
405
|
-
// Build output tag with state attributes
|
|
404
|
+
// Build output tag with state attributes
|
|
406
405
|
const stateAttrs = extractStateAttributes(props);
|
|
407
406
|
const attrStr = stateAttrs ? ` ${stateAttrs}` : '';
|
|
408
407
|
const textContent = label || '';
|
package/src/core/systemPrompt.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* System prompt for the AI agent
|
|
2
|
+
* System prompt for the AI agent.
|
|
3
3
|
*
|
|
4
4
|
* Separated into its own file for maintainability.
|
|
5
|
-
* The prompt uses XML-style tags
|
|
5
|
+
* The prompt uses XML-style tags
|
|
6
6
|
* to give the LLM clear, structured instructions.
|
|
7
7
|
*/
|
|
8
8
|
|
package/src/core/types.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Core types for the
|
|
2
|
+
* Core types for the React Native AI SDK.
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
// ─── Agent Modes ──────────────────────────────────────────────
|
|
@@ -61,27 +61,27 @@ export interface AgentStep {
|
|
|
61
61
|
export interface AgentConfig {
|
|
62
62
|
apiKey: string;
|
|
63
63
|
model?: string;
|
|
64
|
-
language?: 'en' | 'ar';
|
|
65
64
|
|
|
66
|
-
|
|
65
|
+
|
|
66
|
+
/** Maximum steps per task */
|
|
67
67
|
maxSteps?: number;
|
|
68
68
|
|
|
69
|
-
// ─── Element Gating
|
|
69
|
+
// ─── Element Gating ──
|
|
70
70
|
|
|
71
71
|
/**
|
|
72
72
|
* React refs of elements the AI must NOT interact with.
|
|
73
|
-
*
|
|
73
|
+
* Refs of elements the AI must NOT interact with.
|
|
74
74
|
* The Fiber walker skips any node whose ref matches one in this list.
|
|
75
75
|
*/
|
|
76
76
|
interactiveBlacklist?: React.RefObject<any>[];
|
|
77
77
|
|
|
78
78
|
/**
|
|
79
79
|
* If set, the AI can ONLY interact with these elements.
|
|
80
|
-
*
|
|
80
|
+
* If set, AI can ONLY interact with these elements.
|
|
81
81
|
*/
|
|
82
82
|
interactiveWhitelist?: React.RefObject<any>[];
|
|
83
83
|
|
|
84
|
-
// ─── Lifecycle Hooks
|
|
84
|
+
// ─── Lifecycle Hooks ───────
|
|
85
85
|
|
|
86
86
|
/** Called before each agent step. */
|
|
87
87
|
onBeforeStep?: (stepCount: number) => Promise<void> | void;
|
|
@@ -95,25 +95,25 @@ export interface AgentConfig {
|
|
|
95
95
|
/** Called after task completes (success or failure). */
|
|
96
96
|
onAfterTask?: (result: ExecutionResult) => Promise<void> | void;
|
|
97
97
|
|
|
98
|
-
// ─── Content Masking
|
|
98
|
+
// ─── Content Masking ──────────
|
|
99
99
|
|
|
100
100
|
/**
|
|
101
101
|
* Transform dehydrated screen text before sending to LLM.
|
|
102
102
|
* Use to mask sensitive data (credit cards, PII, etc).
|
|
103
|
-
*
|
|
103
|
+
* Transform screen content before the LLM sees it (for data masking).
|
|
104
104
|
*/
|
|
105
105
|
transformScreenContent?: (content: string) => Promise<string> | string;
|
|
106
106
|
|
|
107
|
-
// ─── Custom Tools
|
|
107
|
+
// ─── Custom Tools ─────────────────────
|
|
108
108
|
|
|
109
109
|
/**
|
|
110
110
|
* Override or remove built-in tools.
|
|
111
111
|
* Set tool to `null` to remove it (e.g., `{ navigate: null }`).
|
|
112
|
-
*
|
|
112
|
+
* Override or remove built-in tools (null = remove).
|
|
113
113
|
*/
|
|
114
114
|
customTools?: Record<string, ToolDefinition | null>;
|
|
115
115
|
|
|
116
|
-
// ─── Instructions
|
|
116
|
+
// ─── Instructions ────────────────────
|
|
117
117
|
|
|
118
118
|
/** Instructions to guide the agent's behavior. */
|
|
119
119
|
instructions?: {
|
|
@@ -122,12 +122,12 @@ export interface AgentConfig {
|
|
|
122
122
|
/**
|
|
123
123
|
* Dynamic per-screen instructions callback.
|
|
124
124
|
* Called before each step to get instructions for the current screen.
|
|
125
|
-
*
|
|
125
|
+
* Per-screen instructions callback.
|
|
126
126
|
*/
|
|
127
127
|
getScreenInstructions?: (screenName: string) => string | undefined | null;
|
|
128
128
|
};
|
|
129
129
|
|
|
130
|
-
/** Delay between steps in ms
|
|
130
|
+
/** Delay between steps in ms. */
|
|
131
131
|
stepDelay?: number;
|
|
132
132
|
|
|
133
133
|
// ─── Status Updates ──────────────────────────────────────────────────────
|
|
@@ -146,7 +146,7 @@ export interface AgentConfig {
|
|
|
146
146
|
|
|
147
147
|
/**
|
|
148
148
|
* Callback for when agent needs user input (ask_user tool).
|
|
149
|
-
*
|
|
149
|
+
* The agent loop blocks until the user responds.
|
|
150
150
|
* If not set, ask_user tool will break the loop (legacy behavior).
|
|
151
151
|
* @example onAskUser: (q) => new Promise(resolve => showPrompt(q, resolve))
|
|
152
152
|
*/
|
package/src/index.ts
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-models.d.mts","sourceRoot":"","sources":["../../fetch-models.mjs"],"names":[],"mappings":""}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"list-all-models.d.mts","sourceRoot":"","sources":["../../list-all-models.mjs"],"names":[],"mappings":""}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"list-models.d.mts","sourceRoot":"","sources":["../../list-models.mjs"],"names":[],"mappings":""}
|