mobile-device-mcp 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +90 -46
- package/dist/ai/analyzer.d.ts +85 -1
- package/dist/ai/analyzer.d.ts.map +1 -1
- package/dist/ai/analyzer.js +406 -17
- package/dist/ai/analyzer.js.map +1 -1
- package/dist/ai/element-search.d.ts.map +1 -1
- package/dist/ai/element-search.js +197 -29
- package/dist/ai/element-search.js.map +1 -1
- package/dist/drivers/android/adb.d.ts +7 -0
- package/dist/drivers/android/adb.d.ts.map +1 -1
- package/dist/drivers/android/adb.js +13 -1
- package/dist/drivers/android/adb.js.map +1 -1
- package/dist/drivers/android/companion-client.d.ts +60 -0
- package/dist/drivers/android/companion-client.d.ts.map +1 -0
- package/dist/drivers/android/companion-client.js +232 -0
- package/dist/drivers/android/companion-client.js.map +1 -0
- package/dist/drivers/android/index.d.ts +92 -0
- package/dist/drivers/android/index.d.ts.map +1 -1
- package/dist/drivers/android/index.js +371 -21
- package/dist/drivers/android/index.js.map +1 -1
- package/dist/drivers/flutter/index.d.ts +219 -0
- package/dist/drivers/flutter/index.d.ts.map +1 -0
- package/dist/drivers/flutter/index.js +670 -0
- package/dist/drivers/flutter/index.js.map +1 -0
- package/dist/drivers/flutter/vm-service.d.ts +147 -0
- package/dist/drivers/flutter/vm-service.d.ts.map +1 -0
- package/dist/drivers/flutter/vm-service.js +221 -0
- package/dist/drivers/flutter/vm-service.js.map +1 -0
- package/dist/drivers/ios/index.d.ts +49 -0
- package/dist/drivers/ios/index.d.ts.map +1 -0
- package/dist/drivers/ios/index.js +236 -0
- package/dist/drivers/ios/index.js.map +1 -0
- package/dist/drivers/ios/simctl.d.ts +12 -0
- package/dist/drivers/ios/simctl.d.ts.map +1 -0
- package/dist/drivers/ios/simctl.js +53 -0
- package/dist/drivers/ios/simctl.js.map +1 -0
- package/dist/recording/generator.d.ts +4 -0
- package/dist/recording/generator.d.ts.map +1 -0
- package/dist/recording/generator.js +128 -0
- package/dist/recording/generator.js.map +1 -0
- package/dist/recording/recorder.d.ts +21 -0
- package/dist/recording/recorder.d.ts.map +1 -0
- package/dist/recording/recorder.js +43 -0
- package/dist/recording/recorder.js.map +1 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +12 -2
- package/dist/server.js.map +1 -1
- package/dist/tools/ai-tools.d.ts.map +1 -1
- package/dist/tools/ai-tools.js +103 -0
- package/dist/tools/ai-tools.js.map +1 -1
- package/dist/tools/app-tools.js +1 -1
- package/dist/tools/app-tools.js.map +1 -1
- package/dist/tools/flutter-tools.d.ts +10 -0
- package/dist/tools/flutter-tools.d.ts.map +1 -0
- package/dist/tools/flutter-tools.js +407 -0
- package/dist/tools/flutter-tools.js.map +1 -0
- package/dist/tools/index.d.ts +11 -2
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +62 -8
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/interaction-tools.js +10 -10
- package/dist/tools/interaction-tools.js.map +1 -1
- package/dist/tools/ios-tools.d.ts +4 -0
- package/dist/tools/ios-tools.d.ts.map +1 -0
- package/dist/tools/ios-tools.js +122 -0
- package/dist/tools/ios-tools.js.map +1 -0
- package/dist/tools/log-tools.js +1 -1
- package/dist/tools/log-tools.js.map +1 -1
- package/dist/tools/recording-tools.d.ts +4 -0
- package/dist/tools/recording-tools.d.ts.map +1 -0
- package/dist/tools/recording-tools.js +138 -0
- package/dist/tools/recording-tools.js.map +1 -0
- package/dist/tools/screen-tools.js +3 -3
- package/dist/tools/screen-tools.js.map +1 -1
- package/dist/tools/video-tools.d.ts +4 -0
- package/dist/tools/video-tools.d.ts.map +1 -0
- package/dist/tools/video-tools.js +101 -0
- package/dist/tools/video-tools.js.map +1 -0
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +6 -2
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# mobile-device-mcp
|
|
2
2
|
|
|
3
|
-
MCP server that gives AI coding assistants (Claude Code, Cursor, Windsurf) the ability to **see and interact with mobile devices**.
|
|
3
|
+
MCP server that gives AI coding assistants (Claude Code, Cursor, Windsurf) the ability to **see and interact with mobile devices**. 34 tools for screenshots, UI inspection, touch interaction, AI-powered visual analysis, and Flutter widget tree inspection.
|
|
4
4
|
|
|
5
5
|
> AI assistants can read your code but can't see your phone. This fixes that.
|
|
6
6
|
|
|
@@ -28,57 +28,68 @@ Without this tool: With this tool:
|
|
|
28
28
|
- Android device/emulator connected via ADB
|
|
29
29
|
- ADB installed (Android SDK Platform Tools)
|
|
30
30
|
|
|
31
|
-
###
|
|
31
|
+
### Setup (One-time, 30 seconds)
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
# Clone and build
|
|
35
|
-
git clone https://github.com/saranshbamania/mobile-device-mcp.git
|
|
36
|
-
cd mobile-device-mcp
|
|
37
|
-
npm install
|
|
38
|
-
npm run build
|
|
33
|
+
1. **Get a Google AI key** (free tier available): [aistudio.google.com/apikey](https://aistudio.google.com/apikey)
|
|
39
34
|
|
|
40
|
-
|
|
41
|
-
node dist/index.js
|
|
42
|
-
```
|
|
43
|
-
|
|
44
|
-
### Configure with Claude Code
|
|
45
|
-
|
|
46
|
-
Add to your Claude Code MCP settings (`~/.claude/settings.json`):
|
|
35
|
+
2. **Add `.mcp.json` to your project root:**
|
|
47
36
|
|
|
48
37
|
```json
|
|
49
38
|
{
|
|
50
39
|
"mcpServers": {
|
|
51
40
|
"mobile-device": {
|
|
52
|
-
"
|
|
53
|
-
"
|
|
41
|
+
"type": "stdio",
|
|
42
|
+
"command": "npx",
|
|
43
|
+
"args": ["-y", "mobile-device-mcp"],
|
|
54
44
|
"env": {
|
|
55
|
-
"GOOGLE_API_KEY": "your-google-api-key"
|
|
56
|
-
"ANTHROPIC_API_KEY": "your-anthropic-api-key"
|
|
45
|
+
"GOOGLE_API_KEY": "your-google-api-key"
|
|
57
46
|
}
|
|
58
47
|
}
|
|
59
48
|
}
|
|
60
49
|
}
|
|
61
50
|
```
|
|
62
51
|
|
|
63
|
-
|
|
52
|
+
3. **Open your AI coding assistant** from that directory. That's it.
|
|
64
53
|
|
|
65
|
-
|
|
54
|
+
The server starts and stops automatically — you never run it manually. Your AI assistant manages it as a background process via the MCP protocol.
|
|
55
|
+
|
|
56
|
+
### Verify It Works
|
|
57
|
+
|
|
58
|
+
**Claude Code:** type `/mcp` — you should see `mobile-device: Connected`
|
|
59
|
+
|
|
60
|
+
**Cursor:** check MCP panel in settings
|
|
61
|
+
|
|
62
|
+
Then just talk to your phone:
|
|
66
63
|
|
|
67
|
-
```json
|
|
68
|
-
{
|
|
69
|
-
"mcpServers": {
|
|
70
|
-
"mobile-device": {
|
|
71
|
-
"command": "node",
|
|
72
|
-
"args": ["/path/to/mobile-device-mcp/dist/index.js"],
|
|
73
|
-
"env": {
|
|
74
|
-
"GOOGLE_API_KEY": "your-google-api-key"
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
64
|
```
|
|
65
|
+
You: "Open my app, tap the login button, type test@email.com in the email field"
|
|
66
|
+
AI: [takes screenshot → sees the screen → smart_tap("login button") → smart_type("email field", "test@email.com")]
|
|
80
67
|
|
|
81
|
-
|
|
68
|
+
You: "Find all the bugs on this screen"
|
|
69
|
+
AI: [analyze_screen → inspects layout, checks for overflow, missing labels, broken states]
|
|
70
|
+
|
|
71
|
+
You: "Navigate to settings and verify dark mode works"
|
|
72
|
+
AI: [smart_tap("settings") → take_screenshot → smart_tap("dark mode toggle") → visual_diff → reports result]
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
No test scripts. No manual screenshots. Just describe what you want in plain English.
|
|
76
|
+
|
|
77
|
+
### Works with Any AI Coding Assistant
|
|
78
|
+
|
|
79
|
+
| Tool | Config file | Docs |
|
|
80
|
+
|------|------------|------|
|
|
81
|
+
| **Claude Code** | `.mcp.json` in project root | [claude.ai/docs](https://claude.ai/docs) |
|
|
82
|
+
| **Cursor** | `.cursor/mcp.json` | [cursor.com/docs](https://cursor.com/docs) |
|
|
83
|
+
| **VS Code + Copilot** | MCP settings | [code.visualstudio.com](https://code.visualstudio.com) |
|
|
84
|
+
| **Windsurf** | MCP settings | [windsurf.com](https://windsurf.com) |
|
|
85
|
+
|
|
86
|
+
All use the same JSON config — just put it in the right file for your editor.
|
|
87
|
+
|
|
88
|
+
### Drop Into Any Project
|
|
89
|
+
|
|
90
|
+
Copy `.mcp.json` into any mobile project — Flutter, React Native, Kotlin, Swift — and your AI assistant gets device superpowers in that directory. No global install needed.
|
|
91
|
+
|
|
92
|
+
## Tools (34 total)
|
|
82
93
|
|
|
83
94
|
### Phase 1 — Device Control (18 tools)
|
|
84
95
|
|
|
@@ -87,7 +98,7 @@ Add to `.cursor/mcp.json`:
|
|
|
87
98
|
| `list_devices` | List all connected Android devices/emulators |
|
|
88
99
|
| `get_device_info` | Model, manufacturer, Android version, SDK level |
|
|
89
100
|
| `get_screen_size` | Screen resolution in pixels |
|
|
90
|
-
| `take_screenshot` | Capture
|
|
101
|
+
| `take_screenshot` | Capture screenshot (PNG or JPEG, configurable quality & resize) |
|
|
91
102
|
| `get_ui_elements` | Get the accessibility/UI element tree as structured JSON |
|
|
92
103
|
| `tap` | Tap at coordinates |
|
|
93
104
|
| `double_tap` | Double tap at coordinates |
|
|
@@ -118,18 +129,43 @@ These tools use AI vision (Claude or Gemini) to understand what's on screen. Req
|
|
|
118
129
|
| `extract_text` | Extract all visible text from the screen (AI-powered OCR) |
|
|
119
130
|
| `verify_screen` | Verify an assertion: *"the login was successful"*, *"error message is showing"* |
|
|
120
131
|
|
|
132
|
+
### Phase 3 — Flutter Widget Tree (8 tools)
|
|
133
|
+
|
|
134
|
+
These tools connect to a running Flutter app in debug/profile mode via the Dart VM Service Protocol. Maps every widget to its source code location (`file:line`).
|
|
135
|
+
|
|
136
|
+
| Tool | What it does |
|
|
137
|
+
|------|-------------|
|
|
138
|
+
| `flutter_connect` | Discover and connect to a running Flutter app on the device |
|
|
139
|
+
| `flutter_disconnect` | Disconnect from the Flutter app and clean up resources |
|
|
140
|
+
| `flutter_get_widget_tree` | Get the full widget tree (summary or detailed) |
|
|
141
|
+
| `flutter_get_widget_details` | Get detailed properties of a specific widget by ID |
|
|
142
|
+
| `flutter_find_widget` | Search the widget tree by type, text, or description |
|
|
143
|
+
| `flutter_get_source_map` | Map every widget to its source code location (file:line:column) |
|
|
144
|
+
| `flutter_screenshot_widget` | Screenshot a specific widget in isolation |
|
|
145
|
+
| `flutter_debug_paint` | Toggle debug paint overlay (shows widget boundaries & padding) |
|
|
146
|
+
|
|
147
|
+
## Performance
|
|
148
|
+
|
|
149
|
+
The server is optimized to minimize latency and AI token costs:
|
|
150
|
+
|
|
151
|
+
- **3-tier element search**: local text match (<1ms) → cached AI → fresh AI. `smart_tap` is 37x faster than naive AI calls.
|
|
152
|
+
- **Screenshot compression**: AI tools auto-compress to JPEG q=80, 720w — **65% smaller** (251KB → 88KB) with zero quality loss. Saves ~55K tokens per screenshot.
|
|
153
|
+
- **Parallel capture**: Screenshot + UI tree fetched simultaneously via `Promise.all()`.
|
|
154
|
+
- **TTL caching**: 3-second cache avoids redundant ADB calls for rapid-fire tool usage.
|
|
155
|
+
|
|
121
156
|
## Environment Variables
|
|
122
157
|
|
|
123
158
|
| Variable | Description | Default |
|
|
124
159
|
|----------|-------------|---------|
|
|
125
160
|
| `ANTHROPIC_API_KEY` | Anthropic API key for Claude vision | — |
|
|
126
|
-
| `GOOGLE_API_KEY` or `GEMINI_API_KEY` | Google API key for Gemini vision | — |
|
|
161
|
+
| `GOOGLE_API_KEY` or `GEMINI_API_KEY` | Google API key for Gemini vision (recommended — cheapest) | — |
|
|
127
162
|
| `MCP_AI_PROVIDER` | Force AI provider: `"anthropic"` or `"google"` | Auto-detected |
|
|
128
|
-
| `MCP_AI_MODEL` | Override AI model | `
|
|
163
|
+
| `MCP_AI_MODEL` | Override AI model | `gemini-2.5-flash` / `claude-sonnet-4-20250514` |
|
|
129
164
|
| `MCP_ADB_PATH` | Custom ADB binary path | Auto-discovered |
|
|
130
165
|
| `MCP_DEFAULT_DEVICE` | Default device serial | Auto-discovered |
|
|
131
|
-
| `MCP_SCREENSHOT_FORMAT` | `"png"` or `"jpeg"` | `
|
|
166
|
+
| `MCP_SCREENSHOT_FORMAT` | `"png"` or `"jpeg"` | `jpeg` |
|
|
132
167
|
| `MCP_SCREENSHOT_QUALITY` | JPEG quality (1-100) | `80` |
|
|
168
|
+
| `MCP_SCREENSHOT_MAX_WIDTH` | Resize screenshots to this max width | `720` |
|
|
133
169
|
| `MCP_AI_SCREENSHOT` | Send screenshots to AI (`"true"`/`"false"`) | `true` |
|
|
134
170
|
| `MCP_AI_UITREE` | Send UI tree to AI (`"true"`/`"false"`) | `true` |
|
|
135
171
|
|
|
@@ -149,14 +185,19 @@ src/
|
|
|
149
185
|
│ ├── interaction-tools.ts # Touch, type, keys
|
|
150
186
|
│ ├── app-tools.ts # App management
|
|
151
187
|
│ ├── log-tools.ts # Logcat
|
|
152
|
-
│
|
|
188
|
+
│ ├── ai-tools.ts # AI-powered tools
|
|
189
|
+
│ └── flutter-tools.ts # Flutter widget inspection tools
|
|
190
|
+
├── drivers/flutter/ # Dart VM Service driver
|
|
191
|
+
│ ├── index.ts # FlutterDriver (discovery, inspection, source mapping)
|
|
192
|
+
│ └── vm-service.ts # JSON-RPC 2.0 WebSocket client (DDS redirect handling)
|
|
153
193
|
├── ai/ # AI visual analysis engine
|
|
154
194
|
│ ├── client.ts # Multi-provider client (Anthropic + Google)
|
|
155
195
|
│ ├── prompts.ts # System prompts & UI element summarizer
|
|
156
|
-
│
|
|
196
|
+
│ ├── analyzer.ts # ScreenAnalyzer orchestrator
|
|
197
|
+
│ └── element-search.ts # Local element search (no AI needed)
|
|
157
198
|
└── utils/
|
|
158
199
|
├── discovery.ts # ADB auto-discovery
|
|
159
|
-
└── image.ts # PNG parsing
|
|
200
|
+
└── image.ts # PNG parsing, JPEG compression, bilinear resize
|
|
160
201
|
```
|
|
161
202
|
|
|
162
203
|
## Roadmap
|
|
@@ -164,17 +205,20 @@ src/
|
|
|
164
205
|
- [x] Phase 1: Android ADB device control (18 tools)
|
|
165
206
|
- [x] Phase 2: AI visual analysis layer (8 tools)
|
|
166
207
|
- [x] Multi-provider AI (Anthropic Claude + Google Gemini)
|
|
167
|
-
- [
|
|
208
|
+
- [x] Performance optimization (3-tier search, caching, parallel capture)
|
|
209
|
+
- [x] Screenshot compression pipeline (JPEG, resize, configurable quality)
|
|
210
|
+
- [x] npm publish (`npx mobile-device-mcp`)
|
|
211
|
+
- [x] Phase 3: Flutter widget tree integration (8 tools, Dart VM Service Protocol)
|
|
168
212
|
- [ ] Phase 4: iOS support (simulators via xcrun simctl, devices via idevice)
|
|
169
213
|
- [ ] Phase 5: Monetization (license keys, usage analytics)
|
|
170
|
-
- [ ] npm publish (`npx mobile-device-mcp`)
|
|
171
|
-
- [ ] Screenshot compression pipeline (JPEG, thumbnail mode)
|
|
172
214
|
- [ ] Multi-device orchestration
|
|
173
215
|
|
|
174
216
|
## Tested On
|
|
175
217
|
|
|
176
|
-
- Pixel 8, Android 16, SDK 36 —
|
|
177
|
-
-
|
|
218
|
+
- Pixel 8, Android 16, SDK 36 — 44/44 tests passed (22 device + 10 AI + 12 Flutter)
|
|
219
|
+
- Flutter 3.41.3, metroping app (debug mode)
|
|
220
|
+
- Google Gemini 2.5 Flash
|
|
221
|
+
- Windows 11 + wireless ADB
|
|
178
222
|
|
|
179
223
|
## License
|
|
180
224
|
|
package/dist/ai/analyzer.d.ts
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import type { ScreenAnalysis, ElementMatch, VisualDiff, ActionPlan, ScreenVerification, AnalyzedElement, DeviceDriver, ScreenshotOptions } from "../types.js";
|
|
2
2
|
import { AIClient } from "./client.js";
|
|
3
|
+
import type { FlutterDriver } from "../drivers/flutter/index.js";
|
|
3
4
|
export declare class ScreenAnalyzer {
|
|
4
5
|
private client;
|
|
5
6
|
private driver;
|
|
6
7
|
private config;
|
|
8
|
+
private flutterDriver?;
|
|
7
9
|
private cache;
|
|
8
10
|
private cacheTTL;
|
|
11
|
+
private pendingPrefetch;
|
|
12
|
+
/** Track when getUIElements returns empty (Flutter apps) to avoid redundant calls. */
|
|
13
|
+
private lastUIElementsEmpty;
|
|
9
14
|
/** Screenshot options used for AI analysis (compressed for performance). */
|
|
10
15
|
private screenshotOptions;
|
|
11
16
|
constructor(client: AIClient, driver: DeviceDriver, config: {
|
|
@@ -15,7 +20,7 @@ export declare class ScreenAnalyzer {
|
|
|
15
20
|
apiKey: string;
|
|
16
21
|
model: string;
|
|
17
22
|
maxTokens: number;
|
|
18
|
-
}, screenshotOptions?: ScreenshotOptions);
|
|
23
|
+
}, screenshotOptions?: ScreenshotOptions, flutterDriver?: FlutterDriver | undefined);
|
|
19
24
|
/**
|
|
20
25
|
* Analyze the current screen state — identifies elements, screen
|
|
21
26
|
* type, visible text, and actionable suggestions.
|
|
@@ -64,6 +69,70 @@ export declare class ScreenAnalyzer {
|
|
|
64
69
|
field: AnalyzedElement | null;
|
|
65
70
|
message: string;
|
|
66
71
|
}>;
|
|
72
|
+
/**
|
|
73
|
+
* Detect and handle system dialogs / popups (permission requests,
|
|
74
|
+
* update prompts, system alerts). Can dismiss, accept, or auto-handle.
|
|
75
|
+
*/
|
|
76
|
+
handlePopup(deviceId: string, action?: "dismiss" | "accept" | "auto"): Promise<{
|
|
77
|
+
found: boolean;
|
|
78
|
+
popup_type?: string;
|
|
79
|
+
action_taken?: string;
|
|
80
|
+
message: string;
|
|
81
|
+
}>;
|
|
82
|
+
/**
|
|
83
|
+
* Fill multiple form fields in a single operation. For each field,
|
|
84
|
+
* finds the matching input element, clears it, and types the new value.
|
|
85
|
+
*/
|
|
86
|
+
fillForm(deviceId: string, fields: Record<string, string>): Promise<{
|
|
87
|
+
success: boolean;
|
|
88
|
+
filled: {
|
|
89
|
+
field: string;
|
|
90
|
+
success: boolean;
|
|
91
|
+
message: string;
|
|
92
|
+
}[];
|
|
93
|
+
message: string;
|
|
94
|
+
}>;
|
|
95
|
+
/**
|
|
96
|
+
* Wait for a specific element to appear on screen.
|
|
97
|
+
* More reliable and faster than generic wait_for_settle — returns
|
|
98
|
+
* as soon as the target element is found instead of requiring
|
|
99
|
+
* two identical consecutive UI dumps.
|
|
100
|
+
*/
|
|
101
|
+
waitForElement(deviceId: string, query: string, options?: {
|
|
102
|
+
/** Max time to wait in ms (default: 5000) */
|
|
103
|
+
timeout?: number;
|
|
104
|
+
/** Interval between polls in ms (default: 300) */
|
|
105
|
+
pollInterval?: number;
|
|
106
|
+
}): Promise<{
|
|
107
|
+
found: boolean;
|
|
108
|
+
element?: AnalyzedElement;
|
|
109
|
+
confidence?: number;
|
|
110
|
+
polls: number;
|
|
111
|
+
elapsed_ms: number;
|
|
112
|
+
message: string;
|
|
113
|
+
}>;
|
|
114
|
+
/**
|
|
115
|
+
* Wait for the screen to settle after a navigation or action.
|
|
116
|
+
* Polls the UI tree until two consecutive dumps are structurally identical,
|
|
117
|
+
* meaning the screen has stopped animating/loading.
|
|
118
|
+
*
|
|
119
|
+
* @param deviceId - Device ID
|
|
120
|
+
* @param options - Configuration for the settle detection
|
|
121
|
+
* @returns Info about whether the screen settled and how long it took
|
|
122
|
+
*/
|
|
123
|
+
waitForScreenSettle(deviceId: string, options?: {
|
|
124
|
+
/** Max time to wait in ms (default: 3000) */
|
|
125
|
+
timeout?: number;
|
|
126
|
+
/** Interval between polls in ms (default: 500) */
|
|
127
|
+
pollInterval?: number;
|
|
128
|
+
/** Minimum wait before first poll in ms (default: 300) */
|
|
129
|
+
initialDelay?: number;
|
|
130
|
+
}): Promise<{
|
|
131
|
+
settled: boolean;
|
|
132
|
+
polls: number;
|
|
133
|
+
elapsed_ms: number;
|
|
134
|
+
message: string;
|
|
135
|
+
}>;
|
|
67
136
|
/**
|
|
68
137
|
* Fetch UI elements for the device, using the cache when valid.
|
|
69
138
|
* This is a lightweight call (no screenshot) used by the local
|
|
@@ -86,6 +155,11 @@ export declare class ScreenAnalyzer {
|
|
|
86
155
|
* expensive uiautomator dump calls on every interaction.
|
|
87
156
|
*/
|
|
88
157
|
private invalidateCache;
|
|
158
|
+
/**
|
|
159
|
+
* Pre-fetch screenshot + UI tree in the background after an action.
|
|
160
|
+
* The next findElement/captureContext call will use this pre-fetched data.
|
|
161
|
+
*/
|
|
162
|
+
private startPrefetch;
|
|
89
163
|
/**
|
|
90
164
|
* Throw if the AI client is not available (no API key configured).
|
|
91
165
|
*/
|
|
@@ -94,5 +168,15 @@ export declare class ScreenAnalyzer {
|
|
|
94
168
|
* Promise-based delay helper.
|
|
95
169
|
*/
|
|
96
170
|
private delay;
|
|
171
|
+
/**
|
|
172
|
+
* Create a structural hash of the UI tree for comparison.
|
|
173
|
+
* Compares element count, text content, and bounds positions.
|
|
174
|
+
* Excludes status bar elements (top 72px) to ignore clock changes.
|
|
175
|
+
*/
|
|
176
|
+
private hashUITree;
|
|
177
|
+
/**
|
|
178
|
+
* Flatten a nested UIElement tree into a flat array.
|
|
179
|
+
*/
|
|
180
|
+
private flattenUIElements;
|
|
97
181
|
}
|
|
98
182
|
//# sourceMappingURL=analyzer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../../src/ai/analyzer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,kBAAkB,EAClB,eAAe,EAEf,YAAY,EACZ,iBAAiB,EAGlB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../../src/ai/analyzer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,kBAAkB,EAClB,eAAe,EAEf,YAAY,EACZ,iBAAiB,EAGlB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAYvC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAqKjE,qBAAa,cAAc;IAevB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IASd,OAAO,CAAC,aAAa,CAAC;IAzBxB,OAAO,CAAC,KAAK,CAGN;IACP,OAAO,CAAC,QAAQ,CAAgB;IAChC,OAAO,CAAC,eAAe,CAA8B;IAErD,sFAAsF;IACtF,OAAO,CAAC,mBAAmB,CAAkB;IAE7C,4EAA4E;IAC5E,OAAO,CAAC,iBAAiB,CAAoB;gBAGnC,MAAM,EAAE,QAAQ,EAChB,MAAM,EAAE,YAAY,EACpB,MAAM,EAAE;QACd,QAAQ,EAAE,WAAW,GAAG,QAAQ,CAAC;QACjC,qBAAqB,EAAE,OAAO,CAAC;QAC/B,iBAAiB,EAAE,OAAO,CAAC;QAC3B,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;KACnB,EACD,iBAAiB,CAAC,EAAE,iBAAiB,EAC7B,aAAa,CAAC,EAAE,aAAa,YAAA;IAcvC;;;OAGG;IACG,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAsB9D;;;;;;OAMG;IACG,WAAW,CACf,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,YAAY,CAAC;IA+ExB;;OAEG;IACG,cAAc,CAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,UAAU,CAAC;IAmCtB;;;OAGG;IACG,kBAAkB,CACtB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,UAAU,CAAC;IAkCtB;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA+BtD;;OAEG;IACG,YAAY,CAChB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,kBAAkB,CAAC;IAsB9B;;;OAGG;IACG,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,kBAAkB,EAAE,MAAM,GACzB,OAAO,CAAC;QACT,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,eAAe,GAAG,IAAI,CAAC;QAC/B,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IAsCF;;;OAGG;IACG,SAAS,CACb,QAAQ,EAAE,MAAM,EAChB,gBAAgB,EAAE,MAAM,EACxB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC;QACT,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,EAAE,eAAe,GAAG,IAAI,CAAC;QAC9B,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IAuCF;;;OAGG;IACG,WAAW,CACf,QAAQ,EAAE,MAAM,EAChB,MAAM,GAAE,SAAS,GAAG,QAAQ,GAAG,MAAe,GAC7C,OAAO,CAAC;QACT,KAAK,EAAE,OAAO,CAAC;QACf,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IA6FF;;;OAGG;IACG,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC7B,OAAO,CAAC;QACT,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,OAAO,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAC/D,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IA8DF;;;;;OAKG;IACG,cAAc,CAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;QACP,6CAA6C;QAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,kDAAkD;QAClD,YAAY,CAAC,EAAE,MAAM,CAAC;KAClB,GACL,OAAO,CAAC;QACT,KAAK,EAAE,OAAO,CAAC;QACf,OAAO,CAAC,EAAE,eAAe,CAAC;QAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IAkDF;;;;;;;;OAQG;IACG,mBAAmB,CACvB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;QACP,6CAA6C;QAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,kDAAkD;QAClD,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,0DAA0D;QAC1D,YAAY,CAAC,EAAE,MAAM,CAAC;KAClB,GACL,OAAO,CAAC;QACT,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IA2DF;;;;OAIG;YACW,aAAa;IA+B3B;;;OAGG;YACW,cAAc;IA8D5B;;OAEG;IACH,OAAO,CAAC,YAAY;IAIpB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAWvB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAQ7B;;OAEG;IACH,OAAO,CAAC,KAAK;IAIb;;;;OAIG;IACH,OAAO,CAAC,UAAU;IAmBlB;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAY1B"}
|