mobile-device-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +181 -0
- package/dist/ai/analyzer.d.ts +98 -0
- package/dist/ai/analyzer.d.ts.map +1 -0
- package/dist/ai/analyzer.js +451 -0
- package/dist/ai/analyzer.js.map +1 -0
- package/dist/ai/client.d.ts +92 -0
- package/dist/ai/client.d.ts.map +1 -0
- package/dist/ai/client.js +281 -0
- package/dist/ai/client.js.map +1 -0
- package/dist/ai/element-search.d.ts +12 -0
- package/dist/ai/element-search.d.ts.map +1 -0
- package/dist/ai/element-search.js +387 -0
- package/dist/ai/element-search.js.map +1 -0
- package/dist/ai/prompts.d.ts +27 -0
- package/dist/ai/prompts.d.ts.map +1 -0
- package/dist/ai/prompts.js +153 -0
- package/dist/ai/prompts.js.map +1 -0
- package/dist/drivers/android/adb.d.ts +21 -0
- package/dist/drivers/android/adb.d.ts.map +1 -0
- package/dist/drivers/android/adb.js +122 -0
- package/dist/drivers/android/adb.js.map +1 -0
- package/dist/drivers/android/index.d.ts +70 -0
- package/dist/drivers/android/index.d.ts.map +1 -0
- package/dist/drivers/android/index.js +529 -0
- package/dist/drivers/android/index.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +131 -0
- package/dist/index.js.map +1 -0
- package/dist/server.d.ts +13 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +41 -0
- package/dist/server.js.map +1 -0
- package/dist/tools/ai-tools.d.ts +11 -0
- package/dist/tools/ai-tools.d.ts.map +1 -0
- package/dist/tools/ai-tools.js +238 -0
- package/dist/tools/ai-tools.js.map +1 -0
- package/dist/tools/app-tools.d.ts +4 -0
- package/dist/tools/app-tools.d.ts.map +1 -0
- package/dist/tools/app-tools.js +222 -0
- package/dist/tools/app-tools.js.map +1 -0
- package/dist/tools/device-tools.d.ts +4 -0
- package/dist/tools/device-tools.d.ts.map +1 -0
- package/dist/tools/device-tools.js +104 -0
- package/dist/tools/device-tools.js.map +1 -0
- package/dist/tools/index.d.ts +21 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +30 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/interaction-tools.d.ts +4 -0
- package/dist/tools/interaction-tools.d.ts.map +1 -0
- package/dist/tools/interaction-tools.js +304 -0
- package/dist/tools/interaction-tools.js.map +1 -0
- package/dist/tools/log-tools.d.ts +4 -0
- package/dist/tools/log-tools.d.ts.map +1 -0
- package/dist/tools/log-tools.js +60 -0
- package/dist/tools/log-tools.js.map +1 -0
- package/dist/tools/screen-tools.d.ts +4 -0
- package/dist/tools/screen-tools.d.ts.map +1 -0
- package/dist/tools/screen-tools.js +105 -0
- package/dist/tools/screen-tools.js.map +1 -0
- package/dist/types.d.ts +219 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +19 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/discovery.d.ts +20 -0
- package/dist/utils/discovery.d.ts.map +1 -0
- package/dist/utils/discovery.js +156 -0
- package/dist/utils/discovery.js.map +1 -0
- package/dist/utils/image.d.ts +46 -0
- package/dist/utils/image.d.ts.map +1 -0
- package/dist/utils/image.js +170 -0
- package/dist/utils/image.js.map +1 -0
- package/package.json +69 -0
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
// ============================================================
|
|
2
|
+
// ScreenAnalyzer — Orchestrates screenshots, UI trees, and AI
|
|
3
|
+
// calls to provide smart visual analysis of mobile device screens.
|
|
4
|
+
// ============================================================
|
|
5
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
6
|
+
import { GoogleGenerativeAI } from "@google/generative-ai";
|
|
7
|
+
import { PROMPTS, buildAnalyzeScreenPrompt, buildFindElementPrompt, buildSuggestActionsPrompt, buildVisualDiffPrompt, buildExtractTextPrompt, buildVerifyScreenPrompt, summarizeUIElements, } from "./prompts.js";
|
|
8
|
+
import { searchElementsLocally } from "./element-search.js";
|
|
9
|
+
// ----------------------------------------------------------
|
|
10
|
+
// Two-image analysis helper (used for visual diff)
|
|
11
|
+
// ----------------------------------------------------------
|
|
12
|
+
/**
|
|
13
|
+
* Sends two screenshots to the AI provider in a single message.
|
|
14
|
+
* This bypasses AIClient (which only supports one image) to allow
|
|
15
|
+
* before/after comparison in a single request.
|
|
16
|
+
*
|
|
17
|
+
* Supports both Anthropic and Google Gemini providers.
|
|
18
|
+
*/
|
|
19
|
+
async function analyzeWithTwoImages(provider, apiKey, model, maxTokens, systemPrompt, userPrompt, beforeBase64, afterBase64, beforeMimeType = "image/png", afterMimeType = "image/png") {
|
|
20
|
+
if (provider === "google") {
|
|
21
|
+
return analyzeWithTwoImagesGemini(apiKey, model, systemPrompt, userPrompt, beforeBase64, afterBase64, beforeMimeType, afterMimeType);
|
|
22
|
+
}
|
|
23
|
+
return analyzeWithTwoImagesAnthropic(apiKey, model, maxTokens, systemPrompt, userPrompt, beforeBase64, afterBase64, beforeMimeType, afterMimeType);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Anthropic implementation: two images in a single message.
|
|
27
|
+
*/
|
|
28
|
+
async function analyzeWithTwoImagesAnthropic(apiKey, model, maxTokens, systemPrompt, userPrompt, beforeBase64, afterBase64, beforeMimeType = "image/png", afterMimeType = "image/png") {
|
|
29
|
+
const anthropic = new Anthropic({ apiKey });
|
|
30
|
+
const response = await anthropic.messages.create({
|
|
31
|
+
model,
|
|
32
|
+
max_tokens: maxTokens,
|
|
33
|
+
system: systemPrompt,
|
|
34
|
+
messages: [
|
|
35
|
+
{
|
|
36
|
+
role: "user",
|
|
37
|
+
content: [
|
|
38
|
+
{
|
|
39
|
+
type: "image",
|
|
40
|
+
source: {
|
|
41
|
+
type: "base64",
|
|
42
|
+
media_type: beforeMimeType,
|
|
43
|
+
data: beforeBase64,
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
{ type: "text", text: "This is the BEFORE screenshot." },
|
|
47
|
+
{
|
|
48
|
+
type: "image",
|
|
49
|
+
source: {
|
|
50
|
+
type: "base64",
|
|
51
|
+
media_type: afterMimeType,
|
|
52
|
+
data: afterBase64,
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
type: "text",
|
|
57
|
+
text: "This is the AFTER screenshot.\n\n" + userPrompt,
|
|
58
|
+
},
|
|
59
|
+
],
|
|
60
|
+
},
|
|
61
|
+
],
|
|
62
|
+
});
|
|
63
|
+
const block = response.content[0];
|
|
64
|
+
if (block.type === "text")
|
|
65
|
+
return block.text;
|
|
66
|
+
throw new Error("Unexpected response type from Anthropic API.");
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Google Gemini implementation: two images in a single message.
|
|
70
|
+
*/
|
|
71
|
+
async function analyzeWithTwoImagesGemini(apiKey, model, systemPrompt, userPrompt, beforeBase64, afterBase64, beforeMimeType = "image/png", afterMimeType = "image/png") {
|
|
72
|
+
const genAI = new GoogleGenerativeAI(apiKey);
|
|
73
|
+
const geminiModel = genAI.getGenerativeModel({ model });
|
|
74
|
+
const result = await geminiModel.generateContent({
|
|
75
|
+
systemInstruction: systemPrompt,
|
|
76
|
+
contents: [
|
|
77
|
+
{
|
|
78
|
+
role: "user",
|
|
79
|
+
parts: [
|
|
80
|
+
{ inlineData: { mimeType: beforeMimeType, data: beforeBase64 } },
|
|
81
|
+
{ text: "This is the BEFORE screenshot." },
|
|
82
|
+
{ inlineData: { mimeType: afterMimeType, data: afterBase64 } },
|
|
83
|
+
{ text: "This is the AFTER screenshot.\n\n" + userPrompt },
|
|
84
|
+
],
|
|
85
|
+
},
|
|
86
|
+
],
|
|
87
|
+
});
|
|
88
|
+
return result.response.text();
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Parse a raw string as JSON, handling markdown code fences.
|
|
92
|
+
*/
|
|
93
|
+
function parseJSONResponse(raw) {
|
|
94
|
+
try {
|
|
95
|
+
return JSON.parse(raw);
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
// Fall through to extraction.
|
|
99
|
+
}
|
|
100
|
+
const fenceMatch = raw.match(/```(?:json)?\s*\n?([\s\S]*?)```/);
|
|
101
|
+
if (fenceMatch) {
|
|
102
|
+
try {
|
|
103
|
+
return JSON.parse(fenceMatch[1].trim());
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
// Fall through.
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
const objectMatch = raw.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
|
|
110
|
+
if (objectMatch) {
|
|
111
|
+
try {
|
|
112
|
+
return JSON.parse(objectMatch[1]);
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
// Fall through.
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
throw new Error(`Failed to parse AI response as JSON. Raw response:\n${raw.slice(0, 500)}`);
|
|
119
|
+
}
|
|
120
|
+
export class ScreenAnalyzer {
|
|
121
|
+
client;
|
|
122
|
+
driver;
|
|
123
|
+
config;
|
|
124
|
+
cache = {};
|
|
125
|
+
cacheTTL = 3000; // 3 seconds
|
|
126
|
+
/** Screenshot options used for AI analysis (compressed for performance). */
|
|
127
|
+
screenshotOptions;
|
|
128
|
+
constructor(client, driver, config, screenshotOptions) {
|
|
129
|
+
this.client = client;
|
|
130
|
+
this.driver = driver;
|
|
131
|
+
this.config = config;
|
|
132
|
+
// Default: JPEG at quality 80, resize to 720px for AI — saves tokens
|
|
133
|
+
this.screenshotOptions = screenshotOptions ?? {
|
|
134
|
+
format: "jpeg",
|
|
135
|
+
quality: 80,
|
|
136
|
+
maxWidth: 720,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
// ----------------------------------------------------------
|
|
140
|
+
// Public API
|
|
141
|
+
// ----------------------------------------------------------
|
|
142
|
+
/**
|
|
143
|
+
* Analyze the current screen state — identifies elements, screen
|
|
144
|
+
* type, visible text, and actionable suggestions.
|
|
145
|
+
*/
|
|
146
|
+
async analyzeScreen(deviceId) {
|
|
147
|
+
try {
|
|
148
|
+
this.assertClientAvailable();
|
|
149
|
+
const ctx = await this.captureContext(deviceId);
|
|
150
|
+
const summarized = ctx.uiElements
|
|
151
|
+
? summarizeUIElements(ctx.uiElements)
|
|
152
|
+
: undefined;
|
|
153
|
+
const userPrompt = buildAnalyzeScreenPrompt(summarized);
|
|
154
|
+
return await this.client.analyzeJSON({
|
|
155
|
+
systemPrompt: PROMPTS.ANALYZE_SCREEN,
|
|
156
|
+
userPrompt,
|
|
157
|
+
screenshot: ctx.screenshot?.base64,
|
|
158
|
+
screenshotMimeType: ctx.screenshot ? `image/${ctx.screenshot.format}` : undefined,
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
catch (error) {
|
|
162
|
+
throw new Error(`analyzeScreen failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Find an element on screen by natural language description.
|
|
167
|
+
*
|
|
168
|
+
* Fast path: searches the UI tree locally first. If a high-confidence
|
|
169
|
+
* match is found (>0.7), returns immediately without calling the AI.
|
|
170
|
+
* Falls back to AI for ambiguous or complex queries.
|
|
171
|
+
*/
|
|
172
|
+
async findElement(deviceId, query) {
|
|
173
|
+
try {
|
|
174
|
+
// --- Fast path: local UI tree search (no AI call) ---
|
|
175
|
+
if (this.config.analyzeWithUITree) {
|
|
176
|
+
const uiElements = await this.getUIElements(deviceId);
|
|
177
|
+
if (uiElements && uiElements.length > 0) {
|
|
178
|
+
const localMatch = searchElementsLocally(uiElements, query);
|
|
179
|
+
if (localMatch.found && localMatch.confidence > 0.5) {
|
|
180
|
+
return localMatch;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
// --- Slow path: AI-powered search ---
|
|
185
|
+
this.assertClientAvailable();
|
|
186
|
+
const ctx = await this.captureContext(deviceId);
|
|
187
|
+
const summarized = ctx.uiElements
|
|
188
|
+
? summarizeUIElements(ctx.uiElements)
|
|
189
|
+
: undefined;
|
|
190
|
+
const userPrompt = buildFindElementPrompt(query, summarized);
|
|
191
|
+
return await this.client.analyzeJSON({
|
|
192
|
+
systemPrompt: PROMPTS.FIND_ELEMENT,
|
|
193
|
+
userPrompt,
|
|
194
|
+
screenshot: ctx.screenshot?.base64,
|
|
195
|
+
screenshotMimeType: ctx.screenshot ? `image/${ctx.screenshot.format}` : undefined,
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
catch (error) {
|
|
199
|
+
throw new Error(`findElement failed for query "${query}": ${error instanceof Error ? error.message : String(error)}`);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Plan a sequence of actions to achieve a goal on the current screen.
|
|
204
|
+
*/
|
|
205
|
+
async suggestActions(deviceId, goal) {
|
|
206
|
+
try {
|
|
207
|
+
this.assertClientAvailable();
|
|
208
|
+
const ctx = await this.captureContext(deviceId);
|
|
209
|
+
const summarized = ctx.uiElements
|
|
210
|
+
? summarizeUIElements(ctx.uiElements)
|
|
211
|
+
: undefined;
|
|
212
|
+
let currentApp;
|
|
213
|
+
try {
|
|
214
|
+
const appInfo = await this.driver.getCurrentApp(deviceId);
|
|
215
|
+
currentApp = appInfo.packageName;
|
|
216
|
+
}
|
|
217
|
+
catch {
|
|
218
|
+
// getCurrentApp may fail on some devices — proceed without it.
|
|
219
|
+
}
|
|
220
|
+
const userPrompt = buildSuggestActionsPrompt(goal, currentApp, summarized);
|
|
221
|
+
return await this.client.analyzeJSON({
|
|
222
|
+
systemPrompt: PROMPTS.SUGGEST_ACTIONS,
|
|
223
|
+
userPrompt,
|
|
224
|
+
screenshot: ctx.screenshot?.base64,
|
|
225
|
+
screenshotMimeType: ctx.screenshot ? `image/${ctx.screenshot.format}` : undefined,
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
catch (error) {
|
|
229
|
+
throw new Error(`suggestActions failed for goal "${goal}": ${error instanceof Error ? error.message : String(error)}`);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Compare the current screen with a previous screenshot and
|
|
234
|
+
* describe what changed.
|
|
235
|
+
*/
|
|
236
|
+
async compareScreenshots(deviceId, beforeBase64) {
|
|
237
|
+
try {
|
|
238
|
+
this.assertClientAvailable();
|
|
239
|
+
// Always take a fresh screenshot for the "after" state.
|
|
240
|
+
const afterScreenshot = await this.driver.takeScreenshot(deviceId, this.screenshotOptions);
|
|
241
|
+
const userPrompt = buildVisualDiffPrompt();
|
|
242
|
+
// Before image might be PNG (from take_screenshot tool) or JPEG (from cached).
|
|
243
|
+
// Detect from base64 header: JPEG starts with /9j/, PNG starts with iVBOR.
|
|
244
|
+
const beforeMimeType = beforeBase64.startsWith("/9j/") ? "image/jpeg" : "image/png";
|
|
245
|
+
const afterMimeType = `image/${afterScreenshot.format}`;
|
|
246
|
+
const raw = await analyzeWithTwoImages(this.config.provider, this.config.apiKey, this.config.model, this.config.maxTokens, PROMPTS.VISUAL_DIFF, userPrompt, beforeBase64, afterScreenshot.base64, beforeMimeType, afterMimeType);
|
|
247
|
+
return parseJSONResponse(raw);
|
|
248
|
+
}
|
|
249
|
+
catch (error) {
|
|
250
|
+
throw new Error(`compareScreenshots failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Extract all readable text from the current screen.
|
|
255
|
+
*/
|
|
256
|
+
async extractText(deviceId) {
|
|
257
|
+
try {
|
|
258
|
+
this.assertClientAvailable();
|
|
259
|
+
// Always need a screenshot for OCR — ignore UITree config.
|
|
260
|
+
const screenshot = await this.driver.takeScreenshot(deviceId, this.screenshotOptions);
|
|
261
|
+
const userPrompt = buildExtractTextPrompt();
|
|
262
|
+
const result = await this.client.analyzeJSON({
|
|
263
|
+
systemPrompt: PROMPTS.EXTRACT_TEXT,
|
|
264
|
+
userPrompt,
|
|
265
|
+
screenshot: screenshot.base64,
|
|
266
|
+
screenshotMimeType: `image/${screenshot.format}`,
|
|
267
|
+
});
|
|
268
|
+
return result.texts;
|
|
269
|
+
}
|
|
270
|
+
catch (error) {
|
|
271
|
+
throw new Error(`extractText failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Verify an assertion about the current screen state.
|
|
276
|
+
*/
|
|
277
|
+
async verifyScreen(deviceId, assertion) {
|
|
278
|
+
try {
|
|
279
|
+
this.assertClientAvailable();
|
|
280
|
+
const ctx = await this.captureContext(deviceId);
|
|
281
|
+
const summarized = ctx.uiElements
|
|
282
|
+
? summarizeUIElements(ctx.uiElements)
|
|
283
|
+
: undefined;
|
|
284
|
+
const userPrompt = buildVerifyScreenPrompt(assertion, summarized);
|
|
285
|
+
return await this.client.analyzeJSON({
|
|
286
|
+
systemPrompt: PROMPTS.VERIFY_SCREEN,
|
|
287
|
+
userPrompt,
|
|
288
|
+
screenshot: ctx.screenshot?.base64,
|
|
289
|
+
screenshotMimeType: ctx.screenshot ? `image/${ctx.screenshot.format}` : undefined,
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
catch (error) {
|
|
293
|
+
throw new Error(`verifyScreen failed for assertion "${assertion}": ${error instanceof Error ? error.message : String(error)}`);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Smart tap: find an element by natural language description and
|
|
298
|
+
* tap its center.
|
|
299
|
+
*/
|
|
300
|
+
async smartTap(deviceId, elementDescription) {
|
|
301
|
+
try {
|
|
302
|
+
const match = await this.findElement(deviceId, elementDescription);
|
|
303
|
+
if (match.found && match.element && match.confidence > 0.5) {
|
|
304
|
+
await this.driver.tap(deviceId, match.element.bounds.centerX, match.element.bounds.centerY);
|
|
305
|
+
this.invalidateCache();
|
|
306
|
+
return {
|
|
307
|
+
success: true,
|
|
308
|
+
tapped: match.element,
|
|
309
|
+
message: `Tapped: ${match.element.description}`,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
return {
|
|
313
|
+
success: false,
|
|
314
|
+
tapped: null,
|
|
315
|
+
message: `Could not find element: ${elementDescription}`,
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
catch (error) {
|
|
319
|
+
throw new Error(`smartTap failed for "${elementDescription}": ${error instanceof Error ? error.message : String(error)}`);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Smart type: find an input field by description, tap to focus it,
|
|
324
|
+
* then type the provided text.
|
|
325
|
+
*/
|
|
326
|
+
async smartType(deviceId, fieldDescription, text) {
|
|
327
|
+
try {
|
|
328
|
+
const match = await this.findElement(deviceId, fieldDescription);
|
|
329
|
+
if (match.found && match.element && match.confidence > 0.5) {
|
|
330
|
+
// Tap to focus the field first.
|
|
331
|
+
await this.driver.tap(deviceId, match.element.bounds.centerX, match.element.bounds.centerY);
|
|
332
|
+
// Brief delay for focus to settle.
|
|
333
|
+
await this.delay(300);
|
|
334
|
+
// Type the text.
|
|
335
|
+
await this.driver.typeText(deviceId, text);
|
|
336
|
+
this.invalidateCache();
|
|
337
|
+
return {
|
|
338
|
+
success: true,
|
|
339
|
+
field: match.element,
|
|
340
|
+
message: `Typed into: ${match.element.description}`,
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
return {
|
|
344
|
+
success: false,
|
|
345
|
+
field: null,
|
|
346
|
+
message: `Could not find field: ${fieldDescription}`,
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
catch (error) {
|
|
350
|
+
throw new Error(`smartType failed for field "${fieldDescription}": ${error instanceof Error ? error.message : String(error)}`);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
// ----------------------------------------------------------
|
|
354
|
+
// Internal helpers
|
|
355
|
+
// ----------------------------------------------------------
|
|
356
|
+
/**
|
|
357
|
+
* Fetch UI elements for the device, using the cache when valid.
|
|
358
|
+
* This is a lightweight call (no screenshot) used by the local
|
|
359
|
+
* element search fast path.
|
|
360
|
+
*/
|
|
361
|
+
async getUIElements(deviceId) {
|
|
362
|
+
if (this.cache.uiElements &&
|
|
363
|
+
this.isCacheValid(this.cache.uiElements.timestamp)) {
|
|
364
|
+
return this.cache.uiElements.data;
|
|
365
|
+
}
|
|
366
|
+
try {
|
|
367
|
+
const elements = await this.driver.getUIElements(deviceId, {
|
|
368
|
+
interactiveOnly: false,
|
|
369
|
+
});
|
|
370
|
+
this.cache.uiElements = { data: elements, timestamp: Date.now() };
|
|
371
|
+
return elements;
|
|
372
|
+
}
|
|
373
|
+
catch {
|
|
374
|
+
return undefined;
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Capture screenshot and/or UI elements based on config flags.
|
|
379
|
+
* Uses caching with TTL and parallel capture for performance.
|
|
380
|
+
*/
|
|
381
|
+
async captureContext(deviceId) {
|
|
382
|
+
const ctx = {};
|
|
383
|
+
const now = Date.now();
|
|
384
|
+
// Check cache for valid screenshot
|
|
385
|
+
const cachedScreenshot = this.config.analyzeWithScreenshot &&
|
|
386
|
+
this.cache.screenshot &&
|
|
387
|
+
this.isCacheValid(this.cache.screenshot.timestamp)
|
|
388
|
+
? this.cache.screenshot.data
|
|
389
|
+
: undefined;
|
|
390
|
+
// Check cache for valid UI elements
|
|
391
|
+
const cachedUIElements = this.config.analyzeWithUITree &&
|
|
392
|
+
this.cache.uiElements &&
|
|
393
|
+
this.isCacheValid(this.cache.uiElements.timestamp)
|
|
394
|
+
? this.cache.uiElements.data
|
|
395
|
+
: undefined;
|
|
396
|
+
if (cachedScreenshot) {
|
|
397
|
+
ctx.screenshot = cachedScreenshot;
|
|
398
|
+
}
|
|
399
|
+
if (cachedUIElements) {
|
|
400
|
+
ctx.uiElements = cachedUIElements;
|
|
401
|
+
}
|
|
402
|
+
// Capture missing data in parallel
|
|
403
|
+
const promises = [];
|
|
404
|
+
if (this.config.analyzeWithScreenshot && !cachedScreenshot) {
|
|
405
|
+
promises.push(this.driver.takeScreenshot(deviceId, this.screenshotOptions).then((s) => {
|
|
406
|
+
ctx.screenshot = s;
|
|
407
|
+
this.cache.screenshot = { data: s, timestamp: now };
|
|
408
|
+
}));
|
|
409
|
+
}
|
|
410
|
+
if (this.config.analyzeWithUITree && !cachedUIElements) {
|
|
411
|
+
promises.push(this.driver
|
|
412
|
+
.getUIElements(deviceId, { interactiveOnly: false })
|
|
413
|
+
.then((e) => {
|
|
414
|
+
ctx.uiElements = e;
|
|
415
|
+
this.cache.uiElements = { data: e, timestamp: now };
|
|
416
|
+
}));
|
|
417
|
+
}
|
|
418
|
+
await Promise.all(promises);
|
|
419
|
+
return ctx;
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* Check if a cached entry is still valid based on TTL.
|
|
423
|
+
*/
|
|
424
|
+
isCacheValid(timestamp) {
|
|
425
|
+
return Date.now() - timestamp < this.cacheTTL;
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Invalidate the cache — call after actions that change the screen.
|
|
429
|
+
* Only clears the screenshot. UI element positions (button layout)
|
|
430
|
+
* rarely change after a tap, so keep the UI tree cache to avoid
|
|
431
|
+
* expensive uiautomator dump calls on every interaction.
|
|
432
|
+
*/
|
|
433
|
+
invalidateCache() {
|
|
434
|
+
this.cache.screenshot = undefined;
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* Throw if the AI client is not available (no API key configured).
|
|
438
|
+
*/
|
|
439
|
+
assertClientAvailable() {
|
|
440
|
+
if (!this.client.isAvailable()) {
|
|
441
|
+
throw new Error("AI client is not available. Set ANTHROPIC_API_KEY to enable AI features.");
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Promise-based delay helper.
|
|
446
|
+
*/
|
|
447
|
+
delay(ms) {
|
|
448
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
//# sourceMappingURL=analyzer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"analyzer.js","sourceRoot":"","sources":["../../src/ai/analyzer.ts"],"names":[],"mappings":"AAAA,+DAA+D;AAC/D,8DAA8D;AAC9D,mEAAmE;AACnE,+DAA+D;AAE/D,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAe3D,OAAO,EACL,OAAO,EACP,wBAAwB,EACxB,sBAAsB,EACtB,yBAAyB,EACzB,qBAAqB,EACrB,sBAAsB,EACtB,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,6DAA6D;AAC7D,mDAAmD;AACnD,6DAA6D;AAE7D;;;;;;GAMG;AACH,KAAK,UAAU,oBAAoB,CACjC,QAAgC,EAChC,MAAc,EACd,KAAa,EACb,SAAiB,EACjB,YAAoB,EACpB,UAAkB,EAClB,YAAoB,EACpB,WAAmB,EACnB,iBAAyB,WAAW,EACpC,gBAAwB,WAAW;IAEnC,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,OAAO,0BAA0B,CAC/B,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAClE,cAAc,EAAE,aAAa,CAC9B,CAAC;IACJ,CAAC;IACD,OAAO,6BAA6B,CAClC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAC7E,cAAc,EAAE,aAAa,CAC9B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,6BAA6B,CAC1C,MAAc,EACd,KAAa,EACb,SAAiB,EACjB,YAAoB,EACpB,UAAkB,EAClB,YAAoB,EACpB,WAAmB,EACnB,iBAAyB,WAAW,EACpC,gBAAwB,WAAW;IAEnC,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC/C,KAAK;QACL,UAAU,EAAE,SAAS;QACrB,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,OAAO;wBACb,MAAM,EAAE;4BACN,IAAI,EAAE,QAAQ;4BACd,UAAU,EAAE,cAAyE;4BACrF,IAAI,EAAE,YAAY;yBACnB;qBACF;oBACD,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,gCAAgC,EAAE;oBACxD;wBACE,IAAI,EAAE,OAAO;wBACb,MAAM,EAAE;4BACN,IAAI,EAAE,QAAQ;4BACd,UAAU,EAAE,aAAwE;4BACpF,IAAI,EAAE,WAAW;yBAClB;qBACF;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,mCAAmC,GAAG,UAAU;qBACvD;iBACF;aACF;SACF;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAClC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC;IAC7C,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;AAClE,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,0BAA0B,CACvC,MAAc,EACd,KAAa,EACb,YAAoB,EACpB,UAAkB,EAClB,YAAoB,EACpB,WAAmB,EACnB,iBAAyB,WAAW,EACpC,gBAAwB,WAAW;IAEnC,MAAM,KAAK,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,KAAK,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,eAAe,CAAC;QAC/C,iBAAiB,EAAE,YAAY;QAC/B,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE;oBACL,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE;oBAChE,EAAE,IAAI,EAAE,gCAAgC,EAAE;oBAC1C,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE;oBAC9D,EAAE,IAAI,EAAE,mCAAmC,GAAG,UAAU,EAAE;iBAC3D;aACF;SACF;KACF,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAI,GAAW;IACvC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,8BAA8B;IAChC,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAChE,IAAI,UAAU,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAM,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,gBAAgB;QAClB,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC3D,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAM,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,gBAAgB;QAClB,CAAC;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CACb,uDAAuD,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC3E,CAAC;AACJ,CAAC;AAWD,MAAM,OAAO,cAAc;IAWf;IACA;IACA;IAZF,KAAK,GAGT,EAAE,CAAC;IACC,QAAQ,GAAW,IAAI,CAAC,CAAC,YAAY;IAE7C,4EAA4E;IACpE,iBAAiB,CAAoB;IAE7C,YACU,MAAgB,EAChB,MAAoB,EACpB,MAOP,EACD,iBAAqC;QAV7B,WAAM,GAAN,MAAM,CAAU;QAChB,WAAM,GAAN,MAAM,CAAc;QACpB,WAAM,GAAN,MAAM,CAOb;QAGD,qEAAqE;QACrE,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,IAAI;YAC5C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE;YACX,QAAQ,EAAE,GAAG;SACd,CAAC;IACJ,CAAC;IAED,6DAA6D;IAC7D,aAAa;IACb,6DAA6D;IAE7D;;;OAGG;IACH,KAAK,CAAC,aAAa,CAAC,QAAgB;QAClC,IAAI,CAAC;YACH,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAChD,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU;gBAC/B,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC;gBACrC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,UAAU,GAAG,wBAAwB,CAAC,UAAU,CAAC,CAAC;YAExD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAiB;gBACnD,YAAY,EAAE,OAAO,CAAC,cAAc;gBACpC,UAAU;gBACV,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,MAAM;gBAClC,kBAAkB,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS;aAClF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,WAAW,CACf,QAAgB,EAChB,KAAa;QAEb,IAAI,CAAC;YACH,uDAAuD;YACvD,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAClC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;gBACtD,IAAI,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxC,MAAM,UAAU,GAAG,qBAAqB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;oBAC5D,IAAI,UAAU,CAAC,KAAK,IAAI,UAAU,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;wBACpD,OAAO,UAAU,CAAC;oBACpB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,uCAAuC;YACvC,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAChD,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU;gBAC/B,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC;gBACrC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,UAAU,GAAG,sBAAsB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;YAE7D,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAe;gBACjD,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,UAAU;gBACV,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,MAAM;gBAClC,kBAAkB,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS;aAClF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,iCAAiC,KAAK,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACrG,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAClB,QAAgB,EAChB,IAAY;QAEZ,IAAI,CAAC;YACH,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAChD,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU;gBAC/B,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC;gBACrC,CAAC,CAAC,SAAS,CAAC;YAEd,IAAI,UAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;gBAC1D,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;YACnC,CAAC;YAAC,MAAM,CAAC;gBACP,+DAA+D;YACjE,CAAC;YAED,MAAM,UAAU,GAAG,yBAAyB,CAC1C,IAAI,EACJ,UAAU,EACV,UAAU,CACX,CAAC;YAEF,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAa;gBAC/C,YAAY,EAAE,OAAO,CAAC,eAAe;gBACrC,UAAU;gBACV,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,MAAM;gBAClC,kBAAkB,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS;aAClF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,mCAAmC,IAAI,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACtG,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,kBAAkB,CACtB,QAAgB,EAChB,YAAoB;QAEpB,IAAI,CAAC;YACH,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAE7B,wDAAwD;YACxD,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAC3F,MAAM,UAAU,GAAG,qBAAqB,EAAE,CAAC;YAE3C,+EAA+E;YAC/E,2EAA2E;YAC3E,MAAM,cAAc,GAAG,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC;YACpF,MAAM,aAAa,GAAG,SAAS,eAAe,CAAC,MAAM,EAAE,CAAC;YAExD,MAAM,GAAG,GAAG,MAAM,oBAAoB,CACpC,IAAI,CAAC,MAAM,CAAC,QAAQ,EACpB,IAAI,CAAC,MAAM,CAAC,MAAM,EAClB,IAAI,CAAC,MAAM,CAAC,KAAK,EACjB,IAAI,CAAC,MAAM,CAAC,SAAS,EACrB,OAAO,CAAC,WAAW,EACnB,UAAU,EACV,YAAY,EACZ,eAAe,CAAC,MAAM,EACtB,cAAc,EACd,aAAa,CACd,CAAC;YAEF,OAAO,iBAAiB,CAAa,GAAG,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,QAAgB;QAChC,IAAI,CAAC;YACH,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAE7B,2DAA2D;YAC3D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACtF,MAAM,UAAU,GAAG,sBAAsB,EAAE,CAAC;YAE5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAsB;gBAChE,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,UAAU;gBACV,UAAU,EAAE,UAAU,CAAC,MAAM;gBAC7B,kBAAkB,EAAE,SAAS,UAAU,CAAC,MAAM,EAAE;aACjD,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,uBAAuB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAChF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAChB,QAAgB,EAChB,SAAiB;QAEjB,IAAI,CAAC;YACH,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAChD,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU;gBAC/B,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC;gBACrC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,UAAU,GAAG,uBAAuB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YAElE,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAqB;gBACvD,YAAY,EAAE,OAAO,CAAC,aAAa;gBACnC,UAAU;gBACV,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,MAAM;gBAClC,kBAAkB,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS;aAClF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,sCAAsC,SAAS,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9G,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CACZ,QAAgB,EAChB,kBAA0B;QAM1B,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;YAEnE,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;gBAC3D,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CACnB,QAAQ,EACR,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAC5B,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAC7B,CAAC;gBACF,IAAI,CAAC,eAAe,EAAE,CAAC;gBACvB,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,KAAK,CAAC,OAAO;oBACrB,OAAO,EAAE,WAAW,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE;iBAChD,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,IAAI;gBACZ,OAAO,EAAE,2BAA2B,kBAAkB,EAAE;aACzD,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,wBAAwB,kBAAkB,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACzG,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CACb,QAAgB,EAChB,gBAAwB,EACxB,IAAY;QAMZ,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;YAEjE,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;gBAC3D,gCAAgC;gBAChC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CACnB,QAAQ,EACR,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAC5B,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAC7B,CAAC;gBAEF,mCAAmC;gBACnC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAEtB,iBAAiB;gBACjB,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAC3C,IAAI,CAAC,eAAe,EAAE,CAAC;gBAEvB,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,KAAK,EAAE,KAAK,CAAC,OAAO;oBACpB,OAAO,EAAE,eAAe,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE;iBACpD,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,yBAAyB,gBAAgB,EAAE;aACrD,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,+BAA+B,gBAAgB,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9G,CAAC;QACJ,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,mBAAmB;IACnB,6DAA6D;IAE7D;;;;OAIG;IACK,KAAK,CAAC,aAAa,CAAC,QAAgB;QAC1C,IACE,IAAI,CAAC,KAAK,CAAC,UAAU;YACrB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAClD,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;QACpC,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE;gBACzD,eAAe,EAAE,KAAK;aACvB,CAAC,CAAC;YACH,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YAClE,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,cAAc,CAAC,QAAgB;QAC3C,MAAM,GAAG,GAAoB,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,mCAAmC;QACnC,MAAM,gBAAgB,GACpB,IAAI,CAAC,MAAM,CAAC,qBAAqB;YACjC,IAAI,CAAC,KAAK,CAAC,UAAU;YACrB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;YAChD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI;YAC5B,CAAC,CAAC,SAAS,CAAC;QAEhB,oCAAoC;QACpC,MAAM,gBAAgB,GACpB,IAAI,CAAC,MAAM,CAAC,iBAAiB;YAC7B,IAAI,CAAC,KAAK,CAAC,UAAU;YACrB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;YAChD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI;YAC5B,CAAC,CAAC,SAAS,CAAC;QAEhB,IAAI,gBAAgB,EAAE,CAAC;YACrB,GAAG,CAAC,UAAU,GAAG,gBAAgB,CAAC;QACpC,CAAC;QACD,IAAI,gBAAgB,EAAE,CAAC;YACrB,GAAG,CAAC,UAAU,GAAG,gBAAgB,CAAC;QACpC,CAAC;QAED,mCAAmC;QACnC,MAAM,QAAQ,GAAoB,EAAE,CAAC;QAErC,IAAI,IAAI,CAAC,MAAM,CAAC,qBAAqB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3D,QAAQ,CAAC,IAAI,CACX,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACtE,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC;gBACnB,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;YACtD,CAAC,CAAC,CACH,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvD,QAAQ,CAAC,IAAI,CACX,IAAI,CAAC,MAAM;iBACR,aAAa,CAAC,QAAQ,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;iBACnD,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACV,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC;gBACnB,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;YACtD,CAAC,CAAC,CACL,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE5B,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,SAAiB;QACpC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC;IAChD,CAAC;IAED;;;;;OAKG;IACK,eAAe;QACrB,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC;IACpC,CAAC;IAED;;OAEG;IACK,qBAAqB;QAC3B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CACb,0EAA0E,CAC3E,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;CACF"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import type { AIConfig } from "../types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Options accepted by {@link AIClient.analyze} and {@link AIClient.analyzeJSON}.
|
|
4
|
+
*/
|
|
5
|
+
export interface AnalyzeOptions {
|
|
6
|
+
systemPrompt: string;
|
|
7
|
+
userPrompt: string;
|
|
8
|
+
/** Base-64 encoded screenshot (PNG by default). */
|
|
9
|
+
screenshot?: string;
|
|
10
|
+
/** MIME type for the screenshot. Defaults to `"image/png"`. */
|
|
11
|
+
screenshotMimeType?: string;
|
|
12
|
+
/** Override the default max tokens for this request. */
|
|
13
|
+
maxTokens?: number;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Wraps both the Anthropic and Google Generative AI SDKs for all
|
|
17
|
+
* AI visual-analysis features.
|
|
18
|
+
*
|
|
19
|
+
* Create one instance and reuse it — the underlying HTTP client is
|
|
20
|
+
* allocated once in the constructor.
|
|
21
|
+
*/
|
|
22
|
+
export declare class AIClient {
|
|
23
|
+
private readonly anthropicClient?;
|
|
24
|
+
private readonly geminiModel?;
|
|
25
|
+
private readonly config;
|
|
26
|
+
constructor(config: AIConfig);
|
|
27
|
+
/**
|
|
28
|
+
* Returns `true` when a non-empty API key has been configured,
|
|
29
|
+
* meaning AI features can be used.
|
|
30
|
+
*/
|
|
31
|
+
isAvailable(): boolean;
|
|
32
|
+
/**
|
|
33
|
+
* Send a prompt (with an optional screenshot) to the configured
|
|
34
|
+
* AI provider and return the plain-text response.
|
|
35
|
+
*/
|
|
36
|
+
analyze(options: AnalyzeOptions): Promise<string>;
|
|
37
|
+
/**
|
|
38
|
+
* Convenience wrapper: call {@link analyze}, then parse the response
|
|
39
|
+
* as JSON. Handles responses wrapped in markdown code fences.
|
|
40
|
+
*/
|
|
41
|
+
analyzeJSON<T>(options: AnalyzeOptions): Promise<T>;
|
|
42
|
+
private analyzeWithAnthropic;
|
|
43
|
+
/**
|
|
44
|
+
* Build the `content` array for the Anthropic user message.
|
|
45
|
+
*/
|
|
46
|
+
private buildAnthropicContent;
|
|
47
|
+
/**
|
|
48
|
+
* Call `messages.create` with a single manual retry for 429 / 529.
|
|
49
|
+
*/
|
|
50
|
+
private createWithRetryAnthropic;
|
|
51
|
+
/**
|
|
52
|
+
* Determine whether an Anthropic error is a retryable 429 or 529.
|
|
53
|
+
*/
|
|
54
|
+
private isRetryableAnthropic;
|
|
55
|
+
/**
|
|
56
|
+
* Extract the text content from the first Anthropic content block.
|
|
57
|
+
*/
|
|
58
|
+
private extractAnthropicText;
|
|
59
|
+
private analyzeWithGemini;
|
|
60
|
+
/**
|
|
61
|
+
* Call Gemini generateContent with retries for 429 / 503 errors.
|
|
62
|
+
* Parses the RetryInfo delay from Google API error messages.
|
|
63
|
+
*/
|
|
64
|
+
private createWithRetryGemini;
|
|
65
|
+
/**
|
|
66
|
+
* Determine whether a Google API error is retryable (429 / 503).
|
|
67
|
+
*/
|
|
68
|
+
private isRetryableGemini;
|
|
69
|
+
/**
|
|
70
|
+
* Extract the retry delay (in ms) from a Google API error message.
|
|
71
|
+
* Looks for patterns like 'retryDelay":"23s"' or 'Please retry in 23.2s'.
|
|
72
|
+
*/
|
|
73
|
+
private extractRetryDelay;
|
|
74
|
+
/**
|
|
75
|
+
* Throw immediately when the API key is absent so callers get a
|
|
76
|
+
* clear message instead of an opaque 401.
|
|
77
|
+
*/
|
|
78
|
+
private assertAvailable;
|
|
79
|
+
/**
|
|
80
|
+
* Parse a string as JSON, stripping markdown code fences if present.
|
|
81
|
+
*/
|
|
82
|
+
private parseJSON;
|
|
83
|
+
/**
|
|
84
|
+
* Wrap unknown errors into Error instances with descriptive messages.
|
|
85
|
+
*/
|
|
86
|
+
private wrapError;
|
|
87
|
+
/**
|
|
88
|
+
* Promise-based delay helper.
|
|
89
|
+
*/
|
|
90
|
+
private delay;
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/ai/client.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,mDAAmD;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAWD;;;;;;GAMG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAY;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAkB;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAW;gBAEtB,MAAM,EAAE,QAAQ;IAoB5B;;;OAGG;IACH,WAAW,IAAI,OAAO;IAItB;;;OAGG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;IASvD;;;OAGG;IACG,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC;YAS3C,oBAAoB;IAsBlC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAsB7B;;OAEG;YACW,wBAAwB;IAetC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAQ5B;;OAEG;IACH,OAAO,CAAC,oBAAoB;YAmBd,iBAAiB;IAqB/B;;;OAGG;YACW,qBAAqB;IA6BnC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IASzB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAuBzB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAQvB;;OAEG;IACH,OAAO,CAAC,SAAS;IAiCjB;;OAEG;IACH,OAAO,CAAC,SAAS;IAYjB;;OAEG;IACH,OAAO,CAAC,KAAK;CAGd"}
|