appium-mcp 1.97.2 → 1.99.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +55 -2
- package/dist/locators/element-filter.d.ts +3 -1
- package/dist/locators/element-filter.d.ts.map +1 -1
- package/dist/locators/element-filter.js +44 -14
- package/dist/locators/element-filter.js.map +1 -1
- package/dist/locators/generate-all-locators.d.ts +6 -1
- package/dist/locators/generate-all-locators.d.ts.map +1 -1
- package/dist/locators/generate-all-locators.js +1 -1
- package/dist/locators/generate-all-locators.js.map +1 -1
- package/dist/locators/locator-generation.d.ts +4 -3
- package/dist/locators/locator-generation.d.ts.map +1 -1
- package/dist/locators/locator-generation.js +31 -10
- package/dist/locators/locator-generation.js.map +1 -1
- package/dist/locators/types.d.ts +3 -0
- package/dist/locators/types.d.ts.map +1 -0
- package/dist/locators/types.js +2 -0
- package/dist/locators/types.js.map +1 -0
- package/dist/telemetry/init.d.ts +2 -0
- package/dist/telemetry/init.d.ts.map +1 -1
- package/dist/telemetry/init.js +14 -1
- package/dist/telemetry/init.js.map +1 -1
- package/dist/tools/gestures/schema.d.ts.map +1 -1
- package/dist/tools/gestures/schema.js +7 -15
- package/dist/tools/gestures/schema.js.map +1 -1
- package/dist/tools/test-generation/locators.d.ts.map +1 -1
- package/dist/tools/test-generation/locators.js +19 -5
- package/dist/tools/test-generation/locators.js.map +1 -1
- package/dist/ui/inspector-screenshot-script.d.ts +3 -0
- package/dist/ui/inspector-screenshot-script.d.ts.map +1 -0
- package/dist/ui/inspector-screenshot-script.js +164 -0
- package/dist/ui/inspector-screenshot-script.js.map +1 -0
- package/dist/ui/mcp-ui-utils.d.ts.map +1 -1
- package/dist/ui/mcp-ui-utils.js +21 -44
- package/dist/ui/mcp-ui-utils.js.map +1 -1
- package/dist/ui/page-source-inspector-app.d.ts.map +1 -1
- package/dist/ui/page-source-inspector-app.js +198 -119
- package/dist/ui/page-source-inspector-app.js.map +1 -1
- package/dist/ui/xml-inspector-scripts.d.ts +4 -0
- package/dist/ui/xml-inspector-scripts.d.ts.map +1 -0
- package/dist/ui/xml-inspector-scripts.js +91 -0
- package/dist/ui/xml-inspector-scripts.js.map +1 -0
- package/package.json +2 -1
- package/server.json +2 -2
- package/src/locators/element-filter.ts +50 -16
- package/src/locators/generate-all-locators.ts +13 -7
- package/src/locators/locator-generation.ts +43 -10
- package/src/locators/types.ts +2 -0
- package/src/telemetry/init.ts +18 -1
- package/src/tools/gestures/schema.ts +7 -17
- package/src/tools/test-generation/locators.ts +31 -7
- package/src/ui/inspector-screenshot-script.ts +163 -0
- package/src/ui/mcp-ui-utils.ts +22 -44
- package/src/ui/page-source-inspector-app.ts +199 -119
- package/src/ui/xml-inspector-scripts.ts +91 -0
|
@@ -7,6 +7,12 @@ import type {FilterOptions} from './element-filter.js';
|
|
|
7
7
|
import {getSuggestedLocators} from './locator-generation.js';
|
|
8
8
|
import {xmlToJSON} from './source-parsing.js';
|
|
9
9
|
import type {JSONElement} from './source-parsing.js';
|
|
10
|
+
import type {DriverName} from './types.js';
|
|
11
|
+
|
|
12
|
+
export interface LocatorGenerationOptions extends FilterOptions {
|
|
13
|
+
/** Additional XPath attributes, tried before the built-in attributes. */
|
|
14
|
+
xpathAttributes?: string[];
|
|
15
|
+
}
|
|
10
16
|
|
|
11
17
|
export interface ElementWithLocators {
|
|
12
18
|
tagName: string;
|
|
@@ -31,8 +37,8 @@ export interface ElementWithLocators {
|
|
|
31
37
|
export function generateAllElementLocators(
|
|
32
38
|
sourceXML: string,
|
|
33
39
|
isNative: boolean = true,
|
|
34
|
-
automationName:
|
|
35
|
-
filters:
|
|
40
|
+
automationName: DriverName,
|
|
41
|
+
filters: LocatorGenerationOptions = {},
|
|
36
42
|
): ElementWithLocators[] {
|
|
37
43
|
const sourceJSON = xmlToJSON(sourceXML);
|
|
38
44
|
const results: ElementWithLocators[] = [];
|
|
@@ -76,8 +82,8 @@ function processElement(
|
|
|
76
82
|
element: JSONElement,
|
|
77
83
|
sourceXML: string,
|
|
78
84
|
isNative: boolean,
|
|
79
|
-
automationName:
|
|
80
|
-
filters:
|
|
85
|
+
automationName: DriverName,
|
|
86
|
+
filters: LocatorGenerationOptions,
|
|
81
87
|
results: ElementWithLocators[],
|
|
82
88
|
): void {
|
|
83
89
|
if (!shouldIncludeElement(element, filters, isNative, automationName)) {
|
|
@@ -85,7 +91,7 @@ function processElement(
|
|
|
85
91
|
}
|
|
86
92
|
|
|
87
93
|
try {
|
|
88
|
-
const strategyMap = getSuggestedLocators(element, sourceXML, isNative, automationName);
|
|
94
|
+
const strategyMap = getSuggestedLocators(element, sourceXML, isNative, automationName, filters.xpathAttributes);
|
|
89
95
|
results.push(transformElementWithLocators(element, strategyMap));
|
|
90
96
|
} catch (error) {
|
|
91
97
|
log.error(`Error generating locators for element at path ${element.path}:`, error);
|
|
@@ -99,8 +105,8 @@ function traverseAndProcessElements(
|
|
|
99
105
|
element: JSONElement | null,
|
|
100
106
|
sourceXML: string,
|
|
101
107
|
isNative: boolean,
|
|
102
|
-
automationName:
|
|
103
|
-
filters:
|
|
108
|
+
automationName: DriverName,
|
|
109
|
+
filters: LocatorGenerationOptions,
|
|
104
110
|
results: ElementWithLocators[],
|
|
105
111
|
): void {
|
|
106
112
|
if (!element) {
|
|
@@ -7,6 +7,7 @@ import log from '../logger.js';
|
|
|
7
7
|
import {isEmpty, omitNilValues} from '../utils/collection.js';
|
|
8
8
|
import {childNodesOf, domToXML, findDOMNodeByPath, xmlToDOM} from './source-parsing.js';
|
|
9
9
|
import type {JSONElement, ElementAttributes} from './source-parsing.js';
|
|
10
|
+
import type {DriverName} from './types.js';
|
|
10
11
|
|
|
11
12
|
// Attributes on nodes that are likely to be unique to the node so we should consider first when
|
|
12
13
|
// suggesting xpath locators. These are considered IN ORDER.
|
|
@@ -84,7 +85,8 @@ export function getComplexSuggestedLocators(
|
|
|
84
85
|
path: string,
|
|
85
86
|
sourceDoc: XMLDocument,
|
|
86
87
|
isNative: boolean,
|
|
87
|
-
automationName:
|
|
88
|
+
automationName: DriverName,
|
|
89
|
+
xpathAttributes: string[] = [],
|
|
88
90
|
): Record<string, string> {
|
|
89
91
|
const complexLocators: Record<string, string | null> = {};
|
|
90
92
|
const domNode = findDOMNodeByPath(path, sourceDoc);
|
|
@@ -103,7 +105,7 @@ export function getComplexSuggestedLocators(
|
|
|
103
105
|
}
|
|
104
106
|
}
|
|
105
107
|
}
|
|
106
|
-
complexLocators.xpath = getOptimalXPath(sourceDoc, domNode);
|
|
108
|
+
complexLocators.xpath = getOptimalXPath(sourceDoc, domNode, xpathAttributes);
|
|
107
109
|
|
|
108
110
|
// Remove entries for locators where the optimal selector could not be found
|
|
109
111
|
return omitNilValues(complexLocators);
|
|
@@ -116,11 +118,18 @@ export function getSuggestedLocators(
|
|
|
116
118
|
selectedElement: JSONElement,
|
|
117
119
|
sourceXML: string,
|
|
118
120
|
isNative: boolean,
|
|
119
|
-
automationName:
|
|
121
|
+
automationName: DriverName,
|
|
122
|
+
xpathAttributes: string[] = [],
|
|
120
123
|
): [string, string][] {
|
|
121
124
|
const sourceDoc = xmlToDOM(sourceXML);
|
|
122
125
|
const simpleLocators = getSimpleSuggestedLocators(selectedElement.attributes, sourceDoc, isNative);
|
|
123
|
-
const complexLocators = getComplexSuggestedLocators(
|
|
126
|
+
const complexLocators = getComplexSuggestedLocators(
|
|
127
|
+
selectedElement.path,
|
|
128
|
+
sourceDoc,
|
|
129
|
+
isNative,
|
|
130
|
+
automationName,
|
|
131
|
+
xpathAttributes,
|
|
132
|
+
);
|
|
124
133
|
|
|
125
134
|
// Combine all locators
|
|
126
135
|
const allLocators = {...simpleLocators, ...complexLocators};
|
|
@@ -171,14 +180,20 @@ export function getSuggestedLocators(
|
|
|
171
180
|
/**
|
|
172
181
|
* Get an optimal XPath for a Node
|
|
173
182
|
*/
|
|
174
|
-
export function getOptimalXPath(doc: XMLDocument, domNode: XMLNode): string | null {
|
|
183
|
+
export function getOptimalXPath(doc: XMLDocument, domNode: XMLNode, xpathAttributes: string[] = []): string | null {
|
|
175
184
|
try {
|
|
176
185
|
// BASE CASE #1: If this isn't an element, we're above the root, return empty string
|
|
177
186
|
if (!domNode.nodeName || domNode.nodeType !== 1) {
|
|
178
187
|
return '';
|
|
179
188
|
}
|
|
180
189
|
|
|
181
|
-
|
|
190
|
+
// Attribute names are data, not XPath expressions. Namespaced attributes need
|
|
191
|
+
// namespace resolution and are intentionally not accepted here.
|
|
192
|
+
if (xpathAttributes.some((name) => !/^[A-Za-z_][A-Za-z0-9_.-]*$/.test(name))) {
|
|
193
|
+
throw new Error('Invalid XPath attribute name');
|
|
194
|
+
}
|
|
195
|
+
const preferredAttributes = [...new Set([...xpathAttributes, ...UNIQUE_XPATH_ATTRIBUTES])];
|
|
196
|
+
const attrsForPairs = [...new Set([...preferredAttributes, ...MAYBE_UNIQUE_XPATH_ATTRIBUTES])];
|
|
182
197
|
const attrPairsPermutations: [string, string][] = attrsForPairs.flatMap((v1, i) =>
|
|
183
198
|
attrsForPairs.slice(i + 1).map((v2) => [v1, v2] as [string, string]),
|
|
184
199
|
);
|
|
@@ -186,7 +201,7 @@ export function getOptimalXPath(doc: XMLDocument, domNode: XMLNode): string | nu
|
|
|
186
201
|
const cases = [
|
|
187
202
|
// BASE CASE #2: If this node has a unique attribute or content attribute, return an absolute
|
|
188
203
|
// XPath with that attribute
|
|
189
|
-
|
|
204
|
+
preferredAttributes,
|
|
190
205
|
|
|
191
206
|
// BASE CASE #3: If this node has a unique pair of attributes including 'maybe' attributes,
|
|
192
207
|
// return an xpath based on that pair
|
|
@@ -246,7 +261,7 @@ export function getOptimalXPath(doc: XMLDocument, domNode: XMLNode): string | nu
|
|
|
246
261
|
}
|
|
247
262
|
|
|
248
263
|
// Make a recursive call to this nodes parents and prepend it to this xpath
|
|
249
|
-
return (domNode.parentNode ? getOptimalXPath(doc, domNode.parentNode) : '') + xpath;
|
|
264
|
+
return (domNode.parentNode ? getOptimalXPath(doc, domNode.parentNode, xpathAttributes) : '') + xpath;
|
|
250
265
|
} catch (error) {
|
|
251
266
|
// If there's an unexpected exception, abort
|
|
252
267
|
logLocatorError('XPath', error);
|
|
@@ -514,13 +529,13 @@ function getUniqueXPath(
|
|
|
514
529
|
if (!attr1Value || !attr2Value) {
|
|
515
530
|
continue;
|
|
516
531
|
}
|
|
517
|
-
xpath = `//${tagForXpath}[@${attr1Name}
|
|
532
|
+
xpath = `//${tagForXpath}[@${attr1Name}=${xpathLiteral(attr1Value)} and @${attr2Name}=${xpathLiteral(attr2Value)}]`;
|
|
518
533
|
} else {
|
|
519
534
|
const attrValue = (domNode as XMLElement).getAttribute?.(attrName as string);
|
|
520
535
|
if (!attrValue) {
|
|
521
536
|
continue;
|
|
522
537
|
}
|
|
523
|
-
xpath = `//${tagForXpath}[@${attrName}
|
|
538
|
+
xpath = `//${tagForXpath}[@${attrName}=${xpathLiteral(attrValue)}]`;
|
|
524
539
|
}
|
|
525
540
|
const [isUnique, indexIfNotUnique] = determineXpathUniqueness(xpath, doc, domNode);
|
|
526
541
|
if (isUnique) {
|
|
@@ -547,3 +562,21 @@ function getUniqueXPath(
|
|
|
547
562
|
function logLocatorError(strategy: string, error: any): void {
|
|
548
563
|
log.error(`The most optimal ${strategy} could not be determined because an error was thrown: '${error}'`);
|
|
549
564
|
}
|
|
565
|
+
|
|
566
|
+
/** XPath 1.0 has no backslash escaping; mixed quotes require concat(). */
|
|
567
|
+
function xpathLiteral(value: string): string {
|
|
568
|
+
if (!value.includes('"')) {
|
|
569
|
+
return `"${value}"`;
|
|
570
|
+
}
|
|
571
|
+
if (!value.includes("'")) {
|
|
572
|
+
return `'${value}'`;
|
|
573
|
+
}
|
|
574
|
+
return (
|
|
575
|
+
'concat(' +
|
|
576
|
+
value
|
|
577
|
+
.split('"')
|
|
578
|
+
.map((part) => `"${part}"`)
|
|
579
|
+
.join(`, '"', `) +
|
|
580
|
+
')'
|
|
581
|
+
);
|
|
582
|
+
}
|
package/src/telemetry/init.ts
CHANGED
|
@@ -11,6 +11,20 @@ let sdkStarted = false;
|
|
|
11
11
|
let shutdownRegistered = false;
|
|
12
12
|
let sdk: {start(): void; shutdown(): Promise<void>} | undefined;
|
|
13
13
|
|
|
14
|
+
/** Resolve trace-specific configuration before the general OTLP setting. */
|
|
15
|
+
export function getTraceExportProtocol(): 'http/json' | 'grpc' {
|
|
16
|
+
const protocol =
|
|
17
|
+
process.env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL?.trim() ||
|
|
18
|
+
process.env.OTEL_EXPORTER_OTLP_PROTOCOL?.trim() ||
|
|
19
|
+
'http/json';
|
|
20
|
+
if (protocol !== 'http/json' && protocol !== 'grpc') {
|
|
21
|
+
throw new Error(
|
|
22
|
+
'Unsupported OTLP trace protocol. Use http/json or grpc in OTEL_EXPORTER_OTLP_TRACES_PROTOCOL or OTEL_EXPORTER_OTLP_PROTOCOL.',
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
return protocol;
|
|
26
|
+
}
|
|
27
|
+
|
|
14
28
|
/**
|
|
15
29
|
* Initializes the OpenTelemetry SDK if telemetry is enabled and not already started.
|
|
16
30
|
* This function is idempotent and safe to call multiple times; the SDK will only
|
|
@@ -22,9 +36,12 @@ export async function initializeOpenTelemetry(): Promise<void> {
|
|
|
22
36
|
return;
|
|
23
37
|
}
|
|
24
38
|
|
|
39
|
+
const protocol = getTraceExportProtocol();
|
|
25
40
|
const [{NodeSDK}, {OTLPTraceExporter}] = await Promise.all([
|
|
26
41
|
import('@opentelemetry/sdk-node'),
|
|
27
|
-
|
|
42
|
+
protocol === 'grpc'
|
|
43
|
+
? import('@opentelemetry/exporter-trace-otlp-grpc')
|
|
44
|
+
: import('@opentelemetry/exporter-trace-otlp-http'),
|
|
28
45
|
]);
|
|
29
46
|
|
|
30
47
|
const nodeSdk = new NodeSDK({
|
|
@@ -43,19 +43,12 @@ export const gestureSchema = z.object({
|
|
|
43
43
|
.enum(GESTURE_ACTIONS)
|
|
44
44
|
.describe(
|
|
45
45
|
[
|
|
46
|
-
'
|
|
47
|
-
'
|
|
48
|
-
'
|
|
49
|
-
'
|
|
50
|
-
'
|
|
51
|
-
'
|
|
52
|
-
'- swipe: dismiss a card, switch screens or tabs, navigate a carousel, or pull-to-refresh (use speed=fast).',
|
|
53
|
-
'For scroll/swipe, provide direction or all four custom coordinates: x, y, endX, endY.',
|
|
54
|
-
'- pinch_zoom: zoom in (scale > 1) or out (scale < 1) on maps, images, or other zoomable views. Requires scale.',
|
|
55
|
-
'- scroll_to_element: scroll to find a target using strategy and selector. Direction is up/down, default down.',
|
|
56
|
-
'Stops when the element is found, page source is unchanged after a scroll (likely end of content), or maxScrollAttempts is reached.',
|
|
57
|
-
'Adjust distance with scrollDistance (0.05–1) or scrollDistancePreset (small/medium/large).',
|
|
58
|
-
'- back: trigger system back navigation.',
|
|
46
|
+
'tap/double_tap/long_press: requires elementUUID or both x/y.',
|
|
47
|
+
'scroll/swipe: requires direction or x/y/endX/endY.',
|
|
48
|
+
'pinch_zoom: requires scale (<1 out, >1 in).',
|
|
49
|
+
'scroll_to_element: requires strategy/selector; direction up/down (default down).',
|
|
50
|
+
'Stops on a match, unchanged page source, or maxScrollAttempts. Adjust distance with scrollDistance or scrollDistancePreset.',
|
|
51
|
+
'back: system back navigation.',
|
|
59
52
|
].join('\n'),
|
|
60
53
|
),
|
|
61
54
|
|
|
@@ -130,10 +123,7 @@ export const gestureSchema = z.object({
|
|
|
130
123
|
.min(0.01)
|
|
131
124
|
.max(10)
|
|
132
125
|
.optional()
|
|
133
|
-
.describe(
|
|
134
|
-
'Required for pinch_zoom. Scale < 1 zooms out (fingers close); scale > 1 zooms in (fingers spread). ' +
|
|
135
|
-
'Examples: 0.5 zooms out; 2.0 zooms in 2x.',
|
|
136
|
-
),
|
|
126
|
+
.describe('Required for pinch_zoom: <1 zooms out, >1 zooms in (e.g. 0.5 or 2 for 2x).'),
|
|
137
127
|
velocity: z
|
|
138
128
|
.number()
|
|
139
129
|
.min(0.1)
|
|
@@ -14,6 +14,7 @@ import {z} from 'zod';
|
|
|
14
14
|
|
|
15
15
|
import {getCurrentContext, getPageSource} from '../../command.js';
|
|
16
16
|
import {generateAllElementLocators} from '../../locators/generate-all-locators.js';
|
|
17
|
+
import type {DriverName} from '../../locators/types.js';
|
|
17
18
|
import {LOCATOR_GENERATOR_URI} from '../../resources/locator-generator.js';
|
|
18
19
|
import {isAndroidUiautomator2DriverSession, isXCUITestDriverSession} from '../../session-store.js';
|
|
19
20
|
import {clientSupportsMcpApps, isMcpAppsEnabled} from '../../ui/mcp-apps.js';
|
|
@@ -24,16 +25,37 @@ export default function generateLocators(server: FastMCP): void {
|
|
|
24
25
|
const mcpAppsEnabled = isMcpAppsEnabled();
|
|
25
26
|
server.addTool({
|
|
26
27
|
name: 'generate_locators',
|
|
27
|
-
description: `Generate
|
|
28
|
+
description: `Generate current-page locator candidates for inspection. Defaults: controls for UiAutomator2/XCUITest/Web; all elements except hierarchy for other native drivers.`,
|
|
28
29
|
_meta: mcpAppsEnabled ? {ui: {resourceUri: LOCATOR_GENERATOR_URI}} : undefined,
|
|
29
30
|
parameters: z.object({
|
|
31
|
+
query: z
|
|
32
|
+
.string()
|
|
33
|
+
.optional()
|
|
34
|
+
.describe(
|
|
35
|
+
'Case-insensitive literal substring of an element tag or attribute value (name, label, text, ID, etc.). Searches all element types, including hidden elements. Blank or omitted keeps the default driver-specific filter.',
|
|
36
|
+
),
|
|
37
|
+
xpathAttributes: z
|
|
38
|
+
.array(
|
|
39
|
+
z
|
|
40
|
+
.string()
|
|
41
|
+
.trim()
|
|
42
|
+
.regex(/^[A-Za-z_][A-Za-z0-9_.-]*$/),
|
|
43
|
+
)
|
|
44
|
+
.max(20)
|
|
45
|
+
.optional()
|
|
46
|
+
.describe(
|
|
47
|
+
'Additional XPath attribute names in priority order, e.g. AutomationId or data-testid. Tried before built-in attributes; defaults remain as fallback. No namespace prefixes or XPath expressions.',
|
|
48
|
+
),
|
|
30
49
|
sessionId: z.string().optional().describe('Session ID; defaults to the active session.'),
|
|
31
50
|
}),
|
|
32
51
|
annotations: {
|
|
33
52
|
readOnlyHint: true,
|
|
34
53
|
openWorldHint: false,
|
|
35
54
|
},
|
|
36
|
-
execute: async (
|
|
55
|
+
execute: async (
|
|
56
|
+
args: {sessionId?: string; query?: string; xpathAttributes?: string[]},
|
|
57
|
+
context: any,
|
|
58
|
+
): Promise<any> => {
|
|
37
59
|
const {log} = context;
|
|
38
60
|
log.info('Getting page source');
|
|
39
61
|
const resolved = await resolveDriver(args.sessionId);
|
|
@@ -54,20 +76,22 @@ export default function generateLocators(server: FastMCP): void {
|
|
|
54
76
|
: driver.capabilities;
|
|
55
77
|
const driverName = String(
|
|
56
78
|
capabilities['appium:automationName'] ?? capabilities.automationName ?? '',
|
|
57
|
-
).toLowerCase();
|
|
79
|
+
).toLowerCase() as DriverName;
|
|
58
80
|
// Read the driver's current context: a cached NATIVE_APP value can be stale
|
|
59
81
|
// after attaching to a session that already has an active WebView.
|
|
60
82
|
const isNative = ['uiautomator2', 'xcuitest'].includes(driverName)
|
|
61
83
|
? (await getCurrentContext(driver)) === 'NATIVE_APP'
|
|
62
84
|
: !capabilities.browserName;
|
|
63
85
|
|
|
64
|
-
const
|
|
65
|
-
|
|
86
|
+
const query = args.query?.trim();
|
|
87
|
+
const elementsWithLocators = generateAllElementLocators(pageSource, isNative, driverName, {
|
|
88
|
+
...(query ? {query, fetchableOnly: false} : {fetchableOnly: true}),
|
|
89
|
+
...(args.xpathAttributes?.length ? {xpathAttributes: [...new Set(args.xpathAttributes)]} : {}),
|
|
66
90
|
});
|
|
67
91
|
|
|
68
92
|
const textResponse = textResult(
|
|
69
93
|
JSON.stringify({
|
|
70
|
-
interactableElements,
|
|
94
|
+
interactableElements: elementsWithLocators,
|
|
71
95
|
message: 'Page source retrieved successfully',
|
|
72
96
|
instruction: `These are the locators for the current page. Use this to generate code for the current page.
|
|
73
97
|
Using the template provided by generate://code-with-locators resource.`,
|
|
@@ -83,7 +107,7 @@ export default function generateLocators(server: FastMCP): void {
|
|
|
83
107
|
return addUIResourceToResponse(textResponse, () =>
|
|
84
108
|
createUIResource(
|
|
85
109
|
`ui://appium-mcp/locator-generator/${Date.now()}`,
|
|
86
|
-
createLocatorGeneratorUI(
|
|
110
|
+
createLocatorGeneratorUI(elementsWithLocators),
|
|
87
111
|
),
|
|
88
112
|
);
|
|
89
113
|
} catch (err: unknown) {
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/** Browser controller for the optional screenshot and XML bounds overlay. */
|
|
2
|
+
export const INSPECTOR_SCREENSHOT_SCRIPT = String.raw`
|
|
3
|
+
function createScreenshotInspector(callTool, setSource, readSourceResult) {
|
|
4
|
+
const button = document.getElementById('takeScreenshotButton');
|
|
5
|
+
const panel = document.getElementById('screenshotPanel');
|
|
6
|
+
const image = document.getElementById('inspectorScreenshot');
|
|
7
|
+
const overlay = document.getElementById('elementOverlay');
|
|
8
|
+
const select = document.getElementById('screenshotElement');
|
|
9
|
+
const status = document.getElementById('screenshotStatus');
|
|
10
|
+
const details = document.getElementById('selectedXmlElement');
|
|
11
|
+
let elements = [];
|
|
12
|
+
let screen;
|
|
13
|
+
let query = '';
|
|
14
|
+
|
|
15
|
+
function readBounds(element) {
|
|
16
|
+
// Drivers differ in attribute casing. Prefer the exact lowercase spelling if both exist.
|
|
17
|
+
function attribute(name) {
|
|
18
|
+
if (element.hasAttribute(name)) return element.getAttribute(name);
|
|
19
|
+
return Array.from(element.attributes).find(attr => attr.name.toLowerCase() === name)?.value;
|
|
20
|
+
}
|
|
21
|
+
const android = (attribute('bounds') || '').match(/^\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]$/);
|
|
22
|
+
if (android) {
|
|
23
|
+
const [, x, y, right, bottom] = android.map(Number);
|
|
24
|
+
return {x, y, width: right - x, height: bottom - y};
|
|
25
|
+
}
|
|
26
|
+
const coordinates = ['x', 'y', 'width', 'height'].map(attribute);
|
|
27
|
+
if (!coordinates.every(value => value !== undefined && value.trim() !== '')) return;
|
|
28
|
+
const values = coordinates.map(Number);
|
|
29
|
+
if (!values.every(Number.isFinite)) return;
|
|
30
|
+
const [x, y, width, height] = values;
|
|
31
|
+
return {x, y, width, height};
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function showSelection() {
|
|
35
|
+
overlay.hidden = true;
|
|
36
|
+
details.textContent = '';
|
|
37
|
+
if (!select.value) return;
|
|
38
|
+
const element = elements[Number(select.value) - 1];
|
|
39
|
+
if (!element) return;
|
|
40
|
+
// Display only the selected opening tag, not its entire subtree.
|
|
41
|
+
const clone = element.cloneNode(false);
|
|
42
|
+
details.textContent = new XMLSerializer().serializeToString(clone);
|
|
43
|
+
if (!screen) {
|
|
44
|
+
status.textContent = 'Take a screenshot to highlight this element.';
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const rect = readBounds(element);
|
|
48
|
+
if (!rect || rect.width <= 0 || rect.height <= 0) {
|
|
49
|
+
status.textContent = 'This element has no usable screen bounds.';
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
if (element.getAttribute('visible') === 'false' || element.getAttribute('displayed') === 'false') {
|
|
53
|
+
status.textContent = 'This element is marked hidden in the XML.';
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
const left = Math.max(0, rect.x);
|
|
57
|
+
const top = Math.max(0, rect.y);
|
|
58
|
+
const right = Math.min(screen.width, rect.x + rect.width);
|
|
59
|
+
const bottom = Math.min(screen.height, rect.y + rect.height);
|
|
60
|
+
if (right <= left || bottom <= top) {
|
|
61
|
+
status.textContent = 'This element is outside the captured screen.';
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
overlay.style.left = (left / screen.width * 100) + '%';
|
|
65
|
+
overlay.style.top = (top / screen.height * 100) + '%';
|
|
66
|
+
overlay.style.width = ((right - left) / screen.width * 100) + '%';
|
|
67
|
+
overlay.style.height = ((bottom - top) / screen.height * 100) + '%';
|
|
68
|
+
overlay.hidden = false;
|
|
69
|
+
status.textContent = 'XML bounds: x=' + rect.x + ', y=' + rect.y + ', width=' + rect.width + ', height=' + rect.height;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function filter(nextQuery) {
|
|
73
|
+
query = nextQuery.trim().toLowerCase();
|
|
74
|
+
const previous = select.value;
|
|
75
|
+
select.textContent = '';
|
|
76
|
+
const placeholder = document.createElement('option');
|
|
77
|
+
placeholder.value = '';
|
|
78
|
+
placeholder.textContent = 'Select an XML element to highlight';
|
|
79
|
+
select.appendChild(placeholder);
|
|
80
|
+
let retained = false;
|
|
81
|
+
elements.forEach((element, index) => {
|
|
82
|
+
const attributes = Array.from(element.attributes).map(attr => attr.value);
|
|
83
|
+
if (query && ![element.tagName, ...attributes].some(value => value.toLowerCase().includes(query))) return;
|
|
84
|
+
const option = document.createElement('option');
|
|
85
|
+
option.value = String(index + 1);
|
|
86
|
+
const name = element.getAttribute('name') || element.getAttribute('label') || element.getAttribute('text') || element.getAttribute('resource-id') || element.getAttribute('id');
|
|
87
|
+
option.textContent = (index + 1) + '. ' + element.tagName + (name ? ' — ' + name : '');
|
|
88
|
+
select.appendChild(option);
|
|
89
|
+
if (option.value === previous) retained = true;
|
|
90
|
+
});
|
|
91
|
+
select.value = retained ? previous : '';
|
|
92
|
+
showSelection();
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function updateSource(source) {
|
|
96
|
+
screen = undefined;
|
|
97
|
+
image.hidden = true;
|
|
98
|
+
overlay.hidden = true;
|
|
99
|
+
elements = [];
|
|
100
|
+
select.value = '';
|
|
101
|
+
if (source.trim().startsWith('<')) {
|
|
102
|
+
const doc = new DOMParser().parseFromString(source, 'text/xml');
|
|
103
|
+
if (!doc.getElementsByTagName('parsererror').length) elements = Array.from(doc.getElementsByTagName('*'));
|
|
104
|
+
}
|
|
105
|
+
filter(query);
|
|
106
|
+
status.textContent = 'XML updated. Take a screenshot to refresh the preview.';
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function assertSuccess(result) {
|
|
110
|
+
if (result?.isError) throw new Error((result.content || []).filter(item => item.type === 'text').map(item => item.text).join('\n') || 'Tool failed');
|
|
111
|
+
return result;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
async function capture() {
|
|
115
|
+
if (button.disabled) return;
|
|
116
|
+
button.disabled = true;
|
|
117
|
+
panel.hidden = false;
|
|
118
|
+
screen = undefined;
|
|
119
|
+
image.hidden = true;
|
|
120
|
+
overlay.hidden = true;
|
|
121
|
+
status.textContent = 'Capturing screenshot and refreshing XML…';
|
|
122
|
+
try {
|
|
123
|
+
const sourceResult = assertSuccess(await callTool('appium_get_page_source', {saveToFile: false}));
|
|
124
|
+
const sizeResult = assertSuccess(await callTool('appium_get_window_size', {}));
|
|
125
|
+
const result = assertSuccess(await callTool('appium_screenshot', {returnRawBase64: true}));
|
|
126
|
+
const sizeText = (sizeResult.content || []).filter(item => item.type === 'text').map(item => item.text).join(' ');
|
|
127
|
+
const size = sizeText.match(/Width:\s*([\d.]+),\s*Height:\s*([\d.]+)/);
|
|
128
|
+
if (!size || !(Number(size[1]) > 0 && Number(size[2]) > 0)) throw new Error('Screen size is unavailable');
|
|
129
|
+
const picture = result.structuredContent?.screenshot || (result.content || []).find(item => item.type === 'image');
|
|
130
|
+
if (!picture?.data || !['image/png', 'image/jpeg'].includes(picture.mimeType)) throw new Error('No supported screenshot returned');
|
|
131
|
+
setSource(readSourceResult(sourceResult));
|
|
132
|
+
screen = {width: Number(size[1]), height: Number(size[2])};
|
|
133
|
+
image.src = 'data:' + picture.mimeType + ';base64,' + picture.data;
|
|
134
|
+
image.hidden = false;
|
|
135
|
+
status.textContent = 'Select an XML element below. Search filters the list.';
|
|
136
|
+
} catch (error) {
|
|
137
|
+
status.textContent = 'Screenshot failed: ' + error.message;
|
|
138
|
+
} finally {
|
|
139
|
+
button.disabled = false;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
image.addEventListener('load', () => {
|
|
144
|
+
if (!screen) return;
|
|
145
|
+
// Pixel density is handled by percentages; orientation changes are not.
|
|
146
|
+
const ratio = image.naturalWidth / image.naturalHeight;
|
|
147
|
+
if (Math.abs(ratio / (screen.width / screen.height) - 1) > 0.03) {
|
|
148
|
+
screen = undefined;
|
|
149
|
+
overlay.hidden = true;
|
|
150
|
+
status.textContent = 'Screenshot and screen dimensions differ. Capture again before highlighting.';
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
image.addEventListener('error', () => {
|
|
154
|
+
screen = undefined;
|
|
155
|
+
image.hidden = true;
|
|
156
|
+
overlay.hidden = true;
|
|
157
|
+
status.textContent = 'Unable to display screenshot.';
|
|
158
|
+
});
|
|
159
|
+
button.addEventListener('click', capture);
|
|
160
|
+
select.addEventListener('change', showSelection);
|
|
161
|
+
return {updateSource, filter};
|
|
162
|
+
}
|
|
163
|
+
`;
|
package/src/ui/mcp-ui-utils.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import {XML_FORMAT_SCRIPT, XML_SEARCH_SCRIPT} from './xml-inspector-scripts.js';
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* MCP-UI Utility Functions
|
|
3
5
|
*
|
|
@@ -940,7 +942,7 @@ export function createPageSourceInspectorUI(pageSource: string): string {
|
|
|
940
942
|
background: #1e1e1e;
|
|
941
943
|
color: #d4d4d4;
|
|
942
944
|
padding: 0;
|
|
943
|
-
overflow:
|
|
945
|
+
overflow: auto;
|
|
944
946
|
}
|
|
945
947
|
summary {
|
|
946
948
|
padding: 12px 16px;
|
|
@@ -987,7 +989,9 @@ export function createPageSourceInspectorUI(pageSource: string): string {
|
|
|
987
989
|
color: #999;
|
|
988
990
|
}
|
|
989
991
|
.viewer {
|
|
990
|
-
|
|
992
|
+
height: 320px;
|
|
993
|
+
min-height: 160px;
|
|
994
|
+
resize: vertical;
|
|
991
995
|
overflow: auto;
|
|
992
996
|
padding: 16px;
|
|
993
997
|
}
|
|
@@ -1020,6 +1024,10 @@ export function createPageSourceInspectorUI(pageSource: string): string {
|
|
|
1020
1024
|
outline: none;
|
|
1021
1025
|
border-color: #007AFF;
|
|
1022
1026
|
}
|
|
1027
|
+
mark { background: #ffd700; color: #000; }
|
|
1028
|
+
mark[data-active] { outline: 2px solid #91baff; background: #ff963c; }
|
|
1029
|
+
.search-status { padding: 6px 16px; font-size: 12px; color: #bbb; }
|
|
1030
|
+
.search-status:empty { display: none; }
|
|
1023
1031
|
</style>
|
|
1024
1032
|
</head>
|
|
1025
1033
|
<body>
|
|
@@ -1036,11 +1044,14 @@ export function createPageSourceInspectorUI(pageSource: string): string {
|
|
|
1036
1044
|
<button class="btn" onclick="generateLocators()">Generate Locators</button>
|
|
1037
1045
|
</div>
|
|
1038
1046
|
</div>
|
|
1039
|
-
<div class="
|
|
1047
|
+
<div id="searchStatus" class="search-status" role="status" aria-live="polite"></div>
|
|
1048
|
+
<div class="viewer" title="Drag the bottom-right corner to resize vertically">
|
|
1040
1049
|
<pre class="xml-content" id="xmlContent">${escapedSource}</pre>
|
|
1041
1050
|
</div>
|
|
1042
1051
|
</details>
|
|
1043
1052
|
<script>
|
|
1053
|
+
${XML_FORMAT_SCRIPT}
|
|
1054
|
+
${XML_SEARCH_SCRIPT}
|
|
1044
1055
|
function copyToClipboard() {
|
|
1045
1056
|
const text = document.getElementById('xmlContent').textContent;
|
|
1046
1057
|
navigator.clipboard.writeText(text).then(() => {
|
|
@@ -1049,13 +1060,9 @@ export function createPageSourceInspectorUI(pageSource: string): string {
|
|
|
1049
1060
|
}
|
|
1050
1061
|
|
|
1051
1062
|
function formatXML() {
|
|
1052
|
-
const content = document.getElementById('xmlContent').textContent;
|
|
1053
1063
|
try {
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
const serializer = new XMLSerializer();
|
|
1057
|
-
const formatted = serializer.serializeToString(xmlDoc);
|
|
1058
|
-
document.getElementById('xmlContent').textContent = formatted;
|
|
1064
|
+
displayedSource = formatXml(originalSource);
|
|
1065
|
+
renderSource(document.getElementById('searchBox').value);
|
|
1059
1066
|
} catch (e) {
|
|
1060
1067
|
alert('Failed to format XML');
|
|
1061
1068
|
}
|
|
@@ -1066,50 +1073,21 @@ export function createPageSourceInspectorUI(pageSource: string): string {
|
|
|
1066
1073
|
type: 'tool',
|
|
1067
1074
|
payload: {
|
|
1068
1075
|
toolName: 'generate_locators',
|
|
1069
|
-
params: {}
|
|
1076
|
+
params: {query: document.getElementById('searchBox').value.trim()}
|
|
1070
1077
|
}
|
|
1071
1078
|
}, '*');
|
|
1072
1079
|
}
|
|
1073
1080
|
|
|
1074
1081
|
const xmlContent = document.getElementById('xmlContent');
|
|
1075
1082
|
const originalSource = xmlContent.textContent;
|
|
1083
|
+
let displayedSource = originalSource;
|
|
1076
1084
|
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
const textLower = text.toLowerCase();
|
|
1080
|
-
let position = 0;
|
|
1081
|
-
|
|
1082
|
-
while (position < text.length) {
|
|
1083
|
-
const matchIndex = textLower.indexOf(searchLower, position);
|
|
1084
|
-
if (matchIndex === -1) {
|
|
1085
|
-
container.appendChild(document.createTextNode(text.slice(position)));
|
|
1086
|
-
return;
|
|
1087
|
-
}
|
|
1088
|
-
|
|
1089
|
-
if (matchIndex > position) {
|
|
1090
|
-
container.appendChild(
|
|
1091
|
-
document.createTextNode(text.slice(position, matchIndex))
|
|
1092
|
-
);
|
|
1093
|
-
}
|
|
1094
|
-
|
|
1095
|
-
const mark = document.createElement('mark');
|
|
1096
|
-
mark.style.background = '#ffd700';
|
|
1097
|
-
mark.style.color = '#000';
|
|
1098
|
-
mark.textContent = text.slice(matchIndex, matchIndex + searchTerm.length);
|
|
1099
|
-
container.appendChild(mark);
|
|
1100
|
-
position = matchIndex + searchTerm.length;
|
|
1101
|
-
}
|
|
1102
|
-
}
|
|
1103
|
-
|
|
1085
|
+
const searchBox = document.getElementById('searchBox');
|
|
1086
|
+
const xmlSearch = createXmlSearch(xmlContent, document.getElementById('searchStatus'));
|
|
1104
1087
|
function renderSource(searchTerm = '') {
|
|
1105
|
-
|
|
1106
|
-
if (!searchTerm) {
|
|
1107
|
-
xmlContent.textContent = originalSource;
|
|
1108
|
-
return;
|
|
1109
|
-
}
|
|
1110
|
-
|
|
1111
|
-
appendTextWithHighlights(xmlContent, originalSource, searchTerm);
|
|
1088
|
+
xmlSearch.render(displayedSource, searchTerm);
|
|
1112
1089
|
}
|
|
1090
|
+
searchBox.addEventListener('keydown', xmlSearch.handleKeyDown);
|
|
1113
1091
|
|
|
1114
1092
|
// Search functionality
|
|
1115
1093
|
document.getElementById('searchBox').addEventListener('input', (e) => {
|