appium-mcp 1.74.2 → 1.75.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/dist/command.d.ts +6 -0
- package/dist/command.d.ts.map +1 -1
- package/dist/command.js +14 -0
- package/dist/command.js.map +1 -1
- package/dist/tests/tools/gestures/ai-element.test.d.ts +2 -0
- package/dist/tests/tools/gestures/ai-element.test.d.ts.map +1 -0
- package/dist/tests/tools/gestures/ai-element.test.js +93 -0
- package/dist/tests/tools/gestures/ai-element.test.js.map +1 -0
- package/dist/tools/gestures/gesture.d.ts.map +1 -1
- package/dist/tools/gestures/gesture.js +10 -1
- package/dist/tools/gestures/gesture.js.map +1 -1
- package/dist/tools/gestures/handlers/ai-element.d.ts +56 -0
- package/dist/tools/gestures/handlers/ai-element.d.ts.map +1 -0
- package/dist/tools/gestures/handlers/ai-element.js +95 -0
- package/dist/tools/gestures/handlers/ai-element.js.map +1 -0
- package/dist/tools/gestures/handlers/pinch.d.ts.map +1 -1
- package/dist/tools/gestures/handlers/pinch.js +31 -13
- package/dist/tools/gestures/handlers/pinch.js.map +1 -1
- package/dist/tools/gestures/handlers/swipe-scroll.d.ts.map +1 -1
- package/dist/tools/gestures/handlers/swipe-scroll.js +16 -2
- package/dist/tools/gestures/handlers/swipe-scroll.js.map +1 -1
- package/dist/tools/gestures/handlers/tap.d.ts.map +1 -1
- package/dist/tools/gestures/handlers/tap.js +51 -41
- package/dist/tools/gestures/handlers/tap.js.map +1 -1
- package/dist/tools/gestures/schema.d.ts +2 -1
- package/dist/tools/gestures/schema.d.ts.map +1 -1
- package/dist/tools/gestures/schema.js +3 -1
- package/dist/tools/gestures/schema.js.map +1 -1
- package/package.json +1 -1
- package/server.json +2 -2
- package/src/command.ts +14 -0
- package/src/resources/submodules.zip +0 -0
- package/src/tools/gestures/gesture.ts +16 -1
- package/src/tools/gestures/handlers/ai-element.ts +140 -0
- package/src/tools/gestures/handlers/pinch.ts +35 -18
- package/src/tools/gestures/handlers/swipe-scroll.ts +20 -7
- package/src/tools/gestures/handlers/tap.ts +56 -50
- package/src/tools/gestures/schema.ts +3 -1
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { getPlatformName, PLATFORM } from '../../../session-store.js';
|
|
2
|
-
import { execute,
|
|
2
|
+
import { execute, getWindowRect, performActions } from '../../../command.js';
|
|
3
3
|
import { errorResult, textResult, toolErrorMessage, } from '../../tool-response.js';
|
|
4
|
+
import { isAIEnabled } from '../../ai/config.js';
|
|
5
|
+
import { aiDisabledResult, isAiElementUUID, resolveTargetRect, } from './ai-element.js';
|
|
4
6
|
const SCROLL_DURATION_MS = 800;
|
|
5
7
|
const SCROLL_INITIAL_PAUSE_MS = 250;
|
|
6
8
|
/** Pointer move duration for programmatic vertical scroll (matches legacy appium_scroll). */
|
|
@@ -75,6 +77,9 @@ const FLIPPED_DIRECTION = {
|
|
|
75
77
|
};
|
|
76
78
|
export async function handleScroll(driver, args) {
|
|
77
79
|
try {
|
|
80
|
+
if (isAiElementUUID(args.elementUUID) && !isAIEnabled()) {
|
|
81
|
+
return aiDisabledResult();
|
|
82
|
+
}
|
|
78
83
|
const platform = getPlatformName(driver);
|
|
79
84
|
// Android scroll follows the scrollbar convention (down = reveal content below),
|
|
80
85
|
// so flip the user's direction before computing the W3C drag coords.
|
|
@@ -109,6 +114,9 @@ export async function handleScroll(driver, args) {
|
|
|
109
114
|
}
|
|
110
115
|
export async function handleSwipe(driver, args) {
|
|
111
116
|
try {
|
|
117
|
+
if (isAiElementUUID(args.elementUUID) && !isAIEnabled()) {
|
|
118
|
+
return aiDisabledResult();
|
|
119
|
+
}
|
|
112
120
|
const coordsResult = await resolveCoords(driver, args);
|
|
113
121
|
if ('error' in coordsResult) {
|
|
114
122
|
return errorResult(`swipe: ${coordsResult.error}`);
|
|
@@ -190,7 +198,13 @@ function coordsForDirection(direction, rect) {
|
|
|
190
198
|
async function resolveCoords(driver, args) {
|
|
191
199
|
if (args.direction) {
|
|
192
200
|
if (args.elementUUID) {
|
|
193
|
-
|
|
201
|
+
// ai-element UUIDs are coordinate UUIDs, not real element ids; their
|
|
202
|
+
// rect is synthesised from the bbox in the UUID itself rather than
|
|
203
|
+
// fetched from the driver.
|
|
204
|
+
const rect = await resolveTargetRect(driver, args.elementUUID);
|
|
205
|
+
if ('error' in rect) {
|
|
206
|
+
return rect;
|
|
207
|
+
}
|
|
194
208
|
return coordsForDirection(args.direction, {
|
|
195
209
|
x: rect.x,
|
|
196
210
|
y: rect.y,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"swipe-scroll.js","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/swipe-scroll.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,
|
|
1
|
+
{"version":3,"file":"swipe-scroll.js","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/swipe-scroll.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EACL,WAAW,EACX,UAAU,EACV,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,iBAAiB,GAClB,MAAM,iBAAiB,CAAC;AAazB,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,uBAAuB,GAAG,GAAG,CAAC;AACpC,6FAA6F;AAC7F,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAOpC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,MAAsB,EACtB,OAA8B;IAE9B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CACvC,MAAM,EACN,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,QAAQ,CACjB,CAAC;IAEF,IAAI,eAAe,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,OAAO,EAAE,CAAC;QACjD,MAAM,cAAc,CAAC,MAAM,EAAE;YAC3B;gBACE,IAAI,EAAE,SAAS;gBACf,EAAE,EAAE,SAAS;gBACb,UAAU,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE;gBACpC,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE;oBAC1D,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,EAAE;oBAClC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE;oBACpD;wBACE,IAAI,EAAE,aAAa;wBACnB,QAAQ,EAAE,uBAAuB;wBACjC,CAAC,EAAE,MAAM;wBACT,CAAC,EAAE,IAAI;qBACR;oBACD,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;iBACjC;aACF;SACF,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,eAAe,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,GAAG,EAAE,CAAC;QACpD,MAAM,OAAO,CAAC,MAAM,EAAE,gBAAgB,EAAE;YACtC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,MAAM;YACN,MAAM;YACN,IAAI,EAAE,MAAM;YACZ,IAAI;SACL,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CACb,yBAAyB,eAAe,CAAC,MAAM,CAAC,uCAAuC,CACxF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CACvB,MAAc,EACd,SAAwB,EACxB,QAAgB;IAEhB,MAAM,GAAG,GAAG,MAAM,GAAG,GAAG,CAAC;IACzB,MAAM,QAAQ,GAAG,MAAM,GAAG,GAAG,GAAG,QAAQ,CAAC;IACzC,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,QAAQ,CAAC;YAClC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,QAAQ,CAAC;SACjC,CAAC;IACJ,CAAC;IACD,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,QAAQ,CAAC;QAClC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,QAAQ,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,MAAM,oBAAoB,GAAG;IAC3B,IAAI,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE;IAC1C,MAAM,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE;IAC5C,IAAI,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE;CAChC,CAAC;AAEX,MAAM,iBAAiB,GAAiC;IACtD,EAAE,EAAE,MAAM;IACV,IAAI,EAAE,IAAI;IACV,IAAI,EAAE,OAAO;IACb,KAAK,EAAE,MAAM;CACd,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAsB,EACtB,IAAiB;IAEjB,IAAI,CAAC;QACH,IAAI,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACxD,OAAO,gBAAgB,EAAE,CAAC;QAC5B,CAAC;QACD,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QACzC,iFAAiF;QACjF,qEAAqE;QACrE,MAAM,UAAU,GACd,QAAQ,KAAK,QAAQ,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS;YAC7C,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE;YAC3D,CAAC,CAAC,IAAI,CAAC;QACX,MAAM,YAAY,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,YAAY,EAAE,CAAC;YAC5B,OAAO,WAAW,CAAC,WAAW,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,kBAAkB,CAAC;QAErD,IAAI,QAAQ,KAAK,QAAQ,CAAC,GAAG,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACrE,MAAM,OAAO,CAAC,MAAM,EAAE,gBAAgB,EAAE;gBACtC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,uBAAuB,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO,UAAU,CACf,IAAI,CAAC,SAAS;YACZ,CAAC,CAAC,yBAAyB,IAAI,CAAC,SAAS,GAAG;YAC5C,CAAC,CAAC,+BAA+B,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,SAAS,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,IAAI,CAC3G,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,WAAW,CAAC,qBAAqB,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAsB,EACtB,IAAiB;IAEjB,IAAI,CAAC;QACH,IAAI,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACxD,OAAO,gBAAgB,EAAE,CAAC;QAC5B,CAAC;QACD,MAAM,YAAY,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACvD,IAAI,OAAO,IAAI,YAAY,EAAE,CAAC;YAC5B,OAAO,WAAW,CAAC,UAAU,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC;QACrC,MAAM,OAAO,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC;QACnD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QAC1C,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QAEzC,6FAA6F;QAC7F,IACE,QAAQ,KAAK,QAAQ,CAAC,GAAG;YACzB,IAAI,CAAC,SAAS;YACd,CAAC,IAAI,CAAC,WAAW;YACjB,KAAK,KAAK,MAAM,EAChB,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YACxE,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC;oBACH,MAAM,OAAO,CAAC,MAAM,EAAE,+BAA+B,EAAE;wBACrD,KAAK,EAAE,MAAM,CAAC,MAAM;wBACpB,KAAK,EAAE,MAAM,CAAC,MAAM;wBACpB,GAAG,EAAE,MAAM,CAAC,IAAI;wBAChB,GAAG,EAAE,MAAM,CAAC,IAAI;wBAChB,QAAQ,EAAE,QAAQ,GAAG,IAAI;qBAC1B,CAAC,CAAC;gBACL,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;gBAC/D,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,UAAU,CACf,IAAI,CAAC,SAAS;YACZ,CAAC,CAAC,uBAAuB,IAAI,CAAC,SAAS,WAAW,KAAK,IAAI;YAC3D,CAAC,CAAC,6BAA6B,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,SAAS,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,YAAY,KAAK,IAAI,CAC1H,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,WAAW,CAAC,oBAAoB,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,SAAoB,EAAE,IAAU;IAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrD,QAAQ,SAAS,EAAE,CAAC;QAClB,KAAK,MAAM;YACT,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;gBAC7C,MAAM,EAAE,OAAO;gBACf,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;gBAC3C,IAAI,EAAE,OAAO;aACd,CAAC;QACJ,KAAK,OAAO;YACV,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;gBAC7C,MAAM,EAAE,OAAO;gBACf,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;gBAC3C,IAAI,EAAE,OAAO;aACd,CAAC;QACJ,KAAK,IAAI;YACP,OAAO;gBACL,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;gBAC9C,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;aAC7C,CAAC;QACJ,KAAK,MAAM;YACT,OAAO;gBACL,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;gBAC9C,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;aAC7C,CAAC;IACN,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,MAAsB,EACtB,IAAiB;IAEjB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QACnB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,qEAAqE;YACrE,mEAAmE;YACnE,2BAA2B;YAC3B,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC/D,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,kBAAkB,CAAC,IAAI,CAAC,SAAS,EAAE;gBACxC,CAAC,EAAE,IAAI,CAAC,CAAC;gBACT,CAAC,EAAE,IAAI,CAAC,CAAC;gBACT,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC,CAAC;QACL,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,kBAAkB,CAAC,IAAI,CAAC,SAAS,EAAE;YACxC,CAAC,EAAE,CAAC;YACJ,CAAC,EAAE,CAAC;YACJ,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;IACL,CAAC;IAED,IACE,IAAI,CAAC,CAAC,KAAK,SAAS;QACpB,IAAI,CAAC,CAAC,KAAK,SAAS;QACpB,IAAI,CAAC,IAAI,KAAK,SAAS;QACvB,IAAI,CAAC,IAAI,KAAK,SAAS,EACvB,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;IAC9E,CAAC;IAED,OAAO;QACL,KAAK,EACH,6EAA6E;KAChF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,MAAsB,EACtB,MAAc,EACd,QAAgB,EAChB,YAAoB;IAEpB,MAAM,OAAO,GAAG;QACd,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;QACxE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,EAAE;QAClC,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE;QACjE,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;KACjC,CAAC;IACF,MAAM,cAAc,CAAC,MAAM,EAAE;QAC3B;YACE,IAAI,EAAE,SAAS;YACf,EAAE,EAAE,SAAS;YACb,UAAU,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE;YACpC,OAAO;SACR;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tap.d.ts","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/tap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"tap.d.ts","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/tap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAoBhE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,wBAAsB,SAAS,CAC7B,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,aAAa,CAAC,CAqCxB;AAED,wBAAsB,eAAe,CACnC,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,aAAa,CAAC,CAwDxB;AAED,wBAAsB,eAAe,CACnC,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,aAAa,CAAC,CA0ExB"}
|
|
@@ -2,23 +2,21 @@ import { getPlatformName, PLATFORM } from '../../../session-store.js';
|
|
|
2
2
|
import { elementClick, execute, getElementRect, performActions, } from '../../../command.js';
|
|
3
3
|
import { errorResult, textResult, textResultWithPrimaryElementId, toolErrorMessage, } from '../../tool-response.js';
|
|
4
4
|
import { isAIEnabled } from '../../ai/config.js';
|
|
5
|
-
|
|
6
|
-
const AI_DISABLED_REJECTION = `Received an ai-element: UUID, but the appium_ai tool is not registered ` +
|
|
7
|
-
`(AI_VISION_ENABLED is not set to true). Use appium_find_element to get a real ` +
|
|
8
|
-
`element UUID, or enable AI_VISION_ENABLED=true with the required AI_VISION_* keys.`;
|
|
5
|
+
import { aiDisabledResult, isAiElementUUID, parseAiElement, } from './ai-element.js';
|
|
9
6
|
export async function handleTap(driver, args) {
|
|
10
7
|
try {
|
|
11
8
|
if (args.elementUUID) {
|
|
12
|
-
if (args.elementUUID
|
|
9
|
+
if (isAiElementUUID(args.elementUUID)) {
|
|
13
10
|
if (!isAIEnabled()) {
|
|
14
|
-
return
|
|
11
|
+
return aiDisabledResult();
|
|
15
12
|
}
|
|
16
|
-
const parsed =
|
|
13
|
+
const parsed = parseAiElement(args.elementUUID);
|
|
17
14
|
if ('error' in parsed) {
|
|
18
15
|
return errorResult(parsed.error);
|
|
19
16
|
}
|
|
20
|
-
|
|
21
|
-
|
|
17
|
+
const { x, y } = parsed.center;
|
|
18
|
+
await performActions(driver, w3cTapAt(x, y));
|
|
19
|
+
return textResultWithPrimaryElementId(args.elementUUID, `Successfully tapped at AI element coordinates (${x}, ${y}).`);
|
|
22
20
|
}
|
|
23
21
|
await elementClick(driver, args.elementUUID);
|
|
24
22
|
return textResultWithPrimaryElementId(args.elementUUID, `Successfully tapped element ${args.elementUUID}.`);
|
|
@@ -42,9 +40,23 @@ export async function handleDoubleTap(driver, args) {
|
|
|
42
40
|
// other JS-bridged) gesture handlers. Resolve elements to coordinates
|
|
43
41
|
// and use the same W3C sequence everywhere, verified against vodqa.
|
|
44
42
|
if (args.elementUUID) {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
43
|
+
if (isAiElementUUID(args.elementUUID)) {
|
|
44
|
+
if (!isAIEnabled()) {
|
|
45
|
+
return aiDisabledResult();
|
|
46
|
+
}
|
|
47
|
+
const parsed = parseAiElement(args.elementUUID);
|
|
48
|
+
if ('error' in parsed) {
|
|
49
|
+
return errorResult(parsed.error);
|
|
50
|
+
}
|
|
51
|
+
// ai-element UUIDs are coordinates, not real element ids — use W3C
|
|
52
|
+
// double-tap at the parsed centre (same as main for real elements).
|
|
53
|
+
({ x, y } = parsed.center);
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
const coords = await resolveCoordsFromElement(driver, args.elementUUID);
|
|
57
|
+
x = coords.x;
|
|
58
|
+
y = coords.y;
|
|
59
|
+
}
|
|
48
60
|
}
|
|
49
61
|
else if (args.x !== undefined && args.y !== undefined) {
|
|
50
62
|
x = args.x;
|
|
@@ -83,22 +95,36 @@ export async function handleLongPress(driver, args) {
|
|
|
83
95
|
let x;
|
|
84
96
|
let y;
|
|
85
97
|
if (args.elementUUID) {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
await execute(driver, 'mobile: touchAndHold', {
|
|
90
|
-
elementId: args.elementUUID,
|
|
91
|
-
duration: duration / 1000,
|
|
92
|
-
});
|
|
93
|
-
return textResultWithPrimaryElementId(args.elementUUID, `Successfully long pressed element ${args.elementUUID} for ${duration}ms.`);
|
|
98
|
+
if (isAiElementUUID(args.elementUUID)) {
|
|
99
|
+
if (!isAIEnabled()) {
|
|
100
|
+
return aiDisabledResult();
|
|
94
101
|
}
|
|
95
|
-
|
|
96
|
-
|
|
102
|
+
const parsed = parseAiElement(args.elementUUID);
|
|
103
|
+
if ('error' in parsed) {
|
|
104
|
+
return errorResult(parsed.error);
|
|
105
|
+
}
|
|
106
|
+
// ai-element UUIDs are coordinates, not real element ids — skip the
|
|
107
|
+
// iOS `mobile: touchAndHold` fast-path and long-press at the parsed centre.
|
|
108
|
+
({ x, y } = parsed.center);
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
const platform = getPlatformName(driver);
|
|
112
|
+
if (platform === PLATFORM.ios) {
|
|
113
|
+
try {
|
|
114
|
+
await execute(driver, 'mobile: touchAndHold', {
|
|
115
|
+
elementId: args.elementUUID,
|
|
116
|
+
duration: duration / 1000,
|
|
117
|
+
});
|
|
118
|
+
return textResultWithPrimaryElementId(args.elementUUID, `Successfully long pressed element ${args.elementUUID} for ${duration}ms.`);
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
// fall through to W3C Actions fallback
|
|
122
|
+
}
|
|
97
123
|
}
|
|
124
|
+
const coords = await resolveCoordsFromElement(driver, args.elementUUID);
|
|
125
|
+
x = coords.x;
|
|
126
|
+
y = coords.y;
|
|
98
127
|
}
|
|
99
|
-
const coords = await resolveCoordsFromElement(driver, args.elementUUID);
|
|
100
|
-
x = coords.x;
|
|
101
|
-
y = coords.y;
|
|
102
128
|
}
|
|
103
129
|
else if (args.x !== undefined && args.y !== undefined) {
|
|
104
130
|
x = args.x;
|
|
@@ -126,22 +152,6 @@ export async function handleLongPress(driver, args) {
|
|
|
126
152
|
return errorResult(`Failed to perform long_press. ${toolErrorMessage(err)}`);
|
|
127
153
|
}
|
|
128
154
|
}
|
|
129
|
-
function parseAiElementCoords(uuid) {
|
|
130
|
-
const parts = uuid.split(':');
|
|
131
|
-
if (parts.length < 2) {
|
|
132
|
-
return { error: 'Invalid AI element UUID format.' };
|
|
133
|
-
}
|
|
134
|
-
const coords = parts[1].split(',');
|
|
135
|
-
if (coords.length < 2) {
|
|
136
|
-
return { error: 'Invalid AI element coordinates format.' };
|
|
137
|
-
}
|
|
138
|
-
const x = parseInt(coords[0], 10);
|
|
139
|
-
const y = parseInt(coords[1], 10);
|
|
140
|
-
if (isNaN(x) || isNaN(y)) {
|
|
141
|
-
return { error: 'Invalid AI element coordinates: not numbers.' };
|
|
142
|
-
}
|
|
143
|
-
return { x, y };
|
|
144
|
-
}
|
|
145
155
|
function w3cTapAt(x, y) {
|
|
146
156
|
return [
|
|
147
157
|
{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tap.js","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/tap.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,EACL,YAAY,EACZ,OAAO,EACP,cAAc,EACd,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,WAAW,EACX,UAAU,EACV,8BAA8B,EAC9B,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"tap.js","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/tap.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,EACL,YAAY,EACZ,OAAO,EACP,cAAc,EACd,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,WAAW,EACX,UAAU,EACV,8BAA8B,EAC9B,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,iBAAiB,CAAC;AAGzB,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,MAAsB,EACtB,IAAiB;IAEjB,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBACtC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;oBACnB,OAAO,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;gBACD,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAChD,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;oBACtB,OAAO,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnC,CAAC;gBACD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;gBAC/B,MAAM,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAC7C,OAAO,8BAA8B,CACnC,IAAI,CAAC,WAAW,EAChB,kDAAkD,CAAC,KAAK,CAAC,IAAI,CAC9D,CAAC;YACJ,CAAC;YACD,MAAM,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC7C,OAAO,8BAA8B,CACnC,IAAI,CAAC,WAAW,EAChB,+BAA+B,IAAI,CAAC,WAAW,GAAG,CACnD,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACjD,OAAO,WAAW,CAChB,+DAA+D,CAChE,CAAC;QACJ,CAAC;QACD,MAAM,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,OAAO,UAAU,CACf,uCAAuC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAC7D,CAAC;IACJ,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,WAAW,CAAC,0BAA0B,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAsB,EACtB,IAAiB;IAEjB,IAAI,CAAC;QACH,IAAI,CAAS,CAAC;QACd,IAAI,CAAS,CAAC;QAEd,sEAAsE;QACtE,sEAAsE;QACtE,sEAAsE;QACtE,oEAAoE;QACpE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBACtC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;oBACnB,OAAO,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;gBACD,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAChD,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;oBACtB,OAAO,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnC,CAAC;gBACD,mEAAmE;gBACnE,oEAAoE;gBACpE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACxE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;gBACb,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;YACf,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACxD,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YACX,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QACb,CAAC;aAAM,CAAC;YACN,OAAO,WAAW,CAChB,sEAAsE,CACvE,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,CAAC,MAAM,EAAE;YAC3B;gBACE,IAAI,EAAE,SAAS;gBACf,EAAE,EAAE,SAAS;gBACb,UAAU,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE;gBACpC,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE;oBAC3C,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,EAAE;oBAClC,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;oBAChC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE;oBAChC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,EAAE;oBAClC,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;iBACjC;aACF;SACF,CAAC,CAAC;QACH,OAAO,UAAU,CAAC,kCAAkC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnE,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,WAAW,CAChB,iCAAiC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CACzD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAsB,EACtB,IAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;QACvC,IAAI,QAAQ,GAAG,GAAG,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;YACvC,OAAO,WAAW,CAChB,uDAAuD,CACxD,CAAC;QACJ,CAAC;QAED,IAAI,CAAS,CAAC;QACd,IAAI,CAAS,CAAC;QAEd,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBACtC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;oBACnB,OAAO,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;gBACD,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAChD,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;oBACtB,OAAO,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnC,CAAC;gBACD,oEAAoE;gBACpE,4EAA4E;gBAC5E,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;gBACzC,IAAI,QAAQ,KAAK,QAAQ,CAAC,GAAG,EAAE,CAAC;oBAC9B,IAAI,CAAC;wBACH,MAAM,OAAO,CAAC,MAAM,EAAE,sBAAsB,EAAE;4BAC5C,SAAS,EAAE,IAAI,CAAC,WAAW;4BAC3B,QAAQ,EAAE,QAAQ,GAAG,IAAI;yBAC1B,CAAC,CAAC;wBACH,OAAO,8BAA8B,CACnC,IAAI,CAAC,WAAW,EAChB,qCAAqC,IAAI,CAAC,WAAW,QAAQ,QAAQ,KAAK,CAC3E,CAAC;oBACJ,CAAC;oBAAC,MAAM,CAAC;wBACP,uCAAuC;oBACzC,CAAC;gBACH,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACxE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;gBACb,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;YACf,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACxD,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YACX,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QACb,CAAC;aAAM,CAAC;YACN,OAAO,WAAW,CAChB,sEAAsE,CACvE,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,CAAC,MAAM,EAAE;YAC3B;gBACE,IAAI,EAAE,SAAS;gBACf,EAAE,EAAE,SAAS;gBACb,UAAU,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE;gBACpC,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE;oBAC1C,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,EAAE;oBAClC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE;oBAC3B,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;iBACjC;aACF;SACF,CAAC,CAAC;QACH,OAAO,UAAU,CACf,iCAAiC,CAAC,KAAK,CAAC,SAAS,QAAQ,KAAK,CAC/D,CAAC;IACJ,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,WAAW,CAChB,iCAAiC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CACzD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,CAAS;IACpC,OAAO;QACL;YACE,IAAI,EAAE,SAAS;YACf,EAAE,EAAE,SAAS;YACb,UAAU,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE;YACpC,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE;gBAC1C,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,EAAE;gBAClC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE;gBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;aACjC;SACF;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAAsB,EACtB,WAAmB;IAEnB,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACvD,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;QACtC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;KACxC,CAAC;AACJ,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
export declare const GESTURE_ACTIONS: readonly ["tap", "double_tap", "long_press", "scroll", "swipe", "pinch_zoom", "scroll_to_element"];
|
|
2
|
+
export declare const GESTURE_ACTIONS: readonly ["tap", "double_tap", "long_press", "scroll", "swipe", "pinch_zoom", "scroll_to_element", "back"];
|
|
3
3
|
export type GestureAction = (typeof GESTURE_ACTIONS)[number];
|
|
4
4
|
export declare const SWIPE_SPEEDS: readonly ["slow", "normal", "fast"];
|
|
5
5
|
export type SwipeSpeed = (typeof SWIPE_SPEEDS)[number];
|
|
@@ -15,6 +15,7 @@ export declare const gestureSchema: z.ZodObject<{
|
|
|
15
15
|
swipe: "swipe";
|
|
16
16
|
pinch_zoom: "pinch_zoom";
|
|
17
17
|
scroll_to_element: "scroll_to_element";
|
|
18
|
+
back: "back";
|
|
18
19
|
}>;
|
|
19
20
|
elementUUID: z.ZodOptional<z.ZodString>;
|
|
20
21
|
x: z.ZodOptional<z.ZodNumber>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/tools/gestures/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,eAAO,MAAM,eAAe,
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/tools/gestures/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,eAAO,MAAM,eAAe,4GASlB,CAAC;AAEX,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7D,eAAO,MAAM,YAAY,qCAAsC,CAAC;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvD,eAAO,MAAM,uBAAuB,uCAAwC,CAAC;AAC7E,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE5E,eAAO,MAAM,kBAAkB,yJAUrB,CAAC;AAEX,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA0JxB,CAAC;AAEH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAC"}
|
|
@@ -12,6 +12,7 @@ export const GESTURE_ACTIONS = [
|
|
|
12
12
|
'swipe',
|
|
13
13
|
'pinch_zoom',
|
|
14
14
|
'scroll_to_element',
|
|
15
|
+
'back',
|
|
15
16
|
];
|
|
16
17
|
export const SWIPE_SPEEDS = ['slow', 'normal', 'fast'];
|
|
17
18
|
export const SCROLL_DISTANCE_PRESETS = ['small', 'medium', 'large'];
|
|
@@ -38,7 +39,8 @@ export const gestureSchema = z.object({
|
|
|
38
39
|
`pinch_zoom: zoom in (scale > 1) or out (scale < 1) on maps, images, or any zoomable view. ` +
|
|
39
40
|
`scroll_to_element: scroll until a specific element is on screen (strategy + selector + direction up|down). ` +
|
|
40
41
|
`Stops when the element is found, page source is unchanged after a scroll (end of scrollable content), or maxScrollAttempts is reached. ` +
|
|
41
|
-
`Optional scrollDistance (0.05–1) or scrollDistancePreset (small|medium|large)
|
|
42
|
+
`Optional scrollDistance (0.05–1) or scrollDistancePreset (small|medium|large). ` +
|
|
43
|
+
`back: triggers the system back navigation (e.g., Android back button or iOS navigation controller pop).`),
|
|
42
44
|
elementUUID: elementUUIDScheme
|
|
43
45
|
.optional()
|
|
44
46
|
.describe(`UUID of the element to act on. ` +
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/tools/gestures/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,MAAM,YAAY,GAAG,WAAW,EAAE;IAChC,CAAC,CAAC,oFAAoF;IACtF,CAAC,CAAC,EAAE,CAAC;AAEP,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK;IACL,YAAY;IACZ,YAAY;IACZ,QAAQ;IACR,OAAO;IACP,YAAY;IACZ,mBAAmB;
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/tools/gestures/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,MAAM,YAAY,GAAG,WAAW,EAAE;IAChC,CAAC,CAAC,oFAAoF;IACtF,CAAC,CAAC,EAAE,CAAC;AAEP,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK;IACL,YAAY;IACZ,YAAY;IACZ,QAAQ;IACR,OAAO;IACP,YAAY;IACZ,mBAAmB;IACnB,MAAM;CACE,CAAC;AAIX,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAU,CAAC;AAGhE,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAU,CAAC;AAG7E,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,kBAAkB;IAClB,IAAI;IACJ,uBAAuB;IACvB,kBAAkB;IAClB,sBAAsB;IACtB,OAAO;IACP,MAAM;IACN,YAAY;IACZ,cAAc;CACN,CAAC;AAEX,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,MAAM,EAAE,CAAC;SACN,IAAI,CAAC,eAAe,CAAC;SACrB,QAAQ,CACP,sBAAsB;QACpB,uCAAuC;QACvC,uFAAuF;QACvF,sEAAsE;QACtE,0DAA0D;QAC1D,2GAA2G;QAC3G,4FAA4F;QAC5F,6GAA6G;QAC7G,yIAAyI;QACzI,iFAAiF;QACjF,yGAAyG,CAC5G;IAEH,WAAW,EAAE,iBAAiB;SAC3B,QAAQ,EAAE;SACV,QAAQ,CACP,iCAAiC;QAC/B,YAAY;QACZ,oDAAoD;QACpD,iIAAiI,CACpI;IAEH,CAAC,EAAE,CAAC;SACD,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,QAAQ,CACP,gBAAgB;QACd,4EAA4E;QAC5E,oFAAoF;QACpF,mFAAmF,CACtF;IACH,CAAC,EAAE,CAAC;SACD,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,QAAQ,CACP,gBAAgB;QACd,+CAA+C;QAC/C,2DAA2D;QAC3D,mFAAmF,CACtF;IACH,IAAI,EAAE,CAAC;SACJ,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,QAAQ,CACP,uEAAuE,CACxE;IACH,IAAI,EAAE,CAAC;SACJ,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,QAAQ,CACP,uEAAuE,CACxE;IAEH,SAAS,EAAE,CAAC;SACT,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;SACrC,QAAQ,EAAE;SACV,QAAQ,CACP,gGAAgG;QAC9F,+FAA+F,CAClG;IAEH,KAAK,EAAE,CAAC;SACL,IAAI,CAAC,YAAY,CAAC;SAClB,QAAQ,EAAE;SACV,QAAQ,CACP,8KAA8K,CAC/K;IAEH,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,KAAK,CAAC;SACV,QAAQ,EAAE;SACV,QAAQ,CACP,2FAA2F;QACzF,oFAAoF,CACvF;IAEH,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,QAAQ,CACP,wJAAwJ,CACzJ;IACH,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,QAAQ,CACP,8EAA8E,CAC/E;IAEH,QAAQ,EAAE,CAAC;SACR,IAAI,CAAC,kBAAkB,CAAC;SACxB,QAAQ,EAAE;SACV,QAAQ,CACP,qDAAqD;QACnD,gPAAgP;QAChP,sCAAsC,CACzC;IACH,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,0DAA0D,CAAC;IAEvE,iBAAiB,EAAE,CAAC;SACjB,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,OAAO,CAAC,EAAE,CAAC;SACX,QAAQ,CACP,oGAAoG,CACrG;IAEH,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,QAAQ,CACP,oGAAoG;QAClG,qGAAqG,CACxG;IAEH,oBAAoB,EAAE,CAAC;SACpB,IAAI,CAAC,uBAAuB,CAAC;SAC7B,QAAQ,EAAE;SACV,QAAQ,CACP,0HAA0H;QACxH,qCAAqC,CACxC;IAEH,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,4DAA4D,CAAC;CAC1E,CAAC,CAAC"}
|
package/package.json
CHANGED
package/server.json
CHANGED
|
@@ -3,12 +3,12 @@
|
|
|
3
3
|
"name": "io.github.appium/appium-mcp",
|
|
4
4
|
"title": "MCP Appium - Mobile Development and Automation Server",
|
|
5
5
|
"description": "MCP server for Appium mobile automation on iOS and Android devices with test creation tools.",
|
|
6
|
-
"version": "1.
|
|
6
|
+
"version": "1.75.0",
|
|
7
7
|
"packages": [
|
|
8
8
|
{
|
|
9
9
|
"registryType": "npm",
|
|
10
10
|
"identifier": "appium-mcp",
|
|
11
|
-
"version": "1.
|
|
11
|
+
"version": "1.75.0",
|
|
12
12
|
"transport": {
|
|
13
13
|
"type": "stdio"
|
|
14
14
|
}
|
package/src/command.ts
CHANGED
|
@@ -558,3 +558,17 @@ export async function setClipboard(
|
|
|
558
558
|
contentType: 'plaintext',
|
|
559
559
|
});
|
|
560
560
|
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* Perform the "back" action on the device, which typically navigates back in the app or UI.
|
|
564
|
+
* @param driver
|
|
565
|
+
* @returns
|
|
566
|
+
*/
|
|
567
|
+
export async function back(driver: DriverInstance): Promise<void> {
|
|
568
|
+
if (isAndroidUiautomator2DriverSession(driver)) {
|
|
569
|
+
return await driver.back();
|
|
570
|
+
} else if (isXCUITestDriverSession(driver)) {
|
|
571
|
+
return await driver.back();
|
|
572
|
+
}
|
|
573
|
+
return await (driver as Client).back();
|
|
574
|
+
}
|
|
Binary file
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import type { ContentResult, FastMCP } from 'fastmcp';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
errorResult,
|
|
4
|
+
resolveDriver,
|
|
5
|
+
textResult,
|
|
6
|
+
toolErrorMessage,
|
|
7
|
+
} from '../tool-response.js';
|
|
3
8
|
import { GESTURE_ACTIONS, gestureSchema, type GestureArgs } from './schema.js';
|
|
4
9
|
import { handleTap, handleDoubleTap, handleLongPress } from './handlers/tap.js';
|
|
5
10
|
import { handleScroll, handleSwipe } from './handlers/swipe-scroll.js';
|
|
6
11
|
import { handlePinchZoom } from './handlers/pinch.js';
|
|
7
12
|
import { handleScrollToElement } from './handlers/scroll-to-element.js';
|
|
13
|
+
import { back } from '../../command.js';
|
|
8
14
|
|
|
9
15
|
export default function gesture(server: FastMCP): void {
|
|
10
16
|
server.addTool({
|
|
@@ -30,6 +36,15 @@ export default function gesture(server: FastMCP): void {
|
|
|
30
36
|
const { driver } = resolved;
|
|
31
37
|
|
|
32
38
|
switch (args.action) {
|
|
39
|
+
case 'back':
|
|
40
|
+
try {
|
|
41
|
+
await back(driver);
|
|
42
|
+
return textResult('Successfully performed back action.');
|
|
43
|
+
} catch (err: unknown) {
|
|
44
|
+
return errorResult(
|
|
45
|
+
`Failed to perform back action. ${toolErrorMessage(err)}`
|
|
46
|
+
);
|
|
47
|
+
}
|
|
33
48
|
case 'tap':
|
|
34
49
|
return handleTap(driver, args);
|
|
35
50
|
case 'double_tap':
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Helpers for the special "ai-element:" UUIDs produced by the appium_ai
|
|
3
|
+
* find_element handler.
|
|
4
|
+
*
|
|
5
|
+
* The format is:
|
|
6
|
+
*
|
|
7
|
+
* ai-element:<cx>,<cy>:<x0>,<y0>,<x1>,<y1>
|
|
8
|
+
*
|
|
9
|
+
* where (cx, cy) is the visual centre of the target and (x0,y0,x1,y1) is
|
|
10
|
+
* the bounding box returned by the vision model. Both are pixel
|
|
11
|
+
* coordinates in the screenshot's coordinate space, NOT real WebDriver
|
|
12
|
+
* element ids — so they MUST NOT be passed to driver.getElementRect or
|
|
13
|
+
* to platform-native commands like `mobile: doubleTap`, `mobile: pinch`,
|
|
14
|
+
* etc., which require a real element id.
|
|
15
|
+
*
|
|
16
|
+
* Centralising the prefix, the parser, and the rect-resolution helper
|
|
17
|
+
* here keeps every gesture handler honest: if it ever needs a rect for
|
|
18
|
+
* a UUID, it goes through `resolveTargetRect` and gets the right thing
|
|
19
|
+
* for both AI and traditional UUIDs.
|
|
20
|
+
*/
|
|
21
|
+
import type { ContentResult } from 'fastmcp';
|
|
22
|
+
import type { Rect } from '@appium/types';
|
|
23
|
+
import { getElementRect } from '../../../command.js';
|
|
24
|
+
import type { DriverInstance } from '../../../session-store.js';
|
|
25
|
+
import { errorResult } from '../../tool-response.js';
|
|
26
|
+
|
|
27
|
+
export const AI_ELEMENT_PREFIX = 'ai-element:';
|
|
28
|
+
|
|
29
|
+
export const AI_DISABLED_REJECTION =
|
|
30
|
+
`Received an ai-element: UUID, but the appium_ai tool is not registered ` +
|
|
31
|
+
`(AI_VISION_ENABLED is not set to true). Use appium_find_element to get a real ` +
|
|
32
|
+
`element UUID, or enable AI_VISION_ENABLED=true with the required AI_VISION_* keys.`;
|
|
33
|
+
|
|
34
|
+
export type ParsedAiElement = {
|
|
35
|
+
center: { x: number; y: number };
|
|
36
|
+
rect: Rect;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* When centre-only AI UUIDs have no bbox, direction-based swipe/scroll uses
|
|
41
|
+
* `rect.width` / `rect.height` with 0.2 / 0.8 edges; an effective 1x1 rect
|
|
42
|
+
* makes start/end identical after rounding. Keep a minimum inset area so those
|
|
43
|
+
* gestures still move visibly while staying centred on the parsed point.
|
|
44
|
+
*
|
|
45
|
+
* Values are screenshot pixels; callers can clamp against the window if needed.
|
|
46
|
+
*/
|
|
47
|
+
const AI_FALLBACK_RECT_WIDTH = 100;
|
|
48
|
+
const AI_FALLBACK_RECT_HEIGHT = 100;
|
|
49
|
+
|
|
50
|
+
export function isAiElementUUID(uuid: string | undefined): uuid is string {
|
|
51
|
+
return typeof uuid === 'string' && uuid.startsWith(AI_ELEMENT_PREFIX);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Parse an `ai-element:` UUID into a centre point and a synthetic rect.
|
|
56
|
+
*
|
|
57
|
+
* If the bbox segment is present and well-formed, the rect describes the
|
|
58
|
+
* full bounding box. Otherwise we fall back to a small rect centred on
|
|
59
|
+
* `(cx, cy)`, so directional swipe/scroll still has usable start/end deltas.
|
|
60
|
+
*/
|
|
61
|
+
export function parseAiElement(
|
|
62
|
+
uuid: string
|
|
63
|
+
): ParsedAiElement | { error: string } {
|
|
64
|
+
const parts = uuid.split(':');
|
|
65
|
+
if (parts.length < 2) {
|
|
66
|
+
return { error: 'Invalid ai-element UUID: missing coordinate segment.' };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const [cxStr, cyStr] = (parts[1] ?? '').split(',');
|
|
70
|
+
const cx = Number.parseInt(cxStr ?? '', 10);
|
|
71
|
+
const cy = Number.parseInt(cyStr ?? '', 10);
|
|
72
|
+
if (!Number.isFinite(cx) || !Number.isFinite(cy)) {
|
|
73
|
+
return {
|
|
74
|
+
error: `Invalid ai-element UUID: centre coordinates are not numbers ('${parts[1]}').`,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
let rect = rectAroundCenter(cx, cy);
|
|
79
|
+
if (parts[2]) {
|
|
80
|
+
const bbox = parts[2].split(',').map((v) => Number.parseInt(v, 10));
|
|
81
|
+
if (
|
|
82
|
+
bbox.length === 4 &&
|
|
83
|
+
bbox.every((n) => Number.isFinite(n)) &&
|
|
84
|
+
bbox[2] > bbox[0] &&
|
|
85
|
+
bbox[3] > bbox[1]
|
|
86
|
+
) {
|
|
87
|
+
rect = {
|
|
88
|
+
x: bbox[0],
|
|
89
|
+
y: bbox[1],
|
|
90
|
+
width: bbox[2] - bbox[0],
|
|
91
|
+
height: bbox[3] - bbox[1],
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return { center: { x: cx, y: cy }, rect };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Single dispatcher for "give me a rect for this UUID":
|
|
101
|
+
* - ai-element UUID → rect synthesised from the bbox / centre fallback, no driver call
|
|
102
|
+
* - traditional UUID → `getElementRect` on the driver
|
|
103
|
+
*
|
|
104
|
+
* Returns `{ error }` for malformed AI UUIDs or for driver lookup failures (`getElementRect`
|
|
105
|
+
* rejects). Call sites avoid try/catch and handle both cases the same way.
|
|
106
|
+
*/
|
|
107
|
+
export async function resolveTargetRect(
|
|
108
|
+
driver: DriverInstance,
|
|
109
|
+
elementUUID: string
|
|
110
|
+
): Promise<Rect | { error: string }> {
|
|
111
|
+
if (isAiElementUUID(elementUUID)) {
|
|
112
|
+
const parsed = parseAiElement(elementUUID);
|
|
113
|
+
if ('error' in parsed) {
|
|
114
|
+
return parsed;
|
|
115
|
+
}
|
|
116
|
+
return parsed.rect;
|
|
117
|
+
}
|
|
118
|
+
try {
|
|
119
|
+
return await getElementRect(driver, elementUUID);
|
|
120
|
+
} catch (err: unknown) {
|
|
121
|
+
return {
|
|
122
|
+
error: err instanceof Error ? err.message : String(err),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function aiDisabledResult(): ContentResult {
|
|
128
|
+
return errorResult(AI_DISABLED_REJECTION);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function rectAroundCenter(cx: number, cy: number): Rect {
|
|
132
|
+
const w = AI_FALLBACK_RECT_WIDTH;
|
|
133
|
+
const h = AI_FALLBACK_RECT_HEIGHT;
|
|
134
|
+
return {
|
|
135
|
+
x: cx - Math.floor(w / 2),
|
|
136
|
+
y: cy - Math.floor(h / 2),
|
|
137
|
+
width: w,
|
|
138
|
+
height: h,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import type { ContentResult } from 'fastmcp';
|
|
2
2
|
import type { DriverInstance } from '../../../session-store.js';
|
|
3
3
|
import { getPlatformName, PLATFORM } from '../../../session-store.js';
|
|
4
|
-
import {
|
|
5
|
-
execute,
|
|
6
|
-
getElementRect,
|
|
7
|
-
getWindowRect,
|
|
8
|
-
performActions,
|
|
9
|
-
} from '../../../command.js';
|
|
4
|
+
import { execute, getWindowRect, performActions } from '../../../command.js';
|
|
10
5
|
import {
|
|
11
6
|
errorResult,
|
|
12
7
|
textResult,
|
|
13
8
|
toolErrorMessage,
|
|
14
9
|
} from '../../tool-response.js';
|
|
10
|
+
import { isAIEnabled } from '../../ai/config.js';
|
|
11
|
+
import {
|
|
12
|
+
aiDisabledResult,
|
|
13
|
+
isAiElementUUID,
|
|
14
|
+
resolveTargetRect,
|
|
15
|
+
} from './ai-element.js';
|
|
15
16
|
import type { GestureArgs } from '../schema.js';
|
|
16
17
|
|
|
17
18
|
const DEFAULT_VELOCITY = 2.2;
|
|
@@ -96,6 +97,14 @@ export async function handlePinchZoom(
|
|
|
96
97
|
}
|
|
97
98
|
const useCustomCoords =
|
|
98
99
|
!args.elementUUID && args.x !== undefined && args.y !== undefined;
|
|
100
|
+
// ai-element UUIDs are coordinate UUIDs, not real element ids: drive the
|
|
101
|
+
// gesture from the bbox and skip the iOS/Android native paths that need
|
|
102
|
+
// a real `elementId`.
|
|
103
|
+
const isAiUUID = isAiElementUUID(args.elementUUID);
|
|
104
|
+
if (isAiUUID && !isAIEnabled()) {
|
|
105
|
+
return aiDisabledResult();
|
|
106
|
+
}
|
|
107
|
+
const useCoordPath = useCustomCoords || isAiUUID;
|
|
99
108
|
|
|
100
109
|
try {
|
|
101
110
|
const platform = getPlatformName(driver);
|
|
@@ -107,7 +116,10 @@ export async function handlePinchZoom(
|
|
|
107
116
|
|
|
108
117
|
if (args.elementUUID) {
|
|
109
118
|
windowRect = await getWindowRect(driver);
|
|
110
|
-
const rect = await
|
|
119
|
+
const rect = await resolveTargetRect(driver, args.elementUUID);
|
|
120
|
+
if ('error' in rect) {
|
|
121
|
+
return errorResult(rect.error);
|
|
122
|
+
}
|
|
111
123
|
({ cx, cy, spread } = resolveElementPinchTarget(rect, windowRect));
|
|
112
124
|
} else if (useCustomCoords) {
|
|
113
125
|
windowRect = await getWindowRect(driver);
|
|
@@ -151,10 +163,11 @@ export async function handlePinchZoom(
|
|
|
151
163
|
],
|
|
152
164
|
},
|
|
153
165
|
]);
|
|
154
|
-
} else if (
|
|
155
|
-
// Zoom in at a
|
|
156
|
-
// with a region centered at (cx, cy). spread is
|
|
157
|
-
// fits within the window.
|
|
166
|
+
} else if (useCoordPath && platform === PLATFORM.android) {
|
|
167
|
+
// Zoom in at a coordinate-driven center on Android: use the native
|
|
168
|
+
// pinchOpenGesture with a region centered at (cx, cy). spread is
|
|
169
|
+
// pre-clamped so the region fits within the window. Also covers
|
|
170
|
+
// ai-element UUIDs, whose synthetic rect is used to pick the region.
|
|
158
171
|
const percent = Math.min(0.99, 1 - 1 / scale);
|
|
159
172
|
await execute(driver, 'mobile: pinchOpenGesture', {
|
|
160
173
|
left: cx - spread,
|
|
@@ -163,8 +176,10 @@ export async function handlePinchZoom(
|
|
|
163
176
|
height: 2 * spread,
|
|
164
177
|
percent,
|
|
165
178
|
});
|
|
166
|
-
} else if (
|
|
167
|
-
// Zoom in at a
|
|
179
|
+
} else if (useCoordPath) {
|
|
180
|
+
// Zoom in at a coordinate-driven center on iOS using W3C Actions.
|
|
181
|
+
// Same path is used for ai-element UUIDs because `mobile: pinch`
|
|
182
|
+
// requires a real elementId we don't have.
|
|
168
183
|
const startSpread = Math.max(1, Math.floor(spread / scale));
|
|
169
184
|
const endSpread = spread;
|
|
170
185
|
const duration = Math.floor((1 / Math.abs(velocity)) * 1000);
|
|
@@ -224,11 +239,13 @@ export async function handlePinchZoom(
|
|
|
224
239
|
}
|
|
225
240
|
|
|
226
241
|
const direction = scale < 1 ? 'out' : 'in';
|
|
227
|
-
const target =
|
|
228
|
-
? `element ${
|
|
229
|
-
:
|
|
230
|
-
? `
|
|
231
|
-
:
|
|
242
|
+
const target = isAiUUID
|
|
243
|
+
? `AI element coordinates (${cx}, ${cy})`
|
|
244
|
+
: args.elementUUID
|
|
245
|
+
? `element ${args.elementUUID}`
|
|
246
|
+
: useCustomCoords
|
|
247
|
+
? `coordinates (${cx}, ${cy})`
|
|
248
|
+
: 'screen';
|
|
232
249
|
return textResult(
|
|
233
250
|
`Successfully pinched ${direction} (scale=${scale}) on ${target}.`
|
|
234
251
|
);
|