@midscene/harmony 1.8.0 → 1.8.1

package/dist/es/index.mjs

@@ -1,7 +1,7 @@
 import node_assert from "node:assert";
 import node_fs, { accessSync, constants } from "node:fs";
-import { getMidsceneLocationSchema, z } from "@midscene/core";
-import { defineAction, defineActionClearInput, defineActionCursorMove, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionLongPress, defineActionScroll, defineActionSwipe, defineActionTap, normalizeMobileSwipeParam } from "@midscene/core/device";
+import { z } from "@midscene/core";
+import { createDefaultMobileActions, defineAction } from "@midscene/core/device";
 import { getTmpFile, sleep } from "@midscene/core/utils";
 import { createImgBase64ByFormat } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
@@ -221,15 +221,6 @@ function device_define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-const harmonyInputParamSchema = z.object({
-    value: z.string().describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
-    mode: z.preprocess((val)=>'append' === val ? 'typeOnly' : val, z["enum"]([
-        'replace',
-        'clear',
-        'typeOnly'
-    ]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "typeOnly" - type the value directly without clearing the field first; "clear" - attempt to clear the field (limited support on HarmonyOS).')),
-    locate: getMidsceneLocationSchema().describe('The input field to be filled').optional()
-});
 const defaultScrollUntilTimes = 10;
 const defaultFastSwipeSpeed = 2000;
 const maxScrollDistance = 9999999;
@@ -270,89 +261,15 @@ const keyNameAliasMap = {
 };
 class HarmonyDevice {
     actionSpace() {
+        const mobileActionContext = {
+            input: this.inputPrimitives,
+            size: ()=>this.size(),
+            sleep: async (timeMs)=>{
+                await sleep(timeMs);
+            }
+        };
         const defaultActions = [
-            defineActionTap(async (param)=>{
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot tap');
-                await this.tap(element.center[0], element.center[1]);
-            }),
-            defineActionDoubleClick(async (param)=>{
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot double click');
-                await this.doubleTap(element.center[0], element.center[1]);
-            }),
-            defineAction({
-                name: 'Input',
-                description: 'Input text into the input field',
-                interfaceAlias: 'aiInput',
-                paramSchema: harmonyInputParamSchema,
-                sample: {
-                    value: 'test@example.com',
-                    locate: {
-                        prompt: 'the email input field'
-                    }
-                },
-                call: async (param)=>{
-                    const element = param.locate;
-                    if ('clear' === param.mode) return void await this.clearInput(element);
-                    if (!param || !param.value) return;
-                    const shouldReplace = 'typeOnly' !== param.mode;
-                    await this.inputText(param.value, element, shouldReplace);
-                }
-            }),
-            defineActionScroll(async (param)=>{
-                const element = param.locate;
-                const startingPoint = element ? {
-                    left: element.center[0],
-                    top: element.center[1]
-                } : void 0;
-                const scrollToEventName = param?.scrollType;
-                if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
-                else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
-                else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
-                else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
-                else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
-                else {
-                    if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
-                    else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
-                    else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
-                    else throw new Error(`Unknown scroll direction: ${param.direction}`);
-                    else await this.scrollDown(param?.distance ?? void 0, startingPoint);
-                    await sleep(500);
-                }
-            }),
-            defineActionDragAndDrop(async (param)=>{
-                const from = param.from;
-                const to = param.to;
-                node_assert(from, 'missing "from" param for drag and drop');
-                node_assert(to, 'missing "to" param for drag and drop');
-                const hdc = await this.getHdc();
-                await hdc.drag(from.center[0], from.center[1], to.center[0], to.center[1]);
-            }),
-            defineActionSwipe(async (param)=>{
-                const { startPoint, endPoint, duration, repeatCount } = normalizeMobileSwipeParam(param, await this.size());
-                const hdc = await this.getHdc();
-                for(let i = 0; i < repeatCount; i++)await hdc.swipe(startPoint.x, startPoint.y, endPoint.x, endPoint.y, duration ? Math.round(duration) : void 0);
-            }),
-            defineActionKeyboardPress(async (param)=>{
-                await this.keyboardPress(param.keyName);
-            }),
-            defineActionCursorMove(async (param)=>{
-                const arrowKey = 'left' === param.direction ? 'ArrowLeft' : 'ArrowRight';
-                const times = param.times ?? 1;
-                for(let i = 0; i < times; i++){
-                    await this.keyboardPress(arrowKey);
-                    await sleep(100);
-                }
-            }),
-            defineActionLongPress(async (param)=>{
-                const element = param.locate;
-                if (!element) throw new Error('LongPress requires an element to be located');
-                await this.longPress(element.center[0], element.center[1]);
-            }),
-            defineActionClearInput(async (param)=>{
-                await this.clearInput(param.locate);
-            })
+            ...createDefaultMobileActions(mobileActionContext)
         ];
         const platformSpecificActions = Object.values(createPlatformActions(this));
         const customActions = this.customActions ?? [];
@@ -362,6 +279,27 @@ class HarmonyDevice {
             ...customActions
         ];
     }
+    async performActionScroll(param) {
+        const element = param.locate;
+        const startingPoint = element ? {
+            left: element.center[0],
+            top: element.center[1]
+        } : void 0;
+        const scrollToEventName = param?.scrollType;
+        if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
+        else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
+        else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
+        else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
+        else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
+        else {
+            if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
+            else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
+            else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
+            else throw new Error(`Unknown scroll direction: ${param.direction}`);
+            else await this.scrollDown(param?.distance ?? void 0, startingPoint);
+            await sleep(500);
+        }
+    }
     describe() {
         return this.descriptionText || `DeviceId: ${this.deviceId}`;
     }
@@ -492,23 +430,23 @@ class HarmonyDevice {
         }
         throw new Error('Screenshot buffer is empty after retries');
     }
-    async tap(x, y) {
+    async tapPoint(point) {
         this.lastTapPosition = {
-            x,
-            y
+            x: point.x,
+            y: point.y
         };
         const hdc = await this.getHdc();
-        await hdc.click(x, y);
+        await hdc.click(point.x, point.y);
     }
-    async doubleTap(x, y) {
+    async doubleTapPoint(point) {
         const hdc = await this.getHdc();
-        await hdc.doubleClick(x, y);
+        await hdc.doubleClick(point.x, point.y);
     }
-    async longPress(x, y) {
+    async longPressPoint(point) {
         const hdc = await this.getHdc();
-        await hdc.longClick(x, y);
+        await hdc.longClick(point.x, point.y);
     }
-    async inputText(text, element, shouldReplace) {
+    async typeText(text, element, shouldReplace) {
         if (!text) return;
         const hdc = await this.getHdc();
         let x;
@@ -539,7 +477,7 @@ class HarmonyDevice {
         }
         await hdc.clearTextField(100);
     }
-    async keyboardPress(key) {
+    async pressKey(key) {
         const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
         const harmonyKey = harmonyKeyCodeMap[normalizedKey] ?? key;
         const hdc = await this.getHdc();
@@ -739,6 +677,37 @@ class HarmonyDevice {
         device_define_property(this, "interfaceType", 'harmony');
         device_define_property(this, "uri", void 0);
         device_define_property(this, "options", void 0);
+        device_define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: (point)=>this.tapPoint(point),
+                doubleClick: (point)=>this.doubleTapPoint(point),
+                longPress: (point)=>this.longPressPoint(point),
+                dragAndDrop: async (from, to)=>{
+                    const hdc = await this.getHdc();
+                    await hdc.drag(from.x, from.y, to.x, to.y);
+                }
+            },
+            keyboard: {
+                keyboardPress: (keyName)=>this.pressKey(keyName),
+                typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true),
+                clearInput: (target)=>this.clearInput(target),
+                cursorMove: async (direction, times = 1)=>{
+                    const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
+                    for(let i = 0; i < times; i++)await this.pressKey(arrowKey);
+                }
+            },
+            touch: {
+                swipe: async (start, end, opts)=>{
+                    const duration = opts?.duration;
+                    const repeatCount = opts?.repeat ?? 1;
+                    const hdc = await this.getHdc();
+                    for(let i = 0; i < repeatCount; i++)await hdc.swipe(start.x, start.y, end.x, end.y, duration ? Math.round(duration) : void 0);
+                }
+            },
+            scroll: {
+                scroll: (param)=>this.performActionScroll(param)
+            }
+        });
         device_define_property(this, "remoteScreenshotPath", '/data/local/tmp/ms_screen.jpeg');
         device_define_property(this, "localScreenshotPath", null);
         node_assert(deviceId, 'deviceId is required for HarmonyDevice');

package/dist/es/mcp-server.mjs

@@ -1,12 +1,12 @@
 import { BaseMCPServer, createMCPServerLauncher } from "@midscene/shared/mcp";
-import { getMidsceneLocationSchema, z } from "@midscene/core";
+import { z } from "@midscene/core";
 import { getDebug } from "@midscene/shared/logger";
 import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
 import { Agent } from "@midscene/core/agent";
 import { mergeAndNormalizeAppNameMapping, normalizeForComparison, repeat } from "@midscene/shared/utils";
 import node_assert from "node:assert";
 import node_fs, { accessSync, constants } from "node:fs";
-import { defineAction, defineActionClearInput, defineActionCursorMove, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionLongPress, defineActionScroll, defineActionSwipe, defineActionTap, normalizeMobileSwipeParam } from "@midscene/core/device";
+import { createDefaultMobileActions, defineAction } from "@midscene/core/device";
 import { getTmpFile, sleep } from "@midscene/core/utils";
 import { createImgBase64ByFormat } from "@midscene/shared/img";
 import { execFile } from "node:child_process";
@@ -250,15 +250,6 @@ function device_define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-const harmonyInputParamSchema = z.object({
-    value: z.string().describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
-    mode: z.preprocess((val)=>'append' === val ? 'typeOnly' : val, z["enum"]([
-        'replace',
-        'clear',
-        'typeOnly'
-    ]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "typeOnly" - type the value directly without clearing the field first; "clear" - attempt to clear the field (limited support on HarmonyOS).')),
-    locate: getMidsceneLocationSchema().describe('The input field to be filled').optional()
-});
 const defaultScrollUntilTimes = 10;
 const defaultFastSwipeSpeed = 2000;
 const maxScrollDistance = 9999999;
@@ -299,89 +290,15 @@ const keyNameAliasMap = {
 };
 class HarmonyDevice {
     actionSpace() {
+        const mobileActionContext = {
+            input: this.inputPrimitives,
+            size: ()=>this.size(),
+            sleep: async (timeMs)=>{
+                await sleep(timeMs);
+            }
+        };
         const defaultActions = [
-            defineActionTap(async (param)=>{
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot tap');
-                await this.tap(element.center[0], element.center[1]);
-            }),
-            defineActionDoubleClick(async (param)=>{
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot double click');
-                await this.doubleTap(element.center[0], element.center[1]);
-            }),
-            defineAction({
-                name: 'Input',
-                description: 'Input text into the input field',
-                interfaceAlias: 'aiInput',
-                paramSchema: harmonyInputParamSchema,
-                sample: {
-                    value: 'test@example.com',
-                    locate: {
-                        prompt: 'the email input field'
-                    }
-                },
-                call: async (param)=>{
-                    const element = param.locate;
-                    if ('clear' === param.mode) return void await this.clearInput(element);
-                    if (!param || !param.value) return;
-                    const shouldReplace = 'typeOnly' !== param.mode;
-                    await this.inputText(param.value, element, shouldReplace);
-                }
-            }),
-            defineActionScroll(async (param)=>{
-                const element = param.locate;
-                const startingPoint = element ? {
-                    left: element.center[0],
-                    top: element.center[1]
-                } : void 0;
-                const scrollToEventName = param?.scrollType;
-                if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
-                else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
-                else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
-                else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
-                else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
-                else {
-                    if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
-                    else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
-                    else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
-                    else throw new Error(`Unknown scroll direction: ${param.direction}`);
-                    else await this.scrollDown(param?.distance ?? void 0, startingPoint);
-                    await sleep(500);
-                }
-            }),
-            defineActionDragAndDrop(async (param)=>{
-                const from = param.from;
-                const to = param.to;
-                node_assert(from, 'missing "from" param for drag and drop');
-                node_assert(to, 'missing "to" param for drag and drop');
-                const hdc = await this.getHdc();
-                await hdc.drag(from.center[0], from.center[1], to.center[0], to.center[1]);
-            }),
-            defineActionSwipe(async (param)=>{
-                const { startPoint, endPoint, duration, repeatCount } = normalizeMobileSwipeParam(param, await this.size());
-                const hdc = await this.getHdc();
-                for(let i = 0; i < repeatCount; i++)await hdc.swipe(startPoint.x, startPoint.y, endPoint.x, endPoint.y, duration ? Math.round(duration) : void 0);
-            }),
-            defineActionKeyboardPress(async (param)=>{
-                await this.keyboardPress(param.keyName);
-            }),
-            defineActionCursorMove(async (param)=>{
-                const arrowKey = 'left' === param.direction ? 'ArrowLeft' : 'ArrowRight';
-                const times = param.times ?? 1;
-                for(let i = 0; i < times; i++){
-                    await this.keyboardPress(arrowKey);
-                    await sleep(100);
-                }
-            }),
-            defineActionLongPress(async (param)=>{
-                const element = param.locate;
-                if (!element) throw new Error('LongPress requires an element to be located');
-                await this.longPress(element.center[0], element.center[1]);
-            }),
-            defineActionClearInput(async (param)=>{
-                await this.clearInput(param.locate);
-            })
+            ...createDefaultMobileActions(mobileActionContext)
         ];
         const platformSpecificActions = Object.values(createPlatformActions(this));
         const customActions = this.customActions ?? [];
@@ -391,6 +308,27 @@ class HarmonyDevice {
             ...customActions
         ];
     }
+    async performActionScroll(param) {
+        const element = param.locate;
+        const startingPoint = element ? {
+            left: element.center[0],
+            top: element.center[1]
+        } : void 0;
+        const scrollToEventName = param?.scrollType;
+        if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
+        else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
+        else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
+        else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
+        else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
+        else {
+            if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
+            else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
+            else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
+            else throw new Error(`Unknown scroll direction: ${param.direction}`);
+            else await this.scrollDown(param?.distance ?? void 0, startingPoint);
+            await sleep(500);
+        }
+    }
     describe() {
         return this.descriptionText || `DeviceId: ${this.deviceId}`;
     }
@@ -521,23 +459,23 @@ class HarmonyDevice {
         }
         throw new Error('Screenshot buffer is empty after retries');
     }
-    async tap(x, y) {
+    async tapPoint(point) {
         this.lastTapPosition = {
-            x,
-            y
+            x: point.x,
+            y: point.y
         };
         const hdc = await this.getHdc();
-        await hdc.click(x, y);
+        await hdc.click(point.x, point.y);
     }
-    async doubleTap(x, y) {
+    async doubleTapPoint(point) {
         const hdc = await this.getHdc();
-        await hdc.doubleClick(x, y);
+        await hdc.doubleClick(point.x, point.y);
     }
-    async longPress(x, y) {
+    async longPressPoint(point) {
         const hdc = await this.getHdc();
-        await hdc.longClick(x, y);
+        await hdc.longClick(point.x, point.y);
     }
-    async inputText(text, element, shouldReplace) {
+    async typeText(text, element, shouldReplace) {
         if (!text) return;
         const hdc = await this.getHdc();
         let x;
@@ -568,7 +506,7 @@ class HarmonyDevice {
         }
         await hdc.clearTextField(100);
     }
-    async keyboardPress(key) {
+    async pressKey(key) {
         const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
         const harmonyKey = harmonyKeyCodeMap[normalizedKey] ?? key;
         const hdc = await this.getHdc();
@@ -768,6 +706,37 @@ class HarmonyDevice {
         device_define_property(this, "interfaceType", 'harmony');
         device_define_property(this, "uri", void 0);
         device_define_property(this, "options", void 0);
+        device_define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: (point)=>this.tapPoint(point),
+                doubleClick: (point)=>this.doubleTapPoint(point),
+                longPress: (point)=>this.longPressPoint(point),
+                dragAndDrop: async (from, to)=>{
+                    const hdc = await this.getHdc();
+                    await hdc.drag(from.x, from.y, to.x, to.y);
+                }
+            },
+            keyboard: {
+                keyboardPress: (keyName)=>this.pressKey(keyName),
+                typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true),
+                clearInput: (target)=>this.clearInput(target),
+                cursorMove: async (direction, times = 1)=>{
+                    const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
+                    for(let i = 0; i < times; i++)await this.pressKey(arrowKey);
+                }
+            },
+            touch: {
+                swipe: async (start, end, opts)=>{
+                    const duration = opts?.duration;
+                    const repeatCount = opts?.repeat ?? 1;
+                    const hdc = await this.getHdc();
+                    for(let i = 0; i < repeatCount; i++)await hdc.swipe(start.x, start.y, end.x, end.y, duration ? Math.round(duration) : void 0);
+                }
+            },
+            scroll: {
+                scroll: (param)=>this.performActionScroll(param)
+            }
+        });
         device_define_property(this, "remoteScreenshotPath", '/data/local/tmp/ms_screen.jpeg');
         device_define_property(this, "localScreenshotPath", null);
         node_assert(deviceId, 'deviceId is required for HarmonyDevice');
@@ -1026,7 +995,7 @@ class HarmonyMCPServer extends BaseMCPServer {
     constructor(toolsManager){
         super({
             name: '@midscene/harmony-mcp',
-            version: "1.8.0",
+            version: "1.8.1",
             description: 'Control the HarmonyOS device using natural language commands'
         }, toolsManager);
     }