@midscene/harmony 1.8.0 → 1.8.1

package/dist/lib/bin.js

@@ -279,15 +279,6 @@ function device_define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-const harmonyInputParamSchema = core_namespaceObject.z.object({
-    value: core_namespaceObject.z.string().describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
-    mode: core_namespaceObject.z.preprocess((val)=>'append' === val ? 'typeOnly' : val, core_namespaceObject.z["enum"]([
-        'replace',
-        'clear',
-        'typeOnly'
-    ]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "typeOnly" - type the value directly without clearing the field first; "clear" - attempt to clear the field (limited support on HarmonyOS).')),
-    locate: (0, core_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be filled').optional()
-});
 const defaultScrollUntilTimes = 10;
 const defaultFastSwipeSpeed = 2000;
 const maxScrollDistance = 9999999;
@@ -328,89 +319,15 @@ const keyNameAliasMap = {
 };
 class device_HarmonyDevice {
     actionSpace() {
+        const mobileActionContext = {
+            input: this.inputPrimitives,
+            size: ()=>this.size(),
+            sleep: async (timeMs)=>{
+                await (0, core_utils_namespaceObject.sleep)(timeMs);
+            }
+        };
         const defaultActions = [
-            (0, device_namespaceObject.defineActionTap)(async (param)=>{
-                const element = param.locate;
-                external_node_assert_default()(element, 'Element not found, cannot tap');
-                await this.tap(element.center[0], element.center[1]);
-            }),
-            (0, device_namespaceObject.defineActionDoubleClick)(async (param)=>{
-                const element = param.locate;
-                external_node_assert_default()(element, 'Element not found, cannot double click');
-                await this.doubleTap(element.center[0], element.center[1]);
-            }),
-            (0, device_namespaceObject.defineAction)({
-                name: 'Input',
-                description: 'Input text into the input field',
-                interfaceAlias: 'aiInput',
-                paramSchema: harmonyInputParamSchema,
-                sample: {
-                    value: 'test@example.com',
-                    locate: {
-                        prompt: 'the email input field'
-                    }
-                },
-                call: async (param)=>{
-                    const element = param.locate;
-                    if ('clear' === param.mode) return void await this.clearInput(element);
-                    if (!param || !param.value) return;
-                    const shouldReplace = 'typeOnly' !== param.mode;
-                    await this.inputText(param.value, element, shouldReplace);
-                }
-            }),
-            (0, device_namespaceObject.defineActionScroll)(async (param)=>{
-                const element = param.locate;
-                const startingPoint = element ? {
-                    left: element.center[0],
-                    top: element.center[1]
-                } : void 0;
-                const scrollToEventName = param?.scrollType;
-                if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
-                else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
-                else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
-                else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
-                else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
-                else {
-                    if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
-                    else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
-                    else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
-                    else throw new Error(`Unknown scroll direction: ${param.direction}`);
-                    else await this.scrollDown(param?.distance ?? void 0, startingPoint);
-                    await (0, core_utils_namespaceObject.sleep)(500);
-                }
-            }),
-            (0, device_namespaceObject.defineActionDragAndDrop)(async (param)=>{
-                const from = param.from;
-                const to = param.to;
-                external_node_assert_default()(from, 'missing "from" param for drag and drop');
-                external_node_assert_default()(to, 'missing "to" param for drag and drop');
-                const hdc = await this.getHdc();
-                await hdc.drag(from.center[0], from.center[1], to.center[0], to.center[1]);
-            }),
-            (0, device_namespaceObject.defineActionSwipe)(async (param)=>{
-                const { startPoint, endPoint, duration, repeatCount } = (0, device_namespaceObject.normalizeMobileSwipeParam)(param, await this.size());
-                const hdc = await this.getHdc();
-                for(let i = 0; i < repeatCount; i++)await hdc.swipe(startPoint.x, startPoint.y, endPoint.x, endPoint.y, duration ? Math.round(duration) : void 0);
-            }),
-            (0, device_namespaceObject.defineActionKeyboardPress)(async (param)=>{
-                await this.keyboardPress(param.keyName);
-            }),
-            (0, device_namespaceObject.defineActionCursorMove)(async (param)=>{
-                const arrowKey = 'left' === param.direction ? 'ArrowLeft' : 'ArrowRight';
-                const times = param.times ?? 1;
-                for(let i = 0; i < times; i++){
-                    await this.keyboardPress(arrowKey);
-                    await (0, core_utils_namespaceObject.sleep)(100);
-                }
-            }),
-            (0, device_namespaceObject.defineActionLongPress)(async (param)=>{
-                const element = param.locate;
-                if (!element) throw new Error('LongPress requires an element to be located');
-                await this.longPress(element.center[0], element.center[1]);
-            }),
-            (0, device_namespaceObject.defineActionClearInput)(async (param)=>{
-                await this.clearInput(param.locate);
-            })
+            ...(0, device_namespaceObject.createDefaultMobileActions)(mobileActionContext)
         ];
         const platformSpecificActions = Object.values(createPlatformActions(this));
         const customActions = this.customActions ?? [];
@@ -420,6 +337,27 @@ class device_HarmonyDevice {
             ...customActions
         ];
     }
+    async performActionScroll(param) {
+        const element = param.locate;
+        const startingPoint = element ? {
+            left: element.center[0],
+            top: element.center[1]
+        } : void 0;
+        const scrollToEventName = param?.scrollType;
+        if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
+        else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
+        else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
+        else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
+        else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
+        else {
+            if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
+            else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
+            else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
+            else throw new Error(`Unknown scroll direction: ${param.direction}`);
+            else await this.scrollDown(param?.distance ?? void 0, startingPoint);
+            await (0, core_utils_namespaceObject.sleep)(500);
+        }
+    }
     describe() {
         return this.descriptionText || `DeviceId: ${this.deviceId}`;
     }
@@ -550,23 +488,23 @@ class device_HarmonyDevice {
         }
         throw new Error('Screenshot buffer is empty after retries');
     }
-    async tap(x, y) {
+    async tapPoint(point) {
         this.lastTapPosition = {
-            x,
-            y
+            x: point.x,
+            y: point.y
         };
         const hdc = await this.getHdc();
-        await hdc.click(x, y);
+        await hdc.click(point.x, point.y);
     }
-    async doubleTap(x, y) {
+    async doubleTapPoint(point) {
         const hdc = await this.getHdc();
-        await hdc.doubleClick(x, y);
+        await hdc.doubleClick(point.x, point.y);
     }
-    async longPress(x, y) {
+    async longPressPoint(point) {
         const hdc = await this.getHdc();
-        await hdc.longClick(x, y);
+        await hdc.longClick(point.x, point.y);
     }
-    async inputText(text, element, shouldReplace) {
+    async typeText(text, element, shouldReplace) {
         if (!text) return;
         const hdc = await this.getHdc();
         let x;
@@ -597,7 +535,7 @@ class device_HarmonyDevice {
         }
         await hdc.clearTextField(100);
     }
-    async keyboardPress(key) {
+    async pressKey(key) {
         const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
         const harmonyKey = harmonyKeyCodeMap[normalizedKey] ?? key;
         const hdc = await this.getHdc();
@@ -797,6 +735,37 @@ class device_HarmonyDevice {
         device_define_property(this, "interfaceType", 'harmony');
         device_define_property(this, "uri", void 0);
         device_define_property(this, "options", void 0);
+        device_define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: (point)=>this.tapPoint(point),
+                doubleClick: (point)=>this.doubleTapPoint(point),
+                longPress: (point)=>this.longPressPoint(point),
+                dragAndDrop: async (from, to)=>{
+                    const hdc = await this.getHdc();
+                    await hdc.drag(from.x, from.y, to.x, to.y);
+                }
+            },
+            keyboard: {
+                keyboardPress: (keyName)=>this.pressKey(keyName),
+                typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true),
+                clearInput: (target)=>this.clearInput(target),
+                cursorMove: async (direction, times = 1)=>{
+                    const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
+                    for(let i = 0; i < times; i++)await this.pressKey(arrowKey);
+                }
+            },
+            touch: {
+                swipe: async (start, end, opts)=>{
+                    const duration = opts?.duration;
+                    const repeatCount = opts?.repeat ?? 1;
+                    const hdc = await this.getHdc();
+                    for(let i = 0; i < repeatCount; i++)await hdc.swipe(start.x, start.y, end.x, end.y, duration ? Math.round(duration) : void 0);
+                }
+            },
+            scroll: {
+                scroll: (param)=>this.performActionScroll(param)
+            }
+        });
         device_define_property(this, "remoteScreenshotPath", '/data/local/tmp/ms_screen.jpeg');
         device_define_property(this, "localScreenshotPath", null);
         external_node_assert_default()(deviceId, 'deviceId is required for HarmonyDevice');

package/dist/lib/cli.js

@@ -275,15 +275,6 @@ function device_define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
-const harmonyInputParamSchema = core_namespaceObject.z.object({
-    value: core_namespaceObject.z.string().describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
-    mode: core_namespaceObject.z.preprocess((val)=>'append' === val ? 'typeOnly' : val, core_namespaceObject.z["enum"]([
-        'replace',
-        'clear',
-        'typeOnly'
-    ]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "typeOnly" - type the value directly without clearing the field first; "clear" - attempt to clear the field (limited support on HarmonyOS).')),
-    locate: (0, core_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be filled').optional()
-});
 const defaultScrollUntilTimes = 10;
 const defaultFastSwipeSpeed = 2000;
 const maxScrollDistance = 9999999;
@@ -324,89 +315,15 @@ const keyNameAliasMap = {
 };
 class HarmonyDevice {
     actionSpace() {
+        const mobileActionContext = {
+            input: this.inputPrimitives,
+            size: ()=>this.size(),
+            sleep: async (timeMs)=>{
+                await (0, core_utils_namespaceObject.sleep)(timeMs);
+            }
+        };
         const defaultActions = [
-            (0, device_namespaceObject.defineActionTap)(async (param)=>{
-                const element = param.locate;
-                external_node_assert_default()(element, 'Element not found, cannot tap');
-                await this.tap(element.center[0], element.center[1]);
-            }),
-            (0, device_namespaceObject.defineActionDoubleClick)(async (param)=>{
-                const element = param.locate;
-                external_node_assert_default()(element, 'Element not found, cannot double click');
-                await this.doubleTap(element.center[0], element.center[1]);
-            }),
-            (0, device_namespaceObject.defineAction)({
-                name: 'Input',
-                description: 'Input text into the input field',
-                interfaceAlias: 'aiInput',
-                paramSchema: harmonyInputParamSchema,
-                sample: {
-                    value: 'test@example.com',
-                    locate: {
-                        prompt: 'the email input field'
-                    }
-                },
-                call: async (param)=>{
-                    const element = param.locate;
-                    if ('clear' === param.mode) return void await this.clearInput(element);
-                    if (!param || !param.value) return;
-                    const shouldReplace = 'typeOnly' !== param.mode;
-                    await this.inputText(param.value, element, shouldReplace);
-                }
-            }),
-            (0, device_namespaceObject.defineActionScroll)(async (param)=>{
-                const element = param.locate;
-                const startingPoint = element ? {
-                    left: element.center[0],
-                    top: element.center[1]
-                } : void 0;
-                const scrollToEventName = param?.scrollType;
-                if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
-                else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
-                else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
-                else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
-                else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
-                else {
-                    if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
-                    else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
-                    else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
-                    else throw new Error(`Unknown scroll direction: ${param.direction}`);
-                    else await this.scrollDown(param?.distance ?? void 0, startingPoint);
-                    await (0, core_utils_namespaceObject.sleep)(500);
-                }
-            }),
-            (0, device_namespaceObject.defineActionDragAndDrop)(async (param)=>{
-                const from = param.from;
-                const to = param.to;
-                external_node_assert_default()(from, 'missing "from" param for drag and drop');
-                external_node_assert_default()(to, 'missing "to" param for drag and drop');
-                const hdc = await this.getHdc();
-                await hdc.drag(from.center[0], from.center[1], to.center[0], to.center[1]);
-            }),
-            (0, device_namespaceObject.defineActionSwipe)(async (param)=>{
-                const { startPoint, endPoint, duration, repeatCount } = (0, device_namespaceObject.normalizeMobileSwipeParam)(param, await this.size());
-                const hdc = await this.getHdc();
-                for(let i = 0; i < repeatCount; i++)await hdc.swipe(startPoint.x, startPoint.y, endPoint.x, endPoint.y, duration ? Math.round(duration) : void 0);
-            }),
-            (0, device_namespaceObject.defineActionKeyboardPress)(async (param)=>{
-                await this.keyboardPress(param.keyName);
-            }),
-            (0, device_namespaceObject.defineActionCursorMove)(async (param)=>{
-                const arrowKey = 'left' === param.direction ? 'ArrowLeft' : 'ArrowRight';
-                const times = param.times ?? 1;
-                for(let i = 0; i < times; i++){
-                    await this.keyboardPress(arrowKey);
-                    await (0, core_utils_namespaceObject.sleep)(100);
-                }
-            }),
-            (0, device_namespaceObject.defineActionLongPress)(async (param)=>{
-                const element = param.locate;
-                if (!element) throw new Error('LongPress requires an element to be located');
-                await this.longPress(element.center[0], element.center[1]);
-            }),
-            (0, device_namespaceObject.defineActionClearInput)(async (param)=>{
-                await this.clearInput(param.locate);
-            })
+            ...(0, device_namespaceObject.createDefaultMobileActions)(mobileActionContext)
         ];
         const platformSpecificActions = Object.values(createPlatformActions(this));
         const customActions = this.customActions ?? [];
@@ -416,6 +333,27 @@ class HarmonyDevice {
             ...customActions
         ];
     }
+    async performActionScroll(param) {
+        const element = param.locate;
+        const startingPoint = element ? {
+            left: element.center[0],
+            top: element.center[1]
+        } : void 0;
+        const scrollToEventName = param?.scrollType;
+        if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
+        else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
+        else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
+        else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
+        else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
+        else {
+            if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
+            else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
+            else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
+            else throw new Error(`Unknown scroll direction: ${param.direction}`);
+            else await this.scrollDown(param?.distance ?? void 0, startingPoint);
+            await (0, core_utils_namespaceObject.sleep)(500);
+        }
+    }
     describe() {
         return this.descriptionText || `DeviceId: ${this.deviceId}`;
     }
@@ -546,23 +484,23 @@ class HarmonyDevice {
         }
         throw new Error('Screenshot buffer is empty after retries');
     }
-    async tap(x, y) {
+    async tapPoint(point) {
         this.lastTapPosition = {
-            x,
-            y
+            x: point.x,
+            y: point.y
         };
         const hdc = await this.getHdc();
-        await hdc.click(x, y);
+        await hdc.click(point.x, point.y);
     }
-    async doubleTap(x, y) {
+    async doubleTapPoint(point) {
         const hdc = await this.getHdc();
-        await hdc.doubleClick(x, y);
+        await hdc.doubleClick(point.x, point.y);
     }
-    async longPress(x, y) {
+    async longPressPoint(point) {
         const hdc = await this.getHdc();
-        await hdc.longClick(x, y);
+        await hdc.longClick(point.x, point.y);
     }
-    async inputText(text, element, shouldReplace) {
+    async typeText(text, element, shouldReplace) {
         if (!text) return;
         const hdc = await this.getHdc();
         let x;
@@ -593,7 +531,7 @@ class HarmonyDevice {
         }
         await hdc.clearTextField(100);
     }
-    async keyboardPress(key) {
+    async pressKey(key) {
         const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
         const harmonyKey = harmonyKeyCodeMap[normalizedKey] ?? key;
         const hdc = await this.getHdc();
@@ -793,6 +731,37 @@ class HarmonyDevice {
         device_define_property(this, "interfaceType", 'harmony');
         device_define_property(this, "uri", void 0);
         device_define_property(this, "options", void 0);
+        device_define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: (point)=>this.tapPoint(point),
+                doubleClick: (point)=>this.doubleTapPoint(point),
+                longPress: (point)=>this.longPressPoint(point),
+                dragAndDrop: async (from, to)=>{
+                    const hdc = await this.getHdc();
+                    await hdc.drag(from.x, from.y, to.x, to.y);
+                }
+            },
+            keyboard: {
+                keyboardPress: (keyName)=>this.pressKey(keyName),
+                typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true),
+                clearInput: (target)=>this.clearInput(target),
+                cursorMove: async (direction, times = 1)=>{
+                    const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
+                    for(let i = 0; i < times; i++)await this.pressKey(arrowKey);
+                }
+            },
+            touch: {
+                swipe: async (start, end, opts)=>{
+                    const duration = opts?.duration;
+                    const repeatCount = opts?.repeat ?? 1;
+                    const hdc = await this.getHdc();
+                    for(let i = 0; i < repeatCount; i++)await hdc.swipe(start.x, start.y, end.x, end.y, duration ? Math.round(duration) : void 0);
+                }
+            },
+            scroll: {
+                scroll: (param)=>this.performActionScroll(param)
+            }
+        });
         device_define_property(this, "remoteScreenshotPath", '/data/local/tmp/ms_screen.jpeg');
         device_define_property(this, "localScreenshotPath", null);
         external_node_assert_default()(deviceId, 'deviceId is required for HarmonyDevice');
@@ -1047,7 +1016,7 @@ class HarmonyMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
 const tools = new HarmonyMidsceneTools();
 (0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-harmony', {
     stripPrefix: 'harmony_',
-    version: "1.8.0",
+    version: "1.8.1",
     extraCommands: (0, core_namespaceObject.createReportCliCommands)()
 }).catch((e)=>{
     process.exit((0, cli_namespaceObject.reportCLIError)(e));