@midscene/harmony 1.8.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/bin.mjs +72 -103
- package/dist/es/cli.mjs +73 -104
- package/dist/es/index.mjs +72 -103
- package/dist/es/mcp-server.mjs +73 -104
- package/dist/lib/bin.js +70 -101
- package/dist/lib/cli.js +71 -102
- package/dist/lib/index.js +70 -101
- package/dist/lib/mcp-server.js +71 -102
- package/dist/types/index.d.ts +8 -6
- package/dist/types/mcp-server.d.ts +8 -6
- package/package.json +4 -4
- package/static/index.html +1 -1
- package/static/static/css/{index.63b028da.css → index.26c9c911.css} +2 -2
- package/static/static/css/{index.63b028da.css.map → index.26c9c911.css.map} +1 -1
- package/static/static/js/{889.c8e2e995.js → 596.5426be9e.js} +27 -27
- package/static/static/js/596.5426be9e.js.map +1 -0
- package/static/static/js/index.acaa5ec1.js +914 -0
- package/static/static/js/index.acaa5ec1.js.map +1 -0
- package/static/static/js/889.c8e2e995.js.map +0 -1
- package/static/static/js/index.7d3d953d.js +0 -914
- package/static/static/js/index.7d3d953d.js.map +0 -1
- /package/static/static/js/{889.c8e2e995.js.LICENSE.txt → 596.5426be9e.js.LICENSE.txt} +0 -0
- /package/static/static/js/{index.7d3d953d.js.LICENSE.txt → index.acaa5ec1.js.LICENSE.txt} +0 -0
package/dist/lib/index.js
CHANGED
|
@@ -258,15 +258,6 @@ function device_define_property(obj, key, value) {
|
|
|
258
258
|
else obj[key] = value;
|
|
259
259
|
return obj;
|
|
260
260
|
}
|
|
261
|
-
const harmonyInputParamSchema = core_namespaceObject.z.object({
|
|
262
|
-
value: core_namespaceObject.z.string().describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
|
|
263
|
-
mode: core_namespaceObject.z.preprocess((val)=>'append' === val ? 'typeOnly' : val, core_namespaceObject.z["enum"]([
|
|
264
|
-
'replace',
|
|
265
|
-
'clear',
|
|
266
|
-
'typeOnly'
|
|
267
|
-
]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "typeOnly" - type the value directly without clearing the field first; "clear" - attempt to clear the field (limited support on HarmonyOS).')),
|
|
268
|
-
locate: (0, core_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be filled').optional()
|
|
269
|
-
});
|
|
270
261
|
const defaultScrollUntilTimes = 10;
|
|
271
262
|
const defaultFastSwipeSpeed = 2000;
|
|
272
263
|
const maxScrollDistance = 9999999;
|
|
@@ -307,89 +298,15 @@ const keyNameAliasMap = {
|
|
|
307
298
|
};
|
|
308
299
|
class HarmonyDevice {
|
|
309
300
|
actionSpace() {
|
|
301
|
+
const mobileActionContext = {
|
|
302
|
+
input: this.inputPrimitives,
|
|
303
|
+
size: ()=>this.size(),
|
|
304
|
+
sleep: async (timeMs)=>{
|
|
305
|
+
await (0, utils_namespaceObject.sleep)(timeMs);
|
|
306
|
+
}
|
|
307
|
+
};
|
|
310
308
|
const defaultActions = [
|
|
311
|
-
(0, device_namespaceObject.
|
|
312
|
-
const element = param.locate;
|
|
313
|
-
external_node_assert_default()(element, 'Element not found, cannot tap');
|
|
314
|
-
await this.tap(element.center[0], element.center[1]);
|
|
315
|
-
}),
|
|
316
|
-
(0, device_namespaceObject.defineActionDoubleClick)(async (param)=>{
|
|
317
|
-
const element = param.locate;
|
|
318
|
-
external_node_assert_default()(element, 'Element not found, cannot double click');
|
|
319
|
-
await this.doubleTap(element.center[0], element.center[1]);
|
|
320
|
-
}),
|
|
321
|
-
(0, device_namespaceObject.defineAction)({
|
|
322
|
-
name: 'Input',
|
|
323
|
-
description: 'Input text into the input field',
|
|
324
|
-
interfaceAlias: 'aiInput',
|
|
325
|
-
paramSchema: harmonyInputParamSchema,
|
|
326
|
-
sample: {
|
|
327
|
-
value: 'test@example.com',
|
|
328
|
-
locate: {
|
|
329
|
-
prompt: 'the email input field'
|
|
330
|
-
}
|
|
331
|
-
},
|
|
332
|
-
call: async (param)=>{
|
|
333
|
-
const element = param.locate;
|
|
334
|
-
if ('clear' === param.mode) return void await this.clearInput(element);
|
|
335
|
-
if (!param || !param.value) return;
|
|
336
|
-
const shouldReplace = 'typeOnly' !== param.mode;
|
|
337
|
-
await this.inputText(param.value, element, shouldReplace);
|
|
338
|
-
}
|
|
339
|
-
}),
|
|
340
|
-
(0, device_namespaceObject.defineActionScroll)(async (param)=>{
|
|
341
|
-
const element = param.locate;
|
|
342
|
-
const startingPoint = element ? {
|
|
343
|
-
left: element.center[0],
|
|
344
|
-
top: element.center[1]
|
|
345
|
-
} : void 0;
|
|
346
|
-
const scrollToEventName = param?.scrollType;
|
|
347
|
-
if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
|
|
348
|
-
else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
|
|
349
|
-
else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
|
|
350
|
-
else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
|
|
351
|
-
else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
|
|
352
|
-
else {
|
|
353
|
-
if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
|
|
354
|
-
else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
|
|
355
|
-
else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
|
|
356
|
-
else throw new Error(`Unknown scroll direction: ${param.direction}`);
|
|
357
|
-
else await this.scrollDown(param?.distance ?? void 0, startingPoint);
|
|
358
|
-
await (0, utils_namespaceObject.sleep)(500);
|
|
359
|
-
}
|
|
360
|
-
}),
|
|
361
|
-
(0, device_namespaceObject.defineActionDragAndDrop)(async (param)=>{
|
|
362
|
-
const from = param.from;
|
|
363
|
-
const to = param.to;
|
|
364
|
-
external_node_assert_default()(from, 'missing "from" param for drag and drop');
|
|
365
|
-
external_node_assert_default()(to, 'missing "to" param for drag and drop');
|
|
366
|
-
const hdc = await this.getHdc();
|
|
367
|
-
await hdc.drag(from.center[0], from.center[1], to.center[0], to.center[1]);
|
|
368
|
-
}),
|
|
369
|
-
(0, device_namespaceObject.defineActionSwipe)(async (param)=>{
|
|
370
|
-
const { startPoint, endPoint, duration, repeatCount } = (0, device_namespaceObject.normalizeMobileSwipeParam)(param, await this.size());
|
|
371
|
-
const hdc = await this.getHdc();
|
|
372
|
-
for(let i = 0; i < repeatCount; i++)await hdc.swipe(startPoint.x, startPoint.y, endPoint.x, endPoint.y, duration ? Math.round(duration) : void 0);
|
|
373
|
-
}),
|
|
374
|
-
(0, device_namespaceObject.defineActionKeyboardPress)(async (param)=>{
|
|
375
|
-
await this.keyboardPress(param.keyName);
|
|
376
|
-
}),
|
|
377
|
-
(0, device_namespaceObject.defineActionCursorMove)(async (param)=>{
|
|
378
|
-
const arrowKey = 'left' === param.direction ? 'ArrowLeft' : 'ArrowRight';
|
|
379
|
-
const times = param.times ?? 1;
|
|
380
|
-
for(let i = 0; i < times; i++){
|
|
381
|
-
await this.keyboardPress(arrowKey);
|
|
382
|
-
await (0, utils_namespaceObject.sleep)(100);
|
|
383
|
-
}
|
|
384
|
-
}),
|
|
385
|
-
(0, device_namespaceObject.defineActionLongPress)(async (param)=>{
|
|
386
|
-
const element = param.locate;
|
|
387
|
-
if (!element) throw new Error('LongPress requires an element to be located');
|
|
388
|
-
await this.longPress(element.center[0], element.center[1]);
|
|
389
|
-
}),
|
|
390
|
-
(0, device_namespaceObject.defineActionClearInput)(async (param)=>{
|
|
391
|
-
await this.clearInput(param.locate);
|
|
392
|
-
})
|
|
309
|
+
...(0, device_namespaceObject.createDefaultMobileActions)(mobileActionContext)
|
|
393
310
|
];
|
|
394
311
|
const platformSpecificActions = Object.values(createPlatformActions(this));
|
|
395
312
|
const customActions = this.customActions ?? [];
|
|
@@ -399,6 +316,27 @@ class HarmonyDevice {
|
|
|
399
316
|
...customActions
|
|
400
317
|
];
|
|
401
318
|
}
|
|
319
|
+
async performActionScroll(param) {
|
|
320
|
+
const element = param.locate;
|
|
321
|
+
const startingPoint = element ? {
|
|
322
|
+
left: element.center[0],
|
|
323
|
+
top: element.center[1]
|
|
324
|
+
} : void 0;
|
|
325
|
+
const scrollToEventName = param?.scrollType;
|
|
326
|
+
if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
|
|
327
|
+
else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
|
|
328
|
+
else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
|
|
329
|
+
else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
|
|
330
|
+
else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
|
|
331
|
+
else {
|
|
332
|
+
if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
|
|
333
|
+
else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
|
|
334
|
+
else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
|
|
335
|
+
else throw new Error(`Unknown scroll direction: ${param.direction}`);
|
|
336
|
+
else await this.scrollDown(param?.distance ?? void 0, startingPoint);
|
|
337
|
+
await (0, utils_namespaceObject.sleep)(500);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
402
340
|
describe() {
|
|
403
341
|
return this.descriptionText || `DeviceId: ${this.deviceId}`;
|
|
404
342
|
}
|
|
@@ -529,23 +467,23 @@ class HarmonyDevice {
|
|
|
529
467
|
}
|
|
530
468
|
throw new Error('Screenshot buffer is empty after retries');
|
|
531
469
|
}
|
|
532
|
-
async
|
|
470
|
+
async tapPoint(point) {
|
|
533
471
|
this.lastTapPosition = {
|
|
534
|
-
x,
|
|
535
|
-
y
|
|
472
|
+
x: point.x,
|
|
473
|
+
y: point.y
|
|
536
474
|
};
|
|
537
475
|
const hdc = await this.getHdc();
|
|
538
|
-
await hdc.click(x, y);
|
|
476
|
+
await hdc.click(point.x, point.y);
|
|
539
477
|
}
|
|
540
|
-
async
|
|
478
|
+
async doubleTapPoint(point) {
|
|
541
479
|
const hdc = await this.getHdc();
|
|
542
|
-
await hdc.doubleClick(x, y);
|
|
480
|
+
await hdc.doubleClick(point.x, point.y);
|
|
543
481
|
}
|
|
544
|
-
async
|
|
482
|
+
async longPressPoint(point) {
|
|
545
483
|
const hdc = await this.getHdc();
|
|
546
|
-
await hdc.longClick(x, y);
|
|
484
|
+
await hdc.longClick(point.x, point.y);
|
|
547
485
|
}
|
|
548
|
-
async
|
|
486
|
+
async typeText(text, element, shouldReplace) {
|
|
549
487
|
if (!text) return;
|
|
550
488
|
const hdc = await this.getHdc();
|
|
551
489
|
let x;
|
|
@@ -576,7 +514,7 @@ class HarmonyDevice {
|
|
|
576
514
|
}
|
|
577
515
|
await hdc.clearTextField(100);
|
|
578
516
|
}
|
|
579
|
-
async
|
|
517
|
+
async pressKey(key) {
|
|
580
518
|
const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
|
|
581
519
|
const harmonyKey = harmonyKeyCodeMap[normalizedKey] ?? key;
|
|
582
520
|
const hdc = await this.getHdc();
|
|
@@ -776,6 +714,37 @@ class HarmonyDevice {
|
|
|
776
714
|
device_define_property(this, "interfaceType", 'harmony');
|
|
777
715
|
device_define_property(this, "uri", void 0);
|
|
778
716
|
device_define_property(this, "options", void 0);
|
|
717
|
+
device_define_property(this, "inputPrimitives", {
|
|
718
|
+
pointer: {
|
|
719
|
+
tap: (point)=>this.tapPoint(point),
|
|
720
|
+
doubleClick: (point)=>this.doubleTapPoint(point),
|
|
721
|
+
longPress: (point)=>this.longPressPoint(point),
|
|
722
|
+
dragAndDrop: async (from, to)=>{
|
|
723
|
+
const hdc = await this.getHdc();
|
|
724
|
+
await hdc.drag(from.x, from.y, to.x, to.y);
|
|
725
|
+
}
|
|
726
|
+
},
|
|
727
|
+
keyboard: {
|
|
728
|
+
keyboardPress: (keyName)=>this.pressKey(keyName),
|
|
729
|
+
typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true),
|
|
730
|
+
clearInput: (target)=>this.clearInput(target),
|
|
731
|
+
cursorMove: async (direction, times = 1)=>{
|
|
732
|
+
const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
|
|
733
|
+
for(let i = 0; i < times; i++)await this.pressKey(arrowKey);
|
|
734
|
+
}
|
|
735
|
+
},
|
|
736
|
+
touch: {
|
|
737
|
+
swipe: async (start, end, opts)=>{
|
|
738
|
+
const duration = opts?.duration;
|
|
739
|
+
const repeatCount = opts?.repeat ?? 1;
|
|
740
|
+
const hdc = await this.getHdc();
|
|
741
|
+
for(let i = 0; i < repeatCount; i++)await hdc.swipe(start.x, start.y, end.x, end.y, duration ? Math.round(duration) : void 0);
|
|
742
|
+
}
|
|
743
|
+
},
|
|
744
|
+
scroll: {
|
|
745
|
+
scroll: (param)=>this.performActionScroll(param)
|
|
746
|
+
}
|
|
747
|
+
});
|
|
779
748
|
device_define_property(this, "remoteScreenshotPath", '/data/local/tmp/ms_screen.jpeg');
|
|
780
749
|
device_define_property(this, "localScreenshotPath", null);
|
|
781
750
|
external_node_assert_default()(deviceId, 'deviceId is required for HarmonyDevice');
|
package/dist/lib/mcp-server.js
CHANGED
|
@@ -291,15 +291,6 @@ function device_define_property(obj, key, value) {
|
|
|
291
291
|
else obj[key] = value;
|
|
292
292
|
return obj;
|
|
293
293
|
}
|
|
294
|
-
const harmonyInputParamSchema = core_namespaceObject.z.object({
|
|
295
|
-
value: core_namespaceObject.z.string().describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
|
|
296
|
-
mode: core_namespaceObject.z.preprocess((val)=>'append' === val ? 'typeOnly' : val, core_namespaceObject.z["enum"]([
|
|
297
|
-
'replace',
|
|
298
|
-
'clear',
|
|
299
|
-
'typeOnly'
|
|
300
|
-
]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "typeOnly" - type the value directly without clearing the field first; "clear" - attempt to clear the field (limited support on HarmonyOS).')),
|
|
301
|
-
locate: (0, core_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be filled').optional()
|
|
302
|
-
});
|
|
303
294
|
const defaultScrollUntilTimes = 10;
|
|
304
295
|
const defaultFastSwipeSpeed = 2000;
|
|
305
296
|
const maxScrollDistance = 9999999;
|
|
@@ -340,89 +331,15 @@ const keyNameAliasMap = {
|
|
|
340
331
|
};
|
|
341
332
|
class HarmonyDevice {
|
|
342
333
|
actionSpace() {
|
|
334
|
+
const mobileActionContext = {
|
|
335
|
+
input: this.inputPrimitives,
|
|
336
|
+
size: ()=>this.size(),
|
|
337
|
+
sleep: async (timeMs)=>{
|
|
338
|
+
await (0, core_utils_namespaceObject.sleep)(timeMs);
|
|
339
|
+
}
|
|
340
|
+
};
|
|
343
341
|
const defaultActions = [
|
|
344
|
-
(0, device_namespaceObject.
|
|
345
|
-
const element = param.locate;
|
|
346
|
-
external_node_assert_default()(element, 'Element not found, cannot tap');
|
|
347
|
-
await this.tap(element.center[0], element.center[1]);
|
|
348
|
-
}),
|
|
349
|
-
(0, device_namespaceObject.defineActionDoubleClick)(async (param)=>{
|
|
350
|
-
const element = param.locate;
|
|
351
|
-
external_node_assert_default()(element, 'Element not found, cannot double click');
|
|
352
|
-
await this.doubleTap(element.center[0], element.center[1]);
|
|
353
|
-
}),
|
|
354
|
-
(0, device_namespaceObject.defineAction)({
|
|
355
|
-
name: 'Input',
|
|
356
|
-
description: 'Input text into the input field',
|
|
357
|
-
interfaceAlias: 'aiInput',
|
|
358
|
-
paramSchema: harmonyInputParamSchema,
|
|
359
|
-
sample: {
|
|
360
|
-
value: 'test@example.com',
|
|
361
|
-
locate: {
|
|
362
|
-
prompt: 'the email input field'
|
|
363
|
-
}
|
|
364
|
-
},
|
|
365
|
-
call: async (param)=>{
|
|
366
|
-
const element = param.locate;
|
|
367
|
-
if ('clear' === param.mode) return void await this.clearInput(element);
|
|
368
|
-
if (!param || !param.value) return;
|
|
369
|
-
const shouldReplace = 'typeOnly' !== param.mode;
|
|
370
|
-
await this.inputText(param.value, element, shouldReplace);
|
|
371
|
-
}
|
|
372
|
-
}),
|
|
373
|
-
(0, device_namespaceObject.defineActionScroll)(async (param)=>{
|
|
374
|
-
const element = param.locate;
|
|
375
|
-
const startingPoint = element ? {
|
|
376
|
-
left: element.center[0],
|
|
377
|
-
top: element.center[1]
|
|
378
|
-
} : void 0;
|
|
379
|
-
const scrollToEventName = param?.scrollType;
|
|
380
|
-
if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
|
|
381
|
-
else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
|
|
382
|
-
else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
|
|
383
|
-
else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
|
|
384
|
-
else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
|
|
385
|
-
else {
|
|
386
|
-
if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
|
|
387
|
-
else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
|
|
388
|
-
else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
|
|
389
|
-
else throw new Error(`Unknown scroll direction: ${param.direction}`);
|
|
390
|
-
else await this.scrollDown(param?.distance ?? void 0, startingPoint);
|
|
391
|
-
await (0, core_utils_namespaceObject.sleep)(500);
|
|
392
|
-
}
|
|
393
|
-
}),
|
|
394
|
-
(0, device_namespaceObject.defineActionDragAndDrop)(async (param)=>{
|
|
395
|
-
const from = param.from;
|
|
396
|
-
const to = param.to;
|
|
397
|
-
external_node_assert_default()(from, 'missing "from" param for drag and drop');
|
|
398
|
-
external_node_assert_default()(to, 'missing "to" param for drag and drop');
|
|
399
|
-
const hdc = await this.getHdc();
|
|
400
|
-
await hdc.drag(from.center[0], from.center[1], to.center[0], to.center[1]);
|
|
401
|
-
}),
|
|
402
|
-
(0, device_namespaceObject.defineActionSwipe)(async (param)=>{
|
|
403
|
-
const { startPoint, endPoint, duration, repeatCount } = (0, device_namespaceObject.normalizeMobileSwipeParam)(param, await this.size());
|
|
404
|
-
const hdc = await this.getHdc();
|
|
405
|
-
for(let i = 0; i < repeatCount; i++)await hdc.swipe(startPoint.x, startPoint.y, endPoint.x, endPoint.y, duration ? Math.round(duration) : void 0);
|
|
406
|
-
}),
|
|
407
|
-
(0, device_namespaceObject.defineActionKeyboardPress)(async (param)=>{
|
|
408
|
-
await this.keyboardPress(param.keyName);
|
|
409
|
-
}),
|
|
410
|
-
(0, device_namespaceObject.defineActionCursorMove)(async (param)=>{
|
|
411
|
-
const arrowKey = 'left' === param.direction ? 'ArrowLeft' : 'ArrowRight';
|
|
412
|
-
const times = param.times ?? 1;
|
|
413
|
-
for(let i = 0; i < times; i++){
|
|
414
|
-
await this.keyboardPress(arrowKey);
|
|
415
|
-
await (0, core_utils_namespaceObject.sleep)(100);
|
|
416
|
-
}
|
|
417
|
-
}),
|
|
418
|
-
(0, device_namespaceObject.defineActionLongPress)(async (param)=>{
|
|
419
|
-
const element = param.locate;
|
|
420
|
-
if (!element) throw new Error('LongPress requires an element to be located');
|
|
421
|
-
await this.longPress(element.center[0], element.center[1]);
|
|
422
|
-
}),
|
|
423
|
-
(0, device_namespaceObject.defineActionClearInput)(async (param)=>{
|
|
424
|
-
await this.clearInput(param.locate);
|
|
425
|
-
})
|
|
342
|
+
...(0, device_namespaceObject.createDefaultMobileActions)(mobileActionContext)
|
|
426
343
|
];
|
|
427
344
|
const platformSpecificActions = Object.values(createPlatformActions(this));
|
|
428
345
|
const customActions = this.customActions ?? [];
|
|
@@ -432,6 +349,27 @@ class HarmonyDevice {
|
|
|
432
349
|
...customActions
|
|
433
350
|
];
|
|
434
351
|
}
|
|
352
|
+
async performActionScroll(param) {
|
|
353
|
+
const element = param.locate;
|
|
354
|
+
const startingPoint = element ? {
|
|
355
|
+
left: element.center[0],
|
|
356
|
+
top: element.center[1]
|
|
357
|
+
} : void 0;
|
|
358
|
+
const scrollToEventName = param?.scrollType;
|
|
359
|
+
if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
|
|
360
|
+
else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
|
|
361
|
+
else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
|
|
362
|
+
else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
|
|
363
|
+
else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
|
|
364
|
+
else {
|
|
365
|
+
if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
|
|
366
|
+
else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
|
|
367
|
+
else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
|
|
368
|
+
else throw new Error(`Unknown scroll direction: ${param.direction}`);
|
|
369
|
+
else await this.scrollDown(param?.distance ?? void 0, startingPoint);
|
|
370
|
+
await (0, core_utils_namespaceObject.sleep)(500);
|
|
371
|
+
}
|
|
372
|
+
}
|
|
435
373
|
describe() {
|
|
436
374
|
return this.descriptionText || `DeviceId: ${this.deviceId}`;
|
|
437
375
|
}
|
|
@@ -562,23 +500,23 @@ class HarmonyDevice {
|
|
|
562
500
|
}
|
|
563
501
|
throw new Error('Screenshot buffer is empty after retries');
|
|
564
502
|
}
|
|
565
|
-
async
|
|
503
|
+
async tapPoint(point) {
|
|
566
504
|
this.lastTapPosition = {
|
|
567
|
-
x,
|
|
568
|
-
y
|
|
505
|
+
x: point.x,
|
|
506
|
+
y: point.y
|
|
569
507
|
};
|
|
570
508
|
const hdc = await this.getHdc();
|
|
571
|
-
await hdc.click(x, y);
|
|
509
|
+
await hdc.click(point.x, point.y);
|
|
572
510
|
}
|
|
573
|
-
async
|
|
511
|
+
async doubleTapPoint(point) {
|
|
574
512
|
const hdc = await this.getHdc();
|
|
575
|
-
await hdc.doubleClick(x, y);
|
|
513
|
+
await hdc.doubleClick(point.x, point.y);
|
|
576
514
|
}
|
|
577
|
-
async
|
|
515
|
+
async longPressPoint(point) {
|
|
578
516
|
const hdc = await this.getHdc();
|
|
579
|
-
await hdc.longClick(x, y);
|
|
517
|
+
await hdc.longClick(point.x, point.y);
|
|
580
518
|
}
|
|
581
|
-
async
|
|
519
|
+
async typeText(text, element, shouldReplace) {
|
|
582
520
|
if (!text) return;
|
|
583
521
|
const hdc = await this.getHdc();
|
|
584
522
|
let x;
|
|
@@ -609,7 +547,7 @@ class HarmonyDevice {
|
|
|
609
547
|
}
|
|
610
548
|
await hdc.clearTextField(100);
|
|
611
549
|
}
|
|
612
|
-
async
|
|
550
|
+
async pressKey(key) {
|
|
613
551
|
const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
|
|
614
552
|
const harmonyKey = harmonyKeyCodeMap[normalizedKey] ?? key;
|
|
615
553
|
const hdc = await this.getHdc();
|
|
@@ -809,6 +747,37 @@ class HarmonyDevice {
|
|
|
809
747
|
device_define_property(this, "interfaceType", 'harmony');
|
|
810
748
|
device_define_property(this, "uri", void 0);
|
|
811
749
|
device_define_property(this, "options", void 0);
|
|
750
|
+
device_define_property(this, "inputPrimitives", {
|
|
751
|
+
pointer: {
|
|
752
|
+
tap: (point)=>this.tapPoint(point),
|
|
753
|
+
doubleClick: (point)=>this.doubleTapPoint(point),
|
|
754
|
+
longPress: (point)=>this.longPressPoint(point),
|
|
755
|
+
dragAndDrop: async (from, to)=>{
|
|
756
|
+
const hdc = await this.getHdc();
|
|
757
|
+
await hdc.drag(from.x, from.y, to.x, to.y);
|
|
758
|
+
}
|
|
759
|
+
},
|
|
760
|
+
keyboard: {
|
|
761
|
+
keyboardPress: (keyName)=>this.pressKey(keyName),
|
|
762
|
+
typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true),
|
|
763
|
+
clearInput: (target)=>this.clearInput(target),
|
|
764
|
+
cursorMove: async (direction, times = 1)=>{
|
|
765
|
+
const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
|
|
766
|
+
for(let i = 0; i < times; i++)await this.pressKey(arrowKey);
|
|
767
|
+
}
|
|
768
|
+
},
|
|
769
|
+
touch: {
|
|
770
|
+
swipe: async (start, end, opts)=>{
|
|
771
|
+
const duration = opts?.duration;
|
|
772
|
+
const repeatCount = opts?.repeat ?? 1;
|
|
773
|
+
const hdc = await this.getHdc();
|
|
774
|
+
for(let i = 0; i < repeatCount; i++)await hdc.swipe(start.x, start.y, end.x, end.y, duration ? Math.round(duration) : void 0);
|
|
775
|
+
}
|
|
776
|
+
},
|
|
777
|
+
scroll: {
|
|
778
|
+
scroll: (param)=>this.performActionScroll(param)
|
|
779
|
+
}
|
|
780
|
+
});
|
|
812
781
|
device_define_property(this, "remoteScreenshotPath", '/data/local/tmp/ms_screen.jpeg');
|
|
813
782
|
device_define_property(this, "localScreenshotPath", null);
|
|
814
783
|
external_node_assert_default()(deviceId, 'deviceId is required for HarmonyDevice');
|
|
@@ -1067,7 +1036,7 @@ class HarmonyMCPServer extends mcp_namespaceObject.BaseMCPServer {
|
|
|
1067
1036
|
constructor(toolsManager){
|
|
1068
1037
|
super({
|
|
1069
1038
|
name: '@midscene/harmony-mcp',
|
|
1070
|
-
version: "1.8.
|
|
1039
|
+
version: "1.8.1",
|
|
1071
1040
|
description: 'Control the HarmonyOS device using natural language commands'
|
|
1072
1041
|
}, toolsManager);
|
|
1073
1042
|
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ import type { ElementInfo } from '@midscene/shared/extractor';
|
|
|
9
9
|
import { HarmonyDeviceOpt } from '@midscene/core/device';
|
|
10
10
|
import { InitArgSpec } from '@midscene/shared/mcp/base-tools';
|
|
11
11
|
import { InterfaceType } from '@midscene/core';
|
|
12
|
-
import {
|
|
12
|
+
import { MobileInputPrimitives } from '@midscene/core/device';
|
|
13
13
|
import { overrideAIConfig } from '@midscene/shared/env';
|
|
14
14
|
import { PlaygroundPlatformDescriptor } from '@midscene/playground';
|
|
15
15
|
import { Point } from '@midscene/core';
|
|
@@ -61,7 +61,9 @@ export declare class HarmonyDevice implements AbstractInterface {
|
|
|
61
61
|
interfaceType: InterfaceType;
|
|
62
62
|
uri: string | undefined;
|
|
63
63
|
options?: HarmonyDeviceOpt;
|
|
64
|
+
readonly inputPrimitives: MobileInputPrimitives;
|
|
64
65
|
actionSpace(): DeviceAction<any>[];
|
|
66
|
+
private performActionScroll;
|
|
65
67
|
constructor(deviceId: string, options?: HarmonyDeviceOpt);
|
|
66
68
|
describe(): string;
|
|
67
69
|
connect(): Promise<HdcClient>;
|
|
@@ -83,12 +85,12 @@ export declare class HarmonyDevice implements AbstractInterface {
|
|
|
83
85
|
private remoteScreenshotPath;
|
|
84
86
|
private localScreenshotPath;
|
|
85
87
|
screenshotBase64(): Promise<string>;
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
88
|
+
private tapPoint;
|
|
89
|
+
private doubleTapPoint;
|
|
90
|
+
private longPressPoint;
|
|
91
|
+
private typeText;
|
|
90
92
|
clearInput(element?: ElementInfo): Promise<void>;
|
|
91
|
-
|
|
93
|
+
private pressKey;
|
|
92
94
|
scroll(deltaX: number, deltaY: number, speed?: number): Promise<void>;
|
|
93
95
|
private scrollInDirection;
|
|
94
96
|
scrollDown(distance?: number, startPoint?: Point): Promise<void>;
|
|
@@ -12,7 +12,7 @@ import { InitArgSpec } from '@midscene/shared/mcp/base-tools';
|
|
|
12
12
|
import { InterfaceType } from '@midscene/core';
|
|
13
13
|
import { LaunchMCPServerOptions } from '@midscene/shared/mcp';
|
|
14
14
|
import { LaunchMCPServerResult } from '@midscene/shared/mcp';
|
|
15
|
-
import {
|
|
15
|
+
import { MobileInputPrimitives } from '@midscene/core/device';
|
|
16
16
|
import { Point } from '@midscene/core';
|
|
17
17
|
import { Size } from '@midscene/core';
|
|
18
18
|
import { Tool } from '@midscene/shared/mcp';
|
|
@@ -59,7 +59,9 @@ declare class HarmonyDevice implements AbstractInterface {
|
|
|
59
59
|
interfaceType: InterfaceType;
|
|
60
60
|
uri: string | undefined;
|
|
61
61
|
options?: HarmonyDeviceOpt;
|
|
62
|
+
readonly inputPrimitives: MobileInputPrimitives;
|
|
62
63
|
actionSpace(): DeviceAction<any>[];
|
|
64
|
+
private performActionScroll;
|
|
63
65
|
constructor(deviceId: string, options?: HarmonyDeviceOpt);
|
|
64
66
|
describe(): string;
|
|
65
67
|
connect(): Promise<HdcClient>;
|
|
@@ -81,12 +83,12 @@ declare class HarmonyDevice implements AbstractInterface {
|
|
|
81
83
|
private remoteScreenshotPath;
|
|
82
84
|
private localScreenshotPath;
|
|
83
85
|
screenshotBase64(): Promise<string>;
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
86
|
+
private tapPoint;
|
|
87
|
+
private doubleTapPoint;
|
|
88
|
+
private longPressPoint;
|
|
89
|
+
private typeText;
|
|
88
90
|
clearInput(element?: ElementInfo): Promise<void>;
|
|
89
|
-
|
|
91
|
+
private pressKey;
|
|
90
92
|
scroll(deltaX: number, deltaY: number, speed?: number): Promise<void>;
|
|
91
93
|
private scrollInDirection;
|
|
92
94
|
scrollDown(distance?: number, startPoint?: Point): Promise<void>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/harmony",
|
|
3
|
-
"version": "1.8.
|
|
3
|
+
"version": "1.8.1",
|
|
4
4
|
"description": "HarmonyOS automation library for Midscene",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"HarmonyOS UI automation",
|
|
@@ -41,9 +41,9 @@
|
|
|
41
41
|
"dependencies": {
|
|
42
42
|
"@inquirer/prompts": "^7.8.6",
|
|
43
43
|
"open": "10.1.0",
|
|
44
|
-
"@midscene/
|
|
45
|
-
"@midscene/
|
|
46
|
-
"@midscene/core": "1.8.
|
|
44
|
+
"@midscene/shared": "1.8.1",
|
|
45
|
+
"@midscene/playground": "1.8.1",
|
|
46
|
+
"@midscene/core": "1.8.1"
|
|
47
47
|
},
|
|
48
48
|
"devDependencies": {
|
|
49
49
|
"@rslib/core": "^0.18.3",
|
package/static/index.html
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
<!doctype html><html><head><link rel="icon" href="/favicon.ico"><title>Midscene Playground</title><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><script defer src="/static/js/lib-react.7b1abe58.js"></script><script defer src="/static/js/
|
|
1
|
+
<!doctype html><html><head><link rel="icon" href="/favicon.ico"><title>Midscene Playground</title><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><script defer src="/static/js/lib-react.7b1abe58.js"></script><script defer src="/static/js/596.5426be9e.js"></script><script defer src="/static/js/index.acaa5ec1.js"></script><link href="/static/css/index.26c9c911.css" rel="stylesheet"></head><body><div id="root"></div></body></html>
|