@midscene/harmony 1.8.7 → 1.8.8-beta-20260601092817.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/bin.mjs +92 -12
- package/dist/es/cli.mjs +93 -13
- package/dist/es/index.mjs +92 -12
- package/dist/es/mcp-server.mjs +93 -13
- package/dist/lib/bin.js +92 -12
- package/dist/lib/cli.js +93 -13
- package/dist/lib/index.js +92 -12
- package/dist/lib/mcp-server.js +93 -13
- package/dist/types/index.d.ts +32 -5
- package/dist/types/mcp-server.d.ts +32 -5
- package/package.json +4 -4
- package/static/index.html +1 -1
- package/static/static/js/{index.6becfe23.js → index.122f2630.js} +29 -29
- package/static/static/js/index.122f2630.js.map +1 -0
- package/static/static/js/index.6becfe23.js.map +0 -1
- /package/static/static/js/{index.6becfe23.js.LICENSE.txt → index.122f2630.js.LICENSE.txt} +0 -0
package/dist/lib/bin.js
CHANGED
|
@@ -156,6 +156,13 @@ class HdcClient {
|
|
|
156
156
|
async screenshot(remotePath) {
|
|
157
157
|
return await this.shell(`snapshot_display -f ${remotePath}`);
|
|
158
158
|
}
|
|
159
|
+
async dumpLayout() {
|
|
160
|
+
const remotePath = '/data/local/tmp/midscene_layout.json';
|
|
161
|
+
const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
|
|
162
|
+
const jsonStart = output.indexOf('{');
|
|
163
|
+
if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
|
|
164
|
+
return output.slice(jsonStart);
|
|
165
|
+
}
|
|
159
166
|
async click(x, y) {
|
|
160
167
|
await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
|
|
161
168
|
}
|
|
@@ -203,9 +210,17 @@ class HdcClient {
|
|
|
203
210
|
await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
|
|
204
211
|
}
|
|
205
212
|
async clearTextField(length = 100) {
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
213
|
+
if (length <= 0) return;
|
|
214
|
+
const MAX_KEYS_PER_CALL = 3;
|
|
215
|
+
const cmds = [];
|
|
216
|
+
let remaining = length;
|
|
217
|
+
while(remaining > 0){
|
|
218
|
+
const n = Math.min(MAX_KEYS_PER_CALL, remaining);
|
|
219
|
+
const codes = Array(n).fill('2055').join(' ');
|
|
220
|
+
cmds.push(`uitest uiInput keyEvent ${codes}`);
|
|
221
|
+
remaining -= n;
|
|
222
|
+
}
|
|
223
|
+
await this.shell(cmds.join(';'));
|
|
209
224
|
}
|
|
210
225
|
async startAbility(bundleName, abilityName) {
|
|
211
226
|
const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
|
|
@@ -286,6 +301,47 @@ const scrollQuadrantDivisions = 4;
|
|
|
286
301
|
const screenEdgeMargin = 50;
|
|
287
302
|
const debugDevice = (0, logger_namespaceObject.getDebug)('harmony:device');
|
|
288
303
|
let screenshotResizeScaleWarned = false;
|
|
304
|
+
const INPUT_FIELD_TYPES = new Set([
|
|
305
|
+
'TextInput',
|
|
306
|
+
'TextArea',
|
|
307
|
+
'SearchField'
|
|
308
|
+
]);
|
|
309
|
+
function parseBounds(raw) {
|
|
310
|
+
if ('string' != typeof raw) return null;
|
|
311
|
+
const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
|
|
312
|
+
if (!m) return null;
|
|
313
|
+
return {
|
|
314
|
+
x1: Number(m[1]),
|
|
315
|
+
y1: Number(m[2]),
|
|
316
|
+
x2: Number(m[3]),
|
|
317
|
+
y2: Number(m[4])
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
function collectInputFields(layout) {
|
|
321
|
+
const fields = [];
|
|
322
|
+
const visit = (node)=>{
|
|
323
|
+
if (!node || 'object' != typeof node) return;
|
|
324
|
+
const n = node;
|
|
325
|
+
const attrs = n.attributes ?? {};
|
|
326
|
+
if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
|
|
327
|
+
const bounds = parseBounds(attrs.bounds);
|
|
328
|
+
if (bounds) fields.push({
|
|
329
|
+
text: String(attrs.text ?? ''),
|
|
330
|
+
bounds
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
for (const child of n.children ?? [])visit(child);
|
|
334
|
+
};
|
|
335
|
+
visit(layout);
|
|
336
|
+
return fields;
|
|
337
|
+
}
|
|
338
|
+
function pickFieldByPoint(fields, point) {
|
|
339
|
+
const [px, py] = point;
|
|
340
|
+
return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
|
|
341
|
+
}
|
|
342
|
+
function pickLongestField(fields) {
|
|
343
|
+
return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
|
|
344
|
+
}
|
|
289
345
|
const harmonyKeyCodeMap = {
|
|
290
346
|
Enter: '2054',
|
|
291
347
|
Backspace: '2055',
|
|
@@ -521,12 +577,13 @@ class device_HarmonyDevice {
|
|
|
521
577
|
if (shouldReplace) {
|
|
522
578
|
await hdc.click(x, y);
|
|
523
579
|
await (0, core_utils_namespaceObject.sleep)(100);
|
|
524
|
-
await
|
|
580
|
+
const length = await this.resolveClearLength(element);
|
|
581
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
525
582
|
await (0, core_utils_namespaceObject.sleep)(100);
|
|
526
583
|
}
|
|
527
584
|
await hdc.inputText(x, y, text);
|
|
528
|
-
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
|
|
529
|
-
if (shouldAutoDismissKeyboard) await this.hideKeyboard();
|
|
585
|
+
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
|
|
586
|
+
if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
|
|
530
587
|
}
|
|
531
588
|
async clearInput(element) {
|
|
532
589
|
const hdc = await this.getHdc();
|
|
@@ -534,7 +591,24 @@ class device_HarmonyDevice {
|
|
|
534
591
|
await hdc.click(element.center[0], element.center[1]);
|
|
535
592
|
await (0, core_utils_namespaceObject.sleep)(100);
|
|
536
593
|
}
|
|
537
|
-
await
|
|
594
|
+
const length = await this.resolveClearLength(element);
|
|
595
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
596
|
+
}
|
|
597
|
+
async resolveClearLength(element) {
|
|
598
|
+
const PADDING = 2;
|
|
599
|
+
const FALLBACK_LENGTH = 100;
|
|
600
|
+
try {
|
|
601
|
+
const hdc = await this.getHdc();
|
|
602
|
+
const layoutJson = await hdc.dumpLayout();
|
|
603
|
+
const layout = JSON.parse(layoutJson);
|
|
604
|
+
const fields = collectInputFields(layout);
|
|
605
|
+
if (0 === fields.length) return FALLBACK_LENGTH;
|
|
606
|
+
const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
|
|
607
|
+
return target.text.length + PADDING;
|
|
608
|
+
} catch (e) {
|
|
609
|
+
debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
|
|
610
|
+
return FALLBACK_LENGTH;
|
|
611
|
+
}
|
|
538
612
|
}
|
|
539
613
|
async pressKey(key) {
|
|
540
614
|
const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
|
|
@@ -697,9 +771,11 @@ class device_HarmonyDevice {
|
|
|
697
771
|
const hdc = await this.getHdc();
|
|
698
772
|
await hdc.keyEvent('RecentApps');
|
|
699
773
|
}
|
|
700
|
-
async hideKeyboard() {
|
|
774
|
+
async hideKeyboard(options) {
|
|
701
775
|
const hdc = await this.getHdc();
|
|
702
|
-
|
|
776
|
+
const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
|
|
777
|
+
const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
|
|
778
|
+
await hdc.keyEvent(key);
|
|
703
779
|
}
|
|
704
780
|
async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
|
|
705
781
|
const hdc = await this.getHdc();
|
|
@@ -748,9 +824,13 @@ class device_HarmonyDevice {
|
|
|
748
824
|
},
|
|
749
825
|
keyboard: {
|
|
750
826
|
keyboardPress: (keyName)=>this.pressKey(keyName),
|
|
751
|
-
typeText: (value, opts)=>
|
|
752
|
-
|
|
753
|
-
|
|
827
|
+
typeText: (value, opts)=>{
|
|
828
|
+
const harmonyOpts = opts;
|
|
829
|
+
return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
|
|
830
|
+
autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
|
|
831
|
+
keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
|
|
832
|
+
});
|
|
833
|
+
},
|
|
754
834
|
clearInput: (target)=>this.clearInput(target),
|
|
755
835
|
cursorMove: async (direction, times = 1)=>{
|
|
756
836
|
const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
|
package/dist/lib/cli.js
CHANGED
|
@@ -152,6 +152,13 @@ class HdcClient {
|
|
|
152
152
|
async screenshot(remotePath) {
|
|
153
153
|
return await this.shell(`snapshot_display -f ${remotePath}`);
|
|
154
154
|
}
|
|
155
|
+
async dumpLayout() {
|
|
156
|
+
const remotePath = '/data/local/tmp/midscene_layout.json';
|
|
157
|
+
const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
|
|
158
|
+
const jsonStart = output.indexOf('{');
|
|
159
|
+
if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
|
|
160
|
+
return output.slice(jsonStart);
|
|
161
|
+
}
|
|
155
162
|
async click(x, y) {
|
|
156
163
|
await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
|
|
157
164
|
}
|
|
@@ -199,9 +206,17 @@ class HdcClient {
|
|
|
199
206
|
await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
|
|
200
207
|
}
|
|
201
208
|
async clearTextField(length = 100) {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
209
|
+
if (length <= 0) return;
|
|
210
|
+
const MAX_KEYS_PER_CALL = 3;
|
|
211
|
+
const cmds = [];
|
|
212
|
+
let remaining = length;
|
|
213
|
+
while(remaining > 0){
|
|
214
|
+
const n = Math.min(MAX_KEYS_PER_CALL, remaining);
|
|
215
|
+
const codes = Array(n).fill('2055').join(' ');
|
|
216
|
+
cmds.push(`uitest uiInput keyEvent ${codes}`);
|
|
217
|
+
remaining -= n;
|
|
218
|
+
}
|
|
219
|
+
await this.shell(cmds.join(';'));
|
|
205
220
|
}
|
|
206
221
|
async startAbility(bundleName, abilityName) {
|
|
207
222
|
const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
|
|
@@ -282,6 +297,47 @@ const scrollQuadrantDivisions = 4;
|
|
|
282
297
|
const screenEdgeMargin = 50;
|
|
283
298
|
const debugDevice = (0, logger_namespaceObject.getDebug)('harmony:device');
|
|
284
299
|
let screenshotResizeScaleWarned = false;
|
|
300
|
+
const INPUT_FIELD_TYPES = new Set([
|
|
301
|
+
'TextInput',
|
|
302
|
+
'TextArea',
|
|
303
|
+
'SearchField'
|
|
304
|
+
]);
|
|
305
|
+
function parseBounds(raw) {
|
|
306
|
+
if ('string' != typeof raw) return null;
|
|
307
|
+
const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
|
|
308
|
+
if (!m) return null;
|
|
309
|
+
return {
|
|
310
|
+
x1: Number(m[1]),
|
|
311
|
+
y1: Number(m[2]),
|
|
312
|
+
x2: Number(m[3]),
|
|
313
|
+
y2: Number(m[4])
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
function collectInputFields(layout) {
|
|
317
|
+
const fields = [];
|
|
318
|
+
const visit = (node)=>{
|
|
319
|
+
if (!node || 'object' != typeof node) return;
|
|
320
|
+
const n = node;
|
|
321
|
+
const attrs = n.attributes ?? {};
|
|
322
|
+
if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
|
|
323
|
+
const bounds = parseBounds(attrs.bounds);
|
|
324
|
+
if (bounds) fields.push({
|
|
325
|
+
text: String(attrs.text ?? ''),
|
|
326
|
+
bounds
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
for (const child of n.children ?? [])visit(child);
|
|
330
|
+
};
|
|
331
|
+
visit(layout);
|
|
332
|
+
return fields;
|
|
333
|
+
}
|
|
334
|
+
function pickFieldByPoint(fields, point) {
|
|
335
|
+
const [px, py] = point;
|
|
336
|
+
return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
|
|
337
|
+
}
|
|
338
|
+
function pickLongestField(fields) {
|
|
339
|
+
return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
|
|
340
|
+
}
|
|
285
341
|
const harmonyKeyCodeMap = {
|
|
286
342
|
Enter: '2054',
|
|
287
343
|
Backspace: '2055',
|
|
@@ -517,12 +573,13 @@ class HarmonyDevice {
|
|
|
517
573
|
if (shouldReplace) {
|
|
518
574
|
await hdc.click(x, y);
|
|
519
575
|
await (0, core_utils_namespaceObject.sleep)(100);
|
|
520
|
-
await
|
|
576
|
+
const length = await this.resolveClearLength(element);
|
|
577
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
521
578
|
await (0, core_utils_namespaceObject.sleep)(100);
|
|
522
579
|
}
|
|
523
580
|
await hdc.inputText(x, y, text);
|
|
524
|
-
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
|
|
525
|
-
if (shouldAutoDismissKeyboard) await this.hideKeyboard();
|
|
581
|
+
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
|
|
582
|
+
if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
|
|
526
583
|
}
|
|
527
584
|
async clearInput(element) {
|
|
528
585
|
const hdc = await this.getHdc();
|
|
@@ -530,7 +587,24 @@ class HarmonyDevice {
|
|
|
530
587
|
await hdc.click(element.center[0], element.center[1]);
|
|
531
588
|
await (0, core_utils_namespaceObject.sleep)(100);
|
|
532
589
|
}
|
|
533
|
-
await
|
|
590
|
+
const length = await this.resolveClearLength(element);
|
|
591
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
592
|
+
}
|
|
593
|
+
async resolveClearLength(element) {
|
|
594
|
+
const PADDING = 2;
|
|
595
|
+
const FALLBACK_LENGTH = 100;
|
|
596
|
+
try {
|
|
597
|
+
const hdc = await this.getHdc();
|
|
598
|
+
const layoutJson = await hdc.dumpLayout();
|
|
599
|
+
const layout = JSON.parse(layoutJson);
|
|
600
|
+
const fields = collectInputFields(layout);
|
|
601
|
+
if (0 === fields.length) return FALLBACK_LENGTH;
|
|
602
|
+
const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
|
|
603
|
+
return target.text.length + PADDING;
|
|
604
|
+
} catch (e) {
|
|
605
|
+
debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
|
|
606
|
+
return FALLBACK_LENGTH;
|
|
607
|
+
}
|
|
534
608
|
}
|
|
535
609
|
async pressKey(key) {
|
|
536
610
|
const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
|
|
@@ -693,9 +767,11 @@ class HarmonyDevice {
|
|
|
693
767
|
const hdc = await this.getHdc();
|
|
694
768
|
await hdc.keyEvent('RecentApps');
|
|
695
769
|
}
|
|
696
|
-
async hideKeyboard() {
|
|
770
|
+
async hideKeyboard(options) {
|
|
697
771
|
const hdc = await this.getHdc();
|
|
698
|
-
|
|
772
|
+
const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
|
|
773
|
+
const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
|
|
774
|
+
await hdc.keyEvent(key);
|
|
699
775
|
}
|
|
700
776
|
async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
|
|
701
777
|
const hdc = await this.getHdc();
|
|
@@ -744,9 +820,13 @@ class HarmonyDevice {
|
|
|
744
820
|
},
|
|
745
821
|
keyboard: {
|
|
746
822
|
keyboardPress: (keyName)=>this.pressKey(keyName),
|
|
747
|
-
typeText: (value, opts)=>
|
|
748
|
-
|
|
749
|
-
|
|
823
|
+
typeText: (value, opts)=>{
|
|
824
|
+
const harmonyOpts = opts;
|
|
825
|
+
return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
|
|
826
|
+
autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
|
|
827
|
+
keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
|
|
828
|
+
});
|
|
829
|
+
},
|
|
750
830
|
clearInput: (target)=>this.clearInput(target),
|
|
751
831
|
cursorMove: async (direction, times = 1)=>{
|
|
752
832
|
const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
|
|
@@ -1013,7 +1093,7 @@ class HarmonyMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
1013
1093
|
const tools = new HarmonyMidsceneTools();
|
|
1014
1094
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-harmony', {
|
|
1015
1095
|
stripPrefix: 'harmony_',
|
|
1016
|
-
version: "1.8.
|
|
1096
|
+
version: "1.8.8-beta-20260601092817.0",
|
|
1017
1097
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
1018
1098
|
}).catch((e)=>{
|
|
1019
1099
|
process.exit((0, cli_namespaceObject.reportCLIError)(e));
|
package/dist/lib/index.js
CHANGED
|
@@ -135,6 +135,13 @@ class HdcClient {
|
|
|
135
135
|
async screenshot(remotePath) {
|
|
136
136
|
return await this.shell(`snapshot_display -f ${remotePath}`);
|
|
137
137
|
}
|
|
138
|
+
async dumpLayout() {
|
|
139
|
+
const remotePath = '/data/local/tmp/midscene_layout.json';
|
|
140
|
+
const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
|
|
141
|
+
const jsonStart = output.indexOf('{');
|
|
142
|
+
if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
|
|
143
|
+
return output.slice(jsonStart);
|
|
144
|
+
}
|
|
138
145
|
async click(x, y) {
|
|
139
146
|
await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
|
|
140
147
|
}
|
|
@@ -182,9 +189,17 @@ class HdcClient {
|
|
|
182
189
|
await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
|
|
183
190
|
}
|
|
184
191
|
async clearTextField(length = 100) {
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
192
|
+
if (length <= 0) return;
|
|
193
|
+
const MAX_KEYS_PER_CALL = 3;
|
|
194
|
+
const cmds = [];
|
|
195
|
+
let remaining = length;
|
|
196
|
+
while(remaining > 0){
|
|
197
|
+
const n = Math.min(MAX_KEYS_PER_CALL, remaining);
|
|
198
|
+
const codes = Array(n).fill('2055').join(' ');
|
|
199
|
+
cmds.push(`uitest uiInput keyEvent ${codes}`);
|
|
200
|
+
remaining -= n;
|
|
201
|
+
}
|
|
202
|
+
await this.shell(cmds.join(';'));
|
|
188
203
|
}
|
|
189
204
|
async startAbility(bundleName, abilityName) {
|
|
190
205
|
const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
|
|
@@ -265,6 +280,47 @@ const scrollQuadrantDivisions = 4;
|
|
|
265
280
|
const screenEdgeMargin = 50;
|
|
266
281
|
const debugDevice = (0, logger_namespaceObject.getDebug)('harmony:device');
|
|
267
282
|
let screenshotResizeScaleWarned = false;
|
|
283
|
+
const INPUT_FIELD_TYPES = new Set([
|
|
284
|
+
'TextInput',
|
|
285
|
+
'TextArea',
|
|
286
|
+
'SearchField'
|
|
287
|
+
]);
|
|
288
|
+
function parseBounds(raw) {
|
|
289
|
+
if ('string' != typeof raw) return null;
|
|
290
|
+
const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
|
|
291
|
+
if (!m) return null;
|
|
292
|
+
return {
|
|
293
|
+
x1: Number(m[1]),
|
|
294
|
+
y1: Number(m[2]),
|
|
295
|
+
x2: Number(m[3]),
|
|
296
|
+
y2: Number(m[4])
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
function collectInputFields(layout) {
|
|
300
|
+
const fields = [];
|
|
301
|
+
const visit = (node)=>{
|
|
302
|
+
if (!node || 'object' != typeof node) return;
|
|
303
|
+
const n = node;
|
|
304
|
+
const attrs = n.attributes ?? {};
|
|
305
|
+
if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
|
|
306
|
+
const bounds = parseBounds(attrs.bounds);
|
|
307
|
+
if (bounds) fields.push({
|
|
308
|
+
text: String(attrs.text ?? ''),
|
|
309
|
+
bounds
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
for (const child of n.children ?? [])visit(child);
|
|
313
|
+
};
|
|
314
|
+
visit(layout);
|
|
315
|
+
return fields;
|
|
316
|
+
}
|
|
317
|
+
function pickFieldByPoint(fields, point) {
|
|
318
|
+
const [px, py] = point;
|
|
319
|
+
return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
|
|
320
|
+
}
|
|
321
|
+
function pickLongestField(fields) {
|
|
322
|
+
return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
|
|
323
|
+
}
|
|
268
324
|
const harmonyKeyCodeMap = {
|
|
269
325
|
Enter: '2054',
|
|
270
326
|
Backspace: '2055',
|
|
@@ -500,12 +556,13 @@ class HarmonyDevice {
|
|
|
500
556
|
if (shouldReplace) {
|
|
501
557
|
await hdc.click(x, y);
|
|
502
558
|
await (0, utils_namespaceObject.sleep)(100);
|
|
503
|
-
await
|
|
559
|
+
const length = await this.resolveClearLength(element);
|
|
560
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
504
561
|
await (0, utils_namespaceObject.sleep)(100);
|
|
505
562
|
}
|
|
506
563
|
await hdc.inputText(x, y, text);
|
|
507
|
-
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
|
|
508
|
-
if (shouldAutoDismissKeyboard) await this.hideKeyboard();
|
|
564
|
+
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
|
|
565
|
+
if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
|
|
509
566
|
}
|
|
510
567
|
async clearInput(element) {
|
|
511
568
|
const hdc = await this.getHdc();
|
|
@@ -513,7 +570,24 @@ class HarmonyDevice {
|
|
|
513
570
|
await hdc.click(element.center[0], element.center[1]);
|
|
514
571
|
await (0, utils_namespaceObject.sleep)(100);
|
|
515
572
|
}
|
|
516
|
-
await
|
|
573
|
+
const length = await this.resolveClearLength(element);
|
|
574
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
575
|
+
}
|
|
576
|
+
async resolveClearLength(element) {
|
|
577
|
+
const PADDING = 2;
|
|
578
|
+
const FALLBACK_LENGTH = 100;
|
|
579
|
+
try {
|
|
580
|
+
const hdc = await this.getHdc();
|
|
581
|
+
const layoutJson = await hdc.dumpLayout();
|
|
582
|
+
const layout = JSON.parse(layoutJson);
|
|
583
|
+
const fields = collectInputFields(layout);
|
|
584
|
+
if (0 === fields.length) return FALLBACK_LENGTH;
|
|
585
|
+
const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
|
|
586
|
+
return target.text.length + PADDING;
|
|
587
|
+
} catch (e) {
|
|
588
|
+
debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
|
|
589
|
+
return FALLBACK_LENGTH;
|
|
590
|
+
}
|
|
517
591
|
}
|
|
518
592
|
async pressKey(key) {
|
|
519
593
|
const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
|
|
@@ -676,9 +750,11 @@ class HarmonyDevice {
|
|
|
676
750
|
const hdc = await this.getHdc();
|
|
677
751
|
await hdc.keyEvent('RecentApps');
|
|
678
752
|
}
|
|
679
|
-
async hideKeyboard() {
|
|
753
|
+
async hideKeyboard(options) {
|
|
680
754
|
const hdc = await this.getHdc();
|
|
681
|
-
|
|
755
|
+
const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
|
|
756
|
+
const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
|
|
757
|
+
await hdc.keyEvent(key);
|
|
682
758
|
}
|
|
683
759
|
async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
|
|
684
760
|
const hdc = await this.getHdc();
|
|
@@ -727,9 +803,13 @@ class HarmonyDevice {
|
|
|
727
803
|
},
|
|
728
804
|
keyboard: {
|
|
729
805
|
keyboardPress: (keyName)=>this.pressKey(keyName),
|
|
730
|
-
typeText: (value, opts)=>
|
|
731
|
-
|
|
732
|
-
|
|
806
|
+
typeText: (value, opts)=>{
|
|
807
|
+
const harmonyOpts = opts;
|
|
808
|
+
return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
|
|
809
|
+
autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
|
|
810
|
+
keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
|
|
811
|
+
});
|
|
812
|
+
},
|
|
733
813
|
clearInput: (target)=>this.clearInput(target),
|
|
734
814
|
cursorMove: async (direction, times = 1)=>{
|
|
735
815
|
const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
|
package/dist/lib/mcp-server.js
CHANGED
|
@@ -168,6 +168,13 @@ class HdcClient {
|
|
|
168
168
|
async screenshot(remotePath) {
|
|
169
169
|
return await this.shell(`snapshot_display -f ${remotePath}`);
|
|
170
170
|
}
|
|
171
|
+
async dumpLayout() {
|
|
172
|
+
const remotePath = '/data/local/tmp/midscene_layout.json';
|
|
173
|
+
const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
|
|
174
|
+
const jsonStart = output.indexOf('{');
|
|
175
|
+
if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
|
|
176
|
+
return output.slice(jsonStart);
|
|
177
|
+
}
|
|
171
178
|
async click(x, y) {
|
|
172
179
|
await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
|
|
173
180
|
}
|
|
@@ -215,9 +222,17 @@ class HdcClient {
|
|
|
215
222
|
await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
|
|
216
223
|
}
|
|
217
224
|
async clearTextField(length = 100) {
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
225
|
+
if (length <= 0) return;
|
|
226
|
+
const MAX_KEYS_PER_CALL = 3;
|
|
227
|
+
const cmds = [];
|
|
228
|
+
let remaining = length;
|
|
229
|
+
while(remaining > 0){
|
|
230
|
+
const n = Math.min(MAX_KEYS_PER_CALL, remaining);
|
|
231
|
+
const codes = Array(n).fill('2055').join(' ');
|
|
232
|
+
cmds.push(`uitest uiInput keyEvent ${codes}`);
|
|
233
|
+
remaining -= n;
|
|
234
|
+
}
|
|
235
|
+
await this.shell(cmds.join(';'));
|
|
221
236
|
}
|
|
222
237
|
async startAbility(bundleName, abilityName) {
|
|
223
238
|
const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
|
|
@@ -298,6 +313,47 @@ const scrollQuadrantDivisions = 4;
|
|
|
298
313
|
const screenEdgeMargin = 50;
|
|
299
314
|
const debugDevice = (0, logger_namespaceObject.getDebug)('harmony:device');
|
|
300
315
|
let screenshotResizeScaleWarned = false;
|
|
316
|
+
const INPUT_FIELD_TYPES = new Set([
|
|
317
|
+
'TextInput',
|
|
318
|
+
'TextArea',
|
|
319
|
+
'SearchField'
|
|
320
|
+
]);
|
|
321
|
+
function parseBounds(raw) {
|
|
322
|
+
if ('string' != typeof raw) return null;
|
|
323
|
+
const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
|
|
324
|
+
if (!m) return null;
|
|
325
|
+
return {
|
|
326
|
+
x1: Number(m[1]),
|
|
327
|
+
y1: Number(m[2]),
|
|
328
|
+
x2: Number(m[3]),
|
|
329
|
+
y2: Number(m[4])
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
function collectInputFields(layout) {
|
|
333
|
+
const fields = [];
|
|
334
|
+
const visit = (node)=>{
|
|
335
|
+
if (!node || 'object' != typeof node) return;
|
|
336
|
+
const n = node;
|
|
337
|
+
const attrs = n.attributes ?? {};
|
|
338
|
+
if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
|
|
339
|
+
const bounds = parseBounds(attrs.bounds);
|
|
340
|
+
if (bounds) fields.push({
|
|
341
|
+
text: String(attrs.text ?? ''),
|
|
342
|
+
bounds
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
for (const child of n.children ?? [])visit(child);
|
|
346
|
+
};
|
|
347
|
+
visit(layout);
|
|
348
|
+
return fields;
|
|
349
|
+
}
|
|
350
|
+
function pickFieldByPoint(fields, point) {
|
|
351
|
+
const [px, py] = point;
|
|
352
|
+
return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
|
|
353
|
+
}
|
|
354
|
+
function pickLongestField(fields) {
|
|
355
|
+
return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
|
|
356
|
+
}
|
|
301
357
|
const harmonyKeyCodeMap = {
|
|
302
358
|
Enter: '2054',
|
|
303
359
|
Backspace: '2055',
|
|
@@ -533,12 +589,13 @@ class HarmonyDevice {
|
|
|
533
589
|
if (shouldReplace) {
|
|
534
590
|
await hdc.click(x, y);
|
|
535
591
|
await (0, core_utils_namespaceObject.sleep)(100);
|
|
536
|
-
await
|
|
592
|
+
const length = await this.resolveClearLength(element);
|
|
593
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
537
594
|
await (0, core_utils_namespaceObject.sleep)(100);
|
|
538
595
|
}
|
|
539
596
|
await hdc.inputText(x, y, text);
|
|
540
|
-
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
|
|
541
|
-
if (shouldAutoDismissKeyboard) await this.hideKeyboard();
|
|
597
|
+
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
|
|
598
|
+
if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
|
|
542
599
|
}
|
|
543
600
|
async clearInput(element) {
|
|
544
601
|
const hdc = await this.getHdc();
|
|
@@ -546,7 +603,24 @@ class HarmonyDevice {
|
|
|
546
603
|
await hdc.click(element.center[0], element.center[1]);
|
|
547
604
|
await (0, core_utils_namespaceObject.sleep)(100);
|
|
548
605
|
}
|
|
549
|
-
await
|
|
606
|
+
const length = await this.resolveClearLength(element);
|
|
607
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
608
|
+
}
|
|
609
|
+
async resolveClearLength(element) {
|
|
610
|
+
const PADDING = 2;
|
|
611
|
+
const FALLBACK_LENGTH = 100;
|
|
612
|
+
try {
|
|
613
|
+
const hdc = await this.getHdc();
|
|
614
|
+
const layoutJson = await hdc.dumpLayout();
|
|
615
|
+
const layout = JSON.parse(layoutJson);
|
|
616
|
+
const fields = collectInputFields(layout);
|
|
617
|
+
if (0 === fields.length) return FALLBACK_LENGTH;
|
|
618
|
+
const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
|
|
619
|
+
return target.text.length + PADDING;
|
|
620
|
+
} catch (e) {
|
|
621
|
+
debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
|
|
622
|
+
return FALLBACK_LENGTH;
|
|
623
|
+
}
|
|
550
624
|
}
|
|
551
625
|
async pressKey(key) {
|
|
552
626
|
const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
|
|
@@ -709,9 +783,11 @@ class HarmonyDevice {
|
|
|
709
783
|
const hdc = await this.getHdc();
|
|
710
784
|
await hdc.keyEvent('RecentApps');
|
|
711
785
|
}
|
|
712
|
-
async hideKeyboard() {
|
|
786
|
+
async hideKeyboard(options) {
|
|
713
787
|
const hdc = await this.getHdc();
|
|
714
|
-
|
|
788
|
+
const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
|
|
789
|
+
const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
|
|
790
|
+
await hdc.keyEvent(key);
|
|
715
791
|
}
|
|
716
792
|
async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
|
|
717
793
|
const hdc = await this.getHdc();
|
|
@@ -760,9 +836,13 @@ class HarmonyDevice {
|
|
|
760
836
|
},
|
|
761
837
|
keyboard: {
|
|
762
838
|
keyboardPress: (keyName)=>this.pressKey(keyName),
|
|
763
|
-
typeText: (value, opts)=>
|
|
764
|
-
|
|
765
|
-
|
|
839
|
+
typeText: (value, opts)=>{
|
|
840
|
+
const harmonyOpts = opts;
|
|
841
|
+
return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
|
|
842
|
+
autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
|
|
843
|
+
keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
|
|
844
|
+
});
|
|
845
|
+
},
|
|
766
846
|
clearInput: (target)=>this.clearInput(target),
|
|
767
847
|
cursorMove: async (direction, times = 1)=>{
|
|
768
848
|
const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
|
|
@@ -1033,7 +1113,7 @@ class HarmonyMCPServer extends mcp_namespaceObject.BaseMCPServer {
|
|
|
1033
1113
|
constructor(toolsManager){
|
|
1034
1114
|
super({
|
|
1035
1115
|
name: '@midscene/harmony-mcp',
|
|
1036
|
-
version: "1.8.
|
|
1116
|
+
version: "1.8.8-beta-20260601092817.0",
|
|
1037
1117
|
description: 'Control the HarmonyOS device using natural language commands'
|
|
1038
1118
|
}, toolsManager);
|
|
1039
1119
|
}
|