@midscene/harmony 1.8.7 → 1.8.8-beta-20260601092817.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/bin.mjs +92 -12
- package/dist/es/cli.mjs +93 -13
- package/dist/es/index.mjs +92 -12
- package/dist/es/mcp-server.mjs +93 -13
- package/dist/lib/bin.js +92 -12
- package/dist/lib/cli.js +93 -13
- package/dist/lib/index.js +92 -12
- package/dist/lib/mcp-server.js +93 -13
- package/dist/types/index.d.ts +32 -5
- package/dist/types/mcp-server.d.ts +32 -5
- package/package.json +4 -4
- package/static/index.html +1 -1
- package/static/static/js/{index.6becfe23.js → index.122f2630.js} +29 -29
- package/static/static/js/index.122f2630.js.map +1 -0
- package/static/static/js/index.6becfe23.js.map +0 -1
- /package/static/static/js/{index.6becfe23.js.LICENSE.txt → index.122f2630.js.LICENSE.txt} +0 -0
package/dist/es/bin.mjs
CHANGED
|
@@ -130,6 +130,13 @@ class HdcClient {
|
|
|
130
130
|
async screenshot(remotePath) {
|
|
131
131
|
return await this.shell(`snapshot_display -f ${remotePath}`);
|
|
132
132
|
}
|
|
133
|
+
async dumpLayout() {
|
|
134
|
+
const remotePath = '/data/local/tmp/midscene_layout.json';
|
|
135
|
+
const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
|
|
136
|
+
const jsonStart = output.indexOf('{');
|
|
137
|
+
if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
|
|
138
|
+
return output.slice(jsonStart);
|
|
139
|
+
}
|
|
133
140
|
async click(x, y) {
|
|
134
141
|
await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
|
|
135
142
|
}
|
|
@@ -177,9 +184,17 @@ class HdcClient {
|
|
|
177
184
|
await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
|
|
178
185
|
}
|
|
179
186
|
async clearTextField(length = 100) {
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
187
|
+
if (length <= 0) return;
|
|
188
|
+
const MAX_KEYS_PER_CALL = 3;
|
|
189
|
+
const cmds = [];
|
|
190
|
+
let remaining = length;
|
|
191
|
+
while(remaining > 0){
|
|
192
|
+
const n = Math.min(MAX_KEYS_PER_CALL, remaining);
|
|
193
|
+
const codes = Array(n).fill('2055').join(' ');
|
|
194
|
+
cmds.push(`uitest uiInput keyEvent ${codes}`);
|
|
195
|
+
remaining -= n;
|
|
196
|
+
}
|
|
197
|
+
await this.shell(cmds.join(';'));
|
|
183
198
|
}
|
|
184
199
|
async startAbility(bundleName, abilityName) {
|
|
185
200
|
const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
|
|
@@ -260,6 +275,47 @@ const scrollQuadrantDivisions = 4;
|
|
|
260
275
|
const screenEdgeMargin = 50;
|
|
261
276
|
const debugDevice = getDebug('harmony:device');
|
|
262
277
|
let screenshotResizeScaleWarned = false;
|
|
278
|
+
const INPUT_FIELD_TYPES = new Set([
|
|
279
|
+
'TextInput',
|
|
280
|
+
'TextArea',
|
|
281
|
+
'SearchField'
|
|
282
|
+
]);
|
|
283
|
+
function parseBounds(raw) {
|
|
284
|
+
if ('string' != typeof raw) return null;
|
|
285
|
+
const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
|
|
286
|
+
if (!m) return null;
|
|
287
|
+
return {
|
|
288
|
+
x1: Number(m[1]),
|
|
289
|
+
y1: Number(m[2]),
|
|
290
|
+
x2: Number(m[3]),
|
|
291
|
+
y2: Number(m[4])
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
function collectInputFields(layout) {
|
|
295
|
+
const fields = [];
|
|
296
|
+
const visit = (node)=>{
|
|
297
|
+
if (!node || 'object' != typeof node) return;
|
|
298
|
+
const n = node;
|
|
299
|
+
const attrs = n.attributes ?? {};
|
|
300
|
+
if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
|
|
301
|
+
const bounds = parseBounds(attrs.bounds);
|
|
302
|
+
if (bounds) fields.push({
|
|
303
|
+
text: String(attrs.text ?? ''),
|
|
304
|
+
bounds
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
for (const child of n.children ?? [])visit(child);
|
|
308
|
+
};
|
|
309
|
+
visit(layout);
|
|
310
|
+
return fields;
|
|
311
|
+
}
|
|
312
|
+
function pickFieldByPoint(fields, point) {
|
|
313
|
+
const [px, py] = point;
|
|
314
|
+
return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
|
|
315
|
+
}
|
|
316
|
+
function pickLongestField(fields) {
|
|
317
|
+
return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
|
|
318
|
+
}
|
|
263
319
|
const harmonyKeyCodeMap = {
|
|
264
320
|
Enter: '2054',
|
|
265
321
|
Backspace: '2055',
|
|
@@ -495,12 +551,13 @@ class device_HarmonyDevice {
|
|
|
495
551
|
if (shouldReplace) {
|
|
496
552
|
await hdc.click(x, y);
|
|
497
553
|
await sleep(100);
|
|
498
|
-
await
|
|
554
|
+
const length = await this.resolveClearLength(element);
|
|
555
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
499
556
|
await sleep(100);
|
|
500
557
|
}
|
|
501
558
|
await hdc.inputText(x, y, text);
|
|
502
|
-
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
|
|
503
|
-
if (shouldAutoDismissKeyboard) await this.hideKeyboard();
|
|
559
|
+
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
|
|
560
|
+
if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
|
|
504
561
|
}
|
|
505
562
|
async clearInput(element) {
|
|
506
563
|
const hdc = await this.getHdc();
|
|
@@ -508,7 +565,24 @@ class device_HarmonyDevice {
|
|
|
508
565
|
await hdc.click(element.center[0], element.center[1]);
|
|
509
566
|
await sleep(100);
|
|
510
567
|
}
|
|
511
|
-
await
|
|
568
|
+
const length = await this.resolveClearLength(element);
|
|
569
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
570
|
+
}
|
|
571
|
+
async resolveClearLength(element) {
|
|
572
|
+
const PADDING = 2;
|
|
573
|
+
const FALLBACK_LENGTH = 100;
|
|
574
|
+
try {
|
|
575
|
+
const hdc = await this.getHdc();
|
|
576
|
+
const layoutJson = await hdc.dumpLayout();
|
|
577
|
+
const layout = JSON.parse(layoutJson);
|
|
578
|
+
const fields = collectInputFields(layout);
|
|
579
|
+
if (0 === fields.length) return FALLBACK_LENGTH;
|
|
580
|
+
const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
|
|
581
|
+
return target.text.length + PADDING;
|
|
582
|
+
} catch (e) {
|
|
583
|
+
debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
|
|
584
|
+
return FALLBACK_LENGTH;
|
|
585
|
+
}
|
|
512
586
|
}
|
|
513
587
|
async pressKey(key) {
|
|
514
588
|
const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
|
|
@@ -671,9 +745,11 @@ class device_HarmonyDevice {
|
|
|
671
745
|
const hdc = await this.getHdc();
|
|
672
746
|
await hdc.keyEvent('RecentApps');
|
|
673
747
|
}
|
|
674
|
-
async hideKeyboard() {
|
|
748
|
+
async hideKeyboard(options) {
|
|
675
749
|
const hdc = await this.getHdc();
|
|
676
|
-
|
|
750
|
+
const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
|
|
751
|
+
const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
|
|
752
|
+
await hdc.keyEvent(key);
|
|
677
753
|
}
|
|
678
754
|
async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
|
|
679
755
|
const hdc = await this.getHdc();
|
|
@@ -722,9 +798,13 @@ class device_HarmonyDevice {
|
|
|
722
798
|
},
|
|
723
799
|
keyboard: {
|
|
724
800
|
keyboardPress: (keyName)=>this.pressKey(keyName),
|
|
725
|
-
typeText: (value, opts)=>
|
|
726
|
-
|
|
727
|
-
|
|
801
|
+
typeText: (value, opts)=>{
|
|
802
|
+
const harmonyOpts = opts;
|
|
803
|
+
return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
|
|
804
|
+
autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
|
|
805
|
+
keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
|
|
806
|
+
});
|
|
807
|
+
},
|
|
728
808
|
clearInput: (target)=>this.clearInput(target),
|
|
729
809
|
cursorMove: async (direction, times = 1)=>{
|
|
730
810
|
const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
|
package/dist/es/cli.mjs
CHANGED
|
@@ -127,6 +127,13 @@ class HdcClient {
|
|
|
127
127
|
async screenshot(remotePath) {
|
|
128
128
|
return await this.shell(`snapshot_display -f ${remotePath}`);
|
|
129
129
|
}
|
|
130
|
+
async dumpLayout() {
|
|
131
|
+
const remotePath = '/data/local/tmp/midscene_layout.json';
|
|
132
|
+
const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
|
|
133
|
+
const jsonStart = output.indexOf('{');
|
|
134
|
+
if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
|
|
135
|
+
return output.slice(jsonStart);
|
|
136
|
+
}
|
|
130
137
|
async click(x, y) {
|
|
131
138
|
await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
|
|
132
139
|
}
|
|
@@ -174,9 +181,17 @@ class HdcClient {
|
|
|
174
181
|
await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
|
|
175
182
|
}
|
|
176
183
|
async clearTextField(length = 100) {
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
184
|
+
if (length <= 0) return;
|
|
185
|
+
const MAX_KEYS_PER_CALL = 3;
|
|
186
|
+
const cmds = [];
|
|
187
|
+
let remaining = length;
|
|
188
|
+
while(remaining > 0){
|
|
189
|
+
const n = Math.min(MAX_KEYS_PER_CALL, remaining);
|
|
190
|
+
const codes = Array(n).fill('2055').join(' ');
|
|
191
|
+
cmds.push(`uitest uiInput keyEvent ${codes}`);
|
|
192
|
+
remaining -= n;
|
|
193
|
+
}
|
|
194
|
+
await this.shell(cmds.join(';'));
|
|
180
195
|
}
|
|
181
196
|
async startAbility(bundleName, abilityName) {
|
|
182
197
|
const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
|
|
@@ -257,6 +272,47 @@ const scrollQuadrantDivisions = 4;
|
|
|
257
272
|
const screenEdgeMargin = 50;
|
|
258
273
|
const debugDevice = getDebug('harmony:device');
|
|
259
274
|
let screenshotResizeScaleWarned = false;
|
|
275
|
+
const INPUT_FIELD_TYPES = new Set([
|
|
276
|
+
'TextInput',
|
|
277
|
+
'TextArea',
|
|
278
|
+
'SearchField'
|
|
279
|
+
]);
|
|
280
|
+
function parseBounds(raw) {
|
|
281
|
+
if ('string' != typeof raw) return null;
|
|
282
|
+
const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
|
|
283
|
+
if (!m) return null;
|
|
284
|
+
return {
|
|
285
|
+
x1: Number(m[1]),
|
|
286
|
+
y1: Number(m[2]),
|
|
287
|
+
x2: Number(m[3]),
|
|
288
|
+
y2: Number(m[4])
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
function collectInputFields(layout) {
|
|
292
|
+
const fields = [];
|
|
293
|
+
const visit = (node)=>{
|
|
294
|
+
if (!node || 'object' != typeof node) return;
|
|
295
|
+
const n = node;
|
|
296
|
+
const attrs = n.attributes ?? {};
|
|
297
|
+
if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
|
|
298
|
+
const bounds = parseBounds(attrs.bounds);
|
|
299
|
+
if (bounds) fields.push({
|
|
300
|
+
text: String(attrs.text ?? ''),
|
|
301
|
+
bounds
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
for (const child of n.children ?? [])visit(child);
|
|
305
|
+
};
|
|
306
|
+
visit(layout);
|
|
307
|
+
return fields;
|
|
308
|
+
}
|
|
309
|
+
function pickFieldByPoint(fields, point) {
|
|
310
|
+
const [px, py] = point;
|
|
311
|
+
return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
|
|
312
|
+
}
|
|
313
|
+
function pickLongestField(fields) {
|
|
314
|
+
return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
|
|
315
|
+
}
|
|
260
316
|
const harmonyKeyCodeMap = {
|
|
261
317
|
Enter: '2054',
|
|
262
318
|
Backspace: '2055',
|
|
@@ -492,12 +548,13 @@ class HarmonyDevice {
|
|
|
492
548
|
if (shouldReplace) {
|
|
493
549
|
await hdc.click(x, y);
|
|
494
550
|
await sleep(100);
|
|
495
|
-
await
|
|
551
|
+
const length = await this.resolveClearLength(element);
|
|
552
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
496
553
|
await sleep(100);
|
|
497
554
|
}
|
|
498
555
|
await hdc.inputText(x, y, text);
|
|
499
|
-
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
|
|
500
|
-
if (shouldAutoDismissKeyboard) await this.hideKeyboard();
|
|
556
|
+
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
|
|
557
|
+
if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
|
|
501
558
|
}
|
|
502
559
|
async clearInput(element) {
|
|
503
560
|
const hdc = await this.getHdc();
|
|
@@ -505,7 +562,24 @@ class HarmonyDevice {
|
|
|
505
562
|
await hdc.click(element.center[0], element.center[1]);
|
|
506
563
|
await sleep(100);
|
|
507
564
|
}
|
|
508
|
-
await
|
|
565
|
+
const length = await this.resolveClearLength(element);
|
|
566
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
567
|
+
}
|
|
568
|
+
async resolveClearLength(element) {
|
|
569
|
+
const PADDING = 2;
|
|
570
|
+
const FALLBACK_LENGTH = 100;
|
|
571
|
+
try {
|
|
572
|
+
const hdc = await this.getHdc();
|
|
573
|
+
const layoutJson = await hdc.dumpLayout();
|
|
574
|
+
const layout = JSON.parse(layoutJson);
|
|
575
|
+
const fields = collectInputFields(layout);
|
|
576
|
+
if (0 === fields.length) return FALLBACK_LENGTH;
|
|
577
|
+
const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
|
|
578
|
+
return target.text.length + PADDING;
|
|
579
|
+
} catch (e) {
|
|
580
|
+
debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
|
|
581
|
+
return FALLBACK_LENGTH;
|
|
582
|
+
}
|
|
509
583
|
}
|
|
510
584
|
async pressKey(key) {
|
|
511
585
|
const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
|
|
@@ -668,9 +742,11 @@ class HarmonyDevice {
|
|
|
668
742
|
const hdc = await this.getHdc();
|
|
669
743
|
await hdc.keyEvent('RecentApps');
|
|
670
744
|
}
|
|
671
|
-
async hideKeyboard() {
|
|
745
|
+
async hideKeyboard(options) {
|
|
672
746
|
const hdc = await this.getHdc();
|
|
673
|
-
|
|
747
|
+
const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
|
|
748
|
+
const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
|
|
749
|
+
await hdc.keyEvent(key);
|
|
674
750
|
}
|
|
675
751
|
async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
|
|
676
752
|
const hdc = await this.getHdc();
|
|
@@ -719,9 +795,13 @@ class HarmonyDevice {
|
|
|
719
795
|
},
|
|
720
796
|
keyboard: {
|
|
721
797
|
keyboardPress: (keyName)=>this.pressKey(keyName),
|
|
722
|
-
typeText: (value, opts)=>
|
|
723
|
-
|
|
724
|
-
|
|
798
|
+
typeText: (value, opts)=>{
|
|
799
|
+
const harmonyOpts = opts;
|
|
800
|
+
return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
|
|
801
|
+
autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
|
|
802
|
+
keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
|
|
803
|
+
});
|
|
804
|
+
},
|
|
725
805
|
clearInput: (target)=>this.clearInput(target),
|
|
726
806
|
cursorMove: async (direction, times = 1)=>{
|
|
727
807
|
const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
|
|
@@ -988,7 +1068,7 @@ class HarmonyMidsceneTools extends BaseMidsceneTools {
|
|
|
988
1068
|
const tools = new HarmonyMidsceneTools();
|
|
989
1069
|
runToolsCLI(tools, 'midscene-harmony', {
|
|
990
1070
|
stripPrefix: 'harmony_',
|
|
991
|
-
version: "1.8.
|
|
1071
|
+
version: "1.8.8-beta-20260601092817.0",
|
|
992
1072
|
extraCommands: createReportCliCommands()
|
|
993
1073
|
}).catch((e)=>{
|
|
994
1074
|
process.exit(reportCLIError(e));
|
package/dist/es/index.mjs
CHANGED
|
@@ -98,6 +98,13 @@ class HdcClient {
|
|
|
98
98
|
async screenshot(remotePath) {
|
|
99
99
|
return await this.shell(`snapshot_display -f ${remotePath}`);
|
|
100
100
|
}
|
|
101
|
+
async dumpLayout() {
|
|
102
|
+
const remotePath = '/data/local/tmp/midscene_layout.json';
|
|
103
|
+
const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
|
|
104
|
+
const jsonStart = output.indexOf('{');
|
|
105
|
+
if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
|
|
106
|
+
return output.slice(jsonStart);
|
|
107
|
+
}
|
|
101
108
|
async click(x, y) {
|
|
102
109
|
await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
|
|
103
110
|
}
|
|
@@ -145,9 +152,17 @@ class HdcClient {
|
|
|
145
152
|
await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
|
|
146
153
|
}
|
|
147
154
|
async clearTextField(length = 100) {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
155
|
+
if (length <= 0) return;
|
|
156
|
+
const MAX_KEYS_PER_CALL = 3;
|
|
157
|
+
const cmds = [];
|
|
158
|
+
let remaining = length;
|
|
159
|
+
while(remaining > 0){
|
|
160
|
+
const n = Math.min(MAX_KEYS_PER_CALL, remaining);
|
|
161
|
+
const codes = Array(n).fill('2055').join(' ');
|
|
162
|
+
cmds.push(`uitest uiInput keyEvent ${codes}`);
|
|
163
|
+
remaining -= n;
|
|
164
|
+
}
|
|
165
|
+
await this.shell(cmds.join(';'));
|
|
151
166
|
}
|
|
152
167
|
async startAbility(bundleName, abilityName) {
|
|
153
168
|
const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
|
|
@@ -228,6 +243,47 @@ const scrollQuadrantDivisions = 4;
|
|
|
228
243
|
const screenEdgeMargin = 50;
|
|
229
244
|
const debugDevice = getDebug('harmony:device');
|
|
230
245
|
let screenshotResizeScaleWarned = false;
|
|
246
|
+
const INPUT_FIELD_TYPES = new Set([
|
|
247
|
+
'TextInput',
|
|
248
|
+
'TextArea',
|
|
249
|
+
'SearchField'
|
|
250
|
+
]);
|
|
251
|
+
function parseBounds(raw) {
|
|
252
|
+
if ('string' != typeof raw) return null;
|
|
253
|
+
const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
|
|
254
|
+
if (!m) return null;
|
|
255
|
+
return {
|
|
256
|
+
x1: Number(m[1]),
|
|
257
|
+
y1: Number(m[2]),
|
|
258
|
+
x2: Number(m[3]),
|
|
259
|
+
y2: Number(m[4])
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
function collectInputFields(layout) {
|
|
263
|
+
const fields = [];
|
|
264
|
+
const visit = (node)=>{
|
|
265
|
+
if (!node || 'object' != typeof node) return;
|
|
266
|
+
const n = node;
|
|
267
|
+
const attrs = n.attributes ?? {};
|
|
268
|
+
if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
|
|
269
|
+
const bounds = parseBounds(attrs.bounds);
|
|
270
|
+
if (bounds) fields.push({
|
|
271
|
+
text: String(attrs.text ?? ''),
|
|
272
|
+
bounds
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
for (const child of n.children ?? [])visit(child);
|
|
276
|
+
};
|
|
277
|
+
visit(layout);
|
|
278
|
+
return fields;
|
|
279
|
+
}
|
|
280
|
+
function pickFieldByPoint(fields, point) {
|
|
281
|
+
const [px, py] = point;
|
|
282
|
+
return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
|
|
283
|
+
}
|
|
284
|
+
function pickLongestField(fields) {
|
|
285
|
+
return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
|
|
286
|
+
}
|
|
231
287
|
const harmonyKeyCodeMap = {
|
|
232
288
|
Enter: '2054',
|
|
233
289
|
Backspace: '2055',
|
|
@@ -463,12 +519,13 @@ class HarmonyDevice {
|
|
|
463
519
|
if (shouldReplace) {
|
|
464
520
|
await hdc.click(x, y);
|
|
465
521
|
await sleep(100);
|
|
466
|
-
await
|
|
522
|
+
const length = await this.resolveClearLength(element);
|
|
523
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
467
524
|
await sleep(100);
|
|
468
525
|
}
|
|
469
526
|
await hdc.inputText(x, y, text);
|
|
470
|
-
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
|
|
471
|
-
if (shouldAutoDismissKeyboard) await this.hideKeyboard();
|
|
527
|
+
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
|
|
528
|
+
if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
|
|
472
529
|
}
|
|
473
530
|
async clearInput(element) {
|
|
474
531
|
const hdc = await this.getHdc();
|
|
@@ -476,7 +533,24 @@ class HarmonyDevice {
|
|
|
476
533
|
await hdc.click(element.center[0], element.center[1]);
|
|
477
534
|
await sleep(100);
|
|
478
535
|
}
|
|
479
|
-
await
|
|
536
|
+
const length = await this.resolveClearLength(element);
|
|
537
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
538
|
+
}
|
|
539
|
+
async resolveClearLength(element) {
|
|
540
|
+
const PADDING = 2;
|
|
541
|
+
const FALLBACK_LENGTH = 100;
|
|
542
|
+
try {
|
|
543
|
+
const hdc = await this.getHdc();
|
|
544
|
+
const layoutJson = await hdc.dumpLayout();
|
|
545
|
+
const layout = JSON.parse(layoutJson);
|
|
546
|
+
const fields = collectInputFields(layout);
|
|
547
|
+
if (0 === fields.length) return FALLBACK_LENGTH;
|
|
548
|
+
const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
|
|
549
|
+
return target.text.length + PADDING;
|
|
550
|
+
} catch (e) {
|
|
551
|
+
debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
|
|
552
|
+
return FALLBACK_LENGTH;
|
|
553
|
+
}
|
|
480
554
|
}
|
|
481
555
|
async pressKey(key) {
|
|
482
556
|
const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
|
|
@@ -639,9 +713,11 @@ class HarmonyDevice {
|
|
|
639
713
|
const hdc = await this.getHdc();
|
|
640
714
|
await hdc.keyEvent('RecentApps');
|
|
641
715
|
}
|
|
642
|
-
async hideKeyboard() {
|
|
716
|
+
async hideKeyboard(options) {
|
|
643
717
|
const hdc = await this.getHdc();
|
|
644
|
-
|
|
718
|
+
const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
|
|
719
|
+
const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
|
|
720
|
+
await hdc.keyEvent(key);
|
|
645
721
|
}
|
|
646
722
|
async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
|
|
647
723
|
const hdc = await this.getHdc();
|
|
@@ -690,9 +766,13 @@ class HarmonyDevice {
|
|
|
690
766
|
},
|
|
691
767
|
keyboard: {
|
|
692
768
|
keyboardPress: (keyName)=>this.pressKey(keyName),
|
|
693
|
-
typeText: (value, opts)=>
|
|
694
|
-
|
|
695
|
-
|
|
769
|
+
typeText: (value, opts)=>{
|
|
770
|
+
const harmonyOpts = opts;
|
|
771
|
+
return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
|
|
772
|
+
autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
|
|
773
|
+
keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
|
|
774
|
+
});
|
|
775
|
+
},
|
|
696
776
|
clearInput: (target)=>this.clearInput(target),
|
|
697
777
|
cursorMove: async (direction, times = 1)=>{
|
|
698
778
|
const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
|
package/dist/es/mcp-server.mjs
CHANGED
|
@@ -127,6 +127,13 @@ class HdcClient {
|
|
|
127
127
|
async screenshot(remotePath) {
|
|
128
128
|
return await this.shell(`snapshot_display -f ${remotePath}`);
|
|
129
129
|
}
|
|
130
|
+
async dumpLayout() {
|
|
131
|
+
const remotePath = '/data/local/tmp/midscene_layout.json';
|
|
132
|
+
const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
|
|
133
|
+
const jsonStart = output.indexOf('{');
|
|
134
|
+
if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
|
|
135
|
+
return output.slice(jsonStart);
|
|
136
|
+
}
|
|
130
137
|
async click(x, y) {
|
|
131
138
|
await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
|
|
132
139
|
}
|
|
@@ -174,9 +181,17 @@ class HdcClient {
|
|
|
174
181
|
await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
|
|
175
182
|
}
|
|
176
183
|
async clearTextField(length = 100) {
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
184
|
+
if (length <= 0) return;
|
|
185
|
+
const MAX_KEYS_PER_CALL = 3;
|
|
186
|
+
const cmds = [];
|
|
187
|
+
let remaining = length;
|
|
188
|
+
while(remaining > 0){
|
|
189
|
+
const n = Math.min(MAX_KEYS_PER_CALL, remaining);
|
|
190
|
+
const codes = Array(n).fill('2055').join(' ');
|
|
191
|
+
cmds.push(`uitest uiInput keyEvent ${codes}`);
|
|
192
|
+
remaining -= n;
|
|
193
|
+
}
|
|
194
|
+
await this.shell(cmds.join(';'));
|
|
180
195
|
}
|
|
181
196
|
async startAbility(bundleName, abilityName) {
|
|
182
197
|
const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
|
|
@@ -257,6 +272,47 @@ const scrollQuadrantDivisions = 4;
|
|
|
257
272
|
const screenEdgeMargin = 50;
|
|
258
273
|
const debugDevice = getDebug('harmony:device');
|
|
259
274
|
let screenshotResizeScaleWarned = false;
|
|
275
|
+
const INPUT_FIELD_TYPES = new Set([
|
|
276
|
+
'TextInput',
|
|
277
|
+
'TextArea',
|
|
278
|
+
'SearchField'
|
|
279
|
+
]);
|
|
280
|
+
function parseBounds(raw) {
|
|
281
|
+
if ('string' != typeof raw) return null;
|
|
282
|
+
const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
|
|
283
|
+
if (!m) return null;
|
|
284
|
+
return {
|
|
285
|
+
x1: Number(m[1]),
|
|
286
|
+
y1: Number(m[2]),
|
|
287
|
+
x2: Number(m[3]),
|
|
288
|
+
y2: Number(m[4])
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
function collectInputFields(layout) {
|
|
292
|
+
const fields = [];
|
|
293
|
+
const visit = (node)=>{
|
|
294
|
+
if (!node || 'object' != typeof node) return;
|
|
295
|
+
const n = node;
|
|
296
|
+
const attrs = n.attributes ?? {};
|
|
297
|
+
if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
|
|
298
|
+
const bounds = parseBounds(attrs.bounds);
|
|
299
|
+
if (bounds) fields.push({
|
|
300
|
+
text: String(attrs.text ?? ''),
|
|
301
|
+
bounds
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
for (const child of n.children ?? [])visit(child);
|
|
305
|
+
};
|
|
306
|
+
visit(layout);
|
|
307
|
+
return fields;
|
|
308
|
+
}
|
|
309
|
+
function pickFieldByPoint(fields, point) {
|
|
310
|
+
const [px, py] = point;
|
|
311
|
+
return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
|
|
312
|
+
}
|
|
313
|
+
function pickLongestField(fields) {
|
|
314
|
+
return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
|
|
315
|
+
}
|
|
260
316
|
const harmonyKeyCodeMap = {
|
|
261
317
|
Enter: '2054',
|
|
262
318
|
Backspace: '2055',
|
|
@@ -492,12 +548,13 @@ class HarmonyDevice {
|
|
|
492
548
|
if (shouldReplace) {
|
|
493
549
|
await hdc.click(x, y);
|
|
494
550
|
await sleep(100);
|
|
495
|
-
await
|
|
551
|
+
const length = await this.resolveClearLength(element);
|
|
552
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
496
553
|
await sleep(100);
|
|
497
554
|
}
|
|
498
555
|
await hdc.inputText(x, y, text);
|
|
499
|
-
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
|
|
500
|
-
if (shouldAutoDismissKeyboard) await this.hideKeyboard();
|
|
556
|
+
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
|
|
557
|
+
if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
|
|
501
558
|
}
|
|
502
559
|
async clearInput(element) {
|
|
503
560
|
const hdc = await this.getHdc();
|
|
@@ -505,7 +562,24 @@ class HarmonyDevice {
|
|
|
505
562
|
await hdc.click(element.center[0], element.center[1]);
|
|
506
563
|
await sleep(100);
|
|
507
564
|
}
|
|
508
|
-
await
|
|
565
|
+
const length = await this.resolveClearLength(element);
|
|
566
|
+
if (length > 0) await hdc.clearTextField(length);
|
|
567
|
+
}
|
|
568
|
+
async resolveClearLength(element) {
|
|
569
|
+
const PADDING = 2;
|
|
570
|
+
const FALLBACK_LENGTH = 100;
|
|
571
|
+
try {
|
|
572
|
+
const hdc = await this.getHdc();
|
|
573
|
+
const layoutJson = await hdc.dumpLayout();
|
|
574
|
+
const layout = JSON.parse(layoutJson);
|
|
575
|
+
const fields = collectInputFields(layout);
|
|
576
|
+
if (0 === fields.length) return FALLBACK_LENGTH;
|
|
577
|
+
const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
|
|
578
|
+
return target.text.length + PADDING;
|
|
579
|
+
} catch (e) {
|
|
580
|
+
debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
|
|
581
|
+
return FALLBACK_LENGTH;
|
|
582
|
+
}
|
|
509
583
|
}
|
|
510
584
|
async pressKey(key) {
|
|
511
585
|
const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
|
|
@@ -668,9 +742,11 @@ class HarmonyDevice {
|
|
|
668
742
|
const hdc = await this.getHdc();
|
|
669
743
|
await hdc.keyEvent('RecentApps');
|
|
670
744
|
}
|
|
671
|
-
async hideKeyboard() {
|
|
745
|
+
async hideKeyboard(options) {
|
|
672
746
|
const hdc = await this.getHdc();
|
|
673
|
-
|
|
747
|
+
const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
|
|
748
|
+
const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
|
|
749
|
+
await hdc.keyEvent(key);
|
|
674
750
|
}
|
|
675
751
|
async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
|
|
676
752
|
const hdc = await this.getHdc();
|
|
@@ -719,9 +795,13 @@ class HarmonyDevice {
|
|
|
719
795
|
},
|
|
720
796
|
keyboard: {
|
|
721
797
|
keyboardPress: (keyName)=>this.pressKey(keyName),
|
|
722
|
-
typeText: (value, opts)=>
|
|
723
|
-
|
|
724
|
-
|
|
798
|
+
typeText: (value, opts)=>{
|
|
799
|
+
const harmonyOpts = opts;
|
|
800
|
+
return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
|
|
801
|
+
autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
|
|
802
|
+
keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
|
|
803
|
+
});
|
|
804
|
+
},
|
|
725
805
|
clearInput: (target)=>this.clearInput(target),
|
|
726
806
|
cursorMove: async (direction, times = 1)=>{
|
|
727
807
|
const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
|
|
@@ -992,7 +1072,7 @@ class HarmonyMCPServer extends BaseMCPServer {
|
|
|
992
1072
|
constructor(toolsManager){
|
|
993
1073
|
super({
|
|
994
1074
|
name: '@midscene/harmony-mcp',
|
|
995
|
-
version: "1.8.
|
|
1075
|
+
version: "1.8.8-beta-20260601092817.0",
|
|
996
1076
|
description: 'Control the HarmonyOS device using natural language commands'
|
|
997
1077
|
}, toolsManager);
|
|
998
1078
|
}
|