@midscene/harmony 1.8.7 → 1.8.8-beta-20260601092605.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/lib/bin.js CHANGED
@@ -156,6 +156,13 @@ class HdcClient {
156
156
  async screenshot(remotePath) {
157
157
  return await this.shell(`snapshot_display -f ${remotePath}`);
158
158
  }
159
+ async dumpLayout() {
160
+ const remotePath = '/data/local/tmp/midscene_layout.json';
161
+ const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
162
+ const jsonStart = output.indexOf('{');
163
+ if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
164
+ return output.slice(jsonStart);
165
+ }
159
166
  async click(x, y) {
160
167
  await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
161
168
  }
@@ -203,9 +210,17 @@ class HdcClient {
203
210
  await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
204
211
  }
205
212
  async clearTextField(length = 100) {
206
- const keys = [];
207
- for(let i = 0; i < length; i++)keys.push('2055', '2071');
208
- await this.keyEvent(...keys);
213
+ if (length <= 0) return;
214
+ const MAX_KEYS_PER_CALL = 3;
215
+ const cmds = [];
216
+ let remaining = length;
217
+ while(remaining > 0){
218
+ const n = Math.min(MAX_KEYS_PER_CALL, remaining);
219
+ const codes = Array(n).fill('2055').join(' ');
220
+ cmds.push(`uitest uiInput keyEvent ${codes}`);
221
+ remaining -= n;
222
+ }
223
+ await this.shell(cmds.join(';'));
209
224
  }
210
225
  async startAbility(bundleName, abilityName) {
211
226
  const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
@@ -286,6 +301,47 @@ const scrollQuadrantDivisions = 4;
286
301
  const screenEdgeMargin = 50;
287
302
  const debugDevice = (0, logger_namespaceObject.getDebug)('harmony:device');
288
303
  let screenshotResizeScaleWarned = false;
304
+ const INPUT_FIELD_TYPES = new Set([
305
+ 'TextInput',
306
+ 'TextArea',
307
+ 'SearchField'
308
+ ]);
309
+ function parseBounds(raw) {
310
+ if ('string' != typeof raw) return null;
311
+ const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
312
+ if (!m) return null;
313
+ return {
314
+ x1: Number(m[1]),
315
+ y1: Number(m[2]),
316
+ x2: Number(m[3]),
317
+ y2: Number(m[4])
318
+ };
319
+ }
320
+ function collectInputFields(layout) {
321
+ const fields = [];
322
+ const visit = (node)=>{
323
+ if (!node || 'object' != typeof node) return;
324
+ const n = node;
325
+ const attrs = n.attributes ?? {};
326
+ if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
327
+ const bounds = parseBounds(attrs.bounds);
328
+ if (bounds) fields.push({
329
+ text: String(attrs.text ?? ''),
330
+ bounds
331
+ });
332
+ }
333
+ for (const child of n.children ?? [])visit(child);
334
+ };
335
+ visit(layout);
336
+ return fields;
337
+ }
338
+ function pickFieldByPoint(fields, point) {
339
+ const [px, py] = point;
340
+ return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
341
+ }
342
+ function pickLongestField(fields) {
343
+ return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
344
+ }
289
345
  const harmonyKeyCodeMap = {
290
346
  Enter: '2054',
291
347
  Backspace: '2055',
@@ -521,12 +577,13 @@ class device_HarmonyDevice {
521
577
  if (shouldReplace) {
522
578
  await hdc.click(x, y);
523
579
  await (0, core_utils_namespaceObject.sleep)(100);
524
- await hdc.clearTextField(100);
580
+ const length = await this.resolveClearLength(element);
581
+ if (length > 0) await hdc.clearTextField(length);
525
582
  await (0, core_utils_namespaceObject.sleep)(100);
526
583
  }
527
584
  await hdc.inputText(x, y, text);
528
- const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
529
- if (shouldAutoDismissKeyboard) await this.hideKeyboard();
585
+ const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
586
+ if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
530
587
  }
531
588
  async clearInput(element) {
532
589
  const hdc = await this.getHdc();
@@ -534,7 +591,24 @@ class device_HarmonyDevice {
534
591
  await hdc.click(element.center[0], element.center[1]);
535
592
  await (0, core_utils_namespaceObject.sleep)(100);
536
593
  }
537
- await hdc.clearTextField(100);
594
+ const length = await this.resolveClearLength(element);
595
+ if (length > 0) await hdc.clearTextField(length);
596
+ }
597
+ async resolveClearLength(element) {
598
+ const PADDING = 2;
599
+ const FALLBACK_LENGTH = 100;
600
+ try {
601
+ const hdc = await this.getHdc();
602
+ const layoutJson = await hdc.dumpLayout();
603
+ const layout = JSON.parse(layoutJson);
604
+ const fields = collectInputFields(layout);
605
+ if (0 === fields.length) return FALLBACK_LENGTH;
606
+ const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
607
+ return target.text.length + PADDING;
608
+ } catch (e) {
609
+ debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
610
+ return FALLBACK_LENGTH;
611
+ }
538
612
  }
539
613
  async pressKey(key) {
540
614
  const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
@@ -697,9 +771,11 @@ class device_HarmonyDevice {
697
771
  const hdc = await this.getHdc();
698
772
  await hdc.keyEvent('RecentApps');
699
773
  }
700
- async hideKeyboard() {
774
+ async hideKeyboard(options) {
701
775
  const hdc = await this.getHdc();
702
- await hdc.keyEvent('Back');
776
+ const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
777
+ const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
778
+ await hdc.keyEvent(key);
703
779
  }
704
780
  async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
705
781
  const hdc = await this.getHdc();
@@ -748,9 +824,13 @@ class device_HarmonyDevice {
748
824
  },
749
825
  keyboard: {
750
826
  keyboardPress: (keyName)=>this.pressKey(keyName),
751
- typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true, {
752
- autoDismissKeyboard: opts?.autoDismissKeyboard
753
- }),
827
+ typeText: (value, opts)=>{
828
+ const harmonyOpts = opts;
829
+ return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
830
+ autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
831
+ keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
832
+ });
833
+ },
754
834
  clearInput: (target)=>this.clearInput(target),
755
835
  cursorMove: async (direction, times = 1)=>{
756
836
  const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
package/dist/lib/cli.js CHANGED
@@ -152,6 +152,13 @@ class HdcClient {
152
152
  async screenshot(remotePath) {
153
153
  return await this.shell(`snapshot_display -f ${remotePath}`);
154
154
  }
155
+ async dumpLayout() {
156
+ const remotePath = '/data/local/tmp/midscene_layout.json';
157
+ const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
158
+ const jsonStart = output.indexOf('{');
159
+ if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
160
+ return output.slice(jsonStart);
161
+ }
155
162
  async click(x, y) {
156
163
  await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
157
164
  }
@@ -199,9 +206,17 @@ class HdcClient {
199
206
  await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
200
207
  }
201
208
  async clearTextField(length = 100) {
202
- const keys = [];
203
- for(let i = 0; i < length; i++)keys.push('2055', '2071');
204
- await this.keyEvent(...keys);
209
+ if (length <= 0) return;
210
+ const MAX_KEYS_PER_CALL = 3;
211
+ const cmds = [];
212
+ let remaining = length;
213
+ while(remaining > 0){
214
+ const n = Math.min(MAX_KEYS_PER_CALL, remaining);
215
+ const codes = Array(n).fill('2055').join(' ');
216
+ cmds.push(`uitest uiInput keyEvent ${codes}`);
217
+ remaining -= n;
218
+ }
219
+ await this.shell(cmds.join(';'));
205
220
  }
206
221
  async startAbility(bundleName, abilityName) {
207
222
  const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
@@ -282,6 +297,47 @@ const scrollQuadrantDivisions = 4;
282
297
  const screenEdgeMargin = 50;
283
298
  const debugDevice = (0, logger_namespaceObject.getDebug)('harmony:device');
284
299
  let screenshotResizeScaleWarned = false;
300
+ const INPUT_FIELD_TYPES = new Set([
301
+ 'TextInput',
302
+ 'TextArea',
303
+ 'SearchField'
304
+ ]);
305
+ function parseBounds(raw) {
306
+ if ('string' != typeof raw) return null;
307
+ const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
308
+ if (!m) return null;
309
+ return {
310
+ x1: Number(m[1]),
311
+ y1: Number(m[2]),
312
+ x2: Number(m[3]),
313
+ y2: Number(m[4])
314
+ };
315
+ }
316
+ function collectInputFields(layout) {
317
+ const fields = [];
318
+ const visit = (node)=>{
319
+ if (!node || 'object' != typeof node) return;
320
+ const n = node;
321
+ const attrs = n.attributes ?? {};
322
+ if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
323
+ const bounds = parseBounds(attrs.bounds);
324
+ if (bounds) fields.push({
325
+ text: String(attrs.text ?? ''),
326
+ bounds
327
+ });
328
+ }
329
+ for (const child of n.children ?? [])visit(child);
330
+ };
331
+ visit(layout);
332
+ return fields;
333
+ }
334
+ function pickFieldByPoint(fields, point) {
335
+ const [px, py] = point;
336
+ return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
337
+ }
338
+ function pickLongestField(fields) {
339
+ return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
340
+ }
285
341
  const harmonyKeyCodeMap = {
286
342
  Enter: '2054',
287
343
  Backspace: '2055',
@@ -517,12 +573,13 @@ class HarmonyDevice {
517
573
  if (shouldReplace) {
518
574
  await hdc.click(x, y);
519
575
  await (0, core_utils_namespaceObject.sleep)(100);
520
- await hdc.clearTextField(100);
576
+ const length = await this.resolveClearLength(element);
577
+ if (length > 0) await hdc.clearTextField(length);
521
578
  await (0, core_utils_namespaceObject.sleep)(100);
522
579
  }
523
580
  await hdc.inputText(x, y, text);
524
- const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
525
- if (shouldAutoDismissKeyboard) await this.hideKeyboard();
581
+ const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
582
+ if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
526
583
  }
527
584
  async clearInput(element) {
528
585
  const hdc = await this.getHdc();
@@ -530,7 +587,24 @@ class HarmonyDevice {
530
587
  await hdc.click(element.center[0], element.center[1]);
531
588
  await (0, core_utils_namespaceObject.sleep)(100);
532
589
  }
533
- await hdc.clearTextField(100);
590
+ const length = await this.resolveClearLength(element);
591
+ if (length > 0) await hdc.clearTextField(length);
592
+ }
593
+ async resolveClearLength(element) {
594
+ const PADDING = 2;
595
+ const FALLBACK_LENGTH = 100;
596
+ try {
597
+ const hdc = await this.getHdc();
598
+ const layoutJson = await hdc.dumpLayout();
599
+ const layout = JSON.parse(layoutJson);
600
+ const fields = collectInputFields(layout);
601
+ if (0 === fields.length) return FALLBACK_LENGTH;
602
+ const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
603
+ return target.text.length + PADDING;
604
+ } catch (e) {
605
+ debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
606
+ return FALLBACK_LENGTH;
607
+ }
534
608
  }
535
609
  async pressKey(key) {
536
610
  const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
@@ -693,9 +767,11 @@ class HarmonyDevice {
693
767
  const hdc = await this.getHdc();
694
768
  await hdc.keyEvent('RecentApps');
695
769
  }
696
- async hideKeyboard() {
770
+ async hideKeyboard(options) {
697
771
  const hdc = await this.getHdc();
698
- await hdc.keyEvent('Back');
772
+ const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
773
+ const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
774
+ await hdc.keyEvent(key);
699
775
  }
700
776
  async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
701
777
  const hdc = await this.getHdc();
@@ -744,9 +820,13 @@ class HarmonyDevice {
744
820
  },
745
821
  keyboard: {
746
822
  keyboardPress: (keyName)=>this.pressKey(keyName),
747
- typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true, {
748
- autoDismissKeyboard: opts?.autoDismissKeyboard
749
- }),
823
+ typeText: (value, opts)=>{
824
+ const harmonyOpts = opts;
825
+ return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
826
+ autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
827
+ keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
828
+ });
829
+ },
750
830
  clearInput: (target)=>this.clearInput(target),
751
831
  cursorMove: async (direction, times = 1)=>{
752
832
  const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
@@ -1013,7 +1093,7 @@ class HarmonyMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
1013
1093
  const tools = new HarmonyMidsceneTools();
1014
1094
  (0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-harmony', {
1015
1095
  stripPrefix: 'harmony_',
1016
- version: "1.8.7",
1096
+ version: "1.8.8-beta-20260601092605.0",
1017
1097
  extraCommands: (0, core_namespaceObject.createReportCliCommands)()
1018
1098
  }).catch((e)=>{
1019
1099
  process.exit((0, cli_namespaceObject.reportCLIError)(e));
package/dist/lib/index.js CHANGED
@@ -135,6 +135,13 @@ class HdcClient {
135
135
  async screenshot(remotePath) {
136
136
  return await this.shell(`snapshot_display -f ${remotePath}`);
137
137
  }
138
+ async dumpLayout() {
139
+ const remotePath = '/data/local/tmp/midscene_layout.json';
140
+ const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
141
+ const jsonStart = output.indexOf('{');
142
+ if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
143
+ return output.slice(jsonStart);
144
+ }
138
145
  async click(x, y) {
139
146
  await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
140
147
  }
@@ -182,9 +189,17 @@ class HdcClient {
182
189
  await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
183
190
  }
184
191
  async clearTextField(length = 100) {
185
- const keys = [];
186
- for(let i = 0; i < length; i++)keys.push('2055', '2071');
187
- await this.keyEvent(...keys);
192
+ if (length <= 0) return;
193
+ const MAX_KEYS_PER_CALL = 3;
194
+ const cmds = [];
195
+ let remaining = length;
196
+ while(remaining > 0){
197
+ const n = Math.min(MAX_KEYS_PER_CALL, remaining);
198
+ const codes = Array(n).fill('2055').join(' ');
199
+ cmds.push(`uitest uiInput keyEvent ${codes}`);
200
+ remaining -= n;
201
+ }
202
+ await this.shell(cmds.join(';'));
188
203
  }
189
204
  async startAbility(bundleName, abilityName) {
190
205
  const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
@@ -265,6 +280,47 @@ const scrollQuadrantDivisions = 4;
265
280
  const screenEdgeMargin = 50;
266
281
  const debugDevice = (0, logger_namespaceObject.getDebug)('harmony:device');
267
282
  let screenshotResizeScaleWarned = false;
283
+ const INPUT_FIELD_TYPES = new Set([
284
+ 'TextInput',
285
+ 'TextArea',
286
+ 'SearchField'
287
+ ]);
288
+ function parseBounds(raw) {
289
+ if ('string' != typeof raw) return null;
290
+ const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
291
+ if (!m) return null;
292
+ return {
293
+ x1: Number(m[1]),
294
+ y1: Number(m[2]),
295
+ x2: Number(m[3]),
296
+ y2: Number(m[4])
297
+ };
298
+ }
299
+ function collectInputFields(layout) {
300
+ const fields = [];
301
+ const visit = (node)=>{
302
+ if (!node || 'object' != typeof node) return;
303
+ const n = node;
304
+ const attrs = n.attributes ?? {};
305
+ if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
306
+ const bounds = parseBounds(attrs.bounds);
307
+ if (bounds) fields.push({
308
+ text: String(attrs.text ?? ''),
309
+ bounds
310
+ });
311
+ }
312
+ for (const child of n.children ?? [])visit(child);
313
+ };
314
+ visit(layout);
315
+ return fields;
316
+ }
317
+ function pickFieldByPoint(fields, point) {
318
+ const [px, py] = point;
319
+ return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
320
+ }
321
+ function pickLongestField(fields) {
322
+ return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
323
+ }
268
324
  const harmonyKeyCodeMap = {
269
325
  Enter: '2054',
270
326
  Backspace: '2055',
@@ -500,12 +556,13 @@ class HarmonyDevice {
500
556
  if (shouldReplace) {
501
557
  await hdc.click(x, y);
502
558
  await (0, utils_namespaceObject.sleep)(100);
503
- await hdc.clearTextField(100);
559
+ const length = await this.resolveClearLength(element);
560
+ if (length > 0) await hdc.clearTextField(length);
504
561
  await (0, utils_namespaceObject.sleep)(100);
505
562
  }
506
563
  await hdc.inputText(x, y, text);
507
- const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
508
- if (shouldAutoDismissKeyboard) await this.hideKeyboard();
564
+ const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
565
+ if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
509
566
  }
510
567
  async clearInput(element) {
511
568
  const hdc = await this.getHdc();
@@ -513,7 +570,24 @@ class HarmonyDevice {
513
570
  await hdc.click(element.center[0], element.center[1]);
514
571
  await (0, utils_namespaceObject.sleep)(100);
515
572
  }
516
- await hdc.clearTextField(100);
573
+ const length = await this.resolveClearLength(element);
574
+ if (length > 0) await hdc.clearTextField(length);
575
+ }
576
+ async resolveClearLength(element) {
577
+ const PADDING = 2;
578
+ const FALLBACK_LENGTH = 100;
579
+ try {
580
+ const hdc = await this.getHdc();
581
+ const layoutJson = await hdc.dumpLayout();
582
+ const layout = JSON.parse(layoutJson);
583
+ const fields = collectInputFields(layout);
584
+ if (0 === fields.length) return FALLBACK_LENGTH;
585
+ const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
586
+ return target.text.length + PADDING;
587
+ } catch (e) {
588
+ debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
589
+ return FALLBACK_LENGTH;
590
+ }
517
591
  }
518
592
  async pressKey(key) {
519
593
  const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
@@ -676,9 +750,11 @@ class HarmonyDevice {
676
750
  const hdc = await this.getHdc();
677
751
  await hdc.keyEvent('RecentApps');
678
752
  }
679
- async hideKeyboard() {
753
+ async hideKeyboard(options) {
680
754
  const hdc = await this.getHdc();
681
- await hdc.keyEvent('Back');
755
+ const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
756
+ const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
757
+ await hdc.keyEvent(key);
682
758
  }
683
759
  async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
684
760
  const hdc = await this.getHdc();
@@ -727,9 +803,13 @@ class HarmonyDevice {
727
803
  },
728
804
  keyboard: {
729
805
  keyboardPress: (keyName)=>this.pressKey(keyName),
730
- typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true, {
731
- autoDismissKeyboard: opts?.autoDismissKeyboard
732
- }),
806
+ typeText: (value, opts)=>{
807
+ const harmonyOpts = opts;
808
+ return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
809
+ autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
810
+ keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
811
+ });
812
+ },
733
813
  clearInput: (target)=>this.clearInput(target),
734
814
  cursorMove: async (direction, times = 1)=>{
735
815
  const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
@@ -168,6 +168,13 @@ class HdcClient {
168
168
  async screenshot(remotePath) {
169
169
  return await this.shell(`snapshot_display -f ${remotePath}`);
170
170
  }
171
+ async dumpLayout() {
172
+ const remotePath = '/data/local/tmp/midscene_layout.json';
173
+ const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
174
+ const jsonStart = output.indexOf('{');
175
+ if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
176
+ return output.slice(jsonStart);
177
+ }
171
178
  async click(x, y) {
172
179
  await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
173
180
  }
@@ -215,9 +222,17 @@ class HdcClient {
215
222
  await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
216
223
  }
217
224
  async clearTextField(length = 100) {
218
- const keys = [];
219
- for(let i = 0; i < length; i++)keys.push('2055', '2071');
220
- await this.keyEvent(...keys);
225
+ if (length <= 0) return;
226
+ const MAX_KEYS_PER_CALL = 3;
227
+ const cmds = [];
228
+ let remaining = length;
229
+ while(remaining > 0){
230
+ const n = Math.min(MAX_KEYS_PER_CALL, remaining);
231
+ const codes = Array(n).fill('2055').join(' ');
232
+ cmds.push(`uitest uiInput keyEvent ${codes}`);
233
+ remaining -= n;
234
+ }
235
+ await this.shell(cmds.join(';'));
221
236
  }
222
237
  async startAbility(bundleName, abilityName) {
223
238
  const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
@@ -298,6 +313,47 @@ const scrollQuadrantDivisions = 4;
298
313
  const screenEdgeMargin = 50;
299
314
  const debugDevice = (0, logger_namespaceObject.getDebug)('harmony:device');
300
315
  let screenshotResizeScaleWarned = false;
316
+ const INPUT_FIELD_TYPES = new Set([
317
+ 'TextInput',
318
+ 'TextArea',
319
+ 'SearchField'
320
+ ]);
321
+ function parseBounds(raw) {
322
+ if ('string' != typeof raw) return null;
323
+ const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
324
+ if (!m) return null;
325
+ return {
326
+ x1: Number(m[1]),
327
+ y1: Number(m[2]),
328
+ x2: Number(m[3]),
329
+ y2: Number(m[4])
330
+ };
331
+ }
332
+ function collectInputFields(layout) {
333
+ const fields = [];
334
+ const visit = (node)=>{
335
+ if (!node || 'object' != typeof node) return;
336
+ const n = node;
337
+ const attrs = n.attributes ?? {};
338
+ if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
339
+ const bounds = parseBounds(attrs.bounds);
340
+ if (bounds) fields.push({
341
+ text: String(attrs.text ?? ''),
342
+ bounds
343
+ });
344
+ }
345
+ for (const child of n.children ?? [])visit(child);
346
+ };
347
+ visit(layout);
348
+ return fields;
349
+ }
350
+ function pickFieldByPoint(fields, point) {
351
+ const [px, py] = point;
352
+ return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
353
+ }
354
+ function pickLongestField(fields) {
355
+ return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
356
+ }
301
357
  const harmonyKeyCodeMap = {
302
358
  Enter: '2054',
303
359
  Backspace: '2055',
@@ -533,12 +589,13 @@ class HarmonyDevice {
533
589
  if (shouldReplace) {
534
590
  await hdc.click(x, y);
535
591
  await (0, core_utils_namespaceObject.sleep)(100);
536
- await hdc.clearTextField(100);
592
+ const length = await this.resolveClearLength(element);
593
+ if (length > 0) await hdc.clearTextField(length);
537
594
  await (0, core_utils_namespaceObject.sleep)(100);
538
595
  }
539
596
  await hdc.inputText(x, y, text);
540
- const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
541
- if (shouldAutoDismissKeyboard) await this.hideKeyboard();
597
+ const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
598
+ if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
542
599
  }
543
600
  async clearInput(element) {
544
601
  const hdc = await this.getHdc();
@@ -546,7 +603,24 @@ class HarmonyDevice {
546
603
  await hdc.click(element.center[0], element.center[1]);
547
604
  await (0, core_utils_namespaceObject.sleep)(100);
548
605
  }
549
- await hdc.clearTextField(100);
606
+ const length = await this.resolveClearLength(element);
607
+ if (length > 0) await hdc.clearTextField(length);
608
+ }
609
+ async resolveClearLength(element) {
610
+ const PADDING = 2;
611
+ const FALLBACK_LENGTH = 100;
612
+ try {
613
+ const hdc = await this.getHdc();
614
+ const layoutJson = await hdc.dumpLayout();
615
+ const layout = JSON.parse(layoutJson);
616
+ const fields = collectInputFields(layout);
617
+ if (0 === fields.length) return FALLBACK_LENGTH;
618
+ const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
619
+ return target.text.length + PADDING;
620
+ } catch (e) {
621
+ debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
622
+ return FALLBACK_LENGTH;
623
+ }
550
624
  }
551
625
  async pressKey(key) {
552
626
  const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
@@ -709,9 +783,11 @@ class HarmonyDevice {
709
783
  const hdc = await this.getHdc();
710
784
  await hdc.keyEvent('RecentApps');
711
785
  }
712
- async hideKeyboard() {
786
+ async hideKeyboard(options) {
713
787
  const hdc = await this.getHdc();
714
- await hdc.keyEvent('Back');
788
+ const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
789
+ const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
790
+ await hdc.keyEvent(key);
715
791
  }
716
792
  async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
717
793
  const hdc = await this.getHdc();
@@ -760,9 +836,13 @@ class HarmonyDevice {
760
836
  },
761
837
  keyboard: {
762
838
  keyboardPress: (keyName)=>this.pressKey(keyName),
763
- typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true, {
764
- autoDismissKeyboard: opts?.autoDismissKeyboard
765
- }),
839
+ typeText: (value, opts)=>{
840
+ const harmonyOpts = opts;
841
+ return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
842
+ autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
843
+ keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
844
+ });
845
+ },
766
846
  clearInput: (target)=>this.clearInput(target),
767
847
  cursorMove: async (direction, times = 1)=>{
768
848
  const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
@@ -1033,7 +1113,7 @@ class HarmonyMCPServer extends mcp_namespaceObject.BaseMCPServer {
1033
1113
  constructor(toolsManager){
1034
1114
  super({
1035
1115
  name: '@midscene/harmony-mcp',
1036
- version: "1.8.7",
1116
+ version: "1.8.8-beta-20260601092605.0",
1037
1117
  description: 'Control the HarmonyOS device using natural language commands'
1038
1118
  }, toolsManager);
1039
1119
  }