@midscene/harmony 1.8.7 → 1.8.8-beta-20260601092605.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/es/bin.mjs CHANGED
@@ -130,6 +130,13 @@ class HdcClient {
130
130
  async screenshot(remotePath) {
131
131
  return await this.shell(`snapshot_display -f ${remotePath}`);
132
132
  }
133
+ async dumpLayout() {
134
+ const remotePath = '/data/local/tmp/midscene_layout.json';
135
+ const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
136
+ const jsonStart = output.indexOf('{');
137
+ if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
138
+ return output.slice(jsonStart);
139
+ }
133
140
  async click(x, y) {
134
141
  await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
135
142
  }
@@ -177,9 +184,17 @@ class HdcClient {
177
184
  await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
178
185
  }
179
186
  async clearTextField(length = 100) {
180
- const keys = [];
181
- for(let i = 0; i < length; i++)keys.push('2055', '2071');
182
- await this.keyEvent(...keys);
187
+ if (length <= 0) return;
188
+ const MAX_KEYS_PER_CALL = 3;
189
+ const cmds = [];
190
+ let remaining = length;
191
+ while(remaining > 0){
192
+ const n = Math.min(MAX_KEYS_PER_CALL, remaining);
193
+ const codes = Array(n).fill('2055').join(' ');
194
+ cmds.push(`uitest uiInput keyEvent ${codes}`);
195
+ remaining -= n;
196
+ }
197
+ await this.shell(cmds.join(';'));
183
198
  }
184
199
  async startAbility(bundleName, abilityName) {
185
200
  const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
@@ -260,6 +275,47 @@ const scrollQuadrantDivisions = 4;
260
275
  const screenEdgeMargin = 50;
261
276
  const debugDevice = getDebug('harmony:device');
262
277
  let screenshotResizeScaleWarned = false;
278
+ const INPUT_FIELD_TYPES = new Set([
279
+ 'TextInput',
280
+ 'TextArea',
281
+ 'SearchField'
282
+ ]);
283
+ function parseBounds(raw) {
284
+ if ('string' != typeof raw) return null;
285
+ const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
286
+ if (!m) return null;
287
+ return {
288
+ x1: Number(m[1]),
289
+ y1: Number(m[2]),
290
+ x2: Number(m[3]),
291
+ y2: Number(m[4])
292
+ };
293
+ }
294
+ function collectInputFields(layout) {
295
+ const fields = [];
296
+ const visit = (node)=>{
297
+ if (!node || 'object' != typeof node) return;
298
+ const n = node;
299
+ const attrs = n.attributes ?? {};
300
+ if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
301
+ const bounds = parseBounds(attrs.bounds);
302
+ if (bounds) fields.push({
303
+ text: String(attrs.text ?? ''),
304
+ bounds
305
+ });
306
+ }
307
+ for (const child of n.children ?? [])visit(child);
308
+ };
309
+ visit(layout);
310
+ return fields;
311
+ }
312
+ function pickFieldByPoint(fields, point) {
313
+ const [px, py] = point;
314
+ return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
315
+ }
316
+ function pickLongestField(fields) {
317
+ return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
318
+ }
263
319
  const harmonyKeyCodeMap = {
264
320
  Enter: '2054',
265
321
  Backspace: '2055',
@@ -495,12 +551,13 @@ class device_HarmonyDevice {
495
551
  if (shouldReplace) {
496
552
  await hdc.click(x, y);
497
553
  await sleep(100);
498
- await hdc.clearTextField(100);
554
+ const length = await this.resolveClearLength(element);
555
+ if (length > 0) await hdc.clearTextField(length);
499
556
  await sleep(100);
500
557
  }
501
558
  await hdc.inputText(x, y, text);
502
- const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
503
- if (shouldAutoDismissKeyboard) await this.hideKeyboard();
559
+ const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
560
+ if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
504
561
  }
505
562
  async clearInput(element) {
506
563
  const hdc = await this.getHdc();
@@ -508,7 +565,24 @@ class device_HarmonyDevice {
508
565
  await hdc.click(element.center[0], element.center[1]);
509
566
  await sleep(100);
510
567
  }
511
- await hdc.clearTextField(100);
568
+ const length = await this.resolveClearLength(element);
569
+ if (length > 0) await hdc.clearTextField(length);
570
+ }
571
+ async resolveClearLength(element) {
572
+ const PADDING = 2;
573
+ const FALLBACK_LENGTH = 100;
574
+ try {
575
+ const hdc = await this.getHdc();
576
+ const layoutJson = await hdc.dumpLayout();
577
+ const layout = JSON.parse(layoutJson);
578
+ const fields = collectInputFields(layout);
579
+ if (0 === fields.length) return FALLBACK_LENGTH;
580
+ const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
581
+ return target.text.length + PADDING;
582
+ } catch (e) {
583
+ debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
584
+ return FALLBACK_LENGTH;
585
+ }
512
586
  }
513
587
  async pressKey(key) {
514
588
  const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
@@ -671,9 +745,11 @@ class device_HarmonyDevice {
671
745
  const hdc = await this.getHdc();
672
746
  await hdc.keyEvent('RecentApps');
673
747
  }
674
- async hideKeyboard() {
748
+ async hideKeyboard(options) {
675
749
  const hdc = await this.getHdc();
676
- await hdc.keyEvent('Back');
750
+ const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
751
+ const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
752
+ await hdc.keyEvent(key);
677
753
  }
678
754
  async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
679
755
  const hdc = await this.getHdc();
@@ -722,9 +798,13 @@ class device_HarmonyDevice {
722
798
  },
723
799
  keyboard: {
724
800
  keyboardPress: (keyName)=>this.pressKey(keyName),
725
- typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true, {
726
- autoDismissKeyboard: opts?.autoDismissKeyboard
727
- }),
801
+ typeText: (value, opts)=>{
802
+ const harmonyOpts = opts;
803
+ return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
804
+ autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
805
+ keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
806
+ });
807
+ },
728
808
  clearInput: (target)=>this.clearInput(target),
729
809
  cursorMove: async (direction, times = 1)=>{
730
810
  const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
package/dist/es/cli.mjs CHANGED
@@ -127,6 +127,13 @@ class HdcClient {
127
127
  async screenshot(remotePath) {
128
128
  return await this.shell(`snapshot_display -f ${remotePath}`);
129
129
  }
130
+ async dumpLayout() {
131
+ const remotePath = '/data/local/tmp/midscene_layout.json';
132
+ const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
133
+ const jsonStart = output.indexOf('{');
134
+ if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
135
+ return output.slice(jsonStart);
136
+ }
130
137
  async click(x, y) {
131
138
  await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
132
139
  }
@@ -174,9 +181,17 @@ class HdcClient {
174
181
  await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
175
182
  }
176
183
  async clearTextField(length = 100) {
177
- const keys = [];
178
- for(let i = 0; i < length; i++)keys.push('2055', '2071');
179
- await this.keyEvent(...keys);
184
+ if (length <= 0) return;
185
+ const MAX_KEYS_PER_CALL = 3;
186
+ const cmds = [];
187
+ let remaining = length;
188
+ while(remaining > 0){
189
+ const n = Math.min(MAX_KEYS_PER_CALL, remaining);
190
+ const codes = Array(n).fill('2055').join(' ');
191
+ cmds.push(`uitest uiInput keyEvent ${codes}`);
192
+ remaining -= n;
193
+ }
194
+ await this.shell(cmds.join(';'));
180
195
  }
181
196
  async startAbility(bundleName, abilityName) {
182
197
  const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
@@ -257,6 +272,47 @@ const scrollQuadrantDivisions = 4;
257
272
  const screenEdgeMargin = 50;
258
273
  const debugDevice = getDebug('harmony:device');
259
274
  let screenshotResizeScaleWarned = false;
275
+ const INPUT_FIELD_TYPES = new Set([
276
+ 'TextInput',
277
+ 'TextArea',
278
+ 'SearchField'
279
+ ]);
280
+ function parseBounds(raw) {
281
+ if ('string' != typeof raw) return null;
282
+ const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
283
+ if (!m) return null;
284
+ return {
285
+ x1: Number(m[1]),
286
+ y1: Number(m[2]),
287
+ x2: Number(m[3]),
288
+ y2: Number(m[4])
289
+ };
290
+ }
291
+ function collectInputFields(layout) {
292
+ const fields = [];
293
+ const visit = (node)=>{
294
+ if (!node || 'object' != typeof node) return;
295
+ const n = node;
296
+ const attrs = n.attributes ?? {};
297
+ if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
298
+ const bounds = parseBounds(attrs.bounds);
299
+ if (bounds) fields.push({
300
+ text: String(attrs.text ?? ''),
301
+ bounds
302
+ });
303
+ }
304
+ for (const child of n.children ?? [])visit(child);
305
+ };
306
+ visit(layout);
307
+ return fields;
308
+ }
309
+ function pickFieldByPoint(fields, point) {
310
+ const [px, py] = point;
311
+ return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
312
+ }
313
+ function pickLongestField(fields) {
314
+ return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
315
+ }
260
316
  const harmonyKeyCodeMap = {
261
317
  Enter: '2054',
262
318
  Backspace: '2055',
@@ -492,12 +548,13 @@ class HarmonyDevice {
492
548
  if (shouldReplace) {
493
549
  await hdc.click(x, y);
494
550
  await sleep(100);
495
- await hdc.clearTextField(100);
551
+ const length = await this.resolveClearLength(element);
552
+ if (length > 0) await hdc.clearTextField(length);
496
553
  await sleep(100);
497
554
  }
498
555
  await hdc.inputText(x, y, text);
499
- const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
500
- if (shouldAutoDismissKeyboard) await this.hideKeyboard();
556
+ const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
557
+ if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
501
558
  }
502
559
  async clearInput(element) {
503
560
  const hdc = await this.getHdc();
@@ -505,7 +562,24 @@ class HarmonyDevice {
505
562
  await hdc.click(element.center[0], element.center[1]);
506
563
  await sleep(100);
507
564
  }
508
- await hdc.clearTextField(100);
565
+ const length = await this.resolveClearLength(element);
566
+ if (length > 0) await hdc.clearTextField(length);
567
+ }
568
+ async resolveClearLength(element) {
569
+ const PADDING = 2;
570
+ const FALLBACK_LENGTH = 100;
571
+ try {
572
+ const hdc = await this.getHdc();
573
+ const layoutJson = await hdc.dumpLayout();
574
+ const layout = JSON.parse(layoutJson);
575
+ const fields = collectInputFields(layout);
576
+ if (0 === fields.length) return FALLBACK_LENGTH;
577
+ const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
578
+ return target.text.length + PADDING;
579
+ } catch (e) {
580
+ debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
581
+ return FALLBACK_LENGTH;
582
+ }
509
583
  }
510
584
  async pressKey(key) {
511
585
  const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
@@ -668,9 +742,11 @@ class HarmonyDevice {
668
742
  const hdc = await this.getHdc();
669
743
  await hdc.keyEvent('RecentApps');
670
744
  }
671
- async hideKeyboard() {
745
+ async hideKeyboard(options) {
672
746
  const hdc = await this.getHdc();
673
- await hdc.keyEvent('Back');
747
+ const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
748
+ const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
749
+ await hdc.keyEvent(key);
674
750
  }
675
751
  async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
676
752
  const hdc = await this.getHdc();
@@ -719,9 +795,13 @@ class HarmonyDevice {
719
795
  },
720
796
  keyboard: {
721
797
  keyboardPress: (keyName)=>this.pressKey(keyName),
722
- typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true, {
723
- autoDismissKeyboard: opts?.autoDismissKeyboard
724
- }),
798
+ typeText: (value, opts)=>{
799
+ const harmonyOpts = opts;
800
+ return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
801
+ autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
802
+ keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
803
+ });
804
+ },
725
805
  clearInput: (target)=>this.clearInput(target),
726
806
  cursorMove: async (direction, times = 1)=>{
727
807
  const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
@@ -988,7 +1068,7 @@ class HarmonyMidsceneTools extends BaseMidsceneTools {
988
1068
  const tools = new HarmonyMidsceneTools();
989
1069
  runToolsCLI(tools, 'midscene-harmony', {
990
1070
  stripPrefix: 'harmony_',
991
- version: "1.8.7",
1071
+ version: "1.8.8-beta-20260601092605.0",
992
1072
  extraCommands: createReportCliCommands()
993
1073
  }).catch((e)=>{
994
1074
  process.exit(reportCLIError(e));
package/dist/es/index.mjs CHANGED
@@ -98,6 +98,13 @@ class HdcClient {
98
98
  async screenshot(remotePath) {
99
99
  return await this.shell(`snapshot_display -f ${remotePath}`);
100
100
  }
101
+ async dumpLayout() {
102
+ const remotePath = '/data/local/tmp/midscene_layout.json';
103
+ const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
104
+ const jsonStart = output.indexOf('{');
105
+ if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
106
+ return output.slice(jsonStart);
107
+ }
101
108
  async click(x, y) {
102
109
  await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
103
110
  }
@@ -145,9 +152,17 @@ class HdcClient {
145
152
  await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
146
153
  }
147
154
  async clearTextField(length = 100) {
148
- const keys = [];
149
- for(let i = 0; i < length; i++)keys.push('2055', '2071');
150
- await this.keyEvent(...keys);
155
+ if (length <= 0) return;
156
+ const MAX_KEYS_PER_CALL = 3;
157
+ const cmds = [];
158
+ let remaining = length;
159
+ while(remaining > 0){
160
+ const n = Math.min(MAX_KEYS_PER_CALL, remaining);
161
+ const codes = Array(n).fill('2055').join(' ');
162
+ cmds.push(`uitest uiInput keyEvent ${codes}`);
163
+ remaining -= n;
164
+ }
165
+ await this.shell(cmds.join(';'));
151
166
  }
152
167
  async startAbility(bundleName, abilityName) {
153
168
  const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
@@ -228,6 +243,47 @@ const scrollQuadrantDivisions = 4;
228
243
  const screenEdgeMargin = 50;
229
244
  const debugDevice = getDebug('harmony:device');
230
245
  let screenshotResizeScaleWarned = false;
246
+ const INPUT_FIELD_TYPES = new Set([
247
+ 'TextInput',
248
+ 'TextArea',
249
+ 'SearchField'
250
+ ]);
251
+ function parseBounds(raw) {
252
+ if ('string' != typeof raw) return null;
253
+ const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
254
+ if (!m) return null;
255
+ return {
256
+ x1: Number(m[1]),
257
+ y1: Number(m[2]),
258
+ x2: Number(m[3]),
259
+ y2: Number(m[4])
260
+ };
261
+ }
262
+ function collectInputFields(layout) {
263
+ const fields = [];
264
+ const visit = (node)=>{
265
+ if (!node || 'object' != typeof node) return;
266
+ const n = node;
267
+ const attrs = n.attributes ?? {};
268
+ if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
269
+ const bounds = parseBounds(attrs.bounds);
270
+ if (bounds) fields.push({
271
+ text: String(attrs.text ?? ''),
272
+ bounds
273
+ });
274
+ }
275
+ for (const child of n.children ?? [])visit(child);
276
+ };
277
+ visit(layout);
278
+ return fields;
279
+ }
280
+ function pickFieldByPoint(fields, point) {
281
+ const [px, py] = point;
282
+ return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
283
+ }
284
+ function pickLongestField(fields) {
285
+ return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
286
+ }
231
287
  const harmonyKeyCodeMap = {
232
288
  Enter: '2054',
233
289
  Backspace: '2055',
@@ -463,12 +519,13 @@ class HarmonyDevice {
463
519
  if (shouldReplace) {
464
520
  await hdc.click(x, y);
465
521
  await sleep(100);
466
- await hdc.clearTextField(100);
522
+ const length = await this.resolveClearLength(element);
523
+ if (length > 0) await hdc.clearTextField(length);
467
524
  await sleep(100);
468
525
  }
469
526
  await hdc.inputText(x, y, text);
470
- const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
471
- if (shouldAutoDismissKeyboard) await this.hideKeyboard();
527
+ const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
528
+ if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
472
529
  }
473
530
  async clearInput(element) {
474
531
  const hdc = await this.getHdc();
@@ -476,7 +533,24 @@ class HarmonyDevice {
476
533
  await hdc.click(element.center[0], element.center[1]);
477
534
  await sleep(100);
478
535
  }
479
- await hdc.clearTextField(100);
536
+ const length = await this.resolveClearLength(element);
537
+ if (length > 0) await hdc.clearTextField(length);
538
+ }
539
+ async resolveClearLength(element) {
540
+ const PADDING = 2;
541
+ const FALLBACK_LENGTH = 100;
542
+ try {
543
+ const hdc = await this.getHdc();
544
+ const layoutJson = await hdc.dumpLayout();
545
+ const layout = JSON.parse(layoutJson);
546
+ const fields = collectInputFields(layout);
547
+ if (0 === fields.length) return FALLBACK_LENGTH;
548
+ const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
549
+ return target.text.length + PADDING;
550
+ } catch (e) {
551
+ debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
552
+ return FALLBACK_LENGTH;
553
+ }
480
554
  }
481
555
  async pressKey(key) {
482
556
  const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
@@ -639,9 +713,11 @@ class HarmonyDevice {
639
713
  const hdc = await this.getHdc();
640
714
  await hdc.keyEvent('RecentApps');
641
715
  }
642
- async hideKeyboard() {
716
+ async hideKeyboard(options) {
643
717
  const hdc = await this.getHdc();
644
- await hdc.keyEvent('Back');
718
+ const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
719
+ const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
720
+ await hdc.keyEvent(key);
645
721
  }
646
722
  async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
647
723
  const hdc = await this.getHdc();
@@ -690,9 +766,13 @@ class HarmonyDevice {
690
766
  },
691
767
  keyboard: {
692
768
  keyboardPress: (keyName)=>this.pressKey(keyName),
693
- typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true, {
694
- autoDismissKeyboard: opts?.autoDismissKeyboard
695
- }),
769
+ typeText: (value, opts)=>{
770
+ const harmonyOpts = opts;
771
+ return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
772
+ autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
773
+ keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
774
+ });
775
+ },
696
776
  clearInput: (target)=>this.clearInput(target),
697
777
  cursorMove: async (direction, times = 1)=>{
698
778
  const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
@@ -127,6 +127,13 @@ class HdcClient {
127
127
  async screenshot(remotePath) {
128
128
  return await this.shell(`snapshot_display -f ${remotePath}`);
129
129
  }
130
+ async dumpLayout() {
131
+ const remotePath = '/data/local/tmp/midscene_layout.json';
132
+ const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
133
+ const jsonStart = output.indexOf('{');
134
+ if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
135
+ return output.slice(jsonStart);
136
+ }
130
137
  async click(x, y) {
131
138
  await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
132
139
  }
@@ -174,9 +181,17 @@ class HdcClient {
174
181
  await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
175
182
  }
176
183
  async clearTextField(length = 100) {
177
- const keys = [];
178
- for(let i = 0; i < length; i++)keys.push('2055', '2071');
179
- await this.keyEvent(...keys);
184
+ if (length <= 0) return;
185
+ const MAX_KEYS_PER_CALL = 3;
186
+ const cmds = [];
187
+ let remaining = length;
188
+ while(remaining > 0){
189
+ const n = Math.min(MAX_KEYS_PER_CALL, remaining);
190
+ const codes = Array(n).fill('2055').join(' ');
191
+ cmds.push(`uitest uiInput keyEvent ${codes}`);
192
+ remaining -= n;
193
+ }
194
+ await this.shell(cmds.join(';'));
180
195
  }
181
196
  async startAbility(bundleName, abilityName) {
182
197
  const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
@@ -257,6 +272,47 @@ const scrollQuadrantDivisions = 4;
257
272
  const screenEdgeMargin = 50;
258
273
  const debugDevice = getDebug('harmony:device');
259
274
  let screenshotResizeScaleWarned = false;
275
+ const INPUT_FIELD_TYPES = new Set([
276
+ 'TextInput',
277
+ 'TextArea',
278
+ 'SearchField'
279
+ ]);
280
+ function parseBounds(raw) {
281
+ if ('string' != typeof raw) return null;
282
+ const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
283
+ if (!m) return null;
284
+ return {
285
+ x1: Number(m[1]),
286
+ y1: Number(m[2]),
287
+ x2: Number(m[3]),
288
+ y2: Number(m[4])
289
+ };
290
+ }
291
+ function collectInputFields(layout) {
292
+ const fields = [];
293
+ const visit = (node)=>{
294
+ if (!node || 'object' != typeof node) return;
295
+ const n = node;
296
+ const attrs = n.attributes ?? {};
297
+ if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
298
+ const bounds = parseBounds(attrs.bounds);
299
+ if (bounds) fields.push({
300
+ text: String(attrs.text ?? ''),
301
+ bounds
302
+ });
303
+ }
304
+ for (const child of n.children ?? [])visit(child);
305
+ };
306
+ visit(layout);
307
+ return fields;
308
+ }
309
+ function pickFieldByPoint(fields, point) {
310
+ const [px, py] = point;
311
+ return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
312
+ }
313
+ function pickLongestField(fields) {
314
+ return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
315
+ }
260
316
  const harmonyKeyCodeMap = {
261
317
  Enter: '2054',
262
318
  Backspace: '2055',
@@ -492,12 +548,13 @@ class HarmonyDevice {
492
548
  if (shouldReplace) {
493
549
  await hdc.click(x, y);
494
550
  await sleep(100);
495
- await hdc.clearTextField(100);
551
+ const length = await this.resolveClearLength(element);
552
+ if (length > 0) await hdc.clearTextField(length);
496
553
  await sleep(100);
497
554
  }
498
555
  await hdc.inputText(x, y, text);
499
- const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
500
- if (shouldAutoDismissKeyboard) await this.hideKeyboard();
556
+ const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
557
+ if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
501
558
  }
502
559
  async clearInput(element) {
503
560
  const hdc = await this.getHdc();
@@ -505,7 +562,24 @@ class HarmonyDevice {
505
562
  await hdc.click(element.center[0], element.center[1]);
506
563
  await sleep(100);
507
564
  }
508
- await hdc.clearTextField(100);
565
+ const length = await this.resolveClearLength(element);
566
+ if (length > 0) await hdc.clearTextField(length);
567
+ }
568
+ async resolveClearLength(element) {
569
+ const PADDING = 2;
570
+ const FALLBACK_LENGTH = 100;
571
+ try {
572
+ const hdc = await this.getHdc();
573
+ const layoutJson = await hdc.dumpLayout();
574
+ const layout = JSON.parse(layoutJson);
575
+ const fields = collectInputFields(layout);
576
+ if (0 === fields.length) return FALLBACK_LENGTH;
577
+ const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
578
+ return target.text.length + PADDING;
579
+ } catch (e) {
580
+ debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
581
+ return FALLBACK_LENGTH;
582
+ }
509
583
  }
510
584
  async pressKey(key) {
511
585
  const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
@@ -668,9 +742,11 @@ class HarmonyDevice {
668
742
  const hdc = await this.getHdc();
669
743
  await hdc.keyEvent('RecentApps');
670
744
  }
671
- async hideKeyboard() {
745
+ async hideKeyboard(options) {
672
746
  const hdc = await this.getHdc();
673
- await hdc.keyEvent('Back');
747
+ const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
748
+ const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
749
+ await hdc.keyEvent(key);
674
750
  }
675
751
  async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
676
752
  const hdc = await this.getHdc();
@@ -719,9 +795,13 @@ class HarmonyDevice {
719
795
  },
720
796
  keyboard: {
721
797
  keyboardPress: (keyName)=>this.pressKey(keyName),
722
- typeText: (value, opts)=>opts?.focusOnly ? Promise.resolve() : this.typeText(value, opts?.target, opts?.replace ?? true, {
723
- autoDismissKeyboard: opts?.autoDismissKeyboard
724
- }),
798
+ typeText: (value, opts)=>{
799
+ const harmonyOpts = opts;
800
+ return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
801
+ autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
802
+ keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
803
+ });
804
+ },
725
805
  clearInput: (target)=>this.clearInput(target),
726
806
  cursorMove: async (direction, times = 1)=>{
727
807
  const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
@@ -992,7 +1072,7 @@ class HarmonyMCPServer extends BaseMCPServer {
992
1072
  constructor(toolsManager){
993
1073
  super({
994
1074
  name: '@midscene/harmony-mcp',
995
- version: "1.8.7",
1075
+ version: "1.8.8-beta-20260601092605.0",
996
1076
  description: 'Control the HarmonyOS device using natural language commands'
997
1077
  }, toolsManager);
998
1078
  }