@midscene/harmony 1.9.8 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1116 +0,0 @@
1
- import { BaseMCPServer, createMCPServerLauncher } from "@midscene/shared/mcp";
2
- import { z } from "@midscene/core";
3
- import { getDebug } from "@midscene/shared/logger";
4
- import { agentBehaviorInitArgShape, extractAgentBehaviorInitArgs, getAgentInitArgsSignature, shouldRebuildAgentForInitArgs } from "@midscene/shared/mcp/agent-behavior-init-args";
5
- import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
6
- import { Agent } from "@midscene/core/agent";
7
- import { mergeAndNormalizeAppNameMapping, normalizeForComparison, repeat } from "@midscene/shared/utils";
8
- import node_assert from "node:assert";
9
- import node_fs, { accessSync, constants } from "node:fs";
10
- import { createDefaultMobileActions, defineAction } from "@midscene/core/device";
11
- import { getTmpFile, sleep } from "@midscene/core/utils";
12
- import { createImgBase64ByFormat } from "@midscene/shared/img";
13
- import { execFile } from "node:child_process";
14
- import { promisify } from "node:util";
15
- const defaultAppNameMapping = {
16
- 设置: 'com.huawei.hmos.settings',
17
- Settings: 'com.huawei.hmos.settings',
18
- 相机: 'com.huawei.hmos.camera',
19
- Camera: 'com.huawei.hmos.camera',
20
- 图库: 'com.huawei.hmos.photos',
21
- Gallery: 'com.huawei.hmos.photos',
22
- 日历: 'com.huawei.hmos.calendar',
23
- Calendar: 'com.huawei.hmos.calendar',
24
- 时钟: 'com.huawei.hmos.clock',
25
- Clock: 'com.huawei.hmos.clock',
26
- 计算器: 'com.huawei.hmos.calculator',
27
- Calculator: 'com.huawei.hmos.calculator',
28
- 文件管理: 'com.huawei.hmos.filemanager',
29
- 备忘录: 'com.huawei.hmos.notepad',
30
- 联系人: 'com.huawei.hmos.contacts',
31
- 电话: 'com.huawei.hmos.phone',
32
- 信息: 'com.huawei.hmos.message',
33
- 邮件: 'com.huawei.hmos.email',
34
- 浏览器: 'com.huawei.hmos.browser',
35
- Browser: 'com.huawei.hmos.browser',
36
- 应用市场: 'com.huawei.appmarket',
37
- AppGallery: 'com.huawei.appmarket',
38
- 华为音乐: 'com.huawei.hmsapp.music',
39
- Music: 'com.huawei.hmsapp.music',
40
- 华为视频: 'com.huawei.hmos.video',
41
- 天气: 'com.huawei.hmos.weather',
42
- Weather: 'com.huawei.hmos.weather',
43
- 抖音: 'com.ss.hm.ugc.aweme',
44
- 支付宝: 'com.alipay.mobile.client',
45
- 高德地图: 'com.amap.hmapp',
46
- 百度: 'com.baidu.baiduapp',
47
- 携程: 'com.ctrip.harmonynext'
48
- };
49
- function _define_property(obj, key, value) {
50
- if (key in obj) Object.defineProperty(obj, key, {
51
- value: value,
52
- enumerable: true,
53
- configurable: true,
54
- writable: true
55
- });
56
- else obj[key] = value;
57
- return obj;
58
- }
59
- const execFileAsync = promisify(execFile);
60
- const debugHdc = getDebug('harmony:hdc');
61
- function resolveHdcPath(hdcPath) {
62
- if (hdcPath) return hdcPath;
63
- if (process.env.HDC_HOME) {
64
- const envPath = `${process.env.HDC_HOME}/hdc`;
65
- debugHdc(`Using HDC from HDC_HOME: ${envPath}`);
66
- return envPath;
67
- }
68
- const homeDir = process.env.HOME || process.env.USERPROFILE || '';
69
- const commonPaths = [
70
- `${homeDir}/Library/HarmonyOS/next/command-line-tools/sdk/default/openharmony/toolchains/hdc`,
71
- `${homeDir}/Library/HarmonyOS/sdk/hmscore/3.1.0/toolchains/hdc`
72
- ];
73
- for (const p of commonPaths)try {
74
- accessSync(p, constants.X_OK);
75
- debugHdc(`Found HDC at: ${p}`);
76
- return p;
77
- } catch {}
78
- return 'hdc';
79
- }
80
- class HdcClient {
81
- buildArgs(args) {
82
- if (this.deviceId) return [
83
- '-t',
84
- this.deviceId,
85
- ...args
86
- ];
87
- return args;
88
- }
89
- async exec(...args) {
90
- let release;
91
- const prev = this.execMutex;
92
- this.execMutex = new Promise((r)=>{
93
- release = r;
94
- });
95
- await prev;
96
- const fullArgs = this.buildArgs(args);
97
- debugHdc(`hdc ${fullArgs.join(' ')}`);
98
- try {
99
- const { stdout, stderr } = await execFileAsync(this.hdcPath, fullArgs, {
100
- timeout: this.timeout,
101
- maxBuffer: 52428800
102
- });
103
- if (stderr?.trim()) debugHdc(`hdc stderr: ${stderr.trim()}`);
104
- debugHdc(`hdc ${fullArgs.join(' ')} end`);
105
- return stdout;
106
- } catch (error) {
107
- if (error.killed && error.stdout?.trim()) {
108
- debugHdc('hdc process was killed but stdout is available, treating as success');
109
- return error.stdout;
110
- }
111
- debugHdc(`hdc error: ${error.message}`);
112
- throw new Error(`HDC command failed: hdc ${fullArgs.join(' ')}: ${error.message}`, {
113
- cause: error
114
- });
115
- } finally{
116
- release();
117
- }
118
- }
119
- async shell(command) {
120
- return this.exec('shell', command);
121
- }
122
- async fileSend(localPath, remotePath) {
123
- await this.exec('file', 'send', localPath, remotePath);
124
- }
125
- async fileRecv(remotePath, localPath) {
126
- await this.exec('file', 'recv', remotePath, localPath);
127
- }
128
- async screenshot(remotePath) {
129
- return await this.shell(`snapshot_display -f ${remotePath}`);
130
- }
131
- async dumpLayout() {
132
- const remotePath = '/data/local/tmp/midscene_layout.json';
133
- const output = await this.shell(`uitest dumpLayout -p ${remotePath} && cat ${remotePath}`);
134
- const jsonStart = output.indexOf('{');
135
- if (jsonStart < 0) throw new Error(`dumpLayout: no JSON body in output: ${output.slice(0, 200)}`);
136
- return output.slice(jsonStart);
137
- }
138
- async click(x, y) {
139
- await this.shell(`uitest uiInput click ${Math.round(x)} ${Math.round(y)}`);
140
- }
141
- async doubleClick(x, y) {
142
- await this.shell(`uitest uiInput doubleClick ${Math.round(x)} ${Math.round(y)}`);
143
- }
144
- async longClick(x, y) {
145
- await this.shell(`uitest uiInput longClick ${Math.round(x)} ${Math.round(y)}`);
146
- }
147
- async swipe(fromX, fromY, toX, toY, speed) {
148
- const args = [
149
- Math.round(fromX),
150
- Math.round(fromY),
151
- Math.round(toX),
152
- Math.round(toY)
153
- ];
154
- if (void 0 !== speed) args.push(Math.round(speed));
155
- await this.shell(`uitest uiInput swipe ${args.join(' ')}`);
156
- }
157
- async fling(fromX, fromY, toX, toY, speed) {
158
- const args = [
159
- Math.round(fromX),
160
- Math.round(fromY),
161
- Math.round(toX),
162
- Math.round(toY)
163
- ];
164
- if (void 0 !== speed) args.push(Math.round(speed));
165
- await this.shell(`uitest uiInput fling ${args.join(' ')}`);
166
- }
167
- async drag(fromX, fromY, toX, toY, speed) {
168
- const args = [
169
- Math.round(fromX),
170
- Math.round(fromY),
171
- Math.round(toX),
172
- Math.round(toY)
173
- ];
174
- if (void 0 !== speed) args.push(Math.round(speed));
175
- await this.shell(`uitest uiInput drag ${args.join(' ')}`);
176
- }
177
- async inputText(x, y, text) {
178
- const escapedText = text.replace(/'/g, "'\\''");
179
- await this.shell(`uitest uiInput inputText ${Math.round(x)} ${Math.round(y)} '${escapedText}'`);
180
- }
181
- async keyEvent(...keys) {
182
- await this.shell(`uitest uiInput keyEvent ${keys.join(' ')}`);
183
- }
184
- async clearTextField(length = 100) {
185
- if (length <= 0) return;
186
- const MAX_KEYS_PER_CALL = 3;
187
- const cmds = [];
188
- let remaining = length;
189
- while(remaining > 0){
190
- const n = Math.min(MAX_KEYS_PER_CALL, remaining);
191
- const codes = Array(n).fill('2055').join(' ');
192
- cmds.push(`uitest uiInput keyEvent ${codes}`);
193
- remaining -= n;
194
- }
195
- await this.shell(cmds.join(';'));
196
- }
197
- async startAbility(bundleName, abilityName) {
198
- const output = await this.shell(`aa start -a ${abilityName} -b ${bundleName}`);
199
- if (output.includes('error:')) throw new Error(`Failed to start ${bundleName}/${abilityName}: ${output.trim()}`);
200
- }
201
- async queryMainAbility(bundleName) {
202
- const output = await this.shell(`bm dump -n ${bundleName}`);
203
- const names = [];
204
- for (const match of output.matchAll(/"name"\s*:\s*"([^"]+)"/g))names.push(match[1]);
205
- for (const candidate of [
206
- 'EntryAbility',
207
- 'MainAbility',
208
- `${bundleName}.MainAbility`
209
- ])if (names.includes(candidate)) return candidate;
210
- return names.find((n)=>n !== bundleName && n.endsWith('Ability') && !n.includes('Extension') && !n.includes('Service') && !n.includes('Form') && !n.includes('Dialog'));
211
- }
212
- async forceStop(bundleName) {
213
- const output = await this.shell(`aa force-stop ${bundleName}`);
214
- if (output.includes('error:')) throw new Error(`Failed to force stop ${bundleName}: ${output.trim()}`);
215
- }
216
- async getScreenInfo() {
217
- const stdout = await this.shell('hidumper -s RenderService -a screen');
218
- const renderDimensionPattern = 'render\\s+(?:size|resolution)\\s*[:=]\\s*(\\d{3,5})x(\\d{3,5})';
219
- const activeFoldMatch = stdout.match(/foldScreenId:(\d+),\s*isConnected:\d+,\s*isPowerOn:1/);
220
- if (activeFoldMatch) {
221
- const activeId = activeFoldMatch[1];
222
- const screenRegex = new RegExp(`screen\\[\\d+\\]:\\s*id=${activeId},.*?${renderDimensionPattern}`);
223
- const screenMatch = stdout.match(screenRegex);
224
- if (screenMatch) {
225
- debugHdc(`Foldable screen detected, active screen id=${activeId}: ${screenMatch[1]}x${screenMatch[2]}`);
226
- return {
227
- width: Number.parseInt(screenMatch[1], 10),
228
- height: Number.parseInt(screenMatch[2], 10)
229
- };
230
- }
231
- }
232
- const renderSizeMatch = stdout.match(new RegExp(renderDimensionPattern));
233
- if (renderSizeMatch) return {
234
- width: Number.parseInt(renderSizeMatch[1], 10),
235
- height: Number.parseInt(renderSizeMatch[2], 10)
236
- };
237
- const displayStdout = await this.shell('hidumper -s DisplayManagerService -a');
238
- const displayMatch = displayStdout.match(/activeModes.*?(\d{3,5}),\s*(\d{3,5})/);
239
- if (displayMatch) return {
240
- width: Number.parseInt(displayMatch[1], 10),
241
- height: Number.parseInt(displayMatch[2], 10)
242
- };
243
- throw new Error(`Failed to get screen size from HDC. RenderService output: ${stdout}`);
244
- }
245
- async listTargets() {
246
- const stdout = await this.exec('list', 'targets');
247
- return stdout.trim().split('\n').map((line)=>line.trim()).filter((line)=>line.length > 0 && !line.startsWith('['));
248
- }
249
- constructor(options){
250
- _define_property(this, "hdcPath", void 0);
251
- _define_property(this, "deviceId", void 0);
252
- _define_property(this, "timeout", void 0);
253
- _define_property(this, "execMutex", Promise.resolve());
254
- this.hdcPath = resolveHdcPath(options.hdcPath);
255
- this.deviceId = options.deviceId ?? '';
256
- this.timeout = options.timeout ?? 60000;
257
- }
258
- }
259
- function device_define_property(obj, key, value) {
260
- if (key in obj) Object.defineProperty(obj, key, {
261
- value: value,
262
- enumerable: true,
263
- configurable: true,
264
- writable: true
265
- });
266
- else obj[key] = value;
267
- return obj;
268
- }
269
- const defaultScrollUntilTimes = 10;
270
- const defaultFastSwipeSpeed = 2000;
271
- const maxScrollDistance = 9999999;
272
- const scrollQuadrantDivisions = 4;
273
- const screenEdgeMargin = 50;
274
- const debugDevice = getDebug('harmony:device');
275
- let screenshotResizeScaleWarned = false;
276
- const INPUT_FIELD_TYPES = new Set([
277
- 'TextInput',
278
- 'TextArea',
279
- 'SearchField'
280
- ]);
281
- function parseBounds(raw) {
282
- if ('string' != typeof raw) return null;
283
- const m = raw.match(/\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/);
284
- if (!m) return null;
285
- return {
286
- x1: Number(m[1]),
287
- y1: Number(m[2]),
288
- x2: Number(m[3]),
289
- y2: Number(m[4])
290
- };
291
- }
292
- function collectInputFields(layout) {
293
- const fields = [];
294
- const visit = (node)=>{
295
- if (!node || 'object' != typeof node) return;
296
- const n = node;
297
- const attrs = n.attributes ?? {};
298
- if (INPUT_FIELD_TYPES.has(String(attrs.type))) {
299
- const bounds = parseBounds(attrs.bounds);
300
- if (bounds) fields.push({
301
- text: String(attrs.text ?? ''),
302
- bounds
303
- });
304
- }
305
- for (const child of n.children ?? [])visit(child);
306
- };
307
- visit(layout);
308
- return fields;
309
- }
310
- function pickFieldByPoint(fields, point) {
311
- const [px, py] = point;
312
- return fields.find(({ bounds: b })=>px >= b.x1 && px <= b.x2 && py >= b.y1 && py <= b.y2);
313
- }
314
- function pickLongestField(fields) {
315
- return fields.reduce((a, b)=>b.text.length > a.text.length ? b : a);
316
- }
317
- const harmonyKeyCodeMap = {
318
- Enter: '2054',
319
- Backspace: '2055',
320
- Tab: '2049',
321
- Escape: '2070',
322
- Home: 'Home',
323
- ArrowUp: '2012',
324
- ArrowDown: '2013',
325
- ArrowLeft: '2014',
326
- ArrowRight: '2015',
327
- Space: '2050',
328
- Delete: '2071'
329
- };
330
- const keyNameAliasMap = {
331
- enter: 'Enter',
332
- backspace: 'Backspace',
333
- tab: 'Tab',
334
- escape: 'Escape',
335
- esc: 'Escape',
336
- home: 'Home',
337
- space: 'Space',
338
- delete: 'Delete',
339
- arrowup: 'ArrowUp',
340
- arrowdown: 'ArrowDown',
341
- arrowleft: 'ArrowLeft',
342
- arrowright: 'ArrowRight',
343
- up: 'ArrowUp',
344
- down: 'ArrowDown',
345
- left: 'ArrowLeft',
346
- right: 'ArrowRight'
347
- };
348
- class HarmonyDevice {
349
- actionSpace() {
350
- const mobileActionContext = {
351
- input: this.inputPrimitives,
352
- size: ()=>this.size(),
353
- sleep: async (timeMs)=>{
354
- await sleep(timeMs);
355
- }
356
- };
357
- const defaultActions = [
358
- ...createDefaultMobileActions(mobileActionContext)
359
- ];
360
- const platformSpecificActions = Object.values(createPlatformActions(this));
361
- const customActions = this.customActions ?? [];
362
- return [
363
- ...defaultActions,
364
- ...platformSpecificActions,
365
- ...customActions
366
- ];
367
- }
368
- async performActionScroll(param) {
369
- const element = param.locate;
370
- const startingPoint = element ? {
371
- left: element.center[0],
372
- top: element.center[1]
373
- } : void 0;
374
- const scrollToEventName = param?.scrollType;
375
- if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
376
- else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
377
- else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
378
- else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
379
- else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
380
- else {
381
- if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance ?? void 0, startingPoint);
382
- else if ('left' === param.direction) await this.scrollLeft(param.distance ?? void 0, startingPoint);
383
- else if ('right' === param.direction) await this.scrollRight(param.distance ?? void 0, startingPoint);
384
- else throw new Error(`Unknown scroll direction: ${param.direction}`);
385
- else await this.scrollDown(param?.distance ?? void 0, startingPoint);
386
- await sleep(500);
387
- }
388
- }
389
- describe() {
390
- return this.descriptionText || `DeviceId: ${this.deviceId}`;
391
- }
392
- async connect() {
393
- const hdc = await this.getHdc();
394
- return hdc;
395
- }
396
- async getHdc() {
397
- if (this.destroyed) throw new Error(`HarmonyDevice ${this.deviceId} has been destroyed and cannot execute HDC commands`);
398
- if (this.hdc) return this.hdc;
399
- if (this.connecting) return this.connecting;
400
- this.connecting = (async ()=>{
401
- debugDevice(`Initializing HDC with device ID: ${this.deviceId}`);
402
- try {
403
- this.hdc = new HdcClient({
404
- hdcPath: this.options?.hdcPath,
405
- deviceId: this.deviceId
406
- });
407
- const screenInfo = await this.hdc.getScreenInfo();
408
- this.cachedScreenSize = screenInfo;
409
- this.descriptionText = `DeviceId: ${this.deviceId}\nScreenSize: ${screenInfo.width}x${screenInfo.height}`;
410
- debugDevice('HDC initialized successfully', this.descriptionText);
411
- return this.hdc;
412
- } catch (e) {
413
- debugDevice(`Failed to initialize HDC: ${e}`);
414
- throw new Error(`Unable to connect to device ${this.deviceId}: ${e}`);
415
- } finally{
416
- this.connecting = null;
417
- }
418
- })();
419
- return this.connecting;
420
- }
421
- setAppNameMapping(mapping) {
422
- this.appNameMapping = mapping;
423
- }
424
- resolvePackageName(appName) {
425
- const normalizedAppName = normalizeForComparison(appName);
426
- return this.appNameMapping[normalizedAppName];
427
- }
428
- async launch(uri) {
429
- const hdc = await this.getHdc();
430
- this.uri = uri;
431
- try {
432
- debugDevice(`Launching app: ${uri}`);
433
- if (uri.startsWith('http://') || uri.startsWith('https://') || uri.includes('://')) {
434
- const sanitizedUri = uri.replace(/[`$\\;"'|&<>(){}]/g, '');
435
- await hdc.shell(`aa start -U ${sanitizedUri}`);
436
- } else if (uri.includes('/')) {
437
- const [bundleName, abilityName] = uri.split('/');
438
- await hdc.startAbility(bundleName, abilityName);
439
- } else {
440
- const bundleName = this.resolvePackageName(uri) ?? uri;
441
- try {
442
- await hdc.startAbility(bundleName, 'EntryAbility');
443
- } catch (e) {
444
- if (!e.message?.includes('resolve ability')) throw e;
445
- const mainAbility = await hdc.queryMainAbility(bundleName);
446
- if (!mainAbility) throw new Error(`Cannot find a launchable ability for ${bundleName}`);
447
- debugDevice(`EntryAbility not found, using discovered ability: ${mainAbility}`);
448
- await hdc.startAbility(bundleName, mainAbility);
449
- }
450
- }
451
- debugDevice(`Successfully launched: ${uri}`);
452
- } catch (error) {
453
- debugDevice(`Error launching ${uri}: ${error}`);
454
- throw new Error(`Failed to launch ${uri}: ${error.message}`, {
455
- cause: error
456
- });
457
- }
458
- return this;
459
- }
460
- async terminate(uri) {
461
- const bundlePart = uri.includes('/') ? uri.split('/')[0] : uri;
462
- const resolved = this.resolvePackageName(bundlePart) ?? bundlePart;
463
- const hdc = await this.getHdc();
464
- try {
465
- debugDevice(`Terminating app: ${resolved}`);
466
- await hdc.forceStop(resolved);
467
- debugDevice(`Successfully terminated: ${resolved}`);
468
- } catch (error) {
469
- debugDevice(`Error terminating ${resolved}: ${error}`);
470
- throw new Error(`Failed to terminate ${resolved}: ${error.message}`, {
471
- cause: error
472
- });
473
- }
474
- }
475
- async getScreenSize() {
476
- if (this.cachedScreenSize) return this.cachedScreenSize;
477
- const hdc = await this.getHdc();
478
- const screenInfo = await hdc.getScreenInfo();
479
- this.cachedScreenSize = screenInfo;
480
- return screenInfo;
481
- }
482
- async size() {
483
- const screenInfo = await this.getScreenSize();
484
- return {
485
- width: screenInfo.width,
486
- height: screenInfo.height
487
- };
488
- }
489
- async screenshotBase64() {
490
- debugDevice('screenshotBase64 begin');
491
- const hdc = await this.getHdc();
492
- if (!this.localScreenshotPath) this.localScreenshotPath = getTmpFile('jpeg');
493
- const maxAttempts = 2;
494
- for(let attempt = 1; attempt <= maxAttempts; attempt++){
495
- const snapshotOutput = await hdc.screenshot(this.remoteScreenshotPath);
496
- const dimMatch = snapshotOutput.match(/width\s+(\d+),\s*height\s+(\d+)/);
497
- if (dimMatch) {
498
- const w = Number.parseInt(dimMatch[1], 10);
499
- const h = Number.parseInt(dimMatch[2], 10);
500
- if (this.cachedScreenSize && (this.cachedScreenSize.width !== w || this.cachedScreenSize.height !== h)) {
501
- debugDevice(`Screen size changed: ${this.cachedScreenSize.width}x${this.cachedScreenSize.height} -> ${w}x${h}`);
502
- this.cachedScreenSize = {
503
- width: w,
504
- height: h
505
- };
506
- }
507
- }
508
- await hdc.fileRecv(this.remoteScreenshotPath, this.localScreenshotPath);
509
- const screenshotBuffer = await node_fs.promises.readFile(this.localScreenshotPath);
510
- if (screenshotBuffer && screenshotBuffer.length > 0) {
511
- debugDevice(`Screenshot captured: ${screenshotBuffer.length} bytes`);
512
- return createImgBase64ByFormat('jpeg', screenshotBuffer.toString('base64'));
513
- }
514
- debugDevice(`Screenshot buffer empty (attempt ${attempt}/${maxAttempts})`);
515
- if (attempt < maxAttempts) await sleep(200);
516
- }
517
- throw new Error('Screenshot buffer is empty after retries');
518
- }
519
- async tapPoint(point) {
520
- this.lastTapPosition = {
521
- x: point.x,
522
- y: point.y
523
- };
524
- const hdc = await this.getHdc();
525
- await hdc.click(point.x, point.y);
526
- }
527
- async doubleTapPoint(point) {
528
- const hdc = await this.getHdc();
529
- await hdc.doubleClick(point.x, point.y);
530
- }
531
- async longPressPoint(point) {
532
- const hdc = await this.getHdc();
533
- await hdc.longClick(point.x, point.y);
534
- }
535
- async typeText(text, element, shouldReplace, options) {
536
- if (!text) return;
537
- const hdc = await this.getHdc();
538
- let x;
539
- let y;
540
- if (element) [x, y] = element.center;
541
- else if (this.lastTapPosition) {
542
- x = this.lastTapPosition.x;
543
- y = this.lastTapPosition.y;
544
- } else {
545
- const { width, height } = await this.size();
546
- x = Math.round(width / 2);
547
- y = Math.round(height / 2);
548
- }
549
- if (shouldReplace) {
550
- await hdc.click(x, y);
551
- await sleep(100);
552
- const length = await this.resolveClearLength(element);
553
- if (length > 0) await hdc.clearTextField(length);
554
- await sleep(100);
555
- }
556
- await hdc.inputText(x, y, text);
557
- const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
558
- if (shouldAutoDismissKeyboard) await this.hideKeyboard(options);
559
- }
560
- async clearInput(element) {
561
- const hdc = await this.getHdc();
562
- if (element) {
563
- await hdc.click(element.center[0], element.center[1]);
564
- await sleep(100);
565
- }
566
- const length = await this.resolveClearLength(element);
567
- if (length > 0) await hdc.clearTextField(length);
568
- }
569
- async resolveClearLength(element) {
570
- const PADDING = 2;
571
- const FALLBACK_LENGTH = 100;
572
- try {
573
- const hdc = await this.getHdc();
574
- const layoutJson = await hdc.dumpLayout();
575
- const layout = JSON.parse(layoutJson);
576
- const fields = collectInputFields(layout);
577
- if (0 === fields.length) return FALLBACK_LENGTH;
578
- const target = element ? pickFieldByPoint(fields, element.center) ?? pickLongestField(fields) : pickLongestField(fields);
579
- return target.text.length + PADDING;
580
- } catch (e) {
581
- debugDevice(`resolveClearLength: layout probe failed, falling back to ${FALLBACK_LENGTH}: ${e}`);
582
- return FALLBACK_LENGTH;
583
- }
584
- }
585
- async pressKey(key) {
586
- const normalizedKey = keyNameAliasMap[key.toLowerCase()] ?? key;
587
- const harmonyKey = harmonyKeyCodeMap[normalizedKey] ?? key;
588
- const hdc = await this.getHdc();
589
- await hdc.keyEvent(harmonyKey);
590
- }
591
- async scroll(deltaX, deltaY, speed) {
592
- if (0 === deltaX && 0 === deltaY) throw new Error('Scroll distance cannot be zero in both directions');
593
- const { width, height } = await this.size();
594
- const n = scrollQuadrantDivisions;
595
- const startX = Math.round(deltaX < 0 ? width / n * (n - 1) : width / n);
596
- const startY = Math.round(deltaY < 0 ? height / n * (n - 1) : height / n);
597
- const maxPositiveDeltaX = startX;
598
- const maxNegativeDeltaX = width - startX;
599
- const maxPositiveDeltaY = startY;
600
- const maxNegativeDeltaY = height - startY;
601
- deltaX = Math.max(-maxNegativeDeltaX, Math.min(deltaX, maxPositiveDeltaX));
602
- deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY));
603
- const endX = Math.round(Math.max(screenEdgeMargin, Math.min(width - screenEdgeMargin, startX - deltaX)));
604
- const endY = Math.round(Math.max(screenEdgeMargin, Math.min(height - screenEdgeMargin, startY - deltaY)));
605
- const hdc = await this.getHdc();
606
- await hdc.fling(startX, startY, endX, endY, speed ?? defaultFastSwipeSpeed);
607
- }
608
- async scrollInDirection(direction, distance, startPoint) {
609
- const { width, height } = await this.size();
610
- const isVertical = 'up' === direction || 'down' === direction;
611
- const scrollDistance = Math.round(distance ?? (isVertical ? height : width));
612
- if (startPoint) {
613
- const hdc = await this.getHdc();
614
- const sx = Math.round(startPoint.left);
615
- const sy = Math.round(startPoint.top);
616
- const endPoints = {
617
- down: {
618
- x: sx,
619
- y: Math.max(screenEdgeMargin, sy - scrollDistance)
620
- },
621
- up: {
622
- x: sx,
623
- y: Math.min(height - screenEdgeMargin, sy + scrollDistance)
624
- },
625
- left: {
626
- x: Math.min(width - screenEdgeMargin, sx + scrollDistance),
627
- y: sy
628
- },
629
- right: {
630
- x: Math.max(screenEdgeMargin, sx - scrollDistance),
631
- y: sy
632
- }
633
- };
634
- const end = endPoints[direction];
635
- await hdc.fling(sx, sy, end.x, end.y, defaultFastSwipeSpeed);
636
- return;
637
- }
638
- const deltas = {
639
- down: [
640
- 0,
641
- scrollDistance
642
- ],
643
- up: [
644
- 0,
645
- -scrollDistance
646
- ],
647
- left: [
648
- -scrollDistance,
649
- 0
650
- ],
651
- right: [
652
- scrollDistance,
653
- 0
654
- ]
655
- };
656
- const [dx, dy] = deltas[direction];
657
- await this.scroll(dx, dy);
658
- }
659
- async scrollDown(distance, startPoint) {
660
- await this.scrollInDirection('down', distance, startPoint);
661
- }
662
- async scrollUp(distance, startPoint) {
663
- await this.scrollInDirection('up', distance, startPoint);
664
- }
665
- async scrollLeft(distance, startPoint) {
666
- await this.scrollInDirection('left', distance, startPoint);
667
- }
668
- async scrollRight(distance, startPoint) {
669
- await this.scrollInDirection('right', distance, startPoint);
670
- }
671
- async scrollUntilEdge(direction, startPoint) {
672
- if (startPoint) {
673
- const { width, height } = await this.size();
674
- const hdc = await this.getHdc();
675
- const sx = Math.round(startPoint.left);
676
- const sy = Math.round(startPoint.top);
677
- const flingTargets = {
678
- up: {
679
- x: sx,
680
- y: Math.round(height) - screenEdgeMargin
681
- },
682
- down: {
683
- x: sx,
684
- y: screenEdgeMargin
685
- },
686
- left: {
687
- x: Math.round(width) - screenEdgeMargin,
688
- y: sy
689
- },
690
- right: {
691
- x: screenEdgeMargin,
692
- y: sy
693
- }
694
- };
695
- const target = flingTargets[direction];
696
- await repeat(defaultScrollUntilTimes, ()=>hdc.fling(sx, sy, target.x, target.y, defaultFastSwipeSpeed));
697
- await sleep(1000);
698
- return;
699
- }
700
- const deltas = {
701
- up: [
702
- 0,
703
- -maxScrollDistance
704
- ],
705
- down: [
706
- 0,
707
- maxScrollDistance
708
- ],
709
- left: [
710
- -maxScrollDistance,
711
- 0
712
- ],
713
- right: [
714
- maxScrollDistance,
715
- 0
716
- ]
717
- };
718
- const [dx, dy] = deltas[direction];
719
- await repeat(defaultScrollUntilTimes, ()=>this.scroll(dx, dy, defaultFastSwipeSpeed));
720
- await sleep(1000);
721
- }
722
- async scrollUntilTop(startPoint) {
723
- await this.scrollUntilEdge('up', startPoint);
724
- }
725
- async scrollUntilBottom(startPoint) {
726
- await this.scrollUntilEdge('down', startPoint);
727
- }
728
- async scrollUntilLeft(startPoint) {
729
- await this.scrollUntilEdge('left', startPoint);
730
- }
731
- async scrollUntilRight(startPoint) {
732
- await this.scrollUntilEdge('right', startPoint);
733
- }
734
- async back() {
735
- const hdc = await this.getHdc();
736
- await hdc.keyEvent('Back');
737
- }
738
- async home() {
739
- const hdc = await this.getHdc();
740
- await hdc.keyEvent('Home');
741
- }
742
- async recentApps() {
743
- const hdc = await this.getHdc();
744
- await hdc.keyEvent('RecentApps');
745
- }
746
- async hideKeyboard(options) {
747
- const hdc = await this.getHdc();
748
- const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
749
- const key = 'back-first' === keyboardDismissStrategy ? 'Back' : harmonyKeyCodeMap.Escape;
750
- await hdc.keyEvent(key);
751
- }
752
- async getDeviceLocalTimeString(format = 'YYYY-MM-DD HH:mm:ss') {
753
- const hdc = await this.getHdc();
754
- try {
755
- const stdout = await hdc.shell('date +%Y-%m-%dT%H:%M:%S');
756
- const match = stdout.trim().match(/^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})$/);
757
- if (!match) throw new Error(`Invalid device time format: ${stdout}`);
758
- const [, year, month, day, hours, minutes, seconds] = match;
759
- const timeString = format.replace('YYYY', year).replace('MM', month).replace('DD', day).replace('HH', hours).replace('mm', minutes).replace('ss', seconds);
760
- debugDevice(`Got device local time: ${timeString}`);
761
- return `${timeString} (${format})`;
762
- } catch (error) {
763
- debugDevice(`Failed to get device local time: ${error}`);
764
- throw new Error(`Failed to get device local time: ${error}`);
765
- }
766
- }
767
- async destroy() {
768
- if (this.destroyed) return;
769
- this.destroyed = true;
770
- this.cachedScreenSize = null;
771
- this.hdc = null;
772
- this.connecting = null;
773
- }
774
- constructor(deviceId, options){
775
- device_define_property(this, "deviceId", void 0);
776
- device_define_property(this, "hdc", null);
777
- device_define_property(this, "connecting", null);
778
- device_define_property(this, "destroyed", false);
779
- device_define_property(this, "descriptionText", void 0);
780
- device_define_property(this, "customActions", void 0);
781
- device_define_property(this, "cachedScreenSize", null);
782
- device_define_property(this, "appNameMapping", {});
783
- device_define_property(this, "lastTapPosition", null);
784
- device_define_property(this, "interfaceType", 'harmony');
785
- device_define_property(this, "uri", void 0);
786
- device_define_property(this, "options", void 0);
787
- device_define_property(this, "inputPrimitives", {
788
- pointer: {
789
- tap: (point)=>this.tapPoint(point),
790
- doubleClick: (point)=>this.doubleTapPoint(point),
791
- longPress: (point)=>this.longPressPoint(point),
792
- dragAndDrop: async (from, to)=>{
793
- const hdc = await this.getHdc();
794
- await hdc.drag(from.x, from.y, to.x, to.y);
795
- }
796
- },
797
- keyboard: {
798
- keyboardPress: (keyName)=>this.pressKey(keyName),
799
- typeText: (value, opts)=>{
800
- const harmonyOpts = opts;
801
- return harmonyOpts?.focusOnly ? Promise.resolve() : this.typeText(value, harmonyOpts?.target, harmonyOpts?.replace ?? true, {
802
- autoDismissKeyboard: harmonyOpts?.autoDismissKeyboard,
803
- keyboardDismissStrategy: harmonyOpts?.keyboardDismissStrategy
804
- });
805
- },
806
- clearInput: (target)=>this.clearInput(target),
807
- cursorMove: async (direction, times = 1)=>{
808
- const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
809
- for(let i = 0; i < times; i++)await this.pressKey(arrowKey);
810
- }
811
- },
812
- touch: {
813
- swipe: async (start, end, opts)=>{
814
- const duration = opts?.duration;
815
- const repeatCount = opts?.repeat ?? 1;
816
- const hdc = await this.getHdc();
817
- for(let i = 0; i < repeatCount; i++)await hdc.swipe(start.x, start.y, end.x, end.y, duration ? Math.round(duration) : void 0);
818
- }
819
- },
820
- scroll: {
821
- scroll: (param)=>this.performActionScroll(param)
822
- }
823
- });
824
- device_define_property(this, "remoteScreenshotPath", '/data/local/tmp/ms_screen.jpeg');
825
- device_define_property(this, "localScreenshotPath", null);
826
- node_assert(deviceId, 'deviceId is required for HarmonyDevice');
827
- this.deviceId = deviceId;
828
- this.options = options;
829
- this.customActions = options?.customActions;
830
- if (options?.screenshotResizeScale !== void 0 && !screenshotResizeScaleWarned) {
831
- screenshotResizeScaleWarned = true;
832
- console.warn('[midscene] screenshotResizeScale is deprecated. Use screenshotShrinkFactor in AgentOpt instead.');
833
- }
834
- }
835
- }
836
- const runHdcShellParamSchema = z.object({
837
- command: z.string().describe('HDC shell command to execute')
838
- });
839
- const launchParamSchema = z.object({
840
- uri: z.string().describe('App name, bundle name, or URL to launch. Prioritize using the exact bundle name or URL the user has provided. If none provided, use the accurate app name.')
841
- });
842
- const terminateParamSchema = z.object({
843
- uri: z.string().describe('Bundle name or app name to terminate. Prioritize using the exact bundle name the user provided. If the bundle is unknown, use the accurate app name shown on screen, such as Settings or Music.')
844
- });
845
- const createPlatformActions = (device)=>({
846
- RunHdcShell: defineAction({
847
- name: 'RunHdcShell',
848
- description: 'Execute HDC shell command on HarmonyOS device',
849
- interfaceAlias: 'runHdcShell',
850
- paramSchema: runHdcShellParamSchema,
851
- sample: {
852
- command: 'hidumper -s WindowManagerService -a'
853
- },
854
- call: async (param)=>{
855
- if (!param.command || '' === param.command.trim()) throw new Error('RunHdcShell requires a non-empty command parameter');
856
- const hdc = await device.getHdc();
857
- return await hdc.shell(param.command);
858
- }
859
- }),
860
- Launch: defineAction({
861
- name: 'Launch',
862
- description: 'Launch a HarmonyOS app or URL',
863
- interfaceAlias: 'launch',
864
- paramSchema: launchParamSchema,
865
- sample: {
866
- uri: 'com.example.app'
867
- },
868
- call: async (param)=>{
869
- if (!param.uri || '' === param.uri.trim()) throw new Error('Launch requires a non-empty uri parameter');
870
- await device.launch(param.uri);
871
- }
872
- }),
873
- Terminate: defineAction({
874
- name: 'Terminate',
875
- description: 'Terminate (force-stop) a HarmonyOS app by bundle name or mapped app name',
876
- interfaceAlias: 'terminate',
877
- paramSchema: terminateParamSchema,
878
- call: async (param)=>{
879
- if (!param.uri || '' === param.uri.trim()) throw new Error('Terminate requires a non-empty uri parameter');
880
- await device.terminate(param.uri);
881
- }
882
- }),
883
- HarmonyBackButton: defineAction({
884
- name: 'HarmonyBackButton',
885
- description: 'Trigger the system "back" operation on HarmonyOS devices',
886
- call: async ()=>{
887
- await device.back();
888
- }
889
- }),
890
- HarmonyHomeButton: defineAction({
891
- name: 'HarmonyHomeButton',
892
- description: 'Trigger the system "home" operation on HarmonyOS devices',
893
- call: async ()=>{
894
- await device.home();
895
- }
896
- }),
897
- HarmonyRecentAppsButton: defineAction({
898
- name: 'HarmonyRecentAppsButton',
899
- description: 'Trigger the system "recent apps" operation on HarmonyOS devices',
900
- call: async ()=>{
901
- await device.recentApps();
902
- }
903
- })
904
- });
905
- const debugUtils = getDebug('harmony:utils');
906
- async function getConnectedDevices(hdcPath, options = {}) {
907
- try {
908
- const hdc = new HdcClient({
909
- hdcPath,
910
- timeout: options.timeout
911
- });
912
- const targets = await hdc.listTargets();
913
- const devices = targets.map((deviceId)=>({
914
- deviceId
915
- }));
916
- debugUtils(`Found ${devices.length} connected devices: `, devices);
917
- return devices;
918
- } catch (error) {
919
- console.error('Failed to get device list:', error);
920
- throw new Error(`Unable to get connected HarmonyOS device list, please ensure HDC is properly configured: ${error.message}`, {
921
- cause: error
922
- });
923
- }
924
- }
925
- function agent_define_property(obj, key, value) {
926
- if (key in obj) Object.defineProperty(obj, key, {
927
- value: value,
928
- enumerable: true,
929
- configurable: true,
930
- writable: true
931
- });
932
- else obj[key] = value;
933
- return obj;
934
- }
935
- const debugAgent = getDebug('harmony:agent');
936
- class HarmonyAgent extends Agent {
937
- async launch(uri) {
938
- const action = this.wrapActionInActionSpace('Launch');
939
- return action({
940
- uri
941
- });
942
- }
943
- async terminate(uri) {
944
- const action = this.wrapActionInActionSpace('Terminate');
945
- return action({
946
- uri
947
- });
948
- }
949
- async runHdcShell(command) {
950
- const action = this.wrapActionInActionSpace('RunHdcShell');
951
- return action({
952
- command
953
- });
954
- }
955
- createActionWrapper(name) {
956
- const action = this.wrapActionInActionSpace(name);
957
- return (...args)=>action(args[0]);
958
- }
959
- constructor(device, opts){
960
- super(device, opts), agent_define_property(this, "back", void 0), agent_define_property(this, "home", void 0), agent_define_property(this, "recentApps", void 0), agent_define_property(this, "appNameMapping", void 0);
961
- this.appNameMapping = mergeAndNormalizeAppNameMapping(defaultAppNameMapping, opts?.appNameMapping);
962
- device.setAppNameMapping(this.appNameMapping);
963
- this.back = this.createActionWrapper('HarmonyBackButton');
964
- this.home = this.createActionWrapper('HarmonyHomeButton');
965
- this.recentApps = this.createActionWrapper('HarmonyRecentAppsButton');
966
- }
967
- }
968
- async function agentFromHdcDevice(deviceId, opts) {
969
- if (!deviceId) {
970
- const devices = await getConnectedDevices(opts?.hdcPath);
971
- if (0 === devices.length) throw new Error('No HarmonyOS devices found. Please connect a HarmonyOS device and ensure HDC is properly configured. Run `hdc list targets` to verify device connection.');
972
- deviceId = devices[0].deviceId;
973
- debugAgent('deviceId not specified, will use the first device (id = %s)', deviceId);
974
- }
975
- const device = new HarmonyDevice(deviceId, opts || {});
976
- await device.connect();
977
- return new HarmonyAgent(device, opts);
978
- }
979
- function mcp_tools_define_property(obj, key, value) {
980
- if (key in obj) Object.defineProperty(obj, key, {
981
- value: value,
982
- enumerable: true,
983
- configurable: true,
984
- writable: true
985
- });
986
- else obj[key] = value;
987
- return obj;
988
- }
989
- const debug = getDebug('mcp:harmony-tools');
990
- function adaptHarmonyInitArgs(extracted) {
991
- if (!extracted) return;
992
- const initArgs = {
993
- ...'string' == typeof extracted.deviceId ? {
994
- deviceId: extracted.deviceId
995
- } : {},
996
- ...extractAgentBehaviorInitArgs(extracted) ?? {}
997
- };
998
- return Object.keys(initArgs).length > 0 ? initArgs : void 0;
999
- }
1000
- class HarmonyMidsceneTools extends BaseMidsceneTools {
1001
- getCliReportSessionName() {
1002
- return 'midscene-harmony';
1003
- }
1004
- createTemporaryDevice() {
1005
- return new HarmonyDevice('temp-for-action-space', {});
1006
- }
1007
- async ensureAgent(initArgs) {
1008
- const deviceId = initArgs?.deviceId;
1009
- const nextSignature = getAgentInitArgsSignature(initArgs);
1010
- if (this.agent && shouldRebuildAgentForInitArgs(this.lastInitArgsSignature, nextSignature)) {
1011
- try {
1012
- await this.agent.destroy?.();
1013
- } catch (error) {
1014
- debug('Failed to destroy agent during cleanup:', error);
1015
- }
1016
- this.agent = void 0;
1017
- }
1018
- if (this.agent) return this.agent;
1019
- debug('Creating Harmony agent with deviceId:', deviceId || 'auto-detect');
1020
- const reportOptions = this.readCliReportAgentOptions();
1021
- const agent = await agentFromHdcDevice(deviceId, {
1022
- autoDismissKeyboard: false,
1023
- ...extractAgentBehaviorInitArgs(initArgs) ?? {},
1024
- ...reportOptions ?? {}
1025
- });
1026
- this.agent = agent;
1027
- this.lastInitArgsSignature = nextSignature;
1028
- return agent;
1029
- }
1030
- preparePlatformTools() {
1031
- return [
1032
- {
1033
- name: 'harmony_connect',
1034
- description: 'Connect to HarmonyOS device via HDC. If deviceId not provided, uses the first available device.',
1035
- schema: this.getAgentInitArgSchema(),
1036
- cli: this.getAgentInitArgCliMetadata(),
1037
- handler: async (args)=>{
1038
- const initArgs = this.extractAgentInitParam(args);
1039
- const deviceId = initArgs?.deviceId;
1040
- const reportSession = this.createNewCliReportSession(deviceId ?? 'auto');
1041
- this.commitCliReportSession(reportSession);
1042
- if (this.agent) {
1043
- try {
1044
- await this.agent.destroy?.();
1045
- } catch (error) {
1046
- debug('Failed to destroy agent during connect:', error);
1047
- }
1048
- this.agent = void 0;
1049
- this.lastInitArgsSignature = void 0;
1050
- }
1051
- const agent = await this.ensureAgent(initArgs);
1052
- const screenshot = await agent.page.screenshotBase64();
1053
- return {
1054
- content: [
1055
- {
1056
- type: 'text',
1057
- text: `Connected to HarmonyOS device${deviceId ? `: ${deviceId}` : ' (auto-detected)'}`
1058
- },
1059
- ...this.buildScreenshotContent(screenshot)
1060
- ],
1061
- isError: false
1062
- };
1063
- }
1064
- },
1065
- {
1066
- name: 'harmony_disconnect',
1067
- description: 'Disconnect from current HarmonyOS device and release HDC resources',
1068
- schema: {},
1069
- handler: this.createDisconnectHandler('HarmonyOS device')
1070
- }
1071
- ];
1072
- }
1073
- constructor(...args){
1074
- super(...args), mcp_tools_define_property(this, "lastInitArgsSignature", void 0), mcp_tools_define_property(this, "initArgSpec", {
1075
- namespace: 'harmony',
1076
- shape: {
1077
- deviceId: z.string().optional().describe('HarmonyOS device ID (from hdc list targets)'),
1078
- ...agentBehaviorInitArgShape
1079
- },
1080
- cli: {
1081
- preferBareKeys: true
1082
- },
1083
- adapt: adaptHarmonyInitArgs
1084
- });
1085
- }
1086
- }
1087
- class HarmonyMCPServer extends BaseMCPServer {
1088
- createToolsManager() {
1089
- return new HarmonyMidsceneTools();
1090
- }
1091
- constructor(toolsManager){
1092
- super({
1093
- name: '@midscene/harmony-mcp',
1094
- version: "1.9.8",
1095
- description: 'Control the HarmonyOS device using natural language commands'
1096
- }, toolsManager);
1097
- }
1098
- }
1099
- function mcpServerForAgent(agent) {
1100
- return createMCPServerLauncher({
1101
- agent,
1102
- platformName: 'HarmonyOS',
1103
- ToolsManagerClass: HarmonyMidsceneTools,
1104
- MCPServerClass: HarmonyMCPServer
1105
- });
1106
- }
1107
- async function mcpKitForAgent(agent) {
1108
- const toolsManager = new HarmonyMidsceneTools();
1109
- toolsManager.setAgent(agent);
1110
- await toolsManager.initTools();
1111
- return {
1112
- description: 'Midscene MCP Kit for HarmonyOS automation',
1113
- tools: toolsManager.getToolDefinitions()
1114
- };
1115
- }
1116
- export { HarmonyMCPServer, mcpKitForAgent, mcpServerForAgent };