@midscene/computer 1.2.3-beta-20260122071913.0 → 1.2.3-beta-20260122082712.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/es/index.mjs CHANGED
@@ -5,6 +5,7 @@ import { actionHoverParamSchema, defineAction, defineActionClearInput, defineAct
5
5
  import { sleep } from "@midscene/core/utils";
6
6
  import { createImgBase64ByFormat } from "@midscene/shared/img";
7
7
  import { getDebug } from "@midscene/shared/logger";
8
+ import clipboardy from "clipboardy";
8
9
  import screenshot_desktop from "screenshot-desktop";
9
10
  import { Agent } from "@midscene/core/agent";
10
11
  import { overrideAIConfig } from "@midscene/shared/env";
@@ -29,6 +30,8 @@ const INPUT_CLEAR_DELAY = 150;
29
30
  const SCROLL_REPEAT_COUNT = 10;
30
31
  const SCROLL_STEP_DELAY = 100;
31
32
  const SCROLL_COMPLETE_DELAY = 500;
33
+ const INPUT_STRATEGY_ALWAYS_CLIPBOARD = 'always-clipboard';
34
+ const INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII = 'clipboard-for-non-ascii';
32
35
  let device_libnut = null;
33
36
  let libnutLoadError = null;
34
37
  async function getLibnut() {
@@ -173,6 +176,40 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
173
176
  throw new Error(`Failed to get screen size: ${error}`);
174
177
  }
175
178
  }
179
+ shouldUseClipboardForText(text) {
180
+ const hasNonAscii = /[\x80-\uFFFF]/.test(text);
181
+ return hasNonAscii;
182
+ }
183
+ async typeViaClipboard(text) {
184
+ node_assert(device_libnut, 'libnut not initialized');
185
+ debugDevice('Using clipboard to input text', {
186
+ textLength: text.length,
187
+ preview: text.substring(0, 20)
188
+ });
189
+ const oldClipboard = await clipboardy.read().catch(()=>'');
190
+ try {
191
+ await clipboardy.write(text);
192
+ await sleep(50);
193
+ const modifier = 'darwin' === process.platform ? 'command' : 'control';
194
+ device_libnut.keyTap('v', [
195
+ modifier
196
+ ]);
197
+ await sleep(100);
198
+ } finally{
199
+ if (oldClipboard) await clipboardy.write(oldClipboard).catch(()=>{
200
+ debugDevice('Failed to restore clipboard content');
201
+ });
202
+ }
203
+ }
204
+ async smartTypeString(text) {
205
+ node_assert(device_libnut, 'libnut not initialized');
206
+ if ('darwin' === process.platform) return void device_libnut.typeString(text);
207
+ const inputStrategy = this.options?.inputStrategy ?? INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII;
208
+ if (inputStrategy === INPUT_STRATEGY_ALWAYS_CLIPBOARD) return void await this.typeViaClipboard(text);
209
+ const shouldUseClipboard = this.shouldUseClipboardForText(text);
210
+ if (shouldUseClipboard) await this.typeViaClipboard(text);
211
+ else device_libnut.typeString(text);
212
+ }
176
213
  actionSpace() {
177
214
  const defaultActions = [
178
215
  defineActionTap(async (param)=>{
@@ -250,7 +287,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
250
287
  }
251
288
  if ('clear' === param.mode) return;
252
289
  if (!param.value) return;
253
- device_libnut.typeString(param.value);
290
+ await this.smartTypeString(param.value);
254
291
  }
255
292
  }),
256
293
  defineActionScroll(async (param)=>{
@@ -7,6 +7,7 @@ import { actionHoverParamSchema, defineAction, defineActionClearInput, defineAct
7
7
  import { sleep } from "@midscene/core/utils";
8
8
  import { createImgBase64ByFormat } from "@midscene/shared/img";
9
9
  import { getDebug } from "@midscene/shared/logger";
10
+ import clipboardy from "clipboardy";
10
11
  import screenshot_desktop from "screenshot-desktop";
11
12
  function _define_property(obj, key, value) {
12
13
  if (key in obj) Object.defineProperty(obj, key, {
@@ -29,6 +30,8 @@ const INPUT_CLEAR_DELAY = 150;
29
30
  const SCROLL_REPEAT_COUNT = 10;
30
31
  const SCROLL_STEP_DELAY = 100;
31
32
  const SCROLL_COMPLETE_DELAY = 500;
33
+ const INPUT_STRATEGY_ALWAYS_CLIPBOARD = 'always-clipboard';
34
+ const INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII = 'clipboard-for-non-ascii';
32
35
  let libnut = null;
33
36
  let libnutLoadError = null;
34
37
  async function getLibnut() {
@@ -173,6 +176,40 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
173
176
  throw new Error(`Failed to get screen size: ${error}`);
174
177
  }
175
178
  }
179
+ shouldUseClipboardForText(text) {
180
+ const hasNonAscii = /[\x80-\uFFFF]/.test(text);
181
+ return hasNonAscii;
182
+ }
183
+ async typeViaClipboard(text) {
184
+ node_assert(libnut, 'libnut not initialized');
185
+ debugDevice('Using clipboard to input text', {
186
+ textLength: text.length,
187
+ preview: text.substring(0, 20)
188
+ });
189
+ const oldClipboard = await clipboardy.read().catch(()=>'');
190
+ try {
191
+ await clipboardy.write(text);
192
+ await sleep(50);
193
+ const modifier = 'darwin' === process.platform ? 'command' : 'control';
194
+ libnut.keyTap('v', [
195
+ modifier
196
+ ]);
197
+ await sleep(100);
198
+ } finally{
199
+ if (oldClipboard) await clipboardy.write(oldClipboard).catch(()=>{
200
+ debugDevice('Failed to restore clipboard content');
201
+ });
202
+ }
203
+ }
204
+ async smartTypeString(text) {
205
+ node_assert(libnut, 'libnut not initialized');
206
+ if ('darwin' === process.platform) return void libnut.typeString(text);
207
+ const inputStrategy = this.options?.inputStrategy ?? INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII;
208
+ if (inputStrategy === INPUT_STRATEGY_ALWAYS_CLIPBOARD) return void await this.typeViaClipboard(text);
209
+ const shouldUseClipboard = this.shouldUseClipboardForText(text);
210
+ if (shouldUseClipboard) await this.typeViaClipboard(text);
211
+ else libnut.typeString(text);
212
+ }
176
213
  actionSpace() {
177
214
  const defaultActions = [
178
215
  defineActionTap(async (param)=>{
@@ -250,7 +287,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
250
287
  }
251
288
  if ('clear' === param.mode) return;
252
289
  if (!param.value) return;
253
- libnut.typeString(param.value);
290
+ await this.smartTypeString(param.value);
254
291
  }
255
292
  }),
256
293
  defineActionScroll(async (param)=>{
package/dist/lib/index.js CHANGED
@@ -51,6 +51,8 @@ const device_namespaceObject = require("@midscene/core/device");
51
51
  const utils_namespaceObject = require("@midscene/core/utils");
52
52
  const img_namespaceObject = require("@midscene/shared/img");
53
53
  const logger_namespaceObject = require("@midscene/shared/logger");
54
+ const external_clipboardy_namespaceObject = require("clipboardy");
55
+ var external_clipboardy_default = /*#__PURE__*/ __webpack_require__.n(external_clipboardy_namespaceObject);
54
56
  const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
55
57
  var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
56
58
  function _define_property(obj, key, value) {
@@ -74,6 +76,8 @@ const INPUT_CLEAR_DELAY = 150;
74
76
  const SCROLL_REPEAT_COUNT = 10;
75
77
  const SCROLL_STEP_DELAY = 100;
76
78
  const SCROLL_COMPLETE_DELAY = 500;
79
+ const INPUT_STRATEGY_ALWAYS_CLIPBOARD = 'always-clipboard';
80
+ const INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII = 'clipboard-for-non-ascii';
77
81
  let device_libnut = null;
78
82
  let libnutLoadError = null;
79
83
  async function getLibnut() {
@@ -218,6 +222,40 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
218
222
  throw new Error(`Failed to get screen size: ${error}`);
219
223
  }
220
224
  }
225
+ shouldUseClipboardForText(text) {
226
+ const hasNonAscii = /[\x80-\uFFFF]/.test(text);
227
+ return hasNonAscii;
228
+ }
229
+ async typeViaClipboard(text) {
230
+ external_node_assert_default()(device_libnut, 'libnut not initialized');
231
+ debugDevice('Using clipboard to input text', {
232
+ textLength: text.length,
233
+ preview: text.substring(0, 20)
234
+ });
235
+ const oldClipboard = await external_clipboardy_default().read().catch(()=>'');
236
+ try {
237
+ await external_clipboardy_default().write(text);
238
+ await (0, utils_namespaceObject.sleep)(50);
239
+ const modifier = 'darwin' === process.platform ? 'command' : 'control';
240
+ device_libnut.keyTap('v', [
241
+ modifier
242
+ ]);
243
+ await (0, utils_namespaceObject.sleep)(100);
244
+ } finally{
245
+ if (oldClipboard) await external_clipboardy_default().write(oldClipboard).catch(()=>{
246
+ debugDevice('Failed to restore clipboard content');
247
+ });
248
+ }
249
+ }
250
+ async smartTypeString(text) {
251
+ external_node_assert_default()(device_libnut, 'libnut not initialized');
252
+ if ('darwin' === process.platform) return void device_libnut.typeString(text);
253
+ const inputStrategy = this.options?.inputStrategy ?? INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII;
254
+ if (inputStrategy === INPUT_STRATEGY_ALWAYS_CLIPBOARD) return void await this.typeViaClipboard(text);
255
+ const shouldUseClipboard = this.shouldUseClipboardForText(text);
256
+ if (shouldUseClipboard) await this.typeViaClipboard(text);
257
+ else device_libnut.typeString(text);
258
+ }
221
259
  actionSpace() {
222
260
  const defaultActions = [
223
261
  (0, device_namespaceObject.defineActionTap)(async (param)=>{
@@ -295,7 +333,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
295
333
  }
296
334
  if ('clear' === param.mode) return;
297
335
  if (!param.value) return;
298
- device_libnut.typeString(param.value);
336
+ await this.smartTypeString(param.value);
299
337
  }
300
338
  }),
301
339
  (0, device_namespaceObject.defineActionScroll)(async (param)=>{
@@ -50,6 +50,8 @@ const device_namespaceObject = require("@midscene/core/device");
50
50
  const utils_namespaceObject = require("@midscene/core/utils");
51
51
  const img_namespaceObject = require("@midscene/shared/img");
52
52
  const logger_namespaceObject = require("@midscene/shared/logger");
53
+ const external_clipboardy_namespaceObject = require("clipboardy");
54
+ var external_clipboardy_default = /*#__PURE__*/ __webpack_require__.n(external_clipboardy_namespaceObject);
53
55
  const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
54
56
  var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
55
57
  function _define_property(obj, key, value) {
@@ -73,6 +75,8 @@ const INPUT_CLEAR_DELAY = 150;
73
75
  const SCROLL_REPEAT_COUNT = 10;
74
76
  const SCROLL_STEP_DELAY = 100;
75
77
  const SCROLL_COMPLETE_DELAY = 500;
78
+ const INPUT_STRATEGY_ALWAYS_CLIPBOARD = 'always-clipboard';
79
+ const INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII = 'clipboard-for-non-ascii';
76
80
  let libnut = null;
77
81
  let libnutLoadError = null;
78
82
  async function getLibnut() {
@@ -217,6 +221,40 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
217
221
  throw new Error(`Failed to get screen size: ${error}`);
218
222
  }
219
223
  }
224
+ shouldUseClipboardForText(text) {
225
+ const hasNonAscii = /[\x80-\uFFFF]/.test(text);
226
+ return hasNonAscii;
227
+ }
228
+ async typeViaClipboard(text) {
229
+ external_node_assert_default()(libnut, 'libnut not initialized');
230
+ debugDevice('Using clipboard to input text', {
231
+ textLength: text.length,
232
+ preview: text.substring(0, 20)
233
+ });
234
+ const oldClipboard = await external_clipboardy_default().read().catch(()=>'');
235
+ try {
236
+ await external_clipboardy_default().write(text);
237
+ await (0, utils_namespaceObject.sleep)(50);
238
+ const modifier = 'darwin' === process.platform ? 'command' : 'control';
239
+ libnut.keyTap('v', [
240
+ modifier
241
+ ]);
242
+ await (0, utils_namespaceObject.sleep)(100);
243
+ } finally{
244
+ if (oldClipboard) await external_clipboardy_default().write(oldClipboard).catch(()=>{
245
+ debugDevice('Failed to restore clipboard content');
246
+ });
247
+ }
248
+ }
249
+ async smartTypeString(text) {
250
+ external_node_assert_default()(libnut, 'libnut not initialized');
251
+ if ('darwin' === process.platform) return void libnut.typeString(text);
252
+ const inputStrategy = this.options?.inputStrategy ?? INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII;
253
+ if (inputStrategy === INPUT_STRATEGY_ALWAYS_CLIPBOARD) return void await this.typeViaClipboard(text);
254
+ const shouldUseClipboard = this.shouldUseClipboardForText(text);
255
+ if (shouldUseClipboard) await this.typeViaClipboard(text);
256
+ else libnut.typeString(text);
257
+ }
220
258
  actionSpace() {
221
259
  const defaultActions = [
222
260
  (0, device_namespaceObject.defineActionTap)(async (param)=>{
@@ -294,7 +332,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
294
332
  }
295
333
  if ('clear' === param.mode) return;
296
334
  if (!param.value) return;
297
- libnut.typeString(param.value);
335
+ await this.smartTypeString(param.value);
298
336
  }
299
337
  }),
300
338
  (0, device_namespaceObject.defineActionScroll)(async (param)=>{
@@ -37,6 +37,26 @@ export declare class ComputerDevice implements AbstractInterface {
37
37
  connect(): Promise<void>;
38
38
  screenshotBase64(): Promise<string>;
39
39
  size(): Promise<Size>;
40
+ /**
41
+ * Check if text contains non-ASCII characters
42
+ * Matches: Chinese, Japanese, Korean, Latin extended characters (café, niño), emoji, etc.
43
+ */
44
+ private shouldUseClipboardForText;
45
+ /**
46
+ * Type text via clipboard (paste)
47
+ * This method:
48
+ * 1. Saves the old clipboard content
49
+ * 2. Writes new content to clipboard
50
+ * 3. Simulates paste shortcut (Ctrl+V / Cmd+V)
51
+ * 4. Restores old clipboard content
52
+ */
53
+ private typeViaClipboard;
54
+ /**
55
+ * Smart type string with platform-specific strategy
56
+ * - macOS: Always use libnut (native support for non-ASCII)
57
+ * - Windows/Linux: Use clipboard for non-ASCII characters
58
+ */
59
+ private smartTypeString;
40
60
  actionSpace(): DeviceAction<any>[];
41
61
  destroy(): Promise<void>;
42
62
  url(): Promise<string>;
@@ -45,6 +65,7 @@ export declare class ComputerDevice implements AbstractInterface {
45
65
  export declare interface ComputerDeviceOpt {
46
66
  displayId?: string;
47
67
  customActions?: DeviceAction<any>[];
68
+ inputStrategy?: 'always-clipboard' | 'clipboard-for-non-ascii';
48
69
  }
49
70
 
50
71
  export declare interface DisplayInfo {
@@ -29,6 +29,26 @@ declare class ComputerDevice implements AbstractInterface {
29
29
  connect(): Promise<void>;
30
30
  screenshotBase64(): Promise<string>;
31
31
  size(): Promise<Size>;
32
+ /**
33
+ * Check if text contains non-ASCII characters
34
+ * Matches: Chinese, Japanese, Korean, Latin extended characters (café, niño), emoji, etc.
35
+ */
36
+ private shouldUseClipboardForText;
37
+ /**
38
+ * Type text via clipboard (paste)
39
+ * This method:
40
+ * 1. Saves the old clipboard content
41
+ * 2. Writes new content to clipboard
42
+ * 3. Simulates paste shortcut (Ctrl+V / Cmd+V)
43
+ * 4. Restores old clipboard content
44
+ */
45
+ private typeViaClipboard;
46
+ /**
47
+ * Smart type string with platform-specific strategy
48
+ * - macOS: Always use libnut (native support for non-ASCII)
49
+ * - Windows/Linux: Use clipboard for non-ASCII characters
50
+ */
51
+ private smartTypeString;
32
52
  actionSpace(): DeviceAction<any>[];
33
53
  destroy(): Promise<void>;
34
54
  url(): Promise<string>;
@@ -37,6 +57,7 @@ declare class ComputerDevice implements AbstractInterface {
37
57
  declare interface ComputerDeviceOpt {
38
58
  displayId?: string;
39
59
  customActions?: DeviceAction<any>[];
60
+ inputStrategy?: 'always-clipboard' | 'clipboard-for-non-ascii';
40
61
  }
41
62
 
42
63
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@midscene/computer",
3
- "version": "1.2.3-beta-20260122071913.0",
3
+ "version": "1.2.3-beta-20260122082712.0",
4
4
  "description": "Midscene.js Computer Desktop Automation",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -26,9 +26,10 @@
26
26
  },
27
27
  "dependencies": {
28
28
  "@computer-use/libnut": "^4.2.0",
29
+ "clipboardy": "^4.0.0",
29
30
  "screenshot-desktop": "^1.15.3",
30
- "@midscene/core": "1.2.3-beta-20260122071913.0",
31
- "@midscene/shared": "1.2.3-beta-20260122071913.0"
31
+ "@midscene/core": "1.2.3-beta-20260122082712.0",
32
+ "@midscene/shared": "1.2.3-beta-20260122082712.0"
32
33
  },
33
34
  "devDependencies": {
34
35
  "@rslib/core": "^0.18.3",
package/src/device.ts CHANGED
@@ -25,6 +25,7 @@ import {
25
25
  import { sleep } from '@midscene/core/utils';
26
26
  import { createImgBase64ByFormat } from '@midscene/shared/img';
27
27
  import { getDebug } from '@midscene/shared/logger';
28
+ import clipboardy from 'clipboardy';
28
29
  import screenshot from 'screenshot-desktop';
29
30
 
30
31
  // Type definitions
@@ -63,6 +64,10 @@ const SCROLL_REPEAT_COUNT = 10;
63
64
  const SCROLL_STEP_DELAY = 100;
64
65
  const SCROLL_COMPLETE_DELAY = 500;
65
66
 
67
+ // Input strategy constants
68
+ const INPUT_STRATEGY_ALWAYS_CLIPBOARD = 'always-clipboard';
69
+ const INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII = 'clipboard-for-non-ascii';
70
+
66
71
  // Lazy load libnut with fallback
67
72
  let libnut: LibNut | null = null;
68
73
  let libnutLoadError: Error | null = null;
@@ -185,6 +190,7 @@ export interface ComputerDeviceOpt {
185
190
  displayId?: string;
186
191
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
187
192
  customActions?: DeviceAction<any>[];
193
+ inputStrategy?: 'always-clipboard' | 'clipboard-for-non-ascii';
188
194
  }
189
195
 
190
196
  export class ComputerDevice implements AbstractInterface {
@@ -289,6 +295,87 @@ Available Displays: ${displays.length > 0 ? displays.map((d) => d.name).join(',
289
295
  }
290
296
  }
291
297
 
298
+ /**
299
+ * Check if text contains non-ASCII characters
300
+ * Matches: Chinese, Japanese, Korean, Latin extended characters (café, niño), emoji, etc.
301
+ */
302
+ private shouldUseClipboardForText(text: string): boolean {
303
+ // Check for any character with code point >= 128 (non-ASCII)
304
+ const hasNonAscii = /[\x80-\uFFFF]/.test(text);
305
+ return hasNonAscii;
306
+ }
307
+
308
+ /**
309
+ * Type text via clipboard (paste)
310
+ * This method:
311
+ * 1. Saves the old clipboard content
312
+ * 2. Writes new content to clipboard
313
+ * 3. Simulates paste shortcut (Ctrl+V / Cmd+V)
314
+ * 4. Restores old clipboard content
315
+ */
316
+ private async typeViaClipboard(text: string): Promise<void> {
317
+ assert(libnut, 'libnut not initialized');
318
+ debugDevice('Using clipboard to input text', {
319
+ textLength: text.length,
320
+ preview: text.substring(0, 20),
321
+ });
322
+
323
+ // 1. Save old clipboard content
324
+ const oldClipboard = await clipboardy.read().catch(() => '');
325
+
326
+ try {
327
+ // 2. Write new content to clipboard
328
+ await clipboardy.write(text);
329
+ await sleep(50);
330
+
331
+ // 3. Simulate paste shortcut
332
+ const modifier = process.platform === 'darwin' ? 'command' : 'control';
333
+ libnut.keyTap('v', [modifier]);
334
+ await sleep(100);
335
+ } finally {
336
+ // 4. Restore old clipboard content
337
+ if (oldClipboard) {
338
+ await clipboardy.write(oldClipboard).catch(() => {
339
+ // Silent fail - don't affect main flow
340
+ debugDevice('Failed to restore clipboard content');
341
+ });
342
+ }
343
+ }
344
+ }
345
+
346
+ /**
347
+ * Smart type string with platform-specific strategy
348
+ * - macOS: Always use libnut (native support for non-ASCII)
349
+ * - Windows/Linux: Use clipboard for non-ASCII characters
350
+ */
351
+ private async smartTypeString(text: string): Promise<void> {
352
+ assert(libnut, 'libnut not initialized');
353
+
354
+ // macOS: use libnut directly (native Chinese support)
355
+ if (process.platform === 'darwin') {
356
+ libnut.typeString(text);
357
+ return;
358
+ }
359
+
360
+ // Windows/Linux: use smart strategy
361
+ const inputStrategy =
362
+ this.options?.inputStrategy ?? INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII;
363
+
364
+ if (inputStrategy === INPUT_STRATEGY_ALWAYS_CLIPBOARD) {
365
+ await this.typeViaClipboard(text);
366
+ return;
367
+ }
368
+
369
+ // clipboard-for-non-ascii strategy: intelligent detection
370
+ const shouldUseClipboard = this.shouldUseClipboardForText(text);
371
+
372
+ if (shouldUseClipboard) {
373
+ await this.typeViaClipboard(text);
374
+ } else {
375
+ libnut.typeString(text);
376
+ }
377
+ }
378
+
292
379
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
293
380
  actionSpace(): DeviceAction<any>[] {
294
381
  const defaultActions: DeviceAction<any>[] = [
@@ -401,7 +488,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d) => d.name).join(',
401
488
  return;
402
489
  }
403
490
 
404
- libnut.typeString(param.value);
491
+ await this.smartTypeString(param.value);
405
492
  },
406
493
  }),
407
494
 
@@ -0,0 +1,153 @@
1
+ import { sleep } from '@midscene/core/utils';
2
+ import { beforeAll, describe, it, vi } from 'vitest';
3
+ import { ComputerAgent, ComputerDevice } from '../../src';
4
+
5
+ vi.setConfig({
6
+ testTimeout: 240 * 1000,
7
+ });
8
+
9
+ describe('chinese and non-ASCII input', () => {
10
+ let agent: ComputerAgent;
11
+
12
+ beforeAll(async () => {
13
+ const device = new ComputerDevice({
14
+ inputStrategy: 'clipboard-for-non-ascii',
15
+ });
16
+ agent = new ComputerAgent(device, {
17
+ aiActionContext:
18
+ 'You are testing text input on a desktop computer. Focus on testing Chinese and other non-ASCII character input.',
19
+ });
20
+ await device.connect();
21
+ });
22
+
23
+ it(
24
+ 'should input Chinese text in browser search',
25
+ async () => {
26
+ const isMac = process.platform === 'darwin';
27
+
28
+ // Open browser (using platform-specific shortcuts)
29
+ if (isMac) {
30
+ await agent.aiAct('press Cmd+Space to open Spotlight');
31
+ await sleep(1000);
32
+ await agent.aiAct('type "Safari" and press Enter');
33
+ } else {
34
+ await agent.aiAct('press Windows key');
35
+ await sleep(1000);
36
+ await agent.aiAct('type "Chrome" or "Edge" and press Enter');
37
+ }
38
+
39
+ await sleep(3000);
40
+
41
+ // Wait for browser to open
42
+ await agent.aiWaitFor('Browser window is open');
43
+
44
+ // Navigate to a search engine
45
+ await agent.aiAct('click on address bar');
46
+ await sleep(500);
47
+ await agent.aiAct('type "google.com" and press Enter');
48
+
49
+ await sleep(3000);
50
+
51
+ // Test Chinese input
52
+ await agent.aiAct('click on search box');
53
+ await sleep(500);
54
+
55
+ // Input Chinese text
56
+ await agent.aiAct('type "你好世界"');
57
+ await sleep(1000);
58
+
59
+ // Verify Chinese text was input correctly
60
+ await agent.aiAssert('The search box contains Chinese text "你好世界"');
61
+
62
+ // Clear and test Japanese
63
+ await agent.aiAct('clear the search box');
64
+ await sleep(500);
65
+ await agent.aiAct('type "こんにちは"');
66
+ await sleep(1000);
67
+
68
+ // Verify Japanese text
69
+ await agent.aiAssert('The search box contains Japanese text');
70
+
71
+ // Clear and test emoji
72
+ await agent.aiAct('clear the search box');
73
+ await sleep(500);
74
+ await agent.aiAct('type "Hello 😀🎉"');
75
+ await sleep(1000);
76
+
77
+ // Verify emoji input
78
+ await agent.aiAssert('The search box contains text with emoji');
79
+
80
+ // Clear and test mixed text
81
+ await agent.aiAct('clear the search box');
82
+ await sleep(500);
83
+ await agent.aiAct('type "Hello 你好 World"');
84
+ await sleep(1000);
85
+
86
+ // Verify mixed text
87
+ await agent.aiAssert(
88
+ 'The search box contains mixed English and Chinese text',
89
+ );
90
+
91
+ // Close browser
92
+ if (isMac) {
93
+ await agent.aiAct('press Cmd+Q to close Safari');
94
+ } else {
95
+ await agent.aiAct('press Alt+F4 to close browser');
96
+ }
97
+ },
98
+ 720 * 1000,
99
+ );
100
+
101
+ it(
102
+ 'should use always-clipboard strategy',
103
+ async () => {
104
+ // Create a new device with always-clipboard strategy
105
+ const deviceAlways = new ComputerDevice({
106
+ inputStrategy: 'always-clipboard',
107
+ });
108
+ const agentAlways = new ComputerAgent(deviceAlways, {
109
+ aiActionContext: 'You are testing text input using clipboard.',
110
+ });
111
+ await deviceAlways.connect();
112
+
113
+ const isMac = process.platform === 'darwin';
114
+
115
+ // Open a text editor
116
+ if (isMac) {
117
+ await agentAlways.aiAct('press Cmd+Space to open Spotlight');
118
+ await sleep(1000);
119
+ await agentAlways.aiAct('type "TextEdit" and press Enter');
120
+ } else {
121
+ await agentAlways.aiAct('press Windows key');
122
+ await sleep(1000);
123
+ await agentAlways.aiAct('type "Notepad" and press Enter');
124
+ }
125
+
126
+ await sleep(2000);
127
+
128
+ // Wait for text editor to open
129
+ await agentAlways.aiWaitFor('Text editor is open');
130
+
131
+ // Test ASCII input (should also use clipboard with always-clipboard strategy)
132
+ await agentAlways.aiAct('type "Hello World"');
133
+ await sleep(1000);
134
+
135
+ // Verify ASCII text
136
+ await agentAlways.aiAssert('The text editor contains "Hello World"');
137
+
138
+ // Close text editor without saving
139
+ if (isMac) {
140
+ await agentAlways.aiAct('press Cmd+Q');
141
+ await sleep(500);
142
+ await agentAlways.aiAct('click "Don\'t Save" button if it appears');
143
+ } else {
144
+ await agentAlways.aiAct('press Alt+F4');
145
+ await sleep(500);
146
+ await agentAlways.aiAct('click "Don\'t Save" button if it appears');
147
+ }
148
+
149
+ await deviceAlways.destroy();
150
+ },
151
+ 720 * 1000,
152
+ );
153
+ });