@midscene/computer 1.2.3-beta-20260122071913.0 → 1.2.3-beta-20260122082712.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/index.mjs +38 -1
- package/dist/es/mcp-server.mjs +38 -1
- package/dist/lib/index.js +39 -1
- package/dist/lib/mcp-server.js +39 -1
- package/dist/types/index.d.ts +21 -0
- package/dist/types/mcp-server.d.ts +21 -0
- package/package.json +4 -3
- package/src/device.ts +88 -1
- package/tests/ai/chinese-input.test.ts +153 -0
- package/tests/unit-test/input-strategy.test.ts +58 -0
- package/tsconfig.tsbuildinfo +1 -1
package/dist/es/index.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import { actionHoverParamSchema, defineAction, defineActionClearInput, defineAct
|
|
|
5
5
|
import { sleep } from "@midscene/core/utils";
|
|
6
6
|
import { createImgBase64ByFormat } from "@midscene/shared/img";
|
|
7
7
|
import { getDebug } from "@midscene/shared/logger";
|
|
8
|
+
import clipboardy from "clipboardy";
|
|
8
9
|
import screenshot_desktop from "screenshot-desktop";
|
|
9
10
|
import { Agent } from "@midscene/core/agent";
|
|
10
11
|
import { overrideAIConfig } from "@midscene/shared/env";
|
|
@@ -29,6 +30,8 @@ const INPUT_CLEAR_DELAY = 150;
|
|
|
29
30
|
const SCROLL_REPEAT_COUNT = 10;
|
|
30
31
|
const SCROLL_STEP_DELAY = 100;
|
|
31
32
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
33
|
+
const INPUT_STRATEGY_ALWAYS_CLIPBOARD = 'always-clipboard';
|
|
34
|
+
const INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII = 'clipboard-for-non-ascii';
|
|
32
35
|
let device_libnut = null;
|
|
33
36
|
let libnutLoadError = null;
|
|
34
37
|
async function getLibnut() {
|
|
@@ -173,6 +176,40 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
173
176
|
throw new Error(`Failed to get screen size: ${error}`);
|
|
174
177
|
}
|
|
175
178
|
}
|
|
179
|
+
shouldUseClipboardForText(text) {
|
|
180
|
+
const hasNonAscii = /[\x80-\uFFFF]/.test(text);
|
|
181
|
+
return hasNonAscii;
|
|
182
|
+
}
|
|
183
|
+
async typeViaClipboard(text) {
|
|
184
|
+
node_assert(device_libnut, 'libnut not initialized');
|
|
185
|
+
debugDevice('Using clipboard to input text', {
|
|
186
|
+
textLength: text.length,
|
|
187
|
+
preview: text.substring(0, 20)
|
|
188
|
+
});
|
|
189
|
+
const oldClipboard = await clipboardy.read().catch(()=>'');
|
|
190
|
+
try {
|
|
191
|
+
await clipboardy.write(text);
|
|
192
|
+
await sleep(50);
|
|
193
|
+
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
194
|
+
device_libnut.keyTap('v', [
|
|
195
|
+
modifier
|
|
196
|
+
]);
|
|
197
|
+
await sleep(100);
|
|
198
|
+
} finally{
|
|
199
|
+
if (oldClipboard) await clipboardy.write(oldClipboard).catch(()=>{
|
|
200
|
+
debugDevice('Failed to restore clipboard content');
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
async smartTypeString(text) {
|
|
205
|
+
node_assert(device_libnut, 'libnut not initialized');
|
|
206
|
+
if ('darwin' === process.platform) return void device_libnut.typeString(text);
|
|
207
|
+
const inputStrategy = this.options?.inputStrategy ?? INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII;
|
|
208
|
+
if (inputStrategy === INPUT_STRATEGY_ALWAYS_CLIPBOARD) return void await this.typeViaClipboard(text);
|
|
209
|
+
const shouldUseClipboard = this.shouldUseClipboardForText(text);
|
|
210
|
+
if (shouldUseClipboard) await this.typeViaClipboard(text);
|
|
211
|
+
else device_libnut.typeString(text);
|
|
212
|
+
}
|
|
176
213
|
actionSpace() {
|
|
177
214
|
const defaultActions = [
|
|
178
215
|
defineActionTap(async (param)=>{
|
|
@@ -250,7 +287,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
250
287
|
}
|
|
251
288
|
if ('clear' === param.mode) return;
|
|
252
289
|
if (!param.value) return;
|
|
253
|
-
|
|
290
|
+
await this.smartTypeString(param.value);
|
|
254
291
|
}
|
|
255
292
|
}),
|
|
256
293
|
defineActionScroll(async (param)=>{
|
package/dist/es/mcp-server.mjs
CHANGED
|
@@ -7,6 +7,7 @@ import { actionHoverParamSchema, defineAction, defineActionClearInput, defineAct
|
|
|
7
7
|
import { sleep } from "@midscene/core/utils";
|
|
8
8
|
import { createImgBase64ByFormat } from "@midscene/shared/img";
|
|
9
9
|
import { getDebug } from "@midscene/shared/logger";
|
|
10
|
+
import clipboardy from "clipboardy";
|
|
10
11
|
import screenshot_desktop from "screenshot-desktop";
|
|
11
12
|
function _define_property(obj, key, value) {
|
|
12
13
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
@@ -29,6 +30,8 @@ const INPUT_CLEAR_DELAY = 150;
|
|
|
29
30
|
const SCROLL_REPEAT_COUNT = 10;
|
|
30
31
|
const SCROLL_STEP_DELAY = 100;
|
|
31
32
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
33
|
+
const INPUT_STRATEGY_ALWAYS_CLIPBOARD = 'always-clipboard';
|
|
34
|
+
const INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII = 'clipboard-for-non-ascii';
|
|
32
35
|
let libnut = null;
|
|
33
36
|
let libnutLoadError = null;
|
|
34
37
|
async function getLibnut() {
|
|
@@ -173,6 +176,40 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
173
176
|
throw new Error(`Failed to get screen size: ${error}`);
|
|
174
177
|
}
|
|
175
178
|
}
|
|
179
|
+
shouldUseClipboardForText(text) {
|
|
180
|
+
const hasNonAscii = /[\x80-\uFFFF]/.test(text);
|
|
181
|
+
return hasNonAscii;
|
|
182
|
+
}
|
|
183
|
+
async typeViaClipboard(text) {
|
|
184
|
+
node_assert(libnut, 'libnut not initialized');
|
|
185
|
+
debugDevice('Using clipboard to input text', {
|
|
186
|
+
textLength: text.length,
|
|
187
|
+
preview: text.substring(0, 20)
|
|
188
|
+
});
|
|
189
|
+
const oldClipboard = await clipboardy.read().catch(()=>'');
|
|
190
|
+
try {
|
|
191
|
+
await clipboardy.write(text);
|
|
192
|
+
await sleep(50);
|
|
193
|
+
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
194
|
+
libnut.keyTap('v', [
|
|
195
|
+
modifier
|
|
196
|
+
]);
|
|
197
|
+
await sleep(100);
|
|
198
|
+
} finally{
|
|
199
|
+
if (oldClipboard) await clipboardy.write(oldClipboard).catch(()=>{
|
|
200
|
+
debugDevice('Failed to restore clipboard content');
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
async smartTypeString(text) {
|
|
205
|
+
node_assert(libnut, 'libnut not initialized');
|
|
206
|
+
if ('darwin' === process.platform) return void libnut.typeString(text);
|
|
207
|
+
const inputStrategy = this.options?.inputStrategy ?? INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII;
|
|
208
|
+
if (inputStrategy === INPUT_STRATEGY_ALWAYS_CLIPBOARD) return void await this.typeViaClipboard(text);
|
|
209
|
+
const shouldUseClipboard = this.shouldUseClipboardForText(text);
|
|
210
|
+
if (shouldUseClipboard) await this.typeViaClipboard(text);
|
|
211
|
+
else libnut.typeString(text);
|
|
212
|
+
}
|
|
176
213
|
actionSpace() {
|
|
177
214
|
const defaultActions = [
|
|
178
215
|
defineActionTap(async (param)=>{
|
|
@@ -250,7 +287,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
250
287
|
}
|
|
251
288
|
if ('clear' === param.mode) return;
|
|
252
289
|
if (!param.value) return;
|
|
253
|
-
|
|
290
|
+
await this.smartTypeString(param.value);
|
|
254
291
|
}
|
|
255
292
|
}),
|
|
256
293
|
defineActionScroll(async (param)=>{
|
package/dist/lib/index.js
CHANGED
|
@@ -51,6 +51,8 @@ const device_namespaceObject = require("@midscene/core/device");
|
|
|
51
51
|
const utils_namespaceObject = require("@midscene/core/utils");
|
|
52
52
|
const img_namespaceObject = require("@midscene/shared/img");
|
|
53
53
|
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
54
|
+
const external_clipboardy_namespaceObject = require("clipboardy");
|
|
55
|
+
var external_clipboardy_default = /*#__PURE__*/ __webpack_require__.n(external_clipboardy_namespaceObject);
|
|
54
56
|
const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
|
|
55
57
|
var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
|
|
56
58
|
function _define_property(obj, key, value) {
|
|
@@ -74,6 +76,8 @@ const INPUT_CLEAR_DELAY = 150;
|
|
|
74
76
|
const SCROLL_REPEAT_COUNT = 10;
|
|
75
77
|
const SCROLL_STEP_DELAY = 100;
|
|
76
78
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
79
|
+
const INPUT_STRATEGY_ALWAYS_CLIPBOARD = 'always-clipboard';
|
|
80
|
+
const INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII = 'clipboard-for-non-ascii';
|
|
77
81
|
let device_libnut = null;
|
|
78
82
|
let libnutLoadError = null;
|
|
79
83
|
async function getLibnut() {
|
|
@@ -218,6 +222,40 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
218
222
|
throw new Error(`Failed to get screen size: ${error}`);
|
|
219
223
|
}
|
|
220
224
|
}
|
|
225
|
+
shouldUseClipboardForText(text) {
|
|
226
|
+
const hasNonAscii = /[\x80-\uFFFF]/.test(text);
|
|
227
|
+
return hasNonAscii;
|
|
228
|
+
}
|
|
229
|
+
async typeViaClipboard(text) {
|
|
230
|
+
external_node_assert_default()(device_libnut, 'libnut not initialized');
|
|
231
|
+
debugDevice('Using clipboard to input text', {
|
|
232
|
+
textLength: text.length,
|
|
233
|
+
preview: text.substring(0, 20)
|
|
234
|
+
});
|
|
235
|
+
const oldClipboard = await external_clipboardy_default().read().catch(()=>'');
|
|
236
|
+
try {
|
|
237
|
+
await external_clipboardy_default().write(text);
|
|
238
|
+
await (0, utils_namespaceObject.sleep)(50);
|
|
239
|
+
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
240
|
+
device_libnut.keyTap('v', [
|
|
241
|
+
modifier
|
|
242
|
+
]);
|
|
243
|
+
await (0, utils_namespaceObject.sleep)(100);
|
|
244
|
+
} finally{
|
|
245
|
+
if (oldClipboard) await external_clipboardy_default().write(oldClipboard).catch(()=>{
|
|
246
|
+
debugDevice('Failed to restore clipboard content');
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
async smartTypeString(text) {
|
|
251
|
+
external_node_assert_default()(device_libnut, 'libnut not initialized');
|
|
252
|
+
if ('darwin' === process.platform) return void device_libnut.typeString(text);
|
|
253
|
+
const inputStrategy = this.options?.inputStrategy ?? INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII;
|
|
254
|
+
if (inputStrategy === INPUT_STRATEGY_ALWAYS_CLIPBOARD) return void await this.typeViaClipboard(text);
|
|
255
|
+
const shouldUseClipboard = this.shouldUseClipboardForText(text);
|
|
256
|
+
if (shouldUseClipboard) await this.typeViaClipboard(text);
|
|
257
|
+
else device_libnut.typeString(text);
|
|
258
|
+
}
|
|
221
259
|
actionSpace() {
|
|
222
260
|
const defaultActions = [
|
|
223
261
|
(0, device_namespaceObject.defineActionTap)(async (param)=>{
|
|
@@ -295,7 +333,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
295
333
|
}
|
|
296
334
|
if ('clear' === param.mode) return;
|
|
297
335
|
if (!param.value) return;
|
|
298
|
-
|
|
336
|
+
await this.smartTypeString(param.value);
|
|
299
337
|
}
|
|
300
338
|
}),
|
|
301
339
|
(0, device_namespaceObject.defineActionScroll)(async (param)=>{
|
package/dist/lib/mcp-server.js
CHANGED
|
@@ -50,6 +50,8 @@ const device_namespaceObject = require("@midscene/core/device");
|
|
|
50
50
|
const utils_namespaceObject = require("@midscene/core/utils");
|
|
51
51
|
const img_namespaceObject = require("@midscene/shared/img");
|
|
52
52
|
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
53
|
+
const external_clipboardy_namespaceObject = require("clipboardy");
|
|
54
|
+
var external_clipboardy_default = /*#__PURE__*/ __webpack_require__.n(external_clipboardy_namespaceObject);
|
|
53
55
|
const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
|
|
54
56
|
var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
|
|
55
57
|
function _define_property(obj, key, value) {
|
|
@@ -73,6 +75,8 @@ const INPUT_CLEAR_DELAY = 150;
|
|
|
73
75
|
const SCROLL_REPEAT_COUNT = 10;
|
|
74
76
|
const SCROLL_STEP_DELAY = 100;
|
|
75
77
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
78
|
+
const INPUT_STRATEGY_ALWAYS_CLIPBOARD = 'always-clipboard';
|
|
79
|
+
const INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII = 'clipboard-for-non-ascii';
|
|
76
80
|
let libnut = null;
|
|
77
81
|
let libnutLoadError = null;
|
|
78
82
|
async function getLibnut() {
|
|
@@ -217,6 +221,40 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
217
221
|
throw new Error(`Failed to get screen size: ${error}`);
|
|
218
222
|
}
|
|
219
223
|
}
|
|
224
|
+
shouldUseClipboardForText(text) {
|
|
225
|
+
const hasNonAscii = /[\x80-\uFFFF]/.test(text);
|
|
226
|
+
return hasNonAscii;
|
|
227
|
+
}
|
|
228
|
+
async typeViaClipboard(text) {
|
|
229
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
230
|
+
debugDevice('Using clipboard to input text', {
|
|
231
|
+
textLength: text.length,
|
|
232
|
+
preview: text.substring(0, 20)
|
|
233
|
+
});
|
|
234
|
+
const oldClipboard = await external_clipboardy_default().read().catch(()=>'');
|
|
235
|
+
try {
|
|
236
|
+
await external_clipboardy_default().write(text);
|
|
237
|
+
await (0, utils_namespaceObject.sleep)(50);
|
|
238
|
+
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
239
|
+
libnut.keyTap('v', [
|
|
240
|
+
modifier
|
|
241
|
+
]);
|
|
242
|
+
await (0, utils_namespaceObject.sleep)(100);
|
|
243
|
+
} finally{
|
|
244
|
+
if (oldClipboard) await external_clipboardy_default().write(oldClipboard).catch(()=>{
|
|
245
|
+
debugDevice('Failed to restore clipboard content');
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
async smartTypeString(text) {
|
|
250
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
251
|
+
if ('darwin' === process.platform) return void libnut.typeString(text);
|
|
252
|
+
const inputStrategy = this.options?.inputStrategy ?? INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII;
|
|
253
|
+
if (inputStrategy === INPUT_STRATEGY_ALWAYS_CLIPBOARD) return void await this.typeViaClipboard(text);
|
|
254
|
+
const shouldUseClipboard = this.shouldUseClipboardForText(text);
|
|
255
|
+
if (shouldUseClipboard) await this.typeViaClipboard(text);
|
|
256
|
+
else libnut.typeString(text);
|
|
257
|
+
}
|
|
220
258
|
actionSpace() {
|
|
221
259
|
const defaultActions = [
|
|
222
260
|
(0, device_namespaceObject.defineActionTap)(async (param)=>{
|
|
@@ -294,7 +332,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
294
332
|
}
|
|
295
333
|
if ('clear' === param.mode) return;
|
|
296
334
|
if (!param.value) return;
|
|
297
|
-
|
|
335
|
+
await this.smartTypeString(param.value);
|
|
298
336
|
}
|
|
299
337
|
}),
|
|
300
338
|
(0, device_namespaceObject.defineActionScroll)(async (param)=>{
|
package/dist/types/index.d.ts
CHANGED
|
@@ -37,6 +37,26 @@ export declare class ComputerDevice implements AbstractInterface {
|
|
|
37
37
|
connect(): Promise<void>;
|
|
38
38
|
screenshotBase64(): Promise<string>;
|
|
39
39
|
size(): Promise<Size>;
|
|
40
|
+
/**
|
|
41
|
+
* Check if text contains non-ASCII characters
|
|
42
|
+
* Matches: Chinese, Japanese, Korean, Latin extended characters (café, niño), emoji, etc.
|
|
43
|
+
*/
|
|
44
|
+
private shouldUseClipboardForText;
|
|
45
|
+
/**
|
|
46
|
+
* Type text via clipboard (paste)
|
|
47
|
+
* This method:
|
|
48
|
+
* 1. Saves the old clipboard content
|
|
49
|
+
* 2. Writes new content to clipboard
|
|
50
|
+
* 3. Simulates paste shortcut (Ctrl+V / Cmd+V)
|
|
51
|
+
* 4. Restores old clipboard content
|
|
52
|
+
*/
|
|
53
|
+
private typeViaClipboard;
|
|
54
|
+
/**
|
|
55
|
+
* Smart type string with platform-specific strategy
|
|
56
|
+
* - macOS: Always use libnut (native support for non-ASCII)
|
|
57
|
+
* - Windows/Linux: Use clipboard for non-ASCII characters
|
|
58
|
+
*/
|
|
59
|
+
private smartTypeString;
|
|
40
60
|
actionSpace(): DeviceAction<any>[];
|
|
41
61
|
destroy(): Promise<void>;
|
|
42
62
|
url(): Promise<string>;
|
|
@@ -45,6 +65,7 @@ export declare class ComputerDevice implements AbstractInterface {
|
|
|
45
65
|
export declare interface ComputerDeviceOpt {
|
|
46
66
|
displayId?: string;
|
|
47
67
|
customActions?: DeviceAction<any>[];
|
|
68
|
+
inputStrategy?: 'always-clipboard' | 'clipboard-for-non-ascii';
|
|
48
69
|
}
|
|
49
70
|
|
|
50
71
|
export declare interface DisplayInfo {
|
|
@@ -29,6 +29,26 @@ declare class ComputerDevice implements AbstractInterface {
|
|
|
29
29
|
connect(): Promise<void>;
|
|
30
30
|
screenshotBase64(): Promise<string>;
|
|
31
31
|
size(): Promise<Size>;
|
|
32
|
+
/**
|
|
33
|
+
* Check if text contains non-ASCII characters
|
|
34
|
+
* Matches: Chinese, Japanese, Korean, Latin extended characters (café, niño), emoji, etc.
|
|
35
|
+
*/
|
|
36
|
+
private shouldUseClipboardForText;
|
|
37
|
+
/**
|
|
38
|
+
* Type text via clipboard (paste)
|
|
39
|
+
* This method:
|
|
40
|
+
* 1. Saves the old clipboard content
|
|
41
|
+
* 2. Writes new content to clipboard
|
|
42
|
+
* 3. Simulates paste shortcut (Ctrl+V / Cmd+V)
|
|
43
|
+
* 4. Restores old clipboard content
|
|
44
|
+
*/
|
|
45
|
+
private typeViaClipboard;
|
|
46
|
+
/**
|
|
47
|
+
* Smart type string with platform-specific strategy
|
|
48
|
+
* - macOS: Always use libnut (native support for non-ASCII)
|
|
49
|
+
* - Windows/Linux: Use clipboard for non-ASCII characters
|
|
50
|
+
*/
|
|
51
|
+
private smartTypeString;
|
|
32
52
|
actionSpace(): DeviceAction<any>[];
|
|
33
53
|
destroy(): Promise<void>;
|
|
34
54
|
url(): Promise<string>;
|
|
@@ -37,6 +57,7 @@ declare class ComputerDevice implements AbstractInterface {
|
|
|
37
57
|
declare interface ComputerDeviceOpt {
|
|
38
58
|
displayId?: string;
|
|
39
59
|
customActions?: DeviceAction<any>[];
|
|
60
|
+
inputStrategy?: 'always-clipboard' | 'clipboard-for-non-ascii';
|
|
40
61
|
}
|
|
41
62
|
|
|
42
63
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/computer",
|
|
3
|
-
"version": "1.2.3-beta-
|
|
3
|
+
"version": "1.2.3-beta-20260122082712.0",
|
|
4
4
|
"description": "Midscene.js Computer Desktop Automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -26,9 +26,10 @@
|
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
28
|
"@computer-use/libnut": "^4.2.0",
|
|
29
|
+
"clipboardy": "^4.0.0",
|
|
29
30
|
"screenshot-desktop": "^1.15.3",
|
|
30
|
-
"@midscene/core": "1.2.3-beta-
|
|
31
|
-
"@midscene/shared": "1.2.3-beta-
|
|
31
|
+
"@midscene/core": "1.2.3-beta-20260122082712.0",
|
|
32
|
+
"@midscene/shared": "1.2.3-beta-20260122082712.0"
|
|
32
33
|
},
|
|
33
34
|
"devDependencies": {
|
|
34
35
|
"@rslib/core": "^0.18.3",
|
package/src/device.ts
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
import { sleep } from '@midscene/core/utils';
|
|
26
26
|
import { createImgBase64ByFormat } from '@midscene/shared/img';
|
|
27
27
|
import { getDebug } from '@midscene/shared/logger';
|
|
28
|
+
import clipboardy from 'clipboardy';
|
|
28
29
|
import screenshot from 'screenshot-desktop';
|
|
29
30
|
|
|
30
31
|
// Type definitions
|
|
@@ -63,6 +64,10 @@ const SCROLL_REPEAT_COUNT = 10;
|
|
|
63
64
|
const SCROLL_STEP_DELAY = 100;
|
|
64
65
|
const SCROLL_COMPLETE_DELAY = 500;
|
|
65
66
|
|
|
67
|
+
// Input strategy constants
|
|
68
|
+
const INPUT_STRATEGY_ALWAYS_CLIPBOARD = 'always-clipboard';
|
|
69
|
+
const INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII = 'clipboard-for-non-ascii';
|
|
70
|
+
|
|
66
71
|
// Lazy load libnut with fallback
|
|
67
72
|
let libnut: LibNut | null = null;
|
|
68
73
|
let libnutLoadError: Error | null = null;
|
|
@@ -185,6 +190,7 @@ export interface ComputerDeviceOpt {
|
|
|
185
190
|
displayId?: string;
|
|
186
191
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
187
192
|
customActions?: DeviceAction<any>[];
|
|
193
|
+
inputStrategy?: 'always-clipboard' | 'clipboard-for-non-ascii';
|
|
188
194
|
}
|
|
189
195
|
|
|
190
196
|
export class ComputerDevice implements AbstractInterface {
|
|
@@ -289,6 +295,87 @@ Available Displays: ${displays.length > 0 ? displays.map((d) => d.name).join(',
|
|
|
289
295
|
}
|
|
290
296
|
}
|
|
291
297
|
|
|
298
|
+
/**
|
|
299
|
+
* Check if text contains non-ASCII characters
|
|
300
|
+
* Matches: Chinese, Japanese, Korean, Latin extended characters (café, niño), emoji, etc.
|
|
301
|
+
*/
|
|
302
|
+
private shouldUseClipboardForText(text: string): boolean {
|
|
303
|
+
// Check for any character with code point >= 128 (non-ASCII)
|
|
304
|
+
const hasNonAscii = /[\x80-\uFFFF]/.test(text);
|
|
305
|
+
return hasNonAscii;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* Type text via clipboard (paste)
|
|
310
|
+
* This method:
|
|
311
|
+
* 1. Saves the old clipboard content
|
|
312
|
+
* 2. Writes new content to clipboard
|
|
313
|
+
* 3. Simulates paste shortcut (Ctrl+V / Cmd+V)
|
|
314
|
+
* 4. Restores old clipboard content
|
|
315
|
+
*/
|
|
316
|
+
private async typeViaClipboard(text: string): Promise<void> {
|
|
317
|
+
assert(libnut, 'libnut not initialized');
|
|
318
|
+
debugDevice('Using clipboard to input text', {
|
|
319
|
+
textLength: text.length,
|
|
320
|
+
preview: text.substring(0, 20),
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
// 1. Save old clipboard content
|
|
324
|
+
const oldClipboard = await clipboardy.read().catch(() => '');
|
|
325
|
+
|
|
326
|
+
try {
|
|
327
|
+
// 2. Write new content to clipboard
|
|
328
|
+
await clipboardy.write(text);
|
|
329
|
+
await sleep(50);
|
|
330
|
+
|
|
331
|
+
// 3. Simulate paste shortcut
|
|
332
|
+
const modifier = process.platform === 'darwin' ? 'command' : 'control';
|
|
333
|
+
libnut.keyTap('v', [modifier]);
|
|
334
|
+
await sleep(100);
|
|
335
|
+
} finally {
|
|
336
|
+
// 4. Restore old clipboard content
|
|
337
|
+
if (oldClipboard) {
|
|
338
|
+
await clipboardy.write(oldClipboard).catch(() => {
|
|
339
|
+
// Silent fail - don't affect main flow
|
|
340
|
+
debugDevice('Failed to restore clipboard content');
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* Smart type string with platform-specific strategy
|
|
348
|
+
* - macOS: Always use libnut (native support for non-ASCII)
|
|
349
|
+
* - Windows/Linux: Use clipboard for non-ASCII characters
|
|
350
|
+
*/
|
|
351
|
+
private async smartTypeString(text: string): Promise<void> {
|
|
352
|
+
assert(libnut, 'libnut not initialized');
|
|
353
|
+
|
|
354
|
+
// macOS: use libnut directly (native Chinese support)
|
|
355
|
+
if (process.platform === 'darwin') {
|
|
356
|
+
libnut.typeString(text);
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// Windows/Linux: use smart strategy
|
|
361
|
+
const inputStrategy =
|
|
362
|
+
this.options?.inputStrategy ?? INPUT_STRATEGY_CLIPBOARD_FOR_NON_ASCII;
|
|
363
|
+
|
|
364
|
+
if (inputStrategy === INPUT_STRATEGY_ALWAYS_CLIPBOARD) {
|
|
365
|
+
await this.typeViaClipboard(text);
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// clipboard-for-non-ascii strategy: intelligent detection
|
|
370
|
+
const shouldUseClipboard = this.shouldUseClipboardForText(text);
|
|
371
|
+
|
|
372
|
+
if (shouldUseClipboard) {
|
|
373
|
+
await this.typeViaClipboard(text);
|
|
374
|
+
} else {
|
|
375
|
+
libnut.typeString(text);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
292
379
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
293
380
|
actionSpace(): DeviceAction<any>[] {
|
|
294
381
|
const defaultActions: DeviceAction<any>[] = [
|
|
@@ -401,7 +488,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d) => d.name).join(',
|
|
|
401
488
|
return;
|
|
402
489
|
}
|
|
403
490
|
|
|
404
|
-
|
|
491
|
+
await this.smartTypeString(param.value);
|
|
405
492
|
},
|
|
406
493
|
}),
|
|
407
494
|
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { sleep } from '@midscene/core/utils';
|
|
2
|
+
import { beforeAll, describe, it, vi } from 'vitest';
|
|
3
|
+
import { ComputerAgent, ComputerDevice } from '../../src';
|
|
4
|
+
|
|
5
|
+
vi.setConfig({
|
|
6
|
+
testTimeout: 240 * 1000,
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
describe('chinese and non-ASCII input', () => {
|
|
10
|
+
let agent: ComputerAgent;
|
|
11
|
+
|
|
12
|
+
beforeAll(async () => {
|
|
13
|
+
const device = new ComputerDevice({
|
|
14
|
+
inputStrategy: 'clipboard-for-non-ascii',
|
|
15
|
+
});
|
|
16
|
+
agent = new ComputerAgent(device, {
|
|
17
|
+
aiActionContext:
|
|
18
|
+
'You are testing text input on a desktop computer. Focus on testing Chinese and other non-ASCII character input.',
|
|
19
|
+
});
|
|
20
|
+
await device.connect();
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
it(
|
|
24
|
+
'should input Chinese text in browser search',
|
|
25
|
+
async () => {
|
|
26
|
+
const isMac = process.platform === 'darwin';
|
|
27
|
+
|
|
28
|
+
// Open browser (using platform-specific shortcuts)
|
|
29
|
+
if (isMac) {
|
|
30
|
+
await agent.aiAct('press Cmd+Space to open Spotlight');
|
|
31
|
+
await sleep(1000);
|
|
32
|
+
await agent.aiAct('type "Safari" and press Enter');
|
|
33
|
+
} else {
|
|
34
|
+
await agent.aiAct('press Windows key');
|
|
35
|
+
await sleep(1000);
|
|
36
|
+
await agent.aiAct('type "Chrome" or "Edge" and press Enter');
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
await sleep(3000);
|
|
40
|
+
|
|
41
|
+
// Wait for browser to open
|
|
42
|
+
await agent.aiWaitFor('Browser window is open');
|
|
43
|
+
|
|
44
|
+
// Navigate to a search engine
|
|
45
|
+
await agent.aiAct('click on address bar');
|
|
46
|
+
await sleep(500);
|
|
47
|
+
await agent.aiAct('type "google.com" and press Enter');
|
|
48
|
+
|
|
49
|
+
await sleep(3000);
|
|
50
|
+
|
|
51
|
+
// Test Chinese input
|
|
52
|
+
await agent.aiAct('click on search box');
|
|
53
|
+
await sleep(500);
|
|
54
|
+
|
|
55
|
+
// Input Chinese text
|
|
56
|
+
await agent.aiAct('type "你好世界"');
|
|
57
|
+
await sleep(1000);
|
|
58
|
+
|
|
59
|
+
// Verify Chinese text was input correctly
|
|
60
|
+
await agent.aiAssert('The search box contains Chinese text "你好世界"');
|
|
61
|
+
|
|
62
|
+
// Clear and test Japanese
|
|
63
|
+
await agent.aiAct('clear the search box');
|
|
64
|
+
await sleep(500);
|
|
65
|
+
await agent.aiAct('type "こんにちは"');
|
|
66
|
+
await sleep(1000);
|
|
67
|
+
|
|
68
|
+
// Verify Japanese text
|
|
69
|
+
await agent.aiAssert('The search box contains Japanese text');
|
|
70
|
+
|
|
71
|
+
// Clear and test emoji
|
|
72
|
+
await agent.aiAct('clear the search box');
|
|
73
|
+
await sleep(500);
|
|
74
|
+
await agent.aiAct('type "Hello 😀🎉"');
|
|
75
|
+
await sleep(1000);
|
|
76
|
+
|
|
77
|
+
// Verify emoji input
|
|
78
|
+
await agent.aiAssert('The search box contains text with emoji');
|
|
79
|
+
|
|
80
|
+
// Clear and test mixed text
|
|
81
|
+
await agent.aiAct('clear the search box');
|
|
82
|
+
await sleep(500);
|
|
83
|
+
await agent.aiAct('type "Hello 你好 World"');
|
|
84
|
+
await sleep(1000);
|
|
85
|
+
|
|
86
|
+
// Verify mixed text
|
|
87
|
+
await agent.aiAssert(
|
|
88
|
+
'The search box contains mixed English and Chinese text',
|
|
89
|
+
);
|
|
90
|
+
|
|
91
|
+
// Close browser
|
|
92
|
+
if (isMac) {
|
|
93
|
+
await agent.aiAct('press Cmd+Q to close Safari');
|
|
94
|
+
} else {
|
|
95
|
+
await agent.aiAct('press Alt+F4 to close browser');
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
720 * 1000,
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
it(
|
|
102
|
+
'should use always-clipboard strategy',
|
|
103
|
+
async () => {
|
|
104
|
+
// Create a new device with always-clipboard strategy
|
|
105
|
+
const deviceAlways = new ComputerDevice({
|
|
106
|
+
inputStrategy: 'always-clipboard',
|
|
107
|
+
});
|
|
108
|
+
const agentAlways = new ComputerAgent(deviceAlways, {
|
|
109
|
+
aiActionContext: 'You are testing text input using clipboard.',
|
|
110
|
+
});
|
|
111
|
+
await deviceAlways.connect();
|
|
112
|
+
|
|
113
|
+
const isMac = process.platform === 'darwin';
|
|
114
|
+
|
|
115
|
+
// Open a text editor
|
|
116
|
+
if (isMac) {
|
|
117
|
+
await agentAlways.aiAct('press Cmd+Space to open Spotlight');
|
|
118
|
+
await sleep(1000);
|
|
119
|
+
await agentAlways.aiAct('type "TextEdit" and press Enter');
|
|
120
|
+
} else {
|
|
121
|
+
await agentAlways.aiAct('press Windows key');
|
|
122
|
+
await sleep(1000);
|
|
123
|
+
await agentAlways.aiAct('type "Notepad" and press Enter');
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
await sleep(2000);
|
|
127
|
+
|
|
128
|
+
// Wait for text editor to open
|
|
129
|
+
await agentAlways.aiWaitFor('Text editor is open');
|
|
130
|
+
|
|
131
|
+
// Test ASCII input (should also use clipboard with always-clipboard strategy)
|
|
132
|
+
await agentAlways.aiAct('type "Hello World"');
|
|
133
|
+
await sleep(1000);
|
|
134
|
+
|
|
135
|
+
// Verify ASCII text
|
|
136
|
+
await agentAlways.aiAssert('The text editor contains "Hello World"');
|
|
137
|
+
|
|
138
|
+
// Close text editor without saving
|
|
139
|
+
if (isMac) {
|
|
140
|
+
await agentAlways.aiAct('press Cmd+Q');
|
|
141
|
+
await sleep(500);
|
|
142
|
+
await agentAlways.aiAct('click "Don\'t Save" button if it appears');
|
|
143
|
+
} else {
|
|
144
|
+
await agentAlways.aiAct('press Alt+F4');
|
|
145
|
+
await sleep(500);
|
|
146
|
+
await agentAlways.aiAct('click "Don\'t Save" button if it appears');
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
await deviceAlways.destroy();
|
|
150
|
+
},
|
|
151
|
+
720 * 1000,
|
|
152
|
+
);
|
|
153
|
+
});
|