@midscene/computer 1.12.4 → 1.12.5-beta-20260907092148.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +55 -8
- package/dist/es/index.mjs +55 -8
- package/dist/lib/cli.js +55 -8
- package/dist/lib/index.js +55 -8
- package/dist/types/index.d.ts +12 -1
- package/package.json +3 -3
package/dist/es/cli.mjs
CHANGED
|
@@ -91,6 +91,29 @@ class ComputerInputDriver {
|
|
|
91
91
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
92
92
|
else lib.keyTap(key);
|
|
93
93
|
}
|
|
94
|
+
keyToggle(key, state, modifiers) {
|
|
95
|
+
const lib = this.getLibnutOrThrow('keyToggle');
|
|
96
|
+
if (void 0 !== modifiers) lib.keyToggle(key, state, modifiers);
|
|
97
|
+
else lib.keyToggle(key, state);
|
|
98
|
+
}
|
|
99
|
+
async keyTapWithExplicitModifiers(key, modifiers, delayMs) {
|
|
100
|
+
const uniqueModifiers = [
|
|
101
|
+
...new Set(modifiers)
|
|
102
|
+
];
|
|
103
|
+
if (0 === uniqueModifiers.length) return void this.keyTap(key);
|
|
104
|
+
const pressedModifiers = [];
|
|
105
|
+
try {
|
|
106
|
+
for (const modifier of uniqueModifiers){
|
|
107
|
+
this.keyToggle(modifier, 'down');
|
|
108
|
+
pressedModifiers.push(modifier);
|
|
109
|
+
}
|
|
110
|
+
await this.delay(delayMs);
|
|
111
|
+
this.keyTap(key);
|
|
112
|
+
await this.delay(delayMs);
|
|
113
|
+
} finally{
|
|
114
|
+
for (const modifier of pressedModifiers.reverse())this.releaseKey(modifier);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
94
117
|
typeString(text) {
|
|
95
118
|
this.getLibnutOrThrow('typeString').typeString(text);
|
|
96
119
|
}
|
|
@@ -163,6 +186,15 @@ class ComputerInputDriver {
|
|
|
163
186
|
this.options.debug(`Failed to release mouse button ${button}: ${error}`);
|
|
164
187
|
}
|
|
165
188
|
}
|
|
189
|
+
releaseKey(key) {
|
|
190
|
+
try {
|
|
191
|
+
const libnut = this.options.getLibnut();
|
|
192
|
+
node_assert(libnut, 'libnut not initialized');
|
|
193
|
+
libnut.keyToggle(key, 'up');
|
|
194
|
+
} catch (error) {
|
|
195
|
+
this.options.debug(`Failed to release key ${key}: ${error}`);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
166
198
|
rejectPendingInputDelays() {
|
|
167
199
|
const error = this.createDestroyedError('in-flight input');
|
|
168
200
|
for (const waitRef of this.pendingInputDelayWaits){
|
|
@@ -523,8 +555,10 @@ async function captureScreenshot(options) {
|
|
|
523
555
|
}
|
|
524
556
|
const SMOOTH_MOVE_STEPS_TAP = 8;
|
|
525
557
|
const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
558
|
+
const SMOOTH_MOVE_STEPS_DRAG = 20;
|
|
526
559
|
const SMOOTH_MOVE_DELAY_TAP = 8;
|
|
527
560
|
const SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
561
|
+
const SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
528
562
|
const MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
529
563
|
const CLICK_SETTLE_DELAY = 50;
|
|
530
564
|
const CLICK_HOLD_DURATION = 100;
|
|
@@ -1151,7 +1185,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1151
1185
|
}
|
|
1152
1186
|
async healthCheck(displays) {
|
|
1153
1187
|
console.log('[HealthCheck] Starting health check...');
|
|
1154
|
-
console.log("[HealthCheck] @midscene/computer v1.12.
|
|
1188
|
+
console.log("[HealthCheck] @midscene/computer v1.12.5-beta-20260907092148.0");
|
|
1155
1189
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1156
1190
|
const screenshotTimeout = 15000;
|
|
1157
1191
|
let timeoutId;
|
|
@@ -1387,6 +1421,8 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1387
1421
|
modifiers,
|
|
1388
1422
|
driver: this.useAppleScript ? "applescript" : 'libnut'
|
|
1389
1423
|
});
|
|
1424
|
+
const shortcutDelay = this.options?.keyboardShortcutDelay ?? 0;
|
|
1425
|
+
if (!this.useAppleScript && modifiers.length > 0 && shortcutDelay > 0) return void await this.inputDriver.keyTapWithExplicitModifiers(key, modifiers, shortcutDelay);
|
|
1390
1426
|
this.inputDriver.sendKey(key, modifiers);
|
|
1391
1427
|
}
|
|
1392
1428
|
resolveUntargetedScrollPoint(screenSize) {
|
|
@@ -1628,7 +1664,10 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1628
1664
|
await this.moveDisplayPointer(from, 'Mouse did not reach the drag start target');
|
|
1629
1665
|
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
1630
1666
|
await this.inputDriver.delay(100);
|
|
1631
|
-
await this.moveDisplayPointer(to, 'Mouse did not reach the drag end target'
|
|
1667
|
+
await this.moveDisplayPointer(to, 'Mouse did not reach the drag end target', {
|
|
1668
|
+
smoothSteps: SMOOTH_MOVE_STEPS_DRAG,
|
|
1669
|
+
smoothDelay: SMOOTH_MOVE_DELAY_DRAG
|
|
1670
|
+
});
|
|
1632
1671
|
await this.inputDriver.delay(100);
|
|
1633
1672
|
});
|
|
1634
1673
|
}
|
|
@@ -1663,6 +1702,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1663
1702
|
}
|
|
1664
1703
|
}
|
|
1665
1704
|
});
|
|
1705
|
+
if (options?.keyboardShortcutDelay !== void 0 && (!Number.isFinite(options.keyboardShortcutDelay) || options.keyboardShortcutDelay < 0)) throw new Error('keyboardShortcutDelay must be a finite non-negative number');
|
|
1666
1706
|
this.options = options;
|
|
1667
1707
|
this.displayId = options?.displayId;
|
|
1668
1708
|
this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
|
|
@@ -2067,10 +2107,10 @@ function rdp_device_define_property(obj, key, value) {
|
|
|
2067
2107
|
const device_debug = getDebug('rdp:device');
|
|
2068
2108
|
const device_SMOOTH_MOVE_STEPS_TAP = 8;
|
|
2069
2109
|
const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
2070
|
-
const
|
|
2110
|
+
const device_SMOOTH_MOVE_STEPS_DRAG = 12;
|
|
2071
2111
|
const device_SMOOTH_MOVE_DELAY_TAP = 8;
|
|
2072
2112
|
const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
2073
|
-
const
|
|
2113
|
+
const device_SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
2074
2114
|
const device_MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
2075
2115
|
const device_CLICK_HOLD_DURATION = 50;
|
|
2076
2116
|
const DRAG_HOLD_DURATION = 100;
|
|
@@ -2273,8 +2313,8 @@ class RDPDevice {
|
|
|
2273
2313
|
await this.backend.mouseButton('left', 'down');
|
|
2274
2314
|
await sleep(DRAG_HOLD_DURATION);
|
|
2275
2315
|
await this.movePointer(Math.round(to.x), Math.round(to.y), {
|
|
2276
|
-
steps:
|
|
2277
|
-
stepDelayMs:
|
|
2316
|
+
steps: device_SMOOTH_MOVE_STEPS_DRAG,
|
|
2317
|
+
stepDelayMs: device_SMOOTH_MOVE_DELAY_DRAG
|
|
2278
2318
|
});
|
|
2279
2319
|
await sleep(DRAG_HOLD_DURATION);
|
|
2280
2320
|
await this.backend.mouseButton('left', 'up');
|
|
@@ -2365,6 +2405,7 @@ function createLocalComputerDevice(opts) {
|
|
|
2365
2405
|
keyboardTypeDelay: opts?.keyboardTypeDelay,
|
|
2366
2406
|
inputStrategy: opts?.inputStrategy,
|
|
2367
2407
|
keyboardDriver: opts?.keyboardDriver,
|
|
2408
|
+
keyboardShortcutDelay: opts?.keyboardShortcutDelay,
|
|
2368
2409
|
headless: opts?.headless,
|
|
2369
2410
|
xvfbResolution: opts?.xvfbResolution,
|
|
2370
2411
|
keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
|
|
@@ -2422,6 +2463,7 @@ const computerInitArgShape = {
|
|
|
2422
2463
|
headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
|
|
2423
2464
|
keyboardTypeDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between keystrokes. In "legacy" mode, positive values enable key-by-key input; zero or omitted uses clipboard input.'),
|
|
2424
2465
|
inputStrategy: z["enum"](inputStrategies).optional().describe('Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.'),
|
|
2466
|
+
keyboardShortcutDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between modifier transitions and the main key for local libnut shortcuts. Positive values can improve compatibility with full-screen remote-control clients. Ignored in RDP mode and by the macOS AppleScript driver.'),
|
|
2425
2467
|
host: z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
|
|
2426
2468
|
port: z.number().optional().describe('RDP port (default 3389). Requires host.'),
|
|
2427
2469
|
username: z.string().optional().describe('RDP username. Requires host.'),
|
|
@@ -2438,7 +2480,7 @@ const computerInitArgShape = {
|
|
|
2438
2480
|
function adaptComputerInitArgs(extracted) {
|
|
2439
2481
|
if (!extracted || 0 === Object.keys(extracted).length) return;
|
|
2440
2482
|
if (extracted.host) {
|
|
2441
|
-
const { displayId: _d, headless: _h, ...rdpFields } = extracted;
|
|
2483
|
+
const { displayId: _d, headless: _h, keyboardShortcutDelay: _s, ...rdpFields } = extracted;
|
|
2442
2484
|
const host = normalizeRdpHost(extracted.host);
|
|
2443
2485
|
return {
|
|
2444
2486
|
mode: 'rdp',
|
|
@@ -2452,6 +2494,7 @@ function adaptComputerInitArgs(extracted) {
|
|
|
2452
2494
|
headless: extracted.headless,
|
|
2453
2495
|
keyboardTypeDelay: extracted.keyboardTypeDelay,
|
|
2454
2496
|
inputStrategy: extracted.inputStrategy,
|
|
2497
|
+
keyboardShortcutDelay: extracted.keyboardShortcutDelay,
|
|
2455
2498
|
...extractAgentBehaviorInitArgs(extracted) ?? {}
|
|
2456
2499
|
};
|
|
2457
2500
|
}
|
|
@@ -2503,6 +2546,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2503
2546
|
const headless = opts?.mode === 'local' ? opts.headless : void 0;
|
|
2504
2547
|
const keyboardTypeDelay = opts?.keyboardTypeDelay;
|
|
2505
2548
|
const inputStrategy = opts?.inputStrategy;
|
|
2549
|
+
const keyboardShortcutDelay = opts?.mode === 'local' ? opts.keyboardShortcutDelay : void 0;
|
|
2506
2550
|
agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
2507
2551
|
const agentOpts = {
|
|
2508
2552
|
...displayId ? {
|
|
@@ -2517,6 +2561,9 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2517
2561
|
...void 0 !== inputStrategy ? {
|
|
2518
2562
|
inputStrategy
|
|
2519
2563
|
} : {},
|
|
2564
|
+
...void 0 !== keyboardShortcutDelay ? {
|
|
2565
|
+
keyboardShortcutDelay
|
|
2566
|
+
} : {},
|
|
2520
2567
|
...this.options.keepXvfbAliveUntilProcessExit ? {
|
|
2521
2568
|
keepXvfbAliveUntilProcessExit: true
|
|
2522
2569
|
} : {},
|
|
@@ -2601,7 +2648,7 @@ const tools = new ComputerMidsceneTools({
|
|
|
2601
2648
|
});
|
|
2602
2649
|
runToolsCLI(tools, 'midscene-computer', {
|
|
2603
2650
|
stripPrefix: 'computer_',
|
|
2604
|
-
version: "1.12.
|
|
2651
|
+
version: "1.12.5-beta-20260907092148.0",
|
|
2605
2652
|
extraCommands: createReportCliCommands()
|
|
2606
2653
|
}).catch((e)=>{
|
|
2607
2654
|
process.exit(reportCLIError(e));
|
package/dist/es/index.mjs
CHANGED
|
@@ -91,6 +91,29 @@ class ComputerInputDriver {
|
|
|
91
91
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
92
92
|
else lib.keyTap(key);
|
|
93
93
|
}
|
|
94
|
+
keyToggle(key, state, modifiers) {
|
|
95
|
+
const lib = this.getLibnutOrThrow('keyToggle');
|
|
96
|
+
if (void 0 !== modifiers) lib.keyToggle(key, state, modifiers);
|
|
97
|
+
else lib.keyToggle(key, state);
|
|
98
|
+
}
|
|
99
|
+
async keyTapWithExplicitModifiers(key, modifiers, delayMs) {
|
|
100
|
+
const uniqueModifiers = [
|
|
101
|
+
...new Set(modifiers)
|
|
102
|
+
];
|
|
103
|
+
if (0 === uniqueModifiers.length) return void this.keyTap(key);
|
|
104
|
+
const pressedModifiers = [];
|
|
105
|
+
try {
|
|
106
|
+
for (const modifier of uniqueModifiers){
|
|
107
|
+
this.keyToggle(modifier, 'down');
|
|
108
|
+
pressedModifiers.push(modifier);
|
|
109
|
+
}
|
|
110
|
+
await this.delay(delayMs);
|
|
111
|
+
this.keyTap(key);
|
|
112
|
+
await this.delay(delayMs);
|
|
113
|
+
} finally{
|
|
114
|
+
for (const modifier of pressedModifiers.reverse())this.releaseKey(modifier);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
94
117
|
typeString(text) {
|
|
95
118
|
this.getLibnutOrThrow('typeString').typeString(text);
|
|
96
119
|
}
|
|
@@ -163,6 +186,15 @@ class ComputerInputDriver {
|
|
|
163
186
|
this.options.debug(`Failed to release mouse button ${button}: ${error}`);
|
|
164
187
|
}
|
|
165
188
|
}
|
|
189
|
+
releaseKey(key) {
|
|
190
|
+
try {
|
|
191
|
+
const libnut = this.options.getLibnut();
|
|
192
|
+
node_assert(libnut, 'libnut not initialized');
|
|
193
|
+
libnut.keyToggle(key, 'up');
|
|
194
|
+
} catch (error) {
|
|
195
|
+
this.options.debug(`Failed to release key ${key}: ${error}`);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
166
198
|
rejectPendingInputDelays() {
|
|
167
199
|
const error = this.createDestroyedError('in-flight input');
|
|
168
200
|
for (const waitRef of this.pendingInputDelayWaits){
|
|
@@ -523,8 +555,10 @@ async function captureScreenshot(options) {
|
|
|
523
555
|
}
|
|
524
556
|
const SMOOTH_MOVE_STEPS_TAP = 8;
|
|
525
557
|
const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
558
|
+
const SMOOTH_MOVE_STEPS_DRAG = 20;
|
|
526
559
|
const SMOOTH_MOVE_DELAY_TAP = 8;
|
|
527
560
|
const SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
561
|
+
const SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
528
562
|
const MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
529
563
|
const CLICK_SETTLE_DELAY = 50;
|
|
530
564
|
const CLICK_HOLD_DURATION = 100;
|
|
@@ -1151,7 +1185,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1151
1185
|
}
|
|
1152
1186
|
async healthCheck(displays) {
|
|
1153
1187
|
console.log('[HealthCheck] Starting health check...');
|
|
1154
|
-
console.log("[HealthCheck] @midscene/computer v1.12.
|
|
1188
|
+
console.log("[HealthCheck] @midscene/computer v1.12.5-beta-20260907092148.0");
|
|
1155
1189
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1156
1190
|
const screenshotTimeout = 15000;
|
|
1157
1191
|
let timeoutId;
|
|
@@ -1387,6 +1421,8 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1387
1421
|
modifiers,
|
|
1388
1422
|
driver: this.useAppleScript ? "applescript" : 'libnut'
|
|
1389
1423
|
});
|
|
1424
|
+
const shortcutDelay = this.options?.keyboardShortcutDelay ?? 0;
|
|
1425
|
+
if (!this.useAppleScript && modifiers.length > 0 && shortcutDelay > 0) return void await this.inputDriver.keyTapWithExplicitModifiers(key, modifiers, shortcutDelay);
|
|
1390
1426
|
this.inputDriver.sendKey(key, modifiers);
|
|
1391
1427
|
}
|
|
1392
1428
|
resolveUntargetedScrollPoint(screenSize) {
|
|
@@ -1628,7 +1664,10 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1628
1664
|
await this.moveDisplayPointer(from, 'Mouse did not reach the drag start target');
|
|
1629
1665
|
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
1630
1666
|
await this.inputDriver.delay(100);
|
|
1631
|
-
await this.moveDisplayPointer(to, 'Mouse did not reach the drag end target'
|
|
1667
|
+
await this.moveDisplayPointer(to, 'Mouse did not reach the drag end target', {
|
|
1668
|
+
smoothSteps: SMOOTH_MOVE_STEPS_DRAG,
|
|
1669
|
+
smoothDelay: SMOOTH_MOVE_DELAY_DRAG
|
|
1670
|
+
});
|
|
1632
1671
|
await this.inputDriver.delay(100);
|
|
1633
1672
|
});
|
|
1634
1673
|
}
|
|
@@ -1663,6 +1702,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1663
1702
|
}
|
|
1664
1703
|
}
|
|
1665
1704
|
});
|
|
1705
|
+
if (options?.keyboardShortcutDelay !== void 0 && (!Number.isFinite(options.keyboardShortcutDelay) || options.keyboardShortcutDelay < 0)) throw new Error('keyboardShortcutDelay must be a finite non-negative number');
|
|
1666
1706
|
this.options = options;
|
|
1667
1707
|
this.displayId = options?.displayId;
|
|
1668
1708
|
this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
|
|
@@ -2110,10 +2150,10 @@ function rdp_device_define_property(obj, key, value) {
|
|
|
2110
2150
|
const device_debug = getDebug('rdp:device');
|
|
2111
2151
|
const device_SMOOTH_MOVE_STEPS_TAP = 8;
|
|
2112
2152
|
const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
2113
|
-
const
|
|
2153
|
+
const device_SMOOTH_MOVE_STEPS_DRAG = 12;
|
|
2114
2154
|
const device_SMOOTH_MOVE_DELAY_TAP = 8;
|
|
2115
2155
|
const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
2116
|
-
const
|
|
2156
|
+
const device_SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
2117
2157
|
const device_MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
2118
2158
|
const device_CLICK_HOLD_DURATION = 50;
|
|
2119
2159
|
const DRAG_HOLD_DURATION = 100;
|
|
@@ -2316,8 +2356,8 @@ class RDPDevice {
|
|
|
2316
2356
|
await this.backend.mouseButton('left', 'down');
|
|
2317
2357
|
await sleep(DRAG_HOLD_DURATION);
|
|
2318
2358
|
await this.movePointer(Math.round(to.x), Math.round(to.y), {
|
|
2319
|
-
steps:
|
|
2320
|
-
stepDelayMs:
|
|
2359
|
+
steps: device_SMOOTH_MOVE_STEPS_DRAG,
|
|
2360
|
+
stepDelayMs: device_SMOOTH_MOVE_DELAY_DRAG
|
|
2321
2361
|
});
|
|
2322
2362
|
await sleep(DRAG_HOLD_DURATION);
|
|
2323
2363
|
await this.backend.mouseButton('left', 'up');
|
|
@@ -2408,6 +2448,7 @@ function createLocalComputerDevice(opts) {
|
|
|
2408
2448
|
keyboardTypeDelay: opts?.keyboardTypeDelay,
|
|
2409
2449
|
inputStrategy: opts?.inputStrategy,
|
|
2410
2450
|
keyboardDriver: opts?.keyboardDriver,
|
|
2451
|
+
keyboardShortcutDelay: opts?.keyboardShortcutDelay,
|
|
2411
2452
|
headless: opts?.headless,
|
|
2412
2453
|
xvfbResolution: opts?.xvfbResolution,
|
|
2413
2454
|
keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
|
|
@@ -2465,6 +2506,7 @@ const computerInitArgShape = {
|
|
|
2465
2506
|
headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
|
|
2466
2507
|
keyboardTypeDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between keystrokes. In "legacy" mode, positive values enable key-by-key input; zero or omitted uses clipboard input.'),
|
|
2467
2508
|
inputStrategy: z["enum"](inputStrategies).optional().describe('Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.'),
|
|
2509
|
+
keyboardShortcutDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between modifier transitions and the main key for local libnut shortcuts. Positive values can improve compatibility with full-screen remote-control clients. Ignored in RDP mode and by the macOS AppleScript driver.'),
|
|
2468
2510
|
host: z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
|
|
2469
2511
|
port: z.number().optional().describe('RDP port (default 3389). Requires host.'),
|
|
2470
2512
|
username: z.string().optional().describe('RDP username. Requires host.'),
|
|
@@ -2481,7 +2523,7 @@ const computerInitArgShape = {
|
|
|
2481
2523
|
function adaptComputerInitArgs(extracted) {
|
|
2482
2524
|
if (!extracted || 0 === Object.keys(extracted).length) return;
|
|
2483
2525
|
if (extracted.host) {
|
|
2484
|
-
const { displayId: _d, headless: _h, ...rdpFields } = extracted;
|
|
2526
|
+
const { displayId: _d, headless: _h, keyboardShortcutDelay: _s, ...rdpFields } = extracted;
|
|
2485
2527
|
const host = normalizeRdpHost(extracted.host);
|
|
2486
2528
|
return {
|
|
2487
2529
|
mode: 'rdp',
|
|
@@ -2495,6 +2537,7 @@ function adaptComputerInitArgs(extracted) {
|
|
|
2495
2537
|
headless: extracted.headless,
|
|
2496
2538
|
keyboardTypeDelay: extracted.keyboardTypeDelay,
|
|
2497
2539
|
inputStrategy: extracted.inputStrategy,
|
|
2540
|
+
keyboardShortcutDelay: extracted.keyboardShortcutDelay,
|
|
2498
2541
|
...extractAgentBehaviorInitArgs(extracted) ?? {}
|
|
2499
2542
|
};
|
|
2500
2543
|
}
|
|
@@ -2546,6 +2589,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2546
2589
|
const headless = opts?.mode === 'local' ? opts.headless : void 0;
|
|
2547
2590
|
const keyboardTypeDelay = opts?.keyboardTypeDelay;
|
|
2548
2591
|
const inputStrategy = opts?.inputStrategy;
|
|
2592
|
+
const keyboardShortcutDelay = opts?.mode === 'local' ? opts.keyboardShortcutDelay : void 0;
|
|
2549
2593
|
agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
2550
2594
|
const agentOpts = {
|
|
2551
2595
|
...displayId ? {
|
|
@@ -2560,6 +2604,9 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2560
2604
|
...void 0 !== inputStrategy ? {
|
|
2561
2605
|
inputStrategy
|
|
2562
2606
|
} : {},
|
|
2607
|
+
...void 0 !== keyboardShortcutDelay ? {
|
|
2608
|
+
keyboardShortcutDelay
|
|
2609
|
+
} : {},
|
|
2563
2610
|
...this.options.keepXvfbAliveUntilProcessExit ? {
|
|
2564
2611
|
keepXvfbAliveUntilProcessExit: true
|
|
2565
2612
|
} : {},
|
|
@@ -2640,7 +2687,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2640
2687
|
}
|
|
2641
2688
|
}
|
|
2642
2689
|
function version() {
|
|
2643
|
-
const currentVersion = "1.12.
|
|
2690
|
+
const currentVersion = "1.12.5-beta-20260907092148.0";
|
|
2644
2691
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2645
2692
|
return currentVersion;
|
|
2646
2693
|
}
|
package/dist/lib/cli.js
CHANGED
|
@@ -116,6 +116,29 @@ class ComputerInputDriver {
|
|
|
116
116
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
117
117
|
else lib.keyTap(key);
|
|
118
118
|
}
|
|
119
|
+
keyToggle(key, state, modifiers) {
|
|
120
|
+
const lib = this.getLibnutOrThrow('keyToggle');
|
|
121
|
+
if (void 0 !== modifiers) lib.keyToggle(key, state, modifiers);
|
|
122
|
+
else lib.keyToggle(key, state);
|
|
123
|
+
}
|
|
124
|
+
async keyTapWithExplicitModifiers(key, modifiers, delayMs) {
|
|
125
|
+
const uniqueModifiers = [
|
|
126
|
+
...new Set(modifiers)
|
|
127
|
+
];
|
|
128
|
+
if (0 === uniqueModifiers.length) return void this.keyTap(key);
|
|
129
|
+
const pressedModifiers = [];
|
|
130
|
+
try {
|
|
131
|
+
for (const modifier of uniqueModifiers){
|
|
132
|
+
this.keyToggle(modifier, 'down');
|
|
133
|
+
pressedModifiers.push(modifier);
|
|
134
|
+
}
|
|
135
|
+
await this.delay(delayMs);
|
|
136
|
+
this.keyTap(key);
|
|
137
|
+
await this.delay(delayMs);
|
|
138
|
+
} finally{
|
|
139
|
+
for (const modifier of pressedModifiers.reverse())this.releaseKey(modifier);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
119
142
|
typeString(text) {
|
|
120
143
|
this.getLibnutOrThrow('typeString').typeString(text);
|
|
121
144
|
}
|
|
@@ -188,6 +211,15 @@ class ComputerInputDriver {
|
|
|
188
211
|
this.options.debug(`Failed to release mouse button ${button}: ${error}`);
|
|
189
212
|
}
|
|
190
213
|
}
|
|
214
|
+
releaseKey(key) {
|
|
215
|
+
try {
|
|
216
|
+
const libnut = this.options.getLibnut();
|
|
217
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
218
|
+
libnut.keyToggle(key, 'up');
|
|
219
|
+
} catch (error) {
|
|
220
|
+
this.options.debug(`Failed to release key ${key}: ${error}`);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
191
223
|
rejectPendingInputDelays() {
|
|
192
224
|
const error = this.createDestroyedError('in-flight input');
|
|
193
225
|
for (const waitRef of this.pendingInputDelayWaits){
|
|
@@ -549,8 +581,10 @@ async function captureScreenshot(options) {
|
|
|
549
581
|
}
|
|
550
582
|
const SMOOTH_MOVE_STEPS_TAP = 8;
|
|
551
583
|
const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
584
|
+
const SMOOTH_MOVE_STEPS_DRAG = 20;
|
|
552
585
|
const SMOOTH_MOVE_DELAY_TAP = 8;
|
|
553
586
|
const SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
587
|
+
const SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
554
588
|
const MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
555
589
|
const CLICK_SETTLE_DELAY = 50;
|
|
556
590
|
const CLICK_HOLD_DURATION = 100;
|
|
@@ -1177,7 +1211,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1177
1211
|
}
|
|
1178
1212
|
async healthCheck(displays) {
|
|
1179
1213
|
console.log('[HealthCheck] Starting health check...');
|
|
1180
|
-
console.log("[HealthCheck] @midscene/computer v1.12.
|
|
1214
|
+
console.log("[HealthCheck] @midscene/computer v1.12.5-beta-20260907092148.0");
|
|
1181
1215
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1182
1216
|
const screenshotTimeout = 15000;
|
|
1183
1217
|
let timeoutId;
|
|
@@ -1413,6 +1447,8 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1413
1447
|
modifiers,
|
|
1414
1448
|
driver: this.useAppleScript ? "applescript" : 'libnut'
|
|
1415
1449
|
});
|
|
1450
|
+
const shortcutDelay = this.options?.keyboardShortcutDelay ?? 0;
|
|
1451
|
+
if (!this.useAppleScript && modifiers.length > 0 && shortcutDelay > 0) return void await this.inputDriver.keyTapWithExplicitModifiers(key, modifiers, shortcutDelay);
|
|
1416
1452
|
this.inputDriver.sendKey(key, modifiers);
|
|
1417
1453
|
}
|
|
1418
1454
|
resolveUntargetedScrollPoint(screenSize) {
|
|
@@ -1654,7 +1690,10 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1654
1690
|
await this.moveDisplayPointer(from, 'Mouse did not reach the drag start target');
|
|
1655
1691
|
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
1656
1692
|
await this.inputDriver.delay(100);
|
|
1657
|
-
await this.moveDisplayPointer(to, 'Mouse did not reach the drag end target'
|
|
1693
|
+
await this.moveDisplayPointer(to, 'Mouse did not reach the drag end target', {
|
|
1694
|
+
smoothSteps: SMOOTH_MOVE_STEPS_DRAG,
|
|
1695
|
+
smoothDelay: SMOOTH_MOVE_DELAY_DRAG
|
|
1696
|
+
});
|
|
1658
1697
|
await this.inputDriver.delay(100);
|
|
1659
1698
|
});
|
|
1660
1699
|
}
|
|
@@ -1689,6 +1728,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1689
1728
|
}
|
|
1690
1729
|
}
|
|
1691
1730
|
});
|
|
1731
|
+
if (options?.keyboardShortcutDelay !== void 0 && (!Number.isFinite(options.keyboardShortcutDelay) || options.keyboardShortcutDelay < 0)) throw new Error('keyboardShortcutDelay must be a finite non-negative number');
|
|
1692
1732
|
this.options = options;
|
|
1693
1733
|
this.displayId = options?.displayId;
|
|
1694
1734
|
this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
|
|
@@ -2094,10 +2134,10 @@ function rdp_device_define_property(obj, key, value) {
|
|
|
2094
2134
|
const device_debug = (0, logger_namespaceObject.getDebug)('rdp:device');
|
|
2095
2135
|
const device_SMOOTH_MOVE_STEPS_TAP = 8;
|
|
2096
2136
|
const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
2097
|
-
const
|
|
2137
|
+
const device_SMOOTH_MOVE_STEPS_DRAG = 12;
|
|
2098
2138
|
const device_SMOOTH_MOVE_DELAY_TAP = 8;
|
|
2099
2139
|
const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
2100
|
-
const
|
|
2140
|
+
const device_SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
2101
2141
|
const device_MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
2102
2142
|
const device_CLICK_HOLD_DURATION = 50;
|
|
2103
2143
|
const DRAG_HOLD_DURATION = 100;
|
|
@@ -2300,8 +2340,8 @@ class RDPDevice {
|
|
|
2300
2340
|
await this.backend.mouseButton('left', 'down');
|
|
2301
2341
|
await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
|
|
2302
2342
|
await this.movePointer(Math.round(to.x), Math.round(to.y), {
|
|
2303
|
-
steps:
|
|
2304
|
-
stepDelayMs:
|
|
2343
|
+
steps: device_SMOOTH_MOVE_STEPS_DRAG,
|
|
2344
|
+
stepDelayMs: device_SMOOTH_MOVE_DELAY_DRAG
|
|
2305
2345
|
});
|
|
2306
2346
|
await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
|
|
2307
2347
|
await this.backend.mouseButton('left', 'up');
|
|
@@ -2392,6 +2432,7 @@ function createLocalComputerDevice(opts) {
|
|
|
2392
2432
|
keyboardTypeDelay: opts?.keyboardTypeDelay,
|
|
2393
2433
|
inputStrategy: opts?.inputStrategy,
|
|
2394
2434
|
keyboardDriver: opts?.keyboardDriver,
|
|
2435
|
+
keyboardShortcutDelay: opts?.keyboardShortcutDelay,
|
|
2395
2436
|
headless: opts?.headless,
|
|
2396
2437
|
xvfbResolution: opts?.xvfbResolution,
|
|
2397
2438
|
keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
|
|
@@ -2449,6 +2490,7 @@ const computerInitArgShape = {
|
|
|
2449
2490
|
headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
|
|
2450
2491
|
keyboardTypeDelay: core_namespaceObject.z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between keystrokes. In "legacy" mode, positive values enable key-by-key input; zero or omitted uses clipboard input.'),
|
|
2451
2492
|
inputStrategy: core_namespaceObject.z["enum"](device_namespaceObject.inputStrategies).optional().describe('Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.'),
|
|
2493
|
+
keyboardShortcutDelay: core_namespaceObject.z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between modifier transitions and the main key for local libnut shortcuts. Positive values can improve compatibility with full-screen remote-control clients. Ignored in RDP mode and by the macOS AppleScript driver.'),
|
|
2452
2494
|
host: core_namespaceObject.z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
|
|
2453
2495
|
port: core_namespaceObject.z.number().optional().describe('RDP port (default 3389). Requires host.'),
|
|
2454
2496
|
username: core_namespaceObject.z.string().optional().describe('RDP username. Requires host.'),
|
|
@@ -2465,7 +2507,7 @@ const computerInitArgShape = {
|
|
|
2465
2507
|
function adaptComputerInitArgs(extracted) {
|
|
2466
2508
|
if (!extracted || 0 === Object.keys(extracted).length) return;
|
|
2467
2509
|
if (extracted.host) {
|
|
2468
|
-
const { displayId: _d, headless: _h, ...rdpFields } = extracted;
|
|
2510
|
+
const { displayId: _d, headless: _h, keyboardShortcutDelay: _s, ...rdpFields } = extracted;
|
|
2469
2511
|
const host = normalizeRdpHost(extracted.host);
|
|
2470
2512
|
return {
|
|
2471
2513
|
mode: 'rdp',
|
|
@@ -2479,6 +2521,7 @@ function adaptComputerInitArgs(extracted) {
|
|
|
2479
2521
|
headless: extracted.headless,
|
|
2480
2522
|
keyboardTypeDelay: extracted.keyboardTypeDelay,
|
|
2481
2523
|
inputStrategy: extracted.inputStrategy,
|
|
2524
|
+
keyboardShortcutDelay: extracted.keyboardShortcutDelay,
|
|
2482
2525
|
...(0, agent_behavior_init_args_namespaceObject.extractAgentBehaviorInitArgs)(extracted) ?? {}
|
|
2483
2526
|
};
|
|
2484
2527
|
}
|
|
@@ -2530,6 +2573,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2530
2573
|
const headless = opts?.mode === 'local' ? opts.headless : void 0;
|
|
2531
2574
|
const keyboardTypeDelay = opts?.keyboardTypeDelay;
|
|
2532
2575
|
const inputStrategy = opts?.inputStrategy;
|
|
2576
|
+
const keyboardShortcutDelay = opts?.mode === 'local' ? opts.keyboardShortcutDelay : void 0;
|
|
2533
2577
|
agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
2534
2578
|
const agentOpts = {
|
|
2535
2579
|
...displayId ? {
|
|
@@ -2544,6 +2588,9 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2544
2588
|
...void 0 !== inputStrategy ? {
|
|
2545
2589
|
inputStrategy
|
|
2546
2590
|
} : {},
|
|
2591
|
+
...void 0 !== keyboardShortcutDelay ? {
|
|
2592
|
+
keyboardShortcutDelay
|
|
2593
|
+
} : {},
|
|
2547
2594
|
...this.options.keepXvfbAliveUntilProcessExit ? {
|
|
2548
2595
|
keepXvfbAliveUntilProcessExit: true
|
|
2549
2596
|
} : {},
|
|
@@ -2628,7 +2675,7 @@ const tools = new ComputerMidsceneTools({
|
|
|
2628
2675
|
});
|
|
2629
2676
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
|
|
2630
2677
|
stripPrefix: 'computer_',
|
|
2631
|
-
version: "1.12.
|
|
2678
|
+
version: "1.12.5-beta-20260907092148.0",
|
|
2632
2679
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
2633
2680
|
}).catch((e)=>{
|
|
2634
2681
|
process.exit((0, cli_namespaceObject.reportCLIError)(e));
|
package/dist/lib/index.js
CHANGED
|
@@ -142,6 +142,29 @@ class ComputerInputDriver {
|
|
|
142
142
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
143
143
|
else lib.keyTap(key);
|
|
144
144
|
}
|
|
145
|
+
keyToggle(key, state, modifiers) {
|
|
146
|
+
const lib = this.getLibnutOrThrow('keyToggle');
|
|
147
|
+
if (void 0 !== modifiers) lib.keyToggle(key, state, modifiers);
|
|
148
|
+
else lib.keyToggle(key, state);
|
|
149
|
+
}
|
|
150
|
+
async keyTapWithExplicitModifiers(key, modifiers, delayMs) {
|
|
151
|
+
const uniqueModifiers = [
|
|
152
|
+
...new Set(modifiers)
|
|
153
|
+
];
|
|
154
|
+
if (0 === uniqueModifiers.length) return void this.keyTap(key);
|
|
155
|
+
const pressedModifiers = [];
|
|
156
|
+
try {
|
|
157
|
+
for (const modifier of uniqueModifiers){
|
|
158
|
+
this.keyToggle(modifier, 'down');
|
|
159
|
+
pressedModifiers.push(modifier);
|
|
160
|
+
}
|
|
161
|
+
await this.delay(delayMs);
|
|
162
|
+
this.keyTap(key);
|
|
163
|
+
await this.delay(delayMs);
|
|
164
|
+
} finally{
|
|
165
|
+
for (const modifier of pressedModifiers.reverse())this.releaseKey(modifier);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
145
168
|
typeString(text) {
|
|
146
169
|
this.getLibnutOrThrow('typeString').typeString(text);
|
|
147
170
|
}
|
|
@@ -214,6 +237,15 @@ class ComputerInputDriver {
|
|
|
214
237
|
this.options.debug(`Failed to release mouse button ${button}: ${error}`);
|
|
215
238
|
}
|
|
216
239
|
}
|
|
240
|
+
releaseKey(key) {
|
|
241
|
+
try {
|
|
242
|
+
const libnut = this.options.getLibnut();
|
|
243
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
244
|
+
libnut.keyToggle(key, 'up');
|
|
245
|
+
} catch (error) {
|
|
246
|
+
this.options.debug(`Failed to release key ${key}: ${error}`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
217
249
|
rejectPendingInputDelays() {
|
|
218
250
|
const error = this.createDestroyedError('in-flight input');
|
|
219
251
|
for (const waitRef of this.pendingInputDelayWaits){
|
|
@@ -575,8 +607,10 @@ async function captureScreenshot(options) {
|
|
|
575
607
|
}
|
|
576
608
|
const SMOOTH_MOVE_STEPS_TAP = 8;
|
|
577
609
|
const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
610
|
+
const SMOOTH_MOVE_STEPS_DRAG = 20;
|
|
578
611
|
const SMOOTH_MOVE_DELAY_TAP = 8;
|
|
579
612
|
const SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
613
|
+
const SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
580
614
|
const MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
581
615
|
const CLICK_SETTLE_DELAY = 50;
|
|
582
616
|
const CLICK_HOLD_DURATION = 100;
|
|
@@ -1203,7 +1237,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1203
1237
|
}
|
|
1204
1238
|
async healthCheck(displays) {
|
|
1205
1239
|
console.log('[HealthCheck] Starting health check...');
|
|
1206
|
-
console.log("[HealthCheck] @midscene/computer v1.12.
|
|
1240
|
+
console.log("[HealthCheck] @midscene/computer v1.12.5-beta-20260907092148.0");
|
|
1207
1241
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1208
1242
|
const screenshotTimeout = 15000;
|
|
1209
1243
|
let timeoutId;
|
|
@@ -1439,6 +1473,8 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1439
1473
|
modifiers,
|
|
1440
1474
|
driver: this.useAppleScript ? "applescript" : 'libnut'
|
|
1441
1475
|
});
|
|
1476
|
+
const shortcutDelay = this.options?.keyboardShortcutDelay ?? 0;
|
|
1477
|
+
if (!this.useAppleScript && modifiers.length > 0 && shortcutDelay > 0) return void await this.inputDriver.keyTapWithExplicitModifiers(key, modifiers, shortcutDelay);
|
|
1442
1478
|
this.inputDriver.sendKey(key, modifiers);
|
|
1443
1479
|
}
|
|
1444
1480
|
resolveUntargetedScrollPoint(screenSize) {
|
|
@@ -1680,7 +1716,10 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1680
1716
|
await this.moveDisplayPointer(from, 'Mouse did not reach the drag start target');
|
|
1681
1717
|
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
1682
1718
|
await this.inputDriver.delay(100);
|
|
1683
|
-
await this.moveDisplayPointer(to, 'Mouse did not reach the drag end target'
|
|
1719
|
+
await this.moveDisplayPointer(to, 'Mouse did not reach the drag end target', {
|
|
1720
|
+
smoothSteps: SMOOTH_MOVE_STEPS_DRAG,
|
|
1721
|
+
smoothDelay: SMOOTH_MOVE_DELAY_DRAG
|
|
1722
|
+
});
|
|
1684
1723
|
await this.inputDriver.delay(100);
|
|
1685
1724
|
});
|
|
1686
1725
|
}
|
|
@@ -1715,6 +1754,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1715
1754
|
}
|
|
1716
1755
|
}
|
|
1717
1756
|
});
|
|
1757
|
+
if (options?.keyboardShortcutDelay !== void 0 && (!Number.isFinite(options.keyboardShortcutDelay) || options.keyboardShortcutDelay < 0)) throw new Error('keyboardShortcutDelay must be a finite non-negative number');
|
|
1718
1758
|
this.options = options;
|
|
1719
1759
|
this.displayId = options?.displayId;
|
|
1720
1760
|
this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
|
|
@@ -2164,10 +2204,10 @@ function rdp_device_define_property(obj, key, value) {
|
|
|
2164
2204
|
const device_debug = (0, logger_namespaceObject.getDebug)('rdp:device');
|
|
2165
2205
|
const device_SMOOTH_MOVE_STEPS_TAP = 8;
|
|
2166
2206
|
const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
2167
|
-
const
|
|
2207
|
+
const device_SMOOTH_MOVE_STEPS_DRAG = 12;
|
|
2168
2208
|
const device_SMOOTH_MOVE_DELAY_TAP = 8;
|
|
2169
2209
|
const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
2170
|
-
const
|
|
2210
|
+
const device_SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
2171
2211
|
const device_MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
2172
2212
|
const device_CLICK_HOLD_DURATION = 50;
|
|
2173
2213
|
const DRAG_HOLD_DURATION = 100;
|
|
@@ -2370,8 +2410,8 @@ class RDPDevice {
|
|
|
2370
2410
|
await this.backend.mouseButton('left', 'down');
|
|
2371
2411
|
await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
|
|
2372
2412
|
await this.movePointer(Math.round(to.x), Math.round(to.y), {
|
|
2373
|
-
steps:
|
|
2374
|
-
stepDelayMs:
|
|
2413
|
+
steps: device_SMOOTH_MOVE_STEPS_DRAG,
|
|
2414
|
+
stepDelayMs: device_SMOOTH_MOVE_DELAY_DRAG
|
|
2375
2415
|
});
|
|
2376
2416
|
await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
|
|
2377
2417
|
await this.backend.mouseButton('left', 'up');
|
|
@@ -2462,6 +2502,7 @@ function createLocalComputerDevice(opts) {
|
|
|
2462
2502
|
keyboardTypeDelay: opts?.keyboardTypeDelay,
|
|
2463
2503
|
inputStrategy: opts?.inputStrategy,
|
|
2464
2504
|
keyboardDriver: opts?.keyboardDriver,
|
|
2505
|
+
keyboardShortcutDelay: opts?.keyboardShortcutDelay,
|
|
2465
2506
|
headless: opts?.headless,
|
|
2466
2507
|
xvfbResolution: opts?.xvfbResolution,
|
|
2467
2508
|
keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
|
|
@@ -2522,6 +2563,7 @@ const computerInitArgShape = {
|
|
|
2522
2563
|
headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
|
|
2523
2564
|
keyboardTypeDelay: core_namespaceObject.z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between keystrokes. In "legacy" mode, positive values enable key-by-key input; zero or omitted uses clipboard input.'),
|
|
2524
2565
|
inputStrategy: core_namespaceObject.z["enum"](device_namespaceObject.inputStrategies).optional().describe('Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.'),
|
|
2566
|
+
keyboardShortcutDelay: core_namespaceObject.z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between modifier transitions and the main key for local libnut shortcuts. Positive values can improve compatibility with full-screen remote-control clients. Ignored in RDP mode and by the macOS AppleScript driver.'),
|
|
2525
2567
|
host: core_namespaceObject.z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
|
|
2526
2568
|
port: core_namespaceObject.z.number().optional().describe('RDP port (default 3389). Requires host.'),
|
|
2527
2569
|
username: core_namespaceObject.z.string().optional().describe('RDP username. Requires host.'),
|
|
@@ -2538,7 +2580,7 @@ const computerInitArgShape = {
|
|
|
2538
2580
|
function adaptComputerInitArgs(extracted) {
|
|
2539
2581
|
if (!extracted || 0 === Object.keys(extracted).length) return;
|
|
2540
2582
|
if (extracted.host) {
|
|
2541
|
-
const { displayId: _d, headless: _h, ...rdpFields } = extracted;
|
|
2583
|
+
const { displayId: _d, headless: _h, keyboardShortcutDelay: _s, ...rdpFields } = extracted;
|
|
2542
2584
|
const host = normalizeRdpHost(extracted.host);
|
|
2543
2585
|
return {
|
|
2544
2586
|
mode: 'rdp',
|
|
@@ -2552,6 +2594,7 @@ function adaptComputerInitArgs(extracted) {
|
|
|
2552
2594
|
headless: extracted.headless,
|
|
2553
2595
|
keyboardTypeDelay: extracted.keyboardTypeDelay,
|
|
2554
2596
|
inputStrategy: extracted.inputStrategy,
|
|
2597
|
+
keyboardShortcutDelay: extracted.keyboardShortcutDelay,
|
|
2555
2598
|
...(0, agent_behavior_init_args_namespaceObject.extractAgentBehaviorInitArgs)(extracted) ?? {}
|
|
2556
2599
|
};
|
|
2557
2600
|
}
|
|
@@ -2603,6 +2646,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2603
2646
|
const headless = opts?.mode === 'local' ? opts.headless : void 0;
|
|
2604
2647
|
const keyboardTypeDelay = opts?.keyboardTypeDelay;
|
|
2605
2648
|
const inputStrategy = opts?.inputStrategy;
|
|
2649
|
+
const keyboardShortcutDelay = opts?.mode === 'local' ? opts.keyboardShortcutDelay : void 0;
|
|
2606
2650
|
agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
2607
2651
|
const agentOpts = {
|
|
2608
2652
|
...displayId ? {
|
|
@@ -2617,6 +2661,9 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2617
2661
|
...void 0 !== inputStrategy ? {
|
|
2618
2662
|
inputStrategy
|
|
2619
2663
|
} : {},
|
|
2664
|
+
...void 0 !== keyboardShortcutDelay ? {
|
|
2665
|
+
keyboardShortcutDelay
|
|
2666
|
+
} : {},
|
|
2620
2667
|
...this.options.keepXvfbAliveUntilProcessExit ? {
|
|
2621
2668
|
keepXvfbAliveUntilProcessExit: true
|
|
2622
2669
|
} : {},
|
|
@@ -2698,7 +2745,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2698
2745
|
}
|
|
2699
2746
|
const env_namespaceObject = require("@midscene/shared/env");
|
|
2700
2747
|
function version() {
|
|
2701
|
-
const currentVersion = "1.12.
|
|
2748
|
+
const currentVersion = "1.12.5-beta-20260907092148.0";
|
|
2702
2749
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2703
2750
|
return currentVersion;
|
|
2704
2751
|
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -177,6 +177,17 @@ export declare interface ComputerDeviceOpt extends ComputerDeviceInputOpt {
|
|
|
177
177
|
* - 'libnut': Use libnut's keyTap (faster but may not work with some TUI apps)
|
|
178
178
|
*/
|
|
179
179
|
keyboardDriver?: 'applescript' | 'libnut';
|
|
180
|
+
/**
|
|
181
|
+
* Delay in milliseconds between explicit modifier transitions and the main
|
|
182
|
+
* key for local libnut shortcuts. A positive value changes modified
|
|
183
|
+
* shortcuts from one `keyTap` call into modifier-down, main-key, and
|
|
184
|
+
* modifier-up phases. This can help foreground clients whose full-screen
|
|
185
|
+
* keyboard capture misses rapidly synthesized modifier state changes.
|
|
186
|
+
*
|
|
187
|
+
* Ignored by the macOS AppleScript driver and RDP mode.
|
|
188
|
+
* @default 0
|
|
189
|
+
*/
|
|
190
|
+
keyboardShortcutDelay?: number;
|
|
180
191
|
/**
|
|
181
192
|
* Headless mode via Xvfb (Linux only).
|
|
182
193
|
* - true: start Xvfb virtual display
|
|
@@ -210,7 +221,7 @@ export declare type ComputerInterface = ComputerDevice | RDPDevice;
|
|
|
210
221
|
/** Init args for the local desktop agent (macOS/Windows/Linux). */
|
|
211
222
|
declare type ComputerLocalInitArgs = {
|
|
212
223
|
mode: 'local';
|
|
213
|
-
} & Pick<ComputerDeviceOpt, 'displayId' | 'headless'> & Pick<ComputerDeviceOpt, 'inputStrategy' | 'keyboardTypeDelay'> & AgentBehaviorInitArgs;
|
|
224
|
+
} & Pick<ComputerDeviceOpt, 'displayId' | 'headless'> & Pick<ComputerDeviceOpt, 'inputStrategy' | 'keyboardTypeDelay' | 'keyboardShortcutDelay'> & AgentBehaviorInitArgs;
|
|
214
225
|
|
|
215
226
|
/**
|
|
216
227
|
* Computer-specific tools manager
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/computer",
|
|
3
|
-
"version": "1.12.
|
|
3
|
+
"version": "1.12.5-beta-20260907092148.0",
|
|
4
4
|
"description": "Midscene.js Computer Desktop Automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -33,8 +33,8 @@
|
|
|
33
33
|
"@computer-use/libnut": "^4.2.0",
|
|
34
34
|
"clipboardy": "^4.0.0",
|
|
35
35
|
"screenshot-desktop": "^1.15.3",
|
|
36
|
-
"@midscene/core": "1.12.
|
|
37
|
-
"@midscene/shared": "1.12.
|
|
36
|
+
"@midscene/core": "1.12.5-beta-20260907092148.0",
|
|
37
|
+
"@midscene/shared": "1.12.5-beta-20260907092148.0"
|
|
38
38
|
},
|
|
39
39
|
"optionalDependencies": {
|
|
40
40
|
"node-mac-permissions": "2.5.0"
|