@midscene/computer 1.12.5 → 1.12.6-beta-20260909033030.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -49
- package/dist/es/cli.mjs +269 -155
- package/dist/es/index.mjs +269 -155
- package/dist/lib/cli.js +269 -155
- package/dist/lib/index.js +269 -155
- package/dist/types/index.d.ts +39 -1
- package/package.json +3 -3
package/dist/lib/index.js
CHANGED
|
@@ -68,6 +68,207 @@ const img_namespaceObject = require("@midscene/shared/img");
|
|
|
68
68
|
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
69
69
|
const external_screenshot_desktop_namespaceObject = require("screenshot-desktop");
|
|
70
70
|
var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
|
|
71
|
+
const US_SHIFTED_CHARACTER_KEYS = new Map([
|
|
72
|
+
[
|
|
73
|
+
'~',
|
|
74
|
+
'`'
|
|
75
|
+
],
|
|
76
|
+
[
|
|
77
|
+
'!',
|
|
78
|
+
'1'
|
|
79
|
+
],
|
|
80
|
+
[
|
|
81
|
+
'@',
|
|
82
|
+
'2'
|
|
83
|
+
],
|
|
84
|
+
[
|
|
85
|
+
'#',
|
|
86
|
+
'3'
|
|
87
|
+
],
|
|
88
|
+
[
|
|
89
|
+
'$',
|
|
90
|
+
'4'
|
|
91
|
+
],
|
|
92
|
+
[
|
|
93
|
+
'%',
|
|
94
|
+
'5'
|
|
95
|
+
],
|
|
96
|
+
[
|
|
97
|
+
'^',
|
|
98
|
+
'6'
|
|
99
|
+
],
|
|
100
|
+
[
|
|
101
|
+
'&',
|
|
102
|
+
'7'
|
|
103
|
+
],
|
|
104
|
+
[
|
|
105
|
+
'*',
|
|
106
|
+
'8'
|
|
107
|
+
],
|
|
108
|
+
[
|
|
109
|
+
'(',
|
|
110
|
+
'9'
|
|
111
|
+
],
|
|
112
|
+
[
|
|
113
|
+
')',
|
|
114
|
+
'0'
|
|
115
|
+
],
|
|
116
|
+
[
|
|
117
|
+
'_',
|
|
118
|
+
'-'
|
|
119
|
+
],
|
|
120
|
+
[
|
|
121
|
+
'+',
|
|
122
|
+
'='
|
|
123
|
+
],
|
|
124
|
+
[
|
|
125
|
+
'{',
|
|
126
|
+
'['
|
|
127
|
+
],
|
|
128
|
+
[
|
|
129
|
+
'}',
|
|
130
|
+
']'
|
|
131
|
+
],
|
|
132
|
+
[
|
|
133
|
+
'|',
|
|
134
|
+
'\\'
|
|
135
|
+
],
|
|
136
|
+
[
|
|
137
|
+
':',
|
|
138
|
+
';'
|
|
139
|
+
],
|
|
140
|
+
[
|
|
141
|
+
'"',
|
|
142
|
+
"'"
|
|
143
|
+
],
|
|
144
|
+
[
|
|
145
|
+
'<',
|
|
146
|
+
','
|
|
147
|
+
],
|
|
148
|
+
[
|
|
149
|
+
'>',
|
|
150
|
+
'.'
|
|
151
|
+
],
|
|
152
|
+
[
|
|
153
|
+
'?',
|
|
154
|
+
'/'
|
|
155
|
+
]
|
|
156
|
+
]);
|
|
157
|
+
function resolveUSShiftedKey(character) {
|
|
158
|
+
if (/^[A-Z]$/.test(character)) return character.toLowerCase();
|
|
159
|
+
return US_SHIFTED_CHARACTER_KEYS.get(character);
|
|
160
|
+
}
|
|
161
|
+
const debugKeyboard = (0, logger_namespaceObject.getDebug)('computer:keyboard');
|
|
162
|
+
const APPLE_SCRIPT_KEY_CODES = {
|
|
163
|
+
return: 36,
|
|
164
|
+
enter: 36,
|
|
165
|
+
tab: 48,
|
|
166
|
+
space: 49,
|
|
167
|
+
backspace: 51,
|
|
168
|
+
delete: 51,
|
|
169
|
+
escape: 53,
|
|
170
|
+
forwarddelete: 117,
|
|
171
|
+
left: 123,
|
|
172
|
+
right: 124,
|
|
173
|
+
down: 125,
|
|
174
|
+
up: 126,
|
|
175
|
+
home: 115,
|
|
176
|
+
end: 119,
|
|
177
|
+
pageup: 116,
|
|
178
|
+
pagedown: 121,
|
|
179
|
+
f1: 122,
|
|
180
|
+
f2: 120,
|
|
181
|
+
f3: 99,
|
|
182
|
+
f4: 118,
|
|
183
|
+
f5: 96,
|
|
184
|
+
f6: 97,
|
|
185
|
+
f7: 98,
|
|
186
|
+
f8: 100,
|
|
187
|
+
f9: 101,
|
|
188
|
+
f10: 109,
|
|
189
|
+
f11: 103,
|
|
190
|
+
f12: 111
|
|
191
|
+
};
|
|
192
|
+
const APPLE_SCRIPT_MODIFIER_KEYS = {
|
|
193
|
+
command: 'command',
|
|
194
|
+
cmd: 'command',
|
|
195
|
+
control: 'control',
|
|
196
|
+
ctrl: 'control',
|
|
197
|
+
shift: 'shift',
|
|
198
|
+
alt: 'option',
|
|
199
|
+
option: 'option',
|
|
200
|
+
meta: 'command'
|
|
201
|
+
};
|
|
202
|
+
function buildKeyCommand(key) {
|
|
203
|
+
const keyCode = APPLE_SCRIPT_KEY_CODES[key.toLowerCase()];
|
|
204
|
+
if (void 0 !== keyCode) return `key code ${keyCode}`;
|
|
205
|
+
const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
206
|
+
return `keystroke "${escapedKey}"`;
|
|
207
|
+
}
|
|
208
|
+
function resolveModifierKeys(modifiers) {
|
|
209
|
+
return modifiers.map((modifier)=>APPLE_SCRIPT_MODIFIER_KEYS[modifier.toLowerCase()]).filter((modifier)=>void 0 !== modifier);
|
|
210
|
+
}
|
|
211
|
+
function buildLogicalKeyPress(key, modifiers) {
|
|
212
|
+
const modifierKeys = resolveModifierKeys(modifiers);
|
|
213
|
+
const modifierClause = modifierKeys.length ? ` using {${modifierKeys.map((modifier)=>`${modifier} down`).join(', ')}}` : '';
|
|
214
|
+
return `tell application "System Events" to ${buildKeyCommand(key)}${modifierClause}`;
|
|
215
|
+
}
|
|
216
|
+
function resolvePhysicalKey(key, modifiers) {
|
|
217
|
+
const resolvedModifiers = [
|
|
218
|
+
...modifiers
|
|
219
|
+
];
|
|
220
|
+
const shiftedBaseKey = resolveUSShiftedKey(key);
|
|
221
|
+
if (void 0 !== shiftedBaseKey) {
|
|
222
|
+
resolvedModifiers.push('shift');
|
|
223
|
+
return {
|
|
224
|
+
key: shiftedBaseKey,
|
|
225
|
+
modifiers: resolvedModifiers
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
return {
|
|
229
|
+
key,
|
|
230
|
+
modifiers: resolvedModifiers
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
function buildPhysicalKeyPress(key, modifiers) {
|
|
234
|
+
const resolved = resolvePhysicalKey(key, modifiers);
|
|
235
|
+
const modifierKeys = [
|
|
236
|
+
...new Set(resolveModifierKeys(resolved.modifiers))
|
|
237
|
+
];
|
|
238
|
+
const keyCommand = buildKeyCommand(resolved.key);
|
|
239
|
+
if (0 === modifierKeys.length) return `tell application "System Events" to ${keyCommand}`;
|
|
240
|
+
const releaseCommands = [
|
|
241
|
+
...modifierKeys
|
|
242
|
+
].reverse().map((modifier)=>`key up ${modifier}`);
|
|
243
|
+
return [
|
|
244
|
+
'tell application "System Events"',
|
|
245
|
+
'try',
|
|
246
|
+
...modifierKeys.map((modifier)=>`key down ${modifier}`),
|
|
247
|
+
keyCommand,
|
|
248
|
+
'on error errorMessage number errorNumber',
|
|
249
|
+
...releaseCommands,
|
|
250
|
+
'error errorMessage number errorNumber',
|
|
251
|
+
'end try',
|
|
252
|
+
...releaseCommands,
|
|
253
|
+
'end tell'
|
|
254
|
+
].join('\n');
|
|
255
|
+
}
|
|
256
|
+
function buildAppleScriptKeyPress(key, modifiers = [], eventMode = 'logical') {
|
|
257
|
+
return 'physical' === eventMode ? buildPhysicalKeyPress(key, modifiers) : buildLogicalKeyPress(key, modifiers);
|
|
258
|
+
}
|
|
259
|
+
function sendKeyViaAppleScript(key, modifiers = [], eventMode = 'logical') {
|
|
260
|
+
const script = buildAppleScriptKeyPress(key, modifiers, eventMode);
|
|
261
|
+
debugKeyboard('sendKeyViaAppleScript', {
|
|
262
|
+
key,
|
|
263
|
+
modifiers,
|
|
264
|
+
eventMode,
|
|
265
|
+
script
|
|
266
|
+
});
|
|
267
|
+
(0, external_node_child_process_namespaceObject.execFileSync)("osascript", [
|
|
268
|
+
'-e',
|
|
269
|
+
script
|
|
270
|
+
]);
|
|
271
|
+
}
|
|
71
272
|
const external_node_assert_namespaceObject = require("node:assert");
|
|
72
273
|
var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
|
|
73
274
|
function _define_property(obj, key, value) {
|
|
@@ -142,6 +343,29 @@ class ComputerInputDriver {
|
|
|
142
343
|
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
143
344
|
else lib.keyTap(key);
|
|
144
345
|
}
|
|
346
|
+
keyToggle(key, state, modifiers) {
|
|
347
|
+
const lib = this.getLibnutOrThrow('keyToggle');
|
|
348
|
+
if (void 0 !== modifiers) lib.keyToggle(key, state, modifiers);
|
|
349
|
+
else lib.keyToggle(key, state);
|
|
350
|
+
}
|
|
351
|
+
async keyTapWithExplicitModifiers(key, modifiers, delayMs) {
|
|
352
|
+
const uniqueModifiers = [
|
|
353
|
+
...new Set(modifiers)
|
|
354
|
+
];
|
|
355
|
+
if (0 === uniqueModifiers.length) return void this.keyTap(key);
|
|
356
|
+
const pressedModifiers = [];
|
|
357
|
+
try {
|
|
358
|
+
for (const modifier of uniqueModifiers){
|
|
359
|
+
this.keyToggle(modifier, 'down');
|
|
360
|
+
pressedModifiers.push(modifier);
|
|
361
|
+
}
|
|
362
|
+
await this.delay(delayMs);
|
|
363
|
+
this.keyTap(key);
|
|
364
|
+
await this.delay(delayMs);
|
|
365
|
+
} finally{
|
|
366
|
+
for (const modifier of pressedModifiers.reverse())this.releaseKey(modifier);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
145
369
|
typeString(text) {
|
|
146
370
|
this.getLibnutOrThrow('typeString').typeString(text);
|
|
147
371
|
}
|
|
@@ -214,6 +438,15 @@ class ComputerInputDriver {
|
|
|
214
438
|
this.options.debug(`Failed to release mouse button ${button}: ${error}`);
|
|
215
439
|
}
|
|
216
440
|
}
|
|
441
|
+
releaseKey(key) {
|
|
442
|
+
try {
|
|
443
|
+
const libnut = this.options.getLibnut();
|
|
444
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
445
|
+
libnut.keyToggle(key, 'up');
|
|
446
|
+
} catch (error) {
|
|
447
|
+
this.options.debug(`Failed to release key ${key}: ${error}`);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
217
450
|
rejectPendingInputDelays() {
|
|
218
451
|
const error = this.createDestroyedError('in-flight input');
|
|
219
452
|
for (const waitRef of this.pendingInputDelayWaits){
|
|
@@ -595,92 +828,6 @@ const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
|
595
828
|
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
596
829
|
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
597
830
|
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
598
|
-
const LINUX_SHIFTED_CHARACTER_KEYS = new Map([
|
|
599
|
-
[
|
|
600
|
-
'~',
|
|
601
|
-
'`'
|
|
602
|
-
],
|
|
603
|
-
[
|
|
604
|
-
'!',
|
|
605
|
-
'1'
|
|
606
|
-
],
|
|
607
|
-
[
|
|
608
|
-
'@',
|
|
609
|
-
'2'
|
|
610
|
-
],
|
|
611
|
-
[
|
|
612
|
-
'#',
|
|
613
|
-
'3'
|
|
614
|
-
],
|
|
615
|
-
[
|
|
616
|
-
'$',
|
|
617
|
-
'4'
|
|
618
|
-
],
|
|
619
|
-
[
|
|
620
|
-
'%',
|
|
621
|
-
'5'
|
|
622
|
-
],
|
|
623
|
-
[
|
|
624
|
-
'^',
|
|
625
|
-
'6'
|
|
626
|
-
],
|
|
627
|
-
[
|
|
628
|
-
'&',
|
|
629
|
-
'7'
|
|
630
|
-
],
|
|
631
|
-
[
|
|
632
|
-
'*',
|
|
633
|
-
'8'
|
|
634
|
-
],
|
|
635
|
-
[
|
|
636
|
-
'(',
|
|
637
|
-
'9'
|
|
638
|
-
],
|
|
639
|
-
[
|
|
640
|
-
')',
|
|
641
|
-
'0'
|
|
642
|
-
],
|
|
643
|
-
[
|
|
644
|
-
'_',
|
|
645
|
-
'-'
|
|
646
|
-
],
|
|
647
|
-
[
|
|
648
|
-
'+',
|
|
649
|
-
'='
|
|
650
|
-
],
|
|
651
|
-
[
|
|
652
|
-
'{',
|
|
653
|
-
'['
|
|
654
|
-
],
|
|
655
|
-
[
|
|
656
|
-
'}',
|
|
657
|
-
']'
|
|
658
|
-
],
|
|
659
|
-
[
|
|
660
|
-
'|',
|
|
661
|
-
'\\'
|
|
662
|
-
],
|
|
663
|
-
[
|
|
664
|
-
':',
|
|
665
|
-
';'
|
|
666
|
-
],
|
|
667
|
-
[
|
|
668
|
-
'"',
|
|
669
|
-
"'"
|
|
670
|
-
],
|
|
671
|
-
[
|
|
672
|
-
'<',
|
|
673
|
-
','
|
|
674
|
-
],
|
|
675
|
-
[
|
|
676
|
-
'>',
|
|
677
|
-
'.'
|
|
678
|
-
],
|
|
679
|
-
[
|
|
680
|
-
'?',
|
|
681
|
-
'/'
|
|
682
|
-
]
|
|
683
|
-
]);
|
|
684
831
|
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
685
832
|
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
686
833
|
const EDGE_SCROLL_SPEC = {
|
|
@@ -717,67 +864,6 @@ const EDGE_SCROLL_SPEC = {
|
|
|
717
864
|
]
|
|
718
865
|
}
|
|
719
866
|
};
|
|
720
|
-
const APPLESCRIPT_KEY_CODE_MAP = {
|
|
721
|
-
return: 36,
|
|
722
|
-
enter: 36,
|
|
723
|
-
tab: 48,
|
|
724
|
-
space: 49,
|
|
725
|
-
backspace: 51,
|
|
726
|
-
delete: 51,
|
|
727
|
-
escape: 53,
|
|
728
|
-
forwarddelete: 117,
|
|
729
|
-
left: 123,
|
|
730
|
-
right: 124,
|
|
731
|
-
down: 125,
|
|
732
|
-
up: 126,
|
|
733
|
-
home: 115,
|
|
734
|
-
end: 119,
|
|
735
|
-
pageup: 116,
|
|
736
|
-
pagedown: 121,
|
|
737
|
-
f1: 122,
|
|
738
|
-
f2: 120,
|
|
739
|
-
f3: 99,
|
|
740
|
-
f4: 118,
|
|
741
|
-
f5: 96,
|
|
742
|
-
f6: 97,
|
|
743
|
-
f7: 98,
|
|
744
|
-
f8: 100,
|
|
745
|
-
f9: 101,
|
|
746
|
-
f10: 109,
|
|
747
|
-
f11: 103,
|
|
748
|
-
f12: 111
|
|
749
|
-
};
|
|
750
|
-
const APPLESCRIPT_MODIFIER_MAP = {
|
|
751
|
-
command: 'command down',
|
|
752
|
-
cmd: 'command down',
|
|
753
|
-
control: 'control down',
|
|
754
|
-
ctrl: 'control down',
|
|
755
|
-
shift: 'shift down',
|
|
756
|
-
alt: 'option down',
|
|
757
|
-
option: 'option down',
|
|
758
|
-
meta: 'command down'
|
|
759
|
-
};
|
|
760
|
-
function sendKeyViaAppleScript(key, modifiers = []) {
|
|
761
|
-
const lowerKey = key.toLowerCase();
|
|
762
|
-
const keyCode = APPLESCRIPT_KEY_CODE_MAP[lowerKey];
|
|
763
|
-
const modifierParts = modifiers.map((m)=>APPLESCRIPT_MODIFIER_MAP[m.toLowerCase()]).filter(Boolean);
|
|
764
|
-
const modifierStr = modifierParts.length > 0 ? ` using {${modifierParts.join(', ')}}` : '';
|
|
765
|
-
let script;
|
|
766
|
-
if (void 0 !== keyCode) script = `tell application "System Events" to key code ${keyCode}${modifierStr}`;
|
|
767
|
-
else {
|
|
768
|
-
const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
769
|
-
script = `tell application "System Events" to keystroke "${escapedKey}"${modifierStr}`;
|
|
770
|
-
}
|
|
771
|
-
debugDevice('sendKeyViaAppleScript', {
|
|
772
|
-
key,
|
|
773
|
-
modifiers,
|
|
774
|
-
script
|
|
775
|
-
});
|
|
776
|
-
(0, external_node_child_process_namespaceObject.execFileSync)("osascript", [
|
|
777
|
-
'-e',
|
|
778
|
-
script
|
|
779
|
-
]);
|
|
780
|
-
}
|
|
781
867
|
function escapePowershellSingleQuoted(value) {
|
|
782
868
|
return value.replace(/'/g, "''");
|
|
783
869
|
}
|
|
@@ -1205,7 +1291,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1205
1291
|
}
|
|
1206
1292
|
async healthCheck(displays) {
|
|
1207
1293
|
console.log('[HealthCheck] Starting health check...');
|
|
1208
|
-
console.log("[HealthCheck] @midscene/computer v1.12.
|
|
1294
|
+
console.log("[HealthCheck] @midscene/computer v1.12.6-beta-20260909033030.0");
|
|
1209
1295
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1210
1296
|
const screenshotTimeout = 15000;
|
|
1211
1297
|
let timeoutId;
|
|
@@ -1382,7 +1468,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1382
1468
|
]);
|
|
1383
1469
|
else {
|
|
1384
1470
|
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
1385
|
-
this.
|
|
1471
|
+
await this.sendLocalModifiedKey('v', [
|
|
1386
1472
|
modifier
|
|
1387
1473
|
]);
|
|
1388
1474
|
}
|
|
@@ -1399,12 +1485,16 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1399
1485
|
}
|
|
1400
1486
|
async typeStringWithDelay(text, keyboardTypeDelay) {
|
|
1401
1487
|
await (0, device_namespaceObject.sendTextSequentially)(text.replace(/\r\n?/g, '\n'), {
|
|
1402
|
-
sendCharacter: (character)=>{
|
|
1403
|
-
const linuxShiftedKey = 'linux' === process.platform ?
|
|
1488
|
+
sendCharacter: async (character)=>{
|
|
1489
|
+
const linuxShiftedKey = 'linux' === process.platform ? US_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
|
|
1490
|
+
const pacedShiftedKey = !this.useAppleScript && (this.options?.keyboardShortcutDelay ?? 0) > 0 ? resolveUSShiftedKey(character) : void 0;
|
|
1404
1491
|
if ('\n' === character) this.inputDriver.sendKey('return');
|
|
1405
1492
|
else if ('\t' === character) this.inputDriver.sendKey('tab');
|
|
1406
1493
|
else if (' ' === character) this.inputDriver.sendKey('space');
|
|
1407
1494
|
else if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript(character);
|
|
1495
|
+
else if (void 0 !== pacedShiftedKey) await this.sendLocalModifiedKey(pacedShiftedKey, [
|
|
1496
|
+
'shift'
|
|
1497
|
+
]);
|
|
1408
1498
|
else if (void 0 !== linuxShiftedKey) this.inputDriver.keyTap(linuxShiftedKey, [
|
|
1409
1499
|
'shift'
|
|
1410
1500
|
]);
|
|
@@ -1425,12 +1515,17 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1425
1515
|
return;
|
|
1426
1516
|
}
|
|
1427
1517
|
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
1428
|
-
this.
|
|
1518
|
+
await this.sendLocalModifiedKey('a', [
|
|
1429
1519
|
modifier
|
|
1430
1520
|
]);
|
|
1431
1521
|
await this.inputDriver.delay(50);
|
|
1432
1522
|
this.inputDriver.keyTap('backspace');
|
|
1433
1523
|
}
|
|
1524
|
+
async sendLocalModifiedKey(key, modifiers) {
|
|
1525
|
+
const shortcutDelay = this.options?.keyboardShortcutDelay ?? 0;
|
|
1526
|
+
if (modifiers.length > 0 && shortcutDelay > 0) return void await this.inputDriver.keyTapWithExplicitModifiers(key, modifiers, shortcutDelay);
|
|
1527
|
+
this.inputDriver.keyTap(key, modifiers);
|
|
1528
|
+
}
|
|
1434
1529
|
async pressKeyboardShortcut(keyName) {
|
|
1435
1530
|
const keys = keyName.split('+');
|
|
1436
1531
|
const modifiers = keys.slice(0, -1).map(normalizeKeyName);
|
|
@@ -1441,6 +1536,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1441
1536
|
modifiers,
|
|
1442
1537
|
driver: this.useAppleScript ? "applescript" : 'libnut'
|
|
1443
1538
|
});
|
|
1539
|
+
if (!this.useAppleScript && modifiers.length > 0 && (this.options?.keyboardShortcutDelay ?? 0) > 0) return void await this.sendLocalModifiedKey(key, modifiers);
|
|
1444
1540
|
this.inputDriver.sendKey(key, modifiers);
|
|
1445
1541
|
}
|
|
1446
1542
|
resolveUntargetedScrollPoint(screenSize) {
|
|
@@ -1594,7 +1690,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1594
1690
|
device_define_property(this, "inputDriver", new ComputerInputDriver({
|
|
1595
1691
|
getLibnut: ()=>device_libnut,
|
|
1596
1692
|
useAppleScript: ()=>this.useAppleScript,
|
|
1597
|
-
sendKeyViaAppleScript,
|
|
1693
|
+
sendKeyViaAppleScript: (key, modifiers)=>sendKeyViaAppleScript(key, modifiers, this.options?.keyboardEventMode ?? 'logical'),
|
|
1598
1694
|
runPhasedScroll,
|
|
1599
1695
|
debug: (message)=>debugDevice(message)
|
|
1600
1696
|
}));
|
|
@@ -1720,6 +1816,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1720
1816
|
}
|
|
1721
1817
|
}
|
|
1722
1818
|
});
|
|
1819
|
+
if (options?.keyboardShortcutDelay !== void 0 && (!Number.isFinite(options.keyboardShortcutDelay) || options.keyboardShortcutDelay < 0)) throw new Error('keyboardShortcutDelay must be a finite non-negative number');
|
|
1723
1820
|
this.options = options;
|
|
1724
1821
|
this.displayId = options?.displayId;
|
|
1725
1822
|
this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
|
|
@@ -2467,6 +2564,8 @@ function createLocalComputerDevice(opts) {
|
|
|
2467
2564
|
keyboardTypeDelay: opts?.keyboardTypeDelay,
|
|
2468
2565
|
inputStrategy: opts?.inputStrategy,
|
|
2469
2566
|
keyboardDriver: opts?.keyboardDriver,
|
|
2567
|
+
keyboardEventMode: opts?.keyboardEventMode,
|
|
2568
|
+
keyboardShortcutDelay: opts?.keyboardShortcutDelay,
|
|
2470
2569
|
headless: opts?.headless,
|
|
2471
2570
|
xvfbResolution: opts?.xvfbResolution,
|
|
2472
2571
|
keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
|
|
@@ -2527,6 +2626,11 @@ const computerInitArgShape = {
|
|
|
2527
2626
|
headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
|
|
2528
2627
|
keyboardTypeDelay: core_namespaceObject.z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between keystrokes. In "legacy" mode, positive values enable key-by-key input; zero or omitted uses clipboard input.'),
|
|
2529
2628
|
inputStrategy: core_namespaceObject.z["enum"](device_namespaceObject.inputStrategies).optional().describe('Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.'),
|
|
2629
|
+
keyboardEventMode: core_namespaceObject.z["enum"]([
|
|
2630
|
+
'logical',
|
|
2631
|
+
'physical'
|
|
2632
|
+
]).optional().describe('macOS AppleScript keyboard event mode for local control. "logical" (default) uses a compact keystroke command; "physical" sends explicit modifier transitions for foreground apps that require separate key state changes. Both modes use AppleScript, and "physical" is not hardware input. It assumes an en-US layout for shifted punctuation and requires sequential text input or a positive keyboardTypeDelay. Ignored in RDP mode and outside the macOS AppleScript driver.'),
|
|
2633
|
+
keyboardShortcutDelay: core_namespaceObject.z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds around modifier transitions for local libnut keyboard events. Positive values pace shortcuts and the implicit Shift used by uppercase or en-US punctuation during sequential input. Ignored in RDP mode and by the macOS AppleScript driver.'),
|
|
2530
2634
|
host: core_namespaceObject.z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
|
|
2531
2635
|
port: core_namespaceObject.z.number().optional().describe('RDP port (default 3389). Requires host.'),
|
|
2532
2636
|
username: core_namespaceObject.z.string().optional().describe('RDP username. Requires host.'),
|
|
@@ -2543,7 +2647,7 @@ const computerInitArgShape = {
|
|
|
2543
2647
|
function adaptComputerInitArgs(extracted) {
|
|
2544
2648
|
if (!extracted || 0 === Object.keys(extracted).length) return;
|
|
2545
2649
|
if (extracted.host) {
|
|
2546
|
-
const { displayId: _d, headless: _h, ...rdpFields } = extracted;
|
|
2650
|
+
const { displayId: _d, headless: _h, keyboardEventMode: _k, keyboardShortcutDelay: _s, ...rdpFields } = extracted;
|
|
2547
2651
|
const host = normalizeRdpHost(extracted.host);
|
|
2548
2652
|
return {
|
|
2549
2653
|
mode: 'rdp',
|
|
@@ -2557,6 +2661,8 @@ function adaptComputerInitArgs(extracted) {
|
|
|
2557
2661
|
headless: extracted.headless,
|
|
2558
2662
|
keyboardTypeDelay: extracted.keyboardTypeDelay,
|
|
2559
2663
|
inputStrategy: extracted.inputStrategy,
|
|
2664
|
+
keyboardEventMode: extracted.keyboardEventMode,
|
|
2665
|
+
keyboardShortcutDelay: extracted.keyboardShortcutDelay,
|
|
2560
2666
|
...(0, agent_behavior_init_args_namespaceObject.extractAgentBehaviorInitArgs)(extracted) ?? {}
|
|
2561
2667
|
};
|
|
2562
2668
|
}
|
|
@@ -2608,6 +2714,8 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2608
2714
|
const headless = opts?.mode === 'local' ? opts.headless : void 0;
|
|
2609
2715
|
const keyboardTypeDelay = opts?.keyboardTypeDelay;
|
|
2610
2716
|
const inputStrategy = opts?.inputStrategy;
|
|
2717
|
+
const keyboardEventMode = opts?.mode === 'local' ? opts.keyboardEventMode : void 0;
|
|
2718
|
+
const keyboardShortcutDelay = opts?.mode === 'local' ? opts.keyboardShortcutDelay : void 0;
|
|
2611
2719
|
agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
2612
2720
|
const agentOpts = {
|
|
2613
2721
|
...displayId ? {
|
|
@@ -2622,6 +2730,12 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2622
2730
|
...void 0 !== inputStrategy ? {
|
|
2623
2731
|
inputStrategy
|
|
2624
2732
|
} : {},
|
|
2733
|
+
...void 0 !== keyboardEventMode ? {
|
|
2734
|
+
keyboardEventMode
|
|
2735
|
+
} : {},
|
|
2736
|
+
...void 0 !== keyboardShortcutDelay ? {
|
|
2737
|
+
keyboardShortcutDelay
|
|
2738
|
+
} : {},
|
|
2625
2739
|
...this.options.keepXvfbAliveUntilProcessExit ? {
|
|
2626
2740
|
keepXvfbAliveUntilProcessExit: true
|
|
2627
2741
|
} : {},
|
|
@@ -2703,7 +2817,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2703
2817
|
}
|
|
2704
2818
|
const env_namespaceObject = require("@midscene/shared/env");
|
|
2705
2819
|
function version() {
|
|
2706
|
-
const currentVersion = "1.12.
|
|
2820
|
+
const currentVersion = "1.12.6-beta-20260909033030.0";
|
|
2707
2821
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2708
2822
|
return currentVersion;
|
|
2709
2823
|
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -139,6 +139,7 @@ export declare class ComputerDevice implements AbstractInterface {
|
|
|
139
139
|
private smartTypeString;
|
|
140
140
|
private typeStringWithDelay;
|
|
141
141
|
private selectAllAndDelete;
|
|
142
|
+
private sendLocalModifiedKey;
|
|
142
143
|
private pressKeyboardShortcut;
|
|
143
144
|
private resolveUntargetedScrollPoint;
|
|
144
145
|
private moveMouseToScrollTarget;
|
|
@@ -177,6 +178,32 @@ export declare interface ComputerDeviceOpt extends ComputerDeviceInputOpt {
|
|
|
177
178
|
* - 'libnut': Use libnut's keyTap (faster but may not work with some TUI apps)
|
|
178
179
|
*/
|
|
179
180
|
keyboardDriver?: 'applescript' | 'libnut';
|
|
181
|
+
/**
|
|
182
|
+
* How the macOS AppleScript keyboard driver represents modifier keys.
|
|
183
|
+
* `logical` keeps the default compact `keystroke ... using` behavior.
|
|
184
|
+
* `physical` emits explicit modifier key-down/key-up transitions for
|
|
185
|
+
* foreground apps that require separate modifier state changes, including
|
|
186
|
+
* some remote-control clients and virtual machine consoles. Both modes use
|
|
187
|
+
* AppleScript System Events; `physical` does not emit hardware events.
|
|
188
|
+
* Physical mode assumes an en-US layout for shifted punctuation and may type
|
|
189
|
+
* base keys in native macOS applications. Text input must use sequential
|
|
190
|
+
* input or a positive `keyboardTypeDelay` to emit individual keys.
|
|
191
|
+
*
|
|
192
|
+
* Ignored outside macOS and when `keyboardDriver` is `libnut`.
|
|
193
|
+
* @default 'logical'
|
|
194
|
+
*/
|
|
195
|
+
keyboardEventMode?: KeyboardEventMode;
|
|
196
|
+
/**
|
|
197
|
+
* Delay in milliseconds around explicit modifier transitions for local
|
|
198
|
+
* libnut keyboard events. A positive value changes modified shortcuts and
|
|
199
|
+
* shifted en-US text characters into modifier-down, main-key, and
|
|
200
|
+
* modifier-up phases. This can help foreground clients whose full-screen
|
|
201
|
+
* keyboard capture misses rapidly synthesized modifier state changes.
|
|
202
|
+
*
|
|
203
|
+
* Ignored by the macOS AppleScript driver and RDP mode.
|
|
204
|
+
* @default 0
|
|
205
|
+
*/
|
|
206
|
+
keyboardShortcutDelay?: number;
|
|
180
207
|
/**
|
|
181
208
|
* Headless mode via Xvfb (Linux only).
|
|
182
209
|
* - true: start Xvfb virtual display
|
|
@@ -210,7 +237,7 @@ export declare type ComputerInterface = ComputerDevice | RDPDevice;
|
|
|
210
237
|
/** Init args for the local desktop agent (macOS/Windows/Linux). */
|
|
211
238
|
declare type ComputerLocalInitArgs = {
|
|
212
239
|
mode: 'local';
|
|
213
|
-
} & Pick<ComputerDeviceOpt, 'displayId' | 'headless'> & Pick<ComputerDeviceOpt, 'inputStrategy' | 'keyboardTypeDelay'> & AgentBehaviorInitArgs;
|
|
240
|
+
} & Pick<ComputerDeviceOpt, 'displayId' | 'headless'> & Pick<ComputerDeviceOpt, 'inputStrategy' | 'keyboardTypeDelay' | 'keyboardEventMode' | 'keyboardShortcutDelay'> & AgentBehaviorInitArgs;
|
|
214
241
|
|
|
215
242
|
/**
|
|
216
243
|
* Computer-specific tools manager
|
|
@@ -297,6 +324,17 @@ export declare class HelperProcessRDPBackendClient implements RDPBackendClient {
|
|
|
297
324
|
private shutdownHelper;
|
|
298
325
|
}
|
|
299
326
|
|
|
327
|
+
/**
|
|
328
|
+
* Modifier delivery mode for the macOS AppleScript keyboard backend.
|
|
329
|
+
*
|
|
330
|
+
* `logical` uses the compact `keystroke ... using` form. `physical` emits
|
|
331
|
+
* explicit modifier transitions for foreground applications that require
|
|
332
|
+
* separate key state changes. Both modes use AppleScript System Events;
|
|
333
|
+
* `physical` does not emit hardware events. It assumes an en-US mapping for
|
|
334
|
+
* shifted punctuation and should be enabled only when needed.
|
|
335
|
+
*/
|
|
336
|
+
export declare type KeyboardEventMode = 'logical' | 'physical';
|
|
337
|
+
|
|
300
338
|
export declare type LocalComputerAgentOpt = BaseComputerAgentOpt & Omit<ComputerDeviceOpt, keyof ComputerAgentSharedDeviceOpt>;
|
|
301
339
|
|
|
302
340
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/computer",
|
|
3
|
-
"version": "1.12.
|
|
3
|
+
"version": "1.12.6-beta-20260909033030.0",
|
|
4
4
|
"description": "Midscene.js Computer Desktop Automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -33,8 +33,8 @@
|
|
|
33
33
|
"@computer-use/libnut": "^4.2.0",
|
|
34
34
|
"clipboardy": "^4.0.0",
|
|
35
35
|
"screenshot-desktop": "^1.15.3",
|
|
36
|
-
"@midscene/core": "1.12.
|
|
37
|
-
"@midscene/shared": "1.12.
|
|
36
|
+
"@midscene/core": "1.12.6-beta-20260909033030.0",
|
|
37
|
+
"@midscene/shared": "1.12.6-beta-20260909033030.0"
|
|
38
38
|
},
|
|
39
39
|
"optionalDependencies": {
|
|
40
40
|
"node-mac-permissions": "2.5.0"
|