@midscene/computer 1.9.8 → 1.10.1-beta-20260624112700.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +83 -11
- package/dist/es/index.mjs +83 -11
- package/dist/lib/cli.js +83 -11
- package/dist/lib/index.js +83 -11
- package/dist/types/index.d.ts +19 -5
- package/package.json +3 -8
- package/dist/es/mcp-server.mjs +0 -2088
- package/dist/lib/mcp-server.js +0 -2141
- package/dist/types/mcp-server.d.ts +0 -245
package/dist/es/cli.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { createReportCliCommands, z } from "@midscene/core";
|
|
2
2
|
import { reportCLIError, runToolsCLI } from "@midscene/shared/cli";
|
|
3
|
+
import { agentBehaviorInitArgShape, extractAgentBehaviorInitArgs, getAgentInitArgsSignature, shouldRebuildAgentForInitArgs } from "@midscene/shared/agent-tools/agent-behavior-init-args";
|
|
4
|
+
import { BaseMidsceneTools } from "@midscene/shared/agent-tools/base-tools";
|
|
3
5
|
import { getDebug } from "@midscene/shared/logger";
|
|
4
|
-
import { agentBehaviorInitArgShape, extractAgentBehaviorInitArgs, getAgentInitArgsSignature, shouldRebuildAgentForInitArgs } from "@midscene/shared/mcp/agent-behavior-init-args";
|
|
5
|
-
import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
|
|
6
6
|
import { Agent } from "@midscene/core/agent";
|
|
7
7
|
import { execFileSync, execSync, spawn, spawnSync } from "node:child_process";
|
|
8
8
|
import { chmodSync, existsSync, statSync } from "node:fs";
|
|
@@ -372,6 +372,42 @@ function sendKeyViaAppleScript(key, modifiers = []) {
|
|
|
372
372
|
script
|
|
373
373
|
]);
|
|
374
374
|
}
|
|
375
|
+
const POWERSHELL_TIMEOUT_MS = 15000;
|
|
376
|
+
const POWERSHELL_MAX_BUFFER = 67108864;
|
|
377
|
+
function escapePowershellSingleQuoted(value) {
|
|
378
|
+
return value.replace(/'/g, "''");
|
|
379
|
+
}
|
|
380
|
+
function runPowershell(script) {
|
|
381
|
+
const encoded = Buffer.from(script, 'utf16le').toString('base64');
|
|
382
|
+
return execFileSync('powershell.exe', [
|
|
383
|
+
'-NoProfile',
|
|
384
|
+
'-NonInteractive',
|
|
385
|
+
'-ExecutionPolicy',
|
|
386
|
+
'Bypass',
|
|
387
|
+
'-EncodedCommand',
|
|
388
|
+
encoded
|
|
389
|
+
], {
|
|
390
|
+
encoding: 'utf8',
|
|
391
|
+
timeout: POWERSHELL_TIMEOUT_MS,
|
|
392
|
+
maxBuffer: POWERSHELL_MAX_BUFFER,
|
|
393
|
+
windowsHide: true
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
function listWindowsDisplays() {
|
|
397
|
+
const script = `
|
|
398
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
399
|
+
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
400
|
+
[PSCustomObject]@{ id = $_.DeviceName; name = $_.DeviceName; primary = $_.Primary }
|
|
401
|
+
}
|
|
402
|
+
ConvertTo-Json @($s) -Compress
|
|
403
|
+
`.trim();
|
|
404
|
+
const parsed = JSON.parse(runPowershell(script).trim());
|
|
405
|
+
return parsed.map((d)=>({
|
|
406
|
+
id: String(d.id),
|
|
407
|
+
name: d.name || String(d.id),
|
|
408
|
+
primary: d.primary || false
|
|
409
|
+
}));
|
|
410
|
+
}
|
|
375
411
|
let device_libnut = null;
|
|
376
412
|
let libnutLoadError = null;
|
|
377
413
|
async function getLibnut() {
|
|
@@ -638,6 +674,7 @@ class ComputerDevice {
|
|
|
638
674
|
}
|
|
639
675
|
static async listDisplays() {
|
|
640
676
|
try {
|
|
677
|
+
if ('win32' === process.platform) return listWindowsDisplays();
|
|
641
678
|
const displays = await screenshot_desktop.listDisplays();
|
|
642
679
|
return displays.map((d)=>({
|
|
643
680
|
id: String(d.id),
|
|
@@ -695,7 +732,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
695
732
|
}
|
|
696
733
|
async healthCheck() {
|
|
697
734
|
console.log('[HealthCheck] Starting health check...');
|
|
698
|
-
console.log("[HealthCheck] @midscene/computer v1.
|
|
735
|
+
console.log("[HealthCheck] @midscene/computer v1.10.1-beta-20260624112700.0");
|
|
699
736
|
console.log('[HealthCheck] Taking screenshot...');
|
|
700
737
|
const screenshotTimeout = 15000;
|
|
701
738
|
let timeoutId;
|
|
@@ -764,6 +801,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
764
801
|
debugDevice('Taking screenshot', {
|
|
765
802
|
displayId: this.displayId
|
|
766
803
|
});
|
|
804
|
+
if ('win32' === process.platform) return this.screenshotViaPowershell();
|
|
767
805
|
const options = {
|
|
768
806
|
format: 'png'
|
|
769
807
|
};
|
|
@@ -797,6 +835,40 @@ Please follow these steps:
|
|
|
797
835
|
Original error: ${lastRawMessage}`);
|
|
798
836
|
throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
|
|
799
837
|
}
|
|
838
|
+
screenshotViaPowershell() {
|
|
839
|
+
const deviceName = this.displayId ? String(this.displayId) : '';
|
|
840
|
+
const target = deviceName ? `[System.Windows.Forms.Screen]::AllScreens | Where-Object { $_.DeviceName -eq '${escapePowershellSingleQuoted(deviceName)}' } | Select-Object -First 1` : '$null';
|
|
841
|
+
const script = `
|
|
842
|
+
$ErrorActionPreference = 'Stop'
|
|
843
|
+
Add-Type -AssemblyName System.Windows.Forms, System.Drawing
|
|
844
|
+
Add-Type @"
|
|
845
|
+
using System;
|
|
846
|
+
using System.Runtime.InteropServices;
|
|
847
|
+
public class NutDpi { [DllImport("user32.dll")] public static extern bool SetProcessDPIAware(); }
|
|
848
|
+
"@
|
|
849
|
+
[NutDpi]::SetProcessDPIAware() | Out-Null
|
|
850
|
+
$screen = ${target}
|
|
851
|
+
if (-not $screen) { $screen = [System.Windows.Forms.Screen]::PrimaryScreen }
|
|
852
|
+
$b = $screen.Bounds
|
|
853
|
+
$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height)
|
|
854
|
+
$g = [System.Drawing.Graphics]::FromImage($bmp)
|
|
855
|
+
$g.CopyFromScreen($b.X, $b.Y, 0, 0, $bmp.Size)
|
|
856
|
+
$ms = New-Object System.IO.MemoryStream
|
|
857
|
+
$bmp.Save($ms, [System.Drawing.Imaging.ImageFormat]::Png)
|
|
858
|
+
[Console]::Out.Write([Convert]::ToBase64String($ms.ToArray()))
|
|
859
|
+
$g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
860
|
+
`.trim();
|
|
861
|
+
let stdout;
|
|
862
|
+
try {
|
|
863
|
+
stdout = runPowershell(script);
|
|
864
|
+
} catch (error) {
|
|
865
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
866
|
+
throw new Error(`Failed to take screenshot on Windows: ${message}`);
|
|
867
|
+
}
|
|
868
|
+
const body = stdout.trim();
|
|
869
|
+
if (!body) throw new Error('Failed to take screenshot on Windows: PowerShell returned no image data');
|
|
870
|
+
return createImgBase64ByFormat('png', body);
|
|
871
|
+
}
|
|
800
872
|
async size() {
|
|
801
873
|
if (this.displayGeometry) return {
|
|
802
874
|
width: Math.round(this.displayGeometry.bounds.width),
|
|
@@ -1874,7 +1946,7 @@ async function agentForRDPComputer(opts) {
|
|
|
1874
1946
|
await device.connect();
|
|
1875
1947
|
return new ComputerAgent(device, opts);
|
|
1876
1948
|
}
|
|
1877
|
-
function
|
|
1949
|
+
function agent_tools_define_property(obj, key, value) {
|
|
1878
1950
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
1879
1951
|
value: value,
|
|
1880
1952
|
enumerable: true,
|
|
@@ -1884,7 +1956,7 @@ function mcp_tools_define_property(obj, key, value) {
|
|
|
1884
1956
|
else obj[key] = value;
|
|
1885
1957
|
return obj;
|
|
1886
1958
|
}
|
|
1887
|
-
const
|
|
1959
|
+
const agent_tools_debug = getDebug('agent-tools:computer');
|
|
1888
1960
|
const RDP_SECURITY_PROTOCOLS = [
|
|
1889
1961
|
'auto',
|
|
1890
1962
|
'tls',
|
|
@@ -1952,14 +2024,14 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
1952
2024
|
try {
|
|
1953
2025
|
await this.agent.destroy?.();
|
|
1954
2026
|
} catch (error) {
|
|
1955
|
-
|
|
2027
|
+
agent_tools_debug('Failed to destroy agent during cleanup:', error);
|
|
1956
2028
|
}
|
|
1957
2029
|
this.agent = void 0;
|
|
1958
2030
|
}
|
|
1959
2031
|
if (this.agent) return this.agent;
|
|
1960
2032
|
const reportOptions = this.readCliReportAgentOptions();
|
|
1961
2033
|
if (opts?.mode === 'rdp') {
|
|
1962
|
-
|
|
2034
|
+
agent_tools_debug('Creating RDP Computer agent for host:', opts.host);
|
|
1963
2035
|
const { mode: _mode, ...rdpFields } = opts;
|
|
1964
2036
|
const agent = await agentForRDPComputer({
|
|
1965
2037
|
...rdpFields,
|
|
@@ -1971,7 +2043,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
1971
2043
|
}
|
|
1972
2044
|
const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
|
|
1973
2045
|
const headless = opts?.mode === 'local' ? opts.headless : void 0;
|
|
1974
|
-
|
|
2046
|
+
agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
1975
2047
|
const agentOpts = {
|
|
1976
2048
|
...displayId ? {
|
|
1977
2049
|
displayId
|
|
@@ -2002,7 +2074,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2002
2074
|
try {
|
|
2003
2075
|
await this.agent.destroy?.();
|
|
2004
2076
|
} catch (error) {
|
|
2005
|
-
|
|
2077
|
+
agent_tools_debug('Failed to destroy agent during connect:', error);
|
|
2006
2078
|
}
|
|
2007
2079
|
this.agent = void 0;
|
|
2008
2080
|
this.lastInitArgsSignature = void 0;
|
|
@@ -2045,7 +2117,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2045
2117
|
];
|
|
2046
2118
|
}
|
|
2047
2119
|
constructor(...args){
|
|
2048
|
-
super(...args),
|
|
2120
|
+
super(...args), agent_tools_define_property(this, "lastInitArgsSignature", void 0), agent_tools_define_property(this, "initArgSpec", {
|
|
2049
2121
|
namespace: 'computer',
|
|
2050
2122
|
shape: computerInitArgShape,
|
|
2051
2123
|
cli: {
|
|
@@ -2058,7 +2130,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2058
2130
|
const tools = new ComputerMidsceneTools();
|
|
2059
2131
|
runToolsCLI(tools, 'midscene-computer', {
|
|
2060
2132
|
stripPrefix: 'computer_',
|
|
2061
|
-
version: "1.
|
|
2133
|
+
version: "1.10.1-beta-20260624112700.0",
|
|
2062
2134
|
extraCommands: createReportCliCommands()
|
|
2063
2135
|
}).catch((e)=>{
|
|
2064
2136
|
process.exit(reportCLIError(e));
|
package/dist/es/index.mjs
CHANGED
|
@@ -13,8 +13,8 @@ import { Agent } from "@midscene/core/agent";
|
|
|
13
13
|
import { once } from "node:events";
|
|
14
14
|
import { createInterface } from "node:readline";
|
|
15
15
|
import { z } from "@midscene/core";
|
|
16
|
-
import { agentBehaviorInitArgShape, extractAgentBehaviorInitArgs, getAgentInitArgsSignature, shouldRebuildAgentForInitArgs } from "@midscene/shared/
|
|
17
|
-
import { BaseMidsceneTools } from "@midscene/shared/
|
|
16
|
+
import { agentBehaviorInitArgShape, extractAgentBehaviorInitArgs, getAgentInitArgsSignature, shouldRebuildAgentForInitArgs } from "@midscene/shared/agent-tools/agent-behavior-init-args";
|
|
17
|
+
import { BaseMidsceneTools } from "@midscene/shared/agent-tools/base-tools";
|
|
18
18
|
import { overrideAIConfig } from "@midscene/shared/env";
|
|
19
19
|
function _define_property(obj, key, value) {
|
|
20
20
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
@@ -372,6 +372,42 @@ function sendKeyViaAppleScript(key, modifiers = []) {
|
|
|
372
372
|
script
|
|
373
373
|
]);
|
|
374
374
|
}
|
|
375
|
+
const POWERSHELL_TIMEOUT_MS = 15000;
|
|
376
|
+
const POWERSHELL_MAX_BUFFER = 67108864;
|
|
377
|
+
function escapePowershellSingleQuoted(value) {
|
|
378
|
+
return value.replace(/'/g, "''");
|
|
379
|
+
}
|
|
380
|
+
function runPowershell(script) {
|
|
381
|
+
const encoded = Buffer.from(script, 'utf16le').toString('base64');
|
|
382
|
+
return execFileSync('powershell.exe', [
|
|
383
|
+
'-NoProfile',
|
|
384
|
+
'-NonInteractive',
|
|
385
|
+
'-ExecutionPolicy',
|
|
386
|
+
'Bypass',
|
|
387
|
+
'-EncodedCommand',
|
|
388
|
+
encoded
|
|
389
|
+
], {
|
|
390
|
+
encoding: 'utf8',
|
|
391
|
+
timeout: POWERSHELL_TIMEOUT_MS,
|
|
392
|
+
maxBuffer: POWERSHELL_MAX_BUFFER,
|
|
393
|
+
windowsHide: true
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
function listWindowsDisplays() {
|
|
397
|
+
const script = `
|
|
398
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
399
|
+
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
400
|
+
[PSCustomObject]@{ id = $_.DeviceName; name = $_.DeviceName; primary = $_.Primary }
|
|
401
|
+
}
|
|
402
|
+
ConvertTo-Json @($s) -Compress
|
|
403
|
+
`.trim();
|
|
404
|
+
const parsed = JSON.parse(runPowershell(script).trim());
|
|
405
|
+
return parsed.map((d)=>({
|
|
406
|
+
id: String(d.id),
|
|
407
|
+
name: d.name || String(d.id),
|
|
408
|
+
primary: d.primary || false
|
|
409
|
+
}));
|
|
410
|
+
}
|
|
375
411
|
let device_libnut = null;
|
|
376
412
|
let libnutLoadError = null;
|
|
377
413
|
async function getLibnut() {
|
|
@@ -638,6 +674,7 @@ class ComputerDevice {
|
|
|
638
674
|
}
|
|
639
675
|
static async listDisplays() {
|
|
640
676
|
try {
|
|
677
|
+
if ('win32' === process.platform) return listWindowsDisplays();
|
|
641
678
|
const displays = await screenshot_desktop.listDisplays();
|
|
642
679
|
return displays.map((d)=>({
|
|
643
680
|
id: String(d.id),
|
|
@@ -695,7 +732,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
695
732
|
}
|
|
696
733
|
async healthCheck() {
|
|
697
734
|
console.log('[HealthCheck] Starting health check...');
|
|
698
|
-
console.log("[HealthCheck] @midscene/computer v1.
|
|
735
|
+
console.log("[HealthCheck] @midscene/computer v1.10.1-beta-20260624112700.0");
|
|
699
736
|
console.log('[HealthCheck] Taking screenshot...');
|
|
700
737
|
const screenshotTimeout = 15000;
|
|
701
738
|
let timeoutId;
|
|
@@ -764,6 +801,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
764
801
|
debugDevice('Taking screenshot', {
|
|
765
802
|
displayId: this.displayId
|
|
766
803
|
});
|
|
804
|
+
if ('win32' === process.platform) return this.screenshotViaPowershell();
|
|
767
805
|
const options = {
|
|
768
806
|
format: 'png'
|
|
769
807
|
};
|
|
@@ -797,6 +835,40 @@ Please follow these steps:
|
|
|
797
835
|
Original error: ${lastRawMessage}`);
|
|
798
836
|
throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
|
|
799
837
|
}
|
|
838
|
+
screenshotViaPowershell() {
|
|
839
|
+
const deviceName = this.displayId ? String(this.displayId) : '';
|
|
840
|
+
const target = deviceName ? `[System.Windows.Forms.Screen]::AllScreens | Where-Object { $_.DeviceName -eq '${escapePowershellSingleQuoted(deviceName)}' } | Select-Object -First 1` : '$null';
|
|
841
|
+
const script = `
|
|
842
|
+
$ErrorActionPreference = 'Stop'
|
|
843
|
+
Add-Type -AssemblyName System.Windows.Forms, System.Drawing
|
|
844
|
+
Add-Type @"
|
|
845
|
+
using System;
|
|
846
|
+
using System.Runtime.InteropServices;
|
|
847
|
+
public class NutDpi { [DllImport("user32.dll")] public static extern bool SetProcessDPIAware(); }
|
|
848
|
+
"@
|
|
849
|
+
[NutDpi]::SetProcessDPIAware() | Out-Null
|
|
850
|
+
$screen = ${target}
|
|
851
|
+
if (-not $screen) { $screen = [System.Windows.Forms.Screen]::PrimaryScreen }
|
|
852
|
+
$b = $screen.Bounds
|
|
853
|
+
$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height)
|
|
854
|
+
$g = [System.Drawing.Graphics]::FromImage($bmp)
|
|
855
|
+
$g.CopyFromScreen($b.X, $b.Y, 0, 0, $bmp.Size)
|
|
856
|
+
$ms = New-Object System.IO.MemoryStream
|
|
857
|
+
$bmp.Save($ms, [System.Drawing.Imaging.ImageFormat]::Png)
|
|
858
|
+
[Console]::Out.Write([Convert]::ToBase64String($ms.ToArray()))
|
|
859
|
+
$g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
860
|
+
`.trim();
|
|
861
|
+
let stdout;
|
|
862
|
+
try {
|
|
863
|
+
stdout = runPowershell(script);
|
|
864
|
+
} catch (error) {
|
|
865
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
866
|
+
throw new Error(`Failed to take screenshot on Windows: ${message}`);
|
|
867
|
+
}
|
|
868
|
+
const body = stdout.trim();
|
|
869
|
+
if (!body) throw new Error('Failed to take screenshot on Windows: PowerShell returned no image data');
|
|
870
|
+
return createImgBase64ByFormat('png', body);
|
|
871
|
+
}
|
|
800
872
|
async size() {
|
|
801
873
|
if (this.displayGeometry) return {
|
|
802
874
|
width: Math.round(this.displayGeometry.bounds.width),
|
|
@@ -1917,7 +1989,7 @@ async function agentForRDPComputer(opts) {
|
|
|
1917
1989
|
await device.connect();
|
|
1918
1990
|
return new ComputerAgent(device, opts);
|
|
1919
1991
|
}
|
|
1920
|
-
function
|
|
1992
|
+
function agent_tools_define_property(obj, key, value) {
|
|
1921
1993
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
1922
1994
|
value: value,
|
|
1923
1995
|
enumerable: true,
|
|
@@ -1927,7 +1999,7 @@ function mcp_tools_define_property(obj, key, value) {
|
|
|
1927
1999
|
else obj[key] = value;
|
|
1928
2000
|
return obj;
|
|
1929
2001
|
}
|
|
1930
|
-
const
|
|
2002
|
+
const agent_tools_debug = getDebug('agent-tools:computer');
|
|
1931
2003
|
const RDP_SECURITY_PROTOCOLS = [
|
|
1932
2004
|
'auto',
|
|
1933
2005
|
'tls',
|
|
@@ -1995,14 +2067,14 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
1995
2067
|
try {
|
|
1996
2068
|
await this.agent.destroy?.();
|
|
1997
2069
|
} catch (error) {
|
|
1998
|
-
|
|
2070
|
+
agent_tools_debug('Failed to destroy agent during cleanup:', error);
|
|
1999
2071
|
}
|
|
2000
2072
|
this.agent = void 0;
|
|
2001
2073
|
}
|
|
2002
2074
|
if (this.agent) return this.agent;
|
|
2003
2075
|
const reportOptions = this.readCliReportAgentOptions();
|
|
2004
2076
|
if (opts?.mode === 'rdp') {
|
|
2005
|
-
|
|
2077
|
+
agent_tools_debug('Creating RDP Computer agent for host:', opts.host);
|
|
2006
2078
|
const { mode: _mode, ...rdpFields } = opts;
|
|
2007
2079
|
const agent = await agentForRDPComputer({
|
|
2008
2080
|
...rdpFields,
|
|
@@ -2014,7 +2086,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2014
2086
|
}
|
|
2015
2087
|
const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
|
|
2016
2088
|
const headless = opts?.mode === 'local' ? opts.headless : void 0;
|
|
2017
|
-
|
|
2089
|
+
agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
2018
2090
|
const agentOpts = {
|
|
2019
2091
|
...displayId ? {
|
|
2020
2092
|
displayId
|
|
@@ -2045,7 +2117,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2045
2117
|
try {
|
|
2046
2118
|
await this.agent.destroy?.();
|
|
2047
2119
|
} catch (error) {
|
|
2048
|
-
|
|
2120
|
+
agent_tools_debug('Failed to destroy agent during connect:', error);
|
|
2049
2121
|
}
|
|
2050
2122
|
this.agent = void 0;
|
|
2051
2123
|
this.lastInitArgsSignature = void 0;
|
|
@@ -2088,7 +2160,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2088
2160
|
];
|
|
2089
2161
|
}
|
|
2090
2162
|
constructor(...args){
|
|
2091
|
-
super(...args),
|
|
2163
|
+
super(...args), agent_tools_define_property(this, "lastInitArgsSignature", void 0), agent_tools_define_property(this, "initArgSpec", {
|
|
2092
2164
|
namespace: 'computer',
|
|
2093
2165
|
shape: computerInitArgShape,
|
|
2094
2166
|
cli: {
|
|
@@ -2099,7 +2171,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2099
2171
|
}
|
|
2100
2172
|
}
|
|
2101
2173
|
function version() {
|
|
2102
|
-
const currentVersion = "1.
|
|
2174
|
+
const currentVersion = "1.10.1-beta-20260624112700.0";
|
|
2103
2175
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2104
2176
|
return currentVersion;
|
|
2105
2177
|
}
|
package/dist/lib/cli.js
CHANGED
|
@@ -26,9 +26,9 @@ var __webpack_require__ = {};
|
|
|
26
26
|
var __webpack_exports__ = {};
|
|
27
27
|
const core_namespaceObject = require("@midscene/core");
|
|
28
28
|
const cli_namespaceObject = require("@midscene/shared/cli");
|
|
29
|
+
const agent_behavior_init_args_namespaceObject = require("@midscene/shared/agent-tools/agent-behavior-init-args");
|
|
30
|
+
const base_tools_namespaceObject = require("@midscene/shared/agent-tools/base-tools");
|
|
29
31
|
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
30
|
-
const agent_behavior_init_args_namespaceObject = require("@midscene/shared/mcp/agent-behavior-init-args");
|
|
31
|
-
const base_tools_namespaceObject = require("@midscene/shared/mcp/base-tools");
|
|
32
32
|
const agent_namespaceObject = require("@midscene/core/agent");
|
|
33
33
|
const external_node_child_process_namespaceObject = require("node:child_process");
|
|
34
34
|
const external_node_fs_namespaceObject = require("node:fs");
|
|
@@ -398,6 +398,42 @@ function sendKeyViaAppleScript(key, modifiers = []) {
|
|
|
398
398
|
script
|
|
399
399
|
]);
|
|
400
400
|
}
|
|
401
|
+
const POWERSHELL_TIMEOUT_MS = 15000;
|
|
402
|
+
const POWERSHELL_MAX_BUFFER = 67108864;
|
|
403
|
+
function escapePowershellSingleQuoted(value) {
|
|
404
|
+
return value.replace(/'/g, "''");
|
|
405
|
+
}
|
|
406
|
+
function runPowershell(script) {
|
|
407
|
+
const encoded = Buffer.from(script, 'utf16le').toString('base64');
|
|
408
|
+
return (0, external_node_child_process_namespaceObject.execFileSync)('powershell.exe', [
|
|
409
|
+
'-NoProfile',
|
|
410
|
+
'-NonInteractive',
|
|
411
|
+
'-ExecutionPolicy',
|
|
412
|
+
'Bypass',
|
|
413
|
+
'-EncodedCommand',
|
|
414
|
+
encoded
|
|
415
|
+
], {
|
|
416
|
+
encoding: 'utf8',
|
|
417
|
+
timeout: POWERSHELL_TIMEOUT_MS,
|
|
418
|
+
maxBuffer: POWERSHELL_MAX_BUFFER,
|
|
419
|
+
windowsHide: true
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
function listWindowsDisplays() {
|
|
423
|
+
const script = `
|
|
424
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
425
|
+
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
426
|
+
[PSCustomObject]@{ id = $_.DeviceName; name = $_.DeviceName; primary = $_.Primary }
|
|
427
|
+
}
|
|
428
|
+
ConvertTo-Json @($s) -Compress
|
|
429
|
+
`.trim();
|
|
430
|
+
const parsed = JSON.parse(runPowershell(script).trim());
|
|
431
|
+
return parsed.map((d)=>({
|
|
432
|
+
id: String(d.id),
|
|
433
|
+
name: d.name || String(d.id),
|
|
434
|
+
primary: d.primary || false
|
|
435
|
+
}));
|
|
436
|
+
}
|
|
401
437
|
let device_libnut = null;
|
|
402
438
|
let libnutLoadError = null;
|
|
403
439
|
async function getLibnut() {
|
|
@@ -664,6 +700,7 @@ class ComputerDevice {
|
|
|
664
700
|
}
|
|
665
701
|
static async listDisplays() {
|
|
666
702
|
try {
|
|
703
|
+
if ('win32' === process.platform) return listWindowsDisplays();
|
|
667
704
|
const displays = await external_screenshot_desktop_default().listDisplays();
|
|
668
705
|
return displays.map((d)=>({
|
|
669
706
|
id: String(d.id),
|
|
@@ -721,7 +758,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
721
758
|
}
|
|
722
759
|
async healthCheck() {
|
|
723
760
|
console.log('[HealthCheck] Starting health check...');
|
|
724
|
-
console.log("[HealthCheck] @midscene/computer v1.
|
|
761
|
+
console.log("[HealthCheck] @midscene/computer v1.10.1-beta-20260624112700.0");
|
|
725
762
|
console.log('[HealthCheck] Taking screenshot...');
|
|
726
763
|
const screenshotTimeout = 15000;
|
|
727
764
|
let timeoutId;
|
|
@@ -790,6 +827,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
790
827
|
debugDevice('Taking screenshot', {
|
|
791
828
|
displayId: this.displayId
|
|
792
829
|
});
|
|
830
|
+
if ('win32' === process.platform) return this.screenshotViaPowershell();
|
|
793
831
|
const options = {
|
|
794
832
|
format: 'png'
|
|
795
833
|
};
|
|
@@ -823,6 +861,40 @@ Please follow these steps:
|
|
|
823
861
|
Original error: ${lastRawMessage}`);
|
|
824
862
|
throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
|
|
825
863
|
}
|
|
864
|
+
screenshotViaPowershell() {
|
|
865
|
+
const deviceName = this.displayId ? String(this.displayId) : '';
|
|
866
|
+
const target = deviceName ? `[System.Windows.Forms.Screen]::AllScreens | Where-Object { $_.DeviceName -eq '${escapePowershellSingleQuoted(deviceName)}' } | Select-Object -First 1` : '$null';
|
|
867
|
+
const script = `
|
|
868
|
+
$ErrorActionPreference = 'Stop'
|
|
869
|
+
Add-Type -AssemblyName System.Windows.Forms, System.Drawing
|
|
870
|
+
Add-Type @"
|
|
871
|
+
using System;
|
|
872
|
+
using System.Runtime.InteropServices;
|
|
873
|
+
public class NutDpi { [DllImport("user32.dll")] public static extern bool SetProcessDPIAware(); }
|
|
874
|
+
"@
|
|
875
|
+
[NutDpi]::SetProcessDPIAware() | Out-Null
|
|
876
|
+
$screen = ${target}
|
|
877
|
+
if (-not $screen) { $screen = [System.Windows.Forms.Screen]::PrimaryScreen }
|
|
878
|
+
$b = $screen.Bounds
|
|
879
|
+
$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height)
|
|
880
|
+
$g = [System.Drawing.Graphics]::FromImage($bmp)
|
|
881
|
+
$g.CopyFromScreen($b.X, $b.Y, 0, 0, $bmp.Size)
|
|
882
|
+
$ms = New-Object System.IO.MemoryStream
|
|
883
|
+
$bmp.Save($ms, [System.Drawing.Imaging.ImageFormat]::Png)
|
|
884
|
+
[Console]::Out.Write([Convert]::ToBase64String($ms.ToArray()))
|
|
885
|
+
$g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
886
|
+
`.trim();
|
|
887
|
+
let stdout;
|
|
888
|
+
try {
|
|
889
|
+
stdout = runPowershell(script);
|
|
890
|
+
} catch (error) {
|
|
891
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
892
|
+
throw new Error(`Failed to take screenshot on Windows: ${message}`);
|
|
893
|
+
}
|
|
894
|
+
const body = stdout.trim();
|
|
895
|
+
if (!body) throw new Error('Failed to take screenshot on Windows: PowerShell returned no image data');
|
|
896
|
+
return (0, img_namespaceObject.createImgBase64ByFormat)('png', body);
|
|
897
|
+
}
|
|
826
898
|
async size() {
|
|
827
899
|
if (this.displayGeometry) return {
|
|
828
900
|
width: Math.round(this.displayGeometry.bounds.width),
|
|
@@ -1901,7 +1973,7 @@ async function agentForRDPComputer(opts) {
|
|
|
1901
1973
|
await device.connect();
|
|
1902
1974
|
return new ComputerAgent(device, opts);
|
|
1903
1975
|
}
|
|
1904
|
-
function
|
|
1976
|
+
function agent_tools_define_property(obj, key, value) {
|
|
1905
1977
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
1906
1978
|
value: value,
|
|
1907
1979
|
enumerable: true,
|
|
@@ -1911,7 +1983,7 @@ function mcp_tools_define_property(obj, key, value) {
|
|
|
1911
1983
|
else obj[key] = value;
|
|
1912
1984
|
return obj;
|
|
1913
1985
|
}
|
|
1914
|
-
const
|
|
1986
|
+
const agent_tools_debug = (0, logger_namespaceObject.getDebug)('agent-tools:computer');
|
|
1915
1987
|
const RDP_SECURITY_PROTOCOLS = [
|
|
1916
1988
|
'auto',
|
|
1917
1989
|
'tls',
|
|
@@ -1979,14 +2051,14 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
1979
2051
|
try {
|
|
1980
2052
|
await this.agent.destroy?.();
|
|
1981
2053
|
} catch (error) {
|
|
1982
|
-
|
|
2054
|
+
agent_tools_debug('Failed to destroy agent during cleanup:', error);
|
|
1983
2055
|
}
|
|
1984
2056
|
this.agent = void 0;
|
|
1985
2057
|
}
|
|
1986
2058
|
if (this.agent) return this.agent;
|
|
1987
2059
|
const reportOptions = this.readCliReportAgentOptions();
|
|
1988
2060
|
if (opts?.mode === 'rdp') {
|
|
1989
|
-
|
|
2061
|
+
agent_tools_debug('Creating RDP Computer agent for host:', opts.host);
|
|
1990
2062
|
const { mode: _mode, ...rdpFields } = opts;
|
|
1991
2063
|
const agent = await agentForRDPComputer({
|
|
1992
2064
|
...rdpFields,
|
|
@@ -1998,7 +2070,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
1998
2070
|
}
|
|
1999
2071
|
const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
|
|
2000
2072
|
const headless = opts?.mode === 'local' ? opts.headless : void 0;
|
|
2001
|
-
|
|
2073
|
+
agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
2002
2074
|
const agentOpts = {
|
|
2003
2075
|
...displayId ? {
|
|
2004
2076
|
displayId
|
|
@@ -2029,7 +2101,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2029
2101
|
try {
|
|
2030
2102
|
await this.agent.destroy?.();
|
|
2031
2103
|
} catch (error) {
|
|
2032
|
-
|
|
2104
|
+
agent_tools_debug('Failed to destroy agent during connect:', error);
|
|
2033
2105
|
}
|
|
2034
2106
|
this.agent = void 0;
|
|
2035
2107
|
this.lastInitArgsSignature = void 0;
|
|
@@ -2072,7 +2144,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2072
2144
|
];
|
|
2073
2145
|
}
|
|
2074
2146
|
constructor(...args){
|
|
2075
|
-
super(...args),
|
|
2147
|
+
super(...args), agent_tools_define_property(this, "lastInitArgsSignature", void 0), agent_tools_define_property(this, "initArgSpec", {
|
|
2076
2148
|
namespace: 'computer',
|
|
2077
2149
|
shape: computerInitArgShape,
|
|
2078
2150
|
cli: {
|
|
@@ -2085,7 +2157,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2085
2157
|
const tools = new ComputerMidsceneTools();
|
|
2086
2158
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
|
|
2087
2159
|
stripPrefix: 'computer_',
|
|
2088
|
-
version: "1.
|
|
2160
|
+
version: "1.10.1-beta-20260624112700.0",
|
|
2089
2161
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
2090
2162
|
}).catch((e)=>{
|
|
2091
2163
|
process.exit((0, cli_namespaceObject.reportCLIError)(e));
|