@midscene/computer 1.7.5-beta-20260420032657.0 → 1.7.5-beta-20260420052829.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +39 -14
- package/dist/es/index.mjs +39 -14
- package/dist/es/mcp-server.mjs +40 -14
- package/dist/lib/cli.js +40 -15
- package/dist/lib/index.js +40 -15
- package/dist/lib/mcp-server.js +40 -14
- package/dist/types/index.d.ts +8 -4
- package/dist/types/mcp-server.d.ts +8 -4
- package/package.json +3 -3
package/dist/es/cli.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createReportCliCommands, getMidsceneLocationSchema, z } from "@midscene/core";
|
|
2
2
|
import { CLIError, runToolsCLI } from "@midscene/shared/cli";
|
|
3
3
|
import { getDebug } from "@midscene/shared/logger";
|
|
4
|
-
import { BaseMidsceneTools } from "@midscene/shared/mcp";
|
|
4
|
+
import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
|
|
5
5
|
import { Agent } from "@midscene/core/agent";
|
|
6
6
|
import node_assert from "node:assert";
|
|
7
7
|
import { execFileSync, execSync, spawn, spawnSync } from "node:child_process";
|
|
@@ -409,7 +409,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
409
409
|
}
|
|
410
410
|
async healthCheck() {
|
|
411
411
|
console.log('[HealthCheck] Starting health check...');
|
|
412
|
-
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-
|
|
412
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260420052829.0");
|
|
413
413
|
console.log('[HealthCheck] Taking screenshot...');
|
|
414
414
|
const screenshotTimeout = 15000;
|
|
415
415
|
let timeoutId;
|
|
@@ -815,13 +815,29 @@ async function agentFromComputer(opts) {
|
|
|
815
815
|
await device.connect();
|
|
816
816
|
return new ComputerAgent(device, opts);
|
|
817
817
|
}
|
|
818
|
+
function mcp_tools_define_property(obj, key, value) {
|
|
819
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
820
|
+
value: value,
|
|
821
|
+
enumerable: true,
|
|
822
|
+
configurable: true,
|
|
823
|
+
writable: true
|
|
824
|
+
});
|
|
825
|
+
else obj[key] = value;
|
|
826
|
+
return obj;
|
|
827
|
+
}
|
|
818
828
|
const debug = getDebug('mcp:computer-tools');
|
|
829
|
+
const computerInitArgShape = {
|
|
830
|
+
displayId: z.string().optional().describe('Display ID (from computer_list_displays)'),
|
|
831
|
+
headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
|
|
832
|
+
};
|
|
819
833
|
class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
820
834
|
createTemporaryDevice() {
|
|
821
835
|
return new ComputerDevice({});
|
|
822
836
|
}
|
|
823
|
-
async ensureAgent(
|
|
824
|
-
|
|
837
|
+
async ensureAgent(opts) {
|
|
838
|
+
const displayId = opts?.displayId;
|
|
839
|
+
const headless = opts?.headless;
|
|
840
|
+
if (this.agent && (void 0 !== displayId || void 0 !== headless)) {
|
|
825
841
|
try {
|
|
826
842
|
await this.agent.destroy?.();
|
|
827
843
|
} catch (error) {
|
|
@@ -831,7 +847,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
831
847
|
}
|
|
832
848
|
if (this.agent) return this.agent;
|
|
833
849
|
debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
834
|
-
const
|
|
850
|
+
const agentOpts = {
|
|
835
851
|
...displayId ? {
|
|
836
852
|
displayId
|
|
837
853
|
} : {},
|
|
@@ -839,7 +855,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
839
855
|
headless
|
|
840
856
|
} : {}
|
|
841
857
|
};
|
|
842
|
-
const agent = await agentFromComputer(Object.keys(
|
|
858
|
+
const agent = await agentFromComputer(Object.keys(agentOpts).length > 0 ? agentOpts : void 0);
|
|
843
859
|
this.agent = agent;
|
|
844
860
|
return agent;
|
|
845
861
|
}
|
|
@@ -848,18 +864,17 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
848
864
|
{
|
|
849
865
|
name: 'computer_connect',
|
|
850
866
|
description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
|
|
851
|
-
schema:
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
const agent = await this.ensureAgent(displayId, headless);
|
|
867
|
+
schema: this.getAgentInitArgSchema(),
|
|
868
|
+
cli: this.getAgentInitArgCliMetadata(),
|
|
869
|
+
handler: async (args)=>{
|
|
870
|
+
const initArgs = this.extractAgentInitParam(args);
|
|
871
|
+
const agent = await this.ensureAgent(initArgs);
|
|
857
872
|
const screenshot = await agent.interface.screenshotBase64();
|
|
858
873
|
return {
|
|
859
874
|
content: [
|
|
860
875
|
{
|
|
861
876
|
type: 'text',
|
|
862
|
-
text: `Connected to computer${displayId ? ` (Display: ${displayId})` : ' (Primary display)'}`
|
|
877
|
+
text: `Connected to computer${initArgs?.displayId ? ` (Display: ${initArgs.displayId})` : ' (Primary display)'}`
|
|
863
878
|
},
|
|
864
879
|
...this.buildScreenshotContent(screenshot)
|
|
865
880
|
]
|
|
@@ -890,11 +905,21 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
890
905
|
}
|
|
891
906
|
];
|
|
892
907
|
}
|
|
908
|
+
constructor(...args){
|
|
909
|
+
super(...args), mcp_tools_define_property(this, "initArgSpec", {
|
|
910
|
+
namespace: 'computer',
|
|
911
|
+
shape: computerInitArgShape,
|
|
912
|
+
cli: {
|
|
913
|
+
preferBareKeys: true
|
|
914
|
+
},
|
|
915
|
+
adapt: (extracted)=>extracted
|
|
916
|
+
});
|
|
917
|
+
}
|
|
893
918
|
}
|
|
894
919
|
const tools = new ComputerMidsceneTools();
|
|
895
920
|
runToolsCLI(tools, 'midscene-computer', {
|
|
896
921
|
stripPrefix: 'computer_',
|
|
897
|
-
version: "1.7.5-beta-
|
|
922
|
+
version: "1.7.5-beta-20260420052829.0",
|
|
898
923
|
extraCommands: createReportCliCommands()
|
|
899
924
|
}).catch((e)=>{
|
|
900
925
|
if (!(e instanceof CLIError)) console.error(e);
|
package/dist/es/index.mjs
CHANGED
|
@@ -11,7 +11,7 @@ import { createImgBase64ByFormat } from "@midscene/shared/img";
|
|
|
11
11
|
import { getDebug } from "@midscene/shared/logger";
|
|
12
12
|
import screenshot_desktop from "screenshot-desktop";
|
|
13
13
|
import { Agent } from "@midscene/core/agent";
|
|
14
|
-
import { BaseMidsceneTools } from "@midscene/shared/mcp";
|
|
14
|
+
import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
|
|
15
15
|
import { overrideAIConfig } from "@midscene/shared/env";
|
|
16
16
|
const debugXvfb = getDebug('computer:xvfb');
|
|
17
17
|
function checkXvfbInstalled() {
|
|
@@ -409,7 +409,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
409
409
|
}
|
|
410
410
|
async healthCheck() {
|
|
411
411
|
console.log('[HealthCheck] Starting health check...');
|
|
412
|
-
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-
|
|
412
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260420052829.0");
|
|
413
413
|
console.log('[HealthCheck] Taking screenshot...');
|
|
414
414
|
const screenshotTimeout = 15000;
|
|
415
415
|
let timeoutId;
|
|
@@ -815,13 +815,29 @@ async function agentFromComputer(opts) {
|
|
|
815
815
|
await device.connect();
|
|
816
816
|
return new ComputerAgent(device, opts);
|
|
817
817
|
}
|
|
818
|
+
function mcp_tools_define_property(obj, key, value) {
|
|
819
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
820
|
+
value: value,
|
|
821
|
+
enumerable: true,
|
|
822
|
+
configurable: true,
|
|
823
|
+
writable: true
|
|
824
|
+
});
|
|
825
|
+
else obj[key] = value;
|
|
826
|
+
return obj;
|
|
827
|
+
}
|
|
818
828
|
const debug = getDebug('mcp:computer-tools');
|
|
829
|
+
const computerInitArgShape = {
|
|
830
|
+
displayId: z.string().optional().describe('Display ID (from computer_list_displays)'),
|
|
831
|
+
headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
|
|
832
|
+
};
|
|
819
833
|
class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
820
834
|
createTemporaryDevice() {
|
|
821
835
|
return new ComputerDevice({});
|
|
822
836
|
}
|
|
823
|
-
async ensureAgent(
|
|
824
|
-
|
|
837
|
+
async ensureAgent(opts) {
|
|
838
|
+
const displayId = opts?.displayId;
|
|
839
|
+
const headless = opts?.headless;
|
|
840
|
+
if (this.agent && (void 0 !== displayId || void 0 !== headless)) {
|
|
825
841
|
try {
|
|
826
842
|
await this.agent.destroy?.();
|
|
827
843
|
} catch (error) {
|
|
@@ -831,7 +847,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
831
847
|
}
|
|
832
848
|
if (this.agent) return this.agent;
|
|
833
849
|
debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
834
|
-
const
|
|
850
|
+
const agentOpts = {
|
|
835
851
|
...displayId ? {
|
|
836
852
|
displayId
|
|
837
853
|
} : {},
|
|
@@ -839,7 +855,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
839
855
|
headless
|
|
840
856
|
} : {}
|
|
841
857
|
};
|
|
842
|
-
const agent = await agentFromComputer(Object.keys(
|
|
858
|
+
const agent = await agentFromComputer(Object.keys(agentOpts).length > 0 ? agentOpts : void 0);
|
|
843
859
|
this.agent = agent;
|
|
844
860
|
return agent;
|
|
845
861
|
}
|
|
@@ -848,18 +864,17 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
848
864
|
{
|
|
849
865
|
name: 'computer_connect',
|
|
850
866
|
description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
|
|
851
|
-
schema:
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
const agent = await this.ensureAgent(displayId, headless);
|
|
867
|
+
schema: this.getAgentInitArgSchema(),
|
|
868
|
+
cli: this.getAgentInitArgCliMetadata(),
|
|
869
|
+
handler: async (args)=>{
|
|
870
|
+
const initArgs = this.extractAgentInitParam(args);
|
|
871
|
+
const agent = await this.ensureAgent(initArgs);
|
|
857
872
|
const screenshot = await agent.interface.screenshotBase64();
|
|
858
873
|
return {
|
|
859
874
|
content: [
|
|
860
875
|
{
|
|
861
876
|
type: 'text',
|
|
862
|
-
text: `Connected to computer${displayId ? ` (Display: ${displayId})` : ' (Primary display)'}`
|
|
877
|
+
text: `Connected to computer${initArgs?.displayId ? ` (Display: ${initArgs.displayId})` : ' (Primary display)'}`
|
|
863
878
|
},
|
|
864
879
|
...this.buildScreenshotContent(screenshot)
|
|
865
880
|
]
|
|
@@ -890,9 +905,19 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
890
905
|
}
|
|
891
906
|
];
|
|
892
907
|
}
|
|
908
|
+
constructor(...args){
|
|
909
|
+
super(...args), mcp_tools_define_property(this, "initArgSpec", {
|
|
910
|
+
namespace: 'computer',
|
|
911
|
+
shape: computerInitArgShape,
|
|
912
|
+
cli: {
|
|
913
|
+
preferBareKeys: true
|
|
914
|
+
},
|
|
915
|
+
adapt: (extracted)=>extracted
|
|
916
|
+
});
|
|
917
|
+
}
|
|
893
918
|
}
|
|
894
919
|
function version() {
|
|
895
|
-
const currentVersion = "1.7.5-beta-
|
|
920
|
+
const currentVersion = "1.7.5-beta-20260420052829.0";
|
|
896
921
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
897
922
|
return currentVersion;
|
|
898
923
|
}
|
package/dist/es/mcp-server.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BaseMCPServer,
|
|
1
|
+
import { BaseMCPServer, createMCPServerLauncher } from "@midscene/shared/mcp";
|
|
2
2
|
import { Agent } from "@midscene/core/agent";
|
|
3
3
|
import node_assert from "node:assert";
|
|
4
4
|
import { execFileSync, execSync, spawn, spawnSync } from "node:child_process";
|
|
@@ -12,6 +12,7 @@ import { sleep } from "@midscene/core/utils";
|
|
|
12
12
|
import { createImgBase64ByFormat } from "@midscene/shared/img";
|
|
13
13
|
import { getDebug } from "@midscene/shared/logger";
|
|
14
14
|
import screenshot_desktop from "screenshot-desktop";
|
|
15
|
+
import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
|
|
15
16
|
const debugXvfb = getDebug('computer:xvfb');
|
|
16
17
|
function checkXvfbInstalled() {
|
|
17
18
|
try {
|
|
@@ -408,7 +409,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
408
409
|
}
|
|
409
410
|
async healthCheck() {
|
|
410
411
|
console.log('[HealthCheck] Starting health check...');
|
|
411
|
-
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-
|
|
412
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260420052829.0");
|
|
412
413
|
console.log('[HealthCheck] Taking screenshot...');
|
|
413
414
|
const screenshotTimeout = 15000;
|
|
414
415
|
let timeoutId;
|
|
@@ -814,13 +815,29 @@ async function agentFromComputer(opts) {
|
|
|
814
815
|
await device.connect();
|
|
815
816
|
return new ComputerAgent(device, opts);
|
|
816
817
|
}
|
|
818
|
+
function mcp_tools_define_property(obj, key, value) {
|
|
819
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
820
|
+
value: value,
|
|
821
|
+
enumerable: true,
|
|
822
|
+
configurable: true,
|
|
823
|
+
writable: true
|
|
824
|
+
});
|
|
825
|
+
else obj[key] = value;
|
|
826
|
+
return obj;
|
|
827
|
+
}
|
|
817
828
|
const debug = getDebug('mcp:computer-tools');
|
|
829
|
+
const computerInitArgShape = {
|
|
830
|
+
displayId: z.string().optional().describe('Display ID (from computer_list_displays)'),
|
|
831
|
+
headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
|
|
832
|
+
};
|
|
818
833
|
class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
819
834
|
createTemporaryDevice() {
|
|
820
835
|
return new ComputerDevice({});
|
|
821
836
|
}
|
|
822
|
-
async ensureAgent(
|
|
823
|
-
|
|
837
|
+
async ensureAgent(opts) {
|
|
838
|
+
const displayId = opts?.displayId;
|
|
839
|
+
const headless = opts?.headless;
|
|
840
|
+
if (this.agent && (void 0 !== displayId || void 0 !== headless)) {
|
|
824
841
|
try {
|
|
825
842
|
await this.agent.destroy?.();
|
|
826
843
|
} catch (error) {
|
|
@@ -830,7 +847,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
830
847
|
}
|
|
831
848
|
if (this.agent) return this.agent;
|
|
832
849
|
debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
833
|
-
const
|
|
850
|
+
const agentOpts = {
|
|
834
851
|
...displayId ? {
|
|
835
852
|
displayId
|
|
836
853
|
} : {},
|
|
@@ -838,7 +855,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
838
855
|
headless
|
|
839
856
|
} : {}
|
|
840
857
|
};
|
|
841
|
-
const agent = await agentFromComputer(Object.keys(
|
|
858
|
+
const agent = await agentFromComputer(Object.keys(agentOpts).length > 0 ? agentOpts : void 0);
|
|
842
859
|
this.agent = agent;
|
|
843
860
|
return agent;
|
|
844
861
|
}
|
|
@@ -847,18 +864,17 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
847
864
|
{
|
|
848
865
|
name: 'computer_connect',
|
|
849
866
|
description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
|
|
850
|
-
schema:
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
const agent = await this.ensureAgent(displayId, headless);
|
|
867
|
+
schema: this.getAgentInitArgSchema(),
|
|
868
|
+
cli: this.getAgentInitArgCliMetadata(),
|
|
869
|
+
handler: async (args)=>{
|
|
870
|
+
const initArgs = this.extractAgentInitParam(args);
|
|
871
|
+
const agent = await this.ensureAgent(initArgs);
|
|
856
872
|
const screenshot = await agent.interface.screenshotBase64();
|
|
857
873
|
return {
|
|
858
874
|
content: [
|
|
859
875
|
{
|
|
860
876
|
type: 'text',
|
|
861
|
-
text: `Connected to computer${displayId ? ` (Display: ${displayId})` : ' (Primary display)'}`
|
|
877
|
+
text: `Connected to computer${initArgs?.displayId ? ` (Display: ${initArgs.displayId})` : ' (Primary display)'}`
|
|
862
878
|
},
|
|
863
879
|
...this.buildScreenshotContent(screenshot)
|
|
864
880
|
]
|
|
@@ -889,6 +905,16 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
889
905
|
}
|
|
890
906
|
];
|
|
891
907
|
}
|
|
908
|
+
constructor(...args){
|
|
909
|
+
super(...args), mcp_tools_define_property(this, "initArgSpec", {
|
|
910
|
+
namespace: 'computer',
|
|
911
|
+
shape: computerInitArgShape,
|
|
912
|
+
cli: {
|
|
913
|
+
preferBareKeys: true
|
|
914
|
+
},
|
|
915
|
+
adapt: (extracted)=>extracted
|
|
916
|
+
});
|
|
917
|
+
}
|
|
892
918
|
}
|
|
893
919
|
class ComputerMCPServer extends BaseMCPServer {
|
|
894
920
|
createToolsManager() {
|
|
@@ -897,7 +923,7 @@ class ComputerMCPServer extends BaseMCPServer {
|
|
|
897
923
|
constructor(toolsManager){
|
|
898
924
|
super({
|
|
899
925
|
name: '@midscene/computer-mcp',
|
|
900
|
-
version: "1.7.5-beta-
|
|
926
|
+
version: "1.7.5-beta-20260420052829.0",
|
|
901
927
|
description: 'Control the computer desktop using natural language commands'
|
|
902
928
|
}, toolsManager);
|
|
903
929
|
}
|
package/dist/lib/cli.js
CHANGED
|
@@ -27,7 +27,7 @@ var __webpack_exports__ = {};
|
|
|
27
27
|
const core_namespaceObject = require("@midscene/core");
|
|
28
28
|
const cli_namespaceObject = require("@midscene/shared/cli");
|
|
29
29
|
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
30
|
-
const
|
|
30
|
+
const base_tools_namespaceObject = require("@midscene/shared/mcp/base-tools");
|
|
31
31
|
const agent_namespaceObject = require("@midscene/core/agent");
|
|
32
32
|
const external_node_assert_namespaceObject = require("node:assert");
|
|
33
33
|
var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
|
|
@@ -437,7 +437,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
437
437
|
}
|
|
438
438
|
async healthCheck() {
|
|
439
439
|
console.log('[HealthCheck] Starting health check...');
|
|
440
|
-
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-
|
|
440
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260420052829.0");
|
|
441
441
|
console.log('[HealthCheck] Taking screenshot...');
|
|
442
442
|
const screenshotTimeout = 15000;
|
|
443
443
|
let timeoutId;
|
|
@@ -843,13 +843,29 @@ async function agentFromComputer(opts) {
|
|
|
843
843
|
await device.connect();
|
|
844
844
|
return new ComputerAgent(device, opts);
|
|
845
845
|
}
|
|
846
|
+
function mcp_tools_define_property(obj, key, value) {
|
|
847
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
848
|
+
value: value,
|
|
849
|
+
enumerable: true,
|
|
850
|
+
configurable: true,
|
|
851
|
+
writable: true
|
|
852
|
+
});
|
|
853
|
+
else obj[key] = value;
|
|
854
|
+
return obj;
|
|
855
|
+
}
|
|
846
856
|
const debug = (0, logger_namespaceObject.getDebug)('mcp:computer-tools');
|
|
847
|
-
|
|
857
|
+
const computerInitArgShape = {
|
|
858
|
+
displayId: core_namespaceObject.z.string().optional().describe('Display ID (from computer_list_displays)'),
|
|
859
|
+
headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
|
|
860
|
+
};
|
|
861
|
+
class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools {
|
|
848
862
|
createTemporaryDevice() {
|
|
849
863
|
return new ComputerDevice({});
|
|
850
864
|
}
|
|
851
|
-
async ensureAgent(
|
|
852
|
-
|
|
865
|
+
async ensureAgent(opts) {
|
|
866
|
+
const displayId = opts?.displayId;
|
|
867
|
+
const headless = opts?.headless;
|
|
868
|
+
if (this.agent && (void 0 !== displayId || void 0 !== headless)) {
|
|
853
869
|
try {
|
|
854
870
|
await this.agent.destroy?.();
|
|
855
871
|
} catch (error) {
|
|
@@ -859,7 +875,7 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
859
875
|
}
|
|
860
876
|
if (this.agent) return this.agent;
|
|
861
877
|
debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
862
|
-
const
|
|
878
|
+
const agentOpts = {
|
|
863
879
|
...displayId ? {
|
|
864
880
|
displayId
|
|
865
881
|
} : {},
|
|
@@ -867,7 +883,7 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
867
883
|
headless
|
|
868
884
|
} : {}
|
|
869
885
|
};
|
|
870
|
-
const agent = await agentFromComputer(Object.keys(
|
|
886
|
+
const agent = await agentFromComputer(Object.keys(agentOpts).length > 0 ? agentOpts : void 0);
|
|
871
887
|
this.agent = agent;
|
|
872
888
|
return agent;
|
|
873
889
|
}
|
|
@@ -876,18 +892,17 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
876
892
|
{
|
|
877
893
|
name: 'computer_connect',
|
|
878
894
|
description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
|
|
879
|
-
schema:
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
const agent = await this.ensureAgent(displayId, headless);
|
|
895
|
+
schema: this.getAgentInitArgSchema(),
|
|
896
|
+
cli: this.getAgentInitArgCliMetadata(),
|
|
897
|
+
handler: async (args)=>{
|
|
898
|
+
const initArgs = this.extractAgentInitParam(args);
|
|
899
|
+
const agent = await this.ensureAgent(initArgs);
|
|
885
900
|
const screenshot = await agent.interface.screenshotBase64();
|
|
886
901
|
return {
|
|
887
902
|
content: [
|
|
888
903
|
{
|
|
889
904
|
type: 'text',
|
|
890
|
-
text: `Connected to computer${displayId ? ` (Display: ${displayId})` : ' (Primary display)'}`
|
|
905
|
+
text: `Connected to computer${initArgs?.displayId ? ` (Display: ${initArgs.displayId})` : ' (Primary display)'}`
|
|
891
906
|
},
|
|
892
907
|
...this.buildScreenshotContent(screenshot)
|
|
893
908
|
]
|
|
@@ -918,11 +933,21 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
918
933
|
}
|
|
919
934
|
];
|
|
920
935
|
}
|
|
936
|
+
constructor(...args){
|
|
937
|
+
super(...args), mcp_tools_define_property(this, "initArgSpec", {
|
|
938
|
+
namespace: 'computer',
|
|
939
|
+
shape: computerInitArgShape,
|
|
940
|
+
cli: {
|
|
941
|
+
preferBareKeys: true
|
|
942
|
+
},
|
|
943
|
+
adapt: (extracted)=>extracted
|
|
944
|
+
});
|
|
945
|
+
}
|
|
921
946
|
}
|
|
922
947
|
const tools = new ComputerMidsceneTools();
|
|
923
948
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
|
|
924
949
|
stripPrefix: 'computer_',
|
|
925
|
-
version: "1.7.5-beta-
|
|
950
|
+
version: "1.7.5-beta-20260420052829.0",
|
|
926
951
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
927
952
|
}).catch((e)=>{
|
|
928
953
|
if (!(e instanceof cli_namespaceObject.CLIError)) console.error(e);
|
package/dist/lib/index.js
CHANGED
|
@@ -458,7 +458,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
458
458
|
}
|
|
459
459
|
async healthCheck() {
|
|
460
460
|
console.log('[HealthCheck] Starting health check...');
|
|
461
|
-
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-
|
|
461
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260420052829.0");
|
|
462
462
|
console.log('[HealthCheck] Taking screenshot...');
|
|
463
463
|
const screenshotTimeout = 15000;
|
|
464
464
|
let timeoutId;
|
|
@@ -865,14 +865,30 @@ async function agentFromComputer(opts) {
|
|
|
865
865
|
await device.connect();
|
|
866
866
|
return new ComputerAgent(device, opts);
|
|
867
867
|
}
|
|
868
|
-
const
|
|
868
|
+
const base_tools_namespaceObject = require("@midscene/shared/mcp/base-tools");
|
|
869
|
+
function mcp_tools_define_property(obj, key, value) {
|
|
870
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
871
|
+
value: value,
|
|
872
|
+
enumerable: true,
|
|
873
|
+
configurable: true,
|
|
874
|
+
writable: true
|
|
875
|
+
});
|
|
876
|
+
else obj[key] = value;
|
|
877
|
+
return obj;
|
|
878
|
+
}
|
|
869
879
|
const debug = (0, logger_namespaceObject.getDebug)('mcp:computer-tools');
|
|
870
|
-
|
|
880
|
+
const computerInitArgShape = {
|
|
881
|
+
displayId: core_namespaceObject.z.string().optional().describe('Display ID (from computer_list_displays)'),
|
|
882
|
+
headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
|
|
883
|
+
};
|
|
884
|
+
class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools {
|
|
871
885
|
createTemporaryDevice() {
|
|
872
886
|
return new ComputerDevice({});
|
|
873
887
|
}
|
|
874
|
-
async ensureAgent(
|
|
875
|
-
|
|
888
|
+
async ensureAgent(opts) {
|
|
889
|
+
const displayId = opts?.displayId;
|
|
890
|
+
const headless = opts?.headless;
|
|
891
|
+
if (this.agent && (void 0 !== displayId || void 0 !== headless)) {
|
|
876
892
|
try {
|
|
877
893
|
await this.agent.destroy?.();
|
|
878
894
|
} catch (error) {
|
|
@@ -882,7 +898,7 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
882
898
|
}
|
|
883
899
|
if (this.agent) return this.agent;
|
|
884
900
|
debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
885
|
-
const
|
|
901
|
+
const agentOpts = {
|
|
886
902
|
...displayId ? {
|
|
887
903
|
displayId
|
|
888
904
|
} : {},
|
|
@@ -890,7 +906,7 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
890
906
|
headless
|
|
891
907
|
} : {}
|
|
892
908
|
};
|
|
893
|
-
const agent = await agentFromComputer(Object.keys(
|
|
909
|
+
const agent = await agentFromComputer(Object.keys(agentOpts).length > 0 ? agentOpts : void 0);
|
|
894
910
|
this.agent = agent;
|
|
895
911
|
return agent;
|
|
896
912
|
}
|
|
@@ -899,18 +915,17 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
899
915
|
{
|
|
900
916
|
name: 'computer_connect',
|
|
901
917
|
description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
|
|
902
|
-
schema:
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
const agent = await this.ensureAgent(displayId, headless);
|
|
918
|
+
schema: this.getAgentInitArgSchema(),
|
|
919
|
+
cli: this.getAgentInitArgCliMetadata(),
|
|
920
|
+
handler: async (args)=>{
|
|
921
|
+
const initArgs = this.extractAgentInitParam(args);
|
|
922
|
+
const agent = await this.ensureAgent(initArgs);
|
|
908
923
|
const screenshot = await agent.interface.screenshotBase64();
|
|
909
924
|
return {
|
|
910
925
|
content: [
|
|
911
926
|
{
|
|
912
927
|
type: 'text',
|
|
913
|
-
text: `Connected to computer${displayId ? ` (Display: ${displayId})` : ' (Primary display)'}`
|
|
928
|
+
text: `Connected to computer${initArgs?.displayId ? ` (Display: ${initArgs.displayId})` : ' (Primary display)'}`
|
|
914
929
|
},
|
|
915
930
|
...this.buildScreenshotContent(screenshot)
|
|
916
931
|
]
|
|
@@ -941,10 +956,20 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
941
956
|
}
|
|
942
957
|
];
|
|
943
958
|
}
|
|
959
|
+
constructor(...args){
|
|
960
|
+
super(...args), mcp_tools_define_property(this, "initArgSpec", {
|
|
961
|
+
namespace: 'computer',
|
|
962
|
+
shape: computerInitArgShape,
|
|
963
|
+
cli: {
|
|
964
|
+
preferBareKeys: true
|
|
965
|
+
},
|
|
966
|
+
adapt: (extracted)=>extracted
|
|
967
|
+
});
|
|
968
|
+
}
|
|
944
969
|
}
|
|
945
970
|
const env_namespaceObject = require("@midscene/shared/env");
|
|
946
971
|
function version() {
|
|
947
|
-
const currentVersion = "1.7.5-beta-
|
|
972
|
+
const currentVersion = "1.7.5-beta-20260420052829.0";
|
|
948
973
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
949
974
|
return currentVersion;
|
|
950
975
|
}
|
package/dist/lib/mcp-server.js
CHANGED
|
@@ -452,7 +452,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
452
452
|
}
|
|
453
453
|
async healthCheck() {
|
|
454
454
|
console.log('[HealthCheck] Starting health check...');
|
|
455
|
-
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-
|
|
455
|
+
console.log("[HealthCheck] @midscene/computer v1.7.5-beta-20260420052829.0");
|
|
456
456
|
console.log('[HealthCheck] Taking screenshot...');
|
|
457
457
|
const screenshotTimeout = 15000;
|
|
458
458
|
let timeoutId;
|
|
@@ -858,13 +858,30 @@ async function agentFromComputer(opts) {
|
|
|
858
858
|
await device.connect();
|
|
859
859
|
return new ComputerAgent(device, opts);
|
|
860
860
|
}
|
|
861
|
+
const base_tools_namespaceObject = require("@midscene/shared/mcp/base-tools");
|
|
862
|
+
function mcp_tools_define_property(obj, key, value) {
|
|
863
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
864
|
+
value: value,
|
|
865
|
+
enumerable: true,
|
|
866
|
+
configurable: true,
|
|
867
|
+
writable: true
|
|
868
|
+
});
|
|
869
|
+
else obj[key] = value;
|
|
870
|
+
return obj;
|
|
871
|
+
}
|
|
861
872
|
const debug = (0, logger_namespaceObject.getDebug)('mcp:computer-tools');
|
|
862
|
-
|
|
873
|
+
const computerInitArgShape = {
|
|
874
|
+
displayId: core_namespaceObject.z.string().optional().describe('Display ID (from computer_list_displays)'),
|
|
875
|
+
headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
|
|
876
|
+
};
|
|
877
|
+
class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools {
|
|
863
878
|
createTemporaryDevice() {
|
|
864
879
|
return new ComputerDevice({});
|
|
865
880
|
}
|
|
866
|
-
async ensureAgent(
|
|
867
|
-
|
|
881
|
+
async ensureAgent(opts) {
|
|
882
|
+
const displayId = opts?.displayId;
|
|
883
|
+
const headless = opts?.headless;
|
|
884
|
+
if (this.agent && (void 0 !== displayId || void 0 !== headless)) {
|
|
868
885
|
try {
|
|
869
886
|
await this.agent.destroy?.();
|
|
870
887
|
} catch (error) {
|
|
@@ -874,7 +891,7 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
874
891
|
}
|
|
875
892
|
if (this.agent) return this.agent;
|
|
876
893
|
debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
877
|
-
const
|
|
894
|
+
const agentOpts = {
|
|
878
895
|
...displayId ? {
|
|
879
896
|
displayId
|
|
880
897
|
} : {},
|
|
@@ -882,7 +899,7 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
882
899
|
headless
|
|
883
900
|
} : {}
|
|
884
901
|
};
|
|
885
|
-
const agent = await agentFromComputer(Object.keys(
|
|
902
|
+
const agent = await agentFromComputer(Object.keys(agentOpts).length > 0 ? agentOpts : void 0);
|
|
886
903
|
this.agent = agent;
|
|
887
904
|
return agent;
|
|
888
905
|
}
|
|
@@ -891,18 +908,17 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
891
908
|
{
|
|
892
909
|
name: 'computer_connect',
|
|
893
910
|
description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
|
|
894
|
-
schema:
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
const agent = await this.ensureAgent(displayId, headless);
|
|
911
|
+
schema: this.getAgentInitArgSchema(),
|
|
912
|
+
cli: this.getAgentInitArgCliMetadata(),
|
|
913
|
+
handler: async (args)=>{
|
|
914
|
+
const initArgs = this.extractAgentInitParam(args);
|
|
915
|
+
const agent = await this.ensureAgent(initArgs);
|
|
900
916
|
const screenshot = await agent.interface.screenshotBase64();
|
|
901
917
|
return {
|
|
902
918
|
content: [
|
|
903
919
|
{
|
|
904
920
|
type: 'text',
|
|
905
|
-
text: `Connected to computer${displayId ? ` (Display: ${displayId})` : ' (Primary display)'}`
|
|
921
|
+
text: `Connected to computer${initArgs?.displayId ? ` (Display: ${initArgs.displayId})` : ' (Primary display)'}`
|
|
906
922
|
},
|
|
907
923
|
...this.buildScreenshotContent(screenshot)
|
|
908
924
|
]
|
|
@@ -933,6 +949,16 @@ class ComputerMidsceneTools extends mcp_namespaceObject.BaseMidsceneTools {
|
|
|
933
949
|
}
|
|
934
950
|
];
|
|
935
951
|
}
|
|
952
|
+
constructor(...args){
|
|
953
|
+
super(...args), mcp_tools_define_property(this, "initArgSpec", {
|
|
954
|
+
namespace: 'computer',
|
|
955
|
+
shape: computerInitArgShape,
|
|
956
|
+
cli: {
|
|
957
|
+
preferBareKeys: true
|
|
958
|
+
},
|
|
959
|
+
adapt: (extracted)=>extracted
|
|
960
|
+
});
|
|
961
|
+
}
|
|
936
962
|
}
|
|
937
963
|
class ComputerMCPServer extends mcp_namespaceObject.BaseMCPServer {
|
|
938
964
|
createToolsManager() {
|
|
@@ -941,7 +967,7 @@ class ComputerMCPServer extends mcp_namespaceObject.BaseMCPServer {
|
|
|
941
967
|
constructor(toolsManager){
|
|
942
968
|
super({
|
|
943
969
|
name: '@midscene/computer-mcp',
|
|
944
|
-
version: "1.7.5-beta-
|
|
970
|
+
version: "1.7.5-beta-20260420052829.0",
|
|
945
971
|
description: 'Control the computer desktop using natural language commands'
|
|
946
972
|
}, toolsManager);
|
|
947
973
|
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { AbstractInterface } from '@midscene/core/device';
|
|
2
2
|
import { Agent } from '@midscene/core/agent';
|
|
3
3
|
import { AgentOpt } from '@midscene/core/agent';
|
|
4
|
-
import { BaseMidsceneTools } from '@midscene/shared/mcp';
|
|
4
|
+
import { BaseMidsceneTools } from '@midscene/shared/mcp/base-tools';
|
|
5
5
|
import { DeviceAction } from '@midscene/core';
|
|
6
|
+
import { InitArgSpec } from '@midscene/shared/mcp/base-tools';
|
|
6
7
|
import { InterfaceType } from '@midscene/core';
|
|
7
8
|
import { overrideAIConfig } from '@midscene/shared/env';
|
|
8
9
|
import { Size } from '@midscene/core';
|
|
9
|
-
import { ToolDefinition } from '@midscene/shared/mcp';
|
|
10
|
+
import type { ToolDefinition } from '@midscene/shared/mcp/types';
|
|
10
11
|
|
|
11
12
|
declare interface AccessibilityCheckResult {
|
|
12
13
|
hasPermission: boolean;
|
|
@@ -113,13 +114,16 @@ export declare interface ComputerDeviceOpt {
|
|
|
113
114
|
xvfbResolution?: string;
|
|
114
115
|
}
|
|
115
116
|
|
|
117
|
+
declare type ComputerInitArgs = Pick<ComputerDeviceOpt, 'displayId' | 'headless'>;
|
|
118
|
+
|
|
116
119
|
/**
|
|
117
120
|
* Computer-specific tools manager
|
|
118
121
|
* Extends BaseMidsceneTools to provide desktop automation tools
|
|
119
122
|
*/
|
|
120
|
-
export declare class ComputerMidsceneTools extends BaseMidsceneTools<ComputerAgent> {
|
|
123
|
+
export declare class ComputerMidsceneTools extends BaseMidsceneTools<ComputerAgent, ComputerInitArgs> {
|
|
124
|
+
protected readonly initArgSpec: InitArgSpec<ComputerInitArgs>;
|
|
121
125
|
protected createTemporaryDevice(): ComputerDevice;
|
|
122
|
-
protected ensureAgent(
|
|
126
|
+
protected ensureAgent(opts?: ComputerInitArgs): Promise<ComputerAgent>;
|
|
123
127
|
/**
|
|
124
128
|
* Provide Computer-specific platform tools
|
|
125
129
|
*/
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { AbstractInterface } from '@midscene/core/device';
|
|
2
2
|
import { Agent } from '@midscene/core/agent';
|
|
3
3
|
import { BaseMCPServer } from '@midscene/shared/mcp';
|
|
4
|
-
import { BaseMidsceneTools } from '@midscene/shared/mcp';
|
|
4
|
+
import { BaseMidsceneTools } from '@midscene/shared/mcp/base-tools';
|
|
5
5
|
import { DeviceAction } from '@midscene/core';
|
|
6
|
+
import { InitArgSpec } from '@midscene/shared/mcp/base-tools';
|
|
6
7
|
import { InterfaceType } from '@midscene/core';
|
|
7
8
|
import { LaunchMCPServerOptions } from '@midscene/shared/mcp';
|
|
8
9
|
import { LaunchMCPServerResult } from '@midscene/shared/mcp';
|
|
9
10
|
import { Size } from '@midscene/core';
|
|
10
11
|
import { Tool } from '@midscene/shared/mcp';
|
|
11
|
-
import { ToolDefinition } from '@midscene/shared/mcp';
|
|
12
|
+
import type { ToolDefinition } from '@midscene/shared/mcp/types';
|
|
12
13
|
|
|
13
14
|
declare class ComputerAgent extends Agent<ComputerDevice> {
|
|
14
15
|
}
|
|
@@ -84,6 +85,8 @@ declare interface ComputerDeviceOpt {
|
|
|
84
85
|
xvfbResolution?: string;
|
|
85
86
|
}
|
|
86
87
|
|
|
88
|
+
declare type ComputerInitArgs = Pick<ComputerDeviceOpt, 'displayId' | 'headless'>;
|
|
89
|
+
|
|
87
90
|
/**
|
|
88
91
|
* Computer MCP Server
|
|
89
92
|
* Provides MCP tools for computer desktop automation
|
|
@@ -97,9 +100,10 @@ export declare class ComputerMCPServer extends BaseMCPServer {
|
|
|
97
100
|
* Computer-specific tools manager
|
|
98
101
|
* Extends BaseMidsceneTools to provide desktop automation tools
|
|
99
102
|
*/
|
|
100
|
-
declare class ComputerMidsceneTools extends BaseMidsceneTools<ComputerAgent> {
|
|
103
|
+
declare class ComputerMidsceneTools extends BaseMidsceneTools<ComputerAgent, ComputerInitArgs> {
|
|
104
|
+
protected readonly initArgSpec: InitArgSpec<ComputerInitArgs>;
|
|
101
105
|
protected createTemporaryDevice(): ComputerDevice;
|
|
102
|
-
protected ensureAgent(
|
|
106
|
+
protected ensureAgent(opts?: ComputerInitArgs): Promise<ComputerAgent>;
|
|
103
107
|
/**
|
|
104
108
|
* Provide Computer-specific platform tools
|
|
105
109
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/computer",
|
|
3
|
-
"version": "1.7.5-beta-
|
|
3
|
+
"version": "1.7.5-beta-20260420052829.0",
|
|
4
4
|
"description": "Midscene.js Computer Desktop Automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -36,8 +36,8 @@
|
|
|
36
36
|
"@computer-use/libnut": "^4.2.0",
|
|
37
37
|
"clipboardy": "^4.0.0",
|
|
38
38
|
"screenshot-desktop": "^1.15.3",
|
|
39
|
-
"@midscene/
|
|
40
|
-
"@midscene/
|
|
39
|
+
"@midscene/core": "1.7.5-beta-20260420052829.0",
|
|
40
|
+
"@midscene/shared": "1.7.5-beta-20260420052829.0"
|
|
41
41
|
},
|
|
42
42
|
"optionalDependencies": {
|
|
43
43
|
"node-mac-permissions": "2.5.0"
|