@midscene/computer-playground 1.6.0 → 1.6.1-beta-20260327104111.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/bin.mjs +83 -26
- package/dist/es/index.mjs +163 -0
- package/dist/lib/bin.js +82 -25
- package/dist/lib/index.js +210 -0
- package/dist/types/index.d.ts +25 -0
- package/package.json +5 -5
- package/static/index.html +1 -1
- package/static/static/css/index.1c539585.css +2 -0
- package/static/static/css/index.1c539585.css.map +1 -0
- package/static/static/js/259.c41de517.js +301 -0
- package/static/static/js/{515.8d886e3b.js.LICENSE.txt → 259.c41de517.js.LICENSE.txt} +31 -0
- package/static/static/js/259.c41de517.js.map +1 -0
- package/static/static/js/async/236.a5d2c1b1.js +2 -0
- package/static/static/js/async/236.a5d2c1b1.js.map +1 -0
- package/static/static/js/async/985.9205ccfb.js +2 -0
- package/static/static/js/async/{985.5765d7fb.js.map → 985.9205ccfb.js.map} +1 -1
- package/static/static/js/{index.1227e63a.js → index.48e77d8c.js} +59 -59
- package/static/static/js/index.48e77d8c.js.map +1 -0
- package/static/static/css/index.4373803e.css +0 -2
- package/static/static/css/index.4373803e.css.map +0 -1
- package/static/static/js/515.8d886e3b.js +0 -271
- package/static/static/js/515.8d886e3b.js.map +0 -1
- package/static/static/js/async/985.5765d7fb.js +0 -2
- package/static/static/js/index.1227e63a.js.map +0 -1
- /package/static/static/js/{index.1227e63a.js.LICENSE.txt → index.48e77d8c.js.LICENSE.txt} +0 -0
package/dist/es/bin.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import node_os from "node:os";
|
|
2
2
|
import node_path from "node:path";
|
|
3
|
-
import { ComputerDevice, agentFromComputer, checkAccessibilityPermission } from "@midscene/computer";
|
|
3
|
+
import { ComputerDevice, agentFromComputer, checkAccessibilityPermission, getConnectedDisplays } from "@midscene/computer";
|
|
4
4
|
import { createScreenshotPreviewDescriptor, definePlaygroundPlatform, launchPreparedPlaygroundPlatform } from "@midscene/playground";
|
|
5
5
|
import puppeteer from "puppeteer";
|
|
6
6
|
import { PLAYGROUND_SERVER_PORT } from "@midscene/shared/constants";
|
|
@@ -60,47 +60,104 @@ const computerPlaygroundPlatform = definePlaygroundPlatform({
|
|
|
60
60
|
description: "Computer playground platform descriptor",
|
|
61
61
|
async prepare (options) {
|
|
62
62
|
const accessibilityCheck = checkAccessibilityPermission(true);
|
|
63
|
-
if (!accessibilityCheck.hasPermission) {
|
|
64
|
-
console.error('❌ Permission Error:\n');
|
|
65
|
-
console.error(accessibilityCheck.error);
|
|
66
|
-
process.exit(1);
|
|
67
|
-
}
|
|
68
63
|
const staticDir = options?.staticDir || node_path.join(__dirname, '../../static');
|
|
69
64
|
const availablePort = await findAvailablePort(PLAYGROUND_SERVER_PORT);
|
|
70
65
|
if (availablePort !== PLAYGROUND_SERVER_PORT) console.log(`⚠️ Port ${PLAYGROUND_SERVER_PORT} is busy, using port ${availablePort} instead`);
|
|
66
|
+
const sessionManager = {
|
|
67
|
+
async getSetupSchema () {
|
|
68
|
+
const displays = await getConnectedDisplays();
|
|
69
|
+
const defaultDisplay = displays.find((display)=>display.primary) || displays[0];
|
|
70
|
+
return {
|
|
71
|
+
title: 'Connect Computer Agent',
|
|
72
|
+
description: accessibilityCheck.hasPermission ? 'Create a Computer Agent for the selected display.' : accessibilityCheck.error,
|
|
73
|
+
primaryActionLabel: 'Create Agent',
|
|
74
|
+
fields: [
|
|
75
|
+
{
|
|
76
|
+
key: 'displayId',
|
|
77
|
+
label: 'Display',
|
|
78
|
+
type: 'select',
|
|
79
|
+
required: true,
|
|
80
|
+
options: displays.map((display)=>({
|
|
81
|
+
label: display.name,
|
|
82
|
+
value: String(display.id),
|
|
83
|
+
description: display.primary ? 'Primary display' : void 0
|
|
84
|
+
})),
|
|
85
|
+
defaultValue: defaultDisplay ? String(defaultDisplay.id) : '',
|
|
86
|
+
placeholder: 'Select a display'
|
|
87
|
+
}
|
|
88
|
+
],
|
|
89
|
+
targets: displays.map((display)=>({
|
|
90
|
+
id: String(display.id),
|
|
91
|
+
label: display.name,
|
|
92
|
+
description: display.primary ? 'Primary display' : void 0,
|
|
93
|
+
isDefault: display.primary
|
|
94
|
+
}))
|
|
95
|
+
};
|
|
96
|
+
},
|
|
97
|
+
async listTargets () {
|
|
98
|
+
const displays = await getConnectedDisplays();
|
|
99
|
+
return displays.map((display)=>({
|
|
100
|
+
id: String(display.id),
|
|
101
|
+
label: display.name,
|
|
102
|
+
description: display.primary ? 'Primary display' : void 0,
|
|
103
|
+
isDefault: display.primary
|
|
104
|
+
}));
|
|
105
|
+
},
|
|
106
|
+
async createSession (input) {
|
|
107
|
+
if (!accessibilityCheck.hasPermission) throw new Error(accessibilityCheck.error || 'Accessibility permission is required');
|
|
108
|
+
const displayId = input?.displayId === void 0 || null === input.displayId ? void 0 : String(input.displayId);
|
|
109
|
+
const agent = await agentFromComputer(displayId ? {
|
|
110
|
+
displayId
|
|
111
|
+
} : void 0);
|
|
112
|
+
const displays = await getConnectedDisplays();
|
|
113
|
+
const selectedDisplay = displays.find((display)=>display.id === displayId) || displays.find((display)=>display.primary) || displays[0];
|
|
114
|
+
return {
|
|
115
|
+
agent,
|
|
116
|
+
agentFactory: ()=>agentFromComputer(selectedDisplay ? {
|
|
117
|
+
displayId: selectedDisplay.id
|
|
118
|
+
} : void 0),
|
|
119
|
+
preview: createScreenshotPreviewDescriptor({
|
|
120
|
+
title: 'Desktop preview'
|
|
121
|
+
}),
|
|
122
|
+
displayName: selectedDisplay?.name || 'Desktop',
|
|
123
|
+
metadata: {
|
|
124
|
+
displayId: selectedDisplay?.id,
|
|
125
|
+
executionUx: 'countdown-before-run'
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
};
|
|
71
130
|
return {
|
|
72
131
|
platformId: 'computer',
|
|
73
132
|
title: 'Midscene Computer Playground',
|
|
74
|
-
|
|
133
|
+
sessionManager,
|
|
75
134
|
launchOptions: {
|
|
76
135
|
port: availablePort,
|
|
77
136
|
openBrowser: false,
|
|
78
137
|
verbose: false,
|
|
79
|
-
staticPath: staticDir
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
windowController.restore();
|
|
93
|
-
return originalSend(body);
|
|
94
|
-
};
|
|
95
|
-
next();
|
|
96
|
-
});
|
|
138
|
+
staticPath: staticDir
|
|
139
|
+
},
|
|
140
|
+
executionHooks: {
|
|
141
|
+
async beforeExecute () {
|
|
142
|
+
const windowController = options?.getWindowController?.();
|
|
143
|
+
if (!windowController) return void console.warn('⚠️ Window controller not initialized yet, skipping window control');
|
|
144
|
+
await new Promise((resolve)=>setTimeout(resolve, 1500));
|
|
145
|
+
await windowController.minimize();
|
|
146
|
+
},
|
|
147
|
+
async afterExecute () {
|
|
148
|
+
const windowController = options?.getWindowController?.();
|
|
149
|
+
if (!windowController) return;
|
|
150
|
+
await windowController.restore();
|
|
97
151
|
}
|
|
98
152
|
},
|
|
99
153
|
preview: createScreenshotPreviewDescriptor({
|
|
100
154
|
title: 'Desktop preview'
|
|
101
155
|
}),
|
|
102
156
|
metadata: {
|
|
103
|
-
executionUx: 'countdown-before-run'
|
|
157
|
+
executionUx: 'countdown-before-run',
|
|
158
|
+
sessionConnected: false,
|
|
159
|
+
setupState: accessibilityCheck.hasPermission ? 'required' : 'blocked',
|
|
160
|
+
setupBlockingReason: accessibilityCheck.error
|
|
104
161
|
}
|
|
105
162
|
};
|
|
106
163
|
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import node_path from "node:path";
|
|
2
|
+
import { agentFromComputer, checkAccessibilityPermission, getConnectedDisplays } from "@midscene/computer";
|
|
3
|
+
import { createScreenshotPreviewDescriptor, definePlaygroundPlatform } from "@midscene/playground";
|
|
4
|
+
import { PLAYGROUND_SERVER_PORT } from "@midscene/shared/constants";
|
|
5
|
+
import { findAvailablePort } from "@midscene/shared/node";
|
|
6
|
+
const computerPlaygroundPlatform = definePlaygroundPlatform({
|
|
7
|
+
id: 'computer',
|
|
8
|
+
title: 'Midscene Computer Playground',
|
|
9
|
+
description: "Computer playground platform descriptor",
|
|
10
|
+
async prepare (options) {
|
|
11
|
+
const accessibilityCheck = checkAccessibilityPermission(true);
|
|
12
|
+
const staticDir = options?.staticDir || node_path.join(__dirname, '../../static');
|
|
13
|
+
const availablePort = await findAvailablePort(PLAYGROUND_SERVER_PORT);
|
|
14
|
+
if (availablePort !== PLAYGROUND_SERVER_PORT) console.log(`⚠️ Port ${PLAYGROUND_SERVER_PORT} is busy, using port ${availablePort} instead`);
|
|
15
|
+
const sessionManager = {
|
|
16
|
+
async getSetupSchema () {
|
|
17
|
+
const displays = await getConnectedDisplays();
|
|
18
|
+
const defaultDisplay = displays.find((display)=>display.primary) || displays[0];
|
|
19
|
+
return {
|
|
20
|
+
title: 'Connect Computer Agent',
|
|
21
|
+
description: accessibilityCheck.hasPermission ? 'Create a Computer Agent for the selected display.' : accessibilityCheck.error,
|
|
22
|
+
primaryActionLabel: 'Create Agent',
|
|
23
|
+
fields: [
|
|
24
|
+
{
|
|
25
|
+
key: 'displayId',
|
|
26
|
+
label: 'Display',
|
|
27
|
+
type: 'select',
|
|
28
|
+
required: true,
|
|
29
|
+
options: displays.map((display)=>({
|
|
30
|
+
label: display.name,
|
|
31
|
+
value: String(display.id),
|
|
32
|
+
description: display.primary ? 'Primary display' : void 0
|
|
33
|
+
})),
|
|
34
|
+
defaultValue: defaultDisplay ? String(defaultDisplay.id) : '',
|
|
35
|
+
placeholder: 'Select a display'
|
|
36
|
+
}
|
|
37
|
+
],
|
|
38
|
+
targets: displays.map((display)=>({
|
|
39
|
+
id: String(display.id),
|
|
40
|
+
label: display.name,
|
|
41
|
+
description: display.primary ? 'Primary display' : void 0,
|
|
42
|
+
isDefault: display.primary
|
|
43
|
+
}))
|
|
44
|
+
};
|
|
45
|
+
},
|
|
46
|
+
async listTargets () {
|
|
47
|
+
const displays = await getConnectedDisplays();
|
|
48
|
+
return displays.map((display)=>({
|
|
49
|
+
id: String(display.id),
|
|
50
|
+
label: display.name,
|
|
51
|
+
description: display.primary ? 'Primary display' : void 0,
|
|
52
|
+
isDefault: display.primary
|
|
53
|
+
}));
|
|
54
|
+
},
|
|
55
|
+
async createSession (input) {
|
|
56
|
+
if (!accessibilityCheck.hasPermission) throw new Error(accessibilityCheck.error || 'Accessibility permission is required');
|
|
57
|
+
const displayId = input?.displayId === void 0 || null === input.displayId ? void 0 : String(input.displayId);
|
|
58
|
+
const agent = await agentFromComputer(displayId ? {
|
|
59
|
+
displayId
|
|
60
|
+
} : void 0);
|
|
61
|
+
const displays = await getConnectedDisplays();
|
|
62
|
+
const selectedDisplay = displays.find((display)=>display.id === displayId) || displays.find((display)=>display.primary) || displays[0];
|
|
63
|
+
return {
|
|
64
|
+
agent,
|
|
65
|
+
agentFactory: ()=>agentFromComputer(selectedDisplay ? {
|
|
66
|
+
displayId: selectedDisplay.id
|
|
67
|
+
} : void 0),
|
|
68
|
+
preview: createScreenshotPreviewDescriptor({
|
|
69
|
+
title: 'Desktop preview'
|
|
70
|
+
}),
|
|
71
|
+
displayName: selectedDisplay?.name || 'Desktop',
|
|
72
|
+
metadata: {
|
|
73
|
+
displayId: selectedDisplay?.id,
|
|
74
|
+
executionUx: 'countdown-before-run'
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
return {
|
|
80
|
+
platformId: 'computer',
|
|
81
|
+
title: 'Midscene Computer Playground',
|
|
82
|
+
sessionManager,
|
|
83
|
+
launchOptions: {
|
|
84
|
+
port: availablePort,
|
|
85
|
+
openBrowser: false,
|
|
86
|
+
verbose: false,
|
|
87
|
+
staticPath: staticDir
|
|
88
|
+
},
|
|
89
|
+
executionHooks: {
|
|
90
|
+
async beforeExecute () {
|
|
91
|
+
const windowController = options?.getWindowController?.();
|
|
92
|
+
if (!windowController) return void console.warn('⚠️ Window controller not initialized yet, skipping window control');
|
|
93
|
+
await new Promise((resolve)=>setTimeout(resolve, 1500));
|
|
94
|
+
await windowController.minimize();
|
|
95
|
+
},
|
|
96
|
+
async afterExecute () {
|
|
97
|
+
const windowController = options?.getWindowController?.();
|
|
98
|
+
if (!windowController) return;
|
|
99
|
+
await windowController.restore();
|
|
100
|
+
}
|
|
101
|
+
},
|
|
102
|
+
preview: createScreenshotPreviewDescriptor({
|
|
103
|
+
title: 'Desktop preview'
|
|
104
|
+
}),
|
|
105
|
+
metadata: {
|
|
106
|
+
executionUx: 'countdown-before-run',
|
|
107
|
+
sessionConnected: false,
|
|
108
|
+
setupState: accessibilityCheck.hasPermission ? 'required' : 'blocked',
|
|
109
|
+
setupBlockingReason: accessibilityCheck.error
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
function _define_property(obj, key, value) {
|
|
115
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
116
|
+
value: value,
|
|
117
|
+
enumerable: true,
|
|
118
|
+
configurable: true,
|
|
119
|
+
writable: true
|
|
120
|
+
});
|
|
121
|
+
else obj[key] = value;
|
|
122
|
+
return obj;
|
|
123
|
+
}
|
|
124
|
+
class BrowserWindowController {
|
|
125
|
+
async minimize() {
|
|
126
|
+
try {
|
|
127
|
+
await this.session.send('Browser.setWindowBounds', {
|
|
128
|
+
windowId: this.windowId,
|
|
129
|
+
bounds: {
|
|
130
|
+
windowState: 'minimized'
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
console.log('🔽 Window minimized, starting task execution...');
|
|
134
|
+
} catch (error) {
|
|
135
|
+
console.warn('⚠️ Failed to minimize window:', error);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
async restore() {
|
|
139
|
+
try {
|
|
140
|
+
await Promise.all([
|
|
141
|
+
this.session.send('Browser.setWindowBounds', {
|
|
142
|
+
windowId: this.windowId,
|
|
143
|
+
bounds: {
|
|
144
|
+
windowState: 'normal'
|
|
145
|
+
}
|
|
146
|
+
}),
|
|
147
|
+
this.page.bringToFront()
|
|
148
|
+
]);
|
|
149
|
+
console.log('🔼 Window restored');
|
|
150
|
+
} catch (error) {
|
|
151
|
+
console.warn('⚠️ Failed to restore window:', error);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
constructor(session, page, windowId){
|
|
155
|
+
_define_property(this, "session", void 0);
|
|
156
|
+
_define_property(this, "page", void 0);
|
|
157
|
+
_define_property(this, "windowId", void 0);
|
|
158
|
+
this.session = session;
|
|
159
|
+
this.page = page;
|
|
160
|
+
this.windowId = windowId;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
export { BrowserWindowController, computerPlaygroundPlatform };
|
package/dist/lib/bin.js
CHANGED
|
@@ -86,47 +86,104 @@ const computerPlaygroundPlatform = (0, playground_namespaceObject.definePlaygrou
|
|
|
86
86
|
description: "Computer playground platform descriptor",
|
|
87
87
|
async prepare (options) {
|
|
88
88
|
const accessibilityCheck = (0, computer_namespaceObject.checkAccessibilityPermission)(true);
|
|
89
|
-
if (!accessibilityCheck.hasPermission) {
|
|
90
|
-
console.error('❌ Permission Error:\n');
|
|
91
|
-
console.error(accessibilityCheck.error);
|
|
92
|
-
process.exit(1);
|
|
93
|
-
}
|
|
94
89
|
const staticDir = options?.staticDir || external_node_path_default().join(__dirname, '../../static');
|
|
95
90
|
const availablePort = await (0, node_namespaceObject.findAvailablePort)(constants_namespaceObject.PLAYGROUND_SERVER_PORT);
|
|
96
91
|
if (availablePort !== constants_namespaceObject.PLAYGROUND_SERVER_PORT) console.log(`⚠️ Port ${constants_namespaceObject.PLAYGROUND_SERVER_PORT} is busy, using port ${availablePort} instead`);
|
|
92
|
+
const sessionManager = {
|
|
93
|
+
async getSetupSchema () {
|
|
94
|
+
const displays = await (0, computer_namespaceObject.getConnectedDisplays)();
|
|
95
|
+
const defaultDisplay = displays.find((display)=>display.primary) || displays[0];
|
|
96
|
+
return {
|
|
97
|
+
title: 'Connect Computer Agent',
|
|
98
|
+
description: accessibilityCheck.hasPermission ? 'Create a Computer Agent for the selected display.' : accessibilityCheck.error,
|
|
99
|
+
primaryActionLabel: 'Create Agent',
|
|
100
|
+
fields: [
|
|
101
|
+
{
|
|
102
|
+
key: 'displayId',
|
|
103
|
+
label: 'Display',
|
|
104
|
+
type: 'select',
|
|
105
|
+
required: true,
|
|
106
|
+
options: displays.map((display)=>({
|
|
107
|
+
label: display.name,
|
|
108
|
+
value: String(display.id),
|
|
109
|
+
description: display.primary ? 'Primary display' : void 0
|
|
110
|
+
})),
|
|
111
|
+
defaultValue: defaultDisplay ? String(defaultDisplay.id) : '',
|
|
112
|
+
placeholder: 'Select a display'
|
|
113
|
+
}
|
|
114
|
+
],
|
|
115
|
+
targets: displays.map((display)=>({
|
|
116
|
+
id: String(display.id),
|
|
117
|
+
label: display.name,
|
|
118
|
+
description: display.primary ? 'Primary display' : void 0,
|
|
119
|
+
isDefault: display.primary
|
|
120
|
+
}))
|
|
121
|
+
};
|
|
122
|
+
},
|
|
123
|
+
async listTargets () {
|
|
124
|
+
const displays = await (0, computer_namespaceObject.getConnectedDisplays)();
|
|
125
|
+
return displays.map((display)=>({
|
|
126
|
+
id: String(display.id),
|
|
127
|
+
label: display.name,
|
|
128
|
+
description: display.primary ? 'Primary display' : void 0,
|
|
129
|
+
isDefault: display.primary
|
|
130
|
+
}));
|
|
131
|
+
},
|
|
132
|
+
async createSession (input) {
|
|
133
|
+
if (!accessibilityCheck.hasPermission) throw new Error(accessibilityCheck.error || 'Accessibility permission is required');
|
|
134
|
+
const displayId = input?.displayId === void 0 || null === input.displayId ? void 0 : String(input.displayId);
|
|
135
|
+
const agent = await (0, computer_namespaceObject.agentFromComputer)(displayId ? {
|
|
136
|
+
displayId
|
|
137
|
+
} : void 0);
|
|
138
|
+
const displays = await (0, computer_namespaceObject.getConnectedDisplays)();
|
|
139
|
+
const selectedDisplay = displays.find((display)=>display.id === displayId) || displays.find((display)=>display.primary) || displays[0];
|
|
140
|
+
return {
|
|
141
|
+
agent,
|
|
142
|
+
agentFactory: ()=>(0, computer_namespaceObject.agentFromComputer)(selectedDisplay ? {
|
|
143
|
+
displayId: selectedDisplay.id
|
|
144
|
+
} : void 0),
|
|
145
|
+
preview: (0, playground_namespaceObject.createScreenshotPreviewDescriptor)({
|
|
146
|
+
title: 'Desktop preview'
|
|
147
|
+
}),
|
|
148
|
+
displayName: selectedDisplay?.name || 'Desktop',
|
|
149
|
+
metadata: {
|
|
150
|
+
displayId: selectedDisplay?.id,
|
|
151
|
+
executionUx: 'countdown-before-run'
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
};
|
|
97
156
|
return {
|
|
98
157
|
platformId: 'computer',
|
|
99
158
|
title: 'Midscene Computer Playground',
|
|
100
|
-
|
|
159
|
+
sessionManager,
|
|
101
160
|
launchOptions: {
|
|
102
161
|
port: availablePort,
|
|
103
162
|
openBrowser: false,
|
|
104
163
|
verbose: false,
|
|
105
|
-
staticPath: staticDir
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
windowController.restore();
|
|
119
|
-
return originalSend(body);
|
|
120
|
-
};
|
|
121
|
-
next();
|
|
122
|
-
});
|
|
164
|
+
staticPath: staticDir
|
|
165
|
+
},
|
|
166
|
+
executionHooks: {
|
|
167
|
+
async beforeExecute () {
|
|
168
|
+
const windowController = options?.getWindowController?.();
|
|
169
|
+
if (!windowController) return void console.warn('⚠️ Window controller not initialized yet, skipping window control');
|
|
170
|
+
await new Promise((resolve)=>setTimeout(resolve, 1500));
|
|
171
|
+
await windowController.minimize();
|
|
172
|
+
},
|
|
173
|
+
async afterExecute () {
|
|
174
|
+
const windowController = options?.getWindowController?.();
|
|
175
|
+
if (!windowController) return;
|
|
176
|
+
await windowController.restore();
|
|
123
177
|
}
|
|
124
178
|
},
|
|
125
179
|
preview: (0, playground_namespaceObject.createScreenshotPreviewDescriptor)({
|
|
126
180
|
title: 'Desktop preview'
|
|
127
181
|
}),
|
|
128
182
|
metadata: {
|
|
129
|
-
executionUx: 'countdown-before-run'
|
|
183
|
+
executionUx: 'countdown-before-run',
|
|
184
|
+
sessionConnected: false,
|
|
185
|
+
setupState: accessibilityCheck.hasPermission ? 'required' : 'blocked',
|
|
186
|
+
setupBlockingReason: accessibilityCheck.error
|
|
130
187
|
}
|
|
131
188
|
};
|
|
132
189
|
}
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __webpack_require__ = {};
|
|
3
|
+
(()=>{
|
|
4
|
+
__webpack_require__.n = (module)=>{
|
|
5
|
+
var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
|
|
6
|
+
__webpack_require__.d(getter, {
|
|
7
|
+
a: getter
|
|
8
|
+
});
|
|
9
|
+
return getter;
|
|
10
|
+
};
|
|
11
|
+
})();
|
|
12
|
+
(()=>{
|
|
13
|
+
__webpack_require__.d = (exports1, definition)=>{
|
|
14
|
+
for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
|
|
15
|
+
enumerable: true,
|
|
16
|
+
get: definition[key]
|
|
17
|
+
});
|
|
18
|
+
};
|
|
19
|
+
})();
|
|
20
|
+
(()=>{
|
|
21
|
+
__webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
|
|
22
|
+
})();
|
|
23
|
+
(()=>{
|
|
24
|
+
__webpack_require__.r = (exports1)=>{
|
|
25
|
+
if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
|
|
26
|
+
value: 'Module'
|
|
27
|
+
});
|
|
28
|
+
Object.defineProperty(exports1, '__esModule', {
|
|
29
|
+
value: true
|
|
30
|
+
});
|
|
31
|
+
};
|
|
32
|
+
})();
|
|
33
|
+
var __webpack_exports__ = {};
|
|
34
|
+
__webpack_require__.r(__webpack_exports__);
|
|
35
|
+
__webpack_require__.d(__webpack_exports__, {
|
|
36
|
+
BrowserWindowController: ()=>BrowserWindowController,
|
|
37
|
+
computerPlaygroundPlatform: ()=>computerPlaygroundPlatform
|
|
38
|
+
});
|
|
39
|
+
const external_node_path_namespaceObject = require("node:path");
|
|
40
|
+
var external_node_path_default = /*#__PURE__*/ __webpack_require__.n(external_node_path_namespaceObject);
|
|
41
|
+
const computer_namespaceObject = require("@midscene/computer");
|
|
42
|
+
const playground_namespaceObject = require("@midscene/playground");
|
|
43
|
+
const constants_namespaceObject = require("@midscene/shared/constants");
|
|
44
|
+
const node_namespaceObject = require("@midscene/shared/node");
|
|
45
|
+
const computerPlaygroundPlatform = (0, playground_namespaceObject.definePlaygroundPlatform)({
|
|
46
|
+
id: 'computer',
|
|
47
|
+
title: 'Midscene Computer Playground',
|
|
48
|
+
description: "Computer playground platform descriptor",
|
|
49
|
+
async prepare (options) {
|
|
50
|
+
const accessibilityCheck = (0, computer_namespaceObject.checkAccessibilityPermission)(true);
|
|
51
|
+
const staticDir = options?.staticDir || external_node_path_default().join(__dirname, '../../static');
|
|
52
|
+
const availablePort = await (0, node_namespaceObject.findAvailablePort)(constants_namespaceObject.PLAYGROUND_SERVER_PORT);
|
|
53
|
+
if (availablePort !== constants_namespaceObject.PLAYGROUND_SERVER_PORT) console.log(`⚠️ Port ${constants_namespaceObject.PLAYGROUND_SERVER_PORT} is busy, using port ${availablePort} instead`);
|
|
54
|
+
const sessionManager = {
|
|
55
|
+
async getSetupSchema () {
|
|
56
|
+
const displays = await (0, computer_namespaceObject.getConnectedDisplays)();
|
|
57
|
+
const defaultDisplay = displays.find((display)=>display.primary) || displays[0];
|
|
58
|
+
return {
|
|
59
|
+
title: 'Connect Computer Agent',
|
|
60
|
+
description: accessibilityCheck.hasPermission ? 'Create a Computer Agent for the selected display.' : accessibilityCheck.error,
|
|
61
|
+
primaryActionLabel: 'Create Agent',
|
|
62
|
+
fields: [
|
|
63
|
+
{
|
|
64
|
+
key: 'displayId',
|
|
65
|
+
label: 'Display',
|
|
66
|
+
type: 'select',
|
|
67
|
+
required: true,
|
|
68
|
+
options: displays.map((display)=>({
|
|
69
|
+
label: display.name,
|
|
70
|
+
value: String(display.id),
|
|
71
|
+
description: display.primary ? 'Primary display' : void 0
|
|
72
|
+
})),
|
|
73
|
+
defaultValue: defaultDisplay ? String(defaultDisplay.id) : '',
|
|
74
|
+
placeholder: 'Select a display'
|
|
75
|
+
}
|
|
76
|
+
],
|
|
77
|
+
targets: displays.map((display)=>({
|
|
78
|
+
id: String(display.id),
|
|
79
|
+
label: display.name,
|
|
80
|
+
description: display.primary ? 'Primary display' : void 0,
|
|
81
|
+
isDefault: display.primary
|
|
82
|
+
}))
|
|
83
|
+
};
|
|
84
|
+
},
|
|
85
|
+
async listTargets () {
|
|
86
|
+
const displays = await (0, computer_namespaceObject.getConnectedDisplays)();
|
|
87
|
+
return displays.map((display)=>({
|
|
88
|
+
id: String(display.id),
|
|
89
|
+
label: display.name,
|
|
90
|
+
description: display.primary ? 'Primary display' : void 0,
|
|
91
|
+
isDefault: display.primary
|
|
92
|
+
}));
|
|
93
|
+
},
|
|
94
|
+
async createSession (input) {
|
|
95
|
+
if (!accessibilityCheck.hasPermission) throw new Error(accessibilityCheck.error || 'Accessibility permission is required');
|
|
96
|
+
const displayId = input?.displayId === void 0 || null === input.displayId ? void 0 : String(input.displayId);
|
|
97
|
+
const agent = await (0, computer_namespaceObject.agentFromComputer)(displayId ? {
|
|
98
|
+
displayId
|
|
99
|
+
} : void 0);
|
|
100
|
+
const displays = await (0, computer_namespaceObject.getConnectedDisplays)();
|
|
101
|
+
const selectedDisplay = displays.find((display)=>display.id === displayId) || displays.find((display)=>display.primary) || displays[0];
|
|
102
|
+
return {
|
|
103
|
+
agent,
|
|
104
|
+
agentFactory: ()=>(0, computer_namespaceObject.agentFromComputer)(selectedDisplay ? {
|
|
105
|
+
displayId: selectedDisplay.id
|
|
106
|
+
} : void 0),
|
|
107
|
+
preview: (0, playground_namespaceObject.createScreenshotPreviewDescriptor)({
|
|
108
|
+
title: 'Desktop preview'
|
|
109
|
+
}),
|
|
110
|
+
displayName: selectedDisplay?.name || 'Desktop',
|
|
111
|
+
metadata: {
|
|
112
|
+
displayId: selectedDisplay?.id,
|
|
113
|
+
executionUx: 'countdown-before-run'
|
|
114
|
+
}
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
return {
|
|
119
|
+
platformId: 'computer',
|
|
120
|
+
title: 'Midscene Computer Playground',
|
|
121
|
+
sessionManager,
|
|
122
|
+
launchOptions: {
|
|
123
|
+
port: availablePort,
|
|
124
|
+
openBrowser: false,
|
|
125
|
+
verbose: false,
|
|
126
|
+
staticPath: staticDir
|
|
127
|
+
},
|
|
128
|
+
executionHooks: {
|
|
129
|
+
async beforeExecute () {
|
|
130
|
+
const windowController = options?.getWindowController?.();
|
|
131
|
+
if (!windowController) return void console.warn('⚠️ Window controller not initialized yet, skipping window control');
|
|
132
|
+
await new Promise((resolve)=>setTimeout(resolve, 1500));
|
|
133
|
+
await windowController.minimize();
|
|
134
|
+
},
|
|
135
|
+
async afterExecute () {
|
|
136
|
+
const windowController = options?.getWindowController?.();
|
|
137
|
+
if (!windowController) return;
|
|
138
|
+
await windowController.restore();
|
|
139
|
+
}
|
|
140
|
+
},
|
|
141
|
+
preview: (0, playground_namespaceObject.createScreenshotPreviewDescriptor)({
|
|
142
|
+
title: 'Desktop preview'
|
|
143
|
+
}),
|
|
144
|
+
metadata: {
|
|
145
|
+
executionUx: 'countdown-before-run',
|
|
146
|
+
sessionConnected: false,
|
|
147
|
+
setupState: accessibilityCheck.hasPermission ? 'required' : 'blocked',
|
|
148
|
+
setupBlockingReason: accessibilityCheck.error
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
function _define_property(obj, key, value) {
|
|
154
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
155
|
+
value: value,
|
|
156
|
+
enumerable: true,
|
|
157
|
+
configurable: true,
|
|
158
|
+
writable: true
|
|
159
|
+
});
|
|
160
|
+
else obj[key] = value;
|
|
161
|
+
return obj;
|
|
162
|
+
}
|
|
163
|
+
class BrowserWindowController {
|
|
164
|
+
async minimize() {
|
|
165
|
+
try {
|
|
166
|
+
await this.session.send('Browser.setWindowBounds', {
|
|
167
|
+
windowId: this.windowId,
|
|
168
|
+
bounds: {
|
|
169
|
+
windowState: 'minimized'
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
console.log('🔽 Window minimized, starting task execution...');
|
|
173
|
+
} catch (error) {
|
|
174
|
+
console.warn('⚠️ Failed to minimize window:', error);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
async restore() {
|
|
178
|
+
try {
|
|
179
|
+
await Promise.all([
|
|
180
|
+
this.session.send('Browser.setWindowBounds', {
|
|
181
|
+
windowId: this.windowId,
|
|
182
|
+
bounds: {
|
|
183
|
+
windowState: 'normal'
|
|
184
|
+
}
|
|
185
|
+
}),
|
|
186
|
+
this.page.bringToFront()
|
|
187
|
+
]);
|
|
188
|
+
console.log('🔼 Window restored');
|
|
189
|
+
} catch (error) {
|
|
190
|
+
console.warn('⚠️ Failed to restore window:', error);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
constructor(session, page, windowId){
|
|
194
|
+
_define_property(this, "session", void 0);
|
|
195
|
+
_define_property(this, "page", void 0);
|
|
196
|
+
_define_property(this, "windowId", void 0);
|
|
197
|
+
this.session = session;
|
|
198
|
+
this.page = page;
|
|
199
|
+
this.windowId = windowId;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
exports.BrowserWindowController = __webpack_exports__.BrowserWindowController;
|
|
203
|
+
exports.computerPlaygroundPlatform = __webpack_exports__.computerPlaygroundPlatform;
|
|
204
|
+
for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
205
|
+
"BrowserWindowController",
|
|
206
|
+
"computerPlaygroundPlatform"
|
|
207
|
+
].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
|
|
208
|
+
Object.defineProperty(exports, '__esModule', {
|
|
209
|
+
value: true
|
|
210
|
+
});
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { CDPSession } from 'puppeteer';
|
|
2
|
+
import type { Page } from 'puppeteer';
|
|
3
|
+
import { PlaygroundPlatformDescriptor } from '@midscene/playground';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Controls browser window state (minimize/restore) during task execution
|
|
7
|
+
* to avoid interference between the playground UI and desktop automation.
|
|
8
|
+
*/
|
|
9
|
+
export declare class BrowserWindowController {
|
|
10
|
+
private session;
|
|
11
|
+
private page;
|
|
12
|
+
private windowId;
|
|
13
|
+
constructor(session: CDPSession, page: Page, windowId: number);
|
|
14
|
+
minimize(): Promise<void>;
|
|
15
|
+
restore(): Promise<void>;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export declare interface ComputerPlatformOptions {
|
|
19
|
+
staticDir?: string;
|
|
20
|
+
getWindowController?: () => BrowserWindowController | null;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export declare const computerPlaygroundPlatform: PlaygroundPlatformDescriptor<ComputerPlatformOptions | undefined>;
|
|
24
|
+
|
|
25
|
+
export { }
|