ft-scout 9.0.2 → 9.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.firebase/hosting.d2Vi.cache +7 -7
- package/README.md +76 -83
- package/bin/src/commands/agent.d.ts.map +1 -1
- package/bin/src/commands/agent.js +13 -223
- package/bin/src/commands/agent.js.map +1 -1
- package/bin/src/commands/plan.d.ts.map +1 -1
- package/bin/src/commands/plan.js +45 -18
- package/bin/src/commands/plan.js.map +1 -1
- package/bin/src/engine/actionVerifier.d.ts.map +1 -1
- package/bin/src/engine/actionVerifier.js +61 -9
- package/bin/src/engine/actionVerifier.js.map +1 -1
- package/bin/src/engine/agentEngine.d.ts.map +1 -1
- package/bin/src/engine/agentEngine.js +459 -547
- package/bin/src/engine/agentEngine.js.map +1 -1
- package/bin/src/engine/agentState.d.ts.map +1 -1
- package/bin/src/engine/agentState.js +0 -64
- package/bin/src/engine/agentState.js.map +1 -1
- package/bin/src/engine/browser/browserController.d.ts.map +1 -1
- package/bin/src/engine/browser/browserController.js +8 -0
- package/bin/src/engine/browser/browserController.js.map +1 -1
- package/bin/src/engine/computer/computerController.d.ts.map +1 -1
- package/bin/src/engine/computer/computerController.js +479 -269
- package/bin/src/engine/computer/computerController.js.map +1 -1
- package/bin/src/engine/computer/index.d.ts.map +1 -1
- package/bin/src/engine/computer/index.js +1 -0
- package/bin/src/engine/computer/index.js.map +1 -1
- package/bin/src/engine/computer/mockComputer.d.ts.map +1 -0
- package/bin/src/engine/computer/mockComputer.js +164 -0
- package/bin/src/engine/computer/mockComputer.js.map +1 -0
- package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -0
- package/bin/src/engine/runtime/agentRuntime.js +829 -0
- package/bin/src/engine/runtime/agentRuntime.js.map +1 -0
- package/bin/src/engine/runtime/index.d.ts.map +1 -0
- package/bin/src/engine/runtime/index.js +2 -0
- package/bin/src/engine/runtime/index.js.map +1 -0
- package/bin/src/engine/safety.d.ts.map +1 -1
- package/bin/src/engine/safety.js +49 -25
- package/bin/src/engine/safety.js.map +1 -1
- package/bin/src/engine/toolRegistry.d.ts.map +1 -1
- package/bin/src/engine/toolRegistry.js +162 -81
- package/bin/src/engine/toolRegistry.js.map +1 -1
- package/bin/src/engine/voiceEngine.d.ts.map +1 -1
- package/bin/src/engine/voiceEngine.js +0 -1
- package/bin/src/engine/voiceEngine.js.map +1 -1
- package/bin/src/engine/workspace/agentWorkspace.d.ts.map +1 -0
- package/bin/src/engine/workspace/agentWorkspace.js +484 -0
- package/bin/src/engine/workspace/agentWorkspace.js.map +1 -0
- package/bin/src/engine/workspace/index.d.ts.map +1 -0
- package/bin/src/engine/workspace/index.js +2 -0
- package/bin/src/engine/workspace/index.js.map +1 -0
- package/bin/src/index.js +4 -5
- package/bin/src/index.js.map +1 -1
- package/bin/src/server/server.d.ts.map +1 -1
- package/bin/src/server/server.js +25 -4
- package/bin/src/server/server.js.map +1 -1
- package/bin/src/utils/auth.d.ts.map +1 -1
- package/bin/src/utils/auth.js +67 -49
- package/bin/src/utils/auth.js.map +1 -1
- package/bin/src/utils/branding.js +1 -1
- package/bin/src/utils/payment.d.ts.map +1 -1
- package/bin/src/utils/payment.js +15 -1
- package/bin/src/utils/payment.js.map +1 -1
- package/bin/src/utils/pricing.js +1 -1
- package/bin/src/utils/pricing.js.map +1 -1
- package/bin/src/utils/razorpay.d.ts.map +1 -1
- package/bin/src/utils/razorpay.js +10 -2
- package/bin/src/utils/razorpay.js.map +1 -1
- package/downloads/invoice-2026-03.pdf +1 -0
- package/package.json +1 -1
- package/web/app.js +54 -135
- package/web/styles.css +2 -2
- package/bin/src/engine/appControl.d.ts.map +0 -1
- package/bin/src/engine/appControl.js +0 -2320
- package/bin/src/engine/appControl.js.map +0 -1
- package/bin/src/engine/liveScreenEngine.d.ts.map +0 -1
- package/bin/src/engine/liveScreenEngine.js +0 -421
- package/bin/src/engine/liveScreenEngine.js.map +0 -1
- package/bin/src/engine/scripts/winCapture.ps1 +0 -248
- package/downloads/INVOICE_2026_ACME.pdf +0 -1
|
@@ -1,295 +1,505 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
export class
|
|
7
|
-
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { exec } from 'child_process';
|
|
4
|
+
import { promisify } from 'util';
|
|
5
|
+
const execAsync = promisify(exec);
|
|
6
|
+
export class SystemComputerController {
|
|
7
|
+
workspaceDir;
|
|
8
|
+
screenshotsDir;
|
|
9
|
+
expectedApplication = null;
|
|
10
|
+
activeWindow = null;
|
|
11
|
+
constructor(workspaceDir = process.cwd()) {
|
|
12
|
+
this.workspaceDir = workspaceDir;
|
|
13
|
+
this.screenshotsDir = path.join(this.workspaceDir, '.ft', 'screenshots');
|
|
8
14
|
try {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
output: `Captured screen to ${shotPath}`,
|
|
13
|
-
details: { path: shotPath },
|
|
14
|
-
};
|
|
15
|
-
}
|
|
16
|
-
catch (err) {
|
|
17
|
-
return {
|
|
18
|
-
success: false,
|
|
19
|
-
output: `Failed to capture screen: ${err.message}`,
|
|
20
|
-
};
|
|
15
|
+
if (!fs.existsSync(this.screenshotsDir)) {
|
|
16
|
+
fs.mkdirSync(this.screenshotsDir, { recursive: true });
|
|
17
|
+
}
|
|
21
18
|
}
|
|
19
|
+
catch { }
|
|
22
20
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
return {
|
|
26
|
-
success: res.success,
|
|
27
|
-
output: res.output,
|
|
28
|
-
details: res.details,
|
|
29
|
-
};
|
|
30
|
-
}
|
|
31
|
-
async double_click(x, y) {
|
|
32
|
-
const res = await executeInApp('desktop', 'click_app', { x, y, double: true });
|
|
33
|
-
return {
|
|
34
|
-
success: res.success,
|
|
35
|
-
output: res.output,
|
|
36
|
-
details: res.details,
|
|
37
|
-
};
|
|
38
|
-
}
|
|
39
|
-
async type(text) {
|
|
40
|
-
const res = await executeInApp('desktop', 'type_text', { text });
|
|
41
|
-
return {
|
|
42
|
-
success: res.success,
|
|
43
|
-
output: res.output,
|
|
44
|
-
details: res.details,
|
|
45
|
-
};
|
|
46
|
-
}
|
|
47
|
-
async key_press(key) {
|
|
48
|
-
const res = await executeInApp('desktop', 'send_keys', { key });
|
|
49
|
-
return {
|
|
50
|
-
success: res.success,
|
|
51
|
-
output: res.output,
|
|
52
|
-
details: res.details,
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
async hotkey(keys) {
|
|
56
|
-
const keyCombo = Array.isArray(keys) ? keys.join('+') : keys;
|
|
57
|
-
const res = await executeInApp('desktop', 'key_combo', { keyCombo });
|
|
58
|
-
return {
|
|
59
|
-
success: res.success,
|
|
60
|
-
output: res.output,
|
|
61
|
-
details: res.details,
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
|
-
async scroll(direction, amount = 4) {
|
|
65
|
-
const res = await executeInApp('desktop', 'scroll', { direction, amount });
|
|
66
|
-
return {
|
|
67
|
-
success: res.success,
|
|
68
|
-
output: res.output,
|
|
69
|
-
details: res.details,
|
|
70
|
-
};
|
|
71
|
-
}
|
|
72
|
-
async move_mouse(x, y) {
|
|
73
|
-
const res = await executeInApp('desktop', 'move_mouse', { x, y });
|
|
74
|
-
return {
|
|
75
|
-
success: res.success,
|
|
76
|
-
output: res.output,
|
|
77
|
-
details: res.details,
|
|
78
|
-
};
|
|
79
|
-
}
|
|
80
|
-
async drag(start, end) {
|
|
81
|
-
const res = await executeInApp('desktop', 'drag', { startX: start.x, startY: start.y, endX: end.x, endY: end.y });
|
|
82
|
-
return {
|
|
83
|
-
success: res.success,
|
|
84
|
-
output: res.output,
|
|
85
|
-
details: res.details,
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
async wait(durationMs) {
|
|
89
|
-
await new Promise((r) => setTimeout(r, Math.min(10000, durationMs)));
|
|
90
|
-
return {
|
|
91
|
-
success: true,
|
|
92
|
-
output: `Waited for ${durationMs}ms`,
|
|
93
|
-
details: { durationMs },
|
|
94
|
-
};
|
|
21
|
+
setExpectedApplication(app) {
|
|
22
|
+
this.expectedApplication = app;
|
|
95
23
|
}
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
return {
|
|
99
|
-
success: res.success,
|
|
100
|
-
output: res.output,
|
|
101
|
-
details: res.details,
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
async close_application(name) {
|
|
105
|
-
const res = await executeInApp(name, 'close_app');
|
|
106
|
-
return {
|
|
107
|
-
success: res.success,
|
|
108
|
-
output: res.output,
|
|
109
|
-
details: res.details,
|
|
110
|
-
};
|
|
24
|
+
getExpectedApplication() {
|
|
25
|
+
return this.expectedApplication;
|
|
111
26
|
}
|
|
112
|
-
async
|
|
113
|
-
const
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
output: res.output,
|
|
117
|
-
details: res.details,
|
|
118
|
-
};
|
|
119
|
-
}
|
|
120
|
-
async inspect_page() {
|
|
121
|
-
const res = await executeInApp('desktop', 'analyze_screen');
|
|
122
|
-
return {
|
|
123
|
-
success: res.success,
|
|
124
|
-
output: res.output,
|
|
125
|
-
details: res.details,
|
|
126
|
-
};
|
|
127
|
-
}
|
|
128
|
-
async get_active_window() {
|
|
129
|
-
const res = await getScreenInfo();
|
|
130
|
-
return {
|
|
131
|
-
success: res.success,
|
|
132
|
-
output: res.output,
|
|
133
|
-
details: res.details,
|
|
134
|
-
};
|
|
27
|
+
async runPowerShell(script) {
|
|
28
|
+
const encoded = Buffer.from(script, 'utf16le').toString('base64');
|
|
29
|
+
const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand ${encoded}`);
|
|
30
|
+
return (stdout || '').trim();
|
|
135
31
|
}
|
|
136
|
-
}
|
|
137
|
-
/**
|
|
138
|
-
* MockComputerController: Fast, deterministic in-memory computer controller
|
|
139
|
-
* for automated unit tests, CI pipelines, and desktop simulations.
|
|
140
|
-
*/
|
|
141
|
-
export class MockComputerController {
|
|
142
|
-
cursor = { x: 0, y: 0 };
|
|
143
|
-
activeApp = null;
|
|
144
|
-
openApps = new Set();
|
|
145
|
-
typedHistory = [];
|
|
146
|
-
actionLog = [];
|
|
147
32
|
async screenshot() {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
33
|
+
const timestamp = new Date().toISOString();
|
|
34
|
+
const filename = `screen_${Date.now()}.png`;
|
|
35
|
+
const filePath = path.join(this.screenshotsDir, filename);
|
|
36
|
+
if (process.platform === 'win32') {
|
|
37
|
+
try {
|
|
38
|
+
const psScript = `
|
|
39
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
40
|
+
Add-Type -AssemblyName System.Drawing
|
|
41
|
+
$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
|
|
42
|
+
$bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
|
|
43
|
+
$graphics = [System.Drawing.Graphics]::FromImage($bmp)
|
|
44
|
+
$graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
|
|
45
|
+
$bmp.Save('${filePath.replace(/'/g, "''")}', [System.Drawing.Imaging.ImageFormat]::Png)
|
|
46
|
+
$graphics.Dispose()
|
|
47
|
+
$bmp.Dispose()
|
|
48
|
+
Write-Output "$($bounds.Width)x$($bounds.Height)"
|
|
49
|
+
`.trim();
|
|
50
|
+
const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
|
|
51
|
+
const dims = stdout.trim().split('x');
|
|
52
|
+
const width = parseInt(dims[0] || '1920', 10);
|
|
53
|
+
const height = parseInt(dims[1] || '1080', 10);
|
|
54
|
+
const ui = await this.inspectUI();
|
|
55
|
+
return {
|
|
56
|
+
path: filePath,
|
|
57
|
+
width,
|
|
58
|
+
height,
|
|
59
|
+
activeWindow: ui.activeWindow,
|
|
60
|
+
timestamp,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
return {
|
|
65
|
+
width: 1920,
|
|
66
|
+
height: 1080,
|
|
67
|
+
activeWindow: 'Desktop',
|
|
68
|
+
timestamp,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
else if (process.platform === 'darwin') {
|
|
73
|
+
try {
|
|
74
|
+
await execAsync(`screencapture -x "${filePath}"`);
|
|
75
|
+
return { path: filePath, width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
return { width: 1920, height: 1080, timestamp };
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
// Linux fallback (scrot or import)
|
|
83
|
+
try {
|
|
84
|
+
await execAsync(`scrot "${filePath}"`);
|
|
85
|
+
return { path: filePath, width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
return { width: 1920, height: 1080, timestamp };
|
|
89
|
+
}
|
|
90
|
+
}
|
|
167
91
|
}
|
|
168
|
-
async
|
|
169
|
-
|
|
92
|
+
async click(x, y, options) {
|
|
93
|
+
const btn = options?.button || 'left';
|
|
94
|
+
const count = options?.clicks || 1;
|
|
95
|
+
if (process.platform === 'win32') {
|
|
96
|
+
try {
|
|
97
|
+
const psScript = `
|
|
98
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
99
|
+
[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${Math.round(x)}, ${Math.round(y)})
|
|
100
|
+
$src = @"
|
|
101
|
+
using System;
|
|
102
|
+
using System.Runtime.InteropServices;
|
|
103
|
+
public class WinMouse {
|
|
104
|
+
[DllImport("user32.dll")]
|
|
105
|
+
public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
|
|
106
|
+
}
|
|
107
|
+
"@
|
|
108
|
+
Add-Type -TypeDefinition $src -ErrorAction SilentlyContinue
|
|
109
|
+
# MOUSEEVENTF_LEFTDOWN = 0x0002, MOUSEEVENTF_LEFTUP = 0x0004
|
|
110
|
+
# MOUSEEVENTF_RIGHTDOWN = 0x0008, MOUSEEVENTF_RIGHTUP = 0x0010
|
|
111
|
+
${btn === 'right'
|
|
112
|
+
? `[WinMouse]::mouse_event(0x0008, 0, 0, 0, 0); Start-Sleep -Milliseconds 50; [WinMouse]::mouse_event(0x0010, 0, 0, 0, 0)`
|
|
113
|
+
: `for ($i=0; $i -lt ${count}; $i++) { [WinMouse]::mouse_event(0x0002, 0, 0, 0, 0); Start-Sleep -Milliseconds 40; [WinMouse]::mouse_event(0x0004, 0, 0, 0, 0); Start-Sleep -Milliseconds 50 }`}
|
|
114
|
+
`.trim();
|
|
115
|
+
await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
|
|
116
|
+
return { success: true, output: `Clicked ${btn} button at (${x}, ${y})` };
|
|
117
|
+
}
|
|
118
|
+
catch (err) {
|
|
119
|
+
return { success: false, output: `Click failed at (${x}, ${y}): ${err?.message || String(err)}` };
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return { success: true, output: `Simulated click at (${x}, ${y}) on platform ${process.platform}` };
|
|
170
123
|
}
|
|
171
|
-
async
|
|
172
|
-
return {
|
|
173
|
-
format: 'base64',
|
|
174
|
-
width: 1920,
|
|
175
|
-
height: 1080,
|
|
176
|
-
data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==',
|
|
177
|
-
};
|
|
178
|
-
}
|
|
179
|
-
async double_click(x, y) {
|
|
180
|
-
this.cursor = { x, y };
|
|
181
|
-
this.actionLog.push({ action: 'double_click', params: { x, y } });
|
|
182
|
-
return {
|
|
183
|
-
success: true,
|
|
184
|
-
output: `Mock double-clicked at (${x}, ${y}).`,
|
|
185
|
-
details: { x, y },
|
|
186
|
-
};
|
|
124
|
+
async doubleClick(x, y) {
|
|
125
|
+
return this.click(x, y, { button: 'left', clicks: 2 });
|
|
187
126
|
}
|
|
188
127
|
async type(text) {
|
|
189
|
-
|
|
190
|
-
|
|
128
|
+
if (!text)
|
|
129
|
+
return { success: true, output: 'Typed empty string', action: 'type' };
|
|
130
|
+
if (process.platform === 'win32') {
|
|
131
|
+
try {
|
|
132
|
+
const targetApp = this.expectedApplication || 'Notepad';
|
|
133
|
+
const escaped = text
|
|
134
|
+
.replace(/\{/g, '{{}')
|
|
135
|
+
.replace(/\}/g, '{}}')
|
|
136
|
+
.replace(/\+/g, '{+}')
|
|
137
|
+
.replace(/\^/g, '{^}')
|
|
138
|
+
.replace(/%/g, '{%}')
|
|
139
|
+
.replace(/~/g, '{~}')
|
|
140
|
+
.replace(/\(/g, '{(}')
|
|
141
|
+
.replace(/\)/g, '{)}')
|
|
142
|
+
.replace(/\[/g, '{[}')
|
|
143
|
+
.replace(/\]/g, '{]}');
|
|
144
|
+
const psScript = `
|
|
145
|
+
$wshell = New-Object -ComObject Wscript.Shell
|
|
146
|
+
$null = $wshell.AppActivate('${targetApp}')
|
|
147
|
+
Start-Sleep -Milliseconds 250
|
|
148
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
149
|
+
[System.Windows.Forms.SendKeys]::SendWait("${escaped.replace(/"/g, '`"')}")
|
|
150
|
+
`.trim();
|
|
151
|
+
await this.runPowerShell(psScript);
|
|
152
|
+
this.activeWindow = targetApp;
|
|
153
|
+
return {
|
|
154
|
+
success: true,
|
|
155
|
+
output: `Typed text (${text.length} chars) into active application "${targetApp}"`,
|
|
156
|
+
action: 'type',
|
|
157
|
+
activeWindow: targetApp,
|
|
158
|
+
expectedWindow: targetApp,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
catch (err) {
|
|
162
|
+
return {
|
|
163
|
+
success: false,
|
|
164
|
+
output: `Type error on active window "${this.expectedApplication || 'foreground'}": ${err?.message || String(err)}`,
|
|
165
|
+
action: 'type',
|
|
166
|
+
error: err?.message || String(err),
|
|
167
|
+
recoverable: true,
|
|
168
|
+
expectedWindow: this.expectedApplication || undefined,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
}
|
|
191
172
|
return {
|
|
192
173
|
success: true,
|
|
193
|
-
output: `
|
|
194
|
-
|
|
174
|
+
output: `Simulated type (${text.length} chars)`,
|
|
175
|
+
action: 'type',
|
|
176
|
+
activeWindow: this.expectedApplication || 'Simulated Window',
|
|
195
177
|
};
|
|
196
178
|
}
|
|
197
|
-
async
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
179
|
+
async keyPress(key) {
|
|
180
|
+
const keyMap = {
|
|
181
|
+
enter: '{ENTER}',
|
|
182
|
+
return: '{ENTER}',
|
|
183
|
+
tab: '{TAB}',
|
|
184
|
+
escape: '{ESC}',
|
|
185
|
+
esc: '{ESC}',
|
|
186
|
+
backspace: '{BACKSPACE}',
|
|
187
|
+
delete: '{DEL}',
|
|
188
|
+
up: '{UP}',
|
|
189
|
+
down: '{DOWN}',
|
|
190
|
+
left: '{LEFT}',
|
|
191
|
+
right: '{RIGHT}',
|
|
192
|
+
space: ' ',
|
|
193
|
+
home: '{HOME}',
|
|
194
|
+
end: '{END}',
|
|
195
|
+
pageup: '{PGUP}',
|
|
196
|
+
pagedown: '{PGDN}',
|
|
203
197
|
};
|
|
198
|
+
const mapped = keyMap[key.toLowerCase()] || key;
|
|
199
|
+
if (process.platform === 'win32') {
|
|
200
|
+
try {
|
|
201
|
+
const targetApp = this.expectedApplication || '';
|
|
202
|
+
const psScript = `
|
|
203
|
+
$wshell = New-Object -ComObject Wscript.Shell
|
|
204
|
+
${targetApp ? `$null = $wshell.AppActivate('${targetApp}')\nStart-Sleep -Milliseconds 150` : ''}
|
|
205
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
206
|
+
[System.Windows.Forms.SendKeys]::SendWait("${mapped}")
|
|
207
|
+
`.trim();
|
|
208
|
+
await this.runPowerShell(psScript);
|
|
209
|
+
return {
|
|
210
|
+
success: true,
|
|
211
|
+
output: `Pressed key: ${key}`,
|
|
212
|
+
action: 'key_press',
|
|
213
|
+
activeWindow: targetApp || undefined,
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
return {
|
|
218
|
+
success: false,
|
|
219
|
+
output: `KeyPress error: ${err?.message || String(err)}`,
|
|
220
|
+
action: 'key_press',
|
|
221
|
+
error: err?.message || String(err),
|
|
222
|
+
recoverable: true,
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
return { success: true, output: `Simulated key press: ${key}`, action: 'key_press' };
|
|
204
227
|
}
|
|
205
228
|
async hotkey(keys) {
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
229
|
+
if (!keys || keys.length === 0)
|
|
230
|
+
return { success: false, output: 'No keys provided for hotkey', action: 'hotkey' };
|
|
231
|
+
if (process.platform === 'win32') {
|
|
232
|
+
try {
|
|
233
|
+
const targetApp = this.expectedApplication || '';
|
|
234
|
+
let prefix = '';
|
|
235
|
+
let mainKey = '';
|
|
236
|
+
for (const k of keys) {
|
|
237
|
+
const lk = k.toLowerCase();
|
|
238
|
+
if (lk === 'ctrl' || lk === 'control')
|
|
239
|
+
prefix += '^';
|
|
240
|
+
else if (lk === 'alt')
|
|
241
|
+
prefix += '%';
|
|
242
|
+
else if (lk === 'shift')
|
|
243
|
+
prefix += '+';
|
|
244
|
+
else
|
|
245
|
+
mainKey = k;
|
|
246
|
+
}
|
|
247
|
+
const sendPattern = `${prefix}${mainKey.toLowerCase()}`;
|
|
248
|
+
const psScript = `
|
|
249
|
+
$wshell = New-Object -ComObject Wscript.Shell
|
|
250
|
+
${targetApp ? `$null = $wshell.AppActivate('${targetApp}')\nStart-Sleep -Milliseconds 150` : ''}
|
|
251
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
252
|
+
[System.Windows.Forms.SendKeys]::SendWait("${sendPattern}")
|
|
253
|
+
`.trim();
|
|
254
|
+
await this.runPowerShell(psScript);
|
|
255
|
+
return {
|
|
256
|
+
success: true,
|
|
257
|
+
output: `Executed hotkey: ${keys.join('+')} on "${targetApp || 'active window'}"`,
|
|
258
|
+
action: 'hotkey',
|
|
259
|
+
activeWindow: targetApp || undefined,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
catch (err) {
|
|
263
|
+
return {
|
|
264
|
+
success: false,
|
|
265
|
+
output: `Hotkey error: ${err?.message || String(err)}`,
|
|
266
|
+
action: 'hotkey',
|
|
267
|
+
error: err?.message || String(err),
|
|
268
|
+
recoverable: true,
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return { success: true, output: `Simulated hotkey: ${keys.join('+')}`, action: 'hotkey' };
|
|
273
|
+
}
|
|
274
|
+
async scroll(deltaX, deltaY) {
|
|
275
|
+
if (process.platform === 'win32') {
|
|
276
|
+
try {
|
|
277
|
+
const psScript = `
|
|
278
|
+
$src = @"
|
|
279
|
+
using System;
|
|
280
|
+
using System.Runtime.InteropServices;
|
|
281
|
+
public class WinMouseScroll {
|
|
282
|
+
[DllImport("user32.dll")]
|
|
283
|
+
public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
|
|
284
|
+
}
|
|
285
|
+
"@
|
|
286
|
+
Add-Type -TypeDefinition $src -ErrorAction SilentlyContinue
|
|
287
|
+
# MOUSEEVENTF_WHEEL = 0x0800
|
|
288
|
+
[WinMouseScroll]::mouse_event(0x0800, 0, 0, [uint32]${Math.round(-deltaY * 120)}, 0)
|
|
289
|
+
`.trim();
|
|
290
|
+
await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
|
|
291
|
+
return { success: true, output: `Scrolled deltaY=${deltaY}, deltaX=${deltaX}` };
|
|
292
|
+
}
|
|
293
|
+
catch (err) {
|
|
294
|
+
return { success: false, output: `Scroll error: ${err?.message || String(err)}` };
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return { success: true, output: `Simulated scroll deltaY=${deltaY}` };
|
|
298
|
+
}
|
|
299
|
+
async mouseMove(x, y) {
|
|
300
|
+
if (process.platform === 'win32') {
|
|
301
|
+
try {
|
|
302
|
+
const psScript = `
|
|
303
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
304
|
+
[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${Math.round(x)}, ${Math.round(y)})
|
|
305
|
+
`.trim();
|
|
306
|
+
await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
|
|
307
|
+
return { success: true, output: `Moved mouse cursor to (${x}, ${y})` };
|
|
308
|
+
}
|
|
309
|
+
catch (err) {
|
|
310
|
+
return { success: false, output: `MouseMove error: ${err?.message || String(err)}` };
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
return { success: true, output: `Simulated mouse move to (${x}, ${y})` };
|
|
314
|
+
}
|
|
315
|
+
async drag(fromX, fromY, toX, toY) {
|
|
316
|
+
await this.mouseMove(fromX, fromY);
|
|
317
|
+
if (process.platform === 'win32') {
|
|
318
|
+
try {
|
|
319
|
+
const psScript = `
|
|
320
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
321
|
+
$src = @"
|
|
322
|
+
using System;
|
|
323
|
+
using System.Runtime.InteropServices;
|
|
324
|
+
public class WinMouseDrag {
|
|
325
|
+
[DllImport("user32.dll")]
|
|
326
|
+
public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
|
|
327
|
+
}
|
|
328
|
+
"@
|
|
329
|
+
Add-Type -TypeDefinition $src -ErrorAction SilentlyContinue
|
|
330
|
+
[WinMouseDrag]::mouse_event(0x0002, 0, 0, 0, 0)
|
|
331
|
+
[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${Math.round(toX)}, ${Math.round(toY)})
|
|
332
|
+
Start-Sleep -Milliseconds 100
|
|
333
|
+
[WinMouseDrag]::mouse_event(0x0004, 0, 0, 0, 0)
|
|
334
|
+
`.trim();
|
|
335
|
+
await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
|
|
336
|
+
return { success: true, output: `Dragged from (${fromX}, ${fromY}) to (${toX}, ${toY})` };
|
|
337
|
+
}
|
|
338
|
+
catch (err) {
|
|
339
|
+
return { success: false, output: `Drag error: ${err?.message || String(err)}` };
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
return { success: true, output: `Simulated drag from (${fromX}, ${fromY}) to (${toX}, ${toY})` };
|
|
239
343
|
}
|
|
240
|
-
async wait(
|
|
241
|
-
|
|
242
|
-
return {
|
|
243
|
-
success: true,
|
|
244
|
-
output: `Mock waited ${durationMs}ms.`,
|
|
245
|
-
details: { durationMs },
|
|
246
|
-
};
|
|
344
|
+
async wait(ms) {
|
|
345
|
+
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
|
|
247
346
|
}
|
|
248
|
-
async
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
347
|
+
async openApplication(appName, args = []) {
|
|
348
|
+
const appLower = appName.toLowerCase().trim();
|
|
349
|
+
const argsStr = args.join(' ');
|
|
350
|
+
try {
|
|
351
|
+
if (process.platform === 'win32') {
|
|
352
|
+
if (appLower === 'code' || appLower === 'vscode') {
|
|
353
|
+
await execAsync(`start code ${argsStr}`);
|
|
354
|
+
this.expectedApplication = 'Visual Studio Code';
|
|
355
|
+
}
|
|
356
|
+
else if (appLower === 'explorer' || appLower === 'file explorer') {
|
|
357
|
+
await execAsync(`start explorer ${argsStr || '.'}`);
|
|
358
|
+
this.expectedApplication = 'File Explorer';
|
|
359
|
+
}
|
|
360
|
+
else if (appLower === 'notepad') {
|
|
361
|
+
await execAsync(`start notepad ${argsStr}`);
|
|
362
|
+
this.expectedApplication = 'Notepad';
|
|
363
|
+
}
|
|
364
|
+
else if (appLower === 'terminal' || appLower === 'cmd' || appLower === 'powershell') {
|
|
365
|
+
await execAsync(`start powershell ${argsStr}`);
|
|
366
|
+
this.expectedApplication = 'PowerShell';
|
|
367
|
+
}
|
|
368
|
+
else {
|
|
369
|
+
await execAsync(`start "" "${appName}" ${argsStr}`);
|
|
370
|
+
this.expectedApplication = appName;
|
|
371
|
+
}
|
|
372
|
+
await this.wait(350);
|
|
373
|
+
if (this.expectedApplication) {
|
|
374
|
+
await this.switchWindow(this.expectedApplication);
|
|
375
|
+
}
|
|
376
|
+
return {
|
|
377
|
+
success: true,
|
|
378
|
+
output: `Launched and focused application: "${this.expectedApplication || appName}"`,
|
|
379
|
+
action: 'open_app',
|
|
380
|
+
activeWindow: this.expectedApplication || appName,
|
|
381
|
+
expectedWindow: this.expectedApplication || appName,
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
else if (process.platform === 'darwin') {
|
|
385
|
+
await execAsync(`open -a "${appName}" ${argsStr}`);
|
|
386
|
+
this.expectedApplication = appName;
|
|
387
|
+
}
|
|
388
|
+
else {
|
|
389
|
+
await execAsync(`xdg-open "${appName}" ${argsStr} &`);
|
|
390
|
+
this.expectedApplication = appName;
|
|
391
|
+
}
|
|
392
|
+
return {
|
|
393
|
+
success: true,
|
|
394
|
+
output: `Launched application: "${appName}"`,
|
|
395
|
+
action: 'open_app',
|
|
396
|
+
activeWindow: appName,
|
|
397
|
+
expectedWindow: appName,
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
catch (err) {
|
|
401
|
+
return {
|
|
402
|
+
success: false,
|
|
403
|
+
output: `Failed to launch "${appName}": ${err?.message || String(err)}`,
|
|
404
|
+
action: 'open_app',
|
|
405
|
+
error: err?.message || String(err),
|
|
406
|
+
recoverable: true,
|
|
407
|
+
};
|
|
408
|
+
}
|
|
268
409
|
}
|
|
269
|
-
async
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
410
|
+
async closeApplication(appName) {
|
|
411
|
+
try {
|
|
412
|
+
if (process.platform === 'win32') {
|
|
413
|
+
const cleanName = appName.replace(/\.exe$/i, '');
|
|
414
|
+
await execAsync(`powershell -NoProfile -Command "Stop-Process -Name '${cleanName}' -Force -ErrorAction SilentlyContinue"`);
|
|
415
|
+
}
|
|
416
|
+
else {
|
|
417
|
+
await execAsync(`pkill -f "${appName}"`);
|
|
418
|
+
}
|
|
419
|
+
if (this.expectedApplication && this.expectedApplication.toLowerCase().includes(appName.toLowerCase())) {
|
|
420
|
+
this.expectedApplication = null;
|
|
421
|
+
}
|
|
422
|
+
return { success: true, output: `Closed application: "${appName}"`, action: 'close_app' };
|
|
423
|
+
}
|
|
424
|
+
catch (err) {
|
|
425
|
+
return {
|
|
426
|
+
success: false,
|
|
427
|
+
output: `Failed to close "${appName}": ${err?.message || String(err)}`,
|
|
428
|
+
action: 'close_app',
|
|
429
|
+
error: err?.message || String(err),
|
|
430
|
+
};
|
|
431
|
+
}
|
|
278
432
|
}
|
|
279
|
-
async
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
433
|
+
async switchWindow(windowTitleOrApp) {
|
|
434
|
+
if (process.platform === 'win32') {
|
|
435
|
+
try {
|
|
436
|
+
const psScript = `
|
|
437
|
+
$wshell = New-Object -ComObject Wscript.Shell
|
|
438
|
+
$result = $wshell.AppActivate("${windowTitleOrApp.replace(/"/g, '`"')}")
|
|
439
|
+
if (-not $result) {
|
|
440
|
+
$p = Get-Process | Where-Object { $_.MainWindowTitle -like "*${windowTitleOrApp.replace(/"/g, '`"')}*" -or $_.ProcessName -like "*${windowTitleOrApp.replace(/"/g, '`"')}*" } | Select-Object -First 1
|
|
441
|
+
if ($p) {
|
|
442
|
+
$null = $wshell.AppActivate($p.Id)
|
|
443
|
+
$result = $true
|
|
286
444
|
}
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
445
|
+
}
|
|
446
|
+
Start-Sleep -Milliseconds 150
|
|
447
|
+
if ($result) { Write-Output "OK" } else { Write-Output "NOT_FOUND" }
|
|
448
|
+
`.trim();
|
|
449
|
+
const stdout = await this.runPowerShell(psScript);
|
|
450
|
+
this.activeWindow = windowTitleOrApp;
|
|
451
|
+
this.expectedApplication = windowTitleOrApp;
|
|
452
|
+
if (stdout.includes('OK')) {
|
|
453
|
+
return {
|
|
454
|
+
success: true,
|
|
455
|
+
output: `Focused window matching: "${windowTitleOrApp}"`,
|
|
456
|
+
action: 'switch_window',
|
|
457
|
+
activeWindow: windowTitleOrApp,
|
|
458
|
+
expectedWindow: windowTitleOrApp,
|
|
459
|
+
};
|
|
460
|
+
}
|
|
461
|
+
return {
|
|
462
|
+
success: false,
|
|
463
|
+
output: `Could not find or focus window: "${windowTitleOrApp}". Try computer_inspect_ui to list open windows.`,
|
|
464
|
+
action: 'switch_window',
|
|
465
|
+
recoverable: true,
|
|
466
|
+
};
|
|
467
|
+
}
|
|
468
|
+
catch (err) {
|
|
469
|
+
return {
|
|
470
|
+
success: false,
|
|
471
|
+
output: `Window switch error for "${windowTitleOrApp}": ${err?.message || String(err)}`,
|
|
472
|
+
action: 'switch_window',
|
|
473
|
+
error: err?.message || String(err),
|
|
474
|
+
recoverable: true,
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
this.activeWindow = windowTitleOrApp;
|
|
479
|
+
return { success: true, output: `Switched window to: "${windowTitleOrApp}"`, action: 'switch_window' };
|
|
480
|
+
}
|
|
481
|
+
async inspectUI() {
|
|
482
|
+
if (process.platform === 'win32') {
|
|
483
|
+
try {
|
|
484
|
+
const psScript = `
|
|
485
|
+
Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle.Trim().Length -gt 0 } | Select-Object -ExpandProperty MainWindowTitle
|
|
486
|
+
`.trim();
|
|
487
|
+
const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
|
|
488
|
+
const titles = stdout
|
|
489
|
+
.split('\n')
|
|
490
|
+
.map((t) => t.trim())
|
|
491
|
+
.filter((t) => t.length > 0);
|
|
492
|
+
const active = titles.length > 0 ? titles[0] : 'Desktop';
|
|
493
|
+
return {
|
|
494
|
+
activeWindow: active,
|
|
495
|
+
windows: titles,
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
catch {
|
|
499
|
+
return { activeWindow: 'Desktop', windows: ['Desktop'] };
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
return { activeWindow: 'Desktop', windows: ['Desktop', 'Terminal'] };
|
|
293
503
|
}
|
|
294
504
|
}
|
|
295
505
|
//# sourceMappingURL=computerController.js.map
|