ft-scout 9.0.2 → 9.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/.firebase/hosting.d2Vi.cache +7 -7
  2. package/README.md +76 -83
  3. package/bin/src/commands/agent.d.ts.map +1 -1
  4. package/bin/src/commands/agent.js +13 -223
  5. package/bin/src/commands/agent.js.map +1 -1
  6. package/bin/src/commands/plan.d.ts.map +1 -1
  7. package/bin/src/commands/plan.js +45 -18
  8. package/bin/src/commands/plan.js.map +1 -1
  9. package/bin/src/engine/actionVerifier.d.ts.map +1 -1
  10. package/bin/src/engine/actionVerifier.js +61 -9
  11. package/bin/src/engine/actionVerifier.js.map +1 -1
  12. package/bin/src/engine/agentEngine.d.ts.map +1 -1
  13. package/bin/src/engine/agentEngine.js +459 -547
  14. package/bin/src/engine/agentEngine.js.map +1 -1
  15. package/bin/src/engine/agentState.d.ts.map +1 -1
  16. package/bin/src/engine/agentState.js +0 -64
  17. package/bin/src/engine/agentState.js.map +1 -1
  18. package/bin/src/engine/browser/browserController.d.ts.map +1 -1
  19. package/bin/src/engine/browser/browserController.js +8 -0
  20. package/bin/src/engine/browser/browserController.js.map +1 -1
  21. package/bin/src/engine/computer/computerController.d.ts.map +1 -1
  22. package/bin/src/engine/computer/computerController.js +479 -269
  23. package/bin/src/engine/computer/computerController.js.map +1 -1
  24. package/bin/src/engine/computer/index.d.ts.map +1 -1
  25. package/bin/src/engine/computer/index.js +1 -0
  26. package/bin/src/engine/computer/index.js.map +1 -1
  27. package/bin/src/engine/computer/mockComputer.d.ts.map +1 -0
  28. package/bin/src/engine/computer/mockComputer.js +164 -0
  29. package/bin/src/engine/computer/mockComputer.js.map +1 -0
  30. package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -0
  31. package/bin/src/engine/runtime/agentRuntime.js +829 -0
  32. package/bin/src/engine/runtime/agentRuntime.js.map +1 -0
  33. package/bin/src/engine/runtime/index.d.ts.map +1 -0
  34. package/bin/src/engine/runtime/index.js +2 -0
  35. package/bin/src/engine/runtime/index.js.map +1 -0
  36. package/bin/src/engine/safety.d.ts.map +1 -1
  37. package/bin/src/engine/safety.js +49 -25
  38. package/bin/src/engine/safety.js.map +1 -1
  39. package/bin/src/engine/toolRegistry.d.ts.map +1 -1
  40. package/bin/src/engine/toolRegistry.js +162 -81
  41. package/bin/src/engine/toolRegistry.js.map +1 -1
  42. package/bin/src/engine/voiceEngine.d.ts.map +1 -1
  43. package/bin/src/engine/voiceEngine.js +0 -1
  44. package/bin/src/engine/voiceEngine.js.map +1 -1
  45. package/bin/src/engine/workspace/agentWorkspace.d.ts.map +1 -0
  46. package/bin/src/engine/workspace/agentWorkspace.js +484 -0
  47. package/bin/src/engine/workspace/agentWorkspace.js.map +1 -0
  48. package/bin/src/engine/workspace/index.d.ts.map +1 -0
  49. package/bin/src/engine/workspace/index.js +2 -0
  50. package/bin/src/engine/workspace/index.js.map +1 -0
  51. package/bin/src/index.js +4 -5
  52. package/bin/src/index.js.map +1 -1
  53. package/bin/src/server/server.d.ts.map +1 -1
  54. package/bin/src/server/server.js +25 -4
  55. package/bin/src/server/server.js.map +1 -1
  56. package/bin/src/utils/auth.d.ts.map +1 -1
  57. package/bin/src/utils/auth.js +67 -49
  58. package/bin/src/utils/auth.js.map +1 -1
  59. package/bin/src/utils/branding.js +1 -1
  60. package/bin/src/utils/payment.d.ts.map +1 -1
  61. package/bin/src/utils/payment.js +15 -1
  62. package/bin/src/utils/payment.js.map +1 -1
  63. package/bin/src/utils/pricing.js +1 -1
  64. package/bin/src/utils/pricing.js.map +1 -1
  65. package/bin/src/utils/razorpay.d.ts.map +1 -1
  66. package/bin/src/utils/razorpay.js +10 -2
  67. package/bin/src/utils/razorpay.js.map +1 -1
  68. package/downloads/invoice-2026-03.pdf +1 -0
  69. package/package.json +1 -1
  70. package/web/app.js +54 -135
  71. package/web/styles.css +2 -2
  72. package/bin/src/engine/appControl.d.ts.map +0 -1
  73. package/bin/src/engine/appControl.js +0 -2320
  74. package/bin/src/engine/appControl.js.map +0 -1
  75. package/bin/src/engine/liveScreenEngine.d.ts.map +0 -1
  76. package/bin/src/engine/liveScreenEngine.js +0 -421
  77. package/bin/src/engine/liveScreenEngine.js.map +0 -1
  78. package/bin/src/engine/scripts/winCapture.ps1 +0 -248
  79. package/downloads/INVOICE_2026_ACME.pdf +0 -1
@@ -1,295 +1,505 @@
1
- import { openApp, executeInApp, takeoverControl, releaseTakeover, captureScreen, analyzeScreen, getScreenInfo, } from '../appControl.js';
2
- /**
3
- * OSComputerController: Production implementation driving host desktop OS
4
- * (Windows, macOS, Linux) through Scout's platform automation engine.
5
- */
6
- export class OSComputerController {
7
- async screenshot() {
1
+ import fs from 'fs';
2
+ import path from 'path';
3
+ import { exec } from 'child_process';
4
+ import { promisify } from 'util';
5
+ const execAsync = promisify(exec);
6
+ export class SystemComputerController {
7
+ workspaceDir;
8
+ screenshotsDir;
9
+ expectedApplication = null;
10
+ activeWindow = null;
11
+ constructor(workspaceDir = process.cwd()) {
12
+ this.workspaceDir = workspaceDir;
13
+ this.screenshotsDir = path.join(this.workspaceDir, '.ft', 'screenshots');
8
14
  try {
9
- const shotPath = await captureScreen();
10
- return {
11
- success: true,
12
- output: `Captured screen to ${shotPath}`,
13
- details: { path: shotPath },
14
- };
15
- }
16
- catch (err) {
17
- return {
18
- success: false,
19
- output: `Failed to capture screen: ${err.message}`,
20
- };
15
+ if (!fs.existsSync(this.screenshotsDir)) {
16
+ fs.mkdirSync(this.screenshotsDir, { recursive: true });
17
+ }
21
18
  }
19
+ catch { }
22
20
  }
23
- async click(x, y, button = 'left') {
24
- const res = await executeInApp('desktop', 'click_app', { x, y, button });
25
- return {
26
- success: res.success,
27
- output: res.output,
28
- details: res.details,
29
- };
30
- }
31
- async double_click(x, y) {
32
- const res = await executeInApp('desktop', 'click_app', { x, y, double: true });
33
- return {
34
- success: res.success,
35
- output: res.output,
36
- details: res.details,
37
- };
38
- }
39
- async type(text) {
40
- const res = await executeInApp('desktop', 'type_text', { text });
41
- return {
42
- success: res.success,
43
- output: res.output,
44
- details: res.details,
45
- };
46
- }
47
- async key_press(key) {
48
- const res = await executeInApp('desktop', 'send_keys', { key });
49
- return {
50
- success: res.success,
51
- output: res.output,
52
- details: res.details,
53
- };
54
- }
55
- async hotkey(keys) {
56
- const keyCombo = Array.isArray(keys) ? keys.join('+') : keys;
57
- const res = await executeInApp('desktop', 'key_combo', { keyCombo });
58
- return {
59
- success: res.success,
60
- output: res.output,
61
- details: res.details,
62
- };
63
- }
64
- async scroll(direction, amount = 4) {
65
- const res = await executeInApp('desktop', 'scroll', { direction, amount });
66
- return {
67
- success: res.success,
68
- output: res.output,
69
- details: res.details,
70
- };
71
- }
72
- async move_mouse(x, y) {
73
- const res = await executeInApp('desktop', 'move_mouse', { x, y });
74
- return {
75
- success: res.success,
76
- output: res.output,
77
- details: res.details,
78
- };
79
- }
80
- async drag(start, end) {
81
- const res = await executeInApp('desktop', 'drag', { startX: start.x, startY: start.y, endX: end.x, endY: end.y });
82
- return {
83
- success: res.success,
84
- output: res.output,
85
- details: res.details,
86
- };
87
- }
88
- async wait(durationMs) {
89
- await new Promise((r) => setTimeout(r, Math.min(10000, durationMs)));
90
- return {
91
- success: true,
92
- output: `Waited for ${durationMs}ms`,
93
- details: { durationMs },
94
- };
21
+ setExpectedApplication(app) {
22
+ this.expectedApplication = app;
95
23
  }
96
- async open_application(name, target) {
97
- const res = await openApp(name, target);
98
- return {
99
- success: res.success,
100
- output: res.output,
101
- details: res.details,
102
- };
103
- }
104
- async close_application(name) {
105
- const res = await executeInApp(name, 'close_app');
106
- return {
107
- success: res.success,
108
- output: res.output,
109
- details: res.details,
110
- };
24
+ getExpectedApplication() {
25
+ return this.expectedApplication;
111
26
  }
112
- async switch_application(name) {
113
- const res = await executeInApp('desktop', 'focus_window', { app: name });
114
- return {
115
- success: res.success,
116
- output: res.output,
117
- details: res.details,
118
- };
119
- }
120
- async inspect_page() {
121
- const res = await executeInApp('desktop', 'analyze_screen');
122
- return {
123
- success: res.success,
124
- output: res.output,
125
- details: res.details,
126
- };
127
- }
128
- async get_active_window() {
129
- const res = await getScreenInfo();
130
- return {
131
- success: res.success,
132
- output: res.output,
133
- details: res.details,
134
- };
27
+ async runPowerShell(script) {
28
+ const encoded = Buffer.from(script, 'utf16le').toString('base64');
29
+ const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand ${encoded}`);
30
+ return (stdout || '').trim();
135
31
  }
136
- }
137
- /**
138
- * MockComputerController: Fast, deterministic in-memory computer controller
139
- * for automated unit tests, CI pipelines, and desktop simulations.
140
- */
141
- export class MockComputerController {
142
- cursor = { x: 0, y: 0 };
143
- activeApp = null;
144
- openApps = new Set();
145
- typedHistory = [];
146
- actionLog = [];
147
32
  async screenshot() {
148
- this.actionLog.push({ action: 'screenshot', params: {} });
149
- return {
150
- success: true,
151
- output: 'Mock screenshot captured: 1920x1080 virtual desktop.',
152
- details: { width: 1920, height: 1080, format: 'png' },
153
- };
154
- }
155
- async click(x, y, button) {
156
- const btn = typeof button === 'object' ? (button.button || 'left') : (button || 'left');
157
- this.cursor = { x, y };
158
- this.actionLog.push({ action: 'click', params: { x, y, button: btn } });
159
- return {
160
- success: true,
161
- output: `Mock clicked ${btn} button at (${x}, ${y}).`,
162
- details: { x, y, button: btn },
163
- };
164
- }
165
- async getWindowList() {
166
- return Array.from(this.openApps).length > 0 ? Array.from(this.openApps) : ['Code', 'Chrome', 'Terminal'];
33
+ const timestamp = new Date().toISOString();
34
+ const filename = `screen_${Date.now()}.png`;
35
+ const filePath = path.join(this.screenshotsDir, filename);
36
+ if (process.platform === 'win32') {
37
+ try {
38
+ const psScript = `
39
+ Add-Type -AssemblyName System.Windows.Forms
40
+ Add-Type -AssemblyName System.Drawing
41
+ $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
42
+ $bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
43
+ $graphics = [System.Drawing.Graphics]::FromImage($bmp)
44
+ $graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
45
+ $bmp.Save('${filePath.replace(/'/g, "''")}', [System.Drawing.Imaging.ImageFormat]::Png)
46
+ $graphics.Dispose()
47
+ $bmp.Dispose()
48
+ Write-Output "$($bounds.Width)x$($bounds.Height)"
49
+ `.trim();
50
+ const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
51
+ const dims = stdout.trim().split('x');
52
+ const width = parseInt(dims[0] || '1920', 10);
53
+ const height = parseInt(dims[1] || '1080', 10);
54
+ const ui = await this.inspectUI();
55
+ return {
56
+ path: filePath,
57
+ width,
58
+ height,
59
+ activeWindow: ui.activeWindow,
60
+ timestamp,
61
+ };
62
+ }
63
+ catch (err) {
64
+ return {
65
+ width: 1920,
66
+ height: 1080,
67
+ activeWindow: 'Desktop',
68
+ timestamp,
69
+ };
70
+ }
71
+ }
72
+ else if (process.platform === 'darwin') {
73
+ try {
74
+ await execAsync(`screencapture -x "${filePath}"`);
75
+ return { path: filePath, width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
76
+ }
77
+ catch {
78
+ return { width: 1920, height: 1080, timestamp };
79
+ }
80
+ }
81
+ else {
82
+ // Linux fallback (scrot or import)
83
+ try {
84
+ await execAsync(`scrot "${filePath}"`);
85
+ return { path: filePath, width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
86
+ }
87
+ catch {
88
+ return { width: 1920, height: 1080, timestamp };
89
+ }
90
+ }
167
91
  }
168
- async typeText(text) {
169
- return this.type(text);
92
+ async click(x, y, options) {
93
+ const btn = options?.button || 'left';
94
+ const count = options?.clicks || 1;
95
+ if (process.platform === 'win32') {
96
+ try {
97
+ const psScript = `
98
+ Add-Type -AssemblyName System.Windows.Forms
99
+ [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${Math.round(x)}, ${Math.round(y)})
100
+ $src = @"
101
+ using System;
102
+ using System.Runtime.InteropServices;
103
+ public class WinMouse {
104
+ [DllImport("user32.dll")]
105
+ public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
106
+ }
107
+ "@
108
+ Add-Type -TypeDefinition $src -ErrorAction SilentlyContinue
109
+ # MOUSEEVENTF_LEFTDOWN = 0x0002, MOUSEEVENTF_LEFTUP = 0x0004
110
+ # MOUSEEVENTF_RIGHTDOWN = 0x0008, MOUSEEVENTF_RIGHTUP = 0x0010
111
+ ${btn === 'right'
112
+ ? `[WinMouse]::mouse_event(0x0008, 0, 0, 0, 0); Start-Sleep -Milliseconds 50; [WinMouse]::mouse_event(0x0010, 0, 0, 0, 0)`
113
+ : `for ($i=0; $i -lt ${count}; $i++) { [WinMouse]::mouse_event(0x0002, 0, 0, 0, 0); Start-Sleep -Milliseconds 40; [WinMouse]::mouse_event(0x0004, 0, 0, 0, 0); Start-Sleep -Milliseconds 50 }`}
114
+ `.trim();
115
+ await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
116
+ return { success: true, output: `Clicked ${btn} button at (${x}, ${y})` };
117
+ }
118
+ catch (err) {
119
+ return { success: false, output: `Click failed at (${x}, ${y}): ${err?.message || String(err)}` };
120
+ }
121
+ }
122
+ return { success: true, output: `Simulated click at (${x}, ${y}) on platform ${process.platform}` };
170
123
  }
171
- async takeScreenshot() {
172
- return {
173
- format: 'base64',
174
- width: 1920,
175
- height: 1080,
176
- data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==',
177
- };
178
- }
179
- async double_click(x, y) {
180
- this.cursor = { x, y };
181
- this.actionLog.push({ action: 'double_click', params: { x, y } });
182
- return {
183
- success: true,
184
- output: `Mock double-clicked at (${x}, ${y}).`,
185
- details: { x, y },
186
- };
124
+ async doubleClick(x, y) {
125
+ return this.click(x, y, { button: 'left', clicks: 2 });
187
126
  }
188
127
  async type(text) {
189
- this.typedHistory.push(text);
190
- this.actionLog.push({ action: 'type', params: { text } });
128
+ if (!text)
129
+ return { success: true, output: 'Typed empty string', action: 'type' };
130
+ if (process.platform === 'win32') {
131
+ try {
132
+ const targetApp = this.expectedApplication || 'Notepad';
133
+ const escaped = text
134
+ .replace(/\{/g, '{{}')
135
+ .replace(/\}/g, '{}}')
136
+ .replace(/\+/g, '{+}')
137
+ .replace(/\^/g, '{^}')
138
+ .replace(/%/g, '{%}')
139
+ .replace(/~/g, '{~}')
140
+ .replace(/\(/g, '{(}')
141
+ .replace(/\)/g, '{)}')
142
+ .replace(/\[/g, '{[}')
143
+ .replace(/\]/g, '{]}');
144
+ const psScript = `
145
+ $wshell = New-Object -ComObject Wscript.Shell
146
+ $null = $wshell.AppActivate('${targetApp}')
147
+ Start-Sleep -Milliseconds 250
148
+ Add-Type -AssemblyName System.Windows.Forms
149
+ [System.Windows.Forms.SendKeys]::SendWait("${escaped.replace(/"/g, '`"')}")
150
+ `.trim();
151
+ await this.runPowerShell(psScript);
152
+ this.activeWindow = targetApp;
153
+ return {
154
+ success: true,
155
+ output: `Typed text (${text.length} chars) into active application "${targetApp}"`,
156
+ action: 'type',
157
+ activeWindow: targetApp,
158
+ expectedWindow: targetApp,
159
+ };
160
+ }
161
+ catch (err) {
162
+ return {
163
+ success: false,
164
+ output: `Type error on active window "${this.expectedApplication || 'foreground'}": ${err?.message || String(err)}`,
165
+ action: 'type',
166
+ error: err?.message || String(err),
167
+ recoverable: true,
168
+ expectedWindow: this.expectedApplication || undefined,
169
+ };
170
+ }
171
+ }
191
172
  return {
192
173
  success: true,
193
- output: `Mock typed "${text}" into active application ${this.activeApp || 'desktop'}.`,
194
- details: { text, activeApp: this.activeApp },
174
+ output: `Simulated type (${text.length} chars)`,
175
+ action: 'type',
176
+ activeWindow: this.expectedApplication || 'Simulated Window',
195
177
  };
196
178
  }
197
- async key_press(key) {
198
- this.actionLog.push({ action: 'key_press', params: { key } });
199
- return {
200
- success: true,
201
- output: `Mock pressed key "${key}".`,
202
- details: { key },
179
+ async keyPress(key) {
180
+ const keyMap = {
181
+ enter: '{ENTER}',
182
+ return: '{ENTER}',
183
+ tab: '{TAB}',
184
+ escape: '{ESC}',
185
+ esc: '{ESC}',
186
+ backspace: '{BACKSPACE}',
187
+ delete: '{DEL}',
188
+ up: '{UP}',
189
+ down: '{DOWN}',
190
+ left: '{LEFT}',
191
+ right: '{RIGHT}',
192
+ space: ' ',
193
+ home: '{HOME}',
194
+ end: '{END}',
195
+ pageup: '{PGUP}',
196
+ pagedown: '{PGDN}',
203
197
  };
198
+ const mapped = keyMap[key.toLowerCase()] || key;
199
+ if (process.platform === 'win32') {
200
+ try {
201
+ const targetApp = this.expectedApplication || '';
202
+ const psScript = `
203
+ $wshell = New-Object -ComObject Wscript.Shell
204
+ ${targetApp ? `$null = $wshell.AppActivate('${targetApp}')\nStart-Sleep -Milliseconds 150` : ''}
205
+ Add-Type -AssemblyName System.Windows.Forms
206
+ [System.Windows.Forms.SendKeys]::SendWait("${mapped}")
207
+ `.trim();
208
+ await this.runPowerShell(psScript);
209
+ return {
210
+ success: true,
211
+ output: `Pressed key: ${key}`,
212
+ action: 'key_press',
213
+ activeWindow: targetApp || undefined,
214
+ };
215
+ }
216
+ catch (err) {
217
+ return {
218
+ success: false,
219
+ output: `KeyPress error: ${err?.message || String(err)}`,
220
+ action: 'key_press',
221
+ error: err?.message || String(err),
222
+ recoverable: true,
223
+ };
224
+ }
225
+ }
226
+ return { success: true, output: `Simulated key press: ${key}`, action: 'key_press' };
204
227
  }
205
228
  async hotkey(keys) {
206
- const combo = Array.isArray(keys) ? keys.join('+') : keys;
207
- this.actionLog.push({ action: 'hotkey', params: { combo } });
208
- return {
209
- success: true,
210
- output: `Mock pressed hotkey combo "${combo}".`,
211
- details: { combo },
212
- };
213
- }
214
- async scroll(direction, amount = 4) {
215
- this.actionLog.push({ action: 'scroll', params: { direction, amount } });
216
- return {
217
- success: true,
218
- output: `Mock scrolled ${direction} by ${amount} units.`,
219
- details: { direction, amount },
220
- };
221
- }
222
- async move_mouse(x, y) {
223
- this.cursor = { x, y };
224
- this.actionLog.push({ action: 'move_mouse', params: { x, y } });
225
- return {
226
- success: true,
227
- output: `Mock moved cursor to (${x}, ${y}).`,
228
- details: { x, y },
229
- };
230
- }
231
- async drag(start, end) {
232
- this.cursor = { ...end };
233
- this.actionLog.push({ action: 'drag', params: { start, end } });
234
- return {
235
- success: true,
236
- output: `Mock dragged from (${start.x}, ${start.y}) to (${end.x}, ${end.y}).`,
237
- details: { start, end },
238
- };
229
+ if (!keys || keys.length === 0)
230
+ return { success: false, output: 'No keys provided for hotkey', action: 'hotkey' };
231
+ if (process.platform === 'win32') {
232
+ try {
233
+ const targetApp = this.expectedApplication || '';
234
+ let prefix = '';
235
+ let mainKey = '';
236
+ for (const k of keys) {
237
+ const lk = k.toLowerCase();
238
+ if (lk === 'ctrl' || lk === 'control')
239
+ prefix += '^';
240
+ else if (lk === 'alt')
241
+ prefix += '%';
242
+ else if (lk === 'shift')
243
+ prefix += '+';
244
+ else
245
+ mainKey = k;
246
+ }
247
+ const sendPattern = `${prefix}${mainKey.toLowerCase()}`;
248
+ const psScript = `
249
+ $wshell = New-Object -ComObject Wscript.Shell
250
+ ${targetApp ? `$null = $wshell.AppActivate('${targetApp}')\nStart-Sleep -Milliseconds 150` : ''}
251
+ Add-Type -AssemblyName System.Windows.Forms
252
+ [System.Windows.Forms.SendKeys]::SendWait("${sendPattern}")
253
+ `.trim();
254
+ await this.runPowerShell(psScript);
255
+ return {
256
+ success: true,
257
+ output: `Executed hotkey: ${keys.join('+')} on "${targetApp || 'active window'}"`,
258
+ action: 'hotkey',
259
+ activeWindow: targetApp || undefined,
260
+ };
261
+ }
262
+ catch (err) {
263
+ return {
264
+ success: false,
265
+ output: `Hotkey error: ${err?.message || String(err)}`,
266
+ action: 'hotkey',
267
+ error: err?.message || String(err),
268
+ recoverable: true,
269
+ };
270
+ }
271
+ }
272
+ return { success: true, output: `Simulated hotkey: ${keys.join('+')}`, action: 'hotkey' };
273
+ }
274
+ async scroll(deltaX, deltaY) {
275
+ if (process.platform === 'win32') {
276
+ try {
277
+ const psScript = `
278
+ $src = @"
279
+ using System;
280
+ using System.Runtime.InteropServices;
281
+ public class WinMouseScroll {
282
+ [DllImport("user32.dll")]
283
+ public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
284
+ }
285
+ "@
286
+ Add-Type -TypeDefinition $src -ErrorAction SilentlyContinue
287
+ # MOUSEEVENTF_WHEEL = 0x0800
288
+ [WinMouseScroll]::mouse_event(0x0800, 0, 0, [uint32]${Math.round(-deltaY * 120)}, 0)
289
+ `.trim();
290
+ await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
291
+ return { success: true, output: `Scrolled deltaY=${deltaY}, deltaX=${deltaX}` };
292
+ }
293
+ catch (err) {
294
+ return { success: false, output: `Scroll error: ${err?.message || String(err)}` };
295
+ }
296
+ }
297
+ return { success: true, output: `Simulated scroll deltaY=${deltaY}` };
298
+ }
299
+ async mouseMove(x, y) {
300
+ if (process.platform === 'win32') {
301
+ try {
302
+ const psScript = `
303
+ Add-Type -AssemblyName System.Windows.Forms
304
+ [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${Math.round(x)}, ${Math.round(y)})
305
+ `.trim();
306
+ await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
307
+ return { success: true, output: `Moved mouse cursor to (${x}, ${y})` };
308
+ }
309
+ catch (err) {
310
+ return { success: false, output: `MouseMove error: ${err?.message || String(err)}` };
311
+ }
312
+ }
313
+ return { success: true, output: `Simulated mouse move to (${x}, ${y})` };
314
+ }
315
+ async drag(fromX, fromY, toX, toY) {
316
+ await this.mouseMove(fromX, fromY);
317
+ if (process.platform === 'win32') {
318
+ try {
319
+ const psScript = `
320
+ Add-Type -AssemblyName System.Windows.Forms
321
+ $src = @"
322
+ using System;
323
+ using System.Runtime.InteropServices;
324
+ public class WinMouseDrag {
325
+ [DllImport("user32.dll")]
326
+ public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
327
+ }
328
+ "@
329
+ Add-Type -TypeDefinition $src -ErrorAction SilentlyContinue
330
+ [WinMouseDrag]::mouse_event(0x0002, 0, 0, 0, 0)
331
+ [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${Math.round(toX)}, ${Math.round(toY)})
332
+ Start-Sleep -Milliseconds 100
333
+ [WinMouseDrag]::mouse_event(0x0004, 0, 0, 0, 0)
334
+ `.trim();
335
+ await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
336
+ return { success: true, output: `Dragged from (${fromX}, ${fromY}) to (${toX}, ${toY})` };
337
+ }
338
+ catch (err) {
339
+ return { success: false, output: `Drag error: ${err?.message || String(err)}` };
340
+ }
341
+ }
342
+ return { success: true, output: `Simulated drag from (${fromX}, ${fromY}) to (${toX}, ${toY})` };
239
343
  }
240
- async wait(durationMs) {
241
- this.actionLog.push({ action: 'wait', params: { durationMs } });
242
- return {
243
- success: true,
244
- output: `Mock waited ${durationMs}ms.`,
245
- details: { durationMs },
246
- };
344
+ async wait(ms) {
345
+ return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
247
346
  }
248
- async open_application(name, target) {
249
- this.openApps.add(name);
250
- this.activeApp = name;
251
- this.actionLog.push({ action: 'open_application', params: { name, target } });
252
- return {
253
- success: true,
254
- output: `Mock opened application "${name}" with target "${target || 'default'}".`,
255
- details: { name, target, active: true },
256
- };
257
- }
258
- async close_application(name) {
259
- this.openApps.delete(name);
260
- if (this.activeApp === name)
261
- this.activeApp = null;
262
- this.actionLog.push({ action: 'close_application', params: { name } });
263
- return {
264
- success: true,
265
- output: `Mock closed application "${name}".`,
266
- details: { name },
267
- };
347
+ async openApplication(appName, args = []) {
348
+ const appLower = appName.toLowerCase().trim();
349
+ const argsStr = args.join(' ');
350
+ try {
351
+ if (process.platform === 'win32') {
352
+ if (appLower === 'code' || appLower === 'vscode') {
353
+ await execAsync(`start code ${argsStr}`);
354
+ this.expectedApplication = 'Visual Studio Code';
355
+ }
356
+ else if (appLower === 'explorer' || appLower === 'file explorer') {
357
+ await execAsync(`start explorer ${argsStr || '.'}`);
358
+ this.expectedApplication = 'File Explorer';
359
+ }
360
+ else if (appLower === 'notepad') {
361
+ await execAsync(`start notepad ${argsStr}`);
362
+ this.expectedApplication = 'Notepad';
363
+ }
364
+ else if (appLower === 'terminal' || appLower === 'cmd' || appLower === 'powershell') {
365
+ await execAsync(`start powershell ${argsStr}`);
366
+ this.expectedApplication = 'PowerShell';
367
+ }
368
+ else {
369
+ await execAsync(`start "" "${appName}" ${argsStr}`);
370
+ this.expectedApplication = appName;
371
+ }
372
+ await this.wait(350);
373
+ if (this.expectedApplication) {
374
+ await this.switchWindow(this.expectedApplication);
375
+ }
376
+ return {
377
+ success: true,
378
+ output: `Launched and focused application: "${this.expectedApplication || appName}"`,
379
+ action: 'open_app',
380
+ activeWindow: this.expectedApplication || appName,
381
+ expectedWindow: this.expectedApplication || appName,
382
+ };
383
+ }
384
+ else if (process.platform === 'darwin') {
385
+ await execAsync(`open -a "${appName}" ${argsStr}`);
386
+ this.expectedApplication = appName;
387
+ }
388
+ else {
389
+ await execAsync(`xdg-open "${appName}" ${argsStr} &`);
390
+ this.expectedApplication = appName;
391
+ }
392
+ return {
393
+ success: true,
394
+ output: `Launched application: "${appName}"`,
395
+ action: 'open_app',
396
+ activeWindow: appName,
397
+ expectedWindow: appName,
398
+ };
399
+ }
400
+ catch (err) {
401
+ return {
402
+ success: false,
403
+ output: `Failed to launch "${appName}": ${err?.message || String(err)}`,
404
+ action: 'open_app',
405
+ error: err?.message || String(err),
406
+ recoverable: true,
407
+ };
408
+ }
268
409
  }
269
- async switch_application(name) {
270
- this.openApps.add(name);
271
- this.activeApp = name;
272
- this.actionLog.push({ action: 'switch_application', params: { name } });
273
- return {
274
- success: true,
275
- output: `Mock focused application "${name}".`,
276
- details: { name },
277
- };
410
+ async closeApplication(appName) {
411
+ try {
412
+ if (process.platform === 'win32') {
413
+ const cleanName = appName.replace(/\.exe$/i, '');
414
+ await execAsync(`powershell -NoProfile -Command "Stop-Process -Name '${cleanName}' -Force -ErrorAction SilentlyContinue"`);
415
+ }
416
+ else {
417
+ await execAsync(`pkill -f "${appName}"`);
418
+ }
419
+ if (this.expectedApplication && this.expectedApplication.toLowerCase().includes(appName.toLowerCase())) {
420
+ this.expectedApplication = null;
421
+ }
422
+ return { success: true, output: `Closed application: "${appName}"`, action: 'close_app' };
423
+ }
424
+ catch (err) {
425
+ return {
426
+ success: false,
427
+ output: `Failed to close "${appName}": ${err?.message || String(err)}`,
428
+ action: 'close_app',
429
+ error: err?.message || String(err),
430
+ };
431
+ }
278
432
  }
279
- async inspect_page() {
280
- this.actionLog.push({ action: 'inspect_page', params: {} });
281
- return {
282
- success: true,
283
- output: `Mock visual inspection of active app "${this.activeApp || 'desktop'}": 3 windows, 12 controls detected.`,
284
- details: { activeApp: this.activeApp, controlsCount: 12 },
285
- };
433
+ async switchWindow(windowTitleOrApp) {
434
+ if (process.platform === 'win32') {
435
+ try {
436
+ const psScript = `
437
+ $wshell = New-Object -ComObject Wscript.Shell
438
+ $result = $wshell.AppActivate("${windowTitleOrApp.replace(/"/g, '`"')}")
439
+ if (-not $result) {
440
+ $p = Get-Process | Where-Object { $_.MainWindowTitle -like "*${windowTitleOrApp.replace(/"/g, '`"')}*" -or $_.ProcessName -like "*${windowTitleOrApp.replace(/"/g, '`"')}*" } | Select-Object -First 1
441
+ if ($p) {
442
+ $null = $wshell.AppActivate($p.Id)
443
+ $result = $true
286
444
  }
287
- async get_active_window() {
288
- return {
289
- success: true,
290
- output: `Active window: "${this.activeApp || 'Desktop'}"`,
291
- details: { activeApp: this.activeApp },
292
- };
445
+ }
446
+ Start-Sleep -Milliseconds 150
447
+ if ($result) { Write-Output "OK" } else { Write-Output "NOT_FOUND" }
448
+ `.trim();
449
+ const stdout = await this.runPowerShell(psScript);
450
+ this.activeWindow = windowTitleOrApp;
451
+ this.expectedApplication = windowTitleOrApp;
452
+ if (stdout.includes('OK')) {
453
+ return {
454
+ success: true,
455
+ output: `Focused window matching: "${windowTitleOrApp}"`,
456
+ action: 'switch_window',
457
+ activeWindow: windowTitleOrApp,
458
+ expectedWindow: windowTitleOrApp,
459
+ };
460
+ }
461
+ return {
462
+ success: false,
463
+ output: `Could not find or focus window: "${windowTitleOrApp}". Try computer_inspect_ui to list open windows.`,
464
+ action: 'switch_window',
465
+ recoverable: true,
466
+ };
467
+ }
468
+ catch (err) {
469
+ return {
470
+ success: false,
471
+ output: `Window switch error for "${windowTitleOrApp}": ${err?.message || String(err)}`,
472
+ action: 'switch_window',
473
+ error: err?.message || String(err),
474
+ recoverable: true,
475
+ };
476
+ }
477
+ }
478
+ this.activeWindow = windowTitleOrApp;
479
+ return { success: true, output: `Switched window to: "${windowTitleOrApp}"`, action: 'switch_window' };
480
+ }
481
+ async inspectUI() {
482
+ if (process.platform === 'win32') {
483
+ try {
484
+ const psScript = `
485
+ Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle.Trim().Length -gt 0 } | Select-Object -ExpandProperty MainWindowTitle
486
+ `.trim();
487
+ const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
488
+ const titles = stdout
489
+ .split('\n')
490
+ .map((t) => t.trim())
491
+ .filter((t) => t.length > 0);
492
+ const active = titles.length > 0 ? titles[0] : 'Desktop';
493
+ return {
494
+ activeWindow: active,
495
+ windows: titles,
496
+ };
497
+ }
498
+ catch {
499
+ return { activeWindow: 'Desktop', windows: ['Desktop'] };
500
+ }
501
+ }
502
+ return { activeWindow: 'Desktop', windows: ['Desktop', 'Terminal'] };
293
503
  }
294
504
  }
295
505
  //# sourceMappingURL=computerController.js.map