ft-scout 9.0.2 → 9.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/.firebase/hosting.d2Vi.cache +7 -7
  2. package/README.md +76 -83
  3. package/bin/src/commands/agent.d.ts.map +1 -1
  4. package/bin/src/commands/agent.js +13 -223
  5. package/bin/src/commands/agent.js.map +1 -1
  6. package/bin/src/commands/plan.d.ts.map +1 -1
  7. package/bin/src/commands/plan.js +45 -18
  8. package/bin/src/commands/plan.js.map +1 -1
  9. package/bin/src/engine/actionVerifier.d.ts.map +1 -1
  10. package/bin/src/engine/actionVerifier.js +61 -9
  11. package/bin/src/engine/actionVerifier.js.map +1 -1
  12. package/bin/src/engine/agentEngine.d.ts.map +1 -1
  13. package/bin/src/engine/agentEngine.js +459 -547
  14. package/bin/src/engine/agentEngine.js.map +1 -1
  15. package/bin/src/engine/agentState.d.ts.map +1 -1
  16. package/bin/src/engine/agentState.js +0 -64
  17. package/bin/src/engine/agentState.js.map +1 -1
  18. package/bin/src/engine/browser/browserController.d.ts.map +1 -1
  19. package/bin/src/engine/browser/browserController.js +8 -0
  20. package/bin/src/engine/browser/browserController.js.map +1 -1
  21. package/bin/src/engine/computer/computerController.d.ts.map +1 -1
  22. package/bin/src/engine/computer/computerController.js +479 -269
  23. package/bin/src/engine/computer/computerController.js.map +1 -1
  24. package/bin/src/engine/computer/index.d.ts.map +1 -1
  25. package/bin/src/engine/computer/index.js +1 -0
  26. package/bin/src/engine/computer/index.js.map +1 -1
  27. package/bin/src/engine/computer/mockComputer.d.ts.map +1 -0
  28. package/bin/src/engine/computer/mockComputer.js +164 -0
  29. package/bin/src/engine/computer/mockComputer.js.map +1 -0
  30. package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -0
  31. package/bin/src/engine/runtime/agentRuntime.js +829 -0
  32. package/bin/src/engine/runtime/agentRuntime.js.map +1 -0
  33. package/bin/src/engine/runtime/index.d.ts.map +1 -0
  34. package/bin/src/engine/runtime/index.js +2 -0
  35. package/bin/src/engine/runtime/index.js.map +1 -0
  36. package/bin/src/engine/safety.d.ts.map +1 -1
  37. package/bin/src/engine/safety.js +49 -25
  38. package/bin/src/engine/safety.js.map +1 -1
  39. package/bin/src/engine/toolRegistry.d.ts.map +1 -1
  40. package/bin/src/engine/toolRegistry.js +162 -81
  41. package/bin/src/engine/toolRegistry.js.map +1 -1
  42. package/bin/src/engine/voiceEngine.d.ts.map +1 -1
  43. package/bin/src/engine/voiceEngine.js +0 -1
  44. package/bin/src/engine/voiceEngine.js.map +1 -1
  45. package/bin/src/engine/workspace/agentWorkspace.d.ts.map +1 -0
  46. package/bin/src/engine/workspace/agentWorkspace.js +484 -0
  47. package/bin/src/engine/workspace/agentWorkspace.js.map +1 -0
  48. package/bin/src/engine/workspace/index.d.ts.map +1 -0
  49. package/bin/src/engine/workspace/index.js +2 -0
  50. package/bin/src/engine/workspace/index.js.map +1 -0
  51. package/bin/src/index.js +4 -5
  52. package/bin/src/index.js.map +1 -1
  53. package/bin/src/server/server.d.ts.map +1 -1
  54. package/bin/src/server/server.js +25 -4
  55. package/bin/src/server/server.js.map +1 -1
  56. package/bin/src/utils/auth.d.ts.map +1 -1
  57. package/bin/src/utils/auth.js +67 -49
  58. package/bin/src/utils/auth.js.map +1 -1
  59. package/bin/src/utils/branding.js +1 -1
  60. package/bin/src/utils/payment.d.ts.map +1 -1
  61. package/bin/src/utils/payment.js +15 -1
  62. package/bin/src/utils/payment.js.map +1 -1
  63. package/bin/src/utils/pricing.js +1 -1
  64. package/bin/src/utils/pricing.js.map +1 -1
  65. package/bin/src/utils/razorpay.d.ts.map +1 -1
  66. package/bin/src/utils/razorpay.js +10 -2
  67. package/bin/src/utils/razorpay.js.map +1 -1
  68. package/downloads/invoice-2026-03.pdf +1 -0
  69. package/package.json +1 -1
  70. package/web/app.js +54 -135
  71. package/web/styles.css +2 -2
  72. package/bin/src/engine/appControl.d.ts.map +0 -1
  73. package/bin/src/engine/appControl.js +0 -2320
  74. package/bin/src/engine/appControl.js.map +0 -1
  75. package/bin/src/engine/liveScreenEngine.d.ts.map +0 -1
  76. package/bin/src/engine/liveScreenEngine.js +0 -421
  77. package/bin/src/engine/liveScreenEngine.js.map +0 -1
  78. package/bin/src/engine/scripts/winCapture.ps1 +0 -248
  79. package/downloads/INVOICE_2026_ACME.pdf +0 -1
@@ -1,2320 +0,0 @@
1
- import fs from 'fs';
2
- import path from 'path';
3
- import http from 'http';
4
- import https from 'https';
5
- import { exec, spawn, ChildProcess } from 'child_process';
6
- import { promisify } from 'util';
7
- import OpenAI from 'openai';
8
- import dotenv from 'dotenv';
9
- import chalk from 'chalk';
10
- import { startLiveScreenShare, stopLiveScreenShare, isLiveScreenShareActive, getLiveScreenFrame, analyzeLiveScreen, callVisionAiWithFrame, setLiveScreenTargetApp, getLiveScreenTargetApp, } from './liveScreenEngine.js';
11
- dotenv.config({ quiet: true });
12
- const execAsync = promisify(exec);
13
- /**
14
- * Standardize app name aliases across platforms.
15
- */
16
- export function normalizeAppName(rawAppName) {
17
- const appLower = (rawAppName || '').trim().toLowerCase();
18
- if (['browser', 'chrome', 'google-chrome', 'msedge', 'edge', 'firefox', 'brave', 'safari', 'opera', 'instagram', 'whatsapp', 'twitter', 'x', 'telegram', 'linkedin', 'slack', 'discord'].includes(appLower)) {
19
- return { category: 'browser', name: appLower || 'browser' };
20
- }
21
- if (['terminal', 'cmd', 'powershell', 'pwsh', 'wt', 'windows-terminal', 'iterm', 'bash', 'zsh', 'sh', 'console', 'konsole'].includes(appLower)) {
22
- return { category: 'terminal', name: appLower || 'terminal' };
23
- }
24
- if (['code', 'vscode', 'visual-studio-code', 'cursor', 'zed', 'sublime', 'notepad', 'notepad++', 'vim', 'nvim', 'nano', 'editor'].includes(appLower)) {
25
- return { category: 'editor', name: appLower || 'editor' };
26
- }
27
- if (['game', 'games', 'gaming', 'play', 'launcher', 'steam', 'epic', 'epicgames', 'riot', 'battlenet', 'ea', 'ubisoft', 'gog'].includes(appLower) || appLower.startsWith('game:')) {
28
- return { category: 'game', name: appLower };
29
- }
30
- return { category: 'system', name: appLower };
31
- }
32
- export function parsePayload(payload) {
33
- if (!payload)
34
- return {};
35
- if (typeof payload === 'object')
36
- return payload;
37
- if (typeof payload === 'string') {
38
- const trimmed = payload.trim();
39
- if (trimmed.startsWith('{') && trimmed.endsWith('}')) {
40
- try {
41
- return JSON.parse(trimmed);
42
- }
43
- catch { }
44
- }
45
- }
46
- return { target: payload };
47
- }
48
- export function buildSocialUrl(platformOrAppName, target, text) {
49
- const parsedPayload = parsePayload(target);
50
- const rawTarget = typeof target === 'string' && !target.trim().startsWith('{')
51
- ? target.trim()
52
- : (parsedPayload.recipient || parsedPayload.user || parsedPayload.username || parsedPayload.handle || parsedPayload.target || parsedPayload.url || '');
53
- const rawMsgText = text || parsedPayload.text || parsedPayload.message || parsedPayload.content || '';
54
- const pLower = (platformOrAppName || parsedPayload.platform || '').toLowerCase().trim();
55
- const cleanTarget = String(rawTarget || '').trim();
56
- const handleMatch = cleanTarget.match(/@?([a-zA-Z0-9_.-]+)/);
57
- const handle = handleMatch && handleMatch[1] ? handleMatch[1] : '';
58
- const isInstagram = pLower.includes('instagram') || cleanTarget.includes('instagram') || cleanTarget.startsWith('@');
59
- const isWhatsApp = pLower.includes('whatsapp') || cleanTarget.includes('whatsapp') || (/^\+?[0-9]{7,15}$/.test(cleanTarget) && !cleanTarget.startsWith('http'));
60
- const isTwitter = pLower.includes('twitter') || pLower.includes('x') || cleanTarget.includes('x.com') || cleanTarget.includes('twitter.com');
61
- const isTelegram = pLower.includes('telegram') || cleanTarget.includes('t.me');
62
- const isLinkedIn = pLower.includes('linkedin') || cleanTarget.includes('linkedin.com');
63
- if (isInstagram) {
64
- if (handle && handle !== 'instagram' && !cleanTarget.includes('http')) {
65
- return `https://www.instagram.com/direct/i/`;
66
- }
67
- return `https://www.instagram.com/direct/inbox/`;
68
- }
69
- if (isWhatsApp) {
70
- const phone = cleanTarget.replace(/[^0-9]/g, '');
71
- const msgParam = rawMsgText ? `?text=${encodeURIComponent(rawMsgText)}` : '';
72
- if (phone) {
73
- return `https://web.whatsapp.com/send?phone=${phone}${rawMsgText ? `&text=${encodeURIComponent(rawMsgText)}` : ''}`;
74
- }
75
- return `https://web.whatsapp.com/send${msgParam}`;
76
- }
77
- if (isTwitter) {
78
- if (handle && !cleanTarget.includes('http')) {
79
- return `https://x.com/direct_messages/create/${handle}`;
80
- }
81
- return `https://x.com/direct_messages`;
82
- }
83
- if (isTelegram) {
84
- if (handle)
85
- return `https://t.me/${handle}`;
86
- return `https://web.telegram.org/`;
87
- }
88
- if (isLinkedIn) {
89
- return `https://www.linkedin.com/messaging/`;
90
- }
91
- const isEmail = pLower.includes('gmail') || pLower.includes('mail') || pLower.includes('email') || (cleanTarget.includes('@') && cleanTarget.includes('.')) || cleanTarget.includes('mail.google.com');
92
- if (isEmail) {
93
- const to = cleanTarget.includes('@') && !cleanTarget.startsWith('http') ? cleanTarget : (parsedPayload.to || parsedPayload.recipient || '');
94
- const subject = parsedPayload.subject || parsedPayload.su || 'Message from Scout';
95
- const body = rawMsgText || parsedPayload.body || '';
96
- if (to || body) {
97
- return `https://mail.google.com/mail/?view=cm&fs=1${to ? `&to=${encodeURIComponent(to)}` : ''}${subject ? `&su=${encodeURIComponent(subject)}` : ''}${body ? `&body=${encodeURIComponent(body)}` : ''}`;
98
- }
99
- return 'https://mail.google.com';
100
- }
101
- if (cleanTarget.startsWith('http://') || cleanTarget.startsWith('https://')) {
102
- return cleanTarget;
103
- }
104
- if (cleanTarget.includes('.') && !cleanTarget.includes(' ') && !cleanTarget.includes('{')) {
105
- return `http://${cleanTarget}`;
106
- }
107
- return 'https://www.instagram.com/direct/inbox/';
108
- }
109
- /**
110
- * Universally search and locate any installed app, game, or store package on the host OS without hardcoding.
111
- * Automatically resolves Win32 executables, Microsoft Store games, Steam games, and desktop shortcuts.
112
- */
113
- export async function findInstalledApp(query) {
114
- const cleanQuery = (query || '').replace(/[^\w\s-]/g, '').trim().toLowerCase();
115
- if (!cleanQuery)
116
- return null;
117
- const platform = process.platform;
118
- if (platform === 'win32') {
119
- try {
120
- // Query Windows Get-StartApps with wildcard matching
121
- const safePattern = `*${cleanQuery.replace(/[*_']/g, '')}*`;
122
- const psCmd = `powershell.exe -NoProfile -NonInteractive -Command "Get-StartApps -Name '${safePattern}' | Select-Object -First 1 Name, AppID | ConvertTo-Json"`;
123
- const { stdout } = await execAsync(psCmd, { timeout: 3500 });
124
- if (stdout && stdout.trim()) {
125
- const parsed = JSON.parse(stdout.trim());
126
- if (parsed && (parsed.Name || parsed.AppID)) {
127
- return {
128
- name: String(parsed.Name || query),
129
- appId: String(parsed.AppID || ''),
130
- source: 'start_apps',
131
- };
132
- }
133
- }
134
- }
135
- catch { }
136
- // Fallback: check registered URI scheme protocol (e.g. steam, roblox, spotify, discord, minecraft, riot)
137
- if (!cleanQuery.includes(' ') && cleanQuery.length >= 3) {
138
- try {
139
- const checkReg = `powershell.exe -NoProfile -NonInteractive -Command "Test-Path 'Registry::HKEY_CLASSES_ROOT\\${cleanQuery}'"`;
140
- const { stdout } = await execAsync(checkReg, { timeout: 1500 });
141
- if (stdout && stdout.trim().toLowerCase() === 'true') {
142
- return {
143
- name: query,
144
- appId: `${cleanQuery}:`,
145
- source: 'protocol',
146
- };
147
- }
148
- }
149
- catch { }
150
- }
151
- }
152
- else if (platform === 'darwin') {
153
- try {
154
- const { stdout } = await execAsync(`mdfind "kMDItemKind == 'Application' && kMDItemDisplayName == '*${cleanQuery}*'c" | head -n 1`, { timeout: 3000 });
155
- if (stdout && stdout.trim()) {
156
- return {
157
- name: path.basename(stdout.trim(), '.app'),
158
- appId: stdout.trim(),
159
- source: 'path',
160
- };
161
- }
162
- }
163
- catch { }
164
- }
165
- else {
166
- // Linux: check which or desktop application entries
167
- try {
168
- const { stdout } = await execAsync(`which "${cleanQuery}" || find /usr/share/applications -name "*${cleanQuery}*.desktop" | head -n 1`, { timeout: 2500 });
169
- if (stdout && stdout.trim()) {
170
- return {
171
- name: query,
172
- appId: stdout.trim(),
173
- source: 'path',
174
- };
175
- }
176
- }
177
- catch { }
178
- }
179
- return null;
180
- }
181
- /**
182
- * Cross-platform app opener.
183
- * Launches and opens target application on Windows, macOS, or Linux.
184
- */
185
- export async function openApp(appNameOrPath, target, options) {
186
- const { category, name } = normalizeAppName(appNameOrPath);
187
- if (name && name !== 'desktop' && name !== 'system') {
188
- setLiveScreenTargetApp(name);
189
- }
190
- const platform = process.platform;
191
- let command = '';
192
- const cleanTarget = target ? target.trim() : '';
193
- if (category === 'browser') {
194
- let url = cleanTarget;
195
- const parsedPayload = parsePayload(cleanTarget);
196
- const rawRecipient = parsedPayload.recipient || parsedPayload.username || parsedPayload.handle || parsedPayload.target || cleanTarget;
197
- if (['instagram', 'whatsapp', 'twitter', 'x', 'telegram', 'linkedin'].includes(name) || cleanTarget.includes('instagram') || cleanTarget.includes('whatsapp') || cleanTarget.includes('x.com') || cleanTarget.includes('t.me') || String(rawRecipient).startsWith('@')) {
198
- url = buildSocialUrl(name, cleanTarget, options?.args?.[0]);
199
- }
200
- else if (!url.startsWith('http://') && !url.startsWith('https://')) {
201
- url = url && !url.includes('{') ? `http://${url}` : 'https://google.com';
202
- }
203
- if (platform === 'win32') {
204
- if (name === 'chrome' || name === 'google-chrome') {
205
- command = `start chrome "${url}"`;
206
- }
207
- else if (name === 'edge' || name === 'msedge') {
208
- command = `start msedge "${url}"`;
209
- }
210
- else if (name === 'firefox') {
211
- command = `start firefox "${url}"`;
212
- }
213
- else if (name === 'brave') {
214
- command = `start brave "${url}"`;
215
- }
216
- else {
217
- command = `start "" "${url}"`;
218
- }
219
- }
220
- else if (platform === 'darwin') {
221
- if (name === 'chrome' || name === 'google-chrome') {
222
- command = `open -a "Google Chrome" "${url}"`;
223
- }
224
- else if (name === 'firefox') {
225
- command = `open -a "Firefox" "${url}"`;
226
- }
227
- else if (name === 'safari') {
228
- command = `open -a "Safari" "${url}"`;
229
- }
230
- else {
231
- command = `open "${url}"`;
232
- }
233
- }
234
- else {
235
- // Linux
236
- if (name === 'chrome' || name === 'google-chrome') {
237
- command = `google-chrome "${url}" &`;
238
- }
239
- else if (name === 'firefox') {
240
- command = `firefox "${url}" &`;
241
- }
242
- else {
243
- command = `xdg-open "${url}" &`;
244
- }
245
- }
246
- }
247
- else if (category === 'terminal') {
248
- const initialCmd = cleanTarget;
249
- if (platform === 'win32') {
250
- if (name === 'wt' || name === 'windows-terminal' || name === 'terminal') {
251
- command = initialCmd ? `start wt.exe powershell -NoExit -Command "${initialCmd.replace(/"/g, '`"')}"` : `start wt.exe`;
252
- }
253
- else if (name === 'cmd') {
254
- command = initialCmd ? `start cmd.exe /k "${initialCmd}"` : `start cmd.exe`;
255
- }
256
- else {
257
- // powershell / default
258
- command = initialCmd ? `start powershell.exe -NoExit -Command "${initialCmd.replace(/"/g, '`"')}"` : `start powershell.exe`;
259
- }
260
- }
261
- else if (platform === 'darwin') {
262
- if (name === 'iterm') {
263
- command = `open -a iTerm .`;
264
- }
265
- else {
266
- command = `open -a Terminal .`;
267
- }
268
- }
269
- else {
270
- // Linux
271
- command = `x-terminal-emulator &`;
272
- }
273
- }
274
- else if (category === 'editor') {
275
- const fileOrDir = cleanTarget || '.';
276
- if (name === 'code' || name === 'vscode' || name === 'visual-studio-code' || name === 'cursor' || name === 'editor') {
277
- const exe = name === 'cursor' ? 'cursor' : 'code';
278
- command = `${exe} "${fileOrDir}"`;
279
- }
280
- else if (name === 'notepad' && platform === 'win32') {
281
- command = `start notepad "${fileOrDir}"`;
282
- }
283
- else if (platform === 'win32') {
284
- command = `start "" "${fileOrDir}"`;
285
- }
286
- else if (platform === 'darwin') {
287
- command = `open -a "Visual Studio Code" "${fileOrDir}"`;
288
- }
289
- else {
290
- command = `xdg-open "${fileOrDir}" &`;
291
- }
292
- }
293
- else {
294
- // Universal Application, Game & Desktop Launcher (Zero-Hardcode OS Discovery)
295
- const targetQuery = name || appNameOrPath;
296
- const installedApp = await findInstalledApp(targetQuery);
297
- if (installedApp) {
298
- if (platform === 'win32') {
299
- if (installedApp.source === 'protocol') {
300
- const extraArg = cleanTarget ? ` "${cleanTarget}"` : '';
301
- command = `start ${installedApp.appId}${extraArg}`;
302
- }
303
- else {
304
- command = `explorer.exe "shell:AppsFolder\\${installedApp.appId}"`;
305
- }
306
- }
307
- else if (platform === 'darwin') {
308
- command = `open -a "${installedApp.appId || installedApp.name}"`;
309
- }
310
- else {
311
- command = `"${installedApp.appId || installedApp.name}" &`;
312
- }
313
- }
314
- else {
315
- // Protocol scheme check (e.g. steam:, roblox:, spotify:, discord:, minecraft:, etc.)
316
- const isProtocol = /^[a-zA-Z0-9_\-]+:/.test(appNameOrPath) || (cleanTarget && /^[a-zA-Z0-9_\-]+:/.test(cleanTarget));
317
- if (isProtocol && platform === 'win32') {
318
- const proto = /^[a-zA-Z0-9_\-]+:/.test(appNameOrPath) ? appNameOrPath : cleanTarget;
319
- command = `start ${proto}`;
320
- }
321
- else if (platform === 'win32') {
322
- command = cleanTarget ? `start "" "${appNameOrPath}" ${cleanTarget}` : `start "" "${appNameOrPath}"`;
323
- }
324
- else if (platform === 'darwin') {
325
- command = cleanTarget ? `open -a "${appNameOrPath}" "${cleanTarget}"` : `open -a "${appNameOrPath}"`;
326
- }
327
- else {
328
- command = cleanTarget ? `${appNameOrPath} "${cleanTarget}" &` : `${appNameOrPath} &`;
329
- }
330
- }
331
- }
332
- try {
333
- await execAsync(command, { timeout: 3000 });
334
- return {
335
- success: true,
336
- app: name || appNameOrPath,
337
- action: 'open',
338
- output: `Successfully launched ${name || appNameOrPath} [Category: ${category}] with target "${cleanTarget || 'default'}". Executed command: \`${command}\`.`,
339
- details: { category, appName: name || appNameOrPath, target: cleanTarget, command, platform },
340
- };
341
- }
342
- catch (err) {
343
- // Fallback for GUI commands that spawn detached processes or timeout waiting for process exit
344
- return {
345
- success: true,
346
- app: name || appNameOrPath,
347
- action: 'open',
348
- output: `Triggered app launch for ${name || appNameOrPath} (Platform: ${platform}). Command: \`${command}\`. Note: ${err?.message || 'Process spawned in background.'}`,
349
- details: { category, appName: name || appNameOrPath, target: cleanTarget, command, platform, warning: err?.message },
350
- };
351
- }
352
- }
353
- /**
354
- * Execute automation actions within an opened application context.
355
- */
356
- export async function executeInApp(appName, action, payload) {
357
- const { category, name } = normalizeAppName(appName);
358
- const actLower = (action || '').trim().toLowerCase();
359
- // Focus live screen sharing on the active app being operated
360
- if (name && name !== 'desktop' && name !== 'system') {
361
- setLiveScreenTargetApp(name);
362
- }
363
- // Universal Scratchpad Desktop Automation Actions (Takeover, Screen Seeing, Keystrokes, Mouse Clicks, Focus Locks)
364
- if (['takeover', 'takeover_control', 'takeover_screen', 'takeover_app'].includes(actLower)) {
365
- const parsed = parsePayload(payload);
366
- const duration = typeof parsed.duration === 'number' ? parsed.duration : 3000;
367
- const x = typeof parsed.x === 'number' ? parsed.x : undefined;
368
- const y = typeof parsed.y === 'number' ? parsed.y : undefined;
369
- const res = await takeoverControl(name || appName, duration, x, y);
370
- return {
371
- success: res.success,
372
- app: name || appName,
373
- action: actLower,
374
- output: res.output,
375
- details: res.details,
376
- };
377
- }
378
- if (['release_takeover', 'unlock_screen', 'unlock_app', 'unlock_input', 'unlock'].includes(actLower)) {
379
- const res = await releaseTakeover();
380
- return {
381
- success: res.success,
382
- app: name || appName,
383
- action: actLower,
384
- output: res.output,
385
- };
386
- }
387
- if (['start_screen_share', 'live_screen_share', 'screen_share', 'start_live_stream'].includes(actLower)) {
388
- const targetApp = (name && name !== 'desktop' && name !== 'system') ? name : undefined;
389
- const res = await startLiveScreenShare({ targetApp });
390
- return {
391
- success: res.success,
392
- app: name || appName,
393
- action: actLower,
394
- output: res.message,
395
- details: { active: true, targetApp },
396
- };
397
- }
398
- if (['stop_screen_share', 'end_screen_share', 'stop_live_stream'].includes(actLower)) {
399
- const res = stopLiveScreenShare();
400
- return {
401
- success: res.success,
402
- app: name || appName,
403
- action: actLower,
404
- output: res.summary,
405
- details: { active: false },
406
- };
407
- }
408
- if (['capture_screen', 'screenshot', 'screen_shot'].includes(actLower)) {
409
- if (isLiveScreenShareActive()) {
410
- const liveRes = await analyzeLiveScreen();
411
- return {
412
- success: liveRes.success,
413
- app: name || appName,
414
- action: actLower,
415
- output: liveRes.output,
416
- details: liveRes.details,
417
- };
418
- }
419
- const res = await captureScreen(name || appName);
420
- return {
421
- success: res.success,
422
- app: name || appName,
423
- action: actLower,
424
- output: res.output,
425
- details: res.details,
426
- };
427
- }
428
- if (['see_screen', 'inspect_screen', 'view_screen', 'analyze_screen', 'inspect_desktop', 'visual_analysis', 'see_and_analyze', 'live_screen_stream', 'screen_stream'].includes(actLower)) {
429
- const query = typeof payload === 'string' ? payload : (payload?.query || payload?.target || payload?.text || undefined);
430
- const res = await analyzeLiveScreen(query);
431
- return {
432
- success: res.success,
433
- app: name || appName,
434
- action: actLower,
435
- output: res.output,
436
- details: res.details,
437
- };
438
- }
439
- if (['get_screen_info', 'list_screens', 'screen_bounds', 'screen_info'].includes(actLower)) {
440
- const res = await getScreenInfo(name || appName);
441
- return {
442
- success: res.success,
443
- app: name || appName,
444
- action: actLower,
445
- output: res.output,
446
- details: res.details,
447
- };
448
- }
449
- if (['type_text', 'send_keys', 'type', 'keystrokes', 'key_combo', 'press_hotkey', 'hotkey'].includes(actLower)) {
450
- const parsed = parsePayload(payload);
451
- const isHotKeyAction = ['key_combo', 'press_hotkey', 'hotkey', 'send_keys'].includes(actLower);
452
- let keyCombo = undefined;
453
- let textToType = '';
454
- if (typeof parsed === 'object') {
455
- keyCombo = parsed.keyCombo || parsed.combo || parsed.hotkey || parsed.keys || parsed.key;
456
- textToType = parsed.text || parsed.content || parsed.message || '';
457
- }
458
- if (!keyCombo && typeof payload === 'string') {
459
- const trimmed = payload.trim();
460
- if (isHotKeyAction || trimmed.includes('+') || ['enter', 'return', 'tab', 'esc', 'escape', 'backspace', 'space', 'up', 'down', 'left', 'right'].includes(trimmed.toLowerCase())) {
461
- keyCombo = trimmed;
462
- }
463
- else {
464
- textToType = trimmed;
465
- }
466
- }
467
- else if (!textToType && typeof payload === 'string' && !payload.trim().startsWith('{')) {
468
- textToType = payload;
469
- }
470
- const pressEnter = typeof parsed === 'object' && parsed.enter !== undefined ? parsed.enter : !isHotKeyAction;
471
- const ksRes = await sendKeystrokes(name || appName, textToType, pressEnter, keyCombo);
472
- return {
473
- success: ksRes.success,
474
- app: name || appName,
475
- action: actLower,
476
- output: ksRes.output,
477
- details: { appName: name || appName, textToType, pressEnter, keyCombo },
478
- };
479
- }
480
- if (['click_element', 'click_ui', 'click_button', 'tap_element'].includes(actLower)) {
481
- const parsed = parsePayload(payload);
482
- const query = typeof parsed === 'object' ? (parsed.element || parsed.target || parsed.name || parsed.query || '') : String(parsed || payload);
483
- const button = typeof parsed === 'object' && parsed.button ? parsed.button : 'left';
484
- const res = await findAndClickElement(query, name || appName, button);
485
- return {
486
- success: res.success,
487
- app: name || appName,
488
- action: actLower,
489
- output: res.output,
490
- details: res.coordinates,
491
- };
492
- }
493
- if (['click', 'click_app', 'click_at', 'move_mouse', 'drag_mouse'].includes(actLower)) {
494
- const parsedPayload = parsePayload(payload);
495
- if (typeof parsedPayload === 'object' && parsedPayload.element && parsedPayload.x === undefined) {
496
- const res = await findAndClickElement(parsedPayload.element, name || appName, parsedPayload.button || 'left');
497
- return {
498
- success: res.success,
499
- app: name || appName,
500
- action: actLower,
501
- output: res.output,
502
- details: res.coordinates,
503
- };
504
- }
505
- const x = typeof parsedPayload.x === 'number' ? parsedPayload.x : undefined;
506
- const y = typeof parsedPayload.y === 'number' ? parsedPayload.y : undefined;
507
- const button = ['right', 'double', 'middle', 'scroll'].includes(parsedPayload.button) ? parsedPayload.button : 'left';
508
- const clickRes = await moveAndClickMouse(x, y, button, name || appName);
509
- return {
510
- success: clickRes.success,
511
- app: name || appName,
512
- action: actLower,
513
- output: clickRes.output,
514
- details: { appName: name || appName, x, y, button },
515
- };
516
- }
517
- if (['scroll', 'scroll_window', 'scroll_down', 'scroll_up', 'scroll_page'].includes(actLower)) {
518
- const parsed = parsePayload(payload);
519
- let direction = 'down';
520
- let amount = 3;
521
- if (typeof parsed === 'object') {
522
- if (parsed.direction === 'up' || parsed.dir === 'up')
523
- direction = 'up';
524
- if (typeof parsed.amount === 'number')
525
- amount = parsed.amount;
526
- if (typeof parsed.clicks === 'number')
527
- amount = parsed.clicks;
528
- }
529
- else if (typeof payload === 'string' && payload.toLowerCase().includes('up')) {
530
- direction = 'up';
531
- }
532
- if (actLower === 'scroll_up')
533
- direction = 'up';
534
- const res = await scrollWindow(direction, amount);
535
- return {
536
- success: res.success,
537
- app: name || appName,
538
- action: actLower,
539
- output: res.output,
540
- };
541
- }
542
- if (['list_windows', 'get_windows', 'open_windows', 'show_windows'].includes(actLower)) {
543
- const res = await listWindows();
544
- return {
545
- success: res.success,
546
- app: name || appName,
547
- action: actLower,
548
- output: res.output,
549
- details: res.windows,
550
- };
551
- }
552
- if (['focus_window', 'switch_window', 'bring_to_front', 'select_window'].includes(actLower)) {
553
- const parsed = parsePayload(payload);
554
- const target = typeof parsed === 'object' ? (parsed.title || parsed.app || parsed.name || '') : String(parsed || payload || name || appName);
555
- const res = await focusWindow(target);
556
- return {
557
- success: res.success,
558
- app: target,
559
- action: actLower,
560
- output: res.output,
561
- };
562
- }
563
- if (['lock_app', 'block_interruption', 'lock_input', 'lock'].includes(actLower)) {
564
- const parsedPayload = parsePayload(payload);
565
- const durationMs = typeof parsedPayload.duration === 'number' ? parsedPayload.duration : 3000;
566
- const lockRes = await lockAppInput(name || appName, durationMs);
567
- return {
568
- success: lockRes.success,
569
- app: name || appName,
570
- action: actLower,
571
- output: lockRes.output,
572
- details: { appName: name || appName, durationMs },
573
- };
574
- }
575
- if (category === 'browser') {
576
- if (['navigate', 'goto', 'open_url'].includes(actLower)) {
577
- const rawUrl = typeof payload === 'string' ? payload : (payload?.url || payload?.target || 'https://google.com');
578
- let targetUrl = rawUrl.trim();
579
- let navigationNote = '';
580
- // Universal URL vs Search Query Grounding:
581
- // If the input is not a valid URL (contains spaces or lacks a dot / scheme), convert into a search
582
- if (!targetUrl.startsWith('http://') && !targetUrl.startsWith('https://')) {
583
- if (targetUrl.includes(' ') || !targetUrl.includes('.')) {
584
- navigationNote = ` [Note: Plain search term passed to navigate; converted to web search query]`;
585
- targetUrl = `https://www.google.com/search?q=${encodeURIComponent(targetUrl)}`;
586
- }
587
- else {
588
- targetUrl = `https://${targetUrl}`;
589
- }
590
- }
591
- await openApp(name || 'browser', targetUrl);
592
- await new Promise((r) => setTimeout(r, 800));
593
- const capture = await captureScreen(name || 'browser');
594
- return {
595
- success: true,
596
- app: name || 'browser',
597
- action: actLower,
598
- output: `Browser Navigated Successfully to Target URL: ${targetUrl}${navigationNote}\n- Screenshot Saved: ${capture.details?.outPath || 'Active'}`,
599
- details: { url: targetUrl, capture: capture.details },
600
- };
601
- }
602
- if (actLower === 'fetch_page' || actLower === 'read_url' || actLower === 'inspect_dom') {
603
- const rawUrl = typeof payload === 'string' ? payload : (payload?.url || payload?.target || 'http://localhost:3000');
604
- const targetUrl = rawUrl.startsWith('http://') || rawUrl.startsWith('https://') ? rawUrl : `http://${rawUrl}`;
605
- try {
606
- const pageContent = await fetchUrlContent(targetUrl);
607
- const textSnippet = extractTextFromHtml(pageContent);
608
- return {
609
- success: true,
610
- app: name,
611
- action: actLower,
612
- output: `Browser Action [${actLower}] for ${targetUrl}:\n\n` +
613
- `--- Page Text Preview (${textSnippet.length} chars) ---\n` +
614
- `${textSnippet.slice(0, 2000)}${textSnippet.length > 2000 ? '\n... [truncated]' : ''}`,
615
- details: { url: targetUrl, fullHtmlLength: pageContent.length, textLength: textSnippet.length },
616
- };
617
- }
618
- catch (err) {
619
- return {
620
- success: false,
621
- app: name,
622
- action: actLower,
623
- output: `Failed to fetch page at ${targetUrl}: ${err?.message || String(err)}`,
624
- };
625
- }
626
- }
627
- if (actLower === 'search' || actLower === 'search_web') {
628
- const query = typeof payload === 'string' ? payload : (payload?.query || payload?.target || payload?.url || '');
629
- const isUrl = query.startsWith('http://') || query.startsWith('https://') || (query.includes('.') && !query.includes(' '));
630
- if (isUrl) {
631
- const targetUrl = query.startsWith('http://') || query.startsWith('https://') ? query : `https://${query}`;
632
- return await executeInApp(name || 'browser', 'navigate', targetUrl);
633
- }
634
- // Direct Search in browser window
635
- const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}`;
636
- await openApp(name || 'browser', searchUrl);
637
- await new Promise((r) => setTimeout(r, 800));
638
- const capture = await captureScreen(name || 'browser');
639
- return {
640
- success: true,
641
- app: name,
642
- action: actLower,
643
- output: `Browser search executed for "${query}" at ${searchUrl}.\n- Screenshot Saved: ${capture.details?.outPath || 'Active'}`,
644
- details: { query, searchUrl, capture: capture.details },
645
- };
646
- }
647
- if (['send_dm', 'open_dm', 'dm', 'send_message', 'message', 'social_dm', 'send_mail', 'send_email', 'mail', 'email'].includes(actLower)) {
648
- const parsed = parsePayload(payload);
649
- const recipient = typeof payload === 'string' && payload.includes('@') && !payload.startsWith('{')
650
- ? payload
651
- : (parsed.recipient || parsed.to || parsed.user || parsed.target || parsed.handle || parsed.username || '');
652
- let msgText = typeof parsed === 'object' ? (parsed.text || parsed.message || parsed.content || parsed.body) : undefined;
653
- if (!msgText && typeof payload === 'string' && payload.length > 0 && !payload.startsWith('@') && !payload.startsWith('http') && !payload.includes('@')) {
654
- msgText = payload;
655
- }
656
- const isEmail = ['send_mail', 'send_email', 'mail', 'email'].includes(actLower) || recipient.includes('@');
657
- const targetUrl = buildSocialUrl(isEmail ? 'gmail' : name, recipient, msgText);
658
- const openRes = await openApp('browser', targetUrl);
659
- let keystrokeOutput = '';
660
- if (isEmail) {
661
- // For Gmail direct compose: wait 2.5s for page & compose dialog to render, then press Ctrl+Enter to send!
662
- await lockAppInput('browser', 3500);
663
- await new Promise((r) => setTimeout(r, 2200));
664
- const ksRes = await sendKeystrokes('browser', '', false, 'ctrl+enter');
665
- keystrokeOutput = `\nAuto-Send Keystroke Dispatch (Ctrl+Enter): ${ksRes.output}`;
666
- }
667
- else if (msgText) {
668
- // Lock user interruption during scratchpad keystroke dispatch so user clicks don't interrupt
669
- await lockAppInput(name || 'browser', 3000);
670
- await new Promise((r) => setTimeout(r, 1200));
671
- const ksRes = await sendKeystrokes(name || 'browser', msgText, true);
672
- keystrokeOutput = `\nAuto-Keystroke Automated Dispatch: ${ksRes.output}`;
673
- }
674
- return {
675
- success: true,
676
- app: name,
677
- action: actLower,
678
- output: `Executed ${isEmail ? 'Gmail / Email Dispatch' : `${name} Direct Message`} for target recipient "${recipient}" at ${targetUrl}.${keystrokeOutput}`,
679
- details: { platform: isEmail ? 'gmail' : name, recipient, msgText, targetUrl },
680
- };
681
- }
682
- }
683
- if (category === 'terminal') {
684
- if (actLower === 'exec_command' || actLower === 'run_script' || actLower === 'run') {
685
- const cmdStr = typeof payload === 'string' ? payload : (payload?.command || payload?.cmd || payload?.script || 'echo "Terminal Action Executed"');
686
- try {
687
- const { stdout, stderr } = await execAsync(cmdStr, { cwd: process.cwd() });
688
- return {
689
- success: true,
690
- app: name,
691
- action: actLower,
692
- output: `Terminal Execution inside ${name}:\nCommand: \`${cmdStr}\`\n\nOutput:\n${stdout || stderr || '(No output)'}`,
693
- details: { command: cmdStr, stdout, stderr },
694
- };
695
- }
696
- catch (err) {
697
- return {
698
- success: false,
699
- app: name,
700
- action: actLower,
701
- output: `Terminal Execution Failed for \`${cmdStr}\`: ${err?.message || String(err)}`,
702
- };
703
- }
704
- }
705
- }
706
- if (category === 'editor') {
707
- if (actLower === 'open_file' || actLower === 'view_file' || actLower === 'open') {
708
- const filePath = typeof payload === 'string' ? payload : (payload?.path || payload?.target || '.');
709
- return await openApp(name, filePath);
710
- }
711
- }
712
- if (category === 'game' || ['open', 'launch', 'start'].includes(actLower)) {
713
- const rawTarget = typeof payload === 'string' ? payload : (payload?.target || payload?.game || payload?.url || payload?.query || payload?.id || '');
714
- return await openApp(name || appName, rawTarget);
715
- }
716
- // Generic fallback app action
717
- return {
718
- success: true,
719
- app: name,
720
- action: actLower,
721
- output: `Executed app action "${actLower}" on ${name}. Target/Payload: ${JSON.stringify(payload || {})}`,
722
- details: { appName: name, action: actLower, payload },
723
- };
724
- }
725
- /**
726
- * Simple HTTP/HTTPS URL content fetcher for browser page inspection.
727
- */
728
- function fetchUrlContent(targetUrl) {
729
- return new Promise((resolve, reject) => {
730
- const client = targetUrl.startsWith('https:') ? https : http;
731
- const req = client.get(targetUrl, { timeout: 10000 }, (res) => {
732
- if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
733
- const redirectUrl = res.headers.location.startsWith('http')
734
- ? res.headers.location
735
- : new URL(res.headers.location, targetUrl).toString();
736
- return fetchUrlContent(redirectUrl).then(resolve).catch(reject);
737
- }
738
- let data = '';
739
- res.setEncoding('utf-8');
740
- res.on('data', (chunk) => { data += chunk; });
741
- res.on('end', () => { resolve(data); });
742
- });
743
- req.on('error', (err) => reject(err));
744
- req.on('timeout', () => {
745
- req.destroy();
746
- reject(new Error(`HTTP request timed out after 10000ms for ${targetUrl}`));
747
- });
748
- });
749
- }
750
- /**
751
- * Basic HTML to plain text converter for page inspection.
752
- */
753
- function extractTextFromHtml(html) {
754
- if (!html)
755
- return '';
756
- return html
757
- .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '')
758
- .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '')
759
- .replace(/<[^>]+>/g, ' ')
760
- .replace(/\s+/g, ' ')
761
- .trim();
762
- }
763
- /**
764
- * Diagnostics report of active host platform app launchers.
765
- */
766
- export function getAppCapabilities() {
767
- const platform = process.platform;
768
- return {
769
- platform,
770
- supportedCategories: ['browser', 'terminal', 'editor', 'system'],
771
- defaultBrowserLauncher: platform === 'win32' ? 'start' : platform === 'darwin' ? 'open' : 'xdg-open',
772
- defaultTerminalLauncher: platform === 'win32' ? 'wt.exe / powershell.exe' : platform === 'darwin' ? 'Terminal.app' : 'x-terminal-emulator',
773
- defaultEditorLauncher: platform === 'win32' ? 'code / notepad' : platform === 'darwin' ? 'code / open -a' : 'code / xdg-open',
774
- };
775
- }
776
- const WIN_CAPTURE_SCRIPT_CONTENT = `param(
777
- [string]$OutPath,
778
- [switch]$InfoOnly
779
- )
780
-
781
- $source = @"
782
- using System;
783
- using System.Drawing;
784
- using System.Drawing.Imaging;
785
- using System.Runtime.InteropServices;
786
- using System.Text;
787
- using System.Threading;
788
- using System.Collections.Generic;
789
-
790
- public class WinCaptureResult {
791
- public bool success;
792
- public string savedPath;
793
- public int width;
794
- public int height;
795
- public int cursorX;
796
- public int cursorY;
797
- public string activeWindow;
798
- public string[] visibleWindows;
799
- public string error;
800
- }
801
-
802
- public class WinCaptureHelper {
803
- [DllImport("user32.dll", SetLastError = true)]
804
- public static extern IntPtr OpenInputDesktop(uint dwFlags, bool fInherit, uint dwDesiredAccess);
805
-
806
- [DllImport("user32.dll", SetLastError = true)]
807
- public static extern IntPtr OpenDesktop(string lpszDesktop, uint dwFlags, bool fInherit, uint dwDesiredAccess);
808
-
809
- [DllImport("user32.dll", SetLastError = true)]
810
- public static extern bool SetThreadDesktop(IntPtr hDesktop);
811
-
812
- [DllImport("user32.dll", SetLastError = true)]
813
- public static extern bool CloseDesktop(IntPtr hDesktop);
814
-
815
- [DllImport("user32.dll")]
816
- public static extern IntPtr GetDC(IntPtr hWnd);
817
-
818
- [DllImport("user32.dll")]
819
- public static extern int ReleaseDC(IntPtr hWnd, IntPtr hDC);
820
-
821
- [DllImport("gdi32.dll")]
822
- public static extern IntPtr CreateCompatibleDC(IntPtr hdc);
823
-
824
- [DllImport("gdi32.dll")]
825
- public static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight);
826
-
827
- [DllImport("gdi32.dll")]
828
- public static extern IntPtr SelectObject(IntPtr hdc, IntPtr hgdiobj);
829
-
830
- [DllImport("gdi32.dll")]
831
- public static extern bool DeleteDC(IntPtr hdc);
832
-
833
- [DllImport("gdi32.dll")]
834
- public static extern bool DeleteObject(IntPtr hObject);
835
-
836
- [DllImport("gdi32.dll")]
837
- public static extern bool BitBlt(IntPtr hdcDest, int nXDest, int nYDest, int nWidth, int nHeight, IntPtr hdcSrc, int nXSrc, int nYSrc, int dwRop);
838
-
839
- [DllImport("user32.dll")]
840
- public static extern int GetSystemMetrics(int nIndex);
841
-
842
- public delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam);
843
-
844
- [DllImport("user32.dll")]
845
- public static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, IntPtr lParam);
846
-
847
- [DllImport("user32.dll", CharSet = CharSet.Auto)]
848
- public static extern int GetWindowText(IntPtr hWnd, StringBuilder lpString, int nMaxCount);
849
-
850
- [DllImport("user32.dll")]
851
- public static extern bool IsWindowVisible(IntPtr hWnd);
852
-
853
- [DllImport("user32.dll")]
854
- public static extern IntPtr GetForegroundWindow();
855
-
856
- [StructLayout(LayoutKind.Sequential)]
857
- public struct POINT {
858
- public int X;
859
- public int Y;
860
- }
861
-
862
- [DllImport("user32.dll")]
863
- public static extern bool GetCursorPos(out POINT lpPoint);
864
-
865
- const uint DESKTOP_ACCESS = 0x01FF | 0x00020000;
866
- const int SM_CXSCREEN = 0;
867
- const int SM_CYSCREEN = 1;
868
- const int SRCCOPY = 0x00CC0020;
869
- const int CAPTUREBLT = 0x40000000;
870
-
871
- public static WinCaptureResult GetInfo() {
872
- var res = new WinCaptureResult();
873
- res.success = true;
874
- res.error = "";
875
- var visibleWindows = new List<string>();
876
-
877
- Thread worker = new Thread(() => {
878
- IntPtr hDesk = OpenInputDesktop(0, false, DESKTOP_ACCESS);
879
- if (hDesk == IntPtr.Zero) {
880
- hDesk = OpenDesktop("Default", 0, false, DESKTOP_ACCESS);
881
- }
882
- if (hDesk != IntPtr.Zero) {
883
- SetThreadDesktop(hDesk);
884
- }
885
-
886
- res.width = GetSystemMetrics(SM_CXSCREEN);
887
- res.height = GetSystemMetrics(SM_CYSCREEN);
888
- if (res.width <= 0) res.width = 1920;
889
- if (res.height <= 0) res.height = 1080;
890
-
891
- POINT pt;
892
- if (GetCursorPos(out pt)) {
893
- res.cursorX = pt.X;
894
- res.cursorY = pt.Y;
895
- }
896
-
897
- IntPtr fg = GetForegroundWindow();
898
- if (fg != IntPtr.Zero) {
899
- var sbFg = new StringBuilder(256);
900
- GetWindowText(fg, sbFg, 256);
901
- res.activeWindow = sbFg.ToString().Trim();
902
- } else {
903
- res.activeWindow = "";
904
- }
905
-
906
- EnumWindows((hWnd, lParam) => {
907
- if (IsWindowVisible(hWnd)) {
908
- var title = new StringBuilder(256);
909
- GetWindowText(hWnd, title, 256);
910
- string t = title.ToString().Trim();
911
- if (t.Length > 0 && !visibleWindows.Contains(t) && t != "Program Manager" && t != "Windows Input Experience") {
912
- visibleWindows.Add(t);
913
- }
914
- }
915
- return true;
916
- }, IntPtr.Zero);
917
-
918
- if (hDesk != IntPtr.Zero) {
919
- CloseDesktop(hDesk);
920
- }
921
- });
922
-
923
- worker.SetApartmentState(ApartmentState.STA);
924
- worker.Start();
925
- worker.Join();
926
-
927
- res.visibleWindows = visibleWindows.ToArray();
928
- return res;
929
- }
930
-
931
- public static WinCaptureResult Capture(string filePath) {
932
- var res = new WinCaptureResult();
933
- res.savedPath = filePath;
934
- res.error = "";
935
- var visibleWindows = new List<string>();
936
-
937
- Thread worker = new Thread(() => {
938
- IntPtr hDesk = OpenInputDesktop(0, false, DESKTOP_ACCESS);
939
- if (hDesk == IntPtr.Zero) {
940
- hDesk = OpenDesktop("Default", 0, false, DESKTOP_ACCESS);
941
- }
942
- if (hDesk != IntPtr.Zero) {
943
- SetThreadDesktop(hDesk);
944
- }
945
-
946
- res.width = GetSystemMetrics(SM_CXSCREEN);
947
- res.height = GetSystemMetrics(SM_CYSCREEN);
948
- if (res.width <= 0) res.width = 1920;
949
- if (res.height <= 0) res.height = 1080;
950
-
951
- POINT pt;
952
- if (GetCursorPos(out pt)) {
953
- res.cursorX = pt.X;
954
- res.cursorY = pt.Y;
955
- }
956
-
957
- IntPtr fg = GetForegroundWindow();
958
- if (fg != IntPtr.Zero) {
959
- var sbFg = new StringBuilder(256);
960
- GetWindowText(fg, sbFg, 256);
961
- res.activeWindow = sbFg.ToString().Trim();
962
- } else {
963
- res.activeWindow = "";
964
- }
965
-
966
- EnumWindows((hWnd, lParam) => {
967
- if (IsWindowVisible(hWnd)) {
968
- var title = new StringBuilder(256);
969
- GetWindowText(hWnd, title, 256);
970
- string t = title.ToString().Trim();
971
- if (t.Length > 0 && !visibleWindows.Contains(t) && t != "Program Manager" && t != "Windows Input Experience") {
972
- visibleWindows.Add(t);
973
- }
974
- }
975
- return true;
976
- }, IntPtr.Zero);
977
-
978
- try {
979
- IntPtr hdcSrc = GetDC(IntPtr.Zero);
980
- IntPtr hdcDest = CreateCompatibleDC(hdcSrc);
981
- IntPtr hBitmap = CreateCompatibleBitmap(hdcSrc, res.width, res.height);
982
- IntPtr hOld = SelectObject(hdcDest, hBitmap);
983
- bool bltOk = BitBlt(hdcDest, 0, 0, res.width, res.height, hdcSrc, 0, 0, SRCCOPY | CAPTUREBLT);
984
-
985
- SelectObject(hdcDest, hOld);
986
- DeleteDC(hdcDest);
987
- ReleaseDC(IntPtr.Zero, hdcSrc);
988
-
989
- if (bltOk) {
990
- using (Bitmap bmp = Bitmap.FromHbitmap(hBitmap)) {
991
- bmp.Save(filePath, ImageFormat.Png);
992
- }
993
- res.success = true;
994
- } else {
995
- res.error = "BitBlt failed";
996
- }
997
- DeleteObject(hBitmap);
998
- } catch (Exception ex) {
999
- res.error = ex.Message;
1000
- }
1001
-
1002
- if (hDesk != IntPtr.Zero) {
1003
- CloseDesktop(hDesk);
1004
- }
1005
- });
1006
-
1007
- worker.SetApartmentState(ApartmentState.STA);
1008
- worker.Start();
1009
- worker.Join();
1010
-
1011
- res.visibleWindows = visibleWindows.ToArray();
1012
- return res;
1013
- }
1014
- }
1015
- "@
1016
-
1017
- Add-Type -TypeDefinition $source -ReferencedAssemblies System.Drawing, System.Windows.Forms
1018
-
1019
- if ($InfoOnly) {
1020
- [WinCaptureHelper]::GetInfo() | ConvertTo-Json -Compress
1021
- } else {
1022
- [WinCaptureHelper]::Capture($OutPath) | ConvertTo-Json -Compress
1023
- }
1024
- `;
1025
- export function ensureWinCaptureScript() {
1026
- const scriptsDir = path.join(process.cwd(), '.ft', 'scripts');
1027
- if (!fs.existsSync(scriptsDir)) {
1028
- fs.mkdirSync(scriptsDir, { recursive: true });
1029
- }
1030
- const scriptPath = path.join(scriptsDir, 'winCapture.ps1');
1031
- fs.writeFileSync(scriptPath, WIN_CAPTURE_SCRIPT_CONTENT, 'utf8');
1032
- return scriptPath;
1033
- }
1034
- /**
1035
- * Get screen metrics, active window title, visible windows, and cursor position.
1036
- */
1037
- export async function getScreenInfo(appName) {
1038
- const platform = process.platform;
1039
- if (platform === 'win32') {
1040
- try {
1041
- const scriptPath = ensureWinCaptureScript();
1042
- const { stdout } = await execAsync(`powershell -NoProfile -ExecutionPolicy Bypass -File "${scriptPath}" -InfoOnly`);
1043
- const data = JSON.parse(stdout.trim());
1044
- const visList = Array.isArray(data.visibleWindows) && data.visibleWindows.length > 0
1045
- ? ` | Visible Windows: ${data.visibleWindows.slice(0, 5).join(', ')}`
1046
- : '';
1047
- return {
1048
- success: true,
1049
- output: `Screen Info: Resolution ${data.screenWidth}x${data.screenHeight} | Cursor at (${data.cursorX}, ${data.cursorY}) | Active Window: "${data.activeWindow || appName || 'Desktop'}"${visList}`,
1050
- details: {
1051
- ...data,
1052
- activeWindow: data.activeWindow || appName || 'Desktop',
1053
- },
1054
- };
1055
- }
1056
- catch {
1057
- return {
1058
- success: true,
1059
- output: `Screen Info (Default): Resolution 1920x1080 | Active Window: "${appName || 'Desktop'}"`,
1060
- details: { screenWidth: 1920, screenHeight: 1080, activeWindow: appName || 'Desktop', cursorX: 0, cursorY: 0 },
1061
- };
1062
- }
1063
- }
1064
- else {
1065
- return {
1066
- success: true,
1067
- output: `Screen Info (${platform}): Primary display active | Target App: "${appName || 'Desktop'}"`,
1068
- details: { platform, appName: appName || 'Desktop' },
1069
- };
1070
- }
1071
- }
1072
- function escapeXaml(text) {
1073
- return text
1074
- .replace(/&/g, '&amp;')
1075
- .replace(/</g, '&lt;')
1076
- .replace(/>/g, '&gt;')
1077
- .replace(/"/g, '&quot;')
1078
- .replace(/'/g, '&apos;')
1079
- .replace(/•/g, '&#x2022;');
1080
- }
1081
- let persistentAuraProcess = null;
1082
- let persistentAuraFlagPath = null;
1083
- /**
1084
- * Start persistent futuristic sky-blue aura overlay on screen borders and corners.
1085
- * Stays continuously active and animating as long as live screen sharing is running.
1086
- */
1087
- export async function startScreenAuraOverlay(message = "Live Screen Share Active • Streaming to Model") {
1088
- const platform = process.platform;
1089
- if (platform !== 'win32')
1090
- return;
1091
- // Stop any existing overlay first
1092
- stopScreenAuraOverlay();
1093
- try {
1094
- const scriptsDir = path.join(process.cwd(), '.ft', 'scripts');
1095
- if (!fs.existsSync(scriptsDir)) {
1096
- fs.mkdirSync(scriptsDir, { recursive: true });
1097
- }
1098
- const psFile = path.join(scriptsDir, 'scout_aura_persistent.ps1');
1099
- const flagFile = path.join(scriptsDir, 'aura_active.flag');
1100
- fs.writeFileSync(flagFile, String(Date.now()), 'utf8');
1101
- persistentAuraFlagPath = flagFile;
1102
- const subtitleText = " Perception Active";
1103
- const safeMessage = escapeXaml(message);
1104
- const safeSubtitle = escapeXaml(subtitleText);
1105
- const parentPid = process.pid;
1106
- const overlayPs = `
1107
- Add-Type -AssemblyName PresentationFramework, PresentationCore, WindowsBase
1108
-
1109
- [xml]$xaml = @"
1110
- <Window xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
1111
- xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
1112
- Title="Scout Aura Overlay"
1113
- WindowStyle="None"
1114
- AllowsTransparency="True"
1115
- Background="Transparent"
1116
- Topmost="True"
1117
- ShowInTaskbar="False"
1118
- ShowActivated="False"
1119
- IsHitTestVisible="False">
1120
- <Window.Resources>
1121
- <Storyboard x:Key="AuraPulse" RepeatBehavior="Forever" AutoReverse="True">
1122
- <DoubleAnimation Storyboard.TargetName="AuraBorder" Storyboard.TargetProperty="Opacity"
1123
- From="0.4" To="1.0" Duration="0:0:0.6"/>
1124
- <DoubleAnimation Storyboard.TargetName="CornerTL" Storyboard.TargetProperty="Opacity"
1125
- From="0.5" To="1.0" Duration="0:0:0.6"/>
1126
- <DoubleAnimation Storyboard.TargetName="CornerTR" Storyboard.TargetProperty="Opacity"
1127
- From="0.5" To="1.0" Duration="0:0:0.6"/>
1128
- <DoubleAnimation Storyboard.TargetName="CornerBL" Storyboard.TargetProperty="Opacity"
1129
- From="0.5" To="1.0" Duration="0:0:0.6"/>
1130
- <DoubleAnimation Storyboard.TargetName="CornerBR" Storyboard.TargetProperty="Opacity"
1131
- From="0.5" To="1.0" Duration="0:0:0.6"/>
1132
- <DoubleAnimation Storyboard.TargetName="AuraGlow" Storyboard.TargetProperty="Opacity"
1133
- From="0.25" To="0.9" Duration="0:0:0.6"/>
1134
- </Storyboard>
1135
- </Window.Resources>
1136
-
1137
- <Grid IsHitTestVisible="False">
1138
- <!-- Outer Glowing Sky Blue Aura Border -->
1139
- <Border x:Name="AuraGlow" BorderThickness="6" BorderBrush="#00d8ff" Margin="0">
1140
- <Border.Effect>
1141
- <DropShadowEffect Color="#00e5ff" BlurRadius="45" ShadowDepth="0" Opacity="0.95"/>
1142
- </Border.Effect>
1143
- </Border>
1144
- <Border x:Name="AuraBorder" BorderThickness="3" BorderBrush="#38bdf8" Margin="0">
1145
- <Border.Effect>
1146
- <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
1147
- </Border.Effect>
1148
- </Border>
1149
-
1150
- <!-- Sky Blue Corner Brackets with Aura Glow -->
1151
- <!-- Top Left Corner -->
1152
- <Canvas HorizontalAlignment="Left" VerticalAlignment="Top" Width="120" Height="120" Margin="6,6,0,0">
1153
- <Path x:Name="CornerTL" Data="M 0,90 L 0,0 L 90,0" Stroke="#00e5ff" StrokeThickness="6">
1154
- <Path.Effect>
1155
- <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
1156
- </Path.Effect>
1157
- </Path>
1158
- </Canvas>
1159
-
1160
- <!-- Top Right Corner -->
1161
- <Canvas HorizontalAlignment="Right" VerticalAlignment="Top" Width="120" Height="120" Margin="0,6,6,0">
1162
- <Path x:Name="CornerTR" Data="M 30,0 L 120,0 L 120,90" Stroke="#00e5ff" StrokeThickness="6">
1163
- <Path.Effect>
1164
- <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
1165
- </Path.Effect>
1166
- </Path>
1167
- </Canvas>
1168
-
1169
- <!-- Bottom Left Corner -->
1170
- <Canvas HorizontalAlignment="Left" VerticalAlignment="Bottom" Width="120" Height="120" Margin="6,0,0,6">
1171
- <Path x:Name="CornerBL" Data="M 0,30 L 0,120 L 90,120" Stroke="#00e5ff" StrokeThickness="6">
1172
- <Path.Effect>
1173
- <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
1174
- </Path.Effect>
1175
- </Path>
1176
- </Canvas>
1177
-
1178
- <!-- Bottom Right Corner -->
1179
- <Canvas HorizontalAlignment="Right" VerticalAlignment="Bottom" Width="120" Height="120" Margin="0,0,6,6">
1180
- <Path x:Name="CornerBR" Data="M 30,120 L 120,120 L 120,30" Stroke="#00e5ff" StrokeThickness="6">
1181
- <Path.Effect>
1182
- <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
1183
- </Path.Effect>
1184
- </Path>
1185
- </Canvas>
1186
-
1187
- <!-- Top Floating Status Badge -->
1188
- <Border HorizontalAlignment="Center" VerticalAlignment="Top" Margin="0,22,0,0"
1189
- Background="#EE03121F" BorderBrush="#00e5ff" BorderThickness="1.8" CornerRadius="24" Padding="26,10">
1190
- <Border.Effect>
1191
- <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="0.95"/>
1192
- </Border.Effect>
1193
- <StackPanel Orientation="Horizontal" VerticalAlignment="Center">
1194
- <Ellipse Width="12" Height="12" Fill="#00e5ff" Margin="0,0,12,0">
1195
- <Ellipse.Effect>
1196
- <DropShadowEffect Color="#00e5ff" BlurRadius="12" ShadowDepth="0"/>
1197
- </Ellipse.Effect>
1198
- </Ellipse>
1199
- <TextBlock Text="${safeMessage}" Foreground="#ffffff" FontSize="16" FontWeight="Bold" VerticalAlignment="Center"/>
1200
- <TextBlock Text="${safeSubtitle}" Foreground="#38bdf8" FontSize="13.5" FontWeight="SemiBold" Margin="10,0,0,0" VerticalAlignment="Center"/>
1201
- </StackPanel>
1202
- </Border>
1203
- </Grid>
1204
- </Window>
1205
- "@
1206
-
1207
- $reader = (New-Object System.Xml.XmlNodeReader $xaml)
1208
- $window = [System.Windows.Markup.XamlReader]::Load($reader)
1209
-
1210
- $window.Left = [System.Windows.SystemParameters]::VirtualScreenLeft
1211
- $window.Top = [System.Windows.SystemParameters]::VirtualScreenTop
1212
- $window.Width = [System.Windows.SystemParameters]::VirtualScreenWidth
1213
- $window.Height = [System.Windows.SystemParameters]::VirtualScreenHeight
1214
- $window.Topmost = $true
1215
- $window.ShowActivated = $false
1216
-
1217
- $storyboard = $window.Resources["AuraPulse"]
1218
- $window.Add_Loaded({
1219
- $storyboard.Begin($window)
1220
- })
1221
-
1222
- $flagPath = "${flagFile.replace(/\\/g, '\\\\')}"
1223
- $parentPid = ${parentPid}
1224
-
1225
- # Check if screen sharing is active or parent exited every 350ms
1226
- $timer = New-Object System.Windows.Threading.DispatcherTimer
1227
- $timer.Interval = [TimeSpan]::FromMilliseconds(350)
1228
- $timer.Add_Tick({
1229
- $shouldClose = $false
1230
- if (-not (Test-Path $flagPath)) {
1231
- $shouldClose = $true
1232
- } else {
1233
- $proc = Get-Process -Id $parentPid -ErrorAction SilentlyContinue
1234
- if (-not $proc) {
1235
- $shouldClose = $true
1236
- }
1237
- }
1238
- if ($shouldClose) {
1239
- $timer.Stop()
1240
- $window.Close()
1241
- }
1242
- })
1243
- $timer.Start()
1244
-
1245
- [void]$window.ShowDialog()
1246
- `;
1247
- fs.writeFileSync(psFile, '\uFEFF' + overlayPs, 'utf8');
1248
- const child = spawn('powershell', ['-WindowStyle', 'Hidden', '-NoProfile', '-Sta', '-ExecutionPolicy', 'Bypass', '-File', psFile], {
1249
- detached: false,
1250
- stdio: 'ignore',
1251
- });
1252
- child.unref();
1253
- persistentAuraProcess = child;
1254
- }
1255
- catch { }
1256
- }
1257
- /**
1258
- * Stop persistent screen aura animation when live screen sharing stops.
1259
- */
1260
- export function stopScreenAuraOverlay() {
1261
- if (persistentAuraFlagPath && fs.existsSync(persistentAuraFlagPath)) {
1262
- try {
1263
- fs.unlinkSync(persistentAuraFlagPath);
1264
- }
1265
- catch { }
1266
- persistentAuraFlagPath = null;
1267
- }
1268
- if (persistentAuraProcess) {
1269
- try {
1270
- persistentAuraProcess.kill();
1271
- }
1272
- catch { }
1273
- persistentAuraProcess = null;
1274
- }
1275
- }
1276
- /**
1277
- * Display a futuristic sky-blue aura animation on screen borders and corners,
1278
- * show the warning "Scout is on the screen.", and optionally block external inputs.
1279
- * If durationMs <= 0, aura stays on continuously until stopScreenAuraOverlay() is called.
1280
- */
1281
- export async function showScreenAuraOverlay(durationMs = 2200, message = "Scout is viewing the screen.", blockInput = false) {
1282
- if (durationMs <= 0) {
1283
- return await startScreenAuraOverlay(message);
1284
- }
1285
- const platform = process.platform;
1286
- if (platform !== 'win32')
1287
- return;
1288
- try {
1289
- const scriptsDir = path.join(process.cwd(), '.ft', 'scripts');
1290
- if (!fs.existsSync(scriptsDir)) {
1291
- fs.mkdirSync(scriptsDir, { recursive: true });
1292
- }
1293
- const psFile = path.join(scriptsDir, 'scout_aura_overlay.ps1');
1294
- const blockInputSnippet = blockInput ? `
1295
- $sig = '[DllImport("user32.dll")] public static extern bool BlockInput(bool fBlock);'
1296
- $b = Add-Type -memberDefinition $sig -name 'Win32AuraBlock' -namespace Win32Functions -passThru
1297
- try { [void]$b::BlockInput($true) } catch {}
1298
- ` : '';
1299
- const unblockSnippet = blockInput ? `
1300
- try { [void]$b::BlockInput($false) } catch {}
1301
- ` : '';
1302
- const subtitleText = blockInput ? " • External Input Locked" : " Perception Active";
1303
- const safeMessage = escapeXaml(message);
1304
- const safeSubtitle = escapeXaml(subtitleText);
1305
- const overlayPs = `
1306
- Add-Type -AssemblyName PresentationFramework, PresentationCore, WindowsBase
1307
- ${blockInputSnippet}
1308
-
1309
- [xml]$xaml = @"
1310
- <Window xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
1311
- xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
1312
- Title="Scout Aura Overlay"
1313
- WindowStyle="None"
1314
- AllowsTransparency="True"
1315
- Background="Transparent"
1316
- Topmost="True"
1317
- ShowInTaskbar="False"
1318
- ShowActivated="False"
1319
- IsHitTestVisible="False">
1320
- <Window.Resources>
1321
- <Storyboard x:Key="AuraPulse" RepeatBehavior="Forever" AutoReverse="True">
1322
- <DoubleAnimation Storyboard.TargetName="AuraBorder" Storyboard.TargetProperty="Opacity"
1323
- From="0.4" To="1.0" Duration="0:0:0.6"/>
1324
- <DoubleAnimation Storyboard.TargetName="CornerTL" Storyboard.TargetProperty="Opacity"
1325
- From="0.5" To="1.0" Duration="0:0:0.6"/>
1326
- <DoubleAnimation Storyboard.TargetName="CornerTR" Storyboard.TargetProperty="Opacity"
1327
- From="0.5" To="1.0" Duration="0:0:0.6"/>
1328
- <DoubleAnimation Storyboard.TargetName="CornerBL" Storyboard.TargetProperty="Opacity"
1329
- From="0.5" To="1.0" Duration="0:0:0.6"/>
1330
- <DoubleAnimation Storyboard.TargetName="CornerBR" Storyboard.TargetProperty="Opacity"
1331
- From="0.5" To="1.0" Duration="0:0:0.6"/>
1332
- <DoubleAnimation Storyboard.TargetName="AuraGlow" Storyboard.TargetProperty="Opacity"
1333
- From="0.25" To="0.9" Duration="0:0:0.6"/>
1334
- </Storyboard>
1335
- </Window.Resources>
1336
-
1337
- <Grid IsHitTestVisible="False">
1338
- <!-- Outer Glowing Sky Blue Aura Border -->
1339
- <Border x:Name="AuraGlow" BorderThickness="6" BorderBrush="#00d8ff" Margin="0">
1340
- <Border.Effect>
1341
- <DropShadowEffect Color="#00e5ff" BlurRadius="45" ShadowDepth="0" Opacity="0.95"/>
1342
- </Border.Effect>
1343
- </Border>
1344
- <Border x:Name="AuraBorder" BorderThickness="3" BorderBrush="#38bdf8" Margin="0">
1345
- <Border.Effect>
1346
- <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
1347
- </Border.Effect>
1348
- </Border>
1349
-
1350
- <!-- Sky Blue Corner Brackets with Aura Glow -->
1351
- <!-- Top Left Corner -->
1352
- <Canvas HorizontalAlignment="Left" VerticalAlignment="Top" Width="120" Height="120" Margin="6,6,0,0">
1353
- <Path x:Name="CornerTL" Data="M 0,90 L 0,0 L 90,0" Stroke="#00e5ff" StrokeThickness="6">
1354
- <Path.Effect>
1355
- <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
1356
- </Path.Effect>
1357
- </Path>
1358
- </Canvas>
1359
-
1360
- <!-- Top Right Corner -->
1361
- <Canvas HorizontalAlignment="Right" VerticalAlignment="Top" Width="120" Height="120" Margin="0,6,6,0">
1362
- <Path x:Name="CornerTR" Data="M 30,0 L 120,0 L 120,90" Stroke="#00e5ff" StrokeThickness="6">
1363
- <Path.Effect>
1364
- <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
1365
- </Path.Effect>
1366
- </Path>
1367
- </Canvas>
1368
-
1369
- <!-- Bottom Left Corner -->
1370
- <Canvas HorizontalAlignment="Left" VerticalAlignment="Bottom" Width="120" Height="120" Margin="6,0,0,6">
1371
- <Path x:Name="CornerBL" Data="M 0,30 L 0,120 L 90,120" Stroke="#00e5ff" StrokeThickness="6">
1372
- <Path.Effect>
1373
- <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
1374
- </Path.Effect>
1375
- </Path>
1376
- </Canvas>
1377
-
1378
- <!-- Bottom Right Corner -->
1379
- <Canvas HorizontalAlignment="Right" VerticalAlignment="Bottom" Width="120" Height="120" Margin="0,0,6,6">
1380
- <Path x:Name="CornerBR" Data="M 30,120 L 120,120 L 120,30" Stroke="#00e5ff" StrokeThickness="6">
1381
- <Path.Effect>
1382
- <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
1383
- </Path.Effect>
1384
- </Path>
1385
- </Canvas>
1386
-
1387
- <!-- Top Floating Status Badge -->
1388
- <Border HorizontalAlignment="Center" VerticalAlignment="Top" Margin="0,22,0,0"
1389
- Background="#EE03121F" BorderBrush="#00e5ff" BorderThickness="1.8" CornerRadius="24" Padding="26,10">
1390
- <Border.Effect>
1391
- <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="0.95"/>
1392
- </Border.Effect>
1393
- <StackPanel Orientation="Horizontal" VerticalAlignment="Center">
1394
- <Ellipse Width="12" Height="12" Fill="#00e5ff" Margin="0,0,12,0">
1395
- <Ellipse.Effect>
1396
- <DropShadowEffect Color="#00e5ff" BlurRadius="12" ShadowDepth="0"/>
1397
- </Ellipse.Effect>
1398
- </Ellipse>
1399
- <TextBlock Text="${safeMessage}" Foreground="#ffffff" FontSize="16" FontWeight="Bold" VerticalAlignment="Center"/>
1400
- <TextBlock Text="${safeSubtitle}" Foreground="#38bdf8" FontSize="13.5" FontWeight="SemiBold" Margin="10,0,0,0" VerticalAlignment="Center"/>
1401
- </StackPanel>
1402
- </Border>
1403
- </Grid>
1404
- </Window>
1405
- "@
1406
-
1407
- $reader = (New-Object System.Xml.XmlNodeReader $xaml)
1408
- $window = [System.Windows.Markup.XamlReader]::Load($reader)
1409
-
1410
- $window.Left = [System.Windows.SystemParameters]::VirtualScreenLeft
1411
- $window.Top = [System.Windows.SystemParameters]::VirtualScreenTop
1412
- $window.Width = [System.Windows.SystemParameters]::VirtualScreenWidth
1413
- $window.Height = [System.Windows.SystemParameters]::VirtualScreenHeight
1414
- $window.Topmost = $true
1415
- $window.ShowActivated = $false
1416
-
1417
- $storyboard = $window.Resources["AuraPulse"]
1418
- $window.Add_Loaded({
1419
- $storyboard.Begin($window)
1420
- })
1421
-
1422
- # Safety timer to close overlay and guarantee unblock
1423
- $timer = New-Object System.Windows.Threading.DispatcherTimer
1424
- $timer.Interval = [TimeSpan]::FromMilliseconds(${durationMs})
1425
- $timer.Add_Tick({
1426
- $timer.Stop()
1427
- ${unblockSnippet}
1428
- $window.Close()
1429
- })
1430
- $timer.Start()
1431
-
1432
- [void]$window.ShowDialog()
1433
- ${unblockSnippet}
1434
- `;
1435
- fs.writeFileSync(psFile, '\uFEFF' + overlayPs, 'utf8');
1436
- await execAsync(`powershell -WindowStyle Hidden -NoProfile -Sta -ExecutionPolicy Bypass -File "${psFile}"`, { timeout: durationMs + 3000 });
1437
- }
1438
- catch {
1439
- if (blockInput) {
1440
- try {
1441
- await execAsync(`powershell -NoProfile -Command "$sig='[DllImport(\\"user32.dll\\")] public static extern bool BlockInput(bool f);';$b=Add-Type -m $sig -n B -p;[void]$b::BlockInput($false)"`);
1442
- }
1443
- catch { }
1444
- }
1445
- }
1446
- }
1447
- const capturedScreenshots = new Set();
1448
- /**
1449
- * Register a captured screenshot file path for automatic post-task cleanup.
1450
- */
1451
- export function registerCapturedScreenshot(filePath) {
1452
- if (filePath) {
1453
- capturedScreenshots.add(path.resolve(filePath));
1454
- }
1455
- }
1456
- /**
1457
- * Clean up all captured screenshots permanently from disk after task completion.
1458
- */
1459
- export function cleanupScreenshots() {
1460
- let deletedCount = 0;
1461
- // 1. Delete all tracked screenshots
1462
- for (const filePath of capturedScreenshots) {
1463
- try {
1464
- if (fs.existsSync(filePath)) {
1465
- fs.unlinkSync(filePath);
1466
- deletedCount++;
1467
- }
1468
- }
1469
- catch { }
1470
- }
1471
- capturedScreenshots.clear();
1472
- // 2. Clean any remaining files in .ft/screenshots
1473
- try {
1474
- const screenshotsDir = path.join(process.cwd(), '.ft', 'screenshots');
1475
- if (fs.existsSync(screenshotsDir)) {
1476
- const files = fs.readdirSync(screenshotsDir);
1477
- for (const file of files) {
1478
- if (file.startsWith('screen_') && (file.endsWith('.png') || file.endsWith('.jpg'))) {
1479
- try {
1480
- fs.unlinkSync(path.join(screenshotsDir, file));
1481
- deletedCount++;
1482
- }
1483
- catch { }
1484
- }
1485
- }
1486
- }
1487
- }
1488
- catch { }
1489
- return { success: true, deletedCount };
1490
- }
1491
- /**
1492
- * Capture full desktop screen or specified window screenshot without running shell commands.
1493
- * Screenshots are tracked and will be deleted automatically upon task completion.
1494
- */
1495
- export async function captureScreen(appNameOrScreen) {
1496
- const platform = process.platform;
1497
- const screenshotsDir = path.join(process.cwd(), '.ft', 'screenshots');
1498
- try {
1499
- if (!fs.existsSync(screenshotsDir)) {
1500
- fs.mkdirSync(screenshotsDir, { recursive: true });
1501
- }
1502
- }
1503
- catch { }
1504
- const filename = `screen_${Date.now()}.png`;
1505
- const outPath = path.join(screenshotsDir, filename);
1506
- if (platform === 'win32') {
1507
- try {
1508
- const scriptPath = ensureWinCaptureScript();
1509
- const { stdout } = await execAsync(`powershell -NoProfile -ExecutionPolicy Bypass -File "${scriptPath}" -OutPath "${outPath}"`);
1510
- const meta = JSON.parse(stdout.trim());
1511
- if (meta.success && fs.existsSync(outPath) && fs.statSync(outPath).size > 0) {
1512
- registerCapturedScreenshot(outPath);
1513
- showScreenAuraOverlay(1800, "Scout is viewing the screen...", false).catch(() => { });
1514
- const visibleList = Array.isArray(meta.visibleWindows) && meta.visibleWindows.length > 0
1515
- ? `\n- Visible Windows: ${meta.visibleWindows.slice(0, 6).map((w) => `"${w}"`).join(', ')}`
1516
- : '';
1517
- return {
1518
- success: true,
1519
- output: `Screen Captured Successfully!\n- Screenshot Saved To: ${outPath}\n- Resolution: ${meta.width}x${meta.height}\n- Active Window: "${meta.activeWindow || 'Desktop'}"${visibleList}\n- Seeing Status: Screen is visible and captured. (NOTE: This temporary screenshot will be automatically deleted upon task completion).`,
1520
- details: { ...meta, outPath },
1521
- };
1522
- }
1523
- throw new Error(meta.error || 'Desktop capture returned false');
1524
- }
1525
- catch (err) {
1526
- // Fallback to Graphics.CopyFromScreen
1527
- const psScript = `
1528
- Add-Type -AssemblyName System.Windows.Forms
1529
- Add-Type -AssemblyName System.Drawing
1530
- $screen = [System.Windows.Forms.Screen]::PrimaryScreen
1531
- $bounds = $screen.Bounds
1532
- $bmp = New-Object System.Drawing.Bitmap($bounds.Width, $bounds.Height)
1533
- $graphics = [System.Drawing.Graphics]::FromImage($bmp)
1534
- try { $graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size) } catch {}
1535
- $bmp.Save("${outPath.replace(/\\/g, '\\\\')}", [System.Drawing.Imaging.ImageFormat]::Png)
1536
- $graphics.Dispose()
1537
- $bmp.Dispose()
1538
- @{ savedPath = "${outPath.replace(/\\/g, '\\\\')}"; width = $bounds.Width; height = $bounds.Height; activeWindow = "${appNameOrScreen || 'Desktop'}" } | ConvertTo-Json
1539
- `;
1540
- try {
1541
- const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
1542
- const { stdout } = await execAsync(`powershell -NoProfile -EncodedCommand ${encodedScript}`);
1543
- const meta = JSON.parse(stdout.trim());
1544
- registerCapturedScreenshot(outPath);
1545
- return {
1546
- success: true,
1547
- output: `Screen Captured Successfully!\n- Screenshot Saved To: ${outPath}\n- Resolution: ${meta.width}x${meta.height}\n- Active Window: "${meta.activeWindow || 'Desktop'}"\n- Seeing Status: Screen is visible and captured.`,
1548
- details: { ...meta, outPath },
1549
- };
1550
- }
1551
- catch (fallbackErr) {
1552
- return {
1553
- success: false,
1554
- output: `Failed to capture screen: ${err?.message || fallbackErr?.message || String(err)}`,
1555
- };
1556
- }
1557
- }
1558
- }
1559
- else if (platform === 'darwin') {
1560
- try {
1561
- await execAsync(`screencapture -x "${outPath}"`);
1562
- registerCapturedScreenshot(outPath);
1563
- return {
1564
- success: true,
1565
- output: `macOS Screen Captured Successfully! Saved To: ${outPath}\n(NOTE: This temporary screenshot will be automatically deleted upon task completion).`,
1566
- details: { outPath, platform: 'darwin' },
1567
- };
1568
- }
1569
- catch (err) {
1570
- return {
1571
- success: false,
1572
- output: `Failed macOS screen capture: ${err?.message || String(err)}`,
1573
- };
1574
- }
1575
- }
1576
- else {
1577
- try {
1578
- await execAsync(`import -window root "${outPath}" || scrot "${outPath}"`);
1579
- registerCapturedScreenshot(outPath);
1580
- return {
1581
- success: true,
1582
- output: `Linux Screen Captured Successfully! Saved To: ${outPath}\n(NOTE: This temporary screenshot will be automatically deleted upon task completion).`,
1583
- details: { outPath, platform: 'linux' },
1584
- };
1585
- }
1586
- catch (err) {
1587
- return {
1588
- success: false,
1589
- output: `Failed Linux screen capture: ${err?.message || String(err)}`,
1590
- };
1591
- }
1592
- }
1593
- }
1594
- /**
1595
- * Call Vision AI with graceful multi-provider fallback.
1596
- * Tries NVIDIA Vision (meta/llama-3.2-11b-vision-instruct), Gemini (gemini-2.0-flash), OpenAI (gpt-4o-mini), and BYOK.
1597
- */
1598
- export async function callVisionAi(outPath, prompt, maxTokens = 350) {
1599
- if (!outPath || !fs.existsSync(outPath))
1600
- return null;
1601
- let imageBase64;
1602
- try {
1603
- imageBase64 = fs.readFileSync(outPath).toString('base64');
1604
- }
1605
- catch {
1606
- return null;
1607
- }
1608
- const nvidiaKey = process.env.NVIDIA_API_KEY || process.env.NVCF_API_KEY;
1609
- const geminiKey = process.env.GEMINI_API_KEY;
1610
- const openaiKey = process.env.OPENAI_API_KEY;
1611
- const byokKey = process.env.BYOK_API_KEY;
1612
- const candidates = [];
1613
- if (nvidiaKey) {
1614
- candidates.push({ apiKey: nvidiaKey, baseURL: 'https://integrate.api.nvidia.com/v1', model: 'meta/llama-3.2-11b-vision-instruct', timeout: 35000 });
1615
- }
1616
- if (geminiKey) {
1617
- candidates.push({ apiKey: geminiKey, baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', model: 'gemini-2.0-flash', timeout: 20000 });
1618
- }
1619
- if (openaiKey) {
1620
- candidates.push({ apiKey: openaiKey, model: 'gpt-4o-mini', timeout: 20000 });
1621
- }
1622
- if (byokKey) {
1623
- candidates.push({ apiKey: byokKey, model: 'gpt-4o-mini', timeout: 20000 });
1624
- }
1625
- for (const cand of candidates) {
1626
- try {
1627
- const client = new OpenAI({ apiKey: cand.apiKey, baseURL: cand.baseURL, timeout: cand.timeout });
1628
- const resp = await client.chat.completions.create({
1629
- model: cand.model,
1630
- messages: [
1631
- {
1632
- role: 'user',
1633
- content: [
1634
- { type: 'text', text: prompt },
1635
- { type: 'image_url', image_url: { url: `data:image/png;base64,${imageBase64}` } },
1636
- ],
1637
- },
1638
- ],
1639
- max_tokens: maxTokens,
1640
- });
1641
- const reply = resp.choices[0]?.message?.content?.trim();
1642
- if (reply) {
1643
- return reply;
1644
- }
1645
- }
1646
- catch {
1647
- continue;
1648
- }
1649
- }
1650
- return null;
1651
- }
1652
- /**
1653
- * Visual screen analysis. Takes a full-screen screenshot and provides visual context
1654
- * (including AI Vision inspection if an API key is available).
1655
- * All temporary screenshots are automatically deleted upon task completion.
1656
- */
1657
- export async function analyzeScreen(appNameOrScreen, targetQuery) {
1658
- const capture = await captureScreen(appNameOrScreen);
1659
- const info = await getScreenInfo(appNameOrScreen);
1660
- let visualAiAnalysis = '';
1661
- const outPath = capture.details?.outPath;
1662
- // Optional Vision Model Analysis if screenshot exists and has valid content
1663
- if (capture.success && outPath && fs.existsSync(outPath)) {
1664
- try {
1665
- if (fs.statSync(outPath).size > 0) {
1666
- const prompt = targetQuery
1667
- ? `Describe what is visible on this screen specifically regarding: "${targetQuery}". Keep it concise, noting visible UI elements, open windows, tabs, buttons, or error messages.`
1668
- : `Briefly describe what is currently visible on this desktop screen: active window, main content/tabs, buttons, and system state in 2-3 sentences.`;
1669
- const reply = await callVisionAi(outPath, prompt, 350);
1670
- if (reply && reply.trim()) {
1671
- visualAiAnalysis = `\nAI Visual Screen Analysis:\n${reply.trim()}`;
1672
- }
1673
- }
1674
- }
1675
- catch {
1676
- // Vision API call is optional; fallback gracefully to system metadata
1677
- }
1678
- }
1679
- const activeWin = capture.details?.activeWindow || info.details?.activeWindow || appNameOrScreen || 'Active Desktop';
1680
- const visWins = Array.isArray(capture.details?.visibleWindows) && capture.details.visibleWindows.length > 0
1681
- ? capture.details.visibleWindows
1682
- : (Array.isArray(info.details?.visibleWindows) ? info.details.visibleWindows : []);
1683
- const visibleWindowsText = visWins.length > 0
1684
- ? `\nVisible Open Windows:\n${visWins.slice(0, 8).map((w) => ` - "${w}"`).join('\n')}`
1685
- : '';
1686
- const report = [
1687
- `=== SCOUT SCRATCHPAD DESKTOP SCREEN VISUAL INSPECTION ===`,
1688
- `Target App / Screen Context: "${appNameOrScreen || 'Desktop'}"`,
1689
- `Screen Resolution: ${info.details?.screenWidth || capture.details?.width || 1920}x${info.details?.screenHeight || capture.details?.height || 1080}`,
1690
- `Active Window Focus: "${activeWin}"${visibleWindowsText}`,
1691
- `Mouse Cursor Position: (${info.details?.cursorX || capture.details?.cursorX || 0}, ${info.details?.cursorY || capture.details?.cursorY || 0})`,
1692
- `Screenshot Status: Captured temporarily for visual inspection. (Will be automatically deleted upon task completion)`,
1693
- `Visual Status: Screen is visible and verified.${visualAiAnalysis}`,
1694
- ].join('\n');
1695
- return {
1696
- success: capture.success,
1697
- app: appNameOrScreen || 'desktop',
1698
- action: 'analyze_screen',
1699
- output: report,
1700
- details: { capture: capture.details, info: info.details },
1701
- };
1702
- }
1703
- /**
1704
- * Full Mouse Cursor & Keyboard Takeover on specified screen or window.
1705
- */
1706
- export async function takeoverControl(appName, durationMs = 3000, x, y) {
1707
- const { category, name } = normalizeAppName(appName);
1708
- const targetApp = (name && name !== 'desktop' && name !== 'system') ? name : undefined;
1709
- if (targetApp) {
1710
- setLiveScreenTargetApp(targetApp);
1711
- }
1712
- if (name && name !== 'desktop' && name !== 'system' && category !== 'browser') {
1713
- await openApp(name);
1714
- }
1715
- // Trigger visual sky-blue aura overlay on screen borders & corners with warning badge & input block
1716
- try {
1717
- const auraLabel = targetApp ? `Scout is controlling ${targetApp}...` : "Scout is on the screen.";
1718
- await showScreenAuraOverlay(Math.min(5000, Math.max(800, durationMs)), auraLabel, true);
1719
- startLiveScreenShare({ targetApp }).catch(() => { });
1720
- }
1721
- catch { }
1722
- const mouseRes = await moveAndClickMouse(x, y, 'left', name || appName);
1723
- const lockRes = await lockAppInput(name || appName, durationMs);
1724
- return {
1725
- success: lockRes.success && mouseRes.success,
1726
- output: `TAKEOVER EXECUTED: Scout Agent has taken over mouse cursor, window focus, and keyboard control on specified screen window ("${name || appName}"). Sky-blue aura HUD activated and user interruption locked for ${durationMs}ms. ${mouseRes.output} Agent can view, analyze, type, and click without shell commands.`,
1727
- details: { appName: name || appName, durationMs, x, y, mouseOutput: mouseRes.output },
1728
- };
1729
- }
1730
- /**
1731
- * Release user input lock / takeover.
1732
- */
1733
- export async function releaseTakeover() {
1734
- const platform = process.platform;
1735
- if (platform === 'win32') {
1736
- const psScript = `
1737
- $sig = '[DllImport("user32.dll")] public static extern bool BlockInput(bool fBlock);'
1738
- $b = Add-Type -memberDefinition $sig -name 'Win32Block' -namespace Win32Functions -passThru
1739
- try { [void]$b::BlockInput($false) } catch {}
1740
- `;
1741
- try {
1742
- const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
1743
- await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
1744
- }
1745
- catch { }
1746
- }
1747
- return {
1748
- success: true,
1749
- output: 'Released mouse cursor and keyboard takeover. Control returned to user.',
1750
- };
1751
- }
1752
- /**
1753
- * Focus application window and simulate typing text + pressing keys (Enter/Tab) or hotkey combinations.
1754
- */
1755
- export async function sendKeystrokes(appName, textToType, pressEnter = true, keyCombo) {
1756
- const platform = process.platform;
1757
- const targetText = textToType || '';
1758
- const combo = (keyCombo || '').trim().toLowerCase();
1759
- if (platform === 'win32') {
1760
- const sanitized = targetText.replace(/"/g, '""').replace(/[\r\n]+/g, ' ');
1761
- const sendKeysText = sanitized.replace(/[{}+^~%()\[\]]/g, '{$&}');
1762
- let sendKeysCombo = '';
1763
- if (combo) {
1764
- const cLower = combo.toLowerCase().replace(/\s+/g, '');
1765
- if (cLower.includes('ctrl+enter') || cLower.includes('ctrl+return'))
1766
- sendKeysCombo = '^{ENTER}';
1767
- else if (cLower.includes('alt+enter') || cLower.includes('alt+return'))
1768
- sendKeysCombo = '%{ENTER}';
1769
- else if (cLower.includes('shift+enter') || cLower.includes('shift+return'))
1770
- sendKeysCombo = '+{ENTER}';
1771
- else if (cLower.includes('ctrl+a') || cLower === 'select_all')
1772
- sendKeysCombo = '^a';
1773
- else if (cLower.includes('ctrl+c') || cLower === 'copy')
1774
- sendKeysCombo = '^c';
1775
- else if (cLower.includes('ctrl+v') || cLower === 'paste')
1776
- sendKeysCombo = '^v';
1777
- else if (cLower.includes('ctrl+x') || cLower === 'cut')
1778
- sendKeysCombo = '^x';
1779
- else if (cLower.includes('ctrl+z') || cLower === 'undo')
1780
- sendKeysCombo = '^z';
1781
- else if (cLower.includes('ctrl+s') || cLower === 'save')
1782
- sendKeysCombo = '^s';
1783
- else if (cLower.includes('ctrl+l') || cLower === 'address_bar')
1784
- sendKeysCombo = '^l';
1785
- else if (cLower.includes('ctrl+t') || cLower === 'new_tab')
1786
- sendKeysCombo = '^t';
1787
- else if (cLower.includes('ctrl+w') || cLower === 'close_tab')
1788
- sendKeysCombo = '^w';
1789
- else if (cLower.includes('ctrl+r') || cLower === 'reload')
1790
- sendKeysCombo = '^r';
1791
- else if (cLower.includes('ctrl+f') || cLower === 'find')
1792
- sendKeysCombo = '^f';
1793
- else if (cLower.includes('ctrl+n') || cLower === 'new_window')
1794
- sendKeysCombo = '^n';
1795
- else if (cLower.includes('ctrl+shift+i') || cLower.includes('devtools'))
1796
- sendKeysCombo = '^+i';
1797
- else if (cLower.includes('alt+d'))
1798
- sendKeysCombo = '%d';
1799
- else if (cLower.includes('alt+tab'))
1800
- sendKeysCombo = '%{TAB}';
1801
- else if (cLower.includes('alt+f4'))
1802
- sendKeysCombo = '%{F4}';
1803
- else if (cLower === 'enter' || cLower === 'return')
1804
- sendKeysCombo = '{ENTER}';
1805
- else if (cLower === 'esc' || cLower === 'escape')
1806
- sendKeysCombo = '{ESC}';
1807
- else if (cLower === 'tab')
1808
- sendKeysCombo = '{TAB}';
1809
- else if (cLower === 'backspace')
1810
- sendKeysCombo = '{BACKSPACE}';
1811
- else if (cLower === 'space')
1812
- sendKeysCombo = ' ';
1813
- else if (cLower === 'up')
1814
- sendKeysCombo = '{UP}';
1815
- else if (cLower === 'down')
1816
- sendKeysCombo = '{DOWN}';
1817
- else if (cLower === 'left')
1818
- sendKeysCombo = '{LEFT}';
1819
- else if (cLower === 'right')
1820
- sendKeysCombo = '{RIGHT}';
1821
- else
1822
- sendKeysCombo = combo;
1823
- }
1824
- const psScript = `
1825
- $sig = @'
1826
- [DllImport("user32.dll")] public static extern bool BlockInput(bool fBlock);
1827
- [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
1828
- [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
1829
- [DllImport("user32.dll")] public static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo);
1830
- '@
1831
- $b = Add-Type -memberDefinition $sig -name ('Win32Block_' + (Get-Random)) -namespace Win32Functions -passThru
1832
-
1833
- try { [void]$b::BlockInput($true) } catch {}
1834
-
1835
- Add-Type -AssemblyName System.Windows.Forms
1836
- $ws = New-Object -ComObject WScript.Shell
1837
-
1838
- try {
1839
- $proc = Get-Process -Name chrome, msedge, firefox, brave, opera, "${appName}" -ErrorAction SilentlyContinue | Where-Object { $_.MainWindowHandle -ne 0 } | Select-Object -First 1
1840
- if ($proc) {
1841
- $b::keybd_event(0x12, 0, 0, [UIntPtr]::Zero)
1842
- $b::keybd_event(0x12, 0, 2, [UIntPtr]::Zero)
1843
- [void]$b::ShowWindow($proc.MainWindowHandle, 9)
1844
- [void]$b::SetForegroundWindow($proc.MainWindowHandle)
1845
- Start-Sleep -Milliseconds 250
1846
- } else {
1847
- $activated = $ws.AppActivate("${appName}")
1848
- if (-not $activated) { $activated = $ws.AppActivate("Chrome") }
1849
- if (-not $activated) { $activated = $ws.AppActivate("Edge") }
1850
- if (-not $activated) { $activated = $ws.AppActivate("Firefox") }
1851
- if (-not $activated) { $activated = $ws.AppActivate("Brave") }
1852
- if (-not $activated) { $activated = $ws.AppActivate("Notepad") }
1853
- if (-not $activated) { $activated = $ws.AppActivate("Visual Studio Code") }
1854
- Start-Sleep -Milliseconds 300
1855
- }
1856
- } catch {}
1857
-
1858
- Start-Sleep -Milliseconds 250
1859
-
1860
- ${sendKeysCombo ? `$ws.SendKeys("${sendKeysCombo}")` : ''}
1861
-
1862
- ${sanitized ? `
1863
- $copied = $false
1864
- try {
1865
- Set-Clipboard -Value "${sanitized}"
1866
- $copied = $true
1867
- } catch {
1868
- try {
1869
- [System.Windows.Forms.Clipboard]::SetText("${sanitized}")
1870
- $copied = $true
1871
- } catch {}
1872
- }
1873
- if ($copied) {
1874
- $ws.SendKeys("^v")
1875
- } else {
1876
- $ws.SendKeys("${sendKeysText}")
1877
- }
1878
- ` : ''}
1879
-
1880
- if (${pressEnter ? '$true' : '$false'}) {
1881
- Start-Sleep -Milliseconds 200
1882
- $ws.SendKeys("~")
1883
- }
1884
-
1885
- try { [void]$b::BlockInput($false) } catch {}
1886
- `;
1887
- try {
1888
- const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
1889
- await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
1890
- return {
1891
- success: true,
1892
- output: `Successfully focused "${appName}" window and sent keyboard takeover inputs ("${targetText}"${combo ? ` [Hotkey: ${combo}]` : ''}${pressEnter ? ' + [ENTER]' : ''}).`
1893
- };
1894
- }
1895
- catch (err) {
1896
- return {
1897
- success: false,
1898
- output: `Failed to send keystrokes to "${appName}": ${err?.message || String(err)}`
1899
- };
1900
- }
1901
- }
1902
- else if (platform === 'darwin') {
1903
- const escaped = targetText.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
1904
- const script = `
1905
- tell application "System Events"
1906
- ${escaped ? `keystroke "${escaped}"` : ''}
1907
- ${pressEnter ? 'key code 36' : ''}
1908
- end tell
1909
- `;
1910
- try {
1911
- await execAsync(`osascript -e '${script.replace(/'/g, "'\\''")}'`);
1912
- return {
1913
- success: true,
1914
- output: `Sent macOS keyboard takeover keystrokes ("${targetText}").`
1915
- };
1916
- }
1917
- catch (err) {
1918
- return {
1919
- success: false,
1920
- output: `Failed macOS osascript keystrokes: ${err?.message || String(err)}`
1921
- };
1922
- }
1923
- }
1924
- else {
1925
- try {
1926
- const escaped = targetText.replace(/"/g, '\\"');
1927
- await execAsync(`xdotool type "${escaped}" ${pressEnter ? '&& xdotool key Return' : ''}`);
1928
- return {
1929
- success: true,
1930
- output: `Sent Linux keyboard takeover keystrokes ("${targetText}").`
1931
- };
1932
- }
1933
- catch (err) {
1934
- return {
1935
- success: false,
1936
- output: `xdotool keystroke error: ${err?.message || String(err)}`
1937
- };
1938
- }
1939
- }
1940
- }
1941
- /**
1942
- * Move cursor and perform OS mouse click at target coordinates or active window center.
1943
- */
1944
- export async function moveAndClickMouse(x, y, button = 'left', appName) {
1945
- const platform = process.platform;
1946
- if (platform === 'win32') {
1947
- const psScript = `
1948
- Add-Type -AssemblyName System.Drawing
1949
- Add-Type -AssemblyName System.Windows.Forms
1950
- $sig = '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int cButtons, int dwExtraInfo); [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);'
1951
- $m = Add-Type -memberDefinition $sig -name 'Win32Mouse' -namespace Win32Functions -passThru
1952
-
1953
- ${(appName && appName !== 'desktop' && appName !== 'system') ? `
1954
- try {
1955
- $proc = Get-Process -Name chrome, msedge, firefox, brave, "${appName}" -ErrorAction SilentlyContinue | Where-Object { $_.MainWindowHandle -ne 0 } | Select-Object -First 1
1956
- if ($proc) {
1957
- $sigFw = '[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);'
1958
- $fw = Add-Type -memberDefinition $sigFw -name 'Win32FwMouse' -namespace Win32Functions -passThru
1959
- [void]$fw::SetForegroundWindow($proc.MainWindowHandle)
1960
- Start-Sleep -Milliseconds 250
1961
- } else {
1962
- $ws = New-Object -ComObject WScript.Shell
1963
- [void]$ws.AppActivate("${appName}")
1964
- Start-Sleep -Milliseconds 200
1965
- }
1966
- } catch {}
1967
- ` : ''}
1968
-
1969
- $screen = [System.Windows.Forms.Screen]::PrimaryScreen
1970
- $targetX = ${x !== undefined ? Math.round(x) : '[int]($screen.Bounds.Width / 2)'}
1971
- $targetY = ${y !== undefined ? Math.round(y) : '[int]($screen.Bounds.Height / 2)'}
1972
-
1973
- [void]$m::SetCursorPos($targetX, $targetY)
1974
- try { [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point($targetX, $targetY) } catch {}
1975
- Start-Sleep -Milliseconds 200
1976
-
1977
- # MOUSEEVENTF_LEFTDOWN = 0x0002, MOUSEEVENTF_LEFTUP = 0x0004
1978
- # MOUSEEVENTF_RIGHTDOWN = 0x0008, MOUSEEVENTF_RIGHTUP = 0x0010
1979
- # MOUSEEVENTF_MIDDLEDOWN = 0x0020, MOUSEEVENTF_MIDDLEUP = 0x0040
1980
- # MOUSEEVENTF_WHEEL = 0x0800
1981
- ${button === 'right' ? '$m::mouse_event(0x0008, 0, 0, 0, 0); $m::mouse_event(0x0010, 0, 0, 0, 0);' :
1982
- button === 'middle' ? '$m::mouse_event(0x0020, 0, 0, 0, 0); $m::mouse_event(0x0040, 0, 0, 0, 0);' :
1983
- button === 'scroll' ? '$m::mouse_event(0x0800, 0, 0, -120, 0);' :
1984
- '$m::mouse_event(0x0002, 0, 0, 0, 0); $m::mouse_event(0x0004, 0, 0, 0, 0);'}
1985
- ${button === 'double' ? 'Start-Sleep -Milliseconds 100; $m::mouse_event(0x0002, 0, 0, 0, 0); $m::mouse_event(0x0004, 0, 0, 0, 0);' : ''}
1986
- `;
1987
- try {
1988
- const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
1989
- await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
1990
- return {
1991
- success: true,
1992
- output: `Executed mouse cursor takeover ${button} click at screen coordinates (${x !== undefined ? x : 'screen center'}, ${y !== undefined ? y : 'screen center'}).`
1993
- };
1994
- }
1995
- catch (err) {
1996
- return {
1997
- success: false,
1998
- output: `Failed mouse takeover action: ${err?.message || String(err)}`
1999
- };
2000
- }
2001
- }
2002
- else if (platform === 'darwin') {
2003
- const script = `
2004
- tell application "System Events"
2005
- ${(x !== undefined && y !== undefined) ? `click at {${x}, ${y}}` : 'click'}
2006
- end tell
2007
- `;
2008
- try {
2009
- await execAsync(`osascript -e '${script.replace(/'/g, "'\\''")}'`);
2010
- return {
2011
- success: true,
2012
- output: `Executed macOS mouse cursor click.`
2013
- };
2014
- }
2015
- catch (err) {
2016
- return {
2017
- success: false,
2018
- output: `Failed macOS mouse click: ${err?.message || String(err)}`
2019
- };
2020
- }
2021
- }
2022
- else {
2023
- try {
2024
- const cmd = (x !== undefined && y !== undefined)
2025
- ? `xdotool mousemove ${x} ${y} click ${button === 'right' ? '3' : '1'}`
2026
- : `xdotool click ${button === 'right' ? '3' : '1'}`;
2027
- await execAsync(cmd);
2028
- return {
2029
- success: true,
2030
- output: `Executed Linux mouse click.`
2031
- };
2032
- }
2033
- catch (err) {
2034
- return {
2035
- success: false,
2036
- output: `xdotool click error: ${err?.message || String(err)}`
2037
- };
2038
- }
2039
- }
2040
- }
2041
- /**
2042
- * Prevent user interruption on targeted application window during scratchpad execution.
2043
- * Focuses window and temporarily locks input / enforces frontmost focus while Scout is working.
2044
- */
2045
- export async function lockAppInput(appName, durationMs = 2000) {
2046
- const platform = process.platform;
2047
- if (platform === 'win32') {
2048
- const psScript = `
2049
- $sig = '[DllImport("user32.dll")] public static extern bool BlockInput(bool fBlock);'
2050
- $b = Add-Type -memberDefinition $sig -name 'Win32Block' -namespace Win32Functions -passThru
2051
-
2052
- $ws = New-Object -ComObject WScript.Shell
2053
- $activated = $ws.AppActivate("${appName}")
2054
- if (-not $activated) { $activated = $ws.AppActivate("Chrome") }
2055
- if (-not $activated) { $activated = $ws.AppActivate("Edge") }
2056
- if (-not $activated) { $activated = $ws.AppActivate("Firefox") }
2057
- if (-not $activated) { $activated = $ws.AppActivate("Brave") }
2058
-
2059
- try { [void]$b::BlockInput($true) } catch {}
2060
- Start-Sleep -Milliseconds ${Math.min(10000, Math.max(500, durationMs))}
2061
- try { [void]$b::BlockInput($false) } catch {}
2062
- `;
2063
- try {
2064
- const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
2065
- await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
2066
- return {
2067
- success: true,
2068
- output: `Locked user interruption on "${appName}" window for ${durationMs}ms while Scout Scratchpad was working.`
2069
- };
2070
- }
2071
- catch (err) {
2072
- return {
2073
- success: false,
2074
- output: `Focus lock applied for "${appName}".`
2075
- };
2076
- }
2077
- }
2078
- else if (platform === 'darwin') {
2079
- const script = `
2080
- tell application "System Events"
2081
- tell application "${appName}" to activate
2082
- end tell
2083
- `;
2084
- try {
2085
- await execAsync(`osascript -e '${script.replace(/'/g, "'\\''")}'`);
2086
- return {
2087
- success: true,
2088
- output: `Enforced frontmost focus on "${appName}" to prevent user interruption on macOS.`
2089
- };
2090
- }
2091
- catch (err) {
2092
- return {
2093
- success: false,
2094
- output: `macOS focus enforcement notice: ${err?.message || String(err)}`
2095
- };
2096
- }
2097
- }
2098
- else {
2099
- return {
2100
- success: true,
2101
- output: `Enforced scratchpad focus lock on Linux for "${appName}".`
2102
- };
2103
- }
2104
- }
2105
- /**
2106
- * List all active top-level application windows.
2107
- */
2108
- export async function listWindows() {
2109
- const platform = process.platform;
2110
- if (platform === 'win32') {
2111
- const psScript = `
2112
- Get-Process | Where-Object { $_.MainWindowTitle -ne '' } | Select-Object Id, ProcessName, MainWindowTitle, MainWindowHandle | ConvertTo-Json
2113
- `;
2114
- try {
2115
- const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
2116
- const { stdout } = await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
2117
- let rawList = [];
2118
- try {
2119
- rawList = JSON.parse(stdout.trim());
2120
- }
2121
- catch { }
2122
- if (!Array.isArray(rawList))
2123
- rawList = rawList ? [rawList] : [];
2124
- const windows = rawList.map((w) => ({
2125
- title: String(w.MainWindowTitle || ''),
2126
- processName: String(w.ProcessName || ''),
2127
- handle: String(w.MainWindowHandle || ''),
2128
- }));
2129
- const formatted = windows.map((w, idx) => ` ${idx + 1}. [${w.processName}] "${w.title}"`).join('\n');
2130
- return {
2131
- success: true,
2132
- output: `Active Application Windows (${windows.length}):\n${formatted || 'No active top-level windows with title found.'}`,
2133
- windows,
2134
- };
2135
- }
2136
- catch (err) {
2137
- return { success: false, output: `Failed to list windows: ${err?.message || String(err)}`, windows: [] };
2138
- }
2139
- }
2140
- return { success: true, output: 'Window listing only supported on Windows.', windows: [] };
2141
- }
2142
- /**
2143
- * Bring a target window to front and focus it.
2144
- */
2145
- export async function focusWindow(appNameOrTitle) {
2146
- const platform = process.platform;
2147
- if (platform === 'win32') {
2148
- const psScript = `
2149
- $target = "${appNameOrTitle.replace(/"/g, '`"')}"
2150
- $proc = Get-Process | Where-Object { ($_.MainWindowTitle -like "*$target*" -or $_.ProcessName -like "*$target*") -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
2151
- if ($proc) {
2152
- $sig = '[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);'
2153
- $w = Add-Type -memberDefinition $sig -name 'Win32Fw' -namespace Win32Functions -passThru
2154
- [void]$w::ShowWindow($proc.MainWindowHandle, 9)
2155
- [void]$w::SetForegroundWindow($proc.MainWindowHandle)
2156
- Write-Output "Focused '$($proc.MainWindowTitle)' ($($proc.ProcessName))"
2157
- } else {
2158
- $ws = New-Object -ComObject WScript.Shell
2159
- $activated = $ws.AppActivate($target)
2160
- if ($activated) { Write-Output "Activated '$target' via WScript.Shell" } else { Write-Output "Window '$target' not found" }
2161
- }
2162
- `;
2163
- try {
2164
- const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
2165
- const { stdout } = await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
2166
- return { success: true, output: stdout.trim() || `Focus request completed for "${appNameOrTitle}".` };
2167
- }
2168
- catch (err) {
2169
- return { success: false, output: `Could not focus window: ${err?.message || String(err)}` };
2170
- }
2171
- }
2172
- return { success: true, output: `Focus request sent for "${appNameOrTitle}".` };
2173
- }
2174
- /**
2175
- * Scroll mouse wheel up or down by specified notch amount.
2176
- */
2177
- export async function scrollWindow(direction = 'down', amount = 3) {
2178
- const platform = process.platform;
2179
- if (platform === 'win32') {
2180
- const clicks = Math.max(1, Math.min(20, amount));
2181
- const delta = direction === 'up' ? 120 * clicks : -120 * clicks;
2182
- const psScript = `
2183
- Add-Type -AssemblyName System.Windows.Forms
2184
- $sig = '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int cButtons, int dwExtraInfo);'
2185
- $m = Add-Type -memberDefinition $sig -name 'Win32Scroll' -namespace Win32Functions -passThru
2186
- $m::mouse_event(0x0800, 0, 0, ${delta}, 0)
2187
- `;
2188
- try {
2189
- const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
2190
- await execAsync(`powershell -NoProfile -EncodedCommand ${encodedScript}`);
2191
- return { success: true, output: `Scrolled mouse wheel ${direction} by ${clicks} notch(es).` };
2192
- }
2193
- catch (err) {
2194
- return { success: false, output: `Failed to scroll: ${err?.message || String(err)}` };
2195
- }
2196
- }
2197
- return { success: true, output: `Scrolled ${direction}` };
2198
- }
2199
- /**
2200
- * Visually ground and click a target UI element by name or visual description.
2201
- */
2202
- export async function findAndClickElement(elementQuery, appName, button = 'left') {
2203
- let screenWidth = 1920;
2204
- let screenHeight = 1080;
2205
- let targetX = undefined;
2206
- let targetY = undefined;
2207
- // Use in-memory live screen frame first without writing screenshot files to disk
2208
- const liveFrame = await getLiveScreenFrame();
2209
- if (liveFrame) {
2210
- screenWidth = liveFrame.width;
2211
- screenHeight = liveFrame.height;
2212
- try {
2213
- const prompt = `Look at this screen and locate the UI element: "${elementQuery}".
2214
- Return ONLY a valid JSON object with the center point of this element in 0-1000 normalized coordinates:
2215
- {"point": [y, x]}
2216
- Where y is between 0 and 1000 (top to bottom) and x is between 0 and 1000 (left to right).
2217
- If not found, return {"point": null}. Output nothing else.`;
2218
- const reply = await callVisionAiWithFrame(liveFrame, prompt, 80);
2219
- if (reply) {
2220
- const match = reply.match(/\{[\s\S]*"point"[\s\S]*\}/);
2221
- if (match) {
2222
- const parsed = JSON.parse(match[0]);
2223
- if (Array.isArray(parsed.point) && parsed.point.length === 2) {
2224
- const normY = Number(parsed.point[0]);
2225
- const normX = Number(parsed.point[1]);
2226
- if (!isNaN(normX) && !isNaN(normY)) {
2227
- targetX = Math.round((normX / 1000) * screenWidth);
2228
- targetY = Math.round((normY / 1000) * screenHeight);
2229
- }
2230
- }
2231
- }
2232
- }
2233
- }
2234
- catch { }
2235
- }
2236
- else {
2237
- const capture = await captureScreen(appName);
2238
- if (capture.success && capture.details?.outPath && fs.existsSync(capture.details.outPath)) {
2239
- screenWidth = capture.details.width || 1920;
2240
- screenHeight = capture.details.height || 1080;
2241
- try {
2242
- const prompt = `Look at this screenshot and locate the UI element: "${elementQuery}".
2243
- Return ONLY a valid JSON object with the center point of this element in 0-1000 normalized coordinates:
2244
- {"point": [y, x]}
2245
- Where y is between 0 and 1000 (top to bottom) and x is between 0 and 1000 (left to right).
2246
- If not found, return {"point": null}. Output nothing else.`;
2247
- const reply = await callVisionAi(capture.details.outPath, prompt, 80);
2248
- if (reply) {
2249
- const match = reply.match(/\{[\s\S]*"point"[\s\S]*\}/);
2250
- if (match) {
2251
- const parsed = JSON.parse(match[0]);
2252
- if (Array.isArray(parsed.point) && parsed.point.length === 2) {
2253
- const normY = Number(parsed.point[0]);
2254
- const normX = Number(parsed.point[1]);
2255
- if (!isNaN(normX) && !isNaN(normY)) {
2256
- targetX = Math.round((normX / 1000) * screenWidth);
2257
- targetY = Math.round((normY / 1000) * screenHeight);
2258
- }
2259
- }
2260
- }
2261
- }
2262
- }
2263
- catch { }
2264
- }
2265
- }
2266
- if ((targetX === undefined || targetY === undefined) && process.platform === 'win32') {
2267
- const psScript = `
2268
- Add-Type -AssemblyName UIAutomationClient
2269
- Add-Type -AssemblyName UIAutomationTypes
2270
- $target = "${elementQuery.replace(/"/g, '""')}"
2271
- $root = [System.Windows.Automation.AutomationElement]::RootElement
2272
-
2273
- $condName = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $target)
2274
- $el = $root.FindFirst([System.Windows.Automation.TreeScope]::Descendants, $condName)
2275
-
2276
- if (-not $el) {
2277
- $all = $root.FindAll([System.Windows.Automation.TreeScope]::Descendants, [System.Windows.Automation.Condition]::TrueCondition)
2278
- foreach ($item in $all) {
2279
- if ($item.Current.Name -and $item.Current.Name.ToLower().Contains($target.ToLower())) {
2280
- $el = $item
2281
- break
2282
- }
2283
- }
2284
- }
2285
-
2286
- if ($el) {
2287
- $rect = $el.Current.BoundingRectangle
2288
- if ($rect -and $rect.Width -gt 0 -and $rect.Height -gt 0) {
2289
- $cx = [int]($rect.X + ($rect.Width / 2))
2290
- $cy = [int]($rect.Y + ($rect.Height / 2))
2291
- @{ x = $cx; y = $cy; name = $el.Current.Name } | ConvertTo-Json
2292
- }
2293
- }
2294
- `;
2295
- try {
2296
- const { stdout } = await execAsync(`powershell -NoProfile -Command "${psScript.replace(/\r?\n/g, ' ')}"`);
2297
- if (stdout.trim()) {
2298
- const uia = JSON.parse(stdout.trim());
2299
- if (uia.x && uia.y) {
2300
- targetX = uia.x;
2301
- targetY = uia.y;
2302
- }
2303
- }
2304
- }
2305
- catch { }
2306
- }
2307
- if (targetX !== undefined && targetY !== undefined) {
2308
- const clickRes = await moveAndClickMouse(targetX, targetY, button, appName);
2309
- return {
2310
- success: clickRes.success,
2311
- output: `Visual Grounding Success: Located "${elementQuery}" at screen coordinates (${targetX}, ${targetY}). ${clickRes.output}`,
2312
- coordinates: { x: targetX, y: targetY },
2313
- };
2314
- }
2315
- return {
2316
- success: false,
2317
- output: `Could not visually or structurally locate UI element "${elementQuery}" on screen. Try calling analyze_screen to view current screen contents, or provide explicit (x, y) coordinates.`,
2318
- };
2319
- }
2320
- //# sourceMappingURL=appControl.js.map