ft-scout 9.0.2 → 9.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.firebase/hosting.d2Vi.cache +7 -7
- package/README.md +76 -83
- package/bin/src/commands/agent.d.ts.map +1 -1
- package/bin/src/commands/agent.js +13 -223
- package/bin/src/commands/agent.js.map +1 -1
- package/bin/src/commands/plan.d.ts.map +1 -1
- package/bin/src/commands/plan.js +45 -18
- package/bin/src/commands/plan.js.map +1 -1
- package/bin/src/engine/actionVerifier.d.ts.map +1 -1
- package/bin/src/engine/actionVerifier.js +61 -9
- package/bin/src/engine/actionVerifier.js.map +1 -1
- package/bin/src/engine/agentEngine.d.ts.map +1 -1
- package/bin/src/engine/agentEngine.js +459 -547
- package/bin/src/engine/agentEngine.js.map +1 -1
- package/bin/src/engine/agentState.d.ts.map +1 -1
- package/bin/src/engine/agentState.js +0 -64
- package/bin/src/engine/agentState.js.map +1 -1
- package/bin/src/engine/browser/browserController.d.ts.map +1 -1
- package/bin/src/engine/browser/browserController.js +8 -0
- package/bin/src/engine/browser/browserController.js.map +1 -1
- package/bin/src/engine/computer/computerController.d.ts.map +1 -1
- package/bin/src/engine/computer/computerController.js +479 -269
- package/bin/src/engine/computer/computerController.js.map +1 -1
- package/bin/src/engine/computer/index.d.ts.map +1 -1
- package/bin/src/engine/computer/index.js +1 -0
- package/bin/src/engine/computer/index.js.map +1 -1
- package/bin/src/engine/computer/mockComputer.d.ts.map +1 -0
- package/bin/src/engine/computer/mockComputer.js +164 -0
- package/bin/src/engine/computer/mockComputer.js.map +1 -0
- package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -0
- package/bin/src/engine/runtime/agentRuntime.js +829 -0
- package/bin/src/engine/runtime/agentRuntime.js.map +1 -0
- package/bin/src/engine/runtime/index.d.ts.map +1 -0
- package/bin/src/engine/runtime/index.js +2 -0
- package/bin/src/engine/runtime/index.js.map +1 -0
- package/bin/src/engine/safety.d.ts.map +1 -1
- package/bin/src/engine/safety.js +49 -25
- package/bin/src/engine/safety.js.map +1 -1
- package/bin/src/engine/toolRegistry.d.ts.map +1 -1
- package/bin/src/engine/toolRegistry.js +162 -81
- package/bin/src/engine/toolRegistry.js.map +1 -1
- package/bin/src/engine/voiceEngine.d.ts.map +1 -1
- package/bin/src/engine/voiceEngine.js +0 -1
- package/bin/src/engine/voiceEngine.js.map +1 -1
- package/bin/src/engine/workspace/agentWorkspace.d.ts.map +1 -0
- package/bin/src/engine/workspace/agentWorkspace.js +484 -0
- package/bin/src/engine/workspace/agentWorkspace.js.map +1 -0
- package/bin/src/engine/workspace/index.d.ts.map +1 -0
- package/bin/src/engine/workspace/index.js +2 -0
- package/bin/src/engine/workspace/index.js.map +1 -0
- package/bin/src/index.js +4 -5
- package/bin/src/index.js.map +1 -1
- package/bin/src/server/server.d.ts.map +1 -1
- package/bin/src/server/server.js +25 -4
- package/bin/src/server/server.js.map +1 -1
- package/bin/src/utils/auth.d.ts.map +1 -1
- package/bin/src/utils/auth.js +67 -49
- package/bin/src/utils/auth.js.map +1 -1
- package/bin/src/utils/branding.js +1 -1
- package/bin/src/utils/payment.d.ts.map +1 -1
- package/bin/src/utils/payment.js +15 -1
- package/bin/src/utils/payment.js.map +1 -1
- package/bin/src/utils/pricing.js +1 -1
- package/bin/src/utils/pricing.js.map +1 -1
- package/bin/src/utils/razorpay.d.ts.map +1 -1
- package/bin/src/utils/razorpay.js +10 -2
- package/bin/src/utils/razorpay.js.map +1 -1
- package/downloads/invoice-2026-03.pdf +1 -0
- package/package.json +1 -1
- package/web/app.js +54 -135
- package/web/styles.css +2 -2
- package/bin/src/engine/appControl.d.ts.map +0 -1
- package/bin/src/engine/appControl.js +0 -2320
- package/bin/src/engine/appControl.js.map +0 -1
- package/bin/src/engine/liveScreenEngine.d.ts.map +0 -1
- package/bin/src/engine/liveScreenEngine.js +0 -421
- package/bin/src/engine/liveScreenEngine.js.map +0 -1
- package/bin/src/engine/scripts/winCapture.ps1 +0 -248
- package/downloads/INVOICE_2026_ACME.pdf +0 -1
|
@@ -1,2320 +0,0 @@
|
|
|
1
|
-
import fs from 'fs';
|
|
2
|
-
import path from 'path';
|
|
3
|
-
import http from 'http';
|
|
4
|
-
import https from 'https';
|
|
5
|
-
import { exec, spawn, ChildProcess } from 'child_process';
|
|
6
|
-
import { promisify } from 'util';
|
|
7
|
-
import OpenAI from 'openai';
|
|
8
|
-
import dotenv from 'dotenv';
|
|
9
|
-
import chalk from 'chalk';
|
|
10
|
-
import { startLiveScreenShare, stopLiveScreenShare, isLiveScreenShareActive, getLiveScreenFrame, analyzeLiveScreen, callVisionAiWithFrame, setLiveScreenTargetApp, getLiveScreenTargetApp, } from './liveScreenEngine.js';
|
|
11
|
-
dotenv.config({ quiet: true });
|
|
12
|
-
const execAsync = promisify(exec);
|
|
13
|
-
/**
|
|
14
|
-
* Standardize app name aliases across platforms.
|
|
15
|
-
*/
|
|
16
|
-
export function normalizeAppName(rawAppName) {
|
|
17
|
-
const appLower = (rawAppName || '').trim().toLowerCase();
|
|
18
|
-
if (['browser', 'chrome', 'google-chrome', 'msedge', 'edge', 'firefox', 'brave', 'safari', 'opera', 'instagram', 'whatsapp', 'twitter', 'x', 'telegram', 'linkedin', 'slack', 'discord'].includes(appLower)) {
|
|
19
|
-
return { category: 'browser', name: appLower || 'browser' };
|
|
20
|
-
}
|
|
21
|
-
if (['terminal', 'cmd', 'powershell', 'pwsh', 'wt', 'windows-terminal', 'iterm', 'bash', 'zsh', 'sh', 'console', 'konsole'].includes(appLower)) {
|
|
22
|
-
return { category: 'terminal', name: appLower || 'terminal' };
|
|
23
|
-
}
|
|
24
|
-
if (['code', 'vscode', 'visual-studio-code', 'cursor', 'zed', 'sublime', 'notepad', 'notepad++', 'vim', 'nvim', 'nano', 'editor'].includes(appLower)) {
|
|
25
|
-
return { category: 'editor', name: appLower || 'editor' };
|
|
26
|
-
}
|
|
27
|
-
if (['game', 'games', 'gaming', 'play', 'launcher', 'steam', 'epic', 'epicgames', 'riot', 'battlenet', 'ea', 'ubisoft', 'gog'].includes(appLower) || appLower.startsWith('game:')) {
|
|
28
|
-
return { category: 'game', name: appLower };
|
|
29
|
-
}
|
|
30
|
-
return { category: 'system', name: appLower };
|
|
31
|
-
}
|
|
32
|
-
export function parsePayload(payload) {
|
|
33
|
-
if (!payload)
|
|
34
|
-
return {};
|
|
35
|
-
if (typeof payload === 'object')
|
|
36
|
-
return payload;
|
|
37
|
-
if (typeof payload === 'string') {
|
|
38
|
-
const trimmed = payload.trim();
|
|
39
|
-
if (trimmed.startsWith('{') && trimmed.endsWith('}')) {
|
|
40
|
-
try {
|
|
41
|
-
return JSON.parse(trimmed);
|
|
42
|
-
}
|
|
43
|
-
catch { }
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
return { target: payload };
|
|
47
|
-
}
|
|
48
|
-
export function buildSocialUrl(platformOrAppName, target, text) {
|
|
49
|
-
const parsedPayload = parsePayload(target);
|
|
50
|
-
const rawTarget = typeof target === 'string' && !target.trim().startsWith('{')
|
|
51
|
-
? target.trim()
|
|
52
|
-
: (parsedPayload.recipient || parsedPayload.user || parsedPayload.username || parsedPayload.handle || parsedPayload.target || parsedPayload.url || '');
|
|
53
|
-
const rawMsgText = text || parsedPayload.text || parsedPayload.message || parsedPayload.content || '';
|
|
54
|
-
const pLower = (platformOrAppName || parsedPayload.platform || '').toLowerCase().trim();
|
|
55
|
-
const cleanTarget = String(rawTarget || '').trim();
|
|
56
|
-
const handleMatch = cleanTarget.match(/@?([a-zA-Z0-9_.-]+)/);
|
|
57
|
-
const handle = handleMatch && handleMatch[1] ? handleMatch[1] : '';
|
|
58
|
-
const isInstagram = pLower.includes('instagram') || cleanTarget.includes('instagram') || cleanTarget.startsWith('@');
|
|
59
|
-
const isWhatsApp = pLower.includes('whatsapp') || cleanTarget.includes('whatsapp') || (/^\+?[0-9]{7,15}$/.test(cleanTarget) && !cleanTarget.startsWith('http'));
|
|
60
|
-
const isTwitter = pLower.includes('twitter') || pLower.includes('x') || cleanTarget.includes('x.com') || cleanTarget.includes('twitter.com');
|
|
61
|
-
const isTelegram = pLower.includes('telegram') || cleanTarget.includes('t.me');
|
|
62
|
-
const isLinkedIn = pLower.includes('linkedin') || cleanTarget.includes('linkedin.com');
|
|
63
|
-
if (isInstagram) {
|
|
64
|
-
if (handle && handle !== 'instagram' && !cleanTarget.includes('http')) {
|
|
65
|
-
return `https://www.instagram.com/direct/i/`;
|
|
66
|
-
}
|
|
67
|
-
return `https://www.instagram.com/direct/inbox/`;
|
|
68
|
-
}
|
|
69
|
-
if (isWhatsApp) {
|
|
70
|
-
const phone = cleanTarget.replace(/[^0-9]/g, '');
|
|
71
|
-
const msgParam = rawMsgText ? `?text=${encodeURIComponent(rawMsgText)}` : '';
|
|
72
|
-
if (phone) {
|
|
73
|
-
return `https://web.whatsapp.com/send?phone=${phone}${rawMsgText ? `&text=${encodeURIComponent(rawMsgText)}` : ''}`;
|
|
74
|
-
}
|
|
75
|
-
return `https://web.whatsapp.com/send${msgParam}`;
|
|
76
|
-
}
|
|
77
|
-
if (isTwitter) {
|
|
78
|
-
if (handle && !cleanTarget.includes('http')) {
|
|
79
|
-
return `https://x.com/direct_messages/create/${handle}`;
|
|
80
|
-
}
|
|
81
|
-
return `https://x.com/direct_messages`;
|
|
82
|
-
}
|
|
83
|
-
if (isTelegram) {
|
|
84
|
-
if (handle)
|
|
85
|
-
return `https://t.me/${handle}`;
|
|
86
|
-
return `https://web.telegram.org/`;
|
|
87
|
-
}
|
|
88
|
-
if (isLinkedIn) {
|
|
89
|
-
return `https://www.linkedin.com/messaging/`;
|
|
90
|
-
}
|
|
91
|
-
const isEmail = pLower.includes('gmail') || pLower.includes('mail') || pLower.includes('email') || (cleanTarget.includes('@') && cleanTarget.includes('.')) || cleanTarget.includes('mail.google.com');
|
|
92
|
-
if (isEmail) {
|
|
93
|
-
const to = cleanTarget.includes('@') && !cleanTarget.startsWith('http') ? cleanTarget : (parsedPayload.to || parsedPayload.recipient || '');
|
|
94
|
-
const subject = parsedPayload.subject || parsedPayload.su || 'Message from Scout';
|
|
95
|
-
const body = rawMsgText || parsedPayload.body || '';
|
|
96
|
-
if (to || body) {
|
|
97
|
-
return `https://mail.google.com/mail/?view=cm&fs=1${to ? `&to=${encodeURIComponent(to)}` : ''}${subject ? `&su=${encodeURIComponent(subject)}` : ''}${body ? `&body=${encodeURIComponent(body)}` : ''}`;
|
|
98
|
-
}
|
|
99
|
-
return 'https://mail.google.com';
|
|
100
|
-
}
|
|
101
|
-
if (cleanTarget.startsWith('http://') || cleanTarget.startsWith('https://')) {
|
|
102
|
-
return cleanTarget;
|
|
103
|
-
}
|
|
104
|
-
if (cleanTarget.includes('.') && !cleanTarget.includes(' ') && !cleanTarget.includes('{')) {
|
|
105
|
-
return `http://${cleanTarget}`;
|
|
106
|
-
}
|
|
107
|
-
return 'https://www.instagram.com/direct/inbox/';
|
|
108
|
-
}
|
|
109
|
-
/**
|
|
110
|
-
* Universally search and locate any installed app, game, or store package on the host OS without hardcoding.
|
|
111
|
-
* Automatically resolves Win32 executables, Microsoft Store games, Steam games, and desktop shortcuts.
|
|
112
|
-
*/
|
|
113
|
-
export async function findInstalledApp(query) {
|
|
114
|
-
const cleanQuery = (query || '').replace(/[^\w\s-]/g, '').trim().toLowerCase();
|
|
115
|
-
if (!cleanQuery)
|
|
116
|
-
return null;
|
|
117
|
-
const platform = process.platform;
|
|
118
|
-
if (platform === 'win32') {
|
|
119
|
-
try {
|
|
120
|
-
// Query Windows Get-StartApps with wildcard matching
|
|
121
|
-
const safePattern = `*${cleanQuery.replace(/[*_']/g, '')}*`;
|
|
122
|
-
const psCmd = `powershell.exe -NoProfile -NonInteractive -Command "Get-StartApps -Name '${safePattern}' | Select-Object -First 1 Name, AppID | ConvertTo-Json"`;
|
|
123
|
-
const { stdout } = await execAsync(psCmd, { timeout: 3500 });
|
|
124
|
-
if (stdout && stdout.trim()) {
|
|
125
|
-
const parsed = JSON.parse(stdout.trim());
|
|
126
|
-
if (parsed && (parsed.Name || parsed.AppID)) {
|
|
127
|
-
return {
|
|
128
|
-
name: String(parsed.Name || query),
|
|
129
|
-
appId: String(parsed.AppID || ''),
|
|
130
|
-
source: 'start_apps',
|
|
131
|
-
};
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
catch { }
|
|
136
|
-
// Fallback: check registered URI scheme protocol (e.g. steam, roblox, spotify, discord, minecraft, riot)
|
|
137
|
-
if (!cleanQuery.includes(' ') && cleanQuery.length >= 3) {
|
|
138
|
-
try {
|
|
139
|
-
const checkReg = `powershell.exe -NoProfile -NonInteractive -Command "Test-Path 'Registry::HKEY_CLASSES_ROOT\\${cleanQuery}'"`;
|
|
140
|
-
const { stdout } = await execAsync(checkReg, { timeout: 1500 });
|
|
141
|
-
if (stdout && stdout.trim().toLowerCase() === 'true') {
|
|
142
|
-
return {
|
|
143
|
-
name: query,
|
|
144
|
-
appId: `${cleanQuery}:`,
|
|
145
|
-
source: 'protocol',
|
|
146
|
-
};
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
catch { }
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
else if (platform === 'darwin') {
|
|
153
|
-
try {
|
|
154
|
-
const { stdout } = await execAsync(`mdfind "kMDItemKind == 'Application' && kMDItemDisplayName == '*${cleanQuery}*'c" | head -n 1`, { timeout: 3000 });
|
|
155
|
-
if (stdout && stdout.trim()) {
|
|
156
|
-
return {
|
|
157
|
-
name: path.basename(stdout.trim(), '.app'),
|
|
158
|
-
appId: stdout.trim(),
|
|
159
|
-
source: 'path',
|
|
160
|
-
};
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
catch { }
|
|
164
|
-
}
|
|
165
|
-
else {
|
|
166
|
-
// Linux: check which or desktop application entries
|
|
167
|
-
try {
|
|
168
|
-
const { stdout } = await execAsync(`which "${cleanQuery}" || find /usr/share/applications -name "*${cleanQuery}*.desktop" | head -n 1`, { timeout: 2500 });
|
|
169
|
-
if (stdout && stdout.trim()) {
|
|
170
|
-
return {
|
|
171
|
-
name: query,
|
|
172
|
-
appId: stdout.trim(),
|
|
173
|
-
source: 'path',
|
|
174
|
-
};
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
catch { }
|
|
178
|
-
}
|
|
179
|
-
return null;
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* Cross-platform app opener.
|
|
183
|
-
* Launches and opens target application on Windows, macOS, or Linux.
|
|
184
|
-
*/
|
|
185
|
-
export async function openApp(appNameOrPath, target, options) {
|
|
186
|
-
const { category, name } = normalizeAppName(appNameOrPath);
|
|
187
|
-
if (name && name !== 'desktop' && name !== 'system') {
|
|
188
|
-
setLiveScreenTargetApp(name);
|
|
189
|
-
}
|
|
190
|
-
const platform = process.platform;
|
|
191
|
-
let command = '';
|
|
192
|
-
const cleanTarget = target ? target.trim() : '';
|
|
193
|
-
if (category === 'browser') {
|
|
194
|
-
let url = cleanTarget;
|
|
195
|
-
const parsedPayload = parsePayload(cleanTarget);
|
|
196
|
-
const rawRecipient = parsedPayload.recipient || parsedPayload.username || parsedPayload.handle || parsedPayload.target || cleanTarget;
|
|
197
|
-
if (['instagram', 'whatsapp', 'twitter', 'x', 'telegram', 'linkedin'].includes(name) || cleanTarget.includes('instagram') || cleanTarget.includes('whatsapp') || cleanTarget.includes('x.com') || cleanTarget.includes('t.me') || String(rawRecipient).startsWith('@')) {
|
|
198
|
-
url = buildSocialUrl(name, cleanTarget, options?.args?.[0]);
|
|
199
|
-
}
|
|
200
|
-
else if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
|
201
|
-
url = url && !url.includes('{') ? `http://${url}` : 'https://google.com';
|
|
202
|
-
}
|
|
203
|
-
if (platform === 'win32') {
|
|
204
|
-
if (name === 'chrome' || name === 'google-chrome') {
|
|
205
|
-
command = `start chrome "${url}"`;
|
|
206
|
-
}
|
|
207
|
-
else if (name === 'edge' || name === 'msedge') {
|
|
208
|
-
command = `start msedge "${url}"`;
|
|
209
|
-
}
|
|
210
|
-
else if (name === 'firefox') {
|
|
211
|
-
command = `start firefox "${url}"`;
|
|
212
|
-
}
|
|
213
|
-
else if (name === 'brave') {
|
|
214
|
-
command = `start brave "${url}"`;
|
|
215
|
-
}
|
|
216
|
-
else {
|
|
217
|
-
command = `start "" "${url}"`;
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
else if (platform === 'darwin') {
|
|
221
|
-
if (name === 'chrome' || name === 'google-chrome') {
|
|
222
|
-
command = `open -a "Google Chrome" "${url}"`;
|
|
223
|
-
}
|
|
224
|
-
else if (name === 'firefox') {
|
|
225
|
-
command = `open -a "Firefox" "${url}"`;
|
|
226
|
-
}
|
|
227
|
-
else if (name === 'safari') {
|
|
228
|
-
command = `open -a "Safari" "${url}"`;
|
|
229
|
-
}
|
|
230
|
-
else {
|
|
231
|
-
command = `open "${url}"`;
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
else {
|
|
235
|
-
// Linux
|
|
236
|
-
if (name === 'chrome' || name === 'google-chrome') {
|
|
237
|
-
command = `google-chrome "${url}" &`;
|
|
238
|
-
}
|
|
239
|
-
else if (name === 'firefox') {
|
|
240
|
-
command = `firefox "${url}" &`;
|
|
241
|
-
}
|
|
242
|
-
else {
|
|
243
|
-
command = `xdg-open "${url}" &`;
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
else if (category === 'terminal') {
|
|
248
|
-
const initialCmd = cleanTarget;
|
|
249
|
-
if (platform === 'win32') {
|
|
250
|
-
if (name === 'wt' || name === 'windows-terminal' || name === 'terminal') {
|
|
251
|
-
command = initialCmd ? `start wt.exe powershell -NoExit -Command "${initialCmd.replace(/"/g, '`"')}"` : `start wt.exe`;
|
|
252
|
-
}
|
|
253
|
-
else if (name === 'cmd') {
|
|
254
|
-
command = initialCmd ? `start cmd.exe /k "${initialCmd}"` : `start cmd.exe`;
|
|
255
|
-
}
|
|
256
|
-
else {
|
|
257
|
-
// powershell / default
|
|
258
|
-
command = initialCmd ? `start powershell.exe -NoExit -Command "${initialCmd.replace(/"/g, '`"')}"` : `start powershell.exe`;
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
else if (platform === 'darwin') {
|
|
262
|
-
if (name === 'iterm') {
|
|
263
|
-
command = `open -a iTerm .`;
|
|
264
|
-
}
|
|
265
|
-
else {
|
|
266
|
-
command = `open -a Terminal .`;
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
else {
|
|
270
|
-
// Linux
|
|
271
|
-
command = `x-terminal-emulator &`;
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
else if (category === 'editor') {
|
|
275
|
-
const fileOrDir = cleanTarget || '.';
|
|
276
|
-
if (name === 'code' || name === 'vscode' || name === 'visual-studio-code' || name === 'cursor' || name === 'editor') {
|
|
277
|
-
const exe = name === 'cursor' ? 'cursor' : 'code';
|
|
278
|
-
command = `${exe} "${fileOrDir}"`;
|
|
279
|
-
}
|
|
280
|
-
else if (name === 'notepad' && platform === 'win32') {
|
|
281
|
-
command = `start notepad "${fileOrDir}"`;
|
|
282
|
-
}
|
|
283
|
-
else if (platform === 'win32') {
|
|
284
|
-
command = `start "" "${fileOrDir}"`;
|
|
285
|
-
}
|
|
286
|
-
else if (platform === 'darwin') {
|
|
287
|
-
command = `open -a "Visual Studio Code" "${fileOrDir}"`;
|
|
288
|
-
}
|
|
289
|
-
else {
|
|
290
|
-
command = `xdg-open "${fileOrDir}" &`;
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
else {
|
|
294
|
-
// Universal Application, Game & Desktop Launcher (Zero-Hardcode OS Discovery)
|
|
295
|
-
const targetQuery = name || appNameOrPath;
|
|
296
|
-
const installedApp = await findInstalledApp(targetQuery);
|
|
297
|
-
if (installedApp) {
|
|
298
|
-
if (platform === 'win32') {
|
|
299
|
-
if (installedApp.source === 'protocol') {
|
|
300
|
-
const extraArg = cleanTarget ? ` "${cleanTarget}"` : '';
|
|
301
|
-
command = `start ${installedApp.appId}${extraArg}`;
|
|
302
|
-
}
|
|
303
|
-
else {
|
|
304
|
-
command = `explorer.exe "shell:AppsFolder\\${installedApp.appId}"`;
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
else if (platform === 'darwin') {
|
|
308
|
-
command = `open -a "${installedApp.appId || installedApp.name}"`;
|
|
309
|
-
}
|
|
310
|
-
else {
|
|
311
|
-
command = `"${installedApp.appId || installedApp.name}" &`;
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
else {
|
|
315
|
-
// Protocol scheme check (e.g. steam:, roblox:, spotify:, discord:, minecraft:, etc.)
|
|
316
|
-
const isProtocol = /^[a-zA-Z0-9_\-]+:/.test(appNameOrPath) || (cleanTarget && /^[a-zA-Z0-9_\-]+:/.test(cleanTarget));
|
|
317
|
-
if (isProtocol && platform === 'win32') {
|
|
318
|
-
const proto = /^[a-zA-Z0-9_\-]+:/.test(appNameOrPath) ? appNameOrPath : cleanTarget;
|
|
319
|
-
command = `start ${proto}`;
|
|
320
|
-
}
|
|
321
|
-
else if (platform === 'win32') {
|
|
322
|
-
command = cleanTarget ? `start "" "${appNameOrPath}" ${cleanTarget}` : `start "" "${appNameOrPath}"`;
|
|
323
|
-
}
|
|
324
|
-
else if (platform === 'darwin') {
|
|
325
|
-
command = cleanTarget ? `open -a "${appNameOrPath}" "${cleanTarget}"` : `open -a "${appNameOrPath}"`;
|
|
326
|
-
}
|
|
327
|
-
else {
|
|
328
|
-
command = cleanTarget ? `${appNameOrPath} "${cleanTarget}" &` : `${appNameOrPath} &`;
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
try {
|
|
333
|
-
await execAsync(command, { timeout: 3000 });
|
|
334
|
-
return {
|
|
335
|
-
success: true,
|
|
336
|
-
app: name || appNameOrPath,
|
|
337
|
-
action: 'open',
|
|
338
|
-
output: `Successfully launched ${name || appNameOrPath} [Category: ${category}] with target "${cleanTarget || 'default'}". Executed command: \`${command}\`.`,
|
|
339
|
-
details: { category, appName: name || appNameOrPath, target: cleanTarget, command, platform },
|
|
340
|
-
};
|
|
341
|
-
}
|
|
342
|
-
catch (err) {
|
|
343
|
-
// Fallback for GUI commands that spawn detached processes or timeout waiting for process exit
|
|
344
|
-
return {
|
|
345
|
-
success: true,
|
|
346
|
-
app: name || appNameOrPath,
|
|
347
|
-
action: 'open',
|
|
348
|
-
output: `Triggered app launch for ${name || appNameOrPath} (Platform: ${platform}). Command: \`${command}\`. Note: ${err?.message || 'Process spawned in background.'}`,
|
|
349
|
-
details: { category, appName: name || appNameOrPath, target: cleanTarget, command, platform, warning: err?.message },
|
|
350
|
-
};
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
/**
|
|
354
|
-
* Execute automation actions within an opened application context.
|
|
355
|
-
*/
|
|
356
|
-
export async function executeInApp(appName, action, payload) {
|
|
357
|
-
const { category, name } = normalizeAppName(appName);
|
|
358
|
-
const actLower = (action || '').trim().toLowerCase();
|
|
359
|
-
// Focus live screen sharing on the active app being operated
|
|
360
|
-
if (name && name !== 'desktop' && name !== 'system') {
|
|
361
|
-
setLiveScreenTargetApp(name);
|
|
362
|
-
}
|
|
363
|
-
// Universal Scratchpad Desktop Automation Actions (Takeover, Screen Seeing, Keystrokes, Mouse Clicks, Focus Locks)
|
|
364
|
-
if (['takeover', 'takeover_control', 'takeover_screen', 'takeover_app'].includes(actLower)) {
|
|
365
|
-
const parsed = parsePayload(payload);
|
|
366
|
-
const duration = typeof parsed.duration === 'number' ? parsed.duration : 3000;
|
|
367
|
-
const x = typeof parsed.x === 'number' ? parsed.x : undefined;
|
|
368
|
-
const y = typeof parsed.y === 'number' ? parsed.y : undefined;
|
|
369
|
-
const res = await takeoverControl(name || appName, duration, x, y);
|
|
370
|
-
return {
|
|
371
|
-
success: res.success,
|
|
372
|
-
app: name || appName,
|
|
373
|
-
action: actLower,
|
|
374
|
-
output: res.output,
|
|
375
|
-
details: res.details,
|
|
376
|
-
};
|
|
377
|
-
}
|
|
378
|
-
if (['release_takeover', 'unlock_screen', 'unlock_app', 'unlock_input', 'unlock'].includes(actLower)) {
|
|
379
|
-
const res = await releaseTakeover();
|
|
380
|
-
return {
|
|
381
|
-
success: res.success,
|
|
382
|
-
app: name || appName,
|
|
383
|
-
action: actLower,
|
|
384
|
-
output: res.output,
|
|
385
|
-
};
|
|
386
|
-
}
|
|
387
|
-
if (['start_screen_share', 'live_screen_share', 'screen_share', 'start_live_stream'].includes(actLower)) {
|
|
388
|
-
const targetApp = (name && name !== 'desktop' && name !== 'system') ? name : undefined;
|
|
389
|
-
const res = await startLiveScreenShare({ targetApp });
|
|
390
|
-
return {
|
|
391
|
-
success: res.success,
|
|
392
|
-
app: name || appName,
|
|
393
|
-
action: actLower,
|
|
394
|
-
output: res.message,
|
|
395
|
-
details: { active: true, targetApp },
|
|
396
|
-
};
|
|
397
|
-
}
|
|
398
|
-
if (['stop_screen_share', 'end_screen_share', 'stop_live_stream'].includes(actLower)) {
|
|
399
|
-
const res = stopLiveScreenShare();
|
|
400
|
-
return {
|
|
401
|
-
success: res.success,
|
|
402
|
-
app: name || appName,
|
|
403
|
-
action: actLower,
|
|
404
|
-
output: res.summary,
|
|
405
|
-
details: { active: false },
|
|
406
|
-
};
|
|
407
|
-
}
|
|
408
|
-
if (['capture_screen', 'screenshot', 'screen_shot'].includes(actLower)) {
|
|
409
|
-
if (isLiveScreenShareActive()) {
|
|
410
|
-
const liveRes = await analyzeLiveScreen();
|
|
411
|
-
return {
|
|
412
|
-
success: liveRes.success,
|
|
413
|
-
app: name || appName,
|
|
414
|
-
action: actLower,
|
|
415
|
-
output: liveRes.output,
|
|
416
|
-
details: liveRes.details,
|
|
417
|
-
};
|
|
418
|
-
}
|
|
419
|
-
const res = await captureScreen(name || appName);
|
|
420
|
-
return {
|
|
421
|
-
success: res.success,
|
|
422
|
-
app: name || appName,
|
|
423
|
-
action: actLower,
|
|
424
|
-
output: res.output,
|
|
425
|
-
details: res.details,
|
|
426
|
-
};
|
|
427
|
-
}
|
|
428
|
-
if (['see_screen', 'inspect_screen', 'view_screen', 'analyze_screen', 'inspect_desktop', 'visual_analysis', 'see_and_analyze', 'live_screen_stream', 'screen_stream'].includes(actLower)) {
|
|
429
|
-
const query = typeof payload === 'string' ? payload : (payload?.query || payload?.target || payload?.text || undefined);
|
|
430
|
-
const res = await analyzeLiveScreen(query);
|
|
431
|
-
return {
|
|
432
|
-
success: res.success,
|
|
433
|
-
app: name || appName,
|
|
434
|
-
action: actLower,
|
|
435
|
-
output: res.output,
|
|
436
|
-
details: res.details,
|
|
437
|
-
};
|
|
438
|
-
}
|
|
439
|
-
if (['get_screen_info', 'list_screens', 'screen_bounds', 'screen_info'].includes(actLower)) {
|
|
440
|
-
const res = await getScreenInfo(name || appName);
|
|
441
|
-
return {
|
|
442
|
-
success: res.success,
|
|
443
|
-
app: name || appName,
|
|
444
|
-
action: actLower,
|
|
445
|
-
output: res.output,
|
|
446
|
-
details: res.details,
|
|
447
|
-
};
|
|
448
|
-
}
|
|
449
|
-
if (['type_text', 'send_keys', 'type', 'keystrokes', 'key_combo', 'press_hotkey', 'hotkey'].includes(actLower)) {
|
|
450
|
-
const parsed = parsePayload(payload);
|
|
451
|
-
const isHotKeyAction = ['key_combo', 'press_hotkey', 'hotkey', 'send_keys'].includes(actLower);
|
|
452
|
-
let keyCombo = undefined;
|
|
453
|
-
let textToType = '';
|
|
454
|
-
if (typeof parsed === 'object') {
|
|
455
|
-
keyCombo = parsed.keyCombo || parsed.combo || parsed.hotkey || parsed.keys || parsed.key;
|
|
456
|
-
textToType = parsed.text || parsed.content || parsed.message || '';
|
|
457
|
-
}
|
|
458
|
-
if (!keyCombo && typeof payload === 'string') {
|
|
459
|
-
const trimmed = payload.trim();
|
|
460
|
-
if (isHotKeyAction || trimmed.includes('+') || ['enter', 'return', 'tab', 'esc', 'escape', 'backspace', 'space', 'up', 'down', 'left', 'right'].includes(trimmed.toLowerCase())) {
|
|
461
|
-
keyCombo = trimmed;
|
|
462
|
-
}
|
|
463
|
-
else {
|
|
464
|
-
textToType = trimmed;
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
|
-
else if (!textToType && typeof payload === 'string' && !payload.trim().startsWith('{')) {
|
|
468
|
-
textToType = payload;
|
|
469
|
-
}
|
|
470
|
-
const pressEnter = typeof parsed === 'object' && parsed.enter !== undefined ? parsed.enter : !isHotKeyAction;
|
|
471
|
-
const ksRes = await sendKeystrokes(name || appName, textToType, pressEnter, keyCombo);
|
|
472
|
-
return {
|
|
473
|
-
success: ksRes.success,
|
|
474
|
-
app: name || appName,
|
|
475
|
-
action: actLower,
|
|
476
|
-
output: ksRes.output,
|
|
477
|
-
details: { appName: name || appName, textToType, pressEnter, keyCombo },
|
|
478
|
-
};
|
|
479
|
-
}
|
|
480
|
-
if (['click_element', 'click_ui', 'click_button', 'tap_element'].includes(actLower)) {
|
|
481
|
-
const parsed = parsePayload(payload);
|
|
482
|
-
const query = typeof parsed === 'object' ? (parsed.element || parsed.target || parsed.name || parsed.query || '') : String(parsed || payload);
|
|
483
|
-
const button = typeof parsed === 'object' && parsed.button ? parsed.button : 'left';
|
|
484
|
-
const res = await findAndClickElement(query, name || appName, button);
|
|
485
|
-
return {
|
|
486
|
-
success: res.success,
|
|
487
|
-
app: name || appName,
|
|
488
|
-
action: actLower,
|
|
489
|
-
output: res.output,
|
|
490
|
-
details: res.coordinates,
|
|
491
|
-
};
|
|
492
|
-
}
|
|
493
|
-
if (['click', 'click_app', 'click_at', 'move_mouse', 'drag_mouse'].includes(actLower)) {
|
|
494
|
-
const parsedPayload = parsePayload(payload);
|
|
495
|
-
if (typeof parsedPayload === 'object' && parsedPayload.element && parsedPayload.x === undefined) {
|
|
496
|
-
const res = await findAndClickElement(parsedPayload.element, name || appName, parsedPayload.button || 'left');
|
|
497
|
-
return {
|
|
498
|
-
success: res.success,
|
|
499
|
-
app: name || appName,
|
|
500
|
-
action: actLower,
|
|
501
|
-
output: res.output,
|
|
502
|
-
details: res.coordinates,
|
|
503
|
-
};
|
|
504
|
-
}
|
|
505
|
-
const x = typeof parsedPayload.x === 'number' ? parsedPayload.x : undefined;
|
|
506
|
-
const y = typeof parsedPayload.y === 'number' ? parsedPayload.y : undefined;
|
|
507
|
-
const button = ['right', 'double', 'middle', 'scroll'].includes(parsedPayload.button) ? parsedPayload.button : 'left';
|
|
508
|
-
const clickRes = await moveAndClickMouse(x, y, button, name || appName);
|
|
509
|
-
return {
|
|
510
|
-
success: clickRes.success,
|
|
511
|
-
app: name || appName,
|
|
512
|
-
action: actLower,
|
|
513
|
-
output: clickRes.output,
|
|
514
|
-
details: { appName: name || appName, x, y, button },
|
|
515
|
-
};
|
|
516
|
-
}
|
|
517
|
-
if (['scroll', 'scroll_window', 'scroll_down', 'scroll_up', 'scroll_page'].includes(actLower)) {
|
|
518
|
-
const parsed = parsePayload(payload);
|
|
519
|
-
let direction = 'down';
|
|
520
|
-
let amount = 3;
|
|
521
|
-
if (typeof parsed === 'object') {
|
|
522
|
-
if (parsed.direction === 'up' || parsed.dir === 'up')
|
|
523
|
-
direction = 'up';
|
|
524
|
-
if (typeof parsed.amount === 'number')
|
|
525
|
-
amount = parsed.amount;
|
|
526
|
-
if (typeof parsed.clicks === 'number')
|
|
527
|
-
amount = parsed.clicks;
|
|
528
|
-
}
|
|
529
|
-
else if (typeof payload === 'string' && payload.toLowerCase().includes('up')) {
|
|
530
|
-
direction = 'up';
|
|
531
|
-
}
|
|
532
|
-
if (actLower === 'scroll_up')
|
|
533
|
-
direction = 'up';
|
|
534
|
-
const res = await scrollWindow(direction, amount);
|
|
535
|
-
return {
|
|
536
|
-
success: res.success,
|
|
537
|
-
app: name || appName,
|
|
538
|
-
action: actLower,
|
|
539
|
-
output: res.output,
|
|
540
|
-
};
|
|
541
|
-
}
|
|
542
|
-
if (['list_windows', 'get_windows', 'open_windows', 'show_windows'].includes(actLower)) {
|
|
543
|
-
const res = await listWindows();
|
|
544
|
-
return {
|
|
545
|
-
success: res.success,
|
|
546
|
-
app: name || appName,
|
|
547
|
-
action: actLower,
|
|
548
|
-
output: res.output,
|
|
549
|
-
details: res.windows,
|
|
550
|
-
};
|
|
551
|
-
}
|
|
552
|
-
if (['focus_window', 'switch_window', 'bring_to_front', 'select_window'].includes(actLower)) {
|
|
553
|
-
const parsed = parsePayload(payload);
|
|
554
|
-
const target = typeof parsed === 'object' ? (parsed.title || parsed.app || parsed.name || '') : String(parsed || payload || name || appName);
|
|
555
|
-
const res = await focusWindow(target);
|
|
556
|
-
return {
|
|
557
|
-
success: res.success,
|
|
558
|
-
app: target,
|
|
559
|
-
action: actLower,
|
|
560
|
-
output: res.output,
|
|
561
|
-
};
|
|
562
|
-
}
|
|
563
|
-
if (['lock_app', 'block_interruption', 'lock_input', 'lock'].includes(actLower)) {
|
|
564
|
-
const parsedPayload = parsePayload(payload);
|
|
565
|
-
const durationMs = typeof parsedPayload.duration === 'number' ? parsedPayload.duration : 3000;
|
|
566
|
-
const lockRes = await lockAppInput(name || appName, durationMs);
|
|
567
|
-
return {
|
|
568
|
-
success: lockRes.success,
|
|
569
|
-
app: name || appName,
|
|
570
|
-
action: actLower,
|
|
571
|
-
output: lockRes.output,
|
|
572
|
-
details: { appName: name || appName, durationMs },
|
|
573
|
-
};
|
|
574
|
-
}
|
|
575
|
-
if (category === 'browser') {
|
|
576
|
-
if (['navigate', 'goto', 'open_url'].includes(actLower)) {
|
|
577
|
-
const rawUrl = typeof payload === 'string' ? payload : (payload?.url || payload?.target || 'https://google.com');
|
|
578
|
-
let targetUrl = rawUrl.trim();
|
|
579
|
-
let navigationNote = '';
|
|
580
|
-
// Universal URL vs Search Query Grounding:
|
|
581
|
-
// If the input is not a valid URL (contains spaces or lacks a dot / scheme), convert into a search
|
|
582
|
-
if (!targetUrl.startsWith('http://') && !targetUrl.startsWith('https://')) {
|
|
583
|
-
if (targetUrl.includes(' ') || !targetUrl.includes('.')) {
|
|
584
|
-
navigationNote = ` [Note: Plain search term passed to navigate; converted to web search query]`;
|
|
585
|
-
targetUrl = `https://www.google.com/search?q=${encodeURIComponent(targetUrl)}`;
|
|
586
|
-
}
|
|
587
|
-
else {
|
|
588
|
-
targetUrl = `https://${targetUrl}`;
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
await openApp(name || 'browser', targetUrl);
|
|
592
|
-
await new Promise((r) => setTimeout(r, 800));
|
|
593
|
-
const capture = await captureScreen(name || 'browser');
|
|
594
|
-
return {
|
|
595
|
-
success: true,
|
|
596
|
-
app: name || 'browser',
|
|
597
|
-
action: actLower,
|
|
598
|
-
output: `Browser Navigated Successfully to Target URL: ${targetUrl}${navigationNote}\n- Screenshot Saved: ${capture.details?.outPath || 'Active'}`,
|
|
599
|
-
details: { url: targetUrl, capture: capture.details },
|
|
600
|
-
};
|
|
601
|
-
}
|
|
602
|
-
if (actLower === 'fetch_page' || actLower === 'read_url' || actLower === 'inspect_dom') {
|
|
603
|
-
const rawUrl = typeof payload === 'string' ? payload : (payload?.url || payload?.target || 'http://localhost:3000');
|
|
604
|
-
const targetUrl = rawUrl.startsWith('http://') || rawUrl.startsWith('https://') ? rawUrl : `http://${rawUrl}`;
|
|
605
|
-
try {
|
|
606
|
-
const pageContent = await fetchUrlContent(targetUrl);
|
|
607
|
-
const textSnippet = extractTextFromHtml(pageContent);
|
|
608
|
-
return {
|
|
609
|
-
success: true,
|
|
610
|
-
app: name,
|
|
611
|
-
action: actLower,
|
|
612
|
-
output: `Browser Action [${actLower}] for ${targetUrl}:\n\n` +
|
|
613
|
-
`--- Page Text Preview (${textSnippet.length} chars) ---\n` +
|
|
614
|
-
`${textSnippet.slice(0, 2000)}${textSnippet.length > 2000 ? '\n... [truncated]' : ''}`,
|
|
615
|
-
details: { url: targetUrl, fullHtmlLength: pageContent.length, textLength: textSnippet.length },
|
|
616
|
-
};
|
|
617
|
-
}
|
|
618
|
-
catch (err) {
|
|
619
|
-
return {
|
|
620
|
-
success: false,
|
|
621
|
-
app: name,
|
|
622
|
-
action: actLower,
|
|
623
|
-
output: `Failed to fetch page at ${targetUrl}: ${err?.message || String(err)}`,
|
|
624
|
-
};
|
|
625
|
-
}
|
|
626
|
-
}
|
|
627
|
-
if (actLower === 'search' || actLower === 'search_web') {
|
|
628
|
-
const query = typeof payload === 'string' ? payload : (payload?.query || payload?.target || payload?.url || '');
|
|
629
|
-
const isUrl = query.startsWith('http://') || query.startsWith('https://') || (query.includes('.') && !query.includes(' '));
|
|
630
|
-
if (isUrl) {
|
|
631
|
-
const targetUrl = query.startsWith('http://') || query.startsWith('https://') ? query : `https://${query}`;
|
|
632
|
-
return await executeInApp(name || 'browser', 'navigate', targetUrl);
|
|
633
|
-
}
|
|
634
|
-
// Direct Search in browser window
|
|
635
|
-
const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}`;
|
|
636
|
-
await openApp(name || 'browser', searchUrl);
|
|
637
|
-
await new Promise((r) => setTimeout(r, 800));
|
|
638
|
-
const capture = await captureScreen(name || 'browser');
|
|
639
|
-
return {
|
|
640
|
-
success: true,
|
|
641
|
-
app: name,
|
|
642
|
-
action: actLower,
|
|
643
|
-
output: `Browser search executed for "${query}" at ${searchUrl}.\n- Screenshot Saved: ${capture.details?.outPath || 'Active'}`,
|
|
644
|
-
details: { query, searchUrl, capture: capture.details },
|
|
645
|
-
};
|
|
646
|
-
}
|
|
647
|
-
if (['send_dm', 'open_dm', 'dm', 'send_message', 'message', 'social_dm', 'send_mail', 'send_email', 'mail', 'email'].includes(actLower)) {
|
|
648
|
-
const parsed = parsePayload(payload);
|
|
649
|
-
const recipient = typeof payload === 'string' && payload.includes('@') && !payload.startsWith('{')
|
|
650
|
-
? payload
|
|
651
|
-
: (parsed.recipient || parsed.to || parsed.user || parsed.target || parsed.handle || parsed.username || '');
|
|
652
|
-
let msgText = typeof parsed === 'object' ? (parsed.text || parsed.message || parsed.content || parsed.body) : undefined;
|
|
653
|
-
if (!msgText && typeof payload === 'string' && payload.length > 0 && !payload.startsWith('@') && !payload.startsWith('http') && !payload.includes('@')) {
|
|
654
|
-
msgText = payload;
|
|
655
|
-
}
|
|
656
|
-
const isEmail = ['send_mail', 'send_email', 'mail', 'email'].includes(actLower) || recipient.includes('@');
|
|
657
|
-
const targetUrl = buildSocialUrl(isEmail ? 'gmail' : name, recipient, msgText);
|
|
658
|
-
const openRes = await openApp('browser', targetUrl);
|
|
659
|
-
let keystrokeOutput = '';
|
|
660
|
-
if (isEmail) {
|
|
661
|
-
// For Gmail direct compose: wait 2.5s for page & compose dialog to render, then press Ctrl+Enter to send!
|
|
662
|
-
await lockAppInput('browser', 3500);
|
|
663
|
-
await new Promise((r) => setTimeout(r, 2200));
|
|
664
|
-
const ksRes = await sendKeystrokes('browser', '', false, 'ctrl+enter');
|
|
665
|
-
keystrokeOutput = `\nAuto-Send Keystroke Dispatch (Ctrl+Enter): ${ksRes.output}`;
|
|
666
|
-
}
|
|
667
|
-
else if (msgText) {
|
|
668
|
-
// Lock user interruption during scratchpad keystroke dispatch so user clicks don't interrupt
|
|
669
|
-
await lockAppInput(name || 'browser', 3000);
|
|
670
|
-
await new Promise((r) => setTimeout(r, 1200));
|
|
671
|
-
const ksRes = await sendKeystrokes(name || 'browser', msgText, true);
|
|
672
|
-
keystrokeOutput = `\nAuto-Keystroke Automated Dispatch: ${ksRes.output}`;
|
|
673
|
-
}
|
|
674
|
-
return {
|
|
675
|
-
success: true,
|
|
676
|
-
app: name,
|
|
677
|
-
action: actLower,
|
|
678
|
-
output: `Executed ${isEmail ? 'Gmail / Email Dispatch' : `${name} Direct Message`} for target recipient "${recipient}" at ${targetUrl}.${keystrokeOutput}`,
|
|
679
|
-
details: { platform: isEmail ? 'gmail' : name, recipient, msgText, targetUrl },
|
|
680
|
-
};
|
|
681
|
-
}
|
|
682
|
-
}
|
|
683
|
-
if (category === 'terminal') {
|
|
684
|
-
if (actLower === 'exec_command' || actLower === 'run_script' || actLower === 'run') {
|
|
685
|
-
const cmdStr = typeof payload === 'string' ? payload : (payload?.command || payload?.cmd || payload?.script || 'echo "Terminal Action Executed"');
|
|
686
|
-
try {
|
|
687
|
-
const { stdout, stderr } = await execAsync(cmdStr, { cwd: process.cwd() });
|
|
688
|
-
return {
|
|
689
|
-
success: true,
|
|
690
|
-
app: name,
|
|
691
|
-
action: actLower,
|
|
692
|
-
output: `Terminal Execution inside ${name}:\nCommand: \`${cmdStr}\`\n\nOutput:\n${stdout || stderr || '(No output)'}`,
|
|
693
|
-
details: { command: cmdStr, stdout, stderr },
|
|
694
|
-
};
|
|
695
|
-
}
|
|
696
|
-
catch (err) {
|
|
697
|
-
return {
|
|
698
|
-
success: false,
|
|
699
|
-
app: name,
|
|
700
|
-
action: actLower,
|
|
701
|
-
output: `Terminal Execution Failed for \`${cmdStr}\`: ${err?.message || String(err)}`,
|
|
702
|
-
};
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
|
-
}
|
|
706
|
-
if (category === 'editor') {
|
|
707
|
-
if (actLower === 'open_file' || actLower === 'view_file' || actLower === 'open') {
|
|
708
|
-
const filePath = typeof payload === 'string' ? payload : (payload?.path || payload?.target || '.');
|
|
709
|
-
return await openApp(name, filePath);
|
|
710
|
-
}
|
|
711
|
-
}
|
|
712
|
-
if (category === 'game' || ['open', 'launch', 'start'].includes(actLower)) {
|
|
713
|
-
const rawTarget = typeof payload === 'string' ? payload : (payload?.target || payload?.game || payload?.url || payload?.query || payload?.id || '');
|
|
714
|
-
return await openApp(name || appName, rawTarget);
|
|
715
|
-
}
|
|
716
|
-
// Generic fallback app action
|
|
717
|
-
return {
|
|
718
|
-
success: true,
|
|
719
|
-
app: name,
|
|
720
|
-
action: actLower,
|
|
721
|
-
output: `Executed app action "${actLower}" on ${name}. Target/Payload: ${JSON.stringify(payload || {})}`,
|
|
722
|
-
details: { appName: name, action: actLower, payload },
|
|
723
|
-
};
|
|
724
|
-
}
|
|
725
|
-
/**
|
|
726
|
-
* Simple HTTP/HTTPS URL content fetcher for browser page inspection.
|
|
727
|
-
*/
|
|
728
|
-
function fetchUrlContent(targetUrl) {
|
|
729
|
-
return new Promise((resolve, reject) => {
|
|
730
|
-
const client = targetUrl.startsWith('https:') ? https : http;
|
|
731
|
-
const req = client.get(targetUrl, { timeout: 10000 }, (res) => {
|
|
732
|
-
if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
733
|
-
const redirectUrl = res.headers.location.startsWith('http')
|
|
734
|
-
? res.headers.location
|
|
735
|
-
: new URL(res.headers.location, targetUrl).toString();
|
|
736
|
-
return fetchUrlContent(redirectUrl).then(resolve).catch(reject);
|
|
737
|
-
}
|
|
738
|
-
let data = '';
|
|
739
|
-
res.setEncoding('utf-8');
|
|
740
|
-
res.on('data', (chunk) => { data += chunk; });
|
|
741
|
-
res.on('end', () => { resolve(data); });
|
|
742
|
-
});
|
|
743
|
-
req.on('error', (err) => reject(err));
|
|
744
|
-
req.on('timeout', () => {
|
|
745
|
-
req.destroy();
|
|
746
|
-
reject(new Error(`HTTP request timed out after 10000ms for ${targetUrl}`));
|
|
747
|
-
});
|
|
748
|
-
});
|
|
749
|
-
}
|
|
750
|
-
/**
|
|
751
|
-
* Basic HTML to plain text converter for page inspection.
|
|
752
|
-
*/
|
|
753
|
-
function extractTextFromHtml(html) {
|
|
754
|
-
if (!html)
|
|
755
|
-
return '';
|
|
756
|
-
return html
|
|
757
|
-
.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '')
|
|
758
|
-
.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '')
|
|
759
|
-
.replace(/<[^>]+>/g, ' ')
|
|
760
|
-
.replace(/\s+/g, ' ')
|
|
761
|
-
.trim();
|
|
762
|
-
}
|
|
763
|
-
/**
|
|
764
|
-
* Diagnostics report of active host platform app launchers.
|
|
765
|
-
*/
|
|
766
|
-
export function getAppCapabilities() {
|
|
767
|
-
const platform = process.platform;
|
|
768
|
-
return {
|
|
769
|
-
platform,
|
|
770
|
-
supportedCategories: ['browser', 'terminal', 'editor', 'system'],
|
|
771
|
-
defaultBrowserLauncher: platform === 'win32' ? 'start' : platform === 'darwin' ? 'open' : 'xdg-open',
|
|
772
|
-
defaultTerminalLauncher: platform === 'win32' ? 'wt.exe / powershell.exe' : platform === 'darwin' ? 'Terminal.app' : 'x-terminal-emulator',
|
|
773
|
-
defaultEditorLauncher: platform === 'win32' ? 'code / notepad' : platform === 'darwin' ? 'code / open -a' : 'code / xdg-open',
|
|
774
|
-
};
|
|
775
|
-
}
|
|
776
|
-
const WIN_CAPTURE_SCRIPT_CONTENT = `param(
|
|
777
|
-
[string]$OutPath,
|
|
778
|
-
[switch]$InfoOnly
|
|
779
|
-
)
|
|
780
|
-
|
|
781
|
-
$source = @"
|
|
782
|
-
using System;
|
|
783
|
-
using System.Drawing;
|
|
784
|
-
using System.Drawing.Imaging;
|
|
785
|
-
using System.Runtime.InteropServices;
|
|
786
|
-
using System.Text;
|
|
787
|
-
using System.Threading;
|
|
788
|
-
using System.Collections.Generic;
|
|
789
|
-
|
|
790
|
-
public class WinCaptureResult {
|
|
791
|
-
public bool success;
|
|
792
|
-
public string savedPath;
|
|
793
|
-
public int width;
|
|
794
|
-
public int height;
|
|
795
|
-
public int cursorX;
|
|
796
|
-
public int cursorY;
|
|
797
|
-
public string activeWindow;
|
|
798
|
-
public string[] visibleWindows;
|
|
799
|
-
public string error;
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
public class WinCaptureHelper {
|
|
803
|
-
[DllImport("user32.dll", SetLastError = true)]
|
|
804
|
-
public static extern IntPtr OpenInputDesktop(uint dwFlags, bool fInherit, uint dwDesiredAccess);
|
|
805
|
-
|
|
806
|
-
[DllImport("user32.dll", SetLastError = true)]
|
|
807
|
-
public static extern IntPtr OpenDesktop(string lpszDesktop, uint dwFlags, bool fInherit, uint dwDesiredAccess);
|
|
808
|
-
|
|
809
|
-
[DllImport("user32.dll", SetLastError = true)]
|
|
810
|
-
public static extern bool SetThreadDesktop(IntPtr hDesktop);
|
|
811
|
-
|
|
812
|
-
[DllImport("user32.dll", SetLastError = true)]
|
|
813
|
-
public static extern bool CloseDesktop(IntPtr hDesktop);
|
|
814
|
-
|
|
815
|
-
[DllImport("user32.dll")]
|
|
816
|
-
public static extern IntPtr GetDC(IntPtr hWnd);
|
|
817
|
-
|
|
818
|
-
[DllImport("user32.dll")]
|
|
819
|
-
public static extern int ReleaseDC(IntPtr hWnd, IntPtr hDC);
|
|
820
|
-
|
|
821
|
-
[DllImport("gdi32.dll")]
|
|
822
|
-
public static extern IntPtr CreateCompatibleDC(IntPtr hdc);
|
|
823
|
-
|
|
824
|
-
[DllImport("gdi32.dll")]
|
|
825
|
-
public static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight);
|
|
826
|
-
|
|
827
|
-
[DllImport("gdi32.dll")]
|
|
828
|
-
public static extern IntPtr SelectObject(IntPtr hdc, IntPtr hgdiobj);
|
|
829
|
-
|
|
830
|
-
[DllImport("gdi32.dll")]
|
|
831
|
-
public static extern bool DeleteDC(IntPtr hdc);
|
|
832
|
-
|
|
833
|
-
[DllImport("gdi32.dll")]
|
|
834
|
-
public static extern bool DeleteObject(IntPtr hObject);
|
|
835
|
-
|
|
836
|
-
[DllImport("gdi32.dll")]
|
|
837
|
-
public static extern bool BitBlt(IntPtr hdcDest, int nXDest, int nYDest, int nWidth, int nHeight, IntPtr hdcSrc, int nXSrc, int nYSrc, int dwRop);
|
|
838
|
-
|
|
839
|
-
[DllImport("user32.dll")]
|
|
840
|
-
public static extern int GetSystemMetrics(int nIndex);
|
|
841
|
-
|
|
842
|
-
public delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam);
|
|
843
|
-
|
|
844
|
-
[DllImport("user32.dll")]
|
|
845
|
-
public static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, IntPtr lParam);
|
|
846
|
-
|
|
847
|
-
[DllImport("user32.dll", CharSet = CharSet.Auto)]
|
|
848
|
-
public static extern int GetWindowText(IntPtr hWnd, StringBuilder lpString, int nMaxCount);
|
|
849
|
-
|
|
850
|
-
[DllImport("user32.dll")]
|
|
851
|
-
public static extern bool IsWindowVisible(IntPtr hWnd);
|
|
852
|
-
|
|
853
|
-
[DllImport("user32.dll")]
|
|
854
|
-
public static extern IntPtr GetForegroundWindow();
|
|
855
|
-
|
|
856
|
-
[StructLayout(LayoutKind.Sequential)]
|
|
857
|
-
public struct POINT {
|
|
858
|
-
public int X;
|
|
859
|
-
public int Y;
|
|
860
|
-
}
|
|
861
|
-
|
|
862
|
-
[DllImport("user32.dll")]
|
|
863
|
-
public static extern bool GetCursorPos(out POINT lpPoint);
|
|
864
|
-
|
|
865
|
-
const uint DESKTOP_ACCESS = 0x01FF | 0x00020000;
|
|
866
|
-
const int SM_CXSCREEN = 0;
|
|
867
|
-
const int SM_CYSCREEN = 1;
|
|
868
|
-
const int SRCCOPY = 0x00CC0020;
|
|
869
|
-
const int CAPTUREBLT = 0x40000000;
|
|
870
|
-
|
|
871
|
-
public static WinCaptureResult GetInfo() {
|
|
872
|
-
var res = new WinCaptureResult();
|
|
873
|
-
res.success = true;
|
|
874
|
-
res.error = "";
|
|
875
|
-
var visibleWindows = new List<string>();
|
|
876
|
-
|
|
877
|
-
Thread worker = new Thread(() => {
|
|
878
|
-
IntPtr hDesk = OpenInputDesktop(0, false, DESKTOP_ACCESS);
|
|
879
|
-
if (hDesk == IntPtr.Zero) {
|
|
880
|
-
hDesk = OpenDesktop("Default", 0, false, DESKTOP_ACCESS);
|
|
881
|
-
}
|
|
882
|
-
if (hDesk != IntPtr.Zero) {
|
|
883
|
-
SetThreadDesktop(hDesk);
|
|
884
|
-
}
|
|
885
|
-
|
|
886
|
-
res.width = GetSystemMetrics(SM_CXSCREEN);
|
|
887
|
-
res.height = GetSystemMetrics(SM_CYSCREEN);
|
|
888
|
-
if (res.width <= 0) res.width = 1920;
|
|
889
|
-
if (res.height <= 0) res.height = 1080;
|
|
890
|
-
|
|
891
|
-
POINT pt;
|
|
892
|
-
if (GetCursorPos(out pt)) {
|
|
893
|
-
res.cursorX = pt.X;
|
|
894
|
-
res.cursorY = pt.Y;
|
|
895
|
-
}
|
|
896
|
-
|
|
897
|
-
IntPtr fg = GetForegroundWindow();
|
|
898
|
-
if (fg != IntPtr.Zero) {
|
|
899
|
-
var sbFg = new StringBuilder(256);
|
|
900
|
-
GetWindowText(fg, sbFg, 256);
|
|
901
|
-
res.activeWindow = sbFg.ToString().Trim();
|
|
902
|
-
} else {
|
|
903
|
-
res.activeWindow = "";
|
|
904
|
-
}
|
|
905
|
-
|
|
906
|
-
EnumWindows((hWnd, lParam) => {
|
|
907
|
-
if (IsWindowVisible(hWnd)) {
|
|
908
|
-
var title = new StringBuilder(256);
|
|
909
|
-
GetWindowText(hWnd, title, 256);
|
|
910
|
-
string t = title.ToString().Trim();
|
|
911
|
-
if (t.Length > 0 && !visibleWindows.Contains(t) && t != "Program Manager" && t != "Windows Input Experience") {
|
|
912
|
-
visibleWindows.Add(t);
|
|
913
|
-
}
|
|
914
|
-
}
|
|
915
|
-
return true;
|
|
916
|
-
}, IntPtr.Zero);
|
|
917
|
-
|
|
918
|
-
if (hDesk != IntPtr.Zero) {
|
|
919
|
-
CloseDesktop(hDesk);
|
|
920
|
-
}
|
|
921
|
-
});
|
|
922
|
-
|
|
923
|
-
worker.SetApartmentState(ApartmentState.STA);
|
|
924
|
-
worker.Start();
|
|
925
|
-
worker.Join();
|
|
926
|
-
|
|
927
|
-
res.visibleWindows = visibleWindows.ToArray();
|
|
928
|
-
return res;
|
|
929
|
-
}
|
|
930
|
-
|
|
931
|
-
public static WinCaptureResult Capture(string filePath) {
|
|
932
|
-
var res = new WinCaptureResult();
|
|
933
|
-
res.savedPath = filePath;
|
|
934
|
-
res.error = "";
|
|
935
|
-
var visibleWindows = new List<string>();
|
|
936
|
-
|
|
937
|
-
Thread worker = new Thread(() => {
|
|
938
|
-
IntPtr hDesk = OpenInputDesktop(0, false, DESKTOP_ACCESS);
|
|
939
|
-
if (hDesk == IntPtr.Zero) {
|
|
940
|
-
hDesk = OpenDesktop("Default", 0, false, DESKTOP_ACCESS);
|
|
941
|
-
}
|
|
942
|
-
if (hDesk != IntPtr.Zero) {
|
|
943
|
-
SetThreadDesktop(hDesk);
|
|
944
|
-
}
|
|
945
|
-
|
|
946
|
-
res.width = GetSystemMetrics(SM_CXSCREEN);
|
|
947
|
-
res.height = GetSystemMetrics(SM_CYSCREEN);
|
|
948
|
-
if (res.width <= 0) res.width = 1920;
|
|
949
|
-
if (res.height <= 0) res.height = 1080;
|
|
950
|
-
|
|
951
|
-
POINT pt;
|
|
952
|
-
if (GetCursorPos(out pt)) {
|
|
953
|
-
res.cursorX = pt.X;
|
|
954
|
-
res.cursorY = pt.Y;
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
IntPtr fg = GetForegroundWindow();
|
|
958
|
-
if (fg != IntPtr.Zero) {
|
|
959
|
-
var sbFg = new StringBuilder(256);
|
|
960
|
-
GetWindowText(fg, sbFg, 256);
|
|
961
|
-
res.activeWindow = sbFg.ToString().Trim();
|
|
962
|
-
} else {
|
|
963
|
-
res.activeWindow = "";
|
|
964
|
-
}
|
|
965
|
-
|
|
966
|
-
EnumWindows((hWnd, lParam) => {
|
|
967
|
-
if (IsWindowVisible(hWnd)) {
|
|
968
|
-
var title = new StringBuilder(256);
|
|
969
|
-
GetWindowText(hWnd, title, 256);
|
|
970
|
-
string t = title.ToString().Trim();
|
|
971
|
-
if (t.Length > 0 && !visibleWindows.Contains(t) && t != "Program Manager" && t != "Windows Input Experience") {
|
|
972
|
-
visibleWindows.Add(t);
|
|
973
|
-
}
|
|
974
|
-
}
|
|
975
|
-
return true;
|
|
976
|
-
}, IntPtr.Zero);
|
|
977
|
-
|
|
978
|
-
try {
|
|
979
|
-
IntPtr hdcSrc = GetDC(IntPtr.Zero);
|
|
980
|
-
IntPtr hdcDest = CreateCompatibleDC(hdcSrc);
|
|
981
|
-
IntPtr hBitmap = CreateCompatibleBitmap(hdcSrc, res.width, res.height);
|
|
982
|
-
IntPtr hOld = SelectObject(hdcDest, hBitmap);
|
|
983
|
-
bool bltOk = BitBlt(hdcDest, 0, 0, res.width, res.height, hdcSrc, 0, 0, SRCCOPY | CAPTUREBLT);
|
|
984
|
-
|
|
985
|
-
SelectObject(hdcDest, hOld);
|
|
986
|
-
DeleteDC(hdcDest);
|
|
987
|
-
ReleaseDC(IntPtr.Zero, hdcSrc);
|
|
988
|
-
|
|
989
|
-
if (bltOk) {
|
|
990
|
-
using (Bitmap bmp = Bitmap.FromHbitmap(hBitmap)) {
|
|
991
|
-
bmp.Save(filePath, ImageFormat.Png);
|
|
992
|
-
}
|
|
993
|
-
res.success = true;
|
|
994
|
-
} else {
|
|
995
|
-
res.error = "BitBlt failed";
|
|
996
|
-
}
|
|
997
|
-
DeleteObject(hBitmap);
|
|
998
|
-
} catch (Exception ex) {
|
|
999
|
-
res.error = ex.Message;
|
|
1000
|
-
}
|
|
1001
|
-
|
|
1002
|
-
if (hDesk != IntPtr.Zero) {
|
|
1003
|
-
CloseDesktop(hDesk);
|
|
1004
|
-
}
|
|
1005
|
-
});
|
|
1006
|
-
|
|
1007
|
-
worker.SetApartmentState(ApartmentState.STA);
|
|
1008
|
-
worker.Start();
|
|
1009
|
-
worker.Join();
|
|
1010
|
-
|
|
1011
|
-
res.visibleWindows = visibleWindows.ToArray();
|
|
1012
|
-
return res;
|
|
1013
|
-
}
|
|
1014
|
-
}
|
|
1015
|
-
"@
|
|
1016
|
-
|
|
1017
|
-
Add-Type -TypeDefinition $source -ReferencedAssemblies System.Drawing, System.Windows.Forms
|
|
1018
|
-
|
|
1019
|
-
if ($InfoOnly) {
|
|
1020
|
-
[WinCaptureHelper]::GetInfo() | ConvertTo-Json -Compress
|
|
1021
|
-
} else {
|
|
1022
|
-
[WinCaptureHelper]::Capture($OutPath) | ConvertTo-Json -Compress
|
|
1023
|
-
}
|
|
1024
|
-
`;
|
|
1025
|
-
export function ensureWinCaptureScript() {
|
|
1026
|
-
const scriptsDir = path.join(process.cwd(), '.ft', 'scripts');
|
|
1027
|
-
if (!fs.existsSync(scriptsDir)) {
|
|
1028
|
-
fs.mkdirSync(scriptsDir, { recursive: true });
|
|
1029
|
-
}
|
|
1030
|
-
const scriptPath = path.join(scriptsDir, 'winCapture.ps1');
|
|
1031
|
-
fs.writeFileSync(scriptPath, WIN_CAPTURE_SCRIPT_CONTENT, 'utf8');
|
|
1032
|
-
return scriptPath;
|
|
1033
|
-
}
|
|
1034
|
-
/**
|
|
1035
|
-
* Get screen metrics, active window title, visible windows, and cursor position.
|
|
1036
|
-
*/
|
|
1037
|
-
export async function getScreenInfo(appName) {
|
|
1038
|
-
const platform = process.platform;
|
|
1039
|
-
if (platform === 'win32') {
|
|
1040
|
-
try {
|
|
1041
|
-
const scriptPath = ensureWinCaptureScript();
|
|
1042
|
-
const { stdout } = await execAsync(`powershell -NoProfile -ExecutionPolicy Bypass -File "${scriptPath}" -InfoOnly`);
|
|
1043
|
-
const data = JSON.parse(stdout.trim());
|
|
1044
|
-
const visList = Array.isArray(data.visibleWindows) && data.visibleWindows.length > 0
|
|
1045
|
-
? ` | Visible Windows: ${data.visibleWindows.slice(0, 5).join(', ')}`
|
|
1046
|
-
: '';
|
|
1047
|
-
return {
|
|
1048
|
-
success: true,
|
|
1049
|
-
output: `Screen Info: Resolution ${data.screenWidth}x${data.screenHeight} | Cursor at (${data.cursorX}, ${data.cursorY}) | Active Window: "${data.activeWindow || appName || 'Desktop'}"${visList}`,
|
|
1050
|
-
details: {
|
|
1051
|
-
...data,
|
|
1052
|
-
activeWindow: data.activeWindow || appName || 'Desktop',
|
|
1053
|
-
},
|
|
1054
|
-
};
|
|
1055
|
-
}
|
|
1056
|
-
catch {
|
|
1057
|
-
return {
|
|
1058
|
-
success: true,
|
|
1059
|
-
output: `Screen Info (Default): Resolution 1920x1080 | Active Window: "${appName || 'Desktop'}"`,
|
|
1060
|
-
details: { screenWidth: 1920, screenHeight: 1080, activeWindow: appName || 'Desktop', cursorX: 0, cursorY: 0 },
|
|
1061
|
-
};
|
|
1062
|
-
}
|
|
1063
|
-
}
|
|
1064
|
-
else {
|
|
1065
|
-
return {
|
|
1066
|
-
success: true,
|
|
1067
|
-
output: `Screen Info (${platform}): Primary display active | Target App: "${appName || 'Desktop'}"`,
|
|
1068
|
-
details: { platform, appName: appName || 'Desktop' },
|
|
1069
|
-
};
|
|
1070
|
-
}
|
|
1071
|
-
}
|
|
1072
|
-
function escapeXaml(text) {
|
|
1073
|
-
return text
|
|
1074
|
-
.replace(/&/g, '&')
|
|
1075
|
-
.replace(/</g, '<')
|
|
1076
|
-
.replace(/>/g, '>')
|
|
1077
|
-
.replace(/"/g, '"')
|
|
1078
|
-
.replace(/'/g, ''')
|
|
1079
|
-
.replace(/•/g, '•');
|
|
1080
|
-
}
|
|
1081
|
-
let persistentAuraProcess = null;
|
|
1082
|
-
let persistentAuraFlagPath = null;
|
|
1083
|
-
/**
|
|
1084
|
-
* Start persistent futuristic sky-blue aura overlay on screen borders and corners.
|
|
1085
|
-
* Stays continuously active and animating as long as live screen sharing is running.
|
|
1086
|
-
*/
|
|
1087
|
-
export async function startScreenAuraOverlay(message = "Live Screen Share Active • Streaming to Model") {
|
|
1088
|
-
const platform = process.platform;
|
|
1089
|
-
if (platform !== 'win32')
|
|
1090
|
-
return;
|
|
1091
|
-
// Stop any existing overlay first
|
|
1092
|
-
stopScreenAuraOverlay();
|
|
1093
|
-
try {
|
|
1094
|
-
const scriptsDir = path.join(process.cwd(), '.ft', 'scripts');
|
|
1095
|
-
if (!fs.existsSync(scriptsDir)) {
|
|
1096
|
-
fs.mkdirSync(scriptsDir, { recursive: true });
|
|
1097
|
-
}
|
|
1098
|
-
const psFile = path.join(scriptsDir, 'scout_aura_persistent.ps1');
|
|
1099
|
-
const flagFile = path.join(scriptsDir, 'aura_active.flag');
|
|
1100
|
-
fs.writeFileSync(flagFile, String(Date.now()), 'utf8');
|
|
1101
|
-
persistentAuraFlagPath = flagFile;
|
|
1102
|
-
const subtitleText = " Perception Active";
|
|
1103
|
-
const safeMessage = escapeXaml(message);
|
|
1104
|
-
const safeSubtitle = escapeXaml(subtitleText);
|
|
1105
|
-
const parentPid = process.pid;
|
|
1106
|
-
const overlayPs = `
|
|
1107
|
-
Add-Type -AssemblyName PresentationFramework, PresentationCore, WindowsBase
|
|
1108
|
-
|
|
1109
|
-
[xml]$xaml = @"
|
|
1110
|
-
<Window xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
|
1111
|
-
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
|
1112
|
-
Title="Scout Aura Overlay"
|
|
1113
|
-
WindowStyle="None"
|
|
1114
|
-
AllowsTransparency="True"
|
|
1115
|
-
Background="Transparent"
|
|
1116
|
-
Topmost="True"
|
|
1117
|
-
ShowInTaskbar="False"
|
|
1118
|
-
ShowActivated="False"
|
|
1119
|
-
IsHitTestVisible="False">
|
|
1120
|
-
<Window.Resources>
|
|
1121
|
-
<Storyboard x:Key="AuraPulse" RepeatBehavior="Forever" AutoReverse="True">
|
|
1122
|
-
<DoubleAnimation Storyboard.TargetName="AuraBorder" Storyboard.TargetProperty="Opacity"
|
|
1123
|
-
From="0.4" To="1.0" Duration="0:0:0.6"/>
|
|
1124
|
-
<DoubleAnimation Storyboard.TargetName="CornerTL" Storyboard.TargetProperty="Opacity"
|
|
1125
|
-
From="0.5" To="1.0" Duration="0:0:0.6"/>
|
|
1126
|
-
<DoubleAnimation Storyboard.TargetName="CornerTR" Storyboard.TargetProperty="Opacity"
|
|
1127
|
-
From="0.5" To="1.0" Duration="0:0:0.6"/>
|
|
1128
|
-
<DoubleAnimation Storyboard.TargetName="CornerBL" Storyboard.TargetProperty="Opacity"
|
|
1129
|
-
From="0.5" To="1.0" Duration="0:0:0.6"/>
|
|
1130
|
-
<DoubleAnimation Storyboard.TargetName="CornerBR" Storyboard.TargetProperty="Opacity"
|
|
1131
|
-
From="0.5" To="1.0" Duration="0:0:0.6"/>
|
|
1132
|
-
<DoubleAnimation Storyboard.TargetName="AuraGlow" Storyboard.TargetProperty="Opacity"
|
|
1133
|
-
From="0.25" To="0.9" Duration="0:0:0.6"/>
|
|
1134
|
-
</Storyboard>
|
|
1135
|
-
</Window.Resources>
|
|
1136
|
-
|
|
1137
|
-
<Grid IsHitTestVisible="False">
|
|
1138
|
-
<!-- Outer Glowing Sky Blue Aura Border -->
|
|
1139
|
-
<Border x:Name="AuraGlow" BorderThickness="6" BorderBrush="#00d8ff" Margin="0">
|
|
1140
|
-
<Border.Effect>
|
|
1141
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="45" ShadowDepth="0" Opacity="0.95"/>
|
|
1142
|
-
</Border.Effect>
|
|
1143
|
-
</Border>
|
|
1144
|
-
<Border x:Name="AuraBorder" BorderThickness="3" BorderBrush="#38bdf8" Margin="0">
|
|
1145
|
-
<Border.Effect>
|
|
1146
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
|
|
1147
|
-
</Border.Effect>
|
|
1148
|
-
</Border>
|
|
1149
|
-
|
|
1150
|
-
<!-- Sky Blue Corner Brackets with Aura Glow -->
|
|
1151
|
-
<!-- Top Left Corner -->
|
|
1152
|
-
<Canvas HorizontalAlignment="Left" VerticalAlignment="Top" Width="120" Height="120" Margin="6,6,0,0">
|
|
1153
|
-
<Path x:Name="CornerTL" Data="M 0,90 L 0,0 L 90,0" Stroke="#00e5ff" StrokeThickness="6">
|
|
1154
|
-
<Path.Effect>
|
|
1155
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
|
|
1156
|
-
</Path.Effect>
|
|
1157
|
-
</Path>
|
|
1158
|
-
</Canvas>
|
|
1159
|
-
|
|
1160
|
-
<!-- Top Right Corner -->
|
|
1161
|
-
<Canvas HorizontalAlignment="Right" VerticalAlignment="Top" Width="120" Height="120" Margin="0,6,6,0">
|
|
1162
|
-
<Path x:Name="CornerTR" Data="M 30,0 L 120,0 L 120,90" Stroke="#00e5ff" StrokeThickness="6">
|
|
1163
|
-
<Path.Effect>
|
|
1164
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
|
|
1165
|
-
</Path.Effect>
|
|
1166
|
-
</Path>
|
|
1167
|
-
</Canvas>
|
|
1168
|
-
|
|
1169
|
-
<!-- Bottom Left Corner -->
|
|
1170
|
-
<Canvas HorizontalAlignment="Left" VerticalAlignment="Bottom" Width="120" Height="120" Margin="6,0,0,6">
|
|
1171
|
-
<Path x:Name="CornerBL" Data="M 0,30 L 0,120 L 90,120" Stroke="#00e5ff" StrokeThickness="6">
|
|
1172
|
-
<Path.Effect>
|
|
1173
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
|
|
1174
|
-
</Path.Effect>
|
|
1175
|
-
</Path>
|
|
1176
|
-
</Canvas>
|
|
1177
|
-
|
|
1178
|
-
<!-- Bottom Right Corner -->
|
|
1179
|
-
<Canvas HorizontalAlignment="Right" VerticalAlignment="Bottom" Width="120" Height="120" Margin="0,0,6,6">
|
|
1180
|
-
<Path x:Name="CornerBR" Data="M 30,120 L 120,120 L 120,30" Stroke="#00e5ff" StrokeThickness="6">
|
|
1181
|
-
<Path.Effect>
|
|
1182
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
|
|
1183
|
-
</Path.Effect>
|
|
1184
|
-
</Path>
|
|
1185
|
-
</Canvas>
|
|
1186
|
-
|
|
1187
|
-
<!-- Top Floating Status Badge -->
|
|
1188
|
-
<Border HorizontalAlignment="Center" VerticalAlignment="Top" Margin="0,22,0,0"
|
|
1189
|
-
Background="#EE03121F" BorderBrush="#00e5ff" BorderThickness="1.8" CornerRadius="24" Padding="26,10">
|
|
1190
|
-
<Border.Effect>
|
|
1191
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="0.95"/>
|
|
1192
|
-
</Border.Effect>
|
|
1193
|
-
<StackPanel Orientation="Horizontal" VerticalAlignment="Center">
|
|
1194
|
-
<Ellipse Width="12" Height="12" Fill="#00e5ff" Margin="0,0,12,0">
|
|
1195
|
-
<Ellipse.Effect>
|
|
1196
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="12" ShadowDepth="0"/>
|
|
1197
|
-
</Ellipse.Effect>
|
|
1198
|
-
</Ellipse>
|
|
1199
|
-
<TextBlock Text="${safeMessage}" Foreground="#ffffff" FontSize="16" FontWeight="Bold" VerticalAlignment="Center"/>
|
|
1200
|
-
<TextBlock Text="${safeSubtitle}" Foreground="#38bdf8" FontSize="13.5" FontWeight="SemiBold" Margin="10,0,0,0" VerticalAlignment="Center"/>
|
|
1201
|
-
</StackPanel>
|
|
1202
|
-
</Border>
|
|
1203
|
-
</Grid>
|
|
1204
|
-
</Window>
|
|
1205
|
-
"@
|
|
1206
|
-
|
|
1207
|
-
$reader = (New-Object System.Xml.XmlNodeReader $xaml)
|
|
1208
|
-
$window = [System.Windows.Markup.XamlReader]::Load($reader)
|
|
1209
|
-
|
|
1210
|
-
$window.Left = [System.Windows.SystemParameters]::VirtualScreenLeft
|
|
1211
|
-
$window.Top = [System.Windows.SystemParameters]::VirtualScreenTop
|
|
1212
|
-
$window.Width = [System.Windows.SystemParameters]::VirtualScreenWidth
|
|
1213
|
-
$window.Height = [System.Windows.SystemParameters]::VirtualScreenHeight
|
|
1214
|
-
$window.Topmost = $true
|
|
1215
|
-
$window.ShowActivated = $false
|
|
1216
|
-
|
|
1217
|
-
$storyboard = $window.Resources["AuraPulse"]
|
|
1218
|
-
$window.Add_Loaded({
|
|
1219
|
-
$storyboard.Begin($window)
|
|
1220
|
-
})
|
|
1221
|
-
|
|
1222
|
-
$flagPath = "${flagFile.replace(/\\/g, '\\\\')}"
|
|
1223
|
-
$parentPid = ${parentPid}
|
|
1224
|
-
|
|
1225
|
-
# Check if screen sharing is active or parent exited every 350ms
|
|
1226
|
-
$timer = New-Object System.Windows.Threading.DispatcherTimer
|
|
1227
|
-
$timer.Interval = [TimeSpan]::FromMilliseconds(350)
|
|
1228
|
-
$timer.Add_Tick({
|
|
1229
|
-
$shouldClose = $false
|
|
1230
|
-
if (-not (Test-Path $flagPath)) {
|
|
1231
|
-
$shouldClose = $true
|
|
1232
|
-
} else {
|
|
1233
|
-
$proc = Get-Process -Id $parentPid -ErrorAction SilentlyContinue
|
|
1234
|
-
if (-not $proc) {
|
|
1235
|
-
$shouldClose = $true
|
|
1236
|
-
}
|
|
1237
|
-
}
|
|
1238
|
-
if ($shouldClose) {
|
|
1239
|
-
$timer.Stop()
|
|
1240
|
-
$window.Close()
|
|
1241
|
-
}
|
|
1242
|
-
})
|
|
1243
|
-
$timer.Start()
|
|
1244
|
-
|
|
1245
|
-
[void]$window.ShowDialog()
|
|
1246
|
-
`;
|
|
1247
|
-
fs.writeFileSync(psFile, '\uFEFF' + overlayPs, 'utf8');
|
|
1248
|
-
const child = spawn('powershell', ['-WindowStyle', 'Hidden', '-NoProfile', '-Sta', '-ExecutionPolicy', 'Bypass', '-File', psFile], {
|
|
1249
|
-
detached: false,
|
|
1250
|
-
stdio: 'ignore',
|
|
1251
|
-
});
|
|
1252
|
-
child.unref();
|
|
1253
|
-
persistentAuraProcess = child;
|
|
1254
|
-
}
|
|
1255
|
-
catch { }
|
|
1256
|
-
}
|
|
1257
|
-
/**
|
|
1258
|
-
* Stop persistent screen aura animation when live screen sharing stops.
|
|
1259
|
-
*/
|
|
1260
|
-
export function stopScreenAuraOverlay() {
|
|
1261
|
-
if (persistentAuraFlagPath && fs.existsSync(persistentAuraFlagPath)) {
|
|
1262
|
-
try {
|
|
1263
|
-
fs.unlinkSync(persistentAuraFlagPath);
|
|
1264
|
-
}
|
|
1265
|
-
catch { }
|
|
1266
|
-
persistentAuraFlagPath = null;
|
|
1267
|
-
}
|
|
1268
|
-
if (persistentAuraProcess) {
|
|
1269
|
-
try {
|
|
1270
|
-
persistentAuraProcess.kill();
|
|
1271
|
-
}
|
|
1272
|
-
catch { }
|
|
1273
|
-
persistentAuraProcess = null;
|
|
1274
|
-
}
|
|
1275
|
-
}
|
|
1276
|
-
/**
|
|
1277
|
-
* Display a futuristic sky-blue aura animation on screen borders and corners,
|
|
1278
|
-
* show the warning "Scout is on the screen.", and optionally block external inputs.
|
|
1279
|
-
* If durationMs <= 0, aura stays on continuously until stopScreenAuraOverlay() is called.
|
|
1280
|
-
*/
|
|
1281
|
-
export async function showScreenAuraOverlay(durationMs = 2200, message = "Scout is viewing the screen.", blockInput = false) {
|
|
1282
|
-
if (durationMs <= 0) {
|
|
1283
|
-
return await startScreenAuraOverlay(message);
|
|
1284
|
-
}
|
|
1285
|
-
const platform = process.platform;
|
|
1286
|
-
if (platform !== 'win32')
|
|
1287
|
-
return;
|
|
1288
|
-
try {
|
|
1289
|
-
const scriptsDir = path.join(process.cwd(), '.ft', 'scripts');
|
|
1290
|
-
if (!fs.existsSync(scriptsDir)) {
|
|
1291
|
-
fs.mkdirSync(scriptsDir, { recursive: true });
|
|
1292
|
-
}
|
|
1293
|
-
const psFile = path.join(scriptsDir, 'scout_aura_overlay.ps1');
|
|
1294
|
-
const blockInputSnippet = blockInput ? `
|
|
1295
|
-
$sig = '[DllImport("user32.dll")] public static extern bool BlockInput(bool fBlock);'
|
|
1296
|
-
$b = Add-Type -memberDefinition $sig -name 'Win32AuraBlock' -namespace Win32Functions -passThru
|
|
1297
|
-
try { [void]$b::BlockInput($true) } catch {}
|
|
1298
|
-
` : '';
|
|
1299
|
-
const unblockSnippet = blockInput ? `
|
|
1300
|
-
try { [void]$b::BlockInput($false) } catch {}
|
|
1301
|
-
` : '';
|
|
1302
|
-
const subtitleText = blockInput ? " • External Input Locked" : " Perception Active";
|
|
1303
|
-
const safeMessage = escapeXaml(message);
|
|
1304
|
-
const safeSubtitle = escapeXaml(subtitleText);
|
|
1305
|
-
const overlayPs = `
|
|
1306
|
-
Add-Type -AssemblyName PresentationFramework, PresentationCore, WindowsBase
|
|
1307
|
-
${blockInputSnippet}
|
|
1308
|
-
|
|
1309
|
-
[xml]$xaml = @"
|
|
1310
|
-
<Window xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
|
1311
|
-
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
|
1312
|
-
Title="Scout Aura Overlay"
|
|
1313
|
-
WindowStyle="None"
|
|
1314
|
-
AllowsTransparency="True"
|
|
1315
|
-
Background="Transparent"
|
|
1316
|
-
Topmost="True"
|
|
1317
|
-
ShowInTaskbar="False"
|
|
1318
|
-
ShowActivated="False"
|
|
1319
|
-
IsHitTestVisible="False">
|
|
1320
|
-
<Window.Resources>
|
|
1321
|
-
<Storyboard x:Key="AuraPulse" RepeatBehavior="Forever" AutoReverse="True">
|
|
1322
|
-
<DoubleAnimation Storyboard.TargetName="AuraBorder" Storyboard.TargetProperty="Opacity"
|
|
1323
|
-
From="0.4" To="1.0" Duration="0:0:0.6"/>
|
|
1324
|
-
<DoubleAnimation Storyboard.TargetName="CornerTL" Storyboard.TargetProperty="Opacity"
|
|
1325
|
-
From="0.5" To="1.0" Duration="0:0:0.6"/>
|
|
1326
|
-
<DoubleAnimation Storyboard.TargetName="CornerTR" Storyboard.TargetProperty="Opacity"
|
|
1327
|
-
From="0.5" To="1.0" Duration="0:0:0.6"/>
|
|
1328
|
-
<DoubleAnimation Storyboard.TargetName="CornerBL" Storyboard.TargetProperty="Opacity"
|
|
1329
|
-
From="0.5" To="1.0" Duration="0:0:0.6"/>
|
|
1330
|
-
<DoubleAnimation Storyboard.TargetName="CornerBR" Storyboard.TargetProperty="Opacity"
|
|
1331
|
-
From="0.5" To="1.0" Duration="0:0:0.6"/>
|
|
1332
|
-
<DoubleAnimation Storyboard.TargetName="AuraGlow" Storyboard.TargetProperty="Opacity"
|
|
1333
|
-
From="0.25" To="0.9" Duration="0:0:0.6"/>
|
|
1334
|
-
</Storyboard>
|
|
1335
|
-
</Window.Resources>
|
|
1336
|
-
|
|
1337
|
-
<Grid IsHitTestVisible="False">
|
|
1338
|
-
<!-- Outer Glowing Sky Blue Aura Border -->
|
|
1339
|
-
<Border x:Name="AuraGlow" BorderThickness="6" BorderBrush="#00d8ff" Margin="0">
|
|
1340
|
-
<Border.Effect>
|
|
1341
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="45" ShadowDepth="0" Opacity="0.95"/>
|
|
1342
|
-
</Border.Effect>
|
|
1343
|
-
</Border>
|
|
1344
|
-
<Border x:Name="AuraBorder" BorderThickness="3" BorderBrush="#38bdf8" Margin="0">
|
|
1345
|
-
<Border.Effect>
|
|
1346
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
|
|
1347
|
-
</Border.Effect>
|
|
1348
|
-
</Border>
|
|
1349
|
-
|
|
1350
|
-
<!-- Sky Blue Corner Brackets with Aura Glow -->
|
|
1351
|
-
<!-- Top Left Corner -->
|
|
1352
|
-
<Canvas HorizontalAlignment="Left" VerticalAlignment="Top" Width="120" Height="120" Margin="6,6,0,0">
|
|
1353
|
-
<Path x:Name="CornerTL" Data="M 0,90 L 0,0 L 90,0" Stroke="#00e5ff" StrokeThickness="6">
|
|
1354
|
-
<Path.Effect>
|
|
1355
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
|
|
1356
|
-
</Path.Effect>
|
|
1357
|
-
</Path>
|
|
1358
|
-
</Canvas>
|
|
1359
|
-
|
|
1360
|
-
<!-- Top Right Corner -->
|
|
1361
|
-
<Canvas HorizontalAlignment="Right" VerticalAlignment="Top" Width="120" Height="120" Margin="0,6,6,0">
|
|
1362
|
-
<Path x:Name="CornerTR" Data="M 30,0 L 120,0 L 120,90" Stroke="#00e5ff" StrokeThickness="6">
|
|
1363
|
-
<Path.Effect>
|
|
1364
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
|
|
1365
|
-
</Path.Effect>
|
|
1366
|
-
</Path>
|
|
1367
|
-
</Canvas>
|
|
1368
|
-
|
|
1369
|
-
<!-- Bottom Left Corner -->
|
|
1370
|
-
<Canvas HorizontalAlignment="Left" VerticalAlignment="Bottom" Width="120" Height="120" Margin="6,0,0,6">
|
|
1371
|
-
<Path x:Name="CornerBL" Data="M 0,30 L 0,120 L 90,120" Stroke="#00e5ff" StrokeThickness="6">
|
|
1372
|
-
<Path.Effect>
|
|
1373
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
|
|
1374
|
-
</Path.Effect>
|
|
1375
|
-
</Path>
|
|
1376
|
-
</Canvas>
|
|
1377
|
-
|
|
1378
|
-
<!-- Bottom Right Corner -->
|
|
1379
|
-
<Canvas HorizontalAlignment="Right" VerticalAlignment="Bottom" Width="120" Height="120" Margin="0,0,6,6">
|
|
1380
|
-
<Path x:Name="CornerBR" Data="M 30,120 L 120,120 L 120,30" Stroke="#00e5ff" StrokeThickness="6">
|
|
1381
|
-
<Path.Effect>
|
|
1382
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
|
|
1383
|
-
</Path.Effect>
|
|
1384
|
-
</Path>
|
|
1385
|
-
</Canvas>
|
|
1386
|
-
|
|
1387
|
-
<!-- Top Floating Status Badge -->
|
|
1388
|
-
<Border HorizontalAlignment="Center" VerticalAlignment="Top" Margin="0,22,0,0"
|
|
1389
|
-
Background="#EE03121F" BorderBrush="#00e5ff" BorderThickness="1.8" CornerRadius="24" Padding="26,10">
|
|
1390
|
-
<Border.Effect>
|
|
1391
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="0.95"/>
|
|
1392
|
-
</Border.Effect>
|
|
1393
|
-
<StackPanel Orientation="Horizontal" VerticalAlignment="Center">
|
|
1394
|
-
<Ellipse Width="12" Height="12" Fill="#00e5ff" Margin="0,0,12,0">
|
|
1395
|
-
<Ellipse.Effect>
|
|
1396
|
-
<DropShadowEffect Color="#00e5ff" BlurRadius="12" ShadowDepth="0"/>
|
|
1397
|
-
</Ellipse.Effect>
|
|
1398
|
-
</Ellipse>
|
|
1399
|
-
<TextBlock Text="${safeMessage}" Foreground="#ffffff" FontSize="16" FontWeight="Bold" VerticalAlignment="Center"/>
|
|
1400
|
-
<TextBlock Text="${safeSubtitle}" Foreground="#38bdf8" FontSize="13.5" FontWeight="SemiBold" Margin="10,0,0,0" VerticalAlignment="Center"/>
|
|
1401
|
-
</StackPanel>
|
|
1402
|
-
</Border>
|
|
1403
|
-
</Grid>
|
|
1404
|
-
</Window>
|
|
1405
|
-
"@
|
|
1406
|
-
|
|
1407
|
-
$reader = (New-Object System.Xml.XmlNodeReader $xaml)
|
|
1408
|
-
$window = [System.Windows.Markup.XamlReader]::Load($reader)
|
|
1409
|
-
|
|
1410
|
-
$window.Left = [System.Windows.SystemParameters]::VirtualScreenLeft
|
|
1411
|
-
$window.Top = [System.Windows.SystemParameters]::VirtualScreenTop
|
|
1412
|
-
$window.Width = [System.Windows.SystemParameters]::VirtualScreenWidth
|
|
1413
|
-
$window.Height = [System.Windows.SystemParameters]::VirtualScreenHeight
|
|
1414
|
-
$window.Topmost = $true
|
|
1415
|
-
$window.ShowActivated = $false
|
|
1416
|
-
|
|
1417
|
-
$storyboard = $window.Resources["AuraPulse"]
|
|
1418
|
-
$window.Add_Loaded({
|
|
1419
|
-
$storyboard.Begin($window)
|
|
1420
|
-
})
|
|
1421
|
-
|
|
1422
|
-
# Safety timer to close overlay and guarantee unblock
|
|
1423
|
-
$timer = New-Object System.Windows.Threading.DispatcherTimer
|
|
1424
|
-
$timer.Interval = [TimeSpan]::FromMilliseconds(${durationMs})
|
|
1425
|
-
$timer.Add_Tick({
|
|
1426
|
-
$timer.Stop()
|
|
1427
|
-
${unblockSnippet}
|
|
1428
|
-
$window.Close()
|
|
1429
|
-
})
|
|
1430
|
-
$timer.Start()
|
|
1431
|
-
|
|
1432
|
-
[void]$window.ShowDialog()
|
|
1433
|
-
${unblockSnippet}
|
|
1434
|
-
`;
|
|
1435
|
-
fs.writeFileSync(psFile, '\uFEFF' + overlayPs, 'utf8');
|
|
1436
|
-
await execAsync(`powershell -WindowStyle Hidden -NoProfile -Sta -ExecutionPolicy Bypass -File "${psFile}"`, { timeout: durationMs + 3000 });
|
|
1437
|
-
}
|
|
1438
|
-
catch {
|
|
1439
|
-
if (blockInput) {
|
|
1440
|
-
try {
|
|
1441
|
-
await execAsync(`powershell -NoProfile -Command "$sig='[DllImport(\\"user32.dll\\")] public static extern bool BlockInput(bool f);';$b=Add-Type -m $sig -n B -p;[void]$b::BlockInput($false)"`);
|
|
1442
|
-
}
|
|
1443
|
-
catch { }
|
|
1444
|
-
}
|
|
1445
|
-
}
|
|
1446
|
-
}
|
|
1447
|
-
const capturedScreenshots = new Set();
|
|
1448
|
-
/**
|
|
1449
|
-
* Register a captured screenshot file path for automatic post-task cleanup.
|
|
1450
|
-
*/
|
|
1451
|
-
export function registerCapturedScreenshot(filePath) {
|
|
1452
|
-
if (filePath) {
|
|
1453
|
-
capturedScreenshots.add(path.resolve(filePath));
|
|
1454
|
-
}
|
|
1455
|
-
}
|
|
1456
|
-
/**
|
|
1457
|
-
* Clean up all captured screenshots permanently from disk after task completion.
|
|
1458
|
-
*/
|
|
1459
|
-
export function cleanupScreenshots() {
|
|
1460
|
-
let deletedCount = 0;
|
|
1461
|
-
// 1. Delete all tracked screenshots
|
|
1462
|
-
for (const filePath of capturedScreenshots) {
|
|
1463
|
-
try {
|
|
1464
|
-
if (fs.existsSync(filePath)) {
|
|
1465
|
-
fs.unlinkSync(filePath);
|
|
1466
|
-
deletedCount++;
|
|
1467
|
-
}
|
|
1468
|
-
}
|
|
1469
|
-
catch { }
|
|
1470
|
-
}
|
|
1471
|
-
capturedScreenshots.clear();
|
|
1472
|
-
// 2. Clean any remaining files in .ft/screenshots
|
|
1473
|
-
try {
|
|
1474
|
-
const screenshotsDir = path.join(process.cwd(), '.ft', 'screenshots');
|
|
1475
|
-
if (fs.existsSync(screenshotsDir)) {
|
|
1476
|
-
const files = fs.readdirSync(screenshotsDir);
|
|
1477
|
-
for (const file of files) {
|
|
1478
|
-
if (file.startsWith('screen_') && (file.endsWith('.png') || file.endsWith('.jpg'))) {
|
|
1479
|
-
try {
|
|
1480
|
-
fs.unlinkSync(path.join(screenshotsDir, file));
|
|
1481
|
-
deletedCount++;
|
|
1482
|
-
}
|
|
1483
|
-
catch { }
|
|
1484
|
-
}
|
|
1485
|
-
}
|
|
1486
|
-
}
|
|
1487
|
-
}
|
|
1488
|
-
catch { }
|
|
1489
|
-
return { success: true, deletedCount };
|
|
1490
|
-
}
|
|
1491
|
-
/**
|
|
1492
|
-
* Capture full desktop screen or specified window screenshot without running shell commands.
|
|
1493
|
-
* Screenshots are tracked and will be deleted automatically upon task completion.
|
|
1494
|
-
*/
|
|
1495
|
-
export async function captureScreen(appNameOrScreen) {
|
|
1496
|
-
const platform = process.platform;
|
|
1497
|
-
const screenshotsDir = path.join(process.cwd(), '.ft', 'screenshots');
|
|
1498
|
-
try {
|
|
1499
|
-
if (!fs.existsSync(screenshotsDir)) {
|
|
1500
|
-
fs.mkdirSync(screenshotsDir, { recursive: true });
|
|
1501
|
-
}
|
|
1502
|
-
}
|
|
1503
|
-
catch { }
|
|
1504
|
-
const filename = `screen_${Date.now()}.png`;
|
|
1505
|
-
const outPath = path.join(screenshotsDir, filename);
|
|
1506
|
-
if (platform === 'win32') {
|
|
1507
|
-
try {
|
|
1508
|
-
const scriptPath = ensureWinCaptureScript();
|
|
1509
|
-
const { stdout } = await execAsync(`powershell -NoProfile -ExecutionPolicy Bypass -File "${scriptPath}" -OutPath "${outPath}"`);
|
|
1510
|
-
const meta = JSON.parse(stdout.trim());
|
|
1511
|
-
if (meta.success && fs.existsSync(outPath) && fs.statSync(outPath).size > 0) {
|
|
1512
|
-
registerCapturedScreenshot(outPath);
|
|
1513
|
-
showScreenAuraOverlay(1800, "Scout is viewing the screen...", false).catch(() => { });
|
|
1514
|
-
const visibleList = Array.isArray(meta.visibleWindows) && meta.visibleWindows.length > 0
|
|
1515
|
-
? `\n- Visible Windows: ${meta.visibleWindows.slice(0, 6).map((w) => `"${w}"`).join(', ')}`
|
|
1516
|
-
: '';
|
|
1517
|
-
return {
|
|
1518
|
-
success: true,
|
|
1519
|
-
output: `Screen Captured Successfully!\n- Screenshot Saved To: ${outPath}\n- Resolution: ${meta.width}x${meta.height}\n- Active Window: "${meta.activeWindow || 'Desktop'}"${visibleList}\n- Seeing Status: Screen is visible and captured. (NOTE: This temporary screenshot will be automatically deleted upon task completion).`,
|
|
1520
|
-
details: { ...meta, outPath },
|
|
1521
|
-
};
|
|
1522
|
-
}
|
|
1523
|
-
throw new Error(meta.error || 'Desktop capture returned false');
|
|
1524
|
-
}
|
|
1525
|
-
catch (err) {
|
|
1526
|
-
// Fallback to Graphics.CopyFromScreen
|
|
1527
|
-
const psScript = `
|
|
1528
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
1529
|
-
Add-Type -AssemblyName System.Drawing
|
|
1530
|
-
$screen = [System.Windows.Forms.Screen]::PrimaryScreen
|
|
1531
|
-
$bounds = $screen.Bounds
|
|
1532
|
-
$bmp = New-Object System.Drawing.Bitmap($bounds.Width, $bounds.Height)
|
|
1533
|
-
$graphics = [System.Drawing.Graphics]::FromImage($bmp)
|
|
1534
|
-
try { $graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size) } catch {}
|
|
1535
|
-
$bmp.Save("${outPath.replace(/\\/g, '\\\\')}", [System.Drawing.Imaging.ImageFormat]::Png)
|
|
1536
|
-
$graphics.Dispose()
|
|
1537
|
-
$bmp.Dispose()
|
|
1538
|
-
@{ savedPath = "${outPath.replace(/\\/g, '\\\\')}"; width = $bounds.Width; height = $bounds.Height; activeWindow = "${appNameOrScreen || 'Desktop'}" } | ConvertTo-Json
|
|
1539
|
-
`;
|
|
1540
|
-
try {
|
|
1541
|
-
const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
|
|
1542
|
-
const { stdout } = await execAsync(`powershell -NoProfile -EncodedCommand ${encodedScript}`);
|
|
1543
|
-
const meta = JSON.parse(stdout.trim());
|
|
1544
|
-
registerCapturedScreenshot(outPath);
|
|
1545
|
-
return {
|
|
1546
|
-
success: true,
|
|
1547
|
-
output: `Screen Captured Successfully!\n- Screenshot Saved To: ${outPath}\n- Resolution: ${meta.width}x${meta.height}\n- Active Window: "${meta.activeWindow || 'Desktop'}"\n- Seeing Status: Screen is visible and captured.`,
|
|
1548
|
-
details: { ...meta, outPath },
|
|
1549
|
-
};
|
|
1550
|
-
}
|
|
1551
|
-
catch (fallbackErr) {
|
|
1552
|
-
return {
|
|
1553
|
-
success: false,
|
|
1554
|
-
output: `Failed to capture screen: ${err?.message || fallbackErr?.message || String(err)}`,
|
|
1555
|
-
};
|
|
1556
|
-
}
|
|
1557
|
-
}
|
|
1558
|
-
}
|
|
1559
|
-
else if (platform === 'darwin') {
|
|
1560
|
-
try {
|
|
1561
|
-
await execAsync(`screencapture -x "${outPath}"`);
|
|
1562
|
-
registerCapturedScreenshot(outPath);
|
|
1563
|
-
return {
|
|
1564
|
-
success: true,
|
|
1565
|
-
output: `macOS Screen Captured Successfully! Saved To: ${outPath}\n(NOTE: This temporary screenshot will be automatically deleted upon task completion).`,
|
|
1566
|
-
details: { outPath, platform: 'darwin' },
|
|
1567
|
-
};
|
|
1568
|
-
}
|
|
1569
|
-
catch (err) {
|
|
1570
|
-
return {
|
|
1571
|
-
success: false,
|
|
1572
|
-
output: `Failed macOS screen capture: ${err?.message || String(err)}`,
|
|
1573
|
-
};
|
|
1574
|
-
}
|
|
1575
|
-
}
|
|
1576
|
-
else {
|
|
1577
|
-
try {
|
|
1578
|
-
await execAsync(`import -window root "${outPath}" || scrot "${outPath}"`);
|
|
1579
|
-
registerCapturedScreenshot(outPath);
|
|
1580
|
-
return {
|
|
1581
|
-
success: true,
|
|
1582
|
-
output: `Linux Screen Captured Successfully! Saved To: ${outPath}\n(NOTE: This temporary screenshot will be automatically deleted upon task completion).`,
|
|
1583
|
-
details: { outPath, platform: 'linux' },
|
|
1584
|
-
};
|
|
1585
|
-
}
|
|
1586
|
-
catch (err) {
|
|
1587
|
-
return {
|
|
1588
|
-
success: false,
|
|
1589
|
-
output: `Failed Linux screen capture: ${err?.message || String(err)}`,
|
|
1590
|
-
};
|
|
1591
|
-
}
|
|
1592
|
-
}
|
|
1593
|
-
}
|
|
1594
|
-
/**
|
|
1595
|
-
* Call Vision AI with graceful multi-provider fallback.
|
|
1596
|
-
* Tries NVIDIA Vision (meta/llama-3.2-11b-vision-instruct), Gemini (gemini-2.0-flash), OpenAI (gpt-4o-mini), and BYOK.
|
|
1597
|
-
*/
|
|
1598
|
-
export async function callVisionAi(outPath, prompt, maxTokens = 350) {
|
|
1599
|
-
if (!outPath || !fs.existsSync(outPath))
|
|
1600
|
-
return null;
|
|
1601
|
-
let imageBase64;
|
|
1602
|
-
try {
|
|
1603
|
-
imageBase64 = fs.readFileSync(outPath).toString('base64');
|
|
1604
|
-
}
|
|
1605
|
-
catch {
|
|
1606
|
-
return null;
|
|
1607
|
-
}
|
|
1608
|
-
const nvidiaKey = process.env.NVIDIA_API_KEY || process.env.NVCF_API_KEY;
|
|
1609
|
-
const geminiKey = process.env.GEMINI_API_KEY;
|
|
1610
|
-
const openaiKey = process.env.OPENAI_API_KEY;
|
|
1611
|
-
const byokKey = process.env.BYOK_API_KEY;
|
|
1612
|
-
const candidates = [];
|
|
1613
|
-
if (nvidiaKey) {
|
|
1614
|
-
candidates.push({ apiKey: nvidiaKey, baseURL: 'https://integrate.api.nvidia.com/v1', model: 'meta/llama-3.2-11b-vision-instruct', timeout: 35000 });
|
|
1615
|
-
}
|
|
1616
|
-
if (geminiKey) {
|
|
1617
|
-
candidates.push({ apiKey: geminiKey, baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', model: 'gemini-2.0-flash', timeout: 20000 });
|
|
1618
|
-
}
|
|
1619
|
-
if (openaiKey) {
|
|
1620
|
-
candidates.push({ apiKey: openaiKey, model: 'gpt-4o-mini', timeout: 20000 });
|
|
1621
|
-
}
|
|
1622
|
-
if (byokKey) {
|
|
1623
|
-
candidates.push({ apiKey: byokKey, model: 'gpt-4o-mini', timeout: 20000 });
|
|
1624
|
-
}
|
|
1625
|
-
for (const cand of candidates) {
|
|
1626
|
-
try {
|
|
1627
|
-
const client = new OpenAI({ apiKey: cand.apiKey, baseURL: cand.baseURL, timeout: cand.timeout });
|
|
1628
|
-
const resp = await client.chat.completions.create({
|
|
1629
|
-
model: cand.model,
|
|
1630
|
-
messages: [
|
|
1631
|
-
{
|
|
1632
|
-
role: 'user',
|
|
1633
|
-
content: [
|
|
1634
|
-
{ type: 'text', text: prompt },
|
|
1635
|
-
{ type: 'image_url', image_url: { url: `data:image/png;base64,${imageBase64}` } },
|
|
1636
|
-
],
|
|
1637
|
-
},
|
|
1638
|
-
],
|
|
1639
|
-
max_tokens: maxTokens,
|
|
1640
|
-
});
|
|
1641
|
-
const reply = resp.choices[0]?.message?.content?.trim();
|
|
1642
|
-
if (reply) {
|
|
1643
|
-
return reply;
|
|
1644
|
-
}
|
|
1645
|
-
}
|
|
1646
|
-
catch {
|
|
1647
|
-
continue;
|
|
1648
|
-
}
|
|
1649
|
-
}
|
|
1650
|
-
return null;
|
|
1651
|
-
}
|
|
1652
|
-
/**
|
|
1653
|
-
* Visual screen analysis. Takes a full-screen screenshot and provides visual context
|
|
1654
|
-
* (including AI Vision inspection if an API key is available).
|
|
1655
|
-
* All temporary screenshots are automatically deleted upon task completion.
|
|
1656
|
-
*/
|
|
1657
|
-
export async function analyzeScreen(appNameOrScreen, targetQuery) {
|
|
1658
|
-
const capture = await captureScreen(appNameOrScreen);
|
|
1659
|
-
const info = await getScreenInfo(appNameOrScreen);
|
|
1660
|
-
let visualAiAnalysis = '';
|
|
1661
|
-
const outPath = capture.details?.outPath;
|
|
1662
|
-
// Optional Vision Model Analysis if screenshot exists and has valid content
|
|
1663
|
-
if (capture.success && outPath && fs.existsSync(outPath)) {
|
|
1664
|
-
try {
|
|
1665
|
-
if (fs.statSync(outPath).size > 0) {
|
|
1666
|
-
const prompt = targetQuery
|
|
1667
|
-
? `Describe what is visible on this screen specifically regarding: "${targetQuery}". Keep it concise, noting visible UI elements, open windows, tabs, buttons, or error messages.`
|
|
1668
|
-
: `Briefly describe what is currently visible on this desktop screen: active window, main content/tabs, buttons, and system state in 2-3 sentences.`;
|
|
1669
|
-
const reply = await callVisionAi(outPath, prompt, 350);
|
|
1670
|
-
if (reply && reply.trim()) {
|
|
1671
|
-
visualAiAnalysis = `\nAI Visual Screen Analysis:\n${reply.trim()}`;
|
|
1672
|
-
}
|
|
1673
|
-
}
|
|
1674
|
-
}
|
|
1675
|
-
catch {
|
|
1676
|
-
// Vision API call is optional; fallback gracefully to system metadata
|
|
1677
|
-
}
|
|
1678
|
-
}
|
|
1679
|
-
const activeWin = capture.details?.activeWindow || info.details?.activeWindow || appNameOrScreen || 'Active Desktop';
|
|
1680
|
-
const visWins = Array.isArray(capture.details?.visibleWindows) && capture.details.visibleWindows.length > 0
|
|
1681
|
-
? capture.details.visibleWindows
|
|
1682
|
-
: (Array.isArray(info.details?.visibleWindows) ? info.details.visibleWindows : []);
|
|
1683
|
-
const visibleWindowsText = visWins.length > 0
|
|
1684
|
-
? `\nVisible Open Windows:\n${visWins.slice(0, 8).map((w) => ` - "${w}"`).join('\n')}`
|
|
1685
|
-
: '';
|
|
1686
|
-
const report = [
|
|
1687
|
-
`=== SCOUT SCRATCHPAD DESKTOP SCREEN VISUAL INSPECTION ===`,
|
|
1688
|
-
`Target App / Screen Context: "${appNameOrScreen || 'Desktop'}"`,
|
|
1689
|
-
`Screen Resolution: ${info.details?.screenWidth || capture.details?.width || 1920}x${info.details?.screenHeight || capture.details?.height || 1080}`,
|
|
1690
|
-
`Active Window Focus: "${activeWin}"${visibleWindowsText}`,
|
|
1691
|
-
`Mouse Cursor Position: (${info.details?.cursorX || capture.details?.cursorX || 0}, ${info.details?.cursorY || capture.details?.cursorY || 0})`,
|
|
1692
|
-
`Screenshot Status: Captured temporarily for visual inspection. (Will be automatically deleted upon task completion)`,
|
|
1693
|
-
`Visual Status: Screen is visible and verified.${visualAiAnalysis}`,
|
|
1694
|
-
].join('\n');
|
|
1695
|
-
return {
|
|
1696
|
-
success: capture.success,
|
|
1697
|
-
app: appNameOrScreen || 'desktop',
|
|
1698
|
-
action: 'analyze_screen',
|
|
1699
|
-
output: report,
|
|
1700
|
-
details: { capture: capture.details, info: info.details },
|
|
1701
|
-
};
|
|
1702
|
-
}
|
|
1703
|
-
/**
|
|
1704
|
-
* Full Mouse Cursor & Keyboard Takeover on specified screen or window.
|
|
1705
|
-
*/
|
|
1706
|
-
export async function takeoverControl(appName, durationMs = 3000, x, y) {
|
|
1707
|
-
const { category, name } = normalizeAppName(appName);
|
|
1708
|
-
const targetApp = (name && name !== 'desktop' && name !== 'system') ? name : undefined;
|
|
1709
|
-
if (targetApp) {
|
|
1710
|
-
setLiveScreenTargetApp(targetApp);
|
|
1711
|
-
}
|
|
1712
|
-
if (name && name !== 'desktop' && name !== 'system' && category !== 'browser') {
|
|
1713
|
-
await openApp(name);
|
|
1714
|
-
}
|
|
1715
|
-
// Trigger visual sky-blue aura overlay on screen borders & corners with warning badge & input block
|
|
1716
|
-
try {
|
|
1717
|
-
const auraLabel = targetApp ? `Scout is controlling ${targetApp}...` : "Scout is on the screen.";
|
|
1718
|
-
await showScreenAuraOverlay(Math.min(5000, Math.max(800, durationMs)), auraLabel, true);
|
|
1719
|
-
startLiveScreenShare({ targetApp }).catch(() => { });
|
|
1720
|
-
}
|
|
1721
|
-
catch { }
|
|
1722
|
-
const mouseRes = await moveAndClickMouse(x, y, 'left', name || appName);
|
|
1723
|
-
const lockRes = await lockAppInput(name || appName, durationMs);
|
|
1724
|
-
return {
|
|
1725
|
-
success: lockRes.success && mouseRes.success,
|
|
1726
|
-
output: `TAKEOVER EXECUTED: Scout Agent has taken over mouse cursor, window focus, and keyboard control on specified screen window ("${name || appName}"). Sky-blue aura HUD activated and user interruption locked for ${durationMs}ms. ${mouseRes.output} Agent can view, analyze, type, and click without shell commands.`,
|
|
1727
|
-
details: { appName: name || appName, durationMs, x, y, mouseOutput: mouseRes.output },
|
|
1728
|
-
};
|
|
1729
|
-
}
|
|
1730
|
-
/**
|
|
1731
|
-
* Release user input lock / takeover.
|
|
1732
|
-
*/
|
|
1733
|
-
export async function releaseTakeover() {
|
|
1734
|
-
const platform = process.platform;
|
|
1735
|
-
if (platform === 'win32') {
|
|
1736
|
-
const psScript = `
|
|
1737
|
-
$sig = '[DllImport("user32.dll")] public static extern bool BlockInput(bool fBlock);'
|
|
1738
|
-
$b = Add-Type -memberDefinition $sig -name 'Win32Block' -namespace Win32Functions -passThru
|
|
1739
|
-
try { [void]$b::BlockInput($false) } catch {}
|
|
1740
|
-
`;
|
|
1741
|
-
try {
|
|
1742
|
-
const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
|
|
1743
|
-
await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
|
|
1744
|
-
}
|
|
1745
|
-
catch { }
|
|
1746
|
-
}
|
|
1747
|
-
return {
|
|
1748
|
-
success: true,
|
|
1749
|
-
output: 'Released mouse cursor and keyboard takeover. Control returned to user.',
|
|
1750
|
-
};
|
|
1751
|
-
}
|
|
1752
|
-
/**
|
|
1753
|
-
* Focus application window and simulate typing text + pressing keys (Enter/Tab) or hotkey combinations.
|
|
1754
|
-
*/
|
|
1755
|
-
export async function sendKeystrokes(appName, textToType, pressEnter = true, keyCombo) {
|
|
1756
|
-
const platform = process.platform;
|
|
1757
|
-
const targetText = textToType || '';
|
|
1758
|
-
const combo = (keyCombo || '').trim().toLowerCase();
|
|
1759
|
-
if (platform === 'win32') {
|
|
1760
|
-
const sanitized = targetText.replace(/"/g, '""').replace(/[\r\n]+/g, ' ');
|
|
1761
|
-
const sendKeysText = sanitized.replace(/[{}+^~%()\[\]]/g, '{$&}');
|
|
1762
|
-
let sendKeysCombo = '';
|
|
1763
|
-
if (combo) {
|
|
1764
|
-
const cLower = combo.toLowerCase().replace(/\s+/g, '');
|
|
1765
|
-
if (cLower.includes('ctrl+enter') || cLower.includes('ctrl+return'))
|
|
1766
|
-
sendKeysCombo = '^{ENTER}';
|
|
1767
|
-
else if (cLower.includes('alt+enter') || cLower.includes('alt+return'))
|
|
1768
|
-
sendKeysCombo = '%{ENTER}';
|
|
1769
|
-
else if (cLower.includes('shift+enter') || cLower.includes('shift+return'))
|
|
1770
|
-
sendKeysCombo = '+{ENTER}';
|
|
1771
|
-
else if (cLower.includes('ctrl+a') || cLower === 'select_all')
|
|
1772
|
-
sendKeysCombo = '^a';
|
|
1773
|
-
else if (cLower.includes('ctrl+c') || cLower === 'copy')
|
|
1774
|
-
sendKeysCombo = '^c';
|
|
1775
|
-
else if (cLower.includes('ctrl+v') || cLower === 'paste')
|
|
1776
|
-
sendKeysCombo = '^v';
|
|
1777
|
-
else if (cLower.includes('ctrl+x') || cLower === 'cut')
|
|
1778
|
-
sendKeysCombo = '^x';
|
|
1779
|
-
else if (cLower.includes('ctrl+z') || cLower === 'undo')
|
|
1780
|
-
sendKeysCombo = '^z';
|
|
1781
|
-
else if (cLower.includes('ctrl+s') || cLower === 'save')
|
|
1782
|
-
sendKeysCombo = '^s';
|
|
1783
|
-
else if (cLower.includes('ctrl+l') || cLower === 'address_bar')
|
|
1784
|
-
sendKeysCombo = '^l';
|
|
1785
|
-
else if (cLower.includes('ctrl+t') || cLower === 'new_tab')
|
|
1786
|
-
sendKeysCombo = '^t';
|
|
1787
|
-
else if (cLower.includes('ctrl+w') || cLower === 'close_tab')
|
|
1788
|
-
sendKeysCombo = '^w';
|
|
1789
|
-
else if (cLower.includes('ctrl+r') || cLower === 'reload')
|
|
1790
|
-
sendKeysCombo = '^r';
|
|
1791
|
-
else if (cLower.includes('ctrl+f') || cLower === 'find')
|
|
1792
|
-
sendKeysCombo = '^f';
|
|
1793
|
-
else if (cLower.includes('ctrl+n') || cLower === 'new_window')
|
|
1794
|
-
sendKeysCombo = '^n';
|
|
1795
|
-
else if (cLower.includes('ctrl+shift+i') || cLower.includes('devtools'))
|
|
1796
|
-
sendKeysCombo = '^+i';
|
|
1797
|
-
else if (cLower.includes('alt+d'))
|
|
1798
|
-
sendKeysCombo = '%d';
|
|
1799
|
-
else if (cLower.includes('alt+tab'))
|
|
1800
|
-
sendKeysCombo = '%{TAB}';
|
|
1801
|
-
else if (cLower.includes('alt+f4'))
|
|
1802
|
-
sendKeysCombo = '%{F4}';
|
|
1803
|
-
else if (cLower === 'enter' || cLower === 'return')
|
|
1804
|
-
sendKeysCombo = '{ENTER}';
|
|
1805
|
-
else if (cLower === 'esc' || cLower === 'escape')
|
|
1806
|
-
sendKeysCombo = '{ESC}';
|
|
1807
|
-
else if (cLower === 'tab')
|
|
1808
|
-
sendKeysCombo = '{TAB}';
|
|
1809
|
-
else if (cLower === 'backspace')
|
|
1810
|
-
sendKeysCombo = '{BACKSPACE}';
|
|
1811
|
-
else if (cLower === 'space')
|
|
1812
|
-
sendKeysCombo = ' ';
|
|
1813
|
-
else if (cLower === 'up')
|
|
1814
|
-
sendKeysCombo = '{UP}';
|
|
1815
|
-
else if (cLower === 'down')
|
|
1816
|
-
sendKeysCombo = '{DOWN}';
|
|
1817
|
-
else if (cLower === 'left')
|
|
1818
|
-
sendKeysCombo = '{LEFT}';
|
|
1819
|
-
else if (cLower === 'right')
|
|
1820
|
-
sendKeysCombo = '{RIGHT}';
|
|
1821
|
-
else
|
|
1822
|
-
sendKeysCombo = combo;
|
|
1823
|
-
}
|
|
1824
|
-
const psScript = `
|
|
1825
|
-
$sig = @'
|
|
1826
|
-
[DllImport("user32.dll")] public static extern bool BlockInput(bool fBlock);
|
|
1827
|
-
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
|
|
1828
|
-
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
|
|
1829
|
-
[DllImport("user32.dll")] public static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo);
|
|
1830
|
-
'@
|
|
1831
|
-
$b = Add-Type -memberDefinition $sig -name ('Win32Block_' + (Get-Random)) -namespace Win32Functions -passThru
|
|
1832
|
-
|
|
1833
|
-
try { [void]$b::BlockInput($true) } catch {}
|
|
1834
|
-
|
|
1835
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
1836
|
-
$ws = New-Object -ComObject WScript.Shell
|
|
1837
|
-
|
|
1838
|
-
try {
|
|
1839
|
-
$proc = Get-Process -Name chrome, msedge, firefox, brave, opera, "${appName}" -ErrorAction SilentlyContinue | Where-Object { $_.MainWindowHandle -ne 0 } | Select-Object -First 1
|
|
1840
|
-
if ($proc) {
|
|
1841
|
-
$b::keybd_event(0x12, 0, 0, [UIntPtr]::Zero)
|
|
1842
|
-
$b::keybd_event(0x12, 0, 2, [UIntPtr]::Zero)
|
|
1843
|
-
[void]$b::ShowWindow($proc.MainWindowHandle, 9)
|
|
1844
|
-
[void]$b::SetForegroundWindow($proc.MainWindowHandle)
|
|
1845
|
-
Start-Sleep -Milliseconds 250
|
|
1846
|
-
} else {
|
|
1847
|
-
$activated = $ws.AppActivate("${appName}")
|
|
1848
|
-
if (-not $activated) { $activated = $ws.AppActivate("Chrome") }
|
|
1849
|
-
if (-not $activated) { $activated = $ws.AppActivate("Edge") }
|
|
1850
|
-
if (-not $activated) { $activated = $ws.AppActivate("Firefox") }
|
|
1851
|
-
if (-not $activated) { $activated = $ws.AppActivate("Brave") }
|
|
1852
|
-
if (-not $activated) { $activated = $ws.AppActivate("Notepad") }
|
|
1853
|
-
if (-not $activated) { $activated = $ws.AppActivate("Visual Studio Code") }
|
|
1854
|
-
Start-Sleep -Milliseconds 300
|
|
1855
|
-
}
|
|
1856
|
-
} catch {}
|
|
1857
|
-
|
|
1858
|
-
Start-Sleep -Milliseconds 250
|
|
1859
|
-
|
|
1860
|
-
${sendKeysCombo ? `$ws.SendKeys("${sendKeysCombo}")` : ''}
|
|
1861
|
-
|
|
1862
|
-
${sanitized ? `
|
|
1863
|
-
$copied = $false
|
|
1864
|
-
try {
|
|
1865
|
-
Set-Clipboard -Value "${sanitized}"
|
|
1866
|
-
$copied = $true
|
|
1867
|
-
} catch {
|
|
1868
|
-
try {
|
|
1869
|
-
[System.Windows.Forms.Clipboard]::SetText("${sanitized}")
|
|
1870
|
-
$copied = $true
|
|
1871
|
-
} catch {}
|
|
1872
|
-
}
|
|
1873
|
-
if ($copied) {
|
|
1874
|
-
$ws.SendKeys("^v")
|
|
1875
|
-
} else {
|
|
1876
|
-
$ws.SendKeys("${sendKeysText}")
|
|
1877
|
-
}
|
|
1878
|
-
` : ''}
|
|
1879
|
-
|
|
1880
|
-
if (${pressEnter ? '$true' : '$false'}) {
|
|
1881
|
-
Start-Sleep -Milliseconds 200
|
|
1882
|
-
$ws.SendKeys("~")
|
|
1883
|
-
}
|
|
1884
|
-
|
|
1885
|
-
try { [void]$b::BlockInput($false) } catch {}
|
|
1886
|
-
`;
|
|
1887
|
-
try {
|
|
1888
|
-
const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
|
|
1889
|
-
await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
|
|
1890
|
-
return {
|
|
1891
|
-
success: true,
|
|
1892
|
-
output: `Successfully focused "${appName}" window and sent keyboard takeover inputs ("${targetText}"${combo ? ` [Hotkey: ${combo}]` : ''}${pressEnter ? ' + [ENTER]' : ''}).`
|
|
1893
|
-
};
|
|
1894
|
-
}
|
|
1895
|
-
catch (err) {
|
|
1896
|
-
return {
|
|
1897
|
-
success: false,
|
|
1898
|
-
output: `Failed to send keystrokes to "${appName}": ${err?.message || String(err)}`
|
|
1899
|
-
};
|
|
1900
|
-
}
|
|
1901
|
-
}
|
|
1902
|
-
else if (platform === 'darwin') {
|
|
1903
|
-
const escaped = targetText.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
1904
|
-
const script = `
|
|
1905
|
-
tell application "System Events"
|
|
1906
|
-
${escaped ? `keystroke "${escaped}"` : ''}
|
|
1907
|
-
${pressEnter ? 'key code 36' : ''}
|
|
1908
|
-
end tell
|
|
1909
|
-
`;
|
|
1910
|
-
try {
|
|
1911
|
-
await execAsync(`osascript -e '${script.replace(/'/g, "'\\''")}'`);
|
|
1912
|
-
return {
|
|
1913
|
-
success: true,
|
|
1914
|
-
output: `Sent macOS keyboard takeover keystrokes ("${targetText}").`
|
|
1915
|
-
};
|
|
1916
|
-
}
|
|
1917
|
-
catch (err) {
|
|
1918
|
-
return {
|
|
1919
|
-
success: false,
|
|
1920
|
-
output: `Failed macOS osascript keystrokes: ${err?.message || String(err)}`
|
|
1921
|
-
};
|
|
1922
|
-
}
|
|
1923
|
-
}
|
|
1924
|
-
else {
|
|
1925
|
-
try {
|
|
1926
|
-
const escaped = targetText.replace(/"/g, '\\"');
|
|
1927
|
-
await execAsync(`xdotool type "${escaped}" ${pressEnter ? '&& xdotool key Return' : ''}`);
|
|
1928
|
-
return {
|
|
1929
|
-
success: true,
|
|
1930
|
-
output: `Sent Linux keyboard takeover keystrokes ("${targetText}").`
|
|
1931
|
-
};
|
|
1932
|
-
}
|
|
1933
|
-
catch (err) {
|
|
1934
|
-
return {
|
|
1935
|
-
success: false,
|
|
1936
|
-
output: `xdotool keystroke error: ${err?.message || String(err)}`
|
|
1937
|
-
};
|
|
1938
|
-
}
|
|
1939
|
-
}
|
|
1940
|
-
}
|
|
1941
|
-
/**
|
|
1942
|
-
* Move cursor and perform OS mouse click at target coordinates or active window center.
|
|
1943
|
-
*/
|
|
1944
|
-
export async function moveAndClickMouse(x, y, button = 'left', appName) {
|
|
1945
|
-
const platform = process.platform;
|
|
1946
|
-
if (platform === 'win32') {
|
|
1947
|
-
const psScript = `
|
|
1948
|
-
Add-Type -AssemblyName System.Drawing
|
|
1949
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
1950
|
-
$sig = '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int cButtons, int dwExtraInfo); [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);'
|
|
1951
|
-
$m = Add-Type -memberDefinition $sig -name 'Win32Mouse' -namespace Win32Functions -passThru
|
|
1952
|
-
|
|
1953
|
-
${(appName && appName !== 'desktop' && appName !== 'system') ? `
|
|
1954
|
-
try {
|
|
1955
|
-
$proc = Get-Process -Name chrome, msedge, firefox, brave, "${appName}" -ErrorAction SilentlyContinue | Where-Object { $_.MainWindowHandle -ne 0 } | Select-Object -First 1
|
|
1956
|
-
if ($proc) {
|
|
1957
|
-
$sigFw = '[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);'
|
|
1958
|
-
$fw = Add-Type -memberDefinition $sigFw -name 'Win32FwMouse' -namespace Win32Functions -passThru
|
|
1959
|
-
[void]$fw::SetForegroundWindow($proc.MainWindowHandle)
|
|
1960
|
-
Start-Sleep -Milliseconds 250
|
|
1961
|
-
} else {
|
|
1962
|
-
$ws = New-Object -ComObject WScript.Shell
|
|
1963
|
-
[void]$ws.AppActivate("${appName}")
|
|
1964
|
-
Start-Sleep -Milliseconds 200
|
|
1965
|
-
}
|
|
1966
|
-
} catch {}
|
|
1967
|
-
` : ''}
|
|
1968
|
-
|
|
1969
|
-
$screen = [System.Windows.Forms.Screen]::PrimaryScreen
|
|
1970
|
-
$targetX = ${x !== undefined ? Math.round(x) : '[int]($screen.Bounds.Width / 2)'}
|
|
1971
|
-
$targetY = ${y !== undefined ? Math.round(y) : '[int]($screen.Bounds.Height / 2)'}
|
|
1972
|
-
|
|
1973
|
-
[void]$m::SetCursorPos($targetX, $targetY)
|
|
1974
|
-
try { [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point($targetX, $targetY) } catch {}
|
|
1975
|
-
Start-Sleep -Milliseconds 200
|
|
1976
|
-
|
|
1977
|
-
# MOUSEEVENTF_LEFTDOWN = 0x0002, MOUSEEVENTF_LEFTUP = 0x0004
|
|
1978
|
-
# MOUSEEVENTF_RIGHTDOWN = 0x0008, MOUSEEVENTF_RIGHTUP = 0x0010
|
|
1979
|
-
# MOUSEEVENTF_MIDDLEDOWN = 0x0020, MOUSEEVENTF_MIDDLEUP = 0x0040
|
|
1980
|
-
# MOUSEEVENTF_WHEEL = 0x0800
|
|
1981
|
-
${button === 'right' ? '$m::mouse_event(0x0008, 0, 0, 0, 0); $m::mouse_event(0x0010, 0, 0, 0, 0);' :
|
|
1982
|
-
button === 'middle' ? '$m::mouse_event(0x0020, 0, 0, 0, 0); $m::mouse_event(0x0040, 0, 0, 0, 0);' :
|
|
1983
|
-
button === 'scroll' ? '$m::mouse_event(0x0800, 0, 0, -120, 0);' :
|
|
1984
|
-
'$m::mouse_event(0x0002, 0, 0, 0, 0); $m::mouse_event(0x0004, 0, 0, 0, 0);'}
|
|
1985
|
-
${button === 'double' ? 'Start-Sleep -Milliseconds 100; $m::mouse_event(0x0002, 0, 0, 0, 0); $m::mouse_event(0x0004, 0, 0, 0, 0);' : ''}
|
|
1986
|
-
`;
|
|
1987
|
-
try {
|
|
1988
|
-
const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
|
|
1989
|
-
await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
|
|
1990
|
-
return {
|
|
1991
|
-
success: true,
|
|
1992
|
-
output: `Executed mouse cursor takeover ${button} click at screen coordinates (${x !== undefined ? x : 'screen center'}, ${y !== undefined ? y : 'screen center'}).`
|
|
1993
|
-
};
|
|
1994
|
-
}
|
|
1995
|
-
catch (err) {
|
|
1996
|
-
return {
|
|
1997
|
-
success: false,
|
|
1998
|
-
output: `Failed mouse takeover action: ${err?.message || String(err)}`
|
|
1999
|
-
};
|
|
2000
|
-
}
|
|
2001
|
-
}
|
|
2002
|
-
else if (platform === 'darwin') {
|
|
2003
|
-
const script = `
|
|
2004
|
-
tell application "System Events"
|
|
2005
|
-
${(x !== undefined && y !== undefined) ? `click at {${x}, ${y}}` : 'click'}
|
|
2006
|
-
end tell
|
|
2007
|
-
`;
|
|
2008
|
-
try {
|
|
2009
|
-
await execAsync(`osascript -e '${script.replace(/'/g, "'\\''")}'`);
|
|
2010
|
-
return {
|
|
2011
|
-
success: true,
|
|
2012
|
-
output: `Executed macOS mouse cursor click.`
|
|
2013
|
-
};
|
|
2014
|
-
}
|
|
2015
|
-
catch (err) {
|
|
2016
|
-
return {
|
|
2017
|
-
success: false,
|
|
2018
|
-
output: `Failed macOS mouse click: ${err?.message || String(err)}`
|
|
2019
|
-
};
|
|
2020
|
-
}
|
|
2021
|
-
}
|
|
2022
|
-
else {
|
|
2023
|
-
try {
|
|
2024
|
-
const cmd = (x !== undefined && y !== undefined)
|
|
2025
|
-
? `xdotool mousemove ${x} ${y} click ${button === 'right' ? '3' : '1'}`
|
|
2026
|
-
: `xdotool click ${button === 'right' ? '3' : '1'}`;
|
|
2027
|
-
await execAsync(cmd);
|
|
2028
|
-
return {
|
|
2029
|
-
success: true,
|
|
2030
|
-
output: `Executed Linux mouse click.`
|
|
2031
|
-
};
|
|
2032
|
-
}
|
|
2033
|
-
catch (err) {
|
|
2034
|
-
return {
|
|
2035
|
-
success: false,
|
|
2036
|
-
output: `xdotool click error: ${err?.message || String(err)}`
|
|
2037
|
-
};
|
|
2038
|
-
}
|
|
2039
|
-
}
|
|
2040
|
-
}
|
|
2041
|
-
/**
|
|
2042
|
-
* Prevent user interruption on targeted application window during scratchpad execution.
|
|
2043
|
-
* Focuses window and temporarily locks input / enforces frontmost focus while Scout is working.
|
|
2044
|
-
*/
|
|
2045
|
-
export async function lockAppInput(appName, durationMs = 2000) {
|
|
2046
|
-
const platform = process.platform;
|
|
2047
|
-
if (platform === 'win32') {
|
|
2048
|
-
const psScript = `
|
|
2049
|
-
$sig = '[DllImport("user32.dll")] public static extern bool BlockInput(bool fBlock);'
|
|
2050
|
-
$b = Add-Type -memberDefinition $sig -name 'Win32Block' -namespace Win32Functions -passThru
|
|
2051
|
-
|
|
2052
|
-
$ws = New-Object -ComObject WScript.Shell
|
|
2053
|
-
$activated = $ws.AppActivate("${appName}")
|
|
2054
|
-
if (-not $activated) { $activated = $ws.AppActivate("Chrome") }
|
|
2055
|
-
if (-not $activated) { $activated = $ws.AppActivate("Edge") }
|
|
2056
|
-
if (-not $activated) { $activated = $ws.AppActivate("Firefox") }
|
|
2057
|
-
if (-not $activated) { $activated = $ws.AppActivate("Brave") }
|
|
2058
|
-
|
|
2059
|
-
try { [void]$b::BlockInput($true) } catch {}
|
|
2060
|
-
Start-Sleep -Milliseconds ${Math.min(10000, Math.max(500, durationMs))}
|
|
2061
|
-
try { [void]$b::BlockInput($false) } catch {}
|
|
2062
|
-
`;
|
|
2063
|
-
try {
|
|
2064
|
-
const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
|
|
2065
|
-
await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
|
|
2066
|
-
return {
|
|
2067
|
-
success: true,
|
|
2068
|
-
output: `Locked user interruption on "${appName}" window for ${durationMs}ms while Scout Scratchpad was working.`
|
|
2069
|
-
};
|
|
2070
|
-
}
|
|
2071
|
-
catch (err) {
|
|
2072
|
-
return {
|
|
2073
|
-
success: false,
|
|
2074
|
-
output: `Focus lock applied for "${appName}".`
|
|
2075
|
-
};
|
|
2076
|
-
}
|
|
2077
|
-
}
|
|
2078
|
-
else if (platform === 'darwin') {
|
|
2079
|
-
const script = `
|
|
2080
|
-
tell application "System Events"
|
|
2081
|
-
tell application "${appName}" to activate
|
|
2082
|
-
end tell
|
|
2083
|
-
`;
|
|
2084
|
-
try {
|
|
2085
|
-
await execAsync(`osascript -e '${script.replace(/'/g, "'\\''")}'`);
|
|
2086
|
-
return {
|
|
2087
|
-
success: true,
|
|
2088
|
-
output: `Enforced frontmost focus on "${appName}" to prevent user interruption on macOS.`
|
|
2089
|
-
};
|
|
2090
|
-
}
|
|
2091
|
-
catch (err) {
|
|
2092
|
-
return {
|
|
2093
|
-
success: false,
|
|
2094
|
-
output: `macOS focus enforcement notice: ${err?.message || String(err)}`
|
|
2095
|
-
};
|
|
2096
|
-
}
|
|
2097
|
-
}
|
|
2098
|
-
else {
|
|
2099
|
-
return {
|
|
2100
|
-
success: true,
|
|
2101
|
-
output: `Enforced scratchpad focus lock on Linux for "${appName}".`
|
|
2102
|
-
};
|
|
2103
|
-
}
|
|
2104
|
-
}
|
|
2105
|
-
/**
|
|
2106
|
-
* List all active top-level application windows.
|
|
2107
|
-
*/
|
|
2108
|
-
export async function listWindows() {
|
|
2109
|
-
const platform = process.platform;
|
|
2110
|
-
if (platform === 'win32') {
|
|
2111
|
-
const psScript = `
|
|
2112
|
-
Get-Process | Where-Object { $_.MainWindowTitle -ne '' } | Select-Object Id, ProcessName, MainWindowTitle, MainWindowHandle | ConvertTo-Json
|
|
2113
|
-
`;
|
|
2114
|
-
try {
|
|
2115
|
-
const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
|
|
2116
|
-
const { stdout } = await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
|
|
2117
|
-
let rawList = [];
|
|
2118
|
-
try {
|
|
2119
|
-
rawList = JSON.parse(stdout.trim());
|
|
2120
|
-
}
|
|
2121
|
-
catch { }
|
|
2122
|
-
if (!Array.isArray(rawList))
|
|
2123
|
-
rawList = rawList ? [rawList] : [];
|
|
2124
|
-
const windows = rawList.map((w) => ({
|
|
2125
|
-
title: String(w.MainWindowTitle || ''),
|
|
2126
|
-
processName: String(w.ProcessName || ''),
|
|
2127
|
-
handle: String(w.MainWindowHandle || ''),
|
|
2128
|
-
}));
|
|
2129
|
-
const formatted = windows.map((w, idx) => ` ${idx + 1}. [${w.processName}] "${w.title}"`).join('\n');
|
|
2130
|
-
return {
|
|
2131
|
-
success: true,
|
|
2132
|
-
output: `Active Application Windows (${windows.length}):\n${formatted || 'No active top-level windows with title found.'}`,
|
|
2133
|
-
windows,
|
|
2134
|
-
};
|
|
2135
|
-
}
|
|
2136
|
-
catch (err) {
|
|
2137
|
-
return { success: false, output: `Failed to list windows: ${err?.message || String(err)}`, windows: [] };
|
|
2138
|
-
}
|
|
2139
|
-
}
|
|
2140
|
-
return { success: true, output: 'Window listing only supported on Windows.', windows: [] };
|
|
2141
|
-
}
|
|
2142
|
-
/**
|
|
2143
|
-
* Bring a target window to front and focus it.
|
|
2144
|
-
*/
|
|
2145
|
-
export async function focusWindow(appNameOrTitle) {
|
|
2146
|
-
const platform = process.platform;
|
|
2147
|
-
if (platform === 'win32') {
|
|
2148
|
-
const psScript = `
|
|
2149
|
-
$target = "${appNameOrTitle.replace(/"/g, '`"')}"
|
|
2150
|
-
$proc = Get-Process | Where-Object { ($_.MainWindowTitle -like "*$target*" -or $_.ProcessName -like "*$target*") -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
|
|
2151
|
-
if ($proc) {
|
|
2152
|
-
$sig = '[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);'
|
|
2153
|
-
$w = Add-Type -memberDefinition $sig -name 'Win32Fw' -namespace Win32Functions -passThru
|
|
2154
|
-
[void]$w::ShowWindow($proc.MainWindowHandle, 9)
|
|
2155
|
-
[void]$w::SetForegroundWindow($proc.MainWindowHandle)
|
|
2156
|
-
Write-Output "Focused '$($proc.MainWindowTitle)' ($($proc.ProcessName))"
|
|
2157
|
-
} else {
|
|
2158
|
-
$ws = New-Object -ComObject WScript.Shell
|
|
2159
|
-
$activated = $ws.AppActivate($target)
|
|
2160
|
-
if ($activated) { Write-Output "Activated '$target' via WScript.Shell" } else { Write-Output "Window '$target' not found" }
|
|
2161
|
-
}
|
|
2162
|
-
`;
|
|
2163
|
-
try {
|
|
2164
|
-
const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
|
|
2165
|
-
const { stdout } = await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
|
|
2166
|
-
return { success: true, output: stdout.trim() || `Focus request completed for "${appNameOrTitle}".` };
|
|
2167
|
-
}
|
|
2168
|
-
catch (err) {
|
|
2169
|
-
return { success: false, output: `Could not focus window: ${err?.message || String(err)}` };
|
|
2170
|
-
}
|
|
2171
|
-
}
|
|
2172
|
-
return { success: true, output: `Focus request sent for "${appNameOrTitle}".` };
|
|
2173
|
-
}
|
|
2174
|
-
/**
|
|
2175
|
-
* Scroll mouse wheel up or down by specified notch amount.
|
|
2176
|
-
*/
|
|
2177
|
-
export async function scrollWindow(direction = 'down', amount = 3) {
|
|
2178
|
-
const platform = process.platform;
|
|
2179
|
-
if (platform === 'win32') {
|
|
2180
|
-
const clicks = Math.max(1, Math.min(20, amount));
|
|
2181
|
-
const delta = direction === 'up' ? 120 * clicks : -120 * clicks;
|
|
2182
|
-
const psScript = `
|
|
2183
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
2184
|
-
$sig = '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int cButtons, int dwExtraInfo);'
|
|
2185
|
-
$m = Add-Type -memberDefinition $sig -name 'Win32Scroll' -namespace Win32Functions -passThru
|
|
2186
|
-
$m::mouse_event(0x0800, 0, 0, ${delta}, 0)
|
|
2187
|
-
`;
|
|
2188
|
-
try {
|
|
2189
|
-
const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
|
|
2190
|
-
await execAsync(`powershell -NoProfile -EncodedCommand ${encodedScript}`);
|
|
2191
|
-
return { success: true, output: `Scrolled mouse wheel ${direction} by ${clicks} notch(es).` };
|
|
2192
|
-
}
|
|
2193
|
-
catch (err) {
|
|
2194
|
-
return { success: false, output: `Failed to scroll: ${err?.message || String(err)}` };
|
|
2195
|
-
}
|
|
2196
|
-
}
|
|
2197
|
-
return { success: true, output: `Scrolled ${direction}` };
|
|
2198
|
-
}
|
|
2199
|
-
/**
|
|
2200
|
-
* Visually ground and click a target UI element by name or visual description.
|
|
2201
|
-
*/
|
|
2202
|
-
export async function findAndClickElement(elementQuery, appName, button = 'left') {
|
|
2203
|
-
let screenWidth = 1920;
|
|
2204
|
-
let screenHeight = 1080;
|
|
2205
|
-
let targetX = undefined;
|
|
2206
|
-
let targetY = undefined;
|
|
2207
|
-
// Use in-memory live screen frame first without writing screenshot files to disk
|
|
2208
|
-
const liveFrame = await getLiveScreenFrame();
|
|
2209
|
-
if (liveFrame) {
|
|
2210
|
-
screenWidth = liveFrame.width;
|
|
2211
|
-
screenHeight = liveFrame.height;
|
|
2212
|
-
try {
|
|
2213
|
-
const prompt = `Look at this screen and locate the UI element: "${elementQuery}".
|
|
2214
|
-
Return ONLY a valid JSON object with the center point of this element in 0-1000 normalized coordinates:
|
|
2215
|
-
{"point": [y, x]}
|
|
2216
|
-
Where y is between 0 and 1000 (top to bottom) and x is between 0 and 1000 (left to right).
|
|
2217
|
-
If not found, return {"point": null}. Output nothing else.`;
|
|
2218
|
-
const reply = await callVisionAiWithFrame(liveFrame, prompt, 80);
|
|
2219
|
-
if (reply) {
|
|
2220
|
-
const match = reply.match(/\{[\s\S]*"point"[\s\S]*\}/);
|
|
2221
|
-
if (match) {
|
|
2222
|
-
const parsed = JSON.parse(match[0]);
|
|
2223
|
-
if (Array.isArray(parsed.point) && parsed.point.length === 2) {
|
|
2224
|
-
const normY = Number(parsed.point[0]);
|
|
2225
|
-
const normX = Number(parsed.point[1]);
|
|
2226
|
-
if (!isNaN(normX) && !isNaN(normY)) {
|
|
2227
|
-
targetX = Math.round((normX / 1000) * screenWidth);
|
|
2228
|
-
targetY = Math.round((normY / 1000) * screenHeight);
|
|
2229
|
-
}
|
|
2230
|
-
}
|
|
2231
|
-
}
|
|
2232
|
-
}
|
|
2233
|
-
}
|
|
2234
|
-
catch { }
|
|
2235
|
-
}
|
|
2236
|
-
else {
|
|
2237
|
-
const capture = await captureScreen(appName);
|
|
2238
|
-
if (capture.success && capture.details?.outPath && fs.existsSync(capture.details.outPath)) {
|
|
2239
|
-
screenWidth = capture.details.width || 1920;
|
|
2240
|
-
screenHeight = capture.details.height || 1080;
|
|
2241
|
-
try {
|
|
2242
|
-
const prompt = `Look at this screenshot and locate the UI element: "${elementQuery}".
|
|
2243
|
-
Return ONLY a valid JSON object with the center point of this element in 0-1000 normalized coordinates:
|
|
2244
|
-
{"point": [y, x]}
|
|
2245
|
-
Where y is between 0 and 1000 (top to bottom) and x is between 0 and 1000 (left to right).
|
|
2246
|
-
If not found, return {"point": null}. Output nothing else.`;
|
|
2247
|
-
const reply = await callVisionAi(capture.details.outPath, prompt, 80);
|
|
2248
|
-
if (reply) {
|
|
2249
|
-
const match = reply.match(/\{[\s\S]*"point"[\s\S]*\}/);
|
|
2250
|
-
if (match) {
|
|
2251
|
-
const parsed = JSON.parse(match[0]);
|
|
2252
|
-
if (Array.isArray(parsed.point) && parsed.point.length === 2) {
|
|
2253
|
-
const normY = Number(parsed.point[0]);
|
|
2254
|
-
const normX = Number(parsed.point[1]);
|
|
2255
|
-
if (!isNaN(normX) && !isNaN(normY)) {
|
|
2256
|
-
targetX = Math.round((normX / 1000) * screenWidth);
|
|
2257
|
-
targetY = Math.round((normY / 1000) * screenHeight);
|
|
2258
|
-
}
|
|
2259
|
-
}
|
|
2260
|
-
}
|
|
2261
|
-
}
|
|
2262
|
-
}
|
|
2263
|
-
catch { }
|
|
2264
|
-
}
|
|
2265
|
-
}
|
|
2266
|
-
if ((targetX === undefined || targetY === undefined) && process.platform === 'win32') {
|
|
2267
|
-
const psScript = `
|
|
2268
|
-
Add-Type -AssemblyName UIAutomationClient
|
|
2269
|
-
Add-Type -AssemblyName UIAutomationTypes
|
|
2270
|
-
$target = "${elementQuery.replace(/"/g, '""')}"
|
|
2271
|
-
$root = [System.Windows.Automation.AutomationElement]::RootElement
|
|
2272
|
-
|
|
2273
|
-
$condName = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $target)
|
|
2274
|
-
$el = $root.FindFirst([System.Windows.Automation.TreeScope]::Descendants, $condName)
|
|
2275
|
-
|
|
2276
|
-
if (-not $el) {
|
|
2277
|
-
$all = $root.FindAll([System.Windows.Automation.TreeScope]::Descendants, [System.Windows.Automation.Condition]::TrueCondition)
|
|
2278
|
-
foreach ($item in $all) {
|
|
2279
|
-
if ($item.Current.Name -and $item.Current.Name.ToLower().Contains($target.ToLower())) {
|
|
2280
|
-
$el = $item
|
|
2281
|
-
break
|
|
2282
|
-
}
|
|
2283
|
-
}
|
|
2284
|
-
}
|
|
2285
|
-
|
|
2286
|
-
if ($el) {
|
|
2287
|
-
$rect = $el.Current.BoundingRectangle
|
|
2288
|
-
if ($rect -and $rect.Width -gt 0 -and $rect.Height -gt 0) {
|
|
2289
|
-
$cx = [int]($rect.X + ($rect.Width / 2))
|
|
2290
|
-
$cy = [int]($rect.Y + ($rect.Height / 2))
|
|
2291
|
-
@{ x = $cx; y = $cy; name = $el.Current.Name } | ConvertTo-Json
|
|
2292
|
-
}
|
|
2293
|
-
}
|
|
2294
|
-
`;
|
|
2295
|
-
try {
|
|
2296
|
-
const { stdout } = await execAsync(`powershell -NoProfile -Command "${psScript.replace(/\r?\n/g, ' ')}"`);
|
|
2297
|
-
if (stdout.trim()) {
|
|
2298
|
-
const uia = JSON.parse(stdout.trim());
|
|
2299
|
-
if (uia.x && uia.y) {
|
|
2300
|
-
targetX = uia.x;
|
|
2301
|
-
targetY = uia.y;
|
|
2302
|
-
}
|
|
2303
|
-
}
|
|
2304
|
-
}
|
|
2305
|
-
catch { }
|
|
2306
|
-
}
|
|
2307
|
-
if (targetX !== undefined && targetY !== undefined) {
|
|
2308
|
-
const clickRes = await moveAndClickMouse(targetX, targetY, button, appName);
|
|
2309
|
-
return {
|
|
2310
|
-
success: clickRes.success,
|
|
2311
|
-
output: `Visual Grounding Success: Located "${elementQuery}" at screen coordinates (${targetX}, ${targetY}). ${clickRes.output}`,
|
|
2312
|
-
coordinates: { x: targetX, y: targetY },
|
|
2313
|
-
};
|
|
2314
|
-
}
|
|
2315
|
-
return {
|
|
2316
|
-
success: false,
|
|
2317
|
-
output: `Could not visually or structurally locate UI element "${elementQuery}" on screen. Try calling analyze_screen to view current screen contents, or provide explicit (x, y) coordinates.`,
|
|
2318
|
-
};
|
|
2319
|
-
}
|
|
2320
|
-
//# sourceMappingURL=appControl.js.map
|