ft-scout 1.0.8 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.firebase/hosting.d2Vi.cache +9 -0
- package/.firebaserc +14 -0
- package/AUTONOMOUS_FEATURES.md +192 -0
- package/README.md +482 -32
- package/bin/ft.js +2 -0
- package/bin/src/commands/agent.d.ts.map +1 -0
- package/bin/src/commands/agent.js +1230 -0
- package/bin/src/commands/agent.js.map +1 -0
- package/bin/src/commands/audit.d.ts.map +1 -0
- package/bin/src/commands/audit.js +104 -0
- package/bin/src/commands/audit.js.map +1 -0
- package/bin/src/commands/authCommand.d.ts.map +1 -0
- package/bin/src/commands/authCommand.js +38 -0
- package/bin/src/commands/authCommand.js.map +1 -0
- package/bin/src/commands/benchmark.d.ts.map +1 -0
- package/bin/src/commands/benchmark.js +135 -0
- package/bin/src/commands/benchmark.js.map +1 -0
- package/bin/src/commands/brief.d.ts.map +1 -1
- package/bin/src/commands/brief.js +28 -19
- package/bin/src/commands/brief.js.map +1 -1
- package/bin/src/commands/configCommand.d.ts.map +1 -0
- package/bin/src/commands/configCommand.js +53 -0
- package/bin/src/commands/configCommand.js.map +1 -0
- package/bin/src/commands/dashboard.d.ts.map +1 -0
- package/bin/src/commands/dashboard.js +41 -0
- package/bin/src/commands/dashboard.js.map +1 -0
- package/bin/src/commands/history.d.ts.map +1 -0
- package/bin/src/commands/history.js +20 -0
- package/bin/src/commands/history.js.map +1 -0
- package/bin/src/commands/init.d.ts.map +1 -1
- package/bin/src/commands/init.js +411 -78
- package/bin/src/commands/init.js.map +1 -1
- package/bin/src/commands/login.d.ts.map +1 -0
- package/bin/src/commands/login.js +190 -0
- package/bin/src/commands/login.js.map +1 -0
- package/bin/src/commands/logout.d.ts.map +1 -0
- package/bin/src/commands/logout.js +15 -0
- package/bin/src/commands/logout.js.map +1 -0
- package/bin/src/commands/mcp.d.ts.map +1 -0
- package/bin/src/commands/mcp.js +393 -0
- package/bin/src/commands/mcp.js.map +1 -0
- package/bin/src/commands/owners.js +6 -6
- package/bin/src/commands/owners.js.map +1 -1
- package/bin/src/commands/plan.d.ts.map +1 -0
- package/bin/src/commands/plan.js +152 -0
- package/bin/src/commands/plan.js.map +1 -0
- package/bin/src/commands/quota.d.ts.map +1 -0
- package/bin/src/commands/quota.js +32 -0
- package/bin/src/commands/quota.js.map +1 -0
- package/bin/src/commands/recommend.d.ts.map +1 -0
- package/bin/src/commands/recommend.js +46 -0
- package/bin/src/commands/recommend.js.map +1 -0
- package/bin/src/commands/risky.d.ts.map +1 -1
- package/bin/src/commands/risky.js +235 -39
- package/bin/src/commands/risky.js.map +1 -1
- package/bin/src/commands/setup.d.ts.map +1 -1
- package/bin/src/commands/setup.js +411 -16
- package/bin/src/commands/setup.js.map +1 -1
- package/bin/src/commands/signup.d.ts.map +1 -0
- package/bin/src/commands/signup.js +118 -0
- package/bin/src/commands/signup.js.map +1 -0
- package/bin/src/commands/undo.d.ts.map +1 -0
- package/bin/src/commands/undo.js +59 -0
- package/bin/src/commands/undo.js.map +1 -0
- package/bin/src/commands/web.d.ts.map +1 -0
- package/bin/src/commands/web.js +90 -0
- package/bin/src/commands/web.js.map +1 -0
- package/bin/src/discovery/config.d.ts.map +1 -0
- package/bin/src/discovery/config.js +41 -0
- package/bin/src/discovery/config.js.map +1 -0
- package/bin/src/discovery/display.d.ts.map +1 -0
- package/bin/src/discovery/display.js +57 -0
- package/bin/src/discovery/display.js.map +1 -0
- package/bin/src/discovery/messages.d.ts.map +1 -0
- package/bin/src/discovery/messages.js +7 -0
- package/bin/src/discovery/messages.js.map +1 -0
- package/bin/src/discovery/signals.d.ts.map +1 -0
- package/bin/src/discovery/signals.js +33 -0
- package/bin/src/discovery/signals.js.map +1 -0
- package/bin/src/engine/actionVerifier.d.ts.map +1 -0
- package/bin/src/engine/actionVerifier.js +283 -0
- package/bin/src/engine/actionVerifier.js.map +1 -0
- package/bin/src/engine/agentEngine.d.ts.map +1 -0
- package/bin/src/engine/agentEngine.js +2188 -0
- package/bin/src/engine/agentEngine.js.map +1 -0
- package/bin/src/engine/agentState.d.ts.map +1 -0
- package/bin/src/engine/agentState.js +381 -0
- package/bin/src/engine/agentState.js.map +1 -0
- package/bin/src/engine/browser/browserController.d.ts.map +1 -0
- package/bin/src/engine/browser/browserController.js +737 -0
- package/bin/src/engine/browser/browserController.js.map +1 -0
- package/bin/src/engine/browser/index.d.ts.map +1 -0
- package/bin/src/engine/browser/index.js +2 -0
- package/bin/src/engine/browser/index.js.map +1 -0
- package/bin/src/engine/computer/computerController.d.ts.map +1 -0
- package/bin/src/engine/computer/computerController.js +773 -0
- package/bin/src/engine/computer/computerController.js.map +1 -0
- package/bin/src/engine/computer/computerDiagnostic.d.ts.map +1 -0
- package/bin/src/engine/computer/computerDiagnostic.js +71 -0
- package/bin/src/engine/computer/computerDiagnostic.js.map +1 -0
- package/bin/src/engine/computer/index.d.ts.map +1 -0
- package/bin/src/engine/computer/index.js +4 -0
- package/bin/src/engine/computer/index.js.map +1 -0
- package/bin/src/engine/computer/mockComputer.d.ts.map +1 -0
- package/bin/src/engine/computer/mockComputer.js +260 -0
- package/bin/src/engine/computer/mockComputer.js.map +1 -0
- package/bin/src/engine/computer/scoutDriver.cs +1161 -0
- package/bin/src/engine/computer/scout_driver.exe +0 -0
- package/bin/src/engine/git.d.ts.map +1 -1
- package/bin/src/engine/git.js +71 -0
- package/bin/src/engine/git.js.map +1 -1
- package/bin/src/engine/llm.d.ts.map +1 -1
- package/bin/src/engine/llm.js +1272 -23
- package/bin/src/engine/llm.js.map +1 -1
- package/bin/src/engine/planner.d.ts.map +1 -0
- package/bin/src/engine/planner.js +321 -0
- package/bin/src/engine/planner.js.map +1 -0
- package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -0
- package/bin/src/engine/runtime/agentRuntime.js +1258 -0
- package/bin/src/engine/runtime/agentRuntime.js.map +1 -0
- package/bin/src/engine/runtime/index.d.ts.map +1 -0
- package/bin/src/engine/runtime/index.js +2 -0
- package/bin/src/engine/runtime/index.js.map +1 -0
- package/bin/src/engine/safety.d.ts.map +1 -0
- package/bin/src/engine/safety.js +316 -0
- package/bin/src/engine/safety.js.map +1 -0
- package/bin/src/engine/scoutAudioRecorder.cs +215 -0
- package/bin/src/engine/scout_audio_recorder.exe +0 -0
- package/bin/src/engine/toolRegistry.d.ts.map +1 -0
- package/bin/src/engine/toolRegistry.js +521 -0
- package/bin/src/engine/toolRegistry.js.map +1 -0
- package/bin/src/engine/verifier.d.ts.map +1 -0
- package/bin/src/engine/verifier.js +414 -0
- package/bin/src/engine/verifier.js.map +1 -0
- package/bin/src/engine/voiceEngine.d.ts.map +1 -0
- package/bin/src/engine/voiceEngine.js +474 -0
- package/bin/src/engine/voiceEngine.js.map +1 -0
- package/bin/src/engine/web.d.ts.map +1 -0
- package/bin/src/engine/web.js +161 -0
- package/bin/src/engine/web.js.map +1 -0
- package/bin/src/engine/workspace/agentWorkspace.d.ts.map +1 -0
- package/bin/src/engine/workspace/agentWorkspace.js +505 -0
- package/bin/src/engine/workspace/agentWorkspace.js.map +1 -0
- package/bin/src/engine/workspace/index.d.ts.map +1 -0
- package/bin/src/engine/workspace/index.js +2 -0
- package/bin/src/engine/workspace/index.js.map +1 -0
- package/bin/src/index.js +365 -23
- package/bin/src/index.js.map +1 -1
- package/bin/src/mcp/client.d.ts.map +1 -0
- package/bin/src/mcp/client.js +127 -0
- package/bin/src/mcp/client.js.map +1 -0
- package/bin/src/mcp/discovery.d.ts.map +1 -0
- package/bin/src/mcp/discovery.js +91 -0
- package/bin/src/mcp/discovery.js.map +1 -0
- package/bin/src/mcp/health.d.ts.map +1 -0
- package/bin/src/mcp/health.js +44 -0
- package/bin/src/mcp/health.js.map +1 -0
- package/bin/src/mcp/index.d.ts.map +1 -0
- package/bin/src/mcp/index.js +15 -0
- package/bin/src/mcp/index.js.map +1 -0
- package/bin/src/mcp/manager.d.ts.map +1 -0
- package/bin/src/mcp/manager.js +648 -0
- package/bin/src/mcp/manager.js.map +1 -0
- package/bin/src/mcp/permissions.d.ts.map +1 -0
- package/bin/src/mcp/permissions.js +50 -0
- package/bin/src/mcp/permissions.js.map +1 -0
- package/bin/src/mcp/persistence.d.ts.map +1 -0
- package/bin/src/mcp/persistence.js +171 -0
- package/bin/src/mcp/persistence.js.map +1 -0
- package/bin/src/mcp/recovery.d.ts.map +1 -0
- package/bin/src/mcp/recovery.js +75 -0
- package/bin/src/mcp/recovery.js.map +1 -0
- package/bin/src/mcp/telemetry.d.ts.map +1 -0
- package/bin/src/mcp/telemetry.js +82 -0
- package/bin/src/mcp/telemetry.js.map +1 -0
- package/bin/src/mcp/toolRouter.d.ts.map +1 -0
- package/bin/src/mcp/toolRouter.js +127 -0
- package/bin/src/mcp/toolRouter.js.map +1 -0
- package/bin/src/mcp/transport.d.ts.map +1 -0
- package/bin/src/mcp/transport.js +328 -0
- package/bin/src/mcp/transport.js.map +1 -0
- package/bin/src/mcp/types.d.ts.map +1 -0
- package/bin/src/mcp/types.js +6 -0
- package/bin/src/mcp/types.js.map +1 -0
- package/bin/src/server/server.d.ts.map +1 -0
- package/bin/src/server/server.js +989 -0
- package/bin/src/server/server.js.map +1 -0
- package/bin/src/utils/admin.d.ts.map +1 -0
- package/bin/src/utils/admin.js +231 -0
- package/bin/src/utils/admin.js.map +1 -0
- package/bin/src/utils/auditLogger.d.ts.map +1 -0
- package/bin/src/utils/auditLogger.js +70 -0
- package/bin/src/utils/auditLogger.js.map +1 -0
- package/bin/src/utils/auth.d.ts.map +1 -0
- package/bin/src/utils/auth.js +679 -0
- package/bin/src/utils/auth.js.map +1 -0
- package/bin/src/utils/branding.d.ts.map +1 -0
- package/bin/src/utils/branding.js +14 -0
- package/bin/src/utils/branding.js.map +1 -0
- package/bin/src/utils/config.d.ts.map +1 -1
- package/bin/src/utils/config.js +124 -3
- package/bin/src/utils/config.js.map +1 -1
- package/bin/src/utils/email.d.ts.map +1 -0
- package/bin/src/utils/email.js +267 -0
- package/bin/src/utils/email.js.map +1 -0
- package/bin/src/utils/firebaseAuth.d.ts.map +1 -0
- package/bin/src/utils/firebaseAuth.js +174 -0
- package/bin/src/utils/firebaseAuth.js.map +1 -0
- package/bin/src/utils/holoseneAuth.d.ts.map +1 -0
- package/bin/src/utils/holoseneAuth.js +307 -0
- package/bin/src/utils/holoseneAuth.js.map +1 -0
- package/bin/src/utils/language.d.ts.map +1 -0
- package/bin/src/utils/language.js +181 -0
- package/bin/src/utils/language.js.map +1 -0
- package/bin/src/utils/markdown.d.ts.map +1 -0
- package/bin/src/utils/markdown.js +215 -0
- package/bin/src/utils/markdown.js.map +1 -0
- package/bin/src/utils/payment.d.ts.map +1 -0
- package/bin/src/utils/payment.js +290 -0
- package/bin/src/utils/payment.js.map +1 -0
- package/bin/src/utils/pricing.d.ts.map +1 -0
- package/bin/src/utils/pricing.js +165 -0
- package/bin/src/utils/pricing.js.map +1 -0
- package/bin/src/utils/razorpay.d.ts.map +1 -0
- package/bin/src/utils/razorpay.js +144 -0
- package/bin/src/utils/razorpay.js.map +1 -0
- package/bin/src/utils/session.d.ts.map +1 -0
- package/bin/src/utils/session.js +82 -0
- package/bin/src/utils/session.js.map +1 -0
- package/bin/src/utils/updateCheck.d.ts.map +1 -0
- package/bin/src/utils/updateCheck.js +108 -0
- package/bin/src/utils/updateCheck.js.map +1 -0
- package/downloads/invoice-2026-03.pdf +1 -0
- package/firebase.json +74 -0
- package/package.json +49 -46
- package/web/ScoutFavicon.png +0 -0
- package/web/app.js +3944 -0
- package/web/styles.css +3081 -0
- package/.ft/context.json +0 -18
- package/email-template.html +0 -223
- package/email-template.txt +0 -45
- package/scripts/send-brevo-campaign.cjs +0 -167
- package/scripts/send-email.js +0 -77
|
@@ -0,0 +1,773 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { exec, execFile, spawn } from 'child_process';
|
|
4
|
+
import { promisify } from 'util';
|
|
5
|
+
import { fileURLToPath } from 'url';
|
|
6
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
7
|
+
const __dirname = path.dirname(__filename);
|
|
8
|
+
const execAsync = promisify(exec);
|
|
9
|
+
const execFileAsync = promisify(execFile);
|
|
10
|
+
export class SystemComputerController {
|
|
11
|
+
workspaceDir;
|
|
12
|
+
screenshotsDir;
|
|
13
|
+
expectedApplication = null;
|
|
14
|
+
activeWindow = null;
|
|
15
|
+
driverExePath = null;
|
|
16
|
+
driverInitAttempted = false;
|
|
17
|
+
constructor(workspaceDir = process.cwd()) {
|
|
18
|
+
this.workspaceDir = workspaceDir;
|
|
19
|
+
this.screenshotsDir = path.join(this.workspaceDir, '.ft', 'screenshots');
|
|
20
|
+
try {
|
|
21
|
+
if (!fs.existsSync(this.screenshotsDir)) {
|
|
22
|
+
fs.mkdirSync(this.screenshotsDir, { recursive: true });
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
catch { }
|
|
26
|
+
}
|
|
27
|
+
setExpectedApplication(app) {
|
|
28
|
+
this.expectedApplication = app;
|
|
29
|
+
}
|
|
30
|
+
getExpectedApplication() {
|
|
31
|
+
return this.expectedApplication;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Resolves or builds the native scout_driver.exe for Windows.
|
|
35
|
+
*/
|
|
36
|
+
async ensureDriver() {
|
|
37
|
+
if (process.platform !== 'win32') {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
const csSourceCandidates = [
|
|
41
|
+
path.join(__dirname, 'scoutDriver.cs'),
|
|
42
|
+
path.join(this.workspaceDir, 'bin', 'src', 'engine', 'computer', 'scoutDriver.cs'),
|
|
43
|
+
];
|
|
44
|
+
const csSource = csSourceCandidates.find((p) => fs.existsSync(p));
|
|
45
|
+
const cscExe = 'C:\\Windows\\Microsoft.NET\\Framework64\\v4.0.30319\\csc.exe';
|
|
46
|
+
if (this.driverExePath && fs.existsSync(this.driverExePath)) {
|
|
47
|
+
// If C# source exists and is newer than the cached binary, recompile to pick up changes
|
|
48
|
+
if (csSource && fs.existsSync(cscExe)) {
|
|
49
|
+
try {
|
|
50
|
+
const srcMtime = fs.statSync(csSource).mtimeMs;
|
|
51
|
+
const exeMtime = fs.statSync(this.driverExePath).mtimeMs;
|
|
52
|
+
if (srcMtime > exeMtime) {
|
|
53
|
+
await execAsync(`"${cscExe}" /nologo /optimize /t:exe /out:"${this.driverExePath}" /r:System.Windows.Forms.dll,System.Drawing.dll "${csSource}"`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
catch { }
|
|
57
|
+
}
|
|
58
|
+
return this.driverExePath;
|
|
59
|
+
}
|
|
60
|
+
// Candidate locations
|
|
61
|
+
const candidates = [
|
|
62
|
+
path.join(__dirname, 'scout_driver.exe'),
|
|
63
|
+
path.join(this.workspaceDir, 'bin', 'src', 'engine', 'computer', 'scout_driver.exe'),
|
|
64
|
+
path.join(this.workspaceDir, '.ft', 'bin', 'scout_driver.exe'),
|
|
65
|
+
];
|
|
66
|
+
for (const c of candidates) {
|
|
67
|
+
if (fs.existsSync(c)) {
|
|
68
|
+
// Check if source is newer than candidate exe
|
|
69
|
+
if (csSource && fs.existsSync(cscExe)) {
|
|
70
|
+
try {
|
|
71
|
+
const srcMtime = fs.statSync(csSource).mtimeMs;
|
|
72
|
+
const exeMtime = fs.statSync(c).mtimeMs;
|
|
73
|
+
if (srcMtime > exeMtime) {
|
|
74
|
+
await execAsync(`"${cscExe}" /nologo /optimize /t:exe /out:"${c}" /r:System.Windows.Forms.dll,System.Drawing.dll "${csSource}"`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
catch { }
|
|
78
|
+
}
|
|
79
|
+
this.driverExePath = c;
|
|
80
|
+
return c;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
if (this.driverInitAttempted) {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
this.driverInitAttempted = true;
|
|
87
|
+
// Try compiling using csc.exe if no candidate binary found
|
|
88
|
+
if (csSource && fs.existsSync(cscExe)) {
|
|
89
|
+
const outDir = path.join(this.workspaceDir, '.ft', 'bin');
|
|
90
|
+
try {
|
|
91
|
+
if (!fs.existsSync(outDir))
|
|
92
|
+
fs.mkdirSync(outDir, { recursive: true });
|
|
93
|
+
const targetExe = path.join(outDir, 'scout_driver.exe');
|
|
94
|
+
await execAsync(`"${cscExe}" /nologo /optimize /t:exe /out:"${targetExe}" /r:System.Windows.Forms.dll,System.Drawing.dll "${csSource}"`);
|
|
95
|
+
if (fs.existsSync(targetExe)) {
|
|
96
|
+
this.driverExePath = targetExe;
|
|
97
|
+
return targetExe;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
catch { }
|
|
101
|
+
}
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Runs an action using native driver or fallback.
|
|
106
|
+
*/
|
|
107
|
+
async runDriverAction(action, args = []) {
|
|
108
|
+
const driver = await this.ensureDriver();
|
|
109
|
+
const strArgs = args.map((a) => String(a));
|
|
110
|
+
if (driver) {
|
|
111
|
+
try {
|
|
112
|
+
const { stdout } = await execFileAsync(driver, [action, ...strArgs], {
|
|
113
|
+
windowsHide: true,
|
|
114
|
+
timeout: 10000,
|
|
115
|
+
});
|
|
116
|
+
const trimmed = (stdout || '').trim();
|
|
117
|
+
if (trimmed.startsWith('{') && trimmed.endsWith('}')) {
|
|
118
|
+
try {
|
|
119
|
+
return JSON.parse(trimmed);
|
|
120
|
+
}
|
|
121
|
+
catch { }
|
|
122
|
+
}
|
|
123
|
+
return { success: true, output: trimmed };
|
|
124
|
+
}
|
|
125
|
+
catch (err) {
|
|
126
|
+
return {
|
|
127
|
+
success: false,
|
|
128
|
+
output: `Driver execution error: ${err?.message || String(err)}`,
|
|
129
|
+
error: err?.message || String(err),
|
|
130
|
+
recoverable: true,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
// PowerShell fallback with input desktop attachment
|
|
135
|
+
return await this.runPowerShellFallback(action, strArgs);
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Reliable PowerShell fallback with thread input desktop attachment
|
|
139
|
+
*/
|
|
140
|
+
async runPowerShellFallback(action, args) {
|
|
141
|
+
const script = `
|
|
142
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
143
|
+
Add-Type -AssemblyName System.Drawing
|
|
144
|
+
|
|
145
|
+
$src = @"
|
|
146
|
+
using System;
|
|
147
|
+
using System.Drawing;
|
|
148
|
+
using System.Drawing.Imaging;
|
|
149
|
+
using System.Threading;
|
|
150
|
+
using System.Runtime.InteropServices;
|
|
151
|
+
public class WinDriverFallback {
|
|
152
|
+
[DllImport("user32.dll", SetLastError = true)]
|
|
153
|
+
public static extern IntPtr OpenWindowStation(string lpszWinSta, bool fInherit, uint dwDesiredAccess);
|
|
154
|
+
[DllImport("user32.dll", SetLastError = true)]
|
|
155
|
+
public static extern bool SetProcessWindowStation(IntPtr hWinSta);
|
|
156
|
+
[DllImport("user32.dll", SetLastError = true)]
|
|
157
|
+
public static extern IntPtr OpenInputDesktop(uint dwFlags, bool fInherit, uint dwDesiredAccess);
|
|
158
|
+
[DllImport("user32.dll", SetLastError = true)]
|
|
159
|
+
public static extern IntPtr OpenDesktop(string lpszDesktop, uint dwFlags, bool fInherit, uint dwDesiredAccess);
|
|
160
|
+
[DllImport("user32.dll", SetLastError = true)]
|
|
161
|
+
public static extern bool SetThreadDesktop(IntPtr hDesktop);
|
|
162
|
+
[DllImport("user32.dll", SetLastError = true)]
|
|
163
|
+
public static extern bool GetCursorPos(out POINT lpPoint);
|
|
164
|
+
[DllImport("user32.dll", SetLastError = true)]
|
|
165
|
+
public static extern bool SetCursorPos(int X, int Y);
|
|
166
|
+
[DllImport("user32.dll", SetLastError = true)]
|
|
167
|
+
public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
|
|
168
|
+
[DllImport("user32.dll")]
|
|
169
|
+
public static extern int GetSystemMetrics(int nIndex);
|
|
170
|
+
[DllImport("user32.dll")]
|
|
171
|
+
public static extern bool SetProcessDPIAware();
|
|
172
|
+
|
|
173
|
+
[StructLayout(LayoutKind.Sequential)]
|
|
174
|
+
public struct POINT { public int X; public int Y; }
|
|
175
|
+
|
|
176
|
+
public static void Attach() {
|
|
177
|
+
try {
|
|
178
|
+
SetProcessDPIAware();
|
|
179
|
+
IntPtr hw = OpenWindowStation("WinSta0", false, 0x10000000);
|
|
180
|
+
if (hw != IntPtr.Zero) SetProcessWindowStation(hw);
|
|
181
|
+
IntPtr hd = OpenInputDesktop(0, false, 0x10000000);
|
|
182
|
+
if (hd == IntPtr.Zero) hd = OpenDesktop("Default", 0, false, 0x10000000);
|
|
183
|
+
if (hd != IntPtr.Zero) SetThreadDesktop(hd);
|
|
184
|
+
} catch {}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
"@
|
|
188
|
+
Add-Type -TypeDefinition $src -ReferencedAssemblies System.Drawing
|
|
189
|
+
|
|
190
|
+
[WinDriverFallback]::Attach()
|
|
191
|
+
$act = "${action.replace(/"/g, '`"')}".ToLower()
|
|
192
|
+
|
|
193
|
+
if ($act -eq "move") {
|
|
194
|
+
$x = [int]${Number(args[0]) || 0}
|
|
195
|
+
$y = [int]${Number(args[1]) || 0}
|
|
196
|
+
[WinDriverFallback]::SetCursorPos($x, $y)
|
|
197
|
+
Write-Output "{\`"success\`":true,\`"action\`":\`"move\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
|
|
198
|
+
} elseif ($act -eq "click" -or $act -eq "left_click") {
|
|
199
|
+
$x = [int]${Number(args[0]) || 0}
|
|
200
|
+
$y = [int]${Number(args[1]) || 0}
|
|
201
|
+
[WinDriverFallback]::SetCursorPos($x, $y)
|
|
202
|
+
Start-Sleep -Milliseconds 30
|
|
203
|
+
[WinDriverFallback]::mouse_event(0x0002, 0, 0, 0, 0)
|
|
204
|
+
Start-Sleep -Milliseconds 40
|
|
205
|
+
[WinDriverFallback]::mouse_event(0x0004, 0, 0, 0, 0)
|
|
206
|
+
Write-Output "{\`"success\`":true,\`"action\`":\`"click\`",\`"button\`":\`"left\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
|
|
207
|
+
} elseif ($act -eq "right_click") {
|
|
208
|
+
$x = [int]${Number(args[0]) || 0}
|
|
209
|
+
$y = [int]${Number(args[1]) || 0}
|
|
210
|
+
[WinDriverFallback]::SetCursorPos($x, $y)
|
|
211
|
+
Start-Sleep -Milliseconds 30
|
|
212
|
+
[WinDriverFallback]::mouse_event(0x0008, 0, 0, 0, 0)
|
|
213
|
+
Start-Sleep -Milliseconds 40
|
|
214
|
+
[WinDriverFallback]::mouse_event(0x0010, 0, 0, 0, 0)
|
|
215
|
+
Write-Output "{\`"success\`":true,\`"action\`":\`"right_click\`",\`"button\`":\`"right\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
|
|
216
|
+
} elseif ($act -eq "double_click") {
|
|
217
|
+
$x = [int]${Number(args[0]) || 0}
|
|
218
|
+
$y = [int]${Number(args[1]) || 0}
|
|
219
|
+
[WinDriverFallback]::SetCursorPos($x, $y)
|
|
220
|
+
Start-Sleep -Milliseconds 30
|
|
221
|
+
[WinDriverFallback]::mouse_event(0x0002, 0, 0, 0, 0)
|
|
222
|
+
Start-Sleep -Milliseconds 40
|
|
223
|
+
[WinDriverFallback]::mouse_event(0x0004, 0, 0, 0, 0)
|
|
224
|
+
Start-Sleep -Milliseconds 80
|
|
225
|
+
[WinDriverFallback]::mouse_event(0x0002, 0, 0, 0, 0)
|
|
226
|
+
Start-Sleep -Milliseconds 40
|
|
227
|
+
[WinDriverFallback]::mouse_event(0x0004, 0, 0, 0, 0)
|
|
228
|
+
Write-Output "{\`"success\`":true,\`"action\`":\`"double_click\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
|
|
229
|
+
} elseif ($act -eq "screenshot") {
|
|
230
|
+
$w = [WinDriverFallback]::GetSystemMetrics(0)
|
|
231
|
+
$h = [WinDriverFallback]::GetSystemMetrics(1)
|
|
232
|
+
if ($w -le 0) { $w = 1920 }
|
|
233
|
+
if ($h -le 0) { $h = 1080 }
|
|
234
|
+
$p = "${(args[0] || '').replace(/\\/g, '\\\\').replace(/"/g, '`"')}"
|
|
235
|
+
$bmp = New-Object System.Drawing.Bitmap $w, $h
|
|
236
|
+
$g = [System.Drawing.Graphics]::FromImage($bmp)
|
|
237
|
+
$g.CopyFromScreen(0, 0, 0, 0, (New-Object System.Drawing.Size($w, $h)), [System.Drawing.CopyPixelOperation]::SourceCopy)
|
|
238
|
+
$bmp.Save($p, [System.Drawing.Imaging.ImageFormat]::Png)
|
|
239
|
+
$g.Dispose()
|
|
240
|
+
$bmp.Dispose()
|
|
241
|
+
Write-Output "{\`"success\`":true,\`"action\`":\`"screenshot\`",\`"width\`":$w,\`"height\`":$h,\`"path\`":\`"$p\`"}"
|
|
242
|
+
} else {
|
|
243
|
+
Write-Output "{\`"success\`":true,\`"action\`":\`"$act\`"}"
|
|
244
|
+
}
|
|
245
|
+
`;
|
|
246
|
+
try {
|
|
247
|
+
const encoded = Buffer.from(script, 'utf16le').toString('base64');
|
|
248
|
+
const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand ${encoded}`);
|
|
249
|
+
const trimmed = (stdout || '').trim();
|
|
250
|
+
if (trimmed.startsWith('{') && trimmed.endsWith('}')) {
|
|
251
|
+
return JSON.parse(trimmed);
|
|
252
|
+
}
|
|
253
|
+
return { success: true, output: trimmed };
|
|
254
|
+
}
|
|
255
|
+
catch (err) {
|
|
256
|
+
return { success: false, output: `PowerShell fallback failed: ${err?.message || String(err)}`, error: err?.message, recoverable: true };
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
async screenshot() {
|
|
260
|
+
const timestamp = new Date().toISOString();
|
|
261
|
+
const filename = `screen_${Date.now()}.png`;
|
|
262
|
+
const filePath = path.join(this.screenshotsDir, filename);
|
|
263
|
+
if (process.platform === 'win32') {
|
|
264
|
+
const res = await this.runDriverAction('screenshot', [filePath]);
|
|
265
|
+
if (res && res.success) {
|
|
266
|
+
const activeWin = res.activeWindow || (await this.inspectUI()).activeWindow;
|
|
267
|
+
this.activeWindow = activeWin;
|
|
268
|
+
return {
|
|
269
|
+
path: filePath,
|
|
270
|
+
width: Number(res.width) || 1920,
|
|
271
|
+
height: Number(res.height) || 1080,
|
|
272
|
+
activeWindow: activeWin,
|
|
273
|
+
timestamp,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
else if (process.platform === 'darwin') {
|
|
278
|
+
try {
|
|
279
|
+
await execAsync(`screencapture -x "${filePath}"`);
|
|
280
|
+
return { path: filePath, width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
|
|
281
|
+
}
|
|
282
|
+
catch {
|
|
283
|
+
return { width: 1920, height: 1080, timestamp };
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
try {
|
|
288
|
+
await execAsync(`scrot "${filePath}"`);
|
|
289
|
+
return { path: filePath, width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
|
|
290
|
+
}
|
|
291
|
+
catch {
|
|
292
|
+
return { width: 1920, height: 1080, timestamp };
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
return { width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
|
|
296
|
+
}
|
|
297
|
+
async click(x, y, options) {
|
|
298
|
+
const btn = options?.button || 'left';
|
|
299
|
+
const count = options?.clicks || 1;
|
|
300
|
+
if (process.platform === 'win32') {
|
|
301
|
+
const act = count === 2 ? 'double_click' : btn === 'right' ? 'right_click' : 'click';
|
|
302
|
+
const res = await this.runDriverAction(act, [x, y, btn]);
|
|
303
|
+
const success = Boolean(res?.success);
|
|
304
|
+
const coords = res?.coordinates || { x, y };
|
|
305
|
+
const activeWin = res?.activeWindow || this.activeWindow || undefined;
|
|
306
|
+
return {
|
|
307
|
+
success,
|
|
308
|
+
output: success
|
|
309
|
+
? `Clicked ${btn} button at (${coords.x}, ${coords.y})`
|
|
310
|
+
: `Click failed at (${x}, ${y}): ${res?.error || 'Unknown error'}`,
|
|
311
|
+
action: 'click',
|
|
312
|
+
coordinates: coords,
|
|
313
|
+
button: btn,
|
|
314
|
+
error: res?.error,
|
|
315
|
+
recoverable: res?.recoverable ?? true,
|
|
316
|
+
activeWindow: activeWin,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
return {
|
|
320
|
+
success: true,
|
|
321
|
+
output: `Simulated click ${btn} at (${x}, ${y})`,
|
|
322
|
+
action: 'click',
|
|
323
|
+
coordinates: { x, y },
|
|
324
|
+
button: btn,
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
async leftClick(x, y) {
|
|
328
|
+
return this.click(x, y, { button: 'left', clicks: 1 });
|
|
329
|
+
}
|
|
330
|
+
async rightClick(x, y) {
|
|
331
|
+
return this.click(x, y, { button: 'right', clicks: 1 });
|
|
332
|
+
}
|
|
333
|
+
async doubleClick(x, y) {
|
|
334
|
+
if (process.platform === 'win32') {
|
|
335
|
+
const res = await this.runDriverAction('double_click', [x, y]);
|
|
336
|
+
const success = Boolean(res?.success);
|
|
337
|
+
const coords = res?.coordinates || { x, y };
|
|
338
|
+
return {
|
|
339
|
+
success,
|
|
340
|
+
output: success
|
|
341
|
+
? `Double-clicked at (${coords.x}, ${coords.y})`
|
|
342
|
+
: `Double-click failed at (${x}, ${y}): ${res?.error || 'Unknown error'}`,
|
|
343
|
+
action: 'double_click',
|
|
344
|
+
coordinates: coords,
|
|
345
|
+
button: 'left',
|
|
346
|
+
error: res?.error,
|
|
347
|
+
recoverable: res?.recoverable ?? true,
|
|
348
|
+
activeWindow: res?.activeWindow || this.activeWindow || undefined,
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
return {
|
|
352
|
+
success: true,
|
|
353
|
+
output: `Simulated double-click at (${x}, ${y})`,
|
|
354
|
+
action: 'double_click',
|
|
355
|
+
coordinates: { x, y },
|
|
356
|
+
button: 'left',
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
async mouseDown(x, y, button) {
|
|
360
|
+
const btn = button || 'left';
|
|
361
|
+
if (process.platform === 'win32') {
|
|
362
|
+
const res = await this.runDriverAction('mouse_down', [x ?? -1, y ?? -1, btn]);
|
|
363
|
+
const coords = res?.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined);
|
|
364
|
+
return {
|
|
365
|
+
success: Boolean(res?.success),
|
|
366
|
+
output: `Mouse down (${btn}) at (${coords?.x ?? 'current'}, ${coords?.y ?? 'current'})`,
|
|
367
|
+
action: 'mouse_down',
|
|
368
|
+
button: btn,
|
|
369
|
+
coordinates: coords,
|
|
370
|
+
error: res?.error,
|
|
371
|
+
recoverable: true,
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
return {
|
|
375
|
+
success: true,
|
|
376
|
+
output: `Simulated mouse down (${btn})`,
|
|
377
|
+
action: 'mouse_down',
|
|
378
|
+
button: btn,
|
|
379
|
+
coordinates: x !== undefined && y !== undefined ? { x, y } : undefined,
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
async mouseUp(x, y, button) {
|
|
383
|
+
const btn = button || 'left';
|
|
384
|
+
if (process.platform === 'win32') {
|
|
385
|
+
const res = await this.runDriverAction('mouse_up', [x ?? -1, y ?? -1, btn]);
|
|
386
|
+
const coords = res?.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined);
|
|
387
|
+
return {
|
|
388
|
+
success: Boolean(res?.success),
|
|
389
|
+
output: `Mouse up (${btn}) at (${coords?.x ?? 'current'}, ${coords?.y ?? 'current'})`,
|
|
390
|
+
action: 'mouse_up',
|
|
391
|
+
button: btn,
|
|
392
|
+
coordinates: coords,
|
|
393
|
+
error: res?.error,
|
|
394
|
+
recoverable: true,
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
return {
|
|
398
|
+
success: true,
|
|
399
|
+
output: `Simulated mouse up (${btn})`,
|
|
400
|
+
action: 'mouse_up',
|
|
401
|
+
button: btn,
|
|
402
|
+
coordinates: x !== undefined && y !== undefined ? { x, y } : undefined,
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
async mouseMove(x, y) {
|
|
406
|
+
return this.move(x, y);
|
|
407
|
+
}
|
|
408
|
+
async move(x, y) {
|
|
409
|
+
if (process.platform === 'win32') {
|
|
410
|
+
const res = await this.runDriverAction('move', [x, y]);
|
|
411
|
+
const success = Boolean(res?.success);
|
|
412
|
+
const coords = res?.coordinates || { x, y };
|
|
413
|
+
return {
|
|
414
|
+
success,
|
|
415
|
+
output: success
|
|
416
|
+
? `Moved mouse cursor to (${coords.x}, ${coords.y})`
|
|
417
|
+
: `MouseMove failed to (${x}, ${y}): ${res?.error || 'Unknown error'}`,
|
|
418
|
+
action: 'move',
|
|
419
|
+
coordinates: coords,
|
|
420
|
+
error: res?.error,
|
|
421
|
+
recoverable: res?.recoverable ?? true,
|
|
422
|
+
activeWindow: res?.activeWindow || this.activeWindow || undefined,
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
return {
|
|
426
|
+
success: true,
|
|
427
|
+
output: `Simulated mouse move to (${x}, ${y})`,
|
|
428
|
+
action: 'move',
|
|
429
|
+
coordinates: { x, y },
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
async drag(startX, startY, endX, endY) {
|
|
433
|
+
if (process.platform === 'win32') {
|
|
434
|
+
const res = await this.runDriverAction('drag', [startX, startY, endX, endY]);
|
|
435
|
+
const success = Boolean(res?.success);
|
|
436
|
+
return {
|
|
437
|
+
success,
|
|
438
|
+
output: success
|
|
439
|
+
? `Dragged from (${startX}, ${startY}) to (${endX}, ${endY})`
|
|
440
|
+
: `Drag failed: ${res?.error || 'Unknown error'}`,
|
|
441
|
+
action: 'drag',
|
|
442
|
+
from: { x: startX, y: startY },
|
|
443
|
+
to: { x: endX, y: endY },
|
|
444
|
+
coordinates: res?.coordinates || { x: endX, y: endY },
|
|
445
|
+
error: res?.error,
|
|
446
|
+
recoverable: res?.recoverable ?? true,
|
|
447
|
+
};
|
|
448
|
+
}
|
|
449
|
+
return {
|
|
450
|
+
success: true,
|
|
451
|
+
output: `Simulated drag from (${startX}, ${startY}) to (${endX}, ${endY})`,
|
|
452
|
+
action: 'drag',
|
|
453
|
+
from: { x: startX, y: startY },
|
|
454
|
+
to: { x: endX, y: endY },
|
|
455
|
+
coordinates: { x: endX, y: endY },
|
|
456
|
+
};
|
|
457
|
+
}
|
|
458
|
+
async scroll(deltaX, deltaY) {
|
|
459
|
+
if (process.platform === 'win32') {
|
|
460
|
+
const res = await this.runDriverAction('scroll', [deltaX, deltaY]);
|
|
461
|
+
return {
|
|
462
|
+
success: Boolean(res?.success),
|
|
463
|
+
output: `Scrolled deltaY=${deltaY}, deltaX=${deltaX}`,
|
|
464
|
+
action: 'scroll',
|
|
465
|
+
details: { deltaX, deltaY },
|
|
466
|
+
};
|
|
467
|
+
}
|
|
468
|
+
return {
|
|
469
|
+
success: true,
|
|
470
|
+
output: `Simulated scroll deltaY=${deltaY}, deltaX=${deltaX}`,
|
|
471
|
+
action: 'scroll',
|
|
472
|
+
details: { deltaX, deltaY },
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
async getMousePosition() {
|
|
476
|
+
if (process.platform === 'win32') {
|
|
477
|
+
const res = await this.runDriverAction('position', []);
|
|
478
|
+
if (res && res.coordinates) {
|
|
479
|
+
return res.coordinates;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
return { x: 0, y: 0 };
|
|
483
|
+
}
|
|
484
|
+
async runDiagnostic() {
|
|
485
|
+
if (process.platform === 'win32') {
|
|
486
|
+
const res = await this.runDriverAction('diagnostic', []);
|
|
487
|
+
return {
|
|
488
|
+
success: Boolean(res?.success),
|
|
489
|
+
output: `Diagnostic: Screen ${res?.width}x${res?.height}, Mouse (${res?.mouse?.x}, ${res?.mouse?.y}), Active: "${res?.activeWindow}", Windows: ${res?.windowCount}`,
|
|
490
|
+
action: 'diagnostic',
|
|
491
|
+
coordinates: res?.mouse,
|
|
492
|
+
activeWindow: res?.activeWindow,
|
|
493
|
+
details: res,
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
return {
|
|
497
|
+
success: true,
|
|
498
|
+
output: `Diagnostic on ${process.platform}: Screen 1920x1080, Mouse (0, 0), Active: "Desktop"`,
|
|
499
|
+
action: 'diagnostic',
|
|
500
|
+
coordinates: { x: 0, y: 0 },
|
|
501
|
+
activeWindow: 'Desktop',
|
|
502
|
+
};
|
|
503
|
+
}
|
|
504
|
+
async type(text) {
|
|
505
|
+
if (!text)
|
|
506
|
+
return { success: true, output: 'Typed empty string', action: 'type' };
|
|
507
|
+
if (process.platform === 'win32') {
|
|
508
|
+
const targetApp = this.expectedApplication || '';
|
|
509
|
+
if (targetApp) {
|
|
510
|
+
await this.switchWindow(targetApp);
|
|
511
|
+
}
|
|
512
|
+
const res = await this.runDriverAction('type', [text]);
|
|
513
|
+
const success = Boolean(res?.success);
|
|
514
|
+
const activeWin = res?.activeWindow || targetApp || 'foreground';
|
|
515
|
+
this.activeWindow = activeWin;
|
|
516
|
+
return {
|
|
517
|
+
success,
|
|
518
|
+
output: success
|
|
519
|
+
? `Typed text (${text.length} chars) into "${activeWin}"`
|
|
520
|
+
: `Type failed: ${res?.error || 'Unknown error'}`,
|
|
521
|
+
action: 'type',
|
|
522
|
+
error: res?.error,
|
|
523
|
+
recoverable: true,
|
|
524
|
+
activeWindow: activeWin,
|
|
525
|
+
expectedWindow: this.expectedApplication || undefined,
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
return {
|
|
529
|
+
success: true,
|
|
530
|
+
output: `Simulated type (${text.length} chars)`,
|
|
531
|
+
action: 'type',
|
|
532
|
+
activeWindow: this.expectedApplication || 'Simulated Window',
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
async keyPress(key) {
|
|
536
|
+
if (process.platform === 'win32') {
|
|
537
|
+
const targetApp = this.expectedApplication || '';
|
|
538
|
+
if (targetApp) {
|
|
539
|
+
await this.switchWindow(targetApp);
|
|
540
|
+
}
|
|
541
|
+
const res = await this.runDriverAction('key', [key]);
|
|
542
|
+
const success = Boolean(res?.success);
|
|
543
|
+
return {
|
|
544
|
+
success,
|
|
545
|
+
output: success ? `Pressed key: ${key}` : `KeyPress failed: ${res?.error || 'Unknown error'}`,
|
|
546
|
+
action: 'key_press',
|
|
547
|
+
error: res?.error,
|
|
548
|
+
recoverable: true,
|
|
549
|
+
activeWindow: res?.activeWindow || targetApp || undefined,
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
return { success: true, output: `Simulated key press: ${key}`, action: 'key_press' };
|
|
553
|
+
}
|
|
554
|
+
async hotkey(keys) {
|
|
555
|
+
if (!keys || keys.length === 0) {
|
|
556
|
+
return { success: false, output: 'No keys provided for hotkey', action: 'hotkey', error: 'No keys provided' };
|
|
557
|
+
}
|
|
558
|
+
if (process.platform === 'win32') {
|
|
559
|
+
const targetApp = this.expectedApplication || '';
|
|
560
|
+
if (targetApp) {
|
|
561
|
+
await this.switchWindow(targetApp);
|
|
562
|
+
}
|
|
563
|
+
const res = await this.runDriverAction('hotkey', [keys.join('+')]);
|
|
564
|
+
const success = Boolean(res?.success);
|
|
565
|
+
const isClosingHotkey = keys.some((k) => {
|
|
566
|
+
const lk = String(k).toLowerCase().trim();
|
|
567
|
+
return lk === 'f4' || lk === 'w';
|
|
568
|
+
});
|
|
569
|
+
if (isClosingHotkey && success && this.expectedApplication) {
|
|
570
|
+
this.expectedApplication = null;
|
|
571
|
+
}
|
|
572
|
+
return {
|
|
573
|
+
success,
|
|
574
|
+
output: success ? `Executed hotkey: ${keys.join('+')}` : `Hotkey failed: ${res?.error || 'Unknown error'}`,
|
|
575
|
+
action: 'hotkey',
|
|
576
|
+
error: res?.error,
|
|
577
|
+
recoverable: true,
|
|
578
|
+
activeWindow: res?.activeWindow || targetApp || undefined,
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
return { success: true, output: `Simulated hotkey: ${keys.join('+')}`, action: 'hotkey' };
|
|
582
|
+
}
|
|
583
|
+
async wait(ms) {
|
|
584
|
+
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
|
|
585
|
+
}
|
|
586
|
+
async openApplication(appName, args = []) {
|
|
587
|
+
const appLower = appName.toLowerCase().trim();
|
|
588
|
+
const argsStr = args.join(' ');
|
|
589
|
+
try {
|
|
590
|
+
if (process.platform === 'win32') {
|
|
591
|
+
const driverRes = await this.runDriverAction('launch', [appName]);
|
|
592
|
+
if (!driverRes || !driverRes.success) {
|
|
593
|
+
if (appLower === 'code' || appLower === 'vscode') {
|
|
594
|
+
await execAsync(`start code ${argsStr}`);
|
|
595
|
+
}
|
|
596
|
+
else if (appLower === 'explorer' || appLower === 'file explorer') {
|
|
597
|
+
await execAsync(`start explorer ${argsStr || '.'}`);
|
|
598
|
+
}
|
|
599
|
+
else if (appLower === 'notepad') {
|
|
600
|
+
await execAsync(`start notepad ${argsStr}`);
|
|
601
|
+
}
|
|
602
|
+
else if (appLower === 'terminal' || appLower === 'cmd' || appLower === 'powershell') {
|
|
603
|
+
await execAsync(`start powershell ${argsStr}`);
|
|
604
|
+
}
|
|
605
|
+
else {
|
|
606
|
+
await execAsync(`start "" "${appName}" ${argsStr}`);
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
this.expectedApplication = appName;
|
|
610
|
+
// Observation-based wait: poll inspectUI to confirm window appeared
|
|
611
|
+
const delays = [400, 800, 1200];
|
|
612
|
+
for (const delay of delays) {
|
|
613
|
+
await this.wait(delay);
|
|
614
|
+
const ui = await this.inspectUI();
|
|
615
|
+
const match = ui.windows.some(w => w.toLowerCase().includes(appName.toLowerCase()));
|
|
616
|
+
if (match || (ui.activeWindow && ui.activeWindow.toLowerCase().includes(appName.toLowerCase()))) {
|
|
617
|
+
break;
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
if (this.expectedApplication) {
|
|
621
|
+
await this.switchWindow(this.expectedApplication);
|
|
622
|
+
}
|
|
623
|
+
return {
|
|
624
|
+
success: true,
|
|
625
|
+
output: `Launched and focused application: "${this.expectedApplication || appName}"`,
|
|
626
|
+
action: 'open_app',
|
|
627
|
+
activeWindow: this.expectedApplication || appName,
|
|
628
|
+
expectedWindow: this.expectedApplication || appName,
|
|
629
|
+
};
|
|
630
|
+
}
|
|
631
|
+
else if (process.platform === 'darwin') {
|
|
632
|
+
await execAsync(`open -a "${appName}" ${argsStr}`);
|
|
633
|
+
this.expectedApplication = appName;
|
|
634
|
+
}
|
|
635
|
+
else {
|
|
636
|
+
await execAsync(`xdg-open "${appName}" ${argsStr} &`);
|
|
637
|
+
this.expectedApplication = appName;
|
|
638
|
+
}
|
|
639
|
+
return {
|
|
640
|
+
success: true,
|
|
641
|
+
output: `Launched application: "${appName}"`,
|
|
642
|
+
action: 'open_app',
|
|
643
|
+
activeWindow: appName,
|
|
644
|
+
expectedWindow: appName,
|
|
645
|
+
};
|
|
646
|
+
}
|
|
647
|
+
catch (err) {
|
|
648
|
+
return {
|
|
649
|
+
success: false,
|
|
650
|
+
output: `Failed to launch "${appName}": ${err?.message || String(err)}`,
|
|
651
|
+
action: 'open_app',
|
|
652
|
+
error: err?.message || String(err),
|
|
653
|
+
recoverable: true,
|
|
654
|
+
};
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
async closeApplication(appName) {
|
|
658
|
+
try {
|
|
659
|
+
if (process.platform === 'win32') {
|
|
660
|
+
const cleanName = appName.replace(/\.exe$/i, '');
|
|
661
|
+
const driverRes = await this.runDriverAction('close_window', [appName]);
|
|
662
|
+
if (!driverRes || !driverRes.success) {
|
|
663
|
+
await execAsync(`powershell -NoProfile -Command "Stop-Process -Name '${cleanName}' -Force -ErrorAction SilentlyContinue"`);
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
else {
|
|
667
|
+
await execAsync(`pkill -f "${appName}"`);
|
|
668
|
+
}
|
|
669
|
+
if (this.expectedApplication && this.expectedApplication.toLowerCase().includes(appName.toLowerCase())) {
|
|
670
|
+
this.expectedApplication = null;
|
|
671
|
+
}
|
|
672
|
+
return { success: true, output: `Closed application: "${appName}"`, action: 'close_app' };
|
|
673
|
+
}
|
|
674
|
+
catch (err) {
|
|
675
|
+
return {
|
|
676
|
+
success: false,
|
|
677
|
+
output: `Failed to close "${appName}": ${err?.message || String(err)}`,
|
|
678
|
+
action: 'close_app',
|
|
679
|
+
error: err?.message || String(err),
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
async switchWindow(windowTitleOrApp) {
|
|
684
|
+
if (process.platform === 'win32') {
|
|
685
|
+
const res = await this.runDriverAction('focus_window', [windowTitleOrApp]);
|
|
686
|
+
if (res && res.success) {
|
|
687
|
+
this.activeWindow = res.activeWindow || windowTitleOrApp;
|
|
688
|
+
this.expectedApplication = windowTitleOrApp;
|
|
689
|
+
return {
|
|
690
|
+
success: true,
|
|
691
|
+
output: `Focused window matching: "${windowTitleOrApp}"`,
|
|
692
|
+
action: 'switch_window',
|
|
693
|
+
activeWindow: this.activeWindow || undefined,
|
|
694
|
+
expectedWindow: windowTitleOrApp,
|
|
695
|
+
};
|
|
696
|
+
}
|
|
697
|
+
// Fallback to WScript.Shell AppActivate
|
|
698
|
+
try {
|
|
699
|
+
const script = `
|
|
700
|
+
$wshell = New-Object -ComObject Wscript.Shell
|
|
701
|
+
$result = $wshell.AppActivate("${windowTitleOrApp.replace(/"/g, '`"')}")
|
|
702
|
+
if (-not $result) {
|
|
703
|
+
$p = Get-Process | Where-Object { $_.MainWindowTitle -like "*${windowTitleOrApp.replace(/"/g, '`"')}*" -or $_.ProcessName -like "*${windowTitleOrApp.replace(/"/g, '`"')}*" } | Select-Object -First 1
|
|
704
|
+
if ($p) {
|
|
705
|
+
$null = $wshell.AppActivate($p.Id)
|
|
706
|
+
$result = $true
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
if ($result) { Write-Output "OK" } else { Write-Output "NOT_FOUND" }
|
|
710
|
+
`;
|
|
711
|
+
const encoded = Buffer.from(script, 'utf16le').toString('base64');
|
|
712
|
+
const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand ${encoded}`);
|
|
713
|
+
if (stdout.includes('OK')) {
|
|
714
|
+
this.activeWindow = windowTitleOrApp;
|
|
715
|
+
this.expectedApplication = windowTitleOrApp;
|
|
716
|
+
return {
|
|
717
|
+
success: true,
|
|
718
|
+
output: `Focused window matching: "${windowTitleOrApp}"`,
|
|
719
|
+
action: 'switch_window',
|
|
720
|
+
activeWindow: windowTitleOrApp,
|
|
721
|
+
expectedWindow: windowTitleOrApp,
|
|
722
|
+
};
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
catch { }
|
|
726
|
+
return {
|
|
727
|
+
success: false,
|
|
728
|
+
output: `Could not find or focus window: "${windowTitleOrApp}". Try computer_inspect_ui to list open windows.`,
|
|
729
|
+
action: 'switch_window',
|
|
730
|
+
recoverable: true,
|
|
731
|
+
};
|
|
732
|
+
}
|
|
733
|
+
this.activeWindow = windowTitleOrApp;
|
|
734
|
+
return { success: true, output: `Switched window to: "${windowTitleOrApp}"`, action: 'switch_window' };
|
|
735
|
+
}
|
|
736
|
+
async getWindowBounds(windowTitleOrApp) {
|
|
737
|
+
if (process.platform === 'win32') {
|
|
738
|
+
const res = await this.runDriverAction('window_bounds', [windowTitleOrApp]);
|
|
739
|
+
if (res && res.success && typeof res.x === 'number') {
|
|
740
|
+
return { x: res.x, y: res.y, width: res.width, height: res.height };
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
return null;
|
|
744
|
+
}
|
|
745
|
+
async inspectUI() {
|
|
746
|
+
if (process.platform === 'win32') {
|
|
747
|
+
const res = await this.runDriverAction('list_windows', []);
|
|
748
|
+
if (res && res.success && Array.isArray(res.windows)) {
|
|
749
|
+
return {
|
|
750
|
+
activeWindow: res.activeWindow || 'Desktop',
|
|
751
|
+
windows: res.windows,
|
|
752
|
+
};
|
|
753
|
+
}
|
|
754
|
+
try {
|
|
755
|
+
const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -Command "Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle.Trim().Length -gt 0 } | Select-Object -ExpandProperty MainWindowTitle"`);
|
|
756
|
+
const titles = stdout
|
|
757
|
+
.split('\n')
|
|
758
|
+
.map((t) => t.trim())
|
|
759
|
+
.filter((t) => t.length > 0);
|
|
760
|
+
const active = titles.length > 0 ? titles[0] : 'Desktop';
|
|
761
|
+
return {
|
|
762
|
+
activeWindow: active,
|
|
763
|
+
windows: titles,
|
|
764
|
+
};
|
|
765
|
+
}
|
|
766
|
+
catch {
|
|
767
|
+
return { activeWindow: 'Desktop', windows: ['Desktop'] };
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
return { activeWindow: 'Desktop', windows: ['Desktop', 'Terminal'] };
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
//# sourceMappingURL=computerController.js.map
|