aiden-runtime 3.16.0

package/dist/core/computerControl.js

@@ -0,0 +1,439 @@
+"use strict";
+// ============================================================
+// DevOS — Autonomous AI Execution System
+// Copyright (c) 2026 Shiva Deore. All rights reserved.
+// ============================================================
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.moveMouse = moveMouse;
+exports.clickMouse = clickMouse;
+exports.typeText = typeText;
+exports.pressKey = pressKey;
+exports.takeScreenshot = takeScreenshot;
+exports.readScreen = readScreen;
+exports.openBrowser = openBrowser;
+exports.focusWindow = focusWindow;
+exports.getScreenSize = getScreenSize;
+exports.visionLoop = visionLoop;
+exports.executeWithVisionRetry = executeWithVisionRetry;
+exports.executeWithFallback = executeWithFallback;
+// core/computerControl.ts — Mouse, keyboard, screenshot, and
+// vision-loop computer control.
+// Pure PowerShell implementation — zero native dependencies.
+// Works on any Windows machine without Visual Studio or nut-js.
+const child_process_1 = require("child_process");
+const util_1 = require("util");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const auditTrail_1 = require("./auditTrail");
+const execAsync = (0, util_1.promisify)(child_process_1.exec);
+const SCREENSHOTS_DIR = path_1.default.join(process.cwd(), 'workspace', 'screenshots');
+try {
+    fs_1.default.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
+}
+catch { }
+// ── PowerShell helpers ────────────────────────────────────────
+// Run a simple one-liner PowerShell command
+async function ps(script) {
+    try {
+        const escaped = script.replace(/"/g, '\\"');
+        const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -Command "${escaped}"`, { timeout: 15000 });
+        return stdout.trim();
+    }
+    catch (e) {
+        return e.message || '';
+    }
+}
+// Write a .ps1 file and execute it — avoids all quoting issues for complex scripts
+async function psFile(script) {
+    const tmpFile = path_1.default.join(process.cwd(), 'workspace', `ps_${Date.now()}.ps1`);
+    fs_1.default.writeFileSync(tmpFile, script, 'utf8');
+    try {
+        const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "${tmpFile}"`, { timeout: 15000 });
+        try {
+            fs_1.default.unlinkSync(tmpFile);
+        }
+        catch { }
+        return stdout.trim();
+    }
+    catch (e) {
+        try {
+            fs_1.default.unlinkSync(tmpFile);
+        }
+        catch { }
+        return e.message || '';
+    }
+}
+// ── MOUSE ──────────────────────────────────────────────────────
+async function moveMouse(x, y) {
+    await psFile(`
+Add-Type -AssemblyName System.Windows.Forms
+[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x}, ${y})
+`);
+    return `Mouse moved to (${x}, ${y})`;
+}
+async function clickMouse(x, y, button = 'left', double = false) {
+    const clicks = double ? 2 : 1;
+    const isRight = button === 'right';
+    await psFile(`
+Add-Type @"
+using System;
+using System.Runtime.InteropServices;
+public class MouseClick {
+  [DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y);
+  [DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int cButtons, int dwExtraInfo);
+  public const int MOUSEEVENTF_LEFTDOWN  = 0x02;
+  public const int MOUSEEVENTF_LEFTUP   = 0x04;
+  public const int MOUSEEVENTF_RIGHTDOWN = 0x08;
+  public const int MOUSEEVENTF_RIGHTUP  = 0x10;
+}
+"@
+[MouseClick]::SetCursorPos(${x}, ${y})
+Start-Sleep -Milliseconds 100
+for ($i = 0; $i -lt ${clicks}; $i++) {
+  ${isRight
+        ? `[MouseClick]::mouse_event([MouseClick]::MOUSEEVENTF_RIGHTDOWN, 0, 0, 0, 0)
+  Start-Sleep -Milliseconds 50
+  [MouseClick]::mouse_event([MouseClick]::MOUSEEVENTF_RIGHTUP, 0, 0, 0, 0)`
+        : `[MouseClick]::mouse_event([MouseClick]::MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0)
+  Start-Sleep -Milliseconds 50
+  [MouseClick]::mouse_event([MouseClick]::MOUSEEVENTF_LEFTUP, 0, 0, 0, 0)`}
+  Start-Sleep -Milliseconds 80
+}
+`);
+    return `${double ? 'Double-clicked' : 'Clicked'} ${button} at (${x}, ${y})`;
+}
+// ── KEYBOARD ───────────────────────────────────────────────────
+async function typeText(text) {
+    // WScript.Shell SendKeys — most reliable for text input
+    const safe = text.replace(/'/g, "''").replace(/[+^%~(){}]/g, '{$&}');
+    await psFile(`
+$wsh = New-Object -ComObject WScript.Shell
+Start-Sleep -Milliseconds 200
+$wsh.SendKeys('${safe}')
+`);
+    return `Typed: ${text.slice(0, 50)}${text.length > 50 ? '...' : ''}`;
+}
+async function pressKey(key) {
+    const keyMap = {
+        enter: '{ENTER}',
+        tab: '{TAB}',
+        escape: '{ESC}',
+        esc: '{ESC}',
+        backspace: '{BACKSPACE}',
+        delete: '{DELETE}',
+        up: '{UP}',
+        down: '{DOWN}',
+        left: '{LEFT}',
+        right: '{RIGHT}',
+        home: '{HOME}',
+        end: '{END}',
+        pageup: '{PGUP}',
+        pagedown: '{PGDN}',
+        f1: '{F1}', f2: '{F2}', f3: '{F3}', f4: '{F4}',
+        f5: '{F5}', f6: '{F6}', f7: '{F7}', f8: '{F8}',
+        f11: '{F11}', f12: '{F12}',
+        'ctrl+c': '^c',
+        'ctrl+v': '^v',
+        'ctrl+a': '^a',
+        'ctrl+z': '^z',
+        'ctrl+s': '^s',
+        'ctrl+t': '^t',
+        'ctrl+w': '^w',
+        'ctrl+l': '^l',
+        'ctrl+r': '^r',
+        'ctrl_c': '^c',
+        'ctrl_v': '^v',
+        'ctrl_a': '^a',
+        'ctrl_z': '^z',
+        'ctrl_l': '^l',
+        'alt+f4': '%{F4}',
+        win: '{LWIN}',
+    };
+    const mapped = keyMap[key.toLowerCase()] || `{${key.toUpperCase()}}`;
+    await psFile(`
+$wsh = New-Object -ComObject WScript.Shell
+Start-Sleep -Milliseconds 100
+$wsh.SendKeys('${mapped}')
+`);
+    return `Pressed: ${key}`;
+}
+// ── SCREENSHOT ─────────────────────────────────────────────────
+async function takeScreenshot() {
+    const filename = `screenshot_${Date.now()}.png`;
+    const filepath = path_1.default.join(SCREENSHOTS_DIR, filename);
+    const escaped = filepath.replace(/\\/g, '\\\\');
+    await psFile(`
+Add-Type -AssemblyName System.Windows.Forms
+Add-Type -AssemblyName System.Drawing
+$screen   = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
+$bitmap   = New-Object System.Drawing.Bitmap($screen.Width, $screen.Height)
+$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+$graphics.CopyFromScreen($screen.Location, [System.Drawing.Point]::Empty, $screen.Size)
+$bitmap.Save('${escaped}')
+$graphics.Dispose()
+$bitmap.Dispose()
+`);
+    // Trim old screenshots — keep only last 10
+    try {
+        const files = fs_1.default.readdirSync(SCREENSHOTS_DIR)
+            .filter(f => f.endsWith('.png'))
+            .sort();
+        if (files.length > 10) {
+            files.slice(0, files.length - 10).forEach(f => {
+                try {
+                    fs_1.default.unlinkSync(path_1.default.join(SCREENSHOTS_DIR, f));
+                }
+                catch { }
+            });
+        }
+    }
+    catch { }
+    if (fs_1.default.existsSync(filepath))
+        return filepath;
+    throw new Error('Screenshot failed — file not created');
+}
+async function readScreen() {
+    const filepath = await takeScreenshot();
+    return `Screenshot saved: ${filepath}`;
+}
+// ── BROWSER ────────────────────────────────────────────────────
+async function openBrowser(url) {
+    const safeUrl = url.replace(/'/g, '%27');
+    await psFile(`Start-Process '${safeUrl}'`);
+    // Wait for browser to load
+    await new Promise(r => setTimeout(r, 3000));
+    return `Opened browser: ${url}`;
+}
+async function focusWindow(title) {
+    await psFile(`
+Add-Type @"
+using System;
+using System.Runtime.InteropServices;
+public class Win32 {
+  [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
+  [DllImport("user32.dll")] public static extern IntPtr FindWindow(string lpClassName, string lpWindowName);
+}
+"@
+$hwnd = [Win32]::FindWindow([NullString]::Value, "${title}")
+if ($hwnd -ne [IntPtr]::Zero) { [Win32]::SetForegroundWindow($hwnd) }
+`);
+    await new Promise(r => setTimeout(r, 500));
+    return `Focused window: ${title}`;
+}
+// ── SCREEN SIZE ────────────────────────────────────────────────
+async function getScreenSize() {
+    const result = await psFile(`
+Add-Type -AssemblyName System.Windows.Forms
+$screen = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
+Write-Output "$($screen.Width)x$($screen.Height)"
+`);
+    const [w, h] = result.split('x').map(Number);
+    return { width: w || 1920, height: h || 1080 };
+}
+// ── VISION LOOP ────────────────────────────────────────────────
+// Iterative see → decide → act loop driven by the active LLM.
+async function visionLoop(goal, maxSteps = 10, callLLM) {
+    const results = [];
+    for (let step = 1; step <= maxSteps; step++) {
+        // Take screenshot
+        let screenshotPath;
+        try {
+            screenshotPath = await takeScreenshot();
+        }
+        catch (e) {
+            return `Vision loop failed at step ${step}: screenshot error — ${e.message}`;
+        }
+        const prompt = `
+You are controlling a Windows computer to achieve this goal: "${goal}"
+
+Steps completed so far:
+${results.length > 0 ? results.map((r, i) => `${i + 1}. ${r}`).join('\n') : 'None yet'}
+
+Screenshot taken at: ${screenshotPath}
+
+Decide the next action. Respond with ONLY a JSON object:
+{
+  "action": "click|type|key|scroll|done|failed",
+  "x": 500,
+  "y": 300,
+  "text": "text to type if action is type",
+  "key": "key name if action is key",
+  "reason": "why this action",
+  "confidence": 0.85,
+  "goal_complete": false
+}
+
+Rules:
+- Use "done" when the goal is fully complete
+- Use "failed" if you cannot proceed
+- confidence below 0.5 → use "done" and explain in reason
+- For browser: click address bar (ctrl+l) → type URL → press enter
+`;
+        let actionJson;
+        try {
+            const response = await callLLM(prompt);
+            const jsonMatch = response.match(/\{[\s\S]*\}/);
+            if (!jsonMatch)
+                throw new Error('No JSON in response');
+            actionJson = JSON.parse(jsonMatch[0]);
+        }
+        catch {
+            results.push(`Step ${step}: LLM parse error — skipping`);
+            continue;
+        }
+        if (actionJson.goal_complete || actionJson.action === 'done') {
+            results.push(`Goal complete: ${actionJson.reason}`);
+            break;
+        }
+        if (actionJson.action === 'failed') {
+            return `Vision loop failed: ${actionJson.reason}`;
+        }
+        if ((actionJson.confidence ?? 1) < 0.5) {
+            return `Vision loop stopped: low confidence (${actionJson.confidence}) — ${actionJson.reason}`;
+        }
+        // Execute action
+        let result = '';
+        switch (actionJson.action) {
+            case 'click':
+                result = await clickMouse(actionJson.x ?? 0, actionJson.y ?? 0);
+                await new Promise(r => setTimeout(r, 800));
+                break;
+            case 'type':
+                result = await typeText(actionJson.text || '');
+                await new Promise(r => setTimeout(r, 500));
+                break;
+            case 'key':
+                result = await pressKey(actionJson.key || 'enter');
+                await new Promise(r => setTimeout(r, 500));
+                break;
+            case 'scroll':
+                await psFile(`
+$wsh = New-Object -ComObject WScript.Shell
+$wsh.SendKeys('{PGDN}')
+`);
+                result = 'Scrolled down';
+                await new Promise(r => setTimeout(r, 500));
+                break;
+            default:
+                result = `Unknown action: ${actionJson.action}`;
+        }
+        results.push(`Step ${step}: ${actionJson.action} — ${result} (${actionJson.reason})`);
+        console.log(`[VisionLoop] Step ${step}: ${actionJson.action} — ${actionJson.reason}`);
+    }
+    return results.join('\n');
+}
+/**
+ * runVisionLoop — wraps the existing visionLoop with a confidence-scored result.
+ * Confidence is derived from whether the output indicates completion vs failure.
+ */
+async function runVisionLoop(task) {
+    try {
+        // Provide a no-op LLM stub — real use goes through the tool registry which
+        // injects the actual callLLM; this layer only needs to determine success/confidence.
+        const output = await visionLoop(task, 10, async (prompt) => {
+            // If visionLoop is called standalone here, we can't call the real LLM.
+            // Return a sentinel that triggers the "failed" path gracefully.
+            return JSON.stringify({ action: 'failed', reason: 'standalone_run_no_llm', confidence: 0 });
+        });
+        if (output.includes('Goal complete')) {
+            return { success: true, confidence: 1.0, output };
+        }
+        if (output.includes('low confidence')) {
+            const match = output.match(/confidence \(([0-9.]+)\)/);
+            const conf = match ? parseFloat(match[1]) : 0.3;
+            return { success: false, confidence: conf, output };
+        }
+        if (output.includes('failed')) {
+            return { success: false, confidence: 0.0, error: output };
+        }
+        // Partial output — screenshot was taken but completion unclear
+        return { success: false, confidence: 0.3, output };
+    }
+    catch (e) {
+        return { success: false, confidence: 0.0, error: e.message };
+    }
+}
+/**
+ * executeWithVisionRetry — retries vision loop up to maxAttempts times.
+ * Returns early if confidence > 0.7 on any attempt.
+ */
+async function executeWithVisionRetry(task, maxAttempts = 3) {
+    for (let i = 0; i < maxAttempts; i++) {
+        const result = await runVisionLoop(task);
+        if (result.success && result.confidence > 0.7)
+            return result;
+        await new Promise(r => setTimeout(r, 1000 * (i + 1)));
+    }
+    return { success: false, confidence: 0, error: 'vision_failed_after_retries' };
+}
+/**
+ * executePowerShell — thin wrapper to run a task as a PowerShell command (Tier 2).
+ */
+async function executePowerShell(task) {
+    try {
+        const output = await psFile(task);
+        const success = !output.toLowerCase().includes('error') && output.length > 0;
+        return { success, output };
+    }
+    catch (e) {
+        return { success: false, output: e.message };
+    }
+}
+/**
+ * executeWithFallback — 4-tier escalation ladder.
+ *
+ * Tier 2: PowerShell direct execution
+ * Tier 3: VisionLoop with retries
+ * Tier 4: Log escalation, return clear message for manual intervention
+ *
+ * (Tier 1 = direct API / native tool call; handled upstream by the tool registry)
+ */
+async function executeWithFallback(task) {
+    // Tier 2 — PowerShell
+    try {
+        const psResult = await executePowerShell(task);
+        if (psResult.success)
+            return { success: true, tier: 2, output: psResult.output };
+    }
+    catch { }
+    // Tier 3 — VisionLoop with retries
+    const visionResult = await executeWithVisionRetry(task);
+    if (visionResult.success)
+        return { success: true, tier: 3, output: visionResult.output };
+    // Tier 4 — Escalation: log and return a clear message
+    auditTrail_1.auditTrail.record({
+        action: 'system',
+        tool: 'computer_control',
+        input: task.slice(0, 200),
+        durationMs: 0,
+        success: false,
+        error: 'Escalated to Tier 4 — vision confidence too low',
+    });
+    return {
+        success: false,
+        tier: 4,
+        error: 'All automated tiers failed. This task requires Claude Computer Use or manual intervention.',
+    };
+}
+/*
+MANUAL TEST — run in PowerShell from DevOS root:
+  node -e "const cc = require('./core/computerControl'); cc.takeScreenshot().then(p => console.log('Screenshot:', p)).catch(console.error);"
+
+Expected: Screenshot saved to workspace/screenshots/screenshot_[timestamp].png
+
+Full flow test — ask Aiden:
+  "open chrome, go to google.com, search for batman, tell me what you see"
+
+Expected plan:
+  Step 1: open_browser({ url: 'https://www.google.com' })
+  Step 2: wait({ ms: 2000 })
+  Step 3: keyboard_press({ key: 'ctrl+l' })
+  Step 4: keyboard_type({ text: 'batman' })
+  Step 5: keyboard_press({ key: 'enter' })
+  Step 6: wait({ ms: 1500 })
+  Step 7: screenshot()
+  Step 8: screen_read()  -- describe what's visible
+*/