@usejarvis/brain 0.1.0

package/src/actions/app-control/linux.ts

@@ -0,0 +1,273 @@
+import type { AppController, WindowInfo, UIElement } from './interface.ts';
+import { $ } from 'bun';
+
+export class LinuxAppController implements AppController {
+  private async checkTool(tool: string): Promise<boolean> {
+    try {
+      await $`which ${tool}`.quiet();
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  private async ensureTool(tool: string): Promise<void> {
+    if (!(await this.checkTool(tool))) {
+      throw new Error(
+        `Required tool '${tool}' not found. Please install it:\n` +
+        `  Ubuntu/Debian: sudo apt install ${tool}\n` +
+        `  Fedora: sudo dnf install ${tool}\n` +
+        `  Arch: sudo pacman -S ${tool}`
+      );
+    }
+  }
+
+  async getActiveWindow(): Promise<WindowInfo> {
+    await this.ensureTool('xdotool');
+    await this.ensureTool('xprop');
+
+    try {
+      const windowId = (await $`xdotool getactivewindow`.text()).trim();
+
+      const xpropOutput = await $`xprop -id ${windowId}`.text();
+
+      const title = this.extractXpropValue(xpropOutput, 'WM_NAME') || 'Unknown';
+      const className = this.extractXpropValue(xpropOutput, 'WM_CLASS') || 'Unknown';
+
+      const geometryOutput = await $`xdotool getwindowgeometry ${windowId}`.text();
+      const bounds = this.parseGeometry(geometryOutput);
+
+      const pid = parseInt(this.extractXpropValue(xpropOutput, '_NET_WM_PID') || '0', 10);
+
+      return {
+        pid,
+        title,
+        className,
+        bounds,
+        focused: true,
+      };
+    } catch (error) {
+      throw new Error(`Failed to get active window: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  async getWindowTree(pid: number): Promise<UIElement[]> {
+    // TODO: Implement using AT-SPI2 (Assistive Technology Service Provider Interface)
+    // This requires complex bindings to the AT-SPI D-Bus interface
+    // For now, return empty array with informative error
+    console.warn(
+      `getWindowTree not yet implemented for Linux.\n` +
+      `Requires AT-SPI2 integration via D-Bus. Consider using:\n` +
+      `  - python-atspi library\n` +
+      `  - Accerciser tool for exploration\n` +
+      `  - Direct D-Bus bindings`
+    );
+    return [];
+  }
+
+  async listWindows(): Promise<WindowInfo[]> {
+    await this.ensureTool('xdotool');
+    await this.ensureTool('xprop');
+
+    try {
+      const hasWmctrl = await this.checkTool('wmctrl');
+
+      let windowIds: string[];
+
+      if (hasWmctrl) {
+        const wmctrlOutput = await $`wmctrl -l -p`.text();
+        windowIds = wmctrlOutput
+          .split('\n')
+          .filter(line => line.trim())
+          .map(line => line.split(/\s+/)[0] || '');
+      } else {
+        const xdotoolOutput = await $`xdotool search --name "."`.text();
+        windowIds = xdotoolOutput.split('\n').filter(id => id.trim());
+      }
+
+      const windows: WindowInfo[] = [];
+      const activeWindowId = (await $`xdotool getactivewindow`.text()).trim();
+
+      for (const windowId of windowIds) {
+        if (!windowId) continue;
+
+        try {
+          const xpropOutput = await $`xprop -id ${windowId}`.text();
+          const title = this.extractXpropValue(xpropOutput, 'WM_NAME') || 'Unknown';
+          const className = this.extractXpropValue(xpropOutput, 'WM_CLASS') || 'Unknown';
+          const pid = parseInt(this.extractXpropValue(xpropOutput, '_NET_WM_PID') || '0', 10);
+
+          const geometryOutput = await $`xdotool getwindowgeometry ${windowId}`.text();
+          const bounds = this.parseGeometry(geometryOutput);
+
+          windows.push({
+            pid,
+            title,
+            className,
+            bounds,
+            focused: windowId === activeWindowId,
+          });
+        } catch {
+          // Skip windows that can't be queried
+          continue;
+        }
+      }
+
+      return windows;
+    } catch (error) {
+      throw new Error(`Failed to list windows: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  async clickElement(element: UIElement): Promise<void> {
+    await this.ensureTool('xdotool');
+
+    try {
+      const centerX = element.bounds.x + element.bounds.width / 2;
+      const centerY = element.bounds.y + element.bounds.height / 2;
+
+      await $`xdotool mousemove ${Math.round(centerX)} ${Math.round(centerY)}`;
+      await $`xdotool click 1`;
+    } catch (error) {
+      throw new Error(`Failed to click element: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  async typeText(text: string): Promise<void> {
+    await this.ensureTool('xdotool');
+
+    try {
+      await $`xdotool type --clearmodifiers ${text}`;
+    } catch (error) {
+      throw new Error(`Failed to type text: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  async pressKeys(keys: string[]): Promise<void> {
+    await this.ensureTool('xdotool');
+
+    try {
+      const keyString = keys.join('+');
+      await $`xdotool key --clearmodifiers ${keyString}`;
+    } catch (error) {
+      throw new Error(`Failed to press keys: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  async captureScreen(): Promise<Buffer> {
+    const hasImport = await this.checkTool('import');
+    const hasScrot = await this.checkTool('scrot');
+
+    if (!hasImport && !hasScrot) {
+      throw new Error(
+        `No screenshot tool found. Please install one:\n` +
+        `  ImageMagick: sudo apt install imagemagick\n` +
+        `  Scrot: sudo apt install scrot`
+      );
+    }
+
+    try {
+      const tmpFile = `/tmp/jarvis-screen-${Date.now()}.png`;
+
+      if (hasImport) {
+        await $`import -window root ${tmpFile}`;
+      } else {
+        await $`scrot ${tmpFile}`;
+      }
+
+      const buffer = await Bun.file(tmpFile).arrayBuffer();
+      await $`rm ${tmpFile}`;
+
+      return Buffer.from(buffer);
+    } catch (error) {
+      throw new Error(`Failed to capture screen: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  async captureWindow(pid: number): Promise<Buffer> {
+    await this.ensureTool('xdotool');
+    const hasImport = await this.checkTool('import');
+
+    if (!hasImport) {
+      throw new Error(
+        `ImageMagick not found. Please install:\n` +
+        `  sudo apt install imagemagick`
+      );
+    }
+
+    try {
+      const windowId = await this.findWindowByPid(pid);
+
+      const tmpFile = `/tmp/jarvis-window-${pid}-${Date.now()}.png`;
+      await $`import -window ${windowId} ${tmpFile}`;
+
+      const buffer = await Bun.file(tmpFile).arrayBuffer();
+      await $`rm ${tmpFile}`;
+
+      return Buffer.from(buffer);
+    } catch (error) {
+      throw new Error(`Failed to capture window: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  async focusWindow(pid: number): Promise<void> {
+    await this.ensureTool('xdotool');
+
+    try {
+      const windowId = await this.findWindowByPid(pid);
+      await $`xdotool windowactivate ${windowId}`;
+    } catch (error) {
+      throw new Error(`Failed to focus window: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  private async findWindowByPid(pid: number): Promise<string> {
+    const windowIds = (await $`xdotool search --name "."`.text())
+      .split('\n')
+      .filter(id => id.trim());
+
+    for (const windowId of windowIds) {
+      try {
+        const xpropOutput = await $`xprop -id ${windowId}`.text();
+        const windowPid = parseInt(this.extractXpropValue(xpropOutput, '_NET_WM_PID') || '0', 10);
+
+        if (windowPid === pid) {
+          return windowId;
+        }
+      } catch {
+        continue;
+      }
+    }
+
+    throw new Error(`No window found for PID ${pid}`);
+  }
+
+  private extractXpropValue(output: string, property: string): string | null {
+    const regex = new RegExp(`^${property}\\(.*?\\)\\s*=\\s*(.+)$`, 'm');
+    const match = output.match(regex);
+
+    if (!match || !match[1]) {
+      return null;
+    }
+
+    let value = match[1].trim();
+
+    value = value.replace(/^"(.*)"$/, '$1');
+    value = value.replace(/^{([^}]*)}.*$/, '$1');
+    value = value.replace(/^"([^"]*)".*$/, '$1');
+
+    return value;
+  }
+
+  private parseGeometry(geometryOutput: string): { x: number; y: number; width: number; height: number } {
+    const positionMatch = geometryOutput.match(/Position:\s*(\d+),(\d+)/);
+    const geometryMatch = geometryOutput.match(/Geometry:\s*(\d+)x(\d+)/);
+
+    const x = positionMatch?.[1] ? parseInt(positionMatch[1], 10) : 0;
+    const y = positionMatch?.[2] ? parseInt(positionMatch[2], 10) : 0;
+    const width = geometryMatch?.[1] ? parseInt(geometryMatch[1], 10) : 0;
+    const height = geometryMatch?.[2] ? parseInt(geometryMatch[2], 10) : 0;
+
+    return { x, y, width, height };
+  }
+}

package/src/actions/app-control/macos.ts

@@ -0,0 +1,54 @@
+import type { AppController, WindowInfo, UIElement } from './interface.ts';
+
+export class MacAppController implements AppController {
+  private notImplemented(method: string): never {
+    throw new Error(
+      `${method} not yet implemented for macOS.\n\n` +
+      `TODO: Implement using one of:\n` +
+      `  - AXUIElement API via N-API native bindings\n` +
+      `  - AppleScript automation\n` +
+      `  - node-mac-automation package\n` +
+      `  - Swift/Objective-C bridge via FFI\n\n` +
+      `Reference:\n` +
+      `  - https://developer.apple.com/documentation/applicationservices/axuielement\n` +
+      `  - https://developer.apple.com/library/archive/documentation/Accessibility/Conceptual/AccessibilityMacOSX/\n` +
+      `  - https://github.com/sveinbjornt/AXElementsTester`
+    );
+  }
+
+  async getActiveWindow(): Promise<WindowInfo> {
+    this.notImplemented('getActiveWindow');
+  }
+
+  async getWindowTree(pid: number): Promise<UIElement[]> {
+    this.notImplemented('getWindowTree');
+  }
+
+  async listWindows(): Promise<WindowInfo[]> {
+    this.notImplemented('listWindows');
+  }
+
+  async clickElement(element: UIElement): Promise<void> {
+    this.notImplemented('clickElement');
+  }
+
+  async typeText(text: string): Promise<void> {
+    this.notImplemented('typeText');
+  }
+
+  async pressKeys(keys: string[]): Promise<void> {
+    this.notImplemented('pressKeys');
+  }
+
+  async captureScreen(): Promise<Buffer> {
+    this.notImplemented('captureScreen');
+  }
+
+  async captureWindow(pid: number): Promise<Buffer> {
+    this.notImplemented('captureWindow');
+  }
+
+  async focusWindow(pid: number): Promise<void> {
+    this.notImplemented('focusWindow');
+  }
+}

package/src/actions/app-control/sidecar-launcher.test.ts

@@ -0,0 +1,23 @@
+import { test, expect, describe } from 'bun:test';
+import { findSidecarExecutable, getWSLHostIP } from './sidecar-launcher.ts';
+
+describe('sidecar-launcher', () => {
+  test('findSidecarExecutable returns string or null', () => {
+    const result = findSidecarExecutable();
+    expect(result === null || typeof result === 'string').toBe(true);
+  });
+
+  test('getWSLHostIP returns a string', () => {
+    const ip = getWSLHostIP();
+    expect(typeof ip).toBe('string');
+    expect(ip.length).toBeGreaterThan(0);
+  });
+
+  test('getWSLHostIP returns localhost or valid IP', () => {
+    const ip = getWSLHostIP();
+    // Either "localhost" or an IPv4 address
+    const isLocalhost = ip === 'localhost';
+    const isIP = /^\d+\.\d+\.\d+\.\d+$/.test(ip);
+    expect(isLocalhost || isIP).toBe(true);
+  });
+});

package/src/actions/app-control/sidecar-launcher.ts

@@ -0,0 +1,286 @@
+/**
+ * Sidecar Launcher — Find, Launch, and Manage the Desktop Bridge
+ *
+ * Auto-detects the desktop-bridge.exe sidecar, launches it from WSL,
+ * and polls TCP to confirm it's ready. Mirrors the pattern in
+ * src/actions/browser/chrome-launcher.ts.
+ */
+
+import { spawn, type Subprocess } from 'bun';
+import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { createConnection } from 'node:net';
+import { WSLBridge } from '../terminal/wsl-bridge.ts';
+
+export type RunningSidecar = {
+  proc: Subprocess | null; // null if externally managed
+  port: number;
+  host: string;
+  startedAt: number;
+  exePath: string;
+};
+
+const DEFAULT_PORT = 9224;
+
+/**
+ * Find the desktop-bridge.exe sidecar on disk.
+ * Checks: ~/.jarvis/sidecar/ then repo build output.
+ */
+export function findSidecarExecutable(): string | null {
+  const candidates: string[] = [];
+
+  if (WSLBridge.isWSL()) {
+    // WSL: check the Windows-side paths via /mnt/c/
+    // First try USERPROFILE-based path (most common)
+    try {
+      const userProfileResult = Bun.spawnSync(['cmd.exe', '/C', 'echo', '%USERPROFILE%']);
+      const userProfile = userProfileResult.stdout.toString().trim();
+      if (userProfile && !userProfile.includes('%')) {
+        const drive = userProfile.charAt(0).toLowerCase();
+        const rest = userProfile.slice(2).replace(/\\/g, '/');
+        const wslPath = `/mnt/${drive}${rest}/.jarvis/sidecar/desktop-bridge.exe`;
+        candidates.push(wslPath);
+      }
+    } catch {
+      // Fall back to common paths
+    }
+
+    candidates.push('/mnt/c/Users/' + (process.env.USER ?? 'user') + '/.jarvis/sidecar/desktop-bridge.exe');
+  } else {
+    // Native Windows
+    candidates.push(join(homedir(), '.jarvis', 'sidecar', 'desktop-bridge.exe'));
+  }
+
+  // Also check repo build output (both custom -o path and default dotnet publish path)
+  const repoBase = join(import.meta.dir, '../../../sidecar/desktop-bridge');
+  candidates.push(join(repoBase, 'bin', 'publish', 'desktop-bridge.exe'));
+  candidates.push(join(repoBase, 'bin', 'Release', 'net10.0-windows', 'win-x64', 'publish', 'desktop-bridge.exe'));
+  candidates.push(join(repoBase, 'bin', 'Release', 'net9.0-windows', 'win-x64', 'publish', 'desktop-bridge.exe'));
+  candidates.push(join(repoBase, 'bin', 'Release', 'net8.0-windows', 'win-x64', 'publish', 'desktop-bridge.exe'));
+
+  for (const path of candidates) {
+    if (existsSync(path)) {
+      return path;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Get the Windows host IP when WSL2 doesn't have mirrored networking.
+ * Parses /etc/resolv.conf for the nameserver entry.
+ */
+export function getWSLHostIP(): string {
+  try {
+    const resolv = readFileSync('/etc/resolv.conf', 'utf-8');
+    const match = resolv.match(/nameserver\s+(\d+\.\d+\.\d+\.\d+)/);
+    if (match) {
+      return match[1]!;
+    }
+  } catch {
+    // ignore
+  }
+  return 'localhost';
+}
+
+/**
+ * Determine the host to connect to the sidecar.
+ * With WSL2 mirrored networking, localhost works.
+ * Without it, we use the Windows host IP.
+ */
+function getSidecarHost(): string {
+  if (!WSLBridge.isWSL()) {
+    return 'localhost';
+  }
+
+  // Check if mirrored networking is active (localhost works)
+  // WSL2 mirrored mode sets networkingMode in .wslconfig
+  // Simplest check: if localhost resolves to Windows, it's mirrored
+  // For now, try localhost first; the connect logic handles fallback
+  return 'localhost';
+}
+
+/**
+ * Check if the sidecar is already running on the given port.
+ */
+export async function isSidecarRunning(port: number = DEFAULT_PORT): Promise<boolean> {
+  const host = getSidecarHost();
+  const hosts = [host];
+
+  // If on WSL and host is localhost, also try the WSL host IP as fallback
+  if (WSLBridge.isWSL() && host === 'localhost') {
+    const hostIP = getWSLHostIP();
+    if (hostIP !== 'localhost') {
+      hosts.push(hostIP);
+    }
+  }
+
+  for (const h of hosts) {
+    try {
+      const alive = await pingTcp(h, port, 2000);
+      if (alive) return true;
+    } catch {
+      // Try next host
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Launch the desktop-bridge sidecar.
+ * Auto-detects the executable and spawns it.
+ */
+export async function launchSidecar(port: number = DEFAULT_PORT): Promise<RunningSidecar> {
+  const exePath = findSidecarExecutable();
+  if (!exePath) {
+    throw new Error(
+      'Desktop bridge sidecar not found.\n' +
+      'Build it with: bun run scripts/build-sidecar.ts\n' +
+      'Requires .NET 8 SDK on Windows.'
+    );
+  }
+
+  console.log(`[SidecarLauncher] Launching: ${exePath}`);
+
+  // Spawn the sidecar
+  const proc = spawn([exePath, '--port', String(port)], {
+    stdout: 'ignore',
+    stderr: 'ignore',
+  });
+
+  const startedAt = Date.now();
+  const host = getSidecarHost();
+
+  // Poll TCP for up to 10s
+  const deadline = Date.now() + 10_000;
+  let reachable = false;
+  let connectedHost = host;
+
+  while (Date.now() < deadline) {
+    try {
+      if (await pingTcp(host, port, 1000)) {
+        reachable = true;
+        connectedHost = host;
+        break;
+      }
+    } catch {
+      // Not ready yet
+    }
+
+    // Also try WSL host IP if localhost fails
+    if (WSLBridge.isWSL() && host === 'localhost') {
+      const hostIP = getWSLHostIP();
+      if (hostIP !== 'localhost') {
+        try {
+          if (await pingTcp(hostIP, port, 1000)) {
+            reachable = true;
+            connectedHost = hostIP;
+            break;
+          }
+        } catch {
+          // Not ready
+        }
+      }
+    }
+
+    await Bun.sleep(300);
+  }
+
+  if (!reachable) {
+    try { proc.kill(); } catch {}
+    throw new Error(
+      `Sidecar started but not reachable on port ${port} after 10s.\n` +
+      `Binary: ${exePath}`
+    );
+  }
+
+  console.log(`[SidecarLauncher] Sidecar ready on ${connectedHost}:${port} (pid ${proc.pid})`);
+
+  return { proc, port, host: connectedHost, startedAt, exePath };
+}
+
+/**
+ * Stop a running sidecar. Sends shutdown command, then kills process.
+ */
+export async function stopSidecar(running: RunningSidecar): Promise<void> {
+  // Try graceful shutdown via JSON-RPC
+  try {
+    const shutdown = JSON.stringify({ jsonrpc: '2.0', method: 'shutdown', params: {}, id: 0 }) + '\n';
+    await sendTcpRaw(running.host, running.port, shutdown, 2000);
+  } catch {
+    // ignore
+  }
+
+  // Kill process if we spawned it
+  if (running.proc) {
+    try { running.proc.kill(); } catch {}
+
+    // Wait for exit
+    const deadline = Date.now() + 3000;
+    while (Date.now() < deadline) {
+      if (running.proc.exitCode !== null) break;
+      await Bun.sleep(100);
+    }
+
+    try { running.proc.kill(9); } catch {}
+  }
+
+  console.log('[SidecarLauncher] Sidecar stopped');
+}
+
+// --- TCP helpers ---
+
+function pingTcp(host: string, port: number, timeoutMs: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    const socket = createConnection({ host, port, timeout: timeoutMs }, () => {
+      // Send ping JSON-RPC
+      const ping = JSON.stringify({ jsonrpc: '2.0', method: 'ping', params: {}, id: -1 }) + '\n';
+      socket.write(ping);
+    });
+
+    let data = '';
+
+    socket.on('data', (chunk) => {
+      data += chunk.toString();
+      if (data.includes('pong')) {
+        socket.destroy();
+        resolve(true);
+      }
+    });
+
+    socket.on('error', () => {
+      socket.destroy();
+      resolve(false);
+    });
+
+    socket.on('timeout', () => {
+      socket.destroy();
+      resolve(false);
+    });
+
+    // Safety timeout
+    setTimeout(() => {
+      socket.destroy();
+      resolve(false);
+    }, timeoutMs + 500);
+  });
+}
+
+function sendTcpRaw(host: string, port: number, data: string, timeoutMs: number): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const socket = createConnection({ host, port, timeout: timeoutMs }, () => {
+      socket.write(data, () => {
+        socket.destroy();
+        resolve();
+      });
+    });
+    socket.on('error', reject);
+    socket.on('timeout', () => {
+      socket.destroy();
+      reject(new Error('Timeout'));
+    });
+  });
+}

package/src/actions/app-control/windows.ts

@@ -0,0 +1,44 @@
+import type { AppController, WindowInfo, UIElement } from './interface.ts';
+
+/**
+ * Windows App Controller — stub.
+ * Previously delegated to the C# DesktopController sidecar.
+ * TODO: Implement local platform-native commands or route via Go sidecar.
+ */
+export class WindowsAppController implements AppController {
+  async getActiveWindow(): Promise<WindowInfo> {
+    throw new Error('WindowsAppController: Not implemented. Use sidecar desktop tools with a target parameter.');
+  }
+
+  async getWindowTree(_pid: number): Promise<UIElement[]> {
+    throw new Error('WindowsAppController: Not implemented. Use sidecar desktop tools with a target parameter.');
+  }
+
+  async listWindows(): Promise<WindowInfo[]> {
+    throw new Error('WindowsAppController: Not implemented. Use sidecar desktop tools with a target parameter.');
+  }
+
+  async clickElement(_element: UIElement): Promise<void> {
+    throw new Error('WindowsAppController: Not implemented. Use sidecar desktop tools with a target parameter.');
+  }
+
+  async typeText(_text: string): Promise<void> {
+    throw new Error('WindowsAppController: Not implemented. Use sidecar desktop tools with a target parameter.');
+  }
+
+  async pressKeys(_keys: string[]): Promise<void> {
+    throw new Error('WindowsAppController: Not implemented. Use sidecar desktop tools with a target parameter.');
+  }
+
+  async captureScreen(): Promise<Buffer> {
+    throw new Error('WindowsAppController: Not implemented. Use sidecar desktop tools with a target parameter.');
+  }
+
+  async captureWindow(_pid: number): Promise<Buffer> {
+    throw new Error('WindowsAppController: Not implemented. Use sidecar desktop tools with a target parameter.');
+  }
+
+  async focusWindow(_pid: number): Promise<void> {
+    throw new Error('WindowsAppController: Not implemented. Use sidecar desktop tools with a target parameter.');
+  }
+}