@usejarvis/brain 0.1.0

package/src/actions/browser/session.ts

@@ -0,0 +1,437 @@
+/**
+ * Browser Controller — High-level browser automation
+ *
+ * Wraps CDPClient with user-friendly operations:
+ * navigate, snapshot (interactive elements with IDs), click, type, screenshot.
+ *
+ * The snapshot approach: each interactive element gets a numeric [id].
+ * The LLM sees these IDs and references them in click/type commands.
+ */
+
+import { CDPClient } from './cdp.ts';
+import { STEALTH_SCRIPT } from './stealth.ts';
+import { launchChrome, stopChrome, type RunningBrowser } from './chrome-launcher.ts';
+
+export type PageElement = {
+  id: number;
+  tag: string;
+  text: string;
+  attrs: Record<string, string>;
+};
+
+export type PageSnapshot = {
+  title: string;
+  url: string;
+  text: string;
+  elements: PageElement[];
+};
+
+// JS function injected into the page to extract interactive elements
+const SNAPSHOT_SCRIPT = `(() => {
+  const els = [];
+  const seen = new WeakSet();
+  const sel = [
+    'a', 'button', 'input', 'select', 'textarea', 'summary',
+    '[role="button"]', '[role="link"]', '[role="tab"]', '[role="textbox"]',
+    '[role="combobox"]', '[role="menuitem"]', '[role="option"]',
+    '[onclick]', '[contenteditable="true"]', '[tabindex="0"]',
+    '[data-testid]'
+  ].join(', ');
+  document.querySelectorAll(sel).forEach((el) => {
+    // Skip duplicates (child of already-captured parent)
+    if (seen.has(el)) return;
+    seen.add(el);
+
+    const rect = el.getBoundingClientRect();
+    if (rect.width === 0 || rect.height === 0) return;
+    if (rect.width < 5 || rect.height < 5) return;
+    const style = window.getComputedStyle(el);
+    if (style.visibility === 'hidden') return;
+    if (style.display === 'none') return;
+    if (style.opacity === '0') return;
+
+    const tag = el.tagName.toLowerCase();
+    const text = (el.innerText || el.textContent || '').trim().replace(/\\s+/g, ' ').slice(0, 100);
+    const attrs = {};
+    for (const a of ['href', 'name', 'placeholder', 'type', 'value', 'aria-label', 'title', 'id', 'role', 'data-testid', 'contenteditable']) {
+      const v = el.getAttribute(a);
+      if (v) attrs[a] = v.slice(0, 200);
+    }
+    els.push({
+      tag,
+      text,
+      attrs,
+      x: Math.round(rect.x + rect.width / 2),
+      y: Math.round(rect.y + rect.height / 2)
+    });
+  });
+
+  // Assign sequential IDs
+  els.forEach((el, i) => { el.id = i + 1; });
+
+  // Get visible text, clean up whitespace
+  let bodyText = document.body.innerText || '';
+  bodyText = bodyText.replace(/\\n{3,}/g, '\\n\\n').trim().slice(0, 8000);
+
+  return {
+    title: document.title,
+    url: location.href,
+    text: bodyText,
+    elements: els
+  };
+})()`;
+
+export class BrowserController {
+  private cdp: CDPClient;
+  private port: number;
+  private profileDir: string | undefined;
+  private _connected = false;
+  private runningBrowser: RunningBrowser | null = null;
+  // Coordinates stored from last snapshot — not sent to LLM
+  private elementCoords = new Map<number, { x: number; y: number }>();
+
+  constructor(port: number = 9222, profileDir?: string) {
+    this.cdp = new CDPClient();
+    this.port = port;
+    this.profileDir = profileDir;
+  }
+
+  /**
+   * Check if Chrome CDP is already reachable on the debug port.
+   */
+  async isAvailable(): Promise<boolean> {
+    try {
+      const res = await fetch(`http://127.0.0.1:${this.port}/json/version`, {
+        signal: AbortSignal.timeout(2000),
+      });
+      return res.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Connect to Chrome. If Chrome isn't running, auto-launches it
+   * with CDP enabled and an isolated profile. No user setup required.
+   */
+  async connect(): Promise<void> {
+    if (this._connected) return;
+
+    // If Chrome isn't running, launch it automatically
+    if (!(await this.isAvailable())) {
+      console.log('[BrowserController] Chrome not detected, launching automatically...');
+      this.runningBrowser = await launchChrome(this.port, this.profileDir);
+    }
+
+    // Discover page targets
+    const listRes = await fetch(`http://127.0.0.1:${this.port}/json/list`);
+    if (!listRes.ok) {
+      throw new Error('Chrome CDP not reachable after launch');
+    }
+
+    const targets = await listRes.json() as Array<{
+      type: string;
+      webSocketDebuggerUrl: string;
+    }>;
+
+    let pageTarget = targets.find(t => t.type === 'page');
+
+    if (!pageTarget) {
+      // Create a new tab
+      const newRes = await fetch(`http://127.0.0.1:${this.port}/json/new?about:blank`);
+      pageTarget = await newRes.json() as any;
+    }
+
+    if (!pageTarget?.webSocketDebuggerUrl) {
+      throw new Error('No page target found and could not create one');
+    }
+
+    // Connect CDP to the page
+    await this.cdp.connect(pageTarget.webSocketDebuggerUrl);
+
+    // Enable required CDP domains
+    await this.cdp.send('Page.enable');
+    await this.cdp.send('Runtime.enable');
+    await this.cdp.send('DOM.enable');
+
+    // Inject stealth scripts for all future navigations
+    await this.cdp.send('Page.addScriptToEvaluateOnNewDocument', {
+      source: STEALTH_SCRIPT,
+    });
+
+    this._connected = true;
+    console.log('[BrowserController] Connected to Chrome');
+  }
+
+  /**
+   * Navigate to a URL and wait for the page to load.
+   */
+  async navigate(url: string): Promise<PageSnapshot> {
+    await this.ensureConnected();
+
+    const loadPromise = this.cdp.waitForEvent('Page.loadEventFired', 30000);
+
+    try {
+      await this.cdp.send('Page.navigate', { url });
+    } catch (err) {
+      // If navigate fails, suppress the dangling loadPromise timeout
+      loadPromise.catch(() => {});
+      throw err;
+    }
+
+    try {
+      await loadPromise;
+    } catch {
+      // Page.loadEventFired timeout — page may still be usable (SPAs, slow loads)
+      console.warn(`[BrowserController] Page load timeout for ${url}, continuing anyway`);
+    }
+
+    // Wait for JS to settle
+    await Bun.sleep(800);
+
+    return this.snapshot();
+  }
+
+  /**
+   * Get a snapshot of the current page: text content + numbered interactive elements.
+   */
+  async snapshot(): Promise<PageSnapshot> {
+    await this.ensureConnected();
+
+    const result = await this.cdp.send('Runtime.evaluate', {
+      expression: SNAPSHOT_SCRIPT,
+      returnByValue: true,
+      awaitPromise: true,
+    });
+
+    if (result.exceptionDetails) {
+      throw new Error(`Snapshot failed: ${JSON.stringify(result.exceptionDetails)}`);
+    }
+
+    const data = result.result.value as PageSnapshot & {
+      elements: Array<PageElement & { x: number; y: number }>;
+    };
+
+    // Store coordinates locally, strip from LLM-facing data
+    this.elementCoords.clear();
+    const cleanElements: PageElement[] = [];
+
+    for (const el of data.elements) {
+      this.elementCoords.set(el.id, { x: el.x, y: el.y });
+      cleanElements.push({
+        id: el.id,
+        tag: el.tag,
+        text: el.text,
+        attrs: el.attrs,
+      });
+    }
+
+    return {
+      title: data.title,
+      url: data.url,
+      text: data.text,
+      elements: cleanElements,
+    };
+  }
+
+  /**
+   * Click an element by its snapshot ID.
+   */
+  async click(elementId: number): Promise<string> {
+    await this.ensureConnected();
+
+    const coords = this.elementCoords.get(elementId);
+    if (!coords) {
+      return `Error: Element [${elementId}] not found. Run browser_snapshot first.`;
+    }
+
+    await this.cdp.send('Input.dispatchMouseEvent', {
+      type: 'mousePressed',
+      x: coords.x,
+      y: coords.y,
+      button: 'left',
+      clickCount: 1,
+    });
+    await this.cdp.send('Input.dispatchMouseEvent', {
+      type: 'mouseReleased',
+      x: coords.x,
+      y: coords.y,
+      button: 'left',
+      clickCount: 1,
+    });
+
+    // Wait for navigation/changes
+    await Bun.sleep(1000);
+
+    return `Clicked element [${elementId}]`;
+  }
+
+  /**
+   * Type text into an input element by its snapshot ID.
+   * Optionally press Enter after typing.
+   */
+  async type(elementId: number, text: string, submit: boolean = false): Promise<string> {
+    await this.ensureConnected();
+
+    // Click to focus the element
+    const clickResult = await this.click(elementId);
+    if (clickResult.startsWith('Error:')) return clickResult;
+
+    await Bun.sleep(200);
+
+    // Clear existing content
+    await this.cdp.send('Input.dispatchKeyEvent', {
+      type: 'keyDown',
+      key: 'a',
+      code: 'KeyA',
+      windowsVirtualKeyCode: 65,
+      nativeVirtualKeyCode: 65,
+      modifiers: 2, // Ctrl
+    });
+    await this.cdp.send('Input.dispatchKeyEvent', {
+      type: 'keyUp',
+      key: 'a',
+      code: 'KeyA',
+      windowsVirtualKeyCode: 65,
+      nativeVirtualKeyCode: 65,
+      modifiers: 2,
+    });
+
+    // Insert text (like paste — much more reliable than char-by-char)
+    await this.cdp.send('Input.insertText', { text });
+
+    let result = `Typed "${text}" into element [${elementId}]`;
+
+    if (submit) {
+      await Bun.sleep(100);
+      await this.pressEnter();
+      // Wait for page load after submit
+      await Bun.sleep(2000);
+      result += ' and pressed Enter';
+    }
+
+    return result;
+  }
+
+  /**
+   * Press Enter key.
+   */
+  async pressEnter(): Promise<void> {
+    await this.cdp.send('Input.dispatchKeyEvent', {
+      type: 'rawKeyDown',
+      key: 'Enter',
+      code: 'Enter',
+      windowsVirtualKeyCode: 13,
+      nativeVirtualKeyCode: 13,
+    });
+    await this.cdp.send('Input.dispatchKeyEvent', {
+      type: 'char',
+      key: 'Enter',
+      code: 'Enter',
+      windowsVirtualKeyCode: 13,
+      nativeVirtualKeyCode: 13,
+    });
+    await this.cdp.send('Input.dispatchKeyEvent', {
+      type: 'keyUp',
+      key: 'Enter',
+      code: 'Enter',
+      windowsVirtualKeyCode: 13,
+      nativeVirtualKeyCode: 13,
+    });
+  }
+
+  /**
+   * Scroll the page up or down.
+   * direction: 'down' or 'up'
+   * amount: pixels to scroll (default: one viewport height)
+   */
+  async scroll(direction: 'up' | 'down' = 'down', amount?: number): Promise<string> {
+    await this.ensureConnected();
+
+    const viewportHeight = (await this.evaluate('window.innerHeight') as number) || 600;
+    const scrollAmount = amount ?? viewportHeight;
+
+    const pixels = direction === 'down' ? scrollAmount : -scrollAmount;
+
+    await this.evaluate(`window.scrollBy(0, ${pixels})`);
+    await Bun.sleep(500); // Wait for lazy-loaded content
+
+    return `Scrolled ${direction} by ${scrollAmount}px`;
+  }
+
+  /**
+   * Take a screenshot and save to a file.
+   */
+  async screenshot(filePath: string = '/tmp/jarvis-screenshot.png'): Promise<string> {
+    await this.ensureConnected();
+
+    const result = await this.cdp.send('Page.captureScreenshot', { format: 'png' });
+    const buffer = Buffer.from(result.data, 'base64');
+
+    await Bun.write(filePath, buffer);
+    return filePath;
+  }
+
+  /**
+   * Take a screenshot and return raw base64 data (for vision/LLM).
+   */
+  async screenshotBuffer(): Promise<{ base64: string; mimeType: string }> {
+    await this.ensureConnected();
+    const result = await this.cdp.send('Page.captureScreenshot', { format: 'png' });
+    return { base64: result.data, mimeType: 'image/png' };
+  }
+
+  /**
+   * Evaluate arbitrary JavaScript in the page context.
+   */
+  async evaluate(expression: string): Promise<unknown> {
+    await this.ensureConnected();
+
+    const result = await this.cdp.send('Runtime.evaluate', {
+      expression,
+      returnByValue: true,
+      awaitPromise: true,
+    });
+
+    if (result.exceptionDetails) {
+      throw new Error(`JS error: ${JSON.stringify(result.exceptionDetails)}`);
+    }
+
+    return result.result.value;
+  }
+
+  /**
+   * Disconnect from Chrome. If we auto-launched Chrome, stop it too.
+   */
+  async disconnect(): Promise<void> {
+    if (this._connected) {
+      await this.cdp.close();
+      this._connected = false;
+      this.elementCoords.clear();
+      console.log('[BrowserController] Disconnected');
+    }
+
+    // Stop the Chrome process we launched (if any)
+    if (this.runningBrowser) {
+      await stopChrome(this.runningBrowser);
+      this.runningBrowser = null;
+    }
+  }
+
+  get connected(): boolean {
+    return this._connected;
+  }
+
+  private async ensureConnected(): Promise<void> {
+    if (this._connected && !this.cdp.isOpen) {
+      // Connection went stale — reset and reconnect
+      console.warn('[BrowserController] CDP connection stale, reconnecting...');
+      this._connected = false;
+      this.elementCoords.clear();
+    }
+
+    if (!this._connected) {
+      await this.connect();
+    }
+  }
+}

package/src/actions/browser/stealth.ts

@@ -0,0 +1,49 @@
+/**
+ * Browser Stealth — Anti-detection scripts
+ *
+ * Injected into every new document via Page.addScriptToEvaluateOnNewDocument
+ * to hide automation fingerprints.
+ */
+
+export const STEALTH_SCRIPT = `
+  // Hide webdriver flag
+  Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+
+  // Fake plugins array (real browsers have plugins)
+  Object.defineProperty(navigator, 'plugins', {
+    get: () => {
+      const plugins = [
+        { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer' },
+        { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai' },
+        { name: 'Native Client', filename: 'internal-nacl-plugin' },
+      ];
+      plugins.length = 3;
+      return plugins;
+    }
+  });
+
+  // Fake languages
+  Object.defineProperty(navigator, 'languages', {
+    get: () => ['en-US', 'en']
+  });
+
+  // Remove automation-related properties from window
+  delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
+  delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
+  delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
+
+  // Fix chrome.runtime to look like a real browser
+  if (!window.chrome) window.chrome = {};
+  if (!window.chrome.runtime) window.chrome.runtime = {};
+
+  // Fix permissions query
+  const originalQuery = window.navigator.permissions?.query;
+  if (originalQuery) {
+    window.navigator.permissions.query = (parameters) => {
+      if (parameters.name === 'notifications') {
+        return Promise.resolve({ state: Notification.permission });
+      }
+      return originalQuery(parameters);
+    };
+  }
+`;

package/src/actions/index.ts

@@ -0,0 +1,20 @@
+// App Control exports
+export { getAppController } from './app-control/interface.ts';
+export type { AppController, WindowInfo, UIElement } from './app-control/interface.ts';
+export { LinuxAppController } from './app-control/linux.ts';
+export { WindowsAppController } from './app-control/windows.ts';
+export { MacAppController } from './app-control/macos.ts';
+
+// Browser exports
+export { CDPClient as CDPBrowser } from './browser/cdp.ts';
+export type { PageElement as BrowserTab } from './browser/session.ts';
+export { BrowserController as BrowserSession } from './browser/session.ts';
+
+// Terminal exports
+export { TerminalExecutor } from './terminal/executor.ts';
+export type { CommandResult, ExecuteOptions } from './terminal/executor.ts';
+export { WSLBridge } from './terminal/wsl-bridge.ts';
+
+// Tools exports
+export { ToolRegistry } from './tools/registry.ts';
+export type { ToolDefinition, ToolParameter } from './tools/registry.ts';

package/src/actions/terminal/executor.ts

@@ -0,0 +1,157 @@
+import { spawn, type Subprocess } from 'bun';
+
+export type CommandResult = {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+  duration: number;
+};
+
+export type ExecuteOptions = {
+  cwd?: string;
+  env?: Record<string, string>;
+  timeout?: number;
+};
+
+export class TerminalExecutor {
+  private shell: string;
+  private defaultTimeout: number;
+
+  constructor(opts?: { shell?: string; timeout?: number }) {
+    this.shell = opts?.shell ?? TerminalExecutor.detectShell();
+    this.defaultTimeout = opts?.timeout ?? 30000;
+  }
+
+  async execute(command: string, opts?: ExecuteOptions): Promise<CommandResult> {
+    const startTime = Date.now();
+    const timeout = opts?.timeout ?? this.defaultTimeout;
+
+    try {
+      const proc = spawn({
+        cmd: [this.shell, '-c', command],
+        cwd: opts?.cwd,
+        env: { ...process.env, ...opts?.env },
+        stdout: 'pipe',
+        stderr: 'pipe',
+      });
+
+      const timeoutId = setTimeout(() => {
+        proc.kill();
+      }, timeout);
+
+      const [stdout, stderr, exitCode] = await Promise.all([
+        this.readStream(proc.stdout),
+        this.readStream(proc.stderr),
+        proc.exited,
+      ]);
+
+      clearTimeout(timeoutId);
+
+      const duration = Date.now() - startTime;
+
+      return {
+        stdout: stdout.toString('utf-8'),
+        stderr: stderr.toString('utf-8'),
+        exitCode: exitCode ?? 0,
+        duration,
+      };
+    } catch (error) {
+      const duration = Date.now() - startTime;
+
+      if (duration >= timeout) {
+        throw new Error(`Command timed out after ${timeout}ms: ${command}`);
+      }
+
+      throw new Error(`Command execution failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  async *stream(command: string, opts?: { cwd?: string; env?: Record<string, string> }): AsyncIterable<string> {
+    const proc = spawn({
+      cmd: [this.shell, '-c', command],
+      cwd: opts?.cwd,
+      env: { ...process.env, ...opts?.env },
+      stdout: 'pipe',
+      stderr: 'pipe',
+    });
+
+    const decoder = new TextDecoder();
+
+    if (!proc.stdout) {
+      throw new Error('Failed to get stdout stream');
+    }
+
+    const reader = proc.stdout.getReader();
+
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+
+        if (done) {
+          break;
+        }
+
+        if (value) {
+          yield decoder.decode(value, { stream: true });
+        }
+      }
+    } finally {
+      reader.releaseLock();
+      proc.kill();
+    }
+  }
+
+  static detectShell(): string {
+    const platform = process.platform;
+
+    if (platform === 'win32') {
+      return process.env.COMSPEC ?? 'powershell.exe';
+    }
+
+    if (process.env.SHELL) {
+      return process.env.SHELL;
+    }
+
+    return '/bin/bash';
+  }
+
+  getShell(): string {
+    return this.shell;
+  }
+
+  private async readStream(stream: ReadableStream<Uint8Array> | null): Promise<Buffer> {
+    if (!stream) {
+      return Buffer.from('');
+    }
+
+    const reader = stream.getReader();
+    const chunks: Uint8Array[] = [];
+
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+
+        if (done) {
+          break;
+        }
+
+        if (value) {
+          chunks.push(value);
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+
+    const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+    const result = new Uint8Array(totalLength);
+
+    let offset = 0;
+    for (const chunk of chunks) {
+      result.set(chunk, offset);
+      offset += chunk.length;
+    }
+
+    return Buffer.from(result);
+  }
+}