@glittering0717/ai-browser 0.0.0-stage → 1.2.0

package/src/main/mcp_tools.js

@@ -0,0 +1,289 @@
+// main/mcp_tools.js — the curated MCP tool list.
+// Exported separately so tests can assert on it without triggering the
+// mcp_server's main() side effects (spawning Electron / opening stdio).
+// Tool count and descriptions are kept minimal to reduce per-message token
+// cost in MCP clients that inject the full tool list on every turn.
+//
+// annotations 遵循 MCP Tool annotation 约定（title / readOnlyHint /
+// destructiveHint / openWorldHint），Smithery / Cursor / Claude 等目录与客户端
+// 会读取并渲染成徽标。只放布尔结论、不放长句，把额外 token 压到最低。
+export const MCP_TOOLS = [
+  {
+    name: 'browse_navigate',
+    title: 'Navigate tab',
+    description: 'Navigate the active/focused tab to a URL.',
+    annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        url: { type: 'string', description: 'URL to navigate to.' },
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+      },
+      required: ['url']
+    }
+  },
+  {
+    name: 'browse_get_tree',
+    title: 'Read semantic tree',
+    description: 'Return the page semantic UI tree (roles, labels, actions, bounds, urls).',
+    annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        focused_only: { type: 'boolean', description: 'Return only the focused element subtree.', default: false },
+        ax: { type: 'boolean', description: 'Use the AX read layer (links carry url).', default: false },
+        subset: { type: 'string', enum: ['interactive', 'full'], description: 'Prune layout-only branches (token saver).' },
+        mode: { type: 'string', enum: ['diff', 'full'], description: 'Return only new/hidden interactive nodes since last read.' },
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+      }
+    }
+  },
+  {
+    name: 'browse_act',
+    title: 'Act on element',
+    description: 'Act on an element by data-ai-id: click, type (append at caret), setContent (replace all), clear, focus, hover, scroll_to, upload, get_value (read-only), drag, press, wheel.',
+    annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        action: { type: 'string', enum: ['click', 'type', 'clear', 'focus', 'hover', 'scroll_to', 'upload', 'get_value', 'drag', 'press', 'wheel'], description: 'Action.' },
+        target: { type: 'string', description: 'Element data-ai-id from the semantic tree. Omit for press (keys go to the current focus); for drag you may also omit it and pass from_x/from_y to start from a bare point.' },
+        text: { type: 'string', description: 'Text to type (type only).' },
+        value: { type: 'string', description: 'Value for select.' },
+        offset: { type: 'integer', description: 'get_value: start offset in Unicode code points (default 0).' },
+        limit: { type: 'integer', description: 'get_value: max code points to return (default 20000, hard cap 1000000).' },
+        url: { type: 'string', description: 'For click: click the link whose href matches this URL instead of guessing the target label.' },
+        keep_tab: { type: 'boolean', description: 'For click: keep the current tab active instead of auto-following a newly opened tab.' },
+        file: { type: 'string', description: 'Absolute path to upload (upload only, target must be a file input).' },
+        timeout_ms: { type: 'integer', description: 'Override the action deadline in ms (default 15000; type/setContent/drag/press scale with text length / duration / repeat; 1s floor, 300s cap). A timeout returns success=false, timed_out=true and warns the action may have partially applied.' },
+        to_target: { type: 'string', description: 'drag: data-ai-id to drop onto (e.g. drag from one node\'s output to another node to connect them).' },
+        dx: { type: 'number', description: 'drag: horizontal distance to move (use with dy instead of to_target).' },
+        dy: { type: 'number', description: 'drag: vertical distance to move.' },
+        to_x: { type: 'number', description: 'drag: absolute viewport x to end at (overrides dx).' },
+        to_y: { type: 'number', description: 'drag: absolute viewport y to end at.' },
+        from_x: { type: 'number', description: 'drag: explicit viewport x to start the drag at (no target needed — e.g. panning empty canvas).' },
+        from_y: { type: 'number', description: 'drag: explicit viewport y to start the drag at.' },
+        from_anchor: { type: 'string', enum: ['center', 'left', 'right', 'top', 'bottom'], description: 'drag: which point of target to grab; a side prefers that node\'s connection handle (default center).' },
+        to_anchor: { type: 'string', enum: ['center', 'left', 'right', 'top', 'bottom'], description: 'drag: which point of to_target to drop on.' },
+        hold: { type: 'array', items: { type: 'string' }, description: 'drag: modifier keys held during the drag (e.g. ["Shift"]).' },
+        button: { type: 'string', enum: ['left', 'middle', 'right'], description: 'drag: mouse button (default left). Canvas apps often pan on middle/right drag.' },
+        key: { type: 'string', description: 'press: a key name such as Delete, Escape, ArrowRight, Enter, or a combo like "Meta+a".' },
+        delta_x: { type: 'number', description: 'wheel: horizontal scroll amount (positive scrolls right).' },
+        delta_y: { type: 'number', description: 'wheel: vertical scroll amount. Negative scrolls up / zooms out; add hold:["Control"] for canvas zoom.' },
+        keys: { type: 'array', items: { type: 'string' }, description: 'press: sequence of keys to press in order.' },
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+      },
+      required: ['action']
+    }
+  },
+  {
+    name: 'browse_evaluate',
+    title: 'Run page JavaScript',
+    description: 'Run a JS expression in the page context and return its value.',
+    annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        js: { type: 'string', description: 'Page-context JS expression (not statements). Max 5000 chars.' },
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+      },
+      required: ['js']
+    }
+  },
+  {
+    name: 'browse_scroll',
+    title: 'Scroll page',
+    description: 'Scroll the page or an element into view.',
+    annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        direction: { type: 'string', enum: ['up', 'down'], default: 'down' },
+        amount: { type: 'integer', default: 500, description: 'Pixels (ignored if target set).' },
+        target: { type: 'string', description: 'data-ai-id to scroll into view.' },
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+      }
+    }
+  },
+  {
+    name: 'browse_wait',
+    title: 'Wait for condition',
+    description: 'Poll the page until a condition holds or timeout. Conditions: button_enabled, modal_appeared, text_contains, url_contains.',
+    annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        condition: { type: 'string', enum: ['button_enabled', 'modal_appeared', 'text_contains', 'url_contains'] },
+        target: { type: 'string', description: 'Button label (button_enabled).' },
+        text: { type: 'string', description: 'Text/URL fragment (text_contains / url_contains).' },
+        timeout_ms: { type: 'integer', default: 10000 },
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+      },
+      required: ['condition']
+    }
+  },
+  {
+    name: 'browse_list_tabs',
+    title: 'List tabs',
+    description: 'List open tabs (id, url, title, active).',
+    annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },
+    inputSchema: { type: 'object', properties: {} }
+  },
+  {
+    name: 'browse_new_tab',
+    title: 'New tab',
+    description: 'Open a new tab, optionally at a URL.',
+    annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        url: { type: 'string', description: 'Optional URL.' }
+      }
+    }
+  },
+  {
+    name: 'browse_close_tab',
+    title: 'Close tab',
+    description: 'Close a tab by id.',
+    annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        tab: { type: 'integer', description: 'Tab id.' }
+      },
+      required: ['tab']
+    }
+  },
+  {
+    name: 'browse_set_active_tab',
+    title: 'Switch active tab',
+    description: 'Switch the active tab.',
+    annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        tab: { type: 'integer', description: 'Tab id.' }
+      },
+      required: ['tab']
+    }
+  },
+  {
+    name: 'browse_network',
+    title: 'Inspect network',
+    description: 'Query captured HTTP requests (Chrome Network panel): list/get/clear/configure. Captures method, headers and POST bodies; sensitive headers redacted by default.',
+    annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        operation: { type: 'string', enum: ['list', 'get', 'clear', 'configure'], description: 'Operation.' },
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' },
+        network_id: { type: 'string', description: 'get: id from list.' },
+        method: { type: 'array', items: { type: 'string' }, description: 'list: filter by HTTP method.' },
+        url_contains: { type: 'string', description: 'list: URL substring filter.' },
+        status_min: { type: 'integer', description: 'list: minimum status.' },
+        status_max: { type: 'integer', description: 'list: maximum status.' },
+        resource_type: { type: 'array', items: { type: 'string' }, description: 'list: XHR/Fetch/Document/...' },
+        state: { type: 'array', items: { type: 'string' }, description: 'list: pending/finished/failed.' },
+        started_after: { type: 'string', description: 'list: ISO timestamp.' },
+        started_before: { type: 'string', description: 'list: ISO timestamp.' },
+        limit: { type: 'integer', description: 'list: page size (default 50, max 200).' },
+        before_seq: { type: 'integer', description: 'list: cursor from pagination.next_before_seq.' },
+        include_request_headers: { type: 'boolean' },
+        include_request_body: { type: 'boolean' },
+        include_response_headers: { type: 'boolean' },
+        include_response_body: { type: 'boolean' },
+        include_sensitive_headers: { type: 'boolean', description: 'Requires AI_BROWSER_NETWORK_SENSITIVE=1.' },
+        request_body_offset: { type: 'integer' },
+        response_body_offset: { type: 'integer' },
+        body_limit: { type: 'integer', description: 'Body page size (default 65536, max 262144).' },
+        enabled: { type: 'boolean', description: 'configure: turn capture on/off.' },
+        capture_bodies: { type: 'string', enum: ['none', 'api', 'all'], description: 'configure.' }
+      },
+      required: ['operation']
+    }
+  },
+  {
+    name: 'browse_canvas',
+    title: 'Read canvas',
+    description: 'Read <canvas> through captured draw calls (not pixels): list canvases, read texts/regions, configure, or capture a PNG fallback. 2D canvas text comes from fillText, so no OCR needed; WebGL is opaque.',
+    annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        operation: { type: 'string', enum: ['list', 'read', 'configure', 'capture'], description: 'Operation.' },
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' },
+        canvas_id: { type: 'string', description: 'read/capture: id from list.' },
+        view: { type: 'string', enum: ['summary', 'calls'], description: 'read: summary (default) or raw draw calls.' },
+        since_seq: { type: 'integer', description: 'read: cursor from pagination.next_seq.' },
+        limit: { type: 'integer', description: 'read: page size (default 200, max 1000).' },
+        mode: { type: 'string', enum: ['off', 'semantic', 'trace'], description: 'configure: recording mode.' },
+        clear: { type: 'boolean', description: 'configure: drop retained calls.' }
+      },
+      required: ['operation']
+    }
+  },
+  {
+    name: 'browse_diff',
+    title: 'Snapshot and diff',
+    description: 'Prove an action changed the page: snapshot a baseline, then diff to get added/removed/changed paths with before/after values. Success is not evidence.',
+    annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        operation: { type: 'string', enum: ['snapshot', 'diff'], description: 'Operation.' },
+        js: { type: 'string', description: 'JSON-serializable expression to capture. Omit for a site-agnostic DOM summary (role/label/value/disabled/size per element).' },
+        label: { type: 'string', description: 'snapshot: a note for your own bookkeeping.' },
+        snapshot: { type: 'string', description: 'diff: the id returned by snapshot.' },
+        rearm: { type: 'boolean', description: 'diff: make the current state the new baseline, so you can loop "assert no change" over many steps.' },
+        forget: { type: 'boolean', description: 'diff: drop the stored baseline when done.' },
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+      },
+      required: ['operation']
+    }
+  },
+  {
+    name: 'browse_capabilities',
+    title: 'Probe capabilities',
+    description: 'Probe what a page affords and which route to take: DOM vs canvas, same-origin iframes, app objects on window, JSON endpoints, plus a recommended plan with evidence.',
+    annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        include_network: { type: 'boolean', description: 'Include observed JSON-ish endpoints (default true).' },
+        network_limit: { type: 'integer', description: 'How many captured requests to scan.' },
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+      }
+    }
+  },
+  {
+    name: 'browse_repaint',
+    title: 'Force repaint',
+    description: "Force the compositor to redraw a tab's BrowserView when the window shows a stale page; returns repaint_count so the repaint is provable.",
+    annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+      }
+    }
+  },
+  {
+    name: 'browse_downloads',
+    title: 'List downloads',
+    description: 'List downloads for a tab with their on-disk paths. Use after clicking an export/download button — downloads are asynchronous, so poll this rather than expecting the path in the click response.',
+    annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        tab: { type: 'integer', description: 'Tab ID (defaults to the active tab).' },
+      },
+    },
+  },
+  {
+    name: 'browse_quit',
+    title: 'Quit browser',
+    description: 'Gracefully shut down the AI Browser (releases resources).',
+    annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },
+    inputSchema: { type: 'object', properties: {} }
+  }
+];

package/src/main/mcp_ws.js

@@ -0,0 +1,178 @@
+// main/mcp_ws.js — Persistent WebSocket JSON-RPC client used by the MCP server.
+// One WebSocket is opened lazily and reused for every RPC; requests carry
+// incrementing ids routed through an id -> resolver map, so concurrent calls
+// on the same connection resolve to the correct response. Server-side event
+// notifications (broadcast with no jsonrpc id) are dropped: the MCP surface is
+// request/response only (server→client notifications are dropped here, which is
+// why live-push tools are not exposed over MCP — see below)
+// (enables CDP monitors + network-body cache; does not stream live events).
+//
+// #5: robust auto-reconnect — when the connection drops (Electron relaunch /
+// NetworkService wedge), `getWs` retries with exponential backoff until the
+// server is reachable again (reconnectTimeout budget) instead of failing the
+// first call after a drop. Calls wait on the connection first, so the RPC
+// per-call `timeout` only starts after the socket is OPEN (a relaunch window is
+// absorbed by backoff, not counted as a hung call).
+//
+// MW-4 race: a stale socket's deferred close/error events must never clobber the
+// refs of a freshly re-opened connection. Two guards: (a) a socket isn't
+// adopted as `_ws` until its 'open' fires, and (b) close/error handlers mutate
+// state only via detachIfCurrent(ws), which no-ops if `_ws` already points at a
+// different (newer) socket.
+import WebSocket from 'ws';
+
+export function createWsClient({
+  url,
+  timeout = 15000,
+  reconnectTimeout = 120000, // total budget to keep retrying after a drop
+  retryBase = 500,           // initial backoff; doubles per attempt, capped
+  maxRetryDelay = 10000,
+  ensureRunning = null,      // Lazy launch hook: async; called on connection
+                             // failure so the server can spawn Electron on demand.
+} = {}) {
+  let _ws = null;        // current OPEN socket (or null)
+  let _connecting = null; // in-flight getWs() promise (dedupes concurrent callers)
+  let _rpcId = 0;
+  const _pending = new Map();
+
+  const wait = (ms) => new Promise((r) => setTimeout(r, ms));
+
+  function flushPending(err) {
+    for (const [, cb] of _pending) cb({ error: { message: err.message } });
+    _pending.clear();
+  }
+
+  // Clear out our ref to `ws` ONLY if it is still the current socket. Prevents a
+  // stale handler from nulling a newer connection (MW-4).
+  function detachIfCurrent(ws) {
+    if (_ws === ws) _ws = null;
+  }
+
+  // Persistent handlers for an OPEN socket: route messages, drop the socket on
+  // error/close (guarded) and reject any in-flight calls so they fail fast while
+  // the NEXT call reconnects.
+  function attachPersistent(ws) {
+    ws.on('message', (data) => {
+      let msg;
+      try { msg = JSON.parse(data.toString()); } catch { return; }
+      const cb = msg.id !== undefined ? _pending.get(msg.id) : undefined;
+      if (cb) { _pending.delete(msg.id); cb(msg); }
+      // Notifications (no id) are dropped — see the createWsClient comment.
+    });
+    ws.on('error', (e) => {
+      detachIfCurrent(ws);
+      flushPending(new Error('WebSocket error: ' + e.message));
+    });
+    ws.on('close', () => {
+      detachIfCurrent(ws);
+      flushPending(new Error('WebSocket closed'));
+    });
+  }
+
+  function getWs() {
+    if (_ws && _ws.readyState === WebSocket.OPEN) return Promise.resolve(_ws);
+    if (_connecting) return _connecting;
+
+    _connecting = (async () => {
+      const deadline = Date.now() + reconnectTimeout;
+      let attempt = 0;
+      let launchErr = null;   // ensureRunning 的失败原因，超时那一刻报给调用方
+      while (true) {
+        // A concurrent open socket appearing (e.g. rude open racing the loop) wins.
+        if (_ws && _ws.readyState === WebSocket.OPEN) return _ws;
+
+        const ws = new WebSocket(url);
+        // Connect-phase: handlers only settle the local await; the socket is not
+        // yet adopted as _ws, so a failed attempt cannot corrupt live state.
+        let open = false;
+        let connectionErr = null;
+        await new Promise((res) => {
+          const settle = () => res();
+          const to = setTimeout(settle, 5000);
+          ws.once('open', () => { open = true; clearTimeout(to); settle(); });
+          ws.once('error', (e) => { connectionErr = e; clearTimeout(to); settle(); });
+          ws.once('close', () => { clearTimeout(to); settle(); });
+        });
+
+        if (open) {
+          _ws = ws;
+          attachPersistent(ws);
+          return ws;
+        }
+
+        // Attempt failed — release this socket's connect-phase listeners fully so
+        // its later error/close can't touch live state (MW-4).
+        try { ws.removeAllListeners(); ws.close(); } catch (_e) {}
+        const err = connectionErr || new Error('connection refused');
+        // eslint-disable-next-line no-console
+        if (attempt > 0 && attempt % 4 === 0) console.error('[mcp] ws reconnect attempt', attempt, '-', err.message);
+
+        // Lazy launch: the first failure triggers the injected runner (spawns
+        // Electron on demand). ensureElectronRunning is idempotent — no-op when
+        // the port is already listening — so calling it on every retry is safe and
+        // also re-launches after a manual kill of Electron.
+        //
+        // ⚠️ 它的失败**不能吞**。此前这里是 `catch (_e) {}`，于是"浏览器根本没起来"
+        // 和"浏览器起来但 WS 还没好"表现完全一样：agent 先重试 120 秒，最后只拿到
+        // "WS reconnect timeout: connection refused"，没有任何线索。现在把原因累积
+        // 起来，超时那一刻一并报出。
+        if (typeof ensureRunning === 'function') {
+          try { await ensureRunning(); launchErr = null; }
+          catch (e) { launchErr = e; }
+        }
+
+        if (Date.now() >= deadline) {
+          const suffix = launchErr
+            ? `; browser launch failed: ${launchErr.message}`
+            : '';
+          throw new Error('WS reconnect timeout: ' + err.message + suffix);
+        }
+        await wait(Math.min(retryBase * 2 ** attempt++, maxRetryDelay));
+      }
+    })().finally(() => { _connecting = null; });
+
+    return _connecting;
+  }
+
+  // JSON-RPC call; resolves to the result, rejects on {error} / timeout /
+  // connection failure. Uses the shared connection via getWs().
+  function call(method, params = {}) {
+    return getWs().then((conn) => new Promise((resolve, reject) => {
+      const id = ++_rpcId;
+      const timer = setTimeout(() => {
+        if (_pending.has(id)) { _pending.delete(id); reject(new Error('WS timeout')); }
+      }, timeout);
+      _pending.set(id, (msg) => {
+        clearTimeout(timer);
+        if (msg.error) reject(new Error(msg.error.message || 'RPC error'));
+        else resolve(msg.result);
+      });
+      try {
+        conn.send(JSON.stringify({ jsonrpc: '2.0', id, method, params }));
+      } catch (e) {
+        clearTimeout(timer);
+        _pending.delete(id);
+        reject(e);
+      }
+    }));
+  }
+
+  // Force-sever the underlying connection. The NEXT call reconnects (MW-4).
+  // Detaches the socket's own handlers first so their deferred close/error
+  // events can't clobber the refs of a freshly re-opened connection created
+  // right after this returns (rapid reconnect race). In-flight calls are
+  // rejected.
+  function close() {
+    const ws = _ws;
+    _ws = null;
+    _connecting = null;
+    if (ws) {
+      ws.removeAllListeners('close');
+      ws.removeAllListeners('error');
+      try { ws.close(); } catch (_e) {}
+    }
+    flushPending(new Error('client closed'));
+  }
+
+  return { call, close };
+}