@glittering0717/ai-browser 0.0.0-stage → 1.2.0

package/src/preload/watcher.cjs

@@ -0,0 +1,241 @@
+// preload/watcher.cjs — Mutation/state/captcha/message/JS-error watchers (CommonJS)
+// Split out of bridge.js v6.9. No Electron dependency — side effects are
+// emitted through an injected `onEvent(eventType, data)` callback so the
+// module can be unit-tested under jsdom (bridge.js passes ipcRenderer.send).
+// Exports: startTracking(callbacks), stopTracking()
+
+// ============================================================
+// 4. CAPTCHA + MESSAGE WATCHERS
+// ============================================================
+
+var observer = null;
+var lastCaptchaSig = "";
+var lastMessageSig = "";
+var mutationQueue = [];
+var mutationFlushTimer = null;
+var scanTimer = null;
+var MUTATION_FLUSH_DELAY = 500;
+var SCAN_INTERVAL = 5000;
+
+function scanCaptcha() {
+  var byCSS = document.querySelectorAll("[class*=captcha],[class*=verify],[class*=slider],[class*=slide],[class*=drag],[id*=captcha],[id*=verify],[class*=JDJRV],[class*=geetest],[class*=yidun],[class*=small-jd]");
+  var byAlt = document.querySelectorAll("img[alt*=验证码],img[alt*=captcha],img[alt*=verify],img[alt*=滑块],img[alt*=slide]");
+  var all = [];
+  for (var bi = 0; bi < byCSS.length; bi++) all.push(byCSS[bi]);
+  for (var ai = 0; ai < byAlt.length; ai++) { if (all.indexOf(byAlt[ai]) < 0) all.push(byAlt[ai]); }
+
+  var visible = [];
+  for (var i = 0; i < all.length; i++) {
+    var r = all[i].getBoundingClientRect();
+    if (r.width >= 0 || r.height >= 0) {
+      visible.push({
+        tag: all[i].tagName,
+        cls: String(all[i].className || "").slice(0, 80),
+        id: all[i].id || "",
+        text: String(all[i].textContent || "").trim().slice(0, 50),
+        bounds: { x: Math.round(r.x), y: Math.round(r.y), width: Math.round(r.width), height: Math.round(r.height) }
+      });
+    }
+  }
+
+  // Also scan for canvases in captcha areas
+  var canvases = document.querySelectorAll("canvas");
+  for (var ci = 0; ci < canvases.length; ci++) {
+    var cr = canvases[ci].getBoundingClientRect();
+    if (cr.width > 10 && cr.height > 10) {
+      // Check if canvas is inside a captcha-like container
+      var parent = canvases[ci].parentElement;
+      var pCls = (parent && parent.className || "") + (parent && parent.id || "");
+      if (/captcha|verify|slider|slide|drag|yidun|geetest/i.test(pCls)) {
+        visible.push({
+          tag: "CANVAS",
+          cls: (canvases[ci].className || "").slice(0, 40),
+          id: canvases[ci].id || "",
+          text: "[canvas captcha]",
+          bounds: { x: Math.round(cr.x), y: Math.round(cr.y), width: Math.round(cr.width), height: Math.round(cr.height) },
+          parent_cls: pCls.slice(0, 60)
+        });
+      }
+    }
+  }
+
+  return visible;
+}
+
+function scanMessages() {
+  var selectors = [
+    "[class*=toast]", "[class*=message]", "[class*=notification]", "[class*=alert]",
+    "[class*=snackbar]", "[class*=notice]", "[class*=tip]",
+    "[role=alert]", "[role=status]", "[class*=banner]"
+  ];
+  var msgs = [];
+  for (var si = 0; si < selectors.length; si++) {
+    var els = document.querySelectorAll(selectors[si]);
+    for (var ei = 0; ei < els.length; ei++) {
+      if (els[ei].offsetParent === null && window.getComputedStyle(els[ei]).display === "none") continue;
+      var r = els[ei].getBoundingClientRect();
+      var t = (els[ei].textContent || "").trim();
+      if (t && t.length < 200) {
+        msgs.push({ text: t.slice(0, 100), bounds: { x: Math.round(r.x), y: Math.round(r.y), width: Math.round(r.width), height: Math.round(r.height) } });
+      }
+    }
+  }
+  return msgs;
+}
+
+function runScan(emit) {
+  scanTimer = null;
+  var cap = scanCaptcha();
+  if (cap.length > 0) {
+    var sig = JSON.stringify(cap.map(function(c) { return c.cls + "|" + c.bounds.width + "x" + c.bounds.height; }));
+    if (sig !== lastCaptchaSig) {
+      lastCaptchaSig = sig;
+      emit("captcha_appeared", { elements: cap });
+    }
+  }
+  var msg = scanMessages();
+  if (msg.length > 0) {
+    var msig = JSON.stringify(msg.map(function(m) { return m.text; }));
+    if (msig !== lastMessageSig) {
+      lastMessageSig = msig;
+      emit("message_appeared", { messages: msg });
+    }
+  }
+}
+
+function flushMutations(emit) {
+  mutationFlushTimer = null;
+  var mutations = mutationQueue;
+  mutationQueue = [];
+  if (mutations.length === 0) return;
+
+  var stateChanges = [];
+
+  for (var mi = 0; mi < mutations.length; mi++) {
+    var m = mutations[mi];
+
+    if (m.type === "attributes") {
+      var attrName = (m.attributeName || "").toLowerCase();
+      var trackedAttrs = ["disabled", "class", "open", "checked", "aria-expanded", "aria-selected", "hidden", "value", "readonly"];
+      if (trackedAttrs.indexOf(attrName) >= 0) {
+        var el = m.target;
+        if (el instanceof HTMLElement) {
+          var aiId = el.getAttribute("data-ai-id");
+          if (aiId) {
+            stateChanges.push({
+              type: "state_changed",
+              targetId: aiId,
+              attribute: attrName,
+              value: el.getAttribute(attrName)
+            });
+          }
+        }
+      }
+    }
+
+    if (m.type === "childList") {
+      var parentId = m.target instanceof HTMLElement ? m.target.getAttribute("data-ai-id") : null;
+      if (m.addedNodes.length > 0 || m.removedNodes.length > 0) {
+        stateChanges.push({
+          type: "dom_changed",
+          parentId: parentId,
+          added: m.addedNodes.length,
+          removed: m.removedNodes.length
+        });
+      }
+    }
+  }
+
+  if (stateChanges.length > 0) {
+    var batch = [];
+    for (var si = 0; si < Math.min(stateChanges.length, 20); si++) batch.push(stateChanges[si]);
+    emit("state_changed", { changes: batch });
+  }
+}
+
+function bindObserver(emit) {
+  if (observer) observer.disconnect();
+  mutationQueue = [];
+  if (mutationFlushTimer) {
+    clearTimeout(mutationFlushTimer);
+    mutationFlushTimer = null;
+  }
+  if (scanTimer) {
+    clearInterval(scanTimer);
+    scanTimer = null;
+  }
+
+  observer = new MutationObserver(function(mutations) {
+    mutationQueue.push.apply(mutationQueue, mutations);
+    if (!mutationFlushTimer) {
+      mutationFlushTimer = setTimeout(function() { flushMutations(emit); }, MUTATION_FLUSH_DELAY);
+    }
+  });
+
+  observer.observe(document.body, {
+    childList: true,
+    subtree: true,
+    attributes: true,
+    attributeFilter: ["disabled", "class", "open", "checked", "aria-expanded", "aria-selected", "hidden", "value", "readonly"]
+  });
+
+  // Captcha/message scan runs on its own low-frequency timer to avoid
+  // blocking the renderer with querySelectorAll during page load.
+  scanTimer = setInterval(function() { runScan(emit); }, SCAN_INTERVAL);
+  setTimeout(function() { runScan(emit); }, 1000);
+}
+
+// ============================================================
+// JS error capture
+// ============================================================
+
+var oldOnError = null;
+
+function hookJsErrors(emit) {
+  oldOnError = window.onerror;
+  window.onerror = function(msg, url, line, col, err) {
+    emit("js_error", { message: String(msg), url: String(url), line: line, col: col, stack: (err && err.stack ? String(err.stack).slice(0, 300) : "") });
+    if (oldOnError) return oldOnError.apply(this, arguments);
+    return false;
+  };
+
+  window.addEventListener("unhandledrejection", function(ev) {
+    var reason = ev.reason;
+    var msg = reason instanceof Error ? reason.message : String(reason);
+    emit("js_error", { message: msg, type: "unhandledrejection", stack: (reason instanceof Error && reason.stack ? String(reason.stack).slice(0, 300) : "") });
+  });
+}
+
+// ============================================================
+// Public API
+// ============================================================
+
+function startTracking(callbacks) {
+  callbacks = callbacks || {};
+  function nullEmit() {}
+  var emit = typeof callbacks.onEvent === "function" ? callbacks.onEvent : nullEmit;
+  bindObserver(emit);
+  hookJsErrors(emit);
+}
+
+function stopTracking() {
+  if (observer) {
+    observer.disconnect();
+    observer = null;
+  }
+  mutationQueue = [];
+  if (mutationFlushTimer) {
+    clearTimeout(mutationFlushTimer);
+    mutationFlushTimer = null;
+  }
+  if (scanTimer) {
+    clearInterval(scanTimer);
+    scanTimer = null;
+  }
+  if (oldOnError !== null) {
+    window.onerror = oldOnError;
+    oldOnError = null;
+  }
+}
+
+module.exports = { startTracking, stopTracking };

package/src/renderer/tab_bar.html

@@ -0,0 +1,115 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<style>
+  * { margin: 0; padding: 0; box-sizing: border-box; }
+  body {
+    height: 36px;
+    background: #1a1a1a;
+    display: flex;
+    align-items: center;
+    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
+    font-size: 12px;
+    color: #ccc;
+    user-select: none;
+    -webkit-app-region: drag;
+  }
+  #tabs {
+    display: flex;
+    flex: 1;
+    overflow: hidden;
+    -webkit-app-region: no-drag;
+  }
+  .tab {
+    display: flex;
+    align-items: center;
+    max-width: 200px;
+    min-width: 80px;
+    padding: 0 12px;
+    height: 36px;
+    cursor: pointer;
+    border-right: 1px solid #2a2a2a;
+    transition: background 0.15s;
+  }
+  .tab:hover { background: #2a2a2a; }
+  .tab.active { background: #2d2d30; color: #fff; border-bottom: 2px solid #007acc; }
+  .tab-title {
+    flex: 1;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+    margin-right: 8px;
+  }
+  .tab-close {
+    width: 16px;
+    height: 16px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    border-radius: 3px;
+    font-size: 14px;
+    color: #666;
+    cursor: pointer;
+  }
+  .tab-close:hover { background: #3a3a3a; color: #fff; }
+  #new-tab {
+    width: 36px;
+    height: 36px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 20px;
+    cursor: pointer;
+    color: #888;
+    -webkit-app-region: no-drag;
+  }
+  #new-tab:hover { background: #2a2a2a; color: #fff; }
+  .tab-id { color: #555; font-size: 10px; margin-right: 4px; }
+</style>
+</head>
+<body>
+<div id="tabs"></div>
+<div id="new-tab">+</div>
+<script>
+const { ipcRenderer } = require('electron');
+
+async function renderTabs() {
+  try {
+    const tabs = await ipcRenderer.invoke('tab:list');
+    const container = document.getElementById('tabs');
+    container.innerHTML = '';
+    for (const tab of tabs) {
+      const el = document.createElement('div');
+      el.className = 'tab' + (tab.active ? ' active' : '');
+      const title = (tab.title || tab.url || 'New Tab').slice(0, 30);
+      el.innerHTML = '<span class="tab-id">[' + tab.id + ']</span>' +
+        '<span class="tab-title">' + title + '</span>' +
+        '<span class="tab-close" data-id="' + tab.id + '">&times;</span>';
+      el.addEventListener('click', function(e) {
+        if (e.target.classList.contains('tab-close')) {
+          ipcRenderer.send('tab:close', parseInt(e.target.dataset.id));
+        } else {
+          ipcRenderer.send('tab:activate', tab.id);
+        }
+      });
+      container.appendChild(el);
+    }
+  } catch(e) {
+    // ipcRenderer not ready yet
+  }
+}
+
+// expose for main process to call
+window.renderTabs = renderTabs;
+
+document.getElementById('new-tab').addEventListener('click', function() {
+  ipcRenderer.send('tab:new');
+});
+
+// Initial render + periodic refresh
+renderTabs();
+setInterval(renderTabs, 1500);
+</script>
+</body>
+</html>

package/src/shared/config.js

@@ -0,0 +1,32 @@
+// shared/config.js — Centralized runtime configuration, single source of truth.
+// Replaces hardcoded port 9223 / User-Agent / userData dir scattered across
+// main/. Each value respects an AI_BROWSER_* env override, with defaults that
+// are byte-identical to the previous hardcoded values (no behavior change by
+// default).
+import path from 'node:path';
+import os from 'node:os';
+
+const DEFAULT_UA =
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36';
+
+export const config = {
+  // WebSocket JSON-RPC server + MCP auto-launch probe port
+  wsPort: Number(process.env.AI_BROWSER_PORT) || 9223,
+  // Electron userData dir — session/cookie persistence survives restart
+  userDataDir: process.env.AI_BROWSER_USER_DATA || path.join(os.homedir(), '.ai-browser'),
+  // Spoofed UA to mask the automation fingerprint
+  userAgent: process.env.AI_BROWSER_UA || DEFAULT_UA,
+  // Height of the in-app tab strip below the window title bar
+  tabBarHeight: 36,
+  // The session every tab's BrowserView uses. **Must stay a single source of
+  // truth**: session-level hooks (will-download, permission requests) are
+  // installed on `session.fromPartition(viewPartition)`. Hooking
+  // `session.defaultSession` instead silently does nothing — the views are not
+  // in that session. Measured 2026-10-12: a download was requested by the page,
+  // no will-download ever fired, and no file landed.
+  viewPartition: process.env.AI_BROWSER_VIEW_PARTITION || 'persist:ai-browser',
+};
+
+export function wsUrl(port = config.wsPort) {
+  return `ws://localhost:${port}`;
+}

package/src/shared/deadline.js

@@ -0,0 +1,89 @@
+// shared/deadline.js — 给"会进页面"的调用加超时（纯函数，可无 Electron 单测）
+//
+// # 为什么需要这个文件
+// ws_server 的每个 case 都是 `await pageManager.xxx(...)`，顶层只有 try/catch ——
+// 抓得住"抛错"，抓不住"不返回"。而 Chromium 的 CDP 命令**真的会不返回**：
+// `Input.dispatchMouseEvent` 要等渲染进程 ack，渲染进程主线程被占住时 ack 永远不来。
+//
+// 实测（tools/_oneoff/blocked_renderer_probe.cjs，页面加载后 while 循环占住主线程
+// 120s，单项上限 45s）：
+//
+//     ui.get_tree      5006ms  ERR getTree timeout     ← 本来就有超时
+//     ui.evaluate     10012ms  ERR evaluate timeout    ← CDP 5s + preload 回落 5s
+//     ui.get_focused   5006ms  ERR getTree timeout
+//     ui.capabilities 15014ms  ok
+//     ui.act          45001ms  **仍无回包**            ← 唯一真挂的一条
+//
+// 也就是说这不是"慢"，是 agent 的工具调用永久悬挂、整个会话死掉，而且**没有任何错误
+// 可看**。比抛错严重得多：抛错至少能让 agent 换策略。
+//
+// # 为什么超时必须按动作算，而不是一个固定值
+// 固定小值会误杀合法长操作。实测这些路径自己就会睡很久：
+//   type/setContent 每字 interval_ms（默认 60ms）→ 500 字就是 30s
+//   drag            duration_ms（默认 420ms）+ steps×~40ms，上限 5s
+//   clear           要走编辑器状态提交，多处 80~150ms 等待 + 回读校验
+// 所以预算 = max(基线, 该动作自报的预期耗时 × 2 + 余量)，并留 env / 单次调用的逃生门。
+
+export const DEFAULT_ACTION_DEADLINE_MS = 15000;
+export const MAX_ACTION_DEADLINE_MS = 300000;
+
+/**
+ * 把一个 promise 套上超时。超时抛出的 error 带 `__deadline` 标记，调用方可以
+ * 据此把它翻译成"成功=false + 明确原因"，而不是当成未知异常往上抛。
+ *
+ * 原 promise 迟到拒绝会被接住（不会变成 unhandled rejection）—— 超时之后它仍可能
+ * 完成或失败，那是后台的事，不该把进程吵醒。
+ */
+export function withDeadline(promise, ms, label) {
+  const budget = Math.max(1, Number(ms) || DEFAULT_ACTION_DEADLINE_MS);
+  let timer = null;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => {
+      const err = new Error(`${label} timeout after ${budget}ms`);
+      err.__deadline = true;
+      err.deadline_ms = budget;
+      reject(err);
+    }, budget);
+  });
+  // 防止"超时已报错，原 promise 之后才拒绝"变成 unhandled rejection。
+  Promise.resolve(promise).catch(() => {});
+  return Promise.race([promise, timeout]).finally(() => {
+    if (timer) clearTimeout(timer);
+  });
+}
+
+/**
+ * 算一个动作的合理超时预算（毫秒）。
+ *
+ * 优先级：单次调用 `params.timeout_ms` > env `AI_BROWSER_ACTION_TIMEOUT_MS` >
+ * 按动作自报耗时推算。三者都是显式逃生门：agent 知道自己要 type 3000 字时，
+ * 不该被一个猜出来的默认值卡死。
+ */
+export function actionDeadlineMs(action, params = {}) {
+  const p = params || {};
+  let need = DEFAULT_ACTION_DEADLINE_MS;
+
+  const text = String(p.text || '');
+  if (action === 'type' || action === 'setContent') {
+    const iv = Number(p.interval_ms) > 0 ? Number(p.interval_ms) : 60;
+    need = Math.max(need, text.length * iv + 5000);
+  } else if (action === 'drag') {
+    const dur = Number(p.duration_ms) > 0 ? Number(p.duration_ms) : 420;
+    const steps = Number(p.steps) > 0 ? Number(p.steps) : 14;
+    need = Math.max(need, dur + steps * 40 + 5000);
+  } else if (action === 'press') {
+    const rep = Number(p.repeat) > 0 ? Number(p.repeat) : 1;
+    const iv = Number(p.interval_ms) > 0 ? Number(p.interval_ms) : 60;
+    need = Math.max(need, rep * iv + 5000);
+  } else if (action === 'clear') {
+    // clear 要走编辑器状态提交与回读校验（多处 80~150ms 等待）。
+    need = Math.max(need, 20000);
+  }
+
+  const env = Number(process.env.AI_BROWSER_ACTION_TIMEOUT_MS);
+  if (Number.isFinite(env) && env > 0) need = env;
+  const perCall = Number(p.timeout_ms);
+  if (Number.isFinite(perCall) && perCall > 0) need = perCall;
+
+  return Math.min(Math.max(need, 1000), MAX_ACTION_DEADLINE_MS);
+}

package/src/shared/guards.js

@@ -0,0 +1,17 @@
+// shared/guards.js — Centralized eval-safety guard, single source of truth.
+// Used by BOTH the raw WebSocket path (ws_server.js `ui.evaluate`) and the MCP
+// path (mcp_server.js `browse_evaluate`). Keeping the limits + blocklist in one
+// place prevents one entry being hardened and the other silently left open.
+
+export const EVAL_MAX_LENGTH = 5000;
+const NODE_IDENTIFIER_RE = /\bprocess\.\b|\brequire\s*\(|\bchild_process\b|\bglobalThis\.process\b/;
+
+// Returns null when the script is safe to run, or the error message that
+// should be returned to the caller. Message text is part of the contract —
+// external ws/mcp clients may match on it, so keep it stable.
+export function evaluateGuardError(js) {
+  if (typeof js !== 'string') return 'script must be a string';
+  if (js.length > EVAL_MAX_LENGTH) return `script exceeds ${EVAL_MAX_LENGTH} char limit`;
+  if (NODE_IDENTIFIER_RE.test(js)) return 'script contains disallowed Node-specific identifier';
+  return null;
+}

package/src/shared/protocol.js

@@ -0,0 +1,116 @@
+// shared/protocol.js — JSON-RPC 2.0 subset for AI Browser
+// Used by both main process (WS server) and preload (IPC bridge)
+
+/**
+ * Create a JSON-RPC request string
+ */
+export function createRequest(id, method, params = {}) {
+  return JSON.stringify({ jsonrpc: '2.0', id, method, params });
+}
+
+/**
+ * Create a JSON-RPC success response string
+ */
+export function createResponse(id, result) {
+  return JSON.stringify({ jsonrpc: '2.0', id, result });
+}
+
+/**
+ * Create a JSON-RPC error response string
+ */
+export function createError(id, code, message, data = null) {
+  const error = { code, message };
+  if (data) error.data = data;
+  return JSON.stringify({ jsonrpc: '2.0', id, error });
+}
+
+/**
+ * Create a server-pushed event (no id — notification per JSON-RPC)
+ */
+export function createEvent(event, data) {
+  return JSON.stringify({ jsonrpc: '2.0', method: event, params: data });
+}
+
+/**
+ * Parse a JSON-RPC message string into object
+ */
+export function parseMessage(msg) {
+  try {
+    const obj = JSON.parse(msg);
+    if (obj.jsonrpc !== '2.0') {
+      throw new Error('Invalid JSON-RPC version');
+    }
+    return obj;
+  } catch (e) {
+    throw new Error(`Failed to parse JSON-RPC message: ${e.message}`);
+  }
+}
+
+/**
+ * Every ui.* method the WebSocket server actually dispatches.
+ *
+ * This list had drifted badly: it named 7 methods while the server served 28,
+ * so anyone reading it would conclude most of the API did not exist. A
+ * machine-readable contract that lies is worse than no contract — so
+ * `tests/test_protocol.js` now derives the real set from the `case 'ui.x':`
+ * labels in `src/main/ws_server.js` and fails if the two diverge. Add a method
+ * to the server and this list must follow, or the suite breaks.
+ */
+export const VALID_METHODS = [
+  'ui.get_tree',
+  'ui.act',
+  'ui.navigate',
+  'ui.repaint',
+  'ui.snapshot',
+  'ui.diff',
+  'ui.capabilities',
+  'ui.evaluate',
+  'ui.subscribe',
+  'ui.unsubscribe',
+  'ui.downloads',
+  'ui.get_focused',
+  'ui.peek',
+  'ui.wait',
+  'ui.scroll',
+  'ui.new_tab',
+  'ui.close_tab',
+  'ui.list_tabs',
+  'ui.set_active_tab',
+  'ui.canvas_configure',
+  'ui.canvas_list',
+  'ui.canvas_read',
+  'ui.canvas_capture',
+  'ui.network_list',
+  'ui.network_get',
+  'ui.network_clear',
+  'ui.network_configure',
+  'ui.network_body',
+  'ui.quit',
+];
+
+/**
+ * Standard JSON-RPC error codes
+ */
+export const ERROR_CODES = {
+  PARSE_ERROR: -32700,
+  INVALID_REQUEST: -32600,
+  METHOD_NOT_FOUND: -32601,
+  INVALID_PARAMS: -32602,
+  INTERNAL_ERROR: -32603,
+  ACTION_FAILED: -32001,
+  TARGET_NOT_FOUND: -32002,
+  INVALID_ACTION: -32003,
+};
+
+// For CJS compatibility in Electron preload
+if (typeof module !== 'undefined' && module.exports) {
+  module.exports = {
+    createRequest,
+    createResponse,
+    createError,
+    createEvent,
+    parseMessage,
+    VALID_METHODS,
+    ERROR_CODES,
+  };
+}