@glittering0717/ai-browser 1.2.1 → 1.3.0

package/src/main/mcp_tools.js

@@ -11,13 +11,14 @@ export const MCP_TOOLS = [
   {
     name: 'browse_navigate',
     title: 'Navigate tab',
-    description: 'Navigate the active/focused tab to a URL.',
+    description: 'Navigate the active/focused tab to a URL. Returns as soon as the requested milestone is reached (default: DOM ready) — it does not wait for the load event, which SPAs often never fire.',
     annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true },
     inputSchema: {
       type: 'object',
       properties: {
         url: { type: 'string', description: 'URL to navigate to.' },
-        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' },
+        wait_until: { type: 'string', enum: ['commit', 'domcontentloaded', 'load'], description: 'How far to wait. commit = navigation committed (fastest, content may not be painted); domcontentloaded = DOM ready (default, safe for SPAs); load = also subresources (can hang forever on pages that never fire load).' }
       },
       required: ['url']
     }
@@ -41,20 +42,21 @@ export const MCP_TOOLS = [
   {
     name: 'browse_act',
     title: 'Act on element',
-    description: 'Act on an element by data-ai-id: click, type (append at caret), setContent (replace all), clear, focus, hover, scroll_to, upload, get_value (read-only), drag, press, wheel.',
+    description: 'Act on an element by data-ai-id: click, type (append at caret), setContent (replace all), clear, focus, hover, scroll_to, upload, get_value (read-only), drag, press, wheel, paste.',
     annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true },
     inputSchema: {
       type: 'object',
       properties: {
-        action: { type: 'string', enum: ['click', 'type', 'clear', 'focus', 'hover', 'scroll_to', 'upload', 'get_value', 'drag', 'press', 'wheel'], description: 'Action.' },
-        target: { type: 'string', description: 'Element data-ai-id from the semantic tree. Omit for press (keys go to the current focus); for drag you may also omit it and pass from_x/from_y to start from a bare point.' },
-        text: { type: 'string', description: 'Text to type (type only).' },
+        action: { type: 'string', enum: ['click', 'type', 'clear', 'focus', 'hover', 'scroll_to', 'upload', 'get_value', 'drag', 'press', 'wheel', 'paste'], description: 'Action.' },
+        html: { type: 'string', description: 'paste: HTML to inject (preferred). Editors convert it through their own paste pipeline (Draft.js/ProseMirror/Quill), so headings/code blocks/bold/lists survive. Plain text loses all formatting.' },
+        target: { type: 'string', description: 'Element data-ai-id from the semantic tree (e.g. e:button-12, axf-3-1) — NOT the visible label. Omit for press (keys go to the current focus); for drag you may also omit it and pass from_x/from_y to start from a bare point. upload also accepts a CSS selector (descendant selectors with spaces included), which is the only way to reach upload widgets that have no semantic node.' },
+        text: { type: 'string', description: 'Text to type (type/setContent). For paste: plain-text fallback used when html is absent.' },
         value: { type: 'string', description: 'Value for select.' },
         offset: { type: 'integer', description: 'get_value: start offset in Unicode code points (default 0).' },
         limit: { type: 'integer', description: 'get_value: max code points to return (default 20000, hard cap 1000000).' },
         url: { type: 'string', description: 'For click: click the link whose href matches this URL instead of guessing the target label.' },
         keep_tab: { type: 'boolean', description: 'For click: keep the current tab active instead of auto-following a newly opened tab.' },
-        file: { type: 'string', description: 'Absolute path to upload (upload only, target must be a file input).' },
+        file: { type: 'string', description: 'Absolute path to upload (upload only). target may be the file input itself, the control that opens the picker, or — when the control has no semantic node at all — a CSS selector such as ".upload-button". Works inside cross-process iframes.' },
         timeout_ms: { type: 'integer', description: 'Override the action deadline in ms (default 15000; type/setContent/drag/press scale with text length / duration / repeat; 1s floor, 300s cap). A timeout returns success=false, timed_out=true and warns the action may have partially applied.' },
         to_target: { type: 'string', description: 'drag: data-ai-id to drop onto (e.g. drag from one node\'s output to another node to connect them).' },
         dx: { type: 'number', description: 'drag: horizontal distance to move (use with dy instead of to_target).' },
@@ -79,13 +81,14 @@ export const MCP_TOOLS = [
   {
     name: 'browse_evaluate',
     title: 'Run page JavaScript',
-    description: 'Run a JS expression in the page context and return its value.',
+    description: 'Run a JS expression in the page context and return its value. Promises are awaited (async IIFEs return their resolved value, not {}).',
     annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true },
     inputSchema: {
       type: 'object',
       properties: {
-        js: { type: 'string', description: 'Page-context JS expression (not statements). Max 5000 chars.' },
-        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+        js: { type: 'string', description: 'Page-context JS expression (not statements). Max 5000 chars. Promises are awaited.' },
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' },
+        frame_ref: { type: 'string', description: 'Run inside this cross-process frame (an axf-... id from the tree). Pass it explicitly when touching iframe content: building the selector by string concatenation defeats the automatic route and the top-frame run returns null.' }
       },
       required: ['js']
     }
@@ -122,6 +125,44 @@ export const MCP_TOOLS = [
       required: ['condition']
     }
   },
+  {
+    name: 'browse_auth_state',
+    title: 'Check login / auth-wall state',
+    description: 'Is this tab still logged in, or bounced to a login wall? auth_wall.detected=true means stop debugging and re-authenticate first. logged_in is heuristic; auth_wall is the signal to trust.',
+    annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' }
+      }
+    }
+  },
+  {
+    name: 'browse_screenshot',
+    title: 'Screenshot the viewport',
+    description: 'Screenshot the viewport (base64 in data). Use FIRST when an action misbehaves: the element tree cannot show overlays, canvas, occlusion or glitches. Warns if a visible password input is on screen.',
+    annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        tab: { type: 'integer', description: 'Tab id (default: active tab).' },
+        format: { type: 'string', enum: ['png', 'jpeg', 'webp'], description: 'Image format. Default png. jpeg at quality 85 is ~5x smaller — prefer it for long pages.' },
+        quality: { type: 'number', description: '0-100, jpeg/webp only (ignored for png). Default 85 for jpeg.' },
+        fullPage: { type: 'boolean', description: 'Capture the full scrollable page, not just the viewport. Cannot be combined with clip.' },
+        clip: {
+          type: 'object',
+          description: 'Capture a rectangle in CSS pixels: {x, y, width, height, scale?}. Use to zoom into one element when the viewport shot is too small to read.',
+          properties: {
+            x: { type: 'number' },
+            y: { type: 'number' },
+            width: { type: 'number' },
+            height: { type: 'number' },
+            scale: { type: 'number', description: 'Device scale factor, 0-4. Default 1.' }
+          }
+        }
+      }
+    }
+  },
   {
     name: 'browse_list_tabs',
     title: 'List tabs',
@@ -279,6 +320,19 @@ export const MCP_TOOLS = [
       },
     },
   },
+  {
+    name: 'browse_errors',
+    title: 'Read error log',
+    description: 'Read the main-process error log: uncaught exceptions, crashes, failed navigations, RPC errors. Each entry includes recent calls as context. Returns the on-disk JSONL path.',
+    annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },
+    inputSchema: {
+      type: 'object',
+      properties: {
+        limit: { type: 'integer', description: 'Max entries to return (default 50).' },
+        kind: { type: 'string', description: 'Filter by kind: uncaught_exception / unhandled_rejection / render_process_gone / child_process_gone / nav_fail / ws_error.' },
+      },
+    },
+  },
   {
     name: 'browse_quit',
     title: 'Quit browser',

package/src/main/network_monitor.js

@@ -165,10 +165,22 @@ export class NetworkMonitor {
     this.captureEnabled = opts.captureEnabled !== undefined ? opts.captureEnabled : envFlag('AI_BROWSER_NETWORK_CAPTURE', true);
     this.sensitiveAllowed = opts.sensitiveAllowed !== undefined ? opts.sensitiveAllowed : envFlag('AI_BROWSER_NETWORK_SENSITIVE', false);
     this.globalCaptureBodies = 'api';
+    // 当前活跃 tab（由 page_manager 在 setActive / closeTab 时同步进来）。
+    // 只被 body 淘汰顺序用：null 表示不知道/没有活跃 tab，此时退回纯按新旧。
+    // 注意它可以是 null，所以下面 _enforceBodyLimit 里判等不能假设非空。
+    this.activeTabId = null;
   }
 
   // ==== 配置 ============================================================
 
+  /**
+   * 告知活跃 tab。page_manager 是唯一调用方（setActive / closeTab 换活跃页时）。
+   * 传 null = 没有活跃 tab（最后一个 tab 关掉了）。
+   */
+  setActiveTab(tabId) {
+    this.activeTabId = tabId == null ? null : tabId;
+  }
+
   setBodyFetcher(tabId, fn) { this._fetchers.set(tabId, fn); }
   clearBodyFetcher(tabId) { this._fetchers.delete(tabId); }
 
@@ -204,16 +216,36 @@ export class NetworkMonitor {
 
   _enforceRecordLimit(protectId) {
     while (this._records.size > this.limits.maxRecords) {
-      let victim = null;
-      for (const id of this._order) {
-        if (id === protectId) continue;
-        victim = id; break;
-      }
+      const victim = this._evictionVictim(protectId);
       if (!victim) break;
       this._dropRecord(victim);
     }
   }
 
+  // 淘汰候选：**非活跃 tab 的最旧一条**优先；一条都没有（全是活跃 tab 的，或
+  // 根本不知道活跃 tab）才退到全局最旧。
+  //
+  // 为什么两处淘汰都要挑 tab：全局上限（记录数 / body 字节）是跨 tab 共享的，
+  // 而后台 tab 会持续累积。实测默认页 baidu.com 挂久了之后，agent 正在操作的
+  // 那个 tab 的 POST 请求体被挤成 evicted（e2e/network_ws.cjs N-04/N-05 红）——
+  // 请求确实发生了、body 也确实拿到过，只是被后台噪声顶掉，agent 读到的是
+  // evicted 而不是内容。让活跃 tab 有优先保留权，后台 tab 超限时先牺牲自己。
+  //
+  // activeTabId 为 null（无活跃 tab / 单测未接线）时，`tab_id !== null` 恒真，
+  // 直接返回全局最旧 —— 与改动前完全一致。
+  _evictionVictim(protectId) {
+    const active = this.activeTabId;
+    let fallback = null;
+    for (const id of this._order) {
+      if (id === protectId) continue;
+      const rec = this._records.get(id);
+      if (!rec) continue;
+      if (fallback === null) fallback = id;
+      if (rec.tab_id !== active) return id;
+    }
+    return fallback;
+  }
+
   _dropRecord(networkId) {
     const rec = this._records.get(networkId);
     if (!rec) return;
@@ -225,25 +257,35 @@ export class NetworkMonitor {
     this._droppedRecords += 1;
   }
 
-  // body 超限：优先淘汰最旧的 response body，其次 request body；metadata 保留。
+  // body 超限淘汰：先 response body 后 request body（同一次超限里），同一种 body
+  // 内部按 _order 由旧到新。**并且先淘汰非活跃 tab 的**（见 _evictionVictim 的
+  // 注释：全局 32MB 配额跨 tab 共享，后台 tab 会把活跃 tab 的 body 顶成 evicted）。
+  // metadata 不淘汰，只丢 body 内容，记录仍然查得到。
+  //
+  // 单 tab 场景下行为不变（活跃 tab 就是唯一的 tab，排序等价于纯按新旧）。
   _enforceBodyLimit(protectId) {
     if (this._bodyBytes <= this.limits.maxBodyBytes) return;
-    for (const pass of ['response', 'request']) {
-      for (const id of this._order) {
-        if (this._bodyBytes <= this.limits.maxBodyBytes) return;
-        if (id === protectId) continue;
-        const rec = this._records.get(id);
-        if (!rec) continue;
-        if (pass === 'response' && rec._respBody) {
-          this._bodyBytes -= rec._respBodyBytes || 0;
-          rec._respBody = null; rec._respBodyBytes = 0;
-          rec.response_body_state = 'evicted';
-          rec._respEvicted = true;
-        } else if (pass === 'request' && rec._reqBody) {
-          this._bodyBytes -= rec._reqBodyBytes || 0;
-          rec._reqBody = null; rec._reqBodyBytes = 0;
-          rec.request_body_state = 'evicted';
-          rec._reqEvicted = true;
+    const active = this.activeTabId;
+    // 两轮：先把非活跃 tab 的 body 淘汰掉，仍然超限才动活跃 tab 自己的。
+    for (const phase of ['background', 'active']) {
+      for (const pass of ['response', 'request']) {
+        for (const id of this._order) {
+          if (this._bodyBytes <= this.limits.maxBodyBytes) return;
+          if (id === protectId) continue;
+          const rec = this._records.get(id);
+          if (!rec) continue;
+          if (phase === 'background' ? rec.tab_id === active : rec.tab_id !== active) continue;
+          if (pass === 'response' && rec._respBody) {
+            this._bodyBytes -= rec._respBodyBytes || 0;
+            rec._respBody = null; rec._respBodyBytes = 0;
+            rec.response_body_state = 'evicted';
+            rec._respEvicted = true;
+          } else if (pass === 'request' && rec._reqBody) {
+            this._bodyBytes -= rec._reqBodyBytes || 0;
+            rec._reqBody = null; rec._reqBodyBytes = 0;
+            rec.request_body_state = 'evicted';
+            rec._reqEvicted = true;
+          }
         }
       }
     }
@@ -283,7 +325,19 @@ export class NetworkMonitor {
       rec.response_body_state = overCap ? 'partial' : 'captured';
     }
     this._bodyBytes += bytes;
-    this._enforceBodyLimit(rec.network_id);
+    // "刚存进来的这条不在本轮淘汰里"是给活跃 tab 的特权，不是普适规则。
+    //
+    // 后台 tab 的 body 没有这个权利：否则一条 40KB 的后台响应体会先把活跃
+    // tab 里的小 body 全部挤掉（实测 baidu 默认页的一条响应把活跃 tab 上
+    // "GET-PAYLOAD-OK" 这 28 字节的响应顶成 evicted —— e2e/network_ws.cjs
+    // N-12 间歇红）。让它自己先被牺牲，活跃 tab 的 body 就不受影响：后台的
+    // body 反正没人看，而"活跃 tab 里明明还在配额内的小 body 消失了"是硬伤。
+    //
+    // activeTabId == null（没接线 / 单 tab）时一律按活跃处理 —— 保持改动前的
+    // 行为，不让新增逻辑在没有 tab 语义的场景里改变语义。
+    const active = this.activeTabId;
+    const isActive = active == null || rec.tab_id === active;
+    this._enforceBodyLimit(isActive ? rec.network_id : null);
     return body;
   }
 
@@ -522,6 +576,10 @@ export class NetworkMonitor {
       max_records: this.limits.maxRecords,
       body_bytes: this._bodyBytes,
       max_body_bytes: this.limits.maxBodyBytes,
+      // 淘汰优先级看这个：非活跃 tab 的 body / 记录先被牺牲。null = 不知道活跃
+      // tab（此时按纯新旧淘汰）。暴露出来是因为"我的 body 怎么被 evicted 了"
+      // 这类问题只能靠它定位 —— 曾经它就是个黑盒。
+      active_tab: this.activeTabId,
     };
   }
 
@@ -531,6 +589,11 @@ export class NetworkMonitor {
       enabled: !!cfg.enabled,
       capture_bodies: cfg.capture_bodies,
       since: this._startedAt,
+      // 当前全局占用与淘汰优先级。body 被 evicted 时，先看这里就能分清是
+      // "自己这个 tab 太多了"还是"后台 tab 把自己顶了"。
+      body_bytes: this._bodyBytes,
+      max_body_bytes: this.limits.maxBodyBytes,
+      active_tab: this.activeTabId,
       limits: {
         max_records: this.limits.maxRecords,
         max_body_bytes: this.limits.maxBodyBytes,