minimal-agent 0.10.0 → 0.12.0

package/src/web/browserBridge.ts

@@ -0,0 +1,672 @@
+import { randomUUID } from 'node:crypto';
+import { rm } from 'node:fs/promises';
+import { configService } from '../config/configService.js';
+import { resolveAgentBrowserPath } from '../tools/webbrowser/agentBrowserPath.js';
+import { browserProfilePath, normalizeUrl, sessionArgs, spawnRunner } from '../tools/webbrowser/browser.js';
+import { ensureDaemon, stopDaemon } from '../tools/webbrowser/browserDaemon.js';
+import { browserService } from '../tools/webbrowser/browserService.js';
+import { browserRunRegistry, type BrowserRunContext } from '../tools/webbrowser/browserRunRegistry.js';
+import {
+  createConversationTab,
+  ensureConversationTab,
+  listBrowserTabs,
+} from '../tools/webbrowser/browserTabs.js';
+import {
+  getBrowserStream,
+  type BrowserStreamEvent,
+  type PickedBrowserElement,
+} from '../tools/webbrowser/browserStream.js';
+import type { ServerFrame } from './protocol.js';
+import type { HumanInterventionEvent } from '../humanIntervention.js';
+
+type BrowserClientFrame = {
+  t: string;
+  requestId?: string;
+  [key: string]: unknown;
+};
+
+export type BrowserBridge = {
+  handle(frame: BrowserClientFrame): Promise<void>;
+  dispose(): void;
+};
+
+function requireAllowedDeveloperOrigin(url: string, allowedOrigins: string[]): string {
+  let origin: string;
+  try { origin = new URL(url).origin; } catch { throw new Error('当前页面没有可授权的 HTTP(S) 站点'); }
+  if (!/^https?:\/\//i.test(origin) || !allowedOrigins.includes(origin)) {
+    throw new Error(`Developer Mode 尚未授权当前站点：${origin}。请在“设置 → 浏览器”添加该站点。`);
+  }
+  return origin;
+}
+
+function parseJsonOutput(text: string): unknown {
+  const trimmed = text.trim();
+  if (!trimmed) return undefined;
+  try {
+    const value = JSON.parse(trimmed) as unknown;
+    if (typeof value === 'string') {
+      try { return JSON.parse(value) as unknown; } catch { return value; }
+    }
+    return value;
+  } catch { return trimmed; }
+}
+
+function pickScript(x: number, y: number): string {
+  return `(() => {
+    const el = document.elementFromPoint(${Math.round(x)}, ${Math.round(y)});
+    if (!el) return null;
+    const r = el.getBoundingClientRect();
+    const attrs = {};
+    for (const a of Array.from(el.attributes || []).slice(0, 16)) attrs[a.name] = String(a.value).slice(0, 300);
+    const escape = (v) => CSS.escape(String(v));
+    let selector = el.id ? '#' + escape(el.id) : el.tagName.toLowerCase();
+    if (!el.id && el.classList && el.classList.length) selector += '.' + Array.from(el.classList).slice(0, 2).map(escape).join('.');
+    if (!el.id) {
+      const same = Array.from(document.querySelectorAll(selector));
+      if (same.length > 1) selector += ':nth-of-type(' + (Array.from(el.parentElement?.children || []).filter((n) => n.tagName === el.tagName).indexOf(el) + 1) + ')';
+    }
+    const role = el.getAttribute('role') || ({A:'link',BUTTON:'button',INPUT:'textbox',SELECT:'combobox',TEXTAREA:'textbox',IMG:'img'}[el.tagName] || undefined);
+    const accessibleName = el.getAttribute('aria-label') || el.getAttribute('alt') || el.getAttribute('title') || String(el.innerText || el.textContent || '').trim().slice(0, 160);
+    return { ref: selector, tag: el.tagName.toLowerCase(), role, accessibleName, text: String(el.innerText || el.textContent || '').trim().slice(0, 1000), boundingBox: { x: r.x, y: r.y, width: r.width, height: r.height }, attributes: attrs };
+  })()`;
+}
+
+function mapStreamEvent(event: BrowserStreamEvent): ServerFrame | null {
+  switch (event.type) {
+    case 'frame':
+      return { t: 'browser/frame', data: String(event.data ?? ''), metadata: event.metadata as Record<string, unknown> | undefined,
+        frameWidth: Number(event.frameWidth ?? event.width ?? (event.metadata as Record<string, unknown> | undefined)?.width ?? 0) || undefined,
+        frameHeight: Number(event.frameHeight ?? event.height ?? (event.metadata as Record<string, unknown> | undefined)?.height ?? 0) || undefined,
+        viewportWidth: Number(event.viewportWidth ?? 0) || undefined,
+        viewportHeight: Number(event.viewportHeight ?? 0) || undefined,
+        deviceScaleFactor: Number(event.deviceScaleFactor ?? 1) || 1 };
+    case 'status':
+      return {
+        t: 'browser/status',
+        connected: Boolean(event.connected),
+        screencasting: Boolean(event.screencasting ?? event.screencast),
+        viewportWidth: Number(event.viewportWidth ?? event.width ?? 1280),
+        viewportHeight: Number(event.viewportHeight ?? event.height ?? 720),
+        engine: 'agent-browser',
+      };
+    case 'tabs': {
+      const tabs = Array.isArray(event.tabs) ? event.tabs : [];
+      return { t: 'browser/tabs', tabs: tabs.map((tab) => {
+        const item = tab as Record<string, unknown>;
+        return { id: String(item.tabId ?? item.id ?? ''), url: String(item.url ?? ''), title: typeof item.title === 'string' ? item.title : undefined, active: Boolean(item.active), favicon: typeof item.favicon === 'string' ? item.favicon : undefined };
+      }) };
+    }
+    case 'agent_candidates': {
+      const list = Array.isArray(event.candidates) ? event.candidates : [];
+      return {
+        t: 'browser/agent/candidates',
+        candidates: list.map((item) => {
+          const c = item as Record<string, unknown>;
+          return { ref: String(c.ref ?? ''), line: String(c.line ?? '') };
+        }),
+        ...(typeof event.selector === 'string' ? { selector: event.selector } : {}),
+      };
+    }
+    case 'agent_outcome': return {
+      t: 'browser/agent/outcome',
+      action: String(event.action ?? ''),
+      effect: event.effect === 'unknown' ? 'unknown' : 'unchanged',
+      ...(typeof event.selector === 'string' ? { selector: event.selector } : {}),
+      ...(typeof event.maskedLines === 'number' ? { maskedLines: event.maskedLines } : {}),
+    };
+    case 'url': return { t: 'browser/url', url: String(event.url ?? ''), tabId: typeof event.tabId === 'string' ? event.tabId : undefined };
+    case 'title': return { t: 'browser/title', title: String(event.title ?? ''), tabId: typeof event.tabId === 'string' ? event.tabId : undefined };
+    case 'navigation': return { t: 'browser/navigation', url: String(event.url ?? ''), state: (event.state === 'start' || event.state === 'commit' || event.state === 'domcontentloaded') ? event.state : 'load' };
+    case 'console': return { t: 'browser/console', level: typeof event.level === 'string' ? event.level : undefined, text: String(event.text ?? '') };
+    case 'page_error': return { t: 'browser/page_error', text: String(event.text ?? event.message ?? '') };
+    case 'dialog': return { t: 'browser/dialog', type: (event.dialogType ?? event.kind ?? 'alert') as 'alert' | 'confirm' | 'prompt' | 'beforeunload', message: String(event.message ?? ''), defaultValue: typeof event.defaultValue === 'string' ? event.defaultValue : undefined };
+    case 'error': return { t: 'browser/error', message: String(event.message ?? '浏览器流错误') };
+    case 'mode': return { t: 'browser/mode', mode: event.mode as 'browse' | 'assist' | 'auto' };
+    case 'marked_elements': return { t: 'browser/marked/elements', elements: (event.elements ?? []) as never };
+    case 'agent_acting': return { t: 'browser/agent/acting', acting: Boolean(event.acting), action: typeof event.action === 'string' ? event.action : undefined, target: typeof event.target === 'string' ? event.target : undefined };
+    case 'agent_pointer': return { t: 'browser/agent/pointer', x: Number(event.x ?? 0), y: Number(event.y ?? 0), visible: Boolean(event.visible), action: (typeof event.action === 'string' ? event.action : undefined) as 'move' | 'click' | 'type' | 'scroll' | 'idle' | 'hover' | undefined, elementRef: typeof event.elementRef === 'string' ? event.elementRef : undefined };
+    case 'control_state': return { t: 'browser/control/changed', state: event.state as never };
+    default: return null;
+  }
+}
+
+function mapHumanInterventionEvent(event: HumanInterventionEvent): Record<string, unknown> {
+  if (event.type === 'started') {
+    return {
+      t: 'human-intervention/started',
+      intervention: event.intervention,
+    };
+  }
+  return {
+    t: 'human-intervention/resolved',
+    resolution: event.resolution,
+  };
+}
+
+// —— 用户指针输入的双通路路由 ————————————————————————————————————————————
+// agent-browser daemon 有两条鼠标输入通路（子模块只读，二进制来自 GitHub releases）：
+//  · stream `input_mouse`(native/stream/websocket.rs)——裸转发 CDP dispatchMouseEvent、
+//    **无状态、且不带 `buttons`**（当前按住的键位掩码）。够 hover / 单击 / 滚轮；但「按住拖拽」
+//    必须靠 mouseMoved 上的 buttons=1 表明左键仍按住，缺了它 Chrome 只当悬停 → 拖不动
+//    （滑块/旋转类验证码就卡在这，用户「鼠标动不了」的真因）。
+//  · 命令 `mouse move|down|up`(native/actions.rs handle_mouse*)——走 build_mouse_event_params
+//    (&mut mouse_state)，**有状态**、按已按下键位正确带 `buttons`，能真正拖拽。
+// 故本桥接层（可写侧，符合「agent-browser 只读、修复落 minimal-agent」）做通路分流：
+//  hover / 单击 / 滚轮仍走快的 input_mouse；一旦识别为拖拽手势（按下后发生移动），整段
+//  down→move→up 改走命令通路让 buttons 被 daemon 跟踪。命令通路要 spawn CLI、逐帧太慢，故
+//  拖拽中间帧按节流下发、并用串行队列保序，末帧落到抬起前的真实坐标。
+export interface PointerRouteState {
+  dragging: boolean;
+  dragButton: string;
+  /** 已按下但尚未判定单击/拖拽的按下帧（移动到来=拖拽；直接抬起=单击）。 */
+  pendingPress: BrowserClientFrame | null;
+  lastDragMoveAt: number;
+}
+
+export function createPointerRouteState(): PointerRouteState {
+  return { dragging: false, dragButton: 'left', pendingPress: null, lastDragMoveAt: 0 };
+}
+
+export interface PointerRoutePlan {
+  /** 走快通路 input_mouse 立即下发的帧。 */
+  stream: Array<Record<string, unknown>>;
+  /** 走 daemon 有状态命令通路、按序 enqueue 的 `mouse ...` argv。 */
+  commands: string[][];
+}
+
+/**
+ * 纯函数：按持久路由状态决定一个用户鼠标帧怎么走。就地更新 state，返回待下发的快通路帧
+ * 与待入队的命令通路 argv。抽成纯函数便于单测拖拽判定/单击缓冲/节流，不碰真实 spawn。
+ */
+export function routeUserMouse(
+  state: PointerRouteState,
+  frame: BrowserClientFrame,
+  now: number,
+  dragMoveThrottleMs = 40,
+): PointerRoutePlan {
+  const plan: PointerRoutePlan = { stream: [], commands: [] };
+  const eventType = String(frame.eventType ?? '');
+  const x = Math.round(Number(frame.x ?? 0));
+  const y = Math.round(Number(frame.y ?? 0));
+  const fast = (f: BrowserClientFrame = frame) => plan.stream.push({ type: 'input_mouse', ...f, t: undefined });
+
+  if (eventType === 'mousePressed') {
+    // 缓冲按下：还不知道是单击还是拖拽——先不派发，等下一帧揭晓。
+    state.pendingPress = frame;
+    state.dragging = false;
+    return plan;
+  }
+
+  if (eventType === 'mouseMoved') {
+    if (state.pendingPress && !state.dragging) {
+      // 按下后首次移动 = 拖拽：在命令通路上建立「按住」状态（mouse down 令 daemon 记住 buttons）。
+      const px = Math.round(Number(state.pendingPress.x ?? x));
+      const py = Math.round(Number(state.pendingPress.y ?? y));
+      state.dragButton = String(state.pendingPress.button ?? 'left') || 'left';
+      state.dragging = true;
+      state.pendingPress = null;
+      state.lastDragMoveAt = now;
+      plan.commands.push(['move', String(px), String(py)], ['down', state.dragButton], ['move', String(x), String(y)]);
+      return plan;
+    }
+    if (state.dragging) {
+      // 拖拽中间帧：节流下发命令通路（spawn 昂贵），不再走 input_mouse 避免双通路重复派发。
+      if (now - state.lastDragMoveAt >= dragMoveThrottleMs) {
+        state.lastDragMoveAt = now;
+        plan.commands.push(['move', String(x), String(y)]);
+      }
+      return plan;
+    }
+    fast(); // 普通悬停 → 快通路
+    return plan;
+  }
+
+  if (eventType === 'mouseReleased') {
+    if (state.dragging) {
+      // 收尾：先补末帧到真实抬起坐标（节流可能丢了它），再抬起清 buttons。
+      plan.commands.push(['move', String(x), String(y)], ['up', state.dragButton]);
+      state.dragging = false;
+      return plan;
+    }
+    if (state.pendingPress) {
+      // 纯单击（按下后无移动）：快通路补发缓冲的按下 + 本次抬起，语义仍是一次点击。
+      fast(state.pendingPress);
+      fast();
+      state.pendingPress = null;
+      return plan;
+    }
+    fast();
+    return plan;
+  }
+
+  // mouseWheel / 其它鼠标事件 → 快通路（不涉及按住拖拽）。
+  fast();
+  return plan;
+}
+
+export function createBrowserBridge(send: (frame: ServerFrame) => void, sessionName = 'minimal-agent'): BrowserBridge {
+  let disposed = false;
+  let activeTabId = '';
+  let pageRevision = randomUUID();
+  let runContext: BrowserRunContext | null = null;
+  const stream = getBrowserStream(sessionName);
+  const unsubscribe = stream.subscribe((event) => {
+    if (disposed) return;
+    if (event.type === 'control_state') {
+      const control = event.state as import('../tools/webbrowser/browserControl.js').BrowserControlState | undefined;
+      if (control && control.status !== 'idle' && control.conversationKey) {
+        runContext = browserRunRegistry.find(control.conversationKey, control.runId)
+          ?? browserRunRegistry.ensure({
+            conversationKey: control.conversationKey,
+            runId: control.runId,
+            profileId: 'default',
+            grants: ['observe', 'interact'],
+          });
+      }
+    }
+    if (event.type === 'human_intervention_started' || event.type === 'human_intervention_resolved') {
+      const interventionEvent = event.event as HumanInterventionEvent | undefined;
+      if (interventionEvent) send(mapHumanInterventionEvent(interventionEvent) as unknown as ServerFrame);
+    }
+    if (event.type === 'url' || event.type === 'navigation') pageRevision = randomUUID();
+    if (event.type === 'tabs' && Array.isArray(event.tabs)) {
+      const active = event.tabs.find((tab) => Boolean((tab as Record<string, unknown>).active)) as Record<string, unknown> | undefined;
+      activeTabId = String(active?.tabId ?? active?.id ?? activeTabId);
+    }
+    let mapped = mapStreamEvent(event);
+    const activeOwner = activeTabId ? browserRunRegistry.ownerOf(activeTabId) : undefined;
+    if (runContext && activeTabId && activeOwner !== runContext.conversationKey
+      && mapped && ['browser/frame', 'browser/url', 'browser/title', 'browser/navigation'].includes(mapped.t)) {
+      return;
+    }
+    if (runContext && mapped?.t === 'browser/tabs') {
+      const claimed = new Set(browserRunRegistry.claimedTabs(runContext.conversationKey).map(String));
+      mapped = { ...mapped, tabs: mapped.tabs.filter((tab) => claimed.has(tab.id)) };
+    }
+    if (mapped?.t === 'browser/frame') send({
+      ...mapped,
+      tabId: activeTabId || mapped.tabId,
+      pageRevision,
+      ...(runContext ? { conversationKey: runContext.conversationKey, runId: runContext.runId } : {}),
+    });
+    else if (mapped) send(mapped);
+    if (event.type === 'control_state') {
+      const state = event.state as import('../tools/webbrowser/browserControl.js').BrowserControlState | undefined;
+      // 控制权移交给用户（手动接管或验证挑战）时淡出 AI 指针，避免“谁在操作”的混淆。
+      if (state?.owner === 'user') stream.setAgentPointer({ x: 0, y: 0, visible: false });
+      if (state?.status === 'waiting_human' && state.reason !== 'manual') {
+        const summary = state.reason === 'captcha' ? '页面需要人工完成安全验证'
+          : state.reason === 'otp' ? '页面需要人工输入一次性验证码'
+            : state.reason === 'password' ? '页面需要人工输入密码'
+              : state.reason === 'qr_login' ? '页面需要人工扫码登录'
+            : state.reason === 'login' ? '页面需要人工登录'
+              : state.reason === 'permission' ? '页面需要人工确认权限'
+                : state.reason === 'security_warning' ? '页面需要人工处理安全警告'
+                  : '网站已暂停或拦截当前访问';
+        send({
+          t: 'browser/challenge/detected',
+          reason: state.reason,
+          summary,
+          resumeToken: state.resumeToken,
+          runId: state.runId,
+          ...(state.conversationKey ? { conversationKey: state.conversationKey } : {}),
+        });
+      }
+    }
+  });
+
+  const run = async (args: string[]) => {
+    const bin = await resolveAgentBrowserPath();
+    if (!bin) throw new Error('未找到内置 agent-browser');
+    await ensureDaemon(sessionName);
+    const result = await browserService.run({ sessionName, bin, args: [...sessionArgs(sessionName), ...args], runner: spawnRunner });
+    if (result.code !== 0) throw new Error((result.stderr || result.stdout || `exit ${result.code}`).trim());
+    return result.stdout.trim();
+  };
+
+  const requireContext = (): BrowserRunContext => {
+    if (!runContext) throw new Error('浏览器尚未绑定当前会话，请重新打开工作台');
+    return runContext;
+  };
+
+  const claimContextTab = async (): Promise<string> => {
+    const context = requireContext();
+    const bin = await resolveAgentBrowserPath();
+    if (!bin) throw new Error('未找到内置 agent-browser');
+    await ensureDaemon(sessionName);
+    const tabId = await ensureConversationTab({
+      context,
+      sessionName,
+      bin,
+      globals: sessionArgs(sessionName),
+      runner: spawnRunner,
+    });
+    activeTabId = tabId;
+    return tabId;
+  };
+
+  const createContextTab = async (url = 'about:blank'): Promise<string> => {
+    const context = requireContext();
+    const bin = await resolveAgentBrowserPath();
+    if (!bin) throw new Error('未找到内置 agent-browser');
+    await ensureDaemon(sessionName);
+    const tabId = await createConversationTab({
+      context,
+      sessionName,
+      bin,
+      globals: sessionArgs(sessionName),
+      runner: spawnRunner,
+    }, url);
+    activeTabId = tabId;
+    return tabId;
+  };
+
+  const runOnContextTab = async (args: string[]): Promise<string> => {
+    const tabId = await claimContextTab();
+    const bin = await resolveAgentBrowserPath();
+    if (!bin) throw new Error('未找到内置 agent-browser');
+    const output = await browserService.exclusive({
+      sessionName,
+      bin,
+      runner: spawnRunner,
+    }, async (execute) => {
+      const switched = await execute([...sessionArgs(sessionName), 'tab', tabId]);
+      if (switched.code !== 0) return switched;
+      return execute([...sessionArgs(sessionName), ...args]);
+    });
+    if (output.code !== 0) {
+      throw new Error((output.stderr || output.stdout || `exit ${output.code}`).trim());
+    }
+    return output.stdout.trim();
+  };
+
+  const ensureStarted = async () => {
+    const config = await configService.browser();
+    if (!config.enabled) throw new Error('内置浏览器已在设置中关闭');
+    const info = await ensureDaemon(sessionName);
+    await stream.connect();
+    return info;
+  };
+
+  // 改视口后重连查看者，让下一帧按新尺寸编码（screencast 尺寸绑定在查看者连接时刻）。
+  // 注意：标签切换/新建/关闭/导航不需要在这里重连——agent-browser daemon 在这些命令后会自己
+  // set_cdp_session_id + notify、让 screencast 重定向到新活动标签（见 actions.rs / stream/cdp_loop.rs）。
+  // 真正的坑是「链接点击开的新标签」不在 daemon 的同步清单里，那条路由前端「活动标签变化 → 补发 tab/switch」兜住（useBrowser.ts）。
+  const reconnectViewer = async () => {
+    if (stream.isConnected()) {
+      stream.disconnect();
+      await stream.connect();
+    }
+  };
+
+  const applyViewport = async (widthValue: unknown, heightValue: unknown) => {
+    const width = Math.max(320, Math.round(Number(widthValue)));
+    const height = Math.max(240, Math.round(Number(heightValue)));
+    if (!Number.isFinite(width) || !Number.isFinite(height)) throw new Error('无效的浏览器视口尺寸');
+    await runOnContextTab(['set', 'viewport', String(width), String(height)]);
+    await reconnectViewer();
+  };
+
+  const pick = async (x: number, y: number): Promise<PickedBrowserElement | null> => {
+    const value = parseJsonOutput(await runOnContextTab(['eval', pickScript(x, y)]));
+    if (!value || typeof value !== 'object') return null;
+    return value as PickedBrowserElement;
+  };
+  stream.setPicker(pick);
+
+  // 用户指针输入的双通路路由状态 + 命令通路串行队列（见 routeUserMouse 头注）。
+  const pointerRoute = createPointerRouteState();
+  let mouseCmdChain: Promise<unknown> = Promise.resolve();
+  const enqueueMouseCmd = (args: string[]): void => {
+    // 串行保序：逐条 await 上一条再跑下一条；失败吞掉（拖拽单帧丢失不该中断整段手势）。
+    mouseCmdChain = mouseCmdChain.then(() => runOnContextTab(['mouse', ...args])).catch(() => {});
+  };
+
+  const result = (frame: BrowserClientFrame, ok: boolean, data?: unknown, error?: string) => send({
+    t: 'browser/result', requestId: frame.requestId, action: frame.t, ok,
+    ...(runContext ? { conversationKey: runContext.conversationKey, runId: runContext.runId, tabId: activeTabId || undefined } : {}),
+    ...(typeof data === 'string' ? { content: data } : data === undefined ? {} : { data }),
+    ...(error ? { error } : {}),
+  });
+
+  return {
+    async handle(frame: BrowserClientFrame) {
+      if (disposed) return;
+      try {
+        switch (frame.t) {
+          case 'browser/context/set': {
+            const conversationKey = String(frame.conversationKey ?? '').trim();
+            if (!conversationKey) throw new Error('缺少浏览器会话标识');
+            runContext = browserRunRegistry.ensure({
+              conversationKey,
+              runId: typeof frame.runId === 'string' ? frame.runId : 'workbench',
+              profileId: 'default',
+              grants: ['observe', 'interact'],
+            });
+            const tabId = await claimContextTab();
+            send({
+              t: 'browser/context/changed',
+              requestId: frame.requestId,
+              conversationKey: runContext.conversationKey,
+              runId: runContext.runId,
+              tabId,
+            });
+            return;
+          }
+          case 'browser/start': {
+            requireContext();
+            const info = await ensureStarted();
+            await claimContextTab();
+            const viewport = frame.viewport as { width?: number; height?: number } | undefined;
+            if (viewport?.width && viewport?.height) await applyViewport(viewport.width, viewport.height);
+            send({
+              t: 'browser/started',
+              requestId: frame.requestId,
+              streamPort: info.streamPort,
+              pid: info.pid,
+              sessionId: sessionName,
+              conversationKey: runContext?.conversationKey,
+              runId: runContext?.runId,
+            });
+            result(frame, true, info);
+            return;
+          }
+          case 'browser/stop':
+            stream.disconnect(); await stopDaemon(sessionName); send({ t: 'browser/stopped', requestId: frame.requestId }); return;
+          case 'browser/profile/clear':
+            stream.disconnect(); await stopDaemon(sessionName); await rm(browserProfilePath('default'), { recursive: true, force: true });
+            send({ t: 'browser/stopped', requestId: frame.requestId }); result(frame, true); return;
+          case 'browser/mode':
+            stream.setMode(frame.mode as 'browse' | 'assist' | 'auto'); result(frame, true); return;
+          case 'browser/control/get':
+            requireContext();
+            send({ t: 'browser/control/changed', state: stream.getControlState() }); return;
+          case 'browser/control/takeover':
+            stream.requestHuman('manual', {
+              runId: runContext?.runId,
+              conversationKey: runContext?.conversationKey,
+              ownerId: 'local-owner',
+            });
+            result(frame, true, stream.getControlState()); return;
+          case 'browser/control/resume': {
+            const ok = stream.resumeControl(typeof frame.resumeToken === 'string' ? frame.resumeToken : undefined);
+            result(frame, ok, stream.getControlState(), ok ? undefined : '浏览器接管请求已失效'); return;
+          }
+          case 'browser/control/stop': {
+            const ok = stream.stopControl(typeof frame.resumeToken === 'string' ? frame.resumeToken : undefined);
+            result(frame, ok, stream.getControlState(), ok ? undefined : '浏览器接管请求已失效'); return;
+          }
+          case 'human-intervention/resume': {
+            const ok = stream.resumeIntervention({
+              id: String(frame.interventionId ?? ''),
+              resumeToken: String(frame.resumeToken ?? ''),
+              ownerId: String(frame.ownerId ?? ''),
+              conversationKey: String(frame.conversationKey ?? ''),
+              runId: String(frame.runId ?? ''),
+            });
+            result(frame, ok, stream.getControlState(), ok ? undefined : '浏览器人工介入请求已失效'); return;
+          }
+          case 'human-intervention/cancel': {
+            const ok = stream.cancelIntervention({
+              id: String(frame.interventionId ?? ''),
+              resumeToken: String(frame.resumeToken ?? ''),
+              ownerId: String(frame.ownerId ?? ''),
+              conversationKey: String(frame.conversationKey ?? ''),
+              runId: String(frame.runId ?? ''),
+            });
+            result(frame, ok, stream.getControlState(), ok ? undefined : '浏览器人工介入请求已失效'); return;
+          }
+          case 'browser/user/input': {
+            requireContext();
+            await claimContextTab();
+            const type = String(frame.eventType ?? '');
+            if (!type.startsWith('mouse')) { stream.send({ type: 'input_keyboard', ...frame, t: undefined }); return; }
+            // 鼠标：hover/单击/滚轮走快通路 input_mouse；拖拽手势改走 daemon 带 buttons 的命令通路。
+            const plan = routeUserMouse(pointerRoute, frame, Date.now());
+            for (const f of plan.stream) stream.send(f);
+            for (const args of plan.commands) enqueueMouseCmd(args);
+            return;
+          }
+          case 'browser/navigate': {
+            requireContext();
+            await ensureStarted();
+            const url = normalizeUrl(String(frame.url ?? ''));
+            if (frame.newTab) await createContextTab(url);
+            else {
+              await claimContextTab();
+              await run(['open', url]);
+            }
+            stream.setCurrentUrl(url); result(frame, true); return;
+          }
+          case 'browser/back': await runOnContextTab(['back']); result(frame, true); return;
+          case 'browser/forward': await runOnContextTab(['forward']); result(frame, true); return;
+          case 'browser/reload': await runOnContextTab(['reload']); result(frame, true); return;
+          case 'browser/tab/new': {
+            const tabId = await createContextTab(
+              frame.url ? normalizeUrl(String(frame.url)) : 'about:blank',
+            );
+            result(frame, true, { tabId });
+            return;
+          }
+          case 'browser/tab/close': {
+            const context = requireContext();
+            const tabId = String(frame.tabId ?? context.activeTabId ?? '');
+            if (!tabId) throw new Error('无效的标签标识');
+            browserRunRegistry.assertTab(context, tabId);
+            await run(['tab', 'close', tabId]);
+            browserRunRegistry.releaseTab(context.conversationKey, tabId);
+            activeTabId = context.activeTabId ?? '';
+            result(frame, true);
+            return;
+          }
+          case 'browser/tab/switch': {
+            const context = requireContext();
+            const tabId = String(frame.tabId ?? '');
+            if (!tabId) throw new Error('无效的标签标识');
+            browserRunRegistry.assertTab(context, tabId);
+            await run(['tab', tabId]);
+            context.activeTabId = tabId;
+            activeTabId = tabId;
+            result(frame, true);
+            return;
+          }
+          case 'browser/tabs': {
+            const context = requireContext();
+            const bin = await resolveAgentBrowserPath();
+            if (!bin) throw new Error('未找到内置 agent-browser');
+            const tabs = await listBrowserTabs({
+              sessionName,
+              bin,
+              globals: sessionArgs(sessionName),
+              runner: spawnRunner,
+            });
+            const claimed = new Set(browserRunRegistry.claimedTabs(context.conversationKey));
+            result(frame, true, tabs.filter((tab) => claimed.has(tab.id)));
+            return;
+          }
+          case 'browser/pick': {
+            const element = await pick(Number(frame.x), Number(frame.y));
+            if (!element) { send({ t: 'browser/picked', requestId: frame.requestId, error: '该位置没有可选元素' }); return; }
+            stream.markElement(element);
+            const item = element as PickedBrowserElement & { role?: string; accessibleName?: string };
+            send({ t: 'browser/picked', requestId: frame.requestId, reference: {
+              id: randomUUID(), url: stream.getCurrentUrl(), tabId: activeTabId, selector: item.ref,
+              tag: item.tag, role: item.role, accessibleName: item.accessibleName, text: item.text,
+              attributes: item.attributes ?? {}, boundingBox: item.boundingBox, pageRevision, capturedAt: Date.now(),
+            }, ...element }); return;
+          }
+          case 'browser/mark/element': {
+            const raw = parseJsonOutput(await runOnContextTab(['eval', `(() => { const el=document.querySelector(${JSON.stringify(String(frame.selector))}); if(!el)return null; const r=el.getBoundingClientRect(); return {ref:${JSON.stringify(String(frame.selector))},tag:el.tagName.toLowerCase(),text:String(el.innerText||el.textContent||'').trim().slice(0,1000),boundingBox:{x:r.x,y:r.y,width:r.width,height:r.height}} })()`]));
+            if (!raw || typeof raw !== 'object') throw new Error('未找到该元素');
+            stream.markElement({ ...(raw as PickedBrowserElement), label: typeof frame.label === 'string' ? frame.label : undefined, color: typeof frame.color === 'string' ? frame.color : undefined });
+            result(frame, true); return;
+          }
+          case 'browser/mark/clear': stream.clearMarkedElements(); result(frame, true); return;
+          case 'browser/viewport': await ensureStarted(); await applyViewport(frame.width, frame.height); result(frame, true); return;
+          case 'browser/dialog/accept': await runOnContextTab(frame.text ? ['dialog', 'accept', '--text', String(frame.text)] : ['dialog', 'accept']); result(frame, true); return;
+          case 'browser/dialog/dismiss': await runOnContextTab(['dialog', 'dismiss']); result(frame, true); return;
+          case 'browser/snapshot': {
+            const tree = await runOnContextTab(['snapshot']); send({ t: 'browser/snapshot', requestId: frame.requestId, tree }); return;
+          }
+          case 'browser/getContent': {
+            const text = await runOnContextTab(['get', 'text', String(frame.selector ?? 'body')]); send({ t: 'browser/content', requestId: frame.requestId, text }); return;
+          }
+          case 'browser/click': await runOnContextTab(['click', String(frame.selector)]); result(frame, true); return;
+          case 'browser/fill': await runOnContextTab(['fill', String(frame.selector), String(frame.value ?? '')]); result(frame, true); return;
+          case 'browser/type': await runOnContextTab(['type', String(frame.selector), String(frame.value ?? '')]); result(frame, true); return;
+          case 'browser/hover': await runOnContextTab(['hover', String(frame.selector)]); result(frame, true); return;
+          case 'browser/scroll': {
+            const args = frame.selector ? ['scrollintoview', String(frame.selector)] : ['scroll', String(frame.direction ?? 'down'), String(Math.abs(Number(frame.y ?? 600)))];
+            await runOnContextTab(args); result(frame, true); return;
+          }
+          case 'browser/select': await runOnContextTab(['select', String(frame.selector), ...(Array.isArray(frame.value) ? frame.value.map(String) : [String(frame.value ?? '')])]); result(frame, true); return;
+          case 'browser/press': await runOnContextTab(['press', String(frame.key)]); result(frame, true); return;
+          case 'browser/wait': await runOnContextTab(frame.selector ? ['wait', String(frame.selector)] : frame.text ? ['wait', '--text', String(frame.text)] : ['wait', String(frame.ms ?? 500)]); result(frame, true); return;
+          case 'browser/eval': result(frame, true, parseJsonOutput(await runOnContextTab(['eval', String(frame.js ?? '')]))); return;
+          case 'browser/upload': await runOnContextTab(['upload', String(frame.selector), ...(Array.isArray(frame.files) ? frame.files.map(String) : [])]); result(frame, true); return;
+          case 'browser/drag': await runOnContextTab(['drag', String(frame.fromSelector), String(frame.toSelector)]); result(frame, true); return;
+          case 'browser/screenshot': {
+            const args = ['screenshot', ...(frame.path ? [String(frame.path)] : []), ...(frame.annotate ? ['--annotate'] : []), ...(frame.fullPage ? ['--full'] : [])];
+            const output = await runOnContextTab(args); send({ t: 'browser/screenshot', requestId: frame.requestId, path: typeof frame.path === 'string' ? frame.path : undefined, data: output }); return;
+          }
+          case 'browser/pdf': { const output = await runOnContextTab(['pdf', String(frame.path ?? '')]); result(frame, true, output); return; }
+          case 'browser/cookies/get': send({ t: 'browser/cookies', requestId: frame.requestId, cookies: (parseJsonOutput(await runOnContextTab(['cookies', 'get'])) as never[]) ?? [] }); return;
+          case 'browser/cookies/clear': await runOnContextTab(['cookies', 'clear']); result(frame, true); return;
+          case 'browser/device': await runOnContextTab(frame.preset ? ['set', 'device', String(frame.preset)] : ['set', 'viewport', String(frame.width), String(frame.height)]); result(frame, true); return;
+          case 'browser/media': await runOnContextTab(['set', 'media', String(frame.scheme ?? 'no-preference')]); result(frame, true); return;
+          case 'browser/record/start': await runOnContextTab(['record', 'start', String(frame.path ?? '')]); result(frame, true); return;
+          case 'browser/record/stop': result(frame, true, await runOnContextTab(['record', 'stop'])); return;
+          case 'browser/debug/console':
+          case 'browser/debug/errors':
+          case 'browser/debug/network':
+          case 'browser/debug/styles':
+          case 'browser/debug/vitals':
+          case 'browser/debug/trace/start':
+          case 'browser/debug/trace/stop': {
+            await claimContextTab();
+            const browserConfig = await configService.browser();
+            if (!browserConfig.developerMode) throw new Error('请先在“设置 → 浏览器”开启完整 CDP Developer Mode');
+            requireAllowedDeveloperOrigin(stream.getCurrentUrl(), browserConfig.allowedCdpOrigins);
+            if (frame.t === 'browser/debug/console') result(frame, true, await run(['console']));
+            else if (frame.t === 'browser/debug/errors') result(frame, true, await run(['errors']));
+            else if (frame.t === 'browser/debug/network') result(frame, true, parseJsonOutput(await run(['network', 'requests', '--json'])));
+            else if (frame.t === 'browser/debug/styles') result(frame, true, parseJsonOutput(await run(['get', 'styles', String(frame.selector ?? 'body'), '--json'])));
+            else if (frame.t === 'browser/debug/vitals') result(frame, true, parseJsonOutput(await run(['vitals', ...(frame.url ? [String(frame.url)] : []), '--json'])));
+            else if (frame.t === 'browser/debug/trace/start') { await run(['trace', 'start']); result(frame, true); }
+            else result(frame, true, await run(['trace', 'stop', ...(frame.path ? [String(frame.path)] : [])]));
+            return;
+          }
+          default:
+            result(frame, false, undefined, `暂不支持浏览器操作：${frame.t}`);
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        result(frame, false, undefined, message);
+        send({ t: 'browser/error', message });
+      }
+    },
+    dispose() { disposed = true; unsubscribe(); },
+  };
+}
+
+export type { BrowserClientFrame };