minimal-agent 0.13.0 → 0.14.0

package/src/tools/webbrowser/browserStream.ts

@@ -1,6 +1,13 @@
+import type {
+  HumanInterventionCredential,
+  HumanInterventionEvent,
+} from '../../humanIntervention.js';
+import {
+  BrowserControlCoordinator,
+  type BrowserControlState,
+  type BrowserHumanReason,
+} from './browserControl.js';
 import { getStreamPort, waitForStreamPort } from './browserDaemon.js';
-import { BrowserControlCoordinator, type BrowserControlState, type BrowserHumanReason } from './browserControl.js';
-import type { HumanInterventionCredential, HumanInterventionEvent } from '../../humanIntervention.js';
 
 export type BrowserMode = 'browse' | 'assist' | 'auto';
 export type PickedBrowserElement = {
@@ -12,7 +19,10 @@ export type PickedBrowserElement = {
   boundingBox?: { x: number; y: number; width: number; height: number };
   attributes?: Record<string, string>;
 };
-export type MarkedBrowserElement = PickedBrowserElement & { label?: string; color?: string };
+export type MarkedBrowserElement = PickedBrowserElement & {
+  label?: string;
+  color?: string;
+};
 export type BrowserStreamEvent = Record<string, unknown> & { type: string };
 
 export interface BrowserStream {
@@ -21,6 +31,8 @@ export interface BrowserStream {
   send(frame: unknown): void;
   dispose(): void;
   subscribe(listener: (event: BrowserStreamEvent) => void): () => void;
+  /** Publish a local engine event to Desktop observers (downloads/CDP lifecycle). */
+  publish(event: BrowserStreamEvent): void;
   getStatus(): Record<string, unknown>;
   getPort(): number | null;
   isConnected(): boolean;
@@ -29,7 +41,9 @@ export interface BrowserStream {
   getCurrentUrl(): string;
   setCurrentUrl(url: string): void;
   pickElement(x: number, y: number): Promise<PickedBrowserElement | null>;
-  setPicker(picker: (x: number, y: number) => Promise<PickedBrowserElement | null>): void;
+  setPicker(
+    picker: (x: number, y: number) => Promise<PickedBrowserElement | null>,
+  ): void;
   markElement(element: MarkedBrowserElement): void;
   clearMarkedElements(): void;
   /** Remove a single marked element by ref (does not affect other marks). */
@@ -41,7 +55,10 @@ export interface BrowserStream {
    * 候选只进对话的话，盯着浏览器的人看到的是 AI 忽然停住；推到面板上他可以一眼看出
    * AI 卡在哪，并点一下把正确的那个喂回去 —— 人的判断比模型再猜一轮便宜得多。
    */
-  setAgentCandidates(candidates: Array<{ ref: string; line: string }>, selector?: string): void;
+  setAgentCandidates(
+    candidates: Array<{ ref: string; line: string }>,
+    selector?: string,
+  ): void;
   /**
    * 发布一次动作**做完之后**的生效结论（只在结论不是「变了」时调用）。
    *
@@ -55,14 +72,34 @@ export interface BrowserStream {
     maskedLines?: number;
   }): void;
   /** C4：发布 AI 指针语义终点给可见面板（宿主用 CSS 过渡平滑移动 + 渲染点击动效）。 */
-  setAgentPointer(pointer: { x: number; y: number; visible: boolean; action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle'; elementRef?: string }): void;
+  setAgentPointer(pointer: {
+    x: number;
+    y: number;
+    visible: boolean;
+    action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle';
+    elementRef?: string;
+  }): void;
   getControlState(): BrowserControlState;
   /** 订阅控制权状态变化（立即以当前状态回调一次）；供工具即时感知「用户接管 / 进入 waiting_human」以掐断在跑动作。 */
   subscribeControl(listener: (state: BrowserControlState) => void): () => void;
-  subscribeInterventions(listener: (event: HumanInterventionEvent) => void): () => void;
-  beginAgentAction(input: { runId: string; conversationKey?: string; action?: string }): void;
+  subscribeInterventions(
+    listener: (event: HumanInterventionEvent) => void,
+  ): () => void;
+  beginAgentAction(input: {
+    runId: string;
+    conversationKey?: string;
+    action?: string;
+  }): void;
   finishAgentAction(runId: string): void;
-  requestHuman(reason: BrowserHumanReason, input?: { runId?: string; conversationKey?: string; ownerId?: string; summary?: string }): string;
+  requestHuman(
+    reason: BrowserHumanReason,
+    input?: {
+      runId?: string;
+      conversationKey?: string;
+      ownerId?: string;
+      summary?: string;
+    },
+  ): string;
   takeControl(input?: { runId?: string; conversationKey?: string }): void;
   resumeControl(token?: string): boolean;
   stopControl(token?: string): boolean;
@@ -78,25 +115,46 @@ class SessionBrowserStream implements BrowserStream {
   private port: number | null = null;
   private mode: BrowserMode = 'browse';
   private currentUrl = 'about:blank';
-  private status: Record<string, unknown> = { connected: false, screencasting: false };
-  private picker?: (x: number, y: number) => Promise<PickedBrowserElement | null>;
+  private status: Record<string, unknown> = {
+    connected: false,
+    screencasting: false,
+  };
+  private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
+  private reconnectAttempt = 0;
+  private disposed = false;
+  private picker?: (
+    x: number,
+    y: number,
+  ) => Promise<PickedBrowserElement | null>;
   private marked: MarkedBrowserElement[] = [];
   private control: BrowserControlCoordinator;
 
   constructor(private readonly sessionName: string) {
-    this.control = new BrowserControlCoordinator({ conversationKey: `browser:${sessionName}` });
-    this.control.subscribe((state) => this.emit({ type: 'control_state', state }));
-    this.control.subscribeInterventions((event) => this.emit({
-      type: event.type === 'started' ? 'human_intervention_started' : 'human_intervention_resolved',
-      event,
-    }));
+    this.control = new BrowserControlCoordinator({
+      conversationKey: `browser:${sessionName}`,
+    });
+    this.control.subscribe((state) =>
+      this.emit({ type: 'control_state', state }),
+    );
+    this.control.subscribeInterventions((event) =>
+      this.emit({
+        type:
+          event.type === 'started'
+            ? 'human_intervention_started'
+            : 'human_intervention_resolved',
+        event,
+      }),
+    );
   }
 
   async connect(): Promise<void> {
+    this.disposed = false;
     if (this.socket?.readyState === WebSocket.OPEN) return;
     if (this.connecting) return this.connecting;
     this.connecting = (async () => {
-      this.port = (await getStreamPort(this.sessionName)) ?? (await waitForStreamPort(this.sessionName));
+      this.port =
+        (await getStreamPort(this.sessionName)) ??
+        (await waitForStreamPort(this.sessionName));
       await new Promise<void>((resolve, reject) => {
         const socket = new WebSocket(`ws://127.0.0.1:${this.port}`);
         const timer = setTimeout(() => {
@@ -106,6 +164,7 @@ class SessionBrowserStream implements BrowserStream {
         socket.onopen = () => {
           clearTimeout(timer);
           this.socket = socket;
+          this.reconnectAttempt = 0;
           this.status = { ...this.status, connected: true };
           this.emit({ type: 'status', ...this.status });
           resolve();
@@ -116,40 +175,86 @@ class SessionBrowserStream implements BrowserStream {
         };
         socket.onclose = () => {
           if (this.socket === socket) this.socket = null;
-          this.status = { ...this.status, connected: false, screencasting: false };
+          this.status = {
+            ...this.status,
+            connected: false,
+            screencasting: false,
+          };
           this.emit({ type: 'status', ...this.status });
+          this.scheduleReconnect();
         };
         socket.onmessage = (event) => this.onMessage(event.data);
       });
-    })().finally(() => { this.connecting = null; });
+    })().finally(() => {
+      this.connecting = null;
+    });
     return this.connecting;
   }
 
-  disconnect(): void { this.socket?.close(); this.socket = null; }
-  dispose(): void { this.disconnect(); this.listeners.clear(); }
+  disconnect(): void {
+    this.disposed = true;
+    if (this.reconnectTimer) clearTimeout(this.reconnectTimer);
+    this.reconnectTimer = null;
+    this.socket?.close();
+    this.socket = null;
+  }
+  dispose(): void {
+    this.disconnect();
+    this.listeners.clear();
+  }
   send(frame: unknown): void {
-    if (this.socket?.readyState === WebSocket.OPEN) this.socket.send(JSON.stringify(frame));
+    if (this.socket?.readyState === WebSocket.OPEN)
+      this.socket.send(JSON.stringify(frame));
   }
   subscribe(listener: (event: BrowserStreamEvent) => void): () => void {
     this.listeners.add(listener);
     return () => this.listeners.delete(listener);
   }
-  getStatus(): Record<string, unknown> { return this.status; }
-  getPort(): number | null { return this.port; }
-  isConnected(): boolean { return this.socket?.readyState === WebSocket.OPEN; }
-  setMode(mode: BrowserMode): void { this.mode = mode; this.emit({ type: 'mode', mode }); }
-  getMode(): BrowserMode { return this.mode; }
-  getCurrentUrl(): string { return this.currentUrl; }
-  setCurrentUrl(url: string): void { this.currentUrl = url; this.emit({ type: 'url', url }); }
-  setPicker(picker: (x: number, y: number) => Promise<PickedBrowserElement | null>): void { this.picker = picker; }
+  publish(event: BrowserStreamEvent): void {
+    this.emit(event);
+  }
+  getStatus(): Record<string, unknown> {
+    return this.status;
+  }
+  getPort(): number | null {
+    return this.port;
+  }
+  isConnected(): boolean {
+    return this.socket?.readyState === WebSocket.OPEN;
+  }
+  setMode(mode: BrowserMode): void {
+    this.mode = mode;
+    this.emit({ type: 'mode', mode });
+  }
+  getMode(): BrowserMode {
+    return this.mode;
+  }
+  getCurrentUrl(): string {
+    return this.currentUrl;
+  }
+  setCurrentUrl(url: string): void {
+    this.currentUrl = url;
+    this.emit({ type: 'url', url });
+  }
+  setPicker(
+    picker: (x: number, y: number) => Promise<PickedBrowserElement | null>,
+  ): void {
+    this.picker = picker;
+  }
   pickElement(x: number, y: number): Promise<PickedBrowserElement | null> {
     return this.picker ? this.picker(x, y) : Promise.resolve(null);
   }
   markElement(element: MarkedBrowserElement): void {
-    this.marked = [...this.marked.filter((item) => item.ref !== element.ref), element];
+    this.marked = [
+      ...this.marked.filter((item) => item.ref !== element.ref),
+      element,
+    ];
     this.emit({ type: 'marked_elements', elements: this.marked });
   }
-  clearMarkedElements(): void { this.marked = []; this.emit({ type: 'marked_elements', elements: [] }); }
+  clearMarkedElements(): void {
+    this.marked = [];
+    this.emit({ type: 'marked_elements', elements: [] });
+  }
   unmarkElement(ref: string): void {
     const next = this.marked.filter((item) => item.ref !== ref);
     if (next.length !== this.marked.length) {
@@ -160,7 +265,10 @@ class SessionBrowserStream implements BrowserStream {
   setAgentActing(acting: boolean, action?: string, target?: string): void {
     this.emit({ type: 'agent_acting', acting, action, target });
   }
-  setAgentCandidates(candidates: Array<{ ref: string; line: string }>, selector?: string): void {
+  setAgentCandidates(
+    candidates: Array<{ ref: string; line: string }>,
+    selector?: string,
+  ): void {
     this.emit({ type: 'agent_candidates', candidates, selector });
   }
   setAgentOutcome(outcome: {
@@ -171,39 +279,102 @@ class SessionBrowserStream implements BrowserStream {
   }): void {
     this.emit({ type: 'agent_outcome', ...outcome });
   }
-  setAgentPointer(pointer: { x: number; y: number; visible: boolean; action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle'; elementRef?: string }): void {
+  setAgentPointer(pointer: {
+    x: number;
+    y: number;
+    visible: boolean;
+    action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle';
+    elementRef?: string;
+  }): void {
     // 只发语义终点（目标元素中心 + 动作类型）；平滑移动交给面板 CSS 过渡，内核不做逐帧插值（避免 WS 洪泛 + 双层缓动打架）。
     this.emit({ type: 'agent_pointer', ...pointer });
   }
-  getControlState(): BrowserControlState { return this.control.getState(); }
-  subscribeControl(listener: (state: BrowserControlState) => void): () => void { return this.control.subscribe(listener); }
-  subscribeInterventions(listener: (event: HumanInterventionEvent) => void): () => void { return this.control.subscribeInterventions(listener); }
-  beginAgentAction(input: { runId: string; conversationKey?: string; action?: string }): void { this.control.beginAgent(input); }
-  finishAgentAction(runId: string): void { this.control.finishAgent(runId); }
-  requestHuman(reason: BrowserHumanReason, input?: { runId?: string; conversationKey?: string; ownerId?: string; summary?: string }): string { return this.control.requestHuman(reason, input); }
-  takeControl(input?: { runId?: string; conversationKey?: string }): void { this.control.takeInteractive(input); }
-  resumeControl(token?: string): boolean { return this.control.resume(token); }
-  stopControl(token?: string): boolean { return this.control.stop(token); }
+  getControlState(): BrowserControlState {
+    return this.control.getState();
+  }
+  subscribeControl(listener: (state: BrowserControlState) => void): () => void {
+    return this.control.subscribe(listener);
+  }
+  subscribeInterventions(
+    listener: (event: HumanInterventionEvent) => void,
+  ): () => void {
+    return this.control.subscribeInterventions(listener);
+  }
+  beginAgentAction(input: {
+    runId: string;
+    conversationKey?: string;
+    action?: string;
+  }): void {
+    this.control.beginAgent(input);
+  }
+  finishAgentAction(runId: string): void {
+    this.control.finishAgent(runId);
+  }
+  requestHuman(
+    reason: BrowserHumanReason,
+    input?: {
+      runId?: string;
+      conversationKey?: string;
+      ownerId?: string;
+      summary?: string;
+    },
+  ): string {
+    return this.control.requestHuman(reason, input);
+  }
+  takeControl(input?: { runId?: string; conversationKey?: string }): void {
+    this.control.takeInteractive(input);
+  }
+  resumeControl(token?: string): boolean {
+    return this.control.resume(token);
+  }
+  stopControl(token?: string): boolean {
+    return this.control.stop(token);
+  }
   resumeIntervention(credential: HumanInterventionCredential): boolean {
     return this.control.resumeIntervention(credential);
   }
   cancelIntervention(credential: HumanInterventionCredential): boolean {
     return this.control.cancelIntervention(credential);
   }
-  waitForHuman(signal?: AbortSignal): Promise<'resumed' | 'stopped'> { return this.control.waitForHuman(signal); }
+  waitForHuman(signal?: AbortSignal): Promise<'resumed' | 'stopped'> {
+    return this.control.waitForHuman(signal);
+  }
 
   private emit(event: BrowserStreamEvent): void {
-    for (const listener of this.listeners) listener(event);
+    for (const listener of this.listeners) {
+      try {
+        listener(event);
+      } catch {
+        // A disconnected/failed UI observer must not break browser control or
+        // prevent the remaining observers from receiving the event.
+      }
+    }
+  }
+
+  private scheduleReconnect(): void {
+    if (this.disposed || !this.listeners.size || this.reconnectTimer) return;
+    const delay = Math.min(10_000, 250 * 2 ** this.reconnectAttempt++);
+    this.reconnectTimer = setTimeout(() => {
+      this.reconnectTimer = null;
+      void this.connect().catch(() => this.scheduleReconnect());
+    }, delay);
   }
   private onMessage(data: unknown): void {
     try {
-      const text = typeof data === 'string' ? data : Buffer.from(data as ArrayBuffer).toString('utf8');
+      const text =
+        typeof data === 'string'
+          ? data
+          : Buffer.from(data as ArrayBuffer).toString('utf8');
       const event = JSON.parse(text) as BrowserStreamEvent;
-      if (event.type === 'url' && typeof event.url === 'string') this.currentUrl = event.url;
+      if (event.type === 'url' && typeof event.url === 'string')
+        this.currentUrl = event.url;
       if (event.type === 'status') this.status = { ...this.status, ...event };
       this.emit(event);
     } catch (error) {
-      this.emit({ type: 'error', message: error instanceof Error ? error.message : String(error) });
+      this.emit({
+        type: 'error',
+        message: error instanceof Error ? error.message : String(error),
+      });
     }
   }
 }
@@ -211,6 +382,9 @@ class SessionBrowserStream implements BrowserStream {
 const streams = new Map<string, SessionBrowserStream>();
 export function getBrowserStream(sessionName = 'minimal-agent'): BrowserStream {
   let stream = streams.get(sessionName);
-  if (!stream) { stream = new SessionBrowserStream(sessionName); streams.set(sessionName, stream); }
+  if (!stream) {
+    stream = new SessionBrowserStream(sessionName);
+    streams.set(sessionName, stream);
+  }
   return stream;
 }

package/src/tools/webbrowser/browserTabs.js

@@ -1,5 +1,5 @@
-import { browserService } from './browserService.js';
 import { browserRunRegistry, } from './browserRunRegistry.js';
+import { browserService } from './browserService.js';
 export function parseTabList(text) {
     try {
         const parsed = JSON.parse(text);
@@ -11,13 +11,17 @@ export function parseTabList(text) {
                     : Number.isInteger(tab.index)
                         ? String(tab.index)
                         : '';
-            return id ? [{
-                    id,
-                    active: Boolean(tab.active),
-                    ...(typeof tab.url === 'string' ? { url: tab.url } : {}),
-                    ...(typeof tab.title === 'string' ? { title: tab.title } : {}),
-                    ...(typeof tab.type === 'string' ? { type: tab.type } : {}),
-                }] : [];
+            return id
+                ? [
+                    {
+                        id,
+                        active: Boolean(tab.active),
+                        ...(typeof tab.url === 'string' ? { url: tab.url } : {}),
+                        ...(typeof tab.title === 'string' ? { title: tab.title } : {}),
+                        ...(typeof tab.type === 'string' ? { type: tab.type } : {}),
+                    },
+                ]
+                : [];
         });
     }
     catch {
@@ -40,7 +44,9 @@ function parseListedTabs(result) {
     let reportedCount = 0;
     try {
         const parsed = JSON.parse(result.stdout);
-        reportedCount = Array.isArray(parsed.data?.tabs) ? parsed.data.tabs.length : 0;
+        reportedCount = Array.isArray(parsed.data?.tabs)
+            ? parsed.data.tabs.length
+            : 0;
     }
     catch {
         throw new BrowserTabError('browser_protocol_mismatch', '浏览器标签列表不是有效 JSON');
@@ -108,22 +114,30 @@ export async function ensureConversationTab(input) {
             tabId = claimed.find((candidate) => tabs.some((tab) => tab.id === candidate));
         }
         if (!tabId) {
-            const hasClaimedTabs = browserRunRegistry.claimedTabs(input.context.conversationKey).length > 0;
+            const hasClaimedTabs = browserRunRegistry.claimedTabs(input.context.conversationKey).length >
+                0;
             const reusableBlank = !hasClaimedTabs
-                ? tabs.find((tab) => tab.url === 'about:blank' && browserRunRegistry.ownerOf(tab.id) === undefined)
+                ? tabs.find((tab) => tab.url === 'about:blank' &&
+                    browserRunRegistry.ownerOf(tab.id) === undefined)
                 : undefined;
             if (reusableBlank) {
                 tabId = reusableBlank.id;
             }
             else {
                 const beforeIds = new Set(tabs.map((tab) => tab.id));
-                const created = await run([...input.globals, 'tab', 'new', 'about:blank']);
+                const created = await run([
+                    ...input.globals,
+                    'tab',
+                    'new',
+                    'about:blank',
+                ]);
                 if (created.code !== 0) {
                     throw new BrowserTabError('tab_bootstrap_failed', created.stderr || created.stdout || '无法创建会话浏览器标签');
                 }
                 const after = parseListedTabs(await run([...input.globals, 'tab', 'list', '--json']));
                 const added = after.filter((tab) => !beforeIds.has(tab.id));
-                tabId = (added.find((candidate) => candidate.active) ?? added.at(-1))?.id;
+                tabId = (added.find((candidate) => candidate.active) ?? added.at(-1))
+                    ?.id;
                 if (!tabId) {
                     throw new BrowserTabError('tab_bootstrap_failed', '新标签创建成功，但 Runtime 未返回其标签 ID');
                 }
@@ -139,5 +153,24 @@ export async function ensureConversationTab(input) {
         return tabId;
     });
 }
-/** Compatibility alias for callers compiled against the previous helper name. */
-export const claimConversationTab = ensureConversationTab;
+export async function claimConversationTabById(input, tabId) {
+    return browserService.exclusive({
+        sessionName: input.sessionName,
+        bin: input.bin,
+        runner: input.runner,
+        signal: input.signal,
+    }, async (run) => {
+        const tabs = parseListedTabs(await run([...input.globals, 'tab', 'list', '--json']));
+        if (!tabs.some((tab) => tab.id === tabId)) {
+            throw new BrowserTabError('tab_not_found', `浏览器标签不存在：${tabId}`);
+        }
+        try {
+            browserRunRegistry.claimTab(input.context, tabId);
+        }
+        catch (error) {
+            throw new BrowserTabError('tab_owned_by_other_conversation', error instanceof Error ? error.message : String(error));
+        }
+        assertSwitched(await run([...input.globals, 'tab', tabId]));
+        return tabId;
+    });
+}