minimal-agent 0.13.0 → 0.14.0

package/src/tools/webbrowser/browserAutomation.ts

@@ -0,0 +1,467 @@
+/**
+ * Persistent CDP transport used by BrowserBatch and BrowserCDP.
+ *
+ * The browser daemon is deliberately still owned by agent-browser; this module
+ * only replaces the short lived command clients used for each automation step.
+ */
+import { mkdir, stat } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+
+import type { AgentBrowserRunner } from './browser.js';
+import { sessionArgs } from './browser.js';
+import { ensureDaemon } from './browserDaemon.js';
+
+export type CdpEvent = {
+  sequence: number;
+  method: string;
+  params: Record<string, unknown>;
+  sessionId?: string;
+  at: number;
+};
+export type BrowserDownload = {
+  id: string;
+  url: string;
+  filename: string;
+  path: string;
+  state: 'start' | 'progress' | 'complete' | 'failed' | 'canceled';
+  receivedBytes?: number;
+  totalBytes?: number;
+  mime?: string;
+  error?: string;
+};
+
+export const BROWSER_TAB_MARKER_KEY = '__minimalAgentTabMarker';
+
+type Pending = {
+  resolve: (value: unknown) => void;
+  reject: (reason: Error) => void;
+  timer: ReturnType<typeof setTimeout>;
+  onAbort?: () => void;
+  signal?: AbortSignal;
+};
+
+class CdpConnection {
+  private socket: WebSocket | null = null;
+  private opening: Promise<void> | null = null;
+  private pageTargetId: string | undefined;
+  private pageSessionId: string | undefined;
+  private id = 0;
+  private pending = new Map<number, Pending>();
+  private events: CdpEvent[] = [];
+  private sequence = 0;
+  private listeners = new Set<(event: CdpEvent) => void>();
+
+  constructor(private readonly url: string) {}
+
+  /**
+   * A connection is page-scoped.  It must not be put back in a process-wide
+   * pool and rebound by another conversation while this caller still holds it.
+   */
+  close(): void {
+    const socket = this.socket;
+    this.socket = null;
+    this.pageTargetId = undefined;
+    this.pageSessionId = undefined;
+    socket?.close();
+  }
+
+  get attachedPageSessionId(): string | undefined {
+    return this.pageSessionId;
+  }
+
+  get attachedPageTargetId(): string | undefined {
+    return this.pageTargetId;
+  }
+
+  async evaluateInFrame(
+    frameId: string,
+    expression: string,
+    signal?: AbortSignal,
+  ): Promise<unknown> {
+    const isolated = (await this.send(
+      'Page.createIsolatedWorld',
+      {
+        frameId,
+        worldName: 'minimal-agent-browser',
+        grantUniveralAccess: false,
+      },
+      undefined,
+      30_000,
+      signal,
+    )) as { executionContextId?: unknown };
+    if (typeof isolated.executionContextId !== 'number')
+      throw new Error(`无法创建 frame execution context：${frameId}`);
+    return this.send(
+      'Runtime.evaluate',
+      {
+        expression,
+        contextId: isolated.executionContextId,
+        returnByValue: true,
+        awaitPromise: true,
+      },
+      undefined,
+      30_000,
+      signal,
+    );
+  }
+
+  async frameOwnerOffset(
+    frameId: string,
+    signal?: AbortSignal,
+  ): Promise<{ x: number; y: number }> {
+    // CDP returns the owner quad in top-level viewport coordinates, including
+    // nested cross-origin frames. Do not add ancestor offsets a second time.
+    const owner = (await this.send(
+      'DOM.getFrameOwner',
+      { frameId },
+      undefined,
+      30_000,
+      signal,
+    )) as { backendNodeId?: unknown };
+    if (typeof owner.backendNodeId !== 'number')
+      throw new Error(`无法取得 frame owner：${frameId}`);
+    const box = (await this.send(
+      'DOM.getBoxModel',
+      { backendNodeId: owner.backendNodeId },
+      undefined,
+      30_000,
+      signal,
+    )) as { model?: { border?: unknown } };
+    const border = box.model?.border;
+    if (!Array.isArray(border) || border.length < 2)
+      throw new Error(`无法取得 frame owner 坐标：${frameId}`);
+    const numbers = border.filter(
+      (value): value is number => typeof value === 'number',
+    );
+    if (numbers.length < 2) throw new Error(`frame owner 坐标无效：${frameId}`);
+    return { x: numbers[0]!, y: numbers[1]! };
+  }
+
+  async open(): Promise<void> {
+    if (this.socket?.readyState === WebSocket.OPEN) return;
+    if (this.opening) return this.opening;
+    this.opening = new Promise<void>((resolve, reject) => {
+      const socket = new WebSocket(this.url);
+      const timer = setTimeout(() => {
+        socket.close();
+        reject(new Error('CDP connection timed out'));
+      }, 10_000);
+      socket.onopen = () => {
+        clearTimeout(timer);
+        this.socket = socket;
+        resolve();
+      };
+      socket.onerror = () => {
+        clearTimeout(timer);
+        reject(new Error('Unable to connect to browser CDP'));
+      };
+      socket.onclose = () => {
+        if (this.socket === socket) this.socket = null;
+        this.pageTargetId = undefined;
+        this.pageSessionId = undefined;
+        for (const [id, request] of this.pending) {
+          clearTimeout(request.timer);
+          if (request.signal && request.onAbort)
+            request.signal.removeEventListener('abort', request.onAbort);
+          request.reject(new Error('CDP connection closed'));
+          this.pending.delete(id);
+        }
+      };
+      socket.onmessage = (message) => this.onMessage(String(message.data));
+    }).finally(() => {
+      this.opening = null;
+    });
+    return this.opening;
+  }
+
+  private onMessage(raw: string): void {
+    let data: Record<string, unknown>;
+    try {
+      data = JSON.parse(raw) as Record<string, unknown>;
+    } catch {
+      return;
+    }
+    if (typeof data.id === 'number') {
+      const request = this.pending.get(data.id);
+      if (!request) return;
+      this.pending.delete(data.id);
+      clearTimeout(request.timer);
+      if (request.signal && request.onAbort)
+        request.signal.removeEventListener('abort', request.onAbort);
+      if (data.error) request.reject(new Error(JSON.stringify(data.error)));
+      else request.resolve(data.result ?? {});
+      return;
+    }
+    if (typeof data.method !== 'string') return;
+    const event: CdpEvent = {
+      sequence: ++this.sequence,
+      method: data.method,
+      params: (data.params ?? {}) as Record<string, unknown>,
+      sessionId:
+        typeof data.sessionId === 'string' ? data.sessionId : undefined,
+      at: Date.now(),
+    };
+    if (
+      event.method === 'Target.detachedFromTarget' &&
+      String(event.params.sessionId ?? '') === this.pageSessionId
+    ) {
+      this.pageTargetId = undefined;
+      this.pageSessionId = undefined;
+    }
+    if (
+      (event.method === 'Target.targetDestroyed' ||
+        event.method === 'Target.targetCrashed') &&
+      String(event.params.targetId ?? '') === this.pageTargetId
+    ) {
+      this.pageTargetId = undefined;
+      this.pageSessionId = undefined;
+    }
+    this.events.push(event);
+    if (this.events.length > 500)
+      this.events.splice(0, this.events.length - 500);
+    for (const listener of this.listeners) {
+      try {
+        listener(event);
+      } catch {
+        // One observer must not break delivery to the remaining CDP consumers.
+      }
+    }
+  }
+
+  async send(
+    method: string,
+    params: Record<string, unknown> = {},
+    target?: string,
+    timeoutMs = 30_000,
+    signal?: AbortSignal,
+  ): Promise<unknown> {
+    await this.open();
+    const socket = this.socket;
+    if (!socket || socket.readyState !== WebSocket.OPEN)
+      throw new Error('CDP connection is unavailable');
+    const id = ++this.id;
+    const pageDomain =
+      /^(Accessibility|DOM|Input|Log|Network|Page|Performance|Runtime|Security)\./.test(
+        method,
+      );
+    const effectiveTarget =
+      target ?? (pageDomain ? this.pageSessionId : undefined);
+    return new Promise((resolve, reject) => {
+      const abortError = () =>
+        signal?.reason instanceof Error
+          ? signal.reason
+          : new Error(`CDP ${method} aborted`);
+      const cleanup = () => {
+        const request = this.pending.get(id);
+        if (!request) return false;
+        this.pending.delete(id);
+        clearTimeout(request.timer);
+        if (request.signal && request.onAbort)
+          request.signal.removeEventListener('abort', request.onAbort);
+        return true;
+      };
+      const onAbort = () => {
+        if (cleanup()) reject(abortError());
+      };
+      const timer = setTimeout(() => {
+        if (cleanup()) reject(new Error(`CDP ${method} timed out`));
+      }, timeoutMs);
+      this.pending.set(id, { resolve, reject, timer, onAbort, signal });
+      if (signal?.aborted) {
+        onAbort();
+        return;
+      }
+      signal?.addEventListener('abort', onAbort, { once: true });
+      try {
+        socket.send(
+          JSON.stringify({
+            id,
+            method,
+            params,
+            ...(effectiveTarget ? { sessionId: effectiveTarget } : {}),
+          }),
+        );
+      } catch (error) {
+        if (cleanup())
+          reject(error instanceof Error ? error : new Error(String(error)));
+      }
+    });
+  }
+
+  async attachToPage(
+    preferredUrl?: string,
+    force = false,
+    preferredMarker?: string,
+  ): Promise<void> {
+    if (this.pageSessionId && !force) return;
+    if (force && this.pageSessionId) {
+      const previousSession = this.pageSessionId;
+      this.pageSessionId = undefined;
+      this.pageTargetId = undefined;
+      await this.send('Target.detachFromTarget', {
+        sessionId: previousSession,
+      }).catch(() => undefined);
+    }
+    const targets = (await this.send('Target.getTargets')) as {
+      targetInfos?: Array<{
+        targetId?: unknown;
+        type?: unknown;
+        url?: unknown;
+      }>;
+    };
+    const pages = (targets.targetInfos ?? []).filter(
+      (target) => target.type === 'page' && typeof target.targetId === 'string',
+    );
+    if (!pages.length) throw new Error('浏览器没有可绑定的页面标签');
+    let selected = preferredMarker
+      ? undefined
+      : preferredUrl
+        ? pages.find((target) => target.url === preferredUrl)
+        : undefined;
+    if (preferredMarker) {
+      const marker = JSON.stringify(preferredMarker);
+      for (const candidate of pages) {
+        const attached = (await this.send('Target.attachToTarget', {
+          targetId: candidate.targetId,
+          flatten: true,
+        })) as { sessionId?: unknown };
+        const sessionId = attached.sessionId;
+        if (typeof sessionId !== 'string') continue;
+        const probe = await this.send(
+          'Runtime.evaluate',
+          {
+            expression: `globalThis[${JSON.stringify(BROWSER_TAB_MARKER_KEY)}] === ${marker}`,
+            returnByValue: true,
+          },
+          sessionId,
+        ).catch(() => undefined);
+        if (
+          (probe as { result?: { value?: unknown } } | undefined)?.result
+            ?.value === true
+        ) {
+          selected = candidate;
+          this.pageTargetId = String(candidate.targetId);
+          this.pageSessionId = sessionId;
+          break;
+        }
+        await this.send('Target.detachFromTarget', { sessionId }).catch(
+          () => undefined,
+        );
+      }
+    }
+    if (preferredMarker) {
+      if (!selected) throw new Error('无法通过会话标记绑定到当前浏览器标签');
+      return;
+    }
+    selected ??= pages[pages.length - 1]!;
+    const attached = (await this.send('Target.attachToTarget', {
+      targetId: selected.targetId,
+      flatten: true,
+    })) as { sessionId?: unknown };
+    if (typeof attached.sessionId !== 'string')
+      throw new Error('浏览器未返回页面 target session');
+    this.pageTargetId = selected.targetId as string;
+    this.pageSessionId = attached.sessionId;
+  }
+
+  read(after = 0, methods?: string[], limit = 100): CdpEvent[] {
+    return this.events
+      .filter(
+        (event) =>
+          event.sequence > after &&
+          (!methods?.length || methods.includes(event.method)),
+      )
+      .slice(-Math.max(1, Math.min(limit, 500)));
+  }
+  subscribe(listener: (event: CdpEvent) => void): () => void {
+    this.listeners.add(listener);
+    return () => this.listeners.delete(listener);
+  }
+}
+
+async function cdpUrl(
+  sessionName: string,
+  bin: string,
+  run: AgentBrowserRunner,
+  signal?: AbortSignal,
+): Promise<string> {
+  await ensureDaemon(sessionName, { signal });
+  const output = await run(
+    bin,
+    [...sessionArgs(sessionName), 'get', 'cdp-url'],
+    signal,
+  );
+  const url = `${output.stdout}\n${output.stderr}`.match(
+    /ws:\/\/[^\s"']+/,
+  )?.[0];
+  if (output.code !== 0 || !url) throw new Error('无法取得浏览器 CDP 地址');
+  return url;
+}
+
+export async function getBrowserAutomation(
+  sessionName: string,
+  bin: string,
+  run: AgentBrowserRunner,
+  signal?: AbortSignal,
+  preferredUrl?: string,
+  preferredMarker?: string,
+): Promise<CdpConnection> {
+  const url = await cdpUrl(sessionName, bin, run, signal);
+  // The browser endpoint is shared, but the attached page is not.  Returning
+  // a fresh connection prevents a later conversation from changing the
+  // mutable page session underneath an in-flight batch.
+  const conn = new CdpConnection(url);
+  await conn.open();
+  await conn.attachToPage(preferredUrl, true, preferredMarker);
+  return conn;
+}
+
+export function browserDownloadDir(
+  conversationKey = 'local',
+  runId = 'default',
+): string {
+  const safe = (value: string) =>
+    value.replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 100) || 'default';
+  const root =
+    process.env.MINIMAL_AGENT_HOME ?? path.join(os.homedir(), '.minimal-agent');
+  return path.join(
+    root,
+    'browser',
+    'downloads',
+    safe(conversationKey),
+    safe(runId),
+  );
+}
+
+export async function configureDownloads(
+  cdp: CdpConnection,
+  conversationKey?: string,
+  runId?: string,
+): Promise<string> {
+  const dir = browserDownloadDir(conversationKey, runId);
+  await mkdir(dir, { recursive: true, mode: 0o700 });
+  // Browser domain is current Chrome; Page fallback keeps older Chromium usable.
+  await cdp
+    .send('Browser.setDownloadBehavior', {
+      behavior: 'allow',
+      downloadPath: dir,
+      eventsEnabled: true,
+    })
+    .catch(() =>
+      cdp.send('Page.setDownloadBehavior', {
+        behavior: 'allow',
+        downloadPath: dir,
+      }),
+    );
+  return dir;
+}
+
+export async function finalDownload(
+  pathname: string,
+): Promise<Pick<BrowserDownload, 'path' | 'receivedBytes' | 'mime'>> {
+  const info = await stat(pathname);
+  return { path: pathname, receivedBytes: info.size };
+}
+
+export type { CdpConnection };

package/src/tools/webbrowser/browserAxState.js

@@ -0,0 +1,97 @@
+/** Compact AX state and generation-aware diffs for BrowserBatch observations. */
+function axValue(input) {
+    if (input && typeof input === 'object' && 'value' in input)
+        return input.value;
+    return input;
+}
+function propertyValue(node, key) {
+    const value = node.properties?.find((item) => item.name === key)?.value
+        ?.value;
+    return value === undefined || value === null ? undefined : String(value);
+}
+export function normalizeAxTree(raw) {
+    const root = raw && typeof raw === 'object' ? raw : {};
+    const nodes = Array.isArray(raw)
+        ? raw
+        : Array.isArray(root.nodes)
+            ? root.nodes
+            : [];
+    return nodes
+        .map((value) => {
+        const node = value && typeof value === 'object' ? value : {};
+        const bounds = node?.bounds;
+        return {
+            id: String(node.nodeId ?? node.id ?? ''),
+            ...(node.parentId !== undefined
+                ? { parentId: String(node.parentId) }
+                : {}),
+            ...(axValue(node.role) || node.role
+                ? { role: String(axValue(node.role) ?? node.role) }
+                : {}),
+            ...(axValue(node.name) || node.name
+                ? { name: String(axValue(node.name) ?? node.name) }
+                : {}),
+            ...(propertyValue(node, 'value')
+                ? { value: propertyValue(node, 'value') }
+                : {}),
+            ...(node.ignored !== undefined
+                ? { ignored: Boolean(node.ignored) }
+                : {}),
+            ...(node.backendDOMNodeId !== undefined
+                ? { backendDOMNodeId: Number(node.backendDOMNodeId) }
+                : {}),
+            ...(bounds &&
+                typeof bounds.x === 'number' &&
+                typeof bounds.y === 'number' &&
+                typeof bounds.width === 'number' &&
+                typeof bounds.height === 'number'
+                ? {
+                    bounds: {
+                        x: bounds.x,
+                        y: bounds.y,
+                        width: bounds.width,
+                        height: bounds.height,
+                    },
+                }
+                : {}),
+        };
+    })
+        .filter((node) => node.id.length > 0);
+}
+function nodeKey(node) {
+    return JSON.stringify(node);
+}
+export function diffAxTree(previous, current, generation, pageRevision, forceFull = false) {
+    const next = new Map(current.map((node) => [node.id, node]));
+    if (!previous || forceFull) {
+        return {
+            generation,
+            pageRevision,
+            full: true,
+            added: current,
+            changed: [],
+            removed: [],
+        };
+    }
+    const added = [];
+    const changed = [];
+    for (const node of current) {
+        const old = previous.get(node.id);
+        if (!old)
+            added.push(node);
+        else if (nodeKey(old) !== nodeKey(node))
+            changed.push(node);
+    }
+    const removed = [...previous.keys()].filter((id) => !next.has(id));
+    return { generation, pageRevision, full: false, added, changed, removed };
+}
+export function conciseAxTree(nodes) {
+    return nodes
+        .map((node, index) => {
+        const details = [node.role, node.name, node.value]
+            .filter(Boolean)
+            .join(' ');
+        return `[ax=${node.id}]${details ? ` ${details}` : ''} index=${index}${node.ignored ? ' (ignored)' : ''}`;
+    })
+        .join('\n');
+}

package/src/tools/webbrowser/browserAxState.ts

@@ -0,0 +1,142 @@
+/** Compact AX state and generation-aware diffs for BrowserBatch observations. */
+
+export interface BrowserAxNode {
+  id: string;
+  parentId?: string;
+  role?: string;
+  name?: string;
+  value?: string;
+  ignored?: boolean;
+  backendDOMNodeId?: number;
+  bounds?: { x: number; y: number; width: number; height: number };
+}
+
+export interface BrowserAxDiff {
+  generation: number;
+  pageRevision: number;
+  full: boolean;
+  added: BrowserAxNode[];
+  changed: BrowserAxNode[];
+  removed: string[];
+}
+
+type RawAxProperty = { name?: unknown; value?: { value?: unknown } };
+type RawAxNode = {
+  nodeId?: unknown;
+  id?: unknown;
+  parentId?: unknown;
+  role?: unknown;
+  name?: unknown;
+  ignored?: unknown;
+  backendDOMNodeId?: unknown;
+  bounds?: { x?: unknown; y?: unknown; width?: unknown; height?: unknown };
+  properties?: RawAxProperty[];
+};
+
+function axValue(input: unknown): unknown {
+  if (input && typeof input === 'object' && 'value' in input)
+    return (input as { value?: unknown }).value;
+  return input;
+}
+
+function propertyValue(node: RawAxNode, key: string): string | undefined {
+  const value = node.properties?.find((item) => item.name === key)?.value
+    ?.value;
+  return value === undefined || value === null ? undefined : String(value);
+}
+
+export function normalizeAxTree(raw: unknown): BrowserAxNode[] {
+  const root =
+    raw && typeof raw === 'object' ? (raw as { nodes?: unknown }) : {};
+  const nodes = Array.isArray(raw)
+    ? raw
+    : Array.isArray(root.nodes)
+      ? root.nodes
+      : [];
+  return nodes
+    .map((value: unknown) => {
+      const node =
+        value && typeof value === 'object' ? (value as RawAxNode) : {};
+      const bounds = node?.bounds;
+      return {
+        id: String(node.nodeId ?? node.id ?? ''),
+        ...(node.parentId !== undefined
+          ? { parentId: String(node.parentId) }
+          : {}),
+        ...(axValue(node.role) || node.role
+          ? { role: String(axValue(node.role) ?? node.role) }
+          : {}),
+        ...(axValue(node.name) || node.name
+          ? { name: String(axValue(node.name) ?? node.name) }
+          : {}),
+        ...(propertyValue(node, 'value')
+          ? { value: propertyValue(node, 'value') }
+          : {}),
+        ...(node.ignored !== undefined
+          ? { ignored: Boolean(node.ignored) }
+          : {}),
+        ...(node.backendDOMNodeId !== undefined
+          ? { backendDOMNodeId: Number(node.backendDOMNodeId) }
+          : {}),
+        ...(bounds &&
+        typeof bounds.x === 'number' &&
+        typeof bounds.y === 'number' &&
+        typeof bounds.width === 'number' &&
+        typeof bounds.height === 'number'
+          ? {
+              bounds: {
+                x: bounds.x,
+                y: bounds.y,
+                width: bounds.width,
+                height: bounds.height,
+              },
+            }
+          : {}),
+      } satisfies BrowserAxNode;
+    })
+    .filter((node: BrowserAxNode) => node.id.length > 0);
+}
+
+function nodeKey(node: BrowserAxNode): string {
+  return JSON.stringify(node);
+}
+
+export function diffAxTree(
+  previous: Map<string, BrowserAxNode> | undefined,
+  current: BrowserAxNode[],
+  generation: number,
+  pageRevision: number,
+  forceFull = false,
+): BrowserAxDiff {
+  const next = new Map(current.map((node) => [node.id, node]));
+  if (!previous || forceFull) {
+    return {
+      generation,
+      pageRevision,
+      full: true,
+      added: current,
+      changed: [],
+      removed: [],
+    };
+  }
+  const added: BrowserAxNode[] = [];
+  const changed: BrowserAxNode[] = [];
+  for (const node of current) {
+    const old = previous.get(node.id);
+    if (!old) added.push(node);
+    else if (nodeKey(old) !== nodeKey(node)) changed.push(node);
+  }
+  const removed = [...previous.keys()].filter((id) => !next.has(id));
+  return { generation, pageRevision, full: false, added, changed, removed };
+}
+
+export function conciseAxTree(nodes: BrowserAxNode[]): string {
+  return nodes
+    .map((node, index) => {
+      const details = [node.role, node.name, node.value]
+        .filter(Boolean)
+        .join(' ');
+      return `[ax=${node.id}]${details ? ` ${details}` : ''} index=${index}${node.ignored ? ' (ignored)' : ''}`;
+    })
+    .join('\n');
+}