hyper-agent-browser 0.1.0 → 0.3.1

package/src/cli.ts

@@ -183,6 +183,47 @@ program
     console.log(result);
   });
 
+program
+  .command("wait-idle")
+  .description("Wait for page to be idle (network + DOM stable)")
+  .option("--timeout <ms>", "Timeout in milliseconds", "30000")
+  .option("--strategy <strategy>", "Strategy: network, dom, or both", "both")
+  .option("--network-idle-time <ms>", "Network idle time", "500")
+  .option("--dom-stable-time <ms>", "DOM stable time", "500")
+  .option("--ignore <selectors>", "Ignore DOM changes in selectors (comma-separated)")
+  .action(async (options, command) => {
+    const result = await executeViaDaemon(
+      "wait-idle",
+      {
+        timeout: Number.parseInt(options.timeout),
+        strategy: options.strategy,
+        networkIdleTime: Number.parseInt(options.networkIdleTime),
+        domStableTime: Number.parseInt(options.domStableTime),
+        ignoreSelectors: options.ignore ? options.ignore.split(",") : undefined,
+      },
+      command.parent,
+    );
+    console.log(result);
+  });
+
+program
+  .command("wait-element <selector>")
+  .description("Wait for element state (attached/detached/visible/hidden)")
+  .option("--state <state>", "Element state", "visible")
+  .option("--timeout <ms>", "Timeout in milliseconds", "10000")
+  .action(async (selector, options, command) => {
+    const result = await executeViaDaemon(
+      "wait-element",
+      {
+        selector,
+        state: options.state,
+        timeout: Number.parseInt(options.timeout),
+      },
+      command.parent,
+    );
+    console.log(result);
+  });
+
 // 第一批新增 Action 命令
 program
   .command("check <selector>")
@@ -506,6 +547,115 @@ program
     console.log(result);
   });
 
+// Extract commands
+program
+  .command("extract-table")
+  .description("Extract table data from page")
+  .option("--selector <selector>", "Table selector")
+  .option("--no-headers", "Exclude headers")
+  .option("--max-rows <number>", "Maximum number of rows", "1000")
+  .action(async (options, command) => {
+    const result = await executeViaDaemon(
+      "extract-table",
+      {
+        selector: options.selector,
+        includeHeaders: options.headers,
+        maxRows: Number.parseInt(options.maxRows),
+      },
+      command.parent,
+    );
+    console.log(result);
+  });
+
+program
+  .command("extract-list")
+  .description("Extract list data from page")
+  .option("--selector <selector>", "List container selector")
+  .option("--pattern <pattern>", "Item pattern (auto by default)", "auto")
+  .option("--max-items <number>", "Maximum number of items", "1000")
+  .action(async (options, command) => {
+    const result = await executeViaDaemon(
+      "extract-list",
+      {
+        selector: options.selector,
+        pattern: options.pattern,
+        maxItems: Number.parseInt(options.maxItems),
+      },
+      command.parent,
+    );
+    console.log(result);
+  });
+
+program
+  .command("extract-form")
+  .description("Extract form field data from page")
+  .option("--selector <selector>", "Form selector")
+  .option("--include-disabled", "Include disabled fields")
+  .action(async (options, command) => {
+    const result = await executeViaDaemon(
+      "extract-form",
+      {
+        selector: options.selector,
+        includeDisabled: options.includeDisabled,
+      },
+      command.parent,
+    );
+    console.log(result);
+  });
+
+program
+  .command("extract-meta")
+  .description("Extract page metadata (SEO, OpenGraph, Schema.org)")
+  .option(
+    "--include <types>",
+    "Metadata types (seo,og,twitter,schema,other)",
+    "seo,og,twitter,schema,other",
+  )
+  .action(async (options, command) => {
+    const result = await executeViaDaemon(
+      "extract-meta",
+      {
+        include: options.include.split(","),
+      },
+      command.parent,
+    );
+    console.log(result);
+  });
+
+// Network commands
+program
+  .command("network-start")
+  .description("Start network request monitoring")
+  .option("--filter <types>", "Resource types (xhr,fetch,document,script,image,font)", "xhr,fetch")
+  .option("--url-pattern <pattern>", "URL pattern to match (glob)")
+  .option("--methods <methods>", "HTTP methods (GET,POST,PUT,DELETE)")
+  .action(async (options, command) => {
+    const result = await executeViaDaemon(
+      "network-start",
+      {
+        filter: options.filter.split(","),
+        urlPattern: options.urlPattern,
+        methods: options.methods ? options.methods.split(",") : undefined,
+      },
+      command.parent,
+    );
+    console.log(result);
+  });
+
+program
+  .command("network-stop <listener-id>")
+  .description("Stop network monitoring and get captured requests")
+  .action(async (listenerId, _options, command) => {
+    const result = await executeViaDaemon(
+      "network-stop",
+      {
+        listenerId,
+      },
+      command.parent,
+    );
+    console.log(result);
+  });
+
 // Session commands
 program
   .command("sessions")
@@ -597,7 +747,7 @@ program
   .command("version")
   .description("Show version information")
   .action(() => {
-    console.log("hyper-agent-browser v0.2.0 (with daemon architecture)");
+    console.log("hyper-agent-browser v0.3.1 (with daemon architecture)");
     console.log(`Bun v${Bun.version}`);
     console.log("Patchright v1.55.1");
   });

package/src/commands/extract.ts

@@ -0,0 +1,76 @@
+import type { Page } from "patchright";
+import { FormExtractor } from "../extractors/form-extractor";
+import { ListExtractor } from "../extractors/list-extractor";
+import { MetaExtractor } from "../extractors/meta-extractor";
+import { TableExtractor } from "../extractors/table-extractor";
+
+export interface ExtractTableOptions {
+  selector?: string;
+  includeHeaders?: boolean;
+  maxRows?: number;
+}
+
+export interface ExtractListOptions {
+  selector?: string;
+  pattern?: string;
+  maxItems?: number;
+}
+
+export interface ExtractFormOptions {
+  selector?: string;
+  includeDisabled?: boolean;
+}
+
+export interface ExtractMetaOptions {
+  include?: string[];
+}
+
+/**
+ * 提取表格数据
+ */
+export async function extractTable(page: Page, options: ExtractTableOptions = {}): Promise<string> {
+  const extractor = new TableExtractor();
+  const result = await extractor.extract(page, options.selector, {
+    includeHeaders: options.includeHeaders,
+    maxRows: options.maxRows,
+  });
+
+  return JSON.stringify(result, null, 2);
+}
+
+/**
+ * 提取列表数据
+ */
+export async function extractList(page: Page, options: ExtractListOptions = {}): Promise<string> {
+  const extractor = new ListExtractor();
+  const result = await extractor.extract(page, options.selector, {
+    pattern: options.pattern,
+    maxItems: options.maxItems,
+  });
+
+  return JSON.stringify(result, null, 2);
+}
+
+/**
+ * 提取表单数据
+ */
+export async function extractForm(page: Page, options: ExtractFormOptions = {}): Promise<string> {
+  const extractor = new FormExtractor();
+  const result = await extractor.extract(page, options.selector, {
+    includeDisabled: options.includeDisabled,
+  });
+
+  return JSON.stringify(result, null, 2);
+}
+
+/**
+ * 提取元数据
+ */
+export async function extractMeta(page: Page, options: ExtractMetaOptions = {}): Promise<string> {
+  const extractor = new MetaExtractor();
+  const result = await extractor.extract(page, {
+    include: options.include,
+  });
+
+  return JSON.stringify(result, null, 2);
+}

package/src/commands/network.ts

@@ -0,0 +1,111 @@
+import type { Page } from "patchright";
+import type { Session } from "../session/store";
+import {
+  NetworkListener,
+  type NetworkListenerConfig,
+  generateListenerId,
+} from "../utils/network-listener";
+
+// 存储活动的监听器
+const activeListeners = new Map<string, NetworkListener>();
+
+export interface NetworkStartOptions {
+  filter?: string[];
+  urlPattern?: string;
+  methods?: string[];
+}
+
+export interface NetworkStopResult {
+  listenerId: string;
+  startTime: number;
+  endTime: number;
+  duration: number;
+  totalRequests: number;
+  requests: any[];
+}
+
+/**
+ * 开始网络监听
+ */
+export async function networkStart(
+  page: Page,
+  session: Session,
+  options: NetworkStartOptions = {},
+): Promise<string> {
+  const listenerId = generateListenerId();
+
+  const config: NetworkListenerConfig = {
+    id: listenerId,
+    sessionDir: session.userDataDir.replace(/\/userdata$/, ""),
+    filter: {
+      types: options.filter ?? ["xhr", "fetch"],
+      urlPattern: options.urlPattern,
+      methods: options.methods,
+    },
+    startTime: Date.now(),
+    status: "active",
+  };
+
+  const listener = new NetworkListener(config);
+  await listener.start(page);
+
+  activeListeners.set(listenerId, listener);
+
+  const info = listener.getInfo();
+  return JSON.stringify(info, null, 2);
+}
+
+/**
+ * 停止网络监听并返回数据
+ */
+export async function networkStop(listenerId: string): Promise<string> {
+  const listener = activeListeners.get(listenerId);
+
+  if (!listener) {
+    throw new Error(`Network listener not found: ${listenerId}`);
+  }
+
+  await listener.stop();
+
+  const info = listener.getInfo();
+  const requests = await listener.getRequests();
+
+  const result: NetworkStopResult = {
+    listenerId,
+    startTime: info.startTime,
+    endTime: Date.now(),
+    duration: Date.now() - info.startTime,
+    totalRequests: requests.length,
+    requests,
+  };
+
+  activeListeners.delete(listenerId);
+
+  return JSON.stringify(result, null, 2);
+}
+
+/**
+ * 获取所有活动监听器
+ */
+export function getActiveListeners(): string[] {
+  return Array.from(activeListeners.keys());
+}
+
+/**
+ * 清理会话的所有监听器
+ */
+export async function cleanupSessionListeners(sessionName: string): Promise<void> {
+  const toDelete: string[] = [];
+
+  for (const [id, listener] of activeListeners.entries()) {
+    const info = listener.getInfo();
+    if (info.listenerId.includes(sessionName)) {
+      await listener.stop();
+      toDelete.push(id);
+    }
+  }
+
+  for (const id of toDelete) {
+    activeListeners.delete(id);
+  }
+}

package/src/commands/wait.ts

@@ -0,0 +1,226 @@
+import type { Page } from "patchright";
+
+export interface WaitIdleOptions {
+  timeout?: number;
+  strategy?: "network" | "dom" | "both";
+  networkIdleTime?: number;
+  domStableTime?: number;
+  ignoreSelectors?: string[];
+}
+
+export interface WaitElementOptions {
+  state?: "attached" | "detached" | "visible" | "hidden";
+  timeout?: number;
+}
+
+/**
+ * 等待页面空闲（网络请求完成 + DOM 稳定）
+ */
+export async function waitIdle(page: Page, options: WaitIdleOptions = {}): Promise<void> {
+  const timeout = options.timeout ?? 30000;
+  const strategy = options.strategy ?? "both";
+  const networkIdleTime = options.networkIdleTime ?? 500;
+  const domStableTime = options.domStableTime ?? 500;
+
+  const startTime = Date.now();
+
+  try {
+    if (strategy === "network" || strategy === "both") {
+      await waitNetworkIdle(page, networkIdleTime, timeout);
+    }
+
+    if (strategy === "dom" || strategy === "both") {
+      const remainingTimeout = timeout - (Date.now() - startTime);
+      if (remainingTimeout <= 0) {
+        throw new Error("Timeout before DOM stability check");
+      }
+      await waitDomStable(page, domStableTime, remainingTimeout, options.ignoreSelectors);
+    }
+  } catch (error) {
+    if (error instanceof Error && error.message.includes("Timeout")) {
+      // 获取当前状态用于调试
+      const state = await getPageState(page);
+      throw new WaitTimeoutError(timeout, state);
+    }
+    throw error;
+  }
+}
+
+/**
+ * 等待网络空闲
+ */
+async function waitNetworkIdle(page: Page, _idleTime: number, timeout: number): Promise<void> {
+  try {
+    // 使用 Playwright 的 networkidle 策略
+    await page.waitForLoadState("networkidle", { timeout });
+  } catch (error) {
+    throw new Error(`Network idle timeout after ${timeout}ms`);
+  }
+}
+
+/**
+ * 等待 DOM 稳定
+ */
+async function waitDomStable(
+  page: Page,
+  stableTime: number,
+  timeout: number,
+  ignoreSelectors?: string[],
+): Promise<void> {
+  try {
+    await page.evaluate(
+      ({ stableTime, timeout, ignoreSelectors }) => {
+        return new Promise((resolve, reject) => {
+          const startTime = Date.now();
+          let timer: NodeJS.Timeout;
+
+          const observer = new MutationObserver((mutations) => {
+            // 过滤掉被忽略的区域的变化
+            const relevantMutations = ignoreSelectors
+              ? mutations.filter((mutation) => {
+                  const target = mutation.target as Element;
+                  return !ignoreSelectors.some((selector) => {
+                    try {
+                      return target.matches?.(selector) || target.closest?.(selector);
+                    } catch {
+                      return false;
+                    }
+                  });
+                })
+              : mutations;
+
+            if (relevantMutations.length === 0) {
+              return;
+            }
+
+            // 重置计时器
+            clearTimeout(timer);
+
+            // 检查是否超时
+            if (Date.now() - startTime > timeout) {
+              observer.disconnect();
+              reject(new Error("DOM stability timeout"));
+              return;
+            }
+
+            // 设置新的稳定计时器
+            timer = setTimeout(() => {
+              observer.disconnect();
+              resolve(undefined);
+            }, stableTime);
+          });
+
+          // 开始观察
+          observer.observe(document.body, {
+            childList: true,
+            subtree: true,
+          });
+
+          // 初始计时器（如果 DOM 已经稳定）
+          timer = setTimeout(() => {
+            observer.disconnect();
+            resolve(undefined);
+          }, stableTime);
+
+          // 超时保护
+          setTimeout(() => {
+            clearTimeout(timer);
+            observer.disconnect();
+            reject(new Error("DOM stability timeout"));
+          }, timeout);
+        });
+      },
+      { stableTime, timeout, ignoreSelectors },
+    );
+  } catch (error) {
+    throw new Error(`DOM stable timeout after ${timeout}ms`);
+  }
+}
+
+/**
+ * 等待元素出现/消失/可见/隐藏
+ */
+export async function waitElement(
+  page: Page,
+  selector: string,
+  options: WaitElementOptions = {},
+): Promise<void> {
+  const state = options.state ?? "visible";
+  const timeout = options.timeout ?? 10000;
+
+  try {
+    const { parseSelector } = await import("../utils/selector");
+    const parsed = parseSelector(selector);
+
+    let locator;
+    if (parsed.type === "css") {
+      locator = page.locator(parsed.value);
+    } else if (parsed.type === "text") {
+      locator = page.getByText(parsed.value);
+    } else if (parsed.type === "xpath") {
+      locator = page.locator(`xpath=${parsed.value}`);
+    } else {
+      throw new Error(`Unsupported selector type: ${parsed.type}`);
+    }
+
+    await locator.waitFor({ state, timeout });
+  } catch (error) {
+    if (error instanceof Error && error.message.includes("Timeout")) {
+      throw new WaitElementTimeoutError(selector, state, timeout);
+    }
+    throw error;
+  }
+}
+
+/**
+ * 获取页面当前状态（用于调试）
+ */
+async function getPageState(page: Page): Promise<any> {
+  const state: any = {
+    network: {
+      idle: false,
+      pendingRequests: 0,
+      requests: [],
+    },
+    dom: {
+      stable: false,
+      mutations: 0,
+    },
+  };
+
+  // 简单检测：如果能成功等待 networkidle，说明网络是空闲的
+  try {
+    await page.waitForLoadState("networkidle", { timeout: 100 });
+    state.network.idle = true;
+  } catch {
+    state.network.idle = false;
+  }
+
+  return state;
+}
+
+/**
+ * 等待超时错误
+ */
+class WaitTimeoutError extends Error {
+  constructor(timeout: number, state: any) {
+    const message = `wait-idle timeout after ${timeout}ms`;
+    super(message);
+    this.name = "WaitTimeoutError";
+
+    // 在错误消息中包含状态信息
+    const stateJson = JSON.stringify(state, null, 2);
+    this.message = `${message}\n\nCurrent state:\n${stateJson}\n\nSuggestions:\n- Check pendingRequests and consider adding --exclude-url for slow requests\n- Check recentMutations and consider adding --ignore for animated regions\n- Increase timeout or idle times`;
+  }
+}
+
+/**
+ * 等待元素超时错误
+ */
+class WaitElementTimeoutError extends Error {
+  constructor(selector: string, state: string, timeout: number) {
+    const message = `Element not ${state} after ${timeout}ms: ${selector}`;
+    super(message);
+    this.name = "WaitElementTimeoutError";
+  }
+}