hyper-agent-browser 0.1.0 → 0.3.1

package/src/daemon/browser-pool.ts

@@ -2,55 +2,110 @@ import { BrowserManager } from "../browser/manager";
 import type { BrowserManagerOptions } from "../browser/manager";
 import type { Session } from "../session/store";
 
+interface CachedBrowser {
+  manager: BrowserManager;
+  options: BrowserManagerOptions;
+}
+
 /**
  * BrowserPool 管理多个 Session 的浏览器实例
  */
 export class BrowserPool {
-  private browsers: Map<string, BrowserManager> = new Map();
+  private browsers: Map<string, CachedBrowser> = new Map();
 
   async get(session: Session, options: BrowserManagerOptions = {}): Promise<BrowserManager> {
     const key = session.name;
 
     if (this.browsers.has(key)) {
-      const browser = this.browsers.get(key)!;
+      const cached = this.browsers.get(key)!;
+      const browser = cached.manager;
 
-      // 检查浏览器是否还连接
-      if (browser.isConnected()) {
-        return browser;
-      }
+      // 检查选项是否一致（特别是 headed 模式）
+      const optionsMismatch = this.hasOptionsMismatch(cached.options, options);
 
-      // 浏览器断开连接，尝试重连
-      try {
-        await browser.connect();
+      if (optionsMismatch) {
+        console.log(
+          `Options mismatch for session ${key} (headed: ${cached.options.headed} -> ${options.headed}), recreating browser`,
+        );
+        try {
+          await browser.close();
+        } catch (error) {
+          console.error("Error closing browser for options change:", error);
+        }
+        this.browsers.delete(key);
+        // 继续创建新的浏览器实例
+      } else if (browser.isConnected()) {
+        // 选项一致且浏览器已连接，直接返回
         return browser;
-      } catch (error) {
-        console.error(`Failed to reconnect browser for session ${key}:`, error);
-        // 重连失败，移除旧实例
+      } else {
+        // 浏览器断开连接，尝试重连
+        try {
+          await browser.connect();
+          if (browser.isConnected()) {
+            return browser;
+          }
+        } catch (error) {
+          console.error(`Failed to reconnect browser for session ${key}:`, error);
+        }
+
+        // 重连失败，移除旧实例并创建新的
+        console.log(`Removing stale browser instance for session ${key}`);
         this.browsers.delete(key);
       }
     }
 
     // 创建新的浏览器实例
+    console.log(
+      `Creating new browser instance for session ${key} (headed: ${options.headed ?? false})`,
+    );
     const browser = new BrowserManager(session, options);
     await browser.connect();
-    this.browsers.set(key, browser);
+    this.browsers.set(key, { manager: browser, options: { ...options } });
 
     return browser;
   }
 
+  /**
+   * 检查选项是否有重要变化需要重建浏览器
+   */
+  private hasOptionsMismatch(
+    cached: BrowserManagerOptions,
+    requested: BrowserManagerOptions,
+  ): boolean {
+    // headed 模式变化需要重建浏览器
+    if ((cached.headed ?? false) !== (requested.headed ?? false)) {
+      return true;
+    }
+    // channel 变化也需要重建
+    if (cached.channel && requested.channel && cached.channel !== requested.channel) {
+      return true;
+    }
+    return false;
+  }
+
   async close(sessionName: string): Promise<boolean> {
-    const browser = this.browsers.get(sessionName);
-    if (!browser) {
+    const cached = this.browsers.get(sessionName);
+    if (!cached) {
       return false;
     }
 
-    await browser.close();
+    try {
+      await cached.manager.close();
+    } catch (error) {
+      console.error(`Error closing browser for session ${sessionName}:`, error);
+    }
     this.browsers.delete(sessionName);
     return true;
   }
 
   async closeAll(): Promise<void> {
-    const closePromises = Array.from(this.browsers.values()).map((b) => b.close());
+    const closePromises = Array.from(this.browsers.entries()).map(async ([name, cached]) => {
+      try {
+        await cached.manager.close();
+      } catch (error) {
+        console.error(`Error closing browser for session ${name}:`, error);
+      }
+    });
     await Promise.all(closePromises);
     this.browsers.clear();
   }
@@ -62,4 +117,22 @@ export class BrowserPool {
   size(): number {
     return this.browsers.size;
   }
+
+  /**
+   * 清理所有无效的浏览器实例
+   */
+  async cleanup(): Promise<void> {
+    const toRemove: string[] = [];
+
+    for (const [name, cached] of this.browsers.entries()) {
+      if (!cached.manager.isConnected()) {
+        toRemove.push(name);
+      }
+    }
+
+    for (const name of toRemove) {
+      console.log(`Cleaning up disconnected browser: ${name}`);
+      this.browsers.delete(name);
+    }
+  }
 }

package/src/daemon/server.ts

@@ -1,8 +1,11 @@
 import * as actionCommands from "../commands/actions";
 import * as advancedCommands from "../commands/advanced";
+import * as extractCommands from "../commands/extract";
 import * as getterCommands from "../commands/getters";
 import * as infoCommands from "../commands/info";
 import * as navigationCommands from "../commands/navigation";
+import * as networkCommands from "../commands/network";
+import * as waitCommands from "../commands/wait";
 import { SessionManager } from "../session/manager";
 import { ReferenceStore } from "../snapshot/reference-store";
 import { formatError } from "../utils/errors";
@@ -529,6 +532,69 @@ export class DaemonServer {
             result = `Highlighted: ${args.selector}`;
             break;
 
+          // Wait commands
+          case "wait-idle":
+            await waitCommands.waitIdle(page, {
+              timeout: args.timeout,
+              strategy: args.strategy,
+              networkIdleTime: args.networkIdleTime,
+              domStableTime: args.domStableTime,
+              ignoreSelectors: args.ignoreSelectors,
+            });
+            result = "Page is idle";
+            break;
+
+          case "wait-element":
+            await waitCommands.waitElement(page, args.selector, {
+              state: args.state,
+              timeout: args.timeout,
+            });
+            result = `Element ${args.state || "visible"}: ${args.selector}`;
+            break;
+
+          // Extract commands
+          case "extract-table":
+            result = await extractCommands.extractTable(page, {
+              selector: args.selector,
+              includeHeaders: args.includeHeaders,
+              maxRows: args.maxRows,
+            });
+            break;
+
+          case "extract-list":
+            result = await extractCommands.extractList(page, {
+              selector: args.selector,
+              pattern: args.pattern,
+              maxItems: args.maxItems,
+            });
+            break;
+
+          case "extract-form":
+            result = await extractCommands.extractForm(page, {
+              selector: args.selector,
+              includeDisabled: args.includeDisabled,
+            });
+            break;
+
+          case "extract-meta":
+            result = await extractCommands.extractMeta(page, {
+              include: args.include,
+            });
+            break;
+
+          // Network commands
+          case "network-start":
+            result = await networkCommands.networkStart(page, session, {
+              filter: args.filter,
+              urlPattern: args.urlPattern,
+              methods: args.methods,
+            });
+            break;
+
+          case "network-stop":
+            result = await networkCommands.networkStop(args.listenerId);
+            break;
+
           default:
             throw new Error(`Unknown command: ${command}`);
         }

package/src/extractors/form-extractor.ts

@@ -0,0 +1,153 @@
+import type { Page } from "patchright";
+
+export interface FormData {
+  type: "form";
+  url: string;
+  selector: string;
+  timestamp: number;
+  fields: FormField[];
+}
+
+export interface FormField {
+  ref?: string;
+  name: string;
+  type: string;
+  value?: string;
+  checked?: boolean;
+  label?: string;
+  placeholder?: string;
+  required: boolean;
+  disabled: boolean;
+  readonly?: boolean;
+  options?: string[];
+  selectedOptions?: string[];
+}
+
+export class FormExtractor {
+  /**
+   * 提取表单数据
+   */
+  async extract(
+    page: Page,
+    selector?: string,
+    options: { includeDisabled?: boolean } = {},
+  ): Promise<FormData> {
+    const includeDisabled = options.includeDisabled ?? false;
+
+    const url = page.url();
+    const timestamp = Date.now();
+
+    const result = await page.evaluate(
+      ({ selector, includeDisabled }) => {
+        let container: Element | null = null;
+
+        if (selector) {
+          container = document.querySelector(selector);
+        } else {
+          container = document.querySelector("form") || document.body;
+        }
+
+        if (!container) {
+          throw new Error("No form container found");
+        }
+
+        const fields: FormField[] = [];
+
+        // 查找所有表单控件
+        const inputs = container.querySelectorAll("input, textarea, select");
+
+        for (const input of Array.from(inputs)) {
+          if (
+            !(
+              input instanceof HTMLInputElement ||
+              input instanceof HTMLTextAreaElement ||
+              input instanceof HTMLSelectElement
+            )
+          ) {
+            continue;
+          }
+
+          // 跳过禁用字段（如果不包含）
+          if (!includeDisabled && input.disabled) {
+            continue;
+          }
+
+          const field: FormField = {
+            name: input.name || input.id || "",
+            type:
+              input instanceof HTMLSelectElement
+                ? "select"
+                : (input as HTMLInputElement).type || "text",
+            required: input.required,
+            disabled: input.disabled,
+          };
+
+          // 提取 placeholder
+          if ("placeholder" in input && input.placeholder) {
+            field.placeholder = input.placeholder;
+          }
+
+          // 提取 readonly
+          if ("readOnly" in input) {
+            field.readonly = input.readOnly;
+          }
+
+          // 提取值
+          if (input instanceof HTMLInputElement) {
+            if (input.type === "checkbox" || input.type === "radio") {
+              field.checked = input.checked;
+              field.value = input.value;
+            } else if (input.type !== "password") {
+              // 不提取密码值
+              field.value = input.value;
+            } else {
+              field.value = input.value ? "******" : "";
+            }
+          } else if (input instanceof HTMLTextAreaElement) {
+            field.value = input.value;
+          } else if (input instanceof HTMLSelectElement) {
+            const selectedOptions = Array.from(input.selectedOptions).map((opt) => opt.value);
+            field.selectedOptions = selectedOptions;
+            field.options = Array.from(input.options).map((opt) => opt.value);
+          }
+
+          // 查找关联的 label
+          let label: HTMLLabelElement | null = null;
+
+          if (input.id) {
+            label = document.querySelector(`label[for="${input.id}"]`);
+          }
+
+          if (!label) {
+            // 查找父级 label
+            let parent = input.parentElement;
+            while (parent) {
+              if (parent instanceof HTMLLabelElement) {
+                label = parent;
+                break;
+              }
+              parent = parent.parentElement;
+            }
+          }
+
+          if (label) {
+            field.label = label.textContent?.trim();
+          }
+
+          fields.push(field);
+        }
+
+        return { fields };
+      },
+      { selector: selector || "", includeDisabled },
+    );
+
+    return {
+      type: "form",
+      url,
+      selector: selector || "form",
+      timestamp,
+      fields: result.fields,
+    };
+  }
+}

package/src/extractors/list-extractor.ts

@@ -0,0 +1,213 @@
+import type { Page } from "patchright";
+
+export interface ListData {
+  type: "list";
+  url: string;
+  selector: string;
+  timestamp: number;
+  pattern: string;
+  items: number;
+  data: Record<string, any>[];
+}
+
+export class ListExtractor {
+  /**
+   * 提取列表数据
+   */
+  async extract(
+    page: Page,
+    selector?: string,
+    options: { pattern?: string; maxItems?: number } = {},
+  ): Promise<ListData> {
+    const pattern = options.pattern ?? "auto";
+    const maxItems = options.maxItems ?? Number.POSITIVE_INFINITY;
+
+    const url = page.url();
+    const timestamp = Date.now();
+
+    const result = await page.evaluate(
+      ({ selector, pattern, maxItems }) => {
+        let container: Element | null = null;
+
+        if (selector) {
+          container = document.querySelector(selector);
+        } else {
+          // 查找常见的列表容器
+          container =
+            document.querySelector("ul") ||
+            document.querySelector("ol") ||
+            document.querySelector('[role="list"]') ||
+            document.body;
+        }
+
+        if (!container) {
+          throw new Error("No list container found");
+        }
+
+        // 自动检测重复结构
+        let items: Element[] = [];
+        let detectedPattern = "";
+
+        if (pattern === "auto") {
+          // 尝试不同的模式
+          const patterns = [
+            "li",
+            '[role="listitem"]',
+            ".item",
+            ".product",
+            ".card",
+            "[class*='item']",
+            "[class*='card']",
+            "[class*='product']",
+          ];
+
+          for (const p of patterns) {
+            const elements = Array.from(container.querySelectorAll(p));
+            if (elements.length > 1) {
+              items = elements;
+              detectedPattern = p;
+              break;
+            }
+          }
+
+          // 如果还是找不到，尝试检测相同 class 的兄弟元素
+          if (items.length === 0) {
+            const children = Array.from(container.children);
+            const classMap = new Map<string, Element[]>();
+
+            for (const child of children) {
+              const className = child.className;
+              if (className) {
+                if (!classMap.has(className)) {
+                  classMap.set(className, []);
+                }
+                classMap.get(className)!.push(child);
+              }
+            }
+
+            // 找到最多重复的 class
+            let maxCount = 0;
+            let maxClass = "";
+            classMap.forEach((elements, className) => {
+              if (elements.length > maxCount) {
+                maxCount = elements.length;
+                maxClass = className;
+              }
+            });
+
+            if (maxCount > 1) {
+              items = classMap.get(maxClass)!;
+              detectedPattern = `.${maxClass.split(" ")[0]}`;
+            }
+          }
+        } else {
+          items = Array.from(container.querySelectorAll(pattern));
+          detectedPattern = pattern;
+        }
+
+        if (items.length === 0) {
+          throw new Error("No repeating items found");
+        }
+
+        // 限制数量
+        items = items.slice(0, maxItems);
+
+        // 提取每个 item 的数据
+        const data = items.map((item) => extractItemData(item));
+
+        return {
+          pattern: detectedPattern,
+          data,
+        };
+      },
+      { selector: selector || "", pattern, maxItems },
+    );
+
+    return {
+      type: "list",
+      url,
+      selector: selector || "auto",
+      timestamp,
+      pattern: result.pattern,
+      items: result.data.length,
+      data: result.data,
+    };
+  }
+}
+
+/**
+ * 提取单个列表项的数据
+ */
+function extractItemData(item: Element): Record<string, any> {
+  const data: Record<string, any> = {};
+
+  // 提取 data-* 属性
+  if (item instanceof HTMLElement) {
+    for (const attr of Array.from(item.attributes)) {
+      if (attr.name.startsWith("data-")) {
+        const key = attr.name.substring(5);
+        data[key] = attr.value;
+      }
+    }
+  }
+
+  // 提取 id
+  if (item.id) {
+    data.id = item.id;
+  }
+
+  // 提取文本内容
+  const textNodes: string[] = [];
+  const walker = document.createTreeWalker(item, NodeFilter.SHOW_TEXT, null);
+
+  let node = walker.nextNode();
+  while (node) {
+    const text = node.textContent?.trim();
+    if (text && text.length > 0) {
+      textNodes.push(text);
+    }
+    node = walker.nextNode();
+  }
+
+  if (textNodes.length === 1) {
+    data.text = textNodes[0];
+  } else if (textNodes.length > 1) {
+    data.text = textNodes.join(" ");
+  }
+
+  // 提取链接
+  const link = item.querySelector("a");
+  if (link instanceof HTMLAnchorElement) {
+    data.link = link.href;
+    if (!data.text) {
+      data.text = link.textContent?.trim();
+    }
+  }
+
+  // 提取图片
+  const img = item.querySelector("img");
+  if (img instanceof HTMLImageElement) {
+    data.image = img.src;
+    if (img.alt) {
+      data.imageAlt = img.alt;
+    }
+  }
+
+  // 提取常见字段（通过 class 或元素结构）
+  const title = item.querySelector('[class*="title"], h1, h2, h3, h4, h5, h6');
+  if (title) {
+    data.title = title.textContent?.trim();
+  }
+
+  const price = item.querySelector('[class*="price"]');
+  if (price) {
+    data.price = price.textContent?.trim();
+  }
+
+  const description = item.querySelector('[class*="desc"], [class*="summary"], p');
+  if (description && description !== title) {
+    data.description = description.textContent?.trim();
+  }
+
+  return data;
+}