hyper-agent-browser 0.1.0 → 0.3.0

package/src/daemon/server.ts

@@ -1,8 +1,11 @@
 import * as actionCommands from "../commands/actions";
 import * as advancedCommands from "../commands/advanced";
+import * as extractCommands from "../commands/extract";
 import * as getterCommands from "../commands/getters";
 import * as infoCommands from "../commands/info";
 import * as navigationCommands from "../commands/navigation";
+import * as networkCommands from "../commands/network";
+import * as waitCommands from "../commands/wait";
 import { SessionManager } from "../session/manager";
 import { ReferenceStore } from "../snapshot/reference-store";
 import { formatError } from "../utils/errors";
@@ -529,6 +532,69 @@ export class DaemonServer {
             result = `Highlighted: ${args.selector}`;
             break;
 
+          // Wait commands
+          case "wait-idle":
+            await waitCommands.waitIdle(page, {
+              timeout: args.timeout,
+              strategy: args.strategy,
+              networkIdleTime: args.networkIdleTime,
+              domStableTime: args.domStableTime,
+              ignoreSelectors: args.ignoreSelectors,
+            });
+            result = "Page is idle";
+            break;
+
+          case "wait-element":
+            await waitCommands.waitElement(page, args.selector, {
+              state: args.state,
+              timeout: args.timeout,
+            });
+            result = `Element ${args.state || "visible"}: ${args.selector}`;
+            break;
+
+          // Extract commands
+          case "extract-table":
+            result = await extractCommands.extractTable(page, {
+              selector: args.selector,
+              includeHeaders: args.includeHeaders,
+              maxRows: args.maxRows,
+            });
+            break;
+
+          case "extract-list":
+            result = await extractCommands.extractList(page, {
+              selector: args.selector,
+              pattern: args.pattern,
+              maxItems: args.maxItems,
+            });
+            break;
+
+          case "extract-form":
+            result = await extractCommands.extractForm(page, {
+              selector: args.selector,
+              includeDisabled: args.includeDisabled,
+            });
+            break;
+
+          case "extract-meta":
+            result = await extractCommands.extractMeta(page, {
+              include: args.include,
+            });
+            break;
+
+          // Network commands
+          case "network-start":
+            result = await networkCommands.networkStart(page, session, {
+              filter: args.filter,
+              urlPattern: args.urlPattern,
+              methods: args.methods,
+            });
+            break;
+
+          case "network-stop":
+            result = await networkCommands.networkStop(args.listenerId);
+            break;
+
           default:
             throw new Error(`Unknown command: ${command}`);
         }

package/src/extractors/form-extractor.ts

@@ -0,0 +1,153 @@
+import type { Page } from "patchright";
+
+export interface FormData {
+  type: "form";
+  url: string;
+  selector: string;
+  timestamp: number;
+  fields: FormField[];
+}
+
+export interface FormField {
+  ref?: string;
+  name: string;
+  type: string;
+  value?: string;
+  checked?: boolean;
+  label?: string;
+  placeholder?: string;
+  required: boolean;
+  disabled: boolean;
+  readonly?: boolean;
+  options?: string[];
+  selectedOptions?: string[];
+}
+
+export class FormExtractor {
+  /**
+   * 提取表单数据
+   */
+  async extract(
+    page: Page,
+    selector?: string,
+    options: { includeDisabled?: boolean } = {},
+  ): Promise<FormData> {
+    const includeDisabled = options.includeDisabled ?? false;
+
+    const url = page.url();
+    const timestamp = Date.now();
+
+    const result = await page.evaluate(
+      ({ selector, includeDisabled }) => {
+        let container: Element | null = null;
+
+        if (selector) {
+          container = document.querySelector(selector);
+        } else {
+          container = document.querySelector("form") || document.body;
+        }
+
+        if (!container) {
+          throw new Error("No form container found");
+        }
+
+        const fields: FormField[] = [];
+
+        // 查找所有表单控件
+        const inputs = container.querySelectorAll("input, textarea, select");
+
+        for (const input of Array.from(inputs)) {
+          if (
+            !(
+              input instanceof HTMLInputElement ||
+              input instanceof HTMLTextAreaElement ||
+              input instanceof HTMLSelectElement
+            )
+          ) {
+            continue;
+          }
+
+          // 跳过禁用字段（如果不包含）
+          if (!includeDisabled && input.disabled) {
+            continue;
+          }
+
+          const field: FormField = {
+            name: input.name || input.id || "",
+            type:
+              input instanceof HTMLSelectElement
+                ? "select"
+                : (input as HTMLInputElement).type || "text",
+            required: input.required,
+            disabled: input.disabled,
+          };
+
+          // 提取 placeholder
+          if ("placeholder" in input && input.placeholder) {
+            field.placeholder = input.placeholder;
+          }
+
+          // 提取 readonly
+          if ("readOnly" in input) {
+            field.readonly = input.readOnly;
+          }
+
+          // 提取值
+          if (input instanceof HTMLInputElement) {
+            if (input.type === "checkbox" || input.type === "radio") {
+              field.checked = input.checked;
+              field.value = input.value;
+            } else if (input.type !== "password") {
+              // 不提取密码值
+              field.value = input.value;
+            } else {
+              field.value = input.value ? "******" : "";
+            }
+          } else if (input instanceof HTMLTextAreaElement) {
+            field.value = input.value;
+          } else if (input instanceof HTMLSelectElement) {
+            const selectedOptions = Array.from(input.selectedOptions).map((opt) => opt.value);
+            field.selectedOptions = selectedOptions;
+            field.options = Array.from(input.options).map((opt) => opt.value);
+          }
+
+          // 查找关联的 label
+          let label: HTMLLabelElement | null = null;
+
+          if (input.id) {
+            label = document.querySelector(`label[for="${input.id}"]`);
+          }
+
+          if (!label) {
+            // 查找父级 label
+            let parent = input.parentElement;
+            while (parent) {
+              if (parent instanceof HTMLLabelElement) {
+                label = parent;
+                break;
+              }
+              parent = parent.parentElement;
+            }
+          }
+
+          if (label) {
+            field.label = label.textContent?.trim();
+          }
+
+          fields.push(field);
+        }
+
+        return { fields };
+      },
+      { selector: selector || "", includeDisabled },
+    );
+
+    return {
+      type: "form",
+      url,
+      selector: selector || "form",
+      timestamp,
+      fields: result.fields,
+    };
+  }
+}

package/src/extractors/list-extractor.ts

@@ -0,0 +1,213 @@
+import type { Page } from "patchright";
+
+export interface ListData {
+  type: "list";
+  url: string;
+  selector: string;
+  timestamp: number;
+  pattern: string;
+  items: number;
+  data: Record<string, any>[];
+}
+
+export class ListExtractor {
+  /**
+   * 提取列表数据
+   */
+  async extract(
+    page: Page,
+    selector?: string,
+    options: { pattern?: string; maxItems?: number } = {},
+  ): Promise<ListData> {
+    const pattern = options.pattern ?? "auto";
+    const maxItems = options.maxItems ?? Number.POSITIVE_INFINITY;
+
+    const url = page.url();
+    const timestamp = Date.now();
+
+    const result = await page.evaluate(
+      ({ selector, pattern, maxItems }) => {
+        let container: Element | null = null;
+
+        if (selector) {
+          container = document.querySelector(selector);
+        } else {
+          // 查找常见的列表容器
+          container =
+            document.querySelector("ul") ||
+            document.querySelector("ol") ||
+            document.querySelector('[role="list"]') ||
+            document.body;
+        }
+
+        if (!container) {
+          throw new Error("No list container found");
+        }
+
+        // 自动检测重复结构
+        let items: Element[] = [];
+        let detectedPattern = "";
+
+        if (pattern === "auto") {
+          // 尝试不同的模式
+          const patterns = [
+            "li",
+            '[role="listitem"]',
+            ".item",
+            ".product",
+            ".card",
+            "[class*='item']",
+            "[class*='card']",
+            "[class*='product']",
+          ];
+
+          for (const p of patterns) {
+            const elements = Array.from(container.querySelectorAll(p));
+            if (elements.length > 1) {
+              items = elements;
+              detectedPattern = p;
+              break;
+            }
+          }
+
+          // 如果还是找不到，尝试检测相同 class 的兄弟元素
+          if (items.length === 0) {
+            const children = Array.from(container.children);
+            const classMap = new Map<string, Element[]>();
+
+            for (const child of children) {
+              const className = child.className;
+              if (className) {
+                if (!classMap.has(className)) {
+                  classMap.set(className, []);
+                }
+                classMap.get(className)!.push(child);
+              }
+            }
+
+            // 找到最多重复的 class
+            let maxCount = 0;
+            let maxClass = "";
+            classMap.forEach((elements, className) => {
+              if (elements.length > maxCount) {
+                maxCount = elements.length;
+                maxClass = className;
+              }
+            });
+
+            if (maxCount > 1) {
+              items = classMap.get(maxClass)!;
+              detectedPattern = `.${maxClass.split(" ")[0]}`;
+            }
+          }
+        } else {
+          items = Array.from(container.querySelectorAll(pattern));
+          detectedPattern = pattern;
+        }
+
+        if (items.length === 0) {
+          throw new Error("No repeating items found");
+        }
+
+        // 限制数量
+        items = items.slice(0, maxItems);
+
+        // 提取每个 item 的数据
+        const data = items.map((item) => extractItemData(item));
+
+        return {
+          pattern: detectedPattern,
+          data,
+        };
+      },
+      { selector: selector || "", pattern, maxItems },
+    );
+
+    return {
+      type: "list",
+      url,
+      selector: selector || "auto",
+      timestamp,
+      pattern: result.pattern,
+      items: result.data.length,
+      data: result.data,
+    };
+  }
+}
+
+/**
+ * 提取单个列表项的数据
+ */
+function extractItemData(item: Element): Record<string, any> {
+  const data: Record<string, any> = {};
+
+  // 提取 data-* 属性
+  if (item instanceof HTMLElement) {
+    for (const attr of Array.from(item.attributes)) {
+      if (attr.name.startsWith("data-")) {
+        const key = attr.name.substring(5);
+        data[key] = attr.value;
+      }
+    }
+  }
+
+  // 提取 id
+  if (item.id) {
+    data.id = item.id;
+  }
+
+  // 提取文本内容
+  const textNodes: string[] = [];
+  const walker = document.createTreeWalker(item, NodeFilter.SHOW_TEXT, null);
+
+  let node = walker.nextNode();
+  while (node) {
+    const text = node.textContent?.trim();
+    if (text && text.length > 0) {
+      textNodes.push(text);
+    }
+    node = walker.nextNode();
+  }
+
+  if (textNodes.length === 1) {
+    data.text = textNodes[0];
+  } else if (textNodes.length > 1) {
+    data.text = textNodes.join(" ");
+  }
+
+  // 提取链接
+  const link = item.querySelector("a");
+  if (link instanceof HTMLAnchorElement) {
+    data.link = link.href;
+    if (!data.text) {
+      data.text = link.textContent?.trim();
+    }
+  }
+
+  // 提取图片
+  const img = item.querySelector("img");
+  if (img instanceof HTMLImageElement) {
+    data.image = img.src;
+    if (img.alt) {
+      data.imageAlt = img.alt;
+    }
+  }
+
+  // 提取常见字段（通过 class 或元素结构）
+  const title = item.querySelector('[class*="title"], h1, h2, h3, h4, h5, h6');
+  if (title) {
+    data.title = title.textContent?.trim();
+  }
+
+  const price = item.querySelector('[class*="price"]');
+  if (price) {
+    data.price = price.textContent?.trim();
+  }
+
+  const description = item.querySelector('[class*="desc"], [class*="summary"], p');
+  if (description && description !== title) {
+    data.description = description.textContent?.trim();
+  }
+
+  return data;
+}

package/src/extractors/meta-extractor.ts

@@ -0,0 +1,139 @@
+import type { Page } from "patchright";
+
+export interface MetaData {
+  type: "metadata";
+  url: string;
+  timestamp: number;
+  seo?: {
+    title?: string;
+    description?: string;
+    keywords?: string[];
+    canonical?: string;
+    robots?: string;
+  };
+  og?: Record<string, string>;
+  twitter?: Record<string, string>;
+  schema?: any[];
+  other?: Record<string, string>;
+}
+
+export class MetaExtractor {
+  /**
+   * 提取页面元数据
+   */
+  async extract(page: Page, options: { include?: string[] } = {}): Promise<MetaData> {
+    const include = options.include ?? ["seo", "og", "twitter", "schema", "other"];
+
+    const url = page.url();
+    const timestamp = Date.now();
+
+    const result = await page.evaluate((includeTypes) => {
+      const data: any = {};
+
+      // SEO 基础元数据
+      if (includeTypes.includes("seo")) {
+        data.seo = {
+          title: document.title,
+        };
+
+        const description = document.querySelector('meta[name="description"]');
+        if (description instanceof HTMLMetaElement) {
+          data.seo.description = description.content;
+        }
+
+        const keywords = document.querySelector('meta[name="keywords"]');
+        if (keywords instanceof HTMLMetaElement) {
+          data.seo.keywords = keywords.content.split(",").map((k) => k.trim());
+        }
+
+        const canonical = document.querySelector('link[rel="canonical"]');
+        if (canonical instanceof HTMLLinkElement) {
+          data.seo.canonical = canonical.href;
+        }
+
+        const robots = document.querySelector('meta[name="robots"]');
+        if (robots instanceof HTMLMetaElement) {
+          data.seo.robots = robots.content;
+        }
+      }
+
+      // Open Graph
+      if (includeTypes.includes("og")) {
+        data.og = {};
+        const ogTags = document.querySelectorAll('meta[property^="og:"]');
+        for (const tag of Array.from(ogTags)) {
+          if (tag instanceof HTMLMetaElement) {
+            const property = tag.getAttribute("property");
+            if (property) {
+              const key = property.substring(3); // 移除 "og:" 前缀
+              data.og[key] = tag.content;
+            }
+          }
+        }
+      }
+
+      // Twitter Cards
+      if (includeTypes.includes("twitter")) {
+        data.twitter = {};
+        const twitterTags = document.querySelectorAll('meta[name^="twitter:"]');
+        for (const tag of Array.from(twitterTags)) {
+          if (tag instanceof HTMLMetaElement) {
+            const name = tag.getAttribute("name");
+            if (name) {
+              const key = name.substring(8); // 移除 "twitter:" 前缀
+              data.twitter[key] = tag.content;
+            }
+          }
+        }
+      }
+
+      // Schema.org (JSON-LD)
+      if (includeTypes.includes("schema")) {
+        data.schema = [];
+        const scripts = document.querySelectorAll('script[type="application/ld+json"]');
+        for (const script of Array.from(scripts)) {
+          try {
+            const json = JSON.parse(script.textContent || "");
+            data.schema.push(json);
+          } catch (e) {
+            // 忽略解析错误
+          }
+        }
+      }
+
+      // 其他元数据
+      if (includeTypes.includes("other")) {
+        data.other = {};
+
+        const charset = document.characterSet;
+        if (charset) {
+          data.other.charset = charset;
+        }
+
+        const viewport = document.querySelector('meta[name="viewport"]');
+        if (viewport instanceof HTMLMetaElement) {
+          data.other.viewport = viewport.content;
+        }
+
+        const lang = document.documentElement.lang;
+        if (lang) {
+          data.other.lang = lang;
+        }
+
+        const author = document.querySelector('meta[name="author"]');
+        if (author instanceof HTMLMetaElement) {
+          data.other.author = author.content;
+        }
+      }
+
+      return data;
+    }, include);
+
+    return {
+      type: "metadata",
+      url,
+      timestamp,
+      ...result,
+    };
+  }
+}