hyper-agent-browser 0.1.0 → 0.3.1

package/src/extractors/meta-extractor.ts

@@ -0,0 +1,139 @@
+import type { Page } from "patchright";
+
+export interface MetaData {
+  type: "metadata";
+  url: string;
+  timestamp: number;
+  seo?: {
+    title?: string;
+    description?: string;
+    keywords?: string[];
+    canonical?: string;
+    robots?: string;
+  };
+  og?: Record<string, string>;
+  twitter?: Record<string, string>;
+  schema?: any[];
+  other?: Record<string, string>;
+}
+
+export class MetaExtractor {
+  /**
+   * 提取页面元数据
+   */
+  async extract(page: Page, options: { include?: string[] } = {}): Promise<MetaData> {
+    const include = options.include ?? ["seo", "og", "twitter", "schema", "other"];
+
+    const url = page.url();
+    const timestamp = Date.now();
+
+    const result = await page.evaluate((includeTypes) => {
+      const data: any = {};
+
+      // SEO 基础元数据
+      if (includeTypes.includes("seo")) {
+        data.seo = {
+          title: document.title,
+        };
+
+        const description = document.querySelector('meta[name="description"]');
+        if (description instanceof HTMLMetaElement) {
+          data.seo.description = description.content;
+        }
+
+        const keywords = document.querySelector('meta[name="keywords"]');
+        if (keywords instanceof HTMLMetaElement) {
+          data.seo.keywords = keywords.content.split(",").map((k) => k.trim());
+        }
+
+        const canonical = document.querySelector('link[rel="canonical"]');
+        if (canonical instanceof HTMLLinkElement) {
+          data.seo.canonical = canonical.href;
+        }
+
+        const robots = document.querySelector('meta[name="robots"]');
+        if (robots instanceof HTMLMetaElement) {
+          data.seo.robots = robots.content;
+        }
+      }
+
+      // Open Graph
+      if (includeTypes.includes("og")) {
+        data.og = {};
+        const ogTags = document.querySelectorAll('meta[property^="og:"]');
+        for (const tag of Array.from(ogTags)) {
+          if (tag instanceof HTMLMetaElement) {
+            const property = tag.getAttribute("property");
+            if (property) {
+              const key = property.substring(3); // 移除 "og:" 前缀
+              data.og[key] = tag.content;
+            }
+          }
+        }
+      }
+
+      // Twitter Cards
+      if (includeTypes.includes("twitter")) {
+        data.twitter = {};
+        const twitterTags = document.querySelectorAll('meta[name^="twitter:"]');
+        for (const tag of Array.from(twitterTags)) {
+          if (tag instanceof HTMLMetaElement) {
+            const name = tag.getAttribute("name");
+            if (name) {
+              const key = name.substring(8); // 移除 "twitter:" 前缀
+              data.twitter[key] = tag.content;
+            }
+          }
+        }
+      }
+
+      // Schema.org (JSON-LD)
+      if (includeTypes.includes("schema")) {
+        data.schema = [];
+        const scripts = document.querySelectorAll('script[type="application/ld+json"]');
+        for (const script of Array.from(scripts)) {
+          try {
+            const json = JSON.parse(script.textContent || "");
+            data.schema.push(json);
+          } catch (e) {
+            // 忽略解析错误
+          }
+        }
+      }
+
+      // 其他元数据
+      if (includeTypes.includes("other")) {
+        data.other = {};
+
+        const charset = document.characterSet;
+        if (charset) {
+          data.other.charset = charset;
+        }
+
+        const viewport = document.querySelector('meta[name="viewport"]');
+        if (viewport instanceof HTMLMetaElement) {
+          data.other.viewport = viewport.content;
+        }
+
+        const lang = document.documentElement.lang;
+        if (lang) {
+          data.other.lang = lang;
+        }
+
+        const author = document.querySelector('meta[name="author"]');
+        if (author instanceof HTMLMetaElement) {
+          data.other.author = author.content;
+        }
+      }
+
+      return data;
+    }, include);
+
+    return {
+      type: "metadata",
+      url,
+      timestamp,
+      ...result,
+    };
+  }
+}

package/src/extractors/table-extractor.ts

@@ -0,0 +1,215 @@
+import type { Page } from "patchright";
+
+export interface TableData {
+  type: "table";
+  url: string;
+  selector: string;
+  timestamp: number;
+  headers: string[];
+  rows: number;
+  data: Record<string, string>[];
+}
+
+export class TableExtractor {
+  /**
+   * 提取表格数据
+   */
+  async extract(
+    page: Page,
+    selector?: string,
+    options: { includeHeaders?: boolean; maxRows?: number } = {},
+  ): Promise<TableData> {
+    const maxRows = options.maxRows ?? Number.POSITIVE_INFINITY;
+
+    const url = page.url();
+    const timestamp = Date.now();
+
+    // 在页面上下文中执行提取逻辑
+    const result = await page.evaluate(
+      ({ selector, maxRows }) => {
+        // 查找表格元素
+        let table: HTMLTableElement | null = null;
+
+        if (selector) {
+          const element = document.querySelector(selector);
+          if (element instanceof HTMLTableElement) {
+            table = element;
+          } else if (element) {
+            // 在指定元素内查找表格
+            const nestedTable = element.querySelector("table");
+            if (nestedTable instanceof HTMLTableElement) {
+              table = nestedTable;
+            }
+          }
+        } else {
+          // 查找第一个可见的表格
+          const tables = Array.from(document.querySelectorAll("table"));
+          table =
+            tables.find((t) => {
+              const style = window.getComputedStyle(t);
+              return style.display !== "none" && style.visibility !== "hidden";
+            }) || null;
+        }
+
+        if (!table) {
+          // 尝试查找 ARIA table
+          const ariaTable = selector
+            ? document.querySelector(selector)
+            : document.querySelector('[role="table"], [role="grid"]');
+
+          if (ariaTable) {
+            return extractAriaTable(ariaTable as HTMLElement, maxRows);
+          }
+
+          throw new Error("No table found on page");
+        }
+
+        return extractHtmlTable(table, maxRows);
+      },
+      { selector: selector || "", maxRows },
+    );
+
+    return {
+      type: "table",
+      url,
+      selector: selector || "table",
+      timestamp,
+      headers: result.headers,
+      rows: result.data.length,
+      data: result.data,
+    };
+  }
+}
+
+/**
+ * 在页面上下文中执行的辅助函数
+ */
+
+// 提取 HTML 表格
+function extractHtmlTable(
+  table: HTMLTableElement,
+  maxRows: number,
+): { headers: string[]; data: Record<string, string>[] } {
+  const headers: string[] = [];
+  const data: Record<string, string>[] = [];
+
+  // 提取表头
+  const thead = table.querySelector("thead");
+  const headerRow = thead?.querySelector("tr") || table.querySelector("tr");
+
+  if (headerRow) {
+    const headerCells = Array.from(headerRow.querySelectorAll("th, td"));
+    headerCells.forEach((cell, index) => {
+      const text = cell.textContent?.trim() || "";
+      const headerText = text || `column_${index + 1}`;
+      headers.push(headerText);
+    });
+  }
+
+  // 如果没有表头，根据第一行推断
+  if (headers.length === 0) {
+    const firstRow = table.querySelector("tr");
+    if (firstRow) {
+      const cells = Array.from(firstRow.querySelectorAll("td, th"));
+      cells.forEach((_, index) => {
+        headers.push(`column_${index + 1}`);
+      });
+    }
+  }
+
+  // 提取数据行
+  const tbody = table.querySelector("tbody") || table;
+  const rows = Array.from(tbody.querySelectorAll("tr"));
+
+  // 跳过表头行（如果表头在 tbody 内）
+  const startIndex = thead ? 0 : 1;
+
+  for (let i = startIndex; i < rows.length && data.length < maxRows; i++) {
+    const row = rows[i];
+
+    // 跳过隐藏行
+    const style = window.getComputedStyle(row);
+    if (style.display === "none" || style.visibility === "hidden") {
+      continue;
+    }
+
+    const cells = Array.from(row.querySelectorAll("td, th"));
+    const rowData: Record<string, string> = {};
+
+    cells.forEach((cell, index) => {
+      if (index >= headers.length) return;
+
+      const text = cell.textContent?.trim() || "";
+      // 处理合并单元格
+      const colspan = Number.parseInt(cell.getAttribute("colspan") || "1");
+
+      if (colspan > 1) {
+        // 对于合并的单元格，填充到多个列
+        for (let j = 0; j < colspan && index + j < headers.length; j++) {
+          rowData[headers[index + j]] = text;
+        }
+      } else {
+        rowData[headers[index]] = text;
+      }
+    });
+
+    // 只添加非空行
+    if (Object.values(rowData).some((v) => v !== "")) {
+      data.push(rowData);
+    }
+  }
+
+  return { headers, data };
+}
+
+// 提取 ARIA 表格
+function extractAriaTable(
+  table: HTMLElement,
+  maxRows: number,
+): { headers: string[]; data: Record<string, string>[] } {
+  const headers: string[] = [];
+  const data: Record<string, string>[] = [];
+
+  // 查找表头行
+  const headerRow = table.querySelector('[role="row"]:first-child');
+  if (headerRow) {
+    const headerCells = Array.from(
+      headerRow.querySelectorAll('[role="columnheader"], [role="gridcell"]'),
+    );
+    headerCells.forEach((cell, index) => {
+      const text = cell.textContent?.trim() || "";
+      headers.push(text || `column_${index + 1}`);
+    });
+  }
+
+  // 查找数据行
+  const rows = Array.from(table.querySelectorAll('[role="row"]'));
+
+  // 跳过表头行
+  const startIndex = headerRow ? 1 : 0;
+
+  for (let i = startIndex; i < rows.length && data.length < maxRows; i++) {
+    const row = rows[i];
+
+    // 跳过隐藏行
+    const style = window.getComputedStyle(row);
+    if (style.display === "none" || style.visibility === "hidden") {
+      continue;
+    }
+
+    const cells = Array.from(row.querySelectorAll('[role="gridcell"], [role="cell"]'));
+    const rowData: Record<string, string> = {};
+
+    cells.forEach((cell, index) => {
+      if (index >= headers.length) return;
+      const text = cell.textContent?.trim() || "";
+      rowData[headers[index]] = text;
+    });
+
+    if (Object.values(rowData).some((v) => v !== "")) {
+      data.push(rowData);
+    }
+  }
+
+  return { headers, data };
+}

package/src/snapshot/dom-extractor.ts

@@ -101,30 +101,43 @@ export class DomSnapshotExtractor {
         // Try ID first
         if (el.id) return `#${el.id}`;
 
-        // Generate nth-child selector
-        let selector = el.tagName.toLowerCase();
+        // Build path from element to root (or an element with ID)
+        const path: string[] = [];
         let current: Element | null = el;
 
-        while (current?.parentElement) {
-          const parent: Element = current.parentElement;
-          const siblings = Array.from(parent.children);
-          const index = siblings.indexOf(current);
-
-          if (index >= 0) {
-            selector = `${parent.tagName.toLowerCase()} > ${selector}:nth-child(${index + 1})`;
+        while (current && current !== document.body && path.length < 6) {
+          const parentEl: Element | null = current.parentElement;
+          if (!parentEl) break;
+
+          // Find index among siblings of same tag type
+          const currentTag = current.tagName;
+          const siblings: Element[] = [];
+          for (let i = 0; i < parentEl.children.length; i++) {
+            const child = parentEl.children[i];
+            if (child.tagName === currentTag) {
+              siblings.push(child);
+            }
           }
+          const index = siblings.indexOf(current) + 1;
+
+          // Use nth-of-type for uniqueness among same-tag siblings
+          const segment =
+            siblings.length > 1
+              ? `${current.tagName.toLowerCase()}:nth-of-type(${index})`
+              : current.tagName.toLowerCase();
+
+          path.unshift(segment);
 
-          current = parent;
-          if (current?.id) {
-            selector = `#${current.id} ${selector}`;
+          // Stop if parent has ID
+          if (parentEl.id) {
+            path.unshift(`#${parentEl.id}`);
             break;
           }
 
-          // Limit depth
-          if (selector.split(">").length > 5) break;
+          current = parentEl;
         }
 
-        return selector;
+        return path.join(" > ");
       }
 
       // Traverse DOM

package/src/utils/network-listener.ts

@@ -0,0 +1,247 @@
+import { existsSync } from "node:fs";
+import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { Page } from "patchright";
+
+export interface NetworkListenerConfig {
+  id: string;
+  sessionDir: string;
+  filter: {
+    types: string[];
+    urlPattern?: string;
+    methods?: string[];
+  };
+  startTime: number;
+  status: "active" | "stopped";
+}
+
+export interface NetworkRequest {
+  id: string;
+  url: string;
+  method: string;
+  request: {
+    headers: Record<string, string>;
+    postData?: any;
+  };
+  response?: {
+    status: number;
+    statusText: string;
+    headers: Record<string, string>;
+    body?: any;
+  };
+  timing: {
+    startTime: number;
+    endTime?: number;
+    duration?: number;
+  };
+}
+
+export class NetworkListener {
+  private config: NetworkListenerConfig;
+  private metaFile: string;
+  private requestsFile: string;
+  private requestCount = 0;
+  private cleanup?: () => void;
+
+  constructor(config: NetworkListenerConfig) {
+    this.config = config;
+    const networkDir = join(config.sessionDir, "network", config.id);
+    this.metaFile = join(networkDir, "meta.json");
+    this.requestsFile = join(networkDir, "requests.jsonl");
+  }
+
+  /**
+   * 启动网络监听
+   */
+  async start(page: Page): Promise<void> {
+    // 确保目录存在
+    const networkDir = join(this.config.sessionDir, "network", this.config.id);
+    if (!existsSync(networkDir)) {
+      await mkdir(networkDir, { recursive: true, mode: 0o700 });
+    }
+
+    // 保存元数据
+    await this.saveMeta();
+
+    // 设置请求拦截
+    const requestHandler = async (request: any) => {
+      const resourceType = request.resourceType();
+      const method = request.method();
+      const url = request.url();
+
+      // 过滤资源类型
+      if (!this.shouldCapture(resourceType, method, url)) {
+        return;
+      }
+
+      const requestId = `req_${++this.requestCount}`;
+      const requestData: NetworkRequest = {
+        id: requestId,
+        url,
+        method,
+        request: {
+          headers: request.headers(),
+          postData: request.postData(),
+        },
+        timing: {
+          startTime: Date.now(),
+        },
+      };
+
+      // 等待响应
+      try {
+        const response = await request.response();
+        if (response) {
+          requestData.response = {
+            status: response.status(),
+            statusText: response.statusText(),
+            headers: response.headers(),
+          };
+
+          // 尝试获取响应体
+          try {
+            const contentType = response.headers()["content-type"] || "";
+            if (contentType.includes("application/json")) {
+              requestData.response.body = await response.json();
+            } else if (contentType.includes("text/")) {
+              requestData.response.body = await response.text();
+            }
+          } catch {
+            // 忽略响应体获取失败
+          }
+
+          requestData.timing.endTime = Date.now();
+          requestData.timing.duration = requestData.timing.endTime - requestData.timing.startTime;
+        }
+      } catch {
+        // 请求失败
+      }
+
+      // 追加到文件
+      await this.appendRequest(requestData);
+    };
+
+    page.on("request", requestHandler);
+
+    // 保存清理函数
+    this.cleanup = () => {
+      page.off("request", requestHandler);
+    };
+
+    // 设置自动超时清理（1小时）
+    setTimeout(
+      async () => {
+        if (this.config.status === "active") {
+          await this.stop();
+        }
+      },
+      60 * 60 * 1000,
+    );
+  }
+
+  /**
+   * 停止监听
+   */
+  async stop(): Promise<void> {
+    this.config.status = "stopped";
+    await this.saveMeta();
+
+    if (this.cleanup) {
+      this.cleanup();
+    }
+  }
+
+  /**
+   * 获取捕获的请求
+   */
+  async getRequests(): Promise<NetworkRequest[]> {
+    if (!existsSync(this.requestsFile)) {
+      return [];
+    }
+
+    const content = await readFile(this.requestsFile, "utf-8");
+    const lines = content.trim().split("\n");
+    return lines.filter((line) => line).map((line) => JSON.parse(line));
+  }
+
+  /**
+   * 获取监听器信息
+   */
+  getInfo(): {
+    listenerId: string;
+    startTime: number;
+    status: string;
+    filter: any;
+  } {
+    return {
+      listenerId: this.config.id,
+      startTime: this.config.startTime,
+      status: this.config.status,
+      filter: this.config.filter,
+    };
+  }
+
+  /**
+   * 判断是否应该捕获请求
+   */
+  private shouldCapture(resourceType: string, method: string, url: string): boolean {
+    // 资源类型过滤
+    const typeMap: Record<string, string> = {
+      xhr: "xhr",
+      fetch: "fetch",
+      document: "document",
+      script: "script",
+      image: "image",
+      font: "font",
+      stylesheet: "stylesheet",
+    };
+
+    const mappedType = typeMap[resourceType];
+    if (!mappedType || !this.config.filter.types.includes(mappedType)) {
+      return false;
+    }
+
+    // HTTP 方法过滤
+    if (this.config.filter.methods && this.config.filter.methods.length > 0) {
+      if (!this.config.filter.methods.includes(method)) {
+        return false;
+      }
+    }
+
+    // URL 模式过滤
+    if (this.config.filter.urlPattern) {
+      const pattern = this.config.filter.urlPattern;
+      // 简单的 glob 模式匹配
+      const regex = new RegExp(pattern.replace(/\*/g, ".*").replace(/\?/g, "."));
+      if (!regex.test(url)) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  /**
+   * 保存元数据
+   */
+  private async saveMeta(): Promise<void> {
+    await writeFile(this.metaFile, JSON.stringify(this.config, null, 2));
+  }
+
+  /**
+   * 追加请求数据
+   */
+  private async appendRequest(request: NetworkRequest): Promise<void> {
+    const line = `${JSON.stringify(request)}\n`;
+    await appendFile(this.requestsFile, line);
+  }
+}
+
+/**
+ * 生成唯一的监听器 ID
+ */
+export function generateListenerId(): string {
+  const timestamp = Date.now().toString(36);
+  const random = Math.random().toString(36).substring(2, 8);
+  return `listener_${timestamp}_${random}`;
+}