hyper-agent-browser 0.1.0 → 0.3.0

package/src/extractors/table-extractor.ts

@@ -0,0 +1,215 @@
+import type { Page } from "patchright";
+
+export interface TableData {
+  type: "table";
+  url: string;
+  selector: string;
+  timestamp: number;
+  headers: string[];
+  rows: number;
+  data: Record<string, string>[];
+}
+
+export class TableExtractor {
+  /**
+   * 提取表格数据
+   */
+  async extract(
+    page: Page,
+    selector?: string,
+    options: { includeHeaders?: boolean; maxRows?: number } = {},
+  ): Promise<TableData> {
+    const maxRows = options.maxRows ?? Number.POSITIVE_INFINITY;
+
+    const url = page.url();
+    const timestamp = Date.now();
+
+    // 在页面上下文中执行提取逻辑
+    const result = await page.evaluate(
+      ({ selector, maxRows }) => {
+        // 查找表格元素
+        let table: HTMLTableElement | null = null;
+
+        if (selector) {
+          const element = document.querySelector(selector);
+          if (element instanceof HTMLTableElement) {
+            table = element;
+          } else if (element) {
+            // 在指定元素内查找表格
+            const nestedTable = element.querySelector("table");
+            if (nestedTable instanceof HTMLTableElement) {
+              table = nestedTable;
+            }
+          }
+        } else {
+          // 查找第一个可见的表格
+          const tables = Array.from(document.querySelectorAll("table"));
+          table =
+            tables.find((t) => {
+              const style = window.getComputedStyle(t);
+              return style.display !== "none" && style.visibility !== "hidden";
+            }) || null;
+        }
+
+        if (!table) {
+          // 尝试查找 ARIA table
+          const ariaTable = selector
+            ? document.querySelector(selector)
+            : document.querySelector('[role="table"], [role="grid"]');
+
+          if (ariaTable) {
+            return extractAriaTable(ariaTable as HTMLElement, maxRows);
+          }
+
+          throw new Error("No table found on page");
+        }
+
+        return extractHtmlTable(table, maxRows);
+      },
+      { selector: selector || "", maxRows },
+    );
+
+    return {
+      type: "table",
+      url,
+      selector: selector || "table",
+      timestamp,
+      headers: result.headers,
+      rows: result.data.length,
+      data: result.data,
+    };
+  }
+}
+
+/**
+ * 在页面上下文中执行的辅助函数
+ */
+
+// 提取 HTML 表格
+function extractHtmlTable(
+  table: HTMLTableElement,
+  maxRows: number,
+): { headers: string[]; data: Record<string, string>[] } {
+  const headers: string[] = [];
+  const data: Record<string, string>[] = [];
+
+  // 提取表头
+  const thead = table.querySelector("thead");
+  const headerRow = thead?.querySelector("tr") || table.querySelector("tr");
+
+  if (headerRow) {
+    const headerCells = Array.from(headerRow.querySelectorAll("th, td"));
+    headerCells.forEach((cell, index) => {
+      const text = cell.textContent?.trim() || "";
+      const headerText = text || `column_${index + 1}`;
+      headers.push(headerText);
+    });
+  }
+
+  // 如果没有表头，根据第一行推断
+  if (headers.length === 0) {
+    const firstRow = table.querySelector("tr");
+    if (firstRow) {
+      const cells = Array.from(firstRow.querySelectorAll("td, th"));
+      cells.forEach((_, index) => {
+        headers.push(`column_${index + 1}`);
+      });
+    }
+  }
+
+  // 提取数据行
+  const tbody = table.querySelector("tbody") || table;
+  const rows = Array.from(tbody.querySelectorAll("tr"));
+
+  // 跳过表头行（如果表头在 tbody 内）
+  const startIndex = thead ? 0 : 1;
+
+  for (let i = startIndex; i < rows.length && data.length < maxRows; i++) {
+    const row = rows[i];
+
+    // 跳过隐藏行
+    const style = window.getComputedStyle(row);
+    if (style.display === "none" || style.visibility === "hidden") {
+      continue;
+    }
+
+    const cells = Array.from(row.querySelectorAll("td, th"));
+    const rowData: Record<string, string> = {};
+
+    cells.forEach((cell, index) => {
+      if (index >= headers.length) return;
+
+      const text = cell.textContent?.trim() || "";
+      // 处理合并单元格
+      const colspan = Number.parseInt(cell.getAttribute("colspan") || "1");
+
+      if (colspan > 1) {
+        // 对于合并的单元格，填充到多个列
+        for (let j = 0; j < colspan && index + j < headers.length; j++) {
+          rowData[headers[index + j]] = text;
+        }
+      } else {
+        rowData[headers[index]] = text;
+      }
+    });
+
+    // 只添加非空行
+    if (Object.values(rowData).some((v) => v !== "")) {
+      data.push(rowData);
+    }
+  }
+
+  return { headers, data };
+}
+
+// 提取 ARIA 表格
+function extractAriaTable(
+  table: HTMLElement,
+  maxRows: number,
+): { headers: string[]; data: Record<string, string>[] } {
+  const headers: string[] = [];
+  const data: Record<string, string>[] = [];
+
+  // 查找表头行
+  const headerRow = table.querySelector('[role="row"]:first-child');
+  if (headerRow) {
+    const headerCells = Array.from(
+      headerRow.querySelectorAll('[role="columnheader"], [role="gridcell"]'),
+    );
+    headerCells.forEach((cell, index) => {
+      const text = cell.textContent?.trim() || "";
+      headers.push(text || `column_${index + 1}`);
+    });
+  }
+
+  // 查找数据行
+  const rows = Array.from(table.querySelectorAll('[role="row"]'));
+
+  // 跳过表头行
+  const startIndex = headerRow ? 1 : 0;
+
+  for (let i = startIndex; i < rows.length && data.length < maxRows; i++) {
+    const row = rows[i];
+
+    // 跳过隐藏行
+    const style = window.getComputedStyle(row);
+    if (style.display === "none" || style.visibility === "hidden") {
+      continue;
+    }
+
+    const cells = Array.from(row.querySelectorAll('[role="gridcell"], [role="cell"]'));
+    const rowData: Record<string, string> = {};
+
+    cells.forEach((cell, index) => {
+      if (index >= headers.length) return;
+      const text = cell.textContent?.trim() || "";
+      rowData[headers[index]] = text;
+    });
+
+    if (Object.values(rowData).some((v) => v !== "")) {
+      data.push(rowData);
+    }
+  }
+
+  return { headers, data };
+}

package/src/utils/network-listener.ts

@@ -0,0 +1,247 @@
+import { existsSync } from "node:fs";
+import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { Page } from "patchright";
+
+export interface NetworkListenerConfig {
+  id: string;
+  sessionDir: string;
+  filter: {
+    types: string[];
+    urlPattern?: string;
+    methods?: string[];
+  };
+  startTime: number;
+  status: "active" | "stopped";
+}
+
+export interface NetworkRequest {
+  id: string;
+  url: string;
+  method: string;
+  request: {
+    headers: Record<string, string>;
+    postData?: any;
+  };
+  response?: {
+    status: number;
+    statusText: string;
+    headers: Record<string, string>;
+    body?: any;
+  };
+  timing: {
+    startTime: number;
+    endTime?: number;
+    duration?: number;
+  };
+}
+
+export class NetworkListener {
+  private config: NetworkListenerConfig;
+  private metaFile: string;
+  private requestsFile: string;
+  private requestCount = 0;
+  private cleanup?: () => void;
+
+  constructor(config: NetworkListenerConfig) {
+    this.config = config;
+    const networkDir = join(config.sessionDir, "network", config.id);
+    this.metaFile = join(networkDir, "meta.json");
+    this.requestsFile = join(networkDir, "requests.jsonl");
+  }
+
+  /**
+   * 启动网络监听
+   */
+  async start(page: Page): Promise<void> {
+    // 确保目录存在
+    const networkDir = join(this.config.sessionDir, "network", this.config.id);
+    if (!existsSync(networkDir)) {
+      await mkdir(networkDir, { recursive: true, mode: 0o700 });
+    }
+
+    // 保存元数据
+    await this.saveMeta();
+
+    // 设置请求拦截
+    const requestHandler = async (request: any) => {
+      const resourceType = request.resourceType();
+      const method = request.method();
+      const url = request.url();
+
+      // 过滤资源类型
+      if (!this.shouldCapture(resourceType, method, url)) {
+        return;
+      }
+
+      const requestId = `req_${++this.requestCount}`;
+      const requestData: NetworkRequest = {
+        id: requestId,
+        url,
+        method,
+        request: {
+          headers: request.headers(),
+          postData: request.postData(),
+        },
+        timing: {
+          startTime: Date.now(),
+        },
+      };
+
+      // 等待响应
+      try {
+        const response = await request.response();
+        if (response) {
+          requestData.response = {
+            status: response.status(),
+            statusText: response.statusText(),
+            headers: response.headers(),
+          };
+
+          // 尝试获取响应体
+          try {
+            const contentType = response.headers()["content-type"] || "";
+            if (contentType.includes("application/json")) {
+              requestData.response.body = await response.json();
+            } else if (contentType.includes("text/")) {
+              requestData.response.body = await response.text();
+            }
+          } catch {
+            // 忽略响应体获取失败
+          }
+
+          requestData.timing.endTime = Date.now();
+          requestData.timing.duration = requestData.timing.endTime - requestData.timing.startTime;
+        }
+      } catch {
+        // 请求失败
+      }
+
+      // 追加到文件
+      await this.appendRequest(requestData);
+    };
+
+    page.on("request", requestHandler);
+
+    // 保存清理函数
+    this.cleanup = () => {
+      page.off("request", requestHandler);
+    };
+
+    // 设置自动超时清理（1小时）
+    setTimeout(
+      async () => {
+        if (this.config.status === "active") {
+          await this.stop();
+        }
+      },
+      60 * 60 * 1000,
+    );
+  }
+
+  /**
+   * 停止监听
+   */
+  async stop(): Promise<void> {
+    this.config.status = "stopped";
+    await this.saveMeta();
+
+    if (this.cleanup) {
+      this.cleanup();
+    }
+  }
+
+  /**
+   * 获取捕获的请求
+   */
+  async getRequests(): Promise<NetworkRequest[]> {
+    if (!existsSync(this.requestsFile)) {
+      return [];
+    }
+
+    const content = await readFile(this.requestsFile, "utf-8");
+    const lines = content.trim().split("\n");
+    return lines.filter((line) => line).map((line) => JSON.parse(line));
+  }
+
+  /**
+   * 获取监听器信息
+   */
+  getInfo(): {
+    listenerId: string;
+    startTime: number;
+    status: string;
+    filter: any;
+  } {
+    return {
+      listenerId: this.config.id,
+      startTime: this.config.startTime,
+      status: this.config.status,
+      filter: this.config.filter,
+    };
+  }
+
+  /**
+   * 判断是否应该捕获请求
+   */
+  private shouldCapture(resourceType: string, method: string, url: string): boolean {
+    // 资源类型过滤
+    const typeMap: Record<string, string> = {
+      xhr: "xhr",
+      fetch: "fetch",
+      document: "document",
+      script: "script",
+      image: "image",
+      font: "font",
+      stylesheet: "stylesheet",
+    };
+
+    const mappedType = typeMap[resourceType];
+    if (!mappedType || !this.config.filter.types.includes(mappedType)) {
+      return false;
+    }
+
+    // HTTP 方法过滤
+    if (this.config.filter.methods && this.config.filter.methods.length > 0) {
+      if (!this.config.filter.methods.includes(method)) {
+        return false;
+      }
+    }
+
+    // URL 模式过滤
+    if (this.config.filter.urlPattern) {
+      const pattern = this.config.filter.urlPattern;
+      // 简单的 glob 模式匹配
+      const regex = new RegExp(pattern.replace(/\*/g, ".*").replace(/\?/g, "."));
+      if (!regex.test(url)) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  /**
+   * 保存元数据
+   */
+  private async saveMeta(): Promise<void> {
+    await writeFile(this.metaFile, JSON.stringify(this.config, null, 2));
+  }
+
+  /**
+   * 追加请求数据
+   */
+  private async appendRequest(request: NetworkRequest): Promise<void> {
+    const line = `${JSON.stringify(request)}\n`;
+    await appendFile(this.requestsFile, line);
+  }
+}
+
+/**
+ * 生成唯一的监听器 ID
+ */
+export function generateListenerId(): string {
+  const timestamp = Date.now().toString(36);
+  const random = Math.random().toString(36).substring(2, 8);
+  return `listener_${timestamp}_${random}`;
+}