page-analyzer 1.1.1 → 1.2.1

package/README.md

@@ -151,6 +151,7 @@ const result = await analyzeUrl('https://example.com', {
 | `options.fullPageScreenshot` | `boolean` | 否 | 是否保存整页截图到当前运行目录的 `snapshots/` 并返回文件路径 |
 | `options.blockScreenshots` | `boolean` | 否 | 是否在 LLM 合并区块后，保存每个逻辑区块截图到当前运行目录的 `snapshots/` 并返回文件路径 |
 | `options.waitForImagesLoaded` | `boolean` | 否 | 是否在提取区块、分析和截图前等待页面图片加载完成，默认 `false` |
+| `options.extractorConfig.s3` | `object` | 否 | 截图 S3 上传配置。配置后截图上传到 S3，返回 HTTPS URL；未配置时仍保存到本地 `snapshots/` |
 
 ### analyzePageEvents(input)
 
@@ -249,6 +250,8 @@ const result = await analyzePageEvents({
 
 启用 `blockScreenshots: true` 后，模块会在 LLM 合并区块后再截图。返回结果会包含 `screenshots.blocks`，每项包含逻辑区块序号 `blockIdx` 和对应截图 `path`；区块分析结果中的每个 block 也会额外带上 `blockScreenshotPaths`，每个逻辑区块最多对应一张截图。无法通过 `blockCssPath` 截图的隐藏或空区块会被跳过。
 
+如果配置 `extractorConfig.s3`，截图不会写入本地 `snapshots/`，而是直接上传到 S3；`screenshots.fullPage`、`screenshots.blocks[].path` 和 `blockScreenshotPaths` 会返回 HTTPS URL。上传不会设置 ACL，访问权限沿用 bucket 策略。单张截图上传失败会重试 3 次，仍失败则跳过该截图。
+
 启用 `waitForImagesLoaded: true` 后，模块会先滚动页面触发懒加载，再等待当前 DOM 中的 `<img>` 完成加载或失败，之后再提取区块、分析和截图；等待时间受 `extractorConfig.timeoutMs` 控制。
 
 截图参数启用后的新增输出示例：
@@ -284,6 +287,36 @@ const result = await analyzePageEvents({
 | `blockMaxDepth` | `15` | 区块提取最大 DOM 深度 |
 | `textPreviewMaxChars` | `1200` | 区块文本预览最大长度 |
 | `waitForImagesLoaded` | `false` | 是否在提取区块、分析和截图前等待页面图片加载完成 |
+| `s3` | 无 | 截图 S3 上传配置。配置后截图直接上传到 S3，未配置时保存到本地 |
+
+S3 截图上传示例：
+
+```js
+const result = await analyzeUrl('https://example.com', {
+  fullPageScreenshot: true,
+  blockScreenshots: true,
+  llm: {
+    apiKey: process.env.LLM_API_KEY,
+    apiEndpoint: process.env.LLM_API_ENDPOINT,
+    model: process.env.LLM_MODEL
+  },
+  extractorConfig: {
+    s3: {
+      bucket: 'my-bucket',
+      region: 'ap-northeast-1',
+      prefix: 'page-analyzer/snapshots',
+      publicBaseUrl: 'https://cdn.example.com',
+      credentials: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+        sessionToken: process.env.AWS_SESSION_TOKEN
+      }
+    }
+  }
+});
+```
+
+`extractorConfig.s3.bucket` 和 `extractorConfig.s3.region` 必填。`credentials` 可省略，省略时使用 AWS SDK 默认凭证链。`publicBaseUrl` 可省略，省略时返回 `https://<bucket>.s3.<region>.amazonaws.com/<key>`；配置后返回 `${publicBaseUrl}/<key>`。
 
 ### parserConfig
 

package/index.js

@@ -292,6 +292,7 @@ function buildPageAnalysisResult({
  * @param {boolean} [options.fullPageScreenshot=false] - Save a full-page screenshot to snapshots/ and return its path.
  * @param {boolean} [options.blockScreenshots=false] - Save one screenshot per merged logical block to snapshots/ and return their paths.
  * @param {boolean} [options.waitForImagesLoaded=false] - Wait for page images before extracting and screenshotting.
+ * @param {Object} [options.extractorConfig.s3] - Optional S3 config for uploading screenshots instead of saving locally.
  * @returns {Promise<Object>} Analysis result. Event and idx fields are omitted unless requested.
  */
 export async function analyzeUrl(url, options = {}) {
@@ -323,47 +324,50 @@ export async function analyzeUrl(url, options = {}) {
     blockScreenshots: false,
     waitForImagesLoaded: waitForImagesLoaded ?? extractorConfig?.waitForImagesLoaded
   });
-  const bundle = await extractor.extract(url);
-  console.log(`[page-analyzer] Extracted: ${bundle.blocks.length} blocks, ${bundle.elementGeometries.length} geometries`);
 
-  // Derive domain from URL
-  let domain = '';
-  try { domain = new URL(url).hostname.replace(/^www\./, ''); } catch { /* ignore */ }
-
-  let result = await analyzePageEvents({
-    html: bundle.html,
-    url,
-    blocks: bundle.blocks,
-    elementGeometries: bundle.elementGeometries,
-    llm: llmConfig,
-    knownEventTypes,
-    parserConfig,
-    showEvents,
-    showBlockIdx,
-    screenshots: bundle.screenshots,
-    domain,
-    nodeId: `${domain}-root`
-  });
-
-  if (shouldCaptureBlocks) {
-    const logicalBlocks = Array.isArray(result?.analysis?.block_analysis?.blocks)
-      ? result.analysis.block_analysis.blocks
-      : [];
-    const blockScreenshotsBundle = await extractor.captureUrlScreenshots(url, logicalBlocks, {
-      fullPageScreenshot: false,
-      blockScreenshots: true
+  return await extractor.withPreparedPage(url, async (page, targetUrl) => {
+    const bundle = await extractor.extractPreparedPage(page, targetUrl);
+    console.log(`[page-analyzer] Extracted: ${bundle.blocks.length} blocks, ${bundle.elementGeometries.length} geometries`);
+
+    // Derive domain from URL
+    let domain = '';
+    try { domain = new URL(targetUrl).hostname.replace(/^www\./, ''); } catch { /* ignore */ }
+
+    let result = await analyzePageEvents({
+      html: bundle.html,
+      url: targetUrl,
+      blocks: bundle.blocks,
+      elementGeometries: bundle.elementGeometries,
+      llm: llmConfig,
+      knownEventTypes,
+      parserConfig,
+      showEvents,
+      showBlockIdx,
+      screenshots: bundle.screenshots,
+      domain,
+      nodeId: `${domain}-root`
     });
-    const screenshots = mergeScreenshots(result.screenshots, blockScreenshotsBundle);
-    result = attachLogicalBlockScreenshotPaths(
-      {
-        ...result,
-        ...(screenshots ? { screenshots } : {})
-      },
-      screenshots
-    );
-  }
 
-  return result;
+    if (shouldCaptureBlocks) {
+      const logicalBlocks = Array.isArray(result?.analysis?.block_analysis?.blocks)
+        ? result.analysis.block_analysis.blocks
+        : [];
+      const blockScreenshotsBundle = await extractor.captureScreenshots(page, targetUrl, logicalBlocks, {
+        fullPageScreenshot: false,
+        blockScreenshots: true
+      });
+      const screenshots = mergeScreenshots(result.screenshots, blockScreenshotsBundle);
+      result = attachLogicalBlockScreenshotPaths(
+        {
+          ...result,
+          ...(screenshots ? { screenshots } : {})
+        },
+        screenshots
+      );
+    }
+
+    return result;
+  });
 }
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "page-analyzer",
-  "version": "1.1.1",
+  "version": "1.2.1",
   "type": "module",
   "description": "Standalone page analysis module.",
   "license": "MIT",
@@ -11,6 +11,7 @@
     "viewer": "node scripts/serve-result-viewer.js"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1045.0",
     "cheerio": "^1.2.0",
     "csv-parse": "^5.6.0",
     "playwright": "^1.58.2"

package/page-extractor.js

@@ -5,6 +5,7 @@
 
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
 
 // In-browser block extraction function (serialized into page.evaluate)
 // Imported from the project's extract-blocks script
@@ -49,6 +50,75 @@ function getBlockSelector(block) {
   return selector || '';
 }
 
+function isObject(value) {
+  return value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function normalizeS3Prefix(value) {
+  return String(value || '')
+    .trim()
+    .replace(/^\/+|\/+$/g, '');
+}
+
+function normalizePublicBaseUrl(value) {
+  return String(value || '')
+    .trim()
+    .replace(/\/+$/g, '');
+}
+
+function normalizeS3Config(config) {
+  if (config == null) {
+    return null;
+  }
+  if (!isObject(config)) {
+    throw new Error('extractorConfig.s3 must be an object');
+  }
+
+  const bucket = String(config.bucket || '').trim();
+  const region = String(config.region || '').trim();
+  if (!bucket) {
+    throw new Error('extractorConfig.s3.bucket is required');
+  }
+  if (!region) {
+    throw new Error('extractorConfig.s3.region is required');
+  }
+
+  return {
+    bucket,
+    region,
+    prefix: normalizeS3Prefix(config.prefix),
+    publicBaseUrl: normalizePublicBaseUrl(config.publicBaseUrl),
+    credentials: isObject(config.credentials) ? config.credentials : undefined,
+    client: config.client,
+    maxUploadAttempts: Number.isInteger(config.maxUploadAttempts)
+      ? Math.max(1, config.maxUploadAttempts)
+      : 3
+  };
+}
+
+function joinS3Key(prefix, filename) {
+  return [prefix, filename].filter(Boolean).join('/');
+}
+
+function encodeS3Key(key) {
+  return String(key || '')
+    .split('/')
+    .map((part) => encodeURIComponent(part))
+    .join('/');
+}
+
+function buildS3Url(s3Config, key) {
+  if (s3Config.publicBaseUrl) {
+    return `${s3Config.publicBaseUrl}/${encodeS3Key(key)}`;
+  }
+
+  return `https://${s3Config.bucket}.s3.${s3Config.region}.amazonaws.com/${encodeS3Key(key)}`;
+}
+
+function getErrorMessage(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+
 export class PageExtractor {
   constructor(config = {}) {
     this.config = {
@@ -71,9 +141,11 @@ export class PageExtractor {
       blockScreenshots: Boolean(config.blockScreenshots),
       snapshotDir: typeof config.snapshotDir === 'string' && config.snapshotDir.trim()
         ? path.resolve(process.cwd(), config.snapshotDir)
-        : path.resolve(process.cwd(), 'snapshots')
+        : path.resolve(process.cwd(), 'snapshots'),
+      s3: normalizeS3Config(config.s3)
     };
     this.playwrightModule = null;
+    this.s3Client = null;
   }
 
   async getPlaywright() {
@@ -85,6 +157,60 @@ export class PageExtractor {
     return this.playwrightModule;
   }
 
+  getS3Client() {
+    if (!this.config.s3) {
+      return null;
+    }
+    if (this.config.s3.client) {
+      return this.config.s3.client;
+    }
+    if (this.s3Client) {
+      return this.s3Client;
+    }
+
+    this.s3Client = new S3Client({
+      region: this.config.s3.region,
+      credentials: this.config.s3.credentials,
+      maxAttempts: 1
+    });
+    return this.s3Client;
+  }
+
+  async uploadScreenshotToS3(filename, body) {
+    const s3Config = this.config.s3;
+    if (!s3Config) {
+      throw new Error('S3 is not configured');
+    }
+
+    const key = joinS3Key(s3Config.prefix, filename);
+    const client = this.getS3Client();
+    const commandInput = {
+      Bucket: s3Config.bucket,
+      Key: key,
+      Body: body,
+      ContentType: 'image/png'
+    };
+
+    let lastError = null;
+    for (let attempt = 1; attempt <= s3Config.maxUploadAttempts; attempt += 1) {
+      try {
+        const command = new PutObjectCommand(commandInput);
+        await client.send(command);
+        return buildS3Url(s3Config, key);
+      } catch (error) {
+        lastError = error;
+        if (attempt < s3Config.maxUploadAttempts) {
+          console.warn(
+            `[page-analyzer] Failed to upload ${key} to S3, retrying ` +
+            `(${attempt}/${s3Config.maxUploadAttempts}): ${getErrorMessage(error)}`
+          );
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
   async revealHiddenContent(page) {
     return page.evaluate(() => {
       const CONTENT_THRESHOLD = 20;
@@ -401,14 +527,14 @@ export class PageExtractor {
     });
   }
 
-  async captureBlockScreenshot(page, block, blockPath) {
+  async captureBlockScreenshotData(page, block, screenshotOptions = {}) {
     if (block?.hidden) {
-      return false;
+      return null;
     }
 
     const selector = getBlockSelector(block);
     if (!selector) {
-      return false;
+      return null;
     }
 
     try {
@@ -416,8 +542,7 @@ export class PageExtractor {
       if (await locator.count() > 0) {
         await this.hideExternalFixedOverlays(page, selector);
         try {
-          await locator.screenshot({ path: blockPath });
-          return true;
+          return await locator.screenshot(screenshotOptions);
         } finally {
           await this.restoreExternalFixedOverlays(page);
         }
@@ -426,28 +551,49 @@ export class PageExtractor {
       // Selector-only mode: skip blocks that cannot be captured through CSS.
     }
 
-    return false;
+    return null;
+  }
+
+  async captureBlockScreenshot(page, block, blockPath) {
+    const body = await this.captureBlockScreenshotData(page, block, { path: blockPath });
+    return Boolean(body) || body === undefined;
   }
 
   async captureScreenshots(page, targetUrl, blocks, options = {}) {
     const fullPageScreenshot = options.fullPageScreenshot ?? this.config.fullPageScreenshot;
     const blockScreenshots = options.blockScreenshots ?? this.config.blockScreenshots;
+    const useS3 = Boolean(this.config.s3);
 
     if (!fullPageScreenshot && !blockScreenshots) {
       return null;
     }
 
-    await fs.mkdir(this.config.snapshotDir, { recursive: true });
+    if (!useS3) {
+      await fs.mkdir(this.config.snapshotDir, { recursive: true });
+    }
+
     const prefix = `${createSnapshotSlug(targetUrl)}-${createSnapshotRunId()}`;
     const screenshots = {};
 
     if (fullPageScreenshot) {
-      const fullPagePath = path.join(this.config.snapshotDir, `${prefix}-full-page.png`);
-      await page.screenshot({
-        path: fullPagePath,
-        fullPage: true
-      });
-      screenshots.fullPage = fullPagePath;
+      const fullPageFilename = `${prefix}-full-page.png`;
+      try {
+        if (useS3) {
+          const body = await page.screenshot({ fullPage: true });
+          screenshots.fullPage = await this.uploadScreenshotToS3(fullPageFilename, body);
+        } else {
+          const fullPagePath = path.join(this.config.snapshotDir, fullPageFilename);
+          await page.screenshot({
+            path: fullPagePath,
+            fullPage: true
+          });
+          screenshots.fullPage = fullPagePath;
+        }
+      } catch (error) {
+        console.warn(
+          `[page-analyzer] Failed to capture/upload full-page screenshot: ${getErrorMessage(error)}`
+        );
+      }
     }
 
     if (blockScreenshots) {
@@ -458,8 +604,29 @@ export class PageExtractor {
         const blockIdx = getBlockNumber(block, index);
 
         const blockLabel = String(blockIdx).padStart(3, '0').replace(/[^0-9a-z-]+/gi, '-');
-        const blockPath = path.join(this.config.snapshotDir, `${prefix}-block-${blockLabel}.png`);
+        const blockFilename = `${prefix}-block-${blockLabel}.png`;
         try {
+          if (useS3) {
+            const body = await this.captureBlockScreenshotData(page, block);
+            if (!body) {
+              continue;
+            }
+            const url = await this.uploadScreenshotToS3(blockFilename, body);
+            const screenshotRecord = {
+              blockIdx,
+              path: url
+            };
+            if (typeof block?.blockName === 'string' && block.blockName.trim()) {
+              screenshotRecord.blockName = block.blockName.trim();
+            }
+            if (typeof block?.blockIdxs === 'string' && block.blockIdxs.trim()) {
+              screenshotRecord.blockIdxs = block.blockIdxs.trim();
+            }
+            screenshots.blocks.push(screenshotRecord);
+            continue;
+          }
+
+          const blockPath = path.join(this.config.snapshotDir, blockFilename);
           const captured = await this.captureBlockScreenshot(page, block, blockPath);
           if (captured) {
             const screenshotRecord = {
@@ -475,7 +642,9 @@ export class PageExtractor {
             screenshots.blocks.push(screenshotRecord);
           }
         } catch (error) {
-          console.warn(`[page-analyzer] Failed to capture block ${blockIdx}: ${error.message}`);
+          console.warn(
+            `[page-analyzer] Failed to capture/upload block ${blockIdx}: ${getErrorMessage(error)}`
+          );
         }
       }
     }
@@ -494,11 +663,14 @@ export class PageExtractor {
     await this.waitForImagesLoaded(page);
   }
 
-  async captureUrlScreenshots(url, blocks, options = {}) {
+  async withPreparedPage(url, callback) {
     const targetUrl = String(url || '').trim();
     if (!targetUrl) {
       throw new Error('PageExtractor requires a non-empty URL');
     }
+    if (typeof callback !== 'function') {
+      throw new Error('PageExtractor.withPreparedPage requires a callback');
+    }
 
     const viewport = {
       width: this.config.viewportWidth,
@@ -510,53 +682,60 @@ export class PageExtractor {
     try {
       const page = await browser.newPage({ viewport });
       await this.preparePage(page, targetUrl);
-      return await this.captureScreenshots(page, targetUrl, blocks, options);
+      return await callback(page, targetUrl);
     } finally {
       await browser.close();
     }
   }
 
+  async captureUrlScreenshots(url, blocks, options = {}) {
+    return this.withPreparedPage(url, async (page, targetUrl) => {
+      return await this.captureScreenshots(page, targetUrl, blocks, options);
+    });
+  }
+
   /**
-   * Extract page data: html, blocks, elementGeometries, screenshots
-   * @param {string} url - URL to extract
+   * Extract page data from an already prepared Playwright page:
+   * html, blocks, elementGeometries, screenshots.
+   * When config.s3 is provided, screenshots are uploaded to S3 and returned as URLs.
+   * @param {import('playwright').Page} page - Prepared Playwright page
+   * @param {string} targetUrl - URL loaded in the page
    * @returns {Promise<{html, blocks, elementGeometries, screenshots, pageSize}>}
    */
-  async extract(url) {
-    const targetUrl = String(url || '').trim();
-    if (!targetUrl) {
-      throw new Error('PageExtractor requires a non-empty URL');
-    }
-
+  async extractPreparedPage(page, targetUrl) {
     const viewport = {
       width: this.config.viewportWidth,
       height: this.config.viewportHeight
     };
 
-    const playwright = await this.getPlaywright();
-    const browser = await playwright.chromium.launch({ headless: true });
-    try {
-      const page = await browser.newPage({ viewport });
-      await this.preparePage(page, targetUrl);
+    const html = await page.content();
 
-      const html = await page.content();
+    const minWidth = Math.round(viewport.width * this.config.minBlockWidthRatio);
+    const blocksResult = await page.evaluate(extractBlocksInBrowser, {
+      minHeight: this.config.minBlockHeight,
+      minWidth,
+      maxHeight: Math.round(viewport.height * this.config.blockMaxHeightRatio),
+      maxDepth: this.config.blockMaxDepth,
+      textPreviewMaxChars: this.config.textPreviewMaxChars,
+      debug: false
+    });
+    const blocks = Array.isArray(blocksResult?.blocks) ? blocksResult.blocks : [];
+    const elementGeometries = await this.collectElementGeometries(page);
+    const finalPageSize = await this.collectPageSize(page);
+    const screenshots = await this.captureScreenshots(page, targetUrl, blocks);
 
-      const minWidth = Math.round(viewport.width * this.config.minBlockWidthRatio);
-      const blocksResult = await page.evaluate(extractBlocksInBrowser, {
-        minHeight: this.config.minBlockHeight,
-        minWidth,
-        maxHeight: Math.round(viewport.height * this.config.blockMaxHeightRatio),
-        maxDepth: this.config.blockMaxDepth,
-        textPreviewMaxChars: this.config.textPreviewMaxChars,
-        debug: false
-      });
-      const blocks = Array.isArray(blocksResult?.blocks) ? blocksResult.blocks : [];
-      const elementGeometries = await this.collectElementGeometries(page);
-      const finalPageSize = await this.collectPageSize(page);
-      const screenshots = await this.captureScreenshots(page, targetUrl, blocks);
+    return { html, blocks, elementGeometries, screenshots, pageSize: finalPageSize };
+  }
 
-      return { html, blocks, elementGeometries, screenshots, pageSize: finalPageSize };
-    } finally {
-      await browser.close();
-    }
+  /**
+   * Extract page data: html, blocks, elementGeometries, screenshots.
+   * When config.s3 is provided, screenshots are uploaded to S3 and returned as URLs.
+   * @param {string} url - URL to extract
+   * @returns {Promise<{html, blocks, elementGeometries, screenshots, pageSize}>}
+   */
+  async extract(url) {
+    return this.withPreparedPage(url, async (page, targetUrl) => {
+      return await this.extractPreparedPage(page, targetUrl);
+    });
   }
 }