page-analyzer 1.2.1 → 1.2.2

package/README.md

@@ -250,7 +250,7 @@ const result = await analyzePageEvents({
 
 启用 `blockScreenshots: true` 后，模块会在 LLM 合并区块后再截图。返回结果会包含 `screenshots.blocks`，每项包含逻辑区块序号 `blockIdx` 和对应截图 `path`；区块分析结果中的每个 block 也会额外带上 `blockScreenshotPaths`，每个逻辑区块最多对应一张截图。无法通过 `blockCssPath` 截图的隐藏或空区块会被跳过。
 
-如果配置 `extractorConfig.s3`，截图不会写入本地 `snapshots/`，而是直接上传到 S3；`screenshots.fullPage`、`screenshots.blocks[].path` 和 `blockScreenshotPaths` 会返回 HTTPS URL。上传不会设置 ACL，访问权限沿用 bucket 策略。单张截图上传失败会重试 3 次，仍失败则跳过该截图。
+如果配置 `extractorConfig.s3`，截图不会写入本地 `snapshots/`，而是直接上传到 S3；`screenshots.fullPage`、`screenshots.blocks[].path` 和 `blockScreenshotPaths` 会返回 HTTPS URL。S3 对象 key 使用 `<prefix>/<domain>/<file-md5>.png`，上传前会先检查对象是否已存在，已存在时直接返回对应 URL，避免重复上传和冗余对象。上传不会设置 ACL，访问权限沿用 bucket 策略。单张截图检查或上传失败会重试 3 次，仍失败则跳过该截图。
 
 启用 `waitForImagesLoaded: true` 后，模块会先滚动页面触发懒加载，再等待当前 DOM 中的 `<img>` 完成加载或失败，之后再提取区块、分析和截图；等待时间受 `extractorConfig.timeoutMs` 控制。
 
@@ -316,7 +316,7 @@ const result = await analyzeUrl('https://example.com', {
 });
 ```
 
-`extractorConfig.s3.bucket` 和 `extractorConfig.s3.region` 必填。`credentials` 可省略，省略时使用 AWS SDK 默认凭证链。`publicBaseUrl` 可省略，省略时返回 `https://<bucket>.s3.<region>.amazonaws.com/<key>`；配置后返回 `${publicBaseUrl}/<key>`。
+`extractorConfig.s3.bucket` 和 `extractorConfig.s3.region` 必填。`credentials` 可省略，省略时使用 AWS SDK 默认凭证链。`publicBaseUrl` 可省略，省略时返回 `https://<bucket>.s3.<region>.amazonaws.com/<key>`；配置后返回 `${publicBaseUrl}/<key>`。启用 S3 上传时，需要凭证具备 `s3:GetObject` 和 `s3:PutObject` 权限；如果希望不存在的对象能被稳定识别为 404，还需要对应 bucket/prefix 的 `s3:ListBucket` 权限。
 
 ### parserConfig
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "page-analyzer",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "type": "module",
   "description": "Standalone page analysis module.",
   "license": "MIT",

package/page-extractor.js

@@ -5,7 +5,8 @@
 
 import fs from 'node:fs/promises';
 import path from 'node:path';
-import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { createHash } from 'node:crypto';
+import { HeadObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
 
 // In-browser block extraction function (serialized into page.evaluate)
 // Imported from the project's extract-blocks script
@@ -41,6 +42,24 @@ function createSnapshotRunId() {
     .replace(/^-+|-+$/g, '');
 }
 
+function createS3DomainSegment(url) {
+  const source = String(url || '').trim();
+  try {
+    const parsed = new URL(source);
+    const hostname = parsed.hostname
+      .toLowerCase()
+      .replace(/[^a-z0-9.-]+/g, '-')
+      .replace(/^-+|-+$/g, '');
+    return hostname || 'page';
+  } catch {
+    return createSnapshotSlug(source);
+  }
+}
+
+function createFileMd5(body) {
+  return createHash('md5').update(body).digest('hex');
+}
+
 function getBlockNumber(block, fallbackIndex) {
   return Number.isInteger(block?.blockIdx) ? block.blockIdx : fallbackIndex;
 }
@@ -96,8 +115,8 @@ function normalizeS3Config(config) {
   };
 }
 
-function joinS3Key(prefix, filename) {
-  return [prefix, filename].filter(Boolean).join('/');
+function joinS3Key(...parts) {
+  return parts.filter(Boolean).join('/');
 }
 
 function encodeS3Key(key) {
@@ -119,6 +138,12 @@ function getErrorMessage(error) {
   return error instanceof Error ? error.message : String(error);
 }
 
+function isS3NotFoundError(error) {
+  const statusCode = error?.$metadata?.httpStatusCode;
+  const errorName = String(error?.name || error?.Code || error?.code || '');
+  return statusCode === 404 || errorName === 'NotFound' || errorName === 'NoSuchKey';
+}
+
 export class PageExtractor {
   constructor(config = {}) {
     this.config = {
@@ -176,13 +201,30 @@ export class PageExtractor {
     return this.s3Client;
   }
 
-  async uploadScreenshotToS3(filename, body) {
+  async s3ObjectExists(client, key) {
+    const s3Config = this.config.s3;
+    try {
+      await client.send(new HeadObjectCommand({
+        Bucket: s3Config.bucket,
+        Key: key
+      }));
+      return true;
+    } catch (error) {
+      if (isS3NotFoundError(error)) {
+        return false;
+      }
+      throw error;
+    }
+  }
+
+  async uploadScreenshotToS3(targetUrl, body) {
     const s3Config = this.config.s3;
     if (!s3Config) {
       throw new Error('S3 is not configured');
     }
 
-    const key = joinS3Key(s3Config.prefix, filename);
+    const domain = createS3DomainSegment(targetUrl);
+    const key = joinS3Key(s3Config.prefix, domain, `${createFileMd5(body)}.png`);
     const client = this.getS3Client();
     const commandInput = {
       Bucket: s3Config.bucket,
@@ -194,6 +236,9 @@ export class PageExtractor {
     let lastError = null;
     for (let attempt = 1; attempt <= s3Config.maxUploadAttempts; attempt += 1) {
       try {
+        if (await this.s3ObjectExists(client, key)) {
+          return buildS3Url(s3Config, key);
+        }
         const command = new PutObjectCommand(commandInput);
         await client.send(command);
         return buildS3Url(s3Config, key);
@@ -201,7 +246,7 @@ export class PageExtractor {
         lastError = error;
         if (attempt < s3Config.maxUploadAttempts) {
           console.warn(
-            `[page-analyzer] Failed to upload ${key} to S3, retrying ` +
+            `[page-analyzer] Failed to check/upload ${key} to S3, retrying ` +
             `(${attempt}/${s3Config.maxUploadAttempts}): ${getErrorMessage(error)}`
           );
         }
@@ -572,16 +617,16 @@ export class PageExtractor {
       await fs.mkdir(this.config.snapshotDir, { recursive: true });
     }
 
-    const prefix = `${createSnapshotSlug(targetUrl)}-${createSnapshotRunId()}`;
+    const localPrefix = `${createSnapshotSlug(targetUrl)}-${createSnapshotRunId()}`;
     const screenshots = {};
 
     if (fullPageScreenshot) {
-      const fullPageFilename = `${prefix}-full-page.png`;
       try {
         if (useS3) {
           const body = await page.screenshot({ fullPage: true });
-          screenshots.fullPage = await this.uploadScreenshotToS3(fullPageFilename, body);
+          screenshots.fullPage = await this.uploadScreenshotToS3(targetUrl, body);
         } else {
+          const fullPageFilename = `${localPrefix}-full-page.png`;
           const fullPagePath = path.join(this.config.snapshotDir, fullPageFilename);
           await page.screenshot({
             path: fullPagePath,
@@ -603,15 +648,13 @@ export class PageExtractor {
         const block = blocks[index];
         const blockIdx = getBlockNumber(block, index);
 
-        const blockLabel = String(blockIdx).padStart(3, '0').replace(/[^0-9a-z-]+/gi, '-');
-        const blockFilename = `${prefix}-block-${blockLabel}.png`;
         try {
           if (useS3) {
             const body = await this.captureBlockScreenshotData(page, block);
             if (!body) {
               continue;
             }
-            const url = await this.uploadScreenshotToS3(blockFilename, body);
+            const url = await this.uploadScreenshotToS3(targetUrl, body);
             const screenshotRecord = {
               blockIdx,
               path: url
@@ -626,6 +669,8 @@ export class PageExtractor {
             continue;
           }
 
+          const blockLabel = String(blockIdx).padStart(3, '0').replace(/[^0-9a-z-]+/gi, '-');
+          const blockFilename = `${localPrefix}-block-${blockLabel}.png`;
           const blockPath = path.join(this.config.snapshotDir, blockFilename);
           const captured = await this.captureBlockScreenshot(page, block, blockPath);
           if (captured) {

package/test/smoke.test.js

@@ -1,10 +1,15 @@
 import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
 import { EventAnalyzer } from '../llm/analyzers/event-analyzer/event-analyzer.js';
 import { buildBlockAnalysisArtifact } from '../llm/analyzers/event-analyzer/event-analyzer-blocks.js';
 import { OpenAiProvider } from '../llm/providers/openai-provider.js';
 import { PageExtractor } from '../page-extractor.js';
 import { analyzeUrl } from '../index.js';
 
+function md5(value) {
+  return createHash('md5').update(value).digest('hex');
+}
+
 class FakeProvider {
   constructor() {
     this.calls = [];
@@ -26,9 +31,10 @@ class FakeProvider {
 }
 
 class FakeLocator {
-  constructor({ count = 1, throwOnScreenshot = false } = {}) {
+  constructor({ count = 1, throwOnScreenshot = false, screenshotBodies = null } = {}) {
     this.countValue = count;
     this.throwOnScreenshot = throwOnScreenshot;
+    this.screenshotBodies = Array.isArray(screenshotBodies) ? screenshotBodies : null;
     this.screenshots = [];
   }
 
@@ -41,11 +47,14 @@ class FakeLocator {
   }
 
   async screenshot(options) {
+    const screenshotIndex = this.screenshots.length;
     this.screenshots.push(options);
     if (this.throwOnScreenshot) {
       throw new Error('selector screenshot failed');
     }
-    return Buffer.from(`locator screenshot:${options?.path || 'buffer'}`);
+    return Buffer.from(
+      this.screenshotBodies?.[screenshotIndex] || `locator screenshot:${options?.path || 'buffer'}`
+    );
   }
 }
 
@@ -74,22 +83,36 @@ class FakePage {
 }
 
 class FakeS3Client {
-  constructor({ failPredicate = null } = {}) {
+  constructor({ failPredicate = null, existingKeys = [] } = {}) {
     this.failPredicate = failPredicate;
+    this.existingKeys = new Set(existingKeys);
+    this.headCommands = [];
     this.commands = [];
     this.attemptsByKey = new Map();
   }
 
   async send(command) {
     const input = command.input;
-    this.commands.push(input);
+    if (command.constructor.name === 'HeadObjectCommand') {
+      this.headCommands.push(input);
+      if (this.existingKeys.has(input.Key)) {
+        return {};
+      }
+      const error = new Error(`s3 object not found for ${input.Key}`);
+      error.name = 'NotFound';
+      error.$metadata = { httpStatusCode: 404 };
+      throw error;
+    }
+
     const attempts = (this.attemptsByKey.get(input.Key) || 0) + 1;
     this.attemptsByKey.set(input.Key, attempts);
+    this.commands.push(input);
 
     if (this.failPredicate?.(input, attempts)) {
       throw new Error(`s3 upload failed for ${input.Key}`);
     }
 
+    this.existingKeys.add(input.Key);
     return {};
   }
 }
@@ -348,14 +371,16 @@ async function analyzeWith(options = {}) {
   assert.deepEqual(page.pageScreenshots[0], { fullPage: true });
   assert.equal(locator.screenshots.length, 1);
   assert.deepEqual(locator.screenshots[0], {});
+  assert.equal(s3Client.headCommands.length, 2);
   assert.equal(s3Client.commands.length, 2);
 
   const [fullPageUpload, blockUpload] = s3Client.commands;
   assert.equal(fullPageUpload.Bucket, 'page-analyzer-test');
   assert.equal(fullPageUpload.ContentType, 'image/png');
   assert.equal(Buffer.isBuffer(fullPageUpload.Body), true);
-  assert.match(fullPageUpload.Key, /^page-analyzer\/snapshots\/example-com-demo-.*-full-page\.png$/);
-  assert.match(blockUpload.Key, /^page-analyzer\/snapshots\/example-com-demo-.*-block-000\.png$/);
+  assert.match(fullPageUpload.Key, /^page-analyzer\/snapshots\/example\.com\/[a-f0-9]{32}\.png$/);
+  assert.match(blockUpload.Key, /^page-analyzer\/snapshots\/example\.com\/[a-f0-9]{32}\.png$/);
+  assert.notEqual(fullPageUpload.Key, blockUpload.Key);
 
   assert.equal(
     screenshots.fullPage,
@@ -384,21 +409,49 @@ async function analyzeWith(options = {}) {
   });
 
   const uploadedKey = s3Client.commands[0].Key;
-  assert.match(uploadedKey, /^nested\/prefix\/example-com-demo-.*-block-000\.png$/);
+  assert.match(uploadedKey, /^nested\/prefix\/example\.com\/[a-f0-9]{32}\.png$/);
   assert.equal(
     screenshots.blocks[0].path,
     `https://page-analyzer-test.s3.ap-northeast-1.amazonaws.com/${uploadedKey}`
   );
 }
 
+{
+  const body = Buffer.from('already uploaded screenshot');
+  const existingKey = `page-analyzer/snapshots/example.com/${md5(body)}.png`;
+  const s3Client = new FakeS3Client({
+    existingKeys: [existingKey]
+  });
+  const extractor = new PageExtractor({
+    s3: {
+      bucket: 'page-analyzer-test',
+      region: 'ap-northeast-1',
+      prefix: 'page-analyzer/snapshots',
+      publicBaseUrl: 'https://cdn.example.com',
+      client: s3Client
+    }
+  });
+
+  const url = await extractor.uploadScreenshotToS3('https://example.com/demo', body);
+
+  assert.equal(s3Client.headCommands.length, 1);
+  assert.equal(s3Client.headCommands[0].Key, existingKey);
+  assert.equal(s3Client.commands.length, 0);
+  assert.equal(url, `https://cdn.example.com/${existingKey}`);
+}
+
 {
   const originalWarn = console.warn;
   const warnings = [];
   console.warn = (message) => warnings.push(message);
 
   try {
+    const failingBlockKey =
+      `page-analyzer/snapshots/example.com/${md5('locator screenshot:block-0')}.png`;
+    const successfulBlockKey =
+      `page-analyzer/snapshots/example.com/${md5('locator screenshot:block-1')}.png`;
     const s3Client = new FakeS3Client({
-      failPredicate: (input) => input.Key.endsWith('-block-000.png')
+      failPredicate: (input) => input.Key === failingBlockKey
     });
     const extractor = new PageExtractor({
       s3: {
@@ -409,7 +462,12 @@ async function analyzeWith(options = {}) {
         client: s3Client
       }
     });
-    const locator = new FakeLocator();
+    const locator = new FakeLocator({
+      screenshotBodies: [
+        'locator screenshot:block-0',
+        'locator screenshot:block-1'
+      ]
+    });
     const page = new FakePage(locator);
     const screenshots = await extractor.captureScreenshots(page, 'https://example.com/demo', [
       { blockName: 'Hero', blockCssPath: '#hero' },
@@ -421,8 +479,8 @@ async function analyzeWith(options = {}) {
 
     assert.equal(screenshots.blocks.length, 1);
     assert.equal(screenshots.blocks[0].blockIdx, 1);
-    assert.equal(s3Client.commands.filter((input) => input.Key.endsWith('-block-000.png')).length, 3);
-    assert.equal(s3Client.commands.filter((input) => input.Key.endsWith('-block-001.png')).length, 1);
+    assert.equal(s3Client.commands.filter((input) => input.Key === failingBlockKey).length, 3);
+    assert.equal(s3Client.commands.filter((input) => input.Key === successfulBlockKey).length, 1);
     assert.equal(warnings.some((message) => message.includes('retrying')), true);
     assert.equal(warnings.some((message) => message.includes('Failed to capture/upload block 0')), true);
   } finally {