page-analyzer 1.0.1 → 1.2.0

package/page-extractor.js

@@ -3,6 +3,10 @@
  * Launches headless Chromium, navigates to URL, scrolls, extracts blocks + element geometries + HTML.
  */
 
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+
 // In-browser block extraction function (serialized into page.evaluate)
 // Imported from the project's extract-blocks script
 import {
@@ -11,6 +15,110 @@ import {
   waitForStableHeight
 } from './vendor/extract-blocks.js';
 
+function createSnapshotSlug(url) {
+  let source = String(url || '').trim();
+  try {
+    const parsed = new URL(source);
+    source = `${parsed.hostname}${parsed.pathname}`;
+  } catch {
+    // Keep the raw value for non-URL inputs.
+  }
+
+  const slug = source
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 80);
+
+  return slug || 'page';
+}
+
+function createSnapshotRunId() {
+  return new Date()
+    .toISOString()
+    .replace(/\.\d{3}z$/i, '')
+    .replace(/[^0-9a-z]+/gi, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+function getBlockNumber(block, fallbackIndex) {
+  return Number.isInteger(block?.blockIdx) ? block.blockIdx : fallbackIndex;
+}
+
+function getBlockSelector(block) {
+  const selector = typeof block?.blockCssPath === 'string' ? block.blockCssPath.trim() : '';
+  return selector || '';
+}
+
+function isObject(value) {
+  return value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function normalizeS3Prefix(value) {
+  return String(value || '')
+    .trim()
+    .replace(/^\/+|\/+$/g, '');
+}
+
+function normalizePublicBaseUrl(value) {
+  return String(value || '')
+    .trim()
+    .replace(/\/+$/g, '');
+}
+
+function normalizeS3Config(config) {
+  if (config == null) {
+    return null;
+  }
+  if (!isObject(config)) {
+    throw new Error('extractorConfig.s3 must be an object');
+  }
+
+  const bucket = String(config.bucket || '').trim();
+  const region = String(config.region || '').trim();
+  if (!bucket) {
+    throw new Error('extractorConfig.s3.bucket is required');
+  }
+  if (!region) {
+    throw new Error('extractorConfig.s3.region is required');
+  }
+
+  return {
+    bucket,
+    region,
+    prefix: normalizeS3Prefix(config.prefix),
+    publicBaseUrl: normalizePublicBaseUrl(config.publicBaseUrl),
+    credentials: isObject(config.credentials) ? config.credentials : undefined,
+    client: config.client,
+    maxUploadAttempts: Number.isInteger(config.maxUploadAttempts)
+      ? Math.max(1, config.maxUploadAttempts)
+      : 3
+  };
+}
+
+function joinS3Key(prefix, filename) {
+  return [prefix, filename].filter(Boolean).join('/');
+}
+
+function encodeS3Key(key) {
+  return String(key || '')
+    .split('/')
+    .map((part) => encodeURIComponent(part))
+    .join('/');
+}
+
+function buildS3Url(s3Config, key, filename) {
+  if (s3Config.publicBaseUrl) {
+    return `${s3Config.publicBaseUrl}/${encodeURIComponent(filename)}`;
+  }
+
+  return `https://${s3Config.bucket}.s3.${s3Config.region}.amazonaws.com/${encodeS3Key(key)}`;
+}
+
+function getErrorMessage(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+
 export class PageExtractor {
   constructor(config = {}) {
     this.config = {
@@ -27,9 +135,17 @@ export class PageExtractor {
       blockMaxDepth: Number.isInteger(config.blockMaxDepth) ? Math.max(1, config.blockMaxDepth) : 15,
       textPreviewMaxChars: Number.isInteger(config.textPreviewMaxChars)
         ? Math.max(120, config.textPreviewMaxChars)
-        : 1200
+        : 1200,
+      waitForImagesLoaded: Boolean(config.waitForImagesLoaded),
+      fullPageScreenshot: Boolean(config.fullPageScreenshot),
+      blockScreenshots: Boolean(config.blockScreenshots),
+      snapshotDir: typeof config.snapshotDir === 'string' && config.snapshotDir.trim()
+        ? path.resolve(process.cwd(), config.snapshotDir)
+        : path.resolve(process.cwd(), 'snapshots'),
+      s3: normalizeS3Config(config.s3)
     };
     this.playwrightModule = null;
+    this.s3Client = null;
   }
 
   async getPlaywright() {
@@ -41,17 +157,97 @@ export class PageExtractor {
     return this.playwrightModule;
   }
 
+  getS3Client() {
+    if (!this.config.s3) {
+      return null;
+    }
+    if (this.config.s3.client) {
+      return this.config.s3.client;
+    }
+    if (this.s3Client) {
+      return this.s3Client;
+    }
+
+    this.s3Client = new S3Client({
+      region: this.config.s3.region,
+      credentials: this.config.s3.credentials,
+      maxAttempts: 1
+    });
+    return this.s3Client;
+  }
+
+  async uploadScreenshotToS3(filename, body) {
+    const s3Config = this.config.s3;
+    if (!s3Config) {
+      throw new Error('S3 is not configured');
+    }
+
+    const key = joinS3Key(s3Config.prefix, filename);
+    const client = this.getS3Client();
+    const commandInput = {
+      Bucket: s3Config.bucket,
+      Key: key,
+      Body: body,
+      ContentType: 'image/png'
+    };
+
+    let lastError = null;
+    for (let attempt = 1; attempt <= s3Config.maxUploadAttempts; attempt += 1) {
+      try {
+        const command = new PutObjectCommand(commandInput);
+        await client.send(command);
+        return buildS3Url(s3Config, key, filename);
+      } catch (error) {
+        lastError = error;
+        if (attempt < s3Config.maxUploadAttempts) {
+          console.warn(
+            `[page-analyzer] Failed to upload ${key} to S3, retrying ` +
+            `(${attempt}/${s3Config.maxUploadAttempts}): ${getErrorMessage(error)}`
+          );
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
   async revealHiddenContent(page) {
     return page.evaluate(() => {
       const CONTENT_THRESHOLD = 20;
+      const NON_CONTENT_TAGS = new Set([
+        'SCRIPT',
+        'STYLE',
+        'NOSCRIPT',
+        'TEMPLATE',
+        'META',
+        'LINK',
+        'IFRAME',
+        'OBJECT',
+        'EMBED'
+      ]);
       let opacityCount = 0;
       let displayCount = 0;
 
+      const isLikelyScriptText = (value) => {
+        const text = String(value || '').trim();
+        if (!text) {
+          return false;
+        }
+        return /(_satellite|google_tag_manager|dataLayer|window\.|document\.|function\s*\(|=>|createElement\(|appendChild\(|\.push\(|var\s+\w+\s*=|const\s+\w+\s*=|let\s+\w+\s*=)/.test(text);
+      };
+
       for (const el of document.querySelectorAll('*')) {
+        if (NON_CONTENT_TAGS.has(el.tagName)) {
+          continue;
+        }
+
         const style = getComputedStyle(el);
         if (parseFloat(style.opacity) === 0 && el.getBoundingClientRect().height > 0) {
           const text = (el.innerText || '').trim();
-          if (text.length >= CONTENT_THRESHOLD || el.querySelectorAll('img, video, picture').length > 0) {
+          if (
+            !isLikelyScriptText(text) &&
+            (text.length >= CONTENT_THRESHOLD || el.querySelectorAll('img, video, picture').length > 0)
+          ) {
             el.style.setProperty('opacity', '1', 'important');
             opacityCount += 1;
           }
@@ -68,7 +264,7 @@ export class PageExtractor {
           el.style.setProperty('display', 'block', 'important');
           const text = (el.innerText || '').trim();
 
-          if (text.length >= CONTENT_THRESHOLD) {
+          if (text.length >= CONTENT_THRESHOLD && !isLikelyScriptText(text)) {
             displayCount += 1;
           } else if (originalDisplay) {
             el.style.display = originalDisplay;
@@ -82,6 +278,80 @@ export class PageExtractor {
     });
   }
 
+  async waitForImagesLoaded(page) {
+    if (!this.config.waitForImagesLoaded) {
+      return null;
+    }
+
+    const timeoutMs = this.config.timeoutMs;
+    const result = await page.evaluate(async ({ timeoutMs: waitTimeoutMs }) => {
+      const images = Array.from(document.images || []);
+      const total = images.length;
+      if (total === 0) {
+        return {
+          total,
+          loaded: 0,
+          timedOut: false
+        };
+      }
+
+      const isSettled = (img) => img.complete;
+      const countLoaded = () => images.filter(isSettled).length;
+      const pending = images.filter((img) => !isSettled(img));
+
+      if (pending.length === 0) {
+        return {
+          total,
+          loaded: total,
+          timedOut: false
+        };
+      }
+
+      let timeoutId = null;
+      const waitForImage = (img) => new Promise((resolve) => {
+        if (isSettled(img)) {
+          resolve();
+          return;
+        }
+
+        const done = () => {
+          img.removeEventListener('load', done);
+          img.removeEventListener('error', done);
+          resolve();
+        };
+        img.addEventListener('load', done, { once: true });
+        img.addEventListener('error', done, { once: true });
+      });
+
+      const allImagesDone = Promise.all(pending.map(waitForImage))
+        .then(() => ({ timedOut: false }));
+      const timeout = new Promise((resolve) => {
+        timeoutId = window.setTimeout(() => resolve({ timedOut: true }), waitTimeoutMs);
+      });
+
+      const waitResult = await Promise.race([allImagesDone, timeout]);
+      if (timeoutId !== null) {
+        window.clearTimeout(timeoutId);
+      }
+
+      return {
+        total,
+        loaded: countLoaded(),
+        timedOut: Boolean(waitResult?.timedOut)
+      };
+    }, { timeoutMs });
+
+    if (result?.timedOut) {
+      console.warn(
+        `[page-analyzer] Timed out waiting for images: ${result.loaded}/${result.total} completed`
+      );
+    } else {
+      console.log(`[page-analyzer] Images loaded: ${result?.loaded || 0}/${result?.total || 0}`);
+    }
+
+    return result;
+  }
+
   async collectElementGeometries(page) {
     return page.evaluate(() => {
       const INTERACTIVE_SELECTOR = 'a, button, form, input, select, textarea, [onclick], [role="button"]';
@@ -161,16 +431,246 @@ export class PageExtractor {
     });
   }
 
-  /**
-   * Extract page data: html, blocks, elementGeometries, markdown
-   * @param {string} url - URL to extract
-   * @returns {Promise<{html, blocks, elementGeometries, pageSize}>}
-   */
-  async extract(url) {
+  async collectPageSize(page) {
+    return page.evaluate(() => {
+      const html = document.documentElement;
+      const body = document.body;
+      return {
+        width: Math.max(
+          html?.scrollWidth || 0,
+          html?.offsetWidth || 0,
+          html?.clientWidth || 0,
+          body?.scrollWidth || 0,
+          body?.offsetWidth || 0,
+          body?.clientWidth || 0,
+          window.innerWidth || 0
+        ),
+        height: Math.max(
+          html?.scrollHeight || 0,
+          html?.offsetHeight || 0,
+          html?.clientHeight || 0,
+          body?.scrollHeight || 0,
+          body?.offsetHeight || 0,
+          body?.clientHeight || 0,
+          window.innerHeight || 0
+        )
+      };
+    });
+  }
+
+  async hideExternalFixedOverlays(page, selector) {
+    return page.evaluate((targetSelector) => {
+      const existing = Array.isArray(window.__pageAnalyzerHiddenOverlays)
+        ? window.__pageAnalyzerHiddenOverlays
+        : [];
+      for (const item of existing) {
+        if (!item?.element) continue;
+        if (item.visibilityValue) {
+          item.element.style.setProperty(
+            'visibility',
+            item.visibilityValue,
+            item.visibilityPriority || ''
+          );
+        } else {
+          item.element.style.removeProperty('visibility');
+        }
+      }
+
+      const target = document.querySelector(targetSelector);
+      if (!(target instanceof Element)) {
+        window.__pageAnalyzerHiddenOverlays = [];
+        return 0;
+      }
+
+      const hidden = [];
+      for (const element of document.querySelectorAll('body *')) {
+        if (!(element instanceof HTMLElement)) continue;
+        if (element === target || element.contains(target) || target.contains(element)) continue;
+
+        const style = getComputedStyle(element);
+        if (style.position !== 'fixed' && style.position !== 'sticky') continue;
+
+        const rect = element.getBoundingClientRect();
+        if (rect.width <= 0 || rect.height <= 0) continue;
+
+        hidden.push({
+          element,
+          visibilityValue: element.style.getPropertyValue('visibility'),
+          visibilityPriority: element.style.getPropertyPriority('visibility')
+        });
+        element.style.setProperty('visibility', 'hidden', 'important');
+      }
+
+      window.__pageAnalyzerHiddenOverlays = hidden;
+      return hidden.length;
+    }, selector);
+  }
+
+  async restoreExternalFixedOverlays(page) {
+    await page.evaluate(() => {
+      const hidden = Array.isArray(window.__pageAnalyzerHiddenOverlays)
+        ? window.__pageAnalyzerHiddenOverlays
+        : [];
+      for (const item of hidden) {
+        if (!item?.element) continue;
+        if (item.visibilityValue) {
+          item.element.style.setProperty(
+            'visibility',
+            item.visibilityValue,
+            item.visibilityPriority || ''
+          );
+        } else {
+          item.element.style.removeProperty('visibility');
+        }
+      }
+      window.__pageAnalyzerHiddenOverlays = [];
+    });
+  }
+
+  async captureBlockScreenshotData(page, block, screenshotOptions = {}) {
+    if (block?.hidden) {
+      return null;
+    }
+
+    const selector = getBlockSelector(block);
+    if (!selector) {
+      return null;
+    }
+
+    try {
+      const locator = page.locator(selector).first();
+      if (await locator.count() > 0) {
+        await this.hideExternalFixedOverlays(page, selector);
+        try {
+          return await locator.screenshot(screenshotOptions);
+        } finally {
+          await this.restoreExternalFixedOverlays(page);
+        }
+      }
+    } catch {
+      // Selector-only mode: skip blocks that cannot be captured through CSS.
+    }
+
+    return null;
+  }
+
+  async captureBlockScreenshot(page, block, blockPath) {
+    const body = await this.captureBlockScreenshotData(page, block, { path: blockPath });
+    return Boolean(body) || body === undefined;
+  }
+
+  async captureScreenshots(page, targetUrl, blocks, options = {}) {
+    const fullPageScreenshot = options.fullPageScreenshot ?? this.config.fullPageScreenshot;
+    const blockScreenshots = options.blockScreenshots ?? this.config.blockScreenshots;
+    const useS3 = Boolean(this.config.s3);
+
+    if (!fullPageScreenshot && !blockScreenshots) {
+      return null;
+    }
+
+    if (!useS3) {
+      await fs.mkdir(this.config.snapshotDir, { recursive: true });
+    }
+
+    const prefix = `${createSnapshotSlug(targetUrl)}-${createSnapshotRunId()}`;
+    const screenshots = {};
+
+    if (fullPageScreenshot) {
+      const fullPageFilename = `${prefix}-full-page.png`;
+      try {
+        if (useS3) {
+          const body = await page.screenshot({ fullPage: true });
+          screenshots.fullPage = await this.uploadScreenshotToS3(fullPageFilename, body);
+        } else {
+          const fullPagePath = path.join(this.config.snapshotDir, fullPageFilename);
+          await page.screenshot({
+            path: fullPagePath,
+            fullPage: true
+          });
+          screenshots.fullPage = fullPagePath;
+        }
+      } catch (error) {
+        console.warn(
+          `[page-analyzer] Failed to capture/upload full-page screenshot: ${getErrorMessage(error)}`
+        );
+      }
+    }
+
+    if (blockScreenshots) {
+      screenshots.blocks = [];
+
+      for (let index = 0; index < blocks.length; index += 1) {
+        const block = blocks[index];
+        const blockIdx = getBlockNumber(block, index);
+
+        const blockLabel = String(blockIdx).padStart(3, '0').replace(/[^0-9a-z-]+/gi, '-');
+        const blockFilename = `${prefix}-block-${blockLabel}.png`;
+        try {
+          if (useS3) {
+            const body = await this.captureBlockScreenshotData(page, block);
+            if (!body) {
+              continue;
+            }
+            const url = await this.uploadScreenshotToS3(blockFilename, body);
+            const screenshotRecord = {
+              blockIdx,
+              path: url
+            };
+            if (typeof block?.blockName === 'string' && block.blockName.trim()) {
+              screenshotRecord.blockName = block.blockName.trim();
+            }
+            if (typeof block?.blockIdxs === 'string' && block.blockIdxs.trim()) {
+              screenshotRecord.blockIdxs = block.blockIdxs.trim();
+            }
+            screenshots.blocks.push(screenshotRecord);
+            continue;
+          }
+
+          const blockPath = path.join(this.config.snapshotDir, blockFilename);
+          const captured = await this.captureBlockScreenshot(page, block, blockPath);
+          if (captured) {
+            const screenshotRecord = {
+              blockIdx,
+              path: blockPath
+            };
+            if (typeof block?.blockName === 'string' && block.blockName.trim()) {
+              screenshotRecord.blockName = block.blockName.trim();
+            }
+            if (typeof block?.blockIdxs === 'string' && block.blockIdxs.trim()) {
+              screenshotRecord.blockIdxs = block.blockIdxs.trim();
+            }
+            screenshots.blocks.push(screenshotRecord);
+          }
+        } catch (error) {
+          console.warn(
+            `[page-analyzer] Failed to capture/upload block ${blockIdx}: ${getErrorMessage(error)}`
+          );
+        }
+      }
+    }
+
+    return screenshots;
+  }
+
+  async preparePage(page, targetUrl) {
+    await page.goto(targetUrl, {
+      waitUntil: 'domcontentloaded',
+      timeout: this.config.timeoutMs
+    });
+    await scrollToBottom(page);
+    await waitForStableHeight(page, { maxWait: this.config.timeoutMs });
+    await this.revealHiddenContent(page);
+    await this.waitForImagesLoaded(page);
+  }
+
+  async withPreparedPage(url, callback) {
     const targetUrl = String(url || '').trim();
     if (!targetUrl) {
       throw new Error('PageExtractor requires a non-empty URL');
     }
+    if (typeof callback !== 'function') {
+      throw new Error('PageExtractor.withPreparedPage requires a callback');
+    }
 
     const viewport = {
       width: this.config.viewportWidth,
@@ -181,35 +681,61 @@ export class PageExtractor {
     const browser = await playwright.chromium.launch({ headless: true });
     try {
       const page = await browser.newPage({ viewport });
-      await page.goto(targetUrl, {
-        waitUntil: 'domcontentloaded',
-        timeout: this.config.timeoutMs
-      });
-      await scrollToBottom(page);
-      await waitForStableHeight(page, { maxWait: this.config.timeoutMs });
-      await this.revealHiddenContent(page);
-
-      const html = await page.content();
-      const pageSize = await page.evaluate(() => ({
-        width: document.documentElement.scrollWidth || 0,
-        height: document.documentElement.scrollHeight || 0
-      }));
-
-      const minWidth = Math.round(viewport.width * this.config.minBlockWidthRatio);
-      const blocksResult = await page.evaluate(extractBlocksInBrowser, {
-        minHeight: this.config.minBlockHeight,
-        minWidth,
-        maxHeight: Math.round(viewport.height * this.config.blockMaxHeightRatio),
-        maxDepth: this.config.blockMaxDepth,
-        textPreviewMaxChars: this.config.textPreviewMaxChars,
-        debug: false
-      });
-      const blocks = Array.isArray(blocksResult?.blocks) ? blocksResult.blocks : [];
-      const elementGeometries = await this.collectElementGeometries(page);
-
-      return { html, blocks, elementGeometries, pageSize };
+      await this.preparePage(page, targetUrl);
+      return await callback(page, targetUrl);
     } finally {
       await browser.close();
     }
   }
+
+  async captureUrlScreenshots(url, blocks, options = {}) {
+    return this.withPreparedPage(url, async (page, targetUrl) => {
+      return await this.captureScreenshots(page, targetUrl, blocks, options);
+    });
+  }
+
+  /**
+   * Extract page data from an already prepared Playwright page:
+   * html, blocks, elementGeometries, screenshots.
+   * When config.s3 is provided, screenshots are uploaded to S3 and returned as URLs.
+   * @param {import('playwright').Page} page - Prepared Playwright page
+   * @param {string} targetUrl - URL loaded in the page
+   * @returns {Promise<{html, blocks, elementGeometries, screenshots, pageSize}>}
+   */
+  async extractPreparedPage(page, targetUrl) {
+    const viewport = {
+      width: this.config.viewportWidth,
+      height: this.config.viewportHeight
+    };
+
+    const html = await page.content();
+
+    const minWidth = Math.round(viewport.width * this.config.minBlockWidthRatio);
+    const blocksResult = await page.evaluate(extractBlocksInBrowser, {
+      minHeight: this.config.minBlockHeight,
+      minWidth,
+      maxHeight: Math.round(viewport.height * this.config.blockMaxHeightRatio),
+      maxDepth: this.config.blockMaxDepth,
+      textPreviewMaxChars: this.config.textPreviewMaxChars,
+      debug: false
+    });
+    const blocks = Array.isArray(blocksResult?.blocks) ? blocksResult.blocks : [];
+    const elementGeometries = await this.collectElementGeometries(page);
+    const finalPageSize = await this.collectPageSize(page);
+    const screenshots = await this.captureScreenshots(page, targetUrl, blocks);
+
+    return { html, blocks, elementGeometries, screenshots, pageSize: finalPageSize };
+  }
+
+  /**
+   * Extract page data: html, blocks, elementGeometries, screenshots.
+   * When config.s3 is provided, screenshots are uploaded to S3 and returned as URLs.
+   * @param {string} url - URL to extract
+   * @returns {Promise<{html, blocks, elementGeometries, screenshots, pageSize}>}
+   */
+  async extract(url) {
+    return this.withPreparedPage(url, async (page, targetUrl) => {
+      return await this.extractPreparedPage(page, targetUrl);
+    });
+  }
 }