page-analyzer 1.0.0 → 1.1.1

package/page-extractor.js

@@ -3,6 +3,9 @@
  * Launches headless Chromium, navigates to URL, scrolls, extracts blocks + element geometries + HTML.
  */
 
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
 // In-browser block extraction function (serialized into page.evaluate)
 // Imported from the project's extract-blocks script
 import {
@@ -11,6 +14,41 @@ import {
   waitForStableHeight
 } from './vendor/extract-blocks.js';
 
+function createSnapshotSlug(url) {
+  let source = String(url || '').trim();
+  try {
+    const parsed = new URL(source);
+    source = `${parsed.hostname}${parsed.pathname}`;
+  } catch {
+    // Keep the raw value for non-URL inputs.
+  }
+
+  const slug = source
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 80);
+
+  return slug || 'page';
+}
+
+function createSnapshotRunId() {
+  return new Date()
+    .toISOString()
+    .replace(/\.\d{3}z$/i, '')
+    .replace(/[^0-9a-z]+/gi, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+function getBlockNumber(block, fallbackIndex) {
+  return Number.isInteger(block?.blockIdx) ? block.blockIdx : fallbackIndex;
+}
+
+function getBlockSelector(block) {
+  const selector = typeof block?.blockCssPath === 'string' ? block.blockCssPath.trim() : '';
+  return selector || '';
+}
+
 export class PageExtractor {
   constructor(config = {}) {
     this.config = {
@@ -27,7 +65,13 @@ export class PageExtractor {
       blockMaxDepth: Number.isInteger(config.blockMaxDepth) ? Math.max(1, config.blockMaxDepth) : 15,
       textPreviewMaxChars: Number.isInteger(config.textPreviewMaxChars)
         ? Math.max(120, config.textPreviewMaxChars)
-        : 1200
+        : 1200,
+      waitForImagesLoaded: Boolean(config.waitForImagesLoaded),
+      fullPageScreenshot: Boolean(config.fullPageScreenshot),
+      blockScreenshots: Boolean(config.blockScreenshots),
+      snapshotDir: typeof config.snapshotDir === 'string' && config.snapshotDir.trim()
+        ? path.resolve(process.cwd(), config.snapshotDir)
+        : path.resolve(process.cwd(), 'snapshots')
     };
     this.playwrightModule = null;
   }
@@ -44,14 +88,40 @@ export class PageExtractor {
   async revealHiddenContent(page) {
     return page.evaluate(() => {
       const CONTENT_THRESHOLD = 20;
+      const NON_CONTENT_TAGS = new Set([
+        'SCRIPT',
+        'STYLE',
+        'NOSCRIPT',
+        'TEMPLATE',
+        'META',
+        'LINK',
+        'IFRAME',
+        'OBJECT',
+        'EMBED'
+      ]);
       let opacityCount = 0;
       let displayCount = 0;
 
+      const isLikelyScriptText = (value) => {
+        const text = String(value || '').trim();
+        if (!text) {
+          return false;
+        }
+        return /(_satellite|google_tag_manager|dataLayer|window\.|document\.|function\s*\(|=>|createElement\(|appendChild\(|\.push\(|var\s+\w+\s*=|const\s+\w+\s*=|let\s+\w+\s*=)/.test(text);
+      };
+
       for (const el of document.querySelectorAll('*')) {
+        if (NON_CONTENT_TAGS.has(el.tagName)) {
+          continue;
+        }
+
         const style = getComputedStyle(el);
         if (parseFloat(style.opacity) === 0 && el.getBoundingClientRect().height > 0) {
           const text = (el.innerText || '').trim();
-          if (text.length >= CONTENT_THRESHOLD || el.querySelectorAll('img, video, picture').length > 0) {
+          if (
+            !isLikelyScriptText(text) &&
+            (text.length >= CONTENT_THRESHOLD || el.querySelectorAll('img, video, picture').length > 0)
+          ) {
             el.style.setProperty('opacity', '1', 'important');
             opacityCount += 1;
           }
@@ -68,7 +138,7 @@ export class PageExtractor {
           el.style.setProperty('display', 'block', 'important');
           const text = (el.innerText || '').trim();
 
-          if (text.length >= CONTENT_THRESHOLD) {
+          if (text.length >= CONTENT_THRESHOLD && !isLikelyScriptText(text)) {
             displayCount += 1;
           } else if (originalDisplay) {
             el.style.display = originalDisplay;
@@ -82,6 +152,80 @@ export class PageExtractor {
     });
   }
 
+  async waitForImagesLoaded(page) {
+    if (!this.config.waitForImagesLoaded) {
+      return null;
+    }
+
+    const timeoutMs = this.config.timeoutMs;
+    const result = await page.evaluate(async ({ timeoutMs: waitTimeoutMs }) => {
+      const images = Array.from(document.images || []);
+      const total = images.length;
+      if (total === 0) {
+        return {
+          total,
+          loaded: 0,
+          timedOut: false
+        };
+      }
+
+      const isSettled = (img) => img.complete;
+      const countLoaded = () => images.filter(isSettled).length;
+      const pending = images.filter((img) => !isSettled(img));
+
+      if (pending.length === 0) {
+        return {
+          total,
+          loaded: total,
+          timedOut: false
+        };
+      }
+
+      let timeoutId = null;
+      const waitForImage = (img) => new Promise((resolve) => {
+        if (isSettled(img)) {
+          resolve();
+          return;
+        }
+
+        const done = () => {
+          img.removeEventListener('load', done);
+          img.removeEventListener('error', done);
+          resolve();
+        };
+        img.addEventListener('load', done, { once: true });
+        img.addEventListener('error', done, { once: true });
+      });
+
+      const allImagesDone = Promise.all(pending.map(waitForImage))
+        .then(() => ({ timedOut: false }));
+      const timeout = new Promise((resolve) => {
+        timeoutId = window.setTimeout(() => resolve({ timedOut: true }), waitTimeoutMs);
+      });
+
+      const waitResult = await Promise.race([allImagesDone, timeout]);
+      if (timeoutId !== null) {
+        window.clearTimeout(timeoutId);
+      }
+
+      return {
+        total,
+        loaded: countLoaded(),
+        timedOut: Boolean(waitResult?.timedOut)
+      };
+    }, { timeoutMs });
+
+    if (result?.timedOut) {
+      console.warn(
+        `[page-analyzer] Timed out waiting for images: ${result.loaded}/${result.total} completed`
+      );
+    } else {
+      console.log(`[page-analyzer] Images loaded: ${result?.loaded || 0}/${result?.total || 0}`);
+    }
+
+    return result;
+  }
+
   async collectElementGeometries(page) {
     return page.evaluate(() => {
       const INTERACTIVE_SELECTOR = 'a, button, form, input, select, textarea, [onclick], [role="button"]';
@@ -161,10 +305,221 @@ export class PageExtractor {
     });
   }
 
+  async collectPageSize(page) {
+    return page.evaluate(() => {
+      const html = document.documentElement;
+      const body = document.body;
+      return {
+        width: Math.max(
+          html?.scrollWidth || 0,
+          html?.offsetWidth || 0,
+          html?.clientWidth || 0,
+          body?.scrollWidth || 0,
+          body?.offsetWidth || 0,
+          body?.clientWidth || 0,
+          window.innerWidth || 0
+        ),
+        height: Math.max(
+          html?.scrollHeight || 0,
+          html?.offsetHeight || 0,
+          html?.clientHeight || 0,
+          body?.scrollHeight || 0,
+          body?.offsetHeight || 0,
+          body?.clientHeight || 0,
+          window.innerHeight || 0
+        )
+      };
+    });
+  }
+
+  async hideExternalFixedOverlays(page, selector) {
+    return page.evaluate((targetSelector) => {
+      const existing = Array.isArray(window.__pageAnalyzerHiddenOverlays)
+        ? window.__pageAnalyzerHiddenOverlays
+        : [];
+      for (const item of existing) {
+        if (!item?.element) continue;
+        if (item.visibilityValue) {
+          item.element.style.setProperty(
+            'visibility',
+            item.visibilityValue,
+            item.visibilityPriority || ''
+          );
+        } else {
+          item.element.style.removeProperty('visibility');
+        }
+      }
+
+      const target = document.querySelector(targetSelector);
+      if (!(target instanceof Element)) {
+        window.__pageAnalyzerHiddenOverlays = [];
+        return 0;
+      }
+
+      const hidden = [];
+      for (const element of document.querySelectorAll('body *')) {
+        if (!(element instanceof HTMLElement)) continue;
+        if (element === target || element.contains(target) || target.contains(element)) continue;
+
+        const style = getComputedStyle(element);
+        if (style.position !== 'fixed' && style.position !== 'sticky') continue;
+
+        const rect = element.getBoundingClientRect();
+        if (rect.width <= 0 || rect.height <= 0) continue;
+
+        hidden.push({
+          element,
+          visibilityValue: element.style.getPropertyValue('visibility'),
+          visibilityPriority: element.style.getPropertyPriority('visibility')
+        });
+        element.style.setProperty('visibility', 'hidden', 'important');
+      }
+
+      window.__pageAnalyzerHiddenOverlays = hidden;
+      return hidden.length;
+    }, selector);
+  }
+
+  async restoreExternalFixedOverlays(page) {
+    await page.evaluate(() => {
+      const hidden = Array.isArray(window.__pageAnalyzerHiddenOverlays)
+        ? window.__pageAnalyzerHiddenOverlays
+        : [];
+      for (const item of hidden) {
+        if (!item?.element) continue;
+        if (item.visibilityValue) {
+          item.element.style.setProperty(
+            'visibility',
+            item.visibilityValue,
+            item.visibilityPriority || ''
+          );
+        } else {
+          item.element.style.removeProperty('visibility');
+        }
+      }
+      window.__pageAnalyzerHiddenOverlays = [];
+    });
+  }
+
+  async captureBlockScreenshot(page, block, blockPath) {
+    if (block?.hidden) {
+      return false;
+    }
+
+    const selector = getBlockSelector(block);
+    if (!selector) {
+      return false;
+    }
+
+    try {
+      const locator = page.locator(selector).first();
+      if (await locator.count() > 0) {
+        await this.hideExternalFixedOverlays(page, selector);
+        try {
+          await locator.screenshot({ path: blockPath });
+          return true;
+        } finally {
+          await this.restoreExternalFixedOverlays(page);
+        }
+      }
+    } catch {
+      // Selector-only mode: skip blocks that cannot be captured through CSS.
+    }
+
+    return false;
+  }
+
+  async captureScreenshots(page, targetUrl, blocks, options = {}) {
+    const fullPageScreenshot = options.fullPageScreenshot ?? this.config.fullPageScreenshot;
+    const blockScreenshots = options.blockScreenshots ?? this.config.blockScreenshots;
+
+    if (!fullPageScreenshot && !blockScreenshots) {
+      return null;
+    }
+
+    await fs.mkdir(this.config.snapshotDir, { recursive: true });
+    const prefix = `${createSnapshotSlug(targetUrl)}-${createSnapshotRunId()}`;
+    const screenshots = {};
+
+    if (fullPageScreenshot) {
+      const fullPagePath = path.join(this.config.snapshotDir, `${prefix}-full-page.png`);
+      await page.screenshot({
+        path: fullPagePath,
+        fullPage: true
+      });
+      screenshots.fullPage = fullPagePath;
+    }
+
+    if (blockScreenshots) {
+      screenshots.blocks = [];
+
+      for (let index = 0; index < blocks.length; index += 1) {
+        const block = blocks[index];
+        const blockIdx = getBlockNumber(block, index);
+
+        const blockLabel = String(blockIdx).padStart(3, '0').replace(/[^0-9a-z-]+/gi, '-');
+        const blockPath = path.join(this.config.snapshotDir, `${prefix}-block-${blockLabel}.png`);
+        try {
+          const captured = await this.captureBlockScreenshot(page, block, blockPath);
+          if (captured) {
+            const screenshotRecord = {
+              blockIdx,
+              path: blockPath
+            };
+            if (typeof block?.blockName === 'string' && block.blockName.trim()) {
+              screenshotRecord.blockName = block.blockName.trim();
+            }
+            if (typeof block?.blockIdxs === 'string' && block.blockIdxs.trim()) {
+              screenshotRecord.blockIdxs = block.blockIdxs.trim();
+            }
+            screenshots.blocks.push(screenshotRecord);
+          }
+        } catch (error) {
+          console.warn(`[page-analyzer] Failed to capture block ${blockIdx}: ${error.message}`);
+        }
+      }
+    }
+
+    return screenshots;
+  }
+
+  async preparePage(page, targetUrl) {
+    await page.goto(targetUrl, {
+      waitUntil: 'domcontentloaded',
+      timeout: this.config.timeoutMs
+    });
+    await scrollToBottom(page);
+    await waitForStableHeight(page, { maxWait: this.config.timeoutMs });
+    await this.revealHiddenContent(page);
+    await this.waitForImagesLoaded(page);
+  }
+
+  async captureUrlScreenshots(url, blocks, options = {}) {
+    const targetUrl = String(url || '').trim();
+    if (!targetUrl) {
+      throw new Error('PageExtractor requires a non-empty URL');
+    }
+
+    const viewport = {
+      width: this.config.viewportWidth,
+      height: this.config.viewportHeight
+    };
+
+    const playwright = await this.getPlaywright();
+    const browser = await playwright.chromium.launch({ headless: true });
+    try {
+      const page = await browser.newPage({ viewport });
+      await this.preparePage(page, targetUrl);
+      return await this.captureScreenshots(page, targetUrl, blocks, options);
+    } finally {
+      await browser.close();
+    }
+  }
+
   /**
-   * Extract page data: html, blocks, elementGeometries, markdown
+   * Extract page data: html, blocks, elementGeometries, screenshots
    * @param {string} url - URL to extract
-   * @returns {Promise<{html, blocks, elementGeometries, pageSize}>}
+   * @returns {Promise<{html, blocks, elementGeometries, screenshots, pageSize}>}
    */
   async extract(url) {
     const targetUrl = String(url || '').trim();
@@ -181,19 +536,9 @@ export class PageExtractor {
     const browser = await playwright.chromium.launch({ headless: true });
     try {
       const page = await browser.newPage({ viewport });
-      await page.goto(targetUrl, {
-        waitUntil: 'domcontentloaded',
-        timeout: this.config.timeoutMs
-      });
-      await scrollToBottom(page);
-      await waitForStableHeight(page, { maxWait: this.config.timeoutMs });
-      await this.revealHiddenContent(page);
+      await this.preparePage(page, targetUrl);
 
       const html = await page.content();
-      const pageSize = await page.evaluate(() => ({
-        width: document.documentElement.scrollWidth || 0,
-        height: document.documentElement.scrollHeight || 0
-      }));
 
       const minWidth = Math.round(viewport.width * this.config.minBlockWidthRatio);
       const blocksResult = await page.evaluate(extractBlocksInBrowser, {
@@ -206,8 +551,10 @@ export class PageExtractor {
       });
       const blocks = Array.isArray(blocksResult?.blocks) ? blocksResult.blocks : [];
       const elementGeometries = await this.collectElementGeometries(page);
+      const finalPageSize = await this.collectPageSize(page);
+      const screenshots = await this.captureScreenshots(page, targetUrl, blocks);
 
-      return { html, blocks, elementGeometries, pageSize };
+      return { html, blocks, elementGeometries, screenshots, pageSize: finalPageSize };
     } finally {
       await browser.close();
     }