page-analyzer 1.0.0

package/page-extractor.js

@@ -0,0 +1,215 @@
+/**
+ * Playwright-based page extractor.
+ * Launches headless Chromium, navigates to URL, scrolls, extracts blocks + element geometries + HTML.
+ */
+
+// In-browser block extraction function (serialized into page.evaluate)
+// Imported from the project's extract-blocks script
+import {
+  extractBlocksInBrowser,
+  scrollToBottom,
+  waitForStableHeight
+} from './vendor/extract-blocks.js';
+
+export class PageExtractor {
+  constructor(config = {}) {
+    this.config = {
+      timeoutMs: Number.isInteger(config.timeoutMs) ? Math.max(1000, config.timeoutMs) : 30000,
+      viewportWidth: Number.isInteger(config.viewportWidth) ? Math.max(320, config.viewportWidth) : 1440,
+      viewportHeight: Number.isInteger(config.viewportHeight) ? Math.max(400, config.viewportHeight) : 900,
+      minBlockHeight: Number.isInteger(config.minBlockHeight) ? Math.max(1, config.minBlockHeight) : 40,
+      minBlockWidthRatio: Number.isFinite(config.minBlockWidthRatio)
+        ? Math.min(1, Math.max(0.05, config.minBlockWidthRatio))
+        : 0.25,
+      blockMaxHeightRatio: Number.isFinite(config.blockMaxHeightRatio)
+        ? Math.max(0.5, config.blockMaxHeightRatio)
+        : 1.5,
+      blockMaxDepth: Number.isInteger(config.blockMaxDepth) ? Math.max(1, config.blockMaxDepth) : 15,
+      textPreviewMaxChars: Number.isInteger(config.textPreviewMaxChars)
+        ? Math.max(120, config.textPreviewMaxChars)
+        : 1200
+    };
+    this.playwrightModule = null;
+  }
+
+  async getPlaywright() {
+    if (this.playwrightModule) {
+      return this.playwrightModule;
+    }
+    const mod = await import('playwright');
+    this.playwrightModule = mod.default || mod;
+    return this.playwrightModule;
+  }
+
+  async revealHiddenContent(page) {
+    return page.evaluate(() => {
+      const CONTENT_THRESHOLD = 20;
+      let opacityCount = 0;
+      let displayCount = 0;
+
+      for (const el of document.querySelectorAll('*')) {
+        const style = getComputedStyle(el);
+        if (parseFloat(style.opacity) === 0 && el.getBoundingClientRect().height > 0) {
+          const text = (el.innerText || '').trim();
+          if (text.length >= CONTENT_THRESHOLD || el.querySelectorAll('img, video, picture').length > 0) {
+            el.style.setProperty('opacity', '1', 'important');
+            opacityCount += 1;
+          }
+          continue;
+        }
+
+        if (style.display === 'none') {
+          const parent = el.parentElement;
+          if (parent && getComputedStyle(parent).display === 'none') {
+            continue;
+          }
+
+          const originalDisplay = el.style.display;
+          el.style.setProperty('display', 'block', 'important');
+          const text = (el.innerText || '').trim();
+
+          if (text.length >= CONTENT_THRESHOLD) {
+            displayCount += 1;
+          } else if (originalDisplay) {
+            el.style.display = originalDisplay;
+          } else {
+            el.style.removeProperty('display');
+          }
+        }
+      }
+
+      return { opacityCount, displayCount };
+    });
+  }
+
+  async collectElementGeometries(page) {
+    return page.evaluate(() => {
+      const INTERACTIVE_SELECTOR = 'a, button, form, input, select, textarea, [onclick], [role="button"]';
+      const records = [];
+      const seenNodes = new Set();
+
+      const normalizeTextInPage = (value, maxLength = 240) => String(value || '')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .slice(0, maxLength);
+
+      const normalizeHref = (value) => {
+        const raw = String(value || '').trim();
+        if (!raw) return '';
+        try {
+          const resolved = new URL(raw, location.href);
+          resolved.hash = '';
+          const href = resolved.href;
+          return href.endsWith('/') && href !== `${resolved.origin}/`
+            ? href.slice(0, -1)
+            : href;
+        } catch {
+          return raw;
+        }
+      };
+
+      const buildPath = (el, useNthOfType) => {
+        if (!el || !el.tagName) return '';
+        const parts = [];
+        let current = el;
+        while (current && current.tagName && current.tagName.toLowerCase() !== 'body') {
+          const parent = current.parentElement;
+          if (!parent || parent.tagName.toLowerCase() === 'html') break;
+          const tag = current.tagName.toLowerCase();
+          const siblings = Array.from(parent.children).filter((child) => {
+            if (!(child instanceof Element)) return false;
+            return useNthOfType ? child.tagName.toLowerCase() === tag : true;
+          });
+          const index = siblings.indexOf(current) + 1;
+          const suffix = useNthOfType
+            ? `:nth-of-type(${index})`
+            : `:nth-child(${index})`;
+          parts.unshift(`${tag}${suffix}`);
+          current = parent;
+        }
+        if (parts.length === 0) return '';
+        return `body > ${parts.join(' > ')}`;
+      };
+
+      for (const element of document.querySelectorAll(INTERACTIVE_SELECTOR)) {
+        if (!(element instanceof Element) || seenNodes.has(element)) continue;
+        seenNodes.add(element);
+
+        const rect = element.getBoundingClientRect();
+        if (rect.width <= 0 || rect.height <= 0) continue;
+
+        records.push({
+          tag: element.tagName.toLowerCase(),
+          text: normalizeTextInPage(element.innerText || element.textContent || ''),
+          href: normalizeHref(element.getAttribute('href') || ''),
+          action: normalizeHref(element.getAttribute('action') || ''),
+          formAction: normalizeHref(element.getAttribute('formaction') || ''),
+          name: normalizeTextInPage(element.getAttribute('name') || '', 120),
+          type: normalizeTextInPage(element.getAttribute('type') || '', 40),
+          ariaLabel: normalizeTextInPage(element.getAttribute('aria-label') || '', 120),
+          top: rect.top + window.scrollY,
+          left: rect.left + window.scrollX,
+          width: rect.width,
+          height: rect.height,
+          selectorNthOfType: buildPath(element, true),
+          selectorNthChild: buildPath(element, false),
+          selectorById: element.id ? `[id="${String(element.id).replace(/"/g, '\\"')}"]` : ''
+        });
+      }
+
+      return records;
+    });
+  }
+
+  /**
+   * Extract page data: html, blocks, elementGeometries, markdown
+   * @param {string} url - URL to extract
+   * @returns {Promise<{html, blocks, elementGeometries, pageSize}>}
+   */
+  async extract(url) {
+    const targetUrl = String(url || '').trim();
+    if (!targetUrl) {
+      throw new Error('PageExtractor requires a non-empty URL');
+    }
+
+    const viewport = {
+      width: this.config.viewportWidth,
+      height: this.config.viewportHeight
+    };
+
+    const playwright = await this.getPlaywright();
+    const browser = await playwright.chromium.launch({ headless: true });
+    try {
+      const page = await browser.newPage({ viewport });
+      await page.goto(targetUrl, {
+        waitUntil: 'domcontentloaded',
+        timeout: this.config.timeoutMs
+      });
+      await scrollToBottom(page);
+      await waitForStableHeight(page, { maxWait: this.config.timeoutMs });
+      await this.revealHiddenContent(page);
+
+      const html = await page.content();
+      const pageSize = await page.evaluate(() => ({
+        width: document.documentElement.scrollWidth || 0,
+        height: document.documentElement.scrollHeight || 0
+      }));
+
+      const minWidth = Math.round(viewport.width * this.config.minBlockWidthRatio);
+      const blocksResult = await page.evaluate(extractBlocksInBrowser, {
+        minHeight: this.config.minBlockHeight,
+        minWidth,
+        maxHeight: Math.round(viewport.height * this.config.blockMaxHeightRatio),
+        maxDepth: this.config.blockMaxDepth,
+        textPreviewMaxChars: this.config.textPreviewMaxChars,
+        debug: false
+      });
+      const blocks = Array.isArray(blocksResult?.blocks) ? blocksResult.blocks : [];
+      const elementGeometries = await this.collectElementGeometries(page);
+
+      return { html, blocks, elementGeometries, pageSize };
+    } finally {
+      await browser.close();
+    }
+  }
+}

package/utils/selector-utils.js

@@ -0,0 +1,31 @@
+/**
+ * Escape a value for use inside a CSS attribute selector string.
+ * Escapes backslashes first, then double-quotes.
+ * @param {*} value
+ * @returns {string}
+ */
+export function escapeAttributeValue(value) {
+  return String(value)
+    .replace(/\\/g, '\\\\')
+    .replace(/"/g, '\\"');
+}
+
+/**
+ * Return true for CSS class tokens that are likely stable identifiers
+ * (not transient UI-state classes or hash-like tokens).
+ * @param {string} token
+ * @returns {boolean}
+ */
+export function isLikelyStableClassToken(token) {
+  const normalized = String(token || '').trim();
+  if (!normalized || normalized.length < 2 || normalized.length > 48) {
+    return false;
+  }
+  if (/^(active|selected|open|show|hide|visible|hidden|focus|hover|disabled|enabled)$/i.test(normalized)) {
+    return false;
+  }
+  if (/\d{4,}/.test(normalized)) {
+    return false;
+  }
+  return /^[a-zA-Z0-9_-]+$/.test(normalized);
+}

package/utils/text-utils.js

@@ -0,0 +1,11 @@
+/**
+ * Normalize whitespace-heavy text into a compact single-line string.
+ * Collapses all whitespace sequences to a single space and trims.
+ * @param {*} value
+ * @returns {string}
+ */
+export function normalizeText(value) {
+  return String(value || '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}

package/utils/url-utils.js

@@ -0,0 +1,43 @@
+import { URL } from 'url';
+
+/**
+ * Normalize URL (convert relative to absolute, remove fragments)
+ * @param {string} urlStr - URL string to normalize
+ * @param {string} baseUrl - Base URL for resolving relative links
+ * @param {Object} options - Normalization options
+ * @returns {string} Normalized absolute URL
+ */
+export function normalizeUrl(urlStr, baseUrl, options = {}) {
+  const {
+    skipFragments = true,
+    normalizeTrailingSlash = true
+  } = options;
+
+  if (!urlStr) return '';
+
+  try {
+    urlStr = urlStr.trim();
+
+    if (urlStr.startsWith('javascript:') ||
+        urlStr.startsWith('mailto:') ||
+        urlStr.startsWith('tel:') ||
+        urlStr.startsWith('#')) {
+      return urlStr;
+    }
+
+    const absoluteUrl = new URL(urlStr, baseUrl);
+
+    if (skipFragments) {
+      absoluteUrl.hash = '';
+    }
+
+    let normalized = absoluteUrl.href;
+    if (normalizeTrailingSlash && normalized.endsWith('/') && normalized !== absoluteUrl.origin + '/') {
+      normalized = normalized.slice(0, -1);
+    }
+
+    return normalized;
+  } catch {
+    return urlStr;
+  }
+}