page-analyzer 1.0.0

package/vendor/extract-blocks.js

@@ -0,0 +1,903 @@
+#!/usr/bin/env node
+
+/**
+ * Standalone script: extract visual content blocks from a web page.
+ *
+ * Core logic runs inside Playwright (headless Chromium), identifying blocks
+ * by their rendered layout (height + width) rather than DOM structure.
+ *
+ * Usage:
+ *   node scripts/extract-blocks.js <url-or-html-file> [options]
+ *
+ * Options:
+ *   --min-height <px>       Minimum block height in pixels (default: 80)
+ *   --min-width-ratio <0-1> Minimum block width as ratio of viewport (default: 0.25)
+ *   --max-height <px>       Blocks taller than this get subdivided (default: viewport * 1.5)
+ *   --viewport <WxH>        Viewport size (default: 1440x900)
+ *   --json                  Output JSON instead of CSV
+ *   --out <file>            Write output to file instead of stdout
+ *   --save-html <file>      Save rendered HTML to file
+ *
+ * Examples:
+ *   node scripts/extract-blocks.js https://example.com
+ *   node scripts/extract-blocks.js https://example.com --viewport 1920x1080 --out blocks.csv
+ *   node scripts/extract-blocks.js page.html --min-height 100 --json
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+
+function parseArgs() {
+  const args = process.argv.slice(2);
+
+  if (args.length === 0) {
+    console.error('Usage: node scripts/extract-blocks.js <url-or-html-file> [options]');
+    console.error('');
+    console.error('Options:');
+    console.error('  --min-height <px>       Min block height (default: 80)');
+    console.error('  --min-width-ratio <0-1> Min width as viewport ratio (default: 0.25)');
+    console.error('  --max-height <px>       Subdivide blocks taller than this (default: viewport*1.5)');
+    console.error('  --viewport <WxH>        Viewport size (default: 1440x900)');
+    console.error('  --json                  Output JSON instead of CSV');
+    console.error('  --out <file>            Write to file instead of stdout');
+    console.error('  --save-html <file>      Save rendered HTML');
+    process.exit(1);
+  }
+
+  const opts = {
+    input: args[0],
+    minHeight: 40,
+    minWidthRatio: 0.25,
+    maxHeight: null,        // computed from viewport if not set
+    viewportWidth: 1440,
+    viewportHeight: 900,
+    json: false,
+    out: null,
+    saveHtml: null,
+    debug: false
+  };
+
+  for (let i = 1; i < args.length; i++) {
+    switch (args[i]) {
+      case '--min-height':
+        opts.minHeight = parseInt(args[++i], 10);
+        break;
+      case '--min-width-ratio':
+        opts.minWidthRatio = parseFloat(args[++i]);
+        break;
+      case '--max-height':
+        opts.maxHeight = parseInt(args[++i], 10);
+        break;
+      case '--viewport': {
+        const [w, h] = args[++i].split('x').map(Number);
+        opts.viewportWidth = w || 1440;
+        opts.viewportHeight = h || 900;
+        break;
+      }
+      case '--json':
+        opts.json = true;
+        break;
+      case '--out':
+        opts.out = args[++i];
+        break;
+      case '--save-html':
+        opts.saveHtml = args[++i];
+        break;
+      case '--debug':
+        opts.debug = true;
+        break;
+      default:
+        console.error(`Unknown option: ${args[i]}`);
+        process.exit(1);
+    }
+  }
+
+  if (opts.maxHeight === null) {
+    opts.maxHeight = Math.round(opts.viewportHeight * 1.5);
+  }
+
+  return opts;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function isUrl(input) {
+  return /^https?:\/\//i.test(input);
+}
+
+// ---------------------------------------------------------------------------
+// CSV formatter
+// ---------------------------------------------------------------------------
+
+function escapeCsv(value) {
+  const str = String(value ?? '');
+  if (str.includes(',') || str.includes('"') || str.includes('\n') || str.includes('\r')) {
+    return `"${str.replace(/"/g, '""')}"`;
+  }
+  return str;
+}
+
+export function blocksToCsv(blocks) {
+  const header = 'blockIdx,branchPath,depth,domOrder,tag,fixed,top,left,width,height,blockCssPath,blockPosition,textPreview,childInteractiveCount';
+  const rows = blocks.map(b => [
+    b.blockIdx,
+    escapeCsv(b.branchPath ?? ''),
+    b.depth ?? 0,
+    b.domOrder ?? 0,
+    b.tag,
+    b.fixed ? 1 : 0,
+    Math.round(b.top),
+    Math.round(b.left),
+    Math.round(b.width),
+    Math.round(b.height),
+    escapeCsv(b.blockCssPath ?? ''),
+    escapeCsv(formatBlockPositionForCsv(b.blockPosition ?? {
+      left: b.left,
+      top: b.top,
+      width: b.width,
+      height: b.height
+    })),
+    escapeCsv(b.textPreview),
+    b.childInteractiveCount
+  ].join(','));
+  return [header, ...rows].join('\n');
+}
+
+function formatBlockPositionForCsv(position = {}) {
+  const left = Number(position.left) || 0;
+  const top = Number(position.top) || 0;
+  const width = Number(position.width) || 0;
+  const height = Number(position.height) || 0;
+  return `left=${Math.round(left)};top=${Math.round(top)};height=${Math.round(height)};width=${Math.round(width)}`;
+}
+
+// ---------------------------------------------------------------------------
+// Core: in-browser block extraction (runs inside page.evaluate)
+// ---------------------------------------------------------------------------
+
+/**
+ * This function is serialised and executed inside Chromium via page.evaluate().
+ * It has NO access to Node APIs — pure browser JS only.
+ */
+export function extractBlocksInBrowser(config) {
+  const { minHeight, minWidth, maxHeight, maxDepth, debug } = config;
+  const debugLog = [];
+  const TEXT_PREVIEW_MAX_CHARS = Number.isInteger(config?.textPreviewMaxChars)
+    ? Math.max(120, config.textPreviewMaxChars)
+    : 1200;
+
+  const SKIP_TAGS = new Set([
+    'script', 'style', 'link', 'meta', 'noscript', 'svg', 'br', 'hr',
+    'img', 'video', 'audio', 'canvas', 'iframe', 'object', 'embed'
+  ]);
+
+  const INTERACTIVE_SELECTOR = 'a, button, input, select, textarea, [onclick], [role="button"], [role="link"]';
+
+  /** Minimum text length for a hidden element to qualify as a content block */
+  const HIDDEN_MIN_TEXT = 20;
+
+  /** Hidden elements with text longer than this get subdivided instead of treated as one block */
+  const HIDDEN_SUBDIVIDE_TEXT = 500;
+
+  const blocks = [];
+  let domOrderCounter = 0;
+
+  /**
+   * Check computed visibility of an element.
+   */
+  function isVisible(el) {
+    const style = getComputedStyle(el);
+    if (style.display === 'none') return false;
+    if (style.visibility === 'hidden') return false;
+    if (parseFloat(style.opacity) === 0) return false;
+    return true;
+  }
+
+  /**
+   * Extract image info from an element (for blocks with no/little text).
+   * Returns a string like "img[alt=Photo of house](hero.jpg)" or empty string.
+   */
+  function extractImageInfo(el) {
+    const imgs = el.querySelectorAll('img');
+    if (imgs.length === 0) return '';
+    const parts = [];
+    for (const img of imgs) {
+      const alt = (img.getAttribute('alt') || '').trim();
+      const src = (img.getAttribute('src') || '').trim();
+      // Shorten src to last path segment + query params stripped
+      let shortSrc = '';
+      try {
+        const url = new URL(src, location.href);
+        shortSrc = url.pathname.split('/').filter(Boolean).pop() || '';
+      } catch {
+        shortSrc = src.split('/').pop()?.split('?')[0] || '';
+      }
+      if (shortSrc.length > 60) shortSrc = shortSrc.slice(0, 57) + '...';
+      const altStr = alt ? `alt="${alt}"` : 'no-alt';
+      parts.push(`img[${altStr}](${shortSrc})`);
+      if (parts.length >= 3) break; // limit to 3 images
+    }
+    return parts.join('; ');
+  }
+
+  function normalizeActionValue(value) {
+    const raw = String(value || '').trim();
+    if (!raw) return '';
+    try {
+      const parsed = new URL(raw, location.href);
+      return `${parsed.pathname || '/'}${parsed.search || ''}`;
+    } catch {
+      return raw;
+    }
+  }
+
+  function normalizeActionLabel(node) {
+    const candidates = [
+      node?.innerText,
+      node?.textContent,
+      node?.getAttribute?.('aria-label'),
+      node?.getAttribute?.('name'),
+      node?.getAttribute?.('value'),
+      node?.getAttribute?.('alt')
+    ];
+    for (const value of candidates) {
+      const text = String(value || '')
+        .replace(/\s+/g, ' ')
+        .replace(/[<>\[\]]/g, '')
+        .trim();
+      if (!text) {
+        continue;
+      }
+      if (text.length > 24) {
+        return `${text.slice(0, 21)}...`;
+      }
+      return text;
+    }
+    return '';
+  }
+
+  function extractActionInfo(el) {
+    const parts = [];
+    const seenActionValues = new Set();
+    const interactiveNodes = [];
+
+    if (typeof el.matches === 'function' && el.matches(INTERACTIVE_SELECTOR)) {
+      interactiveNodes.push(el);
+    }
+    for (const node of el.querySelectorAll(INTERACTIVE_SELECTOR)) {
+      interactiveNodes.push(node);
+    }
+
+    for (const node of interactiveNodes) {
+      const href = node.getAttribute('href') || '';
+      const action = node.getAttribute('action') || '';
+      const formAction = node.getAttribute('formaction') || '';
+      const onClick = node.getAttribute('onclick') || '';
+
+      let actionValue = normalizeActionValue(action || formAction || href);
+      if (!actionValue && onClick) {
+        actionValue = 'inline_onclick';
+      }
+      if (!actionValue) {
+        continue;
+      }
+      if (actionValue.length > 120) {
+        actionValue = `${actionValue.slice(0, 117)}...`;
+      }
+      if (seenActionValues.has(actionValue)) {
+        continue;
+      }
+      seenActionValues.add(actionValue);
+
+      const label = normalizeActionLabel(node);
+      const token = label
+        ? `<action[${label}]=${actionValue}>`
+        : `<action=${actionValue}>`;
+      parts.push(token);
+      if (parts.length >= 10) break;
+    }
+
+    return parts.join(' ');
+  }
+
+  function truncateTextPreview(value) {
+    const text = String(value || '');
+    if (text.length <= TEXT_PREVIEW_MAX_CHARS) {
+      return text;
+    }
+
+    let out = text.slice(0, TEXT_PREVIEW_MAX_CHARS);
+    const lastOpen = out.lastIndexOf('<');
+    const lastClose = out.lastIndexOf('>');
+    if (lastOpen > lastClose) {
+      out = out.slice(0, lastOpen);
+    }
+    return out.replace(/[|;, ]+$/g, '').trim();
+  }
+
+  /**
+   * Build the textPreview for a block, including image info if text is sparse.
+   */
+  function buildTextPreview(el) {
+    const text = (el.innerText || '').replace(/\s+/g, ' ').trim();
+    const actionInfo = extractActionInfo(el);
+    if (text.length >= 10) {
+      return actionInfo
+        ? truncateTextPreview(`[interactive-actions] ${actionInfo} | ${text}`)
+        : truncateTextPreview(text);
+    }
+
+    // Text is empty/sparse — try to provide image and action context
+    const imgInfo = extractImageInfo(el);
+    const segments = [];
+    if (text) segments.push(text);
+    if (imgInfo) segments.push(imgInfo);
+    if (actionInfo) segments.push(`[interactive-actions] ${actionInfo}`);
+    if (segments.length > 0) {
+      return truncateTextPreview(segments.join(' | '));
+    }
+
+    return truncateTextPreview(text);
+  }
+
+  /**
+   * Recursive top-down block identification based on visual dimensions.
+   *
+   * Logic:
+   * - Hidden + substantial content → accept as ONE block (no recursion)
+   * - Hidden + sparse content → skip
+   * - Visible + too small → skip
+   * - Visible + good size (≤ maxHeight) → accept as block
+   * - Visible + too tall (> maxHeight) → subdivide into children
+   */
+  const indent = d => '  '.repeat(d);
+
+  function textSnippet(el, len = 60) {
+    return (el.innerText || '').replace(/\s+/g, ' ').trim().slice(0, len);
+  }
+
+  function normalizePositionNumber(value) {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : 0;
+  }
+
+  function buildBlockPosition({ left, top, width, height } = {}) {
+    return {
+      left: normalizePositionNumber(left),
+      top: normalizePositionNumber(top),
+      height: Math.max(0, normalizePositionNumber(height)),
+      width: Math.max(0, normalizePositionNumber(width))
+    };
+  }
+
+  function buildBlockCssPath(el) {
+    if (!(el instanceof Element) || !el.tagName) {
+      return '';
+    }
+
+    const parts = [];
+    let current = el;
+    while (current && current instanceof Element && current.tagName) {
+      const tag = current.tagName.toLowerCase();
+      if (tag === 'body') {
+        break;
+      }
+
+      const parent = current.parentElement;
+      if (!parent || parent.tagName.toLowerCase() === 'html') {
+        break;
+      }
+
+      const siblings = Array.from(parent.children).filter((child) => {
+        return child instanceof Element && child.tagName.toLowerCase() === tag;
+      });
+      const index = siblings.indexOf(current) + 1;
+      parts.unshift(`${tag}:nth-of-type(${Math.max(1, index)})`);
+      current = parent;
+    }
+
+    return parts.length > 0 ? `body > ${parts.join(' > ')}` : 'body';
+  }
+
+  function createBlockRecord(el, fields = {}) {
+    return {
+      ...fields,
+      blockCssPath: buildBlockCssPath(el),
+      blockPosition: buildBlockPosition(fields)
+    };
+  }
+
+  /**
+   * Walk inside hidden containers — since we can't use visual dimensions,
+   * we subdivide by text content length, recursing into children that are
+   * large enough to split further.
+   */
+  function walkHidden(el, depth, branchPath) {
+    if (depth > maxDepth) return;
+    const tag = el.tagName?.toLowerCase();
+    if (!tag || SKIP_TAGS.has(tag)) return;
+
+    const myDomOrder = ++domOrderCounter;
+    const text = (el.innerText || '').trim();
+    const textLen = text.length;
+
+    // Too little content → skip
+    if (textLen < HIDDEN_MIN_TEXT) return;
+
+    // Large child with multiple children → keep subdividing
+    if (textLen >= HIDDEN_SUBDIVIDE_TEXT && el.children.length > 1) {
+      if (debug) {
+        debugLog.push(`${indent(depth)}HIDDEN-sub ↓ ${el.children.length} children (${textLen} chars)`);
+      }
+      const countBefore = blocks.length;
+      const children = Array.from(el.children);
+      for (let i = 0; i < children.length; i++) {
+        walkHidden(children[i], depth + 1, `${branchPath}.${i}`);
+      }
+      if (blocks.length > countBefore) return; // subdivision succeeded
+    }
+
+    // Leaf-ish hidden block — accept
+    if (debug) {
+      const snippet = textSnippet(el);
+      debugLog.push(`${indent(depth)}HIDDEN-sub ✅ | ${snippet}`);
+    }
+    blocks.push(createBlockRecord(el, {
+      el,
+      tag,
+      top: 0,
+      left: 0,
+      width: 0,
+      height: 0,
+      hidden: true,
+      branchPath,
+      depth,
+      domOrder: myDomOrder,
+      textPreview: buildTextPreview(el),
+      childInteractiveCount: el.querySelectorAll(INTERACTIVE_SELECTOR).length,
+    }));
+  }
+
+  function walk(el, depth, branchPath) {
+    if (depth > maxDepth) return;
+
+    const tag = el.tagName.toLowerCase();
+    if (SKIP_TAGS.has(tag)) return;
+
+    const myDomOrder = ++domOrderCounter;
+
+    // Detect fixed/sticky overlays (floating headers, chat widgets, modals).
+    const position = getComputedStyle(el).position;
+    const isFixed = position === 'fixed' || position === 'sticky';
+
+    // Fixed/sticky → treat as a single block (no subdivision), so each overlay
+    // maps to exactly one block row regardless of its height.
+    if (isFixed) {
+      const rect = el.getBoundingClientRect();
+      const preview = buildTextPreview(el);
+      if (debug) {
+        debugLog.push(`${indent(depth)}FIXED(${position}) ${Math.round(rect.width)}x${Math.round(rect.height)} ✅ | ${preview || '(empty)'}`);
+      }
+      blocks.push(createBlockRecord(el, {
+        el,
+        tag,
+        top: rect.top + window.scrollY,
+        left: rect.left + window.scrollX,
+        width: rect.width,
+        height: rect.height,
+        hidden: false,
+        fixed: true,
+        branchPath,
+        depth,
+        domOrder: myDomOrder,
+        textPreview: preview,
+        childInteractiveCount: el.querySelectorAll(INTERACTIVE_SELECTOR).length,
+      }));
+      return;
+    }
+
+    // Hidden element handling
+    if (!isVisible(el)) {
+      const text = (el.innerText || '').trim();
+      const textLen = text.length;
+      const substantial = textLen >= HIDDEN_MIN_TEXT;
+
+      if (!substantial) {
+        if (debug && depth <= 4) {
+          const snippet = textSnippet(el);
+          if (snippet) debugLog.push(`${indent(depth)}HIDDEN → skip  | ${snippet}`);
+        }
+        return;
+      }
+
+      // Large hidden container → subdivide into children
+      if (textLen >= HIDDEN_SUBDIVIDE_TEXT && el.children.length > 1) {
+        if (debug) {
+          debugLog.push(`${indent(depth)}HIDDEN → subdivide (${textLen} chars, ${el.children.length} children)`);
+        }
+        const countBefore = blocks.length;
+        const children = Array.from(el.children);
+        for (let i = 0; i < children.length; i++) {
+          walkHidden(children[i], depth + 1, `${branchPath}.${i}`);
+        }
+        // If subdivision produced nothing, fall through to treat as one block
+        if (blocks.length > countBefore) return;
+        if (debug) {
+          debugLog.push(`${indent(depth)}HIDDEN → no children qualified, treating as one block`);
+        }
+      }
+
+      // Small hidden element or subdivision failed → treat as one block
+      if (debug && depth <= 4) {
+        const snippet = textSnippet(el);
+        debugLog.push(`${indent(depth)}HIDDEN → block | ${snippet}`);
+      }
+      blocks.push(createBlockRecord(el, {
+        el,
+        tag,
+        top: 0,
+        left: 0,
+        width: 0,
+        height: 0,
+        hidden: true,
+        branchPath,
+        depth,
+        domOrder: myDomOrder,
+        textPreview: buildTextPreview(el),
+        childInteractiveCount: el.querySelectorAll(INTERACTIVE_SELECTOR).length,
+      }));
+      return;
+    }
+
+    const rect = el.getBoundingClientRect();
+    const w = rect.width;
+    const h = rect.height;
+
+    // Too small → skip entirely
+    if (w < minWidth || h < minHeight) {
+      if (debug && depth <= 3) {
+        const snippet = textSnippet(el);
+        debugLog.push(`${indent(depth)}${Math.round(w)}x${Math.round(h)} skip | ${snippet || '(empty)'}`);
+      }
+      return;
+    }
+
+    // Good size → accept as block
+    if (h <= maxHeight) {
+      const preview = buildTextPreview(el);
+      if (debug) {
+        debugLog.push(`${indent(depth)}${Math.round(w)}x${Math.round(h)} ✅   | ${preview || '(empty)'}`);
+      }
+      const absTop = rect.top + window.scrollY;
+      const absLeft = rect.left + window.scrollX;
+
+      blocks.push(createBlockRecord(el, {
+        el,
+        tag,
+        top: absTop,
+        left: absLeft,
+        width: w,
+        height: h,
+        hidden: false,
+        fixed: isFixed,
+        branchPath,
+        depth,
+        domOrder: myDomOrder,
+        textPreview: preview,
+        childInteractiveCount: el.querySelectorAll(INTERACTIVE_SELECTOR).length,
+      }));
+      return; // don't recurse further — this element is the block
+    }
+
+    // Too tall → try to subdivide into children
+    if (debug) {
+      debugLog.push(`${indent(depth)}${Math.round(w)}x${Math.round(h)} ↓ ${el.children.length} children`);
+    }
+    const countBefore = blocks.length;
+    const children = Array.from(el.children);
+    for (let i = 0; i < children.length; i++) {
+      walk(children[i], depth + 1, `${branchPath}.${i}`);
+    }
+
+    // If subdivision produced nothing, accept the large element itself
+    if (blocks.length === countBefore) {
+      const preview = buildTextPreview(el);
+      if (debug) {
+        debugLog.push(`${indent(depth)}${Math.round(w)}x${Math.round(h)} ✅ (no children qualified) | ${preview || '(empty)'}`);
+      }
+      const absTop = rect.top + window.scrollY;
+      const absLeft = rect.left + window.scrollX;
+      blocks.push(createBlockRecord(el, {
+        el,
+        tag,
+        top: absTop,
+        left: absLeft,
+        width: w,
+        height: h,
+        hidden: false,
+        fixed: isFixed,
+        branchPath,
+        depth,
+        domOrder: myDomOrder,
+        textPreview: preview,
+        childInteractiveCount: el.querySelectorAll(INTERACTIVE_SELECTOR).length,
+      }));
+    }
+  }
+
+  // Start from body's direct children (skip body itself — it's always full-page)
+  const bodyChildren = Array.from(document.body.children);
+  for (let i = 0; i < bodyChildren.length; i++) {
+    walk(bodyChildren[i], 0, String(i));
+  }
+
+  // ── Deduplication: remove blocks fully contained by another block ─────────
+  // Keep the inner (more specific) block, remove the outer container.
+  const toRemove = new Set();
+  for (let i = 0; i < blocks.length; i++) {
+    for (let j = 0; j < blocks.length; j++) {
+      if (i === j) continue;
+      if (blocks[i].el.contains(blocks[j].el)) {
+        // i contains j → remove i (the outer one)
+        toRemove.add(i);
+      }
+    }
+  }
+
+  const deduped = blocks
+    .filter((_, idx) => !toRemove.has(idx))
+    // Drop empty blocks: no text, no images, no interactive children
+    .filter(b => b.textPreview || b.childInteractiveCount > 0)
+    .map(({ el, ...rest }, idx) => ({ blockIdx: idx, ...rest }));
+
+  return { blocks: deduped, debugLog };
+}
+
+// ---------------------------------------------------------------------------
+// Scroll to bottom: trigger lazy-load / JS rendering
+// ---------------------------------------------------------------------------
+
+/**
+ * Incrementally scroll from top to bottom of the page,
+ * pausing at each step to let lazy content load.
+ */
+export async function scrollToBottom(page, { step = 800, delay = 300, maxScrolls = 30 } = {}) {
+  console.error('📜 Scrolling to bottom...');
+
+  let lastHeight = 0;
+  let scrollCount = 0;
+
+  while (scrollCount < maxScrolls) {
+    const currentHeight = await page.evaluate(() => document.documentElement.scrollHeight);
+
+    // Already at or past the bottom
+    const scrollY = await page.evaluate(() => window.scrollY + window.innerHeight);
+    if (scrollY >= currentHeight) break;
+
+    // Infinite scroll detection: if height grew since last check, page is extending
+    if (currentHeight > lastHeight && scrollCount > 0) {
+      const growth = currentHeight - lastHeight;
+      // If page grew more than 2x viewport in one scroll step → likely infinite scroll, stop
+      const viewportH = await page.evaluate(() => window.innerHeight);
+      if (growth > viewportH * 2) {
+        console.error(`📜 Infinite scroll detected (grew ${growth}px), stopping`);
+        break;
+      }
+    }
+    lastHeight = currentHeight;
+
+    await page.evaluate(s => window.scrollBy(0, s), step);
+    await page.waitForTimeout(delay);
+    scrollCount++;
+  }
+
+  console.error(`📜 Scrolled ${scrollCount} steps`);
+
+  // Scroll back to top
+  await page.evaluate(() => window.scrollTo(0, 0));
+  await page.waitForTimeout(300);
+}
+
+// ---------------------------------------------------------------------------
+// Wait strategy: poll until page height stabilises
+// ---------------------------------------------------------------------------
+
+/**
+ * Poll document.body.scrollHeight every `interval` ms.
+ * Once it stays the same for `stableCount` consecutive checks, return.
+ * Gives up after `maxWait` ms total.
+ */
+export async function waitForStableHeight(page, { interval = 500, stableCount = 3, maxWait = 15000 } = {}) {
+  const t0 = Date.now();
+  let lastHeight = 0;
+  let same = 0;
+
+  while (Date.now() - t0 < maxWait) {
+    const h = await page.evaluate(() => document.documentElement.scrollHeight);
+    if (h === lastHeight) {
+      same++;
+      if (same >= stableCount) {
+        console.error(`⏱  Page stable at height ${h}px (${Date.now() - t0}ms)`);
+        return;
+      }
+    } else {
+      same = 0;
+      lastHeight = h;
+    }
+    await page.waitForTimeout(interval);
+  }
+
+  console.error(`⏱  Max wait ${maxWait}ms reached, proceeding (height=${lastHeight}px)`);
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+async function main() {
+  const opts = parseArgs();
+  const viewport = { width: opts.viewportWidth, height: opts.viewportHeight };
+
+  // ── Load Playwright ──────────────────────────────────────────────────────
+  let pw;
+  try {
+    const mod = await import('playwright');
+    pw = mod.default || mod;
+  } catch {
+    console.error('❌ playwright is required. Install: npm i playwright');
+    process.exit(1);
+  }
+
+  // ── Launch browser & load page ───────────────────────────────────────────
+  const inputLabel = opts.input;
+  console.error(`🌐 Opening: ${inputLabel} (${viewport.width}x${viewport.height})`);
+
+  const browser = await pw.chromium.launch({ headless: true });
+  let page;
+  try {
+    page = await browser.newPage({ viewport });
+
+    if (isUrl(opts.input)) {
+      await page.goto(opts.input, { waitUntil: 'domcontentloaded', timeout: 30000 });
+    } else {
+      const inputPath = path.resolve(opts.input);
+      if (!fs.existsSync(inputPath)) {
+        console.error(`File not found: ${inputPath}`);
+        process.exit(1);
+      }
+      const html = fs.readFileSync(inputPath, 'utf-8');
+      await page.setContent(html, { waitUntil: 'domcontentloaded', timeout: 30000 });
+    }
+
+    // Scroll to bottom to trigger lazy-load content, then wait for stable height
+    await scrollToBottom(page);
+    await waitForStableHeight(page);
+
+    // Force-reveal hidden content so block extraction can see real dimensions.
+    // Two categories:
+    //   1. opacity:0 — scroll-triggered animations (IntersectionObserver) that never fire in headless
+    //   2. display:none — interactive panels (dropdown menus, accordions) with substantial content
+    const revealStats = await page.evaluate(() => {
+      const CONTENT_THRESHOLD = 20; // min text chars to keep a revealed element
+      let opacityCount = 0;
+      let displayCount = 0;
+
+      for (const el of document.querySelectorAll('*')) {
+        const s = getComputedStyle(el);
+
+        // Case 1: opacity:0 with real layout — animation that didn't trigger
+        // Only reveal if the element (or its subtree) has actual content,
+        // to avoid surfacing empty scroll-anchor / animation-placeholder divs.
+        if (parseFloat(s.opacity) === 0 && el.getBoundingClientRect().height > 0) {
+          const text = (el.innerText || '').trim();
+          if (text.length >= CONTENT_THRESHOLD || el.querySelectorAll('img, video, picture').length > 0) {
+            el.style.setProperty('opacity', '1', 'important');
+            opacityCount++;
+          }
+          continue;
+        }
+
+        // Case 2: display:none — dropdown/accordion/menu panel
+        if (s.display === 'none') {
+          // Only process "root" hidden elements (parent is visible).
+          // Children of a display:none parent inherit none; they'll be handled
+          // once we reveal their parent.
+          const parent = el.parentElement;
+          if (parent && getComputedStyle(parent).display === 'none') continue;
+
+          // Temporarily reveal to measure content
+          const origDisplay = el.style.display;
+          el.style.setProperty('display', 'block', 'important');
+          const text = (el.innerText || '').trim();
+
+          if (text.length >= CONTENT_THRESHOLD) {
+            displayCount++;
+            // Keep visible — has substantial content
+          } else {
+            // Revert — not enough content to qualify
+            if (origDisplay) {
+              el.style.display = origDisplay;
+            } else {
+              el.style.removeProperty('display');
+            }
+          }
+        }
+      }
+      return { opacityCount, displayCount };
+    });
+    if (revealStats.opacityCount > 0 || revealStats.displayCount > 0) {
+      console.error(`👁  Force-revealed: ${revealStats.opacityCount} opacity:0, ${revealStats.displayCount} display:none`);
+    }
+
+    // Save rendered HTML if requested
+    if (opts.saveHtml) {
+      const rendered = await page.content();
+      const savePath = path.resolve(opts.saveHtml);
+      fs.writeFileSync(savePath, rendered, 'utf-8');
+      console.error(`💾 HTML saved to: ${savePath}`);
+    }
+
+    const pageSize = await page.evaluate(() => ({
+      w: document.documentElement.scrollWidth,
+      h: document.documentElement.scrollHeight
+    }));
+    console.error(`📄 Page size: ${pageSize.w}x${pageSize.h}`);
+
+    // ── Extract blocks inside browser ────────────────────────────────────
+    const minWidth = Math.round(viewport.width * opts.minWidthRatio);
+    const result = await page.evaluate(extractBlocksInBrowser, {
+      minHeight: opts.minHeight,
+      minWidth,
+      maxHeight: opts.maxHeight,
+      maxDepth: 15,
+      textPreviewMaxChars: 1200,
+      debug: opts.debug
+    });
+
+    const blocks = result.blocks;
+    if (opts.debug && result.debugLog.length > 0) {
+      console.error('\n🔍 Walk tree:');
+      for (const line of result.debugLog) {
+        console.error(line);
+      }
+      console.error('');
+    }
+
+    console.error(`📦 Blocks found: ${blocks.length}`);
+
+    // ── Output ───────────────────────────────────────────────────────────
+    let output;
+    if (opts.json) {
+      output = JSON.stringify(blocks, null, 2);
+    } else {
+      output = blocksToCsv(blocks);
+    }
+
+    if (opts.out) {
+      const outPath = path.resolve(opts.out);
+      fs.writeFileSync(outPath, output, 'utf-8');
+      console.error(`💾 Written to: ${outPath}`);
+    } else {
+      console.log(output);
+    }
+  } finally {
+    await browser.close();
+  }
+}
+
+const cliArgvPath = process.argv[1] ? path.resolve(process.argv[1]) : '';
+const isCliEntry = cliArgvPath && pathToFileURL(cliArgvPath).href === import.meta.url;
+
+if (isCliEntry) {
+  main().catch(err => {
+    console.error(`❌ ${err.message}`);
+    process.exit(1);
+  });
+}