design-clone 1.2.0 → 2.1.0

package/src/core/multi-page-screenshot.js

@@ -16,7 +16,7 @@ import { getBrowser, getPage, disconnectBrowser } from '../utils/browser.js';
 import { captureViewport, VIEWPORTS, DEFAULT_SCROLL_DELAY } from './screenshot.js';
 import { waitForDomStable, waitForPageReady } from './page-readiness.js';
 import { dismissCookieBanner } from './cookie-handler.js';
-import { extractCleanHtml, JS_FRAMEWORK_PATTERNS, MAX_HTML_SIZE } from './html-extractor.js';
+import { extractAndEnhanceHtml, JS_FRAMEWORK_PATTERNS, MAX_HTML_SIZE } from './html-extractor.js';
 import { extractAllCss, MAX_CSS_SIZE } from './css-extractor.js';
 import { filterCssFile } from './filter-css.js';
 
@@ -69,7 +69,7 @@ async function createOutputStructure(outputDir, viewports) {
 
 /**
  * Capture a single page (all viewports + HTML/CSS extraction)
- * @param {Page} page - Puppeteer page instance
+ * @param {Page} page - Playwright page instance
  * @param {Object} pageInfo - Page info { path, name, url }
  * @param {string} outputDir - Output directory
  * @param {Object} options - Capture options
@@ -91,7 +91,7 @@ async function captureSinglePage(page, pageInfo, outputDir, options) {
   try {
     // Navigate to page
     await page.goto(pageInfo.url, {
-      waitUntil: ['load', 'networkidle0'],
+      waitUntil: 'networkidle',
       timeout: options.timeout
     });
 
@@ -104,10 +104,11 @@ async function captureSinglePage(page, pageInfo, outputDir, options) {
     // Extra stabilization
     await waitForDomStable(page, 300, 3000);
 
-    // Extract HTML
+    // Extract HTML with semantic enhancement
     if (options.extractHtml) {
       try {
-        const htmlResult = await extractCleanHtml(page, JS_FRAMEWORK_PATTERNS);
+        const enhanceSemantic = options.enhanceSemantic !== false;
+        const htmlResult = await extractAndEnhanceHtml(page, { enhanceSemantic });
         const htmlSize = Buffer.byteLength(htmlResult.html, 'utf-8');
 
         if (htmlSize > MAX_HTML_SIZE) {
@@ -118,7 +119,9 @@ async function captureSinglePage(page, pageInfo, outputDir, options) {
           result.html = {
             path: htmlPath,
             size: htmlSize,
-            elementCount: htmlResult.elementCount
+            elementCount: htmlResult.elementCount,
+            semanticEnhanced: enhanceSemantic,
+            semanticStats: htmlResult.semanticStats || null
           };
           if (htmlResult.warnings.length > 0) {
             result.warnings.push(...htmlResult.warnings);

package/src/core/page-readiness.js

@@ -32,7 +32,7 @@ export const CONTENT_SELECTORS = [
 
 /**
  * Wait for fonts to finish loading
- * @param {Page} page - Puppeteer page
+ * @param {Page} page - Playwright page
  * @param {number} timeout - Max wait time in ms
  */
 export async function waitForFontsLoaded(page, timeout = FONT_LOAD_TIMEOUT) {
@@ -51,13 +51,13 @@ export async function waitForFontsLoaded(page, timeout = FONT_LOAD_TIMEOUT) {
 
 /**
  * Wait for styles to stabilize (no new style mutations)
- * @param {Page} page - Puppeteer page
+ * @param {Page} page - Playwright page
  * @param {number} stableMs - How long to wait without changes
  * @param {number} timeout - Max wait time in ms
  */
 export async function waitForStylesStable(page, stableMs = 500, timeout = 5000) {
   try {
-    await page.evaluate(async (stable, max) => {
+    await page.evaluate(async ({ stable, max }) => {
       return new Promise((resolve) => {
         let lastChange = Date.now();
         let checkInterval;
@@ -88,7 +88,7 @@ export async function waitForStylesStable(page, stableMs = 500, timeout = 5000)
           }
         }, 100);
       });
-    }, stableMs, timeout);
+    }, { stable: stableMs, max: timeout });
   } catch {
     // Error in style stability check, continue anyway
   }
@@ -96,7 +96,7 @@ export async function waitForStylesStable(page, stableMs = 500, timeout = 5000)
 
 /**
  * Wait for DOM to stabilize (element count unchanged)
- * @param {Page} page - Puppeteer page
+ * @param {Page} page - Playwright page
  * @param {number} threshold - Stability duration in ms
  * @param {number} timeout - Max wait time in ms
  */
@@ -116,7 +116,7 @@ export async function waitForDomStable(page, threshold = DOM_STABLE_THRESHOLD, t
 
 /**
  * Wait for page to be ready (loading complete, content visible)
- * @param {Page} page - Puppeteer page
+ * @param {Page} page - Playwright page
  * @param {number} timeout - Max wait time
  */
 export async function waitForPageReady(page, timeout = PAGE_READY_TIMEOUT) {
@@ -125,7 +125,7 @@ export async function waitForPageReady(page, timeout = PAGE_READY_TIMEOUT) {
   const minContentElements = Math.max(100, initialCount * 2);
 
   while (Date.now() - startTime < timeout) {
-    const result = await page.evaluate((loadingSels, contentSels, minElements) => {
+    const result = await page.evaluate(({ loadingSels, contentSels, minElements }) => {
       const elementCount = document.querySelectorAll('*').length;
 
       const loadingGone = loadingSels.every(sel => {
@@ -148,7 +148,7 @@ export async function waitForPageReady(page, timeout = PAGE_READY_TIMEOUT) {
         loadingGone,
         contentExists
       };
-    }, LOADING_SELECTORS, CONTENT_SELECTORS, minContentElements);
+    }, { loadingSels: LOADING_SELECTORS, contentSels: CONTENT_SELECTORS, minElements: minContentElements });
 
     if (result.ready) break;
     await new Promise(r => setTimeout(r, 200));

package/src/core/screenshot.js

@@ -21,6 +21,8 @@
  *   --video         Record scroll preview video (default: false)
  *   --video-format  Video format: webm, mp4, gif (default: webm)
  *   --video-duration Video duration in ms (default: 12000)
+ *   --section-mode  Enable section-based capture for AI analysis (default: false)
+ *   --no-semantic   Disable WordPress semantic HTML enhancement (default: false)
  */
 
 import path from 'path';
@@ -34,10 +36,12 @@ import { getBrowser, getPage, closeBrowser, disconnectBrowser, parseArgs, output
 import { waitForDomStable, waitForFontsLoaded, waitForStylesStable, waitForPageReady } from './page-readiness.js';
 import { dismissCookieBanner } from './cookie-handler.js';
 import { forceLazyImages, forceAnimatedElementsVisible, triggerLazyLoad, waitForAllImages, LAZY_LOAD_MAX_ITERATIONS } from './lazy-loader.js';
-import { extractCleanHtml, JS_FRAMEWORK_PATTERNS, MAX_HTML_SIZE } from './html-extractor.js';
+import { extractCleanHtml, extractAndEnhanceHtml, JS_FRAMEWORK_PATTERNS, MAX_HTML_SIZE } from './html-extractor.js';
+import { extractContentCounts, generateContentSummary } from './content-counter.js';
 import { extractAllCss, MAX_CSS_SIZE } from './css-extractor.js';
 import { extractComponentDimensions } from './dimension-extractor.js';
 import { buildDimensionsOutput, generateAISummary } from './dimension-output.js';
+import { extractDOMHierarchy } from './dom-tree-analyzer.js';
 import { extractAnimations, generateAnimationsCss, generateAnimationTokens } from './animation-extractor.js';
 import { captureAllHoverStates, generateHoverCss } from './state-capture.js';
 import { captureVideo, hasFfmpeg, FFMPEG_REQUIRED_FORMATS } from './video-capture.js';
@@ -102,7 +106,7 @@ async function compressIfNeeded(filePath, maxSizeMB = 5) {
  * Capture screenshot for a single viewport
  */
 async function captureViewport(page, viewport, outputPath, fullPage = true, maxSize = 5, scrollDelay = DEFAULT_SCROLL_DELAY) {
-  await page.setViewport(VIEWPORTS[viewport]);
+  await page.setViewportSize(VIEWPORTS[viewport]);
   await new Promise(r => setTimeout(r, VIEWPORT_SETTLE_DELAY));
   await waitForDomStable(page, 300, 5000);
   await waitForFontsLoaded(page, 3000);
@@ -110,13 +114,23 @@ async function captureViewport(page, viewport, outputPath, fullPage = true, maxS
 
   const componentDimensions = await extractComponentDimensions(page, viewport);
 
+  // Extract DOM hierarchy (desktop only for efficiency)
+  let domHierarchy = null;
+  if (viewport === 'desktop') {
+    try {
+      domHierarchy = await extractDOMHierarchy(page, { maxDepth: 8 });
+    } catch (err) {
+      console.error(`[WARN] DOM hierarchy extraction failed: ${err.message}`);
+    }
+  }
+
   const lazyStats = await forceLazyImages(page);
   const scrollInfo = await triggerLazyLoad(page, LAZY_LOAD_MAX_ITERATIONS, scrollDelay);
   await forceLazyImages(page);
   const imageStats = await waitForAllImages(page, 15000);
 
   try {
-    await page.waitForNetworkIdle({ timeout: NETWORK_IDLE_TIMEOUT });
+    await page.waitForLoadState('networkidle', { timeout: NETWORK_IDLE_TIMEOUT });
   } catch {
     // Timeout ok
   }
@@ -142,6 +156,7 @@ async function captureViewport(page, viewport, outputPath, fullPage = true, maxS
     path: path.resolve(outputPath),
     dimensions: VIEWPORTS[viewport],
     componentDimensions,
+    domHierarchy,  // DOM hierarchy (desktop only)
     scrollInfo,
     imageStats,
     size: compression.finalSize,
@@ -179,6 +194,7 @@ async function captureMultiViewport() {
   const videoDuration = args['video-duration']
     ? parseInt(args['video-duration'], 10)
     : 12000;
+  const sectionMode = args['section-mode'] === 'true';
 
   for (const vp of requestedViewports) {
     if (!VIEWPORTS[vp]) {
@@ -214,8 +230,8 @@ async function captureMultiViewport() {
         currentHeadless = headless;
 
         if (navigateUrl) {
-          await page.setViewport(VIEWPORTS.desktop);
-          await page.goto(navigateUrl, { waitUntil: ['load', 'networkidle0'], timeout: 60000 });
+          await page.setViewportSize(VIEWPORTS.desktop);
+          await page.goto(navigateUrl, { waitUntil: 'domcontentloaded', timeout: 90000 });
           await new Promise(r => setTimeout(r, 3000));
           cookieResult = await dismissCookieBanner(page);
           await waitForPageReady(page);
@@ -234,9 +250,40 @@ async function captureMultiViewport() {
     if (extractHtml || extractCss) {
       extraction = { html: null, css: null, warnings: [] };
 
+      // Extract content counts BEFORE cleaning HTML (to count hidden items too)
       if (extractHtml) {
         try {
-          const htmlResult = await extractCleanHtml(page, JS_FRAMEWORK_PATTERNS);
+          const contentCounts = await extractContentCounts(page);
+          const countsPath = path.join(args.output, 'content-counts.json');
+          await fs.writeFile(countsPath, JSON.stringify(contentCounts, null, 2), 'utf-8');
+
+          // Generate summary for structure analysis
+          const contentSummary = generateContentSummary(contentCounts);
+          const summaryPath = path.join(args.output, 'content-summary.md');
+          await fs.writeFile(summaryPath, contentSummary, 'utf-8');
+
+          extraction.contentCounts = {
+            path: path.resolve(countsPath),
+            summaryPath: path.resolve(summaryPath),
+            summary: contentCounts.summary
+          };
+
+          if (process.stderr.isTTY) {
+            console.error(`[INFO] Content counts: ${contentCounts.grids.total} grids, ${contentCounts.repeatedItems.total} items`);
+          }
+        } catch (error) {
+          extractionWarnings.push(`Content counting failed: ${error.message}`);
+        }
+      }
+
+      if (extractHtml) {
+        try {
+          // Use semantic enhancement unless --no-semantic flag is set
+          const enhanceSemantic = args['no-semantic'] !== 'true';
+          const htmlResult = enhanceSemantic
+            ? await extractAndEnhanceHtml(page, { enhanceSemantic: true })
+            : await extractCleanHtml(page, JS_FRAMEWORK_PATTERNS);
+
           const html = htmlResult.html;
           const htmlSize = Buffer.byteLength(html, 'utf-8');
 
@@ -246,7 +293,13 @@ async function captureMultiViewport() {
 
           const htmlPath = path.join(args.output, 'source.html');
           await fs.writeFile(htmlPath, html, 'utf-8');
-          extraction.html = { path: path.resolve(htmlPath), size: htmlSize, elementCount: htmlResult.elementCount };
+          extraction.html = {
+            path: path.resolve(htmlPath),
+            size: htmlSize,
+            elementCount: htmlResult.elementCount,
+            semanticEnhanced: enhanceSemantic,
+            semanticStats: htmlResult.semanticStats || null
+          };
           if (htmlResult.warnings.length > 0) extractionWarnings.push(...htmlResult.warnings);
         } catch (error) {
           extraction.html = { error: error.message, failed: true };
@@ -360,7 +413,7 @@ async function captureMultiViewport() {
       }
     }
 
-    // Capture hover states (requires headless mode per Puppeteer #5255)
+    // Capture hover states (requires headless mode per Playwright #5255)
     let hoverResult = null;
     if (captureHover) {
       try {
@@ -449,7 +502,7 @@ async function captureMultiViewport() {
         }
 
         // Use desktop viewport for video
-        await page.setViewport(VIEWPORTS.desktop);
+        await page.setViewportSize(VIEWPORTS.desktop);
         await new Promise(r => setTimeout(r, 1000));
 
         if (process.stderr.isTTY) {
@@ -490,6 +543,77 @@ async function captureMultiViewport() {
     const summaryPath = path.join(args.output, 'dimensions-summary.json');
     await fs.writeFile(summaryPath, JSON.stringify(aiSummary, null, 2));
 
+    // Write DOM hierarchy if available (from desktop capture)
+    const desktopScreenshot = screenshots.find(s => s.viewport === 'desktop');
+    let hierarchyPath = null;
+    if (desktopScreenshot?.domHierarchy) {
+      hierarchyPath = path.join(args.output, 'dom-hierarchy.json');
+      await fs.writeFile(hierarchyPath, JSON.stringify(desktopScreenshot.domHierarchy, null, 2));
+
+      if (process.stderr.isTTY) {
+        const stats = desktopScreenshot.domHierarchy.stats || {};
+        console.error(`[INFO] DOM hierarchy: ${stats.totalNodes || 0} nodes, ${stats.landmarkCount || 0} landmarks, ${stats.extractionTimeMs || 0}ms`);
+      }
+    }
+
+    // Section-based capture (for improved AI analysis)
+    let sectionResult = null;
+    if (sectionMode && desktopScreenshot) {
+      try {
+        // Lazy import section modules
+        const { detectSections } = await import('./section-detector.js');
+        const { cropSections } = await import('./section-cropper.js');
+
+        // Reset to desktop viewport for section detection
+        await page.setViewportSize(VIEWPORTS.desktop);
+        await new Promise(r => setTimeout(r, 500));
+
+        if (process.stderr.isTTY) {
+          console.error('[INFO] Detecting sections...');
+        }
+
+        const sections = await detectSections(page, {
+          padding: 40,
+          minSections: 3,
+          fallbackToViewport: true
+        });
+
+        if (process.stderr.isTTY) {
+          console.error(`[INFO] Found ${sections.length} sections, cropping...`);
+        }
+
+        const croppedResult = await cropSections(
+          desktopScreenshot.path,
+          sections,
+          args.output
+        );
+
+        sectionResult = {
+          enabled: true,
+          count: croppedResult.sections.length,
+          skipped: croppedResult.skipped.length,
+          sections: croppedResult.sections.map(s => ({
+            index: s.index,
+            name: s.name,
+            path: s.relativePath,
+            bounds: s.bounds,
+            role: s.role
+          })),
+          directory: croppedResult.directory,
+          summary: croppedResult.summary
+        };
+
+        if (process.stderr.isTTY) {
+          console.error(`[INFO] Sections: ${croppedResult.sections.length} cropped, ${croppedResult.skipped.length} skipped`);
+        }
+      } catch (err) {
+        sectionResult = { enabled: true, error: err.message };
+        if (process.stderr.isTTY) {
+          console.error(`[WARN] Section processing failed: ${err.message}`);
+        }
+      }
+    }
+
     const totalContainers = Object.values(dimensionsOutput.viewports).reduce((sum, vp) => sum + (vp.containers?.length || 0), 0);
     const totalCards = Object.values(dimensionsOutput.viewports).reduce((sum, vp) => sum + (vp.cards?.length || 0), 0);
     const totalGrids = Object.values(dimensionsOutput.viewports).reduce((sum, vp) => sum + (vp.gridLayouts?.length || 0), 0);
@@ -528,6 +652,11 @@ async function captureMultiViewport() {
         stats: { containers: totalContainers, cards: totalCards, gridLayouts: totalGrids,
           typography: Object.values(dimensionsOutput.viewports).reduce((sum, vp) => sum + (vp.typography?.length || 0), 0) }
       },
+      domHierarchy: desktopScreenshot?.domHierarchy ? {
+        path: path.resolve(hierarchyPath),
+        stats: desktopScreenshot.domHierarchy.stats
+      } : undefined,
+      sections: sectionResult,
       screenshots,
       browserRestarts: browserRestarts.length > 0 ? browserRestarts : undefined,
       scrollDelay,

package/src/core/section-cropper.js

@@ -0,0 +1,209 @@
+/**
+ * Section Cropper
+ *
+ * Crop full-page screenshot into individual section images using Sharp.
+ * Uses section bounds from section-detector.js.
+ *
+ * Usage:
+ *   import { cropSections } from './section-cropper.js';
+ *   const results = await cropSections(screenshotPath, sections, outputDir);
+ */
+
+import path from 'path';
+import fs from 'fs/promises';
+
+// Try to import Sharp
+let sharp = null;
+try {
+  sharp = (await import('sharp')).default;
+} catch {
+  // Sharp not available
+}
+
+// Default configuration
+const DEFAULT_OPTIONS = {
+  minHeight: 100,        // Skip sections smaller than this
+  quality: 90,           // PNG quality
+  compressionLevel: 6,   // PNG compression (0-9)
+  format: 'png'          // Output format
+};
+
+/**
+ * Crop sections from a full-page screenshot
+ * @param {string} screenshotPath - Path to full screenshot
+ * @param {Array} sections - Array of section objects with bounds
+ * @param {string} outputDir - Base output directory
+ * @param {Object} options - Configuration options
+ * @returns {Promise<Array>} Array of cropped section info
+ */
+export async function cropSections(screenshotPath, sections, outputDir, options = {}) {
+  const config = { ...DEFAULT_OPTIONS, ...options };
+
+  // Check Sharp availability
+  if (!sharp) {
+    throw new Error('Sharp is not installed. Run: npm install sharp');
+  }
+
+  // Create sections directory
+  const sectionsDir = path.join(outputDir, 'sections');
+  await fs.mkdir(sectionsDir, { recursive: true });
+
+  // Get source image metadata
+  const metadata = await sharp(screenshotPath).metadata();
+  const imageWidth = metadata.width;
+  const imageHeight = metadata.height;
+
+  const results = [];
+  const skipped = [];
+
+  for (const section of sections) {
+    // Validate and clamp bounds
+    const bounds = validateBounds(section.bounds, imageWidth, imageHeight);
+
+    // Skip tiny sections
+    if (bounds.height < config.minHeight) {
+      skipped.push({
+        index: section.index,
+        name: section.name,
+        reason: `Height ${bounds.height}px < ${config.minHeight}px minimum`
+      });
+      continue;
+    }
+
+    // Skip zero-dimension sections
+    if (bounds.width <= 0 || bounds.height <= 0) {
+      skipped.push({
+        index: section.index,
+        name: section.name,
+        reason: 'Zero or negative dimensions'
+      });
+      continue;
+    }
+
+    // Generate output filename
+    const safeName = sanitizeName(section.name);
+    const filename = `section-${section.index}-${safeName}.png`;
+    const outputPath = path.join(sectionsDir, filename);
+
+    try {
+      // Crop and save
+      await sharp(screenshotPath)
+        .extract({
+          left: bounds.left,
+          top: bounds.top,
+          width: bounds.width,
+          height: bounds.height
+        })
+        .png({
+          quality: config.quality,
+          compressionLevel: config.compressionLevel
+        })
+        .toFile(outputPath);
+
+      results.push({
+        index: section.index,
+        name: section.name,
+        filename,
+        path: outputPath,
+        relativePath: path.join('sections', filename),
+        bounds: {
+          x: bounds.left,
+          y: bounds.top,
+          width: bounds.width,
+          height: bounds.height
+        },
+        role: section.role || 'unknown',
+        selector: section.selector || null
+      });
+    } catch (err) {
+      skipped.push({
+        index: section.index,
+        name: section.name,
+        reason: `Crop error: ${err.message}`
+      });
+    }
+  }
+
+  // Write summary JSON
+  const summary = {
+    source: path.basename(screenshotPath),
+    sourceWidth: imageWidth,
+    sourceHeight: imageHeight,
+    sectionsCount: results.length,
+    skippedCount: skipped.length,
+    sections: results,
+    skipped: skipped.length > 0 ? skipped : undefined,
+    createdAt: new Date().toISOString()
+  };
+
+  const summaryPath = path.join(sectionsDir, 'sections.json');
+  await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2));
+
+  return {
+    sections: results,
+    skipped,
+    summary: summaryPath,
+    directory: sectionsDir
+  };
+}
+
+/**
+ * Validate and clamp bounds to image dimensions
+ * @param {Object} bounds - Section bounds {x, y, width, height}
+ * @param {number} imageWidth - Source image width
+ * @param {number} imageHeight - Source image height
+ * @returns {Object} Validated bounds {left, top, width, height}
+ */
+function validateBounds(bounds, imageWidth, imageHeight) {
+  // Clamp starting position
+  const left = Math.max(0, Math.round(bounds.x));
+  const top = Math.max(0, Math.round(bounds.y));
+
+  // Calculate max possible dimensions
+  const maxWidth = imageWidth - left;
+  const maxHeight = imageHeight - top;
+
+  // Clamp dimensions
+  const width = Math.min(Math.round(bounds.width), maxWidth);
+  const height = Math.min(Math.round(bounds.height), maxHeight);
+
+  return { left, top, width, height };
+}
+
+/**
+ * Sanitize section name for filename
+ * @param {string} name - Section name
+ * @returns {string} Safe filename
+ */
+function sanitizeName(name) {
+  return name
+    .toLowerCase()
+    .replace(/[^a-z0-9-]/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^-|-$/g, '')
+    .substring(0, 50) || 'unnamed';
+}
+
+/**
+ * Check if Sharp is available
+ * @returns {boolean}
+ */
+export function isSharpAvailable() {
+  return sharp !== null;
+}
+
+/**
+ * Get cropper summary for logging
+ * @param {Object} result - Result from cropSections
+ * @returns {Object} Summary object
+ */
+export function getCropperSummary(result) {
+  return {
+    cropped: result.sections.length,
+    skipped: result.skipped.length,
+    directory: result.directory,
+    totalSize: result.sections.reduce((sum, s) => sum + (s.bounds.width * s.bounds.height), 0)
+  };
+}
+
+export { DEFAULT_OPTIONS };