design-clone 1.2.0 → 2.3.0

package/src/core/media/extract-assets.js

@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+/**
+ * Asset Extraction Script for Pixel-Perfect Clone
+ *
+ * Downloads and organizes assets from source website:
+ * - Images (jpg, png, gif, webp, svg)
+ * - Fonts (woff, woff2, ttf, otf)
+ * - CSS-embedded images (background-url)
+ *
+ * Usage:
+ *   node extract-assets.js --url <url> --output <dir> [--verbose]
+ *
+ * Options:
+ *   --url       Target website URL (required)
+ *   --output    Output directory (required)
+ *   --verbose   Show detailed progress
+ *   --timeout   Download timeout in ms (default: 30000)
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { getBrowser, getPage, closeBrowser, disconnectBrowser } from '../../utils/browser.js';
+import { parseArgs, outputJSON, outputError } from '../../utils/helpers.js';
+import { downloadBatch, getSafeFilename, getAssetType } from './extract-assets-downloader.js';
+import { extractCssUrls, extractAssetsFromPage } from './extract-assets-page-scraper.js';
+
+/**
+ * Main extraction function
+ */
+async function extractAssets() {
+  const args = parseArgs(process.argv.slice(2));
+
+  if (!args.url) {
+    outputError(new Error('--url is required'));
+    process.exit(1);
+  }
+  if (!args.output) {
+    outputError(new Error('--output directory is required'));
+    process.exit(1);
+  }
+
+  const verbose = args.verbose === 'true';
+  const timeout = args.timeout ? parseInt(args.timeout) : 30000;
+
+  try {
+    // Create output directories
+    const assetsDir = path.join(args.output, 'assets');
+    await fs.mkdir(path.join(assetsDir, 'images'), { recursive: true });
+    await fs.mkdir(path.join(assetsDir, 'fonts'), { recursive: true });
+    await fs.mkdir(path.join(assetsDir, 'icons'), { recursive: true });
+
+    // Launch browser and navigate
+    const browser = await getBrowser({ headless: args.headless !== 'false' });
+    const page = await getPage(browser);
+
+    if (verbose) console.error(`\n📦 Extracting assets from: ${args.url}\n`);
+
+    await page.goto(args.url, { waitUntil: 'networkidle', timeout: 30000 });
+
+    // Extract assets from page DOM
+    const pageAssets = await extractAssetsFromPage(page, args.url);
+
+    // Collect CSS content for font/background extraction
+    let allCssContent = '';
+
+    const inlineCss = await page.evaluate(() =>
+      Array.from(document.querySelectorAll('style')).map(s => s.textContent).join('\n')
+    );
+    allCssContent += inlineCss;
+
+    const sourceCssPath = path.join(args.output, 'analysis', 'source.css');
+    try {
+      const sourceCss = await fs.readFile(sourceCssPath, 'utf-8');
+      allCssContent += '\n' + sourceCss;
+    } catch { /* source.css not available */ }
+
+    // Combine all URLs and categorize
+    const cssAssetUrls = extractCssUrls(allCssContent, args.url);
+    const allUrls = new Set([...pageAssets.images, ...cssAssetUrls]);
+
+    const downloads = [];
+    const urlMapping = {};
+
+    for (const url of allUrls) {
+      const type = getAssetType(url);
+      const filename = getSafeFilename(url);
+      const destPath = path.join(assetsDir, type === 'other' ? 'images' : type, filename);
+      const relativePath = path.relative(args.output, destPath);
+
+      downloads.push({ url, destPath, type });
+      urlMapping[url] = relativePath;
+    }
+
+    if (verbose) {
+      console.error(`Found ${downloads.length} assets to download:`);
+      console.error(`  - Images: ${downloads.filter(d => d.type === 'images').length}`);
+      console.error(`  - Fonts: ${downloads.filter(d => d.type === 'fonts').length}`);
+      console.error(`  - Icons: ${downloads.filter(d => d.type === 'icons').length}`);
+      console.error('');
+    }
+
+    // Download all assets
+    const parsedConcurrency = args.concurrency ? parseInt(args.concurrency) : NaN;
+    const concurrency = Number.isNaN(parsedConcurrency) ? undefined : parsedConcurrency;
+    const downloadResults = await downloadBatch(downloads, verbose, { maxConcurrent: concurrency });
+
+    // Validate downloaded assets
+    let integrity = null;
+    try {
+      const { validateBatch } = await import('./asset-validator.js');
+      integrity = await validateBatch(assetsDir);
+    } catch { /* validation optional */ }
+
+    // Save inline SVGs
+    let savedSvgs = 0;
+    for (const svg of pageAssets.inlineSvgs) {
+      const filename = `${svg.id.replace(/[^a-zA-Z0-9-_]/g, '_')}.svg`;
+      const svgPath = path.join(assetsDir, 'icons', filename);
+      try {
+        await fs.writeFile(svgPath, svg.content, 'utf-8');
+        savedSvgs++;
+      } catch { /* ignore */ }
+    }
+
+    // Save URL mapping for HTML rewriting
+    const mappingPath = path.join(assetsDir, 'url-mapping.json');
+    await fs.writeFile(mappingPath, JSON.stringify(urlMapping, null, 2));
+
+    // Close browser
+    if (args.close === 'true') {
+      await closeBrowser();
+    } else {
+      await disconnectBrowser();
+    }
+
+    outputJSON({
+      success: true,
+      assetsDir: path.resolve(assetsDir),
+      urlMapping: mappingPath,
+      stats: {
+        total: downloads.length,
+        downloaded: downloadResults.success,
+        failed: downloadResults.failed,
+        skipped: downloadResults.skipped,
+        inlineSvgs: savedSvgs
+      },
+      integrity: integrity || undefined,
+      errors: downloadResults.errors.length > 0 ? downloadResults.errors.slice(0, 10) : undefined
+    });
+    process.exit(0);
+
+  } catch (error) {
+    outputError(error);
+    process.exit(1);
+  }
+}
+
+// Run
+extractAssets();

package/src/core/media/video-capture-convert.js

@@ -0,0 +1,200 @@
+/**
+ * Video Capture Conversion
+ *
+ * ffmpeg dependency management and WebM-to-MP4/GIF format conversion.
+ * Lazy-loads fluent-ffmpeg and @ffmpeg-installer/ffmpeg so the module
+ * can be imported without those packages installed.
+ *
+ * @module video-capture-convert
+ */
+
+import fs from 'fs/promises';
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+/** Formats requiring ffmpeg for conversion */
+export const FFMPEG_REQUIRED_FORMATS = ['mp4', 'gif'];
+
+/** GIF output settings */
+const GIF_DEFAULT_FPS = 10;
+const GIF_DEFAULT_WIDTH = 640;
+
+// ============================================================================
+// ffmpeg Dependency Management
+// ============================================================================
+
+let ffmpeg = null;
+let ffmpegInitialized = false;
+
+/**
+ * Initialize ffmpeg dependencies.
+ * Lazy-loads fluent-ffmpeg and @ffmpeg-installer/ffmpeg.
+ *
+ * @returns {Promise<boolean>} True if ffmpeg is available
+ */
+export async function initFfmpeg() {
+  if (ffmpegInitialized) return ffmpeg !== false;
+
+  ffmpegInitialized = true;
+
+  try {
+    const [fluentFfmpeg, installer] = await Promise.all([
+      import('fluent-ffmpeg'),
+      import('@ffmpeg-installer/ffmpeg')
+    ]);
+
+    ffmpeg = fluentFfmpeg.default;
+    const ffmpegPath = installer.path;
+    ffmpeg.setFfmpegPath(ffmpegPath);
+
+    return true;
+  } catch (importError) {
+    ffmpeg = false;
+
+    if (importError.code !== 'ERR_MODULE_NOT_FOUND') {
+      console.error('[video-capture] ffmpeg initialization error:', importError.message);
+    }
+
+    return false;
+  }
+}
+
+/**
+ * Check if ffmpeg is available for video conversion.
+ *
+ * @returns {Promise<boolean>} True if ffmpeg dependencies are available
+ */
+export async function hasFfmpeg() {
+  return await initFfmpeg();
+}
+
+// ============================================================================
+// Input Validation
+// ============================================================================
+
+/**
+ * Validate output path
+ * @param {string} outputPath - Output file/directory path
+ * @throws {TypeError} If path is invalid
+ */
+export function validatePath(outputPath) {
+  if (!outputPath || typeof outputPath !== 'string') {
+    throw new TypeError('Invalid output path: must be a non-empty string');
+  }
+}
+
+// ============================================================================
+// Format Conversion
+// ============================================================================
+
+/**
+ * Convert WebM to MP4 using ffmpeg.
+ *
+ * Uses H.264 codec with settings optimized for web playback:
+ * - libx264 encoder with fast preset
+ * - CRF 23 for good quality/size balance
+ * - yuv420p pixel format for iOS/Safari compatibility
+ * - faststart flag for progressive playback
+ *
+ * @param {string} inputPath - Path to WebM file
+ * @param {string} outputPath - Path for MP4 output
+ * @returns {Promise<{path: string, format: string}>} Conversion result
+ * @throws {Error} If ffmpeg is not available or conversion fails
+ */
+export async function convertToMp4(inputPath, outputPath) {
+  validatePath(inputPath);
+  validatePath(outputPath);
+
+  const hasFf = await initFfmpeg();
+  if (!hasFf) {
+    throw new Error(
+      'ffmpeg not available. Install: npm install fluent-ffmpeg @ffmpeg-installer/ffmpeg'
+    );
+  }
+
+  return new Promise((resolve, reject) => {
+    ffmpeg(inputPath)
+      .outputOptions([
+        '-c:v libx264',
+        '-preset fast',
+        '-crf 23',
+        '-pix_fmt yuv420p',
+        '-movflags +faststart'
+      ])
+      .output(outputPath)
+      .on('end', () => resolve({ path: outputPath, format: 'mp4' }))
+      .on('error', (err) => reject(new Error(`MP4 conversion failed: ${err.message}`)))
+      .run();
+  });
+}
+
+/**
+ * Convert WebM to GIF using ffmpeg.
+ *
+ * Uses two-pass conversion with palette generation for high-quality output:
+ * 1. Generate optimized palette from video
+ * 2. Create GIF using palette with dithering
+ *
+ * @param {string} inputPath - Path to WebM file
+ * @param {string} outputPath - Path for GIF output
+ * @param {Object} [options={}] - GIF options
+ * @param {number} [options.fps=10] - Output frame rate
+ * @param {number} [options.width=640] - Output width (height auto-calculated)
+ * @returns {Promise<{path: string, format: string}>} Conversion result
+ * @throws {Error} If ffmpeg is not available or conversion fails
+ */
+export async function convertToGif(inputPath, outputPath, options = {}) {
+  validatePath(inputPath);
+  validatePath(outputPath);
+
+  const { fps = GIF_DEFAULT_FPS, width = GIF_DEFAULT_WIDTH } = options;
+
+  const hasFf = await initFfmpeg();
+  if (!hasFf) {
+    throw new Error(
+      'ffmpeg not available. Install: npm install fluent-ffmpeg @ffmpeg-installer/ffmpeg'
+    );
+  }
+
+  const palettePath = inputPath.replace(/\.webm$/i, '-palette.png');
+
+  try {
+    // Pass 1: Generate palette
+    await new Promise((resolve, reject) => {
+      ffmpeg(inputPath)
+        .outputOptions([
+          '-vf',
+          `fps=${fps},scale=${width}:-1:flags=lanczos,palettegen=stats_mode=diff`
+        ])
+        .output(palettePath)
+        .on('end', resolve)
+        .on('error', (err) => reject(new Error(`Palette generation failed: ${err.message}`)))
+        .run();
+    });
+
+    // Pass 2: Create GIF with palette
+    await new Promise((resolve, reject) => {
+      ffmpeg(inputPath)
+        .input(palettePath)
+        .complexFilter([
+          `fps=${fps},scale=${width}:-1:flags=lanczos[x];[x][1:v]paletteuse=dither=bayer:bayer_scale=5`
+        ])
+        .output(outputPath)
+        .on('end', resolve)
+        .on('error', (err) => reject(new Error(`GIF creation failed: ${err.message}`)))
+        .run();
+    });
+
+    return { path: outputPath, format: 'gif' };
+  } finally {
+    try {
+      await fs.unlink(palettePath);
+    } catch (cleanupErr) {
+      if (process.env.DEBUG) {
+        console.error(`[video-capture] Palette cleanup failed: ${cleanupErr.message}`);
+      }
+    }
+  }
+}

package/src/core/media/video-capture.js

@@ -0,0 +1,201 @@
+/**
+ * Video Capture Module
+ *
+ * Record scrolling interactions using Playwright's context-level video
+ * recording. Optionally converts WebM to MP4/GIF via video-capture-convert.js.
+ *
+ * Usage:
+ *   import { captureVideo, hasFfmpeg } from './video-capture.js';
+ *   const result = await captureVideo(page, outputDir, { format: 'webm' });
+ *
+ * @module video-capture
+ */
+
+import path from 'path';
+import fs from 'fs/promises';
+
+import { hasFfmpeg, convertToMp4, convertToGif, validatePath, FFMPEG_REQUIRED_FORMATS } from './video-capture-convert.js';
+import { isTTY } from '../../utils/log.js';
+
+const DEFAULT_DURATION = 12000;        // ms
+const DEFAULT_HOLD_MS = 500;           // hold at top/bottom of scroll
+const MAX_SCROLL_STEPS = 100;          // cap to avoid memory exhaustion
+const VIEWPORT_OVERLAP_FRACTION = 0.5; // fraction of viewport per scroll step
+const DEFAULT_VIDEO_VIEWPORT = { width: 1440, height: 900 };
+
+/** Log to stderr when running in TTY. */
+function log(message) {
+  if (isTTY) console.error(message);
+}
+
+/** Validate that page is a Playwright page instance. */
+function validatePage(page) {
+  if (!page || typeof page.evaluate !== 'function') throw new TypeError('Invalid page object: must be a Playwright page');
+  if (typeof page.context !== 'function') throw new TypeError('Invalid page object: missing context() method');
+}
+
+/**
+ * Record page scroll using a new Playwright context with video enabled.
+ * IMPORTANT: page must be closed before calling video.path() (Playwright requirement).
+ * @param {import('playwright').Browser} browser
+ * @param {string} pageUrl
+ * @param {string} outputDir
+ * @param {{duration?, scrollPauseMs?, holdTopMs?, holdBottomMs?, viewport?}} [options]
+ * @returns {Promise<{path, format, duration, scrollSteps, pageHeight}>}
+ */
+export async function recordScroll(browser, pageUrl, outputDir, options = {}) {
+  if (!browser || typeof browser.newContext !== 'function') {
+    throw new TypeError('Invalid browser: must be a Playwright browser instance');
+  }
+  validatePath(outputDir);
+
+  const {
+    duration = DEFAULT_DURATION,
+    scrollPauseMs = 50,
+    holdTopMs = DEFAULT_HOLD_MS,
+    holdBottomMs = DEFAULT_HOLD_MS,
+    viewport = DEFAULT_VIDEO_VIEWPORT
+  } = options;
+
+  const context = await browser.newContext({
+    recordVideo: { dir: outputDir, size: viewport },
+    viewport
+  });
+
+  const page = await context.newPage();
+  const startTime = Date.now();
+
+  try {
+    await page.goto(pageUrl, { waitUntil: 'networkidle', timeout: 30000 });
+
+    const totalHeight = await page.evaluate(() =>
+      Math.max(document.body.scrollHeight, document.documentElement.scrollHeight)
+    );
+
+    const viewportHeight = viewport.height;
+    const scrollDistance = Math.max(0, totalHeight - viewportHeight);
+    const isScrollable = scrollDistance > 0;
+
+    const rawScrollSteps = isScrollable
+      ? Math.ceil(scrollDistance / (viewportHeight * VIEWPORT_OVERLAP_FRACTION))
+      : 0;
+    const scrollSteps = Math.min(rawScrollSteps, MAX_SCROLL_STEPS);
+
+    const scrollTime = duration - holdTopMs - holdBottomMs;
+    const scrollDelay = scrollSteps > 0
+      ? Math.max(scrollPauseMs, Math.floor(scrollTime / (scrollSteps * 2)))
+      : 0;
+
+    await page.evaluate(() => window.scrollTo({ top: 0, behavior: 'instant' }));
+    await new Promise(r => setTimeout(r, 200 + holdTopMs));
+
+    if (isScrollable && scrollSteps > 0) {
+      const scroll = (i) => page.evaluate(y => window.scrollTo({ top: y, behavior: 'instant' }), (i / scrollSteps) * scrollDistance);
+      for (let i = 1; i <= scrollSteps; i++) { await scroll(i); await new Promise(r => setTimeout(r, scrollDelay)); }
+      await new Promise(r => setTimeout(r, holdBottomMs));
+      for (let i = scrollSteps - 1; i >= 0; i--) { await scroll(i); await new Promise(r => setTimeout(r, scrollDelay)); }
+      await new Promise(r => setTimeout(r, holdTopMs));
+    } else {
+      await new Promise(r => setTimeout(r, scrollTime + holdBottomMs));
+    }
+
+    const actualDuration = Date.now() - startTime;
+    // IMPORTANT: Close page before video.path() (Playwright requirement)
+    await page.close();
+    const video = page.video();
+    const videoPath = video ? await video.path() : null;
+    await context.close();
+    if (!videoPath) throw new Error('Video recording failed - no path returned');
+    return { path: videoPath, format: 'webm', duration: actualDuration, scrollSteps, pageHeight: totalHeight };
+
+  } catch (error) {
+    try {
+      await page.close().catch(() => {});
+      await context.close().catch(() => {});
+    } catch { /* ignore cleanup errors */ }
+    throw error;
+  }
+}
+
+/**
+ * Capture scroll video. Records WebM via new browser context, optionally converts to MP4/GIF.
+ * @param {import('playwright').Page} page - source of browser + URL
+ * @param {string} outputDir
+ * @param {{'webm'|'mp4'|'gif'} format?, number duration?, string filename?} [options]
+ * @returns {Promise<{webm, mp4?, gif?, output, duration, pageHeight, conversionError?}>}
+ */
+export async function captureVideo(page, outputDir, options = {}) {
+  validatePage(page);
+  validatePath(outputDir);
+
+  const { format = 'webm', duration = DEFAULT_DURATION, filename = 'preview' } = options;
+
+  const browser = page.context().browser();
+  const pageUrl = page.url();
+  const viewport = page.viewportSize() || DEFAULT_VIDEO_VIEWPORT;
+
+  if (!browser) {
+    throw new Error('Cannot get browser from page. Ensure page has browser context.');
+  }
+
+  log('[video] Recording scroll...');
+  const recordResult = await recordScroll(browser, pageUrl, outputDir, { duration, viewport });
+  log(`[video] Recorded ${(recordResult.duration / 1000).toFixed(1)}s`);
+
+  // Rename to expected filename (Playwright auto-generates random names)
+  const expectedPath = path.join(outputDir, `${filename}.webm`);
+  if (recordResult.path !== expectedPath) {
+    try {
+      await fs.rename(recordResult.path, expectedPath);
+      recordResult.path = expectedPath;
+    } catch (renameErr) {
+      log(`[video] Could not rename video: ${renameErr.message}`);
+    }
+  }
+
+  const result = {
+    webm: recordResult.path,
+    duration: recordResult.duration,
+    pageHeight: recordResult.pageHeight,
+    output: recordResult.path
+  };
+
+  if (format === 'mp4') {
+    const mp4Path = path.join(outputDir, `${filename}.mp4`);
+    log('[video] Converting to MP4...');
+    try {
+      await convertToMp4(recordResult.path, mp4Path);
+      result.mp4 = mp4Path;
+      result.output = mp4Path;
+      log('[video] MP4 conversion complete');
+    } catch (e) {
+      log(`[video] MP4 conversion failed: ${e.message}`);
+      result.conversionError = e.message;
+    }
+  } else if (format === 'gif') {
+    const gifPath = path.join(outputDir, `${filename}.gif`);
+    log('[video] Converting to GIF...');
+    try {
+      await convertToGif(recordResult.path, gifPath);
+      result.gif = gifPath;
+      result.output = gifPath;
+      log('[video] GIF conversion complete');
+    } catch (e) {
+      log(`[video] GIF conversion failed: ${e.message}`);
+      result.conversionError = e.message;
+    }
+  }
+
+  return result;
+}
+
+// Re-exports for backward compatibility
+export {
+  hasFfmpeg,
+  convertToMp4,
+  convertToGif,
+  FFMPEG_REQUIRED_FORMATS,
+  DEFAULT_DURATION,
+  MAX_SCROLL_STEPS,
+  VIEWPORT_OVERLAP_FRACTION
+};

package/src/core/{cookie-handler.js → page-prep/cookie-handler.js}

@@ -38,7 +38,7 @@ export const COOKIE_REMOVE_SELECTORS = [
 
 /**
  * Dismiss cookie banners by clicking accept or removing elements
- * @param {Page} page - Puppeteer page
+ * @param {Page} page - Playwright page
  * @returns {Promise<{method: string, selector?: string, count?: number}>}
  */
 export async function dismissCookieBanner(page) {