design-clone 2.1.0 → 3.0.0

package/src/core/links/rewrite-links.js

@@ -0,0 +1,173 @@
+/**
+ * Link Rewriting Module
+ *
+ * Rewrites internal links in HTML to point to local .html files.
+ * Preserves external links unchanged.
+ * CSS link rewriting lives in rewrite-links-css-rewriter.js.
+ *
+ * Usage:
+ *   import { rewriteLinks, createPageManifest } from '../links/rewrite-links.js';
+ *   const rewritten = rewriteLinks(html, manifest, { baseUrl });
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import { normalizeUrl } from '../discovery/discover-pages-utils.js';
+import { rewriteCssLinks } from './rewrite-links-css-rewriter.js';
+
+/**
+ * Convert URL path to local filename.
+ * @param {string} urlPath - e.g. '/about', '/services/consulting'
+ * @returns {string} e.g. 'about.html', 'services-consulting.html'
+ */
+export function pathToFilename(urlPath) {
+  if (!urlPath || urlPath === '/' || urlPath === '') return 'index.html';
+
+  const name = urlPath
+    .replace(/^\//, '')
+    .replace(/\/$/, '')
+    .replace(/\//g, '-')
+    .replace(/[^a-z0-9-]/gi, '-')
+    .replace(/-+/g, '-')
+    .toLowerCase();
+
+  return `${name}.html`;
+}
+
+/**
+ * Create page manifest from discovered pages.
+ * @param {Array<{ path, name, url }>} pages
+ * @param {Object} options
+ * @returns {Object} Page manifest
+ */
+export function createPageManifest(pages, options = {}) {
+  const baseUrl = pages[0]?.url ? new URL(pages[0].url).origin : '';
+
+  return {
+    baseUrl,
+    capturedAt: new Date().toISOString(),
+    pages: pages.map(page => ({
+      path:        page.path,
+      name:        page.name,
+      file:        pathToFilename(page.path),
+      originalUrl: page.url
+    })),
+    assets: {
+      css:    'styles.css',
+      tokens: options.hasTokens ? 'tokens.css' : null
+    },
+    stats: options.stats || {}
+  };
+}
+
+/**
+ * Build URL-to-filename mapping from manifest (path + full URL variants).
+ * @param {Object} manifest
+ * @returns {Map<string, string>}
+ */
+function buildUrlMap(manifest) {
+  const urlMap = new Map();
+
+  for (const page of manifest.pages) {
+    if (page.originalUrl) {
+      urlMap.set(page.originalUrl, page.file);
+      urlMap.set(page.originalUrl.replace(/\/$/, ''), page.file);
+    }
+    if (page.path) {
+      urlMap.set(page.path, page.file);
+      if (page.path !== '/') urlMap.set(page.path.replace(/\/$/, ''), page.file);
+    }
+  }
+
+  return urlMap;
+}
+
+/**
+ * Rewrite links in HTML to point to local files.
+ * @param {string} html
+ * @param {Object} manifest
+ * @param {Object} options
+ * @param {string} [options.baseUrl]
+ * @param {boolean} [options.rewriteCss=true]
+ * @param {boolean} [options.injectTokensCss=false]
+ * @returns {string} HTML with rewritten links
+ */
+export function rewriteLinks(html, manifest, options = {}) {
+  const { baseUrl, rewriteCss = true, injectTokensCss = false } = options;
+  const urlMap = buildUrlMap(manifest);
+
+  let result = html;
+
+  // Rewrite <a href="..."> internal links
+  result = result.replace(
+    /(<a\s[^>]*href=["'])([^"']+)(["'][^>]*>)/gi,
+    (match, prefix, href, suffix) => {
+      if (!href ||
+          href.startsWith('javascript:') ||
+          href.startsWith('mailto:') ||
+          href.startsWith('tel:') ||
+          href.startsWith('#')) {
+        return match;
+      }
+
+      let filename = null;
+
+      if (urlMap.has(href)) {
+        filename = urlMap.get(href);
+      } else if (baseUrl) {
+        const normalized = normalizeUrl(baseUrl, href);
+        if (normalized && urlMap.has(normalized)) filename = urlMap.get(normalized);
+      }
+
+      if (filename) {
+        const fragmentMatch = href.match(/#[^#]*$/);
+        const fragment      = fragmentMatch ? fragmentMatch[0] : '';
+        return `${prefix}${filename}${fragment}${suffix}`;
+      }
+
+      return match;
+    }
+  );
+
+  if (rewriteCss) {
+    result = rewriteCssLinks(result, injectTokensCss);
+  }
+
+  return result;
+}
+
+/**
+ * Rewrite links in all HTML files listed in manifest.
+ * @param {string} htmlDir
+ * @param {Object} manifest
+ * @param {Object} options
+ * @returns {Promise<{ processed: string[], errors: Array }>}
+ */
+export async function rewriteAllLinks(htmlDir, manifest, options = {}) {
+  const results = { processed: [], errors: [] };
+
+  for (const page of manifest.pages) {
+    const htmlPath = path.join(htmlDir, page.file);
+    try {
+      const html      = await fs.readFile(htmlPath, 'utf-8');
+      const rewritten = rewriteLinks(html, manifest, options);
+      await fs.writeFile(htmlPath, rewritten, 'utf-8');
+      results.processed.push(page.file);
+    } catch (err) {
+      results.errors.push({ file: page.file, error: err.message });
+    }
+  }
+
+  return results;
+}
+
+// CLI stub
+const isMainModule = process.argv[1] && (
+  process.argv[1].endsWith('rewrite-links.js') ||
+  process.argv[1].includes('rewrite-links')
+);
+
+if (isMainModule) {
+  console.log('rewrite-links.js - Use as module, not CLI');
+  console.log('Exports: rewriteLinks, createPageManifest, pathToFilename, rewriteAllLinks');
+}

package/src/core/media/asset-validator.js

@@ -0,0 +1,118 @@
+/**
+ * Asset Validator
+ *
+ * Validates downloaded assets via magic bytes and size thresholds.
+ * Sanitizes SVG files by stripping script tags and event handlers.
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+
+const MAGIC_BYTES = {
+  png:   [0x89, 0x50, 0x4E, 0x47],
+  jpeg:  [0xFF, 0xD8, 0xFF],
+  gif:   [0x47, 0x49, 0x46],
+  woff2: [0x77, 0x4F, 0x46, 0x32],
+  woff:  [0x77, 0x4F, 0x46, 0x46],
+};
+
+const MIN_SIZES = { images: 100, fonts: 1024 };
+
+/**
+ * Validate a single asset file
+ * @param {string} filePath - Path to the file
+ * @returns {Promise<{valid: boolean, type: string, issues: string[]}>}
+ */
+export async function validateAsset(filePath) {
+  const issues = [];
+  const ext = path.extname(filePath).toLowerCase().slice(1);
+
+  let stat;
+  try {
+    stat = await fs.stat(filePath);
+  } catch {
+    return { valid: false, type: ext, issues: ['File not found'] };
+  }
+
+  // Size check
+  const isFont = ['woff', 'woff2', 'ttf', 'otf', 'eot'].includes(ext);
+  const minSize = isFont ? MIN_SIZES.fonts : MIN_SIZES.images;
+  if (stat.size < minSize) {
+    issues.push(`File too small: ${stat.size} bytes (min: ${minSize})`);
+  }
+
+  // SVG check
+  if (ext === 'svg') {
+    const content = await fs.readFile(filePath, 'utf-8');
+    if (/<script[\s>]/i.test(content)) issues.push('SVG contains script tags');
+    if (/\son\w+\s*=/i.test(content)) issues.push('SVG contains event handlers');
+    if (/javascript\s*:/i.test(content)) issues.push('SVG contains javascript: URIs');
+    if (/<(iframe|object|embed)[\s>]/i.test(content)) issues.push('SVG contains unsafe elements');
+    if (issues.length > 0) {
+      const sanitized = sanitizeSvg(content);
+      await fs.writeFile(filePath, sanitized, 'utf-8');
+      return { valid: true, type: 'svg', issues, sanitized: true };
+    }
+    return { valid: true, type: 'svg', issues: [] };
+  }
+
+  // Magic byte check for known types
+  const magicKey = ext === 'jpg' ? 'jpeg' : ext;
+  const expected = MAGIC_BYTES[magicKey];
+  if (expected && stat.size >= expected.length) {
+    const buf = Buffer.alloc(expected.length);
+    const fh = await fs.open(filePath, 'r');
+    await fh.read(buf, 0, expected.length, 0);
+    await fh.close();
+    const matches = expected.every((b, i) => buf[i] === b);
+    if (!matches) issues.push(`Magic bytes mismatch for ${ext}`);
+  }
+
+  return { valid: issues.length === 0, type: ext, issues };
+}
+
+/**
+ * Validate all assets in a directory
+ * @param {string} assetsDir - Assets root directory
+ * @returns {Promise<{valid: number, invalid: number, sanitized: number, details: Array}>}
+ */
+export async function validateBatch(assetsDir) {
+  const results = { valid: 0, invalid: 0, sanitized: 0, details: [] };
+  const subdirs = ['images', 'fonts', 'icons'];
+
+  for (const sub of subdirs) {
+    const dir = path.join(assetsDir, sub);
+    let files;
+    try { files = await fs.readdir(dir); } catch { continue; }
+    for (const file of files) {
+      const result = await validateAsset(path.join(dir, file));
+      if (result.sanitized) results.sanitized++;
+      if (result.valid) results.valid++;
+      else {
+        results.invalid++;
+        results.details.push({ file: `${sub}/${file}`, ...result });
+      }
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Strip dangerous content from SVG
+ * @param {string} content - SVG content
+ * @returns {string} Sanitized SVG
+ */
+export function sanitizeSvg(content) {
+  return content
+    // Strip <script> blocks
+    .replace(/<script[\s\S]*?<\/script>/gi, '')
+    // Strip dangerous elements: <iframe>, <object>, <embed>
+    .replace(/<(iframe|object|embed)[\s\S]*?<\/\1>/gi, '')
+    .replace(/<(iframe|object|embed)(\s[^>]*)?\s*\/>/gi, '')
+    // Strip javascript: URIs from href, xlink:href, src attributes
+    .replace(/((?:xlink:)?href|src)\s*=\s*["']\s*javascript\s*:[^"']*["']/gi, '')
+    // Strip on* event handler attributes
+    .replace(/\son\w+\s*=\s*["'][^"']*["']/gi, '')
+    .replace(/\son\w+\s*=\s*\{[^}]*\}/gi, '');
+}

package/src/core/media/extract-assets-downloader.js

@@ -0,0 +1,187 @@
+/**
+ * Asset Downloader Utilities
+ *
+ * Handles file downloading with timeout/retry, batch rate-limiting,
+ * safe filename generation, and asset-type classification.
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import https from 'https';
+import http from 'http';
+import { URL } from 'url';
+
+// Asset type configurations
+export const ASSET_TYPES = {
+  images: {
+    extensions: ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico', '.avif'],
+    folder: 'images'
+  },
+  fonts: {
+    extensions: ['.woff', '.woff2', '.ttf', '.otf', '.eot'],
+    folder: 'fonts'
+  },
+  icons: {
+    extensions: ['.svg'],
+    folder: 'icons'
+  }
+};
+
+// Rate limiting configuration
+export const RATE_LIMIT = {
+  maxConcurrent: 10,
+  delayBetweenBatches: 50
+};
+
+/**
+ * Download a file with timeout and retry.
+ * @param {string} url
+ * @param {string} destPath
+ * @param {number} timeout - ms
+ * @param {number} retries
+ * @returns {Promise<{ success: boolean, error?: string }>}
+ */
+export async function downloadFile(url, destPath, timeout = 30000, retries = 2) {
+  const protocol = url.startsWith('https') ? https : http;
+
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    try {
+      await new Promise((resolve, reject) => {
+        const timeoutId = setTimeout(() => reject(new Error('Download timeout')), timeout);
+
+        const request = protocol.get(url, {
+          headers: {
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
+            'Accept': '*/*'
+          }
+        }, (response) => {
+          if (response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
+            clearTimeout(timeoutId);
+            downloadFile(response.headers.location, destPath, timeout, 0).then(resolve).catch(reject);
+            return;
+          }
+
+          if (response.statusCode === 429) {
+            clearTimeout(timeoutId);
+            const retryAfter = parseInt(response.headers['retry-after'] || '0', 10);
+            const backoffMs = retryAfter > 0
+              ? retryAfter * 1000
+              : Math.min(1000 * Math.pow(2, attempt), 8000);
+            reject(new Error(`HTTP 429 (retry after ${backoffMs}ms)`));
+            return;
+          }
+
+          if (response.statusCode !== 200) {
+            clearTimeout(timeoutId);
+            reject(new Error(`HTTP ${response.statusCode}`));
+            return;
+          }
+
+          const chunks = [];
+          response.on('data', chunk => chunks.push(chunk));
+          response.on('end', async () => {
+            clearTimeout(timeoutId);
+            try {
+              const buffer = Buffer.concat(chunks);
+              await fs.mkdir(path.dirname(destPath), { recursive: true });
+              await fs.writeFile(destPath, buffer);
+              resolve({ size: buffer.length });
+            } catch (err) {
+              reject(err);
+            }
+          });
+          response.on('error', err => { clearTimeout(timeoutId); reject(err); });
+        });
+
+        request.on('error', err => { clearTimeout(timeoutId); reject(err); });
+      });
+
+      return { success: true };
+    } catch (err) {
+      if (attempt === retries) return { success: false, error: err.message };
+      const is429 = err.message.includes('HTTP 429');
+      const delay = is429
+        ? parseInt(err.message.match(/(\d+)ms/)?.[1] || '2000')
+        : 500 * (attempt + 1);
+      await new Promise(r => setTimeout(r, delay));
+    }
+  }
+}
+
+/**
+ * Download files in batches with rate limiting.
+ * @param {Array<{ url: string, destPath: string, type: string }>} downloads
+ * @param {boolean} verbose
+ * @returns {Promise<{ success: number, failed: number, skipped: number, errors: Array }>}
+ */
+export async function downloadBatch(downloads, verbose = false, options = {}) {
+  const { maxConcurrent = RATE_LIMIT.maxConcurrent, delayBetween = RATE_LIMIT.delayBetweenBatches } = options;
+  const results = { success: 0, failed: 0, skipped: 0, errors: [] };
+
+  for (let i = 0; i < downloads.length; i += maxConcurrent) {
+    const batch = downloads.slice(i, i + maxConcurrent);
+
+    await Promise.all(batch.map(async ({ url, destPath, type }) => {
+      try {
+        await fs.access(destPath);
+        results.skipped++;
+        return;
+      } catch { /* file doesn't exist, continue */ }
+
+      const result = await downloadFile(url, destPath);
+      if (result.success) {
+        results.success++;
+        if (verbose) console.error(`  ✓ ${type}: ${path.basename(destPath)}`);
+      } else {
+        results.failed++;
+        results.errors.push({ url, error: result.error });
+        if (verbose) console.error(`  ✗ ${type}: ${path.basename(url)} - ${result.error}`);
+      }
+    }));
+
+    if (i + maxConcurrent < downloads.length) {
+      await new Promise(r => setTimeout(r, delayBetween));
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Generate safe filename from URL.
+ * @param {string} url
+ * @returns {string}
+ */
+export function getSafeFilename(url) {
+  try {
+    const urlObj = new URL(url);
+    let filename = path.basename(urlObj.pathname);
+
+    if (urlObj.search) {
+      const hash = Buffer.from(urlObj.search).toString('base64').slice(0, 8);
+      const ext = path.extname(filename);
+      const base = path.basename(filename, ext);
+      filename = `${base}-${hash}${ext}`;
+    }
+
+    filename = filename.replace(/[^a-zA-Z0-9._-]/g, '_');
+    if (!path.extname(filename)) filename += '.bin';
+
+    return filename;
+  } catch {
+    return `asset-${Date.now()}.bin`;
+  }
+}
+
+/**
+ * Determine asset type from URL extension.
+ * @param {string} url
+ * @returns {'fonts'|'icons'|'images'|'other'}
+ */
+export function getAssetType(url) {
+  const ext = path.extname(new URL(url).pathname).toLowerCase();
+  if (ASSET_TYPES.fonts.extensions.includes(ext)) return 'fonts';
+  if (ext === '.svg') return 'icons';
+  if (ASSET_TYPES.images.extensions.includes(ext)) return 'images';
+  return 'other';
+}

package/src/core/media/extract-assets-page-scraper.js

@@ -0,0 +1,115 @@
+/**
+ * Page Asset Scraper
+ *
+ * Extracts asset URLs from a live Playwright page (images, inline SVGs,
+ * CSS stylesheet links) and from raw CSS content (background URLs, font-face).
+ */
+
+import { URL } from 'url';
+
+/**
+ * Parse CSS text for asset URLs (background images + font-face src).
+ * @param {string} cssContent
+ * @param {string} baseUrl
+ * @returns {string[]} Absolute asset URLs
+ */
+export function extractCssUrls(cssContent, baseUrl) {
+  const urls = new Set();
+
+  const bgPattern = /url\s*\(\s*['"]?([^'")\s]+)['"]?\s*\)/gi;
+  let match;
+  while ((match = bgPattern.exec(cssContent)) !== null) {
+    const url = match[1];
+    if (!url.startsWith('data:')) {
+      try { urls.add(new URL(url, baseUrl).href); } catch { /* ignore */ }
+    }
+  }
+
+  const fontPattern = /@font-face\s*\{[^}]*src:\s*([^;]+)/gi;
+  while ((match = fontPattern.exec(cssContent)) !== null) {
+    const srcValue = match[1];
+    const urlPattern = /url\s*\(\s*['"]?([^'")\s]+)['"]?\s*\)/gi;
+    let urlMatch;
+    while ((urlMatch = urlPattern.exec(srcValue)) !== null) {
+      const url = urlMatch[1];
+      if (!url.startsWith('data:')) {
+        try { urls.add(new URL(url, baseUrl).href); } catch { /* ignore */ }
+      }
+    }
+  }
+
+  return Array.from(urls);
+}
+
+/**
+ * Extract all asset URLs and inline SVGs from a Playwright page.
+ * @param {import('playwright').Page} page
+ * @param {string} baseUrl
+ * @returns {Promise<{ images: string[], cssUrls: string[], inlineSvgs: Array<{id: string, content: string}> }>}
+ */
+export async function extractAssetsFromPage(page, baseUrl) {
+  return await page.evaluate((url) => {
+    const imageSet = new Set();
+    const cssUrls = [];
+
+    // img[src]
+    document.querySelectorAll('img[src]').forEach(img => {
+      const src = img.getAttribute('src');
+      if (src && !src.startsWith('data:')) {
+        try { imageSet.add(new URL(src, url).href); } catch { /* ignore */ }
+      }
+    });
+
+    // srcset
+    document.querySelectorAll('[srcset]').forEach(el => {
+      const srcset = el.getAttribute('srcset');
+      if (srcset) {
+        srcset.split(',').forEach(part => {
+          const src = part.trim().split(/\s+/)[0];
+          if (src && !src.startsWith('data:')) {
+            try { imageSet.add(new URL(src, url).href); } catch { /* ignore */ }
+          }
+        });
+      }
+    });
+
+    // Inline background images
+    document.querySelectorAll('[style*="background"]').forEach(el => {
+      const style = el.getAttribute('style');
+      const urlMatch = style.match(/url\s*\(\s*['"]?([^'")\s]+)['"]?\s*\)/i);
+      if (urlMatch && !urlMatch[1].startsWith('data:')) {
+        try { imageSet.add(new URL(urlMatch[1], url).href); } catch { /* ignore */ }
+      }
+    });
+
+    // Favicon and touch icons
+    document.querySelectorAll('link[rel*="icon"]').forEach(link => {
+      const href = link.getAttribute('href');
+      if (href && !href.startsWith('data:')) {
+        try { imageSet.add(new URL(href, url).href); } catch { /* ignore */ }
+      }
+    });
+
+    // Inline SVGs
+    const inlineSvgs = [];
+    document.querySelectorAll('svg').forEach((svg, index) => {
+      const svgContent = svg.outerHTML;
+      if (svgContent.length < 50000) {
+        inlineSvgs.push({
+          id: svg.id || `inline-svg-${index}`,
+          content: svgContent
+        });
+      }
+    });
+
+    // External CSS stylesheets (for font extraction)
+    document.querySelectorAll('link[rel="stylesheet"]').forEach(link => {
+      const href = link.getAttribute('href');
+      if (href) {
+        try { cssUrls.push(new URL(href, url).href); } catch { /* ignore */ }
+      }
+    });
+
+    return { images: Array.from(imageSet), cssUrls, inlineSvgs };
+  }, baseUrl);
+}