design-clone 1.0.0 → 1.1.0

package/src/core/design-tokens.js

@@ -0,0 +1,103 @@
+/**
+ * Design Tokens Extraction Wrapper
+ *
+ * Wraps the Python script for extracting design tokens from screenshots.
+ *
+ * Usage:
+ *   import { extractDesignTokens } from './design-tokens.js';
+ *   const result = await extractDesignTokens(outputDir, cssPath);
+ */
+
+import { spawn } from 'child_process';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+/**
+ * Extract design tokens from screenshots using Gemini Vision API
+ *
+ * @param {string} outputDir - Output directory (contains analysis/desktop/*.png)
+ * @param {string} cssPath - Path to merged CSS file for reference
+ * @returns {Promise<Object>} Result with { success, tokens_json, tokens_css }
+ */
+export async function extractDesignTokens(outputDir, cssPath = null) {
+  const scriptPath = path.resolve(__dirname, '../ai/extract-design-tokens.py');
+  const screenshotsDir = path.join(outputDir, 'analysis', 'desktop');
+
+  // Build args
+  const args = [
+    scriptPath,
+    '--screenshots', screenshotsDir,
+    '--output', outputDir
+  ];
+
+  if (cssPath) {
+    args.push('--css', cssPath);
+  }
+
+  return new Promise((resolve) => {
+    const proc = spawn('python3', args, {
+      stdio: ['ignore', 'pipe', 'pipe'],
+      env: { ...process.env }
+    });
+
+    let stdout = '';
+    let stderr = '';
+
+    proc.stdout.on('data', (data) => {
+      stdout += data.toString();
+    });
+
+    proc.stderr.on('data', (data) => {
+      stderr += data.toString();
+    });
+
+    proc.on('close', (code) => {
+      if (code !== 0) {
+        // Try to parse error from stdout (script outputs JSON errors)
+        try {
+          const errorResult = JSON.parse(stdout);
+          resolve({
+            success: false,
+            error: errorResult.error || 'Unknown error',
+            hint: errorResult.hint || null
+          });
+        } catch {
+          resolve({
+            success: false,
+            error: stderr || `Process exited with code ${code}`
+          });
+        }
+        return;
+      }
+
+      // Parse success result
+      try {
+        const result = JSON.parse(stdout);
+        resolve(result);
+      } catch (err) {
+        resolve({
+          success: false,
+          error: `Failed to parse output: ${err.message}`
+        });
+      }
+    });
+
+    proc.on('error', (err) => {
+      if (err.code === 'ENOENT') {
+        resolve({
+          success: false,
+          error: 'Python3 not found',
+          hint: 'Install Python 3 to enable AI token extraction'
+        });
+      } else {
+        resolve({
+          success: false,
+          error: err.message
+        });
+      }
+    });
+  });
+}

package/src/core/discover-pages.js

@@ -0,0 +1,314 @@
+/**
+ * Page Discovery Module
+ *
+ * Extracts navigation links from a website to discover cloneable pages.
+ * Handles SPA hydration, filters external links, and normalizes URLs.
+ *
+ * Usage:
+ *   import { discoverPages } from './discover-pages.js';
+ *   const result = await discoverPages('https://example.com', { maxPages: 10 });
+ */
+
+import { getBrowser, getPage, disconnectBrowser } from '../utils/browser.js';
+import { waitForDomStable, waitForPageReady } from './page-readiness.js';
+import { dismissCookieBanner } from './cookie-handler.js';
+
+// Navigation selectors in priority order
+const NAV_SELECTORS = [
+  'header nav a',
+  'header a',
+  'nav a',
+  '[role="navigation"] a',
+  '.navbar a',
+  '.nav-menu a',
+  '.navigation a',
+  'footer nav a',
+  'footer a'
+];
+
+// Patterns to exclude from discovered links
+const EXCLUDE_PATTERNS = [
+  /^mailto:/i,
+  /^tel:/i,
+  /^javascript:/i,
+  /^#/,
+  /\.(pdf|jpg|jpeg|png|gif|svg|webp|ico|zip|tar|gz|mp3|mp4|avi|mov)$/i,
+  /facebook\.com/i,
+  /twitter\.com/i,
+  /instagram\.com/i,
+  /linkedin\.com/i,
+  /youtube\.com/i,
+  /tiktok\.com/i
+];
+
+// Default options
+const DEFAULT_OPTIONS = {
+  maxPages: 10,
+  selectors: null,  // Use default NAV_SELECTORS if null
+  includeSubdomains: false,
+  timeout: 30000
+};
+
+/**
+ * Normalize URL for comparison and deduplication
+ * @param {string} baseUrl - Base URL for resolving relative paths
+ * @param {string} href - URL to normalize
+ * @returns {string|null} Normalized URL or null if invalid
+ */
+export function normalizeUrl(baseUrl, href) {
+  if (!href || typeof href !== 'string') return null;
+
+  try {
+    const url = new URL(href, baseUrl);
+
+    // Skip non-http(s) protocols
+    if (!url.protocol.startsWith('http')) return null;
+
+    // Build normalized URL: origin + pathname (no hash, no query)
+    let normalized = url.origin + url.pathname;
+
+    // Remove trailing slash (except for root)
+    if (normalized.endsWith('/') && normalized !== url.origin + '/') {
+      normalized = normalized.slice(0, -1);
+    }
+
+    return normalized;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Check if URL is same domain as base
+ * @param {string} url - URL to check
+ * @param {string} baseDomain - Base domain to compare against
+ * @param {boolean} includeSubdomains - Whether to include subdomains
+ * @returns {boolean}
+ */
+export function isSameDomain(url, baseDomain, includeSubdomains = false) {
+  try {
+    const urlObj = new URL(url);
+    const hostname = urlObj.hostname.toLowerCase();
+    const base = baseDomain.toLowerCase();
+
+    if (hostname === base) return true;
+
+    if (includeSubdomains) {
+      return hostname.endsWith('.' + base);
+    }
+
+    return false;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Extract page name from link text or URL path
+ * @param {string} text - Link text
+ * @param {string} path - URL path
+ * @returns {string} Page name
+ */
+export function extractPageName(text, path) {
+  // Use link text if available and meaningful
+  if (text && text.length > 0 && text.length < 50) {
+    return text;
+  }
+
+  // Extract from path
+  if (!path || path === '/') return 'Home';
+
+  // Get last segment of path
+  const segments = path.split('/').filter(Boolean);
+  if (segments.length === 0) return 'Home';
+
+  const lastSegment = segments[segments.length - 1];
+
+  // Convert kebab-case/snake_case to Title Case
+  return lastSegment
+    .replace(/[-_]/g, ' ')
+    .replace(/\b\w/g, c => c.toUpperCase());
+}
+
+/**
+ * Check if href should be excluded
+ * @param {string} href - URL to check
+ * @returns {boolean}
+ */
+function shouldExclude(href) {
+  if (!href) return true;
+  return EXCLUDE_PATTERNS.some(pattern => pattern.test(href));
+}
+
+/**
+ * Discover pages from a website by extracting navigation links
+ * @param {string} baseUrl - Starting URL to discover from
+ * @param {Object} options - Discovery options
+ * @returns {Promise<Object>} Discovery result
+ */
+export async function discoverPages(baseUrl, options = {}) {
+  const opts = { ...DEFAULT_OPTIONS, ...options };
+  const startTime = Date.now();
+
+  let browser = null;
+  let page = null;
+
+  try {
+    // Parse base URL
+    const baseUrlObj = new URL(baseUrl);
+    const baseDomain = baseUrlObj.hostname;
+
+    // Launch browser
+    browser = await getBrowser({ headless: true });
+    page = await getPage(browser);
+
+    // Navigate to page
+    await page.goto(baseUrl, {
+      waitUntil: ['load', 'networkidle0'],
+      timeout: opts.timeout
+    });
+
+    // Wait for SPA hydration
+    await page.waitForSelector('nav a, header a, [role="navigation"] a', {
+      visible: true,
+      timeout: 5000
+    }).catch(() => {});
+
+    await waitForDomStable(page, 500, 5000);
+
+    // Dismiss cookie banner if present
+    await dismissCookieBanner(page);
+
+    // Wait a bit more for any dynamic content
+    await new Promise(r => setTimeout(r, 1000));
+
+    // Extract links using selectors
+    const selectors = opts.selectors || NAV_SELECTORS;
+    const selectorString = selectors.join(', ');
+
+    const rawLinks = await page.$$eval(selectorString, (elements) => {
+      return elements.map(el => ({
+        href: el.href,
+        text: el.textContent?.trim() || '',
+        tagName: el.tagName
+      }));
+    }).catch(() => []);
+
+    // Process and filter links
+    const seenUrls = new Set();
+    const pages = [];
+
+    // Always include homepage first
+    const homeUrl = normalizeUrl(baseUrl, '/');
+    if (homeUrl) {
+      seenUrls.add(homeUrl);
+      pages.push({
+        path: '/',
+        name: 'Home',
+        url: homeUrl
+      });
+    }
+
+    for (const link of rawLinks) {
+      // Skip excluded patterns
+      if (shouldExclude(link.href)) continue;
+
+      // Normalize URL
+      const normalized = normalizeUrl(baseUrl, link.href);
+      if (!normalized) continue;
+
+      // Skip if already seen
+      if (seenUrls.has(normalized)) continue;
+
+      // Check same domain
+      if (!isSameDomain(normalized, baseDomain, opts.includeSubdomains)) continue;
+
+      // Extract path
+      const urlObj = new URL(normalized);
+      const path = urlObj.pathname;
+
+      // Skip homepage (already added)
+      if (path === '/') continue;
+
+      // Add to results
+      seenUrls.add(normalized);
+      pages.push({
+        path,
+        name: extractPageName(link.text, path),
+        url: normalized
+      });
+
+      // Check max pages limit
+      if (pages.length >= opts.maxPages) break;
+    }
+
+    // Sort by path depth (shallow first)
+    pages.sort((a, b) => {
+      if (a.path === '/') return -1;
+      if (b.path === '/') return 1;
+      const depthA = (a.path.match(/\//g) || []).length;
+      const depthB = (b.path.match(/\//g) || []).length;
+      return depthA - depthB;
+    });
+
+    const duration = Date.now() - startTime;
+
+    return {
+      success: true,
+      baseUrl: baseUrlObj.origin,
+      baseDomain,
+      pages,
+      stats: {
+        totalLinksFound: rawLinks.length,
+        pagesDiscovered: pages.length,
+        durationMs: duration
+      }
+    };
+  } catch (error) {
+    return {
+      success: false,
+      baseUrl,
+      pages: [{
+        path: '/',
+        name: 'Home',
+        url: normalizeUrl(baseUrl, '/') || baseUrl
+      }],
+      error: error.message,
+      stats: {
+        totalLinksFound: 0,
+        pagesDiscovered: 1,
+        durationMs: Date.now() - startTime
+      }
+    };
+  } finally {
+    if (browser) {
+      await disconnectBrowser();
+    }
+  }
+}
+
+// CLI support
+const isMainModule = process.argv[1] && (
+  process.argv[1].endsWith('discover-pages.js') ||
+  process.argv[1].includes('discover-pages')
+);
+
+if (isMainModule) {
+  const url = process.argv[2];
+  const maxPages = parseInt(process.argv[3]) || 10;
+
+  if (!url) {
+    console.error('Usage: node discover-pages.js <url> [maxPages]');
+    process.exit(1);
+  }
+
+  discoverPages(url, { maxPages })
+    .then(result => {
+      console.log(JSON.stringify(result, null, 2));
+      process.exit(result.success ? 0 : 1);
+    })
+    .catch(err => {
+      console.error(JSON.stringify({ success: false, error: err.message }));
+      process.exit(1);
+    });
+}

package/src/core/html-extractor.js

@@ -5,6 +5,8 @@
  * event handlers, and framework-specific attributes.
  */
 
+import { LAYOUT_PROPERTIES } from './css-extractor.js';
+
 // Size limits
 export const MAX_HTML_SIZE = 10 * 1024 * 1024; // 10MB limit
 export const MAX_DOM_ELEMENTS = 50000;          // Warn on large DOMs
@@ -16,6 +18,20 @@ export const JS_FRAMEWORK_PATTERNS = [
   /^data-alpine/i, /^wire:/i, /^@/
 ];
 
+// Properties to inline on critical elements (layout only, not visual)
+// Uses shared LAYOUT_PROPERTIES from css-extractor (DRY)
+export const INLINE_LAYOUT_PROPS = [
+  ...LAYOUT_PROPERTIES.display,
+  ...LAYOUT_PROPERTIES.grid,
+  ...LAYOUT_PROPERTIES.position,
+  ...LAYOUT_PROPERTIES.sizing,
+  ...LAYOUT_PROPERTIES.box.slice(0, 2) // boxSizing, overflow only (skip overflowX/Y, border)
+];
+
+// Criteria for critical elements (no sticky - avoid scroll context side effects)
+export const CRITICAL_DISPLAY = ['flex', 'inline-flex', 'grid', 'inline-grid'];
+export const CRITICAL_POSITION = ['absolute', 'fixed'];
+
 /**
  * Extract and clean HTML from page
  * @param {Page} page - Puppeteer page
@@ -23,7 +39,7 @@ export const JS_FRAMEWORK_PATTERNS = [
  * @returns {Promise<{html: string, warnings: string[], elementCount: number}>}
  */
 export async function extractCleanHtml(page, frameworkPatterns = JS_FRAMEWORK_PATTERNS) {
-  return await page.evaluate((patterns) => {
+  return await page.evaluate((patterns, inlineProps, criticalDisplay, criticalPosition) => {
     const warnings = [];
 
     // Check DOM size
@@ -72,6 +88,58 @@ export async function extractCleanHtml(page, frameworkPatterns = JS_FRAMEWORK_PA
       });
     });
 
+    // Inline computed styles on critical elements (flex/grid/positioned)
+    // Using index-based matching for reliability
+    const inlineStyles = [];
+    let inlinedCount = 0;
+
+    document.querySelectorAll('*').forEach((liveEl, idx) => {
+      const style = getComputedStyle(liveEl);
+      const display = style.display;
+      const position = style.position;
+
+      // Only critical elements (flex/grid containers, absolute/fixed positioned)
+      if (criticalDisplay.includes(display) || criticalPosition.includes(position)) {
+        const props = [];
+        inlineProps.forEach(prop => {
+          const val = style[prop];
+          // Skip defaults/empty values
+          if (val && val !== 'auto' && val !== 'none' && val !== 'normal' &&
+              val !== '0px' && val !== 'static' && val !== 'visible' &&
+              val !== 'content-box') {
+            // Convert camelCase to kebab-case
+            const cssProp = prop.replace(/([A-Z])/g, '-$1').toLowerCase();
+            props.push(`${cssProp}: ${val}`);
+          }
+        });
+
+        // Always include display for critical elements
+        if (!props.some(p => p.startsWith('display:'))) {
+          props.unshift(`display: ${display}`);
+        }
+
+        if (props.length > 0) {
+          inlineStyles.push({ idx, style: props.join('; ') });
+        }
+      }
+    });
+
+    // Apply to cloned doc using index matching
+    const clonedElements = doc.querySelectorAll('*');
+    inlineStyles.forEach(({ idx, style }) => {
+      if (clonedElements[idx]) {
+        const existing = clonedElements[idx].getAttribute('style') || '';
+        clonedElements[idx].setAttribute('style',
+          existing ? `${existing}; ${style}` : style);
+        inlinedCount++;
+      }
+    });
+
+    // Track for warnings
+    if (inlinedCount > 100) {
+      warnings.push(`Inlined ${inlinedCount} critical elements`);
+    }
+
     // Remove hidden elements
     doc.querySelectorAll('[hidden], [style*="display: none"], [style*="display:none"]')
        .forEach(el => el.remove());
@@ -97,6 +165,7 @@ export async function extractCleanHtml(page, frameworkPatterns = JS_FRAMEWORK_PA
                  (document.documentElement.lang || 'en') + '">\n' +
                  doc.innerHTML + '\n</html>';
 
-    return { html, warnings, elementCount };
-  }, frameworkPatterns.map(r => ({ source: r.source, flags: r.flags })));
+    return { html, warnings, elementCount, inlinedCount };
+  }, frameworkPatterns.map(r => ({ source: r.source, flags: r.flags })),
+     INLINE_LAYOUT_PROPS, CRITICAL_DISPLAY, CRITICAL_POSITION);
 }