@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/branding.js

@@ -0,0 +1,234 @@
+/**
+ * Branding and design system extraction from web pages
+ * Extracts colors, fonts, typography, spacing, components, and CSS variables
+ */
+/**
+ * Extract branding and design system from a webpage
+ * This must run inside a Playwright browser context to access computed styles
+ *
+ * @param page - Playwright Page object
+ * @returns Complete branding profile
+ *
+ * @example
+ * ```typescript
+ * const browser = await chromium.launch();
+ * const page = await browser.newPage();
+ * await page.goto('https://example.com');
+ * const branding = await extractBranding(page);
+ * console.log(branding.colors.primary);
+ * ```
+ */
+export async function extractBranding(page) {
+    try {
+        // Run extraction in browser context to access computed styles
+        const extracted = await page.evaluate(() => {
+            const result = {
+                colorScheme: 'light',
+                colors: {},
+                fonts: [],
+                typography: {
+                    fontFamilies: {},
+                    fontSizes: {},
+                    fontWeights: {},
+                    lineHeights: {},
+                },
+                spacing: {},
+                components: {},
+                cssVariables: {},
+            };
+            // Helper to parse RGB/RGBA to hex
+            function rgbToHex(rgb) {
+                const match = rgb.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*[\d.]+)?\)/);
+                if (!match)
+                    return rgb;
+                const [, r, g, b] = match;
+                return '#' + [r, g, b].map(x => parseInt(x).toString(16).padStart(2, '0')).join('');
+            }
+            // Extract all CSS variables from :root
+            const rootStyles = getComputedStyle(document.documentElement);
+            for (let i = 0; i < rootStyles.length; i++) {
+                const prop = rootStyles[i];
+                if (prop.startsWith('--')) {
+                    const value = rootStyles.getPropertyValue(prop).trim();
+                    result.cssVariables[prop] = value;
+                    // Detect color variables
+                    if (value.match(/^#[0-9a-f]{3,8}$/i) || value.match(/^rgba?\(/i) || value.match(/^hsla?\(/i)) {
+                        const colorKey = prop.replace(/^--/, '').replace(/-/g, '_');
+                        result.colors[colorKey] = value.startsWith('rgb') ? rgbToHex(value) : value;
+                    }
+                }
+            }
+            // Detect color scheme (light/dark)
+            const bgColor = rootStyles.backgroundColor || getComputedStyle(document.body).backgroundColor;
+            if (bgColor) {
+                const rgb = bgColor.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)/);
+                if (rgb) {
+                    const [, r, g, b] = rgb.map(Number);
+                    const brightness = (r * 299 + g * 587 + b * 114) / 1000;
+                    result.colorScheme = brightness < 128 ? 'dark' : 'light';
+                    result.colors.background = rgbToHex(bgColor);
+                }
+            }
+            // Extract text colors
+            const bodyStyles = getComputedStyle(document.body);
+            result.colors.textPrimary = rgbToHex(bodyStyles.color);
+            // Find headings for secondary text color
+            const heading = document.querySelector('h1, h2, h3, h4, h5, h6');
+            if (heading) {
+                const headingColor = getComputedStyle(heading).color;
+                if (headingColor !== bodyStyles.color) {
+                    result.colors.textSecondary = rgbToHex(headingColor);
+                }
+            }
+            // Try to detect primary/accent colors from buttons, links, etc.
+            const button = document.querySelector('button, .btn, [role="button"], a.button');
+            if (button) {
+                const btnStyles = getComputedStyle(button);
+                const btnBg = btnStyles.backgroundColor;
+                if (btnBg && !btnBg.includes('rgba(0, 0, 0, 0)')) {
+                    result.colors.primary = rgbToHex(btnBg);
+                }
+            }
+            const link = document.querySelector('a');
+            if (link) {
+                const linkColor = getComputedStyle(link).color;
+                if (!result.colors.primary && linkColor !== bodyStyles.color) {
+                    result.colors.accent = rgbToHex(linkColor);
+                }
+            }
+            // Extract fonts
+            const fontFamiliesSet = new Set();
+            const fontElements = [document.body, ...Array.from(document.querySelectorAll('h1, h2, h3, h4, h5, h6, p, button, input'))];
+            for (const el of fontElements) {
+                if (el) {
+                    const styles = getComputedStyle(el);
+                    const family = styles.fontFamily.split(',')[0].replace(/['"]/g, '').trim();
+                    fontFamiliesSet.add(family);
+                    const tagName = el.tagName.toLowerCase();
+                    result.typography.fontFamilies[tagName] = family;
+                    result.typography.fontSizes[tagName] = styles.fontSize;
+                    result.typography.fontWeights[tagName] = parseInt(styles.fontWeight) || 400;
+                    result.typography.lineHeights[tagName] = styles.lineHeight;
+                }
+            }
+            // Build fonts array with sources
+            fontFamiliesSet.forEach(family => {
+                const fontObj = { family };
+                // Detect Google Fonts
+                const links = document.querySelectorAll('link[href*="fonts.googleapis.com"]');
+                for (const link of links) {
+                    const href = link.getAttribute('href') || '';
+                    if (href.includes(family.replace(/\s+/g, '+'))) {
+                        fontObj.source = 'Google Fonts';
+                        // Extract weights from URL
+                        const weightMatch = href.match(/wght@([0-9;]+)/);
+                        if (weightMatch) {
+                            fontObj.weights = weightMatch[1].split(';').map(Number);
+                        }
+                        break;
+                    }
+                }
+                if (!fontObj.source) {
+                    fontObj.source = 'system';
+                }
+                result.fonts.push(fontObj);
+            });
+            // Extract spacing values
+            const container = document.querySelector('main, .container, .wrapper, #content, [class*="container"]');
+            if (container) {
+                const containerStyles = getComputedStyle(container);
+                result.spacing.containerMaxWidth = containerStyles.maxWidth;
+                result.spacing.borderRadius = containerStyles.borderRadius;
+                // Try to detect base spacing unit
+                const padding = containerStyles.padding;
+                const paddingMatch = padding.match(/(\d+)px/);
+                if (paddingMatch) {
+                    const px = parseInt(paddingMatch[1]);
+                    // Common spacing systems use multiples of 4 or 8
+                    if (px % 8 === 0)
+                        result.spacing.baseUnit = 8;
+                    else if (px % 4 === 0)
+                        result.spacing.baseUnit = 4;
+                }
+            }
+            // Extract common component patterns
+            const componentSelectors = {
+                button: 'button, .btn, [role="button"]',
+                input: 'input[type="text"], input[type="email"], textarea',
+                card: '.card, [class*="card"]',
+                nav: 'nav, .nav, .navigation',
+                header: 'header, .header',
+                footer: 'footer, .footer',
+            };
+            for (const [name, selector] of Object.entries(componentSelectors)) {
+                const el = document.querySelector(selector);
+                if (el) {
+                    const styles = getComputedStyle(el);
+                    result.components[name] = {
+                        backgroundColor: styles.backgroundColor.includes('rgba(0, 0, 0, 0)') ? 'transparent' : rgbToHex(styles.backgroundColor),
+                        color: rgbToHex(styles.color),
+                        borderRadius: styles.borderRadius,
+                        padding: styles.padding,
+                        fontSize: styles.fontSize,
+                        fontWeight: styles.fontWeight,
+                    };
+                }
+            }
+            return result;
+        });
+        // Extract logo and favicon using Playwright
+        const logo = await page.evaluate(() => {
+            const logoSelectors = [
+                'img[alt*="logo" i]',
+                'img[class*="logo" i]',
+                'img[id*="logo" i]',
+                'a.logo img',
+                'a[class*="logo"] img',
+                'header img',
+                '.header img',
+                'nav img:first-of-type',
+            ];
+            for (const selector of logoSelectors) {
+                const img = document.querySelector(selector);
+                if (img?.src)
+                    return img.src;
+            }
+            return undefined;
+        });
+        const favicon = await page.evaluate(() => {
+            const faviconSelectors = [
+                'link[rel="icon"]',
+                'link[rel="shortcut icon"]',
+                'link[rel="apple-touch-icon"]',
+            ];
+            for (const selector of faviconSelectors) {
+                const link = document.querySelector(selector);
+                if (link?.href)
+                    return link.href;
+            }
+            return undefined;
+        });
+        return {
+            ...extracted,
+            logo,
+            favicon,
+        };
+    }
+    catch (error) {
+        // Return minimal branding profile on error
+        console.error('Branding extraction failed:', error);
+        return {
+            colorScheme: 'light',
+            colors: {},
+            fonts: [],
+            typography: {
+                fontFamilies: {},
+                fontSizes: {},
+                fontWeights: {},
+            },
+            spacing: {},
+            components: {},
+            cssVariables: {},
+        };
+    }
+}

package/dist/core/browser-fetch.d.ts

@@ -0,0 +1,323 @@
+/**
+ * Browser-based fetching — uses Playwright via the browser pool.
+ * Handles browserFetch, browserScreenshot, retryFetch, and scrollAndWait.
+ */
+import type { Page } from 'playwright';
+import type { PageAction } from '../types.js';
+import { type FetchResult } from './http-fetch.js';
+/**
+ * Fetch using headless Chromium via Playwright
+ * Slower but can handle JavaScript-heavy sites and bypass some bot detection
+ */
+export declare function browserFetch(url: string, options?: {
+    userAgent?: string;
+    waitMs?: number;
+    timeoutMs?: number;
+    screenshot?: boolean;
+    screenshotFullPage?: boolean;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+    actions?: PageAction[];
+    /** Keep the browser page open after fetch (caller must close page + browser) */
+    keepPageOpen?: boolean;
+    /** Abort signal for internal races/cancellation */
+    signal?: AbortSignal;
+    /**
+     * Path to a persistent Chrome user-data-dir.
+     * When set, bypasses the shared browser pool and keeps the browser alive
+     * between fetches so cookies / session data persist.
+     */
+    profileDir?: string;
+    /** Launch browser in headed (visible) mode. Only meaningful with profileDir or for debugging. */
+    headed?: boolean;
+    /**
+     * Playwright storage state (cookies + localStorage) to inject into the browser context.
+     * When provided, a new BrowserContext is created with this state, which is more reliable
+     * than --user-data-dir for session injection.
+     */
+    storageState?: any;
+    /**
+     * Proxy URL for routing browser requests through a proxy server.
+     * Supports HTTP, HTTPS, and SOCKS5 proxies.
+     * Format: protocol://[user:pass@]host:port
+     * Examples: 'http://proxy.example.com:8080', 'socks5://user:pass@host:1080'
+     */
+    proxy?: string;
+    /** Device emulation: 'desktop' (default), 'mobile', 'tablet' */
+    device?: 'desktop' | 'mobile' | 'tablet';
+    /** Browser viewport width in pixels */
+    viewportWidth?: number;
+    /** Browser viewport height in pixels */
+    viewportHeight?: number;
+    /** Device scale factor (pixel density) for screenshots. Defaults to device profile value. */
+    deviceScaleFactor?: number;
+    /** Wait condition: 'domcontentloaded' (default), 'networkidle', 'load', 'commit' */
+    waitUntil?: string;
+    /** CSS selector to wait for before extracting content */
+    waitSelector?: string;
+    /** Block resource types for faster loading: 'image', 'stylesheet', 'font', 'media', 'script' */
+    blockResources?: string[];
+    /**
+     * Whether the target is a Single-Page Application (Kayak, Google Flights, Expedia, etc).
+     * When true, the DOM stability check uses a longer timeout (12s) to wait for async data loads.
+     * When false (default), a shorter 3s stability window is used.
+     */
+    isSPA?: boolean;
+    /** Language preferences for browser navigator.languages and locale (e.g. ['ja', 'en']) */
+    languages?: string[];
+}): Promise<FetchResult>;
+/**
+ * Capture a screenshot of a URL using headless Chromium via Playwright.
+ */
+export declare function browserScreenshot(url: string, options?: {
+    fullPage?: boolean;
+    width?: number;
+    height?: number;
+    format?: 'png' | 'jpeg';
+    quality?: number;
+    waitMs?: number;
+    timeoutMs?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+    actions?: Array<{
+        type: 'wait' | 'click' | 'scroll' | 'type' | 'fill' | 'select' | 'press' | 'hover' | 'waitForSelector' | 'screenshot';
+        selector?: string;
+        value?: string;
+        key?: string;
+        ms?: number;
+        to?: 'top' | 'bottom' | number | {
+            x: number;
+            y: number;
+        };
+        timeout?: number;
+    }>;
+    scrollThrough?: boolean;
+    selector?: string;
+}): Promise<{
+    buffer: Buffer;
+    finalUrl: string;
+}>;
+/**
+ * Capture screenshots of two URLs and compute a pixel-level visual diff.
+ */
+export declare function browserDiff(url1: string, url2: string, options?: {
+    width?: number;
+    height?: number;
+    fullPage?: boolean;
+    threshold?: number;
+    format?: 'png' | 'jpeg';
+    quality?: number;
+    stealth?: boolean;
+    waitMs?: number;
+    timeoutMs?: number;
+}): Promise<{
+    diffBuffer: Buffer;
+    diffPixels: number;
+    totalPixels: number;
+    diffPercent: number;
+    dimensions: {
+        width: number;
+        height: number;
+    };
+}>;
+/**
+ * Retry a fetch operation with exponential backoff
+ */
+export declare function retryFetch<T>(fn: () => Promise<T>, maxAttempts?: number, baseDelayMs?: number): Promise<T>;
+/**
+ * Scroll to the bottom of the page N times, waiting for the network to
+ * settle between each scroll.  Useful for triggering lazy-loaded content
+ * (infinite scroll, deferred images, etc.).
+ *
+ * @param page   - Playwright Page instance.
+ * @param times  - Number of scroll-and-wait cycles (default: 3).
+ * @returns        The final page HTML after all scrolls complete.
+ */
+export declare function scrollAndWait(page: Page, times?: number): Promise<string>;
+/**
+ * Capture multiple screenshots at evenly distributed scroll positions.
+ * Returns an array of Buffers (one per frame).
+ */
+export declare function browserFilmstrip(url: string, options?: {
+    frames?: number;
+    width?: number;
+    height?: number;
+    format?: 'png' | 'jpeg';
+    quality?: number;
+    waitMs?: number;
+    timeoutMs?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+}): Promise<{
+    frames: Buffer[];
+    finalUrl: string;
+}>;
+/**
+ * Section-aware audit screenshots.
+ * Finds all elements matching a CSS selector and captures a viewport screenshot
+ * scrolled to each one.  Returns one image buffer per matching element.
+ */
+export declare function browserAudit(url: string, options?: {
+    width?: number;
+    height?: number;
+    format?: 'png' | 'jpeg';
+    quality?: number;
+    selector?: string;
+    waitMs?: number;
+    timeoutMs?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+    scrollThrough?: boolean;
+}): Promise<{
+    frames: {
+        index: number;
+        tag: string;
+        id: string;
+        className: string;
+        top: number;
+        height: number;
+        buffer: Buffer;
+    }[];
+    finalUrl: string;
+}>;
+/**
+ * Capture N viewport screenshots at fixed intervals to record CSS animation states.
+ */
+export declare function browserAnimationCapture(url: string, options?: {
+    frames?: number;
+    intervalMs?: number;
+    scrollTo?: number;
+    selector?: string;
+    width?: number;
+    height?: number;
+    format?: 'png' | 'jpeg';
+    quality?: number;
+    waitMs?: number;
+    timeoutMs?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+}): Promise<{
+    frames: {
+        index: number;
+        timestampMs: number;
+        buffer: Buffer;
+    }[];
+    finalUrl: string;
+}>;
+/**
+ * Capture screenshots at multiple viewport widths in a single browser session.
+ * Resizes the viewport between each capture.
+ */
+export declare function browserViewports(url: string, options: {
+    viewports: {
+        width: number;
+        height: number;
+        label?: string;
+    }[];
+    fullPage?: boolean;
+    format?: 'png' | 'jpeg';
+    quality?: number;
+    waitMs?: number;
+    timeoutMs?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+    scrollThrough?: boolean;
+}): Promise<{
+    frames: {
+        width: number;
+        height: number;
+        label: string;
+        buffer: Buffer;
+    }[];
+    finalUrl: string;
+}>;
+export interface DesignAuditResult {
+    score: number;
+    colorScheme: 'light' | 'dark' | 'unknown';
+    spacingViolations: {
+        element: string;
+        property: string;
+        value: number;
+        nearestGridValue: number;
+    }[];
+    touchTargetViolations: {
+        element: string;
+        width: number;
+        height: number;
+        minRequired: number;
+    }[];
+    contrastViolations: {
+        element: string;
+        textColor: string;
+        bgColor: string;
+        ratio: number;
+        required: number;
+        bgResolved?: boolean;
+    }[];
+    typography: {
+        fontSizes: string[];
+        lineHeights: string[];
+        letterSpacings: string[];
+    };
+    spacingScale: number[];
+    accessibilityViolations: {
+        type: 'missing-alt' | 'missing-label' | 'missing-aria' | 'heading-skip' | 'empty-link' | 'empty-button';
+        element: string;
+        details: string;
+    }[];
+    headingStructure: string[];
+    summary: string;
+}
+/**
+ * Extract computed CSS values and validate against design rules.
+ * Returns structured JSON instead of pixel images.
+ */
+export declare function browserDesignAudit(url: string, options?: {
+    rules?: {
+        spacingGrid?: number;
+        minTouchTarget?: number;
+        minContrast?: number;
+    };
+    selector?: string;
+    width?: number;
+    height?: number;
+    waitMs?: number;
+    timeoutMs?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+}): Promise<{
+    audit: DesignAuditResult;
+    finalUrl: string;
+}>;
+/**
+ * Extract structured visual design intelligence from a URL using a browser.
+ * Returns a DesignAnalysis object with effects, palette, layout, type scale,
+ * and quality signals.
+ */
+export declare function browserDesignAnalysis(url: string, options?: {
+    selector?: string;
+    width?: number;
+    height?: number;
+    waitMs?: number;
+    timeoutMs?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    cookies?: string[];
+    stealth?: boolean;
+}): Promise<{
+    analysis: import('./design-analysis.js').DesignAnalysis;
+    finalUrl: string;
+}>;