@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/fetch-cache.js

@@ -0,0 +1,95 @@
+/**
+ * In-memory LRU fetch cache for WebPeel
+ *
+ * Caches pipeline results to avoid redundant fetches for identical requests.
+ * Supports TTL-based expiry and LRU eviction when maxEntries is exceeded.
+ * Exported as a singleton: import { fetchCache } from './fetch-cache.js'
+ */
+export class FetchCache {
+    cache;
+    maxEntries;
+    defaultTTL; // ms
+    hits;
+    misses;
+    constructor(maxEntries = 500, defaultTTLSeconds = 300) {
+        this.cache = new Map();
+        this.maxEntries = maxEntries;
+        this.defaultTTL = defaultTTLSeconds * 1000;
+        this.hits = 0;
+        this.misses = 0;
+    }
+    /**
+     * Generate a stable cache key from url + relevant fetch options.
+     * Different option combinations produce different cache entries.
+     */
+    getKey(url, options = {}) {
+        const render = options.render ? '1' : '0';
+        const stealth = options.stealth ? '1' : '0';
+        const budget = options.budget !== undefined ? String(options.budget) : '';
+        return `${url}|r:${render}|s:${stealth}|b:${budget}`;
+    }
+    /**
+     * Retrieve a cached entry. Returns null if missing or expired.
+     * On hit: entry is moved to the end of the Map (LRU refresh).
+     */
+    get(key) {
+        const entry = this.cache.get(key);
+        if (!entry) {
+            this.misses++;
+            return null;
+        }
+        const ageMs = Date.now() - entry.timestamp;
+        if (ageMs > this.defaultTTL) {
+            // Expired — evict and return null
+            this.cache.delete(key);
+            this.misses++;
+            return null;
+        }
+        // LRU touch: move to end
+        this.cache.delete(key);
+        this.cache.set(key, entry);
+        this.hits++;
+        return entry;
+    }
+    /**
+     * Store an entry in the cache.
+     * If the cache is at capacity, the least recently used entry is evicted.
+     */
+    set(key, entry) {
+        // Remove existing to refresh position
+        if (this.cache.has(key)) {
+            this.cache.delete(key);
+        }
+        this.cache.set(key, entry);
+        // LRU eviction: remove oldest entry (first in Map iteration order)
+        while (this.cache.size > this.maxEntries) {
+            const oldestKey = this.cache.keys().next().value;
+            if (oldestKey !== undefined) {
+                this.cache.delete(oldestKey);
+            }
+            else {
+                break;
+            }
+        }
+    }
+    /** Clear all entries and reset stats. */
+    clear() {
+        this.cache.clear();
+        this.hits = 0;
+        this.misses = 0;
+    }
+    /** Return cache stats. hitRate is in [0, 1]. */
+    stats() {
+        const total = this.hits + this.misses;
+        return {
+            size: this.cache.size,
+            hits: this.hits,
+            misses: this.misses,
+            hitRate: total === 0 ? 0 : Math.round((this.hits / total) * 100) / 100,
+        };
+    }
+}
+/** Singleton fetch cache — shared across all requests (5 min TTL, 500 entries). */
+export const fetchCache = new FetchCache(500, 300);
+/** Singleton search cache — shorter TTL since results change faster (60 s). */
+export const searchCache = new FetchCache(500, 60);

package/dist/core/fetcher.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Core fetching — thin re-export layer for backward compatibility.
+ *
+ * The implementation has been split into focused modules:
+ *   - http-fetch.ts  — Pure HTTP fetching (simpleFetch, SSRF validation, HTTP pool)
+ *   - browser-pool.ts — Browser lifecycle & page pool (getBrowser, cleanup, warmup)
+ *   - browser-fetch.ts — Browser-based fetching (browserFetch, browserScreenshot)
+ */
+export { simpleFetch, type FetchResult } from './http-fetch.js';
+export { cleanup, warmup, closePool, closeProfileBrowser, playwrightLoaded } from './browser-pool.js';
+export { browserFetch, browserScreenshot, browserFilmstrip, browserAudit, browserAnimationCapture, browserViewports, browserDesignAudit, browserDiff, retryFetch, scrollAndWait, browserDesignAnalysis } from './browser-fetch.js';
+export type { DesignAuditResult } from './browser-fetch.js';
+export type { DesignAnalysis, EffectInstance } from './design-analysis.js';

package/dist/core/fetcher.js

@@ -0,0 +1,12 @@
+/**
+ * Core fetching — thin re-export layer for backward compatibility.
+ *
+ * The implementation has been split into focused modules:
+ *   - http-fetch.ts  — Pure HTTP fetching (simpleFetch, SSRF validation, HTTP pool)
+ *   - browser-pool.ts — Browser lifecycle & page pool (getBrowser, cleanup, warmup)
+ *   - browser-fetch.ts — Browser-based fetching (browserFetch, browserScreenshot)
+ */
+// Re-export everything for backward compatibility
+export { simpleFetch } from './http-fetch.js';
+export { cleanup, warmup, closePool, closeProfileBrowser, playwrightLoaded } from './browser-pool.js';
+export { browserFetch, browserScreenshot, browserFilmstrip, browserAudit, browserAnimationCapture, browserViewports, browserDesignAudit, browserDiff, retryFetch, scrollAndWait, browserDesignAnalysis } from './browser-fetch.js';

package/dist/core/google-cache.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Google Cache fallback fetcher.
+ *
+ * When a site blocks direct access (Akamai, PerimeterX, Cloudflare, etc.),
+ * Google's cache at webcache.googleusercontent.com often has a clean copy
+ * that's freely accessible without anti-bot protection.
+ */
+export interface GoogleCacheResult {
+    html: string;
+    url: string;
+    cachedDate?: string;
+    statusCode: number;
+    method: 'google-cache';
+}
+/**
+ * Fetch a cached copy of a URL from Google Cache.
+ *
+ * Returns null if:
+ * - Google returns a 404 (page not in cache)
+ * - Google redirects to the live page (cache unavailable)
+ * - The response looks like a Google search page rather than a cache
+ */
+export declare function fetchGoogleCache(url: string, options?: {
+    timeout?: number;
+}): Promise<GoogleCacheResult | null>;
+/**
+ * Google Cache is always available — no API key or special setup required.
+ */
+export declare function isGoogleCacheAvailable(): boolean;

package/dist/core/google-cache.js

@@ -0,0 +1,180 @@
+/**
+ * Google Cache fallback fetcher.
+ *
+ * When a site blocks direct access (Akamai, PerimeterX, Cloudflare, etc.),
+ * Google's cache at webcache.googleusercontent.com often has a clean copy
+ * that's freely accessible without anti-bot protection.
+ */
+/**
+ * Fetch a cached copy of a URL from Google Cache.
+ *
+ * Returns null if:
+ * - Google returns a 404 (page not in cache)
+ * - Google redirects to the live page (cache unavailable)
+ * - The response looks like a Google search page rather than a cache
+ */
+export async function fetchGoogleCache(url, options) {
+    const timeout = options?.timeout ?? 10000;
+    // Build the Google Cache URL
+    const cacheUrl = `https://webcache.googleusercontent.com/search?q=cache:${encodeURIComponent(url)}`;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+    try {
+        const response = await fetch(cacheUrl, {
+            signal: controller.signal,
+            redirect: 'follow',
+            headers: {
+                // Must look like a real Chrome browser or Google blocks us
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
+                Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+                'Accept-Language': 'en-US,en;q=0.9',
+                'Accept-Encoding': 'gzip, deflate, br',
+                'Cache-Control': 'no-cache',
+                Pragma: 'no-cache',
+                'Sec-Fetch-Dest': 'document',
+                'Sec-Fetch-Mode': 'navigate',
+                'Sec-Fetch-Site': 'none',
+                'Upgrade-Insecure-Requests': '1',
+            },
+        });
+        clearTimeout(timer);
+        // 404 → page not cached
+        if (response.status === 404) {
+            return null;
+        }
+        // If redirected to the live site (Google doesn't have a cache), return null
+        const finalUrl = response.url;
+        if (!finalUrl.includes('webcache.googleusercontent.com') &&
+            !finalUrl.includes('google.com/search')) {
+            // Redirected away from Google cache — cache unavailable
+            return null;
+        }
+        const html = await response.text();
+        // If this looks like a Google search results page (not a cache page), return null
+        if (isGoogleSearchPage(html)) {
+            return null;
+        }
+        // If the page is way too small to be real content, return null
+        if (html.length < 200) {
+            return null;
+        }
+        // Extract the cache date from Google's notice banner
+        const cachedDate = extractCacheDate(html);
+        // Remove Google's wrapper elements and return the cleaned HTML
+        const cleanedHtml = removeGoogleWrapper(html);
+        return {
+            html: cleanedHtml,
+            url,
+            cachedDate,
+            statusCode: 200,
+            method: 'google-cache',
+        };
+    }
+    catch (error) {
+        clearTimeout(timer);
+        if (error instanceof Error && error.name === 'AbortError') {
+            // Timeout
+            return null;
+        }
+        // Network errors → return null (not in cache / unavailable)
+        return null;
+    }
+}
+/**
+ * Detect if a page is a Google search results page rather than a cached page.
+ */
+function isGoogleSearchPage(html) {
+    // Google search pages have these distinctive patterns
+    if (html.includes('<title>Google Search</title>'))
+        return true;
+    if (html.includes('id="search"') && html.includes('class="g"'))
+        return true;
+    // "Did not match any documents" message
+    if (html.includes('did not match any documents'))
+        return true;
+    // Redirect to google.com/search with no cache content
+    if (html.includes('www.google.com/search?') && !html.includes('webcache'))
+        return true;
+    return false;
+}
+/**
+ * Extract the cache date from Google's cache notice banner.
+ *
+ * The notice typically reads:
+ * "It is a snapshot of the page as it appeared on 15 Jan 2025 09:30:12 GMT."
+ */
+function extractCacheDate(html) {
+    // Match date patterns in Google's cache notice
+    // Patterns like: "15 Jan 2025 09:30:12 GMT" or "Jan 15, 2025"
+    const patterns = [
+        /as it appeared on ([^<."]+(?:GMT|UTC))/i,
+        /snapshot.*?on\s+([A-Za-z]+ \d+,?\s+\d{4}[^<."]*)/i,
+        /cached on:?\s*([^<."]+)/i,
+    ];
+    for (const pattern of patterns) {
+        const match = html.match(pattern);
+        if (match && match[1]) {
+            return match[1].trim();
+        }
+    }
+    return undefined;
+}
+/**
+ * Remove Google's wrapper elements from the cached page.
+ *
+ * Google's cache page structure:
+ * 1. A <div style="..."> at the very top with the cache notice
+ * 2. An <hr> separator
+ * 3. The actual cached page content
+ *
+ * We remove Google's branding/notice and return just the original content.
+ */
+function removeGoogleWrapper(html) {
+    let cleaned = html;
+    // Remove the Google cache notice div at the top
+    // It's typically: <div style="...">...</div><hr>
+    // Strategy 1: Find the first <hr> that follows the cache notice and take everything after it
+    const hrIndex = findFirstCacheHr(html);
+    if (hrIndex !== -1) {
+        cleaned = html.slice(hrIndex + 4); // +4 for '<hr>'
+    }
+    // Remove remaining Google-specific elements that might be injected
+    // Google injects a top bar div with id="google-cache-hdr" or similar
+    cleaned = cleaned
+        .replace(/<div[^>]*id=["']google-cache-hdr["'][^>]*>[\s\S]*?<\/div>/i, '')
+        .replace(/<div[^>]*id=["']gbw["'][^>]*>[\s\S]*?<\/div>/i, '')
+        .replace(/<div[^>]*id=["']gb["'][^>]*>[\s\S]*?<\/div>/gi, '');
+    return cleaned.trim();
+}
+/**
+ * Find the index of the <hr> tag that separates Google's cache notice
+ * from the actual page content.
+ *
+ * We look for the first <hr> that appears after Google's cache notice text.
+ */
+function findFirstCacheHr(html) {
+    // Look for the cache notice keywords to confirm we're in a Google cache page
+    const noticeKeywords = [
+        'webcache.googleusercontent',
+        "Google's cache of",
+        'cached version of',
+        'It is a snapshot',
+    ];
+    const hasNotice = noticeKeywords.some((kw) => html.toLowerCase().includes(kw.toLowerCase()));
+    if (!hasNotice) {
+        // Not a standard Google cache page — don't strip anything
+        return -1;
+    }
+    // Find the first <hr>, <hr/>, or <hr /> tag — that's the separator
+    const hrMatch = html.match(/<hr\s*\/?>/i);
+    if (hrMatch && hrMatch.index !== undefined) {
+        return hrMatch.index;
+    }
+    return -1;
+}
+/**
+ * Google Cache is always available — no API key or special setup required.
+ */
+export function isGoogleCacheAvailable() {
+    return true;
+}

package/dist/core/google-serp-parser.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Google SERP Parser — extracts rich structured data from Google search HTML.
+ * Supports organic results, knowledge panel, PAA, featured snippets,
+ * related searches, shopping, news, images, videos, and local pack.
+ */
+export interface GoogleSerpResult {
+    organicResults: Array<{
+        position: number;
+        title: string;
+        url: string;
+        snippet: string;
+        sitelinks?: Array<{
+            title: string;
+            url: string;
+        }>;
+        date?: string;
+        cachedUrl?: string;
+    }>;
+    knowledgePanel?: {
+        title: string;
+        type?: string;
+        description?: string;
+        source?: string;
+        sourceUrl?: string;
+        attributes?: Record<string, string>;
+        imageUrl?: string;
+    };
+    peopleAlsoAsk?: Array<{
+        question: string;
+        snippet?: string;
+        source?: string;
+        sourceUrl?: string;
+    }>;
+    featuredSnippet?: {
+        text: string;
+        source: string;
+        sourceUrl: string;
+        type: 'paragraph' | 'list' | 'table';
+    };
+    relatedSearches?: string[];
+    shoppingResults?: Array<{
+        title: string;
+        price?: string;
+        source?: string;
+        url?: string;
+        imageUrl?: string;
+        rating?: number;
+        reviewCount?: number;
+    }>;
+    newsResults?: Array<{
+        title: string;
+        url: string;
+        source: string;
+        date?: string;
+        snippet?: string;
+        imageUrl?: string;
+    }>;
+    imagePack?: Array<{
+        url: string;
+        imageUrl: string;
+        title?: string;
+    }>;
+    videoResults?: Array<{
+        title: string;
+        url: string;
+        platform?: string;
+        duration?: string;
+        date?: string;
+        thumbnailUrl?: string;
+    }>;
+    localPack?: Array<{
+        name: string;
+        address?: string;
+        rating?: number;
+        reviewCount?: number;
+        type?: string;
+        phone?: string;
+    }>;
+    totalResults?: string;
+    searchTime?: string;
+}
+export declare function parseGoogleSerp(html: string): GoogleSerpResult;