@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/cache.js

@@ -0,0 +1,121 @@
+/**
+ * In-memory LRU response cache.
+ */
+const MAX_ENTRIES = 1000;
+const DEFAULT_TTL_MS = 5 * 60 * 1000; // 5 minutes
+const STALE_WHILE_REVALIDATE_MS = 10 * 60 * 1000; // 10 minutes
+const REVALIDATION_TIMEOUT_MS = 30 * 1000; // 30 seconds — reset revalidating flag if fetch hangs
+let cacheTTL = DEFAULT_TTL_MS;
+const responseCache = new Map();
+function normalizeUrl(url) {
+    try {
+        const normalized = new URL(url);
+        normalized.hash = '';
+        normalized.hostname = normalized.hostname.toLowerCase();
+        if ((normalized.protocol === 'http:' && normalized.port === '80') ||
+            (normalized.protocol === 'https:' && normalized.port === '443')) {
+            normalized.port = '';
+        }
+        if (!normalized.pathname) {
+            normalized.pathname = '/';
+        }
+        const sortedParams = [...normalized.searchParams.entries()]
+            .sort(([a], [b]) => a.localeCompare(b));
+        normalized.search = '';
+        for (const [key, value] of sortedParams) {
+            normalized.searchParams.append(key, value);
+        }
+        return normalized.toString();
+    }
+    catch {
+        return url.trim();
+    }
+}
+function getCacheEntry(key) {
+    const entry = responseCache.get(key);
+    if (!entry) {
+        return null;
+    }
+    const ageMs = Date.now() - entry.timestamp;
+    const maxAgeMs = cacheTTL + STALE_WHILE_REVALIDATE_MS;
+    if (ageMs > maxAgeMs) {
+        responseCache.delete(key);
+        return null;
+    }
+    const stale = ageMs > cacheTTL;
+    // LRU touch: move to the end when read.
+    responseCache.delete(key);
+    responseCache.set(key, entry);
+    return {
+        value: entry.result,
+        stale,
+    };
+}
+function setCacheEntry(key, result) {
+    if (responseCache.has(key)) {
+        responseCache.delete(key);
+    }
+    responseCache.set(key, {
+        result,
+        timestamp: Date.now(),
+    });
+    while (responseCache.size > MAX_ENTRIES) {
+        const oldestKey = responseCache.keys().next().value;
+        if (!oldestKey) {
+            break;
+        }
+        responseCache.delete(oldestKey);
+    }
+}
+export function getCached(url) {
+    const entry = getCacheEntry(normalizeUrl(url));
+    if (!entry || entry.stale) {
+        return null;
+    }
+    return entry.value;
+}
+export function getCachedWithSWR(url) {
+    return getCacheEntry(normalizeUrl(url));
+}
+export function markRevalidating(url) {
+    const key = normalizeUrl(url);
+    const entry = responseCache.get(key);
+    if (!entry) {
+        return false;
+    }
+    const ageMs = Date.now() - entry.timestamp;
+    const maxAgeMs = cacheTTL + STALE_WHILE_REVALIDATE_MS;
+    if (ageMs > maxAgeMs) {
+        responseCache.delete(key);
+        return false;
+    }
+    const stale = ageMs > cacheTTL;
+    if (!stale) {
+        return false;
+    }
+    // If already revalidating, check if the attempt has timed out
+    if (entry.revalidating && entry.revalidatingAt) {
+        if (Date.now() - entry.revalidatingAt < REVALIDATION_TIMEOUT_MS) {
+            return false; // Still within timeout, don't retry
+        }
+        // Timed out — allow a new attempt
+    }
+    entry.revalidating = true;
+    entry.revalidatingAt = Date.now();
+    // LRU touch: move to the end when updated.
+    responseCache.delete(key);
+    responseCache.set(key, entry);
+    return true;
+}
+export function setCached(url, result) {
+    setCacheEntry(normalizeUrl(url), result);
+}
+export function clearCache() {
+    responseCache.clear();
+}
+export function setCacheTTL(ms) {
+    if (!Number.isFinite(ms) || ms <= 0) {
+        throw new Error('Cache TTL must be a positive number of milliseconds');
+    }
+    cacheTTL = ms;
+}

package/dist/core/cf-worker-proxy.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Cloudflare Worker fetch proxy client.
+ *
+ * When PeelTLS fails due to IP blocking, this routes the request through
+ * a Cloudflare Worker running on Cloudflare's edge network. The Worker
+ * has clean IPs that aren't flagged by most anti-bot systems.
+ *
+ * Setup:
+ *   1. Deploy the worker: cd worker && npx wrangler deploy
+ *   2. Set WEBPEEL_CF_WORKER_URL env var to the worker URL
+ *   3. Optionally set WEBPEEL_CF_WORKER_TOKEN for auth
+ *
+ * Free tier: 100,000 requests/day
+ */
+import type { FetchResult } from './fetcher.js';
+export interface CfWorkerProxyOptions {
+    timeout?: number;
+    headers?: Record<string, string>;
+}
+export interface CfWorkerProxyResult extends FetchResult {
+    method: 'cf-worker';
+    edge?: string;
+}
+/**
+ * Check if a CF Worker proxy is configured.
+ */
+export declare function isCfWorkerAvailable(): boolean;
+/**
+ * Fetch a URL through the Cloudflare Worker proxy.
+ * The Worker makes the request from Cloudflare's edge — clean IPs.
+ */
+export declare function cfWorkerFetch(url: string, options?: CfWorkerProxyOptions): Promise<CfWorkerProxyResult>;

package/dist/core/cf-worker-proxy.js

@@ -0,0 +1,87 @@
+/**
+ * Cloudflare Worker fetch proxy client.
+ *
+ * When PeelTLS fails due to IP blocking, this routes the request through
+ * a Cloudflare Worker running on Cloudflare's edge network. The Worker
+ * has clean IPs that aren't flagged by most anti-bot systems.
+ *
+ * Setup:
+ *   1. Deploy the worker: cd worker && npx wrangler deploy
+ *   2. Set WEBPEEL_CF_WORKER_URL env var to the worker URL
+ *   3. Optionally set WEBPEEL_CF_WORKER_TOKEN for auth
+ *
+ * Free tier: 100,000 requests/day
+ */
+const CHROME_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
+    'Sec-Ch-Ua': '"Chromium";v="133", "Google Chrome";v="133"',
+    'Sec-Ch-Ua-Mobile': '?0',
+    'Sec-Ch-Ua-Platform': '"macOS"',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9',
+};
+/**
+ * Check if a CF Worker proxy is configured.
+ */
+export function isCfWorkerAvailable() {
+    return !!getWorkerUrl();
+}
+function getWorkerUrl() {
+    return process.env.WEBPEEL_CF_WORKER_URL;
+}
+function getWorkerToken() {
+    return process.env.WEBPEEL_CF_WORKER_TOKEN;
+}
+/**
+ * Fetch a URL through the Cloudflare Worker proxy.
+ * The Worker makes the request from Cloudflare's edge — clean IPs.
+ */
+export async function cfWorkerFetch(url, options) {
+    const workerUrl = getWorkerUrl();
+    if (!workerUrl) {
+        throw new Error('Cloudflare Worker proxy not configured. Set WEBPEEL_CF_WORKER_URL env var.\n' +
+            'Deploy: cd worker && npx wrangler deploy');
+    }
+    const mergedHeaders = {
+        ...CHROME_HEADERS,
+        ...(options?.headers ?? {}),
+    };
+    const fetchHeaders = {
+        'Content-Type': 'application/json',
+    };
+    const token = getWorkerToken();
+    if (token) {
+        fetchHeaders['Authorization'] = `Bearer ${token}`;
+    }
+    const body = JSON.stringify({
+        url,
+        headers: mergedHeaders,
+        method: 'GET',
+        timeout: Math.round((options?.timeout ?? 30000) / 1000),
+        followRedirects: true,
+    });
+    if (process.env.DEBUG) {
+        console.debug('[webpeel]', 'CF Worker proxy fetch:', url);
+    }
+    const response = await fetch(`${workerUrl}/fetch`, {
+        method: 'POST',
+        headers: fetchHeaders,
+        body,
+        signal: AbortSignal.timeout(options?.timeout ?? 30000),
+    });
+    const result = await response.json();
+    if (result.error) {
+        throw new Error(`CF Worker proxy failed: ${result.error}`);
+    }
+    return {
+        html: result.body,
+        url: result.finalUrl || url,
+        statusCode: result.status,
+        contentType: result.headers?.['content-type'] ?? 'text/html',
+        method: 'cf-worker',
+        edge: result.edge,
+    };
+}

package/dist/core/challenge-detection.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Challenge / bot-protection page detection.
+ *
+ * Analyzes raw HTML (and optional HTTP status code) to determine whether the
+ * response is a bot-challenge or block page rather than real content.
+ *
+ * Design goals:
+ *  - Fast: pure string/regex matching, no DOM parsing required
+ *  - Low false-positive rate: uses confidence scoring, only flags at >= 0.7
+ *  - No external dependencies
+ */
+export type ChallengeType = 'cloudflare' | 'captcha' | 'akamai' | 'perimeterx' | 'datadome' | 'incapsula' | 'generic-block' | 'empty-shell';
+export interface ChallengeDetectionResult {
+    isChallenge: boolean;
+    type?: ChallengeType;
+    /** Confidence score from 0 (not a challenge) to 1 (definitely a challenge). */
+    confidence: number;
+    details?: string;
+}
+/**
+ * Detect whether an HTML response is a bot-challenge or block page.
+ *
+ * @param html       - Raw HTML response body.
+ * @param statusCode - HTTP status code (optional but improves accuracy).
+ */
+export declare function detectChallenge(html: string, statusCode?: number): ChallengeDetectionResult;