@memvid/maw 1.0.0

package/dist/src/crawler/sitemap.js

@@ -0,0 +1,209 @@
+/**
+ * Sitemap.xml parser using sitemapper
+ */
+import * as cheerio from 'cheerio';
+import Sitemapper from 'sitemapper';
+export class SitemapParser {
+    cache = new Map();
+    sitemapper;
+    constructor() {
+        this.sitemapper = new Sitemapper({
+            timeout: 10000,
+            requestHeaders: {
+                'User-Agent': 'maw/1.0 (sitemap crawler)',
+            },
+        });
+    }
+    /**
+     * Parse sitemap for a URL and return all URLs
+     */
+    async parse(url) {
+        try {
+            const parsedUrl = new URL(url);
+            const host = parsedUrl.hostname;
+            // Check cache
+            if (this.cache.has(host)) {
+                return this.cache.get(host);
+            }
+            const urls = [];
+            // Try common sitemap locations
+            const sitemapUrls = [
+                `${parsedUrl.origin}/sitemap.xml`,
+                `${parsedUrl.origin}/sitemap_index.xml`,
+                `${parsedUrl.origin}/sitemap/sitemap.xml`,
+            ];
+            for (const sitemapUrl of sitemapUrls) {
+                try {
+                    const sitemapContent = await this.fetchSitemap(sitemapUrl);
+                    if (sitemapContent) {
+                        const parsed = await this.parseSitemapContent(sitemapContent, parsedUrl.origin);
+                        urls.push(...parsed);
+                        if (urls.length > 0)
+                            break;
+                    }
+                }
+                catch {
+                    // Try next location
+                }
+            }
+            this.cache.set(host, urls);
+            return urls;
+        }
+        catch {
+            return [];
+        }
+    }
+    /**
+     * Parse sitemap from a specific URL
+     */
+    async parseUrl(sitemapUrl) {
+        try {
+            const content = await this.fetchSitemap(sitemapUrl);
+            if (!content)
+                return [];
+            const origin = new URL(sitemapUrl).origin;
+            return this.parseSitemapContent(content, origin);
+        }
+        catch {
+            return [];
+        }
+    }
+    async fetchSitemap(url) {
+        try {
+            const response = await fetch(url, {
+                headers: {
+                    'User-Agent': 'maw/1.0',
+                    'Accept': 'application/xml, text/xml, */*',
+                },
+                signal: AbortSignal.timeout(5000), // 5s timeout - fail fast
+            });
+            if (!response.ok)
+                return null;
+            return await response.text();
+        }
+        catch {
+            return null;
+        }
+    }
+    async parseSitemapContent(content, origin) {
+        const urls = [];
+        const MAX_URLS = 200; // Don't parse more than we need
+        // Check if it's a sitemap index
+        if (content.includes('<sitemapindex')) {
+            const indexUrls = this.parseSitemapIndex(content);
+            // Fetch only first 2 sub-sitemaps (speed over completeness)
+            for (const indexUrl of indexUrls.slice(0, 2)) {
+                if (urls.length >= MAX_URLS)
+                    break; // Early exit
+                try {
+                    const subContent = await this.fetchSitemap(indexUrl);
+                    if (subContent) {
+                        const subUrls = this.parseUrlset(subContent);
+                        urls.push(...subUrls.slice(0, MAX_URLS - urls.length));
+                    }
+                }
+                catch {
+                    // Skip failed sitemaps
+                }
+            }
+        }
+        else {
+            // Regular sitemap - limit to MAX_URLS
+            const parsed = this.parseUrlset(content);
+            urls.push(...parsed.slice(0, MAX_URLS));
+        }
+        return urls.slice(0, MAX_URLS);
+    }
+    parseSitemapIndex(content) {
+        const $ = cheerio.load(content, { xmlMode: true });
+        const urls = [];
+        $('sitemap loc').each((_, el) => {
+            const loc = $(el).text().trim();
+            if (loc)
+                urls.push(loc);
+        });
+        return urls;
+    }
+    parseUrlset(content) {
+        const $ = cheerio.load(content, { xmlMode: true });
+        const urls = [];
+        $('url loc').each((_, el) => {
+            const loc = $(el).text().trim();
+            if (loc)
+                urls.push(loc);
+        });
+        return urls;
+    }
+    /**
+     * Parse sitemap for a URL and return all URLs with metadata
+     * Uses sitemapper for robust parsing of sitemap indexes and news sitemaps
+     */
+    async parseWithMetadata(url) {
+        try {
+            const parsedUrl = new URL(url);
+            const origin = parsedUrl.origin;
+            // Try news sitemaps first (more recent content)
+            const newsSitemapUrls = [
+                `${origin}/sitemap/news.xml`,
+                `${origin}/news-sitemap.xml`,
+                `${origin}/sitemap-news.xml`,
+                `${origin}/sitemaps/news.xml`,
+            ];
+            // Try news sitemaps first
+            for (const sitemapUrl of newsSitemapUrls) {
+                try {
+                    const content = await this.fetchSitemap(sitemapUrl);
+                    if (content && content.includes('<url')) {
+                        const urls = this.parseWithMeta(content);
+                        if (urls.length > 0) {
+                            return urls.slice(0, 1000);
+                        }
+                    }
+                }
+                catch {
+                    // Try next
+                }
+            }
+            // Fall back to sitemapper for regular sitemaps
+            const result = await this.sitemapper.fetch(`${origin}/sitemap.xml`);
+            if (result.sites && result.sites.length > 0) {
+                return result.sites.slice(0, 1000).map(loc => ({ loc }));
+            }
+            // Try sitemap index
+            const indexResult = await this.sitemapper.fetch(`${origin}/sitemap_index.xml`);
+            if (indexResult.sites && indexResult.sites.length > 0) {
+                return indexResult.sites.slice(0, 1000).map(loc => ({ loc }));
+            }
+            return [];
+        }
+        catch {
+            return [];
+        }
+    }
+    /**
+     * Parse sitemap with full metadata
+     */
+    parseWithMeta(content) {
+        const $ = cheerio.load(content, { xmlMode: true });
+        const urls = [];
+        $('url').each((_, el) => {
+            const $el = $(el);
+            const loc = $el.find('loc').text().trim();
+            if (loc) {
+                const entry = { loc };
+                const lastmod = $el.find('lastmod').text().trim();
+                if (lastmod)
+                    entry.lastmod = lastmod;
+                const changefreq = $el.find('changefreq').text().trim();
+                if (changefreq)
+                    entry.changefreq = changefreq;
+                const priority = $el.find('priority').text().trim();
+                if (priority)
+                    entry.priority = parseFloat(priority);
+                urls.push(entry);
+            }
+        });
+        return urls;
+    }
+}
+//# sourceMappingURL=sitemap.js.map

package/dist/src/crawler/sitemap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../../src/crawler/sitemap.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,UAAU,MAAM,YAAY,CAAC;AASpC,MAAM,OAAO,aAAa;IAChB,KAAK,GAA0B,IAAI,GAAG,EAAE,CAAC;IACzC,UAAU,CAAa;IAE/B;QACE,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAC;YAC/B,OAAO,EAAE,KAAK;YACd,cAAc,EAAE;gBACd,YAAY,EAAE,2BAA2B;aAC1C;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,GAAW;QACrB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;YAEhC,cAAc;YACd,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;YAC/B,CAAC;YAED,MAAM,IAAI,GAAa,EAAE,CAAC;YAE1B,+BAA+B;YAC/B,MAAM,WAAW,GAAG;gBAClB,GAAG,SAAS,CAAC,MAAM,cAAc;gBACjC,GAAG,SAAS,CAAC,MAAM,oBAAoB;gBACvC,GAAG,SAAS,CAAC,MAAM,sBAAsB;aAC1C,CAAC;YAEF,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,IAAI,CAAC;oBACH,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;oBAC3D,IAAI,cAAc,EAAE,CAAC;wBACnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;wBAChF,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;wBACrB,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;4BAAE,MAAM;oBAC7B,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,oBAAoB;gBACtB,CAAC;YACH,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC3B,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,UAAkB;QAC/B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YACpD,IAAI,CAAC,OAAO;gBAAE,OAAO,EAAE,CAAC;YAExB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;YAC1C,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO,EAAE;oBACP,YAAY,EAAE,SAAS;oBACvB,QAAQ,EAAE,gCAAgC;iBAC3C;gBACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,yBAAyB;aAC7D,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,OAAO,IAAI,CAAC;YAE9B,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,OAAe,EAAE,MAAc;QAC/D,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,gCAAgC;QAEtD,gCAAgC;QAChC,IAAI,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YAElD,4DAA4D;YAC5D,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC7C,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;oBAAE,MAAM,CAAC,aAAa;gBACjD,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;oBACrD,IAAI,UAAU,EAAE,CAAC;wBACf,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;wBAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;oBACzD,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,uBAAuB;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,sCAAsC;YACtC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1C,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACjC,CAAC;IAEO,iBAAiB,CAAC,OAAe;QACvC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC9B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAChC,IAAI,GAAG;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,WAAW,CAAC,OAAe;QACjC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC1B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAChC,IAAI,GAAG;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB,CAAC,GAAW;QACjC,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;YAEhC,gDAAgD;YAChD,MAAM,eAAe,GAAG;gBACtB,GAAG,MAAM,mBAAmB;gBAC5B,GAAG,MAAM,mBAAmB;gBAC5B,GAAG,MAAM,mBAAmB;gBAC5B,GAAG,MAAM,oBAAoB;aAC9B,CAAC;YAEF,0BAA0B;YAC1B,KAAK,MAAM,UAAU,IAAI,eAAe,EAAE,CAAC;gBACzC,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;oBACpD,IAAI,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBACxC,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;wBACzC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACpB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;wBAC7B,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,WAAW;gBACb,CAAC;YACH,CAAC;YAED,+CAA+C;YAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;YACpE,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YAC3D,CAAC;YAED,oBAAoB;YACpB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,MAAM,oBAAoB,CAAC,CAAC;YAC/E,IAAI,WAAW,CAAC,KAAK,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtD,OAAO,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YAChE,CAAC;YAED,OAAO,EAAE,CAAC;QACZ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,OAAe;QAC3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,GAAiB,EAAE,CAAC;QAE9B,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACtB,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAClB,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAE1C,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,KAAK,GAAe,EAAE,GAAG,EAAE,CAAC;gBAElC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAClD,IAAI,OAAO;oBAAE,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;gBAErC,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACxD,IAAI,UAAU;oBAAE,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC;gBAE9C,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACpD,IAAI,QAAQ;oBAAE,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAEpD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/dist/src/engine/detector.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Block detection heuristics for anti-bot systems
+ */
+export type BlockReason = 'cloudflare' | 'datadome' | 'captcha' | 'rate_limit' | 'forbidden' | 'empty_body' | 'javascript_required' | 'access_denied';
+export interface BlockCheck {
+    blocked: boolean;
+    reason?: BlockReason;
+    confidence: number;
+}
+/**
+ * Check if the response indicates blocking
+ */
+export declare function isBlocked(html: string, statusCode: number, url: string): BlockCheck;
+/**
+ * Quick pre-check based on known protected domains
+ */
+export declare function needsBrowser(url: string): boolean;
+//# sourceMappingURL=detector.d.ts.map

package/dist/src/engine/detector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"detector.d.ts","sourceRoot":"","sources":["../../../src/engine/detector.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,MAAM,WAAW,GACnB,YAAY,GACZ,UAAU,GACV,SAAS,GACT,YAAY,GACZ,WAAW,GACX,YAAY,GACZ,qBAAqB,GACrB,eAAe,CAAC;AAEpB,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AA8DD;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,UAAU,CAuEnF;AAgBD;;GAEG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAoBjD"}

package/dist/src/engine/detector.js

@@ -0,0 +1,155 @@
+/**
+ * Block detection heuristics for anti-bot systems
+ */
+// Cloudflare challenge patterns
+const CLOUDFLARE_PATTERNS = [
+    /Checking your browser/i,
+    /cf-browser-verification/i,
+    /cloudflare/i,
+    /_cf_chl_opt/,
+    /challenge-platform/i,
+    /Just a moment\.\.\./i,
+    /ray id:/i,
+    /cf-turnstile/i,
+];
+// DataDome patterns
+const DATADOME_PATTERNS = [
+    /datadome/i,
+    /dd\.js/,
+    /captcha-delivery\.com/,
+    /geo\.captcha-delivery\.com/,
+];
+// PerimeterX / HUMAN patterns
+const PERIMETERX_PATTERNS = [
+    /perimeterx/i,
+    /px-captcha/i,
+    /_pxhd/,
+    /human challenge/i,
+];
+// Akamai Bot Manager patterns
+const AKAMAI_PATTERNS = [
+    /akamai/i,
+    /ak_bmsc/,
+    /_abck/,
+];
+// Generic bot detection patterns
+const BOT_DETECTION_PATTERNS = [
+    /access denied/i,
+    /please verify you are human/i,
+    /enable javascript/i,
+    /browser.*not supported/i,
+    /automated access/i,
+    /bot detected/i,
+    /please complete the security check/i,
+    /unusual traffic/i,
+    /blocked/i,
+    /forbidden/i,
+    /not allowed/i,
+];
+// Captcha patterns
+const CAPTCHA_PATTERNS = [
+    /recaptcha/i,
+    /hcaptcha/i,
+    /g-recaptcha/,
+    /h-captcha/,
+    /captcha/i,
+    /turnstile/i,
+];
+/**
+ * Check if the response indicates blocking
+ */
+export function isBlocked(html, statusCode, url) {
+    // Status code checks
+    if (statusCode === 403) {
+        return { blocked: true, reason: 'forbidden', confidence: 0.9 };
+    }
+    if (statusCode === 429) {
+        return { blocked: true, reason: 'rate_limit', confidence: 0.95 };
+    }
+    if (statusCode === 503) {
+        // Could be Cloudflare challenge or actual server error
+        if (CLOUDFLARE_PATTERNS.some(p => p.test(html))) {
+            return { blocked: true, reason: 'cloudflare', confidence: 0.95 };
+        }
+    }
+    // Empty or minimal body
+    if (!html || html.length < 500) {
+        // Some pages are legitimately small, check for tell-tale signs
+        if (html && (html.includes('challenge') || html.includes('captcha'))) {
+            return { blocked: true, reason: 'empty_body', confidence: 0.7 };
+        }
+    }
+    // Cloudflare
+    const cfMatches = CLOUDFLARE_PATTERNS.filter(p => p.test(html));
+    if (cfMatches.length >= 2) {
+        return { blocked: true, reason: 'cloudflare', confidence: 0.9 };
+    }
+    // DataDome - only flag if page is SHORT (actual block pages are small)
+    if (DATADOME_PATTERNS.filter(p => p.test(html)).length >= 2 && html.length < 50000) {
+        return { blocked: true, reason: 'datadome', confidence: 0.9 };
+    }
+    // PerimeterX - only flag if page is short
+    if (PERIMETERX_PATTERNS.some(p => p.test(html)) && html.length < 50000) {
+        return { blocked: true, reason: 'access_denied', confidence: 0.85 };
+    }
+    // Akamai - only flag if page is short
+    if (AKAMAI_PATTERNS.filter(p => p.test(html)).length >= 2 && html.length < 50000) {
+        return { blocked: true, reason: 'access_denied', confidence: 0.8 };
+    }
+    // Captcha - only flag if the page is SHORT and has captcha patterns
+    // Long pages with captcha are probably just forms, not block pages
+    const captchaMatches = CAPTCHA_PATTERNS.filter(p => p.test(html)).length;
+    if (captchaMatches >= 2 && html.length < 10000) {
+        // Page is short with multiple captcha patterns - likely a challenge page
+        return { blocked: true, reason: 'captcha', confidence: 0.85 };
+    }
+    // Generic bot detection - only if page is very short
+    if (BOT_DETECTION_PATTERNS.some(p => p.test(html)) && html.length < 3000) {
+        return { blocked: true, reason: 'access_denied', confidence: 0.7 };
+    }
+    // JavaScript required (no actual content)
+    if (html.includes('noscript') &&
+        (html.includes('enable javascript') || html.includes('JavaScript is required')) &&
+        !html.includes('<article') &&
+        !html.includes('<main') &&
+        html.length < 5000) {
+        return { blocked: true, reason: 'javascript_required', confidence: 0.6 };
+    }
+    return { blocked: false, confidence: 0 };
+}
+// Known protected domains that typically need browser
+const KNOWN_PROTECTED_DOMAINS = new Set([
+    'linkedin.com',
+    'instagram.com',
+    'facebook.com',
+    'twitter.com',
+    'x.com',
+    'tiktok.com',
+    'indeed.com',
+    'glassdoor.com',
+    'zillow.com',
+    'yelp.com',
+]);
+/**
+ * Quick pre-check based on known protected domains
+ */
+export function needsBrowser(url) {
+    try {
+        const hostname = new URL(url).hostname.replace('www.', '');
+        // Check exact match
+        if (KNOWN_PROTECTED_DOMAINS.has(hostname)) {
+            return true;
+        }
+        // Check if subdomain of protected domain
+        for (const domain of KNOWN_PROTECTED_DOMAINS) {
+            if (hostname.endsWith(`.${domain}`)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=detector.js.map

package/dist/src/engine/detector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"detector.js","sourceRoot":"","sources":["../../../src/engine/detector.ts"],"names":[],"mappings":"AAAA;;GAEG;AAkBH,gCAAgC;AAChC,MAAM,mBAAmB,GAAG;IAC1B,wBAAwB;IACxB,0BAA0B;IAC1B,aAAa;IACb,aAAa;IACb,qBAAqB;IACrB,sBAAsB;IACtB,UAAU;IACV,eAAe;CAChB,CAAC;AAEF,oBAAoB;AACpB,MAAM,iBAAiB,GAAG;IACxB,WAAW;IACX,QAAQ;IACR,uBAAuB;IACvB,4BAA4B;CAC7B,CAAC;AAEF,8BAA8B;AAC9B,MAAM,mBAAmB,GAAG;IAC1B,aAAa;IACb,aAAa;IACb,OAAO;IACP,kBAAkB;CACnB,CAAC;AAEF,8BAA8B;AAC9B,MAAM,eAAe,GAAG;IACtB,SAAS;IACT,SAAS;IACT,OAAO;CACR,CAAC;AAEF,iCAAiC;AACjC,MAAM,sBAAsB,GAAG;IAC7B,gBAAgB;IAChB,8BAA8B;IAC9B,oBAAoB;IACpB,yBAAyB;IACzB,mBAAmB;IACnB,eAAe;IACf,qCAAqC;IACrC,kBAAkB;IAClB,UAAU;IACV,YAAY;IACZ,cAAc;CACf,CAAC;AAEF,mBAAmB;AACnB,MAAM,gBAAgB,GAAG;IACvB,YAAY;IACZ,WAAW;IACX,aAAa;IACb,WAAW;IACX,UAAU;IACV,YAAY;CACb,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,UAAkB,EAAE,GAAW;IACrE,qBAAqB;IACrB,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QACvB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;IACjE,CAAC;IAED,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QACvB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACnE,CAAC;IAED,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QACvB,uDAAuD;QACvD,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACnE,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC/B,+DAA+D;QAC/D,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;QAClE,CAAC;IACH,CAAC;IAED,aAAa;IACb,MAAM,SAAS,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAChE,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;IAClE,CAAC;IAED,uEAAuE;IACvE,IAAI,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;QACnF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;IAChE,CAAC;IAED,0CAA0C;IAC1C,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;QACvE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACtE,CAAC;IAED,sCAAsC;IACtC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;QACjF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;IACrE,CAAC;IAED,oEAAoE;IACpE,mEAAmE;IACnE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IACzE,IAAI,cAAc,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;QAC/C,yEAAyE;QACzE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAChE,CAAC;IAED,qDAAqD;IACrD,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;QACzE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;IACrE,CAAC;IAED,0CAA0C;IAC1C,IACE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;QACzB,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;QAC/E,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;QAC1B,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,IAAI,EAClB,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;IAC3E,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED,sDAAsD;AACtD,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC;IACtC,cAAc;IACd,eAAe;IACf,cAAc;IACd,aAAa;IACb,OAAO;IACP,YAAY;IACZ,YAAY;IACZ,eAAe;IACf,YAAY;IACZ,UAAU;CACX,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAE3D,oBAAoB;QACpB,IAAI,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,yCAAyC;QACzC,KAAK,MAAM,MAAM,IAAI,uBAAuB,EAAE,CAAC;YAC7C,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC,EAAE,CAAC;gBACpC,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/src/engine/fetch.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Fast HTTP fetch engine - default for most sites
+ */
+export interface FetchOptions {
+    timeout?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+}
+export interface FetchResult {
+    html: string;
+    statusCode: number;
+    headers: Record<string, string>;
+    finalUrl: string;
+}
+export declare class FetchEngine {
+    fetch(url: string, options?: FetchOptions): Promise<FetchResult>;
+}
+//# sourceMappingURL=fetch.d.ts.map

package/dist/src/engine/fetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../../src/engine/fetch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAqBD,qBAAa,WAAW;IAChB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC;CAoC3E"}

package/dist/src/engine/fetch.js

@@ -0,0 +1,53 @@
+/**
+ * Fast HTTP fetch engine - default for most sites
+ */
+const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+const DEFAULT_HEADERS = {
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'Cache-Control': 'no-cache',
+    'Pragma': 'no-cache',
+    'Sec-Ch-Ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
+    'Sec-Ch-Ua-Mobile': '?0',
+    'Sec-Ch-Ua-Platform': '"macOS"',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+    'Upgrade-Insecure-Requests': '1',
+};
+export class FetchEngine {
+    async fetch(url, options = {}) {
+        const controller = new AbortController();
+        const timeout = options.timeout || 5000;
+        const timeoutId = setTimeout(() => controller.abort(), timeout);
+        try {
+            const response = await fetch(url, {
+                headers: {
+                    'User-Agent': options.userAgent || DEFAULT_USER_AGENT,
+                    ...DEFAULT_HEADERS,
+                    ...options.headers,
+                },
+                redirect: 'follow',
+                signal: controller.signal,
+            });
+            clearTimeout(timeoutId);
+            const html = await response.text();
+            return {
+                html,
+                statusCode: response.status,
+                headers: Object.fromEntries(response.headers.entries()),
+                finalUrl: response.url,
+            };
+        }
+        catch (error) {
+            clearTimeout(timeoutId);
+            if (error.name === 'AbortError') {
+                throw new Error(`Timeout after ${timeout}ms`);
+            }
+            throw error;
+        }
+    }
+}
+//# sourceMappingURL=fetch.js.map

package/dist/src/engine/fetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../src/engine/fetch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAeH,MAAM,kBAAkB,GACtB,uHAAuH,CAAC;AAE1H,MAAM,eAAe,GAAG;IACtB,QAAQ,EAAE,kGAAkG;IAC5G,iBAAiB,EAAE,gBAAgB;IACnC,iBAAiB,EAAE,mBAAmB;IACtC,eAAe,EAAE,UAAU;IAC3B,QAAQ,EAAE,UAAU;IACpB,WAAW,EAAE,kEAAkE;IAC/E,kBAAkB,EAAE,IAAI;IACxB,oBAAoB,EAAE,SAAS;IAC/B,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,MAAM;IACxB,gBAAgB,EAAE,IAAI;IACtB,2BAA2B,EAAE,GAAG;CACjC,CAAC;AAEF,MAAM,OAAO,WAAW;IACtB,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,UAAwB,EAAE;QACjD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC;QACxC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;QAEhE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO,EAAE;oBACP,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;oBACrD,GAAG,eAAe;oBAClB,GAAG,OAAO,CAAC,OAAO;iBACnB;gBACD,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEnC,OAAO;gBACL,IAAI;gBACJ,UAAU,EAAE,QAAQ,CAAC,MAAM;gBAC3B,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACvD,QAAQ,EAAE,QAAQ,CAAC,GAAG;aACvB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,iBAAiB,OAAO,IAAI,CAAC,CAAC;YAChD,CAAC;YAED,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}

package/dist/src/engine/index.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Engine Waterfall - orchestrates fetch engines
+ * Tries fast fetch first, escalates to browser only when blocked
+ */
+import { type BlockReason } from './detector.js';
+export interface EngineResult {
+    html: string;
+    statusCode: number;
+    engine: 'fetch' | 'playwright' | 'rebrowser';
+    blocked: boolean;
+    blockReason?: BlockReason;
+    finalUrl: string;
+}
+export interface EngineOptions {
+    timeout?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    forceEngine?: 'fetch' | 'playwright' | 'rebrowser';
+}
+export interface EngineStats {
+    fetch: number;
+    playwright: number;
+    rebrowser: number;
+    blocked: number;
+}
+export declare class EngineWaterfall {
+    private engines;
+    private stats;
+    private browserInstallPromise;
+    fetch(url: string, options?: EngineOptions): Promise<EngineResult>;
+    private fetchWithEngine;
+    private ensureBrowserInstalled;
+    private installBrowser;
+    getStats(): EngineStats;
+    close(): Promise<void>;
+}
+export { isBlocked, needsBrowser } from './detector.js';
+export type { BlockReason, BlockCheck } from './detector.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/engine/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/engine/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EAA2B,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAG1E,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IAC7C,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,WAAW,CAAC,EAAE,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;CACpD;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAmBD,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAA4E;IAC3F,OAAO,CAAC,KAAK,CAAsE;IACnF,OAAO,CAAC,qBAAqB,CAAiC;IAExD,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC;YAmC9D,eAAe;YAqCf,sBAAsB;YAatB,cAAc;IAc5B,QAAQ,IAAI,WAAW;IAIjB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAQ7B;AAED,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACxD,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC"}

package/dist/src/engine/index.js

@@ -0,0 +1,116 @@
+/**
+ * Engine Waterfall - orchestrates fetch engines
+ * Tries fast fetch first, escalates to browser only when blocked
+ */
+import { FetchEngine } from './fetch.js';
+import { PlaywrightEngine, isPlaywrightInstalled } from './playwright.js';
+import { RebrowserEngine } from './rebrowser.js';
+import { isBlocked, needsBrowser } from './detector.js';
+import { createLogger } from '../utils/logger.js';
+const log = createLogger();
+// Engine priority order
+const ENGINES = [
+    { name: 'fetch', timeout: 5000, create: () => new FetchEngine() },
+    { name: 'playwright', timeout: 15000, create: () => new PlaywrightEngine() },
+    { name: 'rebrowser', timeout: 30000, create: () => new RebrowserEngine() },
+];
+export class EngineWaterfall {
+    engines = new Map();
+    stats = { fetch: 0, playwright: 0, rebrowser: 0, blocked: 0 };
+    browserInstallPromise = null;
+    async fetch(url, options = {}) {
+        // Force specific engine if requested
+        if (options.forceEngine) {
+            return this.fetchWithEngine(url, options.forceEngine, options);
+        }
+        // Skip fetch for known protected sites
+        const startIndex = needsBrowser(url) ? 1 : 0;
+        const enginesToTry = ENGINES.slice(startIndex);
+        // Try engines in order until success
+        for (const engineConfig of enginesToTry) {
+            try {
+                const result = await this.fetchWithEngine(url, engineConfig.name, {
+                    ...options,
+                    timeout: options.timeout || engineConfig.timeout,
+                });
+                // Check if response looks blocked
+                if (!result.blocked) {
+                    this.stats[engineConfig.name]++;
+                    return result;
+                }
+                log.dim(`  ${engineConfig.name} blocked: ${result.blockReason}`);
+            }
+            catch (error) {
+                log.dim(`  ${engineConfig.name} failed: ${error.message}`);
+            }
+        }
+        // All engines failed
+        this.stats.blocked++;
+        throw new Error(`All engines failed for ${url}`);
+    }
+    async fetchWithEngine(url, engineName, options) {
+        // Lazy-load browser engines
+        if (engineName === 'playwright' || engineName === 'rebrowser') {
+            if (!await this.ensureBrowserInstalled()) {
+                throw new Error('Browser not available');
+            }
+        }
+        // Get or create engine instance
+        let engine = this.engines.get(engineName);
+        if (!engine) {
+            const config = ENGINES.find(e => e.name === engineName);
+            if (!config)
+                throw new Error(`Unknown engine: ${engineName}`);
+            engine = config.create();
+            this.engines.set(engineName, engine);
+        }
+        // Fetch
+        const result = await engine.fetch(url, options);
+        // Detect blocking
+        const blockCheck = isBlocked(result.html, result.statusCode, url);
+        return {
+            html: result.html,
+            statusCode: result.statusCode,
+            engine: engineName,
+            blocked: blockCheck.blocked,
+            blockReason: blockCheck.reason,
+            finalUrl: result.finalUrl,
+        };
+    }
+    async ensureBrowserInstalled() {
+        if (await isPlaywrightInstalled()) {
+            return true;
+        }
+        // Only show install prompt once
+        if (!this.browserInstallPromise) {
+            this.browserInstallPromise = this.installBrowser();
+        }
+        return this.browserInstallPromise;
+    }
+    async installBrowser() {
+        log.info('\n  Protected site detected. Installing browser...');
+        try {
+            const { execSync } = await import('child_process');
+            execSync('npx playwright install chromium', { stdio: 'inherit' });
+            log.success('  Browser installed successfully.\n');
+            return true;
+        }
+        catch (error) {
+            log.error('  Failed to install browser. Some sites may not work.');
+            return false;
+        }
+    }
+    getStats() {
+        return { ...this.stats };
+    }
+    async close() {
+        for (const engine of this.engines.values()) {
+            if ('close' in engine && typeof engine.close === 'function') {
+                await engine.close();
+            }
+        }
+        this.engines.clear();
+    }
+}
+export { isBlocked, needsBrowser } from './detector.js';
+//# sourceMappingURL=index.js.map

package/dist/src/engine/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/engine/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAoB,MAAM,eAAe,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAyBlD,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;AAU3B,wBAAwB;AACxB,MAAM,OAAO,GAAmB;IAC9B,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,WAAW,EAAE,EAAE;IACjE,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,gBAAgB,EAAE,EAAE;IAC5E,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,eAAe,EAAE,EAAE;CAC3E,CAAC;AAEF,MAAM,OAAO,eAAe;IAClB,OAAO,GAAkE,IAAI,GAAG,EAAE,CAAC;IACnF,KAAK,GAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC3E,qBAAqB,GAA4B,IAAI,CAAC;IAE9D,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,UAAyB,EAAE;QAClD,qCAAqC;QACrC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACjE,CAAC;QAED,uCAAuC;QACvC,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAE/C,qCAAqC;QACrC,KAAK,MAAM,YAAY,IAAI,YAAY,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,YAAY,CAAC,IAAI,EAAE;oBAChE,GAAG,OAAO;oBACV,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,YAAY,CAAC,OAAO;iBACjD,CAAC,CAAC;gBAEH,kCAAkC;gBAClC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBACpB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,MAAM,CAAC;gBAChB,CAAC;gBAED,GAAG,CAAC,GAAG,CAAC,KAAK,YAAY,CAAC,IAAI,aAAa,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;YACnE,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,GAAG,CAAC,GAAG,CAAC,KAAK,YAAY,CAAC,IAAI,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;IACnD,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,GAAW,EACX,UAAsB,EACtB,OAAsB;QAEtB,4BAA4B;QAC5B,IAAI,UAAU,KAAK,YAAY,IAAI,UAAU,KAAK,WAAW,EAAE,CAAC;YAC9D,IAAI,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC;gBACzC,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,gCAAgC;QAChC,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;YACxD,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,UAAU,EAAE,CAAC,CAAC;YAC9D,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACvC,CAAC;QAED,QAAQ;QACR,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAEhD,kBAAkB;QAClB,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAElE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,MAAM,EAAE,UAAU;YAClB,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,WAAW,EAAE,UAAU,CAAC,MAAM;YAC9B,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,IAAI,MAAM,qBAAqB,EAAE,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,gCAAgC;QAChC,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAChC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACrD,CAAC;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,GAAG,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;QAE/D,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;YACnD,QAAQ,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;YAClE,GAAG,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,GAAG,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;YACnE,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,OAAO,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;gBAC5D,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACF;AAED,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC"}