@cosmocoder/mcp-web-docs 1.0.0

package/build/crawler/browser-config.js

@@ -0,0 +1,29 @@
+import { log } from 'crawlee';
+// Suppress Crawlee's stdout logging for MCP compatibility
+// MCP servers must only output JSON-RPC messages to stdout
+log.setLevel(log.LEVELS.OFF);
+export const getBrowserConfig = (requestQueue) => ({
+    maxRequestsPerCrawl: 1000,
+    requestQueue,
+    maxConcurrency: 20,
+    maxRequestsPerMinute: 600,
+    maxRequestRetries: 0,
+    navigationTimeoutSecs: 10,
+    browserPoolOptions: {
+        maxOpenPagesPerBrowser: 5,
+        useFingerprints: false,
+        operationTimeoutSecs: 15,
+        closeInactiveBrowserAfterSecs: 10,
+    },
+    preNavigationHooks: [
+        async ({ page }) => {
+            await page.setViewportSize({ width: 1920, height: 1080 });
+            await page.setExtraHTTPHeaders({
+                'Accept-Language': 'en-US,en;q=0.9',
+                Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+            });
+        },
+    ],
+});
+//# sourceMappingURL=browser-config.js.map

package/build/crawler/browser-config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-config.js","sourceRoot":"","sources":["../../src/crawler/browser-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,GAAG,EAAE,MAAM,SAAS,CAAC;AAExD,0DAA0D;AAC1D,2DAA2D;AAC3D,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAE7B,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,YAAsD,EAAqC,EAAE,CAAC,CAAC;IAC9H,mBAAmB,EAAE,IAAI;IACzB,YAAY;IACZ,cAAc,EAAE,EAAE;IAClB,oBAAoB,EAAE,GAAG;IACzB,iBAAiB,EAAE,CAAC;IACpB,qBAAqB,EAAE,EAAE;IACzB,kBAAkB,EAAE;QAClB,sBAAsB,EAAE,CAAC;QACzB,eAAe,EAAE,KAAK;QACtB,oBAAoB,EAAE,EAAE;QACxB,6BAA6B,EAAE,EAAE;KAClC;IACD,kBAAkB,EAAE;QAClB,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;YACjB,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1D,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBAC7B,iBAAiB,EAAE,gBAAgB;gBACnC,MAAM,EAAE,4EAA4E;gBACpF,YAAY,EACV,2HAA2H;aAC9H,CAAC,CAAC;QACL,CAAC;KACF;CACF,CAAC,CAAC"}

package/build/crawler/browser-config.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/crawler/browser-config.test.js

@@ -0,0 +1,56 @@
+import { getBrowserConfig } from './browser-config.js';
+describe('Browser Config', () => {
+    describe('getBrowserConfig', () => {
+        it('should return config with default values', () => {
+            const mockQueue = {};
+            const config = getBrowserConfig(mockQueue);
+            expect(config.maxRequestsPerCrawl).toBe(1000);
+            expect(config.maxConcurrency).toBe(20);
+            expect(config.maxRequestsPerMinute).toBe(600);
+            expect(config.maxRequestRetries).toBe(0);
+            expect(config.navigationTimeoutSecs).toBe(10);
+        });
+        it('should return config with requestQueue', () => {
+            const mockQueue = { name: 'test-queue' };
+            const config = getBrowserConfig(mockQueue);
+            expect(config.requestQueue).toBe(mockQueue);
+        });
+        it('should have browser pool options', () => {
+            const mockQueue = {};
+            const config = getBrowserConfig(mockQueue);
+            expect(config.browserPoolOptions).toBeDefined();
+            expect(config.browserPoolOptions?.maxOpenPagesPerBrowser).toBe(5);
+            expect(config.browserPoolOptions?.useFingerprints).toBe(false);
+            expect(config.browserPoolOptions?.operationTimeoutSecs).toBe(15);
+            expect(config.browserPoolOptions?.closeInactiveBrowserAfterSecs).toBe(10);
+        });
+        it('should have preNavigationHooks', () => {
+            const mockQueue = {};
+            const config = getBrowserConfig(mockQueue);
+            expect(config.preNavigationHooks).toBeDefined();
+            expect(Array.isArray(config.preNavigationHooks)).toBe(true);
+            expect(config.preNavigationHooks?.length).toBe(1);
+        });
+        it('should configure page in preNavigationHook', async () => {
+            const mockQueue = {};
+            const config = getBrowserConfig(mockQueue);
+            const mockPage = {
+                setViewportSize: vi.fn().mockResolvedValue(undefined),
+                setExtraHTTPHeaders: vi.fn().mockResolvedValue(undefined),
+            };
+            const mockContext = { page: mockPage };
+            const mockGotoOptions = {};
+            const hook = config.preNavigationHooks?.[0];
+            if (hook) {
+                await hook(mockContext, mockGotoOptions);
+            }
+            expect(mockPage.setViewportSize).toHaveBeenCalledWith({ width: 1920, height: 1080 });
+            expect(mockPage.setExtraHTTPHeaders).toHaveBeenCalledWith(expect.objectContaining({
+                'Accept-Language': expect.any(String),
+                Accept: expect.any(String),
+                'User-Agent': expect.any(String),
+            }));
+        });
+    });
+});
+//# sourceMappingURL=browser-config.test.js.map

package/build/crawler/browser-config.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-config.test.js","sourceRoot":"","sources":["../../src/crawler/browser-config.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAIvD,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,SAAS,GAAG,EAAE,IAAI,EAAE,YAAY,EAA4C,CAAC;YACnF,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,WAAW,EAAE,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,sBAAsB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/D,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjE,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,6BAA6B,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,WAAW,EAAE,CAAC;YAChD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,QAAQ,GAAG;gBACf,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;gBACrD,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;aACvC,CAAC;YAErB,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAA+B,CAAC;YACpE,MAAM,eAAe,GAAG,EAA2B,CAAC;YAEpD,MAAM,IAAI,GAAG,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5C,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;YAC3C,CAAC;YAED,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACrF,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,oBAAoB,CACvD,MAAM,CAAC,gBAAgB,CAAC;gBACtB,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBACrC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBAC1B,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;aACjC,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/cheerio.d.ts

@@ -0,0 +1,11 @@
+import { CrawlResult } from '../types.js';
+import { BaseCrawler } from './base.js';
+export declare class CheerioCrawler extends BaseCrawler {
+    private readonly BATCH_SIZE;
+    private readonly FETCH_TIMEOUT;
+    constructor(maxDepth?: number, maxRequestsPerCrawl?: number, onProgress?: (progress: number, description: string) => void);
+    crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
+    private processPageWithRetry;
+    private extractLinks;
+    private extractTitle;
+}

package/build/crawler/cheerio.js

@@ -0,0 +1,134 @@
+import { URL } from 'url';
+import * as cheerio from 'cheerio';
+import { BaseCrawler } from './base.js';
+export class CheerioCrawler extends BaseCrawler {
+    // Batch size for processing URLs
+    BATCH_SIZE = 50;
+    FETCH_TIMEOUT = 30000; // 30 seconds
+    constructor(maxDepth = 4, maxRequestsPerCrawl = 1000, onProgress) {
+        super(maxDepth, maxRequestsPerCrawl, onProgress);
+    }
+    async *crawl(url) {
+        console.debug(`[${this.constructor.name}] Starting crawl from: ${url}`);
+        if (this.isAborting) {
+            console.debug('[CheerioCrawler] Crawl aborted');
+            return;
+        }
+        const startUrl = new URL(url);
+        const baseUrl = this.normalizeUrl(startUrl.toString());
+        // Track pages to crawl
+        const pagesToCrawl = new Map(); // URL -> depth
+        pagesToCrawl.set(baseUrl, 0);
+        while (pagesToCrawl.size > 0 && !this.isAborting) {
+            // Get batch of URLs to process
+            const batchEntries = Array.from(pagesToCrawl.entries()).slice(0, this.BATCH_SIZE);
+            const batch = new Map(batchEntries);
+            // Remove batch from queue
+            batchEntries.forEach(([url]) => pagesToCrawl.delete(url));
+            try {
+                // Process batch in parallel with timeout and rate limiting
+                const results = await Promise.all(Array.from(batch.entries()).map(async ([pageUrl]) => {
+                    // Apply rate limiting
+                    await this.rateLimit();
+                    const result = await this.processPageWithRetry(pageUrl);
+                    return { pageUrl, ...result };
+                }));
+                // Handle results
+                for (const { pageUrl, content, links, error } of results) {
+                    if (error || !content || this.isAborting)
+                        continue;
+                    this.markUrlAsSeen(pageUrl);
+                    yield {
+                        url: pageUrl,
+                        path: this.getPathFromUrl(pageUrl),
+                        content,
+                        title: this.extractTitle(content)
+                    };
+                    // Add new links to queue if within depth limit
+                    const currentDepth = batch.get(pageUrl) || 0;
+                    if (currentDepth < this.maxDepth) {
+                        for (const link of links) {
+                            const normalizedLink = this.normalizeUrl(link);
+                            if (this.shouldCrawl(normalizedLink) && !pagesToCrawl.has(normalizedLink)) {
+                                pagesToCrawl.set(normalizedLink, currentDepth + 1);
+                            }
+                        }
+                    }
+                }
+                // Add delay between batches
+                await new Promise(resolve => setTimeout(resolve, 1000));
+            }
+            catch (e) {
+                console.error('[CheerioCrawler] Error processing batch:', e);
+            }
+        }
+        console.debug('[CheerioCrawler] Crawl completed');
+    }
+    async processPageWithRetry(url) {
+        return this.retryWithBackoff(async () => {
+            try {
+                // Create fetch request with timeout
+                const controller = new AbortController();
+                const timeoutId = setTimeout(() => controller.abort(), this.FETCH_TIMEOUT);
+                const response = await fetch(url, {
+                    signal: controller.signal,
+                    headers: {
+                        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+                    }
+                });
+                clearTimeout(timeoutId);
+                if (!response.ok) {
+                    throw new Error(`HTTP error! status: ${response.status}`);
+                }
+                const content = await response.text();
+                const links = this.extractLinks(content, new URL(url));
+                return { content, links };
+            }
+            catch (e) {
+                if (e instanceof Error) {
+                    return { content: null, links: [], error: e };
+                }
+                return { content: null, links: [], error: new Error('Unknown error occurred') };
+            }
+        });
+    }
+    extractLinks(html, baseUrl) {
+        try {
+            const $ = cheerio.load(html);
+            const links = new Set();
+            // Find all links, including those in navigation elements
+            $('a').each((_, element) => {
+                const href = $(element).attr('href');
+                if (!href)
+                    return;
+                try {
+                    const url = new URL(href, baseUrl);
+                    const normalizedUrl = this.normalizeUrl(url.toString());
+                    // Use BaseCrawler's URL validation
+                    if (this.shouldCrawl(normalizedUrl)) {
+                        links.add(normalizedUrl);
+                    }
+                }
+                catch (e) {
+                    console.debug(`[CheerioCrawler] Invalid URL ${href}:`, e);
+                }
+            });
+            return Array.from(links);
+        }
+        catch (e) {
+            console.error('[CheerioCrawler] Error extracting links:', e);
+            return [];
+        }
+    }
+    extractTitle(html) {
+        try {
+            const $ = cheerio.load(html);
+            return $('title').text().trim() || 'Untitled';
+        }
+        catch (e) {
+            console.error('[CheerioCrawler] Error extracting title:', e);
+            return 'Untitled';
+        }
+    }
+}
+//# sourceMappingURL=cheerio.js.map

package/build/crawler/cheerio.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/crawler/cheerio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAExC,MAAM,OAAO,cAAe,SAAQ,WAAW;IAC7C,iCAAiC;IAChB,UAAU,GAAG,EAAE,CAAC;IAChB,aAAa,GAAG,KAAK,CAAC,CAAC,aAAa;IAErD,YACE,WAAmB,CAAC,EACpB,sBAA8B,IAAI,EAClC,UAA4D;QAE5D,KAAK,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,0BAA0B,GAAG,EAAE,CAAC,CAAC;QAExE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAChD,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEvD,uBAAuB;QACvB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,eAAe;QAC/D,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAE7B,OAAO,YAAY,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACjD,+BAA+B;YAC/B,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAClF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;YAEpC,0BAA0B;YAC1B,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAE1D,IAAI,CAAC;gBACH,2DAA2D;gBAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE;oBAClD,sBAAsB;oBACtB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;oBACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;oBACxD,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,EAAE,CAAC;gBAChC,CAAC,CAAC,CACH,CAAC;gBAEF,iBAAiB;gBACjB,KAAK,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,OAAO,EAAE,CAAC;oBACzD,IAAI,KAAK,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU;wBAAE,SAAS;oBAEnD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;oBAE5B,MAAM;wBACJ,GAAG,EAAE,OAAO;wBACZ,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;wBAClC,OAAO;wBACP,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;qBAClC,CAAC;oBAEF,+CAA+C;oBAC/C,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBAC7C,IAAI,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;4BACzB,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;4BAC/C,IAAI,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;gCAC1E,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;4BACrD,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,4BAA4B;gBAC5B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;YAC1D,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,GAAW;QAK5C,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,IAAI,EAAE;YACtC,IAAI,CAAC;gBACH,oCAAoC;gBACpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;gBAE3E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBAChC,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,YAAY,EAAE,qHAAqH;qBACpI;iBACF,CAAC,CAAC;gBAEH,YAAY,CAAC,SAAS,CAAC,CAAC;gBAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC5D,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBAEvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YAC5B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,YAAY,KAAK,EAAE,CAAC;oBACvB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;gBAChD,CAAC;gBACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClF,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,YAAY,CAAC,IAAY,EAAE,OAAY;QAC7C,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;YAEhC,yDAAyD;YACzD,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACzB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,CAAC,IAAI;oBAAE,OAAO;gBAElB,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;oBACnC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAExD,mCAAmC;oBACnC,IAAI,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;wBACpC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;oBAC3B,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC5D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;YAC7D,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,IAAY;QAC/B,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,UAAU,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;YAC7D,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC;CACF"}

package/build/crawler/chromium.d.ts

@@ -0,0 +1,21 @@
+import { CrawlResult } from '../types.js';
+import { BaseCrawler } from './base.js';
+export declare class ChromiumCrawler extends BaseCrawler {
+    private readonly PCR_CONFIG;
+    private curCrawlCount;
+    private baseHostname;
+    private readonly BATCH_SIZE;
+    private readonly REACT_WAIT_TIME;
+    private readonly NAVIGATION_WAIT_TIME;
+    private readonly MAX_CONCURRENT_PAGES;
+    private readonly PAGE_TIMEOUT;
+    private readonly resourceCache;
+    constructor(maxDepth?: number, maxRequestsPerCrawl?: number, onProgress?: (progress: number, description: string) => void);
+    crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
+    private extractPageContent;
+    private waitForDynamicContent;
+    private gotoPageAndHandleRedirects;
+    private configurePage;
+    private processSinglePage;
+    private getLinksFromPage;
+}