@cosmocoder/mcp-web-docs 1.0.0

package/build/crawler/default.js

@@ -0,0 +1,138 @@
+import { URL } from 'url';
+import * as cheerio from 'cheerio';
+import { BaseCrawler } from './base.js';
+export class DefaultCrawler extends BaseCrawler {
+    BATCH_SIZE = 50;
+    FETCH_TIMEOUT = 30000; // 30 seconds
+    constructor(maxDepth = 4, maxRequestsPerCrawl = 1000, onProgress) {
+        super(maxDepth, maxRequestsPerCrawl, onProgress);
+    }
+    async *crawl(url) {
+        console.debug(`[${this.constructor.name}] Starting crawl from: ${url}`);
+        if (this.isAborting) {
+            console.debug('[DefaultCrawler] Crawl aborted');
+            return;
+        }
+        const startUrl = new URL(url);
+        const baseUrl = this.normalizeUrl(startUrl.toString());
+        // Track pages to process
+        const pagesToCrawl = new Map(); // URL -> depth
+        pagesToCrawl.set(baseUrl, 0);
+        while (pagesToCrawl.size > 0 && !this.isAborting) {
+            // Get batch of URLs to process
+            const batchEntries = Array.from(pagesToCrawl.entries()).slice(0, this.BATCH_SIZE);
+            const batch = new Map(batchEntries);
+            // Remove batch from queue
+            batchEntries.forEach(([url]) => pagesToCrawl.delete(url));
+            try {
+                // Process batch in parallel with timeout and rate limiting
+                const results = await Promise.all(Array.from(batch.entries()).map(async ([pageUrl]) => {
+                    // Apply rate limiting
+                    await this.rateLimit();
+                    const result = await this.processPageWithRetry(pageUrl);
+                    return { pageUrl, ...result };
+                }));
+                // Handle results
+                for (const { pageUrl, content, links, error } of results) {
+                    if (error || !content || this.isAborting)
+                        continue;
+                    this.markUrlAsSeen(pageUrl);
+                    yield {
+                        url: pageUrl,
+                        path: this.getPathFromUrl(pageUrl),
+                        content,
+                        title: this.extractTitle(content)
+                    };
+                    // Add new links to queue if within depth limit
+                    const currentDepth = batch.get(pageUrl) || 0;
+                    if (currentDepth < this.maxDepth) {
+                        for (const link of links) {
+                            const normalizedLink = this.normalizeUrl(link);
+                            if (this.shouldCrawl(normalizedLink) && !pagesToCrawl.has(normalizedLink)) {
+                                pagesToCrawl.set(normalizedLink, currentDepth + 1);
+                            }
+                        }
+                    }
+                    // Check if we've hit the request limit
+                    if (this.seenUrls.size >= this.maxRequestsPerCrawl) {
+                        console.debug('[DefaultCrawler] Max requests reached');
+                        return;
+                    }
+                }
+                // Add delay between batches
+                await new Promise(resolve => setTimeout(resolve, 1000));
+            }
+            catch (e) {
+                console.error('[DefaultCrawler] Error processing batch:', e);
+            }
+        }
+        console.debug('[DefaultCrawler] Crawl completed');
+    }
+    async processPageWithRetry(url) {
+        return this.retryWithBackoff(async () => {
+            try {
+                // Create fetch request with timeout
+                const controller = new AbortController();
+                const timeoutId = setTimeout(() => controller.abort(), this.FETCH_TIMEOUT);
+                const response = await fetch(url, {
+                    signal: controller.signal,
+                    headers: {
+                        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+                    }
+                });
+                clearTimeout(timeoutId);
+                if (!response.ok) {
+                    throw new Error(`HTTP error! status: ${response.status}`);
+                }
+                const content = await response.text();
+                const links = this.extractLinks(content, new URL(url));
+                return { content, links };
+            }
+            catch (e) {
+                if (e instanceof Error) {
+                    return { content: null, links: [], error: e };
+                }
+                return { content: null, links: [], error: new Error('Unknown error occurred') };
+            }
+        });
+    }
+    extractLinks(html, baseUrl) {
+        try {
+            const $ = cheerio.load(html);
+            const links = new Set();
+            // Find all links, including those in navigation elements
+            $('a').each((_, element) => {
+                const href = $(element).attr('href');
+                if (!href)
+                    return;
+                try {
+                    const url = new URL(href, baseUrl);
+                    const normalizedUrl = this.normalizeUrl(url.toString());
+                    // Use BaseCrawler's URL validation
+                    if (this.shouldCrawl(normalizedUrl)) {
+                        links.add(normalizedUrl);
+                    }
+                }
+                catch (e) {
+                    console.debug(`[DefaultCrawler] Invalid URL ${href}:`, e);
+                }
+            });
+            return Array.from(links);
+        }
+        catch (e) {
+            console.error('[DefaultCrawler] Error extracting links:', e);
+            return [];
+        }
+    }
+    extractTitle(html) {
+        try {
+            const $ = cheerio.load(html);
+            return $('title').text().trim() || 'Untitled';
+        }
+        catch (e) {
+            console.error('[DefaultCrawler] Error extracting title:', e);
+            return 'Untitled';
+        }
+    }
+}
+//# sourceMappingURL=default.js.map

package/build/crawler/default.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"default.js","sourceRoot":"","sources":["../../src/crawler/default.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAExC,MAAM,OAAO,cAAe,SAAQ,WAAW;IAC5B,UAAU,GAAG,EAAE,CAAC;IAChB,aAAa,GAAG,KAAK,CAAC,CAAC,aAAa;IAErD,YACE,WAAmB,CAAC,EACpB,sBAA8B,IAAI,EAClC,UAA4D;QAE5D,KAAK,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,0BAA0B,GAAG,EAAE,CAAC,CAAC;QAExE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAChD,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEvD,yBAAyB;QACzB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,eAAe;QAC/D,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAE7B,OAAO,YAAY,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACjD,+BAA+B;YAC/B,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAClF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;YAEpC,0BAA0B;YAC1B,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAE1D,IAAI,CAAC;gBACH,2DAA2D;gBAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE;oBAClD,sBAAsB;oBACtB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;oBACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;oBACxD,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,EAAE,CAAC;gBAChC,CAAC,CAAC,CACH,CAAC;gBAEF,iBAAiB;gBACjB,KAAK,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,OAAO,EAAE,CAAC;oBACzD,IAAI,KAAK,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU;wBAAE,SAAS;oBAEnD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;oBAE5B,MAAM;wBACJ,GAAG,EAAE,OAAO;wBACZ,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;wBAClC,OAAO;wBACP,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;qBAClC,CAAC;oBAEF,+CAA+C;oBAC/C,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBAC7C,IAAI,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;4BACzB,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;4BAC/C,IAAI,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;gCAC1E,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;4BACrD,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,uCAAuC;oBACvC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;wBACnD,OAAO,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;wBACvD,OAAO;oBACT,CAAC;gBACH,CAAC;gBAED,4BAA4B;gBAC5B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;YAC1D,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,GAAW;QAK5C,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,IAAI,EAAE;YACtC,IAAI,CAAC;gBACH,oCAAoC;gBACpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;gBAE3E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBAChC,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,YAAY,EAAE,qHAAqH;qBACpI;iBACF,CAAC,CAAC;gBAEH,YAAY,CAAC,SAAS,CAAC,CAAC;gBAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC5D,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBAEvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YAC5B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,YAAY,KAAK,EAAE,CAAC;oBACvB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;gBAChD,CAAC;gBACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClF,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,YAAY,CAAC,IAAY,EAAE,OAAY;QAC7C,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;YAEhC,yDAAyD;YACzD,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACzB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,CAAC,IAAI;oBAAE,OAAO;gBAElB,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;oBACnC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAExD,mCAAmC;oBACnC,IAAI,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;wBACpC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;oBAC3B,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC5D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;YAC7D,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,IAAY;QAC/B,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,UAAU,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;YAC7D,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC;CACF"}

package/build/crawler/docs-crawler.d.ts

@@ -0,0 +1,26 @@
+import { CrawlResult, DocsCrawlerType, WebCrawler } from '../types.js';
+import { StorageState } from './crawlee-crawler.js';
+export declare class DocsCrawler implements WebCrawler {
+    private readonly maxDepth;
+    private readonly maxRequestsPerCrawl;
+    private readonly githubToken?;
+    private readonly onProgress?;
+    private readonly GITHUB_HOST;
+    private readonly MIN_PAGES;
+    private isAborting;
+    private storageState?;
+    private pathPrefix?;
+    constructor(maxDepth?: number, maxRequestsPerCrawl?: number, githubToken?: string | undefined, onProgress?: ((progress: number, description: string) => void) | undefined);
+    /**
+     * Set an optional path prefix to restrict crawling to URLs under this path.
+     * Only pages whose path starts with this prefix will be crawled.
+     * Example: '/oss/javascript/langchain' would only crawl pages under that path.
+     */
+    setPathPrefix(prefix: string): void;
+    /**
+     * Set authentication storage state (cookies) to use when crawling
+     */
+    setStorageState(state: StorageState): void;
+    crawl(url: string): AsyncGenerator<CrawlResult, DocsCrawlerType>;
+    abort(): void;
+}

package/build/crawler/docs-crawler.js

@@ -0,0 +1,97 @@
+import { URL } from 'url';
+import { CrawleeCrawler } from './crawlee-crawler.js';
+import { GitHubCrawler } from './github.js';
+import { logger } from '../util/logger.js';
+export class DocsCrawler {
+    maxDepth;
+    maxRequestsPerCrawl;
+    githubToken;
+    onProgress;
+    GITHUB_HOST = 'github.com';
+    MIN_PAGES = 2; // Require at least 2 pages for component libraries
+    isAborting = false;
+    storageState;
+    pathPrefix;
+    constructor(maxDepth = 4, maxRequestsPerCrawl = 1000, githubToken, onProgress) {
+        this.maxDepth = maxDepth;
+        this.maxRequestsPerCrawl = maxRequestsPerCrawl;
+        this.githubToken = githubToken;
+        this.onProgress = onProgress;
+    }
+    /**
+     * Set an optional path prefix to restrict crawling to URLs under this path.
+     * Only pages whose path starts with this prefix will be crawled.
+     * Example: '/oss/javascript/langchain' would only crawl pages under that path.
+     */
+    setPathPrefix(prefix) {
+        this.pathPrefix = prefix;
+        logger.info(`[DocsCrawler] Path prefix restriction set: ${prefix}`);
+    }
+    /**
+     * Set authentication storage state (cookies) to use when crawling
+     */
+    setStorageState(state) {
+        this.storageState = state;
+        logger.info(`[DocsCrawler] Set storage state with ${state.cookies?.length || 0} cookies`);
+    }
+    async *crawl(url) {
+        const startUrl = new URL(url);
+        logger.debug(`[DocsCrawler] Starting crawl of ${startUrl}`);
+        if (this.isAborting) {
+            logger.debug('[DocsCrawler] Crawl aborted');
+            return 'crawlee';
+        }
+        // Handle GitHub repositories
+        if (startUrl.host === this.GITHUB_HOST) {
+            logger.debug('[DocsCrawler] Detected GitHub repository');
+            const githubCrawler = new GitHubCrawler(this.maxDepth, this.maxRequestsPerCrawl, this.githubToken, this.onProgress);
+            try {
+                for await (const page of githubCrawler.crawl(url)) {
+                    if (this.isAborting)
+                        break;
+                    yield page;
+                }
+                return 'github';
+            }
+            catch (e) {
+                logger.debug('[DocsCrawler] GitHub crawler failed:', e);
+                // Don't fall through to other crawlers for GitHub URLs
+                throw e;
+            }
+        }
+        // Use Crawlee for all other sites
+        logger.debug('[DocsCrawler] Using Crawlee crawler');
+        const crawleeCrawler = new CrawleeCrawler(this.maxDepth, this.maxRequestsPerCrawl, this.onProgress);
+        // Pass authentication if available
+        if (this.storageState) {
+            crawleeCrawler.setStorageState(this.storageState);
+        }
+        // Pass path prefix restriction if configured
+        if (this.pathPrefix) {
+            crawleeCrawler.setPathPrefix(this.pathPrefix);
+        }
+        let pageCount = 0;
+        try {
+            for await (const page of crawleeCrawler.crawl(url)) {
+                if (this.isAborting)
+                    break;
+                pageCount++;
+                yield page;
+            }
+            if (pageCount >= this.MIN_PAGES) {
+                logger.debug(`[DocsCrawler] Crawlee crawler successful (${pageCount} pages)`);
+                return 'crawlee';
+            }
+            logger.debug(`[DocsCrawler] Crawlee crawler found insufficient pages (${pageCount})`);
+            throw new Error(`Crawlee crawler found only ${pageCount} pages, need at least ${this.MIN_PAGES}`);
+        }
+        catch (e) {
+            logger.debug('[DocsCrawler] Crawlee crawler failed:', e);
+            throw e;
+        }
+    }
+    abort() {
+        this.isAborting = true;
+    }
+}
+//# sourceMappingURL=docs-crawler.js.map

package/build/crawler/docs-crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docs-crawler.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAW;IAQH;IACA;IACA;IACA;IAVF,WAAW,GAAG,YAAY,CAAC;IAC3B,SAAS,GAAG,CAAC,CAAC,CAAC,mDAAmD;IAC3E,UAAU,GAAG,KAAK,CAAC;IACnB,YAAY,CAAgB;IAC5B,UAAU,CAAU;IAE5B,YACmB,WAAmB,CAAC,EACpB,sBAA8B,IAAI,EAClC,WAAoB,EACpB,UAA4D;QAH5D,aAAQ,GAAR,QAAQ,CAAY;QACpB,wBAAmB,GAAnB,mBAAmB,CAAe;QAClC,gBAAW,GAAX,WAAW,CAAS;QACpB,eAAU,GAAV,UAAU,CAAkD;IAC5E,CAAC;IAEJ;;;;OAIG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,8CAA8C,MAAM,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,KAAmB;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5F,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,mCAAmC,QAAQ,EAAE,CAAC,CAAC;QAE5D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC5C,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,6BAA6B;QAC7B,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAEpH,IAAI,CAAC;gBACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClD,IAAI,IAAI,CAAC,UAAU;wBAAE,MAAM;oBAC3B,MAAM,IAAI,CAAC;gBACb,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,sCAAsC,EAAE,CAAC,CAAC,CAAC;gBACxD,uDAAuD;gBACvD,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;QAED,kCAAkC;QAClC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACpD,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEpG,mCAAmC;QACnC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,cAAc,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACpD,CAAC;QAED,6CAA6C;QAC7C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnD,IAAI,IAAI,CAAC,UAAU;oBAAE,MAAM;gBAC3B,SAAS,EAAE,CAAC;gBACZ,MAAM,IAAI,CAAC;YACb,CAAC;YAED,IAAI,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAChC,MAAM,CAAC,KAAK,CAAC,6CAA6C,SAAS,SAAS,CAAC,CAAC;gBAC9E,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,2DAA2D,SAAS,GAAG,CAAC,CAAC;YACtF,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,yBAAyB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACpG,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;CACF"}

package/build/crawler/docs-crawler.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/crawler/docs-crawler.test.js

@@ -0,0 +1,185 @@
+import { DocsCrawler } from './docs-crawler.js';
+const mockGitHubCrawl = vi.fn();
+vi.mock('./github.js', () => ({
+    GitHubCrawler: function () {
+        return {
+            crawl: mockGitHubCrawl,
+        };
+    },
+}));
+const mockCrawleeCrawl = vi.fn();
+const mockSetStorageState = vi.fn();
+vi.mock('./crawlee-crawler.js', () => ({
+    CrawleeCrawler: function () {
+        return {
+            crawl: mockCrawleeCrawl,
+            setStorageState: mockSetStorageState,
+        };
+    },
+}));
+describe('DocsCrawler', () => {
+    let crawler;
+    beforeEach(() => {
+        vi.clearAllMocks();
+        crawler = new DocsCrawler();
+    });
+    describe('constructor', () => {
+        it('should initialize with default values', () => {
+            expect(crawler).toBeDefined();
+        });
+        it('should accept custom parameters', () => {
+            const customCrawler = new DocsCrawler(10, 500, 'github_token', vi.fn());
+            expect(customCrawler).toBeDefined();
+        });
+    });
+    describe('crawl', () => {
+        describe('GitHub URLs', () => {
+            it('should use GitHubCrawler for github.com URLs', async () => {
+                const mockResults = [
+                    { url: 'https://github.com/owner/repo/README.md', path: 'README.md', content: '# README', title: 'README' },
+                ];
+                mockGitHubCrawl.mockImplementation(async function* () {
+                    for (const result of mockResults) {
+                        yield result;
+                    }
+                });
+                const results = [];
+                const generator = crawler.crawl('https://github.com/owner/repo');
+                for await (const result of generator) {
+                    results.push(result);
+                }
+                expect(results).toHaveLength(1);
+                expect(results[0].url).toContain('github.com');
+            });
+            it('should return github type for GitHub URLs', async () => {
+                mockGitHubCrawl.mockImplementation(async function* () {
+                    yield { url: 'https://github.com/owner/repo', path: '/', content: 'test', title: 'Test' };
+                });
+                const generator = crawler.crawl('https://github.com/owner/repo');
+                // Manually iterate to capture the return value
+                let result = await generator.next();
+                while (!result.done) {
+                    result = await generator.next();
+                }
+                const crawlerType = result.value;
+                expect(crawlerType).toBe('github');
+            });
+            it('should propagate errors from GitHubCrawler', async () => {
+                // eslint-disable-next-line require-yield
+                mockGitHubCrawl.mockImplementation(async function* () {
+                    throw new Error('GitHub API error');
+                });
+                const generator = crawler.crawl('https://github.com/owner/repo');
+                await expect(async () => {
+                    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+                    for await (const _ of generator) {
+                        // Just consume results
+                    }
+                }).rejects.toThrow('GitHub API error');
+            });
+        });
+        describe('Non-GitHub URLs', () => {
+            it('should use CrawleeCrawler for non-GitHub URLs', async () => {
+                const mockResults = [
+                    { url: 'https://docs.example.com/guide', path: '/guide', content: '<h1>Guide</h1>', title: 'Guide' },
+                    { url: 'https://docs.example.com/api', path: '/api', content: '<h1>API</h1>', title: 'API' },
+                ];
+                mockCrawleeCrawl.mockImplementation(async function* () {
+                    for (const result of mockResults) {
+                        yield result;
+                    }
+                });
+                const results = [];
+                for await (const result of crawler.crawl('https://docs.example.com')) {
+                    results.push(result);
+                }
+                expect(results).toHaveLength(2);
+            });
+            it('should return crawlee type for sufficient pages', async () => {
+                mockCrawleeCrawl.mockImplementation(async function* () {
+                    yield { url: 'https://example.com/page1', path: '/page1', content: 'Page 1', title: 'Page 1' };
+                    yield { url: 'https://example.com/page2', path: '/page2', content: 'Page 2', title: 'Page 2' };
+                });
+                const generator = crawler.crawl('https://example.com');
+                // Manually iterate to capture the return value
+                let result = await generator.next();
+                while (!result.done) {
+                    result = await generator.next();
+                }
+                const crawlerType = result.value;
+                expect(crawlerType).toBe('crawlee');
+            });
+            it('should throw error when insufficient pages found', async () => {
+                mockCrawleeCrawl.mockImplementation(async function* () {
+                    yield { url: 'https://example.com/page1', path: '/page1', content: 'Page 1', title: 'Page 1' };
+                    // Only 1 page, needs at least 2
+                });
+                const generator = crawler.crawl('https://example.com');
+                await expect(async () => {
+                    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+                    for await (const _ of generator) {
+                        // Just consume results
+                    }
+                }).rejects.toThrow(/found only 1 pages/);
+            });
+        });
+        describe('abort', () => {
+            it('should stop crawling when aborted', async () => {
+                mockCrawleeCrawl.mockImplementation(async function* () {
+                    yield { url: 'https://example.com/page1', path: '/page1', content: 'Page 1', title: 'Page 1' };
+                    yield { url: 'https://example.com/page2', path: '/page2', content: 'Page 2', title: 'Page 2' };
+                    yield { url: 'https://example.com/page3', path: '/page3', content: 'Page 3', title: 'Page 3' };
+                });
+                const results = [];
+                const generator = crawler.crawl('https://example.com');
+                // Get first result
+                const first = await generator.next();
+                if (!first.done) {
+                    results.push(first.value);
+                }
+                // Abort
+                crawler.abort();
+                // Generator should stop yielding after abort (depending on implementation)
+                // The test verifies abort() method exists and is callable
+                expect(results).toHaveLength(1);
+            });
+            it('should return early when already aborting', async () => {
+                crawler.abort();
+                mockCrawleeCrawl.mockImplementation(async function* () {
+                    yield { url: 'https://example.com/page1', path: '/page1', content: 'Page 1', title: 'Page 1' };
+                });
+                const generator = crawler.crawl('https://example.com');
+                const result = await generator.next();
+                // Should return immediately with crawlee type when aborted
+                expect(result.done).toBe(true);
+                expect(result.value).toBe('crawlee');
+            });
+        });
+        describe('setStorageState', () => {
+            it('should set storage state', () => {
+                const storageState = {
+                    cookies: [{ name: 'session', value: 'abc123', domain: 'example.com', path: '/' }],
+                };
+                crawler.setStorageState(storageState);
+                // Verify the method doesn't throw
+                expect(true).toBe(true);
+            });
+            it('should pass storage state to CrawleeCrawler', async () => {
+                const storageState = {
+                    cookies: [{ name: 'session', value: 'abc123', domain: 'example.com', path: '/' }],
+                };
+                crawler.setStorageState(storageState);
+                mockCrawleeCrawl.mockImplementation(async function* () {
+                    yield { url: 'https://example.com/page1', path: '/page1', content: 'Page 1', title: 'Page 1' };
+                    yield { url: 'https://example.com/page2', path: '/page2', content: 'Page 2', title: 'Page 2' };
+                });
+                // eslint-disable-next-line @typescript-eslint/no-unused-vars
+                for await (const _ of crawler.crawl('https://example.com')) {
+                    // Just consume results
+                }
+                expect(mockSetStorageState).toHaveBeenCalledWith(storageState);
+            });
+        });
+    });
+});
+//# sourceMappingURL=docs-crawler.test.js.map

package/build/crawler/docs-crawler.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docs-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAChC,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5B,aAAa,EAAE;QACb,OAAO;YACL,KAAK,EAAE,eAAe;SACvB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACjC,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACpC,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,cAAc,EAAE;QACd,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,eAAe,EAAE,mBAAmB;SACrC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,aAAa,GAAG,IAAI,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;YAC3B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;gBAC5D,MAAM,WAAW,GAAkB;oBACjC,EAAE,GAAG,EAAE,yCAAyC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE;iBAC5G,CAAC;gBAEF,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBAChD,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;wBACjC,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAEjE,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;oBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;gBAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;gBACzD,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBAChD,MAAM,EAAE,GAAG,EAAE,+BAA+B,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;gBAC5F,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAEjE,+CAA+C;gBAC/C,IAAI,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBACpB,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;gBAEjC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;gBAC1D,yCAAyC;gBACzC,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBAChD,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAEjE,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;oBACtB,6DAA6D;oBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;wBAChC,uBAAuB;oBACzB,CAAC;gBACH,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;YAC/B,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;gBAC7D,MAAM,WAAW,GAAkB;oBACjC,EAAE,GAAG,EAAE,gCAAgC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,OAAO,EAAE;oBACpG,EAAE,GAAG,EAAE,8BAA8B,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;iBAC7F,CAAC;gBAEF,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;wBACjC,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC;oBACrE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;gBAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;gBAC/D,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBAC/F,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAEvD,+CAA+C;gBAC/C,IAAI,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBACpB,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;gBAEjC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;gBAChE,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBAC/F,gCAAgC;gBAClC,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAEvD,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;oBACtB,6DAA6D;oBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;wBAChC,uBAAuB;oBACzB,CAAC;gBACH,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;YAC3C,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;YACrB,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;gBACjD,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBAC/F,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBAC/F,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAEvD,mBAAmB;gBACnB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;oBAChB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;gBAED,QAAQ;gBACR,OAAO,CAAC,KAAK,EAAE,CAAC;gBAEhB,2EAA2E;gBAC3E,0DAA0D;gBAC1D,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;gBACzD,OAAO,CAAC,KAAK,EAAE,CAAC;gBAEhB,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAEtC,2DAA2D;gBAC3D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;YAC/B,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;gBAClC,MAAM,YAAY,GAAG;oBACnB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;iBAClF,CAAC;gBAEF,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;gBAEtC,kCAAkC;gBAClC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;gBAC3D,MAAM,YAAY,GAAG;oBACnB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;iBAClF,CAAC;gBAEF,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;gBAEtC,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBAC/F,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,6DAA6D;gBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;oBAC3D,uBAAuB;gBACzB,CAAC;gBAED,MAAM,CAAC,mBAAmB,CAAC,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/factory.d.ts

@@ -0,0 +1,6 @@
+import { BaseCrawler } from './base.js';
+export declare class CrawlerFactory {
+    private static readonly JS_FRAMEWORK_INDICATORS;
+    private static detectSiteType;
+    static createCrawler(url: string, maxRequestsPerCrawl?: number, maxDepth?: number, onProgress?: (progress: number, description: string) => void): Promise<BaseCrawler>;
+}

package/build/crawler/factory.js

@@ -0,0 +1,83 @@
+import { URL } from 'url';
+import { DefaultCrawler } from './default.js';
+import { ChromiumCrawler } from './chromium.js';
+import { CheerioCrawler } from './cheerio.js';
+export class CrawlerFactory {
+    // Common JavaScript framework identifiers
+    static JS_FRAMEWORK_INDICATORS = [
+        'react',
+        'vue',
+        'angular',
+        'next',
+        'nuxt',
+        'gatsby',
+        'docusaurus',
+        'vuepress',
+        'gridsome',
+        'svelte'
+    ];
+    static async detectSiteType(url) {
+        try {
+            const response = await fetch(url, {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+                }
+            });
+            if (!response.ok) {
+                throw new Error(`HTTP error! status: ${response.status}`);
+            }
+            const html = await response.text();
+            // Check for JavaScript frameworks
+            const hasFramework = CrawlerFactory.JS_FRAMEWORK_INDICATORS.some(framework => html.toLowerCase().includes(framework));
+            // Check for JavaScript-heavy indicators
+            const isJsHeavy = (html.includes('data-react') ||
+                html.includes('ng-') ||
+                html.includes('v-') ||
+                html.includes('__NEXT_DATA__') ||
+                html.includes('nuxt') ||
+                html.includes('id="___gatsby"'));
+            return { isJsHeavy, hasFramework };
+        }
+        catch (e) {
+            console.error('[CrawlerFactory] Error detecting site type:', e);
+            return { isJsHeavy: false, hasFramework: false };
+        }
+    }
+    static async createCrawler(url, maxRequestsPerCrawl = 1000, maxDepth = 4, onProgress) {
+        const startUrl = new URL(url);
+        console.debug(`[CrawlerFactory] Creating crawler for ${startUrl}`);
+        // Check if site is JavaScript-heavy first
+        const { isJsHeavy, hasFramework } = await CrawlerFactory.detectSiteType(url);
+        // Try Chromium for JavaScript-heavy sites
+        if (isJsHeavy || hasFramework) {
+            console.debug(`[CrawlerFactory] Site appears to be JavaScript-heavy, using Chromium crawler`);
+            return new ChromiumCrawler(maxDepth, maxRequestsPerCrawl, onProgress);
+        }
+        // Try default crawler
+        try {
+            console.debug(`[CrawlerFactory] Attempting default crawler for ${url}`);
+            const defaultCrawler = new DefaultCrawler(maxDepth, maxRequestsPerCrawl, onProgress);
+            const generator = defaultCrawler.crawl(url);
+            const { value: firstPage, done } = await generator.next();
+            if (!done && firstPage?.content) {
+                console.debug('[CrawlerFactory] Successfully created default crawler');
+                return defaultCrawler;
+            }
+        }
+        catch (e) {
+            console.debug('[CrawlerFactory] Default crawler failed:', e);
+        }
+        // Fall back to Cheerio crawler
+        console.debug(`[CrawlerFactory] Attempting Cheerio crawler for ${url}`);
+        const cheerioCrawler = new CheerioCrawler(maxDepth, maxRequestsPerCrawl, onProgress);
+        const generator = cheerioCrawler.crawl(url);
+        const { value: firstPage, done } = await generator.next();
+        if (!done && firstPage?.content) {
+            console.debug('[CrawlerFactory] Successfully created Cheerio crawler');
+            return cheerioCrawler;
+        }
+        console.error(`[CrawlerFactory] All crawlers failed for ${url}`);
+        throw new Error(`Failed to create crawler for ${url}`);
+    }
+}
+//# sourceMappingURL=factory.js.map

package/build/crawler/factory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"factory.js","sourceRoot":"","sources":["../../src/crawler/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,MAAM,OAAO,cAAc;IACzB,0CAA0C;IAClC,MAAM,CAAU,uBAAuB,GAAG;QAChD,OAAO;QACP,KAAK;QACL,SAAS;QACT,MAAM;QACN,MAAM;QACN,QAAQ;QACR,YAAY;QACZ,UAAU;QACV,UAAU;QACV,QAAQ;KACT,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,GAAW;QAI7C,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO,EAAE;oBACP,YAAY,EAAE,qHAAqH;iBACpI;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5D,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEnC,kCAAkC;YAClC,MAAM,YAAY,GAAG,cAAc,CAAC,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAC3E,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CACvC,CAAC;YAEF,wCAAwC;YACxC,MAAM,SAAS,GAAG,CAChB,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAC3B,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACpB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACnB,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;gBAC9B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACrB,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAChC,CAAC;YAEF,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,CAAC,CAAC,CAAC;YAChE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,aAAa,CACxB,GAAW,EACX,sBAA8B,IAAI,EAClC,WAAmB,CAAC,EACpB,UAA4D;QAE5D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,OAAO,CAAC,KAAK,CAAC,yCAAyC,QAAQ,EAAE,CAAC,CAAC;QAEnE,0CAA0C;QAC1C,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAE7E,0CAA0C;QAC1C,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;YAC9B,OAAO,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAC;YAC9F,OAAO,IAAI,eAAe,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;QACxE,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,mDAAmD,GAAG,EAAE,CAAC,CAAC;YACxE,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;YACrF,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;YAE1D,IAAI,CAAC,IAAI,IAAI,SAAS,EAAE,OAAO,EAAE,CAAC;gBAChC,OAAO,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;gBACvE,OAAO,cAAc,CAAC;YACxB,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,+BAA+B;QAC/B,OAAO,CAAC,KAAK,CAAC,mDAAmD,GAAG,EAAE,CAAC,CAAC;QACxE,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;QACrF,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;QAE1D,IAAI,CAAC,IAAI,IAAI,SAAS,EAAE,OAAO,EAAE,CAAC;YAChC,OAAO,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;YACvE,OAAO,cAAc,CAAC;QACxB,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,4CAA4C,GAAG,EAAE,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,gCAAgC,GAAG,EAAE,CAAC,CAAC;IACzD,CAAC"}

package/build/crawler/github-pages-extractor.d.ts

@@ -0,0 +1,4 @@
+import { ContentExtractor, ExtractedContent } from './content-extractor-types.js';
+export declare class GitHubPagesExtractor implements ContentExtractor {
+    extractContent(document: Document): Promise<ExtractedContent>;
+}

package/build/crawler/github-pages-extractor.js

@@ -0,0 +1,33 @@
+export class GitHubPagesExtractor {
+    async extractContent(document) {
+        // Remove navigation and footer
+        document.querySelectorAll('nav, header, footer').forEach((el) => el.remove());
+        // Get main content
+        const main = document.querySelector('main, article, .markdown-body');
+        if (!main) {
+            return {
+                content: '',
+                metadata: { type: 'overview' },
+            };
+        }
+        const clone = main.cloneNode(true);
+        clone.querySelectorAll('script, style').forEach((el) => el.remove());
+        // Extract title and description
+        const title = clone.querySelector('h1')?.textContent?.trim();
+        const firstParagraph = clone.querySelector('h1 + p')?.textContent?.trim();
+        return {
+            content: clone.textContent?.trim() || '',
+            metadata: {
+                type: 'overview',
+                pattern: {
+                    name: title || '',
+                    type: 'component',
+                    description: firstParagraph || '',
+                    usageContexts: [],
+                    relatedPatterns: [],
+                },
+            },
+        };
+    }
+}
+//# sourceMappingURL=github-pages-extractor.js.map

package/build/crawler/github-pages-extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github-pages-extractor.js","sourceRoot":"","sources":["../../src/crawler/github-pages-extractor.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,oBAAoB;IAC/B,KAAK,CAAC,cAAc,CAAC,QAAkB;QACrC,+BAA+B;QAC/B,QAAQ,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QAE9E,mBAAmB;QACnB,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,+BAA+B,CAAC,CAAC;QACrE,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,QAAQ,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE;aAC/B,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAY,CAAC;QAC9C,KAAK,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QAErE,gCAAgC;QAChC,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAC7D,MAAM,cAAc,GAAG,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAE1E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE;YACxC,QAAQ,EAAE;gBACR,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE;oBACP,IAAI,EAAE,KAAK,IAAI,EAAE;oBACjB,IAAI,EAAE,WAAW;oBACjB,WAAW,EAAE,cAAc,IAAI,EAAE;oBACjC,aAAa,EAAE,EAAE;oBACjB,eAAe,EAAE,EAAE;iBACpB;aACF;SACF,CAAC;IACJ,CAAC;CACF"}

package/build/crawler/github-pages-extractor.test.d.ts

@@ -0,0 +1 @@
+export {};