@cosmocoder/mcp-web-docs 1.0.0 → 1.0.1

package/build/crawler/default.d.ts

@@ -1,11 +0,0 @@
-import { CrawlResult } from '../types.js';
-import { BaseCrawler } from './base.js';
-export declare class DefaultCrawler extends BaseCrawler {
-    private readonly BATCH_SIZE;
-    private readonly FETCH_TIMEOUT;
-    constructor(maxDepth?: number, maxRequestsPerCrawl?: number, onProgress?: (progress: number, description: string) => void);
-    crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
-    private processPageWithRetry;
-    private extractLinks;
-    private extractTitle;
-}

package/build/crawler/default.js

@@ -1,138 +0,0 @@
-import { URL } from 'url';
-import * as cheerio from 'cheerio';
-import { BaseCrawler } from './base.js';
-export class DefaultCrawler extends BaseCrawler {
-    BATCH_SIZE = 50;
-    FETCH_TIMEOUT = 30000; // 30 seconds
-    constructor(maxDepth = 4, maxRequestsPerCrawl = 1000, onProgress) {
-        super(maxDepth, maxRequestsPerCrawl, onProgress);
-    }
-    async *crawl(url) {
-        console.debug(`[${this.constructor.name}] Starting crawl from: ${url}`);
-        if (this.isAborting) {
-            console.debug('[DefaultCrawler] Crawl aborted');
-            return;
-        }
-        const startUrl = new URL(url);
-        const baseUrl = this.normalizeUrl(startUrl.toString());
-        // Track pages to process
-        const pagesToCrawl = new Map(); // URL -> depth
-        pagesToCrawl.set(baseUrl, 0);
-        while (pagesToCrawl.size > 0 && !this.isAborting) {
-            // Get batch of URLs to process
-            const batchEntries = Array.from(pagesToCrawl.entries()).slice(0, this.BATCH_SIZE);
-            const batch = new Map(batchEntries);
-            // Remove batch from queue
-            batchEntries.forEach(([url]) => pagesToCrawl.delete(url));
-            try {
-                // Process batch in parallel with timeout and rate limiting
-                const results = await Promise.all(Array.from(batch.entries()).map(async ([pageUrl]) => {
-                    // Apply rate limiting
-                    await this.rateLimit();
-                    const result = await this.processPageWithRetry(pageUrl);
-                    return { pageUrl, ...result };
-                }));
-                // Handle results
-                for (const { pageUrl, content, links, error } of results) {
-                    if (error || !content || this.isAborting)
-                        continue;
-                    this.markUrlAsSeen(pageUrl);
-                    yield {
-                        url: pageUrl,
-                        path: this.getPathFromUrl(pageUrl),
-                        content,
-                        title: this.extractTitle(content)
-                    };
-                    // Add new links to queue if within depth limit
-                    const currentDepth = batch.get(pageUrl) || 0;
-                    if (currentDepth < this.maxDepth) {
-                        for (const link of links) {
-                            const normalizedLink = this.normalizeUrl(link);
-                            if (this.shouldCrawl(normalizedLink) && !pagesToCrawl.has(normalizedLink)) {
-                                pagesToCrawl.set(normalizedLink, currentDepth + 1);
-                            }
-                        }
-                    }
-                    // Check if we've hit the request limit
-                    if (this.seenUrls.size >= this.maxRequestsPerCrawl) {
-                        console.debug('[DefaultCrawler] Max requests reached');
-                        return;
-                    }
-                }
-                // Add delay between batches
-                await new Promise(resolve => setTimeout(resolve, 1000));
-            }
-            catch (e) {
-                console.error('[DefaultCrawler] Error processing batch:', e);
-            }
-        }
-        console.debug('[DefaultCrawler] Crawl completed');
-    }
-    async processPageWithRetry(url) {
-        return this.retryWithBackoff(async () => {
-            try {
-                // Create fetch request with timeout
-                const controller = new AbortController();
-                const timeoutId = setTimeout(() => controller.abort(), this.FETCH_TIMEOUT);
-                const response = await fetch(url, {
-                    signal: controller.signal,
-                    headers: {
-                        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-                    }
-                });
-                clearTimeout(timeoutId);
-                if (!response.ok) {
-                    throw new Error(`HTTP error! status: ${response.status}`);
-                }
-                const content = await response.text();
-                const links = this.extractLinks(content, new URL(url));
-                return { content, links };
-            }
-            catch (e) {
-                if (e instanceof Error) {
-                    return { content: null, links: [], error: e };
-                }
-                return { content: null, links: [], error: new Error('Unknown error occurred') };
-            }
-        });
-    }
-    extractLinks(html, baseUrl) {
-        try {
-            const $ = cheerio.load(html);
-            const links = new Set();
-            // Find all links, including those in navigation elements
-            $('a').each((_, element) => {
-                const href = $(element).attr('href');
-                if (!href)
-                    return;
-                try {
-                    const url = new URL(href, baseUrl);
-                    const normalizedUrl = this.normalizeUrl(url.toString());
-                    // Use BaseCrawler's URL validation
-                    if (this.shouldCrawl(normalizedUrl)) {
-                        links.add(normalizedUrl);
-                    }
-                }
-                catch (e) {
-                    console.debug(`[DefaultCrawler] Invalid URL ${href}:`, e);
-                }
-            });
-            return Array.from(links);
-        }
-        catch (e) {
-            console.error('[DefaultCrawler] Error extracting links:', e);
-            return [];
-        }
-    }
-    extractTitle(html) {
-        try {
-            const $ = cheerio.load(html);
-            return $('title').text().trim() || 'Untitled';
-        }
-        catch (e) {
-            console.error('[DefaultCrawler] Error extracting title:', e);
-            return 'Untitled';
-        }
-    }
-}
-//# sourceMappingURL=default.js.map

package/build/crawler/default.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"default.js","sourceRoot":"","sources":["../../src/crawler/default.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAExC,MAAM,OAAO,cAAe,SAAQ,WAAW;IAC5B,UAAU,GAAG,EAAE,CAAC;IAChB,aAAa,GAAG,KAAK,CAAC,CAAC,aAAa;IAErD,YACE,WAAmB,CAAC,EACpB,sBAA8B,IAAI,EAClC,UAA4D;QAE5D,KAAK,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,0BAA0B,GAAG,EAAE,CAAC,CAAC;QAExE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAChD,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEvD,yBAAyB;QACzB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,eAAe;QAC/D,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAE7B,OAAO,YAAY,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACjD,+BAA+B;YAC/B,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAClF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;YAEpC,0BAA0B;YAC1B,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAE1D,IAAI,CAAC;gBACH,2DAA2D;gBAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE;oBAClD,sBAAsB;oBACtB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;oBACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;oBACxD,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,EAAE,CAAC;gBAChC,CAAC,CAAC,CACH,CAAC;gBAEF,iBAAiB;gBACjB,KAAK,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,OAAO,EAAE,CAAC;oBACzD,IAAI,KAAK,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU;wBAAE,SAAS;oBAEnD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;oBAE5B,MAAM;wBACJ,GAAG,EAAE,OAAO;wBACZ,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;wBAClC,OAAO;wBACP,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;qBAClC,CAAC;oBAEF,+CAA+C;oBAC/C,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBAC7C,IAAI,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;4BACzB,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;4BAC/C,IAAI,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;gCAC1E,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;4BACrD,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,uCAAuC;oBACvC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;wBACnD,OAAO,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;wBACvD,OAAO;oBACT,CAAC;gBACH,CAAC;gBAED,4BAA4B;gBAC5B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;YAC1D,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,GAAW;QAK5C,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,IAAI,EAAE;YACtC,IAAI,CAAC;gBACH,oCAAoC;gBACpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;gBAE3E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBAChC,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,YAAY,EAAE,qHAAqH;qBACpI;iBACF,CAAC,CAAC;gBAEH,YAAY,CAAC,SAAS,CAAC,CAAC;gBAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC5D,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBAEvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YAC5B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,YAAY,KAAK,EAAE,CAAC;oBACvB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;gBAChD,CAAC;gBACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAClF,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,YAAY,CAAC,IAAY,EAAE,OAAY;QAC7C,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;YAEhC,yDAAyD;YACzD,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACzB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,CAAC,IAAI;oBAAE,OAAO;gBAElB,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;oBACnC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAExD,mCAAmC;oBACnC,IAAI,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;wBACpC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;oBAC3B,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC5D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;YAC7D,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,IAAY;QAC/B,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,UAAU,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;YAC7D,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC;CACF"}

package/build/crawler/factory.d.ts

@@ -1,6 +0,0 @@
-import { BaseCrawler } from './base.js';
-export declare class CrawlerFactory {
-    private static readonly JS_FRAMEWORK_INDICATORS;
-    private static detectSiteType;
-    static createCrawler(url: string, maxRequestsPerCrawl?: number, maxDepth?: number, onProgress?: (progress: number, description: string) => void): Promise<BaseCrawler>;
-}

package/build/crawler/factory.js

@@ -1,83 +0,0 @@
-import { URL } from 'url';
-import { DefaultCrawler } from './default.js';
-import { ChromiumCrawler } from './chromium.js';
-import { CheerioCrawler } from './cheerio.js';
-export class CrawlerFactory {
-    // Common JavaScript framework identifiers
-    static JS_FRAMEWORK_INDICATORS = [
-        'react',
-        'vue',
-        'angular',
-        'next',
-        'nuxt',
-        'gatsby',
-        'docusaurus',
-        'vuepress',
-        'gridsome',
-        'svelte'
-    ];
-    static async detectSiteType(url) {
-        try {
-            const response = await fetch(url, {
-                headers: {
-                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-                }
-            });
-            if (!response.ok) {
-                throw new Error(`HTTP error! status: ${response.status}`);
-            }
-            const html = await response.text();
-            // Check for JavaScript frameworks
-            const hasFramework = CrawlerFactory.JS_FRAMEWORK_INDICATORS.some(framework => html.toLowerCase().includes(framework));
-            // Check for JavaScript-heavy indicators
-            const isJsHeavy = (html.includes('data-react') ||
-                html.includes('ng-') ||
-                html.includes('v-') ||
-                html.includes('__NEXT_DATA__') ||
-                html.includes('nuxt') ||
-                html.includes('id="___gatsby"'));
-            return { isJsHeavy, hasFramework };
-        }
-        catch (e) {
-            console.error('[CrawlerFactory] Error detecting site type:', e);
-            return { isJsHeavy: false, hasFramework: false };
-        }
-    }
-    static async createCrawler(url, maxRequestsPerCrawl = 1000, maxDepth = 4, onProgress) {
-        const startUrl = new URL(url);
-        console.debug(`[CrawlerFactory] Creating crawler for ${startUrl}`);
-        // Check if site is JavaScript-heavy first
-        const { isJsHeavy, hasFramework } = await CrawlerFactory.detectSiteType(url);
-        // Try Chromium for JavaScript-heavy sites
-        if (isJsHeavy || hasFramework) {
-            console.debug(`[CrawlerFactory] Site appears to be JavaScript-heavy, using Chromium crawler`);
-            return new ChromiumCrawler(maxDepth, maxRequestsPerCrawl, onProgress);
-        }
-        // Try default crawler
-        try {
-            console.debug(`[CrawlerFactory] Attempting default crawler for ${url}`);
-            const defaultCrawler = new DefaultCrawler(maxDepth, maxRequestsPerCrawl, onProgress);
-            const generator = defaultCrawler.crawl(url);
-            const { value: firstPage, done } = await generator.next();
-            if (!done && firstPage?.content) {
-                console.debug('[CrawlerFactory] Successfully created default crawler');
-                return defaultCrawler;
-            }
-        }
-        catch (e) {
-            console.debug('[CrawlerFactory] Default crawler failed:', e);
-        }
-        // Fall back to Cheerio crawler
-        console.debug(`[CrawlerFactory] Attempting Cheerio crawler for ${url}`);
-        const cheerioCrawler = new CheerioCrawler(maxDepth, maxRequestsPerCrawl, onProgress);
-        const generator = cheerioCrawler.crawl(url);
-        const { value: firstPage, done } = await generator.next();
-        if (!done && firstPage?.content) {
-            console.debug('[CrawlerFactory] Successfully created Cheerio crawler');
-            return cheerioCrawler;
-        }
-        console.error(`[CrawlerFactory] All crawlers failed for ${url}`);
-        throw new Error(`Failed to create crawler for ${url}`);
-    }
-}
-//# sourceMappingURL=factory.js.map

package/build/crawler/factory.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"factory.js","sourceRoot":"","sources":["../../src/crawler/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,MAAM,OAAO,cAAc;IACzB,0CAA0C;IAClC,MAAM,CAAU,uBAAuB,GAAG;QAChD,OAAO;QACP,KAAK;QACL,SAAS;QACT,MAAM;QACN,MAAM;QACN,QAAQ;QACR,YAAY;QACZ,UAAU;QACV,UAAU;QACV,QAAQ;KACT,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,GAAW;QAI7C,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO,EAAE;oBACP,YAAY,EAAE,qHAAqH;iBACpI;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5D,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEnC,kCAAkC;YAClC,MAAM,YAAY,GAAG,cAAc,CAAC,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAC3E,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CACvC,CAAC;YAEF,wCAAwC;YACxC,MAAM,SAAS,GAAG,CAChB,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAC3B,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACpB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACnB,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;gBAC9B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACrB,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAChC,CAAC;YAEF,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,CAAC,CAAC,CAAC;YAChE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,aAAa,CACxB,GAAW,EACX,sBAA8B,IAAI,EAClC,WAAmB,CAAC,EACpB,UAA4D;QAE5D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,OAAO,CAAC,KAAK,CAAC,yCAAyC,QAAQ,EAAE,CAAC,CAAC;QAEnE,0CAA0C;QAC1C,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAE7E,0CAA0C;QAC1C,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;YAC9B,OAAO,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAC;YAC9F,OAAO,IAAI,eAAe,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;QACxE,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,mDAAmD,GAAG,EAAE,CAAC,CAAC;YACxE,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;YACrF,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;YAE1D,IAAI,CAAC,IAAI,IAAI,SAAS,EAAE,OAAO,EAAE,CAAC;gBAChC,OAAO,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;gBACvE,OAAO,cAAc,CAAC;YACxB,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,+BAA+B;QAC/B,OAAO,CAAC,KAAK,CAAC,mDAAmD,GAAG,EAAE,CAAC,CAAC;QACxE,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,QAAQ,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC;QACrF,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;QAE1D,IAAI,CAAC,IAAI,IAAI,SAAS,EAAE,OAAO,EAAE,CAAC;YAChC,OAAO,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;YACvE,OAAO,cAAc,CAAC;QACxB,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,4CAA4C,GAAG,EAAE,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,gCAAgC,GAAG,EAAE,CAAC,CAAC;IACzD,CAAC"}

package/build/crawler/puppeteer.d.ts

@@ -1,16 +0,0 @@
-import { CrawlResult } from '../types.js';
-import { BaseCrawler } from './base.js';
-export declare class PuppeteerCrawler extends BaseCrawler {
-    private browser?;
-    private readonly userAgent;
-    private readonly LINK_GROUP_SIZE;
-    private curCrawlCount;
-    crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
-    private setupPage;
-    private crawlSitePages;
-    private gotoPageAndHandleRedirects;
-    private processPage;
-    private getLinksFromPage;
-    private groupLinks;
-    abort(): void;
-}

package/build/crawler/puppeteer.js

@@ -1,191 +0,0 @@
-import puppeteer from 'puppeteer';
-import { BaseCrawler } from './base.js';
-import { JSDOM } from 'jsdom';
-import { Readability } from '@mozilla/readability';
-export class PuppeteerCrawler extends BaseCrawler {
-    browser;
-    userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36';
-    LINK_GROUP_SIZE = 2;
-    curCrawlCount = 0;
-    async *crawl(url) {
-        try {
-            this.browser = await puppeteer.launch({
-                headless: true,
-                args: [
-                    '--no-sandbox',
-                    '--disable-setuid-sandbox',
-                    '--disable-dev-shm-usage',
-                    '--disable-gpu',
-                    '--window-size=1280,800'
-                ]
-            });
-            const page = await this.browser.newPage();
-            await this.setupPage(page);
-            const visitedUrls = new Set();
-            yield* this.crawlSitePages(page, new URL(url), 0, visitedUrls);
-        }
-        finally {
-            await this.browser?.close();
-        }
-    }
-    async setupPage(page) {
-        await page.setUserAgent(this.userAgent);
-        await page.setViewport({ width: 1280, height: 800 });
-        // Block only unnecessary resources
-        await page.setRequestInterception(true);
-        page.on('request', request => {
-            const resourceType = request.resourceType();
-            if (['image', 'media', 'font'].includes(resourceType)) {
-                request.abort();
-            }
-            else {
-                request.continue();
-            }
-        });
-        // Handle JavaScript errors
-        page.on('pageerror', error => {
-            console.warn('Page error:', error);
-        });
-        // Handle console messages
-        page.on('console', (msg) => {
-            const type = msg.type();
-            if (type === 'error' || type === 'warn') {
-                console.debug(`Console ${type}:`, msg.text());
-            }
-        });
-    }
-    async *crawlSitePages(page, curUrl, depth, visitedUrls) {
-        const urlStr = curUrl.toString();
-        if (visitedUrls.has(urlStr) || !this.shouldCrawl(urlStr) || depth > this.maxDepth) {
-            return;
-        }
-        try {
-            // Rate limiting
-            await this.rateLimit();
-            // Navigate to page with proper redirect handling
-            await this.gotoPageAndHandleRedirects(page, urlStr);
-            // Extract content
-            const { content, title, links } = await this.processPage(page, curUrl);
-            visitedUrls.add(urlStr);
-            this.markUrlAsSeen(urlStr);
-            this.curCrawlCount++;
-            yield {
-                url: urlStr,
-                path: this.getPathFromUrl(urlStr),
-                content,
-                title
-            };
-            // Process links in batches
-            if (depth < this.maxDepth && this.curCrawlCount < this.maxRequestsPerCrawl) {
-                const linkGroups = this.groupLinks(links);
-                for (const linkGroup of linkGroups) {
-                    for (const link of linkGroup) {
-                        if (this.curCrawlCount >= this.maxRequestsPerCrawl) {
-                            return;
-                        }
-                        yield* this.crawlSitePages(page, new URL(link), depth + 1, visitedUrls);
-                    }
-                }
-            }
-        }
-        catch (error) {
-            console.error(`Error crawling ${urlStr}:`, error);
-        }
-    }
-    async gotoPageAndHandleRedirects(page, url) {
-        const MAX_PAGE_WAIT_MS = 5000;
-        await page.goto(url, {
-            timeout: 0,
-            waitUntil: 'networkidle2'
-        });
-        let responseEventOccurred = false;
-        const responseHandler = () => responseEventOccurred = true;
-        const responseWatcher = new Promise((resolve) => {
-            setTimeout(() => {
-                if (!responseEventOccurred) {
-                    resolve();
-                }
-                else {
-                    setTimeout(() => resolve(), MAX_PAGE_WAIT_MS);
-                }
-            }, 500);
-        });
-        page.on('response', responseHandler);
-        await Promise.race([responseWatcher, page.waitForNavigation()]);
-        page.off('response', responseHandler);
-    }
-    async processPage(page, url) {
-        // Wait for dynamic content
-        try {
-            await page.waitForFunction(() => {
-                const mainContent = document.querySelector('main') || document.querySelector('.content') || document.querySelector('#content');
-                return mainContent && mainContent.children.length > 0;
-            }, { timeout: 5000 });
-        }
-        catch (error) {
-            console.warn('Timeout waiting for main content, proceeding anyway');
-        }
-        // Extract content using Readability
-        const html = await page.content();
-        const dom = new JSDOM(html, { url: url.toString() });
-        const reader = new Readability(dom.window.document, {
-            charThreshold: 20,
-            nbTopCandidates: 5,
-            maxElemsToParse: 10000
-        });
-        const article = reader.parse();
-        if (!article) {
-            throw new Error('Failed to parse page content');
-        }
-        // Extract links
-        const links = await this.getLinksFromPage(page, url);
-        return {
-            content: article.textContent,
-            title: article.title,
-            links
-        };
-    }
-    async getLinksFromPage(page, curUrl) {
-        const links = await page.$$eval('a', (links) => links.map((a) => a.href));
-        const cleanedLinks = links
-            .map(link => {
-            try {
-                const url = new URL(link);
-                url.hash = ''; // Remove hash
-                return url.href;
-            }
-            catch {
-                return null;
-            }
-        })
-            .filter((link) => {
-            if (!link)
-                return false;
-            try {
-                const url = new URL(link);
-                return (url.pathname.startsWith(curUrl.pathname) &&
-                    url.hostname === curUrl.hostname &&
-                    link !== curUrl.href);
-            }
-            catch {
-                return false;
-            }
-        });
-        return Array.from(new Set(cleanedLinks));
-    }
-    groupLinks(links) {
-        return links.reduce((acc, link, i) => {
-            const groupIndex = Math.floor(i / this.LINK_GROUP_SIZE);
-            if (!acc[groupIndex]) {
-                acc.push([]);
-            }
-            acc[groupIndex].push(link);
-            return acc;
-        }, []);
-    }
-    abort() {
-        super.abort();
-        void this.browser?.close();
-    }
-}
-//# sourceMappingURL=puppeteer.js.map

package/build/crawler/puppeteer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"puppeteer.js","sourceRoot":"","sources":["../../src/crawler/puppeteer.ts"],"names":[],"mappings":"AAAA,OAAO,SAA4C,MAAM,WAAW,CAAC;AAErE,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,MAAM,OAAO,gBAAiB,SAAQ,WAAW;IACvC,OAAO,CAAW;IACT,SAAS,GAAG,qHAAqH,CAAC;IAClI,eAAe,GAAG,CAAC,CAAC;IAC7B,aAAa,GAAG,CAAC,CAAC;IAE1B,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC;gBACpC,QAAQ,EAAE,IAAI;gBACd,IAAI,EAAE;oBACJ,cAAc;oBACd,0BAA0B;oBAC1B,yBAAyB;oBACzB,eAAe;oBACf,wBAAwB;iBACzB;aACF,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAE3B,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;YACtC,KAAK,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC;QACjE,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;QAC9B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,IAAU;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAErD,mCAAmC;QACnC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE;YAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YAC5C,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACtD,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,2BAA2B;QAC3B,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE;YAC3B,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,0BAA0B;QAC1B,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAmB,EAAE,EAAE;YACzC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;YACxB,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;gBACxC,OAAO,CAAC,KAAK,CAAC,WAAW,IAAI,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,CAAC,cAAc,CAC3B,IAAU,EACV,MAAW,EACX,KAAa,EACb,WAAwB;QAExB,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;QAEjC,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClF,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,gBAAgB;YAChB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YAEvB,iDAAiD;YACjD,MAAM,IAAI,CAAC,0BAA0B,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEpD,kBAAkB;YAClB,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEvE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACxB,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC3B,IAAI,CAAC,aAAa,EAAE,CAAC;YAErB,MAAM;gBACJ,GAAG,EAAE,MAAM;gBACX,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;gBACjC,OAAO;gBACP,KAAK;aACN,CAAC;YAEF,2BAA2B;YAC3B,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC3E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC1C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;oBACnC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;4BACnD,OAAO;wBACT,CAAC;wBACD,KAAK,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;oBAC1E,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,MAAM,GAAG,EAAE,KAAK,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,0BAA0B,CAAC,IAAU,EAAE,GAAW;QAC9D,MAAM,gBAAgB,GAAG,IAAI,CAAC;QAE9B,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACnB,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,cAAc;SAC1B,CAAC,CAAC;QAEH,IAAI,qBAAqB,GAAG,KAAK,CAAC;QAClC,MAAM,eAAe,GAAG,GAAG,EAAE,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAE3D,MAAM,eAAe,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACpD,UAAU,CAAC,GAAG,EAAE;gBACd,IAAI,CAAC,qBAAqB,EAAE,CAAC;oBAC3B,OAAO,EAAE,CAAC;gBACZ,CAAC;qBAAM,CAAC;oBACN,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,EAAE,gBAAgB,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC,EAAE,GAAG,CAAC,CAAC;QACV,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QACrC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,eAAe,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;QAChE,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;IACxC,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAU,EAAE,GAAQ;QAC5C,2BAA2B;QAC3B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE;gBAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;gBAC/H,OAAO,WAAW,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACxD,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACtE,CAAC;QAED,oCAAoC;QACpC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE;YAClD,aAAa,EAAE,EAAE;YACjB,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,KAAK;SACvB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;QAED,gBAAgB;QAChB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAErD,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,WAAW;YAC5B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,KAAK;SACN,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAU,EAAE,MAAW;QACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAE1E,MAAM,YAAY,GAAG,KAAK;aACvB,GAAG,CAAC,IAAI,CAAC,EAAE;YACV,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC1B,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,cAAc;gBAC7B,OAAO,GAAG,CAAC,IAAI,CAAC;YAClB,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE;YAC/B,IAAI,CAAC,IAAI;gBAAE,OAAO,KAAK,CAAC;YACxB,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC1B,OAAO,CACL,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC;oBACxC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ;oBAChC,IAAI,KAAK,MAAM,CAAC,IAAI,CACrB,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;QAEL,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;IAC3C,CAAC;IAEO,UAAU,CAAC,KAAe;QAChC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;YACnC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;YACxD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBACrB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,CAAC;YACD,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAgB,CAAC,CAAC;IACvB,CAAC;IAED,KAAK;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;IAC7B,CAAC;CACF"}

package/build/embeddings/openai.d.ts

@@ -1,8 +0,0 @@
-import { EmbeddingsProvider } from './types.js';
-export declare class OpenAIEmbeddings implements EmbeddingsProvider {
-    private openai;
-    private cache;
-    readonly dimensions = 1536;
-    constructor(apiKey: string);
-    embed(text: string): Promise<number[]>;
-}

package/build/embeddings/openai.js

@@ -1,56 +0,0 @@
-import OpenAI from 'openai';
-import { logger } from '../util/logger.js';
-export class OpenAIEmbeddings {
-    openai;
-    cache;
-    dimensions = 1536; // text-embedding-3-small dimensions
-    constructor(apiKey) {
-        if (!apiKey) {
-            throw new Error('OpenAI API key is required');
-        }
-        this.openai = new OpenAI({ apiKey });
-        this.cache = new Map();
-    }
-    async embed(text) {
-        // Ensure input is a string and not empty
-        if (!text || typeof text !== 'string') {
-            throw new Error('Input text must be a non-empty string');
-        }
-        // Check cache first
-        const cacheKey = text.slice(0, 1000); // Limit cache key size
-        const cached = this.cache.get(cacheKey);
-        if (cached) {
-            return cached;
-        }
-        try {
-            const cleanText = text.trim();
-            if (!cleanText) {
-                throw new Error('Input text is empty after trimming');
-            }
-            const response = await this.openai.embeddings.create({
-                model: "text-embedding-3-small",
-                input: cleanText,
-                dimensions: this.dimensions
-            });
-            if (!response.data?.[0]?.embedding) {
-                throw new Error('No embedding returned from OpenAI');
-            }
-            const embedding = response.data[0].embedding;
-            // Cache the result
-            this.cache.set(cacheKey, embedding);
-            // Limit cache size to prevent memory issues
-            if (this.cache.size > 1000) {
-                const firstKey = this.cache.keys().next().value;
-                if (firstKey) {
-                    this.cache.delete(firstKey);
-                }
-            }
-            return embedding;
-        }
-        catch (error) {
-            logger.debug('Error generating embedding:', error);
-            throw error;
-        }
-    }
-}
-//# sourceMappingURL=openai.js.map

package/build/embeddings/openai.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/embeddings/openai.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,gBAAgB;IACnB,MAAM,CAAS;IACf,KAAK,CAAwB;IAC5B,UAAU,GAAG,IAAI,CAAC,CAAC,oCAAoC;IAEhE,YAAY,MAAc;QACxB,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,yCAAyC;QACzC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,oBAAoB;QACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,uBAAuB;QAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACxD,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACnD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,SAAS;gBAChB,UAAU,EAAE,IAAI,CAAC,UAAU;aAC5B,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YACvD,CAAC;YAED,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAE7C,mBAAmB;YACnB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAEpC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;gBAChD,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,CAAC;YACnD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}

package/build/rag/cache.d.ts

@@ -1,10 +0,0 @@
-import { GeneratedResponse } from "./response-generator.js";
-export interface CachedResponse {
-    query: string;
-    response: GeneratedResponse;
-}
-export declare class RAGCache {
-    private cache;
-    getCachedResponse(query: string): Promise<CachedResponse | null>;
-    cacheResponse(query: string, response: GeneratedResponse): Promise<void>;
-}

package/build/rag/cache.js

@@ -1,10 +0,0 @@
-export class RAGCache {
-    cache = new Map();
-    async getCachedResponse(query) {
-        return this.cache.get(query) || null;
-    }
-    async cacheResponse(query, response) {
-        this.cache.set(query, { query, response });
-    }
-}
-//# sourceMappingURL=cache.js.map

package/build/rag/cache.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/rag/cache.ts"],"names":[],"mappings":"AAOA,MAAM,OAAO,QAAQ;IACX,KAAK,GAAgC,IAAI,GAAG,EAAE,CAAC;IAEvD,KAAK,CAAC,iBAAiB,CAAC,KAAa;QACnC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAa,EAAE,QAA2B;QAC5D,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC7C,CAAC;CACF"}

package/build/rag/code-generator.d.ts

@@ -1,11 +0,0 @@
-export interface CodeExample {
-    imports: string;
-    props: string;
-    usage: string;
-}
-export declare class CodeGenerator {
-    generateCodeExample(component: any, context: any): Promise<CodeExample>;
-    private generateImports;
-    private generateProps;
-    private generateUsage;
-}

package/build/rag/code-generator.js

@@ -1,30 +0,0 @@
-export class CodeGenerator {
-    async generateCodeExample(component, context) {
-        // Extract component name and props from the component info
-        const componentName = component.name || 'Example';
-        const componentProps = component.props || {};
-        // Generate imports based on context
-        const imports = this.generateImports(componentName, context);
-        // Generate props string
-        const props = this.generateProps(componentProps);
-        // Generate usage example
-        const usage = this.generateUsage(componentName, props);
-        return { imports, props, usage };
-    }
-    generateImports(componentName, context) {
-        // Look for import path in context
-        const importPath = context.importPath || 'example';
-        return `import ${componentName} from '${importPath}';`;
-    }
-    generateProps(props) {
-        return Object.entries(props)
-            .map(([key, value]) => `${key}: ${JSON.stringify(value)}`)
-            .join(', ');
-    }
-    generateUsage(componentName, props) {
-        return props
-            ? `<${componentName} ${props} />`
-            : `<${componentName} />`;
-    }
-}
-//# sourceMappingURL=code-generator.js.map

package/build/rag/code-generator.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"code-generator.js","sourceRoot":"","sources":["../../src/rag/code-generator.ts"],"names":[],"mappings":"AAMA,MAAM,OAAO,aAAa;IACxB,KAAK,CAAC,mBAAmB,CAAC,SAAc,EAAE,OAAY;QACpD,2DAA2D;QAC3D,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC;QAClD,MAAM,cAAc,GAAG,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC;QAE7C,oCAAoC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QAE7D,wBAAwB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;QAEjD,yBAAyB;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IACnC,CAAC;IAEO,eAAe,CAAC,aAAqB,EAAE,OAAY;QACzD,kCAAkC;QAClC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,SAAS,CAAC;QACnD,OAAO,UAAU,aAAa,UAAU,UAAU,IAAI,CAAC;IACzD,CAAC;IAEO,aAAa,CAAC,KAA0B;QAC9C,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;aACzB,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;aACzD,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAEO,aAAa,CAAC,aAAqB,EAAE,KAAa;QACxD,OAAO,KAAK;YACV,CAAC,CAAC,IAAI,aAAa,IAAI,KAAK,KAAK;YACjC,CAAC,CAAC,IAAI,aAAa,KAAK,CAAC;IAC7B,CAAC;CACF"}

package/build/rag/context-assembler.d.ts

@@ -1,23 +0,0 @@
-import { EnhancedChunk } from "../types/rag.js";
-import { ComponentRelationship } from "../crawler/content-extractor-types.js";
-export interface AssembledContext {
-    hierarchicalContext: EnhancedChunk[];
-    relationships: ComponentRelationship[];
-    metadata: {
-        summary: string;
-        topics: string[];
-        complexity: 'basic' | 'intermediate' | 'advanced';
-        prerequisites: string[];
-        frameworks: string[];
-        languages: string[];
-    };
-}
-export declare class ContextAssembler {
-    assembleContext(chunks: EnhancedChunk[]): Promise<AssembledContext>;
-    private groupChunksByType;
-    private extractRelationships;
-    private buildHierarchy;
-    private consolidateMetadata;
-    private generateSummary;
-    private deduplicateRelationships;
-}