@gulibs/safe-coder 0.0.25 → 0.0.27

package/dist/documentation/doc-crawler.js

@@ -1,1162 +0,0 @@
-import { HttpClient } from '../utils/http-client.js';
-import { logger } from '../utils/logger.js';
-import { WebDocumentationBrowser } from './web-doc-browser.js';
-import { LlmsTxtDetector, LlmsTxtDownloader, LlmsTxtParser } from './llms-txt/index.js';
-import { CheckpointManager } from './checkpoint-manager.js';
-import { join } from 'path';
-import { tmpdir } from 'os';
-export class DocumentationCrawler {
-    browser;
-    visitedUrls;
-    urlQueue;
-    crawledPages;
-    errors;
-    options;
-    baseUrl;
-    linkDiscoveryStats;
-    checkpointManager;
-    pagesSinceLastCheckpoint;
-    DOCUMENTATION_PATTERNS = [
-        /\/docs?\//i,
-        /\/documentation/i,
-        /\/guide/i,
-        /\/tutorial/i,
-        /\/api/i,
-        /\/reference/i,
-        /\/manual/i,
-        /\/help/i,
-        /\/about/i,
-        /\/getting-started/i,
-    ];
-    EXCLUDED_PATTERNS = [
-        /\/login/i,
-        /\/signup/i,
-        /\/register/i,
-        /\/checkout/i,
-        /\/cart/i,
-        /\/payment/i,
-        /\/home$/i,
-        // Don't exclude root path - it might be documentation
-        // /^\/$/,
-    ];
-    constructor(httpClient) {
-        this.browser = new WebDocumentationBrowser(httpClient);
-        this.visitedUrls = new Set();
-        this.urlQueue = [];
-        this.crawledPages = [];
-        this.errors = [];
-        this.options = {
-            crawlStrategy: 'bfs', // Default to breadth-first search
-            maxDepth: 3,
-            maxPages: 50,
-            includePaths: [],
-            excludePaths: [],
-            rateLimit: 500, // 500ms default delay
-            maxRetries: 2, // Default 2 retries
-            retryDelay: 1000, // Default 1 second delay before retry
-            useBrowserAutomation: false, // Default to HTTP-only for backward compatibility
-            skipLlmsTxt: false, // Enable llms.txt detection by default
-            workers: 1, // Default to single-threaded crawling
-        };
-        this.baseUrl = new URL('https://example.com');
-        this.linkDiscoveryStats = {
-            totalLinksFound: 0,
-            linksFiltered: {
-                notContent: 0,
-                externalDomain: 0,
-                alreadyVisited: 0,
-                excludedPattern: 0,
-                depthLimit: 0,
-            },
-            linksQueued: 0,
-            pagesDiscovered: 0,
-            pagesCrawled: 0,
-        };
-        this.pagesSinceLastCheckpoint = 0;
-    }
-    /**
-     * Crawl documentation starting from a root URL
-     * Uses HTTP client (axios) exclusively - no browser automation
-     * For SPA sites that require JavaScript rendering, use Cursor/Claude's built-in browser tools
-     * Supports both BFS (breadth-first) and DFS (depth-first) crawl strategies
-     */
-    async crawl(rootUrl, options = {}) {
-        const strategy = options.crawlStrategy || 'bfs';
-        logger.info('Starting documentation crawl using HTTP client (axios)', {
-            url: rootUrl,
-            strategy,
-            method: 'HTTP GET',
-            client: 'axios/HttpClient',
-            note: 'For SPA sites, use Cursor/Claude browser tools to get rendered content first',
-        });
-        // Reset state
-        this.visitedUrls.clear();
-        this.urlQueue = [];
-        this.crawledPages = [];
-        this.errors = [];
-        this.linkDiscoveryStats = {
-            totalLinksFound: 0,
-            linksFiltered: {
-                notContent: 0,
-                externalDomain: 0,
-                alreadyVisited: 0,
-                excludedPattern: 0,
-                depthLimit: 0,
-            },
-            linksQueued: 0,
-            pagesDiscovered: 0,
-            pagesCrawled: 0,
-        };
-        // Merge options
-        this.options = {
-            ...this.options,
-            ...options,
-        };
-        // Parse and validate root URL
-        try {
-            this.baseUrl = new URL(rootUrl);
-        }
-        catch (error) {
-            throw new Error(`Invalid root URL: ${rootUrl}`);
-        }
-        // No longer require documentation-only pages - allow any website with extractable content
-        logger.debug('Starting crawl from URL (permissive mode)', { url: rootUrl });
-        // Setup checkpoint manager if enabled
-        if (this.options.checkpoint?.enabled) {
-            const checkpointFile = this.options.checkpoint.file ||
-                join(tmpdir(), `safe-coder-checkpoint-${this.sanitizeFilename(rootUrl)}.json`);
-            this.checkpointManager = new CheckpointManager(checkpointFile);
-            // Try to resume from checkpoint if requested
-            if (this.options.resume) {
-                const loaded = await this.loadCheckpoint();
-                if (loaded) {
-                    logger.info('Resumed from checkpoint', {
-                        pagesCrawled: this.crawledPages.length,
-                        pendingUrls: this.urlQueue.length,
-                        visitedUrls: this.visitedUrls.size,
-                    });
-                }
-            }
-        }
-        // Try to detect and use llms.txt if available (unless explicitly disabled)
-        if (!this.options.skipLlmsTxt) {
-            await this.tryLlmsTxt(rootUrl);
-        }
-        // Detect SPA and provide warning
-        try {
-            const spaDetection = await this.browser.detectSPA(rootUrl);
-            if (spaDetection.isSPA && spaDetection.confidence !== 'low') {
-                logger.warn('SPA detected at root URL - crawling may be limited', {
-                    url: rootUrl,
-                    confidence: spaDetection.confidence,
-                    indicators: spaDetection.indicators,
-                    suggestion: spaDetection.suggestion,
-                });
-                // Add warning to first page if SPA detected
-                if (spaDetection.suggestion) {
-                    logger.info('SPA Detection Warning', {
-                        message: spaDetection.suggestion,
-                        recommendation: 'Consider using browser automation tools to get fully rendered content before crawling.',
-                    });
-                }
-            }
-        }
-        catch (error) {
-            // SPA detection failure is not critical, continue crawling
-            logger.debug('SPA detection failed, continuing with crawl', {
-                url: rootUrl,
-                error: error instanceof Error ? error.message : String(error),
-            });
-        }
-        // Start crawling from root
-        this.urlQueue.push({ url: rootUrl, depth: 0 });
-        let maxDepthReached = 0;
-        // Process queue - use parallel workers if specified
-        const startTime = Date.now();
-        const workerCount = this.options.workers || 1;
-        if (workerCount > 1) {
-            logger.info('Using parallel crawling', { workers: workerCount });
-            maxDepthReached = await this.crawlWithWorkers(startTime);
-        }
-        else {
-            maxDepthReached = await this.crawlSequential(startTime);
-        }
-        // Update final statistics
-        this.linkDiscoveryStats.pagesDiscovered = this.visitedUrls.size;
-        // Calculate final statistics
-        const totalTime = ((Date.now() - startTime) / 1000).toFixed(2);
-        const avgTimePerPage = this.crawledPages.length > 0
-            ? ((Date.now() - startTime) / this.crawledPages.length / 1000).toFixed(2)
-            : '0';
-        const successRate = this.linkDiscoveryStats.pagesDiscovered > 0
-            ? ((this.crawledPages.length / this.linkDiscoveryStats.pagesDiscovered) * 100).toFixed(1)
-            : '0';
-        // Log crawl completion with comprehensive statistics
-        logger.info('Documentation crawl completed using HTTP client (axios)', {
-            totalPages: this.crawledPages.length,
-            maxDepthReached,
-            errors: this.errors.length,
-            totalTimeSeconds: totalTime,
-            avgTimePerPageSeconds: avgTimePerPage,
-            successRate: `${successRate}%`,
-            method: 'HTTP GET',
-            client: 'axios/HttpClient',
-            linkStats: {
-                totalLinksFound: this.linkDiscoveryStats.totalLinksFound,
-                linksQueued: this.linkDiscoveryStats.linksQueued,
-                linksFiltered: this.linkDiscoveryStats.linksFiltered,
-                pagesDiscovered: this.linkDiscoveryStats.pagesDiscovered,
-                pagesCrawled: this.linkDiscoveryStats.pagesCrawled,
-            },
-            errorBreakdown: this.getErrorBreakdown(),
-        });
-        // Validate if content is sufficient for skill generation
-        const validation = this.canGenerateSkill(this.crawledPages);
-        const abandoned = !validation.canGenerate;
-        const abandonReason = validation.reason;
-        if (abandoned) {
-            logger.warn('Crawl completed but content is insufficient for skill generation', {
-                reason: abandonReason,
-                pagesCrawled: this.crawledPages.length,
-                suggestion: 'Consider crawling more pages or a different website',
-            });
-        }
-        // Clear checkpoint after successful completion
-        if (this.checkpointManager && !abandoned) {
-            await this.clearCheckpoint();
-        }
-        return {
-            pages: this.crawledPages,
-            totalPages: this.crawledPages.length,
-            maxDepthReached,
-            errors: this.errors,
-            linkDiscoveryStats: this.linkDiscoveryStats,
-            abandoned,
-            abandonReason,
-        };
-    }
-    /**
-     * Sequential crawling (single-threaded)
-     */
-    async crawlSequential(startTime) {
-        let maxDepthReached = 0;
-        let lastProgressLog = Date.now();
-        const PROGRESS_LOG_INTERVAL = 5000; // Log progress every 5 seconds
-        while (this.urlQueue.length > 0 && this.crawledPages.length < this.options.maxPages) {
-            // Use different strategies for getting next URL
-            // BFS: shift() - take from front (queue behavior)
-            // DFS: pop() - take from back (stack behavior)
-            const queued = this.options.crawlStrategy === 'dfs' ? this.urlQueue.pop() : this.urlQueue.shift();
-            if (!queued)
-                break;
-            const { url, depth } = queued;
-            // Log progress periodically
-            const now = Date.now();
-            if (now - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
-                const elapsed = ((now - startTime) / 1000).toFixed(1);
-                const pagesPerSecond = (this.crawledPages.length / elapsed).toFixed(2);
-                logger.info('Crawl progress', {
-                    pagesCrawled: this.crawledPages.length,
-                    pagesRemaining: this.urlQueue.length,
-                    maxPages: this.options.maxPages,
-                    errors: this.errors.length,
-                    elapsedSeconds: elapsed,
-                    pagesPerSecond,
-                    currentDepth: depth,
-                    maxDepth: this.options.maxDepth,
-                });
-                lastProgressLog = now;
-            }
-            // Skip if already visited
-            if (this.visitedUrls.has(url)) {
-                continue;
-            }
-            // Check depth limit
-            if (depth > this.options.maxDepth) {
-                continue;
-            }
-            // Mark as visited
-            this.visitedUrls.add(url);
-            maxDepthReached = Math.max(maxDepthReached, depth);
-            await this.processPage(url, depth);
-            // Rate limiting
-            if (this.options.rateLimit > 0 && this.urlQueue.length > 0) {
-                await this.delay(this.options.rateLimit);
-            }
-        }
-        return maxDepthReached;
-    }
-    /**
-     * Parallel crawling with multiple workers
-     */
-    async crawlWithWorkers(startTime) {
-        let maxDepthReached = 0;
-        let lastProgressLog = Date.now();
-        const PROGRESS_LOG_INTERVAL = 5000;
-        const workerCount = this.options.workers || 1;
-        while (this.urlQueue.length > 0 && this.crawledPages.length < this.options.maxPages) {
-            // Log progress periodically
-            const now = Date.now();
-            if (now - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
-                const elapsed = ((now - startTime) / 1000).toFixed(1);
-                const pagesPerSecond = (this.crawledPages.length / elapsed).toFixed(2);
-                logger.info('Crawl progress (parallel)', {
-                    pagesCrawled: this.crawledPages.length,
-                    pagesRemaining: this.urlQueue.length,
-                    maxPages: this.options.maxPages,
-                    errors: this.errors.length,
-                    elapsedSeconds: elapsed,
-                    pagesPerSecond,
-                    workers: workerCount,
-                });
-                lastProgressLog = now;
-            }
-            // Get batch of URLs to process in parallel
-            const batch = [];
-            const batchSize = Math.min(workerCount, this.urlQueue.length, this.options.maxPages - this.crawledPages.length);
-            for (let i = 0; i < batchSize; i++) {
-                const queued = this.options.crawlStrategy === 'dfs' ? this.urlQueue.pop() : this.urlQueue.shift();
-                if (!queued)
-                    break;
-                // Skip if already visited
-                if (this.visitedUrls.has(queued.url)) {
-                    continue;
-                }
-                // Check depth limit
-                if (queued.depth > this.options.maxDepth) {
-                    continue;
-                }
-                // Mark as visited
-                this.visitedUrls.add(queued.url);
-                maxDepthReached = Math.max(maxDepthReached, queued.depth);
-                batch.push(queued);
-            }
-            if (batch.length === 0) {
-                break;
-            }
-            // Process batch in parallel
-            await Promise.all(batch.map(async (queued) => {
-                await this.processPage(queued.url, queued.depth);
-                // Rate limiting (per worker)
-                if (this.options.rateLimit > 0) {
-                    await this.delay(this.options.rateLimit);
-                }
-            }));
-        }
-        return maxDepthReached;
-    }
-    /**
-     * Process a single page (shared by both sequential and parallel crawling)
-     */
-    async processPage(url, depth) {
-        try {
-            // Crawl the page using HTTP GET with retry logic
-            logger.debug('Fetching page via HTTP GET', { url, depth, method: 'HTTP GET', client: 'axios' });
-            const page = await this.fetchPageWithRetry(url);
-            // Check if page has minimal content (possible SPA issue)
-            const contentLength = page.content.length;
-            const linksCount = page.navigationLinks.length;
-            if (contentLength < 200 && linksCount < 3) {
-                logger.warn('Page has minimal content - may be SPA', {
-                    url,
-                    contentLength,
-                    linksCount,
-                    suggestion: 'This page may require JavaScript rendering. Consider using browser automation tools.',
-                });
-            }
-            // Convert to CrawledPage format
-            const crawledPage = {
-                url: page.url,
-                title: page.title,
-                content: page.content,
-                depth,
-                sections: page.sections,
-                navigationLinks: page.navigationLinks,
-                headings: page.headings,
-                codeSamples: page.codeSamples,
-            };
-            this.crawledPages.push(crawledPage);
-            this.linkDiscoveryStats.pagesCrawled++;
-            this.pagesSinceLastCheckpoint++;
-            // Save checkpoint if interval reached
-            if (this.checkpointManager && this.options.checkpoint?.enabled) {
-                const interval = this.options.checkpoint.interval || 10;
-                if (this.pagesSinceLastCheckpoint >= interval) {
-                    await this.saveCheckpoint();
-                    this.pagesSinceLastCheckpoint = 0;
-                }
-            }
-            const totalLinksOnPage = page.navigationLinks.length;
-            this.linkDiscoveryStats.totalLinksFound += totalLinksOnPage;
-            logger.debug('Page fetched and parsed successfully', {
-                url,
-                title: page.title.substring(0, 50),
-                linksFound: totalLinksOnPage,
-                depth,
-            });
-            // Discover and queue new URLs
-            if (depth < this.options.maxDepth) {
-                const discoveryResult = this.discoverDocumentationLinks(page, depth + 1);
-                const newUrls = discoveryResult.discovered;
-                logger.debug('Link discovery completed', {
-                    url,
-                    totalLinksOnPage,
-                    discovered: newUrls.length,
-                    filtered: discoveryResult.filtered,
-                });
-                let queuedCount = 0;
-                let skippedAlreadyVisited = 0;
-                for (const newUrl of newUrls) {
-                    if (!this.visitedUrls.has(newUrl.url)) {
-                        // Also check if it's already in the queue to avoid duplicates
-                        const alreadyInQueue = this.urlQueue.some(q => q.url === newUrl.url);
-                        if (!alreadyInQueue) {
-                            this.urlQueue.push(newUrl);
-                            this.linkDiscoveryStats.linksQueued++;
-                            queuedCount++;
-                        }
-                        else {
-                            skippedAlreadyVisited++;
-                        }
-                    }
-                    else {
-                        skippedAlreadyVisited++;
-                    }
-                }
-                logger.debug('Links queued', {
-                    url,
-                    queued: queuedCount,
-                    skippedAlreadyVisited,
-                    queueLengthAfter: this.urlQueue.length,
-                });
-            }
-            else {
-                this.linkDiscoveryStats.linksFiltered.depthLimit += totalLinksOnPage;
-            }
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            const errorType = this.classifyError(error);
-            this.errors.push({
-                url,
-                error: `${errorType}: ${errorMessage}`,
-            });
-            logger.warn('Page crawl failed', {
-                url,
-                error: errorMessage,
-                errorType,
-                depth,
-                willContinue: true,
-            });
-        }
-    }
-    /**
-     * Discover documentation links from a crawled page
-     */
-    discoverDocumentationLinks(page, nextDepth) {
-        const discovered = [];
-        const filtered = {
-            notContent: 0, // Renamed from notDocumentation
-            externalDomain: 0,
-            alreadyVisited: 0,
-            excludedPattern: 0,
-        };
-        const linkDetails = [];
-        for (const link of page.navigationLinks) {
-            // Only follow internal links
-            if (!link.isInternal) {
-                filtered.externalDomain++;
-                this.linkDiscoveryStats.linksFiltered.externalDomain++;
-                linkDetails.push({ url: link.url, reason: 'not_internal' });
-                continue;
-            }
-            try {
-                const linkUrl = new URL(link.url);
-                // Must be same origin
-                if (linkUrl.origin !== this.baseUrl.origin) {
-                    filtered.externalDomain++;
-                    this.linkDiscoveryStats.linksFiltered.externalDomain++;
-                    linkDetails.push({ url: link.url, reason: 'different_origin' });
-                    continue;
-                }
-                // Check if already visited
-                const normalizedUrl = linkUrl.href.split('#')[0];
-                if (this.visitedUrls.has(normalizedUrl)) {
-                    filtered.alreadyVisited++;
-                    this.linkDiscoveryStats.linksFiltered.alreadyVisited++;
-                    linkDetails.push({ url: link.url, reason: 'already_visited' });
-                    continue;
-                }
-                // Check if it's a valid content path (permissive - only exclude clearly non-content)
-                if (!this.isDocumentationPath(linkUrl.pathname)) {
-                    filtered.notContent++;
-                    this.linkDiscoveryStats.linksFiltered.notContent++;
-                    linkDetails.push({ url: link.url, reason: 'not_content_path', pathname: linkUrl.pathname });
-                    continue;
-                }
-                // Check exclude patterns
-                if (this.shouldExclude(linkUrl.pathname)) {
-                    filtered.excludedPattern++;
-                    this.linkDiscoveryStats.linksFiltered.excludedPattern++;
-                    linkDetails.push({ url: link.url, reason: 'excluded_pattern', pathname: linkUrl.pathname });
-                    continue;
-                }
-                // Check include patterns (if specified)
-                if (this.options.includePaths.length > 0) {
-                    const matchesInclude = this.options.includePaths.some(pattern => linkUrl.pathname.includes(pattern));
-                    if (!matchesInclude) {
-                        filtered.notContent++;
-                        this.linkDiscoveryStats.linksFiltered.notContent++;
-                        linkDetails.push({ url: link.url, reason: 'not_in_include_paths', pathname: linkUrl.pathname });
-                        continue;
-                    }
-                }
-                // Check exclude patterns (if specified)
-                if (this.options.excludePaths.length > 0) {
-                    const matchesExclude = this.options.excludePaths.some(pattern => linkUrl.pathname.includes(pattern));
-                    if (matchesExclude) {
-                        filtered.excludedPattern++;
-                        this.linkDiscoveryStats.linksFiltered.excludedPattern++;
-                        linkDetails.push({ url: link.url, reason: 'matches_exclude_paths', pathname: linkUrl.pathname });
-                        continue;
-                    }
-                }
-                // Add to discovered links
-                discovered.push({
-                    url: normalizedUrl,
-                    depth: nextDepth,
-                });
-                linkDetails.push({ url: link.url, reason: 'accepted' });
-            }
-            catch (error) {
-                // Invalid URL, skip
-                linkDetails.push({ url: link.url, reason: 'invalid_url', error: error instanceof Error ? error.message : String(error) });
-                continue;
-            }
-        }
-        // Log detailed filtering information
-        const filteredLinksByReason = linkDetails
-            .filter(d => d.reason !== 'accepted')
-            .reduce((acc, d) => {
-            if (!acc[d.reason])
-                acc[d.reason] = [];
-            acc[d.reason].push({ url: d.url, pathname: d.pathname, error: d.error });
-            return acc;
-        }, {});
-        logger.info('Link filtering details', {
-            totalLinks: page.navigationLinks.length,
-            discovered: discovered.length,
-            filtered: {
-                notContent: filtered.notContent,
-                externalDomain: filtered.externalDomain,
-                alreadyVisited: filtered.alreadyVisited,
-                excludedPattern: filtered.excludedPattern,
-            },
-            filteredLinksByReason,
-            sampleAcceptedLinks: linkDetails
-                .filter(d => d.reason === 'accepted')
-                .slice(0, 10)
-                .map(d => d.url),
-        });
-        return {
-            discovered,
-            filtered,
-            alreadyVisited: filtered.alreadyVisited,
-            notContent: filtered.notContent,
-            externalDomain: filtered.externalDomain,
-            excludedPattern: filtered.excludedPattern,
-        };
-    }
-    /**
-     * Check if a path should be crawled (permissive - only exclude clearly non-content paths)
-     */
-    isDocumentationPath(pathname) {
-        // Exclude clearly non-content pages
-        if (this.shouldExclude(pathname)) {
-            return false;
-        }
-        // Exclude static resources
-        const looksLikeStaticResource = /\.(?:css|js|json|xml|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|pdf|zip|exe|dmg)$/i.test(pathname);
-        if (looksLikeStaticResource) {
-            return false;
-        }
-        // Exclude API endpoints that are clearly not content (unless they're documentation APIs)
-        // Keep API endpoints that might be documentation (e.g., /api/docs, /docs/api)
-        const looksLikeApiEndpoint = /^\/api\/[^/]+$/i.test(pathname);
-        if (looksLikeApiEndpoint && !pathname.includes('/docs') && !pathname.includes('/documentation')) {
-            return false;
-        }
-        // Allow root path
-        if (pathname === '/' || pathname === '') {
-            return true;
-        }
-        // Exclude paths with file extensions (unless they're HTML pages)
-        const hasFileExtension = /\.[a-z]{2,4}$/i.test(pathname.split('?')[0]);
-        if (hasFileExtension && !pathname.match(/\.(html?|htm)$/i)) {
-            return false;
-        }
-        // Permissive: allow any path that doesn't match exclusion patterns
-        // This allows crawling any website, not just documentation
-        return true;
-    }
-    /**
-     * Check if a path should be excluded
-     */
-    shouldExclude(pathname) {
-        return this.EXCLUDED_PATTERNS.some(pattern => pattern.test(pathname));
-    }
-    /**
-     * Check if crawled content is sufficient for skill generation
-     * Enhanced with multi-dimensional quality metrics
-     */
-    canGenerateSkill(pages) {
-        if (pages.length === 0) {
-            return { canGenerate: false, reason: 'empty_pages' };
-        }
-        const metrics = this.evaluateContentQuality(pages);
-        // All pages are media-only
-        if (metrics.mediaOnlyPages === pages.length && !metrics.hasTextContent) {
-            return { canGenerate: false, reason: 'media_only' };
-        }
-        // No pages have sufficient content
-        if (!metrics.hasSufficientContent) {
-            return { canGenerate: false, reason: 'insufficient_content' };
-        }
-        // No structured content (headings, sections)
-        if (!metrics.hasStructuredContent) {
-            return { canGenerate: false, reason: 'no_structured_content' };
-        }
-        return { canGenerate: true };
-    }
-    /**
-     * Evaluate content quality with multi-dimensional metrics
-     */
-    evaluateContentQuality(pages) {
-        const MIN_CONTENT_LENGTH = 100;
-        let hasSufficientContent = false;
-        let hasStructuredContent = false;
-        let hasTextContent = false;
-        let mediaOnlyCount = 0;
-        let totalContentLength = 0;
-        let totalCodeSamples = 0;
-        // Track content diversity
-        const urlPatterns = new Set();
-        const titlePatterns = new Set();
-        for (const page of pages) {
-            const contentLength = (page.content || '').trim().length;
-            const hasHeadings = page.headings && page.headings.length > 0;
-            const hasText = contentLength > 0;
-            totalContentLength += contentLength;
-            totalCodeSamples += (page.codeSamples || []).length;
-            // Check if page is media-only
-            const hasImages = /<img[^>]*>/i.test(page.content || '');
-            const hasMedia = hasImages || (page.codeSamples && page.codeSamples.length > 0);
-            if (hasMedia && contentLength < MIN_CONTENT_LENGTH) {
-                mediaOnlyCount++;
-            }
-            if (contentLength >= MIN_CONTENT_LENGTH) {
-                hasSufficientContent = true;
-            }
-            if (hasHeadings) {
-                hasStructuredContent = true;
-            }
-            if (hasText) {
-                hasTextContent = true;
-            }
-            // Track diversity
-            try {
-                const urlPath = new URL(page.url).pathname;
-                const pathSegments = urlPath.split('/').filter(s => s);
-                if (pathSegments.length > 0) {
-                    urlPatterns.add(pathSegments[0]);
-                }
-            }
-            catch {
-                // Invalid URL, skip
-            }
-            // Track title diversity
-            const titleWords = page.title.toLowerCase().split(/\s+/).slice(0, 3);
-            titlePatterns.add(titleWords.join(' '));
-        }
-        // Calculate diversity score (0-1)
-        const contentDiversity = Math.min(1, (urlPatterns.size + titlePatterns.size) / (pages.length * 0.5));
-        // Calculate API coverage score (0-1)
-        const pagesWithCode = pages.filter(p => p.codeSamples && p.codeSamples.length > 0).length;
-        const apiCoverage = pages.length > 0 ? pagesWithCode / pages.length : 0;
-        const avgContentLength = pages.length > 0 ? totalContentLength / pages.length : 0;
-        return {
-            hasSufficientContent,
-            hasStructuredContent,
-            hasTextContent,
-            mediaOnlyPages: mediaOnlyCount,
-            contentDiversity,
-            apiCoverage,
-            avgContentLength,
-            totalCodeSamples,
-        };
-    }
-    /**
-     * Check if should continue crawling based on content quality
-     */
-    shouldContinueCrawling(currentPages, maxPages) {
-        if (currentPages >= maxPages) {
-            return false;
-        }
-        // Evaluate quality every 10 pages
-        if (currentPages % 10 === 0 && currentPages > 0) {
-            const metrics = this.evaluateContentQuality(this.crawledPages);
-            // High quality content - can stop early if we have enough
-            if (metrics.hasSufficientContent &&
-                metrics.contentDiversity > 0.7 &&
-                metrics.apiCoverage > 0.5 &&
-                currentPages >= maxPages * 0.5) {
-                logger.info('High quality content detected, considering early stop', {
-                    currentPages,
-                    maxPages,
-                    diversity: metrics.contentDiversity.toFixed(2),
-                    apiCoverage: metrics.apiCoverage.toFixed(2),
-                });
-                // Continue but log the possibility
-            }
-            // Low quality warning
-            if (currentPages >= maxPages * 0.8 && !metrics.hasSufficientContent) {
-                logger.warn('Approaching page limit but content quality is low', {
-                    currentPages,
-                    maxPages,
-                    diversity: metrics.contentDiversity.toFixed(2),
-                    apiCoverage: metrics.apiCoverage.toFixed(2),
-                    suggestion: 'Consider increasing maxPages or refining includePaths',
-                });
-            }
-        }
-        return currentPages < maxPages;
-    }
-    /**
-     * Fetch a page with retry logic
-     * Supports both HTML pages and Markdown files
-     */
-    async fetchPageWithRetry(url, retryCount = 0) {
-        try {
-            // Check if this is a Markdown file
-            if (url.endsWith('.md') || url.includes('.md?') || url.includes('.md#')) {
-                return await this.extractMarkdownContent(url);
-            }
-            // Regular HTML page
-            return await this.browser.browsePage(url);
-        }
-        catch (error) {
-            const errorType = this.classifyError(error);
-            const isRetryable = this.isRetryableError(error);
-            if (isRetryable && retryCount < this.options.maxRetries) {
-                const delay = this.options.retryDelay * (retryCount + 1); // Exponential backoff
-                logger.info('Retrying page fetch', {
-                    url,
-                    retryCount: retryCount + 1,
-                    maxRetries: this.options.maxRetries,
-                    delay,
-                    errorType,
-                });
-                await this.delay(delay);
-                return this.fetchPageWithRetry(url, retryCount + 1);
-            }
-            // Not retryable or max retries reached
-            throw error;
-        }
-    }
-    /**
-     * Extract content from Markdown file
-     * Converts Markdown structure to WebDocumentationPage format
-     */
-    async extractMarkdownContent(url) {
-        logger.debug('Extracting Markdown content', { url });
-        // Fetch raw markdown content
-        const httpClient = new HttpClient();
-        const response = await httpClient.get(url, {
-            responseType: 'text',
-            timeout: 30000,
-        });
-        const markdownContent = response.data;
-        // Parse markdown structure
-        const parsed = this.parseMarkdown(markdownContent, url);
-        return {
-            url,
-            title: parsed.title,
-            content: parsed.content,
-            searchableContent: parsed.content, // Add searchable content for consistency
-            sections: parsed.sections,
-            navigationLinks: parsed.links,
-            headings: parsed.headings,
-            codeSamples: parsed.codeSamples,
-            isDocumentation: true,
-        };
-    }
-    /**
-     * Parse Markdown content into structured data
-     */
-    parseMarkdown(content, url) {
-        const lines = content.split('\n');
-        let title = '';
-        const headings = [];
-        const codeSamples = [];
-        const sections = [];
-        const links = [];
-        const contentLines = [];
-        // Extract title from first h1
-        for (const line of lines) {
-            if (line.startsWith('# ')) {
-                title = line.substring(2).trim();
-                break;
-            }
-        }
-        // Extract headings (h2-h6)
-        const headingRegex = /^(#{2,6})\s+(.+)$/;
-        for (const line of lines) {
-            const match = line.match(headingRegex);
-            if (match) {
-                const level = match[1].length;
-                const text = match[2].trim();
-                const id = text.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-');
-                headings.push({
-                    level: `h${level}`,
-                    text,
-                    id,
-                });
-            }
-        }
-        // Extract code blocks
-        const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
-        let match;
-        while ((match = codeBlockRegex.exec(content)) !== null) {
-            const language = match[1] || 'text';
-            const code = match[2].trim();
-            if (code.length > 10) {
-                codeSamples.push({
-                    code,
-                    language,
-                });
-            }
-        }
-        // Extract content (remove code blocks and headings)
-        let contentWithoutCode = content.replace(codeBlockRegex, '');
-        contentWithoutCode = contentWithoutCode.replace(/^#{1,6}\s+.+$/gm, '');
-        for (const para of contentWithoutCode.split('\n\n')) {
-            const trimmed = para.trim();
-            if (trimmed.length > 20) {
-                contentLines.push(trimmed);
-            }
-        }
-        // Extract links (markdown format)
-        const linkRegex = /\[([^\]]*)\]\(([^)]+)\)/g;
-        while ((match = linkRegex.exec(content)) !== null) {
-            const text = match[1];
-            const linkUrl = match[2].trim();
-            // Skip anchors
-            if (linkUrl.startsWith('#')) {
-                continue;
-            }
-            // Resolve relative URLs
-            let absoluteUrl;
-            try {
-                if (linkUrl.startsWith('http://') || linkUrl.startsWith('https://')) {
-                    absoluteUrl = linkUrl;
-                }
-                else {
-                    absoluteUrl = new URL(linkUrl, url).href;
-                }
-                // Remove fragment
-                absoluteUrl = absoluteUrl.split('#')[0];
-                // Only include .md URLs to avoid client-side rendered HTML pages
-                if (absoluteUrl.endsWith('.md') || absoluteUrl.includes('.md?')) {
-                    const linkOrigin = new URL(absoluteUrl).origin;
-                    const baseOrigin = this.baseUrl.origin;
-                    links.push({
-                        text,
-                        url: absoluteUrl,
-                        isInternal: linkOrigin === baseOrigin,
-                    });
-                }
-            }
-            catch (error) {
-                // Invalid URL, skip
-                logger.debug('Invalid URL in markdown link', { url: linkUrl });
-            }
-        }
-        // Build sections from headings
-        let currentSection = null;
-        let currentContent = [];
-        for (const line of lines) {
-            const headerMatch = line.match(headingRegex);
-            if (headerMatch) {
-                // Save previous section
-                if (currentSection) {
-                    currentSection.content = currentContent.join('\n').trim();
-                    if (currentSection.content.length > 0) {
-                        sections.push(currentSection);
-                    }
-                }
-                // Start new section
-                const text = headerMatch[2].trim();
-                currentSection = {
-                    title: text,
-                    content: '',
-                    anchor: text.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-'),
-                };
-                currentContent = [];
-            }
-            else if (currentSection) {
-                currentContent.push(line);
-            }
-        }
-        // Save last section
-        if (currentSection) {
-            currentSection.content = currentContent.join('\n').trim();
-            if (currentSection.content.length > 0) {
-                sections.push(currentSection);
-            }
-        }
-        return {
-            title: title || 'Untitled',
-            content: contentLines.join('\n\n'),
-            headings,
-            codeSamples,
-            sections,
-            links,
-        };
-    }
-    /**
-     * Classify error type for better error messages
-     */
-    classifyError(error) {
-        if (!(error instanceof Error)) {
-            return 'UnknownError';
-        }
-        const message = error.message.toLowerCase();
-        const errorName = error.name.toLowerCase();
-        // Network errors
-        if (errorName.includes('timeout') || message.includes('timeout')) {
-            return 'TimeoutError';
-        }
-        if (errorName.includes('network') || message.includes('network') || message.includes('econnrefused')) {
-            return 'NetworkError';
-        }
-        if (message.includes('econnreset') || message.includes('socket')) {
-            return 'ConnectionError';
-        }
-        // HTTP errors
-        if (errorName.includes('http') || message.includes('status')) {
-            if (message.includes('404'))
-                return 'NotFoundError';
-            if (message.includes('403'))
-                return 'ForbiddenError';
-            if (message.includes('401'))
-                return 'UnauthorizedError';
-            if (message.includes('429'))
-                return 'RateLimitError';
-            if (message.includes('500') || message.includes('502') || message.includes('503')) {
-                return 'ServerError';
-            }
-            return 'HttpError';
-        }
-        // Content errors
-        if (message.includes('documentation') || message.includes('not appear to be')) {
-            return 'NotDocumentationError';
-        }
-        if (message.includes('spa') || message.includes('javascript')) {
-            return 'SPAError';
-        }
-        return 'UnknownError';
-    }
-    /**
-     * Check if an error is retryable
-     */
-    isRetryableError(error) {
-        if (!(error instanceof Error)) {
-            return false;
-        }
-        const errorType = this.classifyError(error);
-        // Retryable errors
-        const retryableTypes = [
-            'TimeoutError',
-            'NetworkError',
-            'ConnectionError',
-            'RateLimitError',
-            'ServerError', // 500, 502, 503
-        ];
-        return retryableTypes.includes(errorType);
-    }
-    /**
-     * Get error breakdown by type
-     */
-    getErrorBreakdown() {
-        const breakdown = {};
-        for (const error of this.errors) {
-            const errorType = error.error.split(':')[0] || 'UnknownError';
-            breakdown[errorType] = (breakdown[errorType] || 0) + 1;
-        }
-        return breakdown;
-    }
-    /**
-     * Try to detect and use llms.txt for optimized crawling
-     */
-    async tryLlmsTxt(rootUrl) {
-        logger.info('Checking for llms.txt files', { url: rootUrl });
-        try {
-            const detector = new LlmsTxtDetector(rootUrl);
-            const variants = await detector.detectAll();
-            if (variants.length === 0) {
-                logger.info('No llms.txt files found, proceeding with normal crawl');
-                return;
-            }
-            logger.info('Found llms.txt variants', {
-                count: variants.length,
-                variants: variants.map(v => v.variant),
-            });
-            // Download all variants
-            const downloader = new LlmsTxtDownloader();
-            const downloaded = await downloader.downloadAll(variants);
-            if (downloaded.length === 0) {
-                logger.warn('Failed to download any llms.txt variants');
-                return;
-            }
-            // Use the largest variant (most comprehensive)
-            const largest = downloaded.reduce((prev, current) => current.size > prev.size ? current : prev);
-            logger.info('Using llms.txt for URL extraction', {
-                variant: largest.variant,
-                size: largest.size,
-            });
-            // Parse URLs from llms.txt
-            const parser = new LlmsTxtParser(largest.content, rootUrl);
-            const extractedUrls = parser.extractUrls();
-            if (extractedUrls.length > 0) {
-                logger.info('Extracted URLs from llms.txt', {
-                    count: extractedUrls.length,
-                });
-                // Add URLs to queue with depth 0
-                for (const url of extractedUrls) {
-                    if (this.isValidUrl(url) && !this.visitedUrls.has(url)) {
-                        this.urlQueue.push({ url, depth: 0 });
-                    }
-                }
-                logger.info('Added llms.txt URLs to crawl queue', {
-                    added: this.urlQueue.length,
-                });
-            }
-            else {
-                logger.info('No URLs extracted from llms.txt, using normal crawl');
-            }
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            logger.warn('llms.txt detection failed, continuing with normal crawl', {
-                error: errorMessage,
-            });
-            // Continue with normal crawling if llms.txt fails
-        }
-    }
-    /**
-     * Check if a URL is valid for crawling
-     */
-    isValidUrl(url) {
-        try {
-            const parsed = new URL(url);
-            // Must be same origin as base URL
-            if (parsed.origin !== this.baseUrl.origin) {
-                return false;
-            }
-            // Must be http or https
-            if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
-                return false;
-            }
-            return true;
-        }
-        catch {
-            return false;
-        }
-    }
-    /**
-     * Save checkpoint
-     */
-    async saveCheckpoint() {
-        if (!this.checkpointManager) {
-            return;
-        }
-        const checkpointData = {
-            config: this.options,
-            visitedUrls: Array.from(this.visitedUrls),
-            pendingUrls: this.urlQueue,
-            pagesCrawled: this.crawledPages.length,
-            lastUpdated: new Date().toISOString(),
-            baseUrl: this.baseUrl.href,
-        };
-        try {
-            await this.checkpointManager.saveCheckpoint(checkpointData);
-        }
-        catch (error) {
-            logger.warn('Failed to save checkpoint', {
-                error: error instanceof Error ? error.message : String(error),
-            });
-        }
-    }
-    /**
-     * Load checkpoint and restore state
-     */
-    async loadCheckpoint() {
-        if (!this.checkpointManager) {
-            return false;
-        }
-        try {
-            const data = await this.checkpointManager.loadCheckpoint();
-            if (!data) {
-                logger.info('No checkpoint found to resume from');
-                return false;
-            }
-            // Restore state
-            this.visitedUrls = new Set(data.visitedUrls);
-            this.urlQueue = data.pendingUrls;
-            // Note: crawledPages are not restored as they will be regenerated
-            logger.info('State restored from checkpoint', {
-                visitedUrls: this.visitedUrls.size,
-                pendingUrls: this.urlQueue.length,
-                lastUpdated: data.lastUpdated,
-            });
-            return true;
-        }
-        catch (error) {
-            logger.warn('Failed to load checkpoint', {
-                error: error instanceof Error ? error.message : String(error),
-            });
-            return false;
-        }
-    }
-    /**
-     * Clear checkpoint after successful crawl
-     */
-    async clearCheckpoint() {
-        if (this.checkpointManager) {
-            try {
-                await this.checkpointManager.clearCheckpoint();
-            }
-            catch (error) {
-                logger.debug('Failed to clear checkpoint', {
-                    error: error instanceof Error ? error.message : String(error),
-                });
-            }
-        }
-    }
-    /**
-     * Sanitize filename for checkpoint
-     */
-    sanitizeFilename(url) {
-        return url
-            .replace(/[^a-z0-9]/gi, '-')
-            .replace(/-+/g, '-')
-            .substring(0, 64);
-    }
-    /**
-     * Delay helper for rate limiting
-     */
-    delay(ms) {
-        return new Promise(resolve => setTimeout(resolve, ms));
-    }
-}
-//# sourceMappingURL=doc-crawler.js.map