@gulibs/safe-coder 0.0.24 → 0.0.25

package/dist/documentation/doc-crawler.js

@@ -1,5 +1,10 @@
+import { HttpClient } from '../utils/http-client.js';
 import { logger } from '../utils/logger.js';
 import { WebDocumentationBrowser } from './web-doc-browser.js';
+import { LlmsTxtDetector, LlmsTxtDownloader, LlmsTxtParser } from './llms-txt/index.js';
+import { CheckpointManager } from './checkpoint-manager.js';
+import { join } from 'path';
+import { tmpdir } from 'os';
 export class DocumentationCrawler {
     browser;
     visitedUrls;
@@ -9,6 +14,8 @@ export class DocumentationCrawler {
     options;
     baseUrl;
     linkDiscoveryStats;
+    checkpointManager;
+    pagesSinceLastCheckpoint;
     DOCUMENTATION_PATTERNS = [
         /\/docs?\//i,
         /\/documentation/i,
@@ -39,6 +46,7 @@ export class DocumentationCrawler {
         this.crawledPages = [];
         this.errors = [];
         this.options = {
+            crawlStrategy: 'bfs', // Default to breadth-first search
             maxDepth: 3,
             maxPages: 50,
             includePaths: [],
@@ -47,6 +55,8 @@ export class DocumentationCrawler {
             maxRetries: 2, // Default 2 retries
             retryDelay: 1000, // Default 1 second delay before retry
             useBrowserAutomation: false, // Default to HTTP-only for backward compatibility
+            skipLlmsTxt: false, // Enable llms.txt detection by default
+            workers: 1, // Default to single-threaded crawling
         };
         this.baseUrl = new URL('https://example.com');
         this.linkDiscoveryStats = {
@@ -62,15 +72,19 @@ export class DocumentationCrawler {
             pagesDiscovered: 0,
             pagesCrawled: 0,
         };
+        this.pagesSinceLastCheckpoint = 0;
     }
     /**
      * Crawl documentation starting from a root URL
      * Uses HTTP client (axios) exclusively - no browser automation
      * For SPA sites that require JavaScript rendering, use Cursor/Claude's built-in browser tools
+     * Supports both BFS (breadth-first) and DFS (depth-first) crawl strategies
      */
     async crawl(rootUrl, options = {}) {
+        const strategy = options.crawlStrategy || 'bfs';
         logger.info('Starting documentation crawl using HTTP client (axios)', {
             url: rootUrl,
+            strategy,
             method: 'HTTP GET',
             client: 'axios/HttpClient',
             note: 'For SPA sites, use Cursor/Claude browser tools to get rendered content first',
@@ -107,6 +121,27 @@ export class DocumentationCrawler {
         }
         // No longer require documentation-only pages - allow any website with extractable content
         logger.debug('Starting crawl from URL (permissive mode)', { url: rootUrl });
+        // Setup checkpoint manager if enabled
+        if (this.options.checkpoint?.enabled) {
+            const checkpointFile = this.options.checkpoint.file ||
+                join(tmpdir(), `safe-coder-checkpoint-${this.sanitizeFilename(rootUrl)}.json`);
+            this.checkpointManager = new CheckpointManager(checkpointFile);
+            // Try to resume from checkpoint if requested
+            if (this.options.resume) {
+                const loaded = await this.loadCheckpoint();
+                if (loaded) {
+                    logger.info('Resumed from checkpoint', {
+                        pagesCrawled: this.crawledPages.length,
+                        pendingUrls: this.urlQueue.length,
+                        visitedUrls: this.visitedUrls.size,
+                    });
+                }
+            }
+        }
+        // Try to detect and use llms.txt if available (unless explicitly disabled)
+        if (!this.options.skipLlmsTxt) {
+            await this.tryLlmsTxt(rootUrl);
+        }
         // Detect SPA and provide warning
         try {
             const spaDetection = await this.browser.detectSPA(rootUrl);
@@ -136,144 +171,15 @@ export class DocumentationCrawler {
         // Start crawling from root
         this.urlQueue.push({ url: rootUrl, depth: 0 });
         let maxDepthReached = 0;
-        // Process queue
+        // Process queue - use parallel workers if specified
         const startTime = Date.now();
-        let lastProgressLog = Date.now();
-        const PROGRESS_LOG_INTERVAL = 5000; // Log progress every 5 seconds
-        while (this.urlQueue.length > 0 && this.crawledPages.length < this.options.maxPages) {
-            const queued = this.urlQueue.shift();
-            if (!queued)
-                break;
-            const { url, depth } = queued;
-            // Log progress periodically
-            const now = Date.now();
-            if (now - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
-                const elapsed = ((now - startTime) / 1000).toFixed(1);
-                const pagesPerSecond = (this.crawledPages.length / elapsed).toFixed(2);
-                logger.info('Crawl progress', {
-                    pagesCrawled: this.crawledPages.length,
-                    pagesRemaining: this.urlQueue.length,
-                    maxPages: this.options.maxPages,
-                    errors: this.errors.length,
-                    elapsedSeconds: elapsed,
-                    pagesPerSecond,
-                    currentDepth: depth,
-                    maxDepth: this.options.maxDepth,
-                });
-                lastProgressLog = now;
-            }
-            // Skip if already visited
-            if (this.visitedUrls.has(url)) {
-                continue;
-            }
-            // Check depth limit
-            if (depth > this.options.maxDepth) {
-                continue;
-            }
-            // Mark as visited
-            this.visitedUrls.add(url);
-            maxDepthReached = Math.max(maxDepthReached, depth);
-            try {
-                // Crawl the page using HTTP GET with retry logic
-                logger.debug('Fetching page via HTTP GET', { url, depth, method: 'HTTP GET', client: 'axios' });
-                const page = await this.fetchPageWithRetry(url);
-                // Check if page has minimal content (possible SPA issue)
-                const contentLength = page.content.length;
-                const linksCount = page.navigationLinks.length;
-                if (contentLength < 200 && linksCount < 3) {
-                    logger.warn('Page has minimal content - may be SPA', {
-                        url,
-                        contentLength,
-                        linksCount,
-                        suggestion: 'This page may require JavaScript rendering. Consider using browser automation tools.',
-                    });
-                }
-                // Convert to CrawledPage format
-                const crawledPage = {
-                    url: page.url,
-                    title: page.title,
-                    content: page.content,
-                    depth,
-                    sections: page.sections,
-                    navigationLinks: page.navigationLinks,
-                    headings: page.headings,
-                    codeSamples: page.codeSamples,
-                };
-                this.crawledPages.push(crawledPage);
-                this.linkDiscoveryStats.pagesCrawled++;
-                const totalLinksOnPage = page.navigationLinks.length;
-                this.linkDiscoveryStats.totalLinksFound += totalLinksOnPage;
-                logger.debug('Page fetched and parsed successfully', {
-                    url,
-                    title: page.title.substring(0, 50),
-                    linksFound: totalLinksOnPage,
-                    depth,
-                });
-                // Discover and queue new URLs
-                if (depth < this.options.maxDepth) {
-                    const discoveryResult = this.discoverDocumentationLinks(page, depth + 1);
-                    const newUrls = discoveryResult.discovered;
-                    logger.debug('Link discovery completed', {
-                        url,
-                        totalLinksOnPage,
-                        discovered: newUrls.length,
-                        filtered: discoveryResult.filtered,
-                        alreadyVisited: discoveryResult.alreadyVisited,
-                        notContent: discoveryResult.notContent,
-                        externalDomain: discoveryResult.externalDomain,
-                        excludedPattern: discoveryResult.excludedPattern,
-                        queueLengthBefore: this.urlQueue.length,
-                    });
-                    let queuedCount = 0;
-                    let skippedAlreadyVisited = 0;
-                    for (const newUrl of newUrls) {
-                        if (!this.visitedUrls.has(newUrl.url)) {
-                            // Also check if it's already in the queue to avoid duplicates
-                            const alreadyInQueue = this.urlQueue.some(q => q.url === newUrl.url);
-                            if (!alreadyInQueue) {
-                                this.urlQueue.push(newUrl);
-                                this.linkDiscoveryStats.linksQueued++;
-                                queuedCount++;
-                            }
-                            else {
-                                skippedAlreadyVisited++;
-                            }
-                        }
-                        else {
-                            skippedAlreadyVisited++;
-                        }
-                    }
-                    logger.debug('Links queued', {
-                        url,
-                        queued: queuedCount,
-                        skippedAlreadyVisited,
-                        queueLengthAfter: this.urlQueue.length,
-                    });
-                }
-                else {
-                    this.linkDiscoveryStats.linksFiltered.depthLimit += totalLinksOnPage;
-                }
-                // Rate limiting
-                if (this.options.rateLimit > 0 && this.urlQueue.length > 0) {
-                    await this.delay(this.options.rateLimit);
-                }
-            }
-            catch (error) {
-                const errorMessage = error instanceof Error ? error.message : String(error);
-                const errorType = this.classifyError(error);
-                this.errors.push({
-                    url,
-                    error: `${errorType}: ${errorMessage}`,
-                });
-                logger.warn('Page crawl failed', {
-                    url,
-                    error: errorMessage,
-                    errorType,
-                    depth,
-                    willContinue: true,
-                });
-                // Continue crawling other pages
-            }
+        const workerCount = this.options.workers || 1;
+        if (workerCount > 1) {
+            logger.info('Using parallel crawling', { workers: workerCount });
+            maxDepthReached = await this.crawlWithWorkers(startTime);
+        }
+        else {
+            maxDepthReached = await this.crawlSequential(startTime);
         }
         // Update final statistics
         this.linkDiscoveryStats.pagesDiscovered = this.visitedUrls.size;
@@ -315,6 +221,10 @@ export class DocumentationCrawler {
                 suggestion: 'Consider crawling more pages or a different website',
             });
         }
+        // Clear checkpoint after successful completion
+        if (this.checkpointManager && !abandoned) {
+            await this.clearCheckpoint();
+        }
         return {
             pages: this.crawledPages,
             totalPages: this.crawledPages.length,
@@ -325,6 +235,221 @@ export class DocumentationCrawler {
             abandonReason,
         };
     }
+    /**
+     * Sequential crawling (single-threaded)
+     */
+    async crawlSequential(startTime) {
+        let maxDepthReached = 0;
+        let lastProgressLog = Date.now();
+        const PROGRESS_LOG_INTERVAL = 5000; // Log progress every 5 seconds
+        while (this.urlQueue.length > 0 && this.crawledPages.length < this.options.maxPages) {
+            // Use different strategies for getting next URL
+            // BFS: shift() - take from front (queue behavior)
+            // DFS: pop() - take from back (stack behavior)
+            const queued = this.options.crawlStrategy === 'dfs' ? this.urlQueue.pop() : this.urlQueue.shift();
+            if (!queued)
+                break;
+            const { url, depth } = queued;
+            // Log progress periodically
+            const now = Date.now();
+            if (now - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
+                const elapsed = ((now - startTime) / 1000).toFixed(1);
+                const pagesPerSecond = (this.crawledPages.length / elapsed).toFixed(2);
+                logger.info('Crawl progress', {
+                    pagesCrawled: this.crawledPages.length,
+                    pagesRemaining: this.urlQueue.length,
+                    maxPages: this.options.maxPages,
+                    errors: this.errors.length,
+                    elapsedSeconds: elapsed,
+                    pagesPerSecond,
+                    currentDepth: depth,
+                    maxDepth: this.options.maxDepth,
+                });
+                lastProgressLog = now;
+            }
+            // Skip if already visited
+            if (this.visitedUrls.has(url)) {
+                continue;
+            }
+            // Check depth limit
+            if (depth > this.options.maxDepth) {
+                continue;
+            }
+            // Mark as visited
+            this.visitedUrls.add(url);
+            maxDepthReached = Math.max(maxDepthReached, depth);
+            await this.processPage(url, depth);
+            // Rate limiting
+            if (this.options.rateLimit > 0 && this.urlQueue.length > 0) {
+                await this.delay(this.options.rateLimit);
+            }
+        }
+        return maxDepthReached;
+    }
+    /**
+     * Parallel crawling with multiple workers
+     */
+    async crawlWithWorkers(startTime) {
+        let maxDepthReached = 0;
+        let lastProgressLog = Date.now();
+        const PROGRESS_LOG_INTERVAL = 5000;
+        const workerCount = this.options.workers || 1;
+        while (this.urlQueue.length > 0 && this.crawledPages.length < this.options.maxPages) {
+            // Log progress periodically
+            const now = Date.now();
+            if (now - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
+                const elapsed = ((now - startTime) / 1000).toFixed(1);
+                const pagesPerSecond = (this.crawledPages.length / elapsed).toFixed(2);
+                logger.info('Crawl progress (parallel)', {
+                    pagesCrawled: this.crawledPages.length,
+                    pagesRemaining: this.urlQueue.length,
+                    maxPages: this.options.maxPages,
+                    errors: this.errors.length,
+                    elapsedSeconds: elapsed,
+                    pagesPerSecond,
+                    workers: workerCount,
+                });
+                lastProgressLog = now;
+            }
+            // Get batch of URLs to process in parallel
+            const batch = [];
+            const batchSize = Math.min(workerCount, this.urlQueue.length, this.options.maxPages - this.crawledPages.length);
+            for (let i = 0; i < batchSize; i++) {
+                const queued = this.options.crawlStrategy === 'dfs' ? this.urlQueue.pop() : this.urlQueue.shift();
+                if (!queued)
+                    break;
+                // Skip if already visited
+                if (this.visitedUrls.has(queued.url)) {
+                    continue;
+                }
+                // Check depth limit
+                if (queued.depth > this.options.maxDepth) {
+                    continue;
+                }
+                // Mark as visited
+                this.visitedUrls.add(queued.url);
+                maxDepthReached = Math.max(maxDepthReached, queued.depth);
+                batch.push(queued);
+            }
+            if (batch.length === 0) {
+                break;
+            }
+            // Process batch in parallel
+            await Promise.all(batch.map(async (queued) => {
+                await this.processPage(queued.url, queued.depth);
+                // Rate limiting (per worker)
+                if (this.options.rateLimit > 0) {
+                    await this.delay(this.options.rateLimit);
+                }
+            }));
+        }
+        return maxDepthReached;
+    }
+    /**
+     * Process a single page (shared by both sequential and parallel crawling)
+     */
+    async processPage(url, depth) {
+        try {
+            // Crawl the page using HTTP GET with retry logic
+            logger.debug('Fetching page via HTTP GET', { url, depth, method: 'HTTP GET', client: 'axios' });
+            const page = await this.fetchPageWithRetry(url);
+            // Check if page has minimal content (possible SPA issue)
+            const contentLength = page.content.length;
+            const linksCount = page.navigationLinks.length;
+            if (contentLength < 200 && linksCount < 3) {
+                logger.warn('Page has minimal content - may be SPA', {
+                    url,
+                    contentLength,
+                    linksCount,
+                    suggestion: 'This page may require JavaScript rendering. Consider using browser automation tools.',
+                });
+            }
+            // Convert to CrawledPage format
+            const crawledPage = {
+                url: page.url,
+                title: page.title,
+                content: page.content,
+                depth,
+                sections: page.sections,
+                navigationLinks: page.navigationLinks,
+                headings: page.headings,
+                codeSamples: page.codeSamples,
+            };
+            this.crawledPages.push(crawledPage);
+            this.linkDiscoveryStats.pagesCrawled++;
+            this.pagesSinceLastCheckpoint++;
+            // Save checkpoint if interval reached
+            if (this.checkpointManager && this.options.checkpoint?.enabled) {
+                const interval = this.options.checkpoint.interval || 10;
+                if (this.pagesSinceLastCheckpoint >= interval) {
+                    await this.saveCheckpoint();
+                    this.pagesSinceLastCheckpoint = 0;
+                }
+            }
+            const totalLinksOnPage = page.navigationLinks.length;
+            this.linkDiscoveryStats.totalLinksFound += totalLinksOnPage;
+            logger.debug('Page fetched and parsed successfully', {
+                url,
+                title: page.title.substring(0, 50),
+                linksFound: totalLinksOnPage,
+                depth,
+            });
+            // Discover and queue new URLs
+            if (depth < this.options.maxDepth) {
+                const discoveryResult = this.discoverDocumentationLinks(page, depth + 1);
+                const newUrls = discoveryResult.discovered;
+                logger.debug('Link discovery completed', {
+                    url,
+                    totalLinksOnPage,
+                    discovered: newUrls.length,
+                    filtered: discoveryResult.filtered,
+                });
+                let queuedCount = 0;
+                let skippedAlreadyVisited = 0;
+                for (const newUrl of newUrls) {
+                    if (!this.visitedUrls.has(newUrl.url)) {
+                        // Also check if it's already in the queue to avoid duplicates
+                        const alreadyInQueue = this.urlQueue.some(q => q.url === newUrl.url);
+                        if (!alreadyInQueue) {
+                            this.urlQueue.push(newUrl);
+                            this.linkDiscoveryStats.linksQueued++;
+                            queuedCount++;
+                        }
+                        else {
+                            skippedAlreadyVisited++;
+                        }
+                    }
+                    else {
+                        skippedAlreadyVisited++;
+                    }
+                }
+                logger.debug('Links queued', {
+                    url,
+                    queued: queuedCount,
+                    skippedAlreadyVisited,
+                    queueLengthAfter: this.urlQueue.length,
+                });
+            }
+            else {
+                this.linkDiscoveryStats.linksFiltered.depthLimit += totalLinksOnPage;
+            }
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            const errorType = this.classifyError(error);
+            this.errors.push({
+                url,
+                error: `${errorType}: ${errorMessage}`,
+            });
+            logger.warn('Page crawl failed', {
+                url,
+                error: errorMessage,
+                errorType,
+                depth,
+                willContinue: true,
+            });
+        }
+    }
     /**
      * Discover documentation links from a crawled page
      */
@@ -482,22 +607,48 @@ export class DocumentationCrawler {
     }
     /**
      * Check if crawled content is sufficient for skill generation
-     * Similar logic to SkillGenerator but here for early validation
+     * Enhanced with multi-dimensional quality metrics
      */
     canGenerateSkill(pages) {
         if (pages.length === 0) {
             return { canGenerate: false, reason: 'empty_pages' };
         }
+        const metrics = this.evaluateContentQuality(pages);
+        // All pages are media-only
+        if (metrics.mediaOnlyPages === pages.length && !metrics.hasTextContent) {
+            return { canGenerate: false, reason: 'media_only' };
+        }
+        // No pages have sufficient content
+        if (!metrics.hasSufficientContent) {
+            return { canGenerate: false, reason: 'insufficient_content' };
+        }
+        // No structured content (headings, sections)
+        if (!metrics.hasStructuredContent) {
+            return { canGenerate: false, reason: 'no_structured_content' };
+        }
+        return { canGenerate: true };
+    }
+    /**
+     * Evaluate content quality with multi-dimensional metrics
+     */
+    evaluateContentQuality(pages) {
         const MIN_CONTENT_LENGTH = 100;
         let hasSufficientContent = false;
         let hasStructuredContent = false;
         let hasTextContent = false;
         let mediaOnlyCount = 0;
+        let totalContentLength = 0;
+        let totalCodeSamples = 0;
+        // Track content diversity
+        const urlPatterns = new Set();
+        const titlePatterns = new Set();
         for (const page of pages) {
             const contentLength = (page.content || '').trim().length;
             const hasHeadings = page.headings && page.headings.length > 0;
             const hasText = contentLength > 0;
-            // Check if page is media-only (has images but no text)
+            totalContentLength += contentLength;
+            totalCodeSamples += (page.codeSamples || []).length;
+            // Check if page is media-only
             const hasImages = /<img[^>]*>/i.test(page.content || '');
             const hasMedia = hasImages || (page.codeSamples && page.codeSamples.length > 0);
             if (hasMedia && contentLength < MIN_CONTENT_LENGTH) {
@@ -512,26 +663,85 @@ export class DocumentationCrawler {
             if (hasText) {
                 hasTextContent = true;
             }
+            // Track diversity
+            try {
+                const urlPath = new URL(page.url).pathname;
+                const pathSegments = urlPath.split('/').filter(s => s);
+                if (pathSegments.length > 0) {
+                    urlPatterns.add(pathSegments[0]);
+                }
+            }
+            catch {
+                // Invalid URL, skip
+            }
+            // Track title diversity
+            const titleWords = page.title.toLowerCase().split(/\s+/).slice(0, 3);
+            titlePatterns.add(titleWords.join(' '));
         }
-        // All pages are media-only
-        if (mediaOnlyCount === pages.length && !hasTextContent) {
-            return { canGenerate: false, reason: 'media_only' };
-        }
-        // No pages have sufficient content
-        if (!hasSufficientContent) {
-            return { canGenerate: false, reason: 'insufficient_content' };
+        // Calculate diversity score (0-1)
+        const contentDiversity = Math.min(1, (urlPatterns.size + titlePatterns.size) / (pages.length * 0.5));
+        // Calculate API coverage score (0-1)
+        const pagesWithCode = pages.filter(p => p.codeSamples && p.codeSamples.length > 0).length;
+        const apiCoverage = pages.length > 0 ? pagesWithCode / pages.length : 0;
+        const avgContentLength = pages.length > 0 ? totalContentLength / pages.length : 0;
+        return {
+            hasSufficientContent,
+            hasStructuredContent,
+            hasTextContent,
+            mediaOnlyPages: mediaOnlyCount,
+            contentDiversity,
+            apiCoverage,
+            avgContentLength,
+            totalCodeSamples,
+        };
+    }
+    /**
+     * Check if should continue crawling based on content quality
+     */
+    shouldContinueCrawling(currentPages, maxPages) {
+        if (currentPages >= maxPages) {
+            return false;
         }
-        // No structured content (headings, sections)
-        if (!hasStructuredContent) {
-            return { canGenerate: false, reason: 'no_structured_content' };
+        // Evaluate quality every 10 pages
+        if (currentPages % 10 === 0 && currentPages > 0) {
+            const metrics = this.evaluateContentQuality(this.crawledPages);
+            // High quality content - can stop early if we have enough
+            if (metrics.hasSufficientContent &&
+                metrics.contentDiversity > 0.7 &&
+                metrics.apiCoverage > 0.5 &&
+                currentPages >= maxPages * 0.5) {
+                logger.info('High quality content detected, considering early stop', {
+                    currentPages,
+                    maxPages,
+                    diversity: metrics.contentDiversity.toFixed(2),
+                    apiCoverage: metrics.apiCoverage.toFixed(2),
+                });
+                // Continue but log the possibility
+            }
+            // Low quality warning
+            if (currentPages >= maxPages * 0.8 && !metrics.hasSufficientContent) {
+                logger.warn('Approaching page limit but content quality is low', {
+                    currentPages,
+                    maxPages,
+                    diversity: metrics.contentDiversity.toFixed(2),
+                    apiCoverage: metrics.apiCoverage.toFixed(2),
+                    suggestion: 'Consider increasing maxPages or refining includePaths',
+                });
+            }
         }
-        return { canGenerate: true };
+        return currentPages < maxPages;
     }
     /**
      * Fetch a page with retry logic
+     * Supports both HTML pages and Markdown files
      */
     async fetchPageWithRetry(url, retryCount = 0) {
         try {
+            // Check if this is a Markdown file
+            if (url.endsWith('.md') || url.includes('.md?') || url.includes('.md#')) {
+                return await this.extractMarkdownContent(url);
+            }
+            // Regular HTML page
             return await this.browser.browsePage(url);
         }
         catch (error) {
@@ -553,6 +763,166 @@ export class DocumentationCrawler {
             throw error;
         }
     }
+    /**
+     * Extract content from Markdown file
+     * Converts Markdown structure to WebDocumentationPage format
+     */
+    async extractMarkdownContent(url) {
+        logger.debug('Extracting Markdown content', { url });
+        // Fetch raw markdown content
+        const httpClient = new HttpClient();
+        const response = await httpClient.get(url, {
+            responseType: 'text',
+            timeout: 30000,
+        });
+        const markdownContent = response.data;
+        // Parse markdown structure
+        const parsed = this.parseMarkdown(markdownContent, url);
+        return {
+            url,
+            title: parsed.title,
+            content: parsed.content,
+            searchableContent: parsed.content, // Add searchable content for consistency
+            sections: parsed.sections,
+            navigationLinks: parsed.links,
+            headings: parsed.headings,
+            codeSamples: parsed.codeSamples,
+            isDocumentation: true,
+        };
+    }
+    /**
+     * Parse Markdown content into structured data
+     */
+    parseMarkdown(content, url) {
+        const lines = content.split('\n');
+        let title = '';
+        const headings = [];
+        const codeSamples = [];
+        const sections = [];
+        const links = [];
+        const contentLines = [];
+        // Extract title from first h1
+        for (const line of lines) {
+            if (line.startsWith('# ')) {
+                title = line.substring(2).trim();
+                break;
+            }
+        }
+        // Extract headings (h2-h6)
+        const headingRegex = /^(#{2,6})\s+(.+)$/;
+        for (const line of lines) {
+            const match = line.match(headingRegex);
+            if (match) {
+                const level = match[1].length;
+                const text = match[2].trim();
+                const id = text.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-');
+                headings.push({
+                    level: `h${level}`,
+                    text,
+                    id,
+                });
+            }
+        }
+        // Extract code blocks
+        const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
+        let match;
+        while ((match = codeBlockRegex.exec(content)) !== null) {
+            const language = match[1] || 'text';
+            const code = match[2].trim();
+            if (code.length > 10) {
+                codeSamples.push({
+                    code,
+                    language,
+                });
+            }
+        }
+        // Extract content (remove code blocks and headings)
+        let contentWithoutCode = content.replace(codeBlockRegex, '');
+        contentWithoutCode = contentWithoutCode.replace(/^#{1,6}\s+.+$/gm, '');
+        for (const para of contentWithoutCode.split('\n\n')) {
+            const trimmed = para.trim();
+            if (trimmed.length > 20) {
+                contentLines.push(trimmed);
+            }
+        }
+        // Extract links (markdown format)
+        const linkRegex = /\[([^\]]*)\]\(([^)]+)\)/g;
+        while ((match = linkRegex.exec(content)) !== null) {
+            const text = match[1];
+            const linkUrl = match[2].trim();
+            // Skip anchors
+            if (linkUrl.startsWith('#')) {
+                continue;
+            }
+            // Resolve relative URLs
+            let absoluteUrl;
+            try {
+                if (linkUrl.startsWith('http://') || linkUrl.startsWith('https://')) {
+                    absoluteUrl = linkUrl;
+                }
+                else {
+                    absoluteUrl = new URL(linkUrl, url).href;
+                }
+                // Remove fragment
+                absoluteUrl = absoluteUrl.split('#')[0];
+                // Only include .md URLs to avoid client-side rendered HTML pages
+                if (absoluteUrl.endsWith('.md') || absoluteUrl.includes('.md?')) {
+                    const linkOrigin = new URL(absoluteUrl).origin;
+                    const baseOrigin = this.baseUrl.origin;
+                    links.push({
+                        text,
+                        url: absoluteUrl,
+                        isInternal: linkOrigin === baseOrigin,
+                    });
+                }
+            }
+            catch (error) {
+                // Invalid URL, skip
+                logger.debug('Invalid URL in markdown link', { url: linkUrl });
+            }
+        }
+        // Build sections from headings
+        let currentSection = null;
+        let currentContent = [];
+        for (const line of lines) {
+            const headerMatch = line.match(headingRegex);
+            if (headerMatch) {
+                // Save previous section
+                if (currentSection) {
+                    currentSection.content = currentContent.join('\n').trim();
+                    if (currentSection.content.length > 0) {
+                        sections.push(currentSection);
+                    }
+                }
+                // Start new section
+                const text = headerMatch[2].trim();
+                currentSection = {
+                    title: text,
+                    content: '',
+                    anchor: text.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-'),
+                };
+                currentContent = [];
+            }
+            else if (currentSection) {
+                currentContent.push(line);
+            }
+        }
+        // Save last section
+        if (currentSection) {
+            currentSection.content = currentContent.join('\n').trim();
+            if (currentSection.content.length > 0) {
+                sections.push(currentSection);
+            }
+        }
+        return {
+            title: title || 'Untitled',
+            content: contentLines.join('\n\n'),
+            headings,
+            codeSamples,
+            sections,
+            links,
+        };
+    }
     /**
      * Classify error type for better error messages
      */
@@ -625,6 +995,163 @@ export class DocumentationCrawler {
         }
         return breakdown;
     }
+    /**
+     * Try to detect and use llms.txt for optimized crawling
+     */
+    async tryLlmsTxt(rootUrl) {
+        logger.info('Checking for llms.txt files', { url: rootUrl });
+        try {
+            const detector = new LlmsTxtDetector(rootUrl);
+            const variants = await detector.detectAll();
+            if (variants.length === 0) {
+                logger.info('No llms.txt files found, proceeding with normal crawl');
+                return;
+            }
+            logger.info('Found llms.txt variants', {
+                count: variants.length,
+                variants: variants.map(v => v.variant),
+            });
+            // Download all variants
+            const downloader = new LlmsTxtDownloader();
+            const downloaded = await downloader.downloadAll(variants);
+            if (downloaded.length === 0) {
+                logger.warn('Failed to download any llms.txt variants');
+                return;
+            }
+            // Use the largest variant (most comprehensive)
+            const largest = downloaded.reduce((prev, current) => current.size > prev.size ? current : prev);
+            logger.info('Using llms.txt for URL extraction', {
+                variant: largest.variant,
+                size: largest.size,
+            });
+            // Parse URLs from llms.txt
+            const parser = new LlmsTxtParser(largest.content, rootUrl);
+            const extractedUrls = parser.extractUrls();
+            if (extractedUrls.length > 0) {
+                logger.info('Extracted URLs from llms.txt', {
+                    count: extractedUrls.length,
+                });
+                // Add URLs to queue with depth 0
+                for (const url of extractedUrls) {
+                    if (this.isValidUrl(url) && !this.visitedUrls.has(url)) {
+                        this.urlQueue.push({ url, depth: 0 });
+                    }
+                }
+                logger.info('Added llms.txt URLs to crawl queue', {
+                    added: this.urlQueue.length,
+                });
+            }
+            else {
+                logger.info('No URLs extracted from llms.txt, using normal crawl');
+            }
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger.warn('llms.txt detection failed, continuing with normal crawl', {
+                error: errorMessage,
+            });
+            // Continue with normal crawling if llms.txt fails
+        }
+    }
+    /**
+     * Check if a URL is valid for crawling
+     */
+    isValidUrl(url) {
+        try {
+            const parsed = new URL(url);
+            // Must be same origin as base URL
+            if (parsed.origin !== this.baseUrl.origin) {
+                return false;
+            }
+            // Must be http or https
+            if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+                return false;
+            }
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Save checkpoint
+     */
+    async saveCheckpoint() {
+        if (!this.checkpointManager) {
+            return;
+        }
+        const checkpointData = {
+            config: this.options,
+            visitedUrls: Array.from(this.visitedUrls),
+            pendingUrls: this.urlQueue,
+            pagesCrawled: this.crawledPages.length,
+            lastUpdated: new Date().toISOString(),
+            baseUrl: this.baseUrl.href,
+        };
+        try {
+            await this.checkpointManager.saveCheckpoint(checkpointData);
+        }
+        catch (error) {
+            logger.warn('Failed to save checkpoint', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    /**
+     * Load checkpoint and restore state
+     */
+    async loadCheckpoint() {
+        if (!this.checkpointManager) {
+            return false;
+        }
+        try {
+            const data = await this.checkpointManager.loadCheckpoint();
+            if (!data) {
+                logger.info('No checkpoint found to resume from');
+                return false;
+            }
+            // Restore state
+            this.visitedUrls = new Set(data.visitedUrls);
+            this.urlQueue = data.pendingUrls;
+            // Note: crawledPages are not restored as they will be regenerated
+            logger.info('State restored from checkpoint', {
+                visitedUrls: this.visitedUrls.size,
+                pendingUrls: this.urlQueue.length,
+                lastUpdated: data.lastUpdated,
+            });
+            return true;
+        }
+        catch (error) {
+            logger.warn('Failed to load checkpoint', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return false;
+        }
+    }
+    /**
+     * Clear checkpoint after successful crawl
+     */
+    async clearCheckpoint() {
+        if (this.checkpointManager) {
+            try {
+                await this.checkpointManager.clearCheckpoint();
+            }
+            catch (error) {
+                logger.debug('Failed to clear checkpoint', {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+    }
+    /**
+     * Sanitize filename for checkpoint
+     */
+    sanitizeFilename(url) {
+        return url
+            .replace(/[^a-z0-9]/gi, '-')
+            .replace(/-+/g, '-')
+            .substring(0, 64);
+    }
     /**
      * Delay helper for rate limiting
      */