@gulibs/safe-coder 0.0.23 → 0.0.25

package/dist/documentation/doc-crawler.js

@@ -1,5 +1,10 @@
+import { HttpClient } from '../utils/http-client.js';
 import { logger } from '../utils/logger.js';
 import { WebDocumentationBrowser } from './web-doc-browser.js';
+import { LlmsTxtDetector, LlmsTxtDownloader, LlmsTxtParser } from './llms-txt/index.js';
+import { CheckpointManager } from './checkpoint-manager.js';
+import { join } from 'path';
+import { tmpdir } from 'os';
 export class DocumentationCrawler {
     browser;
     visitedUrls;
@@ -9,6 +14,8 @@ export class DocumentationCrawler {
     options;
     baseUrl;
     linkDiscoveryStats;
+    checkpointManager;
+    pagesSinceLastCheckpoint;
     DOCUMENTATION_PATTERNS = [
         /\/docs?\//i,
         /\/documentation/i,
@@ -39,6 +46,7 @@ export class DocumentationCrawler {
         this.crawledPages = [];
         this.errors = [];
         this.options = {
+            crawlStrategy: 'bfs', // Default to breadth-first search
             maxDepth: 3,
             maxPages: 50,
             includePaths: [],
@@ -46,12 +54,15 @@ export class DocumentationCrawler {
             rateLimit: 500, // 500ms default delay
             maxRetries: 2, // Default 2 retries
             retryDelay: 1000, // Default 1 second delay before retry
+            useBrowserAutomation: false, // Default to HTTP-only for backward compatibility
+            skipLlmsTxt: false, // Enable llms.txt detection by default
+            workers: 1, // Default to single-threaded crawling
         };
         this.baseUrl = new URL('https://example.com');
         this.linkDiscoveryStats = {
             totalLinksFound: 0,
             linksFiltered: {
-                notDocumentation: 0,
+                notContent: 0,
                 externalDomain: 0,
                 alreadyVisited: 0,
                 excludedPattern: 0,
@@ -61,15 +72,19 @@ export class DocumentationCrawler {
             pagesDiscovered: 0,
             pagesCrawled: 0,
         };
+        this.pagesSinceLastCheckpoint = 0;
     }
     /**
      * Crawl documentation starting from a root URL
      * Uses HTTP client (axios) exclusively - no browser automation
      * For SPA sites that require JavaScript rendering, use Cursor/Claude's built-in browser tools
+     * Supports both BFS (breadth-first) and DFS (depth-first) crawl strategies
      */
     async crawl(rootUrl, options = {}) {
+        const strategy = options.crawlStrategy || 'bfs';
         logger.info('Starting documentation crawl using HTTP client (axios)', {
             url: rootUrl,
+            strategy,
             method: 'HTTP GET',
             client: 'axios/HttpClient',
             note: 'For SPA sites, use Cursor/Claude browser tools to get rendered content first',
@@ -82,7 +97,7 @@ export class DocumentationCrawler {
         this.linkDiscoveryStats = {
             totalLinksFound: 0,
             linksFiltered: {
-                notDocumentation: 0,
+                notContent: 0,
                 externalDomain: 0,
                 alreadyVisited: 0,
                 excludedPattern: 0,
@@ -104,12 +119,28 @@ export class DocumentationCrawler {
         catch (error) {
             throw new Error(`Invalid root URL: ${rootUrl}`);
         }
-        // Check if root URL is documentation
-        logger.debug('Checking if URL is documentation page (HTTP request)', { url: rootUrl });
-        const isDoc = await this.browser.isDocumentationPage(rootUrl);
-        if (!isDoc) {
-            throw new Error(`The provided URL does not appear to be a documentation page: ${rootUrl}\n` +
-                `Note: For SPA sites that require JavaScript rendering, use Cursor/Claude's browser tools to get rendered HTML first, then process it.`);
+        // No longer require documentation-only pages - allow any website with extractable content
+        logger.debug('Starting crawl from URL (permissive mode)', { url: rootUrl });
+        // Setup checkpoint manager if enabled
+        if (this.options.checkpoint?.enabled) {
+            const checkpointFile = this.options.checkpoint.file ||
+                join(tmpdir(), `safe-coder-checkpoint-${this.sanitizeFilename(rootUrl)}.json`);
+            this.checkpointManager = new CheckpointManager(checkpointFile);
+            // Try to resume from checkpoint if requested
+            if (this.options.resume) {
+                const loaded = await this.loadCheckpoint();
+                if (loaded) {
+                    logger.info('Resumed from checkpoint', {
+                        pagesCrawled: this.crawledPages.length,
+                        pendingUrls: this.urlQueue.length,
+                        visitedUrls: this.visitedUrls.size,
+                    });
+                }
+            }
+        }
+        // Try to detect and use llms.txt if available (unless explicitly disabled)
+        if (!this.options.skipLlmsTxt) {
+            await this.tryLlmsTxt(rootUrl);
         }
         // Detect SPA and provide warning
         try {
@@ -140,12 +171,82 @@ export class DocumentationCrawler {
         // Start crawling from root
         this.urlQueue.push({ url: rootUrl, depth: 0 });
         let maxDepthReached = 0;
-        // Process queue
+        // Process queue - use parallel workers if specified
         const startTime = Date.now();
+        const workerCount = this.options.workers || 1;
+        if (workerCount > 1) {
+            logger.info('Using parallel crawling', { workers: workerCount });
+            maxDepthReached = await this.crawlWithWorkers(startTime);
+        }
+        else {
+            maxDepthReached = await this.crawlSequential(startTime);
+        }
+        // Update final statistics
+        this.linkDiscoveryStats.pagesDiscovered = this.visitedUrls.size;
+        // Calculate final statistics
+        const totalTime = ((Date.now() - startTime) / 1000).toFixed(2);
+        const avgTimePerPage = this.crawledPages.length > 0
+            ? ((Date.now() - startTime) / this.crawledPages.length / 1000).toFixed(2)
+            : '0';
+        const successRate = this.linkDiscoveryStats.pagesDiscovered > 0
+            ? ((this.crawledPages.length / this.linkDiscoveryStats.pagesDiscovered) * 100).toFixed(1)
+            : '0';
+        // Log crawl completion with comprehensive statistics
+        logger.info('Documentation crawl completed using HTTP client (axios)', {
+            totalPages: this.crawledPages.length,
+            maxDepthReached,
+            errors: this.errors.length,
+            totalTimeSeconds: totalTime,
+            avgTimePerPageSeconds: avgTimePerPage,
+            successRate: `${successRate}%`,
+            method: 'HTTP GET',
+            client: 'axios/HttpClient',
+            linkStats: {
+                totalLinksFound: this.linkDiscoveryStats.totalLinksFound,
+                linksQueued: this.linkDiscoveryStats.linksQueued,
+                linksFiltered: this.linkDiscoveryStats.linksFiltered,
+                pagesDiscovered: this.linkDiscoveryStats.pagesDiscovered,
+                pagesCrawled: this.linkDiscoveryStats.pagesCrawled,
+            },
+            errorBreakdown: this.getErrorBreakdown(),
+        });
+        // Validate if content is sufficient for skill generation
+        const validation = this.canGenerateSkill(this.crawledPages);
+        const abandoned = !validation.canGenerate;
+        const abandonReason = validation.reason;
+        if (abandoned) {
+            logger.warn('Crawl completed but content is insufficient for skill generation', {
+                reason: abandonReason,
+                pagesCrawled: this.crawledPages.length,
+                suggestion: 'Consider crawling more pages or a different website',
+            });
+        }
+        // Clear checkpoint after successful completion
+        if (this.checkpointManager && !abandoned) {
+            await this.clearCheckpoint();
+        }
+        return {
+            pages: this.crawledPages,
+            totalPages: this.crawledPages.length,
+            maxDepthReached,
+            errors: this.errors,
+            linkDiscoveryStats: this.linkDiscoveryStats,
+            abandoned,
+            abandonReason,
+        };
+    }
+    /**
+     * Sequential crawling (single-threaded)
+     */
+    async crawlSequential(startTime) {
+        let maxDepthReached = 0;
         let lastProgressLog = Date.now();
         const PROGRESS_LOG_INTERVAL = 5000; // Log progress every 5 seconds
         while (this.urlQueue.length > 0 && this.crawledPages.length < this.options.maxPages) {
-            const queued = this.urlQueue.shift();
+            // Use different strategies for getting next URL
+            // BFS: shift() - take from front (queue behavior)
+            // DFS: pop() - take from back (stack behavior)
+            const queued = this.options.crawlStrategy === 'dfs' ? this.urlQueue.pop() : this.urlQueue.shift();
             if (!queued)
                 break;
             const { url, depth } = queued;
@@ -177,144 +278,177 @@ export class DocumentationCrawler {
             // Mark as visited
             this.visitedUrls.add(url);
             maxDepthReached = Math.max(maxDepthReached, depth);
-            try {
-                // Crawl the page using HTTP GET with retry logic
-                logger.debug('Fetching page via HTTP GET', { url, depth, method: 'HTTP GET', client: 'axios' });
-                const page = await this.fetchPageWithRetry(url);
-                // Check if page has minimal content (possible SPA issue)
-                const contentLength = page.content.length;
-                const linksCount = page.navigationLinks.length;
-                if (contentLength < 200 && linksCount < 3) {
-                    logger.warn('Page has minimal content - may be SPA', {
-                        url,
-                        contentLength,
-                        linksCount,
-                        suggestion: 'This page may require JavaScript rendering. Consider using browser automation tools.',
-                    });
+            await this.processPage(url, depth);
+            // Rate limiting
+            if (this.options.rateLimit > 0 && this.urlQueue.length > 0) {
+                await this.delay(this.options.rateLimit);
+            }
+        }
+        return maxDepthReached;
+    }
+    /**
+     * Parallel crawling with multiple workers
+     */
+    async crawlWithWorkers(startTime) {
+        let maxDepthReached = 0;
+        let lastProgressLog = Date.now();
+        const PROGRESS_LOG_INTERVAL = 5000;
+        const workerCount = this.options.workers || 1;
+        while (this.urlQueue.length > 0 && this.crawledPages.length < this.options.maxPages) {
+            // Log progress periodically
+            const now = Date.now();
+            if (now - lastProgressLog >= PROGRESS_LOG_INTERVAL) {
+                const elapsed = ((now - startTime) / 1000).toFixed(1);
+                const pagesPerSecond = (this.crawledPages.length / elapsed).toFixed(2);
+                logger.info('Crawl progress (parallel)', {
+                    pagesCrawled: this.crawledPages.length,
+                    pagesRemaining: this.urlQueue.length,
+                    maxPages: this.options.maxPages,
+                    errors: this.errors.length,
+                    elapsedSeconds: elapsed,
+                    pagesPerSecond,
+                    workers: workerCount,
+                });
+                lastProgressLog = now;
+            }
+            // Get batch of URLs to process in parallel
+            const batch = [];
+            const batchSize = Math.min(workerCount, this.urlQueue.length, this.options.maxPages - this.crawledPages.length);
+            for (let i = 0; i < batchSize; i++) {
+                const queued = this.options.crawlStrategy === 'dfs' ? this.urlQueue.pop() : this.urlQueue.shift();
+                if (!queued)
+                    break;
+                // Skip if already visited
+                if (this.visitedUrls.has(queued.url)) {
+                    continue;
                 }
-                // Convert to CrawledPage format
-                const crawledPage = {
-                    url: page.url,
-                    title: page.title,
-                    content: page.content,
-                    depth,
-                    sections: page.sections,
-                    navigationLinks: page.navigationLinks,
-                    headings: page.headings,
-                    codeSamples: page.codeSamples,
-                };
-                this.crawledPages.push(crawledPage);
-                this.linkDiscoveryStats.pagesCrawled++;
-                const totalLinksOnPage = page.navigationLinks.length;
-                this.linkDiscoveryStats.totalLinksFound += totalLinksOnPage;
-                logger.debug('Page fetched and parsed successfully', {
+                // Check depth limit
+                if (queued.depth > this.options.maxDepth) {
+                    continue;
+                }
+                // Mark as visited
+                this.visitedUrls.add(queued.url);
+                maxDepthReached = Math.max(maxDepthReached, queued.depth);
+                batch.push(queued);
+            }
+            if (batch.length === 0) {
+                break;
+            }
+            // Process batch in parallel
+            await Promise.all(batch.map(async (queued) => {
+                await this.processPage(queued.url, queued.depth);
+                // Rate limiting (per worker)
+                if (this.options.rateLimit > 0) {
+                    await this.delay(this.options.rateLimit);
+                }
+            }));
+        }
+        return maxDepthReached;
+    }
+    /**
+     * Process a single page (shared by both sequential and parallel crawling)
+     */
+    async processPage(url, depth) {
+        try {
+            // Crawl the page using HTTP GET with retry logic
+            logger.debug('Fetching page via HTTP GET', { url, depth, method: 'HTTP GET', client: 'axios' });
+            const page = await this.fetchPageWithRetry(url);
+            // Check if page has minimal content (possible SPA issue)
+            const contentLength = page.content.length;
+            const linksCount = page.navigationLinks.length;
+            if (contentLength < 200 && linksCount < 3) {
+                logger.warn('Page has minimal content - may be SPA', {
                     url,
-                    title: page.title.substring(0, 50),
-                    linksFound: totalLinksOnPage,
-                    depth,
+                    contentLength,
+                    linksCount,
+                    suggestion: 'This page may require JavaScript rendering. Consider using browser automation tools.',
                 });
-                // Discover and queue new URLs
-                if (depth < this.options.maxDepth) {
-                    const discoveryResult = this.discoverDocumentationLinks(page, depth + 1);
-                    const newUrls = discoveryResult.discovered;
-                    logger.debug('Link discovery completed', {
-                        url,
-                        totalLinksOnPage,
-                        discovered: newUrls.length,
-                        filtered: discoveryResult.filtered,
-                        alreadyVisited: discoveryResult.alreadyVisited,
-                        notDocumentation: discoveryResult.notDocumentation,
-                        externalDomain: discoveryResult.externalDomain,
-                        excludedPattern: discoveryResult.excludedPattern,
-                        queueLengthBefore: this.urlQueue.length,
-                    });
-                    let queuedCount = 0;
-                    let skippedAlreadyVisited = 0;
-                    for (const newUrl of newUrls) {
-                        if (!this.visitedUrls.has(newUrl.url)) {
-                            // Also check if it's already in the queue to avoid duplicates
-                            const alreadyInQueue = this.urlQueue.some(q => q.url === newUrl.url);
-                            if (!alreadyInQueue) {
-                                this.urlQueue.push(newUrl);
-                                this.linkDiscoveryStats.linksQueued++;
-                                queuedCount++;
-                            }
-                            else {
-                                skippedAlreadyVisited++;
-                            }
+            }
+            // Convert to CrawledPage format
+            const crawledPage = {
+                url: page.url,
+                title: page.title,
+                content: page.content,
+                depth,
+                sections: page.sections,
+                navigationLinks: page.navigationLinks,
+                headings: page.headings,
+                codeSamples: page.codeSamples,
+            };
+            this.crawledPages.push(crawledPage);
+            this.linkDiscoveryStats.pagesCrawled++;
+            this.pagesSinceLastCheckpoint++;
+            // Save checkpoint if interval reached
+            if (this.checkpointManager && this.options.checkpoint?.enabled) {
+                const interval = this.options.checkpoint.interval || 10;
+                if (this.pagesSinceLastCheckpoint >= interval) {
+                    await this.saveCheckpoint();
+                    this.pagesSinceLastCheckpoint = 0;
+                }
+            }
+            const totalLinksOnPage = page.navigationLinks.length;
+            this.linkDiscoveryStats.totalLinksFound += totalLinksOnPage;
+            logger.debug('Page fetched and parsed successfully', {
+                url,
+                title: page.title.substring(0, 50),
+                linksFound: totalLinksOnPage,
+                depth,
+            });
+            // Discover and queue new URLs
+            if (depth < this.options.maxDepth) {
+                const discoveryResult = this.discoverDocumentationLinks(page, depth + 1);
+                const newUrls = discoveryResult.discovered;
+                logger.debug('Link discovery completed', {
+                    url,
+                    totalLinksOnPage,
+                    discovered: newUrls.length,
+                    filtered: discoveryResult.filtered,
+                });
+                let queuedCount = 0;
+                let skippedAlreadyVisited = 0;
+                for (const newUrl of newUrls) {
+                    if (!this.visitedUrls.has(newUrl.url)) {
+                        // Also check if it's already in the queue to avoid duplicates
+                        const alreadyInQueue = this.urlQueue.some(q => q.url === newUrl.url);
+                        if (!alreadyInQueue) {
+                            this.urlQueue.push(newUrl);
+                            this.linkDiscoveryStats.linksQueued++;
+                            queuedCount++;
                         }
                         else {
                             skippedAlreadyVisited++;
                         }
                     }
-                    logger.debug('Links queued', {
-                        url,
-                        queued: queuedCount,
-                        skippedAlreadyVisited,
-                        queueLengthAfter: this.urlQueue.length,
-                    });
-                }
-                else {
-                    this.linkDiscoveryStats.linksFiltered.depthLimit += totalLinksOnPage;
-                }
-                // Rate limiting
-                if (this.options.rateLimit > 0 && this.urlQueue.length > 0) {
-                    await this.delay(this.options.rateLimit);
+                    else {
+                        skippedAlreadyVisited++;
+                    }
                 }
-            }
-            catch (error) {
-                const errorMessage = error instanceof Error ? error.message : String(error);
-                const errorType = this.classifyError(error);
-                this.errors.push({
+                logger.debug('Links queued', {
                     url,
-                    error: `${errorType}: ${errorMessage}`,
-                });
-                logger.warn('Page crawl failed', {
-                    url,
-                    error: errorMessage,
-                    errorType,
-                    depth,
-                    willContinue: true,
+                    queued: queuedCount,
+                    skippedAlreadyVisited,
+                    queueLengthAfter: this.urlQueue.length,
                 });
-                // Continue crawling other pages
+            }
+            else {
+                this.linkDiscoveryStats.linksFiltered.depthLimit += totalLinksOnPage;
             }
         }
-        // Update final statistics
-        this.linkDiscoveryStats.pagesDiscovered = this.visitedUrls.size;
-        // Calculate final statistics
-        const totalTime = ((Date.now() - startTime) / 1000).toFixed(2);
-        const avgTimePerPage = this.crawledPages.length > 0
-            ? ((Date.now() - startTime) / this.crawledPages.length / 1000).toFixed(2)
-            : '0';
-        const successRate = this.linkDiscoveryStats.pagesDiscovered > 0
-            ? ((this.crawledPages.length / this.linkDiscoveryStats.pagesDiscovered) * 100).toFixed(1)
-            : '0';
-        // Log crawl completion with comprehensive statistics
-        logger.info('Documentation crawl completed using HTTP client (axios)', {
-            totalPages: this.crawledPages.length,
-            maxDepthReached,
-            errors: this.errors.length,
-            totalTimeSeconds: totalTime,
-            avgTimePerPageSeconds: avgTimePerPage,
-            successRate: `${successRate}%`,
-            method: 'HTTP GET',
-            client: 'axios/HttpClient',
-            linkStats: {
-                totalLinksFound: this.linkDiscoveryStats.totalLinksFound,
-                linksQueued: this.linkDiscoveryStats.linksQueued,
-                linksFiltered: this.linkDiscoveryStats.linksFiltered,
-                pagesDiscovered: this.linkDiscoveryStats.pagesDiscovered,
-                pagesCrawled: this.linkDiscoveryStats.pagesCrawled,
-            },
-            errorBreakdown: this.getErrorBreakdown(),
-        });
-        return {
-            pages: this.crawledPages,
-            totalPages: this.crawledPages.length,
-            maxDepthReached,
-            errors: this.errors,
-            linkDiscoveryStats: this.linkDiscoveryStats,
-        };
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            const errorType = this.classifyError(error);
+            this.errors.push({
+                url,
+                error: `${errorType}: ${errorMessage}`,
+            });
+            logger.warn('Page crawl failed', {
+                url,
+                error: errorMessage,
+                errorType,
+                depth,
+                willContinue: true,
+            });
+        }
     }
     /**
      * Discover documentation links from a crawled page
@@ -322,7 +456,7 @@ export class DocumentationCrawler {
     discoverDocumentationLinks(page, nextDepth) {
         const discovered = [];
         const filtered = {
-            notDocumentation: 0,
+            notContent: 0, // Renamed from notDocumentation
             externalDomain: 0,
             alreadyVisited: 0,
             excludedPattern: 0,
@@ -353,11 +487,11 @@ export class DocumentationCrawler {
                     linkDetails.push({ url: link.url, reason: 'already_visited' });
                     continue;
                 }
-                // Check if it's a documentation path
+                // Check if it's a valid content path (permissive - only exclude clearly non-content)
                 if (!this.isDocumentationPath(linkUrl.pathname)) {
-                    filtered.notDocumentation++;
-                    this.linkDiscoveryStats.linksFiltered.notDocumentation++;
-                    linkDetails.push({ url: link.url, reason: 'not_documentation_path', pathname: linkUrl.pathname });
+                    filtered.notContent++;
+                    this.linkDiscoveryStats.linksFiltered.notContent++;
+                    linkDetails.push({ url: link.url, reason: 'not_content_path', pathname: linkUrl.pathname });
                     continue;
                 }
                 // Check exclude patterns
@@ -371,8 +505,8 @@ export class DocumentationCrawler {
                 if (this.options.includePaths.length > 0) {
                     const matchesInclude = this.options.includePaths.some(pattern => linkUrl.pathname.includes(pattern));
                     if (!matchesInclude) {
-                        filtered.notDocumentation++;
-                        this.linkDiscoveryStats.linksFiltered.notDocumentation++;
+                        filtered.notContent++;
+                        this.linkDiscoveryStats.linksFiltered.notContent++;
                         linkDetails.push({ url: link.url, reason: 'not_in_include_paths', pathname: linkUrl.pathname });
                         continue;
                     }
@@ -413,7 +547,7 @@ export class DocumentationCrawler {
             totalLinks: page.navigationLinks.length,
             discovered: discovered.length,
             filtered: {
-                notDocumentation: filtered.notDocumentation,
+                notContent: filtered.notContent,
                 externalDomain: filtered.externalDomain,
                 alreadyVisited: filtered.alreadyVisited,
                 excludedPattern: filtered.excludedPattern,
@@ -428,52 +562,42 @@ export class DocumentationCrawler {
             discovered,
             filtered,
             alreadyVisited: filtered.alreadyVisited,
-            notDocumentation: filtered.notDocumentation,
+            notContent: filtered.notContent,
             externalDomain: filtered.externalDomain,
             excludedPattern: filtered.excludedPattern,
         };
     }
     /**
-     * Check if a path is a documentation path
+     * Check if a path should be crawled (permissive - only exclude clearly non-content paths)
      */
     isDocumentationPath(pathname) {
-        // Check against documentation patterns
-        const matchesPattern = this.DOCUMENTATION_PATTERNS.some(pattern => pattern.test(pathname));
-        // If it matches a pattern, it's definitely documentation
-        if (matchesPattern) {
+        // Exclude clearly non-content pages
+        if (this.shouldExclude(pathname)) {
+            return false;
+        }
+        // Exclude static resources
+        const looksLikeStaticResource = /\.(?:css|js|json|xml|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|pdf|zip|exe|dmg)$/i.test(pathname);
+        if (looksLikeStaticResource) {
+            return false;
+        }
+        // Exclude API endpoints that are clearly not content (unless they're documentation APIs)
+        // Keep API endpoints that might be documentation (e.g., /api/docs, /docs/api)
+        const looksLikeApiEndpoint = /^\/api\/[^/]+$/i.test(pathname);
+        if (looksLikeApiEndpoint && !pathname.includes('/docs') && !pathname.includes('/documentation')) {
+            return false;
+        }
+        // Allow root path
+        if (pathname === '/' || pathname === '') {
             return true;
         }
-        // Additional check: if the base URL is a documentation page (which we verified at start),
-        // then paths on the same domain are likely documentation too (unless they match excluded patterns)
-        // This helps with sites that have documentation at root level or non-standard paths
-        // Only apply this if the path doesn't match excluded patterns
-        if (!this.shouldExclude(pathname)) {
-            // If pathname is just "/" or empty, it's the root - check if base URL was documentation
-            if (pathname === '/' || pathname === '') {
-                return true; // Root of a documentation site is documentation
-            }
-            // For documentation sites, be more permissive:
-            // 1. If path contains common documentation keywords
-            // 2. If path looks like a documentation structure (no file extensions like .html, .php, etc.)
-            // 3. If path doesn't look like an API endpoint or static resource
-            const hasFileExtension = /\.[a-z]{2,4}$/i.test(pathname.split('?')[0]);
-            const looksLikeStaticResource = /\.(?:css|js|json|xml|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot)$/i.test(pathname);
-            const looksLikeApiEndpoint = /^\/api\/[^/]+$/i.test(pathname) && !/\/docs\/api\//i.test(pathname);
-            // If it's a static resource or API endpoint (not docs), exclude it
-            if (looksLikeStaticResource || (looksLikeApiEndpoint && !pathname.includes('/docs'))) {
-                return false;
-            }
-            // If it has a file extension (but not a static resource), be conservative
-            if (hasFileExtension) {
-                return false;
-            }
-            // For paths without file extensions, check if they contain documentation keywords
-            // OR if they're under common documentation paths
-            const looksLikeDoc = /(?:doc|guide|tutorial|api|reference|manual|help|about|getting-started|examples?)/i.test(pathname);
-            const isUnderDocPath = /^\/(?:docs?|documentation|guides?|tutorials?|api|reference|manual|help|examples?)/i.test(pathname);
-            return looksLikeDoc || isUnderDocPath;
+        // Exclude paths with file extensions (unless they're HTML pages)
+        const hasFileExtension = /\.[a-z]{2,4}$/i.test(pathname.split('?')[0]);
+        if (hasFileExtension && !pathname.match(/\.(html?|htm)$/i)) {
+            return false;
         }
-        return false;
+        // Permissive: allow any path that doesn't match exclusion patterns
+        // This allows crawling any website, not just documentation
+        return true;
     }
     /**
      * Check if a path should be excluded
@@ -481,11 +605,143 @@ export class DocumentationCrawler {
     shouldExclude(pathname) {
         return this.EXCLUDED_PATTERNS.some(pattern => pattern.test(pathname));
     }
+    /**
+     * Check if crawled content is sufficient for skill generation
+     * Enhanced with multi-dimensional quality metrics
+     */
+    canGenerateSkill(pages) {
+        if (pages.length === 0) {
+            return { canGenerate: false, reason: 'empty_pages' };
+        }
+        const metrics = this.evaluateContentQuality(pages);
+        // All pages are media-only
+        if (metrics.mediaOnlyPages === pages.length && !metrics.hasTextContent) {
+            return { canGenerate: false, reason: 'media_only' };
+        }
+        // No pages have sufficient content
+        if (!metrics.hasSufficientContent) {
+            return { canGenerate: false, reason: 'insufficient_content' };
+        }
+        // No structured content (headings, sections)
+        if (!metrics.hasStructuredContent) {
+            return { canGenerate: false, reason: 'no_structured_content' };
+        }
+        return { canGenerate: true };
+    }
+    /**
+     * Evaluate content quality with multi-dimensional metrics
+     */
+    evaluateContentQuality(pages) {
+        const MIN_CONTENT_LENGTH = 100;
+        let hasSufficientContent = false;
+        let hasStructuredContent = false;
+        let hasTextContent = false;
+        let mediaOnlyCount = 0;
+        let totalContentLength = 0;
+        let totalCodeSamples = 0;
+        // Track content diversity
+        const urlPatterns = new Set();
+        const titlePatterns = new Set();
+        for (const page of pages) {
+            const contentLength = (page.content || '').trim().length;
+            const hasHeadings = page.headings && page.headings.length > 0;
+            const hasText = contentLength > 0;
+            totalContentLength += contentLength;
+            totalCodeSamples += (page.codeSamples || []).length;
+            // Check if page is media-only
+            const hasImages = /<img[^>]*>/i.test(page.content || '');
+            const hasMedia = hasImages || (page.codeSamples && page.codeSamples.length > 0);
+            if (hasMedia && contentLength < MIN_CONTENT_LENGTH) {
+                mediaOnlyCount++;
+            }
+            if (contentLength >= MIN_CONTENT_LENGTH) {
+                hasSufficientContent = true;
+            }
+            if (hasHeadings) {
+                hasStructuredContent = true;
+            }
+            if (hasText) {
+                hasTextContent = true;
+            }
+            // Track diversity
+            try {
+                const urlPath = new URL(page.url).pathname;
+                const pathSegments = urlPath.split('/').filter(s => s);
+                if (pathSegments.length > 0) {
+                    urlPatterns.add(pathSegments[0]);
+                }
+            }
+            catch {
+                // Invalid URL, skip
+            }
+            // Track title diversity
+            const titleWords = page.title.toLowerCase().split(/\s+/).slice(0, 3);
+            titlePatterns.add(titleWords.join(' '));
+        }
+        // Calculate diversity score (0-1)
+        const contentDiversity = Math.min(1, (urlPatterns.size + titlePatterns.size) / (pages.length * 0.5));
+        // Calculate API coverage score (0-1)
+        const pagesWithCode = pages.filter(p => p.codeSamples && p.codeSamples.length > 0).length;
+        const apiCoverage = pages.length > 0 ? pagesWithCode / pages.length : 0;
+        const avgContentLength = pages.length > 0 ? totalContentLength / pages.length : 0;
+        return {
+            hasSufficientContent,
+            hasStructuredContent,
+            hasTextContent,
+            mediaOnlyPages: mediaOnlyCount,
+            contentDiversity,
+            apiCoverage,
+            avgContentLength,
+            totalCodeSamples,
+        };
+    }
+    /**
+     * Check if should continue crawling based on content quality
+     */
+    shouldContinueCrawling(currentPages, maxPages) {
+        if (currentPages >= maxPages) {
+            return false;
+        }
+        // Evaluate quality every 10 pages
+        if (currentPages % 10 === 0 && currentPages > 0) {
+            const metrics = this.evaluateContentQuality(this.crawledPages);
+            // High quality content - can stop early if we have enough
+            if (metrics.hasSufficientContent &&
+                metrics.contentDiversity > 0.7 &&
+                metrics.apiCoverage > 0.5 &&
+                currentPages >= maxPages * 0.5) {
+                logger.info('High quality content detected, considering early stop', {
+                    currentPages,
+                    maxPages,
+                    diversity: metrics.contentDiversity.toFixed(2),
+                    apiCoverage: metrics.apiCoverage.toFixed(2),
+                });
+                // Continue but log the possibility
+            }
+            // Low quality warning
+            if (currentPages >= maxPages * 0.8 && !metrics.hasSufficientContent) {
+                logger.warn('Approaching page limit but content quality is low', {
+                    currentPages,
+                    maxPages,
+                    diversity: metrics.contentDiversity.toFixed(2),
+                    apiCoverage: metrics.apiCoverage.toFixed(2),
+                    suggestion: 'Consider increasing maxPages or refining includePaths',
+                });
+            }
+        }
+        return currentPages < maxPages;
+    }
     /**
      * Fetch a page with retry logic
+     * Supports both HTML pages and Markdown files
      */
     async fetchPageWithRetry(url, retryCount = 0) {
         try {
+            // Check if this is a Markdown file
+            if (url.endsWith('.md') || url.includes('.md?') || url.includes('.md#')) {
+                return await this.extractMarkdownContent(url);
+            }
+            // Regular HTML page
             return await this.browser.browsePage(url);
         }
         catch (error) {
@@ -507,6 +763,166 @@ export class DocumentationCrawler {
             throw error;
         }
     }
+    /**
+     * Extract content from Markdown file
+     * Converts Markdown structure to WebDocumentationPage format
+     */
+    async extractMarkdownContent(url) {
+        logger.debug('Extracting Markdown content', { url });
+        // Fetch raw markdown content
+        const httpClient = new HttpClient();
+        const response = await httpClient.get(url, {
+            responseType: 'text',
+            timeout: 30000,
+        });
+        const markdownContent = response.data;
+        // Parse markdown structure
+        const parsed = this.parseMarkdown(markdownContent, url);
+        return {
+            url,
+            title: parsed.title,
+            content: parsed.content,
+            searchableContent: parsed.content, // Add searchable content for consistency
+            sections: parsed.sections,
+            navigationLinks: parsed.links,
+            headings: parsed.headings,
+            codeSamples: parsed.codeSamples,
+            isDocumentation: true,
+        };
+    }
+    /**
+     * Parse Markdown content into structured data
+     */
+    parseMarkdown(content, url) {
+        const lines = content.split('\n');
+        let title = '';
+        const headings = [];
+        const codeSamples = [];
+        const sections = [];
+        const links = [];
+        const contentLines = [];
+        // Extract title from first h1
+        for (const line of lines) {
+            if (line.startsWith('# ')) {
+                title = line.substring(2).trim();
+                break;
+            }
+        }
+        // Extract headings (h2-h6)
+        const headingRegex = /^(#{2,6})\s+(.+)$/;
+        for (const line of lines) {
+            const match = line.match(headingRegex);
+            if (match) {
+                const level = match[1].length;
+                const text = match[2].trim();
+                const id = text.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-');
+                headings.push({
+                    level: `h${level}`,
+                    text,
+                    id,
+                });
+            }
+        }
+        // Extract code blocks
+        const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
+        let match;
+        while ((match = codeBlockRegex.exec(content)) !== null) {
+            const language = match[1] || 'text';
+            const code = match[2].trim();
+            if (code.length > 10) {
+                codeSamples.push({
+                    code,
+                    language,
+                });
+            }
+        }
+        // Extract content (remove code blocks and headings)
+        let contentWithoutCode = content.replace(codeBlockRegex, '');
+        contentWithoutCode = contentWithoutCode.replace(/^#{1,6}\s+.+$/gm, '');
+        for (const para of contentWithoutCode.split('\n\n')) {
+            const trimmed = para.trim();
+            if (trimmed.length > 20) {
+                contentLines.push(trimmed);
+            }
+        }
+        // Extract links (markdown format)
+        const linkRegex = /\[([^\]]*)\]\(([^)]+)\)/g;
+        while ((match = linkRegex.exec(content)) !== null) {
+            const text = match[1];
+            const linkUrl = match[2].trim();
+            // Skip anchors
+            if (linkUrl.startsWith('#')) {
+                continue;
+            }
+            // Resolve relative URLs
+            let absoluteUrl;
+            try {
+                if (linkUrl.startsWith('http://') || linkUrl.startsWith('https://')) {
+                    absoluteUrl = linkUrl;
+                }
+                else {
+                    absoluteUrl = new URL(linkUrl, url).href;
+                }
+                // Remove fragment
+                absoluteUrl = absoluteUrl.split('#')[0];
+                // Only include .md URLs to avoid client-side rendered HTML pages
+                if (absoluteUrl.endsWith('.md') || absoluteUrl.includes('.md?')) {
+                    const linkOrigin = new URL(absoluteUrl).origin;
+                    const baseOrigin = this.baseUrl.origin;
+                    links.push({
+                        text,
+                        url: absoluteUrl,
+                        isInternal: linkOrigin === baseOrigin,
+                    });
+                }
+            }
+            catch (error) {
+                // Invalid URL, skip
+                logger.debug('Invalid URL in markdown link', { url: linkUrl });
+            }
+        }
+        // Build sections from headings
+        let currentSection = null;
+        let currentContent = [];
+        for (const line of lines) {
+            const headerMatch = line.match(headingRegex);
+            if (headerMatch) {
+                // Save previous section
+                if (currentSection) {
+                    currentSection.content = currentContent.join('\n').trim();
+                    if (currentSection.content.length > 0) {
+                        sections.push(currentSection);
+                    }
+                }
+                // Start new section
+                const text = headerMatch[2].trim();
+                currentSection = {
+                    title: text,
+                    content: '',
+                    anchor: text.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-'),
+                };
+                currentContent = [];
+            }
+            else if (currentSection) {
+                currentContent.push(line);
+            }
+        }
+        // Save last section
+        if (currentSection) {
+            currentSection.content = currentContent.join('\n').trim();
+            if (currentSection.content.length > 0) {
+                sections.push(currentSection);
+            }
+        }
+        return {
+            title: title || 'Untitled',
+            content: contentLines.join('\n\n'),
+            headings,
+            codeSamples,
+            sections,
+            links,
+        };
+    }
     /**
      * Classify error type for better error messages
      */
@@ -579,6 +995,163 @@ export class DocumentationCrawler {
         }
         return breakdown;
     }
+    /**
+     * Try to detect and use llms.txt for optimized crawling
+     */
+    async tryLlmsTxt(rootUrl) {
+        logger.info('Checking for llms.txt files', { url: rootUrl });
+        try {
+            const detector = new LlmsTxtDetector(rootUrl);
+            const variants = await detector.detectAll();
+            if (variants.length === 0) {
+                logger.info('No llms.txt files found, proceeding with normal crawl');
+                return;
+            }
+            logger.info('Found llms.txt variants', {
+                count: variants.length,
+                variants: variants.map(v => v.variant),
+            });
+            // Download all variants
+            const downloader = new LlmsTxtDownloader();
+            const downloaded = await downloader.downloadAll(variants);
+            if (downloaded.length === 0) {
+                logger.warn('Failed to download any llms.txt variants');
+                return;
+            }
+            // Use the largest variant (most comprehensive)
+            const largest = downloaded.reduce((prev, current) => current.size > prev.size ? current : prev);
+            logger.info('Using llms.txt for URL extraction', {
+                variant: largest.variant,
+                size: largest.size,
+            });
+            // Parse URLs from llms.txt
+            const parser = new LlmsTxtParser(largest.content, rootUrl);
+            const extractedUrls = parser.extractUrls();
+            if (extractedUrls.length > 0) {
+                logger.info('Extracted URLs from llms.txt', {
+                    count: extractedUrls.length,
+                });
+                // Add URLs to queue with depth 0
+                for (const url of extractedUrls) {
+                    if (this.isValidUrl(url) && !this.visitedUrls.has(url)) {
+                        this.urlQueue.push({ url, depth: 0 });
+                    }
+                }
+                logger.info('Added llms.txt URLs to crawl queue', {
+                    added: this.urlQueue.length,
+                });
+            }
+            else {
+                logger.info('No URLs extracted from llms.txt, using normal crawl');
+            }
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger.warn('llms.txt detection failed, continuing with normal crawl', {
+                error: errorMessage,
+            });
+            // Continue with normal crawling if llms.txt fails
+        }
+    }
+    /**
+     * Check if a URL is valid for crawling
+     */
+    isValidUrl(url) {
+        try {
+            const parsed = new URL(url);
+            // Must be same origin as base URL
+            if (parsed.origin !== this.baseUrl.origin) {
+                return false;
+            }
+            // Must be http or https
+            if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+                return false;
+            }
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Save checkpoint
+     */
+    async saveCheckpoint() {
+        if (!this.checkpointManager) {
+            return;
+        }
+        const checkpointData = {
+            config: this.options,
+            visitedUrls: Array.from(this.visitedUrls),
+            pendingUrls: this.urlQueue,
+            pagesCrawled: this.crawledPages.length,
+            lastUpdated: new Date().toISOString(),
+            baseUrl: this.baseUrl.href,
+        };
+        try {
+            await this.checkpointManager.saveCheckpoint(checkpointData);
+        }
+        catch (error) {
+            logger.warn('Failed to save checkpoint', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    /**
+     * Load checkpoint and restore state
+     */
+    async loadCheckpoint() {
+        if (!this.checkpointManager) {
+            return false;
+        }
+        try {
+            const data = await this.checkpointManager.loadCheckpoint();
+            if (!data) {
+                logger.info('No checkpoint found to resume from');
+                return false;
+            }
+            // Restore state
+            this.visitedUrls = new Set(data.visitedUrls);
+            this.urlQueue = data.pendingUrls;
+            // Note: crawledPages are not restored as they will be regenerated
+            logger.info('State restored from checkpoint', {
+                visitedUrls: this.visitedUrls.size,
+                pendingUrls: this.urlQueue.length,
+                lastUpdated: data.lastUpdated,
+            });
+            return true;
+        }
+        catch (error) {
+            logger.warn('Failed to load checkpoint', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return false;
+        }
+    }
+    /**
+     * Clear checkpoint after successful crawl
+     */
+    async clearCheckpoint() {
+        if (this.checkpointManager) {
+            try {
+                await this.checkpointManager.clearCheckpoint();
+            }
+            catch (error) {
+                logger.debug('Failed to clear checkpoint', {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+    }
+    /**
+     * Sanitize filename for checkpoint
+     */
+    sanitizeFilename(url) {
+        return url
+            .replace(/[^a-z0-9]/gi, '-')
+            .replace(/-+/g, '-')
+            .substring(0, 64);
+    }
     /**
      * Delay helper for rate limiting
      */