@staticn0va/wigolo 0.1.0

package/dist/crawl/crawler.js

@@ -0,0 +1,228 @@
+import { matchesPatterns } from './url-utils.js';
+import { RateLimiter } from './rate-limiter.js';
+import { RobotsParser } from './robots.js';
+import { parseSitemap, parseSitemapIndex, extractSitemapUrlFromRobots } from './sitemap.js';
+import { getConfig } from '../config.js';
+import { createLogger } from '../logger.js';
+const log = createLogger('crawl');
+export class Crawler {
+    fetchFn;
+    rawFetchFn;
+    rateLimiter = new RateLimiter();
+    constructor(fetchFn, rawFetchFn) {
+        this.fetchFn = fetchFn;
+        this.rawFetchFn = rawFetchFn;
+    }
+    async crawl(input) {
+        const strategy = input.strategy ?? 'bfs';
+        const maxDepth = input.max_depth ?? 2;
+        const maxPages = input.max_pages ?? 20;
+        const seedOrigin = new URL(input.url).origin;
+        // Fetch and parse robots.txt if configured
+        const config = getConfig();
+        let robotsParser = null;
+        if (config.respectRobotsTxt) {
+            robotsParser = await this.fetchRobots(seedOrigin);
+        }
+        if (strategy === 'sitemap') {
+            return this.crawlSitemap(input, seedOrigin, maxPages, robotsParser);
+        }
+        const traversalStrategy = strategy === 'map' ? 'bfs' : strategy;
+        return this.crawlTraversal(input, seedOrigin, maxDepth, maxPages, traversalStrategy, robotsParser);
+    }
+    robotsTxtContent = null;
+    async fetchRobots(origin) {
+        try {
+            const result = await this.rawFetchFn(`${origin}/robots.txt`);
+            if (result.statusCode === 200 && result.html) {
+                this.robotsTxtContent = result.html;
+                const parser = new RobotsParser(result.html);
+                const crawlDelay = parser.getCrawlDelay();
+                if (crawlDelay !== null) {
+                    const domain = new URL(origin).hostname;
+                    this.rateLimiter.setRobotsCrawlDelay(domain, crawlDelay);
+                }
+                return parser;
+            }
+        }
+        catch {
+            log.debug('Could not fetch robots.txt', { origin });
+        }
+        return null;
+    }
+    async crawlTraversal(input, seedOrigin, maxDepth, maxPages, strategy, robotsParser) {
+        const visited = new Set();
+        const pages = [];
+        const allLinks = [];
+        // Queue: [url, depth]
+        const queue = [[input.url, 0]];
+        visited.add(input.url);
+        while (queue.length > 0 && pages.length < maxPages) {
+            const item = strategy === 'dfs' ? queue.pop() : queue.shift();
+            const [url, depth] = item;
+            // Check robots.txt
+            if (robotsParser && !robotsParser.isAllowed(new URL(url).pathname)) {
+                log.debug('Blocked by robots.txt', { url });
+                continue;
+            }
+            // Rate limit
+            const release = await this.rateLimiter.acquire(url);
+            let fetchResult;
+            try {
+                fetchResult = await this.fetchFn(url);
+            }
+            catch (err) {
+                log.warn('Fetch failed during crawl', { url, error: String(err) });
+                release();
+                continue;
+            }
+            release();
+            if (fetchResult.error) {
+                log.warn('Fetch returned error', { url, error: fetchResult.error });
+                continue;
+            }
+            pages.push({
+                url: fetchResult.url,
+                title: fetchResult.title,
+                markdown: fetchResult.markdown,
+                depth,
+            });
+            // Discover links for traversal
+            if (depth < maxDepth) {
+                const newLinks = this.filterLinks(fetchResult.links, seedOrigin, visited, input.include_patterns, input.exclude_patterns, robotsParser);
+                for (const link of newLinks) {
+                    visited.add(link);
+                    queue.push([link, depth + 1]);
+                }
+                if (input.extract_links) {
+                    for (const link of fetchResult.links) {
+                        allLinks.push({ from: url, to: link });
+                    }
+                }
+            }
+        }
+        // total_found = all unique URLs discovered (visited set), including unvisited queue items
+        return {
+            pages,
+            total_found: visited.size,
+            crawled: pages.length,
+            ...(input.extract_links ? { links: allLinks } : {}),
+        };
+    }
+    filterLinks(links, seedOrigin, visited, includePatterns, excludePatterns, robotsParser) {
+        const filtered = links.filter((link) => {
+            try {
+                const parsed = new URL(link);
+                if (parsed.origin !== seedOrigin)
+                    return false;
+                if (visited.has(link))
+                    return false;
+                if (!matchesPatterns(link, includePatterns, excludePatterns))
+                    return false;
+                if (robotsParser && !robotsParser.isAllowed(parsed.pathname))
+                    return false;
+                return true;
+            }
+            catch {
+                return false;
+            }
+        });
+        // Prioritize documentation pages over marketing/nav pages
+        return filtered.sort((a, b) => {
+            const aDoc = isDocPage(a) ? 0 : 1;
+            const bDoc = isDocPage(b) ? 0 : 1;
+            return aDoc - bDoc;
+        });
+    }
+    async crawlSitemap(input, seedOrigin, maxPages, robotsParser) {
+        // Discover sitemap URLs (pass already-fetched robots.txt content)
+        let sitemapUrls = await this.discoverSitemapUrls(seedOrigin, this.robotsTxtContent);
+        if (sitemapUrls.length === 0) {
+            log.info('No sitemap found, falling back to BFS');
+            return this.crawlTraversal(input, seedOrigin, input.max_depth ?? 2, maxPages, 'bfs', robotsParser);
+        }
+        // Filter by patterns
+        sitemapUrls = sitemapUrls.filter((url) => matchesPatterns(url, input.include_patterns, input.exclude_patterns));
+        const totalFound = sitemapUrls.length;
+        const toFetch = sitemapUrls.slice(0, maxPages);
+        const pages = [];
+        const allLinks = [];
+        for (const url of toFetch) {
+            if (pages.length >= maxPages)
+                break;
+            if (robotsParser && !robotsParser.isAllowed(new URL(url).pathname))
+                continue;
+            const release = await this.rateLimiter.acquire(url);
+            try {
+                const result = await this.fetchFn(url);
+                release();
+                if (!result.error) {
+                    pages.push({ url: result.url, title: result.title, markdown: result.markdown, depth: 0 });
+                    if (input.extract_links) {
+                        for (const link of result.links) {
+                            allLinks.push({ from: url, to: link });
+                        }
+                    }
+                }
+            }
+            catch (err) {
+                release();
+                log.warn('Sitemap fetch failed', { url, error: String(err) });
+            }
+        }
+        return {
+            pages,
+            total_found: totalFound,
+            crawled: pages.length,
+            ...(input.extract_links ? { links: allLinks } : {}),
+        };
+    }
+    async discoverSitemapUrls(origin, robotsTxt) {
+        const sitemapLocations = [];
+        // Check robots.txt for sitemap references (reuses already-fetched content)
+        if (robotsTxt) {
+            sitemapLocations.push(...extractSitemapUrlFromRobots(robotsTxt));
+        }
+        // Try default location
+        if (sitemapLocations.length === 0) {
+            sitemapLocations.push(`${origin}/sitemap.xml`);
+        }
+        const allUrls = [];
+        for (const sitemapUrl of sitemapLocations) {
+            try {
+                const result = await this.rawFetchFn(sitemapUrl);
+                if (result.statusCode !== 200)
+                    continue;
+                // Check if it's a sitemap index
+                const indexUrls = parseSitemapIndex(result.html);
+                if (indexUrls.length > 0) {
+                    // Fetch each sub-sitemap
+                    for (const subUrl of indexUrls) {
+                        try {
+                            const subResult = await this.rawFetchFn(subUrl);
+                            if (subResult.statusCode === 200) {
+                                allUrls.push(...parseSitemap(subResult.html));
+                            }
+                        }
+                        catch {
+                            // skip failed sub-sitemaps
+                        }
+                    }
+                }
+                else {
+                    allUrls.push(...parseSitemap(result.html));
+                }
+            }
+            catch {
+                // skip failed sitemap fetches
+            }
+        }
+        return allUrls;
+    }
+}
+const DOC_PATH_PATTERNS = ['/docs/', '/guide/', '/api/', '/reference/'];
+function isDocPage(url) {
+    const path = new URL(url).pathname.toLowerCase();
+    return DOC_PATH_PATTERNS.some(p => path.includes(p));
+}
+//# sourceMappingURL=crawler.js.map

package/dist/crawl/crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/crawl/crawler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,cAAc,CAAC;AAC5F,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAKlC,MAAM,OAAO,OAAO;IACV,OAAO,CAAU;IACjB,UAAU,CAAa;IACvB,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IAExC,YAAY,OAAgB,EAAE,UAAsB;QAClD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAiB;QAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC;QACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC;QAEvC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAE7C,2CAA2C;QAC3C,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,IAAI,YAAY,GAAwB,IAAI,CAAC;QAC7C,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,YAAY,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QACpD,CAAC;QAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,iBAAiB,GAAG,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;QAChE,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,YAAY,CAAC,CAAC;IACrG,CAAC;IAEO,gBAAgB,GAAkB,IAAI,CAAC;IAEvC,KAAK,CAAC,WAAW,CAAC,MAAc;QACtC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,MAAM,aAAa,CAAC,CAAC;YAC7D,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC7C,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC;gBACpC,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,UAAU,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC1C,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;oBACxB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;oBACxC,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC3D,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,GAAG,CAAC,KAAK,CAAC,4BAA4B,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,KAAiB,EACjB,UAAkB,EAClB,QAAgB,EAChB,QAAgB,EAChB,QAAuB,EACvB,YAAiC;QAEjC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,KAAK,GAAsB,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAe,EAAE,CAAC;QAEhC,sBAAsB;QACtB,MAAM,KAAK,GAA4B,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;YAChE,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC;YAE1B,mBAAmB;YACnB,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnE,GAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC5C,SAAS;YACX,CAAC;YAED,aAAa;YACb,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEpD,IAAI,WAAwB,CAAC;YAC7B,IAAI,CAAC;gBACH,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACnE,OAAO,EAAE,CAAC;gBACV,SAAS;YACX,CAAC;YAED,OAAO,EAAE,CAAC;YAEV,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;gBACtB,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC;gBACpE,SAAS;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC;gBACT,GAAG,EAAE,WAAW,CAAC,GAAG;gBACpB,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,QAAQ,EAAE,WAAW,CAAC,QAAQ;gBAC9B,KAAK;aACN,CAAC,CAAC;YAEH,+BAA+B;YAC/B,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;gBAExI,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;oBAC5B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAClB,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;oBACxB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;wBACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;oBACzC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,0FAA0F;QAC1F,OAAO;YACL,KAAK;YACL,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,OAAO,EAAE,KAAK,CAAC,MAAM;YACrB,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC;IAEO,WAAW,CACjB,KAAe,EACf,UAAkB,EAClB,OAAoB,EACpB,eAAqC,EACrC,eAAqC,EACrC,YAAiC;QAEjC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YACrC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU;oBAAE,OAAO,KAAK,CAAC;gBAC/C,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,OAAO,KAAK,CAAC;gBACpC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,eAAe,EAAE,eAAe,CAAC;oBAAE,OAAO,KAAK,CAAC;gBAC3E,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;oBAAE,OAAO,KAAK,CAAC;gBAC3E,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,0DAA0D;QAC1D,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC5B,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,OAAO,IAAI,GAAG,IAAI,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAiB,EACjB,UAAkB,EAClB,QAAgB,EAChB,YAAiC;QAEjC,kEAAkE;QAClE,IAAI,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAEpF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,GAAG,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,SAAS,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QACrG,CAAC;QAED,qBAAqB;QACrB,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACvC,eAAe,CAAC,GAAG,EAAE,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,CAAC,CACrE,CAAC;QAEF,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;QACtC,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAsB,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAe,EAAE,CAAC;QAEhC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;gBAAE,MAAM;YAEpC,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;gBAAE,SAAS;YAE7E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEpD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACvC,OAAO,EAAE,CAAC;gBAEV,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;oBAClB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;oBAE1F,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;wBACxB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;4BAChC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;wBACzC,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,CAAC;gBACV,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO;YACL,KAAK;YACL,WAAW,EAAE,UAAU;YACvB,OAAO,EAAE,KAAK,CAAC,MAAM;YACrB,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAc,EAAE,SAAoC;QACpF,MAAM,gBAAgB,GAAa,EAAE,CAAC;QAEtC,2EAA2E;QAC3E,IAAI,SAAS,EAAE,CAAC;YACd,gBAAgB,CAAC,IAAI,CAAC,GAAG,2BAA2B,CAAC,SAAS,CAAC,CAAC,CAAC;QACnE,CAAC;QAED,uBAAuB;QACvB,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,gBAAgB,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,KAAK,MAAM,UAAU,IAAI,gBAAgB,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;gBACjD,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG;oBAAE,SAAS;gBAExC,gCAAgC;gBAChC,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACjD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,yBAAyB;oBACzB,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;wBAC/B,IAAI,CAAC;4BACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;4BAChD,IAAI,SAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gCACjC,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;4BAChD,CAAC;wBACH,CAAC;wBAAC,MAAM,CAAC;4BACP,2BAA2B;wBAC7B,CAAC;oBACH,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,8BAA8B;YAChC,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,MAAM,iBAAiB,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;AAExE,SAAS,SAAS,CAAC,GAAW;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IACjD,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC"}

package/dist/crawl/dedup.d.ts

@@ -0,0 +1,15 @@
+export declare function splitIntoBlocks(markdown: string): string[];
+export declare function normalizeBlockText(text: string): string;
+interface PageInput {
+    url: string;
+    markdown: string;
+}
+interface PageOutput {
+    url: string;
+    markdown: string;
+}
+export declare function deduplicatePages(pages: PageInput[], domain?: string): PageOutput[];
+export declare function getStoredBoilerplate(domain: string): string[];
+export declare function storeBoilerplate(domain: string, hashes: string[]): void;
+export {};
+//# sourceMappingURL=dedup.d.ts.map

package/dist/crawl/dedup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/crawl/dedup.ts"],"names":[],"mappings":"AAGA,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA2B1D;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEvD;AAMD,UAAU,SAAS;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU,EAAE,CA+ClF;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAI7D;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAavE"}

package/dist/crawl/dedup.js

@@ -0,0 +1,93 @@
+import { createHash } from 'node:crypto';
+import { getDatabase } from '../cache/db.js';
+export function splitIntoBlocks(markdown) {
+    if (!markdown.trim())
+        return [];
+    const lines = markdown.split('\n');
+    const headingIndices = [];
+    for (let i = 0; i < lines.length; i++) {
+        const match = lines[i].match(/^(#{1,6})\s+/);
+        if (match) {
+            headingIndices.push({ level: match[1].length, lineIdx: i });
+        }
+    }
+    // If no headings, split by double-newline (paragraph blocks)
+    if (headingIndices.length === 0) {
+        return markdown.split(/\n\n+/).map((b) => b.trim()).filter(Boolean);
+    }
+    // Non-overlapping split: each heading starts a new block, ending at the next heading of ANY level
+    const blocks = [];
+    for (let i = 0; i < headingIndices.length; i++) {
+        const start = headingIndices[i].lineIdx;
+        const end = i + 1 < headingIndices.length ? headingIndices[i + 1].lineIdx : lines.length;
+        blocks.push(lines.slice(start, end).join('\n').trim());
+    }
+    return blocks.filter(Boolean);
+}
+export function normalizeBlockText(text) {
+    return text.toLowerCase().replace(/\s+/g, ' ').trim();
+}
+function hashBlock(text) {
+    return createHash('sha256').update(normalizeBlockText(text)).digest('hex');
+}
+export function deduplicatePages(pages, domain) {
+    if (pages.length <= 1)
+        return pages.map((p) => ({ url: p.url, markdown: p.markdown }));
+    // Pre-load stored boilerplate hashes for this domain
+    const storedHashes = domain ? getStoredBoilerplate(domain) : [];
+    const boilerplateHashes = new Set(storedHashes);
+    // Split each page into blocks and hash them
+    const pageBlocks = pages.map((page) => ({
+        url: page.url,
+        blocks: splitIntoBlocks(page.markdown),
+    }));
+    // Count how many pages each block hash appears in
+    const hashPageCount = new Map();
+    for (const page of pageBlocks) {
+        const seenHashes = new Set();
+        for (const block of page.blocks) {
+            const h = hashBlock(block);
+            if (!seenHashes.has(h)) {
+                seenHashes.add(h);
+                hashPageCount.set(h, (hashPageCount.get(h) ?? 0) + 1);
+            }
+        }
+    }
+    // Mark hashes appearing in >50% of pages as boilerplate
+    const threshold = pages.length / 2;
+    for (const [hash, count] of hashPageCount) {
+        if (count > threshold) {
+            boilerplateHashes.add(hash);
+        }
+    }
+    // Store updated boilerplate hashes for this domain
+    if (domain) {
+        storeBoilerplate(domain, Array.from(boilerplateHashes));
+    }
+    // Strip boilerplate blocks from each page
+    return pageBlocks.map((page) => {
+        const filtered = page.blocks.filter((block) => !boilerplateHashes.has(hashBlock(block)));
+        return {
+            url: page.url,
+            markdown: filtered.join('\n\n'),
+        };
+    });
+}
+export function getStoredBoilerplate(domain) {
+    const db = getDatabase();
+    const rows = db.prepare('SELECT block_hash FROM domain_boilerplate WHERE domain = ?').all(domain);
+    return rows.map(r => r.block_hash);
+}
+export function storeBoilerplate(domain, hashes) {
+    const db = getDatabase();
+    const del = db.prepare('DELETE FROM domain_boilerplate WHERE domain = ?');
+    const insert = db.prepare('INSERT OR IGNORE INTO domain_boilerplate (domain, block_hash, sample_text) VALUES (?, ?, ?)');
+    const tx = db.transaction((items) => {
+        del.run(domain);
+        for (const hash of items) {
+            insert.run(domain, hash, null);
+        }
+    });
+    tx(hashes);
+}
+//# sourceMappingURL=dedup.js.map

package/dist/crawl/dedup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../src/crawl/dedup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,MAAM,UAAU,eAAe,CAAC,QAAgB;IAC9C,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAEhC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,cAAc,GAAyC,EAAE,CAAC;IAEhE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC7C,IAAI,KAAK,EAAE,CAAC;YACV,cAAc,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtE,CAAC;IAED,kGAAkG;IAClG,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACxC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;QACzF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACxD,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC7E,CAAC;AAYD,MAAM,UAAU,gBAAgB,CAAC,KAAkB,EAAE,MAAe;IAClE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEvF,qDAAqD;IACrD,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAS,YAAY,CAAC,CAAC;IAExD,4CAA4C;IAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACtC,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;KACvC,CAAC,CAAC,CAAC;IAEJ,kDAAkD;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QACrC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;YAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAClB,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;QAC1C,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,0CAA0C;IAC1C,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACzF,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;SAChC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAc;IACjD,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,4DAA4D,CAAC,CAAC,GAAG,CAAC,MAAM,CAA6B,CAAC;IAC9H,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,MAAgB;IAC/D,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB,6FAA6F,CAC9F,CAAC;IACF,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,KAAe,EAAE,EAAE;QAC5C,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QACjC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,MAAM,CAAC,CAAC;AACb,CAAC"}

package/dist/crawl/mapper.d.ts

@@ -0,0 +1,17 @@
+import type { MapOutput } from '../types.js';
+interface MapInput {
+    url: string;
+    max_depth?: number;
+    max_pages?: number;
+    include_patterns?: string[];
+    exclude_patterns?: string[];
+}
+export type LightFetchFn = (url: string) => Promise<{
+    html: string;
+    finalUrl: string;
+    statusCode: number;
+}>;
+export declare function extractLinks(html: string, origin: string): string[];
+export declare function mapUrls(input: MapInput, fetchFn: LightFetchFn): Promise<MapOutput>;
+export {};
+//# sourceMappingURL=mapper.d.ts.map

package/dist/crawl/mapper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mapper.d.ts","sourceRoot":"","sources":["../../src/crawl/mapper.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAI7C,UAAU,QAAQ;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7B;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAI5G,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CA+CnE;AAED,wBAAsB,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,SAAS,CAAC,CAqFxF"}

package/dist/crawl/mapper.js

@@ -0,0 +1,178 @@
+import { parseHTML } from 'linkedom';
+import { matchesPatterns } from './url-utils.js';
+import { parseSitemap, parseSitemapIndex, extractSitemapUrlFromRobots } from './sitemap.js';
+import { createLogger } from '../logger.js';
+const log = createLogger('crawl');
+const IGNORED_PROTOCOLS = ['javascript:', 'mailto:', 'tel:', 'data:', 'blob:', 'ftp:'];
+export function extractLinks(html, origin) {
+    if (!html || !html.trim())
+        return [];
+    try {
+        const { document: doc } = parseHTML(html);
+        const anchors = doc.querySelectorAll('a[href]');
+        const seen = new Set();
+        const links = [];
+        const parsedOrigin = new URL(origin).origin;
+        for (const anchor of Array.from(anchors)) {
+            const href = anchor.getAttribute('href');
+            if (!href)
+                continue;
+            const trimmed = href.trim();
+            if (!trimmed)
+                continue;
+            // Skip fragment-only links
+            if (trimmed.startsWith('#'))
+                continue;
+            // Skip non-http protocols
+            if (IGNORED_PROTOCOLS.some((p) => trimmed.toLowerCase().startsWith(p)))
+                continue;
+            try {
+                const resolved = new URL(trimmed, origin);
+                // Same-origin check
+                if (resolved.origin !== parsedOrigin)
+                    continue;
+                // Strip fragment, keep path + query
+                resolved.hash = '';
+                const normalized = resolved.href;
+                if (!seen.has(normalized)) {
+                    seen.add(normalized);
+                    links.push(normalized);
+                }
+            }
+            catch {
+                log.debug('Failed to resolve URL', { href: trimmed, origin });
+            }
+        }
+        return links;
+    }
+    catch (err) {
+        log.debug('Failed to parse HTML for link extraction', { error: String(err) });
+        return [];
+    }
+}
+export async function mapUrls(input, fetchFn) {
+    const maxDepth = input.max_depth ?? 3;
+    const maxPages = input.max_pages ?? 200;
+    let origin;
+    try {
+        origin = new URL(input.url).origin;
+    }
+    catch (err) {
+        return {
+            urls: [],
+            total_found: 0,
+            sitemap_found: false,
+            error: `Invalid seed URL: ${String(err)}`,
+        };
+    }
+    const discovered = new Set([input.url]);
+    const queued = new Set([input.url]);
+    const queue = [{ url: input.url, depth: 0 }];
+    let sitemapFound = false;
+    // Phase 1: Sitemap discovery (best-effort, errors are non-fatal)
+    try {
+        const sitemapUrls = await discoverSitemapUrls(origin, fetchFn);
+        if (sitemapUrls.length > 0) {
+            sitemapFound = true;
+            for (const sitemapUrl of sitemapUrls) {
+                if (discovered.size >= maxPages)
+                    break;
+                if (matchesPatterns(sitemapUrl, input.include_patterns, input.exclude_patterns)) {
+                    discovered.add(sitemapUrl);
+                    // Also queue sitemap URLs for BFS traversal so their links are explored
+                    if (!queued.has(sitemapUrl)) {
+                        queued.add(sitemapUrl);
+                        queue.push({ url: sitemapUrl, depth: 0 });
+                    }
+                }
+            }
+        }
+    }
+    catch (err) {
+        log.debug('Sitemap discovery failed, continuing with BFS only', { error: String(err) });
+    }
+    // Phase 2: BFS link traversal
+    let seedError;
+    while (queue.length > 0 && discovered.size < maxPages) {
+        const current = queue.shift();
+        if (current.depth > maxDepth)
+            continue;
+        try {
+            const { html } = await fetchFn(current.url);
+            const links = extractLinks(html, origin);
+            for (const link of links) {
+                if (discovered.size >= maxPages)
+                    break;
+                if (discovered.has(link))
+                    continue;
+                if (!matchesPatterns(link, input.include_patterns, input.exclude_patterns))
+                    continue;
+                discovered.add(link);
+                // Only queue for further traversal if we haven't hit max depth
+                if (current.depth + 1 <= maxDepth && !queued.has(link)) {
+                    queued.add(link);
+                    queue.push({ url: link, depth: current.depth + 1 });
+                }
+            }
+        }
+        catch (err) {
+            if (current.url === input.url && current.depth === 0) {
+                seedError = String(err);
+                log.warn('Seed URL fetch failed during map', { url: current.url, error: String(err) });
+            }
+            else {
+                log.debug('Child page fetch failed during map, skipping', { url: current.url, error: String(err) });
+            }
+        }
+    }
+    const urls = Array.from(discovered);
+    return {
+        urls,
+        total_found: urls.length,
+        sitemap_found: sitemapFound,
+        ...(seedError !== undefined ? { error: seedError } : {}),
+    };
+}
+async function discoverSitemapUrls(origin, fetchFn) {
+    const sitemapLocations = [];
+    // Try robots.txt first for Sitemap directives
+    try {
+        const { html: robotsTxt } = await fetchFn(`${origin}/robots.txt`);
+        const fromRobots = extractSitemapUrlFromRobots(robotsTxt);
+        sitemapLocations.push(...fromRobots);
+    }
+    catch {
+        log.debug('robots.txt not found, trying default sitemap location');
+    }
+    // Fallback to default /sitemap.xml
+    if (sitemapLocations.length === 0) {
+        sitemapLocations.push(`${origin}/sitemap.xml`);
+    }
+    const allUrls = [];
+    for (const sitemapUrl of sitemapLocations) {
+        try {
+            const { html: sitemapXml } = await fetchFn(sitemapUrl);
+            // Check if it's a sitemap index (contains sub-sitemaps)
+            const indexUrls = parseSitemapIndex(sitemapXml);
+            if (indexUrls.length > 0) {
+                for (const subUrl of indexUrls) {
+                    try {
+                        const { html: subXml } = await fetchFn(subUrl);
+                        allUrls.push(...parseSitemap(subXml));
+                    }
+                    catch {
+                        log.debug('Failed to fetch sub-sitemap', { url: subUrl });
+                    }
+                }
+            }
+            else {
+                allUrls.push(...parseSitemap(sitemapXml));
+            }
+        }
+        catch {
+            log.debug('Sitemap fetch failed', { url: sitemapUrl });
+        }
+    }
+    return allUrls;
+}
+//# sourceMappingURL=mapper.js.map

package/dist/crawl/mapper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mapper.js","sourceRoot":"","sources":["../../src/crawl/mapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,cAAc,CAAC;AAC5F,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAYlC,MAAM,iBAAiB,GAAG,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AAEvF,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,MAAc;IACvD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAErC,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAE5C,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,2BAA2B;YAC3B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,0BAA0B;YAC1B,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBAAE,SAAS;YAEjF,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAE1C,oBAAoB;gBACpB,IAAI,QAAQ,CAAC,MAAM,KAAK,YAAY;oBAAE,SAAS;gBAE/C,oCAAoC;gBACpC,QAAQ,CAAC,IAAI,GAAG,EAAE,CAAC;gBACnB,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAEjC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC1B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;oBACrB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,0CAA0C,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9E,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAe,EAAE,OAAqB;IAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,IAAI,GAAG,CAAC;IAExC,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,EAAE;YACR,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,KAAK;YACpB,KAAK,EAAE,qBAAqB,MAAM,CAAC,GAAG,CAAC,EAAE;SAC1C,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,MAAM,KAAK,GAA0C,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IACpF,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,iEAAiE;IACjE,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,IAAI,UAAU,CAAC,IAAI,IAAI,QAAQ;oBAAE,MAAM;gBACvC,IAAI,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;oBAChF,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;oBAC3B,wEAAwE;oBACxE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;wBAC5B,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;wBACvB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC5C,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,oDAAoD,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,8BAA8B;IAC9B,IAAI,SAA6B,CAAC;IAElC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;QACtD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;QAE/B,IAAI,OAAO,CAAC,KAAK,GAAG,QAAQ;YAAE,SAAS;QAEvC,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,UAAU,CAAC,IAAI,IAAI,QAAQ;oBAAE,MAAM;gBACvC,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,SAAS;gBACnC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,CAAC;oBAAE,SAAS;gBAErF,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAErB,+DAA+D;gBAC/D,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC,IAAI,QAAQ,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACvD,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACjB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,OAAO,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACrD,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;gBACxB,GAAG,CAAC,IAAI,CAAC,kCAAkC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzF,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,KAAK,CAAC,8CAA8C,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtG,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEpC,OAAO;QACL,IAAI;QACJ,WAAW,EAAE,IAAI,CAAC,MAAM;QACxB,aAAa,EAAE,YAAY;QAC3B,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACzD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,MAAc,EAAE,OAAqB;IACtE,MAAM,gBAAgB,GAAa,EAAE,CAAC;IAEtC,8CAA8C;IAC9C,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,MAAM,aAAa,CAAC,CAAC;QAClE,MAAM,UAAU,GAAG,2BAA2B,CAAC,SAAS,CAAC,CAAC;QAC1D,gBAAgB,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,GAAG,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;IACrE,CAAC;IAED,mCAAmC;IACnC,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClC,gBAAgB,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,UAAU,IAAI,gBAAgB,EAAE,CAAC;QAC1C,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;YAEvD,wDAAwD;YACxD,MAAM,SAAS,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;YAChD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;oBAC/B,IAAI,CAAC;wBACH,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;wBAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;oBACxC,CAAC;oBAAC,MAAM,CAAC;wBACP,GAAG,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;oBAC5D,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,GAAG,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/crawl/rate-limiter.d.ts

@@ -0,0 +1,10 @@
+export declare class RateLimiter {
+    private domains;
+    private robotsDelays;
+    setRobotsCrawlDelay(domain: string, delaySeconds: number): void;
+    acquire(url: string): Promise<() => void>;
+    private getOrCreateState;
+    private startRequest;
+    private processQueue;
+}
+//# sourceMappingURL=rate-limiter.d.ts.map

package/dist/crawl/rate-limiter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rate-limiter.d.ts","sourceRoot":"","sources":["../../src/crawl/rate-limiter.ts"],"names":[],"mappings":"AAWA,qBAAa,WAAW;IACtB,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,YAAY,CAA6B;IAEjD,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,IAAI;IAIzD,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC;IAoB/C,OAAO,CAAC,gBAAgB;IA6BxB,OAAO,CAAC,YAAY;IAUpB,OAAO,CAAC,YAAY;CAarB"}