@staticn0va/wigolo 0.1.0 → 0.1.2

package/dist/crawl/crawler.js

@@ -1,228 +1,229 @@
-import { matchesPatterns } from './url-utils.js';
-import { RateLimiter } from './rate-limiter.js';
-import { RobotsParser } from './robots.js';
-import { parseSitemap, parseSitemapIndex, extractSitemapUrlFromRobots } from './sitemap.js';
-import { getConfig } from '../config.js';
-import { createLogger } from '../logger.js';
-const log = createLogger('crawl');
-export class Crawler {
-    fetchFn;
-    rawFetchFn;
-    rateLimiter = new RateLimiter();
-    constructor(fetchFn, rawFetchFn) {
-        this.fetchFn = fetchFn;
-        this.rawFetchFn = rawFetchFn;
+import { matchesPatterns, canonicalForCrawl } from "./url-utils.js";
+import { RateLimiter } from "./rate-limiter.js";
+import { RobotsParser } from "./robots.js";
+import { parseSitemap, parseSitemapIndex, extractSitemapUrlFromRobots } from "./sitemap.js";
+import { probeSitemap } from "./sitemap-first.js";
+import { isIndexingEnabled, indexCrawlResult } from "./index-to-vec.js";
+import { getConfig } from "../config.js";
+import { createLogger } from "../logger.js";
+const log = createLogger("crawl");
+class Crawler {
+  fetchFn;
+  rawFetchFn;
+  rateLimiter = new RateLimiter();
+  constructor(fetchFn, rawFetchFn) {
+    this.fetchFn = fetchFn;
+    this.rawFetchFn = rawFetchFn;
+  }
+  async crawl(input) {
+    const strategy = input.strategy ?? "bfs";
+    const maxDepth = input.max_depth ?? 2;
+    const maxPages = input.max_pages ?? 20;
+    const seedOrigin = new URL(input.url).origin;
+    const config = getConfig();
+    let robotsParser = null;
+    if (config.respectRobotsTxt) {
+      robotsParser = await this.fetchRobots(seedOrigin);
     }
-    async crawl(input) {
-        const strategy = input.strategy ?? 'bfs';
-        const maxDepth = input.max_depth ?? 2;
-        const maxPages = input.max_pages ?? 20;
-        const seedOrigin = new URL(input.url).origin;
-        // Fetch and parse robots.txt if configured
-        const config = getConfig();
-        let robotsParser = null;
-        if (config.respectRobotsTxt) {
-            robotsParser = await this.fetchRobots(seedOrigin);
-        }
-        if (strategy === 'sitemap') {
-            return this.crawlSitemap(input, seedOrigin, maxPages, robotsParser);
+    if (strategy === "auto") {
+      const sitemapUrls = await probeSitemap(seedOrigin, this.rawFetchFn);
+      if (sitemapUrls && sitemapUrls.length > 0) {
+        log.info("auto strategy: using sitemap", { origin: seedOrigin, urls: sitemapUrls.length });
+        return this.crawlFromExplicitUrls(input, sitemapUrls, maxPages, robotsParser);
+      }
+      log.info("auto strategy: no sitemap found, falling back to BFS", { origin: seedOrigin });
+      return this.crawlTraversal(input, seedOrigin, maxDepth, maxPages, "bfs", robotsParser);
+    }
+    if (strategy === "sitemap") {
+      return this.crawlSitemap(input, seedOrigin, maxPages, robotsParser);
+    }
+    const traversalStrategy = strategy === "map" ? "bfs" : strategy;
+    return this.crawlTraversal(input, seedOrigin, maxDepth, maxPages, traversalStrategy, robotsParser);
+  }
+  robotsTxtContent = null;
+  async fetchRobots(origin) {
+    try {
+      const result = await this.rawFetchFn(`${origin}/robots.txt`);
+      if (result.statusCode === 200 && result.html) {
+        this.robotsTxtContent = result.html;
+        const parser = new RobotsParser(result.html);
+        const crawlDelay = parser.getCrawlDelay();
+        if (crawlDelay !== null) {
+          const domain = new URL(origin).hostname;
+          this.rateLimiter.setRobotsCrawlDelay(domain, crawlDelay);
         }
-        const traversalStrategy = strategy === 'map' ? 'bfs' : strategy;
-        return this.crawlTraversal(input, seedOrigin, maxDepth, maxPages, traversalStrategy, robotsParser);
+        return parser;
+      }
+    } catch {
+      log.debug("Could not fetch robots.txt", { origin });
     }
-    robotsTxtContent = null;
-    async fetchRobots(origin) {
-        try {
-            const result = await this.rawFetchFn(`${origin}/robots.txt`);
-            if (result.statusCode === 200 && result.html) {
-                this.robotsTxtContent = result.html;
-                const parser = new RobotsParser(result.html);
-                const crawlDelay = parser.getCrawlDelay();
-                if (crawlDelay !== null) {
-                    const domain = new URL(origin).hostname;
-                    this.rateLimiter.setRobotsCrawlDelay(domain, crawlDelay);
-                }
-                return parser;
-            }
+    return null;
+  }
+  async crawlTraversal(input, seedOrigin, maxDepth, maxPages, strategy, robotsParser) {
+    const visited = /* @__PURE__ */ new Set();
+    const pages = [];
+    const allLinks = [];
+    const indexing = isIndexingEnabled();
+    const queue = [[input.url, 0]];
+    visited.add(canonicalForCrawl(input.url));
+    while (queue.length > 0 && pages.length < maxPages) {
+      const next = strategy === "dfs" ? queue.pop() : queue.shift();
+      const [url, depth] = next;
+      if (robotsParser && !robotsParser.isAllowed(new URL(url).pathname)) {
+        log.debug("Blocked by robots.txt", { url });
+        continue;
+      }
+      const release = await this.rateLimiter.acquire(url);
+      let fetchResult;
+      try {
+        fetchResult = await this.fetchFn(url);
+      } catch (err) {
+        log.warn("Fetch failed during crawl", { url, error: String(err) });
+        release();
+        continue;
+      }
+      release();
+      if (fetchResult.error) {
+        log.warn("Fetch returned error", { url, error: fetchResult.error });
+        continue;
+      }
+      const item = {
+        url: fetchResult.url,
+        title: fetchResult.title,
+        markdown: fetchResult.markdown,
+        depth
+      };
+      pages.push(item);
+      if (indexing) await indexCrawlResult(item);
+      if (depth < maxDepth) {
+        const newLinks = this.filterLinks(fetchResult.links, seedOrigin, visited, input.include_patterns, input.exclude_patterns, robotsParser);
+        for (const link of newLinks) {
+          visited.add(canonicalForCrawl(link));
+          queue.push([link, depth + 1]);
         }
-        catch {
-            log.debug('Could not fetch robots.txt', { origin });
+        if (input.extract_links) {
+          for (const link of fetchResult.links) {
+            allLinks.push({ from: url, to: link });
+          }
         }
-        return null;
+      }
     }
-    async crawlTraversal(input, seedOrigin, maxDepth, maxPages, strategy, robotsParser) {
-        const visited = new Set();
-        const pages = [];
-        const allLinks = [];
-        // Queue: [url, depth]
-        const queue = [[input.url, 0]];
-        visited.add(input.url);
-        while (queue.length > 0 && pages.length < maxPages) {
-            const item = strategy === 'dfs' ? queue.pop() : queue.shift();
-            const [url, depth] = item;
-            // Check robots.txt
-            if (robotsParser && !robotsParser.isAllowed(new URL(url).pathname)) {
-                log.debug('Blocked by robots.txt', { url });
-                continue;
-            }
-            // Rate limit
-            const release = await this.rateLimiter.acquire(url);
-            let fetchResult;
-            try {
-                fetchResult = await this.fetchFn(url);
-            }
-            catch (err) {
-                log.warn('Fetch failed during crawl', { url, error: String(err) });
-                release();
-                continue;
-            }
-            release();
-            if (fetchResult.error) {
-                log.warn('Fetch returned error', { url, error: fetchResult.error });
-                continue;
-            }
-            pages.push({
-                url: fetchResult.url,
-                title: fetchResult.title,
-                markdown: fetchResult.markdown,
-                depth,
-            });
-            // Discover links for traversal
-            if (depth < maxDepth) {
-                const newLinks = this.filterLinks(fetchResult.links, seedOrigin, visited, input.include_patterns, input.exclude_patterns, robotsParser);
-                for (const link of newLinks) {
-                    visited.add(link);
-                    queue.push([link, depth + 1]);
-                }
-                if (input.extract_links) {
-                    for (const link of fetchResult.links) {
-                        allLinks.push({ from: url, to: link });
-                    }
-                }
+    return {
+      pages,
+      total_found: visited.size,
+      crawled: pages.length,
+      ...input.extract_links ? { links: allLinks } : {}
+    };
+  }
+  filterLinks(links, seedOrigin, visited, includePatterns, excludePatterns, robotsParser) {
+    const filtered = links.filter((link) => {
+      try {
+        const parsed = new URL(link);
+        if (parsed.origin !== seedOrigin) return false;
+        if (visited.has(canonicalForCrawl(link))) return false;
+        if (!matchesPatterns(link, includePatterns, excludePatterns)) return false;
+        if (robotsParser && !robotsParser.isAllowed(parsed.pathname)) return false;
+        return true;
+      } catch {
+        return false;
+      }
+    });
+    return filtered.sort((a, b) => {
+      const aDoc = isDocPage(a) ? 0 : 1;
+      const bDoc = isDocPage(b) ? 0 : 1;
+      return aDoc - bDoc;
+    });
+  }
+  async crawlSitemap(input, seedOrigin, maxPages, robotsParser) {
+    const sitemapUrls = await this.discoverSitemapUrls(seedOrigin, this.robotsTxtContent);
+    if (sitemapUrls.length === 0) {
+      log.info("No sitemap found, falling back to BFS");
+      return this.crawlTraversal(input, seedOrigin, input.max_depth ?? 2, maxPages, "bfs", robotsParser);
+    }
+    return this.crawlFromExplicitUrls(input, sitemapUrls, maxPages, robotsParser);
+  }
+  /**
+   * Crawl an explicit list of URLs (e.g. from a sitemap probe). Applies
+   * include/exclude patterns, robots.txt, max_pages, and rate limits the
+   * same way as crawlSitemap.
+   */
+  async crawlFromExplicitUrls(input, urls, maxPages, robotsParser) {
+    const filtered = urls.filter(
+      (url) => matchesPatterns(url, input.include_patterns, input.exclude_patterns)
+    );
+    const totalFound = filtered.length;
+    const toFetch = filtered.slice(0, maxPages);
+    const pages = [];
+    const allLinks = [];
+    const indexing = isIndexingEnabled();
+    for (const url of toFetch) {
+      if (pages.length >= maxPages) break;
+      if (robotsParser && !robotsParser.isAllowed(new URL(url).pathname)) continue;
+      const release = await this.rateLimiter.acquire(url);
+      try {
+        const result = await this.fetchFn(url);
+        release();
+        if (!result.error) {
+          const item = { url: result.url, title: result.title, markdown: result.markdown, depth: 0 };
+          pages.push(item);
+          if (indexing) await indexCrawlResult(item);
+          if (input.extract_links) {
+            for (const link of result.links) {
+              allLinks.push({ from: url, to: link });
             }
+          }
         }
-        // total_found = all unique URLs discovered (visited set), including unvisited queue items
-        return {
-            pages,
-            total_found: visited.size,
-            crawled: pages.length,
-            ...(input.extract_links ? { links: allLinks } : {}),
-        };
+      } catch (err) {
+        release();
+        log.warn("Sitemap fetch failed", { url, error: String(err) });
+      }
     }
-    filterLinks(links, seedOrigin, visited, includePatterns, excludePatterns, robotsParser) {
-        const filtered = links.filter((link) => {
-            try {
-                const parsed = new URL(link);
-                if (parsed.origin !== seedOrigin)
-                    return false;
-                if (visited.has(link))
-                    return false;
-                if (!matchesPatterns(link, includePatterns, excludePatterns))
-                    return false;
-                if (robotsParser && !robotsParser.isAllowed(parsed.pathname))
-                    return false;
-                return true;
-            }
-            catch {
-                return false;
-            }
-        });
-        // Prioritize documentation pages over marketing/nav pages
-        return filtered.sort((a, b) => {
-            const aDoc = isDocPage(a) ? 0 : 1;
-            const bDoc = isDocPage(b) ? 0 : 1;
-            return aDoc - bDoc;
-        });
+    return {
+      pages,
+      total_found: totalFound,
+      crawled: pages.length,
+      ...input.extract_links ? { links: allLinks } : {}
+    };
+  }
+  async discoverSitemapUrls(origin, robotsTxt) {
+    const sitemapLocations = [];
+    if (robotsTxt) {
+      sitemapLocations.push(...extractSitemapUrlFromRobots(robotsTxt));
     }
-    async crawlSitemap(input, seedOrigin, maxPages, robotsParser) {
-        // Discover sitemap URLs (pass already-fetched robots.txt content)
-        let sitemapUrls = await this.discoverSitemapUrls(seedOrigin, this.robotsTxtContent);
-        if (sitemapUrls.length === 0) {
-            log.info('No sitemap found, falling back to BFS');
-            return this.crawlTraversal(input, seedOrigin, input.max_depth ?? 2, maxPages, 'bfs', robotsParser);
-        }
-        // Filter by patterns
-        sitemapUrls = sitemapUrls.filter((url) => matchesPatterns(url, input.include_patterns, input.exclude_patterns));
-        const totalFound = sitemapUrls.length;
-        const toFetch = sitemapUrls.slice(0, maxPages);
-        const pages = [];
-        const allLinks = [];
-        for (const url of toFetch) {
-            if (pages.length >= maxPages)
-                break;
-            if (robotsParser && !robotsParser.isAllowed(new URL(url).pathname))
-                continue;
-            const release = await this.rateLimiter.acquire(url);
-            try {
-                const result = await this.fetchFn(url);
-                release();
-                if (!result.error) {
-                    pages.push({ url: result.url, title: result.title, markdown: result.markdown, depth: 0 });
-                    if (input.extract_links) {
-                        for (const link of result.links) {
-                            allLinks.push({ from: url, to: link });
-                        }
-                    }
-                }
-            }
-            catch (err) {
-                release();
-                log.warn('Sitemap fetch failed', { url, error: String(err) });
-            }
-        }
-        return {
-            pages,
-            total_found: totalFound,
-            crawled: pages.length,
-            ...(input.extract_links ? { links: allLinks } : {}),
-        };
+    if (sitemapLocations.length === 0) {
+      sitemapLocations.push(`${origin}/sitemap.xml`);
     }
-    async discoverSitemapUrls(origin, robotsTxt) {
-        const sitemapLocations = [];
-        // Check robots.txt for sitemap references (reuses already-fetched content)
-        if (robotsTxt) {
-            sitemapLocations.push(...extractSitemapUrlFromRobots(robotsTxt));
-        }
-        // Try default location
-        if (sitemapLocations.length === 0) {
-            sitemapLocations.push(`${origin}/sitemap.xml`);
-        }
-        const allUrls = [];
-        for (const sitemapUrl of sitemapLocations) {
+    const allUrls = [];
+    for (const sitemapUrl of sitemapLocations) {
+      try {
+        const result = await this.rawFetchFn(sitemapUrl);
+        if (result.statusCode !== 200) continue;
+        const indexUrls = parseSitemapIndex(result.html);
+        if (indexUrls.length > 0) {
+          for (const subUrl of indexUrls) {
             try {
-                const result = await this.rawFetchFn(sitemapUrl);
-                if (result.statusCode !== 200)
-                    continue;
-                // Check if it's a sitemap index
-                const indexUrls = parseSitemapIndex(result.html);
-                if (indexUrls.length > 0) {
-                    // Fetch each sub-sitemap
-                    for (const subUrl of indexUrls) {
-                        try {
-                            const subResult = await this.rawFetchFn(subUrl);
-                            if (subResult.statusCode === 200) {
-                                allUrls.push(...parseSitemap(subResult.html));
-                            }
-                        }
-                        catch {
-                            // skip failed sub-sitemaps
-                        }
-                    }
-                }
-                else {
-                    allUrls.push(...parseSitemap(result.html));
-                }
-            }
-            catch {
-                // skip failed sitemap fetches
+              const subResult = await this.rawFetchFn(subUrl);
+              if (subResult.statusCode === 200) {
+                allUrls.push(...parseSitemap(subResult.html));
+              }
+            } catch {
             }
+          }
+        } else {
+          allUrls.push(...parseSitemap(result.html));
         }
-        return allUrls;
+      } catch {
+      }
     }
+    return allUrls;
+  }
 }
-const DOC_PATH_PATTERNS = ['/docs/', '/guide/', '/api/', '/reference/'];
+const DOC_PATH_PATTERNS = ["/docs/", "/guide/", "/api/", "/reference/"];
 function isDocPage(url) {
-    const path = new URL(url).pathname.toLowerCase();
-    return DOC_PATH_PATTERNS.some(p => path.includes(p));
+  const path = new URL(url).pathname.toLowerCase();
+  return DOC_PATH_PATTERNS.some((p) => path.includes(p));
 }
+export {
+  Crawler
+};
 //# sourceMappingURL=crawler.js.map

package/dist/crawl/crawler.js.map

@@ -1 +1 @@
-{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/crawl/crawler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,cAAc,CAAC;AAC5F,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAKlC,MAAM,OAAO,OAAO;IACV,OAAO,CAAU;IACjB,UAAU,CAAa;IACvB,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IAExC,YAAY,OAAgB,EAAE,UAAsB;QAClD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAiB;QAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC;QACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC;QAEvC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAE7C,2CAA2C;QAC3C,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,IAAI,YAAY,GAAwB,IAAI,CAAC;QAC7C,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,YAAY,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QACpD,CAAC;QAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,iBAAiB,GAAG,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;QAChE,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,YAAY,CAAC,CAAC;IACrG,CAAC;IAEO,gBAAgB,GAAkB,IAAI,CAAC;IAEvC,KAAK,CAAC,WAAW,CAAC,MAAc;QACtC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,MAAM,aAAa,CAAC,CAAC;YAC7D,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC7C,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC;gBACpC,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,UAAU,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC1C,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;oBACxB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;oBACxC,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC3D,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,GAAG,CAAC,KAAK,CAAC,4BAA4B,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,KAAiB,EACjB,UAAkB,EAClB,QAAgB,EAChB,QAAgB,EAChB,QAAuB,EACvB,YAAiC;QAEjC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,KAAK,GAAsB,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAe,EAAE,CAAC;QAEhC,sBAAsB;QACtB,MAAM,KAAK,GAA4B,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;YAChE,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC;YAE1B,mBAAmB;YACnB,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnE,GAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC5C,SAAS;YACX,CAAC;YAED,aAAa;YACb,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEpD,IAAI,WAAwB,CAAC;YAC7B,IAAI,CAAC;gBACH,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACnE,OAAO,EAAE,CAAC;gBACV,SAAS;YACX,CAAC;YAED,OAAO,EAAE,CAAC;YAEV,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;gBACtB,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC;gBACpE,SAAS;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC;gBACT,GAAG,EAAE,WAAW,CAAC,GAAG;gBACpB,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,QAAQ,EAAE,WAAW,CAAC,QAAQ;gBAC9B,KAAK;aACN,CAAC,CAAC;YAEH,+BAA+B;YAC/B,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;gBAExI,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;oBAC5B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAClB,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;oBACxB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;wBACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;oBACzC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,0FAA0F;QAC1F,OAAO;YACL,KAAK;YACL,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,OAAO,EAAE,KAAK,CAAC,MAAM;YACrB,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC;IAEO,WAAW,CACjB,KAAe,EACf,UAAkB,EAClB,OAAoB,EACpB,eAAqC,EACrC,eAAqC,EACrC,YAAiC;QAEjC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YACrC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU;oBAAE,OAAO,KAAK,CAAC;gBAC/C,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,OAAO,KAAK,CAAC;gBACpC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,eAAe,EAAE,eAAe,CAAC;oBAAE,OAAO,KAAK,CAAC;gBAC3E,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;oBAAE,OAAO,KAAK,CAAC;gBAC3E,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,0DAA0D;QAC1D,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC5B,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,OAAO,IAAI,GAAG,IAAI,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAiB,EACjB,UAAkB,EAClB,QAAgB,EAChB,YAAiC;QAEjC,kEAAkE;QAClE,IAAI,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAEpF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,GAAG,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,SAAS,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QACrG,CAAC;QAED,qBAAqB;QACrB,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACvC,eAAe,CAAC,GAAG,EAAE,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,CAAC,CACrE,CAAC;QAEF,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;QACtC,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAsB,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAe,EAAE,CAAC;QAEhC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;gBAAE,MAAM;YAEpC,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;gBAAE,SAAS;YAE7E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEpD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACvC,OAAO,EAAE,CAAC;gBAEV,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;oBAClB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;oBAE1F,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;wBACxB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;4BAChC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;wBACzC,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,CAAC;gBACV,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO;YACL,KAAK;YACL,WAAW,EAAE,UAAU;YACvB,OAAO,EAAE,KAAK,CAAC,MAAM;YACrB,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAc,EAAE,SAAoC;QACpF,MAAM,gBAAgB,GAAa,EAAE,CAAC;QAEtC,2EAA2E;QAC3E,IAAI,SAAS,EAAE,CAAC;YACd,gBAAgB,CAAC,IAAI,CAAC,GAAG,2BAA2B,CAAC,SAAS,CAAC,CAAC,CAAC;QACnE,CAAC;QAED,uBAAuB;QACvB,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,gBAAgB,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,KAAK,MAAM,UAAU,IAAI,gBAAgB,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;gBACjD,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG;oBAAE,SAAS;gBAExC,gCAAgC;gBAChC,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACjD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,yBAAyB;oBACzB,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;wBAC/B,IAAI,CAAC;4BACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;4BAChD,IAAI,SAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gCACjC,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;4BAChD,CAAC;wBACH,CAAC;wBAAC,MAAM,CAAC;4BACP,2BAA2B;wBAC7B,CAAC;oBACH,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,8BAA8B;YAChC,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,MAAM,iBAAiB,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;AAExE,SAAS,SAAS,CAAC,GAAW;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IACjD,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC"}
+{"version":3,"sources":["../../src/crawl/crawler.ts"],"sourcesContent":["import type { FetchOutput, CrawlInput, CrawlOutput, CrawlResultItem, LinkEdge, RawFetchResult } from '../types.js';\nimport { matchesPatterns, canonicalForCrawl } from './url-utils.js';\nimport { RateLimiter } from './rate-limiter.js';\nimport { RobotsParser } from './robots.js';\nimport { parseSitemap, parseSitemapIndex, extractSitemapUrlFromRobots } from './sitemap.js';\nimport { probeSitemap } from './sitemap-first.js';\nimport { isIndexingEnabled, indexCrawlResult } from './index-to-vec.js';\nimport { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('crawl');\n\nexport type FetchFn = (url: string) => Promise<FetchOutput>;\nexport type RawFetchFn = (url: string) => Promise<RawFetchResult>;\n\nexport class Crawler {\n  private fetchFn: FetchFn;\n  private rawFetchFn: RawFetchFn;\n  private rateLimiter = new RateLimiter();\n\n  constructor(fetchFn: FetchFn, rawFetchFn: RawFetchFn) {\n    this.fetchFn = fetchFn;\n    this.rawFetchFn = rawFetchFn;\n  }\n\n  async crawl(input: CrawlInput): Promise<CrawlOutput> {\n    const strategy = input.strategy ?? 'bfs';\n    const maxDepth = input.max_depth ?? 2;\n    const maxPages = input.max_pages ?? 20;\n\n    const seedOrigin = new URL(input.url).origin;\n\n    // Fetch and parse robots.txt if configured\n    const config = getConfig();\n    let robotsParser: RobotsParser | null = null;\n    if (config.respectRobotsTxt) {\n      robotsParser = await this.fetchRobots(seedOrigin);\n    }\n\n    if (strategy === 'auto') {\n      const sitemapUrls = await probeSitemap(seedOrigin, this.rawFetchFn);\n      if (sitemapUrls && sitemapUrls.length > 0) {\n        log.info('auto strategy: using sitemap', { origin: seedOrigin, urls: sitemapUrls.length });\n        return this.crawlFromExplicitUrls(input, sitemapUrls, maxPages, robotsParser);\n      }\n      log.info('auto strategy: no sitemap found, falling back to BFS', { origin: seedOrigin });\n      return this.crawlTraversal(input, seedOrigin, maxDepth, maxPages, 'bfs', robotsParser);\n    }\n\n    if (strategy === 'sitemap') {\n      return this.crawlSitemap(input, seedOrigin, maxPages, robotsParser);\n    }\n\n    const traversalStrategy = strategy === 'map' ? 'bfs' : strategy;\n    return this.crawlTraversal(input, seedOrigin, maxDepth, maxPages, traversalStrategy, robotsParser);\n  }\n\n  private robotsTxtContent: string | null = null;\n\n  private async fetchRobots(origin: string): Promise<RobotsParser | null> {\n    try {\n      const result = await this.rawFetchFn(`${origin}/robots.txt`);\n      if (result.statusCode === 200 && result.html) {\n        this.robotsTxtContent = result.html;\n        const parser = new RobotsParser(result.html);\n        const crawlDelay = parser.getCrawlDelay();\n        if (crawlDelay !== null) {\n          const domain = new URL(origin).hostname;\n          this.rateLimiter.setRobotsCrawlDelay(domain, crawlDelay);\n        }\n        return parser;\n      }\n    } catch {\n      log.debug('Could not fetch robots.txt', { origin });\n    }\n    return null;\n  }\n\n  private async crawlTraversal(\n    input: CrawlInput,\n    seedOrigin: string,\n    maxDepth: number,\n    maxPages: number,\n    strategy: 'bfs' | 'dfs',\n    robotsParser: RobotsParser | null,\n  ): Promise<CrawlOutput> {\n    const visited = new Set<string>();\n    const pages: CrawlResultItem[] = [];\n    const allLinks: LinkEdge[] = [];\n    const indexing = isIndexingEnabled();\n\n    // Queue: [url, depth]\n    const queue: Array<[string, number]> = [[input.url, 0]];\n    visited.add(canonicalForCrawl(input.url));\n\n    while (queue.length > 0 && pages.length < maxPages) {\n      const next = strategy === 'dfs' ? queue.pop()! : queue.shift()!;\n      const [url, depth] = next;\n\n      // Check robots.txt\n      if (robotsParser && !robotsParser.isAllowed(new URL(url).pathname)) {\n        log.debug('Blocked by robots.txt', { url });\n        continue;\n      }\n\n      // Rate limit\n      const release = await this.rateLimiter.acquire(url);\n\n      let fetchResult: FetchOutput;\n      try {\n        fetchResult = await this.fetchFn(url);\n      } catch (err) {\n        log.warn('Fetch failed during crawl', { url, error: String(err) });\n        release();\n        continue;\n      }\n\n      release();\n\n      if (fetchResult.error) {\n        log.warn('Fetch returned error', { url, error: fetchResult.error });\n        continue;\n      }\n\n      const item: CrawlResultItem = {\n        url: fetchResult.url,\n        title: fetchResult.title,\n        markdown: fetchResult.markdown,\n        depth,\n      };\n      pages.push(item);\n\n      if (indexing) await indexCrawlResult(item);\n\n      // Discover links for traversal\n      if (depth < maxDepth) {\n        const newLinks = this.filterLinks(fetchResult.links, seedOrigin, visited, input.include_patterns, input.exclude_patterns, robotsParser);\n\n        for (const link of newLinks) {\n          visited.add(canonicalForCrawl(link));\n          queue.push([link, depth + 1]);\n        }\n\n        if (input.extract_links) {\n          for (const link of fetchResult.links) {\n            allLinks.push({ from: url, to: link });\n          }\n        }\n      }\n    }\n\n    // total_found = all unique URLs discovered (visited set), including unvisited queue items\n    return {\n      pages,\n      total_found: visited.size,\n      crawled: pages.length,\n      ...(input.extract_links ? { links: allLinks } : {}),\n    };\n  }\n\n  private filterLinks(\n    links: string[],\n    seedOrigin: string,\n    visited: Set<string>,\n    includePatterns: string[] | undefined,\n    excludePatterns: string[] | undefined,\n    robotsParser: RobotsParser | null,\n  ): string[] {\n    const filtered = links.filter((link) => {\n      try {\n        const parsed = new URL(link);\n        if (parsed.origin !== seedOrigin) return false;\n        if (visited.has(canonicalForCrawl(link))) return false;\n        if (!matchesPatterns(link, includePatterns, excludePatterns)) return false;\n        if (robotsParser && !robotsParser.isAllowed(parsed.pathname)) return false;\n        return true;\n      } catch {\n        return false;\n      }\n    });\n\n    // Prioritize documentation pages over marketing/nav pages\n    return filtered.sort((a, b) => {\n      const aDoc = isDocPage(a) ? 0 : 1;\n      const bDoc = isDocPage(b) ? 0 : 1;\n      return aDoc - bDoc;\n    });\n  }\n\n  async crawlSitemap(\n    input: CrawlInput,\n    seedOrigin: string,\n    maxPages: number,\n    robotsParser: RobotsParser | null,\n  ): Promise<CrawlOutput> {\n    // Discover sitemap URLs (pass already-fetched robots.txt content)\n    const sitemapUrls = await this.discoverSitemapUrls(seedOrigin, this.robotsTxtContent);\n\n    if (sitemapUrls.length === 0) {\n      log.info('No sitemap found, falling back to BFS');\n      return this.crawlTraversal(input, seedOrigin, input.max_depth ?? 2, maxPages, 'bfs', robotsParser);\n    }\n\n    return this.crawlFromExplicitUrls(input, sitemapUrls, maxPages, robotsParser);\n  }\n\n  /**\n   * Crawl an explicit list of URLs (e.g. from a sitemap probe). Applies\n   * include/exclude patterns, robots.txt, max_pages, and rate limits the\n   * same way as crawlSitemap.\n   */\n  private async crawlFromExplicitUrls(\n    input: CrawlInput,\n    urls: string[],\n    maxPages: number,\n    robotsParser: RobotsParser | null,\n  ): Promise<CrawlOutput> {\n    const filtered = urls.filter((url) =>\n      matchesPatterns(url, input.include_patterns, input.exclude_patterns),\n    );\n\n    const totalFound = filtered.length;\n    const toFetch = filtered.slice(0, maxPages);\n    const pages: CrawlResultItem[] = [];\n    const allLinks: LinkEdge[] = [];\n    const indexing = isIndexingEnabled();\n\n    for (const url of toFetch) {\n      if (pages.length >= maxPages) break;\n\n      if (robotsParser && !robotsParser.isAllowed(new URL(url).pathname)) continue;\n\n      const release = await this.rateLimiter.acquire(url);\n\n      try {\n        const result = await this.fetchFn(url);\n        release();\n\n        if (!result.error) {\n          const item: CrawlResultItem = { url: result.url, title: result.title, markdown: result.markdown, depth: 0 };\n          pages.push(item);\n\n          if (indexing) await indexCrawlResult(item);\n\n          if (input.extract_links) {\n            for (const link of result.links) {\n              allLinks.push({ from: url, to: link });\n            }\n          }\n        }\n      } catch (err) {\n        release();\n        log.warn('Sitemap fetch failed', { url, error: String(err) });\n      }\n    }\n\n    return {\n      pages,\n      total_found: totalFound,\n      crawled: pages.length,\n      ...(input.extract_links ? { links: allLinks } : {}),\n    };\n  }\n\n  private async discoverSitemapUrls(origin: string, robotsTxt: string | null | undefined): Promise<string[]> {\n    const sitemapLocations: string[] = [];\n\n    // Check robots.txt for sitemap references (reuses already-fetched content)\n    if (robotsTxt) {\n      sitemapLocations.push(...extractSitemapUrlFromRobots(robotsTxt));\n    }\n\n    // Try default location\n    if (sitemapLocations.length === 0) {\n      sitemapLocations.push(`${origin}/sitemap.xml`);\n    }\n\n    const allUrls: string[] = [];\n\n    for (const sitemapUrl of sitemapLocations) {\n      try {\n        const result = await this.rawFetchFn(sitemapUrl);\n        if (result.statusCode !== 200) continue;\n\n        // Check if it's a sitemap index\n        const indexUrls = parseSitemapIndex(result.html);\n        if (indexUrls.length > 0) {\n          // Fetch each sub-sitemap\n          for (const subUrl of indexUrls) {\n            try {\n              const subResult = await this.rawFetchFn(subUrl);\n              if (subResult.statusCode === 200) {\n                allUrls.push(...parseSitemap(subResult.html));\n              }\n            } catch {\n              // skip failed sub-sitemaps\n            }\n          }\n        } else {\n          allUrls.push(...parseSitemap(result.html));\n        }\n      } catch {\n        // skip failed sitemap fetches\n      }\n    }\n\n    return allUrls;\n  }\n}\n\nconst DOC_PATH_PATTERNS = ['/docs/', '/guide/', '/api/', '/reference/'];\n\nfunction isDocPage(url: string): boolean {\n  const path = new URL(url).pathname.toLowerCase();\n  return DOC_PATH_PATTERNS.some(p => path.includes(p));\n}\n"],"mappings":"AACA,SAAS,iBAAiB,yBAAyB;AACnD,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;AAC7B,SAAS,cAAc,mBAAmB,mCAAmC;AAC7E,SAAS,oBAAoB;AAC7B,SAAS,mBAAmB,wBAAwB;AACpD,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,OAAO;AAKzB,MAAM,QAAQ;AAAA,EACX;AAAA,EACA;AAAA,EACA,cAAc,IAAI,YAAY;AAAA,EAEtC,YAAY,SAAkB,YAAwB;AACpD,SAAK,UAAU;AACf,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,MAAM,OAAyC;AACnD,UAAM,WAAW,MAAM,YAAY;AACnC,UAAM,WAAW,MAAM,aAAa;AACpC,UAAM,WAAW,MAAM,aAAa;AAEpC,UAAM,aAAa,IAAI,IAAI,MAAM,GAAG,EAAE;AAGtC,UAAM,SAAS,UAAU;AACzB,QAAI,eAAoC;AACxC,QAAI,OAAO,kBAAkB;AAC3B,qBAAe,MAAM,KAAK,YAAY,UAAU;AAAA,IAClD;AAEA,QAAI,aAAa,QAAQ;AACvB,YAAM,cAAc,MAAM,aAAa,YAAY,KAAK,UAAU;AAClE,UAAI,eAAe,YAAY,SAAS,GAAG;AACzC,YAAI,KAAK,gCAAgC,EAAE,QAAQ,YAAY,MAAM,YAAY,OAAO,CAAC;AACzF,eAAO,KAAK,sBAAsB,OAAO,aAAa,UAAU,YAAY;AAAA,MAC9E;AACA,UAAI,KAAK,wDAAwD,EAAE,QAAQ,WAAW,CAAC;AACvF,aAAO,KAAK,eAAe,OAAO,YAAY,UAAU,UAAU,OAAO,YAAY;AAAA,IACvF;AAEA,QAAI,aAAa,WAAW;AAC1B,aAAO,KAAK,aAAa,OAAO,YAAY,UAAU,YAAY;AAAA,IACpE;AAEA,UAAM,oBAAoB,aAAa,QAAQ,QAAQ;AACvD,WAAO,KAAK,eAAe,OAAO,YAAY,UAAU,UAAU,mBAAmB,YAAY;AAAA,EACnG;AAAA,EAEQ,mBAAkC;AAAA,EAE1C,MAAc,YAAY,QAA8C;AACtE,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,WAAW,GAAG,MAAM,aAAa;AAC3D,UAAI,OAAO,eAAe,OAAO,OAAO,MAAM;AAC5C,aAAK,mBAAmB,OAAO;AAC/B,cAAM,SAAS,IAAI,aAAa,OAAO,IAAI;AAC3C,cAAM,aAAa,OAAO,cAAc;AACxC,YAAI,eAAe,MAAM;AACvB,gBAAM,SAAS,IAAI,IAAI,MAAM,EAAE;AAC/B,eAAK,YAAY,oBAAoB,QAAQ,UAAU;AAAA,QACzD;AACA,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AACN,UAAI,MAAM,8BAA8B,EAAE,OAAO,CAAC;AAAA,IACpD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,eACZ,OACA,YACA,UACA,UACA,UACA,cACsB;AACtB,UAAM,UAAU,oBAAI,IAAY;AAChC,UAAM,QAA2B,CAAC;AAClC,UAAM,WAAuB,CAAC;AAC9B,UAAM,WAAW,kBAAkB;AAGnC,UAAM,QAAiC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;AACtD,YAAQ,IAAI,kBAAkB,MAAM,GAAG,CAAC;AAExC,WAAO,MAAM,SAAS,KAAK,MAAM,SAAS,UAAU;AAClD,YAAM,OAAO,aAAa,QAAQ,MAAM,IAAI,IAAK,MAAM,MAAM;AAC7D,YAAM,CAAC,KAAK,KAAK,IAAI;AAGrB,UAAI,gBAAgB,CAAC,aAAa,UAAU,IAAI,IAAI,GAAG,EAAE,QAAQ,GAAG;AAClE,YAAI,MAAM,yBAAyB,EAAE,IAAI,CAAC;AAC1C;AAAA,MACF;AAGA,YAAM,UAAU,MAAM,KAAK,YAAY,QAAQ,GAAG;AAElD,UAAI;AACJ,UAAI;AACF,sBAAc,MAAM,KAAK,QAAQ,GAAG;AAAA,MACtC,SAAS,KAAK;AACZ,YAAI,KAAK,6BAA6B,EAAE,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AACjE,gBAAQ;AACR;AAAA,MACF;AAEA,cAAQ;AAER,UAAI,YAAY,OAAO;AACrB,YAAI,KAAK,wBAAwB,EAAE,KAAK,OAAO,YAAY,MAAM,CAAC;AAClE;AAAA,MACF;AAEA,YAAM,OAAwB;AAAA,QAC5B,KAAK,YAAY;AAAA,QACjB,OAAO,YAAY;AAAA,QACnB,UAAU,YAAY;AAAA,QACtB;AAAA,MACF;AACA,YAAM,KAAK,IAAI;AAEf,UAAI,SAAU,OAAM,iBAAiB,IAAI;AAGzC,UAAI,QAAQ,UAAU;AACpB,cAAM,WAAW,KAAK,YAAY,YAAY,OAAO,YAAY,SAAS,MAAM,kBAAkB,MAAM,kBAAkB,YAAY;AAEtI,mBAAW,QAAQ,UAAU;AAC3B,kBAAQ,IAAI,kBAAkB,IAAI,CAAC;AACnC,gBAAM,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC;AAAA,QAC9B;AAEA,YAAI,MAAM,eAAe;AACvB,qBAAW,QAAQ,YAAY,OAAO;AACpC,qBAAS,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,CAAC;AAAA,UACvC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,WAAO;AAAA,MACL;AAAA,MACA,aAAa,QAAQ;AAAA,MACrB,SAAS,MAAM;AAAA,MACf,GAAI,MAAM,gBAAgB,EAAE,OAAO,SAAS,IAAI,CAAC;AAAA,IACnD;AAAA,EACF;AAAA,EAEQ,YACN,OACA,YACA,SACA,iBACA,iBACA,cACU;AACV,UAAM,WAAW,MAAM,OAAO,CAAC,SAAS;AACtC,UAAI;AACF,cAAM,SAAS,IAAI,IAAI,IAAI;AAC3B,YAAI,OAAO,WAAW,WAAY,QAAO;AACzC,YAAI,QAAQ,IAAI,kBAAkB,IAAI,CAAC,EAAG,QAAO;AACjD,YAAI,CAAC,gBAAgB,MAAM,iBAAiB,eAAe,EAAG,QAAO;AACrE,YAAI,gBAAgB,CAAC,aAAa,UAAU,OAAO,QAAQ,EAAG,QAAO;AACrE,eAAO;AAAA,MACT,QAAQ;AACN,eAAO;AAAA,MACT;AAAA,IACF,CAAC;AAGD,WAAO,SAAS,KAAK,CAAC,GAAG,MAAM;AAC7B,YAAM,OAAO,UAAU,CAAC,IAAI,IAAI;AAChC,YAAM,OAAO,UAAU,CAAC,IAAI,IAAI;AAChC,aAAO,OAAO;AAAA,IAChB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,aACJ,OACA,YACA,UACA,cACsB;AAEtB,UAAM,cAAc,MAAM,KAAK,oBAAoB,YAAY,KAAK,gBAAgB;AAEpF,QAAI,YAAY,WAAW,GAAG;AAC5B,UAAI,KAAK,uCAAuC;AAChD,aAAO,KAAK,eAAe,OAAO,YAAY,MAAM,aAAa,GAAG,UAAU,OAAO,YAAY;AAAA,IACnG;AAEA,WAAO,KAAK,sBAAsB,OAAO,aAAa,UAAU,YAAY;AAAA,EAC9E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,sBACZ,OACA,MACA,UACA,cACsB;AACtB,UAAM,WAAW,KAAK;AAAA,MAAO,CAAC,QAC5B,gBAAgB,KAAK,MAAM,kBAAkB,MAAM,gBAAgB;AAAA,IACrE;AAEA,UAAM,aAAa,SAAS;AAC5B,UAAM,UAAU,SAAS,MAAM,GAAG,QAAQ;AAC1C,UAAM,QAA2B,CAAC;AAClC,UAAM,WAAuB,CAAC;AAC9B,UAAM,WAAW,kBAAkB;AAEnC,eAAW,OAAO,SAAS;AACzB,UAAI,MAAM,UAAU,SAAU;AAE9B,UAAI,gBAAgB,CAAC,aAAa,UAAU,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAG;AAEpE,YAAM,UAAU,MAAM,KAAK,YAAY,QAAQ,GAAG;AAElD,UAAI;AACF,cAAM,SAAS,MAAM,KAAK,QAAQ,GAAG;AACrC,gBAAQ;AAER,YAAI,CAAC,OAAO,OAAO;AACjB,gBAAM,OAAwB,EAAE,KAAK,OAAO,KAAK,OAAO,OAAO,OAAO,UAAU,OAAO,UAAU,OAAO,EAAE;AAC1G,gBAAM,KAAK,IAAI;AAEf,cAAI,SAAU,OAAM,iBAAiB,IAAI;AAEzC,cAAI,MAAM,eAAe;AACvB,uBAAW,QAAQ,OAAO,OAAO;AAC/B,uBAAS,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,CAAC;AAAA,YACvC;AAAA,UACF;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AACZ,gBAAQ;AACR,YAAI,KAAK,wBAAwB,EAAE,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,MAC9D;AAAA,IACF;AAEA,WAAO;AAAA,MACL;AAAA,MACA,aAAa;AAAA,MACb,SAAS,MAAM;AAAA,MACf,GAAI,MAAM,gBAAgB,EAAE,OAAO,SAAS,IAAI,CAAC;AAAA,IACnD;AAAA,EACF;AAAA,EAEA,MAAc,oBAAoB,QAAgB,WAAyD;AACzG,UAAM,mBAA6B,CAAC;AAGpC,QAAI,WAAW;AACb,uBAAiB,KAAK,GAAG,4BAA4B,SAAS,CAAC;AAAA,IACjE;AAGA,QAAI,iBAAiB,WAAW,GAAG;AACjC,uBAAiB,KAAK,GAAG,MAAM,cAAc;AAAA,IAC/C;AAEA,UAAM,UAAoB,CAAC;AAE3B,eAAW,cAAc,kBAAkB;AACzC,UAAI;AACF,cAAM,SAAS,MAAM,KAAK,WAAW,UAAU;AAC/C,YAAI,OAAO,eAAe,IAAK;AAG/B,cAAM,YAAY,kBAAkB,OAAO,IAAI;AAC/C,YAAI,UAAU,SAAS,GAAG;AAExB,qBAAW,UAAU,WAAW;AAC9B,gBAAI;AACF,oBAAM,YAAY,MAAM,KAAK,WAAW,MAAM;AAC9C,kBAAI,UAAU,eAAe,KAAK;AAChC,wBAAQ,KAAK,GAAG,aAAa,UAAU,IAAI,CAAC;AAAA,cAC9C;AAAA,YACF,QAAQ;AAAA,YAER;AAAA,UACF;AAAA,QACF,OAAO;AACL,kBAAQ,KAAK,GAAG,aAAa,OAAO,IAAI,CAAC;AAAA,QAC3C;AAAA,MACF,QAAQ;AAAA,MAER;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;AAEA,MAAM,oBAAoB,CAAC,UAAU,WAAW,SAAS,aAAa;AAEtE,SAAS,UAAU,KAAsB;AACvC,QAAM,OAAO,IAAI,IAAI,GAAG,EAAE,SAAS,YAAY;AAC/C,SAAO,kBAAkB,KAAK,OAAK,KAAK,SAAS,CAAC,CAAC;AACrD;","names":[]}

package/dist/crawl/dedup.d.ts

@@ -8,6 +8,7 @@ interface PageOutput {
     url: string;
     markdown: string;
 }
+export declare function stripRepeatedNavigationLines(pages: PageInput[]): PageInput[];
 export declare function deduplicatePages(pages: PageInput[], domain?: string): PageOutput[];
 export declare function getStoredBoilerplate(domain: string): string[];
 export declare function storeBoilerplate(domain: string, hashes: string[]): void;

package/dist/crawl/dedup.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/crawl/dedup.ts"],"names":[],"mappings":"AAGA,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA2B1D;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEvD;AAMD,UAAU,SAAS;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU,EAAE,CA+ClF;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAI7D;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAavE"}
+{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/crawl/dedup.ts"],"names":[],"mappings":"AAGA,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA2B1D;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEvD;AAMD,UAAU,SAAS;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAWD,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,SAAS,EAAE,CAqC5E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU,EAAE,CAiDlF;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAI7D;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAavE"}