@memvid/maw 1.0.0

package/dist/src/crawler/index.js

@@ -0,0 +1,249 @@
+/**
+ * Web crawler with rate limiting
+ */
+import PQueue from 'p-queue';
+import { EngineWaterfall } from '../engine/index.js';
+import { Extractor } from '../extractor/index.js';
+import { RobotsParser } from './robots.js';
+import { SitemapParser } from './sitemap.js';
+import { createLogger } from '../utils/logger.js';
+import { normalizeUrl, shouldSkipUrl, getBaseDomain } from '../utils/url.js';
+import { DedupTracker } from '../utils/dedup.js';
+const DEFAULT_OPTIONS = {
+    depth: 2, // 2 levels deep (fast, good coverage)
+    concurrency: 10, // 10 concurrent requests
+    maxPages: 150, // 150 pages max (good coverage, avoids bloat)
+    rateLimit: 10, // 10 requests/sec
+    timeout: 10000, // 10s timeout (fail fast)
+    respectRobots: true,
+    useSitemap: true,
+};
+const log = createLogger();
+export class Crawler {
+    options;
+    engine;
+    extractor;
+    robots;
+    sitemap;
+    dedup;
+    visited = new Set();
+    queue;
+    baseHosts = new Set();
+    results = [];
+    pending = new Map();
+    constructor(options = {}) {
+        this.options = { ...DEFAULT_OPTIONS, ...options };
+        this.engine = new EngineWaterfall();
+        this.extractor = new Extractor();
+        this.robots = new RobotsParser();
+        this.sitemap = new SitemapParser();
+        this.dedup = new DedupTracker('en'); // Prefer English content
+        // Rate-limited queue
+        this.queue = new PQueue({
+            concurrency: this.options.concurrency,
+            interval: 1000,
+            intervalCap: this.options.rateLimit,
+        });
+    }
+    /**
+     * Crawl URLs and yield results as they complete
+     */
+    async *crawl(startUrls) {
+        const normalizedUrls = startUrls.map(u => normalizeUrl(u));
+        this.baseHosts = new Set(normalizedUrls.map(u => getBaseDomain(u)));
+        // Add start URLs FIRST (don't wait for sitemap)
+        for (const url of normalizedUrls) {
+            this.addToQueue(url, 0);
+        }
+        // Parse robots.txt (fast, ~1 request)
+        if (this.options.respectRobots) {
+            try {
+                await Promise.race([
+                    Promise.all(normalizedUrls.map(url => this.robots.parse(url))),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 3000))
+                ]);
+            }
+            catch {
+                // Timeout or error - continue without robots
+            }
+        }
+        // Get URLs from sitemap (with 5s total timeout)
+        if (this.options.useSitemap) {
+            try {
+                await Promise.race([
+                    this.parseSitemaps(normalizedUrls),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 5000))
+                ]);
+            }
+            catch {
+                // Timeout - continue with what we have
+            }
+        }
+        // Process queue and yield results with idle timeout
+        let lastYieldTime = Date.now();
+        let yieldedCount = 0;
+        const idleTimeout = 15000; // 15 seconds without new results = done
+        while (this.queue.size > 0 || this.queue.pending > 0 || this.results.length > 0) {
+            // Yield available results
+            while (this.results.length > 0) {
+                yield this.results.shift();
+                yieldedCount++;
+                lastYieldTime = Date.now();
+            }
+            // Stop if we've hit maxPages
+            if (yieldedCount >= this.options.maxPages) {
+                break;
+            }
+            // Check for idle timeout (no new results for too long)
+            if (Date.now() - lastYieldTime > idleTimeout) {
+                break;
+            }
+            // Wait a bit for more results
+            await new Promise(r => setTimeout(r, 50));
+        }
+        // Final results
+        while (this.results.length > 0) {
+            yield this.results.shift();
+        }
+    }
+    async parseSitemaps(urls) {
+        for (const url of urls) {
+            try {
+                // Only try first sitemap from robots.txt
+                const robotsSitemaps = this.robots.getSitemaps(url);
+                let sitemapUrls = [];
+                if (robotsSitemaps.length > 0) {
+                    // Just get first sitemap
+                    sitemapUrls = await this.sitemap.parseUrl(robotsSitemaps[0]);
+                }
+                else {
+                    sitemapUrls = await this.sitemap.parse(url);
+                }
+                // Add limited sitemap URLs
+                const limit = Math.min(this.options.maxPages, 50);
+                for (const sUrl of sitemapUrls.slice(0, limit)) {
+                    if (this.shouldCrawl(sUrl, 1)) {
+                        this.addToQueue(sUrl, 1);
+                    }
+                }
+            }
+            catch {
+                // No sitemap - continue
+            }
+        }
+    }
+    addToQueue(url, depth) {
+        const normalized = normalizeUrl(url);
+        if (this.visited.has(normalized))
+            return;
+        if (this.visited.size >= this.options.maxPages)
+            return;
+        this.visited.add(normalized);
+        this.queue.add(async () => {
+            await this.processUrl(url, depth);
+        });
+    }
+    async processUrl(url, depth) {
+        try {
+            const result = await this.engine.fetch(url, {
+                timeout: this.options.timeout,
+                forceEngine: this.options.forceEngine,
+            });
+            if (result.blocked) {
+                log.dim(`  Blocked: ${url}`);
+                return;
+            }
+            if (result.statusCode >= 400) {
+                log.dim(`  HTTP ${result.statusCode}: ${url}`);
+                return;
+            }
+            const extracted = this.extractor.extract(result.html, result.finalUrl);
+            // Skip pages with very little content
+            if (extracted.wordCount < 20) {
+                log.dim(`  Low content: ${url}`);
+                return;
+            }
+            // Add to results
+            this.results.push({
+                url,
+                finalUrl: result.finalUrl,
+                extracted,
+                depth,
+                engine: result.engine,
+            });
+            // Queue discovered links
+            if (depth < this.options.depth) {
+                for (const link of extracted.links) {
+                    if (this.shouldCrawl(link, depth + 1)) {
+                        this.addToQueue(link, depth + 1);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            log.dim(`  Failed: ${url} - ${error.message}`);
+        }
+    }
+    shouldCrawl(url, depth) {
+        try {
+            // Normalize
+            const normalized = normalizeUrl(url);
+            // Already visited
+            if (this.visited.has(normalized))
+                return false;
+            // Max pages
+            if (this.visited.size >= this.options.maxPages)
+                return false;
+            // Depth check
+            if (depth > this.options.depth)
+                return false;
+            // Skip non-HTML
+            if (shouldSkipUrl(url))
+                return false;
+            // Same domain only
+            const domain = getBaseDomain(url);
+            if (!this.baseHosts.has(domain))
+                return false;
+            // Robots.txt
+            if (this.options.respectRobots && !this.robots.isAllowed(url))
+                return false;
+            // Include/exclude patterns
+            if (this.options.includePattern && !this.options.includePattern.test(url))
+                return false;
+            if (this.options.excludePattern && this.options.excludePattern.test(url))
+                return false;
+            // Smart dedup - skip localized versions
+            const dedupResult = this.dedup.shouldSkip(url);
+            if (dedupResult.skip) {
+                log.dim(`  Skip: ${url} (${dedupResult.reason})`);
+                return false;
+            }
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Get engine statistics
+     */
+    getStats() {
+        return {
+            ...this.engine.getStats(),
+            visited: this.visited.size,
+            queued: this.queue.size,
+            pending: this.queue.pending,
+            dedup: this.dedup.getStats(),
+        };
+    }
+    /**
+     * Close all resources
+     */
+    async close() {
+        this.queue.clear();
+        await this.engine.close();
+    }
+}
+export { RobotsParser } from './robots.js';
+export { SitemapParser } from './sitemap.js';
+//# sourceMappingURL=index.js.map

package/dist/src/crawler/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/crawler/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAqB,MAAM,oBAAoB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAsB,MAAM,uBAAuB,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAuBjD,MAAM,eAAe,GAAiB;IACpC,KAAK,EAAE,CAAC,EAAe,sCAAsC;IAC7D,WAAW,EAAE,EAAE,EAAQ,yBAAyB;IAChD,QAAQ,EAAE,GAAG,EAAU,8CAA8C;IACrE,SAAS,EAAE,EAAE,EAAU,kBAAkB;IACzC,OAAO,EAAE,KAAK,EAAS,0BAA0B;IACjD,aAAa,EAAE,IAAI;IACnB,UAAU,EAAE,IAAI;CACjB,CAAC;AAEF,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;AAE3B,MAAM,OAAO,OAAO;IACV,OAAO,CAAe;IACtB,MAAM,CAAkB;IACxB,SAAS,CAAY;IACrB,MAAM,CAAe;IACrB,OAAO,CAAgB;IACvB,KAAK,CAAe;IACpB,OAAO,GAAgB,IAAI,GAAG,EAAE,CAAC;IACjC,KAAK,CAAS;IACd,SAAS,GAAgB,IAAI,GAAG,EAAE,CAAC;IACnC,OAAO,GAAkB,EAAE,CAAC;IAC5B,OAAO,GAAgD,IAAI,GAAG,EAAE,CAAC;IAEzE,YAAY,UAAiC,EAAE;QAC7C,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,GAAG,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,IAAI,aAAa,EAAE,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,yBAAyB;QAE9D,qBAAqB;QACrB,IAAI,CAAC,KAAK,GAAG,IAAI,MAAM,CAAC;YACtB,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW;YACrC,QAAQ,EAAE,IAAI;YACd,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS;SACpC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,KAAK,CAAC,SAAmB;QAC9B,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpE,gDAAgD;QAChD,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;YACjC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC1B,CAAC;QAED,sCAAsC;QACtC,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,IAAI,CAAC;oBACjB,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;oBAC9D,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;iBACjF,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,6CAA6C;YAC/C,CAAC;QACH,CAAC;QAED,gDAAgD;QAChD,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,IAAI,CAAC;oBACjB,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC;oBAClC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;iBACjF,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,uCAAuC;YACzC,CAAC;QACH,CAAC;QAED,oDAAoD;QACpD,IAAI,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,wCAAwC;QAEnE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChF,0BAA0B;YAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAG,CAAC;gBAC5B,YAAY,EAAE,CAAC;gBACf,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,CAAC;YAED,6BAA6B;YAC7B,IAAI,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC1C,MAAM;YACR,CAAC;YAED,uDAAuD;YACvD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,GAAG,WAAW,EAAE,CAAC;gBAC7C,MAAM;YACR,CAAC;YAED,8BAA8B;YAC9B,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;QAED,gBAAgB;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAG,CAAC;QAC9B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,IAAc;QACxC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,yCAAyC;gBACzC,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,WAAW,GAAa,EAAE,CAAC;gBAE/B,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,yBAAyB;oBACzB,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/D,CAAC;qBAAM,CAAC;oBACN,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC9C,CAAC;gBAED,2BAA2B;gBAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBAClD,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;oBAC/C,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;wBAC9B,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;oBAC3B,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,GAAW,EAAE,KAAa;QAC3C,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAErC,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;YAAE,OAAO;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ;YAAE,OAAO;QAEvD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,KAAa;QACjD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE;gBAC1C,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW;aACtC,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,GAAG,CAAC,GAAG,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC;gBAC7B,OAAO;YACT,CAAC;YAED,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;gBAC7B,GAAG,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC,CAAC;gBAC/C,OAAO;YACT,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEvE,sCAAsC;YACtC,IAAI,SAAS,CAAC,SAAS,GAAG,EAAE,EAAE,CAAC;gBAC7B,GAAG,CAAC,GAAG,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAC;gBACjC,OAAO;YACT,CAAC;YAED,iBAAiB;YACjB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAChB,GAAG;gBACH,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,SAAS;gBACT,KAAK;gBACL,MAAM,EAAE,MAAM,CAAC,MAAM;aACtB,CAAC,CAAC;YAEH,yBAAyB;YACzB,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBAC/B,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;oBACnC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;wBACtC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;oBACnC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,GAAG,CAAC,GAAG,CAAC,aAAa,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,GAAW,EAAE,KAAa;QAC5C,IAAI,CAAC;YACH,YAAY;YACZ,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAErC,kBAAkB;YAClB,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE/C,YAAY;YACZ,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAE7D,cAAc;YACd,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;gBAAE,OAAO,KAAK,CAAC;YAE7C,gBAAgB;YAChB,IAAI,aAAa,CAAC,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;YAErC,mBAAmB;YACnB,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE9C,aAAa;YACb,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE5E,2BAA2B;YAC3B,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;YACxF,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;YAEvF,wCAAwC;YACxC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAC/C,IAAI,WAAW,CAAC,IAAI,EAAE,CAAC;gBACrB,GAAG,CAAC,GAAG,CAAC,WAAW,GAAG,KAAK,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;gBAClD,OAAO,KAAK,CAAC;YACf,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO;YACL,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YAC1B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACvB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;YAC3B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE;SAC7B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;CACF;AAED,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC"}

package/dist/src/crawler/robots.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Robots.txt parser
+ */
+export declare class RobotsParser {
+    private rules;
+    private sitemaps;
+    /**
+     * Fetch and parse robots.txt for a URL
+     */
+    parse(url: string): Promise<void>;
+    private parseRobotsTxt;
+    /**
+     * Check if a URL is allowed by robots.txt
+     */
+    isAllowed(url: string, userAgent?: string): boolean;
+    /**
+     * Get crawl delay for a host
+     */
+    getCrawlDelay(url: string, userAgent?: string): number | undefined;
+    /**
+     * Get sitemaps declared in robots.txt
+     */
+    getSitemaps(url: string): string[];
+    private pathMatches;
+}
+//# sourceMappingURL=robots.d.ts.map

package/dist/src/crawler/robots.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../../src/crawler/robots.ts"],"names":[],"mappings":"AAAA;;GAEG;AASH,qBAAa,YAAY;IACvB,OAAO,CAAC,KAAK,CAAwC;IACrD,OAAO,CAAC,QAAQ,CAAoC;IAEpD;;OAEG;IACG,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA6BvC,OAAO,CAAC,cAAc;IAoEtB;;OAEG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,OAAO;IA0ChD;;OAEG;IACH,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,MAAM,GAAG,SAAS;IAiB/D;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE;IASlC,OAAO,CAAC,WAAW;CAoBpB"}

package/dist/src/crawler/robots.js

@@ -0,0 +1,179 @@
+/**
+ * Robots.txt parser
+ */
+export class RobotsParser {
+    rules = new Map();
+    sitemaps = new Map();
+    /**
+     * Fetch and parse robots.txt for a URL
+     */
+    async parse(url) {
+        try {
+            const parsedUrl = new URL(url);
+            const robotsUrl = `${parsedUrl.origin}/robots.txt`;
+            const host = parsedUrl.hostname;
+            // Already parsed
+            if (this.rules.has(host))
+                return;
+            const response = await fetch(robotsUrl, {
+                headers: { 'User-Agent': 'maw/1.0' },
+                signal: AbortSignal.timeout(5000),
+            });
+            if (!response.ok) {
+                // No robots.txt - allow everything
+                this.rules.set(host, []);
+                return;
+            }
+            const text = await response.text();
+            this.parseRobotsTxt(host, text);
+        }
+        catch {
+            // Failed to fetch - allow everything
+            const host = new URL(url).hostname;
+            this.rules.set(host, []);
+        }
+    }
+    parseRobotsTxt(host, text) {
+        const rules = [];
+        const sitemaps = [];
+        let currentRule = null;
+        const lines = text.split('\n');
+        for (const line of lines) {
+            const trimmed = line.trim();
+            // Skip comments and empty lines
+            if (!trimmed || trimmed.startsWith('#'))
+                continue;
+            const colonIndex = trimmed.indexOf(':');
+            if (colonIndex === -1)
+                continue;
+            const directive = trimmed.slice(0, colonIndex).toLowerCase().trim();
+            const value = trimmed.slice(colonIndex + 1).trim();
+            switch (directive) {
+                case 'user-agent':
+                    if (currentRule) {
+                        rules.push(currentRule);
+                    }
+                    currentRule = {
+                        userAgent: value.toLowerCase(),
+                        allow: [],
+                        disallow: [],
+                    };
+                    break;
+                case 'allow':
+                    if (currentRule && value) {
+                        currentRule.allow.push(value);
+                    }
+                    break;
+                case 'disallow':
+                    if (currentRule && value) {
+                        currentRule.disallow.push(value);
+                    }
+                    break;
+                case 'crawl-delay':
+                    if (currentRule) {
+                        const delay = parseFloat(value);
+                        if (!isNaN(delay)) {
+                            currentRule.crawlDelay = delay;
+                        }
+                    }
+                    break;
+                case 'sitemap':
+                    if (value) {
+                        sitemaps.push(value);
+                    }
+                    break;
+            }
+        }
+        if (currentRule) {
+            rules.push(currentRule);
+        }
+        this.rules.set(host, rules);
+        this.sitemaps.set(host, sitemaps);
+    }
+    /**
+     * Check if a URL is allowed by robots.txt
+     */
+    isAllowed(url, userAgent = '*') {
+        try {
+            const parsedUrl = new URL(url);
+            const host = parsedUrl.hostname;
+            const path = parsedUrl.pathname + parsedUrl.search;
+            const hostRules = this.rules.get(host);
+            if (!hostRules || hostRules.length === 0) {
+                return true; // No rules = allow
+            }
+            // Find matching rules (specific user-agent or *)
+            const matchingRules = hostRules.filter(r => r.userAgent === userAgent.toLowerCase() || r.userAgent === '*');
+            if (matchingRules.length === 0) {
+                return true; // No matching rules = allow
+            }
+            // Check rules (more specific rules take precedence)
+            for (const rule of matchingRules) {
+                // Check disallow first
+                for (const disallow of rule.disallow) {
+                    if (this.pathMatches(path, disallow)) {
+                        // Check if there's a more specific allow
+                        for (const allow of rule.allow) {
+                            if (this.pathMatches(path, allow) && allow.length > disallow.length) {
+                                return true;
+                            }
+                        }
+                        return false;
+                    }
+                }
+            }
+            return true;
+        }
+        catch {
+            return true;
+        }
+    }
+    /**
+     * Get crawl delay for a host
+     */
+    getCrawlDelay(url, userAgent = '*') {
+        try {
+            const host = new URL(url).hostname;
+            const hostRules = this.rules.get(host);
+            if (!hostRules)
+                return undefined;
+            const rule = hostRules.find(r => r.userAgent === userAgent.toLowerCase() || r.userAgent === '*');
+            return rule?.crawlDelay;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Get sitemaps declared in robots.txt
+     */
+    getSitemaps(url) {
+        try {
+            const host = new URL(url).hostname;
+            return this.sitemaps.get(host) || [];
+        }
+        catch {
+            return [];
+        }
+    }
+    pathMatches(path, pattern) {
+        // Simple pattern matching (supports * and $ wildcards)
+        if (pattern === '/')
+            return true;
+        // Convert pattern to regex
+        let regex = pattern
+            .replace(/[.+?^${}()|[\]\\]/g, '\\$&') // Escape special chars
+            .replace(/\*/g, '.*'); // * becomes .*
+        // $ at end means exact match
+        if (regex.endsWith('\\$')) {
+            regex = regex.slice(0, -2) + '$';
+        }
+        try {
+            return new RegExp('^' + regex).test(path);
+        }
+        catch {
+            return path.startsWith(pattern);
+        }
+    }
+}
+//# sourceMappingURL=robots.js.map

package/dist/src/crawler/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../../src/crawler/robots.ts"],"names":[],"mappings":"AAAA;;GAEG;AASH,MAAM,OAAO,YAAY;IACf,KAAK,GAA8B,IAAI,GAAG,EAAE,CAAC;IAC7C,QAAQ,GAA0B,IAAI,GAAG,EAAE,CAAC;IAEpD;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,GAAW;QACrB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,SAAS,GAAG,GAAG,SAAS,CAAC,MAAM,aAAa,CAAC;YACnD,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;YAEhC,iBAAiB;YACjB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,OAAO;YAEjC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;gBACtC,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE;gBACpC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;aAClC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,mCAAmC;gBACnC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACzB,OAAO;YACT,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,qCAAqC;YACrC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACnC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,IAAY;QAC/C,MAAM,KAAK,GAAiB,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,IAAI,WAAW,GAAsB,IAAI,CAAC;QAE1C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAE5B,gCAAgC;YAChC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAElD,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACxC,IAAI,UAAU,KAAK,CAAC,CAAC;gBAAE,SAAS;YAEhC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;YACpE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAEnD,QAAQ,SAAS,EAAE,CAAC;gBAClB,KAAK,YAAY;oBACf,IAAI,WAAW,EAAE,CAAC;wBAChB,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBAC1B,CAAC;oBACD,WAAW,GAAG;wBACZ,SAAS,EAAE,KAAK,CAAC,WAAW,EAAE;wBAC9B,KAAK,EAAE,EAAE;wBACT,QAAQ,EAAE,EAAE;qBACb,CAAC;oBACF,MAAM;gBAER,KAAK,OAAO;oBACV,IAAI,WAAW,IAAI,KAAK,EAAE,CAAC;wBACzB,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAChC,CAAC;oBACD,MAAM;gBAER,KAAK,UAAU;oBACb,IAAI,WAAW,IAAI,KAAK,EAAE,CAAC;wBACzB,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnC,CAAC;oBACD,MAAM;gBAER,KAAK,aAAa;oBAChB,IAAI,WAAW,EAAE,CAAC;wBAChB,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;wBAChC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;4BAClB,WAAW,CAAC,UAAU,GAAG,KAAK,CAAC;wBACjC,CAAC;oBACH,CAAC;oBACD,MAAM;gBAER,KAAK,SAAS;oBACZ,IAAI,KAAK,EAAE,CAAC;wBACV,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACvB,CAAC;oBACD,MAAM;YACV,CAAC;QACH,CAAC;QAED,IAAI,WAAW,EAAE,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,GAAG;QACpC,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;YAChC,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;YAEnD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzC,OAAO,IAAI,CAAC,CAAC,mBAAmB;YAClC,CAAC;YAED,iDAAiD;YACjD,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CACpC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,SAAS,KAAK,GAAG,CACpE,CAAC;YAEF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/B,OAAO,IAAI,CAAC,CAAC,4BAA4B;YAC3C,CAAC;YAED,oDAAoD;YACpD,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;gBACjC,uBAAuB;gBACvB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACrC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;wBACrC,yCAAyC;wBACzC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;4BAC/B,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gCACpE,OAAO,IAAI,CAAC;4BACd,CAAC;wBACH,CAAC;wBACD,OAAO,KAAK,CAAC;oBACf,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,GAAW,EAAE,SAAS,GAAG,GAAG;QACxC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAEvC,IAAI,CAAC,SAAS;gBAAE,OAAO,SAAS,CAAC;YAEjC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CACzB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,SAAS,KAAK,GAAG,CACpE,CAAC;YAEF,OAAO,IAAI,EAAE,UAAU,CAAC;QAC1B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,GAAW;QACrB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,IAAY,EAAE,OAAe;QAC/C,uDAAuD;QACvD,IAAI,OAAO,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;QAEjC,2BAA2B;QAC3B,IAAI,KAAK,GAAG,OAAO;aAChB,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,uBAAuB;aAC7D,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,eAAe;QAExC,6BAA6B;QAC7B,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1B,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;QACnC,CAAC;QAED,IAAI,CAAC;YACH,OAAO,IAAI,MAAM,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;CACF"}

package/dist/src/crawler/sitemap.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Sitemap.xml parser using sitemapper
+ */
+export interface SitemapUrl {
+    loc: string;
+    lastmod?: string;
+    changefreq?: string;
+    priority?: number;
+}
+export declare class SitemapParser {
+    private cache;
+    private sitemapper;
+    constructor();
+    /**
+     * Parse sitemap for a URL and return all URLs
+     */
+    parse(url: string): Promise<string[]>;
+    /**
+     * Parse sitemap from a specific URL
+     */
+    parseUrl(sitemapUrl: string): Promise<string[]>;
+    private fetchSitemap;
+    private parseSitemapContent;
+    private parseSitemapIndex;
+    private parseUrlset;
+    /**
+     * Parse sitemap for a URL and return all URLs with metadata
+     * Uses sitemapper for robust parsing of sitemap indexes and news sitemaps
+     */
+    parseWithMetadata(url: string): Promise<SitemapUrl[]>;
+    /**
+     * Parse sitemap with full metadata
+     */
+    parseWithMeta(content: string): SitemapUrl[];
+}
+//# sourceMappingURL=sitemap.d.ts.map

package/dist/src/crawler/sitemap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../../src/crawler/sitemap.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,KAAK,CAAoC;IACjD,OAAO,CAAC,UAAU,CAAa;;IAW/B;;OAEG;IACG,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAuC3C;;OAEG;IACG,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAYvC,YAAY;YAkBZ,mBAAmB;IA8BjC,OAAO,CAAC,iBAAiB;IAYzB,OAAO,CAAC,WAAW;IAYnB;;;OAGG;IACG,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IA8C3D;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,EAAE;CA0B7C"}