@tyroneross/blog-scraper 0.1.0 → 0.3.0

package/dist/lib/web-scrapers/robots-checker.js

@@ -0,0 +1,285 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.globalRobotsChecker = exports.RobotsChecker = void 0;
+class RobotsChecker {
+    constructor() {
+        this.cache = new Map();
+        this.cacheTimeout = 24 * 60 * 60 * 1000; // 24 hours
+        this.userAgent = 'AtomizeNews/1.0';
+        this.requestTimeout = 5000; // 5 seconds
+    }
+    /**
+     * Check if a URL is allowed to be crawled according to robots.txt
+     */
+    async isAllowed(url) {
+        try {
+            const urlObj = new URL(url);
+            const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
+            console.log(`🤖 [Robots] Checking ${url} against ${robotsUrl}`);
+            const robotsTxt = await this.getRobotsTxt(robotsUrl);
+            if (!robotsTxt) {
+                // If robots.txt doesn't exist or can't be fetched, allow by default
+                return {
+                    allowed: true,
+                    sitemaps: [],
+                    reason: 'No robots.txt found - allowing by default'
+                };
+            }
+            const result = this.checkRules(urlObj.pathname, robotsTxt);
+            console.log(`🤖 [Robots] ${result.allowed ? '✅ Allowed' : '❌ Blocked'}: ${url} - ${result.reason}`);
+            return result;
+        }
+        catch (error) {
+            console.warn(`⚠️ [Robots] Error checking robots.txt for ${url}:`, error);
+            // On error, default to allowing the request
+            return {
+                allowed: true,
+                sitemaps: [],
+                reason: `Error checking robots.txt: ${error instanceof Error ? error.message : 'Unknown error'}`
+            };
+        }
+    }
+    /**
+     * Get sitemaps listed in robots.txt for a domain
+     */
+    async getSitemaps(domain) {
+        try {
+            const robotsUrl = `https://${domain}/robots.txt`;
+            const robotsTxt = await this.getRobotsTxt(robotsUrl);
+            return robotsTxt ? robotsTxt.sitemaps : [];
+        }
+        catch (error) {
+            console.warn(`⚠️ [Robots] Error getting sitemaps for ${domain}:`, error);
+            return [];
+        }
+    }
+    /**
+     * Get the recommended crawl delay for a domain
+     */
+    async getCrawlDelay(domain) {
+        try {
+            const robotsUrl = `https://${domain}/robots.txt`;
+            const robotsTxt = await this.getRobotsTxt(robotsUrl);
+            if (!robotsTxt)
+                return undefined;
+            // Find the most specific rule for our user agent
+            const rule = this.findBestMatchingRule(robotsTxt.rules);
+            return rule?.crawlDelay;
+        }
+        catch (error) {
+            console.warn(`⚠️ [Robots] Error getting crawl delay for ${domain}:`, error);
+            return undefined;
+        }
+    }
+    async getRobotsTxt(robotsUrl) {
+        // Check cache first
+        const cached = this.cache.get(robotsUrl);
+        if (cached && Date.now() < cached.expiresAt) {
+            return cached;
+        }
+        try {
+            console.log(`🤖 [Robots] Fetching ${robotsUrl}`);
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), this.requestTimeout);
+            const response = await fetch(robotsUrl, {
+                headers: {
+                    'User-Agent': this.userAgent,
+                },
+                signal: controller.signal,
+            });
+            clearTimeout(timeoutId);
+            if (!response.ok) {
+                if (response.status === 404) {
+                    console.log(`🤖 [Robots] No robots.txt found at ${robotsUrl}`);
+                    return null;
+                }
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+            const text = await response.text();
+            const robotsTxt = this.parseRobotsTxt(text);
+            // Cache the result
+            this.cache.set(robotsUrl, robotsTxt);
+            console.log(`🤖 [Robots] Successfully parsed robots.txt for ${new URL(robotsUrl).hostname}`);
+            return robotsTxt;
+        }
+        catch (error) {
+            if (error instanceof Error && error.name === 'AbortError') {
+                console.warn(`⚠️ [Robots] Timeout fetching ${robotsUrl}`);
+            }
+            else {
+                console.warn(`⚠️ [Robots] Error fetching ${robotsUrl}:`, error);
+            }
+            return null;
+        }
+    }
+    parseRobotsTxt(text) {
+        const lines = text.split('\n').map(line => line.trim()).filter(line => line && !line.startsWith('#'));
+        const rules = [];
+        const globalSitemaps = [];
+        let currentRule = null;
+        for (const line of lines) {
+            const [key, ...valueParts] = line.split(':');
+            const value = valueParts.join(':').trim();
+            const lowerKey = key.toLowerCase().trim();
+            switch (lowerKey) {
+                case 'user-agent':
+                    // Start a new rule
+                    if (currentRule) {
+                        rules.push(this.completeRule(currentRule));
+                    }
+                    currentRule = {
+                        userAgent: value.toLowerCase(),
+                        disallows: [],
+                        allows: [],
+                        sitemaps: []
+                    };
+                    break;
+                case 'disallow':
+                    if (currentRule) {
+                        currentRule.disallows = currentRule.disallows || [];
+                        if (value) {
+                            currentRule.disallows.push(value);
+                        }
+                    }
+                    break;
+                case 'allow':
+                    if (currentRule) {
+                        currentRule.allows = currentRule.allows || [];
+                        if (value) {
+                            currentRule.allows.push(value);
+                        }
+                    }
+                    break;
+                case 'crawl-delay':
+                    if (currentRule && value) {
+                        const delay = parseFloat(value);
+                        if (!isNaN(delay)) {
+                            currentRule.crawlDelay = delay * 1000; // Convert to milliseconds
+                        }
+                    }
+                    break;
+                case 'sitemap':
+                    if (value) {
+                        globalSitemaps.push(value);
+                        if (currentRule) {
+                            currentRule.sitemaps = currentRule.sitemaps || [];
+                            currentRule.sitemaps.push(value);
+                        }
+                    }
+                    break;
+            }
+        }
+        // Don't forget the last rule
+        if (currentRule) {
+            rules.push(this.completeRule(currentRule));
+        }
+        const now = Date.now();
+        return {
+            rules,
+            sitemaps: globalSitemaps,
+            fetchedAt: now,
+            expiresAt: now + this.cacheTimeout
+        };
+    }
+    completeRule(partial) {
+        return {
+            userAgent: partial.userAgent || '*',
+            disallows: partial.disallows || [],
+            allows: partial.allows || [],
+            crawlDelay: partial.crawlDelay,
+            sitemaps: partial.sitemaps || []
+        };
+    }
+    checkRules(path, robotsTxt) {
+        // Find the best matching rule for our user agent
+        const rule = this.findBestMatchingRule(robotsTxt.rules);
+        if (!rule) {
+            return {
+                allowed: true,
+                sitemaps: robotsTxt.sitemaps,
+                reason: 'No applicable rules found'
+            };
+        }
+        // Check Allow rules first (they have priority over Disallow)
+        for (const allowPattern of rule.allows) {
+            if (this.matchesPattern(path, allowPattern)) {
+                return {
+                    allowed: true,
+                    crawlDelay: rule.crawlDelay,
+                    sitemaps: robotsTxt.sitemaps,
+                    reason: `Explicitly allowed by pattern: ${allowPattern}`
+                };
+            }
+        }
+        // Check Disallow rules
+        for (const disallowPattern of rule.disallows) {
+            if (this.matchesPattern(path, disallowPattern)) {
+                return {
+                    allowed: false,
+                    crawlDelay: rule.crawlDelay,
+                    sitemaps: robotsTxt.sitemaps,
+                    reason: `Blocked by pattern: ${disallowPattern}`
+                };
+            }
+        }
+        // If no rules match, allow by default
+        return {
+            allowed: true,
+            crawlDelay: rule.crawlDelay,
+            sitemaps: robotsTxt.sitemaps,
+            reason: 'No matching disallow rules'
+        };
+    }
+    findBestMatchingRule(rules) {
+        // Priority order: exact match for our user agent, then wildcard
+        const exactMatch = rules.find(rule => rule.userAgent === this.userAgent.toLowerCase());
+        if (exactMatch)
+            return exactMatch;
+        const wildcardMatch = rules.find(rule => rule.userAgent === '*');
+        if (wildcardMatch)
+            return wildcardMatch;
+        return null;
+    }
+    matchesPattern(path, pattern) {
+        if (pattern === '') {
+            // Empty disallow means allow everything
+            return false;
+        }
+        if (pattern === '/') {
+            // Root disallow means disallow everything
+            return true;
+        }
+        // Handle wildcards - simplified pattern matching
+        if (pattern.includes('*')) {
+            // Convert robots.txt wildcard pattern to regex
+            const regexPattern = pattern
+                .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') // Escape regex special chars
+                .replace(/\\\*/g, '.*'); // Convert * to .*
+            const regex = new RegExp('^' + regexPattern);
+            return regex.test(path);
+        }
+        // Simple prefix matching for patterns without wildcards
+        return path.startsWith(pattern);
+    }
+    // Clear cache (useful for testing)
+    clearCache() {
+        this.cache.clear();
+    }
+    // Get cache stats
+    getCacheStats() {
+        return {
+            size: this.cache.size,
+            entries: Array.from(this.cache.entries()).map(([url, data]) => ({
+                url,
+                fetchedAt: new Date(data.fetchedAt).toISOString(),
+                expiresAt: new Date(data.expiresAt).toISOString(),
+                rulesCount: data.rules.length,
+                sitemapsCount: data.sitemaps.length
+            }))
+        };
+    }
+}
+exports.RobotsChecker = RobotsChecker;
+// Default global instance
+exports.globalRobotsChecker = new RobotsChecker();
+//# sourceMappingURL=robots-checker.js.map

package/dist/lib/web-scrapers/robots-checker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots-checker.js","sourceRoot":"","sources":["../../../lib/web-scrapers/robots-checker.ts"],"names":[],"mappings":";;;AAeA,MAAa,aAAa;IAA1B;QACU,UAAK,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC5B,iBAAY,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW;QAC/C,cAAS,GAAG,iBAAiB,CAAC;QAC9B,mBAAc,GAAG,IAAI,CAAC,CAAC,YAAY;IAiUtD,CAAC;IA/TC;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,GAAW;QAMzB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC;YAElE,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,YAAY,SAAS,EAAE,CAAC,CAAC;YAEhE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;YACrD,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,oEAAoE;gBACpE,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ,EAAE,EAAE;oBACZ,MAAM,EAAE,2CAA2C;iBACpD,CAAC;YACJ,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAE3D,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,KAAK,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAEpG,OAAO,MAAM,CAAC;QAEhB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,6CAA6C,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACzE,4CAA4C;YAC5C,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE;aACjG,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,MAAc;QAC9B,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,WAAW,MAAM,aAAa,CAAC;YACjD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;YACrD,OAAO,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,0CAA0C,MAAM,GAAG,EAAE,KAAK,CAAC,CAAC;YACzE,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,MAAc;QAChC,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,WAAW,MAAM,aAAa,CAAC;YACjD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;YAErD,IAAI,CAAC,SAAS;gBAAE,OAAO,SAAS,CAAC;YAEjC,iDAAiD;YACjD,MAAM,IAAI,GAAG,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACxD,OAAO,IAAI,EAAE,UAAU,CAAC;QAE1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,6CAA6C,MAAM,GAAG,EAAE,KAAK,CAAC,CAAC;YAC5E,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,SAAiB;QAC1C,oBAAoB;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;YAC5C,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,wBAAwB,SAAS,EAAE,CAAC,CAAC;YAEjD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;YAE5E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;gBACtC,OAAO,EAAE;oBACP,YAAY,EAAE,IAAI,CAAC,SAAS;iBAC7B;gBACD,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,OAAO,CAAC,GAAG,CAAC,sCAAsC,SAAS,EAAE,CAAC,CAAC;oBAC/D,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACrE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAE5C,mBAAmB;YACnB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAErC,OAAO,CAAC,GAAG,CAAC,kDAAkD,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC7F,OAAO,SAAS,CAAC;QAEnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,OAAO,CAAC,IAAI,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAC;YAC5D,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,8BAA8B,SAAS,GAAG,EAAE,KAAK,CAAC,CAAC;YAClE,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACtG,MAAM,KAAK,GAAiB,EAAE,CAAC;QAC/B,MAAM,cAAc,GAAa,EAAE,CAAC;QAEpC,IAAI,WAAW,GAA+B,IAAI,CAAC;QAEnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC7C,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1C,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;YAE1C,QAAQ,QAAQ,EAAE,CAAC;gBACjB,KAAK,YAAY;oBACf,mBAAmB;oBACnB,IAAI,WAAW,EAAE,CAAC;wBAChB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC;oBAC7C,CAAC;oBACD,WAAW,GAAG;wBACZ,SAAS,EAAE,KAAK,CAAC,WAAW,EAAE;wBAC9B,SAAS,EAAE,EAAE;wBACb,MAAM,EAAE,EAAE;wBACV,QAAQ,EAAE,EAAE;qBACb,CAAC;oBACF,MAAM;gBAER,KAAK,UAAU;oBACb,IAAI,WAAW,EAAE,CAAC;wBAChB,WAAW,CAAC,SAAS,GAAG,WAAW,CAAC,SAAS,IAAI,EAAE,CAAC;wBACpD,IAAI,KAAK,EAAE,CAAC;4BACV,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACpC,CAAC;oBACH,CAAC;oBACD,MAAM;gBAER,KAAK,OAAO;oBACV,IAAI,WAAW,EAAE,CAAC;wBAChB,WAAW,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC;wBAC9C,IAAI,KAAK,EAAE,CAAC;4BACV,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACjC,CAAC;oBACH,CAAC;oBACD,MAAM;gBAER,KAAK,aAAa;oBAChB,IAAI,WAAW,IAAI,KAAK,EAAE,CAAC;wBACzB,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;wBAChC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;4BAClB,WAAW,CAAC,UAAU,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,0BAA0B;wBACnE,CAAC;oBACH,CAAC;oBACD,MAAM;gBAER,KAAK,SAAS;oBACZ,IAAI,KAAK,EAAE,CAAC;wBACV,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAC3B,IAAI,WAAW,EAAE,CAAC;4BAChB,WAAW,CAAC,QAAQ,GAAG,WAAW,CAAC,QAAQ,IAAI,EAAE,CAAC;4BAClD,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACnC,CAAC;oBACH,CAAC;oBACD,MAAM;YACV,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,IAAI,WAAW,EAAE,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,OAAO;YACL,KAAK;YACL,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,GAAG;YACd,SAAS,EAAE,GAAG,GAAG,IAAI,CAAC,YAAY;SACnC,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,OAA4B;QAC/C,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;YACnC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE;YAClC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE;YAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,EAAE;SACjC,CAAC;IACJ,CAAC;IAEO,UAAU,CAAC,IAAY,EAAE,SAAoB;QAMnD,iDAAiD;QACjD,MAAM,IAAI,GAAG,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAExD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,SAAS,CAAC,QAAQ;gBAC5B,MAAM,EAAE,2BAA2B;aACpC,CAAC;QACJ,CAAC;QAED,6DAA6D;QAC7D,KAAK,MAAM,YAAY,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACvC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,YAAY,CAAC,EAAE,CAAC;gBAC5C,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,SAAS,CAAC,QAAQ;oBAC5B,MAAM,EAAE,kCAAkC,YAAY,EAAE;iBACzD,CAAC;YACJ,CAAC;QACH,CAAC;QAED,uBAAuB;QACvB,KAAK,MAAM,eAAe,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC7C,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,CAAC;gBAC/C,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,SAAS,CAAC,QAAQ;oBAC5B,MAAM,EAAE,uBAAuB,eAAe,EAAE;iBACjD,CAAC;YACJ,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,OAAO;YACL,OAAO,EAAE,IAAI;YACb,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,QAAQ,EAAE,SAAS,CAAC,QAAQ;YAC5B,MAAM,EAAE,4BAA4B;SACrC,CAAC;IACJ,CAAC;IAEO,oBAAoB,CAAC,KAAmB;QAC9C,gEAAgE;QAChE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;QACvF,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC;QACjE,IAAI,aAAa;YAAE,OAAO,aAAa,CAAC;QAExC,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,OAAe;QAClD,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;YACnB,wCAAwC;YACxC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,OAAO,KAAK,GAAG,EAAE,CAAC;YACpB,0CAA0C;YAC1C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,iDAAiD;QACjD,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,+CAA+C;YAC/C,MAAM,YAAY,GAAG,OAAO;iBACzB,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC,6BAA6B;iBACpE,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB;YAE7C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,GAAG,YAAY,CAAC,CAAC;YAC7C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,wDAAwD;QACxD,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAED,mCAAmC;IACnC,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,kBAAkB;IAClB,aAAa;QACX,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC9D,GAAG;gBACH,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;gBACjD,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;gBACjD,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBAC7B,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;aACpC,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;CACF;AArUD,sCAqUC;AAED,0BAA0B;AACb,QAAA,mBAAmB,GAAG,IAAI,aAAa,EAAE,CAAC"}

package/dist/lib/web-scrapers/rss-discovery.d.ts

@@ -0,0 +1,62 @@
+export interface DiscoveredFeed {
+    url: string;
+    title?: string;
+    type: 'rss' | 'atom' | 'rdf';
+    source: 'link-tag' | 'common-path' | 'content-scan';
+    confidence: number;
+}
+export declare class RSSDiscovery {
+    private readonly userAgent;
+    private readonly timeout;
+    private readonly maxRedirects;
+    /**
+     * Discover RSS feeds from a given URL
+     */
+    discoverFeeds(url: string): Promise<DiscoveredFeed[]>;
+    /**
+     * Check if the URL itself is a direct feed
+     */
+    private checkDirectFeed;
+    /**
+     * Fetch HTML page content
+     */
+    private fetchPage;
+    /**
+     * Extract feed URLs from HTML link tags
+     */
+    private extractFeedsFromHTML;
+    /**
+     * Check common feed paths
+     */
+    private checkCommonPaths;
+    /**
+     * Scan HTML content for feed-like patterns
+     */
+    private scanForFeedContent;
+    /**
+     * Validate if a URL is actually a feed
+     */
+    private validateFeedUrl;
+    /**
+     * Resolve relative URLs to absolute URLs
+     */
+    private resolveUrl;
+    /**
+     * Check if content type indicates a feed
+     */
+    private isFeedContentType;
+    /**
+     * Determine feed type from content type
+     */
+    private determineFeedType;
+    /**
+     * Guess feed type from URL or text
+     */
+    private guessFeedType;
+    /**
+     * Check if a link looks like it could be a feed
+     */
+    private isFeedLikeLink;
+}
+export declare const globalRSSDiscovery: RSSDiscovery;
+//# sourceMappingURL=rss-discovery.d.ts.map

package/dist/lib/web-scrapers/rss-discovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rss-discovery.d.ts","sourceRoot":"","sources":["../../../lib/web-scrapers/rss-discovery.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC;IAC7B,MAAM,EAAE,UAAU,GAAG,aAAa,GAAG,cAAc,CAAC;IACpD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiF;IAC3G,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAK;IAElC;;OAEG;IACG,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAuD3D;;OAEG;YACW,eAAe;IAuC7B;;OAEG;YACW,SAAS;IAmCvB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAsD5B;;OAEG;YACW,gBAAgB;IA+C9B;;OAEG;YACW,kBAAkB;IAqChC;;OAEG;YACW,eAAe;IAgC7B;;OAEG;IACH,OAAO,CAAC,UAAU;IAQlB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IASzB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAOzB;;OAEG;IACH,OAAO,CAAC,aAAa;IAOrB;;OAEG;IACH,OAAO,CAAC,cAAc;CAUvB;AAGD,eAAO,MAAM,kBAAkB,cAAqB,CAAC"}