@tyroneross/blog-scraper 0.1.0 → 0.3.0

package/dist/lib/web-scrapers/rss-discovery.js

@@ -0,0 +1,384 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.globalRSSDiscovery = exports.RSSDiscovery = void 0;
+const cheerio = __importStar(require("cheerio"));
+const scraping_rate_limiter_1 = require("../scraping-rate-limiter");
+const robots_checker_1 = require("./robots-checker");
+class RSSDiscovery {
+    constructor() {
+        this.userAgent = 'Mozilla/5.0 (compatible; AtomizeNews/1.0; +https://atomize-news.vercel.app)';
+        this.timeout = 10000; // 10 seconds
+        this.maxRedirects = 3;
+    }
+    /**
+     * Discover RSS feeds from a given URL
+     */
+    async discoverFeeds(url) {
+        console.log(`🔍 [RSSDiscovery] Starting feed discovery for ${url}`);
+        const feeds = new Map();
+        try {
+            // Step 1: Check if the URL itself is a feed
+            const directFeed = await this.checkDirectFeed(url);
+            if (directFeed) {
+                feeds.set(directFeed.url, directFeed);
+                console.log(`✅ [RSSDiscovery] Direct feed found: ${directFeed.url}`);
+                return Array.from(feeds.values());
+            }
+            // Step 2: Check robots.txt compliance
+            const robotsCheck = await robots_checker_1.globalRobotsChecker.isAllowed(url);
+            if (!robotsCheck.allowed) {
+                console.warn(`🤖 [RSSDiscovery] URL blocked by robots.txt: ${url} - ${robotsCheck.reason}`);
+                return [];
+            }
+            // Step 3: Fetch and parse HTML page
+            const html = await this.fetchPage(url);
+            if (!html) {
+                return [];
+            }
+            // Step 4: Extract feeds from link tags in HTML
+            const linkFeeds = this.extractFeedsFromHTML(html, url);
+            linkFeeds.forEach(feed => feeds.set(feed.url, feed));
+            // Step 5: Try common feed paths if no feeds found in HTML
+            if (feeds.size === 0) {
+                const commonPathFeeds = await this.checkCommonPaths(url);
+                commonPathFeeds.forEach(feed => feeds.set(feed.url, feed));
+            }
+            // Step 6: Content-based feed discovery (look for feed-like content)
+            if (feeds.size === 0) {
+                const contentFeeds = await this.scanForFeedContent(html, url);
+                contentFeeds.forEach(feed => feeds.set(feed.url, feed));
+            }
+            const discoveredFeeds = Array.from(feeds.values());
+            discoveredFeeds.sort((a, b) => b.confidence - a.confidence); // Sort by confidence descending
+            console.log(`🔍 [RSSDiscovery] Discovered ${discoveredFeeds.length} feeds for ${url}`);
+            return discoveredFeeds;
+        }
+        catch (error) {
+            console.error(`❌ [RSSDiscovery] Error discovering feeds for ${url}:`, error);
+            return [];
+        }
+    }
+    /**
+     * Check if the URL itself is a direct feed
+     */
+    async checkDirectFeed(url) {
+        try {
+            const response = await scraping_rate_limiter_1.globalRateLimiter.execute(url, async () => {
+                const controller = new AbortController();
+                const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+                try {
+                    const res = await fetch(url, {
+                        method: 'HEAD',
+                        headers: { 'User-Agent': this.userAgent },
+                        signal: controller.signal,
+                    });
+                    clearTimeout(timeoutId);
+                    return res;
+                }
+                catch (error) {
+                    clearTimeout(timeoutId);
+                    throw error;
+                }
+            });
+            const contentType = response.headers.get('content-type') || '';
+            if (this.isFeedContentType(contentType)) {
+                const type = this.determineFeedType(contentType);
+                return {
+                    url,
+                    type,
+                    source: 'link-tag',
+                    confidence: 1.0
+                };
+            }
+            return null;
+        }
+        catch (error) {
+            // Not a direct feed, continue with other discovery methods
+            return null;
+        }
+    }
+    /**
+     * Fetch HTML page content
+     */
+    async fetchPage(url) {
+        try {
+            return await scraping_rate_limiter_1.globalRateLimiter.execute(url, async () => {
+                const controller = new AbortController();
+                const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+                try {
+                    const response = await fetch(url, {
+                        headers: { 'User-Agent': this.userAgent },
+                        signal: controller.signal,
+                    });
+                    clearTimeout(timeoutId);
+                    if (!response.ok) {
+                        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                    }
+                    const contentType = response.headers.get('content-type') || '';
+                    if (!contentType.includes('text/html')) {
+                        throw new Error(`Not HTML content: ${contentType}`);
+                    }
+                    return await response.text();
+                }
+                catch (error) {
+                    clearTimeout(timeoutId);
+                    throw error;
+                }
+            });
+        }
+        catch (error) {
+            console.error(`❌ [RSSDiscovery] Error fetching page ${url}:`, error);
+            return null;
+        }
+    }
+    /**
+     * Extract feed URLs from HTML link tags
+     */
+    extractFeedsFromHTML(html, baseUrl) {
+        const feeds = [];
+        try {
+            const $ = cheerio.load(html);
+            // Look for RSS/Atom feed links
+            $('link[rel="alternate"]').each((_, element) => {
+                const $link = $(element);
+                const type = $link.attr('type');
+                const href = $link.attr('href');
+                const title = $link.attr('title');
+                if (href && this.isFeedContentType(type || '')) {
+                    const absoluteUrl = this.resolveUrl(href, baseUrl);
+                    if (absoluteUrl) {
+                        feeds.push({
+                            url: absoluteUrl,
+                            title: title || undefined,
+                            type: this.determineFeedType(type || ''),
+                            source: 'link-tag',
+                            confidence: 0.9
+                        });
+                    }
+                }
+            });
+            // Look for other potential feed links
+            $('a[href]').each((_, element) => {
+                const $link = $(element);
+                const href = $link.attr('href');
+                const text = $link.text().toLowerCase().trim();
+                if (href && this.isFeedLikeLink(href, text)) {
+                    const absoluteUrl = this.resolveUrl(href, baseUrl);
+                    if (absoluteUrl && !feeds.some(f => f.url === absoluteUrl)) {
+                        feeds.push({
+                            url: absoluteUrl,
+                            title: $link.text().trim() || undefined,
+                            type: this.guessFeedType(href),
+                            source: 'content-scan',
+                            confidence: 0.6
+                        });
+                    }
+                }
+            });
+        }
+        catch (error) {
+            console.error(`❌ [RSSDiscovery] Error parsing HTML for feeds:`, error);
+        }
+        return feeds;
+    }
+    /**
+     * Check common feed paths
+     */
+    async checkCommonPaths(url) {
+        const baseUrl = new URL(url);
+        const commonPaths = [
+            '/feed/',
+            '/feed.xml',
+            '/rss/',
+            '/rss.xml',
+            '/feeds/',
+            '/feeds.xml',
+            '/atom.xml',
+            '/index.xml',
+            '/blog/feed/',
+            '/blog/rss.xml',
+            '/news/feed/',
+            '/news/rss.xml'
+        ];
+        const feeds = [];
+        for (const path of commonPaths) {
+            try {
+                const testUrl = `${baseUrl.protocol}//${baseUrl.host}${path}`;
+                // Check robots.txt for this specific path
+                const robotsCheck = await robots_checker_1.globalRobotsChecker.isAllowed(testUrl);
+                if (!robotsCheck.allowed) {
+                    continue;
+                }
+                const isValid = await this.validateFeedUrl(testUrl);
+                if (isValid) {
+                    feeds.push({
+                        url: testUrl,
+                        type: this.guessFeedType(path),
+                        source: 'common-path',
+                        confidence: 0.7
+                    });
+                }
+            }
+            catch (error) {
+                // Continue checking other paths
+                continue;
+            }
+        }
+        return feeds;
+    }
+    /**
+     * Scan HTML content for feed-like patterns
+     */
+    async scanForFeedContent(html, baseUrl) {
+        const feeds = [];
+        try {
+            const $ = cheerio.load(html);
+            // Look for URLs in the content that might be feeds
+            const text = $.text();
+            const urlRegex = /https?:\/\/[^\s]+(?:feed|rss|atom)[^\s]*/gi;
+            const matches = text.match(urlRegex);
+            if (matches) {
+                for (const match of matches) {
+                    const cleanUrl = match.replace(/[.,;:!?)]$/, ''); // Remove trailing punctuation
+                    const absoluteUrl = this.resolveUrl(cleanUrl, baseUrl);
+                    if (absoluteUrl && !feeds.some(f => f.url === absoluteUrl)) {
+                        const isValid = await this.validateFeedUrl(absoluteUrl);
+                        if (isValid) {
+                            feeds.push({
+                                url: absoluteUrl,
+                                type: this.guessFeedType(absoluteUrl),
+                                source: 'content-scan',
+                                confidence: 0.5
+                            });
+                        }
+                    }
+                }
+            }
+        }
+        catch (error) {
+            console.error(`❌ [RSSDiscovery] Error scanning content for feeds:`, error);
+        }
+        return feeds;
+    }
+    /**
+     * Validate if a URL is actually a feed
+     */
+    async validateFeedUrl(url) {
+        try {
+            return await scraping_rate_limiter_1.globalRateLimiter.execute(url, async () => {
+                const controller = new AbortController();
+                const timeoutId = setTimeout(() => controller.abort(), 5000); // Shorter timeout for validation
+                try {
+                    const response = await fetch(url, {
+                        method: 'HEAD',
+                        headers: { 'User-Agent': this.userAgent },
+                        signal: controller.signal,
+                    });
+                    clearTimeout(timeoutId);
+                    if (!response.ok) {
+                        return false;
+                    }
+                    const contentType = response.headers.get('content-type') || '';
+                    return this.isFeedContentType(contentType);
+                }
+                catch (error) {
+                    clearTimeout(timeoutId);
+                    return false;
+                }
+            });
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    /**
+     * Resolve relative URLs to absolute URLs
+     */
+    resolveUrl(url, baseUrl) {
+        try {
+            return new URL(url, baseUrl).toString();
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Check if content type indicates a feed
+     */
+    isFeedContentType(contentType) {
+        const lowerType = contentType.toLowerCase();
+        return lowerType.includes('application/rss+xml') ||
+            lowerType.includes('application/atom+xml') ||
+            lowerType.includes('application/rdf+xml') ||
+            lowerType.includes('text/xml') ||
+            lowerType.includes('application/xml');
+    }
+    /**
+     * Determine feed type from content type
+     */
+    determineFeedType(contentType) {
+        const lowerType = contentType.toLowerCase();
+        if (lowerType.includes('atom'))
+            return 'atom';
+        if (lowerType.includes('rdf'))
+            return 'rdf';
+        return 'rss'; // Default to RSS
+    }
+    /**
+     * Guess feed type from URL or text
+     */
+    guessFeedType(urlOrText) {
+        const lower = urlOrText.toLowerCase();
+        if (lower.includes('atom'))
+            return 'atom';
+        if (lower.includes('rdf'))
+            return 'rdf';
+        return 'rss'; // Default to RSS
+    }
+    /**
+     * Check if a link looks like it could be a feed
+     */
+    isFeedLikeLink(href, text) {
+        const lowerHref = href.toLowerCase();
+        const lowerText = text.toLowerCase();
+        const feedKeywords = ['rss', 'feed', 'atom', 'xml', 'syndication'];
+        return feedKeywords.some(keyword => lowerHref.includes(keyword) || lowerText.includes(keyword));
+    }
+}
+exports.RSSDiscovery = RSSDiscovery;
+// Default global instance
+exports.globalRSSDiscovery = new RSSDiscovery();
+//# sourceMappingURL=rss-discovery.js.map

package/dist/lib/web-scrapers/rss-discovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rss-discovery.js","sourceRoot":"","sources":["../../../lib/web-scrapers/rss-discovery.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAmC;AACnC,oEAA6D;AAC7D,qDAAuD;AAUvD,MAAa,YAAY;IAAzB;QACmB,cAAS,GAAG,6EAA6E,CAAC;QAC1F,YAAO,GAAG,KAAK,CAAC,CAAC,aAAa;QAC9B,iBAAY,GAAG,CAAC,CAAC;IA0XpC,CAAC;IAxXC;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,GAAW;QAC7B,OAAO,CAAC,GAAG,CAAC,iDAAiD,GAAG,EAAE,CAAC,CAAC;QAEpE,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;QAEhD,IAAI,CAAC;YACH,4CAA4C;YAC5C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YACnD,IAAI,UAAU,EAAE,CAAC;gBACf,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBACtC,OAAO,CAAC,GAAG,CAAC,uCAAuC,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC;gBACrE,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACpC,CAAC;YAED,sCAAsC;YACtC,MAAM,WAAW,GAAG,MAAM,oCAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,gDAAgD,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC5F,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,oCAAoC;YACpC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,+CAA+C;YAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvD,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAErD,0DAA0D;YAC1D,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACrB,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;gBACzD,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,CAAC;YAED,oEAAoE;YACpE,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACrB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAC9D,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC1D,CAAC;YAED,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACnD,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,gCAAgC;YAE7F,OAAO,CAAC,GAAG,CAAC,gCAAgC,eAAe,CAAC,MAAM,cAAc,GAAG,EAAE,CAAC,CAAC;YACvF,OAAO,eAAe,CAAC;QAEzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,gDAAgD,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YAC7E,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAAC,GAAW;QACvC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,yCAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;gBAC/D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAErE,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;wBAC3B,MAAM,EAAE,MAAM;wBACd,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;wBACzC,MAAM,EAAE,UAAU,CAAC,MAAM;qBAC1B,CAAC,CAAC;oBACH,YAAY,CAAC,SAAS,CAAC,CAAC;oBACxB,OAAO,GAAG,CAAC;gBACb,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,YAAY,CAAC,SAAS,CAAC,CAAC;oBACxB,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE/D,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,EAAE,CAAC;gBACxC,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;gBACjD,OAAO;oBACL,GAAG;oBACH,IAAI;oBACJ,MAAM,EAAE,UAAU;oBAClB,UAAU,EAAE,GAAG;iBAChB,CAAC;YACJ,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2DAA2D;YAC3D,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS,CAAC,GAAW;QACjC,IAAI,CAAC;YACH,OAAO,MAAM,yCAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;gBACrD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAErE,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;wBAChC,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;wBACzC,MAAM,EAAE,UAAU,CAAC,MAAM;qBAC1B,CAAC,CAAC;oBAEH,YAAY,CAAC,SAAS,CAAC,CAAC;oBAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;wBACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBACrE,CAAC;oBAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;wBACvC,MAAM,IAAI,KAAK,CAAC,qBAAqB,WAAW,EAAE,CAAC,CAAC;oBACtD,CAAC;oBAED,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC/B,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,YAAY,CAAC,SAAS,CAAC,CAAC;oBACxB,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,wCAAwC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,IAAY,EAAE,OAAe;QACxD,MAAM,KAAK,GAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,+BAA+B;YAC/B,CAAC,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBAC7C,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;gBACzB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAElC,IAAI,IAAI,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC;oBAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;oBACnD,IAAI,WAAW,EAAE,CAAC;wBAChB,KAAK,CAAC,IAAI,CAAC;4BACT,GAAG,EAAE,WAAW;4BAChB,KAAK,EAAE,KAAK,IAAI,SAAS;4BACzB,IAAI,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,IAAI,EAAE,CAAC;4BACxC,MAAM,EAAE,UAAU;4BAClB,UAAU,EAAE,GAAG;yBAChB,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,sCAAsC;YACtC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBAC/B,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;gBACzB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;gBAE/C,IAAI,IAAI,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;oBAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;oBACnD,IAAI,WAAW,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,WAAW,CAAC,EAAE,CAAC;wBAC3D,KAAK,CAAC,IAAI,CAAC;4BACT,GAAG,EAAE,WAAW;4BAChB,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS;4BACvC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;4BAC9B,MAAM,EAAE,cAAc;4BACtB,UAAU,EAAE,GAAG;yBAChB,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,gDAAgD,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,GAAW;QACxC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,WAAW,GAAG;YAClB,QAAQ;YACR,WAAW;YACX,OAAO;YACP,UAAU;YACV,SAAS;YACT,YAAY;YACZ,WAAW;YACX,YAAY;YACZ,aAAa;YACb,eAAe;YACf,aAAa;YACb,eAAe;SAChB,CAAC;QAEF,MAAM,KAAK,GAAqB,EAAE,CAAC;QAEnC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC;gBAE9D,0CAA0C;gBAC1C,MAAM,WAAW,GAAG,MAAM,oCAAmB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;oBACzB,SAAS;gBACX,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;gBACpD,IAAI,OAAO,EAAE,CAAC;oBACZ,KAAK,CAAC,IAAI,CAAC;wBACT,GAAG,EAAE,OAAO;wBACZ,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;wBAC9B,MAAM,EAAE,aAAa;wBACrB,UAAU,EAAE,GAAG;qBAChB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,gCAAgC;gBAChC,SAAS;YACX,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAAC,IAAY,EAAE,OAAe;QAC5D,MAAM,KAAK,GAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,mDAAmD;YACnD,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YACtB,MAAM,QAAQ,GAAG,4CAA4C,CAAC;YAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAErC,IAAI,OAAO,EAAE,CAAC;gBACZ,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;oBAC5B,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC,8BAA8B;oBAChF,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;oBAEvD,IAAI,WAAW,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,WAAW,CAAC,EAAE,CAAC;wBAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;wBACxD,IAAI,OAAO,EAAE,CAAC;4BACZ,KAAK,CAAC,IAAI,CAAC;gCACT,GAAG,EAAE,WAAW;gCAChB,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC;gCACrC,MAAM,EAAE,cAAc;gCACtB,UAAU,EAAE,GAAG;6BAChB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QAEH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,oDAAoD,EAAE,KAAK,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAAC,GAAW;QACvC,IAAI,CAAC;YACH,OAAO,MAAM,yCAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;gBACrD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,iCAAiC;gBAE/F,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;wBAChC,MAAM,EAAE,MAAM;wBACd,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;wBACzC,MAAM,EAAE,UAAU,CAAC,MAAM;qBAC1B,CAAC,CAAC;oBAEH,YAAY,CAAC,SAAS,CAAC,CAAC;oBAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;wBACjB,OAAO,KAAK,CAAC;oBACf,CAAC;oBAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC/D,OAAO,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;gBAE7C,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,YAAY,CAAC,SAAS,CAAC,CAAC;oBACxB,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,GAAW,EAAE,OAAe;QAC7C,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,WAAmB;QAC3C,MAAM,SAAS,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;QAC5C,OAAO,SAAS,CAAC,QAAQ,CAAC,qBAAqB,CAAC;YACzC,SAAS,CAAC,QAAQ,CAAC,sBAAsB,CAAC;YAC1C,SAAS,CAAC,QAAQ,CAAC,qBAAqB,CAAC;YACzC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;YAC9B,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,WAAmB;QAC3C,MAAM,SAAS,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;QAC5C,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAC;QAC9C,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAC5C,OAAO,KAAK,CAAC,CAAC,iBAAiB;IACjC,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,SAAiB;QACrC,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAC;QAC1C,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACxC,OAAO,KAAK,CAAC,CAAC,iBAAiB;IACjC,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,IAAY,EAAE,IAAY;QAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAErC,MAAM,YAAY,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;QAEnE,OAAO,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CACjC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAC3D,CAAC;IACJ,CAAC;CACF;AA7XD,oCA6XC;AAED,0BAA0B;AACb,QAAA,kBAAkB,GAAG,IAAI,YAAY,EAAE,CAAC"}

package/dist/lib/web-scrapers/sitemap-parser.d.ts

@@ -0,0 +1,65 @@
+export interface SitemapEntry {
+    url: string;
+    lastmod?: Date;
+    changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
+    priority?: number;
+    images?: SitemapImage[];
+    news?: SitemapNews;
+}
+export interface SitemapImage {
+    loc: string;
+    caption?: string;
+    title?: string;
+}
+export interface SitemapNews {
+    title: string;
+    publishedDate?: Date;
+    keywords?: string[];
+}
+export interface SitemapIndex {
+    sitemaps: {
+        loc: string;
+        lastmod?: Date;
+    }[];
+}
+export declare class SitemapParser {
+    private readonly userAgent;
+    private readonly timeout;
+    private readonly maxSitemapSize;
+    private readonly maxEntries;
+    private readonly recentTimeframe;
+    /**
+     * Parse sitemap from URL and return entries
+     */
+    parseSitemap(url: string, options?: {
+        filterRecent?: boolean;
+        maxEntries?: number;
+        includeImages?: boolean;
+        includeNews?: boolean;
+    }): Promise<SitemapEntry[]>;
+    /**
+     * Discover sitemaps from domain
+     */
+    discoverSitemaps(domain: string): Promise<string[]>;
+    /**
+     * Get recent entries from all sitemaps for a domain
+     */
+    getRecentEntries(domain: string, options?: {
+        hoursBack?: number;
+        maxEntries?: number;
+    }): Promise<SitemapEntry[]>;
+    private fetchSitemap;
+    private checkSitemapExists;
+    private isSitemapIndex;
+    private parseSitemapIndex;
+    private parseRegularSitemap;
+    /**
+     * Validate sitemap format
+     */
+    validateSitemapFormat(xml: string): {
+        valid: boolean;
+        errors: string[];
+    };
+}
+export declare const globalSitemapParser: SitemapParser;
+//# sourceMappingURL=sitemap-parser.d.ts.map

package/dist/lib/web-scrapers/sitemap-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap-parser.d.ts","sourceRoot":"","sources":["../../../lib/web-scrapers/sitemap-parser.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,IAAI,CAAC;IACf,UAAU,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC;IACxB,IAAI,CAAC,EAAE,WAAW,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,IAAI,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE;QACR,GAAG,EAAE,MAAM,CAAC;QACZ,OAAO,CAAC,EAAE,IAAI,CAAC;KAChB,EAAE,CAAC;CACL;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiF;IAC3G,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAoB;IACnD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAuB;IAEvD;;OAEG;IACG,YAAY,CAChB,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;QACP,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,WAAW,CAAC,EAAE,OAAO,CAAC;KAClB,GACL,OAAO,CAAC,YAAY,EAAE,CAAC;IA6B1B;;OAEG;IACG,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAwCzD;;OAEG;IACG,gBAAgB,CACpB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAO,GACxD,OAAO,CAAC,YAAY,EAAE,CAAC;YAmCZ,YAAY;YA6CZ,kBAAkB;IA0BhC,OAAO,CAAC,cAAc;YAIR,iBAAiB;IA6E/B,OAAO,CAAC,mBAAmB;IA6G3B;;OAEG;IACH,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE;CAiEzE;AAGD,eAAO,MAAM,mBAAmB,eAAsB,CAAC"}