@tyroneross/blog-scraper 0.1.0 → 0.3.0

package/dist/lib/web-scrapers/content-extractor.js

@@ -0,0 +1,531 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.globalContentExtractor = exports.ContentExtractor = void 0;
+const readability_1 = require("@mozilla/readability");
+const jsdom_1 = require("jsdom");
+const cheerio = __importStar(require("cheerio"));
+const p_limit_1 = __importDefault(require("p-limit"));
+const scraping_rate_limiter_1 = require("../scraping-rate-limiter");
+const robots_checker_1 = require("./robots-checker");
+class ContentExtractor {
+    constructor() {
+        this.userAgent = 'Mozilla/5.0 (compatible; AtomizeNews/1.0; +https://atomize-news.vercel.app)';
+        this.timeout = 15000; // 15 seconds
+        this.maxContentSize = 10 * 1024 * 1024; // 10MB max
+        this.minContentLength = 200; // Minimum 200 characters
+        this.wordsPerMinute = 200; // Average reading speed
+        this.ssrfProtection = {
+            isPrivateIP: (url) => {
+                try {
+                    const urlObj = new URL(url);
+                    const hostname = urlObj.hostname;
+                    // Check for private IP ranges
+                    const privateRanges = [
+                        /^127\./, // 127.0.0.0/8 (loopback)
+                        /^10\./, // 10.0.0.0/8 (private)
+                        /^172\.(1[6-9]|2[0-9]|3[01])\./, // 172.16.0.0/12 (private)
+                        /^192\.168\./, // 192.168.0.0/16 (private)
+                        /^169\.254\./, // 169.254.0.0/16 (link-local)
+                        /^::1$/, // IPv6 loopback
+                        /^fe80:/, // IPv6 link-local
+                        /^fc00:/, // IPv6 unique local
+                        /^fd00:/ // IPv6 unique local
+                    ];
+                    return privateRanges.some(range => range.test(hostname));
+                }
+                catch {
+                    return true; // If we can't parse, block it
+                }
+            },
+            isLocalhost: (url) => {
+                try {
+                    const urlObj = new URL(url);
+                    const hostname = urlObj.hostname.toLowerCase();
+                    return hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1';
+                }
+                catch {
+                    return true;
+                }
+            },
+            isAllowedProtocol: (url) => {
+                try {
+                    const urlObj = new URL(url);
+                    return urlObj.protocol === 'http:' || urlObj.protocol === 'https:';
+                }
+                catch {
+                    return false;
+                }
+            }
+        };
+    }
+    /**
+     * Extract content from a URL
+     */
+    async extractContent(url) {
+        console.log(`📖 [ContentExtractor] Starting content extraction from ${url}`);
+        try {
+            // SSRF protection
+            if (!this.ssrfProtection.isAllowedProtocol(url)) {
+                throw new Error(`Disallowed protocol: ${url}`);
+            }
+            if (this.ssrfProtection.isPrivateIP(url) || this.ssrfProtection.isLocalhost(url)) {
+                throw new Error(`Private/local IP not allowed: ${url}`);
+            }
+            // Check robots.txt compliance
+            const robotsCheck = await robots_checker_1.globalRobotsChecker.isAllowed(url);
+            if (!robotsCheck.allowed) {
+                console.warn(`🤖 [ContentExtractor] URL blocked by robots.txt: ${url} - ${robotsCheck.reason}`);
+                return null;
+            }
+            const html = await this.fetchContent(url);
+            if (!html) {
+                return null;
+            }
+            // Extract content using multiple methods
+            const extracted = await this.extractFromHTML(html, url);
+            if (!extracted) {
+                console.warn(`⚠️ [ContentExtractor] No content extracted from ${url}`);
+                return null;
+            }
+            // Validate extracted content
+            if (extracted.textContent.length < this.minContentLength) {
+                console.warn(`⚠️ [ContentExtractor] Content too short (${extracted.textContent.length} chars): ${url}`);
+                return null;
+            }
+            console.log(`✅ [ContentExtractor] Successfully extracted ${extracted.wordCount} words from ${url}`);
+            return extracted;
+        }
+        catch (error) {
+            console.error(`❌ [ContentExtractor] Error extracting content from ${url}:`, error);
+            return null;
+        }
+    }
+    /**
+     * Extract content from multiple URLs with configurable concurrency
+     */
+    async extractBatch(urls, options = {}) {
+        const concurrency = options.concurrency || 5;
+        console.log(`📖 [ContentExtractor] Starting parallel batch extraction of ${urls.length} URLs (concurrency: ${concurrency})`);
+        const limit = (0, p_limit_1.default)(concurrency);
+        let completed = 0;
+        const results = await Promise.all(urls.map(url => limit(async () => {
+            try {
+                const result = await this.extractContent(url);
+                completed++;
+                options.onProgress?.(completed, urls.length, url);
+                return result;
+            }
+            catch (error) {
+                console.error(`❌ [ContentExtractor] Error in batch extraction for ${url}:`, error);
+                completed++;
+                options.onProgress?.(completed, urls.length, url);
+                return null;
+            }
+        })));
+        const successful = results.filter(Boolean).length;
+        console.log(`📖 [ContentExtractor] Batch complete: ${successful}/${urls.length} successful`);
+        return results;
+    }
+    async fetchContent(url) {
+        try {
+            return await scraping_rate_limiter_1.globalRateLimiter.execute(url, async () => {
+                const controller = new AbortController();
+                const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+                try {
+                    const response = await fetch(url, {
+                        headers: {
+                            'User-Agent': this.userAgent,
+                            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                            'Accept-Language': 'en-US,en;q=0.9'
+                        },
+                        signal: controller.signal,
+                    });
+                    clearTimeout(timeoutId);
+                    if (!response.ok) {
+                        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                    }
+                    const contentLength = response.headers.get('content-length');
+                    if (contentLength && parseInt(contentLength) > this.maxContentSize) {
+                        throw new Error(`Content too large: ${contentLength} bytes`);
+                    }
+                    const html = await response.text();
+                    if (html.length > this.maxContentSize) {
+                        throw new Error(`Content too large: ${html.length} bytes`);
+                    }
+                    return html;
+                }
+                catch (error) {
+                    clearTimeout(timeoutId);
+                    throw error;
+                }
+            });
+        }
+        catch (error) {
+            console.error(`❌ [ContentExtractor] Error fetching content from ${url}:`, error);
+            return null;
+        }
+    }
+    async extractFromHTML(html, url) {
+        const errors = [];
+        try {
+            // Try Readability first (most reliable)
+            const readabilityResult = this.extractWithReadability(html, url);
+            if (readabilityResult && readabilityResult.textContent.length >= this.minContentLength) {
+                return {
+                    ...readabilityResult,
+                    extractionMethod: 'readability',
+                    confidence: 0.9
+                };
+            }
+            else {
+                errors.push('Readability extraction failed or content too short');
+            }
+        }
+        catch (error) {
+            errors.push(`Readability error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+        try {
+            // Fallback to manual extraction
+            const fallbackResult = this.extractWithFallback(html, url);
+            if (fallbackResult && fallbackResult.textContent.length >= this.minContentLength) {
+                return {
+                    ...fallbackResult,
+                    extractionMethod: 'fallback',
+                    confidence: 0.6,
+                    errors
+                };
+            }
+            else {
+                errors.push('Fallback extraction failed or content too short');
+            }
+        }
+        catch (error) {
+            errors.push(`Fallback error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+        // If both methods fail, return null
+        console.error(`❌ [ContentExtractor] All extraction methods failed for ${url}:`, errors);
+        return null;
+    }
+    extractWithReadability(html, url) {
+        try {
+            const dom = new jsdom_1.JSDOM(html, { url });
+            const document = dom.window.document;
+            const reader = new readability_1.Readability(document);
+            const article = reader.parse();
+            if (!article) {
+                return null;
+            }
+            // Extract structured data
+            const structured = this.extractStructuredData(html, url);
+            // Calculate metrics
+            const wordCount = this.countWords(article.textContent ?? '');
+            const readingTime = Math.ceil(wordCount / this.wordsPerMinute);
+            return {
+                url,
+                title: article.title || '',
+                content: article.content || '',
+                textContent: article.textContent || '',
+                excerpt: article.excerpt || undefined,
+                byline: article.byline || undefined,
+                publishedTime: this.extractPublishedTime(html),
+                siteName: article.siteName || this.extractSiteName(html),
+                lang: this.extractLanguage(html),
+                structured,
+                wordCount,
+                readingTime,
+                confidence: 0.9,
+                extractionMethod: 'readability',
+                extractedAt: new Date()
+            };
+        }
+        catch (error) {
+            console.error(`❌ [ContentExtractor] Readability extraction failed:`, error);
+            return null;
+        }
+    }
+    extractWithFallback(html, url) {
+        try {
+            const $ = cheerio.load(html);
+            // Remove unwanted elements
+            const unwantedSelectors = [
+                'script', 'style', 'nav', 'header', 'footer',
+                '.advertisement', '.ads', '.social-share', '.comments',
+                '.sidebar', '.navigation', '.menu', '.popup', '.modal'
+            ];
+            unwantedSelectors.forEach(selector => $(selector).remove());
+            // Try to find the main content
+            let content = '';
+            let title = '';
+            // Extract title
+            title = $('h1').first().text().trim() ||
+                $('title').text().trim() ||
+                $('meta[property="og:title"]').attr('content') || '';
+            // Try different content selectors
+            const contentSelectors = [
+                '.blog-post__body', // Arista blogs, HubSpot blogs
+                'article',
+                '.article-content',
+                '.post-content',
+                '.entry-content',
+                '.content',
+                'main',
+                '#content',
+                '.story-body'
+            ];
+            for (const selector of contentSelectors) {
+                const element = $(selector).first();
+                if (element.length > 0) {
+                    content = element.html() || '';
+                    if (content.length > this.minContentLength) {
+                        break;
+                    }
+                }
+            }
+            // If no specific content area found, try to extract from body
+            if (content.length < this.minContentLength) {
+                content = $('body').html() || '';
+            }
+            if (!content || content.length < this.minContentLength) {
+                return null;
+            }
+            const textContent = $(content).text().trim();
+            const wordCount = this.countWords(textContent);
+            const readingTime = Math.ceil(wordCount / this.wordsPerMinute);
+            // Extract structured data
+            const structured = this.extractStructuredData(html, url);
+            return {
+                url,
+                title,
+                content,
+                textContent,
+                excerpt: textContent.substring(0, 300) + '...',
+                publishedTime: this.extractPublishedTime(html),
+                siteName: this.extractSiteName(html),
+                lang: this.extractLanguage(html),
+                structured,
+                wordCount,
+                readingTime,
+                confidence: 0.6,
+                extractionMethod: 'fallback',
+                extractedAt: new Date()
+            };
+        }
+        catch (error) {
+            console.error(`❌ [ContentExtractor] Fallback extraction failed:`, error);
+            return null;
+        }
+    }
+    extractStructuredData(html, url) {
+        const structured = {};
+        try {
+            const $ = cheerio.load(html);
+            // Extract JSON-LD
+            const jsonLdScripts = [];
+            $('script[type="application/ld+json"]').each((_, element) => {
+                try {
+                    const jsonText = $(element).html();
+                    if (jsonText) {
+                        const data = JSON.parse(jsonText);
+                        jsonLdScripts.push(data);
+                    }
+                }
+                catch {
+                    // Skip malformed JSON-LD
+                }
+            });
+            if (jsonLdScripts.length > 0) {
+                structured.jsonLd = jsonLdScripts;
+            }
+            // Extract OpenGraph tags
+            const openGraph = {};
+            $('meta[property^="og:"]').each((_, element) => {
+                const property = $(element).attr('property');
+                const content = $(element).attr('content');
+                if (property && content) {
+                    openGraph[property] = content;
+                }
+            });
+            if (Object.keys(openGraph).length > 0) {
+                structured.openGraph = openGraph;
+            }
+            // Extract Twitter Card tags
+            const twitterCard = {};
+            $('meta[name^="twitter:"]').each((_, element) => {
+                const name = $(element).attr('name');
+                const content = $(element).attr('content');
+                if (name && content) {
+                    twitterCard[name] = content;
+                }
+            });
+            if (Object.keys(twitterCard).length > 0) {
+                structured.twitterCard = twitterCard;
+            }
+            // Extract microdata (basic support)
+            const microdata = [];
+            $('[itemscope]').each((_, element) => {
+                const $item = $(element);
+                const itemType = $item.attr('itemtype');
+                if (itemType) {
+                    const item = { '@type': itemType };
+                    $item.find('[itemprop]').each((_, propElement) => {
+                        const $prop = $(propElement);
+                        const propName = $prop.attr('itemprop');
+                        const propValue = $prop.attr('content') || $prop.text().trim();
+                        if (propName && propValue) {
+                            item[propName] = propValue;
+                        }
+                    });
+                    microdata.push(item);
+                }
+            });
+            if (microdata.length > 0) {
+                structured.microdata = microdata;
+            }
+        }
+        catch (error) {
+            console.warn(`⚠️ [ContentExtractor] Error extracting structured data:`, error);
+        }
+        return Object.keys(structured).length > 0 ? structured : undefined;
+    }
+    extractPublishedTime(html) {
+        try {
+            const $ = cheerio.load(html);
+            // Try different selectors for published time
+            const timeSelectors = [
+                'meta[property="article:published_time"]',
+                'meta[name="datePublished"]',
+                'meta[name="publishdate"]',
+                'time[datetime]',
+                '.published-date',
+                '.publish-date',
+                '.article-date'
+            ];
+            for (const selector of timeSelectors) {
+                const element = $(selector).first();
+                if (element.length > 0) {
+                    const timeStr = element.attr('content') || element.attr('datetime') || element.text().trim();
+                    if (timeStr) {
+                        const date = new Date(timeStr);
+                        if (!isNaN(date.getTime())) {
+                            return date;
+                        }
+                    }
+                }
+            }
+            return undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    extractSiteName(html) {
+        try {
+            const $ = cheerio.load(html);
+            return $('meta[property="og:site_name"]').attr('content') ||
+                $('meta[name="application-name"]').attr('content') ||
+                undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    extractLanguage(html) {
+        try {
+            const $ = cheerio.load(html);
+            return $('html').attr('lang') ||
+                $('meta[name="language"]').attr('content') ||
+                $('meta[http-equiv="content-language"]').attr('content') ||
+                undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    countWords(text) {
+        if (!text)
+            return 0;
+        return text.trim().split(/\s+/).filter(word => word.length > 0).length;
+    }
+    /**
+     * Validate extracted content quality
+     */
+    validateContent(content) {
+        const issues = [];
+        let score = 1.0;
+        // Check minimum content length
+        if (content.textContent.length < this.minContentLength) {
+            issues.push(`Content too short: ${content.textContent.length} characters`);
+            score -= 0.5;
+        }
+        // Check title quality
+        if (!content.title || content.title.length < 10) {
+            issues.push('Missing or too short title');
+            score -= 0.2;
+        }
+        else if (content.title.length > 200) {
+            issues.push('Title too long');
+            score -= 0.1;
+        }
+        // Check content-to-HTML ratio (detect pages with too much markup)
+        const htmlLength = content.content.length;
+        const textLength = content.textContent.length;
+        const ratio = textLength / htmlLength;
+        if (ratio < 0.1) {
+            issues.push('Low text-to-HTML ratio - may be poorly extracted');
+            score -= 0.2;
+        }
+        // Check for duplicate content indicators
+        const sentences = content.textContent.split('.').filter(s => s.trim().length > 10);
+        const uniqueSentences = new Set(sentences);
+        const duplicateRatio = (sentences.length - uniqueSentences.size) / sentences.length;
+        if (duplicateRatio > 0.3) {
+            issues.push('High duplicate content detected');
+            score -= 0.3;
+        }
+        return {
+            isValid: issues.length === 0 && score >= 0.5,
+            issues,
+            score: Math.max(0, score)
+        };
+    }
+}
+exports.ContentExtractor = ContentExtractor;
+// Default global instance
+exports.globalContentExtractor = new ContentExtractor();
+//# sourceMappingURL=content-extractor.js.map

package/dist/lib/web-scrapers/content-extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-extractor.js","sourceRoot":"","sources":["../../../lib/web-scrapers/content-extractor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sDAAmD;AACnD,iCAA8B;AAC9B,iDAAmC;AACnC,sDAA6B;AAC7B,oEAA6D;AAC7D,qDAAuD;AAsCvD,MAAa,gBAAgB;IAQ3B;QAPiB,cAAS,GAAG,6EAA6E,CAAC;QAC1F,YAAO,GAAG,KAAK,CAAC,CAAC,aAAa;QAC9B,mBAAc,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,WAAW;QAC9C,qBAAgB,GAAG,GAAG,CAAC,CAAC,yBAAyB;QACjD,mBAAc,GAAG,GAAG,CAAC,CAAC,wBAAwB;QAI7D,IAAI,CAAC,cAAc,GAAG;YACpB,WAAW,EAAE,CAAC,GAAW,EAAW,EAAE;gBACpC,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;oBAEjC,8BAA8B;oBAC9B,MAAM,aAAa,GAAG;wBACpB,QAAQ,EAAqB,yBAAyB;wBACtD,OAAO,EAAsB,uBAAuB;wBACpD,+BAA+B,EAAG,0BAA0B;wBAC5D,aAAa,EAAgB,2BAA2B;wBACxD,aAAa,EAAgB,8BAA8B;wBAC3D,OAAO,EAAsB,gBAAgB;wBAC7C,QAAQ,EAAqB,kBAAkB;wBAC/C,QAAQ,EAAqB,oBAAoB;wBACjD,QAAQ,CAAqB,oBAAoB;qBAClD,CAAC;oBAEF,OAAO,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,IAAI,CAAC,CAAC,8BAA8B;gBAC7C,CAAC;YACH,CAAC;YAED,WAAW,EAAE,CAAC,GAAW,EAAW,EAAE;gBACpC,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;oBAC/C,OAAO,QAAQ,KAAK,WAAW,IAAI,QAAQ,KAAK,WAAW,IAAI,QAAQ,KAAK,KAAK,CAAC;gBACpF,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;YAED,iBAAiB,EAAE,CAAC,GAAW,EAAW,EAAE;gBAC1C,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5B,OAAO,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC;gBACrE,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,GAAW;QAC9B,OAAO,CAAC,GAAG,CAAC,0DAA0D,GAAG,EAAE,CAAC,CAAC;QAE7E,IAAI,CAAC;YACH,kBAAkB;YAClB,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChD,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,EAAE,CAAC,CAAC;YACjD,CAAC;YAED,IAAI,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjF,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,EAAE,CAAC,CAAC;YAC1D,CAAC;YAED,8BAA8B;YAC9B,MAAM,WAAW,GAAG,MAAM,oCAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,oDAAoD,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;gBAChG,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,IAAI,CAAC;YACd,CAAC;YAED,yCAAyC;YACzC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAExD,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,mDAAmD,GAAG,EAAE,CAAC,CAAC;gBACvE,OAAO,IAAI,CAAC;YACd,CAAC;YAED,6BAA6B;YAC7B,IAAI,SAAS,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC,4CAA4C,SAAS,CAAC,WAAW,CAAC,MAAM,YAAY,GAAG,EAAE,CAAC,CAAC;gBACxG,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,+CAA+C,SAAS,CAAC,SAAS,eAAe,GAAG,EAAE,CAAC,CAAC;YACpG,OAAO,SAAS,CAAC;QAEnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,sDAAsD,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACnF,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAChB,IAAc,EACd,UAGI,EAAE;QAEN,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,+DAA+D,IAAI,CAAC,MAAM,uBAAuB,WAAW,GAAG,CAAC,CAAC;QAE7H,MAAM,KAAK,GAAG,IAAA,iBAAM,EAAC,WAAW,CAAC,CAAC;QAClC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CACb,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;gBAC9C,SAAS,EAAE,CAAC;gBACZ,OAAO,CAAC,UAAU,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAClD,OAAO,MAAM,CAAC;YAChB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,sDAAsD,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;gBACnF,SAAS,EAAE,CAAC;gBACZ,OAAO,CAAC,UAAU,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAClD,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC,CACH,CACF,CAAC;QAEF,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,yCAAyC,UAAU,IAAI,IAAI,CAAC,MAAM,aAAa,CAAC,CAAC;QAC7F,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,IAAI,CAAC;YACH,OAAO,MAAM,yCAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;gBACrD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAErE,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;wBAChC,OAAO,EAAE;4BACP,YAAY,EAAE,IAAI,CAAC,SAAS;4BAC5B,QAAQ,EAAE,iEAAiE;4BAC3E,iBAAiB,EAAE,gBAAgB;yBACpC;wBACD,MAAM,EAAE,UAAU,CAAC,MAAM;qBAC1B,CAAC,CAAC;oBAEH,YAAY,CAAC,SAAS,CAAC,CAAC;oBAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;wBACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBACrE,CAAC;oBAED,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;oBAC7D,IAAI,aAAa,IAAI,QAAQ,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;wBACnE,MAAM,IAAI,KAAK,CAAC,sBAAsB,aAAa,QAAQ,CAAC,CAAC;oBAC/D,CAAC;oBAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAEnC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;wBACtC,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;oBAC7D,CAAC;oBAED,OAAO,IAAI,CAAC;gBAEd,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,YAAY,CAAC,SAAS,CAAC,CAAC;oBACxB,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,oDAAoD,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACjF,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,IAAY,EAAE,GAAW;QACrD,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,IAAI,CAAC;YACH,wCAAwC;YACxC,MAAM,iBAAiB,GAAG,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACjE,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvF,OAAO;oBACL,GAAG,iBAAiB;oBACpB,gBAAgB,EAAE,aAAa;oBAC/B,UAAU,EAAE,GAAG;iBAChB,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAChG,CAAC;QAED,IAAI,CAAC;YACH,gCAAgC;YAChC,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC3D,IAAI,cAAc,IAAI,cAAc,CAAC,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACjF,OAAO;oBACL,GAAG,cAAc;oBACjB,gBAAgB,EAAE,UAAU;oBAC5B,UAAU,EAAE,GAAG;oBACf,MAAM;iBACP,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,mBAAmB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC7F,CAAC;QAED,oCAAoC;QACpC,OAAO,CAAC,KAAK,CAAC,0DAA0D,GAAG,GAAG,EAAE,MAAM,CAAC,CAAC;QACxF,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,sBAAsB,CAAC,IAAY,EAAE,GAAW;QACtD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,aAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;YAErC,MAAM,MAAM,GAAG,IAAI,yBAAW,CAAC,QAAQ,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;YAE/B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,IAAI,CAAC;YACd,CAAC;YAED,0BAA0B;YAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,oBAAoB;YACpB,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;YAE/D,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE;gBAC1B,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;gBAC9B,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE;gBACtC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,SAAS;gBACrC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,SAAS;gBACnC,aAAa,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC;gBAC9C,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;gBACxD,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;gBAChC,UAAU;gBACV,SAAS;gBACT,WAAW;gBACX,UAAU,EAAE,GAAG;gBACf,gBAAgB,EAAE,aAAa;gBAC/B,WAAW,EAAE,IAAI,IAAI,EAAE;aACxB,CAAC;QAEJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,qDAAqD,EAAE,KAAK,CAAC,CAAC;YAC5E,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,IAAY,EAAE,GAAW;QACnD,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,2BAA2B;YAC3B,MAAM,iBAAiB,GAAG;gBACxB,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ;gBAC5C,gBAAgB,EAAE,MAAM,EAAE,eAAe,EAAE,WAAW;gBACtD,UAAU,EAAE,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ;aACvD,CAAC;YAEF,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAE5D,+BAA+B;YAC/B,IAAI,OAAO,GAAG,EAAE,CAAC;YACjB,IAAI,KAAK,GAAG,EAAE,CAAC;YAEf,gBAAgB;YAChB,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;gBACnC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;gBACxB,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAEvD,kCAAkC;YAClC,MAAM,gBAAgB,GAAG;gBACvB,kBAAkB,EAAO,8BAA8B;gBACvD,SAAS;gBACT,kBAAkB;gBAClB,eAAe;gBACf,gBAAgB;gBAChB,UAAU;gBACV,MAAM;gBACN,UAAU;gBACV,aAAa;aACd,CAAC;YAEF,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;gBACxC,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;gBACpC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;oBAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBAC3C,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YAED,8DAA8D;YAC9D,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC3C,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;YACnC,CAAC;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvD,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;YAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;YAE/D,0BAA0B;YAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,OAAO;gBACL,GAAG;gBACH,KAAK;gBACL,OAAO;gBACP,WAAW;gBACX,OAAO,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;gBAC9C,aAAa,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC;gBAC9C,QAAQ,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;gBACpC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;gBAChC,UAAU;gBACV,SAAS;gBACT,WAAW;gBACX,UAAU,EAAE,GAAG;gBACf,gBAAgB,EAAE,UAAU;gBAC5B,WAAW,EAAE,IAAI,IAAI,EAAE;aACxB,CAAC;QAEJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,kDAAkD,EAAE,KAAK,CAAC,CAAC;YACzE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,qBAAqB,CAAC,IAAY,EAAE,GAAW;QACrD,MAAM,UAAU,GAAmC,EAAE,CAAC;QAEtD,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,kBAAkB;YAClB,MAAM,aAAa,GAAU,EAAE,CAAC;YAChC,CAAC,CAAC,oCAAoC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBAC1D,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;oBACnC,IAAI,QAAQ,EAAE,CAAC;wBACb,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;wBAClC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC3B,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,yBAAyB;gBAC3B,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,UAAU,CAAC,MAAM,GAAG,aAAa,CAAC;YACpC,CAAC;YAED,yBAAyB;YACzB,MAAM,SAAS,GAA2B,EAAE,CAAC;YAC7C,CAAC,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBAC7C,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC7C,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC3C,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;oBACxB,SAAS,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;gBAChC,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC;YACnC,CAAC;YAED,4BAA4B;YAC5B,MAAM,WAAW,GAA2B,EAAE,CAAC;YAC/C,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBAC9C,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC3C,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;oBACpB,WAAW,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;gBAC9B,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxC,UAAU,CAAC,WAAW,GAAG,WAAW,CAAC;YACvC,CAAC;YAED,oCAAoC;YACpC,MAAM,SAAS,GAAU,EAAE,CAAC;YAC5B,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACnC,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;gBACzB,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACxC,IAAI,QAAQ,EAAE,CAAC;oBACb,MAAM,IAAI,GAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;oBACxC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE;wBAC/C,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;wBAC7B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;wBACxC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;wBAC/D,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;4BAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,SAAS,CAAC;wBAC7B,CAAC;oBACH,CAAC,CAAC,CAAC;oBACH,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC;YACnC,CAAC;QAEH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,yDAAyD,EAAE,KAAK,CAAC,CAAC;QACjF,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,CAAC;IAEO,oBAAoB,CAAC,IAAY;QACvC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,6CAA6C;YAC7C,MAAM,aAAa,GAAG;gBACpB,yCAAyC;gBACzC,4BAA4B;gBAC5B,0BAA0B;gBAC1B,gBAAgB;gBAChB,iBAAiB;gBACjB,eAAe;gBACf,eAAe;aAChB,CAAC;YAEF,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;gBACrC,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;gBACpC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;oBAC7F,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC/B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;4BAC3B,OAAO,IAAI,CAAC;wBACd,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,IAAY;QAClC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,OAAO,CAAC,CAAC,+BAA+B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;gBACvD,CAAC,CAAC,+BAA+B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;gBAClD,SAAS,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,IAAY;QAClC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC3B,CAAC,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC1C,CAAC,CAAC,qCAAqC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;gBACxD,SAAS,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,IAAY;QAC7B,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QACpB,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,OAAyB;QAKvC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,KAAK,GAAG,GAAG,CAAC;QAEhB,+BAA+B;QAC/B,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,WAAW,CAAC,MAAM,aAAa,CAAC,CAAC;YAC3E,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC1C,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;aAAM,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC9B,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;QAED,kEAAkE;QAClE,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1C,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC;QAC9C,MAAM,KAAK,GAAG,UAAU,GAAG,UAAU,CAAC;QAEtC,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;YAChE,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;QAED,yCAAyC;QACzC,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QACnF,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC3C,MAAM,cAAc,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAEpF,IAAI,cAAc,GAAG,GAAG,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;QAED,OAAO;YACL,OAAO,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,IAAI,GAAG;YAC5C,MAAM;YACN,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC;SAC1B,CAAC;IACJ,CAAC;CACF;AA7iBD,4CA6iBC;AAED,0BAA0B;AACb,QAAA,sBAAsB,GAAG,IAAI,gBAAgB,EAAE,CAAC"}

package/dist/lib/web-scrapers/html-scraper.d.ts

@@ -0,0 +1,74 @@
+declare const PERPLEXITY_MODELS: {
+    readonly SONAR: "llama-3.1-sonar-small-128k-online";
+    readonly SONAR_PRO: "llama-3.1-sonar-large-128k-online";
+};
+export interface ScrapingConfig {
+    selectors?: {
+        articleLinks?: string[];
+        titleSelectors?: string[];
+        dateSelectors?: string[];
+        excludeSelectors?: string[];
+    };
+    filters?: {
+        minTitleLength?: number;
+        maxTitleLength?: number;
+        includePatterns?: RegExp[];
+        excludePatterns?: RegExp[];
+        allowedDomains?: string[];
+    };
+    limits?: {
+        maxLinksPerPage?: number;
+        maxDepth?: number;
+    };
+    perplexityFallback?: {
+        enabled?: boolean;
+        model?: typeof PERPLEXITY_MODELS[keyof typeof PERPLEXITY_MODELS];
+        useForRobotsBlocked?: boolean;
+        useForParseFailed?: boolean;
+        searchRecency?: 'hour' | 'day' | 'week' | 'month';
+    };
+}
+export interface ExtractedArticle {
+    url: string;
+    title?: string;
+    publishedDate?: Date;
+    description?: string;
+    confidence: number;
+    source: 'link-text' | 'meta-data' | 'structured-data';
+}
+export declare class HTMLScraper {
+    private readonly userAgent;
+    private readonly timeout;
+    private readonly defaultConfig;
+    /**
+     * Extract article links from a webpage
+     */
+    extractArticleLinks(url: string, config?: ScrapingConfig): Promise<ExtractedArticle[]>;
+    /**
+     * Extract articles from multiple pages with pagination support
+     */
+    extractFromMultiplePages(startUrl: string, config?: ScrapingConfig, options?: {
+        maxPages?: number;
+        paginationSelector?: string;
+        nextPagePatterns?: RegExp[];
+    }): Promise<ExtractedArticle[]>;
+    private fetchPage;
+    private parseArticleLinks;
+    private extractArticleInfo;
+    private extractStructuredData;
+    private findNextPageUrls;
+    private deduplicateArticles;
+    private passesFilters;
+    private isLikelyArticleUrl;
+    private parseDate;
+    private resolveUrl;
+    private mergeConfig;
+    /**
+     * Use Perplexity API to extract articles when traditional scraping fails
+     * Requires PERPLEXITY_API_KEY environment variable to be set
+     */
+    private extractWithPerplexity;
+}
+export declare const globalHTMLScraper: HTMLScraper;
+export {};
+//# sourceMappingURL=html-scraper.d.ts.map

package/dist/lib/web-scrapers/html-scraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-scraper.d.ts","sourceRoot":"","sources":["../../../lib/web-scrapers/html-scraper.ts"],"names":[],"mappings":"AAMA,QAAA,MAAM,iBAAiB;;;CAGb,CAAC;AAEX,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE;QACV,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QACxB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC7B,CAAC;IACF,OAAO,CAAC,EAAE;QACR,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;KAC3B,CAAC;IACF,MAAM,CAAC,EAAE;QACP,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,kBAAkB,CAAC,EAAE;QACnB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,KAAK,CAAC,EAAE,OAAO,iBAAiB,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;QACjE,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,aAAa,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,OAAO,CAAC;KACnD,CAAC;CACH;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,IAAI,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,WAAW,GAAG,WAAW,GAAG,iBAAiB,CAAC;CACvD;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiF;IAC3G,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAqE5B;IAEF;;OAEG;IACG,mBAAmB,CACvB,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,cAAmB,GAC1B,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAqD9B;;OAEG;IACG,wBAAwB,CAC5B,QAAQ,EAAE,MAAM,EAChB,MAAM,GAAE,cAAmB,EAC3B,OAAO,GAAE;QACP,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;KACxB,GACL,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAgDhB,SAAS;IAuCvB,OAAO,CAAC,iBAAiB;IA4DzB,OAAO,CAAC,kBAAkB;IAqE1B,OAAO,CAAC,qBAAqB;YAsCf,gBAAgB;IAuC9B,OAAO,CAAC,mBAAmB;IAa3B,OAAO,CAAC,aAAa;IAkCrB,OAAO,CAAC,kBAAkB;IAiB1B,OAAO,CAAC,SAAS;IA8BjB,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,WAAW;IAiCnB;;;OAGG;YACW,qBAAqB;CAyEpC;AAGD,eAAO,MAAM,iBAAiB,aAAoB,CAAC"}