@tyroneross/blog-scraper 0.1.0 → 0.3.0

package/dist/lib/web-scrapers/sitemap-parser.js

@@ -0,0 +1,430 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.globalSitemapParser = exports.SitemapParser = void 0;
+const cheerio = __importStar(require("cheerio"));
+const p_limit_1 = __importDefault(require("p-limit"));
+const scraping_rate_limiter_1 = require("../scraping-rate-limiter");
+const robots_checker_1 = require("./robots-checker");
+class SitemapParser {
+    constructor() {
+        this.userAgent = 'Mozilla/5.0 (compatible; AtomizeNews/1.0; +https://atomize-news.vercel.app)';
+        this.timeout = 15000; // 15 seconds for sitemaps
+        this.maxSitemapSize = 50 * 1024 * 1024; // 50MB max
+        this.maxEntries = 50000; // Max entries per sitemap
+        this.recentTimeframe = 48 * 60 * 60 * 1000; // 48 hours in ms
+    }
+    /**
+     * Parse sitemap from URL and return entries
+     */
+    async parseSitemap(url, options = {}) {
+        console.log(`🗺️ [Sitemap] Starting to parse ${url}`);
+        try {
+            // Check robots.txt compliance
+            const robotsCheck = await robots_checker_1.globalRobotsChecker.isAllowed(url);
+            if (!robotsCheck.allowed) {
+                console.warn(`🤖 [Sitemap] URL blocked by robots.txt: ${url} - ${robotsCheck.reason}`);
+                return [];
+            }
+            const xml = await this.fetchSitemap(url);
+            if (!xml) {
+                return [];
+            }
+            // Detect if this is a sitemap index or regular sitemap
+            if (this.isSitemapIndex(xml)) {
+                return await this.parseSitemapIndex(xml, options);
+            }
+            else {
+                return this.parseRegularSitemap(xml, options);
+            }
+        }
+        catch (error) {
+            console.error(`❌ [Sitemap] Error parsing sitemap ${url}:`, error);
+            return [];
+        }
+    }
+    /**
+     * Discover sitemaps from domain
+     */
+    async discoverSitemaps(domain) {
+        const sitemaps = [];
+        try {
+            // First, check robots.txt for sitemap declarations
+            const robotsSitemaps = await robots_checker_1.globalRobotsChecker.getSitemaps(domain);
+            sitemaps.push(...robotsSitemaps);
+            // Try common sitemap paths
+            const commonPaths = [
+                '/sitemap.xml',
+                '/sitemap_index.xml',
+                '/sitemaps.xml',
+                '/sitemap/',
+                '/news-sitemap.xml'
+            ];
+            for (const path of commonPaths) {
+                const sitemapUrl = `https://${domain}${path}`;
+                // Skip if already found in robots.txt
+                if (sitemaps.includes(sitemapUrl)) {
+                    continue;
+                }
+                const exists = await this.checkSitemapExists(sitemapUrl);
+                if (exists) {
+                    sitemaps.push(sitemapUrl);
+                }
+            }
+            console.log(`🗺️ [Sitemap] Discovered ${sitemaps.length} sitemaps for ${domain}`);
+            return Array.from(new Set(sitemaps)); // Remove duplicates
+        }
+        catch (error) {
+            console.error(`❌ [Sitemap] Error discovering sitemaps for ${domain}:`, error);
+            return [];
+        }
+    }
+    /**
+     * Get recent entries from all sitemaps for a domain
+     */
+    async getRecentEntries(domain, options = {}) {
+        const hoursBack = options.hoursBack || 48;
+        const maxEntries = options.maxEntries || 1000;
+        const sitemaps = await this.discoverSitemaps(domain);
+        const allEntries = [];
+        for (const sitemapUrl of sitemaps) {
+            try {
+                const entries = await this.parseSitemap(sitemapUrl, {
+                    filterRecent: true,
+                    maxEntries: Math.floor(maxEntries / sitemaps.length), // Distribute quota
+                    includeNews: true
+                });
+                allEntries.push(...entries);
+            }
+            catch (error) {
+                console.warn(`⚠️ [Sitemap] Error parsing ${sitemapUrl}:`, error);
+                continue;
+            }
+        }
+        // Filter by time and sort by lastmod
+        const cutoffTime = new Date(Date.now() - (hoursBack * 60 * 60 * 1000));
+        const recentEntries = allEntries
+            .filter(entry => entry.lastmod && entry.lastmod >= cutoffTime)
+            .sort((a, b) => {
+            if (!a.lastmod || !b.lastmod)
+                return 0;
+            return b.lastmod.getTime() - a.lastmod.getTime();
+        })
+            .slice(0, maxEntries);
+        console.log(`🗺️ [Sitemap] Found ${recentEntries.length} recent entries from ${domain}`);
+        return recentEntries;
+    }
+    async fetchSitemap(url) {
+        try {
+            return await scraping_rate_limiter_1.globalRateLimiter.execute(url, async () => {
+                const controller = new AbortController();
+                const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+                try {
+                    const response = await fetch(url, {
+                        headers: {
+                            'User-Agent': this.userAgent,
+                            'Accept': 'application/xml, text/xml, */*',
+                        },
+                        signal: controller.signal,
+                    });
+                    clearTimeout(timeoutId);
+                    if (!response.ok) {
+                        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                    }
+                    const contentLength = response.headers.get('content-length');
+                    if (contentLength && parseInt(contentLength) > this.maxSitemapSize) {
+                        throw new Error(`Sitemap too large: ${contentLength} bytes`);
+                    }
+                    const xml = await response.text();
+                    if (xml.length > this.maxSitemapSize) {
+                        throw new Error(`Sitemap too large: ${xml.length} bytes`);
+                    }
+                    return xml;
+                }
+                catch (error) {
+                    clearTimeout(timeoutId);
+                    throw error;
+                }
+            });
+        }
+        catch (error) {
+            console.error(`❌ [Sitemap] Error fetching ${url}:`, error);
+            return null;
+        }
+    }
+    async checkSitemapExists(url) {
+        try {
+            return await scraping_rate_limiter_1.globalRateLimiter.execute(url, async () => {
+                const controller = new AbortController();
+                const timeoutId = setTimeout(() => controller.abort(), 5000);
+                try {
+                    const response = await fetch(url, {
+                        method: 'HEAD',
+                        headers: { 'User-Agent': this.userAgent },
+                        signal: controller.signal,
+                    });
+                    clearTimeout(timeoutId);
+                    return response.ok;
+                }
+                catch (error) {
+                    clearTimeout(timeoutId);
+                    return false;
+                }
+            });
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    isSitemapIndex(xml) {
+        return xml.includes('<sitemapindex') || xml.includes('</sitemapindex>');
+    }
+    async parseSitemapIndex(xml, options) {
+        console.log(`🗺️ [Sitemap] Parsing sitemap index`);
+        const $ = cheerio.load(xml, { xmlMode: true });
+        const sitemaps = [];
+        const allEntries = [];
+        // Extract sitemap URLs from index
+        $('sitemap').each((_, element) => {
+            const $element = $(element);
+            const loc = $element.find('loc').first().text().trim();
+            if (loc) {
+                sitemaps.push(loc);
+            }
+        });
+        console.log(`🗺️ [Sitemap] Found ${sitemaps.length} sitemaps in index`);
+        // Prioritize US English sitemaps, then other English, then rest
+        const prioritizedSitemaps = sitemaps.sort((a, b) => {
+            const aLower = a.toLowerCase();
+            const bLower = b.toLowerCase();
+            // /en-us/ comes first
+            const aIsEnUs = aLower.includes('/en-us/') || aLower.includes('/en_us/');
+            const bIsEnUs = bLower.includes('/en-us/') || bLower.includes('/en_us/');
+            if (aIsEnUs && !bIsEnUs)
+                return -1;
+            if (bIsEnUs && !aIsEnUs)
+                return 1;
+            // Then other English locales
+            const aIsEnglish = /\/en[-_][a-z]{2}\//i.test(aLower);
+            const bIsEnglish = /\/en[-_][a-z]{2}\//i.test(bLower);
+            if (aIsEnglish && !bIsEnglish)
+                return -1;
+            if (bIsEnglish && !aIsEnglish)
+                return 1;
+            return 0;
+        });
+        // Parse each individual sitemap in PARALLEL (prioritize en-us, limit to 10)
+        const sitemapsToProcess = prioritizedSitemaps.slice(0, 10);
+        const entriesPerSitemap = Math.floor((options.maxEntries || this.maxEntries) / Math.min(sitemaps.length, 10));
+        // Parallel fetching with concurrency limit of 5
+        const limit = (0, p_limit_1.default)(5);
+        console.log(`🗺️ [Sitemap] Fetching ${sitemapsToProcess.length} sitemaps in parallel (concurrency: 5)`);
+        const results = await Promise.allSettled(sitemapsToProcess.map(sitemapUrl => limit(async () => {
+            const sitemapXml = await this.fetchSitemap(sitemapUrl);
+            if (sitemapXml) {
+                return this.parseRegularSitemap(sitemapXml, {
+                    ...options,
+                    maxEntries: entriesPerSitemap
+                });
+            }
+            return [];
+        })));
+        // Collect successful results
+        for (const result of results) {
+            if (result.status === 'fulfilled') {
+                allEntries.push(...result.value);
+            }
+            else {
+                console.warn(`⚠️ [Sitemap] Error parsing sitemap:`, result.reason);
+            }
+        }
+        console.log(`🗺️ [Sitemap] Parallel fetch complete: ${allEntries.length} entries from ${sitemapsToProcess.length} sitemaps`);
+        return allEntries;
+    }
+    parseRegularSitemap(xml, options) {
+        console.log(`🗺️ [Sitemap] Parsing regular sitemap`);
+        const $ = cheerio.load(xml, { xmlMode: true });
+        const entries = [];
+        const maxEntries = options.maxEntries || this.maxEntries;
+        const cutoffTime = options.filterRecent
+            ? new Date(Date.now() - this.recentTimeframe)
+            : null;
+        $('url').each((index, element) => {
+            if (entries.length >= maxEntries) {
+                return false; // Break the loop
+            }
+            const $element = $(element);
+            const loc = $element.find('loc').first().text().trim();
+            if (!loc)
+                return;
+            const entry = { url: loc };
+            // Parse lastmod
+            const lastmodText = $element.find('lastmod').first().text().trim();
+            if (lastmodText) {
+                const lastmod = new Date(lastmodText);
+                if (!isNaN(lastmod.getTime())) {
+                    entry.lastmod = lastmod;
+                }
+            }
+            // Filter by recency if requested
+            if (cutoffTime && entry.lastmod && entry.lastmod < cutoffTime) {
+                return; // Skip this entry
+            }
+            // Parse changefreq
+            const changefreq = $element.find('changefreq').first().text().trim();
+            if (changefreq) {
+                entry.changefreq = changefreq;
+            }
+            // Parse priority
+            const priorityText = $element.find('priority').first().text().trim();
+            if (priorityText) {
+                const priority = parseFloat(priorityText);
+                if (!isNaN(priority)) {
+                    entry.priority = priority;
+                }
+            }
+            // Parse images if requested
+            if (options.includeImages) {
+                const images = [];
+                $element.find('image\\:image').each((_, imgElement) => {
+                    const $img = $(imgElement);
+                    const imgLoc = $img.find('image\\:loc').first().text().trim();
+                    if (imgLoc) {
+                        images.push({
+                            loc: imgLoc,
+                            caption: $img.find('image\\:caption').first().text().trim() || undefined,
+                            title: $img.find('image\\:title').first().text().trim() || undefined,
+                        });
+                    }
+                });
+                if (images.length > 0) {
+                    entry.images = images;
+                }
+            }
+            // Parse news if requested
+            if (options.includeNews) {
+                const $news = $element.find('news\\:news');
+                if ($news.length > 0) {
+                    const title = $news.find('news\\:title').first().text().trim();
+                    if (title) {
+                        entry.news = { title };
+                        const pubDateText = $news.find('news\\:publication_date').first().text().trim();
+                        if (pubDateText) {
+                            const pubDate = new Date(pubDateText);
+                            if (!isNaN(pubDate.getTime())) {
+                                entry.news.publishedDate = pubDate;
+                            }
+                        }
+                        const keywords = $news.find('news\\:keywords').first().text().trim();
+                        if (keywords) {
+                            entry.news.keywords = keywords.split(',').map(k => k.trim());
+                        }
+                    }
+                }
+            }
+            entries.push(entry);
+        });
+        console.log(`🗺️ [Sitemap] Parsed ${entries.length} entries from sitemap`);
+        return entries;
+    }
+    /**
+     * Validate sitemap format
+     */
+    validateSitemapFormat(xml) {
+        const errors = [];
+        try {
+            const $ = cheerio.load(xml, { xmlMode: true });
+            // Check for root element
+            const hasUrlset = $('urlset').length > 0;
+            const hasSitemapIndex = $('sitemapindex').length > 0;
+            if (!hasUrlset && !hasSitemapIndex) {
+                errors.push('Missing required root element: <urlset> or <sitemapindex>');
+            }
+            // Check URL count (for regular sitemaps)
+            if (hasUrlset) {
+                const urlCount = $('url').length;
+                if (urlCount > 50000) {
+                    errors.push(`Too many URLs: ${urlCount} (max: 50,000)`);
+                }
+            }
+            // Validate URL entries
+            $('url').each((index, element) => {
+                const $element = $(element);
+                const loc = $element.find('loc').first().text().trim();
+                if (!loc) {
+                    errors.push(`URL entry ${index + 1} missing <loc> element`);
+                }
+                else {
+                    try {
+                        new URL(loc);
+                    }
+                    catch {
+                        errors.push(`Invalid URL in entry ${index + 1}: ${loc}`);
+                    }
+                }
+                // Validate lastmod format
+                const lastmod = $element.find('lastmod').first().text().trim();
+                if (lastmod) {
+                    const date = new Date(lastmod);
+                    if (isNaN(date.getTime())) {
+                        errors.push(`Invalid lastmod date in entry ${index + 1}: ${lastmod}`);
+                    }
+                }
+                // Validate priority
+                const priority = $element.find('priority').first().text().trim();
+                if (priority) {
+                    const priorityNum = parseFloat(priority);
+                    if (isNaN(priorityNum) || priorityNum < 0 || priorityNum > 1) {
+                        errors.push(`Invalid priority in entry ${index + 1}: ${priority} (must be 0-1)`);
+                    }
+                }
+            });
+        }
+        catch (error) {
+            errors.push(`XML parsing error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+        return {
+            valid: errors.length === 0,
+            errors
+        };
+    }
+}
+exports.SitemapParser = SitemapParser;
+// Default global instance
+exports.globalSitemapParser = new SitemapParser();
+//# sourceMappingURL=sitemap-parser.js.map

package/dist/lib/web-scrapers/sitemap-parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap-parser.js","sourceRoot":"","sources":["../../../lib/web-scrapers/sitemap-parser.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAmC;AACnC,sDAA6B;AAC7B,oEAA6D;AAC7D,qDAAuD;AA8BvD,MAAa,aAAa;IAA1B;QACmB,cAAS,GAAG,6EAA6E,CAAC;QAC1F,YAAO,GAAG,KAAK,CAAC,CAAC,0BAA0B;QAC3C,mBAAc,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,WAAW;QAC9C,eAAU,GAAG,KAAK,CAAC,CAAC,0BAA0B;QAC9C,oBAAe,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,iBAAiB;IAuc3E,CAAC;IArcC;;OAEG;IACH,KAAK,CAAC,YAAY,CAChB,GAAW,EACX,UAKI,EAAE;QAEN,OAAO,CAAC,GAAG,CAAC,mCAAmC,GAAG,EAAE,CAAC,CAAC;QAEtD,IAAI,CAAC;YACH,8BAA8B;YAC9B,MAAM,WAAW,GAAG,MAAM,oCAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,2CAA2C,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;gBACvF,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,uDAAuD;YACvD,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,OAAO,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;QAEH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,qCAAqC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YAClE,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,MAAc;QACnC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,IAAI,CAAC;YACH,mDAAmD;YACnD,MAAM,cAAc,GAAG,MAAM,oCAAmB,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACrE,QAAQ,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC;YAEjC,2BAA2B;YAC3B,MAAM,WAAW,GAAG;gBAClB,cAAc;gBACd,oBAAoB;gBACpB,eAAe;gBACf,WAAW;gBACX,mBAAmB;aACpB,CAAC;YAEF,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;gBAC/B,MAAM,UAAU,GAAG,WAAW,MAAM,GAAG,IAAI,EAAE,CAAC;gBAE9C,sCAAsC;gBACtC,IAAI,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;oBAClC,SAAS;gBACX,CAAC;gBAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;gBACzD,IAAI,MAAM,EAAE,CAAC;oBACX,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,4BAA4B,QAAQ,CAAC,MAAM,iBAAiB,MAAM,EAAE,CAAC,CAAC;YAClF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,oBAAoB;QAE5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,8CAA8C,MAAM,GAAG,EAAE,KAAK,CAAC,CAAC;YAC9E,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CACpB,MAAc,EACd,UAAuD,EAAE;QAEzD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;QAE9C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACrD,MAAM,UAAU,GAAmB,EAAE,CAAC;QAEtC,KAAK,MAAM,UAAU,IAAI,QAAQ,EAAE,CAAC;YAClC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE;oBAClD,YAAY,EAAE,IAAI;oBAClB,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,mBAAmB;oBACzE,WAAW,EAAE,IAAI;iBAClB,CAAC,CAAC;gBACH,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,8BAA8B,UAAU,GAAG,EAAE,KAAK,CAAC,CAAC;gBACjE,SAAS;YACX,CAAC;QACH,CAAC;QAED,qCAAqC;QACrC,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,SAAS,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QACvE,MAAM,aAAa,GAAG,UAAU;aAC7B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,IAAI,UAAU,CAAC;aAC7D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACb,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,OAAO;gBAAE,OAAO,CAAC,CAAC;YACvC,OAAO,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACnD,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAExB,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,wBAAwB,MAAM,EAAE,CAAC,CAAC;QACzF,OAAO,aAAa,CAAC;IACvB,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,IAAI,CAAC;YACH,OAAO,MAAM,yCAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;gBACrD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAErE,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;wBAChC,OAAO,EAAE;4BACP,YAAY,EAAE,IAAI,CAAC,SAAS;4BAC5B,QAAQ,EAAE,gCAAgC;yBAC3C;wBACD,MAAM,EAAE,UAAU,CAAC,MAAM;qBAC1B,CAAC,CAAC;oBAEH,YAAY,CAAC,SAAS,CAAC,CAAC;oBAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;wBACjB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;oBACrE,CAAC;oBAED,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;oBAC7D,IAAI,aAAa,IAAI,QAAQ,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;wBACnE,MAAM,IAAI,KAAK,CAAC,sBAAsB,aAAa,QAAQ,CAAC,CAAC;oBAC/D,CAAC;oBAED,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAElC,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;wBACrC,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,CAAC,MAAM,QAAQ,CAAC,CAAC;oBAC5D,CAAC;oBAED,OAAO,GAAG,CAAC;gBAEb,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,YAAY,CAAC,SAAS,CAAC,CAAC;oBACxB,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,8BAA8B,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YAC3D,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,GAAW;QAC1C,IAAI,CAAC;YACH,OAAO,MAAM,yCAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;gBACrD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;gBAE7D,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;wBAChC,MAAM,EAAE,MAAM;wBACd,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;wBACzC,MAAM,EAAE,UAAU,CAAC,MAAM;qBAC1B,CAAC,CAAC;oBAEH,YAAY,CAAC,SAAS,CAAC,CAAC;oBACxB,OAAO,QAAQ,CAAC,EAAE,CAAC;gBAErB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,YAAY,CAAC,SAAS,CAAC,CAAC;oBACxB,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,GAAW;QAChC,OAAO,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAC1E,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,GAAW,EACX,OAAY;QAEZ,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QAEnD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAmB,EAAE,CAAC;QAEtC,kCAAkC;QAClC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YAC/B,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;YAC5B,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACvD,IAAI,GAAG,EAAE,CAAC;gBACR,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,uBAAuB,QAAQ,CAAC,MAAM,oBAAoB,CAAC,CAAC;QAExE,gEAAgE;QAChE,MAAM,mBAAmB,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACjD,MAAM,MAAM,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YAE/B,sBAAsB;YACtB,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACzE,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,OAAO,IAAI,CAAC,OAAO;gBAAE,OAAO,CAAC,CAAC,CAAC;YACnC,IAAI,OAAO,IAAI,CAAC,OAAO;gBAAE,OAAO,CAAC,CAAC;YAElC,6BAA6B;YAC7B,MAAM,UAAU,GAAG,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtD,MAAM,UAAU,GAAG,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtD,IAAI,UAAU,IAAI,CAAC,UAAU;gBAAE,OAAO,CAAC,CAAC,CAAC;YACzC,IAAI,UAAU,IAAI,CAAC,UAAU;gBAAE,OAAO,CAAC,CAAC;YAExC,OAAO,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;QAEH,4EAA4E;QAC5E,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3D,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QAE9G,gDAAgD;QAChD,MAAM,KAAK,GAAG,IAAA,iBAAM,EAAC,CAAC,CAAC,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,0BAA0B,iBAAiB,CAAC,MAAM,wCAAwC,CAAC,CAAC;QAExG,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,iBAAiB,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CACjC,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YACvD,IAAI,UAAU,EAAE,CAAC;gBACf,OAAO,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE;oBAC1C,GAAG,OAAO;oBACV,UAAU,EAAE,iBAAiB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CACH,CACF,CAAC;QAEF,6BAA6B;QAC7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,qCAAqC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YACrE,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,0CAA0C,UAAU,CAAC,MAAM,iBAAiB,iBAAiB,CAAC,MAAM,WAAW,CAAC,CAAC;QAC7H,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,mBAAmB,CACzB,GAAW,EACX,OAKC;QAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QAErD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC;QACzD,MAAM,UAAU,GAAG,OAAO,CAAC,YAAY;YACrC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;YAC7C,CAAC,CAAC,IAAI,CAAC;QAET,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YAC/B,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBACjC,OAAO,KAAK,CAAC,CAAC,iBAAiB;YACjC,CAAC;YAED,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;YAC5B,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAEvD,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEjB,MAAM,KAAK,GAAiB,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;YAEzC,gBAAgB;YAChB,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACnE,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC;gBACtC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;oBAC9B,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;gBAC1B,CAAC;YACH,CAAC;YAED,iCAAiC;YACjC,IAAI,UAAU,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,GAAG,UAAU,EAAE,CAAC;gBAC9D,OAAO,CAAC,kBAAkB;YAC5B,CAAC;YAED,mBAAmB;YACnB,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACrE,IAAI,UAAU,EAAE,CAAC;gBACf,KAAK,CAAC,UAAU,GAAG,UAAiB,CAAC;YACvC,CAAC;YAED,iBAAiB;YACjB,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACrE,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,QAAQ,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC;gBAC1C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACrB,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;gBAC5B,CAAC;YACH,CAAC;YAED,4BAA4B;YAC5B,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAmB,EAAE,CAAC;gBAClC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE;oBACpD,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;oBAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;oBAC9D,IAAI,MAAM,EAAE,CAAC;wBACX,MAAM,CAAC,IAAI,CAAC;4BACV,GAAG,EAAE,MAAM;4BACX,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS;4BACxE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS;yBACrE,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;gBACxB,CAAC;YACH,CAAC;YAED,0BAA0B;YAC1B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;gBACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC3C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;oBAC/D,IAAI,KAAK,EAAE,CAAC;wBACV,KAAK,CAAC,IAAI,GAAG,EAAE,KAAK,EAAE,CAAC;wBAEvB,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;wBAChF,IAAI,WAAW,EAAE,CAAC;4BAChB,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC;4BACtC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gCAC9B,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;4BACrC,CAAC;wBACH,CAAC;wBAED,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;wBACrE,IAAI,QAAQ,EAAE,CAAC;4BACb,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;wBAC/D,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,CAAC,MAAM,uBAAuB,CAAC,CAAC;QAC3E,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,qBAAqB,CAAC,GAAW;QAC/B,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAE/C,yBAAyB;YACzB,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACzC,MAAM,eAAe,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAErD,IAAI,CAAC,SAAS,IAAI,CAAC,eAAe,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;YAC3E,CAAC;YAED,yCAAyC;YACzC,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;gBACjC,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;oBACrB,MAAM,CAAC,IAAI,CAAC,kBAAkB,QAAQ,gBAAgB,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;YAED,uBAAuB;YACvB,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC/B,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;gBAC5B,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAEvD,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,MAAM,CAAC,IAAI,CAAC,aAAa,KAAK,GAAG,CAAC,wBAAwB,CAAC,CAAC;gBAC9D,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC;wBACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;oBACf,CAAC;oBAAC,MAAM,CAAC;wBACP,MAAM,CAAC,IAAI,CAAC,wBAAwB,KAAK,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;oBAC3D,CAAC;gBACH,CAAC;gBAED,0BAA0B;gBAC1B,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAC/D,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC/B,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,CAAC,iCAAiC,KAAK,GAAG,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC;oBACxE,CAAC;gBACH,CAAC;gBAED,oBAAoB;gBACpB,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACjE,IAAI,QAAQ,EAAE,CAAC;oBACb,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;oBACzC,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;wBAC7D,MAAM,CAAC,IAAI,CAAC,6BAA6B,KAAK,GAAG,CAAC,KAAK,QAAQ,gBAAgB,CAAC,CAAC;oBACnF,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAChG,CAAC;QAED,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC1B,MAAM;SACP,CAAC;IACJ,CAAC;CACF;AA5cD,sCA4cC;AAED,0BAA0B;AACb,QAAA,mBAAmB,GAAG,IAAI,aAAa,EAAE,CAAC"}

package/package.json

@@ -1,69 +1,90 @@
 {
   "name": "@tyroneross/blog-scraper",
-  "version": "0.1.0",
-  "description": "Powerful web scraping SDK for extracting blog articles and content. No LLM required.",
+  "version": "0.3.0",
+  "description": "Intelligent web scraper SDK for extracting blog/news content. Supports RSS, sitemaps, and HTML with automatic detection.",
+  "main": "./dist/lib/index.js",
+  "types": "./dist/lib/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/lib/index.d.ts",
+      "import": "./dist/lib/index.js",
+      "require": "./dist/lib/index.js"
+    }
+  },
+  "files": [
+    "dist/lib",
+    "README.md",
+    "LICENSE"
+  ],
+  "sideEffects": false,
   "keywords": [
     "scraper",
-    "web-scraping",
-    "content-extraction",
+    "web-scraper",
     "blog",
-    "articles",
     "rss",
     "sitemap",
+    "content-extraction",
     "readability",
-    "mozilla-readability"
+    "news",
+    "article"
   ],
   "author": "Tyrone Ross",
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/tyroneross/blog-content-scraper.git",
-    "directory": "packages/sdk"
+    "url": "git+https://github.com/tyroneross/blog-scraper.git"
   },
-  "homepage": "https://github.com/tyroneross/blog-content-scraper#readme",
+  "homepage": "https://github.com/tyroneross/blog-scraper#readme",
   "bugs": {
-    "url": "https://github.com/tyroneross/blog-content-scraper/issues"
+    "url": "https://github.com/tyroneross/blog-scraper/issues"
   },
-  "main": "./dist/index.js",
-  "module": "./dist/index.mjs",
-  "types": "./dist/index.d.ts",
-  "exports": {
-    ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.mjs",
-      "require": "./dist/index.js"
-    }
-  },
-  "files": [
-    "dist",
-    "README.md",
-    "LICENSE"
-  ],
   "scripts": {
-    "build": "tsup src/index.ts --format cjs,esm --dts --clean",
-    "dev": "tsup src/index.ts --format cjs,esm --dts --watch",
+    "dev": "next dev",
+    "build": "next build",
+    "build:sdk": "tsc -p tsconfig.build.json",
+    "start": "next start",
+    "lint": "next lint",
+    "pre-deploy": "./scripts/pre-deploy.sh",
     "typecheck": "tsc --noEmit",
-    "prepublishOnly": "npm run build"
+    "test:f1": "tsx tests/run-f1-tests.ts",
+    "test:f1:dragnet": "tsx tests/dragnet-f1-test.ts",
+    "prepublishOnly": "npm run build:sdk"
   },
   "dependencies": {
-    "@mozilla/readability": "^0.5.0",
+    "@mozilla/readability": "^0.6.0",
     "cheerio": "^1.1.2",
+    "global-agent": "^3.0.0",
     "jsdom": "^24.0.0",
+    "p-limit": "^7.2.0",
+    "playwright": "^1.57.0",
+    "remark-gfm": "^4.0.1",
     "rss-parser": "^3.13.0",
     "turndown": "^7.2.1",
     "zod": "^3.22.0"
   },
   "devDependencies": {
+    "@tailwindcss/postcss": "^4.1.0",
+    "@tailwindcss/typography": "^0.5.19",
+    "@tyroneross/claude-code-debugger": "^1.3.0",
+    "@types/global-agent": "^3.0.0",
     "@types/jsdom": "^21.1.7",
     "@types/node": "^20.0.0",
+    "@types/react": "^19.0.0",
+    "@types/react-dom": "^19.0.0",
     "@types/turndown": "^5.0.5",
-    "tsup": "^8.0.0",
+    "eslint": "^8.0.0",
+    "eslint-config-next": "^15.0.0",
+    "lucide-react": "^0.400.0",
+    "next": "^16.1.1",
+    "postcss": "^8.4.0",
+    "react": "^19.0.0",
+    "react-dom": "^19.0.0",
+    "react-markdown": "^10.1.0",
+    "tailwindcss": "^4.1.0",
+    "tsx": "^4.7.0",
     "typescript": "^5.0.0"
   },
   "engines": {
     "node": ">=18.0.0"
-  },
-  "publishConfig": {
-    "access": "public"
   }
 }