@tyroneross/blog-scraper 0.1.0 → 0.3.0

package/dist/lib/web-scrapers/playwright-scraper.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Playwright-based scraper for JavaScript-rendered pages
+ *
+ * Used as a fallback when static HTML scraping fails (e.g., Next.js, React, Vue sites)
+ * Returns the same ExtractedArticle format as HTMLScraper for consistency
+ */
+import { ExtractedArticle, ScrapingConfig } from './html-scraper';
+export interface PlaywrightScraperConfig extends ScrapingConfig {
+    /** Wait for specific selector before extracting (optional) */
+    waitForSelector?: string;
+    /** Maximum time to wait for page load in ms (default: 30000) */
+    timeout?: number;
+    /** Whether to block images/fonts for faster loading (default: true) */
+    blockMedia?: boolean;
+    /** Custom viewport size */
+    viewport?: {
+        width: number;
+        height: number;
+    };
+}
+export declare class PlaywrightScraper {
+    private browser;
+    private readonly userAgent;
+    private readonly defaultConfig;
+    /**
+     * Initialize browser instance
+     */
+    private getBrowser;
+    /**
+     * Close browser instance
+     */
+    close(): Promise<void>;
+    /**
+     * Extract article links from a JavaScript-rendered page
+     */
+    extractArticleLinks(url: string, config?: PlaywrightScraperConfig): Promise<ExtractedArticle[]>;
+    /**
+     * Fetch fully rendered HTML content from a page
+     * Useful for content extraction on JS-rendered article pages
+     */
+    fetchRenderedContent(url: string, config?: PlaywrightScraperConfig): Promise<string | null>;
+    /**
+     * Check if URL passes filters
+     */
+    private passesFilters;
+    /**
+     * Check if URL looks like an article
+     */
+    private isLikelyArticleUrl;
+    /**
+     * Merge configurations
+     */
+    private mergeConfig;
+}
+export declare function getPlaywrightScraper(): PlaywrightScraper;
+export declare function closePlaywrightScraper(): Promise<void>;
+//# sourceMappingURL=playwright-scraper.d.ts.map

package/dist/lib/web-scrapers/playwright-scraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"playwright-scraper.d.ts","sourceRoot":"","sources":["../../../lib/web-scrapers/playwright-scraper.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAElE,MAAM,WAAW,uBAAwB,SAAQ,cAAc;IAC7D,8DAA8D;IAC9D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gEAAgE;IAChE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uEAAuE;IACvE,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,2BAA2B;IAC3B,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9C;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA2H;IAErJ,OAAO,CAAC,QAAQ,CAAC,aAAa,CAsD5B;IAEF;;OAEG;YACW,UAAU;IAiBxB;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B;;OAEG;IACG,mBAAmB,CACvB,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,uBAA4B,GACnC,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAwJ9B;;;OAGG;IACG,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,uBAA4B,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IA2CrG;;OAEG;IACH,OAAO,CAAC,aAAa;IAiBrB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAe1B;;OAEG;IACH,OAAO,CAAC,WAAW;CA6BpB;AAKD,wBAAgB,oBAAoB,IAAI,iBAAiB,CAKxD;AAGD,wBAAsB,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC,CAK5D"}

package/dist/lib/web-scrapers/playwright-scraper.js

@@ -0,0 +1,355 @@
+"use strict";
+/**
+ * Playwright-based scraper for JavaScript-rendered pages
+ *
+ * Used as a fallback when static HTML scraping fails (e.g., Next.js, React, Vue sites)
+ * Returns the same ExtractedArticle format as HTMLScraper for consistency
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PlaywrightScraper = void 0;
+exports.getPlaywrightScraper = getPlaywrightScraper;
+exports.closePlaywrightScraper = closePlaywrightScraper;
+const playwright_1 = require("playwright");
+class PlaywrightScraper {
+    constructor() {
+        this.browser = null;
+        this.userAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+        this.defaultConfig = {
+            timeout: 30000,
+            blockMedia: true,
+            viewport: { width: 1280, height: 720 },
+            selectors: {
+                articleLinks: [
+                    'article a[href]',
+                    '.article a[href]',
+                    '.post a[href]',
+                    '.story a[href]',
+                    '.news-item a[href]',
+                    '.card a[href]',
+                    '[class*="article"] a[href]',
+                    '[class*="post"] a[href]',
+                    '[class*="news"] a[href]',
+                    '[class*="story"] a[href]',
+                    'h1 a[href]',
+                    'h2 a[href]',
+                    'h3 a[href]',
+                    '.headline a[href]',
+                    '.title a[href]',
+                    // Common list patterns
+                    'ul li a[href]',
+                    '.list-item a[href]',
+                    '[role="listitem"] a[href]',
+                ],
+                excludeSelectors: [
+                    'nav',
+                    'header',
+                    'footer',
+                    '.navigation',
+                    '.menu',
+                    '.sidebar',
+                    '.advertisement',
+                    '.ads',
+                    '.comments',
+                    '.social-share',
+                    '[aria-hidden="true"]',
+                ]
+            },
+            filters: {
+                minTitleLength: 10,
+                maxTitleLength: 300,
+                excludePatterns: [
+                    /\/(tag|category|author|search|archive|login|register|contact|about|privacy|terms)\//i,
+                    /\.(pdf|jpg|jpeg|png|gif|mp4|zip|doc)$/i,
+                    /#/,
+                    /javascript:/i,
+                    /mailto:/i
+                ]
+            },
+            limits: {
+                maxLinksPerPage: 100
+            }
+        };
+    }
+    /**
+     * Initialize browser instance
+     */
+    async getBrowser() {
+        if (!this.browser) {
+            console.log('🎭 [Playwright] Launching browser...');
+            this.browser = await playwright_1.chromium.launch({
+                headless: true,
+                args: [
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-dev-shm-usage',
+                    '--disable-accelerated-2d-canvas',
+                    '--disable-gpu'
+                ]
+            });
+        }
+        return this.browser;
+    }
+    /**
+     * Close browser instance
+     */
+    async close() {
+        if (this.browser) {
+            await this.browser.close();
+            this.browser = null;
+            console.log('🎭 [Playwright] Browser closed');
+        }
+    }
+    /**
+     * Extract article links from a JavaScript-rendered page
+     */
+    async extractArticleLinks(url, config = {}) {
+        console.log(`🎭 [Playwright] Extracting articles from ${url}`);
+        const mergedConfig = this.mergeConfig(this.defaultConfig, config);
+        const browser = await this.getBrowser();
+        const context = await browser.newContext({
+            userAgent: this.userAgent,
+            viewport: mergedConfig.viewport,
+        });
+        const page = await context.newPage();
+        const articles = [];
+        try {
+            // Block unnecessary resources for faster loading
+            if (mergedConfig.blockMedia) {
+                await page.route('**/*', (route) => {
+                    const resourceType = route.request().resourceType();
+                    if (['image', 'font', 'media'].includes(resourceType)) {
+                        route.abort();
+                    }
+                    else {
+                        route.continue();
+                    }
+                });
+            }
+            // Navigate to page
+            console.log(`🎭 [Playwright] Loading ${url}...`);
+            await page.goto(url, {
+                waitUntil: 'networkidle',
+                timeout: mergedConfig.timeout
+            });
+            // Wait for custom selector if specified
+            if (mergedConfig.waitForSelector) {
+                await page.waitForSelector(mergedConfig.waitForSelector, {
+                    timeout: mergedConfig.timeout
+                });
+            }
+            // Give JS a moment to finish rendering
+            await page.waitForTimeout(1000);
+            // Remove excluded elements
+            for (const selector of mergedConfig.selectors?.excludeSelectors || []) {
+                await page.evaluate((sel) => {
+                    document.querySelectorAll(sel).forEach(el => el.remove());
+                }, selector);
+            }
+            // Extract articles using configured selectors
+            const extractedData = await page.evaluate((selectors) => {
+                const results = [];
+                const seenUrls = new Set();
+                for (const selector of selectors) {
+                    const links = document.querySelectorAll(selector);
+                    links.forEach((link) => {
+                        const anchor = link;
+                        const href = anchor.href;
+                        if (!href || seenUrls.has(href))
+                            return;
+                        seenUrls.add(href);
+                        // Get title from link text or nearby heading
+                        let title = anchor.textContent?.trim() || '';
+                        // Try to find better title from parent article/card
+                        const parent = anchor.closest('article, [class*="card"], [class*="post"], [class*="item"], li');
+                        if (parent) {
+                            const heading = parent.querySelector('h1, h2, h3, h4, .title, .headline');
+                            if (heading) {
+                                const headingText = heading.textContent?.trim();
+                                if (headingText && headingText.length > title.length) {
+                                    title = headingText;
+                                }
+                            }
+                            // Get description
+                            const desc = parent.querySelector('p, .excerpt, .summary, .description');
+                            const description = desc?.textContent?.trim();
+                            // Get date
+                            const dateEl = parent.querySelector('time, [datetime], .date, .published');
+                            const date = dateEl?.getAttribute('datetime') || dateEl?.textContent?.trim();
+                            if (title && title.length >= 10) {
+                                results.push({
+                                    url: href,
+                                    title,
+                                    date,
+                                    description: description?.substring(0, 300)
+                                });
+                            }
+                        }
+                        else if (title && title.length >= 10) {
+                            results.push({ url: href, title });
+                        }
+                    });
+                }
+                return results;
+            }, mergedConfig.selectors?.articleLinks || []);
+            // Process and filter results
+            for (const item of extractedData) {
+                if (articles.length >= (mergedConfig.limits?.maxLinksPerPage || 100))
+                    break;
+                // Apply URL filters
+                if (!this.passesFilters(item.url, mergedConfig.filters))
+                    continue;
+                // Parse date if available
+                let publishedDate;
+                if (item.date) {
+                    const parsed = new Date(item.date);
+                    if (!isNaN(parsed.getTime())) {
+                        publishedDate = parsed;
+                    }
+                }
+                // Calculate confidence based on data quality
+                let confidence = 0.6; // Base confidence for Playwright extraction
+                if (publishedDate)
+                    confidence += 0.1;
+                if (item.description)
+                    confidence += 0.1;
+                if (this.isLikelyArticleUrl(item.url))
+                    confidence += 0.1;
+                articles.push({
+                    url: item.url,
+                    title: item.title,
+                    publishedDate,
+                    description: item.description,
+                    confidence: Math.min(confidence, 1.0),
+                    source: 'link-text'
+                });
+            }
+            console.log(`🎭 [Playwright] Extracted ${articles.length} articles from ${url}`);
+        }
+        catch (error) {
+            console.error(`❌ [Playwright] Error extracting from ${url}:`, error);
+        }
+        finally {
+            await context.close();
+        }
+        return articles;
+    }
+    /**
+     * Fetch fully rendered HTML content from a page
+     * Useful for content extraction on JS-rendered article pages
+     */
+    async fetchRenderedContent(url, config = {}) {
+        console.log(`🎭 [Playwright] Fetching rendered content from ${url}`);
+        const mergedConfig = this.mergeConfig(this.defaultConfig, config);
+        const browser = await this.getBrowser();
+        const context = await browser.newContext({
+            userAgent: this.userAgent,
+            viewport: mergedConfig.viewport,
+        });
+        const page = await context.newPage();
+        try {
+            // Don't block images for content extraction - we might need them
+            await page.goto(url, {
+                waitUntil: 'networkidle',
+                timeout: mergedConfig.timeout
+            });
+            // Wait for custom selector if specified
+            if (mergedConfig.waitForSelector) {
+                await page.waitForSelector(mergedConfig.waitForSelector, {
+                    timeout: mergedConfig.timeout
+                });
+            }
+            // Give JS a moment to finish rendering
+            await page.waitForTimeout(1000);
+            // Get the full HTML
+            const html = await page.content();
+            console.log(`🎭 [Playwright] Fetched ${html.length} bytes of rendered HTML`);
+            return html;
+        }
+        catch (error) {
+            console.error(`❌ [Playwright] Error fetching content from ${url}:`, error);
+            return null;
+        }
+        finally {
+            await context.close();
+        }
+    }
+    /**
+     * Check if URL passes filters
+     */
+    passesFilters(url, filters) {
+        if (!filters)
+            return true;
+        // Check exclude patterns
+        if (filters.excludePatterns?.some(pattern => pattern.test(url))) {
+            return false;
+        }
+        // Check include patterns if specified
+        if (filters.includePatterns?.length &&
+            !filters.includePatterns.some(pattern => pattern.test(url))) {
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Check if URL looks like an article
+     */
+    isLikelyArticleUrl(url) {
+        const articlePatterns = [
+            /\/article[s]?\//i,
+            /\/post[s]?\//i,
+            /\/story\//i,
+            /\/stories\//i,
+            /\/news\//i,
+            /\/blog\//i,
+            /\/\d{4}\/\d{2}\/\d{2}\//,
+            /\/\d{4}\/\d{2}\//
+        ];
+        return articlePatterns.some(pattern => pattern.test(url));
+    }
+    /**
+     * Merge configurations
+     */
+    mergeConfig(defaultConfig, userConfig) {
+        return {
+            ...defaultConfig,
+            ...userConfig,
+            selectors: {
+                ...defaultConfig.selectors,
+                ...userConfig.selectors,
+                articleLinks: [
+                    ...(defaultConfig.selectors?.articleLinks || []),
+                    ...(userConfig.selectors?.articleLinks || [])
+                ],
+                excludeSelectors: [
+                    ...(defaultConfig.selectors?.excludeSelectors || []),
+                    ...(userConfig.selectors?.excludeSelectors || [])
+                ]
+            },
+            filters: {
+                ...defaultConfig.filters,
+                ...userConfig.filters
+            },
+            limits: {
+                ...defaultConfig.limits,
+                ...userConfig.limits
+            }
+        };
+    }
+}
+exports.PlaywrightScraper = PlaywrightScraper;
+// Global instance with lazy initialization
+let globalPlaywrightScraper = null;
+function getPlaywrightScraper() {
+    if (!globalPlaywrightScraper) {
+        globalPlaywrightScraper = new PlaywrightScraper();
+    }
+    return globalPlaywrightScraper;
+}
+// Cleanup function for graceful shutdown
+async function closePlaywrightScraper() {
+    if (globalPlaywrightScraper) {
+        await globalPlaywrightScraper.close();
+        globalPlaywrightScraper = null;
+    }
+}
+//# sourceMappingURL=playwright-scraper.js.map

package/dist/lib/web-scrapers/playwright-scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"playwright-scraper.js","sourceRoot":"","sources":["../../../lib/web-scrapers/playwright-scraper.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAmYH,oDAKC;AAGD,wDAKC;AA9YD,2CAAqD;AAcrD,MAAa,iBAAiB;IAA9B;QACU,YAAO,GAAmB,IAAI,CAAC;QACtB,cAAS,GAAG,uHAAuH,CAAC;QAEpI,kBAAa,GAA4B;YACxD,OAAO,EAAE,KAAK;YACd,UAAU,EAAE,IAAI;YAChB,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;YACtC,SAAS,EAAE;gBACT,YAAY,EAAE;oBACZ,iBAAiB;oBACjB,kBAAkB;oBAClB,eAAe;oBACf,gBAAgB;oBAChB,oBAAoB;oBACpB,eAAe;oBACf,4BAA4B;oBAC5B,yBAAyB;oBACzB,yBAAyB;oBACzB,0BAA0B;oBAC1B,YAAY;oBACZ,YAAY;oBACZ,YAAY;oBACZ,mBAAmB;oBACnB,gBAAgB;oBAChB,uBAAuB;oBACvB,eAAe;oBACf,oBAAoB;oBACpB,2BAA2B;iBAC5B;gBACD,gBAAgB,EAAE;oBAChB,KAAK;oBACL,QAAQ;oBACR,QAAQ;oBACR,aAAa;oBACb,OAAO;oBACP,UAAU;oBACV,gBAAgB;oBAChB,MAAM;oBACN,WAAW;oBACX,eAAe;oBACf,sBAAsB;iBACvB;aACF;YACD,OAAO,EAAE;gBACP,cAAc,EAAE,EAAE;gBAClB,cAAc,EAAE,GAAG;gBACnB,eAAe,EAAE;oBACf,sFAAsF;oBACtF,wCAAwC;oBACxC,GAAG;oBACH,cAAc;oBACd,UAAU;iBACX;aACF;YACD,MAAM,EAAE;gBACN,eAAe,EAAE,GAAG;aACrB;SACF,CAAC;IAoTJ,CAAC;IAlTC;;OAEG;IACK,KAAK,CAAC,UAAU;QACtB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;YACpD,IAAI,CAAC,OAAO,GAAG,MAAM,qBAAQ,CAAC,MAAM,CAAC;gBACnC,QAAQ,EAAE,IAAI;gBACd,IAAI,EAAE;oBACJ,cAAc;oBACd,0BAA0B;oBAC1B,yBAAyB;oBACzB,iCAAiC;oBACjC,eAAe;iBAChB;aACF,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CACvB,GAAW,EACX,SAAkC,EAAE;QAEpC,OAAO,CAAC,GAAG,CAAC,4CAA4C,GAAG,EAAE,CAAC,CAAC;QAE/D,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,YAAY,CAAC,QAAQ;SAChC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAuB,EAAE,CAAC;QAExC,IAAI,CAAC;YACH,iDAAiD;YACjD,IAAI,YAAY,CAAC,UAAU,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;oBACjC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;oBACpD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;wBACtD,KAAK,CAAC,KAAK,EAAE,CAAC;oBAChB,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,QAAQ,EAAE,CAAC;oBACnB,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;YAED,mBAAmB;YACnB,OAAO,CAAC,GAAG,CAAC,2BAA2B,GAAG,KAAK,CAAC,CAAC;YACjD,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;gBACnB,SAAS,EAAE,aAAa;gBACxB,OAAO,EAAE,YAAY,CAAC,OAAO;aAC9B,CAAC,CAAC;YAEH,wCAAwC;YACxC,IAAI,YAAY,CAAC,eAAe,EAAE,CAAC;gBACjC,MAAM,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC,eAAe,EAAE;oBACvD,OAAO,EAAE,YAAY,CAAC,OAAO;iBAC9B,CAAC,CAAC;YACL,CAAC;YAED,uCAAuC;YACvC,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAEhC,2BAA2B;YAC3B,KAAK,MAAM,QAAQ,IAAI,YAAY,CAAC,SAAS,EAAE,gBAAgB,IAAI,EAAE,EAAE,CAAC;gBACtE,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,EAAE;oBAC1B,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC5D,CAAC,EAAE,QAAQ,CAAC,CAAC;YACf,CAAC;YAED,8CAA8C;YAC9C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,SAAS,EAAE,EAAE;gBACtD,MAAM,OAAO,GAKR,EAAE,CAAC;gBACR,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;gBAEnC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACjC,MAAM,KAAK,GAAG,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;oBAElD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;wBACrB,MAAM,MAAM,GAAG,IAAyB,CAAC;wBACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;wBAEzB,IAAI,CAAC,IAAI,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;4BAAE,OAAO;wBACxC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;wBAEnB,6CAA6C;wBAC7C,IAAI,KAAK,GAAG,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;wBAE7C,oDAAoD;wBACpD,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,gEAAgE,CAAC,CAAC;wBAChG,IAAI,MAAM,EAAE,CAAC;4BACX,MAAM,OAAO,GAAG,MAAM,CAAC,aAAa,CAAC,mCAAmC,CAAC,CAAC;4BAC1E,IAAI,OAAO,EAAE,CAAC;gCACZ,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;gCAChD,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oCACrD,KAAK,GAAG,WAAW,CAAC;gCACtB,CAAC;4BACH,CAAC;4BAED,kBAAkB;4BAClB,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,CAAC,qCAAqC,CAAC,CAAC;4BACzE,MAAM,WAAW,GAAG,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;4BAE9C,WAAW;4BACX,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,CAAC,qCAAqC,CAAC,CAAC;4BAC3E,MAAM,IAAI,GAAG,MAAM,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;4BAE7E,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gCAChC,OAAO,CAAC,IAAI,CAAC;oCACX,GAAG,EAAE,IAAI;oCACT,KAAK;oCACL,IAAI;oCACJ,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC;iCAC5C,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;6BAAM,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;4BACvC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;wBACrC,CAAC;oBACH,CAAC,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO,OAAO,CAAC;YACjB,CAAC,EAAE,YAAY,CAAC,SAAS,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC;YAE/C,6BAA6B;YAC7B,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;gBACjC,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,eAAe,IAAI,GAAG,CAAC;oBAAE,MAAM;gBAE5E,oBAAoB;gBACpB,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,OAAO,CAAC;oBAAE,SAAS;gBAElE,0BAA0B;gBAC1B,IAAI,aAA+B,CAAC;gBACpC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBACd,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACnC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;wBAC7B,aAAa,GAAG,MAAM,CAAC;oBACzB,CAAC;gBACH,CAAC;gBAED,6CAA6C;gBAC7C,IAAI,UAAU,GAAG,GAAG,CAAC,CAAC,4CAA4C;gBAClE,IAAI,aAAa;oBAAE,UAAU,IAAI,GAAG,CAAC;gBACrC,IAAI,IAAI,CAAC,WAAW;oBAAE,UAAU,IAAI,GAAG,CAAC;gBACxC,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,UAAU,IAAI,GAAG,CAAC;gBAEzD,QAAQ,CAAC,IAAI,CAAC;oBACZ,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,aAAa;oBACb,WAAW,EAAE,IAAI,CAAC,WAAW;oBAC7B,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC;oBACrC,MAAM,EAAE,WAAW;iBACpB,CAAC,CAAC;YACL,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,6BAA6B,QAAQ,CAAC,MAAM,kBAAkB,GAAG,EAAE,CAAC,CAAC;QAEnF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,wCAAwC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QACvE,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,oBAAoB,CAAC,GAAW,EAAE,SAAkC,EAAE;QAC1E,OAAO,CAAC,GAAG,CAAC,kDAAkD,GAAG,EAAE,CAAC,CAAC;QAErE,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,YAAY,CAAC,QAAQ;SAChC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAErC,IAAI,CAAC;YACH,iEAAiE;YACjE,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;gBACnB,SAAS,EAAE,aAAa;gBACxB,OAAO,EAAE,YAAY,CAAC,OAAO;aAC9B,CAAC,CAAC;YAEH,wCAAwC;YACxC,IAAI,YAAY,CAAC,eAAe,EAAE,CAAC;gBACjC,MAAM,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC,eAAe,EAAE;oBACvD,OAAO,EAAE,YAAY,CAAC,OAAO;iBAC9B,CAAC,CAAC;YACL,CAAC;YAED,uCAAuC;YACvC,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAEhC,oBAAoB;YACpB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,CAAC,MAAM,yBAAyB,CAAC,CAAC;YAE7E,OAAO,IAAI,CAAC;QAEd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,8CAA8C,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YAC3E,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,GAAW,EAAE,OAAmC;QACpE,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1B,yBAAyB;QACzB,IAAI,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAChE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,sCAAsC;QACtC,IAAI,OAAO,CAAC,eAAe,EAAE,MAAM;YAC/B,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAChE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,GAAW;QACpC,MAAM,eAAe,GAAG;YACtB,kBAAkB;YAClB,eAAe;YACf,YAAY;YACZ,cAAc;YACd,WAAW;YACX,WAAW;YACX,yBAAyB;YACzB,kBAAkB;SACnB,CAAC;QAEF,OAAO,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACK,WAAW,CACjB,aAAsC,EACtC,UAAmC;QAEnC,OAAO;YACL,GAAG,aAAa;YAChB,GAAG,UAAU;YACb,SAAS,EAAE;gBACT,GAAG,aAAa,CAAC,SAAS;gBAC1B,GAAG,UAAU,CAAC,SAAS;gBACvB,YAAY,EAAE;oBACZ,GAAG,CAAC,aAAa,CAAC,SAAS,EAAE,YAAY,IAAI,EAAE,CAAC;oBAChD,GAAG,CAAC,UAAU,CAAC,SAAS,EAAE,YAAY,IAAI,EAAE,CAAC;iBAC9C;gBACD,gBAAgB,EAAE;oBAChB,GAAG,CAAC,aAAa,CAAC,SAAS,EAAE,gBAAgB,IAAI,EAAE,CAAC;oBACpD,GAAG,CAAC,UAAU,CAAC,SAAS,EAAE,gBAAgB,IAAI,EAAE,CAAC;iBAClD;aACF;YACD,OAAO,EAAE;gBACP,GAAG,aAAa,CAAC,OAAO;gBACxB,GAAG,UAAU,CAAC,OAAO;aACtB;YACD,MAAM,EAAE;gBACN,GAAG,aAAa,CAAC,MAAM;gBACvB,GAAG,UAAU,CAAC,MAAM;aACrB;SACF,CAAC;IACJ,CAAC;CACF;AA9WD,8CA8WC;AAED,2CAA2C;AAC3C,IAAI,uBAAuB,GAA6B,IAAI,CAAC;AAE7D,SAAgB,oBAAoB;IAClC,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAC7B,uBAAuB,GAAG,IAAI,iBAAiB,EAAE,CAAC;IACpD,CAAC;IACD,OAAO,uBAAuB,CAAC;AACjC,CAAC;AAED,yCAAyC;AAClC,KAAK,UAAU,sBAAsB;IAC1C,IAAI,uBAAuB,EAAE,CAAC;QAC5B,MAAM,uBAAuB,CAAC,KAAK,EAAE,CAAC;QACtC,uBAAuB,GAAG,IAAI,CAAC;IACjC,CAAC;AACH,CAAC"}

package/dist/lib/web-scrapers/robots-checker.d.ts

@@ -0,0 +1,42 @@
+export declare class RobotsChecker {
+    private cache;
+    private readonly cacheTimeout;
+    private readonly userAgent;
+    private readonly requestTimeout;
+    /**
+     * Check if a URL is allowed to be crawled according to robots.txt
+     */
+    isAllowed(url: string): Promise<{
+        allowed: boolean;
+        crawlDelay?: number;
+        sitemaps: string[];
+        reason?: string;
+    }>;
+    /**
+     * Get sitemaps listed in robots.txt for a domain
+     */
+    getSitemaps(domain: string): Promise<string[]>;
+    /**
+     * Get the recommended crawl delay for a domain
+     */
+    getCrawlDelay(domain: string): Promise<number | undefined>;
+    private getRobotsTxt;
+    private parseRobotsTxt;
+    private completeRule;
+    private checkRules;
+    private findBestMatchingRule;
+    private matchesPattern;
+    clearCache(): void;
+    getCacheStats(): {
+        size: number;
+        entries: {
+            url: string;
+            fetchedAt: string;
+            expiresAt: string;
+            rulesCount: number;
+            sitemapsCount: number;
+        }[];
+    };
+}
+export declare const globalRobotsChecker: RobotsChecker;
+//# sourceMappingURL=robots-checker.d.ts.map

package/dist/lib/web-scrapers/robots-checker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots-checker.d.ts","sourceRoot":"","sources":["../../../lib/web-scrapers/robots-checker.ts"],"names":[],"mappings":"AAeA,qBAAa,aAAa;IACxB,OAAO,CAAC,KAAK,CAAgC;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuB;IACpD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;IAEvC;;OAEG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QACpC,OAAO,EAAE,OAAO,CAAC;QACjB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IAkCF;;OAEG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAWpD;;OAEG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;YAiBlD,YAAY;IAiD1B,OAAO,CAAC,cAAc;IA+EtB,OAAO,CAAC,YAAY;IAUpB,OAAO,CAAC,UAAU;IAkDlB,OAAO,CAAC,oBAAoB;IAW5B,OAAO,CAAC,cAAc;IA2BtB,UAAU;IAKV,aAAa;;;;;;;;;;CAYd;AAGD,eAAO,MAAM,mBAAmB,eAAsB,CAAC"}