@tyroneross/blog-scraper 0.1.0 → 0.3.0

package/dist/lib/quality-scorer.js

@@ -0,0 +1,376 @@
+"use strict";
+/**
+ * @package @tyroneross/scraper-testing
+ * Article quality scoring system
+ *
+ * No LLM required - uses metadata and content signals to determine article quality
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_ALLOW_PATHS = exports.DEFAULT_DENY_PATHS = exports.DEFAULT_QUALITY_CONFIG = void 0;
+exports.validateContent = validateContent;
+exports.calculateArticleQualityScore = calculateArticleQualityScore;
+exports.isNonEnglishLocalePath = isNonEnglishLocalePath;
+exports.shouldDenyUrl = shouldDenyUrl;
+exports.getQualityBreakdown = getQualityBreakdown;
+/**
+ * Default quality score configuration
+ * These weights were optimized through testing with 1,788 real articles
+ */
+exports.DEFAULT_QUALITY_CONFIG = {
+    contentWeight: 0.60, // Content validation (length, quality, ratio)
+    dateWeight: 0.12, // Publication date presence
+    authorWeight: 0.08, // Author/byline presence
+    schemaWeight: 0.08, // Schema.org metadata
+    readingTimeWeight: 0.12, // Substantial reading time (2+ min)
+    threshold: 0.50, // Minimum score to pass (50%)
+};
+/**
+ * Default patterns to block non-article pages
+ * These cover common non-article paths across websites
+ */
+exports.DEFAULT_DENY_PATHS = [
+    '/',
+    '/index',
+    '/index.html',
+    '/about',
+    '/about/*',
+    '/careers',
+    '/careers/*',
+    '/jobs',
+    '/jobs/*',
+    '/contact',
+    '/contact/*',
+    '/team',
+    '/team/*',
+    '/privacy',
+    '/terms',
+    '/legal/*',
+    '/tag/*',
+    '/tags/*',
+    '/category/*',
+    '/categories/*',
+    '/author/*',
+    '/authors/*',
+    '/archive/*',
+    '/search',
+    '/search/*',
+    // Non-article content pages
+    '/use-cases',
+    '/use-cases/*',
+    '/solutions',
+    '/solutions/*',
+    '/products',
+    '/products/*',
+    '/services',
+    '/services/*',
+    '/partners',
+    '/partners/*',
+    '/support',
+    '/support/*',
+    '/help',
+    '/help/*',
+    '/faq',
+    '/faq/*',
+    '/pricing',
+    '/pricing/*',
+    '/features',
+    '/features/*',
+    '/demo',
+    '/demo/*',
+    '/login',
+    '/signup',
+    '/register',
+    '/account',
+    '/account/*',
+    '/dashboard',
+    '/dashboard/*',
+    '/settings',
+    '/settings/*',
+    '/trust-center',
+    '/trust-center/*',
+    '/ai-trust-center',
+    '/ai-trust-center/*',
+    '/safety',
+    '/safety/*',
+    '/compliance',
+    '/compliance/*',
+    '/certification',
+    '/certification/*',
+    '/industries',
+    '/industries/*',
+    '/platform',
+    '/platform/*',
+    '/developers',
+    '/developers/*',
+    '/documentation',
+    '/documentation/*',
+    '/docs',
+    '/docs/*',
+    '/api',
+    '/api/*',
+    '/download',
+    '/download/*',
+    '/downloads',
+    '/downloads/*',
+    // Non-English language paths (filter to English only)
+    '/cs-cz/*', // Czech
+    '/de-de/*', // German
+    '/de-at/*', // German (Austria)
+    '/de-ch/*', // German (Swiss)
+    '/fr-fr/*', // French
+    '/fr-ca/*', // French (Canada)
+    '/es-es/*', // Spanish
+    '/es-mx/*', // Spanish (Mexico)
+    '/es-la/*', // Spanish (Latin America)
+    '/it-it/*', // Italian
+    '/ja-jp/*', // Japanese
+    '/ko-kr/*', // Korean
+    '/zh-cn/*', // Chinese (Simplified)
+    '/zh-tw/*', // Chinese (Traditional)
+    '/zh-hk/*', // Chinese (Hong Kong)
+    '/pt-br/*', // Portuguese (Brazil)
+    '/pt-pt/*', // Portuguese
+    '/ru-ru/*', // Russian
+    '/pl-pl/*', // Polish
+    '/nl-nl/*', // Dutch
+    '/sv-se/*', // Swedish
+    '/nb-no/*', // Norwegian
+    '/da-dk/*', // Danish
+    '/fi-fi/*', // Finnish
+    '/tr-tr/*', // Turkish
+    '/ar-ae/*', // Arabic
+    '/he-il/*', // Hebrew
+    '/th-th/*', // Thai
+    '/vi-vn/*', // Vietnamese
+    '/id-id/*', // Indonesian
+    // Short language codes
+    '/de/*',
+    '/fr/*',
+    '/es/*',
+    '/it/*',
+    '/ja/*',
+    '/ko/*',
+    '/zh/*',
+    '/pt/*',
+    '/ru/*',
+    '/pl/*',
+    '/nl/*',
+];
+/**
+ * Default patterns for content sections (blog, news, articles)
+ * Used for allow-listing paths when scraping
+ */
+exports.DEFAULT_ALLOW_PATHS = [
+    '/news/*',
+    '/blog/*',
+    '/articles/*',
+    '/posts/*',
+    '/stories/*',
+    '/press/*',
+    '/updates/*',
+    '/announcements/*',
+    '/insights/*',
+    '/resources/*',
+    '/publications/*',
+    '/research/*',
+    '/engineering/*',
+];
+/**
+ * Validate content quality (Tier 2 filtering)
+ * Checks length, title quality, and text-to-HTML ratio
+ *
+ * @param extracted - Extracted content from article
+ * @returns Validation result with score and reasons
+ */
+function validateContent(extracted) {
+    const reasons = [];
+    let score = 1.0; // Start with perfect score, deduct for issues
+    // Check content length (minimum 200 characters)
+    const contentLength = extracted.textContent?.length || 0;
+    if (contentLength < 200) {
+        reasons.push('Content too short (< 200 characters)');
+        score -= 0.5; // Heavy penalty for short content
+    }
+    // Check title quality (10-200 characters)
+    const titleLength = extracted.title?.length || 0;
+    if (titleLength < 10 || titleLength > 200) {
+        reasons.push('Title length invalid (must be 10-200 characters)');
+        score -= 0.2;
+    }
+    // Check text-to-HTML ratio (should be at least 10% text)
+    if (extracted.content && extracted.textContent) {
+        const htmlLength = extracted.content.length;
+        const textLength = extracted.textContent.length;
+        const ratio = textLength / htmlLength;
+        if (ratio < 0.1) {
+            reasons.push('Low text-to-HTML ratio (< 10%)');
+            score -= 0.2;
+        }
+    }
+    // Content must score at least 0.5 to be considered valid
+    const isValid = score >= 0.5;
+    return {
+        isValid,
+        score: Math.max(0, Math.min(1.0, score)), // Clamp between 0-1
+        reasons,
+    };
+}
+/**
+ * Calculate article quality score (Tier 3 filtering)
+ *
+ * Score breakdown:
+ * - Content validation (60%): Length, title quality, text-to-HTML ratio
+ * - Publication date (12%): Articles should have timestamps
+ * - Author/byline (8%): Professional articles cite authors
+ * - Schema.org metadata (8%): Structured data indicates article pages
+ * - Reading time (12%): Substantial content (2+ min read)
+ *
+ * @param extracted - Extracted content from article
+ * @param config - Optional quality score configuration
+ * @returns Quality score between 0-1
+ */
+function calculateArticleQualityScore(extracted, config = {}) {
+    const finalConfig = { ...exports.DEFAULT_QUALITY_CONFIG, ...config };
+    let score = 0;
+    // Tier 2: Content validation (60% weight by default)
+    const validation = validateContent(extracted);
+    score += validation.score * finalConfig.contentWeight;
+    // Tier 3: Article metadata signals
+    // Has publication date (12% weight by default)
+    if (extracted.publishedTime) {
+        score += finalConfig.dateWeight;
+    }
+    // Has author/byline (8% weight by default)
+    if (extracted.byline) {
+        score += finalConfig.authorWeight;
+    }
+    // Has article schema.org metadata (8% weight by default)
+    if (extracted.structured?.jsonLd) {
+        const schemas = Array.isArray(extracted.structured.jsonLd)
+            ? extracted.structured.jsonLd
+            : [extracted.structured.jsonLd];
+        const hasArticleType = schemas.some((s) => {
+            const type = s['@type'];
+            return (type === 'Article' ||
+                type === 'NewsArticle' ||
+                type === 'BlogPosting' ||
+                type === 'TechArticle' ||
+                type === 'ScholarlyArticle');
+        });
+        if (hasArticleType) {
+            score += finalConfig.schemaWeight;
+        }
+    }
+    // Substantial reading time (12% weight by default)
+    // Articles should be at least 2 minutes to read
+    if (extracted.readingTime && extracted.readingTime >= 2) {
+        score += finalConfig.readingTimeWeight;
+    }
+    return Math.min(score, 1.0); // Cap at 1.0
+}
+/**
+ * Regex pattern for locale paths (xx-yy format like /en-us/, /fr-be/)
+ */
+const LOCALE_PATH_REGEX = /^\/[a-z]{2}[-_][a-z]{2}(?:\/|$)/i;
+/**
+ * Regex pattern for US English locale path (/en-us/ only)
+ */
+const US_ENGLISH_LOCALE_REGEX = /^\/en[-_]us(?:\/|$)/i;
+/**
+ * Check if a path should be filtered out (non-US-English locale)
+ *
+ * Returns true (should filter) for:
+ * - /fr-be/, /de-de/, /ja-jp/, /zh-cn/ (non-English locales)
+ * - /en-gb/, /en-au/, /en-ca/ (non-US English locales)
+ *
+ * Returns false (should keep) for:
+ * - /en-us/ (US English only)
+ * - /blog/, /news/, /articles/ (no locale prefix - default to US English)
+ */
+function isNonEnglishLocalePath(path) {
+    // If path has a locale prefix (xx-yy format)
+    if (LOCALE_PATH_REGEX.test(path)) {
+        // Only allow /en-us/
+        return !US_ENGLISH_LOCALE_REGEX.test(path);
+    }
+    // No locale prefix - allow (assume US English)
+    return false;
+}
+/**
+ * Check if a URL should be denied based on path patterns
+ *
+ * @param url - URL to check
+ * @param denyPaths - Patterns to deny (supports wildcards with *)
+ * @returns True if URL should be denied
+ */
+function shouldDenyUrl(url, denyPaths = exports.DEFAULT_DENY_PATHS) {
+    try {
+        const urlObj = new URL(url);
+        const path = urlObj.pathname;
+        // First check for non-English locale patterns (e.g., /fr-be/, /de-ch/)
+        if (isNonEnglishLocalePath(path)) {
+            return true;
+        }
+        return denyPaths.some((pattern) => {
+            // Exact match
+            if (pattern === path)
+                return true;
+            // Wildcard match (e.g., /about/*)
+            if (pattern.endsWith('/*')) {
+                const prefix = pattern.slice(0, -2); // Remove /*
+                return path.startsWith(prefix);
+            }
+            return false;
+        });
+    }
+    catch {
+        return false; // Invalid URL, don't deny
+    }
+}
+/**
+ * Get quality score breakdown for debugging
+ * Useful for understanding why an article scored a certain way
+ *
+ * @param extracted - Extracted content from article
+ * @param config - Optional quality score configuration
+ * @returns Breakdown of quality score components
+ */
+function getQualityBreakdown(extracted, config = {}) {
+    const finalConfig = { ...exports.DEFAULT_QUALITY_CONFIG, ...config };
+    const validation = validateContent(extracted);
+    const breakdown = {
+        contentValidation: validation.score * finalConfig.contentWeight,
+        publishedDate: extracted.publishedTime ? finalConfig.dateWeight : 0,
+        author: extracted.byline ? finalConfig.authorWeight : 0,
+        schema: 0,
+        readingTime: extracted.readingTime && extracted.readingTime >= 2 ? finalConfig.readingTimeWeight : 0,
+        total: 0,
+        passesThreshold: false,
+    };
+    // Check schema
+    if (extracted.structured?.jsonLd) {
+        const schemas = Array.isArray(extracted.structured.jsonLd)
+            ? extracted.structured.jsonLd
+            : [extracted.structured.jsonLd];
+        const hasArticleType = schemas.some((s) => {
+            const type = s['@type'];
+            return (type === 'Article' ||
+                type === 'NewsArticle' ||
+                type === 'BlogPosting' ||
+                type === 'TechArticle' ||
+                type === 'ScholarlyArticle');
+        });
+        if (hasArticleType) {
+            breakdown.schema = finalConfig.schemaWeight;
+        }
+    }
+    breakdown.total =
+        breakdown.contentValidation +
+            breakdown.publishedDate +
+            breakdown.author +
+            breakdown.schema +
+            breakdown.readingTime;
+    breakdown.passesThreshold = breakdown.total >= finalConfig.threshold;
+    return breakdown;
+}
+//# sourceMappingURL=quality-scorer.js.map

package/dist/lib/quality-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"quality-scorer.js","sourceRoot":"","sources":["../../lib/quality-scorer.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAiLH,0CAsCC;AAgBD,oEAoDC;AAuBD,wDAQC;AASD,sCAyBC;AAUD,kDAyDC;AA3ZD;;;GAGG;AACU,QAAA,sBAAsB,GAAiC;IAClE,aAAa,EAAE,IAAI,EAAO,8CAA8C;IACxE,UAAU,EAAE,IAAI,EAAW,4BAA4B;IACvD,YAAY,EAAE,IAAI,EAAS,yBAAyB;IACpD,YAAY,EAAE,IAAI,EAAS,sBAAsB;IACjD,iBAAiB,EAAE,IAAI,EAAI,oCAAoC;IAC/D,SAAS,EAAE,IAAI,EAAY,8BAA8B;CAC1D,CAAC;AAEF;;;GAGG;AACU,QAAA,kBAAkB,GAAG;IAChC,GAAG;IACH,QAAQ;IACR,aAAa;IACb,QAAQ;IACR,UAAU;IACV,UAAU;IACV,YAAY;IACZ,OAAO;IACP,SAAS;IACT,UAAU;IACV,YAAY;IACZ,OAAO;IACP,SAAS;IACT,UAAU;IACV,QAAQ;IACR,UAAU;IACV,QAAQ;IACR,SAAS;IACT,aAAa;IACb,eAAe;IACf,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,SAAS;IACT,WAAW;IACX,4BAA4B;IAC5B,YAAY;IACZ,cAAc;IACd,YAAY;IACZ,cAAc;IACd,WAAW;IACX,aAAa;IACb,WAAW;IACX,aAAa;IACb,WAAW;IACX,aAAa;IACb,UAAU;IACV,YAAY;IACZ,OAAO;IACP,SAAS;IACT,MAAM;IACN,QAAQ;IACR,UAAU;IACV,YAAY;IACZ,WAAW;IACX,aAAa;IACb,OAAO;IACP,SAAS;IACT,QAAQ;IACR,SAAS;IACT,WAAW;IACX,UAAU;IACV,YAAY;IACZ,YAAY;IACZ,cAAc;IACd,WAAW;IACX,aAAa;IACb,eAAe;IACf,iBAAiB;IACjB,kBAAkB;IAClB,oBAAoB;IACpB,SAAS;IACT,WAAW;IACX,aAAa;IACb,eAAe;IACf,gBAAgB;IAChB,kBAAkB;IAClB,aAAa;IACb,eAAe;IACf,WAAW;IACX,aAAa;IACb,aAAa;IACb,eAAe;IACf,gBAAgB;IAChB,kBAAkB;IAClB,OAAO;IACP,SAAS;IACT,MAAM;IACN,QAAQ;IACR,WAAW;IACX,aAAa;IACb,YAAY;IACZ,cAAc;IACd,sDAAsD;IACtD,UAAU,EAAG,QAAQ;IACrB,UAAU,EAAG,SAAS;IACtB,UAAU,EAAG,mBAAmB;IAChC,UAAU,EAAG,iBAAiB;IAC9B,UAAU,EAAG,SAAS;IACtB,UAAU,EAAG,kBAAkB;IAC/B,UAAU,EAAG,UAAU;IACvB,UAAU,EAAG,mBAAmB;IAChC,UAAU,EAAG,0BAA0B;IACvC,UAAU,EAAG,UAAU;IACvB,UAAU,EAAG,WAAW;IACxB,UAAU,EAAG,SAAS;IACtB,UAAU,EAAG,uBAAuB;IACpC,UAAU,EAAG,wBAAwB;IACrC,UAAU,EAAG,sBAAsB;IACnC,UAAU,EAAG,sBAAsB;IACnC,UAAU,EAAG,aAAa;IAC1B,UAAU,EAAG,UAAU;IACvB,UAAU,EAAG,SAAS;IACtB,UAAU,EAAG,QAAQ;IACrB,UAAU,EAAG,UAAU;IACvB,UAAU,EAAG,YAAY;IACzB,UAAU,EAAG,SAAS;IACtB,UAAU,EAAG,UAAU;IACvB,UAAU,EAAG,UAAU;IACvB,UAAU,EAAG,SAAS;IACtB,UAAU,EAAG,SAAS;IACtB,UAAU,EAAG,OAAO;IACpB,UAAU,EAAG,aAAa;IAC1B,UAAU,EAAG,aAAa;IAC1B,uBAAuB;IACvB,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;CACR,CAAC;AAEF;;;GAGG;AACU,QAAA,mBAAmB,GAAG;IACjC,SAAS;IACT,SAAS;IACT,aAAa;IACb,UAAU;IACV,YAAY;IACZ,UAAU;IACV,YAAY;IACZ,kBAAkB;IAClB,aAAa;IACb,cAAc;IACd,iBAAiB;IACjB,aAAa;IACb,gBAAgB;CACjB,CAAC;AAEF;;;;;;GAMG;AACH,SAAgB,eAAe,CAAC,SAA2B;IACzD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAG,GAAG,CAAC,CAAC,8CAA8C;IAE/D,gDAAgD;IAChD,MAAM,aAAa,GAAG,SAAS,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,CAAC;IACzD,IAAI,aAAa,GAAG,GAAG,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACrD,KAAK,IAAI,GAAG,CAAC,CAAC,kCAAkC;IAClD,CAAC;IAED,0CAA0C;IAC1C,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC,CAAC;IACjD,IAAI,WAAW,GAAG,EAAE,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;QACjE,KAAK,IAAI,GAAG,CAAC;IACf,CAAC;IAED,yDAAyD;IACzD,IAAI,SAAS,CAAC,OAAO,IAAI,SAAS,CAAC,WAAW,EAAE,CAAC;QAC/C,MAAM,UAAU,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC;QAC5C,MAAM,UAAU,GAAG,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC;QAChD,MAAM,KAAK,GAAG,UAAU,GAAG,UAAU,CAAC;QAEtC,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;YAC/C,KAAK,IAAI,GAAG,CAAC;QACf,CAAC;IACH,CAAC;IAED,yDAAyD;IACzD,MAAM,OAAO,GAAG,KAAK,IAAI,GAAG,CAAC;IAE7B,OAAO;QACL,OAAO;QACP,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,EAAE,oBAAoB;QAC9D,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAgB,4BAA4B,CAC1C,SAA2B,EAC3B,SAA6B,EAAE;IAE/B,MAAM,WAAW,GAAG,EAAE,GAAG,8BAAsB,EAAE,GAAG,MAAM,EAAE,CAAC;IAC7D,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,qDAAqD;IACrD,MAAM,UAAU,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;IAC9C,KAAK,IAAI,UAAU,CAAC,KAAK,GAAG,WAAW,CAAC,aAAa,CAAC;IAEtD,mCAAmC;IAEnC,+CAA+C;IAC/C,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,WAAW,CAAC,UAAU,CAAC;IAClC,CAAC;IAED,2CAA2C;IAC3C,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;QACrB,KAAK,IAAI,WAAW,CAAC,YAAY,CAAC;IACpC,CAAC;IAED,yDAAyD;IACzD,IAAI,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC;YACxD,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM;YAC7B,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAElC,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE;YAC7C,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,CACL,IAAI,KAAK,SAAS;gBAClB,IAAI,KAAK,aAAa;gBACtB,IAAI,KAAK,aAAa;gBACtB,IAAI,KAAK,aAAa;gBACtB,IAAI,KAAK,kBAAkB,CAC5B,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,cAAc,EAAE,CAAC;YACnB,KAAK,IAAI,WAAW,CAAC,YAAY,CAAC;QACpC,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,gDAAgD;IAChD,IAAI,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;QACxD,KAAK,IAAI,WAAW,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,aAAa;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,iBAAiB,GAAG,kCAAkC,CAAC;AAE7D;;GAEG;AACH,MAAM,uBAAuB,GAAG,sBAAsB,CAAC;AAEvD;;;;;;;;;;GAUG;AACH,SAAgB,sBAAsB,CAAC,IAAY;IACjD,6CAA6C;IAC7C,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,qBAAqB;QACrB,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IACD,+CAA+C;IAC/C,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,aAAa,CAAC,GAAW,EAAE,YAAsB,0BAAkB;IACjF,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;QAE7B,uEAAuE;QACvE,IAAI,sBAAsB,CAAC,IAAI,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;YAChC,cAAc;YACd,IAAI,OAAO,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAC;YAElC,kCAAkC;YAClC,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY;gBACjD,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACjC,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC,CAAC,0BAA0B;IAC1C,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,mBAAmB,CACjC,SAA2B,EAC3B,SAA6B,EAAE;IAU/B,MAAM,WAAW,GAAG,EAAE,GAAG,8BAAsB,EAAE,GAAG,MAAM,EAAE,CAAC;IAC7D,MAAM,UAAU,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;IAE9C,MAAM,SAAS,GAAG;QAChB,iBAAiB,EAAE,UAAU,CAAC,KAAK,GAAG,WAAW,CAAC,aAAa;QAC/D,aAAa,EAAE,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACnE,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,EAAE,CAAC;QACT,WAAW,EAAE,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;QACpG,KAAK,EAAE,CAAC;QACR,eAAe,EAAE,KAAK;KACvB,CAAC;IAEF,eAAe;IACf,IAAI,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC;YACxD,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM;YAC7B,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAElC,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE;YAC7C,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,CACL,IAAI,KAAK,SAAS;gBAClB,IAAI,KAAK,aAAa;gBACtB,IAAI,KAAK,aAAa;gBACtB,IAAI,KAAK,aAAa;gBACtB,IAAI,KAAK,kBAAkB,CAC5B,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,cAAc,EAAE,CAAC;YACnB,SAAS,CAAC,MAAM,GAAG,WAAW,CAAC,YAAY,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,SAAS,CAAC,KAAK;QACb,SAAS,CAAC,iBAAiB;YAC3B,SAAS,CAAC,aAAa;YACvB,SAAS,CAAC,MAAM;YAChB,SAAS,CAAC,MAAM;YAChB,SAAS,CAAC,WAAW,CAAC;IAExB,SAAS,CAAC,eAAe,GAAG,SAAS,CAAC,KAAK,IAAI,WAAW,CAAC,SAAS,CAAC;IAErE,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/lib/rss-utils.d.ts

@@ -0,0 +1,31 @@
+export interface RSSItem {
+    title: string;
+    link: string;
+    pubDate: string;
+    guid: string;
+    content?: string;
+    contentSnippet?: string;
+}
+/**
+ * Creates a content-based hash for article deduplication.
+ * Uses normalized title + date + source instead of URL to handle:
+ * - URL tracking parameters (utm_source, etc.)
+ * - URL redirects (HTTP vs HTTPS, www vs non-www)
+ * - Cross-posting (same article on multiple sites)
+ *
+ * @param title - Article title
+ * @param link - Article URL (legacy, kept for backward compatibility)
+ * @param publishedAt - Publication date (defaults to now)
+ * @param source - Source name (defaults to 'unknown')
+ * @returns SHA-256 hash as hex string
+ */
+export declare function createGuidHash(title: string, link: string, publishedAt?: Date, source?: string): string;
+export declare function fetchRSSFeed(url: string, sourceId?: string): Promise<RSSItem[]>;
+export declare function validateRSSFeed(url: string): Promise<{
+    isValid: boolean;
+    error?: string;
+    feedTitle?: string;
+    itemCount?: number;
+    contentType?: string;
+}>;
+//# sourceMappingURL=rss-utils.d.ts.map

package/dist/lib/rss-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rss-utils.d.ts","sourceRoot":"","sources":["../../lib/rss-utils.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,WAAW,GAAE,IAAiB,EAC9B,MAAM,GAAE,MAAkB,GACzB,MAAM,CA0BR;AAED,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CA4CrF;AAGD,wBAAsB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1D,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC,CA2FD"}

package/dist/lib/rss-utils.js

@@ -0,0 +1,175 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createGuidHash = createGuidHash;
+exports.fetchRSSFeed = fetchRSSFeed;
+exports.validateRSSFeed = validateRSSFeed;
+const rss_parser_1 = __importDefault(require("rss-parser"));
+const crypto_1 = __importDefault(require("crypto"));
+const parser = new rss_parser_1.default({
+    timeout: 15000, // Increased timeout
+    headers: {
+        'User-Agent': 'Mozilla/5.0 (compatible; AtomizeNews/1.0; +https://atomize-news.vercel.app)'
+    }
+});
+/**
+ * Creates a content-based hash for article deduplication.
+ * Uses normalized title + date + source instead of URL to handle:
+ * - URL tracking parameters (utm_source, etc.)
+ * - URL redirects (HTTP vs HTTPS, www vs non-www)
+ * - Cross-posting (same article on multiple sites)
+ *
+ * @param title - Article title
+ * @param link - Article URL (legacy, kept for backward compatibility)
+ * @param publishedAt - Publication date (defaults to now)
+ * @param source - Source name (defaults to 'unknown')
+ * @returns SHA-256 hash as hex string
+ */
+function createGuidHash(title, link, publishedAt = new Date(), source = 'unknown') {
+    // Normalize title: lowercase, collapse whitespace, remove punctuation
+    const normalizedTitle = title
+        .trim()
+        .toLowerCase()
+        .replace(/\s+/g, ' ')
+        .replace(/[^\w\s]/g, ''); // Remove punctuation for better matching
+    // Extract date bucket (YYYY-MM-DD) - same day articles might be duplicates
+    const dateKey = publishedAt.toISOString().split('T')[0];
+    // Normalize source name
+    const normalizedSource = source.toLowerCase().trim();
+    // Create composite key: title | date | source
+    const composite = `${normalizedTitle}|${dateKey}|${normalizedSource}`;
+    // Generate SHA-256 hash
+    const hash = crypto_1.default.createHash('sha256').update(composite).digest('hex');
+    // Log for debugging (in development)
+    if (process.env.NODE_ENV === 'development') {
+        console.log(`🔍 [GuidHash] Generated hash for: "${title.substring(0, 50)}..." from ${source} on ${dateKey}`);
+    }
+    return hash;
+}
+async function fetchRSSFeed(url, sourceId) {
+    const now = new Date();
+    try {
+        console.log(`🔄 [RSS] Fetching feed from ${url}`);
+        const feed = await parser.parseURL(url);
+        if (!feed.items || feed.items.length === 0) {
+            console.warn(`⚠️ [RSS] Feed from ${url} contains no items`);
+            return [];
+        }
+        const items = feed.items.map(item => ({
+            title: item.title || 'Untitled',
+            link: item.link || '',
+            pubDate: item.pubDate || new Date().toISOString(),
+            guid: item.guid || item.link || crypto_1.default.randomUUID(),
+            content: item.content || item['content:encoded'] || '',
+            contentSnippet: item.contentSnippet || ''
+        }));
+        console.log(`✅ [RSS] Successfully fetched ${items.length} items from ${url}`);
+        return items;
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+        console.error(`❌ [RSS] Failed to fetch RSS from ${url}:`, errorMessage);
+        // Log specific error types for debugging
+        if (error instanceof Error) {
+            if (error.message.includes('Invalid character')) {
+                console.error(`🔍 [RSS] XML parsing error - feed may be malformed or contain HTML`);
+            }
+            else if (error.message.includes('timeout')) {
+                console.error(`🔍 [RSS] Request timeout - server may be slow or unreachable`);
+            }
+            else if (error.message.includes('ENOTFOUND')) {
+                console.error(`🔍 [RSS] Domain not found - check URL spelling`);
+            }
+            else if (error.message.includes('ECONNREFUSED')) {
+                console.error(`🔍 [RSS] Connection refused - server may be down`);
+            }
+        }
+        return [];
+    }
+}
+// Enhanced RSS validation with better error handling
+async function validateRSSFeed(url) {
+    try {
+        console.log(`🔍 [Validation] Validating RSS feed: ${url}`);
+        // First check if URL is reachable
+        const response = await fetch(url, {
+            method: 'HEAD',
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (compatible; AtomizeNews/1.0; +https://atomize-news.vercel.app)'
+            },
+            signal: AbortSignal.timeout(10000)
+        });
+        if (!response.ok) {
+            return {
+                isValid: false,
+                error: `HTTP ${response.status}: ${response.statusText}`,
+                contentType: response.headers.get('content-type') || undefined
+            };
+        }
+        const contentType = response.headers.get('content-type') || '';
+        // Check if content type suggests RSS/XML
+        const isRssContentType = contentType.includes('application/rss+xml') ||
+            contentType.includes('application/xml') ||
+            contentType.includes('text/xml') ||
+            contentType.includes('application/atom+xml');
+        if (!isRssContentType && contentType.includes('text/html')) {
+            return {
+                isValid: false,
+                error: 'URL returns HTML content instead of RSS feed',
+                contentType
+            };
+        }
+        // Now fetch and parse the actual RSS content
+        const feed = await parser.parseURL(url);
+        if (!feed.title) {
+            return {
+                isValid: false,
+                error: 'RSS feed has no title - may be malformed',
+                contentType
+            };
+        }
+        if (!feed.items || feed.items.length === 0) {
+            return {
+                isValid: false,
+                error: 'RSS feed contains no items',
+                contentType,
+                feedTitle: feed.title
+            };
+        }
+        console.log(`✅ [Validation] RSS feed validated successfully: ${feed.title} (${feed.items.length} items)`);
+        return {
+            isValid: true,
+            feedTitle: feed.title,
+            itemCount: feed.items.length,
+            contentType
+        };
+    }
+    catch (error) {
+        let errorMessage = 'Unknown validation error';
+        if (error instanceof Error) {
+            if (error.message.includes('timeout') || error.name === 'AbortError') {
+                errorMessage = 'Request timeout - URL may be unreachable';
+            }
+            else if (error.message.includes('Invalid character')) {
+                errorMessage = 'Invalid RSS/XML format - feed may be malformed';
+            }
+            else if (error.message.includes('getaddrinfo ENOTFOUND')) {
+                errorMessage = 'Domain not found - check URL spelling';
+            }
+            else if (error.message.includes('ECONNREFUSED')) {
+                errorMessage = 'Connection refused - server may be down';
+            }
+            else {
+                errorMessage = error.message;
+            }
+        }
+        console.error(`❌ [Validation] RSS validation failed for ${url}:`, errorMessage);
+        return {
+            isValid: false,
+            error: errorMessage
+        };
+    }
+}
+//# sourceMappingURL=rss-utils.js.map

package/dist/lib/rss-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rss-utils.js","sourceRoot":"","sources":["../../lib/rss-utils.ts"],"names":[],"mappings":";;;;;AAgCA,wCA+BC;AAED,oCA4CC;AAGD,0CAiGC;AAjND,4DAAgC;AAChC,oDAA4B;AAE5B,MAAM,MAAM,GAAG,IAAI,oBAAM,CAAC;IACxB,OAAO,EAAE,KAAK,EAAE,oBAAoB;IACpC,OAAO,EAAE;QACP,YAAY,EAAE,6EAA6E;KAC5F;CACF,CAAC,CAAC;AAWH;;;;;;;;;;;;GAYG;AACH,SAAgB,cAAc,CAC5B,KAAa,EACb,IAAY,EACZ,cAAoB,IAAI,IAAI,EAAE,EAC9B,SAAiB,SAAS;IAE1B,sEAAsE;IACtE,MAAM,eAAe,GAAG,KAAK;SAC1B,IAAI,EAAE;SACN,WAAW,EAAE;SACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,yCAAyC;IAErE,2EAA2E;IAC3E,MAAM,OAAO,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAExD,wBAAwB;IACxB,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IAErD,8CAA8C;IAC9C,MAAM,SAAS,GAAG,GAAG,eAAe,IAAI,OAAO,IAAI,gBAAgB,EAAE,CAAC;IAEtE,wBAAwB;IACxB,MAAM,IAAI,GAAG,gBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEzE,qCAAqC;IACrC,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,sCAAsC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,MAAM,OAAO,OAAO,EAAE,CAAC,CAAC;IAC/G,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAEM,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,QAAiB;IAC/D,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IAEvB,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;QAElD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAExC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,CAAC,CAAC;YAC5D,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,UAAU;YAC/B,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACjD,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,gBAAM,CAAC,UAAU,EAAE;YACnD,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE;YACtD,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,EAAE;SAC1C,CAAC,CAAC,CAAC;QAEJ,OAAO,CAAC,GAAG,CAAC,gCAAgC,KAAK,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC,CAAC;QAC9E,OAAO,KAAK,CAAC;IAEf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QAC9E,OAAO,CAAC,KAAK,CAAC,oCAAoC,GAAG,GAAG,EAAE,YAAY,CAAC,CAAC;QAExE,yCAAyC;QACzC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;gBAChD,OAAO,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;YACtF,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7C,OAAO,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;YAChF,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC/C,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;YAClE,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClD,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,qDAAqD;AAC9C,KAAK,UAAU,eAAe,CAAC,GAAW;IAO/C,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,wCAAwC,GAAG,EAAE,CAAC,CAAC;QAE3D,kCAAkC;QAClC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,YAAY,EAAE,6EAA6E;aAC5F;YACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;SACnC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE;gBACxD,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS;aAC/D,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAE/D,yCAAyC;QACzC,MAAM,gBAAgB,GACpB,WAAW,CAAC,QAAQ,CAAC,qBAAqB,CAAC;YAC3C,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC;YACvC,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC;YAChC,WAAW,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC;QAE/C,IAAI,CAAC,gBAAgB,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3D,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,8CAA8C;gBACrD,WAAW;aACZ,CAAC;QACJ,CAAC;QAED,6CAA6C;QAC7C,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAExC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,0CAA0C;gBACjD,WAAW;aACZ,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,4BAA4B;gBACnC,WAAW;gBACX,SAAS,EAAE,IAAI,CAAC,KAAK;aACtB,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,mDAAmD,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC;QAE1G,OAAO;YACL,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,IAAI,CAAC,KAAK;YACrB,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YAC5B,WAAW;SACZ,CAAC;IAEJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,YAAY,GAAG,0BAA0B,CAAC;QAE9C,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACrE,YAAY,GAAG,0CAA0C,CAAC;YAC5D,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;gBACvD,YAAY,GAAG,gDAAgD,CAAC;YAClE,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC;gBAC3D,YAAY,GAAG,uCAAuC,CAAC;YACzD,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClD,YAAY,GAAG,yCAAyC,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,4CAA4C,GAAG,GAAG,EAAE,YAAY,CAAC,CAAC;QAEhF,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,YAAY;SACpB,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/lib/scraping-rate-limiter.d.ts

@@ -0,0 +1,52 @@
+export interface RateLimiterConfig {
+    requestsPerSecond?: number;
+    maxBackoff?: number;
+    maxConcurrent?: number;
+    maxConcurrentPerHost?: number;
+}
+export declare const RATE_LIMITER_PRESETS: {
+    readonly conservative: {
+        readonly requestsPerSecond: 1;
+        readonly maxBackoff: 30000;
+        readonly maxConcurrent: 10;
+        readonly maxConcurrentPerHost: 2;
+    };
+    readonly moderate: {
+        readonly requestsPerSecond: 2;
+        readonly maxBackoff: 30000;
+        readonly maxConcurrent: 20;
+        readonly maxConcurrentPerHost: 3;
+    };
+    readonly aggressive: {
+        readonly requestsPerSecond: 4;
+        readonly maxBackoff: 15000;
+        readonly maxConcurrent: 30;
+        readonly maxConcurrentPerHost: 5;
+    };
+};
+export type RateLimiterPreset = keyof typeof RATE_LIMITER_PRESETS;
+export declare class ScrapingRateLimiter {
+    private hosts;
+    private readonly baseDelay;
+    private readonly maxBackoff;
+    private readonly maxConcurrent;
+    private readonly maxConcurrentPerHost;
+    private activeRequests;
+    constructor(options?: RateLimiterConfig);
+    static fromPreset(preset: RateLimiterPreset): ScrapingRateLimiter;
+    execute<T>(url: string, operation: () => Promise<T>, options?: {
+        priority?: number;
+        maxRetries?: number;
+    }): Promise<T>;
+    private extractHost;
+    private enqueueRequest;
+    private processQueue;
+    private handleRequestError;
+    private shouldRetry;
+    private shouldBackoff;
+    private wait;
+    getStats(): Record<string, any>;
+}
+export declare const globalRateLimiter: ScrapingRateLimiter;
+export declare function createRateLimiter(config: RateLimiterConfig | RateLimiterPreset): ScrapingRateLimiter;
+//# sourceMappingURL=scraping-rate-limiter.d.ts.map

package/dist/lib/scraping-rate-limiter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraping-rate-limiter.d.ts","sourceRoot":"","sources":["../../lib/scraping-rate-limiter.ts"],"names":[],"mappings":"AAmBA,MAAM,WAAW,iBAAiB;IAChC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAGD,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;CAsBvB,CAAC;AAEX,MAAM,MAAM,iBAAiB,GAAG,MAAM,OAAO,oBAAoB,CAAC;AAElE,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,KAAK,CAAgC;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,cAAc,CAAqB;gBAE/B,OAAO,GAAE,iBAAsB;IAO3C,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,iBAAiB,GAAG,mBAAmB;IAI3D,OAAO,CAAC,CAAC,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC3B,OAAO,GAAE;QACP,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;KAChB,GACL,OAAO,CAAC,CAAC,CAAC;IAqBb,OAAO,CAAC,WAAW;IASnB,OAAO,CAAC,cAAc;YAiCR,YAAY;YAiEZ,kBAAkB;IAuChC,OAAO,CAAC,WAAW;IAyBnB,OAAO,CAAC,aAAa;IAerB,OAAO,CAAC,IAAI;IAKZ,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAmBhC;AAGD,eAAO,MAAM,iBAAiB,qBAA6C,CAAC;AAG5E,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,GAAG,iBAAiB,GAAG,mBAAmB,CAKpG"}