@tyroneross/blog-scraper 0.1.0 → 0.3.0

package/dist/lib/scraping-rate-limiter.js

@@ -0,0 +1,238 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.globalRateLimiter = exports.ScrapingRateLimiter = exports.RATE_LIMITER_PRESETS = void 0;
+exports.createRateLimiter = createRateLimiter;
+// Preset configurations for different use cases
+exports.RATE_LIMITER_PRESETS = {
+    // Conservative: for web app with strict limits
+    conservative: {
+        requestsPerSecond: 1,
+        maxBackoff: 30000,
+        maxConcurrent: 10,
+        maxConcurrentPerHost: 2,
+    },
+    // Moderate: good balance of speed and politeness (default for web app)
+    moderate: {
+        requestsPerSecond: 2,
+        maxBackoff: 30000,
+        maxConcurrent: 20,
+        maxConcurrentPerHost: 3,
+    },
+    // Aggressive: for SDK usage with higher limits
+    aggressive: {
+        requestsPerSecond: 4,
+        maxBackoff: 15000,
+        maxConcurrent: 30,
+        maxConcurrentPerHost: 5,
+    },
+};
+class ScrapingRateLimiter {
+    constructor(options = {}) {
+        this.hosts = new Map();
+        this.activeRequests = new Set();
+        this.baseDelay = Math.floor(1000 / (options.requestsPerSecond || 2));
+        this.maxBackoff = options.maxBackoff || 30000;
+        this.maxConcurrent = options.maxConcurrent || 20;
+        this.maxConcurrentPerHost = options.maxConcurrentPerHost || 3;
+    }
+    static fromPreset(preset) {
+        return new ScrapingRateLimiter(exports.RATE_LIMITER_PRESETS[preset]);
+    }
+    async execute(url, operation, options = {}) {
+        const host = this.extractHost(url);
+        if (!host) {
+            throw new Error(`Invalid URL: ${url}`);
+        }
+        return new Promise((resolve, reject) => {
+            const request = {
+                resolve,
+                reject,
+                operation,
+                priority: options.priority || 0,
+                retryCount: 0,
+                maxRetries: options.maxRetries || 3,
+                host
+            };
+            this.enqueueRequest(host, request);
+        });
+    }
+    extractHost(url) {
+        try {
+            const parsed = new URL(url);
+            return parsed.hostname.toLowerCase();
+        }
+        catch {
+            return null;
+        }
+    }
+    enqueueRequest(host, request) {
+        if (!this.hosts.has(host)) {
+            this.hosts.set(host, {
+                lastRequest: 0,
+                backoffUntil: 0,
+                backoffMultiplier: 1,
+                queue: [],
+                processing: false,
+                activeCount: 0
+            });
+        }
+        const hostState = this.hosts.get(host);
+        // Insert request in priority order (higher priority first)
+        const insertIndex = hostState.queue.findIndex(req => req.priority < request.priority);
+        if (insertIndex === -1) {
+            hostState.queue.push(request);
+        }
+        else {
+            hostState.queue.splice(insertIndex, 0, request);
+        }
+        // Start processing if not already running
+        if (!hostState.processing) {
+            this.processQueue(host).catch(error => {
+                console.error(`[RateLimiter] Error processing queue for ${host}:`, error);
+            });
+        }
+    }
+    async processQueue(host) {
+        const hostState = this.hosts.get(host);
+        if (!hostState || hostState.processing) {
+            return;
+        }
+        hostState.processing = true;
+        try {
+            while (hostState.queue.length > 0) {
+                // Check if we're within global concurrent limits
+                if (this.activeRequests.size >= this.maxConcurrent) {
+                    await this.wait(100);
+                    continue;
+                }
+                // Check if we're within per-host concurrent limits
+                if (hostState.activeCount >= this.maxConcurrentPerHost) {
+                    await this.wait(100);
+                    continue;
+                }
+                // Check if we're still in backoff period
+                if (Date.now() < hostState.backoffUntil) {
+                    const waitTime = hostState.backoffUntil - Date.now();
+                    await this.wait(Math.min(waitTime, 1000));
+                    continue;
+                }
+                // Check rate limiting
+                const now = Date.now();
+                const timeSinceLastRequest = now - hostState.lastRequest;
+                if (timeSinceLastRequest < this.baseDelay) {
+                    const waitTime = this.baseDelay - timeSinceLastRequest;
+                    await this.wait(waitTime);
+                    continue;
+                }
+                const request = hostState.queue.shift();
+                const requestId = `${host}-${Date.now()}-${Math.random()}`;
+                this.activeRequests.add(requestId);
+                hostState.activeCount++;
+                try {
+                    hostState.lastRequest = Date.now();
+                    const result = await request.operation();
+                    // Reset backoff on success
+                    hostState.backoffMultiplier = 1;
+                    hostState.backoffUntil = 0;
+                    request.resolve(result);
+                }
+                catch (error) {
+                    await this.handleRequestError(hostState, request, error);
+                }
+                finally {
+                    this.activeRequests.delete(requestId);
+                    hostState.activeCount--;
+                }
+            }
+        }
+        finally {
+            hostState.processing = false;
+        }
+    }
+    async handleRequestError(hostState, request, error) {
+        const shouldRetry = this.shouldRetry(error, request);
+        if (shouldRetry && request.retryCount < request.maxRetries) {
+            request.retryCount++;
+            // Apply exponential backoff for certain errors
+            if (this.shouldBackoff(error)) {
+                const backoffTime = Math.min(this.baseDelay * hostState.backoffMultiplier * Math.pow(2, request.retryCount), this.maxBackoff);
+                hostState.backoffUntil = Date.now() + backoffTime;
+                hostState.backoffMultiplier = Math.min(hostState.backoffMultiplier * 1.5, 10);
+                console.warn(`[RateLimiter] Backing off ${request.host} for ${backoffTime}ms ` +
+                    `(attempt ${request.retryCount}/${request.maxRetries}): ${error.message}`);
+            }
+            // Re-queue the request with lower priority
+            request.priority = Math.max(request.priority - 1, -10);
+            hostState.queue.unshift(request);
+        }
+        else {
+            console.error(`[RateLimiter] Request failed for ${request.host} ` +
+                `(${request.retryCount}/${request.maxRetries} retries): ${error.message}`);
+            request.reject(error);
+        }
+    }
+    shouldRetry(error, request) {
+        // Don't retry if we've exceeded max retries
+        if (request.retryCount >= request.maxRetries) {
+            return false;
+        }
+        // Retry on network errors
+        if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') {
+            return true;
+        }
+        // Retry on HTTP status codes that might be transient
+        if (error.status) {
+            const status = error.status;
+            return status === 408 || status === 429 || status >= 500;
+        }
+        // Retry on timeout errors
+        if (error.name === 'AbortError' || error.message.includes('timeout')) {
+            return true;
+        }
+        return false;
+    }
+    shouldBackoff(error) {
+        // Apply backoff for rate limiting and server errors
+        if (error.status) {
+            const status = error.status;
+            return status === 429 || status >= 500;
+        }
+        // Apply backoff for connection errors
+        if (error.code === 'ECONNREFUSED' || error.code === 'ENOTFOUND') {
+            return true;
+        }
+        return false;
+    }
+    wait(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+    // Utility method to get current queue stats
+    getStats() {
+        const stats = {};
+        this.hosts.forEach((state, host) => {
+            stats[host] = {
+                queueLength: state.queue.length,
+                processing: state.processing,
+                backoffUntil: state.backoffUntil,
+                backoffMultiplier: state.backoffMultiplier,
+                lastRequest: state.lastRequest
+            };
+        });
+        return {
+            hosts: stats,
+            activeRequests: this.activeRequests.size,
+            maxConcurrent: this.maxConcurrent
+        };
+    }
+}
+exports.ScrapingRateLimiter = ScrapingRateLimiter;
+// Default global rate limiter instance (moderate preset for web app)
+exports.globalRateLimiter = ScrapingRateLimiter.fromPreset('moderate');
+// Factory function to create rate limiter with custom config (for SDK users)
+function createRateLimiter(config) {
+    if (typeof config === 'string') {
+        return ScrapingRateLimiter.fromPreset(config);
+    }
+    return new ScrapingRateLimiter(config);
+}
+//# sourceMappingURL=scraping-rate-limiter.js.map

package/dist/lib/scraping-rate-limiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraping-rate-limiter.js","sourceRoot":"","sources":["../../lib/scraping-rate-limiter.ts"],"names":[],"mappings":";;;AA4TA,8CAKC;AAvSD,gDAAgD;AACnC,QAAA,oBAAoB,GAAG;IAClC,+CAA+C;IAC/C,YAAY,EAAE;QACZ,iBAAiB,EAAE,CAAC;QACpB,UAAU,EAAE,KAAK;QACjB,aAAa,EAAE,EAAE;QACjB,oBAAoB,EAAE,CAAC;KACxB;IACD,uEAAuE;IACvE,QAAQ,EAAE;QACR,iBAAiB,EAAE,CAAC;QACpB,UAAU,EAAE,KAAK;QACjB,aAAa,EAAE,EAAE;QACjB,oBAAoB,EAAE,CAAC;KACxB;IACD,+CAA+C;IAC/C,UAAU,EAAE;QACV,iBAAiB,EAAE,CAAC;QACpB,UAAU,EAAE,KAAK;QACjB,aAAa,EAAE,EAAE;QACjB,oBAAoB,EAAE,CAAC;KACxB;CACO,CAAC;AAIX,MAAa,mBAAmB;IAQ9B,YAAY,UAA6B,EAAE;QAPnC,UAAK,GAAG,IAAI,GAAG,EAAqB,CAAC;QAKrC,mBAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QAGzC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,iBAAiB,IAAI,CAAC,CAAC,CAAC,CAAC;QACrE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE,CAAC;QACjD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,IAAI,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,CAAC,UAAU,CAAC,MAAyB;QACzC,OAAO,IAAI,mBAAmB,CAAC,4BAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,OAAO,CACX,GAAW,EACX,SAA2B,EAC3B,UAGI,EAAE;QAEN,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;QACzC,CAAC;QAED,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,MAAM,OAAO,GAAkB;gBAC7B,OAAO;gBACP,MAAM;gBACN,SAAS;gBACT,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,CAAC;gBAC/B,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,CAAC;gBACnC,IAAI;aACL,CAAC;YAEF,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,WAAW,CAAC,GAAW;QAC7B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,OAAO,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,OAAsB;QACzD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE;gBACnB,WAAW,EAAE,CAAC;gBACd,YAAY,EAAE,CAAC;gBACf,iBAAiB,EAAE,CAAC;gBACpB,KAAK,EAAE,EAAE;gBACT,UAAU,EAAE,KAAK;gBACjB,WAAW,EAAE,CAAC;aACf,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;QAExC,2DAA2D;QAC3D,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAC3C,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CACvC,CAAC;QAEF,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;QAClD,CAAC;QAED,0CAA0C;QAC1C,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC;YAC1B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACpC,OAAO,CAAC,KAAK,CAAC,4CAA4C,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;YAC5E,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,IAAY;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,EAAE,CAAC;YACvC,OAAO;QACT,CAAC;QAED,SAAS,CAAC,UAAU,GAAG,IAAI,CAAC;QAE5B,IAAI,CAAC;YACH,OAAO,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,iDAAiD;gBACjD,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACnD,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,mDAAmD;gBACnD,IAAI,SAAS,CAAC,WAAW,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;oBACvD,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,yCAAyC;gBACzC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,YAAY,EAAE,CAAC;oBACxC,MAAM,QAAQ,GAAG,SAAS,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBACrD,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;oBAC1C,SAAS;gBACX,CAAC;gBAED,sBAAsB;gBACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,MAAM,oBAAoB,GAAG,GAAG,GAAG,SAAS,CAAC,WAAW,CAAC;gBACzD,IAAI,oBAAoB,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,GAAG,oBAAoB,CAAC;oBACvD,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC1B,SAAS;gBACX,CAAC;gBAED,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;gBACzC,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC3D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACnC,SAAS,CAAC,WAAW,EAAE,CAAC;gBAExB,IAAI,CAAC;oBACH,SAAS,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC;oBAEzC,2BAA2B;oBAC3B,SAAS,CAAC,iBAAiB,GAAG,CAAC,CAAC;oBAChC,SAAS,CAAC,YAAY,GAAG,CAAC,CAAC;oBAE3B,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAE1B,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;gBAC3D,CAAC;wBAAS,CAAC;oBACT,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;oBACtC,SAAS,CAAC,WAAW,EAAE,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;QAC/B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,SAAoB,EACpB,OAAsB,EACtB,KAAU;QAEV,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAErD,IAAI,WAAW,IAAI,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;YAC3D,OAAO,CAAC,UAAU,EAAE,CAAC;YAErB,+CAA+C;YAC/C,IAAI,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAC1B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,EAC9E,IAAI,CAAC,UAAU,CAChB,CAAC;gBAEF,SAAS,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;gBAClD,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,iBAAiB,GAAG,GAAG,EAAE,EAAE,CAAC,CAAC;gBAE9E,OAAO,CAAC,IAAI,CACV,6BAA6B,OAAO,CAAC,IAAI,QAAQ,WAAW,KAAK;oBACjE,YAAY,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,UAAU,MAAM,KAAK,CAAC,OAAO,EAAE,CAC1E,CAAC;YACJ,CAAC;YAED,2CAA2C;YAC3C,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACvD,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEnC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CACX,oCAAoC,OAAO,CAAC,IAAI,GAAG;gBACnD,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,UAAU,cAAc,KAAK,CAAC,OAAO,EAAE,CAC1E,CAAC;YACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,KAAU,EAAE,OAAsB;QACpD,4CAA4C;QAC5C,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YAC7C,OAAO,KAAK,CAAC;QACf,CAAC;QAED,0BAA0B;QAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAChE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qDAAqD;QACrD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YAC5B,OAAO,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,IAAI,GAAG,CAAC;QAC3D,CAAC;QAED,0BAA0B;QAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,aAAa,CAAC,KAAU;QAC9B,oDAAoD;QACpD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YAC5B,OAAO,MAAM,KAAK,GAAG,IAAI,MAAM,IAAI,GAAG,CAAC;QACzC,CAAC;QAED,sCAAsC;QACtC,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAChE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,IAAI,CAAC,EAAU;QACrB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,4CAA4C;IAC5C,QAAQ;QACN,MAAM,KAAK,GAAwB,EAAE,CAAC;QAEtC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACjC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACZ,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;gBAC/B,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;gBAC1C,WAAW,EAAE,KAAK,CAAC,WAAW;aAC/B,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,cAAc,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI;YACxC,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC;IACJ,CAAC;CACF;AAjQD,kDAiQC;AAED,qEAAqE;AACxD,QAAA,iBAAiB,GAAG,mBAAmB,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;AAE5E,6EAA6E;AAC7E,SAAgB,iBAAiB,CAAC,MAA6C;IAC7E,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,mBAAmB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,IAAI,mBAAmB,CAAC,MAAM,CAAC,CAAC;AACzC,CAAC"}

package/dist/lib/source-orchestrator.d.ts

@@ -0,0 +1,306 @@
+import { z } from 'zod';
+import { DiscoveredFeed } from './web-scrapers/rss-discovery';
+export declare const CandidateArticleSchema: z.ZodObject<{
+    url: z.ZodString;
+    title: z.ZodString;
+    publishedAt: z.ZodDate;
+    content: z.ZodOptional<z.ZodString>;
+    excerpt: z.ZodOptional<z.ZodString>;
+    guid: z.ZodString;
+    confidence: z.ZodNumber;
+    source: z.ZodEnum<["rss", "sitemap", "html", "discovery"]>;
+    extractionMethod: z.ZodEnum<["rss", "sitemap", "html-links", "content-extraction"]>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    title: string;
+    source: "rss" | "sitemap" | "html" | "discovery";
+    confidence: number;
+    extractionMethod: "rss" | "sitemap" | "html-links" | "content-extraction";
+    publishedAt: Date;
+    guid: string;
+    content?: string | undefined;
+    excerpt?: string | undefined;
+    metadata?: Record<string, any> | undefined;
+}, {
+    url: string;
+    title: string;
+    source: "rss" | "sitemap" | "html" | "discovery";
+    confidence: number;
+    extractionMethod: "rss" | "sitemap" | "html-links" | "content-extraction";
+    publishedAt: Date;
+    guid: string;
+    content?: string | undefined;
+    excerpt?: string | undefined;
+    metadata?: Record<string, any> | undefined;
+}>;
+export type CandidateArticle = z.infer<typeof CandidateArticleSchema>;
+export declare const SourceConfigSchema: z.ZodObject<{
+    sourceType: z.ZodEnum<["rss", "sitemap", "html", "auto"]>;
+    allowPaths: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    denyPaths: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    maxDepth: z.ZodOptional<z.ZodNumber>;
+    detectOnly: z.ZodOptional<z.ZodBoolean>;
+    scrapeConfig: z.ZodOptional<z.ZodObject<{
+        selectors: z.ZodOptional<z.ZodObject<{
+            articleLinks: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            titleSelectors: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            dateSelectors: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            excludeSelectors: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strip", z.ZodTypeAny, {
+            excludeSelectors?: string[] | undefined;
+            articleLinks?: string[] | undefined;
+            titleSelectors?: string[] | undefined;
+            dateSelectors?: string[] | undefined;
+        }, {
+            excludeSelectors?: string[] | undefined;
+            articleLinks?: string[] | undefined;
+            titleSelectors?: string[] | undefined;
+            dateSelectors?: string[] | undefined;
+        }>>;
+        filters: z.ZodOptional<z.ZodObject<{
+            minTitleLength: z.ZodOptional<z.ZodNumber>;
+            maxTitleLength: z.ZodOptional<z.ZodNumber>;
+            includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strip", z.ZodTypeAny, {
+            minTitleLength?: number | undefined;
+            maxTitleLength?: number | undefined;
+            excludePatterns?: string[] | undefined;
+            includePatterns?: string[] | undefined;
+        }, {
+            minTitleLength?: number | undefined;
+            maxTitleLength?: number | undefined;
+            excludePatterns?: string[] | undefined;
+            includePatterns?: string[] | undefined;
+        }>>;
+        limits: z.ZodOptional<z.ZodObject<{
+            maxLinksPerPage: z.ZodOptional<z.ZodNumber>;
+            maxPages: z.ZodOptional<z.ZodNumber>;
+        }, "strip", z.ZodTypeAny, {
+            maxLinksPerPage?: number | undefined;
+            maxPages?: number | undefined;
+        }, {
+            maxLinksPerPage?: number | undefined;
+            maxPages?: number | undefined;
+        }>>;
+    }, "strip", z.ZodTypeAny, {
+        filters?: {
+            minTitleLength?: number | undefined;
+            maxTitleLength?: number | undefined;
+            excludePatterns?: string[] | undefined;
+            includePatterns?: string[] | undefined;
+        } | undefined;
+        selectors?: {
+            excludeSelectors?: string[] | undefined;
+            articleLinks?: string[] | undefined;
+            titleSelectors?: string[] | undefined;
+            dateSelectors?: string[] | undefined;
+        } | undefined;
+        limits?: {
+            maxLinksPerPage?: number | undefined;
+            maxPages?: number | undefined;
+        } | undefined;
+    }, {
+        filters?: {
+            minTitleLength?: number | undefined;
+            maxTitleLength?: number | undefined;
+            excludePatterns?: string[] | undefined;
+            includePatterns?: string[] | undefined;
+        } | undefined;
+        selectors?: {
+            excludeSelectors?: string[] | undefined;
+            articleLinks?: string[] | undefined;
+            titleSelectors?: string[] | undefined;
+            dateSelectors?: string[] | undefined;
+        } | undefined;
+        limits?: {
+            maxLinksPerPage?: number | undefined;
+            maxPages?: number | undefined;
+        } | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    sourceType: "rss" | "sitemap" | "html" | "auto";
+    maxDepth?: number | undefined;
+    allowPaths?: string[] | undefined;
+    denyPaths?: string[] | undefined;
+    detectOnly?: boolean | undefined;
+    scrapeConfig?: {
+        filters?: {
+            minTitleLength?: number | undefined;
+            maxTitleLength?: number | undefined;
+            excludePatterns?: string[] | undefined;
+            includePatterns?: string[] | undefined;
+        } | undefined;
+        selectors?: {
+            excludeSelectors?: string[] | undefined;
+            articleLinks?: string[] | undefined;
+            titleSelectors?: string[] | undefined;
+            dateSelectors?: string[] | undefined;
+        } | undefined;
+        limits?: {
+            maxLinksPerPage?: number | undefined;
+            maxPages?: number | undefined;
+        } | undefined;
+    } | undefined;
+}, {
+    sourceType: "rss" | "sitemap" | "html" | "auto";
+    maxDepth?: number | undefined;
+    allowPaths?: string[] | undefined;
+    denyPaths?: string[] | undefined;
+    detectOnly?: boolean | undefined;
+    scrapeConfig?: {
+        filters?: {
+            minTitleLength?: number | undefined;
+            maxTitleLength?: number | undefined;
+            excludePatterns?: string[] | undefined;
+            includePatterns?: string[] | undefined;
+        } | undefined;
+        selectors?: {
+            excludeSelectors?: string[] | undefined;
+            articleLinks?: string[] | undefined;
+            titleSelectors?: string[] | undefined;
+            dateSelectors?: string[] | undefined;
+        } | undefined;
+        limits?: {
+            maxLinksPerPage?: number | undefined;
+            maxPages?: number | undefined;
+        } | undefined;
+    } | undefined;
+}>;
+export type SourceConfig = z.infer<typeof SourceConfigSchema> & {
+    circuitBreaker?: {
+        execute<T>(operation: () => Promise<T>): Promise<T>;
+    };
+};
+export interface OrchestrationResult {
+    articles: CandidateArticle[];
+    sourceInfo: {
+        detectedType: 'rss' | 'sitemap' | 'html';
+        discoveredFeeds?: DiscoveredFeed[];
+        discoveredSitemaps?: string[];
+        extractionStats: {
+            attempted: number;
+            successful: number;
+            failed: number;
+            filtered: number;
+        };
+    };
+    processingTime: number;
+    errors: string[];
+}
+export declare class SourceOrchestrator {
+    private readonly maxArticlesPerSource;
+    private readonly recentTimeframe;
+    /**
+     * Common content section paths - prioritized for news/blog content
+     */
+    private readonly contentSectionPaths;
+    /**
+     * Common blog subdomains to check when scraping root domains
+     * Many companies host their blogs on separate subdomains
+     */
+    private readonly blogSubdomains;
+    /**
+     * Infer path filters from the input URL
+     * e.g., if user enters anthropic.com/news, filter results to /news/* paths
+     */
+    private inferPathFiltersFromUrl;
+    /**
+     * Discover content sections from sitemap when user enters root domain
+     * Returns prioritized list of content paths found
+     */
+    private discoverContentSectionsFromSitemap;
+    /**
+     * Filter sitemap entries to content sections when processing root domain
+     * Also applies non-English locale filtering
+     */
+    private filterToContentSections;
+    /**
+     * Discover blog subdomains for a given domain
+     * e.g., for nvidia.com, check if blogs.nvidia.com exists
+     */
+    private discoverBlogSubdomains;
+    /**
+     * Main orchestration method - determines source type and extracts content
+     */
+    processSource(url: string, config?: SourceConfig): Promise<OrchestrationResult>;
+    /**
+     * Auto-detect source type and process accordingly
+     */
+    private autoDetectAndProcess;
+    /**
+     * Process source with known type
+     */
+    private processKnownType;
+    /**
+     * Process URL as RSS feed
+     */
+    private processAsRSS;
+    /**
+     * Process URL as sitemap
+     */
+    private processAsSitemap;
+    /**
+     * Process URL as HTML page
+     */
+    private processAsHTML;
+    /**
+     * Process URL using Playwright for JavaScript-rendered pages
+     * Used as fallback when static HTML scraping fails
+     */
+    private processAsPlaywright;
+    /**
+     * Apply path filtering based on allowPaths and denyPaths
+     * Also filters out non-English locale paths
+     *
+     * @param articles - Articles to filter
+     * @param config - Source configuration
+     * @param options - Filtering options
+     * @param options.skipAllowFilters - Skip allow path filtering (useful for RSS which is already curated)
+     */
+    private applyPathFilters;
+    /**
+     * Check if a path matches a pattern (supports wildcards)
+     */
+    private matchesPattern;
+    /**
+     * Build scraping configuration from source config
+     */
+    private buildScrapingConfig;
+    /**
+     * Extract title from URL as fallback
+     */
+    private extractTitleFromUrl;
+    /**
+     * Create a consistent GUID for an article
+     */
+    private createGuid;
+    /**
+     * Finalize processing result
+     */
+    private finalizeResult;
+    /**
+     * Extract full content for articles (optional enhancement step)
+     */
+    enhanceWithFullContent(articles: CandidateArticle[], maxArticles?: number, options?: {
+        concurrency?: number;
+        onProgress?: (completed: number, total: number) => void;
+    }): Promise<CandidateArticle[]>;
+    /**
+     * Validate orchestrator configuration
+     */
+    static validateConfig(config: any): SourceConfig;
+    /**
+     * Get source statistics
+     */
+    getSourceStats(url: string): Promise<{
+        robotsCompliant: boolean;
+        hasRSSFeed: boolean;
+        hasSitemap: boolean;
+        detectedType: string;
+        estimatedArticleCount: number;
+    }>;
+}
+export declare const globalSourceOrchestrator: SourceOrchestrator;
+//# sourceMappingURL=source-orchestrator.d.ts.map

package/dist/lib/source-orchestrator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"source-orchestrator.d.ts","sourceRoot":"","sources":["../../lib/source-orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,EAAsB,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAelF,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAWjC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwB7B,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,GAAG;IAC9D,cAAc,CAAC,EAAE;QAAE,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;KAAE,CAAC;CAC1E,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,UAAU,EAAE;QACV,YAAY,EAAE,KAAK,GAAG,SAAS,GAAG,MAAM,CAAC;QACzC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;QACnC,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC9B,eAAe,EAAE;YACf,SAAS,EAAE,MAAM,CAAC;YAClB,UAAU,EAAE,MAAM,CAAC;YACnB,MAAM,EAAE,MAAM,CAAC;YACf,QAAQ,EAAE,MAAM,CAAC;SAClB,CAAC;KACH,CAAC;IACF,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAuB;IAEvD;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAIlC;IAEF;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAG7B;IAEF;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAqC/B;;;OAGG;IACH,OAAO,CAAC,kCAAkC;IAmC1C;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IA4B/B;;;OAGG;YACW,sBAAsB;IAsCpC;;OAEG;IACG,aAAa,CACjB,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,YAAqC,GAC5C,OAAO,CAAC,mBAAmB,CAAC;IA0C/B;;OAEG;YACW,oBAAoB;IAqLlC;;OAEG;YACW,gBAAgB;IAsC9B;;OAEG;YACW,YAAY;IAmC1B;;OAEG;YACW,gBAAgB;IAuC9B;;OAEG;YACW,aAAa;IAoC3B;;;OAGG;YACW,mBAAmB;IAwCjC;;;;;;;;OAQG;IACH,OAAO,CAAC,gBAAgB;IAiDxB;;OAEG;IACH,OAAO,CAAC,cAAc;IAwBtB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA4B3B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAiB3B;;OAEG;IACH,OAAO,CAAC,UAAU;IAIlB;;OAEG;IACH,OAAO,CAAC,cAAc;IA0BtB;;OAEG;IACG,sBAAsB,CAC1B,QAAQ,EAAE,gBAAgB,EAAE,EAC5B,WAAW,GAAE,MAAW,EACxB,OAAO,GAAE;QACP,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KACpD,GACL,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAgD9B;;OAEG;IACH,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,GAAG,GAAG,YAAY;IAWhD;;OAEG;IACG,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QACzC,eAAe,EAAE,OAAO,CAAC;QACzB,UAAU,EAAE,OAAO,CAAC;QACpB,UAAU,EAAE,OAAO,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,qBAAqB,EAAE,MAAM,CAAC;KAC/B,CAAC;CA4BH;AAGD,eAAO,MAAM,wBAAwB,oBAA2B,CAAC"}