@tyroneross/blog-scraper 0.1.0 → 0.3.0

package/dist/lib/index.d.ts

@@ -0,0 +1,96 @@
+/**
+ * @package blog-content-scraper SDK
+ *
+ * Intelligent web scraper for extracting blog/news content from any website.
+ * Supports RSS feeds, sitemaps, and HTML scraping with automatic detection.
+ *
+ * @example
+ * ```typescript
+ * import { scrapeWebsite, createRateLimiter } from 'blog-content-scraper';
+ *
+ * // Simple usage
+ * const result = await scrapeWebsite('https://techcrunch.com');
+ * console.log(result.articles);
+ *
+ * // With options
+ * const result = await scrapeWebsite('https://example.com/blog', {
+ *   maxArticles: 10,
+ *   extractFullContent: true,
+ *   qualityThreshold: 0.5
+ * });
+ * ```
+ */
+export { globalSourceOrchestrator, SourceOrchestrator, CandidateArticleSchema, SourceConfigSchema } from './source-orchestrator';
+export type { CandidateArticle, SourceConfig, OrchestrationResult } from './source-orchestrator';
+export { ScrapingRateLimiter, createRateLimiter, globalRateLimiter, RATE_LIMITER_PRESETS, type RateLimiterConfig, type RateLimiterPreset } from './scraping-rate-limiter';
+export { calculateArticleQualityScore, DEFAULT_QUALITY_CONFIG, DEFAULT_DENY_PATHS, DEFAULT_ALLOW_PATHS, isNonEnglishLocalePath } from './quality-scorer';
+export { CircuitBreaker, circuitBreakers } from './circuit-breaker';
+export type { ScrapedArticle, ScraperTestResult, ScraperTestRequest, ProgressState, ProgressStage, QualityScoreConfig, ContentValidation, ExtractedContent, ScraperPlugin } from './types';
+export { convertToMarkdown } from './formatters/html-to-markdown';
+export { cleanText, stripHTML } from './formatters/text-cleaner';
+export { globalRSSDiscovery, type DiscoveredFeed } from './web-scrapers/rss-discovery';
+export { globalSitemapParser, type SitemapEntry } from './web-scrapers/sitemap-parser';
+export { HTMLScraper, type ExtractedArticle, type ScrapingConfig } from './web-scrapers/html-scraper';
+export { ContentExtractor } from './web-scrapers/content-extractor';
+/**
+ * Simplified scraping function for common use cases
+ *
+ * @param url - Website URL to scrape
+ * @param options - Scraping options
+ * @returns Promise with scraped articles and metadata
+ *
+ * @example
+ * ```typescript
+ * const result = await scrapeWebsite('https://techcrunch.com', {
+ *   maxArticles: 5,
+ *   extractFullContent: true
+ * });
+ *
+ * for (const article of result.articles) {
+ *   console.log(article.title, article.url);
+ * }
+ * ```
+ */
+export declare function scrapeWebsite(url: string, options?: {
+    /** Maximum articles to return (default: 10) */
+    maxArticles?: number;
+    /** Extract full article content (default: true) */
+    extractFullContent?: boolean;
+    /** Minimum quality score 0-1 (default: 0.5) */
+    qualityThreshold?: number;
+    /** Source type: 'auto' | 'rss' | 'sitemap' | 'html' (default: 'auto') */
+    sourceType?: 'auto' | 'rss' | 'sitemap' | 'html';
+    /** URL patterns to allow (e.g., ['/blog/*', '/news/*']) */
+    allowPaths?: string[];
+    /** URL patterns to deny (e.g., ['/about', '/careers/*']) */
+    denyPaths?: string[];
+    /** Progress callback for long-running operations */
+    onProgress?: (completed: number, total: number) => void;
+    /** Abort signal for cancellation */
+    signal?: AbortSignal;
+}): Promise<{
+    url: string;
+    detectedType: "rss" | "sitemap" | "html";
+    articles: {
+        url: string;
+        title: string;
+        publishedDate: string;
+        description: string | undefined;
+        fullContent: string | undefined;
+        fullContentMarkdown: string | undefined;
+        fullContentText: string | undefined;
+        confidence: number;
+        source: "rss" | "sitemap" | "html" | "discovery";
+        qualityScore: number;
+        metadata: Record<string, any> | undefined;
+    }[];
+    stats: {
+        totalDiscovered: number;
+        afterQualityFilter: number;
+        processingTime: number;
+    };
+    discoveredFeeds: import("./web-scrapers/rss-discovery").DiscoveredFeed[] | undefined;
+    discoveredSitemaps: string[] | undefined;
+    errors: string[];
+}>;
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../lib/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAGH,OAAO,EACL,wBAAwB,EACxB,kBAAkB,EAClB,sBAAsB,EACtB,kBAAkB,EACnB,MAAM,uBAAuB,CAAC;AAE/B,YAAY,EACV,gBAAgB,EAChB,YAAY,EACZ,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,EACpB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACvB,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EACL,4BAA4B,EAC5B,sBAAsB,EACtB,kBAAkB,EAClB,mBAAmB,EACnB,sBAAsB,EACvB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EACL,cAAc,EACd,eAAe,EAChB,MAAM,mBAAmB,CAAC;AAG3B,YAAY,EACV,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,iBAAiB,EACjB,gBAAgB,EAChB,aAAa,EACd,MAAM,SAAS,CAAC;AAGjB,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAGjE,OAAO,EAAE,kBAAkB,EAAE,KAAK,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACvF,OAAO,EAAE,mBAAmB,EAAE,KAAK,YAAY,EAAE,MAAM,+BAA+B,CAAC;AACvF,OAAO,EAAE,WAAW,EAAE,KAAK,gBAAgB,EAAE,KAAK,cAAc,EAAE,MAAM,6BAA6B,CAAC;AACtG,OAAO,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AAEpE;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,+CAA+C;IAC/C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,+CAA+C;IAC/C,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS,GAAG,MAAM,CAAC;IACjD,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACxD,oCAAoC;IACpC,MAAM,CAAC,EAAE,WAAW,CAAC;CACjB;;;;;;;;;;;;;;;;;;;;;;;;GA2FP"}

package/dist/lib/index.js

@@ -0,0 +1,184 @@
+"use strict";
+/**
+ * @package blog-content-scraper SDK
+ *
+ * Intelligent web scraper for extracting blog/news content from any website.
+ * Supports RSS feeds, sitemaps, and HTML scraping with automatic detection.
+ *
+ * @example
+ * ```typescript
+ * import { scrapeWebsite, createRateLimiter } from 'blog-content-scraper';
+ *
+ * // Simple usage
+ * const result = await scrapeWebsite('https://techcrunch.com');
+ * console.log(result.articles);
+ *
+ * // With options
+ * const result = await scrapeWebsite('https://example.com/blog', {
+ *   maxArticles: 10,
+ *   extractFullContent: true,
+ *   qualityThreshold: 0.5
+ * });
+ * ```
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ContentExtractor = exports.HTMLScraper = exports.globalSitemapParser = exports.globalRSSDiscovery = exports.stripHTML = exports.cleanText = exports.convertToMarkdown = exports.circuitBreakers = exports.CircuitBreaker = exports.isNonEnglishLocalePath = exports.DEFAULT_ALLOW_PATHS = exports.DEFAULT_DENY_PATHS = exports.DEFAULT_QUALITY_CONFIG = exports.calculateArticleQualityScore = exports.RATE_LIMITER_PRESETS = exports.globalRateLimiter = exports.createRateLimiter = exports.ScrapingRateLimiter = exports.SourceConfigSchema = exports.CandidateArticleSchema = exports.SourceOrchestrator = exports.globalSourceOrchestrator = void 0;
+exports.scrapeWebsite = scrapeWebsite;
+// Main orchestrator
+var source_orchestrator_1 = require("./source-orchestrator");
+Object.defineProperty(exports, "globalSourceOrchestrator", { enumerable: true, get: function () { return source_orchestrator_1.globalSourceOrchestrator; } });
+Object.defineProperty(exports, "SourceOrchestrator", { enumerable: true, get: function () { return source_orchestrator_1.SourceOrchestrator; } });
+Object.defineProperty(exports, "CandidateArticleSchema", { enumerable: true, get: function () { return source_orchestrator_1.CandidateArticleSchema; } });
+Object.defineProperty(exports, "SourceConfigSchema", { enumerable: true, get: function () { return source_orchestrator_1.SourceConfigSchema; } });
+// Rate limiter with presets
+var scraping_rate_limiter_1 = require("./scraping-rate-limiter");
+Object.defineProperty(exports, "ScrapingRateLimiter", { enumerable: true, get: function () { return scraping_rate_limiter_1.ScrapingRateLimiter; } });
+Object.defineProperty(exports, "createRateLimiter", { enumerable: true, get: function () { return scraping_rate_limiter_1.createRateLimiter; } });
+Object.defineProperty(exports, "globalRateLimiter", { enumerable: true, get: function () { return scraping_rate_limiter_1.globalRateLimiter; } });
+Object.defineProperty(exports, "RATE_LIMITER_PRESETS", { enumerable: true, get: function () { return scraping_rate_limiter_1.RATE_LIMITER_PRESETS; } });
+// Quality scoring
+var quality_scorer_1 = require("./quality-scorer");
+Object.defineProperty(exports, "calculateArticleQualityScore", { enumerable: true, get: function () { return quality_scorer_1.calculateArticleQualityScore; } });
+Object.defineProperty(exports, "DEFAULT_QUALITY_CONFIG", { enumerable: true, get: function () { return quality_scorer_1.DEFAULT_QUALITY_CONFIG; } });
+Object.defineProperty(exports, "DEFAULT_DENY_PATHS", { enumerable: true, get: function () { return quality_scorer_1.DEFAULT_DENY_PATHS; } });
+Object.defineProperty(exports, "DEFAULT_ALLOW_PATHS", { enumerable: true, get: function () { return quality_scorer_1.DEFAULT_ALLOW_PATHS; } });
+Object.defineProperty(exports, "isNonEnglishLocalePath", { enumerable: true, get: function () { return quality_scorer_1.isNonEnglishLocalePath; } });
+// Circuit breaker for resilience
+var circuit_breaker_1 = require("./circuit-breaker");
+Object.defineProperty(exports, "CircuitBreaker", { enumerable: true, get: function () { return circuit_breaker_1.CircuitBreaker; } });
+Object.defineProperty(exports, "circuitBreakers", { enumerable: true, get: function () { return circuit_breaker_1.circuitBreakers; } });
+// Formatters
+var html_to_markdown_1 = require("./formatters/html-to-markdown");
+Object.defineProperty(exports, "convertToMarkdown", { enumerable: true, get: function () { return html_to_markdown_1.convertToMarkdown; } });
+var text_cleaner_1 = require("./formatters/text-cleaner");
+Object.defineProperty(exports, "cleanText", { enumerable: true, get: function () { return text_cleaner_1.cleanText; } });
+Object.defineProperty(exports, "stripHTML", { enumerable: true, get: function () { return text_cleaner_1.stripHTML; } });
+// Re-export scraper components for advanced usage
+var rss_discovery_1 = require("./web-scrapers/rss-discovery");
+Object.defineProperty(exports, "globalRSSDiscovery", { enumerable: true, get: function () { return rss_discovery_1.globalRSSDiscovery; } });
+var sitemap_parser_1 = require("./web-scrapers/sitemap-parser");
+Object.defineProperty(exports, "globalSitemapParser", { enumerable: true, get: function () { return sitemap_parser_1.globalSitemapParser; } });
+var html_scraper_1 = require("./web-scrapers/html-scraper");
+Object.defineProperty(exports, "HTMLScraper", { enumerable: true, get: function () { return html_scraper_1.HTMLScraper; } });
+var content_extractor_1 = require("./web-scrapers/content-extractor");
+Object.defineProperty(exports, "ContentExtractor", { enumerable: true, get: function () { return content_extractor_1.ContentExtractor; } });
+/**
+ * Simplified scraping function for common use cases
+ *
+ * @param url - Website URL to scrape
+ * @param options - Scraping options
+ * @returns Promise with scraped articles and metadata
+ *
+ * @example
+ * ```typescript
+ * const result = await scrapeWebsite('https://techcrunch.com', {
+ *   maxArticles: 5,
+ *   extractFullContent: true
+ * });
+ *
+ * for (const article of result.articles) {
+ *   console.log(article.title, article.url);
+ * }
+ * ```
+ */
+async function scrapeWebsite(url, options = {}) {
+    const { globalSourceOrchestrator } = await Promise.resolve().then(() => __importStar(require('./source-orchestrator')));
+    const { calculateArticleQualityScore, DEFAULT_DENY_PATHS } = await Promise.resolve().then(() => __importStar(require('./quality-scorer')));
+    const { convertToMarkdown } = await Promise.resolve().then(() => __importStar(require('./formatters/html-to-markdown')));
+    const { cleanText, stripHTML } = await Promise.resolve().then(() => __importStar(require('./formatters/text-cleaner')));
+    const { maxArticles = 10, extractFullContent = true, qualityThreshold = 0.5, sourceType = 'auto', allowPaths = [], denyPaths = DEFAULT_DENY_PATHS, onProgress, signal } = options;
+    // Check for cancellation
+    if (signal?.aborted) {
+        throw new Error('Operation cancelled');
+    }
+    // Process source
+    const result = await globalSourceOrchestrator.processSource(url, {
+        sourceType,
+        allowPaths,
+        denyPaths,
+        detectOnly: false
+    });
+    // Check for cancellation
+    if (signal?.aborted) {
+        throw new Error('Operation cancelled');
+    }
+    // Enhance with full content if requested
+    let articles = result.articles.slice(0, maxArticles);
+    if (extractFullContent && articles.length > 0) {
+        articles = await globalSourceOrchestrator.enhanceWithFullContent(articles, maxArticles, { onProgress });
+    }
+    // Calculate quality scores and format output
+    const scoredArticles = articles.map(article => {
+        const extracted = {
+            title: article.title,
+            excerpt: article.excerpt,
+            content: article.content,
+            textContent: article.content || '',
+            publishedTime: article.publishedAt.toISOString()
+        };
+        const qualityScore = calculateArticleQualityScore(extracted);
+        const fullContent = extractFullContent ? article.content : undefined;
+        return {
+            url: article.url,
+            title: article.title,
+            publishedDate: article.publishedAt.toISOString(),
+            description: article.excerpt,
+            fullContent,
+            fullContentMarkdown: fullContent ? convertToMarkdown(fullContent) : undefined,
+            fullContentText: fullContent ? cleanText(stripHTML(fullContent)) : undefined,
+            confidence: article.confidence,
+            source: article.source,
+            qualityScore,
+            metadata: article.metadata
+        };
+    });
+    // Filter by quality threshold
+    const filteredArticles = scoredArticles.filter(a => a.qualityScore >= qualityThreshold);
+    return {
+        url,
+        detectedType: result.sourceInfo.detectedType,
+        articles: filteredArticles,
+        stats: {
+            totalDiscovered: result.articles.length,
+            afterQualityFilter: filteredArticles.length,
+            processingTime: result.processingTime
+        },
+        discoveredFeeds: result.sourceInfo.discoveredFeeds,
+        discoveredSitemaps: result.sourceInfo.discoveredSitemaps,
+        errors: result.errors
+    };
+}
+//# sourceMappingURL=index.js.map

package/dist/lib/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmFH,sCA8GC;AA/LD,oBAAoB;AACpB,6DAK+B;AAJ7B,+HAAA,wBAAwB,OAAA;AACxB,yHAAA,kBAAkB,OAAA;AAClB,6HAAA,sBAAsB,OAAA;AACtB,yHAAA,kBAAkB,OAAA;AASpB,4BAA4B;AAC5B,iEAOiC;AAN/B,4HAAA,mBAAmB,OAAA;AACnB,0HAAA,iBAAiB,OAAA;AACjB,0HAAA,iBAAiB,OAAA;AACjB,6HAAA,oBAAoB,OAAA;AAKtB,kBAAkB;AAClB,mDAM0B;AALxB,8HAAA,4BAA4B,OAAA;AAC5B,wHAAA,sBAAsB,OAAA;AACtB,oHAAA,kBAAkB,OAAA;AAClB,qHAAA,mBAAmB,OAAA;AACnB,wHAAA,sBAAsB,OAAA;AAGxB,iCAAiC;AACjC,qDAG2B;AAFzB,iHAAA,cAAc,OAAA;AACd,kHAAA,eAAe,OAAA;AAgBjB,aAAa;AACb,kEAAkE;AAAzD,qHAAA,iBAAiB,OAAA;AAC1B,0DAAiE;AAAxD,yGAAA,SAAS,OAAA;AAAE,yGAAA,SAAS,OAAA;AAE7B,kDAAkD;AAClD,8DAAuF;AAA9E,mHAAA,kBAAkB,OAAA;AAC3B,gEAAuF;AAA9E,qHAAA,mBAAmB,OAAA;AAC5B,4DAAsG;AAA7F,2GAAA,WAAW,OAAA;AACpB,sEAAoE;AAA3D,qHAAA,gBAAgB,OAAA;AAEzB;;;;;;;;;;;;;;;;;;GAkBG;AACI,KAAK,UAAU,aAAa,CACjC,GAAW,EACX,UAiBI,EAAE;IAEN,MAAM,EAAE,wBAAwB,EAAE,GAAG,wDAAa,uBAAuB,GAAC,CAAC;IAC3E,MAAM,EAAE,4BAA4B,EAAE,kBAAkB,EAAE,GAAG,wDAAa,kBAAkB,GAAC,CAAC;IAC9F,MAAM,EAAE,iBAAiB,EAAE,GAAG,wDAAa,+BAA+B,GAAC,CAAC;IAC5E,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,wDAAa,2BAA2B,GAAC,CAAC;IAE3E,MAAM,EACJ,WAAW,GAAG,EAAE,EAChB,kBAAkB,GAAG,IAAI,EACzB,gBAAgB,GAAG,GAAG,EACtB,UAAU,GAAG,MAAM,EACnB,UAAU,GAAG,EAAE,EACf,SAAS,GAAG,kBAAkB,EAC9B,UAAU,EACV,MAAM,EACP,GAAG,OAAO,CAAC;IAEZ,yBAAyB;IACzB,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IAED,iBAAiB;IACjB,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,aAAa,CAAC,GAAG,EAAE;QAC/D,UAAU;QACV,UAAU;QACV,SAAS;QACT,UAAU,EAAE,KAAK;KAClB,CAAC,CAAC;IAEH,yBAAyB;IACzB,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IAED,yCAAyC;IACzC,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IAErD,IAAI,kBAAkB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,QAAQ,GAAG,MAAM,wBAAwB,CAAC,sBAAsB,CAC9D,QAAQ,EACR,WAAW,EACX,EAAE,UAAU,EAAE,CACf,CAAC;IACJ,CAAC;IAED,6CAA6C;IAC7C,MAAM,cAAc,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;QAC5C,MAAM,SAAS,GAAG;YAChB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,WAAW,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;YAClC,aAAa,EAAE,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE;SACjD,CAAC;QAEF,MAAM,YAAY,GAAG,4BAA4B,CAAC,SAAS,CAAC,CAAC;QAC7D,MAAM,WAAW,GAAG,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAErE,OAAO;YACL,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,aAAa,EAAE,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE;YAChD,WAAW,EAAE,OAAO,CAAC,OAAO;YAC5B,WAAW;YACX,mBAAmB,EAAE,WAAW,CAAC,CAAC,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS;YAC7E,eAAe,EAAE,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;YAC5E,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,YAAY;YACZ,QAAQ,EAAE,OAAO,CAAC,QAAQ;SAC3B,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,8BAA8B;IAC9B,MAAM,gBAAgB,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,gBAAgB,CAAC,CAAC;IAExF,OAAO;QACL,GAAG;QACH,YAAY,EAAE,MAAM,CAAC,UAAU,CAAC,YAAY;QAC5C,QAAQ,EAAE,gBAAgB;QAC1B,KAAK,EAAE;YACL,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;YACvC,kBAAkB,EAAE,gBAAgB,CAAC,MAAM;YAC3C,cAAc,EAAE,MAAM,CAAC,cAAc;SACtC;QACD,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC,eAAe;QAClD,kBAAkB,EAAE,MAAM,CAAC,UAAU,CAAC,kBAAkB;QACxD,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC;AACJ,CAAC"}

package/dist/lib/quality-scorer.d.ts

@@ -0,0 +1,83 @@
+/**
+ * @package @tyroneross/scraper-testing
+ * Article quality scoring system
+ *
+ * No LLM required - uses metadata and content signals to determine article quality
+ */
+import { ExtractedContent, QualityScoreConfig, ContentValidation } from './types';
+/**
+ * Default quality score configuration
+ * These weights were optimized through testing with 1,788 real articles
+ */
+export declare const DEFAULT_QUALITY_CONFIG: Required<QualityScoreConfig>;
+/**
+ * Default patterns to block non-article pages
+ * These cover common non-article paths across websites
+ */
+export declare const DEFAULT_DENY_PATHS: string[];
+/**
+ * Default patterns for content sections (blog, news, articles)
+ * Used for allow-listing paths when scraping
+ */
+export declare const DEFAULT_ALLOW_PATHS: string[];
+/**
+ * Validate content quality (Tier 2 filtering)
+ * Checks length, title quality, and text-to-HTML ratio
+ *
+ * @param extracted - Extracted content from article
+ * @returns Validation result with score and reasons
+ */
+export declare function validateContent(extracted: ExtractedContent): ContentValidation;
+/**
+ * Calculate article quality score (Tier 3 filtering)
+ *
+ * Score breakdown:
+ * - Content validation (60%): Length, title quality, text-to-HTML ratio
+ * - Publication date (12%): Articles should have timestamps
+ * - Author/byline (8%): Professional articles cite authors
+ * - Schema.org metadata (8%): Structured data indicates article pages
+ * - Reading time (12%): Substantial content (2+ min read)
+ *
+ * @param extracted - Extracted content from article
+ * @param config - Optional quality score configuration
+ * @returns Quality score between 0-1
+ */
+export declare function calculateArticleQualityScore(extracted: ExtractedContent, config?: QualityScoreConfig): number;
+/**
+ * Check if a path should be filtered out (non-US-English locale)
+ *
+ * Returns true (should filter) for:
+ * - /fr-be/, /de-de/, /ja-jp/, /zh-cn/ (non-English locales)
+ * - /en-gb/, /en-au/, /en-ca/ (non-US English locales)
+ *
+ * Returns false (should keep) for:
+ * - /en-us/ (US English only)
+ * - /blog/, /news/, /articles/ (no locale prefix - default to US English)
+ */
+export declare function isNonEnglishLocalePath(path: string): boolean;
+/**
+ * Check if a URL should be denied based on path patterns
+ *
+ * @param url - URL to check
+ * @param denyPaths - Patterns to deny (supports wildcards with *)
+ * @returns True if URL should be denied
+ */
+export declare function shouldDenyUrl(url: string, denyPaths?: string[]): boolean;
+/**
+ * Get quality score breakdown for debugging
+ * Useful for understanding why an article scored a certain way
+ *
+ * @param extracted - Extracted content from article
+ * @param config - Optional quality score configuration
+ * @returns Breakdown of quality score components
+ */
+export declare function getQualityBreakdown(extracted: ExtractedContent, config?: QualityScoreConfig): {
+    contentValidation: number;
+    publishedDate: number;
+    author: number;
+    schema: number;
+    readingTime: number;
+    total: number;
+    passesThreshold: boolean;
+};
+//# sourceMappingURL=quality-scorer.d.ts.map

package/dist/lib/quality-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"quality-scorer.d.ts","sourceRoot":"","sources":["../../lib/quality-scorer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAElF;;;GAGG;AACH,eAAO,MAAM,sBAAsB,EAAE,QAAQ,CAAC,kBAAkB,CAO/D,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,kBAAkB,UA+H9B,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,mBAAmB,UAc/B,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,gBAAgB,GAAG,iBAAiB,CAsC9E;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,gBAAgB,EAC3B,MAAM,GAAE,kBAAuB,GAC9B,MAAM,CAiDR;AAYD;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAQ5D;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,GAAE,MAAM,EAAuB,GAAG,OAAO,CAyB5F;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,gBAAgB,EAC3B,MAAM,GAAE,kBAAuB,GAC9B;IACD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,OAAO,CAAC;CAC1B,CA8CA"}