@tyroneross/blog-scraper 0.1.0 → 0.3.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Tyrone Ross
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,370 +1,345 @@
-# @tyroneross/blog-scraper
+# Blog Content Scraper
 
-> Powerful web scraping SDK for extracting blog articles and content. No LLM required.
+Intelligent web scraper for extracting blog/news content from any website. Includes both a **web UI** for testing and a **programmatic SDK** for integration.
 
-[![npm version](https://img.shields.io/npm/v/@tyroneross/blog-scraper.svg)](https://www.npmjs.com/package/@tyroneross/blog-scraper)
-[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+## Quick Start (SDK)
 
-## Features
-
-✨ **No LLM needed** - Uses Mozilla Readability (92.2% F1 score) for content extraction
-🎯 **3-tier filtering** - URL patterns → content validation → quality scoring
-⚡ **Fast** - Extracts articles in 2-5 seconds
-🔧 **Modular** - Use high-level API or individual components
-📦 **Zero config** - Works out of the box
-🌐 **Multi-source** - RSS feeds, sitemaps, and HTML pages
+```typescript
+import { scrapeWebsite } from './lib';
 
-## Installation
+const result = await scrapeWebsite('https://techcrunch.com', {
+  maxArticles: 5,
+  extractFullContent: true
+});
 
-```bash
-npm install @tyroneross/blog-scraper
+for (const article of result.articles) {
+  console.log(article.title, article.qualityScore);
+}
 ```
 
-## Quick Start
-
-```typescript
-import { scrape } from '@tyroneross/blog-scraper';
+See [SDK Documentation](#sdk-documentation) below for full API reference.
 
-// Simple usage - scrape a blog
-const result = await scrape('https://example.com/blog');
+---
 
-console.log(`Found ${result.articles.length} articles`);
-console.log(`Processing time: ${result.processingTime}ms`);
+## Web UI
 
-// Access articles
-result.articles.forEach(article => {
-  console.log(article.title);
-  console.log(article.url);
-  console.log(article.fullContentMarkdown); // Markdown format
-  console.log(article.qualityScore); // 0-1 quality score
-});
-```
+Standalone web application for testing web scraping with intelligent content filtering. Built with Next.js, Mozilla Readability, and zero LLM dependencies.
 
-## API Reference
+## Features
 
-### High-Level API (Recommended)
+- ✅ **No configuration needed** - Works immediately
+- 🎯 **3-tier filtering** - URL patterns → content validation → quality scoring
+- ⚡ **Fast** - Mozilla Readability (92.2% F1 score)
+- 📊 **Detailed stats** - See filtering pipeline in action
+- 🎨 **Clean UI** - Built with Tailwind CSS
+- 🚀 **Deploy anywhere** - Vercel, Netlify, Docker, etc.
 
-#### `scrape(url, options?)`
+## Quick Start
 
-Extract articles from a URL with automatic source detection.
+### Local Development
 
-```typescript
-import { scrape } from '@tyroneross/blog-scraper';
-
-const result = await scrape('https://example.com', {
-  // Optional configuration
-  sourceType: 'auto',           // 'auto' | 'rss' | 'sitemap' | 'html'
-  maxArticles: 50,              // Maximum articles to return
-  extractFullContent: true,     // Extract full article content
-  denyPaths: ['/about', '/contact'], // URL patterns to exclude
-  qualityThreshold: 0.6         // Minimum quality score (0-1)
-});
+1. **Install dependencies:**
+```bash
+npm install
 ```
 
-**Returns:**
-```typescript
-{
-  url: string;
-  detectedType: 'rss' | 'sitemap' | 'html';
-  confidence: 'high' | 'medium' | 'low';
-  articles: ScrapedArticle[];
-  extractionStats: {
-    attempted: number;
-    successful: number;
-    failed: number;
-    filtered: number;
-    totalDiscovered: number;
-    afterDenyFilter: number;
-    afterContentValidation: number;
-    afterQualityFilter: number;
-  };
-  processingTime: number;
-  errors: string[];
-  timestamp: string;
-}
+2. **Run dev server:**
+```bash
+npm run dev
 ```
 
-#### `quickScrape(url)`
+3. **Open browser:**
+```
+http://localhost:3000
+```
 
-Fast URL-only extraction (no full content).
+## Deployment
 
-```typescript
-import { quickScrape } from '@tyroneross/blog-scraper';
+### Vercel (Recommended)
 
-const urls = await quickScrape('https://example.com/blog');
-// Returns: ['url1', 'url2', 'url3', ...]
+1. **Install Vercel CLI:**
+```bash
+npm install -g vercel
 ```
 
-### Modular API (Advanced)
-
-Use individual components for granular control.
-
-#### Content Extraction
+2. **Deploy:**
+```bash
+vercel
+```
 
-```typescript
-import { ContentExtractor } from '@tyroneross/blog-scraper';
+3. **Production deploy:**
+```bash
+vercel --prod
+```
 
-const extractor = new ContentExtractor();
-const content = await extractor.extractContent('https://example.com/article');
+### Netlify
 
-console.log(content.title);
-console.log(content.textContent);
-console.log(content.wordCount);
-console.log(content.readingTime);
+1. **Build command:**
+```
+npm run build
 ```
 
-#### Quality Scoring
+2. **Publish directory:**
+```
+.next
+```
 
-```typescript
-import { calculateArticleQualityScore, getQualityBreakdown } from '@tyroneross/blog-scraper';
-
-const score = calculateArticleQualityScore(extractedContent);
-console.log(`Quality score: ${score}`); // 0-1
-
-// Get detailed breakdown
-const breakdown = getQualityBreakdown(extractedContent);
-console.log(breakdown);
-// {
-//   contentValidation: 0.6,
-//   publishedDate: 0.12,
-//   author: 0.08,
-//   schema: 0.08,
-//   readingTime: 0.12,
-//   total: 1.0,
-//   passesThreshold: true
-// }
+3. **Deploy:**
+```bash
+netlify deploy --prod
 ```
 
-#### Custom Quality Configuration
+### Docker
+
+```dockerfile
+FROM node:18-alpine
+WORKDIR /app
+COPY package*.json ./
+RUN npm install
+COPY . .
+RUN npm run build
+EXPOSE 3000
+CMD ["npm", "start"]
+```
 
-```typescript
-import { calculateArticleQualityScore } from '@tyroneross/blog-scraper';
-
-const score = calculateArticleQualityScore(content, {
-  contentWeight: 0.8,        // Increase content importance
-  dateWeight: 0.05,          // Decrease date importance
-  authorWeight: 0.05,
-  schemaWeight: 0.05,
-  readingTimeWeight: 0.05,
-  threshold: 0.7             // Stricter threshold
-});
+```bash
+docker build -t scraper-app .
+docker run -p 3000:3000 scraper-app
 ```
 
-#### RSS Discovery
+## How It Works
 
-```typescript
-import { RSSDiscovery } from '@tyroneross/blog-scraper';
+### 3-Tier Filtering System
+
+**Tier 1: URL Deny Patterns**
+- Blocks /, /about, /careers, /contact, /tag/*, etc.
+- Fast, pattern-based filtering
 
-const discovery = new RSSDiscovery();
-const feeds = await discovery.discoverFeeds('https://example.com');
+**Tier 2: Content Validation**
+- Minimum 200 characters
+- Title length 10-200 characters
+- Text-to-HTML ratio ≥ 10%
 
-feeds.forEach(feed => {
-  console.log(feed.url);
-  console.log(feed.title);
-  console.log(feed.confidence); // 0-1
-});
-```
+**Tier 3: Metadata Scoring**
+- Content quality: 60% weight
+- Publication date: 12% weight
+- Author/byline: 8% weight
+- Schema.org metadata: 8% weight
+- Reading time (2+ min): 12% weight
+- **Default threshold**: 50%
 
-#### Sitemap Parsing
+### Technology Stack
 
-```typescript
-import { SitemapParser } from '@tyroneross/blog-scraper';
+- **Next.js 15** - React framework
+- **TypeScript** - Type safety
+- **Tailwind CSS** - Styling
+- **Mozilla Readability** - Content extraction
+- **JSDOM** - HTML parsing
+- **Zod** - Schema validation
+- **Lucide React** - Icons
 
-const parser = new SitemapParser();
-const entries = await parser.parseSitemap('https://example.com/sitemap.xml');
+## Project Structure
 
-entries.forEach(entry => {
-  console.log(entry.url);
-  console.log(entry.lastmod);
-  console.log(entry.priority);
-});
+```
+scraper-app/
+├── app/
+│   ├── api/scraper-test/      # API route
+│   │   └── route.ts
+│   ├── layout.tsx              # Root layout
+│   ├── page.tsx                # Homepage
+│   └── globals.css             # Global styles
+├── components/
+│   ├── ScraperTester.tsx       # Main UI component
+│   └── ScraperResults.tsx      # Results display
+├── lib/
+│   ├── types.ts                # TypeScript types
+│   ├── quality-scorer.ts       # Quality scoring logic
+│   └── content-extractor.ts    # Content extraction
+├── public/                     # Static assets
+├── package.json
+├── tsconfig.json
+├── tailwind.config.ts
+└── next.config.js
 ```
 
-#### HTML Scraping
+## Environment Variables
 
-```typescript
-import { HTMLScraper } from '@tyroneross/blog-scraper';
-
-const scraper = new HTMLScraper();
-const articles = await scraper.extractFromPage('https://example.com/blog', {
-  selectors: {
-    articleLinks: ['article a', '.post-link'],
-    titleSelectors: ['h1', '.post-title'],
-    dateSelectors: ['time', '.published-date']
-  },
-  filters: {
-    minTitleLength: 10,
-    maxTitleLength: 200
-  }
-});
-```
+No environment variables required! The app works out of the box.
 
-#### Rate Limiting
+## Performance
 
-```typescript
-import { ScrapingRateLimiter } from '@tyroneross/blog-scraper';
+- **Single article:** ~2-5 seconds
+- **Bundle size:** ~150 KB (gzipped)
+- **Zero API costs:** No external APIs used
+- **Memory:** ~100 MB average
 
-// Create custom rate limiter
-const limiter = new ScrapingRateLimiter({
-  maxConcurrent: 5,
-  minTime: 1000  // 1 second between requests
-});
+## Testing
 
-// Use in your scraping logic
-await limiter.execute('example.com', async () => {
-  // Your scraping code here
-});
-```
+### F1 Score Validation
 
-#### Circuit Breaker
+The **92.2% F1 score** claim for Mozilla Readability is validated through automated testing using two approaches:
 
-```typescript
-import { CircuitBreaker } from '@tyroneross/blog-scraper';
+#### 1. Dragnet Benchmark Dataset (Recommended)
 
-const breaker = new CircuitBreaker('my-operation', {
-  failureThreshold: 5,
-  resetTimeout: 60000  // 1 minute
-});
+Uses the established [Dragnet benchmark dataset](https://github.com/seomoz/dragnet_data) - a well-documented, peer-reviewed dataset used in academic research:
 
-const result = await breaker.execute(async () => {
-  // Your operation here
-});
+```bash
+npm run test:f1:dragnet
 ```
 
-## Examples
+**Results: 91.4% F1 score** (0.8% from claimed 92.2%)
+- 📊 Dataset: 414 test articles (20 tested for efficiency)
+- 📚 Source: Published research (2013)
+- ✅ 100% extraction success rate
+- 📈 92.6% Precision, 92.3% Recall
 
-### Example 1: Scrape with Custom Deny Patterns
+#### 2. Custom Test Dataset
 
-```typescript
-import { scrape } from '@tyroneross/blog-scraper';
-
-const result = await scrape('https://techcrunch.com', {
-  denyPaths: [
-    '/',
-    '/about',
-    '/contact',
-    '/tag/*',      // Exclude all tag pages
-    '/author/*'    // Exclude all author pages
-  ],
-  maxArticles: 20
-});
+Quick validation with curated test articles:
+
+```bash
+npm run test:f1
 ```
 
-### Example 2: Build Custom Pipeline
+**Results: 96.3% F1 score**
+- 3 manually-labeled test articles
+- Useful for quick validation and development
 
-```typescript
-import {
-  SourceOrchestrator,
-  ContentExtractor,
-  calculateArticleQualityScore
-} from '@tyroneross/blog-scraper';
-
-// Step 1: Discover articles
-const orchestrator = new SourceOrchestrator();
-const discovered = await orchestrator.processSource('https://example.com', {
-  sourceType: 'auto'
-});
+---
 
-// Step 2: Extract content
-const extractor = new ContentExtractor();
-const extracted = await Promise.all(
-  discovered.articles
-    .slice(0, 10)
-    .map(a => extractor.extractContent(a.url))
-);
+**What is F1 Score?**
+- **Precision**: % of extracted content that is actually article content (not ads/navigation)
+- **Recall**: % of actual article content that was successfully extracted
+- **F1 Score**: Harmonic mean of precision and recall
 
-// Step 3: Score and filter
-const scored = extracted
-  .filter(Boolean)
-  .map(content => ({
-    content,
-    score: calculateArticleQualityScore(content!)
-  }))
-  .filter(item => item.score >= 0.7);
+**Conclusion:** The 92.2% F1 claim is **validated** using the established Dragnet benchmark dataset (91.4% achieved).
 
-console.log(`Found ${scored.length} high-quality articles`);
-```
+See [tests/README.md](./tests/README.md) for detailed testing documentation and how to add new test cases.
 
-### Example 3: RSS-Only Scraping
+## License
 
-```typescript
-import { scrape } from '@tyroneross/blog-scraper';
+MIT
 
-const result = await scrape('https://example.com', {
-  sourceType: 'rss',           // Only use RSS feeds
-  extractFullContent: false,   // Don't extract full content
-  maxArticles: 100
-});
-```
+## Contributing
 
-## How It Works
+Contributions welcome! Areas for improvement:
+- RSS/Sitemap discovery
+- Batch URL processing
+- Export functionality (CSV, JSON)
+- Custom quality scoring
+- Dark mode
 
-### 3-Tier Filtering System
+## Support
 
-**Tier 1: URL Deny Patterns**
-- Fast pattern-based filtering
-- Excludes non-article pages (/, /about, /tag/*, etc.)
-- Customizable patterns
+- Issues: https://github.com/tyroneross/scraper-app/issues
+- Questions: Open a discussion
 
-**Tier 2: Content Validation**
-- Minimum 200 characters
-- Title length 10-200 characters
-- Text-to-HTML ratio ≥ 10%
+---
 
-**Tier 3: Quality Scoring**
-- Content quality: 60% weight
-- Publication date: 12% weight
-- Author/byline: 8% weight
-- Schema.org metadata: 8% weight
-- Reading time: 12% weight
-- Default threshold: 50%
+## SDK Documentation
 
-### Auto-Detection Flow
+The SDK provides programmatic access to the scraping engine without the web UI.
 
-1. Try RSS feed (highest confidence)
-2. Discover RSS feeds from HTML
-3. Try sitemap parsing
-4. Discover sitemaps from domain
-5. Fall back to HTML link extraction
+### Installation
 
-## TypeScript Support
+```bash
+npm install
+```
 
-Full TypeScript support with exported types:
+### Basic Usage
 
 ```typescript
-import type {
-  ScrapedArticle,
-  ScraperTestResult,
-  ScrapeOptions,
-  ExtractedContent,
-  QualityScoreConfig
-} from '@tyroneross/blog-scraper';
+import { scrapeWebsite } from './lib';
+
+const result = await scrapeWebsite('https://example.com/blog', {
+  maxArticles: 10,           // Max articles to return (default: 10)
+  extractFullContent: true,  // Get full article text (default: true)
+  qualityThreshold: 0.5,     // Min quality score 0-1 (default: 0.5)
+  sourceType: 'auto',        // 'auto' | 'rss' | 'sitemap' | 'html'
+  allowPaths: ['/blog/*'],   // Only scrape these paths
+  denyPaths: ['/about'],     // Skip these paths
+  onProgress: (done, total) => console.log(`${done}/${total}`)
+});
 ```
 
-## Performance
+### Response Format
 
-- **Single article extraction:** ~2-5 seconds
-- **Bundle size:** ~150 KB (gzipped)
-- **Memory usage:** ~100 MB average
-- **No external APIs:** Zero API costs
+```typescript
+{
+  url: string;
+  detectedType: 'rss' | 'sitemap' | 'html';
+  articles: Array<{
+    url: string;
+    title: string;
+    publishedDate: string;
+    description?: string;
+    fullContent?: string;          // Raw HTML
+    fullContentMarkdown?: string;  // Formatted markdown
+    fullContentText?: string;      // Plain text
+    qualityScore: number;          // 0-1
+    confidence: number;
+    source: 'rss' | 'sitemap' | 'html';
+  }>;
+  stats: {
+    totalDiscovered: number;
+    afterQualityFilter: number;
+    processingTime: number;
+  };
+  errors: string[];
+}
+```
 
-## Requirements
+### Advanced: Direct Orchestrator
 
-- Node.js ≥ 18.0.0
-- No environment variables needed
+```typescript
+import { globalSourceOrchestrator } from './lib';
 
-## License
+const result = await globalSourceOrchestrator.processSource(url, {
+  sourceType: 'auto',
+  allowPaths: ['/news/*'],
+  denyPaths: ['/about', '/careers/*']
+});
 
-MIT © Tyrone Ross
+// Enhance with full content (parallel processing)
+const enhanced = await globalSourceOrchestrator.enhanceWithFullContent(
+  result.articles,
+  10,
+  { concurrency: 5, onProgress: (done, total) => {} }
+);
+```
 
-## Contributing
+### Rate Limiter Presets
 
-Contributions welcome! Please open an issue or PR.
+```typescript
+import { createRateLimiter } from './lib';
 
-## Support
+const limiter = createRateLimiter('moderate'); // or 'conservative', 'aggressive'
+```
+
+| Preset | Req/s | Max Concurrent | Per Host |
+|--------|-------|----------------|----------|
+| conservative | 1 | 10 | 2 |
+| moderate | 2 | 20 | 3 |
+| aggressive | 4 | 30 | 5 |
+
+### Path Patterns
+
+```typescript
+'/blog/*'      // Matches /blog/anything
+'/news/2024/*' // Matches /news/2024/anything
+'/about'       // Exact match
+```
+
+**Default deny patterns:** `/`, `/about/*`, `/careers/*`, `/contact/*`, `/tag/*`, `/category/*`, `/login`, `/signup`, `/pricing/*`
+
+### Quality Scoring
 
-- [GitHub Issues](https://github.com/tyroneross/blog-content-scraper/issues)
-- [Documentation](https://github.com/tyroneross/blog-content-scraper#readme)
+Score weights:
+- Content quality: 60%
+- Publication date: 12%
+- Author/byline: 8%
+- Schema.org data: 8%
+- Reading time: 12%
 
 ---
 

package/dist/lib/circuit-breaker.d.ts

@@ -0,0 +1,29 @@
+interface CircuitBreakerOptions {
+    failureThreshold: number;
+    timeout: number;
+    resetTimeout: number;
+    name: string;
+}
+export declare class CircuitBreaker {
+    private failures;
+    private lastFailureTime;
+    private state;
+    private options;
+    constructor(options: CircuitBreakerOptions);
+    execute<T>(operation: () => Promise<T>): Promise<T>;
+    private executeWithTimeout;
+    private onSuccess;
+    private onFailure;
+    getState(): {
+        state: "CLOSED" | "OPEN" | "HALF_OPEN";
+        failures: number;
+        lastFailureTime: number;
+    };
+}
+export declare const circuitBreakers: {
+    rss: CircuitBreaker;
+    scraping: CircuitBreaker;
+    scrapingTest: CircuitBreaker;
+};
+export {};
+//# sourceMappingURL=circuit-breaker.d.ts.map

package/dist/lib/circuit-breaker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"circuit-breaker.d.ts","sourceRoot":"","sources":["../../lib/circuit-breaker.ts"],"names":[],"mappings":"AAAA,UAAU,qBAAqB;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,KAAK,CAA6C;IAC1D,OAAO,CAAC,OAAO,CAAwB;gBAE3B,OAAO,EAAE,qBAAqB;IAIpC,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YAoB3C,kBAAkB;IAkBhC,OAAO,CAAC,SAAS;IAKjB,OAAO,CAAC,SAAS;IAUjB,QAAQ;;;;;CAOT;AAGD,eAAO,MAAM,eAAe;;;;CAqB3B,CAAC"}