rubycrawl 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fb671708a756ff233448c5d18ac0bd815eb44ecf8d0a8c893fcf8107f95c182
-  data.tar.gz: 66898cb3f978494123441044319105e027f670557cd61430a184e0e1d105a3ed
+  metadata.gz: c38e6b7b377a04d6baec4756a7bdf749580e5391d42483b9f6f7e50ee0cbd25f
+  data.tar.gz: 8323d9dbe93915b2f81fb6adbd6056b0007ef3ac58a828feb3492d14e02b7423
 SHA512:
-  metadata.gz: d715984a47719f7c022c512bf136c7bd01050b67ea1dba44f9ea2e24b93196525f46c31e95402c18ba2afbceba02a734f8f214ee686522260557589fca7fea01
-  data.tar.gz: 621e0c5ee326f2757c5459ca763bffd24bfa70baa9c33d820003d7bae0da0ce20529e1bff95136893b846fd80efc70129873ee87ccdb7c2bef5926879d01e6ba
+  metadata.gz: 2905355938f1f18c747c83bdcc1360f88c887026d8b2242c00a87727cb32ab9954927a24ea12975d8c89a1f0e358ffab22ed458b2cac1d8a9acfb9537bb03eca
+  data.tar.gz: 556b1d58707d72698a8e537dc41e8a0b4d47656b501b1cbdcff9db1532180d4d28b4f443f505789454da381efa52adf722c501eb85212564809b6571d504ee55

data/README.md

@@ -1,6 +1,7 @@
 # RubyCrawl 🎭
 
 [![Gem Version](https://badge.fury.io/rb/rubycrawl.svg)](https://rubygems.org/gems/rubycrawl)
+[![CI](https://github.com/craft-wise/rubycrawl/actions/workflows/ci.yml/badge.svg)](https://github.com/craft-wise/rubycrawl/actions/workflows/ci.yml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Ruby](https://img.shields.io/badge/ruby-%3E%3D%203.0-red.svg)](https://www.ruby-lang.org/)
 
@@ -16,6 +17,7 @@ RubyCrawl provides **accurate, JavaScript-enabled web scraping** using a pure Ru
 - ✅ **Production-ready** — Auto-retry, error handling, resource optimization
 - ✅ **Multi-page crawling** — BFS algorithm with smart URL deduplication
 - ✅ **Rails-friendly** — Generators, initializers, and ActiveJob integration
+- ✅ **Readability-powered** — Mozilla Readability.js for article-quality extraction, heuristic fallback for all other pages
 
 ```ruby
 # One line to crawl any JavaScript-heavy site
@@ -35,7 +37,7 @@ result.metadata       # Title, description, OG tags, etc.
 - **Simple API**: Clean Ruby interface — zero Ferrum or CDP knowledge required
 - **Resource optimization**: Built-in resource blocking for 2-3x faster crawls
 - **Auto-managed browsers**: Lazy Chrome singleton, isolated page per crawl
-- **Content extraction**: HTML, plain text, clean HTML, Markdown (lazy), links, metadata
+- **Content extraction**: Mozilla Readability.js (primary) + link-density heuristic (fallback) — article-quality `clean_html`, `clean_text`, `clean_markdown`, links, metadata
 - **Multi-page crawling**: BFS crawler with configurable depth limits and URL deduplication
 - **Smart URL handling**: Automatic normalization, tracking parameter removal, same-host filtering
 - **Rails integration**: First-class Rails support with generators and initializers
@@ -102,14 +104,15 @@ require "rubycrawl"
 result = RubyCrawl.crawl("https://example.com")
 
 # Access extracted content
-result.final_url      # Final URL after redirects
-result.clean_text     # Noise-stripped plain text (no nav/footer/ads)
-result.clean_html     # Noise-stripped HTML (same noise removed as clean_text)
-result.raw_text       # Full body.innerText (unfiltered)
-result.html           # Full raw HTML content
-result.links          # Extracted links with url, text, title, rel
-result.metadata       # Title, description, OG tags, etc.
-result.clean_markdown # Markdown converted from clean_html (lazy — first access only)
+result.final_url                   # Final URL after redirects
+result.clean_text                  # Noise-stripped plain text (no nav/footer/ads)
+result.clean_html                  # Noise-stripped HTML (same noise removed as clean_text)
+result.raw_text                    # Full body.innerText (unfiltered)
+result.html                        # Full raw HTML content
+result.links                       # Extracted links with url, text, title, rel
+result.metadata                    # Title, description, OG tags, etc.
+result.metadata['extractor']       # "readability" or "heuristic" — which extractor ran
+result.clean_markdown              # Markdown converted from clean_html (lazy — first access only)
 ```
 
 ## Use Cases
@@ -187,13 +190,38 @@ puts "Indexed #{pages_crawled} pages"
 
 #### Multi-Page Options
 
-| Option            | Default   | Description                          |
-| ----------------- | --------- | ------------------------------------ |
-| `max_pages`       | 50        | Maximum number of pages to crawl     |
-| `max_depth`       | 3         | Maximum link depth from start URL    |
-| `same_host_only`  | true      | Only follow links on the same domain |
-| `wait_until`      | inherited | Page load strategy                   |
-| `block_resources` | inherited | Block images/fonts/CSS               |
+| Option                 | Default   | Description                                         |
+| ---------------------- | --------- | --------------------------------------------------- |
+| `max_pages`            | 50        | Maximum number of pages to crawl                    |
+| `max_depth`            | 3         | Maximum link depth from start URL                   |
+| `same_host_only`       | true      | Only follow links on the same domain                |
+| `wait_until`           | inherited | Page load strategy                                  |
+| `block_resources`      | inherited | Block images/fonts/CSS                              |
+| `respect_robots_txt`   | false     | Honour robots.txt rules and auto-sleep `Crawl-delay`|
+
+#### robots.txt Support
+
+When `respect_robots_txt: true`, RubyCrawl fetches `robots.txt` once at the start of the crawl and:
+
+- Skips any URL disallowed for `User-agent: *`
+- Automatically sleeps the `Crawl-delay` specified in robots.txt between pages
+
+```ruby
+RubyCrawl.crawl_site("https://example.com",
+  respect_robots_txt: true,
+  max_pages: 100
+) do |page|
+  puts page.url
+end
+```
+
+Or enable globally:
+
+```ruby
+RubyCrawl.configure(respect_robots_txt: true)
+```
+
+If robots.txt is unreachable or missing, crawling proceeds normally (fail open).
 
 #### Page Result Object
 
@@ -245,11 +273,12 @@ result = RubyCrawl.crawl(
 
 | Option            | Values                                                      | Default | Description                                         |
 | ----------------- | ----------------------------------------------------------- | ------- | --------------------------------------------------- |
-| `wait_until`      | `"load"`, `"domcontentloaded"`, `"networkidle"`, `"commit"` | `nil`   | When to consider page loaded (nil = Ferrum default) |
-| `block_resources` | `true`, `false`                                             | `nil`   | Block images, fonts, CSS, media for faster crawls   |
-| `max_attempts`    | Integer                                                     | `3`     | Total number of attempts (including the first)      |
-| `timeout`         | Integer (seconds)                                           | `30`    | Browser navigation timeout                          |
-| `headless`        | `true`, `false`                                             | `true`  | Run Chrome headlessly                               |
+| `wait_until`           | `"load"`, `"domcontentloaded"`, `"networkidle"`, `"commit"` | `nil`   | When to consider page loaded (nil = Ferrum default) |
+| `block_resources`      | `true`, `false`                                             | `nil`   | Block images, fonts, CSS, media for faster crawls   |
+| `max_attempts`         | Integer                                                     | `3`     | Total number of attempts (including the first)      |
+| `timeout`              | Integer (seconds)                                           | `30`    | Browser navigation timeout                          |
+| `headless`             | `true`, `false`                                             | `true`  | Run Chrome headlessly                               |
+| `respect_robots_txt`   | `true`, `false`                                             | `false` | Honour robots.txt rules and auto-sleep Crawl-delay  |
 
 **Wait strategies explained:**
 
@@ -318,7 +347,8 @@ result.metadata
 #   "twitter_image"       => "https://...",
 #   "canonical"           => "https://...",
 #   "lang"                => "en",
-#   "charset"             => "UTF-8"
+#   "charset"             => "UTF-8",
+#   "extractor"           => "readability"  # or "heuristic"
 # }
 ```
 
@@ -473,25 +503,45 @@ RubyCrawl uses a single-process architecture:
 ```
 RubyCrawl (public API)
   ↓
-Browser (lib/rubycrawl/browser.rb)  ← Ferrum wrapper
+Browser (lib/rubycrawl/browser.rb)       ← Ferrum wrapper
   ↓
-Ferrum::Browser                     ← Chrome DevTools Protocol (pure Ruby)
+Ferrum::Browser                          ← Chrome DevTools Protocol (pure Ruby)
   ↓
-Chromium                            ← headless browser
+Chromium                                 ← headless browser
+  ↓
+Readability.js → heuristic fallback      ← content extraction (inside browser)
 ```
 
 - Chrome launches once lazily and is reused across all crawls
 - Each crawl gets an isolated page context (own cookies/storage)
-- JS extraction runs inside the browser via `page.evaluate()`
+- Content extraction runs inside the browser via `page.evaluate()`:
+  - **Primary**: Mozilla Readability.js — article-quality extraction for blogs, docs, news
+  - **Fallback**: link-density heuristic — covers marketing pages, homepages, SPAs
+- `result.metadata['extractor']` tells you which path was used (`"readability"` or `"heuristic"`)
 - No separate processes, no HTTP boundary, no Node.js
 
 ## Performance
 
 - **Resource blocking**: Keep `block_resources: true` (default: nil) to skip images/fonts/CSS for 2-3x faster crawls
 - **Wait strategy**: Use `wait_until: "load"` for static sites, `"networkidle"` for SPAs
-- **Concurrency**: Use background jobs (Sidekiq, GoodJob, etc.) for parallel crawling
 - **Browser reuse**: The first crawl is slower (~2s) due to Chrome launch; subsequent crawls are much faster (~200-500ms)
 
+### Parallelism
+
+RubyCrawl does not support parallel page loading within a single process — Ferrum uses one Chrome instance and concurrent access is not thread-safe.
+
+The recommended pattern is **job-level parallelism**: each background job gets its own `RubyCrawl` instance and Chrome process, with natural rate limiting via your job queue's concurrency setting:
+
+```ruby
+# Enqueue independent crawls — each job runs its own Chrome
+urls.each { |url| CrawlJob.perform_later(url) }
+
+# Control concurrency via your queue worker config (Sidekiq, GoodJob, etc.)
+# e.g. Sidekiq concurrency: 3 → 3 Chrome processes crawling in parallel
+```
+
+This also works naturally with `respect_robots_txt: true` — each job respects Crawl-delay independently.
+
 ## Development
 
 ```bash
@@ -499,12 +549,9 @@ git clone git@github.com:craft-wise/rubycrawl.git
 cd rubycrawl
 bin/setup
 
-# Run unit tests (no browser required)
+# Run all tests (Chrome required — installed as a gem dependency)
 bundle exec rspec
 
-# Run integration tests (requires Chrome)
-INTEGRATION=1 bundle exec rspec
-
 # Manual testing
 bin/console
 > RubyCrawl.crawl("https://example.com")
@@ -528,7 +575,9 @@ The gem is available as open source under the terms of the [MIT License](LICENSE
 
 Built with [Ferrum](https://github.com/rubycdp/ferrum) — pure Ruby Chrome DevTools Protocol client.
 
-Powered by [reverse_markdown](https://github.com/xijo/reverse_markdown) for GitHub-flavored Markdown conversion.
+Content extraction powered by [Mozilla Readability.js](https://github.com/mozilla/readability) — the algorithm behind Firefox Reader View.
+
+Markdown conversion powered by [reverse_markdown](https://github.com/xijo/reverse_markdown) for GitHub-flavored output.
 
 ## Support
 

data/lib/rubycrawl/browser/extraction.rb

@@ -3,13 +3,10 @@
 class RubyCrawl
   class Browser
     # JavaScript extraction constants, evaluated inside Chromium via page.evaluate().
-    # Ported verbatim from node/src/index.js — logic is unchanged.
-    # NOISE_SELECTORS is interpolated directly into EXTRACT_CONTENT_JS (no need to
-    # pass as a JS argument as the Node version did).
+    # All constants are IIFEs — Ferrum's page.evaluate() evaluates an expression,
+    # it does NOT call function definitions. Wrapping as (() => { ... })() ensures
+    # the function is immediately invoked and its return value is captured.
     module Extraction
-      # All constants are IIFEs — Ferrum's page.evaluate() evaluates an expression,
-      # it does NOT call function definitions. Wrapping as (() => { ... })() ensures
-      # the function is immediately invoked and its return value is captured.
       EXTRACT_METADATA_JS = <<~JS
         (() => {
           const getMeta = (name) => {
@@ -54,8 +51,7 @@ class RubyCrawl
         (() => (document.body?.innerText || "").trim())()
       JS
 
-      # Semantic noise selectors — covers standard HTML5 elements and ARIA roles.
-      # Interpolated directly into EXTRACT_CONTENT_JS as a string literal.
+      # Semantic noise selectors — used by the heuristic fallback.
       NOISE_SELECTORS = [
         'nav', 'header', 'footer', 'aside',
         '[role="navigation"]', '[role="banner"]', '[role="contentinfo"]',
@@ -64,11 +60,37 @@ class RubyCrawl
         'script', 'style', 'noscript', 'iframe'
       ].join(', ').freeze
 
-      # Removes semantic noise (nav/header/footer/aside + ARIA roles) and high
-      # link-density containers, then returns both clean plain text and clean HTML.
-      # DOM mutations are reversed after extraction so the page is unchanged.
+      # Mozilla Readability.js v0.6.0 — vendored source, read once at load time.
+      # Embedded inside EXTRACT_CONTENT_JS's outer IIFE so Readability is defined
+      # and used within the same Runtime.evaluate expression (Ferrum evaluates a
+      # single expression — separate evaluate calls have separate scopes).
+      READABILITY_JS = File.read(File.join(__dir__, 'readability.js')).freeze
+
+      # Extracts clean article HTML using Mozilla Readability (primary) with a
+      # link-density heuristic as fallback when Readability returns no content.
+      # Everything is wrapped in one outer IIFE so page.evaluate gets a single
+      # expression and Readability is in scope for the extraction logic.
+      # DOM mutations from the fallback path are reversed after extraction.
       EXTRACT_CONTENT_JS = <<~JS.freeze
         (() => {
+          // Mozilla Readability.js v0.6.0 — defined in this IIFE's scope.
+          #{READABILITY_JS}
+
+          // Primary: Mozilla Readability — article-quality extraction.
+          let readabilityDebug = null;
+          try {
+            const docClone = document.cloneNode(true);
+            const reader = new Readability(docClone, { charThreshold: 100 });
+            const article = reader.parse();
+            if (article && article.textContent && article.textContent.trim().length > 200) {
+              return { cleanHtml: article.content, extractor: "readability" };
+            }
+            readabilityDebug = article ? `returned ${article.textContent?.trim().length ?? 0} text chars (below threshold)` : "returned null (no article detected)";
+          } catch (e) {
+            readabilityDebug = `error: ${e.message}`;
+          }
+
+          // Fallback: link-density heuristic (works on nav-heavy / non-article pages).
           const noiseSelectors = #{NOISE_SELECTORS.to_json};
           function linkDensity(el) {
             const total = (el.innerText || "").trim().length;
@@ -98,7 +120,7 @@ class RubyCrawl
           }
           const cleanHtml = document.body.innerHTML;
           removed.reverse().forEach(({ el, parent, next }) => parent.insertBefore(el, next));
-          return { cleanHtml };
+          return { cleanHtml, extractor: "heuristic", debug: readabilityDebug };
         })()
       JS
     end