html2rss 0.18.0 → 0.19.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3dfb5fe2f11c7ef1948abb13942f89489585bce5f3eb2509807e8b6c8bfbfafb
-  data.tar.gz: b2bce87ae6e3450faf38fcad8c839e8e9e762737ec51d6d5b6ef4bfc099aa456
+  metadata.gz: 69268fde80ddaa21f5ca3588de51f63182909714956af3ed8b1ee11a47075dc8
+  data.tar.gz: 045dfb3fec6cebfa8c7d066acd12c056dbf01766bbe1c292642d6d4d9db72055
 SHA512:
-  metadata.gz: c252b80acd7ede0cca3b0c15941a06b8dce703d0bf1835ebabe1be707558a29d72cb175915f39978ae1ee13d9551e7b692589d7d43d956db8fc5d52f52c20453
-  data.tar.gz: e68d7254ffe2c5ad634b7e67d3023756be8fe954c2c96b1429fea91e76b6da11a332d616731209ea60398565767679aa1fc4ecd38e3159886fe77280074e6ec5
+  metadata.gz: de88861fd21375da62549cbed418f5f1550e7adf8e9c6ea98cfce9331944067bc8aa43eacdbdfe3ab6380764e485031f1d6bb3b456e0bf486340864328a5abc8
+  data.tar.gz: 6f65cd2e7dc555c35cb456bff595184331f3df07dafe10c50d6d334102237b0f98a5971c6033d8e56c4504254726ca0eca9757c22b1a697c2a47b8b76681945a

data/README.md

@@ -36,7 +36,7 @@ Please see the [contributing guide](https://html2rss.github.io/get-involved/cont
 ### Core Components
 
 1. **Config** - Loads and validates configuration (YAML/hash)
-2. **RequestService** - Fetches pages using Faraday or Browserless
+2. **RequestService** - Fetches pages using Faraday, Botasaurus, or Browserless
 3. **Selectors** - Extracts content via CSS selectors with extractors/post-processors
 4. **AutoSource** - Auto-detects content using Schema.org, JSON state blobs, semantic HTML, and structural patterns
 5. **RssBuilder** - Assembles Article objects and renders RSS 2.0
@@ -47,6 +47,65 @@ Please see the [contributing guide](https://html2rss.github.io/get-involved/cont
 Config -> Request -> Extraction -> Processing -> Building -> Output
 ```
 
+### Request Strategies
+
+- `auto` (default): pipeline fallback orchestration (`faraday` -> `botasaurus` -> `browserless`) based on extraction outcome and retry policy.
+- `faraday`: direct HTTP fetch.
+- `botasaurus`: delegates fetching to a Botasaurus scrape API. Requires `BOTASAURUS_SCRAPER_URL` (for example `http://localhost:4010`).
+- `browserless`: remote browser rendering via Browserless (`BROWSERLESS_IO_WEBSOCKET_URL` and token as needed).
+
+Auto fallback shares one request budget across all strategy attempts. For pagination-heavy or dynamic pages, increase `request.max_requests` (or `--max-requests`) when retries exhaust the budget.
+
+Auto fallback decisions are hidden at the default `LOG_LEVEL=warn`; run with `LOG_LEVEL=info` to include them in CLI output.
+
+Supported `request.botasaurus` options:
+
+- `navigation_mode` (`auto`, `get`, `google_get`, `google_get_bypass`; default `auto`)
+- `max_retries` (`0..3`; default `2`)
+- `wait_for_selector` (string)
+- `wait_timeout_seconds` (integer)
+- `block_images` (boolean)
+- `block_images_and_css` (boolean)
+- `wait_for_complete_page_load` (boolean)
+- `headless` (boolean, default `false`)
+- `proxy` (string)
+- `user_agent` (string)
+- `window_size` (two-item integer array, for example `[1920, 1080]`)
+- `lang` (string, for example `en-US`)
+
+Minimal YAML config example:
+
+```yaml
+channel:
+  url: https://example.com
+strategy: botasaurus
+auto_source: {}
+request:
+  botasaurus:
+    navigation_mode: auto
+    max_retries: 2
+    headless: false
+```
+
+Example request payload shape:
+
+```json
+{
+  "url": "https://example.com",
+  "navigation_mode": "auto",
+  "max_retries": 2,
+  "headless": false
+}
+```
+
+Example usage:
+
+```bash
+BOTASAURUS_SCRAPER_URL=http://localhost:4010 html2rss auto https://example.com --strategy botasaurus
+```
+
+Policy note: html2rss still enforces local request policy preflight and timeout budget. Botasaurus handles browser navigation/rendering internals, so some policy details are delegated to upstream execution.
+
 ### Config schema workflow
 
 The config schema is generated from the runtime `dry-validation` contracts and exported for client-side tooling.

data/lib/html2rss/articles/deduplicator.rb

@@ -3,6 +3,7 @@
 require 'set' # rubocop:disable Lint/RedundantRequireStatement
 
 module Html2rss
+  # Shared helpers that operate on `RssBuilder::Article` collections.
   module Articles
     ##
     # Deduplicates a list of articles while preserving their original order.

data/lib/html2rss/auto_source/cleanup.rb

@@ -7,14 +7,21 @@ module Html2rss
     # :reek:MissingSafeMethod { enabled: false }
     # It applies various strategies to filter and refine the article list.
     class Cleanup
+      # Default cleanup behavior for auto-sourced article lists.
       DEFAULT_CONFIG = {
         keep_different_domain: false,
         min_words_title: 3
       }.freeze
 
+      # Allowed URL schemes for article filtering.
       VALID_SCHEMES = %w[http https].to_set.freeze
 
       class << self
+        # @param articles [Array<Article>] extracted article candidates
+        # @param url [Html2rss::Url] feed source URL used for same-host filtering
+        # @param keep_different_domain [Boolean] whether to keep off-domain entries
+        # @param min_words_title [Integer] minimum word count for title filtering
+        # @return [Array<Article>] cleaned article list
         def call(articles, url:, keep_different_domain:, min_words_title:)
           Log.debug "Cleanup: start with #{articles.size} articles"
 
@@ -35,6 +42,7 @@ module Html2rss
         #
         # @param articles [Array<Article>] The list of articles to process.
         # @param key [Symbol] The key to deduplicate by.
+        # @return [Array<Article>] the mutated articles array
         def deduplicate_by!(articles, key)
           seen = {}
           articles.reject! do |article|
@@ -47,6 +55,7 @@ module Html2rss
         # Keeps only articles with HTTP or HTTPS URLs.
         #
         # @param articles [Array<Article>] The list of articles to process.
+        # @return [Array<Article>] the mutated articles array
         def keep_only_http_urls!(articles)
           articles.select! { |article| VALID_SCHEMES.include?(article.url&.scheme) }
         end
@@ -56,6 +65,7 @@ module Html2rss
         #
         # @param articles [Array<Article>] The list of articles to process.
         # @param base_url [Html2rss::Url] The source URL to compare against.
+        # @return [Array<Article>] the mutated articles array
         def reject_different_domain!(articles, base_url)
           base_host = base_url.host
           articles.select! { |article| article.url&.host == base_host }
@@ -66,6 +76,7 @@ module Html2rss
         #
         # @param articles [Array<Article>] The list of articles to process.
         # @param min_words_title [Integer] The minimum number of words in the title.
+        # @return [Array<Article>] the mutated articles array
         def keep_only_with_min_words_title!(articles, min_words_title:)
           articles.select! do |article|
             article.title ? word_count_at_least?(article.title, min_words_title) : true

data/lib/html2rss/auto_source/scraper/html.rb

@@ -19,9 +19,12 @@ module Html2rss
       class Html
         include Enumerable
 
+        # Elements ignored when traversing potential article containers.
         TAGS_TO_IGNORE = /(nav|footer|header|svg|script|style)/i
 
+        # Minimum selector frequency required to treat a path as a stable list signal.
         DEFAULT_MINIMUM_SELECTOR_FREQUENCY = 2
+        # Number of most frequent selectors kept for container extraction.
         DEFAULT_USE_TOP_SELECTORS = 5
 
         ##
@@ -53,6 +56,8 @@ module Html2rss
         # @param url [String] The base URL.
         # @param extractor [Class] The extractor class to handle article extraction.
         # @param opts [Hash] Additional options.
+        # @option opts [Integer] :minimum_selector_frequency minimum count before a selector is considered stable
+        # @option opts [Integer] :use_top_selectors number of top selectors to keep
         def initialize(parsed_body, url:, extractor: HtmlExtractor, **opts)
           @parsed_body = parsed_body
           @url = url

data/lib/html2rss/auto_source/scraper/json_state.rb

@@ -5,7 +5,7 @@ require 'json'
 module Html2rss
   class AutoSource
     module Scraper
-      #
+      ##
       # Scrapes JSON state blobs embedded in script tags such as Next.js, Nuxt,
       # or custom window globals. The scraper searches `<script type="application/json">`
       # tags and well-known JavaScript globals for arrays of article-like hashes
@@ -13,7 +13,9 @@ module Html2rss
       class JsonState
         include Enumerable
 
+        # Selector for JSON-only script tags.
         JSON_SCRIPT_SELECTOR = 'script[type="application/json"]'
+        # Regex patterns for known global JavaScript state assignments.
         GLOBAL_ASSIGNMENT_PATTERNS = [
           /(?:window|self|globalThis)\.__NEXT_DATA__\s*=\s*/m,
           /(?:window|self|globalThis)\.__NUXT__\s*=\s*/m,
@@ -28,36 +30,53 @@ module Html2rss
           /(?:window|self|globalThis)\.angular\s*=\s*/m
         ].freeze
 
-        TITLE_KEYS = %w[title headline name text].freeze
-        URL_KEYS = %w[url link href permalink slug path canonicalUrl shortUrl].freeze
-        DESCRIPTION_KEYS = %w[description summary excerpt dek subheading].freeze
-        IMAGE_KEYS = %w[image imageUrl thumbnailUrl thumbnail src featuredImage coverImage heroImage].freeze
-        PUBLISHED_AT_KEYS = %w[published_at publishedAt datePublished date publicationDate pubDate updatedAt updated_at
+        # Preferred keys when extracting title-like values from state payloads.
+        TITLE_KEYS = %i[title headline name text].freeze
+        # Preferred keys when extracting URL-like values from state payloads.
+        URL_KEYS = %i[url link href permalink slug path canonicalUrl shortUrl].freeze
+        # Preferred keys when extracting description-like values from state payloads.
+        DESCRIPTION_KEYS = %i[description summary excerpt dek subheading].freeze
+        # Preferred keys when extracting image-like values from state payloads.
+        IMAGE_KEYS = %i[image imageUrl thumbnailUrl thumbnail src featuredImage coverImage heroImage].freeze
+        # Preferred keys when extracting publication timestamps from state payloads.
+        PUBLISHED_AT_KEYS = %i[published_at publishedAt datePublished date publicationDate pubDate updatedAt updated_at
                                createdAt created_at].freeze
-        CATEGORY_KEYS = %w[categories tags section sections topic topics channel].freeze
-        ID_KEYS = %w[id guid uuid slug key].freeze
+        # Preferred keys when extracting category-like values from state payloads.
+        CATEGORY_KEYS = %i[categories tags section sections topic topics channel].freeze
+        # Preferred keys when extracting identifier-like values from state payloads.
+        ID_KEYS = %i[id guid uuid slug key].freeze
 
         # Scans DOM nodes for JSON payloads containing article data.
         module DocumentScanner
           module_function
 
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @return [Array<Hash, Array>] parsed JSON documents discovered in scripts
           def json_documents(parsed_body)
             script_documents(parsed_body) + assignment_documents(parsed_body)
           end
 
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @return [Array<Hash, Array>] JSON documents extracted from JSON script tags
           def script_documents(parsed_body)
             parsed_body.css(JSON_SCRIPT_SELECTOR).filter_map { parse_json(_1.text) }
           end
 
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @return [Array<Hash, Array>] JSON documents extracted from global assignments
           def assignment_documents(parsed_body)
             parsed_body.css('script').filter_map { parse_assignment(_1.text) }
           end
 
+          # @param text [String] script text that may contain a global assignment
+          # @return [Hash, Array, nil] parsed assignment payload when available
           def parse_assignment(text)
             payload = assignment_payload(text)
             parse_json(payload) if payload
           end
 
+          # @param text [String] script text to inspect for known assignment patterns
+          # @return [String, nil] extracted JSON-like assignment payload
           def assignment_payload(text)
             trimmed = text.to_s.strip
             return if trimmed.empty?
@@ -72,10 +91,14 @@ module Html2rss
             nil
           end
 
+          # @param text [String] text potentially containing JSON-like payloads
+          # @return [String, nil] normalized assignment payload
           def extract_assignment_payload(text)
             extract_json_block(text) || text
           end
 
+          # @param text [String] text potentially containing JSON blocks
+          # @return [String, nil] extracted JSON block spanning balanced brackets
           def extract_json_block(text)
             start_index = text.index(/[\[{]/)
             return unless start_index
@@ -85,6 +108,9 @@ module Html2rss
           end
 
           # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+          # @param text [String] text starting with a JSON object/array opening token
+          # @param start_index [Integer] index where JSON-like content starts
+          # @return [Integer, nil] index where the balanced JSON payload ends
           def scan_for_json_end(text, start_index)
             stack = []
             in_string = false
@@ -121,6 +147,8 @@ module Html2rss
           end
           # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
+          # @param payload [String, nil] JSON payload to parse
+          # @return [Hash, Array, nil] parsed payload or nil when parsing fails
           def parse_json(payload)
             return unless payload
 
@@ -129,6 +157,9 @@ module Html2rss
             parse_js_object(payload, error)
           end
 
+          # @param payload [String] JavaScript object-literal payload
+          # @param _original_error [JSON::ParserError] original JSON parse error
+          # @return [Hash, Array, nil] parsed payload after JavaScript coercion
           def parse_js_object(payload, _original_error)
             coerced = coerce_javascript_object(payload)
             return unless coerced
@@ -141,6 +172,8 @@ module Html2rss
             nil
           end
 
+          # @param payload [String] JavaScript object-literal payload
+          # @return [String] JSON-compatible payload string
           def coerce_javascript_object(payload)
             string = payload.dup
 
@@ -148,12 +181,16 @@ module Html2rss
             strip_trailing_commas(quote_unquoted_keys(string))
           end
 
+          # @param jsonish [String] JSON-like string with potentially unquoted keys
+          # @return [String] payload with unquoted object keys quoted
           def quote_unquoted_keys(jsonish)
             jsonish.gsub(/(\A\s*|[{,\[]\s*)([A-Za-z_]\w*)(\s*:)/) do
               "#{Regexp.last_match(1)}\"#{Regexp.last_match(2)}\"#{Regexp.last_match(3)}"
             end
           end
 
+          # @param jsonish [String] JSON-like string with potential trailing commas
+          # @return [String] payload without trailing commas before closing tokens
           def strip_trailing_commas(jsonish)
             jsonish.gsub(/,(\s*[\]}])/, '\1')
           end
@@ -164,6 +201,9 @@ module Html2rss
         module ValueFinder
           module_function
 
+          # @param object [Hash, Array] candidate container traversed during key lookup
+          # @param keys [Array<Symbol>] keys to probe in order
+          # @return [Object, nil] first matching value
           def fetch(object, keys)
             case object
             when Hash then fetch_from_hash(object, keys)
@@ -171,19 +211,21 @@ module Html2rss
             end
           end
 
+          # @param hash [Hash] hash candidate traversed during key lookup
+          # @param keys [Array<Symbol>] keys to probe in order
+          # @return [Object, nil] first matching value from hash or nested metadata
           def fetch_from_hash(hash, keys)
             keys.each do |key|
-              string_key = key.to_s
-              return hash[string_key] if hash.key?(string_key)
-
-              symbol_key = string_key.to_sym
-              return hash[symbol_key] if hash.key?(symbol_key)
+              return hash[key] if hash.key?(key)
             end
 
-            fetch_nested(hash[:attributes] || hash['attributes'], keys) ||
-              fetch_nested(hash[:data] || hash['data'], keys)
+            fetch_nested(hash[:attributes], keys) ||
+              fetch_nested(hash[:data], keys)
           end
 
+          # @param array [Array] array whose entries may contain target keys
+          # @param keys [Array<Symbol>] keys to probe in order
+          # @return [Object, nil] first matching value from array entries
           def fetch_from_array(array, keys)
             array.each do |entry|
               result = fetch(entry, keys)
@@ -193,6 +235,9 @@ module Html2rss
             nil
           end
 
+          # @param value [Hash, Array, nil] nested value to recurse into
+          # @param keys [Array<Symbol>] keys to probe in order
+          # @return [Object, nil] matching nested value
           def fetch_nested(value, keys)
             fetch(value, keys) if value
           end
@@ -203,6 +248,8 @@ module Html2rss
         module CandidateDetector
           module_function
 
+          # @param document [Hash, Array, Object] candidate document node
+          # @return [Boolean] whether the node contains article-like arrays
           def candidate_array?(document)
             case document
             when Array
@@ -214,6 +261,8 @@ module Html2rss
             end
           end
 
+          # @param value [Hash, Array, Object] candidate nested value
+          # @return [Boolean] whether nested value should be traversed for article candidates
           def traversable_candidate?(value)
             case value
             when Array, Hash then candidate_array?(value)
@@ -221,6 +270,8 @@ module Html2rss
             end
           end
 
+          # @param array [Array<Object>] candidate list of entries
+          # @return [Boolean] whether array includes hash entries with title and URL fields
           def array_of_articles?(array)
             array.any? do |element|
               next unless element.is_a?(Hash)
@@ -229,10 +280,14 @@ module Html2rss
             end
           end
 
+          # @param object [Hash] article candidate object
+          # @return [Object, nil] detected title-like value
           def title_from(object)
             ValueFinder.fetch(object, TITLE_KEYS)
           end
 
+          # @param object [Hash] article candidate object
+          # @return [Object, nil] detected URL-like value
           def url_from(object)
             ValueFinder.fetch(object, URL_KEYS)
           end
@@ -244,6 +299,9 @@ module Html2rss
           module_function
 
           # rubocop:disable Metrics/MethodLength
+          # @param entry [Hash] raw article entry candidate
+          # @param base_url [String, Html2rss::Url] base URL for relative link resolution
+          # @return [Hash{Symbol => Object}, nil] normalized article hash for downstream extraction
           def normalise(entry, base_url:)
             return unless entry.is_a?(Hash)
 
@@ -267,11 +325,18 @@ module Html2rss
           end
           # rubocop:enable Metrics/MethodLength
 
+          # @param value [Object] candidate scalar value
+          # @return [String, nil] normalized non-empty string value
           def string(value)
             trimmed = value.to_s.strip
             trimmed unless trimmed.empty?
           end
 
+          # @param entry [Hash] raw article entry candidate
+          # @param keys [Array<String>] preferred link keys
+          # @param base_url [String, Html2rss::Url] base URL for relative link resolution
+          # @param log_key [String] structured log message key
+          # @return [Html2rss::Url, nil] resolved absolute URL
           def resolve_link(entry, keys:, base_url:, log_key:)
             value = ValueFinder.fetch(entry, keys)
             value = ValueFinder.fetch(value, keys) if value.is_a?(Hash)
@@ -285,6 +350,8 @@ module Html2rss
           end
 
           # rubocop:disable Metrics/MethodLength
+          # @param entry [Hash] raw article entry candidate
+          # @return [Array<String>, nil] normalized unique categories
           def categories(entry)
             raw = ValueFinder.fetch(entry, CATEGORY_KEYS)
             names = case raw
@@ -297,7 +364,7 @@ module Html2rss
             result = names.flat_map do |value|
               case value
               when Hash
-                string(ValueFinder.fetch(value, %w[name title label]))
+                string(ValueFinder.fetch(value, %i[name title label]))
               else
                 string(value)
               end
@@ -308,6 +375,9 @@ module Html2rss
           end
           # rubocop:enable Metrics/MethodLength
 
+          # @param entry [Hash] raw article entry candidate
+          # @param article_url [Html2rss::Url] resolved article URL
+          # @return [String] stable article identifier fallbacking to resolved URL
           def identifier(entry, article_url)
             value = ValueFinder.fetch(entry, ID_KEYS)
             value = ValueFinder.fetch(value, ID_KEYS) if value.is_a?(Hash)
@@ -316,20 +386,28 @@ module Html2rss
         end
         private_constant :ArticleNormalizer
 
+        # @return [Symbol] scraper config key
         def self.options_key = :json_state
 
         class << self
+          # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
           def articles?(parsed_body)
             return false unless parsed_body
 
             DocumentScanner.json_documents(parsed_body).any? { CandidateDetector.candidate_array?(_1) }
           end
 
+          # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
+          # @return [Array<Hash, Array>] parsed JSON documents discovered in the response body
           def json_documents(parsed_body)
             DocumentScanner.json_documents(parsed_body)
           end
         end
 
+        # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
+        # @param url [String, Html2rss::Url] page URL used to resolve relative links
+        # @param _opts [Hash] scraper-specific options
+        # @option _opts [Object] :_reserved reserved for future scraper-specific options
         def initialize(parsed_body, url:, **_opts)
           @parsed_body = parsed_body
           @url = url
@@ -337,6 +415,8 @@ module Html2rss
 
         attr_reader :parsed_body
 
+        # @yield [Hash{Symbol => Object}] normalized article hash
+        # @return [Enumerator, void] article enumerator when no block is given
         def each
           return enum_for(:each) unless block_given?