html2rss 0.18.0 → 0.19.0

data/lib/html2rss.rb

@@ -26,7 +26,7 @@ module Html2rss
   #
   # @param file [String] path to the YAML file
   # @param feed_name [String, nil] optional feed name inside a multi-feed config
-  # @return [Hash<Symbol, Object>] loaded configuration hash
+  # @return [Hash{Symbol => Object}] loaded configuration hash
   def self.config_from_yaml_file(file, feed_name = nil)
     Config.load_yaml(file, feed_name)
   end
@@ -34,23 +34,19 @@ module Html2rss
   ##
   # Returns an RSS object generated from the provided configuration.
   #
-  # @param raw_config [Hash<Symbol, Object>] feed configuration
+  # @param raw_config [Hash{Symbol => Object}] feed configuration
   # @return [RSS::Rss] generated RSS feed
   def self.feed(raw_config)
-    run_pipeline(raw_config) do |response:, config:, articles:|
-      build_rss_feed(response:, config:, articles:)
-    end
+    FeedPipeline.new(raw_config).to_rss
   end
 
   ##
   # Returns a JSONFeed 1.1 hash generated from the provided configuration.
   #
-  # @param raw_config [Hash<Symbol, Object>] feed configuration
+  # @param raw_config [Hash{Symbol => Object}] feed configuration
   # @return [Hash] JSONFeed-compliant hash
   def self.json_feed(raw_config)
-    run_pipeline(raw_config) do |response:, config:, articles:|
-      build_json_feed(response:, config:, articles:)
-    end
+    FeedPipeline.new(raw_config).to_json_feed
   end
 
   ##
@@ -62,7 +58,7 @@ module Html2rss
   # @param max_redirects [Integer, nil] optional redirect limit override
   # @param max_requests [Integer, nil] optional request budget override
   # @return [RSS::Rss] generated RSS feed
-  def self.auto_source(url, strategy: :faraday, items_selector: nil, max_redirects: nil, max_requests: nil)
+  def self.auto_source(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: nil)
     feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:))
   end
 
@@ -75,80 +71,13 @@ module Html2rss
   # @param max_redirects [Integer, nil] optional redirect limit override
   # @param max_requests [Integer, nil] optional request budget override
   # @return [Hash] JSONFeed-compliant hash
-  def self.auto_json_feed(url, strategy: :faraday, items_selector: nil, max_redirects: nil, max_requests: nil)
+  def self.auto_json_feed(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: nil)
     json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:))
   end
 
   class << self
     private
 
-    def run_pipeline(raw_config)
-      # 1. Normalize and validate the user-facing feed config.
-      config = Config.from_hash(raw_config, params: raw_config[:params])
-      runtime_input = RequestSession::RuntimeInput.from_config(config)
-
-      # 2. Fetch the initial page using a shared request session.
-      request_session = RequestSession.from_runtime_input(runtime_input)
-      response = request_session.fetch_initial_response
-
-      # 3. Collect articles from configured selectors and auto-source scrapers.
-      articles = Articles::Deduplicator.new(
-        collect_articles(response:, config:, request_session:)
-      ).call
-
-      # 4. Render the final output format chosen by the public entrypoint.
-      yield response:, config:, articles:
-    end
-
-    def collect_articles(response:, config:, request_session:)
-      selector_articles(response:, config:, request_session:) +
-        auto_source_articles(response:, config:, request_session:)
-    end
-
-    def selector_articles(response:, config:, request_session:) # rubocop:disable Metrics/MethodLength
-      return [] unless (selectors = config.selectors)
-
-      page_responses = if (max_pages = selectors.dig(:items, :pagination, :max_pages))
-                         RequestSession::RelNextPager.new(
-                           session: request_session,
-                           initial_response: response,
-                           max_pages:
-                         ).to_a
-                       else
-                         [response]
-                       end
-
-      page_responses.flat_map do |page_response|
-        Selectors.new(page_response, selectors:, time_zone: config.time_zone).articles
-      end
-    end
-
-    def auto_source_articles(response:, config:, request_session:)
-      return [] unless (auto_source = config.auto_source)
-
-      AutoSource.new(response, auto_source, request_session:).articles
-    end
-
-    def build_rss_feed(response:, config:, articles:)
-      channel = RssBuilder::Channel.new(response, overrides: config.channel)
-
-      RssBuilder.new(channel:, articles:, stylesheets: config.stylesheets).call
-    end
-
-    def build_json_feed(response:, config:, articles:)
-      channel = RssBuilder::Channel.new(response, overrides: config.channel)
-
-      JsonFeedBuilder.new(channel:, articles:).call
-    end
-
-    def explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
-      keys = []
-      keys << :strategy unless strategy == :faraday
-      keys << :max_redirects unless max_redirects.nil?
-      keys << :max_requests unless max_requests.nil?
-      keys
-    end
-
     def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:)
       Config.auto_source_config(
         url:,
@@ -165,6 +94,14 @@ module Html2rss
         explicit_keys: explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
       )
     end
+
+    def explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
+      keys = []
+      keys << :strategy unless strategy.nil? || strategy == Config.default_strategy_name
+      keys << :max_redirects unless max_redirects.nil?
+      keys << :max_requests unless max_requests.nil?
+      keys
+    end
   end
 end
 

data/schema/html2rss-config.schema.json

@@ -3,6 +3,7 @@
   "type": "object",
   "properties": {
     "strategy": {
+      "type": "string",
       "not": {
         "type": "null"
       }
@@ -445,13 +446,94 @@
             }
           },
           "required": []
+        },
+        "botasaurus": {
+          "type": "object",
+          "properties": {
+            "navigation_mode": {
+              "type": "string",
+              "minLength": 1,
+              "enum": [
+                "auto",
+                "get",
+                "google_get",
+                "google_get_bypass"
+              ]
+            },
+            "max_retries": {
+              "type": "integer",
+              "not": {
+                "type": "null"
+              },
+              "minimum": 0,
+              "maximum": 3
+            },
+            "wait_for_selector": {
+              "type": [
+                "null",
+                "string"
+              ]
+            },
+            "wait_timeout_seconds": {
+              "type": "integer",
+              "not": {
+                "type": "null"
+              },
+              "exclusiveMinimum": 0
+            },
+            "block_images": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "block_images_and_css": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "wait_for_complete_page_load": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "headless": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "proxy": {
+              "type": "string",
+              "minLength": 1
+            },
+            "user_agent": {
+              "type": "string",
+              "minLength": 1
+            },
+            "window_size": {
+              "type": "array",
+              "items": {
+                "minLength": 2,
+                "maxLength": 2,
+                "type": "integer",
+                "exclusiveMinimum": 0
+              }
+            },
+            "lang": {
+              "type": "string",
+              "minLength": 1
+            }
+          },
+          "required": []
         }
       },
       "required": []
     }
   },
   "required": [
-    "strategy",
     "channel"
   ],
   "anyOf": [

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.18.0
+  version: 0.19.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -315,6 +315,9 @@ files:
 - lib/html2rss/config/schema.rb
 - lib/html2rss/config/validator.rb
 - lib/html2rss/error.rb
+- lib/html2rss/feed_pipeline.rb
+- lib/html2rss/feed_pipeline/auto_fallback.rb
+- lib/html2rss/hash_util.rb
 - lib/html2rss/html_extractor.rb
 - lib/html2rss/html_extractor/date_extractor.rb
 - lib/html2rss/html_extractor/enclosure_extractor.rb
@@ -331,6 +334,8 @@ files:
 - lib/html2rss/rendering/video_renderer.rb
 - lib/html2rss/request_controls.rb
 - lib/html2rss/request_service.rb
+- lib/html2rss/request_service/botasaurus_contract.rb
+- lib/html2rss/request_service/botasaurus_strategy.rb
 - lib/html2rss/request_service/browserless_strategy.rb
 - lib/html2rss/request_service/budget.rb
 - lib/html2rss/request_service/context.rb
@@ -379,7 +384,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.18.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.19.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: