html2rss 0.29.1 → 0.30.0

data/lib/html2rss/request_service/httpx_strategy.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+require 'httpx'
+require 'httpx/plugins/follow_redirects'
+require 'httpx/plugins/callbacks'
+require 'httpx/plugins/ssrf_filter'
+require 'httpx/plugins/brotli'
+require 'httpx/plugins/fiber_concurrency'
+require 'httpx/plugins/retries'
+
+module Html2rss
+  class RequestService
+    ##
+    # Strategy to use HTTPX for HTTP requests.
+    # Provides native HTTP/2 with ALPN, built-in SSRF protection, streaming byte limits,
+    # and redirect handling without monkey-patching.
+    # rubocop:disable-next Metrics/ClassLength -- terminal redirect retry colocated with HTTPX transport
+    class HttpxStrategy < Strategy
+      class << self
+        # rubocop:disable ThreadSafety/ClassInstanceVariable
+        # @return [HTTPX::Session]
+        def base_session
+          @base_sessions ||= {}
+          @base_sessions[HTTPX::Session] ||= HTTPX
+                                             .plugin(:follow_redirects)
+                                             .plugin(:callbacks)
+                                             .plugin(:brotli)
+                                             .plugin(:fiber_concurrency)
+                                             .plugin(:retries)
+        end
+
+        # @return [HTTPX::Session]
+        def base_ssrf_session
+          @base_ssrf_sessions ||= {}
+          @base_ssrf_sessions[HTTPX::Session] ||= base_session.plugin(:ssrf_filter)
+        end
+        # rubocop:enable ThreadSafety/ClassInstanceVariable
+      end
+
+      ##
+      # @return [ResponseGuard]
+      attr_reader :response_guard
+
+      # Executes the request with runtime policy enforcement, returning the normalized response.
+      #
+      # @return [Response] normalized response
+      def perform_execute
+        deadline = request_deadline
+        @response_guard = ResponseGuard.new(policy: ctx.policy)
+        reset_redirect_tracking!
+        raw_response = request_with_terminal_redirect_retry(response_guard, deadline:)
+        build_response(raw_response)
+      end
+
+      private
+
+      def request_deadline
+        monotonic_now + ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
+      end
+
+      def reset_redirect_tracking!
+        @last_redirect_to = nil
+        @terminal_redirect_retried = false
+        @request_url_override = nil
+        @current_url = request_url
+      end
+
+      def request_with_terminal_redirect_retry(response_guard, deadline:)
+        raw_response = execute_http_request(response_guard, deadline:)
+        return raw_response unless redirect_limit_reached?(raw_response)
+
+        unless terminal_redirect_retryable?(raw_response)
+          raise RedirectLimitReached, "Too many redirects (status #{raw_response.status})"
+        end
+
+        retry_from_terminal_redirect!(raw_response, response_guard, deadline:)
+      end
+
+      def redirect_limit_reached?(response)
+        return false unless response.is_a?(HTTPX::Response)
+        return false if response.status == 304
+        return false unless (300..399).cover?(response.status)
+
+        location = response.headers['location']
+        !location.nil? && !location.strip.empty?
+      end
+
+      def terminal_redirect_url(raw_response)
+        location = raw_response.headers['location']
+        if location && !location.empty?
+          base = raw_response.uri || request_url
+          return normalize_url(URI.join(base.to_s, location))
+        end
+
+        @last_redirect_to
+      end
+
+      def terminal_redirect_retryable?(raw_response)
+        return false if @terminal_redirect_retried
+
+        target = terminal_redirect_url(raw_response)
+        target && target.to_s != request_url.to_s
+      end
+
+      def retry_from_terminal_redirect!(raw_response, response_guard, deadline:)
+        terminal_url = terminal_redirect_url(raw_response)
+        @terminal_redirect_retried = true
+        Log.debug("#{self.class}: redirect limit reached; retrying once from #{terminal_url}")
+        begin_terminal_url_request!(terminal_url)
+        new_response = execute_http_request(response_guard, deadline:, consume_budget: false)
+        if redirect_limit_reached?(new_response)
+          raise RedirectLimitReached, "Too many redirects (status #{new_response.status})"
+        end
+
+        new_response
+      end
+
+      def begin_terminal_url_request!(terminal_url)
+        ctx.policy.validate_request!(url: terminal_url, origin_url: ctx.origin_url, relation: ctx.relation)
+        @request_url_override = terminal_url
+        @last_redirect_to = nil
+        @current_url = terminal_url
+      end
+
+      def request_url
+        @request_url_override || ctx.url
+      end
+
+      def execute_http_request(response_guard, deadline:, consume_budget: true)
+        preflight!(consume_budget:)
+        session = build_session(response_guard, deadline:)
+        response = session.get(request_url.to_s, headers: ctx.headers)
+        raise response.error if response.is_a?(HTTPX::ErrorResponse)
+
+        response
+      end
+
+      def build_session(_response_guard, deadline:)
+        session_client(deadline)
+      end
+
+      def session_client(deadline)
+        session = ctx.policy.allow_private_networks? ? self.class.base_session : self.class.base_ssrf_session
+        session.with(session_options(deadline))
+      end
+
+      def session_options(deadline)
+        remaining = remaining_timeout_seconds(deadline)
+        {
+          timeout: session_timeouts(remaining),
+          max_redirects: ctx.policy.max_redirects,
+          follow_insecure_redirects: false,
+          max_response_body_size: ctx.policy.max_response_bytes,
+          resolver_class: :system,
+          redirect_on: redirect_callback
+        }
+      end
+
+      def session_timeouts(remaining)
+        {
+          connect_timeout: [ctx.policy.connect_timeout_seconds, remaining].min,
+          read_timeout: [ctx.policy.read_timeout_seconds, remaining].min,
+          operation_timeout: remaining,
+          request_timeout: remaining,
+          total_request_timeout: remaining
+        }
+      end
+
+      def redirect_callback
+        lambda do |redirect_uri|
+          to_url = normalize_url(redirect_uri)
+          from_url = @current_url
+          @last_redirect_to = to_url
+          ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
+          @current_url = to_url
+          true
+        end
+      end
+
+      def build_response(response)
+        headers = response.headers.to_h
+        Response.new(
+          body: CompressedBody.decode(response.body.to_s, headers:),
+          headers:,
+          url: response_url(response),
+          status: response.status
+        )
+      end
+
+      def response_url(response)
+        return ctx.url unless (uri = response.uri)
+
+        Html2rss::Url.from_absolute(uri.to_s)
+      end
+
+      def normalize_url(url)
+        Html2rss::Url.from_absolute(url.to_s)
+      end
+
+      def remaining_timeout_seconds(deadline)
+        remaining = deadline - monotonic_now
+        raise RequestTimedOut, 'Request timed out' if remaining <= 0
+
+        remaining
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def timeout_error?(error)
+        error.is_a?(HTTPX::TimeoutError) || super
+      end
+
+      def ssrf_error?(error)
+        error.is_a?(HTTPX::ServerSideRequestForgeryError) || super
+      end
+
+      def response_too_large_error?(error)
+        (error.is_a?(HTTPX::Error) && error.message.include?('maximum response body size exceeded')) || super
+      end
+
+      def connection_error?(error)
+        error.is_a?(HTTPX::ConnectionError) || error.is_a?(HTTPX::TLSError) || super
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/response_guard.rb

@@ -9,22 +9,6 @@ module Html2rss
       # @param policy [Policy] request policy that defines byte ceilings
       def initialize(policy:)
         @policy = policy
-        @streamed_bytes = 0
-      end
-
-      ##
-      # Validates response headers and streamed byte count.
-      #
-      # @param total_bytes [Integer] cumulative byte count received so far
-      # @param headers [Hash, nil] response headers if known
-      # @return [void]
-      # @raise [ResponseTooLarge] if the response exceeds configured limits
-      def inspect_chunk!(total_bytes:, headers: nil)
-        header_length = headers&.fetch('content-length', headers&.fetch('Content-Length', nil))
-        raise_if_too_large!(header_length.to_i, policy.max_response_bytes) if header_length
-
-        @streamed_bytes = total_bytes
-        raise_if_too_large!(@streamed_bytes, policy.max_response_bytes)
       end
 
       ##

data/lib/html2rss/request_service/strategy.rb

@@ -97,11 +97,16 @@ module Html2rss
       # @param error [StandardError]
       # @return [void]
       # @raise [StandardError]
+      # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- error translation dispatch
       def handle_error(error)
         if timeout_error?(error)
           log_timeout!(reason: 'transport')
           Log.debug("#{self.class}: transport timeout message=#{error.message}")
           raise RequestTimedOut, error.message
+        elsif ssrf_error?(error)
+          raise PrivateNetworkDenied, error.message
+        elsif response_too_large_error?(error)
+          raise ResponseTooLarge, "Response exceeded #{ctx.policy.max_response_bytes} bytes"
         elsif connection_error?(error)
           translate_connection_error(error)
         else
@@ -135,14 +140,25 @@ module Html2rss
       # @param error [StandardError]
       # @return [Boolean]
       def timeout_error?(error)
-        error.is_a?(Faraday::TimeoutError) ||
-          error.is_a?(Timeout::Error)
+        error.is_a?(Timeout::Error)
+      end
+
+      # @param _error [StandardError]
+      # @return [Boolean]
+      def ssrf_error?(_error)
+        false
+      end
+
+      # @param _error [StandardError]
+      # @return [Boolean]
+      def response_too_large_error?(_error)
+        false
       end
 
       # @param error [StandardError]
       # @return [Boolean]
       def connection_error?(error)
-        error.is_a?(Faraday::ConnectionFailed) || error.is_a?(Faraday::SSLError)
+        error.is_a?(SocketError) || error.is_a?(SystemCallError)
       end
     end
   end

data/lib/html2rss/request_service.rb

@@ -1,18 +1,13 @@
 # frozen_string_literal: true
 
-require 'singleton'
-require 'forwardable'
-
 module Html2rss
   ##
   # Requests website URLs to retrieve their HTML for further processing.
-  # Provides concrete transport strategies (e.g. Faraday, Botasaurus).
+  # Provides concrete transport strategies (e.g. HttpxStrategy, BotasaurusStrategy).
   #
   # Feed-level +:auto+ is not registered here — {FeedPipeline::StrategyPlan} resolves
   # it to a concrete strategy (or {FeedPipeline::AutoFallback} chain) before execute.
   class RequestService
-    include Singleton
-
     # Raised when an unknown request strategy is requested.
     class UnknownStrategy < Html2rss::Error; end
     # Raised when a URL cannot be parsed or validated.
@@ -59,94 +54,60 @@ module Html2rss
     # Raised when Botasaurus responds but the scrape fails (upstream error, bad payload).
     class BotasaurusServiceError < Html2rss::Error; end
 
-    class << self
-      extend Forwardable
-
-      %i[default_strategy_name
-         default_strategy_name=
-         strategy_names
-         register_strategy
-         unregister_strategy
-         strategy_registered?
-         execute].each do |method|
-        def_delegator :instance, method
-      end
-    end
-
-    def initialize
-      @strategies = {
-        faraday: FaradayStrategy,
-        botasaurus: BotasaurusStrategy,
-        local_file: LocalFileStrategy
-      }
-      @default_strategy_name = :faraday
-    end
+    # Map of supported strategy names to their implementation classes.
+    # @return [Hash{Symbol => Class<Strategy>}]
+    STRATEGIES = {
+      default: HttpxStrategy,
+      httpx: HttpxStrategy,
+      faraday: HttpxStrategy,
+      botasaurus: BotasaurusStrategy,
+      local_file: LocalFileStrategy
+    }.freeze
 
-    # @return [Symbol] the default strategy name
-    attr_reader :default_strategy_name
+    # Canonical default strategy symbol.
+    # @return [Symbol]
+    DEFAULT_STRATEGY_NAME = :default
 
-    ##
-    # Sets the default strategy.
-    # @param strategy [Symbol] the name of the strategy
-    # @return [Symbol] the selected default strategy name
-    # @raise [UnknownStrategy] if the strategy is not registered
-    def default_strategy_name=(strategy)
-      raise UnknownStrategy unless strategy_registered?(strategy)
-
-      @default_strategy_name = strategy.to_sym
-    end
+    class << self
+      # @return [Symbol] the default strategy name
+      def default_strategy_name = DEFAULT_STRATEGY_NAME
 
-    # @return [Array<String>] the names of the registered strategies
-    def strategy_names = @strategies.keys.map(&:to_s)
+      # @return [Array<String>] the names of the registered strategies
+      def strategy_names = STRATEGIES.keys.map(&:to_s)
 
-    ##
-    # Registers a new strategy.
-    # @param name [Symbol] the name of the strategy
-    # @param strategy_class [Class] the class implementing the strategy
-    # @return [Class] the registered strategy class
-    # @raise [ArgumentError] if strategy_class is not a Class
-    def register_strategy(name, strategy_class)
-      unless strategy_class.is_a?(Class)
-        raise ArgumentError, "Expected a Class for strategy, got #{strategy_class.class}"
+      ##
+      # Checks if a strategy is registered.
+      # @param name [Symbol, String] the name of the strategy
+      # @return [Boolean] true if the strategy is registered, false otherwise.
+      def strategy_registered?(name)
+        STRATEGIES.key?(name.to_sym)
       end
 
-      @strategies[name.to_sym] = strategy_class
-    end
-
-    ##
-    # Checks if a strategy is registered.
-    # @param name [Symbol] the name of the strategy
-    # @return [Boolean] true if the strategy is registered, false otherwise.
-    def strategy_registered?(name)
-      @strategies.key?(name.to_sym)
-    end
-
-    ##
-    # Unregisters a strategy.
-    # @param name [Symbol] the name of the strategy
-    # @return [Boolean] true if the strategy was unregistered, false otherwise.
-    # @raise [ArgumentError] if attempting to unregister the default strategy.
-    def unregister_strategy(name) # rubocop:disable Naming/PredicateMethod
-      name_sym = name.to_sym
-      raise ArgumentError, 'Cannot unregister the default strategy.' if name_sym == @default_strategy_name
+      ##
+      # Executes the request using the specified strategy.
+      # @param ctx [Context] the context for the request.
+      # @param strategy [Symbol, String] the strategy to use (defaults to the default strategy).
+      # @return [Response] the response from the executed strategy.
+      # @raise [UnknownStrategy] if the strategy is not registered.
+      def execute(ctx, strategy: default_strategy_name)
+        strategy_sym = strategy.to_sym
+        strategy_class = STRATEGIES.fetch(strategy_sym) do
+          raise UnknownStrategy,
+                "The strategy '#{strategy}' is not known. Available strategies: #{strategy_names.join(', ')}"
+        end
+
+        warn_migration_strategy(strategy_sym) if strategy_sym == :faraday
+        strategy_class.new(ctx).execute
+      end
 
-      !!@strategies.delete(name_sym)
-    end
+      private
 
-    ##
-    # Executes the request using the specified strategy.
-    # @param ctx [Context] the context for the request.
-    # @param strategy [Symbol] the strategy to use (defaults to the default strategy).
-    # @return [Response] the response from the executed strategy.
-    # @raise [ArgumentError] if the context is nil.
-    # @raise [UnknownStrategy] if the strategy is not registered.
-    def execute(ctx, strategy: default_strategy_name)
-      strategy_class = @strategies.fetch(strategy.to_sym) do
-        raise UnknownStrategy,
-              "The strategy '#{strategy}' is not known. Available strategies: #{strategy_names.join(', ')}"
+      def warn_migration_strategy(strategy)
+        message = "RequestService: strategy ':#{strategy}' is deprecated for migration and will be removed " \
+                  "in a future release. Use ':default' instead."
+        warn(message, category: :deprecated)
+        Log.warn(message)
       end
-
-      strategy_class.new(ctx).execute
     end
   end
 end

data/lib/html2rss/test.rb

@@ -197,7 +197,8 @@ module Html2rss
 
         channel_title = feed_result.channel_title
         channel_url = raw_config.dig(:channel, :url).to_s
-        strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] || :faraday
+        strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] ||
+                        RequestService.default_strategy_name
         min_items_passed = item_count >= min_items
         quality_failed = strict_quality && min_items_passed && quality_failure?(quality_report)
         passed = min_items_passed && !quality_failed

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.29.1'
+  VERSION = '0.30.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -33,7 +33,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   # Golden path step 1 (optional); use {.recon} for verdict and native_feed.
   #
   # @param url [String] source page URL
-  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
   # @param deep [Boolean] when true and strategy is :auto, one Botasaurus hop if configured
   # @return [Html2rss::PageRecon::Diagnostics::Report]
   def self.inspect(url, strategy: :auto, deep: false, **)
@@ -45,7 +45,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   # Golden path step 2 (optional); adds verdict beyond {.inspect}.
   #
   # @param url [String, Html2rss::Url] source page URL
-  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
   # @return [Html2rss::Recon::Result]
   def self.recon(url, strategy: :auto, **)
     Recon.call(url, strategy:, **)
@@ -56,7 +56,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   # Golden path step 3.
   #
   # @param url [String] source page URL
-  # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+  # @param strategy [Symbol] request strategy (+:auto+, +:default+, +:botasaurus+)
   # @option options [String, nil] :items_selector optional CSS selector hint for items
   # @option options [Array<String>, nil] :topics optional directory topics override
   # @option options [String, nil] :title optional title override
@@ -140,7 +140,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   # Scrapes multiple URLs in parallel using auto-source article discovery.
   #
   # @param urls [Enumerable<String>] list of URLs to scrape
-  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
   # @param limit [Integer] max articles to keep per URL (default: 10)
   # @param concurrency [Integer] max worker threads (default: 5)
   # @return [Html2rss::Batch::BatchResult]
@@ -152,7 +152,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   # Inspects multiple URLs in parallel with per-URL error isolation.
   #
   # @param urls [Enumerable<String>] list of URLs to inspect
-  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
   # @param concurrency [Integer] max worker threads (default: 5)
   # @return [Html2rss::Batch::BatchResult]
   def self.batch_inspect(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
@@ -163,7 +163,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   # Runs recon across multiple URLs in parallel with per-URL error isolation.
   #
   # @param urls [Enumerable<String>] list of URLs to recon
-  # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+  # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
   # @param concurrency [Integer] max worker threads (default: 5)
   # @option options [String, nil] :cache_dir optional HTML cache directory
   # @return [Html2rss::Batch::BatchResult]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.29.1
+  version: 0.30.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -52,53 +52,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: faraday
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: 2.0.1
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: 2.0.1
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: faraday-follow_redirects
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: faraday-gzip
+  name: httpx
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: kramdown
   requirement: !ruby/object:Gem::Requirement
@@ -431,7 +397,7 @@ files:
 - lib/html2rss/request_service/budget.rb
 - lib/html2rss/request_service/compressed_body.rb
 - lib/html2rss/request_service/context.rb
-- lib/html2rss/request_service/faraday_strategy.rb
+- lib/html2rss/request_service/httpx_strategy.rb
 - lib/html2rss/request_service/local_file_strategy.rb
 - lib/html2rss/request_service/network_guard.rb
 - lib/html2rss/request_service/policy.rb
@@ -502,7 +468,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.29.1
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.30.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: