html2rss 0.17.0 → 0.19.0

data/lib/html2rss/request_service/response.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'nokogiri'
+
 module Html2rss
   class RequestService
     ##
@@ -7,21 +9,67 @@ module Html2rss
     class Response
       ##
       # @param body [String] the body of the response
+      # @param url [Html2rss::Url] the final request URL
       # @param headers [Hash] the headers of the response
-      def initialize(body:, headers: {})
+      # @param status [Integer, nil] the HTTP status code when available
+      def initialize(body:, url:, headers: {}, status: nil)
         @body = body
 
         headers = headers.dup
         headers.transform_keys!(&:to_s)
+        HashUtil.assert_string_keys!(headers, context: 'response headers', deep: false)
 
         @headers = headers
+        @status = status
+        @url = url
       end
 
-      # @return [String] the body of the response
+      # @return [String] the raw body of the response
       attr_reader :body
 
-      # @return [Hash<String, Object>] the headers of the response
+      # @return [Hash{String => Object}] the headers of the response
       attr_reader :headers
+
+      # @return [Integer, nil] the HTTP status code when known
+      attr_reader :status
+
+      # @return [Html2rss::Url] the URL of the response
+      attr_reader :url
+
+      # @return [String] normalized content type header value
+      def content_type = header('content-type').to_s
+
+      # @return [Boolean] whether response content is JSON
+      def json_response? = content_type.include?('application/json')
+
+      # @return [Boolean] whether response content is HTML
+      def html_response? = content_type.include?('text/html')
+
+      ##
+      # @return [Nokogiri::HTML::Document, Hash] the parsed body of the response, frozen object
+      # @raise [UnsupportedResponseContentType] if the content type is not supported
+      def parsed_body
+        @parsed_body ||= if html_response?
+                           Nokogiri::HTML(body).tap do |doc|
+                             # Remove comments from the document to avoid processing irrelevant content
+                             doc.xpath('//comment()').each(&:remove)
+                           end.freeze
+                         elsif json_response?
+                           JSON.parse(body, symbolize_names: true).freeze
+                         else
+                           raise UnsupportedResponseContentType, "Unsupported content type: #{content_type}"
+                         end
+      end
+
+      private
+
+      # @param name [String] canonical header name
+      # @return [Object, nil] header value when present
+      def header(name)
+        headers.fetch(name) do
+          headers.find { |key, _value| key.casecmp?(name) }&.last
+        end
+      end
     end
   end
 end

data/lib/html2rss/request_service/response_guard.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Enforces response-size limits before parsing.
+    class ResponseGuard
+      ##
+      # @param policy [Policy] request policy that defines byte ceilings
+      def initialize(policy:)
+        @policy = policy
+        @streamed_bytes = 0
+      end
+
+      ##
+      # Validates response headers and streamed byte count.
+      #
+      # @param total_bytes [Integer] cumulative byte count received so far
+      # @param headers [Hash, nil] response headers if known
+      # @return [void]
+      # @raise [ResponseTooLarge] if the response exceeds configured limits
+      def inspect_chunk!(total_bytes:, headers: nil)
+        header_length = headers&.fetch('content-length', headers&.fetch('Content-Length', nil))
+        raise_if_too_large!(header_length.to_i, policy.max_response_bytes) if header_length
+
+        @streamed_bytes = total_bytes
+        raise_if_too_large!(@streamed_bytes, policy.max_response_bytes)
+      end
+
+      ##
+      # Validates the final response body after middleware processing.
+      #
+      # @param body [String, nil] final response body
+      # @return [void]
+      # @raise [ResponseTooLarge] if the final body exceeds configured limits
+      # @raise [BlockedSurfaceDetected] if the body matches known anti-bot interstitial signatures
+      def inspect_body!(body)
+        normalized_body = body.to_s
+        size = normalized_body.bytesize
+        raise_if_too_large!(size, policy.max_decompressed_bytes)
+        raise_if_blocked_surface!(normalized_body)
+      end
+
+      private
+
+      attr_reader :policy
+
+      def raise_if_blocked_surface!(body)
+        signature = Html2rss::BlockedSurface.interstitial_signature_for(body)
+        return unless signature
+
+        raise BlockedSurfaceDetected, signature.fetch(:message)
+      end
+
+      def raise_if_too_large!(bytes, limit)
+        return unless bytes > limit
+
+        raise ResponseTooLarge, "Response exceeded #{limit} bytes"
+      end
+    end
+  end
+end

data/lib/html2rss/request_service.rb

@@ -6,13 +6,38 @@ require 'forwardable'
 module Html2rss
   ##
   # Requests website URLs to retrieve their HTML for further processing.
-  # Provides strategies, i.e. to integrate Browserless.io.
+  # Provides strategies, e.g. integrating Browserless.io.
   class RequestService
     include Singleton
 
+    # Raised when an unknown request strategy is requested.
     class UnknownStrategy < Html2rss::Error; end
+    # Raised when a URL cannot be parsed or validated.
     class InvalidUrl < Html2rss::Error; end
+    # Raised when a URL uses an unsupported scheme.
     class UnsupportedUrlScheme < Html2rss::Error; end
+    # Raised when a response type cannot be parsed.
+    class UnsupportedResponseContentType < Html2rss::Error; end
+    # Raised when request limits are exceeded.
+    class RequestBudgetExceeded < Html2rss::Error; end
+    # Raised when policy denies private-network access.
+    class PrivateNetworkDenied < Html2rss::Error; end
+    # Raised when cross-origin follow-up requests are denied.
+    class CrossOriginFollowUpDenied < Html2rss::Error; end
+    # Raised when a response exceeds configured size limits.
+    class ResponseTooLarge < Html2rss::Error; end
+    # Raised when blocked content surfaces are detected.
+    class BlockedSurfaceDetected < Html2rss::Error; end
+    # Raised when a request times out.
+    class RequestTimedOut < Html2rss::Error; end
+    # Raised when Browserless configuration is missing or invalid.
+    class BrowserlessConfigurationError < Html2rss::Error; end
+    # Raised when Browserless cannot be reached.
+    class BrowserlessConnectionFailed < Html2rss::Error; end
+    # Raised when Botasaurus configuration is missing or invalid.
+    class BotasaurusConfigurationError < Html2rss::Error; end
+    # Raised when Botasaurus cannot be reached or returns invalid payloads.
+    class BotasaurusConnectionFailed < Html2rss::Error; end
 
     class << self
       extend Forwardable
@@ -31,6 +56,7 @@ module Html2rss
     def initialize
       @strategies = {
         faraday: FaradayStrategy,
+        botasaurus: BotasaurusStrategy,
         browserless: BrowserlessStrategy
       }
       @default_strategy_name = :faraday
@@ -42,6 +68,7 @@ module Html2rss
     ##
     # Sets the default strategy.
     # @param strategy [Symbol] the name of the strategy
+    # @return [Symbol] the selected default strategy name
     # @raise [UnknownStrategy] if the strategy is not registered
     def default_strategy_name=(strategy)
       raise UnknownStrategy unless strategy_registered?(strategy)
@@ -55,9 +82,13 @@ module Html2rss
     ##
     # Registers a new strategy.
     # @param name [Symbol] the name of the strategy
-    # @param strategy_class [Class] the class of the strategy
+    # @param strategy_class [Class] the class implementing the strategy
+    # @return [Class] the registered strategy class
+    # @raise [ArgumentError] if strategy_class is not a Class
     def register_strategy(name, strategy_class)
-      raise ArgumentError, 'Strategy class must be a Class' unless strategy_class.is_a?(Class)
+      unless strategy_class.is_a?(Class)
+        raise ArgumentError, "Expected a Class for strategy, got #{strategy_class.class}"
+      end
 
       @strategies[name.to_sym] = strategy_class
     end
@@ -65,7 +96,7 @@ module Html2rss
     ##
     # Checks if a strategy is registered.
     # @param name [Symbol] the name of the strategy
-    # @return [Boolean] true if the strategy is registered, false otherwise
+    # @return [Boolean] true if the strategy is registered, false otherwise.
     def strategy_registered?(name)
       @strategies.key?(name.to_sym)
     end
@@ -73,24 +104,28 @@ module Html2rss
     ##
     # Unregisters a strategy.
     # @param name [Symbol] the name of the strategy
-    # @return [Boolean] true if the strategy was unregistered, false otherwise
-    def unregister_strategy(name)
-      raise ArgumentError, 'Cannot unregister the default strategy' if name.to_sym == @default_strategy_name
+    # @return [Boolean] true if the strategy was unregistered, false otherwise.
+    # @raise [ArgumentError] if attempting to unregister the default strategy.
+    def unregister_strategy(name) # rubocop:disable Naming/PredicateMethod
+      name_sym = name.to_sym
+      raise ArgumentError, 'Cannot unregister the default strategy.' if name_sym == @default_strategy_name
 
-      !!@strategies.delete(name.to_sym)
+      !!@strategies.delete(name_sym)
     end
 
     ##
-    # Executes the request.
-    # @param ctx [Context] the context for the request
-    # @param strategy [Symbol] the strategy to use
-    # @return [Response] the response from the strategy
-    # @raise [UnknownStrategy] if the strategy is not known
+    # Executes the request using the specified strategy.
+    # @param ctx [Context] the context for the request.
+    # @param strategy [Symbol] the strategy to use (defaults to the default strategy).
+    # @return [Response] the response from the executed strategy.
+    # @raise [ArgumentError] if the context is nil.
+    # @raise [UnknownStrategy] if the strategy is not registered.
     def execute(ctx, strategy: default_strategy_name)
-      strategy_class = @strategies.fetch(strategy) do
+      strategy_class = @strategies.fetch(strategy.to_sym) do
         raise UnknownStrategy,
-              "The strategy '#{strategy}' is not known. Available strategies are: #{strategy_names.join(', ')}"
+              "The strategy '#{strategy}' is not known. Available strategies: #{strategy_names.join(', ')}"
       end
+
       strategy_class.new(ctx).execute
     end
   end

data/lib/html2rss/request_session/rel_next_pager.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    ##
+    # Traverses a rel=next pagination chain for selector-driven extraction.
+    class RelNextPager
+      include Enumerable
+
+      ##
+      # @param session [RequestSession] request session used to execute follow-ups
+      # @param initial_response [RequestService::Response] first page response
+      # @param max_pages [Integer] configured page budget, including the initial page
+      # @param logger [Logger] logger used for pagination stop reasons
+      def initialize(session:, initial_response:, max_pages:, logger: Html2rss::Log)
+        @session = session
+        @initial_response = initial_response
+        @max_pages = max_pages
+        @logger = logger
+      end
+
+      ##
+      # Iterates over all paginated responses, beginning with the initial response.
+      #
+      # @yield [RequestService::Response] each page response
+      # @return [Enumerator] enumerator when no block is given
+      def each
+        return enum_for(:each) unless block_given?
+
+        yield initial_response
+
+        current_response = initial_response
+        session.effective_page_budget(max_pages).pred.times do
+          next_url = next_page_url(current_response)
+          break unless follow_up_allowed?(next_url)
+
+          current_response = fetch_follow_up_response_or_stop(next_url, current_response.url)
+          break unless current_response
+
+          yield current_response
+        end
+      end
+
+      private
+
+      attr_reader :session, :initial_response, :max_pages, :logger
+
+      def next_page_url(page_response)
+        href = page_response.parsed_body.at_css('link[rel~="next"][href], a[rel~="next"][href]')&.[]('href')
+        return nil if href.nil? || href.empty?
+
+        Html2rss::Url.from_relative(href, page_response.url)
+      end
+
+      def follow_up_allowed?(next_url)
+        next_url && !session.visited?(next_url)
+      end
+
+      def fetch_follow_up_response_or_stop(next_url, origin_url)
+        session.follow_up(url: next_url, relation: :pagination, origin_url:)
+      rescue RequestService::RequestBudgetExceeded => error
+        logger.warn(
+          "#{self.class}: pagination stopped at #{next_url} - #{error.message}. " \
+          "Retry with --max-requests #{session.max_requests + 1} or increase request.max_requests in the config."
+        )
+        nil
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/runtime_input.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    ##
+    # Carries the runtime request inputs needed to build a RequestSession.
+    class RuntimeInput
+      ##
+      # @param config [Html2rss::Config] validated feed config
+      # @return [RuntimeInput] runtime request inputs derived from the config
+      def self.from_config(config)
+        new(
+          url: config.url,
+          headers: config.headers,
+          request: config.request,
+          strategy: config.strategy,
+          request_policy: RuntimePolicy.from_config(config)
+        )
+      end
+
+      ##
+      # @param url [String, Html2rss::Url] initial request URL
+      # @param headers [Hash] normalized request headers
+      # @param request [Hash] validated request options for strategies
+      # @param strategy [Symbol] request strategy to use for the session
+      # @param request_policy [RequestService::Policy] request policy for the session
+      def initialize(url:, headers:, request:, strategy:, request_policy:)
+        @url = Html2rss::Url.from_absolute(url)
+        @headers = normalize_headers(headers).freeze
+        @request = normalize_request(request).freeze
+        @strategy = strategy
+        @request_policy = request_policy
+        freeze
+      end
+
+      ##
+      # @return [Html2rss::Url] initial request URL
+      attr_reader :url
+
+      ##
+      # @return [Hash] normalized request headers
+      attr_reader :headers
+
+      ##
+      # @return [Hash] validated request options for strategies
+      attr_reader :request
+
+      ##
+      # @return [Symbol] request strategy to use for the session
+      attr_reader :strategy
+
+      ##
+      # @return [RequestService::Policy] policy derived from the runtime request inputs
+      attr_reader :request_policy
+
+      private
+
+      def normalize_headers(headers)
+        headers.to_h do |key, value|
+          [key.to_s, value]
+        end
+      end
+
+      def normalize_request(request)
+        normalized = HashUtil.deep_symbolize_keys(request, context: 'request')
+        HashUtil.assert_symbol_keys!(normalized, context: 'request')
+        normalized
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/runtime_policy.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    ##
+    # Builds the runtime request policy for a feed run.
+    class RuntimePolicy
+      ##
+      # @param config [Html2rss::Config] validated feed config
+      # @return [Html2rss::RequestService::Policy] request policy derived from runtime config
+      def self.from_config(config)
+        RequestService::Policy.new(
+          max_requests: effective_max_requests_for(config),
+          max_redirects: config.max_redirects
+        )
+      end
+
+      class << self
+        private
+
+        def effective_max_requests_for(config)
+          return config.max_requests if config.explicit_max_requests?
+
+          [baseline_request_budget_for(config), config.max_requests].max
+        end
+
+        # Reserve enough budget for the initial request plus predictable follow-ups
+        # that the top-level pipeline may trigger during a normal feed build.
+        def baseline_request_budget_for(config)
+          1 + pagination_follow_up_budget_for(config) +
+            known_auto_source_follow_up_budget_for(config) +
+            auto_strategy_fallback_budget_for(config) +
+            browserless_preload_budget_for(config)
+        end
+
+        def auto_strategy_fallback_budget_for(config)
+          return 0 unless config.strategy == :auto
+
+          [FeedPipeline::AutoFallback::CHAIN.size - 1, 0].max
+        end
+
+        def pagination_follow_up_budget_for(config)
+          [config.selectors&.dig(:items, :pagination, :max_pages).to_i - 1, 0].max
+        end
+
+        def known_auto_source_follow_up_budget_for(config)
+          config.auto_source&.dig(:scraper, :wordpress_api, :enabled) ? 1 : 0
+        end
+
+        def browserless_preload_budget_for(config)
+          preload = config.request.dig(:browserless, :preload)
+          return 0 unless preload
+
+          top_level_preload_wait_budget(preload) +
+            click_selector_preload_budget(preload) +
+            scroll_preload_budget(preload)
+        end
+
+        def top_level_preload_wait_budget(preload)
+          preload[:wait_after_ms] ? 2 : 0
+        end
+
+        def click_selector_preload_budget(preload)
+          preload.fetch(:click_selectors, []).sum { preload_action_budget(_1, :max_clicks) }
+        end
+
+        def scroll_preload_budget(preload)
+          scroll = preload[:scroll_down]
+          return 0 unless scroll
+
+          preload_action_budget(scroll, :iterations)
+        end
+
+        def preload_action_budget(config, count_key)
+          action_count = config.fetch(count_key, 1)
+          wait_budget = config[:wait_after_ms] ? action_count : 0
+
+          action_count + wait_budget
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Coordinates multi-request feed builds on top of RequestService.
+  class RequestSession
+    class << self
+      ##
+      # Builds a request session from translated runtime request inputs.
+      #
+      # @param runtime_input [RuntimeInput] translated runtime request inputs
+      # @param budget [RequestService::Budget, nil] optional shared budget for multi-attempt runs
+      # @param logger [Logger] logger used for operational warnings
+      # @return [RequestSession] configured request session
+      def from_runtime_input(runtime_input, budget: nil, logger: Html2rss::Log) # rubocop:disable Metrics/MethodLength
+        context_options = {
+          url: runtime_input.url,
+          headers: runtime_input.headers,
+          request: runtime_input.request,
+          policy: runtime_input.request_policy
+        }
+        context_options[:budget] = budget unless budget.nil?
+
+        new(
+          context: RequestService::Context.new(**context_options),
+          strategy: runtime_input.strategy,
+          logger:
+        )
+      end
+    end
+
+    ##
+    # @param context [RequestService::Context] initial request context
+    # @param strategy [Symbol] request strategy to use for all requests in the session
+    # @param logger [Logger] logger used for operational warnings
+    def initialize(context:, strategy:, logger: Html2rss::Log)
+      @context = context
+      @strategy = strategy
+      @logger = logger
+      @visited_urls = Set.new
+    end
+
+    ##
+    # Executes the initial request for the session.
+    #
+    # @return [RequestService::Response] initial response
+    def fetch_initial_response
+      execute(context).tap { |response| remember!(response.url) }
+    end
+
+    ##
+    # Executes a follow-up request sharing policy, headers, and budget.
+    #
+    # @param url [String, Html2rss::Url] follow-up request url
+    # @param relation [Symbol] why the follow-up is being made
+    # @param origin_url [String, Html2rss::Url] effective origin for same-origin checks
+    # @return [RequestService::Response] follow-up response
+    def follow_up(url:, relation:, origin_url:)
+      execute(context.follow_up(url:, relation:, origin_url:)).tap { |response| remember!(response.url) }
+    end
+
+    ##
+    # Returns the effective page budget after applying the policy ceiling.
+    #
+    # @param requested_pages [Integer] configured page budget
+    # @return [Integer] effective page budget for the session
+    def effective_page_budget(requested_pages)
+      effective_pages = [requested_pages, context.policy.max_requests].min
+      return effective_pages if effective_pages == requested_pages
+
+      logger.warn(
+        "#{self.class}: pagination max_pages=#{requested_pages} " \
+        "exceeds system ceiling=#{context.policy.max_requests}; " \
+        "clamping to #{effective_pages}"
+      )
+      effective_pages
+    end
+
+    ##
+    # Returns the configured request budget for the session.
+    #
+    # @return [Integer] maximum requests allowed for the feed build
+    def max_requests
+      context.policy.max_requests
+    end
+
+    ##
+    # @param url [String, Html2rss::Url] follow-up target URL for the request
+    # @return [Boolean] whether the url was already visited in this session
+    def visited?(url)
+      visited_urls.include?(normalize_url(url))
+    end
+
+    ##
+    # Records a visited url in the session.
+    #
+    # @param url [String, Html2rss::Url] URL used to update relation tracking state
+    # @return [Set<Html2rss::Url>] visited urls
+    def remember!(url)
+      visited_urls.add(normalize_url(url))
+    end
+
+    private
+
+    attr_reader :context, :strategy, :logger, :visited_urls
+
+    def execute(request_context)
+      RequestService.execute(request_context, strategy:).tap do |response|
+        logger.debug(
+          "#{self.class}: relation=#{request_context.relation} " \
+          "request_url=#{request_context.url} final_url=#{response.url} " \
+          "status=#{response.status || 'unknown'} content_type=#{response.content_type.inspect} " \
+          "bytes=#{response.body.bytesize}"
+        )
+      end
+    end
+
+    def normalize_url(url)
+      Html2rss::Url.from_absolute(url)
+    end
+  end
+end