html2rss 0.18.0 → 0.19.0

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require 'json'
+
+##
+# Main html2rss namespace.
+module Html2rss
+  ##
+  # Request transport orchestration and strategies.
+  class RequestService
+    ##
+    # Maps html2rss request/response handling to the botasaurus-scrape-api contract.
+    class BotasaurusContract
+      # Default Botasaurus scrape options when no explicit config is provided.
+      DEFAULT_OPTIONS = {
+        navigation_mode: 'auto',
+        max_retries: 2,
+        headless: false
+      }.freeze
+
+      # Allowlisted request.botasaurus keys forwarded to upstream.
+      OPTION_KEYS = %i[
+        navigation_mode
+        max_retries
+        wait_for_selector
+        wait_timeout_seconds
+        block_images
+        block_images_and_css
+        wait_for_complete_page_load
+        headless
+        proxy
+        user_agent
+        window_size
+        lang
+      ].freeze
+
+      # Parsed Botasaurus response wrapper.
+      class ParsedResponse
+        # Fallback headers when upstream omits response headers.
+        DEFAULT_HEADERS = { 'content-type' => 'text/html' }.freeze
+
+        # @param payload [Hash{String => Object}] parsed Botasaurus response payload
+        # @param transport_status [Integer] HTTP status returned by Botasaurus
+        def initialize(payload:, transport_status:)
+          @payload = payload
+          @transport_status = transport_status
+        end
+
+        # @return [Boolean] true when upstream classified request as challenge blocked
+        def challenge_block? = error_category == 'challenge_block'
+
+        # @return [Boolean] true when upstream returned non-200 or an error payload
+        def upstream_failure?
+          status != 200 || error_message?
+        end
+
+        # @return [String] normalized challenge error message
+        def challenge_message
+          error || 'Botasaurus challenge block detected.'
+        end
+
+        # @return [String] actionable upstream failure summary
+        def upstream_failure_message
+          details = ["status=#{status}"]
+          details << "error_category=#{error_category}" if error_category
+          details << "error=#{error}" if error
+          details << "request_id=#{request_id}" if request_id
+          "Botasaurus scrape failed (#{details.join(', ')})."
+        end
+
+        # @return [String] rendered HTML body from Botasaurus
+        # @raise [BotasaurusConnectionFailed] when html is missing
+        def html
+          value = payload['html']
+          raise BotasaurusConnectionFailed, "Botasaurus response missing required 'html' field" if value.nil?
+
+          value.to_s
+        end
+
+        # @return [Hash{String => String}] normalized response headers
+        def headers
+          raw_headers = payload['headers']
+          return DEFAULT_HEADERS.dup unless raw_headers.is_a?(Hash) && raw_headers.any?
+
+          raw_headers.to_h { |key, value| [key.to_s, value.to_s] }
+        end
+
+        # @return [Integer] resolved status code (payload status_code or transport status)
+        def status
+          status_code = payload['status_code']
+          status_code.is_a?(Integer) ? status_code : transport_status
+        end
+
+        # @return [String, nil] final URL reported by upstream
+        def final_url = payload['final_url']
+
+        private
+
+        attr_reader :payload, :transport_status
+
+        def error = payload['error']
+
+        def request_id = payload['request_id']
+
+        def error_category = payload['error_category']
+
+        def error_message?
+          value = error
+          value.is_a?(String) ? !value.empty? : !value.nil?
+        end
+      end
+
+      ##
+      # @param url [Html2rss::Url] canonical URL to scrape
+      # @param options [Hash] validated request.botasaurus options
+      # @option options [String] :navigation_mode
+      # @option options [Integer] :max_retries
+      # @option options [String] :wait_for_selector
+      # @option options [Integer] :wait_timeout_seconds
+      # @option options [Boolean] :block_images
+      # @option options [Boolean] :block_images_and_css
+      # @option options [Boolean] :wait_for_complete_page_load
+      # @option options [Boolean] :headless
+      # @option options [String] :proxy
+      # @option options [String] :user_agent
+      # @option options [Array<Integer>] :window_size
+      # @option options [String] :lang
+      def initialize(url:, options: {})
+        @url = url
+        @options = options
+      end
+
+      # @return [Hash] payload for POST /scrape
+      def request_payload
+        DEFAULT_OPTIONS.merge(filtered_options).merge(url: url.to_s)
+      end
+
+      # @param transport_response [Faraday::Response] upstream HTTP response
+      # @return [ParsedResponse]
+      # @raise [BotasaurusConnectionFailed] when payload is not valid JSON object
+      def parse_response(transport_response)
+        payload = JSON.parse(transport_response.body.to_s)
+        raise BotasaurusConnectionFailed, 'Botasaurus response must be a JSON object' unless payload.is_a?(Hash)
+
+        ParsedResponse.new(payload:, transport_status: transport_response.status)
+      rescue JSON::ParserError => error
+        raise BotasaurusConnectionFailed, "Botasaurus response JSON parse failed: #{error.message}"
+      end
+
+      private
+
+      attr_reader :url, :options
+
+      def filtered_options
+        OPTION_KEYS.each_with_object({}) do |key, normalized|
+          normalized[key] = options[key] if options.key?(key)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'json'
+
+module Html2rss
+  class RequestService
+    ##
+    # Strategy to delegate fetching to a Botasaurus scrape API.
+    class BotasaurusStrategy < Strategy
+      ##
+      # Executes a Botasaurus-backed request with shared request policy guards.
+      #
+      # @return [Response] normalized request response
+      # @raise [BotasaurusConfigurationError] when BOTASAURUS_SCRAPER_URL is missing or invalid
+      # @raise [BotasaurusConnectionFailed] when Botasaurus cannot be reached or returns an invalid payload
+      # @raise [RequestTimedOut] when the Botasaurus request exceeds configured timeout
+      def execute
+        validate_request!
+        transport_response = client.post('/scrape', JSON.generate(contract.request_payload), content_type_header)
+        parsed_response = contract.parse_response(transport_response)
+        raise_if_challenge_blocked!(parsed_response)
+        raise_if_upstream_failed!(parsed_response)
+        build_response(parsed_response)
+      rescue Faraday::TimeoutError, Timeout::Error => error
+        raise RequestTimedOut, error.message
+      rescue Faraday::ConnectionFailed, Faraday::SSLError => error
+        raise BotasaurusConnectionFailed, "Botasaurus connection failed: #{error.message}"
+      end
+
+      private
+
+      def validate_request!
+        ctx.budget.consume!
+        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
+      end
+
+      def build_response(parsed_response)
+        body = parsed_response.html
+        ResponseGuard.new(policy: ctx.policy).inspect_body!(body)
+
+        Response.new(
+          body:,
+          headers: parsed_response.headers,
+          url: response_url(parsed_response.final_url),
+          status: parsed_response.status
+        )
+      end
+
+      def raise_if_challenge_blocked!(parsed_response)
+        return unless parsed_response.challenge_block?
+
+        raise BlockedSurfaceDetected, "Blocked surface detected: #{parsed_response.challenge_message}"
+      end
+
+      def raise_if_upstream_failed!(parsed_response)
+        return unless parsed_response.upstream_failure?
+
+        raise BotasaurusConnectionFailed, parsed_response.upstream_failure_message
+      end
+
+      def response_url(final_url)
+        return ctx.url if final_url.nil?
+
+        Html2rss::Url.from_absolute(final_url)
+      rescue ArgumentError
+        ctx.url
+      end
+
+      def contract
+        @contract ||= BotasaurusContract.new(url: ctx.url, options: ctx.request.fetch(:botasaurus, {}))
+      end
+
+      def client
+        @client ||= Faraday.new(url: scraper_base_url.to_s, request: request_options)
+      end
+
+      def request_options
+        { timeout: ctx.policy.total_timeout_seconds }
+      end
+
+      def content_type_header
+        { 'Content-Type' => 'application/json' }
+      end
+
+      def scraper_base_url
+        @scraper_base_url ||= begin
+          configured = ENV.fetch('BOTASAURUS_SCRAPER_URL') do
+            raise BotasaurusConfigurationError, 'BOTASAURUS_SCRAPER_URL is required for strategy=botasaurus.'
+          end
+          Html2rss::Url.for_channel(configured)
+        rescue ArgumentError => error
+          raise BotasaurusConfigurationError, "BOTASAURUS_SCRAPER_URL is invalid: #{error.message}"
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/context.rb

@@ -18,8 +18,8 @@ module Html2rss
       # @raise [ArgumentError] if policy or budget is explicitly nil
       def initialize(url:, headers: {}, request: {}, **request_options)
         @url = Html2rss::Url.from_absolute(url)
-        @headers = headers
-        @request = request.freeze
+        @headers = normalize_headers(headers).freeze
+        @request = normalize_request(request).freeze
         assign_request_options(request_options)
       end
 
@@ -85,6 +85,18 @@ module Html2rss
         source = origin_url || @url
         Html2rss::Url.from_absolute(source)
       end
+
+      def normalize_headers(headers)
+        headers.to_h do |key, value|
+          [key.to_s, value]
+        end
+      end
+
+      def normalize_request(request)
+        normalized = HashUtil.deep_symbolize_keys(request, context: 'request')
+        HashUtil.assert_symbol_keys!(normalized, context: 'request')
+        normalized
+      end
     end
   end
 end

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -13,8 +13,11 @@ module Html2rss
       ##
       # Restores buffered streamed bytes so response middleware can process them.
       class StreamingBodyMiddleware < Faraday::Middleware
+        # Request-context key used to store streamed chunks before middleware completion.
         STREAM_BUFFER_KEY = :html2rss_stream_buffer
 
+        # @param env [Faraday::Env] completed response environment
+        # @return [void]
         def on_complete(env)
           buffer = env.request.context&.delete(STREAM_BUFFER_KEY)
           return if buffer.nil? || buffer.empty?
@@ -24,13 +27,12 @@ module Html2rss
       end
 
       ##
-      # NOTE: Unlike BrowserlessStrategy, Faraday does not expose the remote IP after connect.
-      # SSRF protection here is pre-connection only (DNS resolution via Policy).
-      # A DNS rebinding attack between resolution and connect cannot be caught at this layer.
-      #
       # Executes a request with runtime policy enforcement.
       #
       # @return [Response] normalized request response
+      # @note Unlike BrowserlessStrategy, Faraday does not expose the remote IP after connect.
+      #   SSRF protection here is pre-connection only (DNS resolution via Policy).
+      #   A DNS rebinding attack between resolution and connect cannot be caught at this layer.
       def execute
         deadline = request_deadline
         response_guard, response = perform_request(deadline:)

data/lib/html2rss/request_service/policy.rb

@@ -10,7 +10,9 @@ module Html2rss
     # Describes the runtime request envelope for a single feed build.
     class Policy # rubocop:disable Metrics/ClassLength
       MAX_REQUESTS_CEILING = 10
+      # Hostnames treated as local/private surfaces.
       LOCAL_HOSTS = %w[localhost localhost.localdomain metadata.google.internal].to_set.freeze
+      # IP ranges blocked when private networks are disabled.
       BLOCKED_IP_RANGES = [
         IPAddr.new('0.0.0.0/8'),
         IPAddr.new('10.0.0.0/8'),
@@ -26,6 +28,7 @@ module Html2rss
         IPAddr.new('ff00::/8')
       ].freeze
 
+      # Default policy values used when request controls are not explicitly set.
       DEFAULTS = {
         connect_timeout_seconds: 5,
         read_timeout_seconds: 10,
@@ -243,6 +246,7 @@ module Html2rss
       end
     end
 
+    # Shared immutable policy instance used for default request execution.
     Policy::DEFAULT_POLICY = Policy.new
   end
 end

data/lib/html2rss/request_service/response.rb

@@ -17,6 +17,7 @@ module Html2rss
 
         headers = headers.dup
         headers.transform_keys!(&:to_s)
+        HashUtil.assert_string_keys!(headers, context: 'response headers', deep: false)
 
         @headers = headers
         @status = status
@@ -26,7 +27,7 @@ module Html2rss
       # @return [String] the raw body of the response
       attr_reader :body
 
-      # @return [Hash<String, Object>] the headers of the response
+      # @return [Hash{String => Object}] the headers of the response
       attr_reader :headers
 
       # @return [Integer, nil] the HTTP status code when known
@@ -35,8 +36,13 @@ module Html2rss
       # @return [Html2rss::Url] the URL of the response
       attr_reader :url
 
+      # @return [String] normalized content type header value
       def content_type = header('content-type').to_s
+
+      # @return [Boolean] whether response content is JSON
       def json_response? = content_type.include?('application/json')
+
+      # @return [Boolean] whether response content is HTML
       def html_response? = content_type.include?('text/html')
 
       ##
@@ -57,6 +63,8 @@ module Html2rss
 
       private
 
+      # @param name [String] canonical header name
+      # @return [Object, nil] header value when present
       def header(name)
         headers.fetch(name) do
           headers.find { |key, _value| key.casecmp?(name) }&.last

data/lib/html2rss/request_service.rb

@@ -10,18 +10,34 @@ module Html2rss
   class RequestService
     include Singleton
 
+    # Raised when an unknown request strategy is requested.
     class UnknownStrategy < Html2rss::Error; end
+    # Raised when a URL cannot be parsed or validated.
     class InvalidUrl < Html2rss::Error; end
+    # Raised when a URL uses an unsupported scheme.
     class UnsupportedUrlScheme < Html2rss::Error; end
+    # Raised when a response type cannot be parsed.
     class UnsupportedResponseContentType < Html2rss::Error; end
+    # Raised when request limits are exceeded.
     class RequestBudgetExceeded < Html2rss::Error; end
+    # Raised when policy denies private-network access.
     class PrivateNetworkDenied < Html2rss::Error; end
+    # Raised when cross-origin follow-up requests are denied.
     class CrossOriginFollowUpDenied < Html2rss::Error; end
+    # Raised when a response exceeds configured size limits.
     class ResponseTooLarge < Html2rss::Error; end
+    # Raised when blocked content surfaces are detected.
     class BlockedSurfaceDetected < Html2rss::Error; end
+    # Raised when a request times out.
     class RequestTimedOut < Html2rss::Error; end
+    # Raised when Browserless configuration is missing or invalid.
     class BrowserlessConfigurationError < Html2rss::Error; end
+    # Raised when Browserless cannot be reached.
     class BrowserlessConnectionFailed < Html2rss::Error; end
+    # Raised when Botasaurus configuration is missing or invalid.
+    class BotasaurusConfigurationError < Html2rss::Error; end
+    # Raised when Botasaurus cannot be reached or returns invalid payloads.
+    class BotasaurusConnectionFailed < Html2rss::Error; end
 
     class << self
       extend Forwardable
@@ -40,6 +56,7 @@ module Html2rss
     def initialize
       @strategies = {
         faraday: FaradayStrategy,
+        botasaurus: BotasaurusStrategy,
         browserless: BrowserlessStrategy
       }
       @default_strategy_name = :faraday
@@ -51,6 +68,7 @@ module Html2rss
     ##
     # Sets the default strategy.
     # @param strategy [Symbol] the name of the strategy
+    # @return [Symbol] the selected default strategy name
     # @raise [UnknownStrategy] if the strategy is not registered
     def default_strategy_name=(strategy)
       raise UnknownStrategy unless strategy_registered?(strategy)
@@ -65,6 +83,7 @@ module Html2rss
     # Registers a new strategy.
     # @param name [Symbol] the name of the strategy
     # @param strategy_class [Class] the class implementing the strategy
+    # @return [Class] the registered strategy class
     # @raise [ArgumentError] if strategy_class is not a Class
     def register_strategy(name, strategy_class)
       unless strategy_class.is_a?(Class)

data/lib/html2rss/request_session/runtime_input.rb

@@ -26,8 +26,8 @@ module Html2rss
       # @param request_policy [RequestService::Policy] request policy for the session
       def initialize(url:, headers:, request:, strategy:, request_policy:)
         @url = Html2rss::Url.from_absolute(url)
-        @headers = headers.freeze
-        @request = request.freeze
+        @headers = normalize_headers(headers).freeze
+        @request = normalize_request(request).freeze
         @strategy = strategy
         @request_policy = request_policy
         freeze
@@ -52,6 +52,20 @@ module Html2rss
       ##
       # @return [RequestService::Policy] policy derived from the runtime request inputs
       attr_reader :request_policy
+
+      private
+
+      def normalize_headers(headers)
+        headers.to_h do |key, value|
+          [key.to_s, value]
+        end
+      end
+
+      def normalize_request(request)
+        normalized = HashUtil.deep_symbolize_keys(request, context: 'request')
+        HashUtil.assert_symbol_keys!(normalized, context: 'request')
+        normalized
+      end
     end
   end
 end

data/lib/html2rss/request_session/runtime_policy.rb

@@ -29,9 +29,16 @@ module Html2rss
         def baseline_request_budget_for(config)
           1 + pagination_follow_up_budget_for(config) +
             known_auto_source_follow_up_budget_for(config) +
+            auto_strategy_fallback_budget_for(config) +
             browserless_preload_budget_for(config)
         end
 
+        def auto_strategy_fallback_budget_for(config)
+          return 0 unless config.strategy == :auto
+
+          [FeedPipeline::AutoFallback::CHAIN.size - 1, 0].max
+        end
+
         def pagination_follow_up_budget_for(config)
           [config.selectors&.dig(:items, :pagination, :max_pages).to_i - 1, 0].max
         end

data/lib/html2rss/request_session.rb

@@ -9,16 +9,20 @@ module Html2rss
       # Builds a request session from translated runtime request inputs.
       #
       # @param runtime_input [RuntimeInput] translated runtime request inputs
+      # @param budget [RequestService::Budget, nil] optional shared budget for multi-attempt runs
       # @param logger [Logger] logger used for operational warnings
       # @return [RequestSession] configured request session
-      def from_runtime_input(runtime_input, logger: Html2rss::Log)
+      def from_runtime_input(runtime_input, budget: nil, logger: Html2rss::Log) # rubocop:disable Metrics/MethodLength
+        context_options = {
+          url: runtime_input.url,
+          headers: runtime_input.headers,
+          request: runtime_input.request,
+          policy: runtime_input.request_policy
+        }
+        context_options[:budget] = budget unless budget.nil?
+
         new(
-          context: RequestService::Context.new(
-            url: runtime_input.url,
-            headers: runtime_input.headers,
-            request: runtime_input.request,
-            policy: runtime_input.request_policy
-          ),
+          context: RequestService::Context.new(**context_options),
           strategy: runtime_input.strategy,
           logger:
         )
@@ -81,7 +85,7 @@ module Html2rss
     end
 
     ##
-    # @param url [String, Html2rss::Url] url to query
+    # @param url [String, Html2rss::Url] follow-up target URL for the request
     # @return [Boolean] whether the url was already visited in this session
     def visited?(url)
       visited_urls.include?(normalize_url(url))
@@ -90,7 +94,7 @@ module Html2rss
     ##
     # Records a visited url in the session.
     #
-    # @param url [String, Html2rss::Url] url to track
+    # @param url [String, Html2rss::Url] URL used to update relation tracking state
     # @return [Set<Html2rss::Url>] visited urls
     def remember!(url)
       visited_urls.add(normalize_url(url))

data/lib/html2rss/rss_builder/article.rb

@@ -13,10 +13,23 @@ module Html2rss
       include Enumerable
       include Comparable
 
+      # Allowed article attributes accepted by the value object constructor.
       PROVIDED_KEYS = %i[id title description url image author guid published_at enclosures categories scraper].freeze
+      # Separator used to build deterministic deduplication fingerprints.
       DEDUP_FINGERPRINT_SEPARATOR = '#!/'
 
-      # @param options [Hash<Symbol, String>]
+      # @param options [Hash{Symbol => String}]
+      # @option options [String] :id stable article identifier
+      # @option options [String] :title article title
+      # @option options [String] :description article description/content
+      # @option options [String, Html2rss::Url] :url canonical article URL
+      # @option options [String, Html2rss::Url] :image image URL for fallback enclosure rendering
+      # @option options [String] :author author name
+      # @option options [String] :guid explicit GUID override
+      # @option options [String, Time, DateTime] :published_at publication timestamp
+      # @option options [Array<Hash{Symbol => Object}>] :enclosures enclosure attribute hashes
+      # @option options [Array<String>] :categories category labels
+      # @option options [Class] :scraper scraper class that produced the article
       def initialize(**options)
         @to_h = {}
         options.each_pair { |key, value| @to_h[key] = value.freeze if value }
@@ -41,10 +54,13 @@ module Html2rss
         PROVIDED_KEYS.each { |key| yield(key, public_send(key)) }
       end
 
+      # @return [String, nil] stable article identifier
       def id = blank_string_to_nil(@to_h[:id])
 
+      # @return [String, nil] article title
       def title = blank_string_to_nil(@to_h[:title])
 
+      # @return [String] rendered article description
       def description
         @description ||= Rendering::DescriptionBuilder.new(
           base: @to_h[:description],
@@ -82,6 +98,7 @@ module Html2rss
         dedup_from_url || dedup_from_id || dedup_from_guid || hash
       end
 
+      # @return [Array<Html2rss::RssBuilder::Enclosure>] normalized enclosure objects
       def enclosures
         @enclosures ||= Array(@to_h[:enclosures])
                         .map { |enclosure| Html2rss::RssBuilder::Enclosure.new(**enclosure) }
@@ -101,6 +118,7 @@ module Html2rss
         end
       end
 
+      # @return [Array<String>] normalized, unique category names
       def categories
         @categories ||= @to_h[:categories].dup.to_a.tap do |categories|
           categories.map! { |category| category.to_s.strip }
@@ -119,10 +137,13 @@ module Html2rss
         nil
       end
 
+      # @return [Class, nil] scraper class that produced this article
       def scraper
         @to_h[:scraper]
       end
 
+      # @param other [Object] value compared against this article
+      # @return [Integer, nil] comparison result for compatible Article values
       def <=>(other)
         return nil unless other.is_a?(Article)
 

data/lib/html2rss/rss_builder/channel.rb

@@ -7,24 +7,28 @@ module Html2rss
     # 1. the HTML document's <head>.
     # 2. the HTTP response
     class Channel
+      # Fallback RSS ttl (in minutes) when no cache directives are present.
       DEFAULT_TTL_IN_MINUTES = 360
+      # Description template used when no explicit or discovered description exists.
       DEFAULT_DESCRIPTION_TEMPLATE = 'Latest items from %<url>s'
 
       ##
-      #
       # @param response [Html2rss::RequestService::Response]
-      # @param overrides [Hash<Symbol, String>] - Optional, overrides for any channel attribute
+      # @param overrides [Hash{Symbol => String}] optional overrides for channel attributes
       def initialize(response, overrides: {})
         @response = response
         @overrides = overrides
       end
 
+      # @return [String] channel title derived from overrides, document title, or URL
       def title
         @title ||= fetch_title
       end
 
+      # @return [Html2rss::Url] canonical channel URL
       def url = @url ||= Html2rss::Url.from_absolute(@response.url)
 
+      # @return [String] channel description text
       def description
         return overrides[:description] unless overrides[:description].to_s.empty?
 
@@ -35,6 +39,7 @@ module Html2rss
         description
       end
 
+      # @return [Integer] cache time-to-live in minutes
       def ttl
         return overrides[:ttl] if overrides[:ttl]
 
@@ -45,6 +50,7 @@ module Html2rss
         DEFAULT_TTL_IN_MINUTES
       end
 
+      # @return [String, nil] ISO-like language code when available
       def language
         return overrides[:language] if overrides[:language]
 
@@ -57,6 +63,7 @@ module Html2rss
         parsed_body['lang'] || parsed_body.at_css('[lang]')&.[]('lang')
       end
 
+      # @return [String, nil] channel author metadata
       def author
         return overrides[:author] if overrides[:author]
 
@@ -65,8 +72,10 @@ module Html2rss
         parsed_body.at_css('meta[name="author"]')&.[]('content')
       end
 
+      # @return [String, Time] source last-modified timestamp or current time fallback
       def last_build_date = headers['last-modified'] || Time.now
 
+      # @return [Html2rss::Url, nil] channel image URL
       def image
         return overrides[:image] if overrides[:image]