html2rss 0.22.2 → 0.23.0

data/lib/html2rss/request_service/blocked_surface.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Shared anti-bot/interstitial signatures used by request and auto-source flows.
+    #
+    # This module centralizes signature matching so request-time guards and
+    # auto-source surface classification stay consistent.
+    module BlockedSurface
+      # Known interstitial fingerprints used to detect blocked or anti-bot surfaces.
+      INTERSTITIAL_SIGNATURES = [
+        {
+          key: :cloudflare_interstitial,
+          min_matches: 2,
+          patterns: [
+            %r{<title>\s*just a moment\.\.\.\s*</title>}i,
+            /checking your browser before accessing/i,
+            /please (?:enable|turn on) javascript and cookies/i,
+            %r{cdn-cgi/challenge-platform}i,
+            /cloudflare ray id/i
+          ],
+          message: 'Blocked surface detected: Cloudflare anti-bot interstitial page. ' \
+                   'Retry with --strategy browserless, try a more specific public listing URL, ' \
+                   'or run from an environment that can complete anti-bot checks.'
+        }
+      ].freeze
+
+      ##
+      # Returns the first matching interstitial signature for the provided body.
+      #
+      # @param body [String, nil] response body candidate
+      # @return [Hash, nil] signature hash when matched, otherwise nil
+      def self.interstitial_signature_for(body)
+        normalized_body = normalize_body(body)
+        INTERSTITIAL_SIGNATURES.find { |signature| interstitial_signature_match?(normalized_body, signature) }
+      end
+
+      ##
+      # @param body [String, nil] response body candidate
+      # @return [Boolean] true when body matches a known interstitial signature
+      def self.interstitial?(body)
+        !interstitial_signature_for(body).nil?
+      end
+
+      def self.interstitial_signature_match?(body, signature)
+        min_matches = signature.fetch(:min_matches, 1)
+        matches = 0
+
+        signature.fetch(:patterns).each do |pattern|
+          matches += 1 if pattern.match?(body)
+          return true if matches >= min_matches
+        end
+
+        false
+      end
+      private_class_method :interstitial_signature_match?
+
+      def self.normalize_body(body)
+        body.to_s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
+      rescue Encoding::CompatibilityError, Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+        body.to_s.force_encoding(Encoding::UTF_8).scrub
+      end
+      private_class_method :normalize_body
+    end
+  end
+end

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -2,11 +2,7 @@
 
 require 'json'
 
-##
-# Main html2rss namespace.
 module Html2rss
-  ##
-  # Request transport orchestration and strategies.
   class RequestService
     ##
     # Maps html2rss request/response handling to the botasaurus-scrape-api contract.

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -8,44 +8,23 @@ module Html2rss
     ##
     # Strategy to delegate fetching to a Botasaurus scrape API.
     class BotasaurusStrategy < Strategy
-      ##
-      # Executes a Botasaurus-backed request with shared request policy guards.
-      #
-      # @return [Response] normalized request response
-      # @raise [BotasaurusConfigurationError] when BOTASAURUS_SCRAPER_URL is missing or invalid
-      # @raise [BotasaurusConnectionFailed] when Botasaurus cannot be reached or returns an invalid payload
-      # @raise [RequestTimedOut] when the Botasaurus request exceeds configured timeout
-      def execute
-        check_timeout!
-        validate_request!
+      private
+
+      def fetch
         parsed_response = post_scrape_request
         raise_if_challenge_blocked!(parsed_response)
         raise_if_upstream_failed!(parsed_response)
         build_response(parsed_response)
-      rescue Faraday::TimeoutError, Timeout::Error => error
-        raise RequestTimedOut, error.message
-      rescue Faraday::ConnectionFailed, Faraday::SSLError => error
-        raise BotasaurusConnectionFailed, "Botasaurus connection failed: #{error.message}"
       end
 
-      private
-
       def post_scrape_request
         transport_response = client.post('/scrape', JSON.generate(contract.request_payload), content_type_header)
         contract.parse_response(transport_response)
       end
 
-      def validate_request!
-        ctx.budget.consume!
-        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
-      end
-
       def build_response(parsed_response)
-        body = parsed_response.html
-        ResponseGuard.new(policy: ctx.policy).inspect_body!(body)
-
         Response.new(
-          body:,
+          body: parsed_response.html,
           headers: parsed_response.headers,
           url: response_url(parsed_response.final_url),
           status: parsed_response.status
@@ -81,8 +60,7 @@ module Html2rss
       end
 
       def request_options
-        timeout = ctx.budget.remaining_timeout_seconds || ctx.policy.total_timeout_seconds
-        raise RequestTimedOut, 'Request timed out' if timeout <= 0
+        timeout = ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
 
         { timeout: timeout.to_i }
       end
@@ -101,6 +79,10 @@ module Html2rss
           raise BotasaurusConfigurationError, "BOTASAURUS_SCRAPER_URL is invalid: #{error.message}"
         end
       end
+
+      def translate_connection_error(error)
+        raise BotasaurusConnectionFailed, "Botasaurus connection failed: #{error.message}"
+      end
     end
   end
 end

data/lib/html2rss/request_service/browserless_strategy.rb

@@ -31,19 +31,6 @@ module Html2rss
     # are aligned with the default values.
     # @see https://github.com/browserless/browserless/pkgs/container/chromium
     class BrowserlessStrategy < Strategy
-      ##
-      # Executes a Browserless-backed request with the shared request policy.
-      #
-      # @return [Response] normalized request response
-      # @raise [RequestTimedOut] if the browser session exceeds the configured timeout
-      def execute
-        check_timeout!
-        validate_request!
-        execute_browserless_request
-      rescue Puppeteer::TimeoutError => error
-        raise RequestTimedOut, error.message
-      end
-
       ##
       # @return [String] the Browserless websocket endpoint with token query param
       # @raise [ArgumentError] if a custom endpoint is configured without an API token
@@ -58,12 +45,7 @@ module Html2rss
 
       private
 
-      def validate_request!
-        ctx.budget.consume!
-        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
-      end
-
-      def execute_browserless_request
+      def fetch
         connect_with_timeout_support do |browser|
           PuppetCommander.new(ctx, browser).call
         ensure
@@ -72,10 +54,7 @@ module Html2rss
       end
 
       def protocol_timeout_ms
-        timeout = ctx.budget.remaining_timeout_seconds || ctx.policy.total_timeout_seconds
-        raise RequestTimedOut, 'Request timed out' if timeout <= 0
-
-        (timeout * 1000).to_i
+        ctx.budget.effective_timeout_ms(fallback: ctx.policy.total_timeout_seconds)
       end
 
       def connect_with_timeout_support(&)

data/lib/html2rss/request_service/budget.rb

@@ -3,40 +3,78 @@
 module Html2rss
   class RequestService
     ##
-    # Tracks how many outbound requests a single feed build may still perform.
+    # Tracks request slots, interaction budget, and wall-clock deadline for one feed build.
+    #
+    # HTTP fetches consume request slots; Browserless preload clicks/scrolls/waits consume
+    # interaction budget. Deadline remains on `#remaining_timeout_seconds`.
     class Budget
       ##
-      ##
-      # @param max_requests [Integer] the maximum number of requests allowed
-      # @param total_timeout_seconds [Integer, nil] the total timeout for the feed build
-      def initialize(max_requests:, total_timeout_seconds: nil)
-        unless max_requests.is_a?(Integer) && max_requests.positive?
-          raise ArgumentError, 'max_requests must be positive'
-        end
+      # @param max_requests [Integer] maximum HTTP request slots
+      # @param max_interactions [Integer] maximum preload interaction slots (default 0)
+      # @param total_timeout_seconds [Integer, nil] wall-clock timeout for the feed build
+      def initialize(max_requests:, max_interactions: 0, total_timeout_seconds: nil)
+        validate_slot_limits!(max_requests:, max_interactions:)
 
-        @remaining = max_requests
+        @remaining_requests = max_requests
+        @remaining_interactions = max_interactions
         @start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         @total_timeout_seconds = total_timeout_seconds
         @mutex = Mutex.new
       end
 
       ##
-      # Consumes one request from the budget.
+      # Consumes one HTTP request slot.
       #
-      # @return [Integer] remaining request count after consumption
-      # @raise [RequestBudgetExceeded] if no requests remain
+      # @return [Integer] remaining request slots after consumption
+      # @raise [RequestBudgetExceeded] if no request slots remain
       def consume!
+        consume_request!
+      end
+
+      ##
+      # Consumes one HTTP request slot.
+      #
+      # @return [Integer] remaining request slots after consumption
+      # @raise [RequestBudgetExceeded] if no request slots remain
+      def consume_request!
+        @mutex.synchronize do
+          raise RequestBudgetExceeded, 'Request budget exhausted' if @remaining_requests.zero?
+
+          @remaining_requests -= 1
+        end
+      end
+
+      ##
+      # Consumes one Browserless preload interaction slot.
+      #
+      # Preload must not steal pagination HTTP request slots.
+      #
+      # @return [Integer] remaining interaction slots after consumption
+      # @raise [InteractionBudgetExceeded] if no interaction slots remain
+      def consume_interaction!
         @mutex.synchronize do
-          raise RequestBudgetExceeded, 'Request budget exhausted' if @remaining.zero?
+          raise InteractionBudgetExceeded, 'Interaction budget exhausted' if @remaining_interactions.zero?
 
-          @remaining -= 1
+          @remaining_interactions -= 1
         end
       end
 
       ##
-      # @return [Integer] requests still available
+      # @return [Integer] HTTP request slots still available
       def remaining
-        @mutex.synchronize { @remaining }
+        remaining_requests
+      end
+
+      ##
+      # @return [Integer] HTTP request slots still available
+      def remaining_requests
+        @mutex.synchronize { @remaining_requests }
+      end
+
+      ##
+      # @return [Integer] preload interaction slots still available
+      def remaining_interactions
+        @mutex.synchronize { @remaining_interactions }
       end
 
       ##
@@ -48,6 +86,42 @@ module Html2rss
         remaining = @total_timeout_seconds - elapsed
         [remaining, 0.0].max
       end
+
+      ##
+      # Resolves wall-clock seconds for the next adapter attempt.
+      #
+      # Prefers the tracked deadline remainder; falls back to the policy total when
+      # this Budget was constructed without a timeout (common in unit tests).
+      #
+      # @param fallback [Numeric] policy total timeout when remaining is untracked
+      # @return [Float] seconds available for this attempt
+      # @raise [RequestTimedOut] when no time remains
+      def effective_timeout_seconds(fallback:)
+        timeout = remaining_timeout_seconds || fallback
+        raise RequestTimedOut, 'Request timed out' if timeout <= 0
+
+        timeout.to_f
+      end
+
+      ##
+      # @param fallback [Numeric] policy total timeout when remaining is untracked
+      # @return [Integer] milliseconds available for this attempt
+      # @raise [RequestTimedOut] when no time remains
+      def effective_timeout_ms(fallback:)
+        (effective_timeout_seconds(fallback:) * 1000).to_i
+      end
+
+      private
+
+      def validate_slot_limits!(max_requests:, max_interactions:)
+        unless max_requests.is_a?(Integer) && max_requests.positive?
+          raise ArgumentError,
+                'max_requests must be positive'
+        end
+        return if max_interactions.is_a?(Integer) && !max_interactions.negative?
+
+        raise ArgumentError, 'max_interactions must be a non-negative integer'
+      end
     end
   end
 end

data/lib/html2rss/request_service/context.rb

@@ -77,9 +77,7 @@ module Html2rss
         raise ArgumentError, 'policy must not be nil' if @policy.nil?
 
         @origin_url = normalized_origin_url(request_options[:origin_url])
-        @budget = request_options.fetch(:budget) do
-          Budget.new(max_requests: policy.max_requests, total_timeout_seconds: policy.total_timeout_seconds)
-        end
+        @budget = request_options.fetch(:budget) { raise ArgumentError, 'budget is required' }
         raise ArgumentError, 'budget must not be nil' if @budget.nil?
       end
 

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -27,33 +27,24 @@ module Html2rss
       end
 
       ##
-      # Executes a request with runtime policy enforcement.
+      # @return [ResponseGuard]
+      attr_reader :response_guard
+
+      # Executes the request with runtime policy enforcement, returning the normalized response.
       #
-      # @return [Response] normalized request response
-      # @note Unlike BrowserlessStrategy, Faraday does not expose the remote IP after connect.
-      #   SSRF protection here is pre-connection only (DNS resolution via Policy).
-      #   A DNS rebinding attack between resolution and connect cannot be caught at this layer.
-      def execute
-        check_timeout!
+      # @return [Response] normalized response
+      def perform_execute
         deadline = request_deadline
-        response_guard, response = perform_request(deadline:)
-        response_guard.inspect_body!(response.body)
-        build_response(response)
-      rescue Faraday::TimeoutError, Timeout::Error => error
-        raise RequestTimedOut, error.message
+        @response_guard = ResponseGuard.new(policy: ctx.policy)
+        raw_response = faraday_request(response_guard, deadline:, streaming_buffer: true)
+        raw_response = retry_without_streaming(response_guard, deadline:) if retry_without_streaming?(raw_response)
+        build_response(raw_response)
       end
 
       private
 
       def request_deadline
-        monotonic_now + (ctx.budget.remaining_timeout_seconds || ctx.policy.total_timeout_seconds)
-      end
-
-      def perform_request(deadline:)
-        response_guard = ResponseGuard.new(policy: ctx.policy)
-        response = faraday_request(response_guard, deadline:, streaming_buffer: true)
-        response = retry_without_streaming(response_guard, deadline:) if retry_without_streaming?(response)
-        [response_guard, response]
+        monotonic_now + ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
       end
 
       def build_response(response)
@@ -61,13 +52,8 @@ module Html2rss
                      status: response.status)
       end
 
-      def validate_request!(consume_budget: true)
-        ctx.budget.consume! if consume_budget
-        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
-      end
-
       def faraday_request(response_guard, deadline:, streaming_buffer:, consume_budget: true)
-        validate_request!(consume_budget:)
+        preflight!(consume_budget:)
 
         client.get do |req|
           apply_timeouts(req, deadline:)

data/lib/html2rss/request_service/local_file_strategy.rb

@@ -3,15 +3,23 @@
 module Html2rss
   class RequestService
     ##
-    # Strategy to read a local HTML file.
+    # Strategy to read a local HTML file (CLI `--input`).
+    #
+    # Intentionally skips remote budget/policy preflight and ResponseGuard
+    # postflight: this adapter never opens a network connection. SSRF controls
+    # remain on remote strategies; do not use this adapter for remote URLs.
     class LocalFileStrategy < Strategy
+      private
+
+      def skip_preflight? = true
+
+      def skip_postflight? = true
+
       ##
-      # Executes the local file read.
-      #
       # @return [Response] the mock response wrapped around the file contents
       # @raise [ArgumentError] if the local file path is missing
       # @raise [Errno::ENOENT] if the file does not exist
-      def execute
+      def fetch
         file_path = ctx.request[:local_file_path]
         raise ArgumentError, 'Local file path is required for local_file strategy' unless file_path
         raise Errno::ENOENT, "File not found: #{file_path}" unless File.exist?(file_path)

data/lib/html2rss/request_service/network_guard.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require 'ipaddr'
+require 'resolv'
+require 'socket'
+
+module Html2rss
+  class RequestService
+    ##
+    # Enforces public-network reachability for request targets and remote IPs.
+    #
+    # Owns DNS resolution, host denylist, and blocked IP ranges. Callers go through
+    # {Policy} so SSRF/origin rules stay single-homed on the public façade.
+    class NetworkGuard
+      # Hostnames treated as local/private surfaces.
+      LOCAL_HOSTS = %w[localhost localhost.localdomain metadata.google.internal].to_set.freeze
+      # IP ranges blocked when private networks are disabled.
+      BLOCKED_IP_RANGES = [
+        IPAddr.new('0.0.0.0/8'),
+        IPAddr.new('10.0.0.0/8'),
+        IPAddr.new('127.0.0.0/8'),
+        IPAddr.new('169.254.0.0/16'),
+        IPAddr.new('172.16.0.0/12'),
+        IPAddr.new('192.168.0.0/16'),
+        IPAddr.new('224.0.0.0/4'),
+        IPAddr.new('::/128'),
+        IPAddr.new('::1/128'),
+        IPAddr.new('fe80::/10'),
+        IPAddr.new('fc00::/7'),
+        IPAddr.new('ff00::/8')
+      ].freeze
+
+      ##
+      # @param allow_private_networks [Boolean] whether private network targets are allowed
+      # @param resolver [#each_address, #getaddrinfo] DNS resolver used for hostname classification
+      def initialize(allow_private_networks:, resolver: Socket)
+        @allow_private_networks = allow_private_networks ? true : false
+        @resolver = resolver
+        freeze
+      end
+
+      ##
+      # Rejects URLs whose host is denylisted or resolves to a blocked address.
+      #
+      # @param url [Html2rss::Url] destination URL
+      # @return [void]
+      # @raise [PrivateNetworkDenied] if the target resolves to a private address
+      def enforce_public_network!(url)
+        host = url.host
+        return if allow_private_networks?
+        return unless blocked_host?(host) || resolved_ip_addresses(host).any? { |address| blocked_ip?(address) }
+
+        raise PrivateNetworkDenied, "Private network target denied for #{url}"
+      end
+
+      ##
+      # Validates the resolved remote IP for a completed request.
+      #
+      # @param ip [String, nil] remote IP address reported by the client
+      # @param url [Html2rss::Url] URL associated with the response
+      # @return [void]
+      # @raise [PrivateNetworkDenied] if the response came from a blocked address
+      def validate_remote_ip!(ip:, url:)
+        return if allow_private_networks?
+        return if ip.nil? || ip.empty?
+
+        parsed_ip = parse_ip(ip)
+        raise PrivateNetworkDenied, "Remote IP could not be validated for #{url}" unless parsed_ip
+        return unless blocked_ip?(parsed_ip)
+
+        raise PrivateNetworkDenied, "Private network target denied for #{url}"
+      end
+
+      private
+
+      attr_reader :resolver
+
+      def allow_private_networks?
+        @allow_private_networks
+      end
+
+      def blocked_host?(host)
+        LOCAL_HOSTS.include?(host.to_s.downcase)
+      end
+
+      def resolved_ip_addresses(host)
+        literal = parse_ip(host)
+        return [literal] if literal
+
+        if resolver.respond_to?(:each_address)
+          addresses_from_each_address(host)
+        else
+          addresses_from_getaddrinfo(host)
+        end
+      rescue Resolv::ResolvError, SocketError, SystemCallError
+        []
+      end
+
+      def addresses_from_each_address(host)
+        [].tap do |addresses|
+          resolver.each_address(host) do |address|
+            parsed = parse_ip(address)
+            addresses << parsed if parsed
+          end
+        end
+      end
+
+      def addresses_from_getaddrinfo(host)
+        resolver.getaddrinfo(host, nil).filter_map do |entry|
+          parse_ip(entry[3])
+        end
+      end
+
+      def parse_ip(value)
+        IPAddr.new(value)
+      rescue IPAddr::AddressFamilyError, IPAddr::InvalidAddressError
+        nil
+      end
+
+      def blocked_ip?(address)
+        BLOCKED_IP_RANGES.any? { |range| range.include?(address) }
+      end
+    end
+  end
+end