html2rss 0.22.2 → 0.24.0

data/lib/html2rss/request_service/budget.rb

@@ -3,40 +3,78 @@
 module Html2rss
   class RequestService
     ##
-    # Tracks how many outbound requests a single feed build may still perform.
+    # Tracks request slots, interaction budget, and wall-clock deadline for one feed build.
+    #
+    # HTTP fetches consume request slots; Browserless preload clicks/scrolls/waits consume
+    # interaction budget. Deadline remains on `#remaining_timeout_seconds`.
     class Budget
       ##
-      ##
-      # @param max_requests [Integer] the maximum number of requests allowed
-      # @param total_timeout_seconds [Integer, nil] the total timeout for the feed build
-      def initialize(max_requests:, total_timeout_seconds: nil)
-        unless max_requests.is_a?(Integer) && max_requests.positive?
-          raise ArgumentError, 'max_requests must be positive'
-        end
+      # @param max_requests [Integer] maximum HTTP request slots
+      # @param max_interactions [Integer] maximum preload interaction slots (default 0)
+      # @param total_timeout_seconds [Integer, nil] wall-clock timeout for the feed build
+      def initialize(max_requests:, max_interactions: 0, total_timeout_seconds: nil)
+        validate_slot_limits!(max_requests:, max_interactions:)
 
-        @remaining = max_requests
+        @remaining_requests = max_requests
+        @remaining_interactions = max_interactions
         @start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         @total_timeout_seconds = total_timeout_seconds
         @mutex = Mutex.new
       end
 
       ##
-      # Consumes one request from the budget.
+      # Consumes one HTTP request slot.
       #
-      # @return [Integer] remaining request count after consumption
-      # @raise [RequestBudgetExceeded] if no requests remain
+      # @return [Integer] remaining request slots after consumption
+      # @raise [RequestBudgetExceeded] if no request slots remain
       def consume!
+        consume_request!
+      end
+
+      ##
+      # Consumes one HTTP request slot.
+      #
+      # @return [Integer] remaining request slots after consumption
+      # @raise [RequestBudgetExceeded] if no request slots remain
+      def consume_request!
+        @mutex.synchronize do
+          raise RequestBudgetExceeded, 'Request budget exhausted' if @remaining_requests.zero?
+
+          @remaining_requests -= 1
+        end
+      end
+
+      ##
+      # Consumes one Browserless preload interaction slot.
+      #
+      # Preload must not steal pagination HTTP request slots.
+      #
+      # @return [Integer] remaining interaction slots after consumption
+      # @raise [InteractionBudgetExceeded] if no interaction slots remain
+      def consume_interaction!
         @mutex.synchronize do
-          raise RequestBudgetExceeded, 'Request budget exhausted' if @remaining.zero?
+          raise InteractionBudgetExceeded, 'Interaction budget exhausted' if @remaining_interactions.zero?
 
-          @remaining -= 1
+          @remaining_interactions -= 1
         end
       end
 
       ##
-      # @return [Integer] requests still available
+      # @return [Integer] HTTP request slots still available
       def remaining
-        @mutex.synchronize { @remaining }
+        remaining_requests
+      end
+
+      ##
+      # @return [Integer] HTTP request slots still available
+      def remaining_requests
+        @mutex.synchronize { @remaining_requests }
+      end
+
+      ##
+      # @return [Integer] preload interaction slots still available
+      def remaining_interactions
+        @mutex.synchronize { @remaining_interactions }
       end
 
       ##
@@ -44,10 +82,51 @@ module Html2rss
       def remaining_timeout_seconds
         return unless @total_timeout_seconds
 
-        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_time
-        remaining = @total_timeout_seconds - elapsed
+        remaining = @total_timeout_seconds - elapsed_seconds
         [remaining, 0.0].max
       end
+
+      ##
+      # @return [Float] seconds since this budget was created
+      def elapsed_seconds
+        Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_time
+      end
+
+      ##
+      # Resolves wall-clock seconds for the next adapter attempt.
+      #
+      # Prefers the tracked deadline remainder; falls back to the policy total when
+      # this Budget was constructed without a timeout (common in unit tests).
+      #
+      # @param fallback [Numeric] policy total timeout when remaining is untracked
+      # @return [Float] seconds available for this attempt
+      # @raise [RequestTimedOut] when no time remains
+      def effective_timeout_seconds(fallback:)
+        timeout = remaining_timeout_seconds || fallback
+        raise RequestTimedOut, 'Request timed out' if timeout <= 0
+
+        timeout.to_f
+      end
+
+      ##
+      # @param fallback [Numeric] policy total timeout when remaining is untracked
+      # @return [Integer] milliseconds available for this attempt
+      # @raise [RequestTimedOut] when no time remains
+      def effective_timeout_ms(fallback:)
+        (effective_timeout_seconds(fallback:) * 1000).to_i
+      end
+
+      private
+
+      def validate_slot_limits!(max_requests:, max_interactions:)
+        unless max_requests.is_a?(Integer) && max_requests.positive?
+          raise ArgumentError,
+                'max_requests must be positive'
+        end
+        return if max_interactions.is_a?(Integer) && !max_interactions.negative?
+
+        raise ArgumentError, 'max_interactions must be a non-negative integer'
+      end
     end
   end
 end

data/lib/html2rss/request_service/context.rb

@@ -77,9 +77,7 @@ module Html2rss
         raise ArgumentError, 'policy must not be nil' if @policy.nil?
 
         @origin_url = normalized_origin_url(request_options[:origin_url])
-        @budget = request_options.fetch(:budget) do
-          Budget.new(max_requests: policy.max_requests, total_timeout_seconds: policy.total_timeout_seconds)
-        end
+        @budget = request_options.fetch(:budget) { raise ArgumentError, 'budget is required' }
         raise ArgumentError, 'budget must not be nil' if @budget.nil?
       end
 

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -27,33 +27,24 @@ module Html2rss
       end
 
       ##
-      # Executes a request with runtime policy enforcement.
+      # @return [ResponseGuard]
+      attr_reader :response_guard
+
+      # Executes the request with runtime policy enforcement, returning the normalized response.
       #
-      # @return [Response] normalized request response
-      # @note Unlike BrowserlessStrategy, Faraday does not expose the remote IP after connect.
-      #   SSRF protection here is pre-connection only (DNS resolution via Policy).
-      #   A DNS rebinding attack between resolution and connect cannot be caught at this layer.
-      def execute
-        check_timeout!
+      # @return [Response] normalized response
+      def perform_execute
         deadline = request_deadline
-        response_guard, response = perform_request(deadline:)
-        response_guard.inspect_body!(response.body)
-        build_response(response)
-      rescue Faraday::TimeoutError, Timeout::Error => error
-        raise RequestTimedOut, error.message
+        @response_guard = ResponseGuard.new(policy: ctx.policy)
+        raw_response = faraday_request(response_guard, deadline:, streaming_buffer: true)
+        raw_response = retry_without_streaming(response_guard, deadline:) if retry_without_streaming?(raw_response)
+        build_response(raw_response)
       end
 
       private
 
       def request_deadline
-        monotonic_now + (ctx.budget.remaining_timeout_seconds || ctx.policy.total_timeout_seconds)
-      end
-
-      def perform_request(deadline:)
-        response_guard = ResponseGuard.new(policy: ctx.policy)
-        response = faraday_request(response_guard, deadline:, streaming_buffer: true)
-        response = retry_without_streaming(response_guard, deadline:) if retry_without_streaming?(response)
-        [response_guard, response]
+        monotonic_now + ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
       end
 
       def build_response(response)
@@ -61,13 +52,8 @@ module Html2rss
                      status: response.status)
       end
 
-      def validate_request!(consume_budget: true)
-        ctx.budget.consume! if consume_budget
-        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
-      end
-
       def faraday_request(response_guard, deadline:, streaming_buffer:, consume_budget: true)
-        validate_request!(consume_budget:)
+        preflight!(consume_budget:)
 
         client.get do |req|
           apply_timeouts(req, deadline:)

data/lib/html2rss/request_service/local_file_strategy.rb

@@ -3,15 +3,23 @@
 module Html2rss
   class RequestService
     ##
-    # Strategy to read a local HTML file.
+    # Strategy to read a local HTML file (CLI `--input`).
+    #
+    # Intentionally skips remote budget/policy preflight and ResponseGuard
+    # postflight: this adapter never opens a network connection. SSRF controls
+    # remain on remote strategies; do not use this adapter for remote URLs.
     class LocalFileStrategy < Strategy
+      private
+
+      def skip_preflight? = true
+
+      def skip_postflight? = true
+
       ##
-      # Executes the local file read.
-      #
       # @return [Response] the mock response wrapped around the file contents
       # @raise [ArgumentError] if the local file path is missing
       # @raise [Errno::ENOENT] if the file does not exist
-      def execute
+      def fetch
         file_path = ctx.request[:local_file_path]
         raise ArgumentError, 'Local file path is required for local_file strategy' unless file_path
         raise Errno::ENOENT, "File not found: #{file_path}" unless File.exist?(file_path)

data/lib/html2rss/request_service/network_guard.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require 'ipaddr'
+require 'resolv'
+require 'socket'
+
+module Html2rss
+  class RequestService
+    ##
+    # Enforces public-network reachability for request targets and remote IPs.
+    #
+    # Owns DNS resolution, host denylist, and blocked IP ranges. Callers go through
+    # {Policy} so SSRF/origin rules stay single-homed on the public façade.
+    class NetworkGuard
+      # Hostnames treated as local/private surfaces.
+      LOCAL_HOSTS = %w[localhost localhost.localdomain metadata.google.internal].to_set.freeze
+      # IP ranges blocked when private networks are disabled.
+      BLOCKED_IP_RANGES = [
+        IPAddr.new('0.0.0.0/8'),
+        IPAddr.new('10.0.0.0/8'),
+        IPAddr.new('127.0.0.0/8'),
+        IPAddr.new('169.254.0.0/16'),
+        IPAddr.new('172.16.0.0/12'),
+        IPAddr.new('192.168.0.0/16'),
+        IPAddr.new('224.0.0.0/4'),
+        IPAddr.new('::/128'),
+        IPAddr.new('::1/128'),
+        IPAddr.new('fe80::/10'),
+        IPAddr.new('fc00::/7'),
+        IPAddr.new('ff00::/8')
+      ].freeze
+
+      ##
+      # @param allow_private_networks [Boolean] whether private network targets are allowed
+      # @param resolver [#each_address, #getaddrinfo] DNS resolver used for hostname classification
+      def initialize(allow_private_networks:, resolver: Socket)
+        @allow_private_networks = allow_private_networks ? true : false
+        @resolver = resolver
+        freeze
+      end
+
+      ##
+      # Rejects URLs whose host is denylisted or resolves to a blocked address.
+      #
+      # @param url [Html2rss::Url] destination URL
+      # @return [void]
+      # @raise [PrivateNetworkDenied] if the target resolves to a private address
+      def enforce_public_network!(url)
+        host = url.host
+        return if allow_private_networks?
+        return unless blocked_host?(host) || resolved_ip_addresses(host).any? { |address| blocked_ip?(address) }
+
+        raise PrivateNetworkDenied, "Private network target denied for #{url}"
+      end
+
+      ##
+      # Validates the resolved remote IP for a completed request.
+      #
+      # @param ip [String, nil] remote IP address reported by the client
+      # @param url [Html2rss::Url] URL associated with the response
+      # @return [void]
+      # @raise [PrivateNetworkDenied] if the response came from a blocked address
+      def validate_remote_ip!(ip:, url:)
+        return if allow_private_networks?
+        return if ip.nil? || ip.empty?
+
+        parsed_ip = parse_ip(ip)
+        raise PrivateNetworkDenied, "Remote IP could not be validated for #{url}" unless parsed_ip
+        return unless blocked_ip?(parsed_ip)
+
+        raise PrivateNetworkDenied, "Private network target denied for #{url}"
+      end
+
+      private
+
+      attr_reader :resolver
+
+      def allow_private_networks?
+        @allow_private_networks
+      end
+
+      def blocked_host?(host)
+        LOCAL_HOSTS.include?(host.to_s.downcase)
+      end
+
+      def resolved_ip_addresses(host)
+        literal = parse_ip(host)
+        return [literal] if literal
+
+        if resolver.respond_to?(:each_address)
+          addresses_from_each_address(host)
+        else
+          addresses_from_getaddrinfo(host)
+        end
+      rescue Resolv::ResolvError, SocketError, SystemCallError
+        []
+      end
+
+      def addresses_from_each_address(host)
+        [].tap do |addresses|
+          resolver.each_address(host) do |address|
+            parsed = parse_ip(address)
+            addresses << parsed if parsed
+          end
+        end
+      end
+
+      def addresses_from_getaddrinfo(host)
+        resolver.getaddrinfo(host, nil).filter_map do |entry|
+          parse_ip(entry[3])
+        end
+      end
+
+      def parse_ip(value)
+        IPAddr.new(value)
+      rescue IPAddr::AddressFamilyError, IPAddr::InvalidAddressError
+        nil
+      end
+
+      def blocked_ip?(address)
+        BLOCKED_IP_RANGES.any? { |range| range.include?(address) }
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/policy.rb

@@ -1,32 +1,17 @@
 # frozen_string_literal: true
 
-require 'ipaddr'
-require 'resolv'
 require 'socket'
 
 module Html2rss
   class RequestService
     ##
-    # Describes the runtime request envelope for a single feed build.
-    class Policy # rubocop:disable Metrics/ClassLength
+    # Enforcer for the runtime request envelope of a single feed build.
+    #
+    # Public façade for request limits, same-origin/follow-up rules, and
+    # network reachability. DNS/host/IP enforcement lives on {NetworkGuard}.
+    class Policy
+      # Hard ceiling for configured max_requests during a feed build.
       MAX_REQUESTS_CEILING = 10
-      # Hostnames treated as local/private surfaces.
-      LOCAL_HOSTS = %w[localhost localhost.localdomain metadata.google.internal].to_set.freeze
-      # IP ranges blocked when private networks are disabled.
-      BLOCKED_IP_RANGES = [
-        IPAddr.new('0.0.0.0/8'),
-        IPAddr.new('10.0.0.0/8'),
-        IPAddr.new('127.0.0.0/8'),
-        IPAddr.new('169.254.0.0/16'),
-        IPAddr.new('172.16.0.0/12'),
-        IPAddr.new('192.168.0.0/16'),
-        IPAddr.new('224.0.0.0/4'),
-        IPAddr.new('::/128'),
-        IPAddr.new('::1/128'),
-        IPAddr.new('fe80::/10'),
-        IPAddr.new('fc00::/7'),
-        IPAddr.new('ff00::/8')
-      ].freeze
 
       # Default policy values used when request controls are not explicitly set.
       DEFAULTS = {
@@ -71,7 +56,7 @@ module Html2rss
         @max_requests = [validate_positive_integer!(:max_requests, max_requests), MAX_REQUESTS_CEILING].min
         @allow_private_networks = allow_private_networks ? true : false
         @allow_cross_origin_followups = allow_cross_origin_followups ? true : false
-        @resolver = resolver
+        @network_guard = NetworkGuard.new(allow_private_networks: @allow_private_networks, resolver:)
         freeze
       end
 
@@ -116,7 +101,7 @@ module Html2rss
       # @raise [PrivateNetworkDenied] if the target resolves to a private address
       def validate_request!(url:, origin_url:, relation:)
         enforce_same_origin!(url, origin_url, relation)
-        enforce_public_network!(url)
+        network_guard.enforce_public_network!(url)
       end
 
       ##
@@ -144,19 +129,12 @@ module Html2rss
       # @return [void]
       # @raise [PrivateNetworkDenied] if the response came from a blocked address
       def validate_remote_ip!(ip:, url:)
-        return if allow_private_networks?
-        return if ip.nil? || ip.empty?
-
-        parsed_ip = parse_ip(ip)
-        raise PrivateNetworkDenied, "Remote IP could not be validated for #{url}" unless parsed_ip
-        return unless blocked_ip?(parsed_ip)
-
-        raise PrivateNetworkDenied, "Private network target denied for #{url}"
+        network_guard.validate_remote_ip!(ip:, url:)
       end
 
       private
 
-      attr_reader :resolver
+      attr_reader :network_guard
 
       def validate_positive_integer!(name, value)
         raise ArgumentError, "#{name} must be positive" unless value.is_a?(Integer) && value.positive?
@@ -194,56 +172,6 @@ module Html2rss
 
         url.scheme == 'https' ? 443 : 80
       end
-
-      def enforce_public_network!(url)
-        host = url.host
-        return if allow_private_networks?
-        return unless blocked_host?(host) || resolved_ip_addresses(host).any? { |address| blocked_ip?(address) }
-
-        raise PrivateNetworkDenied, "Private network target denied for #{url}"
-      end
-
-      def blocked_host?(host)
-        LOCAL_HOSTS.include?(host.to_s.downcase)
-      end
-
-      def resolved_ip_addresses(host)
-        literal = parse_ip(host)
-        return [literal] if literal
-
-        if resolver.respond_to?(:each_address)
-          addresses_from_each_address(host)
-        else
-          addresses_from_getaddrinfo(host)
-        end
-      rescue Resolv::ResolvError, SocketError, SystemCallError
-        []
-      end
-
-      def addresses_from_each_address(host)
-        [].tap do |addresses|
-          resolver.each_address(host) do |address|
-            parsed = parse_ip(address)
-            addresses << parsed if parsed
-          end
-        end
-      end
-
-      def addresses_from_getaddrinfo(host)
-        resolver.getaddrinfo(host, nil).filter_map do |entry|
-          parse_ip(entry[3])
-        end
-      end
-
-      def parse_ip(value)
-        IPAddr.new(value)
-      rescue IPAddr::AddressFamilyError, IPAddr::InvalidAddressError
-        nil
-      end
-
-      def blocked_ip?(address)
-        BLOCKED_IP_RANGES.any? { |range| range.include?(address) }
-      end
     end
 
     # Shared immutable policy instance used for default request execution.

data/lib/html2rss/request_service/puppet_commander/navigation_guards.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    class PuppetCommander
+      ##
+      # Owns Puppeteer request interception, deferred navigation errors, main-frame
+      # filtering, and redirect-chain validation via {Policy}.
+      #
+      # Does not re-own Policy rules — only calls +validate_*!+ on +ctx.policy+.
+      class NavigationGuards
+        class << self
+          ##
+          # Resolves the final response URL for Response building and IP validation.
+          #
+          # @param navigation_response [Puppeteer::HTTPResponse, nil]
+          # @param fallback_url [String, Html2rss::Url]
+          # @return [Html2rss::Url]
+          def response_url(navigation_response, fallback_url)
+            raw_url = navigation_response&.url || fallback_url.to_s
+            Html2rss::Url.from_absolute(raw_url)
+          end
+        end
+
+        ##
+        # @param ctx [Context] request context providing policy and origin
+        # @param skip_request_resources [Set<String>] resource types to abort
+        def initialize(ctx:, skip_request_resources:)
+          @ctx = ctx
+          @skip_request_resources = skip_request_resources
+          @navigation_error = nil
+          @latest_navigation_response = nil
+          @main_frame = nil
+        end
+
+        ##
+        # Captures the main frame and wires request/response interceptors.
+        #
+        # @param page [Puppeteer::Page] browser page
+        # @return [void]
+        def install!(page)
+          @main_frame = page.main_frame if page.respond_to?(:main_frame)
+          page.request_interception = true
+          page.on('request') { |request| handle_request(request) }
+          page.on('response') { |response| handle_response(response) }
+        end
+
+        ##
+        # Clears deferred error and latest navigation response before goto.
+        #
+        # @return [void]
+        def begin_navigation!
+          @navigation_error = nil
+          @latest_navigation_response = nil
+        end
+
+        ##
+        # Re-raises a deferred navigation error when one was captured.
+        #
+        # @return [void]
+        # @raise [Html2rss::Error] when a navigation request or response validation failed
+        def raise_deferred_error!
+          raise @navigation_error if @navigation_error
+        end
+
+        ##
+        # @return [Puppeteer::HTTPResponse, nil] latest main-frame navigation response
+        attr_reader :latest_navigation_response
+
+        ##
+        # Validates the remote IP of a navigation response via Policy.
+        #
+        # @param navigation_response [Puppeteer::HTTPResponse, nil]
+        # @return [void]
+        def validate_final!(navigation_response)
+          final_url = self.class.response_url(navigation_response, ctx.url)
+          ctx.policy.validate_remote_ip!(ip: remote_ip(navigation_response), url: final_url)
+        end
+
+        private
+
+        attr_reader :ctx, :skip_request_resources, :main_frame
+
+        def handle_request(request)
+          validate_request!(request)
+
+          skip_request_resources.member?(request.resource_type) ? request.abort : request.continue
+        rescue Html2rss::Error => error
+          store_navigation_error(error, navigation_request: request.navigation_request?)
+          request.abort
+        end
+
+        def handle_response(response)
+          @latest_navigation_response = response if main_frame_navigation_response?(response)
+          validate_final!(response)
+        rescue Html2rss::Error => error
+          store_navigation_error(error, navigation_request: response.request.navigation_request?)
+        end
+
+        def validate_request!(request)
+          validate_navigation_redirect_chain!(request)
+          validate_navigation_target!(request)
+        end
+
+        def main_frame_navigation_response?(response)
+          request = response.request
+          return false unless request.navigation_request?
+          return true unless request.respond_to?(:frame)
+
+          frame = request.frame
+          return true if frame.nil?
+          return frame == main_frame unless main_frame.nil?
+          return true unless frame.respond_to?(:parent_frame)
+
+          frame.parent_frame.nil?
+        end
+
+        def remote_ip(navigation_response)
+          navigation_response.remote_address&.ip
+        end
+
+        def request_chain(request)
+          (request.redirect_chain + [request]).map { |entry| request_url(entry) }
+        end
+
+        def request_url(request)
+          Html2rss::Url.from_absolute(request.url)
+        end
+
+        def validate_navigation_redirect_chain!(request)
+          request_chain(request).each_cons(2) do |from_url, to_url|
+            ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
+          end
+        end
+
+        def validate_navigation_target!(request)
+          ctx.policy.validate_request!(url: request_url(request), origin_url: ctx.origin_url, relation: ctx.relation)
+        end
+
+        def store_navigation_error(error, navigation_request:)
+          return unless navigation_request
+
+          @navigation_error = error if @navigation_error.nil?
+        end
+      end
+    end
+  end
+end