html2rss 0.22.2 → 0.23.0

data/lib/html2rss/request_service/policy.rb

@@ -1,32 +1,17 @@
 # frozen_string_literal: true
 
-require 'ipaddr'
-require 'resolv'
 require 'socket'
 
 module Html2rss
   class RequestService
     ##
-    # Describes the runtime request envelope for a single feed build.
-    class Policy # rubocop:disable Metrics/ClassLength
+    # Enforcer for the runtime request envelope of a single feed build.
+    #
+    # Public façade for request limits, same-origin/follow-up rules, and
+    # network reachability. DNS/host/IP enforcement lives on {NetworkGuard}.
+    class Policy
+      # Hard ceiling for configured max_requests during a feed build.
       MAX_REQUESTS_CEILING = 10
-      # Hostnames treated as local/private surfaces.
-      LOCAL_HOSTS = %w[localhost localhost.localdomain metadata.google.internal].to_set.freeze
-      # IP ranges blocked when private networks are disabled.
-      BLOCKED_IP_RANGES = [
-        IPAddr.new('0.0.0.0/8'),
-        IPAddr.new('10.0.0.0/8'),
-        IPAddr.new('127.0.0.0/8'),
-        IPAddr.new('169.254.0.0/16'),
-        IPAddr.new('172.16.0.0/12'),
-        IPAddr.new('192.168.0.0/16'),
-        IPAddr.new('224.0.0.0/4'),
-        IPAddr.new('::/128'),
-        IPAddr.new('::1/128'),
-        IPAddr.new('fe80::/10'),
-        IPAddr.new('fc00::/7'),
-        IPAddr.new('ff00::/8')
-      ].freeze
 
       # Default policy values used when request controls are not explicitly set.
       DEFAULTS = {
@@ -71,7 +56,7 @@ module Html2rss
         @max_requests = [validate_positive_integer!(:max_requests, max_requests), MAX_REQUESTS_CEILING].min
         @allow_private_networks = allow_private_networks ? true : false
         @allow_cross_origin_followups = allow_cross_origin_followups ? true : false
-        @resolver = resolver
+        @network_guard = NetworkGuard.new(allow_private_networks: @allow_private_networks, resolver:)
         freeze
       end
 
@@ -116,7 +101,7 @@ module Html2rss
       # @raise [PrivateNetworkDenied] if the target resolves to a private address
       def validate_request!(url:, origin_url:, relation:)
         enforce_same_origin!(url, origin_url, relation)
-        enforce_public_network!(url)
+        network_guard.enforce_public_network!(url)
       end
 
       ##
@@ -144,19 +129,12 @@ module Html2rss
       # @return [void]
       # @raise [PrivateNetworkDenied] if the response came from a blocked address
       def validate_remote_ip!(ip:, url:)
-        return if allow_private_networks?
-        return if ip.nil? || ip.empty?
-
-        parsed_ip = parse_ip(ip)
-        raise PrivateNetworkDenied, "Remote IP could not be validated for #{url}" unless parsed_ip
-        return unless blocked_ip?(parsed_ip)
-
-        raise PrivateNetworkDenied, "Private network target denied for #{url}"
+        network_guard.validate_remote_ip!(ip:, url:)
       end
 
       private
 
-      attr_reader :resolver
+      attr_reader :network_guard
 
       def validate_positive_integer!(name, value)
         raise ArgumentError, "#{name} must be positive" unless value.is_a?(Integer) && value.positive?
@@ -194,56 +172,6 @@ module Html2rss
 
         url.scheme == 'https' ? 443 : 80
       end
-
-      def enforce_public_network!(url)
-        host = url.host
-        return if allow_private_networks?
-        return unless blocked_host?(host) || resolved_ip_addresses(host).any? { |address| blocked_ip?(address) }
-
-        raise PrivateNetworkDenied, "Private network target denied for #{url}"
-      end
-
-      def blocked_host?(host)
-        LOCAL_HOSTS.include?(host.to_s.downcase)
-      end
-
-      def resolved_ip_addresses(host)
-        literal = parse_ip(host)
-        return [literal] if literal
-
-        if resolver.respond_to?(:each_address)
-          addresses_from_each_address(host)
-        else
-          addresses_from_getaddrinfo(host)
-        end
-      rescue Resolv::ResolvError, SocketError, SystemCallError
-        []
-      end
-
-      def addresses_from_each_address(host)
-        [].tap do |addresses|
-          resolver.each_address(host) do |address|
-            parsed = parse_ip(address)
-            addresses << parsed if parsed
-          end
-        end
-      end
-
-      def addresses_from_getaddrinfo(host)
-        resolver.getaddrinfo(host, nil).filter_map do |entry|
-          parse_ip(entry[3])
-        end
-      end
-
-      def parse_ip(value)
-        IPAddr.new(value)
-      rescue IPAddr::AddressFamilyError, IPAddr::InvalidAddressError
-        nil
-      end
-
-      def blocked_ip?(address)
-        BLOCKED_IP_RANGES.any? { |range| range.include?(address) }
-      end
     end
 
     # Shared immutable policy instance used for default request execution.

data/lib/html2rss/request_service/puppet_commander/navigation_guards.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    class PuppetCommander
+      ##
+      # Owns Puppeteer request interception, deferred navigation errors, main-frame
+      # filtering, and redirect-chain validation via {Policy}.
+      #
+      # Does not re-own Policy rules — only calls +validate_*!+ on +ctx.policy+.
+      class NavigationGuards
+        class << self
+          ##
+          # Resolves the final response URL for Response building and IP validation.
+          #
+          # @param navigation_response [Puppeteer::HTTPResponse, nil]
+          # @param fallback_url [String, Html2rss::Url]
+          # @return [Html2rss::Url]
+          def response_url(navigation_response, fallback_url)
+            raw_url = navigation_response&.url || fallback_url.to_s
+            Html2rss::Url.from_absolute(raw_url)
+          end
+        end
+
+        ##
+        # @param ctx [Context] request context providing policy and origin
+        # @param skip_request_resources [Set<String>] resource types to abort
+        def initialize(ctx:, skip_request_resources:)
+          @ctx = ctx
+          @skip_request_resources = skip_request_resources
+          @navigation_error = nil
+          @latest_navigation_response = nil
+          @main_frame = nil
+        end
+
+        ##
+        # Captures the main frame and wires request/response interceptors.
+        #
+        # @param page [Puppeteer::Page] browser page
+        # @return [void]
+        def install!(page)
+          @main_frame = page.main_frame if page.respond_to?(:main_frame)
+          page.request_interception = true
+          page.on('request') { |request| handle_request(request) }
+          page.on('response') { |response| handle_response(response) }
+        end
+
+        ##
+        # Clears deferred error and latest navigation response before goto.
+        #
+        # @return [void]
+        def begin_navigation!
+          @navigation_error = nil
+          @latest_navigation_response = nil
+        end
+
+        ##
+        # Re-raises a deferred navigation error when one was captured.
+        #
+        # @return [void]
+        # @raise [Html2rss::Error] when a navigation request or response validation failed
+        def raise_deferred_error!
+          raise @navigation_error if @navigation_error
+        end
+
+        ##
+        # @return [Puppeteer::HTTPResponse, nil] latest main-frame navigation response
+        attr_reader :latest_navigation_response
+
+        ##
+        # Validates the remote IP of a navigation response via Policy.
+        #
+        # @param navigation_response [Puppeteer::HTTPResponse, nil]
+        # @return [void]
+        def validate_final!(navigation_response)
+          final_url = self.class.response_url(navigation_response, ctx.url)
+          ctx.policy.validate_remote_ip!(ip: remote_ip(navigation_response), url: final_url)
+        end
+
+        private
+
+        attr_reader :ctx, :skip_request_resources, :main_frame
+
+        def handle_request(request)
+          validate_request!(request)
+
+          skip_request_resources.member?(request.resource_type) ? request.abort : request.continue
+        rescue Html2rss::Error => error
+          store_navigation_error(error, navigation_request: request.navigation_request?)
+          request.abort
+        end
+
+        def handle_response(response)
+          @latest_navigation_response = response if main_frame_navigation_response?(response)
+          validate_final!(response)
+        rescue Html2rss::Error => error
+          store_navigation_error(error, navigation_request: response.request.navigation_request?)
+        end
+
+        def validate_request!(request)
+          validate_navigation_redirect_chain!(request)
+          validate_navigation_target!(request)
+        end
+
+        def main_frame_navigation_response?(response)
+          request = response.request
+          return false unless request.navigation_request?
+          return true unless request.respond_to?(:frame)
+
+          frame = request.frame
+          return true if frame.nil?
+          return frame == main_frame unless main_frame.nil?
+          return true unless frame.respond_to?(:parent_frame)
+
+          frame.parent_frame.nil?
+        end
+
+        def remote_ip(navigation_response)
+          navigation_response.remote_address&.ip
+        end
+
+        def request_chain(request)
+          (request.redirect_chain + [request]).map { |entry| request_url(entry) }
+        end
+
+        def request_url(request)
+          Html2rss::Url.from_absolute(request.url)
+        end
+
+        def validate_navigation_redirect_chain!(request)
+          request_chain(request).each_cons(2) do |from_url, to_url|
+            ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
+          end
+        end
+
+        def validate_navigation_target!(request)
+          ctx.policy.validate_request!(url: request_url(request), origin_url: ctx.origin_url, relation: ctx.relation)
+        end
+
+        def store_navigation_error(error, navigation_request:)
+          return unless navigation_request
+
+          @navigation_error = error if @navigation_error.nil?
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/puppet_commander/preload_runner.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    class PuppetCommander
+      ##
+      # Runs Browserless preload choreography: wait, click, and scroll against
+      # the interaction budget on {Context#budget}.
+      class PreloadRunner
+        ##
+        # @param ctx [Context] request context providing preload config and budget
+        def initialize(ctx:)
+          @ctx = ctx
+        end
+
+        ##
+        # Executes configured preload actions on the page, if any.
+        #
+        # @param page [Puppeteer::Page] browser page after initial navigation
+        # @return [void]
+        def call(page)
+          preload_config = ctx.browserless_preload
+          return unless preload_config
+
+          wait_after(page, preload_config[:wait_after_ms])
+          click_selectors(page, preload_config[:click_selectors]) if preload_config[:click_selectors]
+          scroll_down(page, preload_config[:scroll_down]) if preload_config[:scroll_down]
+          wait_after(page, preload_config[:wait_after_ms])
+        end
+
+        private
+
+        attr_reader :ctx
+
+        def wait_after(page, timeout_ms)
+          return unless timeout_ms
+
+          ctx.budget.consume_interaction!
+          page.wait_for_timeout(timeout_ms)
+        end
+
+        def click_selectors(page, selectors)
+          selectors.each { |selector_config| click_selector(page, selector_config) }
+        end
+
+        def scroll_down(page, config)
+          iterations = config.fetch(:iterations, 1)
+          wait_after_ms = config[:wait_after_ms]
+          previous_height = nil
+
+          iterations.times do
+            updated_height = perform_scroll_iteration(page, wait_after_ms, previous_height)
+            break unless updated_height
+
+            previous_height = updated_height
+          end
+        end
+
+        def click_selector(page, config)
+          selector = config.fetch(:selector)
+          max_clicks = config.fetch(:max_clicks, 1)
+          wait_after_ms = config[:wait_after_ms]
+
+          max_clicks.times do
+            break unless (element = page.query_selector(selector))
+
+            ctx.budget.consume_interaction!
+            element.click
+            wait_after(page, wait_after_ms)
+          end
+        end
+
+        def perform_scroll_iteration(page, wait_after_ms, previous_height)
+          ctx.budget.consume_interaction!
+          page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)')
+          wait_after(page, wait_after_ms)
+
+          current_height = page.evaluate('() => document.body.scrollHeight')
+          return if previous_height && current_height <= previous_height
+
+          current_height
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/puppet_commander.rb

@@ -4,7 +4,11 @@ module Html2rss
   class RequestService
     ##
     # Commands the Puppeteer Browser to the website and builds the Response.
-    class PuppetCommander # rubocop:disable Metrics/ClassLength
+    #
+    # Public interface is {#call}; page setup, navigation, and body collection
+    # are private orchestration steps.
+    class PuppetCommander
+      # Request header names that must not be forwarded to the browser session.
       BROWSER_UNSAFE_HEADERS = %w[
         host connection content-length transfer-encoding
         sec-fetch-dest sec-fetch-mode sec-fetch-site sec-fetch-user
@@ -21,8 +25,9 @@ module Html2rss
                      referer: [ctx.url.scheme, ctx.url.host].join('://'))
         @ctx = ctx
         @browser = browser
-        @skip_request_resources = skip_request_resources
         @referer = referer
+        @navigation_guards = NavigationGuards.new(ctx:, skip_request_resources:)
+        @preload_runner = PreloadRunner.new(ctx:)
       end
 
       ##
@@ -32,233 +37,59 @@ module Html2rss
       def call
         page = new_page
         navigation_response = navigate_to_destination(page, ctx.url)
-        perform_preload(page)
-        raise_navigation_error_if_any
-        final_navigation_response = latest_navigation_response || navigation_response
-        validate_navigation_response!(final_navigation_response)
+        preload_runner.call(page)
+        navigation_guards.raise_deferred_error!
+        final_navigation_response = navigation_guards.latest_navigation_response || navigation_response
+        navigation_guards.validate_final!(final_navigation_response)
         build_response(page, final_navigation_response)
       ensure
         page&.close
       end
 
-      ##
-      # @return [Puppeteer::Page]
-      # @see https://yusukeiwaki.github.io/puppeteer-ruby-docs/Puppeteer/Page.html
+      private
+
+      attr_reader :ctx, :browser, :referer, :navigation_guards, :preload_runner
+
       def new_page
         page = browser.new_page
-        @main_frame = page.main_frame if page.respond_to?(:main_frame)
         configure_page(page)
-        configure_navigation_guards(page)
+        navigation_guards.install!(page)
         page
       end
 
-      ##
-      # @param page [Puppeteer::Page]
-      # @return [void]
       def configure_page(page)
         page.extra_http_headers = browser_headers
         page.default_navigation_timeout = navigation_timeout_ms
         page.default_timeout = navigation_timeout_ms
       end
 
-      ##
-      # @param page [Puppeteer::Page]
-      # @return [void]
-      def configure_navigation_guards(page)
-        page.request_interception = true
-        page.on('request') do |request|
-          handle_request(request)
-        end
-        page.on('response') { |response| handle_response(response) }
-      end
-
-      ##
-      # @param page [Puppeteer::Page] browser page
-      # @param url [Html2rss::Url] target URL
-      # @return [Puppeteer::HTTPResponse, nil] the navigation response if one was produced
       def navigate_to_destination(page, url)
-        @navigation_error = nil
-        @latest_navigation_response = nil
+        navigation_guards.begin_navigation!
         page.goto(url, wait_until: 'networkidle0', referer:, timeout: navigation_timeout_ms).tap do
-          raise_navigation_error_if_any
+          navigation_guards.raise_deferred_error!
         end
       rescue StandardError
-        raise_navigation_error_if_any
+        navigation_guards.raise_deferred_error!
 
         raise
       end
 
-      ##
-      # @param page [Puppeteer::Page] browser page
-      # @return [String] rendered HTML content
-      def body(page) = page.content
-
-      private
-
-      attr_reader :ctx, :browser, :skip_request_resources, :referer, :latest_navigation_response, :main_frame
-
-      ##
-      # Re-raises a deferred navigation error when one was captured.
-      #
-      # @raise [Html2rss::Error] when a navigation request or response validation failed
-      def raise_navigation_error_if_any
-        raise @navigation_error if @navigation_error
-      end
-
       def navigation_timeout_ms
-        ctx.policy.total_timeout_seconds * 1000
+        ctx.budget.effective_timeout_ms(fallback: ctx.policy.total_timeout_seconds)
       end
 
       def browser_headers
         ctx.headers.reject { |key, _| BROWSER_UNSAFE_HEADERS.include?(key.to_s.downcase) }
       end
 
-      def handle_request(request)
-        validate_request!(request)
-
-        skip_request_resources.member?(request.resource_type) ? request.abort : request.continue
-      rescue Html2rss::Error => error
-        store_navigation_error(error, navigation_request: request.navigation_request?)
-        request.abort
-      end
-
-      def handle_response(response)
-        @latest_navigation_response = response if main_frame_navigation_response?(response)
-        validate_response!(response)
-      rescue Html2rss::Error => error
-        store_navigation_error(error, navigation_request: response.request.navigation_request?)
-      end
-
-      def validate_request!(request)
-        validate_navigation_redirect_chain!(request)
-        validate_navigation_target!(request)
-      end
-
-      def main_frame_navigation_response?(response)
-        request = response.request
-        return false unless request.navigation_request?
-        return true unless request.respond_to?(:frame)
-
-        frame = request.frame
-        return true if frame.nil?
-        return frame == main_frame unless main_frame.nil?
-        return true unless frame.respond_to?(:parent_frame)
-
-        frame.parent_frame.nil?
-      end
-
       def build_response(page, navigation_response)
-        page_body = body(page)
-        ResponseGuard.new(policy: ctx.policy).inspect_body!(page_body)
-
         Response.new(
-          body: page_body,
+          body: page.content,
           headers: navigation_response&.headers || {},
-          url: response_url(navigation_response, ctx.url),
+          url: NavigationGuards.response_url(navigation_response, ctx.url),
           status: navigation_response&.status
         )
       end
-
-      def validate_navigation_response!(navigation_response)
-        final_url = response_url(navigation_response, ctx.url)
-        ctx.policy.validate_remote_ip!(ip: remote_ip(navigation_response), url: final_url)
-      end
-
-      def validate_response!(response)
-        validate_navigation_response!(response)
-      end
-
-      def response_url(navigation_response, fallback_url)
-        raw_url = navigation_response&.url || fallback_url.to_s
-        Html2rss::Url.from_absolute(raw_url)
-      end
-
-      def remote_ip(navigation_response)
-        navigation_response.remote_address&.ip
-      end
-
-      def request_chain(request)
-        (request.redirect_chain + [request]).map { |entry| request_url(entry) }
-      end
-
-      def request_url(request)
-        Html2rss::Url.from_absolute(request.url)
-      end
-
-      def validate_navigation_redirect_chain!(request)
-        request_chain(request).each_cons(2) do |from_url, to_url|
-          ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
-        end
-      end
-
-      def validate_navigation_target!(request)
-        ctx.policy.validate_request!(url: request_url(request), origin_url: ctx.origin_url, relation: ctx.relation)
-      end
-
-      def store_navigation_error(error, navigation_request:)
-        return unless navigation_request
-
-        @navigation_error = error if @navigation_error.nil?
-      end
-
-      def perform_preload(page)
-        preload_config = ctx.browserless_preload
-        return unless preload_config
-
-        wait_after(page, preload_config[:wait_after_ms])
-        click_selectors(page, preload_config[:click_selectors]) if preload_config[:click_selectors]
-        scroll_down(page, preload_config[:scroll_down]) if preload_config[:scroll_down]
-        wait_after(page, preload_config[:wait_after_ms])
-      end
-
-      def wait_after(page, timeout_ms)
-        return unless timeout_ms
-
-        ctx.budget.consume!
-        page.wait_for_timeout(timeout_ms)
-      end
-
-      def click_selectors(page, selectors)
-        selectors.each { |selector_config| click_selector(page, selector_config) }
-      end
-
-      def scroll_down(page, config)
-        iterations = config.fetch(:iterations, 1)
-        wait_after_ms = config[:wait_after_ms]
-        previous_height = nil
-
-        iterations.times do
-          updated_height = perform_scroll_iteration(page, wait_after_ms, previous_height)
-          break unless updated_height
-
-          previous_height = updated_height
-        end
-      end
-
-      def click_selector(page, config)
-        selector = config.fetch(:selector)
-        max_clicks = config.fetch(:max_clicks, 1)
-        wait_after_ms = config[:wait_after_ms]
-
-        max_clicks.times do
-          break unless (element = page.query_selector(selector))
-
-          ctx.budget.consume!
-          element.click
-          wait_after(page, wait_after_ms)
-        end
-      end
-
-      def perform_scroll_iteration(page, wait_after_ms, previous_height)
-        ctx.budget.consume!
-        page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)')
-        wait_after(page, wait_after_ms)
-
-        current_height = page.evaluate('() => document.body.scrollHeight')
-        return if previous_height && current_height <= previous_height
-
-        current_height
-      end
     end
   end
 end

data/lib/html2rss/request_service/response_guard.rb

@@ -46,7 +46,7 @@ module Html2rss
       attr_reader :policy
 
       def raise_if_blocked_surface!(body)
-        signature = Html2rss::BlockedSurface.interstitial_signature_for(body)
+        signature = BlockedSurface.interstitial_signature_for(body)
         return unless signature
 
         raise BlockedSurfaceDetected, signature.fetch(:message)