html2rss 0.17.0 → 0.19.0

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -2,6 +2,7 @@
 
 require 'faraday'
 require 'faraday/follow_redirects'
+require 'faraday/gzip'
 
 module Html2rss
   class RequestService
@@ -9,15 +10,146 @@ module Html2rss
     # Strategy to use Faraday for the request.
     # @see https://rubygems.org/gems/faraday
     class FaradayStrategy < Strategy
-      # return [Response]
+      ##
+      # Restores buffered streamed bytes so response middleware can process them.
+      class StreamingBodyMiddleware < Faraday::Middleware
+        # Request-context key used to store streamed chunks before middleware completion.
+        STREAM_BUFFER_KEY = :html2rss_stream_buffer
+
+        # @param env [Faraday::Env] completed response environment
+        # @return [void]
+        def on_complete(env)
+          buffer = env.request.context&.delete(STREAM_BUFFER_KEY)
+          return if buffer.nil? || buffer.empty?
+
+          env.body = buffer
+        end
+      end
+
+      ##
+      # Executes a request with runtime policy enforcement.
+      #
+      # @return [Response] normalized request response
+      # @note Unlike BrowserlessStrategy, Faraday does not expose the remote IP after connect.
+      #   SSRF protection here is pre-connection only (DNS resolution via Policy).
+      #   A DNS rebinding attack between resolution and connect cannot be caught at this layer.
       def execute
-        request = Faraday.new(url: ctx.url, headers: ctx.headers) do |faraday|
-          faraday.use Faraday::FollowRedirects::Middleware
+        deadline = request_deadline
+        response_guard, response = perform_request(deadline:)
+        response_guard.inspect_body!(response.body)
+        build_response(response)
+      rescue Faraday::TimeoutError, Timeout::Error => error
+        raise RequestTimedOut, error.message
+      end
+
+      private
+
+      def request_deadline
+        monotonic_now + ctx.policy.total_timeout_seconds
+      end
+
+      def perform_request(deadline:)
+        response_guard = ResponseGuard.new(policy: ctx.policy)
+        response = faraday_request(response_guard, deadline:, streaming_buffer: true)
+        response = retry_without_streaming(response_guard, deadline:) if retry_without_streaming?(response)
+        [response_guard, response]
+      end
+
+      def build_response(response)
+        Response.new(body: response.body, headers: response.headers, url: response_url(response),
+                     status: response.status)
+      end
+
+      def validate_request!(consume_budget: true)
+        ctx.budget.consume! if consume_budget
+        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
+      end
+
+      def faraday_request(response_guard, deadline:, streaming_buffer:, consume_budget: true)
+        validate_request!(consume_budget:)
+
+        client.get do |req|
+          apply_timeouts(req, deadline:)
+          buffer = prepare_stream_buffer(req) if streaming_buffer
+          req.options.on_data = on_data_callback(response_guard, buffer)
+        end
+      end
+
+      def retry_without_streaming(response_guard, deadline:)
+        faraday_request(response_guard, deadline:, streaming_buffer: false, consume_budget: false)
+      end
+
+      def client
+        @client ||= Faraday.new(url: ctx.url.to_s, headers: ctx.headers) do |faraday|
+          faraday.use Faraday::FollowRedirects::Middleware, limit: ctx.policy.max_redirects, callback: redirect_callback
+          faraday.request :gzip
+          faraday.use StreamingBodyMiddleware
           faraday.adapter Faraday.default_adapter
         end
-        response = request.get
+      end
+
+      def apply_timeouts(request, deadline:)
+        remaining_timeout = remaining_timeout_seconds(deadline)
+        request.options.timeout = remaining_timeout
+        request.options.open_timeout = [ctx.policy.connect_timeout_seconds, remaining_timeout].min
+        request.options.read_timeout = [ctx.policy.read_timeout_seconds, remaining_timeout].min
+      end
+
+      def prepare_stream_buffer(request)
+        request.options.context ||= {}
+        request.options.context[StreamingBodyMiddleware::STREAM_BUFFER_KEY] = +''
+      end
+
+      def on_data_callback(response_guard, buffer)
+        proc do |chunk, total_bytes, env|
+          response_guard.inspect_chunk!(total_bytes:, headers: env&.response_headers)
+          buffer&.<< chunk
+        end
+      end
+
+      def remaining_timeout_seconds(deadline)
+        remaining = deadline - monotonic_now
+        raise RequestTimedOut, 'Request timed out' if remaining <= 0
+
+        remaining
+      end
+
+      def retry_without_streaming?(response)
+        return false if response.body.to_s.empty? == false
+        return false unless response_success?(response)
+
+        final_url = response.env&.url
+        return false unless final_url
+
+        final_url.to_s != ctx.url.to_s
+      end
+
+      def response_success?(response)
+        return true if response.status.nil?
+
+        response.status >= 200 && response.status < 300
+      end
+
+      def response_url(response)
+        return ctx.url unless (url = response.env&.url)
+
+        Html2rss::Url.from_absolute(url.to_s)
+      end
+
+      def redirect_callback
+        lambda do |old_env, new_env|
+          from_url = normalize_url(old_env[:url])
+          to_url = normalize_url(new_env[:url])
+          ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
+        end
+      end
+
+      def normalize_url(url)
+        Html2rss::Url.from_absolute(url.to_s)
+      end
 
-        Response.new(body: response.body, headers: response.headers)
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
     end
   end

data/lib/html2rss/request_service/policy.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+require 'ipaddr'
+require 'resolv'
+require 'socket'
+
+module Html2rss
+  class RequestService
+    ##
+    # Describes the runtime request envelope for a single feed build.
+    class Policy # rubocop:disable Metrics/ClassLength
+      MAX_REQUESTS_CEILING = 10
+      # Hostnames treated as local/private surfaces.
+      LOCAL_HOSTS = %w[localhost localhost.localdomain metadata.google.internal].to_set.freeze
+      # IP ranges blocked when private networks are disabled.
+      BLOCKED_IP_RANGES = [
+        IPAddr.new('0.0.0.0/8'),
+        IPAddr.new('10.0.0.0/8'),
+        IPAddr.new('127.0.0.0/8'),
+        IPAddr.new('169.254.0.0/16'),
+        IPAddr.new('172.16.0.0/12'),
+        IPAddr.new('192.168.0.0/16'),
+        IPAddr.new('224.0.0.0/4'),
+        IPAddr.new('::/128'),
+        IPAddr.new('::1/128'),
+        IPAddr.new('fe80::/10'),
+        IPAddr.new('fc00::/7'),
+        IPAddr.new('ff00::/8')
+      ].freeze
+
+      # Default policy values used when request controls are not explicitly set.
+      DEFAULTS = {
+        connect_timeout_seconds: 5,
+        read_timeout_seconds: 10,
+        total_timeout_seconds: 30,
+        max_redirects: 3,
+        max_response_bytes: 5_242_880,
+        max_decompressed_bytes: 10_485_760,
+        max_requests: 1,
+        allow_private_networks: false,
+        allow_cross_origin_followups: false
+      }.freeze
+
+      ##
+      # @param connect_timeout_seconds [Integer] maximum connection setup time
+      # @param read_timeout_seconds [Integer] maximum read stall time
+      # @param total_timeout_seconds [Integer] maximum total request time
+      # @param max_redirects [Integer] maximum redirect count
+      # @param max_response_bytes [Integer] maximum streamed response bytes
+      # @param max_decompressed_bytes [Integer] maximum final body size
+      # @param max_requests [Integer] maximum requests per feed build
+      # @param allow_private_networks [Boolean] whether private network targets are allowed
+      # @param allow_cross_origin_followups [Boolean] whether follow-up requests may leave the origin host
+      # @param resolver [#each_address] DNS resolver used for hostname classification
+      def initialize(connect_timeout_seconds: DEFAULTS[:connect_timeout_seconds], # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+                     read_timeout_seconds: DEFAULTS[:read_timeout_seconds],
+                     total_timeout_seconds: DEFAULTS[:total_timeout_seconds],
+                     max_redirects: DEFAULTS[:max_redirects],
+                     max_response_bytes: DEFAULTS[:max_response_bytes],
+                     max_decompressed_bytes: DEFAULTS[:max_decompressed_bytes],
+                     max_requests: DEFAULTS[:max_requests],
+                     allow_private_networks: DEFAULTS[:allow_private_networks],
+                     allow_cross_origin_followups: DEFAULTS[:allow_cross_origin_followups],
+                     resolver: Socket)
+        @connect_timeout_seconds = validate_positive_integer!(:connect_timeout_seconds, connect_timeout_seconds)
+        @read_timeout_seconds = validate_positive_integer!(:read_timeout_seconds, read_timeout_seconds)
+        @total_timeout_seconds = validate_positive_integer!(:total_timeout_seconds, total_timeout_seconds)
+        @max_redirects = validate_non_negative_integer!(:max_redirects, max_redirects)
+        @max_response_bytes = validate_positive_integer!(:max_response_bytes, max_response_bytes)
+        @max_decompressed_bytes = validate_positive_integer!(:max_decompressed_bytes, max_decompressed_bytes)
+        @max_requests = [validate_positive_integer!(:max_requests, max_requests), MAX_REQUESTS_CEILING].min
+        @allow_private_networks = allow_private_networks ? true : false
+        @allow_cross_origin_followups = allow_cross_origin_followups ? true : false
+        @resolver = resolver
+        freeze
+      end
+
+      attr_reader :connect_timeout_seconds,
+                  :read_timeout_seconds,
+                  :total_timeout_seconds,
+                  :max_redirects,
+                  :max_response_bytes,
+                  :max_decompressed_bytes,
+                  :max_requests
+
+      ##
+      # @return [Boolean] whether private network targets may be requested
+      def allow_private_networks?
+        @allow_private_networks
+      end
+
+      ##
+      # @return [Boolean] whether follow-up requests may leave the initial origin
+      def allow_cross_origin_followups?
+        @allow_cross_origin_followups
+      end
+
+      ##
+      # Returns the default request policy.
+      #
+      # @return [Policy] a default, frozen policy instance
+      # rubocop:disable Layout/ClassStructure
+      def self.default
+        new
+      end
+      # rubocop:enable Layout/ClassStructure
+
+      ##
+      # Validates whether a request target is permitted for the given context.
+      #
+      # @param url [Html2rss::Url] destination URL
+      # @param origin_url [Html2rss::Url] initial URL of the feed build
+      # @param relation [Symbol] logical reason for the request
+      # @return [void]
+      # @raise [CrossOriginFollowUpDenied] if a follow-up leaves the origin host
+      # @raise [PrivateNetworkDenied] if the target resolves to a private address
+      def validate_request!(url:, origin_url:, relation:)
+        enforce_same_origin!(url, origin_url, relation)
+        enforce_public_network!(url)
+      end
+
+      ##
+      # Validates a redirect hop before it is followed.
+      #
+      # @param from_url [Html2rss::Url] URL that produced the redirect
+      # @param to_url [Html2rss::Url] redirect destination
+      # @param origin_url [Html2rss::Url] initial URL of the feed build
+      # @param relation [Symbol] logical reason for the request
+      # @return [void]
+      # @raise [UnsupportedUrlScheme] if the redirect downgrades from HTTPS to HTTP
+      def validate_redirect!(from_url:, to_url:, origin_url:, relation:)
+        if from_url.scheme == 'https' && to_url.scheme == 'http'
+          raise UnsupportedUrlScheme, 'Redirect downgraded from https to http'
+        end
+
+        validate_request!(url: to_url, origin_url:, relation:)
+      end
+
+      ##
+      # Validates the resolved remote IP for a completed request.
+      #
+      # @param ip [String, nil] remote IP address reported by the client
+      # @param url [Html2rss::Url] URL associated with the response
+      # @return [void]
+      # @raise [PrivateNetworkDenied] if the response came from a blocked address
+      def validate_remote_ip!(ip:, url:)
+        return if allow_private_networks?
+        return if ip.nil? || ip.empty?
+
+        parsed_ip = parse_ip(ip)
+        raise PrivateNetworkDenied, "Remote IP could not be validated for #{url}" unless parsed_ip
+        return unless blocked_ip?(parsed_ip)
+
+        raise PrivateNetworkDenied, "Private network target denied for #{url}"
+      end
+
+      private
+
+      attr_reader :resolver
+
+      def validate_positive_integer!(name, value)
+        raise ArgumentError, "#{name} must be positive" unless value.is_a?(Integer) && value.positive?
+
+        value
+      end
+
+      def validate_non_negative_integer!(name, value)
+        raise ArgumentError, "#{name} must be non-negative" unless value.is_a?(Integer) && !value.negative?
+
+        value
+      end
+
+      def enforce_same_origin!(url, origin_url, relation)
+        return if relation == :initial || allow_cross_origin_followups?
+
+        enforce_follow_up_scheme!(url, origin_url)
+        return if comparable_origin(url) == comparable_origin(origin_url)
+
+        raise CrossOriginFollowUpDenied, "Cross-origin follow-up denied for #{url}"
+      end
+
+      def enforce_follow_up_scheme!(url, origin_url)
+        return unless origin_url.scheme == 'https' && url.scheme == 'http'
+
+        raise UnsupportedUrlScheme, "Follow-up downgraded from https to http for #{url}"
+      end
+
+      def comparable_origin(url)
+        [url.host, normalized_port(url)]
+      end
+
+      def normalized_port(url)
+        return url.port if url.port
+
+        url.scheme == 'https' ? 443 : 80
+      end
+
+      def enforce_public_network!(url)
+        host = url.host
+        return if allow_private_networks?
+        return unless blocked_host?(host) || resolved_ip_addresses(host).any? { |address| blocked_ip?(address) }
+
+        raise PrivateNetworkDenied, "Private network target denied for #{url}"
+      end
+
+      def blocked_host?(host)
+        LOCAL_HOSTS.include?(host.to_s.downcase)
+      end
+
+      def resolved_ip_addresses(host)
+        literal = parse_ip(host)
+        return [literal] if literal
+
+        if resolver.respond_to?(:each_address)
+          addresses_from_each_address(host)
+        else
+          addresses_from_getaddrinfo(host)
+        end
+      rescue Resolv::ResolvError, SocketError, SystemCallError
+        []
+      end
+
+      def addresses_from_each_address(host)
+        [].tap do |addresses|
+          resolver.each_address(host) do |address|
+            parsed = parse_ip(address)
+            addresses << parsed if parsed
+          end
+        end
+      end
+
+      def addresses_from_getaddrinfo(host)
+        resolver.getaddrinfo(host, nil).filter_map do |entry|
+          parse_ip(entry[3])
+        end
+      end
+
+      def parse_ip(value)
+        IPAddr.new(value)
+      rescue IPAddr::AddressFamilyError, IPAddr::InvalidAddressError
+        nil
+      end
+
+      def blocked_ip?(address)
+        BLOCKED_IP_RANGES.any? { |range| range.include?(address) }
+      end
+    end
+
+    # Shared immutable policy instance used for default request execution.
+    Policy::DEFAULT_POLICY = Policy.new
+  end
+end

data/lib/html2rss/request_service/puppet_commander.rb

@@ -4,7 +4,13 @@ module Html2rss
   class RequestService
     ##
     # Commands the Puppeteer Browser to the website and builds the Response.
-    class PuppetCommander
+    class PuppetCommander # rubocop:disable Metrics/ClassLength
+      BROWSER_UNSAFE_HEADERS = %w[
+        host connection content-length transfer-encoding
+        sec-fetch-dest sec-fetch-mode sec-fetch-site sec-fetch-user
+        upgrade-insecure-requests
+      ].to_set.freeze
+
       # @param ctx [Context]
       # @param browser [Puppeteer::Browser]
       # @param skip_request_resources [Set<String>] the resource types not to request
@@ -19,13 +25,18 @@ module Html2rss
         @referer = referer
       end
 
-      # @return [Response]
+      ##
+      # Visits the request URL and normalizes the page into a response object.
+      #
+      # @return [Response] rendered page response
       def call
         page = new_page
-
-        response = navigate_to_destination(page, ctx.url)
-
-        Response.new(body: body(page), headers: response.headers)
+        navigation_response = navigate_to_destination(page, ctx.url)
+        perform_preload(page)
+        raise_navigation_error_if_any
+        final_navigation_response = latest_navigation_response || navigation_response
+        validate_navigation_response!(final_navigation_response)
+        build_response(page, final_navigation_response)
       ensure
         page&.close
       end
@@ -35,27 +46,215 @@ module Html2rss
       # @see https://yusukeiwaki.github.io/puppeteer-ruby-docs/Puppeteer/Page.html
       def new_page
         page = browser.new_page
-        page.extra_http_headers = ctx.headers
+        @main_frame = page.main_frame if page.respond_to?(:main_frame)
+        configure_page(page)
+        configure_navigation_guards(page)
+        page
+      end
 
-        return page if skip_request_resources.empty?
+      ##
+      # @param page [Puppeteer::Page]
+      # @return [void]
+      def configure_page(page)
+        page.extra_http_headers = browser_headers
+        page.default_navigation_timeout = navigation_timeout_ms
+        page.default_timeout = navigation_timeout_ms
+      end
 
+      ##
+      # @param page [Puppeteer::Page]
+      # @return [void]
+      def configure_navigation_guards(page)
         page.request_interception = true
         page.on('request') do |request|
-          skip_request_resources.member?(request.resource_type) ? request.abort : request.continue
+          handle_request(request)
         end
-
-        page
+        page.on('response') { |response| handle_response(response) }
       end
 
+      ##
+      # @param page [Puppeteer::Page] browser page
+      # @param url [Html2rss::Url] target URL
+      # @return [Puppeteer::HTTPResponse, nil] the navigation response if one was produced
       def navigate_to_destination(page, url)
-        page.goto(url, wait_until: 'networkidle0', referer:)
+        @navigation_error = nil
+        @latest_navigation_response = nil
+        page.goto(url, wait_until: 'networkidle0', referer:, timeout: navigation_timeout_ms).tap do
+          raise_navigation_error_if_any
+        end
+      rescue StandardError
+        raise_navigation_error_if_any
+
+        raise
       end
 
+      ##
+      # @param page [Puppeteer::Page] browser page
+      # @return [String] rendered HTML content
       def body(page) = page.content
 
       private
 
-      attr_reader :ctx, :browser, :skip_request_resources, :referer
+      attr_reader :ctx, :browser, :skip_request_resources, :referer, :latest_navigation_response, :main_frame
+
+      def raise_navigation_error_if_any
+        raise @navigation_error if @navigation_error
+      end
+
+      def navigation_timeout_ms
+        ctx.policy.total_timeout_seconds * 1000
+      end
+
+      def browser_headers
+        ctx.headers.reject { |key, _| BROWSER_UNSAFE_HEADERS.include?(key.to_s.downcase) }
+      end
+
+      def handle_request(request)
+        validate_request!(request)
+
+        skip_request_resources.member?(request.resource_type) ? request.abort : request.continue
+      rescue Html2rss::Error => error
+        store_navigation_error(error, navigation_request: request.navigation_request?)
+        request.abort
+      end
+
+      def handle_response(response)
+        @latest_navigation_response = response if main_frame_navigation_response?(response)
+        validate_response!(response)
+      rescue Html2rss::Error => error
+        store_navigation_error(error, navigation_request: response.request.navigation_request?)
+      end
+
+      def validate_request!(request)
+        validate_navigation_redirect_chain!(request)
+        validate_navigation_target!(request)
+      end
+
+      def main_frame_navigation_response?(response)
+        request = response.request
+        return false unless request.navigation_request?
+        return true unless request.respond_to?(:frame)
+
+        frame = request.frame
+        return true if frame.nil?
+        return frame == main_frame unless main_frame.nil?
+        return true unless frame.respond_to?(:parent_frame)
+
+        frame.parent_frame.nil?
+      end
+
+      def build_response(page, navigation_response)
+        page_body = body(page)
+        ResponseGuard.new(policy: ctx.policy).inspect_body!(page_body)
+
+        Response.new(
+          body: page_body,
+          headers: navigation_response&.headers || {},
+          url: response_url(navigation_response, ctx.url),
+          status: navigation_response&.status
+        )
+      end
+
+      def validate_navigation_response!(navigation_response)
+        final_url = response_url(navigation_response, ctx.url)
+        ctx.policy.validate_remote_ip!(ip: remote_ip(navigation_response), url: final_url)
+      end
+
+      def validate_response!(response)
+        validate_navigation_response!(response)
+      end
+
+      def response_url(navigation_response, fallback_url)
+        raw_url = navigation_response&.url || fallback_url.to_s
+        Html2rss::Url.from_absolute(raw_url)
+      end
+
+      def remote_ip(navigation_response)
+        navigation_response.remote_address&.ip
+      end
+
+      def request_chain(request)
+        (request.redirect_chain + [request]).map { |entry| request_url(entry) }
+      end
+
+      def request_url(request)
+        Html2rss::Url.from_absolute(request.url)
+      end
+
+      def validate_navigation_redirect_chain!(request)
+        request_chain(request).each_cons(2) do |from_url, to_url|
+          ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
+        end
+      end
+
+      def validate_navigation_target!(request)
+        ctx.policy.validate_request!(url: request_url(request), origin_url: ctx.origin_url, relation: ctx.relation)
+      end
+
+      def store_navigation_error(error, navigation_request:)
+        return unless navigation_request
+
+        @navigation_error = error if @navigation_error.nil?
+      end
+
+      def perform_preload(page)
+        preload_config = ctx.browserless_preload
+        return unless preload_config
+
+        wait_after(page, preload_config[:wait_after_ms])
+        click_selectors(page, preload_config[:click_selectors]) if preload_config[:click_selectors]
+        scroll_down(page, preload_config[:scroll_down]) if preload_config[:scroll_down]
+        wait_after(page, preload_config[:wait_after_ms])
+      end
+
+      def wait_after(page, timeout_ms)
+        return unless timeout_ms
+
+        ctx.budget.consume!
+        page.wait_for_timeout(timeout_ms)
+      end
+
+      def click_selectors(page, selectors)
+        selectors.each { |selector_config| click_selector(page, selector_config) }
+      end
+
+      def scroll_down(page, config)
+        iterations = config.fetch(:iterations, 1)
+        wait_after_ms = config[:wait_after_ms]
+        previous_height = nil
+
+        iterations.times do
+          updated_height = perform_scroll_iteration(page, wait_after_ms, previous_height)
+          break unless updated_height
+
+          previous_height = updated_height
+        end
+      end
+
+      def click_selector(page, config)
+        selector = config.fetch(:selector)
+        max_clicks = config.fetch(:max_clicks, 1)
+        wait_after_ms = config[:wait_after_ms]
+
+        max_clicks.times do
+          break unless (element = page.query_selector(selector))
+
+          ctx.budget.consume!
+          element.click
+          wait_after(page, wait_after_ms)
+        end
+      end
+
+      def perform_scroll_iteration(page, wait_after_ms, previous_height)
+        ctx.budget.consume!
+        page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)')
+        wait_after(page, wait_after_ms)
+
+        current_height = page.evaluate('() => document.body.scrollHeight')
+        return if previous_height && current_height <= previous_height
+
+        current_height
+      end
     end
   end
 end