html2rss 0.22.2 → 0.24.0

data/lib/html2rss/request_service/puppet_commander/preload_runner.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    class PuppetCommander
+      ##
+      # Runs Browserless preload choreography: wait, click, and scroll against
+      # the interaction budget on {Context#budget}.
+      class PreloadRunner
+        ##
+        # @param ctx [Context] request context providing preload config and budget
+        def initialize(ctx:)
+          @ctx = ctx
+        end
+
+        ##
+        # Executes configured preload actions on the page, if any.
+        #
+        # @param page [Puppeteer::Page] browser page after initial navigation
+        # @return [void]
+        def call(page)
+          preload_config = ctx.browserless_preload
+          return unless preload_config
+
+          wait_after(page, preload_config[:wait_after_ms])
+          click_selectors(page, preload_config[:click_selectors]) if preload_config[:click_selectors]
+          scroll_down(page, preload_config[:scroll_down]) if preload_config[:scroll_down]
+          wait_after(page, preload_config[:wait_after_ms])
+        end
+
+        private
+
+        attr_reader :ctx
+
+        def wait_after(page, timeout_ms)
+          return unless timeout_ms
+
+          ctx.budget.consume_interaction!
+          page.wait_for_timeout(timeout_ms)
+        end
+
+        def click_selectors(page, selectors)
+          selectors.each { |selector_config| click_selector(page, selector_config) }
+        end
+
+        def scroll_down(page, config)
+          iterations = config.fetch(:iterations, 1)
+          wait_after_ms = config[:wait_after_ms]
+          previous_height = nil
+
+          iterations.times do
+            updated_height = perform_scroll_iteration(page, wait_after_ms, previous_height)
+            break unless updated_height
+
+            previous_height = updated_height
+          end
+        end
+
+        def click_selector(page, config)
+          selector = config.fetch(:selector)
+          max_clicks = config.fetch(:max_clicks, 1)
+          wait_after_ms = config[:wait_after_ms]
+
+          max_clicks.times do
+            break unless (element = page.query_selector(selector))
+
+            ctx.budget.consume_interaction!
+            element.click
+            wait_after(page, wait_after_ms)
+          end
+        end
+
+        def perform_scroll_iteration(page, wait_after_ms, previous_height)
+          ctx.budget.consume_interaction!
+          page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)')
+          wait_after(page, wait_after_ms)
+
+          current_height = page.evaluate('() => document.body.scrollHeight')
+          return if previous_height && current_height <= previous_height
+
+          current_height
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/puppet_commander.rb

@@ -4,7 +4,11 @@ module Html2rss
   class RequestService
     ##
     # Commands the Puppeteer Browser to the website and builds the Response.
-    class PuppetCommander # rubocop:disable Metrics/ClassLength
+    #
+    # Public interface is {#call}; page setup, navigation, and body collection
+    # are private orchestration steps.
+    class PuppetCommander
+      # Request header names that must not be forwarded to the browser session.
       BROWSER_UNSAFE_HEADERS = %w[
         host connection content-length transfer-encoding
         sec-fetch-dest sec-fetch-mode sec-fetch-site sec-fetch-user
@@ -21,8 +25,9 @@ module Html2rss
                      referer: [ctx.url.scheme, ctx.url.host].join('://'))
         @ctx = ctx
         @browser = browser
-        @skip_request_resources = skip_request_resources
         @referer = referer
+        @navigation_guards = NavigationGuards.new(ctx:, skip_request_resources:)
+        @preload_runner = PreloadRunner.new(ctx:)
       end
 
       ##
@@ -32,233 +37,59 @@ module Html2rss
       def call
         page = new_page
         navigation_response = navigate_to_destination(page, ctx.url)
-        perform_preload(page)
-        raise_navigation_error_if_any
-        final_navigation_response = latest_navigation_response || navigation_response
-        validate_navigation_response!(final_navigation_response)
+        preload_runner.call(page)
+        navigation_guards.raise_deferred_error!
+        final_navigation_response = navigation_guards.latest_navigation_response || navigation_response
+        navigation_guards.validate_final!(final_navigation_response)
         build_response(page, final_navigation_response)
       ensure
         page&.close
       end
 
-      ##
-      # @return [Puppeteer::Page]
-      # @see https://yusukeiwaki.github.io/puppeteer-ruby-docs/Puppeteer/Page.html
+      private
+
+      attr_reader :ctx, :browser, :referer, :navigation_guards, :preload_runner
+
       def new_page
         page = browser.new_page
-        @main_frame = page.main_frame if page.respond_to?(:main_frame)
         configure_page(page)
-        configure_navigation_guards(page)
+        navigation_guards.install!(page)
         page
       end
 
-      ##
-      # @param page [Puppeteer::Page]
-      # @return [void]
       def configure_page(page)
         page.extra_http_headers = browser_headers
         page.default_navigation_timeout = navigation_timeout_ms
         page.default_timeout = navigation_timeout_ms
       end
 
-      ##
-      # @param page [Puppeteer::Page]
-      # @return [void]
-      def configure_navigation_guards(page)
-        page.request_interception = true
-        page.on('request') do |request|
-          handle_request(request)
-        end
-        page.on('response') { |response| handle_response(response) }
-      end
-
-      ##
-      # @param page [Puppeteer::Page] browser page
-      # @param url [Html2rss::Url] target URL
-      # @return [Puppeteer::HTTPResponse, nil] the navigation response if one was produced
       def navigate_to_destination(page, url)
-        @navigation_error = nil
-        @latest_navigation_response = nil
+        navigation_guards.begin_navigation!
         page.goto(url, wait_until: 'networkidle0', referer:, timeout: navigation_timeout_ms).tap do
-          raise_navigation_error_if_any
+          navigation_guards.raise_deferred_error!
         end
       rescue StandardError
-        raise_navigation_error_if_any
+        navigation_guards.raise_deferred_error!
 
         raise
       end
 
-      ##
-      # @param page [Puppeteer::Page] browser page
-      # @return [String] rendered HTML content
-      def body(page) = page.content
-
-      private
-
-      attr_reader :ctx, :browser, :skip_request_resources, :referer, :latest_navigation_response, :main_frame
-
-      ##
-      # Re-raises a deferred navigation error when one was captured.
-      #
-      # @raise [Html2rss::Error] when a navigation request or response validation failed
-      def raise_navigation_error_if_any
-        raise @navigation_error if @navigation_error
-      end
-
       def navigation_timeout_ms
-        ctx.policy.total_timeout_seconds * 1000
+        ctx.budget.effective_timeout_ms(fallback: ctx.policy.total_timeout_seconds)
       end
 
       def browser_headers
         ctx.headers.reject { |key, _| BROWSER_UNSAFE_HEADERS.include?(key.to_s.downcase) }
       end
 
-      def handle_request(request)
-        validate_request!(request)
-
-        skip_request_resources.member?(request.resource_type) ? request.abort : request.continue
-      rescue Html2rss::Error => error
-        store_navigation_error(error, navigation_request: request.navigation_request?)
-        request.abort
-      end
-
-      def handle_response(response)
-        @latest_navigation_response = response if main_frame_navigation_response?(response)
-        validate_response!(response)
-      rescue Html2rss::Error => error
-        store_navigation_error(error, navigation_request: response.request.navigation_request?)
-      end
-
-      def validate_request!(request)
-        validate_navigation_redirect_chain!(request)
-        validate_navigation_target!(request)
-      end
-
-      def main_frame_navigation_response?(response)
-        request = response.request
-        return false unless request.navigation_request?
-        return true unless request.respond_to?(:frame)
-
-        frame = request.frame
-        return true if frame.nil?
-        return frame == main_frame unless main_frame.nil?
-        return true unless frame.respond_to?(:parent_frame)
-
-        frame.parent_frame.nil?
-      end
-
       def build_response(page, navigation_response)
-        page_body = body(page)
-        ResponseGuard.new(policy: ctx.policy).inspect_body!(page_body)
-
         Response.new(
-          body: page_body,
+          body: page.content,
           headers: navigation_response&.headers || {},
-          url: response_url(navigation_response, ctx.url),
+          url: NavigationGuards.response_url(navigation_response, ctx.url),
           status: navigation_response&.status
         )
       end
-
-      def validate_navigation_response!(navigation_response)
-        final_url = response_url(navigation_response, ctx.url)
-        ctx.policy.validate_remote_ip!(ip: remote_ip(navigation_response), url: final_url)
-      end
-
-      def validate_response!(response)
-        validate_navigation_response!(response)
-      end
-
-      def response_url(navigation_response, fallback_url)
-        raw_url = navigation_response&.url || fallback_url.to_s
-        Html2rss::Url.from_absolute(raw_url)
-      end
-
-      def remote_ip(navigation_response)
-        navigation_response.remote_address&.ip
-      end
-
-      def request_chain(request)
-        (request.redirect_chain + [request]).map { |entry| request_url(entry) }
-      end
-
-      def request_url(request)
-        Html2rss::Url.from_absolute(request.url)
-      end
-
-      def validate_navigation_redirect_chain!(request)
-        request_chain(request).each_cons(2) do |from_url, to_url|
-          ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
-        end
-      end
-
-      def validate_navigation_target!(request)
-        ctx.policy.validate_request!(url: request_url(request), origin_url: ctx.origin_url, relation: ctx.relation)
-      end
-
-      def store_navigation_error(error, navigation_request:)
-        return unless navigation_request
-
-        @navigation_error = error if @navigation_error.nil?
-      end
-
-      def perform_preload(page)
-        preload_config = ctx.browserless_preload
-        return unless preload_config
-
-        wait_after(page, preload_config[:wait_after_ms])
-        click_selectors(page, preload_config[:click_selectors]) if preload_config[:click_selectors]
-        scroll_down(page, preload_config[:scroll_down]) if preload_config[:scroll_down]
-        wait_after(page, preload_config[:wait_after_ms])
-      end
-
-      def wait_after(page, timeout_ms)
-        return unless timeout_ms
-
-        ctx.budget.consume!
-        page.wait_for_timeout(timeout_ms)
-      end
-
-      def click_selectors(page, selectors)
-        selectors.each { |selector_config| click_selector(page, selector_config) }
-      end
-
-      def scroll_down(page, config)
-        iterations = config.fetch(:iterations, 1)
-        wait_after_ms = config[:wait_after_ms]
-        previous_height = nil
-
-        iterations.times do
-          updated_height = perform_scroll_iteration(page, wait_after_ms, previous_height)
-          break unless updated_height
-
-          previous_height = updated_height
-        end
-      end
-
-      def click_selector(page, config)
-        selector = config.fetch(:selector)
-        max_clicks = config.fetch(:max_clicks, 1)
-        wait_after_ms = config[:wait_after_ms]
-
-        max_clicks.times do
-          break unless (element = page.query_selector(selector))
-
-          ctx.budget.consume!
-          element.click
-          wait_after(page, wait_after_ms)
-        end
-      end
-
-      def perform_scroll_iteration(page, wait_after_ms, previous_height)
-        ctx.budget.consume!
-        page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)')
-        wait_after(page, wait_after_ms)
-
-        current_height = page.evaluate('() => document.body.scrollHeight')
-        return if previous_height && current_height <= previous_height
-
-        current_height
-      end
     end
   end
 end

data/lib/html2rss/request_service/response.rb

@@ -7,12 +7,16 @@ module Html2rss
     ##
     # To be used by strategies to provide their response.
     class Response
+      # Default when a strategy does not attach transport telemetry.
+      EMPTY_TRANSPORT_META = {}.freeze
+
       ##
       # @param body [String] the body of the response
       # @param url [Html2rss::Url] the final request URL
       # @param headers [Hash] the headers of the response
       # @param status [Integer, nil] the HTTP status code when available
-      def initialize(body:, url:, headers: {}, status: nil)
+      # @param transport_meta [Hash] allowlisted upstream telemetry (frozen when present)
+      def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META)
         @body = body
 
         headers = headers.dup
@@ -22,6 +26,7 @@ module Html2rss
         @headers = headers
         @status = status
         @url = url
+        @transport_meta = transport_meta.nil? || transport_meta.empty? ? EMPTY_TRANSPORT_META : transport_meta.freeze
       end
 
       # @return [String] the raw body of the response
@@ -36,6 +41,9 @@ module Html2rss
       # @return [Html2rss::Url] the URL of the response
       attr_reader :url
 
+      # @return [Hash] allowlisted upstream transport telemetry
+      attr_reader :transport_meta
+
       # @return [String] normalized content type header value
       def content_type = header('content-type').to_s
 

data/lib/html2rss/request_service/response_guard.rb

@@ -46,7 +46,7 @@ module Html2rss
       attr_reader :policy
 
       def raise_if_blocked_surface!(body)
-        signature = Html2rss::BlockedSurface.interstitial_signature_for(body)
+        signature = BlockedSurface.interstitial_signature_for(body)
         return unless signature
 
         raise BlockedSurfaceDetected, signature.fetch(:message)

data/lib/html2rss/request_service/strategy.rb

@@ -3,7 +3,10 @@
 module Html2rss
   class RequestService
     ##
-    # Defines the interface of every request strategy.
+    # Defines the guarded request interface for every strategy adapter.
+    #
+    # `#execute` always runs timeout check → budget/policy preflight → `#fetch`
+    # → ResponseGuard postflight so new adapters cannot skip shared controls.
     class Strategy
       ##
       # @param ctx [Context] the context for the request
@@ -12,11 +15,17 @@ module Html2rss
       end
 
       ##
-      # Executes the request.
+      # Executes the request through the shared preflight/postflight path.
+      #
       # @return [Response] the response from the strategy
-      # @raise [NotImplementedError] if the method is not implemented by the subclass
+      # @raise [NotImplementedError] if the subclass does not implement `#fetch`
       def execute
-        raise NotImplementedError, 'Subclass must implement #execute'
+        check_timeout!
+        response = perform_execute
+        postflight!(response, response_guard:)
+        response
+      rescue StandardError => error
+        handle_error(error)
       end
 
       private
@@ -24,9 +33,118 @@ module Html2rss
       # @return [Context] the context for the request
       attr_reader :ctx
 
+      ##
+      # Adapter hook: perform the preflight, fetch, and optional retry lifecycle.
+      #
+      # @return [Response] normalized response
+      def perform_execute
+        preflight!
+        fetch
+      end
+
+      ##
+      # Adapter hook: perform the transport-specific fetch after preflight.
+      #
+      # @return [Response] normalized response
+      # @raise [NotImplementedError] when a subclass omits the hook
+      def fetch
+        raise NotImplementedError, 'Subclass must implement #fetch'
+      end
+
+      ##
+      # Consumes a request slot and validates the destination before transport work.
+      #
+      # @param consume_budget [Boolean] whether to decrement the request budget
+      # @return [void]
+      def preflight!(consume_budget: true)
+        return if skip_preflight?
+
+        ctx.budget.consume! if consume_budget
+        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
+      end
+
+      ##
+      # Enforces response-size and blocked-surface checks after transport work.
+      #
+      # @param response [Response, nil] adapter response
+      # @param response_guard [ResponseGuard, nil] existing guard to reuse (e.g. streaming)
+      # @return [void]
+      def postflight!(response, response_guard: nil)
+        return if skip_postflight? || response.nil?
+
+        guard = response_guard || ResponseGuard.new(policy: ctx.policy)
+        guard.inspect_body!(response.body)
+      end
+
+      ##
+      # @return [Boolean] true when remote budget/policy preflight must be skipped
+      def skip_preflight? = false
+
+      ##
+      # @return [Boolean] true when ResponseGuard postflight must be skipped
+      def skip_postflight? = false
+
       def check_timeout!
+        ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
+      rescue RequestTimedOut
+        log_timeout!(reason: 'budget_exhausted')
+        raise
+      end
+
+      # @return [ResponseGuard, nil]
+      def response_guard = nil
+
+      # @param error [StandardError]
+      # @return [void]
+      # @raise [StandardError]
+      def handle_error(error)
+        if timeout_error?(error)
+          log_timeout!(reason: 'transport')
+          Log.debug("#{self.class}: transport timeout message=#{error.message}")
+          raise RequestTimedOut, error.message
+        elsif connection_error?(error)
+          translate_connection_error(error)
+        else
+          raise error
+        end
+      end
+
+      # @param reason [String] timeout classification (budget_exhausted / transport)
+      # @return [void]
+      # rubocop:disable Metrics/AbcSize -- structured timeout fields stay in one log line
+      def log_timeout!(reason:)
         remaining = ctx.budget.remaining_timeout_seconds
-        raise RequestTimedOut, 'Request timed out' if remaining && remaining <= 0
+        remaining_label = remaining.nil? ? 'untracked' : format('%.3f', remaining)
+        detail = [
+          "strategy=#{self.class.name}",
+          "host=#{ctx.url.host}",
+          "elapsed=#{format('%.3f', ctx.budget.elapsed_seconds)}s",
+          "budget_remaining=#{remaining_label}",
+          "reason=#{reason}"
+        ]
+        Log.info("#{self.class}: request timeout #{detail.join(' ')}")
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      # @param error [StandardError]
+      # @return [void]
+      # @raise [StandardError]
+      def translate_connection_error(error)
+        raise error
+      end
+
+      # @param error [StandardError]
+      # @return [Boolean]
+      def timeout_error?(error)
+        error.is_a?(Faraday::TimeoutError) ||
+          error.is_a?(Timeout::Error) ||
+          (defined?(Puppeteer::TimeoutError) && error.is_a?(Puppeteer::TimeoutError))
+      end
+
+      # @param error [StandardError]
+      # @return [Boolean]
+      def connection_error?(error)
+        error.is_a?(Faraday::ConnectionFailed) || error.is_a?(Faraday::SSLError)
       end
     end
   end

data/lib/html2rss/request_service.rb

@@ -6,7 +6,10 @@ require 'forwardable'
 module Html2rss
   ##
   # Requests website URLs to retrieve their HTML for further processing.
-  # Provides strategies, e.g. integrating Browserless.io.
+  # Provides concrete transport strategies (e.g. Faraday, Browserless).
+  #
+  # Feed-level +:auto+ is not registered here — {FeedPipeline::StrategyPlan} resolves
+  # it to a concrete strategy (or {FeedPipeline::AutoFallback} chain) before execute.
   class RequestService
     include Singleton
 
@@ -18,8 +21,10 @@ module Html2rss
     class UnsupportedUrlScheme < Html2rss::Error; end
     # Raised when a response type cannot be parsed.
     class UnsupportedResponseContentType < Html2rss::Error; end
-    # Raised when request limits are exceeded.
+    # Raised when HTTP request slot limits are exceeded.
     class RequestBudgetExceeded < Html2rss::Error; end
+    # Raised when Browserless preload interaction limits are exceeded.
+    class InteractionBudgetExceeded < Html2rss::Error; end
     # Raised when policy denies private-network access.
     class PrivateNetworkDenied < Html2rss::Error; end
     # Raised when cross-origin follow-up requests are denied.

data/lib/html2rss/request_session/pager/base.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Abstract base class for pagination strategies.
+      class Base
+        include Enumerable
+
+        # Default maximum number of pages to fetch if omitted.
+        # @return [Integer]
+        DEFAULT_MAX_PAGES = 5
+
+        # @param session [RequestSession] request session used to execute follow-ups
+        # @param initial_response [RequestService::Response] first page response
+        # @param config [Hash, Integer] pagination configuration or max_pages integer
+        # @param logger [Logger] logger used for pagination stop warnings
+        def initialize(session:, initial_response:, config: {}, logger: Html2rss::Log)
+          @session = session
+          @initial_response = initial_response
+          @config = config.is_a?(Hash) ? config : { max_pages: config }
+          @logger = logger
+        end
+
+        # @yield [RequestService::Response] each page response
+        # @return [Enumerator] enumerator when no block is given
+        def each
+          return enum_for(:each) unless block_given?
+
+          yield initial_response
+
+          current_response = initial_response
+          session.effective_page_budget(max_pages).pred.times do |index|
+            next_url = next_page_url(current_response, page_number: index + 2)
+            break unless follow_up_allowed?(next_url)
+
+            current_response = fetch_follow_up_response_or_stop(next_url, current_response.url)
+            break unless current_response
+
+            yield current_response
+          end
+        end
+
+        private
+
+        attr_reader :session, :initial_response, :config, :logger
+
+        # @return [Integer] configured maximum pages (always a positive Integer)
+        def max_pages
+          value = config.fetch(:max_pages, DEFAULT_MAX_PAGES)
+          value.is_a?(Integer) && value.positive? ? value : DEFAULT_MAX_PAGES
+        end
+
+        # @param next_url [Html2rss::Url, String, nil]
+        # @return [Boolean]
+        def follow_up_allowed?(next_url)
+          !next_url.nil? && !next_url.to_s.empty? && !session.visited?(next_url)
+        end
+
+        # @param next_url [Html2rss::Url, String]
+        # @param origin_url [Html2rss::Url, String]
+        # @return [RequestService::Response, nil]
+        def fetch_follow_up_response_or_stop(next_url, origin_url)
+          session.follow_up(url: next_url, relation: :pagination, origin_url:)
+        rescue RequestService::RequestBudgetExceeded => error
+          logger.warn(
+            "#{self.class}: pagination stopped at #{next_url} - #{error.message}. " \
+            "Retry with --max-requests #{session.max_requests + 1} or increase request.max_requests in the config."
+          )
+          nil
+        end
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer] 1-based target page number (e.g. 2 for second page)
+        # @return [Html2rss::Url, String, nil]
+        def next_page_url(page_response, page_number:)
+          raise NotImplementedError, "#{self.class}#next_page_url must be implemented by subclass"
+        end
+      end
+    end
+  end
+end