html2rss 0.22.2 → 0.23.0

data/lib/html2rss/request_service/strategy.rb

@@ -3,7 +3,10 @@
 module Html2rss
   class RequestService
     ##
-    # Defines the interface of every request strategy.
+    # Defines the guarded request interface for every strategy adapter.
+    #
+    # `#execute` always runs timeout check → budget/policy preflight → `#fetch`
+    # → ResponseGuard postflight so new adapters cannot skip shared controls.
     class Strategy
       ##
       # @param ctx [Context] the context for the request
@@ -12,11 +15,17 @@ module Html2rss
       end
 
       ##
-      # Executes the request.
+      # Executes the request through the shared preflight/postflight path.
+      #
       # @return [Response] the response from the strategy
-      # @raise [NotImplementedError] if the method is not implemented by the subclass
+      # @raise [NotImplementedError] if the subclass does not implement `#fetch`
       def execute
-        raise NotImplementedError, 'Subclass must implement #execute'
+        check_timeout!
+        response = perform_execute
+        postflight!(response, response_guard:)
+        response
+      rescue StandardError => error
+        handle_error(error)
       end
 
       private
@@ -24,9 +33,96 @@ module Html2rss
       # @return [Context] the context for the request
       attr_reader :ctx
 
+      ##
+      # Adapter hook: perform the preflight, fetch, and optional retry lifecycle.
+      #
+      # @return [Response] normalized response
+      def perform_execute
+        preflight!
+        fetch
+      end
+
+      ##
+      # Adapter hook: perform the transport-specific fetch after preflight.
+      #
+      # @return [Response] normalized response
+      # @raise [NotImplementedError] when a subclass omits the hook
+      def fetch
+        raise NotImplementedError, 'Subclass must implement #fetch'
+      end
+
+      ##
+      # Consumes a request slot and validates the destination before transport work.
+      #
+      # @param consume_budget [Boolean] whether to decrement the request budget
+      # @return [void]
+      def preflight!(consume_budget: true)
+        return if skip_preflight?
+
+        ctx.budget.consume! if consume_budget
+        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
+      end
+
+      ##
+      # Enforces response-size and blocked-surface checks after transport work.
+      #
+      # @param response [Response, nil] adapter response
+      # @param response_guard [ResponseGuard, nil] existing guard to reuse (e.g. streaming)
+      # @return [void]
+      def postflight!(response, response_guard: nil)
+        return if skip_postflight? || response.nil?
+
+        guard = response_guard || ResponseGuard.new(policy: ctx.policy)
+        guard.inspect_body!(response.body)
+      end
+
+      ##
+      # @return [Boolean] true when remote budget/policy preflight must be skipped
+      def skip_preflight? = false
+
+      ##
+      # @return [Boolean] true when ResponseGuard postflight must be skipped
+      def skip_postflight? = false
+
       def check_timeout!
-        remaining = ctx.budget.remaining_timeout_seconds
-        raise RequestTimedOut, 'Request timed out' if remaining && remaining <= 0
+        ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
+      end
+
+      # @return [ResponseGuard, nil]
+      def response_guard = nil
+
+      # @param error [StandardError]
+      # @return [void]
+      # @raise [StandardError]
+      def handle_error(error)
+        if timeout_error?(error)
+          raise RequestTimedOut, error.message
+        elsif connection_error?(error)
+          translate_connection_error(error)
+        else
+          raise error
+        end
+      end
+
+      # @param error [StandardError]
+      # @return [void]
+      # @raise [StandardError]
+      def translate_connection_error(error)
+        raise error
+      end
+
+      # @param error [StandardError]
+      # @return [Boolean]
+      def timeout_error?(error)
+        error.is_a?(Faraday::TimeoutError) ||
+          error.is_a?(Timeout::Error) ||
+          (defined?(Puppeteer::TimeoutError) && error.is_a?(Puppeteer::TimeoutError))
+      end
+
+      # @param error [StandardError]
+      # @return [Boolean]
+      def connection_error?(error)
+        error.is_a?(Faraday::ConnectionFailed) || error.is_a?(Faraday::SSLError)
       end
     end
   end

data/lib/html2rss/request_service.rb

@@ -6,7 +6,10 @@ require 'forwardable'
 module Html2rss
   ##
   # Requests website URLs to retrieve their HTML for further processing.
-  # Provides strategies, e.g. integrating Browserless.io.
+  # Provides concrete transport strategies (e.g. Faraday, Browserless).
+  #
+  # Feed-level +:auto+ is not registered here — {FeedPipeline::StrategyPlan} resolves
+  # it to a concrete strategy (or {FeedPipeline::AutoFallback} chain) before execute.
   class RequestService
     include Singleton
 
@@ -18,8 +21,10 @@ module Html2rss
     class UnsupportedUrlScheme < Html2rss::Error; end
     # Raised when a response type cannot be parsed.
     class UnsupportedResponseContentType < Html2rss::Error; end
-    # Raised when request limits are exceeded.
+    # Raised when HTTP request slot limits are exceeded.
     class RequestBudgetExceeded < Html2rss::Error; end
+    # Raised when Browserless preload interaction limits are exceeded.
+    class InteractionBudgetExceeded < Html2rss::Error; end
     # Raised when policy denies private-network access.
     class PrivateNetworkDenied < Html2rss::Error; end
     # Raised when cross-origin follow-up requests are denied.

data/lib/html2rss/request_session/pager/base.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Abstract base class for pagination strategies.
+      class Base
+        include Enumerable
+
+        # Default maximum number of pages to fetch if omitted.
+        # @return [Integer]
+        DEFAULT_MAX_PAGES = 5
+
+        # @param session [RequestSession] request session used to execute follow-ups
+        # @param initial_response [RequestService::Response] first page response
+        # @param config [Hash, Integer] pagination configuration or max_pages integer
+        # @param logger [Logger] logger used for pagination stop warnings
+        def initialize(session:, initial_response:, config: {}, logger: Html2rss::Log)
+          @session = session
+          @initial_response = initial_response
+          @config = config.is_a?(Hash) ? config : { max_pages: config }
+          @logger = logger
+        end
+
+        # @yield [RequestService::Response] each page response
+        # @return [Enumerator] enumerator when no block is given
+        def each
+          return enum_for(:each) unless block_given?
+
+          yield initial_response
+
+          current_response = initial_response
+          session.effective_page_budget(max_pages).pred.times do |index|
+            next_url = next_page_url(current_response, page_number: index + 2)
+            break unless follow_up_allowed?(next_url)
+
+            current_response = fetch_follow_up_response_or_stop(next_url, current_response.url)
+            break unless current_response
+
+            yield current_response
+          end
+        end
+
+        private
+
+        attr_reader :session, :initial_response, :config, :logger
+
+        # @return [Integer] configured maximum pages (always a positive Integer)
+        def max_pages
+          value = config.fetch(:max_pages, DEFAULT_MAX_PAGES)
+          value.is_a?(Integer) && value.positive? ? value : DEFAULT_MAX_PAGES
+        end
+
+        # @param next_url [Html2rss::Url, String, nil]
+        # @return [Boolean]
+        def follow_up_allowed?(next_url)
+          !next_url.nil? && !next_url.to_s.empty? && !session.visited?(next_url)
+        end
+
+        # @param next_url [Html2rss::Url, String]
+        # @param origin_url [Html2rss::Url, String]
+        # @return [RequestService::Response, nil]
+        def fetch_follow_up_response_or_stop(next_url, origin_url)
+          session.follow_up(url: next_url, relation: :pagination, origin_url:)
+        rescue RequestService::RequestBudgetExceeded => error
+          logger.warn(
+            "#{self.class}: pagination stopped at #{next_url} - #{error.message}. " \
+            "Retry with --max-requests #{session.max_requests + 1} or increase request.max_requests in the config."
+          )
+          nil
+        end
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer] 1-based target page number (e.g. 2 for second page)
+        # @return [Html2rss::Url, String, nil]
+        def next_page_url(page_response, page_number:)
+          raise NotImplementedError, "#{self.class}#next_page_url must be implemented by subclass"
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/pager/custom_selector.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Extracts the next-page URL using a configurable CSS or XPath selector.
+      class CustomSelector < Base
+        private
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer]
+        # @return [String, nil]
+        def next_page_url(page_response, page_number: nil) # rubocop:disable Lint/UnusedMethodArgument
+          selector = config[:selector]
+          return nil if selector.nil? || selector.empty?
+
+          node = extract_node(page_response.parsed_body, selector)
+          return nil unless node
+
+          href = node['href'] || node.text
+          return nil if href.nil? || href.strip.empty?
+
+          Html2rss::Url.from_relative(href.strip, page_response.url)
+        end
+
+        # @param parsed_body [Nokogiri::HTML::Document]
+        # @param selector [String]
+        # @return [Nokogiri::XML::Node, nil]
+        def extract_node(parsed_body, selector)
+          parsed_body.at_css(selector) || parsed_body.at_xpath(selector)
+        rescue Nokogiri::CSS::SyntaxError, Nokogiri::XML::XPath::SyntaxError
+          # Pure XPath like `descendant::a` fails LOOKS_LIKE_XPATH / at_css; fall back explicitly.
+          parsed_body.at_xpath(selector)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/pager/json_cursor.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Digs into JSON response payloads to extract next page URLs or cursor tokens.
+      class JsonCursor < Base
+        # Default query parameter name for cursor token pagination.
+        # @return [String]
+        DEFAULT_PARAM = 'cursor'
+
+        private
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer]
+        # @return [String, nil]
+        def next_page_url(page_response, page_number: nil) # rubocop:disable Lint/UnusedMethodArgument
+          json_data = parse_json_body(page_response)
+          return nil unless json_data
+
+          url_from_next_path(json_data, page_response.url) ||
+            url_from_cursor_path(json_data, page_response.url)
+        end
+
+        # @param json_data [Hash]
+        # @param origin_url [Html2rss::Url]
+        # @return [String, nil]
+        def url_from_next_path(json_data, origin_url)
+          return nil unless (next_url_path = config[:next_url_path])
+
+          next_url = dig_path(json_data, next_url_path)
+          Html2rss::Url.from_relative(next_url.to_s, origin_url) if next_url && !next_url.to_s.empty?
+        end
+
+        # @param json_data [Hash]
+        # @param origin_url [Html2rss::Url]
+        # @return [Html2rss::Url, nil]
+        def url_from_cursor_path(json_data, origin_url)
+          return nil unless (cursor_path = config[:cursor_path])
+
+          cursor_val = dig_path(json_data, cursor_path)
+          return nil if cursor_val.nil? || cursor_val.to_s.empty?
+
+          param = config.fetch(:param, DEFAULT_PARAM)
+          url = Html2rss::Url.from_absolute(origin_url)
+          url.with_query_values(url.query_values.merge(param.to_s => cursor_val.to_s))
+        end
+
+        # @param page_response [RequestService::Response]
+        # @return [Hash, nil]
+        def parse_json_body(page_response)
+          JSON.parse(page_response.body)
+        rescue JSON::ParserError
+          nil
+        end
+
+        # @param hash [Hash]
+        # @param path [String, Symbol]
+        # @return [Object, nil]
+        def dig_path(hash, path)
+          return nil unless hash.is_a?(Hash)
+
+          keys = path.to_s.split('.')
+          result = hash
+          keys.each do |key|
+            return nil unless result.is_a?(Hash)
+
+            result = result[key] || result[key.to_sym]
+          end
+          result
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/pager/offset.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Paginate using numeric offset increments in query parameters or URL path templates.
+      class Offset < Base
+        # Default query parameter name for offset pagination.
+        # @return [String]
+        DEFAULT_PARAM = 'offset'
+
+        # Default starting numeric offset value.
+        # @return [Integer]
+        DEFAULT_START_OFFSET = 0
+
+        # Default offset increment per page.
+        # @return [Integer]
+        DEFAULT_INCREMENT = 20
+
+        private
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer]
+        # @return [Html2rss::Url]
+        def next_page_url(page_response, page_number:)
+          param = config.fetch(:param, DEFAULT_PARAM)
+          target_offset_val = target_offset_for(page_number)
+          url = Html2rss::Url.from_absolute(page_response.url)
+
+          if url.to_s.include?('{offset}')
+            Html2rss::Url.from_absolute(url.to_s.gsub('{offset}', target_offset_val.to_s))
+          else
+            url.with_query_values(url.query_values.merge(param.to_s => target_offset_val.to_s))
+          end
+        end
+
+        # @param page_number [Integer]
+        # @return [Integer]
+        def target_offset_for(page_number)
+          start_offset = config.fetch(:start_offset, DEFAULT_START_OFFSET)
+          increment = config.fetch(:increment, DEFAULT_INCREMENT)
+          start_offset + ((page_number - 1) * increment)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/pager/rel_next.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Traverses a rel=next pagination chain for HTML documents.
+      class RelNext < Base
+        private
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer]
+        # @return [String, nil]
+        def next_page_url(page_response, page_number: nil) # rubocop:disable Lint/UnusedMethodArgument
+          href = page_response.parsed_body.at_css('link[rel~="next"][href], a[rel~="next"][href]')&.[]('href')
+          return nil if href.nil? || href.empty?
+
+          Html2rss::Url.from_relative(href, page_response.url)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/pager/url_template.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Paginate using page number increments in query parameters or URL path templates.
+      class UrlTemplate < Base
+        # Default query parameter name for page number pagination.
+        # @return [String]
+        DEFAULT_PARAM = 'page'
+
+        # Default starting page number.
+        # @return [Integer]
+        DEFAULT_START_PAGE = 1
+
+        # Default page number step increment.
+        # @return [Integer]
+        DEFAULT_STEP = 1
+
+        private
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer] 1-based target page number (e.g. 2 for 2nd page)
+        # @return [Html2rss::Url]
+        def next_page_url(page_response, page_number:)
+          param = config.fetch(:param, DEFAULT_PARAM)
+          target_page_val = target_page_for(page_number)
+          url = Html2rss::Url.from_absolute(page_response.url)
+
+          if url.to_s.include?('{page}')
+            Html2rss::Url.from_absolute(url.to_s.gsub('{page}', target_page_val.to_s))
+          else
+            url.with_query_values(url.query_values.merge(param.to_s => target_page_val.to_s))
+          end
+        end
+
+        # @param page_number [Integer]
+        # @return [Integer]
+        def target_page_for(page_number)
+          start_page = config.fetch(:start_page, DEFAULT_START_PAGE)
+          step = config.fetch(:step, DEFAULT_STEP)
+          start_page + ((page_number - 1) * step)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/pager.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    ##
+    # Factory for building pagination strategy instances.
+    module Pager
+      # Mapping of strategy symbols to strategy class implementations.
+      # @return [Hash{Symbol => Class}]
+      STRATEGIES = {
+        rel_next: RelNext,
+        custom_selector: CustomSelector,
+        url_template: UrlTemplate,
+        offset: Offset,
+        json_cursor: JsonCursor
+      }.freeze
+
+      ##
+      # Returns the supported pagination strategy names.
+      #
+      # @return [Array<String>] strategy names derived from {STRATEGIES}
+      def self.strategy_names
+        STRATEGIES.keys.map(&:to_s).freeze
+      end
+
+      ##
+      # Returns the number of request slots needed for the pagination configuration.
+      #
+      # @param config [Hash, Integer, nil] pagination configuration
+      # @return [Integer] number of follow-up requests needed
+      def self.request_slots_for(config)
+        return 0 unless config
+
+        max_pages = normalize_config(config)[:max_pages] || Base::DEFAULT_MAX_PAGES
+        [max_pages - 1, 0].max
+      end
+
+      ##
+      # Returns a pager instance for the provided configuration.
+      #
+      # @param config [Hash, Integer, nil] pagination configuration
+      # @param session [RequestSession] request session used to execute follow-ups
+      # @param initial_response [RequestService::Response] initial page response
+      # @return [RequestSession::Pager::Base] pager strategy instance
+      def self.for(config, session:, initial_response:)
+        normalized_config = normalize_config(config)
+        strategy_name = normalized_config.fetch(:strategy, :rel_next).to_sym
+
+        klass = STRATEGIES.fetch(strategy_name) do
+          raise ArgumentError, "Unknown pagination strategy: #{strategy_name}"
+        end
+
+        klass.new(session:, initial_response:, config: normalized_config)
+      end
+
+      ##
+      # Normalizes integer, hash, or nil inputs into a strategy config hash.
+      #
+      # @param config [Hash, Integer, nil]
+      # @return [Hash]
+      def self.normalize_config(config)
+        case config
+        when Integer
+          { max_pages: config, strategy: :rel_next }
+        when Hash
+          normalized = config.transform_keys(&:to_sym)
+          normalized[:strategy] = (normalized[:strategy] || :rel_next).to_sym
+          normalized
+        else
+          { max_pages: Base::DEFAULT_MAX_PAGES, strategy: :rel_next }
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session.rb

@@ -6,26 +6,22 @@ module Html2rss
   class RequestSession
     class << self
       ##
-      # Builds a request session from translated runtime request inputs.
+      # Builds a request session from config, strategy, and shared budget/policy.
       #
-      # @param runtime_input [RuntimeInput] translated runtime request inputs
-      # @param budget [RequestService::Budget, nil] optional shared budget for multi-attempt runs
+      # Context owns URL/header/request normalization once; callers must not
+      # pre-normalize through a passthrough bag.
+      #
+      # @param config [Html2rss::Config] validated feed config
+      # @param strategy [Symbol] request strategy for the session
+      # @param budget [RequestService::Budget] shared request budget
+      # @param policy [RequestService::Policy] request policy (from FeedPipeline::RuntimePolicy.resources_for)
       # @param logger [Logger] logger used for operational warnings
       # @return [RequestSession] configured request session
-      def from_runtime_input(runtime_input, budget: nil, logger: Html2rss::Log) # rubocop:disable Metrics/MethodLength
-        context_options = {
-          url: runtime_input.url,
-          headers: runtime_input.headers,
-          request: runtime_input.request,
-          policy: runtime_input.request_policy
-        }
-        context_options[:budget] = budget unless budget.nil?
-
-        new(
-          context: RequestService::Context.new(**context_options),
-          strategy: runtime_input.strategy,
-          logger:
+      def build(config:, strategy:, budget:, policy:, logger: Html2rss::Log)
+        context = RequestService::Context.new(
+          url: config.url, headers: config.headers, request: config.request, policy:, budget:
         )
+        new(context:, strategy:, logger:)
       end
     end
 
@@ -48,6 +44,19 @@ module Html2rss
       execute(context).tap { |response| remember!(response.url) }
     end
 
+    ##
+    # Returns an enumerable sequence of page responses for the given initial response
+    # and optional pagination configuration.
+    #
+    # @param initial_response [RequestService::Response] initial page response
+    # @param pagination_config [Hash, Integer, nil] pagination configuration
+    # @return [Enumerable<RequestService::Response>] page responses
+    def page_responses(initial_response, pagination_config: nil)
+      return [initial_response] unless pagination_config
+
+      Pager.for(pagination_config, session: self, initial_response:)
+    end
+
     ##
     # Executes a follow-up request sharing policy, headers, and budget.
     #

data/lib/html2rss/selectors/item_scope.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    ##
+    # Per-item extraction scope: owns the article node and page +base_url+ for one
+    # extraction pass. Channel hashes are injected (and cached) by {Selectors}.
+    #
+    # Distinct from {Context}, which is the post-processor invocation bag (+options+).
+    # Post-processor +config+ is derived from {#channel} — not a parallel bag.
+    ItemScope = Data.define(:item, :base_url, :scraper, :channel) do
+      ##
+      # Selects an attribute using this scope's item and base_url.
+      #
+      # @param name [Symbol, String]
+      # @return [Object, Array<Object>]
+      def select(name) = scraper.select_in_scope(name, self)
+
+      ##
+      # Builds a post-processor {Context} carrying this scope for nested selects.
+      #
+      # @param options [Hash] post-processor options from the selector config
+      # @option options [String] :name post-processor name
+      # @return [Context]
+      def context_for(options:)
+        Context.new(options:, item:, config: { channel: }, scraper:, item_scope: self)
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/base.rb

@@ -29,7 +29,7 @@ module Html2rss
         # @param context [Selectors::Context] call-site context used for richer validation errors
         # @return [void]
         # @raise [InvalidType] if the value is not of the expected type(s)
-        def self.assert_type(value, types = [], name, context:)
+        def self.assert_type(value, types, name, context:)
           return if Array(types).any? { |type| value.is_a?(type) }
 
           options = if context.respond_to?(:options)

data/lib/html2rss/selectors/post_processors/markdown_to_html.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'kramdown'
-require_relative 'sanitize_html'
 
 module Html2rss
   class Selectors

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require 'sanitize'
-require_relative 'html_transformers/transform_urls_to_absolute_ones'
-require_relative 'html_transformers/wrap_img_in_a'
 
 module Html2rss
   class Selectors