html2rss 0.22.2 → 0.24.0

data/lib/html2rss/request_session/pager/custom_selector.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Extracts the next-page URL using a configurable CSS or XPath selector.
+      class CustomSelector < Base
+        private
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer]
+        # @return [String, nil]
+        def next_page_url(page_response, page_number: nil) # rubocop:disable Lint/UnusedMethodArgument
+          selector = config[:selector]
+          return nil if selector.nil? || selector.empty?
+
+          node = extract_node(page_response.parsed_body, selector)
+          return nil unless node
+
+          href = node['href'] || node.text
+          return nil if href.nil? || href.strip.empty?
+
+          Html2rss::Url.from_relative(href.strip, page_response.url)
+        end
+
+        # @param parsed_body [Nokogiri::HTML::Document]
+        # @param selector [String]
+        # @return [Nokogiri::XML::Node, nil]
+        def extract_node(parsed_body, selector)
+          parsed_body.at_css(selector) || parsed_body.at_xpath(selector)
+        rescue Nokogiri::CSS::SyntaxError, Nokogiri::XML::XPath::SyntaxError
+          # Pure XPath like `descendant::a` fails LOOKS_LIKE_XPATH / at_css; fall back explicitly.
+          parsed_body.at_xpath(selector)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/pager/json_cursor.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Digs into JSON response payloads to extract next page URLs or cursor tokens.
+      class JsonCursor < Base
+        # Default query parameter name for cursor token pagination.
+        # @return [String]
+        DEFAULT_PARAM = 'cursor'
+
+        private
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer]
+        # @return [String, nil]
+        def next_page_url(page_response, page_number: nil) # rubocop:disable Lint/UnusedMethodArgument
+          json_data = parse_json_body(page_response)
+          return nil unless json_data
+
+          url_from_next_path(json_data, page_response.url) ||
+            url_from_cursor_path(json_data, page_response.url)
+        end
+
+        # @param json_data [Hash]
+        # @param origin_url [Html2rss::Url]
+        # @return [String, nil]
+        def url_from_next_path(json_data, origin_url)
+          return nil unless (next_url_path = config[:next_url_path])
+
+          next_url = dig_path(json_data, next_url_path)
+          Html2rss::Url.from_relative(next_url.to_s, origin_url) if next_url && !next_url.to_s.empty?
+        end
+
+        # @param json_data [Hash]
+        # @param origin_url [Html2rss::Url]
+        # @return [Html2rss::Url, nil]
+        def url_from_cursor_path(json_data, origin_url)
+          return nil unless (cursor_path = config[:cursor_path])
+
+          cursor_val = dig_path(json_data, cursor_path)
+          return nil if cursor_val.nil? || cursor_val.to_s.empty?
+
+          param = config.fetch(:param, DEFAULT_PARAM)
+          url = Html2rss::Url.from_absolute(origin_url)
+          url.with_query_values(url.query_values.merge(param.to_s => cursor_val.to_s))
+        end
+
+        # @param page_response [RequestService::Response]
+        # @return [Hash, nil]
+        def parse_json_body(page_response)
+          JSON.parse(page_response.body)
+        rescue JSON::ParserError
+          nil
+        end
+
+        # @param hash [Hash]
+        # @param path [String, Symbol]
+        # @return [Object, nil]
+        def dig_path(hash, path)
+          return nil unless hash.is_a?(Hash)
+
+          keys = path.to_s.split('.')
+          result = hash
+          keys.each do |key|
+            return nil unless result.is_a?(Hash)
+
+            result = result[key] || result[key.to_sym]
+          end
+          result
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/pager/offset.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Paginate using numeric offset increments in query parameters or URL path templates.
+      class Offset < Base
+        # Default query parameter name for offset pagination.
+        # @return [String]
+        DEFAULT_PARAM = 'offset'
+
+        # Default starting numeric offset value.
+        # @return [Integer]
+        DEFAULT_START_OFFSET = 0
+
+        # Default offset increment per page.
+        # @return [Integer]
+        DEFAULT_INCREMENT = 20
+
+        private
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer]
+        # @return [Html2rss::Url]
+        def next_page_url(page_response, page_number:)
+          param = config.fetch(:param, DEFAULT_PARAM)
+          target_offset_val = target_offset_for(page_number)
+          url = Html2rss::Url.from_absolute(page_response.url)
+
+          if url.to_s.include?('{offset}')
+            Html2rss::Url.from_absolute(url.to_s.gsub('{offset}', target_offset_val.to_s))
+          else
+            url.with_query_values(url.query_values.merge(param.to_s => target_offset_val.to_s))
+          end
+        end
+
+        # @param page_number [Integer]
+        # @return [Integer]
+        def target_offset_for(page_number)
+          start_offset = config.fetch(:start_offset, DEFAULT_START_OFFSET)
+          increment = config.fetch(:increment, DEFAULT_INCREMENT)
+          start_offset + ((page_number - 1) * increment)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/pager/rel_next.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Traverses a rel=next pagination chain for HTML documents.
+      class RelNext < Base
+        private
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer]
+        # @return [String, nil]
+        def next_page_url(page_response, page_number: nil) # rubocop:disable Lint/UnusedMethodArgument
+          href = page_response.parsed_body.at_css('link[rel~="next"][href], a[rel~="next"][href]')&.[]('href')
+          return nil if href.nil? || href.empty?
+
+          Html2rss::Url.from_relative(href, page_response.url)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/pager/url_template.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    module Pager
+      ##
+      # Paginate using page number increments in query parameters or URL path templates.
+      class UrlTemplate < Base
+        # Default query parameter name for page number pagination.
+        # @return [String]
+        DEFAULT_PARAM = 'page'
+
+        # Default starting page number.
+        # @return [Integer]
+        DEFAULT_START_PAGE = 1
+
+        # Default page number step increment.
+        # @return [Integer]
+        DEFAULT_STEP = 1
+
+        private
+
+        # @param page_response [RequestService::Response]
+        # @param page_number [Integer] 1-based target page number (e.g. 2 for 2nd page)
+        # @return [Html2rss::Url]
+        def next_page_url(page_response, page_number:)
+          param = config.fetch(:param, DEFAULT_PARAM)
+          target_page_val = target_page_for(page_number)
+          url = Html2rss::Url.from_absolute(page_response.url)
+
+          if url.to_s.include?('{page}')
+            Html2rss::Url.from_absolute(url.to_s.gsub('{page}', target_page_val.to_s))
+          else
+            url.with_query_values(url.query_values.merge(param.to_s => target_page_val.to_s))
+          end
+        end
+
+        # @param page_number [Integer]
+        # @return [Integer]
+        def target_page_for(page_number)
+          start_page = config.fetch(:start_page, DEFAULT_START_PAGE)
+          step = config.fetch(:step, DEFAULT_STEP)
+          start_page + ((page_number - 1) * step)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/pager.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    ##
+    # Factory for building pagination strategy instances.
+    module Pager
+      # Mapping of strategy symbols to strategy class implementations.
+      # @return [Hash{Symbol => Class}]
+      STRATEGIES = {
+        rel_next: RelNext,
+        custom_selector: CustomSelector,
+        url_template: UrlTemplate,
+        offset: Offset,
+        json_cursor: JsonCursor
+      }.freeze
+
+      ##
+      # Returns the supported pagination strategy names.
+      #
+      # @return [Array<String>] strategy names derived from {STRATEGIES}
+      def self.strategy_names
+        STRATEGIES.keys.map(&:to_s).freeze
+      end
+
+      ##
+      # Returns the number of request slots needed for the pagination configuration.
+      #
+      # @param config [Hash, Integer, nil] pagination configuration
+      # @return [Integer] number of follow-up requests needed
+      def self.request_slots_for(config)
+        return 0 unless config
+
+        max_pages = normalize_config(config)[:max_pages] || Base::DEFAULT_MAX_PAGES
+        [max_pages - 1, 0].max
+      end
+
+      ##
+      # Returns a pager instance for the provided configuration.
+      #
+      # @param config [Hash, Integer, nil] pagination configuration
+      # @param session [RequestSession] request session used to execute follow-ups
+      # @param initial_response [RequestService::Response] initial page response
+      # @return [RequestSession::Pager::Base] pager strategy instance
+      def self.for(config, session:, initial_response:)
+        normalized_config = normalize_config(config)
+        strategy_name = normalized_config.fetch(:strategy, :rel_next).to_sym
+
+        klass = STRATEGIES.fetch(strategy_name) do
+          raise ArgumentError, "Unknown pagination strategy: #{strategy_name}"
+        end
+
+        klass.new(session:, initial_response:, config: normalized_config)
+      end
+
+      ##
+      # Normalizes integer, hash, or nil inputs into a strategy config hash.
+      #
+      # @param config [Hash, Integer, nil]
+      # @return [Hash]
+      def self.normalize_config(config)
+        case config
+        when Integer
+          { max_pages: config, strategy: :rel_next }
+        when Hash
+          normalized = config.transform_keys(&:to_sym)
+          normalized[:strategy] = (normalized[:strategy] || :rel_next).to_sym
+          normalized
+        else
+          { max_pages: Base::DEFAULT_MAX_PAGES, strategy: :rel_next }
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session.rb

@@ -6,26 +6,22 @@ module Html2rss
   class RequestSession
     class << self
       ##
-      # Builds a request session from translated runtime request inputs.
+      # Builds a request session from config, strategy, and shared budget/policy.
       #
-      # @param runtime_input [RuntimeInput] translated runtime request inputs
-      # @param budget [RequestService::Budget, nil] optional shared budget for multi-attempt runs
+      # Context owns URL/header/request normalization once; callers must not
+      # pre-normalize through a passthrough bag.
+      #
+      # @param config [Html2rss::Config] validated feed config
+      # @param strategy [Symbol] request strategy for the session
+      # @param budget [RequestService::Budget] shared request budget
+      # @param policy [RequestService::Policy] request policy (from FeedPipeline::RuntimePolicy.resources_for)
       # @param logger [Logger] logger used for operational warnings
       # @return [RequestSession] configured request session
-      def from_runtime_input(runtime_input, budget: nil, logger: Html2rss::Log) # rubocop:disable Metrics/MethodLength
-        context_options = {
-          url: runtime_input.url,
-          headers: runtime_input.headers,
-          request: runtime_input.request,
-          policy: runtime_input.request_policy
-        }
-        context_options[:budget] = budget unless budget.nil?
-
-        new(
-          context: RequestService::Context.new(**context_options),
-          strategy: runtime_input.strategy,
-          logger:
+      def build(config:, strategy:, budget:, policy:, logger: Html2rss::Log)
+        context = RequestService::Context.new(
+          url: config.url, headers: config.headers, request: config.request, policy:, budget:
         )
+        new(context:, strategy:, logger:)
       end
     end
 
@@ -48,6 +44,19 @@ module Html2rss
       execute(context).tap { |response| remember!(response.url) }
     end
 
+    ##
+    # Returns an enumerable sequence of page responses for the given initial response
+    # and optional pagination configuration.
+    #
+    # @param initial_response [RequestService::Response] initial page response
+    # @param pagination_config [Hash, Integer, nil] pagination configuration
+    # @return [Enumerable<RequestService::Response>] page responses
+    def page_responses(initial_response, pagination_config: nil)
+      return [initial_response] unless pagination_config
+
+      Pager.for(pagination_config, session: self, initial_response:)
+    end
+
     ##
     # Executes a follow-up request sharing policy, headers, and budget.
     #
@@ -100,6 +109,10 @@ module Html2rss
       visited_urls.add(normalize_url(url))
     end
 
+    ##
+    # @return [Html2rss::Url] the session's current request URL
+    def url = context.url
+
     private
 
     attr_reader :context, :strategy, :logger, :visited_urls

data/lib/html2rss/selectors/item_scope.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    ##
+    # Per-item extraction scope: owns the article node and page +base_url+ for one
+    # extraction pass. Channel hashes are injected (and cached) by {Selectors}.
+    #
+    # Distinct from {Context}, which is the post-processor invocation bag (+options+).
+    # Post-processor +config+ is derived from {#channel} — not a parallel bag.
+    ItemScope = Data.define(:item, :base_url, :scraper, :channel) do
+      ##
+      # Selects an attribute using this scope's item and base_url.
+      #
+      # @param name [Symbol, String]
+      # @return [Object, Array<Object>]
+      def select(name) = scraper.select_in_scope(name, self)
+
+      ##
+      # Builds a post-processor {Context} carrying this scope for nested selects.
+      #
+      # @param options [Hash] post-processor options from the selector config
+      # @option options [String] :name post-processor name
+      # @return [Context]
+      def context_for(options:)
+        Context.new(options:, item:, config: { channel: }, scraper:, item_scope: self)
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/base.rb

@@ -29,7 +29,7 @@ module Html2rss
         # @param context [Selectors::Context] call-site context used for richer validation errors
         # @return [void]
         # @raise [InvalidType] if the value is not of the expected type(s)
-        def self.assert_type(value, types = [], name, context:)
+        def self.assert_type(value, types, name, context:)
           return if Array(types).any? { |type| value.is_a?(type) }
 
           options = if context.respond_to?(:options)

data/lib/html2rss/selectors/post_processors/markdown_to_html.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'kramdown'
-require_relative 'sanitize_html'
 
 module Html2rss
   class Selectors

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require 'sanitize'
-require_relative 'html_transformers/transform_urls_to_absolute_ones'
-require_relative 'html_transformers/wrap_img_in_a'
 
 module Html2rss
   class Selectors

data/lib/html2rss/selectors/post_processors/template.rb

@@ -42,6 +42,10 @@ module Html2rss
 
           string = context[:options]&.dig(:string).to_s
           raise InvalidType, 'The `string` template is absent.' if string.empty?
+
+          return if context.item_scope
+
+          raise MissingOption, 'The post-processor context is missing `item_scope`.', [], cause: nil
         end
 
         ##
@@ -51,8 +55,6 @@ module Html2rss
           super
 
           @options = context[:options] || {}
-          @scraper = context[:scraper]
-          @item = context[:item]
           @string = @options[:string].to_s
           @getter = ->(key) { item_value(key) }
         end
@@ -69,7 +71,9 @@ module Html2rss
         # @return [String]
         def item_value(key)
           key = key.to_sym
-          key == :self ? value : @scraper.select(key, @item)
+          return value if key == :self
+
+          @context.item_scope.select(key)
         end
       end
     end