html2rss 0.16.0 → 0.18.0

data/lib/html2rss/request_service.rb

@@ -6,13 +6,22 @@ require 'forwardable'
 module Html2rss
   ##
   # Requests website URLs to retrieve their HTML for further processing.
-  # Provides strategies, i.e. to integrate Browserless.io.
+  # Provides strategies, e.g. integrating Browserless.io.
   class RequestService
     include Singleton
 
     class UnknownStrategy < Html2rss::Error; end
     class InvalidUrl < Html2rss::Error; end
     class UnsupportedUrlScheme < Html2rss::Error; end
+    class UnsupportedResponseContentType < Html2rss::Error; end
+    class RequestBudgetExceeded < Html2rss::Error; end
+    class PrivateNetworkDenied < Html2rss::Error; end
+    class CrossOriginFollowUpDenied < Html2rss::Error; end
+    class ResponseTooLarge < Html2rss::Error; end
+    class BlockedSurfaceDetected < Html2rss::Error; end
+    class RequestTimedOut < Html2rss::Error; end
+    class BrowserlessConfigurationError < Html2rss::Error; end
+    class BrowserlessConnectionFailed < Html2rss::Error; end
 
     class << self
       extend Forwardable
@@ -55,9 +64,12 @@ module Html2rss
     ##
     # Registers a new strategy.
     # @param name [Symbol] the name of the strategy
-    # @param strategy_class [Class] the class of the strategy
+    # @param strategy_class [Class] the class implementing the strategy
+    # @raise [ArgumentError] if strategy_class is not a Class
     def register_strategy(name, strategy_class)
-      raise ArgumentError, 'Strategy class must be a Class' unless strategy_class.is_a?(Class)
+      unless strategy_class.is_a?(Class)
+        raise ArgumentError, "Expected a Class for strategy, got #{strategy_class.class}"
+      end
 
       @strategies[name.to_sym] = strategy_class
     end
@@ -65,7 +77,7 @@ module Html2rss
     ##
     # Checks if a strategy is registered.
     # @param name [Symbol] the name of the strategy
-    # @return [Boolean] true if the strategy is registered, false otherwise
+    # @return [Boolean] true if the strategy is registered, false otherwise.
     def strategy_registered?(name)
       @strategies.key?(name.to_sym)
     end
@@ -73,24 +85,28 @@ module Html2rss
     ##
     # Unregisters a strategy.
     # @param name [Symbol] the name of the strategy
-    # @return [Boolean] true if the strategy was unregistered, false otherwise
-    def unregister_strategy(name)
-      raise ArgumentError, 'Cannot unregister the default strategy' if name.to_sym == @default_strategy_name
+    # @return [Boolean] true if the strategy was unregistered, false otherwise.
+    # @raise [ArgumentError] if attempting to unregister the default strategy.
+    def unregister_strategy(name) # rubocop:disable Naming/PredicateMethod
+      name_sym = name.to_sym
+      raise ArgumentError, 'Cannot unregister the default strategy.' if name_sym == @default_strategy_name
 
-      !!@strategies.delete(name.to_sym)
+      !!@strategies.delete(name_sym)
     end
 
     ##
-    # Executes the request.
-    # @param ctx [Context] the context for the request
-    # @param strategy [Symbol] the strategy to use
-    # @return [Response] the response from the strategy
-    # @raise [UnknownStrategy] if the strategy is not known
+    # Executes the request using the specified strategy.
+    # @param ctx [Context] the context for the request.
+    # @param strategy [Symbol] the strategy to use (defaults to the default strategy).
+    # @return [Response] the response from the executed strategy.
+    # @raise [ArgumentError] if the context is nil.
+    # @raise [UnknownStrategy] if the strategy is not registered.
     def execute(ctx, strategy: default_strategy_name)
-      strategy_class = @strategies.fetch(strategy) do
+      strategy_class = @strategies.fetch(strategy.to_sym) do
         raise UnknownStrategy,
-              "The strategy '#{strategy}' is not known. Available strategies are: #{strategy_names.join(', ')}"
+              "The strategy '#{strategy}' is not known. Available strategies: #{strategy_names.join(', ')}"
       end
+
       strategy_class.new(ctx).execute
     end
   end

data/lib/html2rss/request_session/rel_next_pager.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    ##
+    # Traverses a rel=next pagination chain for selector-driven extraction.
+    class RelNextPager
+      include Enumerable
+
+      ##
+      # @param session [RequestSession] request session used to execute follow-ups
+      # @param initial_response [RequestService::Response] first page response
+      # @param max_pages [Integer] configured page budget, including the initial page
+      # @param logger [Logger] logger used for pagination stop reasons
+      def initialize(session:, initial_response:, max_pages:, logger: Html2rss::Log)
+        @session = session
+        @initial_response = initial_response
+        @max_pages = max_pages
+        @logger = logger
+      end
+
+      ##
+      # Iterates over all paginated responses, beginning with the initial response.
+      #
+      # @yield [RequestService::Response] each page response
+      # @return [Enumerator] enumerator when no block is given
+      def each
+        return enum_for(:each) unless block_given?
+
+        yield initial_response
+
+        current_response = initial_response
+        session.effective_page_budget(max_pages).pred.times do
+          next_url = next_page_url(current_response)
+          break unless follow_up_allowed?(next_url)
+
+          current_response = fetch_follow_up_response_or_stop(next_url, current_response.url)
+          break unless current_response
+
+          yield current_response
+        end
+      end
+
+      private
+
+      attr_reader :session, :initial_response, :max_pages, :logger
+
+      def next_page_url(page_response)
+        href = page_response.parsed_body.at_css('link[rel~="next"][href], a[rel~="next"][href]')&.[]('href')
+        return nil if href.nil? || href.empty?
+
+        Html2rss::Url.from_relative(href, page_response.url)
+      end
+
+      def follow_up_allowed?(next_url)
+        next_url && !session.visited?(next_url)
+      end
+
+      def fetch_follow_up_response_or_stop(next_url, origin_url)
+        session.follow_up(url: next_url, relation: :pagination, origin_url:)
+      rescue RequestService::RequestBudgetExceeded => error
+        logger.warn(
+          "#{self.class}: pagination stopped at #{next_url} - #{error.message}. " \
+          "Retry with --max-requests #{session.max_requests + 1} or increase request.max_requests in the config."
+        )
+        nil
+      end
+    end
+  end
+end

data/lib/html2rss/request_session/runtime_input.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    ##
+    # Carries the runtime request inputs needed to build a RequestSession.
+    class RuntimeInput
+      ##
+      # @param config [Html2rss::Config] validated feed config
+      # @return [RuntimeInput] runtime request inputs derived from the config
+      def self.from_config(config)
+        new(
+          url: config.url,
+          headers: config.headers,
+          request: config.request,
+          strategy: config.strategy,
+          request_policy: RuntimePolicy.from_config(config)
+        )
+      end
+
+      ##
+      # @param url [String, Html2rss::Url] initial request URL
+      # @param headers [Hash] normalized request headers
+      # @param request [Hash] validated request options for strategies
+      # @param strategy [Symbol] request strategy to use for the session
+      # @param request_policy [RequestService::Policy] request policy for the session
+      def initialize(url:, headers:, request:, strategy:, request_policy:)
+        @url = Html2rss::Url.from_absolute(url)
+        @headers = headers.freeze
+        @request = request.freeze
+        @strategy = strategy
+        @request_policy = request_policy
+        freeze
+      end
+
+      ##
+      # @return [Html2rss::Url] initial request URL
+      attr_reader :url
+
+      ##
+      # @return [Hash] normalized request headers
+      attr_reader :headers
+
+      ##
+      # @return [Hash] validated request options for strategies
+      attr_reader :request
+
+      ##
+      # @return [Symbol] request strategy to use for the session
+      attr_reader :strategy
+
+      ##
+      # @return [RequestService::Policy] policy derived from the runtime request inputs
+      attr_reader :request_policy
+    end
+  end
+end

data/lib/html2rss/request_session/runtime_policy.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestSession
+    ##
+    # Builds the runtime request policy for a feed run.
+    class RuntimePolicy
+      ##
+      # @param config [Html2rss::Config] validated feed config
+      # @return [Html2rss::RequestService::Policy] request policy derived from runtime config
+      def self.from_config(config)
+        RequestService::Policy.new(
+          max_requests: effective_max_requests_for(config),
+          max_redirects: config.max_redirects
+        )
+      end
+
+      class << self
+        private
+
+        def effective_max_requests_for(config)
+          return config.max_requests if config.explicit_max_requests?
+
+          [baseline_request_budget_for(config), config.max_requests].max
+        end
+
+        # Reserve enough budget for the initial request plus predictable follow-ups
+        # that the top-level pipeline may trigger during a normal feed build.
+        def baseline_request_budget_for(config)
+          1 + pagination_follow_up_budget_for(config) +
+            known_auto_source_follow_up_budget_for(config) +
+            browserless_preload_budget_for(config)
+        end
+
+        def pagination_follow_up_budget_for(config)
+          [config.selectors&.dig(:items, :pagination, :max_pages).to_i - 1, 0].max
+        end
+
+        def known_auto_source_follow_up_budget_for(config)
+          config.auto_source&.dig(:scraper, :wordpress_api, :enabled) ? 1 : 0
+        end
+
+        def browserless_preload_budget_for(config)
+          preload = config.request.dig(:browserless, :preload)
+          return 0 unless preload
+
+          top_level_preload_wait_budget(preload) +
+            click_selector_preload_budget(preload) +
+            scroll_preload_budget(preload)
+        end
+
+        def top_level_preload_wait_budget(preload)
+          preload[:wait_after_ms] ? 2 : 0
+        end
+
+        def click_selector_preload_budget(preload)
+          preload.fetch(:click_selectors, []).sum { preload_action_budget(_1, :max_clicks) }
+        end
+
+        def scroll_preload_budget(preload)
+          scroll = preload[:scroll_down]
+          return 0 unless scroll
+
+          preload_action_budget(scroll, :iterations)
+        end
+
+        def preload_action_budget(config, count_key)
+          action_count = config.fetch(count_key, 1)
+          wait_budget = config[:wait_after_ms] ? action_count : 0
+
+          action_count + wait_budget
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_session.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Coordinates multi-request feed builds on top of RequestService.
+  class RequestSession
+    class << self
+      ##
+      # Builds a request session from translated runtime request inputs.
+      #
+      # @param runtime_input [RuntimeInput] translated runtime request inputs
+      # @param logger [Logger] logger used for operational warnings
+      # @return [RequestSession] configured request session
+      def from_runtime_input(runtime_input, logger: Html2rss::Log)
+        new(
+          context: RequestService::Context.new(
+            url: runtime_input.url,
+            headers: runtime_input.headers,
+            request: runtime_input.request,
+            policy: runtime_input.request_policy
+          ),
+          strategy: runtime_input.strategy,
+          logger:
+        )
+      end
+    end
+
+    ##
+    # @param context [RequestService::Context] initial request context
+    # @param strategy [Symbol] request strategy to use for all requests in the session
+    # @param logger [Logger] logger used for operational warnings
+    def initialize(context:, strategy:, logger: Html2rss::Log)
+      @context = context
+      @strategy = strategy
+      @logger = logger
+      @visited_urls = Set.new
+    end
+
+    ##
+    # Executes the initial request for the session.
+    #
+    # @return [RequestService::Response] initial response
+    def fetch_initial_response
+      execute(context).tap { |response| remember!(response.url) }
+    end
+
+    ##
+    # Executes a follow-up request sharing policy, headers, and budget.
+    #
+    # @param url [String, Html2rss::Url] follow-up request url
+    # @param relation [Symbol] why the follow-up is being made
+    # @param origin_url [String, Html2rss::Url] effective origin for same-origin checks
+    # @return [RequestService::Response] follow-up response
+    def follow_up(url:, relation:, origin_url:)
+      execute(context.follow_up(url:, relation:, origin_url:)).tap { |response| remember!(response.url) }
+    end
+
+    ##
+    # Returns the effective page budget after applying the policy ceiling.
+    #
+    # @param requested_pages [Integer] configured page budget
+    # @return [Integer] effective page budget for the session
+    def effective_page_budget(requested_pages)
+      effective_pages = [requested_pages, context.policy.max_requests].min
+      return effective_pages if effective_pages == requested_pages
+
+      logger.warn(
+        "#{self.class}: pagination max_pages=#{requested_pages} " \
+        "exceeds system ceiling=#{context.policy.max_requests}; " \
+        "clamping to #{effective_pages}"
+      )
+      effective_pages
+    end
+
+    ##
+    # Returns the configured request budget for the session.
+    #
+    # @return [Integer] maximum requests allowed for the feed build
+    def max_requests
+      context.policy.max_requests
+    end
+
+    ##
+    # @param url [String, Html2rss::Url] url to query
+    # @return [Boolean] whether the url was already visited in this session
+    def visited?(url)
+      visited_urls.include?(normalize_url(url))
+    end
+
+    ##
+    # Records a visited url in the session.
+    #
+    # @param url [String, Html2rss::Url] url to track
+    # @return [Set<Html2rss::Url>] visited urls
+    def remember!(url)
+      visited_urls.add(normalize_url(url))
+    end
+
+    private
+
+    attr_reader :context, :strategy, :logger, :visited_urls
+
+    def execute(request_context)
+      RequestService.execute(request_context, strategy:).tap do |response|
+        logger.debug(
+          "#{self.class}: relation=#{request_context.relation} " \
+          "request_url=#{request_context.url} final_url=#{response.url} " \
+          "status=#{response.status || 'unknown'} content_type=#{response.content_type.inspect} " \
+          "bytes=#{response.body.bytesize}"
+        )
+      end
+    end
+
+    def normalize_url(url)
+      Html2rss::Url.from_absolute(url)
+    end
+  end
+end

data/lib/html2rss/rss_builder/article.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require 'zlib'
+require 'sanitize'
+require 'nokogiri'
+
+module Html2rss
+  class RssBuilder
+    ##
+    # Article is a simple data object representing an article extracted from a page.
+    # It is enumerable and responds to all keys specified in PROVIDED_KEYS.
+    class Article
+      include Enumerable
+      include Comparable
+
+      PROVIDED_KEYS = %i[id title description url image author guid published_at enclosures categories scraper].freeze
+      DEDUP_FINGERPRINT_SEPARATOR = '#!/'
+
+      # @param options [Hash<Symbol, String>]
+      def initialize(**options)
+        @to_h = {}
+        options.each_pair { |key, value| @to_h[key] = value.freeze if value }
+        @to_h.freeze
+
+        return unless (unknown_keys = options.keys - PROVIDED_KEYS).any?
+
+        Log.warn "Article: unknown keys found: #{unknown_keys.join(', ')}"
+      end
+
+      # Checks if the article is valid based on the presence of URL, ID, and either title or description.
+      # @return [Boolean] True if the article is valid, otherwise false.
+      def valid?
+        !url.to_s.empty? && (!title.to_s.empty? || !description.to_s.empty?) && !id.to_s.empty?
+      end
+
+      # @yield [key, value]
+      # @return [Enumerator] if no block is given
+      def each
+        return enum_for(:each) unless block_given?
+
+        PROVIDED_KEYS.each { |key| yield(key, public_send(key)) }
+      end
+
+      def id = blank_string_to_nil(@to_h[:id])
+
+      def title = blank_string_to_nil(@to_h[:title])
+
+      def description
+        @description ||= Rendering::DescriptionBuilder.new(
+          base: @to_h[:description],
+          title:,
+          url:,
+          enclosures:,
+          image:
+        ).call
+      end
+
+      # @return [Url, nil]
+      def url
+        @url ||= Url.sanitize(@to_h[:url])
+      end
+
+      # @return [Url, nil]
+      def image
+        @image ||= Url.sanitize(@to_h[:image])
+      end
+
+      # @return [String, nil]
+      def author = blank_string_to_nil(@to_h[:author])
+
+      # Generates a unique identifier based on the URL and ID using CRC32.
+      # @return [String]
+      def guid
+        @guid ||= Zlib.crc32(fetch_guid).to_s(36).encode('utf-8')
+      end
+
+      ##
+      # Returns a deterministic fingerprint used to detect duplicate articles.
+      #
+      # @return [String, Integer]
+      def deduplication_fingerprint
+        dedup_from_url || dedup_from_id || dedup_from_guid || hash
+      end
+
+      def enclosures
+        @enclosures ||= Array(@to_h[:enclosures])
+                        .map { |enclosure| Html2rss::RssBuilder::Enclosure.new(**enclosure) }
+      end
+
+      # @return [Html2rss::RssBuilder::Enclosure, nil]
+      def enclosure
+        return @enclosure if defined?(@enclosure)
+
+        case (object = @to_h[:enclosures]&.first)
+        when Hash
+          @enclosure = Html2rss::RssBuilder::Enclosure.new(**object)
+        when nil
+          @enclosure = Html2rss::RssBuilder::Enclosure.new(url: image) if image
+        else
+          Log.warn "Article: unknown enclosure type: #{object.class}"
+        end
+      end
+
+      def categories
+        @categories ||= @to_h[:categories].dup.to_a.tap do |categories|
+          categories.map! { |category| category.to_s.strip }
+          categories.reject!(&:empty?)
+          categories.uniq!
+        end
+      end
+
+      # Parses and returns the published_at time.
+      # @return [DateTime, nil]
+      def published_at
+        return if (string = @to_h[:published_at].to_s.strip).empty?
+
+        @published_at ||= DateTime.parse(string)
+      rescue ArgumentError
+        nil
+      end
+
+      def scraper
+        @to_h[:scraper]
+      end
+
+      def <=>(other)
+        return nil unless other.is_a?(Article)
+
+        0 if other.all? { |key, value| value == public_send(key) ? public_send(key) <=> value : false }
+      end
+
+      private
+
+      def dedup_from_url
+        return unless (value = url)
+
+        [value.to_s, id].compact.join(DEDUP_FINGERPRINT_SEPARATOR)
+      end
+
+      def dedup_from_id
+        return if id.to_s.empty?
+
+        id
+      end
+
+      def dedup_from_guid
+        value = guid
+        return if value.to_s.empty?
+
+        [value, title, description].compact.join(DEDUP_FINGERPRINT_SEPARATOR)
+      end
+
+      def fetch_guid
+        guid = @to_h[:guid].map { |s| s.to_s.strip }.reject(&:empty?).join if @to_h[:guid].is_a?(Array)
+
+        guid || [url, id].join('#!/')
+      end
+
+      def blank_string_to_nil(value)
+        return if value.is_a?(String) && value.strip.empty?
+
+        value
+      end
+    end
+  end
+end

data/lib/html2rss/rss_builder/channel.rb

@@ -1,20 +1,105 @@
 # frozen_string_literal: true
 
 module Html2rss
-  module RssBuilder
+  class RssBuilder
     ##
-    # Builds the <channel> tag (with the provided maker).
+    # Extracts channel information from
+    # 1. the HTML document's <head>.
+    # 2. the HTTP response
     class Channel
+      DEFAULT_TTL_IN_MINUTES = 360
+      DEFAULT_DESCRIPTION_TEMPLATE = 'Latest items from %<url>s'
+
       ##
-      # @param maker [RSS::Maker::RSS20::Channel]
-      # @param config [Html2rss::Config]
-      # @param tags [Set<Symbol>]
-      # @return nil
-      def self.add(maker, config, tags)
-        tags.each { |tag| maker.public_send(:"#{tag}=", config.public_send(tag)) }
-
-        maker.generator = "html2rss V. #{::Html2rss::VERSION}"
-        maker.lastBuildDate = Time.now
+      #
+      # @param response [Html2rss::RequestService::Response]
+      # @param overrides [Hash<Symbol, String>] - Optional, overrides for any channel attribute
+      def initialize(response, overrides: {})
+        @response = response
+        @overrides = overrides
+      end
+
+      def title
+        @title ||= fetch_title
+      end
+
+      def url = @url ||= Html2rss::Url.from_absolute(@response.url)
+
+      def description
+        return overrides[:description] unless overrides[:description].to_s.empty?
+
+        description = parsed_body.at_css('meta[name="description"]')&.[]('content') if html_response?
+
+        return format(DEFAULT_DESCRIPTION_TEMPLATE, url:) if description.to_s.empty?
+
+        description
+      end
+
+      def ttl
+        return overrides[:ttl] if overrides[:ttl]
+
+        if (ttl = headers['cache-control']&.match(/max-age=(\d+)/)&.[](1))
+          return ttl.to_i.fdiv(60).ceil
+        end
+
+        DEFAULT_TTL_IN_MINUTES
+      end
+
+      def language
+        return overrides[:language] if overrides[:language]
+
+        if (language_code = headers['content-language']&.match(/^([a-z]{2})/))
+          return language_code[0]
+        end
+
+        return unless html_response?
+
+        parsed_body['lang'] || parsed_body.at_css('[lang]')&.[]('lang')
+      end
+
+      def author
+        return overrides[:author] if overrides[:author]
+
+        return unless html_response?
+
+        parsed_body.at_css('meta[name="author"]')&.[]('content')
+      end
+
+      def last_build_date = headers['last-modified'] || Time.now
+
+      def image
+        return overrides[:image] if overrides[:image]
+
+        return unless html_response?
+
+        if (image_url = parsed_body.at_css('meta[property="og:image"]')&.[]('content'))
+          Url.sanitize(image_url)
+        end
+      end
+
+      private
+
+      attr_reader :overrides
+
+      def parsed_body = @parsed_body ||= @response.parsed_body
+      def headers = @headers ||= @response.headers
+      def html_response? = @html_response ||= @response.html_response?
+
+      def fetch_title
+        override_title = overrides[:title]
+        return override_title if override_title
+        return parsed_title if parsed_title
+
+        url.channel_titleized
+      end
+
+      def parsed_title
+        return unless html_response?
+
+        title = parsed_body.at_css('head > title')&.text.to_s
+        return if title.empty?
+
+        title.gsub(/\s+/, ' ').strip
       end
     end
   end