html2rss 0.17.0 → 0.19.0

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+##
+# The Html2rss namespace.
+module Html2rss
+  ##
+  # Coordinates feed generation pipeline stages.
+  class FeedPipeline
+    # Retries feed extraction across concrete request strategies for :auto mode.
+    class AutoFallback
+      # Ordered list of concrete request strategies attempted by auto mode.
+      CHAIN = %i[faraday botasaurus browserless].freeze
+
+      # Error classes that should abort auto fallback immediately.
+      NON_FALLBACK_ERRORS = [
+        RequestService::UnknownStrategy,
+        RequestService::InvalidUrl,
+        RequestService::UnsupportedUrlScheme,
+        RequestService::UnsupportedResponseContentType,
+        RequestService::RequestBudgetExceeded,
+        RequestService::PrivateNetworkDenied,
+        RequestService::CrossOriginFollowUpDenied,
+        RequestService::ResponseTooLarge,
+        RequestService::BrowserlessConfigurationError
+      ].freeze
+
+      ##
+      # @param strategies [Array<Symbol>] ordered concrete strategies for fallback
+      # @param budget [RequestService::Budget] shared request budget across retries
+      # @param session_for [Proc] request session factory proc
+      # @param articles_for [Proc] article extraction proc
+      # @return [void]
+      def initialize(strategies:, budget:, session_for:, articles_for:)
+        @strategies = strategies
+        @budget = budget
+        @session_for = session_for
+        @articles_for = articles_for
+      end
+
+      ##
+      # @return [Hash{Symbol => Object}] pipeline state containing :response and :articles
+      def call
+        state, attempts = run_attempts
+        return state if state
+
+        finalize_failure(attempts:)
+      end
+
+      private
+
+      attr_reader :strategies, :budget, :session_for, :articles_for
+
+      def run_attempts
+        state = { result: nil, attempts: [] }
+        strategies.each_with_index do |strategy, index|
+          run_attempt_for(strategy:, next_strategy: strategies[index + 1], state:)
+          break if state.fetch(:result)
+        end
+        [state.fetch(:result), state.fetch(:attempts)]
+      end
+
+      def run_attempt_for(strategy:, next_strategy:, state:)
+        result, attempts = attempt(
+          strategy:,
+          next_strategy:,
+          state: { attempts: state.fetch(:attempts) }
+        )
+        state[:result] = result
+        state[:attempts] = attempts
+      end
+
+      def attempt(strategy:, next_strategy:, state:)
+        request_session = session_for.call(strategy:, budget:)
+        response, state = fetch_response(
+          request_session:,
+          strategy:,
+          next_strategy:,
+          state:
+        )
+        return [nil, state.fetch(:attempts)] unless response
+
+        process_response(response:, strategy:, next_strategy:, request_session:, state:)
+      end
+
+      def fetch_response(request_session:, strategy:, next_strategy:, state:)
+        [request_session.fetch_initial_response, state]
+      rescue *NON_FALLBACK_ERRORS
+        raise
+      rescue StandardError => error
+        state[:attempts] << { strategy:, items_count: nil, error_class: error.class.name }
+        log_warn_fallback_error(strategy:, next_strategy:, error:) if next_strategy
+        Log.debug("#{self.class}: strategy=#{strategy} error=#{error.class}: #{error.message}")
+        [nil, state]
+      end
+
+      def process_response(response:, strategy:, next_strategy:, request_session:, state:)
+        articles = articles_for.call(response:, request_session:)
+        items_count = articles.size
+        state[:attempts] << { strategy:, items_count:, error_class: nil }
+        Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count}")
+        return success_state(response:, strategy:, articles:, state:) if items_count.positive?
+
+        log_info_fallback_zero_items(strategy:, next_strategy:) if next_strategy
+        [nil, state.fetch(:attempts)]
+      end
+
+      def success_state(response:, strategy:, articles:, state:)
+        if state.fetch(:attempts).size > 1
+          Log.info("#{self.class}: auto selected strategy=#{strategy} after attempts=#{state.fetch(:attempts).size}")
+        end
+        [{ response:, articles: }, state.fetch(:attempts)]
+      end
+
+      def finalize_failure(attempts:)
+        raise NoFeedItemsExtracted.new(attempts:)
+      end
+
+      def log_warn_fallback_error(strategy:, next_strategy:, error:)
+        Log.warn("#{self.class}: auto fallback #{strategy} -> #{next_strategy} after error=#{error.class}")
+      end
+
+      def log_info_fallback_zero_items(strategy:, next_strategy:)
+        Log.info("#{self.class}: auto fallback #{strategy} -> #{next_strategy} after zero extracted items")
+      end
+    end
+  end
+end

data/lib/html2rss/feed_pipeline.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Builds feeds from validated config through request, extraction, and rendering stages.
+  class FeedPipeline
+    ##
+    # @param raw_config [Hash{Symbol => Object}] user-provided feed config
+    def initialize(raw_config)
+      @raw_config = raw_config
+    end
+
+    ##
+    # @return [RSS::Rss] generated RSS feed
+    def to_rss
+      run do |response:, config:, articles:|
+        channel = RssBuilder::Channel.new(response, overrides: config.channel)
+        RssBuilder.new(channel:, articles:, stylesheets: config.stylesheets).call
+      end
+    end
+
+    ##
+    # @return [Hash] generated JSONFeed 1.1 payload
+    def to_json_feed
+      run do |response:, config:, articles:|
+        channel = RssBuilder::Channel.new(response, overrides: config.channel)
+        JsonFeedBuilder.new(channel:, articles:).call
+      end
+    end
+
+    private
+
+    attr_reader :raw_config
+
+    def run
+      config = Config.from_hash(raw_config, params: raw_config[:params])
+      state = pipeline_state_for(config)
+      yield response: state.fetch(:response), config:, articles: state.fetch(:articles)
+    end
+
+    def pipeline_state_for(config)
+      if config.strategy == :auto
+        run_auto_pipeline(config)
+      else
+        run_pipeline_for_strategy(config, strategy: config.strategy)
+      end
+    end
+
+    def run_pipeline_for_strategy(config, strategy:, budget: nil)
+      request_session = request_session_for(config, strategy:, budget:)
+      response = request_session.fetch_initial_response
+      articles = deduplicated_articles(response:, config:, request_session:)
+      { response:, articles: }
+    end
+
+    def request_session_for(config, strategy:, budget: nil)
+      RequestSession.from_runtime_input(runtime_input_for(config, strategy:), budget:)
+    end
+
+    def runtime_input_for(config, strategy:)
+      RequestSession::RuntimeInput.new(
+        url: config.url,
+        headers: config.headers,
+        request: config.request,
+        strategy:,
+        request_policy: RequestSession::RuntimePolicy.from_config(config)
+      )
+    end
+
+    def deduplicated_articles(response:, config:, request_session:)
+      Articles::Deduplicator.new(
+        collect_articles(response:, config:, request_session:)
+      ).call
+    end
+
+    def run_auto_pipeline(config)
+      auto_fallback_for(config).call
+    end
+
+    def auto_fallback_for(config)
+      AutoFallback.new(
+        strategies: AutoFallback::CHAIN,
+        budget: auto_pipeline_budget(config),
+        session_for: lambda do |strategy:, budget:|
+          request_session_for(config, strategy:, budget:)
+        end,
+        articles_for: lambda do |response:, request_session:|
+          deduplicated_articles(response:, config:, request_session:)
+        end
+      )
+    end
+
+    def auto_pipeline_budget(config)
+      max_requests = RequestSession::RuntimePolicy.from_config(config).max_requests
+      RequestService::Budget.new(max_requests:)
+    end
+
+    def collect_articles(response:, config:, request_session:)
+      selector_articles(response:, config:, request_session:) +
+        auto_source_articles(response:, config:, request_session:)
+    end
+
+    def selector_articles(response:, config:, request_session:) # rubocop:disable Metrics/MethodLength
+      return [] unless (selectors = config.selectors)
+
+      page_responses = if (max_pages = selectors.dig(:items, :pagination, :max_pages))
+                         RequestSession::RelNextPager.new(
+                           session: request_session,
+                           initial_response: response,
+                           max_pages:
+                         ).to_a
+                       else
+                         [response]
+                       end
+
+      page_responses.flat_map do |page_response|
+        Selectors.new(page_response, selectors:, time_zone: config.time_zone).articles
+      end
+    end
+
+    def auto_source_articles(response:, config:, request_session:)
+      return [] unless (auto_source = config.auto_source)
+
+      AutoSource.new(response, auto_source, request_session:).articles
+    end
+  end
+end

data/lib/html2rss/hash_util.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Html2rss
+  # Shared helpers for hash normalization and structural operations.
+  module HashUtil
+    module_function
+
+    # Deeply duplicates nested arrays and hashes.
+    #
+    # @param object [Object] nested value from configuration or runtime state
+    # @return [Object] deep duplicated object
+    def deep_dup(object)
+      case object
+      in Hash
+        object.transform_values { deep_dup(_1) }
+      in Array
+        object.map { deep_dup(_1) }
+      else
+        object.dup rescue StandardError # rubocop:disable Style/RescueModifier
+      end
+    end
+
+    # Deeply merges nested hashes while replacing non-hash values from override.
+    #
+    # @param base [Hash] base hash
+    # @param override [Hash] override hash
+    # @return [Hash] merged hash
+    def deep_merge(base, override)
+      base.merge(override) do |_key, old_val, new_val|
+        case [old_val, new_val]
+        in [Hash, Hash]
+          deep_merge(old_val, new_val)
+        else
+          new_val
+        end
+      end
+    end
+
+    # Converts string-keyed hashes to symbol-keyed hashes recursively.
+    #
+    # @param object [Object] value to normalize
+    # @param context [String] error context
+    # @return [Object] normalized value
+    def deep_symbolize_keys(object, context: 'hash')
+      case object
+      in Hash
+        object.each_with_object({}) do |(k, v), memo|
+          memo[symbol_key(k, context:)] = deep_symbolize_keys(v, context:)
+        end
+      in Array
+        object.map { deep_symbolize_keys(_1, context:) }
+      else
+        object
+      end
+    end
+
+    # Validates that hash keys are symbols.
+    #
+    # @param value [Object] candidate hash container whose keys must be symbols
+    # @param context [String] error context
+    # @param deep [Boolean] whether nested hashes should also be validated
+    # @return [void]
+    def assert_symbol_keys!(value, context: 'hash', deep: true)
+      return unless value in Hash
+
+      unless value.each_key.all?(Symbol)
+        invalid_key = value.keys.find { _1.class != Symbol }
+        raise ArgumentError, "#{context} must use symbol keys (found #{invalid_key.inspect})"
+      end
+
+      value.each_value { assert_symbol_keys!(_1, context:, deep:) } if deep
+    end
+
+    # Validates that hash keys are strings.
+    #
+    # @param value [Object] candidate hash container whose keys must be strings
+    # @param context [String] error context
+    # @param deep [Boolean] whether nested hashes should also be validated
+    # @return [void]
+    def assert_string_keys!(value, context: 'hash', deep: true)
+      return unless value in Hash
+
+      unless value.each_key.all?(String)
+        invalid_key = value.keys.find { _1.class != String }
+        raise ArgumentError, "#{context} must use string keys (found #{invalid_key.inspect})"
+      end
+
+      value.each_value { assert_string_keys!(_1, context:, deep:) } if deep
+    end
+
+    def symbol_key(key, context:)
+      case key
+      in Symbol then key
+      in String then key.to_sym
+      else
+        raise ArgumentError, "#{context} must use string or symbol keys (found #{key.inspect})"
+      end
+    end
+    private_class_method :symbol_key
+  end
+end

data/lib/html2rss/html_extractor/date_extractor.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    # Extracts the earliest date from an article_tag.
+    class DateExtractor
+      # @param article_tag [Nokogiri::XML::Element] article container node
+      # @return [DateTime, nil]
+      def self.call(article_tag)
+        times = article_tag.css('[datetime]').filter_map do |tag|
+          DateTime.parse(tag['datetime'])
+        rescue ArgumentError, TypeError
+          nil
+        end
+
+        times.min
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor/enclosure_extractor.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # Extracts enclosures from HTML tags using various strategies.
+    class EnclosureExtractor
+      # @param article_tag [Nokogiri::XML::Element] article container node
+      # @param base_url [String, Html2rss::Url] base URL for relative enclosure links
+      # @return [Array<Hash{Symbol => Object}>] normalized enclosure hashes
+      def self.call(article_tag, base_url)
+        [
+          Extractors::Image,
+          Extractors::Media,
+          Extractors::Pdf,
+          Extractors::Iframe,
+          Extractors::Archive
+        ].flat_map { |strategy| strategy.call(article_tag, base_url:) }
+      end
+    end
+
+    # Extraction strategies for enclosure-like media/link tags.
+    module Extractors
+      # Extracts image enclosures from HTML tags.
+      # Finds all image sources and returns them in a format suitable for RSS.
+      class Image
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative image sources
+        # @return [Array<Hash{Symbol => Object}>] image enclosure hashes
+        def self.call(article_tag, base_url:)
+          article_tag.css('img[src]:not([src^="data"])').filter_map do |img|
+            src = img['src'].to_s
+            next if src.empty?
+
+            abs_url = Url.from_relative(src, base_url)
+            {
+              url: abs_url,
+              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
+            }
+          end
+        end
+      end
+
+      # Extracts media enclosures (video/audio) from HTML tags.
+      class Media
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative media sources
+        # @return [Array<Hash{Symbol => Object}>] media enclosure hashes
+        def self.call(article_tag, base_url:)
+          article_tag.css('video source[src], audio source[src], audio[src]').filter_map do |element|
+            src = element['src'].to_s
+            next if src.empty?
+
+            {
+              url: Url.from_relative(src, base_url),
+              type: element['type']
+            }
+          end
+        end
+      end
+
+      # Extracts PDF enclosures from HTML tags.
+      class Pdf
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative PDF links
+        # @return [Array<Hash{Symbol => Object}>] PDF enclosure hashes
+        def self.call(article_tag, base_url:)
+          article_tag.css('a[href$=".pdf"]').filter_map do |link|
+            href = link['href'].to_s
+            next if href.empty?
+
+            abs_url = Url.from_relative(href, base_url)
+            {
+              url: abs_url,
+              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url)
+            }
+          end
+        end
+      end
+
+      # Extracts iframe enclosures from HTML tags.
+      class Iframe
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative iframe links
+        # @return [Array<Hash{Symbol => Object}>] iframe enclosure hashes
+        def self.call(article_tag, base_url:)
+          article_tag.css('iframe[src]').filter_map do |iframe|
+            src = iframe['src']
+            next if src.nil? || src.empty?
+
+            abs_url = Url.from_relative(src, base_url)
+            {
+              url: abs_url,
+              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
+            }
+          end
+        end
+      end
+
+      # Extracts archive enclosures (zip, tar.gz, tgz) from HTML tags.
+      class Archive
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative archive links
+        # @return [Array<Hash{Symbol => Object}>] archive enclosure hashes
+        def self.call(article_tag, base_url:)
+          article_tag.css('a[href$=".zip"], a[href$=".tar.gz"], a[href$=".tgz"]').filter_map do |link|
+            href = link['href'].to_s
+            next if href.empty?
+
+            abs_url = Url.from_relative(href, base_url)
+            {
+              url: abs_url,
+              type: 'application/zip'
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor/image_extractor.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # Image is responsible for extracting image URLs the article_tag.
+    class ImageExtractor
+      # @param article_tag [Nokogiri::XML::Element] article container node
+      # @param base_url [String, Html2rss::Url] base URL for relative image URLs
+      # @return [Html2rss::Url, nil] best candidate image URL
+      def self.call(article_tag, base_url:)
+        img_src = from_source(article_tag) ||
+                  from_img(article_tag) ||
+                  from_style(article_tag)
+
+        Url.from_relative(img_src, base_url) if img_src
+      end
+
+      # @param article_tag [Nokogiri::XML::Element] article container node
+      # @return [String, nil] src attribute from first matching image tag
+      def self.from_img(article_tag)
+        article_tag.at_css('img[src]:not([src^="data"])')&.[]('src')
+      end
+
+      ##
+      # Extracts the largest image source from the srcset attribute
+      # of an img tag or a source tag inside a picture tag.
+      #
+      # @param article_tag [Nokogiri::XML::Element] article container node
+      # @return [String, nil] largest srcset URL candidate
+      # @see <https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Responsive_images>
+      # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#srcset>
+      # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture>
+      def self.from_source(article_tag) # rubocop:disable Metrics/AbcSize
+        hash = article_tag.css('img[srcset], picture > source[srcset]').flat_map do |source|
+          source['srcset'].to_s.scan(/(\S+)\s+(\d+w|\d+h)[\s,]?/).map do |url, width|
+            next if url.nil? || url.start_with?('data:')
+
+            width_value = width.to_i.zero? ? 0 : width.scan(/\d+/).first.to_i
+
+            [width_value, url.strip]
+          end
+        end.compact.to_h
+
+        hash[hash.keys.max]
+      end
+
+      # @param article_tag [Nokogiri::XML::Element] article container node
+      # @return [String, nil] best style-based background image URL
+      def self.from_style(article_tag)
+        article_tag.css('[style*="url"]')
+                   .filter_map { |tag| tag['style'][/url\(['"]?(.*?)['"]?\)/, 1] }
+                   .reject { |src| src.start_with?('data:') }
+                   .max_by(&:size)
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # HtmlExtractor is responsible for extracting details (headline, url, images, etc.)
+  # from an article_tag.
+  class HtmlExtractor
+    # Tags ignored when extracting visible text content from article containers.
+    INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
+    # Heading tags used to prioritize title extraction.
+    HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
+    # Selector used to derive non-headline description nodes.
+    NON_HEADLINE_SELECTOR = (HEADING_TAGS.map { |tag| ":not(#{tag})" } + INVISIBLE_CONTENT_TAGS.to_a).freeze
+
+    # Anchor selector used to identify the canonical article link element.
+    MAIN_ANCHOR_SELECTOR = begin
+      buf = +'a[href]:not([href=""])'
+      %w[# javascript: mailto: tel: file:// sms: data:].each do |prefix|
+        buf << %[:not([href^="#{prefix}"])]
+      end
+      buf.freeze
+    end
+
+    class << self
+      ##
+      # Extracts visible text from a given node and its children.
+      #
+      # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
+      # @param separator [String] separator used to join text fragments (default is a space)
+      # @return [String, nil] the concatenated visible text, or nil if none is found
+      def extract_visible_text(tag, separator: ' ')
+        parts = tag.children.filter_map do |child|
+          next unless visible_child?(child)
+
+          raw_text = child.children.empty? ? child.text : extract_visible_text(child)
+          text = raw_text&.strip
+          text unless text.to_s.empty?
+        end
+
+        parts.join(separator).squeeze(' ').strip unless parts.empty?
+      end
+
+      private
+
+      def visible_child?(node)
+        !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
+          !(node.name == 'a' && node['href']&.start_with?('#'))
+      end
+    end
+
+    ##
+    # @param article_tag [Nokogiri::XML::Node] article-like container to extract from
+    # @param base_url [String, Html2rss::Url] base url used to resolve relative links
+    # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
+    def initialize(article_tag, base_url:, selected_anchor:)
+      raise ArgumentError, 'article_tag is required' unless article_tag
+
+      @article_tag = article_tag
+      @base_url = base_url
+      @selected_anchor = selected_anchor
+    end
+
+    # @return [Hash{Symbol => Object}] extracted article attributes
+    def call
+      {
+        title: extract_title,
+        url: extract_url,
+        image: extract_image,
+        description: extract_description,
+        id: generate_id,
+        published_at: extract_published_at,
+        enclosures: extract_enclosures,
+        categories: extract_categories
+      }
+    end
+
+    private
+
+    attr_reader :article_tag, :base_url, :selected_anchor
+
+    class << self
+      ##
+      # @param article_tag [Nokogiri::XML::Node] article-like container to search within
+      # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
+      def main_anchor_for(article_tag)
+        return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
+
+        article_tag.at_css(MAIN_ANCHOR_SELECTOR)
+      end
+    end
+
+    def extract_url
+      @extract_url ||= begin
+        href = selected_anchor&.[]('href').to_s
+
+        Url.from_relative(href.split('#').first.strip, base_url) unless href.empty?
+      end
+    end
+
+    def extract_title
+      title_source = heading || selected_anchor
+      self.class.extract_visible_text(title_source) if title_source
+    end
+
+    def heading
+      @heading ||= begin
+        heading_tags = article_tag.css(HEADING_TAGS.join(',')).group_by(&:name)
+        smallest_heading = heading_tags.keys.min
+        if smallest_heading
+          heading_tags[smallest_heading]&.max_by do |tag|
+            self.class.extract_visible_text(tag)&.size.to_i
+          end
+        end
+      end
+    end
+
+    def extract_description
+      text = self.class.extract_visible_text(article_tag.css(NON_HEADLINE_SELECTOR), separator: '<br>')
+      return text if text && !text.empty?
+
+      description = self.class.extract_visible_text(article_tag)
+      return nil if description.nil? || description.strip.empty?
+
+      description.strip
+    end
+
+    def generate_id
+      [
+        article_tag['id'],
+        article_tag.at_css('[id]')&.attr('id'),
+        extract_url&.path,
+        extract_url&.query
+      ].compact.reject(&:empty?).first
+    end
+
+    def extract_image = ImageExtractor.call(article_tag, base_url:)
+    def extract_published_at = DateExtractor.call(article_tag)
+    def extract_enclosures = EnclosureExtractor.call(article_tag, base_url)
+    def extract_categories = CategoryExtractor.call(article_tag)
+  end
+end