html2rss 0.20.0 → 0.21.0

data/lib/html2rss/config/class_methods.rb

@@ -132,10 +132,7 @@ module Html2rss
       def default_config
         {
           strategy: default_strategy_name,
-          request: {
-            max_redirects: RequestService::Policy::DEFAULTS[:max_redirects],
-            max_requests: RequestService::Policy::DEFAULTS[:max_requests]
-          },
+          request: default_request_config,
           channel: { time_zone: 'UTC' },
           headers: RequestHeaders.browser_defaults,
           stylesheets: Html2rss.configuration.stylesheets || []
@@ -149,6 +146,14 @@ module Html2rss
 
       private
 
+      def default_request_config
+        {
+          max_redirects: RequestService::Policy::DEFAULTS[:max_redirects],
+          max_requests: RequestService::Policy::DEFAULTS[:max_requests],
+          total_timeout_seconds: RequestService::Policy::DEFAULTS[:total_timeout_seconds]
+        }
+      end
+
       def resolve_effective_config(config, params:)
         effective_config = HashUtil.deep_symbolize_keys(config, context: 'config')
         resolved_params = parameter_defaults(effective_config)

data/lib/html2rss/config/validator.rb

@@ -80,6 +80,7 @@ module Html2rss
       RequestConfig = Dry::Schema.Params do
         optional(:max_redirects).filled(:integer, gteq?: 0)
         optional(:max_requests).filled(:integer, gt?: 0)
+        optional(:total_timeout_seconds).filled(:integer, gt?: 0)
         optional(:browserless).hash(BrowserlessRequestConfig)
         optional(:botasaurus).hash(BotasaurusRequestConfig)
       end

data/lib/html2rss/config.rb

@@ -31,7 +31,8 @@ module Html2rss
       @request_controls = request_controls.with_effective_values(
         strategy: validated_config[:strategy],
         max_redirects: validated_config.dig(:request, :max_redirects),
-        max_requests: validated_config.dig(:request, :max_requests)
+        max_requests: validated_config.dig(:request, :max_requests),
+        total_timeout_seconds: validated_config.dig(:request, :total_timeout_seconds)
       )
     end
 
@@ -41,6 +42,8 @@ module Html2rss
     def max_redirects = request_controls.max_redirects
     # @return [Integer, nil] configured request budget
     def max_requests = request_controls.max_requests
+    # @return [Integer, nil] configured request timeout
+    def total_timeout_seconds = request_controls.total_timeout_seconds
     # @return [Array<Hash>] stylesheet definitions
     def stylesheets = config[:stylesheets]
 

data/lib/html2rss/html_extractor/enclosure_extractor.rb

@@ -5,116 +5,87 @@ module Html2rss
     ##
     # Extracts enclosures from HTML tags using various strategies.
     class EnclosureExtractor
+      # CSS union query covering images, media, PDFs, iframes, and archives.
+      SELECTOR = [
+        'img[src]:not([src^="data"])',
+        'video source[src]',
+        'audio source[src]',
+        'audio[src]',
+        'a[href$=".pdf"]',
+        'iframe[src]',
+        'a[href$=".zip"]',
+        'a[href$=".tar.gz"]',
+        'a[href$=".tgz"]'
+      ].join(',').freeze
+
       # @param article_tag [Nokogiri::XML::Element] article container node
       # @param base_url [String, Html2rss::Url] base URL for relative enclosure links
       # @return [Array<Hash{Symbol => Object}>] normalized enclosure hashes
       def self.call(article_tag, base_url)
-        [
-          Extractors::Image,
-          Extractors::Media,
-          Extractors::Pdf,
-          Extractors::Iframe,
-          Extractors::Archive
-        ].flat_map { |strategy| strategy.call(article_tag, base_url:) }
+        article_tag.css(SELECTOR).filter_map do |element|
+          extract_from_element(element, base_url)
+        end
       end
-    end
 
-    # Extraction strategies for enclosure-like media/link tags.
-    module Extractors
-      # Extracts image enclosures from HTML tags.
-      # Finds all image sources and returns them in a format suitable for RSS.
-      class Image
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative image sources
-        # @return [Array<Hash{Symbol => Object}>] image enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('img[src]:not([src^="data"])').filter_map do |img|
-            src = img['src'].to_s
-            next if src.empty?
-
-            abs_url = Url.from_relative(src, base_url)
-            {
-              url: abs_url,
-              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
-            }
-          end
+      def self.extract_from_element(element, base_url)
+        case element.name
+        when 'img'
+          extract_image(element, base_url)
+        when 'video', 'audio', 'source'
+          extract_media(element, base_url)
+        when 'iframe'
+          extract_iframe(element, base_url)
+        when 'a'
+          extract_a(element, base_url)
         end
       end
 
-      # Extracts media enclosures (video/audio) from HTML tags.
-      class Media
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative media sources
-        # @return [Array<Hash{Symbol => Object}>] media enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('video source[src], audio source[src], audio[src]').filter_map do |element|
-            src = element['src'].to_s
-            next if src.empty?
+      def self.extract_image(img, base_url)
+        src = img['src'].to_s
+        return if src.empty?
 
-            {
-              url: Url.from_relative(src, base_url),
-              type: element['type']
-            }
-          end
-        end
+        abs_url = Url.from_relative(src, base_url)
+        {
+          url: abs_url,
+          type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
+        }
       end
 
-      # Extracts PDF enclosures from HTML tags.
-      class Pdf
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative PDF links
-        # @return [Array<Hash{Symbol => Object}>] PDF enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('a[href$=".pdf"]').filter_map do |link|
-            href = link['href'].to_s
-            next if href.empty?
+      def self.extract_media(element, base_url)
+        src = element['src'].to_s
+        return if src.empty?
 
-            abs_url = Url.from_relative(href, base_url)
-            {
-              url: abs_url,
-              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url)
-            }
-          end
-        end
+        {
+          url: Url.from_relative(src, base_url),
+          type: element['type']
+        }
       end
 
-      # Extracts iframe enclosures from HTML tags.
-      class Iframe
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative iframe links
-        # @return [Array<Hash{Symbol => Object}>] iframe enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('iframe[src]').filter_map do |iframe|
-            src = iframe['src']
-            next if src.nil? || src.empty?
+      def self.extract_iframe(iframe, base_url)
+        src = iframe['src'].to_s
+        return if src.empty?
 
-            abs_url = Url.from_relative(src, base_url)
-            {
-              url: abs_url,
-              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
-            }
-          end
-        end
+        abs_url = Url.from_relative(src, base_url)
+        {
+          url: abs_url,
+          type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
+        }
       end
 
-      # Extracts archive enclosures (zip, tar.gz, tgz) from HTML tags.
-      class Archive
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative archive links
-        # @return [Array<Hash{Symbol => Object}>] archive enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('a[href$=".zip"], a[href$=".tar.gz"], a[href$=".tgz"]').filter_map do |link|
-            href = link['href'].to_s
-            next if href.empty?
+      def self.extract_a(link, base_url)
+        href = link['href'].to_s
+        return if href.empty?
 
-            abs_url = Url.from_relative(href, base_url)
-            {
-              url: abs_url,
-              type: 'application/zip'
-            }
-          end
+        abs_url = Url.from_relative(href, base_url)
+
+        if href.end_with?('.pdf')
+          { url: abs_url, type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url) }
+        else
+          { url: abs_url, type: 'application/zip' }
         end
       end
+
+      private_class_method :extract_from_element, :extract_image, :extract_media, :extract_iframe, :extract_a
     end
   end
 end

data/lib/html2rss/html_extractor/list_candidates.rb

@@ -75,17 +75,11 @@ module Html2rss
       def each_anchor(anchor_filter:)
         return enum_for(:each_anchor, anchor_filter:) unless block_given?
 
-        traversal_root&.traverse do |node|
-          yield node if relevant_anchor?(node, anchor_filter:)
+        traversal_root&.css(HtmlExtractor::MAIN_ANCHOR_SELECTOR)&.each do |node|
+          yield node if anchor_filter.call(node)
         end
       end
 
-      def relevant_anchor?(node, anchor_filter:)
-        node.element? &&
-          node.matches?(HtmlExtractor::MAIN_ANCHOR_SELECTOR) &&
-          anchor_filter.call(node)
-      end
-
       def traversal_root
         parsed_body.at_css('body, html') || parsed_body.root
       end

data/lib/html2rss/html_extractor/semantic_anchor_candidates.rb

@@ -31,6 +31,8 @@ module Html2rss
 
       # Shared context for all anchors in one semantic container.
       class Context
+        attr_reader :container
+
         # Ancestor tags that usually indicate navigation/utility regions.
         UTILITY_LANDMARK_TAGS = %w[nav aside footer menu].freeze
 
@@ -56,7 +58,7 @@ module Html2rss
         def visible_text(node)
           return '' unless node
 
-          HtmlExtractor.extract_visible_text(node).to_s.strip
+          (@visible_texts ||= {}.compare_by_identity)[node] ||= HtmlExtractor.extract_visible_text(node).to_s.strip
         end
 
         # @param anchor [Nokogiri::XML::Node] anchor candidate
@@ -70,12 +72,6 @@ module Html2rss
         def utility_text?(text)
           @link_heuristics.utility_text?(text)
         end
-
-        # @param ancestors [Array<Nokogiri::XML::Node>]
-        # @return [Boolean] true when the anchor lives inside navigation chrome
-        def utility_landmark?(ancestors)
-          ancestors.any? { |node| UTILITY_LANDMARK_TAGS.include?(node.name) }
-        end
       end
 
       # One anchor plus the facts needed to decide whether it represents content.
@@ -131,7 +127,7 @@ module Html2rss
 
         # @return [Boolean] true when visible anchor text has words
         def meaningful_text?
-          text.scan(/\p{Alnum}+/).any?
+          @meaningful_text ||= text.match?(/\p{Alnum}/)
         end
 
         # @return [Boolean] true when the destination route has content signals
@@ -142,8 +138,17 @@ module Html2rss
         # @return [Boolean] true when the anchor is inside the selected heading
         def heading_anchor?
           heading = @context.heading
+          return false unless heading
+
+          curr = @anchor
+          container = @context.container
+          while curr.respond_to?(:parent)
+            return true if curr == heading
+            break if curr == container
 
-          heading && @anchor.ancestors.include?(heading)
+            curr = curr.parent
+          end
+          false
         end
 
         # @return [Boolean] true when anchor text exactly matches heading text
@@ -151,14 +156,14 @@ module Html2rss
           heading_text = @context.heading_text
 
           meaningful_text? &&
-            heading_text.scan(/\p{Alnum}+/).any? &&
+            heading_text.match?(/\p{Alnum}/) &&
             heading_text == text
         end
 
         private
 
         def representative_content_anchor?
-          heading_anchor? || meaningful_text? || content_like_destination?
+          meaningful_text? || content_like_destination? || heading_anchor?
         end
 
         def utility_text_suppressed?
@@ -174,7 +179,19 @@ module Html2rss
         def ineligible_anchor?
           destination_facts.high_confidence_utility_destination ||
             icon_only_anchor? ||
-            @context.utility_landmark?(@anchor.ancestors.to_a)
+            utility_landmark_ancestor?
+        end
+
+        def utility_landmark_ancestor?
+          curr = @anchor.parent
+          container = @context.container
+          while curr.respond_to?(:parent)
+            return true if Context::UTILITY_LANDMARK_TAGS.include?(curr.name)
+            break if curr == container
+
+            curr = curr.parent
+          end
+          false
         end
 
         def icon_only_anchor?

data/lib/html2rss/html_extractor/semantic_containers.rb

@@ -27,43 +27,17 @@ module Html2rss
 
       # @return [Array<Nokogiri::XML::Node>] candidate semantic containers
       def call
-        containers = SELECTORS.each_with_object([]) do |selector, memo|
-          collect_selector_containers(selector, memo)
+        cache = {}.compare_by_identity
+        candidates = @parsed_body.css(SELECTORS.join(',')).reject do |node|
+          HtmlExtractor.ignored_container_path?(node, cache)
         end
 
-        containers.sort_by { document_order.fetch(_1) }
-      end
-
-      private
-
-      def document_order
-        @document_order ||= begin
-          order = {}
-          index = 0
-
-          @parsed_body.traverse do |node|
-            next unless node.element?
-
-            order[node] = index
-            index += 1
-          end
-
-          order.compare_by_identity
-        end
-      end
-
-      def collect_selector_containers(selector, containers)
-        @parsed_body.css(selector).each do |container|
-          next if HtmlExtractor.ignored_container_path?(container)
-          next if seen[container]
-
-          seen[container] = true
-          containers << container
-        end
-      end
-
-      def seen
-        @seen ||= {}.compare_by_identity
+        # Preserve the original post-order traversal intent (specific-first)
+        # by sorting candidates by depth (descending) while keeping original document
+        # order for nodes at the same depth.
+        candidates.each_with_index
+                  .sort_by { |node, index| [-node.ancestors.size, index] }
+                  .map!(&:first)
       end
     end
   end

data/lib/html2rss/html_extractor.rb

@@ -4,15 +4,15 @@ module Html2rss
   ##
   # HtmlExtractor is responsible for extracting details (headline, url, images, etc.)
   # from an article_tag.
-  class HtmlExtractor
+  class HtmlExtractor # rubocop:disable Metrics/ClassLength
     # Tags ignored when extracting visible text content from article containers.
     INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
-    # Element path pattern ignored when traversing candidate article containers.
-    IGNORED_CONTAINER_PATH = /(nav|footer|header|svg|script|style)/i
     # Heading tags used to prioritize title extraction.
     HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
     # Selector used to derive non-headline description nodes.
     NON_HEADLINE_SELECTOR = (HEADING_TAGS.map { |tag| ":not(#{tag})" } + INVISIBLE_CONTENT_TAGS.to_a).freeze
+    # Element tags that indicate ignored DOM chrome when found in a container path.
+    IGNORED_CONTAINER_TAGS = %w[nav footer header svg script style].to_set.freeze
 
     # Anchor selector used to identify the canonical article link element.
     MAIN_ANCHOR_SELECTOR = begin
@@ -42,8 +42,39 @@ module Html2rss
         parts.join(separator).squeeze(' ').strip unless parts.empty?
       end
 
+      ##
+      # @param article_tag [Nokogiri::XML::Node] article-like container to search within
+      # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
+      def main_anchor_for(article_tag)
+        return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
+
+        article_tag.at_css(MAIN_ANCHOR_SELECTOR)
+      end
+
+      ##
+      # @param node [Nokogiri::XML::Node]
+      # @param cache [Hash, nil] identity cache used to store results (must use compare_by_identity)
+      # @return [Boolean] true when the node belongs to ignored DOM chrome
+      def ignored_container_path?(node, cache = nil)
+        return cache[node] if cache&.key?(node)
+
+        res = walk_ignored_container_path?(node)
+        cache[node] = res if cache
+        res
+      end
+
       private
 
+      def walk_ignored_container_path?(node)
+        curr = node
+        while curr.respond_to?(:parent)
+          return true if IGNORED_CONTAINER_TAGS.include?(curr.name)
+
+          curr = curr.parent
+        end
+        false
+      end
+
       def visible_child?(node)
         !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
           !(node.name == 'a' && node['href']&.start_with?('#'))
@@ -80,26 +111,6 @@ module Html2rss
 
     attr_reader :article_tag, :base_url, :selected_anchor
 
-    class << self
-      ##
-      # @param article_tag [Nokogiri::XML::Node] article-like container to search within
-      # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
-      def main_anchor_for(article_tag)
-        return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
-
-        article_tag.at_css(MAIN_ANCHOR_SELECTOR)
-      end
-
-      ##
-      # @param node [Nokogiri::XML::Node, String] node or path to test
-      # @return [Boolean] true when the node belongs to ignored DOM chrome
-      def ignored_container_path?(node)
-        path = node.respond_to?(:path) ? node.path : node.to_s
-
-        path.match?(IGNORED_CONTAINER_PATH)
-      end
-    end
-
     def extract_url
       @extract_url ||= begin
         href = selected_anchor&.[]('href').to_s
@@ -115,14 +126,24 @@ module Html2rss
 
     def heading
       @heading ||= begin
-        heading_tags = article_tag.css(HEADING_TAGS.join(',')).group_by(&:name)
-        smallest_heading = heading_tags.keys.min
-        if smallest_heading
-          heading_tags[smallest_heading]&.max_by do |tag|
-            self.class.extract_visible_text(tag)&.size.to_i
-          end
-        end
+        tags = article_tag.css(HEADING_TAGS.join(','))
+        tags.any? ? select_best_heading(tags) : nil
+      end
+    end
+
+    def select_best_heading(tags)
+      min_tag_name = tags.map(&:name).min
+      best_tag = nil
+      max_size = -1
+
+      tags.each do |tag|
+        next if tag.name != min_tag_name
+
+        size = self.class.extract_visible_text(tag)&.size.to_i
+        (best_tag = tag) && (max_size = size) if size > max_size
       end
+
+      best_tag
     end
 
     def extract_description

data/lib/html2rss/rendering/description_builder.rb

@@ -25,12 +25,12 @@ module Html2rss
       # @param end_of_range [Integer] Optional, defaults to half the text length
       # @return [String]
       def self.remove_pattern_from_start(text, pattern, end_of_range: (text.size * 0.5).to_i)
-        return text unless text.is_a?(String) && pattern.is_a?(String)
+        return text unless text.is_a?(String) && pattern.is_a?(String) && !pattern.empty?
 
         index = text.index(pattern)
-        return text if index.nil? || index >= end_of_range
+        return text if index.nil? || index > end_of_range
 
-        text.gsub(/^(.{0,#{end_of_range}})#{Regexp.escape(pattern)}/, '\1')
+        "#{text[0, index]}#{text[(index + pattern.size)..]}"
       end
 
       # @param base [String] The base text content for the description

data/lib/html2rss/request_controls.rb

@@ -7,7 +7,7 @@ module Html2rss
     # Request-control keys accepted at the top level of feed config.
     TOP_LEVEL_KEYS = %i[strategy].freeze
     # Request-control keys accepted under the nested `request` config.
-    REQUEST_KEYS = %i[max_redirects max_requests].freeze
+    REQUEST_KEYS = %i[max_redirects max_requests total_timeout_seconds].freeze
 
     ##
     # @param config [Hash{Symbol => Object}] raw config input
@@ -20,6 +20,7 @@ module Html2rss
         strategy: config[:strategy],
         max_redirects: request_value_for(config, :max_redirects),
         max_requests: request_value_for(config, :max_requests),
+        total_timeout_seconds: request_value_for(config, :total_timeout_seconds),
         explicit_keys: explicit_keys_for(config)
       )
     end
@@ -47,11 +48,13 @@ module Html2rss
     # @param strategy [Symbol, nil] effective request strategy
     # @param max_redirects [Integer, nil] effective redirect limit
     # @param max_requests [Integer, nil] effective request budget
+    # @param total_timeout_seconds [Integer, nil] effective request timeout
     # @param explicit_keys [Array<Symbol>] controls explicitly supplied by the caller
-    def initialize(strategy: nil, max_redirects: nil, max_requests: nil, explicit_keys: [])
+    def initialize(strategy: nil, max_redirects: nil, max_requests: nil, total_timeout_seconds: nil, explicit_keys: [])
       @strategy = strategy
       @max_redirects = max_redirects
       @max_requests = max_requests
+      @total_timeout_seconds = total_timeout_seconds
       @explicit_keys = explicit_keys.map(&:to_sym).uniq.freeze
       freeze
     end
@@ -68,6 +71,10 @@ module Html2rss
     # @return [Integer, nil] effective request budget
     attr_reader :max_requests
 
+    ##
+    # @return [Integer, nil] effective request timeout
+    attr_reader :total_timeout_seconds
+
     ##
     # @param name [Symbol, String] request control name
     # @return [Boolean] whether the control was explicitly supplied
@@ -79,12 +86,14 @@ module Html2rss
     # @param strategy [Symbol, nil] validated request strategy
     # @param max_redirects [Integer, nil] validated redirect limit
     # @param max_requests [Integer, nil] validated request budget
+    # @param total_timeout_seconds [Integer, nil] validated request timeout
     # @return [RequestControls] controls updated with validated effective values
-    def with_effective_values(strategy:, max_redirects:, max_requests:)
+    def with_effective_values(strategy:, max_redirects:, max_requests:, total_timeout_seconds:)
       self.class.new(
         strategy:,
         max_redirects:,
         max_requests:,
+        total_timeout_seconds:,
         explicit_keys:
       )
     end
@@ -98,6 +107,7 @@ module Html2rss
       config[:strategy] = strategy if explicit?(:strategy)
       apply_request_value(config, :max_redirects, max_redirects)
       apply_request_value(config, :max_requests, max_requests)
+      apply_request_value(config, :total_timeout_seconds, total_timeout_seconds)
       config
     end
 

data/lib/html2rss/request_service/policy.rb

@@ -30,9 +30,9 @@ module Html2rss
 
       # Default policy values used when request controls are not explicitly set.
       DEFAULTS = {
-        connect_timeout_seconds: 5,
-        read_timeout_seconds: 10,
-        total_timeout_seconds: 30,
+        connect_timeout_seconds: Integer(ENV.fetch('HTML2RSS_CONNECT_TIMEOUT_SECONDS', 5)),
+        read_timeout_seconds: Integer(ENV.fetch('HTML2RSS_READ_TIMEOUT_SECONDS', 10)),
+        total_timeout_seconds: Integer(ENV.fetch('HTML2RSS_TOTAL_TIMEOUT_SECONDS', 30)),
         max_redirects: 3,
         max_response_bytes: 5_242_880,
         max_decompressed_bytes: 10_485_760,

data/lib/html2rss/request_session/runtime_policy.rb

@@ -11,7 +11,8 @@ module Html2rss
       def self.from_config(config)
         RequestService::Policy.new(
           max_requests: effective_max_requests_for(config),
-          max_redirects: config.max_redirects
+          max_redirects: config.max_redirects,
+          total_timeout_seconds: config.total_timeout_seconds || RequestService::Policy::DEFAULTS[:total_timeout_seconds]
         )
       end