html2rss 0.24.0 → 0.26.0

data/lib/html2rss/feed_builder/rss.rb

@@ -79,7 +79,7 @@ module Html2rss
         RSS::Maker.make('2.0') do |maker|
           Stylesheet.add(maker, stylesheets)
 
-          make_channel(maker.channel)
+          make_channel(maker)
           make_items(maker)
         end
       end
@@ -96,14 +96,26 @@ module Html2rss
         @stylesheets.map { |style| Stylesheet.new(**style) }
       end
 
+      # rubocop:disable Metrics/AbcSize
       def make_channel(maker)
+        channel_maker = maker.channel
         %i[language title description ttl].each do |key|
-          maker.public_send(:"#{key}=", channel.public_send(key))
+          channel_maker.public_send(:"#{key}=", channel.public_send(key))
         end
 
-        maker.link = channel.url.to_s
-        maker.generator = generator
-        maker.updated = channel.last_build_date
+        channel_maker.managingEditor = channel.author if channel.author
+        channel_maker.author = channel.author if channel.author
+        channel_maker.link = channel.url.to_s
+        channel_maker.generator = generator
+        channel_maker.updated = channel.last_build_date
+
+        make_image(maker.image) if channel.image
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      def make_image(image_maker)
+        image_maker.url = channel.image.to_s
+        image_maker.title = channel.title.to_s
       end
 
       def make_items(maker)

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -19,18 +19,18 @@ module Html2rss
         RequestService::RequestBudgetExceeded,
         RequestService::PrivateNetworkDenied,
         RequestService::CrossOriginFollowUpDenied,
-        RequestService::ResponseTooLarge,
-        RequestService::BrowserlessConfigurationError
+        RequestService::ResponseTooLarge
       ].freeze
 
       ##
       # Mutable run state for one auto-fallback chain: attempts plus selected result.
       class AttemptState
-        attr_reader :attempts, :result
+        attr_reader :attempts, :result, :last_response
 
         def initialize
           @attempts = []
           @result = nil
+          @last_response = nil
         end
 
         # @return [Boolean] true when a strategy already yielded items
@@ -53,6 +53,12 @@ module Html2rss
           @attempts << attempt
         end
 
+        # @param response [RequestService::Response] last response that reached extraction
+        # @return [void]
+        def remember_response(response)
+          @last_response = response
+        end
+
         # @param response [RequestService::Response] successful response
         # @param articles [Array] extracted articles
         # @param dedup_dropped [Integer] articles removed by deduplication
@@ -92,7 +98,7 @@ module Html2rss
         state = run_attempts
         return state.result if state.succeeded?
 
-        finalize_failure(attempts: state.attempts)
+        finalize_failure(attempts: state.attempts, response: state.last_response)
       end
 
       private
@@ -127,6 +133,7 @@ module Html2rss
       end
 
       def process_response(response:, strategy:, next_strategy:, request_session:, state:)
+        state.remember_response(response)
         articles, dedup_dropped = articles_for(response:, request_session:)
         items_count = articles.size
         state.record_items(strategy:, items_count:, transport_meta: response.transport_meta)
@@ -152,8 +159,11 @@ module Html2rss
                  "budget_remaining=#{budget_remaining_label}")
       end
 
-      def finalize_failure(attempts:)
-        raise NoFeedItemsExtracted.new(attempts:)
+      def finalize_failure(attempts:, response:)
+        surface_category = response && AutoSource::Scraper.classify_no_scraper_surface(
+          response.parsed_body, body: response.body
+        )
+        raise NoFeedItemsExtracted.new(attempts:, surface_category:)
       end
 
       # rubocop:disable Metrics/AbcSize, Metrics/MethodLength

data/lib/html2rss/feed_pipeline/runtime_policy.rb

@@ -6,8 +6,7 @@ module Html2rss
     # Planner for the runtime request policy and budgets of a feed run.
     #
     # HTTP request slots (pagination, document GETs, strategy fallback) are planned
-    # separately from Browserless preload interaction budget so preload cannot steal
-    # pagination slots.
+    # from configuration options.
     class RuntimePolicy
       # Policy plus Budget from one config expansion (shared meters for a feed run).
       Resources = Data.define(:policy, :budget)
@@ -34,26 +33,18 @@ module Html2rss
           policy:,
           budget: RequestService::Budget.new(
             max_requests: policy.max_requests,
-            max_interactions: interaction_budget_for(config),
             total_timeout_seconds: policy.total_timeout_seconds
           )
         )
       end
 
       ##
-      # Builds a Budget with separate request and interaction slot pools.
+      # Builds a Budget with request slot pools.
       #
       # @param config [Html2rss::Config] validated feed config
       # @return [Html2rss::RequestService::Budget] shared budget for the feed build
       def self.budget_for(config) = resources_for(config).budget
 
-      ##
-      # @param config [Html2rss::Config] validated feed config
-      # @return [Integer] preload interaction slots derived from browserless preload config
-      def self.interaction_budget_for(config)
-        browserless_preload_budget_for(config)
-      end
-
       class << self
         private
 
@@ -64,44 +55,13 @@ module Html2rss
         end
 
         # Reserve enough HTTP request slots for the initial request plus predictable
-        # follow-ups. Preload interactions are planned separately.
+        # follow-ups.
         def baseline_request_budget_for(config)
           1 +
             RequestSession::Pager.request_slots_for(config.selectors&.dig(:items, :pagination)) +
             AutoSource.request_slots_for(config.auto_source) +
             StrategyPlan.resolve(config.strategy).request_slots
         end
-
-        def browserless_preload_budget_for(config)
-          preload = config.request.dig(:browserless, :preload)
-          return 0 unless preload
-
-          top_level_preload_wait_budget(preload) +
-            click_selector_preload_budget(preload) +
-            scroll_preload_budget(preload)
-        end
-
-        def top_level_preload_wait_budget(preload)
-          preload[:wait_after_ms] ? 2 : 0
-        end
-
-        def click_selector_preload_budget(preload)
-          preload.fetch(:click_selectors, []).sum { preload_action_budget(_1, :max_clicks) }
-        end
-
-        def scroll_preload_budget(preload)
-          scroll = preload[:scroll_down]
-          return 0 unless scroll
-
-          preload_action_budget(scroll, :iterations)
-        end
-
-        def preload_action_budget(config, count_key)
-          action_count = config.fetch(count_key, 1)
-          wait_budget = config[:wait_after_ms] ? action_count : 0
-
-          action_count + wait_budget
-        end
       end
     end
   end

data/lib/html2rss/feed_pipeline.rb

@@ -81,11 +81,22 @@ module Html2rss
       request_session = request_session_for(config, strategy:, resources:)
       response = request_session.fetch_initial_response
       articles, dedup_dropped = deduplicated_articles(config:, response:, request_session:)
+      raise_empty_auto_source!(strategy:, response:) if config.auto_source && articles.empty?
+
       PipelineOutcome.new(
         response:, articles:, dedup_dropped:, selected_strategy: nil, attempt_count: 0, strategy_attempts: []
       )
     end
 
+    def raise_empty_auto_source!(strategy:, response:)
+      raise NoFeedItemsExtracted.new(
+        attempts: [{ strategy:, items_count: 0, error_class: nil }],
+        surface_category: AutoSource::Scraper.classify_no_scraper_surface(
+          response.parsed_body, body: response.body
+        )
+      )
+    end
+
     def run_auto_pipeline(config, resources:)
       AutoFallback.new(
         strategies: AutoFallback::CHAIN,

data/lib/html2rss/feed_result.rb

@@ -69,7 +69,7 @@ module Html2rss
 
     ##
     # @return [Html2rss::Status] frozen scraper tallies, dedup count, and generator formatter.
-    #   {#to_h} on +status+ is a stable consumer contract for observability payloads.
+    #   {Status#to_h} on +status+ is a stable consumer contract for observability payloads.
     attr_reader :status
 
     private

data/lib/html2rss/hash_util.rb

@@ -54,6 +54,23 @@ module Html2rss
       end
     end
 
+    # Converts hash keys (and Symbol values) to strings recursively.
+    #
+    # @param object [Object] value to normalize
+    # @return [Object] normalized value with string keys
+    def deep_stringify_keys(object)
+      case object
+      when Hash
+        object.to_h { |key, value| [key.to_s, deep_stringify_keys(value)] }
+      when Array
+        object.map { deep_stringify_keys(_1) }
+      when Symbol
+        object.to_s
+      else
+        object
+      end
+    end
+
     # Validates that hash keys are symbols.
     #
     # @param value [Object] candidate hash container whose keys must be symbols

data/lib/html2rss/html/article_extractor/category_extractor.rb

@@ -7,20 +7,16 @@ module Html2rss
       # CategoryExtractor is responsible for extracting categories from HTML elements
       # by looking for CSS class names containing common category-related terms.
       class CategoryExtractor
-        # Common category-related terms to look for in class names
-        CATEGORY_TERMS = %w[
-          category categories tag tags topic topics section sections
-          label labels theme themes subject subjects
-        ].freeze
+        # Shared category vocabulary (owned by {ArticleRules::Category}).
+        CATEGORY_TERMS = ArticleRules::Category::CATEGORY_TERMS
+        # Shared category attribute/class pattern (owned by {ArticleRules::Category}).
+        CATEGORY_ATTR_PATTERN = ArticleRules::Category::CATEGORY_ATTR_PATTERN
 
         # CSS selectors to find elements with category-related class names or data attributes
         CATEGORY_SELECTORS = CATEGORY_TERMS.flat_map do |term|
           ["[class*=\"#{term}\"]", "[data-#{term}]", "[#{term}]"]
         end.freeze
 
-        # Regex pattern for matching category-related attribute names
-        CATEGORY_ATTR_PATTERN = /#{CATEGORY_TERMS.join('|')}/i
-
         ##
         # Extracts categories from the given article tag by looking for elements
         # with class names containing common category-related terms.
@@ -30,7 +26,6 @@ module Html2rss
         def self.call(article_tag)
           return [] unless article_tag
 
-          # Single optimized traversal that extracts all category types
           extract_all_categories(article_tag)
             .map(&:strip)
             .reject(&:empty?)
@@ -44,10 +39,7 @@ module Html2rss
         def self.extract_all_categories(article_tag)
           Set.new.tap do |categories|
             article_tag.css(CATEGORY_SELECTORS.join(',')).each do |element|
-              # Extract text categories from elements with category-related class names
-              extract_text_categories!(categories, element) if element['class']&.match?(CATEGORY_ATTR_PATTERN)
-
-              # Extract data categories from all elements
+              extract_text_categories!(categories, element) if ArticleRules::Category.class_match?(element['class'])
               extract_element_data_categories!(categories, element)
             end
           end
@@ -61,10 +53,9 @@ module Html2rss
         # @return [void]
         def self.extract_element_data_categories!(categories, element)
           element.attributes.each_value do |attr|
-            next unless attr.name.match?(CATEGORY_ATTR_PATTERN)
+            next unless ArticleRules::Category.attr_name_match?(attr.name)
 
-            value = attr.value&.strip
-            categories.add(value) if value && !value.empty?
+            ArticleRules::Category.add_text!(categories, attr.value)
           end
         end
 
@@ -85,7 +76,7 @@ module Html2rss
           if anchors.any?
             anchors.each { |node| add_text_to_categories!(categories, node) }
           else
-            extract_split_text_categories!(categories, element)
+            ArticleRules::Category.add_split_text!(categories, element.text)
           end
         end
 
@@ -96,27 +87,10 @@ module Html2rss
         # @param element [Nokogiri::XML::Element] The element to extract text from
         # @return [void]
         def self.add_text_to_categories!(categories, element)
-          text = Navigator.extract_visible_text(element)
-          categories.add(text) if text && !text.empty?
-        end
-
-        ##
-        # Extracts categories from the element's text by splitting on newlines.
-        #
-        # @param categories [Set<String>] Accumulator set
-        # @param element [Nokogiri::XML::Element] The element to extract text from
-        # @return [void]
-        def self.extract_split_text_categories!(categories, element)
-          text = element.text
-          return unless text
-
-          text.split(/\n+/).each do |line|
-            line = line.strip
-            categories.add(line) unless line.empty?
-          end
+          ArticleRules::Category.add_text!(categories, Navigator.extract_visible_text(element))
         end
 
-        private_class_method :add_text_to_categories!, :extract_split_text_categories!
+        private_class_method :add_text_to_categories!
       end
     end
   end

data/lib/html2rss/html/article_extractor/date_extractor.rb

@@ -8,13 +8,8 @@ module Html2rss
         # @param article_tag [Nokogiri::XML::Element] article container node
         # @return [DateTime, nil]
         def self.call(article_tag)
-          times = article_tag.css('[datetime]').filter_map do |tag|
-            DateTime.parse(tag['datetime'])
-          rescue ArgumentError, TypeError
-            nil
-          end
-
-          times.min
+          datetimes = article_tag.css('[datetime]').map { |tag| tag['datetime'] }
+          ArticleRules::Date.earliest(datetimes)
         end
       end
     end

data/lib/html2rss/html/article_extractor/enclosure_extractor.rb

@@ -33,62 +33,17 @@ module Html2rss
         def self.extract_from_element(element, base_url)
           case element.name
           when 'img'
-            extract_image(element, base_url)
+            ArticleRules::Enclosure.from_image(element['src'], base_url)
           when 'video', 'audio', 'source'
-            extract_media(element, base_url)
+            ArticleRules::Enclosure.from_media(element['src'], element['type'], base_url)
           when 'iframe'
-            extract_iframe(element, base_url)
+            ArticleRules::Enclosure.from_iframe(element['src'], base_url)
           when 'a'
-            extract_a(element, base_url)
+            ArticleRules::Enclosure.from_anchor(element['href'], base_url)
           end
         end
 
-        def self.extract_image(img, base_url)
-          src = img['src'].to_s
-          return if src.empty?
-
-          abs_url = Url.from_relative(src, base_url)
-          {
-            url: abs_url,
-            type: Article::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
-          }
-        end
-
-        def self.extract_media(element, base_url)
-          src = element['src'].to_s
-          return if src.empty?
-
-          {
-            url: Url.from_relative(src, base_url),
-            type: element['type']
-          }
-        end
-
-        def self.extract_iframe(iframe, base_url)
-          src = iframe['src'].to_s
-          return if src.empty?
-
-          abs_url = Url.from_relative(src, base_url)
-          {
-            url: abs_url,
-            type: Article::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
-          }
-        end
-
-        def self.extract_a(link, base_url)
-          href = link['href'].to_s
-          return if href.empty?
-
-          abs_url = Url.from_relative(href, base_url)
-
-          if href.end_with?('.pdf')
-            { url: abs_url, type: Article::Enclosure.guess_content_type_from_url(abs_url) }
-          else
-            { url: abs_url, type: 'application/zip' }
-          end
-        end
-
-        private_class_method :extract_from_element, :extract_image, :extract_media, :extract_iframe, :extract_a
+        private_class_method :extract_from_element
       end
     end
   end

data/lib/html2rss/html/article_extractor/image_extractor.rb

@@ -32,27 +32,16 @@ module Html2rss
         # @see <https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Responsive_images>
         # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#srcset>
         # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture>
-        def self.from_source(article_tag) # rubocop:disable Metrics/AbcSize
-          hash = article_tag.css('img[srcset], picture > source[srcset]').flat_map do |source|
-            source['srcset'].to_s.scan(/(\S+)\s+(\d+w|\d+h)[\s,]?/).map do |url, width|
-              next if url.nil? || url.start_with?('data:')
-
-              width_value = width.to_i.zero? ? 0 : width.scan(/\d+/).first.to_i
-
-              [width_value, url.strip]
-            end
-          end.compact.to_h
-
-          hash[hash.keys.max]
+        def self.from_source(article_tag)
+          srcsets = article_tag.css('img[srcset], picture > source[srcset]').map { |source| source['srcset'] }
+          ArticleRules::Image.largest_from_srcsets(srcsets)
         end
 
         # @param article_tag [Nokogiri::XML::Element] article container node
         # @return [String, nil] best style-based background image URL
         def self.from_style(article_tag)
-          article_tag.css('[style*="url"]')
-                     .filter_map { |tag| tag['style'][/url\(['"]?(.*?)['"]?\)/, 1] }
-                     .reject { |src| src.start_with?('data:') }
-                     .max_by(&:size)
+          styles = article_tag.css('[style*="url"]').map { |tag| tag['style'] }
+          ArticleRules::Image.best_from_styles(styles)
         end
       end
     end

data/lib/html2rss/html/article_rules/category.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module ArticleRules
+      ##
+      # Shared category term vocabulary and text accumulation (DOM walk stays in adapters).
+      module Category
+        # Class/data attribute tokens that mark category metadata.
+        CATEGORY_TERMS = %w[
+          category categories tag tags topic topics section sections
+          label labels theme themes subject subjects
+        ].freeze
+
+        # Regex matching category-related attribute or class names.
+        CATEGORY_ATTR_PATTERN = /#{CATEGORY_TERMS.join('|')}/i
+
+        class << self
+          ##
+          # @param name [String]
+          # @return [Boolean]
+          def attr_name_match?(name)
+            name.to_s.match?(CATEGORY_ATTR_PATTERN)
+          end
+
+          ##
+          # @param class_attr [String, nil]
+          # @return [Boolean]
+          def class_match?(class_attr)
+            class_attr.to_s.match?(CATEGORY_ATTR_PATTERN)
+          end
+
+          ##
+          # @param categories [Set<String>]
+          # @param text [String, nil]
+          # @return [void]
+          def add_text!(categories, text)
+            value = text.to_s.strip
+            categories.add(value) unless value.empty?
+          end
+
+          ##
+          # @param categories [Set<String>]
+          # @param text [String, nil]
+          # @return [void]
+          def add_split_text!(categories, text)
+            return unless text
+
+            text.split(/\n+/).each { |line| add_text!(categories, line) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_rules/date.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module ArticleRules
+      ##
+      # DOM-agnostic datetime aggregation for article published_at.
+      module Date
+        class << self
+          ##
+          # @param datetime_strings [Array<String, nil>] raw datetime attribute values
+          # @return [DateTime, nil] earliest successfully parsed instant
+          def earliest(datetime_strings)
+            times = Array(datetime_strings).filter_map do |value|
+              DateTime.parse(value.to_s)
+            rescue ArgumentError, TypeError
+              nil
+            end
+            times.min
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_rules/enclosure.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module ArticleRules
+      ##
+      # DOM-agnostic enclosure URL/type normalization.
+      module Enclosure
+        # Href path suffixes treated as download/archive enclosure links.
+        ARCHIVE_HREF_SUFFIXES = %w[.pdf .zip .tar.gz .tgz].freeze
+
+        class << self
+          ##
+          # @param href [String]
+          # @return [Boolean]
+          def archive_href?(href)
+            ARCHIVE_HREF_SUFFIXES.any? { |suffix| href.to_s.end_with?(suffix) }
+          end
+
+          ##
+          # @param src [String]
+          # @param base_url [String, Html2rss::Url]
+          # @return [Hash{Symbol => Object}, nil]
+          def from_image(src, base_url)
+            return if src.to_s.empty?
+
+            abs = Url.from_relative(src, base_url)
+            { url: abs, type: Article::Enclosure.guess_content_type_from_url(abs, default: 'image/jpeg') }
+          end
+
+          ##
+          # @param src [String]
+          # @param type [String, nil]
+          # @param base_url [String, Html2rss::Url]
+          # @return [Hash{Symbol => Object}, nil]
+          def from_media(src, type, base_url)
+            return if src.to_s.empty?
+
+            { url: Url.from_relative(src, base_url), type: }
+          end
+
+          ##
+          # @param src [String]
+          # @param base_url [String, Html2rss::Url]
+          # @return [Hash{Symbol => Object}, nil]
+          def from_iframe(src, base_url)
+            return if src.to_s.empty?
+
+            abs = Url.from_relative(src, base_url)
+            { url: abs, type: Article::Enclosure.guess_content_type_from_url(abs, default: 'text/html') }
+          end
+
+          ##
+          # @param href [String]
+          # @param base_url [String, Html2rss::Url]
+          # @return [Hash{Symbol => Object}, nil]
+          def from_anchor(href, base_url)
+            return if href.to_s.empty?
+
+            abs = Url.from_relative(href, base_url)
+            type = if href.end_with?('.pdf')
+                     Article::Enclosure.guess_content_type_from_url(abs)
+                   else
+                     'application/zip'
+                   end
+            { url: abs, type: }
+          end
+        end
+      end
+    end
+  end
+end