html2rss 0.32.4 → 0.34.0

data/lib/html2rss/feed_resolution.rb

@@ -8,6 +8,9 @@ module Html2rss
   #
   # {include:file:lib/html2rss/feed_resolution/README.md}
   module FeedResolution
+    # Raised when the tournament or its policy is handed articles that are not an Array.
+    class InvalidError < Html2rss::ContractError; end
+
     # Default entry-resolution options merged into AutoSource::DEFAULT_CONFIG.
     DEFAULT_CONFIG = {
       enabled: true,
@@ -116,17 +119,13 @@ module Html2rss
         return unless resolution.applied
 
         effective = scrape_target.with_effective(resolution.scrape_url)
-        return ApplyOutcome.new(scrape_target: effective, status: :succeeded) if retry_with(
-          resolution, effective:
-        )
-
-        nil
+        ApplyOutcome.new(scrape_target: effective, status: :succeeded) if retry_with(effective:)
       rescue RequestService::RequestBudgetExceeded
         log_budget_exhausted
         nil
-      rescue *NON_FALLBACK_ERRORS
+      rescue *NON_FALLBACK_ERRORS, Html2rss::ContractError
         raise
-      rescue StandardError => error
+      rescue Html2rss::Error => error
         log_retry_failed(error)
         nil
       end
@@ -158,7 +157,7 @@ module Html2rss
       # @return [String, nil]
       def channel_host
         Url.from_absolute(config.url.to_s).host
-      rescue ArgumentError
+      rescue Html2rss::Url::InvalidUrlError
         nil
       end
 
@@ -184,24 +183,20 @@ module Html2rss
         )
       end
 
-      def retry_with(_resolution, effective:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- retry extract path
+      def retry_with(effective:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- retry extract path
         retry_session = pipeline.request_session_for(
           config, strategy:, resources:, scrape_url: effective.effective_url
         )
         retry_response = retry_session.fetch_initial_response
-        retry_articles, dedup_dropped, admission_drops = pipeline.deduplicated_articles(
-          config:, response: retry_response, request_session: retry_session
-        )
+        extraction = pipeline.extract(config:, response: retry_response, request_session: retry_session)
+        items_count = extraction.articles.size
         state.remember_response(retry_response)
-        state.record_items(
-          strategy:, items_count: retry_articles.size, transport_meta: retry_response.transport_meta
-        )
-        return unless retry_articles.size.positive?
+        state.record_items(strategy:, items_count:, transport_meta: retry_response.transport_meta)
+        return unless items_count.positive?
 
         state.succeed!(
-          response: retry_response, articles: retry_articles, dedup_dropped:,
-          selected_strategy: strategy, attempt_count: state.attempts.size, admission_drops:,
-          scrape_target: effective
+          response: retry_response, extraction:, selected_strategy: strategy,
+          attempt_count: state.attempts.size, scrape_target: effective
         )
         true
       end
@@ -218,9 +213,9 @@ module Html2rss
       # @param surface_category [Symbol, nil]
       # rubocop:disable-next Metrics/ParameterLists -- tournament context stays co-located
       def initialize(entry_url:, response:, session:, config:, articles:, surface_category: nil)
-        raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
+        raise InvalidError, 'articles must be an Array' unless articles.is_a?(Array)
 
-        @entry_url = Html2rss::Url.from_absolute(entry_url)
+        @entry_url = Url.from_absolute(entry_url)
         @response = response
         @session = session
         @config = config

data/lib/html2rss/feed_result.rb

@@ -15,17 +15,20 @@ module Html2rss
   #   are re-frozen via private {#marshal_load}. For trusted cache reads only,
   #   +Marshal.load(payload, freeze: true)+ is also safe.
   class FeedResult
+    # Raised when the scrape handle is built from invalid parts.
+    class InvalidError < Html2rss::ContractError; end
+
     ##
     # @param channel [Html2rss::Channel] channel metadata (internal; not exposed on the public API)
     # @param articles [Array<Html2rss::Article>] extracted articles (deduplicated; not exposed)
     # @param status [Html2rss::Status] pipeline telemetry
     # @param stylesheets [Array<Hash>] optional RSS stylesheet configs
+    # @raise [InvalidError] when a part has the wrong type
     def initialize(channel:, articles:, status:, stylesheets: [])
-      raise ArgumentError, 'channel must be a Html2rss::Channel' unless channel.is_a?(Channel)
-      raise ArgumentError, 'status must be a Html2rss::Status' unless status.is_a?(Status)
-
-      articles = Array(articles)
-      raise ArgumentError, 'articles must all be Html2rss::Article' unless articles.all?(Article)
+      raise InvalidError, 'channel must be a Html2rss::Channel' unless channel.is_a?(Channel)
+      raise InvalidError, 'status must be a Html2rss::Status' unless status.is_a?(Status)
+      raise InvalidError, 'articles must be an Array' unless articles.is_a?(Array)
+      raise InvalidError, 'articles must all be Html2rss::Article' unless articles.all?(Article)
 
       @channel = channel
       @articles = articles.dup.freeze
@@ -82,7 +85,9 @@ module Html2rss
     end
 
     def freeze_stylesheets(stylesheets)
-      Array(stylesheets).map do |sheet|
+      raise InvalidError, 'stylesheets must be an Array' unless stylesheets.is_a?(Array)
+
+      stylesheets.map do |sheet|
         sheet.to_h.transform_keys(&:to_sym).transform_values do |value|
           value.is_a?(String) ? value.dup.freeze : value
         end.freeze

data/lib/html2rss/hash_util.rb

@@ -3,116 +3,117 @@
 module Html2rss
   # Shared helpers for hash normalization and structural operations.
   module HashUtil
-    module_function
+    class << self
+      # Deeply duplicates nested arrays and hashes.
+      #
+      # @param object [Object] nested value from configuration or runtime state
+      # @return [Object] deep duplicated object
+      def deep_dup(object)
+        case object
+        in Hash
+          object.transform_values { deep_dup(_1) }
+        in Array
+          object.map { deep_dup(_1) }
+        else
+          object.dup
+        end
+      end
 
-    # Deeply duplicates nested arrays and hashes.
-    #
-    # @param object [Object] nested value from configuration or runtime state
-    # @return [Object] deep duplicated object
-    def deep_dup(object)
-      case object
-      in Hash
-        object.transform_values { deep_dup(_1) }
-      in Array
-        object.map { deep_dup(_1) }
-      else
-        object.dup rescue StandardError # rubocop:disable Style/RescueModifier
+      # Deeply merges nested hashes while replacing non-hash values from override.
+      #
+      # @param base [Hash] base hash
+      # @param override [Hash] override hash
+      # @return [Hash] merged hash
+      def deep_merge(base, override)
+        base.merge(override) do |_key, old_val, new_val|
+          case [old_val, new_val]
+          in [Hash, Hash]
+            deep_merge(old_val, new_val)
+          else
+            new_val
+          end
+        end
       end
-    end
 
-    # Deeply merges nested hashes while replacing non-hash values from override.
-    #
-    # @param base [Hash] base hash
-    # @param override [Hash] override hash
-    # @return [Hash] merged hash
-    def deep_merge(base, override)
-      base.merge(override) do |_key, old_val, new_val|
-        case [old_val, new_val]
-        in [Hash, Hash]
-          deep_merge(old_val, new_val)
+      # Converts string-keyed hashes to symbol-keyed hashes recursively.
+      #
+      # @param object [Object] value to normalize
+      # @param context [String] error context
+      # @return [Object] normalized value
+      def deep_symbolize_keys(object, context: 'hash')
+        case object
+        in Hash
+          object.each_with_object({}) do |(k, v), memo|
+            memo[symbol_key(k, context:)] = deep_symbolize_keys(v, context:)
+          end
+        in Array
+          object.map { deep_symbolize_keys(_1, context:) }
         else
-          new_val
+          object
         end
       end
-    end
 
-    # Converts string-keyed hashes to symbol-keyed hashes recursively.
-    #
-    # @param object [Object] value to normalize
-    # @param context [String] error context
-    # @return [Object] normalized value
-    def deep_symbolize_keys(object, context: 'hash')
-      case object
-      in Hash
-        object.each_with_object({}) do |(k, v), memo|
-          memo[symbol_key(k, context:)] = deep_symbolize_keys(v, context:)
+      # Converts hash keys (and Symbol values) to strings recursively.
+      #
+      # @param object [Object] value to normalize
+      # @return [Object] normalized value with string keys
+      def deep_stringify_keys(object)
+        case object
+        when Hash
+          object.to_h { |key, value| [key.to_s, deep_stringify_keys(value)] }
+        when Array
+          object.map { deep_stringify_keys(_1) }
+        when Symbol
+          object.to_s
+        else
+          object
         end
-      in Array
-        object.map { deep_symbolize_keys(_1, context:) }
-      else
-        object
       end
-    end
 
-    # Converts hash keys (and Symbol values) to strings recursively.
-    #
-    # @param object [Object] value to normalize
-    # @return [Object] normalized value with string keys
-    def deep_stringify_keys(object)
-      case object
-      when Hash
-        object.to_h { |key, value| [key.to_s, deep_stringify_keys(value)] }
-      when Array
-        object.map { deep_stringify_keys(_1) }
-      when Symbol
-        object.to_s
-      else
-        object
-      end
-    end
+      # Validates that hash keys are symbols.
+      #
+      # @param value [Object] candidate hash container whose keys must be symbols
+      # @param context [String] error context
+      # @param deep [Boolean] whether nested hashes should also be validated
+      # @return [void]
+      def assert_symbol_keys!(value, context: 'hash', deep: true)
+        return unless value in Hash
 
-    # Validates that hash keys are symbols.
-    #
-    # @param value [Object] candidate hash container whose keys must be symbols
-    # @param context [String] error context
-    # @param deep [Boolean] whether nested hashes should also be validated
-    # @return [void]
-    def assert_symbol_keys!(value, context: 'hash', deep: true)
-      return unless value in Hash
+        unless value.each_key.all?(Symbol)
+          invalid_key = value.keys.find { _1.class != Symbol }
+          raise InvalidOption, "#{context} must use symbol keys (found #{invalid_key.inspect})"
+        end
 
-      unless value.each_key.all?(Symbol)
-        invalid_key = value.keys.find { _1.class != Symbol }
-        raise ArgumentError, "#{context} must use symbol keys (found #{invalid_key.inspect})"
+        value.each_value { assert_symbol_keys!(_1, context:, deep:) } if deep
       end
 
-      value.each_value { assert_symbol_keys!(_1, context:, deep:) } if deep
-    end
+      # Validates that hash keys are strings.
+      #
+      # @param value [Object] candidate hash container whose keys must be strings
+      # @param context [String] error context
+      # @param deep [Boolean] whether nested hashes should also be validated
+      # @return [void]
+      def assert_string_keys!(value, context: 'hash', deep: true)
+        return unless value in Hash
 
-    # Validates that hash keys are strings.
-    #
-    # @param value [Object] candidate hash container whose keys must be strings
-    # @param context [String] error context
-    # @param deep [Boolean] whether nested hashes should also be validated
-    # @return [void]
-    def assert_string_keys!(value, context: 'hash', deep: true)
-      return unless value in Hash
+        unless value.each_key.all?(String)
+          invalid_key = value.keys.find { _1.class != String }
+          raise InvalidOption, "#{context} must use string keys (found #{invalid_key.inspect})"
+        end
 
-      unless value.each_key.all?(String)
-        invalid_key = value.keys.find { _1.class != String }
-        raise ArgumentError, "#{context} must use string keys (found #{invalid_key.inspect})"
+        value.each_value { assert_string_keys!(_1, context:, deep:) } if deep
       end
 
-      value.each_value { assert_string_keys!(_1, context:, deep:) } if deep
-    end
+      private
 
-    def symbol_key(key, context:)
-      case key
-      in Symbol then key
-      in String then key.to_sym
-      else
-        raise ArgumentError, "#{context} must use string or symbol keys (found #{key.inspect})"
+      def symbol_key(key, context:)
+        case key
+        in Symbol then key
+        in String then key.to_sym
+        else
+          raise InvalidOption, "#{context} must use string or symbol keys (found #{key.inspect})"
+        end
       end
     end
-    private_class_method :symbol_key
   end
 end

data/lib/html2rss/html/article_extractor/category_extractor.rb

@@ -28,24 +28,21 @@ module Html2rss
         def self.call(article_tag, title: nil)
           return [] unless article_tag
 
-          extract_all_categories(article_tag, title:)
-            .map(&:strip)
-            .reject(&:empty?)
+          extract_all_categories(article_tag, title:).to_a
         end
 
         ##
         # Optimized single DOM traversal that extracts all category types.
+        # `Node#css` only yields descendants, so the container itself is never visited.
         #
         # @param article_tag [Nokogiri::XML::Element] The article element
         # @param title [String, nil]
         # @return [Set<String>] Set of category strings
-        def self.extract_all_categories(article_tag, title: nil)
+        def self.extract_all_categories(article_tag, title:)
           Set.new.tap do |categories|
             extract_element_data_categories!(categories, article_tag, title:)
             article_tag.css(CATEGORY_SELECTORS.join(',')).each do |element|
-              next if element == article_tag
-
-              if ArticleRules::Category.class_match?(element['class'])
+              if ArticleRules::Category.class_match?(element['class'].to_s.split)
                 extract_text_categories!(categories, element, title:)
               end
               extract_element_data_categories!(categories, element, title:)
@@ -60,7 +57,7 @@ module Html2rss
         # @param element [Nokogiri::XML::Element] metadata element that may contain category links
         # @param title [String, nil]
         # @return [void]
-        def self.extract_element_data_categories!(categories, element, title: nil)
+        def self.extract_element_data_categories!(categories, element, title:)
           element.attributes.each_value do |attr|
             next unless ArticleRules::Category.attr_name_match?(attr.name)
 
@@ -75,7 +72,7 @@ module Html2rss
         # @param element [Nokogiri::XML::Element] metadata element whose text may contain delimiters
         # @param title [String, nil]
         # @return [void]
-        def self.extract_text_categories!(categories, element, title: nil)
+        def self.extract_text_categories!(categories, element, title:)
           return if category_container?(element)
 
           anchors = element.name == 'a' ? [element] : element.css('a').to_a
@@ -96,7 +93,7 @@ module Html2rss
         # @param element [Nokogiri::XML::Element] The element to extract text from
         # @param title [String, nil]
         # @return [void]
-        def self.add_text_to_categories!(categories, element, title: nil)
+        def self.add_text_to_categories!(categories, element, title:)
           ArticleRules::Category.add_text!(categories, Navigator.extract_visible_text(element), title:)
         end
 

data/lib/html2rss/html/article_extractor/date_extractor.rb

@@ -9,7 +9,7 @@ module Html2rss
         # @param leftover_lines [Array<String>] leftover visible lines (already split)
         # @param time_zone [String] channel time zone for naive values
         # @return [DateTime, nil]
-        def self.call(article_tag, leftover_lines: [], time_zone: 'UTC')
+        def self.call(article_tag, time_zone:, leftover_lines: [])
           datetimes = article_tag.css('[datetime]').map { |tag| tag['datetime'] }
           ArticleRules::Date.earliest(datetimes, leftover_lines:, time_zone:)
         end

data/lib/html2rss/html/article_extractor/enclosure_extractor.rb

@@ -4,46 +4,25 @@ module Html2rss
   module Html
     class ArticleExtractor
       ##
-      # Extracts enclosures from HTML tags using various strategies.
+      # Adapts Nokogiri elements to {ArticleRules::Enclosure}, which owns which elements become enclosures.
       class EnclosureExtractor
-        # CSS union query covering images, media, PDFs, iframes, and archives.
-        SELECTOR = [
-          'img[src]:not([src^="data"])',
-          'video source[src]',
-          'audio source[src]',
-          'audio[src]',
-          'a[href$=".pdf"]',
-          'iframe[src]',
-          'a[href$=".zip"]',
-          'a[href$=".tar.gz"]',
-          'a[href$=".tgz"]'
-        ].join(',').freeze
+        # CSS union of the element names {ArticleRules::Enclosure} may turn into an enclosure.
+        SELECTOR = ArticleRules::Enclosure::ELEMENT_NAMES.join(',').freeze
+        # CSS union of the players a +source+ element must sit inside.
+        PLAYERS = ArticleRules::Enclosure::MEDIA_ELEMENTS.join(',').freeze
 
         # @param article_tag [Nokogiri::XML::Element] article container node
         # @param base_url [String, Html2rss::Url] base URL for relative enclosure links
-        # @return [Array<Hash{Symbol => Object}>] normalized enclosure hashes
+        # @return [Array<Hash{Symbol => Object}>] normalized enclosure hashes in document order, one per URL
         def self.call(article_tag, base_url)
-          return [] unless article_tag
-
-          article_tag.css(SELECTOR).filter_map do |element|
-            extract_from_element(element, base_url)
+          found = article_tag.css(SELECTOR).filter_map do |element|
+            attrs = { src: element['src'], href: element['href'], type: element['type'] }
+            ArticleRules::Enclosure.from_element(element.name, base_url, **attrs) do
+              element.ancestors(PLAYERS).first&.name
+            end
           end
+          ArticleRules::Enclosure.unique(found)
         end
-
-        def self.extract_from_element(element, base_url)
-          case element.name
-          when 'img'
-            ArticleRules::Enclosure.from_image(element['src'], base_url)
-          when 'video', 'audio', 'source'
-            ArticleRules::Enclosure.from_media(element['src'], element['type'], base_url)
-          when 'iframe'
-            ArticleRules::Enclosure.from_iframe(element['src'], base_url)
-          when 'a'
-            ArticleRules::Enclosure.from_anchor(element['href'], base_url)
-          end
-        end
-
-        private_class_method :extract_from_element
       end
     end
   end

data/lib/html2rss/html/article_extractor/heading_extractor.rb

@@ -12,10 +12,9 @@ module Html2rss
         class << self
           ##
           # @param article_tag [Nokogiri::XML::Element] container node
-          # @param fallback_anchorless [Boolean] whether to use fallback search
           # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
           # @return [Nokogiri::XML::Node, nil] the heading node, if found
-          def call(article_tag, fallback_anchorless:, selected_anchor:)
+          def call(article_tag, selected_anchor:)
             tags = article_tag.css(HEADING_TAGS.join(','))
             if tags.any?
               select_best_heading(tags)
@@ -23,7 +22,7 @@ module Html2rss
               labeled = heading_from_aria_or_title(article_tag)
               return labeled if labeled
 
-              fallback_heading(article_tag) if fallback_anchorless && selected_anchor.nil?
+              fallback_heading(article_tag) if selected_anchor.nil?
             end
           end
 

data/lib/html2rss/html/article_extractor/id_generator.rb

@@ -14,16 +14,15 @@ module Html2rss
           # @param heading [Nokogiri::XML::Node, nil] heading node
           # @param url [Html2rss::Url, nil] absolute article URL
           # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
-          # @param fallback_anchorless [Boolean] whether to use fallback hashing
           # @return [String, nil] the generated ID, if any
-          def call(article_tag, heading:, url:, selected_anchor:, fallback_anchorless:)
+          def call(article_tag, heading:, url:, selected_anchor:)
             id_from_dom = parse_id_from_dom(article_tag, url, selected_anchor)
             return id_from_dom if id_from_dom
 
-            heading_text = resolve_heading_text(article_tag, heading, fallback_anchorless)
+            heading_text = resolve_heading_text(article_tag, heading)
             if heading_text && !heading_text.strip.empty?
               generate_slug(heading_text)
-            elsif fallback_anchorless
+            else
               generate_content_hash(article_tag)
             end
           end
@@ -32,22 +31,20 @@ module Html2rss
 
           def parse_id_from_dom(article_tag, url, selected_anchor)
             candidates = [article_tag['id'], article_tag.at_css('[id]')&.attr('id')]
-            candidates += [url&.path, url&.query] if selected_anchor
+            candidates += [url&.path] if selected_anchor
             candidates.compact.reject(&:empty?).first
           end
 
-          def resolve_heading_text(article_tag, heading, fallback_anchorless)
+          def resolve_heading_text(article_tag, heading)
             text = heading ? Navigator::TextExtractor.call(heading) : nil
             if text.nil? || text.strip.empty?
-              fallback_text_node_content(article_tag, fallback_anchorless)
+              fallback_text_node_content(article_tag)
             else
               text
             end
           end
 
-          def fallback_text_node_content(article_tag, fallback_anchorless)
-            return unless fallback_anchorless
-
+          def fallback_text_node_content(article_tag)
             article_tag.xpath('.//text()').find { |t| !t.text.strip.empty? }&.text&.strip
           end
 

data/lib/html2rss/html/article_extractor/image_extractor.rb

@@ -12,7 +12,8 @@ module Html2rss
         def self.call(article_tag, base_url:)
           img_src = from_source(article_tag) ||
                     from_img(article_tag) ||
-                    from_style(article_tag)
+                    from_style(article_tag) ||
+                    from_poster(article_tag)
 
           Url.from_relative(img_src, base_url) if img_src
         end
@@ -43,6 +44,13 @@ module Html2rss
           styles = article_tag.css('[style*="url"]').map { |tag| tag['style'] }
           ArticleRules::Image.best_from_styles(styles)
         end
+
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @return [String, nil] first usable video poster frame: the last resort, when the card has no other image
+        def self.from_poster(article_tag)
+          posters = article_tag.css('video[poster]').map { |video| video['poster'] }
+          posters.find { |src| ArticleRules::Image.remote_src?(src) }
+        end
       end
     end
   end