html2rss 0.22.2 → 0.23.0

data/lib/html2rss/feed_pipeline.rb

@@ -3,7 +3,10 @@
 module Html2rss
   ##
   # Builds feeds from validated config through request, extraction, and rendering stages.
-  class FeedPipeline # rubocop:disable Metrics/ClassLength
+  class FeedPipeline
+    # Bundle of inputs shared by selector and auto-source article collection.
+    ExtractionContext = Data.define(:config, :response, :request_session)
+
     ##
     # @param raw_config [Hash{Symbol => Object}] user-provided feed config
     def initialize(raw_config)
@@ -14,8 +17,8 @@ module Html2rss
     # @return [RSS::Rss] generated RSS feed
     def to_rss
       run do |response:, config:, articles:|
-        channel = RssBuilder::Channel.new(response, overrides: config.channel)
-        RssBuilder.new(channel:, articles:, stylesheets: config.stylesheets).call
+        channel = Html2rss::Channel.new(response, overrides: config.channel)
+        FeedBuilder.build(:rss, channel:, articles:, stylesheets: config.stylesheets)
       end
     end
 
@@ -23,8 +26,8 @@ module Html2rss
     # @return [Hash] generated JSONFeed 1.1 payload
     def to_json_feed
       run do |response:, config:, articles:|
-        channel = RssBuilder::Channel.new(response, overrides: config.channel)
-        JsonFeedBuilder.new(channel:, articles:).call
+        channel = Html2rss::Channel.new(response, overrides: config.channel)
+        FeedBuilder.build(:json_feed, channel:, articles:)
       end
     end
 
@@ -39,98 +42,82 @@ module Html2rss
     end
 
     def pipeline_state_for(config)
-      if config.strategy == :auto
-        run_auto_pipeline(config)
+      plan = StrategyPlan.resolve(config.strategy)
+      resources = RuntimePolicy.resources_for(config)
+      if plan.is_a?(StrategyPlan::Auto)
+        run_auto_pipeline(config, resources:)
       else
-        run_pipeline_for_strategy(config, strategy: config.strategy)
+        run_pipeline_for_strategy(config, strategy: plan.strategy, resources:)
       end
     end
 
-    def run_pipeline_for_strategy(config, strategy:, budget: nil)
-      request_session = request_session_for(config, strategy:, budget:)
+    def run_pipeline_for_strategy(config, strategy:, resources:)
+      request_session = request_session_for(config, strategy:, resources:)
       response = request_session.fetch_initial_response
-      articles = deduplicated_articles(response:, config:, request_session:)
+      articles = deduplicated_articles(
+        ExtractionContext.new(config:, response:, request_session:)
+      )
       { response:, articles: }
     end
 
-    def request_session_for(config, strategy:, budget: nil)
-      RequestSession.from_runtime_input(runtime_input_for(config, strategy:), budget:)
-    end
-
-    def runtime_input_for(config, strategy:)
-      RequestSession::RuntimeInput.new(
-        url: config.url,
-        headers: config.headers,
-        request: config.request,
+    def request_session_for(config, strategy:, resources:)
+      RequestSession.build(
+        config:,
         strategy:,
-        request_policy: RequestSession::RuntimePolicy.from_config(config)
+        budget: resources.budget,
+        policy: resources.policy
       )
     end
 
-    def deduplicated_articles(response:, config:, request_session:)
-      Articles::Deduplicator.new(
-        collect_articles(response:, config:, request_session:)
-      ).call
-    end
-
-    def run_auto_pipeline(config)
-      auto_fallback_for(config).call
+    def deduplicated_articles(extraction)
+      Article::Deduplicator.new(collect_articles(extraction)).call
     end
 
     # rubocop:disable Metrics/MethodLength
-    def auto_fallback_for(config)
+    def run_auto_pipeline(config, resources:)
       AutoFallback.new(
         strategies: AutoFallback::CHAIN,
-        budget: auto_pipeline_budget(config),
+        budget: resources.budget,
         session_for: lambda do |strategy:, budget:|
-          if budget.remaining_timeout_seconds && budget.remaining_timeout_seconds <= 0
-            raise RequestService::RequestTimedOut, 'Request timed out'
-          end
-
-          request_session_for(config, strategy:, budget:)
+          budget.effective_timeout_seconds(fallback: resources.policy.total_timeout_seconds)
+          request_session_for(config, strategy:, resources:)
         end,
         articles_for: lambda do |response:, request_session:|
-          deduplicated_articles(response:, config:, request_session:)
+          deduplicated_articles(ExtractionContext.new(config:, response:, request_session:))
         end
-      )
+      ).call
     end
     # rubocop:enable Metrics/MethodLength
 
-    def auto_pipeline_budget(config)
-      policy = RequestSession::RuntimePolicy.from_config(config)
-      RequestService::Budget.new(
-        max_requests: policy.max_requests,
-        total_timeout_seconds: policy.total_timeout_seconds
-      )
+    def collect_articles(extraction)
+      selector_articles(extraction) + auto_source_articles(extraction)
     end
 
-    def collect_articles(response:, config:, request_session:)
-      selector_articles(response:, config:, request_session:) +
-        auto_source_articles(response:, config:, request_session:)
-    end
-
-    def selector_articles(response:, config:, request_session:) # rubocop:disable Metrics/MethodLength
+    # rubocop:disable Metrics/MethodLength
+    def selector_articles(extraction)
+      config = extraction.config
       return [] unless (selectors = config.selectors)
 
-      page_responses = if (max_pages = selectors.dig(:items, :pagination, :max_pages))
-                         RequestSession::RelNextPager.new(
-                           session: request_session,
-                           initial_response: response,
-                           max_pages:
-                         ).to_a
-                       else
-                         [response]
-                       end
-
-      page_responses.flat_map do |page_response|
-        Selectors.new(page_response, selectors:, time_zone: config.time_zone).articles
+      page_responses = extraction.request_session.page_responses(
+        extraction.response,
+        pagination_config: selectors.dig(:items, :pagination)
+      )
+
+      articles = []
+      page_responses.each do |page_response|
+        page_articles = Selectors.new(page_response, selectors:, time_zone: config.time_zone).articles
+        break if page_articles.empty? && articles.any?
+
+        articles.concat(page_articles)
       end
+      articles
     end
+    # rubocop:enable Metrics/MethodLength
 
-    def auto_source_articles(response:, config:, request_session:)
-      return [] unless (auto_source = config.auto_source)
+    def auto_source_articles(extraction)
+      return [] unless (auto_source = extraction.config.auto_source)
 
-      AutoSource.new(response, auto_source, request_session:).articles
+      AutoSource.new(extraction.response, auto_source, request_session: extraction.request_session).articles
     end
   end
 end

data/lib/html2rss/html/article_extractor/category_extractor.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      ##
+      # CategoryExtractor is responsible for extracting categories from HTML elements
+      # by looking for CSS class names containing common category-related terms.
+      class CategoryExtractor
+        # Common category-related terms to look for in class names
+        CATEGORY_TERMS = %w[
+          category categories tag tags topic topics section sections
+          label labels theme themes subject subjects
+        ].freeze
+
+        # CSS selectors to find elements with category-related class names or data attributes
+        CATEGORY_SELECTORS = CATEGORY_TERMS.flat_map do |term|
+          ["[class*=\"#{term}\"]", "[data-#{term}]", "[#{term}]"]
+        end.freeze
+
+        # Regex pattern for matching category-related attribute names
+        CATEGORY_ATTR_PATTERN = /#{CATEGORY_TERMS.join('|')}/i
+
+        ##
+        # Extracts categories from the given article tag by looking for elements
+        # with class names containing common category-related terms.
+        #
+        # @param article_tag [Nokogiri::XML::Element] The article element to extract categories from
+        # @return [Array<String>] Array of category strings, empty if none found
+        def self.call(article_tag)
+          return [] unless article_tag
+
+          # Single optimized traversal that extracts all category types
+          extract_all_categories(article_tag)
+            .map(&:strip)
+            .reject(&:empty?)
+        end
+
+        ##
+        # Optimized single DOM traversal that extracts all category types.
+        #
+        # @param article_tag [Nokogiri::XML::Element] The article element
+        # @return [Set<String>] Set of category strings
+        def self.extract_all_categories(article_tag)
+          Set.new.tap do |categories|
+            article_tag.css(CATEGORY_SELECTORS.join(',')).each do |element|
+              # Extract text categories from elements with category-related class names
+              extract_text_categories!(categories, element) if element['class']&.match?(CATEGORY_ATTR_PATTERN)
+
+              # Extract data categories from all elements
+              extract_element_data_categories!(categories, element)
+            end
+          end
+        end
+
+        ##
+        # Extracts categories from data attributes of a single element.
+        #
+        # @param categories [Set<String>] Accumulator set
+        # @param element [Nokogiri::XML::Element] metadata element that may contain category links
+        # @return [void]
+        def self.extract_element_data_categories!(categories, element)
+          element.attributes.each_value do |attr|
+            next unless attr.name.match?(CATEGORY_ATTR_PATTERN)
+
+            value = attr.value&.strip
+            categories.add(value) if value && !value.empty?
+          end
+        end
+
+        ##
+        # Extracts text-based categories from elements, splitting content into discrete values.
+        #
+        # @param categories [Set<String>] Accumulator set
+        # @param element [Nokogiri::XML::Element] metadata element whose text may contain delimiters
+        # @return [void]
+        def self.extract_text_categories!(categories, element)
+          if element.name == 'a'
+            add_text_to_categories!(categories, element)
+            return
+          end
+
+          anchors = element.css('a')
+
+          if anchors.any?
+            anchors.each { |node| add_text_to_categories!(categories, node) }
+          else
+            extract_split_text_categories!(categories, element)
+          end
+        end
+
+        ##
+        # Adds the visible text of the given element to the categories set.
+        #
+        # @param categories [Set<String>] Accumulator set
+        # @param element [Nokogiri::XML::Element] The element to extract text from
+        # @return [void]
+        def self.add_text_to_categories!(categories, element)
+          text = Navigator.extract_visible_text(element)
+          categories.add(text) if text && !text.empty?
+        end
+
+        ##
+        # Extracts categories from the element's text by splitting on newlines.
+        #
+        # @param categories [Set<String>] Accumulator set
+        # @param element [Nokogiri::XML::Element] The element to extract text from
+        # @return [void]
+        def self.extract_split_text_categories!(categories, element)
+          text = element.text
+          return unless text
+
+          text.split(/\n+/).each do |line|
+            line = line.strip
+            categories.add(line) unless line.empty?
+          end
+        end
+
+        private_class_method :add_text_to_categories!, :extract_split_text_categories!
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_extractor/date_extractor.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      # Extracts the earliest date from an article_tag.
+      class DateExtractor
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @return [DateTime, nil]
+        def self.call(article_tag)
+          times = article_tag.css('[datetime]').filter_map do |tag|
+            DateTime.parse(tag['datetime'])
+          rescue ArgumentError, TypeError
+            nil
+          end
+
+          times.min
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_extractor/enclosure_extractor.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      ##
+      # Extracts enclosures from HTML tags using various strategies.
+      class EnclosureExtractor
+        # CSS union query covering images, media, PDFs, iframes, and archives.
+        SELECTOR = [
+          'img[src]:not([src^="data"])',
+          'video source[src]',
+          'audio source[src]',
+          'audio[src]',
+          'a[href$=".pdf"]',
+          'iframe[src]',
+          'a[href$=".zip"]',
+          'a[href$=".tar.gz"]',
+          'a[href$=".tgz"]'
+        ].join(',').freeze
+
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative enclosure links
+        # @return [Array<Hash{Symbol => Object}>] normalized enclosure hashes
+        def self.call(article_tag, base_url)
+          return [] unless article_tag
+
+          article_tag.css(SELECTOR).filter_map do |element|
+            extract_from_element(element, base_url)
+          end
+        end
+
+        def self.extract_from_element(element, base_url)
+          case element.name
+          when 'img'
+            extract_image(element, base_url)
+          when 'video', 'audio', 'source'
+            extract_media(element, base_url)
+          when 'iframe'
+            extract_iframe(element, base_url)
+          when 'a'
+            extract_a(element, base_url)
+          end
+        end
+
+        def self.extract_image(img, base_url)
+          src = img['src'].to_s
+          return if src.empty?
+
+          abs_url = Url.from_relative(src, base_url)
+          {
+            url: abs_url,
+            type: Article::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
+          }
+        end
+
+        def self.extract_media(element, base_url)
+          src = element['src'].to_s
+          return if src.empty?
+
+          {
+            url: Url.from_relative(src, base_url),
+            type: element['type']
+          }
+        end
+
+        def self.extract_iframe(iframe, base_url)
+          src = iframe['src'].to_s
+          return if src.empty?
+
+          abs_url = Url.from_relative(src, base_url)
+          {
+            url: abs_url,
+            type: Article::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
+          }
+        end
+
+        def self.extract_a(link, base_url)
+          href = link['href'].to_s
+          return if href.empty?
+
+          abs_url = Url.from_relative(href, base_url)
+
+          if href.end_with?('.pdf')
+            { url: abs_url, type: Article::Enclosure.guess_content_type_from_url(abs_url) }
+          else
+            { url: abs_url, type: 'application/zip' }
+          end
+        end
+
+        private_class_method :extract_from_element, :extract_image, :extract_media, :extract_iframe, :extract_a
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_extractor/heading_extractor.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      ##
+      # HeadingExtractor identifies and returns the best heading element within a container.
+      class HeadingExtractor
+        # Heading tags used to prioritize title extraction.
+        HEADING_TAGS = Navigator::HEADING_TAGS
+
+        class << self
+          ##
+          # @param article_tag [Nokogiri::XML::Element] container node
+          # @param fallback_anchorless [Boolean] whether to use fallback search
+          # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
+          # @return [Nokogiri::XML::Node, nil] the heading node, if found
+          def call(article_tag, fallback_anchorless:, selected_anchor:)
+            tags = article_tag.css(HEADING_TAGS.join(','))
+            if tags.any?
+              select_best_heading(tags)
+            elsif fallback_anchorless && selected_anchor.nil?
+              fallback_heading(article_tag)
+            end
+          end
+
+          private
+
+          def select_best_heading(tags)
+            min_tag_name = tags.map(&:name).min
+            best_tag = nil
+            max_size = -1
+
+            tags.each do |tag|
+              next if tag.name != min_tag_name
+
+              size = Navigator::TextExtractor.call(tag)&.size.to_i
+              (best_tag = tag) && (max_size = size) if size > max_size
+            end
+
+            best_tag
+          end
+
+          def fallback_heading(article_tag)
+            fallback_tags = article_tag.css(
+              'strong, b, [class*="title"], [class*="font-bold"], [class*="font-semibold"]'
+            )
+            fallback_tags.find { |t| !Navigator::TextExtractor.call(t).to_s.strip.empty? }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_extractor/id_generator.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      ##
+      # IdGenerator determines the unique ID for an article container node.
+      class IdGenerator
+        class << self
+          ##
+          # @param article_tag [Nokogiri::XML::Element] container node
+          # @param heading [Nokogiri::XML::Node, nil] heading node
+          # @param url [Html2rss::Url, nil] absolute article URL
+          # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
+          # @param fallback_anchorless [Boolean] whether to use fallback hashing
+          # @return [String, nil] the generated ID, if any
+          def call(article_tag, heading:, url:, selected_anchor:, fallback_anchorless:)
+            id_from_dom = parse_id_from_dom(article_tag, url, selected_anchor)
+            return id_from_dom if id_from_dom
+
+            heading_text = resolve_heading_text(article_tag, heading, fallback_anchorless)
+            if heading_text && !heading_text.strip.empty?
+              generate_slug(heading_text)
+            elsif fallback_anchorless
+              generate_content_hash(article_tag)
+            end
+          end
+
+          private
+
+          def parse_id_from_dom(article_tag, url, selected_anchor)
+            candidates = [article_tag['id'], article_tag.at_css('[id]')&.attr('id')]
+            candidates += [url&.path, url&.query] if selected_anchor
+            candidates.compact.reject(&:empty?).first
+          end
+
+          def resolve_heading_text(article_tag, heading, fallback_anchorless)
+            text = heading ? Navigator::TextExtractor.call(heading) : nil
+            if text.nil? || text.strip.empty?
+              fallback_text_node_content(article_tag, fallback_anchorless)
+            else
+              text
+            end
+          end
+
+          def fallback_text_node_content(article_tag, fallback_anchorless)
+            return unless fallback_anchorless
+
+            article_tag.xpath('.//text()').find { |t| !t.text.strip.empty? }&.text&.strip
+          end
+
+          def generate_slug(text)
+            slug = text.downcase.gsub(/[^a-z0-9]+/, '-')
+            slug = slug[1..] if slug.start_with?('-')
+            slug = slug[0..-2] if slug.end_with?('-')
+            slug unless slug.empty?
+          end
+
+          def generate_content_hash(article_tag)
+            text = Navigator::TextExtractor.call(article_tag).to_s.strip
+            Zlib.crc32(text).to_s(36) unless text.empty?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_extractor/image_extractor.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      ##
+      # Image is responsible for extracting image URLs the article_tag.
+      class ImageExtractor
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative image URLs
+        # @return [Html2rss::Url, nil] best candidate image URL
+        def self.call(article_tag, base_url:)
+          img_src = from_source(article_tag) ||
+                    from_img(article_tag) ||
+                    from_style(article_tag)
+
+          Url.from_relative(img_src, base_url) if img_src
+        end
+
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @return [String, nil] src attribute from first matching image tag
+        def self.from_img(article_tag)
+          article_tag.at_css('img[src]:not([src^="data"])')&.[]('src')
+        end
+
+        ##
+        # Extracts the largest image source from the srcset attribute
+        # of an img tag or a source tag inside a picture tag.
+        #
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @return [String, nil] largest srcset URL candidate
+        # @see <https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Responsive_images>
+        # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#srcset>
+        # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture>
+        def self.from_source(article_tag) # rubocop:disable Metrics/AbcSize
+          hash = article_tag.css('img[srcset], picture > source[srcset]').flat_map do |source|
+            source['srcset'].to_s.scan(/(\S+)\s+(\d+w|\d+h)[\s,]?/).map do |url, width|
+              next if url.nil? || url.start_with?('data:')
+
+              width_value = width.to_i.zero? ? 0 : width.scan(/\d+/).first.to_i
+
+              [width_value, url.strip]
+            end
+          end.compact.to_h
+
+          hash[hash.keys.max]
+        end
+
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @return [String, nil] best style-based background image URL
+        def self.from_style(article_tag)
+          article_tag.css('[style*="url"]')
+                     .filter_map { |tag| tag['style'][/url\(['"]?(.*?)['"]?\)/, 1] }
+                     .reject { |src| src.start_with?('data:') }
+                     .max_by(&:size)
+        end
+      end
+    end
+  end
+end