html2rss 0.22.2 → 0.24.0

data/lib/html2rss/feed_pipeline/runtime_policy.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class FeedPipeline
+    ##
+    # Planner for the runtime request policy and budgets of a feed run.
+    #
+    # HTTP request slots (pagination, document GETs, strategy fallback) are planned
+    # separately from Browserless preload interaction budget so preload cannot steal
+    # pagination slots.
+    class RuntimePolicy
+      # Policy plus Budget from one config expansion (shared meters for a feed run).
+      Resources = Data.define(:policy, :budget)
+
+      ##
+      # @param config [Html2rss::Config] validated feed config
+      # @return [Html2rss::RequestService::Policy] request policy derived from runtime config
+      def self.from_config(config)
+        RequestService::Policy.new(
+          max_requests: effective_max_requests_for(config),
+          max_redirects: config.max_redirects,
+          total_timeout_seconds: config.total_timeout_seconds || RequestService::Policy::DEFAULTS[:total_timeout_seconds]
+        )
+      end
+
+      ##
+      # Builds policy and budget from one `from_config` so FeedPipeline does not dual-own expansion.
+      #
+      # @param config [Html2rss::Config] validated feed config
+      # @return [Resources] shared policy + budget for the feed build
+      def self.resources_for(config)
+        policy = from_config(config)
+        Resources.new(
+          policy:,
+          budget: RequestService::Budget.new(
+            max_requests: policy.max_requests,
+            max_interactions: interaction_budget_for(config),
+            total_timeout_seconds: policy.total_timeout_seconds
+          )
+        )
+      end
+
+      ##
+      # Builds a Budget with separate request and interaction slot pools.
+      #
+      # @param config [Html2rss::Config] validated feed config
+      # @return [Html2rss::RequestService::Budget] shared budget for the feed build
+      def self.budget_for(config) = resources_for(config).budget
+
+      ##
+      # @param config [Html2rss::Config] validated feed config
+      # @return [Integer] preload interaction slots derived from browserless preload config
+      def self.interaction_budget_for(config)
+        browserless_preload_budget_for(config)
+      end
+
+      class << self
+        private
+
+        def effective_max_requests_for(config)
+          return config.max_requests if config.explicit_max_requests?
+
+          [baseline_request_budget_for(config), config.max_requests].max
+        end
+
+        # Reserve enough HTTP request slots for the initial request plus predictable
+        # follow-ups. Preload interactions are planned separately.
+        def baseline_request_budget_for(config)
+          1 +
+            RequestSession::Pager.request_slots_for(config.selectors&.dig(:items, :pagination)) +
+            AutoSource.request_slots_for(config.auto_source) +
+            StrategyPlan.resolve(config.strategy).request_slots
+        end
+
+        def browserless_preload_budget_for(config)
+          preload = config.request.dig(:browserless, :preload)
+          return 0 unless preload
+
+          top_level_preload_wait_budget(preload) +
+            click_selector_preload_budget(preload) +
+            scroll_preload_budget(preload)
+        end
+
+        def top_level_preload_wait_budget(preload)
+          preload[:wait_after_ms] ? 2 : 0
+        end
+
+        def click_selector_preload_budget(preload)
+          preload.fetch(:click_selectors, []).sum { preload_action_budget(_1, :max_clicks) }
+        end
+
+        def scroll_preload_budget(preload)
+          scroll = preload[:scroll_down]
+          return 0 unless scroll
+
+          preload_action_budget(scroll, :iterations)
+        end
+
+        def preload_action_budget(config, count_key)
+          action_count = config.fetch(count_key, 1)
+          wait_budget = config[:wait_after_ms] ? action_count : 0
+
+          action_count + wait_budget
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/feed_pipeline/strategy_plan.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class FeedPipeline
+    ##
+    # Feed-level request plan: +:auto+ (fallback chain) or a concrete transport strategy.
+    #
+    # {RequestService} executes concrete adapters only. +:auto+ is resolved here before
+    # any session or adapter call — plan vs transport locality.
+    class StrategyPlan
+      # Feed-level auto plan: try {AutoFallback::CHAIN} until items are extracted.
+      Auto = Data.define do
+        # @return [Integer]
+        def request_slots
+          [AutoFallback::CHAIN.size - 1, 0].max
+        end
+      end
+
+      # Concrete transport strategy name for {RequestService}.
+      Concrete = Data.define(:strategy) do
+        # @return [Integer]
+        def request_slots = 0
+      end
+
+      # Symbol used in config / configure for the auto plan.
+      AUTO_NAME = :auto
+
+      class << self
+        ##
+        # @param name [Symbol, String] config or configure strategy value
+        # @return [Auto, Concrete] resolved feed-level plan
+        # @raise [ArgumentError] when +name+ is neither +:auto+ nor a registered strategy
+        def resolve(name)
+          normalized = name.to_sym
+          return Auto.new if normalized == AUTO_NAME
+
+          unless RequestService.strategy_registered?(normalized)
+            raise ArgumentError, "unknown strategy plan: #{name.inspect}"
+          end
+
+          Concrete.new(strategy: normalized)
+        end
+
+        ##
+        # @param name [Symbol, String, nil] candidate plan name
+        # @return [Boolean] whether +name+ is a valid feed-level strategy plan
+        def valid?(name)
+          return false unless name.is_a?(Symbol) || name.is_a?(String)
+
+          resolve(name)
+          true
+        rescue ArgumentError
+          false
+        end
+
+        ##
+        # @return [Array<Symbol>] accepted plan names (+:auto+ plus registered strategies)
+        def accepted_names
+          [AUTO_NAME, *RequestService.strategy_names.map(&:to_sym)].uniq
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/feed_pipeline.rb

@@ -3,7 +3,15 @@
 module Html2rss
   ##
   # Builds feeds from validated config through request, extraction, and rendering stages.
-  class FeedPipeline # rubocop:disable Metrics/ClassLength
+  class FeedPipeline
+    # Scrape-finished facts after request + extraction + dedup (before Channel/Status materialize).
+    # selected_strategy: set on :auto success; nil otherwise.
+    # attempt_count: auto attempts attempted; 0 outside :auto.
+    # strategy_attempts: auto attempt hashes (with optional transport_meta); empty outside :auto.
+    PipelineOutcome = Data.define(
+      :response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts
+    )
+
     ##
     # @param raw_config [Hash{Symbol => Object}] user-provided feed config
     def initialize(raw_config)
@@ -11,123 +19,109 @@ module Html2rss
     end
 
     ##
-    # @return [RSS::Rss] generated RSS feed
-    def to_rss
-      run do |response:, config:, articles:|
-        channel = RssBuilder::Channel.new(response, overrides: config.channel)
-        RssBuilder.new(channel:, articles:, stylesheets: config.stylesheets).call
-      end
+    # Runs the pipeline once and returns an opaque, Marshal-cacheable result.
+    #
+    # @return [Html2rss::FeedResult]
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- Status kwargs stay co-located with Channel
+    def to_result
+      config = Config.from_hash(raw_config, params: raw_config[:params])
+      outcome = pipeline_outcome_for(config)
+      channel = Channel.from_response(outcome.response, overrides: config.channel)
+      status = Status.build(
+        articles: outcome.articles,
+        dedup_dropped: outcome.dedup_dropped,
+        selected_strategy: outcome.selected_strategy,
+        attempt_count: outcome.attempt_count,
+        strategy_attempts: outcome.strategy_attempts
+      )
+      FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+    # @api private Host seam for {AutoFallback} (and single-strategy path).
+    # @param config [Html2rss::Config]
+    # @param strategy [Symbol]
+    # @param resources [Html2rss::FeedPipeline::RuntimePolicy::Resources]
+    # @return [Html2rss::RequestSession]
+    def request_session_for(config, strategy:, resources:)
+      RequestSession.build(
+        config:,
+        strategy:,
+        budget: resources.budget,
+        policy: resources.policy
+      )
     end
 
-    ##
-    # @return [Hash] generated JSONFeed 1.1 payload
-    def to_json_feed
-      run do |response:, config:, articles:|
-        channel = RssBuilder::Channel.new(response, overrides: config.channel)
-        JsonFeedBuilder.new(channel:, articles:).call
-      end
+    # @api private Host seam for {AutoFallback} (and single-strategy path).
+    # @param config [Html2rss::Config]
+    # @param response [Html2rss::RequestService::Response]
+    # @param request_session [Html2rss::RequestSession]
+    # @return [Array(Array<Html2rss::Article>, Integer)] unique articles and drop count
+    def deduplicated_articles(config:, response:, request_session:)
+      collected = collect_articles(config:, response:, request_session:)
+      unique = Article::Deduplicator.new(collected).call
+      [unique, collected.size - unique.size]
     end
 
     private
 
     attr_reader :raw_config
 
-    def run
-      config = Config.from_hash(raw_config, params: raw_config[:params])
-      state = pipeline_state_for(config)
-      yield response: state.fetch(:response), config:, articles: state.fetch(:articles)
-    end
-
-    def pipeline_state_for(config)
-      if config.strategy == :auto
-        run_auto_pipeline(config)
+    def pipeline_outcome_for(config)
+      plan = StrategyPlan.resolve(config.strategy)
+      resources = RuntimePolicy.resources_for(config)
+      if plan.is_a?(StrategyPlan::Auto)
+        run_auto_pipeline(config, resources:)
       else
-        run_pipeline_for_strategy(config, strategy: config.strategy)
+        run_pipeline_for_strategy(config, strategy: plan.strategy, resources:)
       end
     end
 
-    def run_pipeline_for_strategy(config, strategy:, budget: nil)
-      request_session = request_session_for(config, strategy:, budget:)
+    def run_pipeline_for_strategy(config, strategy:, resources:)
+      request_session = request_session_for(config, strategy:, resources:)
       response = request_session.fetch_initial_response
-      articles = deduplicated_articles(response:, config:, request_session:)
-      { response:, articles: }
-    end
-
-    def request_session_for(config, strategy:, budget: nil)
-      RequestSession.from_runtime_input(runtime_input_for(config, strategy:), budget:)
-    end
-
-    def runtime_input_for(config, strategy:)
-      RequestSession::RuntimeInput.new(
-        url: config.url,
-        headers: config.headers,
-        request: config.request,
-        strategy:,
-        request_policy: RequestSession::RuntimePolicy.from_config(config)
+      articles, dedup_dropped = deduplicated_articles(config:, response:, request_session:)
+      PipelineOutcome.new(
+        response:, articles:, dedup_dropped:, selected_strategy: nil, attempt_count: 0, strategy_attempts: []
       )
     end
 
-    def deduplicated_articles(response:, config:, request_session:)
-      Articles::Deduplicator.new(
-        collect_articles(response:, config:, request_session:)
+    def run_auto_pipeline(config, resources:)
+      AutoFallback.new(
+        strategies: AutoFallback::CHAIN,
+        budget: resources.budget,
+        pipeline: self,
+        config:,
+        resources:
       ).call
     end
 
-    def run_auto_pipeline(config)
-      auto_fallback_for(config).call
+    def collect_articles(config:, response:, request_session:)
+      selector_articles(config:, response:, request_session:) +
+        auto_source_articles(config:, response:, request_session:)
     end
 
     # rubocop:disable Metrics/MethodLength
-    def auto_fallback_for(config)
-      AutoFallback.new(
-        strategies: AutoFallback::CHAIN,
-        budget: auto_pipeline_budget(config),
-        session_for: lambda do |strategy:, budget:|
-          if budget.remaining_timeout_seconds && budget.remaining_timeout_seconds <= 0
-            raise RequestService::RequestTimedOut, 'Request timed out'
-          end
-
-          request_session_for(config, strategy:, budget:)
-        end,
-        articles_for: lambda do |response:, request_session:|
-          deduplicated_articles(response:, config:, request_session:)
-        end
-      )
-    end
-    # rubocop:enable Metrics/MethodLength
+    def selector_articles(config:, response:, request_session:)
+      return [] unless (selectors = config.selectors)
 
-    def auto_pipeline_budget(config)
-      policy = RequestSession::RuntimePolicy.from_config(config)
-      RequestService::Budget.new(
-        max_requests: policy.max_requests,
-        total_timeout_seconds: policy.total_timeout_seconds
+      page_responses = request_session.page_responses(
+        response,
+        pagination_config: selectors.dig(:items, :pagination)
       )
-    end
 
-    def collect_articles(response:, config:, request_session:)
-      selector_articles(response:, config:, request_session:) +
-        auto_source_articles(response:, config:, request_session:)
-    end
+      articles = []
+      page_responses.each do |page_response|
+        page_articles = Selectors.new(page_response, selectors:, time_zone: config.time_zone).articles
+        break if page_articles.empty? && articles.any?
 
-    def selector_articles(response:, config:, request_session:) # rubocop:disable Metrics/MethodLength
-      return [] unless (selectors = config.selectors)
-
-      page_responses = if (max_pages = selectors.dig(:items, :pagination, :max_pages))
-                         RequestSession::RelNextPager.new(
-                           session: request_session,
-                           initial_response: response,
-                           max_pages:
-                         ).to_a
-                       else
-                         [response]
-                       end
-
-      page_responses.flat_map do |page_response|
-        Selectors.new(page_response, selectors:, time_zone: config.time_zone).articles
+        articles.concat(page_articles)
       end
+      articles
     end
+    # rubocop:enable Metrics/MethodLength
 
-    def auto_source_articles(response:, config:, request_session:)
+    def auto_source_articles(config:, response:, request_session:)
       return [] unless (auto_source = config.auto_source)
 
       AutoSource.new(response, auto_source, request_session:).articles

data/lib/html2rss/feed_result.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Closed Marshal-cacheable handle for one scrape.
+  #
+  # Frozen consumer query/render set (do not grow without an explicit contract change):
+  # {#empty?}, {#channel_title}, {#to_rss}, {#to_json_feed}, and {#status}.
+  #
+  # Non-goals: no Channel reader, no Articles reader, no peephole into scrape internals.
+  # html2rss-web and other consumers must use only this surface.
+  #
+  # @note Gem contract: instances must round-trip through +Marshal.dump+ / +Marshal.load+
+  #   so web can cache one scrape and render RSS or JSON Feed on read. Loaded results
+  #   are re-frozen via private {#marshal_load}. For trusted cache reads only,
+  #   +Marshal.load(payload, freeze: true)+ is also safe.
+  class FeedResult
+    ##
+    # @param channel [Html2rss::Channel] channel metadata (internal; not exposed on the public API)
+    # @param articles [Array<Html2rss::Article>] extracted articles (deduplicated; not exposed)
+    # @param status [Html2rss::Status] pipeline telemetry
+    # @param stylesheets [Array<Hash>] optional RSS stylesheet configs
+    def initialize(channel:, articles:, status:, stylesheets: [])
+      raise ArgumentError, 'channel must be a Html2rss::Channel' unless channel.is_a?(Channel)
+      raise ArgumentError, 'status must be a Html2rss::Status' unless status.is_a?(Status)
+
+      articles = Array(articles)
+      raise ArgumentError, 'articles must all be Html2rss::Article' unless articles.all?(Article)
+
+      @channel = channel
+      @articles = articles.dup.freeze
+      @status = status
+      @stylesheets = freeze_stylesheets(stylesheets)
+      freeze
+    end
+
+    ##
+    # @return [Boolean] true when the scrape produced no items
+    def empty? = @articles.empty?
+
+    ##
+    # Channel title string only — does not expose the channel object.
+    #
+    # @return [String] title from the materialized channel
+    def channel_title = @channel.title
+
+    ##
+    # @return [RSS::Rss] RSS 2.0 document
+    def to_rss
+      FeedBuilder::Rss.new(
+        channel: @channel,
+        articles: @articles,
+        stylesheets: @stylesheets,
+        generator: status.to_generator_comment
+      ).call
+    end
+
+    ##
+    # @param feed_url [String, nil] optional self URL for JSON Feed (+feed_url+)
+    # @return [Hash] JSON Feed 1.1 hash
+    def to_json_feed(feed_url: nil)
+      FeedBuilder::JsonFeed.new(
+        channel: @channel,
+        articles: @articles,
+        feed_url:,
+        user_comment: status.to_generator_comment
+      ).call
+    end
+
+    ##
+    # @return [Html2rss::Status] frozen scraper tallies, dedup count, and generator formatter.
+    #   {#to_h} on +status+ is a stable consumer contract for observability payloads.
+    attr_reader :status
+
+    private
+
+    # Constructor payload for Marshal (avoids default ivar thaw). Rebuilds via {#initialize}.
+    def marshal_dump = [@channel, @articles, @status, @stylesheets]
+
+    def marshal_load((channel, articles, status, stylesheets))
+      initialize(channel:, articles:, status:, stylesheets:)
+    end
+
+    def freeze_stylesheets(stylesheets)
+      Array(stylesheets).map do |sheet|
+        sheet.to_h.transform_keys(&:to_sym).transform_values do |value|
+          value.is_a?(String) ? value.dup.freeze : value
+        end.freeze
+      end.freeze
+    end
+  end
+end

data/lib/html2rss/html/article_extractor/category_extractor.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      ##
+      # CategoryExtractor is responsible for extracting categories from HTML elements
+      # by looking for CSS class names containing common category-related terms.
+      class CategoryExtractor
+        # Common category-related terms to look for in class names
+        CATEGORY_TERMS = %w[
+          category categories tag tags topic topics section sections
+          label labels theme themes subject subjects
+        ].freeze
+
+        # CSS selectors to find elements with category-related class names or data attributes
+        CATEGORY_SELECTORS = CATEGORY_TERMS.flat_map do |term|
+          ["[class*=\"#{term}\"]", "[data-#{term}]", "[#{term}]"]
+        end.freeze
+
+        # Regex pattern for matching category-related attribute names
+        CATEGORY_ATTR_PATTERN = /#{CATEGORY_TERMS.join('|')}/i
+
+        ##
+        # Extracts categories from the given article tag by looking for elements
+        # with class names containing common category-related terms.
+        #
+        # @param article_tag [Nokogiri::XML::Element] The article element to extract categories from
+        # @return [Array<String>] Array of category strings, empty if none found
+        def self.call(article_tag)
+          return [] unless article_tag
+
+          # Single optimized traversal that extracts all category types
+          extract_all_categories(article_tag)
+            .map(&:strip)
+            .reject(&:empty?)
+        end
+
+        ##
+        # Optimized single DOM traversal that extracts all category types.
+        #
+        # @param article_tag [Nokogiri::XML::Element] The article element
+        # @return [Set<String>] Set of category strings
+        def self.extract_all_categories(article_tag)
+          Set.new.tap do |categories|
+            article_tag.css(CATEGORY_SELECTORS.join(',')).each do |element|
+              # Extract text categories from elements with category-related class names
+              extract_text_categories!(categories, element) if element['class']&.match?(CATEGORY_ATTR_PATTERN)
+
+              # Extract data categories from all elements
+              extract_element_data_categories!(categories, element)
+            end
+          end
+        end
+
+        ##
+        # Extracts categories from data attributes of a single element.
+        #
+        # @param categories [Set<String>] Accumulator set
+        # @param element [Nokogiri::XML::Element] metadata element that may contain category links
+        # @return [void]
+        def self.extract_element_data_categories!(categories, element)
+          element.attributes.each_value do |attr|
+            next unless attr.name.match?(CATEGORY_ATTR_PATTERN)
+
+            value = attr.value&.strip
+            categories.add(value) if value && !value.empty?
+          end
+        end
+
+        ##
+        # Extracts text-based categories from elements, splitting content into discrete values.
+        #
+        # @param categories [Set<String>] Accumulator set
+        # @param element [Nokogiri::XML::Element] metadata element whose text may contain delimiters
+        # @return [void]
+        def self.extract_text_categories!(categories, element)
+          if element.name == 'a'
+            add_text_to_categories!(categories, element)
+            return
+          end
+
+          anchors = element.css('a')
+
+          if anchors.any?
+            anchors.each { |node| add_text_to_categories!(categories, node) }
+          else
+            extract_split_text_categories!(categories, element)
+          end
+        end
+
+        ##
+        # Adds the visible text of the given element to the categories set.
+        #
+        # @param categories [Set<String>] Accumulator set
+        # @param element [Nokogiri::XML::Element] The element to extract text from
+        # @return [void]
+        def self.add_text_to_categories!(categories, element)
+          text = Navigator.extract_visible_text(element)
+          categories.add(text) if text && !text.empty?
+        end
+
+        ##
+        # Extracts categories from the element's text by splitting on newlines.
+        #
+        # @param categories [Set<String>] Accumulator set
+        # @param element [Nokogiri::XML::Element] The element to extract text from
+        # @return [void]
+        def self.extract_split_text_categories!(categories, element)
+          text = element.text
+          return unless text
+
+          text.split(/\n+/).each do |line|
+            line = line.strip
+            categories.add(line) unless line.empty?
+          end
+        end
+
+        private_class_method :add_text_to_categories!, :extract_split_text_categories!
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_extractor/date_extractor.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      # Extracts the earliest date from an article_tag.
+      class DateExtractor
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @return [DateTime, nil]
+        def self.call(article_tag)
+          times = article_tag.css('[datetime]').filter_map do |tag|
+            DateTime.parse(tag['datetime'])
+          rescue ArgumentError, TypeError
+            nil
+          end
+
+          times.min
+        end
+      end
+    end
+  end
+end