html2rss 0.22.2 → 0.23.0

data/lib/html2rss/feed_builder/json_feed/item.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedBuilder
+    class JsonFeed
+      ##
+      # Maps an {Html2rss::Article} to a JSONFeed 1.1 item hash.
+      class Item
+        ##
+        # @param article [Html2rss::Article]
+        def initialize(article)
+          @article = article
+        end
+
+        ##
+        # @return [Hash, nil] the JSONFeed-compliant item hash
+        def to_h
+          content = content_fields
+          return if content.empty?
+
+          item_payload.merge(content).compact
+        end
+
+        private
+
+        attr_reader :article
+
+        ##
+        # @return [Hash]
+        def item_payload
+          {
+            id: article.guid,
+            url: article.url&.to_s,
+            title: article.title,
+            image: article.image&.to_s,
+            date_published: article.published_at&.iso8601,
+            authors: author_array,
+            tags:,
+            attachments:
+          }
+        end
+
+        ##
+        # @return [Array<Hash>, nil]
+        def author_array
+          return unless (name = article.author)
+
+          [{ name: }]
+        end
+
+        ##
+        # JSON Feed items must include content_html or content_text.
+        # @return [Hash]
+        def content_fields
+          description = article.description
+          return { content_html: description } if description
+
+          title = article.title
+          return { content_text: title } if title
+
+          {}
+        end
+
+        ##
+        # @return [Array<String>, nil]
+        def tags
+          cats = article.categories
+          cats.empty? ? nil : cats
+        end
+
+        ##
+        # Maps enclosures to JSONFeed attachment objects.
+        # @return [Array<Hash>, nil]
+        def attachments
+          enclosures = article.enclosures
+          return nil if enclosures.empty?
+
+          enclosures.map { |enc| attachment_hash(enc) }
+        end
+
+        ##
+        # @param enclosure [Html2rss::Article::Enclosure]
+        # @return [Hash]
+        def attachment_hash(enclosure)
+          size = enclosure.bytes_length
+
+          {
+            url: enclosure.url.to_s,
+            mime_type: enclosure.type,
+            size_in_bytes: size&.positive? ? size : nil
+          }.compact
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/feed_builder/json_feed.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedBuilder
+    ##
+    # Builds a JSONFeed 1.1 hash from channel metadata and articles.
+    #
+    # @see https://www.jsonfeed.org/version/1.1/
+    class JsonFeed
+      # Official JSON Feed 1.1 schema version URL.
+      VERSION_URL = 'https://jsonfeed.org/version/1.1'
+
+      ##
+      # @param channel [Html2rss::Channel]
+      # @param articles [Array<Html2rss::Article>]
+      def initialize(channel:, articles:)
+        @channel = channel
+        @articles = articles
+      end
+
+      ##
+      # Builds and returns the JSONFeed hash.
+      #
+      # @return [Hash] the JSONFeed-compliant hash
+      def call
+        base_payload.merge(authors: author_array, items: item_hashes).compact
+      end
+
+      private
+
+      attr_reader :channel, :articles
+
+      ##
+      # @return [Hash]
+      def base_payload
+        {
+          version: VERSION_URL,
+          title: channel.title,
+          home_page_url: channel.url.to_s,
+          description: channel.description,
+          language: channel.language,
+          icon: channel.image&.to_s
+        }
+      end
+
+      ##
+      # @return [Array<Hash>]
+      def item_hashes
+        articles.filter_map { |article| Item.new(article).to_h }
+      end
+
+      ##
+      # @return [Array<Hash>, nil]
+      def author_array
+        return unless (name = channel.author)
+
+        [{ name: }]
+      end
+    end
+  end
+end

data/lib/html2rss/feed_builder/rss/stylesheet.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedBuilder
+    class Rss
+      ##
+      # Represents a stylesheet.
+      class Stylesheet
+        class << self
+          ##
+          # Adds the stylesheet XML tags to the RSS.
+          #
+          # @param maker [RSS::Maker::RSS20] RSS maker object.
+          # @param stylesheets [Array<Html2rss::FeedBuilder::Rss::Stylesheet>] Array of stylesheet configurations.
+          # @return [nil]
+          def add(maker, stylesheets)
+            stylesheets.each do |stylesheet|
+              add_stylesheet(maker, stylesheet)
+            end
+          end
+
+          private
+
+          ##
+          # Adds a single Stylesheet to the RSS.
+          #
+          # @param maker [RSS::Maker::RSS20] RSS maker object.
+          # @param stylesheet [Html2rss::FeedBuilder::Rss::Stylesheet] Stylesheet configuration.
+          # @return [nil]
+          def add_stylesheet(maker, stylesheet)
+            maker.xml_stylesheets.new_xml_stylesheet do |xss|
+              xss.href = stylesheet.href
+              xss.type = stylesheet.type
+              xss.media = stylesheet.media
+            end
+          end
+        end
+
+        # Allowed stylesheet MIME types for RSS processing instructions.
+        TYPES = ['text/css', 'text/xsl'].to_set.freeze
+
+        # @param href [String] stylesheet URL
+        # @param type [String] MIME type (`text/css` or `text/xsl`)
+        # @param media [String] media query hint for the stylesheet
+        def initialize(href:, type:, media: 'all')
+          raise ArgumentError, 'stylesheet.href must be a String' unless href.is_a?(String)
+          raise ArgumentError, 'stylesheet.type invalid' unless TYPES.include?(type)
+          raise ArgumentError, 'stylesheet.media must be a String' unless media.is_a?(String)
+
+          @href = href
+          @type = type
+          @media = media
+        end
+        attr_reader :href, :type, :media
+
+        # @return [String] the XML representation of the stylesheet
+        def to_xml
+          <<~XML
+            <?xml-stylesheet href="#{href}" type="#{type}" media="#{media}"?>
+          XML
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/feed_builder/rss.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'rss'
+
+module Html2rss
+  module FeedBuilder
+    ##
+    # Builds an RSS Feed by providing channel, articles and stylesheets.
+    class Rss
+      class << self
+        # @param article [Html2rss::Article] source article
+        # @param item_maker [RSS::Maker::RSS20::ItemsBase::ItemBase] RSS item builder
+        # @return [void]
+        def add_item(article, item_maker)
+          add_item_string_values(article, item_maker)
+          add_item_categories(article, item_maker)
+          add_enclosure(article.enclosure, item_maker)
+          add_item_guid(article, item_maker)
+        end
+
+        # @param enclosure [Html2rss::Article::Enclosure, nil] built enclosure object for the current RSS item
+        # @param maker [RSS::Maker::RSS20::ItemsBase::ItemBase] RSS item builder
+        # @return [void]
+        def add_enclosure(enclosure, maker)
+          return unless enclosure
+
+          maker.enclosure.tap do |enclosure_maker|
+            enclosure_maker.url = enclosure.url.to_s
+            enclosure_maker.type = enclosure.type
+            enclosure_maker.length = enclosure.bytes_length
+          end
+        end
+
+        private
+
+        def add_item_string_values(article, item_maker)
+          %i[title description author].each do |attr|
+            next unless (value = article.send(attr))
+            next if value.empty?
+
+            item_maker.send(:"#{attr}=", value)
+          end
+
+          item_maker.link = article.url.to_s if article.url
+          item_maker.pubDate = article.published_at&.rfc2822
+        end
+
+        def add_item_categories(article, item_maker)
+          article.categories.each { |category| item_maker.categories.new_category.content = category }
+        end
+
+        def add_item_guid(article, item_maker)
+          item_maker.guid.tap do |guid|
+            guid.content = article.guid
+            guid.isPermaLink = false
+          end
+        end
+      end
+
+      ##
+      # @param channel [Html2rss::Channel] The channel information for the RSS feed.
+      # @param articles [Array<Html2rss::Article>] The list of articles to include in the RSS feed.
+      # @param stylesheets [Array<Hash>] An optional array of stylesheet configurations.
+      def initialize(channel:, articles:, stylesheets: [])
+        @channel = channel
+        @articles = articles
+        @stylesheets = stylesheets
+      end
+
+      # @return [RSS::Rss] RSS 2.0 document instance
+      def call
+        RSS::Maker.make('2.0') do |maker|
+          Stylesheet.add(maker, stylesheets)
+
+          make_channel(maker.channel)
+          make_items(maker)
+        end
+      end
+
+      private
+
+      attr_reader :channel, :articles
+
+      def stylesheets
+        @stylesheets.map { |style| Stylesheet.new(**style) }
+      end
+
+      def make_channel(maker)
+        %i[language title description ttl].each do |key|
+          maker.public_send(:"#{key}=", channel.public_send(key))
+        end
+
+        maker.link = channel.url.to_s
+        maker.generator = generator
+        maker.updated = channel.last_build_date
+      end
+
+      def make_items(maker)
+        articles.each do |article|
+          maker.items.new_item { |item_maker| self.class.add_item(article, item_maker) }
+        end
+      end
+
+      def generator
+        scraper_namespace_regex = /(?<namespace>Html2rss|Scraper)::/
+
+        scraper_counts = articles.flat_map(&:scraper).tally.map do |klass, count|
+          scraper_name = klass.to_s.gsub(scraper_namespace_regex, '')
+          "#{scraper_name} (#{count})"
+        end
+
+        "html2rss V. #{Html2rss::VERSION} (scrapers: #{scraper_counts.join(', ')})"
+      end
+    end
+  end
+end

data/lib/html2rss/feed_builder.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Html2rss
+  # Entrypoint and namespace for feed building and formatting (RSS 2.0 / JSON Feed 1.1).
+  module FeedBuilder
+    # Builds the requested feed type from the channel and article list.
+    #
+    # @param type [Symbol] :rss or :json_feed
+    # @param channel [Html2rss::Channel]
+    # @param articles [Array<Html2rss::Article>]
+    # @return [RSS::Rss, Hash] format-compliant representation of the feed
+    def self.build(type, channel:, articles:, **)
+      case type
+      when :rss
+        Rss.new(channel:, articles:, **).call
+      when :json_feed
+        JsonFeed.new(channel:, articles:).call
+      else
+        raise ArgumentError, "Unknown feed type: #{type}"
+      end
+    end
+  end
+end

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -1,14 +1,12 @@
 # frozen_string_literal: true
 
-##
-# The Html2rss namespace.
 module Html2rss
-  ##
-  # Coordinates feed generation pipeline stages.
   class FeedPipeline
-    # Retries feed extraction across concrete request strategies for :auto mode.
+    # Retries feed extraction across concrete request strategies for the :auto plan.
+    #
+    # Owned by {FeedPipeline}; invoked only after {StrategyPlan} resolves +:auto+.
     class AutoFallback
-      # Ordered list of concrete request strategies attempted by auto mode.
+      # Ordered list of concrete request strategies attempted by the :auto plan.
       CHAIN = %i[faraday botasaurus browserless].freeze
 
       # Error classes that should abort auto fallback immediately.
@@ -24,6 +22,41 @@ module Html2rss
         RequestService::BrowserlessConfigurationError
       ].freeze
 
+      ##
+      # Mutable run state for one auto-fallback chain: attempts plus selected result.
+      class AttemptState
+        attr_reader :attempts, :result
+
+        def initialize
+          @attempts = []
+          @result = nil
+        end
+
+        # @return [Boolean] true when a strategy already yielded items
+        def succeeded? = !result.nil?
+
+        # @param strategy [Symbol] strategy that raised
+        # @param error [Exception] caught error
+        # @return [void]
+        def record_error(strategy:, error:)
+          @attempts << { strategy:, items_count: nil, error_class: error.class.name }
+        end
+
+        # @param strategy [Symbol] strategy that returned a response
+        # @param items_count [Integer] extracted article count
+        # @return [void]
+        def record_items(strategy:, items_count:)
+          @attempts << { strategy:, items_count:, error_class: nil }
+        end
+
+        # @param response [RequestService::Response] successful response
+        # @param articles [Array] extracted articles
+        # @return [void]
+        def succeed!(response:, articles:)
+          @result = { response:, articles: }
+        end
+      end
+
       ##
       # @param strategies [Array<Symbol>] ordered concrete strategies for fallback
       # @param budget [RequestService::Budget] shared request budget across retries
@@ -40,10 +73,10 @@ module Html2rss
       ##
       # @return [Hash{Symbol => Object}] pipeline state containing :response and :articles
       def call
-        state, attempts = run_attempts
-        return state if state
+        state = run_attempts
+        return state.result if state.succeeded?
 
-        finalize_failure(attempts:)
+        finalize_failure(attempts: state.attempts)
       end
 
       private
@@ -51,64 +84,48 @@ module Html2rss
       attr_reader :strategies, :budget, :session_for, :articles_for
 
       def run_attempts
-        state = { result: nil, attempts: [] }
-        strategies.each_with_index do |strategy, index|
-          run_attempt_for(strategy:, next_strategy: strategies[index + 1], state:)
-          break if state.fetch(:result)
+        AttemptState.new.tap do |state|
+          strategies.each_with_index do |strategy, index|
+            attempt(strategy:, next_strategy: strategies[index + 1], state:)
+            break if state.succeeded?
+          end
         end
-        [state.fetch(:result), state.fetch(:attempts)]
-      end
-
-      def run_attempt_for(strategy:, next_strategy:, state:)
-        result, attempts = attempt(
-          strategy:,
-          next_strategy:,
-          state: { attempts: state.fetch(:attempts) }
-        )
-        state[:result] = result
-        state[:attempts] = attempts
       end
 
       def attempt(strategy:, next_strategy:, state:)
         request_session = session_for.call(strategy:, budget:)
-        response, state = fetch_response(
-          request_session:,
-          strategy:,
-          next_strategy:,
-          state:
-        )
-        return [nil, state.fetch(:attempts)] unless response
+        response = fetch_response(request_session:, strategy:, next_strategy:, state:)
+        return unless response
 
         process_response(response:, strategy:, next_strategy:, request_session:, state:)
       end
 
       def fetch_response(request_session:, strategy:, next_strategy:, state:)
-        [request_session.fetch_initial_response, state]
+        request_session.fetch_initial_response
       rescue *NON_FALLBACK_ERRORS
         raise
       rescue StandardError => error
-        state[:attempts] << { strategy:, items_count: nil, error_class: error.class.name }
+        state.record_error(strategy:, error:)
         log_warn_fallback_error(strategy:, next_strategy:, error:) if next_strategy
         Log.debug("#{self.class}: strategy=#{strategy} error=#{error.class}: #{error.message}")
-        [nil, state]
+        nil
       end
 
       def process_response(response:, strategy:, next_strategy:, request_session:, state:)
         articles = articles_for.call(response:, request_session:)
         items_count = articles.size
-        state[:attempts] << { strategy:, items_count:, error_class: nil }
+        state.record_items(strategy:, items_count:)
         Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count}")
-        return success_state(response:, strategy:, articles:, state:) if items_count.positive?
+        return record_success(response:, strategy:, articles:, state:) if items_count.positive?
 
         log_info_fallback_zero_items(strategy:, next_strategy:) if next_strategy
-        [nil, state.fetch(:attempts)]
       end
 
-      def success_state(response:, strategy:, articles:, state:)
-        if state.fetch(:attempts).size > 1
-          Log.info("#{self.class}: auto selected strategy=#{strategy} after attempts=#{state.fetch(:attempts).size}")
-        end
-        [{ response:, articles: }, state.fetch(:attempts)]
+      def record_success(response:, strategy:, articles:, state:)
+        state.succeed!(response:, articles:)
+        return unless state.attempts.size > 1
+
+        Log.info("#{self.class}: auto selected strategy=#{strategy} after attempts=#{state.attempts.size}")
       end
 
       def finalize_failure(attempts:)

data/lib/html2rss/feed_pipeline/runtime_policy.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class FeedPipeline
+    ##
+    # Planner for the runtime request policy and budgets of a feed run.
+    #
+    # HTTP request slots (pagination, document GETs, strategy fallback) are planned
+    # separately from Browserless preload interaction budget so preload cannot steal
+    # pagination slots.
+    class RuntimePolicy
+      # Policy plus Budget from one config expansion (shared meters for a feed run).
+      Resources = Data.define(:policy, :budget)
+
+      ##
+      # @param config [Html2rss::Config] validated feed config
+      # @return [Html2rss::RequestService::Policy] request policy derived from runtime config
+      def self.from_config(config)
+        RequestService::Policy.new(
+          max_requests: effective_max_requests_for(config),
+          max_redirects: config.max_redirects,
+          total_timeout_seconds: config.total_timeout_seconds || RequestService::Policy::DEFAULTS[:total_timeout_seconds]
+        )
+      end
+
+      ##
+      # Builds policy and budget from one `from_config` so FeedPipeline does not dual-own expansion.
+      #
+      # @param config [Html2rss::Config] validated feed config
+      # @return [Resources] shared policy + budget for the feed build
+      def self.resources_for(config)
+        policy = from_config(config)
+        Resources.new(
+          policy:,
+          budget: RequestService::Budget.new(
+            max_requests: policy.max_requests,
+            max_interactions: interaction_budget_for(config),
+            total_timeout_seconds: policy.total_timeout_seconds
+          )
+        )
+      end
+
+      ##
+      # Builds a Budget with separate request and interaction slot pools.
+      #
+      # @param config [Html2rss::Config] validated feed config
+      # @return [Html2rss::RequestService::Budget] shared budget for the feed build
+      def self.budget_for(config) = resources_for(config).budget
+
+      ##
+      # @param config [Html2rss::Config] validated feed config
+      # @return [Integer] preload interaction slots derived from browserless preload config
+      def self.interaction_budget_for(config)
+        browserless_preload_budget_for(config)
+      end
+
+      class << self
+        private
+
+        def effective_max_requests_for(config)
+          return config.max_requests if config.explicit_max_requests?
+
+          [baseline_request_budget_for(config), config.max_requests].max
+        end
+
+        # Reserve enough HTTP request slots for the initial request plus predictable
+        # follow-ups. Preload interactions are planned separately.
+        def baseline_request_budget_for(config)
+          1 +
+            RequestSession::Pager.request_slots_for(config.selectors&.dig(:items, :pagination)) +
+            AutoSource.request_slots_for(config.auto_source) +
+            StrategyPlan.resolve(config.strategy).request_slots
+        end
+
+        def browserless_preload_budget_for(config)
+          preload = config.request.dig(:browserless, :preload)
+          return 0 unless preload
+
+          top_level_preload_wait_budget(preload) +
+            click_selector_preload_budget(preload) +
+            scroll_preload_budget(preload)
+        end
+
+        def top_level_preload_wait_budget(preload)
+          preload[:wait_after_ms] ? 2 : 0
+        end
+
+        def click_selector_preload_budget(preload)
+          preload.fetch(:click_selectors, []).sum { preload_action_budget(_1, :max_clicks) }
+        end
+
+        def scroll_preload_budget(preload)
+          scroll = preload[:scroll_down]
+          return 0 unless scroll
+
+          preload_action_budget(scroll, :iterations)
+        end
+
+        def preload_action_budget(config, count_key)
+          action_count = config.fetch(count_key, 1)
+          wait_budget = config[:wait_after_ms] ? action_count : 0
+
+          action_count + wait_budget
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/feed_pipeline/strategy_plan.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class FeedPipeline
+    ##
+    # Feed-level request plan: +:auto+ (fallback chain) or a concrete transport strategy.
+    #
+    # {RequestService} executes concrete adapters only. +:auto+ is resolved here before
+    # any session or adapter call — plan vs transport locality.
+    class StrategyPlan
+      # Feed-level auto plan: try {AutoFallback::CHAIN} until items are extracted.
+      Auto = Data.define do
+        # @return [Integer]
+        def request_slots
+          [AutoFallback::CHAIN.size - 1, 0].max
+        end
+      end
+
+      # Concrete transport strategy name for {RequestService}.
+      Concrete = Data.define(:strategy) do
+        # @return [Integer]
+        def request_slots = 0
+      end
+
+      # Symbol used in config / configure for the auto plan.
+      AUTO_NAME = :auto
+
+      class << self
+        ##
+        # @param name [Symbol, String] config or configure strategy value
+        # @return [Auto, Concrete] resolved feed-level plan
+        # @raise [ArgumentError] when +name+ is neither +:auto+ nor a registered strategy
+        def resolve(name)
+          normalized = name.to_sym
+          return Auto.new if normalized == AUTO_NAME
+
+          unless RequestService.strategy_registered?(normalized)
+            raise ArgumentError, "unknown strategy plan: #{name.inspect}"
+          end
+
+          Concrete.new(strategy: normalized)
+        end
+
+        ##
+        # @param name [Symbol, String, nil] candidate plan name
+        # @return [Boolean] whether +name+ is a valid feed-level strategy plan
+        def valid?(name)
+          return false unless name.is_a?(Symbol) || name.is_a?(String)
+
+          resolve(name)
+          true
+        rescue ArgumentError
+          false
+        end
+
+        ##
+        # @return [Array<Symbol>] accepted plan names (+:auto+ plus registered strategies)
+        def accepted_names
+          [AUTO_NAME, *RequestService.strategy_names.map(&:to_sym)].uniq
+        end
+      end
+    end
+  end
+end