html2rss 0.22.2 → 0.24.0

data/lib/html2rss/feed_builder/json_feed/item.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedBuilder
+    class JsonFeed
+      ##
+      # Maps an {Html2rss::Article} to a JSONFeed 1.1 item hash.
+      class Item
+        ##
+        # @param article [Html2rss::Article]
+        def initialize(article)
+          @article = article
+        end
+
+        ##
+        # @return [Hash, nil] the JSONFeed-compliant item hash
+        def to_h
+          content = content_fields
+          return if content.empty?
+
+          item_payload.merge(content).compact
+        end
+
+        private
+
+        attr_reader :article
+
+        ##
+        # @return [Hash]
+        def item_payload
+          {
+            id: article.guid,
+            url: article.url&.to_s,
+            title: article.title,
+            image: article.image&.to_s,
+            date_published: article.published_at&.iso8601,
+            authors: author_array,
+            tags:,
+            attachments:
+          }
+        end
+
+        ##
+        # @return [Array<Hash>, nil]
+        def author_array
+          return unless (name = article.author)
+
+          [{ name: }]
+        end
+
+        ##
+        # JSON Feed items must include content_html or content_text.
+        # Title is a fallback only when description is absent.
+        #
+        # @return [Hash]
+        def content_fields
+          if (description = ItemPresentation.description_for(article))
+            return content_for(description)
+          end
+
+          return { content_text: article.title } if article.title
+
+          {}
+        end
+
+        ##
+        # @param description [String]
+        # @return [Hash]
+        def content_for(description)
+          html?(description) ? { content_html: description } : { content_text: description }
+        end
+
+        ##
+        # @param value [String]
+        # @return [Boolean]
+        def html?(value)
+          Nokogiri::HTML.fragment(value).children.any?(&:element?)
+        end
+
+        ##
+        # @return [Array<String>, nil]
+        def tags
+          cats = article.categories
+          cats.empty? ? nil : cats
+        end
+
+        ##
+        # Maps enclosures to JSONFeed attachment objects.
+        # @return [Array<Hash>, nil]
+        def attachments
+          enclosures = article.enclosures
+          return nil if enclosures.empty?
+
+          enclosures.map { |enc| attachment_hash(enc) }
+        end
+
+        ##
+        # @param enclosure [Html2rss::Article::Enclosure]
+        # @return [Hash]
+        def attachment_hash(enclosure)
+          size = enclosure.bytes_length
+
+          {
+            url: enclosure.url.to_s,
+            mime_type: enclosure.type,
+            size_in_bytes: size&.positive? ? size : nil
+          }.compact
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/feed_builder/json_feed.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedBuilder
+    ##
+    # Builds a JSONFeed 1.1 hash from channel metadata and articles.
+    #
+    # @see https://www.jsonfeed.org/version/1.1/
+    class JsonFeed
+      # Official JSON Feed 1.1 schema version URL.
+      VERSION_URL = 'https://jsonfeed.org/version/1.1'
+
+      ##
+      # @param channel [Html2rss::Channel]
+      # @param articles [Array<Html2rss::Article>]
+      # @param user_comment [String] required generator comment (from {Status})
+      # @param feed_url [String, nil] optional absolute self URL for the feed (JSON Feed feed_url)
+      def initialize(channel:, articles:, user_comment:, feed_url: nil)
+        raise ArgumentError, 'user_comment must be a non-blank String' if blank_string?(user_comment)
+
+        @channel = channel
+        @articles = articles
+        @feed_url = normalize_feed_url(feed_url)
+        @user_comment = user_comment
+      end
+
+      ##
+      # Builds and returns the JSONFeed hash.
+      #
+      # @return [Hash] the JSONFeed-compliant hash
+      def call
+        base_payload.merge(authors: author_array, items: item_hashes).compact
+      end
+
+      private
+
+      attr_reader :channel, :articles, :feed_url, :user_comment
+
+      def blank_string?(value)
+        !value.is_a?(String) || value.strip.empty?
+      end
+
+      def normalize_feed_url(value)
+        return if value.nil?
+        raise ArgumentError, 'feed_url must be a String or nil' unless value.is_a?(String)
+        return if value.strip.empty?
+
+        Url.from_absolute(value).to_s
+      end
+
+      ##
+      # @return [Hash]
+      def base_payload
+        {
+          version: VERSION_URL,
+          title: channel.title,
+          home_page_url: channel.url.to_s,
+          feed_url:,
+          description: channel.description,
+          user_comment:,
+          language: channel.language,
+          icon: channel.image&.to_s
+        }
+      end
+
+      ##
+      # @return [Array<Hash>]
+      def item_hashes
+        articles.filter_map { |article| Item.new(article).to_h }
+      end
+
+      ##
+      # @return [Array<Hash>, nil]
+      def author_array
+        return unless (name = channel.author)
+
+        [{ name: }]
+      end
+    end
+  end
+end

data/lib/html2rss/feed_builder/rss/stylesheet.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedBuilder
+    class Rss
+      ##
+      # Represents a stylesheet.
+      class Stylesheet
+        class << self
+          ##
+          # Adds the stylesheet XML tags to the RSS.
+          #
+          # @param maker [RSS::Maker::RSS20] RSS maker object.
+          # @param stylesheets [Array<Html2rss::FeedBuilder::Rss::Stylesheet>] Array of stylesheet configurations.
+          # @return [nil]
+          def add(maker, stylesheets)
+            stylesheets.each do |stylesheet|
+              add_stylesheet(maker, stylesheet)
+            end
+          end
+
+          private
+
+          ##
+          # Adds a single Stylesheet to the RSS.
+          #
+          # @param maker [RSS::Maker::RSS20] RSS maker object.
+          # @param stylesheet [Html2rss::FeedBuilder::Rss::Stylesheet] Stylesheet configuration.
+          # @return [nil]
+          def add_stylesheet(maker, stylesheet)
+            maker.xml_stylesheets.new_xml_stylesheet do |xss|
+              xss.href = stylesheet.href
+              xss.type = stylesheet.type
+              xss.media = stylesheet.media
+            end
+          end
+        end
+
+        # Allowed stylesheet MIME types for RSS processing instructions.
+        TYPES = ['text/css', 'text/xsl'].to_set.freeze
+
+        # @param href [String] stylesheet URL
+        # @param type [String] MIME type (`text/css` or `text/xsl`)
+        # @param media [String] media query hint for the stylesheet
+        def initialize(href:, type:, media: 'all')
+          raise ArgumentError, 'stylesheet.href must be a String' unless href.is_a?(String)
+          raise ArgumentError, 'stylesheet.type invalid' unless TYPES.include?(type)
+          raise ArgumentError, 'stylesheet.media must be a String' unless media.is_a?(String)
+
+          @href = href
+          @type = type
+          @media = media
+        end
+        attr_reader :href, :type, :media
+
+        # @return [String] the XML representation of the stylesheet
+        def to_xml
+          <<~XML
+            <?xml-stylesheet href="#{href}" type="#{type}" media="#{media}"?>
+          XML
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/feed_builder/rss.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'rss'
+
+module Html2rss
+  module FeedBuilder
+    ##
+    # Builds an RSS Feed by providing channel, articles and stylesheets.
+    class Rss
+      class << self
+        # @param article [Html2rss::Article] source article
+        # @param item_maker [RSS::Maker::RSS20::ItemsBase::ItemBase] RSS item builder
+        # @return [void]
+        def add_item(article, item_maker)
+          add_item_string_values(article, item_maker)
+          add_item_categories(article, item_maker)
+          add_enclosure(ItemPresentation.rss_enclosure_for(article), item_maker)
+          add_item_guid(article, item_maker)
+        end
+
+        # @param enclosure [Html2rss::Article::Enclosure, nil] built enclosure object for the current RSS item
+        # @param maker [RSS::Maker::RSS20::ItemsBase::ItemBase] RSS item builder
+        # @return [void]
+        def add_enclosure(enclosure, maker)
+          return unless enclosure
+
+          maker.enclosure.tap do |enclosure_maker|
+            enclosure_maker.url = enclosure.url.to_s
+            enclosure_maker.type = enclosure.type
+            enclosure_maker.length = enclosure.bytes_length
+          end
+        end
+
+        private
+
+        def add_item_string_values(article, item_maker)
+          {
+            title: article.title,
+            description: ItemPresentation.description_for(article),
+            author: article.author
+          }.each do |attr, value|
+            next if value.nil? || value.empty?
+
+            item_maker.public_send(:"#{attr}=", value)
+          end
+
+          item_maker.link = article.url.to_s if article.url
+          item_maker.pubDate = article.published_at&.rfc2822
+        end
+
+        def add_item_categories(article, item_maker)
+          article.categories.each { |category| item_maker.categories.new_category.content = category }
+        end
+
+        def add_item_guid(article, item_maker)
+          item_maker.guid.tap do |guid|
+            guid.content = article.guid
+            guid.isPermaLink = false
+          end
+        end
+      end
+
+      ##
+      # @param channel [Html2rss::Channel] The channel information for the RSS feed.
+      # @param articles [Array<Html2rss::Article>] The list of articles to include in the RSS feed.
+      # @param stylesheets [Array<Hash>] An optional array of stylesheet configurations.
+      # @param generator [String] required preformatted generator comment (from {Status})
+      def initialize(channel:, articles:, generator:, stylesheets: [])
+        raise ArgumentError, 'generator must be a non-blank String' if blank_string?(generator)
+
+        @channel = channel
+        @articles = articles
+        @stylesheets = stylesheets
+        @generator = generator
+      end
+
+      # @return [RSS::Rss] RSS 2.0 document instance
+      def call
+        RSS::Maker.make('2.0') do |maker|
+          Stylesheet.add(maker, stylesheets)
+
+          make_channel(maker.channel)
+          make_items(maker)
+        end
+      end
+
+      private
+
+      attr_reader :channel, :articles, :generator
+
+      def blank_string?(value)
+        !value.is_a?(String) || value.strip.empty?
+      end
+
+      def stylesheets
+        @stylesheets.map { |style| Stylesheet.new(**style) }
+      end
+
+      def make_channel(maker)
+        %i[language title description ttl].each do |key|
+          maker.public_send(:"#{key}=", channel.public_send(key))
+        end
+
+        maker.link = channel.url.to_s
+        maker.generator = generator
+        maker.updated = channel.last_build_date
+      end
+
+      def make_items(maker)
+        articles.each do |article|
+          maker.items.new_item { |item_maker| self.class.add_item(article, item_maker) }
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/feed_builder.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Html2rss
+  # Namespace for feed format adapters (RSS 2.0 / JSON Feed 1.1).
+  #
+  # {FeedResult} constructs {Rss} / {JsonFeed} directly — there is no dispatcher
+  # entrypoint on this module. Stylesheets are a scrape-time artifact carried on
+  # FeedResult; +feed_url+ is a render-time JSON Feed-only option.
+  #
+  # Channel field projection:
+  # - Rss: language, title, description, ttl, link, updated
+  # - JsonFeed: title, home_page_url, description, language, icon, authors (+ feed_url)
+  module FeedBuilder
+  end
+end

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -1,15 +1,14 @@
 # frozen_string_literal: true
 
-##
-# The Html2rss namespace.
 module Html2rss
-  ##
-  # Coordinates feed generation pipeline stages.
   class FeedPipeline
-    # Retries feed extraction across concrete request strategies for :auto mode.
+    # Retries feed extraction across concrete request strategies for the :auto plan.
+    #
+    # Owned by {FeedPipeline}; invoked only after {StrategyPlan} resolves +:auto+.
+    # Hosted by the pipeline instance: session + extract call back into FeedPipeline.
     class AutoFallback
-      # Ordered list of concrete request strategies attempted by auto mode.
-      CHAIN = %i[faraday botasaurus browserless].freeze
+      # Ordered list of concrete request strategies attempted by the :auto plan.
+      CHAIN = %i[faraday botasaurus].freeze
 
       # Error classes that should abort auto fallback immediately.
       NON_FALLBACK_ERRORS = [
@@ -24,103 +23,164 @@ module Html2rss
         RequestService::BrowserlessConfigurationError
       ].freeze
 
+      ##
+      # Mutable run state for one auto-fallback chain: attempts plus selected result.
+      class AttemptState
+        attr_reader :attempts, :result
+
+        def initialize
+          @attempts = []
+          @result = nil
+        end
+
+        # @return [Boolean] true when a strategy already yielded items
+        def succeeded? = !result.nil?
+
+        # @param strategy [Symbol] strategy that raised
+        # @param error [Exception] caught error
+        # @return [void]
+        def record_error(strategy:, error:)
+          @attempts << { strategy:, items_count: nil, error_class: error.class.name }
+        end
+
+        # @param strategy [Symbol] strategy that returned a response
+        # @param items_count [Integer] extracted article count
+        # @param transport_meta [Hash, nil] optional allowlisted upstream telemetry
+        # @return [void]
+        def record_items(strategy:, items_count:, transport_meta: nil)
+          attempt = { strategy:, items_count:, error_class: nil }
+          attempt[:transport_meta] = transport_meta if transport_meta && !transport_meta.empty?
+          @attempts << attempt
+        end
+
+        # @param response [RequestService::Response] successful response
+        # @param articles [Array] extracted articles
+        # @param dedup_dropped [Integer] articles removed by deduplication
+        # @param selected_strategy [Symbol] concrete strategy that produced items
+        # @param attempt_count [Integer] number of attempts recorded for this chain
+        # @return [void]
+        def succeed!(response:, articles:, dedup_dropped:, selected_strategy:, attempt_count:)
+          @result = PipelineOutcome.new(
+            response:,
+            articles:,
+            dedup_dropped:,
+            selected_strategy:,
+            attempt_count:,
+            strategy_attempts: attempts
+          )
+        end
+      end
+
       ##
       # @param strategies [Array<Symbol>] ordered concrete strategies for fallback
       # @param budget [RequestService::Budget] shared request budget across retries
-      # @param session_for [Proc] request session factory proc
-      # @param articles_for [Proc] article extraction proc
+      # @param pipeline [Html2rss::FeedPipeline] host for session + article extraction
+      # @param config [Html2rss::Config] validated feed config
+      # @param resources [Html2rss::FeedPipeline::RuntimePolicy::Resources] budget + policy
       # @return [void]
-      def initialize(strategies:, budget:, session_for:, articles_for:)
+      def initialize(strategies:, budget:, pipeline:, config:, resources:)
         @strategies = strategies
         @budget = budget
-        @session_for = session_for
-        @articles_for = articles_for
+        @pipeline = pipeline
+        @config = config
+        @resources = resources
       end
 
       ##
-      # @return [Hash{Symbol => Object}] pipeline state containing :response and :articles
+      # @return [Html2rss::FeedPipeline::PipelineOutcome] scrape-finished state for materialization
       def call
-        state, attempts = run_attempts
-        return state if state
+        state = run_attempts
+        return state.result if state.succeeded?
 
-        finalize_failure(attempts:)
+        finalize_failure(attempts: state.attempts)
       end
 
       private
 
-      attr_reader :strategies, :budget, :session_for, :articles_for
+      attr_reader :strategies, :budget, :pipeline, :config, :resources
 
       def run_attempts
-        state = { result: nil, attempts: [] }
-        strategies.each_with_index do |strategy, index|
-          run_attempt_for(strategy:, next_strategy: strategies[index + 1], state:)
-          break if state.fetch(:result)
+        AttemptState.new.tap do |state|
+          strategies.each_with_index do |strategy, index|
+            attempt(strategy:, next_strategy: strategies[index + 1], state:)
+            break if state.succeeded?
+          end
         end
-        [state.fetch(:result), state.fetch(:attempts)]
-      end
-
-      def run_attempt_for(strategy:, next_strategy:, state:)
-        result, attempts = attempt(
-          strategy:,
-          next_strategy:,
-          state: { attempts: state.fetch(:attempts) }
-        )
-        state[:result] = result
-        state[:attempts] = attempts
       end
 
       def attempt(strategy:, next_strategy:, state:)
-        request_session = session_for.call(strategy:, budget:)
-        response, state = fetch_response(
-          request_session:,
-          strategy:,
-          next_strategy:,
-          state:
-        )
-        return [nil, state.fetch(:attempts)] unless response
+        request_session = pipeline.request_session_for(config, strategy:, resources:)
+        response = fetch_response(request_session:, strategy:, next_strategy:, state:)
+        return unless response
 
         process_response(response:, strategy:, next_strategy:, request_session:, state:)
       end
 
       def fetch_response(request_session:, strategy:, next_strategy:, state:)
-        [request_session.fetch_initial_response, state]
+        request_session.fetch_initial_response
       rescue *NON_FALLBACK_ERRORS
         raise
       rescue StandardError => error
-        state[:attempts] << { strategy:, items_count: nil, error_class: error.class.name }
-        log_warn_fallback_error(strategy:, next_strategy:, error:) if next_strategy
-        Log.debug("#{self.class}: strategy=#{strategy} error=#{error.class}: #{error.message}")
-        [nil, state]
+        state.record_error(strategy:, error:)
+        log_fallback_error(strategy:, next_strategy:, error:, request_session:) if next_strategy
+        nil
       end
 
       def process_response(response:, strategy:, next_strategy:, request_session:, state:)
-        articles = articles_for.call(response:, request_session:)
+        articles, dedup_dropped = articles_for(response:, request_session:)
         items_count = articles.size
-        state[:attempts] << { strategy:, items_count:, error_class: nil }
-        Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count}")
-        return success_state(response:, strategy:, articles:, state:) if items_count.positive?
+        state.record_items(strategy:, items_count:, transport_meta: response.transport_meta)
+        Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count} " \
+                  "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
+                  "budget_remaining=#{budget_remaining_label}")
+        return record_success(response:, strategy:, articles:, dedup_dropped:, state:) if items_count.positive?
 
-        log_info_fallback_zero_items(strategy:, next_strategy:) if next_strategy
-        [nil, state.fetch(:attempts)]
+        log_info_fallback_zero_items(strategy:, next_strategy:, response:) if next_strategy
       end
 
-      def success_state(response:, strategy:, articles:, state:)
-        if state.fetch(:attempts).size > 1
-          Log.info("#{self.class}: auto selected strategy=#{strategy} after attempts=#{state.fetch(:attempts).size}")
-        end
-        [{ response:, articles: }, state.fetch(:attempts)]
+      def articles_for(response:, request_session:)
+        pipeline.deduplicated_articles(config:, response:, request_session:)
+      end
+
+      def record_success(response:, strategy:, articles:, dedup_dropped:, state:)
+        attempt_count = state.attempts.size
+        state.succeed!(response:, articles:, dedup_dropped:, selected_strategy: strategy, attempt_count:)
+        return unless attempt_count > 1
+
+        Log.info("#{self.class}: auto selected strategy=#{strategy} after attempts=#{attempt_count} " \
+                 "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
+                 "budget_remaining=#{budget_remaining_label}")
       end
 
       def finalize_failure(attempts:)
         raise NoFeedItemsExtracted.new(attempts:)
       end
 
-      def log_warn_fallback_error(strategy:, next_strategy:, error:)
-        Log.warn("#{self.class}: auto fallback #{strategy} -> #{next_strategy} after error=#{error.class}")
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def log_fallback_error(strategy:, next_strategy:, error:, request_session:)
+        host = request_session.url.host
+        detail = "host=#{host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
+                 "budget_remaining=#{budget_remaining_label}"
+        if error.is_a?(RequestService::RequestTimedOut)
+          Log.info("#{self.class}: auto fallback #{strategy} -> #{next_strategy} " \
+                   "after timeout=#{error.class} #{detail}")
+        else
+          Log.warn("#{self.class}: auto fallback #{strategy} -> #{next_strategy} " \
+                   "after error=#{error.class} #{detail}")
+        end
+        Log.debug("#{self.class}: strategy=#{strategy} error=#{error.class}: #{error.message} #{detail}")
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+      def log_info_fallback_zero_items(strategy:, next_strategy:, response:)
+        Log.info("#{self.class}: auto fallback #{strategy} -> #{next_strategy} after zero extracted items " \
+                 "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
+                 "budget_remaining=#{budget_remaining_label}")
       end
 
-      def log_info_fallback_zero_items(strategy:, next_strategy:)
-        Log.info("#{self.class}: auto fallback #{strategy} -> #{next_strategy} after zero extracted items")
+      def budget_remaining_label
+        remaining = budget.remaining_timeout_seconds
+        remaining.nil? ? 'untracked' : format('%.3f', remaining)
       end
     end
   end