html2rss 0.22.2 → 0.24.0

data/lib/html2rss/request_session/runtime_input.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class RequestSession
-    ##
-    # Carries the runtime request inputs needed to build a RequestSession.
-    class RuntimeInput
-      ##
-      # @param config [Html2rss::Config] validated feed config
-      # @return [RuntimeInput] runtime request inputs derived from the config
-      def self.from_config(config)
-        new(
-          url: config.url,
-          headers: config.headers,
-          request: config.request,
-          strategy: config.strategy,
-          request_policy: RuntimePolicy.from_config(config)
-        )
-      end
-
-      ##
-      # @param url [String, Html2rss::Url] initial request URL
-      # @param headers [Hash] normalized request headers
-      # @param request [Hash] validated request options for strategies
-      # @param strategy [Symbol] request strategy to use for the session
-      # @param request_policy [RequestService::Policy] request policy for the session
-      def initialize(url:, headers:, request:, strategy:, request_policy:)
-        @url = Html2rss::Url.from_absolute(url)
-        @headers = normalize_headers(headers).freeze
-        @request = normalize_request(request).freeze
-        @strategy = strategy
-        @request_policy = request_policy
-        freeze
-      end
-
-      ##
-      # @return [Html2rss::Url] initial request URL
-      attr_reader :url
-
-      ##
-      # @return [Hash] normalized request headers
-      attr_reader :headers
-
-      ##
-      # @return [Hash] validated request options for strategies
-      attr_reader :request
-
-      ##
-      # @return [Symbol] request strategy to use for the session
-      attr_reader :strategy
-
-      ##
-      # @return [RequestService::Policy] policy derived from the runtime request inputs
-      attr_reader :request_policy
-
-      private
-
-      def normalize_headers(headers)
-        headers.to_h do |key, value|
-          [key.to_s, value]
-        end
-      end
-
-      def normalize_request(request)
-        normalized = HashUtil.deep_symbolize_keys(request, context: 'request')
-        HashUtil.assert_symbol_keys!(normalized, context: 'request')
-        normalized
-      end
-    end
-  end
-end

data/lib/html2rss/request_session/runtime_policy.rb

@@ -1,84 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class RequestSession
-    ##
-    # Builds the runtime request policy for a feed run.
-    class RuntimePolicy
-      ##
-      # @param config [Html2rss::Config] validated feed config
-      # @return [Html2rss::RequestService::Policy] request policy derived from runtime config
-      def self.from_config(config)
-        RequestService::Policy.new(
-          max_requests: effective_max_requests_for(config),
-          max_redirects: config.max_redirects,
-          total_timeout_seconds: config.total_timeout_seconds || RequestService::Policy::DEFAULTS[:total_timeout_seconds]
-        )
-      end
-
-      class << self
-        private
-
-        def effective_max_requests_for(config)
-          return config.max_requests if config.explicit_max_requests?
-
-          [baseline_request_budget_for(config), config.max_requests].max
-        end
-
-        # Reserve enough budget for the initial request plus predictable follow-ups
-        # that the top-level pipeline may trigger during a normal feed build.
-        def baseline_request_budget_for(config)
-          1 + pagination_follow_up_budget_for(config) +
-            known_auto_source_follow_up_budget_for(config) +
-            auto_strategy_fallback_budget_for(config) +
-            browserless_preload_budget_for(config)
-        end
-
-        def auto_strategy_fallback_budget_for(config)
-          return 0 unless config.strategy == :auto
-
-          [FeedPipeline::AutoFallback::CHAIN.size - 1, 0].max
-        end
-
-        def pagination_follow_up_budget_for(config)
-          [config.selectors&.dig(:items, :pagination, :max_pages).to_i - 1, 0].max
-        end
-
-        def known_auto_source_follow_up_budget_for(config)
-          config.auto_source&.dig(:scraper, :wordpress_api, :enabled) ? 1 : 0
-        end
-
-        def browserless_preload_budget_for(config)
-          preload = config.request.dig(:browserless, :preload)
-          return 0 unless preload
-
-          top_level_preload_wait_budget(preload) +
-            click_selector_preload_budget(preload) +
-            scroll_preload_budget(preload)
-        end
-
-        def top_level_preload_wait_budget(preload)
-          preload[:wait_after_ms] ? 2 : 0
-        end
-
-        def click_selector_preload_budget(preload)
-          preload.fetch(:click_selectors, []).sum { preload_action_budget(_1, :max_clicks) }
-        end
-
-        def scroll_preload_budget(preload)
-          scroll = preload[:scroll_down]
-          return 0 unless scroll
-
-          preload_action_budget(scroll, :iterations)
-        end
-
-        def preload_action_budget(config, count_key)
-          action_count = config.fetch(count_key, 1)
-          wait_budget = config[:wait_after_ms] ? action_count : 0
-
-          action_count + wait_budget
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/rss_builder/article.rb

@@ -1,208 +0,0 @@
-# frozen_string_literal: true
-
-require 'zlib'
-require 'sanitize'
-require 'nokogiri'
-
-module Html2rss
-  class RssBuilder
-    ##
-    # Article is a simple data object representing an article extracted from a page.
-    # It is enumerable and responds to all keys specified in PROVIDED_KEYS.
-    # rubocop:disable Metrics/ClassLength
-    class Article
-      include Enumerable
-      include Comparable
-
-      # Allowed article attributes accepted by the value object constructor.
-      PROVIDED_KEYS = %i[id title description url image author guid published_at enclosures categories scraper].freeze
-      # Separator used to build deterministic deduplication fingerprints.
-      DEDUP_FINGERPRINT_SEPARATOR = '#!/'
-      # Sentinel object used to pre-initialize instance variables in the constructor.
-      # This ensures all Article instances share the exact same object shape (Ruby 3.3+ optimization),
-      # preventing performance warnings and slower instance variable access due to shape transitions
-      # when attributes are lazily/conditionally accessed in different sequences.
-      NOT_SET = Object.new.freeze
-
-      # @param options [Hash{Symbol => String}]
-      # @option options [String] :id stable article identifier
-      # @option options [String] :title article title
-      # @option options [String] :description article description/content
-      # @option options [String, Html2rss::Url] :url canonical article URL
-      # @option options [String, Html2rss::Url] :image image URL for fallback enclosure rendering
-      # @option options [String] :author author name
-      # @option options [String] :guid explicit GUID override
-      # @option options [String, Time, DateTime] :published_at publication timestamp
-      # @option options [Array<Hash{Symbol => Object}>] :enclosures enclosure attribute hashes
-      # @option options [Array<String>] :categories category labels
-      # @option options [Class] :scraper scraper class that produced the article
-      def initialize(**options)
-        @to_h = options.each_with_object({}) { |(k, v), h| h[k] = v.freeze if v }.freeze
-
-        @description = @url = @image = @guid = @enclosures = @enclosure = @categories = @published_at = NOT_SET
-
-        return unless (unknown_keys = options.keys - PROVIDED_KEYS).any?
-
-        Log.warn "Article: unknown keys found: #{unknown_keys.join(', ')}"
-      end
-
-      # Checks if the article is valid based on the presence of URL, ID, and either title or description.
-      # @return [Boolean] True if the article is valid, otherwise false.
-      def valid?
-        !url.to_s.empty? && (!title.to_s.empty? || !description.to_s.empty?) && !id.to_s.empty?
-      end
-
-      # @yield [key, value]
-      # @return [Enumerator] if no block is given
-      def each
-        return enum_for(:each) unless block_given?
-
-        PROVIDED_KEYS.each { |key| yield(key, public_send(key)) }
-      end
-
-      # @return [String, nil] stable article identifier
-      def id = blank_string_to_nil(@to_h[:id])
-
-      # @return [String, nil] article title
-      def title = blank_string_to_nil(@to_h[:title])
-
-      # @return [String] rendered article description
-      def description
-        return @description unless @description == NOT_SET
-
-        @description = Rendering::DescriptionBuilder.new(
-          base: @to_h[:description],
-          title:,
-          url:,
-          enclosures:,
-          image:
-        ).call
-      end
-
-      # @return [Url, nil]
-      def url
-        return @url unless @url == NOT_SET
-
-        @url = Url.sanitize(@to_h[:url])
-      end
-
-      # @return [Url, nil]
-      def image
-        return @image unless @image == NOT_SET
-
-        @image = Url.sanitize(@to_h[:image])
-      end
-
-      # @return [String, nil]
-      def author = blank_string_to_nil(@to_h[:author])
-
-      # Generates a unique identifier based on the URL and ID using CRC32.
-      # @return [String]
-      def guid
-        return @guid unless @guid == NOT_SET
-
-        @guid = Zlib.crc32(fetch_guid).to_s(36).encode('utf-8')
-      end
-
-      ##
-      # Returns a deterministic fingerprint used to detect duplicate articles.
-      #
-      # @return [String, Integer]
-      def deduplication_fingerprint
-        dedup_from_url || dedup_from_id || dedup_from_guid || hash
-      end
-
-      # @return [Array<Html2rss::RssBuilder::Enclosure>] normalized enclosure objects
-      def enclosures
-        return @enclosures unless @enclosures == NOT_SET
-
-        @enclosures = Array(@to_h[:enclosures])
-                      .map { |enclosure| Html2rss::RssBuilder::Enclosure.new(**enclosure) }
-      end
-
-      # @return [Html2rss::RssBuilder::Enclosure, nil]
-      def enclosure
-        return @enclosure unless @enclosure == NOT_SET
-
-        @enclosure = case (object = @to_h[:enclosures]&.first)
-                     when Hash
-                       Html2rss::RssBuilder::Enclosure.new(**object)
-                     when nil
-                       Html2rss::RssBuilder::Enclosure.new(url: image) if image
-                     else
-                       Log.warn "Article: unknown enclosure type: #{object.class}"
-                       nil
-                     end
-      end
-
-      # @return [Array<String>] normalized, unique category names
-      def categories
-        return @categories unless @categories == NOT_SET
-
-        @categories = @to_h[:categories].dup.to_a.tap do |categories|
-          categories.map! { |category| category.to_s.strip }
-          categories.reject!(&:empty?)
-          categories.uniq!
-        end
-      end
-
-      # Parses and returns the published_at time.
-      # @return [DateTime, nil]
-      def published_at
-        return @published_at unless @published_at == NOT_SET
-
-        string = @to_h[:published_at].to_s.strip
-        @published_at = string.empty? ? nil : DateTime.parse(string)
-      rescue ArgumentError
-        @published_at = nil
-      end
-
-      # @return [Class, nil] scraper class that produced this article
-      def scraper
-        @to_h[:scraper]
-      end
-
-      # @param other [Object] value compared against this article
-      # @return [Integer, nil] comparison result for compatible Article values
-      def <=>(other)
-        return nil unless other.is_a?(Article)
-
-        0 if other.all? { |key, value| value == public_send(key) ? public_send(key) <=> value : false }
-      end
-
-      private
-
-      def dedup_from_url
-        return unless (value = url)
-
-        [value.to_s, id].compact.join(DEDUP_FINGERPRINT_SEPARATOR)
-      end
-
-      def dedup_from_id
-        return if id.to_s.empty?
-
-        id
-      end
-
-      def dedup_from_guid
-        value = guid
-        return if value.to_s.empty?
-
-        [value, title, description].compact.join(DEDUP_FINGERPRINT_SEPARATOR)
-      end
-
-      def fetch_guid
-        guid = @to_h[:guid].map { |s| s.to_s.strip }.reject(&:empty?).join if @to_h[:guid].is_a?(Array)
-
-        guid || [url, id].join('#!/')
-      end
-
-      def blank_string_to_nil(value)
-        return if value.is_a?(String) && value.strip.empty?
-
-        value
-      end
-    end
-    # rubocop:enable Metrics/ClassLength
-  end
-end

data/lib/html2rss/rss_builder/channel.rb

@@ -1,118 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class RssBuilder
-    ##
-    # Extracts channel information from
-    # 1. the HTML document's <head>.
-    # 2. the HTTP response
-    class Channel
-      # Fallback RSS ttl (in minutes) when no cache directives are present.
-      DEFAULT_TTL_IN_MINUTES = 360
-      # Description template used when no explicit or discovered description exists.
-      DEFAULT_DESCRIPTION_TEMPLATE = 'Latest items from %<url>s'
-
-      ##
-      # @param response [Html2rss::RequestService::Response]
-      # @param overrides [Hash{Symbol => String}] optional overrides for channel attributes
-      def initialize(response, overrides: {})
-        @response = response
-        @overrides = overrides
-      end
-
-      # @return [String] channel title derived from overrides, document title, or URL
-      def title
-        @title ||= fetch_title
-      end
-
-      # @return [Html2rss::Url] canonical channel URL
-      def url = @url ||= Html2rss::Url.from_absolute(@response.url)
-
-      # @return [String] channel description text
-      def description
-        return overrides[:description] unless overrides[:description].to_s.empty?
-
-        description = parsed_body.at_css('meta[name="description"]')&.[]('content') if html_response?
-
-        return format(DEFAULT_DESCRIPTION_TEMPLATE, url:) if description.to_s.empty?
-
-        description
-      end
-
-      # @return [Integer] cache time-to-live in minutes
-      def ttl
-        calculated = if overrides[:ttl]
-                       overrides[:ttl].to_i
-                     elsif (max_age = headers['cache-control']&.match(/max-age=(\d+)/)&.[](1))
-                       max_age.to_i.fdiv(60).ceil
-                     else
-                       DEFAULT_TTL_IN_MINUTES
-                     end
-
-        min_ttl = Html2rss.configuration.min_ttl
-        min_ttl ? [calculated, min_ttl].max : calculated
-      end
-
-      # @return [String, nil] ISO-like language code when available
-      def language
-        return overrides[:language] if overrides[:language]
-
-        if (language_code = headers['content-language']&.match(/^([a-z]{2})/))
-          return language_code[0]
-        end
-
-        return unless html_response?
-
-        parsed_body['lang'] || parsed_body.at_css('[lang]')&.[]('lang')
-      end
-
-      # @return [String, nil] channel author metadata
-      def author
-        return overrides[:author] if overrides[:author]
-
-        return unless html_response?
-
-        parsed_body.at_css('meta[name="author"]')&.[]('content')
-      end
-
-      # @return [String, Time] source last-modified timestamp or current time fallback
-      def last_build_date = headers['last-modified'] || Time.now
-
-      # @return [Html2rss::Url, nil] channel image URL
-      def image
-        return overrides[:image] if overrides[:image]
-
-        return unless html_response?
-
-        if (image_url = parsed_body.at_css('meta[property="og:image"]')&.[]('content'))
-          Url.sanitize(image_url)
-        end
-      end
-
-      private
-
-      attr_reader :overrides
-
-      def parsed_body = @parsed_body ||= @response.parsed_body
-      def headers = @headers ||= @response.headers
-      def html_response? = @html_response ||= @response.html_response?
-
-      def fetch_title
-        override_title = overrides[:title]
-        return override_title if override_title
-        return parsed_title if parsed_title
-
-        url.channel_titleized
-      end
-
-      def parsed_title
-        return unless html_response?
-
-        title = parsed_body.at_css('head > title')&.text.to_s
-        return if title.empty?
-
-        title.gsub(/\s+/, ' ').strip
-      end
-    end
-  end
-end

data/lib/html2rss/rss_builder/stylesheet.rb

@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class RssBuilder
-    ##
-    # Represents a stylesheet.
-    class Stylesheet
-      class << self
-        ##
-        # Adds the stylesheet XML tags to the RSS.
-        #
-        # @param maker [RSS::Maker::RSS20] RSS maker object.
-        # @param stylesheets [Array<Html2rss::RssBuilder::Stylesheet>] Array of stylesheet configurations.
-        # @return [nil]
-        def add(maker, stylesheets)
-          stylesheets.each do |stylesheet|
-            add_stylesheet(maker, stylesheet)
-          end
-        end
-
-        private
-
-        ##
-        # Adds a single Stylesheet to the RSS.
-        #
-        # @param maker [RSS::Maker::RSS20] RSS maker object.
-        # @param stylesheet [Html2rss::RssBuilder::Stylesheet] Stylesheet configuration.
-        # @return [nil]
-        def add_stylesheet(maker, stylesheet)
-          maker.xml_stylesheets.new_xml_stylesheet do |xss|
-            xss.href = stylesheet.href
-            xss.type = stylesheet.type
-            xss.media = stylesheet.media
-          end
-        end
-      end
-
-      # Allowed stylesheet MIME types for RSS processing instructions.
-      TYPES = ['text/css', 'text/xsl'].to_set.freeze
-
-      # @param href [String] stylesheet URL
-      # @param type [String] MIME type (`text/css` or `text/xsl`)
-      # @param media [String] media query hint for the stylesheet
-      def initialize(href:, type:, media: 'all')
-        raise ArgumentError, 'stylesheet.href must be a String' unless href.is_a?(String)
-        raise ArgumentError, 'stylesheet.type invalid' unless TYPES.include?(type)
-        raise ArgumentError, 'stylesheet.media must be a String' unless media.is_a?(String)
-
-        @href = href
-        @type = type
-        @media = media
-      end
-      attr_reader :href, :type, :media
-
-      # @return [String] the XML representation of the stylesheet
-      def to_xml
-        <<~XML
-          <?xml-stylesheet href="#{href}" type="#{type}" media="#{media}"?>
-        XML
-      end
-    end
-  end
-end

data/lib/html2rss/rss_builder.rb

@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-require 'rss'
-
-module Html2rss
-  ##
-  # Builds an RSS Feed by providing channel, articles and stylesheets.
-  class RssBuilder
-    class << self
-      # @param article [Html2rss::RssBuilder::Article] source article
-      # @param item_maker [RSS::Maker::RSS20::ItemsBase::ItemBase] RSS item builder
-      # @return [void]
-      def add_item(article, item_maker)
-        add_item_string_values(article, item_maker)
-        add_item_categories(article, item_maker)
-        Enclosure.add(article.enclosure, item_maker)
-        add_item_guid(article, item_maker)
-      end
-
-      private
-
-      def add_item_string_values(article, item_maker)
-        %i[title description author].each do |attr|
-          next unless (value = article.send(attr))
-          next if value.empty?
-
-          item_maker.send(:"#{attr}=", value)
-        end
-
-        item_maker.link = article.url.to_s if article.url
-        item_maker.pubDate = article.published_at&.rfc2822
-      end
-
-      def add_item_categories(article, item_maker)
-        article.categories.each { |category| item_maker.categories.new_category.content = category }
-      end
-
-      def add_item_guid(article, item_maker)
-        item_maker.guid.tap do |guid|
-          guid.content = article.guid
-          guid.isPermaLink = false
-        end
-      end
-    end
-
-    ##
-    # @param channel [Html2rss::RssBuilder::Channel] The channel information for the RSS feed.
-    # @param articles [Array<Html2rss::RssBuilder::Article>] The list of articles to include in the RSS feed.
-    # @param stylesheets [Array<Hash>] An optional array of stylesheet configurations.
-    def initialize(channel:, articles:, stylesheets: [])
-      @channel = channel
-      @articles = articles
-      @stylesheets = stylesheets
-    end
-
-    # @return [RSS::Rss] RSS 2.0 document instance
-    def call
-      RSS::Maker.make('2.0') do |maker|
-        Stylesheet.add(maker, stylesheets)
-
-        make_channel(maker.channel)
-        make_items(maker)
-      end
-    end
-
-    private
-
-    attr_reader :channel, :articles
-
-    def stylesheets
-      @stylesheets.map { |style| Stylesheet.new(**style) }
-    end
-
-    def make_channel(maker)
-      %i[language title description ttl].each do |key|
-        maker.public_send(:"#{key}=", channel.public_send(key))
-      end
-
-      maker.link = channel.url.to_s
-      maker.generator = generator
-      maker.updated = channel.last_build_date
-    end
-
-    def make_items(maker)
-      articles.each do |article|
-        maker.items.new_item { |item_maker| self.class.add_item(article, item_maker) }
-      end
-    end
-
-    def generator
-      scraper_namespace_regex = /(?<namespace>Html2rss|Scraper)::/
-
-      scraper_counts = articles.flat_map(&:scraper).tally.map do |klass, count|
-        scraper_name = klass.to_s.gsub(scraper_namespace_regex, '')
-        "#{scraper_name} (#{count})"
-      end
-
-      "html2rss V. #{Html2rss::VERSION} (scrapers: #{scraper_counts.join(', ')})"
-    end
-  end
-end