html2rss 0.8.2 → 0.10.0

data/lib/html2rss/config/channel.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'addressable'
+
+module Html2rss
+  class Config
+    ##
+    # Holds the configuration for the feed's channel options.
+    # This contains:
+    #
+    # 1. the RSS channel attributes
+    # 2. html2rss options like json or custom HTTP-headers for the request
+    class Channel
+      ##
+      # @param channel [Hash<Symbol, Object>]
+      # @param params [Hash]
+      def initialize(channel, params: {})
+        raise ArgumentError, 'channel must be a hash' unless channel.is_a?(Hash)
+        raise ArgumentError, 'missing key :url' unless channel[:url].is_a?(String)
+
+        @config = process_params(channel, params.transform_keys(&:to_sym))
+      end
+
+      ##
+      # The HTTP headers to use for the request.
+      #
+      # @return [Hash<Symbol, String>]
+      def headers
+        config.fetch(:headers, {})
+      end
+
+      ##
+      # @return [String]
+      def author
+        config.fetch(:author, 'html2rss')
+      end
+
+      ##
+      # @return [Integer]
+      def ttl
+        config.fetch(:ttl, 360)
+      end
+
+      ##
+      # @return [String]
+      def title
+        config.fetch(:title) { Utils.titleized_url(url) }
+      end
+
+      ##
+      # @return [String] language code
+      def language
+        config.fetch(:language, 'en')
+      end
+
+      ##
+      # @return [String]
+      def description
+        config.fetch(:description) { "Latest items from #{url}." }
+      end
+
+      ##
+      # @return [Addressable::URI]
+      def url
+        Addressable::URI.parse(config[:url]).normalize
+      end
+
+      ##
+      # @return [String] time_zone name
+      def time_zone
+        config.fetch(:time_zone, 'UTC')
+      end
+
+      ##
+      # @return [true, false]
+      def json?
+        config.fetch(:json, false)
+      end
+
+      ##
+      # @param config [Hash<Symbol, Object>]
+      # @return [Set<String>] the required parameter names
+      def self.required_params_for_config(config)
+        config.each_with_object(Set.new) do |(_, value), required_params|
+          required_params.merge(value.scan(/%<([\w_\d]+)>/).flatten) if value.is_a?(String)
+        end
+      end
+
+      private
+
+      # @return [Hash<Symbol, Object>]
+      attr_reader :config
+
+      ##
+      # @param config [Hash<Symbol, Object>]
+      # @param params [Hash<Symbol, String>]
+      # @return [nil]
+      def assert_required_params_presence(config, params)
+        missing_params = self.class.required_params_for_config(config) - params.keys.map(&:to_s)
+        raise ParamsMissing, missing_params.to_a.join(', ') unless missing_params.empty?
+      end
+
+      ##
+      # Sets the variables used in the feed config's channel.
+      #
+      # @param config [Hash<Symbol, Object>]
+      # @param params [Hash<Symbol, Object>]
+      # @return [Hash<Symbol, Object>]
+      def process_params(config, params)
+        assert_required_params_presence(config, params)
+        config.transform_values do |value|
+          value.is_a?(String) ? format(value, params) : value
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/config/selectors.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ##
+    # Holds the configurations of the selectors.
+    class Selectors
+      ITEMS_SELECTOR_NAME = :items
+
+      # Struct to represent a selector with associated attributes for extraction and processing.
+      Selector = Struct.new(:selector, :attribute, :extractor, :post_process, :order, :static, keyword_init: true)
+
+      ##
+      # @param config [Hash<Symbol, Object>]
+      def initialize(config)
+        validate_config(config)
+        @config = config
+      end
+
+      ##
+      # @param name [Symbol]
+      # @return [true, false]
+      def selector?(name)
+        name != ITEMS_SELECTOR_NAME && item_selector_names.include?(name)
+      end
+
+      ##
+      # @param name [Symbol]
+      # @return [Selector]
+      def selector(name)
+        raise ArgumentError, "invalid item's selector name: #{name}" unless selector?(name)
+
+        Selector.new(config[name])
+      end
+
+      ##
+      # @return [Set<Symbol>]
+      def category_selector_names
+        selector_keys_for(:categories)
+      end
+
+      ##
+      # @return [Set<Symbol>]
+      def guid_selector_names
+        selector_keys_for(:guid, default: :title_or_description)
+      end
+
+      ##
+      # Returns the CSS/XPath selector.
+      #
+      # @param name [Symbol]
+      # @return [String]
+      def selector_string(name)
+        Selector.new(config[name]).selector
+      end
+
+      ##
+      # @return [Set<Symbol>]
+      def item_selector_names
+        @item_selector_names ||= config.keys.reject { |key| key == ITEMS_SELECTOR_NAME }.to_set
+      end
+
+      ##
+      # @return [Symbol, nil]
+      def items_order
+        config.dig(ITEMS_SELECTOR_NAME, :order)&.to_sym
+      end
+
+      private
+
+      attr_reader :config
+
+      def validate_config(config)
+        raise ArgumentError, 'selector for items is required' unless config[ITEMS_SELECTOR_NAME].is_a?(Hash)
+      end
+
+      ##
+      # Returns the selector keys for the selector named `name`. If none, returns [default].
+      #
+      # @param name [Symbol]
+      # @param default [String, Symbol]
+      # @return [Set<Symbol>]
+      def selector_keys_for(name, default: nil)
+        config.fetch(name) { Array(default) }.tap do |array|
+          array.reject! { |entry| entry.to_s == '' }
+          array.map!(&:to_sym)
+        end.to_set
+      end
+    end
+  end
+end

data/lib/html2rss/config.rb

@@ -1,91 +1,84 @@
-require 'active_support/core_ext/hash'
+# frozen_string_literal: true
+
+require 'forwardable'
 
 module Html2rss
   ##
   # The Config class abstracts from the config data structure and
   # provides default values.
   class Config
-    def initialize(feed_config, global_config = {})
-      @global_config = global_config.deep_symbolize_keys
-      @feed_config = feed_config.deep_symbolize_keys
-      @channel_config = @feed_config.fetch(:channel, {})
-    end
-
-    def author
-      channel_config.fetch :author, 'html2rss'
-    end
-
-    def ttl
-      channel_config.fetch :ttl, 360
-    end
-
-    def title
-      channel_config.fetch(:title) { generated_title }
-    end
-
-    def generated_title
-      uri = URI(url)
-
-      nicer_path = uri.path.split('/')
-      nicer_path.reject! { |part| part == '' }
-
-      nicer_path.any? ? "#{uri.host}: #{nicer_path.join(' ').titleize}" : uri.host
-    end
-
-    def language
-      channel_config.fetch :language, 'en'
-    end
-
-    def description
-      channel_config.fetch :description, "Latest items from #{url}."
-    end
-
-    def url
-      channel_config.dig :url
-    end
-    alias link url
-
-    def time_zone
-      channel_config.fetch :time_zone, 'UTC'
-    end
-
-    def json?
-      channel_config.fetch :json, false
-    end
-
+    extend Forwardable
+
+    ##
+    # The Error class to be thrown when a feed config requires params, but none
+    # were passed to Config.
+    class ParamsMissing < StandardError; end
+
+    ##
+    # Thrown when the feed config does not contain a value at `:channel`.
+    class ChannelMissing < StandardError; end
+
+    # Struct to store XML Stylesheet attributes
+    Stylesheet = Struct.new(:href, :type, :media, keyword_init: true)
+
+    def_delegator :@channel, :author
+    def_delegator :@channel, :ttl
+    def_delegator :@channel, :title
+    def_delegator :@channel, :language
+    def_delegator :@channel, :description
+    def_delegator :@channel, :url
+    def_delegator :@channel, :url, :link
+    def_delegator :@channel, :time_zone
+    def_delegator :@channel, :json?
+
+    def_delegator :@selectors, :item_selector_names
+    def_delegator :@selectors, :selector?
+    def_delegator :@selectors, :category_selector_names
+    def_delegator :@selectors, :guid_selector_names
+    def_delegator :@selectors, :items_order
+    def_delegator :@selectors, :selector_string
+
+    ##
+    # Initializes the Config object with feed configuration, global settings, and parameters.
+    #
+    # @param feed_config [Hash<Symbol, Object>] The configuration hash containing `:channel` and `:selectors`.
+    # @param global [Hash<Symbol, Object>] Global settings hash.
+    # @param params [Hash<Symbol, String>] Parameters hash.
+    def initialize(feed_config, global = {}, params = {})
+      channel_config = feed_config[:channel]
+      raise ChannelMissing, 'Channel configuration is missing in feed_config' unless channel_config
+
+      @channel = Channel.new(channel_config, params:)
+      @selectors = Selectors.new(feed_config[:selectors])
+      @global = global
+    end
+
+    ##
+    # Retrieves selector attributes merged with channel attributes.
+    #
+    # @param name [Symbol] Selector name.
+    # @return [Hash<Symbol, Object>] Merged attributes hash.
+    def selector_attributes_with_channel(name)
+      @selectors.selector(name).to_h.merge(channel: @channel)
+    end
+
+    ##
+    # Retrieves headers merged from global settings and channel headers.
+    #
+    # @return [Hash] Merged headers hash.
     def headers
-      global_config.fetch(:headers, {}).merge(channel_config.fetch(:headers, {}))
-    end
-
-    def attribute_options(name)
-      feed_config.dig(:selectors).fetch(name, {}).merge(channel: channel_config)
+      @global.fetch(:headers, {}).merge(@channel.headers)
     end
 
-    def attribute?(name)
-      attribute_names.include?(name)
+    ##
+    # Retrieves stylesheets from global settings.
+    #
+    # @return [Array<Stylesheet>] Array of Stylesheet structs.
+    def stylesheets
+      @global.fetch(:stylesheets, []).map { |attributes| Stylesheet.new(attributes) }
     end
 
-    def category_selectors
-      categories = feed_config.dig(:selectors, :categories)
-      return [] unless categories
-
-      categories = categories.keep_if { |category| category.to_s != '' }
-      categories.map!(&:to_sym)
-      categories.uniq!
-
-      categories
-    end
-
-    def selector(name)
-      feed_config.dig(:selectors, name, :selector)
-    end
-
-    def attribute_names
-      @attribute_names ||= feed_config.fetch(:selectors, {}).keys.tap { |attrs| attrs.delete(:items) }
-    end
-
-    private
-
-    attr_reader :feed_config, :channel_config, :global_config
+    # Provides read-only access to the channel object.
+    attr_reader :channel
   end
 end

data/lib/html2rss/item.rb

@@ -1,12 +1,24 @@
-require 'faraday'
-require 'faraday_middleware'
+# frozen_string_literal: true
+
 require 'nokogiri'
 
 module Html2rss
   ##
-  # Takes the selected Nokogiri::HTML and responds to accessors names
+  # Takes the selected Nokogiri::HTML and responds to accessor names
   # defined in the feed config.
+  #
+  # Instances can only be created via `.from_url` and
+  # each represents an internally used "RSS item".
+  # Such an item provides dynamically defined attributes as methods.
   class Item
+    # A context instance is passed to Item Extractors.
+    Context = Struct.new('Context', :options, :item, :config, keyword_init: true)
+    # Class to keep an Item's <enclosure>.
+    Enclosure = Struct.new('Enclosure', :type, :bits_length, :url, keyword_init: true)
+
+    ##
+    # @param xml [Nokogiri::XML::Element]
+    # @param config [Html2rss::Config]
     def initialize(xml, config)
       @xml = xml
       @config = config
@@ -14,84 +26,150 @@ module Html2rss
 
     private_class_method :new
 
+    ##
+    # Checks if the object responds to a method dynamically based on the configuration.
+    #
+    # @param method_name [Symbol]
+    # @param _include_private [true, false]
+    # @return [true, false]
     def respond_to_missing?(method_name, _include_private = false)
-      config.attribute?(method_name) || super
+      config.selector?(method_name) || super
     end
 
+    ##
+    # Dynamically extracts data based on the method name.
+    #
+    # @param method_name [Symbol]
+    # @param _args [Array]
+    # @return [String] extracted value for the selector.
     def method_missing(method_name, *_args)
       return super unless respond_to_missing?(method_name)
 
-      attribute_options = config.attribute_options(method_name)
+      extract(method_name)
+    end
 
-      extractor = ItemExtractors.get_extractor(attribute_options[:extractor])
-      value = extractor.new(xml, attribute_options).get
+    ##
+    # Selects and processes data according to the selector name.
+    #
+    # @param tag [Symbol]
+    # @return [String] the extracted value for the selector.
+    def extract(tag)
+      attribute_options = config.selector_attributes_with_channel(tag.to_sym)
+
+      post_process(
+        ItemExtractors.item_extractor_factory(attribute_options, xml).get,
+        attribute_options.fetch(:post_process, false)
+      )
+    end
 
-      post_process(value, attribute_options.fetch(:post_process, false))
+    ##
+    # Checks if the item is valid accordin to RSS 2.0 spec,
+    # by ensuring it has at least a title or a description.
+    #
+    # @return [true, false]
+    def valid?
+      title_or_description.to_s != ''
     end
 
-    def available_attributes
-      @available_attributes ||= (%i[title link description author comments updated] &
-        @config.attribute_names) - %i[categories enclosure]
+    ##
+    # Returns either the title or the description, preferring title if available.
+    #
+    # @return [String, nil]
+    def title_or_description
+      return title if config.selector?(:title)
+
+      description if config.selector?(:description)
     end
 
     ##
-    # At least a title or a description is required to be a valid RSS 2.0 item.
-    def valid?
-      title = self.title if config.attribute?(:title)
-      description = self.description if config.attribute?(:description)
-      [title, description].join != ''
+    #
+    # @return [String] SHA1 hashed GUID.
+    def guid
+      content = config.guid_selector_names.flat_map { |method_name| public_send(method_name) }.join
+
+      Digest::SHA1.hexdigest(content)
     end
 
     ##
-    # @return [Array]
+    # Retrieves categories for the item based on configured category selectors.
+    #
+    # @return [Array<String>] list of categories.
     def categories
-      config.category_selectors.map(&method(:method_missing))
+      config.category_selector_names.map { |method_name| public_send(method_name) }
     end
 
+    ##
+    # Checks if the item has an enclosure based on configuration.
+    #
+    # @return [true, false]
     def enclosure?
-      config.attribute?(:enclosure)
+      config.selector?(:enclosure)
     end
 
-    def enclosure_url
-      enclosure = Html2rss::Utils.sanitize_url(method_missing(:enclosure))
-
-      Html2rss::Utils.build_absolute_url_from_relative(enclosure, config.url).to_s if enclosure
+    ##
+    # Retrieves enclosure details for the item.
+    #
+    # @return [Enclosure] enclosure details.
+    def enclosure
+      url = enclosure_url
+
+      raise 'An item.enclosure requires an absolute URL' unless url&.absolute?
+
+      Enclosure.new(
+        type: Html2rss::Utils.guess_content_type_from_url(url),
+        bits_length: 0,
+        url: url.to_s
+      )
     end
 
     ##
-    # @return [Array]
+    # Fetches items from a given URL using configuration settings.
+    #
+    # @param url [String] URL to fetch items from.
+    # @param config [Html2rss::Config] Configuration object.
+    # @return [Array<Html2rss::Item>] list of items fetched.
     def self.from_url(url, config)
-      body = get_body_from_url(url, config)
+      body = Utils.request_body_from_url(url, convert_json_to_xml: config.json?, headers: config.headers)
 
-      Nokogiri.HTML(body).css(config.selector(:items))
-              .map { |xml_item| new xml_item, config }
-              .keep_if(&:valid?)
+      Nokogiri.HTML(body)
+              .css(config.selector_string(Config::Selectors::ITEMS_SELECTOR_NAME))
+              .map { |xml| new(xml, config) }
+              .select(&:valid?)
     end
 
     private
 
-    def self.get_body_from_url(url, config)
-      body = Faraday.new(url: url, headers: config.headers) do |faraday|
-        faraday.use FaradayMiddleware::FollowRedirects
-        faraday.adapter Faraday.default_adapter
-      end.get.body
-
-      config.json? ? Html2rss::Utils.object_to_xml(JSON.parse(body)) : body
-    end
-    private_class_method :get_body_from_url
-
-    attr_reader :xml, :config
+    # @return [Nokogiri::XML::Element] XML element representing the item.
+    attr_reader :xml
+    # @return [Html2rss::Config] Configuration object for the item.
+    attr_reader :config
 
+    ##
+    # Processes the extracted value according to post-processing options.
+    #
+    # @param value [String] extracted value.
+    # @param post_process_options [Hash<Symbol, Object>] post-processing options.
+    # @return [String] processed value.
     def post_process(value, post_process_options)
       return value unless post_process_options
 
       [post_process_options].flatten.each do |options|
         value = AttributePostProcessors.get_processor(options[:name])
-                                       .new(value, options: options, item: self, config: @config)
+                                       .new(value, Context.new(options:, item: self, config:))
                                        .get
       end
 
       value
     end
+
+    ##
+    # Retrieves the URL for the enclosure, sanitizing and ensuring it's absolute.
+    #
+    # @return [Addressable::URI, nil] absolute URL of the enclosure.
+    def enclosure_url
+      enclosure = Html2rss::Utils.sanitize_url(extract(:enclosure))
+
+      Html2rss::Utils.build_absolute_url_from_relative(enclosure, config.url) if enclosure
+    end
   end
 end

data/lib/html2rss/item_extractors/attribute.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module Html2rss
   module ItemExtractors
     ##
     # Returns the value of the attribute.
     #
-    # Imagine this +time+ HTML element with a +datetime+ attribute:
+    # Imagine this +time+ HTML tag with a +datetime+ attribute:
     #
     #     <time datetime="2019-07-01">...</time>
     #
@@ -18,19 +20,30 @@ module Html2rss
     # Would return:
     #    '2019-07-01'
     #
-    # In case you're extracting a date or a time, do not forget to parse it
-    # during post processing with
-    # {AttributePostProcessors::ParseTime}[rdoc-ref:Html2rss::AttributePostProcessors::ParseTime].
+    # In case you're extracting a date or a time, consider parsing it
+    # during post processing with {AttributePostProcessors::ParseTime}.
     class Attribute
+      # The available options for the attribute extractor.
+      Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true)
+
+      ##
+      # Initializes the Attribute extractor.
+      #
+      # @param xml [Nokogiri::XML::Element]
+      # @param options [Options]
       def initialize(xml, options)
         @options = options
-        @element = ItemExtractors.element(xml, options)
+        @element = ItemExtractors.element(xml, options.selector)
       end
 
       ##
-      # @return [String]
+      # Retrieves and returns the attribute's value as a string.
+      #
+      # @return [String] The value of the attribute.
       def get
-        @element.attr(@options[:attribute]).to_s
+        @element.attr(@options.attribute).to_s.freeze
+      rescue NoMethodError => error
+        raise "Failed to extract attribute: #{error.message}"
       end
     end
   end

data/lib/html2rss/item_extractors/href.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Html2rss
   module ItemExtractors
     ##
@@ -21,15 +23,29 @@ module Html2rss
     # Would return:
     #    'http://blog-without-a-feed.example.com/posts/latest-findings'
     class Href
+      # The available options for the href (attribute) extractor.
+      Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true)
+
+      ##
+      # Initializes the Href extractor.
+      #
+      # @param xml [Nokogiri::XML::Element]
+      # @param options [Options]
       def initialize(xml, options)
         @options = options
-        element = ItemExtractors.element(xml, options)
-        @href = Html2rss::Utils.sanitize_url(element.attr('href'))
+        @element = ItemExtractors.element(xml, options.selector)
+        @href = @element.attr('href').to_s
       end
 
-      # @return [URI::HTTPS, URI::HTTP]
+      ##
+      # Retrieves and returns the normalized absolute URL.
+      #
+      # @return [String] The absolute URL.
       def get
-        Html2rss::Utils.build_absolute_url_from_relative(@href, @options[:channel][:url])
+        return nil unless @href
+
+        sanitized_href = Html2rss::Utils.sanitize_url(@href)
+        Html2rss::Utils.build_absolute_url_from_relative(sanitized_href, @options.channel.url)
       end
     end
   end