html2rss 0.17.0 → 0.18.0

data/lib/html2rss/config/selectors.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class Config
-    ##
-    # Holds the configurations of the selectors.
-    class Selectors
-      ITEMS_SELECTOR_NAME = :items
-
-      # Struct to represent a selector with associated attributes for extraction and processing.
-      Selector = Struct.new(:selector, :attribute, :extractor, :post_process, :order, :static, :content_type,
-                            keyword_init: true)
-
-      # raised when an invalid selector name is used
-      class InvalidSelectorName < Html2rss::Error; end
-
-      ##
-      # @param config [Hash<Symbol, Object>]
-      def initialize(config)
-        validate_config(config)
-        @config = config
-      end
-
-      ##
-      # @param name [Symbol]
-      # @return [true, false]
-      def selector?(name)
-        name != ITEMS_SELECTOR_NAME && item_selector_names.include?(name)
-      end
-
-      ##
-      # @param name [Symbol]
-      # @return [Selector]
-      def selector(name)
-        raise InvalidSelectorName, "invalid selector name: #{name}" unless selector?(name)
-
-        keywords = config[name].slice(*available_keys)
-
-        if (additional_keys = keywords.keys - available_keys).any?
-          Log.warn "additional keys (#{additional_keys.join(', ')}) present in selector #{name}"
-        end
-
-        Selector.new(keywords)
-      end
-
-      ##
-      # @return [Set<Symbol>]
-      def category_selector_names
-        selector_keys_for(:categories)
-      end
-
-      ##
-      # @return [Set<Symbol>]
-      def guid_selector_names
-        selector_keys_for(:guid, default: :title_or_description)
-      end
-
-      ##
-      # Returns the CSS/XPath selector.
-      #
-      # @param name [Symbol]
-      # @return [String]
-      def selector_string(name)
-        Selector.new(config[name]).selector
-      end
-
-      ##
-      # @return [Set<Symbol>]
-      def item_selector_names
-        @item_selector_names ||= config.keys.reject { |key| key == ITEMS_SELECTOR_NAME }.to_set
-      end
-
-      ##
-      # @return [Symbol, nil]
-      def items_order
-        config.dig(ITEMS_SELECTOR_NAME, :order)&.to_sym
-      end
-
-      private
-
-      attr_reader :config
-
-      def validate_config(config)
-        raise ArgumentError, 'selector for items is required' unless config[ITEMS_SELECTOR_NAME].is_a?(Hash)
-      end
-
-      ##
-      # Returns the selector keys for the selector named `name`. If none, returns [default].
-      #
-      # @param name [Symbol]
-      # @param default [String, Symbol]
-      # @return [Set<Symbol>]
-      def selector_keys_for(name, default: nil)
-        config.fetch(name) { Array(default) }.tap do |array|
-          array.reject! { |entry| entry.to_s == '' }
-          array.map!(&:to_sym)
-        end.to_set
-      end
-
-      def available_keys = @available_keys ||= Selector.members
-    end
-  end
-end

data/lib/html2rss/item.rb

@@ -1,186 +0,0 @@
-# frozen_string_literal: true
-
-require 'nokogiri'
-
-module Html2rss
-  ##
-  # Takes the selected Nokogiri::HTML and responds to accessor names
-  # defined in the feed config.
-  #
-  # Instances can only be created via `.from_url` and
-  # each represents an internally used "RSS item".
-  # Such an item provides dynamically defined attributes as methods.
-  class Item
-    # A context instance is passed to Item Extractors.
-    Context = Struct.new('Context', :options, :item, :config, keyword_init: true)
-    # Class to keep an Item's <enclosure>.
-    Enclosure = Struct.new('Enclosure', :type, :bits_length, :url, keyword_init: true)
-
-    ##
-    # Fetches items from a given URL using configuration settings.
-    #
-    # @param url [Addressable::URI] URL to fetch items from.
-    # @param config [Html2rss::Config] Configuration object.
-    # @return [Array<Html2rss::Item>] list of items fetched.
-    def self.from_url(url, config)
-      ctx = RequestService::Context.new(url:, headers: config.headers)
-
-      body = RequestService.execute(ctx, strategy: config.strategy).body
-      body = ObjectToXmlConverter.new(JSON.parse(body)).call if config.json?
-
-      Nokogiri.HTML(body)
-              .css(config.selector_string(Config::Selectors::ITEMS_SELECTOR_NAME))
-              .map { |xml| new(xml, config) }
-              .select(&:valid?)
-    end
-
-    ##
-    # @param xml [Nokogiri::XML::Element]
-    # @param config [Html2rss::Config]
-    def initialize(xml, config)
-      @xml = xml
-      @config = config
-    end
-
-    private_class_method :new
-
-    ##
-    # Checks if the object responds to a method dynamically based on the configuration.
-    #
-    # @param method_name [Symbol]
-    # @param _include_private [true, false]
-    # @return [true, false]
-    # :reek:BooleanParameter { enabled: false }
-    def respond_to_missing?(method_name, _include_private = false)
-      config.selector?(method_name) || super
-    end
-
-    ##
-    # Dynamically extracts data based on the method name.
-    #
-    # @param method_name [Symbol]
-    # @param _args [Array]
-    # @return [String] extracted value for the selector.
-    def method_missing(method_name, *_args)
-      return super unless respond_to_missing?(method_name)
-
-      extract(method_name)
-    end
-
-    ##
-    # Selects and processes data according to the selector name.
-    #
-    # @param tag [Symbol]
-    # @return [String] the extracted value for the selector.
-    def extract(tag)
-      attribute_options = config.selector_attributes_with_channel(tag.to_sym)
-
-      post_process(
-        ItemExtractors.item_extractor_factory(attribute_options, xml).get,
-        attribute_options.fetch(:post_process, false)
-      )
-    end
-
-    ##
-    # Checks if the item is valid accordin to RSS 2.0 spec,
-    # by ensuring it has at least a title or a description.
-    #
-    # @return [true, false]
-    def valid?
-      title_or_description.to_s != ''
-    end
-
-    ##
-    # Returns either the title or the description, preferring title if available.
-    #
-    # @return [String, nil]
-    def title_or_description
-      return title if config.selector?(:title)
-
-      description if config.selector?(:description)
-    end
-
-    ##
-    #
-    # @return [String] SHA1 hashed GUID.
-    def guid
-      content = config.guid_selector_names.flat_map { |method_name| public_send(method_name) }.join
-
-      Digest::SHA1.hexdigest(content)
-    end
-
-    ##
-    # Retrieves categories for the item based on configured category selectors.
-    #
-    # @return [Array<String>] list of categories.
-    def categories
-      config.category_selector_names
-            .filter_map do |method_name|
-        category = public_send(method_name)
-        category.strip unless category.to_s.empty?
-      end.uniq
-    end
-
-    ##
-    # Checks if the item has an enclosure based on configuration.
-    #
-    # @return [true, false]
-    def enclosure?
-      config.selector?(:enclosure)
-    end
-
-    ##
-    # Retrieves enclosure details for the item.
-    #
-    # @return [Enclosure] enclosure details.
-    def enclosure
-      url = enclosure_url
-
-      raise 'An item.enclosure requires an absolute URL' unless url&.absolute?
-
-      type = config.selector_attributes_with_channel(:enclosure)[:content_type] ||
-             Html2rss::Utils.guess_content_type_from_url(url)
-
-      Enclosure.new(
-        type:,
-        bits_length: 0,
-        url: url.to_s
-      )
-    end
-
-    private
-
-    # @return [Nokogiri::XML::Element] XML element representing the item.
-    attr_reader :xml
-    # @return [Html2rss::Config] Configuration object for the item.
-    attr_reader :config
-
-    ##
-    # Processes the extracted value according to post-processing options.
-    #
-    # @param value [String] extracted value.
-    # @param post_process_options [Hash<Symbol, Object>] post-processing options.
-    # @return [String] processed value.
-    def post_process(value, post_process_options)
-      return value unless post_process_options
-
-      [post_process_options].flatten.each do |options|
-        value = AttributePostProcessors.get_processor(options[:name])
-                                       .new(value, Context.new(options:, item: self, config:))
-                                       .get
-      end
-
-      value
-    end
-
-    ##
-    # Retrieves the URL for the enclosure, sanitizing and ensuring it's absolute.
-    #
-    # @return [Addressable::URI, nil] absolute URL of the enclosure.
-    def enclosure_url
-      enclosure = Html2rss::Utils.sanitize_url(extract(:enclosure))
-
-      Html2rss::Utils.build_absolute_url_from_relative(enclosure, config.url) if enclosure
-    end
-  end
-end

data/lib/html2rss/item_extractors/attribute.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module ItemExtractors
-    ##
-    # Returns the value of the attribute.
-    #
-    # Imagine this +time+ HTML tag with a +datetime+ attribute:
-    #
-    #     <time datetime="2019-07-01">...</time>
-    #
-    # YAML usage example:
-    #
-    #    selectors:
-    #      link:
-    #        selector: time
-    #        extractor: attribute
-    #        attribute: datetime
-    #
-    # Would return:
-    #    '2019-07-01'
-    #
-    # In case you're extracting a date or a time, consider parsing it
-    # during post processing with {AttributePostProcessors::ParseTime}.
-    class Attribute
-      # The available options for the attribute extractor.
-      Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true)
-
-      ##
-      # Initializes the Attribute extractor.
-      #
-      # @param xml [Nokogiri::XML::Element]
-      # @param options [Options]
-      def initialize(xml, options)
-        @options = options
-        @element = ItemExtractors.element(xml, options.selector)
-      end
-
-      ##
-      # Retrieves and returns the attribute's value as a string.
-      #
-      # @return [String] The value of the attribute.
-      def get
-        @element.attr(@options.attribute).to_s.freeze
-      rescue NoMethodError => error
-        raise "Failed to extract attribute: #{error.message}"
-      end
-    end
-  end
-end

data/lib/html2rss/item_extractors/href.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module ItemExtractors
-    ##
-    # Returns the value of the +href+ attribute.
-    # It always returns absolute URLs. If the extracted +href+ value is a
-    # relative URL, it prepends the channel's URL.
-    #
-    # Imagine this +a+ HTML element with a +href+ attribute:
-    #
-    #     <a href="/posts/latest-findings">...</a>
-    #
-    # YAML usage example:
-    #    channel:
-    #      url: http://blog-without-a-feed.example.com
-    #      ...
-    #    selectors:
-    #      link:
-    #        selector: a
-    #        extractor: href
-    #
-    # Would return:
-    #    'http://blog-without-a-feed.example.com/posts/latest-findings'
-    class Href
-      # The available options for the href (attribute) extractor.
-      Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true)
-
-      ##
-      # Initializes the Href extractor.
-      #
-      # @param xml [Nokogiri::XML::Element]
-      # @param options [Options]
-      def initialize(xml, options)
-        @options = options
-        @element = ItemExtractors.element(xml, options.selector)
-        @href = @element.attr('href').to_s
-      end
-
-      ##
-      # Retrieves and returns the normalized absolute URL.
-      #
-      # @return [String] The absolute URL.
-      def get
-        return nil unless @href
-
-        sanitized_href = Html2rss::Utils.sanitize_url(@href)
-        Html2rss::Utils.build_absolute_url_from_relative(sanitized_href, @options.channel.url)
-      end
-    end
-  end
-end

data/lib/html2rss/item_extractors/html.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module ItemExtractors
-    ##
-    # Returns the HTML content of the specified element.
-    #
-    # Example HTML structure:
-    #
-    #     <p>Lorem <b>ipsum</b> dolor ...</p>
-    #
-    # YAML usage example:
-    #
-    #    selectors:
-    #      description:
-    #        selector: p
-    #        extractor: html
-    #
-    # Would return:
-    #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
-    #
-    # Always ensure to sanitize the HTML during post-processing with
-    # {AttributePostProcessors::SanitizeHtml}.
-    class Html
-      # The available options for the html extractor.
-      Options = Struct.new('HtmlOptions', :selector, keyword_init: true)
-
-      ##
-      # Initializes the Html extractor.
-      #
-      # @param xml [Nokogiri::XML::Element]
-      # @param options [Options]
-      def initialize(xml, options)
-        @element = ItemExtractors.element(xml, options.selector)
-      end
-
-      ##
-      # Retrieves and returns the HTML content of the element.
-      #
-      # @return [String] The HTML content.
-      def get
-        @element.to_s
-      end
-    end
-  end
-end

data/lib/html2rss/item_extractors/static.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module ItemExtractors
-    ##
-    # Returns a static value provided in the options.
-    #
-    # Example usage in YAML:
-    #
-    #    selectors:
-    #      author:
-    #        extractor: static
-    #        static: Foobar
-    #
-    # Would return:
-    #    'Foobar'
-    class Static
-      # The available option for the static extractor.
-      Options = Struct.new('StaticOptions', :static, keyword_init: true)
-
-      ##
-      # Initializes the Static extractor.
-      #
-      # @param _xml [nil, Nokogiri::XML::Element] Unused parameter for compatibility with other extractors.
-      # @param options [Options] Options containing the static value.
-      def initialize(_xml, options)
-        @options = options
-      end
-
-      ##
-      # Retrieves and returns the static value.
-      #
-      # @return [String, Symbol] The static value provided in options.
-      def get
-        @options.static
-      end
-    end
-  end
-end

data/lib/html2rss/item_extractors/text.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module ItemExtractors
-    ##
-    # Return the text content of the attribute. This is the default extractor used,
-    # when no extractor is explicitly given.
-    #
-    # Example HTML structure:
-    #
-    #     <p>Lorem <b>ipsum</b> dolor ...</p>
-    #
-    # YAML usage example:
-    #
-    #    selectors:
-    #      description:
-    #        selector: p
-    #        extractor: text
-    #
-    # Would return:
-    #    'Lorem ipsum dolor ...'
-    class Text
-      # The available options for the text extractor.
-      Options = Struct.new('TextOptions', :selector, keyword_init: true)
-
-      ##
-      # Initializes the Text extractor.
-      #
-      # @param xml [Nokogiri::XML::Element]
-      # @param options [Options]
-      def initialize(xml, options)
-        @element = ItemExtractors.element(xml, options.selector)
-      end
-
-      ##
-      # Retrieves and returns the text content of the element.
-      #
-      # @return [String] The text content.
-      def get
-        @element.text.to_s.strip.gsub(/\s+/, ' ')
-      end
-    end
-  end
-end

data/lib/html2rss/item_extractors.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  ##
-  # Provides a namespace for item extractors.
-  module ItemExtractors
-    ##
-    # The Error class to be thrown when an unknown extractor name is requested.
-    class UnknownExtractorName < Html2rss::Error; end
-
-    ##
-    # Maps the extractor name to the class implementing the extractor.
-    #
-    # The key is the name to use in the feed config.
-    NAME_TO_CLASS = {
-      attribute: Attribute,
-      href: Href,
-      html: Html,
-      static: Static,
-      text: Text
-    }.freeze
-
-    ##
-    # Maps the extractor class to its corresponding options class.
-    ITEM_OPTION_CLASSES = Hash.new do |hash, klass|
-      hash[klass] = klass.const_get(:Options)
-    end
-
-    DEFAULT_EXTRACTOR = :text
-
-    ##
-    # Retrieves an element from Nokogiri XML based on the selector.
-    #
-    # @param xml [Nokogiri::XML::Document]
-    # @param selector [String, nil]
-    # @return [Nokogiri::XML::ElementSet] selected XML elements
-    def self.element(xml, selector)
-      selector ? xml.css(selector) : xml
-    end
-
-    ##
-    # Creates an instance of the requested item extractor.
-    #
-    # @param attribute_options [Hash<Symbol, Object>]
-    #   Should contain at least `:extractor` (the name) and required options for that extractor.
-    # @param xml [Nokogiri::XML::Document]
-    # @return [Object] instance of the specified item extractor class
-    def self.item_extractor_factory(attribute_options, xml)
-      extractor_name = attribute_options[:extractor]&.to_sym || DEFAULT_EXTRACTOR
-      extractor_class = find_extractor_class(extractor_name)
-      options_instance = build_options_instance(extractor_class, attribute_options)
-      create_extractor_instance(extractor_class, xml, options_instance)
-    end
-
-    ##
-    # Finds the extractor class based on the name.
-    #
-    # @param extractor_name [Symbol] the name of the extractor
-    # @return [Class] the class implementing the extractor
-    # @raise [UnknownExtractorName] if the extractor class is not found
-    def self.find_extractor_class(extractor_name)
-      NAME_TO_CLASS[extractor_name] || raise(UnknownExtractorName,
-                                             "Unknown extractor name '#{extractor_name}' requested in NAME_TO_CLASS")
-    end
-
-    ##
-    # Builds the options instance for the extractor class.
-    #
-    # @param extractor_class [Class] the class implementing the extractor
-    # @param attribute_options [Hash<Symbol, Object>] the attribute options
-    # @return [Object] an instance of the options class for the extractor
-    def self.build_options_instance(extractor_class, attribute_options)
-      options = attribute_options.slice(*extractor_class::Options.members)
-      ITEM_OPTION_CLASSES[extractor_class].new(options)
-    end
-
-    ##
-    # Creates an instance of the extractor class.
-    #
-    # @param extractor_class [Class] the class implementing the extractor
-    # @param xml [Nokogiri::XML::Document] the XML document
-    # @param options_instance [Object] the options instance
-    # @return [Object] an instance of the extractor class
-    def self.create_extractor_instance(extractor_class, xml, options_instance)
-      extractor_class.new(xml, options_instance)
-    end
-  end
-end

data/lib/html2rss/object_to_xml_converter.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require 'cgi'
-require 'json'
-
-module Html2rss
-  ##
-  # A naive implementation of "Object to XML": converts a Ruby object to XML format.
-  class ObjectToXmlConverter
-    OBJECT_TO_XML_TAGS = {
-      hash: ['<object>', '</object>'],
-      enumerable: ['<array>', '</array>']
-    }.freeze
-
-    ##
-    # @param object [Object] any Ruby object (Hash, Array, String, Symbol, etc.)
-    def initialize(object)
-      @object = object
-    end
-
-    ##
-    # Converts the object to XML format.
-    #
-    # @return [String] representing the object in XML
-    def call
-      object_to_xml(@object)
-    end
-
-    private
-
-    def object_to_xml(object)
-      case object
-      when Hash
-        hash_to_xml(object)
-      when Enumerable
-        enumerable_to_xml(object)
-      else
-        CGI.escapeHTML(object.to_s)
-      end
-    end
-
-    def hash_to_xml(object)
-      prefix, suffix = OBJECT_TO_XML_TAGS[:hash]
-      inner_xml = object.map { |key, value| "<#{key}>#{object_to_xml(value)}</#{key}>" }.join
-
-      "#{prefix}#{inner_xml}#{suffix}"
-    end
-
-    def enumerable_to_xml(object)
-      prefix, suffix = OBJECT_TO_XML_TAGS[:enumerable]
-      inner_xml = object.map { |value| object_to_xml(value) }.join
-
-      "#{prefix}#{inner_xml}#{suffix}"
-    end
-  end
-end