html2rss 0.17.0 → 0.19.0

data/lib/html2rss/selectors/config.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require 'dry-validation'
+
+module Html2rss
+  class Selectors
+    ##
+    # Validates the configuration hash for :selectors.
+    class Config < Dry::Validation::Contract
+      # Required wrapper key used to validate dynamic selector names.
+      NESTING_KEY = :dynamic_keys_workaround
+
+      ##
+      # Validates the configuration of the :items selector
+      class Items < Dry::Validation::Contract
+        params do
+          required(:selector).filled(:string)
+          optional(:order).filled(included_in?: %w[reverse])
+          optional(:enhance).filled(:bool?)
+          optional(:pagination).hash do
+            required(:max_pages).filled(:integer, gt?: 0)
+          end
+        end
+      end
+
+      ##
+      # Validates the configuration of a single selector.
+      class Selector < Dry::Validation::Contract
+        params do
+          optional(:selector)
+          optional(:extractor).filled(:string)
+          optional(:attribute).filled(:string)
+          optional(:static).filled(:string)
+          optional(:post_process).array(:hash)
+        end
+
+        rule(:selector) do
+          key(:selector).failure('`selector` must be a string') if value && !value.is_a?(String)
+        end
+
+        rule(:extractor) do
+          # dependent on the extractor, validate required fields, (i.e. static, attribute)
+          case value
+          when 'attribute'
+            key(:attribute).failure('`attribute` must be a string') unless values[:attribute].is_a?(String)
+          when 'static'
+            key(:static).failure('`static` must be a string') unless values[:static].is_a?(String)
+          end
+        end
+
+        rule(:post_process).each do
+          case (name = value[:name])
+          when 'gsub'
+            key(:pattern).failure('`pattern` must be a string') unless value[:pattern].is_a?(String)
+            key(:replacement).failure('`replacement` must be a string') unless value[:replacement].is_a?(String)
+          when 'substring'
+            key(:start).failure('`start` must be an integer') unless value[:start].is_a?(Integer)
+            key(:end).failure('`end` must be an integer or omitted') if !value[:end].nil? && !value[:end].is_a?(Integer)
+          when 'template'
+            key(:string).failure('`string` must be a string') unless value[:string].is_a?(String)
+          when 'html_to_markdown', 'markdown_to_html', 'parse_time', 'parse_uri', 'sanitize_html'
+            # nothing to validate
+          when nil
+            key(:post_process).failure('Missing post_processor `name`')
+          else
+            key(:post_process).failure("Unknown post_processor `name`: #{name}")
+          end
+        end
+      end
+
+      ##
+      # Validates the configuration of the :enclosure Selector
+      class Enclosure < Selector
+        params do
+          optional(:content_type).filled(:string, format?: %r{^[\w-]+/[\w-]+$})
+        end
+      end
+
+      params do
+        required(NESTING_KEY).hash
+      end
+
+      rule(NESTING_KEY) do
+        value.each_pair do |selector_key, selector|
+          case selector_key.to_sym
+          when Selectors::ITEMS_SELECTOR_KEY
+            Items.new.call(selector).errors.each { |error| key(selector_key).failure(error) }
+          when :enclosure
+            Enclosure.new.call(selector).errors.each { |error| key(selector_key).failure(error) }
+          when :guid, :categories
+            unless selector.is_a?(Array)
+              key(selector_key).failure("`#{selector_key}` must be an array")
+              next
+            end
+
+            key(selector_key).failure("`#{selector_key}` must contain at least one element") if selector.empty?
+
+            selector.each do |name|
+              next if values[NESTING_KEY].key?(name.to_sym)
+
+              key(selector_key).failure("`#{selector_key}` references unspecified `#{name}`")
+            end
+          else
+            # From here on, the selector is found under its "dynamic" selector_key
+            Selector.new.call(selector).errors.each { |error| key(selector_key).failure(error) }
+          end
+        end
+      end
+
+      ##
+      # Shortcut to validate the config.
+      # @param config [Hash] the configuration hash to validate
+      # @return [Dry::Validation::Result] the result of the validation
+      def self.call(config)
+        # dry-validation/schema does not support "Dynamic Keys" yet: https://github.com/dry-rb/dry-schema/issues/37
+        # But :selectors contains mostly "dynamic" keys, as the user defines them to extract article attributes.
+        # --> Validate the dynamic keys manually.
+        # To be able to specify a `rule`, nest the config under NESTING_KEY and mark that as `required`.
+        new.call(NESTING_KEY => config)
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/extractors/attribute.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    module Extractors
+      ##
+      # Returns the value of the attribute.
+      #
+      # Imagine this +time+ HTML tag with a +datetime+ attribute:
+      #
+      #     <time datetime="2019-07-01">...</time>
+      #
+      # YAML usage example:
+      #
+      #    selectors:
+      #      link:
+      #        selector: time
+      #        extractor: attribute
+      #        attribute: datetime
+      #
+      # Would return:
+      #    '2019-07-01'
+      #
+      # In case you're extracting a date or a time, consider parsing it
+      # during post processing with {PostProcessors::ParseTime}.
+      class Attribute
+        # The available options for the attribute extractor.
+        Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+
+        ##
+        # Initializes the Attribute extractor.
+        #
+        # @param xml [Nokogiri::XML::Element]
+        # @param options [Options]
+        # @option options [String] :selector CSS selector used to find the element
+        # @option options [String] :attribute attribute name to extract from the selected element
+        def initialize(xml, options)
+          @options = options
+          @element = Extractors.element(xml, options.selector)
+        end
+
+        ##
+        # Retrieves and returns the attribute's value as a string.
+        #
+        # @return [String] The value of the attribute.
+        def get
+          @element.attr(@options.attribute).to_s
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/extractors/href.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    module Extractors
+      ##
+      # Returns the value of the +href+ attribute.
+      # It always returns absolute URLs. If the extracted +href+ value is a
+      # relative URL, it prepends the channel's URL.
+      #
+      # Imagine this +a+ HTML element with a +href+ attribute:
+      #
+      #     <a href="/posts/latest-findings">...</a>
+      #
+      # YAML usage example:
+      #    channel:
+      #      url: http://blog-without-a-feed.example.com
+      #      ...
+      #    selectors:
+      #      link:
+      #        selector: a
+      #        extractor: href
+      #
+      # Would return:
+      #    'http://blog-without-a-feed.example.com/posts/latest-findings'
+      class Href
+        # The available options for the href (attribute) extractor.
+        Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+
+        ##
+        # Initializes the Href extractor.
+        #
+        # @param xml [Nokogiri::XML::Element]
+        # @param options [Options]
+        # @option options [String] :selector CSS selector used to find the link element
+        # @option options [Hash{Symbol => Object}] :channel channel configuration, including :url
+        def initialize(xml, options)
+          @options = options
+          @element = Extractors.element(xml, options.selector)
+          @href = @element.attr('href').to_s
+        end
+
+        ##
+        # Retrieves and returns the normalized absolute URL.
+        #
+        # @return [String] The absolute URL.
+        def get
+          return nil unless @href
+
+          Url.from_relative(@href, @options.channel[:url])
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/extractors/html.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    module Extractors
+      ##
+      # Returns the HTML content of the specified element.
+      #
+      # Example HTML structure:
+      #
+      #     <p>Lorem <b>ipsum</b> dolor ...</p>
+      #
+      # YAML usage example:
+      #
+      #    selectors:
+      #      description:
+      #        selector: p
+      #        extractor: html
+      #
+      # Would return:
+      #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
+      #
+      # Always ensure to sanitize the HTML during post-processing with
+      # {PostProcessors::SanitizeHtml}.
+      class Html
+        # The available options for the html extractor.
+        Options = Struct.new('HtmlOptions', :selector, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+
+        ##
+        # Initializes the Html extractor.
+        #
+        # @param xml [Nokogiri::XML::Element]
+        # @param options [Options]
+        # @option options [String] :selector CSS selector used to find the element
+        def initialize(xml, options)
+          @element = Extractors.element(xml, options.selector)
+        end
+
+        ##
+        # Retrieves and returns the HTML content of the element.
+        #
+        # @return [String] The HTML content.
+        def get
+          @element.to_s
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/extractors/static.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    module Extractors
+      ##
+      # Returns a static value provided in the options.
+      #
+      # Example usage in YAML:
+      #
+      #    selectors:
+      #      byline:
+      #        extractor: static
+      #        static: Foobar
+      #
+      # Would return:
+      #    'Foobar'
+      class Static
+        # The available option for the static extractor.
+        Options = Struct.new('StaticOptions', :static, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+
+        ##
+        # Initializes the Static extractor.
+        #
+        # @param _xml [nil, Nokogiri::XML::Element] Unused parameter for compatibility with other extractors.
+        # @param options [Options] Options containing the static value.
+        # @option options [String, Symbol] :static static value returned by this extractor
+        def initialize(_xml, options)
+          @options = options
+        end
+
+        ##
+        # Retrieves and returns the static value.
+        #
+        # @return [String, Symbol] The static value provided in options.
+        def get
+          @options.static
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/extractors/text.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    module Extractors
+      ##
+      # Return the text content of the attribute. This is the default extractor used,
+      # when no extractor is explicitly given.
+      #
+      # Example HTML structure:
+      #
+      #     <p>Lorem <b>ipsum</b> dolor ...</p>
+      #
+      # YAML usage example:
+      #
+      #    selectors:
+      #      description:
+      #        selector: p
+      #        extractor: text
+      #
+      # Would return:
+      #    'Lorem ipsum dolor ...'
+      class Text
+        # The available options for the text extractor.
+        Options = Struct.new('TextOptions', :selector, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+
+        ##
+        # Initializes the Text extractor.
+        #
+        # @param xml [Nokogiri::XML::Element]
+        # @param options [Options]
+        # @option options [String] :selector CSS selector used to find the element
+        def initialize(xml, options)
+          @element = Extractors.element(xml, options.selector)
+        end
+
+        ##
+        # Retrieves and returns the text content of the element.
+        #
+        # @return [String] The text content.
+        def get
+          @element.text.to_s.strip.gsub(/\s+/, ' ')
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/extractors.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    ##
+    # Provides a namespace for item extractors.
+    module Extractors
+      ##
+      # Maps the extractor name to the class implementing the extractor.
+      #
+      # The key is the name to use in the feed config.
+      NAME_TO_CLASS = {
+        attribute: Attribute,
+        href: Href,
+        html: Html,
+        static: Static,
+        text: Text
+      }.freeze
+
+      ##
+      # Maps the extractor class to its corresponding options class.
+      ITEM_OPTION_CLASSES = Hash.new do |hash, klass|
+        hash[klass] = klass.const_get(:Options)
+      end
+
+      # Extractor used when none is explicitly configured.
+      DEFAULT_EXTRACTOR = :text
+
+      class << self
+        ##
+        # Retrieves an element from Nokogiri XML based on the selector.
+        #
+        # @param xml [Nokogiri::XML::Document]
+        # @param selector [String, nil]
+        # @return [Nokogiri::XML::ElementSet] selected XML elements
+        def element(xml, selector)
+          selector ? xml.css(selector) : xml
+        end
+
+        # @param attribute_options [Hash{Symbol => Object}]
+        #   Should contain at least `:extractor` (the name) and required options for that extractor.
+        # @param xml [Nokogiri::XML::Document]
+        # @return [Object] instance of the specified item extractor class
+        def get(attribute_options, xml)
+          extractor_class = NAME_TO_CLASS[attribute_options[:extractor]&.to_sym || DEFAULT_EXTRACTOR]
+          options = ITEM_OPTION_CLASSES[extractor_class].new(attribute_options.slice(*extractor_class::Options.members))
+
+          extractor_class.new(xml, options).get
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/object_to_xml_converter.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  class Selectors
+    ##
+    # A naive implementation of "Object to XML": converts a Ruby object to XML format.
+    class ObjectToXmlConverter
+      # Wrapper tags used for top-level collection conversion.
+      OBJECT_TO_XML_TAGS = {
+        hash: ['<object>', '</object>'],
+        array: ['<array>', '</array>']
+      }.freeze
+
+      ##
+      # @param object [Object] any Ruby object (Hash, Array, String, Symbol, etc.)
+      def initialize(object)
+        @object = object
+      end
+
+      ##
+      # Converts the object to XML format.
+      #
+      # @return [String] representing the object in XML
+      def call
+        object_to_xml(@object).tap do |converted|
+          Html2rss::Log.debug("#{self.class}: converted object to XML (#{converted.bytesize} bytes)")
+        end
+      end
+
+      private
+
+      def object_to_xml(object)
+        case object
+        when Hash
+          hash_to_xml(object)
+        when Array
+          array_to_xml(object)
+        else
+          CGI.escapeHTML(object.to_s)
+        end
+      end
+
+      def hash_to_xml(object)
+        prefix, suffix = OBJECT_TO_XML_TAGS[:hash]
+        inner_xml = object.each_with_object(+'') do |(key, value), str|
+          str << "<#{key}>#{object_to_xml(value)}</#{key}>"
+        end
+
+        "#{prefix}#{inner_xml}#{suffix}"
+      end
+
+      def array_to_xml(object)
+        prefix, suffix = OBJECT_TO_XML_TAGS[:array]
+        inner_xml = object.each_with_object(+'') { |value, str| str << object_to_xml(value) }
+
+        "#{prefix}#{inner_xml}#{suffix}"
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/base.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    module PostProcessors
+      ##
+      # All post processors must inherit from this base class and implement `self.validate_args!` and `#get`.
+      class Base
+        # Validates the presence of required options in the context
+        #
+        # @param keys [Array<Symbol>] the keys to check for presence
+        # @param context [Selectors::Context] the context containing options
+        # @return [void]
+        # @raise [MissingOption] if any key is missing
+        def self.expect_options(keys, context)
+          keys.each do |key|
+            unless (options = context[:options]).key?(key)
+              raise MissingOption, "The `#{key}` option is missing in: #{options.inspect}", [],
+                    cause: nil
+            end
+          end
+        end
+
+        # Asserts that the value is of the expected type(s)
+        #
+        # @param value [Object] the value to check
+        # @param types [Array<Class>, Class] the expected type(s)
+        # @param name [String] the name of the option being checked
+        # @param context [Selectors::Context] call-site context used for richer validation errors
+        # @return [void]
+        # @raise [InvalidType] if the value is not of the expected type(s)
+        def self.assert_type(value, types = [], name, context:)
+          return if Array(types).any? { |type| value.is_a?(type) }
+
+          options = if context.respond_to?(:options)
+                      context.options
+                    else
+                      { file: File.basename(caller(1, 1).first.split(':').first) }
+                    end
+          message = "The type of `#{name}` must be #{Array(types).join(' or ')}, " \
+                    "but is: #{value.class} in: #{options.inspect}"
+          raise InvalidType, message, [], cause: nil
+        end
+
+        ##
+        # This method validates the arguments passed to the post processor. Must be implemented by subclasses.
+        #
+        # @param _value [Object] extracted selector value
+        # @param _context [Selectors::Context] post-processor execution context
+        # @return [void]
+        def self.validate_args!(_value, _context)
+          raise NotImplementedError, 'You must implement the `validate_args!` method in the post processor'
+        end
+
+        # Initializes the post processor
+        #
+        # @param value [Object] the value to be processed
+        # @param context [Selectors::Context] runtime selector context and options
+        def initialize(value, context)
+          klass = self.class
+          klass.assert_type(context, Selectors::Context, 'context', context:)
+          klass.validate_args!(value, context)
+
+          @value = value
+          @context = context
+        end
+
+        attr_reader :value, :context
+
+        # Abstract method to be implemented by subclasses
+        #
+        # @return [Object] transformed value
+        # @raise [NotImplementedError] if not implemented in subclass
+        def get
+          raise NotImplementedError, 'You must implement the `get` method in the post processor'
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/gsub.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'regexp_parser'
+
+module Html2rss
+  class Selectors
+    module PostProcessors
+      ##
+      # Imagine this HTML:
+      #    <h1>Foo bar and boo<h1>
+      #
+      # YAML usage example:
+      #    selectors:
+      #      title:
+      #        selector: h1
+      #        post_process:
+      #          name: gsub
+      #          pattern: boo
+      #          replacement: baz
+      #
+      # Would return:
+      #    'Foo bar and baz'
+      #
+      # `pattern` can be a Regexp or a String. If it is a String, it will remove
+      # one pair of surrounding slashes ('/') to keep backwards compatibility
+      # and then parse it to build a Regexp.
+      #
+      # `replacement` can be a String or a Hash.
+      #
+      # See the doc on [String#gsub](https://ruby-doc.org/core/String.html#method-i-gsub) for more info.
+      class Gsub < Base
+        # @param value [String] extracted selector value
+        # @param context [Selectors::Context] post-processor context
+        # @return [void]
+        def self.validate_args!(value, context)
+          assert_type value, String, :value, context:
+          expect_options(%i[replacement pattern], context)
+          assert_type context.dig(:options, :replacement), [String, Hash], :replacement, context:
+        end
+
+        ##
+        # @param value [String]
+        # @param context [Selectors::Context]
+        def initialize(value, context)
+          super
+
+          options = context[:options]
+
+          @replacement = options[:replacement]
+          @pattern = options[:pattern]
+        end
+
+        ##
+        # @return [String]
+        def get
+          value.to_s.gsub(pattern, replacement)
+        end
+
+        private
+
+        attr_accessor :replacement
+
+        ##
+        # @return [Regexp]
+        def pattern
+          @pattern.is_a?(String) ? parse_regexp_string(@pattern) : @pattern
+        end
+
+        ##
+        # Parses the given String and builds a Regexp out of it.
+        #
+        # It will remove one pair of surrounding slashes ('/') from the String
+        # to maintain backwards compatibility before building the Regexp.
+        #
+        # @param string [String]
+        # @return [Regexp]
+        def parse_regexp_string(string)
+          raise ArgumentError, 'must be a string!' unless string.is_a?(String)
+
+          # Only remove surrounding slashes if the string has at least 3 characters
+          # to avoid issues with single character strings like "/"
+          string = string[1..-2] if string.length >= 3 && string.start_with?('/') && string.end_with?('/')
+          Regexp::Parser.parse(string, options: ::Regexp::EXTENDED | ::Regexp::IGNORECASE).to_re
+        end
+      end
+    end
+  end
+end