html2rss 0.6.0 → 0.9.0

data/html2rss.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.description   = 'Give the URL to scrape and some CSS selectors. Get a RSS::Rss instance in return.'
   spec.homepage      = 'https://github.com/gildesmarais/html2rss'
   spec.license       = 'MIT'
-  spec.required_ruby_version = '>= 2.4.0'
+  spec.required_ruby_version = '>= 2.5.0'
 
   if spec.respond_to?(:metadata)
     spec.metadata['allowed_push_host'] = 'https://rubygems.org'
@@ -29,15 +29,19 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'activesupport', '~> 5.0'
+  spec.add_dependency 'activesupport', '>= 5', '< 7'
+  spec.add_dependency 'addressable', '~> 2.7'
   spec.add_dependency 'builder'
-  spec.add_dependency 'faraday', '~> 0.15'
-  spec.add_dependency 'faraday_middleware', '~> 0.13'
-  spec.add_dependency 'hashie', '~> 3.6'
+  spec.add_dependency 'faraday', '~> 1.0'
+  spec.add_dependency 'faraday_middleware'
+  spec.add_dependency 'kramdown'
+  spec.add_dependency 'mime-types', '> 3.0'
   spec.add_dependency 'nokogiri', '>= 1.10', '< 2.0'
-  spec.add_dependency 'reverse_markdown', '~> 1.3'
+  spec.add_dependency 'reverse_markdown', '~> 2.0'
   spec.add_dependency 'sanitize', '~> 5.0'
-  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_dependency 'to_regexp'
+  spec.add_dependency 'zeitwerk'
+  spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'byebug'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop'

data/lib/html2rss.rb

@@ -1,7 +1,8 @@
-require 'html2rss/config'
-require 'html2rss/feed_builder'
-require 'html2rss/version'
-require 'html2rss/utils'
+require 'zeitwerk'
+
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+
 require 'yaml'
 
 ##

data/lib/html2rss/attribute_post_processors.rb

@@ -1,19 +1,13 @@
-require_relative 'attribute_post_processors/html_to_markdown'
-require_relative 'attribute_post_processors/parse_time'
-require_relative 'attribute_post_processors/parse_uri'
-require_relative 'attribute_post_processors/sanitize_html'
-require_relative 'attribute_post_processors/substring'
-require_relative 'attribute_post_processors/template'
-
 module Html2rss
   ##
   # Provides a namespace for attribute post processors.
   module AttributePostProcessors
     def self.get_processor(name)
-      camel_cased_name = name.split('_').map(&:capitalize).join
-      class_name = ['Html2rss', 'AttributePostProcessors', camel_cased_name].join('::')
+      @get_processor ||= Hash.new do |processors, key|
+        processors[key] = Utils.get_class_from_name(key, 'AttributePostProcessors')
+      end
 
-      Object.const_get(class_name)
+      @get_processor[name]
     end
   end
 end

data/lib/html2rss/attribute_post_processors/gsub.rb

@@ -0,0 +1,42 @@
+require 'to_regexp'
+
+module Html2rss
+  module AttributePostProcessors
+    ##
+    #
+    # Imagine this HTML:
+    #    <h1>Foo bar and boo<h1>
+    #
+    # YAML usage example:
+    #    selectors:
+    #      title:
+    #        selector: h1
+    #        post_process:
+    #          name: gsub
+    #          pattern: boo
+    #          replacement: baz
+    #
+    # Would return:
+    #    'Foo bar and baz'
+    #
+    # `pattern` can be a Regexp or a String.
+    #
+    # `replacement` can be a String or a Hash.
+    #
+    # See the doc on [String#gsub](https://ruby-doc.org/core/String.html#method-i-gsub) for more info.
+    class Gsub
+      def initialize(value, env)
+        @value = value
+        options = env[:options]
+        @pattern = options[:pattern].to_regexp || options[:pattern]
+        @replacement = options[:replacement]
+      end
+
+      ##
+      # @return [String]
+      def get
+        @value.to_s.gsub(@pattern, @replacement)
+      end
+    end
+  end
+end

data/lib/html2rss/attribute_post_processors/markdown_to_html.rb

@@ -0,0 +1,45 @@
+require 'kramdown'
+
+module Html2rss
+  module AttributePostProcessors
+    ##
+    # Generates HTML from Markdown.
+    #
+    # It's particularly useful in conjunction with the Template post processor
+    # to generate a description from other selectors.
+    #
+    # YAML usage example:
+    #
+    #    selectors:
+    #      description:
+    #        selector: section
+    #        post_process:
+    #          - name: template
+    #            string: |
+    #              # %s
+    #
+    #              Price: %s
+    #            methods:
+    #              - self
+    #              - price
+    #          - name: markdown_to_html
+    #
+    # Would e.g. return:
+    #
+    #    <h1>Section</h1>
+    #
+    #    <p>Price: 12.34</p>
+    class MarkdownToHtml
+      def initialize(value, env)
+        @value = value
+        @env = env
+      end
+
+      ##
+      # @return [String] formatted in Markdown
+      def get
+        SanitizeHtml.new(Kramdown::Document.new(@value).to_html, @env).get
+      end
+    end
+  end
+end

data/lib/html2rss/attribute_post_processors/parse_time.rb

@@ -17,12 +17,12 @@ module Html2rss
     #        selector: span
     #        post_process:
     #          name: 'parse_time'
+    #          time_zone: 'Europe/Berlin'
     #
     # Would return:
     #    "Tue, 02 Jul 2019 00:00:00 +0200"
     #
     # It uses {https://ruby-doc.org/stdlib-2.5.3/libdoc/time/rdoc/Time.html#method-c-parse Time.parse}.
-    # As of now it ignores time zones and always falls back to the UTC time zone.
     class ParseTime
       def initialize(value, env)
         @value = value.to_s

data/lib/html2rss/attribute_post_processors/parse_uri.rb

@@ -5,7 +5,7 @@ module Html2rss
     #
     # Imagine this HTML structure:
     #
-    #    <span>http://why-not-use-a-link.uh</span>
+    #    <span>http://why-not-use-a-link.uh </span>
     #
     # YAML usage example:
     #
@@ -15,6 +15,7 @@ module Html2rss
     #        extractor: text
     #        post_process:
     #          name: parse_uri
+    #
     # Would return:
     #    'http://why-not-use-a-link.uh'
     class ParseUri
@@ -25,7 +26,7 @@ module Html2rss
       ##
       # @return [String]
       def get
-        URI(@value).to_s
+        URI(Html2rss::Utils.sanitize_url(@value)).to_s
       end
     end
   end

data/lib/html2rss/attribute_post_processors/sanitize_html.rb

@@ -4,10 +4,16 @@ module Html2rss
   module AttributePostProcessors
     ##
     # Returns sanitized HTML code as String.
-    # Adds
     #
-    # - rel="nofollow noopener noreferrer" to a elements
-    # - referrer-policy='no-referrer' to img elements
+    # It adds:
+    #
+    # - `rel="nofollow noopener noreferrer"` to <a> tags
+    # - `referrer-policy='no-referrer'` to <img> tags
+    #
+    # It also:
+    #
+    # - wraps all <img> tags, whose direct parent is not an <a>, into an <a>
+    #   linking to the <img>'s `src`.
     #
     # Imagine this HTML structure:
     #
@@ -21,7 +27,7 @@ module Html2rss
     #
     #    selectors:
     #      description:
-    #        selector: section
+    #        selector: '.section'
     #        extractor: html
     #        post_process:
     #          name: sanitize_html
@@ -29,6 +35,9 @@ module Html2rss
     # Would return:
     #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
     class SanitizeHtml
+      URL_ELEMENTS_WITH_URL_ATTRIBUTE = { 'a' => :href, 'img' => :src }.freeze
+      private_constant :URL_ELEMENTS_WITH_URL_ATTRIBUTE
+
       def initialize(value, env)
         @value = value
         @channel_url = env[:config].url
@@ -41,28 +50,22 @@ module Html2rss
       # - adds target="_blank" to a elements
       # @return [String]
       def get
-        Sanitize.fragment(@value, Sanitize::Config.merge(
-                                    Sanitize::Config::RELAXED,
-                                    attributes: { all: %w[dir lang alt title translate] },
-                                    add_attributes: {
-                                      'a' => {
-                                        'rel' => 'nofollow noopener noreferrer',
-                                        'target' => '_blank'
-                                      },
-                                      'img' => {
-                                        'referrer-policy' => 'no-referrer'
-                                      }
-                                    },
-                                    transformers: [transform_urls_to_absolute_ones]
-                                  )).to_s.split.join(' ')
+        Sanitize.fragment(@value, sanitize_config).to_s.split.join(' ')
       end
 
       private
 
-      URL_ELEMENTS_WITH_URL_ATTRIBUTE = {
-        'a' => :href,
-        'img' => :src
-      }.freeze
+      def sanitize_config
+        Sanitize::Config.merge(
+          Sanitize::Config::RELAXED,
+          attributes: { all: %w[dir lang alt title translate] },
+          add_attributes: {
+            'a' => { 'rel' => 'nofollow noopener noreferrer', 'target' => '_blank' },
+            'img' => { 'referrer-policy' => 'no-referrer' }
+          },
+          transformers: [transform_urls_to_absolute_ones, wrap_img_in_a]
+        )
+      end
 
       def transform_urls_to_absolute_ones
         lambda do |env|
@@ -78,6 +81,23 @@ module Html2rss
           env[:node][url_attribute] = absolute_url
         end
       end
+
+      def wrap_img_in_a
+        lambda do |env|
+          return if env[:node_name] != 'img'
+
+          img = env[:node]
+
+          return if img.parent.name == 'a'
+
+          anchor = Nokogiri::XML::Node.new('a', img)
+          anchor[:href] = img[:src]
+
+          anchor.add_child img.dup
+
+          img.replace(anchor)
+        end
+      end
     end
   end
 end

data/lib/html2rss/attribute_post_processors/substring.rb

@@ -2,9 +2,15 @@ module Html2rss
   module AttributePostProcessors
     ## Returns a defined part of a String.
     #
+    # Both parameters must be an Integer and they can be negative.
     # The +end+ parameter can be omitted, in that case it will not cut the
     # String at the end.
     #
+    # A Regexp or a MatchString is not supported.
+    #
+    # See the [`String#[]`](https://ruby-doc.org/core/String.html#method-i-5B-5D)
+    # documentation for more information.
+    #
     # Imagine this HTML:
     #    <h1>Foo bar and baz<h1>
     #
@@ -13,9 +19,9 @@ module Html2rss
     #      title:
     #        selector: h1
     #        post_process:
-    #         name: substring
-    #         start: 4
-    #         end: 6
+    #          name: substring
+    #          start: 4
+    #          end: 6
     #
     # Would return:
     #    'bar'
@@ -28,8 +34,8 @@ module Html2rss
       ##
       # @return [String]
       def get
-        ending = @options.fetch('end', @value.length).to_i
-        @value[@options['start'].to_i..ending]
+        ending = @options.fetch(:end, @value.length).to_i
+        @value[@options[:start].to_i..ending]
       end
     end
   end

data/lib/html2rss/attribute_post_processors/template.rb

@@ -4,7 +4,8 @@ module Html2rss
   module AttributePostProcessors
     ## Returns a formatted String according to the string pattern.
     #
-    # If +self+ is given as a method, the extracted value will be used.
+    # If +self+ is used, the selectors extracted value will be used.
+    # It uses [Kernel#format](https://ruby-doc.org/core/Kernel.html#method-i-format)
     #
     # Imagine this HTML:
     #    <li>
@@ -22,11 +23,8 @@ module Html2rss
     #      title:
     #        selector: h1
     #        post_process:
-    #         name: template
-    #         string: '%s (%s)'
-    #         methods:
-    #           - self
-    #           - price
+    #          name: template
+    #          string: '%{self} (%{price})'
     #
     # Would return:
     #    'Product (23,42€)'
@@ -35,25 +33,36 @@ module Html2rss
         @value = value
         @options = env[:options]
         @item = env[:item]
+        @string = @options[:string]
       end
 
       ##
-      # - uses {http://ruby-doc.org/core-2.6.3/String.html#method-i-25 String#%}
       # @return [String]
       def get
-        string % methods
+        return format_string_with_methods if @options[:methods]
+
+        names = string.scan(/%[<|{](\w*)[>|}]/)
+        names.flatten!
+        names.compact!
+        names.map!(&:to_sym)
+
+        format(string, names.map { |name| [name, item_value(name)] }.to_h)
       end
 
       private
 
-      def string
-        @options['string']
-      end
+      attr_reader :string
 
       def methods
-        @methods ||= @options['methods'].map do |method|
-          method == 'self' ? @value.to_s : @item.public_send(method.to_sym).to_s
-        end
+        @methods ||= @options[:methods].map(&method(:item_value))
+      end
+
+      def format_string_with_methods
+        string % methods
+      end
+
+      def item_value(method_name)
+        method_name.to_sym == :self ? @value.to_s : @item.public_send(method_name).to_s
       end
     end
   end

data/lib/html2rss/config.rb

@@ -1,71 +1,91 @@
+require 'active_support/core_ext/hash'
+
 module Html2rss
   ##
   # The Config class abstracts from the config data structure and
   # provides default values.
   class Config
     def initialize(feed_config, global_config = {})
-      @global_config = Utils::IndifferentAccessHash.new global_config
-      @feed_config = Utils::IndifferentAccessHash.new feed_config
-      @channel_config = Utils::IndifferentAccessHash.new @feed_config.fetch('channel', {})
+      @global_config = global_config.deep_symbolize_keys
+      @feed_config = feed_config.deep_symbolize_keys
+      @channel_config = @feed_config.fetch(:channel, {})
     end
 
     def author
-      channel_config.fetch 'author', 'html2rss'
+      channel_config.fetch :author, 'html2rss'
     end
 
     def ttl
-      channel_config.fetch 'ttl', 3600
+      channel_config.fetch :ttl, 360
     end
 
     def title
-      channel_config.fetch 'title', 'html2rss generated title'
+      channel_config.fetch(:title) { generated_title }
+    end
+
+    def generated_title
+      uri = URI(url)
+
+      nicer_path = uri.path.split('/')
+      nicer_path.reject! { |part| part == '' }
+
+      nicer_path.any? ? "#{uri.host}: #{nicer_path.join(' ').titleize}" : uri.host
     end
 
     def language
-      channel_config.fetch 'language', 'en'
+      channel_config.fetch :language, 'en'
     end
 
     def description
-      channel_config.fetch 'description', 'A description of my html2rss feed.'
+      channel_config.fetch :description, "Latest items from #{url}."
     end
 
     def url
-      channel_config.dig 'url'
+      channel_config.dig :url
     end
     alias link url
 
     def time_zone
-      channel_config.fetch 'time_zone', 'UTC'
+      channel_config.fetch :time_zone, 'UTC'
     end
 
     def json?
-      channel_config.fetch 'json', false
+      channel_config.fetch :json, false
     end
 
     def headers
-      global_config.fetch('headers', {}).merge(channel_config.fetch('headers', {}))
+      global_config.fetch(:headers, {}).merge(channel_config.fetch(:headers, {}))
     end
 
     def attribute_options(name)
-      feed_config.dig('selectors').fetch(name, {}).merge('channel' => channel_config)
+      feed_config.dig(:selectors).fetch(name, {}).merge(channel: channel_config)
     end
 
     def attribute?(name)
-      attribute_names.include?(name.to_s)
+      attribute_names.include?(name)
     end
 
-    def categories
-      feed_config.dig('selectors').fetch('categories', []).map(&:to_sym)
+    def category_selectors
+      categories = feed_config.dig(:selectors, :categories)
+      return [] unless categories
+
+      categories = categories.keep_if { |category| category.to_s != '' }
+      categories.map!(&:to_sym)
+      categories.uniq!
+
+      categories
     end
 
     def selector(name)
-      feed_config.dig('selectors', name, 'selector')
+      feed_config.dig(:selectors, name, :selector)
     end
 
     def attribute_names
-      @attribute_names ||= feed_config.fetch('selectors', {}).keys.map(&:to_s).tap do |attrs|
-        attrs.delete('items')
-      end
+      @attribute_names ||= feed_config.fetch(:selectors, {}).keys.tap { |attrs| attrs.delete(:items) }
+    end
+
+    def items_order
+      feed_config.dig(:selectors, :items, :order)&.to_sym
     end
 
     private