html2rss 0.8.1 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 134aacbabf1b61aadcdb6d96757d329bde20429027f97c2c3bbd0328bb02cd7b
-  data.tar.gz: 2e6ef3c8a38df3e54983670b17d574bfe771069f662414e58a5c0572ab922b1f
+  metadata.gz: dc1a99cdc487b0367e8af1145d876b84d043cf5abe31fdf8e7662e6ac3ab72c9
+  data.tar.gz: 57c41b955c3d35c7681d2213518d7faca1f5fe536f08fb6715547180b31c3312
 SHA512:
-  metadata.gz: 4e3b80225d14b44820f742f484a418e2d0b90060f7e329f2c89a2786795a1d87bf372f194e873d4a109d8c3649399e0716a41f65479c1b60ef9b8202154a8e48
-  data.tar.gz: ed5c095e4457d11208fa67634e526f1345c64f9d48d48df3eb11b8a5ebeb9cb203a62ae329f5388491d57435b1326bf25771349e3a374dc424094d3da60822e0
+  metadata.gz: 9d47c5c10bacc651a25fa90b1054d5658c7d4a1fe803a22dbcbeda42e931acc55cbbb39b8d7eafaf948bcc497d7961c4dca7c795cd955a7b6bbfe58c6a3925f2
+  data.tar.gz: 8521bcc416c72c86407c99b2e60950c5b3c683ea8aa05012e0ee829e6cfc33e8320564480c99f824d2c2da7f1732d638da26589fc4a4875e0e0a75b3c4e6a1d2

data/CHANGELOG.md

@@ -1,4 +1,13 @@
-# [](https://github.com/gildesmarais/html2rss/compare/v0.8.1...v) (2019-11-08)
+# [](https://github.com/gildesmarais/html2rss/compare/v0.8.2...v) (2019-11-09)
+
+
+
+## [0.8.2](https://github.com/gildesmarais/html2rss/compare/v0.8.1...v0.8.2) (2019-11-09)
+
+
+### Features
+
+* improve url handling by sanitizing and normalizing urls ([#70](https://github.com/gildesmarais/html2rss/issues/70)) ([02cd551](https://github.com/gildesmarais/html2rss/commit/02cd551))
 
 
 

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    html2rss (0.8.1)
+    html2rss (0.8.2)
       activesupport (~> 5.0)
+      addressable (~> 2.7)
       builder
       faraday (~> 0.15)
       faraday_middleware (~> 0.13)
-      hashie (~> 3.6)
       kramdown
       mime-types (> 3.0)
       nokogiri (>= 1.10, < 2.0)
@@ -23,6 +23,8 @@ GEM
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.0)
     builder (3.2.3)
     byebug (11.0.1)
@@ -40,7 +42,6 @@ GEM
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.13.1)
       faraday (>= 0.7.4, < 1.0)
-    hashie (3.6.0)
     i18n (1.7.0)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
@@ -59,6 +60,7 @@ GEM
     parallel (1.18.0)
     parser (2.6.5.0)
       ast (~> 2.4.0)
+    public_suffix (4.0.1)
     rainbow (3.0.0)
     reverse_markdown (1.3.0)
       nokogiri

data/README.md

@@ -61,6 +61,8 @@ The contents of both hashes are explained below.
 | `description` | optional | String  | auto-generated |                                            |
 | `ttl`         | optional | Integer |          `360` | TTL in _minutes_                           |
 | `time_zone`   | optional | String  |        `'UTC'` | TimeZone name                              |
+| `language`    | optional | String  |         `'en'` | Language code                              |
+| `author`      | optional | String  |                | Format: `email (Name)'`                    |
 | `headers`     | optional | Hash    |           `{}` | Set HTTP request headers. See notes below. |
 | `json`        | optional | Boolean |        `false` | Handle JSON response. See notes below.     |
 
@@ -413,10 +415,10 @@ Use this to e.g. have Cookie or Authorization information sent or to spoof the U
       url: 'https://example.com',
       title: "Example with http headers",
       headers: {
-        "User-Agent" => "html2rss-request",
-        "X-Something" => "Foobar",
-        "Authorization" => "Token deadbea7",
-        "Cookie" => "monster=MeWantCookie"
+        "User-Agent": "html2rss-request",
+        "X-Something": "Foobar",
+        "Authorization": "Token deadbea7",
+        "Cookie": "monster=MeWantCookie"
       }
     },
     selectors: {}

data/html2rss.gemspec

@@ -30,10 +30,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activesupport', '~> 5.0'
+  spec.add_dependency 'addressable', '~> 2.7'
   spec.add_dependency 'builder'
   spec.add_dependency 'faraday', '~> 0.15'
   spec.add_dependency 'faraday_middleware', '~> 0.13'
-  spec.add_dependency 'hashie', '~> 3.6'
   spec.add_dependency 'kramdown'
   spec.add_dependency 'mime-types', '> 3.0'
   spec.add_dependency 'nokogiri', '>= 1.10', '< 2.0'

data/lib/html2rss/attribute_post_processors/gsub.rb

@@ -27,8 +27,9 @@ module Html2rss
     class Gsub
       def initialize(value, env)
         @value = value
-        @pattern = env[:options]['pattern'].to_regexp || env[:options]['pattern']
-        @replacement = env[:options]['replacement']
+        options = env[:options]
+        @pattern = options[:pattern].to_regexp || options[:pattern]
+        @replacement = options[:replacement]
       end
 
       ##

data/lib/html2rss/attribute_post_processors/parse_uri.rb

@@ -26,7 +26,7 @@ module Html2rss
       ##
       # @return [String]
       def get
-        URI(@value.to_s.split(' ').join).to_s
+        URI(Html2rss::Utils.sanitize_url(@value)).to_s
       end
     end
   end

data/lib/html2rss/attribute_post_processors/substring.rb

@@ -34,8 +34,8 @@ module Html2rss
       ##
       # @return [String]
       def get
-        ending = @options.fetch('end', @value.length).to_i
-        @value[@options['start'].to_i..ending]
+        ending = @options.fetch(:end, @value.length).to_i
+        @value[@options[:start].to_i..ending]
       end
     end
   end

data/lib/html2rss/attribute_post_processors/template.rb

@@ -33,33 +33,36 @@ module Html2rss
         @value = value
         @options = env[:options]
         @item = env[:item]
+        @string = @options[:string]
       end
 
       ##
       # @return [String]
       def get
-        if @options['methods']
-          string % methods
-        else
-          names = string.scan(/%[<|{](\w*)[>|}]/).flatten
-          names.uniq!
+        return format_string_with_methods if @options[:methods]
 
-          format(string, names.map { |name| [name.to_sym, item_value(name)] }.to_h)
-        end
+        names = string.scan(/%[<|{](\w*)[>|}]/)
+        names.flatten!
+        names.compact!
+        names.map!(&:to_sym)
+
+        format(string, names.map { |name| [name, item_value(name)] }.to_h)
       end
 
       private
 
-      def string
-        @options['string']
-      end
+      attr_reader :string
 
       def methods
-        @methods ||= @options['methods'].map(&method(:item_value))
+        @methods ||= @options[:methods].map(&method(:item_value))
+      end
+
+      def format_string_with_methods
+        string % methods
       end
 
       def item_value(method_name)
-        method_name.to_s == 'self' ? @value.to_s : @item.public_send(method_name.to_sym).to_s
+        method_name.to_sym == :self ? @value.to_s : @item.public_send(method_name).to_s
       end
     end
   end

data/lib/html2rss/config.rb

@@ -1,79 +1,87 @@
+require 'active_support/core_ext/hash'
+
 module Html2rss
   ##
   # The Config class abstracts from the config data structure and
   # provides default values.
   class Config
     def initialize(feed_config, global_config = {})
-      @global_config = Utils::IndifferentAccessHash.new global_config
-      @feed_config = Utils::IndifferentAccessHash.new feed_config
-      @channel_config = Utils::IndifferentAccessHash.new @feed_config.fetch('channel', {})
+      @global_config = global_config.deep_symbolize_keys
+      @feed_config = feed_config.deep_symbolize_keys
+      @channel_config = @feed_config.fetch(:channel, {})
     end
 
     def author
-      channel_config.fetch 'author', 'html2rss'
+      channel_config.fetch :author, 'html2rss'
     end
 
     def ttl
-      channel_config.fetch 'ttl', 360
+      channel_config.fetch :ttl, 360
     end
 
     def title
-      channel_config.fetch 'title' do
-        uri = URI(url)
+      channel_config.fetch(:title) { generated_title }
+    end
+
+    def generated_title
+      uri = URI(url)
 
-        nicer_path = uri.path.split('/')
-        nicer_path.reject! { |p| p == '' }
-        nicer_path.map!(&:titleize)
+      nicer_path = uri.path.split('/')
+      nicer_path.reject! { |part| part == '' }
 
-        nicer_path.any? ? "#{uri.host}: #{nicer_path.join(' ')}" : uri.host
-      end
+      nicer_path.any? ? "#{uri.host}: #{nicer_path.join(' ').titleize}" : uri.host
     end
 
     def language
-      channel_config.fetch 'language', 'en'
+      channel_config.fetch :language, 'en'
     end
 
     def description
-      channel_config.fetch 'description', "Latest items from #{url}."
+      channel_config.fetch :description, "Latest items from #{url}."
     end
 
     def url
-      channel_config.dig 'url'
+      channel_config.dig :url
     end
     alias link url
 
     def time_zone
-      channel_config.fetch 'time_zone', 'UTC'
+      channel_config.fetch :time_zone, 'UTC'
     end
 
     def json?
-      channel_config.fetch 'json', false
+      channel_config.fetch :json, false
     end
 
     def headers
-      global_config.fetch('headers', {}).merge(channel_config.fetch('headers', {}))
+      global_config.fetch(:headers, {}).merge(channel_config.fetch(:headers, {}))
     end
 
     def attribute_options(name)
-      feed_config.dig('selectors').fetch(name, {}).merge('channel' => channel_config)
+      feed_config.dig(:selectors).fetch(name, {}).merge(channel: channel_config)
     end
 
     def attribute?(name)
-      attribute_names.include?(name.to_s)
+      attribute_names.include?(name)
     end
 
-    def categories
-      feed_config.dig('selectors').fetch('categories', []).map(&:to_sym)
+    def category_selectors
+      categories = feed_config.dig(:selectors, :categories)
+      return [] unless categories
+
+      categories = categories.keep_if { |category| category.to_s != '' }
+      categories.map!(&:to_sym)
+      categories.uniq!
+
+      categories
     end
 
     def selector(name)
-      feed_config.dig('selectors', name, 'selector')
+      feed_config.dig(:selectors, name, :selector)
     end
 
     def attribute_names
-      @attribute_names ||= feed_config.fetch('selectors', {}).keys.map(&:to_s).tap do |attrs|
-        attrs.delete('items')
-      end
+      @attribute_names ||= feed_config.fetch(:selectors, {}).keys.tap { |attrs| attrs.delete(:items) }
     end
 
     private

data/lib/html2rss/feed_builder.rb

@@ -18,55 +18,58 @@ module Html2rss
     # @return [RSS:Rss]
     def rss
       RSS::Maker.make('2.0') do |maker|
-        add_channel(maker)
+        add_channel(maker.channel)
 
-        feed_items.map { |feed_item| add_item(feed_item, maker.items.new_item) }
+        items.each { |item| add_item(item, maker.items.new_item) }
       end
     end
 
-    private
+    def self.add_categories(categories, item_maker)
+      categories.each { |category| item_maker.categories.new_category.content = category }
+    end
 
-    attr_reader :config
+    def self.add_enclosure_from_url(url, item_maker)
+      return unless url
 
-    def add_channel(maker)
-      %i[language author title description link ttl].each do |attribute_name|
-        maker.channel.public_send("#{attribute_name}=".to_sym, config.public_send(attribute_name))
-      end
+      enclosure = item_maker.enclosure
+      content_type = MIME::Types.type_for(File.extname(url).delete('.'))
 
-      maker.channel.generator = "html2rss V. #{::Html2rss::VERSION}"
-      maker.channel.lastBuildDate = Time.now.to_s
+      enclosure.type = content_type.any? ? content_type.first.to_s : 'application/octet-stream'
+      enclosure.length = 0
+      enclosure.url = url
     end
 
-    def feed_items
-      @feed_items ||= Item.from_url(config.url, config).keep_if(&:valid?)
+    def self.add_guid(item, item_maker)
+      guid = item_maker.guid
+      guid.content = Digest::SHA1.hexdigest(item.title)
+      guid.isPermaLink = false
     end
 
-    def add_item(feed_item, rss_item)
-      feed_item.available_attributes.each do |attribute_name|
-        rss_item.public_send("#{attribute_name}=".to_sym, feed_item.public_send(attribute_name))
-      end
+    private
 
-      feed_item.categories.each { |category| rss_item.categories.new_category.content = category }
-      add_enclosure_from_url(feed_item.enclosure_url, rss_item) if config.attribute?(:enclosure)
+    attr_reader :config
 
-      add_guid(feed_item, rss_item)
-    end
+    def add_channel(channel_maker)
+      %i[language author title description link ttl].each do |attribute_name|
+        channel_maker.public_send("#{attribute_name}=", config.public_send(attribute_name))
+      end
 
-    def add_enclosure_from_url(url, rss_item)
-      content_type = MIME::Types.type_for(File.extname(url).delete('.'))
+      channel_maker.generator = "html2rss V. #{::Html2rss::VERSION}"
+      channel_maker.lastBuildDate = Time.now
+    end
 
-      rss_item.enclosure.type = if content_type && content_type.first
-                                  content_type.first.to_s
-                                else
-                                  'application/octet-stream'
-                                end
-      rss_item.enclosure.length = 0
-      rss_item.enclosure.url = url
+    def items
+      @items ||= Item.from_url(config.url, config)
     end
 
-    def add_guid(feed_item, rss_item)
-      rss_item.guid.content = Digest::SHA1.hexdigest(feed_item.title)
-      rss_item.guid.isPermaLink = false
+    def add_item(item, item_maker)
+      item.available_attributes.each do |attribute_name|
+        item_maker.public_send("#{attribute_name}=", item.public_send(attribute_name))
+      end
+
+      self.class.add_categories(item.categories, item_maker)
+      self.class.add_enclosure_from_url(item.enclosure_url, item_maker) if item.enclosure?
+      self.class.add_guid(item, item_maker)
     end
   end
 end

data/lib/html2rss/item.rb

@@ -23,15 +23,15 @@ module Html2rss
 
       attribute_options = config.attribute_options(method_name)
 
-      extractor = ItemExtractors.get_extractor(attribute_options['extractor'])
+      extractor = ItemExtractors.get_extractor(attribute_options[:extractor])
       value = extractor.new(xml, attribute_options).get
 
-      post_process(value, attribute_options.fetch('post_process', false))
+      post_process(value, attribute_options.fetch(:post_process, false))
     end
 
     def available_attributes
-      @available_attributes ||= (%w[title link description author comments updated] &
-        @config.attribute_names) - %w[categories enclosure]
+      @available_attributes ||= (%i[title link description author comments updated] &
+        @config.attribute_names) - %i[categories enclosure]
     end
 
     ##
@@ -45,17 +45,17 @@ module Html2rss
     ##
     # @return [Array]
     def categories
-      categories = config.categories
-      categories.map!(&method(:method_missing))
-      categories.uniq!
-      categories.keep_if { |category| category.to_s != '' }
+      config.category_selectors.map(&method(:method_missing))
+    end
+
+    def enclosure?
+      config.attribute?(:enclosure)
     end
 
     def enclosure_url
-      enclosure = method_missing(:enclosure)
-      return if enclosure.to_s == ''
+      enclosure = Html2rss::Utils.sanitize_url(method_missing(:enclosure))
 
-      Html2rss::Utils.build_absolute_url_from_relative(enclosure, config.url).to_s
+      Html2rss::Utils.build_absolute_url_from_relative(enclosure, config.url).to_s if enclosure
     end
 
     ##
@@ -63,7 +63,9 @@ module Html2rss
     def self.from_url(url, config)
       body = get_body_from_url(url, config)
 
-      Nokogiri.HTML(body).css(config.selector('items')).map { |xml_item| new xml_item, config }
+      Nokogiri.HTML(body).css(config.selector(:items))
+              .map { |xml_item| new xml_item, config }
+              .keep_if(&:valid?)
     end
 
     private
@@ -74,7 +76,7 @@ module Html2rss
         faraday.adapter Faraday.default_adapter
       end.get.body
 
-      config.json? ? Html2rss::Utils.hash_to_xml(JSON.parse(body)) : body
+      config.json? ? Html2rss::Utils.object_to_xml(JSON.parse(body)) : body
     end
     private_class_method :get_body_from_url
 
@@ -84,7 +86,7 @@ module Html2rss
       return value unless post_process_options
 
       [post_process_options].flatten.each do |options|
-        value = AttributePostProcessors.get_processor(options['name'])
+        value = AttributePostProcessors.get_processor(options[:name])
                                        .new(value, options: options, item: self, config: @config)
                                        .get
       end

data/lib/html2rss/item_extractors.rb

@@ -15,7 +15,7 @@ module Html2rss
     ##
     # @return [Nokogiri::XML::Element]
     def self.element(xml, options)
-      selector = options['selector']
+      selector = options[:selector]
       selector ? xml.css(selector) : xml
     end
   end

data/lib/html2rss/item_extractors/attribute.rb

@@ -30,7 +30,7 @@ module Html2rss
       ##
       # @return [String]
       def get
-        @element.attr(@options['attribute']).to_s
+        @element.attr(@options[:attribute]).to_s
       end
     end
   end

data/lib/html2rss/item_extractors/href.rb

@@ -24,12 +24,12 @@ module Html2rss
       def initialize(xml, options)
         @options = options
         element = ItemExtractors.element(xml, options)
-        @href = element.attr('href').to_s
+        @href = Html2rss::Utils.sanitize_url(element.attr('href'))
       end
 
       # @return [URI::HTTPS, URI::HTTP]
       def get
-        Html2rss::Utils.build_absolute_url_from_relative(@href, @options['channel']['url'])
+        Html2rss::Utils.build_absolute_url_from_relative(@href, @options[:channel][:url])
       end
     end
   end

data/lib/html2rss/item_extractors/static.rb

@@ -15,13 +15,13 @@ module Html2rss
         @options = options
       end
 
-      # Returns what options['static'] holds.
+      # Returns what options[:static] holds.
       #
       #    options = { static: 'Foobar' }
       #    Static.new(xml, options).get
       #    # => 'Foobar'
       def get
-        @options['static']
+        @options[:static]
       end
     end
   end

data/lib/html2rss/utils.rb

@@ -1,6 +1,6 @@
 require 'active_support/core_ext/hash'
+require 'addressable/uri'
 require 'builder'
-require 'hashie'
 require 'json'
 require 'nokogiri'
 
@@ -8,27 +8,20 @@ module Html2rss
   ##
   # The collecting tank for utility methods.
   module Utils
-    ##
-    # A Hash with indifferent access, build with {https://github.com/intridea/hashie Hashie}.
-    class IndifferentAccessHash < Hash
-      include Hashie::Extensions::MergeInitializer
-      include Hashie::Extensions::IndifferentAccess
-    end
-
-    def self.build_absolute_url_from_relative(url, channel_url)
+    def self.build_absolute_url_from_relative(url, base_url)
       url = URI(url) if url.is_a?(String)
 
       return url if url.absolute?
 
-      URI(channel_url).tap do |uri|
+      URI(base_url).tap do |uri|
         uri.path = url.path.to_s.start_with?('/') ? url.path : "/#{url.path}"
         uri.query = url.query
         uri.fragment = url.fragment if url.fragment
       end
     end
 
-    def self.hash_to_xml(hash)
-      hash.to_xml(skip_instruct: true, skip_types: true)
+    def self.object_to_xml(object)
+      object.to_xml(skip_instruct: true, skip_types: true)
     end
 
     def self.get_class_from_name(snake_cased_name, module_name)
@@ -36,5 +29,12 @@ module Html2rss
       class_name = ['Html2rss', module_name, camel_cased_name].join('::')
       Object.const_get(class_name)
     end
+
+    def self.sanitize_url(url)
+      squished_url = url.to_s.split(' ').join
+      return if squished_url.to_s == ''
+
+      Addressable::URI.parse(squished_url).normalize.to_s
+    end
   end
 end

data/lib/html2rss/version.rb

@@ -1,3 +1,3 @@
 module Html2rss
-  VERSION = '0.8.1'.freeze
+  VERSION = '0.8.2'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.2
 platform: ruby
 authors:
 - Gil Desmarais
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-08 00:00:00.000000000 Z
+date: 2019-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: builder
   requirement: !ruby/object:Gem::Requirement
@@ -66,20 +80,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.13'
-- !ruby/object:Gem::Dependency
-  name: hashie
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.6'
 - !ruby/object:Gem::Dependency
   name: kramdown
   requirement: !ruby/object:Gem::Requirement