html2rss 0.6.0 → 0.9.0

data/lib/html2rss/feed_builder.rb

@@ -1,5 +1,5 @@
 require 'rss'
-require_relative 'item'
+require 'mime/types'
 
 module Html2rss
   ##
@@ -18,42 +18,64 @@ module Html2rss
     # @return [RSS:Rss]
     def rss
       RSS::Maker.make('2.0') do |maker|
-        add_channel(maker)
+        add_channel(maker.channel)
 
-        feed_items.map do |feed_item|
-          add_item(feed_item, maker.items.new_item)
-        end
+        items.each { |item| add_item(item, maker.items.new_item) }
       end
     end
 
+    def self.add_categories(categories, item_maker)
+      categories.each { |category| item_maker.categories.new_category.content = category }
+    end
+
+    def self.add_enclosure_from_url(url, item_maker)
+      return unless url
+
+      enclosure = item_maker.enclosure
+      content_type = MIME::Types.type_for(File.extname(url).delete('.'))
+
+      enclosure.type = content_type.any? ? content_type.first.to_s : 'application/octet-stream'
+      enclosure.length = 0
+      enclosure.url = url
+    end
+
+    def self.add_guid(item, item_maker)
+      guid = item_maker.guid
+      guid.content = Digest::SHA1.hexdigest(item.title)
+      guid.isPermaLink = false
+    end
+
     private
 
     attr_reader :config
 
-    def add_channel(maker)
+    def add_channel(channel_maker)
       %i[language author title description link ttl].each do |attribute_name|
-        maker.channel.public_send("#{attribute_name}=".to_sym, config.public_send(attribute_name))
+        channel_maker.public_send("#{attribute_name}=", config.public_send(attribute_name))
       end
 
-      maker.channel.generator = "html2rss V. #{::Html2rss::VERSION}"
-      maker.channel.lastBuildDate = Time.now.to_s
+      channel_maker.generator = "html2rss V. #{::Html2rss::VERSION}"
+      channel_maker.lastBuildDate = Time.now
     end
 
-    def feed_items
-      @feed_items ||= Item.from_url(config.url, config).keep_if(&:valid?)
-    end
+    def items
+      return @items if defined?(@items)
 
-    def add_item(feed_item, rss_item)
-      feed_item.available_attributes.each do |attribute_name|
-        rss_item.public_send("#{attribute_name}=".to_sym, feed_item.public_send(attribute_name))
-      end
+      items = Item.from_url(config.url, config)
+
+      items.reverse! if config.items_order == :reverse
+
+      @items = items
+    end
 
-      feed_item.categories.each do |category|
-        rss_item.categories.new_category.content = category
+    def add_item(item, item_maker)
+      item.available_attributes.each do |attribute_name|
+        item_maker.public_send("#{attribute_name}=", item.public_send(attribute_name))
       end
 
-      rss_item.guid.content = Digest::SHA1.hexdigest(feed_item.title)
-      rss_item.guid.isPermaLink = false
+      self.class.add_categories(item.categories, item_maker)
+      self.class.add_enclosure_from_url(item.enclosure_url, item_maker) if item.enclosure?
+      self.class.add_guid(item, item_maker)
     end
   end
 end

data/lib/html2rss/item.rb

@@ -1,9 +1,6 @@
 require 'faraday'
 require 'faraday_middleware'
-require 'open-uri'
 require 'nokogiri'
-require_relative 'item_extractors'
-require_relative 'attribute_post_processors'
 
 module Html2rss
   ##
@@ -26,15 +23,15 @@ module Html2rss
 
       attribute_options = config.attribute_options(method_name)
 
-      extractor = ItemExtractors.get_extractor(attribute_options['extractor'])
+      extractor = ItemExtractors.get_extractor(attribute_options[:extractor])
       value = extractor.new(xml, attribute_options).get
 
-      post_process(value, attribute_options.fetch('post_process', false))
+      post_process(value, attribute_options.fetch(:post_process, false))
     end
 
     def available_attributes
-      @available_attributes ||= (%w[title link description author comments updated] &
-        @config.attribute_names) - ['categories']
+      @available_attributes ||= (%i[title link description author comments updated] &
+        @config.attribute_names) - %i[categories enclosure]
     end
 
     ##
@@ -48,10 +45,17 @@ module Html2rss
     ##
     # @return [Array]
     def categories
-      categories = config.categories
-      categories.map!(&method(:method_missing))
-      categories.uniq!
-      categories.keep_if { |category| category.to_s != '' }
+      config.category_selectors.map(&method(:method_missing))
+    end
+
+    def enclosure?
+      config.attribute?(:enclosure)
+    end
+
+    def enclosure_url
+      enclosure = Html2rss::Utils.sanitize_url(method_missing(:enclosure))
+
+      Html2rss::Utils.build_absolute_url_from_relative(enclosure, config.url).to_s if enclosure
     end
 
     ##
@@ -59,20 +63,22 @@ module Html2rss
     def self.from_url(url, config)
       body = get_body_from_url(url, config)
 
-      Nokogiri::HTML(body).css(config.selector('items')).map do |xml_item|
-        new xml_item, config
-      end
+      Nokogiri.HTML(body).css(config.selector(:items))
+              .map { |xml_item| new xml_item, config }
+              .keep_if(&:valid?)
     end
 
     private
 
     def self.get_body_from_url(url, config)
-      body = Faraday.new(url: url, headers: config.headers) do |faraday|
+      request = Faraday.new(url: url, headers: config.headers) do |faraday|
         faraday.use FaradayMiddleware::FollowRedirects
         faraday.adapter Faraday.default_adapter
-      end.get.body
+      end
+
+      body = request.get.body
 
-      config.json? ? Html2rss::Utils.hash_to_xml(JSON.parse(body)) : body
+      config.json? ? Html2rss::Utils.object_to_xml(JSON.parse(body)) : body
     end
     private_class_method :get_body_from_url
 
@@ -82,7 +88,7 @@ module Html2rss
       return value unless post_process_options
 
       [post_process_options].flatten.each do |options|
-        value = AttributePostProcessors.get_processor(options['name'])
+        value = AttributePostProcessors.get_processor(options[:name])
                                        .new(value, options: options, item: self, config: @config)
                                        .get
       end

data/lib/html2rss/item_extractors.rb

@@ -1,29 +1,22 @@
-require_relative 'item_extractors/attribute'
-require_relative 'item_extractors/current_time'
-require_relative 'item_extractors/href'
-require_relative 'item_extractors/html'
-require_relative 'item_extractors/static'
-require_relative 'item_extractors/text'
-
 module Html2rss
   ##
   # Provides a namespace for item extractors.
   module ItemExtractors
     DEFAULT = 'text'.freeze
+    private_constant :DEFAULT
 
     def self.get_extractor(name)
-      @extractors = Hash.new do |hash, key|
-        camel_cased_name = key.split('_').map(&:capitalize).join
-        class_name = ['Html2rss', 'ItemExtractors', camel_cased_name].join('::')
+      @get_extractor ||= Hash.new do |extractors, key|
+        extractors[key] = Utils.get_class_from_name(key || DEFAULT, 'ItemExtractors')
+      end
 
-        hash[key] = Object.const_get(class_name)
-      end[name || DEFAULT]
+      @get_extractor[name]
     end
 
     ##
     # @return [Nokogiri::XML::Element]
     def self.element(xml, options)
-      selector = options['selector']
+      selector = options[:selector]
       selector ? xml.css(selector) : xml
     end
   end

data/lib/html2rss/item_extractors/attribute.rb

@@ -30,7 +30,7 @@ module Html2rss
       ##
       # @return [String]
       def get
-        @element.attr(@options['attribute']).to_s
+        @element.attr(@options[:attribute]).to_s
       end
     end
   end

data/lib/html2rss/item_extractors/href.rb

@@ -24,12 +24,12 @@ module Html2rss
       def initialize(xml, options)
         @options = options
         element = ItemExtractors.element(xml, options)
-        @href = element.attr('href').to_s
+        @href = Html2rss::Utils.sanitize_url(element.attr('href'))
       end
 
       # @return [URI::HTTPS, URI::HTTP]
       def get
-        Html2rss::Utils.build_absolute_url_from_relative(@href, @options['channel']['url'])
+        Html2rss::Utils.build_absolute_url_from_relative(@href, @options[:channel][:url])
       end
     end
   end

data/lib/html2rss/item_extractors/static.rb

@@ -15,13 +15,13 @@ module Html2rss
         @options = options
       end
 
-      # Returns what options['static'] holds.
+      # Returns what options[:static] holds.
       #
       #    options = { static: 'Foobar' }
       #    Static.new(xml, options).get
       #    # => 'Foobar'
       def get
-        @options['static']
+        @options[:static]
       end
     end
   end

data/lib/html2rss/utils.rb

@@ -1,6 +1,6 @@
 require 'active_support/core_ext/hash'
+require 'addressable/uri'
 require 'builder'
-require 'hashie'
 require 'json'
 require 'nokogiri'
 
@@ -8,27 +8,33 @@ module Html2rss
   ##
   # The collecting tank for utility methods.
   module Utils
-    ##
-    # A Hash with indifferent access, build with {https://github.com/intridea/hashie Hashie}.
-    class IndifferentAccessHash < Hash
-      include Hashie::Extensions::MergeInitializer
-      include Hashie::Extensions::IndifferentAccess
-    end
-
-    def self.build_absolute_url_from_relative(url, channel_url)
+    def self.build_absolute_url_from_relative(url, base_url)
       url = URI(url) if url.is_a?(String)
 
       return url if url.absolute?
 
-      URI(channel_url).tap do |uri|
+      URI(base_url).tap do |uri|
         uri.path = url.path.to_s.start_with?('/') ? url.path : "/#{url.path}"
         uri.query = url.query
         uri.fragment = url.fragment if url.fragment
       end
     end
 
-    def self.hash_to_xml(hash)
-      hash.to_xml(root: :html, skip_instruct: true, skip_types: true)
+    def self.object_to_xml(object)
+      object.to_xml(skip_instruct: true, skip_types: true)
+    end
+
+    def self.get_class_from_name(snake_cased_name, module_name)
+      camel_cased_name = snake_cased_name.split('_').map(&:capitalize).join
+      class_name = ['Html2rss', module_name, camel_cased_name].join('::')
+      Object.const_get(class_name)
+    end
+
+    def self.sanitize_url(url)
+      squished_url = url.to_s.split(' ').join
+      return if squished_url.to_s == ''
+
+      Addressable::URI.parse(squished_url).normalize.to_s
     end
   end
 end

data/lib/html2rss/version.rb

@@ -1,3 +1,4 @@
 module Html2rss
-  VERSION = '0.6.0'.freeze
+  VERSION = '0.9.0'.freeze
+  public_constant :VERSION
 end

metadata

@@ -1,29 +1,49 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Gil Desmarais
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-05 00:00:00.000000000 Z
+date: 2020-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: addressable
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: builder
   requirement: !ruby/object:Gem::Requirement
@@ -44,42 +64,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: hashie
+  name: kramdown
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -106,14 +140,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: sanitize
   requirement: !ruby/object:Gem::Requirement
@@ -128,20 +162,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: to_regexp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -277,7 +339,9 @@ files:
 - html2rss.gemspec
 - lib/html2rss.rb
 - lib/html2rss/attribute_post_processors.rb
+- lib/html2rss/attribute_post_processors/gsub.rb
 - lib/html2rss/attribute_post_processors/html_to_markdown.rb
+- lib/html2rss/attribute_post_processors/markdown_to_html.rb
 - lib/html2rss/attribute_post_processors/parse_time.rb
 - lib/html2rss/attribute_post_processors/parse_uri.rb
 - lib/html2rss/attribute_post_processors/sanitize_html.rb
@@ -310,14 +374,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubyforge_project: 
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Returns an RSS::Rss object by scraping a URL.