html2rss 0.5.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1590d66348bc64be15db1b614963cf7d9032a438fca94590cb6e1f70bf6149ed
-  data.tar.gz: 371f1459e0a623b9b62c7cd73f94536762cb7e0abbde2e63a836f82dd06742a2
+  metadata.gz: fdb35d5375dda349c92fb7ccaed71747de5e49b2adfb3a1cfca9a9c251aa86a9
+  data.tar.gz: 6532e1df3c67108c6ae8bf7eaef7a6d5eb228c4d3020f99aa02f8a9d507a625e
 SHA512:
-  metadata.gz: 660f15157d054016e1dc91269d8686d85359e1b9b83cad69c47accb7d3be0e2b8a168802f9d29c8b213c0a4443822c862a9c461fc6e40284371c405ec6425f92
-  data.tar.gz: 36943075ad772b28f7e4c3b8d4a9d1faef6fb0c61ceea7be7dc177d90fc43839d81e44d2cbc215acdaad476d250d584d612a0c10d201fded859e30488f353eb6
+  metadata.gz: 86fbfe46d9ec7b0f21bc0e756da94ed96ecfa635f858336424034875a646f81c08e64610bf8b478d9e0212ea272ea06ee13d6bf33c32cf01dd25363e4ba44868
+  data.tar.gz: dfad9fc0ba59cc8ecd323299b330c5cc6831336da7fd913f4f72a52ee807861d4b604927e73477f33e353314f2107477457cfa4f320024cd63c19e4b0a577cd5

data/.rubocop.yml

@@ -22,7 +22,7 @@ Metrics/MethodLength:
   Max: 15
 
 Documentation:
-  Enabled: false
+  Enabled: true
 
 Style/BlockDelimiters:
   Enabled: true

data/.travis.yml

@@ -9,11 +9,12 @@ before_install:
 bundler_args: "--jobs=3 --retry=3"
 
 rvm:
-  - 2.4.7
-  - 2.5.6
-  - 2.6.4
+  - 2.4.9
+  - 2.5.7
+  - 2.6.5
 
 script:
+  - bundle exec rubocop -F
   - bundle exec rspec
 
 deploy:

data/CHANGELOG.md

@@ -1,4 +1,20 @@
-# [](https://github.com/gildesmarais/html2rss/compare/v0.5.2...v) (2019-09-19)
+# [](https://github.com/gildesmarais/html2rss/compare/v0.6.0...v) (2019-10-05)
+
+
+
+# [0.6.0](https://github.com/gildesmarais/html2rss/compare/v0.5.2...v0.6.0) (2019-10-05)
+
+
+### Bug Fixes
+
+* **specs:** simplecov does not exclude files from spec/ ([#44](https://github.com/gildesmarais/html2rss/issues/44)) ([b0ca780](https://github.com/gildesmarais/html2rss/commit/b0ca780))
+
+
+### Features
+
+* **ci:** run rubocop on ci ([#40](https://github.com/gildesmarais/html2rss/issues/40)) ([f4ec8d1](https://github.com/gildesmarais/html2rss/commit/f4ec8d1))
+* memoize ItemExtractor lookups ([#45](https://github.com/gildesmarais/html2rss/issues/45)) ([e88321c](https://github.com/gildesmarais/html2rss/commit/e88321c))
+* support setting of request headers in feed config ([#41](https://github.com/gildesmarais/html2rss/issues/41)) ([a7aca11](https://github.com/gildesmarais/html2rss/commit/a7aca11)), closes [#38](https://github.com/gildesmarais/html2rss/issues/38)
 
 
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    html2rss (0.5.2)
+    html2rss (0.6.0)
       activesupport (~> 5.0)
       builder
       faraday (~> 0.15)
@@ -26,24 +26,24 @@ GEM
     crass (1.0.4)
     diff-lcs (1.3)
     docile (1.3.2)
-    faraday (0.15.4)
+    faraday (0.16.2)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.13.1)
       faraday (>= 0.7.4, < 1.0)
     hashie (3.6.0)
-    i18n (1.6.0)
+    i18n (1.7.0)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.3)
     json (2.2.0)
     mini_portile2 (2.4.0)
-    minitest (5.11.3)
+    minitest (5.12.2)
     multipart-post (2.1.1)
     nokogiri (1.10.4)
       mini_portile2 (~> 2.4.0)
     nokogumbo (2.0.1)
       nokogiri (~> 1.8, >= 1.8.4)
     parallel (1.17.0)
-    parser (2.6.4.1)
+    parser (2.6.5.0)
       ast (~> 2.4.0)
     rainbow (3.0.0)
     reverse_markdown (1.3.0)
@@ -54,24 +54,24 @@ GEM
       rspec-mocks (~> 3.8.0)
     rspec-core (3.8.2)
       rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.4)
+    rspec-expectations (3.8.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.1)
+    rspec-mocks (3.8.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-support (3.8.2)
-    rubocop (0.74.0)
+    rspec-support (3.8.3)
+    rubocop (0.75.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
       parser (>= 2.6)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-performance (1.4.1)
+    rubocop-performance (1.5.0)
       rubocop (>= 0.71.0)
-    rubocop-rspec (1.35.0)
-      rubocop (>= 0.60.0)
+    rubocop-rspec (1.36.0)
+      rubocop (>= 0.68.1)
     ruby-progressbar (1.10.1)
     sanitize (5.1.0)
       crass (~> 1.0.2)

data/README.md

@@ -48,7 +48,17 @@ Since 0.5.0 it is possible to scrape and process JSON.
 
 Adding `json: true` to the channel config will convert the JSON response to XML.
 
-Example:
+Feed config:
+
+```yaml
+channel:
+  url: https://example.com
+  title: "Example with JSON"
+  json: true
+# ...
+```
+
+Imagine this HTTP response:
 
 ```json
 {
@@ -73,6 +83,25 @@ Your items selector would be `data > datum`, the item's link selector would be `
 
 Under the hood it uses ActiveSupport's [`Hash.to_xml`](https://apidock.com/rails/Hash/to_xml) core extension for the JSON to XML conversion.
 
+## Set any HTTP header in the request
+
+You can add any HTTP headers to the request to the channel URL.
+You can use this to e.g. have Cookie or Authorization information being sent or to overwrite the User-Agent.
+
+```yaml
+channel:
+  url: https://example.com
+  title: "Example with http headers"
+  headers:
+    "User-Agent": "html2rss-request"
+    "X-Something": "Foobar"
+    "Authorization": "Token deadbea7"
+    "Cookie": "monster=MeWantCookie"
+# ...
+```
+
+The headers provided by the channel will be merged into the global headers.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/html2rss.gemspec

@@ -12,9 +12,11 @@ Gem::Specification.new do |spec|
   spec.description   = 'Give the URL to scrape and some CSS selectors. Get a RSS::Rss instance in return.'
   spec.homepage      = 'https://github.com/gildesmarais/html2rss'
   spec.license       = 'MIT'
+  spec.required_ruby_version = '>= 2.4.0'
 
   if spec.respond_to?(:metadata)
     spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+    spec.metadata['changelog_uri'] = 'https://github.com/gildesmarais/html2rss/blob/master/CHANGELOG.md'
   else
     raise 'RubyGems 2.0 or newer is required to protect against ' \
       'public gem pushes.'

data/lib/html2rss.rb

@@ -4,6 +4,9 @@ require 'html2rss/version'
 require 'html2rss/utils'
 require 'yaml'
 
+##
+# The Html2rss namespace.
+# Request HTML from an URL and transform it to a RSS 2.0 object.
 module Html2rss
   ##
   # Returns a RSS object which is generated from the provided file.

data/lib/html2rss/config.rb

@@ -1,4 +1,7 @@
 module Html2rss
+  ##
+  # The Config class abstracts from the config data structure and
+  # provides default values.
   class Config
     def initialize(feed_config, global_config = {})
       @global_config = Utils::IndifferentAccessHash.new global_config
@@ -40,7 +43,7 @@ module Html2rss
     end
 
     def headers
-      global_config.fetch('headers', {})
+      global_config.fetch('headers', {}).merge(channel_config.fetch('headers', {}))
     end
 
     def attribute_options(name)

data/lib/html2rss/feed_builder.rb

@@ -2,6 +2,13 @@ require 'rss'
 require_relative 'item'
 
 module Html2rss
+  ##
+  # The purpose is to build the feed, consisting of
+  #
+  # - the 'channel' and
+  # - the 'item'
+  #
+  # parts.
   class FeedBuilder
     def initialize(config)
       @config = config
@@ -11,10 +18,10 @@ module Html2rss
     # @return [RSS:Rss]
     def rss
       RSS::Maker.make('2.0') do |maker|
-        add_channel_to_maker(maker)
+        add_channel(maker)
 
         feed_items.map do |feed_item|
-          add_item_to_items(feed_item, maker.items)
+          add_item(feed_item, maker.items.new_item)
         end
       end
     end
@@ -23,7 +30,7 @@ module Html2rss
 
     attr_reader :config
 
-    def add_channel_to_maker(maker)
+    def add_channel(maker)
       %i[language author title description link ttl].each do |attribute_name|
         maker.channel.public_send("#{attribute_name}=".to_sym, config.public_send(attribute_name))
       end
@@ -33,24 +40,20 @@ module Html2rss
     end
 
     def feed_items
-      @feed_items ||= Item.from_url config.url, config
+      @feed_items ||= Item.from_url(config.url, config).keep_if(&:valid?)
     end
 
-    def add_item_to_items(feed_item, items)
-      return unless feed_item.valid?
-
-      items.new_item do |rss_item|
-        feed_item.available_attributes.each do |attribute_name|
-          rss_item.public_send("#{attribute_name}=".to_sym, feed_item.public_send(attribute_name))
-        end
-
-        feed_item.categories.each do |category|
-          rss_item.categories.new_category.content = category
-        end
+    def add_item(feed_item, rss_item)
+      feed_item.available_attributes.each do |attribute_name|
+        rss_item.public_send("#{attribute_name}=".to_sym, feed_item.public_send(attribute_name))
+      end
 
-        rss_item.guid.content = Digest::SHA1.hexdigest(feed_item.title)
-        rss_item.guid.isPermaLink = false
+      feed_item.categories.each do |category|
+        rss_item.categories.new_category.content = category
       end
+
+      rss_item.guid.content = Digest::SHA1.hexdigest(feed_item.title)
+      rss_item.guid.isPermaLink = false
     end
   end
 end

data/lib/html2rss/item.rb

@@ -6,6 +6,9 @@ require_relative 'item_extractors'
 require_relative 'attribute_post_processors'
 
 module Html2rss
+  ##
+  # Takes the selected Nokogiri::HTML and responds to accessors names
+  # defined in the feed config.
   class Item
     def initialize(xml, config)
       @xml = xml

data/lib/html2rss/item_extractors.rb

@@ -12,11 +12,12 @@ module Html2rss
     DEFAULT = 'text'.freeze
 
     def self.get_extractor(name)
-      name ||= DEFAULT
-      camel_cased_name = name.split('_').map(&:capitalize).join
-      class_name = ['Html2rss', 'ItemExtractors', camel_cased_name].join('::')
+      @extractors = Hash.new do |hash, key|
+        camel_cased_name = key.split('_').map(&:capitalize).join
+        class_name = ['Html2rss', 'ItemExtractors', camel_cased_name].join('::')
 
-      Object.const_get(class_name)
+        hash[key] = Object.const_get(class_name)
+      end[name || DEFAULT]
     end
 
     ##

data/lib/html2rss/utils.rb

@@ -5,6 +5,8 @@ require 'json'
 require 'nokogiri'
 
 module Html2rss
+  ##
+  # The collecting tank for utility methods.
   module Utils
     ##
     # A Hash with indifferent access, build with {https://github.com/intridea/hashie Hashie}.

data/lib/html2rss/version.rb

@@ -1,3 +1,3 @@
 module Html2rss
-  VERSION = '0.5.2'.freeze
+  VERSION = '0.6.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Gil Desmarais
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-09-19 00:00:00.000000000 Z
+date: 2019-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -301,6 +301,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
+  changelog_uri: https://github.com/gildesmarais/html2rss/blob/master/CHANGELOG.md
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -309,15 +310,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Returns an RSS::Rss object by scraping a URL.