html2rss 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d23c5f38ec394aa7a095ed72fe438bbb7480e56e3f1466afccf8b15640ad3ea
-  data.tar.gz: 9cf409fb2b00a21964e91ee81882dc626b47a9a37a5dd7817b59048197f44e45
+  metadata.gz: 3d7642c21f46ea6086a3ead39b2923fa4c09ddc821719e2c40936846a83e5438
+  data.tar.gz: 8d6efa4ed03261f11e9ec302ea0cda860e83b74d78203c48b6650f1a7d9c106c
 SHA512:
-  metadata.gz: 4650b3337acea8b93a0d73279e401c3790b67ff9e311e21782e1a892e58d5dd156ce7c649e961b36031d046d4c272042c5ecba6e185fc91a42718ef345aea40c
-  data.tar.gz: 76a3d9517ea0f5e6f6f12b1de803cbe1bc1cc5b7f5401a8bf5c3cf3de544b3580ab23bfe9be8964e5fb5b1da8b35184ce1bfea418808c447812b8d99def8897c
+  metadata.gz: f6f1a3c52d9689b18d04a7b0c772ab0b9742202ca2e292b8265f7d26d85989f7201da9ab3de2db81258b82b8271c5789176976ea6c67c42edcb4739a556386f3
+  data.tar.gz: 8d892ec1d5761d6c5338e77f36a86debb2ccc1f1f7bac98ba83489c2aa8a7fe0638a45d355ff6fbd295a417d4df25f9ffc932fd15dd70f5465b0b61ee566cec0

data/.rubocop.yml

@@ -1,3 +1,6 @@
+require:
+  - 'rubocop-performance'
+
 AllCops:
   TargetRubyVersion: 2.4
   DisplayCopNames: true

data/.travis.yml

@@ -9,10 +9,9 @@ before_install:
 bundler_args: "--jobs=3 --retry=3"
 
 rvm:
-  - 2.3.8
-  - 2.4.5
-  - 2.5.3
-  - 2.6.1
+  - 2.4.6
+  - 2.5.4
+  - 2.6.3
 
 script:
   - bundle exec rspec

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    html2rss (0.2.1)
+    html2rss (0.3.0)
       faraday (~> 0.15)
       faraday_middleware (~> 0.13)
       nokogiri (>= 1.10)
@@ -10,34 +10,50 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    byebug (10.0.2)
+    ast (2.4.0)
+    byebug (11.0.1)
     crass (1.0.4)
     diff-lcs (1.3)
-    docile (1.3.1)
+    docile (1.3.2)
     faraday (0.15.4)
       multipart-post (>= 1.2, < 3)
-    faraday_middleware (0.13.0)
+    faraday_middleware (0.13.1)
       faraday (>= 0.7.4, < 1.0)
-    json (2.1.0)
+    jaro_winkler (1.5.3)
+    json (2.2.0)
     mini_portile2 (2.4.0)
-    multipart-post (2.0.0)
-    nokogiri (1.10.1)
+    multipart-post (2.1.1)
+    nokogiri (1.10.3)
       mini_portile2 (~> 2.4.0)
     nokogumbo (2.0.1)
       nokogiri (~> 1.8, >= 1.8.4)
+    parallel (1.17.0)
+    parser (2.6.3.0)
+      ast (~> 2.4.0)
+    rainbow (3.0.0)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
+    rspec-core (3.8.1)
       rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.2)
+    rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+    rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
+    rspec-support (3.8.2)
+    rubocop (0.71.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.6)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    rubocop-performance (1.4.0)
+      rubocop (>= 0.71.0)
+    ruby-progressbar (1.10.1)
     sanitize (5.0.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
@@ -47,18 +63,21 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    vcr (4.0.0)
+    unicode-display_width (1.6.0)
+    vcr (5.0.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 1.16)
-  byebug (~> 10.0)
+  byebug
   html2rss!
   rspec (~> 3.0)
+  rubocop
+  rubocop-performance
   simplecov
-  vcr (~> 4.0)
+  vcr
 
 BUNDLED WITH
-   1.16.6
+   1.17.2

data/README.md

@@ -1,34 +1,23 @@
 ![html2rss logo](https://github.com/gildesmarais/html2rss/raw/master/support/logo.png)
 
-# html2rss [![Build Status](https://travis-ci.org/gildesmarais/html2rss.svg?branch=master)](https://travis-ci.org/gildesmarais/html2rss)
+# html2rss [![Build Status](https://travis-ci.org/gildesmarais/html2rss.svg?branch=master)](https://travis-ci.org/gildesmarais/html2rss) [![Gem Version](https://badge.fury.io/rb/html2rss.svg)](https://badge.fury.io/rb/html2rss)
 
-Request and convert an HTML document to an RSS feed via a config object.
-The config contains the URL to scrape and the selectors needed to extract
-the required information. This gem provides some extractors (e.g. extract
-the information from an HTML attribute).
+Request HTML from an URL and transform it to a Ruby RSS 2.0 object.
 
-Please always check the website's Terms of Service before if its allowed to
-scrape their content!
+**Are you searching for a ready to use "website to RSS" solution?**
+[Check out `html2rss-web`!](https://github.com/gildesmarais/html2rss-web)
 
-## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'html2rss'
-```
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
+Each website needs a feed config which contains the URL to scrape and
+CSS selectors to extract the required information (like title, URL, ...).
+This gem provides [extractors](https://github.com/gildesmarais/html2rss/blob/master/lib/html2rss/item_extractor.rb) (e.g. extract the information from an HTML attribute)
+and [post processors](https://github.com/gildesmarais/html2rss/tree/master/lib/html2rss/attribute_post_processors) to make information retrieval even easier.
 
-    $ gem install html2rss
+## Installation
 
-## Usage
+Add this line to your application's Gemfile: `gem 'html2rss'`
+And then execute: `bundle`
 
-## Usage with a YAML file
+## Usage with a YAML config file
 
 Create a YAML config file. Find an example at [`rspec/config.test.yml`](https://github.com/gildesmarais/html2rss/blob/master/spec/config.test.yml).
 
@@ -36,15 +25,7 @@ Create a YAML config file. Find an example at [`rspec/config.test.yml`](https://
 
 an `RSS:Rss` object.
 
-## Usage in a web application
-
-Find a minimal Sintra app which exposes your feeds to HTTP endpoints here:
-[gildesmarais/html2rss-web](https://github.com/gildesmarais/html2rss-web)
-
-### Tips and tricks
-
-- Check that the channel url does not redirect to a mobile page
-- fiddling with [`curl`](https://github.com/curl/curl) and [`pup`](https://github.com/ericchiang/pup) to find the selectors seems quite efficient
+**Too complicated?** See [`html2rss-configs`](https://github.com/gildesmarais/html2rss-configs) for ready-made feed configs!
 
 ## Development
 
@@ -54,10 +35,15 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/gildesmarais/html2rss.
 
-## Changelog generation
+## Releasing a new version
 
-The `CHANGELOG.md` can be generated automatically with [`standard-changelog`](https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/standard-changelog).
+1. increase version in `lib/version.rb`
+2. `bundle`
+3. commit the changes
+4. `git tag v....`
+5. `git push; git push --tags`
+6. update the changelog, commit and push
 
-## License
+### Changelog generation
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+The `CHANGELOG.md` can be generated automatically with [`standard-changelog`](https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/standard-changelog).

data/html2rss.gemspec

@@ -33,8 +33,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'nokogiri', '>= 1.10'
   spec.add_dependency 'sanitize', '~> 5.0'
   spec.add_development_dependency 'bundler', '~> 1.16'
-  spec.add_development_dependency 'byebug', '~> 10.0'
+  spec.add_development_dependency 'byebug'
   spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop-performance'
   spec.add_development_dependency 'simplecov'
-  spec.add_development_dependency 'vcr', '~> 4.0'
+  spec.add_development_dependency 'vcr'
 end

data/lib/html2rss.rb

@@ -5,7 +5,11 @@ require 'yaml'
 
 module Html2rss
   def self.feed_from_yaml_config(file, name)
-    config = Config.new(YAML.load(File.open(file)).freeze, name)
+    yaml = YAML.load(File.open(file))
+    feed_config = yaml['feeds'][name]
+    global_config = yaml.reject { |k| k == 'feeds' }
+
+    config = Config.new(feed_config, global_config)
     feed(config)
   end
 

data/lib/html2rss/config.rb

@@ -1,11 +1,11 @@
 module Html2rss
   class Config
-    attr_reader :feed_config, :channel_config
+    attr_reader :feed_config, :channel_config, :global_config
 
-    def initialize(config, name)
-      @config = config
-      @feed_config = @config['feeds'][name.to_s]
-      @channel_config = @feed_config['channel']
+    def initialize(feed_config, global_config = {})
+      @global_config = global_config
+      @feed_config = feed_config
+      @channel_config = feed_config.fetch('channel', {})
     end
 
     def author
@@ -34,7 +34,7 @@ module Html2rss
     alias link url
 
     def headers
-      @config.fetch('headers', {})
+      global_config.fetch('headers', {})
     end
 
     def options(name)

data/lib/html2rss/item_extractor.rb

@@ -11,7 +11,7 @@ module Html2rss
         uri = URI(href)
       else
         uri = URI(options['channel']['url'])
-        uri.path = path.start_with?('/') ? path : "/#{path}"
+        uri.path = path.to_s.start_with?('/') ? path : "/#{path}"
         uri.query = query
       end
 

data/lib/html2rss/version.rb

@@ -1,3 +1,3 @@
 module Html2rss
-  VERSION = '0.2.2'.freeze
+  VERSION = '0.3.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Gil Desmarais
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-31 00:00:00.000000000 Z
+date: 2019-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -84,16 +84,16 @@ dependencies:
   name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +108,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -126,16 +154,16 @@ dependencies:
   name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0'
 description: |-
   Create your config object, include the url to scrape,
                           some selectors and get a RSS2 feed in return.
@@ -191,7 +219,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.2
+rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
 summary: Generate RSS feeds by scraping websites by providing a config.