html2rss 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d7642c21f46ea6086a3ead39b2923fa4c09ddc821719e2c40936846a83e5438
-  data.tar.gz: 8d6efa4ed03261f11e9ec302ea0cda860e83b74d78203c48b6650f1a7d9c106c
+  metadata.gz: 6a7c25abdeeccbeb69e1b7942c2f8dd9c7da543a090cc1108debcf3370b88989
+  data.tar.gz: 34dc564b089dfec7e2a220288484dd24fe407743301fcb4c2d74f37c35a2325b
 SHA512:
-  metadata.gz: f6f1a3c52d9689b18d04a7b0c772ab0b9742202ca2e292b8265f7d26d85989f7201da9ab3de2db81258b82b8271c5789176976ea6c67c42edcb4739a556386f3
-  data.tar.gz: 8d892ec1d5761d6c5338e77f36a86debb2ccc1f1f7bac98ba83489c2aa8a7fe0638a45d355ff6fbd295a417d4df25f9ffc932fd15dd70f5465b0b61ee566cec0
+  metadata.gz: a2168f96899c9a53b9b55010c1746ce0dca9e78ec5004903d837fb333b1a158b7c124eaf08ef34f2cb6515164d9dc20a3c16c47fe672c4ba80b1241cf3decdd3
+  data.tar.gz: 0fb315100e1cc6b55d81b363186b3e994e54c0c58e3381ca5bc3c5884fd5b993e0364db311eea64688cdaa72e5c865055b45e8f0268db48773355c035083e9b6

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+# [0.3.1](https://github.com/gildesmarais/html2rss/compare/v0.3.0...v0.3.1) (2019-06-23)
+
+
+### Features
+
+* handle string and symbol keys in config hashes ([#15](https://github.com/gildesmarais/html2rss/issues/15)) ([93ad824](https://github.com/gildesmarais/html2rss/commit/93ad824))
+* support attributes without selector, fallback to root element then ([#16](https://github.com/gildesmarais/html2rss/issues/16)) ([d99ae3d](https://github.com/gildesmarais/html2rss/commit/d99ae3d))
+
+
+# [0.3.0](https://github.com/gildesmarais/html2rss/compare/v0.2.2...v0.3.0) (2019-06-20)
+
+
+### Features
+
+* add rubocop and update development deps ([#13](https://github.com/gildesmarais/html2rss/issues/13)) ([6e06329](https://github.com/gildesmarais/html2rss/commit/6e06329))
+* change Config constructor arguments ([#14](https://github.com/gildesmarais/html2rss/issues/14)) ([21f8746](https://github.com/gildesmarais/html2rss/commit/21f8746))
+
+
+
 # [0.2.1](https://github.com/gildesmarais/html2rss/compare/v0.2.0...v0.2.1) (2018-11-18)
 
 

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    html2rss (0.3.0)
+    html2rss (0.3.1)
       faraday (~> 0.15)
       faraday_middleware (~> 0.13)
-      nokogiri (>= 1.10)
+      hashie (~> 3.6)
+      nokogiri (>= 1.10, < 2.0)
       sanitize (~> 5.0)
 
 GEM
@@ -19,6 +20,7 @@ GEM
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.13.1)
       faraday (>= 0.7.4, < 1.0)
+    hashie (3.6.0)
     jaro_winkler (1.5.3)
     json (2.2.0)
     mini_portile2 (2.4.0)

data/README.md

@@ -17,6 +17,19 @@ and [post processors](https://github.com/gildesmarais/html2rss/tree/master/lib/h
 Add this line to your application's Gemfile: `gem 'html2rss'`
 And then execute: `bundle`
 
+```ruby
+rss = Html2rss.feed(
+  channel: { name: 'StackOverflow: Hot Network Questions', url: 'https://stackoverflow.com' },
+  selectors: {
+    items: { selector: '#hot-network-questions > ul > li' },
+    title: { selector: 'a' },
+    link: { selector: 'a', extractor: 'href' }
+  }
+)
+
+puts rss.to_s
+```
+
 ## Usage with a YAML config file
 
 Create a YAML config file. Find an example at [`rspec/config.test.yml`](https://github.com/gildesmarais/html2rss/blob/master/spec/config.test.yml).

data/html2rss.gemspec

@@ -8,9 +8,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Gil Desmarais']
   spec.email         = ['html2rss@desmarais.de']
 
-  spec.summary       = 'Generate RSS feeds by scraping websites by providing a config.'
-  spec.description   = 'Create your config object, include the url to scrape,
-                        some selectors and get a RSS2 feed in return.'
+  spec.summary       = 'Returns an RSS::Rss object by scraping a URL.'
+  spec.description   = 'Give the URL to scrape and some CSS selectors. Get a RSS::Rss instance in return.'
   spec.homepage      = 'https://github.com/gildesmarais/html2rss'
   spec.license       = 'MIT'
 
@@ -30,7 +29,8 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'faraday', '~> 0.15'
   spec.add_dependency 'faraday_middleware', '~> 0.13'
-  spec.add_dependency 'nokogiri', '>= 1.10'
+  spec.add_dependency 'hashie', '~> 3.6'
+  spec.add_dependency 'nokogiri', '>= 1.10', '< 2.0'
   spec.add_dependency 'sanitize', '~> 5.0'
   spec.add_development_dependency 'bundler', '~> 1.16'
   spec.add_development_dependency 'byebug'

data/lib/html2rss.rb

@@ -5,7 +5,10 @@ require 'yaml'
 
 module Html2rss
   def self.feed_from_yaml_config(file, name)
+    # rubocop:disable Security/YAMLLoad
     yaml = YAML.load(File.open(file))
+    # rubocop:enable Security/YAMLLoad
+
     feed_config = yaml['feeds'][name]
     global_config = yaml.reject { |k| k == 'feeds' }
 
@@ -13,7 +16,23 @@ module Html2rss
     feed(config)
   end
 
+  ##
+  # Returns the RSS object, which is generated from the provided config.
+  #
+  # `config`: can be a Hash or an instance of Html2rss::Config.
+  #
+  # = Example with a Ruby Hash
+  #    Html2rss.feed(
+  #      channel: { name: 'StackOverflow: Hot Network Questions', url: 'https://stackoverflow.com' },
+  #      selectors: {
+  #        items: { selector: '#hot-network-questions > ul > li' },
+  #        title: { selector: 'a' },
+  #        link: { selector: 'a', extractor: 'href' }
+  #      }
+  #    )
   def self.feed(config)
+    config = Config.new(config) unless config.is_a?(Config)
+
     feed = FeedBuilder.new config
     feed.rss
   end

data/lib/html2rss/config.rb

@@ -1,11 +1,18 @@
+require 'hashie'
+
 module Html2rss
   class Config
     attr_reader :feed_config, :channel_config, :global_config
 
+    class IndifferentAccessHash < Hash
+      include Hashie::Extensions::MergeInitializer
+      include Hashie::Extensions::IndifferentAccess
+    end
+
     def initialize(feed_config, global_config = {})
-      @global_config = global_config
-      @feed_config = feed_config
-      @channel_config = feed_config.fetch('channel', {})
+      @global_config = IndifferentAccessHash.new global_config
+      @feed_config = IndifferentAccessHash.new feed_config
+      @channel_config = IndifferentAccessHash.new @feed_config.fetch('channel', {})
     end
 
     def author
@@ -13,7 +20,7 @@ module Html2rss
     end
 
     def ttl
-      (channel_config.fetch 'ttl').to_i || nil
+      channel_config.fetch 'ttl', 3600
     end
 
     def title

data/lib/html2rss/item.rb

@@ -38,9 +38,7 @@ module Html2rss
     end
 
     def valid?
-      return false if [title.to_s, description.to_s].join('') == ''
-
-      true
+      [title.to_s, description.to_s].join('') != ''
     end
 
     def categories

data/lib/html2rss/item_extractor.rb

@@ -1,10 +1,15 @@
 module Html2rss
   module ItemExtractor
-    TEXT = proc { |xml, options| xml.css(options['selector'])&.text&.strip }
-    ATTRIBUTE = proc { |xml, options| xml.css(options['selector']).attr(options['attribute']).to_s }
+    TEXT = proc { |xml, options|
+      element(xml, options)&.text&.strip&.split&.join(' ')
+    }
+
+    ATTRIBUTE = proc { |xml, options|
+      element(xml, options).attr(options['attribute']).to_s
+    }
 
     HREF = proc { |xml, options|
-      href = xml.css(options['selector']).attr('href').to_s
+      href = element(xml, options).attr('href').to_s
       path, query = href.split('?')
 
       if href.start_with?('http')
@@ -18,8 +23,15 @@ module Html2rss
       uri
     }
 
-    HTML = proc { |xml, options| xml.css(options['selector']).to_s }
+    HTML = proc { |xml, options|
+      element(xml, options).to_s
+    }
+
     STATIC = proc { |_xml, options| options['static'] }
     CURRENT_TIME = proc { |_xml, _options| Time.new }
+
+    def self.element(xml, options)
+      options['selector'] ? xml.css(options['selector']) : xml
+    end
   end
 end

data/lib/html2rss/version.rb

@@ -1,3 +1,3 @@
 module Html2rss
-  VERSION = '0.3.0'.freeze
+  VERSION = '0.3.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Gil Desmarais
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-20 00:00:00.000000000 Z
+date: 2019-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -45,6 +59,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.10'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -52,6 +69,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.10'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: sanitize
   requirement: !ruby/object:Gem::Requirement
@@ -164,9 +184,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: |-
-  Create your config object, include the url to scrape,
-                          some selectors and get a RSS2 feed in return.
+description: Give the URL to scrape and some CSS selectors. Get a RSS::Rss instance
+  in return.
 email:
 - html2rss@desmarais.de
 executables: []
@@ -222,5 +241,5 @@ requirements: []
 rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
-summary: Generate RSS feeds by scraping websites by providing a config.
+summary: Returns an RSS::Rss object by scraping a URL.
 test_files: []