RubyGems - html2rss - Versions diffs - 0.3.3 → 0.4.0 - Mend

html2rss 0.3.3 → 0.4.0

Files changed (61) hide show

checksums.yaml +4 -4
data/.rubocop.yml +18 -11
data/.travis.yml +3 -3
data/.yardopts +6 -0
data/Gemfile.lock +23 -5
data/README.md +2 -1
data/docs/Html2rss.html +353 -0
data/docs/Html2rss/AttributePostProcessors.html +203 -0
data/docs/Html2rss/AttributePostProcessors/ParseTime.html +332 -0
data/docs/Html2rss/AttributePostProcessors/ParseUri.html +314 -0
data/docs/Html2rss/AttributePostProcessors/SanitizeHtml.html +346 -0
data/docs/Html2rss/AttributePostProcessors/Substring.html +321 -0
data/docs/Html2rss/AttributePostProcessors/Template.html +336 -0
data/docs/Html2rss/Config.html +795 -0
data/docs/Html2rss/FeedBuilder.html +295 -0
data/docs/Html2rss/Item.html +654 -0
data/docs/Html2rss/ItemExtractors.html +297 -0
data/docs/Html2rss/ItemExtractors/Attribute.html +317 -0
data/docs/Html2rss/ItemExtractors/CurrentTime.html +297 -0
data/docs/Html2rss/ItemExtractors/Href.html +319 -0
data/docs/Html2rss/ItemExtractors/Html.html +314 -0
data/docs/Html2rss/ItemExtractors/Static.html +301 -0
data/docs/Html2rss/ItemExtractors/Text.html +312 -0
data/docs/Html2rss/Utils.html +115 -0
data/docs/Html2rss/Utils/IndifferentAccessHash.html +142 -0
data/docs/_index.html +300 -0
data/docs/class_list.html +51 -0
data/docs/css/common.css +1 -0
data/docs/css/full_list.css +58 -0
data/docs/css/style.css +496 -0
data/docs/file.README.html +135 -0
data/docs/file_list.html +56 -0
data/docs/frames.html +17 -0
data/docs/index.html +135 -0
data/docs/js/app.js +303 -0
data/docs/js/full_list.js +216 -0
data/docs/js/jquery.js +4 -0
data/docs/method_list.html +435 -0
data/docs/top-level-namespace.html +110 -0
data/html2rss.gemspec +3 -0
data/lib/html2rss.rb +19 -4
data/lib/html2rss/attribute_post_processors.rb +5 -3
data/lib/html2rss/attribute_post_processors/parse_time.rb +29 -3
data/lib/html2rss/attribute_post_processors/parse_uri.rb +20 -1
data/lib/html2rss/attribute_post_processors/sanitize_html.rb +65 -3
data/lib/html2rss/attribute_post_processors/substring.rb +24 -3
data/lib/html2rss/attribute_post_processors/template.rb +37 -10
data/lib/html2rss/config.rb +11 -12
data/lib/html2rss/feed_builder.rb +8 -6
data/lib/html2rss/item.rb +28 -19
data/lib/html2rss/item_extractors.rb +29 -0
data/lib/html2rss/item_extractors/attribute.rb +37 -0
data/lib/html2rss/item_extractors/current_time.rb +21 -0
data/lib/html2rss/item_extractors/href.rb +36 -0
data/lib/html2rss/item_extractors/html.rb +34 -0
data/lib/html2rss/item_extractors/static.rb +28 -0
data/lib/html2rss/item_extractors/text.rb +32 -0
data/lib/html2rss/utils.rb +25 -0
data/lib/html2rss/version.rb +1 -1
metadata +88 -4
data/lib/html2rss/item_extractor.rb +0 -37

data/docs/top-level-namespace.html ADDED

@@ -0,0 +1,110 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>
+  Top Level Namespace
+    &mdash; Documentation by YARD 0.9.20
+</title>
+  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
+  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
+<script type="text/javascript" charset="utf-8">
+  pathId = "";
+  relpath = '';
+</script>
+  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
+  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
+  </head>
+  <body>
+    <div class="nav_wrap">
+      <iframe id="nav" src="class_list.html?1"></iframe>
+      <div id="resizer"></div>
+    </div>
+    <div id="main" tabindex="-1">
+      <div id="header">
+        <div id="menu">
+    <a href="_index.html">Index</a> &raquo;
+    <span class="title">Top Level Namespace</span>
+</div>
+        <div id="search">
+    <a class="full_list_link" id="class_list_link"
+        href="class_list.html">
+        <svg width="24" height="24">
+          <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
+        </svg>
+    </a>
+</div>
+        <div class="clear"></div>
+      </div>
+      <div id="content"><h1>Top Level Namespace
+</h1>
+<div class="box_info">
+</div>
+<h2>Defined Under Namespace</h2>
+<p class="children">
+      <strong class="modules">Modules:</strong> <span class='object_link'><a href="Html2rss.html" title="Html2rss (module)">Html2rss</a></span>
+</p>
+</div>
+      <div id="footer">
+  Generated on Sun Jul 14 19:35:05 2019 by
+  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
+  0.9.20 (ruby-2.6.3).
+</div>
+    </div>
+  </body>
+</html>

data/html2rss.gemspec CHANGED

@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
+  spec.add_dependency 'activesupport', '~> 5.0'
   spec.add_dependency 'faraday', '~> 0.15'
   spec.add_dependency 'faraday_middleware', '~> 0.13'
   spec.add_dependency 'hashie', '~> 3.6'
@@ -37,6 +38,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop'
   spec.add_development_dependency 'rubocop-performance'
+  spec.add_development_dependency 'rubocop-rspec'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'vcr'
+  spec.add_development_dependency 'yard'
 end

data/lib/html2rss.rb CHANGED

@@ -1,28 +1,41 @@
 require 'html2rss/config'
 require 'html2rss/feed_builder'
 require 'html2rss/version'
+require 'html2rss/utils'
 require 'yaml'
 module Html2rss
+  ##
+  # Returns a RSS object which is generated from the provided file.
+  #
+  # `file_path`: a File object of a YAML file
+  # `name`: the of the feed
+  #
+  # Example:
+  #
+  #    feed = Html2rss.feed_from_yaml_config(File.join(['spec', 'config.test.yml']), 'nuxt-releases')
+  #    # => #<RSS::Rss:0x00007fb2f6331228
+  # @return [RSS:Rss]
   def self.feed_from_yaml_config(file, name)
     # rubocop:disable Security/YAMLLoad
     yaml = YAML.load(File.open(file))
     # rubocop:enable Security/YAMLLoad
     feed_config = yaml['feeds'][name]
-    global_config = yaml.reject { |k| k == 'feeds' }
+    global_config = yaml.reject { |key| key == 'feeds' }
     config = Config.new(feed_config, global_config)
     feed(config)
   end
   ##
-  # Returns the RSS object, which is generated from the provided config.
+  # Returns a RSS object which is generated from the provided config.
   #
   # `config`: can be a Hash or an instance of Html2rss::Config.
   #
-  # = Example with a Ruby Hash
-  #    Html2rss.feed(
+  # Example:
+  #
+  #    feed = Html2rss.feed(
   #      channel: { name: 'StackOverflow: Hot Network Questions', url: 'https://stackoverflow.com' },
   #      selectors: {
   #        items: { selector: '#hot-network-questions > ul > li' },
@@ -30,6 +43,8 @@ module Html2rss
   #        link: { selector: 'a', extractor: 'href' }
   #      }
   #    )
+  #    # => #<RSS::Rss:0x00007fb2f48d14a0 ...>
+  # @return [RSS:Rss]
   def self.feed(config)
     config = Config.new(config) unless config.is_a?(Config)

data/lib/html2rss/attribute_post_processors.rb CHANGED

@@ -5,10 +5,12 @@ require_relative 'attribute_post_processors/substring'
 require_relative 'attribute_post_processors/template'
 module Html2rss
+  ##
+  # Provides a namespace for attribute post processors.
   module AttributePostProcessors
-    def self.get_processor(options)
-      camel_cased_option = options['name'].split('_').collect(&:capitalize).join
-      class_name = ['Html2rss', 'AttributePostProcessors', camel_cased_option].join('::')
+    def self.get_processor(name)
+      camel_cased_name = name.split('_').map(&:capitalize).join
+      class_name = ['Html2rss', 'AttributePostProcessors', camel_cased_name].join('::')
       Object.const_get(class_name)
     end

data/lib/html2rss/attribute_post_processors/parse_time.rb CHANGED

@@ -1,12 +1,38 @@
+require 'active_support'
+require 'active_support/core_ext/time'
 module Html2rss
   module AttributePostProcessors
+    ##
+    # Returns the {https://www.w3.org/Protocols/rfc822/ RFC822} representation of a time.
+    #
+    # Imagine this HTML structure:
+    #
+    #     <p>Published on <span>2019-07-02</span></p>
+    #
+    # YAML usage example:
+    #
+    #    selectors:
+    #      description:
+    #        selector: span
+    #        post_process:
+    #          name: 'parse_time'
+    #
+    # Would return:
+    #    "Tue, 02 Jul 2019 00:00:00 +0200"
+    #
+    # It uses {https://ruby-doc.org/stdlib-2.5.3/libdoc/time/rdoc/Time.html#method-c-parse Time.parse}.
+    # As of now it ignores time zones and always falls back to the UTC time zone.
     class ParseTime
-      def initialize(value, _options, _item)
-        @value = value
+      def initialize(value, env)
+        @value = value.to_s
+        @time_zone = env[:config].time_zone
       end
+      ##
+      # @return [String] rfc822 formatted time
       def get
-        Time.parse(@value).rfc822
+        Time.use_zone(@time_zone) { Time.zone.parse(@value).rfc822 }
       end
     end
   end

data/lib/html2rss/attribute_post_processors/parse_uri.rb CHANGED

@@ -1,10 +1,29 @@
 module Html2rss
   module AttributePostProcessors
+    ##
+    # Returns the URI as String.
+    #
+    # Imagine this HTML structure:
+    #
+    #    <span>http://why-not-use-a-link.uh</span>
+    #
+    # YAML usage example:
+    #
+    #    selectors:
+    #      link:
+    #        selector: span
+    #        extractor: text
+    #        post_process:
+    #          name: parse_uri
+    # Would return:
+    #    'http://why-not-use-a-link.uh'
     class ParseUri
-      def initialize(value, _options, _item)
+      def initialize(value, _env)
         @value = value
       end
+      ##
+      # @return [String]
       def get
         URI(@value).to_s
       end

data/lib/html2rss/attribute_post_processors/sanitize_html.rb CHANGED

@@ -2,21 +2,83 @@ require 'sanitize'
 module Html2rss
   module AttributePostProcessors
+    ##
+    # Returns sanitized HTML code as String.
+    # Adds
+    #
+    # - rel="nofollow noopener noreferrer" to a elements
+    # - referrer-policy='no-referrer' to img elements
+    #
+    # Imagine this HTML structure:
+    #
+    #     <section>
+    #       Lorem <b>ipsum</b> dolor...
+    #       <iframe src="https://evil.corp/miner"></iframe>
+    #       <script>alert();</script>
+    #     </section>
+    #
+    # YAML usage example:
+    #
+    #    selectors:
+    #      description:
+    #        selector: section
+    #        extractor: html
+    #        post_process:
+    #          name: sanitize_html
+    #
+    # Would return:
+    #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
     class SanitizeHtml
-      def initialize(value, _options, _item)
+      def initialize(value, env)
         @value = value
+        @channel_url = env[:config].url
       end
+      ##
+      # - uses the {https://github.com/rgrove/sanitize sanitize gem}
+      # - uses the config {https://github.com/rgrove/sanitize#sanitizeconfigrelaxed Sanitize::Config::RELAXED}
+      # - adds rel="nofollow noopener noreferrer" to a elements
+      # - adds target="_blank" to a elements
+      # @return [String]
       def get
         Sanitize.fragment(@value, Sanitize::Config.merge(
                                     Sanitize::Config::RELAXED,
+                                    attributes: {
+                                      all: %w[dir lang alt title translate]
+                                    },
                                     add_attributes: {
                                       'a' => {
                                         'rel' => 'nofollow noopener noreferrer',
                                         'target' => '_blank'
+                                      },
+                                      'img' => {
+                                        'referrer-policy' => 'no-referrer'
                                       }
-                                    }
-                                  ))
+                                    },
+                                    transformers: [transform_urls_to_absolute_ones]
+                                  )).to_s.split.join(' ')
+      end
+      private
+      URL_ELEMENTS_WITH_URL_ATTRIBUTE = {
+        'a' => :href,
+        'img' => :src
+      }.freeze
+      def transform_urls_to_absolute_ones
+        lambda do |env|
+          return unless URL_ELEMENTS_WITH_URL_ATTRIBUTE.key?(env[:node_name])
+          url_attribute = URL_ELEMENTS_WITH_URL_ATTRIBUTE[env[:node_name]]
+          url = env[:node][url_attribute]
+          return if URI(url).absolute?
+          absolute_url = Html2rss::Utils.build_absolute_url_from_relative(url, @channel_url)
+          env[:node][url_attribute] = absolute_url
+        end
       end
     end
   end

data/lib/html2rss/attribute_post_processors/substring.rb CHANGED

@@ -1,13 +1,34 @@
 module Html2rss
   module AttributePostProcessors
+    ## Returns a defined part of a String.
+    #
+    # The +end+ parameter can be omitted, in that case it will not cut the
+    # String at the end.
+    #
+    # Imagine this HTML:
+    #    <h1>Foo bar and baz<h1>
+    #
+    # YAML usage example:
+    #    selectors:
+    #      title:
+    #        selector: h1
+    #        post_process:
+    #         name: substring
+    #         start: 4
+    #         end: 6
+    #
+    # Would return:
+    #    'bar'
     class Substring
-      def initialize(value, options, _item)
+      def initialize(value, env)
         @value = value
-        @options = options
+        @options = env[:options]
       end
+      ##
+      # @return [String]
       def get
-        ending = @options.fetch('end', false) ? @options['end'].to_i : @value.length
+        ending = @options.fetch('end', @value.length).to_i
         @value[@options['start'].to_i..ending]
       end
     end

data/lib/html2rss/attribute_post_processors/template.rb CHANGED

@@ -2,13 +2,44 @@ require 'sanitize'
 module Html2rss
   module AttributePostProcessors
+    ## Returns a formatted String according to the string pattern.
+    #
+    # If +self+ is given as a method, the extracted value will be used.
+    #
+    # Imagine this HTML:
+    #    <li>
+    #      <h1>Product</h1>
+    #      <span class="price">23,42€</span>
+    #    </li>
+    #
+    # YAML usage example:
+    #
+    #    selectors:
+    #      items:
+    #        selector: 'li'
+    #      price:
+    #       selector: '.price'
+    #      title:
+    #        selector: h1
+    #        post_process:
+    #         name: template
+    #         string: '%s (%s)'
+    #         methods:
+    #           - self
+    #           - price
+    #
+    # Would return:
+    #    'Product (23,42€)'
     class Template
-      def initialize(value, options, item)
+      def initialize(value, env)
         @value = value
-        @options = options
-        @item = item
+        @options = env[:options]
+        @item = env[:item]
       end
+      ##
+      # - uses {http://ruby-doc.org/core-2.6.3/String.html#method-i-25 String#%}
+      # @return [String]
       def get
         string % methods
       end
@@ -20,13 +51,9 @@ module Html2rss
       end
       def methods
-        @methods ||= @options['methods'].map { |method|
-          if method == 'self'
-            @value
-          else
-            @item.send(method.to_sym)&.to_s
-          end
-        }
+        @methods ||= @options['methods'].map do |method|
+          method == 'self' ? @value.to_s : @item.public_send(method.to_sym).to_s
+        end
       end
     end
   end