RubyGems - html2rss - Versions diffs - 0.3.3 → 0.4.0 - Mend

html2rss 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (61) hide show

checksums.yaml +4 -4
data/.rubocop.yml +18 -11
data/.travis.yml +3 -3
data/.yardopts +6 -0
data/Gemfile.lock +23 -5
data/README.md +2 -1
data/docs/Html2rss.html +353 -0
data/docs/Html2rss/AttributePostProcessors.html +203 -0
data/docs/Html2rss/AttributePostProcessors/ParseTime.html +332 -0
data/docs/Html2rss/AttributePostProcessors/ParseUri.html +314 -0
data/docs/Html2rss/AttributePostProcessors/SanitizeHtml.html +346 -0
data/docs/Html2rss/AttributePostProcessors/Substring.html +321 -0
data/docs/Html2rss/AttributePostProcessors/Template.html +336 -0
data/docs/Html2rss/Config.html +795 -0
data/docs/Html2rss/FeedBuilder.html +295 -0
data/docs/Html2rss/Item.html +654 -0
data/docs/Html2rss/ItemExtractors.html +297 -0
data/docs/Html2rss/ItemExtractors/Attribute.html +317 -0
data/docs/Html2rss/ItemExtractors/CurrentTime.html +297 -0
data/docs/Html2rss/ItemExtractors/Href.html +319 -0
data/docs/Html2rss/ItemExtractors/Html.html +314 -0
data/docs/Html2rss/ItemExtractors/Static.html +301 -0
data/docs/Html2rss/ItemExtractors/Text.html +312 -0
data/docs/Html2rss/Utils.html +115 -0
data/docs/Html2rss/Utils/IndifferentAccessHash.html +142 -0
data/docs/_index.html +300 -0
data/docs/class_list.html +51 -0
data/docs/css/common.css +1 -0
data/docs/css/full_list.css +58 -0
data/docs/css/style.css +496 -0
data/docs/file.README.html +135 -0
data/docs/file_list.html +56 -0
data/docs/frames.html +17 -0
data/docs/index.html +135 -0
data/docs/js/app.js +303 -0
data/docs/js/full_list.js +216 -0
data/docs/js/jquery.js +4 -0
data/docs/method_list.html +435 -0
data/docs/top-level-namespace.html +110 -0
data/html2rss.gemspec +3 -0
data/lib/html2rss.rb +19 -4
data/lib/html2rss/attribute_post_processors.rb +5 -3
data/lib/html2rss/attribute_post_processors/parse_time.rb +29 -3
data/lib/html2rss/attribute_post_processors/parse_uri.rb +20 -1
data/lib/html2rss/attribute_post_processors/sanitize_html.rb +65 -3
data/lib/html2rss/attribute_post_processors/substring.rb +24 -3
data/lib/html2rss/attribute_post_processors/template.rb +37 -10
data/lib/html2rss/config.rb +11 -12
data/lib/html2rss/feed_builder.rb +8 -6
data/lib/html2rss/item.rb +28 -19
data/lib/html2rss/item_extractors.rb +29 -0
data/lib/html2rss/item_extractors/attribute.rb +37 -0
data/lib/html2rss/item_extractors/current_time.rb +21 -0
data/lib/html2rss/item_extractors/href.rb +36 -0
data/lib/html2rss/item_extractors/html.rb +34 -0
data/lib/html2rss/item_extractors/static.rb +28 -0
data/lib/html2rss/item_extractors/text.rb +32 -0
data/lib/html2rss/utils.rb +25 -0
data/lib/html2rss/version.rb +1 -1
metadata +88 -4
data/lib/html2rss/item_extractor.rb +0 -37

data/docs/top-level-namespace.html ADDED

@@ -0,0 +1,110 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>
+  Top Level Namespace
+    &mdash; Documentation by YARD 0.9.20
+</title>
+  <link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
+  <link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
+<script type="text/javascript" charset="utf-8">
+  pathId = "";
+  relpath = '';
+</script>
+  <script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
+  <script type="text/javascript" charset="utf-8" src="js/app.js"></script>
+  </head>
+  <body>
+    <div class="nav_wrap">
+      <iframe id="nav" src="class_list.html?1"></iframe>
+      <div id="resizer"></div>
+    </div>
+    <div id="main" tabindex="-1">
+      <div id="header">
+        <div id="menu">
+    <a href="_index.html">Index</a> &raquo;
+    <span class="title">Top Level Namespace</span>
+</div>
+        <div id="search">
+    <a class="full_list_link" id="class_list_link"
+        href="class_list.html">
+        <svg width="24" height="24">
+          <rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
+          <rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
+        </svg>
+    </a>
+</div>
+        <div class="clear"></div>
+      </div>
+      <div id="content"><h1>Top Level Namespace
+</h1>
+<div class="box_info">
+</div>
+<h2>Defined Under Namespace</h2>
+<p class="children">
+      <strong class="modules">Modules:</strong> <span class='object_link'><a href="Html2rss.html" title="Html2rss (module)">Html2rss</a></span>
+</p>
+</div>
+      <div id="footer">
+  Generated on Sun Jul 14 19:35:05 2019 by
+  <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
+  0.9.20 (ruby-2.6.3).
+</div>
+    </div>
+  </body>
+</html>

data/html2rss.gemspec CHANGED

@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
+  spec.add_dependency 'activesupport', '~> 5.0'
   spec.add_dependency 'faraday', '~> 0.15'
   spec.add_dependency 'faraday_middleware', '~> 0.13'
   spec.add_dependency 'hashie', '~> 3.6'
@@ -37,6 +38,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop'
   spec.add_development_dependency 'rubocop-performance'
+  spec.add_development_dependency 'rubocop-rspec'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'vcr'
+  spec.add_development_dependency 'yard'
 end

data/lib/html2rss.rb CHANGED

@@ -1,28 +1,41 @@
 require 'html2rss/config'
 require 'html2rss/feed_builder'
 require 'html2rss/version'
+require 'html2rss/utils'
 require 'yaml'
 module Html2rss
+  ##
+  # Returns a RSS object which is generated from the provided file.
+  #
+  # `file_path`: a File object of a YAML file
+  # `name`: the of the feed
+  #
+  # Example:
+  #
+  #    feed = Html2rss.feed_from_yaml_config(File.join(['spec', 'config.test.yml']), 'nuxt-releases')
+  #    # => #<RSS::Rss:0x00007fb2f6331228
+  # @return [RSS:Rss]
   def self.feed_from_yaml_config(file, name)
     # rubocop:disable Security/YAMLLoad
     yaml = YAML.load(File.open(file))
     # rubocop:enable Security/YAMLLoad
     feed_config = yaml['feeds'][name]
-    global_config = yaml.reject { |k| k == 'feeds' }
+    global_config = yaml.reject { |key| key == 'feeds' }
     config = Config.new(feed_config, global_config)
     feed(config)
   end
   ##
-  # Returns the RSS object, which is generated from the provided config.
+  # Returns a RSS object which is generated from the provided config.
   #
   # `config`: can be a Hash or an instance of Html2rss::Config.
   #
-  # = Example with a Ruby Hash
-  #    Html2rss.feed(
+  # Example:
+  #
+  #    feed = Html2rss.feed(
   #      channel: { name: 'StackOverflow: Hot Network Questions', url: 'https://stackoverflow.com' },
   #      selectors: {
   #        items: { selector: '#hot-network-questions > ul > li' },
@@ -30,6 +43,8 @@ module Html2rss
   #        link: { selector: 'a', extractor: 'href' }
   #      }
   #    )
+  #    # => #<RSS::Rss:0x00007fb2f48d14a0 ...>
+  # @return [RSS:Rss]
   def self.feed(config)
     config = Config.new(config) unless config.is_a?(Config)

data/lib/html2rss/attribute_post_processors.rb CHANGED

@@ -5,10 +5,12 @@ require_relative 'attribute_post_processors/substring'
 require_relative 'attribute_post_processors/template'
 module Html2rss
+  ##
+  # Provides a namespace for attribute post processors.
   module AttributePostProcessors
-    def self.get_processor(options)
-      camel_cased_option = options['name'].split('_').collect(&:capitalize).join
-      class_name = ['Html2rss', 'AttributePostProcessors', camel_cased_option].join('::')
+    def self.get_processor(name)
+      camel_cased_name = name.split('_').map(&:capitalize).join
+      class_name = ['Html2rss', 'AttributePostProcessors', camel_cased_name].join('::')
       Object.const_get(class_name)
     end

data/lib/html2rss/attribute_post_processors/parse_time.rb CHANGED

@@ -1,12 +1,38 @@
+require 'active_support'
+require 'active_support/core_ext/time'
 module Html2rss
   module AttributePostProcessors
+    ##
+    # Returns the {https://www.w3.org/Protocols/rfc822/ RFC822} representation of a time.
+    #
+    # Imagine this HTML structure:
+    #
+    #     <p>Published on <span>2019-07-02</span></p>
+    #
+    # YAML usage example:
+    #
+    #    selectors:
+    #      description:
+    #        selector: span
+    #        post_process:
+    #          name: 'parse_time'
+    #
+    # Would return:
+    #    "Tue, 02 Jul 2019 00:00:00 +0200"
+    #
+    # It uses {https://ruby-doc.org/stdlib-2.5.3/libdoc/time/rdoc/Time.html#method-c-parse Time.parse}.
+    # As of now it ignores time zones and always falls back to the UTC time zone.
     class ParseTime
-      def initialize(value, _options, _item)
-        @value = value
+      def initialize(value, env)
+        @value = value.to_s
+        @time_zone = env[:config].time_zone
       end
+      ##
+      # @return [String] rfc822 formatted time
       def get
-        Time.parse(@value).rfc822
+        Time.use_zone(@time_zone) { Time.zone.parse(@value).rfc822 }
       end
     end
   end

data/lib/html2rss/attribute_post_processors/parse_uri.rb CHANGED

@@ -1,10 +1,29 @@
 module Html2rss
   module AttributePostProcessors
+    ##
+    # Returns the URI as String.
+    #
+    # Imagine this HTML structure:
+    #
+    #    <span>http://why-not-use-a-link.uh</span>
+    #
+    # YAML usage example:
+    #
+    #    selectors:
+    #      link:
+    #        selector: span
+    #        extractor: text
+    #        post_process:
+    #          name: parse_uri
+    # Would return:
+    #    'http://why-not-use-a-link.uh'
     class ParseUri
-      def initialize(value, _options, _item)
+      def initialize(value, _env)
         @value = value
       end
+      ##
+      # @return [String]
       def get
         URI(@value).to_s
       end

data/lib/html2rss/attribute_post_processors/sanitize_html.rb CHANGED

@@ -2,21 +2,83 @@ require 'sanitize'
 module Html2rss
   module AttributePostProcessors
+    ##
+    # Returns sanitized HTML code as String.
+    # Adds
+    #
+    # - rel="nofollow noopener noreferrer" to a elements
+    # - referrer-policy='no-referrer' to img elements
+    #
+    # Imagine this HTML structure:
+    #
+    #     <section>
+    #       Lorem <b>ipsum</b> dolor...
+    #       <iframe src="https://evil.corp/miner"></iframe>
+    #       <script>alert();</script>
+    #     </section>
+    #
+    # YAML usage example:
+    #
+    #    selectors:
+    #      description:
+    #        selector: section
+    #        extractor: html
+    #        post_process:
+    #          name: sanitize_html
+    #
+    # Would return:
+    #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
     class SanitizeHtml
-      def initialize(value, _options, _item)
+      def initialize(value, env)
         @value = value
+        @channel_url = env[:config].url
       end
+      ##
+      # - uses the {https://github.com/rgrove/sanitize sanitize gem}
+      # - uses the config {https://github.com/rgrove/sanitize#sanitizeconfigrelaxed Sanitize::Config::RELAXED}
+      # - adds rel="nofollow noopener noreferrer" to a elements
+      # - adds target="_blank" to a elements
+      # @return [String]
       def get
         Sanitize.fragment(@value, Sanitize::Config.merge(
                                     Sanitize::Config::RELAXED,
+                                    attributes: {
+                                      all: %w[dir lang alt title translate]
+                                    },
                                     add_attributes: {
                                       'a' => {
                                         'rel' => 'nofollow noopener noreferrer',
                                         'target' => '_blank'
+                                      },
+                                      'img' => {
+                                        'referrer-policy' => 'no-referrer'
                                       }
-                                    }
-                                  ))
+                                    },
+                                    transformers: [transform_urls_to_absolute_ones]
+                                  )).to_s.split.join(' ')
+      end
+      private
+      URL_ELEMENTS_WITH_URL_ATTRIBUTE = {
+        'a' => :href,
+        'img' => :src
+      }.freeze
+      def transform_urls_to_absolute_ones
+        lambda do |env|
+          return unless URL_ELEMENTS_WITH_URL_ATTRIBUTE.key?(env[:node_name])
+          url_attribute = URL_ELEMENTS_WITH_URL_ATTRIBUTE[env[:node_name]]
+          url = env[:node][url_attribute]
+          return if URI(url).absolute?
+          absolute_url = Html2rss::Utils.build_absolute_url_from_relative(url, @channel_url)
+          env[:node][url_attribute] = absolute_url
+        end
       end
     end
   end

data/lib/html2rss/attribute_post_processors/substring.rb CHANGED

@@ -1,13 +1,34 @@
 module Html2rss
   module AttributePostProcessors
+    ## Returns a defined part of a String.
+    #
+    # The +end+ parameter can be omitted, in that case it will not cut the
+    # String at the end.
+    #
+    # Imagine this HTML:
+    #    <h1>Foo bar and baz<h1>
+    #
+    # YAML usage example:
+    #    selectors:
+    #      title:
+    #        selector: h1
+    #        post_process:
+    #         name: substring
+    #         start: 4
+    #         end: 6
+    #
+    # Would return:
+    #    'bar'
     class Substring
-      def initialize(value, options, _item)
+      def initialize(value, env)
         @value = value
-        @options = options
+        @options = env[:options]
       end
+      ##
+      # @return [String]
       def get
-        ending = @options.fetch('end', false) ? @options['end'].to_i : @value.length
+        ending = @options.fetch('end', @value.length).to_i
         @value[@options['start'].to_i..ending]
       end
     end

data/lib/html2rss/attribute_post_processors/template.rb CHANGED

@@ -2,13 +2,44 @@ require 'sanitize'
 module Html2rss
   module AttributePostProcessors
+    ## Returns a formatted String according to the string pattern.
+    #
+    # If +self+ is given as a method, the extracted value will be used.
+    #
+    # Imagine this HTML:
+    #    <li>
+    #      <h1>Product</h1>
+    #      <span class="price">23,42€</span>
+    #    </li>
+    #
+    # YAML usage example:
+    #
+    #    selectors:
+    #      items:
+    #        selector: 'li'
+    #      price:
+    #       selector: '.price'
+    #      title:
+    #        selector: h1
+    #        post_process:
+    #         name: template
+    #         string: '%s (%s)'
+    #         methods:
+    #           - self
+    #           - price
+    #
+    # Would return:
+    #    'Product (23,42€)'
     class Template
-      def initialize(value, options, item)
+      def initialize(value, env)
         @value = value
-        @options = options
-        @item = item
+        @options = env[:options]
+        @item = env[:item]
       end
+      ##
+      # - uses {http://ruby-doc.org/core-2.6.3/String.html#method-i-25 String#%}
+      # @return [String]
       def get
         string % methods
       end
@@ -20,13 +51,9 @@ module Html2rss
       end
       def methods
-        @methods ||= @options['methods'].map { |method|
-          if method == 'self'
-            @value
-          else
-            @item.send(method.to_sym)&.to_s
-          end
-        }
+        @methods ||= @options['methods'].map do |method|
+          method == 'self' ? @value.to_s : @item.public_send(method.to_sym).to_s
+        end
       end
     end
   end