html2rss 0.9.0 → 0.11.0

data/exe/html2rss

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'html2rss/cli'
+
+Html2rss::CLI.start(ARGV)

data/html2rss.gemspec

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'html2rss/version'
@@ -8,46 +10,39 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Gil Desmarais']
   spec.email         = ['html2rss@desmarais.de']
 
-  spec.summary       = 'Returns an RSS::Rss object by scraping a URL.'
-  spec.description   = 'Give the URL to scrape and some CSS selectors. Get a RSS::Rss instance in return.'
-  spec.homepage      = 'https://github.com/gildesmarais/html2rss'
+  spec.summary       = 'Generates RSS feeds from websites by scraping a URL and using CSS selectors to extract item.'
+  spec.description   = 'Supports JSON content, custom HTTP headers, and post-processing of extracted content.'
+  spec.homepage      = 'https://github.com/html2rss/html2rss'
   spec.license       = 'MIT'
-  spec.required_ruby_version = '>= 2.5.0'
+  spec.required_ruby_version = '>= 3.1'
 
   if spec.respond_to?(:metadata)
     spec.metadata['allowed_push_host'] = 'https://rubygems.org'
-    spec.metadata['changelog_uri'] = 'https://github.com/gildesmarais/html2rss/blob/master/CHANGELOG.md'
+    spec.metadata['changelog_uri'] = "#{spec.homepage}/releases/tag/v#{spec.version}"
+    spec.metadata['rubygems_mfa_required'] = 'true'
   else
     raise 'RubyGems 2.0 or newer is required to protect against ' \
-      'public gem pushes.'
+          'public gem pushes.'
   end
 
-  spec.files = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
+  spec.files = `git ls-files -z`.split("\x0").select do |f|
+    f.match(%r{^(lib/|exe/|README.md|LICENSE|html2rss.gemspec)})
   end
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'activesupport', '>= 5', '< 7'
   spec.add_dependency 'addressable', '~> 2.7'
-  spec.add_dependency 'builder'
-  spec.add_dependency 'faraday', '~> 1.0'
-  spec.add_dependency 'faraday_middleware'
+  spec.add_dependency 'faraday', '> 2.0.1', '< 3.0'
+  spec.add_dependency 'faraday-follow_redirects'
   spec.add_dependency 'kramdown'
   spec.add_dependency 'mime-types', '> 3.0'
   spec.add_dependency 'nokogiri', '>= 1.10', '< 2.0'
+  spec.add_dependency 'regexp_parser'
   spec.add_dependency 'reverse_markdown', '~> 2.0'
-  spec.add_dependency 'sanitize', '~> 5.0'
-  spec.add_dependency 'to_regexp'
+  spec.add_dependency 'rss'
+  spec.add_dependency 'sanitize', '~> 6.0'
+  spec.add_dependency 'thor'
+  spec.add_dependency 'tzinfo'
   spec.add_dependency 'zeitwerk'
-  spec.add_development_dependency 'bundler'
-  spec.add_development_dependency 'byebug'
-  spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rubocop'
-  spec.add_development_dependency 'rubocop-performance'
-  spec.add_development_dependency 'rubocop-rspec'
-  spec.add_development_dependency 'simplecov'
-  spec.add_development_dependency 'vcr'
-  spec.add_development_dependency 'yard'
 end

data/lib/html2rss/attribute_post_processors/gsub.rb

@@ -1,9 +1,8 @@
-require 'to_regexp'
+# frozen_string_literal: true
 
 module Html2rss
   module AttributePostProcessors
     ##
-    #
     # Imagine this HTML:
     #    <h1>Foo bar and boo<h1>
     #
@@ -19,23 +18,46 @@ module Html2rss
     # Would return:
     #    'Foo bar and baz'
     #
-    # `pattern` can be a Regexp or a String.
+    # `pattern` can be a Regexp or a String. If it is a String, it will remove
+    # one pair of surrounding slashes ('/') to keep backwards compatibility
+    # and then parse it to build a Regexp.
     #
     # `replacement` can be a String or a Hash.
     #
     # See the doc on [String#gsub](https://ruby-doc.org/core/String.html#method-i-gsub) for more info.
     class Gsub
-      def initialize(value, env)
+      ##
+      # @param value [String]
+      # @param context [Item::Context]
+      def initialize(value, context)
         @value = value
-        options = env[:options]
-        @pattern = options[:pattern].to_regexp || options[:pattern]
-        @replacement = options[:replacement]
+        @options = context[:options]
       end
 
       ##
       # @return [String]
       def get
-        @value.to_s.gsub(@pattern, @replacement)
+        @value.to_s.gsub(pattern, replacement)
+      end
+
+      private
+
+      ##
+      # @return [Regexp]
+      def pattern
+        pattern = @options[:pattern]
+        raise ArgumentError, 'The `pattern` option is missing' unless pattern
+
+        pattern.is_a?(String) ? Utils.build_regexp_from_string(pattern) : pattern
+      end
+
+      ##
+      # @return [Hash, String]
+      def replacement
+        replacement = @options[:replacement]
+        return replacement if replacement.is_a?(String) || replacement.is_a?(Hash)
+
+        raise ArgumentError, 'The `replacement` option must be a String or Hash'
       end
     end
   end

data/lib/html2rss/attribute_post_processors/html_to_markdown.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'reverse_markdown'
 
 module Html2rss
@@ -25,14 +27,17 @@ module Html2rss
     # Would return:
     #    'Lorem **ipsum** dolor'
     class HtmlToMarkdown
+      ##
+      # @param value [String]
+      # @param env [Item::Context]
       def initialize(value, env)
-        @value = SanitizeHtml.new(value, env).get
+        @sanitized_value = SanitizeHtml.new(value, env).get
       end
 
       ##
       # @return [String] formatted in Markdown
       def get
-        ReverseMarkdown.convert @value
+        ReverseMarkdown.convert(@sanitized_value)
       end
     end
   end

data/lib/html2rss/attribute_post_processors/html_transformers/transform_urls_to_absolute_ones.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module AttributePostProcessors
+    module HtmlTransformers
+      ##
+      # Transformer that converts relative URLs to absolute URLs within specified HTML elements.
+      class TransformUrlsToAbsoluteOnes
+        URL_ELEMENTS_WITH_URL_ATTRIBUTE = { 'a' => :href, 'img' => :src }.freeze
+
+        def initialize(channel_url)
+          @channel_url = channel_url
+        end
+
+        ##
+        # Transforms URLs to absolute ones.
+        def call(node_name:, node:, **_env)
+          return unless URL_ELEMENTS_WITH_URL_ATTRIBUTE.key?(node_name)
+
+          url_attribute = URL_ELEMENTS_WITH_URL_ATTRIBUTE[node_name]
+          url = node[url_attribute]
+          node[url_attribute] = Html2rss::Utils.build_absolute_url_from_relative(url, @channel_url)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/attribute_post_processors/html_transformers/wrap_img_in_a.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module AttributePostProcessors
+    module HtmlTransformers
+      ##
+      # Transformer that wraps <img> tags into <a> tags linking to `img.src`.
+      class WrapImgInA
+        ##
+        # Wraps <img> tags into <a> tags that link to `img.src`.
+        #
+        # @param node_name [String]
+        # @param node [Nokogiri::XML::Node]
+        # @return [nil]
+        def call(node_name:, node:, **_env)
+          return unless already_wrapped?(node_name, node)
+
+          wrap_image_in_anchor(node)
+        end
+
+        def already_wrapped?(node_name, node)
+          node_name == 'img' && node.parent.name != 'a'
+        end
+
+        private
+
+        ##
+        # Wraps the <img> node in an <a> tag.
+        #
+        # @param node [Nokogiri::XML::Node]
+        # @return [nil]
+        def wrap_image_in_anchor(node)
+          anchor = Nokogiri::XML::Node.new('a', node.document)
+          anchor['href'] = node['src']
+          node.add_next_sibling(anchor)
+          anchor.add_child(node.remove)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/attribute_post_processors/markdown_to_html.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require 'kramdown'
+require_relative 'sanitize_html'
 
 module Html2rss
   module AttributePostProcessors
@@ -30,15 +33,21 @@ module Html2rss
     #
     #    <p>Price: 12.34</p>
     class MarkdownToHtml
+      ##
+      # @param value [String] Markdown content to convert to HTML
+      # @param env [Item::Context] Context object providing additional environment details
       def initialize(value, env)
         @value = value
         @env = env
       end
 
       ##
-      # @return [String] formatted in Markdown
+      # Converts Markdown to sanitized HTML.
+      #
+      # @return [String] Sanitized HTML content
       def get
-        SanitizeHtml.new(Kramdown::Document.new(@value).to_html, @env).get
+        html_content = Kramdown::Document.new(@value).to_html
+        SanitizeHtml.new(html_content, @env).get
       end
     end
   end

data/lib/html2rss/attribute_post_processors/parse_time.rb

@@ -1,5 +1,7 @@
-require 'active_support'
-require 'active_support/core_ext/time'
+# frozen_string_literal: true
+
+require 'time'
+require_relative '../utils'
 
 module Html2rss
   module AttributePostProcessors
@@ -24,15 +26,20 @@ module Html2rss
     #
     # It uses {https://ruby-doc.org/stdlib-2.5.3/libdoc/time/rdoc/Time.html#method-c-parse Time.parse}.
     class ParseTime
+      ##
+      # @param value [String] the time to parse
+      # @param env [Item::Context] Context object providing additional environment details
       def initialize(value, env)
         @value = value.to_s
         @time_zone = env[:config].time_zone
       end
 
       ##
-      # @return [String] rfc822 formatted time
+      # Converts the provided time string to RFC822 format, taking into account the configured time zone.
+      #
+      # @return [String] RFC822 formatted time
       def get
-        Time.use_zone(@time_zone) { Time.zone.parse(@value).rfc822 }
+        Utils.use_zone(@time_zone) { Time.parse(@value).rfc822 }
       end
     end
   end

data/lib/html2rss/attribute_post_processors/parse_uri.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module Html2rss
   module AttributePostProcessors
     ##
     # Returns the URI as String.
+    # If the URL is relative, it builds an absolute one with the channel's URL as base.
     #
     # Imagine this HTML structure:
     #
@@ -19,14 +22,21 @@ module Html2rss
     # Would return:
     #    'http://why-not-use-a-link.uh'
     class ParseUri
-      def initialize(value, _env)
+      ##
+      # @param value [String]
+      # @param context [Item::Context]
+      def initialize(value, context)
         @value = value
+        @config_url = context.config.url
       end
 
       ##
       # @return [String]
       def get
-        URI(Html2rss::Utils.sanitize_url(@value)).to_s
+        Html2rss::Utils.build_absolute_url_from_relative(
+          Html2rss::Utils.sanitize_url(@value),
+          @config_url
+        ).to_s
       end
     end
   end

data/lib/html2rss/attribute_post_processors/sanitize_html.rb

@@ -1,17 +1,21 @@
+# frozen_string_literal: true
+
 require 'sanitize'
+require_relative 'html_transformers/transform_urls_to_absolute_ones'
+require_relative 'html_transformers/wrap_img_in_a'
 
 module Html2rss
   module AttributePostProcessors
     ##
     # Returns sanitized HTML code as String.
     #
-    # It adds:
+    # It sanitizes by using the [sanitize gem](https://github.com/rgrove/sanitize) with
+    # [Sanitize::Config::RELAXED](https://github.com/rgrove/sanitize#sanitizeconfigrelaxed).
+    #
+    # Furthermore, it adds:
     #
     # - `rel="nofollow noopener noreferrer"` to <a> tags
     # - `referrer-policy='no-referrer'` to <img> tags
-    #
-    # It also:
-    #
     # - wraps all <img> tags, whose direct parent is not an <a>, into an <a>
     #   linking to the <img>'s `src`.
     #
@@ -35,68 +39,60 @@ module Html2rss
     # Would return:
     #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
     class SanitizeHtml
-      URL_ELEMENTS_WITH_URL_ATTRIBUTE = { 'a' => :href, 'img' => :src }.freeze
-      private_constant :URL_ELEMENTS_WITH_URL_ATTRIBUTE
-
+      ##
+      # @param value [String]
+      # @param env [Item::Context]
       def initialize(value, env)
         @value = value
         @channel_url = env[:config].url
       end
 
       ##
-      # - uses the {https://github.com/rgrove/sanitize sanitize gem}
-      # - uses the config {https://github.com/rgrove/sanitize#sanitizeconfigrelaxed Sanitize::Config::RELAXED}
-      # - adds rel="nofollow noopener noreferrer" to a elements
-      # - adds target="_blank" to a elements
       # @return [String]
       def get
-        Sanitize.fragment(@value, sanitize_config).to_s.split.join(' ')
+        sanitized_html = Sanitize.fragment(@value, sanitize_config)
+        sanitized_html.to_s.gsub(/\s+/, ' ').strip
       end
 
       private
 
+      ##
+      # @return [Sanitize::Config]
       def sanitize_config
         Sanitize::Config.merge(
           Sanitize::Config::RELAXED,
           attributes: { all: %w[dir lang alt title translate] },
-          add_attributes: {
-            'a' => { 'rel' => 'nofollow noopener noreferrer', 'target' => '_blank' },
-            'img' => { 'referrer-policy' => 'no-referrer' }
-          },
-          transformers: [transform_urls_to_absolute_ones, wrap_img_in_a]
+          add_attributes:,
+          transformers: [
+            method(:transform_urls_to_absolute_ones),
+            method(:wrap_img_in_a)
+          ]
         )
       end
 
-      def transform_urls_to_absolute_ones
-        lambda do |env|
-          return unless URL_ELEMENTS_WITH_URL_ATTRIBUTE.key?(env[:node_name])
-
-          url_attribute = URL_ELEMENTS_WITH_URL_ATTRIBUTE[env[:node_name]]
-          url = env[:node][url_attribute]
-
-          return if URI(url).absolute?
-
-          absolute_url = Html2rss::Utils.build_absolute_url_from_relative(url, @channel_url)
-
-          env[:node][url_attribute] = absolute_url
-        end
+      def add_attributes
+        {
+          'a' => { 'rel' => 'nofollow noopener noreferrer', 'target' => '_blank' },
+          'img' => { 'referrer-policy' => 'no-referrer' }
+        }
       end
 
-      def wrap_img_in_a
-        lambda do |env|
-          return if env[:node_name] != 'img'
-
-          img = env[:node]
-
-          return if img.parent.name == 'a'
-
-          anchor = Nokogiri::XML::Node.new('a', img)
-          anchor[:href] = img[:src]
-
-          anchor.add_child img.dup
+      ##
+      # Wrapper for transform_urls_to_absolute_ones to pass the channel_url.
+      #
+      # @param env [Hash]
+      # @return [nil]
+      def transform_urls_to_absolute_ones(env)
+        HtmlTransformers::TransformUrlsToAbsoluteOnes.new(@channel_url).call(**env)
+      end
 
-          img.replace(anchor)
-        end
+      ##
+      # Wrapper for wrap_img_in_a.
+      #
+      # @param env [Hash]
+      # @return [nil]
+      def wrap_img_in_a(env)
+        HtmlTransformers::WrapImgInA.new.call(**env)
       end
     end
   end

data/lib/html2rss/attribute_post_processors/substring.rb

@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 module Html2rss
   module AttributePostProcessors
-    ## Returns a defined part of a String.
+    ##
+    # Returns a defined part of a String.
     #
     # Both parameters must be an Integer and they can be negative.
     # The +end+ parameter can be omitted, in that case it will not cut the
@@ -26,16 +29,23 @@ module Html2rss
     # Would return:
     #    'bar'
     class Substring
+      ##
+      # @param value [String] The original string to extract a substring from.
+      # @param env [Item::Context] Context object providing additional environment details.
       def initialize(value, env)
         @value = value
         @options = env[:options]
       end
 
       ##
-      # @return [String]
+      # Extracts the substring from the original string based on the provided start and end indices.
+      #
+      # @return [String] The extracted substring.
       def get
-        ending = @options.fetch(:end, @value.length).to_i
-        @value[@options[:start].to_i..ending]
+        start_index = @options[:start].to_i
+        end_index = @options[:end]&.to_i || @value.length
+
+        @value[start_index..end_index]
       end
     end
   end

data/lib/html2rss/attribute_post_processors/template.rb

@@ -1,25 +1,28 @@
-require 'sanitize'
+# frozen_string_literal: true
 
 module Html2rss
   module AttributePostProcessors
-    ## Returns a formatted String according to the string pattern.
+    ##
+    # Returns a formatted String according to the string pattern.
     #
     # If +self+ is used, the selectors extracted value will be used.
     # It uses [Kernel#format](https://ruby-doc.org/core/Kernel.html#method-i-format)
     #
     # Imagine this HTML:
+    #
     #    <li>
     #      <h1>Product</h1>
     #      <span class="price">23,42€</span>
     #    </li>
     #
+    #
     # YAML usage example:
     #
     #    selectors:
     #      items:
     #        selector: 'li'
     #      price:
-    #       selector: '.price'
+    #        selector: '.price'
     #      title:
     #        selector: h1
     #        post_process:
@@ -29,6 +32,9 @@ module Html2rss
     # Would return:
     #    'Product (23,42€)'
     class Template
+      ##
+      # @param value [String]
+      # @param env [Item::Context]
       def initialize(value, env)
         @value = value
         @options = env[:options]
@@ -39,28 +45,46 @@ module Html2rss
       ##
       # @return [String]
       def get
-        return format_string_with_methods if @options[:methods]
-
-        names = string.scan(/%[<|{](\w*)[>|}]/)
-        names.flatten!
-        names.compact!
-        names.map!(&:to_sym)
-
-        format(string, names.map { |name| [name, item_value(name)] }.to_h)
+        @options[:methods] ? format_string_with_methods : format_string_with_dynamic_params
       end
 
       private
 
+      ##
+      # @return [String] the string containing the template
       attr_reader :string
 
+      ##
+      # @return [Array<String>]
       def methods
-        @methods ||= @options[:methods].map(&method(:item_value))
+        @methods ||= @options[:methods].map { |method_name| item_value(method_name) }
       end
 
+      ##
+      # Formats a string using methods.
+      #
+      # @return [String]
+      # @deprecated Use %<id>s formatting instead. Will be removed in version 1.0.0. See README / Dynamic parameters.
       def format_string_with_methods
+        warn '[DEPRECATION] This method of using params is deprecated and \
+              support for it will be removed in version 1.0.0.\
+              Please use dynamic parameters (i.e. %<id>s, see README.md) instead.'
+
         string % methods
       end
 
+      ##
+      # @return [String]
+      def format_string_with_dynamic_params
+        param_names = string.scan(/%[<|{](\w*)[>|}]/)
+        param_names.flatten!
+
+        format(string, param_names.to_h { |name| [name.to_sym, item_value(name)] })
+      end
+
+      ##
+      # @param method_name [String, Symbol]
+      # @return [String]
       def item_value(method_name)
         method_name.to_sym == :self ? @value.to_s : @item.public_send(method_name).to_s
       end

data/lib/html2rss/attribute_post_processors.rb

@@ -1,13 +1,36 @@
+# frozen_string_literal: true
+
 module Html2rss
   ##
   # Provides a namespace for attribute post processors.
   module AttributePostProcessors
-    def self.get_processor(name)
-      @get_processor ||= Hash.new do |processors, key|
-        processors[key] = Utils.get_class_from_name(key, 'AttributePostProcessors')
-      end
+    ##
+    # Error raised when an unknown post processor name is requested.
+    class UnknownPostProcessorName < Html2rss::Error; end
 
-      @get_processor[name]
+    ##
+    # Maps the post processor name to the class implementing the post processor.
+    #
+    # The key is the name to use in the feed config.
+    NAME_TO_CLASS = {
+      gsub: Gsub,
+      html_to_markdown: HtmlToMarkdown,
+      markdown_to_html: MarkdownToHtml,
+      parse_time: ParseTime,
+      parse_uri: ParseUri,
+      sanitize_html: SanitizeHtml,
+      substring: Substring,
+      template: Template
+    }.freeze
+
+    ##
+    # Retrieves the attribute post processor class based on the given name.
+    #
+    # @param name [Symbol] The name of the post processor.
+    # @return [Class] The attribute post processor class.
+    # @raise [UnknownPostProcessorName] If the requested name is not found in NAME_TO_CLASS.
+    def self.get_processor(name)
+      NAME_TO_CLASS[name.to_sym] || raise(UnknownPostProcessorName, "Can't find a post processor named '#{name}'")
     end
   end
 end

data/lib/html2rss/cli.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative '../html2rss'
+require 'thor'
+
+module Html2rss
+  ##
+  # The Html2rss command line interface.
+  class CLI < Thor
+    def self.exit_on_failure?
+      true
+    end
+
+    desc 'feed YAML_FILE [FEED_NAME] [param=value ...]', 'Print RSS built from the YAML_FILE file to stdout'
+    ##
+    # Prints the feed to STDOUT.
+    #
+    # @param yaml_file [String] Path to the YAML configuration file.
+    # @param options [Array<String>] Additional options including feed name and parameters.
+    # @return [nil]
+    def feed(yaml_file, *options)
+      raise "File '#{yaml_file}' does not exist" unless File.exist?(yaml_file)
+
+      feed_name = options.shift
+      params = options.to_h { |opt| opt.split('=', 2) }
+      puts Html2rss.feed_from_yaml_config(yaml_file, feed_name, params:)
+    end
+  end
+end