html2rss 0.11.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3c8df87caea7cc1dd9855993e9dd4fcb4285b37a5c1f87fb48a3d85e75ce0a51
-  data.tar.gz: 8ab432f13d76f45e30ea65316669c324bcba46b5234a082a6f0670cc445a6958
+  metadata.gz: 7a2bf557dd65533533e07b4581e195f2d2b32ff906831526a4d7aed27a558d71
+  data.tar.gz: f42e5f03649a08219d310a2545413c371f851530c4d323fd68ef783b4b3b5e13
 SHA512:
-  metadata.gz: 7115ee36e47ac65d19b13bb0f2ef0b19ebdb387d2e6fa53128e804b8d22d6a834517b88642f5c3e31e8850e344a2f94b6cb0de6619504603cfeacae52e47f3e2
-  data.tar.gz: b2c9f342cff237d9cafd4817f2570d18803fa9ba844ab306681ac02d12fb7cf147f3b40117ab21c78937325d80623ad27e25c6df4c8f8150fe81be2a07d12817
+  metadata.gz: 724a1fa8ab15ae140278eb9b055f22e7aad12e94627795f7a2f13c78f5421607e39d6ba040821b4c47b69f963cc0180bf8e964ff0b896403cb6305ed1d67dbb5
+  data.tar.gz: a06c2e16b0b51c6b6d2184430efc2a4e8b2812fee413163aa2991567e7608141f1c18189fdded58c8c3383940c4790478cd631abc6a1470ad648b2030fdefaab

data/README.md

@@ -26,26 +26,40 @@ You can also install it as a dependency in your Ruby project:
 
 ## Generating a feed on the CLI
 
-Create a file called `my_config_file.yml` with this example content:
+### using automatic scraping
+
+html2rss offers an automatic scrapting feature. Try it with:
+
+`html2rss auto https://unmatchedstyle.com/`
+
+### creating a feed config file and using it
+
+If the results are not to your satisfaction, you can create a feed config file.
+
+Create a file called `my_config_file.yml` with this sample content:
 
 ```yml
 channel:
-  url: https://stackoverflow.com/questions
+  url: https://unmatchedstyle.com
 selectors:
   items:
-    selector: "#hot-network-questions > ul > li"
+    selector: "article[id^='post-']"
   title:
-    selector: a
+    selector: h2
   link:
     selector: a
     extractor: href
+  description:
+    selector: ".post-content"
+    post_process:
+      - name: sanitize_html
 ```
 
-Build the RSS with: `html2rss feed ./my_config_file.yml`.
+Build the feed from this config with: `html2rss feed ./my_config_file.yml`.
 
 ## Generating a feed with Ruby
 
-Here's a minimal working example in Ruby:
+Here's a minimal working example using Ruby:
 
 ```ruby
 require 'html2rss'
@@ -481,7 +495,7 @@ feeds:
 
 Your feed configs go below `feeds`. Everything else is part of the global config.
 
-Find a full example of a `feeds.yml` at [`spec/feeds.test.yml`](https://github.com/html2rss/html2rss/blob/master/spec/feeds.test.yml).
+Find a full example of a `feeds.yml` at [`spec/fixtures/feeds.test.yml`](https://github.com/html2rss/html2rss/blob/master/spec/fixtures/feeds.test.yml).
 
 Now you can build your feeds like this:
 
@@ -583,8 +597,22 @@ Recommended further readings:
 
 ### Contributing
 
-1. Fork it ( <https://github.com/html2rss/html2rss/fork> )
+Find ideas what to contribute in:
+
+1. <https://github.com/orgs/html2rss/discussions>
+2. the issues tracker: <https://github.com/html2rss/html2rss/issues>
+
+#### Development Helpers
+
+1. `bin/setup`: installs dependencies and sets up the development environment.
+2. `bin/guard`: automatically runs rspec, rubocop and reek when a file changes.
+3. for a modern Ruby development experience: install [`ruby-lsp`](https://github.com/Shopify/ruby-lsp) and integrate it to your IDE:
+   a. [Ruby in Visual Studio Code](https://code.visualstudio.com/docs/languages/ruby)
+
+#### How to submit changes
+
+1. Fork this repo ( <https://github.com/html2rss/html2rss/fork> )
 2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
+3. Implement a commit your changes (`git commit -am 'feat: add XYZ'`)
 4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request
+5. Create a new Pull Request using the Github web UI

data/html2rss.gemspec

@@ -38,6 +38,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'kramdown'
   spec.add_dependency 'mime-types', '> 3.0'
   spec.add_dependency 'nokogiri', '>= 1.10', '< 2.0'
+  spec.add_dependency 'parallel'
   spec.add_dependency 'regexp_parser'
   spec.add_dependency 'reverse_markdown', '~> 2.0'
   spec.add_dependency 'rss'

data/lib/html2rss/attribute_post_processors/base.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Provides a namespace for attribute post processors.
+  module AttributePostProcessors
+    ##
+    # All post processors must inherit from this base class and implement `self.validate_args!` and `#get`.
+    class Base
+      # Validates the presence of required options in the context
+      #
+      # @param keys [Array<Symbol>] the keys to check for presence
+      # @param context [Hash] the context containing options
+      # @raise [MissingOption] if any key is missing
+      def self.expect_options(keys, context)
+        keys.each do |key|
+          unless (options = context[:options]).key?(key)
+            raise MissingOption, "The `#{key}` option is missing in: #{options.inspect}", [],
+                  cause: nil
+          end
+        end
+      end
+
+      # Asserts that the value is of the expected type(s)
+      #
+      # @param value [Object] the value to check
+      # @param types [Array<Class>, Class] the expected type(s)
+      # @param name [String] the name of the option being checked
+      # @param context [Item::Context] the context
+      # @raise [InvalidType] if the value is not of the expected type(s)
+      def self.assert_type(value, types = [], name, context:)
+        types = [types] unless types.is_a?(Array)
+
+        return if types.any? { |type| value.is_a?(type) }
+
+        options = context[:options] if context.is_a?(Hash)
+        options ||= { file: File.basename(caller_locations(1, 1).first.absolute_path) }
+
+        raise InvalidType, format('The type of `%<name>s` must be %<types>s, but is: %<type>s in: %<options>s',
+                                  name:, types: types.join(' or '), type: value.class, options: options.inspect),
+              [], cause: nil
+      end
+
+      ##
+      # This method validates the arguments passed to the post processor. Must be implemented by subclasses.
+      def self.validate_args!(_value, _context)
+        raise NotImplementedError, 'You must implement the `validate_args!` method in the post processor'
+      end
+
+      # Initializes the post processor
+      #
+      # @param value [Object] the value to be processed
+      # @param context [Item::Context] the context
+      def initialize(value, context)
+        klass = self.class
+        # TODO: get rid of Hash
+        klass.assert_type(context, [Item::Context, Hash], 'context', context:)
+        klass.validate_args!(value, context)
+
+        @value = value
+        @context = context
+      end
+
+      attr_reader :value, :context
+
+      # Abstract method to be implemented by subclasses
+      #
+      # @raise [NotImplementedError] if not implemented in subclass
+      def get
+        raise NotImplementedError, 'You must implement the `get` method in the post processor'
+      end
+    end
+  end
+end

data/lib/html2rss/attribute_post_processors/gsub.rb

@@ -25,39 +25,39 @@ module Html2rss
     # `replacement` can be a String or a Hash.
     #
     # See the doc on [String#gsub](https://ruby-doc.org/core/String.html#method-i-gsub) for more info.
-    class Gsub
+    class Gsub < Base
+      def self.validate_args!(value, context)
+        assert_type value, String, :value, context:
+        expect_options(%i[replacement pattern], context)
+        assert_type context.dig(:options, :replacement), [String, Hash], :replacement, context:
+      end
+
       ##
       # @param value [String]
       # @param context [Item::Context]
       def initialize(value, context)
-        @value = value
-        @options = context[:options]
+        super
+
+        options = context[:options]
+
+        @replacement = options[:replacement]
+        @pattern = options[:pattern]
       end
 
       ##
       # @return [String]
       def get
-        @value.to_s.gsub(pattern, replacement)
+        value.to_s.gsub(pattern, replacement)
       end
 
       private
 
+      attr_accessor :replacement
+
       ##
       # @return [Regexp]
       def pattern
-        pattern = @options[:pattern]
-        raise ArgumentError, 'The `pattern` option is missing' unless pattern
-
-        pattern.is_a?(String) ? Utils.build_regexp_from_string(pattern) : pattern
-      end
-
-      ##
-      # @return [Hash, String]
-      def replacement
-        replacement = @options[:replacement]
-        return replacement if replacement.is_a?(String) || replacement.is_a?(Hash)
-
-        raise ArgumentError, 'The `replacement` option must be a String or Hash'
+        @pattern.is_a?(String) ? Utils.build_regexp_from_string(@pattern) : @pattern
       end
     end
   end

data/lib/html2rss/attribute_post_processors/html_to_markdown.rb

@@ -26,18 +26,17 @@ module Html2rss
     #
     # Would return:
     #    'Lorem **ipsum** dolor'
-    class HtmlToMarkdown
-      ##
-      # @param value [String]
-      # @param env [Item::Context]
-      def initialize(value, env)
-        @sanitized_value = SanitizeHtml.new(value, env).get
+    class HtmlToMarkdown < Base
+      def self.validate_args!(value, context)
+        assert_type value, String, :value, context:
       end
 
       ##
       # @return [String] formatted in Markdown
       def get
-        ReverseMarkdown.convert(@sanitized_value)
+        sanitized_value = SanitizeHtml.new(value, context).get
+
+        ReverseMarkdown.convert(sanitized_value)
       end
     end
   end

data/lib/html2rss/attribute_post_processors/markdown_to_html.rb

@@ -32,13 +32,9 @@ module Html2rss
     #    <h1>Section</h1>
     #
     #    <p>Price: 12.34</p>
-    class MarkdownToHtml
-      ##
-      # @param value [String] Markdown content to convert to HTML
-      # @param env [Item::Context] Context object providing additional environment details
-      def initialize(value, env)
-        @value = value
-        @env = env
+    class MarkdownToHtml < Base
+      def self.validate_args!(value, context)
+        assert_type value, String, :value, context:
       end
 
       ##
@@ -46,8 +42,8 @@ module Html2rss
       #
       # @return [String] Sanitized HTML content
       def get
-        html_content = Kramdown::Document.new(@value).to_html
-        SanitizeHtml.new(html_content, @env).get
+        html_content = Kramdown::Document.new(value).to_html
+        SanitizeHtml.new(html_content, context).get
       end
     end
   end

data/lib/html2rss/attribute_post_processors/parse_time.rb

@@ -24,22 +24,22 @@ module Html2rss
     # Would return:
     #    "Tue, 02 Jul 2019 00:00:00 +0200"
     #
-    # It uses {https://ruby-doc.org/stdlib-2.5.3/libdoc/time/rdoc/Time.html#method-c-parse Time.parse}.
-    class ParseTime
-      ##
-      # @param value [String] the time to parse
-      # @param env [Item::Context] Context object providing additional environment details
-      def initialize(value, env)
-        @value = value.to_s
-        @time_zone = env[:config].time_zone
+    # It uses `Time.parse`.
+    class ParseTime < Base
+      def self.validate_args!(value, context)
+        assert_type(value, String, :value, context:)
+        assert_type(context[:config].time_zone, String, :time_zone, context:)
       end
 
       ##
-      # Converts the provided time string to RFC822 format, taking into account the configured time zone.
+      # Converts the provided time string to RFC822 format, taking into account the time_zone.
       #
       # @return [String] RFC822 formatted time
+      # @raise [TZInfo::InvalidTimezoneIdentifier] if the configured time zone is invalid
       def get
-        Utils.use_zone(@time_zone) { Time.parse(@value).rfc822 }
+        time_zone = context[:config].time_zone
+
+        Utils.use_zone(time_zone) { Time.parse(value).rfc822 }
       end
     end
   end

data/lib/html2rss/attribute_post_processors/parse_uri.rb

@@ -21,21 +21,24 @@ module Html2rss
     #
     # Would return:
     #    'http://why-not-use-a-link.uh'
-    class ParseUri
-      ##
-      # @param value [String]
-      # @param context [Item::Context]
-      def initialize(value, context)
-        @value = value
-        @config_url = context.config.url
+    class ParseUri < Base
+      def self.validate_args!(value, context)
+        url_types = [String, URI::HTTP, Addressable::URI].freeze
+
+        assert_type(value, url_types, :value, context:)
+        assert_type(context.config.url, url_types, :url, context:)
+
+        raise ArgumentError, 'The `value` option is missing or empty.' if value.to_s.empty?
       end
 
       ##
       # @return [String]
       def get
+        config_url = context.config.url
+
         Html2rss::Utils.build_absolute_url_from_relative(
-          Html2rss::Utils.sanitize_url(@value),
-          @config_url
+          Html2rss::Utils.sanitize_url(value),
+          config_url
         ).to_s
       end
     end

data/lib/html2rss/attribute_post_processors/sanitize_html.rb

@@ -38,19 +38,26 @@ module Html2rss
     #
     # Would return:
     #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
-    class SanitizeHtml
+    class SanitizeHtml < Base
+      def self.validate_args!(value, context)
+        assert_type value, String, :value, context:
+      end
+
       ##
-      # @param value [String]
-      # @param env [Item::Context]
-      def initialize(value, env)
-        @value = value
-        @channel_url = env[:config].url
+      # Shorthand method to get the sanitized HTML.
+      # @param html [String]
+      # @param url [String, Addressable::URI]
+      def self.get(html, url)
+        raise ArgumentError, 'url must be a String or Addressable::URI' if url.to_s.empty?
+        return nil if html.to_s.empty?
+
+        new(html, { config: Config::Channel.new({ url: }) }).get
       end
 
       ##
       # @return [String]
       def get
-        sanitized_html = Sanitize.fragment(@value, sanitize_config)
+        sanitized_html = Sanitize.fragment(value, sanitize_config)
         sanitized_html.to_s.gsub(/\s+/, ' ').strip
       end
 
@@ -77,13 +84,15 @@ module Html2rss
         }
       end
 
+      def channel_url = context[:config].url
+
       ##
       # Wrapper for transform_urls_to_absolute_ones to pass the channel_url.
       #
       # @param env [Hash]
       # @return [nil]
       def transform_urls_to_absolute_ones(env)
-        HtmlTransformers::TransformUrlsToAbsoluteOnes.new(@channel_url).call(**env)
+        HtmlTransformers::TransformUrlsToAbsoluteOnes.new(channel_url).call(**env)
       end
 
       ##

data/lib/html2rss/attribute_post_processors/substring.rb

@@ -28,13 +28,15 @@ module Html2rss
     #
     # Would return:
     #    'bar'
-    class Substring
-      ##
-      # @param value [String] The original string to extract a substring from.
-      # @param env [Item::Context] Context object providing additional environment details.
-      def initialize(value, env)
-        @value = value
-        @options = env[:options]
+    class Substring < Base
+      def self.validate_args!(value, context)
+        assert_type value, String, :value, context:
+
+        options = context[:options]
+        assert_type options[:start], Integer, :start, context:
+
+        end_index = options[:end]
+        assert_type(end_index, Integer, :end, context:) if end_index
       end
 
       ##
@@ -42,11 +44,29 @@ module Html2rss
       #
       # @return [String] The extracted substring.
       def get
-        start_index = @options[:start].to_i
-        end_index = @options[:end]&.to_i || @value.length
+        value[range]
+      end
+
+      ##
+      # Determines the range for the substring extraction based on the provided start and end indices.
+      #
+      # @return [Range] The range object representing the start and end/Infinity (integers).
+      def range
+        return (start_index..) unless end_index?
+
+        if start_index == end_index
+          raise ArgumentError,
+                'The `start` value must be unequal to the `end` value.'
+        end
 
-        @value[start_index..end_index]
+        (start_index..end_index)
       end
+
+      private
+
+      def end_index?  = !context[:options][:end].to_s.empty?
+      def end_index   = context[:options][:end].to_i
+      def start_index = context[:options][:start].to_i
     end
   end
 end

data/lib/html2rss/attribute_post_processors/template.rb

@@ -31,15 +31,23 @@ module Html2rss
     #
     # Would return:
     #    'Product (23,42€)'
-    class Template
+    class Template < Base
+      def self.validate_args!(value, context)
+        assert_type value, String, :value, context:
+
+        string = context[:options]&.dig(:string).to_s
+        raise InvalidType, 'The `string` template is absent.' if string.empty?
+      end
+
       ##
       # @param value [String]
-      # @param env [Item::Context]
-      def initialize(value, env)
-        @value = value
-        @options = env[:options]
-        @item = env[:item]
-        @string = @options[:string]
+      # @param context [Item::Context]
+      def initialize(value, context)
+        super
+
+        @options = context[:options] || {}
+        @item = context[:item]
+        @string = @options[:string].to_s
       end
 
       ##
@@ -66,9 +74,9 @@ module Html2rss
       # @return [String]
       # @deprecated Use %<id>s formatting instead. Will be removed in version 1.0.0. See README / Dynamic parameters.
       def format_string_with_methods
-        warn '[DEPRECATION] This method of using params is deprecated and \
-              support for it will be removed in version 1.0.0.\
-              Please use dynamic parameters (i.e. %<id>s, see README.md) instead.'
+        Log.warn '[DEPRECATION] This method of using params is deprecated and \
+                  support for it will be removed in version 1.0.0.\
+                  Please use dynamic parameters (i.e. %<id>s, see README.md) instead.'
 
         string % methods
       end
@@ -86,7 +94,7 @@ module Html2rss
       # @param method_name [String, Symbol]
       # @return [String]
       def item_value(method_name)
-        method_name.to_sym == :self ? @value.to_s : @item.public_send(method_name).to_s
+        method_name.to_sym == :self ? value : @item.public_send(method_name).to_s
       end
     end
   end

data/lib/html2rss/attribute_post_processors.rb

@@ -8,6 +8,14 @@ module Html2rss
     # Error raised when an unknown post processor name is requested.
     class UnknownPostProcessorName < Html2rss::Error; end
 
+    ##
+    # Error raised when a required option is missing.
+    class MissingOption < Html2rss::Error; end
+
+    ##
+    # Error raised when an invalid type is provided.
+    class InvalidType < Html2rss::Error; end
+
     ##
     # Maps the post processor name to the class implementing the post processor.
     #

data/lib/html2rss/auto_source/article.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require 'zlib'
+require 'sanitize'
+
+module Html2rss
+  class AutoSource
+    ##
+    # Article is a simple data object representing an article extracted from a page.
+    # It is enumerable and responds to all keys specified in PROVIDED_KEYS.
+    class Article
+      include Enumerable
+      include Comparable
+
+      PROVIDED_KEYS = %i[id title description url image guid published_at scraper].freeze
+
+      # @param options [Hash<Symbol, String>]
+      def initialize(**options)
+        @to_h = {}
+        options.each_pair { |key, value| @to_h[key] = value.freeze if value }
+        @to_h.freeze
+
+        return unless (unknown_keys = options.keys - PROVIDED_KEYS).any?
+
+        Log.warn "Article: unknown keys found: #{unknown_keys.join(', ')}"
+      end
+
+      # Checks if the article is valid based on the presence of URL, ID, and either title or description.
+      # @return [Boolean] True if the article is valid, otherwise false.
+      def valid?
+        !url.to_s.empty? && (!title.to_s.empty? || !description.to_s.empty?) && !id.to_s.empty?
+      end
+
+      # @yield [key, value]
+      # @return [Enumerator] if no block is given
+      def each
+        return enum_for(:each) unless block_given?
+
+        PROVIDED_KEYS.each { |key| yield(key, public_send(key)) }
+      end
+
+      def id
+        @to_h[:id]
+      end
+
+      def title
+        @to_h[:title]
+      end
+
+      def description
+        return @description if defined?(@description)
+
+        return if url.to_s.empty? || @to_h[:description].to_s.empty?
+
+        @description ||= Html2rss::AttributePostProcessors::SanitizeHtml.get(@to_h[:description], url)
+      end
+
+      # @return [Addressable::URI, nil]
+      def url
+        @url ||= Html2rss::Utils.sanitize_url(@to_h[:url])
+      end
+
+      # @return [Addressable::URI, nil]
+      def image
+        @image ||= Html2rss::Utils.sanitize_url(@to_h[:image])
+      end
+
+      # Generates a unique identifier based on the URL and ID using CRC32.
+      # @return [String]
+      def guid
+        @guid ||= Zlib.crc32([url, id].join('#!/')).to_s(36).encode('utf-8')
+      end
+
+      # Parses and returns the published_at time.
+      # @return [Time, nil]
+      def published_at
+        return if (string = @to_h[:published_at].to_s).strip.empty?
+
+        @published_at ||= Time.parse(string)
+      rescue ArgumentError
+        nil
+      end
+
+      def scraper
+        @to_h[:scraper]
+      end
+
+      def <=>(other)
+        return nil unless other.is_a?(Article)
+
+        0 if other.all? { |key, value| value == public_send(key) ? public_send(key) <=> value : false }
+      end
+    end
+  end
+end