html2rss 0.20.0 → 0.21.0

data/lib/html2rss/rss_builder/article.rb

@@ -9,6 +9,7 @@ module Html2rss
     ##
     # Article is a simple data object representing an article extracted from a page.
     # It is enumerable and responds to all keys specified in PROVIDED_KEYS.
+    # rubocop:disable Metrics/ClassLength
     class Article
       include Enumerable
       include Comparable
@@ -17,6 +18,11 @@ module Html2rss
       PROVIDED_KEYS = %i[id title description url image author guid published_at enclosures categories scraper].freeze
       # Separator used to build deterministic deduplication fingerprints.
       DEDUP_FINGERPRINT_SEPARATOR = '#!/'
+      # Sentinel object used to pre-initialize instance variables in the constructor.
+      # This ensures all Article instances share the exact same object shape (Ruby 3.3+ optimization),
+      # preventing performance warnings and slower instance variable access due to shape transitions
+      # when attributes are lazily/conditionally accessed in different sequences.
+      NOT_SET = Object.new.freeze
 
       # @param options [Hash{Symbol => String}]
       # @option options [String] :id stable article identifier
@@ -31,9 +37,9 @@ module Html2rss
       # @option options [Array<String>] :categories category labels
       # @option options [Class] :scraper scraper class that produced the article
       def initialize(**options)
-        @to_h = {}
-        options.each_pair { |key, value| @to_h[key] = value.freeze if value }
-        @to_h.freeze
+        @to_h = options.each_with_object({}) { |(k, v), h| h[k] = v.freeze if v }.freeze
+
+        @description = @url = @image = @guid = @enclosures = @enclosure = @categories = @published_at = NOT_SET
 
         return unless (unknown_keys = options.keys - PROVIDED_KEYS).any?
 
@@ -62,7 +68,9 @@ module Html2rss
 
       # @return [String] rendered article description
       def description
-        @description ||= Rendering::DescriptionBuilder.new(
+        return @description unless @description == NOT_SET
+
+        @description = Rendering::DescriptionBuilder.new(
           base: @to_h[:description],
           title:,
           url:,
@@ -73,12 +81,16 @@ module Html2rss
 
       # @return [Url, nil]
       def url
-        @url ||= Url.sanitize(@to_h[:url])
+        return @url unless @url == NOT_SET
+
+        @url = Url.sanitize(@to_h[:url])
       end
 
       # @return [Url, nil]
       def image
-        @image ||= Url.sanitize(@to_h[:image])
+        return @image unless @image == NOT_SET
+
+        @image = Url.sanitize(@to_h[:image])
       end
 
       # @return [String, nil]
@@ -87,7 +99,9 @@ module Html2rss
       # Generates a unique identifier based on the URL and ID using CRC32.
       # @return [String]
       def guid
-        @guid ||= Zlib.crc32(fetch_guid).to_s(36).encode('utf-8')
+        return @guid unless @guid == NOT_SET
+
+        @guid = Zlib.crc32(fetch_guid).to_s(36).encode('utf-8')
       end
 
       ##
@@ -100,27 +114,32 @@ module Html2rss
 
       # @return [Array<Html2rss::RssBuilder::Enclosure>] normalized enclosure objects
       def enclosures
-        @enclosures ||= Array(@to_h[:enclosures])
-                        .map { |enclosure| Html2rss::RssBuilder::Enclosure.new(**enclosure) }
+        return @enclosures unless @enclosures == NOT_SET
+
+        @enclosures = Array(@to_h[:enclosures])
+                      .map { |enclosure| Html2rss::RssBuilder::Enclosure.new(**enclosure) }
       end
 
       # @return [Html2rss::RssBuilder::Enclosure, nil]
       def enclosure
-        return @enclosure if defined?(@enclosure)
-
-        case (object = @to_h[:enclosures]&.first)
-        when Hash
-          @enclosure = Html2rss::RssBuilder::Enclosure.new(**object)
-        when nil
-          @enclosure = Html2rss::RssBuilder::Enclosure.new(url: image) if image
-        else
-          Log.warn "Article: unknown enclosure type: #{object.class}"
-        end
+        return @enclosure unless @enclosure == NOT_SET
+
+        @enclosure = case (object = @to_h[:enclosures]&.first)
+                     when Hash
+                       Html2rss::RssBuilder::Enclosure.new(**object)
+                     when nil
+                       Html2rss::RssBuilder::Enclosure.new(url: image) if image
+                     else
+                       Log.warn "Article: unknown enclosure type: #{object.class}"
+                       nil
+                     end
       end
 
       # @return [Array<String>] normalized, unique category names
       def categories
-        @categories ||= @to_h[:categories].dup.to_a.tap do |categories|
+        return @categories unless @categories == NOT_SET
+
+        @categories = @to_h[:categories].dup.to_a.tap do |categories|
           categories.map! { |category| category.to_s.strip }
           categories.reject!(&:empty?)
           categories.uniq!
@@ -130,11 +149,12 @@ module Html2rss
       # Parses and returns the published_at time.
       # @return [DateTime, nil]
       def published_at
-        return if (string = @to_h[:published_at].to_s.strip).empty?
+        return @published_at unless @published_at == NOT_SET
 
-        @published_at ||= DateTime.parse(string)
+        string = @to_h[:published_at].to_s.strip
+        @published_at = string.empty? ? nil : DateTime.parse(string)
       rescue ArgumentError
-        nil
+        @published_at = nil
       end
 
       # @return [Class, nil] scraper class that produced this article
@@ -183,5 +203,6 @@ module Html2rss
         value
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/html2rss/rss_builder/enclosure.rb

@@ -16,9 +16,11 @@ module Html2rss
       def self.guess_content_type_from_url(url, default: 'application/octet-stream')
         return default unless url
 
-        url = url.path.split('?').first
+        path = url.path
+        ext = File.extname(path)
+        ext = ext[1..] if ext.start_with?('.')
 
-        content_type = MIME::Types.type_for(File.extname(url).delete('.'))
+        content_type = MIME::Types.type_for(ext)
         content_type.first&.to_s || 'application/octet-stream'
       end
 

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -91,7 +91,6 @@ module Html2rss
         end
 
         ##
-        # Shorthand method to get the sanitized HTML.
         # @param html [String]
         # @param url [String, Html2rss::Url]
         # @return [String, nil]
@@ -102,10 +101,34 @@ module Html2rss
           new(html, context).get
         end
 
+        ##
+        # @param channel_url [String, Html2rss::Url]
+        # @return [Hash] the memoized sanitize configuration
+        # rubocop:disable Metrics/MethodLength, ThreadSafety/ClassInstanceVariable
+        def self.sanitize_config(channel_url)
+          @sanitize_configs ||= {}
+          @sanitize_configs[channel_url] ||= begin
+            config = Sanitize::Config.merge(
+              Sanitize::Config::RELAXED,
+              attributes: { all: %w[dir lang alt title translate] },
+              add_attributes: TAG_ATTRIBUTES,
+              transformers: [
+                lambda { |env|
+                  HtmlTransformers::TransformUrlsToAbsoluteOnes.new(channel_url).call(**env)
+                },
+                ->(env) { HtmlTransformers::WrapImgInA.new.call(**env) }
+              ]
+            )
+            config[:elements].push('audio', 'video', 'source')
+            config.freeze
+          end
+        end
+        # rubocop:enable Metrics/MethodLength, ThreadSafety/ClassInstanceVariable
+
         ##
         # @return [String, nil]
         def get
-          sanitized_html = Sanitize.fragment(value, sanitize_config).to_s
+          sanitized_html = Sanitize.fragment(value, self.class.sanitize_config(channel_url)).to_s
           sanitized_html.gsub!(/\s+/, ' ')
           sanitized_html.strip!
           sanitized_html.empty? ? nil : sanitized_html
@@ -114,40 +137,6 @@ module Html2rss
         private
 
         def channel_url = context.dig(:config, :channel, :url)
-
-        ##
-        # @return [Sanitize::Config]
-        def sanitize_config # rubocop:disable Metrics/MethodLength
-          config = Sanitize::Config.merge(
-            Sanitize::Config::RELAXED,
-            attributes: { all: %w[dir lang alt title translate] },
-            add_attributes: TAG_ATTRIBUTES,
-            transformers: [
-              method(:transform_urls_to_absolute_ones),
-              method(:wrap_img_in_a)
-            ]
-          )
-          config[:elements].push('audio', 'video', 'source')
-          config
-        end
-
-        ##
-        # Wrapper for transform_urls_to_absolute_ones to pass the channel_url.
-        #
-        # @param env [Hash]
-        # @return [nil]
-        def transform_urls_to_absolute_ones(env)
-          HtmlTransformers::TransformUrlsToAbsoluteOnes.new(channel_url).call(**env)
-        end
-
-        ##
-        # Wrapper for wrap_img_in_a.
-        #
-        # @param env [Hash]
-        # @return [nil]
-        def wrap_img_in_a(env)
-          HtmlTransformers::WrapImgInA.new.call(**env)
-        end
       end
     end
   end

data/lib/html2rss/selectors/post_processors/substring.rb

@@ -34,19 +34,17 @@ module Html2rss
         # @param context [Selectors::Context] post-processor context
         # @return [void]
         def self.validate_args!(value, context)
-          assert_type value, String, :value, context:
+          assert_type(value, String, :value, context:)
 
           options = context[:options]
-          assert_type options[:start], Integer, :start, context:
-
-          end_index = options[:end]
-          assert_type(end_index, Integer, :end, context:) if end_index
+          assert_type(options[:start], Integer, :start, context:)
+          assert_type(options[:end], Integer, :end, context:) if options.key?(:end)
         end
 
         ##
         # Extracts the substring from the original string based on the provided start and end indices.
         #
-        # @return [String] The extracted substring.
+        # @return [String, nil] The extracted substring.
         def get
           value[range]
         end
@@ -56,21 +54,16 @@ module Html2rss
         #
         # @return [Range] The range object representing the start and end/Infinity (integers).
         def range
-          return (start_index..) unless end_index?
+          options = context[:options]
+          start = options[:start]
 
-          if start_index == end_index
-            raise ArgumentError,
-                  'The `start` value must be unequal to the `end` value.'
-          end
+          return (start..) unless options.key?(:end)
 
-          (start_index..end_index)
-        end
+          finish = options[:end]
+          raise ArgumentError, 'The `start` value must be unequal to the `end` value.' if start == finish
 
-        private
-
-        def end_index?  = !context[:options][:end].to_s.empty?
-        def end_index   = context[:options][:end].to_i
-        def start_index = context[:options][:start].to_i
+          (start..finish)
+        end
       end
     end
   end

data/lib/html2rss/selectors/post_processors/template.rb

@@ -29,7 +29,7 @@ module Html2rss
       #        selector: h1
       #        post_process:
       #          name: template
-      #          string: '%{self} (%{price})'
+      #          string: '`%{self}` (`%{price}`)'
       #
       # Would return:
       #    'Product (23,42€)'
@@ -54,12 +54,13 @@ module Html2rss
           @scraper = context[:scraper]
           @item = context[:item]
           @string = @options[:string].to_s
+          @getter = ->(key) { item_value(key) }
         end
 
         ##
         # @return [String]
         def get
-          Html2rss::Config::DynamicParams.call(@string, {}, getter: method(:item_value), replace_missing_with: '')
+          Html2rss::Config::DynamicParams.call(@string, {}, getter: @getter, replace_missing_with: '')
         end
 
         private

data/lib/html2rss/selectors.rb

@@ -199,7 +199,10 @@ module Html2rss
     end
 
     def select_regular(_name, item:, config:, base_url:)
-      value = Extractors.get(config.merge(channel: channel_context(base_url)), item)
+      @merged_configs ||= {}
+      merged_config = @merged_configs[[config.object_id, base_url]] ||=
+        config.merge(channel: channel_context(base_url)).freeze
+      value = Extractors.get(merged_config, item)
 
       if value && (post_process_steps = config[:post_process])
         steps = post_process_steps.is_a?(Array) ? post_process_steps : [post_process_steps]
@@ -210,8 +213,9 @@ module Html2rss
     end
 
     def post_process(item, value, post_process_steps, base_url:)
+      pp_context = channel_post_process_context(base_url)
       post_process_steps.each do |options|
-        context = Context.new(config: { channel: { url: base_url, time_zone: @time_zone } },
+        context = Context.new(config: pp_context,
                               item:, scraper: self, options:)
 
         value = PostProcessors.get(options[:name], value, context)
@@ -262,7 +266,11 @@ module Html2rss
     end
 
     def category_node_options(selector_config, base_url:)
-      selector_config.merge(channel: channel_context(base_url), selector: nil)
+      @category_node_configs ||= {}
+      @category_node_configs[[selector_config.object_id, base_url]] ||= selector_config.merge(
+        channel: channel_context(base_url),
+        selector: nil
+      ).freeze
     end
 
     def apply_post_process_steps(item:, value:, post_process_steps:, base_url:)
@@ -288,7 +296,13 @@ module Html2rss
     end
 
     def channel_context(base_url)
-      { url: base_url, time_zone: @time_zone }
+      @channel_contexts ||= {}
+      @channel_contexts[base_url] ||= { url: base_url, time_zone: @time_zone }.freeze
+    end
+
+    def channel_post_process_context(base_url)
+      @channel_pp_contexts ||= {}
+      @channel_pp_contexts[base_url] ||= { channel: channel_context(base_url) }.freeze
     end
 
     # @return [Hash] enclosure details.

data/lib/html2rss/url.rb

@@ -54,8 +54,8 @@ module Html2rss
     # @param raw_url [String] the raw URL string to sanitize
     # @return [Url, nil] the sanitized URL, or nil if no valid URL found
     def self.sanitize(raw_url)
-      matched_urls = raw_url.to_s.scan(%r{(?:(?:https?|ftp|mailto)://|mailto:)[^\s<>"]+})
-      url = matched_urls.first.to_s.strip
+      match = raw_url.to_s.match(%r{(?:(?:https?|ftp|mailto)://|mailto:)[^\s<>"]+})
+      url = match ? match[0].strip : ''
       return nil if url.empty?
 
       new(Addressable::URI.parse(url).normalize)
@@ -125,6 +125,7 @@ module Html2rss
     # @param uri [Addressable::URI] the underlying Addressable::URI object (internal use only)
     def initialize(uri)
       @uri = uri.freeze
+      @path_segments = @uri.path.to_s.split('/').reject(&:empty?).freeze
       freeze
     end
 
@@ -162,7 +163,7 @@ module Html2rss
     # Returns the URL path split into non-empty segments.
     #
     # @return [Array<String>] normalized path segments
-    def path_segments = @uri.path.to_s.split('/').reject(&:empty?)
+    attr_reader :path_segments
 
     ##
     # Returns a copy of the URL with the provided path.

data/lib/html2rss/version.rb

@@ -4,6 +4,6 @@
 # The Html2rss namespace.
 module Html2rss
   # Current application version.
-  VERSION = '0.20.0'
+  VERSION = '0.21.0'
   public_constant :VERSION
 end

data/schema/html2rss-config.schema.json

@@ -379,6 +379,13 @@
           },
           "exclusiveMinimum": 0
         },
+        "total_timeout_seconds": {
+          "type": "integer",
+          "not": {
+            "type": "null"
+          },
+          "exclusiveMinimum": 0
+        },
         "browserless": {
           "type": "object",
           "properties": {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.20.0
+  version: 0.21.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -147,20 +147,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: puppeteer-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -390,7 +376,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.20.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.21.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
@@ -399,7 +385,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="