html2rss 0.20.1 → 0.22.0

data/lib/html2rss/html_extractor.rb

@@ -4,15 +4,13 @@ module Html2rss
   ##
   # HtmlExtractor is responsible for extracting details (headline, url, images, etc.)
   # from an article_tag.
+  # rubocop:disable Metrics/ClassLength
   class HtmlExtractor
-    # Tags ignored when extracting visible text content from article containers.
-    INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
-    # Element path pattern ignored when traversing candidate article containers.
-    IGNORED_CONTAINER_PATH = /(nav|footer|header|svg|script|style)/i
     # Heading tags used to prioritize title extraction.
     HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
-    # Selector used to derive non-headline description nodes.
-    NON_HEADLINE_SELECTOR = (HEADING_TAGS.map { |tag| ":not(#{tag})" } + INVISIBLE_CONTENT_TAGS.to_a).freeze
+
+    # Element tags that indicate ignored DOM chrome when found in a container path.
+    IGNORED_CONTAINER_TAGS = %w[nav footer header svg script style].to_set.freeze
 
     # Anchor selector used to identify the canonical article link element.
     MAIN_ANCHOR_SELECTOR = begin
@@ -26,27 +24,47 @@ module Html2rss
     class << self
       ##
       # Extracts visible text from a given node and its children.
+      # Delegates to TextExtractor.
       #
       # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
       # @param separator [String] separator used to join text fragments (default is a space)
+      # @param exclude_nodes [Array<Nokogiri::XML::Node>, nil] nodes to exclude from extraction
       # @return [String, nil] the concatenated visible text, or nil if none is found
-      def extract_visible_text(tag, separator: ' ')
-        parts = tag.children.filter_map do |child|
-          next unless visible_child?(child)
+      def extract_visible_text(tag, separator: ' ', exclude_nodes: nil)
+        TextExtractor.call(tag, separator:, exclude_nodes:)
+      end
 
-          raw_text = child.children.empty? ? child.text : extract_visible_text(child)
-          text = raw_text&.strip
-          text unless text.to_s.empty?
-        end
+      ##
+      # @param article_tag [Nokogiri::XML::Node] article-like container to search within
+      # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
+      def main_anchor_for(article_tag)
+        return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
+
+        article_tag.at_css(MAIN_ANCHOR_SELECTOR)
+      end
 
-        parts.join(separator).squeeze(' ').strip unless parts.empty?
+      ##
+      # @param node [Nokogiri::XML::Node]
+      # @param cache [Hash, nil] identity cache used to store results (must use compare_by_identity)
+      # @return [Boolean] true when the node belongs to ignored DOM chrome
+      def ignored_container_path?(node, cache = nil)
+        return cache[node] if cache&.key?(node)
+
+        res = walk_ignored_container_path?(node)
+        cache[node] = res if cache
+        res
       end
 
       private
 
-      def visible_child?(node)
-        !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
-          !(node.name == 'a' && node['href']&.start_with?('#'))
+      def walk_ignored_container_path?(node)
+        curr = node
+        while curr.respond_to?(:parent)
+          return true if IGNORED_CONTAINER_TAGS.include?(curr.name)
+
+          curr = curr.parent
+        end
+        false
       end
     end
 
@@ -54,12 +72,14 @@ module Html2rss
     # @param article_tag [Nokogiri::XML::Node] article-like container to extract from
     # @param base_url [String, Html2rss::Url] base url used to resolve relative links
     # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
-    def initialize(article_tag, base_url:, selected_anchor:)
+    # @param fallback_anchorless [Boolean] whether to fall back to anchorless extraction
+    def initialize(article_tag, base_url:, selected_anchor:, fallback_anchorless: false)
       raise ArgumentError, 'article_tag is required' unless article_tag
 
       @article_tag = article_tag
       @base_url = base_url
       @selected_anchor = selected_anchor
+      @fallback_anchorless = fallback_anchorless
     end
 
     # @return [Hash{Symbol => Object}] extracted article attributes
@@ -80,68 +100,66 @@ module Html2rss
 
     attr_reader :article_tag, :base_url, :selected_anchor
 
-    class << self
-      ##
-      # @param article_tag [Nokogiri::XML::Node] article-like container to search within
-      # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
-      def main_anchor_for(article_tag)
-        return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
-
-        article_tag.at_css(MAIN_ANCHOR_SELECTOR)
-      end
-
-      ##
-      # @param node [Nokogiri::XML::Node, String] node or path to test
-      # @return [Boolean] true when the node belongs to ignored DOM chrome
-      def ignored_container_path?(node)
-        path = node.respond_to?(:path) ? node.path : node.to_s
-
-        path.match?(IGNORED_CONTAINER_PATH)
-      end
-    end
-
     def extract_url
       @extract_url ||= begin
         href = selected_anchor&.[]('href').to_s
 
-        Url.from_relative(href.split('#').first.strip, base_url) unless href.empty?
+        if href.empty?
+          anchorless_url_fallback
+        else
+          Url.from_relative(href.split('#').first.strip, base_url)
+        end
       end
     end
 
+    def anchorless_url_fallback
+      return unless @fallback_anchorless
+
+      id = generate_id
+      Url.from_relative("##{id}", base_url) if id
+    end
+
     def extract_title
       title_source = heading || selected_anchor
-      self.class.extract_visible_text(title_source) if title_source
+      if title_source
+        self.class.extract_visible_text(title_source)
+      else
+        fallback_anchorless_title
+      end
+    end
+
+    def fallback_anchorless_title
+      return unless @fallback_anchorless && selected_anchor.nil?
+
+      text_node = article_tag.xpath('.//text()').find { |t| !t.text.strip.empty? }
+      text_node&.text&.strip
     end
 
     def heading
-      @heading ||= begin
-        heading_tags = article_tag.css(HEADING_TAGS.join(',')).group_by(&:name)
-        smallest_heading = heading_tags.keys.min
-        if smallest_heading
-          heading_tags[smallest_heading]&.max_by do |tag|
-            self.class.extract_visible_text(tag)&.size.to_i
-          end
-        end
-      end
+      @heading ||= HeadingExtractor.call(
+        article_tag,
+        fallback_anchorless: @fallback_anchorless,
+        selected_anchor:
+      )
     end
 
     def extract_description
-      text = self.class.extract_visible_text(article_tag.css(NON_HEADLINE_SELECTOR), separator: '<br>')
-      return text if text && !text.empty?
-
-      description = self.class.extract_visible_text(article_tag)
-      return nil if description.nil? || description.strip.empty?
+      exclude = [heading, selected_anchor].compact.to_set
+      description = self.class.extract_visible_text(article_tag, exclude_nodes: exclude)
+      return if description.nil?
 
-      description.strip
+      desc = description.strip
+      desc.empty? ? nil : desc
     end
 
     def generate_id
-      [
-        article_tag['id'],
-        article_tag.at_css('[id]')&.attr('id'),
-        extract_url&.path,
-        extract_url&.query
-      ].compact.reject(&:empty?).first
+      @generate_id ||= IdGenerator.call(
+        article_tag,
+        heading:,
+        url: (selected_anchor ? extract_url : nil),
+        selected_anchor:,
+        fallback_anchorless: @fallback_anchorless
+      )
     end
 
     def extract_image = ImageExtractor.call(article_tag, base_url:)
@@ -149,4 +167,5 @@ module Html2rss
     def extract_enclosures = EnclosureExtractor.call(article_tag, base_url)
     def extract_categories = CategoryExtractor.call(article_tag)
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/html2rss/rendering/description_builder.rb

@@ -25,12 +25,12 @@ module Html2rss
       # @param end_of_range [Integer] Optional, defaults to half the text length
       # @return [String]
       def self.remove_pattern_from_start(text, pattern, end_of_range: (text.size * 0.5).to_i)
-        return text unless text.is_a?(String) && pattern.is_a?(String)
+        return text unless text.is_a?(String) && pattern.is_a?(String) && !pattern.empty?
 
         index = text.index(pattern)
-        return text if index.nil? || index >= end_of_range
+        return text if index.nil? || index > end_of_range
 
-        text.gsub(/^(.{0,#{end_of_range}})#{Regexp.escape(pattern)}/, '\1')
+        "#{text[0, index]}#{text[(index + pattern.size)..]}"
       end
 
       # @param base [String] The base text content for the description

data/lib/html2rss/rendering.rb

@@ -4,6 +4,8 @@ module Html2rss
   # Namespace for HTML rendering logic, used to generate rich content such as
   # images, audio, video, or embedded documents for feed descriptions.
   #
+  # @see Html2rss::Rendering::DescriptionBuilder
+  #
   # @example
   #   Html2rss::Rendering::ImageRenderer.new(
   #     url: "https://example.com/image.jpg",
@@ -16,8 +18,6 @@ module Html2rss
   #     image: "https://example.com/image.jpg",
   #     title: "Example"
   #   )
-  #
-  # @see Html2rss::Rendering::DescriptionBuilder
   module Rendering
   end
 end

data/lib/html2rss/request_service/local_file_strategy.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Strategy to read a local HTML file.
+    class LocalFileStrategy < Strategy
+      ##
+      # Executes the local file read.
+      #
+      # @return [Response] the mock response wrapped around the file contents
+      # @raise [ArgumentError] if the local file path is missing
+      # @raise [Errno::ENOENT] if the file does not exist
+      def execute
+        file_path = ctx.request[:local_file_path]
+        raise ArgumentError, 'Local file path is required for local_file strategy' unless file_path
+        raise Errno::ENOENT, "File not found: #{file_path}" unless File.exist?(file_path)
+
+        body = File.read(file_path)
+        Response.new(
+          body:,
+          headers: { 'content-type' => 'text/html; charset=utf-8' },
+          url: ctx.url,
+          status: 200
+        )
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/puppet_commander.rb

@@ -97,6 +97,10 @@ module Html2rss
 
       attr_reader :ctx, :browser, :skip_request_resources, :referer, :latest_navigation_response, :main_frame
 
+      ##
+      # Re-raises a deferred navigation error when one was captured.
+      #
+      # @raise [Html2rss::Error] when a navigation request or response validation failed
       def raise_navigation_error_if_any
         raise @navigation_error if @navigation_error
       end

data/lib/html2rss/request_service.rb

@@ -57,7 +57,8 @@ module Html2rss
       @strategies = {
         faraday: FaradayStrategy,
         botasaurus: BotasaurusStrategy,
-        browserless: BrowserlessStrategy
+        browserless: BrowserlessStrategy,
+        local_file: LocalFileStrategy
       }
       @default_strategy_name = :faraday
     end

data/lib/html2rss/rss_builder/article.rb

@@ -9,6 +9,7 @@ module Html2rss
     ##
     # Article is a simple data object representing an article extracted from a page.
     # It is enumerable and responds to all keys specified in PROVIDED_KEYS.
+    # rubocop:disable Metrics/ClassLength
     class Article
       include Enumerable
       include Comparable
@@ -17,6 +18,11 @@ module Html2rss
       PROVIDED_KEYS = %i[id title description url image author guid published_at enclosures categories scraper].freeze
       # Separator used to build deterministic deduplication fingerprints.
       DEDUP_FINGERPRINT_SEPARATOR = '#!/'
+      # Sentinel object used to pre-initialize instance variables in the constructor.
+      # This ensures all Article instances share the exact same object shape (Ruby 3.3+ optimization),
+      # preventing performance warnings and slower instance variable access due to shape transitions
+      # when attributes are lazily/conditionally accessed in different sequences.
+      NOT_SET = Object.new.freeze
 
       # @param options [Hash{Symbol => String}]
       # @option options [String] :id stable article identifier
@@ -31,9 +37,9 @@ module Html2rss
       # @option options [Array<String>] :categories category labels
       # @option options [Class] :scraper scraper class that produced the article
       def initialize(**options)
-        @to_h = {}
-        options.each_pair { |key, value| @to_h[key] = value.freeze if value }
-        @to_h.freeze
+        @to_h = options.each_with_object({}) { |(k, v), h| h[k] = v.freeze if v }.freeze
+
+        @description = @url = @image = @guid = @enclosures = @enclosure = @categories = @published_at = NOT_SET
 
         return unless (unknown_keys = options.keys - PROVIDED_KEYS).any?
 
@@ -62,7 +68,9 @@ module Html2rss
 
       # @return [String] rendered article description
       def description
-        @description ||= Rendering::DescriptionBuilder.new(
+        return @description unless @description == NOT_SET
+
+        @description = Rendering::DescriptionBuilder.new(
           base: @to_h[:description],
           title:,
           url:,
@@ -73,12 +81,16 @@ module Html2rss
 
       # @return [Url, nil]
       def url
-        @url ||= Url.sanitize(@to_h[:url])
+        return @url unless @url == NOT_SET
+
+        @url = Url.sanitize(@to_h[:url])
       end
 
       # @return [Url, nil]
       def image
-        @image ||= Url.sanitize(@to_h[:image])
+        return @image unless @image == NOT_SET
+
+        @image = Url.sanitize(@to_h[:image])
       end
 
       # @return [String, nil]
@@ -87,7 +99,9 @@ module Html2rss
       # Generates a unique identifier based on the URL and ID using CRC32.
       # @return [String]
       def guid
-        @guid ||= Zlib.crc32(fetch_guid).to_s(36).encode('utf-8')
+        return @guid unless @guid == NOT_SET
+
+        @guid = Zlib.crc32(fetch_guid).to_s(36).encode('utf-8')
       end
 
       ##
@@ -100,27 +114,32 @@ module Html2rss
 
       # @return [Array<Html2rss::RssBuilder::Enclosure>] normalized enclosure objects
       def enclosures
-        @enclosures ||= Array(@to_h[:enclosures])
-                        .map { |enclosure| Html2rss::RssBuilder::Enclosure.new(**enclosure) }
+        return @enclosures unless @enclosures == NOT_SET
+
+        @enclosures = Array(@to_h[:enclosures])
+                      .map { |enclosure| Html2rss::RssBuilder::Enclosure.new(**enclosure) }
       end
 
       # @return [Html2rss::RssBuilder::Enclosure, nil]
       def enclosure
-        return @enclosure if defined?(@enclosure)
-
-        case (object = @to_h[:enclosures]&.first)
-        when Hash
-          @enclosure = Html2rss::RssBuilder::Enclosure.new(**object)
-        when nil
-          @enclosure = Html2rss::RssBuilder::Enclosure.new(url: image) if image
-        else
-          Log.warn "Article: unknown enclosure type: #{object.class}"
-        end
+        return @enclosure unless @enclosure == NOT_SET
+
+        @enclosure = case (object = @to_h[:enclosures]&.first)
+                     when Hash
+                       Html2rss::RssBuilder::Enclosure.new(**object)
+                     when nil
+                       Html2rss::RssBuilder::Enclosure.new(url: image) if image
+                     else
+                       Log.warn "Article: unknown enclosure type: #{object.class}"
+                       nil
+                     end
       end
 
       # @return [Array<String>] normalized, unique category names
       def categories
-        @categories ||= @to_h[:categories].dup.to_a.tap do |categories|
+        return @categories unless @categories == NOT_SET
+
+        @categories = @to_h[:categories].dup.to_a.tap do |categories|
           categories.map! { |category| category.to_s.strip }
           categories.reject!(&:empty?)
           categories.uniq!
@@ -130,11 +149,12 @@ module Html2rss
       # Parses and returns the published_at time.
       # @return [DateTime, nil]
       def published_at
-        return if (string = @to_h[:published_at].to_s.strip).empty?
+        return @published_at unless @published_at == NOT_SET
 
-        @published_at ||= DateTime.parse(string)
+        string = @to_h[:published_at].to_s.strip
+        @published_at = string.empty? ? nil : DateTime.parse(string)
       rescue ArgumentError
-        nil
+        @published_at = nil
       end
 
       # @return [Class, nil] scraper class that produced this article
@@ -183,5 +203,6 @@ module Html2rss
         value
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/html2rss/rss_builder/enclosure.rb

@@ -16,9 +16,11 @@ module Html2rss
       def self.guess_content_type_from_url(url, default: 'application/octet-stream')
         return default unless url
 
-        url = url.path.split('?').first
+        path = url.path
+        ext = File.extname(path)
+        ext = ext[1..] if ext.start_with?('.')
 
-        content_type = MIME::Types.type_for(File.extname(url).delete('.'))
+        content_type = MIME::Types.type_for(ext)
         content_type.first&.to_s || 'application/octet-stream'
       end
 

data/lib/html2rss/selectors/extractors/attribute.rb

@@ -25,7 +25,7 @@ module Html2rss
       # during post processing with {PostProcessors::ParseTime}.
       class Attribute
         # The available options for the attribute extractor.
-        Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+        Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true)
 
         ##
         # Initializes the Attribute extractor.

data/lib/html2rss/selectors/extractors/href.rb

@@ -25,7 +25,7 @@ module Html2rss
       #    'http://blog-without-a-feed.example.com/posts/latest-findings'
       class Href
         # The available options for the href (attribute) extractor.
-        Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+        Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true)
 
         ##
         # Initializes the Href extractor.

data/lib/html2rss/selectors/extractors/html.rb

@@ -24,7 +24,7 @@ module Html2rss
       # {PostProcessors::SanitizeHtml}.
       class Html
         # The available options for the html extractor.
-        Options = Struct.new('HtmlOptions', :selector, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+        Options = Struct.new('HtmlOptions', :selector, keyword_init: true)
 
         ##
         # Initializes the Html extractor.

data/lib/html2rss/selectors/extractors/static.rb

@@ -17,7 +17,7 @@ module Html2rss
       #    'Foobar'
       class Static
         # The available option for the static extractor.
-        Options = Struct.new('StaticOptions', :static, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+        Options = Struct.new('StaticOptions', :static, keyword_init: true)
 
         ##
         # Initializes the Static extractor.

data/lib/html2rss/selectors/extractors/text.rb

@@ -22,7 +22,7 @@ module Html2rss
       #    'Lorem ipsum dolor ...'
       class Text
         # The available options for the text extractor.
-        Options = Struct.new('TextOptions', :selector, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+        Options = Struct.new('TextOptions', :selector, keyword_init: true)
 
         ##
         # Initializes the Text extractor.

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -91,7 +91,6 @@ module Html2rss
         end
 
         ##
-        # Shorthand method to get the sanitized HTML.
         # @param html [String]
         # @param url [String, Html2rss::Url]
         # @return [String, nil]
@@ -102,11 +101,42 @@ module Html2rss
           new(html, context).get
         end
 
+        ##
+        # @param channel_url [String, Html2rss::Url]
+        # @return [Hash] the memoized sanitize configuration
+        # rubocop:disable Metrics/MethodLength, ThreadSafety/ClassInstanceVariable
+        def self.sanitize_config(channel_url)
+          @sanitize_configs ||= {}
+          @sanitize_configs[channel_url] ||= begin
+            config = Sanitize::Config.merge(
+              Sanitize::Config::RELAXED,
+              attributes: { all: %w[dir lang alt title translate] },
+              add_attributes: TAG_ATTRIBUTES,
+              transformers: [
+                lambda { |env|
+                  HtmlTransformers::TransformUrlsToAbsoluteOnes.new(channel_url).call(**env)
+                },
+                ->(env) { HtmlTransformers::WrapImgInA.new.call(**env) }
+              ]
+            )
+            config[:elements].push('audio', 'video', 'source')
+            config.freeze
+          end
+        end
+        # rubocop:enable Metrics/MethodLength, ThreadSafety/ClassInstanceVariable
+
         ##
         # @return [String, nil]
         def get
-          sanitized_html = Sanitize.fragment(value, sanitize_config).to_s
+          # Temporarily replace newlines with a placeholder to preserve them during space collapsing
+          temp_value = value.to_s.gsub("\n", ' __NEWLINE_PLACEHOLDER__ ')
+          sanitized_html = Sanitize.fragment(temp_value, self.class.sanitize_config(channel_url)).to_s
           sanitized_html.gsub!(/\s+/, ' ')
+
+          # Restore newlines and clean up surrounding whitespace
+          sanitized_html.gsub!(/[ \t\r]*__NEWLINE_PLACEHOLDER__[ \t\r]*/, "\n")
+          sanitized_html.gsub!(/\n{3,}/, "\n\n")
+
           sanitized_html.strip!
           sanitized_html.empty? ? nil : sanitized_html
         end
@@ -114,40 +144,6 @@ module Html2rss
         private
 
         def channel_url = context.dig(:config, :channel, :url)
-
-        ##
-        # @return [Sanitize::Config]
-        def sanitize_config # rubocop:disable Metrics/MethodLength
-          config = Sanitize::Config.merge(
-            Sanitize::Config::RELAXED,
-            attributes: { all: %w[dir lang alt title translate] },
-            add_attributes: TAG_ATTRIBUTES,
-            transformers: [
-              method(:transform_urls_to_absolute_ones),
-              method(:wrap_img_in_a)
-            ]
-          )
-          config[:elements].push('audio', 'video', 'source')
-          config
-        end
-
-        ##
-        # Wrapper for transform_urls_to_absolute_ones to pass the channel_url.
-        #
-        # @param env [Hash]
-        # @return [nil]
-        def transform_urls_to_absolute_ones(env)
-          HtmlTransformers::TransformUrlsToAbsoluteOnes.new(channel_url).call(**env)
-        end
-
-        ##
-        # Wrapper for wrap_img_in_a.
-        #
-        # @param env [Hash]
-        # @return [nil]
-        def wrap_img_in_a(env)
-          HtmlTransformers::WrapImgInA.new.call(**env)
-        end
       end
     end
   end

data/lib/html2rss/selectors/post_processors/substring.rb

@@ -34,19 +34,17 @@ module Html2rss
         # @param context [Selectors::Context] post-processor context
         # @return [void]
         def self.validate_args!(value, context)
-          assert_type value, String, :value, context:
+          assert_type(value, String, :value, context:)
 
           options = context[:options]
-          assert_type options[:start], Integer, :start, context:
-
-          end_index = options[:end]
-          assert_type(end_index, Integer, :end, context:) if end_index
+          assert_type(options[:start], Integer, :start, context:)
+          assert_type(options[:end], Integer, :end, context:) if options.key?(:end)
         end
 
         ##
         # Extracts the substring from the original string based on the provided start and end indices.
         #
-        # @return [String] The extracted substring.
+        # @return [String, nil] The extracted substring.
         def get
           value[range]
         end
@@ -56,21 +54,16 @@ module Html2rss
         #
         # @return [Range] The range object representing the start and end/Infinity (integers).
         def range
-          return (start_index..) unless end_index?
+          options = context[:options]
+          start = options[:start]
 
-          if start_index == end_index
-            raise ArgumentError,
-                  'The `start` value must be unequal to the `end` value.'
-          end
+          return (start..) unless options.key?(:end)
 
-          (start_index..end_index)
-        end
+          finish = options[:end]
+          raise ArgumentError, 'The `start` value must be unequal to the `end` value.' if start == finish
 
-        private
-
-        def end_index?  = !context[:options][:end].to_s.empty?
-        def end_index   = context[:options][:end].to_i
-        def start_index = context[:options][:start].to_i
+          (start..finish)
+        end
       end
     end
   end