html2rss 0.21.0 → 0.22.1

data/lib/html2rss/html_extractor/id_generator.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # IdGenerator determines the unique ID for an article container node.
+    class IdGenerator
+      class << self
+        ##
+        # @param article_tag [Nokogiri::XML::Element] container node
+        # @param heading [Nokogiri::XML::Node, nil] heading node
+        # @param url [Html2rss::Url, nil] absolute article URL
+        # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
+        # @param fallback_anchorless [Boolean] whether to use fallback hashing
+        # @return [String, nil] the generated ID, if any
+        def call(article_tag, heading:, url:, selected_anchor:, fallback_anchorless:)
+          id_from_dom = parse_id_from_dom(article_tag, url, selected_anchor)
+          return id_from_dom if id_from_dom
+
+          heading_text = resolve_heading_text(article_tag, heading, fallback_anchorless)
+          if heading_text && !heading_text.strip.empty?
+            generate_slug(heading_text)
+          elsif fallback_anchorless
+            generate_content_hash(article_tag)
+          end
+        end
+
+        private
+
+        def parse_id_from_dom(article_tag, url, selected_anchor)
+          candidates = [article_tag['id'], article_tag.at_css('[id]')&.attr('id')]
+          candidates += [url&.path, url&.query] if selected_anchor
+          candidates.compact.reject(&:empty?).first
+        end
+
+        def resolve_heading_text(article_tag, heading, fallback_anchorless)
+          text = heading ? TextExtractor.call(heading) : nil
+          if text.nil? || text.strip.empty?
+            fallback_text_node_content(article_tag, fallback_anchorless)
+          else
+            text
+          end
+        end
+
+        def fallback_text_node_content(article_tag, fallback_anchorless)
+          return unless fallback_anchorless
+
+          article_tag.xpath('.//text()').find { |t| !t.text.strip.empty? }&.text&.strip
+        end
+
+        def generate_slug(text)
+          slug = text.downcase.gsub(/[^a-z0-9]+/, '-')
+          slug = slug[1..] if slug.start_with?('-')
+          slug = slug[0..-2] if slug.end_with?('-')
+          slug unless slug.empty?
+        end
+
+        def generate_content_hash(article_tag)
+          text = TextExtractor.call(article_tag).to_s.strip
+          Zlib.crc32(text).to_s(36) unless text.empty?
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor/semantic_anchor_candidates.rb

@@ -138,17 +138,7 @@ module Html2rss
         # @return [Boolean] true when the anchor is inside the selected heading
         def heading_anchor?
           heading = @context.heading
-          return false unless heading
-
-          curr = @anchor
-          container = @context.container
-          while curr.respond_to?(:parent)
-            return true if curr == heading
-            break if curr == container
-
-            curr = curr.parent
-          end
-          false
+          heading && (@anchor == heading || HtmlNavigator.descendant_of?(@anchor, heading))
         end
 
         # @return [Boolean] true when anchor text exactly matches heading text
@@ -183,15 +173,11 @@ module Html2rss
         end
 
         def utility_landmark_ancestor?
-          curr = @anchor.parent
           container = @context.container
-          while curr.respond_to?(:parent)
-            return true if Context::UTILITY_LANDMARK_TAGS.include?(curr.name)
-            break if curr == container
+          condition = proc { |node| node == container || Context::UTILITY_LANDMARK_TAGS.include?(node.name) }
+          landmark = HtmlNavigator.parent_until_condition(@anchor.parent, condition)
 
-            curr = curr.parent
-          end
-          false
+          landmark && landmark != container
         end
 
         def icon_only_anchor?

data/lib/html2rss/html_extractor/semantic_containers.rb

@@ -32,9 +32,34 @@ module Html2rss
           HtmlExtractor.ignored_container_path?(node, cache)
         end
 
-        # Preserve the original post-order traversal intent (specific-first)
-        # by sorting candidates by depth (descending) while keeping original document
-        # order for nodes at the same depth.
+        candidates = filter_nested_containers(candidates)
+        sort_by_depth(candidates)
+      end
+
+      private
+
+      def filter_nested_containers(candidates)
+        candidate_set = Set.new(candidates)
+        rejected = Set.new
+
+        candidates.each do |candidate_b|
+          next if candidate_b.name == 'div'
+
+          find_and_reject_ancestors(candidate_b, candidate_set, rejected)
+        end
+
+        candidates.reject { |c| rejected.include?(c) }
+      end
+
+      def find_and_reject_ancestors(node, candidate_set, rejected)
+        curr = node.parent
+        while curr && !curr.document? && curr.name != 'html'
+          rejected << curr if candidate_set.include?(curr)
+          curr = curr.parent
+        end
+      end
+
+      def sort_by_depth(candidates)
         candidates.each_with_index
                   .sort_by { |node, index| [-node.ancestors.size, index] }
                   .map!(&:first)

data/lib/html2rss/html_extractor/text_extractor.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # TextExtractor extracts visible text from DOM elements, preserving lists
+    # and block spacing while sanitizing white spaces.
+    class TextExtractor
+      # HTML block elements that trigger line breaks or special formatting.
+      BLOCK_TAGS = %w[p div li ul ol h1 h2 h3 h4 h5 h6 tr br].to_set.freeze
+      # Tags ignored when extracting visible text content.
+      INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
+
+      class << self
+        ##
+        # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
+        # @param separator [String] separator used to join text fragments (default is a space)
+        # @param exclude_nodes [Array<Nokogiri::XML::Node>, nil] nodes to exclude from extraction
+        # @return [String, nil] the concatenated visible text, or nil if none is found
+        def call(tag, separator: ' ', exclude_nodes: nil)
+          return tag.text.gsub(/\s+/, ' ').strip if tag.respond_to?(:text?) && tag.text?
+
+          parts = iterate_children(tag, separator, exclude_nodes)
+          return if parts.empty?
+
+          parts.join.squeeze(' ').strip
+        end
+
+        private
+
+        def iterate_children(tag, separator, exclude_nodes)
+          last = false
+          tag.children.each_with_object([]) do |c, p|
+            next if exclude_nodes&.include?(c) || !visible_child?(c)
+
+            text, block = process_child_node(c, separator, exclude_nodes)
+            next if text.empty?
+
+            append_separator!(p, separator, block, last)
+            (p << text) && (last = block)
+          end
+        end
+
+        def process_child_node(child, separator, exclude_nodes)
+          child_text = get_child_text(child, separator, exclude_nodes)
+          return ['', false] if child_text.empty?
+
+          child_text = "- #{child_text}" if child.name == 'li'
+          [child_text, BLOCK_TAGS.include?(child.name)]
+        end
+
+        def get_child_text(child, separator, exclude_nodes)
+          if child.children.empty?
+            child.text.to_s.gsub(/\s+/, ' ').strip
+          else
+            call(child, separator:, exclude_nodes:).to_s.strip
+          end
+        end
+
+        def append_separator!(parts, separator, is_block, last_was_block)
+          return if parts.empty?
+
+          parts << if is_block || last_was_block
+                     (separator == ' ' ? "\n" : separator)
+                   else
+                     ' '
+                   end
+        end
+
+        def visible_child?(node)
+          !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
+            !(node.name == 'a' && node['href']&.start_with?('#'))
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor.rb

@@ -4,13 +4,11 @@ module Html2rss
   ##
   # HtmlExtractor is responsible for extracting details (headline, url, images, etc.)
   # from an article_tag.
-  class HtmlExtractor # rubocop:disable Metrics/ClassLength
-    # Tags ignored when extracting visible text content from article containers.
-    INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
+  # rubocop:disable Metrics/ClassLength
+  class HtmlExtractor
     # Heading tags used to prioritize title extraction.
     HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
-    # Selector used to derive non-headline description nodes.
-    NON_HEADLINE_SELECTOR = (HEADING_TAGS.map { |tag| ":not(#{tag})" } + INVISIBLE_CONTENT_TAGS.to_a).freeze
+
     # Element tags that indicate ignored DOM chrome when found in a container path.
     IGNORED_CONTAINER_TAGS = %w[nav footer header svg script style].to_set.freeze
 
@@ -26,20 +24,14 @@ module Html2rss
     class << self
       ##
       # Extracts visible text from a given node and its children.
+      # Delegates to TextExtractor.
       #
       # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
       # @param separator [String] separator used to join text fragments (default is a space)
+      # @param exclude_nodes [Array<Nokogiri::XML::Node>, nil] nodes to exclude from extraction
       # @return [String, nil] the concatenated visible text, or nil if none is found
-      def extract_visible_text(tag, separator: ' ')
-        parts = tag.children.filter_map do |child|
-          next unless visible_child?(child)
-
-          raw_text = child.children.empty? ? child.text : extract_visible_text(child)
-          text = raw_text&.strip
-          text unless text.to_s.empty?
-        end
-
-        parts.join(separator).squeeze(' ').strip unless parts.empty?
+      def extract_visible_text(tag, separator: ' ', exclude_nodes: nil)
+        TextExtractor.call(tag, separator:, exclude_nodes:)
       end
 
       ##
@@ -55,42 +47,47 @@ module Html2rss
       # @param node [Nokogiri::XML::Node]
       # @param cache [Hash, nil] identity cache used to store results (must use compare_by_identity)
       # @return [Boolean] true when the node belongs to ignored DOM chrome
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
       def ignored_container_path?(node, cache = nil)
         return cache[node] if cache&.key?(node)
 
-        res = walk_ignored_container_path?(node)
-        cache[node] = res if cache
-        res
-      end
+        curr = node
+        visited = []
+        is_ignored = false
 
-      private
+        while curr.respond_to?(:parent) && curr
+          if cache&.key?(curr)
+            is_ignored = cache[curr]
+            break
+          end
 
-      def walk_ignored_container_path?(node)
-        curr = node
-        while curr.respond_to?(:parent)
-          return true if IGNORED_CONTAINER_TAGS.include?(curr.name)
+          if IGNORED_CONTAINER_TAGS.include?(curr.name)
+            is_ignored = true
+            break
+          end
 
+          visited << curr
           curr = curr.parent
         end
-        false
-      end
+        visited.each { |n| cache[n] = is_ignored } if cache
 
-      def visible_child?(node)
-        !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
-          !(node.name == 'a' && node['href']&.start_with?('#'))
+        is_ignored
       end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
     end
 
     ##
     # @param article_tag [Nokogiri::XML::Node] article-like container to extract from
     # @param base_url [String, Html2rss::Url] base url used to resolve relative links
     # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
-    def initialize(article_tag, base_url:, selected_anchor:)
+    # @param fallback_anchorless [Boolean] whether to fall back to anchorless extraction
+    def initialize(article_tag, base_url:, selected_anchor:, fallback_anchorless: false)
       raise ArgumentError, 'article_tag is required' unless article_tag
 
       @article_tag = article_tag
       @base_url = base_url
       @selected_anchor = selected_anchor
+      @fallback_anchorless = fallback_anchorless
     end
 
     # @return [Hash{Symbol => Object}] extracted article attributes
@@ -115,54 +112,73 @@ module Html2rss
       @extract_url ||= begin
         href = selected_anchor&.[]('href').to_s
 
-        Url.from_relative(href.split('#').first.strip, base_url) unless href.empty?
+        if href.empty?
+          anchorless_url_fallback
+        else
+          Url.from_relative(href.split('#').first.strip, base_url)
+        end
       end
     end
 
-    def extract_title
-      title_source = heading || selected_anchor
-      self.class.extract_visible_text(title_source) if title_source
+    def anchorless_url_fallback
+      return unless @fallback_anchorless
+
+      id = generate_id
+      Url.from_relative("##{id}", base_url) if id
     end
 
-    def heading
-      @heading ||= begin
-        tags = article_tag.css(HEADING_TAGS.join(','))
-        tags.any? ? select_best_heading(tags) : nil
-      end
+    # rubocop:disable Metrics/CyclomaticComplexity
+    def extract_title
+      source = heading || selected_anchor
+      title_text = source ? self.class.extract_visible_text(source) : fallback_anchorless_title
+      return unless title_text
+
+      kicker = kicker_node ? self.class.extract_visible_text(kicker_node).to_s.strip : nil
+      kicker && !kicker.empty? && !title_text.include?(kicker) ? "#{kicker}: #{title_text}" : title_text
     end
+    # rubocop:enable Metrics/CyclomaticComplexity
+
+    def fallback_anchorless_title
+      return unless @fallback_anchorless && selected_anchor.nil?
 
-    def select_best_heading(tags)
-      min_tag_name = tags.map(&:name).min
-      best_tag = nil
-      max_size = -1
+      text_node = article_tag.xpath('.//text()').find { |t| !t.text.strip.empty? }
+      text_node&.text&.strip
+    end
 
-      tags.each do |tag|
-        next if tag.name != min_tag_name
+    def heading
+      @heading ||= HeadingExtractor.call(
+        article_tag,
+        fallback_anchorless: @fallback_anchorless,
+        selected_anchor:
+      )
+    end
 
-        size = self.class.extract_visible_text(tag)&.size.to_i
-        (best_tag = tag) && (max_size = size) if size > max_size
+    def kicker_node
+      @kicker_node ||= begin
+        selector = '[data-tb-kicker], [class*="kicker"], [class*="eyebrow"], ' \
+                   '[class*="pre-title"], [class*="pretitle"], [class*="overline"]'
+        node = article_tag.at_css(selector)
+        node && heading && (node == heading || HtmlNavigator.descendant_of?(node, heading)) ? nil : node
       end
-
-      best_tag
     end
 
     def extract_description
-      text = self.class.extract_visible_text(article_tag.css(NON_HEADLINE_SELECTOR), separator: '<br>')
-      return text if text && !text.empty?
-
-      description = self.class.extract_visible_text(article_tag)
-      return nil if description.nil? || description.strip.empty?
+      exclude = [heading, selected_anchor, kicker_node].compact.to_set
+      description = self.class.extract_visible_text(article_tag, exclude_nodes: exclude)
+      return if description.nil?
 
-      description.strip
+      desc = description.strip
+      desc.empty? ? nil : desc
     end
 
     def generate_id
-      [
-        article_tag['id'],
-        article_tag.at_css('[id]')&.attr('id'),
-        extract_url&.path,
-        extract_url&.query
-      ].compact.reject(&:empty?).first
+      @generate_id ||= IdGenerator.call(
+        article_tag,
+        heading:,
+        url: (selected_anchor ? extract_url : nil),
+        selected_anchor:,
+        fallback_anchorless: @fallback_anchorless
+      )
     end
 
     def extract_image = ImageExtractor.call(article_tag, base_url:)
@@ -170,4 +186,5 @@ module Html2rss
     def extract_enclosures = EnclosureExtractor.call(article_tag, base_url)
     def extract_categories = CategoryExtractor.call(article_tag)
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/html2rss/html_navigator.rb

@@ -49,6 +49,23 @@ module Html2rss
 
         current_tag.ancestors(tag_name).first
       end
+
+      ##
+      # Returns true if child_node is a descendant of parent_node.
+      # Walks up using parent pointers to avoid NodeSet allocations.
+      #
+      # @param child_node [Nokogiri::XML::Node] potential descendant
+      # @param parent_node [Nokogiri::XML::Node] potential ancestor
+      # @return [Boolean] true when child_node is a descendant of parent_node
+      def descendant_of?(child_node, parent_node)
+        curr = child_node.respond_to?(:parent) ? child_node.parent : nil
+        while curr
+          return true if curr == parent_node
+
+          curr = curr.respond_to?(:parent) ? curr.parent : nil
+        end
+        false
+      end
     end
   end
 end

data/lib/html2rss/rendering.rb

@@ -4,6 +4,8 @@ module Html2rss
   # Namespace for HTML rendering logic, used to generate rich content such as
   # images, audio, video, or embedded documents for feed descriptions.
   #
+  # @see Html2rss::Rendering::DescriptionBuilder
+  #
   # @example
   #   Html2rss::Rendering::ImageRenderer.new(
   #     url: "https://example.com/image.jpg",
@@ -16,8 +18,6 @@ module Html2rss
   #     image: "https://example.com/image.jpg",
   #     title: "Example"
   #   )
-  #
-  # @see Html2rss::Rendering::DescriptionBuilder
   module Rendering
   end
 end

data/lib/html2rss/request_service/local_file_strategy.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Strategy to read a local HTML file.
+    class LocalFileStrategy < Strategy
+      ##
+      # Executes the local file read.
+      #
+      # @return [Response] the mock response wrapped around the file contents
+      # @raise [ArgumentError] if the local file path is missing
+      # @raise [Errno::ENOENT] if the file does not exist
+      def execute
+        file_path = ctx.request[:local_file_path]
+        raise ArgumentError, 'Local file path is required for local_file strategy' unless file_path
+        raise Errno::ENOENT, "File not found: #{file_path}" unless File.exist?(file_path)
+
+        body = File.read(file_path)
+        Response.new(
+          body:,
+          headers: { 'content-type' => 'text/html; charset=utf-8' },
+          url: ctx.url,
+          status: 200
+        )
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/puppet_commander.rb

@@ -97,6 +97,10 @@ module Html2rss
 
       attr_reader :ctx, :browser, :skip_request_resources, :referer, :latest_navigation_response, :main_frame
 
+      ##
+      # Re-raises a deferred navigation error when one was captured.
+      #
+      # @raise [Html2rss::Error] when a navigation request or response validation failed
       def raise_navigation_error_if_any
         raise @navigation_error if @navigation_error
       end

data/lib/html2rss/request_service.rb

@@ -57,7 +57,8 @@ module Html2rss
       @strategies = {
         faraday: FaradayStrategy,
         botasaurus: BotasaurusStrategy,
-        browserless: BrowserlessStrategy
+        browserless: BrowserlessStrategy,
+        local_file: LocalFileStrategy
       }
       @default_strategy_name = :faraday
     end

data/lib/html2rss/selectors/extractors/attribute.rb

@@ -25,7 +25,7 @@ module Html2rss
       # during post processing with {PostProcessors::ParseTime}.
       class Attribute
         # The available options for the attribute extractor.
-        Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+        Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true)
 
         ##
         # Initializes the Attribute extractor.

data/lib/html2rss/selectors/extractors/href.rb

@@ -25,7 +25,7 @@ module Html2rss
       #    'http://blog-without-a-feed.example.com/posts/latest-findings'
       class Href
         # The available options for the href (attribute) extractor.
-        Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+        Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true)
 
         ##
         # Initializes the Href extractor.

data/lib/html2rss/selectors/extractors/html.rb

@@ -24,7 +24,7 @@ module Html2rss
       # {PostProcessors::SanitizeHtml}.
       class Html
         # The available options for the html extractor.
-        Options = Struct.new('HtmlOptions', :selector, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+        Options = Struct.new('HtmlOptions', :selector, keyword_init: true)
 
         ##
         # Initializes the Html extractor.

data/lib/html2rss/selectors/extractors/static.rb

@@ -17,7 +17,7 @@ module Html2rss
       #    'Foobar'
       class Static
         # The available option for the static extractor.
-        Options = Struct.new('StaticOptions', :static, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+        Options = Struct.new('StaticOptions', :static, keyword_init: true)
 
         ##
         # Initializes the Static extractor.

data/lib/html2rss/selectors/extractors/text.rb

@@ -22,7 +22,7 @@ module Html2rss
       #    'Lorem ipsum dolor ...'
       class Text
         # The available options for the text extractor.
-        Options = Struct.new('TextOptions', :selector, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+        Options = Struct.new('TextOptions', :selector, keyword_init: true)
 
         ##
         # Initializes the Text extractor.

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -128,8 +128,15 @@ module Html2rss
         ##
         # @return [String, nil]
         def get
-          sanitized_html = Sanitize.fragment(value, self.class.sanitize_config(channel_url)).to_s
+          # Temporarily replace newlines with a placeholder to preserve them during space collapsing
+          temp_value = value.to_s.gsub("\n", ' __NEWLINE_PLACEHOLDER__ ')
+          sanitized_html = Sanitize.fragment(temp_value, self.class.sanitize_config(channel_url)).to_s
           sanitized_html.gsub!(/\s+/, ' ')
+
+          # Restore newlines and clean up surrounding whitespace
+          sanitized_html.gsub!(/[ \t\r]*__NEWLINE_PLACEHOLDER__[ \t\r]*/, "\n")
+          sanitized_html.gsub!(/\n{3,}/, "\n\n")
+
           sanitized_html.strip!
           sanitized_html.empty? ? nil : sanitized_html
         end

data/lib/html2rss/selectors.rb

@@ -18,7 +18,7 @@ module Html2rss
     include Enumerable
 
     # A context instance passed to item extractors and post-processors.
-    Context = Struct.new('Context', :options, :item, :config, :scraper, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+    Context = Struct.new('Context', :options, :item, :config, :scraper, keyword_init: true)
 
     # Default selectors options merged into user configuration.
     DEFAULT_CONFIG = { items: { enhance: true } }.freeze
@@ -103,11 +103,15 @@ module Html2rss
     # @param article_tag [Nokogiri::XML::Element] HTML element to extract additional info from.
     # @param base_url [String, Html2rss::Url] base URL for normalization during enhancement
     # @return [Hash] The enhanced article hash.
+    # rubocop:disable Metrics/MethodLength
     def enhance_article_hash(article_hash, article_tag, base_url = @url)
       selected_anchor = HtmlExtractor.main_anchor_for(article_tag)
-      return article_hash unless selected_anchor
-
-      extracted = HtmlExtractor.new(article_tag, base_url:, selected_anchor:).call
+      extracted = HtmlExtractor.new(
+        article_tag,
+        base_url:,
+        selected_anchor:,
+        fallback_anchorless: true
+      ).call
       return article_hash unless extracted
 
       extracted.each_with_object(article_hash) do |(key, value), hash|
@@ -116,6 +120,7 @@ module Html2rss
         hash[key] = value
       end
     end
+    # rubocop:enable Metrics/MethodLength
 
     ##
     # Selects the value for a given attribute from an HTML element.