html2rss 0.32.3 → 0.33.0

data/lib/html2rss/selectors.rb

@@ -132,7 +132,7 @@ module Html2rss
 
       field = @compiled_fields[name]
       return select_compiled(field, scope) if field
-      return select_field(name, scope:, config: nil) if name == :url
+      return select_field(name, scope:) if name == :url
 
       raise InvalidSelectorName, "Selector for '#{name}' is not defined."
     end
@@ -225,15 +225,11 @@ module Html2rss
       value.is_a?(Array) ? value : [value]
     end
 
-    def select_field(selector_key, scope:, config:)
-      return default_item_url(scope.item, scope.base_url) if fallback_url_selector?(selector_key, config, scope.item)
-      raise InvalidSelectorName, "Selector for '#{selector_key}' is not defined." if config.nil?
+    # Only reached for +:url+ without a configured selector, so the item must be an anchor.
+    def select_field(selector_key, scope:)
+      return default_item_url(scope.item, scope.base_url) if anchor_element?(scope.item)
 
-      if SPECIAL_ATTRIBUTES.member?(selector_key)
-        select_special(selector_key, scope:, config:)
-      else
-        select_regular(selector_key, scope:, config:)
-      end
+      raise InvalidSelectorName, "Selector for '#{selector_key}' is not defined."
     end
 
     def select_compiled(field, scope)
@@ -327,6 +323,7 @@ module Html2rss
     end
 
     def post_process(scope, value, post_process_steps)
+      value = step_input(value)
       post_process_steps.each do |step_config|
         value = PostProcessors.call(step_config[:name], value, scope.context_for(step_config:))
       end
@@ -334,9 +331,15 @@ module Html2rss
       value
     end
 
+    # Post-processors work on text, so a URL value (from the href extractor) enters them as a String.
+    def step_input(value)
+      value.is_a?(Url) ? value.to_s : value
+    end
+
     def apply_compiled_post_process(scope, value, steps)
       return value unless value && steps
 
+      value = step_input(value)
       steps.each do |klass, step_config|
         value = klass.new(value, scope.context_for(step_config:)).call
       end
@@ -375,15 +378,8 @@ module Html2rss
     end
 
     def extract_category_text(category)
-      text = case category
-             when Nokogiri::XML::Node, Nokogiri::XML::NodeSet
-               Html2rss::Html::Navigator.extract_visible_text(category)
-             else
-               category&.to_s
-             end
-
-      stripped = text&.strip
-      stripped unless stripped.nil? || stripped.empty?
+      stripped = category.to_s.strip
+      stripped unless stripped.empty?
     end
 
     def node_set_with_multiple_elements?(nodes)

data/lib/html2rss/sst/node.rb

@@ -2,7 +2,7 @@
 
 module Html2rss
   module SST
-    Node = Data.define(:name, :attrs, :own_text, :children, :tag_path)
+    Node = Data.define(:name, :attrs, :own_text, :tails, :children, :tag_path)
 
     ##
     # Immutable simplified semantic tree node. Predicates and traversal live on
@@ -11,12 +11,13 @@ module Html2rss
       ##
       # @param name [Symbol, String]
       # @param attrs [Attrs, nil]
-      # @param own_text [String, nil]
+      # @param own_text [String, nil] all direct text, in document order
+      # @param tails [Array<String>, nil] direct text following each child, parallel to +children+
       # @param children [Array<Node>, nil]
       # @param tag_path [String, nil] index-free ancestry path for list clustering
       # @return [Node]
       # @raise [ArgumentError] on invalid name/attrs/children
-      def self.build(name:, attrs: nil, own_text: nil, children: nil, tag_path: nil)
+      def self.build(name:, attrs: nil, own_text: nil, children: nil, tag_path: nil, tails: nil) # rubocop:disable Metrics/ParameterLists
         attrs_value = coerce_attrs(attrs)
         kids = coerce_children(children)
 
@@ -24,6 +25,7 @@ module Html2rss
           name: coerce_name(name),
           attrs: attrs_value,
           own_text: own_text.to_s.freeze,
+          tails: Array(tails).map { _1.to_s.freeze }.freeze,
           children: kids,
           tag_path: tag_path.to_s.freeze
         )
@@ -52,6 +54,10 @@ module Html2rss
       end
       private_class_method :coerce_children
 
+      ##
+      # @return [String] direct text before the first child
+      def lead_text = own_text.delete_suffix(tails.join)
+
       ##
       # @return [Boolean]
       def link?

data/lib/html2rss/sst/normalizer.rb

@@ -8,9 +8,6 @@ module Html2rss
     # Sole Nokogiri consumer on the heuristic auto-source path. Builds an
     # immutable SST::Document with parent/depth/chrome indices.
     class Normalizer # rubocop:disable Metrics/ClassLength
-      # Raised when no SST nodes survive normalization for the chosen root.
-      class EmptyTree < ArgumentError; end
-
       # Hard ceiling for SST node allocations; beyond this we degrade.
       MAX_NODES = 5_000
 
@@ -64,10 +61,10 @@ module Html2rss
         ##
         # @param input [String, Nokogiri::HTML::Document, Nokogiri::XML::Node]
         #   HTML string or already-parsed node (String is parsed once here)
-        # @return [Document]
-        # @raise [ArgumentError] when input is nil or unsupported
+        # @return [Document, nil] +nil+ when no SST nodes survive normalization
+        # @raise [InvalidOption] when input is nil or unsupported
         def call(input)
-          raise ArgumentError, 'input is required' if input.nil?
+          raise InvalidOption, 'input is required' if input.nil?
 
           new(coerce(input)).call
         end
@@ -81,7 +78,7 @@ module Html2rss
           when String then Nokogiri::HTML(input)
           when Nokogiri::XML::Node then input
           else
-            raise ArgumentError, "expected String or Nokogiri::XML::Node, got #{input.class}"
+            raise InvalidOption, "expected String or Nokogiri::XML::Node, got #{input.class}"
           end
         end
       end
@@ -97,11 +94,11 @@ module Html2rss
       end
 
       ##
-      # @return [Document]
+      # @return [Document, nil] +nil+ when no SST nodes survive normalization
       def call
         root_nk = resolve_root_nk
-        root = normalize_element(root_nk, parent: nil, depth: 0, path: '', chrome: false)
-        raise EmptyTree, 'SST Normalizer produced an empty tree' unless root
+        root = normalize_element(root_nk, depth: 0, path: '', chrome: false)
+        return unless root
 
         index = Index.new(root:, parents: @parents, depths: @depths, ignored_chrome: @ignored_chrome)
         Document.build(root:, index:, degraded: @degraded, node_count: @node_count)
@@ -150,15 +147,15 @@ module Html2rss
       end
 
       # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-      def normalize_element(nk_node, parent:, depth:, path:, chrome:)
+      def normalize_element(nk_node, depth:, path:, chrome:)
         return unless nk_node.element?
 
-        tag = Html2rss::Html::Probe.tag(nk_node)
+        tag = Html::Probe.tag(nk_node)
         return if STRIPPED_TAGS.include?(tag)
 
         if @node_count >= MAX_NODES && !@degraded
           @degraded = true
-          Html2rss::Log.warn({
+          Log.warn({
             event_name: 'sst.normalizer.max_nodes',
             max_nodes: MAX_NODES,
             message: "sst.normalizer MAX_NODES=#{MAX_NODES} breached; degrading to semantic-tag-only"
@@ -168,21 +165,15 @@ module Html2rss
         return if @degraded && !SEMANTIC_DEGRADE_TAGS.include?(tag)
 
         @node_count += 1
-        tag_path = path.empty? ? "/#{tag}" : "#{path}/#{tag}"
+        tag_path = "#{path}/#{tag}"
         attrs = extract_attrs(nk_node)
-        own_text = direct_text(nk_node)
         chrome_here = chrome || IGNORED_CONTAINER_TAGS.include?(tag)
 
-        children = nk_node.element_children.filter_map do |child|
-          normalize_element(child, parent: :pending, depth: depth + 1, path: tag_path, chrome: chrome_here)
-        end.freeze
+        children, lead, tails = normalize_children(nk_node, depth:, path: tag_path, chrome: chrome_here)
 
-        node = Node.build(name: tag, attrs:, own_text:, children:, tag_path:)
-        @parents[node] = parent == :pending ? nil : parent
+        node = Node.build(name: tag, attrs:, own_text: [lead, *tails].join, children:, tag_path:, tails:)
         @depths[node] = depth
         @ignored_chrome[node] = chrome_here
-
-        # Fix parent pointers now that node exists.
         children.each { |child| @parents[child] = node }
 
         node
@@ -213,8 +204,18 @@ module Html2rss
         end
       end
 
-      def direct_text(nk_node)
-        nk_node.children.select(&:text?).map(&:text).join
+      # Walks child nodes once, keeping direct text in the slot of the preceding kept child
+      # (slot 0 is the lead text) so text order survives the element-only tree.
+      def normalize_children(nk_node, depth:, path:, chrome:)
+        children = []
+        slots = [+'']
+        nk_node.children.each do |child|
+          next slots.last << child.text if child.text?
+
+          node = normalize_element(child, depth: depth + 1, path:, chrome:)
+          (children << node) && (slots << +'') if node
+        end
+        [children, slots.first, slots.drop(1)]
       end
     end
   end

data/lib/html2rss/sst/text.rb

@@ -5,77 +5,73 @@ module Html2rss
     ##
     # Visible-text extraction for SST nodes (port of Navigator::TextExtractor).
     module Text
-      module_function
+      class << self
+        ##
+        # @param node [Node]
+        # @param separator [String]
+        # @param exclude [Array<Node>, nil]
+        # @return [String, nil]
+        def extract(node, separator: ' ', exclude: nil)
+          raise ArgumentError, 'node is required' unless node
 
-      ##
-      # @param node [Node]
-      # @param separator [String]
-      # @param exclude [Array<Node>, nil]
-      # @return [String, nil]
-      def extract(node, separator: ' ', exclude: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
-        raise ArgumentError, 'node is required' unless node
-
-        excluded = exclude&.each_with_object({}.compare_by_identity) { |n, h| h[n] = true }
-        if node.children.empty?
-          text = node.own_text.to_s.gsub(/\s+/, ' ').strip
-          return text.empty? ? nil : text
+          excluded = exclude&.each_with_object({}.compare_by_identity) { |n, h| h[n] = true }
+          join_fragments(fragments(node, separator, excluded), separator)
         end
 
-        parts = collect_parts(node, separator, excluded)
-        own = node.own_text.to_s.gsub(/\s+/, ' ').strip
-        parts = [own, *parts] unless own.empty?
-        return if parts.empty?
-
-        parts.join.squeeze(' ').strip
-      end
+        private
 
-      def collect_parts(node, separator, excluded)
-        last_block = false
-        node.children.each_with_object([]) do |child, parts|
-          next if excluded&.[](child)
-          next unless visible_child?(child)
+        # Ordered [text, block?] pairs: lead text, then each child followed by the text after it.
+        def fragments(node, separator, excluded)
+          pairs = [[squish(node.lead_text), false]]
+          node.children.zip(node.tails).each do |child, tail|
+            pairs << child_text_and_block(child, separator, excluded) if shown?(child, excluded)
+            pairs << [squish(tail), false]
+          end
+          pairs
+        end
 
-          text, block = child_text_and_block(child, separator, excluded)
-          next if text.empty?
+        def join_fragments(pairs, separator)
+          parts = []
+          last_block = false
+          pairs.each do |text, block|
+            next if text.empty?
 
-          append_separator!(parts, separator, block, last_block)
-          parts << text
-          last_block = block
+            append_separator!(parts, separator, block, last_block)
+            parts << text
+            last_block = block
+          end
+          parts.join unless parts.empty?
         end
-      end
-      module_function :collect_parts
-      private_class_method :collect_parts
 
-      def child_text_and_block(child, separator, excluded)
-        text = if child.children.empty?
-                 child.own_text.gsub(/\s+/, ' ').strip
-               else
-                 extract(child, separator:, exclude: excluded&.keys).to_s.strip
-               end
-        text = "- #{text}" if child.name == :li && !text.empty?
-        [text, Tags::BLOCK_NAMES.include?(child.name)]
-      end
-      module_function :child_text_and_block
-      private_class_method :child_text_and_block
+        def shown?(child, excluded) = !excluded&.[](child) && visible_child?(child)
 
-      def append_separator!(parts, separator, is_block, last_was_block)
-        return if parts.empty?
+        def squish(text) = text.to_s.gsub(/\s+/, ' ').strip
 
-        parts << if is_block || last_was_block
-                   (separator == ' ' ? "\n" : separator)
+        def child_text_and_block(child, separator, excluded)
+          text = if child.children.empty?
+                   squish(child.own_text)
                  else
-                   ' '
+                   extract(child, separator:, exclude: excluded&.keys).to_s.strip
                  end
-      end
-      module_function :append_separator!
-      private_class_method :append_separator!
+          text = "- #{text}" if child.name == :li && !text.empty?
+          [text, Tags::BLOCK_NAMES.include?(child.name)]
+        end
+
+        def append_separator!(parts, separator, is_block, last_was_block)
+          return if parts.empty?
 
-      def visible_child?(node)
-        !Tags::INVISIBLE_NAMES.include?(node.name) &&
-          !(node.name == :a && node.attrs.href&.start_with?('#'))
+          parts << if is_block || last_was_block
+                     (separator == ' ' ? "\n" : separator)
+                   else
+                     ' '
+                   end
+        end
+
+        def visible_child?(node)
+          !Tags::INVISIBLE_NAMES.include?(node.name) &&
+            !(node.name == :a && node.attrs.href&.start_with?('#'))
+        end
       end
-      module_function :visible_child?
-      private_class_method :visible_child?
     end
   end
 end

data/lib/html2rss/status.rb

@@ -13,6 +13,10 @@ module Html2rss
   # Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
   # Tallies and counters are validated and frozen at construction (including Marshal load).
   class Status
+    # Keys kept in {#to_h} even when zero or empty.
+    ALWAYS_REPORTED = %i[version dedup_dropped].freeze
+    private_constant :ALWAYS_REPORTED
+
     class << self
       ##
       # Builds status from extracted articles and scrape telemetry.
@@ -71,9 +75,9 @@ module Html2rss
       version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
       strategy_attempts: [], admission_drops: {}, entry_url: nil, scrape_url: nil, entry_resolution: nil
     )
-      dedup = Integer(dedup_dropped)
-      attempts = Integer(attempt_count)
-      validate_counters!(dedup:, attempts:, selected_strategy:)
+      dedup = Option.integer(:dedup_dropped, dedup_dropped, min: 0)
+      attempts = Option.integer(:attempt_count, attempt_count, min: 0)
+      validate_selected_strategy!(selected_strategy, attempts)
 
       super(
         version: version.to_s.dup.freeze,
@@ -93,20 +97,8 @@ module Html2rss
     # Observability hash for web (+scraper_status+). Omits empty/absent optional keys.
     #
     # @return [Hash{Symbol => Object}]
-    # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- omit-empty optional keys stay explicit
     def to_h
-      {
-        version:,
-        dedup_dropped:,
-        **(scraper_tallies.any? ? { scraper_tallies: } : {}),
-        **(selected_strategy.nil? ? {} : { selected_strategy: }),
-        **(attempt_count.positive? ? { attempt_count: } : {}),
-        **(strategy_attempts.any? ? { strategy_attempts: } : {}),
-        **(admission_drops.any? ? { admission_drops: } : {}),
-        **(entry_url ? { entry_url: } : {}),
-        **(scrape_url ? { scrape_url: } : {}),
-        **(entry_resolution ? { entry_resolution: } : {})
-      }
+      super.reject { |key, value| !ALWAYS_REPORTED.include?(key) && omit?(value) }
     end
 
     ##
@@ -124,6 +116,14 @@ module Html2rss
 
     private
 
+    def omit?(value)
+      case value
+      when nil then true
+      when Integer then value.zero?
+      else value.empty?
+      end
+    end
+
     def marshal_dump
       [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts,
        admission_drops, entry_url, scrape_url, entry_resolution]
@@ -133,16 +133,16 @@ module Html2rss
       version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count,
         strategy_attempts, admission_drops, entry_url, scrape_url, entry_resolution = payload
       initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:,
-                 strategy_attempts:, admission_drops: admission_drops || {}, entry_url:, scrape_url:,
+                 strategy_attempts:, admission_drops:, entry_url:, scrape_url:,
                  entry_resolution:)
     end
 
     def freeze_tallies(tallies)
-      tallies.to_h.transform_keys(&:to_s).transform_values { |count| Integer(count) }.freeze
+      tallies.to_h { |name, count| [name.to_s, count] }.freeze
     end
 
     def freeze_attempts(attempts)
-      Array(attempts).map { |attempt| attempt.to_h.freeze }.freeze
+      attempts.map { |attempt| attempt.dup.freeze }.freeze
     end
 
     def freeze_optional_string(value)
@@ -161,17 +161,10 @@ module Html2rss
       end.transform_keys(&:to_sym).freeze
     end
 
-    def validate_counters!(dedup:, attempts:, selected_strategy:)
-      raise ArgumentError, 'dedup_dropped must be >= 0' if dedup.negative?
-      raise ArgumentError, 'attempt_count must be >= 0' if attempts.negative?
-
-      unless selected_strategy.nil? || selected_strategy.is_a?(Symbol)
-        raise ArgumentError, 'selected_strategy must be a Symbol or nil'
-      end
-
-      return unless selected_strategy && attempts < 1
-
-      raise ArgumentError, 'attempt_count must be >= 1 when selected_strategy is set'
+    def validate_selected_strategy!(selected_strategy, attempts)
+      return if selected_strategy.nil?
+      raise ArgumentError, 'selected_strategy must be a Symbol or nil' unless selected_strategy.is_a?(Symbol)
+      raise ArgumentError, 'attempt_count must be >= 1 when selected_strategy is set' if attempts < 1
     end
   end
 end

data/lib/html2rss/syndication/alternate_links.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Syndication
+    ##
+    # Scans raw HTML for +<link rel="alternate">+ hints that point at RSS/Atom feeds.
+    module AlternateLinks
+      LINK_TAG_RE = /<link\b[^>]*>/i
+      HREF_RE = /\bhref\s*=\s*["']([^"']+)["']/i
+      TYPE_RE = /\btype\s*=\s*["']([^"']+)["']/i
+      REL_RE = /\brel\s*=\s*["']([^"']+)["']/i
+      private_constant :LINK_TAG_RE, :HREF_RE, :TYPE_RE, :REL_RE
+
+      class << self
+        ##
+        # @param html [String] raw HTML
+        # @return [Array<String>] hrefs of alternate links that look like feeds
+        def hrefs(html)
+          html.scan(LINK_TAG_RE).filter_map { |tag| feed_href(tag) }
+        end
+
+        private
+
+        def feed_href(tag)
+          return unless tag[REL_RE, 1].to_s.downcase.split(/\s+/).include?('alternate')
+
+          href = tag[HREF_RE, 1]
+          return if href.nil? || href.empty?
+
+          href.strip if syndication_type?(tag[TYPE_RE, 1]) || href.match?(/rss|atom|feed|\.xml/i)
+        end
+
+        def syndication_type?(type)
+          type.to_s.downcase.match?(/rss|atom/)
+        end
+      end
+    end
+  end
+end