html2rss 0.23.0 → 0.25.0

data/lib/html2rss/sst/node.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Immutable simplified semantic tree node. Predicates and traversal live on
+    # the type so Scoring/Segmenter never touch Nokogiri.
+    Node = Data.define(:name, :attrs, :own_text, :children, :tag_path) do
+      ##
+      # @param name [Symbol, String]
+      # @param attrs [Attrs, nil]
+      # @param own_text [String, nil]
+      # @param children [Array<Node>, nil]
+      # @param tag_path [String, nil] index-free ancestry path for list clustering
+      # @return [Node]
+      # @raise [ArgumentError] on invalid name/attrs/children
+      def self.build(name:, attrs: nil, own_text: nil, children: nil, tag_path: nil)
+        attrs_value = coerce_attrs(attrs)
+        kids = coerce_children(children)
+
+        new(
+          name: coerce_name(name),
+          attrs: attrs_value,
+          own_text: own_text.to_s.freeze,
+          children: kids,
+          tag_path: tag_path.to_s.freeze
+        )
+      end
+
+      def self.coerce_name(name)
+        raise ArgumentError, 'name is required' if name.nil? || name.to_s.strip.empty?
+
+        name.to_sym
+      end
+      private_class_method :coerce_name
+
+      def self.coerce_attrs(attrs)
+        attrs_value = attrs.nil? ? Attrs.empty : attrs
+        raise ArgumentError, 'attrs must be SST::Attrs' unless attrs_value.is_a?(Attrs)
+
+        attrs_value
+      end
+      private_class_method :coerce_attrs
+
+      def self.coerce_children(children)
+        kids = Array(children).freeze
+        raise ArgumentError, 'children must be SST::Node instances' unless kids.all?(Node)
+
+        kids
+      end
+      private_class_method :coerce_children
+
+      ##
+      # @return [Boolean]
+      def link?
+        name == :a && eligible_href?
+      end
+
+      ##
+      # @return [Boolean]
+      def image? = name == :img
+
+      ##
+      # @return [Boolean]
+      def heading? = Tags::HEADING_NAMES.include?(name)
+
+      ##
+      # @return [Boolean]
+      def utility_landmark? = Tags::UTILITY_LANDMARK_NAMES.include?(name)
+
+      ##
+      # @return [Boolean]
+      def ignored_container_name? = Tags::IGNORED_CONTAINER_NAMES.include?(name)
+
+      ##
+      # @return [Boolean]
+      def eligible_href?
+        href = attrs.href
+        return false if href.nil? || href.empty?
+
+        Tags::SKIP_HREF_PREFIXES.none? { |prefix| href.start_with?(prefix) }
+      end
+
+      ##
+      # Depth-first enumeration of self and descendants.
+      #
+      # @yieldparam node [Node]
+      # @return [Enumerator, Array<Node>]
+      def each_node(&block)
+        return enum_for(:each_node) unless block
+
+        yield self
+        children.each { |child| child.each_node(&block) }
+      end
+
+      ##
+      # @return [Array<Node>] all descendants excluding self
+      def descendants
+        children.flat_map { |child| [child, *child.descendants] }
+      end
+
+      ##
+      # @return [Array<Node>]
+      def find_all(&predicate)
+        each_node.select(&predicate)
+      end
+
+      ##
+      # @return [Node, nil]
+      def find(&predicate)
+        each_node.find(&predicate)
+      end
+
+      ##
+      # Visible text with block newlines, excluding optional subtrees.
+      #
+      # @param separator [String]
+      # @param exclude [Array<Node>, nil]
+      # @return [String, nil]
+      def visible_text(separator: ' ', exclude: nil)
+        if exclude.nil? && separator == ' ' && (index = Index.for_node(self))
+          return index.memo_visible_text(self)
+        end
+
+        Text.extract(self, separator:, exclude:)
+      end
+
+      ##
+      # @return [Integer] alphanumeric word count of visible text
+      def word_count
+        if (index = Index.for_node(self))
+          return index.memo_word_count(self)
+        end
+
+        visible_text.to_s.scan(/\p{Alnum}+/).size
+      end
+
+      ##
+      # @return [Float] words per descendant link
+      def text_density
+        words = word_count
+        links = descendants.count(&:link?)
+        links.zero? ? words.to_f : words.to_f / links
+      end
+    end
+  end
+end

data/lib/html2rss/sst/normalizer.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module Html2rss
+  module SST
+    ##
+    # Sole Nokogiri consumer on the heuristic auto-source path. Builds an
+    # immutable SST::Document with parent/depth/chrome indices.
+    class Normalizer # rubocop:disable Metrics/ClassLength
+      # Hard ceiling for SST node allocations; beyond this we degrade.
+      MAX_NODES = 5_000
+
+      # Tags stripped entirely from the SST.
+      STRIPPED_TAGS = Set['script', 'style', 'noscript', 'iframe', 'svg', 'template'].freeze
+
+      # Kept when MAX_NODES forces semantic-tag-only degrade.
+      SEMANTIC_DEGRADE_TAGS = Set['html', 'body', 'article', 'section', 'li', 'tr', 'div', 'a',
+                                  'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'time', 'img', 'p',
+                                  'ul', 'ol', 'main'].freeze
+
+      # Attribute names preserved in Attrs#raw for extraction.
+      RAW_ATTR_KEEP = /
+        \A(
+          data- |
+          aria- |
+          class |
+          id |
+          href |
+          src |
+          srcset |
+          style |
+          datetime |
+          itemprop |
+          type |
+          category |
+          categories |
+          tag |
+          tags |
+          topic |
+          topics |
+          section |
+          sections |
+          label |
+          labels |
+          theme |
+          themes |
+          subject |
+          subjects
+        )
+      /xi
+
+      class << self
+        ##
+        # @param input [String, Nokogiri::HTML::Document, Nokogiri::XML::Node]
+        #   HTML string or already-parsed node (String is parsed once here)
+        # @return [Document]
+        # @raise [ArgumentError] when input is nil or unsupported
+        def call(input)
+          raise ArgumentError, 'input is required' if input.nil?
+
+          new(coerce(input)).call
+        end
+
+        private
+
+        # @param input [String, Nokogiri::XML::Node]
+        # @return [Nokogiri::XML::Node]
+        def coerce(input)
+          case input
+          when String then Nokogiri::HTML(input)
+          when Nokogiri::XML::Node then input
+          else
+            raise ArgumentError, "expected String or Nokogiri::XML::Node, got #{input.class}"
+          end
+        end
+      end
+
+      # @param parsed_body [Nokogiri::HTML::Document, Nokogiri::XML::Node]
+      def initialize(parsed_body)
+        @parsed_body = parsed_body
+        @node_count = 0
+        @degraded = false
+        @parents = {}.compare_by_identity
+        @depths = {}.compare_by_identity
+        @ignored_chrome = {}.compare_by_identity
+      end
+
+      ##
+      # @return [Document]
+      def call
+        root_nk = @parsed_body.respond_to?(:root) ? (@parsed_body.at_css('html') || @parsed_body.root) : @parsed_body
+        root = normalize_element(root_nk, parent: nil, depth: 0, path: '', chrome: false)
+        raise ArgumentError, 'SST Normalizer produced an empty tree' unless root
+
+        index = Index.new(root:, parents: @parents, depths: @depths, ignored_chrome: @ignored_chrome)
+        Document.build(root:, index:, degraded: @degraded, node_count: @node_count)
+      end
+
+      private
+
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+      def normalize_element(nk_node, parent:, depth:, path:, chrome:)
+        return unless nk_node.respond_to?(:name)
+        return if nk_node.text? || nk_node.comment? || nk_node.cdata?
+
+        tag = nk_node.name.to_s.downcase
+        return if STRIPPED_TAGS.include?(tag)
+        return if @degraded && !SEMANTIC_DEGRADE_TAGS.include?(tag)
+
+        if @node_count >= MAX_NODES && !@degraded
+          @degraded = true
+          Html2rss::Log.warn("sst.normalizer MAX_NODES=#{MAX_NODES} breached; degrading to semantic-tag-only")
+        end
+
+        return if @degraded && !SEMANTIC_DEGRADE_TAGS.include?(tag)
+
+        @node_count += 1
+        tag_path = path.empty? ? "/#{tag}" : "#{path}/#{tag}"
+        attrs = extract_attrs(nk_node)
+        own_text = direct_text(nk_node)
+        chrome_here = chrome || Tags::IGNORED_CONTAINER_NAMES.include?(tag.to_sym)
+
+        children = nk_node.children.filter_map do |child|
+          normalize_element(child, parent: :pending, depth: depth + 1, path: tag_path, chrome: chrome_here)
+        end.freeze
+
+        node = Node.build(name: tag, attrs:, own_text:, children:, tag_path:)
+        @parents[node] = parent == :pending ? nil : parent
+        @depths[node] = depth
+        @ignored_chrome[node] = chrome_here
+
+        # Fix parent pointers now that node exists.
+        children.each { |child| @parents[child] = node }
+
+        node
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+      def extract_attrs(nk_node) # rubocop:disable Metrics/MethodLength
+        Attrs.build(
+          href: nk_node['href'],
+          src: nk_node['src'],
+          id: nk_node['id'],
+          class_names: nk_node['class'].to_s.split(/\s+/).reject(&:empty?),
+          datetime: nk_node['datetime'],
+          itemprop: nk_node['itemprop'],
+          style: nk_node['style'],
+          srcset: nk_node['srcset'],
+          type: nk_node['type'],
+          raw: raw_attrs(nk_node)
+        )
+      end
+
+      def raw_attrs(nk_node)
+        typed = %w[href src id class datetime itemprop style srcset type].to_set
+        nk_node.attribute_nodes.each_with_object({}) do |attr, raw|
+          name = attr.name.to_s
+          next if typed.include?(name)
+          next unless name.match?(RAW_ATTR_KEEP)
+
+          raw[name] = attr.value.to_s
+        end
+      end
+
+      def direct_text(nk_node)
+        nk_node.children.select(&:text?).map(&:text).join
+      end
+    end
+  end
+end

data/lib/html2rss/sst/tags.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Shared tag-name and href constants for SST nodes.
+    module Tags
+      # Heading element names.
+      HEADING_NAMES = %i[h1 h2 h3 h4 h5 h6].to_set.freeze
+      # Block-level names that contribute structural newlines in text.
+      BLOCK_NAMES = %i[p div li ul ol h1 h2 h3 h4 h5 h6 tr br].to_set.freeze
+      # Names skipped when collecting visible text.
+      INVISIBLE_NAMES = %i[svg script noscript style template].to_set.freeze
+      # Landmark names treated as utility chrome.
+      UTILITY_LANDMARK_NAMES = %i[nav aside footer menu].to_set.freeze
+      # Container names ignored as article roots.
+      IGNORED_CONTAINER_NAMES = %i[nav footer header svg script style].to_set.freeze
+      # Names excluded from class/structure clustering.
+      CLUSTER_EXCLUDED_NAMES = %i[html body nav footer header svg script style].to_set.freeze
+      # Href prefixes that never count as content destinations.
+      SKIP_HREF_PREFIXES = ['#', 'javascript:', 'mailto:', 'tel:', 'file://', 'sms:', 'data:'].freeze
+      # Layout-ish names used when resolving nested wrapper groups.
+      LAYOUT_NAMES = %i[div section article li ul ol].to_set.freeze
+    end
+  end
+end

data/lib/html2rss/sst/text.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Visible-text extraction for SST nodes (port of Navigator::TextExtractor).
+    module Text
+      module_function
+
+      ##
+      # @param node [Node]
+      # @param separator [String]
+      # @param exclude [Array<Node>, nil]
+      # @return [String, nil]
+      def extract(node, separator: ' ', exclude: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+        raise ArgumentError, 'node is required' unless node
+
+        excluded = exclude&.each_with_object({}.compare_by_identity) { |n, h| h[n] = true }
+        if node.children.empty?
+          text = node.own_text.to_s.gsub(/\s+/, ' ').strip
+          return text.empty? ? nil : text
+        end
+
+        parts = collect_parts(node, separator, excluded)
+        own = node.own_text.to_s.gsub(/\s+/, ' ').strip
+        parts = [own, *parts] unless own.empty?
+        return if parts.empty?
+
+        parts.join.squeeze(' ').strip
+      end
+
+      def collect_parts(node, separator, excluded)
+        last_block = false
+        node.children.each_with_object([]) do |child, parts|
+          next if excluded&.[](child)
+          next unless visible_child?(child)
+
+          text, block = child_text_and_block(child, separator, excluded)
+          next if text.empty?
+
+          append_separator!(parts, separator, block, last_block)
+          parts << text
+          last_block = block
+        end
+      end
+      module_function :collect_parts
+      private_class_method :collect_parts
+
+      def child_text_and_block(child, separator, excluded)
+        text = if child.children.empty?
+                 child.own_text.gsub(/\s+/, ' ').strip
+               else
+                 extract(child, separator:, exclude: excluded&.keys).to_s.strip
+               end
+        text = "- #{text}" if child.name == :li && !text.empty?
+        [text, Tags::BLOCK_NAMES.include?(child.name)]
+      end
+      module_function :child_text_and_block
+      private_class_method :child_text_and_block
+
+      def append_separator!(parts, separator, is_block, last_was_block)
+        return if parts.empty?
+
+        parts << if is_block || last_was_block
+                   (separator == ' ' ? "\n" : separator)
+                 else
+                   ' '
+                 end
+      end
+      module_function :append_separator!
+      private_class_method :append_separator!
+
+      def visible_child?(node)
+        !Tags::INVISIBLE_NAMES.include?(node.name) &&
+          !(node.name == :a && node.attrs.href&.start_with?('#'))
+      end
+      module_function :visible_child?
+      private_class_method :visible_child?
+    end
+  end
+end

data/lib/html2rss/sst.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Simplified Semantic Tree namespace for the heuristic auto-source pipeline.
+  module SST
+  end
+end

data/lib/html2rss/status.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Shared RSS +generator+ / JSON Feed +user_comment+ formatter string.
+  #
+  # Exposed publicly via {FeedResult#status}. Safe to log without reading articles.
+  # Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
+  # Tallies and counters are validated and frozen at construction (including Marshal load).
+  Status = Data.define(
+    :version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts
+  ) do
+    class << self
+      ##
+      # Builds status from extracted articles and scrape telemetry.
+      #
+      # @param articles [Array<Html2rss::Article>] articles kept after deduplication
+      # @param dedup_dropped [Integer] number of articles removed by deduplication
+      # @param selected_strategy [Symbol, nil] concrete strategy that succeeded under +:auto+ (else +nil+)
+      # @param attempt_count [Integer] auto-fallback attempt count (0 when not under +:auto+)
+      # @param strategy_attempts [Array<Hash>] auto-fallback attempt hashes (empty outside +:auto+)
+      # @return [Html2rss::Status]
+      def build(articles:, dedup_dropped: 0, selected_strategy: nil, attempt_count: 0, strategy_attempts: [])
+        tallies = articles.filter_map(&:scraper).tally.transform_keys { |klass| scraper_name(klass) }
+        new(
+          version: Html2rss::VERSION,
+          scraper_tallies: tallies,
+          dedup_dropped:,
+          selected_strategy:,
+          attempt_count:,
+          strategy_attempts:
+        )
+      end
+
+      ##
+      # @param klass [Class, #to_s] scraper class
+      # @return [String] short scraper label for tallies / generator text
+      def scraper_name(klass)
+        klass.to_s.gsub(/(?:Html2rss|Scraper)::/, '')
+      end
+    end
+
+    ##
+    # @param version [String]
+    # @param scraper_tallies [Hash{String => Integer}]
+    # @param dedup_dropped [Integer]
+    # @param selected_strategy [Symbol, nil]
+    # @param attempt_count [Integer]
+    # @param strategy_attempts [Array<Hash>]
+    # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- Status Data.define members
+    def initialize(
+      version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0, strategy_attempts: []
+    )
+      dedup = Integer(dedup_dropped)
+      attempts = Integer(attempt_count)
+      validate_counters!(dedup:, attempts:, selected_strategy:)
+
+      super(
+        version: version.to_s.dup.freeze,
+        scraper_tallies: freeze_tallies(scraper_tallies),
+        dedup_dropped: dedup,
+        selected_strategy:,
+        attempt_count: attempts,
+        strategy_attempts: freeze_attempts(strategy_attempts)
+      )
+    end
+    # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
+
+    ##
+    # Observability hash for web (+scraper_status+). Omits empty/absent optional keys:
+    # +:scraper_tallies+ when empty, +:selected_strategy+ when +nil+, +:attempt_count+ when zero,
+    # +:strategy_attempts+ when empty.
+    # Data members remain available via readers even when omitted here.
+    #
+    # @return [Hash{Symbol => Object}] always +:version+ (String), +:dedup_dropped+ (Integer);
+    #   optionally +:scraper_tallies+, +:selected_strategy+, +:attempt_count+, +:strategy_attempts+
+    def to_h
+      {
+        version:,
+        dedup_dropped:,
+        **(scraper_tallies.any? ? { scraper_tallies: } : {}),
+        **(selected_strategy.nil? ? {} : { selected_strategy: }),
+        **(attempt_count.positive? ? { attempt_count: } : {}),
+        **(strategy_attempts.any? ? { strategy_attempts: } : {})
+      }
+    end
+
+    ##
+    # Formats the RSS +generator+ string and JSON Feed +user_comment+.
+    # Scraper-focused only — auto strategy summary stays on {#to_h}, not this string.
+    # Omits the +(scrapers: …)+ clause when tallies are empty.
+    #
+    # @return [String]
+    def to_generator_comment
+      return "html2rss V. #{version}" if scraper_tallies.empty?
+
+      counts = scraper_tallies.map { |name, count| "#{name} (#{count})" }
+      "html2rss V. #{version} (scrapers: #{counts.join(', ')})"
+    end
+
+    private
+
+    def marshal_dump
+      [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts]
+    end
+
+    def marshal_load((version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts))
+      initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:, strategy_attempts:)
+    end
+
+    def freeze_tallies(tallies)
+      tallies.to_h.transform_keys(&:to_s).transform_values { |count| Integer(count) }.freeze
+    end
+
+    def freeze_attempts(attempts)
+      Array(attempts).map { |attempt| attempt.to_h.freeze }.freeze
+    end
+
+    def validate_counters!(dedup:, attempts:, selected_strategy:)
+      raise ArgumentError, 'dedup_dropped must be >= 0' if dedup.negative?
+      raise ArgumentError, 'attempt_count must be >= 0' if attempts.negative?
+
+      unless selected_strategy.nil? || selected_strategy.is_a?(Symbol)
+        raise ArgumentError, 'selected_strategy must be a Symbol or nil'
+      end
+
+      return unless selected_strategy && attempts < 1
+
+      raise ArgumentError, 'attempt_count must be >= 1 when selected_strategy is set'
+    end
+  end
+end

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.23.0'
+  VERSION = '0.25.0'
   public_constant :VERSION
 end