html2rss 0.24.0 → 0.26.0

data/lib/html2rss/sst/normalizer.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module Html2rss
+  module SST
+    ##
+    # Sole Nokogiri consumer on the heuristic auto-source path. Builds an
+    # immutable SST::Document with parent/depth/chrome indices.
+    class Normalizer # rubocop:disable Metrics/ClassLength
+      # Hard ceiling for SST node allocations; beyond this we degrade.
+      MAX_NODES = 5_000
+
+      # Tags stripped entirely from the SST.
+      STRIPPED_TAGS = Set['script', 'style', 'noscript', 'iframe', 'svg', 'template'].freeze
+
+      # Kept when MAX_NODES forces semantic-tag-only degrade.
+      SEMANTIC_DEGRADE_TAGS = Set['html', 'body', 'article', 'section', 'li', 'tr', 'div', 'a',
+                                  'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'time', 'img', 'p',
+                                  'ul', 'ol', 'main'].freeze
+
+      # Attribute names preserved in Attrs#raw for extraction.
+      RAW_ATTR_KEEP = /
+        \A(
+          data- |
+          aria- |
+          class |
+          id |
+          href |
+          src |
+          srcset |
+          style |
+          datetime |
+          itemprop |
+          type |
+          category |
+          categories |
+          tag |
+          tags |
+          topic |
+          topics |
+          section |
+          sections |
+          label |
+          labels |
+          theme |
+          themes |
+          subject |
+          subjects
+        )
+      /xi
+
+      class << self
+        ##
+        # @param input [String, Nokogiri::HTML::Document, Nokogiri::XML::Node]
+        #   HTML string or already-parsed node (String is parsed once here)
+        # @return [Document]
+        # @raise [ArgumentError] when input is nil or unsupported
+        def call(input)
+          raise ArgumentError, 'input is required' if input.nil?
+
+          new(coerce(input)).call
+        end
+
+        private
+
+        # @param input [String, Nokogiri::XML::Node]
+        # @return [Nokogiri::XML::Node]
+        def coerce(input)
+          case input
+          when String then Nokogiri::HTML(input)
+          when Nokogiri::XML::Node then input
+          else
+            raise ArgumentError, "expected String or Nokogiri::XML::Node, got #{input.class}"
+          end
+        end
+      end
+
+      # @param parsed_body [Nokogiri::HTML::Document, Nokogiri::XML::Node]
+      def initialize(parsed_body)
+        @parsed_body = parsed_body
+        @node_count = 0
+        @degraded = false
+        @parents = {}.compare_by_identity
+        @depths = {}.compare_by_identity
+        @ignored_chrome = {}.compare_by_identity
+      end
+
+      ##
+      # @return [Document]
+      def call
+        root_nk = @parsed_body.respond_to?(:root) ? (@parsed_body.at_css('html') || @parsed_body.root) : @parsed_body
+        root = normalize_element(root_nk, parent: nil, depth: 0, path: '', chrome: false)
+        raise ArgumentError, 'SST Normalizer produced an empty tree' unless root
+
+        index = Index.new(root:, parents: @parents, depths: @depths, ignored_chrome: @ignored_chrome)
+        Document.build(root:, index:, degraded: @degraded, node_count: @node_count)
+      end
+
+      private
+
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+      def normalize_element(nk_node, parent:, depth:, path:, chrome:)
+        return unless nk_node.respond_to?(:name)
+        return if nk_node.text? || nk_node.comment? || nk_node.cdata?
+
+        tag = nk_node.name.to_s.downcase
+        return if STRIPPED_TAGS.include?(tag)
+        return if @degraded && !SEMANTIC_DEGRADE_TAGS.include?(tag)
+
+        if @node_count >= MAX_NODES && !@degraded
+          @degraded = true
+          Html2rss::Log.warn("sst.normalizer MAX_NODES=#{MAX_NODES} breached; degrading to semantic-tag-only")
+        end
+
+        return if @degraded && !SEMANTIC_DEGRADE_TAGS.include?(tag)
+
+        @node_count += 1
+        tag_path = path.empty? ? "/#{tag}" : "#{path}/#{tag}"
+        attrs = extract_attrs(nk_node)
+        own_text = direct_text(nk_node)
+        chrome_here = chrome || Tags::IGNORED_CONTAINER_NAMES.include?(tag.to_sym)
+
+        children = nk_node.children.filter_map do |child|
+          normalize_element(child, parent: :pending, depth: depth + 1, path: tag_path, chrome: chrome_here)
+        end.freeze
+
+        node = Node.build(name: tag, attrs:, own_text:, children:, tag_path:)
+        @parents[node] = parent == :pending ? nil : parent
+        @depths[node] = depth
+        @ignored_chrome[node] = chrome_here
+
+        # Fix parent pointers now that node exists.
+        children.each { |child| @parents[child] = node }
+
+        node
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+      def extract_attrs(nk_node) # rubocop:disable Metrics/MethodLength
+        Attrs.build(
+          href: nk_node['href'],
+          src: nk_node['src'],
+          id: nk_node['id'],
+          class_names: nk_node['class'].to_s.split(/\s+/).reject(&:empty?),
+          datetime: nk_node['datetime'],
+          itemprop: nk_node['itemprop'],
+          style: nk_node['style'],
+          srcset: nk_node['srcset'],
+          type: nk_node['type'],
+          raw: raw_attrs(nk_node)
+        )
+      end
+
+      def raw_attrs(nk_node)
+        typed = %w[href src id class datetime itemprop style srcset type].to_set
+        nk_node.attribute_nodes.each_with_object({}) do |attr, raw|
+          name = attr.name.to_s
+          next if typed.include?(name)
+          next unless name.match?(RAW_ATTR_KEEP)
+
+          raw[name] = attr.value.to_s
+        end
+      end
+
+      def direct_text(nk_node)
+        nk_node.children.select(&:text?).map(&:text).join
+      end
+    end
+  end
+end

data/lib/html2rss/sst/tags.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Shared tag-name and href constants for SST nodes.
+    module Tags
+      # Heading element names.
+      HEADING_NAMES = %i[h1 h2 h3 h4 h5 h6].to_set.freeze
+      # Block-level names that contribute structural newlines in text.
+      BLOCK_NAMES = %i[p div li ul ol h1 h2 h3 h4 h5 h6 tr br].to_set.freeze
+      # Names skipped when collecting visible text.
+      INVISIBLE_NAMES = %i[svg script noscript style template].to_set.freeze
+      # Landmark names treated as utility chrome.
+      UTILITY_LANDMARK_NAMES = %i[nav aside footer menu].to_set.freeze
+      # Container names ignored as article roots.
+      IGNORED_CONTAINER_NAMES = %i[nav footer header svg script style].to_set.freeze
+      # Names excluded from class/structure clustering.
+      CLUSTER_EXCLUDED_NAMES = %i[html body nav footer header svg script style].to_set.freeze
+      # Href prefixes that never count as content destinations.
+      SKIP_HREF_PREFIXES = ['#', 'javascript:', 'mailto:', 'tel:', 'file://', 'sms:', 'data:'].freeze
+      # Layout-ish names used when resolving nested wrapper groups.
+      LAYOUT_NAMES = %i[div section article li ul ol].to_set.freeze
+    end
+  end
+end

data/lib/html2rss/sst/text.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Visible-text extraction for SST nodes (port of Navigator::TextExtractor).
+    module Text
+      module_function
+
+      ##
+      # @param node [Node]
+      # @param separator [String]
+      # @param exclude [Array<Node>, nil]
+      # @return [String, nil]
+      def extract(node, separator: ' ', exclude: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+        raise ArgumentError, 'node is required' unless node
+
+        excluded = exclude&.each_with_object({}.compare_by_identity) { |n, h| h[n] = true }
+        if node.children.empty?
+          text = node.own_text.to_s.gsub(/\s+/, ' ').strip
+          return text.empty? ? nil : text
+        end
+
+        parts = collect_parts(node, separator, excluded)
+        own = node.own_text.to_s.gsub(/\s+/, ' ').strip
+        parts = [own, *parts] unless own.empty?
+        return if parts.empty?
+
+        parts.join.squeeze(' ').strip
+      end
+
+      def collect_parts(node, separator, excluded)
+        last_block = false
+        node.children.each_with_object([]) do |child, parts|
+          next if excluded&.[](child)
+          next unless visible_child?(child)
+
+          text, block = child_text_and_block(child, separator, excluded)
+          next if text.empty?
+
+          append_separator!(parts, separator, block, last_block)
+          parts << text
+          last_block = block
+        end
+      end
+      module_function :collect_parts
+      private_class_method :collect_parts
+
+      def child_text_and_block(child, separator, excluded)
+        text = if child.children.empty?
+                 child.own_text.gsub(/\s+/, ' ').strip
+               else
+                 extract(child, separator:, exclude: excluded&.keys).to_s.strip
+               end
+        text = "- #{text}" if child.name == :li && !text.empty?
+        [text, Tags::BLOCK_NAMES.include?(child.name)]
+      end
+      module_function :child_text_and_block
+      private_class_method :child_text_and_block
+
+      def append_separator!(parts, separator, is_block, last_was_block)
+        return if parts.empty?
+
+        parts << if is_block || last_was_block
+                   (separator == ' ' ? "\n" : separator)
+                 else
+                   ' '
+                 end
+      end
+      module_function :append_separator!
+      private_class_method :append_separator!
+
+      def visible_child?(node)
+        !Tags::INVISIBLE_NAMES.include?(node.name) &&
+          !(node.name == :a && node.attrs.href&.start_with?('#'))
+      end
+      module_function :visible_child?
+      private_class_method :visible_child?
+    end
+  end
+end

data/lib/html2rss/sst.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Simplified Semantic Tree namespace for the heuristic auto-source pipeline.
+  module SST
+  end
+end

data/lib/html2rss/url.rb

@@ -21,6 +21,7 @@ module Html2rss
   #   url = Url.from_relative('/foo-bar/baz.txt', 'https://example.com')
   #   url.titleized # => "Foo Bar Baz"
   #   url.channel_titleized # => "example.com: Foo Bar Baz"
+  # rubocop:disable Metrics/ClassLength -- value object owns resolve/sanitize/titleize/identity
   class Url
     include Comparable
 
@@ -189,6 +190,14 @@ module Html2rss
       self.class.from_absolute(uri.normalize.to_s)
     end
 
+    ##
+    # Returns a copy of the URL with the fragment removed (dedup / self-link identity).
+    #
+    # @return [Url] this URL when already fragment-free; otherwise a new URL
+    def without_fragment
+      @uri.fragment ? self.class.from_absolute(@uri.omit(:fragment).normalize.to_s) : self
+    end
+
     ##
     # Returns a copy of the URL with the provided query values.
     #
@@ -259,13 +268,7 @@ module Html2rss
     # @param other [Object] the other object to compare with
     # @return [Boolean] true if the URLs are equal
     def ==(other) = other.is_a?(Url) && to_s == other.to_s
-
-    ##
-    # Supports hash-based comparisons by ensuring equality semantics match `hash`.
-    #
-    # @param other [Object] the other object to compare with
-    # @return [Boolean] true if the URLs are considered equal
-    def eql?(other) = other.is_a?(Url) && to_s == other.to_s
+    alias eql? ==
 
     ##
     # Returns the hash code for this URL.
@@ -279,4 +282,5 @@ module Html2rss
     # @return [String] the debug representation
     def inspect = "#<#{self.class}:#{object_id} @uri=#{@uri.inspect}>"
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.24.0'
+  VERSION = '0.26.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -3,7 +3,11 @@
 require 'zeitwerk'
 
 loader = Zeitwerk::Loader.for_gem
-loader.inflector.inflect('cli' => 'CLI')
+loader.inflector.inflect(
+  'cli' => 'CLI',
+  'sst' => 'SST',
+  'mcp' => 'MCP'
+)
 loader.setup
 
 require 'logger'
@@ -67,7 +71,10 @@ module Html2rss
   # rubocop:disable Metrics/ParameterLists
 
   ##
-  # Scrapes the provided URL and returns an RSS object.
+  # Scrapes the provided URL without hand-written selectors and returns an RSS object.
+  #
+  # Builds an auto_source config, then FeedPipeline runs structured scrapers and
+  # (when needed) the SST heuristic path — see docs/auto_source.md.
   #
   # @param url [String] source page URL
   # @param strategy [Symbol] request strategy to use
@@ -75,18 +82,23 @@ module Html2rss
   # @param max_redirects [Integer, nil] optional redirect limit override
   # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
   # @param local_file_path [String, nil] optional local HTML file path
+  # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
   # @return [RSS::Rss] generated RSS feed
   def self.auto_source(url,
                        strategy: :auto,
                        items_selector: nil,
                        max_redirects: nil,
                        max_requests: 4,
-                       local_file_path: nil)
-    feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path:))
+                       local_file_path: nil,
+                       limit: nil)
+    feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
+                                  local_file_path:, limit:))
   end
 
   ##
-  # Scrapes the provided URL and returns a JSONFeed 1.1 hash.
+  # Scrapes the provided URL without hand-written selectors and returns a JSONFeed 1.1 hash.
+  #
+  # Same auto_source pipeline as {.auto_source}; see docs/auto_source.md.
   #
   # @param url [String] source page URL
   # @param strategy [Symbol] request strategy to use
@@ -94,15 +106,47 @@ module Html2rss
   # @param max_redirects [Integer, nil] optional redirect limit override
   # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
   # @param local_file_path [String, nil] optional local HTML file path
+  # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
   # @return [Hash] JSONFeed-compliant hash
   def self.auto_json_feed(url,
                           strategy: :auto,
                           items_selector: nil,
                           max_redirects: nil,
                           max_requests: 4,
-                          local_file_path: nil)
+                          local_file_path: nil,
+                          limit: nil)
     json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
-                                       local_file_path:))
+                                       local_file_path:, limit:))
+  end
+
+  ##
+  # Analyzes a URL and produces a reusable YAML-ready feed config hash.
+  #
+  # Uses auto-source discovery to extract articles, then derives CSS selectors
+  # from the structural analysis.
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+  # @param items_selector [String, nil] optional CSS selector hint for items
+  # @param max_redirects [Integer, nil] optional redirect limit override
+  # @param max_requests [Integer, nil] optional request budget override
+  # @param limit [Integer, nil] max articles to keep
+  # @param local_file_path [String, nil] optional local HTML file path
+  # @return [Hash] feed config hash with +:channel+ and +:selectors+
+  def self.capture(url,
+                   strategy: :auto,
+                   items_selector: nil,
+                   max_redirects: nil,
+                   max_requests: nil,
+                   limit: nil,
+                   local_file_path: nil)
+    Capture.build(url,
+                  strategy:,
+                  items_selector:,
+                  max_redirects:,
+                  max_requests:,
+                  limit:,
+                  local_file_path:).config
   end
 
   # rubocop:enable Metrics/ParameterLists
@@ -154,11 +198,14 @@ module Html2rss
   class << self
     private
 
-    def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path: nil) # rubocop:disable Metrics/ParameterLists
+    # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength
+    def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
+                                 local_file_path: nil, limit: nil)
       config = Config.auto_source_config(
         url:,
         items_selector:,
-        request_controls: shortcut_request_controls(strategy:, max_redirects:, max_requests:)
+        request_controls: Config::RequestControls.from_shortcut(strategy:, max_redirects:, max_requests:),
+        limit:
       )
       if local_file_path
         config[:request] ||= {}
@@ -166,23 +213,7 @@ module Html2rss
       end
       config
     end
-
-    def shortcut_request_controls(strategy:, max_redirects:, max_requests:)
-      Config::RequestControls.new(
-        strategy:,
-        max_redirects:,
-        max_requests:,
-        explicit_keys: explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
-      )
-    end
-
-    def explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
-      keys = []
-      keys << :strategy unless strategy.nil? || strategy == Config.default_strategy_name
-      keys << :max_redirects unless max_redirects.nil?
-      keys << :max_requests unless max_requests.nil?
-      keys
-    end
+    # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
   end
 
   logger.level = defaults.log_level if logger.respond_to?(:level=)