html2rss 0.24.0 → 0.25.0

data/lib/html2rss/sst/normalizer.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module Html2rss
+  module SST
+    ##
+    # Sole Nokogiri consumer on the heuristic auto-source path. Builds an
+    # immutable SST::Document with parent/depth/chrome indices.
+    class Normalizer # rubocop:disable Metrics/ClassLength
+      # Hard ceiling for SST node allocations; beyond this we degrade.
+      MAX_NODES = 5_000
+
+      # Tags stripped entirely from the SST.
+      STRIPPED_TAGS = Set['script', 'style', 'noscript', 'iframe', 'svg', 'template'].freeze
+
+      # Kept when MAX_NODES forces semantic-tag-only degrade.
+      SEMANTIC_DEGRADE_TAGS = Set['html', 'body', 'article', 'section', 'li', 'tr', 'div', 'a',
+                                  'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'time', 'img', 'p',
+                                  'ul', 'ol', 'main'].freeze
+
+      # Attribute names preserved in Attrs#raw for extraction.
+      RAW_ATTR_KEEP = /
+        \A(
+          data- |
+          aria- |
+          class |
+          id |
+          href |
+          src |
+          srcset |
+          style |
+          datetime |
+          itemprop |
+          type |
+          category |
+          categories |
+          tag |
+          tags |
+          topic |
+          topics |
+          section |
+          sections |
+          label |
+          labels |
+          theme |
+          themes |
+          subject |
+          subjects
+        )
+      /xi
+
+      class << self
+        ##
+        # @param input [String, Nokogiri::HTML::Document, Nokogiri::XML::Node]
+        #   HTML string or already-parsed node (String is parsed once here)
+        # @return [Document]
+        # @raise [ArgumentError] when input is nil or unsupported
+        def call(input)
+          raise ArgumentError, 'input is required' if input.nil?
+
+          new(coerce(input)).call
+        end
+
+        private
+
+        # @param input [String, Nokogiri::XML::Node]
+        # @return [Nokogiri::XML::Node]
+        def coerce(input)
+          case input
+          when String then Nokogiri::HTML(input)
+          when Nokogiri::XML::Node then input
+          else
+            raise ArgumentError, "expected String or Nokogiri::XML::Node, got #{input.class}"
+          end
+        end
+      end
+
+      # @param parsed_body [Nokogiri::HTML::Document, Nokogiri::XML::Node]
+      def initialize(parsed_body)
+        @parsed_body = parsed_body
+        @node_count = 0
+        @degraded = false
+        @parents = {}.compare_by_identity
+        @depths = {}.compare_by_identity
+        @ignored_chrome = {}.compare_by_identity
+      end
+
+      ##
+      # @return [Document]
+      def call
+        root_nk = @parsed_body.respond_to?(:root) ? (@parsed_body.at_css('html') || @parsed_body.root) : @parsed_body
+        root = normalize_element(root_nk, parent: nil, depth: 0, path: '', chrome: false)
+        raise ArgumentError, 'SST Normalizer produced an empty tree' unless root
+
+        index = Index.new(root:, parents: @parents, depths: @depths, ignored_chrome: @ignored_chrome)
+        Document.build(root:, index:, degraded: @degraded, node_count: @node_count)
+      end
+
+      private
+
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+      def normalize_element(nk_node, parent:, depth:, path:, chrome:)
+        return unless nk_node.respond_to?(:name)
+        return if nk_node.text? || nk_node.comment? || nk_node.cdata?
+
+        tag = nk_node.name.to_s.downcase
+        return if STRIPPED_TAGS.include?(tag)
+        return if @degraded && !SEMANTIC_DEGRADE_TAGS.include?(tag)
+
+        if @node_count >= MAX_NODES && !@degraded
+          @degraded = true
+          Html2rss::Log.warn("sst.normalizer MAX_NODES=#{MAX_NODES} breached; degrading to semantic-tag-only")
+        end
+
+        return if @degraded && !SEMANTIC_DEGRADE_TAGS.include?(tag)
+
+        @node_count += 1
+        tag_path = path.empty? ? "/#{tag}" : "#{path}/#{tag}"
+        attrs = extract_attrs(nk_node)
+        own_text = direct_text(nk_node)
+        chrome_here = chrome || Tags::IGNORED_CONTAINER_NAMES.include?(tag.to_sym)
+
+        children = nk_node.children.filter_map do |child|
+          normalize_element(child, parent: :pending, depth: depth + 1, path: tag_path, chrome: chrome_here)
+        end.freeze
+
+        node = Node.build(name: tag, attrs:, own_text:, children:, tag_path:)
+        @parents[node] = parent == :pending ? nil : parent
+        @depths[node] = depth
+        @ignored_chrome[node] = chrome_here
+
+        # Fix parent pointers now that node exists.
+        children.each { |child| @parents[child] = node }
+
+        node
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+      def extract_attrs(nk_node) # rubocop:disable Metrics/MethodLength
+        Attrs.build(
+          href: nk_node['href'],
+          src: nk_node['src'],
+          id: nk_node['id'],
+          class_names: nk_node['class'].to_s.split(/\s+/).reject(&:empty?),
+          datetime: nk_node['datetime'],
+          itemprop: nk_node['itemprop'],
+          style: nk_node['style'],
+          srcset: nk_node['srcset'],
+          type: nk_node['type'],
+          raw: raw_attrs(nk_node)
+        )
+      end
+
+      def raw_attrs(nk_node)
+        typed = %w[href src id class datetime itemprop style srcset type].to_set
+        nk_node.attribute_nodes.each_with_object({}) do |attr, raw|
+          name = attr.name.to_s
+          next if typed.include?(name)
+          next unless name.match?(RAW_ATTR_KEEP)
+
+          raw[name] = attr.value.to_s
+        end
+      end
+
+      def direct_text(nk_node)
+        nk_node.children.select(&:text?).map(&:text).join
+      end
+    end
+  end
+end

data/lib/html2rss/sst/tags.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Shared tag-name and href constants for SST nodes.
+    module Tags
+      # Heading element names.
+      HEADING_NAMES = %i[h1 h2 h3 h4 h5 h6].to_set.freeze
+      # Block-level names that contribute structural newlines in text.
+      BLOCK_NAMES = %i[p div li ul ol h1 h2 h3 h4 h5 h6 tr br].to_set.freeze
+      # Names skipped when collecting visible text.
+      INVISIBLE_NAMES = %i[svg script noscript style template].to_set.freeze
+      # Landmark names treated as utility chrome.
+      UTILITY_LANDMARK_NAMES = %i[nav aside footer menu].to_set.freeze
+      # Container names ignored as article roots.
+      IGNORED_CONTAINER_NAMES = %i[nav footer header svg script style].to_set.freeze
+      # Names excluded from class/structure clustering.
+      CLUSTER_EXCLUDED_NAMES = %i[html body nav footer header svg script style].to_set.freeze
+      # Href prefixes that never count as content destinations.
+      SKIP_HREF_PREFIXES = ['#', 'javascript:', 'mailto:', 'tel:', 'file://', 'sms:', 'data:'].freeze
+      # Layout-ish names used when resolving nested wrapper groups.
+      LAYOUT_NAMES = %i[div section article li ul ol].to_set.freeze
+    end
+  end
+end

data/lib/html2rss/sst/text.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Visible-text extraction for SST nodes (port of Navigator::TextExtractor).
+    module Text
+      module_function
+
+      ##
+      # @param node [Node]
+      # @param separator [String]
+      # @param exclude [Array<Node>, nil]
+      # @return [String, nil]
+      def extract(node, separator: ' ', exclude: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+        raise ArgumentError, 'node is required' unless node
+
+        excluded = exclude&.each_with_object({}.compare_by_identity) { |n, h| h[n] = true }
+        if node.children.empty?
+          text = node.own_text.to_s.gsub(/\s+/, ' ').strip
+          return text.empty? ? nil : text
+        end
+
+        parts = collect_parts(node, separator, excluded)
+        own = node.own_text.to_s.gsub(/\s+/, ' ').strip
+        parts = [own, *parts] unless own.empty?
+        return if parts.empty?
+
+        parts.join.squeeze(' ').strip
+      end
+
+      def collect_parts(node, separator, excluded)
+        last_block = false
+        node.children.each_with_object([]) do |child, parts|
+          next if excluded&.[](child)
+          next unless visible_child?(child)
+
+          text, block = child_text_and_block(child, separator, excluded)
+          next if text.empty?
+
+          append_separator!(parts, separator, block, last_block)
+          parts << text
+          last_block = block
+        end
+      end
+      module_function :collect_parts
+      private_class_method :collect_parts
+
+      def child_text_and_block(child, separator, excluded)
+        text = if child.children.empty?
+                 child.own_text.gsub(/\s+/, ' ').strip
+               else
+                 extract(child, separator:, exclude: excluded&.keys).to_s.strip
+               end
+        text = "- #{text}" if child.name == :li && !text.empty?
+        [text, Tags::BLOCK_NAMES.include?(child.name)]
+      end
+      module_function :child_text_and_block
+      private_class_method :child_text_and_block
+
+      def append_separator!(parts, separator, is_block, last_was_block)
+        return if parts.empty?
+
+        parts << if is_block || last_was_block
+                   (separator == ' ' ? "\n" : separator)
+                 else
+                   ' '
+                 end
+      end
+      module_function :append_separator!
+      private_class_method :append_separator!
+
+      def visible_child?(node)
+        !Tags::INVISIBLE_NAMES.include?(node.name) &&
+          !(node.name == :a && node.attrs.href&.start_with?('#'))
+      end
+      module_function :visible_child?
+      private_class_method :visible_child?
+    end
+  end
+end

data/lib/html2rss/sst.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Simplified Semantic Tree namespace for the heuristic auto-source pipeline.
+  module SST
+  end
+end

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.24.0'
+  VERSION = '0.25.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -3,7 +3,10 @@
 require 'zeitwerk'
 
 loader = Zeitwerk::Loader.for_gem
-loader.inflector.inflect('cli' => 'CLI')
+loader.inflector.inflect(
+  'cli' => 'CLI',
+  'sst' => 'SST'
+)
 loader.setup
 
 require 'logger'
@@ -67,7 +70,10 @@ module Html2rss
   # rubocop:disable Metrics/ParameterLists
 
   ##
-  # Scrapes the provided URL and returns an RSS object.
+  # Scrapes the provided URL without hand-written selectors and returns an RSS object.
+  #
+  # Builds an auto_source config, then FeedPipeline runs structured scrapers and
+  # (when needed) the SST heuristic path — see docs/auto_source.md.
   #
   # @param url [String] source page URL
   # @param strategy [Symbol] request strategy to use
@@ -75,18 +81,23 @@ module Html2rss
   # @param max_redirects [Integer, nil] optional redirect limit override
   # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
   # @param local_file_path [String, nil] optional local HTML file path
+  # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
   # @return [RSS::Rss] generated RSS feed
   def self.auto_source(url,
                        strategy: :auto,
                        items_selector: nil,
                        max_redirects: nil,
                        max_requests: 4,
-                       local_file_path: nil)
-    feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path:))
+                       local_file_path: nil,
+                       limit: nil)
+    feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
+                                  local_file_path:, limit:))
   end
 
   ##
-  # Scrapes the provided URL and returns a JSONFeed 1.1 hash.
+  # Scrapes the provided URL without hand-written selectors and returns a JSONFeed 1.1 hash.
+  #
+  # Same auto_source pipeline as {.auto_source}; see docs/auto_source.md.
   #
   # @param url [String] source page URL
   # @param strategy [Symbol] request strategy to use
@@ -94,15 +105,17 @@ module Html2rss
   # @param max_redirects [Integer, nil] optional redirect limit override
   # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
   # @param local_file_path [String, nil] optional local HTML file path
+  # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
   # @return [Hash] JSONFeed-compliant hash
   def self.auto_json_feed(url,
                           strategy: :auto,
                           items_selector: nil,
                           max_redirects: nil,
                           max_requests: 4,
-                          local_file_path: nil)
+                          local_file_path: nil,
+                          limit: nil)
     json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
-                                       local_file_path:))
+                                       local_file_path:, limit:))
   end
 
   # rubocop:enable Metrics/ParameterLists
@@ -154,11 +167,14 @@ module Html2rss
   class << self
     private
 
-    def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path: nil) # rubocop:disable Metrics/ParameterLists
+    # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength
+    def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
+                                 local_file_path: nil, limit: nil)
       config = Config.auto_source_config(
         url:,
         items_selector:,
-        request_controls: shortcut_request_controls(strategy:, max_redirects:, max_requests:)
+        request_controls: Config::RequestControls.from_shortcut(strategy:, max_redirects:, max_requests:),
+        limit:
       )
       if local_file_path
         config[:request] ||= {}
@@ -166,23 +182,7 @@ module Html2rss
       end
       config
     end
-
-    def shortcut_request_controls(strategy:, max_redirects:, max_requests:)
-      Config::RequestControls.new(
-        strategy:,
-        max_redirects:,
-        max_requests:,
-        explicit_keys: explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
-      )
-    end
-
-    def explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
-      keys = []
-      keys << :strategy unless strategy.nil? || strategy == Config.default_strategy_name
-      keys << :max_redirects unless max_redirects.nil?
-      keys << :max_requests unless max_requests.nil?
-      keys
-    end
+    # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
   end
 
   logger.level = defaults.log_level if logger.respond_to?(:level=)

data/schema/html2rss-config.schema.json

@@ -45,6 +45,18 @@
             "null",
             "string"
           ]
+        },
+        "author": {
+          "type": [
+            "null",
+            "string"
+          ]
+        },
+        "image": {
+          "type": [
+            "null",
+            "string"
+          ]
         }
       },
       "required": [
@@ -94,6 +106,13 @@
       "$schema": "http://json-schema.org/draft-06/schema#",
       "type": "object",
       "properties": {
+        "limit": {
+          "type": "integer",
+          "not": {
+            "type": "null"
+          },
+          "exclusiveMinimum": 0
+        },
         "scraper": {
           "type": "object",
           "properties": {
@@ -269,6 +288,7 @@
       },
       "required": [],
       "default": {
+        "limit": 25,
         "scraper": {
           "wordpress_api": {
             "enabled": true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.24.0
+  version: 0.25.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -278,25 +278,13 @@ files:
 - lib/html2rss/article/enclosure.rb
 - lib/html2rss/auto_source.rb
 - lib/html2rss/auto_source/cleanup.rb
-- lib/html2rss/auto_source/discovery.rb
-- lib/html2rss/auto_source/discovery/dom_clustering.rb
-- lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb
-- lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb
-- lib/html2rss/auto_source/discovery/list_candidates.rb
-- lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb
-- lib/html2rss/auto_source/discovery/semantic_containers.rb
-- lib/html2rss/auto_source/discovery/sitemap.rb
-- lib/html2rss/auto_source/link_heuristics.rb
-- lib/html2rss/auto_source/link_heuristics/anchor_signals.rb
-- lib/html2rss/auto_source/link_heuristics/container_assessor.rb
-- lib/html2rss/auto_source/link_heuristics/container_signals.rb
-- lib/html2rss/auto_source/link_heuristics/destination_facts.rb
-- lib/html2rss/auto_source/link_heuristics/href_extractor.rb
-- lib/html2rss/auto_source/link_heuristics/path_classifier.rb
-- lib/html2rss/auto_source/link_heuristics/text_classifier.rb
 - lib/html2rss/auto_source/scraper.rb
 - lib/html2rss/auto_source/scraper/html.rb
 - lib/html2rss/auto_source/scraper/json_state.rb
+- lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb
+- lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb
+- lib/html2rss/auto_source/scraper/json_state/document_scanner.rb
+- lib/html2rss/auto_source/scraper/json_state/value_finder.rb
 - lib/html2rss/auto_source/scraper/meta_oembed.rb
 - lib/html2rss/auto_source/scraper/microdata.rb
 - lib/html2rss/auto_source/scraper/microformats2.rb
@@ -307,10 +295,17 @@ files:
 - lib/html2rss/auto_source/scraper/semantic_html.rb
 - lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb
 - lib/html2rss/auto_source/scraper/sitemap.rb
+- lib/html2rss/auto_source/scraper/sitemap/parser.rb
 - lib/html2rss/auto_source/scraper/wordpress_api.rb
 - lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb
 - lib/html2rss/auto_source/scraper/wordpress_api/page_scope/date_archive_range.rb
 - lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb
+- lib/html2rss/auto_source/segment.rb
+- lib/html2rss/auto_source/segmenter.rb
+- lib/html2rss/auto_source/segmenter/cluster.rb
+- lib/html2rss/auto_source/segmenter/list.rb
+- lib/html2rss/auto_source/segmenter/primary_link.rb
+- lib/html2rss/auto_source/segmenter/semantic.rb
 - lib/html2rss/channel.rb
 - lib/html2rss/cli.rb
 - lib/html2rss/config.rb
@@ -344,15 +339,27 @@ files:
 - lib/html2rss/html/article_extractor/heading_extractor.rb
 - lib/html2rss/html/article_extractor/id_generator.rb
 - lib/html2rss/html/article_extractor/image_extractor.rb
+- lib/html2rss/html/article_rules.rb
+- lib/html2rss/html/article_rules/category.rb
+- lib/html2rss/html/article_rules/date.rb
+- lib/html2rss/html/article_rules/enclosure.rb
+- lib/html2rss/html/article_rules/image.rb
 - lib/html2rss/html/navigator.rb
 - lib/html2rss/html/navigator/text_extractor.rb
 - lib/html2rss/html/rendering.rb
 - lib/html2rss/html/rendering/audio_renderer.rb
 - lib/html2rss/html/rendering/description_builder.rb
+- lib/html2rss/html/rendering/escaped_attributes.rb
 - lib/html2rss/html/rendering/image_renderer.rb
 - lib/html2rss/html/rendering/media_renderer.rb
 - lib/html2rss/html/rendering/pdf_renderer.rb
 - lib/html2rss/html/rendering/video_renderer.rb
+- lib/html2rss/html/sst_article_extractor.rb
+- lib/html2rss/link_destination.rb
+- lib/html2rss/link_destination/destination_facts.rb
+- lib/html2rss/link_destination/noise_policy.rb
+- lib/html2rss/link_destination/path_classifier.rb
+- lib/html2rss/link_destination/text_classifier.rb
 - lib/html2rss/request_service.rb
 - lib/html2rss/request_service/blocked_surface.rb
 - lib/html2rss/request_service/botasaurus_contract.rb
@@ -378,6 +385,15 @@ files:
 - lib/html2rss/request_session/pager/offset.rb
 - lib/html2rss/request_session/pager/rel_next.rb
 - lib/html2rss/request_session/pager/url_template.rb
+- lib/html2rss/scoring.rb
+- lib/html2rss/scoring/anchor_score.rb
+- lib/html2rss/scoring/cluster_scorer.rb
+- lib/html2rss/scoring/container_assessor.rb
+- lib/html2rss/scoring/engine.rb
+- lib/html2rss/scoring/link_resolver.rb
+- lib/html2rss/scoring/observation.rb
+- lib/html2rss/scoring/ranked_segment.rb
+- lib/html2rss/scoring/score.rb
 - lib/html2rss/selectors.rb
 - lib/html2rss/selectors/extractors.rb
 - lib/html2rss/selectors/extractors/attribute.rb
@@ -399,6 +415,14 @@ files:
 - lib/html2rss/selectors/post_processors/sanitize_html.rb
 - lib/html2rss/selectors/post_processors/substring.rb
 - lib/html2rss/selectors/post_processors/template.rb
+- lib/html2rss/sst.rb
+- lib/html2rss/sst/attrs.rb
+- lib/html2rss/sst/document.rb
+- lib/html2rss/sst/index.rb
+- lib/html2rss/sst/node.rb
+- lib/html2rss/sst/normalizer.rb
+- lib/html2rss/sst/tags.rb
+- lib/html2rss/sst/text.rb
 - lib/html2rss/status.rb
 - lib/html2rss/url.rb
 - lib/html2rss/version.rb
@@ -409,7 +433,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.24.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.25.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:

data/lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    module Discovery
-      class DomClustering
-        ##
-        # Scores candidate DOM groups using heading, time, date, and word-count signals.
-        class GroupScorer
-          def initialize
-            @text_words = {}.compare_by_identity
-            @has_date = {}.compare_by_identity
-          end
-
-          # @param groups [Hash{String => Array<Nokogiri::XML::Node>}] candidate DOM groups
-          # @return [Array<Nokogiri::XML::Node>] nodes from the highest-scoring group
-          def select_best_group(groups)
-            best_nodes = []
-            best_score = -1
-
-            groups.each_value do |nodes|
-              score = score_group(nodes)
-              next if score.negative?
-
-              (best_nodes = nodes) && (best_score = score) if score > best_score
-            end
-
-            best_nodes
-          end
-
-          # @param nodes [Array<Nokogiri::XML::Node>]
-          # @return [Float] average visible word count across nodes
-          def avg_words(nodes)
-            nodes.sum { |n| text_words(n) } / nodes.size.to_f
-          end
-
-          private
-
-          def score_group(nodes)
-            avg_w = avg_words(nodes)
-            return -1 if avg_w < 5
-
-            score = nodes.size + (avg_w / 5.0)
-            score += 20 if nodes_heading?(nodes)
-            score += 20 if nodes_time?(nodes)
-            score += 40 if nodes_date?(nodes)
-            score
-          end
-
-          def nodes_heading?(nodes)
-            nodes.any? do |n|
-              n.at_css(Html2rss::Html::Navigator::HEADING_TAGS.join(',')) ||
-                n.at_css('.font-bold, .font-semibold')
-            end
-          end
-
-          def nodes_time?(nodes)
-            nodes.any? { |n| n.at_css('time, [datetime]') }
-          end
-
-          def nodes_date?(nodes)
-            nodes.any? { |n| date?(n) }
-          end
-
-          def text_words(node)
-            @text_words[node] ||= Html2rss::Html::Navigator.extract_visible_text(node).to_s.scan(/\p{Alnum}+/).size
-          end
-
-          def date?(node)
-            @has_date[node] ||= begin
-              text = Html2rss::Html::Navigator.extract_visible_text(node).to_s
-              text.match?(%r{\b\d{4}[-/]\d{2}[-/]\d{2}\b}) ||
-                text.match?(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i)
-            end
-          end
-        end
-      end
-    end
-  end
-end