html2rss 0.20.1 → 0.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3eb20836fb55a5e33d114634c7c20bf2a65afd1a923af1bba82896797fdb099d
-  data.tar.gz: 97c00923e0ca5744cf82f22d54aa428cc784cba3e259bbcdcd9252928f9bc3c0
+  metadata.gz: 9ff7cdc4e25f3abc6da000e4f672d6832fb6027e49885aecc0cad38329c5e6ae
+  data.tar.gz: e42c216e328bb2c56971dd58871f023f15e672563e27741c56c6cf7fe4cb322a
 SHA512:
-  metadata.gz: 36431edafddcca32a53f562a75fbcd77fae969ca8c8fa7c2b6de77f121add2ee3c34c2b6e2e7b57109742780d62eeddde823b1a9bf8f4c1aaeed08a0ae4e5c90
-  data.tar.gz: d3eafa9cbbecc5ccaded7b21508e0af1c43337999cb5651f2d879df5217c1aa1f3e5484bcfd6354037d709715db1cc106f2946dd592745c0ac58252bcdd26ac8
+  metadata.gz: e52812f947561b9a52537f1b28c530f63e116194642d13ff526fac1ad32f02d7ea6ff8ca9b5ee16e2d7f686e1babec1908ca51399ba42ef9461ed3dbe0d02117
+  data.tar.gz: 4754495a5947aca6de71846d1c88128d9fc1826e1014fd00806f58e7cd1e1575dc3bd2380ced3aeecd0cd6d51e0366ddee7a1f7db0220750e13f66645de2edde

data/html2rss.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.description   = 'Supports JSON content, custom HTTP headers, and post-processing of extracted content.'
   spec.homepage      = 'https://github.com/html2rss/html2rss'
   spec.license       = 'MIT'
-  spec.required_ruby_version = '>= 3.2'
+  spec.required_ruby_version = '>= 3.3'
 
   if spec.respond_to?(:metadata)
     spec.metadata['allowed_push_host'] = 'https://rubygems.org'
@@ -41,7 +41,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'kramdown'
   spec.add_dependency 'mime-types', '> 3.0'
   spec.add_dependency 'nokogiri', '>= 1.10', '< 2.0'
-  spec.add_dependency 'parallel'
   spec.add_dependency 'puppeteer-ruby'
   spec.add_dependency 'regexp_parser'
   spec.add_dependency 'reverse_markdown', '~> 3.0'

data/lib/html2rss/auto_source/scraper/html/class_clustering.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class Html
+        ##
+        # ClassClustering clusters DOM elements on anchorless pages by class lists and scores
+        # candidate groups to find the best list of content cards/articles.
+        # rubocop:disable Metrics/ClassLength
+        class ClassClustering
+          # Node tags considered layout containers
+          LAYOUT_TAG_NAMES = Set['div', 'section', 'article'].freeze
+          # HTML/layout tags excluded from candidate nodes
+          EXCLUDED_TAGS = Set['html', 'body', 'nav', 'footer', 'header', 'svg', 'script', 'style'].freeze
+
+          class << self
+            ##
+            # Clusters elements in parsed_body and returns the best set of content card nodes.
+            #
+            # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+            # @param minimum_selector_frequency [Integer] minimum frequency for class groups
+            # @return [Array<Nokogiri::XML::Node>] candidate nodes of the top-scoring class group
+            def call(parsed_body, minimum_selector_frequency:)
+              new(parsed_body, minimum_selector_frequency:).call
+            end
+          end
+
+          # @param parsed_body [Nokogiri::HTML::Document]
+          # @param minimum_selector_frequency [Integer]
+          def initialize(parsed_body, minimum_selector_frequency:)
+            @parsed_body = parsed_body
+            @minimum_frequency = minimum_selector_frequency
+            @text_words = {}.compare_by_identity
+            @has_date = {}.compare_by_identity
+          end
+
+          # @return [Array<Nokogiri::XML::Node>]
+          def call
+            candidate_groups = collect_candidate_groups
+            return [] if candidate_groups.empty?
+
+            non_containers = filter_containers(candidate_groups)
+            final_groups = filter_1_to_1_overlap(non_containers)
+
+            select_best_group(final_groups)
+          end
+
+          private
+
+          def collect_candidate_groups
+            class_groups = Hash.new { |h, k| h[k] = [] }
+            cache = {}.compare_by_identity
+
+            @parsed_body.css('[class]').each { |node| add_node_to_groups(node, class_groups, cache) }
+
+            class_groups.select { |_, nodes| nodes.size >= @minimum_frequency }
+          end
+
+          def add_node_to_groups(node, class_groups, cache)
+            return if EXCLUDED_TAGS.include?(node.name) || HtmlExtractor.ignored_container_path?(node, cache)
+
+            cls = normalize_class(node['class'])
+            class_groups[cls] << node unless cls.empty?
+          end
+
+          def normalize_class(class_attr)
+            class_str = class_attr.to_s.strip
+            return '' if class_str.empty?
+
+            # Bypass split/sort/join allocation for single-class lists
+            if class_str.include?(' ')
+              class_str.split(/\s+/).sort.join(' ')
+            else
+              class_str
+            end
+          end
+
+          # Discard group A if any node of A contains > 1 node of another group B
+          def filter_containers(groups)
+            groups.reject do |cls_a, nodes_a|
+              groups.any? { |cls_b, nodes_b| cls_a != cls_b && container_of?(nodes_a, nodes_b) }
+            end
+          end
+
+          def container_of?(nodes_a, nodes_b)
+            return false unless LAYOUT_TAG_NAMES.include?(nodes_b.first.name)
+
+            nodes_a.any? do |node_a|
+              nodes_b.count { |node_b| node_a != node_b && node_b.ancestors.include?(node_a) } > 1
+            end
+          end
+
+          # If group A contains group B, and they have the same size:
+          # - If B (the descendant) contains >= 80% of A's words, AND B's tag is div/section/article,
+          #   B is the actual content card. Discard A.
+          # - Otherwise, B is a sub-element (header, metadata line, button). Discard B.
+          def filter_1_to_1_overlap(groups)
+            discarded = Set.new
+            groups.each_key do |cls_a|
+              groups.each_key do |cls_b|
+                next if cls_a == cls_b || discarded.include?(cls_a) || discarded.include?(cls_b)
+
+                resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
+              end
+            end
+
+            groups.except(*discarded)
+          end
+
+          def resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
+            nodes_a = groups[cls_a]
+            nodes_b = groups[cls_b]
+            return if nodes_a.size != nodes_b.size
+            return unless nodes_a.zip(nodes_b).all? { |a, b| a != b && b.ancestors.include?(a) }
+
+            discarded << (keep_descendant?(nodes_a, nodes_b) ? cls_a : cls_b)
+          end
+
+          def keep_descendant?(nodes_a, nodes_b)
+            avg_words(nodes_b) >= 0.8 * avg_words(nodes_a) &&
+              LAYOUT_TAG_NAMES.include?(nodes_b.first.name)
+          end
+
+          def select_best_group(groups)
+            best_nodes = []
+            best_score = -1
+
+            groups.each_value do |nodes|
+              score = score_group(nodes)
+              next if score.negative?
+
+              (best_nodes = nodes) && (best_score = score) if score > best_score
+            end
+
+            best_nodes
+          end
+
+          def score_group(nodes)
+            avg_w = avg_words(nodes)
+            return -1 if avg_w < 5
+
+            score = nodes.size + (avg_w / 5.0)
+            score += 20 if nodes_heading?(nodes)
+            score += 20 if nodes_time?(nodes)
+            score += 40 if nodes_date?(nodes)
+            score
+          end
+
+          def nodes_heading?(nodes)
+            nodes.any? do |n|
+              n.at_css(HtmlExtractor::HEADING_TAGS.join(',')) ||
+                n.at_css('.font-bold, .font-semibold')
+            end
+          end
+
+          def nodes_time?(nodes)
+            nodes.any? { |n| n.at_css('time, [datetime]') }
+          end
+
+          def nodes_date?(nodes)
+            nodes.any? { |n| date?(n) }
+          end
+
+          def avg_words(nodes)
+            nodes.sum { |n| text_words(n) } / nodes.size.to_f
+          end
+
+          def text_words(node)
+            @text_words[node] ||= HtmlExtractor.extract_visible_text(node).to_s.scan(/\p{Alnum}+/).size
+          end
+
+          def date?(node)
+            @has_date[node] ||= begin
+              text = HtmlExtractor.extract_visible_text(node).to_s
+              text.match?(%r{\b\d{4}[-/]\d{2}[-/]\d{2}\b}) ||
+                text.match?(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i)
+            end
+          end
+          # rubocop:enable Metrics/ClassLength
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/html.rb

@@ -62,7 +62,9 @@ module Html2rss
           @url = url
           @extractor = extractor
           @opts = opts
+          @fallback_anchorless = opts.fetch(:fallback_anchorless, false)
           @link_heuristics = LinkHeuristics.new(url)
+          @ignored_cache = {}.compare_by_identity
         end
 
         attr_reader :parsed_body
@@ -73,10 +75,13 @@ module Html2rss
         def each
           return enum_for(:each) unless block_given?
 
-          each_article_tag do |article_tag, selected_anchor|
-            article_hash = extract_article(article_tag, selected_anchor:)
-            yield article_hash if article_hash
-          end
+          articles.each { yield _1 }
+        end
+
+        ##
+        # @return [Boolean] true when the scraper can likely extract articles
+        def extractable?
+          articles.any?
         end
 
         ##
@@ -91,7 +96,7 @@ module Html2rss
         # @return [Boolean] true when the node is a good extraction boundary
         def article_tag_condition?(node)
           # Ignore tags that are below ignored DOM chrome.
-          return false if HtmlExtractor.ignored_container_path?(node)
+          return false if HtmlExtractor.ignored_container_path?(node, @ignored_cache)
           return true if %w[body html].include?(node.name)
           return false unless (parent = node.parent)
 
@@ -100,14 +105,41 @@ module Html2rss
 
         private
 
+        def articles
+          @articles ||= begin
+            extracted = each_article_tag.filter_map do |article_tag, selected_anchor|
+              extract_article(article_tag, selected_anchor:)
+            end
+
+            extracted += find_anchorless_articles if @fallback_anchorless
+            extracted
+          end
+        end
+
+        def find_anchorless_articles
+          ClassClustering.call(parsed_body, minimum_selector_frequency:).map do |node|
+            @extractor.new(node, base_url: @url, selected_anchor: nil, fallback_anchorless: true).call
+          end
+        end
+
+        ##
+        # @return [Integer]
         def minimum_selector_frequency = @opts[:minimum_selector_frequency] || DEFAULT_MINIMUM_SELECTOR_FREQUENCY
+
+        ##
+        # @return [Boolean]
         def use_top_selectors = @opts[:use_top_selectors] || DEFAULT_USE_TOP_SELECTORS
 
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @return [Integer]
         def anchor_count(node)
-          @anchor_counts ||= {}
-          @anchor_counts[node.path] ||= node.name == 'a' ? 1 : node.css('a').size
+          (@anchor_counts ||= {}.compare_by_identity)[node] ||= node.name == 'a' ? 1 : node.css('a').size
         end
 
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @return [Boolean]
         def relevant_anchor?(node)
           destination_facts = @link_heuristics.destination_facts(node)
           return false unless destination_facts
@@ -115,14 +147,24 @@ module Html2rss
           !noise_anchor?(node, destination_facts)
         end
 
+        ##
+        # @yield [article_tag, selected_anchor]
+        # @yieldparam article_tag [Nokogiri::XML::Node]
+        # @yieldparam selected_anchor [Nokogiri::XML::Node]
+        # @return [Enumerator, nil]
         def each_article_tag(&block)
           return enum_for(:each_article_tag) unless block
 
-          list_candidates.each_article_tag(anchor_filter: method(:relevant_anchor?),
-                                           boundary_condition: method(:article_tag_condition?),
-                                           &block)
+          anchor_filter = ->(node) { relevant_anchor?(node) }
+          boundary_condition = ->(node) { article_tag_condition?(node) }
+
+          list_candidates.each_article_tag(anchor_filter:, boundary_condition:, &block)
         end
 
+        ##
+        # @param article_tag [Nokogiri::XML::Node]
+        # @param selected_anchor [Nokogiri::XML::Node, nil]
+        # @return [Hash, nil]
         def extract_article(article_tag, selected_anchor: nil)
           selected_anchor ||= preferred_anchor_for(article_tag)
           return unless selected_anchor
@@ -131,18 +173,28 @@ module Html2rss
           @extractor.new(article_tag, base_url: @url, selected_anchor:).call
         end
 
+        ##
+        # @param anchor [Nokogiri::XML::Node]
+        # @param destination_facts [DestinationFacts]
+        # @return [Boolean]
         def noise_anchor?(anchor, destination_facts) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
           return true unless destination_facts
 
-          text = HtmlExtractor.extract_visible_text(anchor).to_s.strip
+          (@noise_anchors ||= {}.compare_by_identity)[anchor] ||= begin
+            text = HtmlExtractor.extract_visible_text(anchor).to_s.strip
 
-          destination_facts.taxonomy_path ||
-            short_utility_label?(text, destination_facts) ||
-            (@link_heuristics.recommended_text?(text) && destination_facts.shallow) ||
-            (@link_heuristics.utility_prefix_text?(text) && destination_facts.high_confidence_utility_destination) ||
-            (@link_heuristics.utility_text?(text) && destination_facts.vanity_path)
+            destination_facts.taxonomy_path ||
+              short_utility_label?(text, destination_facts) ||
+              (@link_heuristics.recommended_text?(text) && destination_facts.shallow) ||
+              (@link_heuristics.utility_prefix_text?(text) && destination_facts.high_confidence_utility_destination) ||
+              (@link_heuristics.utility_text?(text) && destination_facts.vanity_path)
+          end
         end
 
+        ##
+        # @param text [String]
+        # @param destination_facts [DestinationFacts]
+        # @return [Boolean]
         def short_utility_label?(text, destination_facts)
           destination_facts.utility_path &&
             !destination_facts.content_path &&
@@ -150,11 +202,16 @@ module Html2rss
             text.scan(/\p{Alnum}+/).size <= 3
         end
 
+        ##
+        # @param article_tag [Nokogiri::XML::Node]
+        # @return [Nokogiri::XML::Node, nil]
         def preferred_anchor_for(article_tag)
           article_tag.css(HtmlExtractor::MAIN_ANCHOR_SELECTOR).find { relevant_anchor?(_1) } ||
             HtmlExtractor.main_anchor_for(article_tag)
         end
 
+        ##
+        # @return [HtmlExtractor::ListCandidates]
         def list_candidates
           HtmlExtractor::ListCandidates.new(
             parsed_body,

data/lib/html2rss/auto_source/scraper/json_state.rb

@@ -30,6 +30,9 @@ module Html2rss
           /(?:window|self|globalThis)\.angular\s*=\s*/m
         ].freeze
 
+        # Combined regex for faster matching of global assignments.
+        GLOBAL_ASSIGNMENT_REGEXP = Regexp.union(GLOBAL_ASSIGNMENT_PATTERNS).freeze
+
         # Preferred keys when extracting title-like values from state payloads.
         TITLE_KEYS = %i[title headline name text].freeze
         # Preferred keys when extracting URL-like values from state payloads.
@@ -53,7 +56,12 @@ module Html2rss
           # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
           # @return [Array<Hash, Array>] parsed JSON documents discovered in scripts
           def json_documents(parsed_body)
-            script_documents(parsed_body) + assignment_documents(parsed_body)
+            # Use identity-based cache to avoid double-parsing of the same document.
+            # WeakMap allows the Nokogiri Document (key) to be garbage collected.
+            # rubocop:disable ThreadSafety/ClassInstanceVariable
+            (@cache ||= ObjectSpace::WeakMap.new)[parsed_body] ||=
+              script_documents(parsed_body) + assignment_documents(parsed_body)
+            # rubocop:enable ThreadSafety/ClassInstanceVariable
           end
 
           # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
@@ -80,15 +88,10 @@ module Html2rss
           def assignment_payload(text)
             trimmed = text.to_s.strip
             return if trimmed.empty?
+            return unless trimmed.match?(GLOBAL_ASSIGNMENT_REGEXP)
 
-            GLOBAL_ASSIGNMENT_PATTERNS.each do |pattern|
-              next unless trimmed.match?(pattern)
-
-              payload = trimmed.sub(pattern, '')
-              return extract_assignment_payload(payload)
-            end
-
-            nil
+            payload = trimmed.sub(GLOBAL_ASSIGNMENT_REGEXP, '')
+            extract_assignment_payload(payload)
           end
 
           # @param text [String] text potentially containing JSON-like payloads
@@ -116,8 +119,10 @@ module Html2rss
             in_string = false
             escape = false
 
-            text.each_char.with_index do |char, index|
-              next if index < start_index
+            i = start_index
+            len = text.length
+            while i < len
+              char = text[i]
 
               if in_string
                 if escape
@@ -127,24 +132,22 @@ module Html2rss
                 elsif char == '"'
                   in_string = false
                 end
-                next
-              end
-
-              case char
-              when '"'
-                in_string = true
-              when '{'
-                stack << '}'
-              when '['
-                stack << ']'
-              when '}', ']'
-                expected = stack.pop
-                return index if expected == char && stack.empty?
+              else
+                case char
+                when '"' then in_string = true
+                when '{' then stack << '}'
+                when '[' then stack << ']'
+                when '}', ']'
+                  expected = stack.pop
+                  return i if expected == char && stack.empty?
+                end
               end
+              i += 1
             end
 
             nil
           end
+
           # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
           # @param payload [String, nil] JSON payload to parse
@@ -184,8 +187,9 @@ module Html2rss
           # @param jsonish [String] JSON-like string with potentially unquoted keys
           # @return [String] payload with unquoted object keys quoted
           def quote_unquoted_keys(jsonish)
-            jsonish.gsub(/(\A\s*|[{,\[]\s*)([A-Za-z_]\w*)(\s*:)/) do
-              "#{Regexp.last_match(1)}\"#{Regexp.last_match(2)}\"#{Regexp.last_match(3)}"
+            jsonish.gsub(/(?<prefix>\A\s*|[{,\[]\s*)(?<key>[A-Za-z_]\w*)(?<suffix>\s*:)/) do
+              captures = Regexp.last_match.named_captures(symbolize_names: true)
+              "#{captures[:prefix]}\"#{captures[:key]}\"#{captures[:suffix]}"
             end
           end
 
@@ -301,7 +305,7 @@ module Html2rss
           # rubocop:disable Metrics/MethodLength
           # @param entry [Hash] raw article entry candidate
           # @param base_url [String, Html2rss::Url] base URL for relative link resolution
-          # @return [Hash{Symbol => Object}, nil] normalized article hash for downstream extraction
+          # @return [Hash{Symbol => Object, nil}] normalized article hash for downstream extraction
           def normalise(entry, base_url:)
             return unless entry.is_a?(Hash)
 
@@ -415,12 +419,17 @@ module Html2rss
 
         attr_reader :parsed_body
 
+        # @return [Boolean] true when the page contains article-like arrays in JSON state
+        def extractable?
+          json_documents.any? { CandidateDetector.candidate_array?(_1) }
+        end
+
         # @yield [Hash{Symbol => Object}] normalized article hash
         # @return [Enumerator, void] article enumerator when no block is given
         def each
           return enum_for(:each) unless block_given?
 
-          DocumentScanner.json_documents(parsed_body).each do |document|
+          json_documents.each do |document|
             discover_articles(document) do |article|
               yield article if article
             end
@@ -431,6 +440,10 @@ module Html2rss
 
         attr_reader :url
 
+        def json_documents
+          self.class.json_documents(parsed_body)
+        end
+
         def discover_articles(document, &block)
           case document
           when Array then handle_array(document, &block)