html2rss 0.24.0 → 0.26.0

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -9,6 +9,7 @@ module Html2rss
     class BotasaurusContract
       # Default Botasaurus scrape options when no explicit config is provided.
       DEFAULT_OPTIONS = {
+        execution_mode: 'auto',
         navigation_mode: 'auto',
         max_retries: 1,
         headless: false
@@ -16,23 +17,30 @@ module Html2rss
 
       # Allowlisted request.botasaurus keys forwarded to upstream.
       OPTION_KEYS = %i[
+        execution_mode
         navigation_mode
         max_retries
         wait_for_selector
         wait_timeout_seconds
+        scroll
+        scroll_to_bottom
         block_images
         block_images_and_css
+        block_trackers
         wait_for_complete_page_load
         headless
         proxy
         user_agent
         window_size
         lang
+        headers
+        cookies
       ].freeze
 
       # Allowlisted upstream response keys exposed as Response#transport_meta.
       META_KEYS = %w[
         request_id strategy_used render_ms challenge_detected blocked_detected attempts error_category
+        execution_tier detected_challenge
       ].freeze
 
       # Remaining seconds at or below which Botasaurus retries are disabled.
@@ -45,6 +53,10 @@ module Html2rss
       class ParsedResponse
         # Fallback headers when upstream omits response headers.
         DEFAULT_HEADERS = { 'content-type' => 'text/html' }.freeze
+        # Per-body size cap for captured XHR JSON (defense-in-depth vs upstream).
+        MAX_XHR_BODY_BYTES = 500_000
+        # Aggregate size cap across all captured XHR bodies.
+        MAX_XHR_AGGREGATE_BYTES = 2_000_000
 
         # @param payload [Hash{String => Object}] parsed Botasaurus response payload
         # @param transport_status [Integer] HTTP status returned by Botasaurus
@@ -58,7 +70,7 @@ module Html2rss
 
         # @return [Boolean] true when upstream returned non-200 or an error payload
         def upstream_failure?
-          status != 200 || error_message?
+          transport_status != 200 || status != 200 || error_message?
         end
 
         # @return [String] normalized challenge error message
@@ -76,10 +88,10 @@ module Html2rss
         end
 
         # @return [String] rendered HTML body from Botasaurus
-        # @raise [BotasaurusConnectionFailed] when html is missing
+        # @raise [BotasaurusServiceError] when html is missing
         def html
           value = payload['html']
-          raise BotasaurusConnectionFailed, "Botasaurus response missing required 'html' field" if value.nil?
+          raise BotasaurusServiceError, "Botasaurus response missing required 'html' field" if value.nil?
 
           value.to_s
         end
@@ -104,6 +116,21 @@ module Html2rss
         # @return [Hash{String => Object}] allowlisted upstream telemetry (frozen)
         def transport_meta = payload.slice(*META_KEYS).compact.freeze
 
+        # @return [Array<Hash{String => Object}>] size-capped XHR/fetch captures (absent → [])
+        def xhr_responses
+          raw = payload['xhr_responses']
+          return [] unless raw.is_a?(Array)
+
+          aggregate_bytes = 0
+          raw.filter_map do |entry|
+            normalized = normalize_xhr_entry(entry)
+            next unless normalized
+            next unless (aggregate_bytes = advance_xhr_budget(normalized, aggregate_bytes))
+
+            normalized
+          end
+        end
+
         private
 
         attr_reader :payload, :transport_status
@@ -118,50 +145,103 @@ module Html2rss
           value = error
           value.is_a?(String) ? !value.empty? : !value.nil?
         end
+
+        def normalize_xhr_entry(entry)
+          return unless entry.is_a?(Hash)
+
+          body = xhr_field(entry, :body)
+          return unless body.is_a?(String) && !body.empty?
+
+          {
+            'url' => xhr_field(entry, :url).to_s,
+            'body' => body,
+            'headers' => xhr_headers(xhr_field(entry, :headers)),
+            'status_code' => xhr_status_code(xhr_field(entry, :status_code))
+          }
+        end
+
+        # @return [Integer, nil] next aggregate byte total when accepted
+        def advance_xhr_budget(normalized, aggregate_bytes)
+          body_bytes = normalized.fetch('body').bytesize
+          return if body_bytes > MAX_XHR_BODY_BYTES
+          return if aggregate_bytes + body_bytes > MAX_XHR_AGGREGATE_BYTES
+
+          aggregate_bytes + body_bytes
+        end
+
+        def xhr_field(entry, key)
+          entry[key.to_s] || entry[key]
+        end
+
+        def xhr_headers(raw)
+          return {} unless raw.is_a?(Hash)
+
+          raw.to_h { |key, value| [key.to_s, value.to_s] }
+        end
+
+        def xhr_status_code(value)
+          value.is_a?(Integer) ? value : nil
+        end
       end
 
       ##
       # @param url [Html2rss::Url] canonical URL to scrape
+      # @param headers [Hash] request headers from context
       # @param options [Hash] validated request.botasaurus options
       # @param remaining_timeout_seconds [Numeric, nil] shared request budget remainder for clamps
+      # @option options [String] :execution_mode
       # @option options [String] :navigation_mode
       # @option options [Integer] :max_retries
       # @option options [String] :wait_for_selector
       # @option options [Integer] :wait_timeout_seconds
       # @option options [Boolean] :block_images
       # @option options [Boolean] :block_images_and_css
+      # @option options [Boolean] :block_trackers
       # @option options [Boolean] :wait_for_complete_page_load
       # @option options [Boolean] :headless
       # @option options [String] :proxy
       # @option options [String] :user_agent
       # @option options [Array<Integer>] :window_size
       # @option options [String] :lang
-      def initialize(url:, options: {}, remaining_timeout_seconds: nil)
+      # @option options [Hash] :cookies
+      # @option options [Hash] :headers
+      def initialize(url:, headers: {}, options: {}, remaining_timeout_seconds: nil)
         @url = url
+        @headers = headers
         @options = options
         @remaining_timeout_seconds = remaining_timeout_seconds
       end
 
       # @return [Hash] payload for POST /scrape (budget-clamped when remaining is known)
       def request_payload
-        DEFAULT_OPTIONS.merge(filtered_options).merge(url: url.to_s).then { clamp_for_budget(_1) }
+        payload = DEFAULT_OPTIONS.merge(filtered_options)
+        forwarded_headers = merged_headers
+        payload[:headers] = forwarded_headers if forwarded_headers&.any?
+        payload.merge(url: url.to_s).then { clamp_for_budget(_1) }
       end
 
       # @param transport_response [Faraday::Response] upstream HTTP response
       # @return [ParsedResponse]
-      # @raise [BotasaurusConnectionFailed] when payload is not valid JSON object
+      # @raise [BotasaurusServiceError] when payload is not valid JSON object
       def parse_response(transport_response)
         payload = JSON.parse(transport_response.body.to_s)
-        raise BotasaurusConnectionFailed, 'Botasaurus response must be a JSON object' unless payload.is_a?(Hash)
+        raise BotasaurusServiceError, 'Botasaurus response must be a JSON object' unless payload.is_a?(Hash)
 
         ParsedResponse.new(payload:, transport_status: transport_response.status)
       rescue JSON::ParserError => error
-        raise BotasaurusConnectionFailed, "Botasaurus response JSON parse failed: #{error.message}"
+        raise BotasaurusServiceError, "Botasaurus response JSON parse failed: #{error.message}"
       end
 
       private
 
-      attr_reader :url, :options, :remaining_timeout_seconds
+      attr_reader :url, :headers, :options, :remaining_timeout_seconds
+
+      def merged_headers
+        explicit = options[:headers] || {}
+        base = headers.is_a?(Hash) ? headers : {}
+        merged = base.merge(explicit).transform_keys(&:to_s).compact
+        merged.empty? ? nil : merged
+      end
 
       def filtered_options
         OPTION_KEYS.each_with_object({}) do |key, normalized|

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -28,7 +28,8 @@ module Html2rss
           headers: parsed_response.headers,
           url: response_url(parsed_response.final_url),
           status: parsed_response.status,
-          transport_meta: parsed_response.transport_meta
+          transport_meta: parsed_response.transport_meta,
+          captured_responses: parsed_response.xhr_responses
         )
       end
 
@@ -41,7 +42,7 @@ module Html2rss
       def raise_if_upstream_failed!(parsed_response)
         return unless parsed_response.upstream_failure?
 
-        raise BotasaurusConnectionFailed, parsed_response.upstream_failure_message
+        raise BotasaurusServiceError, parsed_response.upstream_failure_message
       end
 
       def response_url(final_url)
@@ -55,6 +56,7 @@ module Html2rss
       def contract
         @contract ||= BotasaurusContract.new(
           url: ctx.url,
+          headers: ctx.headers,
           options: ctx.request.fetch(:botasaurus, {}),
           remaining_timeout_seconds: attempt_timeout_seconds
         )

data/lib/html2rss/request_service/budget.rb

@@ -3,20 +3,17 @@
 module Html2rss
   class RequestService
     ##
-    # Tracks request slots, interaction budget, and wall-clock deadline for one feed build.
+    # Tracks HTTP request slots and wall-clock deadline for one feed build.
     #
-    # HTTP fetches consume request slots; Browserless preload clicks/scrolls/waits consume
-    # interaction budget. Deadline remains on `#remaining_timeout_seconds`.
+    # HTTP fetches consume request slots. Deadline remains on `#remaining_timeout_seconds`.
     class Budget
       ##
       # @param max_requests [Integer] maximum HTTP request slots
-      # @param max_interactions [Integer] maximum preload interaction slots (default 0)
       # @param total_timeout_seconds [Integer, nil] wall-clock timeout for the feed build
-      def initialize(max_requests:, max_interactions: 0, total_timeout_seconds: nil)
-        validate_slot_limits!(max_requests:, max_interactions:)
+      def initialize(max_requests:, total_timeout_seconds: nil)
+        validate_slot_limits!(max_requests:)
 
         @remaining_requests = max_requests
-        @remaining_interactions = max_interactions
         @start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         @total_timeout_seconds = total_timeout_seconds
         @mutex = Mutex.new
@@ -44,21 +41,6 @@ module Html2rss
         end
       end
 
-      ##
-      # Consumes one Browserless preload interaction slot.
-      #
-      # Preload must not steal pagination HTTP request slots.
-      #
-      # @return [Integer] remaining interaction slots after consumption
-      # @raise [InteractionBudgetExceeded] if no interaction slots remain
-      def consume_interaction!
-        @mutex.synchronize do
-          raise InteractionBudgetExceeded, 'Interaction budget exhausted' if @remaining_interactions.zero?
-
-          @remaining_interactions -= 1
-        end
-      end
-
       ##
       # @return [Integer] HTTP request slots still available
       def remaining
@@ -71,12 +53,6 @@ module Html2rss
         @mutex.synchronize { @remaining_requests }
       end
 
-      ##
-      # @return [Integer] preload interaction slots still available
-      def remaining_interactions
-        @mutex.synchronize { @remaining_interactions }
-      end
-
       ##
       # @return [Float, nil] the remaining timeout in seconds, or nil if not tracked
       def remaining_timeout_seconds
@@ -118,14 +94,10 @@ module Html2rss
 
       private
 
-      def validate_slot_limits!(max_requests:, max_interactions:)
-        unless max_requests.is_a?(Integer) && max_requests.positive?
-          raise ArgumentError,
-                'max_requests must be positive'
-        end
-        return if max_interactions.is_a?(Integer) && !max_interactions.negative?
+      def validate_slot_limits!(max_requests:)
+        return if max_requests.is_a?(Integer) && max_requests.positive?
 
-        raise ArgumentError, 'max_interactions must be a non-negative integer'
+        raise ArgumentError, 'max_requests must be positive'
       end
     end
   end

data/lib/html2rss/request_service/context.rb

@@ -32,12 +32,6 @@ module Html2rss
       # @return [Hash] the request specific options
       attr_reader :request
 
-      # @return [Hash] browserless specific options
-      def browserless = request.fetch(:browserless, {})
-
-      # @return [Hash, nil] preload options for browserless requests
-      def browserless_preload = browserless[:preload]
-
       # @return [Symbol] the request relation
       attr_reader :relation
 

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -9,6 +9,7 @@ module Html2rss
     ##
     # Strategy to use Faraday for the request.
     # @see https://rubygems.org/gems/faraday
+    # rubocop:disable Metrics/ClassLength -- terminal redirect retry colocated with Faraday transport
     class FaradayStrategy < Strategy
       ##
       # Restores buffered streamed bytes so response middleware can process them.
@@ -36,7 +37,8 @@ module Html2rss
       def perform_execute
         deadline = request_deadline
         @response_guard = ResponseGuard.new(policy: ctx.policy)
-        raw_response = faraday_request(response_guard, deadline:, streaming_buffer: true)
+        reset_redirect_tracking!
+        raw_response = request_with_terminal_redirect_retry(response_guard, deadline:)
         raw_response = retry_without_streaming(response_guard, deadline:) if retry_without_streaming?(raw_response)
         build_response(raw_response)
       end
@@ -47,6 +49,45 @@ module Html2rss
         monotonic_now + ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
       end
 
+      def reset_redirect_tracking!
+        @last_redirect_to = nil
+        @terminal_redirect_retried = false
+        @request_url_override = nil
+      end
+
+      def request_with_terminal_redirect_retry(response_guard, deadline:)
+        faraday_request(response_guard, deadline:, streaming_buffer: true)
+      rescue Faraday::FollowRedirects::RedirectLimitReached => error
+        raise error unless terminal_redirect_retryable?
+
+        retry_from_terminal_redirect!(response_guard, deadline:)
+      end
+
+      def terminal_redirect_retryable?
+        return false if @terminal_redirect_retried || @last_redirect_to.nil?
+
+        @last_redirect_to.to_s != request_url.to_s
+      end
+
+      def retry_from_terminal_redirect!(response_guard, deadline:)
+        terminal_url = @last_redirect_to
+        @terminal_redirect_retried = true
+        Log.debug("#{self.class}: redirect limit reached; retrying once from #{terminal_url}")
+        begin_terminal_url_request!(terminal_url)
+        faraday_request(response_guard, deadline:, streaming_buffer: true, consume_budget: false)
+      end
+
+      def begin_terminal_url_request!(terminal_url)
+        ctx.policy.validate_request!(url: terminal_url, origin_url: ctx.origin_url, relation: ctx.relation)
+        @request_url_override = terminal_url
+        @client = nil
+        @last_redirect_to = nil
+      end
+
+      def request_url
+        @request_url_override || ctx.url
+      end
+
       def build_response(response)
         Response.new(body: response.body, headers: response.headers, url: response_url(response),
                      status: response.status)
@@ -66,14 +107,55 @@ module Html2rss
         faraday_request(response_guard, deadline:, streaming_buffer: false, consume_budget: false)
       end
 
+      ##
+      # Validates the remote socket peer IP during Net::HTTP start.
+      module PeerIpValidator
+        module_function
+
+        # @param http [Net::HTTP] connection to configure
+        # @param policy [Policy] request policy
+        # @return [void]
+        def install!(http, policy:)
+          orig_start = http.method(:start)
+          http.define_singleton_method(:start) do |&block|
+            orig_start.call do |opened_http|
+              PeerIpValidator.validate!(opened_http, policy:)
+              block.call(opened_http)
+            end
+          end
+        end
+
+        # @param opened_http [Net::HTTP] active connection
+        # @param policy [Policy] request policy
+        # @return [void]
+        def validate!(opened_http, policy:)
+          scheme = opened_http.use_ssl? ? 'https' : 'http'
+          url = Html2rss::Url.from_absolute("#{scheme}://#{opened_http.address}:#{opened_http.port}")
+          policy.validate_remote_ip!(ip: peer_ip_for(opened_http), url:)
+        end
+
+        # @param opened_http [Net::HTTP]
+        # @return [String, nil]
+        def peer_ip_for(opened_http)
+          sock = opened_http.instance_variable_get(:@socket)
+          io = sock.respond_to?(:io) ? sock.io : sock
+          peeraddr = io.respond_to?(:peeraddr) ? io.peeraddr : nil
+          peeraddr&.[](3) || peeraddr&.[](2)
+        end
+      end
+
+      # rubocop:disable Metrics/AbcSize
       def client
-        @client ||= Faraday.new(url: ctx.url.to_s, headers: ctx.headers) do |faraday|
+        @client ||= Faraday.new(url: request_url.to_s, headers: ctx.headers) do |faraday|
           faraday.use Faraday::FollowRedirects::Middleware, limit: ctx.policy.max_redirects, callback: redirect_callback
           faraday.request :gzip
           faraday.use StreamingBodyMiddleware
-          faraday.adapter Faraday.default_adapter
+          faraday.adapter Faraday.default_adapter do |http|
+            PeerIpValidator.install!(http, policy: ctx.policy)
+          end
         end
       end
+      # rubocop:enable Metrics/AbcSize
 
       def apply_timeouts(request, deadline:)
         remaining_timeout = remaining_timeout_seconds(deadline)
@@ -127,6 +209,7 @@ module Html2rss
         lambda do |old_env, new_env|
           from_url = normalize_url(old_env[:url])
           to_url = normalize_url(new_env[:url])
+          @last_redirect_to = to_url
           ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
         end
       end
@@ -139,5 +222,6 @@ module Html2rss
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/html2rss/request_service/network_guard.rb

@@ -48,7 +48,10 @@ module Html2rss
       def enforce_public_network!(url)
         host = url.host
         return if allow_private_networks?
-        return unless blocked_host?(host) || resolved_ip_addresses(host).any? { |address| blocked_ip?(address) }
+        raise PrivateNetworkDenied, "Private network target denied for #{url}" if blocked_host?(host)
+
+        addresses = resolved_ip_addresses(host)
+        return unless addresses.empty? || addresses.any? { |address| blocked_ip?(address) }
 
         raise PrivateNetworkDenied, "Private network target denied for #{url}"
       end
@@ -62,9 +65,8 @@ module Html2rss
       # @raise [PrivateNetworkDenied] if the response came from a blocked address
       def validate_remote_ip!(ip:, url:)
         return if allow_private_networks?
-        return if ip.nil? || ip.empty?
 
-        parsed_ip = parse_ip(ip)
+        parsed_ip = ip && !ip.to_s.empty? ? parse_ip(ip) : nil
         raise PrivateNetworkDenied, "Remote IP could not be validated for #{url}" unless parsed_ip
         return unless blocked_ip?(parsed_ip)
 

data/lib/html2rss/request_service/policy.rb

@@ -18,7 +18,7 @@ module Html2rss
         connect_timeout_seconds: Integer(ENV.fetch('HTML2RSS_CONNECT_TIMEOUT_SECONDS', 5)),
         read_timeout_seconds: Integer(ENV.fetch('HTML2RSS_READ_TIMEOUT_SECONDS', 10)),
         total_timeout_seconds: Integer(ENV.fetch('HTML2RSS_TOTAL_TIMEOUT_SECONDS', 30)),
-        max_redirects: 3,
+        max_redirects: 5,
         max_response_bytes: 5_242_880,
         max_decompressed_bytes: 10_485_760,
         max_requests: 1,

data/lib/html2rss/request_service/response.rb

@@ -9,6 +9,8 @@ module Html2rss
     class Response
       # Default when a strategy does not attach transport telemetry.
       EMPTY_TRANSPORT_META = {}.freeze
+      # Default when a strategy does not capture sub-resource responses.
+      EMPTY_CAPTURED_RESPONSES = [].freeze
 
       ##
       # @param body [String] the body of the response
@@ -16,7 +18,10 @@ module Html2rss
       # @param headers [Hash] the headers of the response
       # @param status [Integer, nil] the HTTP status code when available
       # @param transport_meta [Hash] allowlisted upstream telemetry (frozen when present)
-      def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META)
+      # @param captured_responses [Array<Hash>] JSON XHR/fetch bodies captured during browser scrapes
+      # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists -- transport + capture fields stay co-located
+      def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META,
+                     captured_responses: EMPTY_CAPTURED_RESPONSES)
         @body = body
 
         headers = headers.dup
@@ -27,7 +32,13 @@ module Html2rss
         @status = status
         @url = url
         @transport_meta = transport_meta.nil? || transport_meta.empty? ? EMPTY_TRANSPORT_META : transport_meta.freeze
+        @captured_responses = if captured_responses.nil? || captured_responses.empty?
+                                EMPTY_CAPTURED_RESPONSES
+                              else
+                                captured_responses.freeze
+                              end
       end
+      # rubocop:enable Metrics/MethodLength, Metrics/ParameterLists
 
       # @return [String] the raw body of the response
       attr_reader :body
@@ -44,6 +55,9 @@ module Html2rss
       # @return [Hash] allowlisted upstream transport telemetry
       attr_reader :transport_meta
 
+      # @return [Array<Hash>] captured JSON XHR/fetch responses (empty when unsupported)
+      attr_reader :captured_responses
+
       # @return [String] normalized content type header value
       def content_type = header('content-type').to_s
 

data/lib/html2rss/request_service/strategy.rb

@@ -137,8 +137,7 @@ module Html2rss
       # @return [Boolean]
       def timeout_error?(error)
         error.is_a?(Faraday::TimeoutError) ||
-          error.is_a?(Timeout::Error) ||
-          (defined?(Puppeteer::TimeoutError) && error.is_a?(Puppeteer::TimeoutError))
+          error.is_a?(Timeout::Error)
       end
 
       # @param error [StandardError]

data/lib/html2rss/request_service.rb

@@ -6,7 +6,7 @@ require 'forwardable'
 module Html2rss
   ##
   # Requests website URLs to retrieve their HTML for further processing.
-  # Provides concrete transport strategies (e.g. Faraday, Browserless).
+  # Provides concrete transport strategies (e.g. Faraday, Botasaurus).
   #
   # Feed-level +:auto+ is not registered here — {FeedPipeline::StrategyPlan} resolves
   # it to a concrete strategy (or {FeedPipeline::AutoFallback} chain) before execute.
@@ -23,8 +23,6 @@ module Html2rss
     class UnsupportedResponseContentType < Html2rss::Error; end
     # Raised when HTTP request slot limits are exceeded.
     class RequestBudgetExceeded < Html2rss::Error; end
-    # Raised when Browserless preload interaction limits are exceeded.
-    class InteractionBudgetExceeded < Html2rss::Error; end
     # Raised when policy denies private-network access.
     class PrivateNetworkDenied < Html2rss::Error; end
     # Raised when cross-origin follow-up requests are denied.
@@ -35,14 +33,12 @@ module Html2rss
     class BlockedSurfaceDetected < Html2rss::Error; end
     # Raised when a request times out.
     class RequestTimedOut < Html2rss::Error; end
-    # Raised when Browserless configuration is missing or invalid.
-    class BrowserlessConfigurationError < Html2rss::Error; end
-    # Raised when Browserless cannot be reached.
-    class BrowserlessConnectionFailed < Html2rss::Error; end
     # Raised when Botasaurus configuration is missing or invalid.
     class BotasaurusConfigurationError < Html2rss::Error; end
-    # Raised when Botasaurus cannot be reached or returns invalid payloads.
+    # Raised when the Botasaurus service cannot be reached (network / DNS / SSL).
     class BotasaurusConnectionFailed < Html2rss::Error; end
+    # Raised when Botasaurus responds but the scrape fails (upstream error, bad payload).
+    class BotasaurusServiceError < Html2rss::Error; end
 
     class << self
       extend Forwardable
@@ -62,7 +58,6 @@ module Html2rss
       @strategies = {
         faraday: FaradayStrategy,
         botasaurus: BotasaurusStrategy,
-        browserless: BrowserlessStrategy,
         local_file: LocalFileStrategy
       }
       @default_strategy_name = :faraday

data/lib/html2rss/scoring/cluster_scorer.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Scores DomClustering-style node groups (heading/time/date/word density).
+    # Sole home for cluster-group ranking weights.
+    class ClusterScorer
+      # Minimum average visible words before a group is eligible.
+      MIN_AVG_WORDS = 5
+      # Bonus when any node has a heading or bold title class.
+      HEADING_BONUS = 20
+      # Bonus when any node has a time/datetime marker.
+      TIME_BONUS = 20
+      # Bonus when any node has date-like visible text.
+      DATE_BONUS = 40
+
+      def initialize
+        @text_words = {}.compare_by_identity
+        @has_date = {}.compare_by_identity
+      end
+
+      ##
+      # @param groups [Hash{String => Array<SST::Node>}]
+      # @return [Array<SST::Node>]
+      def select_best_group(groups)
+        best_nodes = []
+        best_score = -1
+
+        groups.each_value do |nodes|
+          score = score_group(nodes)
+          next if score.negative?
+
+          (best_nodes = nodes) && (best_score = score) if score > best_score
+        end
+        best_nodes
+      end
+
+      ##
+      # @param nodes [Array<SST::Node>]
+      # @return [Float]
+      def avg_words(nodes)
+        nodes.sum { |n| text_words(n) } / nodes.size.to_f
+      end
+
+      private
+
+      def score_group(nodes)
+        avg_w = avg_words(nodes)
+        return -1 if avg_w < MIN_AVG_WORDS
+
+        score = nodes.size + (avg_w / 5.0)
+        score += HEADING_BONUS if nodes_heading?(nodes)
+        score += TIME_BONUS if nodes_time?(nodes)
+        score += DATE_BONUS if nodes_date?(nodes)
+        score
+      end
+
+      def nodes_heading?(nodes)
+        nodes.any? do |n|
+          n.find(&:heading?) ||
+            n.attrs.class_names.any? { |c| c.match?(/font-bold|font-semibold/) }
+        end
+      end
+
+      def nodes_time?(nodes)
+        nodes.any? { |n| n.find { |d| d.name == :time || d.attrs.datetime } }
+      end
+
+      def nodes_date?(nodes)
+        nodes.any? { |n| date?(n) }
+      end
+
+      def text_words(node)
+        @text_words[node] ||= node.word_count
+      end
+
+      def date?(node)
+        @has_date[node] ||= begin
+          text = node.visible_text.to_s
+          text.match?(%r{\b\d{4}[-/]\d{2}[-/]\d{2}\b}) ||
+            text.match?(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i)
+        end
+      end
+    end
+  end
+end