html2rss 0.25.0 → 0.26.0

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -9,6 +9,7 @@ module Html2rss
     class BotasaurusContract
       # Default Botasaurus scrape options when no explicit config is provided.
       DEFAULT_OPTIONS = {
+        execution_mode: 'auto',
         navigation_mode: 'auto',
         max_retries: 1,
         headless: false
@@ -16,23 +17,30 @@ module Html2rss
 
       # Allowlisted request.botasaurus keys forwarded to upstream.
       OPTION_KEYS = %i[
+        execution_mode
         navigation_mode
         max_retries
         wait_for_selector
         wait_timeout_seconds
+        scroll
+        scroll_to_bottom
         block_images
         block_images_and_css
+        block_trackers
         wait_for_complete_page_load
         headless
         proxy
         user_agent
         window_size
         lang
+        headers
+        cookies
       ].freeze
 
       # Allowlisted upstream response keys exposed as Response#transport_meta.
       META_KEYS = %w[
         request_id strategy_used render_ms challenge_detected blocked_detected attempts error_category
+        execution_tier detected_challenge
       ].freeze
 
       # Remaining seconds at or below which Botasaurus retries are disabled.
@@ -45,6 +53,10 @@ module Html2rss
       class ParsedResponse
         # Fallback headers when upstream omits response headers.
         DEFAULT_HEADERS = { 'content-type' => 'text/html' }.freeze
+        # Per-body size cap for captured XHR JSON (defense-in-depth vs upstream).
+        MAX_XHR_BODY_BYTES = 500_000
+        # Aggregate size cap across all captured XHR bodies.
+        MAX_XHR_AGGREGATE_BYTES = 2_000_000
 
         # @param payload [Hash{String => Object}] parsed Botasaurus response payload
         # @param transport_status [Integer] HTTP status returned by Botasaurus
@@ -58,7 +70,7 @@ module Html2rss
 
         # @return [Boolean] true when upstream returned non-200 or an error payload
         def upstream_failure?
-          status != 200 || error_message?
+          transport_status != 200 || status != 200 || error_message?
         end
 
         # @return [String] normalized challenge error message
@@ -76,10 +88,10 @@ module Html2rss
         end
 
         # @return [String] rendered HTML body from Botasaurus
-        # @raise [BotasaurusConnectionFailed] when html is missing
+        # @raise [BotasaurusServiceError] when html is missing
         def html
           value = payload['html']
-          raise BotasaurusConnectionFailed, "Botasaurus response missing required 'html' field" if value.nil?
+          raise BotasaurusServiceError, "Botasaurus response missing required 'html' field" if value.nil?
 
           value.to_s
         end
@@ -104,6 +116,21 @@ module Html2rss
         # @return [Hash{String => Object}] allowlisted upstream telemetry (frozen)
         def transport_meta = payload.slice(*META_KEYS).compact.freeze
 
+        # @return [Array<Hash{String => Object}>] size-capped XHR/fetch captures (absent → [])
+        def xhr_responses
+          raw = payload['xhr_responses']
+          return [] unless raw.is_a?(Array)
+
+          aggregate_bytes = 0
+          raw.filter_map do |entry|
+            normalized = normalize_xhr_entry(entry)
+            next unless normalized
+            next unless (aggregate_bytes = advance_xhr_budget(normalized, aggregate_bytes))
+
+            normalized
+          end
+        end
+
         private
 
         attr_reader :payload, :transport_status
@@ -118,50 +145,103 @@ module Html2rss
           value = error
           value.is_a?(String) ? !value.empty? : !value.nil?
         end
+
+        def normalize_xhr_entry(entry)
+          return unless entry.is_a?(Hash)
+
+          body = xhr_field(entry, :body)
+          return unless body.is_a?(String) && !body.empty?
+
+          {
+            'url' => xhr_field(entry, :url).to_s,
+            'body' => body,
+            'headers' => xhr_headers(xhr_field(entry, :headers)),
+            'status_code' => xhr_status_code(xhr_field(entry, :status_code))
+          }
+        end
+
+        # @return [Integer, nil] next aggregate byte total when accepted
+        def advance_xhr_budget(normalized, aggregate_bytes)
+          body_bytes = normalized.fetch('body').bytesize
+          return if body_bytes > MAX_XHR_BODY_BYTES
+          return if aggregate_bytes + body_bytes > MAX_XHR_AGGREGATE_BYTES
+
+          aggregate_bytes + body_bytes
+        end
+
+        def xhr_field(entry, key)
+          entry[key.to_s] || entry[key]
+        end
+
+        def xhr_headers(raw)
+          return {} unless raw.is_a?(Hash)
+
+          raw.to_h { |key, value| [key.to_s, value.to_s] }
+        end
+
+        def xhr_status_code(value)
+          value.is_a?(Integer) ? value : nil
+        end
       end
 
       ##
       # @param url [Html2rss::Url] canonical URL to scrape
+      # @param headers [Hash] request headers from context
       # @param options [Hash] validated request.botasaurus options
       # @param remaining_timeout_seconds [Numeric, nil] shared request budget remainder for clamps
+      # @option options [String] :execution_mode
       # @option options [String] :navigation_mode
       # @option options [Integer] :max_retries
       # @option options [String] :wait_for_selector
       # @option options [Integer] :wait_timeout_seconds
       # @option options [Boolean] :block_images
       # @option options [Boolean] :block_images_and_css
+      # @option options [Boolean] :block_trackers
       # @option options [Boolean] :wait_for_complete_page_load
       # @option options [Boolean] :headless
       # @option options [String] :proxy
       # @option options [String] :user_agent
       # @option options [Array<Integer>] :window_size
       # @option options [String] :lang
-      def initialize(url:, options: {}, remaining_timeout_seconds: nil)
+      # @option options [Hash] :cookies
+      # @option options [Hash] :headers
+      def initialize(url:, headers: {}, options: {}, remaining_timeout_seconds: nil)
         @url = url
+        @headers = headers
         @options = options
         @remaining_timeout_seconds = remaining_timeout_seconds
       end
 
       # @return [Hash] payload for POST /scrape (budget-clamped when remaining is known)
       def request_payload
-        DEFAULT_OPTIONS.merge(filtered_options).merge(url: url.to_s).then { clamp_for_budget(_1) }
+        payload = DEFAULT_OPTIONS.merge(filtered_options)
+        forwarded_headers = merged_headers
+        payload[:headers] = forwarded_headers if forwarded_headers&.any?
+        payload.merge(url: url.to_s).then { clamp_for_budget(_1) }
       end
 
       # @param transport_response [Faraday::Response] upstream HTTP response
       # @return [ParsedResponse]
-      # @raise [BotasaurusConnectionFailed] when payload is not valid JSON object
+      # @raise [BotasaurusServiceError] when payload is not valid JSON object
       def parse_response(transport_response)
         payload = JSON.parse(transport_response.body.to_s)
-        raise BotasaurusConnectionFailed, 'Botasaurus response must be a JSON object' unless payload.is_a?(Hash)
+        raise BotasaurusServiceError, 'Botasaurus response must be a JSON object' unless payload.is_a?(Hash)
 
         ParsedResponse.new(payload:, transport_status: transport_response.status)
       rescue JSON::ParserError => error
-        raise BotasaurusConnectionFailed, "Botasaurus response JSON parse failed: #{error.message}"
+        raise BotasaurusServiceError, "Botasaurus response JSON parse failed: #{error.message}"
       end
 
       private
 
-      attr_reader :url, :options, :remaining_timeout_seconds
+      attr_reader :url, :headers, :options, :remaining_timeout_seconds
+
+      def merged_headers
+        explicit = options[:headers] || {}
+        base = headers.is_a?(Hash) ? headers : {}
+        merged = base.merge(explicit).transform_keys(&:to_s).compact
+        merged.empty? ? nil : merged
+      end
 
       def filtered_options
         OPTION_KEYS.each_with_object({}) do |key, normalized|

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -28,7 +28,8 @@ module Html2rss
           headers: parsed_response.headers,
           url: response_url(parsed_response.final_url),
           status: parsed_response.status,
-          transport_meta: parsed_response.transport_meta
+          transport_meta: parsed_response.transport_meta,
+          captured_responses: parsed_response.xhr_responses
         )
       end
 
@@ -41,7 +42,7 @@ module Html2rss
       def raise_if_upstream_failed!(parsed_response)
         return unless parsed_response.upstream_failure?
 
-        raise BotasaurusConnectionFailed, parsed_response.upstream_failure_message
+        raise BotasaurusServiceError, parsed_response.upstream_failure_message
       end
 
       def response_url(final_url)
@@ -55,6 +56,7 @@ module Html2rss
       def contract
         @contract ||= BotasaurusContract.new(
           url: ctx.url,
+          headers: ctx.headers,
           options: ctx.request.fetch(:botasaurus, {}),
           remaining_timeout_seconds: attempt_timeout_seconds
         )

data/lib/html2rss/request_service/budget.rb

@@ -3,20 +3,17 @@
 module Html2rss
   class RequestService
     ##
-    # Tracks request slots, interaction budget, and wall-clock deadline for one feed build.
+    # Tracks HTTP request slots and wall-clock deadline for one feed build.
     #
-    # HTTP fetches consume request slots; Browserless preload clicks/scrolls/waits consume
-    # interaction budget. Deadline remains on `#remaining_timeout_seconds`.
+    # HTTP fetches consume request slots. Deadline remains on `#remaining_timeout_seconds`.
     class Budget
       ##
       # @param max_requests [Integer] maximum HTTP request slots
-      # @param max_interactions [Integer] maximum preload interaction slots (default 0)
       # @param total_timeout_seconds [Integer, nil] wall-clock timeout for the feed build
-      def initialize(max_requests:, max_interactions: 0, total_timeout_seconds: nil)
-        validate_slot_limits!(max_requests:, max_interactions:)
+      def initialize(max_requests:, total_timeout_seconds: nil)
+        validate_slot_limits!(max_requests:)
 
         @remaining_requests = max_requests
-        @remaining_interactions = max_interactions
         @start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         @total_timeout_seconds = total_timeout_seconds
         @mutex = Mutex.new
@@ -44,21 +41,6 @@ module Html2rss
         end
       end
 
-      ##
-      # Consumes one Browserless preload interaction slot.
-      #
-      # Preload must not steal pagination HTTP request slots.
-      #
-      # @return [Integer] remaining interaction slots after consumption
-      # @raise [InteractionBudgetExceeded] if no interaction slots remain
-      def consume_interaction!
-        @mutex.synchronize do
-          raise InteractionBudgetExceeded, 'Interaction budget exhausted' if @remaining_interactions.zero?
-
-          @remaining_interactions -= 1
-        end
-      end
-
       ##
       # @return [Integer] HTTP request slots still available
       def remaining
@@ -71,12 +53,6 @@ module Html2rss
         @mutex.synchronize { @remaining_requests }
       end
 
-      ##
-      # @return [Integer] preload interaction slots still available
-      def remaining_interactions
-        @mutex.synchronize { @remaining_interactions }
-      end
-
       ##
       # @return [Float, nil] the remaining timeout in seconds, or nil if not tracked
       def remaining_timeout_seconds
@@ -118,14 +94,10 @@ module Html2rss
 
       private
 
-      def validate_slot_limits!(max_requests:, max_interactions:)
-        unless max_requests.is_a?(Integer) && max_requests.positive?
-          raise ArgumentError,
-                'max_requests must be positive'
-        end
-        return if max_interactions.is_a?(Integer) && !max_interactions.negative?
+      def validate_slot_limits!(max_requests:)
+        return if max_requests.is_a?(Integer) && max_requests.positive?
 
-        raise ArgumentError, 'max_interactions must be a non-negative integer'
+        raise ArgumentError, 'max_requests must be positive'
       end
     end
   end

data/lib/html2rss/request_service/context.rb

@@ -32,12 +32,6 @@ module Html2rss
       # @return [Hash] the request specific options
       attr_reader :request
 
-      # @return [Hash] browserless specific options
-      def browserless = request.fetch(:browserless, {})
-
-      # @return [Hash, nil] preload options for browserless requests
-      def browserless_preload = browserless[:preload]
-
       # @return [Symbol] the request relation
       attr_reader :relation
 

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -9,6 +9,7 @@ module Html2rss
     ##
     # Strategy to use Faraday for the request.
     # @see https://rubygems.org/gems/faraday
+    # rubocop:disable Metrics/ClassLength -- terminal redirect retry colocated with Faraday transport
     class FaradayStrategy < Strategy
       ##
       # Restores buffered streamed bytes so response middleware can process them.
@@ -36,7 +37,8 @@ module Html2rss
       def perform_execute
         deadline = request_deadline
         @response_guard = ResponseGuard.new(policy: ctx.policy)
-        raw_response = faraday_request(response_guard, deadline:, streaming_buffer: true)
+        reset_redirect_tracking!
+        raw_response = request_with_terminal_redirect_retry(response_guard, deadline:)
         raw_response = retry_without_streaming(response_guard, deadline:) if retry_without_streaming?(raw_response)
         build_response(raw_response)
       end
@@ -47,6 +49,45 @@ module Html2rss
         monotonic_now + ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
       end
 
+      def reset_redirect_tracking!
+        @last_redirect_to = nil
+        @terminal_redirect_retried = false
+        @request_url_override = nil
+      end
+
+      def request_with_terminal_redirect_retry(response_guard, deadline:)
+        faraday_request(response_guard, deadline:, streaming_buffer: true)
+      rescue Faraday::FollowRedirects::RedirectLimitReached => error
+        raise error unless terminal_redirect_retryable?
+
+        retry_from_terminal_redirect!(response_guard, deadline:)
+      end
+
+      def terminal_redirect_retryable?
+        return false if @terminal_redirect_retried || @last_redirect_to.nil?
+
+        @last_redirect_to.to_s != request_url.to_s
+      end
+
+      def retry_from_terminal_redirect!(response_guard, deadline:)
+        terminal_url = @last_redirect_to
+        @terminal_redirect_retried = true
+        Log.debug("#{self.class}: redirect limit reached; retrying once from #{terminal_url}")
+        begin_terminal_url_request!(terminal_url)
+        faraday_request(response_guard, deadline:, streaming_buffer: true, consume_budget: false)
+      end
+
+      def begin_terminal_url_request!(terminal_url)
+        ctx.policy.validate_request!(url: terminal_url, origin_url: ctx.origin_url, relation: ctx.relation)
+        @request_url_override = terminal_url
+        @client = nil
+        @last_redirect_to = nil
+      end
+
+      def request_url
+        @request_url_override || ctx.url
+      end
+
       def build_response(response)
         Response.new(body: response.body, headers: response.headers, url: response_url(response),
                      status: response.status)
@@ -105,7 +146,7 @@ module Html2rss
 
       # rubocop:disable Metrics/AbcSize
       def client
-        @client ||= Faraday.new(url: ctx.url.to_s, headers: ctx.headers) do |faraday|
+        @client ||= Faraday.new(url: request_url.to_s, headers: ctx.headers) do |faraday|
           faraday.use Faraday::FollowRedirects::Middleware, limit: ctx.policy.max_redirects, callback: redirect_callback
           faraday.request :gzip
           faraday.use StreamingBodyMiddleware
@@ -168,6 +209,7 @@ module Html2rss
         lambda do |old_env, new_env|
           from_url = normalize_url(old_env[:url])
           to_url = normalize_url(new_env[:url])
+          @last_redirect_to = to_url
           ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
         end
       end
@@ -180,5 +222,6 @@ module Html2rss
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/html2rss/request_service/policy.rb

@@ -18,7 +18,7 @@ module Html2rss
         connect_timeout_seconds: Integer(ENV.fetch('HTML2RSS_CONNECT_TIMEOUT_SECONDS', 5)),
         read_timeout_seconds: Integer(ENV.fetch('HTML2RSS_READ_TIMEOUT_SECONDS', 10)),
         total_timeout_seconds: Integer(ENV.fetch('HTML2RSS_TOTAL_TIMEOUT_SECONDS', 30)),
-        max_redirects: 3,
+        max_redirects: 5,
         max_response_bytes: 5_242_880,
         max_decompressed_bytes: 10_485_760,
         max_requests: 1,

data/lib/html2rss/request_service/response.rb

@@ -9,6 +9,8 @@ module Html2rss
     class Response
       # Default when a strategy does not attach transport telemetry.
       EMPTY_TRANSPORT_META = {}.freeze
+      # Default when a strategy does not capture sub-resource responses.
+      EMPTY_CAPTURED_RESPONSES = [].freeze
 
       ##
       # @param body [String] the body of the response
@@ -16,7 +18,10 @@ module Html2rss
       # @param headers [Hash] the headers of the response
       # @param status [Integer, nil] the HTTP status code when available
       # @param transport_meta [Hash] allowlisted upstream telemetry (frozen when present)
-      def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META)
+      # @param captured_responses [Array<Hash>] JSON XHR/fetch bodies captured during browser scrapes
+      # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists -- transport + capture fields stay co-located
+      def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META,
+                     captured_responses: EMPTY_CAPTURED_RESPONSES)
         @body = body
 
         headers = headers.dup
@@ -27,7 +32,13 @@ module Html2rss
         @status = status
         @url = url
         @transport_meta = transport_meta.nil? || transport_meta.empty? ? EMPTY_TRANSPORT_META : transport_meta.freeze
+        @captured_responses = if captured_responses.nil? || captured_responses.empty?
+                                EMPTY_CAPTURED_RESPONSES
+                              else
+                                captured_responses.freeze
+                              end
       end
+      # rubocop:enable Metrics/MethodLength, Metrics/ParameterLists
 
       # @return [String] the raw body of the response
       attr_reader :body
@@ -44,6 +55,9 @@ module Html2rss
       # @return [Hash] allowlisted upstream transport telemetry
       attr_reader :transport_meta
 
+      # @return [Array<Hash>] captured JSON XHR/fetch responses (empty when unsupported)
+      attr_reader :captured_responses
+
       # @return [String] normalized content type header value
       def content_type = header('content-type').to_s
 

data/lib/html2rss/request_service/strategy.rb

@@ -137,8 +137,7 @@ module Html2rss
       # @return [Boolean]
       def timeout_error?(error)
         error.is_a?(Faraday::TimeoutError) ||
-          error.is_a?(Timeout::Error) ||
-          (defined?(Puppeteer::TimeoutError) && error.is_a?(Puppeteer::TimeoutError))
+          error.is_a?(Timeout::Error)
       end
 
       # @param error [StandardError]

data/lib/html2rss/request_service.rb

@@ -6,7 +6,7 @@ require 'forwardable'
 module Html2rss
   ##
   # Requests website URLs to retrieve their HTML for further processing.
-  # Provides concrete transport strategies (e.g. Faraday, Browserless).
+  # Provides concrete transport strategies (e.g. Faraday, Botasaurus).
   #
   # Feed-level +:auto+ is not registered here — {FeedPipeline::StrategyPlan} resolves
   # it to a concrete strategy (or {FeedPipeline::AutoFallback} chain) before execute.
@@ -23,8 +23,6 @@ module Html2rss
     class UnsupportedResponseContentType < Html2rss::Error; end
     # Raised when HTTP request slot limits are exceeded.
     class RequestBudgetExceeded < Html2rss::Error; end
-    # Raised when Browserless preload interaction limits are exceeded.
-    class InteractionBudgetExceeded < Html2rss::Error; end
     # Raised when policy denies private-network access.
     class PrivateNetworkDenied < Html2rss::Error; end
     # Raised when cross-origin follow-up requests are denied.
@@ -35,14 +33,12 @@ module Html2rss
     class BlockedSurfaceDetected < Html2rss::Error; end
     # Raised when a request times out.
     class RequestTimedOut < Html2rss::Error; end
-    # Raised when Browserless configuration is missing or invalid.
-    class BrowserlessConfigurationError < Html2rss::Error; end
-    # Raised when Browserless cannot be reached.
-    class BrowserlessConnectionFailed < Html2rss::Error; end
     # Raised when Botasaurus configuration is missing or invalid.
     class BotasaurusConfigurationError < Html2rss::Error; end
-    # Raised when Botasaurus cannot be reached or returns invalid payloads.
+    # Raised when the Botasaurus service cannot be reached (network / DNS / SSL).
     class BotasaurusConnectionFailed < Html2rss::Error; end
+    # Raised when Botasaurus responds but the scrape fails (upstream error, bad payload).
+    class BotasaurusServiceError < Html2rss::Error; end
 
     class << self
       extend Forwardable
@@ -62,7 +58,6 @@ module Html2rss
       @strategies = {
         faraday: FaradayStrategy,
         botasaurus: BotasaurusStrategy,
-        browserless: BrowserlessStrategy,
         local_file: LocalFileStrategy
       }
       @default_strategy_name = :faraday

data/lib/html2rss/scoring/container_assessor.rb

@@ -8,21 +8,15 @@ module Html2rss
       # Microdata itemprop values treated as publish/update markers.
       PUBLISH_ITEMPROPS = %w[datePublished dateModified].freeze
 
-      # Matches content-like tokens in class/id strings.
-      CONTENT_TOKEN_REGEXP = begin
-        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:content)
-        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
-      end.freeze
-
-      # Matches utility/junk tokens in class/id strings.
-      JUNK_TOKEN_REGEXP = begin
-        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:utility)
-        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
-      end.freeze
-
       # @param text_classifier [LinkDestination::TextClassifier]
-      def initialize(text_classifier: LinkDestination::TextClassifier.new)
+      # @param content_token_regexp [Regexp]
+      # @param junk_token_regexp [Regexp]
+      def initialize(text_classifier: LinkDestination::TextClassifier.new,
+                     content_token_regexp: LinkResolver::CONTENT_TOKEN_REGEXP,
+                     junk_token_regexp: LinkResolver::JUNK_TOKEN_REGEXP)
         @text_classifier = text_classifier
+        @content_token_regexp = content_token_regexp
+        @junk_token_regexp = junk_token_regexp
       end
 
       ##
@@ -41,8 +35,8 @@ module Html2rss
           publish_marker: publish_marker?(container),
           descriptive_context: descriptive_context?(container.visible_text, title),
           article_container: container.name == :article,
-          content_tokens: tokens.match?(CONTENT_TOKEN_REGEXP),
-          junk_tokens: tokens.match?(JUNK_TOKEN_REGEXP),
+          content_tokens: tokens.match?(@content_token_regexp),
+          junk_tokens: tokens.match?(@junk_token_regexp),
           utility_prefix_title: @text_classifier.utility_prefix?(title),
           recommended_title: @text_classifier.recommended?(title),
           utility_path: destination_facts&.utility_path,

data/lib/html2rss/scoring/engine.rb

@@ -10,7 +10,17 @@ module Html2rss
       # Minimum composite score retained by {#rank_top} (precision floor).
       SCORE_FLOOR = 0.0
 
-      # Quality feature triples: [FeatureId, predicate, weight].
+      FEATURE_IDS = %i[
+        title_word_count_ge3 title_word_count_ge7 path_length_gt6 content_path
+        publish_marker descriptive_context article_container content_tokens
+        non_content_utility_path utility_prefix_title_short shallow weak_container
+        recommended_title_non_content high_confidence_junk_path junk_tokens
+        heading_anchor heading_text_match meaningful_text content_like_destination
+        cluster_size cluster_avg_words cluster_heading cluster_time cluster_date
+      ].to_set.freeze
+      private_constant :FEATURE_IDS
+
+      # Quality feature triples: [feature_id, predicate, weight].
       QUALITY_RULES = [
         [:title_word_count_ge3, ->(o) { o.title_word_count >= 3 }, 40],
         [:title_word_count_ge7, ->(o) { o.title_word_count >= 7 }, 15],
@@ -22,7 +32,7 @@ module Html2rss
         [:content_tokens, lambda(&:content_tokens), 10]
       ].freeze
 
-      # Junk feature triples: [FeatureId, predicate, weight].
+      # Junk feature triples: [feature_id, predicate, weight].
       JUNK_RULES = [
         [:non_content_utility_path, ->(o) { o.utility_path && !o.content_path && !o.strong_post_suffix }, 25],
         [:utility_prefix_title_short, ->(o) { o.utility_prefix_title && o.title_word_count <= 6 }, 15],
@@ -33,7 +43,17 @@ module Html2rss
         [:junk_tokens, lambda(&:junk_tokens), 15]
       ].freeze
 
-      (QUALITY_RULES + JUNK_RULES).each { |feature_id, _, _| FeatureId.assert!(feature_id) }
+      ##
+      # @param id [Symbol]
+      # @return [Symbol]
+      # @raise [ArgumentError] when id is not in the closed set
+      def self.assert_feature_id!(id)
+        raise ArgumentError, "unknown feature: #{id.inspect}" unless FEATURE_IDS.include?(id)
+
+        id
+      end
+
+      (QUALITY_RULES + JUNK_RULES).each { |feature_id, _, _| assert_feature_id!(feature_id) }
 
       # @param link_resolver [LinkResolver]
       def initialize(link_resolver:)
@@ -78,9 +98,9 @@ module Html2rss
 
         quality, quality_parts = apply_rules(QUALITY_RULES, obs)
         junk, junk_parts = apply_rules(JUNK_RULES, obs)
-        RankedSegment.build(
+        ranked(
           segment:,
-          score: Score.build(
+          score: build_score(
             composite: quality - junk,
             quality:,
             junk:,
@@ -96,6 +116,30 @@ module Html2rss
         end
         [parts.values.sum, parts]
       end
+
+      def build_score(composite:, quality: nil, junk: nil, breakdown: nil)
+        raise ArgumentError, 'composite must be Numeric' unless composite.is_a?(Numeric)
+
+        Score.new(
+          composite: composite.to_f,
+          quality: (quality.nil? ? composite : quality).to_f,
+          junk: (junk || 0).to_f,
+          breakdown: normalize_breakdown(breakdown)
+        )
+      end
+
+      def normalize_breakdown(breakdown)
+        return Score::EMPTY_BREAKDOWN if breakdown.nil?
+
+        breakdown.transform_keys { |id| self.class.assert_feature_id!(id) }.freeze
+      end
+
+      def ranked(segment:, score:)
+        raise ArgumentError, 'segment must be AutoSource::Segment' unless segment.is_a?(Html2rss::AutoSource::Segment)
+        raise ArgumentError, 'score must be Scoring::Score' unless score.is_a?(Score)
+
+        RankedSegment.new(segment:, score:)
+      end
     end
   end
 end

data/lib/html2rss/scoring/link_resolver.rb

@@ -8,6 +8,18 @@ module Html2rss
       # Captures the href portion before a fragment for memoization keys.
       HREF_BASE_PATTERN = /\A([^#]*)/
 
+      # Matches content-like tokens in class/id strings (from PathClassifier vocabulary).
+      CONTENT_TOKEN_REGEXP = begin
+        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:content)
+        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
+      end.freeze
+
+      # Matches utility/junk tokens in class/id strings (from PathClassifier vocabulary).
+      JUNK_TOKEN_REGEXP = begin
+        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:utility)
+        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
+      end.freeze
+
       # @param base_url [String, Html2rss::Url]
       def initialize(base_url)
         @base_url = base_url